tr(1): backslash(): fix octal escape parsing

There are two bugs in backslash():

1. 8 and 9 are not octal digits.  If we see '8' or '9' we should
   terminate the octal escape.

2. We return octal escape values larger than UCHAR_MAX even though
   tr(1) is (currently) a byte-oriented program and values larger
   than UCHAR_MAX make no sense.

So, fix them both.  In particular, (a) stop parsing if we see
characters outside of '0'-'7' and (b) escaped octal values larger than
UCHAR_MAX are a terminal error.

While here, some cleanup:

- Check for empty escapes at the top of the function.  This simplifies
  later cases.

- Use the for-loop conditional to terminate octal escape parsing after
  three characters.

- Use an ANSI-style function definition.

We can fix the switch-statement indentation later, in a larger KNF
patch.

ok millert@

1 files changed, 25 insertions, 18 deletions

diff --git a/usr.bin/tr/str.c b/usr.bin/tr/str.c
index 6b5f1f02d37..26e706d29ba 100644
--- a/usr.bin/tr/str.c
+++ b/usr.bin/tr/str.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: str.c,v 1.12 2012/12/05 23:20:26 deraadt Exp $	*/
+/*	$OpenBSD: str.c,v 1.13 2021/10/31 21:34:16 cheloha Exp $	*/
 /*	$NetBSD: str.c,v 1.7 1995/08/31 22:13:47 jtc Exp $	*/
 
 /*-
@@ -32,6 +32,7 @@
 
 #include <sys/types.h>
 
+#include <assert.h>
 #include <errno.h>
 #include <stddef.h>
 #include <stdio.h>
@@ -280,25 +281,34 @@ genseq(s)
  * an escape code or a literal character.
  */
 static int
-backslash(s)
-	STR *s;
+backslash(STR *s)
 {
-	int ch, cnt, val;
+	size_t i;
+	int ch, val;
 
-	for (cnt = val = 0;;) {
-		ch = *++s->str;
-		if (!isascii(ch) || !isdigit(ch))
-			break;
-		val = val * 8 + ch - '0';
-		if (++cnt == 3) {
-			++s->str;
+	assert(*s->str == '\\');
+	s->str++;
+
+	/* Empty escapes become plain backslashes. */
+	if (*s->str == '\0') {
+		s->state = EOS;
+		return ('\\');
+	}
+
+	val = 0;
+	for (i = 0; i < 3; i++) {
+		if (s->str[i] < '0' || '7' < s->str[i])
 			break;
-		}
+		val = val * 8 + s->str[i] - '0';
 	}
-	if (cnt)
+	if (i > 0) {
+		if (val > UCHAR_MAX)
+			errx(1, "octal value out of range: %d", val);
+		s->str += i;
 		return (val);
-	if (ch != '\0')
-		++s->str;
+	}
+
+	ch = *s->str++;
 	switch (ch) {
 		case 'a':			/* escape characters */
 			return ('\7');
@@ -314,9 +324,6 @@ backslash(s)
 			return ('\t');
 		case 'v':
 			return ('\13');
-		case '\0':			/*  \" -> \ */
-			s->state = EOS;
-			return ('\\');
 		default:			/* \x" -> x */
 			return (ch);
 	}