finally make our m4 SusV3-compliant.

- changecom and changequote have a simple definition (that matches gnu-m4,
coincidentally, so we no longer need two distinct modes for these)

- off-by-one bug in -s, this finally works.

- reorder main parser loop, so that we can use alphabetic constructs in
quotes/comments.

- rename putback to pushback, this matches comments, and makes more sense.

- more uniform (and updated) description of changequote/changecom.

- new, systematic regression tests of comments/quotes.

- framework to test -s: one perl script to reconstitute `full' files with
all line numbers, so that we can verify the output without needing a
complete match.

okay otto@, fries@

9 files changed, 270 insertions, 29 deletions

diff --git a/regress/usr.bin/m4/Makefile b/regress/usr.bin/m4/Makefile
index 66e6e1a4fe5..6b89ec7a220 100644
--- a/regress/usr.bin/m4/Makefile
+++ b/regress/usr.bin/m4/Makefile
@@ -1,4 +1,4 @@
-#	$OpenBSD: Makefile,v 1.21 2005/05/17 20:37:11 espie Exp $
+#	$OpenBSD: Makefile,v 1.22 2005/09/06 15:33:21 espie Exp $
 
 FIBOMAX=25
 M4=m4
@@ -10,7 +10,7 @@ REGRESS_TARGETS= test-ff_after_dnl test-m4wrap test-m4wrap2 \
     test-m4wrap3 test-gm4wrap3 test-fibo \
     test-patterns trip test-strangequotes test-redef test-quotes \
     test-weird test-args test-esyscmd test-eval test-gnupatterns \
-    test-gnupatterns2
+    test-gnupatterns2 test-comments test-synch1 test-synch1bis
 
 test-ff_after_dnl: ff_after_dnl.m4
 	${M4} ff_after_dnl.m4 | diff - ${.CURDIR}/ff_after_dnl.out
@@ -52,8 +52,9 @@ test-quotes:
 	${M4} ${.CURDIR}/quotes.m4 2>&1|	\
 		sed -e 's,\( *\).*/quotes.m4,\1quotes.m4,' | \
 		diff - ${.CURDIR}/quotes.out
-	${M4} -g ${.CURDIR}/quotes.m4 2>&1|diff - ${.CURDIR}/quotes.gnu.out
 
+test-comments:
+	${M4} ${.CURDIR}/comments.m4 | diff - ${.CURDIR}/comments.out
 test-strangequotes: strangequotes.m4
 	${M4} strangequotes.m4| diff - ${.CURDIR}/strangequotes.out
 
@@ -72,6 +73,14 @@ test-esyscmd:
 test-eval:
 	${M4} ${.CURDIR}/eval.m4 | diff -u - ${.CURDIR}/eval.out
 
+test-synch1:
+	${M4} -s ${.CURDIR}/synch1.m4|perl ${.CURDIR}/reconstitute|\
+		grep MARK| diff - ${.CURDIR}/synch1.out
+
+test-synch1bis:
+	${M4} -s <${.CURDIR}/synch1.m4|perl ${.CURDIR}/reconstitute|\
+		grep MARK| diff - ${.CURDIR}/synch1bis.out
+
 .PHONY:	${REGRESS_TARGETS}
 
 .include <bsd.regress.mk>
diff --git a/regress/usr.bin/m4/comments.m4 b/regress/usr.bin/m4/comments.m4
new file mode 100644
index 00000000000..f24d9e6eff3
--- /dev/null
+++ b/regress/usr.bin/m4/comments.m4
@@ -0,0 +1,58 @@
+dnl $OpenBSD: comments.m4,v 1.1 2005/09/06 15:33:21 espie Exp $
+dnl checking the way changecom works.
+1: normal
+define(`comment', `COMMENT')dnl
+define(`p', 'XXX')dnl
+# this is a comment
+>> this is a comment
+p this is a comment
+p this is a comment q comment too
+
+2: `changecom(>>)dnl'
+changecom(>>)dnl
+# this is a comment
+>> this is a comment
+p this is a comment
+p this is a comment q comment too
+
+3: `changecom dnl'
+changecom dnl
+# this is a comment
+>> this is a comment
+p this is a comment
+p this is a comment q comment too
+
+4: `changecom()dnl'
+changecom()dnl
+# this is a comment
+>> this is a comment
+p this is a comment
+p this is a comment q comment too
+
+5: `changecom(,)dnl'
+changecom(,)dnl
+# this is a comment
+>> this is a comment
+p this is a comment
+p this is a comment q comment too
+
+6: `changecom(`p',q)dnl'
+changecom(`p',q)dnl
+# this is a comment
+>> this is a comment
+p this is a comment
+p this is a comment q comment too
+
+7: `changecom(`p')dnl'
+changecom(`p')dnl
+# this is a comment
+>> this is a comment
+p this is a comment
+p this is a comment q comment too
+
+8: `changecom(#)dnl'
+changecom(#)dnl
+# this is a comment
+>> this is a comment
+p this is a comment
+p this is a comment q comment too
diff --git a/regress/usr.bin/m4/comments.out b/regress/usr.bin/m4/comments.out
new file mode 100644
index 00000000000..157f107a8a2
--- /dev/null
+++ b/regress/usr.bin/m4/comments.out
@@ -0,0 +1,47 @@
+1: normal
+# this is a comment
+>> this is a COMMENT
+'XXX' this is a COMMENT
+'XXX' this is a COMMENT q COMMENT too
+
+2: changecom(>>)dnl
+# this is a COMMENT
+>> this is a comment
+'XXX' this is a COMMENT
+'XXX' this is a COMMENT q COMMENT too
+
+3: changecom dnl
+ # this is a COMMENT
+>> this is a COMMENT
+'XXX' this is a COMMENT
+'XXX' this is a COMMENT q COMMENT too
+
+4: changecom()dnl
+# this is a COMMENT
+>> this is a COMMENT
+'XXX' this is a COMMENT
+'XXX' this is a COMMENT q COMMENT too
+
+5: changecom(,)dnl
+# this is a COMMENT
+>> this is a COMMENT
+'XXX' this is a COMMENT
+'XXX' this is a COMMENT q COMMENT too
+
+6: changecom(`p',q)dnl
+# this is a COMMENT
+>> this is a COMMENT
+p this is a comment
+p this is a comment q COMMENT too
+
+7: changecom(`p')dnl
+# this is a COMMENT
+>> this is a COMMENT
+p this is a comment
+p this is a comment q comment too
+
+8: changecom(#)dnl
+# this is a comment
+>> this is a COMMENT
+'XXX' this is a COMMENT
+'XXX' this is a COMMENT q COMMENT too
diff --git a/regress/usr.bin/m4/quotes.m4 b/regress/usr.bin/m4/quotes.m4
index 08414663974..891ddcef24e 100644
--- a/regress/usr.bin/m4/quotes.m4
+++ b/regress/usr.bin/m4/quotes.m4
@@ -1,21 +1,57 @@
-dnl $OpenBSD: quotes.m4,v 1.1 2001/09/29 15:49:18 espie Exp $
+dnl $OpenBSD: quotes.m4,v 1.2 2005/09/06 15:33:21 espie Exp $
 dnl Checking the way changequote() is supposed to work
-dnl in both normal and gnu emulation mode
+define(`string',`STRING')dnl
+1: normal
+`quoted string'
+[quoted string]
+normal string
+`half quoted string
+going up to that string'
+
+2: kill quotes
 changequote()dnl
-`No quotes left'
-changequote([,])dnl
-[Quotes]changequote
-`No quotes left'
+`quoted string'
+[quoted string]
+normal string
+`half quoted string
+going up to that string'
+
+3: normal changed quote
 changequote([,])dnl
+`quoted string'
+[quoted string]
+normal string
+`half quoted string
+going up to that string'
+
+4: empty quotes, kill them too
 changequote(,)dnl
-`No quotes left'
-changequote([,])dnl
-dnl same thing with comments, so first:
-define([comment], [COMMENT])dnl
-# this is a comment
-changecom(>>)dnl
-# this is a comment
-changecom
-# this is a comment
-changecom(,)dnl
-# this is a comment
+`quoted string'
+[quoted string]
+normal string
+`half quoted string
+going up to that string'
+
+5: start quote only
+changequote(`)dnl
+`quoted string'
+[quoted string]
+normal string
+`half quoted string
+going up to that string'
+
+6: normal quotes are back
+changequote
+`quoted string'
+[quoted string]
+normal string
+`half quoted string
+going up to that string'
+
+7: start quote+empty end quote
+changequote([,)dnl
+`quoted string'
+[quoted string]
+normal string
+`half quoted string
+going up to that string'
diff --git a/regress/usr.bin/m4/quotes.out b/regress/usr.bin/m4/quotes.out
index 555590d2a2b..cf34ba42a8d 100644
--- a/regress/usr.bin/m4/quotes.out
+++ b/regress/usr.bin/m4/quotes.out
@@ -1,9 +1,49 @@
-No quotes left
-Quotes
-No quotes left
-No quotes left'
-changequote([,)# this is a comment
-# this is a comment
-
-# this is a comment
-# this is a comment
+m4: unclosed quote:
+   quotes.m4 at line 54
+1: normal
+quoted string
+[quoted STRING]
+normal STRING
+half quoted string
+going up to that string
+
+2: kill quotes
+`quoted STRING'
+[quoted STRING]
+normal STRING
+`half quoted STRING
+going up to that STRING'
+
+3: normal changed quote
+`quoted STRING'
+quoted string
+normal STRING
+`half quoted STRING
+going up to that STRING'
+
+4: empty quotes, kill them too
+`quoted STRING'
+[quoted STRING]
+normal STRING
+`half quoted STRING
+going up to that STRING'
+
+5: start quote only
+quoted string'[quoted STRING]
+normal STRING
+half quoted stringgoing up to that STRING'
+
+6: normal quotes are back
+
+quoted string
+[quoted STRING]
+normal STRING
+half quoted string
+going up to that string
+
+7: start quote+empty end quote
+`quoted STRING'
+quoted string]
+normal string
+`half quoted string
+going up to that string'
diff --git a/regress/usr.bin/m4/reconstitute b/regress/usr.bin/m4/reconstitute
new file mode 100644
index 00000000000..98554bc3886
--- /dev/null
+++ b/regress/usr.bin/m4/reconstitute
@@ -0,0 +1,24 @@
+#! /usr/bin/perl
+# $OpenBSD: reconstitute,v 1.1 2005/09/06 15:33:21 espie Exp $
+
+# Written by Marc Espie, 2005
+# Public domain
+
+# This simple perl script puts back line numbers everywhere.
+# This is suitable for testing synchronization, as we don't really
+# care how many synchronization marks we emit, as long as the line
+# numbers match
+
+use File::Basename;
+
+my ($lineno, $file) = (-1, "<unknown>");
+
+while (<>) {
+	if (m/^#line\s+(\d+)\s+\"(.*)\"/) {
+		($lineno, $file) = ($1, $2);
+		$file=basename($file);
+	} else {
+		print "$file:$lineno:$_";
+		$lineno++;
+	}
+}
diff --git a/regress/usr.bin/m4/synch1.m4 b/regress/usr.bin/m4/synch1.m4
new file mode 100644
index 00000000000..becd550d3fd
--- /dev/null
+++ b/regress/usr.bin/m4/synch1.m4
@@ -0,0 +1,13 @@
+This is a test for multi MARK1
+line synchronization stuff MARK2
+# Let's include comments to see MARK3
+dnl
+dnl
+`quoted string' MARK4
+define(`A',`big string
+with a newline embedded')dnl
+Return to normal MARK5
+Let's try A
+And let's see where we are now. MARK6
+Okay, some more input.
+MARK7
diff --git a/regress/usr.bin/m4/synch1.out b/regress/usr.bin/m4/synch1.out
new file mode 100644
index 00000000000..3da7c42b549
--- /dev/null
+++ b/regress/usr.bin/m4/synch1.out
@@ -0,0 +1,7 @@
+synch1.m4:1:This is a test for multi MARK1
+synch1.m4:2:line synchronization stuff MARK2
+synch1.m4:3:# Let's include comments to see MARK3
+synch1.m4:4:quoted string MARK4
+synch1.m4:7:Return to normal MARK5
+synch1.m4:11:And let's see where we are now. MARK6
+synch1.m4:13:MARK7
diff --git a/regress/usr.bin/m4/synch1bis.out b/regress/usr.bin/m4/synch1bis.out
new file mode 100644
index 00000000000..aa765e4f62a
--- /dev/null
+++ b/regress/usr.bin/m4/synch1bis.out
@@ -0,0 +1,7 @@
+stdin:1:This is a test for multi MARK1
+stdin:2:line synchronization stuff MARK2
+stdin:3:# Let's include comments to see MARK3
+stdin:4:quoted string MARK4
+stdin:7:Return to normal MARK5
+stdin:11:And let's see where we are now. MARK6
+stdin:13:MARK7