Unify roff macro argument parsing (in roff.c, roff_userdef()) and man macro

argument parsing (in man_argv.c, man_args()), both having different bugs,
to use one common macro argument parser (in mandoc.c, mandoc_getarg()),
because from the point of view of roff, man macros are just roff macros,
hence their arguments are parsed in exactly the same way.

While doing so, fix these bugs:
 * Escaped blanks (i.e. those preceded by an odd number of backslashes)
   were mishandled as argument separators in unquoted arguments to
   user-defined roff macros.
 * Unescaped blanks preceded by an even number of backslashes were not
   recognized as argument separators in unquoted arguments to man macros.
 * Escaped backslashes (i.e. pairs of backslashes) were not reduced
   to single backslashes both in unquoted and quoted arguments both
   to user-defined roff macros and to man macros.
 * Escaped quotes (i.e. pairs of quotes inside quoted arguments) were
   not reduced to single quotes in man macros.

OK kristaps@

Note that mdoc macro argument parsing is yet another beast for no good
reason and is probably afflicted by similar bugs.  But i don't attempt
to fix that right now because it is intricately entangled with lots of
unrelated high-level mdoc(7) functionality, like delimiter handling and
column list phrase handling.  Disentagling that would waste too much
time now.

10 files changed, 356 insertions, 118 deletions

diff --git a/regress/usr.bin/mandoc/roff/Makefile b/regress/usr.bin/mandoc/roff/Makefile
index 33c2617c79e..cf54fb8af15 100644
--- a/regress/usr.bin/mandoc/roff/Makefile
+++ b/regress/usr.bin/mandoc/roff/Makefile
@@ -1,6 +1,6 @@
-# $OpenBSD: Makefile,v 1.2 2010/12/09 20:56:30 schwarze Exp $
+# $OpenBSD: Makefile,v 1.3 2011/01/03 22:27:21 schwarze Exp $
 
-SUBDIR+= cond string
+SUBDIR+= args cond string
 
 groff groff-clean: _SUBDIRUSE
 
diff --git a/regress/usr.bin/mandoc/roff/args/Makefile b/regress/usr.bin/mandoc/roff/args/Makefile
new file mode 100644
index 00000000000..3df35e48910
--- /dev/null
+++ b/regress/usr.bin/mandoc/roff/args/Makefile
@@ -0,0 +1,6 @@
+# $OpenBSD: Makefile,v 1.1 2011/01/03 22:27:21 schwarze Exp $
+
+REGRESS_TARGETS=roff man
+GROFF_TARGETS=roff man
+
+.include <bsd.regress.mk>
diff --git a/regress/usr.bin/mandoc/roff/args/man.in b/regress/usr.bin/mandoc/roff/args/man.in
new file mode 100644
index 00000000000..190e66d5709
--- /dev/null
+++ b/regress/usr.bin/mandoc/roff/args/man.in
@@ -0,0 +1,113 @@
+.TH ARGS-MAN 1 "January 1, 2011"
+.SH NAME
+args-man - arguments to man macros
+.SH DESCRIPTION
+standard unquoted:
+.IB one two
+text
+.br
+escaped blanks:
+.IB one\ one two\ two
+text
+.br
+escaped 'e' character:
+.IB one\eone two
+text
+.br
+.\"escaped backslash before blank:
+.\"IB one\\ two
+.\"text
+.\"br
+escaped backslash before 'e' character:
+.IB one\\e two
+text
+.br
+double inter-argument space:
+.IB one  two
+text
+.br
+triple inter-argument space:
+.IB one   two
+text
+.br
+single eol blank:
+.IB one two 
+text
+.br
+double eol blank:
+.IB one two  
+text
+.br
+triple eol blank:
+.IB one two   
+text
+.br
+standard quoted:
+.IB "one" "two"
+text
+.br
+quoted quotes:
+.IB "one""one" """two"""
+text
+.br
+quoted whitespace:
+.IB "one one" "two two"
+text
+.br
+escaped 'e' characters:
+.IB "one \e one" "\e"
+text
+.br
+escaped backslash before blank:
+.IB "one\\ one" "\\ "
+text
+.br
+escaped backslash before 'e' character:
+.IB "one\\eone" "\\e"
+text
+.br
+double inter-argument space:
+.IB "one one"  "two two"
+text
+.br
+triple inter-argument space:
+.IB "one one"   "two two"
+text
+.br
+missing inter-argument space:
+.IB "one one"two\ two
+text
+.br
+single eol blank:
+.IB "one one" "two two" 
+text
+.br
+double eol blank:
+.IB "one one" "two two"  
+text
+.br
+triple eol blank:
+.IB "one one" "two two"   
+text
+.br
+.\"trailing blanks in arguments:
+.\"IB "one " "two "
+.\"text
+.\"br
+unterminated quotes:
+.IB "one
+.IB one "two
+text
+.br
+.\"single trailing blank in unterminated quotes:
+.\"IB "one 
+.\"IB one "two 
+.\"text
+.\"br
+.\"double trailing blank in unterminated quotes:
+.\"IB "one  
+.\"IB one "two  
+.\"text
+.\"br
+backslash at eol:
+.IB one two\
diff --git a/regress/usr.bin/mandoc/roff/args/man.out_ascii b/regress/usr.bin/mandoc/roff/args/man.out_ascii
new file mode 100644
index 00000000000..3b945f483ed
--- /dev/null
+++ b/regress/usr.bin/mandoc/roff/args/man.out_ascii
@@ -0,0 +1,34 @@
+ARGS-MAN(1)                                                        ARGS-MAN(1)
+
+
+
+NNAAMMEE
+       args-man - arguments to man macros
+
+DDEESSCCRRIIPPTTIIOONN
+       standard unquoted: _o_n_ettwwoo text
+       escaped blanks: _o_n_e _o_n_ettwwoo ttwwoo text
+       escaped 'e' character: _o_n_e_\_o_n_ettwwoo text
+       escaped backslash before 'e' character: _o_n_e_\ttwwoo text
+       double inter-argument space: _o_n_ettwwoo text
+       triple inter-argument space: _o_n_ettwwoo text
+       single eol blank: _o_n_ettwwoo text
+       double eol blank: _o_n_ettwwoo text
+       triple eol blank: _o_n_ettwwoo text
+       standard quoted: _o_n_ettwwoo text
+       quoted quotes: _o_n_e_"_o_n_e""ttwwoo"" text
+       quoted whitespace: _o_n_e _o_n_ettwwoo ttwwoo text
+       escaped 'e' characters: _o_n_e _\ _o_n_e\\ text
+       escaped backslash before blank: _o_n_e _o_n_e  text
+       escaped backslash before 'e' character: _o_n_e_\_o_n_e\\ text
+       double inter-argument space: _o_n_e _o_n_ettwwoo ttwwoo text
+       triple inter-argument space: _o_n_e _o_n_ettwwoo ttwwoo text
+       missing inter-argument space: _o_n_e _o_n_ettwwoo ttwwoo text
+       single eol blank: _o_n_e _o_n_ettwwoo ttwwoo text
+       double eol blank: _o_n_e _o_n_ettwwoo ttwwoo text
+       triple eol blank: _o_n_e _o_n_ettwwoo ttwwoo text
+       unterminated quotes: _o_n_e _o_n_ettwwoo text
+       backslash at eol: _o_n_ettwwoo
+
+
+
diff --git a/regress/usr.bin/mandoc/roff/args/roff.in b/regress/usr.bin/mandoc/roff/args/roff.in
new file mode 100644
index 00000000000..ff9a5781cc5
--- /dev/null
+++ b/regress/usr.bin/mandoc/roff/args/roff.in
@@ -0,0 +1,65 @@
+.TH ARGS-ROFF 1 "January 1, 2011"
+.SH NAME
+args-roff - arguments to roff macros
+.SH DESCRIPTION
+.de test
+(\\$1) (\\$2)
+.br
+..
+standard unquoted:
+.test one two
+escaped blanks:
+.test one\ one two\ two
+escaped 'e' character:
+.test one\eone two
+escaped backslash before blank:
+.test one\\ two
+escaped backslash before 'e' character:
+.test one\\e two
+double inter-argument space:
+.test one  two
+triple inter-argument space:
+.test one   two
+single eol blank:
+.test one two 
+double eol blank:
+.test one two  
+triple eol blank:
+.test one two   
+standard quoted:
+.test "one" "two"
+quoted quotes:
+.test "one""one" """two"""
+quoted whitespace:
+.test "one one" "two two"
+escaped 'e' characters:
+.test "one \e one" "\e"
+escaped backslash before blank:
+.test "one\\ one" "\\ "
+escaped backslash before 'e' character:
+.test "one\\eone" "\\e"
+double inter-argument space:
+.test "one one"  "two two"
+triple inter-argument space:
+.test "one one"   "two two"
+missing inter-argument space:
+.test "one one"two\ two
+single eol blank:
+.test "one one" "two two" 
+double eol blank:
+.test "one one" "two two"  
+triple eol blank:
+.test "one one" "two two"   
+trailing blanks in arguments:
+.test "one " "two "
+unterminated quotes:
+.\"test "one
+.test one "two
+single trailing blank in unterminated quotes:
+.\"test "one 
+.test one "two 
+double trailing blank in unterminated quotes:
+.\"test "one  
+.test one "two  
+backslash at eol:
+.test one two\
diff --git a/regress/usr.bin/mandoc/roff/args/roff.out_ascii b/regress/usr.bin/mandoc/roff/args/roff.out_ascii
new file mode 100644
index 00000000000..e8192088e82
--- /dev/null
+++ b/regress/usr.bin/mandoc/roff/args/roff.out_ascii
@@ -0,0 +1,38 @@
+ARGS-ROFF(1)                                                      ARGS-ROFF(1)
+
+
+
+NNAAMMEE
+       args-roff - arguments to roff macros
+
+DDEESSCCRRIIPPTTIIOONN
+       standard unquoted: (one) (two)
+       escaped blanks: (one one) (two two)
+       escaped 'e' character: (one\one) (two)
+       escaped backslash before blank: (one) (two)
+       escaped backslash before 'e' character: (one\) (two)
+       double inter-argument space: (one) (two)
+       triple inter-argument space: (one) (two)
+       single eol blank: (one) (two)
+       double eol blank: (one) (two)
+       triple eol blank: (one) (two)
+       standard quoted: (one) (two)
+       quoted quotes: (one"one) ("two")
+       quoted whitespace: (one one) (two two)
+       escaped 'e' characters: (one \ one) (\)
+       escaped backslash before blank: (one one) ( )
+       escaped backslash before 'e' character: (one\one) (\)
+       double inter-argument space: (one one) (two two)
+       triple inter-argument space: (one one) (two two)
+       missing inter-argument space: (one one) (two two)
+       single eol blank: (one one) (two two)
+       double eol blank: (one one) (two two)
+       triple eol blank: (one one) (two two)
+       trailing blanks in arguments: (one ) (two )
+       unterminated quotes: (one) (two)
+       single trailing blank in unterminated quotes: (one) (two )
+       double trailing blank in unterminated quotes: (one) (two  )
+       backslash at eol: (one) (two)
+
+
+
diff --git a/usr.bin/mandoc/libmandoc.h b/usr.bin/mandoc/libmandoc.h
index c79a3c46689..c066e6db9c3 100644
--- a/usr.bin/mandoc/libmandoc.h
+++ b/usr.bin/mandoc/libmandoc.h
@@ -1,4 +1,4 @@
-/*	$Id: libmandoc.h,v 1.7 2010/07/16 00:34:33 schwarze Exp $ */
+/*	$Id: libmandoc.h,v 1.8 2011/01/03 22:27:21 schwarze Exp $ */
 /*
  * Copyright (c) 2009, 2010 Kristaps Dzonsons <kristaps@bsd.lv>
  *
@@ -24,6 +24,7 @@ void		*mandoc_calloc(size_t, size_t);
 char		*mandoc_strdup(const char *);
 void		*mandoc_malloc(size_t);
 void		*mandoc_realloc(void *, size_t);
+char		*mandoc_getarg(char **, mandocmsg, void *, int, int *);
 time_t		 mandoc_a2time(int, const char *);
 #define		 MTIME_CANONICAL	(1 << 0)
 #define		 MTIME_REDUCED		(1 << 1)
diff --git a/usr.bin/mandoc/man_argv.c b/usr.bin/mandoc/man_argv.c
index 60fd3d42b22..25271da7a04 100644
--- a/usr.bin/mandoc/man_argv.c
+++ b/usr.bin/mandoc/man_argv.c
@@ -1,6 +1,6 @@
-/*	$Id: man_argv.c,v 1.3 2010/07/31 23:42:04 schwarze Exp $ */
+/*	$Id: man_argv.c,v 1.4 2011/01/03 22:27:21 schwarze Exp $ */
 /*
- * Copyright (c) 2008, 2009 Kristaps Dzonsons <kristaps@bsd.lv>
+ * Copyright (c) 2011 Ingo Schwarze <schwarze@openbsd.org>
  *
  * Permission to use, copy, modify, and distribute this software for any
  * purpose with or without fee is hereby granted, provided that the above
@@ -17,84 +17,24 @@
 #include <sys/types.h>
 
 #include <assert.h>
-#include <stdlib.h>
-#include <string.h>
 
 #include "mandoc.h"
 #include "libman.h"
+#include "libmandoc.h"
 
 
 int
 man_args(struct man *m, int line, int *pos, char *buf, char **v)
 {
+	char	 *start;
 
 	assert(*pos);
-	assert(' ' != buf[*pos]);
+	*v = start = buf + *pos;
+	assert(' ' != *start);
 
-	if (0 == buf[*pos])
+	if ('\0' == *start)
 		return(ARGS_EOLN);
 
-	*v = &buf[*pos];
-
-	/* 
-	 * Process a quoted literal.  A quote begins with a double-quote
-	 * and ends with a double-quote NOT preceded by a double-quote.
-	 * Whitespace is NOT involved in literal termination.
-	 */
-
-	if ('\"' == buf[*pos]) {
-		*v = &buf[++(*pos)];
-
-		for ( ; buf[*pos]; (*pos)++) {
-			if ('\"' != buf[*pos])
-				continue;
-			if ('\"' != buf[*pos + 1])
-				break;
-			(*pos)++;
-		}
-
-		if (0 == buf[*pos]) {
-			if ( ! man_pmsg(m, line, *pos, MANDOCERR_BADQUOTE))
-				return(ARGS_ERROR);
-			return(ARGS_QWORD);
-		}
-
-		buf[(*pos)++] = 0;
-
-		if (0 == buf[*pos])
-			return(ARGS_QWORD);
-
-		while (' ' == buf[*pos])
-			(*pos)++;
-
-		if (0 == buf[*pos])
-			if ( ! man_pmsg(m, line, *pos, MANDOCERR_EOLNSPACE))
-				return(ARGS_ERROR);
-
-		return(ARGS_QWORD);
-	}
-
-	/* 
-	 * A non-quoted term progresses until either the end of line or
-	 * a non-escaped whitespace.
-	 */
-
-	for ( ; buf[*pos]; (*pos)++)
-		if (' ' == buf[*pos] && '\\' != buf[*pos - 1])
-			break;
-
-	if (0 == buf[*pos])
-		return(ARGS_WORD);
-
-	buf[(*pos)++] = 0;
-
-	while (' ' == buf[*pos])
-		(*pos)++;
-
-	if (0 == buf[*pos])
-		if ( ! man_pmsg(m, line, *pos, MANDOCERR_EOLNSPACE))
-			return(ARGS_ERROR);
-
-	return(ARGS_WORD);
+	*v = mandoc_getarg(v, m->msg, m->data, line, pos);
+	return('"' == *start ? ARGS_QWORD : ARGS_WORD);
 }
-
diff --git a/usr.bin/mandoc/mandoc.c b/usr.bin/mandoc/mandoc.c
index cd8707adc84..92bd1a3454b 100644
--- a/usr.bin/mandoc/mandoc.c
+++ b/usr.bin/mandoc/mandoc.c
@@ -1,14 +1,15 @@
-/*	$Id: mandoc.c,v 1.20 2010/09/27 21:25:28 schwarze Exp $ */
+/*	$Id: mandoc.c,v 1.21 2011/01/03 22:27:21 schwarze Exp $ */
 /*
  * Copyright (c) 2008, 2009, 2010 Kristaps Dzonsons <kristaps@bsd.lv>
+ * Copyright (c) 2011 Ingo Schwarze <schwarze@openbsd.org>
  *
  * Permission to use, copy, modify, and distribute this software for any
  * purpose with or without fee is hereby granted, provided that the above
  * copyright notice and this permission notice appear in all copies.
  *
- * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHORS DISCLAIM ALL WARRANTIES
  * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
- * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR
  * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
  * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
  * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
@@ -279,6 +280,83 @@ mandoc_strdup(const char *ptr)
 	return(p);
 }
 
+/*
+ * Parse a quoted or unquoted roff-style request or macro argument.
+ * Return a pointer to the parsed argument, which is either the original
+ * pointer or advanced by one byte in case the argument is quoted.
+ * Null-terminate the argument in place.
+ * Collapse pairs of quotes inside quoted arguments.
+ * Advance the argument pointer to the next argument,
+ * or to the null byte terminating the argument line.
+ */
+char *
+mandoc_getarg(char **cpp, mandocmsg msg, void *data, int ln, int *pos)
+{
+	char	 *start, *cp;
+	int	  quoted, pairs, white;
+
+	/* Quoting can only start with a new word. */
+	start = *cpp;
+	if ('"' == *start) {
+		quoted = 1;
+		start++;
+	} else
+		quoted = 0;
+
+	pairs = 0;
+	white = 0;
+	for (cp = start; '\0' != *cp; cp++) {
+		/* Move left after quoted quotes and escaped backslashes. */
+		if (pairs)
+			cp[-pairs] = cp[0];
+		if ('\\' == cp[0]) {
+			if ('\\' == cp[1]) {
+				/* Poor man's copy mode. */
+				pairs++;
+				cp++;
+			} else if (0 == quoted && ' ' == cp[1])
+				/* Skip escaped blanks. */
+				cp++;
+		} else if (0 == quoted) {
+			if (' ' == cp[0]) {
+				/* Unescaped blanks end unquoted args. */
+				white = 1;
+				break;
+			}
+		} else if ('"' == cp[0]) {
+			if ('"' == cp[1]) {
+				/* Quoted quotes collapse. */
+				pairs++;
+				cp++;
+			} else {
+				/* Unquoted quotes end quoted args. */
+				quoted = 2;
+				break;
+			}
+		}
+	}
+
+	/* Quoted argument without a closing quote. */
+	if (1 == quoted && msg)
+		(*msg)(MANDOCERR_BADQUOTE, data, ln, *pos, NULL);
+
+	/* Null-terminate this argument and move to the next one. */
+	if (pairs)
+		cp[-pairs] = '\0';
+	if ('\0' != *cp) {
+		*cp++ = '\0';
+		while (' ' == *cp)
+			cp++;
+	}
+	*pos += (cp - start) + (quoted ? 1 : 0);
+	*cpp = cp;
+
+	if ('\0' == *cp && msg && (white || ' ' == cp[-1]))
+		(*msg)(MANDOCERR_EOLNSPACE, data, ln, *pos, NULL);
+
+	return(start);
+}
+
 
 static int
 a2time(time_t *t, const char *fmt, const char *p)
diff --git a/usr.bin/mandoc/roff.c b/usr.bin/mandoc/roff.c
index a7337063603..49dd9fb0047 100644
--- a/usr.bin/mandoc/roff.c
+++ b/usr.bin/mandoc/roff.c
@@ -1,7 +1,7 @@
-/*	$Id: roff.c,v 1.24 2010/12/21 01:30:58 schwarze Exp $ */
+/*	$Id: roff.c,v 1.25 2011/01/03 22:27:21 schwarze Exp $ */
 /*
  * Copyright (c) 2010 Kristaps Dzonsons <kristaps@bsd.lv>
- * Copyright (c) 2010 Ingo Schwarze <schwarze@openbsd.org>
+ * Copyright (c) 2010, 2011 Ingo Schwarze <schwarze@openbsd.org>
  *
  * Permission to use, copy, modify, and distribute this software for any
  * purpose with or without fee is hereby granted, provided that the above
@@ -1124,53 +1124,16 @@ roff_userdef(ROFF_ARGS)
 {
 	const char	 *arg[9];
 	char		 *cp, *n1, *n2;
-	int		  i, quoted, pairs;
+	int		  i;
 
 	/*
 	 * Collect pointers to macro argument strings
 	 * and null-terminate them.
 	 */
 	cp = *bufp + pos;
-	for (i = 0; i < 9; i++) {
-		/* Quoting can only start with a new word. */
-		if ('"' == *cp) {
-			quoted = 1;
-			cp++;
-		} else
-			quoted = 0;
-		arg[i] = cp;
-		for (pairs = 0; '\0' != *cp; cp++) {
-			/* Unquoted arguments end at blanks. */
-			if (0 == quoted) {
-				if (' ' == *cp)
-					break;
-				continue;
-			}
-			/* After pairs of quotes, move left. */
-			if (pairs)
-				cp[-pairs] = cp[0];
-			/* Pairs of quotes do not end words, ... */
-			if ('"' == cp[0] && '"' == cp[1]) {
-				pairs++;
-				cp++;
-				continue;
-			}
-			/* ... but solitary quotes do. */
-			if ('"' != *cp)
-				continue;
-			if (pairs)
-				cp[-pairs] = '\0';
-			*cp = ' ';
-			break;
-		}
-		/* Last argument; the remaining ones are empty strings. */
-		if ('\0' == *cp)
-			continue;
-		/* Null-terminate argument and move to the next one. */
-		*cp++ = '\0';
-		while (' ' == *cp)
-			cp++;
-	}
+	for (i = 0; i < 9; i++)
+		arg[i] = '\0' == *cp ? NULL :
+		    mandoc_getarg(&cp, r->msg, r->data, ln, &pos);
 
 	/*
 	 * Expand macro arguments.