Synch with my current work.

Numerous changes:
- generate can build several tables
- style cleanup
- statistics code
- use variable names throughout (struct Name)
- recursive variables everywhere
- faster parser (pass buffer along instead of allocating multiple copies)
- correct parser. Handles comments everywhere, and ; correctly
- more string intervals
- simplified dir.c, less recursion.
- extended for loops
- sinclude()
- finished removing extra junk from Lst_*
- handles ${@D} and friends in a simpler way
- cleaned up and modular VarModifiers handling.
- recognizes some gnu Makefile usages and errors out about them.

Additionally, some extra functionality is defined by FEATURES. The set of
functionalities is currently hardcoded to OpenBSD defaults, but this may
include support for some NetBSD extensions, like ODE modifiers.

Backed by miod@ and millert@, who finally got sick of my endless patches...

1 files changed, 231 insertions, 317 deletions

diff --git a/usr.bin/make/lowparse.c b/usr.bin/make/lowparse.c
index 9563ab47fad..7e74ce66b48 100644
--- a/usr.bin/make/lowparse.c
+++ b/usr.bin/make/lowparse.c
@@ -1,4 +1,5 @@
-/*	$OpenBSD: lowparse.c,v 1.5 2000/11/24 14:27:19 espie Exp $ */
+/*	$OpenPackages$ */
+/*	$OpenBSD: lowparse.c,v 1.6 2001/05/03 13:41:07 espie Exp $ */
 
 /* low-level parsing functions. */
 
@@ -42,32 +43,31 @@ static LIST	    fileNames;	/* file names to free at end */
 
 /* Definitions for handling #include specifications */
 typedef struct IFile_ {
-    char           	*fname;	/* name of file */
-    unsigned long      	lineno;	/* line number */
-    FILE 		*F;	/* open stream */
-    char 		*str;	/* read from char area */	
-    char 		*ptr;	/* where we are */
-    char 		*end;	/* don't overdo it */
+    char		*fname; /* name of file */
+    unsigned long	lineno; /* line number */
+    FILE		*F;	/* open stream */
+    char		*str;	/* read from char area */
+    char		*ptr;	/* where we are */
+    char		*end;	/* don't overdo it */
 } IFile;
 
 static IFile	*current;
 
-
-
-static LIST     includes;  	/* stack of IFiles generated by
+static LIST	includes;	/* stack of IFiles generated by
 				 * #includes */
 
-static IFile *new_ifile __P((char *, FILE *));
-static IFile *new_istring __P((char *, char *, unsigned long));
-static void free_ifile __P((IFile *));
-static void ParseVErrorInternal __P((char *, unsigned long, int, char *, va_list));
-static int newline __P((void));
-#define ParseReadc()	current->ptr < current->end ? *current->ptr++ : newline()
-static void ParseUnreadc __P((char));
+static IFile *new_ifile(char *, FILE *);
+static IFile *new_istring(char *, char *, unsigned long);
+static void free_ifile(IFile *);
+static void ParseVErrorInternal(char *, unsigned long, int, char *, va_list);
+static int newline(void);
+static int skiptoendofline(void);
+static void ParseFoldLF(Buffer, int);
+static int ParseSkipEmptyLines(Buffer);
 static int	    fatals = 0;
 
 /*-
- * ParseVErrorInternal  --
+ * ParseVErrorInternal	--
  *	Error message abort function for parsing. Prints out the context
  *	of the error (line number and file) as well as the message with
  *	two optional arguments.
@@ -96,7 +96,7 @@ ParseVErrorInternal(va_alist)
 }
 
 /*-
- * Parse_Error  --
+ * Parse_Error	--
  *	External interface to ParseVErrorInternal; uses the default filename
  *	Line number.
  */
@@ -148,9 +148,9 @@ free_ifile(ifile)
     IFile *ifile;
 {
     if (ifile->F)
-    	(void)fclose(ifile->F);
+	(void)fclose(ifile->F);
     else
-    	free(ifile->str);
+	free(ifile->str);
     /* Note we can't free the file names yet, as they are embedded in GN for
      * error reports. */
     free(ifile);
@@ -165,7 +165,7 @@ new_istring(str, name, lineno)
     IFile *ifile;
 
     ifile = emalloc(sizeof(*ifile));
-    /* No malloc, name is always taken from an existing ifile */
+    /* No malloc, name is always taken from an already existing ifile */
     ifile->fname = name;
     ifile->F = NULL;
     /* Strings are used from for loops... */
@@ -188,8 +188,8 @@ new_istring(str, name, lineno)
  */
 void
 Parse_FromString(str, lineno)
-    char 		*str;
-    unsigned long 	lineno;
+    char		*str;
+    unsigned long	lineno;
 {
     if (DEBUG(FOR))
 	(void)fprintf(stderr, "%s\n----\n", str);
@@ -200,19 +200,19 @@ Parse_FromString(str, lineno)
 }
 
 
-void 
+void
 Parse_FromFile(name, stream)
     char		*name;
     FILE		*stream;
 {
     if (current != NULL)
-    	Lst_AtFront(&includes, current);
+	Lst_AtFront(&includes, current);
     current = new_ifile(name, stream);
 }
 
 /*-
  *---------------------------------------------------------------------
- * Parse_NextFile -- 
+ * Parse_NextFile --
  *	Called when EOF is reached in the current file. If we were reading
  *	an include file, the includes stack is popped and things set up
  *	to go back to reading the previous file at the previous location.
@@ -235,12 +235,11 @@ Parse_NextFile()
 	if (current != NULL)
 	    free_ifile(current);
 	current = next;
-    	return TRUE;
+	return TRUE;
     } else
-    	return FALSE;
+	return FALSE;
 }
 
-
 /* guts for ParseReadc. Grab a new line off fgetln when we hit "\n" */
 static int
 newline()
@@ -259,325 +258,240 @@ newline()
     return EOF;
 }
 
-void 
-ParseUnreadc(c)
-	char c;
+#define ParseReadc()	current->ptr < current->end ? *current->ptr++ : newline()
+
+/* Take advantage of fgetln: we don't have to scan a whole line to skip it.
+ */
+static int
+skiptoendofline()
 {
-	current->ptr--;
-	*current->ptr = c;
+    if (current->F) {
+	if (current->end - current->ptr > 1)
+	    current->ptr = current->end - 1;
+	if (*current->ptr == '\n')
+	    return *current->ptr++;
+	return EOF;
+    } else {
+	int c;
+
+	do {
+	    c = ParseReadc();
+	} while (c != '\n' && c != EOF);
+	return c;
+    }
 }
 
-/* ParseSkipLine():
- *	Grab the next line
+
+/* ParseSkipGetLine():
+ *	Return the first logical line that starts with '.'
  */
 char *
-ParseSkipLine(skip)
-    int skip; 		/* Skip lines that don't start with . */
+ParseSkipGetLine(linebuf)
+    Buffer linebuf;
 {
-    char *line;
-    int c, lastc;
-    BUFFER buf;
+    int c;
+
+	/* If first char isn't dot, skip to end of line, handling \ */
+    while ((c = ParseReadc()) != '.') {
+	for (;c != '\n'; c = ParseReadc()) {
+	    if (c == '\\') {
+		c = ParseReadc();
+		if (c == '\n')
+		    current->lineno++;
+	    }
+	    if (c == EOF) {
+		Parse_Error(PARSE_FATAL, "Unclosed conditional");
+		return NULL;
+	    }
+	}
+	current->lineno++;
+    }
 
-    Buf_Init(&buf, MAKE_BSIZE);
+	/* This is the line we need to copy */
+    return ParseGetLine(linebuf, "conditional");
+}
 
+/* Grab logical line into linebuf.
+ * The first character is already in c. */
+static void
+ParseFoldLF(linebuf, c)
+    Buffer linebuf;
+    int c;
+{
     for (;;) {
-        Buf_Reset(&buf);
-        lastc = '\0';
-
-        while (((c = ParseReadc()) != '\n' || lastc == '\\')
-               && c != EOF) {
-            if (c == '\n') {
-                Buf_ReplaceLastChar(&buf, ' ');
-                current->lineno++;
-
-                while ((c = ParseReadc()) == ' ' || c == '\t');
-
-                if (c == EOF)
-                    break;
-            }
-
-            Buf_AddChar(&buf, c);
-            lastc = c;
-        }
-
-        line = Buf_Retrieve(&buf);
-        current->lineno++;
-	    /* allow for non-newline terminated lines while skipping */
-	if (line[0] == '.')
+	if (c == '\n') {
+	    current->lineno++;
 	    break;
-
-        if (c == EOF) {
-            Parse_Error(PARSE_FATAL, "Unclosed conditional/for loop");
-            Buf_Destroy(&buf);
-            return NULL;
-        }
-	if (skip == 0)
+	}
+	if (c == EOF)
 	    break;
-
+	Buf_AddChar(linebuf, c);
+	c = ParseReadc();
+	while (c == '\\') {
+	    c = ParseReadc();
+	    if (c == '\n') {
+		Buf_AddSpace(linebuf);
+		current->lineno++;
+		do {
+		    c = ParseReadc();
+		} while (c == ' ' || c == '\t');
+	    } else {
+		Buf_AddChar(linebuf, '\\');
+		if (c == '\\') {
+		    Buf_AddChar(linebuf, '\\');
+		    c = ParseReadc();
+		}
+		break;
+	    }
+	}
     }
-
-    return line;
 }
 
-
-/*-
- *---------------------------------------------------------------------
- * ParseReadLine --
- *	Read an entire line from the input file. Called only by Parse_File.
- *	To facilitate escaped newlines and what have you, a character is
- *	buffered in 'lastc', which is '\0' when no characters have been
- *	read. When we break out of the loop, c holds the terminating
- *	character and lastc holds a character that should be added to
- *	the line (unless we don't read anything but a terminator).
- *
- * Results:
- *	A line w/o its newline
- *
- * Side Effects:
- *	Only those associated with reading a character
- *---------------------------------------------------------------------
+/* ParseGetLine:
+ *	Simply get line, no parsing beyond \
  */
 char *
-ParseReadLine ()
+ParseGetLine(linebuf, type)
+    Buffer	linebuf;
+    const char	*type;
 {
-    BUFFER  	  buf;	    	/* Buffer for current line */
-    register int  c;	      	/* the current character */
-    register int  lastc;    	/* The most-recent character */
-    Boolean	  semiNL;     	/* treat semi-colons as newlines */
-    Boolean	  ignDepOp;   	/* TRUE if should ignore dependency operators
-				 * for the purposes of setting semiNL */
-    Boolean 	  ignComment;	/* TRUE if should ignore comments (in a
-				 * shell command */
-    char 	  *line;    	/* Result */
-    char          *ep;		/* to strip trailing blanks */
-
-    semiNL = FALSE;
-    ignDepOp = FALSE;
-    ignComment = FALSE;
-
-    /*
-     * Handle special-characters at the beginning of the line. Either a
-     * leading tab (shell command) or pound-sign (possible conditional)
-     * forces us to ignore comments and dependency operators and treat
-     * semi-colons as semi-colons (by leaving semiNL FALSE). This also
-     * discards completely blank lines.
-     */
-    for (;;) {
-	c = ParseReadc();
+    int c;
 
-	if (c == '\t') {
-	    ignComment = ignDepOp = TRUE;
-	    break;
-	} else if (c == '\n') {
+    Buf_Reset(linebuf);
+    c = ParseReadc();
+    if (c == EOF) {
+	Parse_Error(PARSE_FATAL, "Unclosed %s", type);
+	return NULL;
+    }
+
+    /* Handle '\' at beginning of line, since \\n needs special treatment */
+    while (c == '\\') {
+	c = ParseReadc();
+	if (c == '\n') {
 	    current->lineno++;
-	} else if (c == '#') {
-	    ParseUnreadc(c);
-	    break;
+	    do {
+		c = ParseReadc();
+	    } while (c == ' ' || c == '\t');
 	} else {
-	    /*
-	     * Anything else breaks out without doing anything
-	     */
+	    Buf_AddChar(linebuf, '\\');
+	    if (c == '\\') {
+		Buf_AddChar(linebuf, '\\');
+		c = ParseReadc();
+	    }
 	    break;
 	}
     }
+    ParseFoldLF(linebuf, c);
 
-    if (c != EOF) {
-	lastc = c;
-	Buf_Init(&buf, MAKE_BSIZE);
-
-	while (((c = ParseReadc ()) != '\n' || (lastc == '\\')) &&
-	       (c != EOF))
-	{
-test_char:
-	    switch(c) {
-	    case '\n':
-		/*
-		 * Escaped newline: read characters until a non-space or an
-		 * unescaped newline and replace them all by a single space.
-		 * This is done by storing the space over the backslash and
-		 * dropping through with the next nonspace. If it is a
-		 * semi-colon and semiNL is TRUE, it will be recognized as a
-		 * newline in the code below this...
-		 */
-		current->lineno++;
-		lastc = ' ';
-		while ((c = ParseReadc ()) == ' ' || c == '\t') {
-		    continue;
-		}
-		if (c == EOF || c == '\n') {
-		    goto line_read;
-		} else {
-		    /*
-		     * Check for comments, semiNL's, etc. -- easier than
-		     * ParseUnreadc(c); continue;
-		     */
-		    goto test_char;
-		}
-		/*NOTREACHED*/
-		break;
+    return Buf_Retrieve(linebuf);
+}
 
-	    case ';':
-		/*
-		 * Semi-colon: Need to see if it should be interpreted as a
-		 * newline
-		 */
-		if (semiNL) {
-		    /*
-		     * To make sure the command that may be following this
-		     * semi-colon begins with a tab, we push one back into the
-		     * input stream. This will overwrite the semi-colon in the
-		     * buffer. If there is no command following, this does no
-		     * harm, since the newline remains in the buffer and the
-		     * whole line is ignored.
-		     */
-		    ParseUnreadc('\t');
-		    goto line_read;
-		}
-		break;
-	    case '=':
-		if (!semiNL) {
-		    /*
-		     * Haven't seen a dependency operator before this, so this
-		     * must be a variable assignment -- don't pay attention to
-		     * dependency operators after this.
-		     */
-		    ignDepOp = TRUE;
-		} else if (lastc == ':' || lastc == '!') {
-		    /*
-		     * Well, we've seen a dependency operator already, but it
-		     * was the previous character, so this is really just an
-		     * expanded variable assignment. Revert semi-colons to
-		     * being just semi-colons again and ignore any more
-		     * dependency operators.
-		     *
-		     * XXX: Note that a line like "foo : a:=b" will blow up,
-		     * but who'd write a line like that anyway?
-		     */
-		    ignDepOp = TRUE; semiNL = FALSE;
-		}
-		break;
-	    case '#':
-		if (!ignComment) {
-		    if (
-#if 0
-		    compatMake &&
-#endif
-		    (lastc != '\\')) {
-			/*
-			 * If the character is a hash mark and it isn't escaped
-			 * (or we're being compatible), the thing is a comment.
-			 * Skip to the end of the line.
-			 */
-			do {
-			    c = ParseReadc();
-			} while ((c != '\n') && (c != EOF));
-			goto line_read;
-		    } else {
-			/*
-			 * Don't add the backslash. Just let the # get copied
-			 * over.
-			 */
-			lastc = c;
-			continue;
+/* Skip all empty lines, and return the first `useful' character.
+ * (This does skip blocks of comments at high speed, which justifies
+ * the complexity of the function.)
+ */
+static int
+ParseSkipEmptyLines(linebuf)
+    Buffer	linebuf;
+{
+    int 	c;		/* the current character */
+
+    for (;;) {
+	Buf_Reset(linebuf);
+	c = ParseReadc();
+	/* Strip leading spaces, fold on '\n' */
+	if (c == ' ') {
+	    do {
+		c = ParseReadc();
+	    } while (c == ' ' || c == '\t');
+	    while (c == '\\') {
+		c = ParseReadc();
+		if (c == '\n') {
+		    current->lineno++;
+		    do {
+			c = ParseReadc();
+		    } while (c == ' ' || c == '\t');
+		} else {
+		    Buf_AddChar(linebuf, '\\');
+		    if (c == '\\') {
+			Buf_AddChar(linebuf, '\\');
+			c = ParseReadc();
 		    }
+		    if (c == EOF)
+			return '\n';
+		    else
+			return c;
 		}
-		break;
-	    case ':':
-	    case '!':
-		if (!ignDepOp && (c == ':' || c == '!')) {
-		    /*
-		     * A semi-colon is recognized as a newline only on
-		     * dependency lines. Dependency lines are lines with a
-		     * colon or an exclamation point. Ergo...
-		     */
-		    semiNL = TRUE;
-		}
-		break;
 	    }
-	    /*
-	     * Copy in the previous character and save this one in lastc.
-	     */
-	    Buf_AddChar(&buf, lastc);
-	    lastc = c;
-
-	}
-    line_read:
-	current->lineno++;
-
-	if (lastc != '\0') 
-	    Buf_AddChar(&buf, lastc);
-	line = Buf_Retrieve(&buf);
-
-	/*
-	 * Strip trailing blanks and tabs from the line.
-	 * Do not strip a blank or tab that is preceeded by
-	 * a '\'
-	 */
-	ep = line;
-	while (*ep)
-	    ++ep;
-	while (ep > line + 1 && (ep[-1] == ' ' || ep[-1] == '\t')) {
-	    if (ep > line + 1 && ep[-2] == '\\')
-		break;
-	    --ep;
+	    assert(c != '\t');
 	}
-	*ep = 0;
-
-	if (line[0] == '.') {
-	    /*
-	     * The line might be a conditional. Ask the conditional module
-	     * about it and act accordingly
-	     */
-	    switch (Cond_Eval (line)) {
-	    case COND_SKIP:
-		/*
-		 * Skip to next conditional that evaluates to COND_PARSE.
-		 */
-		do {
-		    free (line);
-		    line = ParseSkipLine(1);
-		} while (line && Cond_Eval(line) != COND_PARSE);
-		if (line == NULL)
-		    break;
-		/* FALLTHROUGH */
-	    case COND_PARSE:
-		free(line);
-		line = ParseReadLine();
-		break;
-	    case COND_INVALID:
-	    	{
-		For *loop;
-
-		loop = For_Eval(line);
-		if (loop != NULL) {
-		    Boolean ok;
-
-		    free(line);
+	if (c == '#')
+	    c = skiptoendofline();
+	/* Almost identical to spaces, except this occurs after comments
+	 * have been taken care of, and we keep the tab itself.  */
+	if (c == '\t') {
+	    Buf_AddChar(linebuf, '\t');
+	    do {
+		c = ParseReadc();
+	    } while (c == ' ' || c == '\t');
+	    while (c == '\\') {
+		c = ParseReadc();
+		if (c == '\n') {
+		    current->lineno++;
 		    do {
-			/* Find the matching endfor.  */
-			line = ParseSkipLine(0);
-			if (line == NULL) {
-			    Parse_Error(PARSE_FATAL,
-				     "Unexpected end of file in for loop.\n");
-			    return line;
-			}
-			ok = For_Accumulate(loop, line);
-			free(line);
-		    } while (ok);
-		    For_Run(loop);
-		    line = ParseReadLine();
-		}
-		break;
+			c = ParseReadc();
+		    } while (c == ' ' || c == '\t');
+		} else {
+		    Buf_AddChar(linebuf, '\\');
+		    if (c == '\\') {
+			Buf_AddChar(linebuf, '\\');
+			c = ParseReadc();
+		    }
+		    if (c == EOF)
+			return '\n';
+		    else
+			return c;
+		    return c;
 		}
 	    }
 	}
-	return (line);
+	if (c == '\n')
+	    current->lineno++;
+	else
+	    return c;
+    }
+}
 
-    } else {
-	/*
-	 * Hit end-of-file, so return a NULL line to indicate this.
-	 */
-	return((char *)NULL);
+/*-
+ *---------------------------------------------------------------------
+ * ParseReadLine --
+ *	Read an entire line from the input file.
+ *
+ * Results:
+ *	A line without a new line, or NULL at EOF.
+ *
+ * Notes:
+ *	Removes beginning and trailing blanks (but keeps first tab).
+ *	Updates line numbers, handles escaped newlines, and skip over
+ *	uninteresting lines.
+ *	All but trivial comments can't be handled at this point, because we
+ *	don't know yet which lines are shell commands or not.
+ *---------------------------------------------------------------------
+ */
+char *
+ParseReadLine(linebuf)
+    Buffer	linebuf;
+{
+    int 	c;		/* the current character */
+
+    c = ParseSkipEmptyLines(linebuf);
+
+    if (c == EOF)
+	return NULL;
+    else {
+	ParseFoldLF(linebuf, c);
+	Buf_KillTrailingSpaces(linebuf);
+	return Buf_Retrieve(linebuf);
     }
 }
 
@@ -607,13 +521,13 @@ LowParse_End()
     Lst_Destroy(&includes, NOFREE);	/* Should be empty now */
 }
 #endif
-	
+
 
 void
 Finish_Errors()
 {
     if (current != NULL) {
-    	free_ifile(current);
+	free_ifile(current);
 	current = NULL;
     }
     if (fatals) {