- implement lazy-parsing of rcs files, that is only parse as much as we

  need. this can save us much work, particularly with very large rcs files.

first of a few important performance improvements.

ok joris@

6 files changed, 177 insertions, 74 deletions

diff --git a/usr.bin/cvs/rcs.c b/usr.bin/cvs/rcs.c
index 9a56fa3b5ea..b7a5eb25a82 100644
--- a/usr.bin/cvs/rcs.c
+++ b/usr.bin/cvs/rcs.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: rcs.c,v 1.116 2005/12/24 04:10:51 joris Exp $	*/
+/*	$OpenBSD: rcs.c,v 1.117 2005/12/27 16:05:20 niallo Exp $	*/
 /*
  * Copyright (c) 2004 Jean-Francois Brousseau <jfb@openbsd.org>
  * All rights reserved.
@@ -45,6 +45,7 @@
 
 #define RCS_BUFSIZE	16384
 #define RCS_BUFEXTSIZE	8192
+#define RCS_KWEXP_SIZE  1024
 
 
 /* RCS token types */
@@ -269,10 +270,13 @@ int rcs_errno = RCS_ERR_NOERR;
 
 
 static int	rcs_write(RCSFILE *);
-static int	rcs_parse(RCSFILE *);
+static int	rcs_parse_init(RCSFILE *);
 static int	rcs_parse_admin(RCSFILE *);
 static int	rcs_parse_delta(RCSFILE *);
+static int      rcs_parse_deltas(RCSFILE *, RCSNUM *);
 static int	rcs_parse_deltatext(RCSFILE *);
+static int      rcs_parse_deltatexts(RCSFILE *, RCSNUM *);
+static int      rcs_parse_desc(RCSFILE *, RCSNUM *);
 
 static int	rcs_parse_access(RCSFILE *);
 static int	rcs_parse_symbols(RCSFILE *);
@@ -345,7 +349,7 @@ rcs_open(const char *path, int flags, ...)
 	TAILQ_INIT(&(rfp->rf_locks));
 
 	if (rfp->rf_flags & RCS_CREATE) {
-	} else if (rcs_parse(rfp) < 0) {
+	} else if (rcs_parse_init(rfp) < 0) {
 		rcs_close(rfp);
 		return (NULL);
 	}
@@ -426,6 +430,7 @@ rcs_close(RCSFILE *rfp)
 		xfree(rfp->rf_expand);
 	if (rfp->rf_desc != NULL)
 		xfree(rfp->rf_desc);
+	rcs_freepdata(rfp->rf_pdata);
 	xfree(rfp);
 }
 
@@ -453,6 +458,9 @@ rcs_write(RCSFILE *rfp)
 
 	from_fd = to_fd = fd = -1;
 
+	/* Write operations need the whole file parsed */
+	rcs_parse_deltatexts(rfp, NULL);
+
 	if (rfp->rf_flags & RCS_SYNCED)
 		return (0);
 
@@ -660,7 +668,7 @@ rcs_head_get(RCSFILE *file)
  * <rev>, which must reference a valid revision within the file.
  */
 int
-rcs_head_set(RCSFILE *file, const RCSNUM *rev)
+rcs_head_set(RCSFILE *file, RCSNUM *rev)
 {
 	struct rcs_delta *rd;
 
@@ -1233,12 +1241,16 @@ rcs_getrev(RCSFILE *rfp, RCSNUM *frev)
 		return (NULL);
 	}
 
+	/* No matter what, we're going to need up the the description parsed */
+	rcs_parse_desc(rfp, NULL);
+
 	rdp = rcs_findrev(rfp, rfp->rf_head);
 	if (rdp == NULL) {
 		cvs_log(LP_ERR, "failed to get RCS HEAD revision");
 		return (NULL);
 	}
-
+	if (rdp->rd_tlen == 0)
+		rcs_parse_deltatexts(rfp, rfp->rf_head);
 	len = rdp->rd_tlen;
 	if (len == 0) {
 		rbuf = cvs_buf_alloc(1, 0);
@@ -1260,8 +1272,10 @@ rcs_getrev(RCSFILE *rfp, RCSNUM *frev)
 				cvs_buf_free(rbuf);
 				return (NULL);
 			}
-
 			cvs_buf_putc(rbuf, '\0');
+			/* check if we have parsed this rev's deltatext */
+			if (rdp->rd_tlen == 0)
+				rcs_parse_deltatexts(rfp, rdp->rd_num);
 
 			bp = cvs_buf_release(rbuf);
 			rbuf = cvs_patchfile((char *)bp, (char *)rdp->rd_text,
@@ -1434,13 +1448,23 @@ rcs_rev_remove(RCSFILE *rf, RCSNUM *rev)
  * Returns a pointer to the delta on success, or NULL on failure.
  */
 struct rcs_delta *
-rcs_findrev(RCSFILE *rfp, const RCSNUM *rev)
+rcs_findrev(RCSFILE *rfp, RCSNUM *rev)
 {
 	u_int cmplen;
-	struct rcs_delta *rdp;
+	struct rcs_delta *rdp, *enddelta;
 	struct rcs_dlist *hp;
 	int found;
 
+	/*
+	 * We need to do more parsing if the last revision in the linked list
+	 * is greater than the requested revision.
+	 */
+	enddelta = TAILQ_LAST(&(rfp->rf_delta), rcs_dlist);
+	if ((enddelta == NULL)
+	    || (rcsnum_cmp(enddelta->rd_num, rev, -1) == -1)) {
+		rcs_parse_deltas(rfp, rev);
+	}
+
 	cmplen = 2;
 	hp = &(rfp->rf_delta);
 
@@ -1582,18 +1606,128 @@ rcs_kflag_usage(void)
 	    "\t-kb\tGenerate binary file unmodified (merges not allowed).\n");
 }
 
+/* rcs_parse_deltas()
+ *
+ * Parse deltas. If <rev> is not NULL, parse only as far as that
+ * revision. If <rev> is NULL, parse all deltas.
+ *
+ * Returns 0 on success, -1 on failure.
+ */
+static int
+rcs_parse_deltas(RCSFILE *rfp, RCSNUM *rev)
+{
+	int ret;
+	struct rcs_delta *enddelta;
+	if ((rfp->rf_flags & PARSED_DELTAS)
+	    || (rfp->rf_flags & RCS_CREATE))
+		return (0);
+	for (;;) {
+		ret = rcs_parse_delta(rfp);
+		if (rev != NULL) {
+			enddelta = TAILQ_LAST(&(rfp->rf_delta), rcs_dlist);
+			if (rcsnum_cmp(enddelta->rd_num, rev, -1) == 0)
+				break;
+		}
+		if (ret == 0) {
+			rfp->rf_flags |= PARSED_DELTAS;
+			break;
+		}
+		else if (ret == -1)
+			fatal("error parsing deltas");
+	}
+	return (0);
+}
+
+/* rcs_parse_deltatexts()
+ *
+ * Parse deltatexts. If <rev> is not NULL, parse only as far as that
+ * revision. If <rev> is NULL, parse everything.
+ *
+ * Returns 0 on success, -1 on failure.
+ */
+static int
+rcs_parse_deltatexts(RCSFILE *rfp, RCSNUM *rev)
+{
+	int ret;
+	struct rcs_delta *rdp;
+	if ((rfp->rf_flags & PARSED_DELTATEXTS)
+	    || (rfp->rf_flags & RCS_CREATE))
+		return (0);
+	if (!(rfp->rf_flags & PARSED_DESC))
+		rcs_parse_desc(rfp, rev);
+	for (;;) {
+		if (rev != NULL) {
+			rdp = rcs_findrev(rfp, rev);
+			if (rdp->rd_text != NULL)
+				break;
+			else
+				ret = rcs_parse_deltatext(rfp);
+		} else
+			ret = rcs_parse_deltatext(rfp);
+		if (ret == 0) {
+			rfp->rf_flags |= PARSED_DELTATEXTS;
+			break;
+		}
+		else if (ret == -1) {
+			fatal("problem parsing deltatexts");
+		}
+	}
+
+	return (0);
+}
+
+/* rcs_parse_desc()
+ *
+ * Parse RCS description.
+ *
+ * Returns 0 on success, -1 on failure.
+ */
+static int
+rcs_parse_desc(RCSFILE *rfp, RCSNUM *rev)
+{
+	int ret = 0;
+	if ((rfp->rf_flags & PARSED_DESC)
+	    || (rfp->rf_flags & RCS_CREATE))
+		return (0);
+        if (!(rfp->rf_flags & PARSED_DELTAS))
+		rcs_parse_deltas(rfp, rev);
+	/* do parsing */
+	ret = rcs_gettok(rfp);
+	if (ret != RCS_TOK_DESC) {
+		rcs_errno = RCS_ERR_PARSE;
+		cvs_log(LP_ERR, "token `%s' found where RCS desc expected",
+		    RCS_TOKSTR(rfp));
+		fatal("problem parsing RCS desc");
+		return (-1);
+	}
+
+	ret = rcs_gettok(rfp);
+	if (ret != RCS_TOK_STRING) {
+		rcs_errno = RCS_ERR_PARSE;
+		cvs_log(LP_ERR, "token `%s' found where RCS desc expected",
+		    RCS_TOKSTR(rfp));
+		fatal("problem parsing RCS desc");
+	}
+
+	rfp->rf_desc = xstrdup(RCS_TOKSTR(rfp));
+	rfp->rf_flags |= PARSED_DESC;
+	return (0);
+}
+
 /*
- * rcs_parse()
+ * rcs_parse_init()
  *
- * Parse the contents of file <path>, which are in the RCS format.
+ * Initial parsing of file <path>, which are in the RCS format.
+ * Just does admin section
  * Returns 0 on success, or -1 on failure.
  */
 static int
-rcs_parse(RCSFILE *rfp)
+rcs_parse_init(RCSFILE *rfp)
 {
-	int ret;
+	int ret, count;
 	struct rcs_pdata *pdp;
 
+	count = 0;
 	if (rfp->rf_flags & RCS_PARSED)
 		return (0);
 
@@ -1616,53 +1750,13 @@ rcs_parse(RCSFILE *rfp)
 
 	if ((ret = rcs_parse_admin(rfp)) < 0) {
 		rcs_freepdata(pdp);
-		return (-1);
-	} else if (ret == RCS_TOK_NUM) {
-		for (;;) {
-			ret = rcs_parse_delta(rfp);
-			if (ret == 0)
-				break;
-			else if (ret == -1) {
-				rcs_freepdata(pdp);
-				return (-1);
-			}
-		}
-	}
-
-	ret = rcs_gettok(rfp);
-	if (ret != RCS_TOK_DESC) {
-		rcs_errno = RCS_ERR_PARSE;
-		cvs_log(LP_ERR, "token `%s' found where RCS desc expected",
-		    RCS_TOKSTR(rfp));
-		rcs_freepdata(pdp);
-		return (-1);
+		fatal("could not parse admin data");
 	}
 
-	ret = rcs_gettok(rfp);
-	if (ret != RCS_TOK_STRING) {
-		rcs_errno = RCS_ERR_PARSE;
-		cvs_log(LP_ERR, "token `%s' found where RCS desc expected",
-		    RCS_TOKSTR(rfp));
-		rcs_freepdata(pdp);
-		return (-1);
-	}
-
-	rfp->rf_desc = xstrdup(RCS_TOKSTR(rfp));
-	for (;;) {
-		ret = rcs_parse_deltatext(rfp);
-		if (ret == 0)
-			break;
-		else if (ret == -1) {
-			rcs_freepdata(pdp);
-			return (-1);
-		}
-	}
-
-	rcs_freepdata(pdp);
-
-	rfp->rf_pdata = NULL;
-	rfp->rf_flags |= RCS_PARSED | RCS_SYNCED;
+	if (rfp->rf_flags & RCS_PARSE_FULLY)
+		rcs_parse_deltatexts(rfp, NULL);
 
+	rfp->rf_flags |= RCS_SYNCED;
 	return (0);
 }
 
@@ -2670,6 +2764,9 @@ rcs_deltatext_set(RCSFILE *rfp, RCSNUM *rev, const char *dtext)
 	size_t len;
 	struct rcs_delta *rdp;
 
+	/* Write operations require full parsing */
+	rcs_parse_deltatexts(rfp, NULL);
+
 	if ((rdp = rcs_findrev(rfp, rev)) == NULL)
 		return (-1);
 
diff --git a/usr.bin/cvs/rcs.h b/usr.bin/cvs/rcs.h
index e4335181fcf..550edd7d606 100644
--- a/usr.bin/cvs/rcs.h
+++ b/usr.bin/cvs/rcs.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: rcs.h,v 1.42 2005/12/08 18:56:10 joris Exp $	*/
+/*	$OpenBSD: rcs.h,v 1.43 2005/12/27 16:05:21 niallo Exp $	*/
 /*
  * Copyright (c) 2004 Jean-Francois Brousseau <jfb@openbsd.org>
  * All rights reserved.
@@ -83,15 +83,21 @@
 
 
 /* file flags */
-#define RCS_READ	0x01
-#define RCS_WRITE	0x02
-#define RCS_RDWR	(RCS_READ|RCS_WRITE)
-#define RCS_CREATE	0x04		/* create the file */
+#define RCS_READ	  (1<<0)
+#define RCS_WRITE	  (1<<1)
+#define RCS_RDWR	  (RCS_READ|RCS_WRITE)
+#define RCS_CREATE	  (1<<2)  /* create the file */
+#define RCS_PARSE_FULLY   (1<<3)  /* fully parse it on open */
 
 /* internal flags */
-#define RCS_PARSED	0x010000	/* file has been parsed */
-#define RCS_SYNCED	0x020000	/* in-mem copy is sync with disk copy */
-#define RCS_SLOCK	0x040000	/* strict lock */
+#define RCS_PARSED	  (1<<4)  /* file has been parsed */
+#define RCS_SYNCED	  (1<<5)  /* in-mem copy is sync with disk copy */
+#define RCS_SLOCK	  (1<<6)  /* strict lock */
+
+/* parser flags */
+#define PARSED_DELTAS     (1<<7)  /* all deltas are parsed */
+#define PARSED_DESC       (1<<8)  /* the description is parsed */
+#define PARSED_DELTATEXTS (1<<9)  /* all delta texts are parsed */
 
 /* delta flags */
 #define RCS_RD_DEAD	0x01	/* dead */
@@ -193,13 +199,13 @@ extern int rcs_errno;
 RCSFILE			*rcs_open(const char *, int, ...);
 void			 rcs_close(RCSFILE *);
 const RCSNUM		*rcs_head_get(RCSFILE *);
-int			 rcs_head_set(RCSFILE *, const RCSNUM *);
+int			 rcs_head_set(RCSFILE *, RCSNUM *);
 const RCSNUM		*rcs_branch_get(RCSFILE *);
 int			 rcs_branch_set(RCSFILE *, const RCSNUM *);
 int			 rcs_access_add(RCSFILE *, const char *);
 int			 rcs_access_remove(RCSFILE *, const char *);
 int			 rcs_access_check(RCSFILE *, const char *);
-struct rcs_delta	*rcs_findrev(RCSFILE *, const RCSNUM *);
+struct rcs_delta	*rcs_findrev(RCSFILE *, RCSNUM *);
 int			 rcs_sym_add(RCSFILE *, const char *, RCSNUM *);
 int			 rcs_sym_remove(RCSFILE *, const char *);
 RCSNUM			*rcs_sym_getrev(RCSFILE *, const char *);
diff --git a/usr.bin/rcs/ci.c b/usr.bin/rcs/ci.c
index 03acfbfaef1..45316432033 100644
--- a/usr.bin/rcs/ci.c
+++ b/usr.bin/rcs/ci.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: ci.c,v 1.89 2005/12/23 00:59:55 joris Exp $	*/
+/*	$OpenBSD: ci.c,v 1.90 2005/12/27 16:05:21 niallo Exp $	*/
 /*
  * Copyright (c) 2005 Niall O'Higgins <niallo@openbsd.org>
  * All rights reserved.
@@ -107,7 +107,7 @@ checkin_main(int argc, char **argv)
 	pb.fmode = pb.flags = status = 0;
 
 	pb.flags = INTERACTIVE;
-	pb.openflags = RCS_RDWR|RCS_CREATE;
+	pb.openflags = RCS_RDWR|RCS_CREATE|RCS_PARSE_FULLY;
 
 	while ((ch = rcs_getopt(argc, argv, CI_OPTSTRING)) != -1) {
 		switch (ch) {
diff --git a/usr.bin/rcs/co.c b/usr.bin/rcs/co.c
index d2aaffaafa7..109b3f9c8d1 100644
--- a/usr.bin/rcs/co.c
+++ b/usr.bin/rcs/co.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: co.c,v 1.49 2005/12/23 00:59:56 joris Exp $	*/
+/*	$OpenBSD: co.c,v 1.50 2005/12/27 16:05:21 niallo Exp $	*/
 /*
  * Copyright (c) 2005 Joris Vink <joris@openbsd.org>
  * All rights reserved.
@@ -156,7 +156,7 @@ checkout_main(int argc, char **argv)
 			continue;
 		}
 
-		if ((file = rcs_open(fpath, RCS_RDWR)) == NULL)
+		if ((file = rcs_open(fpath, RCS_RDWR|RCS_PARSE_FULLY)) == NULL)
 			continue;
 
 		if (flags & PRESERVETIME)
diff --git a/usr.bin/rcs/rcsdiff.c b/usr.bin/rcs/rcsdiff.c
index ad19ddc1a8c..cf625668525 100644
--- a/usr.bin/rcs/rcsdiff.c
+++ b/usr.bin/rcs/rcsdiff.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: rcsdiff.c,v 1.27 2005/12/20 09:04:17 xsa Exp $	*/
+/*	$OpenBSD: rcsdiff.c,v 1.28 2005/12/27 16:05:21 niallo Exp $	*/
 /*
  * Copyright (c) 2005 Joris Vink <joris@openbsd.org>
  * All rights reserved.
@@ -121,7 +121,7 @@ rcsdiff_main(int argc, char **argv)
 		if (rcs_statfile(argv[i], fpath, sizeof(fpath)) < 0)
 			continue;
 
-		if ((file = rcs_open(fpath, RCS_READ)) == NULL)
+		if ((file = rcs_open(fpath, RCS_READ|RCS_PARSE_FULLY)) == NULL)
 			continue;
 
 		if (kflag != RCS_KWEXP_ERR)
diff --git a/usr.bin/rcs/rcsprog.c b/usr.bin/rcs/rcsprog.c
index 27bc3d81897..5e3dde89a63 100644
--- a/usr.bin/rcs/rcsprog.c
+++ b/usr.bin/rcs/rcsprog.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: rcsprog.c,v 1.57 2005/12/24 16:37:46 xsa Exp $	*/
+/*	$OpenBSD: rcsprog.c,v 1.58 2005/12/27 16:05:21 niallo Exp $	*/
 /*
  * Copyright (c) 2005 Jean-Francois Brousseau <jfb@openbsd.org>
  * All rights reserved.
@@ -383,7 +383,7 @@ rcs_main(int argc, char **argv)
 
 	kflag = lkmode = -1;
 	fmode = 0;
-	flags = RCS_RDWR;
+	flags = RCS_RDWR|RCS_PARSE_FULLY;
 	descfile = nflag = NULL;
 	logstr = alist = comment = elist = NULL;