Make the mandocdb(8) format endian-neutral by storing integer data

in network byte order; from kristaps@.

Hopefully, this finishes the database format.
This commit requires another rebuild of your mandoc databases
by running "sudo mandocdb".

3 files changed, 41 insertions, 39 deletions

diff --git a/usr.bin/mandoc/apropos_db.c b/usr.bin/mandoc/apropos_db.c
index 6212ce5ab74..519da73782b 100644
--- a/usr.bin/mandoc/apropos_db.c
+++ b/usr.bin/mandoc/apropos_db.c
@@ -1,4 +1,4 @@
-/*	$Id: apropos_db.c,v 1.11 2011/11/29 22:30:56 schwarze Exp $ */
+/*	$Id: apropos_db.c,v 1.12 2011/12/03 14:53:12 schwarze Exp $ */
 /*
  * Copyright (c) 2011 Kristaps Dzonsons <kristaps@bsd.lv>
  * Copyright (c) 2011 Ingo Schwarze <schwarze@openbsd.org>
@@ -15,6 +15,7 @@
  * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
  * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
  */
+#include <sys/types.h>
 #include <assert.h>
 #include <fcntl.h>
 #include <regex.h>
@@ -114,8 +115,9 @@ static	const struct type types[] = {
 };
 
 static	DB	*btree_open(void);
-static	int	 btree_read(const DBT *,
-			const struct mchars *, char **);
+static	int	 btree_read(const DBT *, const DBT *,
+			const struct mchars *, 
+			struct db_val *, char **);
 static	int	 expreval(const struct expr *, int *);
 static	void	 exprexec(const struct expr *,
 			const char *, uint64_t, struct rec *);
@@ -144,6 +146,7 @@ btree_open(void)
 	DB		*db;
 
 	memset(&info, 0, sizeof(BTREEINFO));
+	info.lorder = 4321;
 	info.flags = R_DUP;
 
 	db = dbopen(MANDOC_DB, O_RDONLY, 0, DB_BTREE, &info);
@@ -158,17 +161,24 @@ btree_open(void)
  * Return 0 if the database is insane, else 1.
  */
 static int
-btree_read(const DBT *v, const struct mchars *mc, char **buf)
+btree_read(const DBT *k, const DBT *v, 
+		const struct mchars *mc, 
+		struct db_val *dbv, char **buf)
 {
+	const struct db_val *vp;
 
-	/* Sanity: are we nil-terminated? */
-
-	assert(v->size > 0);
+	/* Are our sizes sane? */
+	if (k->size < 2 || sizeof(struct db_val) != v->size)
+		return(0);
 
-	if ('\0' != ((char *)v->data)[(int)v->size - 1])
+	/* Is our string nil-terminated? */
+	if ('\0' != ((const char *)k->data)[(int)k->size - 1])
 		return(0);
 
-	norm_string((char *)v->data, mc, buf);
+	vp = v->data;
+	norm_string((const char *)k->data, mc, buf);
+	dbv->rec = betoh32(vp->rec);
+	dbv->mask = betoh64(vp->mask);
 	return(1);
 }
 
@@ -442,14 +452,12 @@ single_search(struct rectree *tree, const struct opts *opts,
 		struct mchars *mc, int vol)
 {
 	int		 root, leaf, ch;
-	uint64_t	 mask;
 	DBT		 key, val;
 	DB		*btree, *idx;
 	char		*buf;
-	recno_t		 rec;
 	struct rec	*rs;
 	struct rec	 r;
-	struct db_val	*vbuf;
+	struct db_val	 vb;
 
 	root	= -1;
 	leaf	= -1;
@@ -469,20 +477,14 @@ single_search(struct rectree *tree, const struct opts *opts,
 	}
 
 	while (0 == (ch = (*btree->seq)(btree, &key, &val, R_NEXT))) {
-		if (key.size < 2 || sizeof(struct db_val) != val.size)
-			break;
-		if ( ! btree_read(&key, mc, &buf))
+		if ( ! btree_read(&key, &val, mc, &vb, &buf))
 			break;
 
-		vbuf = val.data;
-		rec = vbuf->rec;
-		mask = vbuf->mask;
-
 		/*
 		 * See if this keyword record matches any of the
 		 * expressions we have stored.
 		 */
-		if ( ! exprmark(expr, buf, mask, NULL))
+		if ( ! exprmark(expr, buf, vb.mask, NULL))
 			continue;
 
 		/*
@@ -492,10 +494,10 @@ single_search(struct rectree *tree, const struct opts *opts,
 		 */
 
 		for (leaf = root; leaf >= 0; )
-			if (rec > rs[leaf].res.rec &&
+			if (vb.rec > rs[leaf].res.rec &&
 					rs[leaf].rhs >= 0)
 				leaf = rs[leaf].rhs;
-			else if (rec < rs[leaf].res.rec &&
+			else if (vb.rec < rs[leaf].res.rec &&
 					rs[leaf].lhs >= 0)
 				leaf = rs[leaf].lhs;
 			else
@@ -507,9 +509,9 @@ single_search(struct rectree *tree, const struct opts *opts,
 		 * try to evaluate it now and continue anyway.
 		 */
 
-		if (leaf >= 0 && rs[leaf].res.rec == rec) {
+		if (leaf >= 0 && rs[leaf].res.rec == vb.rec) {
 			if (0 == rs[leaf].matched)
-				exprexec(expr, buf, mask, &rs[leaf]);
+				exprexec(expr, buf, vb.mask, &rs[leaf]);
 			continue;
 		}
 
@@ -519,7 +521,7 @@ single_search(struct rectree *tree, const struct opts *opts,
 		 * database, then begin partial evaluation.
 		 */
 
-		key.data = &rec;
+		key.data = &vb.rec;
 		key.size = sizeof(recno_t);
 
 		if (0 != (*idx->get)(idx, &key, &val, 0))
@@ -543,12 +545,12 @@ single_search(struct rectree *tree, const struct opts *opts,
 		rs[tree->len].matches =
 			mandoc_calloc(terms, sizeof(int));
 
-		exprexec(expr, buf, mask, &rs[tree->len]);
+		exprexec(expr, buf, vb.mask, &rs[tree->len]);
 
 		/* Append to our tree. */
 
 		if (leaf >= 0) {
-			if (rec > rs[leaf].res.rec)
+			if (vb.rec > rs[leaf].res.rec)
 				rs[leaf].rhs = tree->len;
 			else
 				rs[leaf].lhs = tree->len;
@@ -570,6 +572,7 @@ static void
 recfree(struct rec *rec)
 {
 
+	free(rec->res.type);
 	free(rec->res.file);
 	free(rec->res.cat);
 	free(rec->res.title);
diff --git a/usr.bin/mandoc/mandocdb.8 b/usr.bin/mandoc/mandocdb.8
index 916bd881e92..7d9b30a014d 100644
--- a/usr.bin/mandoc/mandocdb.8
+++ b/usr.bin/mandoc/mandocdb.8
@@ -1,4 +1,4 @@
-.\"	$Id: mandocdb.8,v 1.10 2011/12/01 23:22:09 schwarze Exp $
+.\"	$Id: mandocdb.8,v 1.11 2011/12/03 14:53:12 schwarze Exp $
 .\"
 .\" Copyright (c) 2011 Kristaps Dzonsons <kristaps@bsd.lv>
 .\"
@@ -14,7 +14,7 @@
 .\" ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
 .\" OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 .\"
-.Dd $Mdocdate: December 1 2011 $
+.Dd $Mdocdate: December 3 2011 $
 .Dt MANDOCDB 8
 .Os
 .Sh NAME
@@ -137,11 +137,11 @@ The keyword database,
 is a
 .Xr btree 3
 database of NUL-terminated keywords (record length is non-zero string
-length plus one) mapping to a 8-byte binary field consisting of the
-keyword type and source
+length plus one) mapping to a 12-byte binary field consisting of the
+64-bit keyword type and 32-bit source
 .Sx Index Database
-record number.
-The type, a 64-bit bit-mask in host order, consists of the following
+record number, both in network-byte order.
+The type bit-mask consists of the following
 values mapping into
 .Xr mdoc 7
 macro identifiers:
diff --git a/usr.bin/mandoc/mandocdb.c b/usr.bin/mandoc/mandocdb.c
index 356df828e2e..0205feac77c 100644
--- a/usr.bin/mandoc/mandocdb.c
+++ b/usr.bin/mandoc/mandocdb.c
@@ -1,4 +1,4 @@
-/*	$Id: mandocdb.c,v 1.16 2011/12/01 23:22:09 schwarze Exp $ */
+/*	$Id: mandocdb.c,v 1.17 2011/12/03 14:53:12 schwarze Exp $ */
 /*
  * Copyright (c) 2011 Kristaps Dzonsons <kristaps@bsd.lv>
  * Copyright (c) 2011 Ingo Schwarze <schwarze@openbsd.org>
@@ -320,6 +320,7 @@ mandocdb(int argc, char *argv[])
 	argv += optind;
 
 	memset(&info, 0, sizeof(BTREEINFO));
+	info.lorder = 4321;
 	info.flags = R_DUP;
 
 	mp = mparse_alloc(MPARSE_AUTO, MANDOCLEVEL_FATAL, NULL, NULL);
@@ -617,15 +618,13 @@ index_merge(const struct of *of, struct mparse *mp,
 		 * into the database.
 		 */
 
-		vbuf.rec = rec;
+		vbuf.rec = htobe32(rec);
 		seq = R_FIRST;
 		while (0 == (ch = (*hash->seq)(hash, &key, &val, seq))) {
 			seq = R_NEXT;
-
-			vbuf.mask = *(uint64_t *)val.data;
+			vbuf.mask = htobe64(*(uint64_t *)val.data);
 			val.size = sizeof(struct db_val);
 			val.data = &vbuf;
-
 			dbt_put(db, dbf, &key, &val);
 		}
 		if (ch < 0) {
@@ -723,7 +722,7 @@ index_prune(const struct of *ofile, DB *db, const char *dbf,
 				break;
 
 			vbuf = val.data;
-			if (*maxrec != vbuf->rec)
+			if (*maxrec != betoh32(vbuf->rec))
 				continue;
 
 			if ((ch = (*db->del)(db, &key, R_CURSOR)) < 0)