diff options
Diffstat (limited to 'usr.bin/mandoc')
| -rw-r--r-- | usr.bin/mandoc/Makefile | 9 | ||||
| -rw-r--r-- | usr.bin/mandoc/apropos.c | 96 | ||||
| -rw-r--r-- | usr.bin/mandoc/apropos_db.c | 866 | ||||
| -rw-r--r-- | usr.bin/mandoc/apropos_db.h | 73 | ||||
| -rw-r--r-- | usr.bin/mandoc/mandocdb.c | 3170 | ||||
| -rw-r--r-- | usr.bin/mandoc/mandocdb.h | 62 | ||||
| -rw-r--r-- | usr.bin/mandoc/mansearch.c | 565 | ||||
| -rw-r--r-- | usr.bin/mandoc/mansearch.h | 87 |
8 files changed, 2328 insertions, 2600 deletions
diff --git a/usr.bin/mandoc/Makefile b/usr.bin/mandoc/Makefile index 0c0c5139022..6f5463699d9 100644 --- a/usr.bin/mandoc/Makefile +++ b/usr.bin/mandoc/Makefile @@ -1,9 +1,10 @@ -# $OpenBSD: Makefile,v 1.71 2013/10/06 23:59:59 schwarze Exp $ +# $OpenBSD: Makefile,v 1.72 2013/12/31 00:40:19 schwarze Exp $ .include <bsd.own.mk> -CFLAGS+=-DVERSION=\"1.12.2\" -CFLAGS+=-W -Wall -Wstrict-prototypes -Wno-unused-parameter +CFLAGS += -DVERSION=\"1.13.0\" +CFLAGS += -W -Wall -Wstrict-prototypes -Wno-unused-parameter +LDFLAGS += -lsqlite3 SRCS= roff.c tbl.c tbl_opts.c tbl_layout.c tbl_data.c eqn.c mandoc.c read.c SRCS+= mdoc_macro.c mdoc.c mdoc_hash.c \ @@ -14,7 +15,7 @@ SRCS+= main.c mdoc_term.c chars.c term.c tree.c man_term.c eqn_term.c SRCS+= mdoc_man.c SRCS+= html.c mdoc_html.c man_html.c out.c eqn_html.c SRCS+= term_ps.c term_ascii.c tbl_term.c tbl_html.c -SRCS+= manpath.c mandocdb.c apropos_db.c apropos.c +SRCS+= manpath.c mandocdb.c mansearch.c apropos.c PROG= mandoc diff --git a/usr.bin/mandoc/apropos.c b/usr.bin/mandoc/apropos.c index 3b58c061e53..d9cf3d1659c 100644 --- a/usr.bin/mandoc/apropos.c +++ b/usr.bin/mandoc/apropos.c @@ -1,7 +1,6 @@ -/* $Id: apropos.c,v 1.17 2013/07/12 11:01:42 schwarze Exp $ */ +/* $Id: apropos.c,v 1.18 2013/12/31 00:40:19 schwarze Exp $ */ /* - * Copyright (c) 2011, 2012 Kristaps Dzonsons <kristaps@bsd.lv> - * Copyright (c) 2011 Ingo Schwarze <schwarze@openbsd.org> + * Copyright (c) 2012 Kristaps Dzonsons <kristaps@bsd.lv> * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above @@ -19,30 +18,26 @@ #include <assert.h> #include <getopt.h> +#include <stdint.h> #include <stdio.h> #include <stdlib.h> #include <string.h> +#include <unistd.h> -#include "apropos_db.h" -#include "mandoc.h" #include "manpath.h" - -static int cmp(const void *, const void *); -static void list(struct res *, size_t, void *); - -static char *progname; +#include "mansearch.h" int apropos(int argc, char *argv[]) { - int ch, rc, whatis; - struct res *res; + int ch, whatis; + struct mansearch search; + size_t i, sz; + struct manpage *res; struct manpaths paths; - size_t terms, ressz; - struct opts opts; - struct expr *e; char *defpaths, *auxpaths; char *conf_file; + char *progname; extern char *optarg; extern int optind; @@ -55,13 +50,10 @@ apropos(int argc, char *argv[]) whatis = (0 == strncmp(progname, "whatis", 6)); memset(&paths, 0, sizeof(struct manpaths)); - memset(&opts, 0, sizeof(struct opts)); + memset(&search, 0, sizeof(struct mansearch)); - ressz = 0; - res = NULL; auxpaths = defpaths = NULL; conf_file = NULL; - e = NULL; while (-1 != (ch = getopt(argc, argv, "C:M:m:S:s:"))) switch (ch) { @@ -75,10 +67,10 @@ apropos(int argc, char *argv[]) auxpaths = optarg; break; case ('S'): - opts.arch = optarg; + search.arch = optarg; break; case ('s'): - opts.cat = optarg; + search.sec = optarg; break; default: goto usage; @@ -90,64 +82,28 @@ apropos(int argc, char *argv[]) if (0 == argc) goto usage; - rc = 0; + search.deftype = whatis ? TYPE_Nm : TYPE_Nm | TYPE_Nd; + search.flags = whatis ? MANSEARCH_WHATIS : 0; manpath_parse(&paths, conf_file, defpaths, auxpaths); + ch = mansearch(&search, &paths, argc, argv, &res, &sz); + manpath_free(&paths); - e = whatis ? termcomp(argc, argv, &terms) : - exprcomp(argc, argv, &terms); - - if (NULL == e) { - fprintf(stderr, "%s: Bad expression\n", progname); - goto out; - } - - rc = apropos_search - (paths.sz, paths.paths, &opts, - e, terms, NULL, &ressz, &res, list); + if (0 == ch) + goto usage; - if (0 == rc) { - fprintf(stderr, "%s: Bad database\n", progname); - goto out; + for (i = 0; i < sz; i++) { + printf("%s - %s\n", res[i].names, res[i].desc); + free(res[i].file); + free(res[i].names); + free(res[i].desc); } -out: - manpath_free(&paths); - resfree(res, ressz); - exprfree(e); - return(rc ? EXIT_SUCCESS : EXIT_FAILURE); - + free(res); + return(sz ? EXIT_SUCCESS : EXIT_FAILURE); usage: fprintf(stderr, "usage: %s [-C file] [-M path] [-m path] " "[-S arch] [-s section]%s ...\n", progname, whatis ? " name" : "\n expression"); return(EXIT_FAILURE); } - -/* ARGSUSED */ -static void -list(struct res *res, size_t sz, void *arg) -{ - size_t i; - - qsort(res, sz, sizeof(struct res), cmp); - - for (i = 0; i < sz; i++) { - if ( ! res[i].matched) - continue; - printf("%s(%s%s%s) - %.70s\n", - res[i].title, - res[i].cat, - *res[i].arch ? "/" : "", - *res[i].arch ? res[i].arch : "", - res[i].desc); - } -} - -static int -cmp(const void *p1, const void *p2) -{ - - return(strcasecmp(((const struct res *)p1)->title, - ((const struct res *)p2)->title)); -} diff --git a/usr.bin/mandoc/apropos_db.c b/usr.bin/mandoc/apropos_db.c deleted file mode 100644 index 6c2c0c638c3..00000000000 --- a/usr.bin/mandoc/apropos_db.c +++ /dev/null @@ -1,866 +0,0 @@ -/* $Id: apropos_db.c,v 1.19 2013/10/05 21:17:29 schwarze Exp $ */ -/* - * Copyright (c) 2011, 2012 Kristaps Dzonsons <kristaps@bsd.lv> - * Copyright (c) 2011 Ingo Schwarze <schwarze@openbsd.org> - * - * Permission to use, copy, modify, and distribute this software for any - * purpose with or without fee is hereby granted, provided that the above - * copyright notice and this permission notice appear in all copies. - * - * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - */ -#include <sys/param.h> -#include <sys/types.h> - -#include <assert.h> -#include <fcntl.h> -#include <regex.h> -#include <stdarg.h> -#include <stdint.h> -#include <stdlib.h> -#include <string.h> -#include <unistd.h> -#include <db.h> - -#include "mandocdb.h" -#include "apropos_db.h" -#include "mandoc.h" - -#define RESFREE(_x) \ - do { \ - free((_x)->file); \ - free((_x)->cat); \ - free((_x)->title); \ - free((_x)->arch); \ - free((_x)->desc); \ - free((_x)->matches); \ - } while (/*CONSTCOND*/0) - -struct expr { - int regex; /* is regex? */ - int index; /* index in match array */ - uint64_t mask; /* type-mask */ - int and; /* is rhs of logical AND? */ - char *v; /* search value */ - regex_t re; /* compiled re, if regex */ - struct expr *next; /* next in sequence */ - struct expr *subexpr; -}; - -struct type { - uint64_t mask; - const char *name; -}; - -struct rectree { - struct res *node; /* record array for dir tree */ - int len; /* length of record array */ -}; - -static const struct type types[] = { - { TYPE_An, "An" }, - { TYPE_Ar, "Ar" }, - { TYPE_At, "At" }, - { TYPE_Bsx, "Bsx" }, - { TYPE_Bx, "Bx" }, - { TYPE_Cd, "Cd" }, - { TYPE_Cm, "Cm" }, - { TYPE_Dv, "Dv" }, - { TYPE_Dx, "Dx" }, - { TYPE_Em, "Em" }, - { TYPE_Er, "Er" }, - { TYPE_Ev, "Ev" }, - { TYPE_Fa, "Fa" }, - { TYPE_Fl, "Fl" }, - { TYPE_Fn, "Fn" }, - { TYPE_Fn, "Fo" }, - { TYPE_Ft, "Ft" }, - { TYPE_Fx, "Fx" }, - { TYPE_Ic, "Ic" }, - { TYPE_In, "In" }, - { TYPE_Lb, "Lb" }, - { TYPE_Li, "Li" }, - { TYPE_Lk, "Lk" }, - { TYPE_Ms, "Ms" }, - { TYPE_Mt, "Mt" }, - { TYPE_Nd, "Nd" }, - { TYPE_Nm, "Nm" }, - { TYPE_Nx, "Nx" }, - { TYPE_Ox, "Ox" }, - { TYPE_Pa, "Pa" }, - { TYPE_Rs, "Rs" }, - { TYPE_Sh, "Sh" }, - { TYPE_Ss, "Ss" }, - { TYPE_St, "St" }, - { TYPE_Sy, "Sy" }, - { TYPE_Tn, "Tn" }, - { TYPE_Va, "Va" }, - { TYPE_Va, "Vt" }, - { TYPE_Xr, "Xr" }, - { UINT64_MAX, "any" }, - { 0, NULL } -}; - -static DB *btree_open(void); -static int btree_read(const DBT *, const DBT *, - const struct mchars *, - uint64_t *, recno_t *, char **); -static int expreval(const struct expr *, int *); -static void exprexec(const struct expr *, - const char *, uint64_t, struct res *); -static int exprmark(const struct expr *, - const char *, uint64_t, int *); -static struct expr *exprexpr(int, char *[], int *, int *, size_t *); -static struct expr *exprterm(char *, int); -static DB *index_open(void); -static int index_read(const DBT *, const DBT *, int, - const struct mchars *, struct res *); -static void norm_string(const char *, - const struct mchars *, char **); -static size_t norm_utf8(unsigned int, char[7]); -static int single_search(struct rectree *, const struct opts *, - const struct expr *, size_t terms, - struct mchars *, int); - -/* - * Open the keyword mandoc-db database. - */ -static DB * -btree_open(void) -{ - BTREEINFO info; - DB *db; - - memset(&info, 0, sizeof(BTREEINFO)); - info.lorder = 4321; - info.flags = R_DUP; - - db = dbopen(MANDOC_DB, O_RDONLY, 0, DB_BTREE, &info); - if (NULL != db) - return(db); - - return(NULL); -} - -/* - * Read a keyword from the database and normalise it. - * Return 0 if the database is insane, else 1. - */ -static int -btree_read(const DBT *k, const DBT *v, const struct mchars *mc, - uint64_t *mask, recno_t *rec, char **buf) -{ - uint64_t vbuf[2]; - - /* Are our sizes sane? */ - if (k->size < 2 || sizeof(vbuf) != v->size) - return(0); - - /* Is our string nil-terminated? */ - if ('\0' != ((const char *)k->data)[(int)k->size - 1]) - return(0); - - norm_string((const char *)k->data, mc, buf); - memcpy(vbuf, v->data, v->size); - *mask = betoh64(vbuf[0]); - *rec = betoh64(vbuf[1]); - return(1); -} - -/* - * Take a Unicode codepoint and produce its UTF-8 encoding. - * This isn't the best way to do this, but it works. - * The magic numbers are from the UTF-8 packaging. - * They're not as scary as they seem: read the UTF-8 spec for details. - */ -static size_t -norm_utf8(unsigned int cp, char out[7]) -{ - int rc; - - rc = 0; - - if (cp <= 0x0000007F) { - rc = 1; - out[0] = (char)cp; - } else if (cp <= 0x000007FF) { - rc = 2; - out[0] = (cp >> 6 & 31) | 192; - out[1] = (cp & 63) | 128; - } else if (cp <= 0x0000FFFF) { - rc = 3; - out[0] = (cp >> 12 & 15) | 224; - out[1] = (cp >> 6 & 63) | 128; - out[2] = (cp & 63) | 128; - } else if (cp <= 0x001FFFFF) { - rc = 4; - out[0] = (cp >> 18 & 7) | 240; - out[1] = (cp >> 12 & 63) | 128; - out[2] = (cp >> 6 & 63) | 128; - out[3] = (cp & 63) | 128; - } else if (cp <= 0x03FFFFFF) { - rc = 5; - out[0] = (cp >> 24 & 3) | 248; - out[1] = (cp >> 18 & 63) | 128; - out[2] = (cp >> 12 & 63) | 128; - out[3] = (cp >> 6 & 63) | 128; - out[4] = (cp & 63) | 128; - } else if (cp <= 0x7FFFFFFF) { - rc = 6; - out[0] = (cp >> 30 & 1) | 252; - out[1] = (cp >> 24 & 63) | 128; - out[2] = (cp >> 18 & 63) | 128; - out[3] = (cp >> 12 & 63) | 128; - out[4] = (cp >> 6 & 63) | 128; - out[5] = (cp & 63) | 128; - } else - return(0); - - out[rc] = '\0'; - return((size_t)rc); -} - -/* - * Normalise strings from the index and database. - * These strings are escaped as defined by mandoc_char(7) along with - * other goop in mandoc.h (e.g., soft hyphens). - * This function normalises these into a nice UTF-8 string. - * Returns 0 if the database is fucked. - */ -static void -norm_string(const char *val, const struct mchars *mc, char **buf) -{ - size_t sz, bsz; - char utfbuf[7]; - const char *seq, *cpp; - int len, u, pos; - enum mandoc_esc esc; - static const char res[] = { '\\', '\t', - ASCII_NBRSP, ASCII_HYPH, '\0' }; - - /* Pre-allocate by the length of the input */ - - bsz = strlen(val) + 1; - *buf = mandoc_realloc(*buf, bsz); - pos = 0; - - while ('\0' != *val) { - /* - * Halt on the first escape sequence. - * This also halts on the end of string, in which case - * we just copy, fallthrough, and exit the loop. - */ - if ((sz = strcspn(val, res)) > 0) { - memcpy(&(*buf)[pos], val, sz); - pos += (int)sz; - val += (int)sz; - } - - if (ASCII_HYPH == *val) { - (*buf)[pos++] = '-'; - val++; - continue; - } else if ('\t' == *val || ASCII_NBRSP == *val) { - (*buf)[pos++] = ' '; - val++; - continue; - } else if ('\\' != *val) - break; - - /* Read past the slash. */ - - val++; - u = 0; - - /* - * Parse the escape sequence and see if it's a - * predefined character or special character. - */ - - esc = mandoc_escape(&val, &seq, &len); - if (ESCAPE_ERROR == esc) - break; - - /* - * XXX - this just does UTF-8, but we need to know - * beforehand whether we should do text substitution. - */ - - switch (esc) { - case (ESCAPE_SPECIAL): - if (0 != (u = mchars_spec2cp(mc, seq, len))) - break; - /* FALLTHROUGH */ - default: - continue; - } - - /* - * If we have a Unicode codepoint, try to convert that - * to a UTF-8 byte string. - */ - - cpp = utfbuf; - if (0 == (sz = norm_utf8(u, utfbuf))) - continue; - - /* Copy the rendered glyph into the stream. */ - - sz = strlen(cpp); - bsz += sz; - - *buf = mandoc_realloc(*buf, bsz); - - memcpy(&(*buf)[pos], cpp, sz); - pos += (int)sz; - } - - (*buf)[pos] = '\0'; -} - -/* - * Open the filename-index mandoc-db database. - * Returns NULL if opening failed. - */ -static DB * -index_open(void) -{ - DB *db; - - db = dbopen(MANDOC_IDX, O_RDONLY, 0, DB_RECNO, NULL); - if (NULL != db) - return(db); - - return(NULL); -} - -/* - * Safely unpack from an index file record into the structure. - * Returns 1 if an entry was unpacked, 0 if the database is insane. - */ -static int -index_read(const DBT *key, const DBT *val, int index, - const struct mchars *mc, struct res *rec) -{ - size_t left; - char *np, *cp; - char type; - -#define INDEX_BREAD(_dst) \ - do { \ - if (NULL == (np = memchr(cp, '\0', left))) \ - return(0); \ - norm_string(cp, mc, &(_dst)); \ - left -= (np - cp) + 1; \ - cp = np + 1; \ - } while (/* CONSTCOND */ 0) - - if (0 == (left = val->size)) - return(0); - - cp = val->data; - assert(sizeof(recno_t) == key->size); - memcpy(&rec->rec, key->data, key->size); - rec->volume = index; - - if ('d' == (type = *cp++)) - rec->type = RESTYPE_MDOC; - else if ('a' == type) - rec->type = RESTYPE_MAN; - else if ('c' == type) - rec->type = RESTYPE_CAT; - else - return(0); - - left--; - INDEX_BREAD(rec->file); - INDEX_BREAD(rec->cat); - INDEX_BREAD(rec->title); - INDEX_BREAD(rec->arch); - INDEX_BREAD(rec->desc); - return(1); -} - -/* - * Search mandocdb databases in paths for expression "expr". - * Filter out by "opts". - * Call "res" with the results, which may be zero. - * Return 0 if there was a database error, else return 1. - */ -int -apropos_search(int pathsz, char **paths, const struct opts *opts, - const struct expr *expr, size_t terms, void *arg, - size_t *sz, struct res **resp, - void (*res)(struct res *, size_t, void *)) -{ - struct rectree tree; - struct mchars *mc; - int i; - - memset(&tree, 0, sizeof(struct rectree)); - - mc = mchars_alloc(); - *sz = 0; - *resp = NULL; - - /* - * Main loop. Change into the directory containing manpage - * databases. Run our expession over each database in the set. - */ - - for (i = 0; i < pathsz; i++) { - assert('/' == paths[i][0]); - if (chdir(paths[i])) - continue; - if (single_search(&tree, opts, expr, terms, mc, i)) - continue; - - resfree(tree.node, tree.len); - mchars_free(mc); - return(0); - } - - (*res)(tree.node, tree.len, arg); - *sz = tree.len; - *resp = tree.node; - mchars_free(mc); - return(1); -} - -static int -single_search(struct rectree *tree, const struct opts *opts, - const struct expr *expr, size_t terms, - struct mchars *mc, int vol) -{ - int root, leaf, ch; - DBT key, val; - DB *btree, *idx; - char *buf; - struct res *rs; - struct res r; - uint64_t mask; - recno_t rec; - - root = -1; - leaf = -1; - btree = NULL; - idx = NULL; - buf = NULL; - rs = tree->node; - - memset(&r, 0, sizeof(struct res)); - - if (NULL == (btree = btree_open())) - return(1); - - if (NULL == (idx = index_open())) { - (*btree->close)(btree); - return(1); - } - - while (0 == (ch = (*btree->seq)(btree, &key, &val, R_NEXT))) { - if ( ! btree_read(&key, &val, mc, &mask, &rec, &buf)) - break; - - /* - * See if this keyword record matches any of the - * expressions we have stored. - */ - if ( ! exprmark(expr, buf, mask, NULL)) - continue; - - /* - * O(log n) scan for prior records. Since a record - * number is unbounded, this has decent performance over - * a complex hash function. - */ - - for (leaf = root; leaf >= 0; ) - if (rec > rs[leaf].rec && - rs[leaf].rhs >= 0) - leaf = rs[leaf].rhs; - else if (rec < rs[leaf].rec && - rs[leaf].lhs >= 0) - leaf = rs[leaf].lhs; - else - break; - - /* - * If we find a record, see if it has already evaluated - * to true. If it has, great, just keep going. If not, - * try to evaluate it now and continue anyway. - */ - - if (leaf >= 0 && rs[leaf].rec == rec) { - if (0 == rs[leaf].matched) - exprexec(expr, buf, mask, &rs[leaf]); - continue; - } - - /* - * We have a new file to examine. - * Extract the manpage's metadata from the index - * database, then begin partial evaluation. - */ - - key.data = &rec; - key.size = sizeof(recno_t); - - if (0 != (*idx->get)(idx, &key, &val, 0)) - break; - - r.lhs = r.rhs = -1; - if ( ! index_read(&key, &val, vol, mc, &r)) - break; - - /* XXX: this should be elsewhere, I guess? */ - - if (opts->cat && strcasecmp(opts->cat, r.cat)) - continue; - - if (opts->arch && *r.arch) - if (strcasecmp(opts->arch, r.arch)) - continue; - - tree->node = rs = mandoc_realloc - (rs, (tree->len + 1) * sizeof(struct res)); - - memcpy(&rs[tree->len], &r, sizeof(struct res)); - memset(&r, 0, sizeof(struct res)); - rs[tree->len].matches = - mandoc_calloc(terms, sizeof(int)); - - exprexec(expr, buf, mask, &rs[tree->len]); - - /* Append to our tree. */ - - if (leaf >= 0) { - if (rec > rs[leaf].rec) - rs[leaf].rhs = tree->len; - else - rs[leaf].lhs = tree->len; - } else - root = tree->len; - - tree->len++; - } - - (*btree->close)(btree); - (*idx->close)(idx); - - free(buf); - RESFREE(&r); - return(1 == ch); -} - -void -resfree(struct res *rec, size_t sz) -{ - size_t i; - - for (i = 0; i < sz; i++) - RESFREE(&rec[i]); - free(rec); -} - -/* - * Compile a list of straight-up terms. - * The arguments are re-written into ~[[:<:]]term[[:>:]], or "term" - * surrounded by word boundaries, then pumped through exprterm(). - * Terms are case-insensitive. - * This emulates whatis(1) behaviour. - */ -struct expr * -termcomp(int argc, char *argv[], size_t *tt) -{ - char *buf; - int pos; - struct expr *e, *next; - size_t sz; - - buf = NULL; - e = NULL; - *tt = 0; - - for (pos = argc - 1; pos >= 0; pos--) { - sz = strlen(argv[pos]) + 18; - buf = mandoc_realloc(buf, sz); - strlcpy(buf, "Nm~[[:<:]]", sz); - strlcat(buf, argv[pos], sz); - strlcat(buf, "[[:>:]]", sz); - if (NULL == (next = exprterm(buf, 0))) { - free(buf); - exprfree(e); - return(NULL); - } - next->next = e; - e = next; - (*tt)++; - } - - free(buf); - return(e); -} - -/* - * Compile a sequence of logical expressions. - * See apropos.1 for a grammar of this sequence. - */ -struct expr * -exprcomp(int argc, char *argv[], size_t *tt) -{ - int pos, lvl; - struct expr *e; - - pos = lvl = 0; - *tt = 0; - - e = exprexpr(argc, argv, &pos, &lvl, tt); - - if (0 == lvl && pos >= argc) - return(e); - - exprfree(e); - return(NULL); -} - -/* - * Compile an array of tokens into an expression. - * An informal expression grammar is defined in apropos(1). - * Return NULL if we fail doing so. All memory will be cleaned up. - * Return the root of the expression sequence if alright. - */ -static struct expr * -exprexpr(int argc, char *argv[], int *pos, int *lvl, size_t *tt) -{ - struct expr *e, *first, *next; - int log; - - first = next = NULL; - - for ( ; *pos < argc; (*pos)++) { - e = next; - - /* - * Close out a subexpression. - */ - - if (NULL != e && 0 == strcmp(")", argv[*pos])) { - if (--(*lvl) < 0) - goto err; - break; - } - - /* - * Small note: if we're just starting, don't let "-a" - * and "-o" be considered logical operators: they're - * just tokens unless pairwise joining, in which case we - * record their existence (or assume "OR"). - */ - log = 0; - - if (NULL != e && 0 == strcmp("-a", argv[*pos])) - log = 1; - else if (NULL != e && 0 == strcmp("-o", argv[*pos])) - log = 2; - - if (log > 0 && ++(*pos) >= argc) - goto err; - - /* - * Now we parse the term part. This can begin with - * "-i", in which case the expression is case - * insensitive. - */ - - if (0 == strcmp("(", argv[*pos])) { - ++(*pos); - ++(*lvl); - next = mandoc_calloc(1, sizeof(struct expr)); - next->subexpr = exprexpr(argc, argv, pos, lvl, tt); - if (NULL == next->subexpr) { - free(next); - next = NULL; - } - } else if (0 == strcmp("-i", argv[*pos])) { - if (++(*pos) >= argc) - goto err; - next = exprterm(argv[*pos], 0); - } else - next = exprterm(argv[*pos], 1); - - if (NULL == next) - goto err; - - next->and = log == 1; - next->index = (int)(*tt)++; - - /* Append to our chain of expressions. */ - - if (NULL == first) { - assert(NULL == e); - first = next; - } else { - assert(NULL != e); - e->next = next; - } - } - - return(first); -err: - exprfree(first); - return(NULL); -} - -/* - * Parse a terminal expression with the grammar as defined in - * apropos(1). - * Return NULL if we fail the parse. - */ -static struct expr * -exprterm(char *buf, int cs) -{ - struct expr e; - struct expr *p; - char *key; - int i; - - memset(&e, 0, sizeof(struct expr)); - - /* Choose regex or substring match. */ - - if (NULL == (e.v = strpbrk(buf, "=~"))) { - e.regex = 0; - e.v = buf; - } else { - e.regex = '~' == *e.v; - *e.v++ = '\0'; - } - - /* Determine the record types to search for. */ - - e.mask = 0; - if (buf < e.v) { - while (NULL != (key = strsep(&buf, ","))) { - i = 0; - while (types[i].mask && - strcmp(types[i].name, key)) - i++; - e.mask |= types[i].mask; - } - } - if (0 == e.mask) - e.mask = TYPE_Nm | TYPE_Nd; - - if (e.regex) { - i = REG_EXTENDED | REG_NOSUB | (cs ? 0 : REG_ICASE); - if (regcomp(&e.re, e.v, i)) - return(NULL); - } - - e.v = mandoc_strdup(e.v); - - p = mandoc_calloc(1, sizeof(struct expr)); - memcpy(p, &e, sizeof(struct expr)); - return(p); -} - -void -exprfree(struct expr *p) -{ - struct expr *pp; - - while (NULL != p) { - if (p->subexpr) - exprfree(p->subexpr); - if (p->regex) - regfree(&p->re); - free(p->v); - pp = p->next; - free(p); - p = pp; - } -} - -static int -exprmark(const struct expr *p, const char *cp, - uint64_t mask, int *ms) -{ - - for ( ; p; p = p->next) { - if (p->subexpr) { - if (exprmark(p->subexpr, cp, mask, ms)) - return(1); - continue; - } else if ( ! (mask & p->mask)) - continue; - - if (p->regex) { - if (regexec(&p->re, cp, 0, NULL, 0)) - continue; - } else if (NULL == strcasestr(cp, p->v)) - continue; - - if (NULL == ms) - return(1); - else - ms[p->index] = 1; - } - - return(0); -} - -static int -expreval(const struct expr *p, int *ms) -{ - int match; - - /* - * AND has precedence over OR. Analysis is left-right, though - * it doesn't matter because there are no side-effects. - * Thus, step through pairwise ANDs and accumulate their Boolean - * evaluation. If we encounter a single true AND collection or - * standalone term, the whole expression is true (by definition - * of OR). - */ - - for (match = 0; p && ! match; p = p->next) { - /* Evaluate a subexpression, if applicable. */ - if (p->subexpr && ! ms[p->index]) - ms[p->index] = expreval(p->subexpr, ms); - - match = ms[p->index]; - for ( ; p->next && p->next->and; p = p->next) { - /* Evaluate a subexpression, if applicable. */ - if (p->next->subexpr && ! ms[p->next->index]) - ms[p->next->index] = - expreval(p->next->subexpr, ms); - match = match && ms[p->next->index]; - } - } - - return(match); -} - -/* - * First, update the array of terms for which this expression evaluates - * to true. - * Second, logically evaluate all terms over the updated array of truth - * values. - * If this evaluates to true, mark the expression as satisfied. - */ -static void -exprexec(const struct expr *e, const char *cp, - uint64_t mask, struct res *r) -{ - - assert(0 == r->matched); - exprmark(e, cp, mask, r->matches); - r->matched = expreval(e, r->matches); -} diff --git a/usr.bin/mandoc/apropos_db.h b/usr.bin/mandoc/apropos_db.h deleted file mode 100644 index 78cc1559318..00000000000 --- a/usr.bin/mandoc/apropos_db.h +++ /dev/null @@ -1,73 +0,0 @@ -/* $Id: apropos_db.h,v 1.11 2012/04/15 11:54:47 schwarze Exp $ */ -/* - * Copyright (c) 2011, 2012 Kristaps Dzonsons <kristaps@bsd.lv> - * - * Permission to use, copy, modify, and distribute this software for any - * purpose with or without fee is hereby granted, provided that the above - * copyright notice and this permission notice appear in all copies. - * - * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - */ -#ifndef APROPOS_H -#define APROPOS_H - -enum restype { - RESTYPE_MAN, /* man(7) file */ - RESTYPE_MDOC, /* mdoc(7) file */ - RESTYPE_CAT /* pre-formatted file */ -}; - -struct res { - enum restype type; /* input file type */ - char *file; /* file in file-system */ - char *cat; /* category (3p, 3, etc.) */ - char *title; /* title (FOO, etc.) */ - char *arch; /* arch (or empty string) */ - char *desc; /* description (from Nd) */ - unsigned int rec; /* record in index */ - /* - * The index volume. This indexes into the array of directories - * searched for manual page databases. - */ - unsigned int volume; - /* - * The following fields are used internally. - * - * Maintain a binary tree for checking the uniqueness of `rec' - * when adding elements to the results array. - * Since the results array is dynamic, use offset in the array - * instead of a pointer to the structure. - */ - int lhs; - int rhs; - int matched; /* expression is true */ - int *matches; /* partial truth evaluations */ -}; - -struct opts { - const char *arch; /* restrict to architecture */ - const char *cat; /* restrict to manual section */ -}; - -__BEGIN_DECLS - -struct expr; - -int apropos_search(int, char **, const struct opts *, - const struct expr *, size_t, - void *, size_t *, struct res **, - void (*)(struct res *, size_t, void *)); -struct expr *exprcomp(int, char *[], size_t *); -void exprfree(struct expr *); -void resfree(struct res *, size_t); -struct expr *termcomp(int, char *[], size_t *); - -__END_DECLS - -#endif /*!APROPOS_H*/ diff --git a/usr.bin/mandoc/mandocdb.c b/usr.bin/mandoc/mandocdb.c index 754c09af0d6..84847d97d0a 100644 --- a/usr.bin/mandoc/mandocdb.c +++ b/usr.bin/mandoc/mandocdb.c @@ -1,7 +1,7 @@ -/* $Id: mandocdb.c,v 1.46 2013/11/21 01:47:10 schwarze Exp $ */ +/* $Id: mandocdb.c,v 1.47 2013/12/31 00:40:19 schwarze Exp $ */ /* - * Copyright (c) 2011 Kristaps Dzonsons <kristaps@bsd.lv> - * Copyright (c) 2011 Ingo Schwarze <schwarze@openbsd.org> + * Copyright (c) 2011, 2012 Kristaps Dzonsons <kristaps@bsd.lv> + * Copyright (c) 2011, 2012, 2013 Ingo Schwarze <schwarze@openbsd.org> * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above @@ -15,283 +15,314 @@ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */ -#include <sys/types.h> +#include <sys/stat.h> #include <assert.h> #include <ctype.h> -#include <dirent.h> #include <errno.h> #include <fcntl.h> +#include <fts.h> #include <getopt.h> #include <limits.h> +#include <stddef.h> #include <stdio.h> #include <stdint.h> #include <stdlib.h> #include <string.h> #include <unistd.h> -#include <db.h> -#include "man.h" +#include <ohash.h> +#include <sqlite3.h> + #include "mdoc.h" +#include "man.h" #include "mandoc.h" -#include "mandocdb.h" #include "manpath.h" +#include "mansearch.h" + +#define SQL_EXEC(_v) \ + if (SQLITE_OK != sqlite3_exec(db, (_v), NULL, NULL, NULL)) \ + fprintf(stderr, "%s\n", sqlite3_errmsg(db)) +#define SQL_BIND_TEXT(_s, _i, _v) \ + if (SQLITE_OK != sqlite3_bind_text \ + ((_s), (_i)++, (_v), -1, SQLITE_STATIC)) \ + fprintf(stderr, "%s\n", sqlite3_errmsg(db)) +#define SQL_BIND_INT(_s, _i, _v) \ + if (SQLITE_OK != sqlite3_bind_int \ + ((_s), (_i)++, (_v))) \ + fprintf(stderr, "%s\n", sqlite3_errmsg(db)) +#define SQL_BIND_INT64(_s, _i, _v) \ + if (SQLITE_OK != sqlite3_bind_int64 \ + ((_s), (_i)++, (_v))) \ + fprintf(stderr, "%s\n", sqlite3_errmsg(db)) +#define SQL_STEP(_s) \ + if (SQLITE_DONE != sqlite3_step((_s))) \ + fprintf(stderr, "%s\n", sqlite3_errmsg(db)) -#define MANDOC_BUFSZ BUFSIZ -#define MANDOC_SLOP 1024 - -#define MANDOC_SRC 0x1 -#define MANDOC_FORM 0x2 - -/* Access to the mandoc database on disk. */ - -struct mdb { - char idxn[PATH_MAX]; /* index db filename */ - char dbn[PATH_MAX]; /* keyword db filename */ - DB *idx; /* index recno database */ - DB *db; /* keyword btree database */ +enum op { + OP_DEFAULT = 0, /* new dbs from dir list or default config */ + OP_CONFFILE, /* new databases from custom config file */ + OP_UPDATE, /* delete/add entries in existing database */ + OP_DELETE, /* delete entries from existing database */ + OP_TEST /* change no databases, report potential problems */ }; -/* Stack of temporarily unused index records. */ - -struct recs { - recno_t *stack; /* pointer to a malloc'ed array */ - size_t size; /* number of allocated slots */ - size_t cur; /* current number of empty records */ - recno_t last; /* last record number in the index */ +enum form { + FORM_NONE, /* format is unknown */ + FORM_SRC, /* format is -man or -mdoc */ + FORM_CAT /* format is cat */ }; -/* Tiny list for files. No need to bring in QUEUE. */ +struct str { + char *utf8; /* key in UTF-8 form */ + const struct mpage *mpage; /* if set, the owning parse */ + uint64_t mask; /* bitmask in sequence */ + char key[]; /* the string itself */ +}; -struct of { - char *fname; /* heap-allocated */ - char *sec; - char *arch; - char *title; - int src_form; - struct of *next; /* NULL for last one */ - struct of *first; /* first in list */ +struct inodev { + ino_t st_ino; + dev_t st_dev; }; -/* Buffer for storing growable data. */ +struct mpage { + struct inodev inodev; /* used for hashing routine */ + enum form form; /* format from file content */ + char *sec; /* section from file content */ + char *arch; /* architecture from file content */ + char *title; /* title from file content */ + char *desc; /* description from file content */ + struct mlink *mlinks; /* singly linked list */ +}; -struct buf { - char *cp; - size_t len; /* current length */ - size_t size; /* total buffer size */ +struct mlink { + char file[PATH_MAX]; /* filename rel. to manpath */ + enum form dform; /* format from directory */ + enum form fform; /* format from file name suffix */ + char *dsec; /* section from directory */ + char *arch; /* architecture from directory */ + char *name; /* name from file name (not empty) */ + char *fsec; /* section from file name suffix */ + struct mlink *next; /* singly linked list */ }; -/* Operation we're going to perform. */ +struct title { + char *title; /* name(sec/arch) given inside the file */ + char *file; /* file name in case of mismatch */ +}; -enum op { - OP_DEFAULT = 0, /* new dbs from dir list or default config */ - OP_CONFFILE, /* new databases from custom config file */ - OP_UPDATE, /* delete/add entries in existing database */ - OP_DELETE, /* delete entries from existing database */ - OP_TEST /* change no databases, report potential problems */ +enum stmt { + STMT_DELETE_PAGE = 0, /* delete mpage */ + STMT_INSERT_PAGE, /* insert mpage */ + STMT_INSERT_LINK, /* insert mlink */ + STMT_INSERT_KEY, /* insert parsed key */ + STMT__MAX }; -#define MAN_ARGS DB *hash, \ - struct buf *buf, \ - struct buf *dbuf, \ - const struct man_node *n -#define MDOC_ARGS DB *hash, \ - struct buf *buf, \ - struct buf *dbuf, \ - const struct mdoc_node *n, \ - const struct mdoc_meta *m - -static void buf_appendmdoc(struct buf *, - const struct mdoc_node *, int); -static void buf_append(struct buf *, const char *); -static void buf_appendb(struct buf *, - const void *, size_t); -static void dbt_put(DB *, const char *, DBT *, DBT *); -static void hash_put(DB *, const struct buf *, uint64_t); -static void hash_reset(DB **); -static void index_merge(const struct of *, struct mparse *, - struct buf *, struct buf *, DB *, - struct mdb *, struct recs *); -static void index_prune(const struct of *, struct mdb *, - struct recs *); -static void ofile_argbuild(int, char *[], struct of **, - const char *); -static void ofile_dirbuild(const char *, const char *, - const char *, int, struct of **); -static void ofile_free(struct of *); -static void pformatted(DB *, struct buf *, - struct buf *, const struct of *); -static int pman_node(MAN_ARGS); -static void pmdoc_node(MDOC_ARGS); -static int pmdoc_head(MDOC_ARGS); -static int pmdoc_body(MDOC_ARGS); -static int pmdoc_Fd(MDOC_ARGS); -static int pmdoc_In(MDOC_ARGS); -static int pmdoc_Fn(MDOC_ARGS); -static int pmdoc_Nd(MDOC_ARGS); -static int pmdoc_Nm(MDOC_ARGS); -static int pmdoc_Sh(MDOC_ARGS); -static int pmdoc_St(MDOC_ARGS); -static int pmdoc_Xr(MDOC_ARGS); - -#define MDOCF_CHILD 0x01 /* Automatically index child nodes. */ +typedef int (*mdoc_fp)(struct mpage *, const struct mdoc_node *); struct mdoc_handler { - int (*fp)(MDOC_ARGS); /* Optional handler. */ - uint64_t mask; /* Set unless handler returns 0. */ - int flags; /* For use by pmdoc_node. */ + mdoc_fp fp; /* optional handler */ + uint64_t mask; /* set unless handler returns 0 */ }; +static void dbclose(int); +static void dbindex(const struct mpage *, struct mchars *); +static int dbopen(int); +static void dbprune(void); +static void filescan(const char *); +static void *hash_alloc(size_t, void *); +static void hash_free(void *, size_t, void *); +static void *hash_halloc(size_t, void *); +static void mlink_add(struct mlink *, const struct stat *); +static void mlink_free(struct mlink *); +static void mlinks_undupe(struct mpage *); +static void mpages_free(void); +static void mpages_merge(struct mchars *, struct mparse *, int); +static void parse_cat(struct mpage *); +static void parse_man(struct mpage *, const struct man_node *); +static void parse_mdoc(struct mpage *, const struct mdoc_node *); +static int parse_mdoc_body(struct mpage *, const struct mdoc_node *); +static int parse_mdoc_head(struct mpage *, const struct mdoc_node *); +static int parse_mdoc_Fd(struct mpage *, const struct mdoc_node *); +static int parse_mdoc_Fn(struct mpage *, const struct mdoc_node *); +static int parse_mdoc_In(struct mpage *, const struct mdoc_node *); +static int parse_mdoc_Nd(struct mpage *, const struct mdoc_node *); +static int parse_mdoc_Nm(struct mpage *, const struct mdoc_node *); +static int parse_mdoc_Sh(struct mpage *, const struct mdoc_node *); +static int parse_mdoc_St(struct mpage *, const struct mdoc_node *); +static int parse_mdoc_Xr(struct mpage *, const struct mdoc_node *); +static void putkey(const struct mpage *, + const char *, uint64_t); +static void putkeys(const struct mpage *, + const char *, size_t, uint64_t); +static void putmdockey(const struct mpage *, + const struct mdoc_node *, uint64_t); +static void say(const char *, const char *, ...); +static int set_basedir(const char *); +static int treescan(void); +static size_t utf8(unsigned int, char [7]); +static void utf8key(struct mchars *, struct str *); + +static char *progname; +static int use_all; /* use all found files */ +static int nodb; /* no database changes */ +static int verb; /* print what we're doing */ +static int warnings; /* warn about crap */ +static int exitcode; /* to be returned by main */ +static enum op op; /* operational mode */ +static char basedir[PATH_MAX]; /* current base directory */ +static struct ohash mpages; /* table of distinct manual pages */ +static struct ohash mlinks; /* table of directory entries */ +static struct ohash strings; /* table of all strings */ +static sqlite3 *db = NULL; /* current database */ +static sqlite3_stmt *stmts[STMT__MAX]; /* current statements */ + static const struct mdoc_handler mdocs[MDOC_MAX] = { - { NULL, 0, 0 }, /* Ap */ - { NULL, 0, 0 }, /* Dd */ - { NULL, 0, 0 }, /* Dt */ - { NULL, 0, 0 }, /* Os */ - { pmdoc_Sh, TYPE_Sh, MDOCF_CHILD }, /* Sh */ - { pmdoc_head, TYPE_Ss, MDOCF_CHILD }, /* Ss */ - { NULL, 0, 0 }, /* Pp */ - { NULL, 0, 0 }, /* D1 */ - { NULL, 0, 0 }, /* Dl */ - { NULL, 0, 0 }, /* Bd */ - { NULL, 0, 0 }, /* Ed */ - { NULL, 0, 0 }, /* Bl */ - { NULL, 0, 0 }, /* El */ - { NULL, 0, 0 }, /* It */ - { NULL, 0, 0 }, /* Ad */ - { NULL, TYPE_An, MDOCF_CHILD }, /* An */ - { NULL, TYPE_Ar, MDOCF_CHILD }, /* Ar */ - { NULL, TYPE_Cd, MDOCF_CHILD }, /* Cd */ - { NULL, TYPE_Cm, MDOCF_CHILD }, /* Cm */ - { NULL, TYPE_Dv, MDOCF_CHILD }, /* Dv */ - { NULL, TYPE_Er, MDOCF_CHILD }, /* Er */ - { NULL, TYPE_Ev, MDOCF_CHILD }, /* Ev */ - { NULL, 0, 0 }, /* Ex */ - { NULL, TYPE_Fa, MDOCF_CHILD }, /* Fa */ - { pmdoc_Fd, TYPE_In, 0 }, /* Fd */ - { NULL, TYPE_Fl, MDOCF_CHILD }, /* Fl */ - { pmdoc_Fn, 0, 0 }, /* Fn */ - { NULL, TYPE_Ft, MDOCF_CHILD }, /* Ft */ - { NULL, TYPE_Ic, MDOCF_CHILD }, /* Ic */ - { pmdoc_In, TYPE_In, 0 }, /* In */ - { NULL, TYPE_Li, MDOCF_CHILD }, /* Li */ - { pmdoc_Nd, TYPE_Nd, MDOCF_CHILD }, /* Nd */ - { pmdoc_Nm, TYPE_Nm, MDOCF_CHILD }, /* Nm */ - { NULL, 0, 0 }, /* Op */ - { NULL, 0, 0 }, /* Ot */ - { NULL, TYPE_Pa, MDOCF_CHILD }, /* Pa */ - { NULL, 0, 0 }, /* Rv */ - { pmdoc_St, TYPE_St, 0 }, /* St */ - { NULL, TYPE_Va, MDOCF_CHILD }, /* Va */ - { pmdoc_body, TYPE_Va, MDOCF_CHILD }, /* Vt */ - { pmdoc_Xr, TYPE_Xr, 0 }, /* Xr */ - { NULL, 0, 0 }, /* %A */ - { NULL, 0, 0 }, /* %B */ - { NULL, 0, 0 }, /* %D */ - { NULL, 0, 0 }, /* %I */ - { NULL, 0, 0 }, /* %J */ - { NULL, 0, 0 }, /* %N */ - { NULL, 0, 0 }, /* %O */ - { NULL, 0, 0 }, /* %P */ - { NULL, 0, 0 }, /* %R */ - { NULL, 0, 0 }, /* %T */ - { NULL, 0, 0 }, /* %V */ - { NULL, 0, 0 }, /* Ac */ - { NULL, 0, 0 }, /* Ao */ - { NULL, 0, 0 }, /* Aq */ - { NULL, TYPE_At, MDOCF_CHILD }, /* At */ - { NULL, 0, 0 }, /* Bc */ - { NULL, 0, 0 }, /* Bf */ - { NULL, 0, 0 }, /* Bo */ - { NULL, 0, 0 }, /* Bq */ - { NULL, TYPE_Bsx, MDOCF_CHILD }, /* Bsx */ - { NULL, TYPE_Bx, MDOCF_CHILD }, /* Bx */ - { NULL, 0, 0 }, /* Db */ - { NULL, 0, 0 }, /* Dc */ - { NULL, 0, 0 }, /* Do */ - { NULL, 0, 0 }, /* Dq */ - { NULL, 0, 0 }, /* Ec */ - { NULL, 0, 0 }, /* Ef */ - { NULL, TYPE_Em, MDOCF_CHILD }, /* Em */ - { NULL, 0, 0 }, /* Eo */ - { NULL, TYPE_Fx, MDOCF_CHILD }, /* Fx */ - { NULL, TYPE_Ms, MDOCF_CHILD }, /* Ms */ - { NULL, 0, 0 }, /* No */ - { NULL, 0, 0 }, /* Ns */ - { NULL, TYPE_Nx, MDOCF_CHILD }, /* Nx */ - { NULL, TYPE_Ox, MDOCF_CHILD }, /* Ox */ - { NULL, 0, 0 }, /* Pc */ - { NULL, 0, 0 }, /* Pf */ - { NULL, 0, 0 }, /* Po */ - { NULL, 0, 0 }, /* Pq */ - { NULL, 0, 0 }, /* Qc */ - { NULL, 0, 0 }, /* Ql */ - { NULL, 0, 0 }, /* Qo */ - { NULL, 0, 0 }, /* Qq */ - { NULL, 0, 0 }, /* Re */ - { NULL, 0, 0 }, /* Rs */ - { NULL, 0, 0 }, /* Sc */ - { NULL, 0, 0 }, /* So */ - { NULL, 0, 0 }, /* Sq */ - { NULL, 0, 0 }, /* Sm */ - { NULL, 0, 0 }, /* Sx */ - { NULL, TYPE_Sy, MDOCF_CHILD }, /* Sy */ - { NULL, TYPE_Tn, MDOCF_CHILD }, /* Tn */ - { NULL, 0, 0 }, /* Ux */ - { NULL, 0, 0 }, /* Xc */ - { NULL, 0, 0 }, /* Xo */ - { pmdoc_head, TYPE_Fn, 0 }, /* Fo */ - { NULL, 0, 0 }, /* Fc */ - { NULL, 0, 0 }, /* Oo */ - { NULL, 0, 0 }, /* Oc */ - { NULL, 0, 0 }, /* Bk */ - { NULL, 0, 0 }, /* Ek */ - { NULL, 0, 0 }, /* Bt */ - { NULL, 0, 0 }, /* Hf */ - { NULL, 0, 0 }, /* Fr */ - { NULL, 0, 0 }, /* Ud */ - { NULL, TYPE_Lb, MDOCF_CHILD }, /* Lb */ - { NULL, 0, 0 }, /* Lp */ - { NULL, TYPE_Lk, MDOCF_CHILD }, /* Lk */ - { NULL, TYPE_Mt, MDOCF_CHILD }, /* Mt */ - { NULL, 0, 0 }, /* Brq */ - { NULL, 0, 0 }, /* Bro */ - { NULL, 0, 0 }, /* Brc */ - { NULL, 0, 0 }, /* %C */ - { NULL, 0, 0 }, /* Es */ - { NULL, 0, 0 }, /* En */ - { NULL, TYPE_Dx, MDOCF_CHILD }, /* Dx */ - { NULL, 0, 0 }, /* %Q */ - { NULL, 0, 0 }, /* br */ - { NULL, 0, 0 }, /* sp */ - { NULL, 0, 0 }, /* %U */ - { NULL, 0, 0 }, /* Ta */ + { NULL, 0 }, /* Ap */ + { NULL, 0 }, /* Dd */ + { NULL, 0 }, /* Dt */ + { NULL, 0 }, /* Os */ + { parse_mdoc_Sh, TYPE_Sh }, /* Sh */ + { parse_mdoc_head, TYPE_Ss }, /* Ss */ + { NULL, 0 }, /* Pp */ + { NULL, 0 }, /* D1 */ + { NULL, 0 }, /* Dl */ + { NULL, 0 }, /* Bd */ + { NULL, 0 }, /* Ed */ + { NULL, 0 }, /* Bl */ + { NULL, 0 }, /* El */ + { NULL, 0 }, /* It */ + { NULL, 0 }, /* Ad */ + { NULL, TYPE_An }, /* An */ + { NULL, TYPE_Ar }, /* Ar */ + { NULL, TYPE_Cd }, /* Cd */ + { NULL, TYPE_Cm }, /* Cm */ + { NULL, TYPE_Dv }, /* Dv */ + { NULL, TYPE_Er }, /* Er */ + { NULL, TYPE_Ev }, /* Ev */ + { NULL, 0 }, /* Ex */ + { NULL, TYPE_Fa }, /* Fa */ + { parse_mdoc_Fd, 0 }, /* Fd */ + { NULL, TYPE_Fl }, /* Fl */ + { parse_mdoc_Fn, 0 }, /* Fn */ + { NULL, TYPE_Ft }, /* Ft */ + { NULL, TYPE_Ic }, /* Ic */ + { parse_mdoc_In, TYPE_In }, /* In */ + { NULL, TYPE_Li }, /* Li */ + { parse_mdoc_Nd, TYPE_Nd }, /* Nd */ + { parse_mdoc_Nm, TYPE_Nm }, /* Nm */ + { NULL, 0 }, /* Op */ + { NULL, 0 }, /* Ot */ + { NULL, TYPE_Pa }, /* Pa */ + { NULL, 0 }, /* Rv */ + { parse_mdoc_St, 0 }, /* St */ + { NULL, TYPE_Va }, /* Va */ + { parse_mdoc_body, TYPE_Va }, /* Vt */ + { parse_mdoc_Xr, 0 }, /* Xr */ + { NULL, 0 }, /* %A */ + { NULL, 0 }, /* %B */ + { NULL, 0 }, /* %D */ + { NULL, 0 }, /* %I */ + { NULL, 0 }, /* %J */ + { NULL, 0 }, /* %N */ + { NULL, 0 }, /* %O */ + { NULL, 0 }, /* %P */ + { NULL, 0 }, /* %R */ + { NULL, 0 }, /* %T */ + { NULL, 0 }, /* %V */ + { NULL, 0 }, /* Ac */ + { NULL, 0 }, /* Ao */ + { NULL, 0 }, /* Aq */ + { NULL, TYPE_At }, /* At */ + { NULL, 0 }, /* Bc */ + { NULL, 0 }, /* Bf */ + { NULL, 0 }, /* Bo */ + { NULL, 0 }, /* Bq */ + { NULL, TYPE_Bsx }, /* Bsx */ + { NULL, TYPE_Bx }, /* Bx */ + { NULL, 0 }, /* Db */ + { NULL, 0 }, /* Dc */ + { NULL, 0 }, /* Do */ + { NULL, 0 }, /* Dq */ + { NULL, 0 }, /* Ec */ + { NULL, 0 }, /* Ef */ + { NULL, TYPE_Em }, /* Em */ + { NULL, 0 }, /* Eo */ + { NULL, TYPE_Fx }, /* Fx */ + { NULL, TYPE_Ms }, /* Ms */ + { NULL, 0 }, /* No */ + { NULL, 0 }, /* Ns */ + { NULL, TYPE_Nx }, /* Nx */ + { NULL, TYPE_Ox }, /* Ox */ + { NULL, 0 }, /* Pc */ + { NULL, 0 }, /* Pf */ + { NULL, 0 }, /* Po */ + { NULL, 0 }, /* Pq */ + { NULL, 0 }, /* Qc */ + { NULL, 0 }, /* Ql */ + { NULL, 0 }, /* Qo */ + { NULL, 0 }, /* Qq */ + { NULL, 0 }, /* Re */ + { NULL, 0 }, /* Rs */ + { NULL, 0 }, /* Sc */ + { NULL, 0 }, /* So */ + { NULL, 0 }, /* Sq */ + { NULL, 0 }, /* Sm */ + { NULL, 0 }, /* Sx */ + { NULL, TYPE_Sy }, /* Sy */ + { NULL, TYPE_Tn }, /* Tn */ + { NULL, 0 }, /* Ux */ + { NULL, 0 }, /* Xc */ + { NULL, 0 }, /* Xo */ + { parse_mdoc_head, 0 }, /* Fo */ + { NULL, 0 }, /* Fc */ + { NULL, 0 }, /* Oo */ + { NULL, 0 }, /* Oc */ + { NULL, 0 }, /* Bk */ + { NULL, 0 }, /* Ek */ + { NULL, 0 }, /* Bt */ + { NULL, 0 }, /* Hf */ + { NULL, 0 }, /* Fr */ + { NULL, 0 }, /* Ud */ + { NULL, TYPE_Lb }, /* Lb */ + { NULL, 0 }, /* Lp */ + { NULL, TYPE_Lk }, /* Lk */ + { NULL, TYPE_Mt }, /* Mt */ + { NULL, 0 }, /* Brq */ + { NULL, 0 }, /* Bro */ + { NULL, 0 }, /* Brc */ + { NULL, 0 }, /* %C */ + { NULL, 0 }, /* Es */ + { NULL, 0 }, /* En */ + { NULL, TYPE_Dx }, /* Dx */ + { NULL, 0 }, /* %Q */ + { NULL, 0 }, /* br */ + { NULL, 0 }, /* sp */ + { NULL, 0 }, /* %U */ + { NULL, 0 }, /* Ta */ }; -static const char *progname; -static int use_all; /* Use all directories and files. */ -static int verb; /* Output verbosity level. */ -static int warnings; /* Potential problems in manuals. */ - int mandocdb(int argc, char *argv[]) { - struct mparse *mp; /* parse sequence */ - struct manpaths dirs; - struct mdb mdb; - struct recs recs; - enum op op; /* current operation */ - const char *dir; - char *cp; - char pbuf[PATH_MAX]; - int ch, i, flags; - DB *hash; /* temporary keyword hashtable */ - BTREEINFO info; /* btree configuration */ - size_t sz1, sz2, ipath; - struct buf buf, /* keyword buffer */ - dbuf; /* description buffer */ - struct of *of; /* list of files for processing */ - extern int optind; - extern char *optarg; + int ch, i; + size_t j, sz; + const char *path_arg; + struct mchars *mc; + struct manpaths dirs; + struct mparse *mp; + struct ohash_info mpages_info, mlinks_info; + + memset(stmts, 0, STMT__MAX * sizeof(sqlite3_stmt *)); + memset(&dirs, 0, sizeof(struct manpaths)); + + mpages_info.alloc = mlinks_info.alloc = hash_alloc; + mpages_info.halloc = mlinks_info.halloc = hash_halloc; + mpages_info.hfree = mlinks_info.hfree = hash_free; + + mpages_info.key_offset = offsetof(struct mpage, inodev); + mlinks_info.key_offset = offsetof(struct mlink, file); progname = strrchr(argv[0], '/'); if (progname == NULL) @@ -299,57 +330,47 @@ mandocdb(int argc, char *argv[]) else ++progname; - memset(&dirs, 0, sizeof(struct manpaths)); - memset(&mdb, 0, sizeof(struct mdb)); - memset(&recs, 0, sizeof(struct recs)); + /* + * We accept a few different invocations. + * The CHECKOP macro makes sure that invocation styles don't + * clobber each other. + */ +#define CHECKOP(_op, _ch) do \ + if (OP_DEFAULT != (_op)) { \ + fprintf(stderr, "-%c: Conflicting option\n", (_ch)); \ + goto usage; \ + } while (/*CONSTCOND*/0) - of = NULL; - mp = NULL; - hash = NULL; + path_arg = NULL; op = OP_DEFAULT; - dir = NULL; - while (-1 != (ch = getopt(argc, argv, "aC:d:tu:vW"))) + while (-1 != (ch = getopt(argc, argv, "aC:d:ntu:vW"))) switch (ch) { case ('a'): use_all = 1; break; case ('C'): - if (op) { - fprintf(stderr, - "-C: conflicting options\n"); - goto usage; - } - dir = optarg; + CHECKOP(op, ch); + path_arg = optarg; op = OP_CONFFILE; break; case ('d'): - if (op) { - fprintf(stderr, - "-d: conflicting options\n"); - goto usage; - } - dir = optarg; + CHECKOP(op, ch); + path_arg = optarg; op = OP_UPDATE; break; + case ('n'): + nodb = 1; + break; case ('t'): + CHECKOP(op, ch); dup2(STDOUT_FILENO, STDERR_FILENO); - if (op) { - fprintf(stderr, - "-t: conflicting options\n"); - goto usage; - } op = OP_TEST; - use_all = 1; - warnings = 1; + nodb = warnings = 1; break; case ('u'): - if (op) { - fprintf(stderr, - "-u: conflicting options\n"); - goto usage; - } - dir = optarg; + CHECKOP(op, ch); + path_arg = optarg; op = OP_DELETE; break; case ('v'): @@ -366,282 +387,584 @@ mandocdb(int argc, char *argv[]) argv += optind; if (OP_CONFFILE == op && argc > 0) { - fprintf(stderr, "-C: too many arguments\n"); + fprintf(stderr, "-C: Too many arguments\n"); goto usage; } - memset(&info, 0, sizeof(BTREEINFO)); - info.lorder = 4321; - info.flags = R_DUP; - - mp = mparse_alloc(MPARSE_AUTO, MANDOCLEVEL_FATAL, NULL, NULL, NULL); - - memset(&buf, 0, sizeof(struct buf)); - memset(&dbuf, 0, sizeof(struct buf)); + exitcode = (int)MANDOCLEVEL_OK; + mp = mparse_alloc(MPARSE_AUTO, + MANDOCLEVEL_FATAL, NULL, NULL, NULL); + mc = mchars_alloc(); - buf.size = dbuf.size = MANDOC_BUFSZ; + ohash_init(&mpages, 6, &mpages_info); + ohash_init(&mlinks, 6, &mlinks_info); - buf.cp = mandoc_malloc(buf.size); - dbuf.cp = mandoc_malloc(dbuf.size); + if (OP_UPDATE == op || OP_DELETE == op || OP_TEST == op) { + /* + * Force processing all files. + */ + use_all = 1; - if (OP_TEST == op) { - ofile_argbuild(argc, argv, &of, NULL); - if (NULL == of) + /* + * All of these deal with a specific directory. + * Jump into that directory then collect files specified + * on the command-line. + */ + if (0 == set_basedir(path_arg)) goto out; - index_merge(of, mp, &dbuf, &buf, hash, &mdb, &recs); - goto out; - } + for (i = 0; i < argc; i++) + filescan(argv[i]); + if (0 == dbopen(1)) + goto out; + if (OP_TEST != op) + dbprune(); + if (OP_DELETE != op) + mpages_merge(mc, mp, 0); + dbclose(1); + } else { + /* + * If we have arguments, use them as our manpaths. + * If we don't, grok from manpath(1) or however else + * manpath_parse() wants to do it. + */ + if (argc > 0) { + dirs.paths = mandoc_calloc + (argc, sizeof(char *)); + dirs.sz = (size_t)argc; + for (i = 0; i < argc; i++) + dirs.paths[i] = mandoc_strdup(argv[i]); + } else + manpath_parse(&dirs, path_arg, NULL, NULL); - if (OP_UPDATE == op || OP_DELETE == op) { - if (NULL == realpath(dir, pbuf)) { - perror(dir); - exit((int)MANDOCLEVEL_BADARG); - } - if (strlcat(pbuf, "/", PATH_MAX) >= PATH_MAX) { - fprintf(stderr, "%s: path too long\n", pbuf); - exit((int)MANDOCLEVEL_BADARG); - } + /* + * First scan the tree rooted at a base directory, then + * build a new database and finally move it into place. + * Ignore zero-length directories and strip trailing + * slashes. + */ + for (j = 0; j < dirs.sz; j++) { + sz = strlen(dirs.paths[j]); + if (sz && '/' == dirs.paths[j][sz - 1]) + dirs.paths[j][--sz] = '\0'; + if (0 == sz) + continue; - strlcat(mdb.dbn, pbuf, PATH_MAX); - sz1 = strlcat(mdb.dbn, MANDOC_DB, PATH_MAX); + if (j) { + ohash_init(&mpages, 6, &mpages_info); + ohash_init(&mlinks, 6, &mlinks_info); + } - strlcat(mdb.idxn, pbuf, PATH_MAX); - sz2 = strlcat(mdb.idxn, MANDOC_IDX, PATH_MAX); + if (0 == set_basedir(dirs.paths[j])) + goto out; + if (0 == treescan()) + goto out; + if (0 == set_basedir(dirs.paths[j])) + goto out; + if (0 == dbopen(0)) + goto out; - if (sz1 >= PATH_MAX || sz2 >= PATH_MAX) { - fprintf(stderr, "%s: path too long\n", mdb.idxn); - exit((int)MANDOCLEVEL_BADARG); + mpages_merge(mc, mp, warnings && !use_all); + dbclose(0); + + if (j + 1 < dirs.sz) { + mpages_free(); + ohash_delete(&mpages); + ohash_delete(&mlinks); + } } + } +out: + set_basedir(NULL); + manpath_free(&dirs); + mchars_free(mc); + mparse_free(mp); + mpages_free(); + ohash_delete(&mpages); + ohash_delete(&mlinks); + return(exitcode); +usage: + fprintf(stderr, "usage: %s [-anvW] [-C file]\n" + " %s [-anvW] dir ...\n" + " %s [-nvW] -d dir [file ...]\n" + " %s [-nvW] -u dir [file ...]\n" + " %s -t file ...\n", + progname, progname, progname, + progname, progname); - flags = O_CREAT | O_RDWR; - mdb.db = dbopen(mdb.dbn, flags, 0644, DB_BTREE, &info); - mdb.idx = dbopen(mdb.idxn, flags, 0644, DB_RECNO, NULL); + return((int)MANDOCLEVEL_BADARG); +} - if (NULL == mdb.db) { - perror(mdb.dbn); - exit((int)MANDOCLEVEL_SYSERR); - } else if (NULL == mdb.idx) { - perror(mdb.idxn); - exit((int)MANDOCLEVEL_SYSERR); - } +/* + * Scan a directory tree rooted at "basedir" for manpages. + * We use fts(), scanning directory parts along the way for clues to our + * section and architecture. + * + * If use_all has been specified, grok all files. + * If not, sanitise paths to the following: + * + * [./]man*[/<arch>]/<name>.<section> + * or + * [./]cat<section>[/<arch>]/<name>.0 + * + * TODO: accomodate for multi-language directories. + */ +static int +treescan(void) +{ + FTS *f; + FTSENT *ff; + struct mlink *mlink; + int dform; + char *fsec; + const char *dsec, *arch, *cp, *path; + const char *argv[2]; - ofile_argbuild(argc, argv, &of, pbuf); + argv[0] = "."; + argv[1] = (char *)NULL; - if (NULL == of) - goto out; + /* + * Walk through all components under the directory, using the + * logical descent of files. + */ + f = fts_open((char * const *)argv, FTS_LOGICAL, NULL); + if (NULL == f) { + exitcode = (int)MANDOCLEVEL_SYSERR; + say("", NULL); + return(0); + } - index_prune(of, &mdb, &recs); + dsec = arch = NULL; + dform = FORM_NONE; + while (NULL != (ff = fts_read(f))) { + path = ff->fts_path + 2; /* - * Go to the root of the respective manual tree. - * This must work or no manuals may be found (they're - * indexed relative to the root). + * If we're a regular file, add an mlink by using the + * stored directory data and handling the filename. */ + if (FTS_F == ff->fts_info) { + if (0 == strcmp(path, MANDOC_DB)) + continue; + if ( ! use_all && ff->fts_level < 2) { + if (warnings) + say(path, "Extraneous file"); + continue; + } else if (NULL == (fsec = + strrchr(ff->fts_name, '.'))) { + if ( ! use_all) { + if (warnings) + say(path, + "No filename suffix"); + continue; + } + } else if (0 == strcmp(++fsec, "html")) { + if (warnings) + say(path, "Skip html"); + continue; + } else if (0 == strcmp(fsec, "gz")) { + if (warnings) + say(path, "Skip gz"); + continue; + } else if (0 == strcmp(fsec, "ps")) { + if (warnings) + say(path, "Skip ps"); + continue; + } else if (0 == strcmp(fsec, "pdf")) { + if (warnings) + say(path, "Skip pdf"); + continue; + } else if ( ! use_all && + ((FORM_SRC == dform && strcmp(fsec, dsec)) || + (FORM_CAT == dform && strcmp(fsec, "0")))) { + if (warnings) + say(path, "Wrong filename suffix"); + continue; + } else + fsec[-1] = '\0'; + mlink = mandoc_calloc(1, sizeof(struct mlink)); + strlcpy(mlink->file, path, sizeof(mlink->file)); + mlink->dform = dform; + if (NULL != dsec) + mlink->dsec = mandoc_strdup(dsec); + if (NULL != arch) + mlink->arch = mandoc_strdup(arch); + mlink->name = mandoc_strdup(ff->fts_name); + if (NULL != fsec) + mlink->fsec = mandoc_strdup(fsec); + mlink_add(mlink, ff->fts_statp); + continue; + } else if (FTS_D != ff->fts_info && + FTS_DP != ff->fts_info) { + if (warnings) + say(path, "Not a regular file"); + continue; + } - if (OP_UPDATE == op) { - if (-1 == chdir(dir)) { - perror(dir); - exit((int)MANDOCLEVEL_SYSERR); + switch (ff->fts_level) { + case (0): + /* Ignore the root directory. */ + break; + case (1): + /* + * This might contain manX/ or catX/. + * Try to infer this from the name. + * If we're not in use_all, enforce it. + */ + dsec = NULL; + dform = FORM_NONE; + cp = ff->fts_name; + if (FTS_DP == ff->fts_info) + break; + + if (0 == strncmp(cp, "man", 3)) { + dform = FORM_SRC; + dsec = cp + 3; + } else if (0 == strncmp(cp, "cat", 3)) { + dform = FORM_CAT; + dsec = cp + 3; } - index_merge(of, mp, &dbuf, &buf, hash, - &mdb, &recs); + + if (NULL != dsec || use_all) + break; + + if (warnings) + say(path, "Unknown directory part"); + fts_set(f, ff, FTS_SKIP); + break; + case (2): + /* + * Possibly our architecture. + * If we're descending, keep tabs on it. + */ + arch = NULL; + if (FTS_DP != ff->fts_info && NULL != dsec) + arch = ff->fts_name; + break; + default: + if (FTS_DP == ff->fts_info || use_all) + break; + if (warnings) + say(path, "Extraneous directory part"); + fts_set(f, ff, FTS_SKIP); + break; } + } - goto out; + fts_close(f); + return(1); +} + +/* + * Add a file to the mlinks table. + * Do not verify that it's a "valid" looking manpage (we'll do that + * later). + * + * Try to infer the manual section, architecture, and page name from the + * path, assuming it looks like + * + * [./]man*[/<arch>]/<name>.<section> + * or + * [./]cat<section>[/<arch>]/<name>.0 + * + * See treescan() for the fts(3) version of this. + */ +static void +filescan(const char *file) +{ + char buf[PATH_MAX]; + struct stat st; + struct mlink *mlink; + char *p, *start; + + assert(use_all); + + if (0 == strncmp(file, "./", 2)) + file += 2; + + if (NULL == realpath(file, buf)) { + exitcode = (int)MANDOCLEVEL_BADARG; + say(file, NULL); + return; + } else if (OP_TEST != op && strstr(buf, basedir) != buf) { + exitcode = (int)MANDOCLEVEL_BADARG; + say("", "%s: outside base directory", buf); + return; + } else if (-1 == stat(buf, &st)) { + exitcode = (int)MANDOCLEVEL_BADARG; + say(file, NULL); + return; + } else if ( ! (S_IFREG & st.st_mode)) { + exitcode = (int)MANDOCLEVEL_BADARG; + say(file, "Not a regular file"); + return; } + start = buf + strlen(basedir); + mlink = mandoc_calloc(1, sizeof(struct mlink)); + strlcpy(mlink->file, start, sizeof(mlink->file)); /* - * Configure the directories we're going to scan. - * If we have command-line arguments, use them. - * If not, we use man(1)'s method (see mandocdb.8). + * First try to guess our directory structure. + * If we find a separator, try to look for man* or cat*. + * If we find one of these and what's underneath is a directory, + * assume it's an architecture. */ + if (NULL != (p = strchr(start, '/'))) { + *p++ = '\0'; + if (0 == strncmp(start, "man", 3)) { + mlink->dform = FORM_SRC; + mlink->dsec = mandoc_strdup(start + 3); + } else if (0 == strncmp(start, "cat", 3)) { + mlink->dform = FORM_CAT; + mlink->dsec = mandoc_strdup(start + 3); + } - if (argc > 0) { - dirs.paths = mandoc_calloc(argc, sizeof(char *)); - dirs.sz = argc; - for (i = 0; i < argc; i++) { - if (NULL == (cp = realpath(argv[i], pbuf))) { - perror(argv[i]); - goto out; - } - dirs.paths[i] = mandoc_strdup(cp); + start = p; + if (NULL != mlink->dsec && NULL != (p = strchr(start, '/'))) { + *p++ = '\0'; + mlink->arch = mandoc_strdup(start); + start = p; } - } else - manpath_parse(&dirs, dir, NULL, NULL); + } - for (ipath = 0; ipath < dirs.sz; ipath++) { + /* + * Now check the file suffix. + * Suffix of `.0' indicates a catpage, `.1-9' is a manpage. + */ + p = strrchr(start, '\0'); + while (p-- > start && '/' != *p && '.' != *p) + /* Loop. */ ; - /* - * Go to the root of the respective manual tree. - * This must work or no manuals may be found: - * They are indexed relative to the root. - */ + if ('.' == *p) { + *p++ = '\0'; + mlink->fsec = mandoc_strdup(p); + } - if (-1 == chdir(dirs.paths[ipath])) { - perror(dirs.paths[ipath]); - exit((int)MANDOCLEVEL_SYSERR); - } + /* + * Now try to parse the name. + * Use the filename portion of the path. + */ + mlink->name = start; + if (NULL != (p = strrchr(start, '/'))) { + mlink->name = p + 1; + *p = '\0'; + } + mlink->name = mandoc_strdup(mlink->name); - /* Create a new database in two temporary files. */ + mlink_add(mlink, &st); +} - flags = O_CREAT | O_EXCL | O_RDWR; - while (NULL == mdb.db) { - strlcpy(mdb.dbn, MANDOC_DB, PATH_MAX); - strlcat(mdb.dbn, ".XXXXXXXXXX", PATH_MAX); - if (NULL == mktemp(mdb.dbn)) { - perror(mdb.dbn); - exit((int)MANDOCLEVEL_SYSERR); - } - mdb.db = dbopen(mdb.dbn, flags, 0644, - DB_BTREE, &info); - if (NULL == mdb.db && EEXIST != errno) { - perror(mdb.dbn); - exit((int)MANDOCLEVEL_SYSERR); - } - } - while (NULL == mdb.idx) { - strlcpy(mdb.idxn, MANDOC_IDX, PATH_MAX); - strlcat(mdb.idxn, ".XXXXXXXXXX", PATH_MAX); - if (NULL == mktemp(mdb.idxn)) { - perror(mdb.idxn); - unlink(mdb.dbn); - exit((int)MANDOCLEVEL_SYSERR); - } - mdb.idx = dbopen(mdb.idxn, flags, 0644, - DB_RECNO, NULL); - if (NULL == mdb.idx && EEXIST != errno) { - perror(mdb.idxn); - unlink(mdb.dbn); - exit((int)MANDOCLEVEL_SYSERR); - } - } +static void +mlink_add(struct mlink *mlink, const struct stat *st) +{ + struct inodev inodev; + struct mpage *mpage; + unsigned int slot; + + assert(NULL != mlink->file); + + if (NULL == mlink->dsec) + mlink->dsec = mandoc_strdup(""); + if (NULL == mlink->arch) + mlink->arch = mandoc_strdup(""); + if (NULL == mlink->name) + mlink->name = mandoc_strdup(""); + if (NULL == mlink->fsec) + mlink->fsec = mandoc_strdup(""); + + if ('0' == *mlink->fsec) { + free(mlink->fsec); + mlink->fsec = mandoc_strdup(mlink->dsec); + mlink->fform = FORM_CAT; + } else if ('1' <= *mlink->fsec && '9' >= *mlink->fsec) + mlink->fform = FORM_SRC; + else + mlink->fform = FORM_NONE; + + slot = ohash_qlookup(&mlinks, mlink->file); + assert(NULL == ohash_find(&mlinks, slot)); + ohash_insert(&mlinks, slot, mlink); + + inodev.st_ino = st->st_ino; + inodev.st_dev = st->st_dev; + slot = ohash_lookup_memory(&mpages, (char *)&inodev, + sizeof(struct inodev), inodev.st_ino); + mpage = ohash_find(&mpages, slot); + if (NULL == mpage) { + mpage = mandoc_calloc(1, sizeof(struct mpage)); + mpage->inodev.st_ino = inodev.st_ino; + mpage->inodev.st_dev = inodev.st_dev; + ohash_insert(&mpages, slot, mpage); + } else + mlink->next = mpage->mlinks; + mpage->mlinks = mlink; +} - /* - * Search for manuals and fill the new database. - */ +static void +mlink_free(struct mlink *mlink) +{ - ofile_dirbuild(".", "", "", 0, &of); + free(mlink->dsec); + free(mlink->arch); + free(mlink->name); + free(mlink->fsec); + free(mlink); +} - if (NULL != of) { - index_merge(of, mp, &dbuf, &buf, hash, - &mdb, &recs); - ofile_free(of); - of = NULL; +static void +mpages_free(void) +{ + struct mpage *mpage; + struct mlink *mlink; + unsigned int slot; + + mpage = ohash_first(&mpages, &slot); + while (NULL != mpage) { + while (NULL != (mlink = mpage->mlinks)) { + mpage->mlinks = mlink->next; + mlink_free(mlink); } + free(mpage->sec); + free(mpage->arch); + free(mpage->title); + free(mpage->desc); + free(mpage); + mpage = ohash_next(&mpages, &slot); + } +} - (*mdb.db->close)(mdb.db); - (*mdb.idx->close)(mdb.idx); - mdb.db = NULL; - mdb.idx = NULL; - - /* - * Replace the old database with the new one. - * This is not perfectly atomic, - * but i cannot think of a better way. - */ - - if (-1 == rename(mdb.dbn, MANDOC_DB)) { - perror(MANDOC_DB); - unlink(mdb.dbn); - unlink(mdb.idxn); - exit((int)MANDOCLEVEL_SYSERR); +/* + * For each mlink to the mpage, check whether the path looks like + * it is formatted, and if it does, check whether a source manual + * exists by the same name, ignoring the suffix. + * If both conditions hold, drop the mlink. + */ +static void +mlinks_undupe(struct mpage *mpage) +{ + char buf[PATH_MAX]; + struct mlink **prev; + struct mlink *mlink; + char *bufp; + + mpage->form = FORM_CAT; + prev = &mpage->mlinks; + while (NULL != (mlink = *prev)) { + if (FORM_CAT != mlink->dform) { + mpage->form = FORM_NONE; + goto nextlink; } - if (-1 == rename(mdb.idxn, MANDOC_IDX)) { - perror(MANDOC_IDX); - unlink(MANDOC_DB); - unlink(MANDOC_IDX); - unlink(mdb.idxn); - exit((int)MANDOCLEVEL_SYSERR); + if (strlcpy(buf, mlink->file, PATH_MAX) >= PATH_MAX) { + if (warnings) + say(mlink->file, "Filename too long"); + goto nextlink; } + bufp = strstr(buf, "cat"); + assert(NULL != bufp); + memcpy(bufp, "man", 3); + if (NULL != (bufp = strrchr(buf, '.'))) + *++bufp = '\0'; + strlcat(buf, mlink->dsec, PATH_MAX); + if (NULL == ohash_find(&mlinks, + ohash_qlookup(&mlinks, buf))) + goto nextlink; + if (warnings) + say(mlink->file, "Man source exists: %s", buf); + if (use_all) + goto nextlink; + *prev = mlink->next; + mlink_free(mlink); + continue; +nextlink: + prev = &(*prev)->next; } - -out: - if (mdb.db) - (*mdb.db->close)(mdb.db); - if (mdb.idx) - (*mdb.idx->close)(mdb.idx); - if (hash) - (*hash->close)(hash); - if (mp) - mparse_free(mp); - - manpath_free(&dirs); - ofile_free(of); - free(buf.cp); - free(dbuf.cp); - free(recs.stack); - - return(MANDOCLEVEL_OK); - -usage: - fprintf(stderr, - "usage: %s [-avvv] [-C file] | dir ... | -t file ...\n" - " -d dir [file ...] | " - "-u dir [file ...]\n", - progname); - - return((int)MANDOCLEVEL_BADARG); } -void -index_merge(const struct of *of, struct mparse *mp, - struct buf *dbuf, struct buf *buf, DB *hash, - struct mdb *mdb, struct recs *recs) +/* + * Run through the files in the global vector "mpages" + * and add them to the database specified in "basedir". + * + * This handles the parsing scheme itself, using the cues of directory + * and filename to determine whether the file is parsable or not. + */ +static void +mpages_merge(struct mchars *mc, struct mparse *mp, int check_reachable) { - recno_t rec; - int ch, skip; - DBT key, val; - DB *files; /* temporary file name table */ - struct mdoc *mdoc; - struct man *man; - const char *fn, *msec, *march, *mtitle; - char *p; - uint64_t mask; - size_t sv; - unsigned seq; - uint64_t vbuf[2]; - char type; - - static char emptystring[] = ""; - - if (warnings) { - files = NULL; - hash_reset(&files); + struct ohash title_table; + struct ohash_info title_info, str_info; + struct mpage *mpage; + struct mdoc *mdoc; + struct man *man; + struct title *title_entry; + char *title_str; + const char *cp; + int match; + unsigned int pslot, tslot; + enum mandoclevel lvl; + + str_info.alloc = hash_alloc; + str_info.halloc = hash_halloc; + str_info.hfree = hash_free; + str_info.key_offset = offsetof(struct str, key); + + if (check_reachable) { + title_info.alloc = hash_alloc; + title_info.halloc = hash_halloc; + title_info.hfree = hash_free; + title_info.key_offset = offsetof(struct title, title); + ohash_init(&title_table, 6, &title_info); } - rec = 0; - for (of = of->first; of; of = of->next) { - fn = of->fname; + mpage = ohash_first(&mpages, &pslot); + while (NULL != mpage) { + mlinks_undupe(mpage); + if (NULL == mpage->mlinks) { + mpage = ohash_next(&mpages, &pslot); + continue; + } + + ohash_init(&strings, 6, &str_info); + mparse_reset(mp); + mdoc = NULL; + man = NULL; + match = 1; /* * Try interpreting the file as mdoc(7) or man(7) * source code, unless it is already known to be * formatted. Fall back to formatted mode. */ - - mparse_reset(mp); - mdoc = NULL; - man = NULL; - - if ((MANDOC_SRC & of->src_form || - ! (MANDOC_FORM & of->src_form)) && - MANDOCLEVEL_FATAL > mparse_readfd(mp, -1, fn)) - mparse_result(mp, &mdoc, &man); + if (FORM_CAT != mpage->mlinks->dform || + FORM_CAT != mpage->mlinks->fform) { + lvl = mparse_readfd(mp, -1, mpage->mlinks->file); + if (lvl < MANDOCLEVEL_FATAL) + mparse_result(mp, &mdoc, &man); + } if (NULL != mdoc) { - msec = mdoc_meta(mdoc)->msec; - march = mdoc_meta(mdoc)->arch; - if (NULL == march) - march = ""; - mtitle = mdoc_meta(mdoc)->title; + mpage->form = FORM_SRC; + mpage->sec = + mandoc_strdup(mdoc_meta(mdoc)->msec); + mpage->arch = mdoc_meta(mdoc)->arch; + mpage->arch = mandoc_strdup( + NULL == mpage->arch ? "" : mpage->arch); + mpage->title = + mandoc_strdup(mdoc_meta(mdoc)->title); } else if (NULL != man) { - msec = man_meta(man)->msec; - march = ""; - mtitle = man_meta(man)->title; + mpage->form = FORM_SRC; + mpage->sec = + mandoc_strdup(man_meta(man)->msec); + mpage->arch = + mandoc_strdup(mpage->mlinks->arch); + mpage->title = + mandoc_strdup(man_meta(man)->title); } else { - msec = of->sec; - march = of->arch; - mtitle = of->title; + mpage->form = FORM_CAT; + mpage->sec = + mandoc_strdup(mpage->mlinks->dsec); + mpage->arch = + mandoc_strdup(mpage->mlinks->arch); + mpage->title = + mandoc_strdup(mpage->mlinks->name); } /* @@ -653,16 +976,13 @@ index_merge(const struct of *of, struct mparse *mp, * section, like encrypt(1) = makekey(8). Do not skip * manuals for such reasons. */ - - skip = 0; - assert(of->sec); - assert(msec); - if (warnings) - if (strcasecmp(msec, of->sec)) - fprintf(stderr, "%s: " - "section \"%s\" manual " - "in \"%s\" directory\n", - fn, msec, of->sec); + if (warnings && !use_all && FORM_SRC == mpage->form && + strcasecmp(mpage->sec, mpage->mlinks->dsec)) { + match = 0; + say(mpage->mlinks->file, "Section \"%s\" " + "manual in %s directory", + mpage->sec, mpage->mlinks->dsec); + } /* * Manual page directories exist for each kernel @@ -678,26 +998,30 @@ index_merge(const struct of *of, struct mparse *mp, * Thus, warn about architecture mismatches, * but don't skip manuals for this reason. */ + if (warnings && !use_all && + strcasecmp(mpage->arch, mpage->mlinks->arch)) { + match = 0; + say(mpage->mlinks->file, "Architecture \"%s\" " + "manual in \"%s\" directory", + mpage->arch, mpage->mlinks->arch); + } + if (warnings && !use_all && + strcasecmp(mpage->title, mpage->mlinks->name)) + match = 0; - assert(of->arch); - assert(march); - if (warnings) - if (strcasecmp(march, of->arch)) - fprintf(stderr, "%s: " - "architecture \"%s\" manual " - "in \"%s\" directory\n", - fn, march, of->arch); - - /* - * By default, skip a file if the title given - * in the file disagrees with the file name. - * Do not warn, this happens for all MLINKs. - */ + putkey(mpage, mpage->mlinks->name, TYPE_Nm); - assert(of->title); - assert(mtitle); - if (strcasecmp(mtitle, of->title)) - skip = 1; + if (NULL != mdoc) { + if (NULL != (cp = mdoc_meta(mdoc)->name)) + putkey(mpage, cp, TYPE_Nm); + assert(NULL == mpage->desc); + parse_mdoc(mpage, mdoc_node(mdoc)); + putkey(mpage, NULL != mpage->desc ? + mpage->desc : mpage->mlinks->name, TYPE_Nd); + } else if (NULL != man) + parse_man(mpage, man_node(man)); + else + parse_cat(mpage); /* * Build a title string for the file. If it matches @@ -705,390 +1029,386 @@ index_merge(const struct of *of, struct mparse *mp, * found; else, remember it as missing. */ - if (warnings) { - buf->len = 0; - buf_appendb(buf, mtitle, strlen(mtitle)); - buf_appendb(buf, "(", 1); - buf_appendb(buf, msec, strlen(msec)); - if ('\0' != *march) { - buf_appendb(buf, "/", 1); - buf_appendb(buf, march, strlen(march)); - } - buf_appendb(buf, ")", 2); - for (p = buf->cp; '\0' != *p; p++) - *p = tolower((unsigned char)*p); - key.data = buf->cp; - key.size = buf->len; - val.data = NULL; - val.size = 0; - if (0 == skip) - val.data = emptystring; - else { - ch = (*files->get)(files, &key, &val, 0); - if (ch < 0) { - perror("hash"); - exit((int)MANDOCLEVEL_SYSERR); - } else if (ch > 0) { - val.data = (void *)fn; - val.size = strlen(fn) + 1; - } else - val.data = NULL; - } - if (NULL != val.data && - (*files->put)(files, &key, &val, 0) < 0) { - perror("hash"); + if (check_reachable) { + if (-1 == asprintf(&title_str, "%s(%s%s%s)", + mpage->title, mpage->sec, + '\0' == *mpage->arch ? "" : "/", + mpage->arch)) { + perror(NULL); exit((int)MANDOCLEVEL_SYSERR); } + tslot = ohash_qlookup(&title_table, title_str); + title_entry = ohash_find(&title_table, tslot); + if (NULL == title_entry) { + title_entry = mandoc_malloc( + sizeof(struct title)); + title_entry->title = title_str; + title_entry->file = mandoc_strdup( + match ? "" : mpage->mlinks->file); + ohash_insert(&title_table, tslot, + title_entry); + } else { + if (match) + *title_entry->file = '\0'; + free(title_str); + } } - if (skip && !use_all) - continue; - - /* - * The index record value consists of a nil-terminated - * filename, a nil-terminated manual section, and a - * nil-terminated description. Use the actual - * location of the file, such that the user can find - * it with man(1). Since the description may not be - * set, we set a sentinel to see if we're going to - * write a nil byte in its place. - */ - - dbuf->len = 0; - type = mdoc ? 'd' : (man ? 'a' : 'c'); - buf_appendb(dbuf, &type, 1); - buf_appendb(dbuf, fn, strlen(fn) + 1); - buf_appendb(dbuf, of->sec, strlen(of->sec) + 1); - buf_appendb(dbuf, of->title, strlen(of->title) + 1); - buf_appendb(dbuf, of->arch, strlen(of->arch) + 1); - - sv = dbuf->len; + dbindex(mpage, mc); + ohash_delete(&strings); + mpage = ohash_next(&mpages, &pslot); + } - /* - * Collect keyword/mask pairs. - * Each pair will become a new btree node. - */ + if (check_reachable) { + title_entry = ohash_first(&title_table, &tslot); + while (NULL != title_entry) { + if ('\0' != *title_entry->file) + say(title_entry->file, + "Probably unreachable, title is %s", + title_entry->title); + free(title_entry->title); + free(title_entry->file); + free(title_entry); + title_entry = ohash_next(&title_table, &tslot); + } + ohash_delete(&title_table); + } +} - hash_reset(&hash); - if (mdoc) - pmdoc_node(hash, buf, dbuf, - mdoc_node(mdoc), mdoc_meta(mdoc)); - else if (man) - pman_node(hash, buf, dbuf, man_node(man)); - else - pformatted(hash, buf, dbuf, of); +static void +parse_cat(struct mpage *mpage) +{ + FILE *stream; + char *line, *p, *title; + size_t len, plen, titlesz; - /* Test mode, do not access any database. */ + if (NULL == (stream = fopen(mpage->mlinks->file, "r"))) { + if (warnings) + say(mpage->mlinks->file, NULL); + return; + } - if (NULL == mdb->db || NULL == mdb->idx) - continue; + /* Skip to first blank line. */ - /* - * Make sure the file name is always registered - * as an .Nm search key. - */ - buf->len = 0; - buf_append(buf, of->title); - hash_put(hash, buf, TYPE_Nm); + while (NULL != (line = fgetln(stream, &len))) + if ('\n' == *line) + break; - /* - * Reclaim an empty index record, if available. - * Use its record number for all new btree nodes. - */ + /* + * Assume the first line that is not indented + * is the first section header. Skip to it. + */ - if (recs->cur > 0) { - recs->cur--; - rec = recs->stack[(int)recs->cur]; - } else if (recs->last > 0) { - rec = recs->last; - recs->last = 0; - } else - rec++; - vbuf[1] = htobe64(rec); + while (NULL != (line = fgetln(stream, &len))) + if ('\n' != *line && ' ' != *line) + break; + + /* + * Read up until the next section into a buffer. + * Strip the leading and trailing newline from each read line, + * appending a trailing space. + * Ignore empty (whitespace-only) lines. + */ - /* - * Copy from the in-memory hashtable of pending - * keyword/mask pairs into the database. - */ + titlesz = 0; + title = NULL; - seq = R_FIRST; - while (0 == (ch = (*hash->seq)(hash, &key, &val, seq))) { - seq = R_NEXT; - assert(sizeof(uint64_t) == val.size); - memcpy(&mask, val.data, val.size); - vbuf[0] = htobe64(mask); - val.size = sizeof(vbuf); - val.data = &vbuf; - dbt_put(mdb->db, mdb->dbn, &key, &val); - } - if (ch < 0) { - perror("hash"); - unlink(mdb->dbn); - unlink(mdb->idxn); - exit((int)MANDOCLEVEL_SYSERR); + while (NULL != (line = fgetln(stream, &len))) { + if (' ' != *line || '\n' != line[len - 1]) + break; + while (len > 0 && isspace((unsigned char)*line)) { + line++; + len--; } + if (1 == len) + continue; + title = mandoc_realloc(title, titlesz + len); + memcpy(title + titlesz, line, len); + titlesz += len; + title[titlesz - 1] = ' '; + } - /* - * Apply to the index. If we haven't had a description - * set, put an empty one in now. - */ - - if (dbuf->len == sv) - buf_appendb(dbuf, "", 1); + /* + * If no page content can be found, or the input line + * is already the next section header, or there is no + * trailing newline, reuse the page title as the page + * description. + */ - key.data = &rec; - key.size = sizeof(recno_t); + if (NULL == title || '\0' == *title) { + if (warnings) + say(mpage->mlinks->file, + "Cannot find NAME section"); + assert(NULL == mpage->desc); + mpage->desc = mandoc_strdup(mpage->mlinks->name); + putkey(mpage, mpage->mlinks->name, TYPE_Nd); + fclose(stream); + free(title); + return; + } - val.data = dbuf->cp; - val.size = dbuf->len; + title = mandoc_realloc(title, titlesz + 1); + title[titlesz] = '\0'; - if (verb) - printf("%s: adding to index\n", fn); + /* + * Skip to the first dash. + * Use the remaining line as the description (no more than 70 + * bytes). + */ - dbt_put(mdb->idx, mdb->idxn, &key, &val); + if (NULL != (p = strstr(title, "- "))) { + for (p += 2; ' ' == *p || '\b' == *p; p++) + /* Skip to next word. */ ; + } else { + if (warnings) + say(mpage->mlinks->file, + "No dash in title line"); + p = title; } - /* - * Iterate the remembered file titles and check that - * all files can be found by their main title. - */ + plen = strlen(p); - if (warnings) { - seq = R_FIRST; - while (0 == (*files->seq)(files, &key, &val, seq)) { - seq = R_NEXT; - if (val.size) - fprintf(stderr, "%s: probably " - "unreachable, title is %s\n", - (char *)val.data, (char *)key.data); - } - (*files->close)(files); + /* Strip backspace-encoding from line. */ + + while (NULL != (line = memchr(p, '\b', plen))) { + len = line - p; + if (0 == len) { + memmove(line, line + 1, plen--); + continue; + } + memmove(line - 1, line + 1, plen - len); + plen -= 2; } + + assert(NULL == mpage->desc); + mpage->desc = mandoc_strdup(p); + putkey(mpage, mpage->desc, TYPE_Nd); + fclose(stream); + free(title); } /* - * Scan through all entries in the index file `idx' and prune those - * entries in `ofile'. - * Pruning consists of removing from `db', then invalidating the entry - * in `idx' (zeroing its value size). + * Put a type/word pair into the word database for this particular file. */ static void -index_prune(const struct of *ofile, struct mdb *mdb, struct recs *recs) +putkey(const struct mpage *mpage, const char *value, uint64_t type) { - const struct of *of; - const char *fn; - uint64_t vbuf[2]; - unsigned seq, sseq; - DBT key, val; - int ch; - - recs->cur = 0; - seq = R_FIRST; - while (0 == (ch = (*mdb->idx->seq)(mdb->idx, &key, &val, seq))) { - seq = R_NEXT; - assert(sizeof(recno_t) == key.size); - memcpy(&recs->last, key.data, key.size); - /* Deleted records are zero-sized. Skip them. */ + assert(NULL != value); + putkeys(mpage, value, strlen(value), type); +} - if (0 == val.size) - goto cont; +/* + * Grok all nodes at or below a certain mdoc node into putkey(). + */ +static void +putmdockey(const struct mpage *mpage, + const struct mdoc_node *n, uint64_t m) +{ - /* - * Make sure we're sane. - * Read past our mdoc/man/cat type to the next string, - * then make sure it's bounded by a NUL. - * Failing any of these, we go into our error handler. - */ + for ( ; NULL != n; n = n->next) { + if (NULL != n->child) + putmdockey(mpage, n->child, m); + if (MDOC_TEXT == n->type) + putkey(mpage, n->string, m); + } +} - fn = (char *)val.data + 1; - if (NULL == memchr(fn, '\0', val.size - 1)) - break; +static void +parse_man(struct mpage *mpage, const struct man_node *n) +{ + const struct man_node *head, *body; + char *start, *sv, *title; + char byte; + size_t sz, titlesz; - /* - * Search for the file in those we care about. - * XXX: build this into a tree. Too slow. - */ + if (NULL == n) + return; - for (of = ofile->first; of; of = of->next) - if (0 == strcmp(fn, of->fname)) - break; + /* + * We're only searching for one thing: the first text child in + * the BODY of a NAME section. Since we don't keep track of + * sections in -man, run some hoops to find out whether we're in + * the correct section or not. + */ - if (NULL == of) - continue; + if (MAN_BODY == n->type && MAN_SH == n->tok) { + body = n; + assert(body->parent); + if (NULL != (head = body->parent->head) && + 1 == head->nchild && + NULL != (head = (head->child)) && + MAN_TEXT == head->type && + 0 == strcmp(head->string, "NAME") && + NULL != (body = body->child) && + MAN_TEXT == body->type) { - /* - * Search through the keyword database, throwing out all - * references to our file. - */ + title = NULL; + titlesz = 0; - sseq = R_FIRST; - while (0 == (ch = (*mdb->db->seq)(mdb->db, - &key, &val, sseq))) { - sseq = R_NEXT; - if (sizeof(vbuf) != val.size) - break; + /* + * Suck the entire NAME section into memory. + * Yes, we might run away. + * But too many manuals have big, spread-out + * NAME sections over many lines. + */ - memcpy(vbuf, val.data, val.size); - if (recs->last != betoh64(vbuf[1])) - continue; + for ( ; NULL != body; body = body->next) { + if (MAN_TEXT != body->type) + break; + if (0 == (sz = strlen(body->string))) + continue; + title = mandoc_realloc + (title, titlesz + sz + 1); + memcpy(title + titlesz, body->string, sz); + titlesz += sz + 1; + title[titlesz - 1] = ' '; + } + if (NULL == title) + return; - if ((ch = (*mdb->db->del)(mdb->db, - &key, R_CURSOR)) < 0) - break; - } + title = mandoc_realloc(title, titlesz + 1); + title[titlesz] = '\0'; - if (ch < 0) { - perror(mdb->dbn); - exit((int)MANDOCLEVEL_SYSERR); - } else if (1 != ch) { - fprintf(stderr, "%s: corrupt database\n", - mdb->dbn); - exit((int)MANDOCLEVEL_SYSERR); - } + /* Skip leading space. */ - if (verb) - printf("%s: deleting from index\n", fn); + sv = title; + while (isspace((unsigned char)*sv)) + sv++; - val.size = 0; - ch = (*mdb->idx->put)(mdb->idx, &key, &val, R_CURSOR); + if (0 == (sz = strlen(sv))) { + free(title); + return; + } - if (ch < 0) - break; -cont: - if (recs->cur >= recs->size) { - recs->size += MANDOC_SLOP; - recs->stack = mandoc_realloc(recs->stack, - recs->size * sizeof(recno_t)); - } + /* Erase trailing space. */ - recs->stack[(int)recs->cur] = recs->last; - recs->cur++; - } + start = &sv[sz - 1]; + while (start > sv && isspace((unsigned char)*start)) + *start-- = '\0'; - if (ch < 0) { - perror(mdb->idxn); - exit((int)MANDOCLEVEL_SYSERR); - } else if (1 != ch) { - fprintf(stderr, "%s: corrupt index\n", mdb->idxn); - exit((int)MANDOCLEVEL_SYSERR); - } + if (start == sv) { + free(title); + return; + } - recs->last++; -} + start = sv; -/* - * Grow the buffer (if necessary) and copy in a binary string. - */ -static void -buf_appendb(struct buf *buf, const void *cp, size_t sz) -{ + /* + * Go through a special heuristic dance here. + * Conventionally, one or more manual names are + * comma-specified prior to a whitespace, then a + * dash, then a description. Try to puzzle out + * the name parts here. + */ - /* Overshoot by MANDOC_BUFSZ. */ + for ( ;; ) { + sz = strcspn(start, " ,"); + if ('\0' == start[sz]) + break; - while (buf->len + sz >= buf->size) { - buf->size = buf->len + sz + MANDOC_BUFSZ; - buf->cp = mandoc_realloc(buf->cp, buf->size); - } + byte = start[sz]; + start[sz] = '\0'; - memcpy(buf->cp + (int)buf->len, cp, sz); - buf->len += sz; -} + putkey(mpage, start, TYPE_Nm); -/* - * Append a nil-terminated string to the buffer. - * This can be invoked multiple times. - * The buffer string will be nil-terminated. - * If invoked multiple times, a space is put between strings. - */ -static void -buf_append(struct buf *buf, const char *cp) -{ - size_t sz; + if (' ' == byte) { + start += sz + 1; + break; + } - if (0 == (sz = strlen(cp))) - return; + assert(',' == byte); + start += sz + 1; + while (' ' == *start) + start++; + } - if (buf->len) - buf->cp[(int)buf->len - 1] = ' '; + if (sv == start) { + putkey(mpage, start, TYPE_Nm); + free(title); + return; + } - buf_appendb(buf, cp, sz + 1); -} + while (isspace((unsigned char)*start)) + start++; -/* - * Recursively add all text from a given node. - * This is optimised for general mdoc nodes in this context, which do - * not consist of subexpressions and having a recursive call for n->next - * would be wasteful. - * The "f" variable should be 0 unless called from pmdoc_Nd for the - * description buffer, which does not start at the beginning of the - * buffer. - */ -static void -buf_appendmdoc(struct buf *buf, const struct mdoc_node *n, int f) -{ + if (0 == strncmp(start, "-", 1)) + start += 1; + else if (0 == strncmp(start, "\\-\\-", 4)) + start += 4; + else if (0 == strncmp(start, "\\-", 2)) + start += 2; + else if (0 == strncmp(start, "\\(en", 4)) + start += 4; + else if (0 == strncmp(start, "\\(em", 4)) + start += 4; - for ( ; n; n = n->next) { - if (n->child) - buf_appendmdoc(buf, n->child, f); + while (' ' == *start) + start++; - if (MDOC_TEXT == n->type && f) { - f = 0; - buf_appendb(buf, n->string, - strlen(n->string) + 1); - } else if (MDOC_TEXT == n->type) - buf_append(buf, n->string); + assert(NULL == mpage->desc); + mpage->desc = mandoc_strdup(start); + putkey(mpage, mpage->desc, TYPE_Nd); + free(title); + return; + } + } + for (n = n->child; n; n = n->next) { + if (NULL != mpage->desc) + break; + parse_man(mpage, n); } } static void -hash_reset(DB **db) +parse_mdoc(struct mpage *mpage, const struct mdoc_node *n) { - DB *hash; - if (NULL != (hash = *db)) - (*hash->close)(hash); - - *db = dbopen(NULL, O_CREAT|O_RDWR, 0644, DB_HASH, NULL); - if (NULL == *db) { - perror("hash"); - exit((int)MANDOCLEVEL_SYSERR); + assert(NULL != n); + for (n = n->child; NULL != n; n = n->next) { + switch (n->type) { + case (MDOC_ELEM): + /* FALLTHROUGH */ + case (MDOC_BLOCK): + /* FALLTHROUGH */ + case (MDOC_HEAD): + /* FALLTHROUGH */ + case (MDOC_BODY): + /* FALLTHROUGH */ + case (MDOC_TAIL): + if (NULL != mdocs[n->tok].fp) + if (0 == (*mdocs[n->tok].fp)(mpage, n)) + break; + if (mdocs[n->tok].mask) + putmdockey(mpage, n->child, + mdocs[n->tok].mask); + break; + default: + assert(MDOC_ROOT != n->type); + continue; + } + if (NULL != n->child) + parse_mdoc(mpage, n); } } -/* ARGSUSED */ -static int -pmdoc_head(MDOC_ARGS) -{ - - return(MDOC_HEAD == n->type); -} - -/* ARGSUSED */ static int -pmdoc_body(MDOC_ARGS) -{ - - return(MDOC_BODY == n->type); -} - -/* ARGSUSED */ -static int -pmdoc_Fd(MDOC_ARGS) +parse_mdoc_Fd(struct mpage *mpage, const struct mdoc_node *n) { const char *start, *end; size_t sz; - if (SEC_SYNOPSIS != n->sec) - return(0); - if (NULL == (n = n->child) || MDOC_TEXT != n->type) + if (SEC_SYNOPSIS != n->sec || + NULL == (n = n->child) || + MDOC_TEXT != n->type) return(0); /* * Only consider those `Fd' macro fields that begin with an * "inclusion" token (versus, e.g., #define). */ + if (strcmp("#include", n->string)) return(0); @@ -1111,121 +1431,121 @@ pmdoc_Fd(MDOC_ARGS) if ('>' == *end || '"' == *end) end--; - assert(end >= start); - - buf_appendb(buf, start, (size_t)(end - start + 1)); - buf_appendb(buf, "", 1); + if (end > start) + putkeys(mpage, start, end - start + 1, TYPE_In); return(1); } -/* ARGSUSED */ static int -pmdoc_In(MDOC_ARGS) +parse_mdoc_In(struct mpage *mpage, const struct mdoc_node *n) { - if (NULL == n->child || MDOC_TEXT != n->child->type) + if (NULL != n->child && MDOC_TEXT == n->child->type) return(0); - buf_append(buf, n->child->string); + putkey(mpage, n->child->string, TYPE_In); return(1); } -/* ARGSUSED */ static int -pmdoc_Fn(MDOC_ARGS) +parse_mdoc_Fn(struct mpage *mpage, const struct mdoc_node *n) { - struct mdoc_node *nn; const char *cp; - nn = n->child; - - if (NULL == nn || MDOC_TEXT != nn->type) + if (NULL == (n = n->child) || MDOC_TEXT != n->type) return(0); - /* .Fn "struct type *name" "char *arg" */ - - cp = strrchr(nn->string, ' '); - if (NULL == cp) - cp = nn->string; + /* + * Parse: .Fn "struct type *name" "char *arg". + * First strip away pointer symbol. + * Then store the function name, then type. + * Finally, store the arguments. + */ - /* Strip away pointer symbol. */ + if (NULL == (cp = strrchr(n->string, ' '))) + cp = n->string; while ('*' == *cp) cp++; - /* Store the function name. */ - - buf_append(buf, cp); - hash_put(hash, buf, TYPE_Fn); - - /* Store the function type. */ + putkey(mpage, cp, TYPE_Fn); - if (nn->string < cp) { - buf->len = 0; - buf_appendb(buf, nn->string, cp - nn->string); - buf_appendb(buf, "", 1); - hash_put(hash, buf, TYPE_Ft); - } - - /* Store the arguments. */ + if (n->string < cp) + putkeys(mpage, n->string, cp - n->string, TYPE_Ft); - for (nn = nn->next; nn; nn = nn->next) { - if (MDOC_TEXT != nn->type) - continue; - buf->len = 0; - buf_append(buf, nn->string); - hash_put(hash, buf, TYPE_Fa); - } + for (n = n->next; NULL != n; n = n->next) + if (MDOC_TEXT == n->type) + putkey(mpage, n->string, TYPE_Fa); return(0); } -/* ARGSUSED */ static int -pmdoc_St(MDOC_ARGS) +parse_mdoc_St(struct mpage *mpage, const struct mdoc_node *n) { if (NULL == n->child || MDOC_TEXT != n->child->type) return(0); - buf_append(buf, n->child->string); + putkey(mpage, n->child->string, TYPE_St); return(1); } -/* ARGSUSED */ static int -pmdoc_Xr(MDOC_ARGS) +parse_mdoc_Xr(struct mpage *mpage, const struct mdoc_node *n) { + char *cp; if (NULL == (n = n->child)) return(0); - buf_appendb(buf, n->string, strlen(n->string)); - - if (NULL != (n = n->next)) { - buf_appendb(buf, ".", 1); - buf_appendb(buf, n->string, strlen(n->string) + 1); - } else - buf_appendb(buf, ".", 2); + if (NULL == n->next) { + putkey(mpage, n->string, TYPE_Xr); + return(0); + } - return(1); + if (-1 == asprintf(&cp, "%s(%s)", n->string, n->next->string)) { + perror(NULL); + exit((int)MANDOCLEVEL_SYSERR); + } + putkey(mpage, cp, TYPE_Xr); + free(cp); + return(0); } -/* ARGSUSED */ static int -pmdoc_Nd(MDOC_ARGS) +parse_mdoc_Nd(struct mpage *mpage, const struct mdoc_node *n) { + size_t sz; if (MDOC_BODY != n->type) return(0); - buf_appendmdoc(dbuf, n->child, 1); + /* + * Special-case the `Nd' because we need to put the description + * into the document table. + */ + + for (n = n->child; NULL != n; n = n->next) { + if (MDOC_TEXT == n->type) { + if (NULL != mpage->desc) { + sz = strlen(mpage->desc) + + strlen(n->string) + 2; + mpage->desc = mandoc_realloc( + mpage->desc, sz); + strlcat(mpage->desc, " ", sz); + strlcat(mpage->desc, n->string, sz); + } else + mpage->desc = mandoc_strdup(n->string); + } + if (NULL != n->child) + parse_mdoc_Nd(mpage, n); + } return(1); } -/* ARGSUSED */ static int -pmdoc_Nm(MDOC_ARGS) +parse_mdoc_Nm(struct mpage *mpage, const struct mdoc_node *n) { if (SEC_NAME == n->sec) @@ -1233,758 +1553,558 @@ pmdoc_Nm(MDOC_ARGS) else if (SEC_SYNOPSIS != n->sec || MDOC_HEAD != n->type) return(0); - if (NULL == n->child) - buf_append(buf, m->name); - return(1); } -/* ARGSUSED */ static int -pmdoc_Sh(MDOC_ARGS) +parse_mdoc_Sh(struct mpage *mpage, const struct mdoc_node *n) { return(SEC_CUSTOM == n->sec && MDOC_HEAD == n->type); } -static void -hash_put(DB *db, const struct buf *buf, uint64_t mask) +static int +parse_mdoc_head(struct mpage *mpage, const struct mdoc_node *n) { - uint64_t oldmask; - DBT key, val; - int rc; - - if (buf->len < 2) - return; - - key.data = buf->cp; - key.size = buf->len; - if ((rc = (*db->get)(db, &key, &val, 0)) < 0) { - perror("hash"); - exit((int)MANDOCLEVEL_SYSERR); - } else if (0 == rc) { - assert(sizeof(uint64_t) == val.size); - memcpy(&oldmask, val.data, val.size); - mask |= oldmask; - } - - val.data = &mask; - val.size = sizeof(uint64_t); - - if ((rc = (*db->put)(db, &key, &val, 0)) < 0) { - perror("hash"); - exit((int)MANDOCLEVEL_SYSERR); - } + return(MDOC_HEAD == n->type); } -static void -dbt_put(DB *db, const char *dbn, DBT *key, DBT *val) +static int +parse_mdoc_body(struct mpage *mpage, const struct mdoc_node *n) { - assert(key->size); - assert(val->size); - - if (0 == (*db->put)(db, key, val, 0)) - return; - - perror(dbn); - exit((int)MANDOCLEVEL_SYSERR); - /* NOTREACHED */ + return(MDOC_BODY == n->type); } /* - * Call out to per-macro handlers after clearing the persistent database - * key. If the macro sets the database key, flush it to the database. + * Add a string to the hash table for the current manual. + * Each string has a bitmask telling which macros it belongs to. + * When we finish the manual, we'll dump the table. */ static void -pmdoc_node(MDOC_ARGS) +putkeys(const struct mpage *mpage, + const char *cp, size_t sz, uint64_t v) { + struct str *s; + unsigned int slot; + const char *end; - if (NULL == n) + if (0 == sz) return; - switch (n->type) { - case (MDOC_HEAD): - /* FALLTHROUGH */ - case (MDOC_BODY): - /* FALLTHROUGH */ - case (MDOC_TAIL): - /* FALLTHROUGH */ - case (MDOC_BLOCK): - /* FALLTHROUGH */ - case (MDOC_ELEM): - buf->len = 0; + end = cp + sz; + slot = ohash_qlookupi(&strings, cp, &end); + s = ohash_find(&strings, slot); - /* - * Both NULL handlers and handlers returning true - * request using the data. Only skip the element - * when the handler returns false. - */ - - if (NULL != mdocs[n->tok].fp && - 0 == (*mdocs[n->tok].fp)(hash, buf, dbuf, n, m)) - break; - - /* - * For many macros, use the text from all children. - * Set zero flags for macros not needing this. - * In that case, the handler must fill the buffer. - */ - - if (MDOCF_CHILD & mdocs[n->tok].flags) - buf_appendmdoc(buf, n->child, 0); - - /* - * Cover the most common case: - * Automatically stage one string per element. - * Set a zero mask for macros not needing this. - * Additional staging can be done in the handler. - */ - - if (mdocs[n->tok].mask) - hash_put(hash, buf, mdocs[n->tok].mask); - break; - default: - break; + if (NULL != s && mpage == s->mpage) { + s->mask |= v; + return; + } else if (NULL == s) { + s = mandoc_calloc(sizeof(struct str) + sz + 1, 1); + memcpy(s->key, cp, sz); + ohash_insert(&strings, slot, s); } - - pmdoc_node(hash, buf, dbuf, n->child, m); - pmdoc_node(hash, buf, dbuf, n->next, m); + s->mpage = mpage; + s->mask = v; } -static int -pman_node(MAN_ARGS) +/* + * Take a Unicode codepoint and produce its UTF-8 encoding. + * This isn't the best way to do this, but it works. + * The magic numbers are from the UTF-8 packaging. + * They're not as scary as they seem: read the UTF-8 spec for details. + */ +static size_t +utf8(unsigned int cp, char out[7]) { - const struct man_node *head, *body; - char *start, *sv, *title; - size_t sz, titlesz; - - if (NULL == n) + size_t rc; + + rc = 0; + if (cp <= 0x0000007F) { + rc = 1; + out[0] = (char)cp; + } else if (cp <= 0x000007FF) { + rc = 2; + out[0] = (cp >> 6 & 31) | 192; + out[1] = (cp & 63) | 128; + } else if (cp <= 0x0000FFFF) { + rc = 3; + out[0] = (cp >> 12 & 15) | 224; + out[1] = (cp >> 6 & 63) | 128; + out[2] = (cp & 63) | 128; + } else if (cp <= 0x001FFFFF) { + rc = 4; + out[0] = (cp >> 18 & 7) | 240; + out[1] = (cp >> 12 & 63) | 128; + out[2] = (cp >> 6 & 63) | 128; + out[3] = (cp & 63) | 128; + } else if (cp <= 0x03FFFFFF) { + rc = 5; + out[0] = (cp >> 24 & 3) | 248; + out[1] = (cp >> 18 & 63) | 128; + out[2] = (cp >> 12 & 63) | 128; + out[3] = (cp >> 6 & 63) | 128; + out[4] = (cp & 63) | 128; + } else if (cp <= 0x7FFFFFFF) { + rc = 6; + out[0] = (cp >> 30 & 1) | 252; + out[1] = (cp >> 24 & 63) | 128; + out[2] = (cp >> 18 & 63) | 128; + out[3] = (cp >> 12 & 63) | 128; + out[4] = (cp >> 6 & 63) | 128; + out[5] = (cp & 63) | 128; + } else return(0); - /* - * We're only searching for one thing: the first text child in - * the BODY of a NAME section. Since we don't keep track of - * sections in -man, run some hoops to find out whether we're in - * the correct section or not. - */ - - if (MAN_BODY == n->type && MAN_SH == n->tok) { - body = n; - assert(body->parent); - if (NULL != (head = body->parent->head) && - 1 == head->nchild && - NULL != (head = (head->child)) && - MAN_TEXT == head->type && - 0 == strcmp(head->string, "NAME") && - NULL != (body = body->child) && - MAN_TEXT == body->type) { - - title = NULL; - titlesz = 0; - /* - * Suck the entire NAME section into memory. - * Yes, we might run away. - * But too many manuals have big, spread-out - * NAME sections over many lines. - */ - for ( ; NULL != body; body = body->next) { - if (MAN_TEXT != body->type) - break; - if (0 == (sz = strlen(body->string))) - continue; - title = mandoc_realloc - (title, titlesz + sz + 1); - memcpy(title + titlesz, body->string, sz); - titlesz += sz + 1; - title[(int)titlesz - 1] = ' '; - } - if (NULL == title) - return(0); - - title = mandoc_realloc(title, titlesz + 1); - title[(int)titlesz] = '\0'; - - /* Skip leading space. */ - - sv = title; - while (isspace((unsigned char)*sv)) - sv++; + out[rc] = '\0'; + return(rc); +} - if (0 == (sz = strlen(sv))) { - free(title); - return(0); - } +/* + * Store the UTF-8 version of a key, or alias the pointer if the key has + * no UTF-8 transcription marks in it. + */ +static void +utf8key(struct mchars *mc, struct str *key) +{ + size_t sz, bsz, pos; + char utfbuf[7], res[5]; + char *buf; + const char *seq, *cpp, *val; + int len, u; + enum mandoc_esc esc; - /* Erase trailing space. */ + assert(NULL == key->utf8); - start = &sv[sz - 1]; - while (start > sv && isspace((unsigned char)*start)) - *start-- = '\0'; + res[0] = '\\'; + res[1] = '\t'; + res[2] = ASCII_NBRSP; + res[3] = ASCII_HYPH; + res[4] = '\0'; - if (start == sv) { - free(title); - return(0); - } + val = key->key; + bsz = strlen(val); - start = sv; + /* + * Pre-check: if we have no stop-characters, then set the + * pointer as ourselvse and get out of here. + */ + if (strcspn(val, res) == bsz) { + key->utf8 = key->key; + return; + } - /* - * Go through a special heuristic dance here. - * This is why -man manuals are great! - * (I'm being sarcastic: my eyes are bleeding.) - * Conventionally, one or more manual names are - * comma-specified prior to a whitespace, then a - * dash, then a description. Try to puzzle out - * the name parts here. - */ + /* Pre-allocate by the length of the input */ - for ( ;; ) { - sz = strcspn(start, " ,"); - if ('\0' == start[(int)sz]) - break; + buf = mandoc_malloc(++bsz); + pos = 0; - buf->len = 0; - buf_appendb(buf, start, sz); - buf_appendb(buf, "", 1); + while ('\0' != *val) { + /* + * Halt on the first escape sequence. + * This also halts on the end of string, in which case + * we just copy, fallthrough, and exit the loop. + */ + if ((sz = strcspn(val, res)) > 0) { + memcpy(&buf[pos], val, sz); + pos += sz; + val += sz; + } - hash_put(hash, buf, TYPE_Nm); + if (ASCII_HYPH == *val) { + buf[pos++] = '-'; + val++; + continue; + } else if ('\t' == *val || ASCII_NBRSP == *val) { + buf[pos++] = ' '; + val++; + continue; + } else if ('\\' != *val) + break; - if (' ' == start[(int)sz]) { - start += (int)sz + 1; - break; - } + /* Read past the slash. */ - assert(',' == start[(int)sz]); - start += (int)sz + 1; - while (' ' == *start) - start++; - } + val++; + u = 0; - buf->len = 0; + /* + * Parse the escape sequence and see if it's a + * predefined character or special character. + */ + esc = mandoc_escape + ((const char **)&val, &seq, &len); + if (ESCAPE_ERROR == esc) + break; - if (sv == start) { - buf_append(buf, start); - free(title); - return(1); - } + if (ESCAPE_SPECIAL != esc) + continue; + if (0 == (u = mchars_spec2cp(mc, seq, len))) + continue; - while (isspace((unsigned char)*start)) - start++; + /* + * If we have a Unicode codepoint, try to convert that + * to a UTF-8 byte string. + */ + cpp = utfbuf; + if (0 == (sz = utf8(u, utfbuf))) + continue; - if (0 == strncmp(start, "-", 1)) - start += 1; - else if (0 == strncmp(start, "\\-\\-", 4)) - start += 4; - else if (0 == strncmp(start, "\\-", 2)) - start += 2; - else if (0 == strncmp(start, "\\(en", 4)) - start += 4; - else if (0 == strncmp(start, "\\(em", 4)) - start += 4; + /* Copy the rendered glyph into the stream. */ - while (' ' == *start) - start++; + sz = strlen(cpp); + bsz += sz; - sz = strlen(start) + 1; - buf_appendb(dbuf, start, sz); - buf_appendb(buf, start, sz); + buf = mandoc_realloc(buf, bsz); - hash_put(hash, buf, TYPE_Nd); - free(title); - } + memcpy(&buf[pos], cpp, sz); + pos += sz; } - for (n = n->child; n; n = n->next) - if (pman_node(hash, buf, dbuf, n)) - return(1); - - return(0); + buf[pos] = '\0'; + key->utf8 = buf; } /* - * Parse a formatted manual page. - * By necessity, this involves rather crude guesswork. + * Flush the current page's terms (and their bits) into the database. + * Wrap the entire set of additions in a transaction to make sqlite be a + * little faster. + * Also, UTF-8-encode the description at the last possible moment. */ static void -pformatted(DB *hash, struct buf *buf, - struct buf *dbuf, const struct of *of) +dbindex(const struct mpage *mpage, struct mchars *mc) { - FILE *stream; - char *line, *p, *title; - size_t len, plen, titlesz; - - if (NULL == (stream = fopen(of->fname, "r"))) { - if (warnings) - perror(of->fname); - return; - } - - /* - * Always use the title derived from the filename up front, - * do not even try to find it in the file. This also makes - * sure we don't end up with an orphan index record, even if - * the file content turns out to be completely unintelligible. - */ - - buf->len = 0; - buf_append(buf, of->title); - hash_put(hash, buf, TYPE_Nm); - - /* Skip to first blank line. */ - - while (NULL != (line = fgetln(stream, &len))) - if ('\n' == *line) - break; - - /* - * Assume the first line that is not indented - * is the first section header. Skip to it. - */ - - while (NULL != (line = fgetln(stream, &len))) - if ('\n' != *line && ' ' != *line) - break; - - /* - * Read up until the next section into a buffer. - * Strip the leading and trailing newline from each read line, - * appending a trailing space. - * Ignore empty (whitespace-only) lines. - */ - - titlesz = 0; - title = NULL; - - while (NULL != (line = fgetln(stream, &len))) { - if (' ' != *line || '\n' != line[(int)len - 1]) - break; - while (len > 0 && isspace((unsigned char)*line)) { - line++; - len--; - } - if (1 == len) - continue; - title = mandoc_realloc(title, titlesz + len); - memcpy(title + titlesz, line, len); - titlesz += len; - title[(int)titlesz - 1] = ' '; - } - + struct mlink *mlink; + struct str *key; + const char *desc; + int64_t recno; + size_t i; + unsigned int slot; - /* - * If no page content can be found, or the input line - * is already the next section header, or there is no - * trailing newline, reuse the page title as the page - * description. - */ + if (verb) + say(mpage->mlinks->file, "Adding to index"); - if (NULL == title || '\0' == *title) { - if (warnings) - fprintf(stderr, "%s: cannot find NAME section\n", - of->fname); - buf_appendb(dbuf, buf->cp, buf->size); - hash_put(hash, buf, TYPE_Nd); - fclose(stream); - free(title); + if (nodb) return; + + desc = ""; + if (NULL != mpage->desc && '\0' != *mpage->desc) { + key = ohash_find(&strings, + ohash_qlookup(&strings, mpage->desc)); + assert(NULL != key); + if (NULL == key->utf8) + utf8key(mc, key); + desc = key->utf8; } - title = mandoc_realloc(title, titlesz + 1); - title[(int)titlesz] = '\0'; + SQL_EXEC("BEGIN TRANSACTION"); + i = 1; /* - * Skip to the first dash. - * Use the remaining line as the description (no more than 70 - * bytes). + * XXX The following three lines are obsolete + * and only kept for backward compatibility + * until apropos(1) and friends have caught up. */ - - if (NULL != (p = strstr(title, "- "))) { - for (p += 2; ' ' == *p || '\b' == *p; p++) - /* Skip to next word. */ ; - } else { - if (warnings) - fprintf(stderr, "%s: no dash in title line\n", - of->fname); - p = title; + SQL_BIND_TEXT(stmts[STMT_INSERT_PAGE], i, mpage->mlinks->file); + SQL_BIND_TEXT(stmts[STMT_INSERT_PAGE], i, mpage->mlinks->dsec); + SQL_BIND_TEXT(stmts[STMT_INSERT_PAGE], i, mpage->mlinks->arch); + SQL_BIND_TEXT(stmts[STMT_INSERT_PAGE], i, desc); + SQL_BIND_INT(stmts[STMT_INSERT_PAGE], i, FORM_SRC == mpage->form); + SQL_STEP(stmts[STMT_INSERT_PAGE]); + recno = sqlite3_last_insert_rowid(db); + sqlite3_reset(stmts[STMT_INSERT_PAGE]); + + for (mlink = mpage->mlinks; mlink; mlink = mlink->next) { + i = 1; + SQL_BIND_TEXT(stmts[STMT_INSERT_LINK], i, mlink->file); + SQL_BIND_TEXT(stmts[STMT_INSERT_LINK], i, mlink->dsec); + SQL_BIND_TEXT(stmts[STMT_INSERT_LINK], i, mlink->arch); + SQL_BIND_TEXT(stmts[STMT_INSERT_LINK], i, mlink->name); + SQL_BIND_INT64(stmts[STMT_INSERT_LINK], i, recno); + SQL_STEP(stmts[STMT_INSERT_LINK]); + sqlite3_reset(stmts[STMT_INSERT_LINK]); } - plen = strlen(p); - - /* Strip backspace-encoding from line. */ - - while (NULL != (line = memchr(p, '\b', plen))) { - len = line - p; - if (0 == len) { - memmove(line, line + 1, plen--); - continue; - } - memmove(line - 1, line + 1, plen - len); - plen -= 2; + for (key = ohash_first(&strings, &slot); NULL != key; + key = ohash_next(&strings, &slot)) { + assert(key->mpage == mpage); + if (NULL == key->utf8) + utf8key(mc, key); + i = 1; + SQL_BIND_INT64(stmts[STMT_INSERT_KEY], i, key->mask); + SQL_BIND_TEXT(stmts[STMT_INSERT_KEY], i, key->utf8); + SQL_BIND_INT64(stmts[STMT_INSERT_KEY], i, recno); + SQL_STEP(stmts[STMT_INSERT_KEY]); + sqlite3_reset(stmts[STMT_INSERT_KEY]); + if (key->utf8 != key->key) + free(key->utf8); + free(key); } - buf_appendb(dbuf, p, plen + 1); - buf->len = 0; - buf_appendb(buf, p, plen + 1); - hash_put(hash, buf, TYPE_Nd); - fclose(stream); - free(title); + SQL_EXEC("END TRANSACTION"); } static void -ofile_argbuild(int argc, char *argv[], struct of **of, - const char *basedir) +dbprune(void) { - char buf[PATH_MAX]; - char pbuf[PATH_MAX]; - const char *sec, *arch, *title; - char *relpath, *p; - int i, src_form; - struct of *nof; - - for (i = 0; i < argc; i++) { - if (NULL == (relpath = realpath(argv[i], pbuf))) { - perror(argv[i]); - continue; - } - if (NULL != basedir) { - if (strstr(pbuf, basedir) != pbuf) { - fprintf(stderr, "%s: file outside " - "base directory %s\n", - pbuf, basedir); - continue; - } - relpath = pbuf + strlen(basedir); - } - - /* - * Try to infer the manual section, architecture and - * page title from the path, assuming it looks like - * man*[/<arch>]/<title>.<section> or - * cat<section>[/<arch>]/<title>.0 - */ - - if (strlcpy(buf, relpath, sizeof(buf)) >= sizeof(buf)) { - fprintf(stderr, "%s: path too long\n", relpath); - continue; - } - sec = arch = title = ""; - src_form = 0; - p = strrchr(buf, '\0'); - while (p-- > buf) { - if ('\0' == *sec && '.' == *p) { - sec = p + 1; - *p = '\0'; - if ('0' == *sec) - src_form |= MANDOC_FORM; - else if ('1' <= *sec && '9' >= *sec) - src_form |= MANDOC_SRC; - continue; - } - if ('/' != *p) - continue; - if ('\0' == *title) { - title = p + 1; - *p = '\0'; - continue; - } - if (0 == strncmp("man", p + 1, 3)) - src_form |= MANDOC_SRC; - else if (0 == strncmp("cat", p + 1, 3)) - src_form |= MANDOC_FORM; - else - arch = p + 1; - break; - } - if ('\0' == *title) { - if (warnings) - fprintf(stderr, - "%s: cannot deduce title " - "from filename\n", - relpath); - title = buf; - } - - /* - * Build the file structure. - */ + struct mpage *mpage; + struct mlink *mlink; + size_t i; + unsigned int slot; - nof = mandoc_calloc(1, sizeof(struct of)); - nof->fname = mandoc_strdup(relpath); - nof->sec = mandoc_strdup(sec); - nof->arch = mandoc_strdup(arch); - nof->title = mandoc_strdup(title); - nof->src_form = src_form; - - /* - * Add the structure to the list. - */ + if (nodb) + return; - if (verb > 1) - printf("%s: scheduling\n", relpath); - if (NULL == *of) { - *of = nof; - (*of)->first = nof; - } else { - nof->first = (*of)->first; - (*of)->next = nof; - *of = nof; - } + mpage = ohash_first(&mpages, &slot); + while (NULL != mpage) { + mlink = mpage->mlinks; + i = 1; + SQL_BIND_TEXT(stmts[STMT_DELETE_PAGE], i, mlink->file); + SQL_STEP(stmts[STMT_DELETE_PAGE]); + sqlite3_reset(stmts[STMT_DELETE_PAGE]); + if (verb) + say(mlink->file, "Deleted from index"); + mpage = ohash_next(&mpages, &slot); } } /* - * Recursively build up a list of files to parse. - * We use this instead of ftw() and so on because I don't want global - * variables hanging around. - * This ignores the mandoc.db and mandoc.index files, but assumes that - * everything else is a manual. - * Pass in a pointer to a NULL structure for the first invocation. + * Close an existing database and its prepared statements. + * If "real" is not set, rename the temporary file into the real one. */ static void -ofile_dirbuild(const char *dir, const char* psec, const char *parch, - int p_src_form, struct of **of) +dbclose(int real) { - char buf[PATH_MAX]; - size_t sz; - DIR *d; - const char *fn, *sec, *arch; - char *p, *q, *suffix; - struct of *nof; - struct dirent *dp; - int src_form; - - if (NULL == (d = opendir(dir))) { - if (warnings) - perror(dir); + size_t i; + + if (nodb) return; + + for (i = 0; i < STMT__MAX; i++) { + sqlite3_finalize(stmts[i]); + stmts[i] = NULL; } - while (NULL != (dp = readdir(d))) { - fn = dp->d_name; + sqlite3_close(db); + db = NULL; - if ('.' == *fn) - continue; + if (real) + return; - src_form = p_src_form; + if (-1 == rename(MANDOC_DB "~", MANDOC_DB)) { + exitcode = (int)MANDOCLEVEL_SYSERR; + say(MANDOC_DB, NULL); + } +} - if (DT_DIR == dp->d_type) { - sec = psec; - arch = parch; +/* + * This is straightforward stuff. + * Open a database connection to a "temporary" database, then open a set + * of prepared statements we'll use over and over again. + * If "real" is set, we use the existing database; if not, we truncate a + * temporary one. + * Must be matched by dbclose(). + */ +static int +dbopen(int real) +{ + const char *file, *sql; + int rc, ofl; - /* - * By default, only use directories called: - * man<section>/[<arch>/] or - * cat<section>/[<arch>/] - */ + if (nodb) + return(1); - if ('\0' == *sec) { - if(0 == strncmp("man", fn, 3)) { - src_form |= MANDOC_SRC; - sec = fn + 3; - } else if (0 == strncmp("cat", fn, 3)) { - src_form |= MANDOC_FORM; - sec = fn + 3; - } else { - if (warnings) fprintf(stderr, - "%s/%s: bad section\n", - dir, fn); - if (use_all) - sec = fn; - else - continue; - } - } else if ('\0' == *arch) { - if (NULL != strchr(fn, '.')) { - if (warnings) fprintf(stderr, - "%s/%s: bad architecture\n", - dir, fn); - if (0 == use_all) - continue; - } - arch = fn; - } else { - if (warnings) fprintf(stderr, "%s/%s: " - "excessive subdirectory\n", dir, fn); - if (0 == use_all) - continue; - } + ofl = SQLITE_OPEN_READWRITE; + if (0 == real) { + file = MANDOC_DB "~"; + if (-1 == remove(file) && ENOENT != errno) { + exitcode = (int)MANDOCLEVEL_SYSERR; + say(file, NULL); + return(0); + } + ofl |= SQLITE_OPEN_EXCLUSIVE; + } else + file = MANDOC_DB; + + rc = sqlite3_open_v2(file, &db, ofl, NULL); + if (SQLITE_OK == rc) + goto prepare_statements; + if (SQLITE_CANTOPEN != rc) { + exitcode = (int)MANDOCLEVEL_SYSERR; + say(file, NULL); + return(0); + } - buf[0] = '\0'; - strlcat(buf, dir, PATH_MAX); - strlcat(buf, "/", PATH_MAX); - sz = strlcat(buf, fn, PATH_MAX); + sqlite3_close(db); + db = NULL; - if (PATH_MAX <= sz) { - if (warnings) fprintf(stderr, "%s/%s: " - "path too long\n", dir, fn); - continue; - } + if (SQLITE_OK != (rc = sqlite3_open(file, &db))) { + exitcode = (int)MANDOCLEVEL_SYSERR; + say(file, NULL); + return(0); + } - if (verb > 1) - printf("%s: scanning\n", buf); + /* + * XXX The first three columns in table mpages are obsolete + * and only kept for backward compatibility + * until apropos(1) and friends have caught up. + */ + sql = "CREATE TABLE \"mpages\" (\n" + " \"file\" TEXT NOT NULL,\n" + " \"sec\" TEXT NOT NULL,\n" + " \"arch\" TEXT NOT NULL,\n" + " \"desc\" TEXT NOT NULL,\n" + " \"form\" INTEGER NOT NULL,\n" + " \"id\" INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL\n" + ");\n" + "\n" + "CREATE TABLE \"mlinks\" (\n" + " \"file\" TEXT NOT NULL,\n" + " \"sec\" TEXT NOT NULL,\n" + " \"arch\" TEXT NOT NULL,\n" + " \"name\" TEXT NOT NULL,\n" + " \"pageid\" INTEGER NOT NULL REFERENCES mpages(id) " + "ON DELETE CASCADE,\n" + " \"id\" INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL\n" + ");\n" + "\n" + "CREATE TABLE \"keys\" (\n" + " \"bits\" INTEGER NOT NULL,\n" + " \"key\" TEXT NOT NULL,\n" + " \"pageid\" INTEGER NOT NULL REFERENCES mpages(id) " + "ON DELETE CASCADE,\n" + " \"id\" INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL\n" + ");\n" + "\n" + "CREATE INDEX \"key_index\" ON keys (key);\n"; + + if (SQLITE_OK != sqlite3_exec(db, sql, NULL, NULL, NULL)) { + exitcode = (int)MANDOCLEVEL_SYSERR; + say(file, "%s", sqlite3_errmsg(db)); + return(0); + } - ofile_dirbuild(buf, sec, arch, src_form, of); - continue; - } +prepare_statements: + SQL_EXEC("PRAGMA foreign_keys = ON"); + sql = "DELETE FROM mpages where file=?"; + sqlite3_prepare_v2(db, sql, -1, &stmts[STMT_DELETE_PAGE], NULL); + sql = "INSERT INTO mpages " + "(file,sec,arch,desc,form) VALUES (?,?,?,?,?)"; + sqlite3_prepare_v2(db, sql, -1, &stmts[STMT_INSERT_PAGE], NULL); + sql = "INSERT INTO mlinks " + "(file,sec,arch,name,pageid) VALUES (?,?,?,?,?)"; + sqlite3_prepare_v2(db, sql, -1, &stmts[STMT_INSERT_LINK], NULL); + sql = "INSERT INTO keys " + "(bits,key,pageid) VALUES (?,?,?)"; + sqlite3_prepare_v2(db, sql, -1, &stmts[STMT_INSERT_KEY], NULL); - if (DT_REG != dp->d_type) { - if (warnings) - fprintf(stderr, - "%s/%s: not a regular file\n", - dir, fn); - continue; - } - if (!strcmp(MANDOC_DB, fn) || !strcmp(MANDOC_IDX, fn)) - continue; - if ('\0' == *psec) { - if (warnings) - fprintf(stderr, - "%s/%s: file outside section\n", - dir, fn); - if (0 == use_all) - continue; - } + /* + * When opening a new database, we can turn off + * synchronous mode for much better performance. + */ - /* - * By default, skip files where the file name suffix - * does not agree with the section directory - * they are located in. - */ + if (real) + SQL_EXEC("PRAGMA synchronous = OFF"); - suffix = strrchr(fn, '.'); - if (NULL == suffix) { - if (warnings) - fprintf(stderr, - "%s/%s: no filename suffix\n", - dir, fn); - if (0 == use_all) - continue; - } else if ((MANDOC_SRC & src_form && - strcmp(suffix + 1, psec)) || - (MANDOC_FORM & src_form && - strcmp(suffix + 1, "0"))) { - if (warnings) - fprintf(stderr, - "%s/%s: wrong filename suffix\n", - dir, fn); - if (0 == use_all) - continue; - if ('0' == suffix[1]) - src_form |= MANDOC_FORM; - else if ('1' <= suffix[1] && '9' >= suffix[1]) - src_form |= MANDOC_SRC; - } + return(1); +} - /* - * Skip formatted manuals if a source version is - * available. Ignore the age: it is very unlikely - * that people install newer formatted base manuals - * when they used to have source manuals before, - * and in ports, old manuals get removed on update. - */ - if (0 == use_all && MANDOC_FORM & src_form && - '\0' != *psec) { - buf[0] = '\0'; - strlcat(buf, dir, PATH_MAX); - p = strrchr(buf, '/'); - if ('\0' != *parch && NULL != p) - for (p--; p > buf; p--) - if ('/' == *p) - break; - if (NULL == p) - p = buf; - else - p++; - if (0 == strncmp("cat", p, 3)) - memcpy(p, "man", 3); - strlcat(buf, "/", PATH_MAX); - sz = strlcat(buf, fn, PATH_MAX); - if (sz >= PATH_MAX) { - if (warnings) fprintf(stderr, - "%s/%s: path too long\n", - dir, fn); - continue; - } - q = strrchr(buf, '.'); - if (NULL != q && p < q++) { - *q = '\0'; - sz = strlcat(buf, psec, PATH_MAX); - if (sz >= PATH_MAX) { - if (warnings) fprintf(stderr, - "%s/%s: path too long\n", - dir, fn); - continue; - } - if (0 == access(buf, R_OK)) - continue; - } - } +static void * +hash_halloc(size_t sz, void *arg) +{ - buf[0] = '\0'; - assert('.' == dir[0]); - if ('/' == dir[1]) { - strlcat(buf, dir + 2, PATH_MAX); - strlcat(buf, "/", PATH_MAX); - } - sz = strlcat(buf, fn, PATH_MAX); - if (sz >= PATH_MAX) { - if (warnings) fprintf(stderr, - "%s/%s: path too long\n", dir, fn); - continue; - } + return(mandoc_calloc(sz, 1)); +} - nof = mandoc_calloc(1, sizeof(struct of)); - nof->fname = mandoc_strdup(buf); - nof->sec = mandoc_strdup(psec); - nof->arch = mandoc_strdup(parch); - nof->src_form = src_form; +static void * +hash_alloc(size_t sz, void *arg) +{ - /* - * Remember the file name without the extension, - * to be used as the page title in the database. - */ + return(mandoc_malloc(sz)); +} - if (NULL != suffix) - *suffix = '\0'; - nof->title = mandoc_strdup(fn); +static void +hash_free(void *p, size_t sz, void *arg) +{ - /* - * Add the structure to the list. - */ + free(p); +} - if (verb > 1) - printf("%s: scheduling\n", buf); +static int +set_basedir(const char *targetdir) +{ + static char startdir[PATH_MAX]; + static int fd; - if (NULL == *of) { - *of = nof; - (*of)->first = nof; - } else { - nof->first = (*of)->first; - (*of)->next = nof; - *of = nof; + /* + * Remember where we started by keeping a fd open to the origin + * path component: throughout this utility, we chdir() a lot to + * handle relative paths, and by doing this, we can return to + * the starting point. + */ + if ('\0' == *startdir) { + if (NULL == getcwd(startdir, PATH_MAX)) { + exitcode = (int)MANDOCLEVEL_SYSERR; + if (NULL != targetdir) + say(".", NULL); + return(0); + } + if (-1 == (fd = open(startdir, O_RDONLY, 0))) { + exitcode = (int)MANDOCLEVEL_SYSERR; + say(startdir, NULL); + return(0); + } + if (NULL == targetdir) + targetdir = startdir; + } else { + if (-1 == fd) + return(0); + if (-1 == fchdir(fd)) { + close(fd); + basedir[0] = '\0'; + exitcode = (int)MANDOCLEVEL_SYSERR; + say(startdir, NULL); + return(0); + } + if (NULL == targetdir) { + close(fd); + return(1); } } - - closedir(d); + if (NULL == realpath(targetdir, basedir)) { + basedir[0] = '\0'; + exitcode = (int)MANDOCLEVEL_BADARG; + say(targetdir, NULL); + return(0); + } else if (-1 == chdir(basedir)) { + exitcode = (int)MANDOCLEVEL_BADARG; + say("", NULL); + return(0); + } + return(1); } static void -ofile_free(struct of *of) +say(const char *file, const char *format, ...) { - struct of *nof; - - if (NULL != of) - of = of->first; - - while (NULL != of) { - nof = of->next; - free(of->fname); - free(of->sec); - free(of->arch); - free(of->title); - free(of); - of = nof; + va_list ap; + + if ('\0' != *basedir) + fprintf(stderr, "%s", basedir); + if ('\0' != *basedir && '\0' != *file) + fputs("//", stderr); + if ('\0' != *file) + fprintf(stderr, "%s", file); + fputs(": ", stderr); + + if (NULL == format) { + perror(NULL); + return; } + + va_start(ap, format); + vfprintf(stderr, format, ap); + va_end(ap); + + fputc('\n', stderr); } diff --git a/usr.bin/mandoc/mandocdb.h b/usr.bin/mandoc/mandocdb.h deleted file mode 100644 index af61f3ff89f..00000000000 --- a/usr.bin/mandoc/mandocdb.h +++ /dev/null @@ -1,62 +0,0 @@ -/* $Id: mandocdb.h,v 1.6 2012/01/09 01:59:08 schwarze Exp $ */ -/* - * Copyright (c) 2011 Kristaps Dzonsons <kristaps@bsd.lv> - * - * Permission to use, copy, modify, and distribute this software for any - * purpose with or without fee is hereby granted, provided that the above - * copyright notice and this permission notice appear in all copies. - * - * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - */ -#ifndef MANDOCDB_H -#define MANDOCDB_H - -#define MANDOC_DB "mandoc.db" -#define MANDOC_IDX "mandoc.index" - -#define TYPE_An 0x0000000000000001ULL -#define TYPE_Ar 0x0000000000000002ULL -#define TYPE_At 0x0000000000000004ULL -#define TYPE_Bsx 0x0000000000000008ULL -#define TYPE_Bx 0x0000000000000010ULL -#define TYPE_Cd 0x0000000000000020ULL -#define TYPE_Cm 0x0000000000000040ULL -#define TYPE_Dv 0x0000000000000080ULL -#define TYPE_Dx 0x0000000000000100ULL -#define TYPE_Em 0x0000000000000200ULL -#define TYPE_Er 0x0000000000000400ULL -#define TYPE_Ev 0x0000000000000800ULL -#define TYPE_Fa 0x0000000000001000ULL -#define TYPE_Fl 0x0000000000002000ULL -#define TYPE_Fn 0x0000000000004000ULL -#define TYPE_Ft 0x0000000000008000ULL -#define TYPE_Fx 0x0000000000010000ULL -#define TYPE_Ic 0x0000000000020000ULL -#define TYPE_In 0x0000000000040000ULL -#define TYPE_Lb 0x0000000000080000ULL -#define TYPE_Li 0x0000000000100000ULL -#define TYPE_Lk 0x0000000000200000ULL -#define TYPE_Ms 0x0000000000400000ULL -#define TYPE_Mt 0x0000000000800000ULL -#define TYPE_Nd 0x0000000001000000ULL -#define TYPE_Nm 0x0000000002000000ULL -#define TYPE_Nx 0x0000000004000000ULL -#define TYPE_Ox 0x0000000008000000ULL -#define TYPE_Pa 0x0000000010000000ULL -#define TYPE_Rs 0x0000000020000000ULL -#define TYPE_Sh 0x0000000040000000ULL -#define TYPE_Ss 0x0000000080000000ULL -#define TYPE_St 0x0000000100000000ULL -#define TYPE_Sy 0x0000000200000000ULL -#define TYPE_Tn 0x0000000400000000ULL -#define TYPE_Va 0x0000000800000000ULL -#define TYPE_Vt 0x0000001000000000ULL -#define TYPE_Xr 0x0000002000000000ULL - -#endif /*!MANDOCDB_H */ diff --git a/usr.bin/mandoc/mansearch.c b/usr.bin/mandoc/mansearch.c new file mode 100644 index 00000000000..222fd58e43e --- /dev/null +++ b/usr.bin/mandoc/mansearch.c @@ -0,0 +1,565 @@ +/* $Id: mansearch.c,v 1.1 2013/12/31 00:40:19 schwarze Exp $ */ +/* + * Copyright (c) 2012 Kristaps Dzonsons <kristaps@bsd.lv> + * Copyright (c) 2013 Ingo Schwarze <schwarze@openbsd.org> + * + * Permission to use, copy, modify, and distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ +#include <assert.h> +#include <fcntl.h> +#include <getopt.h> +#include <limits.h> +#include <regex.h> +#include <stdio.h> +#include <stdint.h> +#include <stddef.h> +#include <stdlib.h> +#include <string.h> +#include <unistd.h> + +#include <ohash.h> +#include <sqlite3.h> + +#include "mandoc.h" +#include "manpath.h" +#include "mansearch.h" + +#define SQL_BIND_TEXT(_db, _s, _i, _v) \ + do { if (SQLITE_OK != sqlite3_bind_text \ + ((_s), (_i)++, (_v), -1, SQLITE_STATIC)) \ + fprintf(stderr, "%s\n", sqlite3_errmsg((_db))); \ + } while (0) +#define SQL_BIND_INT64(_db, _s, _i, _v) \ + do { if (SQLITE_OK != sqlite3_bind_int64 \ + ((_s), (_i)++, (_v))) \ + fprintf(stderr, "%s\n", sqlite3_errmsg((_db))); \ + } while (0) +#define SQL_BIND_BLOB(_db, _s, _i, _v) \ + do { if (SQLITE_OK != sqlite3_bind_blob \ + ((_s), (_i)++, (&_v), sizeof(_v), SQLITE_STATIC)) \ + fprintf(stderr, "%s\n", sqlite3_errmsg((_db))); \ + } while (0) + +struct expr { + uint64_t bits; /* type-mask */ + const char *substr; /* to search for, if applicable */ + regex_t regexp; /* compiled regexp, if applicable */ + struct expr *next; /* next in sequence */ +}; + +struct match { + uint64_t id; /* identifier in database */ + char *file; /* relative filepath of manpage */ + char *desc; /* description of manpage */ + int form; /* 0 == catpage */ +}; + +struct type { + uint64_t bits; + const char *name; +}; + +static const struct type types[] = { + { TYPE_An, "An" }, + { TYPE_Ar, "Ar" }, + { TYPE_At, "At" }, + { TYPE_Bsx, "Bsx" }, + { TYPE_Bx, "Bx" }, + { TYPE_Cd, "Cd" }, + { TYPE_Cm, "Cm" }, + { TYPE_Dv, "Dv" }, + { TYPE_Dx, "Dx" }, + { TYPE_Em, "Em" }, + { TYPE_Er, "Er" }, + { TYPE_Ev, "Ev" }, + { TYPE_Fa, "Fa" }, + { TYPE_Fl, "Fl" }, + { TYPE_Fn, "Fn" }, + { TYPE_Fn, "Fo" }, + { TYPE_Ft, "Ft" }, + { TYPE_Fx, "Fx" }, + { TYPE_Ic, "Ic" }, + { TYPE_In, "In" }, + { TYPE_Lb, "Lb" }, + { TYPE_Li, "Li" }, + { TYPE_Lk, "Lk" }, + { TYPE_Ms, "Ms" }, + { TYPE_Mt, "Mt" }, + { TYPE_Nd, "Nd" }, + { TYPE_Nm, "Nm" }, + { TYPE_Nx, "Nx" }, + { TYPE_Ox, "Ox" }, + { TYPE_Pa, "Pa" }, + { TYPE_Rs, "Rs" }, + { TYPE_Sh, "Sh" }, + { TYPE_Ss, "Ss" }, + { TYPE_St, "St" }, + { TYPE_Sy, "Sy" }, + { TYPE_Tn, "Tn" }, + { TYPE_Va, "Va" }, + { TYPE_Va, "Vt" }, + { TYPE_Xr, "Xr" }, + { ~0ULL, "any" }, + { 0ULL, NULL } +}; + +static void *hash_alloc(size_t, void *); +static void hash_free(void *, size_t, void *); +static void *hash_halloc(size_t, void *); +static struct expr *exprcomp(const struct mansearch *, + int, char *[]); +static void exprfree(struct expr *); +static struct expr *exprterm(const struct mansearch *, char *, int); +static void sql_match(sqlite3_context *context, + int argc, sqlite3_value **argv); +static void sql_regexp(sqlite3_context *context, + int argc, sqlite3_value **argv); +static char *sql_statement(const struct expr *, + const char *, const char *); + +int +mansearch(const struct mansearch *search, + const struct manpaths *paths, + int argc, char *argv[], + struct manpage **res, size_t *sz) +{ + int fd, rc, c; + int64_t id; + char buf[PATH_MAX]; + char *sql, *newnames; + const char *oldnames, *sep1, *name, *sec, *sep2, *arch; + struct manpage *mpage; + struct expr *e, *ep; + sqlite3 *db; + sqlite3_stmt *s; + struct match *mp; + struct ohash_info info; + struct ohash htab; + unsigned int idx; + size_t i, j, cur, maxres; + + memset(&info, 0, sizeof(struct ohash_info)); + + info.halloc = hash_halloc; + info.alloc = hash_alloc; + info.hfree = hash_free; + info.key_offset = offsetof(struct match, id); + + *sz = cur = maxres = 0; + sql = NULL; + *res = NULL; + fd = -1; + e = NULL; + rc = 0; + + if (0 == argc) + goto out; + if (NULL == (e = exprcomp(search, argc, argv))) + goto out; + + /* + * Save a descriptor to the current working directory. + * Since pathnames in the "paths" variable might be relative, + * and we'll be chdir()ing into them, we need to keep a handle + * on our current directory from which to start the chdir(). + */ + + if (NULL == getcwd(buf, PATH_MAX)) { + perror(NULL); + goto out; + } else if (-1 == (fd = open(buf, O_RDONLY, 0))) { + perror(buf); + goto out; + } + + sql = sql_statement(e, search->arch, search->sec); + + /* + * Loop over the directories (containing databases) for us to + * search. + * Don't let missing/bad databases/directories phase us. + * In each, try to open the resident database and, if it opens, + * scan it for our match expression. + */ + + for (i = 0; i < paths->sz; i++) { + if (-1 == fchdir(fd)) { + perror(buf); + free(*res); + break; + } else if (-1 == chdir(paths->paths[i])) { + perror(paths->paths[i]); + continue; + } + + c = sqlite3_open_v2 + (MANDOC_DB, &db, + SQLITE_OPEN_READONLY, NULL); + + if (SQLITE_OK != c) { + perror(MANDOC_DB); + sqlite3_close(db); + continue; + } + + /* + * Define the SQL functions for substring + * and regular expression matching. + */ + + c = sqlite3_create_function(db, "match", 2, + SQLITE_ANY, NULL, sql_match, NULL, NULL); + assert(SQLITE_OK == c); + c = sqlite3_create_function(db, "regexp", 2, + SQLITE_ANY, NULL, sql_regexp, NULL, NULL); + assert(SQLITE_OK == c); + + j = 1; + c = sqlite3_prepare_v2(db, sql, -1, &s, NULL); + if (SQLITE_OK != c) + fprintf(stderr, "%s\n", sqlite3_errmsg(db)); + + if (NULL != search->arch) + SQL_BIND_TEXT(db, s, j, search->arch); + if (NULL != search->sec) + SQL_BIND_TEXT(db, s, j, search->sec); + + for (ep = e; NULL != ep; ep = ep->next) { + if (NULL == ep->substr) { + SQL_BIND_BLOB(db, s, j, ep->regexp); + } else + SQL_BIND_TEXT(db, s, j, ep->substr); + SQL_BIND_INT64(db, s, j, ep->bits); + } + + memset(&htab, 0, sizeof(struct ohash)); + ohash_init(&htab, 4, &info); + + /* + * Hash each entry on its [unique] document identifier. + * This is a uint64_t. + * Instead of using a hash function, simply convert the + * uint64_t to a uint32_t, the hash value's type. + * This gives good performance and preserves the + * distribution of buckets in the table. + */ + while (SQLITE_ROW == (c = sqlite3_step(s))) { + id = sqlite3_column_int64(s, 0); + idx = ohash_lookup_memory + (&htab, (char *)&id, + sizeof(uint64_t), (uint32_t)id); + + if (NULL != ohash_find(&htab, idx)) + continue; + + mp = mandoc_calloc(1, sizeof(struct match)); + mp->id = id; + mp->file = mandoc_strdup + ((char *)sqlite3_column_text(s, 3)); + mp->desc = mandoc_strdup + ((char *)sqlite3_column_text(s, 4)); + mp->form = sqlite3_column_int(s, 5); + ohash_insert(&htab, idx, mp); + } + + if (SQLITE_DONE != c) + fprintf(stderr, "%s\n", sqlite3_errmsg(db)); + + sqlite3_finalize(s); + + c = sqlite3_prepare_v2(db, + "SELECT * FROM mlinks WHERE pageid=?", + -1, &s, NULL); + if (SQLITE_OK != c) + fprintf(stderr, "%s\n", sqlite3_errmsg(db)); + + for (mp = ohash_first(&htab, &idx); + NULL != mp; + mp = ohash_next(&htab, &idx)) { + if (cur + 1 > maxres) { + maxres += 1024; + *res = mandoc_realloc + (*res, maxres * sizeof(struct manpage)); + } + mpage = *res + cur; + if (-1 == asprintf(&mpage->file, "%s/%s", + paths->paths[i], mp->file)) { + perror(0); + exit((int)MANDOCLEVEL_SYSERR); + } + mpage->names = NULL; + mpage->desc = mp->desc; + mpage->form = mp->form; + + j = 1; + SQL_BIND_INT64(db, s, j, mp->id); + while (SQLITE_ROW == (c = sqlite3_step(s))) { + if (NULL == mpage->names) { + oldnames = ""; + sep1 = ""; + } else { + oldnames = mpage->names; + sep1 = ", "; + } + sec = sqlite3_column_text(s, 1); + arch = sqlite3_column_text(s, 2); + name = sqlite3_column_text(s, 3); + sep2 = '\0' == *arch ? "" : "/"; + if (-1 == asprintf(&newnames, + "%s%s%s(%s%s%s)", oldnames, sep1, + name, sec, sep2, arch)) { + perror(0); + exit((int)MANDOCLEVEL_SYSERR); + } + free(mpage->names); + mpage->names = newnames; + } + if (SQLITE_DONE != c) + fprintf(stderr, "%s\n", sqlite3_errmsg(db)); + sqlite3_reset(s); + + free(mp->file); + free(mp); + cur++; + } + + sqlite3_finalize(s); + sqlite3_close(db); + ohash_delete(&htab); + } + rc = 1; +out: + exprfree(e); + if (-1 != fd) + close(fd); + free(sql); + *sz = cur; + return(rc); +} + +/* + * Implement substring match as an application-defined SQL function. + * Using the SQL LIKE or GLOB operators instead would be a bad idea + * because that would require escaping metacharacters in the string + * being searched for. + */ +static void +sql_match(sqlite3_context *context, int argc, sqlite3_value **argv) +{ + + assert(2 == argc); + sqlite3_result_int(context, NULL != strcasestr( + (const char *)sqlite3_value_text(argv[1]), + (const char *)sqlite3_value_text(argv[0]))); +} + +/* + * Implement regular expression match + * as an application-defined SQL function. + */ +static void +sql_regexp(sqlite3_context *context, int argc, sqlite3_value **argv) +{ + + assert(2 == argc); + sqlite3_result_int(context, !regexec( + (regex_t *)sqlite3_value_blob(argv[0]), + (const char *)sqlite3_value_text(argv[1]), + 0, NULL, 0)); +} + +/* + * Prepare the search SQL statement. + * We search for any of the words specified in our match expression. + * We filter the per-doc AND expressions when collecting results. + */ +static char * +sql_statement(const struct expr *e, const char *arch, const char *sec) +{ + char *sql; + const char *substr = "(key MATCH ? AND bits & ?)"; + const char *regexp = "(key REGEXP ? AND bits & ?)"; + const char *andarch = "arch = ? AND "; + const char *andsec = "sec = ? AND "; + size_t substrsz; + size_t regexpsz; + size_t sz; + + sql = mandoc_strdup + ("SELECT pageid,bits,key,file,desc,form,sec,arch " + "FROM keys " + "INNER JOIN mpages ON mpages.id=keys.pageid " + "WHERE "); + sz = strlen(sql); + substrsz = strlen(substr); + regexpsz = strlen(regexp); + + if (NULL != arch) { + sz += strlen(andarch) + 1; + sql = mandoc_realloc(sql, sz); + strlcat(sql, andarch, sz); + } + + if (NULL != sec) { + sz += strlen(andsec) + 1; + sql = mandoc_realloc(sql, sz); + strlcat(sql, andsec, sz); + } + + sz += 2; + sql = mandoc_realloc(sql, sz); + strlcat(sql, "(", sz); + + for ( ; NULL != e; e = e->next) { + sz += (NULL == e->substr ? regexpsz : substrsz) + + (NULL == e->next ? 3 : 5); + sql = mandoc_realloc(sql, sz); + strlcat(sql, NULL == e->substr ? regexp : substr, sz); + strlcat(sql, NULL == e->next ? ");" : " OR ", sz); + } + + return(sql); +} + +/* + * Compile a set of string tokens into an expression. + * Tokens in "argv" are assumed to be individual expression atoms (e.g., + * "(", "foo=bar", etc.). + */ +static struct expr * +exprcomp(const struct mansearch *search, int argc, char *argv[]) +{ + int i, cs; + struct expr *first, *next, *cur; + + first = cur = NULL; + + for (i = 0; i < argc; i++) { + if (0 == strcmp("-i", argv[i])) { + if (++i >= argc) + return(NULL); + cs = 0; + } else + cs = 1; + next = exprterm(search, argv[i], cs); + if (NULL == next) { + exprfree(first); + return(NULL); + } + if (NULL != first) { + cur->next = next; + cur = next; + } else + cur = first = next; + } + + return(first); +} + +static struct expr * +exprterm(const struct mansearch *search, char *buf, int cs) +{ + struct expr *e; + char *key, *v; + size_t i; + + if ('\0' == *buf) + return(NULL); + + e = mandoc_calloc(1, sizeof(struct expr)); + + /*"whatis" mode uses an opaque string and default fields. */ + + if (MANSEARCH_WHATIS & search->flags) { + e->substr = buf; + e->bits = search->deftype; + return(e); + } + + /* + * If no =~ is specified, search with equality over names and + * descriptions. + * If =~ begins the phrase, use name and description fields. + */ + + if (NULL == (v = strpbrk(buf, "=~"))) { + e->substr = buf; + e->bits = search->deftype; + return(e); + } else if (v == buf) + e->bits = search->deftype; + + if ('~' == *v++) { + if (regcomp(&e->regexp, v, + REG_EXTENDED | REG_NOSUB | (cs ? 0 : REG_ICASE))) { + free(e); + return(NULL); + } + } else + e->substr = v; + v[-1] = '\0'; + + /* + * Parse out all possible fields. + * If the field doesn't resolve, bail. + */ + + while (NULL != (key = strsep(&buf, ","))) { + if ('\0' == *key) + continue; + i = 0; + while (types[i].bits && + strcasecmp(types[i].name, key)) + i++; + if (0 == types[i].bits) { + free(e); + return(NULL); + } + e->bits |= types[i].bits; + } + + return(e); +} + +static void +exprfree(struct expr *p) +{ + struct expr *pp; + + while (NULL != p) { + pp = p->next; + free(p); + p = pp; + } +} + +static void * +hash_halloc(size_t sz, void *arg) +{ + + return(mandoc_calloc(sz, 1)); +} + +static void * +hash_alloc(size_t sz, void *arg) +{ + + return(mandoc_malloc(sz)); +} + +static void +hash_free(void *p, size_t sz, void *arg) +{ + + free(p); +} diff --git a/usr.bin/mandoc/mansearch.h b/usr.bin/mandoc/mansearch.h new file mode 100644 index 00000000000..bcb06962ee1 --- /dev/null +++ b/usr.bin/mandoc/mansearch.h @@ -0,0 +1,87 @@ +/* $Id: mansearch.h,v 1.1 2013/12/31 00:40:19 schwarze Exp $ */ +/* + * Copyright (c) 2012 Kristaps Dzonsons <kristaps@bsd.lv> + * + * Permission to use, copy, modify, and distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ +#ifndef MANSEARCH_H +#define MANSEARCH_H + +#define MANDOC_DB "mandoc.db" + +#define TYPE_An 0x0000000000000001ULL +#define TYPE_Ar 0x0000000000000002ULL +#define TYPE_At 0x0000000000000004ULL +#define TYPE_Bsx 0x0000000000000008ULL +#define TYPE_Bx 0x0000000000000010ULL +#define TYPE_Cd 0x0000000000000020ULL +#define TYPE_Cm 0x0000000000000040ULL +#define TYPE_Dv 0x0000000000000080ULL +#define TYPE_Dx 0x0000000000000100ULL +#define TYPE_Em 0x0000000000000200ULL +#define TYPE_Er 0x0000000000000400ULL +#define TYPE_Ev 0x0000000000000800ULL +#define TYPE_Fa 0x0000000000001000ULL +#define TYPE_Fl 0x0000000000002000ULL +#define TYPE_Fn 0x0000000000004000ULL +#define TYPE_Ft 0x0000000000008000ULL +#define TYPE_Fx 0x0000000000010000ULL +#define TYPE_Ic 0x0000000000020000ULL +#define TYPE_In 0x0000000000040000ULL +#define TYPE_Lb 0x0000000000080000ULL +#define TYPE_Li 0x0000000000100000ULL +#define TYPE_Lk 0x0000000000200000ULL +#define TYPE_Ms 0x0000000000400000ULL +#define TYPE_Mt 0x0000000000800000ULL +#define TYPE_Nd 0x0000000001000000ULL +#define TYPE_Nm 0x0000000002000000ULL +#define TYPE_Nx 0x0000000004000000ULL +#define TYPE_Ox 0x0000000008000000ULL +#define TYPE_Pa 0x0000000010000000ULL +#define TYPE_Rs 0x0000000020000000ULL +#define TYPE_Sh 0x0000000040000000ULL +#define TYPE_Ss 0x0000000080000000ULL +#define TYPE_St 0x0000000100000000ULL +#define TYPE_Sy 0x0000000200000000ULL +#define TYPE_Tn 0x0000000400000000ULL +#define TYPE_Va 0x0000000800000000ULL +#define TYPE_Vt 0x0000001000000000ULL +#define TYPE_Xr 0x0000002000000000ULL + +__BEGIN_DECLS + +struct manpage { + char *file; /* to be prefixed by manpath */ + char *names; /* a list of names with sections */ + char *desc; /* description of manpage */ + int form; /* 0 == catpage */ +}; + +struct mansearch { + const char *arch; /* architecture/NULL */ + const char *sec; /* mansection/NULL */ + uint64_t deftype; /* type if no key */ + int flags; +#define MANSEARCH_WHATIS 0x01 /* whatis mode: equality, no key */ +}; + +int mansearch(const struct mansearch *cfg, /* options */ + const struct manpaths *paths, /* manpaths */ + int argc, /* size of argv */ + char *argv[], /* search terms */ + struct manpage **res, /* results */ + size_t *ressz); /* results returned */ + +__END_DECLS + +#endif /*!MANSEARCH_H*/ |