Remove compaction and indexing from ldapd. It is better done by a separate

process now that the btree can pick up the changes automatically.

6 files changed, 8 insertions, 425 deletions

diff --git a/usr.sbin/ldapd/Makefile b/usr.sbin/ldapd/Makefile
index c67863c181e..661d911b637 100644
--- a/usr.sbin/ldapd/Makefile
+++ b/usr.sbin/ldapd/Makefile
@@ -1,11 +1,11 @@
-#	$OpenBSD: Makefile,v 1.4 2010/06/06 17:49:56 miod Exp $
+#	$OpenBSD: Makefile,v 1.5 2010/06/23 12:40:19 martinh Exp $
 
 PROG=		ldapd
 MAN=		ldapd.8 ldapd.conf.5
 SRCS=		ber.c log.c control.c \
 		util.c ldapd.c ldape.c conn.c attributes.c namespace.c \
 		btree.c filter.c search.c parse.y \
-		auth.c modify.c index.c ssl.c ssl_privsep.c compact.c \
+		auth.c modify.c index.c ssl.c ssl_privsep.c \
 		validate.c uuid.c
 
 LDADD=		-levent -lssl -lcrypto -lz -lutil
diff --git a/usr.sbin/ldapd/compact.c b/usr.sbin/ldapd/compact.c
deleted file mode 100644
index 8afd55e0ff8..00000000000
--- a/usr.sbin/ldapd/compact.c
+++ /dev/null
@@ -1,185 +0,0 @@
-/*	$OpenBSD: compact.c,v 1.2 2010/05/31 18:29:04 martinh Exp $ */
-
-/*
- * Copyright (c) 2010 Martin Hedenfalk <martin@bzero.se>
- *
- * Permission to use, copy, modify, and distribute this software for any
- * purpose with or without fee is hereby granted, provided that the above
- * copyright notice and this permission notice appear in all copies.
- *
- * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
- * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
- * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
- * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
- * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
- * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
- * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
- */
-
-#include <sys/queue.h>
-#include <sys/types.h>
-#include <sys/wait.h>
-
-#include <assert.h>
-#include <signal.h>
-#include <stdlib.h>
-#include <unistd.h>
-
-#include "ldapd.h"
-
-static int		 continue_compaction(struct ctl_conn *c);
-static void		 stop_compaction(struct ctl_conn *c);
-
-void
-check_compaction(pid_t pid, int status)
-{
-	struct ctl_conn		*c;
-
-	if ((c = control_connbypid(pid)) == NULL)
-		return;
-
-	if (WIFEXITED(status)) {
-		log_debug("compaction process %d exited with status %d",
-		    pid, WEXITSTATUS(status));
-		if (WEXITSTATUS(status) == 0) {
-			control_report_compaction(c, 0);
-			continue_compaction(c);
-			return;
-		}
-	} else if (WIFSIGNALED(status))
-		log_warn("compaction process %d exited due to signal %d",
-		    pid, WTERMSIG(status));
-	else
-		log_debug("compaction process %d exited", pid);
-
-	/* Compaction failed, no need to continue (disk might be full).
-	 */
-	control_report_compaction(c, 1);
-	if (c->ns != NULL)
-		c->ns->compacting = 0;
-	c->ns = NULL;
-}
-
-static pid_t
-compact(struct btree *bt)
-{
-	pid_t		 pid;
-
-	pid = fork();
-	if (pid < 0) {
-		log_warn("compaction monitor: fork");
-		return -1;
-	}
-	if (pid > 0)
-		return pid;
-
-	signal(SIGINT, SIG_IGN);
-	signal(SIGTERM, SIG_IGN);
-	signal(SIGCHLD, SIG_IGN);
-	signal(SIGHUP, SIG_IGN);
-	signal(SIGPIPE, SIG_IGN);
-
-	setproctitle("compacting %s", btree_get_path(bt));
-	_exit(btree_compact(bt));
-}
-
-static int
-continue_compaction(struct ctl_conn *c)
-{
-again:
-	switch (c->state) {
-	case COMPACT_DATA:
-		log_info("compacting namespace %s (entries)", c->ns->suffix);
-		if (namespace_commit(c->ns) != 0) {
-			control_report_compaction(c, 1);
-			goto fail;
-		}
-		c->ns->compacting = 1;
-		if ((c->pid = compact(c->ns->data_db)) < 0)
-			goto fail;
-		c->state = COMPACT_INDX;
-		break;
-	case COMPACT_INDX:
-		if (namespace_reopen_data(c->ns) != 0)
-			goto fail;
-		log_info("compacting namespace %s (index)", c->ns->suffix);
-		if ((c->pid = compact(c->ns->indx_db)) < 0)
-			goto fail;
-		c->state = COMPACT_DONE;
-		break;
-	case COMPACT_DONE:
-		if (namespace_reopen_indx(c->ns) != 0)
-			goto fail;
-		c->ns->compacting = 0;
-		c->pid = 0;
-		namespace_queue_schedule(c->ns);
-
-		if (c->all) {
-			/* Proceed with the next namespace that isn't
-			 * already being compacted or indexed.
-			 */
-			while ((c->ns = TAILQ_NEXT(c->ns, next)) != NULL) {
-				if (!c->ns->compacting && !c->ns->indexing)
-					break;
-			}
-		} else
-			c->ns = NULL;
-
-		if (c->ns == NULL) {
-			control_end(c);
-			return 0;
-		}
-		c->state = COMPACT_DATA;
-		goto again;
-		break;
-	default:
-		assert(0);
-	}
-
-	return 0;
-
-fail:
-	control_end(c);
-	namespace_remove(c->ns);
-	c->ns = NULL;
-	return -1;
-}
-
-/* Run compaction for the given namespace, or all namespaces if ns is NULL.
- *
- * Returns 0 on success, or -1 on error.
- */
-int
-run_compaction(struct ctl_conn *c, struct namespace *ns)
-{
-	if (ns == NULL) {
-		c->all = 1;
-		c->ns = TAILQ_FIRST(&conf->namespaces);
-	} else {
-		c->all = 0;
-		c->ns = ns;
-	}
-
-	c->closecb = stop_compaction;
-	c->state = COMPACT_DATA;
-	return continue_compaction(c);
-}
-
-static void
-stop_compaction(struct ctl_conn *c)
-{
-	if (c->pid != 0) {
-		log_info("stopping compaction process %i", c->pid);
-		if (kill(c->pid, SIGKILL) != 0)
-			log_warn("failed to stop compaction process");
-		c->pid = 0;
-	}
-
-	if (c->ns != NULL) {
-		log_info("stopped compacting namespace %s", c->ns->suffix);
-		c->ns->compacting = 0;
-		namespace_queue_schedule(c->ns);
-	}
-	c->ns = NULL;
-}
-
diff --git a/usr.sbin/ldapd/control.c b/usr.sbin/ldapd/control.c
index db77ffdda43..5652506167e 100644
--- a/usr.sbin/ldapd/control.c
+++ b/usr.sbin/ldapd/control.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: control.c,v 1.1 2010/05/31 17:36:31 martinh Exp $	*/
+/*	$OpenBSD: control.c,v 1.2 2010/06/23 12:40:19 martinh Exp $	*/
 
 /*
  * Copyright (c) 2010 Martin Hedenfalk <martin@bzero.se>
@@ -163,18 +163,6 @@ control_connbyfd(int fd)
 	return (c);
 }
 
-struct ctl_conn *
-control_connbypid(pid_t pid)
-{
-	struct ctl_conn	*c;
-
-	for (c = TAILQ_FIRST(&ctl_conns); c != NULL && c->pid != pid;
-	    c = TAILQ_NEXT(c, entry))
-		;	/* nothing */
-
-	return (c);
-}
-
 void
 control_close(int fd)
 {
@@ -188,9 +176,6 @@ control_close(int fd)
 	msgbuf_clear(&c->iev.ibuf.w);
 	TAILQ_REMOVE(&ctl_conns, c, entry);
 
-	if (c->closecb)
-		c->closecb(c);
-
 	event_del(&c->iev.ev);
 	close(c->iev.ibuf.fd);
 	free(c);
@@ -300,12 +285,6 @@ control_dispatch_imsg(int fd, short event, void *arg)
 
 			log_verbose(verbose);
 			break;
-		case IMSG_CTL_COMPACT:
-			run_compaction(c, NULL);
-			break;
-		case IMSG_CTL_INDEX:
-			run_indexer(c, NULL);
-			break;
 		default:
 			log_debug("control_dispatch_imsg: "
 			    "error handling imsg %d", imsg.hdr.type);
@@ -337,43 +316,3 @@ control_end(struct ctl_conn *c)
 	imsg_event_add(&c->iev);
 }
 
-void
-control_report_compaction(struct ctl_conn *c, int status)
-{
-	struct compaction_status	 cs;
-
-	/* Report compaction status to the requesting control conn.
-	 */
-	bzero(&cs, sizeof(cs));
-	strlcpy(cs.suffix, c->ns->suffix, sizeof(cs.suffix));
-	cs.db = c->state;
-	cs.status = status;
-	imsg_compose(&c->iev.ibuf, IMSG_CTL_COMPACT_STATUS, 0,
-	    c->iev.ibuf.pid, -1, &cs, sizeof(cs));
-
-	imsg_event_add(&c->iev);
-}
-
-
-void
-control_report_indexer(struct ctl_conn *c, int status)
-{
-	const struct btree_stat	*st;
-	struct indexer_status	 is;
-
-	/* Report indexer status to the requesting control conn.
-	 */
-	bzero(&is, sizeof(is));
-	strlcpy(is.suffix, c->ns->suffix, sizeof(is.suffix));
-	st = btree_stat(c->ns->data_db);
-	is.entries = st->entries;
-	is.ncomplete = c->ncomplete;
-	is.status = status;
-	log_debug("reporting indexer status %ju/%ju for ns %s",
-	    (intmax_t)is.ncomplete, (intmax_t)is.entries, c->ns->suffix);
-	imsg_compose(&c->iev.ibuf, IMSG_CTL_INDEX_STATUS, 0,
-	    c->iev.ibuf.pid, -1, &is, sizeof(is));
-
-	imsg_event_add(&c->iev);
-}
-
diff --git a/usr.sbin/ldapd/index.c b/usr.sbin/ldapd/index.c
index 743afa11be7..13e01dd407e 100644
--- a/usr.sbin/ldapd/index.c
+++ b/usr.sbin/ldapd/index.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: index.c,v 1.4 2010/06/11 08:45:06 martinh Exp $ */
+/*	$OpenBSD: index.c,v 1.5 2010/06/23 12:40:19 martinh Exp $ */
 
 /*
  * Copyright (c) 2009 Martin Hedenfalk <martin@bzero.se>
@@ -79,10 +79,7 @@
 
 #include "ldapd.h"
 
-static void	 continue_indexer(int fd, short why, void *arg);
-static void	 stop_indexer(struct ctl_conn *c);
-
-int
+static int
 index_attribute(struct namespace *ns, char *attr, struct btval *dn,
     struct ber_element *a)
 {
@@ -160,7 +157,7 @@ index_rdn(struct namespace *ns, struct btval *dn)
 	return 0;
 }
 
-int
+static int
 unindex_attribute(struct namespace *ns, char *attr, struct btval *dn,
     struct ber_element *a)
 {
@@ -351,142 +348,3 @@ index_to_dn(struct namespace *ns, struct btval *indx, struct btval *dn)
 	return 0;
 }
 
-/* Return the next namespace that isn't already being indexed or compacted.
- */
-static struct namespace *
-next_namespace(struct namespace *ns)
-{
-	if (ns == NULL)
-		ns = TAILQ_FIRST(&conf->namespaces);
-	else
-		ns = TAILQ_NEXT(ns, next);
-
-	do {
-		if (ns == NULL || (!ns->indexing && !ns->compacting))
-			break;
-	} while ((ns = TAILQ_NEXT(ns, next)) != NULL);
-
-	return ns;
-}
-
-static void
-continue_indexer(int fd, short why, void *arg)
-{
-	struct ctl_conn		*c = arg;
-	struct ber_element	*elm;
-	struct btval		 key, val;
-	struct timeval		 tv;
-	int			 i, rc = BT_FAIL;
-
-	if (c->cursor == NULL) {
-		log_info("begin indexing namespace %s", c->ns->suffix);
-		c->ncomplete = 0;
-		c->ns->indexing = 1;
-		c->cursor = btree_cursor_open(c->ns->data_db);
-		if (c->cursor == NULL) {
-			log_warn("failed to open cursor");
-			goto fail;
-		}
-	}
-
-	bzero(&key, sizeof(key));
-	bzero(&val, sizeof(val));
-
-	tv.tv_sec = 0;
-	tv.tv_usec = 0;
-
-	if (namespace_begin(c->ns) != 0) {
-		tv.tv_usec = 50000;
-		evtimer_add(&c->ev, &tv);
-		return;
-	}
-
-	for (i = 0; i < 40; i++) {
-		rc = btree_cursor_get(c->cursor, &key, &val, BT_NEXT);
-		if (rc != BT_SUCCESS)
-			break;
-		if ((elm = namespace_db2ber(c->ns, &val)) == NULL)
-			continue;
-		rc = index_entry(c->ns, &key, elm);
-		ber_free_elements(elm);
-		btval_reset(&key);
-		btval_reset(&val);
-		if (rc != 0)
-			goto fail;
-		++c->ncomplete;
-	}
-
-	if (namespace_commit(c->ns) != 0)
-		goto fail;
-
-	control_report_indexer(c, 0);
-
-	if (rc == BT_NOTFOUND) {
-		log_info("done indexing namespace %s", c->ns->suffix);
-		btree_cursor_close(c->cursor);
-		c->cursor = NULL;
-		c->ns->indexing = 0;
-		control_report_indexer(c, 1);
-
-		if (c->all)
-			c->ns = next_namespace(c->ns);
-		else
-			c->ns = NULL;
-
-		if (c->ns == NULL) {
-			log_info("done indexing all namespaces");
-			return;
-		}
-	} else if (rc != BT_SUCCESS)
-		goto fail;
-
-	evtimer_add(&c->ev, &tv);
-	return;
-
-fail:
-	if (c->ns != NULL)
-		control_report_indexer(c, -1);
-	control_end(c);
-	stop_indexer(c);
-}
-
-/* Run indexing for the given namespace, or all namespaces if ns is NULL.
- *
- * Returns 0 on success, or -1 on error.
- */
-int
-run_indexer(struct ctl_conn *c, struct namespace *ns)
-{
-	if (ns == NULL) {
-		c->all = 1;
-		c->ns = next_namespace(NULL);
-	} else {
-		c->all = 0;
-		c->ns = ns;
-	}
-
-	if (c->ns == NULL) {
-		control_end(c);
-	} else {
-		c->closecb = stop_indexer;
-		evtimer_set(&c->ev, continue_indexer, c);
-		continue_indexer(0, 0, c);
-	}
-
-	return 0;
-}
-
-static void
-stop_indexer(struct ctl_conn *c)
-{
-	if (c->ns != NULL) {
-		log_info("stopped indexing namespace %s", c->ns->suffix);
-		c->ns->indexing = 0;
-	}
-	btree_cursor_close(c->cursor);
-	c->cursor = NULL;
-	c->ns = NULL;
-	c->closecb = NULL;
-	evtimer_del(&c->ev);
-}
-
diff --git a/usr.sbin/ldapd/ldapd.h b/usr.sbin/ldapd/ldapd.h
index a0b62c7f440..82fb87a9e14 100644
--- a/usr.sbin/ldapd/ldapd.h
+++ b/usr.sbin/ldapd/ldapd.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: ldapd.h,v 1.6 2010/06/15 19:30:26 martinh Exp $ */
+/*	$OpenBSD: ldapd.h,v 1.7 2010/06/23 12:40:19 martinh Exp $ */
 
 /*
  * Copyright (c) 2009, 2010 Martin Hedenfalk <martin@bzero.se>
@@ -117,8 +117,6 @@ struct namespace {
 	struct attr_index_list	 indices;
 	unsigned int		 cache_size;
 	unsigned int		 index_cache_size;
-	int			 compacting;	/* true if being compacted */
-	int			 indexing;	/* true if being indexed */
 	struct request_queue	 request_queue;
 	struct event		 ev_queue;
 	unsigned int		 queued_requests;
@@ -410,16 +408,6 @@ struct ctl_conn {
 #define CTL_CONN_NOTIFY		 0x01
 #define CTL_CONN_LOCKED		 0x02		/* restricted mode */
 	struct imsgev		 iev;
-
-	ctl_close_func		 closecb;
-
-	enum comp_state		 state;
-	pid_t			 pid;		/* compaction process */
-	struct namespace	*ns;		/* compacted or indexed */
-	uint64_t		 ncomplete;	/* completed entries */
-	int			 all;		/* 1: traverse all namespaces */
-	struct cursor		*cursor;
-	struct event		 ev;
 };
 TAILQ_HEAD(ctl_connlist, ctl_conn);
 extern  struct ctl_connlist ctl_conns;
@@ -443,11 +431,6 @@ int			 imsg_compose_event(struct imsgev *iev, u_int16_t type,
 			    u_int16_t datalen);
 int			 imsg_event_handle(struct imsgev *iev, short event);
 
-/* compact.c */
-int			 run_compaction(struct ctl_conn *c,
-				struct namespace *ns);
-void			 check_compaction(pid_t pid, int status);
-
 /* conn.c */
 extern struct conn_list	 conn_list;
 struct conn		*conn_by_fd(int fd);
@@ -530,17 +513,12 @@ char			*ldap_now(void);
 
 /* control.c */
 void			 control_init(struct control_sock *);
-struct ctl_conn		*control_connbypid(pid_t);
 void			 control_listen(struct control_sock *);
 void			 control_accept(int, short, void *);
 void			 control_dispatch_imsg(int, short, void *);
 void			 control_imsg_forward(struct imsg *);
 void			 control_cleanup(struct control_sock *);
 void			 control_end(struct ctl_conn *c);
-void			 control_report_compaction(struct ctl_conn *c,
-				int status);
-void			 control_report_indexer( struct ctl_conn *c,
-				int status);
 
 /* filter.c */
 int			 ldap_matches_filter(struct ber_element *root,
@@ -599,18 +577,12 @@ int			 ber2db(struct ber_element *root, struct btval *val,
 struct ber_element	*db2ber(struct btval *val, int compression_level);
 
 /* index.c */
-int			 index_namespace(struct namespace *ns);
 int			 index_entry(struct namespace *ns, struct btval *dn,
 				struct ber_element *elm);
 int			 unindex_entry(struct namespace *ns, struct btval *dn,
 				struct ber_element *elm);
-int			 index_attribute(struct namespace *ns, char *attr,
-				struct btval *dn, struct ber_element *a);
-int			 unindex_attribute(struct namespace *ns, char *attr,
-				struct btval *dn, struct ber_element *a);
 int			 index_to_dn(struct namespace *ns, struct btval *indx,
 				struct btval *dn);
-int			 run_indexer(struct ctl_conn *c, struct namespace *ns);
 
 /* ssl.c */
 void	 ssl_init(void);
diff --git a/usr.sbin/ldapd/ldape.c b/usr.sbin/ldapd/ldape.c
index 912255191f6..a3bcdd50c06 100644
--- a/usr.sbin/ldapd/ldape.c
+++ b/usr.sbin/ldapd/ldape.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: ldape.c,v 1.5 2010/06/15 15:12:54 martinh Exp $ */
+/*	$OpenBSD: ldape.c,v 1.6 2010/06/23 12:40:19 martinh Exp $ */
 
 /*
  * Copyright (c) 2009, 2010 Martin Hedenfalk <martin@bzero.se>
@@ -58,7 +58,6 @@ ldape_sig_handler(int sig, short why, void *data)
 			pid = waitpid(WAIT_ANY, &status, WNOHANG);
 			if (pid <= 0)
 				break;
-			check_compaction(pid, status);
 		}
 		return;
 	}