Make modify and simple auth requests open their own transactions, as search

already does. Trigger a reopen imsg request if either the data or index
databases are compacted. Queue the failed request and try again when the
file is reopened.

Compaction can now be done by a separate process, and ldapd will pick up
the change and reopen the file.

5 files changed, 181 insertions, 137 deletions

diff --git a/usr.sbin/ldapd/auth.c b/usr.sbin/ldapd/auth.c
index 2d461ef20ec..4f26aad63e3 100644
--- a/usr.sbin/ldapd/auth.c
+++ b/usr.sbin/ldapd/auth.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: auth.c,v 1.1 2010/05/31 17:36:31 martinh Exp $ */
+/*	$OpenBSD: auth.c,v 1.2 2010/06/15 15:47:56 martinh Exp $ */
 
 /*
  * Copyright (c) 2009, 2010 Martin Hedenfalk <martin@bzero.se>
@@ -20,6 +20,7 @@
 #include <sys/queue.h>
 #include <netinet/in.h>
 
+#include <errno.h>
 #include <openssl/sha.h>
 #include <pwd.h>
 #include <resolv.h>		/* for b64_pton */
@@ -286,7 +287,14 @@ ldap_auth_simple(struct request *req, char *binddn, struct ber_element *auth)
 		    LDAP_SCOPE_BASE))
 			return LDAP_INSUFFICIENT_ACCESS;
 
-		elm = namespace_get(ns, binddn);
+		if (ns->data_db == NULL ||
+		    ((elm = namespace_get(ns, binddn)) == NULL &&
+		     errno == EAGAIN)) {
+			if (namespace_queue_request(ns, req) != 0)
+				return LDAP_BUSY;
+			return -1;	/* Database is being reopened. */
+		}
+
 		if (elm != NULL)
 			pw = ldap_get_attribute(elm, "userPassword");
 		if (pw != NULL) {
@@ -355,7 +363,8 @@ ldap_bind(struct request *req)
 
 	switch (auth->be_type) {
 	case LDAP_AUTH_SIMPLE:
-		rc = ldap_auth_simple(req, binddn, auth);
+		if ((rc = ldap_auth_simple(req, binddn, auth)) < 0)
+			return rc;
 		break;
 	case LDAP_AUTH_SASL:
 		if ((rc = ldap_auth_sasl(req, binddn, auth)) == LDAP_SUCCESS)
diff --git a/usr.sbin/ldapd/ldapd.h b/usr.sbin/ldapd/ldapd.h
index d87fa5797b1..e50e4e0eff0 100644
--- a/usr.sbin/ldapd/ldapd.h
+++ b/usr.sbin/ldapd/ldapd.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: ldapd.h,v 1.4 2010/06/15 15:12:54 martinh Exp $ */
+/*	$OpenBSD: ldapd.h,v 1.5 2010/06/15 15:47:56 martinh Exp $ */
 
 /*
  * Copyright (c) 2009, 2010 Martin Hedenfalk <martin@bzero.se>
@@ -492,14 +492,15 @@ int			 namespace_del(struct namespace *ns, char *dn);
 struct namespace	*namespace_for_base(const char *basedn);
 int			 namespace_has_index(struct namespace *ns,
 				const char *attr, enum index_type type);
+int			 namespace_begin_txn(struct namespace *ns,
+				struct btree_txn **data_txn,
+				struct btree_txn **indx_txn, int rdonly);
 int			 namespace_begin(struct namespace *ns);
 int			 namespace_commit(struct namespace *ns);
 void			 namespace_abort(struct namespace *ns);
 int			 namespace_queue_request(struct namespace *ns,
 				struct request *req);
 void			 namespace_queue_schedule(struct namespace *ns);
-int			 namespace_should_queue(struct namespace *ns,
-				struct request *req);
 void			 namespace_cancel_conn(struct conn *conn);
 
 int			 namespace_ber2db(struct namespace *ns,
diff --git a/usr.sbin/ldapd/modify.c b/usr.sbin/ldapd/modify.c
index 90a944025c7..883cd58c39d 100644
--- a/usr.sbin/ldapd/modify.c
+++ b/usr.sbin/ldapd/modify.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: modify.c,v 1.1 2010/05/31 17:36:31 martinh Exp $ */
+/*	$OpenBSD: modify.c,v 1.2 2010/06/15 15:47:56 martinh Exp $ */
 
 /*
  * Copyright (c) 2009, 2010 Martin Hedenfalk <martin@bzero.se>
@@ -29,6 +29,7 @@
 int
 ldap_delete(struct request *req)
 {
+	int			 rc;
 	char			*dn;
 	struct namespace	*ns;
 
@@ -46,20 +47,29 @@ ldap_delete(struct request *req)
 	if (!authorized(req->conn, ns, ACI_WRITE, dn, LDAP_SCOPE_BASE))
 		return ldap_respond(req, LDAP_INSUFFICIENT_ACCESS);
 
-	if (namespace_should_queue(ns, req)) {
+	if ((rc = namespace_begin(ns)) == BT_DEAD) {
 		if (namespace_queue_request(ns, req) != 0)
 			return ldap_respond(req, LDAP_BUSY);
 		return LDAP_BUSY;
-	}
+	} else if (rc != BT_SUCCESS)
+		return ldap_respond(req, LDAP_OTHER);
 
 	switch (namespace_del(ns, dn)) {
 	case BT_NOTFOUND:
-		return ldap_respond(req, LDAP_NO_SUCH_OBJECT);
+		rc = LDAP_NO_SUCH_OBJECT;
+		break;
 	case BT_SUCCESS:
-		return ldap_respond(req, LDAP_SUCCESS);
+		rc = LDAP_SUCCESS;
+		break;
 	default:
-		return ldap_respond(req, LDAP_OTHER);
+		rc = LDAP_OTHER;
+		break;
 	}
+
+	namespace_commit(ns);
+	if (rc >= 0)
+		ldap_respond(req, rc);
+	return rc;
 }
 
 int
@@ -89,11 +99,12 @@ ldap_add(struct request *req)
 	if (!authorized(req->conn, ns, ACI_WRITE, dn, LDAP_SCOPE_BASE) != 0)
 		return ldap_respond(req, LDAP_INSUFFICIENT_ACCESS);
 
-	if (namespace_should_queue(ns, req)) {
+	if ((rc = namespace_begin(ns)) == BT_DEAD) {
 		if (namespace_queue_request(ns, req) != 0)
 			return ldap_respond(req, LDAP_BUSY);
 		return LDAP_BUSY;
-	}
+	} else if (rc != BT_SUCCESS)
+		return ldap_respond(req, LDAP_OTHER);
 
 	/* add operational attributes
 	 */
@@ -112,16 +123,25 @@ ldap_add(struct request *req)
 	ldap_add_attribute(attrs, "entryUUID", set);
 
 	if ((rc = validate_entry(dn, attrs, ns->relax)) != LDAP_SUCCESS)
-		return ldap_respond(req, rc);
+		goto done;
 
 	switch (namespace_add(ns, dn, attrs)) {
 	case BT_SUCCESS:
-		return ldap_respond(req, LDAP_SUCCESS);
+		rc = LDAP_SUCCESS;
+		break;
 	case BT_EXISTS:
-		return ldap_respond(req, LDAP_ALREADY_EXISTS);
+		rc = LDAP_ALREADY_EXISTS;
+		break;
 	default:
-		return ldap_respond(req, LDAP_OTHER);
+		rc = LDAP_OTHER;
+		break;
 	}
+
+done:
+	namespace_commit(ns);
+	if (rc >= 0)
+		ldap_respond(req, rc);
+	return rc;
 }
 
 int
@@ -152,32 +172,40 @@ ldap_modify(struct request *req)
 	if (!authorized(req->conn, ns, ACI_WRITE, dn, LDAP_SCOPE_BASE) != 0)
 		return ldap_respond(req, LDAP_INSUFFICIENT_ACCESS);
 
-	if (namespace_should_queue(ns, req)) {
+	if ((rc = namespace_begin(ns)) == BT_DEAD) {
 		if (namespace_queue_request(ns, req) != 0)
 			return ldap_respond(req, LDAP_BUSY);
 		return LDAP_BUSY;
-	}
+	} else if (rc != BT_SUCCESS)
+		return ldap_respond(req, LDAP_OTHER);
 
-	if ((entry = namespace_get(ns, dn)) == NULL)
-		return ldap_respond(req, LDAP_NO_SUCH_OBJECT);
+	if ((entry = namespace_get(ns, dn)) == NULL) {
+		rc = LDAP_NO_SUCH_OBJECT;
+		goto done;
+	}
 
 	for (mod = mods->be_sub; mod; mod = mod->be_next) {
-		if (ber_scanf_elements(mod, "{E{se", &op, &attr, &vals) != 0)
-			return ldap_respond(req, LDAP_PROTOCOL_ERROR);
+		if (ber_scanf_elements(mod, "{E{se", &op, &attr, &vals) != 0) {
+			rc = LDAP_PROTOCOL_ERROR;
+			goto done;
+		}
 
 		if ((at = lookup_attribute(attr)) == NULL && !ns->relax) {
 			log_debug("unknown attribute type %s", attr);
-			return ldap_respond(req, LDAP_NO_SUCH_ATTRIBUTE);
+			rc = LDAP_NO_SUCH_ATTRIBUTE;
+			goto done;
 		}
 		if (at != NULL && at->immutable) {
 			log_debug("attempt to modify immutable attribute %s",
 			    attr);
-			return ldap_respond(req, LDAP_CONSTRAINT_VIOLATION);
+			rc = LDAP_CONSTRAINT_VIOLATION;
+			goto done;
 		}
 		if (at != NULL && at->usage != USAGE_USER_APP) {
 			log_debug("attempt to modify operational attribute %s",
 			    attr);
-			return ldap_respond(req, LDAP_CONSTRAINT_VIOLATION);
+			rc = LDAP_CONSTRAINT_VIOLATION;
+			goto done;
 		}
 
 		a = ldap_get_attribute(entry, attr);
@@ -209,7 +237,7 @@ ldap_modify(struct request *req)
 	}
 
 	if ((rc = validate_entry(dn, entry, ns->relax)) != LDAP_SUCCESS)
-		return ldap_respond(req, rc);
+		goto done;
 
 	set = ber_add_set(NULL);
 	ber_add_string(set, req->conn->binddn ?: "");
@@ -227,11 +255,20 @@ ldap_modify(struct request *req)
 
 	switch (namespace_update(ns, dn, entry)) {
 	case BT_SUCCESS:
-		return ldap_respond(req, LDAP_SUCCESS);
+		rc = LDAP_SUCCESS;
+		break;
 	case BT_EXISTS:
-		return ldap_respond(req, LDAP_ALREADY_EXISTS);
+		rc = LDAP_ALREADY_EXISTS;
+		break;
 	default:
-		return ldap_respond(req, LDAP_OTHER);
+		rc = LDAP_OTHER;
+		break;
 	}
+
+done:
+	namespace_commit(ns);
+	if (rc >= 0)
+		ldap_respond(req, rc);
+	return rc;
 }
 
diff --git a/usr.sbin/ldapd/namespace.c b/usr.sbin/ldapd/namespace.c
index 250a5985224..cb50b0e5405 100644
--- a/usr.sbin/ldapd/namespace.c
+++ b/usr.sbin/ldapd/namespace.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: namespace.c,v 1.5 2010/06/15 15:12:54 martinh Exp $ */
+/*	$OpenBSD: namespace.c,v 1.6 2010/06/15 15:47:56 martinh Exp $ */
 
 /*
  * Copyright (c) 2009, 2010 Martin Hedenfalk <martin@bzero.se>
@@ -20,6 +20,7 @@
 #include <sys/queue.h>
 
 #include <assert.h>
+#include <errno.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
@@ -59,19 +60,36 @@ namespace_new(const char *suffix)
 }
 
 int
-namespace_begin(struct namespace *ns)
+namespace_begin_txn(struct namespace *ns, struct btree_txn **data_txn,
+    struct btree_txn **indx_txn, int rdonly)
 {
-	if (ns->data_txn != NULL)
-		return -1;
-
-	if ((ns->data_txn = btree_txn_begin(ns->data_db, 0)) == NULL ||
-	    (ns->indx_txn = btree_txn_begin(ns->indx_db, 0)) == NULL) {
-		btree_txn_abort(ns->data_txn);
-		ns->data_txn = NULL;
-		return -1;
+	int	rc = BT_FAIL;
+
+	if (ns->data_db == NULL || ns->indx_db == NULL)
+		return BT_DEAD;
+
+	if ((*data_txn = btree_txn_begin(ns->data_db, rdonly)) == NULL ||
+	    (*indx_txn = btree_txn_begin(ns->indx_db, rdonly)) == NULL) {
+		if (errno == EAGAIN) {
+			if (*data_txn == NULL)
+				namespace_reopen_data(ns);
+			else
+				namespace_reopen_indx(ns);
+			rc = BT_DEAD;
+		}
+		log_warn("failed to open transaction");
+		btree_txn_abort(*data_txn);
+		*data_txn = NULL;
+		return rc;
 	}
 
-	return 0;
+	return BT_SUCCESS;
+}
+
+int
+namespace_begin(struct namespace *ns)
+{
+	return namespace_begin_txn(ns, &ns->data_txn, &ns->indx_txn, 0);
 }
 
 int
@@ -141,34 +159,43 @@ namespace_open(struct namespace *ns)
 	return 0;
 }
 
-int
-namespace_reopen_data(struct namespace *ns)
+static int
+namespace_reopen(const char *path)
 {
-	uint32_t	 old_flags;
+	struct open_req		 req;
 
-	log_info("reopening namespace %s (entries)", ns->suffix);
-	old_flags = btree_get_flags(ns->data_db);
-	btree_close(ns->data_db);
-	ns->data_db = btree_open(ns->data_path, old_flags, 0644);
-	if (ns->data_db == NULL)
+	log_debug("asking parent to open %s", path);
+
+	bzero(&req, sizeof(req));
+	if (strlcpy(req.path, path, sizeof(req.path)) >= sizeof(req.path)) {
+		log_warnx("%s: path truncated", __func__);
 		return -1;
+	}
 
-	return 0;
+	return imsg_compose_event(iev_ldapd, IMSG_LDAPD_OPEN, 0, 0, -1, &req,
+	    sizeof(req));
 }
 
 int
-namespace_reopen_indx(struct namespace *ns)
+namespace_reopen_data(struct namespace *ns)
 {
-	uint32_t	 old_flags;
-
-	log_info("reopening namespace %s (index)", ns->suffix);
-	old_flags = btree_get_flags(ns->indx_db);
-	btree_close(ns->indx_db);
-	ns->indx_db = btree_open(ns->indx_path, old_flags, 0644);
-	if (ns->indx_db == NULL)
-		return -1;
+	if (ns->data_db != NULL) {
+		btree_close(ns->data_db);
+		ns->data_db = NULL;
+		return namespace_reopen(ns->data_path);
+	}
+	return 1;
+}
 
-	return 0;
+int
+namespace_reopen_indx(struct namespace *ns)
+{
+	if (ns->indx_db != NULL) {
+		btree_close(ns->indx_db);
+		ns->indx_db = NULL;
+		return namespace_reopen(ns->indx_path);
+	}
+	return 1;
 }
 
 static int
@@ -255,10 +282,14 @@ namespace_find(struct namespace *ns, char *dn)
 	key.data = dn;
 	key.size = strlen(dn);
 
-	switch (btree_get(ns->data_db, &key, &val)) {
+	switch (btree_txn_get(ns->data_db, ns->data_txn, &key, &val)) {
 	case BT_FAIL:
 		log_warn("%s", dn);
 		return NULL;
+	case BT_DEAD:
+		log_warn("%s", dn);
+		namespace_reopen_data(ns);
+		return NULL;
 	case BT_NOTFOUND:
 		log_debug("%s: dn not found", dn);
 		return NULL;
@@ -404,8 +435,8 @@ namespace_put(struct namespace *ns, char *dn, struct ber_element *root,
 	struct btval		 key, val;
 
 	assert(ns != NULL);
-	assert(ns->data_txn == NULL);
-	assert(ns->indx_txn == NULL);
+	assert(ns->data_txn != NULL);
+	assert(ns->indx_txn != NULL);
 
 	bzero(&key, sizeof(key));
 	key.data = dn;
@@ -414,9 +445,6 @@ namespace_put(struct namespace *ns, char *dn, struct ber_element *root,
 	if (namespace_ber2db(ns, root, &val) != 0)
 		return BT_FAIL;
 
-	if (namespace_begin(ns) != 0)
-		return BT_FAIL;
-
 	rc = btree_txn_put(NULL, ns->data_txn, &key, &val,
 	    update ? 0 : BT_NOOVERWRITE);
 	if (rc != BT_SUCCESS) {
@@ -424,27 +452,18 @@ namespace_put(struct namespace *ns, char *dn, struct ber_element *root,
 			log_debug("%s: already exists", dn);
 		else
 			log_warn("%s", dn);
-		goto fail;
+		goto done;
 	}
 
 	/* FIXME: if updating, try harder to just update changed indices.
 	 */
 	if (update && unindex_entry(ns, &key, root) != BT_SUCCESS)
-		goto fail;
+		goto done;
 
-	if (index_entry(ns, &key, root) != 0)
-		goto fail;
-
-	if (namespace_commit(ns) != BT_SUCCESS)
-		goto fail;
+	rc = index_entry(ns, &key, root);
 
+done:
 	btval_reset(&val);
-	return 0;
-
-fail:
-	namespace_abort(ns);
-	btval_reset(&val);
-
 	return rc;
 }
 
@@ -468,8 +487,8 @@ namespace_del(struct namespace *ns, char *dn)
 	struct btval		 key, data;
 
 	assert(ns != NULL);
-	assert(ns->indx_txn == NULL);
-	assert(ns->data_txn == NULL);
+	assert(ns->indx_txn != NULL);
+	assert(ns->data_txn != NULL);
 
 	bzero(&key, sizeof(key));
 	bzero(&data, sizeof(data));
@@ -477,31 +496,11 @@ namespace_del(struct namespace *ns, char *dn)
 	key.data = dn;
 	key.size = strlen(key.data);
 
-	if (namespace_begin(ns) != 0)
-		return BT_FAIL;
-
 	rc = btree_txn_del(NULL, ns->data_txn, &key, &data);
-	if (rc != BT_SUCCESS)
-		goto fail;
-
-	if ((root = namespace_db2ber(ns, &data)) == NULL ||
-	    (rc = unindex_entry(ns, &key, root)) != 0)
-		goto fail;
+	if (rc == BT_SUCCESS && (root = namespace_db2ber(ns, &data)) != NULL)
+		rc = unindex_entry(ns, &key, root);
 
-	if (btree_txn_commit(ns->data_txn) != BT_SUCCESS)
-		goto fail;
-	ns->data_txn = NULL;
-	if (btree_txn_commit(ns->indx_txn) != BT_SUCCESS)
-		goto fail;
-	ns->indx_txn = NULL;
-
-	btval_reset(&data);
-	return 0;
-
-fail:
-	namespace_abort(ns);
 	btval_reset(&data);
-
 	return rc;
 }
 
@@ -540,7 +539,7 @@ namespace_has_index(struct namespace *ns, const char *attr,
 	return 0;
 }
 
-/* Queues modification requests while the namespace is being compacted.
+/* Queues modification requests while the namespace is being reopened.
  */
 int
 namespace_queue_request(struct namespace *ns, struct request *req)
@@ -561,16 +560,19 @@ namespace_queue_replay(int fd, short event, void *data)
 	struct namespace	*ns = data;
 	struct request		*req;
 
+	if (ns->data_db == NULL || ns->indx_db == NULL) {
+		log_debug("%s: database is being reopened", ns->suffix);
+		return;		/* Database is being reopened. */
+	}
+
 	if ((req = TAILQ_FIRST(&ns->request_queue)) == NULL)
 		return;
 	TAILQ_REMOVE(&ns->request_queue, req, next);
 
+	log_debug("replaying queued request");
 	req->replayed = 1;
 	request_dispatch(req);
 	ns->queued_requests--;
-
-	if (!ns->compacting)
-		namespace_queue_schedule(ns);
 }
 
 void
@@ -578,25 +580,11 @@ namespace_queue_schedule(struct namespace *ns)
 {
 	struct timeval	 tv;
 
-	tv.tv_sec = 0;
-	tv.tv_usec = 0;
-	evtimer_add(&ns->ev_queue, &tv);
-}
-
-int
-namespace_should_queue(struct namespace *ns, struct request *req)
-{
-	if (ns->compacting) {
-		log_debug("namespace %s is being compacted", ns->suffix);
-		return 1;
+	if (!evtimer_pending(&ns->ev_queue, NULL)) {
+		tv.tv_sec = 0;
+		tv.tv_usec = 0;
+		evtimer_add(&ns->ev_queue, &tv);
 	}
-
-	if (ns->queued_requests > 0 && !req->replayed) {
-		log_debug("namespace %s is being replayed", ns->suffix);
-		return 1;
-	}
-
-	return 0;
 }
 
 /* Cancel all queued requests from the given connection. Drops matching
diff --git a/usr.sbin/ldapd/search.c b/usr.sbin/ldapd/search.c
index 4b2882a2701..d23cf53ec19 100644
--- a/usr.sbin/ldapd/search.c
+++ b/usr.sbin/ldapd/search.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: search.c,v 1.4 2010/06/11 12:02:03 martinh Exp $ */
+/*	$OpenBSD: search.c,v 1.5 2010/06/15 15:47:56 martinh Exp $ */
 
 /*
  * Copyright (c) 2009, 2010 Martin Hedenfalk <martin@bzero.se>
@@ -20,9 +20,10 @@
 #include <sys/types.h>
 #include <sys/tree.h>
 
+#include <errno.h>
 #include <event.h>
-#include <string.h>
 #include <stdlib.h>
+#include <string.h>
 #include <time.h>
 
 #include "ldapd.h"
@@ -169,9 +170,11 @@ search_close(struct search *search)
 	btree_txn_abort(search->data_txn);
 	btree_txn_abort(search->indx_txn);
 
-	log_debug("finished search on msgid %lld", search->req->msgid);
+	if (search->req != NULL) {
+		log_debug("finished search on msgid %lld", search->req->msgid);
+		request_free(search->req);
+	}
 	TAILQ_REMOVE(&search->conn->searches, search, next);
-	request_free(search->req);
 	filter_free(search->plan);
 	free(search);
 	--stats.searches;
@@ -698,6 +701,7 @@ ldap_search(struct request *req)
 {
 	long long		 reason = LDAP_OTHER;
 	struct search		*search = NULL;
+	int			 rc;
 
 	if (stats.searches > MAX_SEARCHES) {
 		log_warnx("refusing more than %u concurrent searches",
@@ -782,13 +786,18 @@ ldap_search(struct request *req)
 		goto done;
 	}
 
-	search->data_txn = btree_txn_begin(search->ns->data_db, 1);
-	if (search->data_txn != NULL)
-		search->indx_txn = btree_txn_begin(search->ns->indx_db, 1);
-	if (search->indx_txn == NULL) {
-		btree_txn_abort(search->data_txn);
-		search->data_txn = NULL;
-		reason = LDAP_OPERATIONS_ERROR;
+	if ((rc = namespace_begin_txn(search->ns,
+	    &search->data_txn, &search->indx_txn, 1)) != BT_SUCCESS) {
+		if (rc == BT_DEAD) {
+			if (namespace_queue_request(search->ns, req) != 0) {
+				reason = LDAP_BUSY;
+				goto done;
+			}
+			search->req = NULL;	/* keep the scheduled request */
+			search_close(search);
+			return 0;
+		}
+		reason = LDAP_OTHER;
 		goto done;
 	}