smtpd makes too many assumptions about the structure and layout of its

disk-based queue, it makes it near impossible to make changes to it without
editing twenty files... how am i going to implement mongodb support ? :-)

bring a new queue_backend API which hides the details of the disk-based
queue to smtpd. it is not "plugged in" yet and I'm filling the holes.

5 files changed, 241 insertions, 4 deletions

diff --git a/usr.sbin/smtpd/queue_backend.c b/usr.sbin/smtpd/queue_backend.c
new file mode 100644
index 00000000000..aa9847b929c
--- /dev/null
+++ b/usr.sbin/smtpd/queue_backend.c
@@ -0,0 +1,123 @@
+/*	$OpenBSD: queue_backend.c,v 1.5 2011/04/14 17:06:43 gilles Exp $	*/
+
+/*
+ * Copyright (c) 2011 Gilles Chehade <gilles@openbsd.org>
+ *
+ * Permission to use, copy, modify, and distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#include <sys/types.h>
+#include <sys/queue.h>
+#include <sys/tree.h>
+#include <sys/param.h>
+#include <sys/socket.h>
+#include <sys/stat.h>
+
+#include <event.h>
+#include <imsg.h>
+#include <libgen.h>
+#include <pwd.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+
+#include "smtpd.h"
+#include "log.h"
+
+int	queue_fsqueue_message(struct smtpd *, enum queue_kind,
+    enum queue_op, char *);
+int	queue_fsqueue_envelope(struct smtpd *, enum queue_kind,
+    enum queue_op , struct message *);
+
+struct queue_backend queue_backends[] = {
+	{ QT_FS,
+	  queue_fsqueue_message,
+	  queue_fsqueue_envelope }
+};
+
+struct queue_backend *
+queue_backend_lookup(enum queue_type type)
+{
+	u_int8_t i;
+
+	for (i = 0; i < nitems(queue_backends); ++i)
+		if (queue_backends[i].type == type)
+			break;
+
+	if (i == nitems(queue_backends))
+		fatalx("invalid queue type");
+
+	return &queue_backends[i];
+}
+
+int
+queue_message_create(struct smtpd *env, enum queue_kind qkind, char *msgid)
+{
+	return env->sc_queue->message(env, qkind, QOP_CREATE, msgid);
+}
+
+int
+queue_message_delete(struct smtpd *env, enum queue_kind qkind, char *msgid)
+{
+	return env->sc_queue->message(env, qkind, QOP_DELETE, msgid);
+}
+
+int
+queue_message_commit(struct smtpd *env, enum queue_kind qkind, char *msgid)
+{
+	return env->sc_queue->message(env, qkind, QOP_COMMIT, msgid);
+}
+
+int
+queue_message_fd_r(struct smtpd *env, enum queue_kind qkind, char *msgid)
+{
+	return env->sc_queue->message(env, qkind, QOP_FD_R, msgid);
+}
+
+int
+queue_message_fd_rw(struct smtpd *env, enum queue_kind qkind, char *msgid)
+{
+	return env->sc_queue->message(env, qkind, QOP_FD_RW, msgid);
+}
+
+int
+queue_envelope_create(struct smtpd *env, enum queue_kind qkind,
+    struct message *envelope)
+{
+	return env->sc_queue->envelope(env, qkind, QOP_CREATE, envelope);
+}
+
+int
+queue_envelope_delete(struct smtpd *env, enum queue_kind qkind,
+    struct message *envelope)
+{
+	return env->sc_queue->envelope(env, qkind, QOP_DELETE, envelope);
+}
+
+int
+queue_envelope_load(struct smtpd *env, enum queue_kind qkind,
+    char *evpid, struct message *envelope)
+{
+	if (strlcpy(envelope->message_uid, evpid, sizeof(envelope->message_uid))
+	    >= sizeof(envelope->message_uid))
+		fatalx("queue_envelope_load: envelope ID truncation.");
+	return env->sc_queue->envelope(env, qkind, QOP_LOAD, envelope);
+}
+
+int
+queue_envelope_update(struct smtpd *env, enum queue_kind qkind,
+    struct message *envelope)
+{
+	return env->sc_queue->envelope(env, qkind, QOP_UPDATE, envelope);
+}
diff --git a/usr.sbin/smtpd/queue_fsqueue.c b/usr.sbin/smtpd/queue_fsqueue.c
new file mode 100644
index 00000000000..ee2766c69cd
--- /dev/null
+++ b/usr.sbin/smtpd/queue_fsqueue.c
@@ -0,0 +1,55 @@
+/*	$OpenBSD: queue_fsqueue.c,v 1.1 2011/04/14 17:06:43 gilles Exp $	*/
+
+/*
+ * Copyright (c) 2011 Gilles Chehade <gilles@openbsd.org>
+ *
+ * Permission to use, copy, modify, and distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#include <sys/types.h>
+#include <sys/queue.h>
+#include <sys/tree.h>
+#include <sys/param.h>
+#include <sys/socket.h>
+#include <sys/stat.h>
+
+#include <event.h>
+#include <imsg.h>
+#include <libgen.h>
+#include <pwd.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+
+#include "smtpd.h"
+#include "log.h"
+
+int	queue_fsqueue_message(struct smtpd *, enum queue_kind,
+    enum queue_op, char *);
+int	queue_fsqueue_envelope(struct smtpd *, enum queue_kind,
+    enum queue_op , struct message *);
+
+int
+queue_fsqueue_message(struct smtpd *env, enum queue_kind qkind,
+    enum queue_op qop, char *msgid)
+{
+	return 0;
+}
+
+int
+queue_fsqueue_envelope(struct smtpd *env, enum queue_kind qkind,
+    enum queue_op qop, struct message *envelope)
+{
+	return 0;
+}
diff --git a/usr.sbin/smtpd/smtpd.c b/usr.sbin/smtpd/smtpd.c
index 894a0e5068d..f1719b6ef00 100644
--- a/usr.sbin/smtpd/smtpd.c
+++ b/usr.sbin/smtpd/smtpd.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: smtpd.c,v 1.117 2011/04/13 20:53:18 gilles Exp $	*/
+/*	$OpenBSD: smtpd.c,v 1.118 2011/04/14 17:06:43 gilles Exp $	*/
 
 /*
  * Copyright (c) 2008 Gilles Chehade <gilles@openbsd.org>
@@ -491,6 +491,10 @@ main(int argc, char *argv[])
 	if (!setup_spool(env.sc_pw->pw_uid, 0))
 		errx(1, "invalid directory permissions");
 
+	env.sc_queue = queue_backend_lookup(QT_FS);
+	if (env.sc_queue == NULL)
+		errx(1, "could not find queue backend");
+
 	log_init(debug);
 	log_verbose(verbose);
 
diff --git a/usr.sbin/smtpd/smtpd.h b/usr.sbin/smtpd/smtpd.h
index 1f112c94af9..d1d84cccc58 100644
--- a/usr.sbin/smtpd/smtpd.h
+++ b/usr.sbin/smtpd/smtpd.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: smtpd.h,v 1.208 2011/04/13 20:53:18 gilles Exp $	*/
+/*	$OpenBSD: smtpd.h,v 1.209 2011/04/14 17:06:43 gilles Exp $	*/
 
 /*
  * Copyright (c) 2008 Gilles Chehade <gilles@openbsd.org>
@@ -634,6 +634,7 @@ struct smtpd {
 	struct passwd				*sc_pw;
 	char					 sc_hostname[MAXHOSTNAMELEN];
 	struct ramqueue				 sc_rqueue;
+	struct queue_backend			*sc_queue;
 
 	TAILQ_HEAD(listenerlist, listener)	*sc_listeners;
 	TAILQ_HEAD(maplist, map)		*sc_maps, *sc_maps_reload;
@@ -901,6 +902,40 @@ struct map_virtual {
 };
 
 
+/* queue structures */
+enum queue_type {
+	QT_INVALID=0,
+	QT_FS
+};
+
+enum queue_kind {
+	Q_INVALID=0,
+	Q_ENQUEUE,
+	Q_INCOMING,
+	Q_QUEUE,
+	Q_PURGE,
+	Q_OFFLINE,
+	Q_BOUNCE
+};
+
+enum queue_op {
+	QOP_INVALID=0,
+	QOP_CREATE,
+	QOP_DELETE,
+	QOP_UPDATE,
+	QOP_COMMIT,
+	QOP_LOAD,
+	QOP_FD_R,
+	QOP_FD_RW
+};
+
+struct queue_backend {
+	enum queue_type	type;
+	int (*message)(struct smtpd *, enum queue_kind, enum queue_op, char *);
+	int (*envelope)(struct smtpd *, enum queue_kind, enum queue_op,
+		struct message *);
+};
+
 extern void (*imsg_callback)(struct smtpd *, struct imsgev *, struct imsg *);
 
 
@@ -1017,6 +1052,25 @@ void		 queue_commit_envelopes(struct smtpd *, struct message*);
 u_int16_t	 queue_hash(char *);
 
 
+/* queue_backend.c */
+struct queue_backend *queue_backend_lookup(enum queue_type);
+int	queue_message_create(struct smtpd *, enum queue_kind, char *);
+int	queue_message_delete(struct smtpd *, enum queue_kind, char *);
+int	queue_message_commit(struct smtpd *, enum queue_kind, char *);
+int	queue_message_fd_r(struct smtpd *, enum queue_kind, char *);
+int	queue_message_fd_rw(struct smtpd *, enum queue_kind, char *);
+int	queue_envelope_create(struct smtpd *, enum queue_kind,
+    struct message *);
+int	queue_envelope_delete(struct smtpd *, enum queue_kind,
+    struct message *);
+int	queue_envelope_load(struct smtpd *, enum queue_kind,
+    char *, struct message *);
+int	queue_envelope_update(struct smtpd *, enum queue_kind,
+    struct message *);
+
+
+
+
 /* queue_shared.c */
 int		 queue_create_layout_message(char *, char *);
 void		 queue_delete_layout_message(char *, char *);
diff --git a/usr.sbin/smtpd/smtpd/Makefile b/usr.sbin/smtpd/smtpd/Makefile
index b244726c86d..df5313662b0 100644
--- a/usr.sbin/smtpd/smtpd/Makefile
+++ b/usr.sbin/smtpd/smtpd/Makefile
@@ -1,4 +1,4 @@
-#	$OpenBSD: Makefile,v 1.23 2011/04/13 20:53:18 gilles Exp $
+#	$OpenBSD: Makefile,v 1.24 2011/04/14 17:06:43 gilles Exp $
 
 PROG=		smtpd
 SRCS=		aliases.c authenticate.c bounce.c client.c		\
@@ -7,7 +7,8 @@ SRCS=		aliases.c authenticate.c bounce.c client.c		\
 		mfa.c mta.c parse.y queue.c queue_shared.c ruleset.c	\
 		runner.c smtp.c smtp_session.c smtpd.c ssl.c		\
 		ssl_privsep.c util.c asr.c print.c pack.c dname.c	\
-		res_random.c sockaddr.c ramqueue.c
+		res_random.c sockaddr.c ramqueue.c queue_backend.c	\
+		queue_fsqueue.c
 
 MAN=		smtpd.8 smtpd.conf.5
 BINDIR=		/usr/sbin