Sync with our smtpd repo:

* first bricks of ldap and sqlite support (not finished but both working)
* new table API to replace map API, all lookups are done through tables
* improved handling of temporary errors throughout the daemon
* improved scheduler and mta logic: connection reuse, optimizes batches
* improved queue: more tolerant to admin errors, new layout, less disk-IO
* improved memory usage under high load
* SSL certs/keys isolated to lookup process to avoid facing network
* VIRTUAL support improved, fully virtual setups possible now
* runtime tracing of processes through smtpctl trace
* ssl_privsep.c sync-ed with relayd
* ssl.c no longer contains smtpd specific interfaces
* smtpd-specific ssl bits moved to ssl_smtpd.c
* update mail address in copyright

FLUSH YOUR QUEUE. FLUSH YOUR QUEUE. FLUSH YOUR QUEUE. FLUSH YOUR QUEUE.

smtpd.conf(5) simplified, it will require adaptations

ok eric@

1 files changed, 214 insertions, 48 deletions

diff --git a/usr.sbin/smtpd/smtpctl.c b/usr.sbin/smtpd/smtpctl.c
index aa063e1f1f3..8c6992750c9 100644
--- a/usr.sbin/smtpd/smtpctl.c
+++ b/usr.sbin/smtpd/smtpctl.c
@@ -1,6 +1,7 @@
-/*	$OpenBSD: smtpctl.c,v 1.98 2012/11/23 09:25:44 eric Exp $	*/
+/*	$OpenBSD: smtpctl.c,v 1.99 2013/01/26 09:37:23 gilles Exp $	*/
 
 /*
+ * Copyright (c) 2006 Gilles Chehade <gilles@poolp.org>
  * Copyright (c) 2006 Pierre-Yves Ritschard <pyr@openbsd.org>
  * Copyright (c) 2005 Claudio Jeker <claudio@openbsd.org>
  * Copyright (c) 2004, 2005 Esben Norby <norby@openbsd.org>
@@ -65,6 +66,8 @@ static int action_schedule_all(void);
 
 static int action_show_queue(void);
 static int action_show_queue_message(uint32_t);
+static uint32_t trace_convert(uint32_t);
+static uint32_t profile_convert(uint32_t);
 
 int proctype;
 struct imsgbuf	*ibuf;
@@ -76,6 +79,9 @@ struct smtpd	*env = NULL;
 
 time_t now;
 
+struct queue_backend queue_backend_null;
+struct queue_backend queue_backend_ram;
+
 __dead void
 usage(void)
 {
@@ -98,12 +104,18 @@ setup_env(struct smtpd *smtpd)
 
 	if ((env->sc_pw = getpwnam(SMTPD_USER)) == NULL)
 		errx(1, "unknown user %s", SMTPD_USER);
+	if ((env->sc_pw = pw_dup(env->sc_pw)) == NULL)
+		err(1, NULL);
 
-	env->sc_queue = queue_backend_lookup("fs");
-	if (env->sc_queue == NULL)
-		errx(1, "could not find queue backend");
+	env->sc_pwqueue = getpwnam(SMTPD_QUEUE_USER);
+	if (env->sc_pwqueue)
+		env->sc_pwqueue = pw_dup(env->sc_pwqueue);
+	else
+		env->sc_pwqueue = pw_dup(env->sc_pw);
+	if (env->sc_pwqueue == NULL)
+		err(1, NULL);
 
-	if (!env->sc_queue->init(0))
+	if (!queue_init("fs", 0))
 		errx(1, "invalid directory permissions");
 }
 
@@ -168,7 +180,8 @@ main(int argc, char *argv[])
 	uint64_t		ulval;
 	char			name[MAX_LINE_SIZE];
 	int			done = 0;
-	int			verbose = 0;
+	int			verb = 0;
+	int			profile = 0;
 	int			action = -1;
 
 	/* parse options */
@@ -219,52 +232,54 @@ main(int argc, char *argv[])
 		/* not reached */
 
 	case SCHEDULE:
-		ulval = strtoevpid(res->data);
-		imsg_compose(ibuf, IMSG_SCHEDULER_SCHEDULE, 0, 0, -1, &ulval,
+		if (! strcmp(res->data, "all"))
+			return action_schedule_all();
+
+		if ((ulval = text_to_evpid(res->data)) == 0)
+			errx(1, "invalid msgid/evpid");
+		imsg_compose(ibuf, IMSG_CTL_SCHEDULE, 0, 0, -1, &ulval,
 		    sizeof(ulval));
 		break;
 	case REMOVE:
-		ulval = strtoevpid(res->data);
-		imsg_compose(ibuf, IMSG_SCHEDULER_REMOVE, 0, 0, -1, &ulval,
+		if ((ulval = text_to_evpid(res->data)) == 0)
+			errx(1, "invalid msgid/evpid");
+		imsg_compose(ibuf, IMSG_CTL_REMOVE, 0, 0, -1, &ulval,
 		    sizeof(ulval));
 		break;
 	case SHOW_QUEUE:
 		return action_show_queue();
-	case SCHEDULE_ALL:
-		return action_schedule_all();
 	case SHUTDOWN:
 		imsg_compose(ibuf, IMSG_CTL_SHUTDOWN, 0, 0, -1, NULL, 0);
 		break;
 	case PAUSE_MDA:
-		imsg_compose(ibuf, IMSG_QUEUE_PAUSE_MDA, 0, 0, -1, NULL, 0);
+		imsg_compose(ibuf, IMSG_CTL_PAUSE_MDA, 0, 0, -1, NULL, 0);
 		break;
 	case PAUSE_MTA:
-		imsg_compose(ibuf, IMSG_QUEUE_PAUSE_MTA, 0, 0, -1, NULL, 0);
+		imsg_compose(ibuf, IMSG_CTL_PAUSE_MTA, 0, 0, -1, NULL, 0);
 		break;
 	case PAUSE_SMTP:
-		imsg_compose(ibuf, IMSG_SMTP_PAUSE, 0, 0, -1, NULL, 0);
+		imsg_compose(ibuf, IMSG_CTL_PAUSE_SMTP, 0, 0, -1, NULL, 0);
 		break;
 	case RESUME_MDA:
-		imsg_compose(ibuf, IMSG_QUEUE_RESUME_MDA, 0, 0, -1, NULL, 0);
+		imsg_compose(ibuf, IMSG_CTL_RESUME_MDA, 0, 0, -1, NULL, 0);
 		break;
 	case RESUME_MTA:
-		imsg_compose(ibuf, IMSG_QUEUE_RESUME_MTA, 0, 0, -1, NULL, 0);
+		imsg_compose(ibuf, IMSG_CTL_RESUME_MTA, 0, 0, -1, NULL, 0);
 		break;
 	case RESUME_SMTP:
-		imsg_compose(ibuf, IMSG_SMTP_RESUME, 0, 0, -1, NULL, 0);
+		imsg_compose(ibuf, IMSG_CTL_RESUME_SMTP, 0, 0, -1, NULL, 0);
 		break;
 	case SHOW_STATS:
 		imsg_compose(ibuf, IMSG_STATS, 0, 0, -1, NULL, 0);
 		break;
-	case UPDATE_MAP:
+	case UPDATE_TABLE:
 		if (strlcpy(name, res->data, sizeof name) >= sizeof name)
-			errx(1, "map name too long.");
-		imsg_compose(ibuf, IMSG_LKA_UPDATE_MAP, 0, 0, -1,
+			errx(1, "table name too long.");
+		imsg_compose(ibuf, IMSG_LKA_UPDATE_TABLE, 0, 0, -1,
 		    name, strlen(name) + 1);
 		done = 1;
 		break;
 	case MONITOR:
-		done = 1;
 		while (1) {
 			imsg_compose(ibuf, IMSG_DIGEST, 0, 0, -1, NULL, 0);
 			flush();
@@ -275,14 +290,65 @@ main(int argc, char *argv[])
 		}
 		break;
 	case LOG_VERBOSE:
-		verbose = 1;
+		verb = TRACE_VERBOSE;
 		/* FALLTHROUGH */
 	case LOG_BRIEF:
-		imsg_compose(ibuf, IMSG_CTL_VERBOSE, 0, 0, -1, &verbose,
-		    sizeof(verbose));
+		imsg_compose(ibuf, IMSG_CTL_VERBOSE, 0, 0, -1, &verb,
+		    sizeof(verb));
 		printf("logging request sent.\n");
 		done = 1;
 		break;
+
+	case LOG_TRACE_IMSG:
+	case LOG_TRACE_IO:
+	case LOG_TRACE_SMTP:
+	case LOG_TRACE_MFA:
+	case LOG_TRACE_MTA:
+	case LOG_TRACE_BOUNCE:
+	case LOG_TRACE_SCHEDULER:
+	case LOG_TRACE_STAT:
+	case LOG_TRACE_RULES:
+	case LOG_TRACE_IMSG_SIZE:
+	case LOG_TRACE_ALL:
+		verb = trace_convert(action);
+		imsg_compose(ibuf, IMSG_CTL_TRACE, 0, 0, -1, &verb,
+		    sizeof(verb));
+		done = 1;
+		break;
+
+	case LOG_UNTRACE_IMSG:
+	case LOG_UNTRACE_IO:
+	case LOG_UNTRACE_SMTP:
+	case LOG_UNTRACE_MFA:
+	case LOG_UNTRACE_MTA:
+	case LOG_UNTRACE_BOUNCE:
+	case LOG_UNTRACE_SCHEDULER:
+	case LOG_UNTRACE_STAT:
+	case LOG_UNTRACE_RULES:
+	case LOG_UNTRACE_IMSG_SIZE:
+	case LOG_UNTRACE_ALL:
+		verb = trace_convert(action);
+		imsg_compose(ibuf, IMSG_CTL_UNTRACE, 0, 0, -1, &verb,
+		    sizeof(verb));
+		done = 1;
+		break;
+
+	case LOG_PROFILE_IMSG:
+	case LOG_PROFILE_QUEUE:
+		profile = profile_convert(action);
+		imsg_compose(ibuf, IMSG_CTL_PROFILE, 0, 0, -1, &profile,
+		    sizeof(profile));
+		done = 1;
+		break;
+
+	case LOG_UNPROFILE_IMSG:
+	case LOG_UNPROFILE_QUEUE:
+		profile = profile_convert(action);
+		imsg_compose(ibuf, IMSG_CTL_UNPROFILE, 0, 0, -1, &profile,
+		    sizeof(profile));
+		done = 1;
+		break;
+
 	default:
 		errx(1, "unknown request (%d)", action);
 	}
@@ -303,6 +369,32 @@ main(int argc, char *argv[])
 		case RESUME_SMTP:
 		case LOG_VERBOSE:
 		case LOG_BRIEF:
+		case LOG_TRACE_IMSG:
+		case LOG_TRACE_IO:
+		case LOG_TRACE_SMTP:
+		case LOG_TRACE_MFA:
+		case LOG_TRACE_MTA:
+		case LOG_TRACE_BOUNCE:
+		case LOG_TRACE_SCHEDULER:
+		case LOG_TRACE_STAT:
+		case LOG_TRACE_RULES:
+		case LOG_TRACE_IMSG_SIZE:
+		case LOG_TRACE_ALL:
+		case LOG_UNTRACE_IMSG:
+		case LOG_UNTRACE_IO:
+		case LOG_UNTRACE_SMTP:
+		case LOG_UNTRACE_MFA:
+		case LOG_UNTRACE_MTA:
+		case LOG_UNTRACE_BOUNCE:
+		case LOG_UNTRACE_SCHEDULER:
+		case LOG_UNTRACE_STAT:
+		case LOG_UNTRACE_RULES:
+		case LOG_UNTRACE_IMSG_SIZE:
+		case LOG_UNTRACE_ALL:
+		case LOG_PROFILE_IMSG:
+		case LOG_PROFILE_QUEUE:
+		case LOG_UNPROFILE_IMSG:
+		case LOG_UNPROFILE_QUEUE:
 			done = show_command_output(&imsg);
 			break;
 		case SHOW_STATS:
@@ -311,7 +403,7 @@ main(int argc, char *argv[])
 			break;
 		case NONE:
 			break;
-		case UPDATE_MAP:
+		case UPDATE_TABLE:
 			break;
 		case MONITOR:
 
@@ -340,13 +432,13 @@ action_show_queue_message(uint32_t msgid)
     nextbatch:
 
 	found = 0;
-	imsg_compose(ibuf, IMSG_SCHEDULER_ENVELOPES, 0, 0, -1,
+	imsg_compose(ibuf, IMSG_CTL_LIST_ENVELOPES, 0, 0, -1,
 	    &evpid, sizeof evpid);
 	flush();
 
 	while (1) {
 		next_message(&imsg);
-		if (imsg.hdr.type != IMSG_SCHEDULER_ENVELOPES)
+		if (imsg.hdr.type != IMSG_CTL_LIST_ENVELOPES)
 			errx(1, "unexpected message %i", imsg.hdr.type);
 
 		if (imsg.hdr.len == sizeof imsg.hdr) {
@@ -375,11 +467,11 @@ action_show_queue(void)
 	now = time(NULL);
 
 	do {
-		imsg_compose(ibuf, IMSG_SCHEDULER_MESSAGES, 0, 0, -1,
+		imsg_compose(ibuf, IMSG_CTL_LIST_MESSAGES, 0, 0, -1,
 		    &msgid, sizeof msgid);
 		flush();
 		next_message(&imsg);
-		if (imsg.hdr.type != IMSG_SCHEDULER_MESSAGES)
+		if (imsg.hdr.type != IMSG_CTL_LIST_MESSAGES)
 			errx(1, "unexpected message type %i", imsg.hdr.type);
 		msgids = imsg.data;
 		n = (imsg.hdr.len - sizeof imsg.hdr) / sizeof (*msgids);
@@ -408,11 +500,11 @@ action_schedule_all(void)
 
 	from = 0;
 	while (1) {
-		imsg_compose(ibuf, IMSG_SCHEDULER_MESSAGES, 0, 0, -1,
+		imsg_compose(ibuf, IMSG_CTL_LIST_MESSAGES, 0, 0, -1,
 		    &from, sizeof from);
 		flush();
 		next_message(&imsg);
-		if (imsg.hdr.type != IMSG_SCHEDULER_MESSAGES)
+		if (imsg.hdr.type != IMSG_CTL_LIST_MESSAGES)
 			errx(1, "unexpected message type %i", imsg.hdr.type);
 		msgids = imsg.data;
 		n = (imsg.hdr.len - sizeof imsg.hdr) / sizeof (*msgids);
@@ -421,7 +513,7 @@ action_schedule_all(void)
 
 		for (i = 0; i < n; i++) {
 			evpid = msgids[i];
-			imsg_compose(ibuf, IMSG_SCHEDULER_SCHEDULE, 0,
+			imsg_compose(ibuf, IMSG_CTL_SCHEDULE, 0,
 			    0, -1, &evpid, sizeof(evpid));
 		}
 		from = msgids[n - 1] + 1;
@@ -518,7 +610,7 @@ show_stats_output(void)
 }
 
 static void
-show_queue(int flags)
+show_queue(flags)
 {
 	struct envelope	 envelope;
 	int		 r;
@@ -541,23 +633,23 @@ show_queue_envelope(struct envelope *e, int online)
 
 	status[0] = '\0';
 
-	getflag(&e->flags, DF_BOUNCE, "bounce",
+	getflag(&e->flags, EF_BOUNCE, "bounce",
 	    status, sizeof(status));
-	getflag(&e->flags, DF_AUTHENTICATED, "auth",
+	getflag(&e->flags, EF_AUTHENTICATED, "auth",
 	    status, sizeof(status));
-	getflag(&e->flags, DF_INTERNAL, "internal",
+	getflag(&e->flags, EF_INTERNAL, "internal",
 	    status, sizeof(status));
 
 	if (online) {
-		if (e->flags & DF_PENDING)
+		if (e->flags & EF_PENDING)
 			snprintf(runstate, sizeof runstate, "pending|%zi",
 			    (ssize_t)(e->nexttry - now));
-		else if (e->flags & DF_INFLIGHT)
+		else if (e->flags & EF_INFLIGHT)
 			snprintf(runstate, sizeof runstate, "inflight|%zi",
 			    (ssize_t)(now - e->lasttry));
 		else
 			snprintf(runstate, sizeof runstate, "invalid|");
-		e->flags &= ~(DF_PENDING|DF_INFLIGHT);
+		e->flags &= ~(EF_PENDING|EF_INFLIGHT);
 	}
 	else
 		strlcpy(runstate, "offline|", sizeof runstate);
@@ -638,13 +730,15 @@ show_envelope(const char *s)
 	char	 buf[MAXPATHLEN];
 	uint64_t evpid;
 
-	evpid = strtoevpid(s);
-	if (! bsnprintf(buf, sizeof(buf), "%s%s/%02x/%08x%s/%016" PRIx64,
-	    PATH_SPOOL,
-	    PATH_QUEUE,
-	    evpid_to_msgid(evpid) & 0xff,
-	    evpid_to_msgid(evpid),
-	    PATH_ENVELOPES, evpid))
+	if ((evpid = text_to_evpid(s)) == 0)
+		errx(1, "invalid msgid/evpid");
+
+	if (! bsnprintf(buf, sizeof(buf), "%s%s/%02x/%08x/%016" PRIx64,
+		PATH_SPOOL,
+		PATH_QUEUE,
+		(evpid_to_msgid(evpid) & 0xff000000) >> 24,
+		evpid_to_msgid(evpid),
+		evpid))
 		errx(1, "unable to retrieve envelope");
 
 	display(buf);
@@ -655,8 +749,12 @@ show_message(const char *s)
 {
 	char	 buf[MAXPATHLEN];
 	uint32_t msgid;
+	uint64_t evpid;
+
+	if ((evpid = text_to_evpid(s)) == 0)
+		errx(1, "invalid msgid/evpid");
 
-	msgid = evpid_to_msgid(strtoevpid(s));
+	msgid = evpid_to_msgid(evpid);
 	if (! bsnprintf(buf, sizeof(buf), "%s%s/%02x/%08x/message",
 	    PATH_SPOOL,
 	    PATH_QUEUE,
@@ -715,3 +813,71 @@ show_monitor(struct stat_digest *d)
 	last = *d;
 	count++;
 }
+
+static uint32_t
+trace_convert(uint32_t trace)
+{
+	switch (trace) {
+	case LOG_TRACE_IMSG:
+	case LOG_UNTRACE_IMSG:
+		return TRACE_IMSG;
+
+	case LOG_TRACE_IO:
+	case LOG_UNTRACE_IO:
+		return TRACE_IO;
+
+	case LOG_TRACE_SMTP:
+	case LOG_UNTRACE_SMTP:
+		return TRACE_SMTP;
+
+	case LOG_TRACE_MFA:
+	case LOG_UNTRACE_MFA:
+		return TRACE_MFA;
+
+	case LOG_TRACE_MTA:
+	case LOG_UNTRACE_MTA:
+		return TRACE_MTA;
+
+	case LOG_TRACE_BOUNCE:
+	case LOG_UNTRACE_BOUNCE:
+		return TRACE_BOUNCE;
+
+	case LOG_TRACE_SCHEDULER:
+	case LOG_UNTRACE_SCHEDULER:
+		return TRACE_SCHEDULER;
+
+	case LOG_TRACE_STAT:
+	case LOG_UNTRACE_STAT:
+		return TRACE_STAT;
+
+	case LOG_TRACE_RULES:
+	case LOG_UNTRACE_RULES:
+		return TRACE_RULES;
+
+	case LOG_TRACE_IMSG_SIZE:
+	case LOG_UNTRACE_IMSG_SIZE:
+		return TRACE_IMSGSIZE;
+
+	case LOG_TRACE_ALL:
+	case LOG_UNTRACE_ALL:
+		return ~TRACE_VERBOSE;
+	}
+
+	return 0;
+}
+
+static uint32_t
+profile_convert(uint32_t prof)
+{
+	switch (prof) {
+	case LOG_PROFILE_IMSG:
+	case LOG_UNPROFILE_IMSG:
+		return PROFILE_IMSG;
+
+	case LOG_PROFILE_QUEUE:
+	case LOG_UNPROFILE_QUEUE:
+		return PROFILE_QUEUE;
+	}
+
+	return 0;
+}