The RSA engine (used by pony) has to wait for a response from the

privileged process (lka) and receive the imsgs in a while loop
synchronously.  But the lka also sends other imsgs (DNS etc.) that can
still be queued up in the buffer when waiting for the RSA response.
This only happens under load with many concurrent connections.  For
now, we just call the pony imsg handler for non-RSA imsgs that are
already in the buffer.

ok gilles@ eric@ blambert@

4 files changed, 32 insertions, 10 deletions

diff --git a/usr.sbin/smtpd/ca.c b/usr.sbin/smtpd/ca.c
index 36fbd22b9b7..6a4e771feb1 100644
--- a/usr.sbin/smtpd/ca.c
+++ b/usr.sbin/smtpd/ca.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: ca.c,v 1.4 2014/04/29 19:13:13 reyk Exp $	*/
+/*	$OpenBSD: ca.c,v 1.5 2014/04/30 08:23:42 reyk Exp $	*/
 
 /*
  * Copyright (c) 2014 Reyk Floeter <reyk@openbsd.org>
@@ -53,6 +53,8 @@ static int	 rsae_verify(int dtype, const u_char *m, u_int, const u_char *,
 		    u_int, const RSA *);
 static int	 rsae_keygen(RSA *, int, BIGNUM *, BN_GENCB *);
 
+static uint64_t	 rsae_reqid = 0;
+
 void
 ca_init(void)
 {
@@ -164,8 +166,10 @@ ca_imsg(struct mproc *p, struct imsg *imsg)
 	size_t			 flen, tlen, padding;
 	struct pki		*pki;
 	int			 ret = 0;
+	uint64_t		 id;
 
 	m_msg(&m, imsg);
+	m_get_id(&m, &id);
 	m_get_string(&m, &pkiname);
 	m_get_data(&m, &from, &flen);
 	m_get_size(&m, &tlen);
@@ -192,6 +196,7 @@ ca_imsg(struct mproc *p, struct imsg *imsg)
 	}
 
 	m_create(p, imsg->hdr.type, 0, 0, -1);
+	m_add_id(p, id);
 	m_add_int(p, ret);
 	if (ret > 0)
 		m_add_data(p, to, (size_t)ret);
@@ -236,6 +241,7 @@ rsae_send_imsg(int flen, const u_char *from, u_char *to, RSA *rsa,
 	char		*pkiname;
 	size_t		 tlen;
 	struct msg	 m;
+	uint64_t	 id;
 
 	if ((pkiname = RSA_get_ex_data(rsa, 0)) == NULL)
 		return (0);
@@ -245,6 +251,8 @@ rsae_send_imsg(int flen, const u_char *from, u_char *to, RSA *rsa,
 	 * operation in OpenSSL's engine layer.
 	 */
 	m_create(p_lka, cmd, 0, 0, -1);
+	rsae_reqid++;
+	m_add_id(p_lka, rsae_reqid);
 	m_add_string(p_lka, pkiname);
 	m_add_data(p_lka, (const void *)from, (size_t)flen);
 	m_add_size(p_lka, (size_t)RSA_size(rsa));
@@ -264,10 +272,24 @@ rsae_send_imsg(int flen, const u_char *from, u_char *to, RSA *rsa,
 				fatalx("imsg_get error");
 			if (n == 0)
 				break;
-			if (imsg.hdr.type != cmd)
-				fatalx("invalid response");
+
+			log_imsg(PROC_PONY, PROC_LKA, &imsg);
+
+			switch (imsg.hdr.type) {
+			case IMSG_CA_PRIVENC:
+			case IMSG_CA_PRIVDEC:
+				break;
+			default:
+				/* Another imsg is queued up in the buffer */
+				pony_imsg(p_lka, &imsg);
+				imsg_free(&imsg);
+				continue;
+			}
 
 			m_msg(&m, &imsg);
+			m_get_id(&m, &id);
+			if (id != rsae_reqid)
+				fatalx("invalid response id");
 			m_get_int(&m, &ret);
 			if (ret > 0)
 				m_get_data(&m, &toptr, &tlen);
diff --git a/usr.sbin/smtpd/pony.c b/usr.sbin/smtpd/pony.c
index 2b3e59226ed..754480a27d6 100644
--- a/usr.sbin/smtpd/pony.c
+++ b/usr.sbin/smtpd/pony.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: pony.c,v 1.3 2014/04/29 19:13:13 reyk Exp $	*/
+/*	$OpenBSD: pony.c,v 1.4 2014/04/30 08:23:42 reyk Exp $	*/
 
 /*
  * Copyright (c) 2014 Gilles Chehade <gilles@poolp.org>
@@ -43,11 +43,10 @@ void mda_imsg(struct mproc *, struct imsg *);
 void mta_imsg(struct mproc *, struct imsg *);
 void smtp_imsg(struct mproc *, struct imsg *);
 
-static void pony_imsg(struct mproc *, struct imsg *);
 static void pony_shutdown(void);
 static void pony_sig_handler(int, short, void *);
 
-static void
+void
 pony_imsg(struct mproc *p, struct imsg *imsg)
 {
 	struct msg	m;
diff --git a/usr.sbin/smtpd/smtpd.c b/usr.sbin/smtpd/smtpd.c
index 482f9440740..bdfbed4b15c 100644
--- a/usr.sbin/smtpd/smtpd.c
+++ b/usr.sbin/smtpd/smtpd.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: smtpd.c,v 1.224 2014/04/29 21:04:17 reyk Exp $	*/
+/*	$OpenBSD: smtpd.c,v 1.225 2014/04/30 08:23:42 reyk Exp $	*/
 
 /*
  * Copyright (c) 2008 Gilles Chehade <gilles@poolp.org>
@@ -73,7 +73,6 @@ static void	offline_done(void);
 static int	offline_enqueue(char *);
 
 static void	purge_task(void);
-static void	log_imsg(int, int, struct imsg *);
 static int	parent_auth_user(const char *, const char *);
 static void	load_pki_tree(void);
 static void	load_pki_keys(void);
@@ -1192,7 +1191,7 @@ imsg_dispatch(struct mproc *p, struct imsg *imsg)
 	}
 }
 
-static void
+void
 log_imsg(int to, int from, struct imsg *imsg)
 {
 
diff --git a/usr.sbin/smtpd/smtpd.h b/usr.sbin/smtpd/smtpd.h
index 670a470a3c7..8cba92068a5 100644
--- a/usr.sbin/smtpd/smtpd.h
+++ b/usr.sbin/smtpd/smtpd.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: smtpd.h,v 1.457 2014/04/29 19:13:14 reyk Exp $	*/
+/*	$OpenBSD: smtpd.h,v 1.458 2014/04/30 08:23:43 reyk Exp $	*/
 
 /*
  * Copyright (c) 2008 Gilles Chehade <gilles@poolp.org>
@@ -1303,6 +1303,7 @@ time_t scheduler_compute_schedule(struct scheduler_info *);
 
 /* pony.c */
 pid_t pony(void);
+void pony_imsg(struct mproc *, struct imsg *);
 
 
 /* smtp.c */
@@ -1325,6 +1326,7 @@ void post_fork(int);
 const char *proc_name(enum smtp_proc_type);
 const char *proc_title(enum smtp_proc_type);
 const char *imsg_to_str(int);
+void log_imsg(int, int, struct imsg *);
 
 
 /* ssl_smtpd.c */