Introduce privsep for private keys:

- Move RSA private keys to a new separate process instead of copying
them to the relays.  A custom RSA engine is used by the SSL/TLS code
of the relay processes to send RSA private key encryption/decryption
(also used for sign/verify) requests to the new "ca" processes instead
of operating on the private key directly.

- Each relay process gets its own related ca process.  Setting
"prefork 5" in the config file will spawn 10 processes (5 relay, 5
ca).  This diff also reduces the default number of relay processes
from 5 to 3 which should be suitable in most installations without a
very heavy load.

- Don't keep text versions of the keys in memory, parse them once and
keep the binary representation.  This might still be the case in
OpenSSL's internals but will be fixed in the library.

This diff doesn't prevent something like "heartbleed" but adds an
additional mitigation to prevent leakage of the private keys from the
processes doing SSL/TLS.

With feedback from many
ok benno@

9 files changed, 652 insertions, 100 deletions

diff --git a/usr.sbin/relayd/Makefile b/usr.sbin/relayd/Makefile
index 8e92b9012cf..6266eaa117e 100644
--- a/usr.sbin/relayd/Makefile
+++ b/usr.sbin/relayd/Makefile
@@ -1,11 +1,12 @@
-#	$OpenBSD: Makefile,v 1.25 2014/04/14 12:58:04 blambert Exp $
+#	$OpenBSD: Makefile,v 1.26 2014/04/18 13:55:26 reyk Exp $
 
 PROG=		relayd
-SRCS=		parse.y log.c control.c ssl.c ssl_privsep.c \
-		relayd.c pfe.c pfe_filter.c pfe_route.c hce.c relay.c \
-		relay_http.c relay_udp.c carp.c check_icmp.c check_tcp.c \
-		check_script.c name2id.c snmp.c shuffle.c proc.c config.c \
-		agentx.c
+SRCS=		parse.y
+SRCS+=		agentx.c ca.c carp.c check_icmp.c check_script.c \
+		check_tcp.c config.c control.c hce.c log.c name2id.c \
+		pfe.c pfe_filter.c pfe_route.c proc.c \
+		relay.c relay_http.c relay_udp.c relayd.c \
+		shuffle.c snmp.c ssl.c ssl_privsep.c
 MAN=		relayd.8 relayd.conf.5
 
 LDADD=		-levent -lssl -lcrypto -lutil
diff --git a/usr.sbin/relayd/ca.c b/usr.sbin/relayd/ca.c
new file mode 100644
index 00000000000..c08a64d4de7
--- /dev/null
+++ b/usr.sbin/relayd/ca.c
@@ -0,0 +1,431 @@
+/*	$OpenBSD: ca.c,v 1.1 2014/04/18 13:55:26 reyk Exp $	*/
+
+/*
+ * Copyright (c) 2014 Reyk Floeter <reyk@openbsd.org>
+ *
+ * Permission to use, copy, modify, and distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#include <sys/param.h>
+#include <sys/queue.h>
+#include <sys/socket.h>
+#include <sys/uio.h>
+
+#include <net/if.h>
+#include <netinet/in.h>
+
+#include <limits.h>
+#include <event.h>
+#include <fcntl.h>
+#include <unistd.h>
+#include <string.h>
+#include <stdlib.h>
+#include <errno.h>
+
+#include <openssl/pem.h>
+#include <openssl/evp.h>
+#include <openssl/rsa.h>
+#include <openssl/engine.h>
+
+#include "relayd.h"
+
+void	 ca_init(struct privsep *, struct privsep_proc *p, void *);
+void	 ca_launch(void);
+
+int	 ca_dispatch_parent(int, struct privsep_proc *, struct imsg *);
+int	 ca_dispatch_relay(int, struct privsep_proc *, struct imsg *);
+
+int	 rsae_pub_enc(int, const u_char *, u_char *, RSA *, int);
+int	 rsae_pub_dec(int,const u_char *, u_char *, RSA *, int);
+int	 rsae_priv_enc(int, const u_char *, u_char *, RSA *, int);
+int	 rsae_priv_dec(int, const u_char *, u_char *, RSA *, int);
+int	 rsae_mod_exp(BIGNUM *, const BIGNUM *, RSA *, BN_CTX *);
+int	 rsae_bn_mod_exp(BIGNUM *, const BIGNUM *, const BIGNUM *,
+	    const BIGNUM *, BN_CTX *, BN_MONT_CTX *);
+int	 rsae_init(RSA *);
+int	 rsae_finish(RSA *);
+int	 rsae_sign(int, const u_char *, u_int,
+	    u_char *, u_int *, const RSA *);
+int	 rsae_verify(int dtype, const u_char *m, u_int,
+	    const u_char *, u_int, const RSA *);
+int	 rsae_keygen(RSA *, int, BIGNUM *, BN_GENCB *);
+
+static struct relayd *env = NULL;
+extern int		 proc_id;
+
+static struct privsep_proc procs[] = {
+	{ "parent",	PROC_PARENT,	ca_dispatch_parent },
+	{ "relay",	PROC_RELAY,	ca_dispatch_relay },
+};
+
+pid_t
+ca(struct privsep *ps, struct privsep_proc *p)
+{
+	env = ps->ps_env;
+
+	return (proc_run(ps, p, procs, nitems(procs), ca_init, NULL));
+}
+
+void
+ca_init(struct privsep *ps, struct privsep_proc *p, void *arg)
+{
+	if (config_init(ps->ps_env) == -1)
+		fatal("failed to initialize configuration");
+
+	proc_id = p->p_instance;
+	env->sc_id = getpid() & 0xffff;
+}
+
+void
+ca_launch(void)
+{
+	BIO		*in = NULL;
+	EVP_PKEY	*pkey = NULL;
+	struct relay	*rlay;
+
+	TAILQ_FOREACH(rlay, env->sc_relays, rl_entry) {
+		if ((rlay->rl_conf.flags & (F_SSL|F_SSLCLIENT)) == 0)
+			continue;
+
+		if ((in = BIO_new_mem_buf(rlay->rl_ssl_key,
+		    rlay->rl_conf.ssl_key_len)) == NULL)
+			fatalx("ca_launch: key");
+
+		if ((pkey = PEM_read_bio_PrivateKey(in,
+		    NULL, NULL, NULL)) == NULL)
+			fatalx("ca_launch: PEM");
+
+		purge_key(&rlay->rl_ssl_key, rlay->rl_conf.ssl_key_len);
+		purge_key(&rlay->rl_ssl_cert, rlay->rl_conf.ssl_cert_len);
+
+		rlay->rl_ssl_pkey = pkey;
+
+		BIO_free(in);
+	}
+}
+
+int
+ca_dispatch_parent(int fd, struct privsep_proc *p, struct imsg *imsg)
+{
+	switch (imsg->hdr.type) {
+	case IMSG_CFG_RELAY:
+		config_getrelay(env, imsg);
+		break;
+	case IMSG_CFG_DONE:
+		config_getcfg(env, imsg);
+		break;
+	case IMSG_CTL_START:
+		ca_launch();
+		break;
+	case IMSG_CTL_RESET:
+		config_getreset(env, imsg);
+		break;
+	default:
+		return (-1);
+	}
+
+	return (0);
+}
+
+static EVP_PKEY *
+ca_get_key(objid_t id)
+{
+	struct relay	*rlay;
+
+	if ((rlay = relay_find(env, id)) == NULL)
+		return (NULL);
+
+	return (rlay->rl_ssl_pkey);
+}
+
+int
+ca_dispatch_relay(int fd, struct privsep_proc *p, struct imsg *imsg)
+{
+	struct ctl_keyop	 cko;
+	EVP_PKEY		*pkey;
+	RSA			*rsa;
+	u_char			*from = NULL, *to = NULL;
+	struct iovec		 iov[2];
+	int			 c = 0;
+
+	switch (imsg->hdr.type) {
+	case IMSG_CA_PRIVENC:
+	case IMSG_CA_PRIVDEC:
+		IMSG_SIZE_CHECK(imsg, (&cko));
+		bcopy(imsg->data, &cko, sizeof(cko));
+		if (cko.cko_proc > env->sc_prefork_relay)
+			fatalx("ca_dispatch_relay: "
+			    "invalid relay proc");
+		if (IMSG_DATA_SIZE(imsg) != (sizeof(cko) + cko.cko_flen))
+			fatalx("ca_dispatch_relay: "
+			    "invalid key operation");
+		if ((pkey = ca_get_key(cko.cko_id)) == NULL ||
+		    (rsa = EVP_PKEY_get1_RSA(pkey)) == NULL)
+			fatalx("ca_dispatch_relay: "
+			    "invalid relay key or id");
+
+		from = (u_char *)imsg->data + sizeof(cko);
+		if ((to = calloc(1, cko.cko_tlen)) == NULL)
+			fatalx("ca_dispatch_relay: calloc");
+
+		switch (imsg->hdr.type) {
+		case IMSG_CA_PRIVENC:
+			cko.cko_tlen = RSA_private_encrypt(cko.cko_flen,
+			    from, to, rsa, cko.cko_padding);
+			break;
+		case IMSG_CA_PRIVDEC:
+			cko.cko_tlen = RSA_private_decrypt(cko.cko_flen,
+			    from, to, rsa, cko.cko_padding);
+			break;
+		}
+
+		iov[c].iov_base = &cko;
+		iov[c++].iov_len = sizeof(cko);
+		if (cko.cko_tlen) {
+			iov[c].iov_base = to;
+			iov[c++].iov_len = cko.cko_tlen;
+		}
+
+		proc_composev_imsg(env->sc_ps, PROC_RELAY, cko.cko_proc,
+		    imsg->hdr.type, -1, iov, c);
+
+		free(to);
+		RSA_free(rsa);
+		break;
+	default:
+		return (-1);
+	}
+
+	return (0);
+}
+
+/*
+ * RSA privsep engine (called from unprivileged processes)
+ */
+
+const RSA_METHOD *rsa_default = NULL;
+
+static RSA_METHOD rsae_method = {
+	"RSA privsep engine",
+	rsae_pub_enc,
+	rsae_pub_dec,
+	rsae_priv_enc,
+	rsae_priv_dec,
+	rsae_mod_exp,
+	rsae_bn_mod_exp,
+	rsae_init,
+	rsae_finish,
+	0,
+	NULL,
+	rsae_sign,
+	rsae_verify,
+	rsae_keygen
+};
+
+static int
+rsae_send_imsg(int flen, const u_char *from, u_char *to,
+    RSA *rsa,int padding, u_int cmd)
+{
+	struct ctl_keyop cko;
+	int		 ret = 0;
+	u_int32_t	*id;
+	struct iovec	 iov[2];
+	struct imsgbuf	*ibuf;
+	struct imsgev	*iev;
+	struct imsg	 imsg;
+	int		 n, done = 0, cnt = 0;
+	u_char		*toptr;
+
+	if ((id = RSA_get_app_data(rsa)) == NULL)
+		return (0);
+
+	iev = proc_iev(env->sc_ps, PROC_CA, proc_id);
+	ibuf = &iev->ibuf;
+
+	/*
+	 * XXX this could be nicer...
+	 */
+
+	cko.cko_id = *id;
+	cko.cko_proc = proc_id;
+	cko.cko_flen = flen;
+	cko.cko_tlen = RSA_size(rsa);
+	cko.cko_padding = padding;
+
+	iov[cnt].iov_base = &cko;
+	iov[cnt++].iov_len = sizeof(cko);
+	iov[cnt].iov_base = from;
+	iov[cnt++].iov_len = flen;
+
+	/*
+	 * Send a synchronous imsg because we cannot defer the RSA
+	 * operation in OpenSSL's engine layer.
+	 */
+	imsg_composev(ibuf, cmd, 0, 0, -1, iov, cnt);
+	imsg_flush(ibuf);
+
+	while (!done) {
+		if ((n = imsg_read(ibuf)) == -1)
+			fatalx("imsg_read");
+		if (n == 0)
+			fatalx("pipe closed");
+
+		while (!done) {
+			if ((n = imsg_get(ibuf, &imsg)) == -1)
+				fatalx("imsg_get error");
+			if (n == 0)
+				break;
+			if (imsg.hdr.type != cmd)
+				fatalx("invalid response");
+
+			IMSG_SIZE_CHECK(&imsg, (&cko));
+			memcpy(&cko, imsg.data, sizeof(cko));
+			if (IMSG_DATA_SIZE(&imsg) !=
+			    (sizeof(cko) + cko.cko_tlen))
+				fatalx("data size");
+
+			ret = cko.cko_tlen;
+			if (ret) {
+				toptr = (u_char *)imsg.data + sizeof(cko);
+				memcpy(to, toptr, ret);
+			}
+			done = 1;			
+
+			imsg_free(&imsg);
+		}
+	}
+	imsg_event_add(iev);
+
+	return (ret);
+}
+
+int
+rsae_pub_enc(int flen,const u_char *from,
+    u_char *to, RSA *rsa,int padding)
+{
+	DPRINTF("%s:%d", __func__, __LINE__);
+	return (rsa_default->rsa_pub_enc(flen, from, to, rsa, padding));
+}
+
+int
+rsae_pub_dec(int flen,const u_char *from, u_char
+    *to, RSA *rsa,int padding)
+{
+	DPRINTF("%s:%d", __func__, __LINE__);
+	return (rsa_default->rsa_pub_dec(flen, from, to, rsa, padding));
+}
+
+int
+rsae_priv_enc(int flen, const u_char *from, u_char *to,
+    RSA *rsa, int padding)
+{
+	DPRINTF("%s:%d", __func__, __LINE__);
+	return (rsae_send_imsg(flen, from, to, rsa, padding,
+	    IMSG_CA_PRIVENC));
+}
+
+int
+rsae_priv_dec(int flen, const u_char *from, u_char *to,
+    RSA *rsa, int padding)
+{
+	DPRINTF("%s:%d", __func__, __LINE__);
+	return (rsae_send_imsg(flen, from, to, rsa, padding,
+	    IMSG_CA_PRIVDEC));
+}
+
+int
+rsae_mod_exp(BIGNUM *r0, const BIGNUM *I, RSA *rsa, BN_CTX *ctx)
+{
+	DPRINTF("%s:%d", __func__, __LINE__);
+	return (rsa_default->rsa_mod_exp(r0, I, rsa, ctx));
+}
+
+int
+rsae_bn_mod_exp(BIGNUM *r, const BIGNUM *a, const BIGNUM *p, const
+    BIGNUM *m, BN_CTX *ctx, BN_MONT_CTX *m_ctx)
+{
+	DPRINTF("%s:%d", __func__, __LINE__);
+	return (rsa_default->bn_mod_exp(r, a, p, m, ctx, m_ctx));
+}
+
+int
+rsae_init(RSA *rsa)
+{
+	DPRINTF("%s:%d", __func__, __LINE__);
+	if (rsa_default->init == NULL)
+		return (1);
+	return (rsa_default->init(rsa));
+}
+
+int
+rsae_finish(RSA *rsa)
+{
+	DPRINTF("%s:%d", __func__, __LINE__);
+	if (rsa_default->finish == NULL)
+		return (1);
+	return (rsa_default->finish(rsa));
+}
+
+int
+rsae_sign(int type, const u_char *m, u_int
+    m_length, u_char *sigret, u_int *siglen, const RSA *rsa)
+{
+	DPRINTF("%s:%d", __func__, __LINE__);
+	return (rsa_default->rsa_sign(type, m, m_length, sigret, siglen, rsa));
+}
+
+int
+rsae_verify(int dtype, const u_char *m, u_int m_length,
+    const u_char *sigbuf, u_int siglen, const RSA *rsa)
+{
+	DPRINTF("%s:%d", __func__, __LINE__);
+	return (rsa_default->rsa_verify(dtype, m, m_length, sigbuf, siglen, rsa));
+}
+
+int
+rsae_keygen(RSA *rsa, int bits, BIGNUM *e, BN_GENCB *cb)
+{
+	DPRINTF("%s:%d", __func__, __LINE__);
+	return (rsa_default->rsa_keygen(rsa, bits, e, cb));
+}
+
+int
+ca_engine_init(struct relayd *x_env)
+{
+	ENGINE	*e;
+
+	if (env == NULL)
+		env = x_env;
+
+	if ((e = ENGINE_get_default_RSA()) == NULL ||
+	    (rsa_default = ENGINE_get_RSA(e)) == NULL)
+		return (-1);
+
+	if (rsa_default->rsa_mod_exp == NULL)
+		rsae_method.rsa_mod_exp = NULL;
+	if (rsa_default->rsa_mod_exp == NULL)
+		rsae_method.rsa_mod_exp = NULL;
+	if (rsa_default->bn_mod_exp == NULL)
+		rsae_method.bn_mod_exp = NULL;
+	if (rsa_default->rsa_keygen == NULL)
+		rsae_method.rsa_keygen = NULL;
+	rsae_method.flags = rsa_default->flags |
+	    RSA_METHOD_FLAG_NO_CHECK;
+	rsae_method.app_data = rsa_default->app_data;
+
+	if (!ENGINE_set_RSA(e, &rsae_method) ||
+	    !ENGINE_set_default_RSA(e))
+		return (-1);
+
+	return (0);
+}
diff --git a/usr.sbin/relayd/config.c b/usr.sbin/relayd/config.c
index 318a9e8bd5e..c9e45ecb279 100644
--- a/usr.sbin/relayd/config.c
+++ b/usr.sbin/relayd/config.c
@@ -1,7 +1,7 @@
-/*	$OpenBSD: config.c,v 1.11 2014/02/24 06:55:11 jsg Exp $	*/
+/*	$OpenBSD: config.c,v 1.12 2014/04/18 13:55:26 reyk Exp $	*/
 
 /*
- * Copyright (c) 2011 Reyk Floeter <reyk@openbsd.org>
+ * Copyright (c) 2011 - 2014 Reyk Floeter <reyk@openbsd.org>
  *
  * Permission to use, copy, modify, and distribute this software for any
  * purpose with or without fee is hereby granted, provided that the above
@@ -64,8 +64,9 @@ config_init(struct relayd *env)
 		ps->ps_what[PROC_PARENT] = CONFIG_ALL;
 		ps->ps_what[PROC_PFE] = CONFIG_ALL & ~CONFIG_PROTOS;
 		ps->ps_what[PROC_HCE] = CONFIG_TABLES;
-		ps->ps_what[PROC_RELAY] =
-		    CONFIG_TABLES|CONFIG_RELAYS|CONFIG_PROTOS;
+		ps->ps_what[PROC_CA] = CONFIG_RELAYS;
+		ps->ps_what[PROC_RELAY] = CONFIG_RELAYS|
+		    CONFIG_TABLES|CONFIG_PROTOS|CONFIG_CA_ENGINE;
 	}
 
 	/* Other configuration */
@@ -245,6 +246,7 @@ config_getcfg(struct relayd *env, struct imsg *imsg)
 	struct table		*tb;
 	struct host		*h, *ph;
 	struct ctl_flags	 cf;
+	u_int			 what;
 
 	if (IMSG_DATA_SIZE(imsg) != sizeof(cf))
 		return (0); /* ignore */
@@ -254,7 +256,9 @@ config_getcfg(struct relayd *env, struct imsg *imsg)
 	env->sc_opts = cf.cf_opts;
 	env->sc_flags = cf.cf_flags;
 
-	if (ps->ps_what[privsep_process] & CONFIG_TABLES) {
+	what = ps->ps_what[privsep_process];
+
+	if (what & CONFIG_TABLES) {
 		/* Update the tables */
 		TAILQ_FOREACH(tb, env->sc_tables, entry) {
 			TAILQ_FOREACH(h, &tb->hosts, entry) {
@@ -267,8 +271,12 @@ config_getcfg(struct relayd *env, struct imsg *imsg)
 		}
 	}
 
-	if (env->sc_flags & (F_SSL|F_SSLCLIENT))
+	if (env->sc_flags & (F_SSL|F_SSLCLIENT)) {
 		ssl_init(env);
+		if ((what & CONFIG_CA_ENGINE) &&
+		    (ca_engine_init(env)) == -1)
+			fatal("CA engine failed");
+	}
 
 	if (privsep_process != PROC_PARENT)
 		proc_compose_imsg(env->sc_ps, PROC_PARENT, -1,
@@ -802,45 +810,52 @@ config_setrelay(struct relayd *env, struct relay *rlay)
 	struct privsep		*ps = env->sc_ps;
 	struct ctl_relaytable	 crt;
 	struct relay_table	*rlt;
+	struct relay_config	 rl;
 	int			 id;
 	int			 fd, n, m;
 	struct iovec		 iov[6];
 	size_t			 c;
+	u_int			 what;
 
 	/* opens listening sockets etc. */
 	if (relay_privinit(rlay) == -1)
 		return (-1);
 
 	for (id = 0; id < PROC_MAX; id++) {
-		if ((ps->ps_what[id] & CONFIG_RELAYS) == 0 ||
-		    id == privsep_process)
+		what = ps->ps_what[id];
+
+		if ((what & CONFIG_RELAYS) == 0 || id == privsep_process)
 			continue;
 
 		DPRINTF("%s: sending relay %s to %s fd %d", __func__,
 		    rlay->rl_conf.name, ps->ps_title[id], rlay->rl_s);
 
+		memcpy(&rl, &rlay->rl_conf, sizeof(rl));
+
 		c = 0;
-		iov[c].iov_base = &rlay->rl_conf;
-		iov[c++].iov_len = sizeof(rlay->rl_conf);
-		if (rlay->rl_conf.ssl_cert_len) {
+		iov[c].iov_base = &rl;
+		iov[c++].iov_len = sizeof(rl);
+		if (rl.ssl_cert_len) {
 			iov[c].iov_base = rlay->rl_ssl_cert;
-			iov[c++].iov_len = rlay->rl_conf.ssl_cert_len;
+			iov[c++].iov_len = rl.ssl_cert_len;
 		}
-		if (rlay->rl_conf.ssl_key_len) {
+		if ((what & CONFIG_CA_ENGINE) == 0 &&
+		    rl.ssl_key_len) {
 			iov[c].iov_base = rlay->rl_ssl_key;
-			iov[c++].iov_len = rlay->rl_conf.ssl_key_len;
-		}
-		if (rlay->rl_conf.ssl_ca_len) {
+			iov[c++].iov_len = rl.ssl_key_len;
+		} else
+			rl.ssl_key_len = 0;
+		if (rl.ssl_ca_len) {
 			iov[c].iov_base = rlay->rl_ssl_ca;
-			iov[c++].iov_len = rlay->rl_conf.ssl_ca_len;
+			iov[c++].iov_len = rl.ssl_ca_len;
 		}
-		if (rlay->rl_conf.ssl_cacert_len) {
+		if (rl.ssl_cacert_len) {
 			iov[c].iov_base = rlay->rl_ssl_cacert;
-			iov[c++].iov_len = rlay->rl_conf.ssl_cacert_len;
+			iov[c++].iov_len = rl.ssl_cacert_len;
 		}
-		if (rlay->rl_conf.ssl_cakey_len) {
+		if (rl.ssl_cakey_len) {
 			iov[c].iov_base = rlay->rl_ssl_cakey;
-			iov[c++].iov_len = rlay->rl_conf.ssl_cakey_len;
+			iov[c++].iov_len = rl.ssl_cakey_len;
 		}
 
 		if (id == PROC_RELAY) {
@@ -858,6 +873,9 @@ config_setrelay(struct relayd *env, struct relay *rlay)
 			    iov, c);
 		}
 
+		if ((what & CONFIG_TABLES) == 0)
+			continue;
+
 		/* Now send the tables associated to this relay */
 		TAILQ_FOREACH(rlt, &rlay->rl_tables, rlt_entry) {
 			crt.id = rlt->rlt_table->conf.id;
diff --git a/usr.sbin/relayd/relay.c b/usr.sbin/relayd/relay.c
index 8c73ded7bcd..ad7b7f86b65 100644
--- a/usr.sbin/relayd/relay.c
+++ b/usr.sbin/relayd/relay.c
@@ -1,7 +1,7 @@
-/*	$OpenBSD: relay.c,v 1.167 2013/09/09 17:57:44 reyk Exp $	*/
+/*	$OpenBSD: relay.c,v 1.168 2014/04/18 13:55:26 reyk Exp $	*/
 
 /*
- * Copyright (c) 2006 - 2013 Reyk Floeter <reyk@openbsd.org>
+ * Copyright (c) 2006 - 2014 Reyk Floeter <reyk@openbsd.org>
  *
  * Permission to use, copy, modify, and distribute this software for any
  * purpose with or without fee is hereby granted, provided that the above
@@ -52,6 +52,8 @@ int		 relay_dispatch_parent(int, struct privsep_proc *,
 		    struct imsg *);
 int		 relay_dispatch_pfe(int, struct privsep_proc *,
 		    struct imsg *);
+int		 relay_dispatch_ca(int, struct privsep_proc *,
+		    struct imsg *);
 void		 relay_shutdown(void);
 
 void		 relay_nodedebug(const char *, struct protonode *);
@@ -96,6 +98,7 @@ int				 proc_id;
 static struct privsep_proc procs[] = {
 	{ "parent",	PROC_PARENT,	relay_dispatch_parent },
 	{ "pfe",	PROC_PFE,	relay_dispatch_pfe },
+	{ "ca",		PROC_CA,	relay_dispatch_ca }
 };
 
 pid_t
@@ -1782,6 +1785,12 @@ relay_dispatch_pfe(int fd, struct privsep_proc *p, struct imsg *imsg)
 }
 
 int
+relay_dispatch_ca(int fd, struct privsep_proc *p, struct imsg *imsg)
+{
+	return (-1);
+}
+
+int
 relay_dispatch_parent(int fd, struct privsep_proc *p, struct imsg *imsg)
 {
 	struct rsession		*con;
@@ -1904,9 +1913,11 @@ relay_ssl_ctx_create(struct relay *rlay)
 		goto err;
 
 	log_debug("%s: loading private key", __func__);
-	if (!ssl_ctx_use_private_key(ctx, rlay->rl_ssl_key,
-	    rlay->rl_conf.ssl_key_len))
+	if (!ssl_ctx_fake_private_key(ctx,
+	    &rlay->rl_conf.id, rlay->rl_ssl_cert, rlay->rl_conf.ssl_cert_len,
+	    &rlay->rl_ssl_x509, &rlay->rl_ssl_pkey))
 		goto err;
+
 	if (!SSL_CTX_check_private_key(ctx))
 		goto err;
 
@@ -1915,6 +1926,9 @@ relay_ssl_ctx_create(struct relay *rlay)
 	    strlen(rlay->rl_conf.name)))
 		goto err;
 
+	/* The text versions of the keys/certs are not needed anymore */
+	purge_key(&rlay->rl_ssl_cert, rlay->rl_conf.ssl_cert_len);
+
 	return (ctx);
 
  err:
@@ -2087,7 +2101,7 @@ relay_ssl_connect(int fd, short event, void *arg)
 		    SSL_get_peer_certificate(con->se_out.ssl)) != NULL) {
 			con->se_in.sslcert =
 			    ssl_update_certificate(servercert,
-			    rlay->rl_ssl_key, rlay->rl_conf.ssl_key_len,
+			    rlay->rl_ssl_pkey,
 			    rlay->rl_ssl_cakey, rlay->rl_conf.ssl_cakey_len,
 			    rlay->rl_ssl_cacert, rlay->rl_conf.ssl_cacert_len);
 		} else
diff --git a/usr.sbin/relayd/relayd.c b/usr.sbin/relayd/relayd.c
index cb8055190eb..6051bd55278 100644
--- a/usr.sbin/relayd/relayd.c
+++ b/usr.sbin/relayd/relayd.c
@@ -1,7 +1,7 @@
-/*	$OpenBSD: relayd.c,v 1.119 2014/03/16 18:38:30 guenther Exp $	*/
+/*	$OpenBSD: relayd.c,v 1.120 2014/04/18 13:55:26 reyk Exp $	*/
 
 /*
- * Copyright (c) 2007, 2008 Reyk Floeter <reyk@openbsd.org>
+ * Copyright (c) 2007 - 2014 Reyk Floeter <reyk@openbsd.org>
  * Copyright (c) 2006 Pierre-Yves Ritschard <pyr@openbsd.org>
  *
  * Permission to use, copy, modify, and distribute this software for any
@@ -57,6 +57,8 @@ int		 parent_dispatch_pfe(int, struct privsep_proc *, struct imsg *);
 int		 parent_dispatch_hce(int, struct privsep_proc *, struct imsg *);
 int		 parent_dispatch_relay(int, struct privsep_proc *,
 		    struct imsg *);
+int		 parent_dispatch_ca(int, struct privsep_proc *,
+		    struct imsg *);
 int		 bindany(struct ctl_bindany *);
 
 struct relayd			*relayd_env;
@@ -64,7 +66,8 @@ struct relayd			*relayd_env;
 static struct privsep_proc procs[] = {
 	{ "pfe",	PROC_PFE, parent_dispatch_pfe, pfe },
 	{ "hce",	PROC_HCE, parent_dispatch_hce, hce },
-	{ "relay",	PROC_RELAY, parent_dispatch_relay, relay }
+	{ "relay",	PROC_RELAY, parent_dispatch_relay, relay },
+	{ "ca",		PROC_CA, parent_dispatch_ca, ca }
 };
 
 void
@@ -222,6 +225,7 @@ main(int argc, char *argv[])
 		log_info("startup");
 
 	ps->ps_instances[PROC_RELAY] = env->sc_prefork_relay;
+	ps->ps_instances[PROC_CA] = env->sc_prefork_relay;
 	proc_init(ps, procs, nitems(procs));
 
 	setproctitle("parent");
@@ -300,8 +304,8 @@ parent_configure(struct relayd *env)
 		config_setrelay(env, rlay);
 	}
 
-	/* HCE, PFE and the preforked relays need to reload their config. */
-	env->sc_reload = 2 + env->sc_prefork_relay;
+	/* HCE, PFE, CA and the relays need to reload their config. */
+	env->sc_reload = 2 + (2 * env->sc_prefork_relay);
 
 	for (id = 0; id < PROC_MAX; id++) {
 		if (id == privsep_process)
@@ -326,7 +330,7 @@ parent_configure(struct relayd *env)
 	ret = 0;
 
  done:
-	config_purge(env, CONFIG_ALL);
+	config_purge(env, CONFIG_ALL & ~CONFIG_RELAYS);
 	return (ret);
 }
 
@@ -513,6 +517,22 @@ parent_dispatch_relay(int fd, struct privsep_proc *p, struct imsg *imsg)
 	return (0);
 }
 
+int
+parent_dispatch_ca(int fd, struct privsep_proc *p, struct imsg *imsg)
+{
+	struct relayd		*env = p->p_env;
+
+	switch (imsg->hdr.type) {
+	case IMSG_CFG_DONE:
+		parent_configure_done(env);
+		break;
+	default:
+		return (-1);
+	}
+
+	return (0);
+}
+
 void
 purge_tree(struct proto_tree *tree)
 {
@@ -568,6 +588,20 @@ purge_table(struct tablelist *head, struct table *table)
 }
 
 void
+purge_key(char **ptr, off_t len)
+{
+	char	*key = *ptr;
+
+	if (key == NULL || len == 0)
+		return;
+
+	explicit_bzero(key, len);
+	free(key);
+
+	*ptr = NULL;
+}
+
+void
 purge_relay(struct relayd *env, struct relay *rlay)
 {
 	struct rsession		*con;
@@ -590,14 +624,22 @@ purge_relay(struct relayd *env, struct relay *rlay)
 	if (rlay->rl_dstbev != NULL)
 		bufferevent_free(rlay->rl_dstbev);
 
+	purge_key(&rlay->rl_ssl_cert, rlay->rl_conf.ssl_cert_len);
+	purge_key(&rlay->rl_ssl_key, rlay->rl_conf.ssl_key_len);
+	purge_key(&rlay->rl_ssl_ca, rlay->rl_conf.ssl_ca_len);
+	purge_key(&rlay->rl_ssl_cakey, rlay->rl_conf.ssl_cakey_len);
+
+	if (rlay->rl_ssl_x509 != NULL) {
+		X509_free(rlay->rl_ssl_x509);
+		rlay->rl_ssl_x509 = NULL;
+	}
+	if (rlay->rl_ssl_pkey != NULL) {
+		EVP_PKEY_free(rlay->rl_ssl_pkey);
+		rlay->rl_ssl_pkey = NULL;
+	}
+
 	if (rlay->rl_ssl_ctx != NULL)
 		SSL_CTX_free(rlay->rl_ssl_ctx);
-	if (rlay->rl_ssl_cert != NULL)
-		free(rlay->rl_ssl_cert);
-	if (rlay->rl_ssl_key != NULL)
-		free(rlay->rl_ssl_key);
-	if (rlay->rl_ssl_ca != NULL)
-		free(rlay->rl_ssl_ca);
 
 	while ((rlt = TAILQ_FIRST(&rlay->rl_tables))) {
 		TAILQ_REMOVE(&rlay->rl_tables, rlt, rlt_entry);
diff --git a/usr.sbin/relayd/relayd.conf.5 b/usr.sbin/relayd/relayd.conf.5
index 3a25aafb2ad..1dc1b6ec695 100644
--- a/usr.sbin/relayd/relayd.conf.5
+++ b/usr.sbin/relayd/relayd.conf.5
@@ -1,4 +1,4 @@
-.\"	$OpenBSD: relayd.conf.5,v 1.141 2014/04/14 15:24:25 jmc Exp $
+.\"	$OpenBSD: relayd.conf.5,v 1.142 2014/04/18 13:55:26 reyk Exp $
 .\"
 .\" Copyright (c) 2006, 2007 Reyk Floeter <reyk@openbsd.org>
 .\" Copyright (c) 2006, 2007 Pierre-Yves Ritschard <pyr@openbsd.org>
@@ -15,7 +15,7 @@
 .\" ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
 .\" OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 .\"
-.Dd $Mdocdate: April 14 2014 $
+.Dd $Mdocdate: April 18 2014 $
 .Dt RELAYD.CONF 5
 .Os
 .Sh NAME
@@ -145,7 +145,7 @@ relayed connections.
 This increases the performance and prevents delays when connecting
 to a relay.
 .Xr relayd 8
-runs 5 relay processes by default and every process will handle
+runs 3 relay processes by default and every process will handle
 all configured relays.
 .It Ic snmp Oo Ic trap Oc Op Qq Ar path
 Send an SNMP trap when the state of a host changes.
diff --git a/usr.sbin/relayd/relayd.h b/usr.sbin/relayd/relayd.h
index ddc120bc81a..bce53dff249 100644
--- a/usr.sbin/relayd/relayd.h
+++ b/usr.sbin/relayd/relayd.h
@@ -1,7 +1,7 @@
-/*	$OpenBSD: relayd.h,v 1.174 2014/04/18 12:02:37 reyk Exp $	*/
+/*	$OpenBSD: relayd.h,v 1.175 2014/04/18 13:55:26 reyk Exp $	*/
 
 /*
- * Copyright (c) 2006 - 2012 Reyk Floeter <reyk@openbsd.org>
+ * Copyright (c) 2006 - 2014 Reyk Floeter <reyk@openbsd.org>
  * Copyright (c) 2006, 2007 Pierre-Yves Ritschard <pyr@openbsd.org>
  * Copyright (c) 2003, 2004 Henning Brauer <henning@openbsd.org>
  *
@@ -54,7 +54,7 @@
 #define RELAY_MAX_SESSIONS	1024
 #define RELAY_TIMEOUT		600
 #define RELAY_CACHESIZE		-1	/* use default size */
-#define RELAY_NUMPROC		5
+#define RELAY_NUMPROC		3
 #define RELAY_MAXPROC		32
 #define RELAY_MAXHOSTS		32
 #define RELAY_STATINTERVAL	60
@@ -69,6 +69,7 @@
 #define CONFIG_PROTOS		0x08
 #define CONFIG_ROUTES		0x10
 #define CONFIG_RTS		0x20
+#define CONFIG_CA_ENGINE	0x40
 #define CONFIG_ALL		0xff
 
 #define SMALL_READ_BUF_SIZE	1024
@@ -229,6 +230,14 @@ struct ctl_bindany {
 	int			 bnd_proto;
 };
 
+struct ctl_keyop {
+	objid_t			 cko_id;
+	int			 cko_proc;
+	int			 cko_flen;
+	int			 cko_tlen;
+	int			 cko_padding;
+};
+
 struct ctl_stats {
 	objid_t			 id;
 	int			 proc;
@@ -657,8 +666,13 @@ struct relay {
 	struct event		 rl_evt;
 
 	SSL_CTX			*rl_ssl_ctx;
+
 	char			*rl_ssl_cert;
+	X509			*rl_ssl_x509;
+
 	char			*rl_ssl_key;
+	EVP_PKEY		*rl_ssl_pkey;
+
 	char			*rl_ssl_ca;
 	char			*rl_ssl_cacert;
 	char			*rl_ssl_cakey;
@@ -828,7 +842,9 @@ enum imsg_type {
 	IMSG_CFG_PROTONODE,
 	IMSG_CFG_RELAY,
 	IMSG_CFG_RELAY_TABLE,
-	IMSG_CFG_DONE
+	IMSG_CFG_DONE,
+	IMSG_CA_PRIVENC,
+	IMSG_CA_PRIVDEC
 };
 
 enum privsep_procid {
@@ -837,6 +853,7 @@ enum privsep_procid {
 	PROC_HCE,
 	PROC_RELAY,
 	PROC_PFE,
+	PROC_CA,
 	PROC_MAX
 } privsep_process;
 
@@ -1068,14 +1085,19 @@ void	 ssl_transaction(struct ctl_tcp_event *);
 SSL_CTX	*ssl_ctx_create(struct relayd *);
 void	 ssl_error(const char *, const char *);
 char	*ssl_load_key(struct relayd *, const char *, off_t *, char *);
-X509	*ssl_update_certificate(X509 *, char *, off_t,
+X509	*ssl_update_certificate(X509 *, EVP_PKEY *,
 	    char *, off_t, char *, off_t);
+int	 ssl_ctx_fake_private_key(SSL_CTX *, void *, char *, off_t,
+	    X509 **, EVP_PKEY **);
 
 /* ssl_privsep.c */
-int	 ssl_ctx_use_private_key(SSL_CTX *, char *, off_t);
 int	 ssl_ctx_use_certificate_chain(SSL_CTX *, char *, off_t);
 int	 ssl_ctx_load_verify_memory(SSL_CTX *, char *, off_t);
 
+/* ca.c */
+pid_t	 ca(struct privsep *, struct privsep_proc *);
+int	 ca_engine_init(struct relayd *);
+
 /* relayd.c */
 struct host	*host_find(struct relayd *, objid_t);
 struct table	*table_find(struct relayd *, objid_t);
@@ -1096,6 +1118,7 @@ struct relay	*relay_findbyname(struct relayd *, const char *);
 struct relay	*relay_findbyaddr(struct relayd *, struct relay_config *);
 int		 expand_string(char *, size_t, const char *, const char *);
 void		 translate_string(char *);
+void		 purge_key(char **, off_t);
 void		 purge_tree(struct proto_tree *);
 void		 purge_table(struct tablelist *, struct table *);
 void		 purge_relay(struct relayd *, struct relay *);
diff --git a/usr.sbin/relayd/ssl.c b/usr.sbin/relayd/ssl.c
index 8b8c6198eff..33bf43f26a1 100644
--- a/usr.sbin/relayd/ssl.c
+++ b/usr.sbin/relayd/ssl.c
@@ -1,7 +1,7 @@
-/*	$OpenBSD: ssl.c,v 1.19 2013/05/31 20:23:37 benno Exp $	*/
+/*	$OpenBSD: ssl.c,v 1.20 2014/04/18 13:55:26 reyk Exp $	*/
 
 /*
- * Copyright (c) 2007-2013 Reyk Floeter <reyk@openbsd.org>
+ * Copyright (c) 2007 - 2014 Reyk Floeter <reyk@openbsd.org>
  * Copyright (c) 2006 Pierre-Yves Ritschard <pyr@openbsd.org>
  *
  * Permission to use, copy, modify, and distribute this software for any
@@ -359,12 +359,12 @@ ssl_load_key(struct relayd *env, const char *name, off_t *len, char *pass)
 }
 
 X509 *
-ssl_update_certificate(X509 *oldcert, char *keystr, off_t keylen,
+ssl_update_certificate(X509 *oldcert, EVP_PKEY *pkey,
     char *cakeystr, off_t cakeylen, char *cacertstr, off_t cacertlen)
 {
 	char		 name[2][SSL_NAME_SIZE];
 	X509		*cert = NULL, *cacert = NULL;
-	EVP_PKEY	*key = NULL, *cakey = NULL;
+	EVP_PKEY	*cakey = NULL;
 	BIO		*bio = NULL;
 
 	name[0][0] = name[1][0] = '\0';
@@ -374,13 +374,6 @@ ssl_update_certificate(X509 *oldcert, char *keystr, off_t keylen,
 	    name[1], sizeof(name[1])))
 		goto done;
 
-	/* Get SSL key */
-	if ((bio = BIO_new_mem_buf(keystr, keylen)) == NULL)
-		goto done;
-	if ((key = PEM_read_bio_PrivateKey(bio, &key,
-	    ssl_password_cb, NULL)) == NULL)
-		goto done;
-
 	/* Get CA key */
 	BIO_free_all(bio);
 	if ((bio = BIO_new_mem_buf(cakeystr, cakeylen)) == NULL)
@@ -401,7 +394,7 @@ ssl_update_certificate(X509 *oldcert, char *keystr, off_t keylen,
 		goto done;
 
 	/* Update certificate key and use our CA as the issuer */
-	X509_set_pubkey(cert, key);
+	X509_set_pubkey(cert, pkey);
 	X509_set_issuer_name(cert, X509_get_subject_name(cacert));
 
 	/* Sign with our CA */
@@ -423,8 +416,6 @@ ssl_update_certificate(X509 *oldcert, char *keystr, off_t keylen,
 		ssl_error(__func__, name[0]);
 	if (bio != NULL)
 		BIO_free_all(bio);
-	if (key != NULL)
-		EVP_PKEY_free(key);
 	if (cacert != NULL)
 		X509_free(cacert);
 	if (cakey != NULL)
@@ -432,3 +423,67 @@ ssl_update_certificate(X509 *oldcert, char *keystr, off_t keylen,
 
 	return (cert);
 }
+
+int
+ssl_ctx_fake_private_key(SSL_CTX *ctx, void *data, char *buf, off_t len,
+    X509 **x509ptr, EVP_PKEY **pkeyptr)
+{
+	int		 ret = 0;
+	BIO		*in;
+	X509		*x509 = NULL;
+	EVP_PKEY	*pkey = NULL;
+	RSA		*rsa = NULL;
+
+	if ((in = BIO_new_mem_buf(buf, len)) == NULL) {
+		SSLerr(SSL_F_SSL_CTX_USE_PRIVATEKEY, ERR_R_BUF_LIB);
+		return (0);
+	}
+
+	if ((x509 = PEM_read_bio_X509(in, NULL,
+	    ctx->default_passwd_callback,
+	    ctx->default_passwd_callback_userdata)) == NULL) {
+		SSLerr(SSL_F_SSL_CTX_USE_PRIVATEKEY, ERR_R_PEM_LIB);
+		goto fail;
+	}
+
+	if ((pkey = X509_get_pubkey(x509)) == NULL) {
+		SSLerr(SSL_F_SSL_CTX_USE_PRIVATEKEY, ERR_R_X509_LIB);
+		goto fail;
+	}
+
+	if ((rsa = EVP_PKEY_get1_RSA(pkey)) == NULL) {
+		SSLerr(SSL_F_SSL_CTX_USE_PRIVATEKEY, ERR_R_EVP_LIB);
+		goto fail;
+	}
+
+	RSA_set_app_data(rsa, data);
+
+	/*
+	 * Use the public key as the "private" key - the secret key
+	 * parameters are hidden in an extra process that will be
+	 * contacted by the RSA engine.  The SSL/TLS library needs at
+	 * least the public key parameters in the current process.
+	 */
+	if (!SSL_CTX_use_PrivateKey(ctx, pkey)) {
+		SSLerr(SSL_F_SSL_CTX_USE_PRIVATEKEY, ERR_R_SSL_LIB);
+		goto fail;
+	}
+
+	*x509ptr = x509;
+	*pkeyptr = pkey;
+	ret = 1;
+
+	goto done;
+
+ fail:
+	if (pkey != NULL)
+		EVP_PKEY_free(pkey);
+	if (x509 != NULL)
+		X509_free(x509);
+
+ done:
+	if (in != NULL)
+		BIO_free(in);
+
+	return ret;
+}
diff --git a/usr.sbin/relayd/ssl_privsep.c b/usr.sbin/relayd/ssl_privsep.c
index bb58c052606..203a2700f58 100644
--- a/usr.sbin/relayd/ssl_privsep.c
+++ b/usr.sbin/relayd/ssl_privsep.c
@@ -1,4 +1,4 @@
-/*      $OpenBSD: ssl_privsep.c,v 1.9 2012/10/04 20:53:30 reyk Exp $    */
+/*      $OpenBSD: ssl_privsep.c,v 1.10 2014/04/18 13:55:26 reyk Exp $    */
 
 /* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com)
  * All rights reserved.
@@ -76,7 +76,6 @@
 #include <openssl/pem.h>
 #include <openssl/ssl.h>
 
-int	 ssl_ctx_use_private_key(SSL_CTX *, char *, off_t);
 int	 ssl_ctx_use_certificate_chain(SSL_CTX *, char *, off_t);
 int	 ssl_ctx_load_verify_memory(SSL_CTX *, char *, off_t);
 int	 ssl_by_mem_ctrl(X509_LOOKUP *, int, const char *, long, char **);
@@ -97,37 +96,6 @@ X509_LOOKUP_METHOD x509_mem_lookup = {
 #define X509_L_ADD_MEM	3
 
 int
-ssl_ctx_use_private_key(SSL_CTX *ctx, char *buf, off_t len)
-{
-	int		 ret;
-	BIO		*in;
-	EVP_PKEY	*pkey;
-
-	ret = 0;
-
-	if ((in = BIO_new_mem_buf(buf, len)) == NULL) {
-		SSLerr(SSL_F_SSL_CTX_USE_PRIVATEKEY_FILE, ERR_R_BUF_LIB);
-		return 0;
-	}
-
-	pkey = PEM_read_bio_PrivateKey(in, NULL,
-	    ctx->default_passwd_callback,
-	    ctx->default_passwd_callback_userdata);
-
-	if (pkey == NULL) {
-		SSLerr(SSL_F_SSL_CTX_USE_PRIVATEKEY_FILE, ERR_R_PEM_LIB);
-		goto end;
-	}
-	ret = SSL_CTX_use_PrivateKey(ctx, pkey);
-	EVP_PKEY_free(pkey);
-end:
-	if (in != NULL)
-		BIO_free(in);
-	return ret;
-}
-
-
-int
 ssl_ctx_use_certificate_chain(SSL_CTX *ctx, char *buf, off_t len)
 {
 	int		 ret;