Teach httpd/proc.c how to fork+exec.

This commit implemented the basic functions to proc.c to make it not rely
on global variables, malloc()ed memory and CLOEXEC pipes.

Fix child proc titles from reyk@
ok reyk@, florian@

3 files changed, 334 insertions, 173 deletions

diff --git a/usr.sbin/httpd/httpd.c b/usr.sbin/httpd/httpd.c
index 4c73252e5f7..8eaa781bbf0 100644
--- a/usr.sbin/httpd/httpd.c
+++ b/usr.sbin/httpd/httpd.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: httpd.c,v 1.59 2016/08/30 13:46:37 rzalamena Exp $	*/
+/*	$OpenBSD: httpd.c,v 1.60 2016/09/01 09:47:03 rzalamena Exp $	*/
 
 /*
  * Copyright (c) 2014 Reyk Floeter <reyk@openbsd.org>
@@ -117,8 +117,13 @@ main(int argc, char *argv[])
 	struct httpd		*env;
 	struct privsep		*ps;
 	const char		*conffile = CONF_FILE;
+	struct privsep_proc	*p;
+	enum privsep_procid	 proc_id = PROC_PARENT;
+	int			 proc_instance = 0;
+	const char		*errp, *title = NULL;
+	int			 argc0 = argc;
 
-	while ((c = getopt(argc, argv, "dD:nf:v")) != -1) {
+	while ((c = getopt(argc, argv, "dD:nf:I:P:v")) != -1) {
 		switch (c) {
 		case 'd':
 			debug = 2;
@@ -139,6 +144,18 @@ main(int argc, char *argv[])
 			verbose++;
 			opts |= HTTPD_OPT_VERBOSE;
 			break;
+		case 'P':
+			title = optarg;
+			proc_id = proc_getid(procs, nitems(procs), title);
+			if (proc_id == PROC_MAX)
+				fatalx("invalid process name");
+			break;
+		case 'I':
+			proc_instance = strtonum(optarg, 0,
+			    PROC_MAX_INSTANCES, &errp);
+			if (errp)
+				fatalx("invalid process instance");
+			break;
 		default:
 			usage();
 		}
@@ -177,15 +194,11 @@ main(int argc, char *argv[])
 	log_init(debug, LOG_DAEMON);
 	log_verbose(verbose);
 
-	if (!debug && daemon(1, 0) == -1)
-		err(1, "failed to daemonize");
-
 	if (env->sc_opts & HTTPD_OPT_NOACTION)
 		ps->ps_noaction = 1;
-	else
-		log_info("startup");
 
 	ps->ps_instances[PROC_SERVER] = env->sc_prefork_server;
+	ps->ps_instance = proc_instance;
 
 	if (env->sc_chroot == NULL)
 		env->sc_chroot = ps->ps_pw->pw_dir;
@@ -198,8 +211,32 @@ main(int argc, char *argv[])
 			errx(1, "malloc failed");
 	}
 
-	proc_init(ps, procs, nitems(procs));
+	if (proc_id != PROC_PARENT) {
+		p = NULL;
+		for (proc = 0; proc < nitems(procs); proc++) {
+			if (procs[proc].p_id != proc_id)
+				continue;
+
+			p = &procs[proc];
+			break;
+		}
+		if (p == NULL || p->p_init == NULL)
+			fatalx("%s: process %d missing process initialization",
+			    __func__, proc_id);
+
+		ps->ps_title[proc_id] = title;
+		p->p_init(ps, p);
+
+		fatalx("failed to initiate child process");
+	}
+
+	proc_init(ps, procs, nitems(procs), argc0, argv);
 	log_procinit("parent");
+	if (!debug && daemon(1, 0) == -1)
+		err(1, "failed to daemonize");
+
+	if (ps->ps_noaction == 0)
+		log_info("startup");
 
 	if (pledge("stdio rpath wpath cpath inet dns ioctl sendfd",
 	    NULL) == -1)
@@ -219,7 +256,7 @@ main(int argc, char *argv[])
 	signal_add(&ps->ps_evsigpipe, NULL);
 	signal_add(&ps->ps_evsigusr1, NULL);
 
-	proc_listen(ps, procs, nitems(procs));
+	proc_connect(ps);
 
 	if (load_config(env->sc_conffile, env) == -1) {
 		proc_kill(env->sc_ps);
diff --git a/usr.sbin/httpd/httpd.h b/usr.sbin/httpd/httpd.h
index 9e12cde6c7a..efed85d0c8a 100644
--- a/usr.sbin/httpd/httpd.h
+++ b/usr.sbin/httpd/httpd.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: httpd.h,v 1.114 2016/08/30 14:31:53 rzalamena Exp $	*/
+/*	$OpenBSD: httpd.h,v 1.115 2016/09/01 09:47:03 rzalamena Exp $	*/
 
 /*
  * Copyright (c) 2006 - 2015 Reyk Floeter <reyk@openbsd.org>
@@ -198,6 +198,7 @@ enum imsg_type {
 	IMSG_CTL_OK,
 	IMSG_CTL_FAIL,
 	IMSG_CTL_VERBOSE,
+	IMSG_CTL_PROCFD,
 	IMSG_CTL_RESET,
 	IMSG_CTL_SHUTDOWN,
 	IMSG_CTL_RELOAD,
@@ -226,6 +227,11 @@ enum privsep_procid {
 /* Attach the control socket to the following process */
 #define PROC_CONTROL	PROC_LOGGER
 
+/* Define default parent socket number */
+#define PARENT_SOCK_FILENO	3
+
+#define PROC_MAX_INSTANCES	128
+
 struct privsep_pipes {
 	int				*pp_pipes[PROC_MAX];
 };
@@ -270,6 +276,11 @@ struct privsep_proc {
 	struct httpd		*p_env;
 };
 
+struct privsep_fd {
+	enum privsep_procid		 pf_procid;
+	unsigned int			 pf_instance;
+};
+
 enum fcgistate {
 	FCGI_READ_HEADER,
 	FCGI_READ_CONTENT,
@@ -687,11 +698,13 @@ __dead void fatalx(const char *, ...)
 	    __attribute__((__format__ (printf, 1, 2)));
 
 /* proc.c */
-void	 proc_init(struct privsep *, struct privsep_proc *, unsigned int);
+enum privsep_procid
+	    proc_getid(struct privsep_proc *, unsigned int, const char *);
+void	 proc_init(struct privsep *, struct privsep_proc *, unsigned int, int, char **);
 void	 proc_kill(struct privsep *);
-void	 proc_listen(struct privsep *, struct privsep_proc *, size_t);
+void	 proc_connect(struct privsep *);
 void	 proc_dispatch(int, short event, void *);
-pid_t	 proc_run(struct privsep *, struct privsep_proc *,
+void	 proc_run(struct privsep *, struct privsep_proc *,
 	    struct privsep_proc *, unsigned int,
 	    void (*)(struct privsep *, struct privsep_proc *, void *), void *);
 void	 proc_range(struct privsep *, enum privsep_procid, int *, int *);
diff --git a/usr.sbin/httpd/proc.c b/usr.sbin/httpd/proc.c
index 2e5c0ae0362..9422993dbc2 100644
--- a/usr.sbin/httpd/proc.c
+++ b/usr.sbin/httpd/proc.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: proc.c,v 1.19 2016/08/30 14:31:53 rzalamena Exp $	*/
+/*	$OpenBSD: proc.c,v 1.20 2016/09/01 09:47:03 rzalamena Exp $	*/
 
 /*
  * Copyright (c) 2010 - 2014 Reyk Floeter <reyk@openbsd.org>
@@ -34,8 +34,14 @@
 
 #include "httpd.h"
 
-void	 proc_open(struct privsep *, struct privsep_proc *,
-	    struct privsep_proc *, size_t);
+void	 proc_exec(struct privsep *, struct privsep_proc *, unsigned int,
+	    int, char **);
+void	 proc_connectpeer(struct privsep *, enum privsep_procid, int,
+	    struct privsep_pipes *);
+void	 proc_setup(struct privsep *, struct privsep_proc *, unsigned int);
+void	 proc_open(struct privsep *, int, int);
+void	 proc_listento(struct privsep *, int, enum privsep_procid,
+	    unsigned int);
 void	 proc_close(struct privsep *);
 int	 proc_ispeer(struct privsep_proc *, unsigned int, enum privsep_procid);
 void	 proc_shutdown(struct privsep_proc *);
@@ -55,16 +61,221 @@ proc_ispeer(struct privsep_proc *procs, unsigned int nproc,
 	return (0);
 }
 
+enum privsep_procid
+proc_getid(struct privsep_proc *procs, unsigned int nproc,
+    const char *proc_name)
+{
+	struct privsep_proc	*p;
+	unsigned int		 proc;
+
+	for (proc = 0; proc < nproc; proc++) {
+		p = &procs[proc];
+		if (strcmp(p->p_title, proc_name))
+			continue;
+
+		return (p->p_id);
+	}
+
+	return (PROC_MAX);
+}
+
+void
+proc_exec(struct privsep *ps, struct privsep_proc *procs, unsigned int nproc,
+    int argc, char **argv)
+{
+	unsigned int		 proc, nargc, i, proc_i;
+	char			**nargv;
+	struct privsep_proc	*p;
+	char			 num[32];
+	int			 fd;
+
+	/* Prepare the new process argv. */
+	nargv = calloc(argc + 5, sizeof(char *));
+	if (nargv == NULL)
+		fatal("%s: calloc", __func__);
+
+	/* Copy call argument first. */
+	nargc = 0;
+	nargv[nargc++] = argv[0];
+
+	/* Set process name argument and save the position. */
+	nargv[nargc++] = "-P";
+	proc_i = nargc;
+	nargc++;
+
+	/* Point process instance arg to stack and copy the original args. */
+	nargv[nargc++] = "-I";
+	nargv[nargc++] = num;
+	for (i = 1; i < (unsigned int) argc; i++)
+		nargv[nargc++] = argv[i];
+
+	nargv[nargc] = NULL;
+
+	for (proc = 0; proc < nproc; proc++) {
+		p = &procs[proc];
+
+		/* Update args with process title. */
+		nargv[proc_i] = (char *) p->p_title;
+
+		/* Fire children processes. */
+		for (i = 0; i < ps->ps_instances[p->p_id]; i++) {
+			/* Update the process instance number. */
+			snprintf(num, sizeof(num), "%u", i);
+
+			fd = ps->ps_pipes[p->p_id][i].pp_pipes[PROC_PARENT][0];
+			ps->ps_pipes[p->p_id][i].pp_pipes[PROC_PARENT][0] = -1;
+
+			switch (fork()) {
+			case -1:
+				fatal("%s: fork", __func__);
+				break;
+			case 0:
+				/* Prepare parent socket. */
+				dup2(fd, PARENT_SOCK_FILENO);
+
+				execvp(argv[0], nargv);
+				fatal("%s: execvp", __func__);
+				break;
+			default:
+				/* Close child end. */
+				close(fd);
+				break;
+			}
+		}
+	}
+	free(nargv);
+}
+
 void
-proc_init(struct privsep *ps, struct privsep_proc *procs, unsigned int nproc)
+proc_connectpeer(struct privsep *ps, enum privsep_procid id, int inst,
+    struct privsep_pipes *pp)
 {
-	unsigned int		 i, j, src, dst;
+	unsigned int		 i, j;
+	struct privsep_fd	 pf;
+
+	for (i = 0; i < PROC_MAX; i++) {
+		/* Parent is already connected with everyone. */
+		if (i == PROC_PARENT)
+			continue;
+
+		for (j = 0; j < ps->ps_instances[i]; j++) {
+			/* Don't send socket to child itself. */
+			if (i == (unsigned int)id &&
+			    j == (unsigned int)inst)
+				continue;
+			if (pp->pp_pipes[i][j] == -1)
+				continue;
+
+			pf.pf_procid = i;
+			pf.pf_instance = j;
+			proc_compose_imsg(ps, id, inst, IMSG_CTL_PROCFD,
+			    -1, pp->pp_pipes[i][j], &pf, sizeof(pf));
+			pp->pp_pipes[i][j] = -1;
+		}
+	}
+}
+
+/* Inter-connect all process except with ourself. */
+void
+proc_connect(struct privsep *ps)
+{
+	unsigned int		 src, i, j;
 	struct privsep_pipes	*pp;
+	struct imsgev		*iev;
 
-	for (src = 0; src < PROC_MAX; src++) {
+	/* Listen on appropriate pipes. */
+	src = privsep_process;
+	pp = &ps->ps_pipes[src][ps->ps_instance];
+
+	for (i = 0; i < PROC_MAX; i++) {
+		/* Don't listen to ourself. */
+		if (i == src)
+			continue;
+
+		for (j = 0; j < ps->ps_instances[i]; j++) {
+			if (pp->pp_pipes[i][j] == -1)
+				continue;
+
+			iev = &ps->ps_ievs[i][j];
+			imsg_init(&iev->ibuf, pp->pp_pipes[i][j]);
+			event_set(&iev->ev, iev->ibuf.fd, iev->events,
+			    iev->handler, iev->data);
+			event_add(&iev->ev, NULL);
+		}
+	}
+
+	/* Exchange pipes between process. */
+	for (i = 0; i < PROC_MAX; i++) {
+		/* Parent is already connected with everyone. */
+		if (i == PROC_PARENT)
+			continue;
+
+		for (j = 0; j < ps->ps_instances[i]; j++)
+			proc_connectpeer(ps, i, j, &ps->ps_pipes[i][j]);
+	}
+}
+
+void
+proc_listento(struct privsep *ps, int fd, enum privsep_procid dst, unsigned int n)
+{
+	struct privsep_pipes	*pp = ps->ps_pp;
+	struct imsgev		*iev;
+
+	if (ps->ps_ievs[dst] == NULL) {
+#if DEBUG > 1
+		log_debug("%s: privsep_process=%d instance=%d dst=%d instance=%d not allocated",
+		    __func__, privsep_process, ps->ps_instance, dst, n);
+#endif /* DEBUG > 1 */
+		close(fd);
+		return;
+	}
+
+	if (pp->pp_pipes[dst][n] != -1) {
+		log_warnx("%s: duplicated descriptor", __func__);
+		close(fd);
+		return;
+	} else
+		pp->pp_pipes[dst][n] = fd;
+
+	iev = &ps->ps_ievs[dst][n];
+	imsg_init(&iev->ibuf, fd);
+	event_set(&iev->ev, iev->ibuf.fd, iev->events, iev->handler, iev->data);
+	event_add(&iev->ev, NULL);
+}
+
+void
+proc_setup(struct privsep *ps, struct privsep_proc *procs, unsigned int nproc)
+{
+	unsigned int		 i, j, src, dst, id;
+	struct privsep_pipes	*pp;
+
+	/* Initialize parent title, ps_instances and procs. */
+	ps->ps_title[PROC_PARENT] = "parent";
+
+	for (src = 0; src < PROC_MAX; src++)
 		/* Default to 1 process instance */
 		if (ps->ps_instances[src] < 1)
 			ps->ps_instances[src] = 1;
+
+	for (src = 0; src < nproc; src++) {
+		procs[src].p_ps = ps;
+		procs[src].p_env = ps->ps_env;
+		if (procs[src].p_cb == NULL)
+			procs[src].p_cb = proc_dispatch_null;
+
+		id = procs[src].p_id;
+		ps->ps_title[id] = procs[src].p_title;
+		if ((ps->ps_ievs[id] = calloc(ps->ps_instances[id],
+		    sizeof(struct imsgev))) == NULL)
+			fatal(__func__);
+
+		/* With this set up, we are ready to call imsg_init(). */
+		for (i = 0; i < ps->ps_instances[id]; i++) {
+			ps->ps_ievs[id][i].handler = proc_dispatch;
+			ps->ps_ievs[id][i].events = EV_READ;
+			ps->ps_ievs[id][i].proc = &procs[src];
+			ps->ps_ievs[id][i].data = &ps->ps_ievs[id][i];
+		}
 	}
 
 	/*
@@ -84,7 +295,7 @@ proc_init(struct privsep *ps, struct privsep_proc *procs, unsigned int nproc)
 		/* Allocate destination array for each process */
 		if ((ps->ps_pipes[src] = calloc(ps->ps_instances[src],
 		    sizeof(struct privsep_pipes))) == NULL)
-			fatal("proc_init: calloc");
+			fatal("%s: calloc", __func__);
 
 		for (i = 0; i < ps->ps_instances[src]; i++) {
 			pp = &ps->ps_pipes[src][i];
@@ -94,7 +305,7 @@ proc_init(struct privsep *ps, struct privsep_proc *procs, unsigned int nproc)
 				if ((pp->pp_pipes[dst] =
 				    calloc(ps->ps_instances[dst],
 				    sizeof(int))) == NULL)
-					fatal("proc_init: calloc");
+					fatal("%s: calloc", __func__);
 
 				/* Mark fd as unused */
 				for (j = 0; j < ps->ps_instances[dst]; j++)
@@ -103,22 +314,25 @@ proc_init(struct privsep *ps, struct privsep_proc *procs, unsigned int nproc)
 		}
 	}
 
-	/*
-	 * Setup and run the parent and its children
-	 */
-	privsep_process = PROC_PARENT;
-	ps->ps_instances[PROC_PARENT] = 1;
-	ps->ps_title[PROC_PARENT] = "parent";
-	ps->ps_pp = &ps->ps_pipes[privsep_process][0];
+	ps->ps_pp = &ps->ps_pipes[privsep_process][ps->ps_instance];
+}
 
-	for (i = 0; i < nproc; i++)
-		ps->ps_title[procs[i].p_id] = procs[i].p_title;
+void
+proc_init(struct privsep *ps, struct privsep_proc *procs, unsigned int nproc,
+    int argc, char **argv)
+{
+	unsigned int		 src, dst;
+
+	privsep_process = PROC_PARENT;
+	proc_setup(ps, procs, nproc);
 
-	proc_open(ps, NULL, procs, nproc);
+	/* Open socketpair()s for everyone. */
+	for (src = 0; src < PROC_MAX; src++)
+		for (dst = 0; dst < PROC_MAX; dst++)
+			proc_open(ps, src, dst);
 
 	/* Engage! */
-	for (i = 0; i < nproc; i++)
-		(*procs[i].p_init)(ps, &procs[i]);
+	proc_exec(ps, procs, nproc, argc, argv);
 }
 
 void
@@ -158,115 +372,30 @@ proc_kill(struct privsep *ps)
 }
 
 void
-proc_open(struct privsep *ps, struct privsep_proc *p,
-    struct privsep_proc *procs, size_t nproc)
+proc_open(struct privsep *ps, int src, int dst)
 {
 	struct privsep_pipes	*pa, *pb;
 	int			 fds[2];
-	unsigned int		 i, j, src, proc;
-
-	if (p == NULL)
-		src = privsep_process; /* parent */
-	else
-		src = p->p_id;
+	unsigned int		 i, j;
 
-	/*
-	 * Open socket pairs for our peers
-	 */
-	for (proc = 0; proc < nproc; proc++) {
-		procs[proc].p_ps = ps;
-		procs[proc].p_env = ps->ps_env;
-		if (procs[proc].p_cb == NULL)
-			procs[proc].p_cb = proc_dispatch_null;
-
-		for (i = 0; i < ps->ps_instances[src]; i++) {
-			for (j = 0; j < ps->ps_instances[procs[proc].p_id];
-			    j++) {
-				pa = &ps->ps_pipes[src][i];
-				pb = &ps->ps_pipes[procs[proc].p_id][j];
-
-				/* Check if fds are already set by peer */
-				if (pa->pp_pipes[procs[proc].p_id][j] != -1)
-					continue;
-
-				if (socketpair(AF_UNIX,
-				    SOCK_STREAM | SOCK_NONBLOCK,
-				    PF_UNSPEC, fds) == -1)
-					fatal("socketpair");
-
-				pa->pp_pipes[procs[proc].p_id][j] = fds[0];
-				pb->pp_pipes[src][i] = fds[1];
-			}
-		}
-	}
-}
-
-void
-proc_listen(struct privsep *ps, struct privsep_proc *procs, size_t nproc)
-{
-	unsigned int		 i, dst, src, n, m;
-	struct privsep_pipes	*pp;
-
-	/*
-	 * Close unused pipes
-	 */
-	for (src = 0; src < PROC_MAX; src++) {
-		for (n = 0; n < ps->ps_instances[src]; n++) {
-			/* Ingore current process */
-			if (src == (unsigned int)privsep_process &&
-			    n == ps->ps_instance)
+	for (i = 0; i < ps->ps_instances[src]; i++) {
+		for (j = 0; j < ps->ps_instances[dst]; j++) {
+			/* Don't create sockets for ourself. */
+			if (src == dst && i == j)
 				continue;
 
-			pp = &ps->ps_pipes[src][n];
-
-			for (dst = 0; dst < PROC_MAX; dst++) {
-				if (src == dst)
-					continue;
-				for (m = 0; m < ps->ps_instances[dst]; m++) {
-					if (pp->pp_pipes[dst][m] == -1)
-						continue;
-
-					/* Close and invalidate fd */
-					close(pp->pp_pipes[dst][m]);
-					pp->pp_pipes[dst][m] = -1;
-				}
-			}
-		}
-	}
-
-	src = privsep_process;
-	ps->ps_pp = pp = &ps->ps_pipes[src][ps->ps_instance];
-
-	/*
-	 * Listen on appropriate pipes
-	 */
-	for (i = 0; i < nproc; i++) {
-		dst = procs[i].p_id;
-
-		if (src == dst)
-			fatal("proc_listen: cannot peer with oneself");
-
-		if ((ps->ps_ievs[dst] = calloc(ps->ps_instances[dst],
-		    sizeof(struct imsgev))) == NULL)
-			fatal("proc_open");
-
-		for (n = 0; n < ps->ps_instances[dst]; n++) {
-			if (pp->pp_pipes[dst][n] == -1)
+			pa = &ps->ps_pipes[src][i];
+			pb = &ps->ps_pipes[dst][j];
+			if (pb->pp_pipes[dst][j] != -1)
 				continue;
 
-			imsg_init(&(ps->ps_ievs[dst][n].ibuf),
-			    pp->pp_pipes[dst][n]);
-			ps->ps_ievs[dst][n].handler = proc_dispatch;
-			ps->ps_ievs[dst][n].events = EV_READ;
-			ps->ps_ievs[dst][n].proc = &procs[i];
-			ps->ps_ievs[dst][n].data = &ps->ps_ievs[dst][n];
-
-			event_set(&(ps->ps_ievs[dst][n].ev),
-			    ps->ps_ievs[dst][n].ibuf.fd,
-			    ps->ps_ievs[dst][n].events,
-			    ps->ps_ievs[dst][n].handler,
-			    ps->ps_ievs[dst][n].data);
-			event_add(&(ps->ps_ievs[dst][n].ev), NULL);
+			if (socketpair(AF_UNIX,
+			    SOCK_STREAM | SOCK_NONBLOCK | SOCK_CLOEXEC,
+			    PF_UNSPEC, fds) == -1)
+				fatal(__func__);
+
+			pa->pp_pipes[dst][j] = fds[0];
+			pb->pp_pipes[src][i] = fds[1];
 		}
 	}
 }
@@ -315,7 +444,7 @@ proc_shutdown(struct privsep_proc *p)
 
 	log_info("%s exiting, pid %d", p->p_title, getpid());
 
-	_exit(0);
+	exit(0);
 }
 
 void
@@ -340,37 +469,22 @@ proc_sig_handler(int sig, short event, void *arg)
 	}
 }
 
-pid_t
+void
 proc_run(struct privsep *ps, struct privsep_proc *p,
     struct privsep_proc *procs, unsigned int nproc,
     void (*run)(struct privsep *, struct privsep_proc *, void *), void *arg)
 {
-	pid_t			 pid;
-	struct passwd		*pw;
+	struct passwd		*pw = ps->ps_pw;
 	const char		*root;
 	struct control_sock	*rcs;
-	unsigned int		 n;
 
 	if (ps->ps_noaction)
-		return (0);
+		exit(0);
 
-	proc_open(ps, p, procs, nproc);
+	log_procinit(p->p_title);
 
-	/* Fork child handlers */
-	switch (pid = fork()) {
-	case -1:
-		fatal("proc_run: cannot fork");
-	case 0:
-		log_procinit(p->p_title);
-
-		/* Set the process group of the current process */
-		setpgid(0, 0);
-		break;
-	default:
-		return (pid);
-	}
-
-	pw = ps->ps_pw;
+	/* Set the process group of the current process */
+	setpgid(0, 0);
 
 	if (p->p_id == PROC_CONTROL && ps->ps_instance == 0) {
 		if (control_init(ps, &ps->ps_csock) == -1)
@@ -400,19 +514,6 @@ proc_run(struct privsep *ps, struct privsep_proc *p,
 	    setresuid(pw->pw_uid, pw->pw_uid, pw->pw_uid))
 		fatal("proc_run: cannot drop privileges");
 
-	/* Fork child handlers */
-	for (n = 1; n < ps->ps_instances[p->p_id]; n++) {
-		if (fork() == 0) {
-			ps->ps_instance = n;
-			break;
-		}
-	}
-
-#ifdef DEBUG
-	log_debug("%s: %s %d/%d, pid %d", __func__, p->p_title,
-	    ps->ps_instance + 1, ps->ps_instances[p->p_id], getpid());
-#endif
-
 	event_init();
 
 	signal_set(&ps->ps_evsigint, SIGINT, proc_sig_handler, p);
@@ -429,8 +530,8 @@ proc_run(struct privsep *ps, struct privsep_proc *p,
 	signal_add(&ps->ps_evsigpipe, NULL);
 	signal_add(&ps->ps_evsigusr1, NULL);
 
-	proc_listen(ps, procs, nproc);
-
+	proc_setup(ps, procs, nproc);
+	proc_listento(ps, PARENT_SOCK_FILENO, PROC_PARENT, 0);
 	if (p->p_id == PROC_CONTROL && ps->ps_instance == 0) {
 		TAILQ_INIT(&ctl_conns);
 		if (control_listen(&ps->ps_csock) == -1)
@@ -440,14 +541,17 @@ proc_run(struct privsep *ps, struct privsep_proc *p,
 				fatalx(__func__);
 	}
 
+#ifdef DEBUG
+	log_debug("%s: %s %d/%d, pid %d", __func__, p->p_title,
+	    ps->ps_instance + 1, ps->ps_instances[p->p_id], getpid());
+#endif
+
 	if (run != NULL)
 		run(ps, p, arg);
 
 	event_dispatch();
 
 	proc_shutdown(p);
-
-	return (0);
 }
 
 void
@@ -461,6 +565,7 @@ proc_dispatch(int fd, short event, void *arg)
 	ssize_t			 n;
 	int			 verbose;
 	const char		*title;
+	struct privsep_fd	 pf;
 
 	title = ps->ps_title[privsep_process];
 	ibuf = &iev->ibuf;
@@ -511,6 +616,12 @@ proc_dispatch(int fd, short event, void *arg)
 			memcpy(&verbose, imsg.data, sizeof(verbose));
 			log_verbose(verbose);
 			break;
+		case IMSG_CTL_PROCFD:
+			IMSG_SIZE_CHECK(&imsg, &pf);
+			memcpy(&pf, imsg.data, sizeof(pf));
+			proc_listento(ps, imsg.fd, pf.pf_procid,
+			    pf.pf_instance);
+			break;
 		default:
 			log_warnx("%s: %s %d got invalid imsg %d peerid %d "
 			    "from %s %d",