Add vmctl send and vmctl receive

ok reyk@ and mlarkin@

12 files changed, 643 insertions, 74 deletions

diff --git a/usr.sbin/vmctl/Makefile b/usr.sbin/vmctl/Makefile
index cf5e25aebc1..61b8ca7b262 100644
--- a/usr.sbin/vmctl/Makefile
+++ b/usr.sbin/vmctl/Makefile
@@ -1,9 +1,11 @@
-#	$OpenBSD: Makefile,v 1.3 2016/10/26 05:26:36 mlarkin Exp $
+#	$OpenBSD: Makefile,v 1.4 2017/07/15 05:05:36 pd Exp $
 
 .if ${MACHINE} == "amd64" || ${MACHINE} == "i386"
 
+.PATH:	${.CURDIR}/../vmd
+
 PROG=	vmctl
-SRCS=	vmctl.c main.c
+SRCS=	vmctl.c main.c atomicio.c
 CFLAGS+= -Wall
 CFLAGS+= -Wstrict-prototypes -Wmissing-prototypes
 CFLAGS+= -Wmissing-declarations
@@ -12,6 +14,7 @@ CFLAGS+= -Wsign-compare
 CFLAGS+= -I${.CURDIR} -I${.CURDIR}/../vmd
 LDADD+=	-lutil
 DPADD+= ${LIBUTIL}
+PATH+=../vmd
 
 .else
 
diff --git a/usr.sbin/vmctl/main.c b/usr.sbin/vmctl/main.c
index d1c0badf8ce..2749f9722ff 100644
--- a/usr.sbin/vmctl/main.c
+++ b/usr.sbin/vmctl/main.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: main.c,v 1.30 2017/07/09 00:51:40 pd Exp $	*/
+/*	$OpenBSD: main.c,v 1.31 2017/07/15 05:05:36 pd Exp $	*/
 
 /*
  * Copyright (c) 2015 Reyk Floeter <reyk@openbsd.org>
@@ -58,6 +58,8 @@ int		 ctl_status(struct parse_result *, int, char *[]);
 int		 ctl_stop(struct parse_result *, int, char *[]);
 int		 ctl_pause(struct parse_result *, int, char *[]);
 int		 ctl_unpause(struct parse_result *, int, char *[]);
+int		 ctl_send(struct parse_result *, int, char *[]);
+int		 ctl_receive(struct parse_result *, int, char *[]);
 
 struct ctl_command ctl_commands[] = {
 	{ "console",	CMD_CONSOLE,	ctl_console,	"id" },
@@ -73,6 +75,8 @@ struct ctl_command ctl_commands[] = {
 	{ "stop",	CMD_STOP,	ctl_stop,	"id" },
 	{ "pause",	CMD_PAUSE,	ctl_pause,	"id" },
 	{ "unpause",	CMD_UNPAUSE,	ctl_unpause,	"id" },
+	{ "send",	CMD_SEND,	ctl_send,	"id",	1},
+	{ "receive",	CMD_RECEIVE,	ctl_receive,	"id" ,	1},
 	{ NULL }
 };
 
@@ -235,6 +239,13 @@ vmmaction(struct parse_result *res)
 	case CMD_UNPAUSE:
 		unpause_vm(res->id, res->name);
 		break;
+	case CMD_SEND:
+		send_vm(res->id, res->name);
+		done = 1;
+		break;
+	case CMD_RECEIVE:
+		vm_receive(res->id, res->name);
+		break;
 	case CMD_CREATE:
 	case NONE:
 		break;
@@ -288,6 +299,9 @@ vmmaction(struct parse_result *res)
 			case CMD_PAUSE:
 				done = pause_vm_complete(&imsg, &ret);
 				break;
+			case CMD_RECEIVE:
+				done = vm_start_complete(&imsg, &ret, 0);
+				break;
 			case CMD_UNPAUSE:
 				done = unpause_vm_complete(&imsg, &ret);
 				break;
@@ -427,6 +441,33 @@ parse_vmid(struct parse_result *res, char *word)
 }
 
 int
+parse_vmname(struct parse_result *res, char *word)
+{
+	const char	*error;
+	uint32_t	 id;
+
+	if (word == NULL) {
+		warnx("missing vmid argument");
+		return (-1);
+	}
+	id = strtonum(word, 0, UINT32_MAX, &error);
+	if (error == NULL) {
+		warnx("invalid vm name");
+		return (-1);
+	} else {
+		if (strlen(word) >= VMM_MAX_NAME_LEN) {
+			warnx("name too long");
+			return (-1);
+		}
+		res->id = 0;
+		if ((res->name = strdup(word)) == NULL)
+			errx(1, "strdup");
+	}
+
+	return (0);
+}
+
+int
 ctl_create(struct parse_result *res, int argc, char *argv[])
 {
 	int		 ch, ret;
@@ -656,6 +697,34 @@ ctl_unpause(struct parse_result *res, int argc, char *argv[])
 	return (vmmaction(res));
 }
 
+int
+ctl_send(struct parse_result *res, int argc, char *argv[])
+{
+	if (pledge("stdio unix sendfd", NULL) == -1)
+		err(1, "pledge");
+	if (argc == 2) {
+		if (parse_vmid(res, argv[1]) == -1)
+			errx(1, "invalid id: %s", argv[1]);
+	} else if (argc != 2)
+		ctl_usage(res->ctl);
+
+	return (vmmaction(res));
+}
+
+int
+ctl_receive(struct parse_result *res, int argc, char *argv[])
+{
+	if (pledge("stdio unix sendfd", NULL) == -1)
+		err(1, "pledge");
+	if (argc == 2) {
+		if (parse_vmname(res, argv[1]) == -1)
+			errx(1, "invalid id: %s", argv[1]);
+	} else if (argc != 2)
+		ctl_usage(res->ctl);
+
+	return (vmmaction(res));
+}
+
 __dead void
 ctl_openconsole(const char *name)
 {
diff --git a/usr.sbin/vmctl/vmctl.8 b/usr.sbin/vmctl/vmctl.8
index 71a0aa4e7f6..353aec23f01 100644
--- a/usr.sbin/vmctl/vmctl.8
+++ b/usr.sbin/vmctl/vmctl.8
@@ -1,4 +1,4 @@
-.\"	$OpenBSD: vmctl.8,v 1.29 2017/04/19 15:38:32 reyk Exp $
+.\"	$OpenBSD: vmctl.8,v 1.30 2017/07/15 05:05:36 pd Exp $
 .\"
 .\" Copyright (c) 2015 Mike Larkin <mlarkin@openbsd.org>
 .\"
@@ -14,7 +14,7 @@
 .\" ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
 .\" OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 .\"
-.Dd $Mdocdate: April 19 2017 $
+.Dd $Mdocdate: July 15 2017 $
 .Dt VMCTL 8
 .Os
 .Sh NAME
@@ -62,6 +62,12 @@ Load additional configuration from the specified file.
 Disable verbose debug logging.
 .It Cm log verbose
 Enable verbose debug logging.
+.It Cm pause Ar id
+Pause a VM with the specified
+.Ar id .
+.It Cm receive Ar name
+Receive a VM from standard input and start it with the specified
+.Ar name .
 .It Cm reload
 Remove all stopped VMs and reload the configuration from the default
 configuration file.
@@ -71,6 +77,10 @@ Reset the running state.
 Reset the configured switches.
 .It Cm reset vms
 Reset and terminate all VMs.
+.It Cm send Ar id
+Send a VM with the specified
+.Ar id
+to standard output and terminate it.
 .It Xo Cm start Ar name
 .Op Fl Lc
 .Op Fl b Ar path
@@ -122,6 +132,9 @@ A graceful shutdown will be attempted if the VM supports the
 device.
 Once stopped, if the VM was not defined in a configuration file, then it is
 removed.
+.It Cm unpause Ar id
+Unpause (resume from a paused state) a vm with the specified
+.Ar id .
 .El
 .Pp
 If the
diff --git a/usr.sbin/vmctl/vmctl.c b/usr.sbin/vmctl/vmctl.c
index 1ae0e1d42e6..3335f30b7d4 100644
--- a/usr.sbin/vmctl/vmctl.c
+++ b/usr.sbin/vmctl/vmctl.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: vmctl.c,v 1.31 2017/07/09 00:51:40 pd Exp $	*/
+/*	$OpenBSD: vmctl.c,v 1.32 2017/07/15 05:05:36 pd Exp $	*/
 
 /*
  * Copyright (c) 2014 Mike Larkin <mlarkin@openbsd.org>
@@ -40,6 +40,7 @@
 
 #include "vmd.h"
 #include "vmctl.h"
+#include "atomicio.h"
 
 extern char *__progname;
 uint32_t info_id;
@@ -197,6 +198,68 @@ vm_start_complete(struct imsg *imsg, int *ret, int autoconnect)
 }
 
 void
+send_vm(uint32_t id, const char *name)
+{
+	struct vmop_id vid;
+	int fds[2], readn, writen;
+	char buf[PAGE_SIZE];
+
+	memset(&vid, 0, sizeof(vid));
+	vid.vid_id = id;
+	if (name != NULL)
+		strlcpy(vid.vid_name, name, sizeof(vid.vid_name));
+	if (socketpair(AF_UNIX, SOCK_STREAM, PF_UNSPEC, fds) == -1) {
+		warnx("%s: socketpair creation failed", __func__);
+	} else {
+		imsg_compose(ibuf, IMSG_VMDOP_SEND_VM_REQUEST, 0, 0, fds[0],
+				&vid, sizeof(vid));
+		imsg_flush(ibuf);
+		while (1) {
+			readn = atomicio(read, fds[1], buf, sizeof(buf));
+			if (!readn)
+				break;
+			writen = atomicio(vwrite, STDOUT_FILENO, buf,
+					readn);
+			if (writen != readn)
+				break;
+		}
+		if (vid.vid_id)
+			warnx("sent vm %d successfully", vid.vid_id);
+		else
+			warnx("sent vm %s successfully", vid.vid_name);
+	}
+}
+
+void
+vm_receive(uint32_t id, const char *name)
+{
+	struct vmop_id vid;
+	int fds[2], readn, writen;
+	char buf[PAGE_SIZE];
+
+	memset(&vid, 0, sizeof(vid));
+	if (name != NULL)
+		strlcpy(vid.vid_name, name, sizeof(vid.vid_name));
+	if (socketpair(AF_UNIX, SOCK_STREAM, PF_UNSPEC, fds) == -1) {
+		warnx("%s: socketpair creation failed", __func__);
+	} else {
+		imsg_compose(ibuf, IMSG_VMDOP_RECEIVE_VM_REQUEST, 0, 0, fds[0],
+		    &vid, sizeof(vid));
+		imsg_flush(ibuf);
+		while (1) {
+			readn = atomicio(read, STDIN_FILENO, buf, sizeof(buf));
+			if (!readn) {
+				close(fds[1]);
+				break;
+			}
+			writen = atomicio(vwrite, fds[1], buf, readn);
+			if (writen != readn)
+				break;
+		}
+	}
+}
+
+void
 pause_vm(uint32_t pause_id, const char *name)
 {
 	struct vmop_id vid;
diff --git a/usr.sbin/vmctl/vmctl.h b/usr.sbin/vmctl/vmctl.h
index ca851dc8933..c68a433d59e 100644
--- a/usr.sbin/vmctl/vmctl.h
+++ b/usr.sbin/vmctl/vmctl.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: vmctl.h,v 1.15 2017/07/09 00:51:40 pd Exp $	*/
+/*	$OpenBSD: vmctl.h,v 1.16 2017/07/15 05:05:36 pd Exp $	*/
 
 /*
  * Copyright (c) 2015 Reyk Floeter <reyk@openbsd.org>
@@ -34,6 +34,8 @@ enum actions {
 	CMD_STOP,
 	CMD_PAUSE,
 	CMD_UNPAUSE,
+	CMD_SEND,
+	CMD_RECEIVE,
 };
 
 struct ctl_command;
@@ -72,6 +74,7 @@ int	 parse_network(struct parse_result *, char *);
 int	 parse_size(struct parse_result *, char *, long long);
 int	 parse_disk(struct parse_result *, char *);
 int	 parse_vmid(struct parse_result *, char *);
+int	 parse_vmname(struct parse_result *, char *);
 void	 parse_free(struct parse_result *);
 int	 parse(int, char *[]);
 __dead void
@@ -88,6 +91,9 @@ void	 pause_vm(uint32_t, const char *);
 int	 pause_vm_complete(struct imsg *, int *);
 void	 unpause_vm(uint32_t, const char *);
 int	 unpause_vm_complete(struct imsg *, int *);
+void	 send_vm(uint32_t, const char *);
+void	 vm_receive(uint32_t, const char *);
+int	 receive_vm_complete(struct imsg *, int *);
 int	 check_info_id(const char *, uint32_t);
 void	 get_info_vm(uint32_t, const char *, int);
 int	 add_info(struct imsg *, int *);
diff --git a/usr.sbin/vmd/config.c b/usr.sbin/vmd/config.c
index 7d738a3cc64..7a37558d9a2 100644
--- a/usr.sbin/vmd/config.c
+++ b/usr.sbin/vmd/config.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: config.c,v 1.31 2017/05/04 08:26:06 reyk Exp $	*/
+/*	$OpenBSD: config.c,v 1.32 2017/07/15 05:05:36 pd Exp $	*/
 
 /*
  * Copyright (c) 2015 Reyk Floeter <reyk@openbsd.org>
@@ -41,6 +41,7 @@
 /* Supported bridge types */
 const char *vmd_descsw[] = { "switch", "bridge", NULL };
 
+
 int
 config_init(struct vmd *env)
 {
@@ -191,28 +192,34 @@ config_setvm(struct privsep *ps, struct vmd_vm *vm, uint32_t peerid, uid_t uid)
 	vm->vm_peerid = peerid;
 	vm->vm_uid = uid;
 
-	if (strlen(vcp->vcp_kernel)) {
-		/* Boot kernel from disk image if path matches the root disk */
-		if (vcp->vcp_ndisks &&
-		    strcmp(vcp->vcp_kernel, vcp->vcp_disks[0]) == 0)
-			vmboot = 1;
-		/* Open external kernel for child */
-		else if ((kernfd = open(vcp->vcp_kernel, O_RDONLY)) == -1) {
-			log_warn("%s: can't open kernel/BIOS boot image %s",
-			    __func__, vcp->vcp_kernel);
-			goto fail;
+	if (!vm->vm_received) {
+		if (strlen(vcp->vcp_kernel)) {
+			/* Boot kernel from disk image if path matches the root
+			 * disk */
+			if (vcp->vcp_ndisks &&
+			    strcmp(vcp->vcp_kernel, vcp->vcp_disks[0]) == 0)
+				vmboot = 1;
+			/* Open external kernel for child */
+			else if ((kernfd = open(vcp->vcp_kernel, O_RDONLY)) ==
+			    -1) {
+				log_warn("%s: can't open kernel/BIOS boot image\
+						%s", __func__, vcp->vcp_kernel);
+				goto fail;
+			}
 		}
-	}
 
-	/*
-	 * Try to open the default BIOS image if no kernel/BIOS has
-	 * been specified.  The BIOS is an external firmware file that is
-	 * typically distributed separately due to an incompatible license.
-	 */
-	if (kernfd == -1 && !vmboot &&
-	    (kernfd = open(VM_DEFAULT_BIOS, O_RDONLY)) == -1) {
-		log_warn("%s: can't open %s", __func__, VM_DEFAULT_BIOS);
-		goto fail;
+		/*
+		 * Try to open the default BIOS image if no kernel/BIOS has been
+		 * specified.  The BIOS is an external firmware file that is
+		 * typically distributed separately due to an incompatible
+		 * license.
+		 */
+		if (kernfd == -1 && !vmboot &&
+		    (kernfd = open(VM_DEFAULT_BIOS, O_RDONLY)) == -1) {
+			log_warn("%s: can't open %s", __func__,
+			    VM_DEFAULT_BIOS);
+			goto fail;
+		}
 	}
 
 	/* Open disk images for child */
@@ -304,9 +311,14 @@ config_setvm(struct privsep *ps, struct vmd_vm *vm, uint32_t peerid, uid_t uid)
 	}
 
 	/* Send VM information */
-	proc_compose_imsg(ps, PROC_VMM, -1,
-	    IMSG_VMDOP_START_VM_REQUEST, vm->vm_vmid, kernfd,
-	    vmc, sizeof(*vmc));
+	if (vm->vm_received)
+		proc_compose_imsg(ps, PROC_VMM, -1,
+		    IMSG_VMDOP_RECEIVE_VM_REQUEST, vm->vm_vmid, fd,  vmc,
+		    sizeof(struct vmop_create_params));
+	else
+		proc_compose_imsg(ps, PROC_VMM, -1,
+		    IMSG_VMDOP_START_VM_REQUEST, vm->vm_vmid, kernfd,
+		    vmc, sizeof(*vmc));
 	for (i = 0; i < vcp->vcp_ndisks; i++) {
 		proc_compose_imsg(ps, PROC_VMM, -1,
 		    IMSG_VMDOP_START_VM_DISK, vm->vm_vmid, diskfds[i],
@@ -318,8 +330,9 @@ config_setvm(struct privsep *ps, struct vmd_vm *vm, uint32_t peerid, uid_t uid)
 		    &i, sizeof(i));
 	}
 
-	proc_compose_imsg(ps, PROC_VMM, -1,
-	    IMSG_VMDOP_START_VM_END, vm->vm_vmid, fd,  NULL, 0);
+	if (!vm->vm_received)
+		proc_compose_imsg(ps, PROC_VMM, -1,
+		    IMSG_VMDOP_START_VM_END, vm->vm_vmid, fd,  NULL, 0);
 
 	free(diskfds);
 	free(tapfds);
@@ -429,7 +442,6 @@ config_getif(struct privsep *ps, struct imsg *imsg)
 		goto fail;
 	}
 	vm->vm_ifs[n].vif_fd = imsg->fd;
-
 	return (0);
  fail:
 	if (imsg->fd != -1)
diff --git a/usr.sbin/vmd/control.c b/usr.sbin/vmd/control.c
index 08baa77cf0d..1af8a0d5e14 100644
--- a/usr.sbin/vmd/control.c
+++ b/usr.sbin/vmd/control.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: control.c,v 1.20 2017/07/09 00:51:40 pd Exp $	*/
+/*	$OpenBSD: control.c,v 1.21 2017/07/15 05:05:36 pd Exp $	*/
 
 /*
  * Copyright (c) 2010-2015 Reyk Floeter <reyk@openbsd.org>
@@ -70,8 +70,9 @@ control_run(struct privsep *ps, struct privsep_proc *p, void *arg)
 	 * cpath - for managing the control socket.
 	 * unix - for the control socket.
 	 * recvfd - for the proc fd exchange.
+	 * sendfd - for send and receive.
 	 */
-	if (pledge("stdio cpath unix recvfd", NULL) == -1)
+	if (pledge("stdio cpath unix recvfd sendfd", NULL) == -1)
 		fatal("pledge");
 }
 
@@ -84,6 +85,8 @@ control_dispatch_vmd(int fd, struct privsep_proc *p, struct imsg *imsg)
 	switch (imsg->hdr.type) {
 	case IMSG_VMDOP_START_VM_RESPONSE:
 	case IMSG_VMDOP_PAUSE_VM_RESPONSE:
+	case IMSG_VMDOP_SEND_VM_RESPONSE:
+	case IMSG_VMDOP_RECEIVE_VM_RESPONSE:
 	case IMSG_VMDOP_UNPAUSE_VM_RESPONSE:
 	case IMSG_VMDOP_TERMINATE_VM_RESPONSE:
 	case IMSG_VMDOP_GET_INFO_VM_DATA:
@@ -96,7 +99,7 @@ control_dispatch_vmd(int fd, struct privsep_proc *p, struct imsg *imsg)
 			return (0);
 		}
 		imsg_compose_event(&c->iev, imsg->hdr.type,
-		    0, 0, -1, imsg->data, IMSG_DATA_SIZE(imsg));
+		    0, 0, imsg->fd, imsg->data, IMSG_DATA_SIZE(imsg));
 		break;
 	case IMSG_VMDOP_CONFIG:
 		config_getconfig(ps->ps_env, imsg);
@@ -368,13 +371,15 @@ control_dispatch_imsg(int fd, short event, void *arg)
 			log_setverbose(v);
 
 			/* FALLTHROUGH */
+		case IMSG_VMDOP_RECEIVE_VM_REQUEST:
+		case IMSG_VMDOP_SEND_VM_REQUEST:
 		case IMSG_VMDOP_PAUSE_VM:
 		case IMSG_VMDOP_UNPAUSE_VM:
 		case IMSG_VMDOP_LOAD:
 		case IMSG_VMDOP_RELOAD:
 		case IMSG_CTL_RESET:
 			if (proc_compose_imsg(ps, PROC_PARENT, -1,
-			    imsg.hdr.type, fd, -1,
+			    imsg.hdr.type, fd, imsg.fd,
 			    imsg.data, IMSG_DATA_SIZE(&imsg)) == -1)
 				goto fail;
 			break;
diff --git a/usr.sbin/vmd/ns8250.c b/usr.sbin/vmd/ns8250.c
index 91c417b13a1..d9f9274de65 100644
--- a/usr.sbin/vmd/ns8250.c
+++ b/usr.sbin/vmd/ns8250.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: ns8250.c,v 1.9 2017/06/07 14:53:28 mlarkin Exp $ */
+/* $OpenBSD: ns8250.c,v 1.10 2017/07/15 05:05:36 pd Exp $ */
 /*
  * Copyright (c) 2016 Mike Larkin <mlarkin@openbsd.org>
  *
@@ -608,6 +608,13 @@ ns8250_restore(int fd, int con_fd, uint32_t vmid)
 	com1_dev.fd = con_fd;
 	com1_dev.irq = 4;
 	com1_dev.rcv_pending = 0;
+	com1_dev.vmid = vmid;
+	com1_dev.byte_out = 0;
+	com1_dev.regs.divlo = 1;
+	com1_dev.baudrate = 115200;
+	com1_dev.rate_tv.tv_usec = 10000;
+	com1_dev.pause_ct = (com1_dev.baudrate / 8) / 1000 * 10;
+	evtimer_set(&com1_dev.rate, ratelimit, NULL);
 
 	event_set(&com1_dev.event, com1_dev.fd, EV_READ | EV_PERSIST,
 	    com_rcv_event, (void *)(intptr_t)vmid);
diff --git a/usr.sbin/vmd/vm.c b/usr.sbin/vmd/vm.c
index 87b980d269c..449407d296d 100644
--- a/usr.sbin/vmd/vm.c
+++ b/usr.sbin/vmd/vm.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: vm.c,v 1.21 2017/07/09 00:51:40 pd Exp $	*/
+/*	$OpenBSD: vm.c,v 1.22 2017/07/15 05:05:36 pd Exp $	*/
 
 /*
  * Copyright (c) 2015 Mike Larkin <mlarkin@openbsd.org>
@@ -60,6 +60,7 @@
 #include "i8259.h"
 #include "ns8250.h"
 #include "mc146818.h"
+#include "atomicio.h"
 
 io_fn_t ioports_map[MAX_PORTS];
 
@@ -73,10 +74,16 @@ void create_memory_map(struct vm_create_params *);
 int alloc_guest_mem(struct vm_create_params *);
 int vmm_create_vm(struct vm_create_params *);
 void init_emulated_hw(struct vmop_create_params *, int *, int *);
+void restore_emulated_hw(struct vm_create_params *,int , int *, int *);
 void vcpu_exit_inout(struct vm_run_params *);
 uint8_t vcpu_exit_pci(struct vm_run_params *);
 int vcpu_pic_intr(uint32_t, uint32_t, uint8_t);
 int loadfile_bios(FILE *, struct vcpu_reg_state *);
+int send_vm(int, struct vm_create_params *);
+int dump_vmr(int , struct vm_mem_range *);
+int dump_mem(int, struct vm_create_params *);
+void restore_vmr(int, struct vm_mem_range *);
+void restore_mem(int, struct vm_create_params *);
 void pause_vm(struct vm_create_params *);
 void unpause_vm(struct vm_create_params *);
 
@@ -257,13 +264,17 @@ start_vm(struct vmd_vm *vm, int fd)
 	FILE			*fp;
 	struct vmboot_params	 vmboot;
 	size_t			 i;
+	struct vm_rwregs_params  vrp;
 
 	/* Child */
 	setproctitle("%s", vcp->vcp_name);
 	log_procinit(vcp->vcp_name);
 
-	create_memory_map(vcp);
+	if (!vm->vm_received)
+		create_memory_map(vcp);
+
 	ret = alloc_guest_mem(vcp);
+
 	if (ret) {
 		errno = ret;
 		fatal("could not allocate guest memory - exiting");
@@ -285,37 +296,46 @@ start_vm(struct vmd_vm *vm, int fd)
 	/*
 	 * pledge in the vm processes:
 	 * stdio - for malloc and basic I/O including events.
+	 * recvfd - for send/recv.
 	 * vmm - for the vmm ioctls and operations.
 	 */
-	if (pledge("stdio vmm", NULL) == -1)
+	if (pledge("stdio vmm recvfd", NULL) == -1)
 		fatal("pledge");
 
-	/*
-	 * Set up default "flat 32 bit" register state - RIP,
-	 * RSP, and GDT info will be set in bootloader
-	 */
-	memcpy(&vrs, &vcpu_init_flat32, sizeof(vrs));
+	if (vm->vm_received) {
+		ret = read(vm->vm_receive_fd, &vrp, sizeof(vrp));
+		if (ret != sizeof(vrp)) {
+			fatal("received incomplete vrp - exiting");
+		}
+		vrs = vrp.vrwp_regs;
+	} else {
+		/*
+		 * Set up default "flat 32 bit" register state - RIP,
+		 * RSP, and GDT info will be set in bootloader
+		 */
+		memcpy(&vrs, &vcpu_init_flat32, sizeof(vrs));
 
-	/* Find and open kernel image */
-	if ((fp = vmboot_open(vm->vm_kernel,
-	    vm->vm_disks[0], &vmboot)) == NULL)
-		fatalx("failed to open kernel - exiting");
+		/* Find and open kernel image */
+		if ((fp = vmboot_open(vm->vm_kernel,
+		    vm->vm_disks[0], &vmboot)) == NULL)
+			fatalx("failed to open kernel - exiting");
 
-	/* Load kernel image */
-	ret = loadfile_elf(fp, vcp, &vrs,
-	    vmboot.vbp_bootdev, vmboot.vbp_howto);
+		/* Load kernel image */
+		ret = loadfile_elf(fp, vcp, &vrs,
+		    vmboot.vbp_bootdev, vmboot.vbp_howto);
 
-	/*
-	 * Try BIOS as a fallback (only if it was provided as an image
-	 * with vm->vm_kernel and not loaded from the disk)
-	 */
-	if (ret && errno == ENOEXEC && vm->vm_kernel != -1)
-		ret = loadfile_bios(fp, &vrs);
+		/*
+		 * Try BIOS as a fallback (only if it was provided as an image
+		 * with vm->vm_kernel and not loaded from the disk)
+		 */
+		if (ret && errno == ENOEXEC && vm->vm_kernel != -1)
+			ret = loadfile_bios(fp, &vrs);
 
-	if (ret)
-		fatal("failed to load kernel or BIOS - exiting");
+		if (ret)
+			fatal("failed to load kernel or BIOS - exiting");
 
-	vmboot_close(fp, &vmboot);
+		vmboot_close(fp, &vmboot);
+	}
 
 	if (vm->vm_kernel != -1)
 		close(vm->vm_kernel);
@@ -329,6 +349,12 @@ start_vm(struct vmd_vm *vm, int fd)
 
 	event_init();
 
+	if (vm->vm_received) {
+		restore_emulated_hw(vcp, vm->vm_receive_fd, nicfds,
+		    vm->vm_disks);
+		restore_mem(vm->vm_receive_fd, vcp);
+	}
+
 	if (vmm_pipe(vm, fd, vm_dispatch_vmm) == -1)
 		fatal("setup vm pipe");
 
@@ -412,6 +438,15 @@ vm_dispatch_vmm(int fd, short event, void *arg)
 			    imsg.hdr.peerid, imsg.hdr.pid, -1, &vmr,
 			    sizeof(vmr));
 			break;
+		case IMSG_VMDOP_SEND_VM_REQUEST:
+			vmr.vmr_id = vm->vm_vmid;
+			vmr.vmr_result = send_vm(imsg.fd,
+			    &vm->vm_params.vmc_params);
+			imsg_compose_event(&vm->vm_iev,
+			    IMSG_VMDOP_SEND_VM_RESPONSE,
+			    imsg.hdr.peerid, imsg.hdr.pid, -1, &vmr,
+			    sizeof(vmr));
+			break;
 		default:
 			fatalx("%s: got invalid imsg %d from %s",
 			    __func__, imsg.hdr.type,
@@ -448,6 +483,150 @@ vm_shutdown(unsigned int cmd)
 	_exit(0);
 }
 
+int
+send_vm(int fd, struct vm_create_params *vcp)
+{
+	struct vm_rwregs_params	   vrp;
+	struct vmop_create_params *vmc;
+	struct vm_terminate_params vtp;
+	struct vm_dump_header	   vmh;
+	unsigned int		   flags = 0;
+	unsigned int		   i;
+	int			   ret = 0;
+
+	memset(&vmh, 0, sizeof(vmh));
+	memcpy(vmh.vmh_signature, VM_DUMP_SIGNATURE, sizeof(vmh.vmh_signature));
+	vmh.vmh_version = VM_DUMP_VERSION;
+	if (atomicio(vwrite, fd, &vmh, sizeof(vmh)) != sizeof(vmh)) {
+		log_warn("%s: failed to send vm dump header", __func__);
+		goto err;
+	}
+
+	pause_vm(vcp);
+
+	vmc = calloc(1, sizeof(struct vmop_create_params));
+	if (vmc == NULL) {
+		log_warn("%s: calloc error geting vmc", __func__);
+		ret = -1;
+		goto err;
+	}
+
+	flags |= VMOP_CREATE_MEMORY;
+	memcpy(&vmc->vmc_params, &current_vm->vm_params, sizeof(struct
+	    vmop_create_params));
+	vmc->vmc_flags = flags;
+	vrp.vrwp_vm_id = vcp->vcp_id;
+	vrp.vrwp_mask = VM_RWREGS_ALL;
+
+	if ((ret = atomicio(vwrite, fd, vmc,
+	    sizeof(struct vmop_create_params)) !=
+	    sizeof(struct vmop_create_params)))
+		goto err;
+
+	for (i = 0; i < vcp->vcp_ncpus; i++) {
+		vrp.vrwp_vcpu_id = i;
+		if ((ret = ioctl(env->vmd_fd, VMM_IOC_READREGS, &vrp))) {
+			log_warn("%s: readregs failed", __func__);
+			goto err;
+		}
+		if ((ret = atomicio(vwrite, fd, &vrp,
+		    sizeof(struct vm_rwregs_params))) !=
+		    sizeof(struct vm_rwregs_params)) {
+			log_warn("%s: dumping registers failed", __func__);
+			goto err;
+		}
+	}
+
+	if ((ret = i8253_dump(fd)))
+		goto err;
+	if ((ret = i8259_dump(fd)))
+		goto err;
+	if ((ret = ns8250_dump(fd)))
+		goto err;
+	if ((ret = mc146818_dump(fd)))
+		goto err;
+	if ((ret = virtio_dump(fd)))
+		goto err;
+	if ((ret = dump_mem(fd, vcp)))
+		goto err;
+
+	vtp.vtp_vm_id = vcp->vcp_id;
+	if (ioctl(env->vmd_fd, VMM_IOC_TERM, &vtp) < 0) {
+		log_warnx("%s: term IOC error: %d, %d", __func__,
+		    errno, ENOENT);
+	}
+err:
+	close(fd);
+	if (ret)
+		unpause_vm(vcp);
+	return ret;
+}
+
+int
+dump_mem(int fd, struct vm_create_params *vcp)
+{
+	unsigned int	i;
+	int		ret;
+	struct		vm_mem_range *vmr;
+
+	for (i = 0; i < vcp->vcp_nmemranges; i++) {
+		vmr = &vcp->vcp_memranges[i];
+		ret = dump_vmr(fd, vmr);
+		if (ret)
+			return ret;
+	}
+	return (0);
+}
+
+void
+restore_mem(int fd, struct vm_create_params *vcp)
+{
+	unsigned int	     i;
+	struct vm_mem_range *vmr;
+
+	for (i = 0; i < vcp->vcp_nmemranges; i++) {
+		vmr = &vcp->vcp_memranges[i];
+		restore_vmr(fd, vmr);
+	}
+}
+
+int
+dump_vmr(int fd, struct vm_mem_range *vmr)
+{
+	size_t	rem = vmr->vmr_size, read=0;
+	char	buf[PAGE_SIZE];
+
+	while (rem > 0) {
+		if(read_mem(vmr->vmr_gpa + read, buf, PAGE_SIZE)) {
+			log_warn("failed to read vmr");
+			return (-1);
+		}
+		if (atomicio(vwrite, fd, buf, sizeof(buf)) != sizeof(buf)) {
+			log_warn("failed to dump vmr");
+			return (-1);
+		}
+		rem = rem - PAGE_SIZE;
+		read = read + PAGE_SIZE;
+	}
+	return (0);
+}
+
+void
+restore_vmr(int fd, struct vm_mem_range *vmr)
+{
+	size_t	rem = vmr->vmr_size, wrote=0;
+	char	buf[PAGE_SIZE];
+
+	while (rem > 0) {
+		if (atomicio(read, fd, buf, sizeof(buf)) != sizeof(buf))
+			fatal("failed to restore vmr");
+		if (write_mem(vmr->vmr_gpa + wrote, buf, PAGE_SIZE))
+			fatal("failed to write vmr");
+		rem = rem - PAGE_SIZE;
+		wrote = wrote + PAGE_SIZE;
+	}
+}
+
 void
 pause_vm(struct vm_create_params *vcp)
 {
@@ -726,6 +905,55 @@ init_emulated_hw(struct vmop_create_params *vmc, int *child_disks,
 	/* Initialize virtio devices */
 	virtio_init(current_vm, child_disks, child_taps);
 }
+/*
+ * restore_emulated_hw
+ *
+ * Restores the userspace hardware emulation from fd
+ */
+void
+restore_emulated_hw(struct vm_create_params *vcp, int fd,
+    int *child_taps, int *child_disks)
+{
+	/* struct vm_create_params *vcp = &vmc->vmc_params; */
+	int i;
+	memset(&ioports_map, 0, sizeof(io_fn_t) * MAX_PORTS);
+
+	/* Init i8253 PIT */
+	i8253_restore(fd, vcp->vcp_id);
+	ioports_map[TIMER_CTRL] = vcpu_exit_i8253;
+	ioports_map[TIMER_BASE + TIMER_CNTR0] = vcpu_exit_i8253;
+	ioports_map[TIMER_BASE + TIMER_CNTR1] = vcpu_exit_i8253;
+	ioports_map[TIMER_BASE + TIMER_CNTR2] = vcpu_exit_i8253;
+
+	/* Init master and slave PICs */
+	i8259_restore(fd);
+	ioports_map[IO_ICU1] = vcpu_exit_i8259;
+	ioports_map[IO_ICU1 + 1] = vcpu_exit_i8259;
+	ioports_map[IO_ICU2] = vcpu_exit_i8259;
+	ioports_map[IO_ICU2 + 1] = vcpu_exit_i8259;
+
+	/* Init ns8250 UART */
+	ns8250_restore(fd, con_fd, vcp->vcp_id);
+	for (i = COM1_DATA; i <= COM1_SCR; i++)
+		ioports_map[i] = vcpu_exit_com;
+
+	/* Init mc146818 RTC */
+	mc146818_restore(fd, vcp->vcp_id);
+	ioports_map[IO_RTC] = vcpu_exit_mc146818;
+	ioports_map[IO_RTC + 1] = vcpu_exit_mc146818;
+
+	/* Initialize PCI */
+	for (i = VMM_PCI_IO_BAR_BASE; i <= VMM_PCI_IO_BAR_END; i++)
+		ioports_map[i] = vcpu_exit_pci;
+
+	ioports_map[PCI_MODE1_ADDRESS_REG] = vcpu_exit_pci;
+	ioports_map[PCI_MODE1_DATA_REG] = vcpu_exit_pci;
+	ioports_map[PCI_MODE1_DATA_REG + 1] = vcpu_exit_pci;
+	ioports_map[PCI_MODE1_DATA_REG + 2] = vcpu_exit_pci;
+	ioports_map[PCI_MODE1_DATA_REG + 3] = vcpu_exit_pci;
+	pci_init();
+	virtio_restore(fd, current_vm, child_disks, child_taps);
+}
 
 /*
  * run_vm
@@ -748,6 +976,7 @@ run_vm(int *child_disks, int *child_taps, struct vmop_create_params *vmc,
     struct vcpu_reg_state *vrs)
 {
 	struct vm_create_params *vcp = &vmc->vmc_params;
+	struct vm_rwregs_params vregsp;
 	uint8_t evdone = 0;
 	size_t i;
 	int ret;
@@ -788,7 +1017,8 @@ run_vm(int *child_disks, int *child_taps, struct vmop_create_params *vmc,
 	log_debug("%s: initializing hardware for vm %s", __func__,
 	    vcp->vcp_name);
 
-	init_emulated_hw(vmc, child_disks, child_taps);
+	if (!current_vm->vm_received)
+		init_emulated_hw(vmc, child_disks, child_taps);
 
 	ret = pthread_mutex_init(&threadmutex, NULL);
 	if (ret) {
@@ -838,6 +1068,19 @@ run_vm(int *child_disks, int *child_taps, struct vmop_create_params *vmc,
 			return (EIO);
 		}
 
+		/* once more becuase reset_cpu changes regs */
+		if (current_vm->vm_received) {
+			vregsp.vrwp_vm_id = vcp->vcp_id;
+			vregsp.vrwp_vcpu_id = i;
+			vregsp.vrwp_regs = *vrs;
+			vregsp.vrwp_mask = VM_RWREGS_ALL;
+			if ((ret = ioctl(env->vmd_fd, VMM_IOC_WRITEREGS,
+			    &vregsp)) < 0) {
+				log_warn("%s: writeregs failed", __func__);
+				return (ret);
+			}
+		}
+
 		ret = pthread_cond_init(&vcpu_run_cond[i], NULL);
 		if (ret) {
 			log_warnx("%s: cannot initialize cond var (%d)",
diff --git a/usr.sbin/vmd/vmd.c b/usr.sbin/vmd/vmd.c
index ca332ae3f28..e12953c6fb4 100644
--- a/usr.sbin/vmd/vmd.c
+++ b/usr.sbin/vmd/vmd.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: vmd.c,v 1.63 2017/07/09 00:51:40 pd Exp $	*/
+/*	$OpenBSD: vmd.c,v 1.64 2017/07/15 05:05:36 pd Exp $	*/
 
 /*
  * Copyright (c) 2015 Reyk Floeter <reyk@openbsd.org>
@@ -40,6 +40,7 @@
 #include <grp.h>
 
 #include "proc.h"
+#include "atomicio.h"
 #include "vmd.h"
 
 __dead void usage(void);
@@ -75,6 +76,7 @@ vmd_dispatch_control(int fd, struct privsep_proc *p, struct imsg *imsg)
 	struct vmop_id			 vid;
 	struct vm_terminate_params	 vtp;
 	struct vmop_result		 vmr;
+	struct vm_dump_header		 vmh;
 	struct vmd_vm			*vm = NULL;
 	char				*str = NULL;
 	uint32_t			 id = 0;
@@ -179,6 +181,82 @@ vmd_dispatch_control(int fd, struct privsep_proc *p, struct imsg *imsg)
 		proc_compose_imsg(ps, PROC_VMM, -1, imsg->hdr.type,
 		    imsg->hdr.peerid, -1, &vid, sizeof(vid));
 		break;
+	case IMSG_VMDOP_SEND_VM_REQUEST:
+		IMSG_SIZE_CHECK(imsg, &vid);
+		memcpy(&vid, imsg->data, sizeof(vid));
+		id = vid.vid_id;
+		if (vid.vid_id == 0) {
+			if ((vm = vm_getbyname(vid.vid_name)) == NULL) {
+				res = ENOENT;
+				cmd = IMSG_VMDOP_SEND_VM_RESPONSE;
+				close(imsg->fd);
+				break;
+			} else {
+				vid.vid_id = vm->vm_vmid;
+			}
+		} else if ((vm = vm_getbyvmid(vid.vid_id)) == NULL) {
+			res = ENOENT;
+			cmd = IMSG_VMDOP_SEND_VM_RESPONSE;
+			close(imsg->fd);
+			break;
+		} else {
+		}
+		vmr.vmr_id = vid.vid_id;
+		log_debug("%s: sending fd to vmctl", __func__);
+		proc_compose_imsg(ps, PROC_VMM, -1, imsg->hdr.type,
+		    imsg->hdr.peerid, imsg->fd, &vid, sizeof(vid));
+		break;
+	case IMSG_VMDOP_RECEIVE_VM_REQUEST:
+		IMSG_SIZE_CHECK(imsg, &vid);
+		memcpy(&vid, imsg->data, sizeof(vid));
+		if (imsg->fd == -1) {
+			log_warnx("%s: invalid fd", __func__);
+			return (-1);
+		}
+		if (atomicio(read, imsg->fd, &vmh, sizeof(vmh)) !=
+		    sizeof(vmh)) {
+			log_warnx("%s: error reading vmh from recevied vm",
+			    __func__);
+			res = EIO;
+			close(imsg->fd);
+			cmd = IMSG_VMDOP_START_VM_RESPONSE;
+			break;
+		}
+		if (vmh.vmh_version != VM_DUMP_VERSION) {
+			log_warnx("%s: incompatible dump version",
+			    __func__);
+			res = ENOENT;
+			close(imsg->fd);
+			cmd = IMSG_VMDOP_SEND_VM_RESPONSE;
+			break;
+		}
+		if (atomicio(read, imsg->fd, &vmc, sizeof(vmc)) !=
+		    sizeof(vmc)) {
+			log_warnx("%s: error reading vmc from recevied vm",
+			    __func__);
+			res = EIO;
+			close(imsg->fd);
+			cmd = IMSG_VMDOP_START_VM_RESPONSE;
+			break;
+		}
+		strlcpy(vmc.vmc_params.vcp_name, vid.vid_name,
+		    sizeof(vmc.vmc_params.vcp_name));
+		vmc.vmc_params.vcp_id = 0;
+
+		ret = vm_register(ps, &vmc, &vm, 0, vmc.vmc_uid);
+		if (ret != 0) {
+			res = errno;
+			cmd = IMSG_VMDOP_START_VM_RESPONSE;
+			close(imsg->fd);
+		} else {
+			vm->vm_received = 1;
+			config_setvm(ps, vm, imsg->hdr.peerid, vmc.vmc_uid);
+			log_debug("%s: sending fd to vmctl", __func__);
+			proc_compose_imsg(ps, PROC_VMM, -1,
+			    IMSG_VMDOP_RECEIVE_VM_END, vm->vm_vmid, imsg->fd,
+			    NULL, 0);
+		}
+		break;
 	default:
 		return (-1);
 	}
@@ -296,6 +374,15 @@ vmd_dispatch_vmm(int fd, struct privsep_proc *p, struct imsg *imsg)
 			vm->vm_shutdown = 1;
 		}
 		break;
+	case IMSG_VMDOP_SEND_VM_RESPONSE:
+		IMSG_SIZE_CHECK(imsg, &vmr);
+		memcpy(&vmr, imsg->data, sizeof(vmr));
+		if ((vm = vm_getbyvmid(vmr.vmr_id)) == NULL)
+			break;
+		if (!vmr.vmr_result)
+			log_info("%s: sent vm %d successfully.",
+			    vm->vm_params.vmc_params.vcp_name,
+			    vm->vm_vmid);
 	case IMSG_VMDOP_TERMINATE_VM_EVENT:
 		IMSG_SIZE_CHECK(imsg, &vmr);
 		memcpy(&vmr, imsg->data, sizeof(vmr));
@@ -572,10 +659,11 @@ vmd_configure(void)
 	 * tty - for openpty.
 	 * proc - run kill to terminate its children safely.
 	 * sendfd - for disks, interfaces and other fds.
+	 * recvfd - for send and receive.
 	 * getpw - lookup user or group id by name.
 	 * chown, fattr - change tty ownership
 	 */
-	if (pledge("stdio rpath wpath proc tty sendfd getpw"
+	if (pledge("stdio rpath wpath proc tty recvfd sendfd getpw"
 	    " chown fattr", NULL) == -1)
 		fatal("pledge");
 
@@ -909,6 +997,7 @@ vm_register(struct privsep *ps, struct vmop_create_params *vmc,
 	vcp = &vmc->vmc_params;
 	vm->vm_pid = -1;
 	vm->vm_tty = -1;
+	vm->vm_receive_fd = -1;
 	vm->vm_paused = 0;
 
 	for (i = 0; i < vcp->vcp_ndisks; i++)
diff --git a/usr.sbin/vmd/vmd.h b/usr.sbin/vmd/vmd.h
index 0cfd4cf546c..a0d2a1589db 100644
--- a/usr.sbin/vmd/vmd.h
+++ b/usr.sbin/vmd/vmd.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: vmd.h,v 1.57 2017/07/09 00:51:40 pd Exp $	*/
+/*	$OpenBSD: vmd.h,v 1.58 2017/07/15 05:05:36 pd Exp $	*/
 
 /*
  * Copyright (c) 2015 Mike Larkin <mlarkin@openbsd.org>
@@ -69,6 +69,11 @@ enum imsg_type {
 	IMSG_VMDOP_PAUSE_VM_RESPONSE,
 	IMSG_VMDOP_UNPAUSE_VM,
 	IMSG_VMDOP_UNPAUSE_VM_RESPONSE,
+	IMSG_VMDOP_SEND_VM_REQUEST,
+	IMSG_VMDOP_SEND_VM_RESPONSE,
+	IMSG_VMDOP_RECEIVE_VM_REQUEST,
+	IMSG_VMDOP_RECEIVE_VM_RESPONSE,
+	IMSG_VMDOP_RECEIVE_VM_END,
 	IMSG_VMDOP_TERMINATE_VM_REQUEST,
 	IMSG_VMDOP_TERMINATE_VM_RESPONSE,
 	IMSG_VMDOP_TERMINATE_VM_EVENT,
@@ -140,6 +145,14 @@ struct vmop_create_params {
 	int64_t			 vmc_gid;
 };
 
+struct vm_dump_header {
+	uint8_t			 vmh_signature[12];
+#define VM_DUMP_SIGNATURE	 VMM_HV_SIGNATURE
+	uint8_t			 vmh_pad[3];
+	uint8_t			 vmh_version;
+#define VM_DUMP_VERSION		 1
+} __packed;
+
 struct vmboot_params {
 	int			 vbp_fd;
 	off_t			 vbp_partoff;
@@ -193,7 +206,9 @@ struct vmd_vm {
 	struct imsgev		 vm_iev;
 	int			 vm_shutdown;
 	uid_t			 vm_uid;
+	int			 vm_received;
 	int			 vm_paused;
+	int			 vm_receive_fd;
 
 	TAILQ_ENTRY(vmd_vm)	 vm_entry;
 };
@@ -304,6 +319,7 @@ int	 vmm_pipe(struct vmd_vm *, int, void (*)(int, short, void *));
 
 /* vm.c */
 int	 start_vm(struct vmd_vm *, int);
+int receive_vm(struct vmd_vm *, int, int);
 __dead void vm_shutdown(unsigned int);
 
 /* control.c */
diff --git a/usr.sbin/vmd/vmm.c b/usr.sbin/vmd/vmm.c
index 2e992d6cb15..c852ff1ac65 100644
--- a/usr.sbin/vmd/vmm.c
+++ b/usr.sbin/vmd/vmm.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: vmm.c,v 1.70 2017/07/09 00:51:40 pd Exp $	*/
+/*	$OpenBSD: vmm.c,v 1.71 2017/07/15 05:05:36 pd Exp $	*/
 
 /*
  * Copyright (c) 2015 Mike Larkin <mlarkin@openbsd.org>
@@ -55,6 +55,7 @@
 
 void vmm_sighdlr(int, short, void *);
 int vmm_start_vm(struct imsg *, uint32_t *);
+int vmm_receive_vm(struct vmd_vm * , int);
 int vmm_dispatch_parent(int, struct privsep_proc *, struct imsg *);
 void vmm_run(struct privsep *, struct privsep_proc *, void *);
 void vmm_dispatch_vm(int, short, void *);
@@ -89,9 +90,10 @@ vmm_run(struct privsep *ps, struct privsep_proc *p, void *arg)
 	 * stdio - for malloc and basic I/O including events.
 	 * vmm - for the vmm ioctls and operations.
 	 * proc - for forking and maitaining vms.
+	 * send - for sending send/recv fds to vm proc.
 	 * recvfd - for disks, interfaces and other fds.
 	 */
-	if (pledge("stdio vmm recvfd proc", NULL) == -1)
+	if (pledge("stdio vmm sendfd recvfd proc", NULL) == -1)
 		fatal("pledge");
 
 	/* Get and terminate all running VMs */
@@ -102,11 +104,12 @@ int
 vmm_dispatch_parent(int fd, struct privsep_proc *p, struct imsg *imsg)
 {
 	struct privsep		*ps = p->p_ps;
-	int			 res = 0, cmd = 0, verbose;
+	int			 res = 0, cmd = 0, verbose, ret;
 	struct vmd_vm		*vm = NULL;
 	struct vm_terminate_params vtp;
-	struct vmop_id vid;
+	struct vmop_id		 vid;
 	struct vmop_result	 vmr;
+	struct vmop_create_params vmc;
 	uint32_t		 id = 0;
 	unsigned int		 mode;
 
@@ -230,6 +233,38 @@ vmm_dispatch_parent(int fd, struct privsep_proc *p, struct imsg *imsg)
 		    imsg->hdr.type, imsg->hdr.peerid, imsg->hdr.pid,
 		    imsg->fd, &vid, sizeof(vid));
 		break;
+	case IMSG_VMDOP_SEND_VM_REQUEST:
+		IMSG_SIZE_CHECK(imsg, &vid);
+		memcpy(&vid, imsg->data, sizeof(vid));
+		id = vid.vid_id;
+		if ((vm = vm_getbyvmid(id)) == NULL) {
+			res = ENOENT;
+			close(imsg->fd);
+			cmd = IMSG_VMDOP_START_VM_RESPONSE;
+			break;
+		}
+		imsg_compose_event(&vm->vm_iev,
+		    imsg->hdr.type, imsg->hdr.peerid, imsg->hdr.pid,
+		    imsg->fd, &vid, sizeof(vid));
+		break;
+	case IMSG_VMDOP_RECEIVE_VM_REQUEST:
+		IMSG_SIZE_CHECK(imsg, &vmc);
+		memcpy(&vmc, imsg->data, sizeof(vmc));
+		ret = vm_register(ps, &vmc, &vm, imsg->hdr.peerid, vmc.vmc_uid);
+		vm->vm_tty = imsg->fd;
+		vm->vm_received = 1;
+		break;
+	case IMSG_VMDOP_RECEIVE_VM_END:
+		if ((vm = vm_getbyvmid(imsg->hdr.peerid)) == NULL) {
+			res = ENOENT;
+			close(imsg->fd);
+			cmd = IMSG_VMDOP_START_VM_RESPONSE;
+			break;
+		}
+		vm->vm_receive_fd = imsg->fd;
+		res = vmm_start_vm(imsg, &id);
+		cmd = IMSG_VMDOP_START_VM_RESPONSE;
+		break;
 	default:
 		return (-1);
 	}
@@ -380,6 +415,7 @@ void
 vmm_dispatch_vm(int fd, short event, void *arg)
 {
 	struct vmd_vm		*vm = arg;
+	struct vmop_result	 vmr;
 	struct imsgev		*iev = &vm->vm_iev;
 	struct imsgbuf		*ibuf = &iev->ibuf;
 	struct imsg		 imsg;
@@ -423,6 +459,11 @@ vmm_dispatch_vm(int fd, short event, void *arg)
 		case IMSG_VMDOP_VM_REBOOT:
 			vm->vm_shutdown = 0;
 			break;
+		case IMSG_VMDOP_SEND_VM_RESPONSE:
+			IMSG_SIZE_CHECK(&imsg, &vmr);
+			memcpy(&vmr, imsg.data, sizeof(vmr));
+			if(!vmr.vmr_result)
+				vm_remove(vm);
 		case IMSG_VMDOP_PAUSE_VM_RESPONSE:
 		case IMSG_VMDOP_UNPAUSE_VM_RESPONSE:
 			for (i = 0; i < sizeof(procs); i++) {
@@ -528,9 +569,11 @@ vmm_start_vm(struct imsg *imsg, uint32_t *id)
 	}
 	vcp = &vm->vm_params.vmc_params;
 
-	if ((vm->vm_tty = imsg->fd) == -1) {
-		log_warnx("%s: can't get tty", __func__);
-		goto err;
+	if (!vm->vm_received) {
+		if ((vm->vm_tty = imsg->fd) == -1) {
+			log_warnx("%s: can't get tty", __func__);
+			goto err;
+		}
 	}
 
 	if (socketpair(AF_UNIX, SOCK_STREAM, PF_UNSPEC, fds) == -1)