Add imsg communication channel between vmd and invividual VMs.

For now, this is only used to forward "log verbose|brief" requests,
but it will be used for better things later.

OK mlarkin@

3 files changed, 165 insertions, 9 deletions

diff --git a/usr.sbin/vmd/vmd.c b/usr.sbin/vmd/vmd.c
index 88cf224ca47..02af5cbc047 100644
--- a/usr.sbin/vmd/vmd.c
+++ b/usr.sbin/vmd/vmd.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: vmd.c,v 1.48 2017/01/09 14:49:22 reyk Exp $	*/
+/*	$OpenBSD: vmd.c,v 1.49 2017/01/11 22:38:10 reyk Exp $	*/
 
 /*
  * Copyright (c) 2015 Reyk Floeter <reyk@openbsd.org>
@@ -65,7 +65,7 @@ int
 vmd_dispatch_control(int fd, struct privsep_proc *p, struct imsg *imsg)
 {
 	struct privsep			*ps = p->p_ps;
-	int				 res = 0, ret = 0, cmd = 0;
+	int				 res = 0, ret = 0, cmd = 0, verbose;
 	unsigned int			 v = 0;
 	struct vmop_create_params	 vmc;
 	struct vmop_id			 vid;
@@ -130,6 +130,14 @@ vmd_dispatch_control(int fd, struct privsep_proc *p, struct imsg *imsg)
 		memcpy(&v, imsg->data, sizeof(v));
 		vmd_reload(v, str);
 		break;
+	case IMSG_CTL_VERBOSE:
+		IMSG_SIZE_CHECK(imsg, &verbose);
+		memcpy(&verbose, imsg->data, sizeof(verbose));
+		log_setverbose(verbose);
+
+		proc_forward_imsg(ps, imsg, PROC_VMM, -1);
+		proc_forward_imsg(ps, imsg, PROC_PRIV, -1);
+		break;
 	default:
 		return (-1);
 	}
@@ -647,6 +655,10 @@ vm_stop(struct vmd_vm *vm, int keeptty)
 
 	vm->vm_running = 0;
 
+	if (vm->vm_iev.ibuf.fd != -1) {
+		event_del(&vm->vm_iev.ev);
+		close(vm->vm_iev.ibuf.fd);
+	}
 	for (i = 0; i < VMM_MAX_DISKS_PER_VM; i++) {
 		if (vm->vm_disks[i] != -1) {
 			close(vm->vm_disks[i]);
@@ -742,6 +754,7 @@ vm_register(struct privsep *ps, struct vmop_create_params *vmc,
 	for (i = 0; i < vcp->vcp_nnics; i++)
 		vm->vm_ifs[i].vif_fd = -1;
 	vm->vm_kernel = -1;
+	vm->vm_iev.ibuf.fd = -1;
 
 	if (++env->vmd_nvm == 0)
 		fatalx("too many vms");
diff --git a/usr.sbin/vmd/vmd.h b/usr.sbin/vmd/vmd.h
index 499f64388c8..c7d3c2120b7 100644
--- a/usr.sbin/vmd/vmd.h
+++ b/usr.sbin/vmd/vmd.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: vmd.h,v 1.40 2016/12/14 21:17:25 reyk Exp $	*/
+/*	$OpenBSD: vmd.h,v 1.41 2017/01/11 22:38:10 reyk Exp $	*/
 
 /*
  * Copyright (c) 2015 Mike Larkin <mlarkin@openbsd.org>
@@ -160,6 +160,7 @@ struct vmd_vm {
 	int			 vm_running;
 	/* When set, VM is not started by default (PROC_PARENT only) */
 	int			 vm_disabled;
+	struct imsgev		 vm_iev;
 	TAILQ_ENTRY(vmd_vm)	 vm_entry;
 };
 TAILQ_HEAD(vmlist, vmd_vm);
diff --git a/usr.sbin/vmd/vmm.c b/usr.sbin/vmd/vmm.c
index 343566edf32..84c5d667d45 100644
--- a/usr.sbin/vmd/vmm.c
+++ b/usr.sbin/vmd/vmm.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: vmm.c,v 1.61 2017/01/08 21:23:32 mlarkin Exp $	*/
+/*	$OpenBSD: vmm.c,v 1.62 2017/01/11 22:38:10 reyk Exp $	*/
 
 /*
  * Copyright (c) 2015 Mike Larkin <mlarkin@openbsd.org>
@@ -83,6 +83,10 @@ int vmm_dispatch_parent(int, struct privsep_proc *, struct imsg *);
 void vmm_run(struct privsep *, struct privsep_proc *, void *);
 int vcpu_pic_intr(uint32_t, uint32_t, uint8_t);
 
+int vmm_pipe(struct vmd_vm *, int, void (*)(int, short, void *));
+void vmm_dispatch_vm(int, short, void *);
+void vm_dispatch_vmm(int, short, void *);
+
 static struct vm_mem_range *find_gpa_range(struct vm_create_params *, paddr_t,
     size_t);
 
@@ -178,7 +182,7 @@ int
 vmm_dispatch_parent(int fd, struct privsep_proc *p, struct imsg *imsg)
 {
 	struct privsep		*ps = p->p_ps;
-	int			 res = 0, cmd = 0;
+	int			 res = 0, cmd = 0, verbose;
 	struct vmd_vm		*vm;
 	struct vm_terminate_params vtp;
 	struct vmop_result	 vmr;
@@ -237,6 +241,18 @@ vmm_dispatch_parent(int fd, struct privsep_proc *p, struct imsg *imsg)
 
 		config_getreset(env, imsg);
 		break;
+	case IMSG_CTL_VERBOSE:
+		IMSG_SIZE_CHECK(imsg, &verbose);
+		memcpy(&verbose, imsg->data, sizeof(verbose));
+		log_setverbose(verbose);
+
+		/* Forward message to each VM process */
+		TAILQ_FOREACH(vm, env->vmd_vms, vm_entry) {
+			imsg_compose_event(&vm->vm_iev,
+			    imsg->hdr.type, imsg->hdr.peerid, imsg->hdr.pid,
+			    -1, &verbose, sizeof(verbose));
+		}
+		break;
 	default:
 		return (-1);
 	}
@@ -346,6 +362,128 @@ vmm_shutdown(void)
 	}
 }
 
+int
+vmm_pipe(struct vmd_vm *vm, int fd, void (*cb)(int, short, void *))
+{
+	struct imsgev	*iev = &vm->vm_iev;
+
+	if (fcntl(fd, F_SETFL, O_NONBLOCK) == -1) {
+		log_warn("failed to set nonblocking mode on vm pipe");
+		return (-1);
+	}
+
+	imsg_init(&iev->ibuf, fd);
+	iev->handler = cb;
+	iev->data = vm;
+	imsg_event_add(iev);
+
+	return (0);
+}
+
+void
+vmm_dispatch_vm(int fd, short event, void *arg)
+{
+	struct vmd_vm		*vm = arg;
+	struct imsgev		*iev = &vm->vm_iev;
+	struct imsgbuf		*ibuf = &iev->ibuf;
+	struct imsg		 imsg;
+	ssize_t			 n;
+
+	if (event & EV_READ) {
+		if ((n = imsg_read(ibuf)) == -1 && errno != EAGAIN)
+			fatal("%s: imsg_read", __func__);
+		if (n == 0) {
+			/* this pipe is dead, so remove the event handler */
+			event_del(&iev->ev);
+			return;
+		}
+	}
+
+	if (event & EV_WRITE) {
+		if ((n = msgbuf_write(&ibuf->w)) == -1 && errno != EAGAIN)
+			fatal("%s: msgbuf_write fd %d", __func__, ibuf->fd);
+		if (n == 0) {
+			/* this pipe is dead, so remove the event handler */
+			event_del(&iev->ev);
+			return;
+		}
+	}
+
+	for (;;) {
+		if ((n = imsg_get(ibuf, &imsg)) == -1)
+			fatal("%s: imsg_get", __func__);
+		if (n == 0)
+			break;
+
+#if DEBUG > 1
+		log_debug("%s: got imsg %d from %s",
+		    __func__, imsg.hdr.type,
+		    vm->vm_params.vmc_params.vcp_name);
+#endif
+
+		switch (imsg.hdr.type) {
+		default:
+			fatalx("%s: got invalid imsg %d from %s",
+			    __func__, imsg.hdr.type,
+			    vm->vm_params.vmc_params.vcp_name);
+		}
+		imsg_free(&imsg);
+	}
+	imsg_event_add(iev);
+}
+
+void
+vm_dispatch_vmm(int fd, short event, void *arg)
+{
+	struct vmd_vm		*vm = arg;
+	struct imsgev		*iev = &vm->vm_iev;
+	struct imsgbuf		*ibuf = &iev->ibuf;
+	struct imsg		 imsg;
+	ssize_t			 n;
+	int			 verbose;
+
+	if (event & EV_READ) {
+		if ((n = imsg_read(ibuf)) == -1 && errno != EAGAIN)
+			fatal("%s: imsg_read", __func__);
+		if (n == 0)
+			_exit(0);
+	}
+
+	if (event & EV_WRITE) {
+		if ((n = msgbuf_write(&ibuf->w)) == -1 && errno != EAGAIN)
+			fatal("%s: msgbuf_write fd %d", __func__, ibuf->fd);
+		if (n == 0)
+			_exit(0);
+	}
+
+	for (;;) {
+		if ((n = imsg_get(ibuf, &imsg)) == -1)
+			fatal("%s: imsg_get", __func__);
+		if (n == 0)
+			break;
+
+#if DEBUG > 1
+		log_debug("%s: got imsg %d from %s",
+		    __func__, imsg.hdr.type,
+		    vm->vm_params.vmc_params.vcp_name);
+#endif
+
+		switch (imsg.hdr.type) {
+		case IMSG_CTL_VERBOSE:
+			IMSG_SIZE_CHECK(&imsg, &verbose);
+			memcpy(&verbose, imsg.data, sizeof(verbose));
+			log_setverbose(verbose);
+			break;
+		default:
+			fatalx("%s: got invalid imsg %d from %s",
+			    __func__, imsg.hdr.type,
+			    vm->vm_params.vmc_params.vcp_name);
+		}
+		imsg_free(&imsg);
+	}
+	imsg_event_add(iev);
+}
+
 /*
  * vcpu_reset
  *
@@ -520,13 +658,15 @@ start_vm(struct imsg *imsg, uint32_t *id)
 		if (read(fds[0], &vcp->vcp_id, sizeof(vcp->vcp_id)) !=
 		    sizeof(vcp->vcp_id))
 			fatal("read vcp id");
-		close(fds[0]);
 
 		if (vcp->vcp_id == 0)
 			goto err;
 
 		*id = vcp->vcp_id;
 
+		if (vmm_pipe(vm, fds[0], vmm_dispatch_vm) == -1)
+			fatal("setup vm pipe");
+
 		return (0);
 	} else {
 		/* Child */
@@ -548,7 +688,6 @@ start_vm(struct imsg *imsg, uint32_t *id)
 		if (write(fds[1], &vcp->vcp_id, sizeof(vcp->vcp_id)) !=
 		    sizeof(vcp->vcp_id))
 			fatal("write vcp id");
-		close(fds[1]);
 
 		if (ret) {
 			errno = ret;
@@ -594,6 +733,11 @@ start_vm(struct imsg *imsg, uint32_t *id)
 		for (i = 0; i < VMM_MAX_NICS_PER_VM; i++)
 			nicfds[i] = vm->vm_ifs[i].vif_fd;
 
+		event_init();
+
+		if (vmm_pipe(vm, fds[1], vm_dispatch_vmm) == -1)
+			fatal("setup vm pipe");
+
 		/* Execute the vcpu run loop(s) for this VM */
 		ret = run_vm(vm->vm_disks, nicfds, vcp, &vrs);
 
@@ -935,8 +1079,6 @@ run_vm(int *child_disks, int *child_taps, struct vm_create_params *vcp,
 	    vcp->vcp_nmemranges > VMM_MAX_MEM_RANGES)
 		return (EINVAL);
 
-	event_init();
-
 	tid = calloc(vcp->vcp_ncpus, sizeof(pthread_t));
 	vrp = calloc(vcp->vcp_ncpus, sizeof(struct vm_run_params *));
 	if (tid == NULL || vrp == NULL) {