Add -w option to vmctl stop to wait for completion of VM termination.

Use it in /etc/rc.d/vmd accordingly.

OK sthen@

10 files changed, 87 insertions, 57 deletions

diff --git a/etc/rc.d/vmd b/etc/rc.d/vmd
index f023ff34954..5db928072bd 100644
--- a/etc/rc.d/vmd
+++ b/etc/rc.d/vmd
@@ -1,6 +1,6 @@
 #!/bin/ksh
 #
-# $OpenBSD: vmd,v 1.7 2018/02/20 10:12:14 sthen Exp $
+# $OpenBSD: vmd,v 1.8 2018/07/11 13:19:47 reyk Exp $
 
 daemon="/usr/sbin/vmd"
 
@@ -17,10 +17,7 @@ list_running() {
 
 rc_stop() {
 	for vm in $(list_running); do
-		_rc_do vmctl stop "$vm"
-		while list_running | fgrep -wq "$vm"; do
-			sleep .1
-		done
+		_rc_do vmctl stop "$vm" -w
 	done
 
 	pkill -T "${daemon_rtable}" -xf "${pexp}"
diff --git a/usr.sbin/vmctl/main.c b/usr.sbin/vmctl/main.c
index 69d93d1a771..7b288d5255c 100644
--- a/usr.sbin/vmctl/main.c
+++ b/usr.sbin/vmctl/main.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: main.c,v 1.36 2018/07/11 09:35:44 reyk Exp $	*/
+/*	$OpenBSD: main.c,v 1.37 2018/07/11 13:19:47 reyk Exp $	*/
 
 /*
  * Copyright (c) 2015 Reyk Floeter <reyk@openbsd.org>
@@ -73,7 +73,7 @@ struct ctl_command ctl_commands[] = {
 	    " [-Lc] [-b image] [-r image] [-m size]\n"
 	    "\t\t[-n switch] [-i count] [-d disk]*" },
 	{ "status",	CMD_STATUS,	ctl_status,	"[id]" },
-	{ "stop",	CMD_STOP,	ctl_stop,	"id [-f]" },
+	{ "stop",	CMD_STOP,	ctl_stop,	"id [-fw]" },
 	{ "pause",	CMD_PAUSE,	ctl_pause,	"id" },
 	{ "unpause",	CMD_UNPAUSE,	ctl_unpause,	"id" },
 	{ "send",	CMD_SEND,	ctl_send,	"id",	1},
@@ -182,6 +182,7 @@ vmmaction(struct parse_result *res)
 	int			 done = 0;
 	int			 n;
 	int			 ret, action;
+	unsigned int		 flags;
 
 	if (ctl_sock == -1) {
 		if ((ctl_sock = socket(AF_UNIX,
@@ -212,7 +213,7 @@ vmmaction(struct parse_result *res)
 		}
 		break;
 	case CMD_STOP:
-		terminate_vm(res->id, res->name, res->force);
+		terminate_vm(res->id, res->name, res->flags);
 		break;
 	case CMD_STATUS:
 		get_info_vm(res->id, res->name, 0);
@@ -254,6 +255,7 @@ vmmaction(struct parse_result *res)
 	}
 
 	action = res->action;
+	flags = res->flags;
 	parse_free(res);
 
 	while (ibuf->w.queued)
@@ -292,7 +294,8 @@ vmmaction(struct parse_result *res)
 				    tty_autoconnect);
 				break;
 			case CMD_STOP:
-				done = terminate_vm_complete(&imsg, &ret);
+				done = terminate_vm_complete(&imsg, &ret,
+				    flags);
 				break;
 			case CMD_CONSOLE:
 			case CMD_STATUS:
@@ -652,10 +655,13 @@ ctl_stop(struct parse_result *res, int argc, char *argv[])
 	argc--;
 	argv++;
 
-	while ((ch = getopt(argc, argv, "f")) != -1) {
+	while ((ch = getopt(argc, argv, "fw")) != -1) {
 		switch (ch) {
 		case 'f':
-			res->force = 1;
+			res->flags |= VMOP_FORCE;
+			break;
+		case 'w':
+			res->flags |= VMOP_WAIT;
 			break;
 		default:
 			ctl_usage(res->ctl);
diff --git a/usr.sbin/vmctl/vmctl.8 b/usr.sbin/vmctl/vmctl.8
index 4bd39460e6f..878b0b6a462 100644
--- a/usr.sbin/vmctl/vmctl.8
+++ b/usr.sbin/vmctl/vmctl.8
@@ -1,4 +1,4 @@
-.\"	$OpenBSD: vmctl.8,v 1.40 2018/07/11 09:35:44 reyk Exp $
+.\"	$OpenBSD: vmctl.8,v 1.41 2018/07/11 13:19:47 reyk Exp $
 .\"
 .\" Copyright (c) 2015 Mike Larkin <mlarkin@openbsd.org>
 .\"
@@ -147,7 +147,7 @@ with '.', '-' or '_'.
 .It Cm status Op Ar id
 Lists VMs running on the host, optionally listing just the selected VM
 .Ar id .
-.It Cm stop Ar id Op Fl f
+.It Cm stop Ar id Op Fl fw
 Stops (terminates) a VM defined by the specified VM
 .Ar id .
 By default,
@@ -156,9 +156,15 @@ a graceful shutdown will be attempted if the VM supports the
 device.
 Once stopped, if the VM was not defined in a configuration file, then it is
 removed.
-The
-.Fl f
-option forcefully stops the VM without attempting a graceful shutdown.
+.Pp
+The following options can be specified when stopping a VM:
+.Bl -tag -width "-w"
+.It Fl f
+Forcefully stop the VM without attempting a graceful shutdown.
+.It Fl w
+Wait until the VM has been terminated.
+.El
+.Pp
 .It Cm unpause Ar id
 Unpause (resume from a paused state) a VM with the specified
 .Ar id .
diff --git a/usr.sbin/vmctl/vmctl.c b/usr.sbin/vmctl/vmctl.c
index fff8afcf680..bfbc2c22801 100644
--- a/usr.sbin/vmctl/vmctl.c
+++ b/usr.sbin/vmctl/vmctl.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: vmctl.c,v 1.51 2018/07/11 09:35:44 reyk Exp $	*/
+/*	$OpenBSD: vmctl.c,v 1.52 2018/07/11 13:19:47 reyk Exp $	*/
 
 /*
  * Copyright (c) 2014 Mike Larkin <mlarkin@openbsd.org>
@@ -410,25 +410,22 @@ unpause_vm_complete(struct imsg *imsg, int *ret)
  * Parameters:
  *  terminate_id: ID of the vm to be terminated
  *  name: optional name of the VM to be terminated
- *  force: forcefully kill the VM process
+ *  flags: VMOP_FORCE or VMOP_WAIT flags
  */
 void
-terminate_vm(uint32_t terminate_id, const char *name, int force)
+terminate_vm(uint32_t terminate_id, const char *name, unsigned int flags)
 {
 	struct vmop_id vid;
-	int cmd;
 
 	memset(&vid, 0, sizeof(vid));
 	vid.vid_id = terminate_id;
 	if (name != NULL)
 		(void)strlcpy(vid.vid_name, name, sizeof(vid.vid_name));
 
-	if (force)
-		cmd = IMSG_VMDOP_KILL_VM_REQUEST;
-	else
-		cmd = IMSG_VMDOP_TERMINATE_VM_REQUEST;
+	vid.vid_flags = flags & (VMOP_FORCE|VMOP_WAIT);
 
-	imsg_compose(ibuf, cmd, 0, 0, -1, &vid, sizeof(vid));
+	imsg_compose(ibuf, IMSG_VMDOP_TERMINATE_VM_REQUEST,
+	    0, 0, -1, &vid, sizeof(vid));
 }
 
 /*
@@ -441,6 +438,7 @@ terminate_vm(uint32_t terminate_id, const char *name, int force)
  * Parameters:
  *  imsg : response imsg received from vmd
  *  ret  : return value
+ *  flags: VMOP_FORCE or VMOP_WAIT flags
  *
  * Return:
  *  Always 1 to indicate we have processed the return message (even if it
@@ -452,7 +450,7 @@ terminate_vm(uint32_t terminate_id, const char *name, int force)
  *   EIO   : terminate_vm command failed
  */
 int
-terminate_vm_complete(struct imsg *imsg, int *ret)
+terminate_vm_complete(struct imsg *imsg, int *ret, unsigned int flags)
 {
 	struct vmop_result *vmr;
 	int res;
@@ -475,8 +473,12 @@ terminate_vm_complete(struct imsg *imsg, int *ret)
 				warn("terminate vm command failed");
 				*ret = EIO;
 			}
+		} else if (flags & VMOP_WAIT) {
+			warnx("terminated vm %d", vmr->vmr_id);
+		} else if (flags & VMOP_FORCE) {
+			warnx("requested to terminate vm %d", vmr->vmr_id);
 		} else {
-			warnx("sent request to terminate vm %d", vmr->vmr_id);
+			warnx("requested to shutdown vm %d", vmr->vmr_id);
 			*ret = 0;
 		}
 	} else {
diff --git a/usr.sbin/vmctl/vmctl.h b/usr.sbin/vmctl/vmctl.h
index 337a5382f86..7fc8af8cbeb 100644
--- a/usr.sbin/vmctl/vmctl.h
+++ b/usr.sbin/vmctl/vmctl.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: vmctl.h,v 1.19 2018/07/11 09:35:44 reyk Exp $	*/
+/*	$OpenBSD: vmctl.h,v 1.20 2018/07/11 13:19:47 reyk Exp $	*/
 
 /*
  * Copyright (c) 2015 Reyk Floeter <reyk@openbsd.org>
@@ -52,9 +52,8 @@ struct parse_result {
 	int			 nnets;
 	size_t			 ndisks;
 	char			**disks;
-	int			 disable;
 	int			 verbose;
-	int			 force;
+	unsigned int		 flags;
 	unsigned int		 mode;
 	struct ctl_command	*ctl;
 };
@@ -86,8 +85,8 @@ int	 create_imagefile(const char *, long);
 int	 vm_start(uint32_t, const char *, int, int, char **, int,
 	    char **, char *, char *);
 int	 vm_start_complete(struct imsg *, int *, int);
-void	 terminate_vm(uint32_t, const char *, int);
-int	 terminate_vm_complete(struct imsg *, int *);
+void	 terminate_vm(uint32_t, const char *, unsigned int);
+int	 terminate_vm_complete(struct imsg *, int *, unsigned int);
 void	 pause_vm(uint32_t, const char *);
 int	 pause_vm_complete(struct imsg *, int *);
 void	 unpause_vm(uint32_t, const char *);
diff --git a/usr.sbin/vmd/config.c b/usr.sbin/vmd/config.c
index fab9af708d4..fc8114a9e5e 100644
--- a/usr.sbin/vmd/config.c
+++ b/usr.sbin/vmd/config.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: config.c,v 1.45 2018/07/10 16:15:51 reyk Exp $	*/
+/*	$OpenBSD: config.c,v 1.46 2018/07/11 13:19:47 reyk Exp $	*/
 
 /*
  * Copyright (c) 2015 Reyk Floeter <reyk@openbsd.org>
@@ -451,6 +451,7 @@ config_getvm(struct privsep *ps, struct imsg *imsg)
 	/* If the fd is -1, the kernel will be searched on the disk */
 	vm->vm_kernel = imsg->fd;
 	vm->vm_running = 1;
+	vm->vm_peerid = (uint32_t)-1;
 
 	return (0);
 
diff --git a/usr.sbin/vmd/control.c b/usr.sbin/vmd/control.c
index 37cc538ccb3..1162c0c207d 100644
--- a/usr.sbin/vmd/control.c
+++ b/usr.sbin/vmd/control.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: control.c,v 1.26 2018/07/11 09:35:44 reyk Exp $	*/
+/*	$OpenBSD: control.c,v 1.27 2018/07/11 13:19:47 reyk Exp $	*/
 
 /*
  * Copyright (c) 2010-2015 Reyk Floeter <reyk@openbsd.org>
@@ -352,7 +352,6 @@ control_dispatch_imsg(int fd, short event, void *arg)
 		switch (imsg.hdr.type) {
 		case IMSG_VMDOP_GET_INFO_VM_REQUEST:
 		case IMSG_VMDOP_TERMINATE_VM_REQUEST:
-		case IMSG_VMDOP_KILL_VM_REQUEST:
 		case IMSG_VMDOP_START_VM_REQUEST:
 		case IMSG_VMDOP_PAUSE_VM:
 		case IMSG_VMDOP_UNPAUSE_VM:
@@ -411,7 +410,6 @@ control_dispatch_imsg(int fd, short event, void *arg)
 			}
 			break;
 		case IMSG_VMDOP_TERMINATE_VM_REQUEST:
-		case IMSG_VMDOP_KILL_VM_REQUEST:
 			if (IMSG_DATA_SIZE(&imsg) < sizeof(vid))
 				goto fail;
 			memcpy(&vid, imsg.data, sizeof(vid));
diff --git a/usr.sbin/vmd/vmd.c b/usr.sbin/vmd/vmd.c
index d4f4e2e71f3..ac67bfd1169 100644
--- a/usr.sbin/vmd/vmd.c
+++ b/usr.sbin/vmd/vmd.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: vmd.c,v 1.92 2018/07/11 10:31:45 reyk Exp $	*/
+/*	$OpenBSD: vmd.c,v 1.93 2018/07/11 13:19:47 reyk Exp $	*/
 
 /*
  * Copyright (c) 2015 Reyk Floeter <reyk@openbsd.org>
@@ -76,10 +76,9 @@ vmd_dispatch_control(int fd, struct privsep_proc *p, struct imsg *imsg)
 {
 	struct privsep			*ps = p->p_ps;
 	int				 res = 0, ret = 0, cmd = 0, verbose;
-	unsigned int			 v = 0;
+	unsigned int			 v = 0, flags;
 	struct vmop_create_params	 vmc;
 	struct vmop_id			 vid;
-	struct vm_terminate_params	 vtp;
 	struct vmop_result		 vmr;
 	struct vm_dump_header		 vmh;
 	struct vmd_vm			*vm = NULL;
@@ -111,9 +110,10 @@ vmd_dispatch_control(int fd, struct privsep_proc *p, struct imsg *imsg)
 		}
 		break;
 	case IMSG_VMDOP_TERMINATE_VM_REQUEST:
-	case IMSG_VMDOP_KILL_VM_REQUEST:
 		IMSG_SIZE_CHECK(imsg, &vid);
 		memcpy(&vid, imsg->data, sizeof(vid));
+		flags = vid.vid_flags;
+
 		if ((id = vid.vid_id) == 0) {
 			/* Lookup vm (id) by name */
 			if ((vm = vm_getbyname(vid.vid_name)) == NULL) {
@@ -121,7 +121,7 @@ vmd_dispatch_control(int fd, struct privsep_proc *p, struct imsg *imsg)
 				cmd = IMSG_VMDOP_TERMINATE_VM_RESPONSE;
 				break;
 			} else if (vm->vm_shutdown &&
-			    imsg->hdr.type != IMSG_VMDOP_KILL_VM_REQUEST) {
+			    (flags & VMOP_FORCE) == 0) {
 				res = EALREADY;
 				cmd = IMSG_VMDOP_TERMINATE_VM_RESPONSE;
 				break;
@@ -141,10 +141,12 @@ vmd_dispatch_control(int fd, struct privsep_proc *p, struct imsg *imsg)
 			cmd = IMSG_VMDOP_TERMINATE_VM_RESPONSE;
 			break;
 		}
-		memset(&vtp, 0, sizeof(vtp));
-		vtp.vtp_vm_id = id;
+
+		memset(&vid, 0, sizeof(vid));
+		vid.vid_id = id;
+		vid.vid_flags = flags;
 		if (proc_compose_imsg(ps, PROC_VMM, -1, imsg->hdr.type,
-		    imsg->hdr.peerid, -1, &vtp, sizeof(vtp)) == -1)
+		    imsg->hdr.peerid, -1, &vid, sizeof(vid)) == -1)
 			return (-1);
 		break;
 	case IMSG_VMDOP_GET_INFO_VM_REQUEST:
@@ -432,6 +434,17 @@ vmd_dispatch_vmm(int fd, struct privsep_proc *p, struct imsg *imsg)
 			vm_stop(vm, 1, __func__);
 			config_setvm(ps, vm, (uint32_t)-1, vm->vm_uid);
 		}
+
+		/* Send a response if a control client is waiting for it */
+		if (imsg->hdr.peerid != (uint32_t)-1) {
+			/* the error is meaningless for deferred responses */
+			vmr.vmr_result = 0;
+
+			if (proc_compose_imsg(ps, PROC_CONTROL, -1,
+			    IMSG_VMDOP_TERMINATE_VM_RESPONSE,
+			    imsg->hdr.peerid, -1, &vmr, sizeof(vmr)) == -1)
+				return (-1);
+		}
 		break;
 	case IMSG_VMDOP_GET_INFO_VM_DATA:
 		IMSG_SIZE_CHECK(imsg, &vir);
diff --git a/usr.sbin/vmd/vmd.h b/usr.sbin/vmd/vmd.h
index 8b1b7a5f058..c25f9642147 100644
--- a/usr.sbin/vmd/vmd.h
+++ b/usr.sbin/vmd/vmd.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: vmd.h,v 1.73 2018/07/11 09:35:44 reyk Exp $	*/
+/*	$OpenBSD: vmd.h,v 1.74 2018/07/11 13:19:47 reyk Exp $	*/
 
 /*
  * Copyright (c) 2015 Mike Larkin <mlarkin@openbsd.org>
@@ -82,7 +82,6 @@ enum imsg_type {
 	IMSG_VMDOP_RECEIVE_VM_RESPONSE,
 	IMSG_VMDOP_RECEIVE_VM_END,
 	IMSG_VMDOP_TERMINATE_VM_REQUEST,
-	IMSG_VMDOP_KILL_VM_REQUEST,
 	IMSG_VMDOP_TERMINATE_VM_RESPONSE,
 	IMSG_VMDOP_TERMINATE_VM_EVENT,
 	IMSG_VMDOP_GET_INFO_VM_REQUEST,
@@ -122,6 +121,9 @@ struct vmop_id {
 	uint32_t		 vid_id;
 	char			 vid_name[VMM_MAX_NAME_LEN];
 	uid_t			 vid_uid;
+	unsigned int		 vid_flags;
+#define VMOP_FORCE		0x01
+#define VMOP_WAIT		0x02
 };
 
 struct vmop_ifreq {
diff --git a/usr.sbin/vmd/vmm.c b/usr.sbin/vmd/vmm.c
index cd721de8989..9beed72df96 100644
--- a/usr.sbin/vmd/vmm.c
+++ b/usr.sbin/vmd/vmm.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: vmm.c,v 1.85 2018/07/11 09:35:44 reyk Exp $	*/
+/*	$OpenBSD: vmm.c,v 1.86 2018/07/11 13:19:47 reyk Exp $	*/
 
 /*
  * Copyright (c) 2015 Mike Larkin <mlarkin@openbsd.org>
@@ -111,7 +111,7 @@ vmm_dispatch_parent(int fd, struct privsep_proc *p, struct imsg *imsg)
 	struct vmop_create_params vmc;
 	uint32_t		 id = 0;
 	pid_t			 pid = 0;
-	unsigned int		 mode;
+	unsigned int		 mode, flags;
 
 	switch (imsg->hdr.type) {
 	case IMSG_VMDOP_START_VM_REQUEST:
@@ -150,17 +150,19 @@ vmm_dispatch_parent(int fd, struct privsep_proc *p, struct imsg *imsg)
 		cmd = IMSG_VMDOP_START_VM_RESPONSE;
 		break;
 	case IMSG_VMDOP_TERMINATE_VM_REQUEST:
-	case IMSG_VMDOP_KILL_VM_REQUEST:
-		IMSG_SIZE_CHECK(imsg, &vtp);
-		memcpy(&vtp, imsg->data, sizeof(vtp));
-		id = vtp.vtp_vm_id;
+		IMSG_SIZE_CHECK(imsg, &vid);
+		memcpy(&vid, imsg->data, sizeof(vid));
+		id = vid.vid_id;
+		flags = vid.vid_flags;
 
 		DPRINTF("%s: recv'ed TERMINATE_VM for %d", __func__, id);
 
+		cmd = IMSG_VMDOP_TERMINATE_VM_RESPONSE;
+
 		if (id == 0) {
 			res = ENOENT;
 		} else if ((vm = vm_getbyvmid(id)) != NULL) {
-			if (imsg->hdr.type == IMSG_VMDOP_KILL_VM_REQUEST) {
+			if (flags & VMOP_FORCE) {
 				vtp.vtp_vm_id = vm_vmid2id(vm->vm_vmid, vm);
 				vm->vm_shutdown = 1;
 				(void)terminate_vm(&vtp);
@@ -189,20 +191,23 @@ vmm_dispatch_parent(int fd, struct privsep_proc *p, struct imsg *imsg)
 				 * Check to see if the VM process is still
 				 * active.  If not, return VMD_VM_STOP_INVALID.
 				 */
-				vtp.vtp_vm_id = vm_vmid2id(vm->vm_vmid, vm);
-				if (vtp.vtp_vm_id == 0) {
+				if (vm_vmid2id(vm->vm_vmid, vm) == 0) {
 					log_debug("%s: no vm running anymore",
 					    __func__);
 					res = VMD_VM_STOP_INVALID;
 				}
 			}
+			if ((flags & VMOP_WAIT) &&
+			    res == 0 && vm->vm_shutdown == 1) {
+				vm->vm_peerid = imsg->hdr.peerid;
+				cmd = 0;
+			}
 		} else {
 			/* vm doesn't exist, cannot stop vm */
 			log_debug("%s: cannot stop vm that is not running",
 			    __func__);
 			res = VMD_VM_STOP_INVALID;
 		}
-		cmd = IMSG_VMDOP_TERMINATE_VM_RESPONSE;
 		break;
 	case IMSG_VMDOP_GET_INFO_VM_REQUEST:
 		res = get_info_vm(ps, imsg, 0);
@@ -387,7 +392,8 @@ vmm_sighdlr(int sig, short event, void *arg)
 				vmr.vmr_id = vm_id2vmid(vmid, vm);
 				if (proc_compose_imsg(ps, PROC_PARENT,
 				    -1, IMSG_VMDOP_TERMINATE_VM_EVENT,
-				    0, -1, &vmr, sizeof(vmr)) == -1)
+				    vm->vm_peerid, -1,
+				    &vmr, sizeof(vmr)) == -1)
 					log_warnx("could not signal "
 					    "termination of VM %u to "
 					    "parent", vm->vm_vmid);