From 668559c106c6c3e1be9e4e2add61f463108adef0 Mon Sep 17 00:00:00 2001 From: Stefan Kempf Date: Fri, 2 Sep 2016 16:23:41 +0000 Subject: Move event handling to separate thread That allows the main thread to handle the termination of a VM on reboot/shutdown or normal exit. The way it works is that VCPUs that terminate for some reason communicate this to a main thread that takes care of shutdown. ok mlarkin@ --- usr.sbin/vmd/vmm.c | 179 ++++++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 150 insertions(+), 29 deletions(-) (limited to 'usr.sbin/vmd/vmm.c') diff --git a/usr.sbin/vmd/vmm.c b/usr.sbin/vmd/vmm.c index 8d0fc3f45f7..2f67f433064 100644 --- a/usr.sbin/vmd/vmm.c +++ b/usr.sbin/vmd/vmm.c @@ -1,4 +1,4 @@ -/* $OpenBSD: vmm.c,v 1.41 2016/09/01 17:09:33 mlarkin Exp $ */ +/* $OpenBSD: vmm.c,v 1.42 2016/09/02 16:23:40 stefan Exp $ */ /* * Copyright (c) 2015 Mike Larkin @@ -68,6 +68,7 @@ int start_vm(struct imsg *, uint32_t *); int terminate_vm(struct vm_terminate_params *); int get_info_vm(struct privsep *, struct imsg *, int); int run_vm(int *, int *, struct vm_create_params *, struct vcpu_reg_state *); +void *event_thread(void *); void *vcpu_run_loop(void *); int vcpu_exit(struct vm_run_params *); int vcpu_reset(uint32_t, uint32_t, struct vcpu_reg_state *); @@ -91,9 +92,13 @@ extern struct vmd *env; extern char *__progname; +pthread_mutex_t threadmutex; +pthread_cond_t threadcond; + pthread_cond_t vcpu_run_cond[VMM_MAX_VCPUS_PER_VM]; pthread_mutex_t vcpu_run_mtx[VMM_MAX_VCPUS_PER_VM]; uint8_t vcpu_hlt[VMM_MAX_VCPUS_PER_VM]; +uint8_t vcpu_done[VMM_MAX_VCPUS_PER_VM]; static struct privsep_proc procs[] = { { "parent", PROC_PARENT, vmm_dispatch_parent }, @@ -880,13 +885,12 @@ int run_vm(int *child_disks, int *child_taps, struct vm_create_params *vcp, struct vcpu_reg_state *vrs) { + uint8_t evdone = 0; size_t i; int ret; - pthread_t *tid; + pthread_t *tid, evtid; struct vm_run_params **vrp; -#if 0 void *exit_status; -#endif if (vcp == NULL) return (EINVAL); @@ -910,8 +914,6 @@ run_vm(int *child_disks, int *child_taps, struct vm_create_params *vcp, vcp->vcp_nmemranges > VMM_MAX_MEM_RANGES) return (EINVAL); - ret = 0; - event_init(); tid = calloc(vcp->vcp_ncpus, sizeof(pthread_t)); @@ -927,6 +929,21 @@ run_vm(int *child_disks, int *child_taps, struct vm_create_params *vcp, init_emulated_hw(vcp, child_disks, child_taps); + ret = pthread_mutex_init(&threadmutex, NULL); + if (ret) { + log_warn("%s: could not initialize thread state mutex", + __func__); + return (ret); + } + ret = pthread_cond_init(&threadcond, NULL); + if (ret) { + log_warn("%s: could not initialize thread state " + "condition variable", __func__); + return (ret); + } + + mutex_lock(&threadmutex); + log_debug("%s: starting vcpu threads for vm %s", __func__, vcp->vcp_name); @@ -980,35 +997,94 @@ run_vm(int *child_disks, int *child_taps, struct vm_create_params *vcp, ret = pthread_create(&tid[i], NULL, vcpu_run_loop, vrp[i]); if (ret) { /* caller will _exit after this return */ + ret = errno; + log_warn("%s: could not create vcpu thread %zu", + __func__, i); return (ret); } } log_debug("%s: waiting on events for VM %s", __func__, vcp->vcp_name); - ret = event_dispatch(); + ret = pthread_create(&evtid, NULL, event_thread, &evdone); + if (ret) { + errno = ret; + log_warn("%s: could not create event thread", __func__); + return (ret); + } -#if 0 - /* XXX need to handle clean exits now */ + for (;;) { + ret = pthread_cond_wait(&threadcond, &threadmutex); + if (ret) { + log_warn("%s: waiting on thread state condition " + "variable failed", __func__); + return (ret); + } + + /* + * Did a VCPU thread exit with an error? => return the first one + */ + for (i = 0; i < vcp->vcp_ncpus; i++) { + if (vcpu_done[i] == 0) + continue; + + if (pthread_join(tid[i], &exit_status)) { + log_warn("%s: failed to join thread %zd - " + "exiting", __progname, i); + return (EIO); + } + + if (exit_status != NULL) { + log_warnx("%s: vm %d vcpu run thread %zd " + "exited abnormally", __progname, + vcp->vcp_id, i); + return (EIO); + } + } + + /* Did the event thread exit? => return with an error */ + if (evdone) { + if (pthread_join(evtid, &exit_status)) { + log_warn("%s: failed to join event thread - " + "exiting", __progname); + return (EIO); + } - /* Wait for all the threads to exit */ - for (i = 0; i < vcp->vcp_ncpus; i++) { - if (pthread_join(tid[i], &exit_status)) { - log_warnx("%s: failed to join thread %zd - " - "exiting", __progname, i); + log_warnx("%s: vm %d event thread exited " + "unexpectedly", __progname, vcp->vcp_id); return (EIO); } - if (exit_status != NULL) { - log_warnx("%s: vm %d vcpu run thread %zd exited " - "abnormally", __progname, vcp->vcp_id, i); - ret = EIO; + /* Did all VCPU threads exit successfully? => return 0 */ + for (i = 0; i < vcp->vcp_ncpus; i++) { + if (vcpu_done[i] == 0) + break; } + if (i == vcp->vcp_ncpus) + return (0); + + /* Some more threads to wait for, start over */ + } -#endif - return (ret); + return (0); } +void * +event_thread(void *arg) +{ + uint8_t *donep = arg; + intptr_t ret; + + ret = event_dispatch(); + + mutex_lock(&threadmutex); + *donep = 1; + pthread_cond_signal(&threadcond); + mutex_unlock(&threadmutex); + + return (void *)ret; + } + /* * vcpu_run_loop * @@ -1026,7 +1102,7 @@ void * vcpu_run_loop(void *arg) { struct vm_run_params *vrp = (struct vm_run_params *)arg; - intptr_t ret; + intptr_t ret = 0; int irq; uint32_t n; @@ -1051,7 +1127,7 @@ vcpu_run_loop(void *arg) log_warnx("%s: can't wait on cond (%d)", __func__, (int)ret); (void)pthread_mutex_unlock(&vcpu_run_mtx[n]); - return ((void *)ret); + break; } } @@ -1059,7 +1135,7 @@ vcpu_run_loop(void *arg) if (ret) { log_warnx("%s: can't unlock mutex on cond (%d)", __func__, (int)ret); - return ((void *)ret); + break; } if (vrp->vrp_irqready && i8259_is_pending()) { @@ -1085,24 +1161,33 @@ vcpu_run_loop(void *arg) ret = errno; log_warn("%s: vm %d / vcpu %d run ioctl failed", __func__, vrp->vrp_vm_id, n); - return ((void *)ret); + break; } /* If the VM is terminating, exit normally */ - if (vrp->vrp_exit_reason == VM_EXIT_TERMINATED) - return (NULL); + if (vrp->vrp_exit_reason == VM_EXIT_TERMINATED) { + ret = (intptr_t)NULL; + break; + } if (vrp->vrp_exit_reason != VM_EXIT_NONE) { /* * vmm(4) needs help handling an exit, handle in * vcpu_exit. */ - if (vcpu_exit(vrp)) - return ((void *)EIO); + if (vcpu_exit(vrp)) { + ret = EIO; + break; + } } } - return (NULL); + mutex_lock(&threadmutex); + vcpu_done[n] = 1; + pthread_cond_signal(&threadcond); + mutex_unlock(&threadmutex); + + return ((void *)ret); } int @@ -1467,3 +1552,39 @@ fd_hasdata(int fd) hasdata = 1; return (hasdata); } + +/* + * mutex_lock + * + * Wrapper function for pthread_mutex_lock that does error checking and that + * exits on failure + */ +void +mutex_lock(pthread_mutex_t *m) +{ + int ret; + + ret = pthread_mutex_lock(m); + if (ret) { + errno = ret; + fatal("could not acquire mutex"); + } +} + +/* + * mutex_unlock + * + * Wrapper function for pthread_mutex_unlock that does error checking and that + * exits on failure + */ +void +mutex_unlock(pthread_mutex_t *m) +{ + int ret; + + ret = pthread_mutex_unlock(m); + if (ret) { + errno = ret; + fatal("could not release mutex"); + } +} -- cgit v1.2.3