Process incoming com data asynchronously to running VCPU

This registers a handler with libevent that is triggered on incoming
data on the com port.

ok mlarkin@

3 files changed, 167 insertions, 90 deletions

diff --git a/usr.sbin/vmd/ns8250.c b/usr.sbin/vmd/ns8250.c
index 3adf0d8a8e3..4347b1f1c9b 100644
--- a/usr.sbin/vmd/ns8250.c
+++ b/usr.sbin/vmd/ns8250.c
@@ -20,23 +20,123 @@
 
 #include <machine/vmmvar.h>
 
+#include <errno.h>
+#include <event.h>
+#include <pthread.h>
 #include <string.h>
 #include <unistd.h>
 
 #include "ns8250.h"
 #include "proc.h"
+#include "vmd.h"
+#include "vmm.h"
 
 extern char *__progname;
-struct ns8250_regs com1_regs;
+struct ns8250_dev com1_dev;
+
+static void com_rcv_event(int, short, void *);
+static void com_rcv(struct ns8250_dev *, uint32_t, uint32_t);
 
 void
-ns8250_init(int fd)
+ns8250_init(int fd, uint32_t vmid)
 {
-	memset(&com1_regs, 0, sizeof(struct ns8250_regs));
-	com1_regs.fd = fd;
+	int ret;
+
+	memset(&com1_dev, 0, sizeof(com1_dev));
+	ret = pthread_mutex_init(&com1_dev.mutex, NULL);
+	if (ret) {
+		errno = ret;
+		fatal("could not initialize com1 mutex");
+	}
+	com1_dev.fd = fd;
+	com1_dev.irq = 4;
+	com1_dev.rcv_pending = 0;
+
+	event_set(&com1_dev.event, com1_dev.fd, EV_READ | EV_PERSIST,
+	    com_rcv_event, (void *)(uint64_t)vmid);
+	event_add(&com1_dev.event, NULL);
+}
+
+static void
+com_rcv_event(int fd, short kind, void *arg)
+{
+	mutex_lock(&com1_dev.mutex);
+
+	/*
+	 * We already have other data pending to be received. The data that
+	 * has become available now will be moved to the com port later.
+	 */
+	if (com1_dev.rcv_pending) {
+		mutex_unlock(&com1_dev.mutex);
+		return;
+	}
+
+	if (com1_dev.regs.lsr & LSR_RXRDY)
+		com1_dev.rcv_pending = 1;
+	else {
+		com_rcv(&com1_dev, (uintptr_t)arg, 0);
+
+		/* If pending interrupt, inject */
+		if ((com1_dev.regs.iir & 0x1) == 0) {
+			/* XXX: vcpu_id */
+			vcpu_assert_pic_irq((uintptr_t)arg, 0, com1_dev.irq);
+		}
+	}
+
+	mutex_unlock(&com1_dev.mutex);
 }
 
 /*
+ * com_rcv
+ *
+ * Move received byte into com data register.
+ * Must be called with the mutex of the com device acquired
+ */
+static void
+com_rcv(struct ns8250_dev *com, uint32_t vm_id, uint32_t vcpu_id)
+{
+	char ch;
+	ssize_t sz;
+
+	/*
+	 * Is there a new character available on com1?
+	 * If so, consume the character, buffer it into the com1 data register
+	 * assert IRQ4, and set the line status register RXRDY bit.
+	 */
+	sz = read(com->fd, &ch, sizeof(char));
+	if (sz == -1) {
+		/*
+		 * If we get EAGAIN, we'll retry and get the character later.
+		 * This error can happen when typing two characters at once
+		 * at the keyboard, for example.
+		 */
+		if (errno != EAGAIN)
+			log_warn("unexpected read error on com device");
+	} else if (sz != 1)
+		log_warnx("unexpected read return value on com device");
+	else {
+		com->regs.lsr |= LSR_RXRDY;
+		com->regs.data = ch;
+		/* XXX these ier and iir bits should be IER_x and IIR_x */
+		if (com->regs.ier & 0x1) {
+			com->regs.iir |= (2 << 1);
+			com->regs.iir &= ~0x1;
+		}
+	}
+
+	/*
+	 * Clear "interrupt pending" by setting IIR low bit to 1 if no
+	 * interrupts are pending
+	 */
+	/* XXX these iir magic numbers should be IIR_x */
+	if ((com->regs.iir & ~0x1) == 0x0)
+		com->regs.iir = 0x1;
+
+	com->rcv_pending = fd_hasdata(com->fd);
+}
+
+
+/*
  * vcpu_process_com_data
  *
  * Emulate in/out instructions to the com1 (ns8250) UART data register
@@ -44,9 +144,12 @@ ns8250_init(int fd)
  * Parameters:
  *  vei: vm exit information from vmm(4) containing information on the in/out
  *      instruction being performed
+ *
+ * Return value:
+ *  interrupt to inject, or 0xFF if nothing to inject
  */
-void
-vcpu_process_com_data(union vm_exit *vei)
+uint8_t
+vcpu_process_com_data(union vm_exit *vei, uint32_t vm_id, uint32_t vcpu_id)
 {
 	/*
 	 * vei_dir == VEI_DIR_OUT : out instruction
@@ -57,12 +160,12 @@ vcpu_process_com_data(union vm_exit *vei)
 	 * reporting)
 	 */
 	if (vei->vei.vei_dir == VEI_DIR_OUT) {
-		write(com1_regs.fd, &vei->vei.vei_data, 1);
-		if (com1_regs.ier & 0x2) {
+		write(com1_dev.fd, &vei->vei.vei_data, 1);
+		if (com1_dev.regs.ier & 0x2) {
 			/* Set TXRDY */
-			com1_regs.iir |= IIR_TXRDY;
+			com1_dev.regs.iir |= IIR_TXRDY;
 			/* Set "interrupt pending" (IIR low bit cleared) */
-			com1_regs.iir &= ~0x1;
+			com1_dev.regs.iir &= ~0x1;
 		}
 	} else {
 		/*
@@ -73,26 +176,34 @@ vcpu_process_com_data(union vm_exit *vei)
 		 * input data and return to the guest. Also clear the
 		 * interrupt info register regardless.
 		 */
-		if (com1_regs.lsr & LSR_RXRDY) {
-			vei->vei.vei_data = com1_regs.data;
-			com1_regs.data = 0x0;
-			com1_regs.lsr &= ~LSR_RXRDY;
+		if (com1_dev.regs.lsr & LSR_RXRDY) {
+			vei->vei.vei_data = com1_dev.regs.data;
+			com1_dev.regs.data = 0x0;
+			com1_dev.regs.lsr &= ~LSR_RXRDY;
 		} else {
-			/* XXX should this be com1_regs.data or 0xff? */
-			vei->vei.vei_data = com1_regs.data;
+			/* XXX should this be com1_dev.data or 0xff? */
+			vei->vei.vei_data = com1_dev.regs.data;
 			log_warnx("guest reading com1 when not ready");
 		}
 
 		/* Reading the data register always clears RXRDY from IIR */
-		com1_regs.iir &= ~IIR_RXRDY;
+		com1_dev.regs.iir &= ~IIR_RXRDY;
 
 		/*
 		 * Clear "interrupt pending" by setting IIR low bit to 1
 		 * if no interrupt are pending
 		 */
-		if (com1_regs.iir == 0x0)
-			com1_regs.iir = 0x1;
+		if (com1_dev.regs.iir == 0x0)
+			com1_dev.regs.iir = 0x1;
+
+		if (com1_dev.rcv_pending)
+			com_rcv(&com1_dev, vm_id, vcpu_id);
 	}
+
+	/* If pending interrupt, make sure it gets injected */
+	if ((com1_dev.regs.iir & 0x1) == 0)
+		return (com1_dev.irq);
+	return (0xFF);
 }
 
 /*
@@ -113,14 +224,14 @@ vcpu_process_com_lcr(union vm_exit *vei)
 	 * Write content to line control register
 	 */
 	if (vei->vei.vei_dir == VEI_DIR_OUT) {
-		com1_regs.lcr = (uint8_t)vei->vei.vei_data;
+		com1_dev.regs.lcr = (uint8_t)vei->vei.vei_data;
 	} else {
 		/*
 		 * vei_dir == VEI_DIR_IN : in instruction
 		 *
 		 * Read line control register
 		 */
-		vei->vei.vei_data = com1_regs.lcr;
+		vei->vei.vei_data = com1_dev.regs.lcr;
 	}
 }
 
@@ -145,7 +256,7 @@ vcpu_process_com_iir(union vm_exit *vei)
 	 * Write to FCR
 	 */
 	if (vei->vei.vei_dir == VEI_DIR_OUT) {
-		com1_regs.fcr = vei->vei.vei_data;
+		com1_dev.regs.fcr = vei->vei.vei_data;
 	} else {
 		/*
 		 * vei_dir == VEI_DIR_IN : in instruction
@@ -153,15 +264,15 @@ vcpu_process_com_iir(union vm_exit *vei)
 		 * Read IIR. Reading the IIR resets the TXRDY bit in the IIR
 		 * after the data is read.
 		 */
-		vei->vei.vei_data = com1_regs.iir;
-		com1_regs.iir &= ~IIR_TXRDY;
+		vei->vei.vei_data = com1_dev.regs.iir;
+		com1_dev.regs.iir &= ~IIR_TXRDY;
 
 		/*
 		 * Clear "interrupt pending" by setting IIR low bit to 1
 		 * if no interrupts are pending
 		 */
-		if (com1_regs.iir == 0x0)
-			com1_regs.iir = 0x1;
+		if (com1_dev.regs.iir == 0x0)
+			com1_dev.regs.iir = 0x1;
 	}
 }
 
@@ -184,14 +295,14 @@ vcpu_process_com_mcr(union vm_exit *vei)
 	 * Write to MCR
 	 */
 	if (vei->vei.vei_dir == VEI_DIR_OUT) {
-		com1_regs.mcr = vei->vei.vei_data;
+		com1_dev.regs.mcr = vei->vei.vei_data;
 	} else {
 		/*
 		 * vei_dir == VEI_DIR_IN : in instruction
 		 *
 		 * Read from MCR
 		 */
-		vei->vei.vei_data = com1_regs.mcr;
+		vei->vei.vei_data = com1_dev.regs.mcr;
 	}
 }
 
@@ -223,7 +334,7 @@ vcpu_process_com_lsr(union vm_exit *vei)
 		 * Read from LSR. We always report TXRDY and TSRE since we
 		 * can process output characters immediately (at any time).
 		 */
-		vei->vei.vei_data = com1_regs.lsr | LSR_TSRE | LSR_TXRDY;
+		vei->vei.vei_data = com1_dev.regs.lsr | LSR_TSRE | LSR_TXRDY;
 	}
 }
 
@@ -254,7 +365,8 @@ vcpu_process_com_msr(union vm_exit *vei)
 		 *
 		 * Read from MSR. We always report DCD, DSR, and CTS.
 		 */
-		vei->vei.vei_data = com1_regs.lsr | MSR_DCD | MSR_DSR | MSR_CTS;
+		vei->vei.vei_data =
+		    com1_dev.regs.lsr | MSR_DCD | MSR_DSR | MSR_CTS;
 	}
 }
 
@@ -280,7 +392,7 @@ vcpu_process_com_scr(union vm_exit *vei)
 	 * Write to SCR
 	 */
 	if (vei->vei.vei_dir == VEI_DIR_OUT) {
-		com1_regs.scr = vei->vei.vei_data;
+		com1_dev.regs.scr = vei->vei.vei_data;
 	} else {
 		/*
 		 * vei_dir == VEI_DIR_IN : in instruction
@@ -289,7 +401,7 @@ vcpu_process_com_scr(union vm_exit *vei)
 		 * a real scratch register, we negate what was written on
 		 * subsequent readback.
 		 */
-		vei->vei.vei_data = ~com1_regs.scr;
+		vei->vei.vei_data = ~com1_dev.regs.scr;
 	}
 }
 
@@ -312,14 +424,14 @@ vcpu_process_com_ier(union vm_exit *vei)
 	 * Write to IER
 	 */
 	if (vei->vei.vei_dir == VEI_DIR_OUT) {
-		com1_regs.ier = vei->vei.vei_data;
+		com1_dev.regs.ier = vei->vei.vei_data;
 	} else {
 		/*
 		 * vei_dir == VEI_DIR_IN : in instruction
 		 *
 		 * Read from IER
 		 */
-		vei->vei.vei_data = com1_regs.ier;
+		vei->vei.vei_data = com1_dev.regs.ier;
 	}
 }
 
@@ -340,8 +452,11 @@ vcpu_process_com_ier(union vm_exit *vei)
 uint8_t
 vcpu_exit_com(struct vm_run_params *vrp)
 {
+	uint8_t intr = 0xFF;
 	union vm_exit *vei = vrp->vrp_exit;
 
+	mutex_lock(&com1_dev.mutex);
+
 	switch (vei->vei.vei_port) {
 	case COM1_LCR:
 		vcpu_process_com_lcr(vei);
@@ -365,50 +480,11 @@ vcpu_exit_com(struct vm_run_params *vrp)
 		vcpu_process_com_scr(vei);
 		break;
 	case COM1_DATA:
-		vcpu_process_com_data(vei);
+		intr = vcpu_process_com_data(vei, vrp->vrp_vm_id,
+		    vrp->vrp_vcpu_id);
 		break;
 	}
 
-	return (0xFF);
-}
-
-/* XXX temporary until this is polled */
-int
-vcpu_com1_needs_intr(void)
-{
-	int need_irq = 0;
-	ssize_t sz;
-	char ch;
-
-	/*
-	 * Is there a new character available on com1?
-	 * If so, consume the character, buffer it into the com1 data register
-	 * assert IRQ4, and set the line status register RXRDY bit.
-	 *
-	 * XXX - move all this com intr checking to another function using poll
-	 */
-	sz = read(com1_regs.fd, &ch, sizeof(char));
-	if (sz == 1) {
-		com1_regs.lsr |= LSR_RXRDY;
-		com1_regs.data = ch;
-		/* XXX these ier and iir bits should be IER_x and IIR_x */
-		if (com1_regs.ier & 0x1) {
-			com1_regs.iir |= (2 << 1);
-			com1_regs.iir &= ~0x1;
-		}
-	}
-
-	/*
-	 * Clear "interrupt pending" by setting IIR low bit to 1 if no
-	 * interrupts are pending
-	 */
-	/* XXX these iir magic numbers should be IIR_x */
-	if ((com1_regs.iir & ~0x1) == 0x0)
-		com1_regs.iir = 0x1;
-
-	/* If pending interrupt, inject */
-	if ((com1_regs.iir & 0x1) == 0)
-		need_irq = 1;
-
-	return need_irq;
+	mutex_unlock(&com1_dev.mutex);
+	return (intr);
 }
diff --git a/usr.sbin/vmd/ns8250.h b/usr.sbin/vmd/ns8250.h
index df50a0400f0..b791b9f8bd5 100644
--- a/usr.sbin/vmd/ns8250.h
+++ b/usr.sbin/vmd/ns8250.h
@@ -39,12 +39,21 @@ struct ns8250_regs {
 	uint8_t mcr;	/* Modem Control Register */
 	uint8_t scr;	/* Scratch Register */
 	uint8_t data;	/* Unread input data */
-	int fd;		/* com port fd */
 };
 
-void ns8250_init(int);
+/* ns8250 UART device state */
+struct ns8250_dev {
+	pthread_mutex_t mutex;
+	struct ns8250_regs regs;
+	struct event event;
+	int fd;
+	int irq;
+	int rcv_pending;
+};
+
+void ns8250_init(int, uint32_t);
 uint8_t vcpu_exit_com(struct vm_run_params *);
-void vcpu_process_com_data(union vm_exit *);
+uint8_t vcpu_process_com_data(union vm_exit *, uint32_t, uint32_t);
 void vcpu_process_com_lcr(union vm_exit *);
 void vcpu_process_com_lsr(union vm_exit *);
 void vcpu_process_com_ier(union vm_exit *);
@@ -52,6 +61,3 @@ void vcpu_process_com_mcr(union vm_exit *);
 void vcpu_process_com_iir(union vm_exit *);
 void vcpu_process_com_msr(union vm_exit *);
 void vcpu_process_com_scr(union vm_exit *);
-
-/* XXX temporary until this is polled */
-int vcpu_com1_needs_intr(void);
diff --git a/usr.sbin/vmd/vmm.c b/usr.sbin/vmd/vmm.c
index 2f67f433064..f5383e1e8c4 100644
--- a/usr.sbin/vmd/vmm.c
+++ b/usr.sbin/vmd/vmm.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: vmm.c,v 1.42 2016/09/02 16:23:40 stefan Exp $	*/
+/*	$OpenBSD: vmm.c,v 1.43 2016/09/02 17:10:08 stefan Exp $	*/
 
 /*
  * Copyright (c) 2015 Mike Larkin <mlarkin@openbsd.org>
@@ -849,7 +849,7 @@ init_emulated_hw(struct vm_create_params *vcp, int *child_disks,
 	ioports_map[IO_ICU2 + 1] = vcpu_exit_i8259;
 
 	/* Init ns8250 UART */
-	ns8250_init(con_fd);
+	ns8250_init(con_fd, vcp->vcp_id);
 	for (i = COM1_DATA; i <= COM1_SCR; i++)
 		ioports_map[i] = vcpu_exit_com;
 
@@ -1322,14 +1322,9 @@ vcpu_exit(struct vm_run_params *vrp)
 	}
 
 	/* XXX this may not be irq 9 all the time */
-	/* XXX change this to poll on the tap interface */
 	if (vionet_process_rx())
 		vcpu_assert_pic_irq(vrp->vrp_vm_id, vrp->vrp_vcpu_id, 9);
 
-	/* XXX temporary until this is polled */
-	if (vcpu_com1_needs_intr())
-		vcpu_assert_pic_irq(vrp->vrp_vm_id, vrp->vrp_vcpu_id, 4);
-
 	vrp->vrp_continue = 1;
 
 	return (0);