Allow 4 vio(4) interfaces in each VM. Also fix a bad interrupt assignment that

caused IRQ9 to be shared between the second disk device and the vio(4)s,
which caused poor network performance.

ok reyk, stefan

6 files changed, 45 insertions, 26 deletions

diff --git a/sys/arch/amd64/include/vmmvar.h b/sys/arch/amd64/include/vmmvar.h
index 13b9d261ee4..d573ee07fe4 100644
--- a/sys/arch/amd64/include/vmmvar.h
+++ b/sys/arch/amd64/include/vmmvar.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: vmmvar.h,v 1.21 2016/10/06 07:37:51 mlarkin Exp $	*/
+/*	$OpenBSD: vmmvar.h,v 1.22 2016/10/12 06:56:54 mlarkin Exp $	*/
 /*
  * Copyright (c) 2014 Mike Larkin <mlarkin@openbsd.org>
  *
@@ -30,7 +30,7 @@
 #define VMM_MAX_KERNEL_PATH	128
 #define VMM_MAX_VCPUS_PER_VM	64
 #define VMM_MAX_VM_MEM_SIZE	(512 * 1024)
-#define VMM_MAX_NICS_PER_VM	2
+#define VMM_MAX_NICS_PER_VM	4
 
 #define VMM_PCI_MMIO_BAR_BASE	0xF0000000
 #define VMM_PCI_MMIO_BAR_END	0xF0FFFFFF
diff --git a/usr.sbin/vmd/pci.c b/usr.sbin/vmd/pci.c
index cfd24eca438..88b2dd6f6f1 100644
--- a/usr.sbin/vmd/pci.c
+++ b/usr.sbin/vmd/pci.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: pci.c,v 1.9 2016/09/01 16:40:06 mlarkin Exp $	*/
+/*	$OpenBSD: pci.c,v 1.10 2016/10/12 06:56:54 mlarkin Exp $	*/
 
 /*
  * Copyright (c) 2015 Mike Larkin <mlarkin@openbsd.org>
@@ -32,7 +32,7 @@ struct pci pci;
 extern char *__progname;
 
 /* PIC IRQs, assigned to devices in order */
-const uint8_t pci_pic_irqs[PCI_MAX_PIC_IRQS] = {3, 5, 9, 10, 11};
+const uint8_t pci_pic_irqs[PCI_MAX_PIC_IRQS] = {3, 5, 7, 9, 10, 11, 14};
 
 /*
  * pci_add_bar
@@ -101,6 +101,26 @@ pci_add_bar(uint8_t id, uint32_t type, void *barfn, void *cookie)
 }
 
 /*
+ * pci_get_dev_irq
+ *
+ * Returns the IRQ for the specified PCI device
+ *
+ * Parameters:
+ *  id: PCI device id to return IRQ for
+ *
+ * Return values:
+ *  The IRQ for the device, or 0xff if no device IRQ assigned
+ */
+uint8_t
+pci_get_dev_irq(uint8_t id)
+{
+	if (pci.pci_devices[id].pd_int)
+		return pci.pci_devices[id].pd_irq;
+	else
+		return 0xFF;
+}
+
+/*
  * pci_add_device
  *
  * Adds a PCI device to the guest VM defined by the supplied parameters.
diff --git a/usr.sbin/vmd/pci.h b/usr.sbin/vmd/pci.h
index dfeefda16ba..d0313514810 100644
--- a/usr.sbin/vmd/pci.h
+++ b/usr.sbin/vmd/pci.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: pci.h,v 1.2 2015/11/22 21:51:32 reyk Exp $	*/
+/*	$OpenBSD: pci.h,v 1.3 2016/10/12 06:56:54 mlarkin Exp $	*/
 
 /*
  * Copyright (c) 2015 Mike Larkin <mlarkin@openbsd.org>
@@ -25,7 +25,7 @@
 #define PCI_BAR_TYPE_IO		0x0
 #define PCI_BAR_TYPE_MMIO	0x1
 
-#define PCI_MAX_PIC_IRQS	5
+#define PCI_MAX_PIC_IRQS	7
 
 typedef int (*pci_cs_fn_t)(int dir, uint8_t reg, uint32_t *data);
 typedef int (*pci_iobar_fn_t)(int dir, uint16_t reg, uint32_t *data, uint8_t *,
@@ -89,3 +89,4 @@ void pci_init(void);
 int pci_add_device(uint8_t *, uint16_t, uint16_t, uint8_t, uint8_t, uint16_t,
     uint16_t, uint8_t, pci_cs_fn_t);
 int pci_add_bar(uint8_t, uint32_t, void *, void *);
+uint8_t pci_get_dev_irq(uint8_t);
diff --git a/usr.sbin/vmd/virtio.c b/usr.sbin/vmd/virtio.c
index 4af04d5cb66..57dbdb9d193 100644
--- a/usr.sbin/vmd/virtio.c
+++ b/usr.sbin/vmd/virtio.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: virtio.c,v 1.21 2016/10/05 17:30:13 reyk Exp $	*/
+/*	$OpenBSD: virtio.c,v 1.22 2016/10/12 06:56:54 mlarkin Exp $	*/
 
 /*
  * Copyright (c) 2015 Mike Larkin <mlarkin@openbsd.org>
@@ -945,13 +945,15 @@ vionet_rx_event(int fd, short kind, void *arg)
  * called on VCPU exit: it can happen that not all data fits into the
  * receive queue of the vionet_dev immediately. So any outstanding data
  * is handled here.
+ *
+ * Parameters:
+ *  vm_id: VM ID of the VM for which to process vionet events
  */
-int
-vionet_process_rx(void)
+void
+vionet_process_rx(uint32_t vm_id)
 {
-	int i, num_enq;
+	int i;
 
-	num_enq = 0;
 	for (i = 0 ; i < nr_vionet; i++) {
 		mutex_lock(&vionet[i].mutex);
 		if (!vionet[i].rx_added) {
@@ -959,16 +961,13 @@ vionet_process_rx(void)
 			continue;
 		}
 
-		if (vionet[i].rx_pending)
-			num_enq += vionet_rx(&vionet[i]);
+		if (vionet[i].rx_pending) {
+			if (vionet_rx(&vionet[i])) {
+				vcpu_assert_pic_irq(vm_id, 0, vionet[i].irq);
+			}
+		}
 		mutex_unlock(&vionet[i].mutex);
 	}
-
-	/*
-	 * XXX returns the number of packets enqueued across all vionet, which
-	 * may not be right for VMs with more than one vionet.
-	 */
-	return (num_enq);
 }
 
 /*
@@ -1295,7 +1294,7 @@ virtio_init(struct vm_create_params *vcp, int *child_disks, int *child_taps)
 			vionet[i].fd = child_taps[i];
 			vionet[i].rx_pending = 0;
 			vionet[i].vm_id = vcp->vcp_id;
-			vionet[i].irq = 9; /* XXX */
+			vionet[i].irq = pci_get_dev_irq(id);
 
 			event_set(&vionet[i].event, vionet[i].fd,
 			    EV_READ | EV_PERSIST, vionet_rx_event, &vionet[i]);
diff --git a/usr.sbin/vmd/virtio.h b/usr.sbin/vmd/virtio.h
index a231ea65026..de048d6b313 100644
--- a/usr.sbin/vmd/virtio.h
+++ b/usr.sbin/vmd/virtio.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: virtio.h,v 1.5 2016/09/02 17:08:28 stefan Exp $	*/
+/*	$OpenBSD: virtio.h,v 1.6 2016/10/12 06:56:54 mlarkin Exp $	*/
 
 /*
  * Copyright (c) 2015 Mike Larkin <mlarkin@openbsd.org>
@@ -152,7 +152,7 @@ void vionet_update_qs(struct vionet_dev *);
 void vionet_update_qa(struct vionet_dev *);
 int vionet_notifyq(struct vionet_dev *);
 void vionet_notify_rx(struct vionet_dev *);
-int vionet_process_rx(void);
+void vionet_process_rx(uint32_t);
 int vionet_enq_rx(struct vionet_dev *, char *, ssize_t, int *);
 
 const char *vioblk_cmd_name(uint32_t);
diff --git a/usr.sbin/vmd/vmm.c b/usr.sbin/vmd/vmm.c
index b4ebbb70f38..bd874cdb75d 100644
--- a/usr.sbin/vmd/vmm.c
+++ b/usr.sbin/vmd/vmm.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: vmm.c,v 1.49 2016/10/06 20:41:28 reyk Exp $	*/
+/*	$OpenBSD: vmm.c,v 1.50 2016/10/12 06:56:54 mlarkin Exp $	*/
 
 /*
  * Copyright (c) 2015 Mike Larkin <mlarkin@openbsd.org>
@@ -1349,9 +1349,8 @@ vcpu_exit(struct vm_run_params *vrp)
 		    __progname, vrp->vrp_exit_reason);
 	}
 
-	/* XXX this may not be irq 9 all the time */
-	if (vionet_process_rx())
-		vcpu_assert_pic_irq(vrp->vrp_vm_id, vrp->vrp_vcpu_id, 9);
+	/* Process any pending traffic */
+	vionet_process_rx(vrp->vrp_vm_id);
 
 	vrp->vrp_continue = 1;