Move xen interrupt handlers to dedicated task queues

Handling receive and transmit for multiple networking interfaces
in a "shared interrupt" within normal interrupt vector code path
introduces too much delay from the hypervisor POV which prevents
it from injecting further completion event interrupts for Rx and
Tx queues.

Additionally, Netfront backend driver includes a mechanism to
detect Rx ring stalls and "turn the carrier off" when the guest
is not replenishing the ring (e.g. due to missing completion
interrupts) that relies on guest waking up periodically and making
sure that the Rx ring completion handling is progressing.

Having tried both task queue + timeout and interrupts + timeout
approaches, it appears that using  the task queue is more flexible
and provides superior performance under heavy network load.

2 files changed, 30 insertions, 13 deletions

diff --git a/sys/dev/pv/xen.c b/sys/dev/pv/xen.c
index 6e13b129688..d534be964a4 100644
--- a/sys/dev/pv/xen.c
+++ b/sys/dev/pv/xen.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: xen.c,v 1.56 2016/04/28 16:40:10 mikeb Exp $	*/
+/*	$OpenBSD: xen.c,v 1.57 2016/07/29 21:27:43 mikeb Exp $	*/
 
 /*
  * Copyright (c) 2015 Mike Belopuhov
@@ -642,13 +642,22 @@ xen_intr(void)
 				continue;
 			}
 			xi->xi_evcnt.ec_count++;
-			if (xi->xi_handler)
-				xi->xi_handler(xi->xi_arg);
+			task_add(xi->xi_taskq, &xi->xi_task);
 		}
 	}
 }
 
 void
+xen_intr_schedule(xen_intr_handle_t xih)
+{
+	struct xen_softc *sc = xen_sc;
+	struct xen_intsrc *xi;
+
+	if ((xi = xen_lookup_intsrc(sc, (evtchn_port_t)xih)) != NULL)
+		task_add(xi->xi_taskq, &xi->xi_task);
+}
+
+void
 xen_intr_signal(xen_intr_handle_t xih)
 {
 	struct xen_softc *sc = xen_sc;
@@ -685,10 +694,17 @@ xen_intr_establish(evtchn_port_t port, xen_intr_handle_t *xih, int domain,
 	if (xi == NULL)
 		return (-1);
 
-	xi->xi_handler = handler;
-	xi->xi_arg = arg;
 	xi->xi_port = (evtchn_port_t)*xih;
 
+	xi->xi_taskq = taskq_create(name, 1, IPL_NET, TASKQ_MPSAFE);
+	if (!xi->xi_taskq) {
+		printf("%s: failed to create interrupt task for %s\n",
+		    sc->sc_dev.dv_xname, name);
+		free(xi, M_DEVBUF, sizeof(*xi));
+		return (-1);
+	}
+	task_set(&xi->xi_task, handler, arg);
+
 	if (port == 0) {
 		/* We're being asked to allocate a new event port */
 		memset(&eau, 0, sizeof(eau));
diff --git a/sys/dev/pv/xenvar.h b/sys/dev/pv/xenvar.h
index a23a615775b..d4e3fc5fc9d 100644
--- a/sys/dev/pv/xenvar.h
+++ b/sys/dev/pv/xenvar.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: xenvar.h,v 1.32 2016/04/19 18:15:41 mikeb Exp $	*/
+/*	$OpenBSD: xenvar.h,v 1.33 2016/07/29 21:27:43 mikeb Exp $	*/
 
 /*
  * Copyright (c) 2015 Mike Belopuhov
@@ -28,13 +28,13 @@
 #endif
 
 struct xen_intsrc {
-	SLIST_ENTRY(xen_intsrc)	  xi_entry;
-	void			(*xi_handler)(void *);
-	void			 *xi_arg;
-	struct evcount		  xi_evcnt;
-	evtchn_port_t		  xi_port;
-	short			  xi_noclose;
-	short			  xi_masked;
+	SLIST_ENTRY(xen_intsrc)	 xi_entry;
+	struct evcount		 xi_evcnt;
+	evtchn_port_t		 xi_port;
+	short			 xi_noclose;
+	short			 xi_masked;
+	struct task		 xi_task;
+	struct taskq		*xi_taskq;
 };
 
 struct xen_gntent {
@@ -114,6 +114,7 @@ typedef uint32_t xen_intr_handle_t;
 void	xen_intr(void);
 void	xen_intr_ack(void);
 void	xen_intr_signal(xen_intr_handle_t);
+void	xen_intr_schedule(xen_intr_handle_t);
 int	xen_intr_establish(evtchn_port_t, xen_intr_handle_t *, int,
 	    void (*)(void *), void *, char *);
 int	xen_intr_disestablish(xen_intr_handle_t);