More interrupt deadlock tomfoolery; turns out the non-XBridge workaround

doesn't appear to trigger the expected interrupt, so use a fugly nanotimeout
instead.

This makes Origin 200 disks not stall as soon as some serious I/O is intended.
They now run multiuser.

2 files changed, 96 insertions, 21 deletions

diff --git a/sys/arch/sgi/xbow/xbridge.c b/sys/arch/sgi/xbow/xbridge.c
index bc655ee5511..b4b38175121 100644
--- a/sys/arch/sgi/xbow/xbridge.c
+++ b/sys/arch/sgi/xbow/xbridge.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: xbridge.c,v 1.16 2009/05/08 18:37:28 miod Exp $	*/
+/*	$OpenBSD: xbridge.c,v 1.17 2009/05/14 21:10:33 miod Exp $	*/
 
 /*
  * Copyright (c) 2008, 2009  Miodrag Vallat.
@@ -27,6 +27,7 @@
 #include <sys/evcount.h>
 #include <sys/malloc.h>
 #include <sys/extent.h>
+#include <sys/timeout.h>
 
 #include <machine/atomic.h>
 #include <machine/autoconf.h>
@@ -499,14 +500,24 @@ struct xbridge_intr {
 	struct	xbridge_softc	*xi_bridge;
 	int	xi_intrsrc;	/* interrupt source on interrupt widget */
 	int	xi_intrbit;	/* interrupt source on BRIDGE */
+	int	xi_device;	/* device slot number */
 
 	int	(*xi_func)(void *);
 	void	*xi_arg;
 
 	struct evcount	xi_count;
 	int	 xi_level;
+
+	struct timeout xi_tmo;	/* XXX deadlock bad workaround */
 };
 
+void	xbridge_timeout(void *);	/* XXX */
+
+/* how our pci_intr_handle_t are constructed... */
+#define	XBRIDGE_INTR_HANDLE(d,b)	(0x100 | ((d) << 3) | (b))
+#define	XBRIDGE_INTR_DEVICE(h)		(((h) >> 3) & 07)
+#define	XBRIDGE_INTR_BIT(h)		((h) & 07)
+
 int
 xbridge_intr_map(struct pci_attach_args *pa, pci_intr_handle_t *ihp)
 {
@@ -532,7 +543,7 @@ xbridge_intr_map(struct pci_attach_args *pa, pci_intr_handle_t *ihp)
 	else
 		intr = device ^ 4;
 
-	*ihp = intr;
+	*ihp = XBRIDGE_INTR_HANDLE(device, intr);
 
 	return 0;
 }
@@ -542,7 +553,7 @@ xbridge_intr_string(void *cookie, pci_intr_handle_t ih)
 {
 	static char str[16];
 
-	snprintf(str, sizeof(str), "irq %d", ih);
+	snprintf(str, sizeof(str), "irq %d", XBRIDGE_INTR_BIT(ih));
 	return(str);
 }
 
@@ -553,7 +564,8 @@ xbridge_intr_establish(void *cookie, pci_intr_handle_t ih, int level,
 	struct xbridge_softc *sc = cookie;
 	struct xbridge_intr *xi;
 	uint32_t int_addr;
-	int intrbit = ih & 0x07;
+	int intrbit = XBRIDGE_INTR_BIT(ih);
+	int device = XBRIDGE_INTR_DEVICE(ih);
 	int intrsrc;
 
 	if (sc->sc_intr[intrbit] != NULL) {
@@ -592,10 +604,12 @@ xbridge_intr_establish(void *cookie, pci_intr_handle_t ih, int level,
 	xi->xi_bridge = sc;
 	xi->xi_intrsrc = intrsrc;
 	xi->xi_intrbit = intrbit;
+	xi->xi_device = device;
 	xi->xi_func = func;
 	xi->xi_arg = arg;
 	xi->xi_level = level;
 	evcount_attach(&xi->xi_count, name, &xi->xi_level, &evcount_intr);
+	timeout_set(&xi->xi_tmo, xbridge_timeout, xi);
 	sc->sc_intr[intrbit] = xi;
 
 	int_addr = ((xbow_intr_widget_register >> 30) & 0x0003ff00) | intrsrc;
@@ -605,23 +619,34 @@ xbridge_intr_establish(void *cookie, pci_intr_handle_t ih, int level,
 	bus_space_write_4(sc->sc_iot, sc->sc_regh, BRIDGE_IER,
 	    bus_space_read_4(sc->sc_iot, sc->sc_regh, BRIDGE_IER) |
 	    (1 << intrbit));
+	/*
+	 * INT_MODE register controls which interrupt pins cause
+	 * ``interrupt clear'' packets to be sent for high->low
+	 * transition.
+	 * We do not want such packets to be sent because we clear
+	 * interrupts ourselves and this would cause interrupts to
+	 * be missed.
+	 */
+#if 0
 	bus_space_write_4(sc->sc_iot, sc->sc_regh, BRIDGE_INT_MODE,
 	    bus_space_read_4(sc->sc_iot, sc->sc_regh, BRIDGE_INT_MODE) |
 	    (1 << intrbit));
+#endif
 	bus_space_write_4(sc->sc_iot, sc->sc_regh, BRIDGE_INT_DEV,
 	    bus_space_read_4(sc->sc_iot, sc->sc_regh, BRIDGE_INT_DEV) |
-	    (intrbit << (intrbit * 3)));
+	    (device << (intrbit * 3)));
 	(void)bus_space_read_4(sc->sc_iot, sc->sc_regh, WIDGET_TFLUSH);
 
-	return (void *)((uint64_t)ih | 8);	/* XXX don't return zero */
+	return (void *)((uint64_t)ih);
 }
 
 void
-xbridge_intr_disestablish(void *cookie, void *ih)
+xbridge_intr_disestablish(void *cookie, void *vih)
 {
 	struct xbridge_softc *sc = cookie;
 	struct xbridge_intr *xi;
-	int intrbit = (uint64_t)ih & 0x07;
+	pci_intr_handle_t ih = (pci_intr_handle_t)(uint64_t)vih;
+	int intrbit = XBRIDGE_INTR_BIT(ih);
 
 	/* should not happen */
 	if ((xi = sc->sc_intr[intrbit]) == NULL)
@@ -652,43 +677,90 @@ xbridge_intr_handler(void *v)
 {
 	struct xbridge_intr *xi = v;
 	struct xbridge_softc *sc = xi->xi_bridge;
+#if 0
 	uint16_t nasid = 0;	/* XXX */
+#endif
 	int rc;
+	int spurious;
 
 	if (xi == NULL) {
-		printf("%s: spurious interrupt on source %d\n",
-		    sc->sc_dev.dv_xname, xi->xi_intrsrc);
+		printf("%s: spurious irq %d\n",
+		    sc->sc_dev.dv_xname, xi->xi_intrbit);
 		return 0;
 	}
 
+	if (!ISSET(sc->sc_flags, XBRIDGE_FLAGS_XBRIDGE))
+		timeout_del(&xi->xi_tmo);
+
+	/*
+	 * Flush PCI write buffers before servicing the interrupt.
+	 */
+	bus_space_read_4(sc->sc_iot, sc->sc_regh,
+	    BRIDGE_DEVICE_WBFLUSH(xi->xi_device));
+
+	if ((bus_space_read_4(sc->sc_iot, sc->sc_regh, BRIDGE_ISR) &
+	    (1 << xi->xi_intrbit)) == 0) {
+		spurious = 1;
+#ifdef DEBUG
+		printf("%s: irq %d but not pending in ISR %08x\n",
+		    sc->sc_dev.dv_xname, xi->xi_intrbit,
+		    bus_space_read_4(sc->sc_iot, sc->sc_regh, BRIDGE_ISR));
+#endif
+	} else
+		spurious = 0;
+
 	if ((rc = (*xi->xi_func)(xi->xi_arg)) != 0)
 		xi->xi_count.ec_count++;
+	if (rc == 0 && spurious == 0)
+		printf("%s: spurious irq %d\n",
+		    sc->sc_dev.dv_xname, xi->xi_intrbit);
 
 	/*
 	 * There is a known BRIDGE race in which, if two interrupts
 	 * on two different pins occur within 60nS of each other,
-	 * further interrupts on the first pin do not cause an interrupt
-	 * to be sent.
+	 * further interrupts on the first pin do not cause an
+	 * interrupt to be sent.
 	 *
-	 * The workaround against this is to check if our interrupt source
-	 * is still active (i.e. another interrupt is pending), in which
-	 * case we force an interrupt anyway.
+	 * The workaround against this is to check if our interrupt
+	 * source is still active (i.e. another interrupt is pending),
+	 * in which case we force an interrupt anyway.
 	 *
-	 * The XBridge even has a nice facility to do this, where we do not
-	 * even have to check if our interrupt is pending.
+	 * The XBridge even has a nice facility to do this, where we
+	 * do not even have to check if our interrupt is pending.
 	 */
 
 	if (ISSET(sc->sc_flags, XBRIDGE_FLAGS_XBRIDGE)) {
 		bus_space_write_4(sc->sc_iot, sc->sc_regh,
 		    BRIDGE_INT_FORCE_PIN(xi->xi_intrbit), 1);
 	} else {
-		if (bus_space_read_4(sc->sc_iot, sc->sc_regh, BRIDGE_ISR) &
-		    (1 << xi->xi_intrbit))
+		if (bus_space_read_4(sc->sc_iot, sc->sc_regh,
+		    BRIDGE_ISR) & (1 << xi->xi_intrbit)) {
+#if 0
+			/* XXX This doesn't appear to work */
 			IP27_RHUB_PI_S(nasid, 0, HUB_IR_CHANGE,
 			    HUB_IR_SET | xi->xi_intrsrc);
+#else
+			timeout_add(&xi->xi_tmo, 1);
+#endif
+		}
 	}
 
-	return rc;
+	return 1;
+}
+
+/*
+ * Bridge (not XBridge) ugly workaround for the interrupt deadlock
+ * problem mentioned above.
+ */
+void
+xbridge_timeout(void *v)
+{
+	struct xbridge_intr *xi = v;
+	struct xbridge_softc *sc = xi->xi_bridge;
+
+	if ((bus_space_read_4(sc->sc_iot, sc->sc_regh, BRIDGE_ISR) &
+	    (1 << xi->xi_intrbit)) != 0)
+		xbridge_intr_handler(v);
 }
 
 /*
diff --git a/sys/arch/sgi/xbow/xbridgereg.h b/sys/arch/sgi/xbow/xbridgereg.h
index e0a176cce03..9025bac47a0 100644
--- a/sys/arch/sgi/xbow/xbridgereg.h
+++ b/sys/arch/sgi/xbow/xbridgereg.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: xbridgereg.h,v 1.4 2009/05/06 20:08:47 miod Exp $	*/
+/*	$OpenBSD: xbridgereg.h,v 1.5 2009/05/14 21:10:33 miod Exp $	*/
 
 /*
  * Copyright (c) 2008 Miodrag Vallat.
@@ -108,6 +108,9 @@
 #define	BRIDGE_DEVIO_SIZE(d) \
 	((d) < 2 ? BRIDGE_DEVIO_LARGE : BRIDGE_DEVIO_SHORT)
 
+
+#define	BRIDGE_DEVICE_WBFLUSH(d)	(0x00000244 + 8 * (d))
+
 /*
  * Read Response Buffer configuration registers
  *