More progress taming the xbow and the pci bridge; still needs code to write,

but (currently commented out) code makes isp happier on IP27 and IP35, to the
point of seeing disks (but considering them offline so far).

3 files changed, 167 insertions, 73 deletions

diff --git a/sys/arch/sgi/xbow/xbow.c b/sys/arch/sgi/xbow/xbow.c
index 63d6b16019d..72a00042e01 100644
--- a/sys/arch/sgi/xbow/xbow.c
+++ b/sys/arch/sgi/xbow/xbow.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: xbow.c,v 1.5 2009/04/19 12:52:33 miod Exp $	*/
+/*	$OpenBSD: xbow.c,v 1.6 2009/05/02 21:30:13 miod Exp $	*/
 
 /*
  * Copyright (c) 2008, 2009 Miodrag Vallat.
@@ -307,7 +307,7 @@ xbowattach(struct device *parent, struct device *self, void *aux)
 	 */
 	xbow_intr_widget_register = (1UL << 47) /* XIO I/O space */ |
 	    ((paddr_t)IP27_RHUB_ADDR(nasid, HUB_IR_CHANGE) -
-	     IP27_NODE_IO_BASE(nasid)) /* HUB register offset */;
+	     IP27_NODE_IO_BASE(0)) /* HUB register offset */;
 
 	/*
 	 * If widget 0 reports itself as a bridge, this is not a
@@ -316,8 +316,8 @@ xbowattach(struct device *parent, struct device *self, void *aux)
 	 */
 	if (vendor == XBOW_VENDOR_SGI4 && product == XBOW_PRODUCT_SGI4_BRIDGE) {
 		/*
-		 * Interrupt widget is #a (this is another facet of this
-		 * bridge).
+		 * Interrupt widget is hardwired to #a (this is another
+		 * facet of this bridge).
 		 */
 		xbow_intr_widget = 0x0a;
 
@@ -325,13 +325,26 @@ xbowattach(struct device *parent, struct device *self, void *aux)
 		    xbowsubmatch_pass2, xbowprint_pass2);
 	} else {
 		/*
+		 * XXX This widget number is actually the Hub part of the
+		 * XXX crossbow, and is where memory and interrupt logic
+		 * XXX resources are connected to.
+		 * XXX The exact widget number ought to be computed from
+		 * XXX the KL configuration graph; I'm hardcoding it for
+		 * XXX now because I am lazy and we only care about the
+		 * XXX first node at the moment. -- miod
+		 */
+		if (sys_config.system_type != SGI_OCTANE)
+			xbow_intr_widget = 0x0a;
+
+		/*
 		 * Enumerate the other widgets.
 		 * We'll do two passes - one to give the first Heart or a Hub a
 		 * chance to setup interrupt routing, and one to attach all
 		 * other widgets.
 		 */
-		xbow_enumerate(self, nasid, 0,
-		    xbowsubmatch_pass1, xbowprint_pass1);
+		if (xbow_intr_widget == 0)
+			xbow_enumerate(self, nasid, 0,
+			    xbowsubmatch_pass1, xbowprint_pass1);
 		xbow_enumerate(self, nasid, xbow_intr_widget,
 		    xbowsubmatch_pass2, xbowprint_pass2);
 	}
diff --git a/sys/arch/sgi/xbow/xbridge.c b/sys/arch/sgi/xbow/xbridge.c
index 95b546cd0c2..7522bd6780f 100644
--- a/sys/arch/sgi/xbow/xbridge.c
+++ b/sys/arch/sgi/xbow/xbridge.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: xbridge.c,v 1.12 2009/04/19 18:37:31 miod Exp $	*/
+/*	$OpenBSD: xbridge.c,v 1.13 2009/05/02 21:30:13 miod Exp $	*/
 
 /*
  * Copyright (c) 2008 Miodrag Vallat.
@@ -54,7 +54,8 @@ struct xbridge_intr;
 
 struct xbridge_softc {
 	struct device	sc_dev;
-	int		sc_rev;
+	int		sc_flags;
+#define	XBRIDGE_FLAGS_XBRIDGE	0x01	/* is XBridge vs Bridge */
 	int		sc_widget;
 	struct mips_pci_chipset sc_pc;
 
@@ -67,6 +68,8 @@ struct xbridge_softc {
 
 	int		sc_intrbit[BRIDGE_NINTRS];
 	struct xbridge_intr	*sc_intr[BRIDGE_NINTRS];
+
+	pcireg_t	sc_devices[BRIDGE_NSLOTS];
 };
 
 const struct cfattach xbridge_ca = {
@@ -160,10 +163,12 @@ xbridge_attach(struct device *parent, struct device *self, void *aux)
 	struct xbow_attach_args *xaa = aux;
 	int i;
 
-	sc->sc_rev = xaa->xaa_revision;
 	sc->sc_widget = xaa->xaa_widget;
 
-	printf(" revision %d\n", sc->sc_rev);
+	printf(" revision %d\n", xaa->xaa_revision);
+	if (xaa->xaa_vendor == XBOW_VENDOR_SGI3 &&
+	    xaa->xaa_product == XBOW_PRODUCT_SGI3_XBRIDGE)
+		sc->sc_flags |= XBRIDGE_FLAGS_XBRIDGE;
 
 	/*
 	 * Map Bridge registers.
@@ -196,7 +201,8 @@ xbridge_attach(struct device *parent, struct device *self, void *aux)
 		    sizeof(*sc->sc_mem_bus_space));
 		sc->sc_mem_bus_space->bus_base += BRIDGE_PCI_MEM_SPACE_BASE;
 
-		if (sc->sc_rev >= 4) {
+		if (!ISSET(sc->sc_flags, XBRIDGE_FLAGS_XBRIDGE) &&
+		    xaa->xaa_revision >= 4) {
 			/* Unrestricted I/O mappings in the large window */
 			bcopy(xaa->xaa_long_tag, sc->sc_io_bus_space,
 			    sizeof(*sc->sc_io_bus_space));
@@ -261,28 +267,62 @@ xbridge_attach(struct device *parent, struct device *self, void *aux)
 	sc->sc_pc.pc_intr_disestablish = xbridge_intr_disestablish;
 
 	/*
-	 * XXX The following magic sequence is supposedly needed for DMA
-	 * XXX to work correctly.  I have no idea what it really does.
+	 * Configure the direct DMA window to access the low 2GB of memory.
 	 */
-	if (sys_config.system_type == SGI_OCTANE) {
+
+	if (sys_config.system_type == SGI_OCTANE)
 		bus_space_write_4(sc->sc_iot, sc->sc_regh, BRIDGE_DIR_MAP,
-		    (xbow_intr_widget << 20) | (1 << 17));
-#if 0
-		bus_space_write_4(sc->sc_iot, sc->sc_regh, 0x284,
+		    (xbow_intr_widget << BRIDGE_DIRMAP_WIDGET_SHIFT) |
+		    BRIDGE_DIRMAP_ADD_512MB);
+	else
+		bus_space_write_4(sc->sc_iot, sc->sc_regh, BRIDGE_DIR_MAP,
+		    xbow_intr_widget << BRIDGE_DIRMAP_WIDGET_SHIFT);
+
+	/*
+	 * Gather device identification for all slots.
+	 * We need this to be able to allocate RRBs correctly, and also
+	 * to be able to check quickly whether a given device is an IOC3.
+	 */
+
+	for (i = 0; i < BRIDGE_NSLOTS; i++) {
+		paddr_t pa;
+
+		pa = sc->sc_regh + BRIDGE_PCI_CFG_SPACE +
+		    (i << 12) + PCI_ID_REG;
+		if (guarded_read_4(pa, &sc->sc_devices[i]) != 0)
+			sc->sc_devices[i] = 0xffffffff;
+	}
+
+#if 0	/* XXX write proper RRB allocation code */
+	if (sys_config.system_type == SGI_OCTANE) {
+		bus_space_write_4(sc->sc_iot, sc->sc_regh, BRIDGE_RRB_EVEN,
 		    0x99889988 | 0x44440000);
-		bus_space_write_4(sc->sc_iot, sc->sc_regh, 0x28c,
+		bus_space_write_4(sc->sc_iot, sc->sc_regh, BRIDGE_RRB_ODD,
 		    0x99889988 | 0x44440000);
-#endif
 	} else {
-		bus_space_write_4(sc->sc_iot, sc->sc_regh, BRIDGE_DIR_MAP,
-		    xbow_intr_widget << 20);
-#if 0
-		bus_space_write_4(sc->sc_iot, sc->sc_regh, 0x284,
+		bus_space_write_4(sc->sc_iot, sc->sc_regh, BRIDGE_RRB_EVEN,
 		    0xba98ba98 | 0x44440000);
-		bus_space_write_4(sc->sc_iot, sc->sc_regh, 0x28c,
+		bus_space_write_4(sc->sc_iot, sc->sc_regh, BRIDGE_RRB_ODD,
 		    0xba98ba98 | 0x44440000);
+	}
 #endif
+
+#if 0	/* XXX write proper I/O mapping allocation code */
+	for (i = 0; i < BRIDGE_NSLOTS; i++) {
+		uint32_t dio;
+
+		if (sc->sc_devices[i] ==
+		    PCI_ID_CODE(PCI_VENDOR_SGI, PCI_PRODUCT_SGI_IOC3))
+			continue;
+
+		dio = bus_space_read_4(sc->sc_iot, sc->sc_regh,
+		    BRIDGE_DEVICE(i));
+		dio |= BRIDGE_DEVICE_SWAP_PMU | BRIDGE_DEVICE_SWAP_DIR |
+		    BRIDGE_DEVICE_COHERENT | BRIDGE_DEVICE_BARRIER;
+		bus_space_write_4(sc->sc_iot, sc->sc_regh, BRIDGE_DEVICE(i),
+		    dio);
 	}
+#endif
 
 	(void)bus_space_read_4(sc->sc_iot, sc->sc_regh, WIDGET_TFLUSH);
 
@@ -293,8 +333,9 @@ xbridge_attach(struct device *parent, struct device *self, void *aux)
 	bus_space_write_4(sc->sc_iot, sc->sc_regh, BRIDGE_IER, 0);
 	bus_space_write_4(sc->sc_iot, sc->sc_regh, BRIDGE_INT_MODE, 0);
 	bus_space_write_4(sc->sc_iot, sc->sc_regh, BRIDGE_INT_DEV, 0);
+
 	bus_space_write_4(sc->sc_iot, sc->sc_regh, WIDGET_INTDEST_ADDR_UPPER,
-	    (xbow_intr_widget_register  >> 32) | (xbow_intr_widget << 16));
+	    (xbow_intr_widget_register >> 32) | (xbow_intr_widget << 16));
 	bus_space_write_4(sc->sc_iot, sc->sc_regh, WIDGET_INTDEST_ADDR_LOWER,
 	    (uint32_t)xbow_intr_widget_register);
 
@@ -382,9 +423,10 @@ pcireg_t
 xbridge_conf_read(void *cookie, pcitag_t tag, int offset)
 {
 	struct xbridge_softc *sc = cookie;
-	pcireg_t id, data;
+	pcireg_t data;
 	int bus, dev, fn;
 	paddr_t pa;
+	int skip;
 	int s;
 
 	/* XXX should actually disable interrupts? */
@@ -400,7 +442,9 @@ xbridge_conf_read(void *cookie, pcitag_t tag, int offset)
 
 	/*
 	 * IOC3 devices only implement a subset of the PCI configuration
-	 * registers.
+	 * registers (supposedly only the first 0x20 bytes, however
+	 * writing to the second BAR also writes to the first).
+	 *
 	 * Depending on which particular model we encounter, things may
 	 * seem to work, or write access to nonexisting registers would
 	 * completely freeze the machine.
@@ -409,12 +453,9 @@ xbridge_conf_read(void *cookie, pcitag_t tag, int offset)
 	 * existing registers ourselves.
 	 */
 
-	if (guarded_read_4(pa + PCI_ID_REG, &id) != 0) {
-		splx(s);
-		return 0xffffffff;
-	}
-
-	if (id == PCI_ID_CODE(PCI_VENDOR_SGI, PCI_PRODUCT_SGI_IOC3)) {
+	skip = 0;
+	if (bus == 0 && sc->sc_devices[dev] ==
+	    PCI_ID_CODE(PCI_VENDOR_SGI, PCI_PRODUCT_SGI_IOC3)) {
 		switch (offset) {
 		case PCI_ID_REG:
 		case PCI_COMMAND_STATUS_REG:
@@ -422,21 +463,21 @@ xbridge_conf_read(void *cookie, pcitag_t tag, int offset)
 		case PCI_BHLC_REG:
 		case PCI_MAPREG_START:
 			/* These registers are implemented. Go ahead. */
-			id = 0;
 			break;
 		case PCI_INTERRUPT_REG:
 			/* This register is not implemented. Fake it. */
 			data = PCI_INTERRUPT_PIN_A << PCI_INTERRUPT_PIN_SHIFT;
+			skip = 1;
 			break;
 		default:
 			/* These registers are not implemented. */
 			data = 0;
+			skip = 1;
 			break;
 		}
-	} else
-		id = 0;
+	}
 
-	if (id == 0) {
+	if (skip == 0) {
 		pa += (fn << 8) + offset;
 		if (guarded_read_4(pa, &data) != 0)
 			data = 0xffffffff;
@@ -450,9 +491,9 @@ void
 xbridge_conf_write(void *cookie, pcitag_t tag, int offset, pcireg_t data)
 {
 	struct xbridge_softc *sc = cookie;
-	pcireg_t id;
 	int bus, dev, fn;
 	paddr_t pa;
+	int skip;
 	int s;
 
 	/* XXX should actually disable interrupts? */
@@ -477,12 +518,9 @@ xbridge_conf_write(void *cookie, pcitag_t tag, int offset, pcireg_t data)
 	 * existing registers ourselves.
 	 */
 
-	if (guarded_read_4(pa + PCI_ID_REG, &id) != 0) {
-		splx(s);
-		return;
-	}
-
-	if (id == PCI_ID_CODE(PCI_VENDOR_SGI, PCI_PRODUCT_SGI_IOC3)) {
+	skip = 0;
+	if (bus == 0 && sc->sc_devices[dev] ==
+	    PCI_ID_CODE(PCI_VENDOR_SGI, PCI_PRODUCT_SGI_IOC3)) {
 		switch (offset) {
 		case PCI_COMMAND_STATUS_REG:
 			/*
@@ -497,16 +535,15 @@ xbridge_conf_write(void *cookie, pcitag_t tag, int offset, pcireg_t data)
 		case PCI_BHLC_REG:
 		case PCI_MAPREG_START:
 			/* These registers are implemented. Go ahead. */
-			id = 0;
 			break;
 		default:
 			/* These registers are not implemented. */
+			skip = 1;
 			break;
 		}
-	} else
-		id = 0;
+	}
 
-	if (id == 0) {
+	if (skip == 0) {
 		pa += (fn << 8) + offset;
 		guarded_write_4(pa, data);
 	}
@@ -579,7 +616,6 @@ xbridge_intr_establish(void *cookie, pci_intr_handle_t ih, int level,
 	uint32_t int_addr;
 	int intrbit = ih & 0x07;
 	int intrsrc;
-	int16_t nasid = 0;	/* XXX */
 
 	if (sc->sc_intr[intrbit] != NULL) {
 		printf("%s: nested interrupts are not supported\n", __func__);
@@ -622,16 +658,7 @@ xbridge_intr_establish(void *cookie, pci_intr_handle_t ih, int level,
 	evcount_attach(&xi->xi_count, name, &xi->xi_level, &evcount_intr);
 	sc->sc_intr[intrbit] = xi;
 
-	switch (sys_config.system_type) {
-	case SGI_OCTANE:
-		int_addr = intrsrc;
-		break;
-	default:
-	case SGI_O200:
-	case SGI_O300:
-		int_addr = 0x20000 | intrsrc | (nasid << 8);
-		break;
-	}
+	int_addr = ((xbow_intr_widget_register >> 30) & 0x0003ff00) | intrsrc;
 
 	bus_space_write_4(sc->sc_iot, sc->sc_regh, BRIDGE_INT_ADDR(intrbit),
 	    int_addr);
@@ -831,16 +858,19 @@ bus_addr_t
 xbridge_pa_to_device(paddr_t pa)
 {
 	/*
-	 * On the Octane, we try to use the direct DMA window whenever
-	 * possible; this allows hardware limited to 32 bit DMA addresses
-	 * to work.
+	 * Try to use the direct DMA window whenever possible; this
+	 * allows hardware limited to 32 bit DMA addresses to work.
+	 *
+	 * XXX There ought to be a specific bus_dma flag to let the
+	 * XXX code know whether the given device can handle 64 bit
+	 * XXX dma addresses or not.
 	 */
 
-	if (sys_config.system_type == SGI_OCTANE) {
+	if (sys_config.system_type == SGI_OCTANE)
 		pa -= IP30_MEMORY_BASE;
-		if (pa < BRIDGE_DMA_DIRECT_LENGTH)
-			return pa + BRIDGE_DMA_DIRECT_BASE;
-	}
+
+	if (pa < BRIDGE_DMA_DIRECT_LENGTH)
+		return pa + BRIDGE_DMA_DIRECT_BASE;
 
 	pa += ((uint64_t)xbow_intr_widget << 60) | (1UL << 56);
 
@@ -853,7 +883,7 @@ xbridge_device_to_pa(bus_addr_t addr)
 	paddr_t pa;
 
 	pa = addr & ((1UL << 56) - 1);
-	if (sys_config.system_type == SGI_OCTANE && pa == (paddr_t)addr)
+	if (pa == (paddr_t)addr)	/* was in direct DMA window */
 		pa = addr - BRIDGE_DMA_DIRECT_BASE;
 
 	if (sys_config.system_type == SGI_OCTANE)
diff --git a/sys/arch/sgi/xbow/xbridgereg.h b/sys/arch/sgi/xbow/xbridgereg.h
index cbb38b32751..81675a01688 100644
--- a/sys/arch/sgi/xbow/xbridgereg.h
+++ b/sys/arch/sgi/xbow/xbridgereg.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: xbridgereg.h,v 1.1 2008/04/07 22:47:40 miod Exp $	*/
+/*	$OpenBSD: xbridgereg.h,v 1.2 2009/05/02 21:30:13 miod Exp $	*/
 
 /*
  * Copyright (c) 2008 Miodrag Vallat.
@@ -26,8 +26,29 @@
 
 #define	BRIDGE_WIDGET_CONTROL_IO_SWAP		0x00800000
 #define	BRIDGE_WIDGET_CONTROL_MEM_SWAP		0x00400000
+#define	BRIDGE_WIDGET_CONTROL_LARGE_PAGES	0x00200000
+
+/*
+ * DMA Direct Window
+ *
+ * The direct map register allows the 2GB direct window to map to
+ * a given widget address space. The upper bits of the XIO address,
+ * identifying the node to access, are provided in the low-order
+ * bits of the register.
+ */
 
 #define	BRIDGE_DIR_MAP			0x00000084
+
+#define	BRIDGE_DIRMAP_WIDGET_SHIFT	20
+#define	BRIDGE_DIRMAP_ADD_512MB		0x00020000	/* add 512MB */
+#define	BRIDGE_DIRMAP_BASE_MASK		0x00001fff
+#define	BRIDGE_DIRMAP_BASE_SHIFT	31
+
+#define	BRIDGE_PCI_MEM_SPACE_BASE	0x0000000040000000ULL
+#define	BRIDGE_PCI_MEM_SPACE_LENGTH	0x0000000040000000ULL
+#define	BRIDGE_PCI_IO_SPACE_BASE	0x0000000100000000ULL
+#define	BRIDGE_PCI_IO_SPACE_LENGTH	0x0000000100000000ULL
+
 #define	BRIDGE_NIC			0x000000b4
 #define	BRIDGE_BUS_TIMEOUT		0x000000c4
 #define	BRIDGE_PCI_CFG			0x000000cc
@@ -47,7 +68,7 @@
 #define	BRIDGE_INT_ADDR(d)		(0x00000134 + 8 * (d))
 
 /*
- * Mapping control
+ * PCI Resource Mapping control
  *
  * There are three ways to map a given device:
  * - memory mapping in the long window, at BRIDGE_PCI_MEM_SPACE_BASE,
@@ -63,15 +84,45 @@
  */
 
 #define	BRIDGE_DEVICE(d)		(0x00000204 + 8 * (d))
+#define	BRIDGE_DEVICE_SWAP_PMU			0x00080000	/* ??? */
+#define	BRIDGE_DEVICE_SWAP_DIR			0x00040000	/* ??? */
+#define	BRIDGE_DEVICE_PRECISE			0x00020000
+#define	BRIDGE_DEVICE_COHERENT			0x00010000
+#define	BRIDGE_DEVICE_BARRIER			0x00008000
 #define	BRIDGE_DEVICE_SWAP			0x00002000
 #define	BRIDGE_DEVICE_IO			0x00001000
 #define	BRIDGE_DEVICE_BASE_MASK			0x00000fff
 #define	BRIDGE_DEVICE_BASE_SHIFT		20
 
-#define	BRIDGE_PCI_MEM_SPACE_BASE	0x0000000040000000ULL
-#define	BRIDGE_PCI_MEM_SPACE_LENGTH	0x0000000040000000ULL
-#define	BRIDGE_PCI_IO_SPACE_BASE	0x0000000100000000ULL
-#define	BRIDGE_PCI_IO_SPACE_LENGTH	0x0000000100000000ULL
+#define	BRIDGE_DEVIO_BASE			0x00200000
+#define	BRIDGE_DEVIO_LARGE			0x00200000
+#define	BRIDGE_DEVIO_SHORT			0x00100000
+
+#define	BRIDGE_DEVIO_OFFS(d) \
+	(BRIDGE_DEVIO_BASE + \
+	 BRIDGE_DEVIO_LARGE * ((d) < 2 ? (d) : 2) + \
+	 BRIDGE_DEVIO_SHORT * ((d) < 2 ? 0 : (d) - 2))
+
+/*
+ * Read Response Buffer configuration registers
+ *
+ * There are 16 RRB, which are shared among the PCI devices.
+ * The following registers provide four bits per RRB, describing
+ * their RRB assignment.
+ *
+ * Since these four bits only assign two bits to map to a PCI slot,
+ * the low-order bit is implied by the RRB register: one controls the
+ * even-numbered PCI slots, while the other controls the odd-numbered
+ * PCI slots.
+ */
+
+#define	BRIDGE_RRB_EVEN			0x00000284
+#define	BRIDGE_RRB_ODD			0x0000028c
+
+#define	RRB_VALID			0x8
+#define	RRB_VCHAN			0x4
+#define	RRB_DEVICE_MASK			0x3
+#define	RRB_SHIFT			4
 
 /*
  * Configuration space