Renumber I/O APICs who clash with other APICs. Also clear the delivery

mode before setting it to fixed, since the MP BIOS may have left something
in that field. This fixed a Quad Xeon we have been loaned over the c2k4.

5 files changed, 98 insertions, 38 deletions

diff --git a/sys/arch/i386/i386/cpu.c b/sys/arch/i386/i386/cpu.c
index fb945ce6a92..4a44510fe90 100644
--- a/sys/arch/i386/i386/cpu.c
+++ b/sys/arch/i386/i386/cpu.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: cpu.c,v 1.5 2004/06/20 19:33:07 aaron Exp $	*/
+/*	$OpenBSD: cpu.c,v 1.6 2004/06/23 17:14:31 niklas Exp $	*/
 /* $NetBSD: cpu.c,v 1.1.2.7 2000/06/26 02:04:05 sommerfeld Exp $ */
 
 /*-
@@ -102,6 +102,7 @@
 #endif
 
 #if NIOAPIC > 0
+#include <machine/i82093reg.h>
 #include <machine/i82093var.h>
 #endif
 
@@ -315,6 +316,10 @@ cpu_attach(parent, self, aux)
 	printf("\n");
 #endif	/* !MULTIPROCESSOR */
 
+	/* Mark this ID as taken if it's in the I/O APIC ID area */
+	if (ci->ci_apicid < IOAPIC_ID_MAX)
+		ioapic_id_map &= ~(1 << ci->ci_apicid);
+	
 #ifdef MULTIPROCESSOR
 	if (mp_verbose) {
 		printf("%s: kstack at 0x%lx for %d bytes\n",
diff --git a/sys/arch/i386/i386/ioapic.c b/sys/arch/i386/i386/ioapic.c
index e5b329372f3..6bcf6d8d840 100644
--- a/sys/arch/i386/i386/ioapic.c
+++ b/sys/arch/i386/i386/ioapic.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: ioapic.c,v 1.2 2004/06/13 21:49:15 niklas Exp $	*/
+/*	$OpenBSD: ioapic.c,v 1.3 2004/06/23 17:14:31 niklas Exp $	*/
 /* 	$NetBSD: ioapic.c,v 1.7 2003/07/14 22:32:40 lukem Exp $	*/
 
 /*-
@@ -115,6 +115,22 @@ int ioapic_cold = 1;
 struct ioapic_softc *ioapics;	 /* head of linked list */
 int nioapics = 0;	   	 /* number attached */
 
+void ioapic_set_id(struct ioapic_softc *);
+
+/*
+ * A bitmap telling what APIC IDs usable for I/O APICs are free.
+ * The size must be at least IOAPIC_ID_MAX bits (16).
+ */
+u_int16_t ioapic_id_map = (1 << IOAPIC_ID_MAX) - 1;
+
+/*
+ * When we renumber I/O APICs we provide a mapping vector giving us the new
+ * ID out of the old BIOS supplied one.  Each item must be able to hold IDs
+ * in [0, IOAPIC_ID_MAX << 1), since we use an extra bit to tell if the ID
+ * has actually been remapped.  
+ */
+u_int8_t ioapic_id_remap[IOAPIC_ID_MAX];
+
 /*
  * Register read/write routines.
  */
@@ -201,6 +217,23 @@ ioapic_match(struct device *parent, void *matchv, void *aux)
 	return (0);
 }
 
+/* Reprogram the APIC ID, and check that it actually got set. */
+void
+ioapic_set_id(struct ioapic_softc *sc) {
+	u_int8_t apic_id;
+
+	ioapic_write(sc, IOAPIC_ID,
+	    (ioapic_read(sc, IOAPIC_ID) & ~IOAPIC_ID_MASK) |
+	    (sc->sc_apicid << IOAPIC_ID_SHIFT));
+
+	apic_id = (ioapic_read(sc, IOAPIC_ID) & IOAPIC_ID_MASK) >>
+	    IOAPIC_ID_SHIFT;
+
+	if (apic_id != sc->sc_apicid)
+		printf(", can't remap to apid %d\n", sc->sc_apicid);
+	else
+		printf(", remapped to apic %d\n", sc->sc_apicid);
+}
 
 /*
  * can't use bus_space_xxx as we don't have a bus handle ...
@@ -211,23 +244,15 @@ ioapic_attach(struct device *parent, struct device *self, void *aux)
 	struct ioapic_softc *sc = (struct ioapic_softc *)self;
 	struct apic_attach_args  *aaa = (struct apic_attach_args *)aux;
 	int apic_id;
+	int8_t new_id;
 	bus_space_handle_t bh;
 	u_int32_t ver_sz;
-	int i;
+	int i, ioapic_found;
 
 	sc->sc_flags = aaa->flags;
 	sc->sc_apicid = aaa->apic_id;
 
-	printf(" apid %d", aaa->apic_id);
-
-	if (ioapic_find(aaa->apic_id) != NULL) {
-		printf(": duplicate apic id (ignored)\n");
-		return;
-	}
-
-	ioapic_add(sc);
-
-	printf(": pa 0x%lx", aaa->apic_address);
+	printf(": apid %d pa 0x%lx", aaa->apic_id, aaa->apic_address);
 
 	if (bus_mem_add_mapping(aaa->apic_address, PAGE_SIZE, 0, &bh) != 0) {
 		printf(", map failed\n");
@@ -236,10 +261,7 @@ ioapic_attach(struct device *parent, struct device *self, void *aux)
 	sc->sc_reg = (volatile u_int32_t *)(bh + IOAPIC_REG);
 	sc->sc_data = (volatile u_int32_t *)(bh + IOAPIC_DATA);
 
-	apic_id = (ioapic_read(sc,IOAPIC_ID) & IOAPIC_ID_MASK) >>
-	    IOAPIC_ID_SHIFT;
 	ver_sz = ioapic_read(sc, IOAPIC_VER);
-
 	sc->sc_apic_vers = (ver_sz & IOAPIC_VER_MASK) >> IOAPIC_VER_SHIFT;
 	sc->sc_apic_sz = (ver_sz & IOAPIC_MAX_MASK) >> IOAPIC_MAX_SHIFT;
 	sc->sc_apic_sz++;
@@ -251,6 +273,38 @@ ioapic_attach(struct device *parent, struct device *self, void *aux)
 
 	printf(", version %x, %d pins\n", sc->sc_apic_vers, sc->sc_apic_sz);
 
+	/*
+	 * If either a LAPIC or an I/O APIC is already at the ID the BIOS
+	 * setup for this I/O APIC, try to find a free ID to use and reprogram
+	 * the chip.  Record this remapping since all references done by the
+	 * MP BIOS will be through the old ID.
+	 */
+	ioapic_found = ioapic_find(sc->sc_apicid) != NULL;
+	if (cpu_info[sc->sc_apicid] != NULL || ioapic_found) {
+		printf("%s: duplicate apic id", sc->sc_dev.dv_xname);
+		new_id = ffs(ioapic_id_map) - 1;
+		if (new_id == -1) {
+			printf(" (and none free, ignoring)\n");
+			return;
+		}
+
+		/*
+		 * If there were many I/O APICs at the same ID, we choose
+		 * to let later references to that ID (in the MP BIOS) refer
+		 * to the first found.
+		 */
+		if (!ioapic_found && !IOAPIC_REMAPPED(sc->sc_apicid))
+			IOAPIC_REMAP(sc->sc_apicid, new_id);
+		sc->sc_apicid = new_id;
+		ioapic_set_id(sc);
+	}
+	ioapic_id_map &= ~(1 << sc->sc_apicid);
+
+	ioapic_add(sc);
+
+	apic_id = (ioapic_read(sc, IOAPIC_ID) & IOAPIC_ID_MASK) >>
+	    IOAPIC_ID_SHIFT;
+
 	sc->sc_pins = malloc(sizeof(struct ioapic_pin) * sc->sc_apic_sz,
 	    M_DEVBUF, M_WAITOK);
 
@@ -267,24 +321,11 @@ ioapic_attach(struct device *parent, struct device *self, void *aux)
 	/*
 	 * In case the APIC is not initialized to the correct ID
 	 * do it now.
-	 * Maybe we should record the original ID for interrupt
-	 * mapping later ...
 	 */
 	if (apic_id != sc->sc_apicid) {
 		printf("%s: misconfigured as apic %d", sc->sc_dev.dv_xname,
 		    apic_id);
-
-		ioapic_write(sc, IOAPIC_ID,
-		    (ioapic_read(sc, IOAPIC_ID) & ~IOAPIC_ID_MASK)
-		    | (sc->sc_apicid << IOAPIC_ID_SHIFT));
-
-		apic_id = (ioapic_read(sc,IOAPIC_ID) & IOAPIC_ID_MASK) >>
-		    IOAPIC_ID_SHIFT;
-
-		if (apic_id != sc->sc_apicid)
-			printf(", can't remap to apid %d\n", sc->sc_apicid);
-		else
-			printf(", remapped to apic %d\n", sc->sc_apicid);
+		ioapic_set_id(sc);
 	}
 #if 0
 	/* output of this was boring. */
@@ -344,6 +385,7 @@ apic_set_redir(struct ioapic_softc *sc, int pin)
 		redlo |= IOAPIC_REDLO_MASK;
 	} else {
 		redlo |= (pp->ip_vector & 0xff);
+		redlo &= ~IOAPIC_REDLO_DEL_MASK;
 		redlo |= (IOAPIC_REDLO_DEL_FIXED << IOAPIC_REDLO_DEL_SHIFT);
 		redlo &= ~IOAPIC_REDLO_DSTMOD;
 
diff --git a/sys/arch/i386/i386/mpbios.c b/sys/arch/i386/i386/mpbios.c
index 689b9ace3f5..ff61d282f6c 100644
--- a/sys/arch/i386/i386/mpbios.c
+++ b/sys/arch/i386/i386/mpbios.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: mpbios.c,v 1.2 2004/06/13 21:49:15 niklas Exp $	*/
+/*	$OpenBSD: mpbios.c,v 1.3 2004/06/23 17:14:31 niklas Exp $	*/
 /*	$NetBSD: mpbios.c,v 1.2 2002/10/01 12:56:57 fvdl Exp $	*/
 
 /*-
@@ -1011,18 +1011,21 @@ mpbios_int(ent, enttype, mpi)
 	struct mp_intr_map *mpi;
 {
 	const struct mpbios_int *entry = (const struct mpbios_int *)ent;
+	struct mpbios_int rw_entry = *entry;
 	struct ioapic_softc *sc = NULL;
 
 	struct mp_intr_map *altmpi;
 	struct mp_bus *mpb;
 
-	u_int32_t id = entry->dst_apic_id;
+	u_int32_t id = IOAPIC_REMAPPED_ID(entry->dst_apic_id);
 	u_int32_t pin = entry->dst_apic_int;
 	u_int32_t bus = entry->src_bus_id;
 	u_int32_t dev = entry->src_bus_irq;
 	u_int32_t type = entry->int_type;
 	u_int32_t flags = entry->int_flags;
 
+	rw_entry.dst_apic_id = id;
+
 	switch (type) {
 	case MPS_INTTYPE_INT:
 		mpb = &(mp_busses[bus]);
@@ -1054,7 +1057,7 @@ mpbios_int(ent, enttype, mpi)
 		return;
 	}
 
-	(*mpb->mb_intr_cfg)(entry, &mpi->redir);
+	(*mpb->mb_intr_cfg)(&rw_entry, &mpi->redir);
 
 	if (enttype == MPS_MCT_IOINT) {
 		sc = ioapic_find(id);
diff --git a/sys/arch/i386/include/i82093reg.h b/sys/arch/i386/include/i82093reg.h
index 07ec03d8991..88f36e469f2 100644
--- a/sys/arch/i386/include/i82093reg.h
+++ b/sys/arch/i386/include/i82093reg.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: i82093reg.h,v 1.2 2004/06/13 21:49:16 niklas Exp $	*/
+/*	$OpenBSD: i82093reg.h,v 1.3 2004/06/23 17:14:31 niklas Exp $	*/
 /* $NetBSD: i82093reg.h,v 1.1.2.2 2000/02/21 18:54:07 sommerfeld Exp $ */
 
 /*-
@@ -53,9 +53,6 @@
  * store the register number of interest in IOAPIC_REG, and store/fetch
  * the real value in IOAPIC_DATA.
  */
-   
-
-
 #define	IOAPIC_REG		0x0000
 #define IOAPIC_DATA		0x0010
 
@@ -67,6 +64,8 @@
 
 #define 	IOAPIC_ID_SHIFT		24
 #define		IOAPIC_ID_MASK		0x0f000000
+#define		IOAPIC_ID_MAX		\
+    ((IOAPIC_ID_MASK >> IOAPIC_ID_SHIFT) + 1)
 
 /* Version, and maximum interrupt pin number. */
   
diff --git a/sys/arch/i386/include/i82093var.h b/sys/arch/i386/include/i82093var.h
index c3d7ce586f3..56a7553be43 100644
--- a/sys/arch/i386/include/i82093var.h
+++ b/sys/arch/i386/include/i82093var.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: i82093var.h,v 1.2 2004/06/13 21:49:16 niklas Exp $	*/
+/*	$OpenBSD: i82093var.h,v 1.3 2004/06/23 17:14:31 niklas Exp $	*/
 /* $NetBSD: i82093var.h,v 1.1 2003/02/26 21:26:10 fvdl Exp $ */
 
 /*-
@@ -85,6 +85,15 @@ struct ioapic_softc {
 #define APIC_IRQ_APIC(x) ((x & APIC_INT_APIC_MASK) >> APIC_INT_APIC_SHIFT)
 #define APIC_IRQ_PIN(x) ((x & APIC_INT_PIN_MASK) >> APIC_INT_PIN_SHIFT)
 
+/* I/O APIC ID remapping helper macros. */
+#define IOAPIC_REMAP_MASK	(IOAPIC_ID_MASK >> IOAPIC_ID_SHIFT)
+#define IOAPIC_REMAP_FLAG	((IOAPIC_REMAP_MASK + 1) << 1)
+#define IOAPIC_REMAP(old_id, new_id) \
+    (ioapic_id_remap[(old_id)] = IOAPIC_REMAP_FLAG | (new_id))
+#define IOAPIC_REMAPPED(id)	(ioapic_id_remap[(id)] & IOAPIC_REMAP_FLAG)
+#define IOAPIC_REMAPPED_ID(id)	\
+    (IOAPIC_REMAPPED(id) ? ioapic_id_remap[(id)] & IOAPIC_REMAP_MASK : (id))
+
 void   *apic_intr_establish(int, int, int, int (*)(void *), void *, char *); 
 void	apic_intr_disestablish(void *);
 
@@ -99,5 +108,7 @@ void	lapic_vectorset(void); /* XXX */
 extern int ioapic_bsp_id;
 extern int nioapics;
 extern struct ioapic_softc *ioapics;
+extern u_int16_t ioapic_id_map;
+extern u_int8_t ioapic_id_remap[];
 
 #endif /* !_I386_I82093VAR_H_ */