Implement Multiple Message MSI support on amd64. This is experimental code

to assist qwx(4) development.  We may remove this code again at some point
in the future.

Multiple Message MSI has some serious design flaws, especially when
combined with the APIC interrupt controller architecture.  It was
superseded by MSI-X.  Unfortunately qwx(4) does not implement MSI-X.

ok stsp@, deraadt@

8 files changed, 181 insertions, 38 deletions

diff --git a/sys/arch/amd64/amd64/i8259.c b/sys/arch/amd64/amd64/i8259.c
index 875d22053ef..d6fa0e3d7f3 100644
--- a/sys/arch/amd64/amd64/i8259.c
+++ b/sys/arch/amd64/amd64/i8259.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: i8259.c,v 1.11 2018/07/27 21:11:31 kettenis Exp $	*/
+/*	$OpenBSD: i8259.c,v 1.12 2024/01/19 18:38:16 kettenis Exp $	*/
 /*	$NetBSD: i8259.c,v 1.2 2003/03/02 18:27:15 fvdl Exp $	*/
 
 /*
@@ -100,6 +100,7 @@ struct pic i8259_pic = {
 	i8259_hwunmask,
 	i8259_setup,
 	i8259_setup,
+	NULL,
 	i8259_stubs,
 	i8259_stubs,
 };
diff --git a/sys/arch/amd64/amd64/intr.c b/sys/arch/amd64/amd64/intr.c
index a3331810a6a..64396b3899a 100644
--- a/sys/arch/amd64/amd64/intr.c
+++ b/sys/arch/amd64/amd64/intr.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: intr.c,v 1.55 2020/12/28 14:23:30 mpi Exp $	*/
+/*	$OpenBSD: intr.c,v 1.56 2024/01/19 18:38:16 kettenis Exp $	*/
 /*	$NetBSD: intr.c,v 1.3 2003/03/03 22:16:20 fvdl Exp $	*/
 
 /*
@@ -310,9 +310,16 @@ other:
 		}
 		return EBUSY;
 found:
-		idtvec = idt_vec_alloc(APIC_LEVEL(level), IDT_INTR_HIGH);
+		if (pic->pic_allocidtvec) {
+			idtvec = pic->pic_allocidtvec(pic, pin,
+			    APIC_LEVEL(level), IDT_INTR_HIGH);
+		} else {
+			idtvec = idt_vec_alloc(APIC_LEVEL(level),
+			    IDT_INTR_HIGH);
+		}
 		if (idtvec == 0) {
-			free(ci->ci_isources[slot], M_DEVBUF, sizeof (struct intrsource));
+			free(ci->ci_isources[slot], M_DEVBUF,
+			    sizeof (struct intrsource));
 			ci->ci_isources[slot] = NULL;
 			return EBUSY;
 		}
diff --git a/sys/arch/amd64/amd64/machdep.c b/sys/arch/amd64/amd64/machdep.c
index 606630a9f0a..ec472ca48bd 100644
--- a/sys/arch/amd64/amd64/machdep.c
+++ b/sys/arch/amd64/amd64/machdep.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: machdep.c,v 1.288 2023/09/08 20:47:22 kn Exp $	*/
+/*	$OpenBSD: machdep.c,v 1.289 2024/01/19 18:38:16 kettenis Exp $	*/
 /*	$NetBSD: machdep.c,v 1.3 2003/05/07 22:58:18 fvdl Exp $	*/
 
 /*-
@@ -1918,6 +1918,29 @@ idt_vec_alloc(int low, int high)
 	return 0;
 }
 
+int
+idt_vec_alloc_range(int low, int high, int num)
+{
+	int i, vec;
+
+	KASSERT(powerof2(num));
+	low = (low + num - 1) & ~(num - 1);
+	high = ((high + 1) & ~(num - 1)) - 1;
+
+	for (vec = low; vec <= high; vec += num) {
+		for (i = 0; i < num; i++) {
+			if (idt_allocmap[vec + i] != 0)
+				break;
+		}
+		if (i == num) {
+			for (i = 0; i < num; i++)
+				idt_allocmap[vec + i] = 1;
+			return vec;
+		}
+	}
+	return 0;
+}
+
 void
 idt_vec_set(int vec, void (*function)(void))
 {
diff --git a/sys/arch/amd64/include/pci_machdep.h b/sys/arch/amd64/include/pci_machdep.h
index 6e5928ec8fd..9abd854f985 100644
--- a/sys/arch/amd64/include/pci_machdep.h
+++ b/sys/arch/amd64/include/pci_machdep.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: pci_machdep.h,v 1.30 2020/10/27 02:39:07 jordan Exp $	*/
+/*	$OpenBSD: pci_machdep.h,v 1.31 2024/01/19 18:38:16 kettenis Exp $	*/
 /*	$NetBSD: pci_machdep.h,v 1.1 2003/02/26 21:26:11 fvdl Exp $	*/
 
 /*
@@ -79,8 +79,11 @@ int		pci_conf_size(pci_chipset_tag_t, pcitag_t);
 pcireg_t	pci_conf_read(pci_chipset_tag_t, pcitag_t, int);
 void		pci_conf_write(pci_chipset_tag_t, pcitag_t, int,
 		    pcireg_t);
+int		pci_intr_enable_msivec(struct pci_attach_args *, int);
 int		pci_intr_map_msi(struct pci_attach_args *,
 		    pci_intr_handle_t *);
+int		pci_intr_map_msivec(struct pci_attach_args *,
+		    int, pci_intr_handle_t *);
 int		pci_intr_map_msix(struct pci_attach_args *,
 		    int, pci_intr_handle_t *);
 int		pci_intr_map(struct pci_attach_args *, pci_intr_handle_t *);
diff --git a/sys/arch/amd64/include/pic.h b/sys/arch/amd64/include/pic.h
index 59846fbeae6..c5b29937dec 100644
--- a/sys/arch/amd64/include/pic.h
+++ b/sys/arch/amd64/include/pic.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: pic.h,v 1.7 2014/12/16 21:20:23 tedu Exp $	*/
+/*	$OpenBSD: pic.h,v 1.8 2024/01/19 18:38:16 kettenis Exp $	*/
 /*	$NetBSD: pic.h,v 1.1 2003/02/26 21:26:11 fvdl Exp $	*/
 
 #ifndef _X86_PIC_H
@@ -22,6 +22,7 @@ struct pic {
         void (*pic_hwunmask)(struct pic *, int);
 	void (*pic_addroute)(struct pic *, struct cpu_info *, int, int, int);
 	void (*pic_delroute)(struct pic *, struct cpu_info *, int, int, int);
+	int (*pic_allocidtvec)(struct pic *, int, int, int);
 	struct intrstub *pic_level_stubs;
 	struct intrstub *pic_edge_stubs;
 };
diff --git a/sys/arch/amd64/include/segments.h b/sys/arch/amd64/include/segments.h
index 6ce184e88d1..87de2c08240 100644
--- a/sys/arch/amd64/include/segments.h
+++ b/sys/arch/amd64/include/segments.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: segments.h,v 1.15 2018/03/29 01:21:02 guenther Exp $	*/
+/*	$OpenBSD: segments.h,v 1.16 2024/01/19 18:38:16 kettenis Exp $	*/
 /*	$NetBSD: segments.h,v 1.1 2003/04/26 18:39:47 fvdl Exp $	*/
 
 /*-
@@ -160,6 +160,7 @@ void set_sys_segment(struct sys_segment_descriptor *, void *, size_t,
 void set_mem_segment(struct mem_segment_descriptor *, void *, size_t,
 			  int, int, int, int, int);
 int idt_vec_alloc(int, int);
+int idt_vec_alloc_range(int, int, int);
 void idt_vec_set(int, void (*)(void));
 void idt_vec_free(int);
 void cpu_init_idt(void);
diff --git a/sys/arch/amd64/pci/pci_machdep.c b/sys/arch/amd64/pci/pci_machdep.c
index 72456c32829..41c65293c4d 100644
--- a/sys/arch/amd64/pci/pci_machdep.c
+++ b/sys/arch/amd64/pci/pci_machdep.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: pci_machdep.c,v 1.77 2021/03/11 11:16:55 jsg Exp $	*/
+/*	$OpenBSD: pci_machdep.c,v 1.78 2024/01/19 18:38:16 kettenis Exp $	*/
 /*	$NetBSD: pci_machdep.c,v 1.3 2003/05/07 21:33:58 fvdl Exp $	*/
 
 /*-
@@ -315,10 +315,21 @@ pci_msix_table_unmap(pci_chipset_tag_t pc, pcitag_t tag,
 	_bus_space_unmap(memt, memh, tblsz * 16, NULL);
 }
 
+/*
+ * We pack the MSI vector number into the lower 8 bits of the PCI tag
+ * and use that as the MSI/MSI-X "PIC" pin number.  This allows us to
+ * address 256 MSI vectors which ought to be enough for anybody.
+ */
+#define PCI_MSI_VEC_MASK	0xff
+#define PCI_MSI_VEC(pin)	((pin) & PCI_MSI_VEC_MASK)
+#define PCI_MSI_TAG(pin)	((pin) & ~PCI_MSI_VEC_MASK)
+#define PCI_MSI_PIN(tag, vec)	((tag) | (vec))
+
 void msi_hwmask(struct pic *, int);
 void msi_hwunmask(struct pic *, int);
 void msi_addroute(struct pic *, struct cpu_info *, int, int, int);
 void msi_delroute(struct pic *, struct cpu_info *, int, int, int);
+int msi_allocidtvec(struct pic *, int, int, int);
 
 struct pic msi_pic = {
 	{0, {NULL}, NULL, 0, "msi", NULL, 0, 0},
@@ -330,6 +341,7 @@ struct pic msi_pic = {
 	msi_hwunmask,
 	msi_addroute,
 	msi_delroute,
+	msi_allocidtvec,
 	NULL,
 	ioapic_edge_stubs
 };
@@ -345,57 +357,157 @@ msi_hwunmask(struct pic *pic, int pin)
 }
 
 void
-msi_addroute(struct pic *pic, struct cpu_info *ci, int pin, int vec, int type)
+msi_addroute(struct pic *pic, struct cpu_info *ci, int pin, int idtvec,
+    int type)
 {
 	pci_chipset_tag_t pc = NULL; /* XXX */
-	pcitag_t tag = pin;
+	pcitag_t tag = PCI_MSI_TAG(pin);
+	int vec = PCI_MSI_VEC(pin);
 	pcireg_t reg, addr;
 	int off;
 
 	if (pci_get_capability(pc, tag, PCI_CAP_MSI, &off, &reg) == 0)
 		panic("%s: no msi capability", __func__);
 
+	if (vec != 0)
+		return;
+
 	addr = 0xfee00000UL | (ci->ci_apicid << 12);
 
 	if (reg & PCI_MSI_MC_C64) {
 		pci_conf_write(pc, tag, off + PCI_MSI_MA, addr);
 		pci_conf_write(pc, tag, off + PCI_MSI_MAU32, 0);
-		pci_conf_write(pc, tag, off + PCI_MSI_MD64, vec);
+		pci_conf_write(pc, tag, off + PCI_MSI_MD64, idtvec);
 	} else {
 		pci_conf_write(pc, tag, off + PCI_MSI_MA, addr);
-		pci_conf_write(pc, tag, off + PCI_MSI_MD32, vec);
+		pci_conf_write(pc, tag, off + PCI_MSI_MD32, idtvec);
 	}
 	pci_conf_write(pc, tag, off, reg | PCI_MSI_MC_MSIE);
 }
 
 void
-msi_delroute(struct pic *pic, struct cpu_info *ci, int pin, int vec, int type)
+msi_delroute(struct pic *pic, struct cpu_info *ci, int pin, int idtvec,
+    int type)
 {
 	pci_chipset_tag_t pc = NULL; /* XXX */
-	pcitag_t tag = pin;
+	pcitag_t tag = PCI_MSI_TAG(pin);
+	int vec = PCI_MSI_VEC(pin);
 	pcireg_t reg;
 	int off;
 
+	if (vec != 0)
+		return;
+
 	if (pci_get_capability(pc, tag, PCI_CAP_MSI, &off, &reg))
 		pci_conf_write(pc, tag, off, reg & ~PCI_MSI_MC_MSIE);
 }
 
 int
+msi_allocidtvec(struct pic *pic, int pin, int low, int high)
+{
+	pci_chipset_tag_t pc = NULL; /* XXX */
+	pcitag_t tag = PCI_MSI_TAG(pin);
+	int vec = PCI_MSI_VEC(pin);
+	int idtvec, mme, off;
+	pcireg_t reg;
+
+	if (pci_get_capability(pc, tag, PCI_CAP_MSI, &off, &reg) == 0)
+		panic("%s: no msi capability", __func__);
+
+	reg = pci_conf_read(pc, tag, off);
+	mme = ((reg & PCI_MSI_MC_MME_MASK) >> PCI_MSI_MC_MME_SHIFT);
+	if (vec >= (1 << mme))
+		return 0;
+
+	if (vec == 0) {
+		idtvec = idt_vec_alloc_range(low, high, (1 << mme));
+		if (reg & PCI_MSI_MC_C64)
+			pci_conf_write(pc, tag, off + PCI_MSI_MD64, idtvec);
+		else
+			pci_conf_write(pc, tag, off + PCI_MSI_MD32, idtvec);
+	} else {
+		if (reg & PCI_MSI_MC_C64)
+			reg = pci_conf_read(pc, tag, off + PCI_MSI_MD64);
+		else
+			reg = pci_conf_read(pc, tag, off + PCI_MSI_MD32);
+		KASSERT(reg > 0);
+		idtvec = reg + vec;
+	}
+
+	return idtvec;
+}
+
+int
+pci_intr_enable_msivec(struct pci_attach_args *pa, int num_vec)
+{
+	pci_chipset_tag_t pc = pa->pa_pc;
+	pcitag_t tag = pa->pa_tag;
+	pcireg_t reg;
+	int mmc, mme, off;
+
+	if ((pa->pa_flags & PCI_FLAGS_MSI_ENABLED) == 0 || mp_busses == NULL ||
+	    pci_get_capability(pc, tag, PCI_CAP_MSI, &off, &reg) == 0)
+		return 1;
+
+	mmc = ((reg & PCI_MSI_MC_MMC_MASK) >> PCI_MSI_MC_MMC_SHIFT);
+	if (num_vec > (1 << mmc))
+		return 1;
+
+	mme = ((reg & PCI_MSI_MC_MME_MASK) >> PCI_MSI_MC_MME_SHIFT);
+	while ((1 << mme) < num_vec)
+		mme++;
+	reg &= ~PCI_MSI_MC_MME_MASK;
+	reg |= (mme << PCI_MSI_MC_MME_SHIFT);
+	pci_conf_write(pc, tag, off, reg);
+
+	return 0;
+}
+
+int
 pci_intr_map_msi(struct pci_attach_args *pa, pci_intr_handle_t *ihp)
 {
 	pci_chipset_tag_t pc = pa->pa_pc;
 	pcitag_t tag = pa->pa_tag;
+	pcireg_t reg;
+	int off;
 
 	if ((pa->pa_flags & PCI_FLAGS_MSI_ENABLED) == 0 || mp_busses == NULL ||
-	    pci_get_capability(pc, tag, PCI_CAP_MSI, NULL, NULL) == 0)
+	    pci_get_capability(pc, tag, PCI_CAP_MSI, &off, &reg) == 0)
 		return 1;
 
+	/* Make sure we only enable one MSI vector. */
+	reg &= ~PCI_MSI_MC_MME_MASK;
+	pci_conf_write(pc, tag, off, reg);
+
 	ihp->tag = tag;
 	ihp->line = APIC_INT_VIA_MSG;
 	ihp->pin = 0;
 	return 0;
 }
 
+int
+pci_intr_map_msivec(struct pci_attach_args *pa, int vec,
+    pci_intr_handle_t *ihp)
+{
+	pci_chipset_tag_t pc = pa->pa_pc;
+	pcitag_t tag = pa->pa_tag;
+	pcireg_t reg;
+	int mme, off;
+
+	if ((pa->pa_flags & PCI_FLAGS_MSI_ENABLED) == 0 || mp_busses == NULL ||
+	    pci_get_capability(pc, tag, PCI_CAP_MSI, &off, &reg) == 0)
+		return 1;
+
+	mme = ((reg & PCI_MSI_MC_MME_MASK) >> PCI_MSI_MC_MME_SHIFT);
+	if (vec > (1 << mme))
+		return 0;
+
+	ihp->tag = PCI_MSI_PIN(tag, vec);
+	ihp->line = APIC_INT_VIA_MSG;
+	ihp->pin = 0;
+	return 0;
+}
+
 void msix_hwmask(struct pic *, int);
 void msix_hwunmask(struct pic *, int);
 void msix_addroute(struct pic *, struct cpu_info *, int, int, int);
@@ -412,19 +524,10 @@ struct pic msix_pic = {
 	msix_addroute,
 	msix_delroute,
 	NULL,
+	NULL,
 	ioapic_edge_stubs
 };
 
-/*
- * We pack the MSI-X vector number into the lower 8 bits of the PCI
- * tag and use that as the MSI-X "PIC" pin number.  This allows us to
- * address 256 MSI-X vectors which ought to be enough for anybody.
- */
-#define PCI_MSIX_VEC_MASK	0xff
-#define PCI_MSIX_VEC(pin)	((pin) & PCI_MSIX_VEC_MASK)
-#define PCI_MSIX_TAG(pin)	((pin) & ~PCI_MSIX_VEC_MASK)
-#define PCI_MSIX_PIN(tag, vec)	((tag) | (vec))
-
 void
 msix_hwmask(struct pic *pic, int pin)
 {
@@ -436,13 +539,14 @@ msix_hwunmask(struct pic *pic, int pin)
 }
 
 void
-msix_addroute(struct pic *pic, struct cpu_info *ci, int pin, int vec, int type)
+msix_addroute(struct pic *pic, struct cpu_info *ci, int pin, int idtvec,
+    int type)
 {
 	pci_chipset_tag_t pc = NULL; /* XXX */
 	bus_space_tag_t memt = X86_BUS_SPACE_MEM; /* XXX */
 	bus_space_handle_t memh;
-	pcitag_t tag = PCI_MSIX_TAG(pin);
-	int entry = PCI_MSIX_VEC(pin);
+	pcitag_t tag = PCI_MSI_TAG(pin);
+	int entry = PCI_MSI_VEC(pin);
 	pcireg_t reg, addr;
 	uint32_t ctrl;
 	int off;
@@ -459,7 +563,7 @@ msix_addroute(struct pic *pic, struct cpu_info *ci, int pin, int vec, int type)
 
 	bus_space_write_4(memt, memh, PCI_MSIX_MA(entry), addr);
 	bus_space_write_4(memt, memh, PCI_MSIX_MAU32(entry), 0);
-	bus_space_write_4(memt, memh, PCI_MSIX_MD(entry), vec);
+	bus_space_write_4(memt, memh, PCI_MSIX_MD(entry), idtvec);
 	bus_space_barrier(memt, memh, PCI_MSIX_MA(entry), 16,
 	    BUS_SPACE_BARRIER_WRITE);
 	ctrl = bus_space_read_4(memt, memh, PCI_MSIX_VC(entry));
@@ -472,13 +576,14 @@ msix_addroute(struct pic *pic, struct cpu_info *ci, int pin, int vec, int type)
 }
 
 void
-msix_delroute(struct pic *pic, struct cpu_info *ci, int pin, int vec, int type)
+msix_delroute(struct pic *pic, struct cpu_info *ci, int pin, int idtvec,
+    int type)
 {
 	pci_chipset_tag_t pc = NULL; /* XXX */
 	bus_space_tag_t memt = X86_BUS_SPACE_MEM; /* XXX */
 	bus_space_handle_t memh;
-	pcitag_t tag = PCI_MSIX_TAG(pin);
-	int entry = PCI_MSIX_VEC(pin);
+	pcitag_t tag = PCI_MSI_TAG(pin);
+	int entry = PCI_MSI_VEC(pin);
 	pcireg_t reg;
 	uint32_t ctrl;
 
@@ -504,7 +609,7 @@ pci_intr_map_msix(struct pci_attach_args *pa, int vec, pci_intr_handle_t *ihp)
 	pcitag_t tag = pa->pa_tag;
 	pcireg_t reg;
 
-	KASSERT(PCI_MSIX_VEC(vec) == vec);
+	KASSERT(PCI_MSI_VEC(vec) == vec);
 
 	if ((pa->pa_flags & PCI_FLAGS_MSI_ENABLED) == 0 || mp_busses == NULL ||
 	    pci_get_capability(pc, tag, PCI_CAP_MSIX, NULL, &reg) == 0)
@@ -513,7 +618,7 @@ pci_intr_map_msix(struct pci_attach_args *pa, int vec, pci_intr_handle_t *ihp)
 	if (vec > PCI_MSIX_MC_TBLSZ(reg))
 		return 1;
 
-	ihp->tag = PCI_MSIX_PIN(tag, vec);
+	ihp->tag = PCI_MSI_PIN(tag, vec);
 	ihp->line = APIC_INT_VIA_MSGX;
 	ihp->pin = 0;
 	return 0;
diff --git a/sys/dev/pci/pcireg.h b/sys/dev/pci/pcireg.h
index 53124eccecb..b839bee9112 100644
--- a/sys/dev/pci/pcireg.h
+++ b/sys/dev/pci/pcireg.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: pcireg.h,v 1.61 2022/06/17 10:08:36 kettenis Exp $	*/
+/*	$OpenBSD: pcireg.h,v 1.62 2024/01/19 18:38:16 kettenis Exp $	*/
 /*	$NetBSD: pcireg.h,v 1.26 2000/05/10 16:58:42 thorpej Exp $	*/
 
 /*
@@ -516,8 +516,10 @@ typedef u_int8_t pci_revision_t;
  */
 #define PCI_MSI_MC		0x00
 #define PCI_MSI_MC_C64		0x00800000
-#define PCI_MSI_MC_MME		0x00700000
-#define PCI_MSI_MC_MMC		0x000e0000
+#define PCI_MSI_MC_MME_MASK	0x00700000
+#define PCI_MSI_MC_MME_SHIFT	20
+#define PCI_MSI_MC_MMC_MASK	0x000e0000
+#define PCI_MSI_MC_MMC_SHIFT	17
 #define PCI_MSI_MC_MSIE		0x00010000
 #define PCI_MSI_MA		0x04
 #define PCI_MSI_MAU32		0x08