Implement suspend/resume support for MSI-X interrupts. Loosely based on

an earlier diff from sf@.

ok jmatthew@, also ok mlarkin@, sf@ for a slightly different earlier version

6 files changed, 264 insertions, 41 deletions

diff --git a/sys/arch/amd64/include/pci_machdep.h b/sys/arch/amd64/include/pci_machdep.h
index 8a456f72b09..2d0cdf16a6e 100644
--- a/sys/arch/amd64/include/pci_machdep.h
+++ b/sys/arch/amd64/include/pci_machdep.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: pci_machdep.h,v 1.27 2018/08/19 08:23:47 kettenis Exp $	*/
+/*	$OpenBSD: pci_machdep.h,v 1.28 2019/06/25 16:46:32 kettenis Exp $	*/
 /*	$NetBSD: pci_machdep.h,v 1.1 2003/02/26 21:26:11 fvdl Exp $	*/
 
 /*
@@ -98,6 +98,13 @@ void		pci_set_powerstate_md(pci_chipset_tag_t, pcitag_t, int, int);
 void		pci_mcfg_init(bus_space_tag_t, bus_addr_t, int, int, int);
 pci_chipset_tag_t pci_lookup_segment(int);
 
+#define __HAVE_PCI_MSIX
+
+int	pci_msix_table_map(pci_chipset_tag_t, pcitag_t,
+	    bus_space_tag_t, bus_space_handle_t *);
+void	pci_msix_table_unmap(pci_chipset_tag_t, pcitag_t,
+	    bus_space_tag_t, bus_space_handle_t);
+
 /*
  * ALL OF THE FOLLOWING ARE MACHINE-DEPENDENT, AND SHOULD NOT BE USED
  * BY PORTABLE CODE.
diff --git a/sys/arch/amd64/pci/pci_machdep.c b/sys/arch/amd64/pci/pci_machdep.c
index 976ef2d3b6b..27e78c7cfa4 100644
--- a/sys/arch/amd64/pci/pci_machdep.c
+++ b/sys/arch/amd64/pci/pci_machdep.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: pci_machdep.c,v 1.70 2019/05/30 22:03:14 kettenis Exp $	*/
+/*	$OpenBSD: pci_machdep.c,v 1.71 2019/06/25 16:46:32 kettenis Exp $	*/
 /*	$NetBSD: pci_machdep.c,v 1.3 2003/05/07 21:33:58 fvdl Exp $	*/
 
 /*-
@@ -326,6 +326,46 @@ pci_conf_write(pci_chipset_tag_t pc, pcitag_t tag, int reg, pcireg_t data)
 	PCI_CONF_UNLOCK();
 }
 
+int
+pci_msix_table_map(pci_chipset_tag_t pc, pcitag_t tag,
+    bus_space_tag_t memt, bus_space_handle_t *memh)
+{
+	bus_addr_t base;
+	pcireg_t reg, table, type;
+	int bir, offset;
+	int off, tblsz;
+
+	if (pci_get_capability(pc, tag, PCI_CAP_MSIX, &off, &reg) == 0)
+		panic("%s: no msix capability", __func__);
+
+	table = pci_conf_read(pc, tag, off + PCI_MSIX_TABLE);
+	bir = (table & PCI_MSIX_TABLE_BIR);
+	offset = (table & PCI_MSIX_TABLE_OFF);
+	tblsz = PCI_MSIX_MC_TBLSZ(reg) + 1;
+
+	bir = PCI_MAPREG_START + bir * 4;
+	type = pci_mapreg_type(pc, tag, bir);
+	if (pci_mapreg_info(pc, tag, bir, type, &base, NULL, NULL) ||
+	    _bus_space_map(memt, base + offset, tblsz * 16, 0, memh))
+		return -1;
+
+	return 0;
+}
+
+void
+pci_msix_table_unmap(pci_chipset_tag_t pc, pcitag_t tag,
+    bus_space_tag_t memt, bus_space_handle_t memh)
+{
+	pcireg_t reg;
+	int tblsz;
+
+	if (pci_get_capability(pc, tag, PCI_CAP_MSIX, NULL, &reg) == 0)
+		panic("%s: no msix capability", __func__);
+
+	tblsz = PCI_MSIX_MC_TBLSZ(reg) + 1;
+	_bus_space_unmap(memt, memh, tblsz * 16, NULL);
+}
+
 void msi_hwmask(struct pic *, int);
 void msi_hwunmask(struct pic *, int);
 void msi_addroute(struct pic *, struct cpu_info *, int, int, int);
@@ -452,29 +492,22 @@ msix_addroute(struct pic *pic, struct cpu_info *ci, int pin, int vec, int type)
 	pci_chipset_tag_t pc = NULL; /* XXX */
 	bus_space_tag_t memt = X86_BUS_SPACE_MEM; /* XXX */
 	bus_space_handle_t memh;
-	bus_addr_t base;
 	pcitag_t tag = PCI_MSIX_TAG(pin);
 	int entry = PCI_MSIX_VEC(pin);
-	pcireg_t reg, addr, table;
+	pcireg_t reg, addr;
 	uint32_t ctrl;
-	int bir, offset;
-	int off, tblsz;
+	int off;
 
 	if (pci_get_capability(pc, tag, PCI_CAP_MSIX, &off, &reg) == 0)
 		panic("%s: no msix capability", __func__);
 
-	addr = 0xfee00000UL | (ci->ci_apicid << 12);
-
-	table = pci_conf_read(pc, tag, off + PCI_MSIX_TABLE);
-	bir = (table & PCI_MSIX_TABLE_BIR);
-	offset = (table & PCI_MSIX_TABLE_OFF);
-	tblsz = PCI_MSIX_MC_TBLSZ(reg) + 1;
+	KASSERT(entry <= PCI_MSIX_MC_TBLSZ(reg));
 
-	bir = PCI_MAPREG_START + bir * 4;
-	if (pci_mem_find(pc, tag, bir, &base, NULL, NULL) ||
-	    _bus_space_map(memt, base + offset, tblsz * 16, 0, &memh))
+	if (pci_msix_table_map(pc, tag, memt, &memh))
 		panic("%s: cannot map registers", __func__);
 
+	addr = 0xfee00000UL | (ci->ci_apicid << 12);
+
 	bus_space_write_4(memt, memh, PCI_MSIX_MA(entry), addr);
 	bus_space_write_4(memt, memh, PCI_MSIX_MAU32(entry), 0);
 	bus_space_write_4(memt, memh, PCI_MSIX_MD(entry), vec);
@@ -484,7 +517,7 @@ msix_addroute(struct pic *pic, struct cpu_info *ci, int pin, int vec, int type)
 	bus_space_write_4(memt, memh, PCI_MSIX_VC(entry),
 	    ctrl & ~PCI_MSIX_VC_MASK);
 
-	_bus_space_unmap(memt, memh, tblsz * 16, NULL);
+	pci_msix_table_unmap(pc, tag, memt, memh);
 
 	pci_conf_write(pc, tag, off, reg | PCI_MSIX_MC_MSIXE);
 }
@@ -495,32 +528,24 @@ msix_delroute(struct pic *pic, struct cpu_info *ci, int pin, int vec, int type)
 	pci_chipset_tag_t pc = NULL; /* XXX */
 	bus_space_tag_t memt = X86_BUS_SPACE_MEM; /* XXX */
 	bus_space_handle_t memh;
-	bus_addr_t base;
 	pcitag_t tag = PCI_MSIX_TAG(pin);
 	int entry = PCI_MSIX_VEC(pin);
-	pcireg_t reg, table;
+	pcireg_t reg;
 	uint32_t ctrl;
-	int bir, offset;
-	int off, tblsz;
 
-	if (pci_get_capability(pc, tag, PCI_CAP_MSIX, &off, &reg) == 0)
+	if (pci_get_capability(pc, tag, PCI_CAP_MSIX, NULL, &reg) == 0)
 		return;
 
-	table = pci_conf_read(pc, tag, off + PCI_MSIX_TABLE);
-	bir = (table & PCI_MSIX_TABLE_BIR);
-	offset = (table & PCI_MSIX_TABLE_OFF);
-	tblsz = PCI_MSIX_MC_TBLSZ(reg) + 1;
+	KASSERT(entry <= PCI_MSIX_MC_TBLSZ(reg));
 
-	bir = PCI_MAPREG_START + bir * 4;
-	if (pci_mem_find(pc, tag, bir, &base, NULL, NULL) ||
-	    _bus_space_map(memt, base + offset, tblsz * 16, 0, &memh))
-		panic("%s: cannot map registers", __func__);
+	if (pci_msix_table_map(pc, tag, memt, &memh))
+		return;
 
 	ctrl = bus_space_read_4(memt, memh, PCI_MSIX_VC(entry));
 	bus_space_write_4(memt, memh, PCI_MSIX_VC(entry),
 	    ctrl | PCI_MSIX_VC_MASK);
 
-	_bus_space_unmap(memt, memh, tblsz * 16, NULL);
+	pci_msix_table_unmap(pc, tag, memt, memh);
 }
 
 int
diff --git a/sys/arch/arm64/dev/pci_machdep.c b/sys/arch/arm64/dev/pci_machdep.c
index 29ca6663075..8df655247b4 100644
--- a/sys/arch/arm64/dev/pci_machdep.c
+++ b/sys/arch/arm64/dev/pci_machdep.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: pci_machdep.c,v 1.3 2019/06/17 11:07:39 kettenis Exp $	*/
+/*	$OpenBSD: pci_machdep.c,v 1.4 2019/06/25 16:46:32 kettenis Exp $	*/
 
 /*
  * Copyright (c) 2019 Mark Kettenis <kettenis@openbsd.org>
@@ -45,14 +45,12 @@ pci_msi_enable(pci_chipset_tag_t pc, pcitag_t tag,
 	pci_conf_write(pc, tag, off, reg | PCI_MSI_MC_MSIE);
 }
 
-void
-pci_msix_enable(pci_chipset_tag_t pc, pcitag_t tag, bus_space_tag_t memt,
-    int vec, bus_addr_t addr, uint32_t data)
+int
+pci_msix_table_map(pci_chipset_tag_t pc, pcitag_t tag,
+    bus_space_tag_t memt, bus_space_handle_t *memh)
 {
-	bus_space_handle_t memh;
 	bus_addr_t base;
 	pcireg_t reg, table, type;
-	uint32_t ctrl;
 	int bir, offset;
 	int off, tblsz;
 
@@ -67,7 +65,41 @@ pci_msix_enable(pci_chipset_tag_t pc, pcitag_t tag, bus_space_tag_t memt,
 	bir = PCI_MAPREG_START + bir * 4;
 	type = pci_mapreg_type(pc, tag, bir);
 	if (pci_mapreg_info(pc, tag, bir, type, &base, NULL, NULL) ||
-	    bus_space_map(memt, base + offset, tblsz * 16, 0, &memh))
+	    bus_space_map(memt, base + offset, tblsz * 16, 0, memh))
+		return -1;
+
+	return 0;
+}
+
+void
+pci_msix_table_unmap(pci_chipset_tag_t pc, pcitag_t tag,
+    bus_space_tag_t memt, bus_space_handle_t memh)
+{
+	pcireg_t reg;
+	int tblsz;
+
+	if (pci_get_capability(pc, tag, PCI_CAP_MSIX, NULL, &reg) == 0)
+		panic("%s: no msix capability", __func__);
+
+	tblsz = PCI_MSIX_MC_TBLSZ(reg) + 1;
+	bus_space_unmap(memt, memh, tblsz * 16);
+}
+
+void
+pci_msix_enable(pci_chipset_tag_t pc, pcitag_t tag, bus_space_tag_t memt,
+    int vec, bus_addr_t addr, uint32_t data)
+{
+	bus_space_handle_t memh;
+	pcireg_t reg;
+	uint32_t ctrl;
+	int off;
+
+	if (pci_get_capability(pc, tag, PCI_CAP_MSIX, &off, &reg) == 0)
+		panic("%s: no msix capability", __func__);
+
+	KASSERT(vec <= PCI_MSIX_MC_TBLSZ(reg));
+
+	if (pci_msix_table_map(pc, tag, memt, &memh))
 		panic("%s: cannot map registers", __func__);
 
 	bus_space_write_4(memt, memh, PCI_MSIX_MA(vec), addr);
@@ -79,7 +111,7 @@ pci_msix_enable(pci_chipset_tag_t pc, pcitag_t tag, bus_space_tag_t memt,
 	bus_space_write_4(memt, memh, PCI_MSIX_VC(vec),
 	    ctrl & ~PCI_MSIX_VC_MASK);
 
-	bus_space_unmap(memt, memh, tblsz * 16);
+	pci_msix_table_unmap(pc, tag, memt, memh);
 
 	pci_conf_write(pc, tag, off, reg | PCI_MSIX_MC_MSIXE);
 }
diff --git a/sys/arch/arm64/include/pci_machdep.h b/sys/arch/arm64/include/pci_machdep.h
index 5507cb98446..24234175250 100644
--- a/sys/arch/arm64/include/pci_machdep.h
+++ b/sys/arch/arm64/include/pci_machdep.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: pci_machdep.h,v 1.5 2019/06/02 18:40:58 kettenis Exp $ */
+/*	$OpenBSD: pci_machdep.h,v 1.6 2019/06/25 16:46:32 kettenis Exp $ */
 
 /*
  * Copyright (c) 2003-2004 Opsycon AB  (www.opsycon.se / www.opsycon.com)
@@ -117,3 +117,10 @@ void	pci_msix_enable(pci_chipset_tag_t, pcitag_t, bus_space_tag_t,
 	    int, bus_addr_t, uint32_t);
 int	_pci_intr_map_msi(struct pci_attach_args *, pci_intr_handle_t *);
 int	_pci_intr_map_msix(struct pci_attach_args *, int, pci_intr_handle_t *);
+
+#define __HAVE_PCI_MSIX
+
+int	pci_msix_table_map(pci_chipset_tag_t, pcitag_t,
+	    bus_space_tag_t, bus_space_handle_t *);
+void	pci_msix_table_unmap(pci_chipset_tag_t, pcitag_t,
+	    bus_space_tag_t, bus_space_handle_t);
diff --git a/sys/dev/pci/pci.c b/sys/dev/pci/pci.c
index ecb59cd7962..1ca2b943df9 100644
--- a/sys/dev/pci/pci.c
+++ b/sys/dev/pci/pci.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: pci.c,v 1.113 2019/06/08 10:27:02 dlg Exp $	*/
+/*	$OpenBSD: pci.c,v 1.114 2019/06/25 16:46:32 kettenis Exp $	*/
 /*	$NetBSD: pci.c,v 1.31 1997/06/06 23:48:04 thorpej Exp $	*/
 
 /*
@@ -53,6 +53,13 @@ void pci_suspend(struct pci_softc *);
 void pci_powerdown(struct pci_softc *);
 void pci_resume(struct pci_softc *);
 
+struct msix_vector {
+	uint32_t mv_ma;
+	uint32_t mv_mau32;
+	uint32_t mv_md;
+	uint32_t mv_vc;
+};
+
 #define NMAPREG			((PCI_MAPREG_END - PCI_MAPREG_START) / \
 				    sizeof(pcireg_t))
 struct pci_dev {
@@ -68,6 +75,8 @@ struct pci_dev {
 	pcireg_t pd_msi_ma;
 	pcireg_t pd_msi_mau32;
 	pcireg_t pd_msi_md;
+	pcireg_t pd_msix_mc;
+	struct msix_vector *pd_msix_table;
 	int pd_pmcsr_state;
 	int pd_vga_decode;
 };
@@ -271,6 +280,9 @@ pci_suspend(struct pci_softc *sc)
 			}
 			pd->pd_msi_mc = reg;
 		}
+
+		pci_suspend_msix(sc->sc_pc, pd->pd_tag, sc->sc_memt,
+		    &pd->pd_msix_mc, pd->pd_msix_table);
 	}
 }
 
@@ -354,6 +366,9 @@ pci_resume(struct pci_softc *sc)
 			pci_conf_write(sc->sc_pc, pd->pd_tag,
 			    off + PCI_MSI_MC, pd->pd_msi_mc);
 		}
+
+		pci_resume_msix(sc->sc_pc, pd->pd_tag, sc->sc_memt,
+		    pd->pd_msix_mc, pd->pd_msix_table);
 	}
 }
 
@@ -531,6 +546,8 @@ pci_probe_device(struct pci_softc *sc, pcitag_t tag,
 			return (0);
 		}
 
+		pd->pd_msix_table = pci_alloc_msix_table(sc->sc_pc, pd->pd_tag);
+
 		s = splhigh();
 		csr = pci_conf_read(pc, tag, PCI_COMMAND_STATUS_REG);
 		if (csr & (PCI_COMMAND_IO_ENABLE | PCI_COMMAND_MEM_ENABLE))
@@ -574,6 +591,7 @@ pci_detach_devices(struct pci_softc *sc, int flags)
 		return (ret);
 
 	for (pd = LIST_FIRST(&sc->sc_devs); pd != NULL; pd = next) {
+		pci_free_msix_table(sc->sc_pc, pd->pd_tag, pd->pd_msix_table);
 		next = LIST_NEXT(pd, pd_next);
 		free(pd, M_DEVBUF, sizeof *pd);
 	}
@@ -1509,3 +1527,128 @@ pci_primary_vga(struct pci_attach_args *pa)
 
 	return (1);
 }
+
+#ifdef __HAVE_PCI_MSIX
+
+struct msix_vector *
+pci_alloc_msix_table(pci_chipset_tag_t pc, pcitag_t tag)
+{
+	struct msix_vector *table;
+	pcireg_t reg;
+	int tblsz;
+
+	if (pci_get_capability(pc, tag, PCI_CAP_MSIX, NULL, &reg) == 0)
+		return NULL;
+
+	tblsz = PCI_MSIX_MC_TBLSZ(reg) + 1;
+	table = mallocarray(tblsz, sizeof(*table), M_DEVBUF, M_WAITOK);
+
+	return table;
+}
+
+void
+pci_free_msix_table(pci_chipset_tag_t pc, pcitag_t tag,
+    struct msix_vector *table)
+{
+	pcireg_t reg;
+	int tblsz;
+
+	if (pci_get_capability(pc, tag, PCI_CAP_MSIX, NULL, &reg) == 0)
+		return;
+
+	tblsz = PCI_MSIX_MC_TBLSZ(reg) + 1;
+	free(table, M_DEVBUF, tblsz * sizeof(*table));
+}
+
+void
+pci_suspend_msix(pci_chipset_tag_t pc, pcitag_t tag,
+    bus_space_tag_t memt, pcireg_t *mc, struct msix_vector *table)
+{
+	bus_space_handle_t memh;
+	pcireg_t reg;
+	int tblsz, i;
+
+	if (pci_get_capability(pc, tag, PCI_CAP_MSIX, NULL, &reg) == 0)
+		return;
+
+	KASSERT(table != NULL);
+
+	if (pci_msix_table_map(pc, tag, memt, &memh))
+		return;
+
+	tblsz = PCI_MSIX_MC_TBLSZ(reg) + 1;
+	for (i = 0; i < tblsz; i++) {
+		table[i].mv_ma = bus_space_read_4(memt, memh, PCI_MSIX_MA(i));
+		table[i].mv_mau32 = bus_space_read_4(memt, memh,
+		    PCI_MSIX_MAU32(i));
+		table[i].mv_md = bus_space_read_4(memt, memh, PCI_MSIX_MD(i));
+		table[i].mv_vc = bus_space_read_4(memt, memh, PCI_MSIX_VC(i));
+	}
+
+	pci_msix_table_unmap(pc, tag, memt, memh);
+	
+	*mc = reg;
+}
+
+void
+pci_resume_msix(pci_chipset_tag_t pc, pcitag_t tag,
+    bus_space_tag_t memt, pcireg_t mc, struct msix_vector *table)
+{
+	bus_space_handle_t memh;
+	pcireg_t reg;
+	int tblsz, i;
+	int off;
+
+	if (pci_get_capability(pc, tag, PCI_CAP_MSIX, &off, &reg) == 0)
+		return;
+
+	KASSERT(table != NULL);
+
+	if (pci_msix_table_map(pc, tag, memt, &memh))
+		return;
+
+	tblsz = PCI_MSIX_MC_TBLSZ(reg) + 1;
+	for (i = 0; i < tblsz; i++) {
+		bus_space_write_4(memt, memh, PCI_MSIX_MA(i), table[i].mv_ma);
+		bus_space_write_4(memt, memh, PCI_MSIX_MAU32(i),
+		    table[i].mv_mau32);
+		bus_space_write_4(memt, memh, PCI_MSIX_MD(i), table[i].mv_md);
+		bus_space_barrier(memt, memh, PCI_MSIX_MA(i), 16,
+		    BUS_SPACE_BARRIER_WRITE);
+		bus_space_write_4(memt, memh, PCI_MSIX_VC(i), table[i].mv_vc);
+		bus_space_barrier(memt, memh, PCI_MSIX_VC(i), 4,
+		    BUS_SPACE_BARRIER_WRITE);
+	}
+
+	pci_msix_table_unmap(pc, tag, memt, memh);
+
+	pci_conf_write(pc, tag, off, mc);
+}
+
+#else
+
+struct msix_vector *
+pci_alloc_msix_table(pci_chipset_tag_t pc, pcitag_t tag)
+{
+	return NULL;
+}
+
+void
+pci_free_msix_table(pci_chipset_tag_t pc, pcitag_t tag,
+    struct msix_vector *table)
+{
+}
+
+void
+pci_suspend_msix(pci_chipset_tag_t pc, pcitag_t tag,
+    bus_space_tag_t memt, pcireg_t *mc, struct msix_vector *table)
+{
+}
+
+void
+pci_resume_msix(pci_chipset_tag_t pc, pcitag_t tag,
+    bus_space_tag_t memt, pcireg_t mc, struct msix_vector *table)
+{
+}
+
+#endif /* __HAVE_PCI_MSIX */
diff --git a/sys/dev/pci/pcivar.h b/sys/dev/pci/pcivar.h
index fdb55e5b545..13d65afadc1 100644
--- a/sys/dev/pci/pcivar.h
+++ b/sys/dev/pci/pcivar.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: pcivar.h,v 1.71 2019/06/17 11:04:06 kettenis Exp $	*/
+/*	$OpenBSD: pcivar.h,v 1.72 2019/06/25 16:46:33 kettenis Exp $	*/
 /*	$NetBSD: pcivar.h,v 1.23 1997/06/06 23:48:05 thorpej Exp $	*/
 
 /*
@@ -238,6 +238,15 @@ int	pci_get_capability(pci_chipset_tag_t, pcitag_t, int,
 int	pci_get_ht_capability(pci_chipset_tag_t, pcitag_t, int,
 	    int *, pcireg_t *);
 
+struct msix_vector;
+
+struct msix_vector *pci_alloc_msix_table(pci_chipset_tag_t, pcitag_t);
+void	pci_free_msix_table(pci_chipset_tag_t, pcitag_t, struct msix_vector *);
+void	pci_suspend_msix(pci_chipset_tag_t, pcitag_t, bus_space_tag_t,
+	    pcireg_t *, struct msix_vector *);
+void	pci_resume_msix(pci_chipset_tag_t, pcitag_t, bus_space_tag_t,
+	    pcireg_t, struct msix_vector *);
+
 uint16_t pci_requester_id(pci_chipset_tag_t, pcitag_t);
 
 struct pci_matchid {