add support for bypassing iommu translation

managing the translation table entries (TTEs) on an iommu is not
free, and is in fact extremely expensive on some platforms. the
flip side of this is that forcing dma through TTEs does provide
some safety and can help during the development of drivers. however,
this has been less true in recent years than it used to be and is
less of a concern now, especially considering the performance
differences on some platforms.

devices have to create dmamaps with BUS_DMA_64BIT to bypass the
iommu because the memory window presented to hardware with direct
access to memory is at an extremely high address. there's no 32bit
bypass access to memory, it has to go through TTEs otherwise.

on an m4000 there are several orders of magnitude performance
difference between a driver with BUS_DMA_64BIT set and one without
it.

hilariously, sun used a pci bridge on a whole generation of machines
that had broken support for dma addresses over 40 bits (or around
there), so devices behind those pci bridges need to have their
dmamap_creates intercepted and any potential BUS_DMA_64BIT flags
cleared on the way to the iommu drivers. this affects at least v215,
v245, and v445, and probably u25s and u45s. it probably explains
why all their onboard nics and disk controllers feel super slow,
and why there was a meme at sun that bcopy was cheaper than dma
when moving packets on and off a nic.

ok kettenis@ deraadt@

9 files changed, 247 insertions, 127 deletions

diff --git a/sys/arch/sparc64/dev/iommu.c b/sys/arch/sparc64/dev/iommu.c
index 8f925c400af..93a6cb25073 100644
--- a/sys/arch/sparc64/dev/iommu.c
+++ b/sys/arch/sparc64/dev/iommu.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: iommu.c,v 1.75 2017/05/25 03:19:39 dlg Exp $	*/
+/*	$OpenBSD: iommu.c,v 1.76 2019/06/25 22:30:55 dlg Exp $	*/
 /*	$NetBSD: iommu.c,v 1.47 2002/02/08 20:03:45 eeh Exp $	*/
 
 /*
@@ -87,6 +87,8 @@ void iommu_dvmamap_print_map(bus_dma_tag_t, struct iommu_state *,
     bus_dmamap_t);
 int iommu_dvmamap_append_range(bus_dma_tag_t, bus_dmamap_t, paddr_t,
     bus_size_t, int, bus_size_t);
+int iommu_dvmamap_insert(bus_dma_tag_t, bus_dmamap_t, bus_addr_t,
+    bus_size_t, int, bus_size_t);
 int64_t iommu_tsb_entry(struct iommu_state *, bus_addr_t);
 void strbuf_reset(struct strbuf_ctl *);
 int iommu_iomap_insert_page(struct iommu_map_state *, paddr_t);
@@ -100,6 +102,25 @@ void iommu_iomap_clear_pages(struct iommu_map_state *);
 void _iommu_dvmamap_sync(bus_dma_tag_t, bus_dma_tag_t, bus_dmamap_t,
     bus_addr_t, bus_size_t, int);
 
+void iommu_hw_enable(struct iommu_state *);
+
+const struct iommu_hw iommu_hw_default = {
+	.ihw_enable	= iommu_hw_enable,
+
+	.ihw_dvma_pa	= IOTTE_PAMASK,
+
+	.ihw_bypass	= 0x3fffUL << 50,
+	.ihw_bypass_nc	= 0,
+	.ihw_bypass_ro	= 0,
+};
+
+void
+iommu_hw_enable(struct iommu_state *is)
+{
+	IOMMUREG_WRITE(is, iommu_tsb, is->is_ptsb);
+	IOMMUREG_WRITE(is, iommu_cr, IOMMUCR_EN | (is->is_tsbsize << 16));
+}
+
 /*
  * Initiate an STC entry flush.
  */
@@ -125,7 +146,8 @@ iommu_strbuf_flush(struct strbuf_ctl *sb, bus_addr_t va)
  *	- create a private DVMA map.
  */
 void
-iommu_init(char *name, struct iommu_state *is, int tsbsize, u_int32_t iovabase)
+iommu_init(char *name, const struct iommu_hw *ihw, struct iommu_state *is,
+    int tsbsize, u_int32_t iovabase)
 {
 	psize_t size;
 	vaddr_t va;
@@ -149,13 +171,9 @@ iommu_init(char *name, struct iommu_state *is, int tsbsize, u_int32_t iovabase)
 	 * be hard-wired, so we read the start and size from the PROM and
 	 * just use those values.
 	 */
-	if (strncmp(name, "pyro", 4) == 0) {
-		is->is_cr = IOMMUREG_READ(is, iommu_cr);
-		is->is_cr &= ~IOMMUCR_FIRE_BE;
-		is->is_cr |= (IOMMUCR_FIRE_SE | IOMMUCR_FIRE_CM_EN |
-		    IOMMUCR_FIRE_TE);
-	} else 
-		is->is_cr = IOMMUCR_EN;
+
+	is->is_hw = ihw;
+
 	is->is_tsbsize = tsbsize;
 	if (iovabase == (u_int32_t)-1) {
 		is->is_dvmabase = IOTSB_VSTART(is->is_tsbsize);
@@ -237,15 +255,6 @@ iommu_init(char *name, struct iommu_state *is, int tsbsize, u_int32_t iovabase)
 	mtx_init(&is->is_mtx, IPL_HIGH);
 
 	/*
-	 * Set the TSB size.  The relevant bits were moved to the TSB
-	 * base register in the PCIe host bridges.
-	 */
-	if (strncmp(name, "pyro", 4) == 0)
-		is->is_ptsb |= is->is_tsbsize;
-	else
-		is->is_cr |= (is->is_tsbsize << 16);
-
-	/*
 	 * Now actually start up the IOMMU.
 	 */
 	iommu_reset(is);
@@ -262,10 +271,7 @@ iommu_reset(struct iommu_state *is)
 {
 	int i;
 
-	IOMMUREG_WRITE(is, iommu_tsb, is->is_ptsb);
-
-	/* Enable IOMMU */
-	IOMMUREG_WRITE(is, iommu_cr, is->is_cr);
+	(*is->is_hw->ihw_enable)(is);
 
 	for (i = 0; i < 2; ++i) {
 		struct strbuf_ctl *sb = is->is_sb[i];
@@ -280,7 +286,7 @@ iommu_reset(struct iommu_state *is)
 			printf(", STC%d enabled", i);
 	}
 
-	if (is->is_flags & IOMMU_FLUSH_CACHE)
+	if (ISSET(is->is_hw->ihw_flags, IOMMU_HW_FLUSH_CACHE))
 		IOMMUREG_WRITE(is, iommu_cache_invalidate, -1ULL);
 }
 
@@ -433,7 +439,7 @@ iommu_extract(struct iommu_state *is, bus_addr_t dva)
 	if (dva >= is->is_dvmabase && dva <= is->is_dvmaend)
 		tte = is->is_tsb[IOTSBSLOT(dva, is->is_tsbsize)];
 
-	return (tte & IOTTE_PAMASK);
+	return (tte & is->is_hw->ihw_dvma_pa);
 }
 
 /*
@@ -601,6 +607,7 @@ iommu_dvmamap_create(bus_dma_tag_t t, bus_dma_tag_t t0, struct strbuf_ctl *sb,
 {
 	int ret;
 	bus_dmamap_t map;
+	struct iommu_state *is = sb->sb_iommu;
 	struct iommu_map_state *ims;
 
 	BUS_DMA_FIND_PARENT(t, _dmamap_create);
@@ -610,6 +617,12 @@ iommu_dvmamap_create(bus_dma_tag_t t, bus_dma_tag_t t0, struct strbuf_ctl *sb,
 	if (ret)
 		return (ret);
 
+	if (flags & BUS_DMA_64BIT) {
+		map->_dm_cookie = is;
+		*dmamap = map;
+		return (0);
+	}
+
 	ims = iommu_iomap_create(atop(round_page(size)));
 
 	if (ims == NULL) {
@@ -641,8 +654,10 @@ iommu_dvmamap_destroy(bus_dma_tag_t t, bus_dma_tag_t t0, bus_dmamap_t map)
 	if (map->dm_nsegs)
 		bus_dmamap_unload(t0, map);
 
-        if (map->_dm_cookie)
-                iommu_iomap_destroy(map->_dm_cookie);
+	if (!ISSET(map->_dm_flags, BUS_DMA_64BIT)) {
+	        if (map->_dm_cookie)
+			iommu_iomap_destroy(map->_dm_cookie);
+	}
 	map->_dm_cookie = NULL;
 
 	BUS_DMA_FIND_PARENT(t, _dmamap_destroy);
@@ -667,36 +682,36 @@ iommu_dvmamap_load(bus_dma_tag_t t, bus_dma_tag_t t0, bus_dmamap_t map,
 	u_long dvmaddr, sgstart, sgend;
 	bus_size_t align, boundary;
 	struct iommu_state *is;
-	struct iommu_map_state *ims = map->_dm_cookie;
+	struct iommu_map_state *ims;
 	pmap_t pmap;
 
-#ifdef DIAGNOSTIC
-	if (ims == NULL)
-		panic("iommu_dvmamap_load: null map state");
-#endif
-#ifdef DEBUG
-	if (ims->ims_sb == NULL)
-		panic("iommu_dvmamap_load: null sb");
-	if (ims->ims_sb->sb_iommu == NULL)
-		panic("iommu_dvmamap_load: null iommu");
-#endif /* DEBUG */
-	is = ims->ims_sb->sb_iommu;
-
-	if (map->dm_nsegs) {
-		/*
-		 * Is it still in use? _bus_dmamap_load should have taken care
-		 * of this.
-		 */
-#ifdef DIAGNOSTIC
-		panic("iommu_dvmamap_load: map still in use");
-#endif
-		bus_dmamap_unload(t0, map);
-	}
-
 	/*
 	 * Make sure that on error condition we return "no valid mappings".
 	 */
-	map->dm_nsegs = 0;
+	KASSERTMSG(map->dm_nsegs == 0, "map still in use");
+
+	if (ISSET(map->_dm_flags, BUS_DMA_64BIT)) {
+		unsigned long bypass;
+		int i;
+
+		is = map->_dm_cookie;
+		bypass = is->is_hw->ihw_bypass;
+
+		/* Bypass translation by the IOMMU. */
+
+		BUS_DMA_FIND_PARENT(t, _dmamap_load);
+		err = (*t->_dmamap_load)(t, t0, map, buf, buflen, p, flags);
+		if (err != 0)
+			return (err);
+
+		for (i = 0; i < map->dm_nsegs; i++)
+			map->dm_segs[i].ds_addr |= bypass;
+
+		return (0);
+	}
+
+	ims = map->_dm_cookie;
+	is = ims->ims_sb->sb_iommu;
 
 	if (buflen < 1 || buflen > map->_dm_size) {
 		DPRINTF(IDB_BUSDMA,
@@ -876,27 +891,9 @@ iommu_dvmamap_load_raw(bus_dma_tag_t t, bus_dma_tag_t t0, bus_dmamap_t map,
 	bus_size_t boundary, align;
 	u_long dvmaddr, sgstart, sgend;
 	struct iommu_state *is;
-	struct iommu_map_state *ims = map->_dm_cookie;
-
-#ifdef DIAGNOSTIC
-	if (ims == NULL)
-		panic("iommu_dvmamap_load_raw: null map state");
-#endif
-#ifdef DEBUG
-	if (ims->ims_sb == NULL)
-		panic("iommu_dvmamap_load_raw: null sb");
-	if (ims->ims_sb->sb_iommu == NULL)
-		panic("iommu_dvmamap_load_raw: null iommu");
-#endif /* DEBUG */
-	is = ims->ims_sb->sb_iommu;
+	struct iommu_map_state *ims;
 
-	if (map->dm_nsegs) {
-		/* Already in use?? */
-#ifdef DIAGNOSTIC
-		panic("iommu_dvmamap_load_raw: map still in use");
-#endif
-		bus_dmamap_unload(t0, map);
-	}
+	KASSERTMSG(map->dm_nsegs == 0, "map stil in use");
 
 	/*
 	 * A boundary presented to bus_dmamem_alloc() takes precedence
@@ -905,6 +902,31 @@ iommu_dvmamap_load_raw(bus_dma_tag_t t, bus_dma_tag_t t0, bus_dmamap_t map,
 	if ((boundary = segs[0]._ds_boundary) == 0)
 		boundary = map->_dm_boundary;
 
+	if (ISSET(map->_dm_flags, BUS_DMA_64BIT)) {
+		unsigned long bypass;
+
+		is = map->_dm_cookie;
+		bypass = is->is_hw->ihw_bypass;
+
+		/* Bypass translation by the IOMMU. */
+		for (i = 0; i < nsegs; i++) {
+			err = iommu_dvmamap_insert(t, map,
+			    bypass | segs[i].ds_addr, segs[i].ds_len,
+			    0, boundary);
+			if (err != 0) {
+				map->dm_nsegs = 0;
+				return (err);
+			}
+		}
+
+		map->dm_mapsize = size;
+
+		return (0);
+	}
+
+	ims = map->_dm_cookie;
+	is = ims->ims_sb->sb_iommu;
+
 	align = MAX(segs[0]._ds_align, PAGE_SIZE);
 
 	/*
@@ -1084,22 +1106,24 @@ iommu_dvmamap_append_range(bus_dma_tag_t t, bus_dmamap_t map, paddr_t pa,
     bus_size_t length, int flags, bus_size_t boundary)
 {
 	struct iommu_map_state *ims = map->_dm_cookie;
-	bus_addr_t sgstart, sgend, bd_mask;
-	bus_dma_segment_t *seg = NULL;
-	int i = map->dm_nsegs;
+	bus_addr_t sgstart = iommu_iomap_translate(ims, pa);
 
-#ifdef DEBUG
-	if (ims == NULL)
-		panic("iommu_dvmamap_append_range: null map state");
-#endif
+	return (iommu_dvmamap_insert(t, map, sgstart, length, flags, boundary));
+}
 
-	sgstart = iommu_iomap_translate(ims, pa);
-	sgend = sgstart + length - 1;
+int
+iommu_dvmamap_insert(bus_dma_tag_t t, bus_dmamap_t map,
+    bus_addr_t sgstart, bus_size_t length, int flags, bus_size_t boundary)
+{
+	bus_addr_t sgend = sgstart + length - 1;
+	bus_addr_t bd_mask;
+	bus_dma_segment_t *seg = NULL;
+	int i = map->dm_nsegs;
 
 #ifdef DIAGNOSTIC
 	if (sgstart == 0 || sgstart > sgend) {
-		printf("append range invalid mapping for %lx "
-		    "(0x%lx - 0x%lx)\n", pa, sgstart, sgend);
+		printf("append range invalid mapping for "
+		    "0x%lx - 0x%lx\n", sgstart, sgend);
 		map->dm_nsegs = 0;
 		return (EINVAL);
 	}
@@ -1298,20 +1322,17 @@ void
 iommu_dvmamap_unload(bus_dma_tag_t t, bus_dma_tag_t t0, bus_dmamap_t map)
 {
 	struct iommu_state *is;
-	struct iommu_map_state *ims = map->_dm_cookie;
+	struct iommu_map_state *ims;
 	bus_addr_t dvmaddr = map->_dm_dvmastart;
 	bus_size_t sgsize = map->_dm_dvmasize;
 	int error;
 
-#ifdef DEBUG
-	if (ims == NULL)
-		panic("iommu_dvmamap_unload: null map state");
-	if (ims->ims_sb == NULL)
-		panic("iommu_dvmamap_unload: null sb");
-	if (ims->ims_sb->sb_iommu == NULL)
-		panic("iommu_dvmamap_unload: null iommu");
-#endif /* DEBUG */
+	if (ISSET(map->_dm_flags, BUS_DMA_64BIT)) {
+		bus_dmamap_unload(t->_parent, map);
+		return;
+	}
 
+	ims = map->_dm_cookie;
 	is = ims->ims_sb->sb_iommu;
 
 	/* Flush the iommu */
@@ -1488,7 +1509,7 @@ iommu_dvmamap_print_map(bus_dma_tag_t t, struct iommu_state *is,
 		break;
 	}
 
-	if (map->_dm_cookie) {
+	if (!ISSET(map->_dm_flags, BUS_DMA_64BIT) && map->_dm_cookie != NULL) {
 		struct iommu_map_state *ims = map->_dm_cookie;
 		struct iommu_page_map *ipm = &ims->ims_map;
 
@@ -1546,19 +1567,19 @@ void
 iommu_dvmamap_sync(bus_dma_tag_t t, bus_dma_tag_t t0, bus_dmamap_t map,
     bus_addr_t offset, bus_size_t len, int ops)
 {
-	struct iommu_map_state *ims = map->_dm_cookie;
+	struct iommu_map_state *ims;
 
-#ifdef DIAGNOSTIC
-	if (ims == NULL)
-		panic("iommu_dvmamap_sync: null map state");
-	if (ims->ims_sb == NULL)
-		panic("iommu_dvmamap_sync: null sb");
-	if (ims->ims_sb->sb_iommu == NULL)
-		panic("iommu_dvmamap_sync: null iommu");
-#endif
 	if (len == 0)
 		return;
 
+	if (map->_dm_flags & BUS_DMA_64BIT) {
+		if (ops & (BUS_DMASYNC_PREWRITE | BUS_DMASYNC_POSTREAD))
+			__membar("#MemIssue");
+		return;
+	}
+
+	ims = map->_dm_cookie;
+
 	if (ops & BUS_DMASYNC_PREWRITE)
 		__membar("#MemIssue");
 
@@ -1622,9 +1643,13 @@ iommu_dvmamem_alloc(bus_dma_tag_t t, bus_dma_tag_t t0, bus_size_t size,
 	    "bound %llx segp %p flags %d\n", (unsigned long long)size,
 	    (unsigned long long)alignment, (unsigned long long)boundary,
 	    segs, flags));
+
+	if ((flags & BUS_DMA_64BIT) == 0)
+		flags |= BUS_DMA_DVMA;
+
 	BUS_DMA_FIND_PARENT(t, _dmamem_alloc);
 	return ((*t->_dmamem_alloc)(t, t0, size, alignment, boundary,
-	    segs, nsegs, rsegs, flags | BUS_DMA_DVMA));
+	    segs, nsegs, rsegs, flags));
 }
 
 void
@@ -1763,7 +1788,7 @@ iommu_iomap_load_map(struct iommu_state *is, struct iommu_map_state *ims,
 
 		/* Flush cache if necessary. */
 		slot = IOTSBSLOT(e->ipe_va, is->is_tsbsize);
-		if (is->is_flags & IOMMU_FLUSH_CACHE &&
+		if (ISSET(is->is_hw->ihw_flags, IOMMU_HW_FLUSH_CACHE) &&
 		    (i == (ipm->ipm_pagecnt - 1) || (slot % 8) == 7))
 			IOMMUREG_WRITE(is, iommu_cache_flush,
 			    is->is_ptsb + slot * 8);
@@ -1788,7 +1813,7 @@ iommu_iomap_unload_map(struct iommu_state *is, struct iommu_map_state *ims)
 
 		/* Flush cache if necessary. */
 		slot = IOTSBSLOT(e->ipe_va, is->is_tsbsize);
-		if (is->is_flags & IOMMU_FLUSH_CACHE &&
+		if (ISSET(is->is_hw->ihw_flags, IOMMU_HW_FLUSH_CACHE) &&
 		    (i == (ipm->ipm_pagecnt - 1) || (slot % 8) == 7))
 			IOMMUREG_WRITE(is, iommu_cache_flush,
 			    is->is_ptsb + slot * 8);
diff --git a/sys/arch/sparc64/dev/iommureg.h b/sys/arch/sparc64/dev/iommureg.h
index db1b0671dbe..b70b6fc4cc4 100644
--- a/sys/arch/sparc64/dev/iommureg.h
+++ b/sys/arch/sparc64/dev/iommureg.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: iommureg.h,v 1.17 2012/08/17 20:46:50 kettenis Exp $	*/
+/*	$OpenBSD: iommureg.h,v 1.18 2019/06/25 22:30:56 dlg Exp $	*/
 /*	$NetBSD: iommureg.h,v 1.6 2001/07/20 00:07:13 eeh Exp $	*/
 
 /*
@@ -90,10 +90,11 @@ struct iommu_strbuf {
 #define IOMMUCR_DE		0x000000000000000002LL	/* Diag enable */
 #define IOMMUCR_EN		0x000000000000000001LL	/* Enable IOMMU */
 
-#define IOMMUCR_FIRE_SE		0x000000000000000400LL	/* Snoop enable */
-#define IOMMUCR_FIRE_CM_EN	0x000000000000000300LL  /* Cache mode enable */
-#define IOMMUCR_FIRE_BE		0x000000000000000002LL	/* Bypass enable */
-#define IOMMUCR_FIRE_TE		0x000000000000000001LL	/* Translation enabled */
+#define IOMMUCR_FIRE_PD		0x000000000000001000UL	/* Process disable */
+#define IOMMUCR_FIRE_SE		0x000000000000000400UL	/* Snoop enable */
+#define IOMMUCR_FIRE_CM_EN	0x000000000000000300UL  /* Cache mode enable */
+#define IOMMUCR_FIRE_BE		0x000000000000000002UL	/* Bypass enable */
+#define IOMMUCR_FIRE_TE		0x000000000000000001UL	/* Translation enabled */
 
 /*
  * IOMMU stuff
diff --git a/sys/arch/sparc64/dev/iommuvar.h b/sys/arch/sparc64/dev/iommuvar.h
index f0a548079c6..b6c7ce06ef7 100644
--- a/sys/arch/sparc64/dev/iommuvar.h
+++ b/sys/arch/sparc64/dev/iommuvar.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: iommuvar.h,v 1.17 2016/05/04 18:26:12 kettenis Exp $	*/
+/*	$OpenBSD: iommuvar.h,v 1.18 2019/06/25 22:30:56 dlg Exp $	*/
 /*	$NetBSD: iommuvar.h,v 1.9 2001/10/07 20:30:41 eeh Exp $	*/
 
 /*
@@ -100,6 +100,21 @@ struct iommu_map_state {
 };
 #define IOMMU_MAP_STREAM	1
 
+struct iommu_hw {
+	void			(*ihw_enable)(struct iommu_state *);
+
+	unsigned long		ihw_dvma_pa;
+
+	unsigned long		ihw_bypass;
+	unsigned long		ihw_bypass_nc;		/* non-cached */
+	unsigned long		ihw_bypass_ro;		/* relaxed ordering */
+
+	unsigned int		ihw_flags;
+#define IOMMU_HW_FLUSH_CACHE		(1 << 0)
+};
+
+extern const struct iommu_hw iommu_hw_default;
+
 /*
  * per-IOMMU state
  */
@@ -112,8 +127,7 @@ struct iommu_state {
 	int64_t			is_cr;		/* Control register value */
 	struct mutex		is_mtx;
 	struct extent		*is_dvmamap;	/* DVMA map for this instance */
-	int			is_flags;
-#define IOMMU_FLUSH_CACHE	0x00000001
+	const struct iommu_hw	*is_hw;
 
 	struct strbuf_ctl	*is_sb[2];	/* Streaming buffers if any */
 
@@ -126,7 +140,8 @@ struct iommu_state {
 };
 
 /* interfaces for PCI/SBus code */
-void	iommu_init(char *, struct iommu_state *, int, u_int32_t);
+void	iommu_init(char *, const struct iommu_hw *, struct iommu_state *,
+    int, u_int32_t);
 void	iommu_reset(struct iommu_state *);
 paddr_t iommu_extract(struct iommu_state *, bus_addr_t);
 int64_t iommu_lookup_tte(struct iommu_state *, bus_addr_t);
diff --git a/sys/arch/sparc64/dev/pci_machdep.c b/sys/arch/sparc64/dev/pci_machdep.c
index b6de170b2c0..9e1417e693a 100644
--- a/sys/arch/sparc64/dev/pci_machdep.c
+++ b/sys/arch/sparc64/dev/pci_machdep.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: pci_machdep.c,v 1.47 2019/06/11 00:45:31 dlg Exp $	*/
+/*	$OpenBSD: pci_machdep.c,v 1.48 2019/06/25 22:30:56 dlg Exp $	*/
 /*	$NetBSD: pci_machdep.c,v 1.22 2001/07/20 00:07:13 eeh Exp $	*/
 
 /*
@@ -57,6 +57,7 @@ int sparc_pci_debug = 0x0;
 #include <machine/openfirm.h>
 #include <dev/pci/pcivar.h>
 #include <dev/pci/pcireg.h>
+#include <dev/pci/pcidevs.h>
 
 #include <dev/ofw/ofw_pci.h>
 
@@ -87,6 +88,46 @@ pci_attach_hook(parent, self, pba)
 }
 
 int
+pci_32bit_dmamap_create(bus_dma_tag_t dt, bus_dma_tag_t t0, bus_size_t size,
+    int nsegments, bus_size_t maxsegsz, bus_size_t boundary, int flags,
+    bus_dmamap_t *dmamp)
+{
+	bus_dma_tag_t pdt = dt->_parent;
+
+	CLR(flags, BUS_DMA_64BIT);
+
+	return ((*pdt->_dmamap_create)(pdt, t0, size, nsegments, maxsegsz,
+	    boundary, flags, dmamp));
+}
+
+int
+pci_probe_device_hook(pci_chipset_tag_t pc, struct pci_attach_args *pa)
+{
+	bus_dma_tag_t dt, pdt;
+
+	if (PCI_VENDOR(pa->pa_id) == PCI_VENDOR_RCC &&
+	    PCI_PRODUCT(pa->pa_id) == PCI_PRODUCT_RCC_PCIE_PCIX) {
+		/*
+		 * These PCI bridges only support 40bit DVA, so intercept
+		 * bus_dmamap_create so we can clear BUS_DMA_64BIT.
+		 */
+
+		dt = malloc(sizeof(*dt), M_DEVBUF, M_NOWAIT | M_ZERO);
+		if (dt == NULL)
+			panic("%s: could not alloc dma tag", __func__);
+
+		pdt = pa->pa_dmat;
+
+		dt->_parent = pdt;
+		dt->_dmamap_create = pci_32bit_dmamap_create;
+
+		pa->pa_dmat = dt;
+	}
+
+	return (0);
+}
+
+int
 pci_bus_maxdevs(pc, busno)
 	pci_chipset_tag_t pc;
 	int busno;
diff --git a/sys/arch/sparc64/dev/psycho.c b/sys/arch/sparc64/dev/psycho.c
index a35cf549d23..e24f804dff6 100644
--- a/sys/arch/sparc64/dev/psycho.c
+++ b/sys/arch/sparc64/dev/psycho.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: psycho.c,v 1.75 2017/05/25 03:19:39 dlg Exp $	*/
+/*	$OpenBSD: psycho.c,v 1.76 2019/06/25 22:30:56 dlg Exp $	*/
 /*	$NetBSD: psycho.c,v 1.39 2001/10/07 20:30:41 eeh Exp $	*/
 
 /*
@@ -902,7 +902,7 @@ psycho_iommu_init(struct psycho_softc *sc, int tsbsize)
 		panic("couldn't malloc iommu name");
 	snprintf(name, 32, "%s dvma", sc->sc_dev.dv_xname);
 
-	iommu_init(name, is, tsbsize, iobase);
+	iommu_init(name, &iommu_hw_default, is, tsbsize, iobase);
 }
 
 /*
diff --git a/sys/arch/sparc64/dev/pyro.c b/sys/arch/sparc64/dev/pyro.c
index d9340a6cd57..2769f71a536 100644
--- a/sys/arch/sparc64/dev/pyro.c
+++ b/sys/arch/sparc64/dev/pyro.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: pyro.c,v 1.32 2019/06/11 00:45:31 dlg Exp $	*/
+/*	$OpenBSD: pyro.c,v 1.33 2019/06/25 22:30:56 dlg Exp $	*/
 
 /*
  * Copyright (c) 2002 Jason L. Wright (jason@thought.net)
@@ -131,6 +131,30 @@ int pyro_msi_eq_intr(void *);
 int pyro_dmamap_create(bus_dma_tag_t, bus_dma_tag_t, bus_size_t, int,
     bus_size_t, bus_size_t, int, bus_dmamap_t *);
 
+void pyro_iommu_enable(struct iommu_state *);
+
+const struct iommu_hw iommu_hw_fire = {
+	.ihw_enable	= pyro_iommu_enable,
+
+	.ihw_dvma_pa	= 0x000007ffffffffffUL,
+
+	.ihw_bypass	= 0xfffc000000000000UL,
+	.ihw_bypass_nc	= 0x0000080000000000UL,
+	.ihw_bypass_ro	= 0,
+};
+
+const struct iommu_hw iommu_hw_oberon = {
+	.ihw_enable	= pyro_iommu_enable,
+
+	.ihw_dvma_pa	= 0x00007fffffffffffUL,
+
+	.ihw_bypass	= 0x7ffc000000000000UL,
+	.ihw_bypass_nc	= 0x0000800000000000UL,
+	.ihw_bypass_ro	= 0x8000000000000000UL,
+
+	.ihw_flags	= IOMMU_HW_FLUSH_CACHE,
+};
+
 #ifdef DDB
 void pyro_xir(void *, int);
 #endif
@@ -266,6 +290,7 @@ pyro_init_iommu(struct pyro_softc *sc, struct pyro_pbm *pbm)
 	int tsbsize = 7;
 	u_int32_t iobase = -1;
 	char *name;
+	const struct iommu_hw *ihw = &iommu_hw_fire;
 
 	is->is_bustag = sc->sc_bust;
 
@@ -282,11 +307,23 @@ pyro_init_iommu(struct pyro_softc *sc, struct pyro_pbm *pbm)
 		panic("couldn't malloc iommu name");
 	snprintf(name, 32, "%s dvma", sc->sc_dv.dv_xname);
 
-	/* On Oberon, we need to flush the cache. */
 	if (sc->sc_oberon)
-		is->is_flags |= IOMMU_FLUSH_CACHE;
+		ihw = &iommu_hw_oberon;
+
+	iommu_init(name, ihw, is, tsbsize, iobase);
+}
+
+void
+pyro_iommu_enable(struct iommu_state *is)
+{
+	unsigned long cr;
+
+	cr = IOMMUREG_READ(is, iommu_cr);
+	cr |= IOMMUCR_FIRE_BE | IOMMUCR_FIRE_SE | IOMMUCR_FIRE_CM_EN |
+	    IOMMUCR_FIRE_TE;
 
-	iommu_init(name, is, tsbsize, iobase);
+	IOMMUREG_WRITE(is, iommu_tsb, is->is_ptsb | is->is_tsbsize);
+	IOMMUREG_WRITE(is, iommu_cr, cr);
 }
 
 void
diff --git a/sys/arch/sparc64/dev/sbus.c b/sys/arch/sparc64/dev/sbus.c
index 7676fc0c0d4..cac8561ce03 100644
--- a/sys/arch/sparc64/dev/sbus.c
+++ b/sys/arch/sparc64/dev/sbus.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: sbus.c,v 1.44 2015/09/19 21:07:04 semarie Exp $	*/
+/*	$OpenBSD: sbus.c,v 1.45 2019/06/25 22:30:56 dlg Exp $	*/
 /*	$NetBSD: sbus.c,v 1.46 2001/10/07 20:30:41 eeh Exp $ */
 
 /*-
@@ -349,7 +349,7 @@ sbus_mb_attach(struct device *parent, struct device *self, void *aux)
 	snprintf(name, 32, "%s dvma", sc->sc_dev.dv_xname);
 
 	printf("%s: ", sc->sc_dev.dv_xname);
-	iommu_init(name, &sc->sc_is, 0, -1);
+	iommu_init(name, &iommu_hw_default, &sc->sc_is, 0, -1);
 
 	/* Initialize Starfire PC interrupt translation. */
 	if (OF_getprop(findroot(), "name", buf, sizeof(buf)) > 0 &&
diff --git a/sys/arch/sparc64/dev/schizo.c b/sys/arch/sparc64/dev/schizo.c
index 33876416b64..3152e77cada 100644
--- a/sys/arch/sparc64/dev/schizo.c
+++ b/sys/arch/sparc64/dev/schizo.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: schizo.c,v 1.68 2017/05/25 03:19:39 dlg Exp $	*/
+/*	$OpenBSD: schizo.c,v 1.69 2019/06/25 22:30:56 dlg Exp $	*/
 
 /*
  * Copyright (c) 2002 Jason L. Wright (jason@thought.net)
@@ -451,7 +451,7 @@ schizo_init_iommu(struct schizo_softc *sc, struct schizo_pbm *pbm)
 		    "using iobase=0x%x, tsbsize=%d\n", iobase, tsbsize));
 	}
 
-	iommu_init(name, is, tsbsize, iobase);
+	iommu_init(name, &iommu_hw_default, is, tsbsize, iobase);
 }
 
 int
diff --git a/sys/arch/sparc64/include/pci_machdep.h b/sys/arch/sparc64/include/pci_machdep.h
index 14bb910c58a..02d7f548dbd 100644
--- a/sys/arch/sparc64/include/pci_machdep.h
+++ b/sys/arch/sparc64/include/pci_machdep.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: pci_machdep.h,v 1.34 2019/06/11 00:45:31 dlg Exp $	*/
+/*	$OpenBSD: pci_machdep.h,v 1.35 2019/06/25 22:30:56 dlg Exp $	*/
 /* $NetBSD: pci_machdep.h,v 1.7 2001/07/20 00:07:14 eeh Exp $ */
 
 /*
@@ -84,10 +84,13 @@ struct sparc_pci_chipset {
 	pcireg_t (*conf_read)(pci_chipset_tag_t, pcitag_t, int);
 	void (*conf_write)(pci_chipset_tag_t, pcitag_t, int, pcireg_t);
 	int (*intr_map)(struct pci_attach_args *, pci_intr_handle_t *);
+	int (*probe_device_hook)(void *, struct pci_attach_args *);
 };
 
 void		pci_attach_hook(struct device *, struct device *,
 				     struct pcibus_attach_args *);
+int		pci_probe_device_hook(pci_chipset_tag_t,
+		    struct pci_attach_args *);
 int		pci_bus_maxdevs(pci_chipset_tag_t, int);
 pcitag_t	pci_make_tag(pci_chipset_tag_t, int, int, int);
 void		pci_decompose_tag(pci_chipset_tag_t, pcitag_t, int *, int *,
@@ -116,8 +119,6 @@ int		sparc64_pci_enumerate_bus(struct pci_softc *,
 
 #define PCI_MACHDEP_ENUMERATE_BUS sparc64_pci_enumerate_bus
 
-#define	pci_probe_device_hook(c, a)	(0)
-
 #define	pci_min_powerstate(c, t)	(PCI_PMCSR_STATE_D3)
 #define	pci_set_powerstate_md(c, t, s, p)