While it seems like we can choose any I/O virtual address for peripheral

devices, this isn't really the case.  It depends on the bus topology of
how devices are connected.  In the case of PCIe, devices are assigned
addresses (in PCI BARs) from the PCI address spaces.  Now if we take an
address from one of these address spaces for our IOVA, transfers from
from a PCI device to that address will terminate inside of the PCI bus.
This is because from the PCI buses' point-of-view, the address we chose
is part of its address space.  To make sure we don't allocate addresses
from there, reserve the PCI addresses in the IOVA.

Note that smmu(4) currently gives each device its own IOVA.  So the PCI
addresses will be reserved only in IOVA from PCI devices, and only the
addresses concerning the PCI bus it is connected to will be reserved.
All other devices behind an smmu(4) will not have any changes to their
IOVA.

ok kettenis@

1 files changed, 69 insertions, 16 deletions

diff --git a/sys/dev/ofw/ofw_misc.c b/sys/dev/ofw/ofw_misc.c
index 389ad22423c..1f562eb89df 100644
--- a/sys/dev/ofw/ofw_misc.c
+++ b/sys/dev/ofw/ofw_misc.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: ofw_misc.c,v 1.32 2021/04/07 17:12:22 kettenis Exp $	*/
+/*	$OpenBSD: ofw_misc.c,v 1.33 2021/06/25 17:41:22 patrick Exp $	*/
 /*
  * Copyright (c) 2017 Mark Kettenis
  *
@@ -896,18 +896,17 @@ iommu_device_do_map(uint32_t phandle, uint32_t *cells, bus_dma_tag_t dmat)
 	return dmat;
 }
 
-bus_dma_tag_t
-iommu_device_map(int node, bus_dma_tag_t dmat)
+int
+iommu_device_lookup(int node, uint32_t *phandle, uint32_t *sid)
 {
-	uint32_t sid = 0;
-	uint32_t phandle = 0;
 	uint32_t *cell;
 	uint32_t *map;
 	int len, icells, ncells;
+	int ret = 1;
 
 	len = OF_getproplen(node, "iommus");
 	if (len <= 0)
-		return dmat;
+		return ret;
 
 	map = malloc(len, M_TEMP, M_WAITOK);
 	OF_getpropintarray(node, "iommus", map, len);
@@ -925,8 +924,9 @@ iommu_device_map(int node, bus_dma_tag_t dmat)
 
 		KASSERT(icells == 1);
 
-		phandle = cell[0];
-		sid = cell[1];
+		*phandle = cell[0];
+		*sid = cell[1];
+		ret = 0;
 		break;
 
 		cell += (1 + icells);
@@ -936,22 +936,23 @@ iommu_device_map(int node, bus_dma_tag_t dmat)
 out:
 	free(map, M_TEMP, len);
 
-	return iommu_device_do_map(phandle, &sid, dmat);
+	return ret;
 }
 
-bus_dma_tag_t
-iommu_device_map_pci(int node, uint32_t rid, bus_dma_tag_t dmat)
+int
+iommu_device_lookup_pci(int node, uint32_t rid, uint32_t *phandle,
+    uint32_t *sid)
 {
-	uint32_t sid_base, sid = 0;
-	uint32_t phandle = 0;
+	uint32_t sid_base;
 	uint32_t *cell;
 	uint32_t *map;
 	uint32_t mask, rid_base;
 	int len, length, icells, ncells;
+	int ret = 1;
 
 	len = OF_getproplen(node, "iommu-map");
 	if (len <= 0)
-		return dmat;
+		return ret;
 
 	map = malloc(len, M_TEMP, M_WAITOK);
 	OF_getpropintarray(node, "iommu-map", map, len);
@@ -976,8 +977,9 @@ iommu_device_map_pci(int node, uint32_t rid, bus_dma_tag_t dmat)
 		sid_base = cell[2];
 		length = cell[3];
 		if (rid >= rid_base && rid < rid_base + length) {
-			sid = sid_base + (rid - rid_base);
-			phandle = cell[1];
+			*sid = sid_base + (rid - rid_base);
+			*phandle = cell[1];
+			ret = 0;
 			break;
 		}
 
@@ -988,5 +990,56 @@ iommu_device_map_pci(int node, uint32_t rid, bus_dma_tag_t dmat)
 out:
 	free(map, M_TEMP, len);
 
+	return ret;
+}
+
+bus_dma_tag_t
+iommu_device_map(int node, bus_dma_tag_t dmat)
+{
+	uint32_t phandle, sid;
+
+	if (iommu_device_lookup(node, &phandle, &sid))
+		return dmat;
+
+	return iommu_device_do_map(phandle, &sid, dmat);
+}
+
+bus_dma_tag_t
+iommu_device_map_pci(int node, uint32_t rid, bus_dma_tag_t dmat)
+{
+	uint32_t phandle, sid;
+
+	if (iommu_device_lookup_pci(node, rid, &phandle, &sid))
+		return dmat;
+
 	return iommu_device_do_map(phandle, &sid, dmat);
 }
+
+void
+iommu_device_do_reserve(uint32_t phandle, uint32_t *cells, bus_addr_t addr,
+    bus_size_t size)
+{
+	struct iommu_device *id;
+
+	if (phandle == 0)
+		return;
+
+	LIST_FOREACH(id, &iommu_devices, id_list) {
+		if (id->id_phandle == phandle) {
+			id->id_reserve(id->id_cookie, cells, addr, size);
+			break;
+		}
+	}
+}
+
+void
+iommu_reserve_region_pci(int node, uint32_t rid, bus_addr_t addr,
+    bus_size_t size)
+{
+	uint32_t phandle, sid;
+
+	if (iommu_device_lookup_pci(node, rid, &phandle, &sid))
+		return;
+
+	return iommu_device_do_reserve(phandle, &sid, addr, size);
+}