Support BUS_DMA_NOCACHE in bus_dma(9). Memory allocations done with

BUS_DMA_NOCACHE (or BUS_DMA_COHERENT if the platform does not have coherent
caches) will use PMAP_NOCACHE when invoking pmap_enter(), to avoid creating
cached mappings, and then evicting them from the cache.

6 files changed, 108 insertions, 69 deletions

diff --git a/sys/arch/loongson/include/bus.h b/sys/arch/loongson/include/bus.h
index d9f15ef1ac3..676c5cd0d49 100644
--- a/sys/arch/loongson/include/bus.h
+++ b/sys/arch/loongson/include/bus.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: bus.h,v 1.3 2010/05/08 21:59:56 miod Exp $	*/
+/*	$OpenBSD: bus.h,v 1.4 2014/03/10 21:32:15 miod Exp $	*/
 
 /*
  * Copyright (c) 2003-2004 Opsycon AB Sweden.  All rights reserved.
@@ -332,18 +332,19 @@ bus_space_barrier(bus_space_tag_t t, bus_space_handle_t h, bus_size_t offset,
 #define BUS_SPACE_BARRIER_READ  0x01		/* force read barrier */
 #define BUS_SPACE_BARRIER_WRITE 0x02		/* force write barrier */
 
-#define	BUS_DMA_WAITOK		0x000
-#define	BUS_DMA_NOWAIT		0x001
-#define	BUS_DMA_ALLOCNOW	0x002
-#define	BUS_DMA_COHERENT	0x008
-#define	BUS_DMA_BUS1		0x010	/* placeholders for bus functions... */
-#define	BUS_DMA_BUS2		0x020
-#define	BUS_DMA_BUS3		0x040
-#define	BUS_DMA_BUS4		0x080
-#define	BUS_DMA_READ		0x100   /* mapping is device -> memory only */
-#define	BUS_DMA_WRITE		0x200   /* mapping is memory -> device only */
-#define	BUS_DMA_STREAMING	0x400   /* hint: sequential, unidirectional */
-#define	BUS_DMA_ZERO		0x800	/* zero memory in dmamem_alloc */
+#define	BUS_DMA_WAITOK		0x0000
+#define	BUS_DMA_NOWAIT		0x0001
+#define	BUS_DMA_ALLOCNOW	0x0002
+#define	BUS_DMA_COHERENT	0x0008
+#define	BUS_DMA_BUS1		0x0010	/* placeholders for bus functions... */
+#define	BUS_DMA_BUS2		0x0020
+#define	BUS_DMA_BUS3		0x0040
+#define	BUS_DMA_BUS4		0x0080
+#define	BUS_DMA_READ		0x0100	/* mapping is device -> memory only */
+#define	BUS_DMA_WRITE		0x0200	/* mapping is memory -> device only */
+#define	BUS_DMA_STREAMING	0x0400	/* hint: sequential, unidirectional */
+#define	BUS_DMA_ZERO		0x0800	/* zero memory in dmamem_alloc */
+#define	BUS_DMA_NOCACHE		0x1000
 
 /* Forwards needed by prototypes below. */
 struct mbuf;
@@ -475,7 +476,7 @@ int	_dmamem_map(bus_dma_tag_t, bus_dma_segment_t *,
 void	_dmamem_unmap(bus_dma_tag_t, caddr_t, size_t);
 paddr_t	_dmamem_mmap(bus_dma_tag_t, bus_dma_segment_t *, int, off_t, int, int);
 int	_dmamem_alloc_range(bus_dma_tag_t, bus_size_t, bus_size_t, bus_size_t,
-	    bus_dma_segment_t *, int, int *, int, vaddr_t, vaddr_t);
+	    bus_dma_segment_t *, int, int *, int, paddr_t, paddr_t);
 
 /*
  *	bus_dmamap_t
diff --git a/sys/arch/loongson/loongson/bus_dma.c b/sys/arch/loongson/loongson/bus_dma.c
index ab3c742a873..b73d3a0569f 100644
--- a/sys/arch/loongson/loongson/bus_dma.c
+++ b/sys/arch/loongson/loongson/bus_dma.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: bus_dma.c,v 1.11 2012/10/03 21:44:51 miod Exp $ */
+/*	$OpenBSD: bus_dma.c,v 1.12 2014/03/10 21:32:15 miod Exp $ */
 
 /*
  * Copyright (c) 2003-2004 Opsycon AB  (www.opsycon.se / www.opsycon.com)
@@ -388,7 +388,7 @@ _dmamem_alloc(bus_dma_tag_t t, bus_size_t size, bus_size_t alignment,
     int flags)
 {
 	return _dmamem_alloc_range(t, size, alignment, boundary,
-	    segs, nsegs, rsegs, flags, (vaddr_t)0, (vaddr_t)-1);
+	    segs, nsegs, rsegs, flags, (paddr_t)0, (paddr_t)-1);
 }
 
 /*
@@ -431,11 +431,11 @@ _dmamem_map(bus_dma_tag_t t, bus_dma_segment_t *segs, int nsegs, size_t size,
 	size_t ssize;
 	paddr_t pa;
 	bus_addr_t addr;
-	int curseg, error;
+	int curseg, error, pmap_flags;
 
 	if (nsegs == 1) {
 		pa = (*t->_device_to_pa)(segs[0].ds_addr);
-		if (flags & BUS_DMA_COHERENT)
+		if (flags & (BUS_DMA_COHERENT | BUS_DMA_NOCACHE))
 			*kvap = (caddr_t)PHYS_TO_XKPHYS(pa, CCA_NC);
 		else
 			*kvap = (caddr_t)PHYS_TO_XKPHYS(pa, CCA_CACHED);
@@ -451,6 +451,9 @@ _dmamem_map(bus_dma_tag_t t, bus_dma_segment_t *segs, int nsegs, size_t size,
 
 	sva = va;
 	ssize = size;
+	pmap_flags = PMAP_WIRED | PMAP_CANFAIL;
+	if (flags & (BUS_DMA_COHERENT | BUS_DMA_NOCACHE))
+		pmap_flags |= PMAP_NOCACHE;
 	for (curseg = 0; curseg < nsegs; curseg++) {
 		for (addr = segs[curseg].ds_addr;
 		    addr < (segs[curseg].ds_addr + segs[curseg].ds_len);
@@ -460,14 +463,22 @@ _dmamem_map(bus_dma_tag_t t, bus_dma_segment_t *segs, int nsegs, size_t size,
 			pa = (*t->_device_to_pa)(addr);
 			error = pmap_enter(pmap_kernel(), va, pa,
 			    VM_PROT_READ | VM_PROT_WRITE, VM_PROT_READ |
-			    VM_PROT_WRITE | PMAP_WIRED | PMAP_CANFAIL);
+			    VM_PROT_WRITE | pmap_flags);
 			if (error) {
 				pmap_update(pmap_kernel());
 				uvm_km_free(kernel_map, sva, ssize);
 				return (error);
 			}
 
-			if (flags & BUS_DMA_COHERENT)
+			/*
+			 * This is redundant with what pmap_enter() did 
+			 * above, but will take care of forcing other
+			 * mappings of the same page (if any) to be
+			 * uncached. 
+			 * If there are no multiple mappings of that 
+			 * page, this amounts to a noop.
+			 */
+			if (flags & (BUS_DMA_COHERENT | BUS_DMA_NOCACHE)) 
 				pmap_page_cache(PHYS_TO_VM_PAGE(pa),
 				    PV_UNCACHED);
 		}
@@ -634,9 +645,9 @@ _dmamap_load_buffer(bus_dma_tag_t t, bus_dmamap_t map, void *buf,
 int
 _dmamem_alloc_range(bus_dma_tag_t t, bus_size_t size, bus_size_t alignment,
     bus_size_t boundary, bus_dma_segment_t *segs, int nsegs, int *rsegs,
-    int flags, vaddr_t low, vaddr_t high)
+    int flags, paddr_t low, paddr_t high)
 {
-	vaddr_t curaddr, lastaddr;
+	paddr_t curaddr, lastaddr;
 	vm_page_t m;
 	struct pglist mlist;
 	int curseg, error, plaflag;
diff --git a/sys/arch/octeon/include/bus.h b/sys/arch/octeon/include/bus.h
index 79c54f1d324..ceacab49c62 100644
--- a/sys/arch/octeon/include/bus.h
+++ b/sys/arch/octeon/include/bus.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: bus.h,v 1.4 2010/12/04 16:46:00 miod Exp $	*/
+/*	$OpenBSD: bus.h,v 1.5 2014/03/10 21:32:15 miod Exp $	*/
 
 /*
  * Copyright (c) 2003-2004 Opsycon AB Sweden.  All rights reserved.
@@ -303,18 +303,19 @@ bus_space_barrier(bus_space_tag_t t, bus_space_handle_t h, bus_size_t offset,
 #define BUS_SPACE_BARRIER_READ  0x01		/* force read barrier */
 #define BUS_SPACE_BARRIER_WRITE 0x02		/* force write barrier */
 
-#define	BUS_DMA_WAITOK		0x000
-#define	BUS_DMA_NOWAIT		0x001
-#define	BUS_DMA_ALLOCNOW	0x002
-#define	BUS_DMA_COHERENT	0x008
-#define	BUS_DMA_BUS1		0x010	/* placeholders for bus functions... */
-#define	BUS_DMA_BUS2		0x020
-#define	BUS_DMA_BUS3		0x040
-#define	BUS_DMA_BUS4		0x080
-#define	BUS_DMA_READ		0x100   /* mapping is device -> memory only */
-#define	BUS_DMA_WRITE		0x200   /* mapping is memory -> device only */
-#define	BUS_DMA_STREAMING	0x400   /* hint: sequential, unidirectional */
-#define	BUS_DMA_ZERO		0x800	/* zero memory in dmamem_alloc */
+#define	BUS_DMA_WAITOK		0x0000
+#define	BUS_DMA_NOWAIT		0x0001
+#define	BUS_DMA_ALLOCNOW	0x0002
+#define	BUS_DMA_COHERENT	0x0008
+#define	BUS_DMA_BUS1		0x0010	/* placeholders for bus functions... */
+#define	BUS_DMA_BUS2		0x0020
+#define	BUS_DMA_BUS3		0x0040
+#define	BUS_DMA_BUS4		0x0080
+#define	BUS_DMA_READ		0x0100	/* mapping is device -> memory only */
+#define	BUS_DMA_WRITE		0x0200	/* mapping is memory -> device only */
+#define	BUS_DMA_STREAMING	0x0400	/* hint: sequential, unidirectional */
+#define	BUS_DMA_ZERO		0x0800	/* zero memory in dmamem_alloc */
+#define	BUS_DMA_NOCACHE		0x1000
 
 /* Forwards needed by prototypes below. */
 struct mbuf;
@@ -446,7 +447,7 @@ int	_dmamem_map(bus_dma_tag_t, bus_dma_segment_t *,
 void	_dmamem_unmap(bus_dma_tag_t, caddr_t, size_t);
 paddr_t	_dmamem_mmap(bus_dma_tag_t, bus_dma_segment_t *, int, off_t, int, int);
 int	_dmamem_alloc_range(bus_dma_tag_t, bus_size_t, bus_size_t, bus_size_t,
-	    bus_dma_segment_t *, int, int *, int, vaddr_t, vaddr_t);
+	    bus_dma_segment_t *, int, int *, int, paddr_t, paddr_t);
 
 /*
  *	bus_dmamap_t
diff --git a/sys/arch/octeon/octeon/bus_dma.c b/sys/arch/octeon/octeon/bus_dma.c
index 35d45c686cf..28728a02f92 100644
--- a/sys/arch/octeon/octeon/bus_dma.c
+++ b/sys/arch/octeon/octeon/bus_dma.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: bus_dma.c,v 1.7 2012/10/03 19:42:54 miod Exp $ */
+/*	$OpenBSD: bus_dma.c,v 1.8 2014/03/10 21:32:15 miod Exp $ */
 
 /*
  * Copyright (c) 2003-2004 Opsycon AB  (www.opsycon.se / www.opsycon.com)
@@ -381,7 +381,7 @@ _dmamem_alloc(bus_dma_tag_t t, bus_size_t size, bus_size_t alignment,
     int flags)
 {
 	return _dmamem_alloc_range(t, size, alignment, boundary,
-	    segs, nsegs, rsegs, flags, (vaddr_t)0, (vaddr_t)-1);
+	    segs, nsegs, rsegs, flags, (paddr_t)0, (paddr_t)-1);
 }
 
 /*
@@ -424,11 +424,11 @@ _dmamem_map(bus_dma_tag_t t, bus_dma_segment_t *segs, int nsegs, size_t size,
 	size_t ssize;
 	paddr_t pa;
 	bus_addr_t addr;
-	int curseg, error;
+	int curseg, error, pmap_flags;
 
 	if (nsegs == 1) {
 		pa = (*t->_device_to_pa)(segs[0].ds_addr);
-		if (flags & BUS_DMA_COHERENT)
+		if (flags & (BUS_DMA_COHERENT | BUS_DMA_NOCACHE))
 			*kvap = (caddr_t)PHYS_TO_XKPHYS(pa, CCA_NC);
 		else
 			*kvap = (caddr_t)PHYS_TO_XKPHYS(pa, CCA_CACHED);
@@ -444,6 +444,9 @@ _dmamem_map(bus_dma_tag_t t, bus_dma_segment_t *segs, int nsegs, size_t size,
 
 	sva = va;
 	ssize = size;
+	pmap_flags = PMAP_WIRED | PMAP_CANFAIL;
+	if (flags & (BUS_DMA_COHERENT | BUS_DMA_NOCACHE))
+		pmap_flags |= PMAP_NOCACHE;
 	for (curseg = 0; curseg < nsegs; curseg++) {
 		for (addr = segs[curseg].ds_addr;
 		    addr < (segs[curseg].ds_addr + segs[curseg].ds_len);
@@ -453,14 +456,22 @@ _dmamem_map(bus_dma_tag_t t, bus_dma_segment_t *segs, int nsegs, size_t size,
 			pa = (*t->_device_to_pa)(addr);
 			error = pmap_enter(pmap_kernel(), va, pa,
 			    VM_PROT_READ | VM_PROT_WRITE, VM_PROT_READ |
-			    VM_PROT_WRITE | PMAP_WIRED | PMAP_CANFAIL);
+			    VM_PROT_WRITE | pmap_flags);
 			if (error) {
 				pmap_update(pmap_kernel());
 				uvm_km_free(kernel_map, sva, ssize);
 				return (error);
 			}
 
-			if (flags & BUS_DMA_COHERENT)
+			/*
+			 * This is redundant with what pmap_enter() did
+			 * above, but will take care of forcing other
+			 * mappings of the same page (if any) to be
+			 * uncached.
+			 * If there are no multiple mappings of that
+			 * page, this amounts to a noop.
+			 */
+			if (flags & (BUS_DMA_COHERENT | BUS_DMA_NOCACHE))
 				pmap_page_cache(PHYS_TO_VM_PAGE(pa),
 				    PV_UNCACHED);
 		}
@@ -627,9 +638,9 @@ _dmamap_load_buffer(bus_dma_tag_t t, bus_dmamap_t map, void *buf,
 int
 _dmamem_alloc_range(bus_dma_tag_t t, bus_size_t size, bus_size_t alignment,
     bus_size_t boundary, bus_dma_segment_t *segs, int nsegs, int *rsegs,
-    int flags, vaddr_t low, vaddr_t high)
+    int flags, paddr_t low, paddr_t high)
 {
-	vaddr_t curaddr, lastaddr;
+	paddr_t curaddr, lastaddr;
 	vm_page_t m;
 	struct pglist mlist;
 	int curseg, error, plaflag;
diff --git a/sys/arch/sgi/include/bus.h b/sys/arch/sgi/include/bus.h
index f43ef116773..7f2a067c2ab 100644
--- a/sys/arch/sgi/include/bus.h
+++ b/sys/arch/sgi/include/bus.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: bus.h,v 1.25 2012/03/15 18:52:55 miod Exp $	*/
+/*	$OpenBSD: bus.h,v 1.26 2014/03/10 21:32:15 miod Exp $	*/
 
 /*
  * Copyright (c) 2003-2004 Opsycon AB Sweden.  All rights reserved.
@@ -209,7 +209,7 @@ bus_space_write_region(8,64)
 static __inline void							      \
 CAT(bus_space_write_raw_region_,n)(bus_space_tag_t bst,			      \
      bus_space_handle_t bsh,						      \
-     bus_addr_t ba, const u_int8_t *x, size_t cnt)		              \
+     bus_addr_t ba, const u_int8_t *x, size_t cnt)			      \
 {									      \
 	cnt >>= ((n) >> 1);						      \
 	while (cnt--) {							      \
@@ -305,18 +305,19 @@ bus_space_barrier(bus_space_tag_t t, bus_space_handle_t h, bus_size_t offs,
 #define BUS_SPACE_BARRIER_READ  0x01		/* force read barrier */
 #define BUS_SPACE_BARRIER_WRITE 0x02		/* force write barrier */
 
-#define	BUS_DMA_WAITOK		0x000
-#define	BUS_DMA_NOWAIT		0x001
-#define	BUS_DMA_ALLOCNOW	0x002
-#define	BUS_DMA_COHERENT	0x008
-#define	BUS_DMA_BUS1		0x010	/* placeholders for bus functions... */
-#define	BUS_DMA_BUS2		0x020
-#define	BUS_DMA_BUS3		0x040
-#define	BUS_DMA_BUS4		0x080
-#define	BUS_DMA_READ		0x100   /* mapping is device -> memory only */
-#define	BUS_DMA_WRITE		0x200   /* mapping is memory -> device only */
-#define	BUS_DMA_STREAMING	0x400   /* hint: sequential, unidirectional */
-#define	BUS_DMA_ZERO		0x800	/* zero memory in dmamem_alloc */
+#define	BUS_DMA_WAITOK		0x0000
+#define	BUS_DMA_NOWAIT		0x0001
+#define	BUS_DMA_ALLOCNOW	0x0002
+#define	BUS_DMA_COHERENT	0x0008
+#define	BUS_DMA_BUS1		0x0010	/* placeholders for bus functions... */
+#define	BUS_DMA_BUS2		0x0020
+#define	BUS_DMA_BUS3		0x0040
+#define	BUS_DMA_BUS4		0x0080
+#define	BUS_DMA_READ		0x0100	/* mapping is device -> memory only */
+#define	BUS_DMA_WRITE		0x0200	/* mapping is memory -> device only */
+#define	BUS_DMA_STREAMING	0x0400	/* hint: sequential, unidirectional */
+#define	BUS_DMA_ZERO		0x0800	/* zero memory in dmamem_alloc */
+#define	BUS_DMA_NOCACHE		0x1000
 
 /* Forwards needed by prototypes below. */
 struct mbuf;
@@ -448,7 +449,7 @@ int	_dmamem_map(bus_dma_tag_t, bus_dma_segment_t *,
 void	_dmamem_unmap(bus_dma_tag_t, caddr_t, size_t);
 paddr_t	_dmamem_mmap(bus_dma_tag_t, bus_dma_segment_t *, int, off_t, int, int);
 int	_dmamem_alloc_range(bus_dma_tag_t, bus_size_t, bus_size_t, bus_size_t,
-	    bus_dma_segment_t *, int, int *, int, vaddr_t, vaddr_t);
+	    bus_dma_segment_t *, int, int *, int, paddr_t, paddr_t);
 
 /*
  *	bus_dmamap_t
diff --git a/sys/arch/sgi/sgi/bus_dma.c b/sys/arch/sgi/sgi/bus_dma.c
index 7419d278b1f..2565e2c7067 100644
--- a/sys/arch/sgi/sgi/bus_dma.c
+++ b/sys/arch/sgi/sgi/bus_dma.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: bus_dma.c,v 1.30 2012/10/03 22:46:09 miod Exp $ */
+/*	$OpenBSD: bus_dma.c,v 1.31 2014/03/10 21:32:15 miod Exp $ */
 
 /*
  * Copyright (c) 2003-2004 Opsycon AB  (www.opsycon.se / www.opsycon.com)
@@ -345,10 +345,12 @@ _dmamap_sync(bus_dma_tag_t t, bus_dmamap_t map, bus_addr_t addr,
 		if (ssize > size)
 			ssize = size;
 
+#ifndef TGT_COHERENT
 		if (IS_XKPHYS(vaddr) && XKPHYS_TO_CCA(vaddr) == CCA_NC) {
 			size -= ssize;
 			ssize = 0;
 		}
+#endif
 
 		if (ssize != 0) {
 #ifdef TGT_COHERENT
@@ -440,7 +442,7 @@ _dmamem_map(bus_dma_tag_t t, bus_dma_segment_t *segs, int nsegs, size_t size,
 	size_t ssize;
 	paddr_t pa;
 	bus_addr_t addr;
-	int curseg, error;
+	int curseg, error, pmap_flags;
 
 #if defined(TGT_INDIGO2)
 	/*
@@ -454,13 +456,16 @@ _dmamem_map(bus_dma_tag_t t, bus_dma_segment_t *segs, int nsegs, size_t size,
 		return EINVAL;
 #endif
 
+#ifdef TGT_COHERENT
+	/* coherent mappings do not need to be uncached on these platforms */
+	flags &= ~BUS_DMA_COHERENT;
+#endif
+
 	if (nsegs == 1) {
 		pa = (*t->_device_to_pa)(segs[0].ds_addr);
-#ifndef TGT_COHERENT
-		if (flags & BUS_DMA_COHERENT)
+		if (flags & (BUS_DMA_COHERENT | BUS_DMA_NOCACHE))
 			*kvap = (caddr_t)PHYS_TO_XKPHYS(pa, CCA_NC);
 		else
-#endif
 			*kvap = (caddr_t)PHYS_TO_XKPHYS(pa, CCA_CACHED);
 		return (0);
 	}
@@ -474,6 +479,9 @@ _dmamem_map(bus_dma_tag_t t, bus_dma_segment_t *segs, int nsegs, size_t size,
 
 	sva = va;
 	ssize = size;
+	pmap_flags = PMAP_WIRED | PMAP_CANFAIL;
+	if (flags & (BUS_DMA_COHERENT | BUS_DMA_NOCACHE))
+		pmap_flags |= PMAP_NOCACHE;
 	for (curseg = 0; curseg < nsegs; curseg++) {
 		for (addr = segs[curseg].ds_addr;
 		    addr < (segs[curseg].ds_addr + segs[curseg].ds_len);
@@ -485,18 +493,24 @@ _dmamem_map(bus_dma_tag_t t, bus_dma_segment_t *segs, int nsegs, size_t size,
 			pa = (*t->_device_to_pa)(addr);
 			error = pmap_enter(pmap_kernel(), va, pa,
 			    VM_PROT_READ | VM_PROT_WRITE, VM_PROT_READ |
-			    VM_PROT_WRITE | PMAP_WIRED | PMAP_CANFAIL);
+			    VM_PROT_WRITE | pmap_flags);
 			if (error) {
 				pmap_update(pmap_kernel());
 				uvm_km_free(kernel_map, sva, ssize);
 				return (error);
 			}
 
-#ifndef TGT_COHERENT
-			if (flags & BUS_DMA_COHERENT)
+			/*
+			 * This is redundant with what pmap_enter() did
+			 * above, but will take care of forcing other
+			 * mappings of the same page (if any) to be
+			 * uncached.
+			 * If there are no multiple mappings of that
+			 * page, this amounts to a noop.
+			 */
+			if (flags & (BUS_DMA_COHERENT | BUS_DMA_NOCACHE))
 				pmap_page_cache(PHYS_TO_VM_PAGE(pa),
 				    PV_UNCACHED);
-#endif
 		}
 		pmap_update(pmap_kernel());
 	}
@@ -670,7 +684,7 @@ _dmamem_alloc_range(bus_dma_tag_t t, bus_size_t size, bus_size_t alignment,
     bus_size_t boundary, bus_dma_segment_t *segs, int nsegs, int *rsegs,
     int flags, paddr_t low, paddr_t high)
 {
-	vaddr_t curaddr, lastaddr;
+	paddr_t curaddr, lastaddr;
 	vm_page_t m;
 	struct pglist mlist;
 	int curseg, error, plaflag;