compiles with minimal mods (From NetBSD)

1 files changed, 991 insertions, 0 deletions

diff --git a/sys/arch/sparc64/dev/iommu.c b/sys/arch/sparc64/dev/iommu.c
new file mode 100644
index 00000000000..1943897a26a
--- /dev/null
+++ b/sys/arch/sparc64/dev/iommu.c
@@ -0,0 +1,991 @@
+/*	$NetBSD: iommu.c,v 1.37 2001/08/06 22:02:58 eeh Exp $	*/
+
+/*
+ * Copyright (c) 1999, 2000 Matthew R. Green
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. The name of the author may not be used to endorse or promote products
+ *    derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+ * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+/*-
+ * Copyright (c) 1998 The NetBSD Foundation, Inc.
+ * All rights reserved.
+ *
+ * This code is derived from software contributed to The NetBSD Foundation
+ * by Paul Kranenburg.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. All advertising materials mentioning features or use of this software
+ *    must display the following acknowledgement:
+ *        This product includes software developed by the NetBSD
+ *        Foundation, Inc. and its contributors.
+ * 4. Neither the name of The NetBSD Foundation nor the names of its
+ *    contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
+ * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/*
+ * Copyright (c) 1992, 1993
+ *	The Regents of the University of California.  All rights reserved.
+ *
+ * This software was developed by the Computer Systems Engineering group
+ * at Lawrence Berkeley Laboratory under DARPA contract BG 91-66 and
+ * contributed to Berkeley.
+ *
+ * All advertising materials mentioning features or use of this software
+ * must display the following acknowledgement:
+ *	This product includes software developed by the University of
+ *	California, Lawrence Berkeley Laboratory.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. All advertising materials mentioning features or use of this software
+ *    must display the following acknowledgement:
+ *	This product includes software developed by the University of
+ *	California, Berkeley and its contributors.
+ * 4. Neither the name of the University nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ *	from: NetBSD: sbus.c,v 1.13 1999/05/23 07:24:02 mrg Exp
+ *	from: @(#)sbus.c	8.1 (Berkeley) 6/11/93
+ */
+
+/*
+ * UltraSPARC IOMMU support; used by both the sbus and pci code.
+ */
+#include <sys/param.h>
+#include <sys/extent.h>
+#include <sys/malloc.h>
+#include <sys/systm.h>
+#include <sys/device.h>
+
+#include <vm/vm.h>
+#include <vm/vm_kern.h>
+#include <uvm/uvm_extern.h>
+
+#include <machine/bus.h>
+#include <sparc64/sparc64/cache.h>
+#include <sparc64/dev/iommureg.h>
+#include <sparc64/dev/iommuvar.h>
+
+#include <machine/autoconf.h>
+#include <machine/cpu.h>
+
+#ifdef DDB
+#include <machine/db_machdep.h>
+#include <ddb/db_sym.h>
+#include <ddb/db_extern.h>
+#endif
+
+#ifdef DEBUG
+#define IDB_BUSDMA	0x1
+#define IDB_IOMMU	0x2
+#define IDB_INFO	0x4
+#define	IDB_SYNC	0x8
+int iommudebug = 0x0;
+#define DPRINTF(l, s)   do { if (iommudebug & l) printf s; } while (0)
+#else
+#define DPRINTF(l, s)
+#endif
+
+#define iommu_strbuf_flush(i,v) bus_space_write_8((i)->is_bustag, \
+	(bus_space_handle_t)(u_long)&(i)->is_sb->strbuf_pgflush, 0, (v))
+static	int iommu_strbuf_flush_done __P((struct iommu_state *));
+
+/*
+ * initialise the UltraSPARC IOMMU (SBUS or PCI):
+ *	- allocate and setup the iotsb.
+ *	- enable the IOMMU
+ *	- initialise the streaming buffers (if they exist)
+ *	- create a private DVMA map.
+ */
+void
+iommu_init(name, is, tsbsize, iovabase)
+	char *name;
+	struct iommu_state *is;
+	int tsbsize;
+	u_int32_t iovabase;
+{
+	psize_t size;
+	vaddr_t va;
+	paddr_t pa;
+	struct vm_page *m;
+	struct pglist mlist;
+
+	/*
+	 * Setup the iommu.
+	 *
+	 * The sun4u iommu is part of the SBUS or PCI controller so we
+	 * will deal with it here..
+	 *
+	 * The IOMMU address space always ends at 0xffffe000, but the starting
+	 * address depends on the size of the map.  The map size is 1024 * 2 ^
+	 * is->is_tsbsize entries, where each entry is 8 bytes.  The start of
+	 * the map can be calculated by (0xffffe000 << (8 + is->is_tsbsize)).
+	 */
+	is->is_cr = (tsbsize << 16) | IOMMUCR_EN;
+	is->is_tsbsize = tsbsize;
+	is->is_dvmabase = iovabase;
+	if (iovabase == -1) is->is_dvmabase = IOTSB_VSTART(is->is_tsbsize);
+
+	/*
+	 * Allocate memory for I/O pagetables.  They need to be physically
+	 * contiguous.
+	 */
+
+	size = NBPG<<(is->is_tsbsize);
+	TAILQ_INIT(&mlist);
+	if (uvm_pglistalloc((psize_t)size, (paddr_t)0, (paddr_t)-1,
+		(paddr_t)NBPG, (paddr_t)0, &mlist, 1, 0) != 0)
+		panic("iommu_init: no memory");
+
+	va = uvm_km_valloc(kernel_map, size);
+	if (va == 0)
+		panic("iommu_init: no memory");
+	is->is_tsb = (int64_t *)va;
+
+	m = TAILQ_FIRST(&mlist);
+	is->is_ptsb = VM_PAGE_TO_PHYS(m);
+
+	/* Map the pages */
+	for (; m != NULL; m = TAILQ_NEXT(m,pageq)) {
+		pa = VM_PAGE_TO_PHYS(m);
+		pmap_enter(pmap_kernel(), va, pa | PMAP_NVC,
+			VM_PROT_READ|VM_PROT_WRITE,
+			VM_PROT_READ|VM_PROT_WRITE|PMAP_WIRED);
+		va += NBPG;
+	}
+	pmap_update();
+	bzero(is->is_tsb, size);
+
+#ifdef DEBUG
+	if (iommudebug & IDB_INFO)
+	{
+		/* Probe the iommu */
+		struct iommureg *regs = is->is_iommu;
+
+		printf("iommu regs at: cr=%lx tsb=%lx flush=%lx\n",
+		    (u_long)&regs->iommu_cr,
+		    (u_long)&regs->iommu_tsb,
+		    (u_long)&regs->iommu_flush);
+		printf("iommu cr=%llx tsb=%llx\n", (unsigned long long)regs->iommu_cr, (unsigned long long)regs->iommu_tsb);
+		printf("TSB base %p phys %llx\n", (void *)is->is_tsb, (unsigned long long)is->is_ptsb);
+		delay(1000000); /* 1 s */
+	}
+#endif
+
+	/*
+	 * Initialize streaming buffer, if it is there.
+	 */
+	if (is->is_sb)
+		(void)pmap_extract(pmap_kernel(), (vaddr_t)&is->is_flush,
+		    (paddr_t *)&is->is_flushpa);
+
+	/*
+	 * now actually start up the IOMMU
+	 */
+	iommu_reset(is);
+
+	/*
+	 * Now all the hardware's working we need to allocate a dvma map.
+	 */
+	printf("DVMA map: %x to %x\n", 
+		(unsigned int)is->is_dvmabase,
+		(unsigned int)(is->is_dvmabase+(size<<10)));
+	is->is_dvmamap = extent_create(name,
+				       is->is_dvmabase, (u_long)IOTSB_VEND,
+				       M_DEVBUF, 0, 0, EX_NOWAIT);
+}
+
+/*
+ * Streaming buffers don't exist on the UltraSPARC IIi; we should have
+ * detected that already and disabled them.  If not, we will notice that
+ * they aren't there when the STRBUF_EN bit does not remain.
+ */
+void
+iommu_reset(is)
+	struct iommu_state *is;
+{
+
+	/* Need to do 64-bit stores */
+	bus_space_write_8(is->is_bustag, 
+			  (bus_space_handle_t)(u_long)&is->is_iommu->iommu_tsb, 
+			  0, is->is_ptsb);
+	/* Enable IOMMU in diagnostic mode */
+	bus_space_write_8(is->is_bustag, 
+			  (bus_space_handle_t)(u_long)&is->is_iommu->iommu_cr, 0, 
+			  is->is_cr|IOMMUCR_DE);
+
+
+	if (!is->is_sb)
+		return;
+
+	/* Enable diagnostics mode? */
+	bus_space_write_8(is->is_bustag, 
+			  (bus_space_handle_t)(u_long)&is->is_sb->strbuf_ctl,
+			  0, STRBUF_EN);
+
+	/* No streaming buffers? Disable them */
+	if (bus_space_read_8(is->is_bustag,
+			     (bus_space_handle_t)(u_long)&is->is_sb->strbuf_ctl, 
+			     0) == 0)
+		is->is_sb = 0;
+}
+
+/*
+ * Here are the iommu control routines. 
+ */
+void
+iommu_enter(is, va, pa, flags)
+	struct iommu_state *is;
+	vaddr_t va;
+	int64_t pa;
+	int flags;
+{
+	int64_t tte;
+
+#ifdef DIAGNOSTIC
+	if (va < is->is_dvmabase)
+		panic("iommu_enter: va %#lx not in DVMA space", va);
+#endif
+
+	tte = MAKEIOTTE(pa, !(flags&BUS_DMA_NOWRITE), !(flags&BUS_DMA_NOCACHE), 
+			(flags&BUS_DMA_STREAMING));
+	
+	/* Is the streamcache flush really needed? */
+	if (is->is_sb) {
+		iommu_strbuf_flush(is, va);
+		iommu_strbuf_flush_done(is);
+	}
+	DPRINTF(IDB_IOMMU, ("Clearing TSB slot %d for va %p\n", 
+		       (int)IOTSBSLOT(va,is->is_tsbsize), (void *)(u_long)va));
+	is->is_tsb[IOTSBSLOT(va,is->is_tsbsize)] = tte;
+	bus_space_write_8(is->is_bustag, (bus_space_handle_t)(u_long)
+			  &is->is_iommu->iommu_flush, 0, va);
+	DPRINTF(IDB_IOMMU, ("iommu_enter: va %lx pa %lx TSB[%lx]@%p=%lx\n",
+		       va, (long)pa, (u_long)IOTSBSLOT(va,is->is_tsbsize), 
+		       (void *)(u_long)&is->is_tsb[IOTSBSLOT(va,is->is_tsbsize)],
+		       (u_long)tte));
+}
+
+/*
+ * iommu_remove: removes mappings created by iommu_enter
+ *
+ * Only demap from IOMMU if flag is set.
+ *
+ * XXX: this function needs better internal error checking.
+ */
+void
+iommu_remove(is, va, len)
+	struct iommu_state *is;
+	vaddr_t va;
+	size_t len;
+{
+
+#ifdef DIAGNOSTIC
+	if (va < is->is_dvmabase)
+		panic("iommu_remove: va 0x%lx not in DVMA space", (u_long)va);
+	if ((long)(va + len) < (long)va)
+		panic("iommu_remove: va 0x%lx + len 0x%lx wraps", 
+		      (long) va, (long) len);
+	if (len & ~0xfffffff) 
+		panic("iommu_remove: rediculous len 0x%lx", (u_long)len);
+#endif
+
+	va = trunc_page(va);
+	DPRINTF(IDB_IOMMU, ("iommu_remove: va %lx TSB[%lx]@%p\n",
+	    va, (u_long)IOTSBSLOT(va,is->is_tsbsize), 
+	    &is->is_tsb[IOTSBSLOT(va,is->is_tsbsize)]));
+	while (len > 0) {
+		DPRINTF(IDB_IOMMU, ("iommu_remove: clearing TSB slot %d for va %p size %lx\n", 
+		    (int)IOTSBSLOT(va,is->is_tsbsize), (void *)(u_long)va, (u_long)len));
+		if (is->is_sb) {
+			DPRINTF(IDB_IOMMU, ("iommu_remove: flushing va %p TSB[%lx]@%p=%lx, %lu bytes left\n", 	       
+			       (void *)(u_long)va, (long)IOTSBSLOT(va,is->is_tsbsize), 
+			       (void *)(u_long)&is->is_tsb[IOTSBSLOT(va,is->is_tsbsize)],
+			       (long)(is->is_tsb[IOTSBSLOT(va,is->is_tsbsize)]), 
+			       (u_long)len));
+			iommu_strbuf_flush(is, va);
+			if (len <= NBPG)
+				iommu_strbuf_flush_done(is);
+			DPRINTF(IDB_IOMMU, ("iommu_remove: flushed va %p TSB[%lx]@%p=%lx, %lu bytes left\n", 	       
+			       (void *)(u_long)va, (long)IOTSBSLOT(va,is->is_tsbsize), 
+			       (void *)(u_long)&is->is_tsb[IOTSBSLOT(va,is->is_tsbsize)],
+			       (long)(is->is_tsb[IOTSBSLOT(va,is->is_tsbsize)]), 
+			       (u_long)len));
+		}
+
+		if (len <= NBPG)
+			len = 0;
+		else
+			len -= NBPG;
+
+		is->is_tsb[IOTSBSLOT(va,is->is_tsbsize)] = 0;
+		bus_space_write_8(is->is_bustag, (bus_space_handle_t)(u_long)
+				  &is->is_iommu->iommu_flush, 0, va);
+		va += NBPG;
+	}
+}
+
+static int 
+iommu_strbuf_flush_done(is)
+	struct iommu_state *is;
+{
+	struct timeval cur, flushtimeout;
+
+#define BUMPTIME(t, usec) { \
+	register volatile struct timeval *tp = (t); \
+	register long us; \
+ \
+	tp->tv_usec = us = tp->tv_usec + (usec); \
+	if (us >= 1000000) { \
+		tp->tv_usec = us - 1000000; \
+		tp->tv_sec++; \
+	} \
+}
+
+	if (!is->is_sb)
+		return (0);
+				
+	/*
+	 * Streaming buffer flushes:
+	 * 
+	 *   1 Tell strbuf to flush by storing va to strbuf_pgflush.  If
+	 *     we're not on a cache line boundary (64-bits):
+	 *   2 Store 0 in flag
+	 *   3 Store pointer to flag in flushsync
+	 *   4 wait till flushsync becomes 0x1
+	 *
+	 * If it takes more than .5 sec, something
+	 * went wrong.
+	 */
+
+	is->is_flush = 0;
+	membar_sync();	/* #StoreStore is prolly enuf. */
+	bus_space_write_8(is->is_bustag, (bus_space_handle_t)(u_long)
+			  &is->is_sb->strbuf_flushsync, 0, is->is_flushpa);
+	membar_sync();	/* Prolly not needed at all. */
+
+	microtime(&flushtimeout); 
+	cur = flushtimeout;
+	BUMPTIME(&flushtimeout, 500000); /* 1/2 sec */
+	
+	DPRINTF(IDB_IOMMU, ("iommu_strbuf_flush_done: flush = %lx at va = %lx pa = %lx now=%lx:%lx until = %lx:%lx\n", 
+		       (long)is->is_flush, (long)&is->is_flush, 
+		       (long)is->is_flushpa, cur.tv_sec, cur.tv_usec, 
+		       flushtimeout.tv_sec, flushtimeout.tv_usec));
+	/* Bypass non-coherent D$ */
+	while (!ldxa(is->is_flushpa, ASI_PHYS_CACHED) && 
+	       ((cur.tv_sec <= flushtimeout.tv_sec) && 
+		(cur.tv_usec <= flushtimeout.tv_usec)))
+		microtime(&cur);
+
+#ifdef DIAGNOSTIC
+	if (!ldxa(is->is_flushpa, ASI_PHYS_CACHED)) {
+		printf("iommu_strbuf_flush_done: flush timeout %p at %p\n",
+		    (void *)(u_long)is->is_flush, 
+		    (void *)(u_long)is->is_flushpa); /* panic? */
+#ifdef DDB
+		Debugger();
+#endif
+	}
+#endif
+	DPRINTF(IDB_IOMMU, ("iommu_strbuf_flush_done: flushed\n"));
+	return (is->is_flush);
+}
+
+/*
+ * IOMMU DVMA operations, common to SBUS and PCI.
+ */
+int
+iommu_dvmamap_load(t, is, map, buf, buflen, p, flags)
+	bus_dma_tag_t t;
+	struct iommu_state *is;
+	bus_dmamap_t map;
+	void *buf;
+	bus_size_t buflen;
+	struct proc *p;
+	int flags;
+{
+	int s;
+	int err;
+	bus_size_t sgsize;
+	paddr_t curaddr;
+	u_long dvmaddr;
+	bus_size_t align, boundary;
+	vaddr_t vaddr = (vaddr_t)buf;
+	pmap_t pmap;
+
+	if (map->dm_nsegs) {
+		/* Already in use?? */
+#ifdef DIAGNOSTIC
+		printf("iommu_dvmamap_load: map still in use\n");
+#endif
+		bus_dmamap_unload(t, map);
+	}
+	/*
+	 * Make sure that on error condition we return "no valid mappings".
+	 */
+	map->dm_nsegs = 0;
+
+	if (buflen > map->_dm_size) {
+		DPRINTF(IDB_BUSDMA,
+		    ("iommu_dvmamap_load(): error %d > %d -- "
+		     "map size exceeded!\n", (int)buflen, (int)map->_dm_size));
+		return (EINVAL);
+	}
+
+	sgsize = round_page(buflen + ((int)vaddr & PGOFSET));
+
+	/*
+	 * A boundary presented to bus_dmamem_alloc() takes precedence
+	 * over boundary in the map.
+	 */
+	if ((boundary = (map->dm_segs[0]._ds_boundary)) == 0)
+		boundary = map->_dm_boundary;
+	align = max(map->dm_segs[0]._ds_align, NBPG);
+	s = splhigh();
+	err = extent_alloc(is->is_dvmamap, sgsize, align, 0,
+	    boundary, EX_NOWAIT|EX_BOUNDZERO, (u_long *)&dvmaddr);
+	splx(s);
+
+#ifdef DEBUG
+	if (err || (dvmaddr == (bus_addr_t)-1))	
+	{ 
+		printf("iommu_dvmamap_load(): extent_alloc(%d, %x) failed!\n",
+		    (int)sgsize, flags);
+#ifdef DDB
+		Debugger();
+#else
+		panic("");
+#endif
+	}		
+#endif	
+	if (err != 0)
+		return (err);
+
+	if (dvmaddr == (bus_addr_t)-1)
+		return (ENOMEM);
+
+	/*
+	 * We always use just one segment.
+	 */
+	map->dm_mapsize = buflen;
+	map->dm_nsegs = 1;
+	map->dm_segs[0].ds_addr = dvmaddr + (vaddr & PGOFSET);
+	map->dm_segs[0].ds_len = buflen;
+
+	if (p != NULL)
+		pmap = p->p_vmspace->vm_map.pmap;
+	else
+		pmap = pmap_kernel();
+
+	dvmaddr = trunc_page(map->dm_segs[0].ds_addr);
+	for (; buflen > 0; ) {
+		/*
+		 * Get the physical address for this page.
+		 */
+		if (pmap_extract(pmap, (vaddr_t)vaddr, &curaddr) == FALSE) {
+			bus_dmamap_unload(t, map);
+			return (-1);
+		}
+
+		/*
+		 * Compute the segment size, and adjust counts.
+		 */
+		sgsize = NBPG - ((u_long)vaddr & PGOFSET);
+		if (buflen < sgsize)
+			sgsize = buflen;
+
+		DPRINTF(IDB_BUSDMA,
+		    ("iommu_dvmamap_load: map %p loading va %p "
+			    "dva %lx at pa %lx\n",
+			    map, (void *)vaddr, (long)dvmaddr,
+			    (long)(curaddr&~(NBPG-1))));
+		iommu_enter(is, trunc_page(dvmaddr), trunc_page(curaddr),
+		    flags);
+			
+		dvmaddr += PAGE_SIZE;
+		vaddr += sgsize;
+		buflen -= sgsize;
+	}
+	return (0);
+}
+
+
+void
+iommu_dvmamap_unload(t, is, map)
+	bus_dma_tag_t t;
+	struct iommu_state *is;
+	bus_dmamap_t map;
+{
+	vaddr_t addr, offset;
+	size_t len;
+	int error, s, i;
+	bus_addr_t dvmaddr;
+	bus_size_t sgsize;
+	paddr_t pa;
+
+	dvmaddr = (map->dm_segs[0].ds_addr & ~PGOFSET);
+	pa = 0;
+	sgsize = 0;
+	for (i = 0; i<map->dm_nsegs; i++) {
+
+		addr = trunc_page(map->dm_segs[i].ds_addr);
+		offset = map->dm_segs[i].ds_addr & PGOFSET;
+		len = map->dm_segs[i].ds_len;
+		if (len == 0 || addr == 0)
+			printf("iommu_dvmamap_unload: map = %p, i = %d, len = %d, addr = %lx\n",
+				map, (int)i, (int)len, (unsigned long)addr);
+
+		DPRINTF(IDB_BUSDMA,
+			("iommu_dvmamap_unload: map %p removing va %lx size %lx\n",
+			 map, (long)addr, (long)len));
+		iommu_remove(is, addr, len);
+		
+		if (trunc_page(pa) == addr)
+			sgsize += trunc_page(len + offset);
+		else 
+			sgsize += round_page(len + offset);
+		pa = addr + offset + len;
+
+	}
+	/* Flush the caches */
+	bus_dmamap_unload(t->_parent, map);
+
+	/* Mark the mappings as invalid. */
+	map->dm_mapsize = 0;
+	map->dm_nsegs = 0;
+	
+	s = splhigh();
+	error = extent_free(is->is_dvmamap, dvmaddr, sgsize, EX_NOWAIT);
+	splx(s);
+	if (error != 0)
+		printf("warning: %qd of DVMA space lost\n", (long long)sgsize);
+}
+
+
+int
+iommu_dvmamap_load_raw(t, is, map, segs, nsegs, flags, size)
+	bus_dma_tag_t t;
+	struct iommu_state *is;
+	bus_dmamap_t map;
+	bus_dma_segment_t *segs;
+	int nsegs;
+	int flags;
+	bus_size_t size;
+{
+	struct vm_page *m;
+	int i, s;
+	int left;
+	int err;
+	bus_size_t sgsize;
+	paddr_t pa;
+	bus_size_t boundary, align;
+	u_long dvmaddr;
+	struct pglist *mlist;
+	int pagesz = PAGE_SIZE;
+
+	if (map->dm_nsegs) {
+		/* Already in use?? */
+#ifdef DIAGNOSTIC
+		printf("iommu_dvmamap_load_raw: map still in use\n");
+#endif
+		bus_dmamap_unload(t, map);
+	}
+	/*
+	 * Make sure that on error condition we return "no valid mappings".
+	 */
+	map->dm_nsegs = 0;
+	/* Count up the total number of pages we need */
+	pa = segs[0].ds_addr;
+	sgsize = 0;
+	for (i=0; i<nsegs; i++) {
+		if (round_page(pa) != round_page(segs[i].ds_addr))
+			sgsize = round_page(sgsize);
+		sgsize += segs[i].ds_len;
+		pa = segs[i].ds_addr + segs[i].ds_len;
+	}
+	sgsize = round_page(sgsize);
+
+	/*
+	 * A boundary presented to bus_dmamem_alloc() takes precedence
+	 * over boundary in the map.
+	 */
+	if ((boundary = segs[0]._ds_boundary) == 0)
+		boundary = map->_dm_boundary;
+
+	align = max(segs[0]._ds_align, NBPG);
+	s = splhigh();
+	err = extent_alloc(is->is_dvmamap, sgsize, align, 0, boundary, 
+	   ((flags & BUS_DMA_NOWAIT) == 0 ? EX_WAITOK : EX_NOWAIT)|EX_BOUNDZERO, 
+	    (u_long *)&dvmaddr);
+	splx(s);
+
+	if (err != 0)
+		return (err);
+
+#ifdef DEBUG
+	if (dvmaddr == (bus_addr_t)-1)	
+	{ 
+		printf("iommu_dvmamap_load_raw(): extent_alloc(%d, %x) failed!\n",
+		    (int)sgsize, flags);
+#ifdef DDB
+		Debugger();
+#else
+		panic("");
+#endif
+	}		
+#endif	
+	if (dvmaddr == (bus_addr_t)-1)
+		return (ENOMEM);
+
+	if ((mlist = segs[0]._ds_mlist) == NULL) {
+		u_long prev_va = NULL;
+		/*
+		 * This segs is made up of individual physical pages,
+		 * probably by _bus_dmamap_load_uio() or 
+		 * _bus_dmamap_load_mbuf().  Ignore the mlist and
+		 * load each segment individually.
+		 */
+
+		/* We'll never end up with less segments than we got as input.
+		   this gives us a chance to fail quickly */
+		if (nsegs > map->_dm_segcnt)
+			return (E2BIG);
+
+		i = 0;
+		dvmaddr += (segs[i].ds_addr & PGOFSET);
+		map->dm_segs[i].ds_addr = dvmaddr;
+		map->dm_segs[i].ds_len = left = segs[i].ds_len;
+		pa = segs[i].ds_addr;
+
+		while (left > 0) {
+			int incr;
+
+			if (sgsize == 0)
+				panic("iommu_dmamap_load_raw: size botch");
+			DPRINTF(IDB_BUSDMA,
+				("iommu_dvmamap_load_raw: map %p loading va %lx at pa %lx\n",
+				 map, (long)dvmaddr, (long)(pa)));
+			/* Enter it if we haven't before. */
+			if (prev_va != trunc_page(dvmaddr))
+				iommu_enter(is, prev_va = trunc_page(dvmaddr), 
+					    trunc_page(pa), flags);
+			incr = min(pagesz, left);
+			dvmaddr += incr;
+			pa += incr;
+			left -= incr;
+
+			/* Next segment */
+			if (left <= 0 && ++i < nsegs) {
+				u_long offset;
+
+				/* 
+				 * If the two segs are on different physical pages
+				 * move to a new virtual page.
+				 */
+				offset = (segs[i].ds_addr & PGOFSET);
+				if (trunc_page(pa) != trunc_page(segs[i].ds_addr))
+					dvmaddr += NBPG;
+
+				pa = segs[i].ds_addr;
+				dvmaddr = trunc_page(dvmaddr) + offset;
+
+				map->dm_segs[i].ds_len = left = segs[i].ds_len;
+				map->dm_segs[i].ds_addr = dvmaddr;
+			}
+		}
+		map->dm_nsegs = i;
+
+		/* bail out if we created more segments than the dmamap is
+		   allowed to carry */
+		if (i > map->_dm_segcnt) {
+			iommu_dvmamap_unload(t, is, map);
+			return (E2BIG);
+		}
+
+		return (0);
+	}
+	/*
+	 * This was allocated with bus_dmamem_alloc.  We only
+	 * have one segment, and the pages are on an `mlist'.
+	 */
+	map->dm_mapsize = size;
+	i = 0;
+	map->dm_segs[0].ds_addr = dvmaddr;
+	map->dm_segs[0].ds_len = size;
+
+	for (m = TAILQ_FIRST(mlist); m != NULL; m = TAILQ_NEXT(m,pageq)) {
+		if (sgsize == 0)
+			panic("iommu_dmamap_load_raw: size botch");
+		pa = VM_PAGE_TO_PHYS(m);
+
+		DPRINTF(IDB_BUSDMA,
+		    ("iommu_dvmamap_load_raw: map %p loading va %lx at pa %lx\n",
+		    map, (long)dvmaddr, (long)(pa)));
+		iommu_enter(is, dvmaddr, pa, flags);
+			
+		dvmaddr += pagesz;
+		sgsize -= pagesz;
+	}
+	map->dm_nsegs = i;
+	return (0);
+}
+
+void
+iommu_dvmamap_sync(t, is, map, offset, len, ops)
+	bus_dma_tag_t t;
+	struct iommu_state *is;
+	bus_dmamap_t map;
+	bus_addr_t offset;
+	bus_size_t len;
+	int ops;
+{
+	vaddr_t va = map->dm_segs[0].ds_addr + offset;
+
+	/*
+	 * We only support one DMA segment; supporting more makes this code
+         * too unweildy.
+	 */
+
+	if (ops & BUS_DMASYNC_PREREAD) {
+		DPRINTF(IDB_SYNC,
+		    ("iommu_dvmamap_sync: syncing va %p len %lu "
+		     "BUS_DMASYNC_PREREAD\n", (void *)(u_long)va, (u_long)len));
+
+		/* Nothing to do */;
+	}
+	if (ops & BUS_DMASYNC_POSTREAD) {
+		DPRINTF(IDB_SYNC,
+		    ("iommu_dvmamap_sync: syncing va %p len %lu "
+		     "BUS_DMASYNC_POSTREAD\n", (void *)(u_long)va, (u_long)len));
+		/* if we have a streaming buffer, flush it here first */
+		if (is->is_sb)
+			while (len > 0) {
+				DPRINTF(IDB_BUSDMA,
+				    ("iommu_dvmamap_sync: flushing va %p, %lu "
+				     "bytes left\n", (void *)(u_long)va, (u_long)len));
+				iommu_strbuf_flush(is, va);
+				if (len <= NBPG) {
+					iommu_strbuf_flush_done(is);
+					len = 0;
+				} else
+					len -= NBPG;
+				va += NBPG;
+			}
+	}
+	if (ops & BUS_DMASYNC_PREWRITE) {
+		DPRINTF(IDB_SYNC,
+		    ("iommu_dvmamap_sync: syncing va %p len %lu "
+		     "BUS_DMASYNC_PREWRITE\n", (void *)(u_long)va, (u_long)len));
+		/* if we have a streaming buffer, flush it here first */
+		if (is->is_sb)
+			while (len > 0) {
+				DPRINTF(IDB_BUSDMA,
+				    ("iommu_dvmamap_sync: flushing va %p, %lu "
+				     "bytes left\n", (void *)(u_long)va, (u_long)len));
+				iommu_strbuf_flush(is, va);
+				if (len <= NBPG) {
+					iommu_strbuf_flush_done(is);
+					len = 0;
+				} else
+					len -= NBPG;
+				va += NBPG;
+			}
+	}
+	if (ops & BUS_DMASYNC_POSTWRITE) {
+		DPRINTF(IDB_SYNC,
+		    ("iommu_dvmamap_sync: syncing va %p len %lu "
+		     "BUS_DMASYNC_POSTWRITE\n", (void *)(u_long)va, (u_long)len));
+		/* Nothing to do */;
+	}
+}
+
+int
+iommu_dvmamem_alloc(t, is, size, alignment, boundary, segs, nsegs, rsegs, flags)
+	bus_dma_tag_t t;
+	struct iommu_state *is;
+	bus_size_t size, alignment, boundary;
+	bus_dma_segment_t *segs;
+	int nsegs;
+	int *rsegs;
+	int flags;
+{
+
+	DPRINTF(IDB_BUSDMA, ("iommu_dvmamem_alloc: sz %llx align %llx bound %llx "
+	   "segp %p flags %d\n", (unsigned long long)size,
+	   (unsigned long long)alignment, (unsigned long long)boundary,
+	   segs, flags));
+	return (bus_dmamem_alloc(t->_parent, size, alignment, boundary,
+	    segs, nsegs, rsegs, flags|BUS_DMA_DVMA));
+}
+
+void
+iommu_dvmamem_free(t, is, segs, nsegs)
+	bus_dma_tag_t t;
+	struct iommu_state *is;
+	bus_dma_segment_t *segs;
+	int nsegs;
+{
+
+	DPRINTF(IDB_BUSDMA, ("iommu_dvmamem_free: segp %p nsegs %d\n",
+	    segs, nsegs));
+	bus_dmamem_free(t->_parent, segs, nsegs);
+}
+
+/*
+ * Map the DVMA mappings into the kernel pmap.
+ * Check the flags to see whether we're streaming or coherent.
+ */
+int
+iommu_dvmamem_map(t, is, segs, nsegs, size, kvap, flags)
+	bus_dma_tag_t t;
+	struct iommu_state *is;
+	bus_dma_segment_t *segs;
+	int nsegs;
+	size_t size;
+	caddr_t *kvap;
+	int flags;
+{
+	struct vm_page *m;
+	vaddr_t va;
+	bus_addr_t addr;
+	struct pglist *mlist;
+	int cbit;
+
+	DPRINTF(IDB_BUSDMA, ("iommu_dvmamem_map: segp %p nsegs %d size %lx\n",
+	    segs, nsegs, size));
+
+	/*
+	 * Allocate some space in the kernel map, and then map these pages
+	 * into this space.
+	 */
+	size = round_page(size);
+	va = uvm_km_valloc(kernel_map, size);
+	if (va == 0)
+		return (ENOMEM);
+
+	*kvap = (caddr_t)va;
+
+	/* 
+	 * digest flags:
+	 */
+	cbit = 0;
+	if (flags & BUS_DMA_COHERENT)	/* Disable vcache */
+		cbit |= PMAP_NVC;
+	if (flags & BUS_DMA_NOCACHE)	/* sideffects */
+		cbit |= PMAP_NC;
+
+	/*
+	 * Now take this and map it into the CPU.
+	 */
+	mlist = segs[0]._ds_mlist;
+	for (m = mlist->tqh_first; m != NULL; m = m->pageq.tqe_next) {
+#ifdef DIAGNOSTIC
+		if (size == 0)
+			panic("iommu_dvmamem_map: size botch");
+#endif
+		addr = VM_PAGE_TO_PHYS(m);
+		DPRINTF(IDB_BUSDMA, ("iommu_dvmamem_map: "
+		    "mapping va %lx at %llx\n", va, (unsigned long long)addr | cbit));
+		pmap_enter(pmap_kernel(), va, addr | cbit,
+		    VM_PROT_READ | VM_PROT_WRITE,
+		    VM_PROT_READ | VM_PROT_WRITE | PMAP_WIRED);
+		va += PAGE_SIZE;
+		size -= PAGE_SIZE;
+	}
+	pmap_update();
+
+	return (0);
+}
+
+/*
+ * Unmap DVMA mappings from kernel
+ */
+void
+iommu_dvmamem_unmap(t, is, kva, size)
+	bus_dma_tag_t t;
+	struct iommu_state *is;
+	caddr_t kva;
+	size_t size;
+{
+	
+	DPRINTF(IDB_BUSDMA, ("iommu_dvmamem_unmap: kvm %p size %lx\n",
+	    kva, size));
+	    
+#ifdef DIAGNOSTIC
+	if ((u_long)kva & PGOFSET)
+		panic("iommu_dvmamem_unmap");
+#endif
+	
+	size = round_page(size);
+	pmap_remove(pmap_kernel(), (vaddr_t)kva, size);
+	pmap_update();
+#if 0
+	/*
+	 * XXX ? is this necessary? i think so and i think other
+	 * implementations are missing it.
+	 */
+	uvm_km_free(kernel_map, (vaddr_t)kva, size);
+#endif
+}