add a sg_dma backend for amd64 bus_dma. This is a lot more clever about

mapping to the gart than the old code, and shouldn't conflict with
bouncebuffers when they're added.

This is essentially the sparc64 iommu code that's been modularised a bit
so I can eventually use the same code for agp-based dma for memory
managed drm drivers.

Now, this would overflow ramdiskA, so iommu and sg_dma are now #ifndef
SMALL_KERNEL.

ok kettenis@, marco@. SMALL_KERNEL discussions with deraadt.

6 files changed, 1088 insertions, 329 deletions

diff --git a/sys/arch/amd64/amd64/bus_dma.c b/sys/arch/amd64/amd64/bus_dma.c
index fa0f31700f1..d1e427547f2 100644
--- a/sys/arch/amd64/amd64/bus_dma.c
+++ b/sys/arch/amd64/amd64/bus_dma.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: bus_dma.c,v 1.26 2009/04/20 00:42:05 oga Exp $	*/
+/*	$OpenBSD: bus_dma.c,v 1.27 2009/04/21 17:05:29 oga Exp $	*/
 /*	$NetBSD: bus_dma.c,v 1.3 2003/05/07 21:33:58 fvdl Exp $	*/
 
 /*-
@@ -646,6 +646,13 @@ _bus_dmamem_alloc_range(bus_dma_tag_t t, bus_size_t size, bus_size_t alignment,
 	/* Always round the size. */
 	size = round_page(size);
 
+	segs[0]._ds_boundary = boundary;
+	segs[0]._ds_align = alignment;
+	if (flags & BUS_DMA_SG) {
+		boundary = 0;
+		alignment = 0;
+	}
+
 	/*
 	 * Allocate pages from the VM system.
 	 */
diff --git a/sys/arch/amd64/amd64/sg_dma.c b/sys/arch/amd64/amd64/sg_dma.c
new file mode 100644
index 00000000000..f9da3094342
--- /dev/null
+++ b/sys/arch/amd64/amd64/sg_dma.c
@@ -0,0 +1,960 @@
+/*	$OpenBSD: sg_dma.c,v 1.1 2009/04/21 17:05:29 oga Exp $	*/
+/*
+ * Copyright (c) 2009 Owain G. Ainsworth <oga@openbsd.org>
+ *
+ * Permission to use, copy, modify, and distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+/*
+ * Copyright (c) 2003 Henric Jungheim
+ * Copyright (c) 2001, 2002 Eduardo Horvath
+ * Copyright (c) 1999, 2000 Matthew R. Green
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. The name of the author may not be used to endorse or promote products
+ *    derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+ * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+/*
+ * Support for scatter/gather style dma through agp or an iommu.
+ */
+#include <sys/param.h>
+#include <sys/extent.h>
+#include <sys/malloc.h>
+#include <sys/systm.h>
+#include <sys/device.h>
+#include <sys/mbuf.h>
+#include <sys/mutex.h>
+
+#include <uvm/uvm_extern.h>
+
+#include <machine/bus.h>
+#include <machine/cpu.h>
+
+#ifndef MAX_DMA_SEGS
+#define MAX_DMA_SEGS	20
+#endif
+
+#ifndef SMALL_KERNEL /* no bigmem needed in ramdisks */
+
+/* 
+ * per-map DVMA page table
+ */
+struct sg_page_entry {
+	SPLAY_ENTRY(sg_page_entry)	spe_node;
+	paddr_t				spe_pa;
+	vaddr_t				spe_va;
+};
+
+/* this should be in the map's dm_cookie. */
+struct sg_page_map {
+	SPLAY_HEAD(sg_page_tree, sg_page_entry) spm_tree;
+	int			 spm_maxpage;	/* Size of allocated page map */
+	int			 spm_pagecnt;	/* Number of entries in use */
+	bus_addr_t		 spm_start;	/* dva when bound */
+	bus_size_t		 spm_size;	/* size of bound map */
+	struct sg_page_entry	 spm_map[1];
+};
+
+int	sg_dmamap_load_seg(bus_dma_tag_t, struct sg_cookie *, bus_dmamap_t,
+	    bus_dma_segment_t *, int, int, bus_size_t, bus_size_t);
+struct sg_page_map *sg_iomap_create(int);
+int	sg_dmamap_append_range(bus_dma_tag_t, bus_dmamap_t, paddr_t,
+	    bus_size_t, int, bus_size_t);
+int	sg_iomap_insert_page(struct sg_page_map *, paddr_t);
+vaddr_t	sg_iomap_translate(struct sg_page_map *, paddr_t);
+void	sg_iomap_load_map(struct sg_cookie *, struct sg_page_map *,
+	    vaddr_t, int);
+void	sg_iomap_unload_map(struct sg_cookie *, struct sg_page_map *);
+void	sg_iomap_destroy(struct sg_page_map *);
+void	sg_iomap_clear_pages(struct sg_page_map *);
+
+struct sg_cookie *
+sg_dmatag_init(char *name, void *hdl, bus_addr_t start, bus_size_t size,
+    void bind(void *, vaddr_t, paddr_t, int),
+    void unbind(void *, vaddr_t), void flush_tlb(void *))
+{
+	struct sg_cookie	*cookie;
+
+	cookie = malloc(sizeof(*cookie), M_DEVBUF, M_NOWAIT|M_ZERO);
+	if (cookie == NULL)
+		return (NULL);
+
+	cookie->sg_ex = extent_create(name, start, start + size - 1,
+	    M_DEVBUF, NULL, NULL, EX_NOWAIT | EX_NOCOALESCE);
+	if (cookie->sg_ex == NULL) {
+		free(cookie, M_DEVBUF);
+		return (NULL);
+	}
+
+	cookie->sg_hdl = hdl;
+	mtx_init(&cookie->sg_mtx, IPL_HIGH);
+	cookie->bind_page = bind;
+	cookie->unbind_page = unbind;
+	cookie->flush_tlb = flush_tlb;
+
+	return (cookie);
+}
+
+void
+sg_dmatag_destroy(struct sg_cookie *cookie)
+{
+	extent_destroy(cookie->sg_ex);
+	free(cookie, M_DEVBUF);
+}
+
+int
+sg_dmamap_create(bus_dma_tag_t t, bus_size_t size, int nsegments,
+    bus_size_t maxsegsz, bus_size_t boundary, int flags, bus_dmamap_t *dmamap)
+{
+	struct sg_page_map	*spm;
+	bus_dmamap_t		 map;
+	int			 ret;
+
+	if ((ret = _bus_dmamap_create(t, size, nsegments, maxsegsz, boundary,
+	    flags, &map)) != 0)
+		return (ret);
+
+	if ((spm = sg_iomap_create(atop(round_page(size)))) == NULL) {
+		_bus_dmamap_destroy(t, map);
+		return (ENOMEM);
+	}
+
+	map->_dm_cookie = spm;
+	*dmamap = map;
+
+	return (0);
+}
+
+void
+sg_dmamap_set_alignment(bus_dma_tag_t tag, bus_dmamap_t dmam,
+    u_long alignment)
+{
+	if (alignment < PAGE_SIZE)
+		return;
+
+	dmam->dm_segs[0]._ds_align = alignment;
+}
+
+void
+sg_dmamap_destroy(bus_dma_tag_t t, bus_dmamap_t map)
+{
+	/*
+	 * The specification (man page) requires a loaded
+	 * map to be unloaded before it is destroyed.
+	 */
+	if (map->dm_nsegs)
+		bus_dmamap_unload(t, map);
+
+        if (map->_dm_cookie)
+                sg_iomap_destroy(map->_dm_cookie);
+	map->_dm_cookie = NULL;
+	_bus_dmamap_destroy(t, map);
+}
+
+/*
+ * Load a contiguous kva buffer into a dmamap.  The physical pages are
+ * not assumed to be contiguous.  Two passes are made through the buffer
+ * and both call pmap_extract() for the same va->pa translations.  It
+ * is possible to run out of pa->dvma mappings; the code should be smart
+ * enough to resize the iomap (when the "flags" permit allocation).  It
+ * is trivial to compute the number of entries required (round the length
+ * up to the page size and then divide by the page size)...
+ */
+int
+sg_dmamap_load(bus_dma_tag_t t, bus_dmamap_t map, void *buf,
+    bus_size_t buflen, struct proc *p, int flags)
+{
+	int err = 0;
+	bus_size_t sgsize;
+	u_long dvmaddr, sgstart, sgend;
+	bus_size_t align, boundary;
+	struct sg_cookie *is = t->_cookie;
+	struct sg_page_map *spm = map->_dm_cookie;
+	pmap_t pmap;
+
+	if (map->dm_nsegs) {
+		/*
+		 * Is it still in use? _bus_dmamap_load should have taken care
+		 * of this.
+		 */
+#ifdef DIAGNOSTIC
+		panic("sg_dmamap_load: map still in use");
+#endif
+		bus_dmamap_unload(t, map);
+	}
+
+	/*
+	 * Make sure that on error condition we return "no valid mappings".
+	 */
+	map->dm_nsegs = 0;
+
+	if (buflen < 1 || buflen > map->_dm_size)
+		return (EINVAL);
+
+	/*
+	 * A boundary presented to bus_dmamem_alloc() takes precedence
+	 * over boundary in the map.
+	 */
+	if ((boundary = (map->dm_segs[0]._ds_boundary)) == 0)
+		boundary = map->_dm_boundary;
+	align = MAX(map->dm_segs[0]._ds_align, PAGE_SIZE);
+
+	pmap = p ? p->p_vmspace->vm_map.pmap : pmap = pmap_kernel();
+
+	/* Count up the total number of pages we need */
+	sg_iomap_clear_pages(spm);
+	{ /* Scope */
+		bus_addr_t a, aend;
+		bus_addr_t addr = (vaddr_t)buf;
+		int seg_len = buflen;
+
+		aend = round_page(addr + seg_len);
+		for (a = trunc_page(addr); a < aend; a += PAGE_SIZE) {
+			paddr_t pa;
+
+			if (pmap_extract(pmap, a, &pa) == FALSE) {
+				printf("iomap pmap error addr 0x%llx\n", a);
+				sg_iomap_clear_pages(spm);
+				return (EFBIG);
+			}
+
+			err = sg_iomap_insert_page(spm, pa);
+			if (err) {
+				printf("iomap insert error: %d for "
+				    "va 0x%llx pa 0x%lx "
+				    "(buf %p len %lld/%llx)\n",
+				    err, a, pa, buf, buflen, buflen);
+				sg_iomap_clear_pages(spm);
+				return (EFBIG);
+			}
+		}
+	}
+	sgsize = spm->spm_pagecnt * PAGE_SIZE;
+
+	mtx_enter(&is->sg_mtx);
+	if (flags & BUS_DMA_24BIT) {
+		sgstart = MAX(is->sg_ex->ex_start, 0xff000000);
+		sgend = MIN(is->sg_ex->ex_end, 0xffffffff);
+	} else {
+		sgstart = is->sg_ex->ex_start;
+		sgend = is->sg_ex->ex_end;
+	}
+
+	/* 
+	 * If our segment size is larger than the boundary we need to 
+	 * split the transfer up into little pieces ourselves.
+	 */
+	err = extent_alloc_subregion(is->sg_ex, sgstart, sgend,
+	    sgsize, align, 0, (sgsize > boundary) ? 0 : boundary, 
+	    EX_NOWAIT | EX_BOUNDZERO, (u_long *)&dvmaddr);
+	mtx_leave(&is->sg_mtx);
+	if (err != 0)
+		return (err);
+
+	/* Set the active DVMA map */
+	spm->spm_start = dvmaddr;
+	spm->spm_size = sgsize;
+
+	map->dm_mapsize = buflen;
+
+	sg_iomap_load_map(is, spm, dvmaddr, flags);
+
+	{ /* Scope */
+		bus_addr_t a, aend;
+		bus_addr_t addr = (vaddr_t)buf;
+		int seg_len = buflen;
+
+		aend = round_page(addr + seg_len);
+		for (a = trunc_page(addr); a < aend; a += PAGE_SIZE) {
+			bus_addr_t pgstart;
+			bus_addr_t pgend;
+			paddr_t pa;
+			int pglen;
+
+			/* Yuck... Redoing the same pmap_extract... */
+			if (pmap_extract(pmap, a, &pa) == FALSE) {
+				printf("iomap pmap error addr 0x%llx\n", a);
+				err = EFBIG;
+				break;
+			}
+
+			pgstart = pa | (MAX(a, addr) & PAGE_MASK);
+			pgend = pa | (MIN(a + PAGE_SIZE - 1,
+			    addr + seg_len - 1) & PAGE_MASK);
+			pglen = pgend - pgstart + 1;
+
+			if (pglen < 1)
+				continue;
+
+			err = sg_dmamap_append_range(t, map, pgstart,
+			    pglen, flags, boundary);
+			if (err == EFBIG)
+				break;
+			else if (err) {
+				printf("iomap load seg page: %d for "
+				    "va 0x%llx pa %lx (%llx - %llx) "
+				    "for %d/0x%x\n",
+				    err, a, pa, pgstart, pgend, pglen, pglen);
+				break;
+			}
+		}
+	}
+	if (err) {
+		sg_iomap_unload_map(is, spm);
+		sg_iomap_clear_pages(spm);
+		map->dm_mapsize = 0;
+		map->dm_nsegs = 0;
+		mtx_enter(&is->sg_mtx);
+		extent_free(is->sg_ex, dvmaddr, sgsize, EX_NOWAIT);
+		spm->spm_start = 0;
+		spm->spm_size = 0;
+		mtx_leave(&is->sg_mtx);
+	}
+
+	return (err);
+}
+
+/*
+ * Load an mbuf into our map. we convert it to some bus_dma_segment_ts then
+ * pass it to load_raw.
+ */
+int
+sg_dmamap_load_mbuf(bus_dma_tag_t t, bus_dmamap_t map, struct mbuf *mb,
+    int flags)
+{
+	/*
+	 * This code is adapted from sparc64, for very fragmented data
+	 * we may need to adapt the algorithm
+	 */
+	bus_dma_segment_t	segs[MAX_DMA_SEGS];
+	size_t			len;
+	int			i;
+
+	/*
+	 * Make sure that on error condition we return "no valid mappings".
+	 */
+	map->dm_mapsize = 0;
+	map->dm_nsegs = 0;
+
+	if (mb->m_pkthdr.len > map->_dm_size)
+		return (EINVAL);
+
+	i = 0;
+	len = 0;
+	while (mb) {
+		vaddr_t	vaddr = mtod(mb, vaddr_t);
+		long	buflen = (long)mb->m_len;
+
+		len += buflen;
+		while (buflen > 0 && i < MAX_DMA_SEGS) {
+			paddr_t	pa;
+			long incr;
+
+			incr = min(buflen, NBPG);
+
+			if (pmap_extract(pmap_kernel(), vaddr, &pa) == FALSE)
+				return EINVAL;
+
+			buflen -= incr;
+			vaddr += incr;
+
+			if (i > 0 && pa == (segs[i - 1].ds_addr +
+			    segs[i - 1].ds_len) && ((segs[i - 1].ds_len + incr)
+			    < map->_dm_maxsegsz)) {
+				/* contigious, great! */
+				segs[i - 1].ds_len += incr;
+				continue;
+			}
+			segs[i].ds_addr = pa;
+			segs[i].ds_len = incr;
+			segs[i]._ds_boundary = 0;
+			segs[i]._ds_align = 0;
+			i++;
+		}
+		mb = mb->m_next;
+		if (mb && i >= MAX_DMA_SEGS) {
+			/* our map, it is too big! */
+			return (EFBIG);
+		}
+	}
+
+	return (sg_dmamap_load_raw(t, map, segs, i, (bus_size_t)len, flags));
+}
+
+/*
+ * Load a uio into the map. Turn it into segments and call load_raw()
+ */
+int
+sg_dmamap_load_uio(bus_dma_tag_t t, bus_dmamap_t map, struct uio *uio,
+    int flags)
+{
+	/*
+	 * loading uios is kinda broken since we can't lock the pages.
+	 * and unlock them at unload. Perhaps page loaning is the answer.
+	 * 'till then we only accept kernel data
+	 */
+	bus_dma_segment_t	segs[MAX_DMA_SEGS];
+	size_t			len;
+	int			i, j;
+
+	/*
+	 * Make sure that on errror we return "no valid mappings".
+	 */
+	map->dm_mapsize = 0;
+	map->dm_nsegs = 0;
+
+	if (uio->uio_resid > map->_dm_size)
+		return (EINVAL);
+
+	if (uio->uio_segflg != UIO_SYSSPACE)
+		return (EOPNOTSUPP);
+
+	i = j = 0;
+	len = 0;
+	while (j < uio->uio_iovcnt) {
+		vaddr_t	vaddr = (vaddr_t)uio->uio_iov[j].iov_base;
+		long	buflen = (long)uio->uio_iov[j].iov_len;
+
+		len += buflen;
+		while (buflen > 0 && i < MAX_DMA_SEGS) {
+			paddr_t pa;
+			long incr;
+
+			incr = min(buflen, NBPG);
+			(void)pmap_extract(pmap_kernel(), vaddr, &pa);
+			buflen -= incr;
+			vaddr += incr;
+
+			if (i > 0 && pa == (segs[i - 1].ds_addr +
+			    segs[i -1].ds_len) && ((segs[i - 1].ds_len + incr)
+			    < map->_dm_maxsegsz)) {
+				/* contigious, yay! */
+				segs[i - 1].ds_len += incr;
+				continue;
+			}
+			segs[i].ds_addr = pa;
+			segs[i].ds_len = incr;
+			segs[i]._ds_boundary = 0;
+			segs[i]._ds_align = 0;
+			i++;
+		}
+		j++;
+		if ((uio->uio_iovcnt - j) && i >= MAX_DMA_SEGS) {
+			/* Our map, is it too big! */
+			return (EFBIG);
+		}
+
+	}
+
+	return (sg_dmamap_load_raw(t, map, segs, i, (bus_size_t)len, flags));
+}
+
+/*
+ * Load a dvmamap from an array of segs.  It calls sg_dmamap_append_range()
+ * or for part of the 2nd pass through the mapping.
+ */
+int
+sg_dmamap_load_raw(bus_dma_tag_t t, bus_dmamap_t map,
+    bus_dma_segment_t *segs, int nsegs, bus_size_t size, int flags)
+{
+	int i;
+	int left;
+	int err = 0;
+	bus_size_t sgsize;
+	bus_size_t boundary, align;
+	u_long dvmaddr, sgstart, sgend;
+	struct sg_cookie *is = t->_cookie;
+	struct sg_page_map *spm = map->_dm_cookie;
+
+	if (map->dm_nsegs) {
+		/* Already in use?? */
+#ifdef DIAGNOSTIC
+		panic("sg_dmamap_load_raw: map still in use");
+#endif
+		bus_dmamap_unload(t, map);
+	}
+
+	/*
+	 * A boundary presented to bus_dmamem_alloc() takes precedence
+	 * over boundary in the map.
+	 */
+	if ((boundary = segs[0]._ds_boundary) == 0)
+		boundary = map->_dm_boundary;
+
+	align = MAX(segs[0]._ds_align, PAGE_SIZE);
+
+	/*
+	 * Make sure that on error condition we return "no valid mappings".
+	 */
+	map->dm_nsegs = 0;
+
+	sg_iomap_clear_pages(spm);
+	/* Count up the total number of pages we need */
+	for (i = 0, left = size; left > 0 && i < nsegs; i++) {
+		bus_addr_t a, aend;
+		bus_size_t len = segs[i].ds_len;
+		bus_addr_t addr = segs[i].ds_addr;
+		int seg_len = MIN(left, len);
+
+		if (len < 1)
+			continue;
+
+		aend = round_page(addr + seg_len);
+		for (a = trunc_page(addr); a < aend; a += PAGE_SIZE) {
+
+			err = sg_iomap_insert_page(spm, a);
+			if (err) {
+				printf("iomap insert error: %d for "
+				    "pa 0x%llx\n", err, a);
+				sg_iomap_clear_pages(spm);
+				return (EFBIG);
+			}
+		}
+
+		left -= seg_len;
+	}
+	sgsize = spm->spm_pagecnt * PAGE_SIZE;
+
+	mtx_enter(&is->sg_mtx);
+	if (flags & BUS_DMA_24BIT) {
+		sgstart = MAX(is->sg_ex->ex_start, 0xff000000);
+		sgend = MIN(is->sg_ex->ex_end, 0xffffffff);
+	} else {
+		sgstart = is->sg_ex->ex_start;
+		sgend = is->sg_ex->ex_end;
+	}
+
+	/* 
+	 * If our segment size is larger than the boundary we need to 
+	 * split the transfer up into little pieces ourselves.
+	 */
+	err = extent_alloc_subregion(is->sg_ex, sgstart, sgend,
+	    sgsize, align, 0, (sgsize > boundary) ? 0 : boundary, 
+	    EX_NOWAIT | EX_BOUNDZERO, (u_long *)&dvmaddr);
+	mtx_leave(&is->sg_mtx);
+
+	if (err != 0)
+		return (err);
+
+	/* Set the active DVMA map */
+	spm->spm_start = dvmaddr;
+	spm->spm_size = sgsize;
+
+	map->dm_mapsize = size;
+
+	sg_iomap_load_map(is, spm, dvmaddr, flags);
+
+	err = sg_dmamap_load_seg(t, is, map, segs, nsegs, flags,
+	    size, boundary);
+
+	if (err) {
+		sg_iomap_unload_map(is, spm);
+		sg_iomap_clear_pages(spm);
+		map->dm_mapsize = 0;
+		map->dm_nsegs = 0;
+		mtx_enter(&is->sg_mtx);
+		extent_free(is->sg_ex, dvmaddr, sgsize, EX_NOWAIT);
+		spm->spm_start = 0;
+		spm->spm_size = 0;
+		mtx_leave(&is->sg_mtx);
+	}
+
+	return (err);
+}
+
+/*
+ * Insert a range of addresses into a loaded map respecting the specified
+ * boundary and alignment restrictions.  The range is specified by its 
+ * physical address and length.  The range cannot cross a page boundary.
+ * This code (along with most of the rest of the function in this file)
+ * assumes that the IOMMU page size is equal to PAGE_SIZE.
+ */
+int
+sg_dmamap_append_range(bus_dma_tag_t t, bus_dmamap_t map, paddr_t pa,
+    bus_size_t length, int flags, bus_size_t boundary)
+{
+	struct sg_page_map *spm = map->_dm_cookie;
+	bus_addr_t sgstart, sgend, bd_mask;
+	bus_dma_segment_t *seg = NULL;
+	int i = map->dm_nsegs;
+
+	sgstart = sg_iomap_translate(spm, pa);
+	sgend = sgstart + length - 1;
+
+#ifdef DIAGNOSTIC
+	if (sgstart == NULL || sgstart > sgend) {
+		printf("append range invalid mapping for %lx "
+		    "(0x%llx - 0x%llx)\n", pa, sgstart, sgend);
+		map->dm_nsegs = 0;
+		return (EINVAL);
+	}
+#endif
+
+#ifdef DEBUG
+	if (trunc_page(sgstart) != trunc_page(sgend)) {
+		printf("append range crossing page boundary! "
+		    "pa %lx length %lld/0x%llx sgstart %llx sgend %llx\n",
+		    pa, length, length, sgstart, sgend);
+	}
+#endif
+
+	/*
+	 * We will attempt to merge this range with the previous entry
+	 * (if there is one).
+	 */
+	if (i > 0) {
+		seg = &map->dm_segs[i - 1];
+		if (sgstart == seg->ds_addr + seg->ds_len) {
+			length += seg->ds_len;
+			sgstart = seg->ds_addr;
+			sgend = sgstart + length - 1;
+		} else
+			seg = NULL;
+	}
+
+	if (seg == NULL) {
+		seg = &map->dm_segs[i];
+		if (++i > map->_dm_segcnt) {
+			map->dm_nsegs = 0;
+			return (EFBIG);
+		}
+	}
+
+	/*
+	 * At this point, "i" is the index of the *next* bus_dma_segment_t
+	 * (the segment count, aka map->dm_nsegs) and "seg" points to the
+	 * *current* entry.  "length", "sgstart", and "sgend" reflect what
+	 * we intend to put in "*seg".  No assumptions should be made about
+	 * the contents of "*seg".  Only "boundary" issue can change this
+	 * and "boundary" is often zero, so explicitly test for that case
+	 * (the test is strictly an optimization).
+	 */ 
+	if (boundary != 0) {
+		bd_mask = ~(boundary - 1);
+
+		while ((sgstart & bd_mask) != (sgend & bd_mask)) {
+			/*
+			 * We are crossing a boundary so fill in the current
+			 * segment with as much as possible, then grab a new
+			 * one.
+			 */
+
+			seg->ds_addr = sgstart;
+			seg->ds_len = boundary - (sgstart & bd_mask);
+
+			sgstart += seg->ds_len; /* sgend stays the same */
+			length -= seg->ds_len;
+
+			seg = &map->dm_segs[i];
+			if (++i > map->_dm_segcnt) {
+				map->dm_nsegs = 0;
+				return (EFBIG);
+			}
+		}
+	}
+
+	seg->ds_addr = sgstart;
+	seg->ds_len = length;
+	map->dm_nsegs = i;
+
+	return (0);
+}
+
+/*
+ * Populate the iomap from a bus_dma_segment_t array.  See note for
+ * sg_dmamap_load() regarding page entry exhaustion of the iomap.
+ * This is less of a problem for load_seg, as the number of pages
+ * is usually similar to the number of segments (nsegs).
+ */
+int
+sg_dmamap_load_seg(bus_dma_tag_t t, struct sg_cookie *is,
+    bus_dmamap_t map, bus_dma_segment_t *segs, int nsegs, int flags,
+    bus_size_t size, bus_size_t boundary)
+{
+	int i;
+	int left;
+	int seg;
+
+	/*
+	 * Keep in mind that each segment could span
+	 * multiple pages and that these are not always
+	 * adjacent. The code is no longer adding dvma
+	 * aliases to the IOMMU.  The STC will not cross
+	 * page boundaries anyway and a IOMMU table walk
+	 * vs. what may be a streamed PCI DMA to a ring
+	 * descriptor is probably a wash.  It eases TLB
+	 * pressure and in the worst possible case, it is
+	 * only as bad a non-IOMMUed architecture.  More
+	 * importantly, the code is not quite as hairy.
+	 * (It's bad enough as it is.)
+	 */
+	left = size;
+	seg = 0;
+	for (i = 0; left > 0 && i < nsegs; i++) {
+		bus_addr_t a, aend;
+		bus_size_t len = segs[i].ds_len;
+		bus_addr_t addr = segs[i].ds_addr;
+		int seg_len = MIN(left, len);
+
+		if (len < 1)
+			continue;
+
+		aend = round_page(addr + seg_len);
+		for (a = trunc_page(addr); a < aend; a += PAGE_SIZE) {
+			bus_addr_t pgstart;
+			bus_addr_t pgend;
+			int pglen;
+			int err;
+
+			pgstart = MAX(a, addr);
+			pgend = MIN(a + PAGE_SIZE - 1, addr + seg_len - 1);
+			pglen = pgend - pgstart + 1;
+			
+			if (pglen < 1)
+				continue;
+
+			err = sg_dmamap_append_range(t, map, pgstart,
+			    pglen, flags, boundary);
+			if (err == EFBIG)
+				return (err);
+			if (err) {
+				printf("iomap load seg page: %d for "
+				    "pa 0x%llx (%llx - %llx for %d/%x\n",
+				    err, a, pgstart, pgend, pglen, pglen);
+				return (err);
+			}
+
+		}
+
+		left -= seg_len;
+	}
+	return (0);
+}
+
+/*
+ * Unload a dvmamap.
+ */
+void
+sg_dmamap_unload(bus_dma_tag_t t, bus_dmamap_t map)
+{
+	struct sg_cookie	*is = t->_cookie;
+	struct sg_page_map	*spm = map->_dm_cookie;
+	bus_addr_t		 dvmaddr = spm->spm_start;
+	bus_size_t		 sgsize = spm->spm_size;
+	int			 error;
+
+	/* Remove the IOMMU entries */
+	sg_iomap_unload_map(is, spm);
+
+	/* Clear the iomap */
+	sg_iomap_clear_pages(spm);
+
+	mtx_enter(&is->sg_mtx);
+	error = extent_free(is->sg_ex, dvmaddr, 
+		sgsize, EX_NOWAIT);
+	spm->spm_start = 0;
+	spm->spm_size = 0;
+	mtx_leave(&is->sg_mtx);
+	if (error != 0)
+		printf("warning: %qd of DVMA space lost\n", sgsize);
+	_bus_dmamap_unload(t, map);
+}
+
+/*
+ * Alloc dma safe memory, telling the backend that we're scatter gather
+ * to ease pressure on the vm.
+ *
+ * This assumes that we can map all physical memory. 
+ */
+int
+sg_dmamem_alloc(bus_dma_tag_t t, bus_size_t size,
+    bus_size_t alignment, bus_size_t boundary, bus_dma_segment_t *segs,
+    int nsegs, int *rsegs, int flags)
+{
+	return (_bus_dmamem_alloc_range(t, size, alignment, boundary,
+	    segs, nsegs, rsegs, flags | BUS_DMA_SG, 0, -1));
+}
+
+/*
+ * Create a new iomap.
+ */
+struct sg_page_map *
+sg_iomap_create(int n)
+{
+	struct sg_page_map	*spm;
+
+	/* Safety for heavily fragmented data, such as mbufs */
+	n += 4;
+	if (n < 16)
+		n = 16;
+
+	spm = malloc(sizeof(*spm) + (n - 1) * sizeof(spm->spm_map[0]),
+		M_DEVBUF, M_NOWAIT | M_ZERO);
+	if (spm == NULL)
+		return (NULL);
+
+	/* Initialize the map. */
+	spm->spm_maxpage = n;
+	SPLAY_INIT(&spm->spm_tree);
+
+	return (spm);
+}
+
+/*
+ * Destroy an iomap.
+ */
+void
+sg_iomap_destroy(struct sg_page_map *spm)
+{
+#ifdef DIAGNOSTIC
+	if (spm->spm_pagecnt > 0)
+		printf("sg_iomap_destroy: %d page entries in use\n",
+		    spm->spm_pagecnt);
+#endif
+
+	free(spm, M_DEVBUF);
+}
+
+/*
+ * Utility function used by splay tree to order page entries by pa.
+ */
+static inline int
+iomap_compare(struct sg_page_entry *a, struct sg_page_entry *b)
+{
+	return ((a->spe_pa > b->spe_pa) ? 1 :
+		(a->spe_pa < b->spe_pa) ? -1 : 0);
+}
+
+SPLAY_PROTOTYPE(sg_page_tree, sg_page_entry, spe_node, iomap_compare);
+
+SPLAY_GENERATE(sg_page_tree, sg_page_entry, spe_node, iomap_compare);
+
+/*
+ * Insert a pa entry in the iomap.
+ */
+int
+sg_iomap_insert_page(struct sg_page_map *spm, paddr_t pa)
+{
+	struct sg_page_entry *e;
+
+	if (spm->spm_pagecnt >= spm->spm_maxpage) {
+		struct sg_page_entry spe;
+
+		spe.spe_pa = pa;
+		if (SPLAY_FIND(sg_page_tree, &spm->spm_tree, &spe))
+			return (0);
+
+		return (ENOMEM);
+	}
+
+	e = &spm->spm_map[spm->spm_pagecnt];
+
+	e->spe_pa = pa;
+	e->spe_va = NULL;
+
+	e = SPLAY_INSERT(sg_page_tree, &spm->spm_tree, e);
+
+	/* Duplicates are okay, but only count them once. */
+	if (e)
+		return (0);
+
+	++spm->spm_pagecnt;
+
+	return (0);
+}
+
+/*
+ * Locate the iomap by filling in the pa->va mapping and inserting it
+ * into the IOMMU tables.
+ */
+void
+sg_iomap_load_map(struct sg_cookie *sc, struct sg_page_map *spm,
+    vaddr_t vmaddr, int flags)
+{
+	struct sg_page_entry	*e;
+	int			 i;
+
+	for (i = 0, e = spm->spm_map; i < spm->spm_pagecnt; ++i, ++e) {
+		e->spe_va = vmaddr;
+		sc->bind_page(sc->sg_hdl, e->spe_va, e->spe_pa, flags);
+		vmaddr += PAGE_SIZE;
+	}
+	sc->flush_tlb(sc->sg_hdl);
+}
+
+/*
+ * Remove the iomap from the IOMMU.
+ */
+void
+sg_iomap_unload_map(struct sg_cookie *sc, struct sg_page_map *spm)
+{
+	struct sg_page_entry	*e;
+	int			 i;
+
+	for (i = 0, e = spm->spm_map; i < spm->spm_pagecnt; ++i, ++e)
+		sc->unbind_page(sc->sg_hdl, e->spe_va);
+	sc->flush_tlb(sc->sg_hdl);
+
+}
+
+/*
+ * Translate a physical address (pa) into a DVMA address.
+ */
+vaddr_t
+sg_iomap_translate(struct sg_page_map *spm, paddr_t pa)
+{
+	struct sg_page_entry	*e, pe;
+	paddr_t			 offset = pa & PAGE_MASK;
+
+	pe.spe_pa = trunc_page(pa);
+
+	e = SPLAY_FIND(sg_page_tree, &spm->spm_tree, &pe);
+
+	if (e == NULL)
+		return (NULL);
+
+	return (e->spe_va | offset);
+}
+
+/*
+ * Clear the iomap table and tree.
+ */
+void
+sg_iomap_clear_pages(struct sg_page_map *spm)
+{
+	spm->spm_pagecnt = 0;
+	SPLAY_INIT(&spm->spm_tree);
+}
+
+
+#endif /* !SMALL_KERNEL */
diff --git a/sys/arch/amd64/conf/files.amd64 b/sys/arch/amd64/conf/files.amd64
index 0c7522be0a7..7101f0b669f 100644
--- a/sys/arch/amd64/conf/files.amd64
+++ b/sys/arch/amd64/conf/files.amd64
@@ -1,4 +1,4 @@
-#	$OpenBSD: files.amd64,v 1.45 2009/04/20 13:26:20 ariane Exp $
+#	$OpenBSD: files.amd64,v 1.46 2009/04/21 17:05:29 oga Exp $
 
 maxpartitions 16
 maxusers 2 16 128
@@ -28,6 +28,7 @@ file	arch/amd64/amd64/lock_machdep.c		multiprocessor
 file	arch/amd64/amd64/intr.c
 file	arch/amd64/amd64/bus_space.c
 file	arch/amd64/amd64/bus_dma.c
+file	arch/amd64/amd64/sg_dma.c
 
 file	arch/amd64/amd64/mptramp.S		multiprocessor
 file	arch/amd64/amd64/ipifuncs.c		multiprocessor
diff --git a/sys/arch/amd64/include/bus.h b/sys/arch/amd64/include/bus.h
index 1bb7ad405ea..8952c1a0678 100644
--- a/sys/arch/amd64/include/bus.h
+++ b/sys/arch/amd64/include/bus.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: bus.h,v 1.16 2009/04/20 00:42:05 oga Exp $	*/
+/*	$OpenBSD: bus.h,v 1.17 2009/04/21 17:05:29 oga Exp $	*/
 /*	$NetBSD: bus.h,v 1.6 1996/11/10 03:19:25 thorpej Exp $	*/
 
 /*-
@@ -66,6 +66,8 @@
 #ifndef _X86_BUS_H_
 #define _X86_BUS_H_
 
+#include <sys/mutex.h>
+
 #include <machine/pio.h>
 
 /*
@@ -447,6 +449,7 @@ void	bus_space_barrier(bus_space_tag_t, bus_space_handle_t,
 #define	BUS_DMA_WRITE		0x0400	/* mapping is memory -> device only */
 #define	BUS_DMA_NOCACHE		0x0800	/* map memory uncached */
 #define	BUS_DMA_ZERO		0x1000	/* zero memory in dmamem_alloc */
+#define	BUS_DMA_SG		0x2000	/* Internal. memory is for SG map */
 
 /* Forwards needed by prototypes below. */
 struct mbuf;
@@ -473,6 +476,13 @@ typedef struct bus_dmamap		*bus_dmamap_t;
 struct bus_dma_segment {
 	bus_addr_t	ds_addr;	/* DMA address */
 	bus_size_t	ds_len;		/* length of transfer */
+	/*
+	 * Ugh. need this so can pass alignment down from bus_dmamem_alloc
+	 * to scatter gather maps. only the first one is used so the rest is
+	 * wasted space. bus_dma could do with fixing the api for this.
+	 */
+	 bus_size_t	_ds_boundary;	/* don't cross */
+	 bus_size_t	_ds_align;	/* align to me */
 };
 typedef struct bus_dma_segment	bus_dma_segment_t;
 
@@ -604,6 +614,41 @@ int	_bus_dmamem_alloc_range(bus_dma_tag_t tag, bus_size_t size,
 	    bus_dma_segment_t *segs, int nsegs, int *rsegs, int flags,
 	    paddr_t low, paddr_t high);
 
+struct extent;
+
+/* Scatter gather bus_dma functions. */
+struct sg_cookie {
+	struct mutex	 sg_mtx;
+	struct extent	*sg_ex;
+	void		*sg_hdl;
+	void		(*bind_page)(void *, vaddr_t, paddr_t, int);
+	void		(*unbind_page)(void *, vaddr_t);
+	void		(*flush_tlb)(void *);
+};
+
+struct sg_cookie	*sg_dmatag_init(char *, void *, bus_addr_t, bus_size_t,
+			    void (*)(void *, vaddr_t, paddr_t, int),
+			    void (*)(void *, vaddr_t), void (*)(void *));
+void	sg_dmatag_destroy(struct sg_cookie *);
+int	sg_dmamap_create(bus_dma_tag_t, bus_size_t, int, bus_size_t,
+	    bus_size_t, int, bus_dmamap_t *);
+void	sg_dmamap_destroy(bus_dma_tag_t, bus_dmamap_t);
+void	sg_dmamap_set_alignment(bus_dma_tag_t, bus_dmamap_t, u_long);
+int	sg_dmamap_load(bus_dma_tag_t, bus_dmamap_t, void *, bus_size_t,
+	    struct proc *, int);
+int	sg_dmamap_load_mbuf(bus_dma_tag_t, bus_dmamap_t,
+	    struct mbuf *, int);
+int	sg_dmamap_load_uio(bus_dma_tag_t, bus_dmamap_t, struct uio *, int);
+int	sg_dmamap_load_raw(bus_dma_tag_t, bus_dmamap_t, bus_dma_segment_t *,
+	    int, bus_size_t, int);
+void	sg_dmamap_unload(bus_dma_tag_t, bus_dmamap_t);
+int	sg_dmamap_load_buffer(bus_dma_tag_t, bus_dmamap_t, void *, bus_size_t,
+	    struct proc *, int, int *, int);
+int	sg_dmamap_load_physarray(bus_dma_tag_t, bus_dmamap_t, paddr_t *,
+	    int, int, int *, int);
+int	sg_dmamem_alloc(bus_dma_tag_t, bus_size_t, bus_size_t, bus_size_t,
+	    bus_dma_segment_t *, int, int *, int);
+
 /*      
  *      paddr_t bus_space_mmap(bus_space_tag_t t, bus_addr_t base,
  *          off_t offset, int prot, int flags);
diff --git a/sys/arch/amd64/pci/iommu.c b/sys/arch/amd64/pci/iommu.c
index 18c11bbf21d..50eeb79493b 100644
--- a/sys/arch/amd64/pci/iommu.c
+++ b/sys/arch/amd64/pci/iommu.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: iommu.c,v 1.27 2009/04/15 23:53:22 oga Exp $	*/
+/*	$OpenBSD: iommu.c,v 1.28 2009/04/21 17:05:29 oga Exp $	*/
 
 /*
  * Copyright (c) 2005 Jason L. Wright (jason@thought.net)
@@ -33,7 +33,6 @@
 #include <sys/errno.h>
 #include <sys/device.h>
 #include <sys/lock.h>
-#include <sys/extent.h>
 #include <sys/malloc.h>
 
 #include <uvm/uvm_extern.h>
@@ -100,14 +99,12 @@
 #define	IOMMU_SIZE		512		/* size in MB */
 #define	IOMMU_ALIGN		IOMMU_SIZE
 
-extern paddr_t avail_end;
-extern struct extent *iomem_ex;
-
 int amdgart_enable = 0;
 
+#ifndef SMALL_KERNEL /* no bigmem in ramdisks */
+
 struct amdgart_softc {
 	pci_chipset_tag_t	 g_pc;
-	struct extent		*g_ex;
 	paddr_t			 g_pa;
 	paddr_t			 g_scribpa;
 	void			*g_scrib;
@@ -118,58 +115,56 @@ struct amdgart_softc {
 	pcitag_t		 g_tags[1];
 };
 
-void amdgart_invalidate_wait(struct amdgart_softc *);
-void amdgart_invalidate(struct amdgart_softc *);
-void amdgart_probe(struct pcibus_attach_args *);
-void amdgart_dumpregs(struct amdgart_softc *);
-int amdgart_iommu_map(struct amdgart_softc *, bus_dmamap_t,
-        bus_dma_segment_t *);
-int amdgart_iommu_unmap(struct amdgart_softc *, bus_dma_segment_t *);
-int amdgart_reload(struct amdgart_softc *, bus_dmamap_t);
-int amdgart_ok(pci_chipset_tag_t, pcitag_t);
-int amdgart_enabled(pci_chipset_tag_t, pcitag_t);
-void amdgart_initpt(struct amdgart_softc *, u_long);
-
-int amdgart_dmamap_create(bus_dma_tag_t, bus_size_t, int, bus_size_t,
-    bus_size_t, int, bus_dmamap_t *);
-void amdgart_dmamap_destroy(bus_dma_tag_t, bus_dmamap_t);
-int amdgart_dmamap_load(bus_dma_tag_t, bus_dmamap_t, void *, bus_size_t,
-    struct proc *, int);
-int amdgart_dmamap_load_mbuf(bus_dma_tag_t, bus_dmamap_t, struct mbuf *, int);
-int amdgart_dmamap_load_uio(bus_dma_tag_t, bus_dmamap_t, struct uio *, int);
-int amdgart_dmamap_load_raw(bus_dma_tag_t, bus_dmamap_t,
-    bus_dma_segment_t *, int, bus_size_t, int);
-void amdgart_dmamap_unload(bus_dma_tag_t, bus_dmamap_t);
-void amdgart_dmamap_sync(bus_dma_tag_t, bus_dmamap_t, bus_addr_t,
-    bus_size_t, int);
-
-int amdgart_dmamem_alloc(bus_dma_tag_t, bus_size_t, bus_size_t, bus_size_t,
-    bus_dma_segment_t *, int, int *, int);
-void amdgart_dmamem_free(bus_dma_tag_t, bus_dma_segment_t *, int);
-int amdgart_dmamem_map(bus_dma_tag_t, bus_dma_segment_t *, int, size_t,
-    caddr_t *, int);
-void amdgart_dmamem_unmap(bus_dma_tag_t, caddr_t, size_t);
-paddr_t amdgart_dmamem_mmap(bus_dma_tag_t, bus_dma_segment_t *, int, off_t,
-    int, int);
+void	amdgart_probe(struct pcibus_attach_args *);
+void	amdgart_dumpregs(struct amdgart_softc *);
+int	amdgart_ok(pci_chipset_tag_t, pcitag_t);
+int	amdgart_enabled(pci_chipset_tag_t, pcitag_t);
+void	amdgart_initpt(struct amdgart_softc *, u_long);
+void	amdgart_bind_page(void *, vaddr_t, paddr_t,  int);
+void	amdgart_unbind_page(void *, vaddr_t);
+void	amdgart_invalidate(void *);
+void	amdgart_invalidate_wait(struct amdgart_softc *);
 
 struct bus_dma_tag amdgart_bus_dma_tag = {
 	NULL,			/* _may_bounce */
-	amdgart_dmamap_create,
-	amdgart_dmamap_destroy,
-	amdgart_dmamap_load,
-	amdgart_dmamap_load_mbuf,
-	amdgart_dmamap_load_uio,
-	amdgart_dmamap_load_raw,
-	amdgart_dmamap_unload,
+	sg_dmamap_create,
+	sg_dmamap_destroy,
+	sg_dmamap_load,
+	sg_dmamap_load_mbuf,
+	sg_dmamap_load_uio,
+	sg_dmamap_load_raw,
+	sg_dmamap_unload,
 	NULL,
-	amdgart_dmamem_alloc,
-	amdgart_dmamem_free,
-	amdgart_dmamem_map,
-	amdgart_dmamem_unmap,
-	amdgart_dmamem_mmap,
+	sg_dmamem_alloc,
+	_bus_dmamem_free,
+	_bus_dmamem_map,
+	_bus_dmamem_unmap,
+	_bus_dmamem_mmap,
 };
 
 void
+amdgart_bind_page(void *handle, vaddr_t offset, paddr_t page,  int flags)
+{
+	struct amdgart_softc	*sc = handle;
+	u_int32_t		 pgno, pte;
+
+	pgno = (offset - sc->g_pa) >> PGSHIFT;
+	pte = GART_PTE_VALID | GART_PTE_COHERENT |
+	    ((page >> 28) & GART_PTE_PHYSHI) | (page & GART_PTE_PHYSLO);
+	sc->g_pte[pgno] = pte;
+}
+
+void
+amdgart_unbind_page(void *handle, vaddr_t offset)
+{
+	struct amdgart_softc	*sc = handle;
+	u_int32_t		 pgno;
+
+	pgno = (offset - sc->g_pa) >> PGSHIFT;
+	sc->g_pte[pgno] = sc->g_scribpte;
+}
+
+void
 amdgart_invalidate_wait(struct amdgart_softc *sc)
 {
 	int	i, n;
@@ -187,8 +182,9 @@ amdgart_invalidate_wait(struct amdgart_softc *sc)
 }
 
 void
-amdgart_invalidate(struct amdgart_softc *sc)
+amdgart_invalidate(void* handle)
 {
+	struct amdgart_softc	*sc = handle;
 	int n;
 
 	for (n = 0; n < sc->g_count; n++)
@@ -257,16 +253,16 @@ static const struct gart_size {
 void
 amdgart_probe(struct pcibus_attach_args *pba)
 {
-	struct amdgart_softc *sc;
-	int dev, count = 0, encount = 0, r, nseg;
-	u_long mapsize, ptesize, gartsize = 0;
-	bus_dma_segment_t seg;
-	pcitag_t tag;
-	pcireg_t v;
-	paddr_t pa;
-	void *scrib = NULL;
-	u_int32_t *pte = NULL;
-	paddr_t ptepa;
+	struct amdgart_softc	*sc;
+	struct sg_cookie	*cookie = NULL;
+	void			*scrib = NULL;
+	u_int32_t		*pte;
+	int			 dev, count = 0, encount = 0, r, nseg;
+	u_long			 mapsize, ptesize, gartsize = 0;
+	bus_dma_segment_t	 seg;
+	pcitag_t		 tag;
+	pcireg_t		 v;
+	paddr_t			 pa, ptepa;
 
 	if (amdgart_enable == 0)
 		return;
@@ -355,13 +351,6 @@ amdgart_probe(struct pcibus_attach_args *pba)
 	}
 	ptepa = seg.ds_addr;
 
-	sc->g_ex = extent_create("iommu", sc->g_pa, sc->g_pa + mapsize - 1,
-	    M_DEVBUF, NULL, NULL, EX_NOWAIT | EX_NOCOALESCE);
-	if (sc->g_ex == NULL) {
-		printf("\nGART: extent create failed");
-		goto err;
-	}
-
 	scrib = malloc(PAGE_SIZE, M_DEVBUF, M_NOWAIT|M_ZERO);
 	if (scrib == NULL) {
 		printf("\nGART: didn't get scribble page");
@@ -377,6 +366,13 @@ amdgart_probe(struct pcibus_attach_args *pba)
 	sc->g_pte = pte;
 	sc->g_dmat = pba->pba_dmat;
 
+	if ((cookie = sg_dmatag_init("iommu", sc, sc->g_pa, mapsize,
+	    amdgart_bind_page, amdgart_unbind_page,
+	    amdgart_invalidate)) == NULL) {
+		printf("\nGART: didn't get dma cookie\n");
+		goto err;
+	}
+
 	for (count = 0, dev = 24; dev < 32; dev++) {
 		tag = pci_make_tag(pba->pba_pc, 0, dev, 3);
 
@@ -436,7 +432,7 @@ amdgart_probe(struct pcibus_attach_args *pba)
 	amdgart_initpt(sc, ptesize / sizeof(*sc->g_pte));
 	sc->g_count = count;
 
-	amdgart_bus_dma_tag._cookie = sc;
+	amdgart_bus_dma_tag._cookie = cookie;
 	pba->pba_dmat = &amdgart_bus_dma_tag;
 
 	return;
@@ -444,10 +440,10 @@ amdgart_probe(struct pcibus_attach_args *pba)
 err:
 	_bus_dmamem_free(pba->pba_dmat, &seg, 1);
 nofreeseg:
-	if (sc->g_ex != NULL)
-		extent_destroy(sc->g_ex);
 	if (scrib != NULL)
 		free(scrib, M_DEVBUF);
+	if (cookie != NULL)
+		sg_dmatag_destroy(cookie);
 	if (sc != NULL)
 		free(sc, M_DEVBUF);
 }
@@ -462,256 +458,4 @@ amdgart_initpt(struct amdgart_softc *sc, u_long nent)
 	amdgart_invalidate(sc);
 }
 
-int
-amdgart_reload(struct amdgart_softc *sc, bus_dmamap_t dmam)
-{
-	int i, j, err;
-
-	for (i = 0; i < dmam->dm_nsegs; i++) {
-		psize_t len;
-
-		len = dmam->dm_segs[i].ds_len;
-		err = amdgart_iommu_map(sc, dmam, &dmam->dm_segs[i]);
-		if (err) {
-			for (j = 0; j < i - 1; j++)
-				amdgart_iommu_unmap(sc, &dmam->dm_segs[j]);
-			return (err);
-		}
-	}
-	return (0);
-}
-
-int
-amdgart_iommu_map(struct amdgart_softc *sc, bus_dmamap_t dmam,
-    bus_dma_segment_t *seg)
-{
-	paddr_t base, end, idx;
-	psize_t alen;
-	u_long res;
-	int err, s;
-	u_int32_t pgno, flags;
-
-	base = trunc_page(seg->ds_addr);
-	end = roundup(seg->ds_addr + seg->ds_len, PAGE_SIZE);
-	alen = end - base;
-
-	s = splhigh();
-	err = extent_alloc(sc->g_ex, alen, PAGE_SIZE, 0, dmam->_dm_boundary,
-	    EX_NOWAIT, &res);
-	splx(s);
-	if (err) {
-		printf("GART: extent_alloc %d\n", err);
-		return (err);
-	}
-
-	seg->ds_addr = res | (seg->ds_addr & PGOFSET);
-
-	for (idx = 0; idx < alen; idx += PAGE_SIZE) {
-		pgno = ((res + idx) - sc->g_pa) >> PGSHIFT;
-		flags = GART_PTE_VALID | GART_PTE_COHERENT |
-		    (((base + idx) >> 28) & GART_PTE_PHYSHI) |
-		     ((base + idx) & GART_PTE_PHYSLO);
-		sc->g_pte[pgno] = flags;
-	}
-
-	return (0);
-}
-
-int
-amdgart_iommu_unmap(struct amdgart_softc *sc, bus_dma_segment_t *seg)
-{
-	paddr_t base, end, idx;
-	psize_t alen;
-	int err, s;
-	u_int32_t pgno;
-
-	base = trunc_page(seg->ds_addr);
-	end = roundup(seg->ds_addr + seg->ds_len, PAGE_SIZE);
-	alen = end - base;
-
-	/*
-	 * order is significant here; invalidate the iommu page table
-	 * entries, then mark them as freed in the extent.
-	 */
-
-	for (idx = 0; idx < alen; idx += PAGE_SIZE) {
-		pgno = ((base - sc->g_pa) + idx) >> PGSHIFT;
-		sc->g_pte[pgno] = sc->g_scribpte;
-	}
-
-	s = splhigh();
-	err = extent_free(sc->g_ex, base, alen, EX_NOWAIT);
-	splx(s);
-	if (err) {
-		/* XXX Shouldn't happen, but if it does, I think we lose. */
-		printf("GART: extent_free %d\n", err);
-		return (err);
-	}
-
-	return (0);
-}
-
-int
-amdgart_dmamap_create(bus_dma_tag_t tag, bus_size_t size, int nsegments,
-    bus_size_t maxsegsz, bus_size_t boundary, int flags, bus_dmamap_t *dmamp)
-{
-	struct amdgart_softc *sc = tag->_cookie;
-
-	return (bus_dmamap_create(sc->g_dmat, size, nsegments,
-	    maxsegsz, boundary, flags, dmamp));
-}
-
-void
-amdgart_dmamap_destroy(bus_dma_tag_t tag, bus_dmamap_t dmam)
-{
-	struct amdgart_softc *sc = tag->_cookie;
-
-	bus_dmamap_destroy(sc->g_dmat, dmam);
-}
-
-int
-amdgart_dmamap_load(bus_dma_tag_t tag, bus_dmamap_t dmam, void *buf,
-    bus_size_t buflen, struct proc *p, int flags)
-{
-	struct amdgart_softc *sc = tag->_cookie;
-	int err;
-
-	err = bus_dmamap_load(sc->g_dmat, dmam, buf, buflen,
-	    p, flags);
-	if (err)
-		return (err);
-	err = amdgart_reload(sc, dmam);
-	if (err)
-		bus_dmamap_unload(sc->g_dmat, dmam);
-	else
-		amdgart_invalidate(sc);
-	return (err);
-}
-
-int
-amdgart_dmamap_load_mbuf(bus_dma_tag_t tag, bus_dmamap_t dmam,
-    struct mbuf *chain, int flags)
-{
-	struct amdgart_softc *sc = tag->_cookie;
-	int err;
-
-	err = bus_dmamap_load_mbuf(sc->g_dmat, dmam,
-	    chain, flags);
-	if (err)
-		return (err);
-	err = amdgart_reload(sc, dmam);
-	if (err)
-		bus_dmamap_unload(sc->g_dmat, dmam);
-	else
-		amdgart_invalidate(sc);
-	return (err);
-}
-
-int
-amdgart_dmamap_load_uio(bus_dma_tag_t tag, bus_dmamap_t dmam,
-    struct uio *uio, int flags)
-{
-	struct amdgart_softc *sc = tag->_cookie;
-	int err;
-
-	err = bus_dmamap_load_uio(sc->g_dmat, dmam, uio, flags);
-	if (err)
-		return (err);
-	err = amdgart_reload(sc, dmam);
-	if (err)
-		bus_dmamap_unload(sc->g_dmat, dmam);
-	else
-		amdgart_invalidate(sc);
-	return (err);
-}
-
-int
-amdgart_dmamap_load_raw(bus_dma_tag_t tag, bus_dmamap_t dmam,
-    bus_dma_segment_t *segs, int nsegs, bus_size_t size, int flags)
-{
-	struct amdgart_softc *sc = tag->_cookie;
-	int err;
-
-	err = bus_dmamap_load_raw(sc->g_dmat, dmam, segs, nsegs,
-	    size, flags);
-	if (err)
-		return (err);
-	err = amdgart_reload(sc, dmam);
-	if (err)
-		bus_dmamap_unload(sc->g_dmat, dmam);
-	else
-		amdgart_invalidate(sc);
-	return (err);
-}
-
-void
-amdgart_dmamap_unload(bus_dma_tag_t tag, bus_dmamap_t dmam)
-{
-	struct amdgart_softc *sc = tag->_cookie;
-	int i;
-
-	for (i = 0; i < dmam->dm_nsegs; i++)
-		amdgart_iommu_unmap(sc, &dmam->dm_segs[i]);
-	amdgart_invalidate(sc);
-	bus_dmamap_unload(sc->g_dmat, dmam);
-}
-
-void
-amdgart_dmamap_sync(bus_dma_tag_t tag, bus_dmamap_t dmam, bus_addr_t offset,
-    bus_size_t size, int ops)
-{
-	struct amdgart_softc *sc = tag->_cookie;
-
-	/*
-	 * XXX how do we deal with non-coherent mappings?  We don't
-	 * XXX allow them right now.
-	 */
-	bus_dmamap_sync(sc->g_dmat, dmam, offset, size, ops);
-}
-
-int
-amdgart_dmamem_alloc(bus_dma_tag_t tag, bus_size_t size, bus_size_t alignment,
-    bus_size_t boundary, bus_dma_segment_t *segs, int nsegs, int *rsegs,
-    int flags)
-{
-	struct amdgart_softc *sc = tag->_cookie;
-
-	return (bus_dmamem_alloc(sc->g_dmat, size, alignment,
-	    boundary, segs, nsegs, rsegs, flags));
-}
-
-void
-amdgart_dmamem_free(bus_dma_tag_t tag, bus_dma_segment_t *segs, int nsegs)
-{
-	struct amdgart_softc *sc = tag->_cookie;
-
-	bus_dmamem_free(sc->g_dmat, segs, nsegs);
-}
-
-int
-amdgart_dmamem_map(bus_dma_tag_t tag, bus_dma_segment_t *segs, int nsegs,
-    size_t size, caddr_t *kvap, int flags)
-{
-	struct amdgart_softc *sc = tag->_cookie;
-
-	return (bus_dmamem_map(sc->g_dmat, segs, nsegs, size,
-	    kvap, flags));
-}
-
-void
-amdgart_dmamem_unmap(bus_dma_tag_t tag, caddr_t kva, size_t size)
-{
-	struct amdgart_softc *sc = tag->_cookie;
-
-	bus_dmamem_unmap(sc->g_dmat, kva, size);
-}
-
-paddr_t
-amdgart_dmamem_mmap(bus_dma_tag_t tag, bus_dma_segment_t *segs, int nsegs,
-    off_t off, int prot, int flags)
-{
-	struct amdgart_softc *sc = tag->_cookie;
-
-	return (bus_dmamem_mmap(sc->g_dmat, segs, nsegs, off,
-	    prot, flags));
-}
+#endif /* !SMALL_KERNEL */
diff --git a/sys/arch/amd64/pci/pci_machdep.c b/sys/arch/amd64/pci/pci_machdep.c
index 78853bf8136..29d371f7067 100644
--- a/sys/arch/amd64/pci/pci_machdep.c
+++ b/sys/arch/amd64/pci/pci_machdep.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: pci_machdep.c,v 1.26 2009/04/13 21:23:16 kettenis Exp $	*/
+/*	$OpenBSD: pci_machdep.c,v 1.27 2009/04/21 17:05:29 oga Exp $	*/
 /*	$NetBSD: pci_machdep.c,v 1.3 2003/05/07 21:33:58 fvdl Exp $	*/
 
 /*-
@@ -175,7 +175,9 @@ pci_attach_hook(struct device *parent, struct device *self,
 {
 	if (pba->pba_bus == 0) {
 		printf(": configuration mode %d", pci_mode);
+#ifndef SMALL_KERNEL
 		amdgart_probe(pba);
+#endif /* !SMALL_KERNEL */
 	}
 }