path: root/sys/dev/pci/drm/i915_drv.c

/*
 * Copyright (c) 2008-2009 Owain G. Ainsworth <oga@openbsd.org>
 *
 * Permission to use, copy, modify, and distribute this software for any
 * purpose with or without fee is hereby granted, provided that the above
 * copyright notice and this permission notice appear in all copies.
 *
 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 */
/*-
 * Copyright © 2008 Intel Corporation
 * Copyright 2003 Tungsten Graphics, Inc., Cedar Park, Texas.
 * copyright 2000 VA Linux Systems, Inc., Sunnyvale, California.
 * All Rights Reserved.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice (including the next
 * paragraph) shall be included in all copies or substantial portions of the
 * Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 * VA LINUX SYSTEMS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
 * OTHER DEALINGS IN THE SOFTWARE.
 *
 * Authors:
 *    Gareth Hughes <gareth@valinux.com>
 *    Eric Anholt <eric@anholt.net>
 *
 */

#include "drmP.h"
#include "drm.h"
#include "i915_drm.h"
#include "i915_drv.h"

#include <machine/pmap.h>

#include <sys/queue.h>
#include <sys/workq.h>
#if 0
#	define INTELDRM_WATCH_COHERENCY
#	define WATCH_INACTIVE
#endif

#define I915_GEM_GPU_DOMAINS	(~(I915_GEM_DOMAIN_CPU | I915_GEM_DOMAIN_GTT))

int	inteldrm_probe(struct device *, void *, void *);
void	inteldrm_attach(struct device *, struct device *, void *);
int	inteldrm_detach(struct device *, int);
int	inteldrm_activate(struct device *, int);
int	inteldrm_ioctl(struct drm_device *, u_long, caddr_t, struct drm_file *);
int	inteldrm_intr(void *);
void	inteldrm_lastclose(struct drm_device *);

void	inteldrm_wrap_ring(struct drm_i915_private *);
int	inteldrm_gmch_match(struct pci_attach_args *);
void	inteldrm_chipset_flush(struct drm_i915_private *);
void	inteldrm_timeout(void *);
void	inteldrm_hangcheck(void *);
void	inteldrm_hung(void *, void *);
void	inteldrm_965_reset(struct drm_i915_private *, u_int8_t);
int	inteldrm_fault(struct drm_obj *, struct uvm_faultinfo *, off_t,
	    vaddr_t, vm_page_t *, int, int, vm_prot_t, int );
void	inteldrm_wipe_mappings(struct drm_obj *);
void	inteldrm_purge_obj(struct drm_obj *);

/* For reset and suspend */
int	inteldrm_save_state(struct drm_i915_private *);
int	inteldrm_restore_state(struct drm_i915_private *);
int	inteldrm_save_display(struct drm_i915_private *);
int	inteldrm_restore_display(struct drm_i915_private *);
void	i915_save_vga(struct drm_i915_private *);
void	i915_restore_vga(struct drm_i915_private *);
void	i915_save_modeset_reg(struct drm_i915_private *);
void	i915_restore_modeset_reg(struct drm_i915_private *);
u_int8_t	i915_read_indexed(struct drm_i915_private *, u_int16_t,
		    u_int16_t, u_int8_t);
void	i915_write_indexed(struct drm_i915_private *, u_int16_t,
	    u_int16_t, u_int8_t, u_int8_t);
void	i915_write_ar(struct drm_i915_private *, u_int16_t, u_int8_t,
	    u_int8_t, u_int16_t);
u_int8_t	i915_read_ar(struct drm_i915_private *, u_int16_t,
		    u_int8_t, u_int16_t);
void	i915_save_palette(struct drm_i915_private *, enum pipe);
void	i915_restore_palette(struct drm_i915_private *, enum pipe);

void	i915_alloc_ifp(struct drm_i915_private *, struct pci_attach_args *);
void	i965_alloc_ifp(struct drm_i915_private *, struct pci_attach_args *);

void	inteldrm_detect_bit_6_swizzle(drm_i915_private_t *,
	    struct pci_attach_args *);

int	inteldrm_setup_mchbar(struct drm_i915_private *,
	    struct pci_attach_args *);
void	inteldrm_teardown_mchbar(struct drm_i915_private *,
	    struct pci_attach_args *, int);

/* Ioctls */
int	i915_gem_init_ioctl(struct drm_device *, void *, struct drm_file *);
int	i915_gem_create_ioctl(struct drm_device *, void *, struct drm_file *);
int	i915_gem_pread_ioctl(struct drm_device *, void *, struct drm_file *);
int	i915_gem_pwrite_ioctl(struct drm_device *, void *, struct drm_file *);
int	i915_gem_set_domain_ioctl(struct drm_device *, void *,
	    struct drm_file *);
int	i915_gem_execbuffer2(struct drm_device *, void *, struct drm_file *);
int	i915_gem_pin_ioctl(struct drm_device *, void *, struct drm_file *);
int	i915_gem_unpin_ioctl(struct drm_device *, void *, struct drm_file *);
int	i915_gem_busy_ioctl(struct drm_device *, void *, struct drm_file *);
int	i915_gem_entervt_ioctl(struct drm_device *, void *, struct drm_file *);
int	i915_gem_leavevt_ioctl(struct drm_device *, void *, struct drm_file *);
int	i915_gem_get_aperture_ioctl(struct drm_device *, void *,
	    struct drm_file *);
int	i915_gem_set_tiling(struct drm_device *, void *, struct drm_file *);
int	i915_gem_get_tiling(struct drm_device *, void *, struct drm_file *);
int	i915_gem_gtt_map_ioctl(struct drm_device *, void *, struct drm_file *);
int	i915_gem_madvise_ioctl(struct drm_device *, void *, struct drm_file *);

/* GEM memory manager functions */
int	i915_gem_init_object(struct drm_obj *);
void	i915_gem_free_object(struct drm_obj *);
int	i915_gem_object_pin(struct drm_obj *, uint32_t, int);
void	i915_gem_object_unpin(struct drm_obj *);
void	i915_gem_retire_requests(struct drm_i915_private *);
void	i915_gem_retire_request(struct drm_i915_private *,
	    struct inteldrm_request *);
void	i915_gem_retire_work_handler(void *, void*);
int	i915_gem_idle(struct drm_i915_private *);
void	i915_gem_object_move_to_active(struct drm_obj *);
void	i915_gem_object_move_off_active(struct drm_obj *);
void	i915_gem_object_move_to_inactive(struct drm_obj *);
uint32_t	i915_add_request(struct drm_i915_private *);
void	inteldrm_process_flushing(struct drm_i915_private *, u_int32_t);
void	i915_move_to_tail(struct inteldrm_obj *, struct i915_gem_list *);
void	i915_list_remove(struct inteldrm_obj *);
int	i915_gem_init_hws(struct drm_i915_private *);
void	i915_gem_cleanup_hws(struct drm_i915_private *);
int	i915_gem_init_ringbuffer(struct drm_i915_private *);
int	inteldrm_start_ring(struct drm_i915_private *);
void	i915_gem_cleanup_ringbuffer(struct drm_i915_private *);
int	i915_gem_ring_throttle(struct drm_device *, struct drm_file *);
int	i915_gem_evict_inactive(struct drm_i915_private *);
int	i915_gem_get_relocs_from_user(struct drm_i915_gem_exec_object2 *,
	    u_int32_t, struct drm_i915_gem_relocation_entry **);
int	i915_gem_put_relocs_to_user(struct drm_i915_gem_exec_object2 *,
	    u_int32_t, struct drm_i915_gem_relocation_entry *);
void	i915_dispatch_gem_execbuffer(struct drm_device *,
	    struct drm_i915_gem_execbuffer2 *, uint64_t);
void	i915_gem_object_set_to_gpu_domain(struct drm_obj *);
int	i915_gem_object_pin_and_relocate(struct drm_obj *,
	    struct drm_file *, struct drm_i915_gem_exec_object2 *,
	    struct drm_i915_gem_relocation_entry *);
int	i915_gem_object_bind_to_gtt(struct drm_obj *, bus_size_t, int);
int	i915_wait_request(struct drm_i915_private *, uint32_t, int);
u_int32_t	i915_gem_flush(struct drm_i915_private *, uint32_t, uint32_t);
int	i915_gem_object_unbind(struct drm_obj *, int);

struct drm_obj	*i915_gem_find_inactive_object(struct drm_i915_private *,
		     size_t);

int	i915_gem_evict_everything(struct drm_i915_private *, int);
int	i915_gem_evict_something(struct drm_i915_private *, size_t, int);
int	i915_gem_object_set_to_gtt_domain(struct drm_obj *, int, int);
int	i915_gem_object_set_to_cpu_domain(struct drm_obj *, int, int);
int	i915_gem_object_flush_gpu_write_domain(struct drm_obj *, int, int);
int	i915_gem_get_fence_reg(struct drm_obj *, int);
int	i915_gem_object_put_fence_reg(struct drm_obj *, int);
bus_size_t	i915_gem_get_gtt_alignment(struct drm_obj *);

bus_size_t	i915_get_fence_size(struct drm_i915_private *, bus_size_t);
int	i915_tiling_ok(struct drm_device *, int, int, int);
int	i915_gem_object_fence_offset_ok(struct drm_obj *, int);
void	i965_write_fence_reg(struct inteldrm_fence *);
void	i915_write_fence_reg(struct inteldrm_fence *);
void	i830_write_fence_reg(struct inteldrm_fence *);
void	i915_gem_bit_17_swizzle(struct drm_obj *);
void	i915_gem_save_bit_17_swizzle(struct drm_obj *);
int	inteldrm_swizzle_page(struct vm_page *page);

/* Debug functions, mostly called from ddb */
void	i915_gem_seqno_info(int);
void	i915_interrupt_info(int);
void	i915_gem_fence_regs_info(int);
void	i915_hws_info(int);
void	i915_batchbuffer_info(int);
void	i915_ringbuffer_data(int);
void	i915_ringbuffer_info(int);
#ifdef WATCH_INACTIVE
void inteldrm_verify_inactive(struct drm_i915_private *, char *, int);
#else
#define inteldrm_verify_inactive(dev,file,line)
#endif

const static struct drm_pcidev inteldrm_pciidlist[] = {
	{PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82830M_IGD,
	    CHIP_I830|CHIP_M},
	{PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82845G_IGD,
	    CHIP_I845G},
	{PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82855GM_IGD,
	    CHIP_I85X|CHIP_M},
	{PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82865G_IGD,
	    CHIP_I865G},
	{PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82915G_IGD_1,
	    CHIP_I915G|CHIP_I9XX},
	{PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_E7221_IGD,
	    CHIP_I915G|CHIP_I9XX},
	{PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82915GM_IGD_1,
	    CHIP_I915GM|CHIP_I9XX|CHIP_M},
	{PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82945G_IGD_1,
	    CHIP_I945G|CHIP_I9XX},
	{PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82945GM_IGD_1,
	    CHIP_I945GM|CHIP_I9XX|CHIP_M},
	{PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82945GME_IGD_1,
	    CHIP_I945GM|CHIP_I9XX|CHIP_M},
	{PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82946GZ_IGD_1,
	    CHIP_I965|CHIP_I9XX},
	{PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82G35_IGD_1,
	    CHIP_I965|CHIP_I9XX},
	{PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82Q965_IGD_1,
	    CHIP_I965|CHIP_I9XX},
	{PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82G965_IGD_1,
	    CHIP_I965|CHIP_I9XX},
	{PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82GM965_IGD_1,
	    CHIP_I965GM|CHIP_I965|CHIP_I9XX|CHIP_M},
	{PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82GME965_IGD_1,
	    CHIP_I965|CHIP_I9XX},
	{PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82G33_IGD_1,
	    CHIP_G33|CHIP_I9XX|CHIP_HWS},
	{PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82Q35_IGD_1,
	    CHIP_G33|CHIP_I9XX|CHIP_HWS},
	{PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82Q33_IGD_1,
	    CHIP_G33|CHIP_I9XX|CHIP_HWS},
	{PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82GM45_IGD_1,
	    CHIP_G4X|CHIP_GM45|CHIP_I965|CHIP_I9XX|CHIP_M|CHIP_HWS},
	{PCI_VENDOR_INTEL, 0x2E02,
	    CHIP_G4X|CHIP_I965|CHIP_I9XX|CHIP_HWS},
	{PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82Q45_IGD_1,
	    CHIP_G4X|CHIP_I965|CHIP_I9XX|CHIP_HWS},
	{PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82G45_IGD_1,
	    CHIP_G4X|CHIP_I965|CHIP_I9XX|CHIP_HWS},
	{PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82G41_IGD_1,
	    CHIP_G4X|CHIP_I965|CHIP_I9XX|CHIP_HWS},
	{0, 0, 0}
};

static const struct drm_driver_info inteldrm_driver = {
	.buf_priv_size		= 1,	/* No dev_priv */
	.file_priv_size		= sizeof(struct inteldrm_file),
	.ioctl			= inteldrm_ioctl,
	.lastclose		= inteldrm_lastclose,
	.vblank_pipes		= 2,
	.get_vblank_counter	= i915_get_vblank_counter,
	.enable_vblank		= i915_enable_vblank,
	.disable_vblank		= i915_disable_vblank,
	.irq_install		= i915_driver_irq_install,
	.irq_uninstall		= i915_driver_irq_uninstall,

#ifdef INTELDRM_GEM
	.gem_init_object	= i915_gem_init_object,
	.gem_free_object	= i915_gem_free_object,
	.gem_fault		= inteldrm_fault,
	.gem_size		= sizeof(struct inteldrm_obj),
#endif /* INTELDRM_GEM */

	.name			= DRIVER_NAME,
	.desc			= DRIVER_DESC,
	.date			= DRIVER_DATE,
	.major			= DRIVER_MAJOR,
	.minor			= DRIVER_MINOR,
	.patchlevel		= DRIVER_PATCHLEVEL,

	.flags			= DRIVER_AGP | DRIVER_AGP_REQUIRE |
				    DRIVER_MTRR | DRIVER_IRQ
#ifdef INTELDRM_GEM
				    | DRIVER_GEM,
#endif /* INTELDRM_GEM */
};

int
inteldrm_probe(struct device *parent, void *match, void *aux)
{
	return (drm_pciprobe((struct pci_attach_args *)aux,
	    inteldrm_pciidlist));
}

/*
 * We're intel IGD, bus 0 function 0 dev 0 should be the GMCH, so it should
 * be Intel
 */
int
inteldrm_gmch_match(struct pci_attach_args *pa)
{
	if (pa->pa_bus == 0 && pa->pa_device == 0 && pa->pa_function == 0 &&
	    PCI_VENDOR(pa->pa_id) == PCI_VENDOR_INTEL &&
	    PCI_CLASS(pa->pa_class) == PCI_CLASS_BRIDGE &&
	    PCI_SUBCLASS(pa->pa_class) == PCI_SUBCLASS_BRIDGE_HOST)
		return (1);
	return (0);
}

void
inteldrm_attach(struct device *parent, struct device *self, void *aux)
{
	struct drm_i915_private	*dev_priv = (struct drm_i915_private *)self;
	struct pci_attach_args	*pa = aux, bpa;
	struct vga_pci_bar	*bar;
	struct drm_device 	*dev;
	const struct drm_pcidev	*id_entry;
	int			 i;

	id_entry = drm_find_description(PCI_VENDOR(pa->pa_id),
	    PCI_PRODUCT(pa->pa_id), inteldrm_pciidlist);
	dev_priv->flags = id_entry->driver_private;
	dev_priv->pci_device = PCI_PRODUCT(pa->pa_id);

	dev_priv->pc = pa->pa_pc;
	dev_priv->tag = pa->pa_tag;
	dev_priv->dmat = pa->pa_dmat;
	dev_priv->bst = pa->pa_memt;

	/* we need to use this api for now due to sharing with intagp */
	bar = vga_pci_bar_info((struct vga_pci_softc *)parent,
	    (IS_I9XX(dev_priv) ? 0 : 1));
	if (bar == NULL) {
		printf(": can't get BAR info\n");
		return;
	}

	dev_priv->regs = vga_pci_bar_map((struct vga_pci_softc *)parent, 
	    bar->addr, 0, 0);
	if (dev_priv->regs == NULL) {
		printf(": can't map mmio space\n");
		return;
	}

	if (pci_intr_map(pa, &dev_priv->ih) != 0) {
		printf(": couldn't map interrupt\n");
		return;
	}

	/*
	 * set up interrupt handler, note that we don't switch the interrupt
	 * on until the X server talks to us, kms will change this.
	 */
	dev_priv->irqh = pci_intr_establish(dev_priv->pc, dev_priv->ih, IPL_TTY,
	    inteldrm_intr, dev_priv, dev_priv->dev.dv_xname);
	if (dev_priv->irqh == NULL) {
		printf(": couldn't  establish interrupt\n");
		return;
	}

	/* Unmask the interrupts that we always want on. */
	dev_priv->irq_mask_reg = ~I915_INTERRUPT_ENABLE_FIX;

#ifdef INTELDRM_GEM
	dev_priv->workq = workq_create("intelrel", 1, IPL_TTY);
	if (dev_priv->workq == NULL) {
		printf("couldn't create workq\n");
		return;
	}

	/* GEM init */
	TAILQ_INIT(&dev_priv->mm.active_list);
	TAILQ_INIT(&dev_priv->mm.flushing_list);
	TAILQ_INIT(&dev_priv->mm.inactive_list);
	TAILQ_INIT(&dev_priv->mm.gpu_write_list);
	TAILQ_INIT(&dev_priv->mm.request_list);
	TAILQ_INIT(&dev_priv->mm.fence_list);
	timeout_set(&dev_priv->mm.retire_timer, inteldrm_timeout, dev_priv);
	timeout_set(&dev_priv->mm.hang_timer, inteldrm_hangcheck, dev_priv);
	dev_priv->mm.next_gem_seqno = 1;
	dev_priv->mm.suspended = 1;
#endif /* INTELDRM_GEM */

	/* For the X server, in kms mode this will not be needed */
	dev_priv->fence_reg_start = 3;

	if (IS_I965G(dev_priv) || IS_I945G(dev_priv) || IS_I945GM(dev_priv) ||
	    IS_G33(dev_priv))
		dev_priv->num_fence_regs = 16;
	else
		dev_priv->num_fence_regs = 8;
	/* Initialise fences to zero, else on some macs we'll get corruption */
	if (IS_I965G(dev_priv)) {
		for (i = 0; i < 16; i++)
			I915_WRITE64(FENCE_REG_965_0 + (i * 8), 0);
	} else {
		for (i = 0; i < 8; i++)
			I915_WRITE(FENCE_REG_830_0 + (i * 4), 0);
		if (IS_I945G(dev_priv) || IS_I945GM(dev_priv) ||
		    IS_G33(dev_priv))
			for (i = 0; i < 8; i++)
				I915_WRITE(FENCE_REG_945_8 + (i * 4), 0);
	}

	if (pci_find_device(&bpa, inteldrm_gmch_match) == 0) {
		printf(": can't find GMCH\n");
		return;
	}

	/* Set up the IFP for chipset flushing */
	if (dev_priv->flags & (CHIP_I915G|CHIP_I915GM|CHIP_I945G|CHIP_I945GM)) {
		i915_alloc_ifp(dev_priv, &bpa);
	} else if (IS_I965G(dev_priv) || IS_G33(dev_priv)) {
		i965_alloc_ifp(dev_priv, &bpa);	
	} else {
		int nsegs;
		/*
		 * I8XX has no flush page mechanism, we fake it by writing until
		 * the cache is empty. allocate a page to scribble on
		 */
		dev_priv->ifp.i8xx.kva = NULL;
		if (bus_dmamem_alloc(pa->pa_dmat, PAGE_SIZE, 0, 0,
		    &dev_priv->ifp.i8xx.seg, 1, &nsegs, BUS_DMA_WAITOK) == 0) {
			if (bus_dmamem_map(pa->pa_dmat, &dev_priv->ifp.i8xx.seg,
			    1, PAGE_SIZE, &dev_priv->ifp.i8xx.kva, 0) != 0) {
				bus_dmamem_free(pa->pa_dmat,
				    &dev_priv->ifp.i8xx.seg, nsegs);
				dev_priv->ifp.i8xx.kva = NULL;
			}
		}
	}

#ifdef INTELDRM_GEM
	inteldrm_detect_bit_6_swizzle(dev_priv, &bpa);
#endif /* INTELDRM_GEM */
	/* Init HWS */
	if (!I915_NEED_GFX_HWS(dev_priv)) {
		if (i915_init_phys_hws(dev_priv, pa->pa_dmat) != 0) {
			printf(": couldn't alloc HWS page\n");
			return;
		}
	}

	printf(": %s\n", pci_intr_string(pa->pa_pc, dev_priv->ih));

	mtx_init(&dev_priv->user_irq_lock, IPL_TTY);

	/* All intel chipsets need to be treated as agp, so just pass one */
	dev_priv->drmdev = drm_attach_pci(&inteldrm_driver, pa, 1, self);

	dev = (struct drm_device *)dev_priv->drmdev;

#ifdef INTELDRM_GEM
	/* XXX would be a lot nicer to get agp info before now */
	uvm_page_physload_flags(atop(dev->agp->base), atop(dev->agp->base +
	    dev->agp->info.ai_aperture_size), atop(dev->agp->base),
	    atop(dev->agp->base + dev->agp->info.ai_aperture_size), 0,
	    PHYSLOAD_DEVICE);
	/* array of vm pages that physload introduced. */
	dev_priv->pgs = PHYS_TO_VM_PAGE(dev->agp->base);
	KASSERT(dev_priv->pgs != NULL);
	/* XXX wc and do earlier */
	if (bus_space_map(dev_priv->bst, dev->agp->base,
	    dev->agp->info.ai_aperture_size, BUS_SPACE_MAP_LINEAR,
	    &dev_priv->aperture_bsh) != 0)
		panic("can't map aperture");
#endif /* INTELDRM_GEM */
}

int
inteldrm_detach(struct device *self, int flags)
{
	struct drm_i915_private *dev_priv = (struct drm_i915_private *)self;

	/* this will quiesce any dma that's going on and kill the timeouts. */
	if (dev_priv->drmdev != NULL) {
		config_detach(dev_priv->drmdev, flags);
		dev_priv->drmdev = NULL;
	}

	i915_free_hws(dev_priv, dev_priv->dmat);

	if (IS_I9XX(dev_priv) && dev_priv->ifp.i9xx.bsh != NULL) {
		bus_space_unmap(dev_priv->ifp.i9xx.bst, dev_priv->ifp.i9xx.bsh,
		    PAGE_SIZE);
	} else if (dev_priv->flags & (CHIP_I830 | CHIP_I845G | CHIP_I85X |
	    CHIP_I865G) && dev_priv->ifp.i8xx.kva != NULL) {
		bus_dmamem_unmap(dev_priv->dmat, dev_priv->ifp.i8xx.kva,
		     PAGE_SIZE);
		bus_dmamem_free(dev_priv->dmat, &dev_priv->ifp.i8xx.seg, 1);
	}

	pci_intr_disestablish(dev_priv->pc, dev_priv->irqh);

	if (dev_priv->regs != NULL)
		vga_pci_bar_unmap(dev_priv->regs);

	return (0);
}

int
inteldrm_activate(struct device *arg, int act)
{
	struct drm_i915_private	*dev_priv = (struct drm_i915_private *)arg;

	switch (act) {
	case DVACT_SUSPEND:
		inteldrm_save_state(dev_priv);
		break;
	case DVACT_RESUME:
		inteldrm_restore_state(dev_priv);
		break;
	}

	return (0);
}

struct cfattach inteldrm_ca = {
	sizeof(struct drm_i915_private), inteldrm_probe, inteldrm_attach,
	inteldrm_detach, inteldrm_activate
};

struct cfdriver inteldrm_cd = {
	0, "inteldrm", DV_DULL
};

int
inteldrm_ioctl(struct drm_device *dev, u_long cmd, caddr_t data,
    struct drm_file *file_priv)
{
	if (file_priv->authenticated == 1) {
		switch (cmd) {
		case DRM_IOCTL_I915_FLUSH:
			return (i915_flush_ioctl(dev, data, file_priv));
		case DRM_IOCTL_I915_BATCHBUFFER:
			return (i915_batchbuffer(dev, data, file_priv));
		case DRM_IOCTL_I915_IRQ_EMIT:
			return (i915_irq_emit(dev, data, file_priv));
		case DRM_IOCTL_I915_IRQ_WAIT:
			return (i915_irq_wait(dev, data, file_priv));
		case DRM_IOCTL_I915_GETPARAM:
			return (i915_getparam(dev, data, file_priv));
		case DRM_IOCTL_I915_CMDBUFFER:
			return (i915_cmdbuffer(dev, data, file_priv));
		case DRM_IOCTL_I915_GET_VBLANK_PIPE:
			return (i915_vblank_pipe_get(dev, data, file_priv));
#ifdef INTELDRM_GEM
		case DRM_IOCTL_I915_GEM_EXECBUFFER2:
			return (i915_gem_execbuffer2(dev, data, file_priv));
		case DRM_IOCTL_I915_GEM_BUSY:
			return (i915_gem_busy_ioctl(dev, data, file_priv));
		case DRM_IOCTL_I915_GEM_THROTTLE:
			return (i915_gem_ring_throttle(dev, file_priv));
		case DRM_IOCTL_I915_GEM_MMAP:
			return (i915_gem_gtt_map_ioctl(dev, data, file_priv));
		case DRM_IOCTL_I915_GEM_CREATE:
			return (i915_gem_create_ioctl(dev, data, file_priv));
		case DRM_IOCTL_I915_GEM_PREAD:
			return (i915_gem_pread_ioctl(dev, data, file_priv));
		case DRM_IOCTL_I915_GEM_PWRITE:
			return (i915_gem_pwrite_ioctl(dev, data, file_priv));
		case DRM_IOCTL_I915_GEM_SET_DOMAIN:
			return (i915_gem_set_domain_ioctl(dev, data,
			    file_priv));
		case DRM_IOCTL_I915_GEM_SET_TILING:
			return (i915_gem_set_tiling(dev, data, file_priv));
		case DRM_IOCTL_I915_GEM_GET_TILING:
			return (i915_gem_get_tiling(dev, data, file_priv));
		case DRM_IOCTL_I915_GEM_GET_APERTURE:
			return (i915_gem_get_aperture_ioctl(dev, data,
			    file_priv));
		case DRM_IOCTL_I915_GEM_MADVISE:
			return (i915_gem_madvise_ioctl(dev, data, file_priv));
#endif /* INTELDRM_GEM */
		default:
			break;
		}
	}

	if (file_priv->master == 1) {
		switch (cmd) {
		case DRM_IOCTL_I915_SETPARAM:
			return (i915_setparam(dev, data, file_priv));
		case DRM_IOCTL_I915_INIT:
			return (i915_dma_init(dev, data, file_priv));
		case DRM_IOCTL_I915_HWS_ADDR:
			return (i915_set_status_page(dev, data, file_priv));
		/* Removed, but still used by userland, so just say `fine' */
		case DRM_IOCTL_I915_INIT_HEAP:
		case DRM_IOCTL_I915_DESTROY_HEAP:
		case DRM_IOCTL_I915_SET_VBLANK_PIPE:
			return (0);
#ifdef INTELDRM_GEM
		case DRM_IOCTL_I915_GEM_INIT:
			return (i915_gem_init_ioctl(dev, data, file_priv));
		case DRM_IOCTL_I915_GEM_ENTERVT:
			return (i915_gem_entervt_ioctl(dev, data, file_priv));
		case DRM_IOCTL_I915_GEM_LEAVEVT:
			return (i915_gem_leavevt_ioctl(dev, data, file_priv));
		case DRM_IOCTL_I915_GEM_PIN:
			return (i915_gem_pin_ioctl(dev, data, file_priv));
		case DRM_IOCTL_I915_GEM_UNPIN:
			return (i915_gem_unpin_ioctl(dev, data, file_priv));
#endif /* INTELDRM_GEM */
		}
	}
	return (EINVAL);
}

int
inteldrm_intr(void *arg)
{
	drm_i915_private_t	*dev_priv = arg;
	struct drm_device	*dev = (struct drm_device *)dev_priv->drmdev;
	u_int32_t		 iir, pipea_stats = 0, pipeb_stats = 0;

	/* we're not set up, don't poke the hw */
	if (dev_priv->hw_status_page == NULL)
		return (0);
	/*
	 * lock is to protect from writes to PIPESTAT and IMR from other cores.
	 */
	mtx_enter(&dev_priv->user_irq_lock);
	iir = I915_READ(IIR);
	if (iir == 0) {
		mtx_leave(&dev_priv->user_irq_lock);
		return (0);
	}

	/*
	 * Clear the PIPE(A|B)STAT regs before the IIR
	 */
	if (iir & I915_DISPLAY_PIPE_A_EVENT_INTERRUPT) {
		pipea_stats = I915_READ(PIPEASTAT);
		I915_WRITE(PIPEASTAT, pipea_stats);
	}
	if (iir & I915_DISPLAY_PIPE_B_EVENT_INTERRUPT) {
		pipeb_stats = I915_READ(PIPEBSTAT);
		I915_WRITE(PIPEBSTAT, pipeb_stats);
	}
#ifdef INTELDRM_GEM
	if (iir & I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT)
		inteldrm_error(dev_priv);
#endif /* INTELDRM_GEM */

	I915_WRITE(IIR, iir);
	(void)I915_READ(IIR); /* Flush posted writes */

	if (dev_priv->sarea_priv != NULL)
		dev_priv->sarea_priv->last_dispatch = READ_BREADCRUMB(dev_priv);

	if (iir & I915_USER_INTERRUPT) {
		wakeup(dev_priv);
#ifdef INTELDRM_GEM
		dev_priv->mm.hang_cnt = 0;
		timeout_add_msec(&dev_priv->mm.hang_timer, 750);
#endif /* INTELDRM_GEM */
	}

	mtx_leave(&dev_priv->user_irq_lock);

	if (pipea_stats & I915_VBLANK_INTERRUPT_STATUS)
		drm_handle_vblank(dev, 0);

	if (pipeb_stats & I915_VBLANK_INTERRUPT_STATUS)
		drm_handle_vblank(dev, 1);

	return (1);
}

u_int32_t
inteldrm_read_hws(struct drm_i915_private *dev_priv, int reg)
{
	struct drm_device	*dev = (struct drm_device *)dev_priv->drmdev;
	u_int32_t		 val;

	/*
	 * When we eventually go GEM only we'll always have a dmamap, so this
	 * madness won't be for long.
	 */
	if (dev_priv->hws_dmamem)
		bus_dmamap_sync(dev->dmat, dev_priv->hws_dmamem->map, 0,
		    PAGE_SIZE, BUS_DMASYNC_POSTREAD);
	
	val = ((volatile u_int32_t *)(dev_priv->hw_status_page))[reg];

	if (dev_priv->hws_dmamem)
		bus_dmamap_sync(dev->dmat, dev_priv->hws_dmamem->map, 0,
		    PAGE_SIZE, BUS_DMASYNC_PREREAD);
	return (val);
}

/*
 * These five ring manipulation functions are protected by dev->dev_lock.
 */
int
inteldrm_wait_ring(struct drm_i915_private *dev_priv, int n)
{
	struct inteldrm_ring	*ring = &dev_priv->ring;
	u_int32_t		 acthd_reg, acthd, last_acthd, last_head;
	int			 i;

	acthd_reg = IS_I965G(dev_priv) ? ACTHD_I965 : ACTHD;
	last_head = I915_READ(PRB0_HEAD) & HEAD_ADDR;
	last_acthd = I915_READ(acthd_reg);

	/* ugh. Could really do with a proper, resettable timer here. */
	for (i = 0; i < 100000; i++) {
		ring->head = I915_READ(PRB0_HEAD) & HEAD_ADDR;
		acthd = I915_READ(acthd_reg);
		ring->space = ring->head - (ring->tail + 8);

		INTELDRM_VPRINTF("%s: head: %x tail: %x space: %x\n", __func__,
			ring->head, ring->tail, ring->space);
		if (ring->space < 0)
			ring->space += ring->size;
		if (ring->space >= n)
			return (0);

		/* Only timeout if the ring isn't chewing away on something */
		if (ring->head != last_head || acthd != last_acthd)
			i = 0;

		last_head = ring->head;
		last_acthd = acthd;
		delay(10);
	}

	return (EBUSY);
}

void
inteldrm_wrap_ring(struct drm_i915_private *dev_priv)
{
	u_int32_t	rem;;

	rem = dev_priv->ring.size - dev_priv->ring.tail;
	if (dev_priv->ring.space < rem &&
	    inteldrm_wait_ring(dev_priv, rem) != 0)
			return; /* XXX */

	dev_priv->ring.space -= rem;

	bus_space_set_region_4(dev_priv->bst, dev_priv->ring.bsh,
	    dev_priv->ring.woffset, MI_NOOP, rem / 4);

	dev_priv->ring.tail = 0;
}

void
inteldrm_begin_ring(struct drm_i915_private *dev_priv, int ncmd)
{
	int	bytes = 4 * ncmd;

	INTELDRM_VPRINTF("%s: %d\n", __func__, ncmd);
	if (dev_priv->ring.tail + bytes > dev_priv->ring.size)
		inteldrm_wrap_ring(dev_priv);
	if (dev_priv->ring.space < bytes)
		inteldrm_wait_ring(dev_priv, bytes);
	dev_priv->ring.woffset = dev_priv->ring.tail;
	dev_priv->ring.tail += bytes;
	dev_priv->ring.tail &= dev_priv->ring.size - 1;
	dev_priv->ring.space -= bytes;
}

void
inteldrm_out_ring(struct drm_i915_private *dev_priv, u_int32_t cmd)
{
	INTELDRM_VPRINTF("%s: %x\n", __func__, cmd);
	bus_space_write_4(dev_priv->bst, dev_priv->ring.bsh,
	    dev_priv->ring.woffset, cmd);
	/*
	 * don't need to deal with wrap here because we padded
	 * the ring out if we would wrap
	 */
	dev_priv->ring.woffset += 4;
}

void
inteldrm_advance_ring(struct drm_i915_private *dev_priv)
{
	INTELDRM_VPRINTF("%s: %x, %x\n", __func__, dev_priv->ring.wspace,
	    dev_priv->ring.woffset);
	I915_WRITE(PRB0_TAIL, dev_priv->ring.tail);
}

void
inteldrm_update_ring(struct drm_i915_private *dev_priv)
{
	struct inteldrm_ring	*ring = &dev_priv->ring;

	ring->head = (I915_READ(PRB0_HEAD) & HEAD_ADDR);
	ring->tail = (I915_READ(PRB0_TAIL) & TAIL_ADDR);
	ring->space = ring->head - (ring->tail + 8);
	if (ring->space < 0)
		ring->space += ring->size;
	INTELDRM_VPRINTF("%s: head: %x tail: %x space: %x\n", __func__,
		ring->head, ring->tail, ring->space);
}

void
i915_alloc_ifp(struct drm_i915_private *dev_priv, struct pci_attach_args *bpa)
{
	bus_addr_t	addr;
	u_int32_t	reg;

	dev_priv->ifp.i9xx.bst = bpa->pa_memt;

	reg = pci_conf_read(bpa->pa_pc, bpa->pa_tag, I915_IFPADDR);
	if (reg & 0x1) {
		addr = (bus_addr_t)reg;
		addr &= ~0x1;
		/* XXX extents ... need data on whether bioses alloc or not. */
		if (bus_space_map(bpa->pa_memt, addr, PAGE_SIZE, 0,
		    &dev_priv->ifp.i9xx.bsh) != 0)
			goto nope;
		return;
	} else if (bpa->pa_memex == NULL || extent_alloc(bpa->pa_memex,
	    PAGE_SIZE, PAGE_SIZE, 0, 0, 0, &addr) || bus_space_map(bpa->pa_memt,
	    addr, PAGE_SIZE, 0, &dev_priv->ifp.i9xx.bsh))
		goto nope;

	pci_conf_write(bpa->pa_pc, bpa->pa_tag, I915_IFPADDR, addr | 0x1);
	
	return;

nope:
	dev_priv->ifp.i9xx.bsh = NULL;
	printf(": no ifp ");
}

void
i965_alloc_ifp(struct drm_i915_private *dev_priv, struct pci_attach_args *bpa)
{
	bus_addr_t	addr;
	u_int32_t	lo, hi;

	dev_priv->ifp.i9xx.bst = bpa->pa_memt;

	hi = pci_conf_read(bpa->pa_pc, bpa->pa_tag, I965_IFPADDR + 4);
	lo = pci_conf_read(bpa->pa_pc, bpa->pa_tag, I965_IFPADDR);
	if (lo & 0x1) {
		addr = (((u_int64_t)hi << 32) | lo);
		addr &= ~0x1;
		/* XXX extents ... need data on whether bioses alloc or not. */
		if (bus_space_map(bpa->pa_memt, addr, PAGE_SIZE, 0,
		    &dev_priv->ifp.i9xx.bsh) != 0)
			goto nope;
		return;
	} else if (bpa->pa_memex == NULL || extent_alloc(bpa->pa_memex,
	    PAGE_SIZE, PAGE_SIZE, 0, 0, 0, &addr) || bus_space_map(bpa->pa_memt,
	    addr, PAGE_SIZE, 0, &dev_priv->ifp.i9xx.bsh))
		goto nope;

	pci_conf_write(bpa->pa_pc, bpa->pa_tag, I965_IFPADDR + 4,
	    upper_32_bits(addr));
	pci_conf_write(bpa->pa_pc, bpa->pa_tag, I965_IFPADDR,
	    (addr & 0xffffffff) | 0x1);
	
	return;

nope:
	dev_priv->ifp.i9xx.bsh = NULL;
	printf(": no ifp ");
}

void
inteldrm_chipset_flush(struct drm_i915_private *dev_priv)
{
	/*
	 * Write to this flush page flushes the chipset write cache.
	 * The write will return when it is done.
	 */
	if (IS_I9XX(dev_priv)) {
	    if (dev_priv->ifp.i9xx.bsh != NULL)
		bus_space_write_4(dev_priv->ifp.i9xx.bst,
		    dev_priv->ifp.i9xx.bsh, 0, 1);
	} else {
		/*
		 * I8XX don't have a flush page mechanism, but do have the
		 * cache. Do it the bruteforce way. we write 1024 byes into
		 * the cache, then clflush them out so they'll kick the stuff
		 * we care about out of the chipset cache.
		 */
		if (dev_priv->ifp.i8xx.kva != NULL) {
			memset(dev_priv->ifp.i8xx.kva, 0, 1024);
			agp_flush_cache_range((vaddr_t)dev_priv->ifp.i8xx.kva,
			    1024);
		}
	}
}

void
inteldrm_lastclose(struct drm_device *dev)
{
	drm_i915_private_t	*dev_priv = dev->dev_private;
#ifdef INTELDRM_GEM
	struct vm_page		*p;
	int			 ret;

	ret = i915_gem_idle(dev_priv);
	if (ret)
		DRM_ERROR("failed to idle hardware: %d\n", ret);

	if (dev_priv->agpdmat != NULL) {
		/*
		 * make sure we nuke everything, we may have mappings that we've
		 * unrefed, but uvm has a reference to them for maps. Make sure
		 * they get unbound and any accesses will segfault.
		 * XXX only do ones in GEM.
		 */
		for (p = dev_priv->pgs; p < dev_priv->pgs +
		    (dev->agp->info.ai_aperture_size / PAGE_SIZE); p++)
			pmap_page_protect(p, VM_PROT_NONE);
		agp_bus_dma_destroy((struct agp_softc *)dev->agp->agpdev,
		    dev_priv->agpdmat);
	}
#endif /* INTELDRM_GEM */
	dev_priv->agpdmat = NULL;


	dev_priv->sarea_priv = NULL;

	i915_dma_cleanup(dev);
}

#ifdef INTELDRM_GEM

int
i915_gem_init_ioctl(struct drm_device *dev, void *data,
		    struct drm_file *file_priv)
{
	drm_i915_private_t *dev_priv = dev->dev_private;
	struct drm_i915_gem_init *args = data;

	DRM_LOCK();

	if (args->gtt_start >= args->gtt_end ||
	    args->gtt_end > dev->agp->info.ai_aperture_size ||
	    (args->gtt_start & PAGE_MASK) != 0 ||
	    (args->gtt_end & PAGE_MASK) != 0) {
		DRM_UNLOCK();
		return (EINVAL);
	}
	/*
	 * putting stuff in the last page of the aperture can cause nasty
	 * problems with prefetch going into unassigned memory. Since we put
	 * a scratch page on all unused aperture pages, just leave the last
	 * page as a spill to prevent gpu hangs.
	 */
	if (args->gtt_end == dev->agp->info.ai_aperture_size)
		args->gtt_end -= 4096;

	if (agp_bus_dma_init((struct agp_softc *)dev->agp->agpdev,
	    dev->agp->base + args->gtt_start, dev->agp->base + args->gtt_end,
	    &dev_priv->agpdmat) != 0) {
		DRM_UNLOCK();
		return (ENOMEM);
	}

	dev->gtt_total = (uint32_t)(args->gtt_end - args->gtt_start);

	DRM_UNLOCK();

	return 0;
}

int
i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
    struct drm_file *file_priv)
{
	struct drm_i915_gem_get_aperture	*args = data;

	args->aper_size = dev->gtt_total;
	args->aper_available_size = (args->aper_size -
	    atomic_read(&dev->pin_memory));

	return (0);
}

/**
 * Creates a new mm object and returns a handle to it.
 */
int
i915_gem_create_ioctl(struct drm_device *dev, void *data,
    struct drm_file *file_priv)
{
	struct drm_i915_gem_create	*args = data;
	struct drm_obj			*obj;
	int				 handle, ret;

	args->size = round_page(args->size);

	/* Allocate the new object */
	obj = drm_gem_object_alloc(dev, args->size);
	if (obj == NULL)
		return (ENOMEM);

	ret = drm_handle_create(file_priv, obj, &handle);
	/* handle has a reference now, drop ours. */
	DRM_LOCK();
	drm_gem_object_unreference(obj);
	DRM_UNLOCK();

	if (ret == 0)
		args->handle = handle;

	return (ret);
}

/**
 * Reads data from the object referenced by handle.
 *
 * On error, the contents of *data are undefined.
 */
int
i915_gem_pread_ioctl(struct drm_device *dev, void *data,
		     struct drm_file *file_priv)
{
	struct drm_i915_private		*dev_priv = dev->dev_private;
	struct drm_i915_gem_pread	*args = data;
	struct drm_obj			*obj;
	struct inteldrm_obj		*obj_priv;
	char				*vaddr;
	bus_space_handle_t	 	 bsh;
	bus_size_t		 	 bsize;
	voff_t				 offset;
	int				 ret;

	obj = drm_gem_object_lookup(dev, file_priv, args->handle);
	if (obj == NULL)
		return (EBADF);
	obj_priv = (struct inteldrm_obj *)obj;

	/*
	 * Bounds check source.
	 */
	DRM_LOCK();
	if (args->offset > obj->size || args->size > obj->size ||
	    args->offset + args->size > obj->size) {
		ret = EINVAL;
		goto out;
	}

	ret = i915_gem_object_pin(obj, 0, 1);
	if (ret)
		goto out;
	ret = i915_gem_object_set_to_gtt_domain(obj, 0, 1);
	if (ret)
		goto unpin;

	obj_priv = (struct inteldrm_obj *)obj;
	offset = obj_priv->gtt_offset + args->offset;

	bsize = round_page(offset + args->size) - trunc_page(offset);

	if ((ret = bus_space_subregion(dev_priv->bst, dev_priv->aperture_bsh,
	    trunc_page(offset), bsize, &bsh)) != 0)
		goto unpin;
	vaddr = bus_space_vaddr(dev->bst, bsh);
	if (vaddr == NULL) {
		ret = EFAULT;
		goto unpin;
	}

	ret = copyout(vaddr + (offset & PAGE_MASK),
	    (char *)(uintptr_t)args->data_ptr, args->size);

	if (ret)
		goto unpin;

unpin:
	i915_gem_object_unpin(obj);
out:
	drm_gem_object_unreference(obj);
	DRM_UNLOCK();

	return (ret);
}


/**
 * Writes data to the object referenced by handle.
 *
 * On error, the contents of the buffer that were to be modified are undefined.
 */
int
i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
    struct drm_file *file_priv)
{
	struct drm_i915_private		*dev_priv = dev->dev_private;
	struct drm_i915_gem_pwrite	*args = data;
	struct drm_obj			*obj;
	struct inteldrm_obj		*obj_priv;
	char 				*vaddr;
	bus_space_handle_t	 	 bsh;
	bus_size_t		 	 bsize;
	off_t			 	 offset;
	int				 ret = 0;

	obj = drm_gem_object_lookup(dev, file_priv, args->handle);
	if (obj == NULL)
		return (EBADF);
	obj_priv = (struct inteldrm_obj *)obj;

	DRM_LOCK();
	/* Bounds check destination. */
	if (args->offset > obj->size || args->size > obj->size ||
	    args->offset + args->size > obj->size) {
		ret = EINVAL;
		goto out;
	}

	ret = i915_gem_object_pin(obj, 0, 1);
	if (ret)
		goto out;
	ret = i915_gem_object_set_to_gtt_domain(obj, 1, 1);
	if (ret)
		goto done;

	obj_priv = (struct inteldrm_obj *)obj;
	offset = obj_priv->gtt_offset + args->offset;
	bsize = round_page(offset + args->size) - trunc_page(offset);

	if ((ret = bus_space_subregion(dev_priv->bst, dev_priv->aperture_bsh,
	    trunc_page(offset), bsize, &bsh)) != 0)
		goto done;
	vaddr = bus_space_vaddr(dev_priv->bst, bsh);
	if (vaddr == NULL) {
		ret = EFAULT;
		goto done;
	}

	ret = copyin((char *)(uintptr_t)args->data_ptr,
	    vaddr + (offset & PAGE_MASK), args->size);


done:
	i915_gem_object_unpin(obj);
out:
	drm_gem_object_unreference(obj);
	DRM_UNLOCK();

	return (ret);
}

/**
 * Called when user space prepares to use an object with the CPU, either through
 * the mmap ioctl's mapping or a GTT mapping.
 */
int
i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
			  struct drm_file *file_priv)
{
	struct drm_i915_gem_set_domain	*args = data;
	struct drm_obj			*obj;
	u_int32_t			 read_domains = args->read_domains;
	u_int32_t			 write_domain = args->write_domain;
	int				 ret;

	/*
	 * Only handle setting domains to types we allow the cpu to see.
	 * while linux allows the CPU domain here, we only allow GTT since that
	 * is all that we let userland near.
	 * Also sanity check that having something in the write domain implies
	 * it's in the read domain, and only that read domain.
	 */
	if ((write_domain | read_domains)  & ~I915_GEM_DOMAIN_GTT ||
	    (write_domain != 0 && read_domains != write_domain))
		return (EINVAL);
		
	obj = drm_gem_object_lookup(dev, file_priv, args->handle);
	if (obj == NULL)
		return (EBADF);

	DRM_LOCK();
	ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0, 1);

	drm_gem_object_unreference(obj);
	DRM_UNLOCK();
	/*
	 * Silently promote `you're not bound, there was nothing to do'
	 * to success, since the client was just asking us to make sure
	 * everything was done.
	 */
	return ((ret == EINVAL) ? 0 : ret);
}

int
i915_gem_gtt_map_ioctl(struct drm_device *dev, void *data,
    struct drm_file *file_priv)
{
	struct drm_i915_gem_mmap	*args = data;
	struct drm_obj			*obj;
	struct inteldrm_obj		*obj_priv;
	vaddr_t				 addr;
	voff_t				 offset;
	vsize_t				 end, nsize;
	int				 ret;

	obj = drm_gem_object_lookup(dev, file_priv, args->handle);
	if (obj == NULL)
		return (EBADF);

	DRM_LOCK();
	obj_priv = (struct inteldrm_obj *)obj;

	/* Check size. Also ensure that the object is not purgeable */
	if (args->size == 0 || args->offset > obj->size || args->size >
	    obj->size || (args->offset + args->size) > obj->size ||
	    i915_obj_purgeable(obj_priv)) {
		ret = EINVAL;
		goto done;
	}

	/* bind to the gtt to speed faulting */
	if (obj_priv->dmamap == NULL) {
		ret = i915_gem_object_bind_to_gtt(obj, 0, 1);
		if (ret)
			goto done;
		i915_gem_object_move_to_inactive(obj);
	}
			

	end = round_page(args->offset + args->size);
	offset = trunc_page(args->offset);
	nsize = end - offset;

	/*
	 * We give our reference from object_lookup to the mmap, so only
	 * must free it in the case that the map fails.
	 */
	addr = uvm_map_hint(curproc, VM_PROT_READ | VM_PROT_WRITE);
	ret = uvm_map_p(&curproc->p_vmspace->vm_map, &addr, nsize, &obj->uobj,
	    offset, 0, UVM_MAPFLAG(UVM_PROT_RW, UVM_PROT_RW,
	    UVM_INH_SHARE, UVM_ADV_RANDOM, 0), curproc);

done:
	if (ret != 0)
		drm_gem_object_unreference(obj);
	DRM_UNLOCK();

	if (ret == 0)
		args->addr_ptr = (uint64_t) addr + (args->offset & PAGE_MASK);

	return (ret);
}

/* called locked */
void
i915_gem_object_move_to_active(struct drm_obj *obj)
{
	struct drm_device	*dev = obj->dev;
	drm_i915_private_t	*dev_priv = dev->dev_private;
	struct inteldrm_obj	*obj_priv = (struct inteldrm_obj *)obj;
	struct inteldrm_fence	*reg;
	u_int32_t		 seqno = dev_priv->mm.next_gem_seqno;

	/* Add a reference if we're newly entering the active list. */
	if (!inteldrm_is_active(obj_priv)) {
		drm_gem_object_reference(obj);
		atomic_setbits_int(&obj_priv->io_flags, I915_ACTIVE);
	}

	if (inteldrm_needs_fence(obj_priv)) {
		reg = &dev_priv->fence_regs[obj_priv->fence_reg];
		reg->last_rendering_seqno = seqno;
	}

	/* Move from whatever list we were on to the tail of execution. */
	i915_move_to_tail(obj_priv, &dev_priv->mm.active_list);
	obj_priv->last_rendering_seqno = seqno;
}

void
i915_gem_object_move_off_active(struct drm_obj *obj)
{
	struct drm_device	*dev = obj->dev;
	struct drm_i915_private	*dev_priv = dev->dev_private;
	struct inteldrm_obj	*obj_priv = (struct inteldrm_obj *)obj;
	struct inteldrm_fence	*reg;

	obj_priv->last_rendering_seqno = 0;
	if (inteldrm_needs_fence(obj_priv)) {
		reg = &dev_priv->fence_regs[obj_priv->fence_reg];
		reg->last_rendering_seqno = 0;
	}
}

/* called locked */
void
i915_gem_object_move_to_inactive(struct drm_obj *obj)
{
	struct drm_device	*dev = obj->dev;
	drm_i915_private_t	*dev_priv = dev->dev_private;
	struct inteldrm_obj	*obj_priv = (struct inteldrm_obj *)obj;

	inteldrm_verify_inactive(dev_priv, __FILE__, __LINE__);
	if (obj_priv->pin_count != 0)
		i915_list_remove(obj_priv);
	else
		i915_move_to_tail(obj_priv, &dev_priv->mm.inactive_list);

	i915_gem_object_move_off_active(obj);
	atomic_clearbits_int(&obj_priv->io_flags, I915_FENCED_EXEC);

	KASSERT((obj_priv->io_flags & I915_GPU_WRITE) == 0);
	if (inteldrm_is_active(obj_priv)) {
		atomic_clearbits_int(&obj_priv->io_flags,
		    I915_ACTIVE);
		drm_gem_object_unreference(obj);
	}
	inteldrm_verify_inactive(dev_priv, __FILE__, __LINE__);
}

void
inteldrm_purge_obj(struct drm_obj *obj)
{
	struct inteldrm_obj *obj_priv = (struct inteldrm_obj *)obj;

	/*
	 * may sleep. We free here instead of deactivate (which
	 * the madvise() syscall would do) because in this case
	 * (userland bo cache and GL_APPLE_object_purgeable objects in
	 * OpenGL) the pages are defined to be freed if they were cleared
	 * so kill them and free up the memory
	 */
	simple_lock(&obj->uao->vmobjlock);
	obj->uao->pgops->pgo_flush(obj->uao, 0, obj->size,
	    PGO_ALLPAGES | PGO_FREE);
	simple_unlock(&obj->uao->vmobjlock);

	/*
	 * If flush failed, it may have halfway through, so just
	 * always mark as purged
	 */
	atomic_setbits_int(&obj_priv->io_flags, I915_PURGED);
}

void
inteldrm_process_flushing(struct drm_i915_private *dev_priv,
    u_int32_t flush_domains)
{
	struct inteldrm_obj		*obj_priv, *next;

	for (obj_priv = TAILQ_FIRST(&dev_priv->mm.gpu_write_list);
	    obj_priv != TAILQ_END(&dev_priv->mm.gpu_write_list);
	    obj_priv = next) {
		struct drm_obj *obj = &(obj_priv->obj);

		next = TAILQ_NEXT(obj_priv, write_list);

		if ((obj->write_domain & flush_domains) == obj->write_domain) {

			obj->write_domain = 0;
			TAILQ_REMOVE(&dev_priv->mm.gpu_write_list,
			    obj_priv, write_list);
			atomic_clearbits_int(&obj_priv->io_flags,
			     I915_GPU_WRITE);
			i915_gem_object_move_to_active(obj);
			/* if we still need the fence, update LRU */
			if (inteldrm_needs_fence(obj_priv)) {
				KASSERT(obj_priv->fence_reg !=
				    I915_FENCE_REG_NONE);
				/* we have a fence, won't sleep, can't fail */
				i915_gem_get_fence_reg(obj, 1);
			}
				
		}
	}
}

/**
 * Creates a new sequence number, emitting a write of it to the status page
 * plus an interrupt, which will trigger and interrupt if they are currently
 * enabled.
 *
 * Must be called with struct_lock held.
 *
 * Returned sequence numbers are nonzero on success.
 */
uint32_t
i915_add_request(struct drm_i915_private *dev_priv)
{
	struct inteldrm_request	*request;
	uint32_t		 seqno;
	int			 was_empty;

	request = drm_calloc(1, sizeof(*request));
	if (request == NULL) {
		printf("%s: failed to allocate request\n", __func__);
		return 0;
	}

	/* Grab the seqno we're going to make this request be, and bump the
	 * next (skipping 0 so it can be the reserved no-seqno value).
	 */
	seqno = dev_priv->mm.next_gem_seqno;
	dev_priv->mm.next_gem_seqno++;
	if (dev_priv->mm.next_gem_seqno == 0)
		dev_priv->mm.next_gem_seqno++;

	BEGIN_LP_RING(4);
	OUT_RING(MI_STORE_DWORD_INDEX);
	OUT_RING(I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
	OUT_RING(seqno);

	OUT_RING(MI_USER_INTERRUPT);
	ADVANCE_LP_RING();

	DRM_DEBUG("%d\n", seqno);

	/* XXX request timing for throttle */
	request->seqno = seqno;
	was_empty = TAILQ_EMPTY(&dev_priv->mm.request_list);
	TAILQ_INSERT_TAIL(&dev_priv->mm.request_list, request, list);

	if (dev_priv->mm.suspended == 0) {
		if (was_empty)
			timeout_add_sec(&dev_priv->mm.retire_timer, 1);
		/* XXX was_empty? */
		timeout_add_msec(&dev_priv->mm.hang_timer, 750);
	}
	return seqno;
}

/**
 * Moves buffers associated only with the given active seqno from the active
 * to inactive list, potentially freeing them.
 *
 * called with and sleeps with the drm_lock.
 */
void
i915_gem_retire_request(struct drm_i915_private *dev_priv,
    struct inteldrm_request *request)
{
	struct inteldrm_obj	*obj_priv;

	/* Move any buffers on the active list that are no longer referenced
	 * by the ringbuffer to the flushing/inactive lists as appropriate.
	 */
	while ((obj_priv  = TAILQ_FIRST(&dev_priv->mm.active_list)) != NULL) {
		struct drm_obj *obj = &obj_priv->obj;

		/* If the seqno being retired doesn't match the oldest in the
		 * list, then the oldest in the list must still be newer than
		 * this seqno.
		 */
		if (obj_priv->last_rendering_seqno != request->seqno)
			return;

		/*
		 * If we're now clean and can be read from, move inactive,
		 * else put on the flushing list to signify that we're not
		 * available quite yet.
		 */
		if (obj->write_domain != 0) {
			KASSERT(inteldrm_is_active(obj_priv));
			i915_move_to_tail(obj_priv,
			    &dev_priv->mm.flushing_list);
			i915_gem_object_move_off_active(obj);
		} else {
			i915_gem_object_move_to_inactive(obj);
		}
	}
}

/**
 * This function clears the request list as sequence numbers are passed.
 */
void
i915_gem_retire_requests(struct drm_i915_private *dev_priv)
{
	struct inteldrm_request	*request;
	uint32_t		 seqno;

	if (dev_priv->hw_status_page == NULL)
		return;

	seqno = i915_get_gem_seqno(dev_priv);

	while ((request = TAILQ_FIRST(&dev_priv->mm.request_list)) != NULL) {
		if (i915_seqno_passed(seqno, request->seqno) ||
		    dev_priv->mm.wedged) {
			i915_gem_retire_request(dev_priv, request);

			TAILQ_REMOVE(&dev_priv->mm.request_list, request, list);
			drm_free(request);
		} else
			break;
	}
}

void
i915_gem_retire_work_handler(void *arg1, void *unused)
{
	drm_i915_private_t	*dev_priv = arg1;
	struct drm_device	*dev = (struct drm_device *)dev_priv->drmdev;

	DRM_LOCK();
	i915_gem_retire_requests(dev_priv);
	if (!TAILQ_EMPTY(&dev_priv->mm.request_list))
		timeout_add_sec(&dev_priv->mm.retire_timer, 1);
	DRM_UNLOCK();
}

/**
 * Waits for a sequence number to be signaled, and cleans up the
 * request and object lists appropriately for that event.
 *
 * Called locked, sleeps with it.
 */
int
i915_wait_request(struct drm_i915_private *dev_priv, uint32_t seqno,
    int interruptible)
{
	int ret = 0;

	KASSERT(seqno != dev_priv->mm.next_gem_seqno);

	/* Check first because poking a wedged chip is bad. */
	if (dev_priv->mm.wedged)
		return (EIO);

	if (seqno == dev_priv->mm.next_gem_seqno) {
		seqno = i915_add_request(dev_priv);
		if (seqno == 0)
			return (ENOMEM);
	}

	if (!i915_seqno_passed(i915_get_gem_seqno(dev_priv), seqno)) {
		mtx_enter(&dev_priv->user_irq_lock);
		i915_user_irq_get(dev_priv);
		while (ret == 0) {
			if (i915_seqno_passed(i915_get_gem_seqno(dev_priv),
			    seqno) || dev_priv->mm.wedged)
				break;
			ret = msleep(dev_priv, &dev_priv->user_irq_lock,
			    PZERO | (interruptible ? PCATCH : 0), "gemwt", 0);
		}
		i915_user_irq_put(dev_priv);
		mtx_leave(&dev_priv->user_irq_lock);
	}
	if (dev_priv->mm.wedged)
		ret = EIO;

	/* Directly dispatch request retiring.  While we have the work queue
	 * to handle this, the waiter on a request often wants an associated
	 * buffer to have made it to the inactive list, and we would need
	 * a separate wait queue to handle that.
	 */
	if (ret == 0)
		i915_gem_retire_requests(dev_priv);

	return (ret);
}

/*
 * flush and invalidate the provided domains
 * if we have successfully queued a gpu flush, then we return a seqno from
 * the request. else (failed or just cpu flushed)  we return 0.
 */
u_int32_t
i915_gem_flush(struct drm_i915_private *dev_priv, uint32_t invalidate_domains,
    uint32_t flush_domains)
{
	uint32_t	cmd;

	if (flush_domains & I915_GEM_DOMAIN_CPU)
		inteldrm_chipset_flush(dev_priv);
	if ((invalidate_domains | flush_domains) & I915_GEM_GPU_DOMAINS) {
		/*
		 * read/write caches:
		 *
		 * I915_GEM_DOMAIN_RENDER is always invalidated, but is
		 * only flushed if MI_NO_WRITE_FLUSH is unset.  On 965, it is
		 * also flushed at 2d versus 3d pipeline switches.
		 *
		 * read-only caches:
		 *
		 * I915_GEM_DOMAIN_SAMPLER is flushed on pre-965 if
		 * MI_READ_FLUSH is set, and is always flushed on 965.
		 *
		 * I915_GEM_DOMAIN_COMMAND may not exist?
		 *
		 * I915_GEM_DOMAIN_INSTRUCTION, which exists on 965, is
		 * invalidated when MI_EXE_FLUSH is set.
		 *
		 * I915_GEM_DOMAIN_VERTEX, which exists on 965, is
		 * invalidated with every MI_FLUSH.
		 *
		 * TLBs:
		 *
		 * On 965, TLBs associated with I915_GEM_DOMAIN_COMMAND
		 * and I915_GEM_DOMAIN_CPU in are invalidated at PTE write and
		 * I915_GEM_DOMAIN_RENDER and I915_GEM_DOMAIN_SAMPLER
		 * are flushed at any MI_FLUSH.
		 */

		cmd = MI_FLUSH | MI_NO_WRITE_FLUSH;
		if ((invalidate_domains | flush_domains) &
		    I915_GEM_DOMAIN_RENDER)
			cmd &= ~MI_NO_WRITE_FLUSH;
		/*
		 * On the 965, the sampler cache always gets flushed
		 * and this bit is reserved.
		 */
		if (!IS_I965G(dev_priv) &&
		    invalidate_domains & I915_GEM_DOMAIN_SAMPLER)
			cmd |= MI_READ_FLUSH;
		if (invalidate_domains & I915_GEM_DOMAIN_INSTRUCTION)
			cmd |= MI_EXE_FLUSH;

		BEGIN_LP_RING(2);
		OUT_RING(cmd);
		OUT_RING(MI_NOOP);
		ADVANCE_LP_RING();
	}

	/* if this is a gpu flush, process the results */
	if (flush_domains & I915_GEM_GPU_DOMAINS) {
		inteldrm_process_flushing(dev_priv, flush_domains);
		return (i915_add_request(dev_priv));
	}

	return (0);
}

/**
 * Unbinds an object from the GTT aperture.
 *
 * XXX track dirty and pass down to uvm (note, DONTNEED buffers are clean).
 */
int
i915_gem_object_unbind(struct drm_obj *obj, int interruptible)
{
	struct drm_device	*dev = obj->dev;
	drm_i915_private_t	*dev_priv = dev->dev_private;
	struct inteldrm_obj	*obj_priv = (struct inteldrm_obj *)obj;
	int			 ret = 0;

	/*
	 * if it's already unbound, or we've already done lastclose, just
	 * let it happen. XXX does this fail to unwire?
	 */
	if (obj_priv->dmamap == NULL || dev_priv->agpdmat == NULL)
		return 0;

	if (obj_priv->pin_count != 0) {
		DRM_ERROR("Attempting to unbind pinned buffer\n");
		return (EINVAL);
	}

	KASSERT(!i915_obj_purged(obj_priv));

	/* Move the object to the CPU domain to ensure that
	 * any possible CPU writes while it's not in the GTT
	 * are flushed when we go to remap it. This will
	 * also ensure that all pending GPU writes are finished
	 * before we unbind.
	 */
	ret = i915_gem_object_set_to_cpu_domain(obj, 1, interruptible);
	if (ret)
		return ret;

	KASSERT(!inteldrm_is_active(obj_priv));

	/* if it's purgeable don't bother dirtying the pages */
	if (i915_obj_purgeable(obj_priv))
		atomic_clearbits_int(&obj_priv->io_flags, I915_DIRTY);
	/*
	 * unload the map, then unwire the backing object.
	 */
	i915_gem_save_bit_17_swizzle(obj);
	bus_dmamap_unload(dev_priv->agpdmat, obj_priv->dmamap);
	uvm_objunwire(obj->uao, 0, obj->size);
	/* XXX persistent dmamap worth the memory? */
	bus_dmamap_destroy(dev_priv->agpdmat, obj_priv->dmamap);
	obj_priv->dmamap = NULL;
	free(obj_priv->dma_segs, M_DRM);
	obj_priv->dma_segs = NULL;
	/* XXX this should change whether we tell uvm the page is dirty */
	atomic_clearbits_int(&obj_priv->io_flags, I915_DIRTY);

	obj_priv->gtt_offset = 0;
	atomic_dec(&dev->gtt_count);
	atomic_sub(obj->size, &dev->gtt_memory);

	/* Remove ourselves from any LRU list if present. */
	i915_list_remove((struct inteldrm_obj *)obj);

	if (i915_obj_purgeable(obj_priv))
		inteldrm_purge_obj(obj);

	return (0);
}

int
i915_gem_evict_something(struct drm_i915_private *dev_priv, size_t min_size,
    int interruptible)
{
	struct drm_obj		*obj;
	struct inteldrm_request	*request;
	struct inteldrm_obj	*obj_priv;
	int			 ret = 0;

	for (;;) {
		i915_gem_retire_requests(dev_priv);

		/* If there's an inactive buffer available now, grab it
		 * and be done.
		 */
		obj = i915_gem_find_inactive_object(dev_priv, min_size);
		if (obj != NULL) {
			obj_priv = (struct inteldrm_obj *)obj;

			KASSERT(obj_priv->pin_count == 0);
			KASSERT(!inteldrm_is_active(obj_priv));

			/* Wait on the rendering and unbind the buffer. */
			return (i915_gem_object_unbind(obj, interruptible));
		}

		/* If we didn't get anything, but the ring is still processing
		 * things, wait for one of those things to finish and hopefully
		 * leave us a buffer to evict.
		 */
		if ((request = TAILQ_FIRST(&dev_priv->mm.request_list))
		    != NULL) {
			ret = i915_wait_request(dev_priv, request->seqno,
			    interruptible);
			if (ret)
				return (ret);

			continue;
		}

		/* If we didn't have anything on the request list but there
		 * are buffers awaiting a flush, emit one and try again.
		 * When we wait on it, those buffers waiting for that flush
		 * will get moved to inactive.
		 */
		TAILQ_FOREACH(obj_priv, &dev_priv->mm.flushing_list, list) {
			obj = &obj_priv->obj;
			if (obj->size >= min_size)
				break;
			obj = NULL;
		}

		if (obj != NULL) {
			if (i915_gem_flush(dev_priv, obj->write_domain,
			    obj->write_domain) == 0)
				return (ENOMEM);
			continue;
		}

		/*
		 * If we didn't do any of the above, there's no single buffer
		 * large enough to swap out for the new one, so just evict
		 * everything and start again. (This should be rare.)
		 */
		if (!TAILQ_EMPTY(&dev_priv->mm.inactive_list))
			return (i915_gem_evict_inactive(dev_priv));
		else
			return (i915_gem_evict_everything(dev_priv,
			    interruptible));
	}
	/* NOTREACHED */
}

struct drm_obj *
i915_gem_find_inactive_object(struct drm_i915_private *dev_priv,
    size_t min_size)
{
	struct drm_obj		*obj, *best = NULL, *first = NULL;
	struct inteldrm_obj	*obj_priv;

	TAILQ_FOREACH(obj_priv, &dev_priv->mm.inactive_list, list) {
		obj = &obj_priv->obj;
		if (obj->size >= min_size) {
			if ((!inteldrm_is_dirty(obj_priv) ||
			    i915_obj_purgeable(obj_priv)) &&
			    (best == NULL || obj->size < best->size)) {
				best = obj;
				if (best->size == min_size)
					return (best);
			}
		}
		if (first == NULL)
			first = obj;
	}

	return ((best != NULL) ? best : first);
}

int
i915_gem_evict_everything(struct drm_i915_private *dev_priv, int interruptible)
{
	u_int32_t	seqno;
	int		ret;

	if (TAILQ_EMPTY(&dev_priv->mm.inactive_list) &&
	    TAILQ_EMPTY(&dev_priv->mm.flushing_list) &&
	    TAILQ_EMPTY(&dev_priv->mm.active_list))
		return (ENOSPC);

	seqno = i915_gem_flush(dev_priv, I915_GEM_GPU_DOMAINS,
	    I915_GEM_GPU_DOMAINS);
	if (seqno == 0)
		return (ENOMEM);

	if ((ret = i915_wait_request(dev_priv, seqno, interruptible)) != 0 ||
	    (ret = i915_gem_evict_inactive(dev_priv)) != 0)
		return (ret);

	/*
	 * All lists should be empty because we flushed the whole queue, then
	 * we evicted the whole shebang, only pinned objects are still bound.
	 */
	KASSERT(TAILQ_EMPTY(&dev_priv->mm.inactive_list));
	KASSERT(TAILQ_EMPTY(&dev_priv->mm.flushing_list));
	KASSERT(TAILQ_EMPTY(&dev_priv->mm.active_list));

	return (0);
}
/*
 * return required GTT alignment for an object, taking into account potential
 * fence register needs
 */
bus_size_t
i915_gem_get_gtt_alignment(struct drm_obj *obj)
{
	struct drm_device	*dev = obj->dev;
	struct drm_i915_private	*dev_priv = dev->dev_private;
	struct inteldrm_obj	*obj_priv = (struct inteldrm_obj *)obj;
	bus_size_t		 start, i;

	/*
	 * Minimum alignment is 4k (GTT page size), but fence registers may
	 * modify this
	 */
	if (IS_I965G(dev_priv) || obj_priv->tiling_mode == I915_TILING_NONE)
		return (4096);

	/*
	 * Older chips need to be aligned to the size of the smallest fence
	 * register that can contain the object.
	 */
	if (IS_I9XX(dev_priv))
		start = 1024 * 1024;
	else
		start = 512 * 1024;

	for (i = start; i < obj->size; i <<= 1)
		;

	return (i);
}

void
i965_write_fence_reg(struct inteldrm_fence *reg)
{
	struct drm_obj		*obj = reg->obj;
	struct drm_device	*dev = obj->dev;
	drm_i915_private_t	*dev_priv = dev->dev_private;
	struct inteldrm_obj	*obj_priv = (struct inteldrm_obj *)obj;
	int			 regnum = obj_priv->fence_reg;
	u_int64_t		 val;

	val = (uint64_t)((obj_priv->gtt_offset + obj->size - 4096) &
		    0xfffff000) << 32;
	val |= obj_priv->gtt_offset & 0xfffff000;
	val |= ((obj_priv->stride / 128) - 1) << I965_FENCE_PITCH_SHIFT;
	if (obj_priv->tiling_mode == I915_TILING_Y)
		val |= 1 << I965_FENCE_TILING_Y_SHIFT;
	val |= I965_FENCE_REG_VALID;

	I915_WRITE64(FENCE_REG_965_0 + (regnum * 8), val);
}

void
i915_write_fence_reg(struct inteldrm_fence *reg)
{
	struct drm_obj		*obj = reg->obj;
	struct drm_device	*dev = obj->dev;
	drm_i915_private_t	*dev_priv = dev->dev_private;
	struct inteldrm_obj	*obj_priv = (struct inteldrm_obj *)obj;
	bus_size_t		 fence_reg;
	u_int32_t		 val;
	u_int32_t		 pitch_val;
	int			 regnum = obj_priv->fence_reg;
	int			 tile_width;

	if ((obj_priv->gtt_offset & ~I915_FENCE_START_MASK) ||
	    (obj_priv->gtt_offset & (obj->size - 1))) {
		DRM_ERROR("object 0x%lx not 1M or size (0x%zx) aligned\n",
		     obj_priv->gtt_offset, obj->size);
		return;
	}

	if (obj_priv->tiling_mode == I915_TILING_Y &&
	    HAS_128_BYTE_Y_TILING(dev_priv))
		tile_width = 128;
	else
		tile_width = 512;

	/* Note: pitch better be a power of two tile widths */
	pitch_val = obj_priv->stride / tile_width;
	pitch_val = ffs(pitch_val) - 1;

	val = obj_priv->gtt_offset;
	if (obj_priv->tiling_mode == I915_TILING_Y)
		val |= 1 << I830_FENCE_TILING_Y_SHIFT;
	val |= I915_FENCE_SIZE_BITS(obj->size);
	val |= pitch_val << I830_FENCE_PITCH_SHIFT;
	val |= I830_FENCE_REG_VALID;

	if (regnum < 8)
		fence_reg = FENCE_REG_830_0 + (regnum * 4);
	else
		fence_reg = FENCE_REG_945_8 + ((regnum - 8) * 4);
	I915_WRITE(fence_reg, val);
}

void
i830_write_fence_reg(struct inteldrm_fence *reg)
{
	struct drm_obj		*obj = reg->obj;
	struct drm_device	*dev = obj->dev;
	struct drm_i915_private	*dev_priv = dev->dev_private;
	struct inteldrm_obj	*obj_priv = (struct inteldrm_obj *)obj;
	int			 regnum = obj_priv->fence_reg;
	u_int32_t		 pitch_val, val;

	if ((obj_priv->gtt_offset & ~I830_FENCE_START_MASK) ||
	    (obj_priv->gtt_offset & (obj->size - 1))) {
		DRM_ERROR("object 0x%08x not 512K or size aligned 0x%lx\n",
		     obj_priv->gtt_offset, obj->size);
		return;
	}

	pitch_val = ffs(obj_priv->stride / 128) - 1;

	val = obj_priv->gtt_offset;
	if (obj_priv->tiling_mode == I915_TILING_Y)
		val |= 1 << I830_FENCE_TILING_Y_SHIFT;
	val |= I830_FENCE_SIZE_BITS(obj->size);
	val |= pitch_val << I830_FENCE_PITCH_SHIFT;
	val |= I830_FENCE_REG_VALID;

	I915_WRITE(FENCE_REG_830_0 + (regnum * 4), val);

}

/*
 * i915_gem_get_fence_reg - set up a fence reg for an object
 *
 * When mapping objects through the GTT, userspace wants to be able to write
 * to them without having to worry about swizzling if the object is tiled.
 *
 * This function walks the fence regs looking for a free one, stealing one
 * if it can't find any.
 *
 * It then sets up the reg based on the object's properties: address, pitch
 * and tiling format.
 */
int
i915_gem_get_fence_reg(struct drm_obj *obj, int interruptible)
{
	struct drm_device	*dev = obj->dev;
	struct drm_i915_private	*dev_priv = dev->dev_private;
	struct inteldrm_obj	*obj_priv = (struct inteldrm_obj *)obj;
	struct inteldrm_obj	*old_obj_priv = NULL;
	struct drm_obj		*old_obj = NULL;
	struct inteldrm_fence	*reg = NULL;
	int			 i, ret, avail;

	/* If our fence is getting used, just update our place in the LRU */
	if (obj_priv->fence_reg != I915_FENCE_REG_NONE) {
		reg = &dev_priv->fence_regs[obj_priv->fence_reg];

		TAILQ_REMOVE(&dev_priv->mm.fence_list, reg, list);
		TAILQ_INSERT_TAIL(&dev_priv->mm.fence_list, reg, list);
		return (0);
	}

	switch (obj_priv->tiling_mode) {
	case I915_TILING_NONE:
		DRM_ERROR("allocating a fence for non-tiled object?\n");
		break;
	case I915_TILING_X:
		if (obj_priv->stride == 0)
			return (EINVAL);
		if (obj_priv->stride & (512 - 1))
			DRM_ERROR("object 0x%08x is X tiled but has non-512B"
			    " pitch\n", obj_priv->gtt_offset);
		break;
	case I915_TILING_Y:
		if (obj_priv->stride == 0)
			return (EINVAL);
		if (obj_priv->stride & (128 - 1))
			DRM_ERROR("object 0x%08x is Y tiled but has non-128B"
			    " pitch\n", obj_priv->gtt_offset);
		break;
	}

	/* First try to find a free reg */
	avail = 0;
	for (i = dev_priv->fence_reg_start; i < dev_priv->num_fence_regs; i++) {
		reg = &dev_priv->fence_regs[i];
		if (reg->obj == NULL)
			break;

		old_obj_priv = (struct inteldrm_obj *)reg->obj;
		if (old_obj_priv->pin_count == 0)
			avail++;
	}

	/* None available, try to steal one or wait for a user to finish */
	if (i == dev_priv->num_fence_regs) {
		if (avail == 0)
			return (ENOMEM);

		TAILQ_FOREACH(reg, &dev_priv->mm.fence_list,
		    list) {
			old_obj = reg->obj;
			old_obj_priv = (struct inteldrm_obj *)old_obj;

			if (old_obj_priv->pin_count)
				continue;

			/* Ref it so that wait_rendering doesn't free it under
			 * us.
			 */
			drm_gem_object_reference(old_obj);

			break;
		}

		i = old_obj_priv->fence_reg;
		reg = &dev_priv->fence_regs[i];

		ret = i915_gem_object_put_fence_reg(old_obj, interruptible);
		drm_gem_object_unreference(old_obj);
		if (ret != 0)
			return (ret);
	}

	obj_priv->fence_reg = i;
	reg->obj = obj;
	TAILQ_INSERT_TAIL(&dev_priv->mm.fence_list, reg, list);

	if (IS_I965G(dev_priv))
		i965_write_fence_reg(reg);
	else if (IS_I9XX(dev_priv))
		i915_write_fence_reg(reg);
	else
		i830_write_fence_reg(reg);

	return 0;
}

int
i915_gem_object_put_fence_reg(struct drm_obj *obj, int interruptible)
{
	struct drm_device	*dev = obj->dev;
	struct drm_i915_private	*dev_priv = dev->dev_private;
	struct inteldrm_obj	*obj_priv = (struct inteldrm_obj *)obj;
	struct inteldrm_fence	*reg;
	int			 ret;

	if (obj_priv->fence_reg == I915_FENCE_REG_NONE)
		return (0);

	/* 
	 * If the last execbuffer we did on the object needed a fence then
	 * we must emit a flush.
	 */
	if (inteldrm_needs_fence(obj_priv)) {
		ret = i915_gem_object_flush_gpu_write_domain(obj, 1,
		    interruptible);
		if (ret != 0)
			return (ret);
	}

	/* if rendering is queued up that depends on the fence, wait for it */
	reg = &dev_priv->fence_regs[obj_priv->fence_reg];
	if (reg->last_rendering_seqno != 0) {
		ret = i915_wait_request(dev_priv, reg->last_rendering_seqno,
		    interruptible);
		if (ret != 0)
			return (ret);
	}

	/* tiling changed, must wipe userspace mappings */
	if ((obj->write_domain | obj->read_domains) & I915_GEM_DOMAIN_GTT) {
		inteldrm_wipe_mappings(obj);
		if (obj->write_domain == I915_GEM_DOMAIN_GTT)
			obj->write_domain = 0;
	}

	if (IS_I965G(dev_priv)) {
		I915_WRITE64(FENCE_REG_965_0 + (obj_priv->fence_reg * 8), 0);
	} else {
		u_int32_t fence_reg;

		if (obj_priv->fence_reg < 8)
			fence_reg = FENCE_REG_830_0 + obj_priv->fence_reg * 4;
		else 
			fence_reg = FENCE_REG_945_8 +
			    (obj_priv->fence_reg - 8) * 4;
		I915_WRITE(fence_reg , 0);
	}

	reg->obj = NULL;
	TAILQ_REMOVE(&dev_priv->mm.fence_list, reg, list);
	obj_priv->fence_reg = I915_FENCE_REG_NONE;

	return (0);
}

int
inteldrm_fault(struct drm_obj *obj, struct uvm_faultinfo *ufi, off_t offset,
    vaddr_t vaddr, vm_page_t *pps, int npages, int centeridx,
    vm_prot_t access_type, int flags)
{
	struct drm_device	*dev = obj->dev;
	struct inteldrm_obj	*obj_priv = (struct inteldrm_obj *)obj;
	paddr_t			 paddr;
	int			 lcv, ret;
	int			 write = !!(access_type & VM_PROT_WRITE);
	vm_prot_t		 mapprot;

	DRM_LOCK();
	/*
	 * XXX is it ok to sleep in fault handlers? If not, we may have some
	 * problems...  (i can has race plz? -> judicious use of
	 * uvmfault_unlockall ahoy)
	 */
	if (obj_priv->dmamap == NULL) {
		ret = i915_gem_object_bind_to_gtt(obj, 0, 0);
		if (ret) {
			printf("%s: failed to bind\n", __func__);
			goto error;
		}
		i915_gem_object_move_to_inactive(obj);
	}

	/*
	 * We could only do this on bind so allow for map_buffer_range
	 * unsynchronised objects (where buffer suballocation
	 * is done by the GL application, however it gives coherency problems
	 * normally.
	 */
	ret = i915_gem_object_set_to_gtt_domain(obj, write, 0);
	if (ret) {
		printf("%s: failed to set to gtt (%d)\n",
		    __func__, ret);
		goto error;
	}

	mapprot = ufi->entry->protection;
	/*
	 * if it's only a read fault, we only put ourselves into the gtt
	 * read domain, so make sure we fault again and set ourselves to write.
	 * this prevents us needing userland to do domain management and get
	 * it wrong, and makes us fully coherent with the gpu re mmap.
	 */
	if (write == 0)
		mapprot &= ~VM_PROT_WRITE;
	/* XXX try and  be more efficient when we do this */
	for (lcv = 0 ; lcv < npages ; lcv++, offset += PAGE_SIZE,
	    vaddr += PAGE_SIZE) {
		if ((flags & PGO_ALLPAGES) == 0 && lcv != centeridx)
			continue;

		if (pps[lcv] == PGO_DONTCARE)
			continue;

		paddr = dev->agp->base + obj_priv->gtt_offset + offset;

		UVMHIST_LOG(maphist,
		    "  MAPPING: device: pm=%p, va=0x%lx, pa=0x%lx, at=%ld",
		    ufi->orig_map->pmap, vaddr, (u_long)paddr, mapprot);
		/* XXX writecombining */
		if (pmap_enter(ufi->orig_map->pmap, vaddr, paddr | PMAP_NOCACHE,
		    mapprot, PMAP_CANFAIL | mapprot) != 0) {
			DRM_UNLOCK();
			printf("%s: enter failed\n", __func__);
			return (VM_PAGER_REFAULT);
		}
	}
	DRM_UNLOCK();
	return (VM_PAGER_OK);

error:
	/*
	 * EIO means we're wedged so when we reset the gpu this will
	 * work, so don't segfault.
	 */
	DRM_UNLOCK();
	return ((ret == EIO) ?  VM_PAGER_REFAULT : VM_PAGER_ERROR);
		
}

void
inteldrm_wipe_mappings(struct drm_obj *obj)
{
	struct inteldrm_obj	*obj_priv = (struct inteldrm_obj *)obj;
	struct drm_device	*dev = obj->dev;
	struct drm_i915_private	*dev_priv = dev->dev_private;
	struct vm_page		*pg;

	/* nuke all our mappings. XXX optimise. */
	for (pg = &dev_priv->pgs[atop(obj_priv->gtt_offset)]; pg !=
	    &dev_priv->pgs[atop(obj_priv->gtt_offset + obj->size)]; pg++)
		pmap_page_protect(pg, VM_PROT_NONE);
}

/**
 * Finds free space in the GTT aperture and binds the object there.
 */
int
i915_gem_object_bind_to_gtt(struct drm_obj *obj, bus_size_t alignment,
    int interruptible)
{
	struct drm_device	*dev = obj->dev;
	drm_i915_private_t	*dev_priv = dev->dev_private;
	struct inteldrm_obj	*obj_priv = (struct inteldrm_obj *)obj;
	int			 ret;

	if (dev_priv->agpdmat == NULL)
		return (EINVAL);
	if (alignment == 0) {
		alignment = i915_gem_get_gtt_alignment(obj);
	} else if (alignment & (i915_gem_get_gtt_alignment(obj) - 1)) {
		DRM_ERROR("Invalid object alignment requested %u\n", alignment);
		return (EINVAL);
	}

	/* Can't bind a purgeable buffer */
	if (i915_obj_purgeable(obj_priv)) {
		printf("tried to bind purgeable buffer");
		return (EINVAL);
	}

	if ((ret = bus_dmamap_create(dev_priv->agpdmat, obj->size, 1,
	    obj->size, 0, BUS_DMA_WAITOK, &obj_priv->dmamap)) != 0) {
		DRM_ERROR("Failed to create dmamap\n");
		return (ret);
	}
	agp_bus_dma_set_alignment(dev_priv->agpdmat, obj_priv->dmamap,
	    alignment);

 search_free:
	/*
	 * the helper function wires the uao then binds it to the aperture for 
	 * us, so all we have to do is set up the dmamap then load it.
	 */
	ret = drm_gem_load_uao(dev_priv->agpdmat, obj_priv->dmamap, obj->uao,
	    obj->size, BUS_DMA_WAITOK | obj_priv->dma_flags,
	    &obj_priv->dma_segs);
	/* XXX NOWAIT? */
	if (ret != 0) {
		/* If the gtt is empty and we're still having trouble
		 * fitting our object in, we're out of memory.
		 */
		if (TAILQ_EMPTY(&dev_priv->mm.inactive_list) &&
		    TAILQ_EMPTY(&dev_priv->mm.flushing_list) &&
		    TAILQ_EMPTY(&dev_priv->mm.active_list)) {
			DRM_ERROR("GTT full, but LRU list empty\n");
			goto error;
		}

		ret = i915_gem_evict_something(dev_priv, obj->size,
		    interruptible);
		if (ret != 0)
			goto error;
		goto search_free;
	}
	i915_gem_bit_17_swizzle(obj);

	obj_priv->gtt_offset = obj_priv->dmamap->dm_segs[0].ds_addr -
	    dev->agp->base;

	atomic_inc(&dev->gtt_count);
	atomic_add(obj->size, &dev->gtt_memory);

	/* Assert that the object is not currently in any GPU domain. As it
	 * wasn't in the GTT, there shouldn't be any way it could have been in
	 * a GPU cache
	 */
	KASSERT((obj->read_domains & I915_GEM_GPU_DOMAINS) == 0);
	KASSERT((obj->write_domain & I915_GEM_GPU_DOMAINS) == 0);

	return (0);

error:
	bus_dmamap_destroy(dev_priv->agpdmat, obj_priv->dmamap);
	obj_priv->dmamap = NULL;
	obj_priv->gtt_offset = 0;
	return (ret);
}

/*
 * Flush the GPU write domain for the object if dirty, then wait for the
 * rendering to complete. When this returns it is safe to unbind from the 
 * GTT or access from the CPU.
 */
int
i915_gem_object_flush_gpu_write_domain(struct drm_obj *obj, int pipelined,
    int interruptible)
{
	struct drm_device	*dev = obj->dev;
	struct drm_i915_private	*dev_priv = dev->dev_private;
	struct inteldrm_obj	*obj_priv = (struct inteldrm_obj *)obj;
	int			 ret = 0;

	if ((obj->write_domain & I915_GEM_GPU_DOMAINS) != 0) {
		/*
		 * Queue the GPU write cache flushing we need.
		 * This call will move stuff form the flushing list to the
		 * active list so all we need to is wait for it.
		 */
		(void)i915_gem_flush(dev_priv, 0, obj->write_domain);
		KASSERT(obj->write_domain == 0);
	}

	/* wait for queued rendering so we know it's flushed and bo is idle */
	if (pipelined == 0 && inteldrm_is_active(obj_priv)) {
		ret =  i915_wait_request(dev_priv,
		    obj_priv->last_rendering_seqno, interruptible);
	}
	return (ret);
}

/* 
 * Moves a single object to the GTT and possibly write domain.
 *
 * This function returns when the move is complete, including waiting on
 * flushes to occur.
 */
int
i915_gem_object_set_to_gtt_domain(struct drm_obj *obj, int write,
    int interruptible)
{
	struct drm_device	*dev = (struct drm_device *)obj->dev;
	struct drm_i915_private	*dev_priv = dev->dev_private;
	struct inteldrm_obj	*obj_priv = (struct inteldrm_obj *)obj;
	int			 ret;

	/* Not valid to be called on unbound objects. */
	if (obj_priv->dmamap == NULL)
		return (EINVAL);
	/* Wait on any GPU rendering and flushing to occur. */
	if ((ret = i915_gem_object_flush_gpu_write_domain(obj, 0,
	    interruptible)) != 0)
		return (ret);

	if (obj->write_domain == I915_GEM_DOMAIN_CPU) {
		/* clflush the pages, and flush chipset cache */
		bus_dmamap_sync(dev_priv->agpdmat, obj_priv->dmamap, 0,
		    obj->size, BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
		inteldrm_chipset_flush(dev_priv);
		obj->write_domain = 0;
	}

	/* We're accessing through the gpu, so grab a new fence register or
	 * update the LRU.
	 */
	if (obj_priv->tiling_mode != I915_TILING_NONE)
		ret = i915_gem_get_fence_reg(obj, interruptible);

	/*
	 * If we're writing through the GTT domain then the CPU and GPU caches
	 * will need to be invalidated at next use.
	 * It should now be out of any other write domains and we can update
	 * to the correct ones
	 */
	KASSERT((obj->write_domain & ~I915_GEM_DOMAIN_GTT) == 0);
	if (write) {
		obj->read_domains = obj->write_domain = I915_GEM_DOMAIN_GTT;
		atomic_setbits_int(&obj_priv->io_flags, I915_DIRTY);
	} else {
		obj->read_domains |= I915_GEM_DOMAIN_GTT;
	}

	return (ret);
}

/*
 * Moves a single object to the CPU read and possibly write domain.
 *
 * This function returns when the move is complete, including waiting on
 * flushes to return.
 */
int
i915_gem_object_set_to_cpu_domain(struct drm_obj *obj, int write,
    int interruptible)
{
	struct drm_device	*dev = obj->dev;
	struct drm_i915_private	*dev_priv = dev->dev_private;
	struct inteldrm_obj	*obj_priv = (struct inteldrm_obj *)obj;
	int			 ret;

	/* Wait on any GPU rendering and flushing to occur. */
	if ((ret = i915_gem_object_flush_gpu_write_domain(obj, 0,
	    interruptible)) != 0)
		return (ret);

	if (obj->write_domain == I915_GEM_DOMAIN_GTT ||
	    (write && obj->read_domains & I915_GEM_DOMAIN_GTT)) {
		/*
		 * No actual flushing is required for the GTT write domain.
		 * Writes to it immeditately go to main memory as far as we
		 * know, so there's no chipset flush. It also doesn't land
		 * in render cache.
		 */
		inteldrm_wipe_mappings(obj);
		if (obj->write_domain == I915_GEM_DOMAIN_GTT)
			obj->write_domain = 0;
	}

	/* remove the fence register since we're not using it anymore */
	if ((ret = i915_gem_object_put_fence_reg(obj, interruptible)) != 0)
		return (ret);

	/* Flush the CPU cache if it's still invalid. */
	if ((obj->read_domains & I915_GEM_DOMAIN_CPU) == 0) {
		bus_dmamap_sync(dev_priv->agpdmat, obj_priv->dmamap, 0,
		    obj->size, BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);

		obj->read_domains |= I915_GEM_DOMAIN_CPU;
	}

	/*
	 * It should now be out of any other write domain, and we can update
	 * the domain value for our changes.
	 */
	KASSERT((obj->write_domain & ~I915_GEM_DOMAIN_CPU) == 0);

	/*
	 * If we're writing through the CPU, then the GPU read domains will
	 * need to be invalidated at next use.
	 */
	if (write)
		obj->read_domains = obj->write_domain = I915_GEM_DOMAIN_CPU;

	return (0);
}

/*
 * Set the next domain for the specified object. This
 * may not actually perform the necessary flushing/invaliding though,
 * as that may want to be batched with other set_domain operations
 *
 * This is (we hope) the only really tricky part of gem. The goal
 * is fairly simple -- track which caches hold bits of the object
 * and make sure they remain coherent. A few concrete examples may
 * help to explain how it works. For shorthand, we use the notation
 * (read_domains, write_domain), e.g. (CPU, CPU) to indicate the
 * a pair of read and write domain masks.
 *
 * Case 1: the batch buffer
 *
 *	1. Allocated
 *	2. Written by CPU
 *	3. Mapped to GTT
 *	4. Read by GPU
 *	5. Unmapped from GTT
 *	6. Freed
 *
 *	Let's take these a step at a time
 *
 *	1. Allocated
 *		Pages allocated from the kernel may still have
 *		cache contents, so we set them to (CPU, CPU) always.
 *	2. Written by CPU (using pwrite)
 *		The pwrite function calls set_domain (CPU, CPU) and
 *		this function does nothing (as nothing changes)
 *	3. Mapped by GTT
 *		This function asserts that the object is not
 *		currently in any GPU-based read or write domains
 *	4. Read by GPU
 *		i915_gem_execbuffer calls set_domain (COMMAND, 0).
 *		As write_domain is zero, this function adds in the
 *		current read domains (CPU+COMMAND, 0).
 *		flush_domains is set to CPU.
 *		invalidate_domains is set to COMMAND
 *		clflush is run to get data out of the CPU caches
 *		then i915_dev_set_domain calls i915_gem_flush to
 *		emit an MI_FLUSH and drm_agp_chipset_flush
 *	5. Unmapped from GTT
 *		i915_gem_object_unbind calls set_domain (CPU, CPU)
 *		flush_domains and invalidate_domains end up both zero
 *		so no flushing/invalidating happens
 *	6. Freed
 *		yay, done
 *
 * Case 2: The shared render buffer
 *
 *	1. Allocated
 *	2. Mapped to GTT
 *	3. Read/written by GPU
 *	4. set_domain to (CPU,CPU)
 *	5. Read/written by CPU
 *	6. Read/written by GPU
 *
 *	1. Allocated
 *		Same as last example, (CPU, CPU)
 *	2. Mapped to GTT
 *		Nothing changes (assertions find that it is not in the GPU)
 *	3. Read/written by GPU
 *		execbuffer calls set_domain (RENDER, RENDER)
 *		flush_domains gets CPU
 *		invalidate_domains gets GPU
 *		clflush (obj)
 *		MI_FLUSH and drm_agp_chipset_flush
 *	4. set_domain (CPU, CPU)
 *		flush_domains gets GPU
 *		invalidate_domains gets CPU
 *		flush_gpu_write (obj) to make sure all drawing is complete.
 *		This will include an MI_FLUSH to get the data from GPU
 *		to memory
 *		clflush (obj) to invalidate the CPU cache
 *		Another MI_FLUSH in i915_gem_flush (eliminate this somehow?)
 *	5. Read/written by CPU
 *		cache lines are loaded and dirtied
 *	6. Read written by GPU
 *		Same as last GPU access
 *
 * Case 3: The constant buffer
 *
 *	1. Allocated
 *	2. Written by CPU
 *	3. Read by GPU
 *	4. Updated (written) by CPU again
 *	5. Read by GPU
 *
 *	1. Allocated
 *		(CPU, CPU)
 *	2. Written by CPU
 *		(CPU, CPU)
 *	3. Read by GPU
 *		(CPU+RENDER, 0)
 *		flush_domains = CPU
 *		invalidate_domains = RENDER
 *		clflush (obj)
 *		MI_FLUSH
 *		drm_agp_chipset_flush
 *	4. Updated (written) by CPU again
 *		(CPU, CPU)
 *		flush_domains = 0 (no previous write domain)
 *		invalidate_domains = 0 (no new read domains)
 *	5. Read by GPU
 *		(CPU+RENDER, 0)
 *		flush_domains = CPU
 *		invalidate_domains = RENDER
 *		clflush (obj)
 *		MI_FLUSH
 *		drm_agp_chipset_flush
 */
void
i915_gem_object_set_to_gpu_domain(struct drm_obj *obj)
{
	struct drm_device	*dev = obj->dev;
	drm_i915_private_t	*dev_priv = dev->dev_private;
	struct inteldrm_obj	*obj_priv = (struct inteldrm_obj *)obj;
	u_int32_t		 invalidate_domains = 0;
	u_int32_t		 flush_domains = 0;

	KASSERT((obj->pending_read_domains & I915_GEM_DOMAIN_CPU) == 0);
	KASSERT(obj->pending_write_domain != I915_GEM_DOMAIN_CPU);
	/*
	 * If the object isn't moving to a new write domain,
	 * let the object stay in multiple read domains
	 */
	if (obj->pending_write_domain == 0)
		obj->pending_read_domains |= obj->read_domains;
	else
		atomic_setbits_int(&obj_priv->io_flags, I915_DIRTY);

	/*
	 * Flush the current write domain if
	 * the new read domains don't match. Invalidate
	 * any read domains which differ from the old
	 * write domain
	 */
	if (obj->write_domain &&
	    obj->write_domain != obj->pending_read_domains) {
		flush_domains |= obj->write_domain;
		invalidate_domains |= obj->pending_read_domains &
		    ~obj->write_domain;
	}
	/*
	 * Invalidate any read caches which may have
	 * stale data. That is, any new read domains.
	 */
	invalidate_domains |= obj->pending_read_domains & ~obj->read_domains;
	/* clflush the cpu now, gpu caches get queued. */
	if ((flush_domains | invalidate_domains) & I915_GEM_DOMAIN_CPU) {
		bus_dmamap_sync(dev_priv->agpdmat, obj_priv->dmamap, 0,
		    obj->size, BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
	}
	if ((flush_domains | invalidate_domains) & I915_GEM_DOMAIN_GTT) {
		inteldrm_wipe_mappings(obj);
	}

	/* The actual obj->write_domain will be updated with
	 * pending_write_domain after we emit the accumulated flush for all of
	 * the domain changes in execuffer (which clears object's write
	 * domains). So if we have a current write domain that we aren't
	 * changing, set pending_write_domain to it.
	 */
	if (flush_domains == 0 && obj->pending_write_domain == 0 &&
	    (obj->pending_read_domains == obj->write_domain ||
	    obj->write_domain == 0))
		obj->pending_write_domain = obj->write_domain;
	obj->read_domains = obj->pending_read_domains;
	obj->pending_read_domains = 0;

	dev->invalidate_domains |= invalidate_domains;
	dev->flush_domains |= flush_domains;
}

/**
 * Pin an object to the GTT and evaluate the relocations landing in it.
 */
int
i915_gem_object_pin_and_relocate(struct drm_obj *obj,
    struct drm_file *file_priv, struct drm_i915_gem_exec_object2 *entry,
    struct drm_i915_gem_relocation_entry *relocs)
{
	struct drm_device			*dev = obj->dev;
	struct drm_i915_private			*dev_priv = dev->dev_private;
	struct drm_obj				*target_obj;
	struct inteldrm_obj			*obj_priv =
						    (struct inteldrm_obj *)obj;
	bus_space_handle_t			 bsh;
	int					 i, ret, needs_fence;

	needs_fence = (entry->flags & EXEC_OBJECT_NEEDS_FENCE) &&
	    obj_priv->tiling_mode != I915_TILING_NONE;
	if (needs_fence)
		atomic_setbits_int(&obj_priv->io_flags, I915_EXEC_NEEDS_FENCE);

	/* Choose the GTT offset for our buffer and put it there. */
	ret = i915_gem_object_pin(obj, (u_int32_t)entry->alignment,
	    needs_fence);
	/* XXX what if already bound at a different alignment? */
	if (ret)
		return ret;

	entry->offset = obj_priv->gtt_offset;

	/* Apply the relocations, using the GTT aperture to avoid cache
	 * flushing requirements.
	 */
	for (i = 0; i < entry->relocation_count; i++) {
		struct drm_i915_gem_relocation_entry *reloc = &relocs[i];
		struct inteldrm_obj *target_obj_priv;
		uint32_t reloc_val, reloc_offset;

		target_obj = drm_gem_object_lookup(obj->dev, file_priv,
		    reloc->target_handle);
		if (target_obj == NULL) {
			i915_gem_object_unpin(obj);
			return (EBADF);
		}
		target_obj_priv = (struct inteldrm_obj *)target_obj;

		/* The target buffer should have appeared before us in the
		 * exec_object list, so it should have a GTT space bound by now.
		 */
		if (target_obj_priv->dmamap == 0) {
			DRM_ERROR("No GTT space found for object %d\n",
				  reloc->target_handle);
			ret = EINVAL;
			goto err;
		}

		/* must be in one write domain and one only */
		if (reloc->write_domain & (reloc->write_domain - 1)) {
			ret = EINVAL;
			goto err;
		}
		if (reloc->read_domains & I915_GEM_DOMAIN_CPU ||
		    reloc->write_domain & I915_GEM_DOMAIN_CPU) {
			DRM_ERROR("relocation with read/write CPU domains: "
			    "obj %p target %d offset %d "
			    "read %08x write %08x", obj,
			    reloc->target_handle, (int)reloc->offset,
			    reloc->read_domains, reloc->write_domain);
			ret = EINVAL;
			goto err;
		}

		if (reloc->write_domain && target_obj->pending_write_domain &&
		    reloc->write_domain != target_obj->pending_write_domain) {
			DRM_ERROR("Write domain conflict: "
				  "obj %p target %d offset %d "
				  "new %08x old %08x\n",
				  obj, reloc->target_handle,
				  (int) reloc->offset,
				  reloc->write_domain,
				  target_obj->pending_write_domain);
			ret = EINVAL;
			goto err;
		}

		target_obj->pending_read_domains |= reloc->read_domains;
		target_obj->pending_write_domain |= reloc->write_domain;


		if (reloc->offset > obj->size - 4) {
			DRM_ERROR("Relocation beyond object bounds: "
				  "obj %p target %d offset %d size %d.\n",
				  obj, reloc->target_handle,
				  (int) reloc->offset, (int) obj->size);
			ret = EINVAL;
			goto err;
		}
		if (reloc->offset & 3) {
			DRM_ERROR("Relocation not 4-byte aligned: "
				  "obj %p target %d offset %d.\n",
				  obj, reloc->target_handle,
				  (int) reloc->offset);
			ret = EINVAL;
			goto err;
		}

		if (reloc->delta > target_obj->size) {
			DRM_ERROR("reloc larger than target\n");
			ret = EINVAL;
			goto err;
		}

		/* Map the page containing the relocation we're going to
		 * perform.
		 */
		reloc_offset = obj_priv->gtt_offset + reloc->offset;
		reloc_val = target_obj_priv->gtt_offset + reloc->delta;

		if ((ret = bus_space_subregion(dev_priv->bst,
		    dev_priv->aperture_bsh, trunc_page(reloc_offset),
		    PAGE_SIZE, &bsh)) != 0) {
			DRM_ERROR("map failed...\n");
			goto err;
		}
		/*
		 * we do this differently to linux, in the case where the
		 * presumed offset matches we actually read to check it's
		 * correct, but at least it won't involve idling the gpu if
		 * it was reading from it before, only if writing (which would
		 * be bad anyway since we're now using it as a command buffer).
		 */
		if (target_obj_priv->gtt_offset == reloc->presumed_offset) {
			ret = i915_gem_object_set_to_gtt_domain(obj, 0, 1);
			if (ret != 0)
				goto err;
			if (bus_space_read_4(dev_priv->bst, bsh,
			    reloc_offset & PAGE_MASK) == reloc_val) {
				drm_gem_object_unreference(target_obj);
				continue;
			}
			DRM_DEBUG("reloc tested and found incorrect\n");
		}

		ret = i915_gem_object_set_to_gtt_domain(obj, 1, 1);
		if (ret != 0)
			goto err;

		bus_space_write_4(dev_priv->bst, bsh, reloc_offset & PAGE_MASK,
		     reloc_val);

		reloc->presumed_offset = target_obj_priv->gtt_offset;

		drm_gem_object_unreference(target_obj);
	}

	return 0;

err:
	/* we always jump to here mid-loop */
	drm_gem_object_unreference(target_obj);
	i915_gem_object_unpin(obj);
	return (ret);
	
}

/** Dispatch a batchbuffer to the ring
 */
void
i915_dispatch_gem_execbuffer(struct drm_device *dev,
    struct drm_i915_gem_execbuffer2 *exec, uint64_t exec_offset)
{
	drm_i915_private_t	*dev_priv = dev->dev_private;
	uint32_t		 exec_start, exec_len;

	exec_start = (uint32_t)exec_offset + exec->batch_start_offset;
	exec_len = (uint32_t)exec->batch_len;

	if (IS_I830(dev_priv) || IS_845G(dev_priv)) {
		BEGIN_LP_RING(6);
		OUT_RING(MI_BATCH_BUFFER);
		OUT_RING(exec_start | MI_BATCH_NON_SECURE);
		OUT_RING(exec_start + exec_len - 4);
		OUT_RING(MI_NOOP);
	} else {
		BEGIN_LP_RING(4);
		if (IS_I965G(dev_priv)) {
			OUT_RING(MI_BATCH_BUFFER_START | (2 << 6) |
			    MI_BATCH_NON_SECURE_I965);
			OUT_RING(exec_start);
		} else {
			OUT_RING(MI_BATCH_BUFFER_START | (2 << 6));
			OUT_RING(exec_start | MI_BATCH_NON_SECURE);
		}
	}

	/*
	 * Ensure that the commands in the batch buffer are
	 * finished before the interrupt fires (from a subsequent request
	 * added). We get back a seqno representing the execution of the
	 * current buffer, which we can wait on.  We would like to mitigate
	 * these interrupts, likely by only creating seqnos occasionally
	 * (so that we have *some* interrupts representing completion of
	 * buffers that we can wait on when trying to clear up gtt space).
	 */
	OUT_RING(MI_FLUSH | MI_NO_WRITE_FLUSH);
	OUT_RING(MI_NOOP);
	ADVANCE_LP_RING();

	inteldrm_verify_inactive(dev_priv, __FILE__, __LINE__);
	/* The sampler always gets flushed on i965 (sigh) */
	if (IS_I965G(dev_priv))
		inteldrm_process_flushing(dev_priv, I915_GEM_DOMAIN_SAMPLER);
}

/* Throttle our rendering by waiting until the ring has completed our requests
 * emitted over 20 msec ago.
 *
 * This should get us reasonable parallelism between CPU and GPU but also
 * relatively low latency when blocking on a particular request to finish.
 */
int
i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file_priv)
{
#if 0
	struct inteldrm_file	*intel_file = (struct inteldrm_file *)file_priv;
	u_int32_t		 seqno;
#endif 
	int			 ret = 0;

	return ret;
}

int
i915_gem_get_relocs_from_user(struct drm_i915_gem_exec_object2 *exec_list,
    u_int32_t buffer_count, struct drm_i915_gem_relocation_entry **relocs)
{
	u_int32_t	reloc_count = 0, reloc_index = 0, i;
	int		ret;

	*relocs = NULL;
	for (i = 0; i < buffer_count; i++) {
		if (reloc_count + exec_list[i].relocation_count < reloc_count)
			return (EINVAL);
		reloc_count += exec_list[i].relocation_count;
	}

	if (reloc_count == 0)
		return (0);

	if (SIZE_MAX / reloc_count < sizeof(**relocs))
		return (EINVAL);
	*relocs = drm_alloc(reloc_count * sizeof(**relocs));
	for (i = 0; i < buffer_count; i++) {
		if ((ret = copyin((void *)(uintptr_t)exec_list[i].relocs_ptr,
		    &(*relocs)[reloc_index], exec_list[i].relocation_count *
		    sizeof(**relocs))) != 0) {
			drm_free(*relocs);
			*relocs = NULL;
			return (ret);
		}
		reloc_index += exec_list[i].relocation_count;
	}

	return (0);
}

int
i915_gem_put_relocs_to_user(struct drm_i915_gem_exec_object2 *exec_list,
    u_int32_t buffer_count, struct drm_i915_gem_relocation_entry *relocs)
{
	u_int32_t	reloc_count = 0, i;
	int		ret = 0;

	if (relocs == NULL)
		return (0);

	for (i = 0; i < buffer_count; i++) {
		if ((ret = copyout(&relocs[reloc_count],
		    (void *)(uintptr_t)exec_list[i].relocs_ptr,
		    exec_list[i].relocation_count * sizeof(*relocs))) != 0)
			break;
		reloc_count += exec_list[i].relocation_count;
	}

	drm_free(relocs);

	return (ret);
}

int
i915_gem_execbuffer2(struct drm_device *dev, void *data,
    struct drm_file *file_priv)
{
	drm_i915_private_t			*dev_priv = dev->dev_private;
	struct drm_i915_gem_execbuffer2		*args = data;
	struct drm_i915_gem_exec_object2	*exec_list = NULL;
	struct drm_i915_gem_relocation_entry	*relocs;
	struct inteldrm_obj			*obj_priv, *batch_obj_priv;
	struct drm_obj				**object_list = NULL;
	struct drm_obj				*batch_obj, *obj;
	size_t					 oflow;
	int					 ret, ret2, i;
	int					 pinned = 0, pin_tries;
	uint32_t				 reloc_index;

	/*
	 * Check for valid execbuffer offset. We can do this early because
	 * bound object are always page aligned, so only the start offset
	 * matters. Also check for integer overflow in the batch offset and size
	 */
	 if ((args->batch_start_offset | args->batch_len) & 0x7 ||
	    args->batch_start_offset + args->batch_len < args->batch_len ||
	    args->batch_start_offset + args->batch_len <
	    args->batch_start_offset)
		return (EINVAL);

	if (args->buffer_count < 1) {
		DRM_ERROR("execbuf with %d buffers\n", args->buffer_count);
		return (EINVAL);
	}
	/* Copy in the exec list from userland, check for overflow */
	oflow = SIZE_MAX / args->buffer_count;
	if (oflow < sizeof(*exec_list) || oflow < sizeof(*object_list))
		return (EINVAL);
	exec_list = drm_alloc(sizeof(*exec_list) * args->buffer_count);
	object_list = drm_alloc(sizeof(*object_list) * args->buffer_count);
	if (exec_list == NULL || object_list == NULL) {
		ret = ENOMEM;
		goto pre_mutex_err;
	}
	ret = copyin((void *)(uintptr_t)args->buffers_ptr, exec_list,
	    sizeof(*exec_list) * args->buffer_count);
	if (ret != 0)
		goto pre_mutex_err;

	ret = i915_gem_get_relocs_from_user(exec_list, args->buffer_count,
	    &relocs);
	if (ret != 0)
		goto pre_mutex_err;

	DRM_LOCK();
	inteldrm_verify_inactive(dev_priv, __FILE__, __LINE__);

	/* XXX check these before we copyin... but we do need the lock */
	if (dev_priv->mm.wedged) {
		ret = EIO;
		goto unlock;
	}

	if (dev_priv->mm.suspended) {
		ret = EBUSY;
		goto unlock;
	}

	/* Look up object handles */
	for (i = 0; i < args->buffer_count; i++) {
		object_list[i] = drm_gem_object_lookup(dev, file_priv,
		    exec_list[i].handle);
		if (object_list[i] == NULL) {
			DRM_ERROR("Invalid object handle %d at index %d\n",
				   exec_list[i].handle, i);
			ret = EBADF;
			goto err;
		}
		obj_priv = (struct inteldrm_obj *)object_list[i];
		if (obj_priv->io_flags & I915_IN_EXEC) {
			DRM_ERROR("Object %p appears more than once in object_list\n",
			    object_list[i]);
			ret = EBADF;
			goto err;
		}
		atomic_setbits_int(&obj_priv->io_flags, I915_IN_EXEC);
	}

	/* Pin and relocate */
	for (pin_tries = 0; ; pin_tries++) {
		ret = pinned = 0;
		reloc_index = 0;

		for (i = 0; i < args->buffer_count; i++) {
			object_list[i]->pending_read_domains = 0;
			object_list[i]->pending_write_domain = 0;
			ret = i915_gem_object_pin_and_relocate(object_list[i],
			    file_priv, &exec_list[i], &relocs[reloc_index]);
			if (ret)
				break;
			pinned++;
			reloc_index += exec_list[i].relocation_count;
		}
		/* success */
		if (ret == 0)
			break;

		/* error other than GTT full, or we've already tried again */
		if (ret != ENOSPC || pin_tries >= 1)
			goto err;

		/* unpin all of our buffers */
		for (i = 0; i < pinned; i++)
			i915_gem_object_unpin(object_list[i]);
		/* evict everyone we can from the aperture */
		ret = i915_gem_evict_everything(dev_priv, 1);
		if (ret)
			goto err;
	}

	/* Set the pending read domains for the batch buffer to COMMAND */
	batch_obj = object_list[args->buffer_count - 1];
	batch_obj_priv = (struct inteldrm_obj *)batch_obj;
	if (args->batch_start_offset + args->batch_len > batch_obj->size ||
	    batch_obj->pending_write_domain) {
		ret = EINVAL;
		goto err;
	}
	batch_obj->pending_read_domains |= I915_GEM_DOMAIN_COMMAND;

	inteldrm_verify_inactive(dev_priv, __FILE__, __LINE__);

	/*
	 * Zero the global flush/invalidate flags. These will be modified as
	 * new domains are computed for each object
	 */
	dev->invalidate_domains = 0;
	dev->flush_domains = 0;

	/* Compute new gpu domains and update invalidate/flush */
	for (i = 0; i < args->buffer_count; i++)
		i915_gem_object_set_to_gpu_domain(object_list[i]);

	inteldrm_verify_inactive(dev_priv, __FILE__, __LINE__);

	/* flush and invalidate any domains that need them. */
	(void)i915_gem_flush(dev_priv, dev->invalidate_domains,
	    dev->flush_domains);

	/*
	 * update the write domains, and fence/gpu write accounting information.
	 * Also do the move to active list here. The lazy seqno accounting will
	 * make sure that they have the correct seqno. If the add_request
	 * fails, then we will wait for a later batch (or one added on the
	 * wait), which will waste some time, but if we're that low on memory
	 * then we could fail in much worse ways.
	 */
	for (i = 0; i < args->buffer_count; i++) {
		obj = object_list[i];
		obj_priv = (struct inteldrm_obj *)obj;

		obj->write_domain = obj->pending_write_domain;
		/*
		 * if we have a write domain, add us to the gpu write list
		 * else we can remove the bit because it has been flushed.
		 */
		if (obj_priv->io_flags & I915_GPU_WRITE)
			TAILQ_REMOVE(&dev_priv->mm.gpu_write_list, obj_priv,
			     write_list);
		if (obj->write_domain) {
			TAILQ_INSERT_TAIL(&dev_priv->mm.gpu_write_list,
			    obj_priv, write_list);
			atomic_setbits_int(&obj_priv->io_flags, I915_GPU_WRITE);
		} else {
			atomic_clearbits_int(&obj_priv->io_flags,
			     I915_GPU_WRITE);
		}
		/* if this batchbuffer needs a fence, then the object is
		 * counted as fenced exec. else any outstanding fence waits
		 * will just wait on the fence last_seqno.
		 */
		if (inteldrm_exec_needs_fence(obj_priv)) {
			atomic_setbits_int(&obj_priv->io_flags,
			    I915_FENCED_EXEC);
		} else {
			atomic_clearbits_int(&obj_priv->io_flags,
			    I915_FENCED_EXEC);
		}
			
		i915_gem_object_move_to_active(object_list[i]);
	}

	inteldrm_verify_inactive(dev_priv, __FILE__, __LINE__);

	/* Exec the batchbuffer */
	/*
	 * XXX make sure that this may never fail by preallocating the request.
	 */
	i915_dispatch_gem_execbuffer(dev, args, batch_obj_priv->gtt_offset);

	/*
	 * move to active associated all previous buffers with the seqno
	 * that this call will emit. so we don't need the return.
	 */
	(void)i915_add_request(dev_priv);

	inteldrm_verify_inactive(dev_priv, __FILE__, __LINE__);

	ret = copyout(exec_list, (void *)(uintptr_t)args->buffers_ptr,
	    sizeof(*exec_list) * args->buffer_count);

err:
	for (i = 0; i < args->buffer_count; i++) {
		if (object_list[i] == NULL)
			break;

		obj_priv = (struct inteldrm_obj *)object_list[i];
		if (i < pinned)
			i915_gem_object_unpin(object_list[i]);

		atomic_clearbits_int(&obj_priv->io_flags, I915_IN_EXEC |
		    I915_EXEC_NEEDS_FENCE);
		drm_gem_object_unreference(object_list[i]);
	}

unlock:
	DRM_UNLOCK();

pre_mutex_err:
	/* update userlands reloc state. */
	ret2 = i915_gem_put_relocs_to_user(exec_list,
	    args->buffer_count, relocs);
	if (ret2 != 0 && ret == 0)
		ret = ret2;

	drm_free(object_list);
	drm_free(exec_list);

	return ret;
}

int
i915_gem_object_pin(struct drm_obj *obj, uint32_t alignment, int needs_fence)
{
	struct drm_device	*dev = obj->dev;
	struct inteldrm_obj	*obj_priv = (struct inteldrm_obj *)obj;
	int			 ret;

	inteldrm_verify_inactive(dev_priv, __FILE__, __LINE__);
	/*
	 * if already bound, but alignment is unsuitable, unbind so we can
	 * fix it. Similarly if we have constraints due to fence registers,
	 * adjust if needed. Note that if we are already pinned we may as well
	 * fail because whatever depends on this alignment will render poorly
	 * otherwise, so just fail the pin (with a printf so we can fix a
	 * wrong userland).
	 */
	if ((alignment && obj_priv->dmamap != NULL &&
	    obj_priv->gtt_offset & (alignment - 1)) || (needs_fence &&
	    !i915_gem_object_fence_offset_ok(obj, obj_priv->tiling_mode))) {
		if (obj_priv->pin_count == 0) {
			ret = i915_gem_object_unbind(obj, 1);
			if (ret)
				return (ret);
		} else {
			DRM_ERROR("repinning an object with bad alignment\n");
		}
			return (EINVAL);
	}

	if (obj_priv->dmamap == NULL) {
		ret = i915_gem_object_bind_to_gtt(obj, alignment, 1);
		if (ret != 0)
			return (ret);
	}

	/*
	 * Pre-965 chips may need a fence register set up in order to
	 * handle tiling properly. GTT mapping may have blown it away so
	 * restore.
	 * With execbuf2 support we don't always need it, but if we do grab
	 * it.
	 */
	if (needs_fence && obj_priv->tiling_mode != I915_TILING_NONE &&
	    (ret = i915_gem_get_fence_reg(obj, 1)) != 0)
		return (ret);

	/* If the object is not active and not pending a flush,
	 * remove it from the inactive list
	 */
	if (++obj_priv->pin_count == 1) {
		atomic_inc(&dev->pin_count);
		atomic_add(obj->size, &dev->pin_memory);
		if (!inteldrm_is_active(obj_priv))
			i915_list_remove(obj_priv);
	}
	inteldrm_verify_inactive(dev_priv, __FILE__, __LINE__);

	return (0);
}

void
i915_gem_object_unpin(struct drm_obj *obj)
{
	struct drm_device	*dev = obj->dev;
	struct inteldrm_obj	*obj_priv = (struct inteldrm_obj *)obj;

	inteldrm_verify_inactive(dev_priv, __FILE__, __LINE__);
	KASSERT(obj_priv->pin_count >= 1);
	KASSERT(obj_priv->dmamap != NULL);

	/* If the object is no longer pinned, and is
	 * neither active nor being flushed, then stick it on
	 * the inactive list
	 */
	if (--obj_priv->pin_count == 0) {
		if (!inteldrm_is_active(obj_priv))
			i915_gem_object_move_to_inactive(obj);
		atomic_dec(&dev->pin_count);
		atomic_sub(obj->size, &dev->pin_memory);
	}
	inteldrm_verify_inactive(dev_priv, __FILE__, __LINE__);
}

int
i915_gem_pin_ioctl(struct drm_device *dev, void *data,
		   struct drm_file *file_priv)
{
	struct drm_i915_gem_pin	*args = data;
	struct drm_obj		*obj;
	struct inteldrm_obj	*obj_priv;
	int			 ret = 0;

	obj = drm_gem_object_lookup(dev, file_priv, args->handle);
	if (obj == NULL)
		return (EBADF);

	obj_priv = (struct inteldrm_obj *)obj;
	DRM_LOCK();
	if (i915_obj_purgeable(obj_priv)) {
		printf("pinning purgeable object\n");
		ret = EINVAL;
		goto out;
	}

	if (++obj_priv->user_pin_count == 1) {
		ret = i915_gem_object_pin(obj, args->alignment, 1);
		if (ret != 0)
			goto out;
	}

	/* XXX - flush the CPU caches for pinned objects
	 * as the X server doesn't manage domains yet
	 */
	i915_gem_object_set_to_gtt_domain(obj, 1, 1);
	args->offset = obj_priv->gtt_offset;

out:
	drm_gem_object_unreference(obj);
	DRM_UNLOCK();

	return (ret);
}

int
i915_gem_unpin_ioctl(struct drm_device *dev, void *data,
		     struct drm_file *file_priv)
{
	struct drm_i915_gem_pin	*args = data;
	struct inteldrm_obj	*obj_priv;
	struct drm_obj		*obj;

	obj = drm_gem_object_lookup(dev, file_priv, args->handle);
	if (obj == NULL)
		return (EBADF);

	DRM_LOCK();
	obj_priv = (struct inteldrm_obj *)obj;
	if (obj_priv->user_pin_count == 0) {
		DRM_UNLOCK();
		return (EINVAL);
	}

	if (--obj_priv->user_pin_count == 0)
		i915_gem_object_unpin(obj);

	drm_gem_object_unreference(obj);
	DRM_UNLOCK();
	return (0);
}

int
i915_gem_busy_ioctl(struct drm_device *dev, void *data,
    struct drm_file *file_priv)
{
	struct drm_i915_private		*dev_priv = dev->dev_private;
	struct drm_i915_gem_busy	*args = data;
	struct drm_obj			*obj;
	struct inteldrm_obj		*obj_priv;

	obj = drm_gem_object_lookup(dev, file_priv, args->handle);
	if (obj == NULL) {
		DRM_ERROR("Bad handle in i915_gem_busy_ioctl(): %d\n",
			  args->handle);
		return (EBADF);
	}

	DRM_LOCK();
	/*
	 * Update the active list for the hardware's current position.
	 * otherwise this will only update on a delayed timer or when
	 * the irq is unmasked. This keeps our working set smaller.
	 */
	i915_gem_retire_requests(dev_priv);

	obj_priv = (struct inteldrm_obj *)obj;
	/*
	 * Don't count being on the flushing list being done. Otherwise, a
	 * buffer left on the flushing list but not getting flushed (because
	 * no one is flushing that domain) won't ever return unbusy and get
	 * reused by libdrm's bo cache. The other expected consumer of this
	 * interface, OpenGL's occlusion queries, also specs that the object
	 * get unbusy "eventually" without any interference.
	 */
	args->busy = inteldrm_is_active(obj_priv) &&
	    obj_priv->last_rendering_seqno != 0;

	drm_gem_object_unreference(obj);
	DRM_UNLOCK();
	return 0;
}

int
i915_gem_madvise_ioctl(struct drm_device *dev, void *data,
    struct drm_file *file_priv)
{
	struct drm_i915_gem_madvise	*args = data;
	struct drm_obj			*obj;
	struct inteldrm_obj		*obj_priv;
	int				 need, ret = 0;

	switch (args->madv) {
	case I915_MADV_DONTNEED:
		need = 0;
		break;
	case I915_MADV_WILLNEED:
		need = 1;
		break;
	default:
		return (EINVAL);
	}

	obj = drm_gem_object_lookup(dev, file_priv, args->handle);
	if (obj == NULL)
		return (EBADF);

	obj_priv = (struct inteldrm_obj *)obj;
	DRM_LOCK();

	/* invalid to madvise on a pinned BO */
	if (obj_priv->pin_count) {
		ret = EINVAL;
		goto out;
	}

	if (!i915_obj_purged(obj_priv)) {
		if (need) {
			atomic_clearbits_int(&obj_priv->io_flags,
			    I915_DONTNEED);
		} else {
			atomic_setbits_int(&obj_priv->io_flags, I915_DONTNEED);
		}
	}


	/* if the object is no longer bound, discard its backing storage */
	if (i915_obj_purgeable(obj_priv) && obj_priv->dmamap == NULL)
		inteldrm_purge_obj(obj);
		
	args->retained = !i915_obj_purged(obj_priv);

out:
	drm_gem_object_unreference(obj);
	DRM_UNLOCK();

	return (ret);
}

int
i915_gem_init_object(struct drm_obj *obj)
{
	struct inteldrm_obj	*obj_priv = (struct inteldrm_obj *)obj;

	/*
	 * We've just allocated pages from the kernel,
	 * so they've just been written by the CPU with
	 * zeros. They'll need to be flushed before we
	 * use them with the GPU.
	 */
	obj->write_domain = I915_GEM_DOMAIN_CPU;
	obj->read_domains = I915_GEM_DOMAIN_CPU;

	/* normal objects don't need special treatment */
	obj_priv->dma_flags = 0;
	obj_priv->fence_reg = I915_FENCE_REG_NONE;

	return 0;
}

/*
 * NOTE all object unreferences in this driver need to hold the DRM_LOCK(),
 * because if they free they poke around in driver structures.
 */
void
i915_gem_free_object(struct drm_obj *obj)
{
	struct inteldrm_obj	*obj_priv = (struct inteldrm_obj *)obj;

	while (obj_priv->pin_count > 0)
		i915_gem_object_unpin(obj);

	i915_gem_object_unbind(obj, 0);
	/* XXX dmatag went away? */
}

/** Unbinds all objects that are on the given buffer list. */
int
i915_gem_evict_inactive(struct drm_i915_private *dev_priv)
{
	struct inteldrm_obj	*obj_priv;
	int			 ret;

	while ((obj_priv = TAILQ_FIRST(&dev_priv->mm.inactive_list)) != NULL) {
		if (obj_priv->pin_count != 0) {
			DRM_ERROR("Pinned object in unbind list\n");
			return (EINVAL);
		}

		if ((ret = i915_gem_object_unbind(&obj_priv->obj, 1)) != 0)
			break;
	}

	return (ret);
}

int
i915_gem_idle(struct drm_i915_private *dev_priv)
{
	struct drm_device	*dev = (struct drm_device *)dev_priv->drmdev;
	int			 ret;

	DRM_LOCK();
	if (dev_priv->mm.suspended || dev_priv->ring.ring_obj == NULL) { 
		DRM_UNLOCK();
		return (0);
	}

	/*
	 * If we're wedged, the workq will clear everything, else this will
	 * empty out the lists for us.
	 */
	if ((ret = i915_gem_evict_everything(dev_priv, 1)) != 0 && ret != ENOSPC) {
		DRM_UNLOCK();
		return (ret);
	}

	/* Hack!  Don't let anybody do execbuf while we don't control the chip.
	 * We need to replace this with a semaphore, or something.
	 */
	dev_priv->mm.suspended = 1;
	/* if we hung then the timer alredy fired. */
	timeout_del(&dev_priv->mm.hang_timer);

	inteldrm_update_ring(dev_priv);
	i915_gem_cleanup_ringbuffer(dev_priv);
	DRM_UNLOCK();

	/* this should be idle now */
	timeout_del(&dev_priv->mm.retire_timer);

	return 0;
}

int
i915_gem_init_hws(struct drm_i915_private *dev_priv)
{
	struct drm_device	*dev = (struct drm_device *)dev_priv->drmdev;
	struct drm_obj		*obj;
	struct inteldrm_obj	*obj_priv;
	int			 ret;

	/* If we need a physical address for the status page, it's already
	 * initialized at driver load time.
	 */
	if (!I915_NEED_GFX_HWS(dev_priv))
		return 0;

	obj = drm_gem_object_alloc(dev, 4096);
	if (obj == NULL) {
		DRM_ERROR("Failed to allocate status page\n");
		return (ENOMEM);
	}
	obj_priv = (struct inteldrm_obj *)obj;
	/*
	 * snooped gtt mapping please .
	 * Normally this flag is only to dmamem_map, but it's been overloaded
	 * for the agp mapping
	 */
	obj_priv->dma_flags = BUS_DMA_COHERENT | BUS_DMA_READ;

	ret = i915_gem_object_pin(obj, 4096, 0);
	if (ret != 0) {
		drm_gem_object_unreference(obj);
		return ret;
	}

	dev_priv->hw_status_page = (void *)vm_map_min(kernel_map);
	obj->uao->pgops->pgo_reference(obj->uao);
	if ((ret = uvm_map(kernel_map, (vaddr_t *)&dev_priv->hw_status_page,
	    PAGE_SIZE, obj->uao, 0, 0, UVM_MAPFLAG(UVM_PROT_RW, UVM_PROT_RW,
	    UVM_INH_SHARE, UVM_ADV_RANDOM, 0))) != 0)
	if (ret != 0) {
		DRM_ERROR("Failed to map status page.\n");
		obj->uao->pgops->pgo_detach(obj->uao);
		memset(&dev_priv->hws_map, 0, sizeof(dev_priv->hws_map));
		i915_gem_object_unpin(obj);
		drm_gem_object_unreference(obj);
		return (EINVAL);
	}
	dev_priv->hws_obj = obj;
	memset(dev_priv->hw_status_page, 0, PAGE_SIZE);
	I915_WRITE(HWS_PGA, obj_priv->gtt_offset);
	I915_READ(HWS_PGA); /* posting read */
	DRM_DEBUG("hws offset: 0x%08x\n", obj_priv->gtt_offset);

	return 0;
}

void
i915_gem_cleanup_hws(struct drm_i915_private *dev_priv)
{
	struct drm_obj		*obj;
	struct inteldrm_obj	*obj_priv;

	if (dev_priv->hws_obj == NULL)
		return;

	obj = dev_priv->hws_obj;
	obj_priv = (struct inteldrm_obj *)obj;

	uvm_unmap(kernel_map, (vaddr_t)dev_priv->hw_status_page,
	    (vaddr_t)dev_priv->hw_status_page + PAGE_SIZE);
	i915_gem_object_unpin(obj);
	drm_gem_object_unreference(obj);
	dev_priv->hws_obj = NULL;

	memset(&dev_priv->hws_map, 0, sizeof(dev_priv->hws_map));
	dev_priv->hw_status_page = NULL;

	/* Write high address into HWS_PGA when disabling. */
	I915_WRITE(HWS_PGA, 0x1ffff000);
}

int
i915_gem_init_ringbuffer(struct drm_i915_private *dev_priv)
{
	struct drm_device	*dev = (struct drm_device *)dev_priv->drmdev;
	struct drm_obj		*obj;
	struct inteldrm_obj	*obj_priv;
	int			 ret;

	ret = i915_gem_init_hws(dev_priv);
	if (ret != 0)
		return ret;

	obj = drm_gem_object_alloc(dev, 128 * 1024);
	if (obj == NULL) {
		DRM_ERROR("Failed to allocate ringbuffer\n");
		ret = ENOMEM;
		goto delhws;
	}
	obj_priv = (struct inteldrm_obj *)obj;

	ret = i915_gem_object_pin(obj, 4096, 0);
	if (ret != 0)
		goto unref;

	/* Set up the kernel mapping for the ring. */
	dev_priv->ring.size = obj->size;

	/* XXX WC */
	if ((ret = bus_space_subregion(dev_priv->bst, dev_priv->aperture_bsh, 
	    obj_priv->gtt_offset, obj->size, &dev_priv->ring.bsh)) != 0) {
		DRM_INFO("can't map ringbuffer\n");
		goto unpin;
	}
	dev_priv->ring.ring_obj = obj;

	if ((ret = inteldrm_start_ring(dev_priv)) != 0)
		goto unmap;

	return (0);

unmap:
unpin:
	memset(&dev_priv->ring, 0, sizeof(dev_priv->ring));
	i915_gem_object_unpin(obj);
unref:
	drm_gem_object_unreference(obj);
delhws:
	i915_gem_cleanup_hws(dev_priv);
	return (ret);
}

int
inteldrm_start_ring(struct drm_i915_private *dev_priv)
{
	struct drm_obj		*obj = dev_priv->ring.ring_obj;
	struct inteldrm_obj	*obj_priv = (struct inteldrm_obj *)obj;
	u_int32_t		 head;

	/* Stop the ring if it's running. */
	I915_WRITE(PRB0_CTL, 0);
	I915_WRITE(PRB0_TAIL, 0);
	I915_WRITE(PRB0_HEAD, 0);

	/* Initialize the ring. */
	I915_WRITE(PRB0_START, obj_priv->gtt_offset);
	head = I915_READ(PRB0_HEAD) & HEAD_ADDR;

	/* G45 ring initialisation fails to reset head to zero */
	if (head != 0) {
		I915_WRITE(PRB0_HEAD, 0);
		DRM_DEBUG("Forced ring head to zero ctl %08x head %08x"
		    "tail %08x start %08x\n", I915_READ(PRB0_CTL),
		    I915_READ(PRB0_HEAD), I915_READ(PRB0_TAIL),
		    I915_READ(PRB0_START));
	}

	I915_WRITE(PRB0_CTL, ((obj->size - 4096) & RING_NR_PAGES) |
	    RING_NO_REPORT | RING_VALID);

	head = I915_READ(PRB0_HEAD) & HEAD_ADDR;
	/* If ring head still != 0, the ring is dead */
	if (head != 0) {
		DRM_ERROR("Ring initialisation failed: ctl %08x head %08x"
		    "tail %08x start %08x\n", I915_READ(PRB0_CTL),
		    I915_READ(PRB0_HEAD), I915_READ(PRB0_TAIL),
		    I915_READ(PRB0_START));
		return (EIO);
	}

	/* Update our cache of the ring state */
	inteldrm_update_ring(dev_priv);

	return (0);
}

void
i915_gem_cleanup_ringbuffer(struct drm_i915_private *dev_priv)
{
	if (dev_priv->ring.ring_obj == NULL)
		return;

	i915_gem_object_unpin(dev_priv->ring.ring_obj);
	drm_gem_object_unreference(dev_priv->ring.ring_obj);
	dev_priv->ring.ring_obj = NULL;
	memset(&dev_priv->ring, 0, sizeof(dev_priv->ring));

	i915_gem_cleanup_hws(dev_priv);
}

int
i915_gem_entervt_ioctl(struct drm_device *dev, void *data,
		       struct drm_file *file_priv)
{
	drm_i915_private_t *dev_priv = dev->dev_private;
	int ret;

	if (dev_priv->mm.wedged) {
		DRM_ERROR("Reenabling wedged hardware, good luck\n");
		dev_priv->mm.wedged = 0;
	}


	DRM_LOCK();
	dev_priv->mm.suspended = 0;

	ret = i915_gem_init_ringbuffer(dev_priv);
	if (ret != 0) {
		DRM_UNLOCK();
		return (ret);
	}

	/* gtt mapping means that the inactive list may not be empty */
	KASSERT(TAILQ_EMPTY(&dev_priv->mm.active_list));
	KASSERT(TAILQ_EMPTY(&dev_priv->mm.flushing_list));
	KASSERT(TAILQ_EMPTY(&dev_priv->mm.request_list));
	DRM_UNLOCK();

	drm_irq_install(dev);

	return (0);
}

int
i915_gem_leavevt_ioctl(struct drm_device *dev, void *data,
		       struct drm_file *file_priv)
{
	struct drm_i915_private	*dev_priv = dev->dev_private;
	int			 ret;

	ret = i915_gem_idle(dev_priv);
	drm_irq_uninstall(dev);
	return (ret);
}

void
inteldrm_timeout(void *arg)
{
	drm_i915_private_t *dev_priv = arg;

	if (workq_add_task(dev_priv->workq, 0, i915_gem_retire_work_handler,
	    dev_priv, NULL) == ENOMEM)
		DRM_ERROR("failed to run retire handler\n");
}

/*
 * handle hung hardware, or error interrupts. for now print debug info.
 */
void
inteldrm_error(struct drm_i915_private *dev_priv)
{
	u_int32_t	eir, ipeir, pgtbl_err, pipea_stats, pipeb_stats;
	u_int8_t	reset = GDRST_RENDER;

	eir = I915_READ(EIR);
	pipea_stats = I915_READ(PIPEASTAT);
	pipeb_stats = I915_READ(PIPEBSTAT);

	/*
	 * only actually check the error bits if we register one.
	 * else we just hung, stay silent.
	 */
	if (eir != 0) {
		printf("render error detected, EIR: 0x%08x\n", eir);
		if (IS_G4X(dev_priv)) {
			if (eir & (GM45_ERROR_MEM_PRIV | GM45_ERROR_CP_PRIV)) {
				ipeir = I915_READ(IPEIR_I965);

				printf("  IPEIR: 0x%08x\n",
				    I915_READ(IPEIR_I965));
				printf("  IPEHR: 0x%08x\n",
				    I915_READ(IPEHR_I965));
				printf("  INSTDONE: 0x%08x\n",
				    I915_READ(INSTDONE_I965));
				printf("  INSTPS: 0x%08x\n",
				    I915_READ(INSTPS));
				printf("  INSTDONE1: 0x%08x\n",
				    I915_READ(INSTDONE1));
				printf("  ACTHD: 0x%08x\n",
				    I915_READ(ACTHD_I965));
				I915_WRITE(IPEIR_I965, ipeir);
				(void)I915_READ(IPEIR_I965);
			}
			if (eir & GM45_ERROR_PAGE_TABLE) {
				pgtbl_err = I915_READ(PGTBL_ER);
				printf("page table error\n");
				printf("  PGTBL_ER: 0x%08x\n", pgtbl_err);
				I915_WRITE(PGTBL_ER, pgtbl_err);
				(void)I915_READ(PGTBL_ER);
				dev_priv->mm.wedged = 1;
				reset = GDRST_FULL;

			}
		} else if (IS_I9XX(dev_priv) && eir & I915_ERROR_PAGE_TABLE) {
			pgtbl_err = I915_READ(PGTBL_ER);
			printf("page table error\n");
			printf("  PGTBL_ER: 0x%08x\n", pgtbl_err);
			I915_WRITE(PGTBL_ER, pgtbl_err);
			(void)I915_READ(PGTBL_ER);
			dev_priv->mm.wedged = 1;
			reset = GDRST_FULL;
		}
		if (eir & I915_ERROR_MEMORY_REFRESH) {
			printf("memory refresh error\n");
			printf("PIPEASTAT: 0x%08x\n",
			       pipea_stats);
			printf("PIPEBSTAT: 0x%08x\n",
			       pipeb_stats);
			/* pipestat has already been acked */
		}
		if (eir & I915_ERROR_INSTRUCTION) {
			printf("instruction error\n");
			printf("  INSTPM: 0x%08x\n",
			       I915_READ(INSTPM));
			if (!IS_I965G(dev_priv)) {
				ipeir = I915_READ(IPEIR);

				printf("  IPEIR: 0x%08x\n",
				       I915_READ(IPEIR));
				printf("  IPEHR: 0x%08x\n",
					   I915_READ(IPEHR));
				printf("  INSTDONE: 0x%08x\n",
					   I915_READ(INSTDONE));
				printf("  ACTHD: 0x%08x\n",
					   I915_READ(ACTHD));
				I915_WRITE(IPEIR, ipeir);
				(void)I915_READ(IPEIR);
			} else {
				ipeir = I915_READ(IPEIR_I965);

				printf("  IPEIR: 0x%08x\n",
				       I915_READ(IPEIR_I965));
				printf("  IPEHR: 0x%08x\n",
				       I915_READ(IPEHR_I965));
				printf("  INSTDONE: 0x%08x\n",
				       I915_READ(INSTDONE_I965));
				printf("  INSTPS: 0x%08x\n",
				       I915_READ(INSTPS));
				printf("  INSTDONE1: 0x%08x\n",
				       I915_READ(INSTDONE1));
				printf("  ACTHD: 0x%08x\n",
				       I915_READ(ACTHD_I965));
				I915_WRITE(IPEIR_I965, ipeir);
				(void)I915_READ(IPEIR_I965);
			}
		}

		I915_WRITE(EIR, eir);
		eir = I915_READ(EIR);
	}
	/*
	 * nasty errors don't clear and need a reset, mask them until we reset
	 * else we'll get infinite interrupt storms.
	 */
	if (eir) {
		/* print so we know that we may want to reset here too */
		if (dev_priv->mm.wedged == 0)
			DRM_ERROR("EIR stuck: 0x%08x, masking\n", eir);
		I915_WRITE(EMR, I915_READ(EMR) | eir);
		I915_WRITE(IIR, I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT);
	}
	/*
	 * if it was a pagetable error, or we were called from hangcheck, then
	 * reset the gpu.
	 */
	if (dev_priv->mm.wedged && workq_add_task(dev_priv->workq, 0,
	    inteldrm_hung, dev_priv, (void *)(uintptr_t)reset) == ENOMEM)
		DRM_INFO("failed to schedule reset task\n");

}

void
inteldrm_hung(void *arg, void *reset_type)
{
	struct drm_i915_private	*dev_priv = arg;
	struct drm_device	*dev = (struct drm_device *)dev_priv->drmdev;
	struct inteldrm_obj	*obj_priv;
	u_int8_t		 reset = (u_int8_t)(uintptr_t)reset_type;

	DRM_LOCK();
	if (HAS_RESET(dev_priv)) {
		DRM_INFO("resetting gpu: ");
		inteldrm_965_reset(dev_priv, reset);
		printf("done!\n");
	} else
		printf("no reset function for chipset.\n");

	/*
	 * Clear out all of the requests and make everything inactive.
	 */
	i915_gem_retire_requests(dev_priv);

	/*
	 * Clear the active and flushing lists to inactive. Since
	 * we've reset the hardware then they're not going to get
	 * flushed or completed otherwise. nuke the domains since
	 * they're now irrelavent.
	 */
	while ((obj_priv = TAILQ_FIRST(&dev_priv->mm.active_list)) != NULL) {
		if (obj_priv->obj.write_domain & I915_GEM_GPU_DOMAINS) {
			TAILQ_REMOVE(&dev_priv->mm.gpu_write_list,
			    obj_priv, write_list);
			atomic_clearbits_int(&obj_priv->io_flags,
			     I915_GPU_WRITE);
			obj_priv->obj.write_domain &= ~I915_GEM_GPU_DOMAINS;
		}
		i915_gem_object_move_to_inactive(&obj_priv->obj);;
	}

	while ((obj_priv = TAILQ_FIRST(&dev_priv->mm.flushing_list)) != NULL) {
		if (obj_priv->obj.write_domain & I915_GEM_GPU_DOMAINS) {
			TAILQ_REMOVE(&dev_priv->mm.gpu_write_list,
			    obj_priv, write_list);
			atomic_clearbits_int(&obj_priv->io_flags,
			     I915_GPU_WRITE);
			obj_priv->obj.write_domain &= ~I915_GEM_GPU_DOMAINS;
		}
		i915_gem_object_move_to_inactive(&obj_priv->obj);
	}

	/* unbind everything */
	(void)i915_gem_evict_inactive(dev_priv);

	if (HAS_RESET(dev_priv))
		dev_priv->mm.wedged = 0;
	DRM_UNLOCK();
}

void
inteldrm_hangcheck(void *arg)
{
	struct drm_i915_private	*dev_priv = arg;
	u_int32_t		 acthd;

	/* are we idle? */
	if (TAILQ_EMPTY(&dev_priv->mm.request_list) ||
	    i915_seqno_passed(i915_get_gem_seqno(dev_priv),
	    TAILQ_LAST(&dev_priv->mm.request_list, i915_request)->seqno)) {
		dev_priv->mm.hang_cnt = 0;
		return;
	}

	if (IS_I965G(dev_priv))
		acthd = I915_READ(ACTHD_I965);
	else
		acthd = I915_READ(ACTHD);

	/* if we've hit ourselves before and the hardware hasn't moved, hung. */
	if (dev_priv->mm.last_acthd == acthd) {
		/* if that's twice we didn't hit it, then we're hung */
		if (++dev_priv->mm.hang_cnt >= 2) {
			dev_priv->mm.hang_cnt = 0;
			/* XXX atomic */
			dev_priv->mm.wedged = 1; 
			DRM_INFO("gpu hung!\n");
			/* XXX locking */
			wakeup(dev_priv);
			inteldrm_error(dev_priv);
			return;
		} 
	} else {
		dev_priv->mm.hang_cnt = 0;
	}

	dev_priv->mm.last_acthd = acthd;
	/* Set ourselves up again, in case we haven't added another batch */
	timeout_add_msec(&dev_priv->mm.hang_timer, 750);
}

void
i915_move_to_tail(struct inteldrm_obj *obj_priv, struct i915_gem_list *head)
{
	i915_list_remove(obj_priv);
	TAILQ_INSERT_TAIL(head, obj_priv, list);
	obj_priv->current_list = head;
}

void
i915_list_remove(struct inteldrm_obj *obj_priv)
{
	if (obj_priv->current_list != NULL)
		TAILQ_REMOVE(obj_priv->current_list, obj_priv, list);
	obj_priv->current_list = NULL;
}

/*
 *
 * Support for managing tiling state of buffer objects.
 *
 * The idea behind tiling is to increase cache hit rates by rearranging
 * pixel data so that a group of pixel accesses are in the same cacheline.
 * Performance improvement from doing this on the back/depth buffer are on
 * the order of 30%.
 *
 * Intel architectures make this somewhat more complicated, though, by
 * adjustments made to addressing of data when the memory is in interleaved
 * mode (matched pairs of DIMMS) to improve memory bandwidth.
 * For interleaved memory, the CPU sends every sequential 64 bytes
 * to an alternate memory channel so it can get the bandwidth from both.
 *
 * The GPU also rearranges its accesses for increased bandwidth to interleaved
 * memory, and it matches what the CPU does for non-tiled.  However, when tiled
 * it does it a little differently, since one walks addresses not just in the
 * X direction but also Y.  So, along with alternating channels when bit
 * 6 of the address flips, it also alternates when other bits flip --  Bits 9
 * (every 512 bytes, an X tile scanline) and 10 (every two X tile scanlines)
 * are common to both the 915 and 965-class hardware.
 *
 * The CPU also sometimes XORs in higher bits as well, to improve
 * bandwidth doing strided access like we do so frequently in graphics.  This
 * is called "Channel XOR Randomization" in the MCH documentation.  The result
 * is that the CPU is XORing in either bit 11 or bit 17 to bit 6 of its address
 * decode.
 *
 * All of this bit 6 XORing has an effect on our memory management,
 * as we need to make sure that the 3d driver can correctly address object
 * contents.
 *
 * If we don't have interleaved memory, all tiling is safe and no swizzling is
 * required.
 *
 * When bit 17 is XORed in, we simply refuse to tile at all.  Bit
 * 17 is not just a page offset, so as we page an object out and back in,
 * individual pages in it will have different bit 17 addresses, resulting in
 * each 64 bytes being swapped with its neighbor!
 *
 * Otherwise, if interleaved, we have to tell the 3d driver what the address
 * swizzling it needs to do is, since it's writing with the CPU to the pages
 * (bit 6 and potentially bit 11 XORed in), and the GPU is reading from the
 * pages (bit 6, 9, and 10 XORed in), resulting in a cumulative bit swizzling
 * required by the CPU of XORing in bit 6, 9, 10, and potentially 11, in order
 * to match what the GPU expects.
 */

#define MCHBAR_I915	0x44
#define MCHBAR_I965	0x48
#define	MCHBAR_SIZE	(4*4096)

#define	DEVEN_REG	0x54
#define	DEVEN_MCHBAR_EN	(1 << 28)


/* 
 * Check the MCHBAR on the host bridge is enabled, and if not allocate it.
 * we do not need to actually map it because we access the bar through it's
 * mirror on the IGD, however, if it is disabled or not allocated then
 * the mirror does not work. *sigh*.
 *
 * we return a trinary state:
 * 0 = already enabled, or can not enable
 * 1 = enabled, needs disable
 * 2 = enabled, needs disable and free.
 */
int
inteldrm_setup_mchbar(struct drm_i915_private *dev_priv,
    struct pci_attach_args *bpa)
{
	u_int64_t	mchbar_addr;
	pcireg_t	tmp, low, high = 0;
	u_long		addr;
	int		reg = IS_I965G(dev_priv) ? MCHBAR_I965 : MCHBAR_I915;
	int		ret = 1, enabled = 0;

	if (IS_I915G(dev_priv) || IS_I915GM(dev_priv)) {
		tmp = pci_conf_read(bpa->pa_pc, bpa->pa_tag, DEVEN_REG);
		enabled = !!(tmp & DEVEN_MCHBAR_EN);
	} else {
		tmp = pci_conf_read(bpa->pa_pc, bpa->pa_tag, reg);
		enabled = tmp & 1;
	}

	if (enabled) {
		return (0);
	}

	if (IS_I965G(dev_priv))
		high = pci_conf_read(bpa->pa_pc, bpa->pa_tag, reg + 4);
	low = pci_conf_read(bpa->pa_pc, bpa->pa_tag, reg);
	mchbar_addr = ((u_int64_t)high << 32) | low;

	/*
	 * XXX need to check to see if it's allocated in the pci resources,
	 * right now we just check to see if there's any address there
	 *
	 * if there's no address, then we allocate one.
	 * note that we can't just use pci_mapreg_map here since some intel
	 * BARs are special in that they set bit 0 to show they're enabled,
	 * this is not handled by generic pci code.
	 */
	if (mchbar_addr == 0) {
		addr = (u_long)mchbar_addr;
		if (bpa->pa_memex == NULL || extent_alloc(bpa->pa_memex,
	            MCHBAR_SIZE, MCHBAR_SIZE, 0, 0, 0, &addr)) {
			return (0); /* just say we don't need to disable */
		} else {
			mchbar_addr = addr;
			ret = 2;
			/* We've allocated it, now fill in the BAR again */
			if (IS_I965G(dev_priv))
				pci_conf_write(bpa->pa_pc, bpa->pa_tag,
				    reg + 4, upper_32_bits(mchbar_addr));
			pci_conf_write(bpa->pa_pc, bpa->pa_tag,
			    reg, mchbar_addr & 0xffffffff);
		}
	}
	/* set the enable bit */
	if (IS_I915G(dev_priv) || IS_I915GM(dev_priv)) {
		pci_conf_write(bpa->pa_pc, bpa->pa_tag, DEVEN_REG,
		    tmp | DEVEN_MCHBAR_EN);
	} else {
		tmp = pci_conf_read(bpa->pa_pc, bpa->pa_tag, reg);
		pci_conf_write(bpa->pa_pc, bpa->pa_tag, reg, tmp | 1);
	}

	return (ret);
}

/*
 * we take the trinary returned from inteldrm_setup_mchbar and clean up after
 * it.
 */
void
inteldrm_teardown_mchbar(struct drm_i915_private *dev_priv,
    struct pci_attach_args *bpa, int disable)
{
	u_int64_t	mchbar_addr;
	pcireg_t	tmp, low, high = 0;
	int		reg = IS_I965G(dev_priv) ? MCHBAR_I965 : MCHBAR_I915;

	switch(disable) {
	case 2:
		if (IS_I965G(dev_priv))
			high = pci_conf_read(bpa->pa_pc, bpa->pa_tag, reg + 4);
		low = pci_conf_read(bpa->pa_pc, bpa->pa_tag, reg);
		mchbar_addr = ((u_int64_t)high << 32) | low;
		extent_free(bpa->pa_memex, mchbar_addr, MCHBAR_SIZE, 0);
		/* FALLTHROUGH */
	case 1:
		if (IS_I915G(dev_priv) || IS_I915GM(dev_priv)) {
			tmp = pci_conf_read(bpa->pa_pc, bpa->pa_tag, DEVEN_REG);
			tmp &= ~DEVEN_MCHBAR_EN;
			pci_conf_write(bpa->pa_pc, bpa->pa_tag, DEVEN_REG, tmp);
		} else {
			tmp = pci_conf_read(bpa->pa_pc, bpa->pa_tag, reg);
			tmp &= ~1;
			pci_conf_write(bpa->pa_pc, bpa->pa_tag, reg, tmp);
		}
		break;
	case 0:
	default:
		break;
	};
}

/**
 * Detects bit 6 swizzling of address lookup between IGD access and CPU
 * access through main memory.
 */
void
inteldrm_detect_bit_6_swizzle(drm_i915_private_t *dev_priv,
    struct pci_attach_args *bpa)
{
	uint32_t	swizzle_x = I915_BIT_6_SWIZZLE_UNKNOWN;
	uint32_t	swizzle_y = I915_BIT_6_SWIZZLE_UNKNOWN;
	int		need_disable;

	if (!IS_I9XX(dev_priv)) {
		/* As far as we know, the 865 doesn't have these bit 6
		 * swizzling issues.
		 */
		swizzle_x = I915_BIT_6_SWIZZLE_NONE;
		swizzle_y = I915_BIT_6_SWIZZLE_NONE;
	} else if (IS_MOBILE(dev_priv)) {
		uint32_t dcc;

		/* try to enable MCHBAR, a lot of biosen disable it */
		need_disable = inteldrm_setup_mchbar(dev_priv, bpa);

		/* On 915-945 and GM965, channel interleave by the CPU is
		 * determined by DCC.  The CPU will alternate based on bit 6
		 * in interleaved mode, and the GPU will then also alternate
		 * on bit 6, 9, and 10 for X, but the CPU may also optionally
		 * alternate based on bit 17 (XOR not disabled and XOR
		 * bit == 17).
		 */
		dcc = I915_READ(DCC);
		switch (dcc & DCC_ADDRESSING_MODE_MASK) {
		case DCC_ADDRESSING_MODE_SINGLE_CHANNEL:
		case DCC_ADDRESSING_MODE_DUAL_CHANNEL_ASYMMETRIC:
			swizzle_x = I915_BIT_6_SWIZZLE_NONE;
			swizzle_y = I915_BIT_6_SWIZZLE_NONE;
			break;
		case DCC_ADDRESSING_MODE_DUAL_CHANNEL_INTERLEAVED:
			if (dcc & DCC_CHANNEL_XOR_DISABLE) {
				/* This is the base swizzling by the GPU for
				 * tiled buffers.
				 */
				swizzle_x = I915_BIT_6_SWIZZLE_9_10;
				swizzle_y = I915_BIT_6_SWIZZLE_9;
			} else if ((dcc & DCC_CHANNEL_XOR_BIT_17) == 0) {
				/* Bit 11 swizzling by the CPU in addition. */
				swizzle_x = I915_BIT_6_SWIZZLE_9_10_11;
				swizzle_y = I915_BIT_6_SWIZZLE_9_11;
			} else {
				/* Bit 17 swizzling by the CPU in addition. */
				swizzle_x = I915_BIT_6_SWIZZLE_9_10_17;
				swizzle_y = I915_BIT_6_SWIZZLE_9_17;
			}
			break;
		}
		if (dcc == 0xffffffff) {
			DRM_ERROR("Couldn't read from MCHBAR.  "
				  "Disabling tiling.\n");
			swizzle_x = I915_BIT_6_SWIZZLE_UNKNOWN;
			swizzle_y = I915_BIT_6_SWIZZLE_UNKNOWN;
		}

		inteldrm_teardown_mchbar(dev_priv, bpa, need_disable);	
	} else {
		/* The 965, G33, and newer, have a very flexible memory
		 * configuration. It will enable dual-channel mode
		 * (interleaving) on as much memory as it can, and the GPU
		 * will additionally sometimes enable different bit 6
		 * swizzling for tiled objects from the CPU.
		 *
		 * Here's what I found on G965:
		 *
		 *    slot fill			memory size	swizzling
		 * 0A   0B	1A	1B	1-ch	2-ch
		 * 512	0	0	0	512	0	O
		 * 512	0	512	0	16	1008	X
		 * 512	0	0	512	16	1008	X
		 * 0	512	0	512	16	1008	X
		 * 1024	1024	1024	0	2048	1024	O
		 *
		 * We could probably detect this based on either the DRB
		 * matching, which was the case for the swizzling required in
		 * the table above, or from the 1-ch value being less than
		 * the minimum size of a rank.
		 */
		if (I915_READ16(C0DRB3) != I915_READ16(C1DRB3)) {
			swizzle_x = I915_BIT_6_SWIZZLE_NONE;
			swizzle_y = I915_BIT_6_SWIZZLE_NONE;
		} else {
			swizzle_x = I915_BIT_6_SWIZZLE_9_10;
			swizzle_y = I915_BIT_6_SWIZZLE_9;
		}
	}

	dev_priv->mm.bit_6_swizzle_x = swizzle_x;
	dev_priv->mm.bit_6_swizzle_y = swizzle_y;
}

int
inteldrm_swizzle_page(struct vm_page *pg)
{
	vaddr_t	 va;
	int	 i;
	u_int8_t temp[64], *vaddr;

#if defined (__HAVE_PMAP_DIRECT)
	va = pmap_map_direct(pg);
#else
	va = uvm_km_valloc(kernel_map, PAGE_SIZE);
	if (va == 0)
		return (ENOMEM);
	pmap_kenter_pa(va, VM_PAGE_TO_PHYS(pg), UVM_PROT_RW);
	pmap_update(pmap_kernel());
#endif
	vaddr = (u_int8_t *)va;

	for (i = 0; i < PAGE_SIZE; i += 128) {
		memcpy(temp, &vaddr[i], 64);
		memcpy(&vaddr[i], &vaddr[i + 64], 64);
		memcpy(&vaddr[i + 64], temp, 64);
	}

#if defined (__HAVE_PMAP_DIRECT)
	pmap_unmap_direct(va);
#else
	pmap_kremove(va, va + PAGE_SIZE);
	pmap_update(pmap_kernel());
	uvm_km_free(kernel_map, va, PAGE_SIZE);
#endif
	return (0);
}

void
i915_gem_bit_17_swizzle(struct drm_obj *obj)
{
	struct drm_device	*dev = obj->dev;
	struct drm_i915_private	*dev_priv = dev->dev_private;
	struct inteldrm_obj	*obj_priv = (struct inteldrm_obj *)obj;
	struct vm_page		*pg;
	bus_dma_segment_t	*segp;
	int			 page_count = obj->size >> PAGE_SHIFT;
	int			 i, n, ret;

	if (dev_priv->mm.bit_6_swizzle_x != I915_BIT_6_SWIZZLE_9_10_17 ||
	    obj_priv->bit_17 == NULL)
		return;

	while (i < page_count) {
		/* compare bit 17 with previous one (in case we swapped).
		 * if they don't match we'll have to swizzle the page
		 */
		if ((((segp->ds_addr + n) >> 17) & 0x1) !=
		    test_bit(i, obj_priv->bit_17)) {
			/* XXX move this to somewhere where we already have pg */
			pg = PHYS_TO_VM_PAGE(segp->ds_addr + n * PAGE_SIZE);
			KASSERT(pg != NULL);
			ret = inteldrm_swizzle_page(pg);
			if (ret)
				return;
			atomic_clearbits_int(&pg->pg_flags, PG_CLEAN);
		}

		if (++n * PAGE_SIZE > segp->ds_len) {
			n = 0;
			segp++;
		}
	}

}

void 
i915_gem_save_bit_17_swizzle(struct drm_obj *obj)
{
	struct drm_device	*dev = obj->dev;
	struct drm_i915_private	*dev_priv = dev->dev_private;
	struct inteldrm_obj	*obj_priv = (struct inteldrm_obj *)obj;
	bus_dma_segment_t	*segp;
	int			 page_count = obj->size >> PAGE_SHIFT, i, n;

	if (dev_priv->mm.bit_6_swizzle_x != I915_BIT_6_SWIZZLE_9_10_17)
		return;

	if (obj_priv->bit_17 == NULL) {
		/* round up number of pages to a multiple of 32 so we know what
		 * size to make the bitmask. XXX this is wasteful with malloc
		 * and a better way should be done
		 */
		size_t nb17 = ((page_count + 31) & ~31)/32;
		obj_priv->bit_17 = drm_alloc(nb17 * sizeof(u_int32_t));
		if (obj_priv-> bit_17 == NULL) {
			return;
		}

	}

	segp = &obj_priv->dma_segs[0];
	n = 0;
	while (i < page_count) {
		if ((segp->ds_addr + (n * PAGE_SIZE)) & (1 << 17))
			set_bit(i, obj_priv->bit_17);
		else
			clear_bit(i, obj_priv->bit_17);

		if (++n * PAGE_SIZE > segp->ds_len) {
			n = 0;
			segp++;
		}
	}
}

bus_size_t
i915_get_fence_size(struct drm_i915_private *dev_priv, bus_size_t size)
{
	bus_size_t	i, start;

	if (IS_I965G(dev_priv)) {
		/* 965 can have fences anywhere, so align to gpu-page size */
		return ((size + (4096 - 1)) & ~(4096 - 1));
	} else {
		/*
		 * Align the size to a power of two greater than the smallest
		 * fence size.
		 */
		if (IS_I9XX(dev_priv))
			start = 1024 * 1024;
		else
			start = 512 * 1024;

		for (i = start; i < size; i <<= 1)
			;

		return (i);
	}
}

int
i915_tiling_ok(struct drm_device *dev, int stride, int size, int tiling_mode)
{
	struct drm_i915_private	*dev_priv = dev->dev_private;
	int			 tile_width;

	/* Linear is always ok */
	if (tiling_mode == I915_TILING_NONE)
		return (1);

	if (!IS_I9XX(dev_priv) || (tiling_mode == I915_TILING_Y &&
	    HAS_128_BYTE_Y_TILING(dev_priv)))
		tile_width = 128;
	else
		tile_width = 512;

	/* Check stride and size constraints */
	if (IS_I965G(dev_priv)) {
		/* fence reg has end address, so size is ok */
		if (stride / 128 > I965_FENCE_MAX_PITCH_VAL)
			return (0);
	} else if (IS_I9XX(dev_priv)) {
		u_int32_t pitch_val = ffs(stride / tile_width) - 1;
		/*
		 * XXX: for Y tiling, max pitch is actually 6 (8k) instead of 4
		 * (2k) on the 945.
		 */
		if (pitch_val > I915_FENCE_MAX_PITCH_VAL ||
		    size > (I830_FENCE_MAX_SIZE_VAL << 20))
			return (0);
	} else {
		u_int32_t pitch_val = ffs(stride / tile_width) - 1;

		if (pitch_val > I830_FENCE_MAX_PITCH_VAL ||
		    size > (I830_FENCE_MAX_SIZE_VAL << 19))
			return (0);
	}

	/* 965+ just needs multiples of the tile width */
	if (IS_I965G(dev_priv))
		return ((stride & (tile_width - 1)) == 0);


	/* Pre-965 needs power-of-two */
	if (stride < tile_width || stride & (stride - 1) ||
	    i915_get_fence_size(dev_priv, size) != size)
		return (0);
	return (1);
}

int
i915_gem_object_fence_offset_ok(struct drm_obj *obj, int tiling_mode)
{
	struct drm_device	*dev = obj->dev;
	struct drm_i915_private	*dev_priv = dev->dev_private;
	struct inteldrm_obj	*obj_priv = (struct inteldrm_obj *)obj;

	if (obj_priv->dmamap == NULL || tiling_mode == I915_TILING_NONE)
		return (1);

	if (!IS_I965G(dev_priv)) {
		if (obj_priv->gtt_offset & (obj->size -1))
			return (0);
		if (IS_I9XX(dev_priv)) {
			if (obj_priv->gtt_offset & ~I915_FENCE_START_MASK)
				return (0);
		} else {
			if (obj_priv->gtt_offset & ~I830_FENCE_START_MASK)
				return (0);
		}
	}
	return (1);
}
/**
 * Sets the tiling mode of an object, returning the required swizzling of
 * bit 6 of addresses in the object.
 */
int
i915_gem_set_tiling(struct drm_device *dev, void *data,
		   struct drm_file *file_priv)
{
	struct drm_i915_gem_set_tiling	*args = data;
	drm_i915_private_t		*dev_priv = dev->dev_private;
	struct drm_obj			*obj;
	struct inteldrm_obj		*obj_priv;
	int				 ret = 0;

	obj = drm_gem_object_lookup(dev, file_priv, args->handle);
	if (obj == NULL)
		return (EBADF);
	obj_priv = (struct inteldrm_obj *)obj;

	if (i915_tiling_ok(dev, args->stride, obj->size,
	    args->tiling_mode) == 0) {
		ret = EINVAL;
		DRM_LOCK(); /* for unref */
		goto out;
	}

	DRM_LOCK();
	if (args->tiling_mode == I915_TILING_NONE) {
		args->swizzle_mode = I915_BIT_6_SWIZZLE_NONE;
		args->stride = 0;
	} else {
		if (args->tiling_mode == I915_TILING_X)
			args->swizzle_mode = dev_priv->mm.bit_6_swizzle_x;
		else
			args->swizzle_mode = dev_priv->mm.bit_6_swizzle_y;
		/* If we can't handle the swizzling, make it untiled. */
		if (args->swizzle_mode == I915_BIT_6_SWIZZLE_UNKNOWN) {
			args->tiling_mode = I915_TILING_NONE;
			args->swizzle_mode = I915_BIT_6_SWIZZLE_NONE;
			args->stride = 0;
		}
	}

	if (args->tiling_mode != obj_priv->tiling_mode ||
	    args->stride != obj_priv->stride) {
		/*
		 * We need to rebind the object if its current allocation no
		 * longer meets the alignment restrictions for its new tiling
		 * mode. Otherwise we can leave it alone, but must clear any
		 * fence register.
		 */
		if (i915_gem_object_fence_offset_ok(obj, args->tiling_mode)) {
			if (obj_priv->pin_count)
				ret = EINVAL;
			else
				ret = i915_gem_object_unbind(obj, 1);
		} else if (obj_priv->fence_reg != I915_FENCE_REG_NONE) {
			ret = i915_gem_object_put_fence_reg(obj, 1);
		} else {
			inteldrm_wipe_mappings(obj);
		}
		if (ret != 0) {
			args->tiling_mode = obj_priv->tiling_mode;
			args->stride = obj_priv->stride;
			goto out;
		}
		obj_priv->tiling_mode = args->tiling_mode;
		obj_priv->stride = args->stride;
	}
	 
out:
	drm_gem_object_unreference(obj);
	DRM_UNLOCK();

	return (ret);
}

/**
 * Returns the current tiling mode and required bit 6 swizzling for the object.
 */
int
i915_gem_get_tiling(struct drm_device *dev, void *data,
		   struct drm_file *file_priv)
{
	struct drm_i915_gem_get_tiling	*args = data;
	drm_i915_private_t		*dev_priv = dev->dev_private;
	struct drm_obj			*obj;
	struct inteldrm_obj		*obj_priv;

	obj = drm_gem_object_lookup(dev, file_priv, args->handle);
	if (obj == NULL)
		return (EBADF);
	obj_priv = (struct inteldrm_obj *)obj;

	DRM_LOCK();

	args->tiling_mode = obj_priv->tiling_mode;
	switch (obj_priv->tiling_mode) {
	case I915_TILING_X:
		args->swizzle_mode = dev_priv->mm.bit_6_swizzle_x;
		break;
	case I915_TILING_Y:
		args->swizzle_mode = dev_priv->mm.bit_6_swizzle_y;
		break;
	case I915_TILING_NONE:
		args->swizzle_mode = I915_BIT_6_SWIZZLE_NONE;
		break;
	default:
		DRM_ERROR("unknown tiling mode\n");
	}

	drm_gem_object_unreference(obj);
	DRM_UNLOCK();

	return 0;
}

#endif /* INTELDRM_GEM */

/**
 * inteldrm_pipe_enabled - check if a pipe is enabled
 * @dev: DRM device
 * @pipe: pipe to check
 *
 * Reading certain registers when the pipe is disabled can hang the chip.
 * Use this routine to make sure the PLL is running and the pipe is active
 * before reading such registers if unsure.
 */
int
inteldrm_pipe_enabled(struct drm_i915_private *dev_priv, int pipe)
{
	bus_size_t	pipeconf = pipe ? PIPEBCONF : PIPEACONF;

	return ((I915_READ(pipeconf) & PIPEACONF_ENABLE) == PIPEACONF_ENABLE);
}

/*
 * Register save/restore for various instances
 */
void
i915_save_palette(struct drm_i915_private *dev_priv, enum pipe pipe)
{
	u_int32_t	*array;
	bus_size_t	 reg = (pipe == PIPE_A ? PALETTE_A : PALETTE_B);
	int		 i;

	if (!inteldrm_pipe_enabled(dev_priv, pipe))
		return;

	if (pipe == PIPE_A)
		array = dev_priv->save_palette_a;
	else
		array = dev_priv->save_palette_b;

	for (i = 0; i < 256; i++)
		array[i] = I915_READ(reg + (i << 2));
}

void
i915_restore_palette(struct drm_i915_private *dev_priv, enum pipe pipe)
{
	u_int32_t	*array;
	bus_size_t	 reg = (pipe == PIPE_A ? PALETTE_A : PALETTE_B);
	int		 i;

	if (!inteldrm_pipe_enabled(dev_priv, pipe))
		return;

	if (pipe == PIPE_A)
		array = dev_priv->save_palette_a;
	else
		array = dev_priv->save_palette_b;

	for(i = 0; i < 256; i++)
		I915_WRITE(reg + (i << 2), array[i]);
}

u_int8_t
i915_read_ar(struct drm_i915_private *dev_priv, u_int16_t st01,
    u_int8_t reg, u_int16_t palette_enable)
{
	I915_READ8(st01);
	I915_WRITE8(VGA_AR_INDEX, palette_enable | reg);
	return I915_READ8(VGA_AR_DATA_READ);
}

void
i915_write_ar(struct drm_i915_private *dev_priv, u_int16_t st01, u_int8_t reg,
    u_int8_t val, u_int16_t palette_enable)
{
	I915_READ8(st01);
	I915_WRITE8(VGA_AR_INDEX, palette_enable | reg);
	I915_WRITE8(VGA_AR_DATA_WRITE, val);
}

u_int8_t
i915_read_indexed(struct drm_i915_private *dev_priv, u_int16_t index_port,
    u_int16_t data_port, u_int8_t reg)
{
	I915_WRITE8(index_port, reg);
	return I915_READ8(data_port);
}

void
i915_write_indexed(struct drm_i915_private *dev_priv, u_int16_t index_port,
    u_int16_t data_port, u_int8_t reg, u_int8_t val)
{
	I915_WRITE8(index_port, reg);
	I915_WRITE8(data_port, val);
}

void
i915_save_vga(struct drm_i915_private *dev_priv)
{
	int i;
	u16 cr_index, cr_data, st01;

	/* VGA color palette registers */
	dev_priv->saveDACMASK = I915_READ8(VGA_DACMASK);

	/* MSR bits */
	dev_priv->saveMSR = I915_READ8(VGA_MSR_READ);
	if (dev_priv->saveMSR & VGA_MSR_CGA_MODE) {
		cr_index = VGA_CR_INDEX_CGA;
		cr_data = VGA_CR_DATA_CGA;
		st01 = VGA_ST01_CGA;
	} else {
		cr_index = VGA_CR_INDEX_MDA;
		cr_data = VGA_CR_DATA_MDA;
		st01 = VGA_ST01_MDA;
	}

	/* CRT controller regs */
	i915_write_indexed(dev_priv, cr_index, cr_data, 0x11,
	    i915_read_indexed(dev_priv, cr_index, cr_data, 0x11) & (~0x80));
	for (i = 0; i <= 0x24; i++)
		dev_priv->saveCR[i] = i915_read_indexed(dev_priv,
		    cr_index, cr_data, i);
	/* Make sure we don't turn off CR group 0 writes */
	dev_priv->saveCR[0x11] &= ~0x80;

	/* Attribute controller registers */
	I915_READ8(st01);
	dev_priv->saveAR_INDEX = I915_READ8(VGA_AR_INDEX);
	for (i = 0; i <= 0x14; i++)
		dev_priv->saveAR[i] = i915_read_ar(dev_priv, st01, i, 0);
	I915_READ8(st01);
	I915_WRITE8(VGA_AR_INDEX, dev_priv->saveAR_INDEX);
	I915_READ8(st01);

	/* Graphics controller registers */
	for (i = 0; i < 9; i++)
		dev_priv->saveGR[i] = i915_read_indexed(dev_priv,
		    VGA_GR_INDEX, VGA_GR_DATA, i);

	dev_priv->saveGR[0x10] = i915_read_indexed(dev_priv,
	    VGA_GR_INDEX, VGA_GR_DATA, 0x10);
	dev_priv->saveGR[0x11] = i915_read_indexed(dev_priv,
	    VGA_GR_INDEX, VGA_GR_DATA, 0x11);
	dev_priv->saveGR[0x18] = i915_read_indexed(dev_priv,
	    VGA_GR_INDEX, VGA_GR_DATA, 0x18);

	/* Sequencer registers */
	for (i = 0; i < 8; i++)
		dev_priv->saveSR[i] = i915_read_indexed(dev_priv,
		    VGA_SR_INDEX, VGA_SR_DATA, i);
}

void
i915_restore_vga(struct drm_i915_private *dev_priv)
{
	u_int16_t	cr_index, cr_data, st01;
	int		i;

	/* MSR bits */
	I915_WRITE8(VGA_MSR_WRITE, dev_priv->saveMSR);
	if (dev_priv->saveMSR & VGA_MSR_CGA_MODE) {
		cr_index = VGA_CR_INDEX_CGA;
		cr_data = VGA_CR_DATA_CGA;
		st01 = VGA_ST01_CGA;
	} else {
		cr_index = VGA_CR_INDEX_MDA;
		cr_data = VGA_CR_DATA_MDA;
		st01 = VGA_ST01_MDA;
	}

	/* Sequencer registers, don't write SR07 */
	for (i = 0; i < 7; i++)
		i915_write_indexed(dev_priv, VGA_SR_INDEX, VGA_SR_DATA, i,
		    dev_priv->saveSR[i]);

	/* CRT controller regs */
	/* Enable CR group 0 writes */
	i915_write_indexed(dev_priv, cr_index, cr_data, 0x11,
	    dev_priv->saveCR[0x11]);
	for (i = 0; i <= 0x24; i++)
		i915_write_indexed(dev_priv, cr_index, cr_data,
		    i, dev_priv->saveCR[i]);

	/* Graphics controller regs */
	for (i = 0; i < 9; i++)
		i915_write_indexed(dev_priv, VGA_GR_INDEX, VGA_GR_DATA, i,
				   dev_priv->saveGR[i]);

	i915_write_indexed(dev_priv, VGA_GR_INDEX, VGA_GR_DATA, 0x10,
			   dev_priv->saveGR[0x10]);
	i915_write_indexed(dev_priv, VGA_GR_INDEX, VGA_GR_DATA, 0x11,
			   dev_priv->saveGR[0x11]);
	i915_write_indexed(dev_priv, VGA_GR_INDEX, VGA_GR_DATA, 0x18,
			   dev_priv->saveGR[0x18]);

	/* Attribute controller registers */
	I915_READ8(st01); /* switch back to index mode */
	for (i = 0; i <= 0x14; i++)
		i915_write_ar(dev_priv, st01, i, dev_priv->saveAR[i], 0);
	I915_READ8(st01); /* switch back to index mode */
	I915_WRITE8(VGA_AR_INDEX, dev_priv->saveAR_INDEX | 0x20);
	I915_READ8(st01);

	/* VGA color palette registers */
	I915_WRITE8(VGA_DACMASK, dev_priv->saveDACMASK);
}

void
i915_save_modeset_reg(struct drm_i915_private *dev_priv)
{
	/* Pipe & plane A info */
	dev_priv->savePIPEACONF = I915_READ(PIPEACONF);
	dev_priv->savePIPEASRC = I915_READ(PIPEASRC);
	dev_priv->saveFPA0 = I915_READ(FPA0);
	dev_priv->saveFPA1 = I915_READ(FPA1);
	dev_priv->saveDPLL_A = I915_READ(DPLL_A);
	if (IS_I965G(dev_priv))
		dev_priv->saveDPLL_A_MD = I915_READ(DPLL_A_MD);
	dev_priv->saveHTOTAL_A = I915_READ(HTOTAL_A);
	dev_priv->saveHBLANK_A = I915_READ(HBLANK_A);
	dev_priv->saveHSYNC_A = I915_READ(HSYNC_A);
	dev_priv->saveVTOTAL_A = I915_READ(VTOTAL_A);
	dev_priv->saveVBLANK_A = I915_READ(VBLANK_A);
	dev_priv->saveVSYNC_A = I915_READ(VSYNC_A);
	dev_priv->saveBCLRPAT_A = I915_READ(BCLRPAT_A);

	dev_priv->saveDSPACNTR = I915_READ(DSPACNTR);
	dev_priv->saveDSPASTRIDE = I915_READ(DSPASTRIDE);
	dev_priv->saveDSPASIZE = I915_READ(DSPASIZE);
	dev_priv->saveDSPAPOS = I915_READ(DSPAPOS);
	dev_priv->saveDSPAADDR = I915_READ(DSPAADDR);
	if (IS_I965G(dev_priv)) {
		dev_priv->saveDSPASURF = I915_READ(DSPASURF);
		dev_priv->saveDSPATILEOFF = I915_READ(DSPATILEOFF);
	}
	i915_save_palette(dev_priv, PIPE_A);
	dev_priv->savePIPEASTAT = I915_READ(PIPEASTAT);

	/* Pipe & plane B info */
	dev_priv->savePIPEBCONF = I915_READ(PIPEBCONF);
	dev_priv->savePIPEBSRC = I915_READ(PIPEBSRC);
	dev_priv->saveFPB0 = I915_READ(FPB0);
	dev_priv->saveFPB1 = I915_READ(FPB1);
	dev_priv->saveDPLL_B = I915_READ(DPLL_B);
	if (IS_I965G(dev_priv))
		dev_priv->saveDPLL_B_MD = I915_READ(DPLL_B_MD);
	dev_priv->saveHTOTAL_B = I915_READ(HTOTAL_B);
	dev_priv->saveHBLANK_B = I915_READ(HBLANK_B);
	dev_priv->saveHSYNC_B = I915_READ(HSYNC_B);
	dev_priv->saveVTOTAL_B = I915_READ(VTOTAL_B);
	dev_priv->saveVBLANK_B = I915_READ(VBLANK_B);
	dev_priv->saveVSYNC_B = I915_READ(VSYNC_B);
	dev_priv->saveBCLRPAT_A = I915_READ(BCLRPAT_A);

	dev_priv->saveDSPBCNTR = I915_READ(DSPBCNTR);
	dev_priv->saveDSPBSTRIDE = I915_READ(DSPBSTRIDE);
	dev_priv->saveDSPBSIZE = I915_READ(DSPBSIZE);
	dev_priv->saveDSPBPOS = I915_READ(DSPBPOS);
	dev_priv->saveDSPBADDR = I915_READ(DSPBADDR);
	if (IS_I965GM(dev_priv) || IS_GM45(dev_priv)) {
		dev_priv->saveDSPBSURF = I915_READ(DSPBSURF);
		dev_priv->saveDSPBTILEOFF = I915_READ(DSPBTILEOFF);
	}
	i915_save_palette(dev_priv, PIPE_B);
	dev_priv->savePIPEBSTAT = I915_READ(PIPEBSTAT);
}

void
i915_restore_modeset_reg(struct drm_i915_private *dev_priv)
{
	/* Pipe & plane A info */
	/* Prime the clock */
	if (dev_priv->saveDPLL_A & DPLL_VCO_ENABLE) {
		I915_WRITE(DPLL_A, dev_priv->saveDPLL_A &
			   ~DPLL_VCO_ENABLE);
		DRM_UDELAY(150);
	}
	I915_WRITE(FPA0, dev_priv->saveFPA0);
	I915_WRITE(FPA1, dev_priv->saveFPA1);
	/* Actually enable it */
	I915_WRITE(DPLL_A, dev_priv->saveDPLL_A);
	DRM_UDELAY(150);
	if (IS_I965G(dev_priv))
		I915_WRITE(DPLL_A_MD, dev_priv->saveDPLL_A_MD);
	DRM_UDELAY(150);

	/* Restore mode */
	I915_WRITE(HTOTAL_A, dev_priv->saveHTOTAL_A);
	I915_WRITE(HBLANK_A, dev_priv->saveHBLANK_A);
	I915_WRITE(HSYNC_A, dev_priv->saveHSYNC_A);
	I915_WRITE(VTOTAL_A, dev_priv->saveVTOTAL_A);
	I915_WRITE(VBLANK_A, dev_priv->saveVBLANK_A);
	I915_WRITE(VSYNC_A, dev_priv->saveVSYNC_A);
	I915_WRITE(BCLRPAT_A, dev_priv->saveBCLRPAT_A);

	/* Restore plane info */
	I915_WRITE(DSPASIZE, dev_priv->saveDSPASIZE);
	I915_WRITE(DSPAPOS, dev_priv->saveDSPAPOS);
	I915_WRITE(PIPEASRC, dev_priv->savePIPEASRC);
	I915_WRITE(DSPAADDR, dev_priv->saveDSPAADDR);
	I915_WRITE(DSPASTRIDE, dev_priv->saveDSPASTRIDE);
	if (IS_I965G(dev_priv)) {
		I915_WRITE(DSPASURF, dev_priv->saveDSPASURF);
		I915_WRITE(DSPATILEOFF, dev_priv->saveDSPATILEOFF);
	}

	I915_WRITE(PIPEACONF, dev_priv->savePIPEACONF);

	i915_restore_palette(dev_priv, PIPE_A);
	/* Enable the plane */
	I915_WRITE(DSPACNTR, dev_priv->saveDSPACNTR);
	I915_WRITE(DSPAADDR, I915_READ(DSPAADDR));

	/* Pipe & plane B info */
	if (dev_priv->saveDPLL_B & DPLL_VCO_ENABLE) {
		I915_WRITE(DPLL_B, dev_priv->saveDPLL_B &
			   ~DPLL_VCO_ENABLE);
		DRM_UDELAY(150);
	}
	I915_WRITE(FPB0, dev_priv->saveFPB0);
	I915_WRITE(FPB1, dev_priv->saveFPB1);
	/* Actually enable it */
	I915_WRITE(DPLL_B, dev_priv->saveDPLL_B);
	DRM_UDELAY(150);
	if (IS_I965G(dev_priv))
		I915_WRITE(DPLL_B_MD, dev_priv->saveDPLL_B_MD);
	DRM_UDELAY(150);

	/* Restore mode */
	I915_WRITE(HTOTAL_B, dev_priv->saveHTOTAL_B);
	I915_WRITE(HBLANK_B, dev_priv->saveHBLANK_B);
	I915_WRITE(HSYNC_B, dev_priv->saveHSYNC_B);
	I915_WRITE(VTOTAL_B, dev_priv->saveVTOTAL_B);
	I915_WRITE(VBLANK_B, dev_priv->saveVBLANK_B);
	I915_WRITE(VSYNC_B, dev_priv->saveVSYNC_B);
	I915_WRITE(BCLRPAT_B, dev_priv->saveBCLRPAT_B);

	/* Restore plane info */
	I915_WRITE(DSPBSIZE, dev_priv->saveDSPBSIZE);
	I915_WRITE(DSPBPOS, dev_priv->saveDSPBPOS);
	I915_WRITE(PIPEBSRC, dev_priv->savePIPEBSRC);
	I915_WRITE(DSPBADDR, dev_priv->saveDSPBADDR);
	I915_WRITE(DSPBSTRIDE, dev_priv->saveDSPBSTRIDE);
	if (IS_I965G(dev_priv)) {
		I915_WRITE(DSPBSURF, dev_priv->saveDSPBSURF);
		I915_WRITE(DSPBTILEOFF, dev_priv->saveDSPBTILEOFF);
	}

	I915_WRITE(PIPEBCONF, dev_priv->savePIPEBCONF);

	i915_restore_palette(dev_priv, PIPE_B);
	/* Enable the plane */
	I915_WRITE(DSPBCNTR, dev_priv->saveDSPBCNTR);
	I915_WRITE(DSPBADDR, I915_READ(DSPBADDR));
}

int
inteldrm_save_display(struct drm_i915_private *dev_priv)
{
	/* Display arbitration control */
	dev_priv->saveDSPARB = I915_READ(DSPARB);

	/* This is only meaningful in non-KMS mode */
	/* Don't save them in KMS mode */
	i915_save_modeset_reg(dev_priv);
	/* Cursor state */
	dev_priv->saveCURACNTR = I915_READ(CURACNTR);
	dev_priv->saveCURAPOS = I915_READ(CURAPOS);
	dev_priv->saveCURABASE = I915_READ(CURABASE);
	dev_priv->saveCURBCNTR = I915_READ(CURBCNTR);
	dev_priv->saveCURBPOS = I915_READ(CURBPOS);
	dev_priv->saveCURBBASE = I915_READ(CURBBASE);
	if (!IS_I9XX(dev_priv))
		dev_priv->saveCURSIZE = I915_READ(CURSIZE);

	/* CRT state */
	dev_priv->saveADPA = I915_READ(ADPA);

	/* LVDS state */
	dev_priv->savePP_CONTROL = I915_READ(PP_CONTROL);
	dev_priv->savePFIT_PGM_RATIOS = I915_READ(PFIT_PGM_RATIOS);
	dev_priv->saveBLC_PWM_CTL = I915_READ(BLC_PWM_CTL);
	if (IS_I965G(dev_priv))
		dev_priv->saveBLC_PWM_CTL2 = I915_READ(BLC_PWM_CTL2);
	if (IS_MOBILE(dev_priv) && !IS_I830(dev_priv))
		dev_priv->saveLVDS = I915_READ(LVDS);
	if (!IS_I830(dev_priv) && !IS_845G(dev_priv))
		dev_priv->savePFIT_CONTROL = I915_READ(PFIT_CONTROL);
	dev_priv->savePP_ON_DELAYS = I915_READ(PP_ON_DELAYS);
	dev_priv->savePP_OFF_DELAYS = I915_READ(PP_OFF_DELAYS);
	dev_priv->savePP_DIVISOR = I915_READ(PP_DIVISOR);

	/* FIXME: save TV & SDVO state */

	/* FBC state */
	dev_priv->saveFBC_CFB_BASE = I915_READ(FBC_CFB_BASE);
	dev_priv->saveFBC_LL_BASE = I915_READ(FBC_LL_BASE);
	dev_priv->saveFBC_CONTROL2 = I915_READ(FBC_CONTROL2);
	dev_priv->saveFBC_CONTROL = I915_READ(FBC_CONTROL);

	/* VGA state */
	dev_priv->saveVGA0 = I915_READ(VGA0);
	dev_priv->saveVGA1 = I915_READ(VGA1);
	dev_priv->saveVGA_PD = I915_READ(VGA_PD);
	dev_priv->saveVGACNTRL = I915_READ(VGACNTRL);

	i915_save_vga(dev_priv);

	return 0;
}

int
inteldrm_restore_display(struct drm_i915_private *dev_priv)
{
	/* Display arbitration */
	I915_WRITE(DSPARB, dev_priv->saveDSPARB);

	/* This is only meaningful in non-KMS mode */
	/* Don't restore them in KMS mode */
	i915_restore_modeset_reg(dev_priv);
	/* Cursor state */
	I915_WRITE(CURAPOS, dev_priv->saveCURAPOS);
	I915_WRITE(CURACNTR, dev_priv->saveCURACNTR);
	I915_WRITE(CURABASE, dev_priv->saveCURABASE);
	I915_WRITE(CURBPOS, dev_priv->saveCURBPOS);
	I915_WRITE(CURBCNTR, dev_priv->saveCURBCNTR);
	I915_WRITE(CURBBASE, dev_priv->saveCURBBASE);
	if (!IS_I9XX(dev_priv))
		I915_WRITE(CURSIZE, dev_priv->saveCURSIZE);

	/* CRT state */
	I915_WRITE(ADPA, dev_priv->saveADPA);

	/* LVDS state */
	if (IS_I965G(dev_priv))
		I915_WRITE(BLC_PWM_CTL2, dev_priv->saveBLC_PWM_CTL2);
	if (IS_MOBILE(dev_priv) && !IS_I830(dev_priv))
		I915_WRITE(LVDS, dev_priv->saveLVDS);
	if (!IS_I830(dev_priv) && !IS_845G(dev_priv))
		I915_WRITE(PFIT_CONTROL, dev_priv->savePFIT_CONTROL);

	I915_WRITE(PFIT_PGM_RATIOS, dev_priv->savePFIT_PGM_RATIOS);
	I915_WRITE(BLC_PWM_CTL, dev_priv->saveBLC_PWM_CTL);
	I915_WRITE(PP_ON_DELAYS, dev_priv->savePP_ON_DELAYS);
	I915_WRITE(PP_OFF_DELAYS, dev_priv->savePP_OFF_DELAYS);
	I915_WRITE(PP_DIVISOR, dev_priv->savePP_DIVISOR);
	I915_WRITE(PP_CONTROL, dev_priv->savePP_CONTROL);

	/* FIXME: restore TV & SDVO state */

	/* FBC info */
	I915_WRITE(FBC_CFB_BASE, dev_priv->saveFBC_CFB_BASE);
	I915_WRITE(FBC_LL_BASE, dev_priv->saveFBC_LL_BASE);
	I915_WRITE(FBC_CONTROL2, dev_priv->saveFBC_CONTROL2);
	I915_WRITE(FBC_CONTROL, dev_priv->saveFBC_CONTROL);

	/* VGA state */
	I915_WRITE(VGACNTRL, dev_priv->saveVGACNTRL);
	I915_WRITE(VGA0, dev_priv->saveVGA0);
	I915_WRITE(VGA1, dev_priv->saveVGA1);
	I915_WRITE(VGA_PD, dev_priv->saveVGA_PD);
	DRM_UDELAY(150);

	i915_restore_vga(dev_priv);

	return 0;
}

int
inteldrm_save_state(struct drm_i915_private *dev_priv)
{
	int i;

	dev_priv->saveLBB = pci_conf_read(dev_priv->pc, dev_priv->tag, LBB);

	/* Render Standby */
	if (IS_I965G(dev_priv) && IS_MOBILE(dev_priv))
		dev_priv->saveRENDERSTANDBY = I915_READ(MCHBAR_RENDER_STANDBY);

	/* Hardware status page */
	dev_priv->saveHWS = I915_READ(HWS_PGA);

	/* Display arbitration control */
	dev_priv->saveDSPARB = I915_READ(DSPARB);

	/* This is only meaningful in non-KMS mode */
	/* Don't save them in KMS mode */
	i915_save_modeset_reg(dev_priv);
	/* Cursor state */
	dev_priv->saveCURACNTR = I915_READ(CURACNTR);
	dev_priv->saveCURAPOS = I915_READ(CURAPOS);
	dev_priv->saveCURABASE = I915_READ(CURABASE);
	dev_priv->saveCURBCNTR = I915_READ(CURBCNTR);
	dev_priv->saveCURBPOS = I915_READ(CURBPOS);
	dev_priv->saveCURBBASE = I915_READ(CURBBASE);
	if (!IS_I9XX(dev_priv))
		dev_priv->saveCURSIZE = I915_READ(CURSIZE);

	/* CRT state */
	dev_priv->saveADPA = I915_READ(ADPA);

	/* LVDS state */
	dev_priv->savePP_CONTROL = I915_READ(PP_CONTROL);
	dev_priv->savePFIT_PGM_RATIOS = I915_READ(PFIT_PGM_RATIOS);
	dev_priv->saveBLC_PWM_CTL = I915_READ(BLC_PWM_CTL);
	if (IS_I965G(dev_priv))
		dev_priv->saveBLC_PWM_CTL2 = I915_READ(BLC_PWM_CTL2);
	if (IS_MOBILE(dev_priv) && !IS_I830(dev_priv))
		dev_priv->saveLVDS = I915_READ(LVDS);
	if (!IS_I830(dev_priv) && !IS_845G(dev_priv))
		dev_priv->savePFIT_CONTROL = I915_READ(PFIT_CONTROL);
	dev_priv->savePP_ON_DELAYS = I915_READ(PP_ON_DELAYS);
	dev_priv->savePP_OFF_DELAYS = I915_READ(PP_OFF_DELAYS);
	dev_priv->savePP_DIVISOR = I915_READ(PP_DIVISOR);

	/* XXX: displayport */
	/* FIXME: save TV & SDVO state */

	/* FBC state */
	dev_priv->saveFBC_CFB_BASE = I915_READ(FBC_CFB_BASE);
	dev_priv->saveFBC_LL_BASE = I915_READ(FBC_LL_BASE);
	dev_priv->saveFBC_CONTROL2 = I915_READ(FBC_CONTROL2);
	dev_priv->saveFBC_CONTROL = I915_READ(FBC_CONTROL);

	/* Interrupt state */
	dev_priv->saveIIR = I915_READ(IIR);
	dev_priv->saveIER = I915_READ(IER);
	dev_priv->saveIMR = I915_READ(IMR);

	/* VGA state */
	dev_priv->saveVGA0 = I915_READ(VGA0);
	dev_priv->saveVGA1 = I915_READ(VGA1);
	dev_priv->saveVGA_PD = I915_READ(VGA_PD);
	dev_priv->saveVGACNTRL = I915_READ(VGACNTRL);

	/* Clock gating state */
	dev_priv->saveD_STATE = I915_READ(D_STATE);
	dev_priv->saveDSPCLK_GATE_D = I915_READ(DSPCLK_GATE_D);

	/* Cache mode state */
	dev_priv->saveCACHE_MODE_0 = I915_READ(CACHE_MODE_0);

	/* Memory Arbitration state */
	dev_priv->saveMI_ARB_STATE = I915_READ(MI_ARB_STATE);

	/* Scratch space */
	for (i = 0; i < 16; i++) {
		dev_priv->saveSWF0[i] = I915_READ(SWF00 + (i << 2));
		dev_priv->saveSWF1[i] = I915_READ(SWF10 + (i << 2));
	}
	for (i = 0; i < 3; i++)
		dev_priv->saveSWF2[i] = I915_READ(SWF30 + (i << 2));

	/* Fences */
	if (IS_I965G(dev_priv)) {
		for (i = 0; i < 16; i++)
			dev_priv->saveFENCE[i] = I915_READ64(FENCE_REG_965_0 +
			    (i * 8));
	} else {
		for (i = 0; i < 8; i++)
			dev_priv->saveFENCE[i] = I915_READ(FENCE_REG_830_0 + (i * 4));

		if (IS_I945G(dev_priv) || IS_I945GM(dev_priv) ||
		    IS_G33(dev_priv))
			for (i = 0; i < 8; i++)
				dev_priv->saveFENCE[i+8] = I915_READ(FENCE_REG_945_8 + (i * 4));
	}
	i915_save_vga(dev_priv);

	return 0;
}

int
inteldrm_restore_state(struct drm_i915_private *dev_priv)
{
	int	i;

	pci_conf_write(dev_priv->pc, dev_priv->tag, LBB, dev_priv->saveLBB);

	/* Render Standby */
	if (IS_I965G(dev_priv) && IS_MOBILE(dev_priv))
		I915_WRITE(MCHBAR_RENDER_STANDBY, dev_priv->saveRENDERSTANDBY);

	/* Hardware status page */
	I915_WRITE(HWS_PGA, dev_priv->saveHWS);

	/* Display arbitration */
	I915_WRITE(DSPARB, dev_priv->saveDSPARB);

	/* Fences */
	if (IS_I965G(dev_priv)) {
		for (i = 0; i < 16; i++)
			I915_WRITE64(FENCE_REG_965_0 + (i * 8), dev_priv->saveFENCE[i]);
	} else {
		for (i = 0; i < 8; i++)
			I915_WRITE(FENCE_REG_830_0 + (i * 4), dev_priv->saveFENCE[i]);
		if (IS_I945G(dev_priv) || IS_I945GM(dev_priv) ||
		    IS_G33(dev_priv))
			for (i = 0; i < 8; i++)
				I915_WRITE(FENCE_REG_945_8 + (i * 4), dev_priv->saveFENCE[i+8]);
	}
	
	/* This is only meaningful in non-KMS mode */
	/* Don't restore them in KMS mode */
	i915_restore_modeset_reg(dev_priv);
	/* Cursor state */
	I915_WRITE(CURAPOS, dev_priv->saveCURAPOS);
	I915_WRITE(CURACNTR, dev_priv->saveCURACNTR);
	I915_WRITE(CURABASE, dev_priv->saveCURABASE);
	I915_WRITE(CURBPOS, dev_priv->saveCURBPOS);
	I915_WRITE(CURBCNTR, dev_priv->saveCURBCNTR);
	I915_WRITE(CURBBASE, dev_priv->saveCURBBASE);
	if (!IS_I9XX(dev_priv))
		I915_WRITE(CURSIZE, dev_priv->saveCURSIZE);

	/* CRT state */
	I915_WRITE(ADPA, dev_priv->saveADPA);

	/* LVDS state */
	if (IS_I965G(dev_priv))
		I915_WRITE(BLC_PWM_CTL2, dev_priv->saveBLC_PWM_CTL2);
	if (IS_MOBILE(dev_priv) && !IS_I830(dev_priv))
		I915_WRITE(LVDS, dev_priv->saveLVDS);
	if (!IS_I830(dev_priv) && !IS_845G(dev_priv))
		I915_WRITE(PFIT_CONTROL, dev_priv->savePFIT_CONTROL);

	I915_WRITE(PFIT_PGM_RATIOS, dev_priv->savePFIT_PGM_RATIOS);
	I915_WRITE(BLC_PWM_CTL, dev_priv->saveBLC_PWM_CTL);
	I915_WRITE(PP_ON_DELAYS, dev_priv->savePP_ON_DELAYS);
	I915_WRITE(PP_OFF_DELAYS, dev_priv->savePP_OFF_DELAYS);
	I915_WRITE(PP_DIVISOR, dev_priv->savePP_DIVISOR);
	I915_WRITE(PP_CONTROL, dev_priv->savePP_CONTROL);

	/* XXX: Display Port state */

	/* FIXME: restore TV & SDVO state */

	/* FBC info */
	I915_WRITE(FBC_CFB_BASE, dev_priv->saveFBC_CFB_BASE);
	I915_WRITE(FBC_LL_BASE, dev_priv->saveFBC_LL_BASE);
	I915_WRITE(FBC_CONTROL2, dev_priv->saveFBC_CONTROL2);
	I915_WRITE(FBC_CONTROL, dev_priv->saveFBC_CONTROL);

	/* VGA state */
	I915_WRITE(VGACNTRL, dev_priv->saveVGACNTRL);
	I915_WRITE(VGA0, dev_priv->saveVGA0);
	I915_WRITE(VGA1, dev_priv->saveVGA1);
	I915_WRITE(VGA_PD, dev_priv->saveVGA_PD);
	DRM_UDELAY(150);

	/* Clock gating state */
	I915_WRITE (D_STATE, dev_priv->saveD_STATE);
	I915_WRITE (DSPCLK_GATE_D, dev_priv->saveDSPCLK_GATE_D);

	/* Cache mode state */
	I915_WRITE (CACHE_MODE_0, dev_priv->saveCACHE_MODE_0 | 0xffff0000);

	/* Memory arbitration state */
	I915_WRITE (MI_ARB_STATE, dev_priv->saveMI_ARB_STATE | 0xffff0000);

	for (i = 0; i < 16; i++) {
		I915_WRITE(SWF00 + (i << 2), dev_priv->saveSWF0[i]);
		I915_WRITE(SWF10 + (i << 2), dev_priv->saveSWF1[i]);
	}
	for (i = 0; i < 3; i++)
		I915_WRITE(SWF30 + (i << 2), dev_priv->saveSWF2[i]);

	i915_restore_vga(dev_priv);

	return 0;
}

#ifdef INTELDRM_GEM
/* 
 * Reset the chip after a hang (965 only)
 *
 * The procedure that should be followed is relatively simple:
 *	- reset the chip using the reset reg
 *	- re-init context state
 *	- re-init Hardware status page
 *	- re-init ringbuffer
 *	- re-init interrupt state
 *	- re-init display
 */
void
inteldrm_965_reset(struct drm_i915_private *dev_priv, u_int8_t flags)
{
	pcireg_t	reg;
	int		i = 0;

	if (flags == GDRST_FULL)
		inteldrm_save_display(dev_priv);

	reg = pci_conf_read(dev_priv->pc, dev_priv->tag, GDRST);
	/*
	 * Set the domains we want to reset, then bit 0 (reset itself).
	 * then we wait for the hardware to clear it.
	 */
	pci_conf_write(dev_priv->pc, dev_priv->tag, GDRST,
	    reg | (u_int32_t)flags | ((flags == GDRST_FULL) ? 0x1 : 0x0));
	delay(50);
	/* don't clobber the rest of the register */
	pci_conf_write(dev_priv->pc, dev_priv->tag, GDRST, reg & 0xfe);

	/* if this fails we're pretty much fucked, but don't loop forever */
	do {
		delay(100);
		reg = pci_conf_read(dev_priv->pc, dev_priv->tag, GDRST);
	} while ((reg & 0x1) && ++i < 10);

	if (reg & 0x1)
		printf("bit 0 not cleared .. ");

	/* put everything back together again */

	/*
	 * GTT is already up (we didn't do a pci-level reset, thank god.
	 *
	 * We don't have to restore the contexts (we don't use them yet).
	 * So, if X is running we need to put the ringbuffer back first.
	 */
	 if (dev_priv->mm.suspended == 0) {
		struct drm_device *dev = (struct drm_device *)dev_priv->drmdev;
		if (inteldrm_start_ring(dev_priv) != 0)
			panic("can't restart ring, we're fucked"); 

		/* put the hardware status page back */
		if (I915_NEED_GFX_HWS(dev_priv))
			I915_WRITE(HWS_PGA, ((struct inteldrm_obj *)
			    dev_priv->hws_obj)->gtt_offset);
		else
			I915_WRITE(HWS_PGA,
			    dev_priv->hws_dmamem->map->dm_segs[0].ds_addr);
		I915_READ(HWS_PGA); /* posting read */

		/* so we remove the handler and can put it back in */
		DRM_UNLOCK();
		drm_irq_uninstall(dev);
		drm_irq_install(dev);
		DRM_LOCK();
	 } else
		printf("not restarting ring...\n");


	 if (flags == GDRST_FULL)
		inteldrm_restore_display(dev_priv);
}
#endif /* INTELDRM_GEM */

/*
 * Debug code from here. 
 */
#ifdef WATCH_INACTIVE
void
inteldrm_verify_inactive(struct drm_i915_private *dev_priv, char *file,
    int line)
{
	struct drm_obj		*obj;
	struct inteldrm_obj	*obj_priv;

	TAILQ_FOREACH(obj_priv, &dev_priv->mm.inactive_list, list) {
		obj = (struct drm_obj *)obj_priv;
		if (obj_priv->pin_count || inteldrm_is_active(obj_priv) ||
		    obj->write_domain & I915_GEM_GPU_DOMAINS)
			DRM_ERROR("inactive %p (p $d a $d w $x) %s:%d\n",
			    obj, obj_priv->pin_count,
			    inteldrm_is_active(obj_priv),
			    obj->write_domain, file, line);
	}
}
#endif /* WATCH_INACTIVE */

#if (INTELDRM_DEBUG > 1)

static const char *get_pin_flag(struct inteldrm_obj *obj_priv)
{
	if (obj_priv->pin_count > 0)
		return "p";
	else
		return " ";
}

static const char *get_tiling_flag(struct inteldrm_obj *obj_priv)
{
    switch (obj_priv->tiling_mode) {
    default:
    case I915_TILING_NONE: return " ";
    case I915_TILING_X: return "X";
    case I915_TILING_Y: return "Y";
    }
}

void
i915_gem_seqno_info(int kdev)
{
	struct drm_device *dev = drm_get_device_from_kdev(kdev);
	drm_i915_private_t *dev_priv = dev->dev_private;

	if (dev_priv->hw_status_page != NULL) {
		printf("Current sequence: %d\n", i915_get_gem_seqno(dev_priv));
	} else {
		printf("Current sequence: hws uninitialized\n");
	}
}


void
i915_interrupt_info(int kdev)
{
	struct drm_device *dev = drm_get_device_from_kdev(kdev);
	drm_i915_private_t *dev_priv = dev->dev_private;

	printf("Interrupt enable:    %08x\n",
		   I915_READ(IER));
	printf("Interrupt identity:  %08x\n",
		   I915_READ(IIR));
	printf("Interrupt mask:      %08x\n",
		   I915_READ(IMR));
	printf("Pipe A stat:         %08x\n",
		   I915_READ(PIPEASTAT));
	printf("Pipe B stat:         %08x\n",
		   I915_READ(PIPEBSTAT));
	printf("Interrupts received: 0\n");
	if (dev_priv->hw_status_page != NULL) {
		printf("Current sequence:    %d\n",
			   i915_get_gem_seqno(dev_priv));
	} else {
		printf("Current sequence:    hws uninitialized\n");
	}
}

void
i915_gem_fence_regs_info(int kdev)
{
	struct drm_device *dev = drm_get_device_from_kdev(kdev);
	drm_i915_private_t *dev_priv = dev->dev_private;
	int i;

	printf("Reserved fences = %d\n", dev_priv->fence_reg_start);
	printf("Total fences = %d\n", dev_priv->num_fence_regs);
	for (i = 0; i < dev_priv->num_fence_regs; i++) {
		struct drm_obj *obj = dev_priv->fence_regs[i].obj;

		if (obj == NULL) {
			printf("Fenced object[%2d] = unused\n", i);
		} else {
			struct inteldrm_obj *obj_priv;

			obj_priv = (struct inteldrm_obj *)obj;
			printf("Fenced object[%2d] = %p: %s "
				   "%08x %08zx %08x %s %08x %08x %d",
				   i, obj, get_pin_flag(obj_priv),
				   obj_priv->gtt_offset,
				   obj->size, obj_priv->stride,
				   get_tiling_flag(obj_priv),
				   obj->read_domains, obj->write_domain,
				   obj_priv->last_rendering_seqno);
			if (obj->name)
				printf(" (name: %d)", obj->name);
			printf("\n");
		}
	}
}

void
i915_hws_info(int kdev)
{
	struct drm_device *dev = drm_get_device_from_kdev(kdev);
	drm_i915_private_t *dev_priv = dev->dev_private;
	int i;
	volatile u32 *hws;

	hws = (volatile u32 *)dev_priv->hw_status_page;
	if (hws == NULL)
		return;

	for (i = 0; i < 4096 / sizeof(u32) / 4; i += 4) {
		printf("0x%08x: 0x%08x 0x%08x 0x%08x 0x%08x\n",
			   i * 4,
			   hws[i], hws[i + 1], hws[i + 2], hws[i + 3]);
	}
}

static void
i915_dump_pages(bus_space_tag_t bst, bus_space_handle_t bsh,
    bus_size_t size)
{
	bus_addr_t	offset = 0;
	int		i = 0;

	/*
	 * this is a bit odd so i don't have to play with the intel
	 * tools too much.
	 */
	for (offset = 0; offset < size; offset += 4, i += 4) {
		if (i == PAGE_SIZE)
			i = 0;
		printf("%08x :  %08x\n", i, bus_space_read_4(bst, bsh,
		    offset));
	}
}

void
i915_batchbuffer_info(int kdev)
{
	struct drm_device	*dev = drm_get_device_from_kdev(kdev);
	drm_i915_private_t	*dev_priv = dev->dev_private;
	struct drm_obj		*obj;
	struct inteldrm_obj	*obj_priv;
	bus_space_handle_t	 bsh;
	int			 ret;

	TAILQ_FOREACH(obj_priv, &dev_priv->mm.active_list, list) {
		obj = &obj_priv->obj;
		if (obj->read_domains & I915_GEM_DOMAIN_COMMAND) {
			if ((ret = bus_space_subregion(dev_priv->bst,
			    dev_priv->aperture_bsh, obj_priv->gtt_offset,
			    obj->size, &bsh)) != 0) {
				DRM_ERROR("Failed to map pages: %d\n", ret);
				return;
			}
			printf("--- gtt_offset = 0x%08x\n",
			    obj_priv->gtt_offset);
			i915_dump_pages(dev_priv->bst, bsh, obj->size);
		}
	}
}

void
i915_ringbuffer_data(int kdev)
{
	struct drm_device	*dev = drm_get_device_from_kdev(kdev);
	drm_i915_private_t	*dev_priv = dev->dev_private;
	bus_size_t		 off;

	if (!dev_priv->ring.ring_obj) {
		printf("No ringbuffer setup\n");
		return;
	}

	for (off = 0; off < dev_priv->ring.size; off += 4)
		printf("%08x :  %08x\n", off, bus_space_read_4(dev_priv->bst,
		    dev_priv->ring.bsh, off));
}

void
i915_ringbuffer_info(int kdev)
{
	struct drm_device	*dev = drm_get_device_from_kdev(kdev);
	drm_i915_private_t	*dev_priv = dev->dev_private;
	u_int32_t		 head, tail;

	head = I915_READ(PRB0_HEAD) & HEAD_ADDR;
	tail = I915_READ(PRB0_TAIL) & TAIL_ADDR;

	printf("RingHead :  %08x\n", head);
	printf("RingTail :  %08x\n", tail);
	printf("RingMask :  %08x\n", dev_priv->ring.size - 1);
	printf("RingSize :  %08lx\n", dev_priv->ring.size);
	printf("Acthd :  %08x\n", I915_READ(IS_I965G(dev_priv) ?
	    ACTHD_I965 : ACTHD));
}

#endif