The locking rework/fix that I promised when I commited GEM.

Before, as well as being kinda nasty there was a very definite race, if
the last reference to an object  was removed by uvm (a map going away),
then the free path happened unlocked, this could cause all kinds of
havoc.

In order to deal with this, move to fine-grained locking. Since uvm
object locks are spinlocks, and we need to sleep in operations that will
wait on the gpu, provide a DRM_BUSY flag that is set on a locked object
that then allows us to unlock and sleep (this is similar to several
things done in uvm on pages and some object types).

The rwlock stays around to ensure that execbuffer can have acces to the
whole gtt, so ioctls that bind to the gtt need a read lock, and
execuffer gets a write lock. otherwise most ioctls just need to busy the
object that they operate on. Lists also have their own locks.

Some cleanup could be done to make this a little prettier, but it is
much more correct than previously.

Tested very very vigorously on 855 (x40) and 965 (x61s), this found numerous
bugs. Also, the I can no longer crash the kernel at will.

A bunch of asserts hidden under DRMLOCKDEBUG have been left in the code for
debugging purposes.

4 files changed, 628 insertions, 256 deletions

diff --git a/sys/dev/pci/drm/drmP.h b/sys/dev/pci/drm/drmP.h
index 5d8f6046216..cb76d42735c 100644
--- a/sys/dev/pci/drm/drmP.h
+++ b/sys/dev/pci/drm/drmP.h
@@ -86,6 +86,8 @@
 #define DRM_CURRENTPID		curproc->p_pid
 #define DRM_LOCK()		rw_enter_write(&dev->dev_lock)
 #define DRM_UNLOCK()		rw_exit_write(&dev->dev_lock)
+#define DRM_READLOCK()		rw_enter_read(&dev->dev_lock)
+#define DRM_READUNLOCK()	rw_exit_read(&dev->dev_lock)
 #define DRM_MAXUNITS		8
 
 /* D_CLONE only supports one device, this will be fixed eventually */
@@ -353,6 +355,22 @@ struct drm_ati_pcigart_info {
 	int			 gart_reg_if;
 };
 
+/*
+ *  Locking protocol:
+ * All drm object are uvm objects, as such they have a reference count and
+ * a lock. On the other hand, operations carries out by the drm may involve
+ * sleeping (waiting for rendering to finish, say), while you wish to have
+ * mutual exclusion on an object. For this reason, all drm-related operations
+ * on drm objects must acquire the DRM_BUSY flag on the object as the first
+ * thing that they do. If the BUSY flag is already on the object, set DRM_WANTED
+ * and sleep until the other locker is done with it. When the BUSY flag is 
+ * acquired then only that flag and a reference is required to do most 
+ * operations on the drm_object. The uvm object is still bound by uvm locking
+ * protocol.
+ *
+ * Subdrivers (radeon, intel, etc) may have other locking requirement, these
+ * requirements will be detailed in those drivers.
+ */
 struct drm_obj {
 	struct uvm_object		 uobj;
 	SPLAY_ENTRY(drm_obj)	 	 entry;
@@ -362,6 +380,13 @@ struct drm_obj {
 	size_t				 size;
 	int				 name;
 	int				 handlecount;
+/* any flags over 0x00000010 are device specific */
+#define	DRM_BUSY	0x00000001
+#define	DRM_WANTED	0x00000002
+	u_int				 do_flags;
+#ifdef DRMLOCKDEBUG			/* to tell owner */
+	struct proc			*holding_proc;
+#endif
 	uint32_t			 read_domains;
 	uint32_t			 write_domain;
 
@@ -640,8 +665,16 @@ int	drm_sg_alloc_ioctl(struct drm_device *, void *, struct drm_file *);
 int	drm_sg_free(struct drm_device *, void *, struct drm_file *);
 
 struct drm_obj *drm_gem_object_alloc(struct drm_device *, size_t);
-void	drm_gem_object_reference(struct drm_obj *);
-void	drm_gem_object_unreference(struct drm_obj *);
+void	 drm_unref(struct uvm_object *);
+void	 drm_ref(struct uvm_object *);
+void	 drm_unref_locked(struct uvm_object *);
+void	 drm_ref_locked(struct uvm_object *);
+void	drm_hold_object_locked(struct drm_obj *);
+void	drm_hold_object(struct drm_obj *);
+void	drm_unhold_object_locked(struct drm_obj *);
+void	drm_unhold_object(struct drm_obj *);
+int	drm_try_hold_object(struct drm_obj *);
+void	drm_unhold_and_unref(struct drm_obj *);
 int	drm_handle_create(struct drm_file *, struct drm_obj *, int *);
 struct drm_obj *drm_gem_object_lookup(struct drm_device *,
 			    struct drm_file *, int );
@@ -651,5 +684,43 @@ int	drm_gem_open_ioctl(struct drm_device *, void *, struct drm_file *);
 int	drm_gem_load_uao(bus_dma_tag_t, bus_dmamap_t, struct uvm_object *,
 	    bus_size_t, int, bus_dma_segment_t **);
 
+static __inline void
+drm_gem_object_reference(struct drm_obj *obj)
+{
+	drm_ref(&obj->uobj);
+}
+
+static __inline void
+drm_gem_object_unreference(struct drm_obj *obj)
+{
+	drm_unref(&obj->uobj);
+}
+
+static __inline void 
+drm_lock_obj(struct drm_obj *obj)
+{
+	simple_lock(&obj->uobj);
+}
+
+static __inline void 
+drm_unlock_obj(struct drm_obj *obj)
+{
+	simple_unlock(&obj->uobj);
+}
+#ifdef DRMLOCKDEBUG
+
+#define DRM_ASSERT_HELD(obj)		\
+	KASSERT(obj->do_flags & DRM_BUSY && obj->holding_proc == curproc)
+#define DRM_OBJ_ASSERT_LOCKED(obj) /* XXX mutexes */
+#define DRM_ASSERT_LOCKED(lock) MUTEX_ASSERT_LOCKED(lock)
+#else
+
+#define DRM_ASSERT_HELD(obj)
+#define DRM_OBJ_ASSERT_LOCKED(obj)
+#define DRM_ASSERT_LOCKED(lock) 
+
+#endif
+
+
 #endif /* __KERNEL__ */
 #endif /* _DRM_P_H_ */
diff --git a/sys/dev/pci/drm/drm_drv.c b/sys/dev/pci/drm/drm_drv.c
index 4dfd6895ec6..7db9a43153f 100644
--- a/sys/dev/pci/drm/drm_drv.c
+++ b/sys/dev/pci/drm/drm_drv.c
@@ -78,8 +78,6 @@ void	 drm_handle_unref(struct drm_obj *);
 
 int	 drm_handle_cmp(struct drm_handle *, struct drm_handle *);
 int	 drm_name_cmp(struct drm_obj *, struct drm_obj *);
-void	 drm_unref(struct uvm_object *);
-void	 drm_ref(struct uvm_object *);
 int	 drm_fault(struct uvm_faultinfo *, vaddr_t, vm_page_t *, int, int,
 	     vm_fault_t, vm_prot_t, int);
 boolean_t	 drm_flush(struct uvm_object *, voff_t, voff_t, int);
@@ -1032,26 +1030,127 @@ struct uvm_pagerops drm_pgops = {
 	drm_flush,
 };
 
+
+void
+drm_hold_object_locked(struct drm_obj *obj)
+{
+	while (obj->do_flags & DRM_BUSY) {
+		atomic_setbits_int(&obj->do_flags, DRM_WANTED);
+		simple_unlock(&uobj->vmobjlock);
+#ifdef DRMLOCKDEBUG
+		{
+		int ret = 0;
+		ret = tsleep(obj, PVM, "drm_hold", 3 * hz); /* XXX msleep */
+		if (ret)
+			printf("still waiting for obj %p, owned by %p\n",
+			    obj, obj->holding_proc);
+		}
+#else
+		tsleep(obj, PVM, "drm_hold", 0); /* XXX msleep */
+#endif
+		simple_lock(&uobj->vmobjlock);
+	}
+#ifdef DRMLOCKDEBUG
+	obj->holding_proc = curproc;
+#endif
+	atomic_setbits_int(&obj->do_flags, DRM_BUSY);
+}
+
+void
+drm_hold_object(struct drm_obj *obj)
+{
+	simple_lock(&obj->uobj->vmobjlock);
+	drm_hold_object_locked(obj);
+	simple_unlock(&obj->uobj->vmobjlock);
+}
+
+int
+drm_try_hold_object(struct drm_obj *obj)
+{
+	simple_lock(&obj->uobj->vmobjlock);
+	/* if the object is free, grab it */
+	if (obj->do_flags & (DRM_BUSY | DRM_WANTED))
+		return (0);
+	atomic_setbits_int(&obj->do_flags, DRM_BUSY);
+#ifdef DRMLOCKDEBUG
+	obj->holding_proc = curproc;
+#endif
+	simple_unlock(&obj->uobj->vmobjlock);
+	return (1);
+}
+
+
+void
+drm_unhold_object_locked(struct drm_obj *obj)
+{
+	if (obj->do_flags & DRM_WANTED)
+		wakeup(obj);
+#ifdef DRMLOCKDEBUG
+	obj->holding_proc = NULL;
+#endif
+	atomic_clearbits_int(&obj->do_flags, DRM_WANTED | DRM_BUSY);	
+}
+
+void
+drm_unhold_object(struct drm_obj *obj)
+{
+	simple_lock(&obj->uobj->vmobjlock);
+	drm_unhold_object_locked(obj);
+	simple_unlock(&obj->uobj->vmobjlock);
+}
+
+void
+drm_ref_locked(struct uvm_object *uobj)
+{
+	uobj->uo_refs++;
+}
+
 void
 drm_ref(struct uvm_object *uobj)
 {
 	simple_lock(&uobj->vmobjlock);
-	uobj->uo_refs++;
+	drm_ref_locked(uobj);
 	simple_unlock(&uobj->vmobjlock);
 }
 
 void
 drm_unref(struct uvm_object *uobj)
 {
+	simple_lock(&uobj->vmobjlock);
+	drm_unref_locked(uobj);
+}
+
+void
+drm_unref_locked(struct uvm_object *uobj)
+{
 	struct drm_obj		*obj = (struct drm_obj *)uobj;
 	struct drm_device	*dev = obj->dev;
 
-	simple_lock(&uobj->vmobjlock);
-	if (--uobj->uo_refs > 0) {
+again:
+	if (uobj->uo_refs > 1) {
+		uobj->uo_refs--;
 		simple_unlock(&uobj->vmobjlock);
 		return;
 	}
 
+	/* inlined version of drm_hold because we want to trylock then sleep */
+	if (obj->do_flags & DRM_BUSY) {
+		atomic_setbits_int(&obj->do_flags, DRM_WANTED);
+		simple_unlock(&uobj->vmobjlock);
+		tsleep(obj, PVM, "drm_unref", 0); /* XXX msleep */
+		simple_lock(&uobj->vmobjlock);
+		goto again;
+	}
+#ifdef DRMLOCKDEBUG
+	obj->holding_proc = curproc;
+#endif
+	atomic_setbits_int(&obj->do_flags, DRM_BUSY);
+	simple_unlock(&obj->vmobjlock);
+	/* We own this thing now. it is on no queues, though it may still
+	 * be bound to the aperture (and on the inactive list, in which case
+	 * idling the buffer is what triggered the free. Since we know no one 
+	 * else can grab it now, we can nuke with impunity.
+	 */
 	if (dev->driver->gem_free_object != NULL)
 		dev->driver->gem_free_object(obj);
 
@@ -1059,10 +1158,22 @@ drm_unref(struct uvm_object *uobj)
 
 	atomic_dec(&dev->obj_count);
 	atomic_sub(obj->size, &dev->obj_memory);
-	simple_unlock(&uobj->vmobjlock);
+	if (obj->do_flags & DRM_WANTED) /* should never happen, not on lists */
+		wakeup(obj);
 	pool_put(&dev->objpl, obj);
 }
 
+/*
+ * convenience function to unreference and unhold an object.
+ */
+void
+drm_unhold_and_unref(struct drm_obj *obj)
+{
+	drm_lock_obj(obj);
+	drm_unhold_object_locked(obj);
+	drm_unref_locked(&obj->uobj);
+}
+
 
 boolean_t	
 drm_flush(struct uvm_object *uobj, voff_t start, voff_t stop, int flags)
@@ -1100,11 +1211,6 @@ drm_fault(struct uvm_faultinfo *ufi, vaddr_t vaddr, vm_page_t *pps,
 	ret = dev->driver->gem_fault(obj, ufi, entry->offset + (vaddr -
 	    entry->start), vaddr, pps, npages, centeridx,
 	    access_type, flags);
-
-	uvmfault_unlockall(ufi, ufi->entry->aref.ar_amap, uobj, NULL);
-	pmap_update(ufi->orig_map->pmap);
-	if (ret != VM_PAGER_OK)
-		uvm_wait("drm_fault");
 	return (ret);
 }
 
@@ -1187,7 +1293,7 @@ drm_gem_object_lookup(struct drm_device *dev, struct drm_file *file_priv,
 	}
 
 	obj = han->obj;
-	drm_gem_object_reference(obj);
+	drm_ref(&obj->uobj);
 	mtx_leave(&file_priv->table_lock);
 
 	return (obj);
@@ -1248,15 +1354,13 @@ again:
 		    &dev->name_tree, obj))
 			goto again;
 		/* name holds a reference to the object */
-		drm_gem_object_reference(obj);
+		drm_ref(&obj->uobj);
 	}
 	mtx_leave(&dev->obj_name_lock);
 
 	args->name = (uint64_t)obj->name;
 
-	DRM_LOCK();
-	drm_gem_object_unreference(obj);
-	DRM_UNLOCK();
+	drm_unref(&obj->uobj);
 
 	return (0);
 }
@@ -1276,17 +1380,15 @@ drm_gem_open_ioctl(struct drm_device *dev, void *data,
 	mtx_enter(&dev->obj_name_lock);
 	obj = SPLAY_FIND(drm_name_tree, &dev->name_tree, &search);
 	if (obj != NULL)
-		drm_gem_object_reference(obj);
+		drm_ref(&obj->uobj);
 	mtx_leave(&dev->obj_name_lock);
 	if (obj == NULL)
 		return (ENOENT);
 
+	/* this gives our reference to the handle */
 	ret = drm_handle_create(file_priv, obj, &handle);
-	/* handle has a reference, drop ours. */
-	DRM_LOCK();
-	drm_gem_object_unreference(obj);
-	DRM_UNLOCK();
 	if (ret) {
+		drm_unref(&obj->uobj);
 		return (ret);
 	}
 
@@ -1296,19 +1398,6 @@ drm_gem_open_ioctl(struct drm_device *dev, void *data,
         return (0);
 }
 
-void
-drm_gem_object_reference(struct drm_obj *obj)
-{
-	drm_ref(&obj->uobj);
-}
-
-void
-drm_gem_object_unreference(struct drm_obj *obj)
-{
-	drm_unref(&obj->uobj);
-}
-
-
 /*
  * grab a reference for a per-open handle. 
  * The object contains a handlecount too because if all handles disappear we
@@ -1318,8 +1407,10 @@ drm_gem_object_unreference(struct drm_obj *obj)
 void
 drm_handle_ref(struct drm_obj *obj)
 {
+	/* we are given the reference from the caller, so just
+	 * crank handlecount.
+	 */
 	obj->handlecount++;
-	drm_gem_object_reference(obj);
 }
 
 /*
@@ -1339,12 +1430,12 @@ drm_handle_unref(struct drm_obj *obj)
 			obj->name = 0;
 			mtx_leave(&dev->obj_name_lock);
 			/* name held a reference to object */
-			drm_gem_object_unreference(obj);
+			drm_unref(&obj->uobj);
 		} else {
 			mtx_leave(&dev->obj_name_lock);
 		}
 	}
-	drm_gem_object_unreference(obj);
+	drm_unref(&obj->uobj);
 }
 
 /*
diff --git a/sys/dev/pci/drm/i915_drv.c b/sys/dev/pci/drm/i915_drv.c
index b879f750688..3ba82a2d717 100644
--- a/sys/dev/pci/drm/i915_drv.c
+++ b/sys/dev/pci/drm/i915_drv.c
@@ -144,6 +144,7 @@ int	i915_gem_idle(struct drm_i915_private *);
 void	i915_gem_object_move_to_active(struct drm_obj *);
 void	i915_gem_object_move_off_active(struct drm_obj *);
 void	i915_gem_object_move_to_inactive(struct drm_obj *);
+void	i915_gem_object_move_to_inactive_locked(struct drm_obj *);
 uint32_t	i915_add_request(struct drm_i915_private *);
 void	inteldrm_process_flushing(struct drm_i915_private *, u_int32_t);
 void	i915_move_to_tail(struct inteldrm_obj *, struct i915_gem_list *);
@@ -450,6 +451,9 @@ inteldrm_attach(struct device *parent, struct device *self, void *aux)
 	printf(": %s\n", pci_intr_string(pa->pa_pc, dev_priv->ih));
 
 	mtx_init(&dev_priv->user_irq_lock, IPL_TTY);
+	mtx_init(&dev_priv->list_lock, IPL_NONE);
+	mtx_init(&dev_priv->request_lock, IPL_NONE);
+	mtx_init(&dev_priv->fence_lock, IPL_NONE);
 
 	/* All intel chipsets need to be treated as agp, so just pass one */
 	dev_priv->drmdev = drm_attach_pci(&inteldrm_driver, pa, 1, self);
@@ -978,9 +982,12 @@ i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
 {
 	struct drm_i915_gem_get_aperture	*args = data;
 
+	/* we need a write lock here to make sure we get the right value */
+	DRM_LOCK();
 	args->aper_size = dev->gtt_total;
 	args->aper_available_size = (args->aper_size -
 	    atomic_read(&dev->pin_memory));
+	DRM_UNLOCK();
 
 	return (0);
 }
@@ -1003,14 +1010,13 @@ i915_gem_create_ioctl(struct drm_device *dev, void *data,
 	if (obj == NULL)
 		return (ENOMEM);
 
+	/* we give our reference to the handle */
 	ret = drm_handle_create(file_priv, obj, &handle);
-	/* handle has a reference now, drop ours. */
-	DRM_LOCK();
-	drm_gem_object_unreference(obj);
-	DRM_UNLOCK();
 
 	if (ret == 0)
 		args->handle = handle;
+	else
+		drm_unref(&obj->uobj);
 
 	return (ret);
 }
@@ -1037,11 +1043,12 @@ i915_gem_pread_ioctl(struct drm_device *dev, void *data,
 	obj = drm_gem_object_lookup(dev, file_priv, args->handle);
 	if (obj == NULL)
 		return (EBADF);
+	DRM_READLOCK();
+	drm_hold_object(obj);
 
 	/*
 	 * Bounds check source.
 	 */
-	DRM_LOCK();
 	if (args->offset > obj->size || args->size > obj->size ||
 	    args->offset + args->size > obj->size) {
 		ret = EINVAL;
@@ -1049,8 +1056,9 @@ i915_gem_pread_ioctl(struct drm_device *dev, void *data,
 	}
 
 	ret = i915_gem_object_pin(obj, 0, 1);
-	if (ret)
+	if (ret) {
 		goto out;
+	}
 	ret = i915_gem_object_set_to_gtt_domain(obj, 0, 1);
 	if (ret)
 		goto unpin;
@@ -1078,8 +1086,8 @@ i915_gem_pread_ioctl(struct drm_device *dev, void *data,
 unpin:
 	i915_gem_object_unpin(obj);
 out:
-	drm_gem_object_unreference(obj);
-	DRM_UNLOCK();
+	drm_unhold_and_unref(obj);
+	DRM_READUNLOCK();
 
 	return (ret);
 }
@@ -1107,8 +1115,9 @@ i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
 	obj = drm_gem_object_lookup(dev, file_priv, args->handle);
 	if (obj == NULL)
 		return (EBADF);
+	DRM_READLOCK();
+	drm_hold_object(obj);
 
-	DRM_LOCK();
 	/* Bounds check destination. */
 	if (args->offset > obj->size || args->size > obj->size ||
 	    args->offset + args->size > obj->size) {
@@ -1117,8 +1126,9 @@ i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
 	}
 
 	ret = i915_gem_object_pin(obj, 0, 1);
-	if (ret)
+	if (ret) {
 		goto out;
+	}
 	ret = i915_gem_object_set_to_gtt_domain(obj, 1, 1);
 	if (ret)
 		goto done;
@@ -1143,8 +1153,8 @@ i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
 done:
 	i915_gem_object_unpin(obj);
 out:
-	drm_gem_object_unreference(obj);
-	DRM_UNLOCK();
+	drm_unhold_and_unref(obj);
+	DRM_READUNLOCK();
 
 	return (ret);
 }
@@ -1177,12 +1187,11 @@ i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
 	obj = drm_gem_object_lookup(dev, file_priv, args->handle);
 	if (obj == NULL)
 		return (EBADF);
+	drm_hold_object(obj);
 
-	DRM_LOCK();
 	ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0, 1);
 
-	drm_gem_object_unreference(obj);
-	DRM_UNLOCK();
+	drm_unhold_and_unref(obj);
 	/*
 	 * Silently promote `you're not bound, there was nothing to do'
 	 * to success, since the client was just asking us to make sure
@@ -1207,7 +1216,9 @@ i915_gem_gtt_map_ioctl(struct drm_device *dev, void *data,
 	if (obj == NULL)
 		return (EBADF);
 
-	DRM_LOCK();
+	/* Since we are doing purely uvm-related operations here we do
+	 * not need to hold the object, a reference alone is sufficient
+	 */
 	obj_priv = (struct inteldrm_obj *)obj;
 
 	/* Check size. Also ensure that the object is not purgeable */
@@ -1233,7 +1244,7 @@ i915_gem_gtt_map_ioctl(struct drm_device *dev, void *data,
 
 done:
 	if (ret != 0)
-		drm_gem_object_unreference(obj);
+		drm_unref(&obj->uobj);
 	DRM_UNLOCK();
 
 	if (ret == 0)
@@ -1252,10 +1263,13 @@ i915_gem_object_move_to_active(struct drm_obj *obj)
 	struct inteldrm_fence	*reg;
 	u_int32_t		 seqno = dev_priv->mm.next_gem_seqno;
 
+	MUTEX_ASSERT_LOCKED(&dev_priv->request_lock);
+	MUTEX_ASSERT_LOCKED(&dev_priv->list_lock);
+
 	/* Add a reference if we're newly entering the active list. */
 	if (!inteldrm_is_active(obj_priv)) {
-		drm_gem_object_reference(obj);
-		atomic_setbits_int(&obj_priv->io_flags, I915_ACTIVE);
+		drm_ref(&obj->uobj);
+		atomic_setbits_int(&obj->do_flags, I915_ACTIVE);
 	}
 
 	if (inteldrm_needs_fence(obj_priv)) {
@@ -1276,21 +1290,43 @@ i915_gem_object_move_off_active(struct drm_obj *obj)
 	struct inteldrm_obj	*obj_priv = (struct inteldrm_obj *)obj;
 	struct inteldrm_fence	*reg;
 
+	MUTEX_ASSERT_LOCKED(&dev_priv->list_lock);
+	DRM_OBJ_ASSERT_LOCKED(obj);
+	
 	obj_priv->last_rendering_seqno = 0;
-	if (inteldrm_needs_fence(obj_priv)) {
+	/* if we have a fence register, then reset the seqno */
+	if (obj_priv->fence_reg != I915_FENCE_REG_NONE) {
 		reg = &dev_priv->fence_regs[obj_priv->fence_reg];
 		reg->last_rendering_seqno = 0;
 	}
 }
 
-/* called locked */
+/* If you call this on an object that you have held, you must have your own
+ * reference, not just the reference from the active list.
+ */
 void
 i915_gem_object_move_to_inactive(struct drm_obj *obj)
 {
 	struct drm_device	*dev = obj->dev;
 	drm_i915_private_t	*dev_priv = dev->dev_private;
+
+	mtx_enter(&dev_priv->list_lock);
+	drm_lock_obj(obj);
+	/* unlocks list lock and object lock */
+	i915_gem_object_move_to_inactive_locked(obj);
+}
+
+/* called locked */
+void
+i915_gem_object_move_to_inactive_locked(struct drm_obj *obj)
+{
+	struct drm_device	*dev = obj->dev;
+	drm_i915_private_t	*dev_priv = dev->dev_private;
 	struct inteldrm_obj	*obj_priv = (struct inteldrm_obj *)obj;
 
+	MUTEX_ASSERT_LOCKED(&dev_priv->list_lock);
+	DRM_OBJ_ASSERT_LOCKED(obj);
+
 	inteldrm_verify_inactive(dev_priv, __FILE__, __LINE__);
 	if (obj_priv->pin_count != 0)
 		i915_list_remove(obj_priv);
@@ -1298,13 +1334,17 @@ i915_gem_object_move_to_inactive(struct drm_obj *obj)
 		i915_move_to_tail(obj_priv, &dev_priv->mm.inactive_list);
 
 	i915_gem_object_move_off_active(obj);
-	atomic_clearbits_int(&obj_priv->io_flags, I915_FENCED_EXEC);
+	atomic_clearbits_int(&obj->do_flags, I915_FENCED_EXEC);
 
-	KASSERT((obj_priv->io_flags & I915_GPU_WRITE) == 0);
+	KASSERT((obj->do_flags & I915_GPU_WRITE) == 0);
+	/* unlock becauase this unref could recurse */
+	mtx_leave(&dev_priv->list_lock);
 	if (inteldrm_is_active(obj_priv)) {
-		atomic_clearbits_int(&obj_priv->io_flags,
+		atomic_clearbits_int(&obj->do_flags,
 		    I915_ACTIVE);
-		drm_gem_object_unreference(obj);
+		drm_unref_locked(&obj->uobj);
+	} else {
+		drm_unlock_obj(obj);
 	}
 	inteldrm_verify_inactive(dev_priv, __FILE__, __LINE__);
 }
@@ -1312,8 +1352,7 @@ i915_gem_object_move_to_inactive(struct drm_obj *obj)
 void
 inteldrm_purge_obj(struct drm_obj *obj)
 {
-	struct inteldrm_obj *obj_priv = (struct inteldrm_obj *)obj;
-
+	DRM_ASSERT_HELD(obj);
 	/*
 	 * may sleep. We free here instead of deactivate (which
 	 * the madvise() syscall would do) because in this case
@@ -1330,7 +1369,7 @@ inteldrm_purge_obj(struct drm_obj *obj)
 	 * If flush failed, it may have halfway through, so just
 	 * always mark as purged
 	 */
-	atomic_setbits_int(&obj_priv->io_flags, I915_PURGED);
+	atomic_setbits_int(&obj->do_flags, I915_PURGED);
 }
 
 void
@@ -1339,6 +1378,8 @@ inteldrm_process_flushing(struct drm_i915_private *dev_priv,
 {
 	struct inteldrm_obj		*obj_priv, *next;
 
+	MUTEX_ASSERT_LOCKED(&dev_priv->request_lock);
+	mtx_enter(&dev_priv->list_lock);
 	for (obj_priv = TAILQ_FIRST(&dev_priv->mm.gpu_write_list);
 	    obj_priv != TAILQ_END(&dev_priv->mm.gpu_write_list);
 	    obj_priv = next) {
@@ -1346,24 +1387,28 @@ inteldrm_process_flushing(struct drm_i915_private *dev_priv,
 
 		next = TAILQ_NEXT(obj_priv, write_list);
 
-		if ((obj->write_domain & flush_domains) == obj->write_domain) {
+		if ((obj->write_domain & flush_domains)) {
 
 			obj->write_domain = 0;
 			TAILQ_REMOVE(&dev_priv->mm.gpu_write_list,
 			    obj_priv, write_list);
-			atomic_clearbits_int(&obj_priv->io_flags,
+			atomic_clearbits_int(&obj->do_flags,
 			     I915_GPU_WRITE);
 			i915_gem_object_move_to_active(obj);
 			/* if we still need the fence, update LRU */
 			if (inteldrm_needs_fence(obj_priv)) {
 				KASSERT(obj_priv->fence_reg !=
 				    I915_FENCE_REG_NONE);
-				/* we have a fence, won't sleep, can't fail */
+				/* we have a fence, won't sleep, can't fail
+				 * since we have the fence we no not need
+				 * to have the object held
+				 */
 				i915_gem_get_fence_reg(obj, 1);
 			}
 				
 		}
 	}
+	mtx_leave(&dev_priv->list_lock);
 }
 
 /**
@@ -1382,6 +1427,8 @@ i915_add_request(struct drm_i915_private *dev_priv)
 	uint32_t		 seqno;
 	int			 was_empty;
 
+	MUTEX_ASSERT_LOCKED(&dev_priv->request_lock);
+
 	request = drm_calloc(1, sizeof(*request));
 	if (request == NULL) {
 		printf("%s: failed to allocate request\n", __func__);
@@ -1432,9 +1479,10 @@ i915_gem_retire_request(struct drm_i915_private *dev_priv,
 {
 	struct inteldrm_obj	*obj_priv;
 
+	MUTEX_ASSERT_LOCKED(&dev_priv->request_lock);
+	mtx_enter(&dev_priv->list_lock);
 	/* Move any buffers on the active list that are no longer referenced
-	 * by the ringbuffer to the flushing/inactive lists as appropriate.
-	 */
+	 * by the ringbuffer to the flushing/inactive lists as appropriate.  */
 	while ((obj_priv  = TAILQ_FIRST(&dev_priv->mm.active_list)) != NULL) {
 		struct drm_obj *obj = &obj_priv->obj;
 
@@ -1443,8 +1491,9 @@ i915_gem_retire_request(struct drm_i915_private *dev_priv,
 		 * this seqno.
 		 */
 		if (obj_priv->last_rendering_seqno != request->seqno)
-			return;
+			break;
 
+		drm_lock_obj(obj);
 		/*
 		 * If we're now clean and can be read from, move inactive,
 		 * else put on the flushing list to signify that we're not
@@ -1455,10 +1504,14 @@ i915_gem_retire_request(struct drm_i915_private *dev_priv,
 			i915_move_to_tail(obj_priv,
 			    &dev_priv->mm.flushing_list);
 			i915_gem_object_move_off_active(obj);
+			drm_unlock_obj(obj);
 		} else {
-			i915_gem_object_move_to_inactive(obj);
+			/* unlocks object for us and drops ref */
+			i915_gem_object_move_to_inactive_locked(obj);
+			mtx_enter(&dev_priv->list_lock);
 		}
 	}
+	mtx_leave(&dev_priv->list_lock);
 }
 
 /**
@@ -1475,29 +1528,30 @@ i915_gem_retire_requests(struct drm_i915_private *dev_priv)
 
 	seqno = i915_get_gem_seqno(dev_priv);
 
+	mtx_enter(&dev_priv->request_lock);
 	while ((request = TAILQ_FIRST(&dev_priv->mm.request_list)) != NULL) {
 		if (i915_seqno_passed(seqno, request->seqno) ||
 		    dev_priv->mm.wedged) {
+			TAILQ_REMOVE(&dev_priv->mm.request_list, request, list);
 			i915_gem_retire_request(dev_priv, request);
+			mtx_leave(&dev_priv->request_lock);
 
-			TAILQ_REMOVE(&dev_priv->mm.request_list, request, list);
 			drm_free(request);
+			mtx_enter(&dev_priv->request_lock);
 		} else
 			break;
 	}
+	mtx_leave(&dev_priv->request_lock);
 }
 
 void
 i915_gem_retire_work_handler(void *arg1, void *unused)
 {
 	drm_i915_private_t	*dev_priv = arg1;
-	struct drm_device	*dev = (struct drm_device *)dev_priv->drmdev;
 
-	DRM_LOCK();
 	i915_gem_retire_requests(dev_priv);
 	if (!TAILQ_EMPTY(&dev_priv->mm.request_list))
 		timeout_add_sec(&dev_priv->mm.retire_timer, 1);
-	DRM_UNLOCK();
 }
 
 /**
@@ -1512,14 +1566,14 @@ i915_wait_request(struct drm_i915_private *dev_priv, uint32_t seqno,
 {
 	int ret = 0;
 
-	KASSERT(seqno != dev_priv->mm.next_gem_seqno);
-
 	/* Check first because poking a wedged chip is bad. */
 	if (dev_priv->mm.wedged)
 		return (EIO);
 
 	if (seqno == dev_priv->mm.next_gem_seqno) {
+		mtx_enter(&dev_priv->request_lock);
 		seqno = i915_add_request(dev_priv);
+		mtx_leave(&dev_priv->request_lock);
 		if (seqno == 0)
 			return (ENOMEM);
 	}
@@ -1561,65 +1615,68 @@ i915_gem_flush(struct drm_i915_private *dev_priv, uint32_t invalidate_domains,
     uint32_t flush_domains)
 {
 	uint32_t	cmd;
+	int		ret = 0;
 
 	if (flush_domains & I915_GEM_DOMAIN_CPU)
 		inteldrm_chipset_flush(dev_priv);
-	if ((invalidate_domains | flush_domains) & I915_GEM_GPU_DOMAINS) {
-		/*
-		 * read/write caches:
-		 *
-		 * I915_GEM_DOMAIN_RENDER is always invalidated, but is
-		 * only flushed if MI_NO_WRITE_FLUSH is unset.  On 965, it is
-		 * also flushed at 2d versus 3d pipeline switches.
-		 *
-		 * read-only caches:
-		 *
-		 * I915_GEM_DOMAIN_SAMPLER is flushed on pre-965 if
-		 * MI_READ_FLUSH is set, and is always flushed on 965.
-		 *
-		 * I915_GEM_DOMAIN_COMMAND may not exist?
-		 *
-		 * I915_GEM_DOMAIN_INSTRUCTION, which exists on 965, is
-		 * invalidated when MI_EXE_FLUSH is set.
-		 *
-		 * I915_GEM_DOMAIN_VERTEX, which exists on 965, is
-		 * invalidated with every MI_FLUSH.
-		 *
-		 * TLBs:
-		 *
-		 * On 965, TLBs associated with I915_GEM_DOMAIN_COMMAND
-		 * and I915_GEM_DOMAIN_CPU in are invalidated at PTE write and
-		 * I915_GEM_DOMAIN_RENDER and I915_GEM_DOMAIN_SAMPLER
-		 * are flushed at any MI_FLUSH.
-		 */
+	if (((invalidate_domains | flush_domains) & I915_GEM_GPU_DOMAINS) == 0) 
+		return (0);
+	/*
+	 * read/write caches:
+	 *
+	 * I915_GEM_DOMAIN_RENDER is always invalidated, but is
+	 * only flushed if MI_NO_WRITE_FLUSH is unset.  On 965, it is
+	 * also flushed at 2d versus 3d pipeline switches.
+	 *
+	 * read-only caches:
+	 *
+	 * I915_GEM_DOMAIN_SAMPLER is flushed on pre-965 if
+	 * MI_READ_FLUSH is set, and is always flushed on 965.
+	 *
+	 * I915_GEM_DOMAIN_COMMAND may not exist?
+	 *
+	 * I915_GEM_DOMAIN_INSTRUCTION, which exists on 965, is
+	 * invalidated when MI_EXE_FLUSH is set.
+	 *
+	 * I915_GEM_DOMAIN_VERTEX, which exists on 965, is
+	 * invalidated with every MI_FLUSH.
+	 *
+	 * TLBs:
+	 *
+	 * On 965, TLBs associated with I915_GEM_DOMAIN_COMMAND
+	 * and I915_GEM_DOMAIN_CPU in are invalidated at PTE write and
+	 * I915_GEM_DOMAIN_RENDER and I915_GEM_DOMAIN_SAMPLER
+	 * are flushed at any MI_FLUSH.
+	 */
 
-		cmd = MI_FLUSH | MI_NO_WRITE_FLUSH;
-		if ((invalidate_domains | flush_domains) &
-		    I915_GEM_DOMAIN_RENDER)
-			cmd &= ~MI_NO_WRITE_FLUSH;
-		/*
-		 * On the 965, the sampler cache always gets flushed
-		 * and this bit is reserved.
-		 */
-		if (!IS_I965G(dev_priv) &&
-		    invalidate_domains & I915_GEM_DOMAIN_SAMPLER)
-			cmd |= MI_READ_FLUSH;
-		if (invalidate_domains & I915_GEM_DOMAIN_INSTRUCTION)
-			cmd |= MI_EXE_FLUSH;
-
-		BEGIN_LP_RING(2);
-		OUT_RING(cmd);
-		OUT_RING(MI_NOOP);
-		ADVANCE_LP_RING();
-	}
+	cmd = MI_FLUSH | MI_NO_WRITE_FLUSH;
+	if ((invalidate_domains | flush_domains) &
+	    I915_GEM_DOMAIN_RENDER)
+		cmd &= ~MI_NO_WRITE_FLUSH;
+	/*
+	 * On the 965, the sampler cache always gets flushed
+	 * and this bit is reserved.
+	 */
+	if (!IS_I965G(dev_priv) &&
+	    invalidate_domains & I915_GEM_DOMAIN_SAMPLER)
+		cmd |= MI_READ_FLUSH;
+	if (invalidate_domains & I915_GEM_DOMAIN_INSTRUCTION)
+		cmd |= MI_EXE_FLUSH;
+
+	mtx_enter(&dev_priv->request_lock);
+	BEGIN_LP_RING(2);
+	OUT_RING(cmd);
+	OUT_RING(MI_NOOP);
+	ADVANCE_LP_RING();
 
 	/* if this is a gpu flush, process the results */
 	if (flush_domains & I915_GEM_GPU_DOMAINS) {
 		inteldrm_process_flushing(dev_priv, flush_domains);
-		return (i915_add_request(dev_priv));
+		ret = i915_add_request(dev_priv);
 	}
+	mtx_leave(&dev_priv->request_lock);
 
-	return (0);
+	return (ret);
 }
 
 /**
@@ -1635,6 +1692,7 @@ i915_gem_object_unbind(struct drm_obj *obj, int interruptible)
 	struct inteldrm_obj	*obj_priv = (struct inteldrm_obj *)obj;
 	int			 ret = 0;
 
+	DRM_ASSERT_HELD(obj);
 	/*
 	 * if it's already unbound, or we've already done lastclose, just
 	 * let it happen. XXX does this fail to unwire?
@@ -1663,8 +1721,7 @@ i915_gem_object_unbind(struct drm_obj *obj, int interruptible)
 
 	/* if it's purgeable don't bother dirtying the pages */
 	if (i915_obj_purgeable(obj_priv))
-		atomic_clearbits_int(&obj_priv->io_flags, I915_DIRTY);
-
+		atomic_clearbits_int(&obj->do_flags, I915_DIRTY);
 	/*
 	 * unload the map, then unwire the backing object.
 	 */
@@ -1677,7 +1734,7 @@ i915_gem_object_unbind(struct drm_obj *obj, int interruptible)
 	free(obj_priv->dma_segs, M_DRM);
 	obj_priv->dma_segs = NULL;
 	/* XXX this should change whether we tell uvm the page is dirty */
-	atomic_clearbits_int(&obj_priv->io_flags, I915_DIRTY);
+	atomic_clearbits_int(&obj->do_flags, I915_DIRTY);
 
 	obj_priv->gtt_offset = 0;
 	atomic_dec(&dev->gtt_count);
@@ -1699,7 +1756,8 @@ i915_gem_evict_something(struct drm_i915_private *dev_priv, size_t min_size,
 	struct drm_obj		*obj;
 	struct inteldrm_request	*request;
 	struct inteldrm_obj	*obj_priv;
-	int			 ret = 0;
+	u_int32_t		 seqno;
+	int			 ret = 0, write_domain = 0;
 
 	for (;;) {
 		i915_gem_retire_requests(dev_priv);
@@ -1710,43 +1768,56 @@ i915_gem_evict_something(struct drm_i915_private *dev_priv, size_t min_size,
 		obj = i915_gem_find_inactive_object(dev_priv, min_size);
 		if (obj != NULL) {
 			obj_priv = (struct inteldrm_obj *)obj;
-
+			/* find inactive object returns the object with a
+			 * reference for us, and held
+			 */
 			KASSERT(obj_priv->pin_count == 0);
 			KASSERT(!inteldrm_is_active(obj_priv));
+			DRM_ASSERT_HELD(obj);
 
 			/* Wait on the rendering and unbind the buffer. */
-			return (i915_gem_object_unbind(obj, interruptible));
+			ret = i915_gem_object_unbind(obj, interruptible);
+			drm_unhold_and_unref(obj);
+			return (ret);
 		}
 
 		/* If we didn't get anything, but the ring is still processing
 		 * things, wait for one of those things to finish and hopefully
 		 * leave us a buffer to evict.
 		 */
+		mtx_enter(&dev_priv->request_lock);
 		if ((request = TAILQ_FIRST(&dev_priv->mm.request_list))
 		    != NULL) {
-			ret = i915_wait_request(dev_priv, request->seqno,
-			    interruptible);
+			seqno = request->seqno;
+			mtx_leave(&dev_priv->request_lock);
+
+			ret = i915_wait_request(dev_priv, seqno, interruptible);
 			if (ret)
 				return (ret);
 
 			continue;
 		}
+		mtx_leave(&dev_priv->request_lock);
 
 		/* If we didn't have anything on the request list but there
 		 * are buffers awaiting a flush, emit one and try again.
 		 * When we wait on it, those buffers waiting for that flush
 		 * will get moved to inactive.
 		 */
+		mtx_enter(&dev_priv->list_lock);
 		TAILQ_FOREACH(obj_priv, &dev_priv->mm.flushing_list, list) {
 			obj = &obj_priv->obj;
-			if (obj->size >= min_size)
+			if (obj->size >= min_size) {
+				write_domain = obj->write_domain;
 				break;
+			}
 			obj = NULL;
 		}
+		mtx_leave(&dev_priv->list_lock);
 
-		if (obj != NULL) {
-			if (i915_gem_flush(dev_priv, obj->write_domain,
-			    obj->write_domain) == 0)
+		if (write_domain) {
+			if (i915_gem_flush(dev_priv, write_domain,
+			    write_domain) == 0)
 				return (ENOMEM);
 			continue;
 		}
@@ -1772,6 +1843,11 @@ i915_gem_find_inactive_object(struct drm_i915_private *dev_priv,
 	struct drm_obj		*obj, *best = NULL, *first = NULL;
 	struct inteldrm_obj	*obj_priv;
 
+	/*
+	 * We don't need references to the object as long as we hold the list
+	 * lock, they won't disappear until we release the lock.
+	 */
+	mtx_enter(&dev_priv->list_lock);
 	TAILQ_FOREACH(obj_priv, &dev_priv->mm.inactive_list, list) {
 		obj = &obj_priv->obj;
 		if (obj->size >= min_size) {
@@ -1780,14 +1856,27 @@ i915_gem_find_inactive_object(struct drm_i915_private *dev_priv,
 			    (best == NULL || obj->size < best->size)) {
 				best = obj;
 				if (best->size == min_size)
-					return (best);
+					break;
 			}
 		}
 		if (first == NULL)
 			first = obj;
 	}
-
-	return ((best != NULL) ? best : first);
+	if (best == NULL)
+		best = first;
+	if (best) {
+		drm_ref(&best->uobj);
+		/*
+		 * if we couldn't grab it, we may as well fail and go
+		 * onto the next step for the sake of simplicity.
+		 */
+		if (drm_try_hold_object(best) == 0) {
+			drm_unref(&best->uobj);
+			best = NULL;
+		}
+	}
+	mtx_leave(&dev_priv->list_lock);
+	return (best);
 }
 
 int
@@ -1980,13 +2069,16 @@ i915_gem_get_fence_reg(struct drm_obj *obj, int interruptible)
 
 	/* If our fence is getting used, just update our place in the LRU */
 	if (obj_priv->fence_reg != I915_FENCE_REG_NONE) {
+		mtx_enter(&dev_priv->fence_lock);
 		reg = &dev_priv->fence_regs[obj_priv->fence_reg];
 
 		TAILQ_REMOVE(&dev_priv->mm.fence_list, reg, list);
 		TAILQ_INSERT_TAIL(&dev_priv->mm.fence_list, reg, list);
+		mtx_leave(&dev_priv->fence_lock);
 		return (0);
 	}
 
+	DRM_ASSERT_HELD(obj);
 	switch (obj_priv->tiling_mode) {
 	case I915_TILING_NONE:
 		DRM_ERROR("allocating a fence for non-tiled object?\n");
@@ -2007,8 +2099,10 @@ i915_gem_get_fence_reg(struct drm_obj *obj, int interruptible)
 		break;
 	}
 
+again:
 	/* First try to find a free reg */
 	avail = 0;
+	mtx_enter(&dev_priv->fence_lock);
 	for (i = dev_priv->fence_reg_start; i < dev_priv->num_fence_regs; i++) {
 		reg = &dev_priv->fence_regs[i];
 		if (reg->obj == NULL)
@@ -2021,8 +2115,10 @@ i915_gem_get_fence_reg(struct drm_obj *obj, int interruptible)
 
 	/* None available, try to steal one or wait for a user to finish */
 	if (i == dev_priv->num_fence_regs) {
-		if (avail == 0)
+		if (avail == 0) {
+			mtx_leave(&dev_priv->fence_lock);
 			return (ENOMEM);
+		}
 
 		TAILQ_FOREACH(reg, &dev_priv->mm.fence_list,
 		    list) {
@@ -2033,20 +2129,31 @@ i915_gem_get_fence_reg(struct drm_obj *obj, int interruptible)
 				continue;
 
 			/* Ref it so that wait_rendering doesn't free it under
-			 * us.
+			 * us. if we can't hold it, it may change state soon
+			 * so grab the next one.
 			 */
-			drm_gem_object_reference(old_obj);
+			drm_ref(&old_obj->uobj);
+			if (drm_try_hold_object(old_obj) == 0) {
+				drm_unref(&old_obj->uobj);
+				continue;
+			}
 
 			break;
 		}
+		mtx_leave(&dev_priv->fence_lock);
 
-		i = old_obj_priv->fence_reg;
-		reg = &dev_priv->fence_regs[i];
+		/* if we tried all of them, give it another whirl. we failed to
+		 * get a hold this go round.
+		 */
+		if (reg == NULL)
+			goto again;
 
 		ret = i915_gem_object_put_fence_reg(old_obj, interruptible);
-		drm_gem_object_unreference(old_obj);
+		drm_unhold_and_unref(old_obj);
 		if (ret != 0)
 			return (ret);
+		/* we should have freed one up now, so relock and re-search */
+		goto again;
 	}
 
 	obj_priv->fence_reg = i;
@@ -2059,6 +2166,7 @@ i915_gem_get_fence_reg(struct drm_obj *obj, int interruptible)
 		i915_write_fence_reg(reg);
 	else
 		i830_write_fence_reg(reg);
+	mtx_leave(&dev_priv->fence_lock);
 
 	return 0;
 }
@@ -2072,6 +2180,7 @@ i915_gem_object_put_fence_reg(struct drm_obj *obj, int interruptible)
 	struct inteldrm_fence	*reg;
 	int			 ret;
 
+	DRM_ASSERT_HELD(obj);
 	if (obj_priv->fence_reg == I915_FENCE_REG_NONE)
 		return (0);
 
@@ -2102,6 +2211,7 @@ i915_gem_object_put_fence_reg(struct drm_obj *obj, int interruptible)
 			obj->write_domain = 0;
 	}
 
+	mtx_enter(&dev_priv->fence_lock);
 	if (IS_I965G(dev_priv)) {
 		I915_WRITE64(FENCE_REG_965_0 + (obj_priv->fence_reg * 8), 0);
 	} else {
@@ -2118,7 +2228,8 @@ i915_gem_object_put_fence_reg(struct drm_obj *obj, int interruptible)
 	reg->obj = NULL;
 	TAILQ_REMOVE(&dev_priv->mm.fence_list, reg, list);
 	obj_priv->fence_reg = I915_FENCE_REG_NONE;
-	atomic_clearbits_int(&obj_priv->io_flags, I915_FENCE_INVALID);
+	mtx_leave(&dev_priv->fence_lock);
+	atomic_clearbits_int(&obj->do_flags, I915_FENCE_INVALID);
 
 	return (0);
 }
@@ -2134,12 +2245,27 @@ inteldrm_fault(struct drm_obj *obj, struct uvm_faultinfo *ufi, off_t offset,
 	int			 lcv, ret;
 	int			 write = !!(access_type & VM_PROT_WRITE);
 	vm_prot_t		 mapprot;
+	boolean_t		 locked = TRUE;
 
-	DRM_LOCK();
-	/*
-	 * XXX this locking is wrong, must be fixed. uvm using simple_locks
-	 * saves us for now.
+	if (rw_enter(&dev->dev_lock, RW_NOSLEEP | RW_READ) != 0) {
+		uvmfault_unlockall(ufi, NULL, &obj->uobj, NULL);
+		DRM_READLOCK();
+		locked = uvmfault_relock(ufi);
+		if (locked)
+			drm_lock_obj(obj);
+	}
+	if (locked)
+		drm_hold_object_locked(obj);
+	else { /* obj already unlocked */
+		return (VM_PAGER_REFAULT);
+	}
+
+	/* we have a hold set on the object now, we can unlock so that we can
+	 * sleep in binding and flushing.
 	 */
+	drm_unlock_obj(obj);
+	
+
 	if (obj_priv->dmamap != NULL &&
 	    (obj_priv->gtt_offset & (i915_gem_get_gtt_alignment(obj) - 1) ||
 	    (!i915_gem_object_fence_offset_ok(obj, obj_priv->tiling_mode)))) {
@@ -2200,20 +2326,29 @@ inteldrm_fault(struct drm_obj *obj, struct uvm_faultinfo *ufi, off_t offset,
 		/* XXX writecombining */
 		if (pmap_enter(ufi->orig_map->pmap, vaddr, paddr | PMAP_NOCACHE,
 		    mapprot, PMAP_CANFAIL | mapprot) != 0) {
-			DRM_UNLOCK();
-			printf("%s: enter failed\n", __func__);
+			drm_unhold_object(obj);
+			uvmfault_unlockall(ufi, ufi->entry->aref.ar_amap,
+			    NULL, NULL);
+			DRM_READUNLOCK();
+			uvm_wait("intelflt");
 			return (VM_PAGER_REFAULT);
 		}
 	}
-	DRM_UNLOCK();
+	drm_unhold_object(obj);
+	uvmfault_unlockall(ufi, ufi->entry->aref.ar_amap, NULL, NULL);
+	DRM_READUNLOCK();
+	pmap_update(ufi->orig_map->pmap);
 	return (VM_PAGER_OK);
 
 error:
 	/*
 	 * EIO means we're wedged so when we reset the gpu this will
-	 * work, so don't segfault.
+	 * work, so don't segfault. XXX only on resettable chips
 	 */
-	DRM_UNLOCK();
+	drm_unhold_object(obj);
+	uvmfault_unlockall(ufi, ufi->entry->aref.ar_amap, NULL, NULL);
+	DRM_READUNLOCK();
+	pmap_update(ufi->orig_map->pmap);
 	return ((ret == EIO) ?  VM_PAGER_REFAULT : VM_PAGER_ERROR);
 		
 }
@@ -2226,6 +2361,7 @@ inteldrm_wipe_mappings(struct drm_obj *obj)
 	struct drm_i915_private	*dev_priv = dev->dev_private;
 	struct vm_page		*pg;
 
+	DRM_ASSERT_HELD(obj);
 	/* make sure any writes hit the bus before we do whatever change
 	 * that prompted us to kill the mappings.
 	 */
@@ -2248,6 +2384,7 @@ i915_gem_object_bind_to_gtt(struct drm_obj *obj, bus_size_t alignment,
 	struct inteldrm_obj	*obj_priv = (struct inteldrm_obj *)obj;
 	int			 ret;
 
+	DRM_ASSERT_HELD(obj);
 	if (dev_priv->agpdmat == NULL)
 		return (EINVAL);
 	if (alignment == 0) {
@@ -2335,6 +2472,7 @@ i915_gem_object_flush_gpu_write_domain(struct drm_obj *obj, int pipelined,
 	struct inteldrm_obj	*obj_priv = (struct inteldrm_obj *)obj;
 	int			 ret = 0;
 
+	DRM_ASSERT_HELD(obj);
 	if ((obj->write_domain & I915_GEM_GPU_DOMAINS) != 0) {
 		/*
 		 * Queue the GPU write cache flushing we need.
@@ -2368,6 +2506,7 @@ i915_gem_object_set_to_gtt_domain(struct drm_obj *obj, int write,
 	struct inteldrm_obj	*obj_priv = (struct inteldrm_obj *)obj;
 	int			 ret;
 
+	DRM_ASSERT_HELD(obj);
 	/* Not valid to be called on unbound objects. */
 	if (obj_priv->dmamap == NULL)
 		return (EINVAL);
@@ -2387,7 +2526,7 @@ i915_gem_object_set_to_gtt_domain(struct drm_obj *obj, int write,
 	/* We're accessing through the gpu, so grab a new fence register or
 	 * update the LRU.
 	 */
-	if (obj_priv->io_flags & I915_FENCE_INVALID) {
+	if (obj->do_flags & I915_FENCE_INVALID) {
 		ret = i915_gem_object_put_fence_reg(obj, interruptible);
 		if (ret)
 			return (ret);
@@ -2404,7 +2543,7 @@ i915_gem_object_set_to_gtt_domain(struct drm_obj *obj, int write,
 	KASSERT((obj->write_domain & ~I915_GEM_DOMAIN_GTT) == 0);
 	if (write) {
 		obj->read_domains = obj->write_domain = I915_GEM_DOMAIN_GTT;
-		atomic_setbits_int(&obj_priv->io_flags, I915_DIRTY);
+		atomic_setbits_int(&obj->do_flags, I915_DIRTY);
 	} else {
 		obj->read_domains |= I915_GEM_DOMAIN_GTT;
 	}
@@ -2427,6 +2566,7 @@ i915_gem_object_set_to_cpu_domain(struct drm_obj *obj, int write,
 	struct inteldrm_obj	*obj_priv = (struct inteldrm_obj *)obj;
 	int			 ret;
 
+	DRM_ASSERT_HELD(obj);
 	/* Wait on any GPU rendering and flushing to occur. */
 	if ((ret = i915_gem_object_flush_gpu_write_domain(obj, 0,
 	    interruptible)) != 0)
@@ -2593,6 +2733,7 @@ i915_gem_object_set_to_gpu_domain(struct drm_obj *obj)
 	u_int32_t		 invalidate_domains = 0;
 	u_int32_t		 flush_domains = 0;
 
+	DRM_ASSERT_HELD(obj);
 	KASSERT((obj->pending_read_domains & I915_GEM_DOMAIN_CPU) == 0);
 	KASSERT(obj->pending_write_domain != I915_GEM_DOMAIN_CPU);
 	/*
@@ -2602,7 +2743,7 @@ i915_gem_object_set_to_gpu_domain(struct drm_obj *obj)
 	if (obj->pending_write_domain == 0)
 		obj->pending_read_domains |= obj->read_domains;
 	else
-		atomic_setbits_int(&obj_priv->io_flags, I915_DIRTY);
+		atomic_setbits_int(&obj->do_flags, I915_DIRTY);
 
 	/*
 	 * Flush the current write domain if
@@ -2663,15 +2804,15 @@ i915_gem_object_pin_and_relocate(struct drm_obj *obj,
 	bus_space_handle_t			 bsh;
 	int					 i, ret, needs_fence;
 
+	DRM_ASSERT_HELD(obj);
 	needs_fence = (entry->flags & EXEC_OBJECT_NEEDS_FENCE) &&
 	    obj_priv->tiling_mode != I915_TILING_NONE;
 	if (needs_fence)
-		atomic_setbits_int(&obj_priv->io_flags, I915_EXEC_NEEDS_FENCE);
+		atomic_setbits_int(&obj->do_flags, I915_EXEC_NEEDS_FENCE);
 
 	/* Choose the GTT offset for our buffer and put it there. */
 	ret = i915_gem_object_pin(obj, (u_int32_t)entry->alignment,
 	    needs_fence);
-	/* XXX what if already bound at a different alignment? */
 	if (ret)
 		return ret;
 
@@ -2687,10 +2828,18 @@ i915_gem_object_pin_and_relocate(struct drm_obj *obj,
 
 		target_obj = drm_gem_object_lookup(obj->dev, file_priv,
 		    reloc->target_handle);
+		/* object must have come before us in the list */
 		if (target_obj == NULL) {
 			i915_gem_object_unpin(obj);
 			return (EBADF);
 		}
+		if ((target_obj->do_flags & I915_IN_EXEC) == 0) {
+			printf("%s: object not already in execbuffer\n",
+			__func__);
+			ret = EBADF;
+			goto err;
+		}
+			
 		target_obj_priv = (struct inteldrm_obj *)target_obj;
 
 		/* The target buffer should have appeared before us in the
@@ -2820,6 +2969,7 @@ i915_dispatch_gem_execbuffer(struct drm_device *dev,
 	drm_i915_private_t	*dev_priv = dev->dev_private;
 	uint32_t		 exec_start, exec_len;
 
+	MUTEX_ASSERT_LOCKED(&dev_priv->request_lock);
 	exec_start = (uint32_t)exec_offset + exec->batch_start_offset;
 	exec_len = (uint32_t)exec->batch_len;
 
@@ -2853,11 +3003,19 @@ i915_dispatch_gem_execbuffer(struct drm_device *dev,
 	OUT_RING(MI_FLUSH | MI_NO_WRITE_FLUSH);
 	OUT_RING(MI_NOOP);
 	ADVANCE_LP_RING();
+	/*
+	 * move to active associated all previous buffers with the seqno
+	 * that this call will emit. so we don't need the return. If it fails
+	 * then the next seqno will take care of it.
+	 */
+	(void)i915_add_request(dev_priv);
 
 	inteldrm_verify_inactive(dev_priv, __FILE__, __LINE__);
+#if 0
 	/* The sampler always gets flushed on i965 (sigh) */
 	if (IS_I965G(dev_priv))
 		inteldrm_process_flushing(dev_priv, I915_GEM_DOMAIN_SAMPLER);
+#endif
 }
 
 /* Throttle our rendering by waiting until the ring has completed our requests
@@ -3004,20 +3162,20 @@ i915_gem_execbuffer2(struct drm_device *dev, void *data,
 	for (i = 0; i < args->buffer_count; i++) {
 		object_list[i] = drm_gem_object_lookup(dev, file_priv,
 		    exec_list[i].handle);
-		if (object_list[i] == NULL) {
+		obj = object_list[i];
+		if (obj == NULL) {
 			DRM_ERROR("Invalid object handle %d at index %d\n",
 				   exec_list[i].handle, i);
 			ret = EBADF;
 			goto err;
 		}
-		obj_priv = (struct inteldrm_obj *)object_list[i];
-		if (obj_priv->io_flags & I915_IN_EXEC) {
+		if (obj->do_flags & I915_IN_EXEC) {
 			DRM_ERROR("Object %p appears more than once in object_list\n",
 			    object_list[i]);
 			ret = EBADF;
 			goto err;
 		}
-		atomic_setbits_int(&obj_priv->io_flags, I915_IN_EXEC);
+		atomic_setbits_int(&obj->do_flags, I915_IN_EXEC);
 	}
 
 	/* Pin and relocate */
@@ -3028,10 +3186,13 @@ i915_gem_execbuffer2(struct drm_device *dev, void *data,
 		for (i = 0; i < args->buffer_count; i++) {
 			object_list[i]->pending_read_domains = 0;
 			object_list[i]->pending_write_domain = 0;
+			drm_hold_object(object_list[i]);
 			ret = i915_gem_object_pin_and_relocate(object_list[i],
 			    file_priv, &exec_list[i], &relocs[reloc_index]);
-			if (ret)
+			if (ret) {
+				drm_unhold_object(object_list[i]);
 				break;
+			}
 			pinned++;
 			reloc_index += exec_list[i].relocation_count;
 		}
@@ -3043,16 +3204,27 @@ i915_gem_execbuffer2(struct drm_device *dev, void *data,
 		if (ret != ENOSPC || pin_tries >= 1)
 			goto err;
 
-		/* unpin all of our buffers */
-		for (i = 0; i < pinned; i++)
+		/*
+		 * unpin all of our buffers and unhold them so they can be
+		 * unbound so we can try and refit everything in the aperture.
+		 */
+		for (i = 0; i < pinned; i++) {
 			i915_gem_object_unpin(object_list[i]);
+			drm_unhold_object(object_list[i]);
+		}
+		pinned = 0;
 		/* evict everyone we can from the aperture */
 		ret = i915_gem_evict_everything(dev_priv, 1);
 		if (ret)
 			goto err;
 	}
 
-	/* Set the pending read domains for the batch buffer to COMMAND */
+	/* If we get here all involved objects are referenced, pinned, relocated
+	 * and held. Now we can finish off the exec processing.
+	 *
+	 * First, set the pending read domains for the batch buffer to
+	 * command.
+	 */
 	batch_obj = object_list[args->buffer_count - 1];
 	batch_obj_priv = (struct inteldrm_obj *)batch_obj;
 	if (args->batch_start_offset + args->batch_len > batch_obj->size ||
@@ -3089,24 +3261,27 @@ i915_gem_execbuffer2(struct drm_device *dev, void *data,
 	 * wait), which will waste some time, but if we're that low on memory
 	 * then we could fail in much worse ways.
 	 */
+	mtx_enter(&dev_priv->request_lock); /* to prevent races on next_seqno */
+	mtx_enter(&dev_priv->list_lock);
 	for (i = 0; i < args->buffer_count; i++) {
 		obj = object_list[i];
 		obj_priv = (struct inteldrm_obj *)obj;
+		drm_lock_obj(obj);
 
 		obj->write_domain = obj->pending_write_domain;
 		/*
 		 * if we have a write domain, add us to the gpu write list
 		 * else we can remove the bit because it has been flushed.
 		 */
-		if (obj_priv->io_flags & I915_GPU_WRITE)
+		if (obj->do_flags & I915_GPU_WRITE)
 			TAILQ_REMOVE(&dev_priv->mm.gpu_write_list, obj_priv,
 			     write_list);
 		if (obj->write_domain) {
 			TAILQ_INSERT_TAIL(&dev_priv->mm.gpu_write_list,
 			    obj_priv, write_list);
-			atomic_setbits_int(&obj_priv->io_flags, I915_GPU_WRITE);
+			atomic_setbits_int(&obj->do_flags, I915_GPU_WRITE);
 		} else {
-			atomic_clearbits_int(&obj_priv->io_flags,
+			atomic_clearbits_int(&obj->do_flags,
 			     I915_GPU_WRITE);
 		}
 		/* if this batchbuffer needs a fence, then the object is
@@ -3114,15 +3289,17 @@ i915_gem_execbuffer2(struct drm_device *dev, void *data,
 		 * will just wait on the fence last_seqno.
 		 */
 		if (inteldrm_exec_needs_fence(obj_priv)) {
-			atomic_setbits_int(&obj_priv->io_flags,
+			atomic_setbits_int(&obj->do_flags,
 			    I915_FENCED_EXEC);
 		} else {
-			atomic_clearbits_int(&obj_priv->io_flags,
+			atomic_clearbits_int(&obj->do_flags,
 			    I915_FENCED_EXEC);
 		}
 			
 		i915_gem_object_move_to_active(object_list[i]);
+		drm_unlock_obj(obj);
 	}
+	mtx_leave(&dev_priv->list_lock);
 
 	inteldrm_verify_inactive(dev_priv, __FILE__, __LINE__);
 
@@ -3131,12 +3308,7 @@ i915_gem_execbuffer2(struct drm_device *dev, void *data,
 	 * XXX make sure that this may never fail by preallocating the request.
 	 */
 	i915_dispatch_gem_execbuffer(dev, args, batch_obj_priv->gtt_offset);
-
-	/*
-	 * move to active associated all previous buffers with the seqno
-	 * that this call will emit. so we don't need the return.
-	 */
-	(void)i915_add_request(dev_priv);
+	mtx_leave(&dev_priv->request_lock);
 
 	inteldrm_verify_inactive(dev_priv, __FILE__, __LINE__);
 
@@ -3148,13 +3320,14 @@ err:
 		if (object_list[i] == NULL)
 			break;
 
-		obj_priv = (struct inteldrm_obj *)object_list[i];
-		if (i < pinned)
-			i915_gem_object_unpin(object_list[i]);
-
-		atomic_clearbits_int(&obj_priv->io_flags, I915_IN_EXEC |
+		atomic_clearbits_int(&object_list[i]->do_flags, I915_IN_EXEC |
 		    I915_EXEC_NEEDS_FENCE);
-		drm_gem_object_unreference(object_list[i]);
+		if (i < pinned) {
+			i915_gem_object_unpin(object_list[i]);
+			drm_unhold_and_unref(object_list[i]);
+		} else {
+			drm_unref(&object_list[i]->uobj);
+		}
 	}
 
 unlock:
@@ -3180,6 +3353,7 @@ i915_gem_object_pin(struct drm_obj *obj, uint32_t alignment, int needs_fence)
 	struct inteldrm_obj	*obj_priv = (struct inteldrm_obj *)obj;
 	int			 ret;
 
+	DRM_ASSERT_HELD(obj);
 	inteldrm_verify_inactive(dev_priv, __FILE__, __LINE__);
 	/*
 	 * if already bound, but alignment is unsuitable, unbind so we can
@@ -3214,7 +3388,7 @@ i915_gem_object_pin(struct drm_obj *obj, uint32_t alignment, int needs_fence)
 	 * it.
 	 */
 	/* if we need a fence now, check that the one we may have is correct */
-	if (needs_fence && obj_priv->io_flags & I915_FENCE_INVALID) {
+	if (needs_fence && obj->do_flags & I915_FENCE_INVALID) {
 		ret= i915_gem_object_put_fence_reg(obj, 1);
 		if (ret)
 			return (ret);
@@ -3246,6 +3420,7 @@ i915_gem_object_unpin(struct drm_obj *obj)
 	inteldrm_verify_inactive(dev_priv, __FILE__, __LINE__);
 	KASSERT(obj_priv->pin_count >= 1);
 	KASSERT(obj_priv->dmamap != NULL);
+	DRM_ASSERT_HELD(obj);
 
 	/* If the object is no longer pinned, and is
 	 * neither active nor being flushed, then stick it on
@@ -3272,11 +3447,12 @@ i915_gem_pin_ioctl(struct drm_device *dev, void *data,
 	obj = drm_gem_object_lookup(dev, file_priv, args->handle);
 	if (obj == NULL)
 		return (EBADF);
+	DRM_LOCK();
+	drm_hold_object(obj);
 
 	obj_priv = (struct inteldrm_obj *)obj;
-	DRM_LOCK();
 	if (i915_obj_purgeable(obj_priv)) {
-		printf("pinning purgeable object\n");
+		printf("%s: pinning purgeable object\n", __func__);
 		ret = EINVAL;
 		goto out;
 	}
@@ -3294,7 +3470,7 @@ i915_gem_pin_ioctl(struct drm_device *dev, void *data,
 	args->offset = obj_priv->gtt_offset;
 
 out:
-	drm_gem_object_unreference(obj);
+	drm_unhold_and_unref(obj);
 	DRM_UNLOCK();
 
 	return (ret);
@@ -3307,24 +3483,28 @@ i915_gem_unpin_ioctl(struct drm_device *dev, void *data,
 	struct drm_i915_gem_pin	*args = data;
 	struct inteldrm_obj	*obj_priv;
 	struct drm_obj		*obj;
+	int			 ret = 0;
 
 	obj = drm_gem_object_lookup(dev, file_priv, args->handle);
 	if (obj == NULL)
 		return (EBADF);
 
 	DRM_LOCK();
+	drm_hold_object(obj);
+
 	obj_priv = (struct inteldrm_obj *)obj;
 	if (obj_priv->user_pin_count == 0) {
-		DRM_UNLOCK();
-		return (EINVAL);
+		ret = EINVAL;
+		goto out;
 	}
 
 	if (--obj_priv->user_pin_count == 0)
 		i915_gem_object_unpin(obj);
 
-	drm_gem_object_unreference(obj);
+out:
+	drm_unhold_and_unref(obj);
 	DRM_UNLOCK();
-	return (0);
+	return (ret);
 }
 
 int
@@ -3343,7 +3523,6 @@ i915_gem_busy_ioctl(struct drm_device *dev, void *data,
 		return (EBADF);
 	}
 
-	DRM_LOCK();
 	/*
 	 * Update the active list for the hardware's current position.
 	 * otherwise this will only update on a delayed timer or when
@@ -3363,8 +3542,7 @@ i915_gem_busy_ioctl(struct drm_device *dev, void *data,
 	args->busy = inteldrm_is_active(obj_priv) &&
 	    obj_priv->last_rendering_seqno != 0;
 
-	drm_gem_object_unreference(obj);
-	DRM_UNLOCK();
+	drm_unref(&obj->uobj);
 	return 0;
 }
 
@@ -3392,8 +3570,8 @@ i915_gem_madvise_ioctl(struct drm_device *dev, void *data,
 	if (obj == NULL)
 		return (EBADF);
 
+	drm_hold_object(obj);
 	obj_priv = (struct inteldrm_obj *)obj;
-	DRM_LOCK();
 
 	/* invalid to madvise on a pinned BO */
 	if (obj_priv->pin_count) {
@@ -3403,10 +3581,10 @@ i915_gem_madvise_ioctl(struct drm_device *dev, void *data,
 
 	if (!i915_obj_purged(obj_priv)) {
 		if (need) {
-			atomic_clearbits_int(&obj_priv->io_flags,
+			atomic_clearbits_int(&obj->do_flags,
 			    I915_DONTNEED);
 		} else {
-			atomic_setbits_int(&obj_priv->io_flags, I915_DONTNEED);
+			atomic_setbits_int(&obj->do_flags, I915_DONTNEED);
 		}
 	}
 
@@ -3418,8 +3596,7 @@ i915_gem_madvise_ioctl(struct drm_device *dev, void *data,
 	args->retained = !i915_obj_purged(obj_priv);
 
 out:
-	drm_gem_object_unreference(obj);
-	DRM_UNLOCK();
+	drm_unhold_and_unref(obj);
 
 	return (ret);
 }
@@ -3454,6 +3631,7 @@ i915_gem_free_object(struct drm_obj *obj)
 {
 	struct inteldrm_obj	*obj_priv = (struct inteldrm_obj *)obj;
 
+	DRM_ASSERT_HELD(obj);
 	while (obj_priv->pin_count > 0)
 		i915_gem_object_unpin(obj);
 
@@ -3468,15 +3646,26 @@ i915_gem_evict_inactive(struct drm_i915_private *dev_priv)
 	struct inteldrm_obj	*obj_priv;
 	int			 ret = 0;
 
+	mtx_enter(&dev_priv->list_lock);
 	while ((obj_priv = TAILQ_FIRST(&dev_priv->mm.inactive_list)) != NULL) {
 		if (obj_priv->pin_count != 0) {
+			ret = EINVAL;
 			DRM_ERROR("Pinned object in unbind list\n");
-			return (EINVAL);
+			break;
 		}
+		/* reference it so that we can frob it outside the lock */
+		drm_ref(&obj_priv->obj.uobj);
+		mtx_leave(&dev_priv->list_lock);
 
-		if ((ret = i915_gem_object_unbind(&obj_priv->obj, 1)) != 0)
+		drm_hold_object(&obj_priv->obj);
+		ret = i915_gem_object_unbind(&obj_priv->obj, 1);
+		drm_unhold_and_unref(&obj_priv->obj);
+
+		mtx_enter(&dev_priv->list_lock);
+		if (ret)
 			break;
 	}
+	mtx_leave(&dev_priv->list_lock);
 
 	return (ret);
 }
@@ -3539,6 +3728,7 @@ i915_gem_init_hws(struct drm_i915_private *dev_priv)
 		return (ENOMEM);
 	}
 	obj_priv = (struct inteldrm_obj *)obj;
+	drm_hold_object(obj);
 	/*
 	 * snooped gtt mapping please .
 	 * Normally this flag is only to dmamem_map, but it's been overloaded
@@ -3548,7 +3738,7 @@ i915_gem_init_hws(struct drm_i915_private *dev_priv)
 
 	ret = i915_gem_object_pin(obj, 4096, 0);
 	if (ret != 0) {
-		drm_gem_object_unreference(obj);
+		drm_unhold_and_unref(obj);
 		return ret;
 	}
 
@@ -3562,9 +3752,10 @@ i915_gem_init_hws(struct drm_i915_private *dev_priv)
 		obj->uao->pgops->pgo_detach(obj->uao);
 		memset(&dev_priv->hws_map, 0, sizeof(dev_priv->hws_map));
 		i915_gem_object_unpin(obj);
-		drm_gem_object_unreference(obj);
+		drm_unhold_and_unref(obj);
 		return (EINVAL);
 	}
+	drm_unhold_object(obj);
 	dev_priv->hws_obj = obj;
 	memset(dev_priv->hw_status_page, 0, PAGE_SIZE);
 	I915_WRITE(HWS_PGA, obj_priv->gtt_offset);
@@ -3586,8 +3777,9 @@ i915_gem_cleanup_hws(struct drm_i915_private *dev_priv)
 
 	uvm_unmap(kernel_map, (vaddr_t)dev_priv->hw_status_page,
 	    (vaddr_t)dev_priv->hw_status_page + PAGE_SIZE);
+	drm_hold_object(obj);
 	i915_gem_object_unpin(obj);
-	drm_gem_object_unreference(obj);
+	drm_unhold_and_unref(obj);
 	dev_priv->hws_obj = NULL;
 
 	memset(&dev_priv->hws_map, 0, sizeof(dev_priv->hws_map));
@@ -3615,6 +3807,7 @@ i915_gem_init_ringbuffer(struct drm_i915_private *dev_priv)
 		ret = ENOMEM;
 		goto delhws;
 	}
+	drm_hold_object(obj);
 	obj_priv = (struct inteldrm_obj *)obj;
 
 	ret = i915_gem_object_pin(obj, 4096, 0);
@@ -3635,6 +3828,7 @@ i915_gem_init_ringbuffer(struct drm_i915_private *dev_priv)
 	if ((ret = inteldrm_start_ring(dev_priv)) != 0)
 		goto unmap;
 
+	drm_unhold_object(obj);
 	return (0);
 
 unmap:
@@ -3642,7 +3836,7 @@ unpin:
 	memset(&dev_priv->ring, 0, sizeof(dev_priv->ring));
 	i915_gem_object_unpin(obj);
 unref:
-	drm_gem_object_unreference(obj);
+	drm_unhold_and_unref(obj);
 delhws:
 	i915_gem_cleanup_hws(dev_priv);
 	return (ret);
@@ -3701,9 +3895,9 @@ i915_gem_cleanup_ringbuffer(struct drm_i915_private *dev_priv)
 {
 	if (dev_priv->ring.ring_obj == NULL)
 		return;
-
+	drm_hold_object(dev_priv->ring.ring_obj);
 	i915_gem_object_unpin(dev_priv->ring.ring_obj);
-	drm_gem_object_unreference(dev_priv->ring.ring_obj);
+	drm_unhold_and_unref(dev_priv->ring.ring_obj);
 	dev_priv->ring.ring_obj = NULL;
 	memset(&dev_priv->ring, 0, sizeof(dev_priv->ring));
 
@@ -3919,27 +4113,35 @@ inteldrm_hung(void *arg, void *reset_type)
 	 * flushed or completed otherwise. nuke the domains since
 	 * they're now irrelavent.
 	 */
+	mtx_enter(&dev_priv->list_lock);
 	while ((obj_priv = TAILQ_FIRST(&dev_priv->mm.active_list)) != NULL) {
+		drm_lock_obj(&obj_priv->obj);
 		if (obj_priv->obj.write_domain & I915_GEM_GPU_DOMAINS) {
 			TAILQ_REMOVE(&dev_priv->mm.gpu_write_list,
 			    obj_priv, write_list);
-			atomic_clearbits_int(&obj_priv->io_flags,
+			atomic_clearbits_int(&obj_priv->obj.do_flags,
 			     I915_GPU_WRITE);
 			obj_priv->obj.write_domain &= ~I915_GEM_GPU_DOMAINS;
 		}
-		i915_gem_object_move_to_inactive(&obj_priv->obj);;
+		/* unlocks object and list */
+		i915_gem_object_move_to_inactive_locked(&obj_priv->obj);;
+		mtx_enter(&dev_priv->list_lock);
 	}
 
 	while ((obj_priv = TAILQ_FIRST(&dev_priv->mm.flushing_list)) != NULL) {
+		drm_lock_obj(&obj_priv->obj);
 		if (obj_priv->obj.write_domain & I915_GEM_GPU_DOMAINS) {
 			TAILQ_REMOVE(&dev_priv->mm.gpu_write_list,
 			    obj_priv, write_list);
-			atomic_clearbits_int(&obj_priv->io_flags,
-			     I915_GPU_WRITE);
+			atomic_clearbits_int(&obj_priv->obj.do_flags,
+			    I915_GPU_WRITE);
 			obj_priv->obj.write_domain &= ~I915_GEM_GPU_DOMAINS;
 		}
-		i915_gem_object_move_to_inactive(&obj_priv->obj);
+		/* unlocks object and list */
+		i915_gem_object_move_to_inactive_locked(&obj_priv->obj);
+		mtx_enter(&dev_priv->list_lock);
 	}
+	mtx_leave(&dev_priv->list_lock);
 
 	/* unbind everything */
 	(void)i915_gem_evict_inactive(dev_priv);
@@ -3955,10 +4157,10 @@ inteldrm_hangcheck(void *arg)
 	struct drm_i915_private	*dev_priv = arg;
 	u_int32_t		 acthd;
 
-	/* are we idle? */
+	/* are we idle? no requests, or ring is empty */
 	if (TAILQ_EMPTY(&dev_priv->mm.request_list) ||
-	    i915_seqno_passed(i915_get_gem_seqno(dev_priv),
-	    TAILQ_LAST(&dev_priv->mm.request_list, i915_request)->seqno)) {
+	    (I915_READ(PRB0_HEAD) & HEAD_ADDR) ==
+	    (I915_READ(PRB0_TAIL) & TAIL_ADDR)) {
 		dev_priv->mm.hang_cnt = 0;
 		return;
 	}
@@ -4496,6 +4698,7 @@ i915_gem_set_tiling(struct drm_device *dev, void *data,
 	if (obj == NULL)
 		return (EBADF);
 	obj_priv = (struct inteldrm_obj *)obj;
+	drm_hold_object(obj);
 
 	if (obj_priv->pin_count == 0) {
 		ret = EBUSY;
@@ -4505,11 +4708,9 @@ i915_gem_set_tiling(struct drm_device *dev, void *data,
 	if (i915_tiling_ok(dev, args->stride, obj->size,
 	    args->tiling_mode) == 0) {
 		ret = EINVAL;
-		DRM_LOCK(); /* for unref */
 		goto out;
 	}
 
-	DRM_LOCK();
 	if (args->tiling_mode == I915_TILING_NONE) {
 		args->swizzle_mode = I915_BIT_6_SWIZZLE_NONE;
 		args->stride = 0;
@@ -4537,15 +4738,14 @@ i915_gem_set_tiling(struct drm_device *dev, void *data,
 		/* fence may no longer be correct, wipe it */
 		inteldrm_wipe_mappings(obj);
 		if (obj_priv->fence_reg != I915_FENCE_REG_NONE)
-			atomic_setbits_int(&obj_priv->io_flags,
+			atomic_setbits_int(&obj->do_flags,
 			    I915_FENCE_INVALID);
 		obj_priv->tiling_mode = args->tiling_mode;
 		obj_priv->stride = args->stride;
 	}
 	 
 out:
-	drm_gem_object_unreference(obj);
-	DRM_UNLOCK();
+	drm_unhold_and_unref(obj);
 
 	return (ret);
 }
@@ -4565,10 +4765,9 @@ i915_gem_get_tiling(struct drm_device *dev, void *data,
 	obj = drm_gem_object_lookup(dev, file_priv, args->handle);
 	if (obj == NULL)
 		return (EBADF);
+	drm_hold_object(obj);
 	obj_priv = (struct inteldrm_obj *)obj;
 
-	DRM_LOCK();
-
 	args->tiling_mode = obj_priv->tiling_mode;
 	switch (obj_priv->tiling_mode) {
 	case I915_TILING_X:
@@ -4584,8 +4783,7 @@ i915_gem_get_tiling(struct drm_device *dev, void *data,
 		DRM_ERROR("unknown tiling mode\n");
 	}
 
-	drm_gem_object_unreference(obj);
-	DRM_UNLOCK();
+	drm_unhold_and_unref(obj);
 
 	return 0;
 }
diff --git a/sys/dev/pci/drm/i915_drv.h b/sys/dev/pci/drm/i915_drv.h
index cdf9d1b76d7..0d887ed3972 100644
--- a/sys/dev/pci/drm/i915_drv.h
+++ b/sys/dev/pci/drm/i915_drv.h
@@ -76,6 +76,14 @@ struct inteldrm_fence {
 	u_int32_t			 last_rendering_seqno;
 };
 
+/*
+ * lock ordering:
+ * exec lock,
+ * request lock
+ * list lock.
+ *
+ * XXX fence lock ,object lock
+ */
 typedef struct drm_i915_private {
 	struct device		 dev;
 	struct device		*drmdev;
@@ -127,9 +135,16 @@ typedef struct drm_i915_private {
 
 	int			 allow_batchbuffer;
 
-	struct inteldrm_fence		fence_regs[16]; /* 965 */
-	int				fence_reg_start; /* 4 by default */
-	int				num_fence_regs; /* 8 pre-965, 16 post */
+	struct mutex		 fence_lock;
+	struct inteldrm_fence	 fence_regs[16]; /* 965 */
+	int			 fence_reg_start; /* 4 by default */
+	int			 num_fence_regs; /* 8 pre-965, 16 post */
+
+	/* protects inactive, flushing, active and exec locks */
+	struct mutex		 list_lock;
+
+	/* protects access to request_list */
+	struct mutex		 request_lock;
 
 	/* Register state */
 	u8 saveLBB;
@@ -353,6 +368,18 @@ struct inteldrm_file {
 #define CHIP_GEN4	0x40000
 #define CHIP_GEN6	0x80000
 
+/* flags we use in drm_obj's do_flags */
+#define I915_ACTIVE		0x0010	/* being used by the gpu. */
+#define I915_IN_EXEC		0x0020	/* being processed in execbuffer */
+#define I915_USER_PINNED	0x0040	/* BO has been pinned from userland */
+#define I915_GPU_WRITE		0x0080	/* BO has been not flushed */
+#define I915_DONTNEED		0x0100	/* BO backing pages purgable */
+#define I915_PURGED		0x0200	/* BO backing pages purged */
+#define I915_DIRTY		0x0400	/* BO written to since last bound */
+#define I915_EXEC_NEEDS_FENCE	0x0800	/* being processed but will need fence*/
+#define I915_FENCED_EXEC	0x1000	/* Most recent exec needs fence */
+#define I915_FENCE_INVALID	0x2000	/* fence has been lazily invalidated */
+
 /** driver private structure attached to each drm_gem_object */
 struct inteldrm_obj {
 	struct drm_obj				 obj;
@@ -367,22 +394,6 @@ struct inteldrm_obj {
 	/* Current offset of the object in GTT space. */
 	bus_addr_t				 gtt_offset;
 	u_int32_t				*bit_17;
-	/*
-	 * This is set if the object is on the active or flushing lists
-	 * (has pending rendering), and is not set if it's on inactive (ready
-	 * to be unbound).
-	 */
-#define I915_ACTIVE		0x0001	/* being used by the gpu. */
-#define I915_IN_EXEC		0x0002	/* being processed in execbuffer */
-#define I915_USER_PINNED	0x0004	/* BO has been pinned from userland */
-#define I915_GPU_WRITE		0x0008	/* BO has been not flushed */
-#define I915_DONTNEED		0x0010	/* BO backing pages purgable */
-#define I915_PURGED		0x0020	/* BO backing pages purged */
-#define I915_DIRTY		0x0040	/* BO written to since last bound */
-#define I915_EXEC_NEEDS_FENCE	0x0080	/* being processed but will need fence*/
-#define I915_FENCED_EXEC	0x0100	/* Most recent exec needs fence */
-#define I915_FENCE_INVALID	0x0200	/* fence has been lazily invalidated */
-	int					 io_flags;
 	/* extra flags to bus_dma */
 	int					 dma_flags; 
 	/* Fence register for this object. needed for tiling. */
@@ -399,11 +410,12 @@ struct inteldrm_obj {
 	u_int32_t				 stride;
 };
 
-#define inteldrm_is_active(obj_priv)	(obj_priv->io_flags & I915_ACTIVE)
-#define inteldrm_is_dirty(obj_priv)	(obj_priv->io_flags & I915_DIRTY)
+#define inteldrm_is_active(obj_priv)	(obj_priv->obj.do_flags & I915_ACTIVE)
+#define inteldrm_is_dirty(obj_priv)	(obj_priv->obj.do_flags & I915_DIRTY)
 #define inteldrm_exec_needs_fence(obj_priv)		\
-	(obj_priv->io_flags & I915_EXEC_NEEDS_FENCE)
-#define inteldrm_needs_fence(obj_priv)	(obj_priv->io_flags & I915_FENCED_EXEC)
+	(obj_priv->obj.do_flags & I915_EXEC_NEEDS_FENCE)
+#define inteldrm_needs_fence(obj_priv)			\
+	(obj_priv->obj.do_flags & I915_FENCED_EXEC)
 
 /**
  * Request queue structure.
@@ -2394,13 +2406,13 @@ i915_get_gem_seqno(struct drm_i915_private *dev_priv)
 static __inline int
 i915_obj_purgeable(struct inteldrm_obj *obj_priv)
 {
-	return (obj_priv->io_flags & I915_DONTNEED);
+	return (obj_priv->obj.do_flags & I915_DONTNEED);
 }
 
 static __inline int
 i915_obj_purged(struct inteldrm_obj *obj_priv)
 {
-	return (obj_priv->io_flags & I915_PURGED);
+	return (obj_priv->obj.do_flags & I915_PURGED);
 }
 
 #endif