Switch the TTM code over to the generic DRM VMA manager.

ok jsg@

6 files changed, 74 insertions, 176 deletions

diff --git a/sys/dev/pci/drm/ttm/ttm_bo.c b/sys/dev/pci/drm/ttm/ttm_bo.c
index 523c25a606c..0f34925f6dd 100644
--- a/sys/dev/pci/drm/ttm/ttm_bo.c
+++ b/sys/dev/pci/drm/ttm/ttm_bo.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: ttm_bo.c,v 1.19 2015/10/23 08:21:58 kettenis Exp $	*/
+/*	$OpenBSD: ttm_bo.c,v 1.20 2017/06/04 14:02:24 kettenis Exp $	*/
 /**************************************************************************
  *
  * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA
@@ -728,14 +728,7 @@ static void ttm_bo_release(struct kref *kref)
 	struct ttm_bo_device *bdev = bo->bdev;
 	struct ttm_mem_type_manager *man = &bdev->man[bo->mem.mem_type];
 
-	write_lock(&bdev->vm_lock);
-	if (likely(bo->vm_node != NULL)) {
-		RB_REMOVE(ttm_bo_device_buffer_objects,
-		    &bdev->addr_space_rb, bo);
-		drm_mm_put_block(bo->vm_node);
-		bo->vm_node = NULL;
-	}
-	write_unlock(&bdev->vm_lock);
+	drm_vma_offset_remove(&bdev->vma_manager, &bo->vma_node);
 	ttm_mem_io_lock(man, false);
 	ttm_mem_io_free_vm(bo);
 	ttm_mem_io_unlock(man);
@@ -1252,6 +1245,7 @@ int ttm_bo_init(struct ttm_bo_device *bdev,
 	bo->acc_size = acc_size;
 	bo->sg = sg;
 	atomic_inc(&bo->glob->bo_count);
+	drm_vma_node_reset(&bo->vma_node);
 
 	ret = ttm_bo_check_placement(bo, placement);
 	if (unlikely(ret != 0))
@@ -1552,10 +1546,7 @@ int ttm_bo_device_release(struct ttm_bo_device *bdev)
 		TTM_DEBUG("Swap list was clean\n");
 	spin_unlock(&glob->lru_lock);
 
-	BUG_ON(!drm_mm_clean(&bdev->addr_space_mm));
-	write_lock(&bdev->vm_lock);
-	drm_mm_takedown(&bdev->addr_space_mm);
-	write_unlock(&bdev->vm_lock);
+	drm_vma_offset_manager_destroy(&bdev->vma_manager);
 
 	return ret;
 }
@@ -1569,7 +1560,6 @@ int ttm_bo_device_init(struct ttm_bo_device *bdev,
 {
 	int ret = -EINVAL;
 
-	rw_init(&bdev->vm_lock, "ttmvm");
 	bdev->driver = driver;
 
 	memset(bdev->man, 0, sizeof(bdev->man));
@@ -1582,9 +1572,8 @@ int ttm_bo_device_init(struct ttm_bo_device *bdev,
 	if (unlikely(ret != 0))
 		goto out_no_sys;
 
-	RB_INIT(&bdev->addr_space_rb);
-	drm_mm_init(&bdev->addr_space_mm, file_page_offset, 0x10000000);
-
+	drm_vma_offset_manager_init(&bdev->vma_manager, file_page_offset,
+				    0x10000000);
 	INIT_DELAYED_WORK(&bdev->wq, ttm_bo_delayed_workqueue);
 	INIT_LIST_HEAD(&bdev->ddestroy);
 	bdev->dev_mapping = NULL;
@@ -1665,14 +1654,6 @@ void ttm_bo_unmap_virtual(struct ttm_buffer_object *bo)
 
 EXPORT_SYMBOL(ttm_bo_unmap_virtual);
 
-static void ttm_bo_vm_insert_rb(struct ttm_buffer_object *bo)
-{
-	struct ttm_bo_device *bdev = bo->bdev;
-
-	/* The caller acquired bdev->vm_lock. */
-	RB_INSERT(ttm_bo_device_buffer_objects, &bdev->addr_space_rb, bo);
-}
-
 /**
  * ttm_bo_setup_vm:
  *
@@ -1687,38 +1668,9 @@ static void ttm_bo_vm_insert_rb(struct ttm_buffer_object *bo)
 static int ttm_bo_setup_vm(struct ttm_buffer_object *bo)
 {
 	struct ttm_bo_device *bdev = bo->bdev;
-	int ret;
-
-retry_pre_get:
-	ret = drm_mm_pre_get(&bdev->addr_space_mm);
-	if (unlikely(ret != 0))
-		return ret;
-
-	write_lock(&bdev->vm_lock);
-	bo->vm_node = drm_mm_search_free(&bdev->addr_space_mm,
-					 bo->mem.num_pages, 0, 0);
 
-	if (unlikely(bo->vm_node == NULL)) {
-		ret = -ENOMEM;
-		goto out_unlock;
-	}
-
-	bo->vm_node = drm_mm_get_block_atomic(bo->vm_node,
-					      bo->mem.num_pages, 0);
-
-	if (unlikely(bo->vm_node == NULL)) {
-		write_unlock(&bdev->vm_lock);
-		goto retry_pre_get;
-	}
-
-	ttm_bo_vm_insert_rb(bo);
-	write_unlock(&bdev->vm_lock);
-	bo->addr_space_offset = ((uint64_t) bo->vm_node->start) << PAGE_SHIFT;
-
-	return 0;
-out_unlock:
-	write_unlock(&bdev->vm_lock);
-	return ret;
+	return drm_vma_offset_add(&bdev->vma_manager, &bo->vma_node,
+				  bo->mem.num_pages);
 }
 
 int ttm_bo_wait(struct ttm_buffer_object *bo,
diff --git a/sys/dev/pci/drm/ttm/ttm_bo_api.h b/sys/dev/pci/drm/ttm/ttm_bo_api.h
index 7fe2e5a15ce..feaa8ed7162 100644
--- a/sys/dev/pci/drm/ttm/ttm_bo_api.h
+++ b/sys/dev/pci/drm/ttm/ttm_bo_api.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: ttm_bo_api.h,v 1.5 2015/09/27 11:09:26 jsg Exp $	*/
+/*	$OpenBSD: ttm_bo_api.h,v 1.6 2017/06/04 14:02:24 kettenis Exp $	*/
 /**************************************************************************
  *
  * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA
@@ -33,6 +33,7 @@
 #define _TTM_BO_API_H_
 
 #include <dev/pci/drm/drmP.h>
+#include <dev/pci/drm/drm_vma_manager.h>
 
 struct ttm_bo_device;
 
@@ -139,7 +140,6 @@ struct ttm_tt;
  * @type: The bo type.
  * @destroy: Destruction function. If NULL, kfree is used.
  * @num_pages: Actual number of pages.
- * @addr_space_offset: Address space offset.
  * @acc_size: Accounted size for this object.
  * @kref: Reference count of this buffer object. When this refcount reaches
  * zero, the object is put on the delayed delete list.
@@ -167,8 +167,7 @@ struct ttm_tt;
  * @reserved: Deadlock-free lock used for synchronization state transitions.
  * @sync_obj: Pointer to a synchronization object.
  * @priv_flags: Flags describing buffer object internal state.
- * @vm_rb: Rb node for the vm rb tree.
- * @vm_node: Address space manager node.
+ * @vma_node: Address space manager node.
  * @offset: The current GPU offset, which can have different meanings
  * depending on the memory type. For SYSTEM type memory, it should be 0.
  * @cur_placement: Hint of current placement.
@@ -197,7 +196,6 @@ struct ttm_buffer_object {
 	enum ttm_bo_type type;
 	void (*destroy) (struct ttm_buffer_object *);
 	unsigned long num_pages;
-	uint64_t addr_space_offset;
 	size_t acc_size;
 
 	/**
@@ -251,13 +249,7 @@ struct ttm_buffer_object {
 	void *sync_obj;
 	unsigned long priv_flags;
 
-	/**
-	 * Members protected by the bdev::vm_lock
-	 */
-
-	RB_ENTRY(ttm_buffer_object) vm_rb;
-	struct drm_mm_node *vm_node;
-
+	struct drm_vma_offset_node vma_node;
 
 	/**
 	 * Special members that are protected by the reserve lock
diff --git a/sys/dev/pci/drm/ttm/ttm_bo_driver.h b/sys/dev/pci/drm/ttm/ttm_bo_driver.h
index 29af2244422..3032f944162 100644
--- a/sys/dev/pci/drm/ttm/ttm_bo_driver.h
+++ b/sys/dev/pci/drm/ttm/ttm_bo_driver.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: ttm_bo_driver.h,v 1.7 2015/09/27 11:09:26 jsg Exp $	*/
+/*	$OpenBSD: ttm_bo_driver.h,v 1.8 2017/06/04 14:02:24 kettenis Exp $	*/
 /**************************************************************************
  *
  * Copyright (c) 2006-2009 Vmware, Inc., Palo Alto, CA., USA
@@ -516,7 +516,7 @@ struct ttm_bo_global {
  * @man: An array of mem_type_managers.
  * @fence_lock: Protects the synchronizing members on *all* bos belonging
  * to this device.
- * @addr_space_mm: Range manager for the device address space.
+ * @vma_manager: Address space manager
  * lru_lock: Spinlock that protects the buffer+device lru lists and
  * ddestroy lists.
  * @val_seq: Current validation sequence.
@@ -534,7 +534,6 @@ struct ttm_bo_device {
 	struct list_head device_list;
 	struct ttm_bo_global *glob;
 	struct ttm_bo_driver *driver;
-	rwlock_t vm_lock;
 	struct ttm_mem_type_manager man[TTM_NUM_MEM_TYPES];
 	spinlock_t fence_lock;
 
@@ -543,10 +542,9 @@ struct ttm_bo_device {
 	bus_dma_tag_t dmat;
 
 	/*
-	 * Protected by the vm lock.
+	 * Protected by internal locks.
 	 */
-	RB_HEAD(ttm_bo_device_buffer_objects, ttm_buffer_object) addr_space_rb;
-	struct drm_mm addr_space_mm;
+	struct drm_vma_offset_manager vma_manager;
 
 	/*
 	 * Protected by the global:lru lock.
@@ -1011,10 +1009,4 @@ int ttm_agp_tt_populate(struct ttm_tt *ttm);
 void ttm_agp_tt_unpopulate(struct ttm_tt *ttm);
 #endif
 
-int	ttm_bo_cmp_rb_tree_items(struct ttm_buffer_object *a,
-	    struct ttm_buffer_object *b);
-
-RB_PROTOTYPE(ttm_bo_device_buffer_objects, ttm_buffer_object, vm_rb,
-    ttm_bo_cmp_rb_tree_items);
-
 #endif
diff --git a/sys/dev/pci/drm/ttm/ttm_bo_manager.c b/sys/dev/pci/drm/ttm/ttm_bo_manager.c
index 92c38ad3c5b..ad948b5117a 100644
--- a/sys/dev/pci/drm/ttm/ttm_bo_manager.c
+++ b/sys/dev/pci/drm/ttm/ttm_bo_manager.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: ttm_bo_manager.c,v 1.5 2015/09/23 23:12:12 kettenis Exp $	*/
+/*	$OpenBSD: ttm_bo_manager.c,v 1.6 2017/06/04 14:02:24 kettenis Exp $	*/
 /**************************************************************************
  *
  * Copyright (c) 2007-2010 VMware, Inc., Palo Alto, CA., USA
@@ -53,34 +53,32 @@ static int ttm_bo_man_get_node(struct ttm_mem_type_manager *man,
 	struct ttm_range_manager *rman = (struct ttm_range_manager *) man->priv;
 	struct drm_mm *mm = &rman->mm;
 	struct drm_mm_node *node = NULL;
+	enum drm_mm_search_flags sflags = DRM_MM_SEARCH_BEST;
 	unsigned long lpfn;
 	int ret;
 
 	lpfn = placement->lpfn;
 	if (!lpfn)
 		lpfn = man->size;
-	do {
-		ret = drm_mm_pre_get(mm);
-		if (unlikely(ret))
-			return ret;
 
-		spin_lock(&rman->lock);
-		node = drm_mm_search_free_in_range(mm,
-					mem->num_pages, mem->page_alignment,
-					placement->fpfn, lpfn, 1);
-		if (unlikely(node == NULL)) {
-			spin_unlock(&rman->lock);
-			return 0;
-		}
-		node = drm_mm_get_block_atomic_range(node, mem->num_pages,
-						     mem->page_alignment,
-						     placement->fpfn,
-						     lpfn);
-		spin_unlock(&rman->lock);
-	} while (node == NULL);
+	node = kzalloc(sizeof(*node), GFP_KERNEL);
+	if (!node)
+		return -ENOMEM;
+
+	spin_lock(&rman->lock);
+	ret = drm_mm_insert_node_in_range_generic(mm, node, mem->num_pages,
+					  mem->page_alignment, 0,
+					  placement->fpfn, lpfn,
+					  sflags);
+	spin_unlock(&rman->lock);
+
+	if (unlikely(ret)) {
+		kfree(node);
+	} else {
+		mem->mm_node = node;
+		mem->start = node->start;
+	}
 
-	mem->mm_node = node;
-	mem->start = node->start;
 	return 0;
 }
 
@@ -91,8 +89,10 @@ static void ttm_bo_man_put_node(struct ttm_mem_type_manager *man,
 
 	if (mem->mm_node) {
 		spin_lock(&rman->lock);
-		drm_mm_put_block(mem->mm_node);
+		drm_mm_remove_node(mem->mm_node);
 		spin_unlock(&rman->lock);
+
+		kfree(mem->mm_node);
 		mem->mm_node = NULL;
 	}
 }
diff --git a/sys/dev/pci/drm/ttm/ttm_bo_util.c b/sys/dev/pci/drm/ttm/ttm_bo_util.c
index e7734e5a799..136814e90a6 100644
--- a/sys/dev/pci/drm/ttm/ttm_bo_util.c
+++ b/sys/dev/pci/drm/ttm/ttm_bo_util.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: ttm_bo_util.c,v 1.15 2015/09/27 11:09:26 jsg Exp $	*/
+/*	$OpenBSD: ttm_bo_util.c,v 1.16 2017/06/04 14:02:24 kettenis Exp $	*/
 /**************************************************************************
  *
  * Copyright (c) 2007-2009 VMware, Inc., Palo Alto, CA., USA
@@ -31,6 +31,7 @@
 
 #include <dev/pci/drm/ttm/ttm_bo_driver.h>
 #include <dev/pci/drm/ttm/ttm_placement.h>
+#include <dev/pci/drm/drm_vma_manager.h>
 
 int	 ttm_mem_reg_ioremap(struct ttm_bo_device *, struct ttm_mem_reg *,
 	     void **);
@@ -447,7 +448,7 @@ static int ttm_buffer_object_transfer(struct ttm_buffer_object *bo,
 	INIT_LIST_HEAD(&fbo->lru);
 	INIT_LIST_HEAD(&fbo->swap);
 	INIT_LIST_HEAD(&fbo->io_reserve_lru);
-	fbo->vm_node = NULL;
+	drm_vma_node_reset(&fbo->vma_node);
 	atomic_set(&fbo->cpu_writers, 0);
 
 	spin_lock(&bdev->fence_lock);
diff --git a/sys/dev/pci/drm/ttm/ttm_bo_vm.c b/sys/dev/pci/drm/ttm/ttm_bo_vm.c
index 905ca27feae..c7fd2c13731 100644
--- a/sys/dev/pci/drm/ttm/ttm_bo_vm.c
+++ b/sys/dev/pci/drm/ttm/ttm_bo_vm.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: ttm_bo_vm.c,v 1.10 2016/04/05 08:22:50 kettenis Exp $	*/
+/*	$OpenBSD: ttm_bo_vm.c,v 1.11 2017/06/04 14:02:24 kettenis Exp $	*/
 /**************************************************************************
  *
  * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA
@@ -34,65 +34,12 @@
 #include <dev/pci/drm/ttm/ttm_module.h>
 #include <dev/pci/drm/ttm/ttm_bo_driver.h>
 #include <dev/pci/drm/ttm/ttm_placement.h>
+#include <dev/pci/drm/drm_vma_manager.h>
 
 #define TTM_BO_VM_NUM_PREFAULT 16
 
 ssize_t	 ttm_bo_fbdev_io(struct ttm_buffer_object *, const char __user *,
 	     char __user *, size_t, off_t *, bool);
-struct ttm_buffer_object *
-	 ttm_bo_vm_lookup_rb(struct ttm_bo_device *, unsigned long,
-	     unsigned long);
-
-#undef RB_ROOT
-#define RB_ROOT(head)	(head)->rbh_root
-
-RB_GENERATE(ttm_bo_device_buffer_objects, ttm_buffer_object, vm_rb,
-    ttm_bo_cmp_rb_tree_items);
-
-int
-ttm_bo_cmp_rb_tree_items(struct ttm_buffer_object *a,
-    struct ttm_buffer_object *b)
-{
-
-	if (a->vm_node->start < b->vm_node->start) {
-		return (-1);
-	} else if (a->vm_node->start > b->vm_node->start) {
-		return (1);
-	} else {
-		return (0);
-	}
-}
-
-struct ttm_buffer_object *
-ttm_bo_vm_lookup_rb(struct ttm_bo_device *bdev,
-		    unsigned long page_start,
-		    unsigned long num_pages)
-{
-	unsigned long cur_offset;
-	struct ttm_buffer_object *bo;
-	struct ttm_buffer_object *best_bo = NULL;
-
-	bo = RB_ROOT(&bdev->addr_space_rb);
-	while (bo != NULL) {
-		cur_offset = bo->vm_node->start;
-		if (page_start >= cur_offset) {
-			best_bo = bo;
-			if (page_start == cur_offset)
-				break;
-			bo = RB_RIGHT(bo, vm_rb);
-		} else
-			bo = RB_LEFT(bo, vm_rb);
-	}
-
-	if (unlikely(best_bo == NULL))
-		return NULL;
-
-	if (unlikely((best_bo->vm_node->start + best_bo->num_pages) <
-		     (page_start + num_pages)))
-		return NULL;
-
-	return best_bo;
-}
 
 int ttm_bo_vm_fault(struct uvm_faultinfo *, vaddr_t, vm_page_t *,
         int, int, vm_fault_t, vm_prot_t, int);
@@ -187,9 +134,9 @@ ttm_bo_vm_fault(struct uvm_faultinfo *ufi, vaddr_t vaddr, vm_page_t *pps,
 	}
 
 	page_offset = ((address - ufi->entry->start) >> PAGE_SHIFT) +
-	    bo->vm_node->start - (ufi->entry->offset >> PAGE_SHIFT);
+	    drm_vma_node_start(&bo->vma_node) - (ufi->entry->offset >> PAGE_SHIFT);
 	page_last = ((ufi->entry->end - ufi->entry->start) >> PAGE_SHIFT) +
-	    bo->vm_node->start - (ufi->entry->offset >> PAGE_SHIFT);
+	    drm_vma_node_start(&bo->vma_node) - (ufi->entry->offset >> PAGE_SHIFT);
 
 	if (unlikely(page_offset >= bo->num_pages)) {
 		retval = VM_PAGER_ERROR;
@@ -301,6 +248,30 @@ struct uvm_pagerops ttm_bo_vm_ops = {
 	.pgo_detach = ttm_bo_vm_detach
 };
 
+static struct ttm_buffer_object *ttm_bo_vm_lookup(struct ttm_bo_device *bdev,
+						  unsigned long offset,
+						  unsigned long pages)
+{
+	struct drm_vma_offset_node *node;
+	struct ttm_buffer_object *bo = NULL;
+
+	drm_vma_offset_lock_lookup(&bdev->vma_manager);
+
+	node = drm_vma_offset_lookup_locked(&bdev->vma_manager, offset, pages);
+	if (likely(node)) {
+		bo = container_of(node, struct ttm_buffer_object, vma_node);
+		if (!kref_get_unless_zero(&bo->kref))
+			bo = NULL;
+	}
+
+	drm_vma_offset_unlock_lookup(&bdev->vma_manager);
+
+	if (!bo)
+		pr_err("Could not find buffer object to map\n");
+
+	return bo;
+}
+
 struct uvm_object *
 ttm_bo_mmap(voff_t off, vsize_t size, struct ttm_bo_device *bdev)
 {
@@ -308,14 +279,9 @@ ttm_bo_mmap(voff_t off, vsize_t size, struct ttm_bo_device *bdev)
 	struct ttm_buffer_object *bo;
 	int ret;
 
-	read_lock(&bdev->vm_lock);
-	bo = ttm_bo_vm_lookup_rb(bdev, off >> PAGE_SHIFT, size >> PAGE_SHIFT);
-	if (likely(bo != NULL) && !kref_get_unless_zero(&bo->kref))
-		bo = NULL;
-	read_unlock(&bdev->vm_lock);
-
-	if (unlikely(bo == NULL)) {
-		pr_err("Could not find buffer object to map\n");
+	bo = ttm_bo_vm_lookup(bdev, off >> PAGE_SHIFT, size >> PAGE_SHIFT);
+	if (unlikely(!bo)) {
+ 		ret = -EINVAL;
 		return NULL;
 	}
 
@@ -383,12 +349,7 @@ ssize_t ttm_bo_io(struct ttm_bo_device *bdev, struct file *filp,
 	bool no_wait = false;
 	bool dummy;
 
-	read_lock(&bdev->vm_lock);
-	bo = ttm_bo_vm_lookup_rb(bdev, dev_offset, 1);
-	if (likely(bo != NULL))
-		ttm_bo_reference(bo);
-	read_unlock(&bdev->vm_lock);
-
+	bo = ttm_bo_vm_lookup(bdev, dev_offset, 1);
 	if (unlikely(bo == NULL))
 		return -EFAULT;
 
@@ -402,7 +363,7 @@ ssize_t ttm_bo_io(struct ttm_bo_device *bdev, struct file *filp,
 	if (unlikely(ret != 0))
 		goto out_unref;
 
-	kmap_offset = dev_offset - bo->vm_node->start;
+	kmap_offset = dev_offset - drm_vma_node_start(&bo->vma_node);
 	if (unlikely(kmap_offset >= bo->num_pages)) {
 		ret = -EFBIG;
 		goto out_unref;