Use a per-UVM object lock to serialize the lower part of the fault handler.

Like the per-amap lock the `vmobjlock' is principally used to serialized
access to objects in the fault handler to allow faults occurring on
different CPUs and different objects to be processed in parallel.

The fault handler now acquires the `vmobjlock' of a given UVM object as
soon as it finds one.  For now a write-lock is always acquired even if
some operations could use a read-lock.

Every pager, corresponding to a different kind of UVM object, now expect
the UVM object to be locked and some operations, like *_get() return it
unlocked.  This is enforced by assertions checking for rw_write_held().

The KERNEL_LOCK() is now pushed to the VFS boundary in the vnode pager.

To ensure the correct amap or object lock is held when modifying a page
many uvm_page* operations are now asserting for the "owner" lock.
However, fields of the "struct vm_page" are still being protected by the
global `pageqlock'.  To prevent lock ordering issues with the new
`vmobjlock' and to reduce differences with NetBSD this lock is now taken
and released for each page instead of around the whole loop.

This commit does not remove the KERNEL_LOCK/UNLOCK() dance.  Unlocking
will follow if there is no fallout.

Ported from NetBSD, tested by many, thanks!

ok kettenis@, kn@

1 files changed, 9 insertions, 6 deletions

diff --git a/sys/uvm/uvm_device.c b/sys/uvm/uvm_device.c
index e5d035f2947..74d9490b326 100644
--- a/sys/uvm/uvm_device.c
+++ b/sys/uvm/uvm_device.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: uvm_device.c,v 1.65 2021/10/23 14:42:08 mpi Exp $	*/
+/*	$OpenBSD: uvm_device.c,v 1.66 2021/12/15 12:53:53 mpi Exp $	*/
 /*	$NetBSD: uvm_device.c,v 1.30 2000/11/25 06:27:59 chs Exp $	*/
 
 /*
@@ -166,7 +166,9 @@ udv_attach(dev_t device, vm_prot_t accessprot, voff_t off, vsize_t size)
 			/*
 			 * bump reference count, unhold, return.
 			 */
+			rw_enter(lcv->u_obj.vmobjlock, RW_WRITE);
 			lcv->u_obj.uo_refs++;
+			rw_exit(lcv->u_obj.vmobjlock);
 
 			mtx_enter(&udv_lock);
 			if (lcv->u_flags & UVM_DEVICE_WANTED)
@@ -228,8 +230,9 @@ udv_attach(dev_t device, vm_prot_t accessprot, voff_t off, vsize_t size)
 static void
 udv_reference(struct uvm_object *uobj)
 {
-	KERNEL_ASSERT_LOCKED();
+	rw_enter(uobj->vmobjlock, RW_WRITE);
 	uobj->uo_refs++;
+	rw_exit(uobj->vmobjlock);
 }
 
 /*
@@ -248,8 +251,10 @@ udv_detach(struct uvm_object *uobj)
 	 * loop until done
 	 */
 again:
+	rw_enter(uobj->vmobjlock, RW_WRITE);
 	if (uobj->uo_refs > 1) {
 		uobj->uo_refs--;
+		rw_exit(uobj->vmobjlock);
 		return;
 	}
 	KASSERT(uobj->uo_npages == 0 && RBT_EMPTY(uvm_objtree, &uobj->memt));
@@ -260,10 +265,7 @@ again:
 	mtx_enter(&udv_lock);
 	if (udv->u_flags & UVM_DEVICE_HOLD) {
 		udv->u_flags |= UVM_DEVICE_WANTED;
-		/*
-		 * lock interleaving. -- this is ok in this case since the
-		 * locks are both IPL_NONE
-		 */
+		rw_exit(uobj->vmobjlock);
 		msleep_nsec(udv, &udv_lock, PVM | PNORELOCK, "udv_detach",
 		    INFSLP);
 		goto again;
@@ -276,6 +278,7 @@ again:
 	if (udv->u_flags & UVM_DEVICE_WANTED)
 		wakeup(udv);
 	mtx_leave(&udv_lock);
+	rw_exit(uobj->vmobjlock);
 
 	uvm_obj_destroy(uobj);
 	free(udv, M_TEMP, sizeof(*udv));