Use a per-UVM object lock to serialize the lower part of the fault handler.

Like the per-amap lock the `vmobjlock' is principally used to serialized
access to objects in the fault handler to allow faults occurring on
different CPUs and different objects to be processed in parallel.

The fault handler now acquires the `vmobjlock' of a given UVM object as
soon as it finds one.  For now a write-lock is always acquired even if
some operations could use a read-lock.

Every pager, corresponding to a different kind of UVM object, now expect
the UVM object to be locked and some operations, like *_get() return it
unlocked.  This is enforced by assertions checking for rw_write_held().

The KERNEL_LOCK() is now pushed to the VFS boundary in the vnode pager.

To ensure the correct amap or object lock is held when modifying a page
many uvm_page* operations are now asserting for the "owner" lock.
However, fields of the "struct vm_page" are still being protected by the
global `pageqlock'.  To prevent lock ordering issues with the new
`vmobjlock' and to reduce differences with NetBSD this lock is now taken
and released for each page instead of around the whole loop.

This commit does not remove the KERNEL_LOCK/UNLOCK() dance.  Unlocking
will follow if there is no fallout.

Ported from NetBSD, tested by many, thanks!

ok kettenis@, kn@

1 files changed, 46 insertions, 10 deletions

diff --git a/sys/uvm/uvm_map.c b/sys/uvm/uvm_map.c
index 4f192f5b5ea..d4e420d4c1e 100644
--- a/sys/uvm/uvm_map.c
+++ b/sys/uvm/uvm_map.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: uvm_map.c,v 1.280 2021/12/07 18:30:26 deraadt Exp $	*/
+/*	$OpenBSD: uvm_map.c,v 1.281 2021/12/15 12:53:53 mpi Exp $	*/
 /*	$NetBSD: uvm_map.c,v 1.86 2000/11/27 08:40:03 chs Exp $	*/
 
 /*
@@ -124,6 +124,8 @@ struct vm_map_entry	*uvm_mapent_alloc(struct vm_map*, int);
 void			 uvm_mapent_free(struct vm_map_entry*);
 void			 uvm_unmap_kill_entry(struct vm_map*,
 			    struct vm_map_entry*);
+void			 uvm_unmap_kill_entry_withlock(struct vm_map *,
+			    struct vm_map_entry *, int);
 void			 uvm_unmap_detach_intrsafe(struct uvm_map_deadq *);
 void			 uvm_mapent_mkfree(struct vm_map*,
 			    struct vm_map_entry*, struct vm_map_entry**,
@@ -499,6 +501,28 @@ uvm_map_reference(struct vm_map *map)
 	atomic_inc_int(&map->ref_count);
 }
 
+void
+uvm_map_lock_entry(struct vm_map_entry *entry)
+{
+	if (entry->aref.ar_amap != NULL) {
+		amap_lock(entry->aref.ar_amap);
+	}
+	if (UVM_ET_ISOBJ(entry)) {
+		rw_enter(entry->object.uvm_obj->vmobjlock, RW_WRITE);
+	}
+}
+
+void
+uvm_map_unlock_entry(struct vm_map_entry *entry)
+{
+	if (UVM_ET_ISOBJ(entry)) {
+		rw_exit(entry->object.uvm_obj->vmobjlock);
+	}
+	if (entry->aref.ar_amap != NULL) {
+		amap_unlock(entry->aref.ar_amap);
+	}
+}
+
 /*
  * Calculate the dused delta.
  */
@@ -2101,7 +2125,8 @@ uvm_mapent_mkfree(struct vm_map *map, struct vm_map_entry *entry,
  * Unwire and release referenced amap and object from map entry.
  */
 void
-uvm_unmap_kill_entry(struct vm_map *map, struct vm_map_entry *entry)
+uvm_unmap_kill_entry_withlock(struct vm_map *map, struct vm_map_entry *entry,
+    int needlock)
 {
 	/* Unwire removed map entry. */
 	if (VM_MAPENT_ISWIRED(entry)) {
@@ -2111,6 +2136,9 @@ uvm_unmap_kill_entry(struct vm_map *map, struct vm_map_entry *entry)
 		KERNEL_UNLOCK();
 	}
 
+	if (needlock)
+		uvm_map_lock_entry(entry);
+
 	/* Entry-type specific code. */
 	if (UVM_ET_ISHOLE(entry)) {
 		/* Nothing to be done for holes. */
@@ -2157,17 +2185,19 @@ uvm_unmap_kill_entry(struct vm_map *map, struct vm_map_entry *entry)
 		 */
 		uvm_km_pgremove(entry->object.uvm_obj, entry->start,
 		    entry->end);
-
-		/*
-		 * null out kernel_object reference, we've just
-		 * dropped it
-		 */
-		entry->etype &= ~UVM_ET_OBJ;
-		entry->object.uvm_obj = NULL;  /* to be safe */
 	} else {
 		/* remove mappings the standard way. */
 		pmap_remove(map->pmap, entry->start, entry->end);
 	}
+
+	if (needlock)
+		uvm_map_unlock_entry(entry);
+}
+
+void
+uvm_unmap_kill_entry(struct vm_map *map, struct vm_map_entry *entry)
+{
+	uvm_unmap_kill_entry_withlock(map, entry, 0);
 }
 
 /*
@@ -2227,7 +2257,7 @@ uvm_unmap_remove(struct vm_map *map, vaddr_t start, vaddr_t end,
 			map->sserial++;
 
 		/* Kill entry. */
-		uvm_unmap_kill_entry(map, entry);
+		uvm_unmap_kill_entry_withlock(map, entry, 1);
 
 		/* Update space usage. */
 		if ((map->flags & VM_MAP_ISVMSPACE) &&
@@ -3420,8 +3450,10 @@ uvm_map_protect(struct vm_map *map, vaddr_t start, vaddr_t end,
 				 */
 				iter->wired_count = 0;
 			}
+			uvm_map_lock_entry(iter);
 			pmap_protect(map->pmap, iter->start, iter->end,
 			    iter->protection & mask);
+			uvm_map_unlock_entry(iter);
 		}
 
 		/*
@@ -3967,11 +3999,13 @@ uvm_mapent_forkcopy(struct vmspace *new_vm, struct vm_map *new_map,
 			 */
 			if (!UVM_ET_ISNEEDSCOPY(old_entry)) {
 				if (old_entry->max_protection & PROT_WRITE) {
+					uvm_map_lock_entry(old_entry);
 					pmap_protect(old_map->pmap,
 					    old_entry->start,
 					    old_entry->end,
 					    old_entry->protection &
 					    ~PROT_WRITE);
+					uvm_map_unlock_entry(old_entry);
 					pmap_update(old_map->pmap);
 				}
 				old_entry->etype |= UVM_ET_NEEDSCOPY;
@@ -4751,9 +4785,11 @@ flush_object:
 		    ((flags & PGO_FREE) == 0 ||
 		     ((entry->max_protection & PROT_WRITE) != 0 &&
 		      (entry->etype & UVM_ET_COPYONWRITE) == 0))) {
+			rw_enter(uobj->vmobjlock, RW_WRITE);
 			rv = uobj->pgops->pgo_flush(uobj,
 			    cp_start - entry->start + entry->offset,
 			    cp_end - entry->start + entry->offset, flags);
+			rw_exit(uobj->vmobjlock);
 
 			if (rv == FALSE)
 				error = EFAULT;