Turn uvm_pagealloc() mp-safe by checking uvmexp global with pageqlock held.

Use a new flag, UVM_PLA_USERESERVE, to tell uvm_pmr_getpages() that using
kernel reserved pages is allowed.

Merge duplicated checks waking the pagedaemon to uvm_pmr_getpages().

Add two more pages to the amount reserved for the kernel to compensate the
fact that the pagedaemon may now consume an additional page.

Document locking of some uvmexp fields.

ok kettenis@

4 files changed, 62 insertions, 59 deletions

diff --git a/sys/uvm/uvm_extern.h b/sys/uvm/uvm_extern.h
index ee8779c36df..7e5e7d6f0dd 100644
--- a/sys/uvm/uvm_extern.h
+++ b/sys/uvm/uvm_extern.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: uvm_extern.h,v 1.154 2020/10/19 08:19:46 mpi Exp $	*/
+/*	$OpenBSD: uvm_extern.h,v 1.155 2020/12/01 13:56:22 mpi Exp $	*/
 /*	$NetBSD: uvm_extern.h,v 1.57 2001/03/09 01:02:12 chs Exp $	*/
 
 /*
@@ -142,14 +142,15 @@ typedef int		vm_prot_t;
 #define	UVM_PGA_ZERO		0x0002	/* returned page must be zeroed */
 
 /*
- * flags for uvm_pglistalloc()
+ * flags for uvm_pglistalloc() also used by uvm_pmr_getpages()
  */
 #define UVM_PLA_WAITOK		0x0001	/* may sleep */
 #define UVM_PLA_NOWAIT		0x0002	/* can't sleep (need one of the two) */
 #define UVM_PLA_ZERO		0x0004	/* zero all pages before returning */
 #define UVM_PLA_TRYCONTIG	0x0008	/* try to allocate contig physmem */
 #define UVM_PLA_FAILOK		0x0010	/* caller can handle failure */
-#define UVM_PLA_NOWAKE		0x0020	/* don't wake the page daemon on failure */
+#define UVM_PLA_NOWAKE		0x0020	/* don't wake page daemon on failure */
+#define UVM_PLA_USERESERVE	0x0040	/* can allocate from kernel reserve */
 
 /*
  * lockflags that control the locking behavior of various functions.
diff --git a/sys/uvm/uvm_page.c b/sys/uvm/uvm_page.c
index 34e0295dd30..ccd2fae6240 100644
--- a/sys/uvm/uvm_page.c
+++ b/sys/uvm/uvm_page.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: uvm_page.c,v 1.152 2020/11/27 12:45:00 mpi Exp $	*/
+/*	$OpenBSD: uvm_page.c,v 1.153 2020/12/01 13:56:22 mpi Exp $	*/
 /*	$NetBSD: uvm_page.c,v 1.44 2000/11/27 08:40:04 chs Exp $	*/
 
 /*
@@ -277,7 +277,7 @@ uvm_page_init(vaddr_t *kvm_startp, vaddr_t *kvm_endp)
 	 * XXXCDC - values may need adjusting
 	 */
 	uvmexp.reserve_pagedaemon = 4;
-	uvmexp.reserve_kernel = 6;
+	uvmexp.reserve_kernel = 8;
 	uvmexp.anonminpct = 10;
 	uvmexp.vnodeminpct = 10;
 	uvmexp.vtextminpct = 5;
@@ -733,32 +733,11 @@ uvm_pglistalloc(psize_t size, paddr_t low, paddr_t high, paddr_t alignment,
 	size = atop(round_page(size));
 
 	/*
-	 * check to see if we need to generate some free pages waking
-	 * the pagedaemon.
-	 */
-	if ((uvmexp.free - BUFPAGES_DEFICIT) < uvmexp.freemin ||
-	    ((uvmexp.free - BUFPAGES_DEFICIT) < uvmexp.freetarg &&
-	    (uvmexp.inactive + BUFPAGES_INACT) < uvmexp.inactarg))
-		wakeup(&uvm.pagedaemon);
-
-	/*
 	 * XXX uvm_pglistalloc is currently only used for kernel
 	 * objects. Unlike the checks in uvm_pagealloc, below, here
-	 * we are always allowed to use the kernel reserve. However, we
-	 * have to enforce the pagedaemon reserve here or allocations
-	 * via this path could consume everything and we can't
-	 * recover in the page daemon.
+	 * we are always allowed to use the kernel reserve.
 	 */
- again:
-	if ((uvmexp.free <= uvmexp.reserve_pagedaemon + size &&
-	    !((curproc == uvm.pagedaemon_proc) ||
-		(curproc == syncerproc)))) {
-		if (flags & UVM_PLA_WAITOK) {
-			uvm_wait("uvm_pglistalloc");
-			goto again;
-		}
-		return (ENOMEM);
-	}
+	flags |= UVM_PLA_USERESERVE;
 
 	if ((high & PAGE_MASK) != PAGE_MASK) {
 		printf("uvm_pglistalloc: Upper boundary 0x%lx "
@@ -871,7 +850,7 @@ uvm_pagerealloc_multi(struct uvm_object *obj, voff_t off, vsize_t size,
 }
 
 /*
- * uvm_pagealloc_strat: allocate vm_page from a particular free list.
+ * uvm_pagealloc: allocate vm_page from a particular free list.
  *
  * => return null if no pages free
  * => wake up pagedaemon if number of free pages drops below low water mark
@@ -886,37 +865,21 @@ uvm_pagealloc(struct uvm_object *obj, voff_t off, struct vm_anon *anon,
 	struct vm_page *pg;
 	struct pglist pgl;
 	int pmr_flags;
-	boolean_t use_reserve;
 
 	KASSERT(obj == NULL || anon == NULL);
+	KASSERT(anon == NULL || off == 0);
 	KASSERT(off == trunc_page(off));
 
-	/*
-	 * check to see if we need to generate some free pages waking
-	 * the pagedaemon.
-	 */
-	if ((uvmexp.free - BUFPAGES_DEFICIT) < uvmexp.freemin ||
-	    ((uvmexp.free - BUFPAGES_DEFICIT) < uvmexp.freetarg &&
-	    (uvmexp.inactive + BUFPAGES_INACT) < uvmexp.inactarg))
-		wakeup(&uvm.pagedaemon);
+	pmr_flags = UVM_PLA_NOWAIT;
 
 	/*
-	 * fail if any of these conditions is true:
-	 * [1]  there really are no free pages, or
-	 * [2]  only kernel "reserved" pages remain and
-	 *        the page isn't being allocated to a kernel object.
-	 * [3]  only pagedaemon "reserved" pages remain and
-	 *        the requestor isn't the pagedaemon.
+	 * We're allowed to use the kernel reserve if the page is
+	 * being allocated to a kernel object.
 	 */
-	use_reserve = (flags & UVM_PGA_USERESERVE) ||
-		(obj && UVM_OBJ_IS_KERN_OBJECT(obj));
-	if ((uvmexp.free <= uvmexp.reserve_kernel && !use_reserve) ||
-	    (uvmexp.free <= uvmexp.reserve_pagedaemon &&
-	     !((curproc == uvm.pagedaemon_proc) ||
-	      (curproc == syncerproc))))
-		goto fail;
+	if ((flags & UVM_PGA_USERESERVE) ||
+	    (obj != NULL && UVM_OBJ_IS_KERN_OBJECT(obj)))
+	    	pmr_flags |= UVM_PLA_USERESERVE;
 
-	pmr_flags = UVM_PLA_NOWAIT;
 	if (flags & UVM_PGA_ZERO)
 		pmr_flags |= UVM_PLA_ZERO;
 	TAILQ_INIT(&pgl);
diff --git a/sys/uvm/uvm_pmemrange.c b/sys/uvm/uvm_pmemrange.c
index d8b0af33533..49c31fc7d67 100644
--- a/sys/uvm/uvm_pmemrange.c
+++ b/sys/uvm/uvm_pmemrange.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: uvm_pmemrange.c,v 1.59 2020/02/18 12:13:40 mpi Exp $	*/
+/*	$OpenBSD: uvm_pmemrange.c,v 1.60 2020/12/01 13:56:22 mpi Exp $	*/
 
 /*
  * Copyright (c) 2009, 2010 Ariane van der Steldt <ariane@stack.nl>
@@ -869,6 +869,7 @@ uvm_pmr_getpages(psize_t count, paddr_t start, paddr_t end, paddr_t align,
 	KASSERT(boundary == 0 || powerof2(boundary));
 	KASSERT(boundary == 0 || maxseg * boundary >= count);
 	KASSERT(TAILQ_EMPTY(result));
+	KASSERT(!(flags & UVM_PLA_WAITOK) ^ !(flags & UVM_PLA_NOWAIT));
 
 	/*
 	 * TRYCONTIG is a noop if you only want a single segment.
@@ -938,8 +939,42 @@ uvm_pmr_getpages(psize_t count, paddr_t start, paddr_t end, paddr_t align,
 	 */
 	desperate = 0;
 
+again:
 	uvm_lock_fpageq();
 
+	/*
+	 * check to see if we need to generate some free pages waking
+	 * the pagedaemon.
+	 */
+	if ((uvmexp.free - BUFPAGES_DEFICIT) < uvmexp.freemin ||
+	    ((uvmexp.free - BUFPAGES_DEFICIT) < uvmexp.freetarg &&
+	    (uvmexp.inactive + BUFPAGES_INACT) < uvmexp.inactarg))
+		wakeup(&uvm.pagedaemon);
+
+	/*
+	 * fail if any of these conditions is true:
+	 * [1]  there really are no free pages, or
+	 * [2]  only kernel "reserved" pages remain and
+	 *        the UVM_PLA_USERESERVE flag wasn't used.
+	 * [3]  only pagedaemon "reserved" pages remain and
+	 *        the requestor isn't the pagedaemon nor the syncer.
+	 */
+	if ((uvmexp.free <= uvmexp.reserve_kernel) &&
+	    !(flags & UVM_PLA_USERESERVE)) {
+		uvm_unlock_fpageq();
+		return ENOMEM;
+	}
+
+	if ((uvmexp.free <= (uvmexp.reserve_pagedaemon + count)) &&
+	    (curproc != uvm.pagedaemon_proc) && (curproc != syncerproc)) {
+		uvm_unlock_fpageq();
+		if (flags & UVM_PLA_WAITOK) {
+			uvm_wait("uvm_pmr_getpages");
+			goto again;
+		}
+		return ENOMEM;
+	}
+
 retry:		/* Return point after sleeping. */
 	fcount = 0;
 	fnsegs = 0;
diff --git a/sys/uvm/uvmexp.h b/sys/uvm/uvmexp.h
index e3603b84d50..270db6b9184 100644
--- a/sys/uvm/uvmexp.h
+++ b/sys/uvm/uvmexp.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: uvmexp.h,v 1.5 2020/04/23 07:57:27 mpi Exp $	*/
+/*	$OpenBSD: uvmexp.h,v 1.6 2020/12/01 13:56:22 mpi Exp $	*/
 
 #ifndef	_UVM_UVMEXP_
 #define	_UVM_UVMEXP_
@@ -39,6 +39,10 @@
 /*
  * uvmexp: global data structures that are exported to parts of the kernel
  * other than the vm system.
+ *
+ *  Locks used to protect struct members in this file:
+ *	I	immutable after creation
+ *	F	uvm_lock_fpageq
  */
 struct uvmexp {
 	/* vm_page constants */
@@ -47,16 +51,16 @@ struct uvmexp {
 	int pageshift;  /* page shift */
 
 	/* vm_page counters */
-	int npages;     /* number of pages we manage */
-	int free;       /* number of free pages */
+	int npages;     /* [I] number of pages we manage */
+	int free;       /* [F] number of free pages */
 	int active;     /* number of active pages */
 	int inactive;   /* number of pages that we free'd but may want back */
 	int paging;	/* number of pages in the process of being paged out */
 	int wired;      /* number of wired pages */
 
-	int zeropages;		/* number of zero'd pages */
-	int reserve_pagedaemon; /* number of pages reserved for pagedaemon */
-	int reserve_kernel;	/* number of pages reserved for kernel */
+	int zeropages;		/* [F] number of zero'd pages */
+	int reserve_pagedaemon; /* [I] # of pages reserved for pagedaemon */
+	int reserve_kernel;	/* [I] # of pages reserved for kernel */
 	int unused01;		/* formerly anonpages */
 	int vnodepages;		/* XXX # of pages used by vnode page cache */
 	int vtextpages;		/* XXX # of pages used by vtext vnodes */