Bring back Mickey's UVM anon change. Testing by thib@, beck@ and

ckuethe@ for a while. Okay beck@, "it is good timing" deraadt@.

14 files changed, 212 insertions, 276 deletions

diff --git a/sys/uvm/uvm.h b/sys/uvm/uvm.h
index 87194aeee40..3cfb93737f8 100644
--- a/sys/uvm/uvm.h
+++ b/sys/uvm/uvm.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: uvm.h,v 1.20 2006/07/13 22:51:26 deraadt Exp $	*/
+/*	$OpenBSD: uvm.h,v 1.21 2007/06/18 21:51:15 pedro Exp $	*/
 /*	$NetBSD: uvm.h,v 1.24 2000/11/27 08:40:02 chs Exp $	*/
 
 /*
@@ -102,10 +102,6 @@ struct uvm {
 	int page_hashmask;		/* hash mask */
 	simple_lock_data_t hashlock;	/* lock on page_hash array */
 
-	/* anon stuff */
-	struct vm_anon *afree;		/* anon free list */
-	simple_lock_data_t afreelock; 	/* lock on anon free list */
-
 	/* static kernel map entry pool */
 	vm_map_entry_t kentry_free;	/* free page pool */
 	simple_lock_data_t kentry_lock;
diff --git a/sys/uvm/uvm_amap.c b/sys/uvm/uvm_amap.c
index 4c14ffd1998..ee105ed3b13 100644
--- a/sys/uvm/uvm_amap.c
+++ b/sys/uvm/uvm_amap.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: uvm_amap.c,v 1.38 2007/06/01 20:10:04 tedu Exp $	*/
+/*	$OpenBSD: uvm_amap.c,v 1.39 2007/06/18 21:51:15 pedro Exp $	*/
 /*	$NetBSD: uvm_amap.c,v 1.27 2000/11/25 06:27:59 chs Exp $	*/
 
 /*
@@ -65,11 +65,27 @@
 
 struct pool uvm_amap_pool;
 
+LIST_HEAD(, vm_amap) amap_list;
+
 /*
  * local functions
  */
 
 static struct vm_amap *amap_alloc1(int, int, int);
+static __inline void amap_list_insert(struct vm_amap *);
+static __inline void amap_list_remove(struct vm_amap *);   
+
+static __inline void
+amap_list_insert(struct vm_amap *amap)
+{
+	LIST_INSERT_HEAD(&amap_list, amap, am_list);
+}
+
+static __inline void
+amap_list_remove(struct vm_amap *amap)
+{ 
+	LIST_REMOVE(amap, am_list);
+}
 
 #ifdef UVM_AMAP_PPREF
 /*
@@ -229,9 +245,11 @@ amap_alloc(vaddr_t sz, vaddr_t padsz, int waitf)
 	AMAP_B2SLOT(padslots, padsz);
 
 	amap = amap_alloc1(slots, padslots, waitf);
-	if (amap)
+	if (amap) {
 		memset(amap->am_anon, 0,
 		    amap->am_maxslot * sizeof(struct vm_anon *));
+		amap_list_insert(amap);
+	}
 
 	UVMHIST_LOG(maphist,"<- done, amap = %p, sz=%lu", amap, sz, 0, 0);
 	return(amap);
@@ -250,6 +268,7 @@ amap_free(struct vm_amap *amap)
 	UVMHIST_FUNC("amap_free"); UVMHIST_CALLED(maphist);
 
 	KASSERT(amap->am_ref == 0 && amap->am_nused == 0);
+	KASSERT((amap->am_flags & AMAP_SWAPOFF) == 0);
 
 	free(amap->am_slots, M_UVMAMAP);
 	free(amap->am_bckptr, M_UVMAMAP);
@@ -466,8 +485,8 @@ amap_share_protect(struct vm_map_entry *entry, vm_prot_t prot)
 		for (lcv = entry->aref.ar_pageoff ; lcv < stop ; lcv++) {
 			if (amap->am_anon[lcv] == NULL)
 				continue;
-			if (amap->am_anon[lcv]->u.an_page != NULL)
-				pmap_page_protect(amap->am_anon[lcv]->u.an_page,
+			if (amap->am_anon[lcv]->an_page != NULL)
+				pmap_page_protect(amap->am_anon[lcv]->an_page,
 						  prot);
 		}
 		return;
@@ -478,8 +497,8 @@ amap_share_protect(struct vm_map_entry *entry, vm_prot_t prot)
 		slot = amap->am_slots[lcv];
 		if (slot < entry->aref.ar_pageoff || slot >= stop)
 			continue;
-		if (amap->am_anon[slot]->u.an_page != NULL)
-			pmap_page_protect(amap->am_anon[slot]->u.an_page, prot);
+		if (amap->am_anon[slot]->an_page != NULL)
+			pmap_page_protect(amap->am_anon[slot]->an_page, prot);
 	}
 	return;
 }
@@ -487,7 +506,7 @@ amap_share_protect(struct vm_map_entry *entry, vm_prot_t prot)
 /*
  * amap_wipeout: wipeout all anon's in an amap; then free the amap!
  *
- * => called from amap_unref when the final reference to an amap is 
+ * => called from amap_unref when the final reference to an amap is
  *	discarded (i.e. when reference count == 1)
  * => the amap should be locked (by the caller)
  */
@@ -500,13 +519,23 @@ amap_wipeout(struct vm_amap *amap)
 	UVMHIST_FUNC("amap_wipeout"); UVMHIST_CALLED(maphist);
 	UVMHIST_LOG(maphist,"(amap=%p)", amap, 0,0,0);
 
+	KASSERT(amap->am_ref == 0);
+
+	if (__predict_false((amap->am_flags & AMAP_SWAPOFF) != 0)) {
+		/*
+		 * amap_swap_off will call us again.
+		 */
+		return;
+	}
+	amap_list_remove(amap);
+
 	for (lcv = 0 ; lcv < amap->am_nused ; lcv++) {
 		int refs;
 
 		slot = amap->am_slots[lcv];
 		anon = amap->am_anon[slot];
 
-		if (anon == NULL || anon->an_ref == 0) 
+		if (anon == NULL || anon->an_ref == 0)
 			panic("amap_wipeout: corrupt amap");
 
 		simple_lock(&anon->an_lock); /* lock anon */
@@ -687,6 +716,8 @@ amap_copy(struct vm_map *map, struct vm_map_entry *entry, int waitf,
 	entry->aref.ar_amap = amap;
 	entry->etype &= ~UVM_ET_NEEDSCOPY;
 
+	amap_list_insert(amap);
+
 	/*
 	 * done!
 	 */
@@ -738,7 +769,7 @@ ReStart:
 		slot = amap->am_slots[lcv];
 		anon = amap->am_anon[slot];
 		simple_lock(&anon->an_lock);
-		pg = anon->u.an_page;
+		pg = anon->an_page;
 
 		/*
 		 * page must be resident since parent is wired
@@ -1036,3 +1067,79 @@ amap_wiperange(struct vm_amap *amap, int slotoff, int slots)
 }
 
 #endif
+
+/*
+ * amap_swap_off: pagein anonymous pages in amaps and drop swap slots.
+ *
+ * => called with swap_syscall_lock held.
+ * => note that we don't always traverse all anons.
+ *    eg. amaps being wiped out, released anons.
+ * => return TRUE if failed.
+ */
+
+boolean_t
+amap_swap_off(int startslot, int endslot)
+{
+	struct vm_amap *am;
+	struct vm_amap *am_next;
+	struct vm_amap marker_prev;
+	struct vm_amap marker_next;
+	boolean_t rv = FALSE;
+
+#if defined(DIAGNOSTIC)
+	memset(&marker_prev, 0, sizeof(marker_prev));
+	memset(&marker_next, 0, sizeof(marker_next));
+#endif /* defined(DIAGNOSTIC) */
+
+	for (am = LIST_FIRST(&amap_list); am != NULL && !rv; am = am_next) {
+		int i;
+
+		LIST_INSERT_BEFORE(am, &marker_prev, am_list);
+		LIST_INSERT_AFTER(am, &marker_next, am_list);
+
+		if (am->am_nused <= 0) {
+			goto next;
+		}
+
+		for (i = 0; i < am->am_nused; i++) {
+			int slot;
+			int swslot;
+			struct vm_anon *anon;
+
+			slot = am->am_slots[i];
+			anon = am->am_anon[slot];
+			simple_lock(&anon->an_lock);
+
+			swslot = anon->an_swslot;
+			if (swslot < startslot || endslot <= swslot) {
+				simple_unlock(&anon->an_lock);
+				continue;
+			}
+
+			am->am_flags |= AMAP_SWAPOFF;
+
+			rv = uvm_anon_pagein(anon);
+
+			am->am_flags &= ~AMAP_SWAPOFF;
+			if (amap_refs(am) == 0) {
+				amap_wipeout(am);
+				am = NULL;
+				break;
+			}
+			if (rv) {
+				break;
+			}
+			i = 0;
+		}
+
+next:
+		KASSERT(LIST_NEXT(&marker_prev, am_list) == &marker_next ||
+		    LIST_NEXT(LIST_NEXT(&marker_prev, am_list), am_list) ==
+		    &marker_next);
+		am_next = LIST_NEXT(&marker_next, am_list);
+		LIST_REMOVE(&marker_prev, am_list);
+		LIST_REMOVE(&marker_next, am_list);
+	}
+
+	return rv;
+}
diff --git a/sys/uvm/uvm_amap.h b/sys/uvm/uvm_amap.h
index 10a619fbd6d..6cdc2a9bae1 100644
--- a/sys/uvm/uvm_amap.h
+++ b/sys/uvm/uvm_amap.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: uvm_amap.h,v 1.16 2007/05/31 21:20:30 thib Exp $	*/
+/*	$OpenBSD: uvm_amap.h,v 1.17 2007/06/18 21:51:15 pedro Exp $	*/
 /*	$NetBSD: uvm_amap.h,v 1.14 2001/02/18 21:19:08 chs Exp $	*/
 
 /*
@@ -116,6 +116,7 @@ AMAP_INLINE				/* drop reference to an amap */
 void		amap_unref(struct vm_amap *, vaddr_t, vsize_t, int);
 					/* remove all anons from amap */
 void		amap_wipeout(struct vm_amap *);
+boolean_t	amap_swap_off(int, int);
 
 /*
  * amap flag values
@@ -123,6 +124,7 @@ void		amap_wipeout(struct vm_amap *);
 
 #define AMAP_SHARED	0x1	/* amap is shared */
 #define AMAP_REFALL	0x2	/* amap_ref: reference entire amap */
+#define AMAP_SWAPOFF	0x4	/* amap_swap_off() is in progress */
 
 #endif /* _KERNEL */
 
@@ -157,6 +159,7 @@ struct vm_amap {
 #ifdef UVM_AMAP_PPREF
 	int *am_ppref;		/* per page reference count (if !NULL) */
 #endif
+	LIST_ENTRY(vm_amap) am_list;
 };
 
 /*
diff --git a/sys/uvm/uvm_amap_i.h b/sys/uvm/uvm_amap_i.h
index 2ed912ab6c0..16fe523e572 100644
--- a/sys/uvm/uvm_amap_i.h
+++ b/sys/uvm/uvm_amap_i.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: uvm_amap_i.h,v 1.17 2007/05/31 21:20:30 thib Exp $	*/
+/*	$OpenBSD: uvm_amap_i.h,v 1.18 2007/06/18 21:51:15 pedro Exp $	*/
 /*	$NetBSD: uvm_amap_i.h,v 1.15 2000/11/25 06:27:59 chs Exp $	*/
 
 /*
@@ -135,9 +135,9 @@ amap_add(aref, offset, anon, replace)
 
 		if (amap->am_anon[slot] == NULL)
 			panic("amap_add: replacing null anon");
-		if (amap->am_anon[slot]->u.an_page != NULL && 
+		if (amap->am_anon[slot]->an_page != NULL && 
 		    (amap->am_flags & AMAP_SHARED) != 0) {
-			pmap_page_protect(amap->am_anon[slot]->u.an_page,
+			pmap_page_protect(amap->am_anon[slot]->an_page,
 			    VM_PROT_NONE);
 			/*
 			 * XXX: suppose page is supposed to be wired somewhere?
@@ -249,7 +249,7 @@ amap_unref(amap, offset, len, all)
 	 * if we are the last reference, free the amap and return.
 	 */
 
-	if (amap->am_ref == 1) {
+	if (amap->am_ref-- == 1) {
 		amap_wipeout(amap);	/* drops final ref and frees */
 		UVMHIST_LOG(maphist,"<- done (was last ref)!", 0, 0, 0, 0);
 		return;			/* no need to unlock */
@@ -258,8 +258,6 @@ amap_unref(amap, offset, len, all)
 	/*
 	 * otherwise just drop the reference count(s)
 	 */
-
-	amap->am_ref--;
 	if (amap->am_ref == 1 && (amap->am_flags & AMAP_SHARED) != 0)
 		amap->am_flags &= ~AMAP_SHARED;	/* clear shared flag */
 #ifdef UVM_AMAP_PPREF
diff --git a/sys/uvm/uvm_anon.c b/sys/uvm/uvm_anon.c
index 47a8b81caf1..673eddc4c2d 100644
--- a/sys/uvm/uvm_anon.c
+++ b/sys/uvm/uvm_anon.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: uvm_anon.c,v 1.27 2007/04/13 18:57:49 art Exp $	*/
+/*	$OpenBSD: uvm_anon.c,v 1.28 2007/06/18 21:51:15 pedro Exp $	*/
 /*	$NetBSD: uvm_anon.c,v 1.10 2000/11/25 06:27:59 chs Exp $	*/
 
 /*
@@ -47,22 +47,7 @@
 #include <uvm/uvm.h>
 #include <uvm/uvm_swap.h>
 
-/*
- * anonblock_list: global list of anon blocks,
- * locked by swap_syscall_lock (since we never remove
- * anything from this list and we only add to it via swapctl(2)).
- */
-
-struct uvm_anonblock {
-	LIST_ENTRY(uvm_anonblock) list;
-	int count;
-	struct vm_anon *anons;
-};
-static LIST_HEAD(anonlist, uvm_anonblock) anonblock_list;
-
-
-static boolean_t anon_pagein(struct vm_anon *);
-
+struct pool uvm_anon_pool;
 
 /*
  * allocate anons
@@ -70,83 +55,9 @@ static boolean_t anon_pagein(struct vm_anon *);
 void
 uvm_anon_init()
 {
-	int nanon = uvmexp.free - (uvmexp.free / 16); /* XXXCDC ??? */
-
-	simple_lock_init(&uvm.afreelock);
-	LIST_INIT(&anonblock_list);
-
-	/*
-	 * Allocate the initial anons.
-	 */
-	uvm_anon_add(nanon);
-}
-
-/*
- * add some more anons to the free pool.  called when we add
- * more swap space.
- *
- * => swap_syscall_lock should be held (protects anonblock_list).
- */
-int
-uvm_anon_add(count)
-	int	count;
-{
-	struct uvm_anonblock *anonblock;
-	struct vm_anon *anon;
-	int lcv, needed;
-
-	simple_lock(&uvm.afreelock);
-	uvmexp.nanonneeded += count;
-	needed = uvmexp.nanonneeded - uvmexp.nanon;
-	simple_unlock(&uvm.afreelock);
-
-	if (needed <= 0) {
-		return 0;
-	}
- 
-	anon = (void *)uvm_km_alloc(kernel_map, sizeof(*anon) * needed);
-
-	/* XXX Should wait for VM to free up. */
-	if (anon == NULL) {
-		printf("uvm_anon_add: can not allocate %d anons\n", needed);
-		panic("uvm_anon_add");
-	}
-
-	MALLOC(anonblock, void *, sizeof(*anonblock), M_UVMAMAP, M_WAITOK);
-
-	anonblock->count = needed;
-	anonblock->anons = anon;
-	LIST_INSERT_HEAD(&anonblock_list, anonblock, list);
-	memset(anon, 0, sizeof(*anon) * needed);
- 
-	simple_lock(&uvm.afreelock);
-	uvmexp.nanon += needed;
-	uvmexp.nfreeanon += needed;
-	for (lcv = 0; lcv < needed; lcv++) {
-		simple_lock_init(&anon->an_lock);
-		anon[lcv].u.an_nxt = uvm.afree;
-		uvm.afree = &anon[lcv];
-		simple_lock_init(&uvm.afree->an_lock);
-	}
-	simple_unlock(&uvm.afreelock);
-	return 0;
-}
-
-/*
- * remove anons from the free pool.
- */
-void
-uvm_anon_remove(count)
-	int count;
-{
-	/*
-	 * we never actually free any anons, to avoid allocation overhead.
-	 * XXX someday we might want to try to free anons.
-	 */
-
-	simple_lock(&uvm.afreelock);
-	uvmexp.nanonneeded -= count;
-	simple_unlock(&uvm.afreelock);
+	pool_init(&uvm_anon_pool, sizeof(struct vm_anon), 0, 0, 0, "anonpl",
+	    &pool_allocator_nointr);
+	pool_sethiwat(&uvm_anon_pool, uvmexp.free / 16);
 }
 
 /*
@@ -155,19 +66,17 @@ uvm_anon_remove(count)
 struct vm_anon *
 uvm_analloc()
 {
-	struct vm_anon *a;
-
-	simple_lock(&uvm.afreelock);
-	a = uvm.afree;
-	if (a) {
-		uvm.afree = a->u.an_nxt;
-		uvmexp.nfreeanon--;
-		a->an_ref = 1;
-		a->an_swslot = 0;
-		a->u.an_page = NULL;		/* so we can free quickly */
+	struct vm_anon *anon;
+
+	anon = pool_get(&uvm_anon_pool, PR_NOWAIT);
+	if (anon) {
+		simple_lock_init(&anon->an_lock);
+		anon->an_ref = 1;
+		anon->an_page = NULL;
+		anon->an_swslot = 0;
+		simple_lock(&anon->an_lock);
 	}
-	simple_unlock(&uvm.afreelock);
-	return(a);
+	return(anon);
 }
 
 /*
@@ -190,7 +99,7 @@ uvm_anfree(anon)
 	 * get page
 	 */
 
-	pg = anon->u.an_page;
+	pg = anon->an_page;
 
 	/*
 	 * if there is a resident page and it is loaned, then anon may not
@@ -257,11 +166,10 @@ uvm_anfree(anon)
 	 * now that we've stripped the data areas from the anon, free the anon
 	 * itself!
 	 */
-	simple_lock(&uvm.afreelock);
-	anon->u.an_nxt = uvm.afree;
-	uvm.afree = anon;
-	uvmexp.nfreeanon++;
-	simple_unlock(&uvm.afreelock);
+	KASSERT(anon->an_page == NULL);
+	KASSERT(anon->an_swslot == 0);
+
+	pool_put(&uvm_anon_pool, anon);
 	UVMHIST_LOG(maphist,"<- done!",0,0,0,0);
 }
 
@@ -284,7 +192,7 @@ uvm_anon_dropswap(anon)
 	uvm_swap_free(anon->an_swslot, 1);
 	anon->an_swslot = 0;
 
-	if (anon->u.an_page == NULL) {
+	if (anon->an_page == NULL) {
 		/* this page is no longer only in swap. */
 		simple_lock(&uvm.swap_data_lock);
 		uvmexp.swpgonly--;
@@ -325,7 +233,7 @@ uvm_anon_lockloanpg(anon)
 	 * not produce an incorrect result.
 	 */
 
-	while (((pg = anon->u.an_page) != NULL) && pg->loan_count != 0) {
+	while (((pg = anon->an_page) != NULL) && pg->loan_count != 0) {
 
 		/*
 		 * quickly check to see if the page has an object before
@@ -387,73 +295,6 @@ uvm_anon_lockloanpg(anon)
 	return(pg);
 }
 
-
-
-/*
- * page in every anon that is paged out to a range of swslots.
- * 
- * swap_syscall_lock should be held (protects anonblock_list).
- */
-
-boolean_t
-anon_swap_off(startslot, endslot)
-	int startslot, endslot;
-{
-	struct uvm_anonblock *anonblock;
-
-	for (anonblock = LIST_FIRST(&anonblock_list);
-	     anonblock != NULL;
-	     anonblock = LIST_NEXT(anonblock, list)) {
-		int i;
-
-		/*
-		 * loop thru all the anons in the anonblock,
-		 * paging in where needed.
-		 */
-
-		for (i = 0; i < anonblock->count; i++) {
-			struct vm_anon *anon = &anonblock->anons[i];
-			int slot;
-
-			/*
-			 * lock anon to work on it.
-			 */
-
-			simple_lock(&anon->an_lock);
-
-			/*
-			 * is this anon's swap slot in range?
-			 */
-
-			slot = anon->an_swslot;
-			if (slot >= startslot && slot < endslot) {
-				boolean_t rv;
-
-				/*
-				 * yup, page it in.
-				 */
-
-				/* locked: anon */
-				rv = anon_pagein(anon);
-				/* unlocked: anon */
-
-				if (rv) {
-					return rv;
-				}
-			} else {
-
-				/*
-				 * nope, unlock and proceed.
-				 */
-
-				simple_unlock(&anon->an_lock);
-			}
-		}
-	}
-	return FALSE;
-}
-
-
 /*
  * fetch an anon's page.
  *
@@ -461,8 +302,8 @@ anon_swap_off(startslot, endslot)
  * => returns TRUE if pagein was aborted due to lack of memory.
  */
 
-static boolean_t
-anon_pagein(anon)
+boolean_t
+uvm_anon_pagein(anon)
 	struct vm_anon *anon;
 {
 	struct vm_page *pg;
@@ -504,7 +345,7 @@ anon_pagein(anon)
 	 * mark it as dirty, clear its swslot and un-busy it.
 	 */
 
-	pg = anon->u.an_page;
+	pg = anon->an_page;
 	uobj = pg->uobject;
 	uvm_swap_free(anon->an_swslot, 1);
 	anon->an_swslot = 0;
diff --git a/sys/uvm/uvm_anon.h b/sys/uvm/uvm_anon.h
index 551802048bf..a454b5572c6 100644
--- a/sys/uvm/uvm_anon.h
+++ b/sys/uvm/uvm_anon.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: uvm_anon.h,v 1.13 2006/07/13 22:51:26 deraadt Exp $	*/
+/*	$OpenBSD: uvm_anon.h,v 1.14 2007/06/18 21:51:15 pedro Exp $	*/
 /*	$NetBSD: uvm_anon.h,v 1.13 2000/12/27 09:17:04 chs Exp $	*/
 
 /*
@@ -49,24 +49,19 @@
  */
 
 struct vm_anon {
+	struct vm_page *an_page;	/* if in RAM [an_lock] */
 	int an_ref;			/* reference count [an_lock] */
-	simple_lock_data_t an_lock;	/* lock for an_ref */
-	union {
-		struct vm_anon *an_nxt;	/* if on free list [afreelock] */
-		struct vm_page *an_page;/* if in RAM [an_lock] */
-	} u;
 	int an_swslot;		/* drum swap slot # (if != 0) 
 				   [an_lock.  also, it is ok to read
 				   an_swslot if we hold an_page PG_BUSY] */
+	simple_lock_data_t an_lock;	/* lock for an_ref */
 };
 
 /*
- * a pool of vm_anon data structures is allocated and put on a global
- * free list at boot time.  vm_anon's on the free list use "an_nxt" as
- * a pointer to the next item on the free list.  for active vm_anon's
- * the data can be in one of the following state: [1] in a vm_page
- * with no backing store allocated yet, [2] in a vm_page with backing
- * store allocated, or [3] paged out to backing store (no vm_page).
+ * for active vm_anon's the data can be in one of the following state:
+ * [1] in a vm_page with no backing store allocated yet, [2] in a vm_page
+ * with backing store allocated, or [3] paged out to backing store
+ * (no vm_page).
  *
  * for pageout in case [2]: if the page has been modified then we must
  * flush it out to backing store, otherwise we can just dump the
@@ -102,11 +97,9 @@ struct vm_aref {
 struct vm_anon *uvm_analloc(void);
 void uvm_anfree(struct vm_anon *);
 void uvm_anon_init(void);
-int uvm_anon_add(int);
-void uvm_anon_remove(int);
 struct vm_page *uvm_anon_lockloanpg(struct vm_anon *);
 void uvm_anon_dropswap(struct vm_anon *);
-boolean_t anon_swap_off(int, int);
+boolean_t uvm_anon_pagein(struct vm_anon *);
 #endif /* _KERNEL */
 
 #endif /* _UVM_UVM_ANON_H_ */
diff --git a/sys/uvm/uvm_fault.c b/sys/uvm/uvm_fault.c
index 38a6a08c8b1..aacf177a865 100644
--- a/sys/uvm/uvm_fault.c
+++ b/sys/uvm/uvm_fault.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: uvm_fault.c,v 1.48 2007/05/31 21:20:30 thib Exp $	*/
+/*	$OpenBSD: uvm_fault.c,v 1.49 2007/06/18 21:51:15 pedro Exp $	*/
 /*	$NetBSD: uvm_fault.c,v 1.51 2000/08/06 00:22:53 thorpej Exp $	*/
 
 /*
@@ -200,7 +200,7 @@ uvmfault_anonflush(anons, n)
 		if (anons[lcv] == NULL)
 			continue;
 		simple_lock(&anons[lcv]->an_lock);
-		pg = anons[lcv]->u.an_page;
+		pg = anons[lcv]->an_page;
 		if (pg && (pg->pg_flags & PG_BUSY) == 0 && pg->loan_count == 0) {
 			uvm_lock_pageq();
 			if (pg->wire_count == 0) {
@@ -306,7 +306,7 @@ uvmfault_anonget(ufi, amap, anon)
 	result = 0;		/* XXX shut up gcc */
 	uvmexp.fltanget++;
         /* bump rusage counters */
-	if (anon->u.an_page)
+	if (anon->an_page)
 		curproc->p_addr->u_stats.p_ru.ru_minflt++;
 	else
 		curproc->p_addr->u_stats.p_ru.ru_majflt++;
@@ -318,7 +318,7 @@ uvmfault_anonget(ufi, amap, anon)
 	while (1) {
 
 		we_own = FALSE;		/* TRUE if we set PG_BUSY on a page */
-		pg = anon->u.an_page;
+		pg = anon->an_page;
 
 		/*
 		 * if there is a resident page and it is loaned, then anon
@@ -462,7 +462,7 @@ uvmfault_anonget(ufi, amap, anon)
 				KASSERT(result != VM_PAGER_PEND);
 
 				/* remove page from anon */
-				anon->u.an_page = NULL;
+				anon->an_page = NULL;
 
 				/*
 				 * remove the swap slot from the anon
@@ -817,14 +817,14 @@ ReFault:
 		anon = anons[lcv];
 		simple_lock(&anon->an_lock);
 		/* ignore loaned pages */
-		if (anon->u.an_page && anon->u.an_page->loan_count == 0 &&
-			(anon->u.an_page->pg_flags & (PG_RELEASED|PG_BUSY)) == 0) {
+		if (anon->an_page && anon->an_page->loan_count == 0 &&
+		    (anon->an_page->pg_flags & (PG_RELEASED|PG_BUSY)) == 0) {
 			uvm_lock_pageq();
-			uvm_pageactivate(anon->u.an_page);	/* reactivate */
+			uvm_pageactivate(anon->an_page);	/* reactivate */
 			uvm_unlock_pageq();
 			UVMHIST_LOG(maphist,
 			    "  MAPPING: n anon: pm=%p, va=0x%lx, pg=%p",
-			    ufi.orig_map->pmap, currva, anon->u.an_page, 0);
+			    ufi.orig_map->pmap, currva, anon->an_page, 0);
 			uvmexp.fltnamap++;
 
 			/*
@@ -834,7 +834,7 @@ ReFault:
 			 */
 
 			(void) pmap_enter(ufi.orig_map->pmap, currva,
-			    VM_PAGE_TO_PHYS(anon->u.an_page),
+			    VM_PAGE_TO_PHYS(anon->an_page),
 			    (anon->an_ref > 1) ? (enter_prot & ~VM_PROT_WRITE) :
 			    enter_prot,
 			    PMAP_CANFAIL |
@@ -1069,7 +1069,7 @@ ReFault:
 	 * uobj is non null if the page is on loan from an object (i.e. uobj)
 	 */
 
-	uobj = anon->u.an_page->uobject;	/* locked by anonget if !NULL */
+	uobj = anon->an_page->uobject;	/* locked by anonget if !NULL */
 
 	/* locked: maps(read), amap, anon, uobj(if one) */
 
@@ -1077,7 +1077,7 @@ ReFault:
 	 * special handling for loaned pages 
 	 */
 
-	if (anon->u.an_page->loan_count) {
+	if (anon->an_page->loan_count) {
 
 		if ((access_type & VM_PROT_WRITE) == 0) {
 			
@@ -1119,19 +1119,19 @@ ReFault:
 				 * (if any)
 				 */
 				/* copy old -> new */
-				uvm_pagecopy(anon->u.an_page, pg);
+				uvm_pagecopy(anon->an_page, pg);
 
 				/* force reload */
-				pmap_page_protect(anon->u.an_page,
+				pmap_page_protect(anon->an_page,
 						  VM_PROT_NONE);
 				uvm_lock_pageq();	  /* KILL loan */
 				if (uobj)
 					/* if we were loaning */
-					anon->u.an_page->loan_count--;
-				anon->u.an_page->uanon = NULL;
+					anon->an_page->loan_count--;
+				anon->an_page->uanon = NULL;
 				/* in case we owned */
 				atomic_clearbits_int(
-				    &anon->u.an_page->pg_flags, PQ_ANON);
+				    &anon->an_page->pg_flags, PQ_ANON);
 				uvm_pageactivate(pg);
 				uvm_unlock_pageq();
 				if (uobj) {
@@ -1140,7 +1140,7 @@ ReFault:
 				}
 
 				/* install new page in anon */
-				anon->u.an_page = pg;
+				anon->an_page = pg;
 				pg->uanon = anon;
 				atomic_setbits_int(&pg->pg_flags, PQ_ANON);
 				atomic_clearbits_int(&pg->pg_flags,
@@ -1195,7 +1195,7 @@ ReFault:
 
 		/* got all resources, replace anon with nanon */
 
-		uvm_pagecopy(oanon->u.an_page, pg);	/* pg now !PG_CLEAN */
+		uvm_pagecopy(oanon->an_page, pg);	/* pg now !PG_CLEAN */
 		/* un-busy! new page */
 		atomic_clearbits_int(&pg->pg_flags, PG_BUSY|PG_FAKE);
 		UVM_PAGE_OWN(pg, NULL);
@@ -1215,7 +1215,7 @@ ReFault:
 
 		uvmexp.flt_anon++;
 		oanon = anon;		/* old, locked anon is same as anon */
-		pg = anon->u.an_page;
+		pg = anon->an_page;
 		if (anon->an_ref > 1)     /* disallow writes to ref > 1 anons */
 			enter_prot = enter_prot & ~VM_PROT_WRITE;
 
diff --git a/sys/uvm/uvm_init.c b/sys/uvm/uvm_init.c
index 448baed5021..611f6fa1e77 100644
--- a/sys/uvm/uvm_init.c
+++ b/sys/uvm/uvm_init.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: uvm_init.c,v 1.15 2007/05/09 11:23:35 art Exp $	*/
+/*	$OpenBSD: uvm_init.c,v 1.16 2007/06/18 21:51:15 pedro Exp $	*/
 /*	$NetBSD: uvm_init.c,v 1.14 2000/06/27 17:29:23 mrg Exp $	*/
 
 /*
@@ -134,11 +134,10 @@ uvm_init()
 	uvm_pager_init();
 
 	/*
-	 * step 8: init anonymous memory systems (both amap and anons)
+	 * step 8: init anonymous memory system
 	 */
 
 	amap_init();		/* init amap module */
-	uvm_anon_init();	/* allocate initial anons */
 
 	/*
 	 * the VM system is now up!  now that malloc is up we can resize the
@@ -170,8 +169,7 @@ uvm_init()
 		panic("uvm_init: cannot reserve dead beef @0x%x\n", DEADBEEF1);
 #endif
 	/*
-	 * done!
+	 * init anonymous memory systems
 	 */
-
-	return;
+	uvm_anon_init();
 }
diff --git a/sys/uvm/uvm_loan.c b/sys/uvm/uvm_loan.c
index c1859f02da4..84cbd82ae84 100644
--- a/sys/uvm/uvm_loan.c
+++ b/sys/uvm/uvm_loan.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: uvm_loan.c,v 1.27 2007/05/31 21:20:30 thib Exp $	*/
+/*	$OpenBSD: uvm_loan.c,v 1.28 2007/06/18 21:51:15 pedro Exp $	*/
 /*	$NetBSD: uvm_loan.c,v 1.22 2000/06/27 17:29:25 mrg Exp $	*/
 
 /*
@@ -332,7 +332,7 @@ uvm_loananon(ufi, output, flags, anon)
 	 */
 	if (flags & UVM_LOAN_TOANON) {
 		simple_lock(&anon->an_lock);
-		pg = anon->u.an_page;
+		pg = anon->an_page;
 		if (pg && (pg->pg_flags & PQ_ANON) != 0 && anon->an_ref == 1)
 			/* read protect it */
 			pmap_page_protect(pg, VM_PROT_READ);
@@ -377,7 +377,7 @@ uvm_loananon(ufi, output, flags, anon)
 	 * we have the page and its owner locked: do the loan now.
 	 */
 
-	pg = anon->u.an_page;
+	pg = anon->an_page;
 	uvm_lock_pageq();
 	if (pg->loan_count == 0)
 		pmap_page_protect(pg, VM_PROT_READ);
@@ -580,7 +580,7 @@ uvm_loanuobj(ufi, output, flags, va)
 		uvmfault_unlockall(ufi, amap, uobj, NULL);
 		return(-1);
 	}
-	anon->u.an_page = pg;
+	anon->an_page = pg;
 	pg->uanon = anon;
 	uvm_lock_pageq();
 	if (pg->loan_count == 0)
diff --git a/sys/uvm/uvm_map.c b/sys/uvm/uvm_map.c
index 0152191b811..221daf57db7 100644
--- a/sys/uvm/uvm_map.c
+++ b/sys/uvm/uvm_map.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: uvm_map.c,v 1.95 2007/06/13 13:32:26 art Exp $	*/
+/*	$OpenBSD: uvm_map.c,v 1.96 2007/06/18 21:51:15 pedro Exp $	*/
 /*	$NetBSD: uvm_map.c,v 1.86 2000/11/27 08:40:03 chs Exp $	*/
 
 /* 
@@ -2927,7 +2927,7 @@ uvm_map_clean(struct vm_map *map, vaddr_t start, vaddr_t end, int flags)
 
 			simple_lock(&anon->an_lock);
 
-			pg = anon->u.an_page;
+			pg = anon->an_page;
 			if (pg == NULL) {
 				simple_unlock(&anon->an_lock);
 				continue;
@@ -3730,9 +3730,9 @@ uvm_page_printit(pg, full, pr)
 	/* cross-verify object/anon */
 	if ((pg->pg_flags & PQ_FREE) == 0) {
 		if (pg->pg_flags & PQ_ANON) {
-			if (pg->uanon == NULL || pg->uanon->u.an_page != pg)
+			if (pg->uanon == NULL || pg->uanon->an_page != pg)
 			    (*pr)("  >>> ANON DOES NOT POINT HERE <<< (%p)\n",
-				(pg->uanon) ? pg->uanon->u.an_page : NULL);
+				(pg->uanon) ? pg->uanon->an_page : NULL);
 			else
 				(*pr)("  anon backpointer is OK\n");
 		} else {
diff --git a/sys/uvm/uvm_mmap.c b/sys/uvm/uvm_mmap.c
index 7a8982a9f7d..af8cc662861 100644
--- a/sys/uvm/uvm_mmap.c
+++ b/sys/uvm/uvm_mmap.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: uvm_mmap.c,v 1.68 2007/05/31 21:20:30 thib Exp $	*/
+/*	$OpenBSD: uvm_mmap.c,v 1.69 2007/06/18 21:51:15 pedro Exp $	*/
 /*	$NetBSD: uvm_mmap.c,v 1.49 2001/02/18 21:19:08 chs Exp $	*/
 
 /*
@@ -331,7 +331,7 @@ sys_mincore(p, v, retval)
 				anon = amap_lookup(&entry->aref,
 				    start - entry->start);
 				/* Don't need to lock anon here. */
-				if (anon != NULL && anon->u.an_page != NULL) {
+				if (anon != NULL && anon->an_page != NULL) {
 					/*
 					 * Anon has the page for this entry
 					 * offset.
diff --git a/sys/uvm/uvm_page.c b/sys/uvm/uvm_page.c
index 2caca1ba938..0390a1b9e46 100644
--- a/sys/uvm/uvm_page.c
+++ b/sys/uvm/uvm_page.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: uvm_page.c,v 1.60 2007/05/18 14:41:55 art Exp $	*/
+/*	$OpenBSD: uvm_page.c,v 1.61 2007/06/18 21:51:15 pedro Exp $	*/
 /*	$NetBSD: uvm_page.c,v 1.44 2000/11/27 08:40:04 chs Exp $	*/
 
 /* 
@@ -1040,7 +1040,7 @@ uvm_pagealloc_strat(obj, off, anon, flags, strat, free_list)
 	pg->pg_flags = PG_BUSY|PG_CLEAN|PG_FAKE;
 	pg->pg_version++;
 	if (anon) {
-		anon->u.an_page = pg;
+		anon->an_page = pg;
 		atomic_setbits_int(&pg->pg_flags, PQ_ANON);
 #ifdef UBC
 		uvm_pgcnt_anon++;
@@ -1168,7 +1168,7 @@ uvm_pagefree(struct vm_page *pg)
 
 		if (saved_loan_count) 
 			return;
-	} else if (saved_loan_count && (pg->pg_flags & PQ_ANON)) {
+	} else if (saved_loan_count && pg->uanon) {
 		/*
 		 * if our page is owned by an anon and is loaned out to the
 		 * kernel then we just want to drop ownership and return.
@@ -1177,6 +1177,7 @@ uvm_pagefree(struct vm_page *pg)
 		 * page as long as we are holding PQ lock.
 		 */
 		atomic_clearbits_int(&pg->pg_flags, PQ_ANON);
+		pg->uanon->an_page = NULL;
 		pg->uanon = NULL;
 		return;
 	}
@@ -1208,11 +1209,12 @@ uvm_pagefree(struct vm_page *pg)
 		pg->wire_count = 0;
 		uvmexp.wired--;
 	}
-#ifdef UBC
 	if (pg->uanon) {
+		pg->uanon->an_page = NULL;
+#ifdef UBC
 		uvm_pgcnt_anon--;
-	}
 #endif
+	}
 
 	/*
 	 * and put on free queue
diff --git a/sys/uvm/uvm_pdaemon.c b/sys/uvm/uvm_pdaemon.c
index 4e6d7757214..1fefe67f2dd 100644
--- a/sys/uvm/uvm_pdaemon.c
+++ b/sys/uvm/uvm_pdaemon.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: uvm_pdaemon.c,v 1.32 2007/04/13 18:57:49 art Exp $	*/
+/*	$OpenBSD: uvm_pdaemon.c,v 1.33 2007/06/18 21:51:15 pedro Exp $	*/
 /*	$NetBSD: uvm_pdaemon.c,v 1.23 2000/08/20 10:24:14 bjh21 Exp $	*/
 
 /* 
@@ -545,7 +545,7 @@ uvmpd_scan_inactive(pglst)
 					KASSERT(anon->an_swslot != 0);
 
 					/* remove from object */
-					anon->u.an_page = NULL;
+					anon->an_page = NULL;
 					simple_unlock(&anon->an_lock);
 				} else {
 					/* pagefree has already removed the
@@ -860,7 +860,7 @@ uvmpd_scan_inactive(pglst)
 			if (p->pg_flags & PG_RELEASED) {
 				if (anon) {
 					/* remove page so we can get nextpg */
-					anon->u.an_page = NULL;
+					anon->an_page = NULL;
 
 					simple_unlock(&anon->an_lock);
 					uvm_anfree(anon);	/* kills anon */
diff --git a/sys/uvm/uvm_swap.c b/sys/uvm/uvm_swap.c
index 0a87cac45b0..f5baa906794 100644
--- a/sys/uvm/uvm_swap.c
+++ b/sys/uvm/uvm_swap.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: uvm_swap.c,v 1.71 2007/06/06 17:15:14 deraadt Exp $	*/
+/*	$OpenBSD: uvm_swap.c,v 1.72 2007/06/18 21:51:15 pedro Exp $	*/
 /*	$NetBSD: uvm_swap.c,v 1.40 2000/11/17 11:39:39 mrg Exp $	*/
 
 /*
@@ -1061,11 +1061,6 @@ swap_on(p, sdp)
 	 */
 	vref(vp);
 
-  	/*
-	 * add anons to reflect the new swap space
-	 */
-	uvm_anon_add(size);
-
 #ifdef UVM_SWAP_ENCRYPT
 	if (uvm_doswapencrypt)
 		uvm_swap_initcrypt(sdp, npages);
@@ -1101,6 +1096,7 @@ swap_off(p, sdp)
 	struct proc *p;
 	struct swapdev *sdp;
 {
+	int error;
 	UVMHIST_FUNC("swap_off"); UVMHIST_CALLED(pdhist);
 	UVMHIST_LOG(pdhist, "  dev=%lx", sdp->swd_dev,0,0,0);
 
@@ -1117,15 +1113,20 @@ swap_off(p, sdp)
 
 	if (uao_swap_off(sdp->swd_drumoffset,
 			 sdp->swd_drumoffset + sdp->swd_drumsize) ||
-	    anon_swap_off(sdp->swd_drumoffset,
+	    amap_swap_off(sdp->swd_drumoffset,
 			  sdp->swd_drumoffset + sdp->swd_drumsize)) {
 		
+		error = ENOMEM;
+	} else if (sdp->swd_npginuse > sdp->swd_npgbad) {
+		error = EBUSY;
+	}
+
+	if (error) {
 		simple_lock(&uvm.swap_data_lock);
 		sdp->swd_flags |= SWF_ENABLE;
 		simple_unlock(&uvm.swap_data_lock);
-		return ENOMEM;
+		return (error);
 	}
-	KASSERT(sdp->swd_npginuse == sdp->swd_npgbad);
 
 	/*
 	 * done with the vnode and saved creds.
@@ -1140,9 +1141,6 @@ swap_off(p, sdp)
 		(void) VOP_CLOSE(sdp->swd_vp, FREAD|FWRITE, p->p_ucred, p);
 	}
 
-	/* remove anons from the system */
-	uvm_anon_remove(sdp->swd_npages);
-
 	simple_lock(&uvm.swap_data_lock);
 	uvmexp.swpages -= sdp->swd_npages;