Put the PG_RELEASED changes diff back in.

This has has been tested very very thoroughly on all archs we have
excepting 88k and 68k. Please see cvs log for the individual commit
messages.

ok beck@, thib@

11 files changed, 207 insertions, 662 deletions

diff --git a/sys/uvm/uvm_aobj.c b/sys/uvm/uvm_aobj.c
index 1bcc615fadc..cc5a075e77f 100644
--- a/sys/uvm/uvm_aobj.c
+++ b/sys/uvm/uvm_aobj.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: uvm_aobj.c,v 1.45 2009/06/16 23:54:57 oga Exp $	*/
+/*	$OpenBSD: uvm_aobj.c,v 1.46 2009/07/22 21:05:37 oga Exp $	*/
 /*	$NetBSD: uvm_aobj.c,v 1.39 2001/02/18 21:19:08 chs Exp $	*/
 
 /*
@@ -139,7 +139,7 @@ struct pool uao_swhash_elt_pool;
  * uvm_aobj: the actual anon-backed uvm_object
  *
  * => the uvm_object is at the top of the structure, this allows
- *   (struct uvm_device *) == (struct uvm_object *)
+ *   (struct uvm_aobj *) == (struct uvm_object *)
  * => only one of u_swslots and u_swhash is used in any given aobj
  */
 
@@ -176,8 +176,6 @@ static void			 uao_free(struct uvm_aobj *);
 static int			 uao_get(struct uvm_object *, voff_t,
 				     vm_page_t *, int *, int, vm_prot_t,
 				     int, int);
-static boolean_t		 uao_releasepg(struct vm_page *,
-				     struct vm_page **);
 static boolean_t		 uao_pagein(struct uvm_aobj *, int, int);
 static boolean_t		 uao_pagein_page(struct uvm_aobj *, int);
 
@@ -194,10 +192,6 @@ struct uvm_pagerops aobj_pager = {
 	NULL,			/* fault */
 	uao_flush,		/* flush */
 	uao_get,		/* get */
-	NULL,			/* put (done by pagedaemon) */
-	NULL,			/* cluster */
-	NULL,			/* mk_pcluster */
-	uao_releasepg		/* releasepg */
 };
 
 /*
@@ -565,7 +559,7 @@ uao_init(void)
 	simple_lock_init(&uao_list_lock);
 
 	/*
-	 * NOTE: Pages fror this pool must not come from a pageable
+	 * NOTE: Pages for this pool must not come from a pageable
 	 * kernel map!
 	 */
 	pool_init(&uao_swhash_elt_pool, sizeof(struct uao_swhash_elt),
@@ -641,8 +635,7 @@ void
 uao_detach_locked(struct uvm_object *uobj)
 {
 	struct uvm_aobj *aobj = (struct uvm_aobj *)uobj;
-	struct vm_page *pg, *next;
-	boolean_t busybody;
+	struct vm_page *pg;
 	UVMHIST_FUNC("uao_detach"); UVMHIST_CALLED(maphist);
 
 	/*
@@ -669,35 +662,26 @@ uao_detach_locked(struct uvm_object *uobj)
 	simple_unlock(&uao_list_lock);
 
 	/*
- 	 * free all the pages that aren't PG_BUSY,
-	 * mark for release any that are.
+	 * Free all pages left in the object. If they're busy, wait
+	 * for them to become available before we kill it.
+	 * Release swap resources then free the page.
  	 */
-	busybody = FALSE;
-	for (pg = TAILQ_FIRST(&uobj->memq); pg != NULL; pg = next) {
-		next = TAILQ_NEXT(pg, listq);
+	uvm_lock_pageq();
+	while((pg = TAILQ_FIRST(&uobj->memq)) != NULL) {
 		if (pg->pg_flags & PG_BUSY) {
-			atomic_setbits_int(&pg->pg_flags, PG_RELEASED);
-			busybody = TRUE;
+			atomic_setbits_int(&pg->pg_flags, PG_WANTED);
+			uvm_unlock_pageq();
+			UVM_UNLOCK_AND_WAIT(pg, &uobj->vmobjlock, 0,
+			    "uao_det", 0);
+			simple_lock(&uobj->vmobjlock);
+			uvm_lock_pageq();
 			continue;
 		}
-
-		/* zap the mappings, free the swap slot, free the page */
 		pmap_page_protect(pg, VM_PROT_NONE);
 		uao_dropswap(&aobj->u_obj, pg->offset >> PAGE_SHIFT);
-		uvm_lock_pageq();
 		uvm_pagefree(pg);
-		uvm_unlock_pageq();
-	}
-
-	/*
- 	 * if we found any busy pages, we're done for now.
- 	 * mark the aobj for death, releasepg will finish up for us.
- 	 */
-	if (busybody) {
-		aobj->u_flags |= UAO_FLAG_KILLME;
-		simple_unlock(&aobj->u_obj.vmobjlock);
-		return;
 	}
+	uvm_unlock_pageq();
 
 	/*
  	 * finally, free the rest.
@@ -729,35 +713,6 @@ uao_detach_locked(struct uvm_object *uobj)
  * => we return TRUE unless we encountered some sort of I/O error
  *	XXXJRT currently never happens, as we never directly initiate
  *	XXXJRT I/O
- *
- * comment on "cleaning" object and PG_BUSY pages:
- *	this routine is holding the lock on the object.  the only time
- *	that is can run into a PG_BUSY page that it does not own is if
- *	some other process has started I/O on the page (e.g. either
- *	a pagein or a pageout).  if the PG_BUSY page is being paged
- *	in, then it can not be dirty (!PG_CLEAN) because no one has
- *	had a change to modify it yet.  if the PG_BUSY page is being
- *	paged out then it means that someone else has already started
- *	cleaning the page for us (how nice!).  in this case, if we
- *	have syncio specified, then after we make our pass through the
- *	object we need to wait for the other PG_BUSY pages to clear
- *	off (i.e. we need to do an iosync).  also note that once a
- *	page is PG_BUSY is must stary in its object until it is un-busyed.
- *	XXXJRT We never actually do this, as we are "flushing" anonymous
- *	XXXJRT memory, which doesn't have persistent backing store.
- *
- * note on page traversal:
- *	we can traverse the pages in an object either by going down the
- *	linked list in "uobj->memq", or we can go over the address range
- *	by page doing hash table lookups for each address.  depending
- *	on how many pages are in the object it may be cheaper to do one
- *	or the other.  we set "by_list" to true if we are using memq.
- *	if the cost of a hash lookup was equal to the cost of the list
- *	traversal we could compare the number of pages in the start->stop
- *	range to the total number of pages in the object.  however, it
- *	seems that a hash table lookup is more expensive than the linked
- *	list traversal, so we multiply the number of pages in the
- *	start->stop range by a penalty which we define below.
  */
 
 #define	UAO_HASH_PENALTY 4	/* XXX: a guess */
@@ -766,19 +721,13 @@ boolean_t
 uao_flush(struct uvm_object *uobj, voff_t start, voff_t stop, int flags)
 {
 	struct uvm_aobj *aobj = (struct uvm_aobj *) uobj;
-	struct vm_page *pp, *ppnext;
-	boolean_t retval, by_list;
+	struct vm_page *pp;
 	voff_t curoff;
 	UVMHIST_FUNC("uao_flush"); UVMHIST_CALLED(maphist);
 
-	curoff = 0;	/* XXX: shut up gcc */
-
-	retval = TRUE;	/* default to success */
-
 	if (flags & PGO_ALLPAGES) {
 		start = 0;
 		stop = aobj->u_pages << PAGE_SHIFT;
-		by_list = TRUE;		/* always go by the list */
 	} else {
 		start = trunc_page(start);
 		stop = round_page(stop);
@@ -787,13 +736,10 @@ uao_flush(struct uvm_object *uobj, voff_t start, voff_t stop, int flags)
 			    "flush (fixed)\n");
 			stop = aobj->u_pages << PAGE_SHIFT;
 		}
-		by_list = (uobj->uo_npages <=
-		    ((stop - start) >> PAGE_SHIFT) * UAO_HASH_PENALTY);
 	}
 
-	UVMHIST_LOG(maphist,
-	    " flush start=0x%lx, stop=0x%lx, by_list=%ld, flags=0x%lx",
-	    (u_long)start, (u_long)stop, by_list, flags);
+	UVMHIST_LOG(maphist, " flush start=0x%lx, stop=0x%lx, flags=0x%lx",
+	    (u_long)start, (u_long)stop, flags, 0);
 
 	/*
 	 * Don't need to do any work here if we're not freeing
@@ -802,44 +748,31 @@ uao_flush(struct uvm_object *uobj, voff_t start, voff_t stop, int flags)
 	if ((flags & (PGO_DEACTIVATE|PGO_FREE)) == 0) {
 		UVMHIST_LOG(maphist,
 		    "<- done (no work to do)",0,0,0,0);
-		return (retval);
+		return (TRUE);
 	}
 
-	/*
-	 * now do it.  note: we must update ppnext in the body of loop or we
-	 * will get stuck.  we need to use ppnext because we may free "pp"
-	 * before doing the next loop.
-	 */
-
-	if (by_list) {
-		pp = TAILQ_FIRST(&uobj->memq);
-	} else {
-		curoff = start;
-		pp = uvm_pagelookup(uobj, curoff);
-	}
-
-	ppnext = NULL;	/* XXX: shut up gcc */
-	uvm_lock_pageq();	/* page queues locked */
-
-	/* locked: both page queues and uobj */
-	for ( ; (by_list && pp != NULL) ||
-	    (!by_list && curoff < stop) ; pp = ppnext) {
-		if (by_list) {
-			ppnext = TAILQ_NEXT(pp, listq);
-
-			/* range check */
-			if (pp->offset < start || pp->offset >= stop)
-				continue;
-		} else {
+	/* locked: uobj */
+	curoff = start;
+	for (;;) {
+		if (curoff < stop) {
+			pp = uvm_pagelookup(uobj, curoff);
 			curoff += PAGE_SIZE;
-			if (curoff < stop)
-				ppnext = uvm_pagelookup(uobj, curoff);
-
-			/* null check */
 			if (pp == NULL)
 				continue;
+		} else {
+			break;
 		}
-		
+
+		/* Make sure page is unbusy, else wait for it. */
+		if (pp->pg_flags & PG_BUSY) {
+			atomic_setbits_int(&pp->pg_flags, PG_WANTED);
+			UVM_UNLOCK_AND_WAIT(pp, &uobj->vmobjlock, 0,
+			    "uaoflsh", 0);
+			simple_lock(&uobj->vmobjlock);
+			curoff -= PAGE_SIZE;
+			continue;
+		}
+
 		switch (flags & (PGO_CLEANIT|PGO_FREE|PGO_DEACTIVATE)) {
 		/*
 		 * XXX In these first 3 cases, we always just
@@ -848,7 +781,9 @@ uao_flush(struct uvm_object *uobj, voff_t start, voff_t stop, int flags)
 		 * XXX in the future.
 		 */
 		case PGO_CLEANIT|PGO_FREE:
+			/* FALLTHROUGH */
 		case PGO_CLEANIT|PGO_DEACTIVATE:
+			/* FALLTHROUGH */
 		case PGO_DEACTIVATE:
  deactivate_it:
 			/* skip the page if it's loaned or wired */
@@ -856,16 +791,13 @@ uao_flush(struct uvm_object *uobj, voff_t start, voff_t stop, int flags)
 			    pp->wire_count != 0)
 				continue;
 
-#ifdef UBC
-			/* ...and deactivate the page. */
-			pmap_clear_reference(pp);
-#else
+			uvm_lock_pageq();
 			/* zap all mappings for the page. */
 			pmap_page_protect(pp, VM_PROT_NONE);
 
 			/* ...and deactivate the page. */
-#endif
 			uvm_pagedeactivate(pp);
+			uvm_unlock_pageq();
 
 			continue;
 
@@ -882,19 +814,13 @@ uao_flush(struct uvm_object *uobj, voff_t start, voff_t stop, int flags)
 			    pp->wire_count != 0)
 				continue;
 
-			/*
-			 * mark the page as released if its busy.
-			 */
-			if (pp->pg_flags & PG_BUSY) {
-				atomic_setbits_int(&pp->pg_flags, PG_RELEASED);
-				continue;
-			}
-
 			/* zap all mappings for the page. */
 			pmap_page_protect(pp, VM_PROT_NONE);
 
 			uao_dropswap(uobj, pp->offset >> PAGE_SHIFT);
+			uvm_lock_pageq();
 			uvm_pagefree(pp);
+			uvm_unlock_pageq();
 
 			continue;
 
@@ -903,11 +829,9 @@ uao_flush(struct uvm_object *uobj, voff_t start, voff_t stop, int flags)
 		}
 	}
 
-	uvm_unlock_pageq();
-
 	UVMHIST_LOG(maphist,
 	    "<- done, rv=%ld",retval,0,0,0);
-	return (retval);
+	return (TRUE);
 }
 
 /*
@@ -989,10 +913,10 @@ uao_get(struct uvm_object *uobj, voff_t offset, struct vm_page **pps,
 			}
 
 			/*
-			 * to be useful must get a non-busy, non-released page
+			 * to be useful must get a non-busy page
 			 */
 			if (ptmp == NULL ||
-			    (ptmp->pg_flags & (PG_BUSY|PG_RELEASED)) != 0) {
+			    (ptmp->pg_flags & PG_BUSY) != 0) {
 				if (lcv == centeridx ||
 				    (flags & PGO_ALLPAGES) != 0)
 					/* need to do a wait or I/O! */
@@ -1099,7 +1023,7 @@ uao_get(struct uvm_object *uobj, voff_t offset, struct vm_page **pps,
 			}
 
 			/* page is there, see if we need to wait on it */
-			if ((ptmp->pg_flags & (PG_BUSY|PG_RELEASED)) != 0) {
+			if ((ptmp->pg_flags & PG_BUSY) != 0) {
 				atomic_setbits_int(&ptmp->pg_flags, PG_WANTED);
 				UVMHIST_LOG(pdhist,
 				    "sleeping, ptmp->flags 0x%lx\n",
@@ -1138,8 +1062,7 @@ uao_get(struct uvm_object *uobj, voff_t offset, struct vm_page **pps,
 		/*
  		 * just zero the page if there's nothing in swap.
  		 */
-		if (swslot == 0)
-		{
+		if (swslot == 0) {
 			/*
 			 * page hasn't existed before, just zero it.
 			 */
@@ -1216,65 +1139,6 @@ uao_get(struct uvm_object *uobj, voff_t offset, struct vm_page **pps,
 }
 
 /*
- * uao_releasepg: handle released page in an aobj
- * 
- * => "pg" is a PG_BUSY [caller owns it], PG_RELEASED page that we need
- *      to dispose of.
- * => caller must handle PG_WANTED case
- * => called with page's object locked, pageq's unlocked
- * => returns TRUE if page's object is still alive, FALSE if we
- *      killed the page's object.    if we return TRUE, then we
- *      return with the object locked.
- * => if (nextpgp != NULL) => we return the next page on the queue, and return
- *                              with the page queues locked [for pagedaemon]
- * => if (nextpgp == NULL) => we return with page queues unlocked [normal case]
- * => we kill the aobj if it is not referenced and we are suppose to
- *      kill it ("KILLME").
- */
-static boolean_t
-uao_releasepg(struct vm_page *pg, struct vm_page **nextpgp /* OUT */)
-{
-	struct uvm_aobj *aobj = (struct uvm_aobj *) pg->uobject;
-
-	KASSERT(pg->pg_flags & PG_RELEASED);
-
-	/*
- 	 * dispose of the page [caller handles PG_WANTED] and swap slot.
- 	 */
-	pmap_page_protect(pg, VM_PROT_NONE);
-	uao_dropswap(&aobj->u_obj, pg->offset >> PAGE_SHIFT);
-	uvm_lock_pageq();
-	if (nextpgp)
-		*nextpgp = TAILQ_NEXT(pg, pageq); /* next page for daemon */
-	uvm_pagefree(pg);
-	if (!nextpgp)
-		uvm_unlock_pageq();		/* keep locked for daemon */
-
-	/*
- 	 * if we're not killing the object, we're done.
- 	 */
-	if ((aobj->u_flags & UAO_FLAG_KILLME) == 0)
-		return TRUE;
-	KASSERT(aobj->u_obj.uo_refs == 0);
-
-	/*
- 	 * if there are still pages in the object, we're done for now.
- 	 */
-	if (aobj->u_obj.uo_npages != 0)
-		return TRUE;
-
-	KASSERT(TAILQ_EMPTY(&aobj->u_obj.memq));
-
-	/*
- 	 * finally, free the rest.
- 	 */
-	uao_free(aobj);
-
-	return FALSE;
-}
-
-
-/*
  * uao_dropswap:  release any swap resources from this aobj page.
  * 
  * => aobj must be locked or have a reference count of 0.
@@ -1476,7 +1340,6 @@ uao_pagein_page(struct uvm_aobj *aobj, int pageidx)
 		return FALSE;
 
 	}
-	KASSERT((pg->pg_flags & PG_RELEASED) == 0);
 
 	/*
 	 * ok, we've got the page now.
diff --git a/sys/uvm/uvm_aobj.h b/sys/uvm/uvm_aobj.h
index c567a181473..27645df5b9b 100644
--- a/sys/uvm/uvm_aobj.h
+++ b/sys/uvm/uvm_aobj.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: uvm_aobj.h,v 1.11 2009/06/16 23:54:57 oga Exp $	*/
+/*	$OpenBSD: uvm_aobj.h,v 1.12 2009/07/22 21:05:37 oga Exp $	*/
 /*	$NetBSD: uvm_aobj.h,v 1.10 2000/01/11 06:57:49 chs Exp $	*/
 
 /*
@@ -55,8 +55,6 @@
 #define UAO_FLAG_KERNSWAP	0x2	/* enable kernel swap */
 
 /* internal flags */
-#define UAO_FLAG_KILLME		0x4	/* aobj should die when last released
-					 * page is no longer PG_BUSY ... */
 #define UAO_FLAG_NOSWAP		0x8	/* aobj can't swap (kernel obj only!) */
 
 #ifdef _KERNEL
diff --git a/sys/uvm/uvm_fault.c b/sys/uvm/uvm_fault.c
index a10fd054e84..e5eb6b8e3b0 100644
--- a/sys/uvm/uvm_fault.c
+++ b/sys/uvm/uvm_fault.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: uvm_fault.c,v 1.57 2009/06/16 23:54:58 oga Exp $	*/
+/*	$OpenBSD: uvm_fault.c,v 1.58 2009/07/22 21:05:37 oga Exp $	*/
 /*	$NetBSD: uvm_fault.c,v 1.51 2000/08/06 00:22:53 thorpej Exp $	*/
 
 /*
@@ -921,10 +921,10 @@ ReFault:
 
 				/*
 				 * if center page is resident and not
-				 * PG_BUSY|PG_RELEASED then pgo_get
-				 * made it PG_BUSY for us and gave
-				 * us a handle to it.   remember this
-				 * page as "uobjpage." (for later use).
+				 * PG_BUSY, then pgo_get made it PG_BUSY
+				 * for us and gave us a handle to it.
+				 * remember this page as "uobjpage."
+				 * (for later use).
 				 */
 				
 				if (lcv == centeridx) {
@@ -966,8 +966,8 @@ ReFault:
 				     (wired ? PMAP_WIRED : 0));
 
 				/* 
-				 * NOTE: page can't be PG_WANTED or PG_RELEASED
-				 * because we've held the lock the whole time
+				 * NOTE: page can't be PG_WANTED because
+				 * we've held the lock the whole time
 				 * we've had the handle.
 				 */
 
@@ -1371,15 +1371,12 @@ Case2:
 		/* locked(!locked): uobj, uobjpage */
 
 		/*
-		 * verify that the page has not be released and re-verify
-		 * that amap slot is still free.   if there is a problem,
-		 * we unlock and clean up.
+		 * Re-verify that amap slot is still free. if there is
+		 * a problem, we unlock and clean up.
 		 */
 
-		if ((uobjpage->pg_flags & PG_RELEASED) != 0 ||
-		    (locked && amap && 
-		    amap_lookup(&ufi.entry->aref,
-		      ufi.orig_rvaddr - ufi.entry->start))) {
+		if (locked && amap && amap_lookup(&ufi.entry->aref,
+		      ufi.orig_rvaddr - ufi.entry->start)) {
 			if (locked) 
 				uvmfault_unlockall(&ufi, amap, NULL, NULL);
 			locked = FALSE;
@@ -1398,17 +1395,6 @@ Case2:
 				/* still holding object lock */
 				wakeup(uobjpage);
 
-			if (uobjpage->pg_flags & PG_RELEASED) {
-				uvmexp.fltpgrele++;
-				KASSERT(uobj->pgops->pgo_releasepg != NULL);
-
-				/* frees page */
-				if (uobj->pgops->pgo_releasepg(uobjpage,NULL))
-					/* unlock if still alive */
-					simple_unlock(&uobj->vmobjlock);
-				goto ReFault;
-			}
-
 			uvm_lock_pageq();
 			/* make sure it is in queues */
 			uvm_pageactivate(uobjpage);
@@ -1423,9 +1409,8 @@ Case2:
 		}
 
 		/*
-		 * we have the data in uobjpage which is PG_BUSY and
-		 * !PG_RELEASED.  we are holding object lock (so the page
-		 * can't be released on us).
+		 * we have the data in uobjpage which is PG_BUSY and we are
+		 * holding object lock.
 		 */
 
 		/* locked: maps(read), amap(if !null), uobj, uobjpage */
@@ -1439,8 +1424,6 @@ Case2:
 	/*
 	 * notes:
 	 *  - at this point uobjpage can not be NULL
-	 *  - at this point uobjpage can not be PG_RELEASED (since we checked
-	 *  for it above)
 	 *  - at this point uobjpage could be PG_WANTED (handle later)
 	 */
 		
@@ -1627,9 +1610,7 @@ Case2:
 			}
 			
 			/*
-			 * dispose of uobjpage.  it can't be PG_RELEASED
-			 * since we still hold the object lock.
-			 * drop handle to uobj as well.
+			 * dispose of uobjpage. drop handle to uobj as well.
 			 */
 
 			if (uobjpage->pg_flags & PG_WANTED)
@@ -1692,11 +1673,6 @@ Case2:
 		if (pg->pg_flags & PG_WANTED)
 			wakeup(pg);		/* lock still held */
 
-		/* 
-		 * note that pg can't be PG_RELEASED since we did not drop
-		 * the object lock since the last time we checked.
-		 */
- 
 		atomic_clearbits_int(&pg->pg_flags, PG_BUSY|PG_FAKE|PG_WANTED);
 		UVM_PAGE_OWN(pg, NULL);
 		uvmfault_unlockall(&ufi, amap, uobj, NULL);
@@ -1736,11 +1712,6 @@ Case2:
 	if (pg->pg_flags & PG_WANTED)
 		wakeup(pg);		/* lock still held */
 
-	/* 
-	 * note that pg can't be PG_RELEASED since we did not drop the object 
-	 * lock since the last time we checked.
-	 */
- 
 	atomic_clearbits_int(&pg->pg_flags, PG_BUSY|PG_FAKE|PG_WANTED);
 	UVM_PAGE_OWN(pg, NULL);
 	uvmfault_unlockall(&ufi, amap, uobj, NULL);
diff --git a/sys/uvm/uvm_km.c b/sys/uvm/uvm_km.c
index e6843631968..f30878ad873 100644
--- a/sys/uvm/uvm_km.c
+++ b/sys/uvm/uvm_km.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: uvm_km.c,v 1.73 2009/06/17 00:13:59 oga Exp $	*/
+/*	$OpenBSD: uvm_km.c,v 1.74 2009/07/22 21:05:37 oga Exp $	*/
 /*	$NetBSD: uvm_km.c,v 1.42 2001/01/14 02:10:01 thorpej Exp $	*/
 
 /* 
@@ -276,8 +276,12 @@ uvm_km_pgremove(struct uvm_object *uobj, vaddr_t start, vaddr_t end)
 		    pp->pg_flags & PG_BUSY, 0, 0);
 
 		if (pp->pg_flags & PG_BUSY) {
-			/* owner must check for this when done */
-			atomic_setbits_int(&pp->pg_flags, PG_RELEASED);
+			atomic_setbits_int(&pp->pg_flags, PG_WANTED);
+			UVM_UNLOCK_AND_WAIT(pp, &uobj->vmobjlock, 0,
+			    "km_pgrm", 0);
+			simple_lock(&uobj->vmobjlock);
+			curoff -= PAGE_SIZE; /* loop back to us */
+			continue;
 		} else {
 			/* free the swap slot... */
 			uao_dropswap(uobj, curoff >> PAGE_SHIFT);
@@ -511,21 +515,6 @@ uvm_km_alloc1(struct vm_map *map, vsize_t size, vsize_t align, boolean_t zeroit)
 	loopva = kva;
 	while (size) {
 		simple_lock(&uvm.kernel_object->vmobjlock);
-		pg = uvm_pagelookup(uvm.kernel_object, offset);
-
-		/*
-		 * if we found a page in an unallocated region, it must be
-		 * released
-		 */
-		if (pg) {
-			if ((pg->pg_flags & PG_RELEASED) == 0)
-				panic("uvm_km_alloc1: non-released page");
-			atomic_setbits_int(&pg->pg_flags, PG_WANTED);
-			UVM_UNLOCK_AND_WAIT(pg, &uvm.kernel_object->vmobjlock,
-			    FALSE, "km_alloc", 0);
-			continue;   /* retry */
-		}
-		
 		/* allocate ram */
 		pg = uvm_pagealloc(uvm.kernel_object, offset, NULL, 0);
 		if (pg) {
diff --git a/sys/uvm/uvm_loan.c b/sys/uvm/uvm_loan.c
index caa6284f1fa..20af9ad7e63 100644
--- a/sys/uvm/uvm_loan.c
+++ b/sys/uvm/uvm_loan.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: uvm_loan.c,v 1.33 2009/06/17 00:13:59 oga Exp $	*/
+/*	$OpenBSD: uvm_loan.c,v 1.34 2009/07/22 21:05:37 oga Exp $	*/
 /*	$NetBSD: uvm_loan.c,v 1.22 2000/06/27 17:29:25 mrg Exp $	*/
 
 /*
@@ -462,14 +462,12 @@ uvm_loanuobj(struct uvm_faultinfo *ufi, void ***output, int flags, vaddr_t va)
 		simple_lock(&uobj->vmobjlock);
 
 		/*
-		 * verify that the page has not be released and re-verify
-		 * that amap slot is still free.   if there is a problem we
-		 * drop our lock (thus force a lookup refresh/retry).
+		 * Re-verify that amap slot is still free. if there is a
+		 * problem we drop our lock (thus force a lookup refresh/retry).
 		 */
 			
-		if ((pg->pg_flags & PG_RELEASED) != 0 ||
-		    (locked && amap && amap_lookup(&ufi->entry->aref,
-		    ufi->orig_rvaddr - ufi->entry->start))) {
+		if (locked && amap && amap_lookup(&ufi->entry->aref,
+		    ufi->orig_rvaddr - ufi->entry->start)) {
 			
 			if (locked)
 				uvmfault_unlockall(ufi, amap, NULL, NULL);
@@ -486,17 +484,6 @@ uvm_loanuobj(struct uvm_faultinfo *ufi, void ***output, int flags, vaddr_t va)
 				/* still holding object lock */
 				wakeup(pg);
 
-			if (pg->pg_flags & PG_RELEASED) {
-#ifdef DIAGNOSTIC
-				if (uobj->pgops->pgo_releasepg == NULL)
-			panic("uvm_loanuobj: object has no releasepg function");
-#endif
-				/* frees page */
-				if (uobj->pgops->pgo_releasepg(pg, NULL))
-					simple_unlock(&uobj->vmobjlock);
-				return (0);
-			}
-
 			uvm_lock_pageq();
 			uvm_pageactivate(pg); /* make sure it is in queues */
 			uvm_unlock_pageq();
@@ -509,8 +496,7 @@ uvm_loanuobj(struct uvm_faultinfo *ufi, void ***output, int flags, vaddr_t va)
 
 	/*
 	 * at this point we have the page we want ("pg") marked PG_BUSY for us
-	 * and we have all data structures locked.   do the loanout.   page can
-	 * not be PG_RELEASED (we caught this above).
+	 * and we have all data structures locked.   do the loanout.
 	 */
 
 	if ((flags & UVM_LOAN_TOANON) == 0) {	/* loan to wired-kernel page? */
diff --git a/sys/uvm/uvm_mmap.c b/sys/uvm/uvm_mmap.c
index d02482e8ab0..34f9f89e8ff 100644
--- a/sys/uvm/uvm_mmap.c
+++ b/sys/uvm/uvm_mmap.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: uvm_mmap.c,v 1.77 2009/07/09 22:29:56 thib Exp $	*/
+/*	$OpenBSD: uvm_mmap.c,v 1.78 2009/07/22 21:05:37 oga Exp $	*/
 /*	$NetBSD: uvm_mmap.c,v 1.49 2001/02/18 21:19:08 chs Exp $	*/
 
 /*
@@ -298,8 +298,7 @@ sys_mincore(struct proc *p, void *v, register_t *retval)
 		 */
 		if (UVM_ET_ISOBJ(entry)) {
 			KASSERT(!UVM_OBJ_IS_KERN_OBJECT(entry->object.uvm_obj));
-			if (entry->object.uvm_obj->pgops->pgo_releasepg
-			    == NULL) {
+			if (entry->object.uvm_obj->pgops->pgo_fault != NULL) {
 				pgi = 1;
 				for (/* nothing */; start < lim;
 				     start += PAGE_SIZE, vec++)
diff --git a/sys/uvm/uvm_page.c b/sys/uvm/uvm_page.c
index 1c6e60df7cf..d0415ed2ea6 100644
--- a/sys/uvm/uvm_page.c
+++ b/sys/uvm/uvm_page.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: uvm_page.c,v 1.91 2009/06/17 00:13:59 oga Exp $	*/
+/*	$OpenBSD: uvm_page.c,v 1.92 2009/07/22 21:05:37 oga Exp $	*/
 /*	$NetBSD: uvm_page.c,v 1.44 2000/11/27 08:40:04 chs Exp $	*/
 
 /* 
@@ -1245,7 +1245,14 @@ uvm_page_unbusy(struct vm_page **pgs, int npgs)
 			UVMHIST_LOG(pdhist, "releasing pg %p", pg,0,0,0);
 			uobj = pg->uobject;
 			if (uobj != NULL) {
-				uobj->pgops->pgo_releasepg(pg, NULL);
+				uvm_lock_pageq();
+				pmap_page_protect(pg, VM_PROT_NONE);
+				/* XXX won't happen right now */
+				if (pg->pg_flags & PQ_ANON)
+					uao_dropswap(uobj,
+					    pg->offset >> PAGE_SHIFT);
+				uvm_pagefree(pg);
+				uvm_unlock_pageq();
 			} else {
 				atomic_clearbits_int(&pg->pg_flags, PG_BUSY);
 				UVM_PAGE_OWN(pg, NULL);
diff --git a/sys/uvm/uvm_pager.c b/sys/uvm/uvm_pager.c
index 38cbd7701bc..ee9048b63d2 100644
--- a/sys/uvm/uvm_pager.c
+++ b/sys/uvm/uvm_pager.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: uvm_pager.c,v 1.53 2009/06/17 00:13:59 oga Exp $	*/
+/*	$OpenBSD: uvm_pager.c,v 1.54 2009/07/22 21:05:37 oga Exp $	*/
 /*	$NetBSD: uvm_pager.c,v 1.36 2000/11/27 18:26:41 chs Exp $	*/
 
 /*
@@ -339,6 +339,8 @@ uvm_pagermapout(vaddr_t kva, int npages)
  *      PGO_ALLPAGES:  all pages in object are valid targets
  *      !PGO_ALLPAGES: use "lo" and "hi" to limit range of cluster
  *      PGO_DOACTCLUST: include active pages in cluster.
+ *	PGO_FREE: set the PG_RELEASED bits on the cluster so they'll be freed
+ *		in async io (caller must clean on error).
  *        NOTE: the caller should clear PG_CLEANCHK bits if PGO_DOACTCLUST.
  *              PG_CLEANCHK is only a hint, but clearing will help reduce
  *		the number of calls we make to the pmap layer.
@@ -440,6 +442,14 @@ uvm_mk_pcluster(struct uvm_object *uobj, struct vm_page **pps, int *npages,
 			atomic_setbits_int(&pclust->pg_flags, PG_BUSY);
 			UVM_PAGE_OWN(pclust, "uvm_mk_pcluster");
 
+			/*
+			 * If we want to free after io is done, and we're
+			 * async, set the released flag
+			 */
+			if ((flags & (PGO_FREE|PGO_SYNCIO)) == PGO_FREE)
+				atomic_setbits_int(&pclust->pg_flags,
+				    PG_RELEASED);
+
 			/* XXX: protect wired page?   see above comment. */
 			pmap_page_protect(pclust, VM_PROT_READ);
 			if (!forward) {
@@ -481,6 +491,7 @@ uvm_mk_pcluster(struct uvm_object *uobj, struct vm_page **pps, int *npages,
  *	PGO_DOACTCLUST: include "PQ_ACTIVE" pages as valid targets
  *	PGO_SYNCIO: do SYNC I/O (no async)
  *	PGO_PDFREECLUST: pagedaemon: drop cluster on successful I/O
+ *	PGO_FREE: tell the aio daemon to free pages in the async case.
  * => start/stop: if (uobj && !PGO_ALLPAGES) limit targets to this range
  *		  if (!uobj) start is the (daddr64_t) of the starting swapblk
  * => return state:
@@ -704,8 +715,6 @@ uvm_pager_dropcluster(struct uvm_object *uobj, struct vm_page *pg,
     struct vm_page **ppsp, int *npages, int flags)
 {
 	int lcv;
-	boolean_t obj_is_alive; 
-	struct uvm_object *saved_uobj;
 
 	/*
 	 * drop all pages but "pg"
@@ -747,9 +756,8 @@ uvm_pager_dropcluster(struct uvm_object *uobj, struct vm_page *pg,
 		}
 
 		/* if page was released, release it.  otherwise un-busy it */
-		if (ppsp[lcv]->pg_flags & PG_RELEASED) {
-
-			if (ppsp[lcv]->pg_flags & PQ_ANON) {
+		if (ppsp[lcv]->pg_flags & PG_RELEASED &&
+		    ppsp[lcv]->pg_flags & PQ_ANON) {
 				/* so that anfree will free */
 				atomic_clearbits_int(&ppsp[lcv]->pg_flags,
 				    PG_BUSY);
@@ -761,34 +769,13 @@ uvm_pager_dropcluster(struct uvm_object *uobj, struct vm_page *pg,
 				uvm_anfree(ppsp[lcv]->uanon);
 
 				continue;
-			}
-
-			/*
-			 * pgo_releasepg will dump the page for us
-			 */
-
-			saved_uobj = ppsp[lcv]->uobject;
-			obj_is_alive =
-			    saved_uobj->pgops->pgo_releasepg(ppsp[lcv], NULL);
-			
-			/* for normal objects, "pg" is still PG_BUSY by us,
-			 * so obj can't die */
-			KASSERT(!uobj || obj_is_alive);
-
-			/* only unlock the object if it is still alive...  */
-			if (obj_is_alive && saved_uobj != uobj)
-				simple_unlock(&saved_uobj->vmobjlock);
-
+		} else {
 			/*
-			 * XXXCDC: suppose uobj died in the pgo_releasepg?
-			 * how pass that
-			 * info up to caller.  we are currently ignoring it...
+			 * if we were planning on async io then we would
+			 * have PG_RELEASED set, clear that with the others.
 			 */
-
-			continue;		/* next page */
-		} else {
 			atomic_clearbits_int(&ppsp[lcv]->pg_flags,
-			    PG_BUSY|PG_WANTED|PG_FAKE);
+			    PG_BUSY|PG_WANTED|PG_FAKE|PG_RELEASED);
 			UVM_PAGE_OWN(ppsp[lcv], NULL);
 		}
 
@@ -812,33 +799,6 @@ uvm_pager_dropcluster(struct uvm_object *uobj, struct vm_page *pg,
 	}
 }
 
-#ifdef UBC
-/*
- * interrupt-context iodone handler for nested i/o bufs.
- *
- * => must be at splbio().
- */
-
-void
-uvm_aio_biodone1(struct buf *bp)
-{
-	struct buf *mbp = bp->b_private;
-
-	splassert(IPL_BIO);
-
-	KASSERT(mbp != bp);
-	if (bp->b_flags & B_ERROR) {
-		mbp->b_flags |= B_ERROR;
-		mbp->b_error = bp->b_error;
-	}
-	mbp->b_resid -= bp->b_bcount;
-	pool_put(&bufpool, bp);
-	if (mbp->b_resid == 0) {
-		biodone(mbp);
-	}
-}
-#endif
-
 /*
  * interrupt-context iodone handler for single-buf i/os
  * or the top-level buf of a nested-buf i/o.
diff --git a/sys/uvm/uvm_pager.h b/sys/uvm/uvm_pager.h
index fea487b7863..4e0151fa2b3 100644
--- a/sys/uvm/uvm_pager.h
+++ b/sys/uvm/uvm_pager.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: uvm_pager.h,v 1.26 2009/06/17 00:13:59 oga Exp $	*/
+/*	$OpenBSD: uvm_pager.h,v 1.27 2009/07/22 21:05:37 oga Exp $	*/
 /*	$NetBSD: uvm_pager.h,v 1.20 2000/11/27 08:40:05 chs Exp $	*/
 
 /*
@@ -109,8 +109,6 @@ struct uvm_pagerops {
 	struct vm_page **	(*pgo_mk_pcluster)(struct uvm_object *,
 				 struct vm_page **, int *, struct vm_page *,
 				 int, voff_t, voff_t);
-						/* release page */
-	boolean_t		(*pgo_releasepg)(struct vm_page *, struct vm_page **);
 };
 
 /* pager flags [mostly for flush] */
diff --git a/sys/uvm/uvm_pdaemon.c b/sys/uvm/uvm_pdaemon.c
index e16ae26ea3b..046368e697a 100644
--- a/sys/uvm/uvm_pdaemon.c
+++ b/sys/uvm/uvm_pdaemon.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: uvm_pdaemon.c,v 1.51 2009/06/26 20:26:02 oga Exp $	*/
+/*	$OpenBSD: uvm_pdaemon.c,v 1.52 2009/07/22 21:05:37 oga Exp $	*/
 /*	$NetBSD: uvm_pdaemon.c,v 1.23 2000/08/20 10:24:14 bjh21 Exp $	*/
 
 /* 
@@ -820,40 +820,25 @@ uvmpd_scan_inactive(struct pglist *pglst)
 			atomic_clearbits_int(&p->pg_flags, PG_BUSY|PG_WANTED);
 			UVM_PAGE_OWN(p, NULL);
 
-			/* released during I/O? */
+			/* released during I/O? Can only happen for anons */
 			if (p->pg_flags & PG_RELEASED) {
-				if (anon) {
-					/*
-					 * remove page so we can get nextpg,
-					 * also zero out anon so we don't use
-					 * it after the free.
-					 */
-					anon->an_page = NULL;
-					p->uanon = NULL;
-
-					simple_unlock(&anon->an_lock);
-					uvm_anfree(anon);	/* kills anon */
-					pmap_page_protect(p, VM_PROT_NONE);
-					anon = NULL;
-					uvm_lock_pageq();
-					nextpg = TAILQ_NEXT(p, pageq);
-					/* free released page */
-					uvm_pagefree(p);
-
-				} else {
-
-					/*
-					 * pgo_releasepg nukes the page and
-					 * gets "nextpg" for us.  it returns
-					 * with the page queues locked (when
-					 * given nextpg ptr).
-					 */
+				KASSERT(anon != NULL);
+				/*
+				 * remove page so we can get nextpg,
+				 * also zero out anon so we don't use
+				 * it after the free.
+				 */
+				anon->an_page = NULL;
+				p->uanon = NULL;
 
-					if (!uobj->pgops->pgo_releasepg(p,
-					    &nextpg))
-						/* uobj died after release */
-						uobj = NULL;
-				}
+				simple_unlock(&anon->an_lock);
+				uvm_anfree(anon);	/* kills anon */
+				pmap_page_protect(p, VM_PROT_NONE);
+				anon = NULL;
+				uvm_lock_pageq();
+				nextpg = TAILQ_NEXT(p, pageq);
+				/* free released page */
+				uvm_pagefree(p);
 			} else {	/* page was not released during I/O */
 				uvm_lock_pageq();
 				nextpg = TAILQ_NEXT(p, pageq);
diff --git a/sys/uvm/uvm_vnode.c b/sys/uvm/uvm_vnode.c
index 465db6d50bd..15130e95ded 100644
--- a/sys/uvm/uvm_vnode.c
+++ b/sys/uvm/uvm_vnode.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: uvm_vnode.c,v 1.67 2009/07/09 22:29:56 thib Exp $	*/
+/*	$OpenBSD: uvm_vnode.c,v 1.68 2009/07/22 21:05:37 oga Exp $	*/
 /*	$NetBSD: uvm_vnode.c,v 1.36 2000/11/24 20:34:01 chs Exp $	*/
 
 /*
@@ -93,7 +93,6 @@ void		 uvn_init(void);
 int		 uvn_io(struct uvm_vnode *, vm_page_t *, int, int, int);
 int		 uvn_put(struct uvm_object *, vm_page_t *, int, boolean_t);
 void		 uvn_reference(struct uvm_object *);
-boolean_t	 uvn_releasepg(struct vm_page *, struct vm_page **);
 
 /*
  * master pager structure
@@ -109,7 +108,6 @@ struct uvm_pagerops uvm_vnodeops = {
 	uvn_put,
 	uvn_cluster,
 	uvm_mk_pcluster, /* use generic version of this: see uvm_pager.c */
-	uvn_releasepg,
 };
 
 /*
@@ -415,31 +413,24 @@ uvn_detach(struct uvm_object *uobj)
 	/*
 	 * given the structure of this pager, the above flush request will
 	 * create the following state: all the pages that were in the object
-	 * have either been free'd or they are marked PG_BUSY|PG_RELEASED.
-	 * the PG_BUSY bit was set either by us or the daemon for async I/O.
-	 * in either case, if we have pages left we can't kill the object
-	 * yet because i/o is pending.  in this case we set the "relkill"
-	 * flag which will cause pgo_releasepg to kill the object once all
-	 * the I/O's are done [pgo_releasepg will be called from the aiodone
-	 * routine or from the page daemon].
+	 * have either been free'd or they are marked PG_BUSY and in the 
+	 * middle of an async io. If we still have pages we set the "relkill"
+	 * state, so that in the case the vnode gets terminated we know 
+	 * to leave it alone. Otherwise we'll kill the vnode when it's empty.
 	 */
 
-	if (uobj->uo_npages) {		/* I/O pending.  iodone will free */
-#ifdef DEBUG
-		/*
-		 * XXXCDC: very unlikely to happen until we have async i/o
-		 * so print a little info message in case it does.
-		 */
-		printf("uvn_detach: vn %p has pages left after flush - "
-		    "relkill mode\n", uobj);
-#endif
-		uvn->u_flags |= UVM_VNODE_RELKILL;
-		simple_unlock(&uobj->vmobjlock);
-		UVMHIST_LOG(maphist,"<- done! (releasepg will kill obj)", 0, 0,
-		    0, 0);
-		return;
+	uvn->u_flags |= UVM_VNODE_RELKILL;
+	/* wait on any outstanding io */
+	while (uobj->uo_npages && uvn->u_flags & UVM_VNODE_RELKILL) {
+		uvn->u_flags |= UVM_VNODE_IOSYNC;
+		UVM_UNLOCK_AND_WAIT(&uvn->u_nio, &uvn->u_obj.vmobjlock, FALSE,
+		    "uvn_term",0);
+		simple_lock(&uvn->u_obj.vmobjlock);
 	}
 
+	if ((uvn->u_flags & UVM_VNODE_RELKILL) == 0)
+		return;
+
 	/*
 	 * kill object now.   note that we can't be on the sync q because
 	 * all references are gone.
@@ -490,8 +481,6 @@ uvn_detach(struct uvm_object *uobj)
  * => the caller must XLOCK and VOP_LOCK the vnode before calling us
  *	[protects us from getting a vnode that is already in the DYING
  *	 state...]
- * => unlike uvn_detach, this function must not return until all the
- *	uvn's pages are disposed of.
  * => in case [2] the uvn is still alive after this call, but all I/O
  *	ops will fail (due to the backing vnode now being "dead").  this
  *	will prob. kill any process using the uvn due to pgo_get failing.
@@ -535,8 +524,8 @@ uvm_vnp_terminate(struct vnode *vp)
 
 	/*
 	 * it is possible that the uvn was detached and is in the relkill
-	 * state [i.e. waiting for async i/o to finish so that releasepg can
-	 * kill object].  we take over the vnode now and cancel the relkill.
+	 * state [i.e. waiting for async i/o to finish].
+	 * we take over the vnode now and cancel the relkill.
 	 * we want to know when the i/o is done so we can recycle right
 	 * away.   note that a uvn can only be in the RELKILL state if it
 	 * has a zero reference count.
@@ -630,72 +619,6 @@ uvm_vnp_terminate(struct vnode *vp)
 }
 
 /*
- * uvn_releasepg: handled a released page in a uvn
- *
- * => "pg" is a PG_BUSY [caller owns it], PG_RELEASED page that we need
- *	to dispose of.
- * => caller must handled PG_WANTED case
- * => called with page's object locked, pageq's unlocked
- * => returns TRUE if page's object is still alive, FALSE if we
- *	killed the page's object.    if we return TRUE, then we
- *	return with the object locked.
- * => if (nextpgp != NULL) => we return pageq.tqe_next here, and return
- *				with the page queues locked [for pagedaemon]
- * => if (nextpgp == NULL) => we return with page queues unlocked [normal case]
- * => we kill the uvn if it is not referenced and we are suppose to
- *	kill it ("relkill").
- */
-
-boolean_t
-uvn_releasepg(struct vm_page *pg, struct vm_page **nextpgp /* OUT */)
-{
-	struct uvm_vnode *uvn = (struct uvm_vnode *) pg->uobject;
-	struct vnode *vp = (struct vnode *)uvn;
-#ifdef DIAGNOSTIC
-	if ((pg->pg_flags & PG_RELEASED) == 0)
-		panic("uvn_releasepg: page not released!");
-#endif
-
-	/*
-	 * dispose of the page [caller handles PG_WANTED]
-	 */
-	pmap_page_protect(pg, VM_PROT_NONE);
-	uvm_lock_pageq();
-	if (nextpgp)
-		*nextpgp = TAILQ_NEXT(pg, pageq); /* next page for daemon */
-	uvm_pagefree(pg);
-	if (!nextpgp)
-		uvm_unlock_pageq();
-
-	/*
-	 * now see if we need to kill the object
-	 */
-	if (uvn->u_flags & UVM_VNODE_RELKILL) {
-		if (uvn->u_obj.uo_refs)
-			panic("uvn_releasepg: kill flag set on referenced "
-			    "object!");
-		if (uvn->u_obj.uo_npages == 0) {
-			if (uvn->u_flags & UVM_VNODE_WRITEABLE) {
-				LIST_REMOVE(uvn, u_wlist);
-			}
-#ifdef DIAGNOSTIC
-			if (!TAILQ_EMPTY(&uvn->u_obj.memq))
-	panic("uvn_releasepg: pages in object with npages == 0");
-#endif
-			if (uvn->u_flags & UVM_VNODE_WANTED)
-				/* still holding object lock */
-				wakeup(uvn);
-
-			uvn->u_flags = 0;		/* DEAD! */
-			simple_unlock(&uvn->u_obj.vmobjlock);
-			vrele(vp);
-			return (FALSE);
-		}
-	}
-	return (TRUE);
-}
-
-/*
  * NOTE: currently we have to use VOP_READ/VOP_WRITE because they go
  * through the buffer cache and allow I/O in any size.  These VOPs use
  * synchronous i/o.  [vs. VOP_STRATEGY which can be async, but doesn't
@@ -729,8 +652,6 @@ uvn_releasepg(struct vm_page *pg, struct vm_page **nextpgp /* OUT */)
  * - if (object->iosync && u_naio == 0) { wakeup &uvn->u_naio }
  * - get "page" structures (atop?).
  * - handle "wanted" pages
- * - handle "released" pages [using pgo_releasepg]
- *   >>> pgo_releasepg may kill the object
  * dont forget to look at "object" wanted flag in all cases.
  */
 
@@ -792,15 +713,13 @@ boolean_t
 uvn_flush(struct uvm_object *uobj, voff_t start, voff_t stop, int flags)
 {
 	struct uvm_vnode *uvn = (struct uvm_vnode *) uobj;
-	struct vm_page *pp, *ppnext, *ptmp;
+	struct vm_page *pp, *ptmp;
 	struct vm_page *pps[MAXBSIZE >> PAGE_SHIFT], **ppsp;
 	int npages, result, lcv;
-	boolean_t retval, need_iosync, by_list, needs_clean, all;
+	boolean_t retval, need_iosync, needs_clean;
 	voff_t curoff;
-	u_short pp_version;
 	UVMHIST_FUNC("uvn_flush"); UVMHIST_CALLED(maphist);
 
-	curoff = 0;	/* XXX: shut up gcc */
 	/*
 	 * get init vals and determine how we are going to traverse object
 	 */
@@ -808,24 +727,16 @@ uvn_flush(struct uvm_object *uobj, voff_t start, voff_t stop, int flags)
 	need_iosync = FALSE;
 	retval = TRUE;		/* return value */
 	if (flags & PGO_ALLPAGES) {
-		all = TRUE;
-		by_list = TRUE;		/* always go by the list */
+		start = 0;
+		stop = round_page(uvn->u_size);
 	} else {
 		start = trunc_page(start);
-		stop = round_page(stop);
-#ifdef DEBUG
-		if (stop > round_page(uvn->u_size))
-			printf("uvn_flush: strange, got an out of range "
-			    "flush (fixed)\n");
-#endif
-		all = FALSE;
-		by_list = (uobj->uo_npages <=
-		    ((stop - start) >> PAGE_SHIFT) * UVN_HASH_PENALTY);
+		stop = MIN(round_page(stop), round_page(uvn->u_size));
 	}
 
 	UVMHIST_LOG(maphist,
-	    " flush start=0x%lx, stop=0x%lx, by_list=%ld, flags=0x%lx",
-	    (u_long)start, (u_long)stop, by_list, flags);
+	    " flush start=0x%lx, stop=0x%lx, flags=0x%lx",
+	    (u_long)start, (u_long)stop, flags, 0);
 
 	/*
 	 * PG_CLEANCHK: this bit is used by the pgo_mk_pcluster function as
@@ -838,75 +749,21 @@ uvn_flush(struct uvm_object *uobj, voff_t start, voff_t stop, int flags)
 	 * [borrowed PG_CLEANCHK idea from FreeBSD VM]
 	 */
 
-	if ((flags & PGO_CLEANIT) != 0 &&
-	    uobj->pgops->pgo_mk_pcluster != NULL) {
-		if (by_list) {
-			TAILQ_FOREACH(pp, &uobj->memq, listq) {
-				if (!all &&
-				    (pp->offset < start || pp->offset >= stop))
-					continue;
+	if ((flags & PGO_CLEANIT) != 0) {
+		KASSERT(uobj->pgops->pgo_mk_pcluster != 0);
+		for (curoff = start ; curoff < stop; curoff += PAGE_SIZE) {
+			if ((pp = uvm_pagelookup(uobj, curoff)) != NULL)
 				atomic_clearbits_int(&pp->pg_flags,
 				    PG_CLEANCHK);
-			}
-
-		} else {   /* by hash */
-			for (curoff = start ; curoff < stop;
-			    curoff += PAGE_SIZE) {
-				pp = uvm_pagelookup(uobj, curoff);
-				if (pp)
-					atomic_clearbits_int(&pp->pg_flags,
-					    PG_CLEANCHK);
-			}
 		}
 	}
 
-	/*
-	 * now do it.   note: we must update ppnext in body of loop or we
-	 * will get stuck.  we need to use ppnext because we may free "pp"
-	 * before doing the next loop.
-	 */
-
-	if (by_list) {
-		pp = TAILQ_FIRST(&uobj->memq);
-	} else {
-		curoff = start;
-		pp = uvm_pagelookup(uobj, curoff);
-	}
-
-	ppnext = NULL;	/* XXX: shut up gcc */
 	ppsp = NULL;		/* XXX: shut up gcc */
 	uvm_lock_pageq();	/* page queues locked */
-
 	/* locked: both page queues and uobj */
-	for ( ; (by_list && pp != NULL) ||
-	  (!by_list && curoff < stop) ; pp = ppnext) {
-
-		if (by_list) {
-
-			/*
-			 * range check
-			 */
-
-			if (!all &&
-			    (pp->offset < start || pp->offset >= stop)) {
-				ppnext = TAILQ_NEXT(pp, listq);
-				continue;
-			}
-
-		} else {
-
-			/*
-			 * null check
-			 */
-
-			curoff += PAGE_SIZE;
-			if (pp == NULL) {
-				if (curoff < stop)
-					ppnext = uvm_pagelookup(uobj, curoff);
-				continue;
-			}
-
-		}
+	for (curoff = start; curoff < stop; curoff += PAGE_SIZE) {
+		if ((pp = uvm_pagelookup(uobj, curoff)) == NULL)
+			continue;
 
 		/*
 		 * handle case where we do not need to clean page (either
@@ -943,37 +800,32 @@ uvn_flush(struct uvm_object *uobj, voff_t start, voff_t stop, int flags)
 		}
 
 		/*
-		 * if we don't need a clean... load ppnext and dispose of pp
+		 * if we don't need a clean... deactivate/free pages then cont.
 		 */
 		if (!needs_clean) {
-			/* load ppnext */
-			if (by_list)
-				ppnext = TAILQ_NEXT(pp, listq);
-			else {
-				if (curoff < stop)
-					ppnext = uvm_pagelookup(uobj, curoff);
-			}
-
-			/* now dispose of pp */
 			if (flags & PGO_DEACTIVATE) {
 				if ((pp->pg_flags & PQ_INACTIVE) == 0 &&
 				    pp->wire_count == 0) {
 					pmap_page_protect(pp, VM_PROT_NONE);
 					uvm_pagedeactivate(pp);
 				}
-
 			} else if (flags & PGO_FREE) {
 				if (pp->pg_flags & PG_BUSY) {
-					/* release busy pages */
 					atomic_setbits_int(&pp->pg_flags,
-					    PG_RELEASED);
+					    PG_WANTED);
+					uvm_unlock_pageq();
+					UVM_UNLOCK_AND_WAIT(pp,
+					    &uobj->vmobjlock, 0, "uvn_flsh", 0);
+					simple_lock(&uobj->vmobjlock);
+					uvm_lock_pageq();
+					curoff -= PAGE_SIZE;
+					continue;
 				} else {
 					pmap_page_protect(pp, VM_PROT_NONE);
 					/* removed page from object */
 					uvm_pagefree(pp);
 				}
 			}
-			/* ppnext is valid so we can continue... */
 			continue;
 		}
 
@@ -989,7 +841,9 @@ uvn_flush(struct uvm_object *uobj, voff_t start, voff_t stop, int flags)
 		atomic_setbits_int(&pp->pg_flags, PG_BUSY);
 		UVM_PAGE_OWN(pp, "uvn_flush");
 		pmap_page_protect(pp, VM_PROT_READ);
-		pp_version = pp->pg_version;
+		/* if we're async, free the page in aiodoned */
+		if ((flags & (PGO_FREE|PGO_SYNCIO)) == PGO_FREE)
+			atomic_setbits_int(&pp->pg_flags, PG_RELEASED);
 ReTry:
 		ppsp = pps;
 		npages = sizeof(pps) / sizeof(struct vm_page *);
@@ -1000,11 +854,11 @@ ReTry:
 		/* unlocked: page queues, uobj */
 
 		/*
-		 * at this point nothing is locked.   if we did an async I/O
-		 * it is remotely possible for the async i/o to complete and
-		 * the page "pp" be freed or what not before we get a chance
-		 * to relock the object.   in order to detect this, we have
-		 * saved the version number of the page in "pp_version".
+		 * if we did an async I/O it is remotely possible for the
+		 * async i/o to complete and the page "pp" be freed or what
+		 * not before we get a chance to relock the object. Therefore,
+		 * we only touch it when it won't be freed, RELEASED took care
+		 * of the rest.
 		 */
 
 		/* relock! */
@@ -1013,7 +867,7 @@ ReTry:
 
 		/*
 		 * VM_PAGER_AGAIN: given the structure of this pager, this
-		 * can only happen when  we are doing async I/O and can't
+		 * can only happen when we are doing async I/O and can't
 		 * map the pages into kernel memory (pager_map) due to lack
 		 * of vm space.   if this happens we drop back to sync I/O.
 		 */
@@ -1031,6 +885,10 @@ ReTry:
 	panic("uvn_flush: PGO_SYNCIO return 'try again' error (impossible)");
 #endif
 			flags |= PGO_SYNCIO;
+			if (flags & PGO_FREE)
+				atomic_clearbits_int(&pp->pg_flags,
+				    PG_RELEASED);
+
 			goto ReTry;
 		}
 
@@ -1042,66 +900,20 @@ ReTry:
 		 */
 
 		/*
-		 * for pending async i/o if we are not deactivating/freeing
-		 * we can move on to the next page.
+		 * for pending async i/o if we are not deactivating
+		 * we can move on to the next page. aiodoned deals with
+		 * the freeing case for us.
 		 */
-
-		if (result == VM_PAGER_PEND) {
-
-			if ((flags & (PGO_DEACTIVATE|PGO_FREE)) == 0) {
-				/*
-				 * no per-page ops: refresh ppnext and continue
-				 */
-				if (by_list) {
-					if (pp->pg_version == pp_version)
-						ppnext = TAILQ_NEXT(pp, listq);
-					else
-						/* reset */
-						ppnext = TAILQ_FIRST(&uobj->memq);
-				} else {
-					if (curoff < stop)
-						ppnext = uvm_pagelookup(uobj,
-						    curoff);
-				}
-				continue;
-			}
-
-			/* need to do anything here? */
-		}
+		if (result == VM_PAGER_PEND && (flags & PGO_DEACTIVATE) == 0)
+			continue;
 
 		/*
-		 * need to look at each page of the I/O operation.  we defer
-		 * processing "pp" until the last trip through this "for" loop
-		 * so that we can load "ppnext" for the main loop after we
-		 * play with the cluster pages [thus the "npages + 1" in the
-		 * loop below].
+		 * need to look at each page of the I/O operation, and do what
+		 * we gotta do.
 		 */
 
-		for (lcv = 0 ; lcv < npages + 1 ; lcv++) {
-
-			/*
-			 * handle ppnext for outside loop, and saving pp
-			 * until the end.
-			 */
-			if (lcv < npages) {
-				if (ppsp[lcv] == pp)
-					continue; /* skip pp until the end */
-				ptmp = ppsp[lcv];
-			} else {
-				ptmp = pp;
-
-				/* set up next page for outer loop */
-				if (by_list) {
-					if (pp->pg_version == pp_version)
-						ppnext = TAILQ_NEXT(pp, listq);
-					else
-						/* reset */
-						ppnext = TAILQ_FIRST(&uobj->memq);
-				} else {
-					if (curoff < stop)
-					ppnext = uvm_pagelookup(uobj, curoff);
-				}
-			}
+		for (lcv = 0 ; lcv < npages; lcv++) {
+			ptmp = ppsp[lcv];
 
 			/*
 			 * verify the page didn't get moved while obj was
@@ -1125,25 +937,10 @@ ReTry:
 				atomic_clearbits_int(&ptmp->pg_flags,
 				    PG_WANTED|PG_BUSY);
 				UVM_PAGE_OWN(ptmp, NULL);
-				if (ptmp->pg_flags & PG_RELEASED) {
-
-					/*
-					 * pgo_releasepg needs to grab the
-					 * pageq lock itself.
-					 */
-					uvm_unlock_pageq();
-					if (!uvn_releasepg(ptmp, NULL))
-						return (TRUE);
-
-					uvm_lock_pageq();	/* relock */
-					continue;		/* next page */
-
-				} else {
-					atomic_setbits_int(&ptmp->pg_flags,
-					    PG_CLEAN|PG_CLEANCHK);
-					if ((flags & PGO_FREE) == 0)
-						pmap_clear_modify(ptmp);
-				}
+				atomic_setbits_int(&ptmp->pg_flags,
+				    PG_CLEAN|PG_CLEANCHK);
+				if ((flags & PGO_FREE) == 0)
+					pmap_clear_modify(ptmp);
 			}
 
 			/*
@@ -1156,29 +953,21 @@ ReTry:
 					pmap_page_protect(ptmp, VM_PROT_NONE);
 					uvm_pagedeactivate(ptmp);
 				}
-
-			} else if (flags & PGO_FREE) {
-				if (result == VM_PAGER_PEND) {
-					if ((ptmp->pg_flags & PG_BUSY) != 0)
-						/* signal for i/o done */
-						atomic_setbits_int(
-						    &ptmp->pg_flags,
-						    PG_RELEASED);
-				} else {
-					if (result != VM_PAGER_OK) {
-						printf("uvn_flush: obj=%p, "
-						   "offset=0x%llx.  error "
-						   "during pageout.\n",
-						    pp->uobject,
-						    (long long)pp->offset);
-						printf("uvn_flush: WARNING: "
-						    "changes to page may be "
-						    "lost!\n");
-						retval = FALSE;
-					}
-					pmap_page_protect(ptmp, VM_PROT_NONE);
-					uvm_pagefree(ptmp);
+			} else if (flags & PGO_FREE &&
+			    result != VM_PAGER_PEND) {
+				if (result != VM_PAGER_OK) {
+					printf("uvn_flush: obj=%p, "
+					   "offset=0x%llx.  error "
+					   "during pageout.\n",
+					    pp->uobject,
+					    (long long)pp->offset);
+					printf("uvn_flush: WARNING: "
+					    "changes to page may be "
+					    "lost!\n");
+					retval = FALSE;
 				}
+				pmap_page_protect(ptmp, VM_PROT_NONE);
+				uvm_pagefree(ptmp);
 			}
 
 		}		/* end of "lcv" for loop */
@@ -1321,7 +1110,7 @@ uvn_get(struct uvm_object *uobj, voff_t offset, struct vm_page **pps,
 
 			/* to be useful must get a non-busy, non-released pg */
 			if (ptmp == NULL ||
-			    (ptmp->pg_flags & (PG_BUSY|PG_RELEASED)) != 0) {
+			    (ptmp->pg_flags & PG_BUSY) != 0) {
 				if (lcv == centeridx || (flags & PGO_ALLPAGES)
 				    != 0)
 				done = FALSE;	/* need to do a wait or I/O! */
@@ -1427,7 +1216,7 @@ uvn_get(struct uvm_object *uobj, voff_t offset, struct vm_page **pps,
 			}
 
 			/* page is there, see if we need to wait on it */
-			if ((ptmp->pg_flags & (PG_BUSY|PG_RELEASED)) != 0) {
+			if ((ptmp->pg_flags & PG_BUSY) != 0) {
 				atomic_setbits_int(&ptmp->pg_flags, PG_WANTED);
 				UVM_UNLOCK_AND_WAIT(ptmp,
 				    &uobj->vmobjlock, FALSE, "uvn_get",0);