summaryrefslogtreecommitdiff
path: root/sys
diff options
context:
space:
mode:
Diffstat (limited to 'sys')
-rw-r--r--sys/uvm/uvm_aobj.c235
-rw-r--r--sys/uvm/uvm_aobj.h4
-rw-r--r--sys/uvm/uvm_fault.c57
-rw-r--r--sys/uvm/uvm_km.c25
-rw-r--r--sys/uvm/uvm_loan.c26
-rw-r--r--sys/uvm/uvm_mmap.c5
-rw-r--r--sys/uvm/uvm_page.c11
-rw-r--r--sys/uvm/uvm_pager.c76
-rw-r--r--sys/uvm/uvm_pager.h4
-rw-r--r--sys/uvm/uvm_pdaemon.c51
-rw-r--r--sys/uvm/uvm_vnode.c375
11 files changed, 207 insertions, 662 deletions
diff --git a/sys/uvm/uvm_aobj.c b/sys/uvm/uvm_aobj.c
index 1bcc615fadc..cc5a075e77f 100644
--- a/sys/uvm/uvm_aobj.c
+++ b/sys/uvm/uvm_aobj.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: uvm_aobj.c,v 1.45 2009/06/16 23:54:57 oga Exp $ */
+/* $OpenBSD: uvm_aobj.c,v 1.46 2009/07/22 21:05:37 oga Exp $ */
/* $NetBSD: uvm_aobj.c,v 1.39 2001/02/18 21:19:08 chs Exp $ */
/*
@@ -139,7 +139,7 @@ struct pool uao_swhash_elt_pool;
* uvm_aobj: the actual anon-backed uvm_object
*
* => the uvm_object is at the top of the structure, this allows
- * (struct uvm_device *) == (struct uvm_object *)
+ * (struct uvm_aobj *) == (struct uvm_object *)
* => only one of u_swslots and u_swhash is used in any given aobj
*/
@@ -176,8 +176,6 @@ static void uao_free(struct uvm_aobj *);
static int uao_get(struct uvm_object *, voff_t,
vm_page_t *, int *, int, vm_prot_t,
int, int);
-static boolean_t uao_releasepg(struct vm_page *,
- struct vm_page **);
static boolean_t uao_pagein(struct uvm_aobj *, int, int);
static boolean_t uao_pagein_page(struct uvm_aobj *, int);
@@ -194,10 +192,6 @@ struct uvm_pagerops aobj_pager = {
NULL, /* fault */
uao_flush, /* flush */
uao_get, /* get */
- NULL, /* put (done by pagedaemon) */
- NULL, /* cluster */
- NULL, /* mk_pcluster */
- uao_releasepg /* releasepg */
};
/*
@@ -565,7 +559,7 @@ uao_init(void)
simple_lock_init(&uao_list_lock);
/*
- * NOTE: Pages fror this pool must not come from a pageable
+ * NOTE: Pages for this pool must not come from a pageable
* kernel map!
*/
pool_init(&uao_swhash_elt_pool, sizeof(struct uao_swhash_elt),
@@ -641,8 +635,7 @@ void
uao_detach_locked(struct uvm_object *uobj)
{
struct uvm_aobj *aobj = (struct uvm_aobj *)uobj;
- struct vm_page *pg, *next;
- boolean_t busybody;
+ struct vm_page *pg;
UVMHIST_FUNC("uao_detach"); UVMHIST_CALLED(maphist);
/*
@@ -669,35 +662,26 @@ uao_detach_locked(struct uvm_object *uobj)
simple_unlock(&uao_list_lock);
/*
- * free all the pages that aren't PG_BUSY,
- * mark for release any that are.
+ * Free all pages left in the object. If they're busy, wait
+ * for them to become available before we kill it.
+ * Release swap resources then free the page.
*/
- busybody = FALSE;
- for (pg = TAILQ_FIRST(&uobj->memq); pg != NULL; pg = next) {
- next = TAILQ_NEXT(pg, listq);
+ uvm_lock_pageq();
+ while((pg = TAILQ_FIRST(&uobj->memq)) != NULL) {
if (pg->pg_flags & PG_BUSY) {
- atomic_setbits_int(&pg->pg_flags, PG_RELEASED);
- busybody = TRUE;
+ atomic_setbits_int(&pg->pg_flags, PG_WANTED);
+ uvm_unlock_pageq();
+ UVM_UNLOCK_AND_WAIT(pg, &uobj->vmobjlock, 0,
+ "uao_det", 0);
+ simple_lock(&uobj->vmobjlock);
+ uvm_lock_pageq();
continue;
}
-
- /* zap the mappings, free the swap slot, free the page */
pmap_page_protect(pg, VM_PROT_NONE);
uao_dropswap(&aobj->u_obj, pg->offset >> PAGE_SHIFT);
- uvm_lock_pageq();
uvm_pagefree(pg);
- uvm_unlock_pageq();
- }
-
- /*
- * if we found any busy pages, we're done for now.
- * mark the aobj for death, releasepg will finish up for us.
- */
- if (busybody) {
- aobj->u_flags |= UAO_FLAG_KILLME;
- simple_unlock(&aobj->u_obj.vmobjlock);
- return;
}
+ uvm_unlock_pageq();
/*
* finally, free the rest.
@@ -729,35 +713,6 @@ uao_detach_locked(struct uvm_object *uobj)
* => we return TRUE unless we encountered some sort of I/O error
* XXXJRT currently never happens, as we never directly initiate
* XXXJRT I/O
- *
- * comment on "cleaning" object and PG_BUSY pages:
- * this routine is holding the lock on the object. the only time
- * that is can run into a PG_BUSY page that it does not own is if
- * some other process has started I/O on the page (e.g. either
- * a pagein or a pageout). if the PG_BUSY page is being paged
- * in, then it can not be dirty (!PG_CLEAN) because no one has
- * had a change to modify it yet. if the PG_BUSY page is being
- * paged out then it means that someone else has already started
- * cleaning the page for us (how nice!). in this case, if we
- * have syncio specified, then after we make our pass through the
- * object we need to wait for the other PG_BUSY pages to clear
- * off (i.e. we need to do an iosync). also note that once a
- * page is PG_BUSY is must stary in its object until it is un-busyed.
- * XXXJRT We never actually do this, as we are "flushing" anonymous
- * XXXJRT memory, which doesn't have persistent backing store.
- *
- * note on page traversal:
- * we can traverse the pages in an object either by going down the
- * linked list in "uobj->memq", or we can go over the address range
- * by page doing hash table lookups for each address. depending
- * on how many pages are in the object it may be cheaper to do one
- * or the other. we set "by_list" to true if we are using memq.
- * if the cost of a hash lookup was equal to the cost of the list
- * traversal we could compare the number of pages in the start->stop
- * range to the total number of pages in the object. however, it
- * seems that a hash table lookup is more expensive than the linked
- * list traversal, so we multiply the number of pages in the
- * start->stop range by a penalty which we define below.
*/
#define UAO_HASH_PENALTY 4 /* XXX: a guess */
@@ -766,19 +721,13 @@ boolean_t
uao_flush(struct uvm_object *uobj, voff_t start, voff_t stop, int flags)
{
struct uvm_aobj *aobj = (struct uvm_aobj *) uobj;
- struct vm_page *pp, *ppnext;
- boolean_t retval, by_list;
+ struct vm_page *pp;
voff_t curoff;
UVMHIST_FUNC("uao_flush"); UVMHIST_CALLED(maphist);
- curoff = 0; /* XXX: shut up gcc */
-
- retval = TRUE; /* default to success */
-
if (flags & PGO_ALLPAGES) {
start = 0;
stop = aobj->u_pages << PAGE_SHIFT;
- by_list = TRUE; /* always go by the list */
} else {
start = trunc_page(start);
stop = round_page(stop);
@@ -787,13 +736,10 @@ uao_flush(struct uvm_object *uobj, voff_t start, voff_t stop, int flags)
"flush (fixed)\n");
stop = aobj->u_pages << PAGE_SHIFT;
}
- by_list = (uobj->uo_npages <=
- ((stop - start) >> PAGE_SHIFT) * UAO_HASH_PENALTY);
}
- UVMHIST_LOG(maphist,
- " flush start=0x%lx, stop=0x%lx, by_list=%ld, flags=0x%lx",
- (u_long)start, (u_long)stop, by_list, flags);
+ UVMHIST_LOG(maphist, " flush start=0x%lx, stop=0x%lx, flags=0x%lx",
+ (u_long)start, (u_long)stop, flags, 0);
/*
* Don't need to do any work here if we're not freeing
@@ -802,44 +748,31 @@ uao_flush(struct uvm_object *uobj, voff_t start, voff_t stop, int flags)
if ((flags & (PGO_DEACTIVATE|PGO_FREE)) == 0) {
UVMHIST_LOG(maphist,
"<- done (no work to do)",0,0,0,0);
- return (retval);
+ return (TRUE);
}
- /*
- * now do it. note: we must update ppnext in the body of loop or we
- * will get stuck. we need to use ppnext because we may free "pp"
- * before doing the next loop.
- */
-
- if (by_list) {
- pp = TAILQ_FIRST(&uobj->memq);
- } else {
- curoff = start;
- pp = uvm_pagelookup(uobj, curoff);
- }
-
- ppnext = NULL; /* XXX: shut up gcc */
- uvm_lock_pageq(); /* page queues locked */
-
- /* locked: both page queues and uobj */
- for ( ; (by_list && pp != NULL) ||
- (!by_list && curoff < stop) ; pp = ppnext) {
- if (by_list) {
- ppnext = TAILQ_NEXT(pp, listq);
-
- /* range check */
- if (pp->offset < start || pp->offset >= stop)
- continue;
- } else {
+ /* locked: uobj */
+ curoff = start;
+ for (;;) {
+ if (curoff < stop) {
+ pp = uvm_pagelookup(uobj, curoff);
curoff += PAGE_SIZE;
- if (curoff < stop)
- ppnext = uvm_pagelookup(uobj, curoff);
-
- /* null check */
if (pp == NULL)
continue;
+ } else {
+ break;
}
-
+
+ /* Make sure page is unbusy, else wait for it. */
+ if (pp->pg_flags & PG_BUSY) {
+ atomic_setbits_int(&pp->pg_flags, PG_WANTED);
+ UVM_UNLOCK_AND_WAIT(pp, &uobj->vmobjlock, 0,
+ "uaoflsh", 0);
+ simple_lock(&uobj->vmobjlock);
+ curoff -= PAGE_SIZE;
+ continue;
+ }
+
switch (flags & (PGO_CLEANIT|PGO_FREE|PGO_DEACTIVATE)) {
/*
* XXX In these first 3 cases, we always just
@@ -848,7 +781,9 @@ uao_flush(struct uvm_object *uobj, voff_t start, voff_t stop, int flags)
* XXX in the future.
*/
case PGO_CLEANIT|PGO_FREE:
+ /* FALLTHROUGH */
case PGO_CLEANIT|PGO_DEACTIVATE:
+ /* FALLTHROUGH */
case PGO_DEACTIVATE:
deactivate_it:
/* skip the page if it's loaned or wired */
@@ -856,16 +791,13 @@ uao_flush(struct uvm_object *uobj, voff_t start, voff_t stop, int flags)
pp->wire_count != 0)
continue;
-#ifdef UBC
- /* ...and deactivate the page. */
- pmap_clear_reference(pp);
-#else
+ uvm_lock_pageq();
/* zap all mappings for the page. */
pmap_page_protect(pp, VM_PROT_NONE);
/* ...and deactivate the page. */
-#endif
uvm_pagedeactivate(pp);
+ uvm_unlock_pageq();
continue;
@@ -882,19 +814,13 @@ uao_flush(struct uvm_object *uobj, voff_t start, voff_t stop, int flags)
pp->wire_count != 0)
continue;
- /*
- * mark the page as released if its busy.
- */
- if (pp->pg_flags & PG_BUSY) {
- atomic_setbits_int(&pp->pg_flags, PG_RELEASED);
- continue;
- }
-
/* zap all mappings for the page. */
pmap_page_protect(pp, VM_PROT_NONE);
uao_dropswap(uobj, pp->offset >> PAGE_SHIFT);
+ uvm_lock_pageq();
uvm_pagefree(pp);
+ uvm_unlock_pageq();
continue;
@@ -903,11 +829,9 @@ uao_flush(struct uvm_object *uobj, voff_t start, voff_t stop, int flags)
}
}
- uvm_unlock_pageq();
-
UVMHIST_LOG(maphist,
"<- done, rv=%ld",retval,0,0,0);
- return (retval);
+ return (TRUE);
}
/*
@@ -989,10 +913,10 @@ uao_get(struct uvm_object *uobj, voff_t offset, struct vm_page **pps,
}
/*
- * to be useful must get a non-busy, non-released page
+ * to be useful must get a non-busy page
*/
if (ptmp == NULL ||
- (ptmp->pg_flags & (PG_BUSY|PG_RELEASED)) != 0) {
+ (ptmp->pg_flags & PG_BUSY) != 0) {
if (lcv == centeridx ||
(flags & PGO_ALLPAGES) != 0)
/* need to do a wait or I/O! */
@@ -1099,7 +1023,7 @@ uao_get(struct uvm_object *uobj, voff_t offset, struct vm_page **pps,
}
/* page is there, see if we need to wait on it */
- if ((ptmp->pg_flags & (PG_BUSY|PG_RELEASED)) != 0) {
+ if ((ptmp->pg_flags & PG_BUSY) != 0) {
atomic_setbits_int(&ptmp->pg_flags, PG_WANTED);
UVMHIST_LOG(pdhist,
"sleeping, ptmp->flags 0x%lx\n",
@@ -1138,8 +1062,7 @@ uao_get(struct uvm_object *uobj, voff_t offset, struct vm_page **pps,
/*
* just zero the page if there's nothing in swap.
*/
- if (swslot == 0)
- {
+ if (swslot == 0) {
/*
* page hasn't existed before, just zero it.
*/
@@ -1216,65 +1139,6 @@ uao_get(struct uvm_object *uobj, voff_t offset, struct vm_page **pps,
}
/*
- * uao_releasepg: handle released page in an aobj
- *
- * => "pg" is a PG_BUSY [caller owns it], PG_RELEASED page that we need
- * to dispose of.
- * => caller must handle PG_WANTED case
- * => called with page's object locked, pageq's unlocked
- * => returns TRUE if page's object is still alive, FALSE if we
- * killed the page's object. if we return TRUE, then we
- * return with the object locked.
- * => if (nextpgp != NULL) => we return the next page on the queue, and return
- * with the page queues locked [for pagedaemon]
- * => if (nextpgp == NULL) => we return with page queues unlocked [normal case]
- * => we kill the aobj if it is not referenced and we are suppose to
- * kill it ("KILLME").
- */
-static boolean_t
-uao_releasepg(struct vm_page *pg, struct vm_page **nextpgp /* OUT */)
-{
- struct uvm_aobj *aobj = (struct uvm_aobj *) pg->uobject;
-
- KASSERT(pg->pg_flags & PG_RELEASED);
-
- /*
- * dispose of the page [caller handles PG_WANTED] and swap slot.
- */
- pmap_page_protect(pg, VM_PROT_NONE);
- uao_dropswap(&aobj->u_obj, pg->offset >> PAGE_SHIFT);
- uvm_lock_pageq();
- if (nextpgp)
- *nextpgp = TAILQ_NEXT(pg, pageq); /* next page for daemon */
- uvm_pagefree(pg);
- if (!nextpgp)
- uvm_unlock_pageq(); /* keep locked for daemon */
-
- /*
- * if we're not killing the object, we're done.
- */
- if ((aobj->u_flags & UAO_FLAG_KILLME) == 0)
- return TRUE;
- KASSERT(aobj->u_obj.uo_refs == 0);
-
- /*
- * if there are still pages in the object, we're done for now.
- */
- if (aobj->u_obj.uo_npages != 0)
- return TRUE;
-
- KASSERT(TAILQ_EMPTY(&aobj->u_obj.memq));
-
- /*
- * finally, free the rest.
- */
- uao_free(aobj);
-
- return FALSE;
-}
-
-
-/*
* uao_dropswap: release any swap resources from this aobj page.
*
* => aobj must be locked or have a reference count of 0.
@@ -1476,7 +1340,6 @@ uao_pagein_page(struct uvm_aobj *aobj, int pageidx)
return FALSE;
}
- KASSERT((pg->pg_flags & PG_RELEASED) == 0);
/*
* ok, we've got the page now.
diff --git a/sys/uvm/uvm_aobj.h b/sys/uvm/uvm_aobj.h
index c567a181473..27645df5b9b 100644
--- a/sys/uvm/uvm_aobj.h
+++ b/sys/uvm/uvm_aobj.h
@@ -1,4 +1,4 @@
-/* $OpenBSD: uvm_aobj.h,v 1.11 2009/06/16 23:54:57 oga Exp $ */
+/* $OpenBSD: uvm_aobj.h,v 1.12 2009/07/22 21:05:37 oga Exp $ */
/* $NetBSD: uvm_aobj.h,v 1.10 2000/01/11 06:57:49 chs Exp $ */
/*
@@ -55,8 +55,6 @@
#define UAO_FLAG_KERNSWAP 0x2 /* enable kernel swap */
/* internal flags */
-#define UAO_FLAG_KILLME 0x4 /* aobj should die when last released
- * page is no longer PG_BUSY ... */
#define UAO_FLAG_NOSWAP 0x8 /* aobj can't swap (kernel obj only!) */
#ifdef _KERNEL
diff --git a/sys/uvm/uvm_fault.c b/sys/uvm/uvm_fault.c
index a10fd054e84..e5eb6b8e3b0 100644
--- a/sys/uvm/uvm_fault.c
+++ b/sys/uvm/uvm_fault.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: uvm_fault.c,v 1.57 2009/06/16 23:54:58 oga Exp $ */
+/* $OpenBSD: uvm_fault.c,v 1.58 2009/07/22 21:05:37 oga Exp $ */
/* $NetBSD: uvm_fault.c,v 1.51 2000/08/06 00:22:53 thorpej Exp $ */
/*
@@ -921,10 +921,10 @@ ReFault:
/*
* if center page is resident and not
- * PG_BUSY|PG_RELEASED then pgo_get
- * made it PG_BUSY for us and gave
- * us a handle to it. remember this
- * page as "uobjpage." (for later use).
+ * PG_BUSY, then pgo_get made it PG_BUSY
+ * for us and gave us a handle to it.
+ * remember this page as "uobjpage."
+ * (for later use).
*/
if (lcv == centeridx) {
@@ -966,8 +966,8 @@ ReFault:
(wired ? PMAP_WIRED : 0));
/*
- * NOTE: page can't be PG_WANTED or PG_RELEASED
- * because we've held the lock the whole time
+ * NOTE: page can't be PG_WANTED because
+ * we've held the lock the whole time
* we've had the handle.
*/
@@ -1371,15 +1371,12 @@ Case2:
/* locked(!locked): uobj, uobjpage */
/*
- * verify that the page has not be released and re-verify
- * that amap slot is still free. if there is a problem,
- * we unlock and clean up.
+ * Re-verify that amap slot is still free. if there is
+ * a problem, we unlock and clean up.
*/
- if ((uobjpage->pg_flags & PG_RELEASED) != 0 ||
- (locked && amap &&
- amap_lookup(&ufi.entry->aref,
- ufi.orig_rvaddr - ufi.entry->start))) {
+ if (locked && amap && amap_lookup(&ufi.entry->aref,
+ ufi.orig_rvaddr - ufi.entry->start)) {
if (locked)
uvmfault_unlockall(&ufi, amap, NULL, NULL);
locked = FALSE;
@@ -1398,17 +1395,6 @@ Case2:
/* still holding object lock */
wakeup(uobjpage);
- if (uobjpage->pg_flags & PG_RELEASED) {
- uvmexp.fltpgrele++;
- KASSERT(uobj->pgops->pgo_releasepg != NULL);
-
- /* frees page */
- if (uobj->pgops->pgo_releasepg(uobjpage,NULL))
- /* unlock if still alive */
- simple_unlock(&uobj->vmobjlock);
- goto ReFault;
- }
-
uvm_lock_pageq();
/* make sure it is in queues */
uvm_pageactivate(uobjpage);
@@ -1423,9 +1409,8 @@ Case2:
}
/*
- * we have the data in uobjpage which is PG_BUSY and
- * !PG_RELEASED. we are holding object lock (so the page
- * can't be released on us).
+ * we have the data in uobjpage which is PG_BUSY and we are
+ * holding object lock.
*/
/* locked: maps(read), amap(if !null), uobj, uobjpage */
@@ -1439,8 +1424,6 @@ Case2:
/*
* notes:
* - at this point uobjpage can not be NULL
- * - at this point uobjpage can not be PG_RELEASED (since we checked
- * for it above)
* - at this point uobjpage could be PG_WANTED (handle later)
*/
@@ -1627,9 +1610,7 @@ Case2:
}
/*
- * dispose of uobjpage. it can't be PG_RELEASED
- * since we still hold the object lock.
- * drop handle to uobj as well.
+ * dispose of uobjpage. drop handle to uobj as well.
*/
if (uobjpage->pg_flags & PG_WANTED)
@@ -1692,11 +1673,6 @@ Case2:
if (pg->pg_flags & PG_WANTED)
wakeup(pg); /* lock still held */
- /*
- * note that pg can't be PG_RELEASED since we did not drop
- * the object lock since the last time we checked.
- */
-
atomic_clearbits_int(&pg->pg_flags, PG_BUSY|PG_FAKE|PG_WANTED);
UVM_PAGE_OWN(pg, NULL);
uvmfault_unlockall(&ufi, amap, uobj, NULL);
@@ -1736,11 +1712,6 @@ Case2:
if (pg->pg_flags & PG_WANTED)
wakeup(pg); /* lock still held */
- /*
- * note that pg can't be PG_RELEASED since we did not drop the object
- * lock since the last time we checked.
- */
-
atomic_clearbits_int(&pg->pg_flags, PG_BUSY|PG_FAKE|PG_WANTED);
UVM_PAGE_OWN(pg, NULL);
uvmfault_unlockall(&ufi, amap, uobj, NULL);
diff --git a/sys/uvm/uvm_km.c b/sys/uvm/uvm_km.c
index e6843631968..f30878ad873 100644
--- a/sys/uvm/uvm_km.c
+++ b/sys/uvm/uvm_km.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: uvm_km.c,v 1.73 2009/06/17 00:13:59 oga Exp $ */
+/* $OpenBSD: uvm_km.c,v 1.74 2009/07/22 21:05:37 oga Exp $ */
/* $NetBSD: uvm_km.c,v 1.42 2001/01/14 02:10:01 thorpej Exp $ */
/*
@@ -276,8 +276,12 @@ uvm_km_pgremove(struct uvm_object *uobj, vaddr_t start, vaddr_t end)
pp->pg_flags & PG_BUSY, 0, 0);
if (pp->pg_flags & PG_BUSY) {
- /* owner must check for this when done */
- atomic_setbits_int(&pp->pg_flags, PG_RELEASED);
+ atomic_setbits_int(&pp->pg_flags, PG_WANTED);
+ UVM_UNLOCK_AND_WAIT(pp, &uobj->vmobjlock, 0,
+ "km_pgrm", 0);
+ simple_lock(&uobj->vmobjlock);
+ curoff -= PAGE_SIZE; /* loop back to us */
+ continue;
} else {
/* free the swap slot... */
uao_dropswap(uobj, curoff >> PAGE_SHIFT);
@@ -511,21 +515,6 @@ uvm_km_alloc1(struct vm_map *map, vsize_t size, vsize_t align, boolean_t zeroit)
loopva = kva;
while (size) {
simple_lock(&uvm.kernel_object->vmobjlock);
- pg = uvm_pagelookup(uvm.kernel_object, offset);
-
- /*
- * if we found a page in an unallocated region, it must be
- * released
- */
- if (pg) {
- if ((pg->pg_flags & PG_RELEASED) == 0)
- panic("uvm_km_alloc1: non-released page");
- atomic_setbits_int(&pg->pg_flags, PG_WANTED);
- UVM_UNLOCK_AND_WAIT(pg, &uvm.kernel_object->vmobjlock,
- FALSE, "km_alloc", 0);
- continue; /* retry */
- }
-
/* allocate ram */
pg = uvm_pagealloc(uvm.kernel_object, offset, NULL, 0);
if (pg) {
diff --git a/sys/uvm/uvm_loan.c b/sys/uvm/uvm_loan.c
index caa6284f1fa..20af9ad7e63 100644
--- a/sys/uvm/uvm_loan.c
+++ b/sys/uvm/uvm_loan.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: uvm_loan.c,v 1.33 2009/06/17 00:13:59 oga Exp $ */
+/* $OpenBSD: uvm_loan.c,v 1.34 2009/07/22 21:05:37 oga Exp $ */
/* $NetBSD: uvm_loan.c,v 1.22 2000/06/27 17:29:25 mrg Exp $ */
/*
@@ -462,14 +462,12 @@ uvm_loanuobj(struct uvm_faultinfo *ufi, void ***output, int flags, vaddr_t va)
simple_lock(&uobj->vmobjlock);
/*
- * verify that the page has not be released and re-verify
- * that amap slot is still free. if there is a problem we
- * drop our lock (thus force a lookup refresh/retry).
+ * Re-verify that amap slot is still free. if there is a
+ * problem we drop our lock (thus force a lookup refresh/retry).
*/
- if ((pg->pg_flags & PG_RELEASED) != 0 ||
- (locked && amap && amap_lookup(&ufi->entry->aref,
- ufi->orig_rvaddr - ufi->entry->start))) {
+ if (locked && amap && amap_lookup(&ufi->entry->aref,
+ ufi->orig_rvaddr - ufi->entry->start)) {
if (locked)
uvmfault_unlockall(ufi, amap, NULL, NULL);
@@ -486,17 +484,6 @@ uvm_loanuobj(struct uvm_faultinfo *ufi, void ***output, int flags, vaddr_t va)
/* still holding object lock */
wakeup(pg);
- if (pg->pg_flags & PG_RELEASED) {
-#ifdef DIAGNOSTIC
- if (uobj->pgops->pgo_releasepg == NULL)
- panic("uvm_loanuobj: object has no releasepg function");
-#endif
- /* frees page */
- if (uobj->pgops->pgo_releasepg(pg, NULL))
- simple_unlock(&uobj->vmobjlock);
- return (0);
- }
-
uvm_lock_pageq();
uvm_pageactivate(pg); /* make sure it is in queues */
uvm_unlock_pageq();
@@ -509,8 +496,7 @@ uvm_loanuobj(struct uvm_faultinfo *ufi, void ***output, int flags, vaddr_t va)
/*
* at this point we have the page we want ("pg") marked PG_BUSY for us
- * and we have all data structures locked. do the loanout. page can
- * not be PG_RELEASED (we caught this above).
+ * and we have all data structures locked. do the loanout.
*/
if ((flags & UVM_LOAN_TOANON) == 0) { /* loan to wired-kernel page? */
diff --git a/sys/uvm/uvm_mmap.c b/sys/uvm/uvm_mmap.c
index d02482e8ab0..34f9f89e8ff 100644
--- a/sys/uvm/uvm_mmap.c
+++ b/sys/uvm/uvm_mmap.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: uvm_mmap.c,v 1.77 2009/07/09 22:29:56 thib Exp $ */
+/* $OpenBSD: uvm_mmap.c,v 1.78 2009/07/22 21:05:37 oga Exp $ */
/* $NetBSD: uvm_mmap.c,v 1.49 2001/02/18 21:19:08 chs Exp $ */
/*
@@ -298,8 +298,7 @@ sys_mincore(struct proc *p, void *v, register_t *retval)
*/
if (UVM_ET_ISOBJ(entry)) {
KASSERT(!UVM_OBJ_IS_KERN_OBJECT(entry->object.uvm_obj));
- if (entry->object.uvm_obj->pgops->pgo_releasepg
- == NULL) {
+ if (entry->object.uvm_obj->pgops->pgo_fault != NULL) {
pgi = 1;
for (/* nothing */; start < lim;
start += PAGE_SIZE, vec++)
diff --git a/sys/uvm/uvm_page.c b/sys/uvm/uvm_page.c
index 1c6e60df7cf..d0415ed2ea6 100644
--- a/sys/uvm/uvm_page.c
+++ b/sys/uvm/uvm_page.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: uvm_page.c,v 1.91 2009/06/17 00:13:59 oga Exp $ */
+/* $OpenBSD: uvm_page.c,v 1.92 2009/07/22 21:05:37 oga Exp $ */
/* $NetBSD: uvm_page.c,v 1.44 2000/11/27 08:40:04 chs Exp $ */
/*
@@ -1245,7 +1245,14 @@ uvm_page_unbusy(struct vm_page **pgs, int npgs)
UVMHIST_LOG(pdhist, "releasing pg %p", pg,0,0,0);
uobj = pg->uobject;
if (uobj != NULL) {
- uobj->pgops->pgo_releasepg(pg, NULL);
+ uvm_lock_pageq();
+ pmap_page_protect(pg, VM_PROT_NONE);
+ /* XXX won't happen right now */
+ if (pg->pg_flags & PQ_ANON)
+ uao_dropswap(uobj,
+ pg->offset >> PAGE_SHIFT);
+ uvm_pagefree(pg);
+ uvm_unlock_pageq();
} else {
atomic_clearbits_int(&pg->pg_flags, PG_BUSY);
UVM_PAGE_OWN(pg, NULL);
diff --git a/sys/uvm/uvm_pager.c b/sys/uvm/uvm_pager.c
index 38cbd7701bc..ee9048b63d2 100644
--- a/sys/uvm/uvm_pager.c
+++ b/sys/uvm/uvm_pager.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: uvm_pager.c,v 1.53 2009/06/17 00:13:59 oga Exp $ */
+/* $OpenBSD: uvm_pager.c,v 1.54 2009/07/22 21:05:37 oga Exp $ */
/* $NetBSD: uvm_pager.c,v 1.36 2000/11/27 18:26:41 chs Exp $ */
/*
@@ -339,6 +339,8 @@ uvm_pagermapout(vaddr_t kva, int npages)
* PGO_ALLPAGES: all pages in object are valid targets
* !PGO_ALLPAGES: use "lo" and "hi" to limit range of cluster
* PGO_DOACTCLUST: include active pages in cluster.
+ * PGO_FREE: set the PG_RELEASED bits on the cluster so they'll be freed
+ * in async io (caller must clean on error).
* NOTE: the caller should clear PG_CLEANCHK bits if PGO_DOACTCLUST.
* PG_CLEANCHK is only a hint, but clearing will help reduce
* the number of calls we make to the pmap layer.
@@ -440,6 +442,14 @@ uvm_mk_pcluster(struct uvm_object *uobj, struct vm_page **pps, int *npages,
atomic_setbits_int(&pclust->pg_flags, PG_BUSY);
UVM_PAGE_OWN(pclust, "uvm_mk_pcluster");
+ /*
+ * If we want to free after io is done, and we're
+ * async, set the released flag
+ */
+ if ((flags & (PGO_FREE|PGO_SYNCIO)) == PGO_FREE)
+ atomic_setbits_int(&pclust->pg_flags,
+ PG_RELEASED);
+
/* XXX: protect wired page? see above comment. */
pmap_page_protect(pclust, VM_PROT_READ);
if (!forward) {
@@ -481,6 +491,7 @@ uvm_mk_pcluster(struct uvm_object *uobj, struct vm_page **pps, int *npages,
* PGO_DOACTCLUST: include "PQ_ACTIVE" pages as valid targets
* PGO_SYNCIO: do SYNC I/O (no async)
* PGO_PDFREECLUST: pagedaemon: drop cluster on successful I/O
+ * PGO_FREE: tell the aio daemon to free pages in the async case.
* => start/stop: if (uobj && !PGO_ALLPAGES) limit targets to this range
* if (!uobj) start is the (daddr64_t) of the starting swapblk
* => return state:
@@ -704,8 +715,6 @@ uvm_pager_dropcluster(struct uvm_object *uobj, struct vm_page *pg,
struct vm_page **ppsp, int *npages, int flags)
{
int lcv;
- boolean_t obj_is_alive;
- struct uvm_object *saved_uobj;
/*
* drop all pages but "pg"
@@ -747,9 +756,8 @@ uvm_pager_dropcluster(struct uvm_object *uobj, struct vm_page *pg,
}
/* if page was released, release it. otherwise un-busy it */
- if (ppsp[lcv]->pg_flags & PG_RELEASED) {
-
- if (ppsp[lcv]->pg_flags & PQ_ANON) {
+ if (ppsp[lcv]->pg_flags & PG_RELEASED &&
+ ppsp[lcv]->pg_flags & PQ_ANON) {
/* so that anfree will free */
atomic_clearbits_int(&ppsp[lcv]->pg_flags,
PG_BUSY);
@@ -761,34 +769,13 @@ uvm_pager_dropcluster(struct uvm_object *uobj, struct vm_page *pg,
uvm_anfree(ppsp[lcv]->uanon);
continue;
- }
-
- /*
- * pgo_releasepg will dump the page for us
- */
-
- saved_uobj = ppsp[lcv]->uobject;
- obj_is_alive =
- saved_uobj->pgops->pgo_releasepg(ppsp[lcv], NULL);
-
- /* for normal objects, "pg" is still PG_BUSY by us,
- * so obj can't die */
- KASSERT(!uobj || obj_is_alive);
-
- /* only unlock the object if it is still alive... */
- if (obj_is_alive && saved_uobj != uobj)
- simple_unlock(&saved_uobj->vmobjlock);
-
+ } else {
/*
- * XXXCDC: suppose uobj died in the pgo_releasepg?
- * how pass that
- * info up to caller. we are currently ignoring it...
+ * if we were planning on async io then we would
+ * have PG_RELEASED set, clear that with the others.
*/
-
- continue; /* next page */
- } else {
atomic_clearbits_int(&ppsp[lcv]->pg_flags,
- PG_BUSY|PG_WANTED|PG_FAKE);
+ PG_BUSY|PG_WANTED|PG_FAKE|PG_RELEASED);
UVM_PAGE_OWN(ppsp[lcv], NULL);
}
@@ -812,33 +799,6 @@ uvm_pager_dropcluster(struct uvm_object *uobj, struct vm_page *pg,
}
}
-#ifdef UBC
-/*
- * interrupt-context iodone handler for nested i/o bufs.
- *
- * => must be at splbio().
- */
-
-void
-uvm_aio_biodone1(struct buf *bp)
-{
- struct buf *mbp = bp->b_private;
-
- splassert(IPL_BIO);
-
- KASSERT(mbp != bp);
- if (bp->b_flags & B_ERROR) {
- mbp->b_flags |= B_ERROR;
- mbp->b_error = bp->b_error;
- }
- mbp->b_resid -= bp->b_bcount;
- pool_put(&bufpool, bp);
- if (mbp->b_resid == 0) {
- biodone(mbp);
- }
-}
-#endif
-
/*
* interrupt-context iodone handler for single-buf i/os
* or the top-level buf of a nested-buf i/o.
diff --git a/sys/uvm/uvm_pager.h b/sys/uvm/uvm_pager.h
index fea487b7863..4e0151fa2b3 100644
--- a/sys/uvm/uvm_pager.h
+++ b/sys/uvm/uvm_pager.h
@@ -1,4 +1,4 @@
-/* $OpenBSD: uvm_pager.h,v 1.26 2009/06/17 00:13:59 oga Exp $ */
+/* $OpenBSD: uvm_pager.h,v 1.27 2009/07/22 21:05:37 oga Exp $ */
/* $NetBSD: uvm_pager.h,v 1.20 2000/11/27 08:40:05 chs Exp $ */
/*
@@ -109,8 +109,6 @@ struct uvm_pagerops {
struct vm_page ** (*pgo_mk_pcluster)(struct uvm_object *,
struct vm_page **, int *, struct vm_page *,
int, voff_t, voff_t);
- /* release page */
- boolean_t (*pgo_releasepg)(struct vm_page *, struct vm_page **);
};
/* pager flags [mostly for flush] */
diff --git a/sys/uvm/uvm_pdaemon.c b/sys/uvm/uvm_pdaemon.c
index e16ae26ea3b..046368e697a 100644
--- a/sys/uvm/uvm_pdaemon.c
+++ b/sys/uvm/uvm_pdaemon.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: uvm_pdaemon.c,v 1.51 2009/06/26 20:26:02 oga Exp $ */
+/* $OpenBSD: uvm_pdaemon.c,v 1.52 2009/07/22 21:05:37 oga Exp $ */
/* $NetBSD: uvm_pdaemon.c,v 1.23 2000/08/20 10:24:14 bjh21 Exp $ */
/*
@@ -820,40 +820,25 @@ uvmpd_scan_inactive(struct pglist *pglst)
atomic_clearbits_int(&p->pg_flags, PG_BUSY|PG_WANTED);
UVM_PAGE_OWN(p, NULL);
- /* released during I/O? */
+ /* released during I/O? Can only happen for anons */
if (p->pg_flags & PG_RELEASED) {
- if (anon) {
- /*
- * remove page so we can get nextpg,
- * also zero out anon so we don't use
- * it after the free.
- */
- anon->an_page = NULL;
- p->uanon = NULL;
-
- simple_unlock(&anon->an_lock);
- uvm_anfree(anon); /* kills anon */
- pmap_page_protect(p, VM_PROT_NONE);
- anon = NULL;
- uvm_lock_pageq();
- nextpg = TAILQ_NEXT(p, pageq);
- /* free released page */
- uvm_pagefree(p);
-
- } else {
-
- /*
- * pgo_releasepg nukes the page and
- * gets "nextpg" for us. it returns
- * with the page queues locked (when
- * given nextpg ptr).
- */
+ KASSERT(anon != NULL);
+ /*
+ * remove page so we can get nextpg,
+ * also zero out anon so we don't use
+ * it after the free.
+ */
+ anon->an_page = NULL;
+ p->uanon = NULL;
- if (!uobj->pgops->pgo_releasepg(p,
- &nextpg))
- /* uobj died after release */
- uobj = NULL;
- }
+ simple_unlock(&anon->an_lock);
+ uvm_anfree(anon); /* kills anon */
+ pmap_page_protect(p, VM_PROT_NONE);
+ anon = NULL;
+ uvm_lock_pageq();
+ nextpg = TAILQ_NEXT(p, pageq);
+ /* free released page */
+ uvm_pagefree(p);
} else { /* page was not released during I/O */
uvm_lock_pageq();
nextpg = TAILQ_NEXT(p, pageq);
diff --git a/sys/uvm/uvm_vnode.c b/sys/uvm/uvm_vnode.c
index 465db6d50bd..15130e95ded 100644
--- a/sys/uvm/uvm_vnode.c
+++ b/sys/uvm/uvm_vnode.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: uvm_vnode.c,v 1.67 2009/07/09 22:29:56 thib Exp $ */
+/* $OpenBSD: uvm_vnode.c,v 1.68 2009/07/22 21:05:37 oga Exp $ */
/* $NetBSD: uvm_vnode.c,v 1.36 2000/11/24 20:34:01 chs Exp $ */
/*
@@ -93,7 +93,6 @@ void uvn_init(void);
int uvn_io(struct uvm_vnode *, vm_page_t *, int, int, int);
int uvn_put(struct uvm_object *, vm_page_t *, int, boolean_t);
void uvn_reference(struct uvm_object *);
-boolean_t uvn_releasepg(struct vm_page *, struct vm_page **);
/*
* master pager structure
@@ -109,7 +108,6 @@ struct uvm_pagerops uvm_vnodeops = {
uvn_put,
uvn_cluster,
uvm_mk_pcluster, /* use generic version of this: see uvm_pager.c */
- uvn_releasepg,
};
/*
@@ -415,31 +413,24 @@ uvn_detach(struct uvm_object *uobj)
/*
* given the structure of this pager, the above flush request will
* create the following state: all the pages that were in the object
- * have either been free'd or they are marked PG_BUSY|PG_RELEASED.
- * the PG_BUSY bit was set either by us or the daemon for async I/O.
- * in either case, if we have pages left we can't kill the object
- * yet because i/o is pending. in this case we set the "relkill"
- * flag which will cause pgo_releasepg to kill the object once all
- * the I/O's are done [pgo_releasepg will be called from the aiodone
- * routine or from the page daemon].
+ * have either been free'd or they are marked PG_BUSY and in the
+ * middle of an async io. If we still have pages we set the "relkill"
+ * state, so that in the case the vnode gets terminated we know
+ * to leave it alone. Otherwise we'll kill the vnode when it's empty.
*/
- if (uobj->uo_npages) { /* I/O pending. iodone will free */
-#ifdef DEBUG
- /*
- * XXXCDC: very unlikely to happen until we have async i/o
- * so print a little info message in case it does.
- */
- printf("uvn_detach: vn %p has pages left after flush - "
- "relkill mode\n", uobj);
-#endif
- uvn->u_flags |= UVM_VNODE_RELKILL;
- simple_unlock(&uobj->vmobjlock);
- UVMHIST_LOG(maphist,"<- done! (releasepg will kill obj)", 0, 0,
- 0, 0);
- return;
+ uvn->u_flags |= UVM_VNODE_RELKILL;
+ /* wait on any outstanding io */
+ while (uobj->uo_npages && uvn->u_flags & UVM_VNODE_RELKILL) {
+ uvn->u_flags |= UVM_VNODE_IOSYNC;
+ UVM_UNLOCK_AND_WAIT(&uvn->u_nio, &uvn->u_obj.vmobjlock, FALSE,
+ "uvn_term",0);
+ simple_lock(&uvn->u_obj.vmobjlock);
}
+ if ((uvn->u_flags & UVM_VNODE_RELKILL) == 0)
+ return;
+
/*
* kill object now. note that we can't be on the sync q because
* all references are gone.
@@ -490,8 +481,6 @@ uvn_detach(struct uvm_object *uobj)
* => the caller must XLOCK and VOP_LOCK the vnode before calling us
* [protects us from getting a vnode that is already in the DYING
* state...]
- * => unlike uvn_detach, this function must not return until all the
- * uvn's pages are disposed of.
* => in case [2] the uvn is still alive after this call, but all I/O
* ops will fail (due to the backing vnode now being "dead"). this
* will prob. kill any process using the uvn due to pgo_get failing.
@@ -535,8 +524,8 @@ uvm_vnp_terminate(struct vnode *vp)
/*
* it is possible that the uvn was detached and is in the relkill
- * state [i.e. waiting for async i/o to finish so that releasepg can
- * kill object]. we take over the vnode now and cancel the relkill.
+ * state [i.e. waiting for async i/o to finish].
+ * we take over the vnode now and cancel the relkill.
* we want to know when the i/o is done so we can recycle right
* away. note that a uvn can only be in the RELKILL state if it
* has a zero reference count.
@@ -630,72 +619,6 @@ uvm_vnp_terminate(struct vnode *vp)
}
/*
- * uvn_releasepg: handled a released page in a uvn
- *
- * => "pg" is a PG_BUSY [caller owns it], PG_RELEASED page that we need
- * to dispose of.
- * => caller must handled PG_WANTED case
- * => called with page's object locked, pageq's unlocked
- * => returns TRUE if page's object is still alive, FALSE if we
- * killed the page's object. if we return TRUE, then we
- * return with the object locked.
- * => if (nextpgp != NULL) => we return pageq.tqe_next here, and return
- * with the page queues locked [for pagedaemon]
- * => if (nextpgp == NULL) => we return with page queues unlocked [normal case]
- * => we kill the uvn if it is not referenced and we are suppose to
- * kill it ("relkill").
- */
-
-boolean_t
-uvn_releasepg(struct vm_page *pg, struct vm_page **nextpgp /* OUT */)
-{
- struct uvm_vnode *uvn = (struct uvm_vnode *) pg->uobject;
- struct vnode *vp = (struct vnode *)uvn;
-#ifdef DIAGNOSTIC
- if ((pg->pg_flags & PG_RELEASED) == 0)
- panic("uvn_releasepg: page not released!");
-#endif
-
- /*
- * dispose of the page [caller handles PG_WANTED]
- */
- pmap_page_protect(pg, VM_PROT_NONE);
- uvm_lock_pageq();
- if (nextpgp)
- *nextpgp = TAILQ_NEXT(pg, pageq); /* next page for daemon */
- uvm_pagefree(pg);
- if (!nextpgp)
- uvm_unlock_pageq();
-
- /*
- * now see if we need to kill the object
- */
- if (uvn->u_flags & UVM_VNODE_RELKILL) {
- if (uvn->u_obj.uo_refs)
- panic("uvn_releasepg: kill flag set on referenced "
- "object!");
- if (uvn->u_obj.uo_npages == 0) {
- if (uvn->u_flags & UVM_VNODE_WRITEABLE) {
- LIST_REMOVE(uvn, u_wlist);
- }
-#ifdef DIAGNOSTIC
- if (!TAILQ_EMPTY(&uvn->u_obj.memq))
- panic("uvn_releasepg: pages in object with npages == 0");
-#endif
- if (uvn->u_flags & UVM_VNODE_WANTED)
- /* still holding object lock */
- wakeup(uvn);
-
- uvn->u_flags = 0; /* DEAD! */
- simple_unlock(&uvn->u_obj.vmobjlock);
- vrele(vp);
- return (FALSE);
- }
- }
- return (TRUE);
-}
-
-/*
* NOTE: currently we have to use VOP_READ/VOP_WRITE because they go
* through the buffer cache and allow I/O in any size. These VOPs use
* synchronous i/o. [vs. VOP_STRATEGY which can be async, but doesn't
@@ -729,8 +652,6 @@ uvn_releasepg(struct vm_page *pg, struct vm_page **nextpgp /* OUT */)
* - if (object->iosync && u_naio == 0) { wakeup &uvn->u_naio }
* - get "page" structures (atop?).
* - handle "wanted" pages
- * - handle "released" pages [using pgo_releasepg]
- * >>> pgo_releasepg may kill the object
* dont forget to look at "object" wanted flag in all cases.
*/
@@ -792,15 +713,13 @@ boolean_t
uvn_flush(struct uvm_object *uobj, voff_t start, voff_t stop, int flags)
{
struct uvm_vnode *uvn = (struct uvm_vnode *) uobj;
- struct vm_page *pp, *ppnext, *ptmp;
+ struct vm_page *pp, *ptmp;
struct vm_page *pps[MAXBSIZE >> PAGE_SHIFT], **ppsp;
int npages, result, lcv;
- boolean_t retval, need_iosync, by_list, needs_clean, all;
+ boolean_t retval, need_iosync, needs_clean;
voff_t curoff;
- u_short pp_version;
UVMHIST_FUNC("uvn_flush"); UVMHIST_CALLED(maphist);
- curoff = 0; /* XXX: shut up gcc */
/*
* get init vals and determine how we are going to traverse object
*/
@@ -808,24 +727,16 @@ uvn_flush(struct uvm_object *uobj, voff_t start, voff_t stop, int flags)
need_iosync = FALSE;
retval = TRUE; /* return value */
if (flags & PGO_ALLPAGES) {
- all = TRUE;
- by_list = TRUE; /* always go by the list */
+ start = 0;
+ stop = round_page(uvn->u_size);
} else {
start = trunc_page(start);
- stop = round_page(stop);
-#ifdef DEBUG
- if (stop > round_page(uvn->u_size))
- printf("uvn_flush: strange, got an out of range "
- "flush (fixed)\n");
-#endif
- all = FALSE;
- by_list = (uobj->uo_npages <=
- ((stop - start) >> PAGE_SHIFT) * UVN_HASH_PENALTY);
+ stop = MIN(round_page(stop), round_page(uvn->u_size));
}
UVMHIST_LOG(maphist,
- " flush start=0x%lx, stop=0x%lx, by_list=%ld, flags=0x%lx",
- (u_long)start, (u_long)stop, by_list, flags);
+ " flush start=0x%lx, stop=0x%lx, flags=0x%lx",
+ (u_long)start, (u_long)stop, flags, 0);
/*
* PG_CLEANCHK: this bit is used by the pgo_mk_pcluster function as
@@ -838,75 +749,21 @@ uvn_flush(struct uvm_object *uobj, voff_t start, voff_t stop, int flags)
* [borrowed PG_CLEANCHK idea from FreeBSD VM]
*/
- if ((flags & PGO_CLEANIT) != 0 &&
- uobj->pgops->pgo_mk_pcluster != NULL) {
- if (by_list) {
- TAILQ_FOREACH(pp, &uobj->memq, listq) {
- if (!all &&
- (pp->offset < start || pp->offset >= stop))
- continue;
+ if ((flags & PGO_CLEANIT) != 0) {
+ KASSERT(uobj->pgops->pgo_mk_pcluster != 0);
+ for (curoff = start ; curoff < stop; curoff += PAGE_SIZE) {
+ if ((pp = uvm_pagelookup(uobj, curoff)) != NULL)
atomic_clearbits_int(&pp->pg_flags,
PG_CLEANCHK);
- }
-
- } else { /* by hash */
- for (curoff = start ; curoff < stop;
- curoff += PAGE_SIZE) {
- pp = uvm_pagelookup(uobj, curoff);
- if (pp)
- atomic_clearbits_int(&pp->pg_flags,
- PG_CLEANCHK);
- }
}
}
- /*
- * now do it. note: we must update ppnext in body of loop or we
- * will get stuck. we need to use ppnext because we may free "pp"
- * before doing the next loop.
- */
-
- if (by_list) {
- pp = TAILQ_FIRST(&uobj->memq);
- } else {
- curoff = start;
- pp = uvm_pagelookup(uobj, curoff);
- }
-
- ppnext = NULL; /* XXX: shut up gcc */
ppsp = NULL; /* XXX: shut up gcc */
uvm_lock_pageq(); /* page queues locked */
-
/* locked: both page queues and uobj */
- for ( ; (by_list && pp != NULL) ||
- (!by_list && curoff < stop) ; pp = ppnext) {
-
- if (by_list) {
-
- /*
- * range check
- */
-
- if (!all &&
- (pp->offset < start || pp->offset >= stop)) {
- ppnext = TAILQ_NEXT(pp, listq);
- continue;
- }
-
- } else {
-
- /*
- * null check
- */
-
- curoff += PAGE_SIZE;
- if (pp == NULL) {
- if (curoff < stop)
- ppnext = uvm_pagelookup(uobj, curoff);
- continue;
- }
-
- }
+ for (curoff = start; curoff < stop; curoff += PAGE_SIZE) {
+ if ((pp = uvm_pagelookup(uobj, curoff)) == NULL)
+ continue;
/*
* handle case where we do not need to clean page (either
@@ -943,37 +800,32 @@ uvn_flush(struct uvm_object *uobj, voff_t start, voff_t stop, int flags)
}
/*
- * if we don't need a clean... load ppnext and dispose of pp
+ * if we don't need a clean... deactivate/free pages then cont.
*/
if (!needs_clean) {
- /* load ppnext */
- if (by_list)
- ppnext = TAILQ_NEXT(pp, listq);
- else {
- if (curoff < stop)
- ppnext = uvm_pagelookup(uobj, curoff);
- }
-
- /* now dispose of pp */
if (flags & PGO_DEACTIVATE) {
if ((pp->pg_flags & PQ_INACTIVE) == 0 &&
pp->wire_count == 0) {
pmap_page_protect(pp, VM_PROT_NONE);
uvm_pagedeactivate(pp);
}
-
} else if (flags & PGO_FREE) {
if (pp->pg_flags & PG_BUSY) {
- /* release busy pages */
atomic_setbits_int(&pp->pg_flags,
- PG_RELEASED);
+ PG_WANTED);
+ uvm_unlock_pageq();
+ UVM_UNLOCK_AND_WAIT(pp,
+ &uobj->vmobjlock, 0, "uvn_flsh", 0);
+ simple_lock(&uobj->vmobjlock);
+ uvm_lock_pageq();
+ curoff -= PAGE_SIZE;
+ continue;
} else {
pmap_page_protect(pp, VM_PROT_NONE);
/* removed page from object */
uvm_pagefree(pp);
}
}
- /* ppnext is valid so we can continue... */
continue;
}
@@ -989,7 +841,9 @@ uvn_flush(struct uvm_object *uobj, voff_t start, voff_t stop, int flags)
atomic_setbits_int(&pp->pg_flags, PG_BUSY);
UVM_PAGE_OWN(pp, "uvn_flush");
pmap_page_protect(pp, VM_PROT_READ);
- pp_version = pp->pg_version;
+ /* if we're async, free the page in aiodoned */
+ if ((flags & (PGO_FREE|PGO_SYNCIO)) == PGO_FREE)
+ atomic_setbits_int(&pp->pg_flags, PG_RELEASED);
ReTry:
ppsp = pps;
npages = sizeof(pps) / sizeof(struct vm_page *);
@@ -1000,11 +854,11 @@ ReTry:
/* unlocked: page queues, uobj */
/*
- * at this point nothing is locked. if we did an async I/O
- * it is remotely possible for the async i/o to complete and
- * the page "pp" be freed or what not before we get a chance
- * to relock the object. in order to detect this, we have
- * saved the version number of the page in "pp_version".
+ * if we did an async I/O it is remotely possible for the
+ * async i/o to complete and the page "pp" be freed or what
+ * not before we get a chance to relock the object. Therefore,
+ * we only touch it when it won't be freed, RELEASED took care
+ * of the rest.
*/
/* relock! */
@@ -1013,7 +867,7 @@ ReTry:
/*
* VM_PAGER_AGAIN: given the structure of this pager, this
- * can only happen when we are doing async I/O and can't
+ * can only happen when we are doing async I/O and can't
* map the pages into kernel memory (pager_map) due to lack
* of vm space. if this happens we drop back to sync I/O.
*/
@@ -1031,6 +885,10 @@ ReTry:
panic("uvn_flush: PGO_SYNCIO return 'try again' error (impossible)");
#endif
flags |= PGO_SYNCIO;
+ if (flags & PGO_FREE)
+ atomic_clearbits_int(&pp->pg_flags,
+ PG_RELEASED);
+
goto ReTry;
}
@@ -1042,66 +900,20 @@ ReTry:
*/
/*
- * for pending async i/o if we are not deactivating/freeing
- * we can move on to the next page.
+ * for pending async i/o if we are not deactivating
+ * we can move on to the next page. aiodoned deals with
+ * the freeing case for us.
*/
-
- if (result == VM_PAGER_PEND) {
-
- if ((flags & (PGO_DEACTIVATE|PGO_FREE)) == 0) {
- /*
- * no per-page ops: refresh ppnext and continue
- */
- if (by_list) {
- if (pp->pg_version == pp_version)
- ppnext = TAILQ_NEXT(pp, listq);
- else
- /* reset */
- ppnext = TAILQ_FIRST(&uobj->memq);
- } else {
- if (curoff < stop)
- ppnext = uvm_pagelookup(uobj,
- curoff);
- }
- continue;
- }
-
- /* need to do anything here? */
- }
+ if (result == VM_PAGER_PEND && (flags & PGO_DEACTIVATE) == 0)
+ continue;
/*
- * need to look at each page of the I/O operation. we defer
- * processing "pp" until the last trip through this "for" loop
- * so that we can load "ppnext" for the main loop after we
- * play with the cluster pages [thus the "npages + 1" in the
- * loop below].
+ * need to look at each page of the I/O operation, and do what
+ * we gotta do.
*/
- for (lcv = 0 ; lcv < npages + 1 ; lcv++) {
-
- /*
- * handle ppnext for outside loop, and saving pp
- * until the end.
- */
- if (lcv < npages) {
- if (ppsp[lcv] == pp)
- continue; /* skip pp until the end */
- ptmp = ppsp[lcv];
- } else {
- ptmp = pp;
-
- /* set up next page for outer loop */
- if (by_list) {
- if (pp->pg_version == pp_version)
- ppnext = TAILQ_NEXT(pp, listq);
- else
- /* reset */
- ppnext = TAILQ_FIRST(&uobj->memq);
- } else {
- if (curoff < stop)
- ppnext = uvm_pagelookup(uobj, curoff);
- }
- }
+ for (lcv = 0 ; lcv < npages; lcv++) {
+ ptmp = ppsp[lcv];
/*
* verify the page didn't get moved while obj was
@@ -1125,25 +937,10 @@ ReTry:
atomic_clearbits_int(&ptmp->pg_flags,
PG_WANTED|PG_BUSY);
UVM_PAGE_OWN(ptmp, NULL);
- if (ptmp->pg_flags & PG_RELEASED) {
-
- /*
- * pgo_releasepg needs to grab the
- * pageq lock itself.
- */
- uvm_unlock_pageq();
- if (!uvn_releasepg(ptmp, NULL))
- return (TRUE);
-
- uvm_lock_pageq(); /* relock */
- continue; /* next page */
-
- } else {
- atomic_setbits_int(&ptmp->pg_flags,
- PG_CLEAN|PG_CLEANCHK);
- if ((flags & PGO_FREE) == 0)
- pmap_clear_modify(ptmp);
- }
+ atomic_setbits_int(&ptmp->pg_flags,
+ PG_CLEAN|PG_CLEANCHK);
+ if ((flags & PGO_FREE) == 0)
+ pmap_clear_modify(ptmp);
}
/*
@@ -1156,29 +953,21 @@ ReTry:
pmap_page_protect(ptmp, VM_PROT_NONE);
uvm_pagedeactivate(ptmp);
}
-
- } else if (flags & PGO_FREE) {
- if (result == VM_PAGER_PEND) {
- if ((ptmp->pg_flags & PG_BUSY) != 0)
- /* signal for i/o done */
- atomic_setbits_int(
- &ptmp->pg_flags,
- PG_RELEASED);
- } else {
- if (result != VM_PAGER_OK) {
- printf("uvn_flush: obj=%p, "
- "offset=0x%llx. error "
- "during pageout.\n",
- pp->uobject,
- (long long)pp->offset);
- printf("uvn_flush: WARNING: "
- "changes to page may be "
- "lost!\n");
- retval = FALSE;
- }
- pmap_page_protect(ptmp, VM_PROT_NONE);
- uvm_pagefree(ptmp);
+ } else if (flags & PGO_FREE &&
+ result != VM_PAGER_PEND) {
+ if (result != VM_PAGER_OK) {
+ printf("uvn_flush: obj=%p, "
+ "offset=0x%llx. error "
+ "during pageout.\n",
+ pp->uobject,
+ (long long)pp->offset);
+ printf("uvn_flush: WARNING: "
+ "changes to page may be "
+ "lost!\n");
+ retval = FALSE;
}
+ pmap_page_protect(ptmp, VM_PROT_NONE);
+ uvm_pagefree(ptmp);
}
} /* end of "lcv" for loop */
@@ -1321,7 +1110,7 @@ uvn_get(struct uvm_object *uobj, voff_t offset, struct vm_page **pps,
/* to be useful must get a non-busy, non-released pg */
if (ptmp == NULL ||
- (ptmp->pg_flags & (PG_BUSY|PG_RELEASED)) != 0) {
+ (ptmp->pg_flags & PG_BUSY) != 0) {
if (lcv == centeridx || (flags & PGO_ALLPAGES)
!= 0)
done = FALSE; /* need to do a wait or I/O! */
@@ -1427,7 +1216,7 @@ uvn_get(struct uvm_object *uobj, voff_t offset, struct vm_page **pps,
}
/* page is there, see if we need to wait on it */
- if ((ptmp->pg_flags & (PG_BUSY|PG_RELEASED)) != 0) {
+ if ((ptmp->pg_flags & PG_BUSY) != 0) {
atomic_setbits_int(&ptmp->pg_flags, PG_WANTED);
UVM_UNLOCK_AND_WAIT(ptmp,
&uobj->vmobjlock, FALSE, "uvn_get",0);