diff options
| -rw-r--r-- | sys/uvm/uvm_aobj.c | 235 | ||||
| -rw-r--r-- | sys/uvm/uvm_aobj.h | 4 | ||||
| -rw-r--r-- | sys/uvm/uvm_fault.c | 57 | ||||
| -rw-r--r-- | sys/uvm/uvm_km.c | 25 | ||||
| -rw-r--r-- | sys/uvm/uvm_loan.c | 26 | ||||
| -rw-r--r-- | sys/uvm/uvm_mmap.c | 5 | ||||
| -rw-r--r-- | sys/uvm/uvm_page.c | 11 | ||||
| -rw-r--r-- | sys/uvm/uvm_pager.c | 76 | ||||
| -rw-r--r-- | sys/uvm/uvm_pager.h | 4 | ||||
| -rw-r--r-- | sys/uvm/uvm_pdaemon.c | 51 | ||||
| -rw-r--r-- | sys/uvm/uvm_vnode.c | 375 |
11 files changed, 207 insertions, 662 deletions
diff --git a/sys/uvm/uvm_aobj.c b/sys/uvm/uvm_aobj.c index 1bcc615fadc..cc5a075e77f 100644 --- a/sys/uvm/uvm_aobj.c +++ b/sys/uvm/uvm_aobj.c @@ -1,4 +1,4 @@ -/* $OpenBSD: uvm_aobj.c,v 1.45 2009/06/16 23:54:57 oga Exp $ */ +/* $OpenBSD: uvm_aobj.c,v 1.46 2009/07/22 21:05:37 oga Exp $ */ /* $NetBSD: uvm_aobj.c,v 1.39 2001/02/18 21:19:08 chs Exp $ */ /* @@ -139,7 +139,7 @@ struct pool uao_swhash_elt_pool; * uvm_aobj: the actual anon-backed uvm_object * * => the uvm_object is at the top of the structure, this allows - * (struct uvm_device *) == (struct uvm_object *) + * (struct uvm_aobj *) == (struct uvm_object *) * => only one of u_swslots and u_swhash is used in any given aobj */ @@ -176,8 +176,6 @@ static void uao_free(struct uvm_aobj *); static int uao_get(struct uvm_object *, voff_t, vm_page_t *, int *, int, vm_prot_t, int, int); -static boolean_t uao_releasepg(struct vm_page *, - struct vm_page **); static boolean_t uao_pagein(struct uvm_aobj *, int, int); static boolean_t uao_pagein_page(struct uvm_aobj *, int); @@ -194,10 +192,6 @@ struct uvm_pagerops aobj_pager = { NULL, /* fault */ uao_flush, /* flush */ uao_get, /* get */ - NULL, /* put (done by pagedaemon) */ - NULL, /* cluster */ - NULL, /* mk_pcluster */ - uao_releasepg /* releasepg */ }; /* @@ -565,7 +559,7 @@ uao_init(void) simple_lock_init(&uao_list_lock); /* - * NOTE: Pages fror this pool must not come from a pageable + * NOTE: Pages for this pool must not come from a pageable * kernel map! */ pool_init(&uao_swhash_elt_pool, sizeof(struct uao_swhash_elt), @@ -641,8 +635,7 @@ void uao_detach_locked(struct uvm_object *uobj) { struct uvm_aobj *aobj = (struct uvm_aobj *)uobj; - struct vm_page *pg, *next; - boolean_t busybody; + struct vm_page *pg; UVMHIST_FUNC("uao_detach"); UVMHIST_CALLED(maphist); /* @@ -669,35 +662,26 @@ uao_detach_locked(struct uvm_object *uobj) simple_unlock(&uao_list_lock); /* - * free all the pages that aren't PG_BUSY, - * mark for release any that are. + * Free all pages left in the object. If they're busy, wait + * for them to become available before we kill it. + * Release swap resources then free the page. */ - busybody = FALSE; - for (pg = TAILQ_FIRST(&uobj->memq); pg != NULL; pg = next) { - next = TAILQ_NEXT(pg, listq); + uvm_lock_pageq(); + while((pg = TAILQ_FIRST(&uobj->memq)) != NULL) { if (pg->pg_flags & PG_BUSY) { - atomic_setbits_int(&pg->pg_flags, PG_RELEASED); - busybody = TRUE; + atomic_setbits_int(&pg->pg_flags, PG_WANTED); + uvm_unlock_pageq(); + UVM_UNLOCK_AND_WAIT(pg, &uobj->vmobjlock, 0, + "uao_det", 0); + simple_lock(&uobj->vmobjlock); + uvm_lock_pageq(); continue; } - - /* zap the mappings, free the swap slot, free the page */ pmap_page_protect(pg, VM_PROT_NONE); uao_dropswap(&aobj->u_obj, pg->offset >> PAGE_SHIFT); - uvm_lock_pageq(); uvm_pagefree(pg); - uvm_unlock_pageq(); - } - - /* - * if we found any busy pages, we're done for now. - * mark the aobj for death, releasepg will finish up for us. - */ - if (busybody) { - aobj->u_flags |= UAO_FLAG_KILLME; - simple_unlock(&aobj->u_obj.vmobjlock); - return; } + uvm_unlock_pageq(); /* * finally, free the rest. @@ -729,35 +713,6 @@ uao_detach_locked(struct uvm_object *uobj) * => we return TRUE unless we encountered some sort of I/O error * XXXJRT currently never happens, as we never directly initiate * XXXJRT I/O - * - * comment on "cleaning" object and PG_BUSY pages: - * this routine is holding the lock on the object. the only time - * that is can run into a PG_BUSY page that it does not own is if - * some other process has started I/O on the page (e.g. either - * a pagein or a pageout). if the PG_BUSY page is being paged - * in, then it can not be dirty (!PG_CLEAN) because no one has - * had a change to modify it yet. if the PG_BUSY page is being - * paged out then it means that someone else has already started - * cleaning the page for us (how nice!). in this case, if we - * have syncio specified, then after we make our pass through the - * object we need to wait for the other PG_BUSY pages to clear - * off (i.e. we need to do an iosync). also note that once a - * page is PG_BUSY is must stary in its object until it is un-busyed. - * XXXJRT We never actually do this, as we are "flushing" anonymous - * XXXJRT memory, which doesn't have persistent backing store. - * - * note on page traversal: - * we can traverse the pages in an object either by going down the - * linked list in "uobj->memq", or we can go over the address range - * by page doing hash table lookups for each address. depending - * on how many pages are in the object it may be cheaper to do one - * or the other. we set "by_list" to true if we are using memq. - * if the cost of a hash lookup was equal to the cost of the list - * traversal we could compare the number of pages in the start->stop - * range to the total number of pages in the object. however, it - * seems that a hash table lookup is more expensive than the linked - * list traversal, so we multiply the number of pages in the - * start->stop range by a penalty which we define below. */ #define UAO_HASH_PENALTY 4 /* XXX: a guess */ @@ -766,19 +721,13 @@ boolean_t uao_flush(struct uvm_object *uobj, voff_t start, voff_t stop, int flags) { struct uvm_aobj *aobj = (struct uvm_aobj *) uobj; - struct vm_page *pp, *ppnext; - boolean_t retval, by_list; + struct vm_page *pp; voff_t curoff; UVMHIST_FUNC("uao_flush"); UVMHIST_CALLED(maphist); - curoff = 0; /* XXX: shut up gcc */ - - retval = TRUE; /* default to success */ - if (flags & PGO_ALLPAGES) { start = 0; stop = aobj->u_pages << PAGE_SHIFT; - by_list = TRUE; /* always go by the list */ } else { start = trunc_page(start); stop = round_page(stop); @@ -787,13 +736,10 @@ uao_flush(struct uvm_object *uobj, voff_t start, voff_t stop, int flags) "flush (fixed)\n"); stop = aobj->u_pages << PAGE_SHIFT; } - by_list = (uobj->uo_npages <= - ((stop - start) >> PAGE_SHIFT) * UAO_HASH_PENALTY); } - UVMHIST_LOG(maphist, - " flush start=0x%lx, stop=0x%lx, by_list=%ld, flags=0x%lx", - (u_long)start, (u_long)stop, by_list, flags); + UVMHIST_LOG(maphist, " flush start=0x%lx, stop=0x%lx, flags=0x%lx", + (u_long)start, (u_long)stop, flags, 0); /* * Don't need to do any work here if we're not freeing @@ -802,44 +748,31 @@ uao_flush(struct uvm_object *uobj, voff_t start, voff_t stop, int flags) if ((flags & (PGO_DEACTIVATE|PGO_FREE)) == 0) { UVMHIST_LOG(maphist, "<- done (no work to do)",0,0,0,0); - return (retval); + return (TRUE); } - /* - * now do it. note: we must update ppnext in the body of loop or we - * will get stuck. we need to use ppnext because we may free "pp" - * before doing the next loop. - */ - - if (by_list) { - pp = TAILQ_FIRST(&uobj->memq); - } else { - curoff = start; - pp = uvm_pagelookup(uobj, curoff); - } - - ppnext = NULL; /* XXX: shut up gcc */ - uvm_lock_pageq(); /* page queues locked */ - - /* locked: both page queues and uobj */ - for ( ; (by_list && pp != NULL) || - (!by_list && curoff < stop) ; pp = ppnext) { - if (by_list) { - ppnext = TAILQ_NEXT(pp, listq); - - /* range check */ - if (pp->offset < start || pp->offset >= stop) - continue; - } else { + /* locked: uobj */ + curoff = start; + for (;;) { + if (curoff < stop) { + pp = uvm_pagelookup(uobj, curoff); curoff += PAGE_SIZE; - if (curoff < stop) - ppnext = uvm_pagelookup(uobj, curoff); - - /* null check */ if (pp == NULL) continue; + } else { + break; } - + + /* Make sure page is unbusy, else wait for it. */ + if (pp->pg_flags & PG_BUSY) { + atomic_setbits_int(&pp->pg_flags, PG_WANTED); + UVM_UNLOCK_AND_WAIT(pp, &uobj->vmobjlock, 0, + "uaoflsh", 0); + simple_lock(&uobj->vmobjlock); + curoff -= PAGE_SIZE; + continue; + } + switch (flags & (PGO_CLEANIT|PGO_FREE|PGO_DEACTIVATE)) { /* * XXX In these first 3 cases, we always just @@ -848,7 +781,9 @@ uao_flush(struct uvm_object *uobj, voff_t start, voff_t stop, int flags) * XXX in the future. */ case PGO_CLEANIT|PGO_FREE: + /* FALLTHROUGH */ case PGO_CLEANIT|PGO_DEACTIVATE: + /* FALLTHROUGH */ case PGO_DEACTIVATE: deactivate_it: /* skip the page if it's loaned or wired */ @@ -856,16 +791,13 @@ uao_flush(struct uvm_object *uobj, voff_t start, voff_t stop, int flags) pp->wire_count != 0) continue; -#ifdef UBC - /* ...and deactivate the page. */ - pmap_clear_reference(pp); -#else + uvm_lock_pageq(); /* zap all mappings for the page. */ pmap_page_protect(pp, VM_PROT_NONE); /* ...and deactivate the page. */ -#endif uvm_pagedeactivate(pp); + uvm_unlock_pageq(); continue; @@ -882,19 +814,13 @@ uao_flush(struct uvm_object *uobj, voff_t start, voff_t stop, int flags) pp->wire_count != 0) continue; - /* - * mark the page as released if its busy. - */ - if (pp->pg_flags & PG_BUSY) { - atomic_setbits_int(&pp->pg_flags, PG_RELEASED); - continue; - } - /* zap all mappings for the page. */ pmap_page_protect(pp, VM_PROT_NONE); uao_dropswap(uobj, pp->offset >> PAGE_SHIFT); + uvm_lock_pageq(); uvm_pagefree(pp); + uvm_unlock_pageq(); continue; @@ -903,11 +829,9 @@ uao_flush(struct uvm_object *uobj, voff_t start, voff_t stop, int flags) } } - uvm_unlock_pageq(); - UVMHIST_LOG(maphist, "<- done, rv=%ld",retval,0,0,0); - return (retval); + return (TRUE); } /* @@ -989,10 +913,10 @@ uao_get(struct uvm_object *uobj, voff_t offset, struct vm_page **pps, } /* - * to be useful must get a non-busy, non-released page + * to be useful must get a non-busy page */ if (ptmp == NULL || - (ptmp->pg_flags & (PG_BUSY|PG_RELEASED)) != 0) { + (ptmp->pg_flags & PG_BUSY) != 0) { if (lcv == centeridx || (flags & PGO_ALLPAGES) != 0) /* need to do a wait or I/O! */ @@ -1099,7 +1023,7 @@ uao_get(struct uvm_object *uobj, voff_t offset, struct vm_page **pps, } /* page is there, see if we need to wait on it */ - if ((ptmp->pg_flags & (PG_BUSY|PG_RELEASED)) != 0) { + if ((ptmp->pg_flags & PG_BUSY) != 0) { atomic_setbits_int(&ptmp->pg_flags, PG_WANTED); UVMHIST_LOG(pdhist, "sleeping, ptmp->flags 0x%lx\n", @@ -1138,8 +1062,7 @@ uao_get(struct uvm_object *uobj, voff_t offset, struct vm_page **pps, /* * just zero the page if there's nothing in swap. */ - if (swslot == 0) - { + if (swslot == 0) { /* * page hasn't existed before, just zero it. */ @@ -1216,65 +1139,6 @@ uao_get(struct uvm_object *uobj, voff_t offset, struct vm_page **pps, } /* - * uao_releasepg: handle released page in an aobj - * - * => "pg" is a PG_BUSY [caller owns it], PG_RELEASED page that we need - * to dispose of. - * => caller must handle PG_WANTED case - * => called with page's object locked, pageq's unlocked - * => returns TRUE if page's object is still alive, FALSE if we - * killed the page's object. if we return TRUE, then we - * return with the object locked. - * => if (nextpgp != NULL) => we return the next page on the queue, and return - * with the page queues locked [for pagedaemon] - * => if (nextpgp == NULL) => we return with page queues unlocked [normal case] - * => we kill the aobj if it is not referenced and we are suppose to - * kill it ("KILLME"). - */ -static boolean_t -uao_releasepg(struct vm_page *pg, struct vm_page **nextpgp /* OUT */) -{ - struct uvm_aobj *aobj = (struct uvm_aobj *) pg->uobject; - - KASSERT(pg->pg_flags & PG_RELEASED); - - /* - * dispose of the page [caller handles PG_WANTED] and swap slot. - */ - pmap_page_protect(pg, VM_PROT_NONE); - uao_dropswap(&aobj->u_obj, pg->offset >> PAGE_SHIFT); - uvm_lock_pageq(); - if (nextpgp) - *nextpgp = TAILQ_NEXT(pg, pageq); /* next page for daemon */ - uvm_pagefree(pg); - if (!nextpgp) - uvm_unlock_pageq(); /* keep locked for daemon */ - - /* - * if we're not killing the object, we're done. - */ - if ((aobj->u_flags & UAO_FLAG_KILLME) == 0) - return TRUE; - KASSERT(aobj->u_obj.uo_refs == 0); - - /* - * if there are still pages in the object, we're done for now. - */ - if (aobj->u_obj.uo_npages != 0) - return TRUE; - - KASSERT(TAILQ_EMPTY(&aobj->u_obj.memq)); - - /* - * finally, free the rest. - */ - uao_free(aobj); - - return FALSE; -} - - -/* * uao_dropswap: release any swap resources from this aobj page. * * => aobj must be locked or have a reference count of 0. @@ -1476,7 +1340,6 @@ uao_pagein_page(struct uvm_aobj *aobj, int pageidx) return FALSE; } - KASSERT((pg->pg_flags & PG_RELEASED) == 0); /* * ok, we've got the page now. diff --git a/sys/uvm/uvm_aobj.h b/sys/uvm/uvm_aobj.h index c567a181473..27645df5b9b 100644 --- a/sys/uvm/uvm_aobj.h +++ b/sys/uvm/uvm_aobj.h @@ -1,4 +1,4 @@ -/* $OpenBSD: uvm_aobj.h,v 1.11 2009/06/16 23:54:57 oga Exp $ */ +/* $OpenBSD: uvm_aobj.h,v 1.12 2009/07/22 21:05:37 oga Exp $ */ /* $NetBSD: uvm_aobj.h,v 1.10 2000/01/11 06:57:49 chs Exp $ */ /* @@ -55,8 +55,6 @@ #define UAO_FLAG_KERNSWAP 0x2 /* enable kernel swap */ /* internal flags */ -#define UAO_FLAG_KILLME 0x4 /* aobj should die when last released - * page is no longer PG_BUSY ... */ #define UAO_FLAG_NOSWAP 0x8 /* aobj can't swap (kernel obj only!) */ #ifdef _KERNEL diff --git a/sys/uvm/uvm_fault.c b/sys/uvm/uvm_fault.c index a10fd054e84..e5eb6b8e3b0 100644 --- a/sys/uvm/uvm_fault.c +++ b/sys/uvm/uvm_fault.c @@ -1,4 +1,4 @@ -/* $OpenBSD: uvm_fault.c,v 1.57 2009/06/16 23:54:58 oga Exp $ */ +/* $OpenBSD: uvm_fault.c,v 1.58 2009/07/22 21:05:37 oga Exp $ */ /* $NetBSD: uvm_fault.c,v 1.51 2000/08/06 00:22:53 thorpej Exp $ */ /* @@ -921,10 +921,10 @@ ReFault: /* * if center page is resident and not - * PG_BUSY|PG_RELEASED then pgo_get - * made it PG_BUSY for us and gave - * us a handle to it. remember this - * page as "uobjpage." (for later use). + * PG_BUSY, then pgo_get made it PG_BUSY + * for us and gave us a handle to it. + * remember this page as "uobjpage." + * (for later use). */ if (lcv == centeridx) { @@ -966,8 +966,8 @@ ReFault: (wired ? PMAP_WIRED : 0)); /* - * NOTE: page can't be PG_WANTED or PG_RELEASED - * because we've held the lock the whole time + * NOTE: page can't be PG_WANTED because + * we've held the lock the whole time * we've had the handle. */ @@ -1371,15 +1371,12 @@ Case2: /* locked(!locked): uobj, uobjpage */ /* - * verify that the page has not be released and re-verify - * that amap slot is still free. if there is a problem, - * we unlock and clean up. + * Re-verify that amap slot is still free. if there is + * a problem, we unlock and clean up. */ - if ((uobjpage->pg_flags & PG_RELEASED) != 0 || - (locked && amap && - amap_lookup(&ufi.entry->aref, - ufi.orig_rvaddr - ufi.entry->start))) { + if (locked && amap && amap_lookup(&ufi.entry->aref, + ufi.orig_rvaddr - ufi.entry->start)) { if (locked) uvmfault_unlockall(&ufi, amap, NULL, NULL); locked = FALSE; @@ -1398,17 +1395,6 @@ Case2: /* still holding object lock */ wakeup(uobjpage); - if (uobjpage->pg_flags & PG_RELEASED) { - uvmexp.fltpgrele++; - KASSERT(uobj->pgops->pgo_releasepg != NULL); - - /* frees page */ - if (uobj->pgops->pgo_releasepg(uobjpage,NULL)) - /* unlock if still alive */ - simple_unlock(&uobj->vmobjlock); - goto ReFault; - } - uvm_lock_pageq(); /* make sure it is in queues */ uvm_pageactivate(uobjpage); @@ -1423,9 +1409,8 @@ Case2: } /* - * we have the data in uobjpage which is PG_BUSY and - * !PG_RELEASED. we are holding object lock (so the page - * can't be released on us). + * we have the data in uobjpage which is PG_BUSY and we are + * holding object lock. */ /* locked: maps(read), amap(if !null), uobj, uobjpage */ @@ -1439,8 +1424,6 @@ Case2: /* * notes: * - at this point uobjpage can not be NULL - * - at this point uobjpage can not be PG_RELEASED (since we checked - * for it above) * - at this point uobjpage could be PG_WANTED (handle later) */ @@ -1627,9 +1610,7 @@ Case2: } /* - * dispose of uobjpage. it can't be PG_RELEASED - * since we still hold the object lock. - * drop handle to uobj as well. + * dispose of uobjpage. drop handle to uobj as well. */ if (uobjpage->pg_flags & PG_WANTED) @@ -1692,11 +1673,6 @@ Case2: if (pg->pg_flags & PG_WANTED) wakeup(pg); /* lock still held */ - /* - * note that pg can't be PG_RELEASED since we did not drop - * the object lock since the last time we checked. - */ - atomic_clearbits_int(&pg->pg_flags, PG_BUSY|PG_FAKE|PG_WANTED); UVM_PAGE_OWN(pg, NULL); uvmfault_unlockall(&ufi, amap, uobj, NULL); @@ -1736,11 +1712,6 @@ Case2: if (pg->pg_flags & PG_WANTED) wakeup(pg); /* lock still held */ - /* - * note that pg can't be PG_RELEASED since we did not drop the object - * lock since the last time we checked. - */ - atomic_clearbits_int(&pg->pg_flags, PG_BUSY|PG_FAKE|PG_WANTED); UVM_PAGE_OWN(pg, NULL); uvmfault_unlockall(&ufi, amap, uobj, NULL); diff --git a/sys/uvm/uvm_km.c b/sys/uvm/uvm_km.c index e6843631968..f30878ad873 100644 --- a/sys/uvm/uvm_km.c +++ b/sys/uvm/uvm_km.c @@ -1,4 +1,4 @@ -/* $OpenBSD: uvm_km.c,v 1.73 2009/06/17 00:13:59 oga Exp $ */ +/* $OpenBSD: uvm_km.c,v 1.74 2009/07/22 21:05:37 oga Exp $ */ /* $NetBSD: uvm_km.c,v 1.42 2001/01/14 02:10:01 thorpej Exp $ */ /* @@ -276,8 +276,12 @@ uvm_km_pgremove(struct uvm_object *uobj, vaddr_t start, vaddr_t end) pp->pg_flags & PG_BUSY, 0, 0); if (pp->pg_flags & PG_BUSY) { - /* owner must check for this when done */ - atomic_setbits_int(&pp->pg_flags, PG_RELEASED); + atomic_setbits_int(&pp->pg_flags, PG_WANTED); + UVM_UNLOCK_AND_WAIT(pp, &uobj->vmobjlock, 0, + "km_pgrm", 0); + simple_lock(&uobj->vmobjlock); + curoff -= PAGE_SIZE; /* loop back to us */ + continue; } else { /* free the swap slot... */ uao_dropswap(uobj, curoff >> PAGE_SHIFT); @@ -511,21 +515,6 @@ uvm_km_alloc1(struct vm_map *map, vsize_t size, vsize_t align, boolean_t zeroit) loopva = kva; while (size) { simple_lock(&uvm.kernel_object->vmobjlock); - pg = uvm_pagelookup(uvm.kernel_object, offset); - - /* - * if we found a page in an unallocated region, it must be - * released - */ - if (pg) { - if ((pg->pg_flags & PG_RELEASED) == 0) - panic("uvm_km_alloc1: non-released page"); - atomic_setbits_int(&pg->pg_flags, PG_WANTED); - UVM_UNLOCK_AND_WAIT(pg, &uvm.kernel_object->vmobjlock, - FALSE, "km_alloc", 0); - continue; /* retry */ - } - /* allocate ram */ pg = uvm_pagealloc(uvm.kernel_object, offset, NULL, 0); if (pg) { diff --git a/sys/uvm/uvm_loan.c b/sys/uvm/uvm_loan.c index caa6284f1fa..20af9ad7e63 100644 --- a/sys/uvm/uvm_loan.c +++ b/sys/uvm/uvm_loan.c @@ -1,4 +1,4 @@ -/* $OpenBSD: uvm_loan.c,v 1.33 2009/06/17 00:13:59 oga Exp $ */ +/* $OpenBSD: uvm_loan.c,v 1.34 2009/07/22 21:05:37 oga Exp $ */ /* $NetBSD: uvm_loan.c,v 1.22 2000/06/27 17:29:25 mrg Exp $ */ /* @@ -462,14 +462,12 @@ uvm_loanuobj(struct uvm_faultinfo *ufi, void ***output, int flags, vaddr_t va) simple_lock(&uobj->vmobjlock); /* - * verify that the page has not be released and re-verify - * that amap slot is still free. if there is a problem we - * drop our lock (thus force a lookup refresh/retry). + * Re-verify that amap slot is still free. if there is a + * problem we drop our lock (thus force a lookup refresh/retry). */ - if ((pg->pg_flags & PG_RELEASED) != 0 || - (locked && amap && amap_lookup(&ufi->entry->aref, - ufi->orig_rvaddr - ufi->entry->start))) { + if (locked && amap && amap_lookup(&ufi->entry->aref, + ufi->orig_rvaddr - ufi->entry->start)) { if (locked) uvmfault_unlockall(ufi, amap, NULL, NULL); @@ -486,17 +484,6 @@ uvm_loanuobj(struct uvm_faultinfo *ufi, void ***output, int flags, vaddr_t va) /* still holding object lock */ wakeup(pg); - if (pg->pg_flags & PG_RELEASED) { -#ifdef DIAGNOSTIC - if (uobj->pgops->pgo_releasepg == NULL) - panic("uvm_loanuobj: object has no releasepg function"); -#endif - /* frees page */ - if (uobj->pgops->pgo_releasepg(pg, NULL)) - simple_unlock(&uobj->vmobjlock); - return (0); - } - uvm_lock_pageq(); uvm_pageactivate(pg); /* make sure it is in queues */ uvm_unlock_pageq(); @@ -509,8 +496,7 @@ uvm_loanuobj(struct uvm_faultinfo *ufi, void ***output, int flags, vaddr_t va) /* * at this point we have the page we want ("pg") marked PG_BUSY for us - * and we have all data structures locked. do the loanout. page can - * not be PG_RELEASED (we caught this above). + * and we have all data structures locked. do the loanout. */ if ((flags & UVM_LOAN_TOANON) == 0) { /* loan to wired-kernel page? */ diff --git a/sys/uvm/uvm_mmap.c b/sys/uvm/uvm_mmap.c index d02482e8ab0..34f9f89e8ff 100644 --- a/sys/uvm/uvm_mmap.c +++ b/sys/uvm/uvm_mmap.c @@ -1,4 +1,4 @@ -/* $OpenBSD: uvm_mmap.c,v 1.77 2009/07/09 22:29:56 thib Exp $ */ +/* $OpenBSD: uvm_mmap.c,v 1.78 2009/07/22 21:05:37 oga Exp $ */ /* $NetBSD: uvm_mmap.c,v 1.49 2001/02/18 21:19:08 chs Exp $ */ /* @@ -298,8 +298,7 @@ sys_mincore(struct proc *p, void *v, register_t *retval) */ if (UVM_ET_ISOBJ(entry)) { KASSERT(!UVM_OBJ_IS_KERN_OBJECT(entry->object.uvm_obj)); - if (entry->object.uvm_obj->pgops->pgo_releasepg - == NULL) { + if (entry->object.uvm_obj->pgops->pgo_fault != NULL) { pgi = 1; for (/* nothing */; start < lim; start += PAGE_SIZE, vec++) diff --git a/sys/uvm/uvm_page.c b/sys/uvm/uvm_page.c index 1c6e60df7cf..d0415ed2ea6 100644 --- a/sys/uvm/uvm_page.c +++ b/sys/uvm/uvm_page.c @@ -1,4 +1,4 @@ -/* $OpenBSD: uvm_page.c,v 1.91 2009/06/17 00:13:59 oga Exp $ */ +/* $OpenBSD: uvm_page.c,v 1.92 2009/07/22 21:05:37 oga Exp $ */ /* $NetBSD: uvm_page.c,v 1.44 2000/11/27 08:40:04 chs Exp $ */ /* @@ -1245,7 +1245,14 @@ uvm_page_unbusy(struct vm_page **pgs, int npgs) UVMHIST_LOG(pdhist, "releasing pg %p", pg,0,0,0); uobj = pg->uobject; if (uobj != NULL) { - uobj->pgops->pgo_releasepg(pg, NULL); + uvm_lock_pageq(); + pmap_page_protect(pg, VM_PROT_NONE); + /* XXX won't happen right now */ + if (pg->pg_flags & PQ_ANON) + uao_dropswap(uobj, + pg->offset >> PAGE_SHIFT); + uvm_pagefree(pg); + uvm_unlock_pageq(); } else { atomic_clearbits_int(&pg->pg_flags, PG_BUSY); UVM_PAGE_OWN(pg, NULL); diff --git a/sys/uvm/uvm_pager.c b/sys/uvm/uvm_pager.c index 38cbd7701bc..ee9048b63d2 100644 --- a/sys/uvm/uvm_pager.c +++ b/sys/uvm/uvm_pager.c @@ -1,4 +1,4 @@ -/* $OpenBSD: uvm_pager.c,v 1.53 2009/06/17 00:13:59 oga Exp $ */ +/* $OpenBSD: uvm_pager.c,v 1.54 2009/07/22 21:05:37 oga Exp $ */ /* $NetBSD: uvm_pager.c,v 1.36 2000/11/27 18:26:41 chs Exp $ */ /* @@ -339,6 +339,8 @@ uvm_pagermapout(vaddr_t kva, int npages) * PGO_ALLPAGES: all pages in object are valid targets * !PGO_ALLPAGES: use "lo" and "hi" to limit range of cluster * PGO_DOACTCLUST: include active pages in cluster. + * PGO_FREE: set the PG_RELEASED bits on the cluster so they'll be freed + * in async io (caller must clean on error). * NOTE: the caller should clear PG_CLEANCHK bits if PGO_DOACTCLUST. * PG_CLEANCHK is only a hint, but clearing will help reduce * the number of calls we make to the pmap layer. @@ -440,6 +442,14 @@ uvm_mk_pcluster(struct uvm_object *uobj, struct vm_page **pps, int *npages, atomic_setbits_int(&pclust->pg_flags, PG_BUSY); UVM_PAGE_OWN(pclust, "uvm_mk_pcluster"); + /* + * If we want to free after io is done, and we're + * async, set the released flag + */ + if ((flags & (PGO_FREE|PGO_SYNCIO)) == PGO_FREE) + atomic_setbits_int(&pclust->pg_flags, + PG_RELEASED); + /* XXX: protect wired page? see above comment. */ pmap_page_protect(pclust, VM_PROT_READ); if (!forward) { @@ -481,6 +491,7 @@ uvm_mk_pcluster(struct uvm_object *uobj, struct vm_page **pps, int *npages, * PGO_DOACTCLUST: include "PQ_ACTIVE" pages as valid targets * PGO_SYNCIO: do SYNC I/O (no async) * PGO_PDFREECLUST: pagedaemon: drop cluster on successful I/O + * PGO_FREE: tell the aio daemon to free pages in the async case. * => start/stop: if (uobj && !PGO_ALLPAGES) limit targets to this range * if (!uobj) start is the (daddr64_t) of the starting swapblk * => return state: @@ -704,8 +715,6 @@ uvm_pager_dropcluster(struct uvm_object *uobj, struct vm_page *pg, struct vm_page **ppsp, int *npages, int flags) { int lcv; - boolean_t obj_is_alive; - struct uvm_object *saved_uobj; /* * drop all pages but "pg" @@ -747,9 +756,8 @@ uvm_pager_dropcluster(struct uvm_object *uobj, struct vm_page *pg, } /* if page was released, release it. otherwise un-busy it */ - if (ppsp[lcv]->pg_flags & PG_RELEASED) { - - if (ppsp[lcv]->pg_flags & PQ_ANON) { + if (ppsp[lcv]->pg_flags & PG_RELEASED && + ppsp[lcv]->pg_flags & PQ_ANON) { /* so that anfree will free */ atomic_clearbits_int(&ppsp[lcv]->pg_flags, PG_BUSY); @@ -761,34 +769,13 @@ uvm_pager_dropcluster(struct uvm_object *uobj, struct vm_page *pg, uvm_anfree(ppsp[lcv]->uanon); continue; - } - - /* - * pgo_releasepg will dump the page for us - */ - - saved_uobj = ppsp[lcv]->uobject; - obj_is_alive = - saved_uobj->pgops->pgo_releasepg(ppsp[lcv], NULL); - - /* for normal objects, "pg" is still PG_BUSY by us, - * so obj can't die */ - KASSERT(!uobj || obj_is_alive); - - /* only unlock the object if it is still alive... */ - if (obj_is_alive && saved_uobj != uobj) - simple_unlock(&saved_uobj->vmobjlock); - + } else { /* - * XXXCDC: suppose uobj died in the pgo_releasepg? - * how pass that - * info up to caller. we are currently ignoring it... + * if we were planning on async io then we would + * have PG_RELEASED set, clear that with the others. */ - - continue; /* next page */ - } else { atomic_clearbits_int(&ppsp[lcv]->pg_flags, - PG_BUSY|PG_WANTED|PG_FAKE); + PG_BUSY|PG_WANTED|PG_FAKE|PG_RELEASED); UVM_PAGE_OWN(ppsp[lcv], NULL); } @@ -812,33 +799,6 @@ uvm_pager_dropcluster(struct uvm_object *uobj, struct vm_page *pg, } } -#ifdef UBC -/* - * interrupt-context iodone handler for nested i/o bufs. - * - * => must be at splbio(). - */ - -void -uvm_aio_biodone1(struct buf *bp) -{ - struct buf *mbp = bp->b_private; - - splassert(IPL_BIO); - - KASSERT(mbp != bp); - if (bp->b_flags & B_ERROR) { - mbp->b_flags |= B_ERROR; - mbp->b_error = bp->b_error; - } - mbp->b_resid -= bp->b_bcount; - pool_put(&bufpool, bp); - if (mbp->b_resid == 0) { - biodone(mbp); - } -} -#endif - /* * interrupt-context iodone handler for single-buf i/os * or the top-level buf of a nested-buf i/o. diff --git a/sys/uvm/uvm_pager.h b/sys/uvm/uvm_pager.h index fea487b7863..4e0151fa2b3 100644 --- a/sys/uvm/uvm_pager.h +++ b/sys/uvm/uvm_pager.h @@ -1,4 +1,4 @@ -/* $OpenBSD: uvm_pager.h,v 1.26 2009/06/17 00:13:59 oga Exp $ */ +/* $OpenBSD: uvm_pager.h,v 1.27 2009/07/22 21:05:37 oga Exp $ */ /* $NetBSD: uvm_pager.h,v 1.20 2000/11/27 08:40:05 chs Exp $ */ /* @@ -109,8 +109,6 @@ struct uvm_pagerops { struct vm_page ** (*pgo_mk_pcluster)(struct uvm_object *, struct vm_page **, int *, struct vm_page *, int, voff_t, voff_t); - /* release page */ - boolean_t (*pgo_releasepg)(struct vm_page *, struct vm_page **); }; /* pager flags [mostly for flush] */ diff --git a/sys/uvm/uvm_pdaemon.c b/sys/uvm/uvm_pdaemon.c index e16ae26ea3b..046368e697a 100644 --- a/sys/uvm/uvm_pdaemon.c +++ b/sys/uvm/uvm_pdaemon.c @@ -1,4 +1,4 @@ -/* $OpenBSD: uvm_pdaemon.c,v 1.51 2009/06/26 20:26:02 oga Exp $ */ +/* $OpenBSD: uvm_pdaemon.c,v 1.52 2009/07/22 21:05:37 oga Exp $ */ /* $NetBSD: uvm_pdaemon.c,v 1.23 2000/08/20 10:24:14 bjh21 Exp $ */ /* @@ -820,40 +820,25 @@ uvmpd_scan_inactive(struct pglist *pglst) atomic_clearbits_int(&p->pg_flags, PG_BUSY|PG_WANTED); UVM_PAGE_OWN(p, NULL); - /* released during I/O? */ + /* released during I/O? Can only happen for anons */ if (p->pg_flags & PG_RELEASED) { - if (anon) { - /* - * remove page so we can get nextpg, - * also zero out anon so we don't use - * it after the free. - */ - anon->an_page = NULL; - p->uanon = NULL; - - simple_unlock(&anon->an_lock); - uvm_anfree(anon); /* kills anon */ - pmap_page_protect(p, VM_PROT_NONE); - anon = NULL; - uvm_lock_pageq(); - nextpg = TAILQ_NEXT(p, pageq); - /* free released page */ - uvm_pagefree(p); - - } else { - - /* - * pgo_releasepg nukes the page and - * gets "nextpg" for us. it returns - * with the page queues locked (when - * given nextpg ptr). - */ + KASSERT(anon != NULL); + /* + * remove page so we can get nextpg, + * also zero out anon so we don't use + * it after the free. + */ + anon->an_page = NULL; + p->uanon = NULL; - if (!uobj->pgops->pgo_releasepg(p, - &nextpg)) - /* uobj died after release */ - uobj = NULL; - } + simple_unlock(&anon->an_lock); + uvm_anfree(anon); /* kills anon */ + pmap_page_protect(p, VM_PROT_NONE); + anon = NULL; + uvm_lock_pageq(); + nextpg = TAILQ_NEXT(p, pageq); + /* free released page */ + uvm_pagefree(p); } else { /* page was not released during I/O */ uvm_lock_pageq(); nextpg = TAILQ_NEXT(p, pageq); diff --git a/sys/uvm/uvm_vnode.c b/sys/uvm/uvm_vnode.c index 465db6d50bd..15130e95ded 100644 --- a/sys/uvm/uvm_vnode.c +++ b/sys/uvm/uvm_vnode.c @@ -1,4 +1,4 @@ -/* $OpenBSD: uvm_vnode.c,v 1.67 2009/07/09 22:29:56 thib Exp $ */ +/* $OpenBSD: uvm_vnode.c,v 1.68 2009/07/22 21:05:37 oga Exp $ */ /* $NetBSD: uvm_vnode.c,v 1.36 2000/11/24 20:34:01 chs Exp $ */ /* @@ -93,7 +93,6 @@ void uvn_init(void); int uvn_io(struct uvm_vnode *, vm_page_t *, int, int, int); int uvn_put(struct uvm_object *, vm_page_t *, int, boolean_t); void uvn_reference(struct uvm_object *); -boolean_t uvn_releasepg(struct vm_page *, struct vm_page **); /* * master pager structure @@ -109,7 +108,6 @@ struct uvm_pagerops uvm_vnodeops = { uvn_put, uvn_cluster, uvm_mk_pcluster, /* use generic version of this: see uvm_pager.c */ - uvn_releasepg, }; /* @@ -415,31 +413,24 @@ uvn_detach(struct uvm_object *uobj) /* * given the structure of this pager, the above flush request will * create the following state: all the pages that were in the object - * have either been free'd or they are marked PG_BUSY|PG_RELEASED. - * the PG_BUSY bit was set either by us or the daemon for async I/O. - * in either case, if we have pages left we can't kill the object - * yet because i/o is pending. in this case we set the "relkill" - * flag which will cause pgo_releasepg to kill the object once all - * the I/O's are done [pgo_releasepg will be called from the aiodone - * routine or from the page daemon]. + * have either been free'd or they are marked PG_BUSY and in the + * middle of an async io. If we still have pages we set the "relkill" + * state, so that in the case the vnode gets terminated we know + * to leave it alone. Otherwise we'll kill the vnode when it's empty. */ - if (uobj->uo_npages) { /* I/O pending. iodone will free */ -#ifdef DEBUG - /* - * XXXCDC: very unlikely to happen until we have async i/o - * so print a little info message in case it does. - */ - printf("uvn_detach: vn %p has pages left after flush - " - "relkill mode\n", uobj); -#endif - uvn->u_flags |= UVM_VNODE_RELKILL; - simple_unlock(&uobj->vmobjlock); - UVMHIST_LOG(maphist,"<- done! (releasepg will kill obj)", 0, 0, - 0, 0); - return; + uvn->u_flags |= UVM_VNODE_RELKILL; + /* wait on any outstanding io */ + while (uobj->uo_npages && uvn->u_flags & UVM_VNODE_RELKILL) { + uvn->u_flags |= UVM_VNODE_IOSYNC; + UVM_UNLOCK_AND_WAIT(&uvn->u_nio, &uvn->u_obj.vmobjlock, FALSE, + "uvn_term",0); + simple_lock(&uvn->u_obj.vmobjlock); } + if ((uvn->u_flags & UVM_VNODE_RELKILL) == 0) + return; + /* * kill object now. note that we can't be on the sync q because * all references are gone. @@ -490,8 +481,6 @@ uvn_detach(struct uvm_object *uobj) * => the caller must XLOCK and VOP_LOCK the vnode before calling us * [protects us from getting a vnode that is already in the DYING * state...] - * => unlike uvn_detach, this function must not return until all the - * uvn's pages are disposed of. * => in case [2] the uvn is still alive after this call, but all I/O * ops will fail (due to the backing vnode now being "dead"). this * will prob. kill any process using the uvn due to pgo_get failing. @@ -535,8 +524,8 @@ uvm_vnp_terminate(struct vnode *vp) /* * it is possible that the uvn was detached and is in the relkill - * state [i.e. waiting for async i/o to finish so that releasepg can - * kill object]. we take over the vnode now and cancel the relkill. + * state [i.e. waiting for async i/o to finish]. + * we take over the vnode now and cancel the relkill. * we want to know when the i/o is done so we can recycle right * away. note that a uvn can only be in the RELKILL state if it * has a zero reference count. @@ -630,72 +619,6 @@ uvm_vnp_terminate(struct vnode *vp) } /* - * uvn_releasepg: handled a released page in a uvn - * - * => "pg" is a PG_BUSY [caller owns it], PG_RELEASED page that we need - * to dispose of. - * => caller must handled PG_WANTED case - * => called with page's object locked, pageq's unlocked - * => returns TRUE if page's object is still alive, FALSE if we - * killed the page's object. if we return TRUE, then we - * return with the object locked. - * => if (nextpgp != NULL) => we return pageq.tqe_next here, and return - * with the page queues locked [for pagedaemon] - * => if (nextpgp == NULL) => we return with page queues unlocked [normal case] - * => we kill the uvn if it is not referenced and we are suppose to - * kill it ("relkill"). - */ - -boolean_t -uvn_releasepg(struct vm_page *pg, struct vm_page **nextpgp /* OUT */) -{ - struct uvm_vnode *uvn = (struct uvm_vnode *) pg->uobject; - struct vnode *vp = (struct vnode *)uvn; -#ifdef DIAGNOSTIC - if ((pg->pg_flags & PG_RELEASED) == 0) - panic("uvn_releasepg: page not released!"); -#endif - - /* - * dispose of the page [caller handles PG_WANTED] - */ - pmap_page_protect(pg, VM_PROT_NONE); - uvm_lock_pageq(); - if (nextpgp) - *nextpgp = TAILQ_NEXT(pg, pageq); /* next page for daemon */ - uvm_pagefree(pg); - if (!nextpgp) - uvm_unlock_pageq(); - - /* - * now see if we need to kill the object - */ - if (uvn->u_flags & UVM_VNODE_RELKILL) { - if (uvn->u_obj.uo_refs) - panic("uvn_releasepg: kill flag set on referenced " - "object!"); - if (uvn->u_obj.uo_npages == 0) { - if (uvn->u_flags & UVM_VNODE_WRITEABLE) { - LIST_REMOVE(uvn, u_wlist); - } -#ifdef DIAGNOSTIC - if (!TAILQ_EMPTY(&uvn->u_obj.memq)) - panic("uvn_releasepg: pages in object with npages == 0"); -#endif - if (uvn->u_flags & UVM_VNODE_WANTED) - /* still holding object lock */ - wakeup(uvn); - - uvn->u_flags = 0; /* DEAD! */ - simple_unlock(&uvn->u_obj.vmobjlock); - vrele(vp); - return (FALSE); - } - } - return (TRUE); -} - -/* * NOTE: currently we have to use VOP_READ/VOP_WRITE because they go * through the buffer cache and allow I/O in any size. These VOPs use * synchronous i/o. [vs. VOP_STRATEGY which can be async, but doesn't @@ -729,8 +652,6 @@ uvn_releasepg(struct vm_page *pg, struct vm_page **nextpgp /* OUT */) * - if (object->iosync && u_naio == 0) { wakeup &uvn->u_naio } * - get "page" structures (atop?). * - handle "wanted" pages - * - handle "released" pages [using pgo_releasepg] - * >>> pgo_releasepg may kill the object * dont forget to look at "object" wanted flag in all cases. */ @@ -792,15 +713,13 @@ boolean_t uvn_flush(struct uvm_object *uobj, voff_t start, voff_t stop, int flags) { struct uvm_vnode *uvn = (struct uvm_vnode *) uobj; - struct vm_page *pp, *ppnext, *ptmp; + struct vm_page *pp, *ptmp; struct vm_page *pps[MAXBSIZE >> PAGE_SHIFT], **ppsp; int npages, result, lcv; - boolean_t retval, need_iosync, by_list, needs_clean, all; + boolean_t retval, need_iosync, needs_clean; voff_t curoff; - u_short pp_version; UVMHIST_FUNC("uvn_flush"); UVMHIST_CALLED(maphist); - curoff = 0; /* XXX: shut up gcc */ /* * get init vals and determine how we are going to traverse object */ @@ -808,24 +727,16 @@ uvn_flush(struct uvm_object *uobj, voff_t start, voff_t stop, int flags) need_iosync = FALSE; retval = TRUE; /* return value */ if (flags & PGO_ALLPAGES) { - all = TRUE; - by_list = TRUE; /* always go by the list */ + start = 0; + stop = round_page(uvn->u_size); } else { start = trunc_page(start); - stop = round_page(stop); -#ifdef DEBUG - if (stop > round_page(uvn->u_size)) - printf("uvn_flush: strange, got an out of range " - "flush (fixed)\n"); -#endif - all = FALSE; - by_list = (uobj->uo_npages <= - ((stop - start) >> PAGE_SHIFT) * UVN_HASH_PENALTY); + stop = MIN(round_page(stop), round_page(uvn->u_size)); } UVMHIST_LOG(maphist, - " flush start=0x%lx, stop=0x%lx, by_list=%ld, flags=0x%lx", - (u_long)start, (u_long)stop, by_list, flags); + " flush start=0x%lx, stop=0x%lx, flags=0x%lx", + (u_long)start, (u_long)stop, flags, 0); /* * PG_CLEANCHK: this bit is used by the pgo_mk_pcluster function as @@ -838,75 +749,21 @@ uvn_flush(struct uvm_object *uobj, voff_t start, voff_t stop, int flags) * [borrowed PG_CLEANCHK idea from FreeBSD VM] */ - if ((flags & PGO_CLEANIT) != 0 && - uobj->pgops->pgo_mk_pcluster != NULL) { - if (by_list) { - TAILQ_FOREACH(pp, &uobj->memq, listq) { - if (!all && - (pp->offset < start || pp->offset >= stop)) - continue; + if ((flags & PGO_CLEANIT) != 0) { + KASSERT(uobj->pgops->pgo_mk_pcluster != 0); + for (curoff = start ; curoff < stop; curoff += PAGE_SIZE) { + if ((pp = uvm_pagelookup(uobj, curoff)) != NULL) atomic_clearbits_int(&pp->pg_flags, PG_CLEANCHK); - } - - } else { /* by hash */ - for (curoff = start ; curoff < stop; - curoff += PAGE_SIZE) { - pp = uvm_pagelookup(uobj, curoff); - if (pp) - atomic_clearbits_int(&pp->pg_flags, - PG_CLEANCHK); - } } } - /* - * now do it. note: we must update ppnext in body of loop or we - * will get stuck. we need to use ppnext because we may free "pp" - * before doing the next loop. - */ - - if (by_list) { - pp = TAILQ_FIRST(&uobj->memq); - } else { - curoff = start; - pp = uvm_pagelookup(uobj, curoff); - } - - ppnext = NULL; /* XXX: shut up gcc */ ppsp = NULL; /* XXX: shut up gcc */ uvm_lock_pageq(); /* page queues locked */ - /* locked: both page queues and uobj */ - for ( ; (by_list && pp != NULL) || - (!by_list && curoff < stop) ; pp = ppnext) { - - if (by_list) { - - /* - * range check - */ - - if (!all && - (pp->offset < start || pp->offset >= stop)) { - ppnext = TAILQ_NEXT(pp, listq); - continue; - } - - } else { - - /* - * null check - */ - - curoff += PAGE_SIZE; - if (pp == NULL) { - if (curoff < stop) - ppnext = uvm_pagelookup(uobj, curoff); - continue; - } - - } + for (curoff = start; curoff < stop; curoff += PAGE_SIZE) { + if ((pp = uvm_pagelookup(uobj, curoff)) == NULL) + continue; /* * handle case where we do not need to clean page (either @@ -943,37 +800,32 @@ uvn_flush(struct uvm_object *uobj, voff_t start, voff_t stop, int flags) } /* - * if we don't need a clean... load ppnext and dispose of pp + * if we don't need a clean... deactivate/free pages then cont. */ if (!needs_clean) { - /* load ppnext */ - if (by_list) - ppnext = TAILQ_NEXT(pp, listq); - else { - if (curoff < stop) - ppnext = uvm_pagelookup(uobj, curoff); - } - - /* now dispose of pp */ if (flags & PGO_DEACTIVATE) { if ((pp->pg_flags & PQ_INACTIVE) == 0 && pp->wire_count == 0) { pmap_page_protect(pp, VM_PROT_NONE); uvm_pagedeactivate(pp); } - } else if (flags & PGO_FREE) { if (pp->pg_flags & PG_BUSY) { - /* release busy pages */ atomic_setbits_int(&pp->pg_flags, - PG_RELEASED); + PG_WANTED); + uvm_unlock_pageq(); + UVM_UNLOCK_AND_WAIT(pp, + &uobj->vmobjlock, 0, "uvn_flsh", 0); + simple_lock(&uobj->vmobjlock); + uvm_lock_pageq(); + curoff -= PAGE_SIZE; + continue; } else { pmap_page_protect(pp, VM_PROT_NONE); /* removed page from object */ uvm_pagefree(pp); } } - /* ppnext is valid so we can continue... */ continue; } @@ -989,7 +841,9 @@ uvn_flush(struct uvm_object *uobj, voff_t start, voff_t stop, int flags) atomic_setbits_int(&pp->pg_flags, PG_BUSY); UVM_PAGE_OWN(pp, "uvn_flush"); pmap_page_protect(pp, VM_PROT_READ); - pp_version = pp->pg_version; + /* if we're async, free the page in aiodoned */ + if ((flags & (PGO_FREE|PGO_SYNCIO)) == PGO_FREE) + atomic_setbits_int(&pp->pg_flags, PG_RELEASED); ReTry: ppsp = pps; npages = sizeof(pps) / sizeof(struct vm_page *); @@ -1000,11 +854,11 @@ ReTry: /* unlocked: page queues, uobj */ /* - * at this point nothing is locked. if we did an async I/O - * it is remotely possible for the async i/o to complete and - * the page "pp" be freed or what not before we get a chance - * to relock the object. in order to detect this, we have - * saved the version number of the page in "pp_version". + * if we did an async I/O it is remotely possible for the + * async i/o to complete and the page "pp" be freed or what + * not before we get a chance to relock the object. Therefore, + * we only touch it when it won't be freed, RELEASED took care + * of the rest. */ /* relock! */ @@ -1013,7 +867,7 @@ ReTry: /* * VM_PAGER_AGAIN: given the structure of this pager, this - * can only happen when we are doing async I/O and can't + * can only happen when we are doing async I/O and can't * map the pages into kernel memory (pager_map) due to lack * of vm space. if this happens we drop back to sync I/O. */ @@ -1031,6 +885,10 @@ ReTry: panic("uvn_flush: PGO_SYNCIO return 'try again' error (impossible)"); #endif flags |= PGO_SYNCIO; + if (flags & PGO_FREE) + atomic_clearbits_int(&pp->pg_flags, + PG_RELEASED); + goto ReTry; } @@ -1042,66 +900,20 @@ ReTry: */ /* - * for pending async i/o if we are not deactivating/freeing - * we can move on to the next page. + * for pending async i/o if we are not deactivating + * we can move on to the next page. aiodoned deals with + * the freeing case for us. */ - - if (result == VM_PAGER_PEND) { - - if ((flags & (PGO_DEACTIVATE|PGO_FREE)) == 0) { - /* - * no per-page ops: refresh ppnext and continue - */ - if (by_list) { - if (pp->pg_version == pp_version) - ppnext = TAILQ_NEXT(pp, listq); - else - /* reset */ - ppnext = TAILQ_FIRST(&uobj->memq); - } else { - if (curoff < stop) - ppnext = uvm_pagelookup(uobj, - curoff); - } - continue; - } - - /* need to do anything here? */ - } + if (result == VM_PAGER_PEND && (flags & PGO_DEACTIVATE) == 0) + continue; /* - * need to look at each page of the I/O operation. we defer - * processing "pp" until the last trip through this "for" loop - * so that we can load "ppnext" for the main loop after we - * play with the cluster pages [thus the "npages + 1" in the - * loop below]. + * need to look at each page of the I/O operation, and do what + * we gotta do. */ - for (lcv = 0 ; lcv < npages + 1 ; lcv++) { - - /* - * handle ppnext for outside loop, and saving pp - * until the end. - */ - if (lcv < npages) { - if (ppsp[lcv] == pp) - continue; /* skip pp until the end */ - ptmp = ppsp[lcv]; - } else { - ptmp = pp; - - /* set up next page for outer loop */ - if (by_list) { - if (pp->pg_version == pp_version) - ppnext = TAILQ_NEXT(pp, listq); - else - /* reset */ - ppnext = TAILQ_FIRST(&uobj->memq); - } else { - if (curoff < stop) - ppnext = uvm_pagelookup(uobj, curoff); - } - } + for (lcv = 0 ; lcv < npages; lcv++) { + ptmp = ppsp[lcv]; /* * verify the page didn't get moved while obj was @@ -1125,25 +937,10 @@ ReTry: atomic_clearbits_int(&ptmp->pg_flags, PG_WANTED|PG_BUSY); UVM_PAGE_OWN(ptmp, NULL); - if (ptmp->pg_flags & PG_RELEASED) { - - /* - * pgo_releasepg needs to grab the - * pageq lock itself. - */ - uvm_unlock_pageq(); - if (!uvn_releasepg(ptmp, NULL)) - return (TRUE); - - uvm_lock_pageq(); /* relock */ - continue; /* next page */ - - } else { - atomic_setbits_int(&ptmp->pg_flags, - PG_CLEAN|PG_CLEANCHK); - if ((flags & PGO_FREE) == 0) - pmap_clear_modify(ptmp); - } + atomic_setbits_int(&ptmp->pg_flags, + PG_CLEAN|PG_CLEANCHK); + if ((flags & PGO_FREE) == 0) + pmap_clear_modify(ptmp); } /* @@ -1156,29 +953,21 @@ ReTry: pmap_page_protect(ptmp, VM_PROT_NONE); uvm_pagedeactivate(ptmp); } - - } else if (flags & PGO_FREE) { - if (result == VM_PAGER_PEND) { - if ((ptmp->pg_flags & PG_BUSY) != 0) - /* signal for i/o done */ - atomic_setbits_int( - &ptmp->pg_flags, - PG_RELEASED); - } else { - if (result != VM_PAGER_OK) { - printf("uvn_flush: obj=%p, " - "offset=0x%llx. error " - "during pageout.\n", - pp->uobject, - (long long)pp->offset); - printf("uvn_flush: WARNING: " - "changes to page may be " - "lost!\n"); - retval = FALSE; - } - pmap_page_protect(ptmp, VM_PROT_NONE); - uvm_pagefree(ptmp); + } else if (flags & PGO_FREE && + result != VM_PAGER_PEND) { + if (result != VM_PAGER_OK) { + printf("uvn_flush: obj=%p, " + "offset=0x%llx. error " + "during pageout.\n", + pp->uobject, + (long long)pp->offset); + printf("uvn_flush: WARNING: " + "changes to page may be " + "lost!\n"); + retval = FALSE; } + pmap_page_protect(ptmp, VM_PROT_NONE); + uvm_pagefree(ptmp); } } /* end of "lcv" for loop */ @@ -1321,7 +1110,7 @@ uvn_get(struct uvm_object *uobj, voff_t offset, struct vm_page **pps, /* to be useful must get a non-busy, non-released pg */ if (ptmp == NULL || - (ptmp->pg_flags & (PG_BUSY|PG_RELEASED)) != 0) { + (ptmp->pg_flags & PG_BUSY) != 0) { if (lcv == centeridx || (flags & PGO_ALLPAGES) != 0) done = FALSE; /* need to do a wait or I/O! */ @@ -1427,7 +1216,7 @@ uvn_get(struct uvm_object *uobj, voff_t offset, struct vm_page **pps, } /* page is there, see if we need to wait on it */ - if ((ptmp->pg_flags & (PG_BUSY|PG_RELEASED)) != 0) { + if ((ptmp->pg_flags & PG_BUSY) != 0) { atomic_setbits_int(&ptmp->pg_flags, PG_WANTED); UVM_UNLOCK_AND_WAIT(ptmp, &uobj->vmobjlock, FALSE, "uvn_get",0); |