diff options
author | Theo de Raadt <deraadt@cvs.openbsd.org> | 1995-10-18 08:53:40 +0000 |
---|---|---|
committer | Theo de Raadt <deraadt@cvs.openbsd.org> | 1995-10-18 08:53:40 +0000 |
commit | d6583bb2a13f329cf0332ef2570eb8bb8fc0e39c (patch) | |
tree | ece253b876159b39c620e62b6c9b1174642e070e /sys/vm |
initial import of NetBSD tree
Diffstat (limited to 'sys/vm')
34 files changed, 15223 insertions, 0 deletions
diff --git a/sys/vm/device_pager.c b/sys/vm/device_pager.c new file mode 100644 index 00000000000..dba1e368707 --- /dev/null +++ b/sys/vm/device_pager.c @@ -0,0 +1,370 @@ +/* $NetBSD: device_pager.c,v 1.18 1994/10/29 07:35:04 cgd Exp $ */ + +/* + * Copyright (c) 1990 University of Utah. + * Copyright (c) 1991, 1993 + * The Regents of the University of California. All rights reserved. + * + * This code is derived from software contributed to Berkeley by + * the Systems Programming Group of the University of Utah Computer + * Science Department. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. All advertising materials mentioning features or use of this software + * must display the following acknowledgement: + * This product includes software developed by the University of + * California, Berkeley and its contributors. + * 4. Neither the name of the University nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * @(#)device_pager.c 8.5 (Berkeley) 1/12/94 + */ + +/* + * Page to/from special files. + */ + +#include <sys/param.h> +#include <sys/systm.h> +#include <sys/conf.h> +#include <sys/mman.h> +#include <sys/malloc.h> + +#include <vm/vm.h> +#include <vm/vm_kern.h> +#include <vm/vm_page.h> +#include <vm/device_pager.h> + +struct pagerlst dev_pager_list; /* list of managed devices */ +struct pglist dev_pager_fakelist; /* list of available vm_page_t's */ + +#ifdef DEBUG +int dpagerdebug = 0; +#define DDB_FOLLOW 0x01 +#define DDB_INIT 0x02 +#define DDB_ALLOC 0x04 +#define DDB_FAIL 0x08 +#endif + +static vm_pager_t dev_pager_alloc + __P((caddr_t, vm_size_t, vm_prot_t, vm_offset_t)); +static void dev_pager_dealloc __P((vm_pager_t)); +static int dev_pager_getpage + __P((vm_pager_t, vm_page_t *, int, boolean_t)); +static boolean_t dev_pager_haspage __P((vm_pager_t, vm_offset_t)); +static void dev_pager_init __P((void)); +static int dev_pager_putpage + __P((vm_pager_t, vm_page_t *, int, boolean_t)); +static vm_page_t dev_pager_getfake __P((vm_offset_t)); +static void dev_pager_putfake __P((vm_page_t)); + +struct pagerops devicepagerops = { + dev_pager_init, + dev_pager_alloc, + dev_pager_dealloc, + dev_pager_getpage, + dev_pager_putpage, + dev_pager_haspage, + vm_pager_clusternull +}; + +static void +dev_pager_init() +{ +#ifdef DEBUG + if (dpagerdebug & DDB_FOLLOW) + printf("dev_pager_init()\n"); +#endif + TAILQ_INIT(&dev_pager_list); + TAILQ_INIT(&dev_pager_fakelist); +} + +static vm_pager_t +dev_pager_alloc(handle, size, prot, foff) + caddr_t handle; + vm_size_t size; + vm_prot_t prot; + vm_offset_t foff; +{ + dev_t dev; + vm_pager_t pager; + int (*mapfunc)(); + vm_object_t object; + dev_pager_t devp; + int npages, off; + +#ifdef DEBUG + if (dpagerdebug & DDB_FOLLOW) + printf("dev_pager_alloc(%x, %x, %x, %x)\n", + handle, size, prot, foff); +#endif +#ifdef DIAGNOSTIC + /* + * Pageout to device, should never happen. + */ + if (handle == NULL) + panic("dev_pager_alloc called"); +#endif + + /* + * Make sure this device can be mapped. + */ + dev = (dev_t)(long)handle; + mapfunc = cdevsw[major(dev)].d_mmap; + if (mapfunc == NULL || mapfunc == enodev || mapfunc == nullop) + return(NULL); + + /* + * Offset should be page aligned. + */ + if (foff & PAGE_MASK) + return(NULL); + + /* + * Check that the specified range of the device allows the + * desired protection. + * + * XXX assumes VM_PROT_* == PROT_* + */ + npages = atop(round_page(size)); + for (off = foff; npages--; off += PAGE_SIZE) + if ((*mapfunc)(dev, off, (int)prot) == -1) + return(NULL); + + /* + * Look up pager, creating as necessary. + */ +top: + pager = vm_pager_lookup(&dev_pager_list, handle); + if (pager == NULL) { + /* + * Allocate and initialize pager structs + */ + pager = (vm_pager_t)malloc(sizeof *pager, M_VMPAGER, M_WAITOK); + if (pager == NULL) + return(NULL); + devp = (dev_pager_t)malloc(sizeof *devp, M_VMPGDATA, M_WAITOK); + if (devp == NULL) { + free((caddr_t)pager, M_VMPAGER); + return(NULL); + } + pager->pg_handle = handle; + pager->pg_ops = &devicepagerops; + pager->pg_type = PG_DEVICE; + pager->pg_flags = 0; + pager->pg_data = devp; + TAILQ_INIT(&devp->devp_pglist); + /* + * Allocate object and associate it with the pager. + */ + object = devp->devp_object = vm_object_allocate(0); + vm_object_enter(object, pager); + vm_object_setpager(object, pager, (vm_offset_t)0, FALSE); + /* + * Finally, put it on the managed list so other can find it. + * First we re-lookup in case someone else beat us to this + * point (due to blocking in the various mallocs). If so, + * we free everything and start over. + */ + if (vm_pager_lookup(&dev_pager_list, handle)) { + free((caddr_t)devp, M_VMPGDATA); + free((caddr_t)pager, M_VMPAGER); + goto top; + } + TAILQ_INSERT_TAIL(&dev_pager_list, pager, pg_list); +#ifdef DEBUG + if (dpagerdebug & DDB_ALLOC) { + printf("dev_pager_alloc: pager %x devp %x object %x\n", + pager, devp, object); + vm_object_print(object, FALSE); + } +#endif + } else { + /* + * vm_object_lookup() gains a reference and also + * removes the object from the cache. + */ + object = vm_object_lookup(pager); +#ifdef DIAGNOSTIC + devp = (dev_pager_t)pager->pg_data; + if (object != devp->devp_object) + panic("dev_pager_setup: bad object"); +#endif + } + return(pager); +} + +static void +dev_pager_dealloc(pager) + vm_pager_t pager; +{ + dev_pager_t devp; + vm_object_t object; + vm_page_t m; + +#ifdef DEBUG + if (dpagerdebug & DDB_FOLLOW) + printf("dev_pager_dealloc(%x)\n", pager); +#endif + TAILQ_REMOVE(&dev_pager_list, pager, pg_list); + /* + * Get the object. + * Note: cannot use vm_object_lookup since object has already + * been removed from the hash chain. + */ + devp = (dev_pager_t)pager->pg_data; + object = devp->devp_object; +#ifdef DEBUG + if (dpagerdebug & DDB_ALLOC) + printf("dev_pager_dealloc: devp %x object %x\n", devp, object); +#endif + /* + * Free up our fake pages. + */ + while ((m = devp->devp_pglist.tqh_first) != NULL) { + TAILQ_REMOVE(&devp->devp_pglist, m, pageq); + dev_pager_putfake(m); + } + free((caddr_t)devp, M_VMPGDATA); + free((caddr_t)pager, M_VMPAGER); +} + +static int +dev_pager_getpage(pager, mlist, npages, sync) + vm_pager_t pager; + vm_page_t *mlist; + int npages; + boolean_t sync; +{ + register vm_object_t object; + vm_offset_t offset, paddr; + vm_page_t page; + dev_t dev; + int (*mapfunc)(), prot; + vm_page_t m; + +#ifdef DEBUG + if (dpagerdebug & DDB_FOLLOW) + printf("dev_pager_getpage(%x, %x, %x, %x)\n", + pager, mlist, npages, sync); +#endif + + if (npages != 1) + panic("dev_pager_getpage: cannot handle multiple pages"); + m = *mlist; + + object = m->object; + dev = (dev_t)(long)pager->pg_handle; + offset = m->offset + object->paging_offset; + prot = PROT_READ; /* XXX should pass in? */ + mapfunc = cdevsw[major(dev)].d_mmap; +#ifdef DIAGNOSTIC + if (mapfunc == NULL || mapfunc == enodev || mapfunc == nullop) + panic("dev_pager_getpage: no map function"); +#endif + paddr = pmap_phys_address((*mapfunc)(dev, (int)offset, prot)); +#ifdef DIAGNOSTIC + if (paddr == -1) + panic("dev_pager_getpage: map function returns error"); +#endif + /* + * Replace the passed in page with our own fake page and free + * up the original. + */ + page = dev_pager_getfake(paddr); + TAILQ_INSERT_TAIL(&((dev_pager_t)pager->pg_data)->devp_pglist, page, + pageq); + vm_object_lock(object); + vm_page_lock_queues(); + vm_page_free(m); + vm_page_insert(page, object, offset); + vm_page_unlock_queues(); + PAGE_WAKEUP(m); + if (offset + PAGE_SIZE > object->size) + object->size = offset + PAGE_SIZE; /* XXX anal */ + vm_object_unlock(object); + + return(VM_PAGER_OK); +} + +static int +dev_pager_putpage(pager, mlist, npages, sync) + vm_pager_t pager; + vm_page_t *mlist; + int npages; + boolean_t sync; +{ +#ifdef DEBUG + if (dpagerdebug & DDB_FOLLOW) + printf("dev_pager_putpage(%x, %x, %x, %x)\n", + pager, mlist, npages, sync); +#endif + if (pager == NULL) + return (FALSE); + panic("dev_pager_putpage called"); +} + +static boolean_t +dev_pager_haspage(pager, offset) + vm_pager_t pager; + vm_offset_t offset; +{ +#ifdef DEBUG + if (dpagerdebug & DDB_FOLLOW) + printf("dev_pager_haspage(%x, %x)\n", pager, offset); +#endif + return(TRUE); +} + +static vm_page_t +dev_pager_getfake(paddr) + vm_offset_t paddr; +{ + vm_page_t m; + int i; + + if (dev_pager_fakelist.tqh_first == NULL) { + m = (vm_page_t)malloc(PAGE_SIZE, M_VMPGDATA, M_WAITOK); + for (i = PAGE_SIZE / sizeof(*m); i > 0; i--) { + TAILQ_INSERT_TAIL(&dev_pager_fakelist, m, pageq); + m++; + } + } + m = dev_pager_fakelist.tqh_first; + TAILQ_REMOVE(&dev_pager_fakelist, m, pageq); + m->flags = PG_BUSY | PG_CLEAN | PG_FAKE | PG_FICTITIOUS; + m->phys_addr = paddr; + m->wire_count = 1; + return(m); +} + +static void +dev_pager_putfake(m) + vm_page_t m; +{ +#ifdef DIAGNOSTIC + if (!(m->flags & PG_FICTITIOUS)) + panic("dev_pager_putfake: bad page"); +#endif + TAILQ_INSERT_TAIL(&dev_pager_fakelist, m, pageq); +} diff --git a/sys/vm/device_pager.h b/sys/vm/device_pager.h new file mode 100644 index 00000000000..6411b824a82 --- /dev/null +++ b/sys/vm/device_pager.h @@ -0,0 +1,55 @@ +/* $NetBSD: device_pager.h,v 1.9 1994/06/29 06:47:41 cgd Exp $ */ + +/* + * Copyright (c) 1990 University of Utah. + * Copyright (c) 1991, 1993 + * The Regents of the University of California. All rights reserved. + * + * This code is derived from software contributed to Berkeley by + * the Systems Programming Group of the University of Utah Computer + * Science Department. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. All advertising materials mentioning features or use of this software + * must display the following acknowledgement: + * This product includes software developed by the University of + * California, Berkeley and its contributors. + * 4. Neither the name of the University nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * @(#)device_pager.h 8.3 (Berkeley) 12/13/93 + */ + +#ifndef _DEVICE_PAGER_ +#define _DEVICE_PAGER_ 1 + +/* + * Device pager private data. + */ +struct devpager { + struct pglist devp_pglist; /* list of pages allocated */ + vm_object_t devp_object; /* object representing this device */ +}; +typedef struct devpager *dev_pager_t; + +#endif /* _DEVICE_PAGER_ */ diff --git a/sys/vm/kern_lock.c b/sys/vm/kern_lock.c new file mode 100644 index 00000000000..91cab2f4543 --- /dev/null +++ b/sys/vm/kern_lock.c @@ -0,0 +1,536 @@ +/* $NetBSD: kern_lock.c,v 1.10 1994/10/30 19:11:09 cgd Exp $ */ + +/* + * Copyright (c) 1991, 1993 + * The Regents of the University of California. All rights reserved. + * + * This code is derived from software contributed to Berkeley by + * The Mach Operating System project at Carnegie-Mellon University. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. All advertising materials mentioning features or use of this software + * must display the following acknowledgement: + * This product includes software developed by the University of + * California, Berkeley and its contributors. + * 4. Neither the name of the University nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * @(#)kern_lock.c 8.1 (Berkeley) 6/11/93 + * + * + * Copyright (c) 1987, 1990 Carnegie-Mellon University. + * All rights reserved. + * + * Authors: Avadis Tevanian, Jr., Michael Wayne Young + * + * Permission to use, copy, modify and distribute this software and + * its documentation is hereby granted, provided that both the copyright + * notice and this permission notice appear in all copies of the + * software, derivative works or modified versions, and any portions + * thereof, and that both notices appear in supporting documentation. + * + * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" + * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND + * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. + * + * Carnegie Mellon requests users of this software to return to + * + * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU + * School of Computer Science + * Carnegie Mellon University + * Pittsburgh PA 15213-3890 + * + * any improvements or extensions that they make and grant Carnegie the + * rights to redistribute these changes. + */ + +/* + * Locking primitives implementation + */ + +#include <sys/param.h> +#include <sys/systm.h> + +#include <vm/vm.h> + +/* XXX */ +#include <sys/proc.h> +typedef void *thread_t; +#define current_thread() ((thread_t)&curproc->p_thread) +/* XXX */ + +#if NCPUS > 1 + +/* + * Module: lock + * Function: + * Provide reader/writer sychronization. + * Implementation: + * Simple interlock on a bit. Readers first interlock + * increment the reader count, then let go. Writers hold + * the interlock (thus preventing further readers), and + * wait for already-accepted readers to go away. + */ + +/* + * The simple-lock routines are the primitives out of which + * the lock package is built. The implementation is left + * to the machine-dependent code. + */ + +#ifdef notdef +/* + * A sample implementation of simple locks. + * assumes: + * boolean_t test_and_set(boolean_t *) + * indivisibly sets the boolean to TRUE + * and returns its old value + * and that setting a boolean to FALSE is indivisible. + */ +/* + * simple_lock_init initializes a simple lock. A simple lock + * may only be used for exclusive locks. + */ + +void simple_lock_init(l) + simple_lock_t l; +{ + *(boolean_t *)l = FALSE; +} + +void simple_lock(l) + simple_lock_t l; +{ + while (test_and_set((boolean_t *)l)) + continue; +} + +void simple_unlock(l) + simple_lock_t l; +{ + *(boolean_t *)l = FALSE; +} + +boolean_t simple_lock_try(l) + simple_lock_t l; +{ + return (!test_and_set((boolean_t *)l)); +} +#endif /* notdef */ +#endif /* NCPUS > 1 */ + +#if NCPUS > 1 +int lock_wait_time = 100; +#else /* NCPUS > 1 */ + + /* + * It is silly to spin on a uni-processor as if we + * thought something magical would happen to the + * want_write bit while we are executing. + */ +int lock_wait_time = 0; +#endif /* NCPUS > 1 */ + + +/* + * Routine: lock_init + * Function: + * Initialize a lock; required before use. + * Note that clients declare the "struct lock" + * variables and then initialize them, rather + * than getting a new one from this module. + */ +void lock_init(l, can_sleep) + lock_t l; + boolean_t can_sleep; +{ + bzero(l, sizeof(lock_data_t)); + simple_lock_init(&l->interlock); + l->want_write = FALSE; + l->want_upgrade = FALSE; + l->read_count = 0; + l->can_sleep = can_sleep; + l->thread = (char *)-1; /* XXX */ + l->recursion_depth = 0; +} + +void lock_sleepable(l, can_sleep) + lock_t l; + boolean_t can_sleep; +{ + simple_lock(&l->interlock); + l->can_sleep = can_sleep; + simple_unlock(&l->interlock); +} + + +/* + * Sleep locks. These use the same data structure and algorithm + * as the spin locks, but the process sleeps while it is waiting + * for the lock. These work on uniprocessor systems. + */ + +void lock_write(l) + register lock_t l; +{ + register int i; + + simple_lock(&l->interlock); + + if (((thread_t)l->thread) == current_thread()) { + /* + * Recursive lock. + */ + l->recursion_depth++; + simple_unlock(&l->interlock); + return; + } + + /* + * Try to acquire the want_write bit. + */ + while (l->want_write) { + if ((i = lock_wait_time) > 0) { + simple_unlock(&l->interlock); + while (--i > 0 && l->want_write) + continue; + simple_lock(&l->interlock); + } + + if (l->can_sleep && l->want_write) { + l->waiting = TRUE; + thread_sleep(l, &l->interlock, FALSE); + simple_lock(&l->interlock); + } + } + l->want_write = TRUE; + + /* Wait for readers (and upgrades) to finish */ + + while ((l->read_count != 0) || l->want_upgrade) { + if ((i = lock_wait_time) > 0) { + simple_unlock(&l->interlock); + while (--i > 0 && (l->read_count != 0 || + l->want_upgrade)) + continue; + simple_lock(&l->interlock); + } + + if (l->can_sleep && (l->read_count != 0 || l->want_upgrade)) { + l->waiting = TRUE; + thread_sleep(l, &l->interlock, FALSE); + simple_lock(&l->interlock); + } + } + simple_unlock(&l->interlock); +} + +void lock_done(l) + register lock_t l; +{ + simple_lock(&l->interlock); + + if (l->read_count != 0) + l->read_count--; + else + if (l->recursion_depth != 0) + l->recursion_depth--; + else + if (l->want_upgrade) + l->want_upgrade = FALSE; + else + l->want_write = FALSE; + + if (l->waiting) { + l->waiting = FALSE; + thread_wakeup(l); + } + simple_unlock(&l->interlock); +} + +void lock_read(l) + register lock_t l; +{ + register int i; + + simple_lock(&l->interlock); + + if (((thread_t)l->thread) == current_thread()) { + /* + * Recursive lock. + */ + l->read_count++; + simple_unlock(&l->interlock); + return; + } + + while (l->want_write || l->want_upgrade) { + if ((i = lock_wait_time) > 0) { + simple_unlock(&l->interlock); + while (--i > 0 && (l->want_write || l->want_upgrade)) + continue; + simple_lock(&l->interlock); + } + + if (l->can_sleep && (l->want_write || l->want_upgrade)) { + l->waiting = TRUE; + thread_sleep(l, &l->interlock, FALSE); + simple_lock(&l->interlock); + } + } + + l->read_count++; + simple_unlock(&l->interlock); +} + +/* + * Routine: lock_read_to_write + * Function: + * Improves a read-only lock to one with + * write permission. If another reader has + * already requested an upgrade to a write lock, + * no lock is held upon return. + * + * Returns TRUE if the upgrade *failed*. + */ +boolean_t lock_read_to_write(l) + register lock_t l; +{ + register int i; + + simple_lock(&l->interlock); + + l->read_count--; + + if (((thread_t)l->thread) == current_thread()) { + /* + * Recursive lock. + */ + l->recursion_depth++; + simple_unlock(&l->interlock); + return(FALSE); + } + + if (l->want_upgrade) { + /* + * Someone else has requested upgrade. + * Since we've released a read lock, wake + * him up. + */ + if (l->waiting) { + l->waiting = FALSE; + thread_wakeup(l); + } + + simple_unlock(&l->interlock); + return (TRUE); + } + + l->want_upgrade = TRUE; + + while (l->read_count != 0) { + if ((i = lock_wait_time) > 0) { + simple_unlock(&l->interlock); + while (--i > 0 && l->read_count != 0) + continue; + simple_lock(&l->interlock); + } + + if (l->can_sleep && l->read_count != 0) { + l->waiting = TRUE; + thread_sleep(l, &l->interlock, FALSE); + simple_lock(&l->interlock); + } + } + + simple_unlock(&l->interlock); + return (FALSE); +} + +void lock_write_to_read(l) + register lock_t l; +{ + simple_lock(&l->interlock); + + l->read_count++; + if (l->recursion_depth != 0) + l->recursion_depth--; + else + if (l->want_upgrade) + l->want_upgrade = FALSE; + else + l->want_write = FALSE; + + if (l->waiting) { + l->waiting = FALSE; + thread_wakeup(l); + } + + simple_unlock(&l->interlock); +} + + +/* + * Routine: lock_try_write + * Function: + * Tries to get a write lock. + * + * Returns FALSE if the lock is not held on return. + */ + +boolean_t lock_try_write(l) + register lock_t l; +{ + + simple_lock(&l->interlock); + + if (((thread_t)l->thread) == current_thread()) { + /* + * Recursive lock + */ + l->recursion_depth++; + simple_unlock(&l->interlock); + return(TRUE); + } + + if (l->want_write || l->want_upgrade || l->read_count) { + /* + * Can't get lock. + */ + simple_unlock(&l->interlock); + return(FALSE); + } + + /* + * Have lock. + */ + + l->want_write = TRUE; + simple_unlock(&l->interlock); + return(TRUE); +} + +/* + * Routine: lock_try_read + * Function: + * Tries to get a read lock. + * + * Returns FALSE if the lock is not held on return. + */ + +boolean_t lock_try_read(l) + register lock_t l; +{ + simple_lock(&l->interlock); + + if (((thread_t)l->thread) == current_thread()) { + /* + * Recursive lock + */ + l->read_count++; + simple_unlock(&l->interlock); + return(TRUE); + } + + if (l->want_write || l->want_upgrade) { + simple_unlock(&l->interlock); + return(FALSE); + } + + l->read_count++; + simple_unlock(&l->interlock); + return(TRUE); +} + +/* + * Routine: lock_try_read_to_write + * Function: + * Improves a read-only lock to one with + * write permission. If another reader has + * already requested an upgrade to a write lock, + * the read lock is still held upon return. + * + * Returns FALSE if the upgrade *failed*. + */ +boolean_t lock_try_read_to_write(l) + register lock_t l; +{ + + simple_lock(&l->interlock); + + if (((thread_t)l->thread) == current_thread()) { + /* + * Recursive lock + */ + l->read_count--; + l->recursion_depth++; + simple_unlock(&l->interlock); + return(TRUE); + } + + if (l->want_upgrade) { + simple_unlock(&l->interlock); + return(FALSE); + } + l->want_upgrade = TRUE; + l->read_count--; + + while (l->read_count != 0) { + l->waiting = TRUE; + thread_sleep(l, &l->interlock, FALSE); + simple_lock(&l->interlock); + } + + simple_unlock(&l->interlock); + return(TRUE); +} + +/* + * Allow a process that has a lock for write to acquire it + * recursively (for read, write, or update). + */ +void lock_set_recursive(l) + lock_t l; +{ + simple_lock(&l->interlock); + if (!l->want_write) { + panic("lock_set_recursive: don't have write lock"); + } + l->thread = (char *) current_thread(); + simple_unlock(&l->interlock); +} + +/* + * Prevent a lock from being re-acquired. + */ +void lock_clear_recursive(l) + lock_t l; +{ + simple_lock(&l->interlock); + if (((thread_t) l->thread) != current_thread()) { + panic("lock_clear_recursive: wrong thread"); + } + if (l->recursion_depth == 0) + l->thread = (char *)-1; /* XXX */ + simple_unlock(&l->interlock); +} diff --git a/sys/vm/lock.h b/sys/vm/lock.h new file mode 100644 index 00000000000..1157b83f356 --- /dev/null +++ b/sys/vm/lock.h @@ -0,0 +1,174 @@ +/* $NetBSD: lock.h,v 1.8 1994/10/30 19:11:11 cgd Exp $ */ + +/* + * Copyright (c) 1991, 1993 + * The Regents of the University of California. All rights reserved. + * + * This code is derived from software contributed to Berkeley by + * The Mach Operating System project at Carnegie-Mellon University. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. All advertising materials mentioning features or use of this software + * must display the following acknowledgement: + * This product includes software developed by the University of + * California, Berkeley and its contributors. + * 4. Neither the name of the University nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * @(#)lock.h 8.1 (Berkeley) 6/11/93 + * + * + * Copyright (c) 1987, 1990 Carnegie-Mellon University. + * All rights reserved. + * + * Authors: Avadis Tevanian, Jr., Michael Wayne Young + * + * Permission to use, copy, modify and distribute this software and + * its documentation is hereby granted, provided that both the copyright + * notice and this permission notice appear in all copies of the + * software, derivative works or modified versions, and any portions + * thereof, and that both notices appear in supporting documentation. + * + * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" + * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND + * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. + * + * Carnegie Mellon requests users of this software to return to + * + * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU + * School of Computer Science + * Carnegie Mellon University + * Pittsburgh PA 15213-3890 + * + * any improvements or extensions that they make and grant Carnegie the + * rights to redistribute these changes. + */ + +/* + * Locking primitives definitions + */ + +#ifndef _LOCK_H_ +#define _LOCK_H_ + +#define NCPUS 1 /* XXX */ + +/* + * A simple spin lock. + */ + +struct slock { + int lock_data; /* in general 1 bit is sufficient */ +}; + +typedef struct slock simple_lock_data_t; +typedef struct slock *simple_lock_t; + +/* + * The general lock structure. Provides for multiple readers, + * upgrading from read to write, and sleeping until the lock + * can be gained. + */ + +struct lock { +#ifdef vax + /* + * Efficient VAX implementation -- see field description below. + */ + unsigned int read_count:16, + want_upgrade:1, + want_write:1, + waiting:1, + can_sleep:1, + :0; + + simple_lock_data_t interlock; +#else /* vax */ +#ifdef ns32000 + /* + * Efficient ns32000 implementation -- + * see field description below. + */ + simple_lock_data_t interlock; + unsigned int read_count:16, + want_upgrade:1, + want_write:1, + waiting:1, + can_sleep:1, + :0; + +#else /* ns32000 */ + /* Only the "interlock" field is used for hardware exclusion; + * other fields are modified with normal instructions after + * acquiring the interlock bit. + */ + simple_lock_data_t + interlock; /* Interlock for remaining fields */ + boolean_t want_write; /* Writer is waiting, or locked for write */ + boolean_t want_upgrade; /* Read-to-write upgrade waiting */ + boolean_t waiting; /* Someone is sleeping on lock */ + boolean_t can_sleep; /* Can attempts to lock go to sleep */ + int read_count; /* Number of accepted readers */ +#endif /* ns32000 */ +#endif /* vax */ + void *thread; /* Thread that has lock, if recursive locking allowed */ + /* (should be thread_t, but but we then have mutually + recursive definitions) */ + int recursion_depth;/* Depth of recursion */ +}; + +typedef struct lock lock_data_t; +typedef struct lock *lock_t; + +#if NCPUS > 1 +__BEGIN_DECLS +void simple_lock __P((simple_lock_t)); +void simple_lock_init __P((simple_lock_t)); +boolean_t simple_lock_try __P((simple_lock_t)); +void simple_unlock __P((simple_lock_t)); +__END_DECLS +#else /* No multiprocessor locking is necessary. */ +#define simple_lock(l) +#define simple_lock_init(l) +#define simple_lock_try(l) (1) /* Always succeeds. */ +#define simple_unlock(l) +#endif + +/* Sleep locks must work even if no multiprocessing. */ + +#define lock_read_done(l) lock_done(l) +#define lock_write_done(l) lock_done(l) + +void lock_clear_recursive __P((lock_t)); +void lock_done __P((lock_t)); +void lock_init __P((lock_t, boolean_t)); +void lock_read __P((lock_t)); +boolean_t lock_read_to_write __P((lock_t)); +void lock_set_recursive __P((lock_t)); +void lock_sleepable __P((lock_t, boolean_t)); +boolean_t lock_try_read __P((lock_t)); +boolean_t lock_try_read_to_write __P((lock_t)); +boolean_t lock_try_write __P((lock_t)); +void lock_write __P((lock_t)); +void lock_write_to_read __P((lock_t)); +#endif /* !_LOCK_H_ */ diff --git a/sys/vm/pmap.h b/sys/vm/pmap.h new file mode 100644 index 00000000000..71d17f0a549 --- /dev/null +++ b/sys/vm/pmap.h @@ -0,0 +1,137 @@ +/* $NetBSD: pmap.h,v 1.10 1995/03/26 20:39:07 jtc Exp $ */ + +/* + * Copyright (c) 1991, 1993 + * The Regents of the University of California. All rights reserved. + * + * This code is derived from software contributed to Berkeley by + * The Mach Operating System project at Carnegie-Mellon University. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. All advertising materials mentioning features or use of this software + * must display the following acknowledgement: + * This product includes software developed by the University of + * California, Berkeley and its contributors. + * 4. Neither the name of the University nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * @(#)pmap.h 8.1 (Berkeley) 6/11/93 + * + * + * Copyright (c) 1987, 1990 Carnegie-Mellon University. + * All rights reserved. + * + * Author: Avadis Tevanian, Jr. + * + * Permission to use, copy, modify and distribute this software and + * its documentation is hereby granted, provided that both the copyright + * notice and this permission notice appear in all copies of the + * software, derivative works or modified versions, and any portions + * thereof, and that both notices appear in supporting documentation. + * + * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" + * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND + * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. + * + * Carnegie Mellon requests users of this software to return to + * + * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU + * School of Computer Science + * Carnegie Mellon University + * Pittsburgh PA 15213-3890 + * + * any improvements or extensions that they make and grant Carnegie the + * rights to redistribute these changes. + */ + +/* + * Machine address mapping definitions -- machine-independent + * section. [For machine-dependent section, see "machine/pmap.h".] + */ + +#ifndef _PMAP_VM_ +#define _PMAP_VM_ + +/* + * Each machine dependent implementation is expected to + * keep certain statistics. They may do this anyway they + * so choose, but are expected to return the statistics + * in the following structure. + */ +struct pmap_statistics { + long resident_count; /* # of pages mapped (total)*/ + long wired_count; /* # of pages wired */ +}; +typedef struct pmap_statistics *pmap_statistics_t; + +#include <machine/pmap.h> + +#ifdef _KERNEL +__BEGIN_DECLS +void *pmap_bootstrap_alloc __P((int)); +void pmap_bootstrap( /* machine dependent */ ); +void pmap_change_wiring __P((pmap_t, vm_offset_t, boolean_t)); +void pmap_clear_modify __P((vm_offset_t pa)); +void pmap_clear_reference __P((vm_offset_t pa)); +void pmap_collect __P((pmap_t)); +void pmap_copy __P((pmap_t, + pmap_t, vm_offset_t, vm_size_t, vm_offset_t)); +void pmap_copy_page __P((vm_offset_t, vm_offset_t)); +pmap_t pmap_create __P((vm_size_t)); +void pmap_destroy __P((pmap_t)); +void pmap_enter __P((pmap_t, + vm_offset_t, vm_offset_t, vm_prot_t, boolean_t)); +vm_offset_t pmap_extract __P((pmap_t, vm_offset_t)); +#ifndef MACHINE_NONCONTIG +void pmap_init __P((vm_offset_t, vm_offset_t)); +#else +void pmap_init __P((void)); +#endif +boolean_t pmap_is_modified __P((vm_offset_t pa)); +boolean_t pmap_is_referenced __P((vm_offset_t pa)); +vm_offset_t pmap_map __P((vm_offset_t, vm_offset_t, vm_offset_t, int)); +void pmap_page_protect __P((vm_offset_t, vm_prot_t)); +void pmap_pageable __P((pmap_t, + vm_offset_t, vm_offset_t, boolean_t)); +vm_offset_t pmap_phys_address __P((int)); +void pmap_pinit __P((pmap_t)); +void pmap_protect __P((pmap_t, + vm_offset_t, vm_offset_t, vm_prot_t)); +void pmap_reference __P((pmap_t)); +void pmap_release __P((pmap_t)); +void pmap_remove __P((pmap_t, vm_offset_t, vm_offset_t)); +void pmap_update __P((void)); +void pmap_zero_page __P((vm_offset_t)); + +#ifdef MACHINE_NONCONTIG +u_int pmap_free_pages __P(()); +void pmap_init __P(()); +boolean_t pmap_next_page __P(()); +void pmap_startup __P(()); +vm_offset_t pmap_steal_memory __P(()); +void pmap_virtual_space __P(()); +#endif +__END_DECLS +#endif + +#endif /* _PMAP_VM_ */ diff --git a/sys/vm/swap_pager.c b/sys/vm/swap_pager.c new file mode 100644 index 00000000000..d69c7f039b3 --- /dev/null +++ b/sys/vm/swap_pager.c @@ -0,0 +1,1033 @@ +/* $NetBSD: swap_pager.c,v 1.24 1994/10/18 06:42:28 cgd Exp $ */ + +/* + * Copyright (c) 1990 University of Utah. + * Copyright (c) 1991, 1993 + * The Regents of the University of California. All rights reserved. + * + * This code is derived from software contributed to Berkeley by + * the Systems Programming Group of the University of Utah Computer + * Science Department. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. All advertising materials mentioning features or use of this software + * must display the following acknowledgement: + * This product includes software developed by the University of + * California, Berkeley and its contributors. + * 4. Neither the name of the University nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * from: Utah $Hdr: swap_pager.c 1.4 91/04/30$ + * + * @(#)swap_pager.c 8.9 (Berkeley) 3/21/94 + */ + +/* + * Quick hack to page to dedicated partition(s). + * TODO: + * Add multiprocessor locks + * Deal with async writes in a better fashion + */ + +#include <sys/param.h> +#include <sys/systm.h> +#include <sys/proc.h> +#include <sys/buf.h> +#include <sys/map.h> +#include <sys/vnode.h> +#include <sys/malloc.h> + +#include <miscfs/specfs/specdev.h> + +#include <vm/vm.h> +#include <vm/vm_page.h> +#include <vm/vm_pageout.h> +#include <vm/swap_pager.h> + +#define NSWSIZES 16 /* size of swtab */ +#define MAXDADDRS 64 /* max # of disk addrs for fixed allocations */ +#ifndef NPENDINGIO +#define NPENDINGIO 64 /* max # of pending cleans */ +#endif + +#ifdef DEBUG +int swpagerdebug = 0x100; +#define SDB_FOLLOW 0x001 +#define SDB_INIT 0x002 +#define SDB_ALLOC 0x004 +#define SDB_IO 0x008 +#define SDB_WRITE 0x010 +#define SDB_FAIL 0x020 +#define SDB_ALLOCBLK 0x040 +#define SDB_FULL 0x080 +#define SDB_ANOM 0x100 +#define SDB_ANOMPANIC 0x200 +#define SDB_CLUSTER 0x400 +#define SDB_PARANOIA 0x800 +#endif + +TAILQ_HEAD(swpclean, swpagerclean); + +struct swpagerclean { + TAILQ_ENTRY(swpagerclean) spc_list; + int spc_flags; + struct buf *spc_bp; + sw_pager_t spc_swp; + vm_offset_t spc_kva; + vm_page_t spc_m; + int spc_npages; +} swcleanlist[NPENDINGIO]; +typedef struct swpagerclean *swp_clean_t; + +/* spc_flags values */ +#define SPC_FREE 0x00 +#define SPC_BUSY 0x01 +#define SPC_DONE 0x02 +#define SPC_ERROR 0x04 + +struct swtab { + vm_size_t st_osize; /* size of object (bytes) */ + int st_bsize; /* vs. size of swap block (DEV_BSIZE units) */ +#ifdef DEBUG + u_long st_inuse; /* number in this range in use */ + u_long st_usecnt; /* total used of this size */ +#endif +} swtab[NSWSIZES+1]; + +#ifdef DEBUG +int swap_pager_poip; /* pageouts in progress */ +int swap_pager_piip; /* pageins in progress */ +#endif + +int swap_pager_maxcluster; /* maximum cluster size */ +int swap_pager_npendingio; /* number of pager clean structs */ + +struct swpclean swap_pager_inuse; /* list of pending page cleans */ +struct swpclean swap_pager_free; /* list of free pager clean structs */ +struct pagerlst swap_pager_list; /* list of "named" anon regions */ + +static void swap_pager_init __P((void)); +static vm_pager_t swap_pager_alloc + __P((caddr_t, vm_size_t, vm_prot_t, vm_offset_t)); +static void swap_pager_clean __P((int)); +#ifdef DEBUG +static void swap_pager_clean_check __P((vm_page_t *, int, int)); +#endif +static void swap_pager_cluster + __P((vm_pager_t, vm_offset_t, + vm_offset_t *, vm_offset_t *)); +static void swap_pager_dealloc __P((vm_pager_t)); +static int swap_pager_getpage + __P((vm_pager_t, vm_page_t *, int, boolean_t)); +static boolean_t swap_pager_haspage __P((vm_pager_t, vm_offset_t)); +static int swap_pager_io __P((sw_pager_t, vm_page_t *, int, int)); +static void swap_pager_iodone __P((struct buf *)); +static int swap_pager_putpage + __P((vm_pager_t, vm_page_t *, int, boolean_t)); + +struct pagerops swappagerops = { + swap_pager_init, + swap_pager_alloc, + swap_pager_dealloc, + swap_pager_getpage, + swap_pager_putpage, + swap_pager_haspage, + swap_pager_cluster +}; + +static void +swap_pager_init() +{ + register swp_clean_t spc; + register int i, bsize; + extern int dmmin, dmmax; + int maxbsize; + +#ifdef DEBUG + if (swpagerdebug & (SDB_FOLLOW|SDB_INIT)) + printf("swpg_init()\n"); +#endif + dfltpagerops = &swappagerops; + TAILQ_INIT(&swap_pager_list); + + /* + * Allocate async IO structures. + * + * XXX it would be nice if we could do this dynamically based on + * the value of nswbuf (since we are ultimately limited by that) + * but neither nswbuf or malloc has been initialized yet. So the + * structs are statically allocated above. + */ + swap_pager_npendingio = NPENDINGIO; + + /* + * Initialize clean lists + */ + TAILQ_INIT(&swap_pager_inuse); + TAILQ_INIT(&swap_pager_free); + for (i = 0, spc = swcleanlist; i < swap_pager_npendingio; i++, spc++) { + TAILQ_INSERT_TAIL(&swap_pager_free, spc, spc_list); + spc->spc_flags = SPC_FREE; + } + + /* + * Calculate the swap allocation constants. + */ + if (dmmin == 0) { + dmmin = DMMIN; + if (dmmin < CLBYTES/DEV_BSIZE) + dmmin = CLBYTES/DEV_BSIZE; + } + if (dmmax == 0) + dmmax = DMMAX; + + /* + * Fill in our table of object size vs. allocation size + */ + bsize = btodb(PAGE_SIZE); + if (bsize < dmmin) + bsize = dmmin; + maxbsize = btodb(sizeof(sw_bm_t) * NBBY * PAGE_SIZE); + if (maxbsize > dmmax) + maxbsize = dmmax; + for (i = 0; i < NSWSIZES; i++) { + swtab[i].st_osize = (vm_size_t) (MAXDADDRS * dbtob(bsize)); + swtab[i].st_bsize = bsize; + if (bsize <= btodb(MAXPHYS)) + swap_pager_maxcluster = dbtob(bsize); +#ifdef DEBUG + if (swpagerdebug & SDB_INIT) + printf("swpg_init: ix %d, size %x, bsize %x\n", + i, swtab[i].st_osize, swtab[i].st_bsize); +#endif + if (bsize >= maxbsize) + break; + bsize *= 2; + } + swtab[i].st_osize = 0; + swtab[i].st_bsize = bsize; +} + +/* + * Allocate a pager structure and associated resources. + * Note that if we are called from the pageout daemon (handle == NULL) + * we should not wait for memory as it could resulting in deadlock. + */ +static vm_pager_t +swap_pager_alloc(handle, size, prot, foff) + caddr_t handle; + register vm_size_t size; + vm_prot_t prot; + vm_offset_t foff; +{ + register vm_pager_t pager; + register sw_pager_t swp; + struct swtab *swt; + int waitok; + +#ifdef DEBUG + if (swpagerdebug & (SDB_FOLLOW|SDB_ALLOC)) + printf("swpg_alloc(%x, %x, %x)\n", handle, size, prot); +#endif + /* + * If this is a "named" anonymous region, look it up and + * return the appropriate pager if it exists. + */ + if (handle) { + pager = vm_pager_lookup(&swap_pager_list, handle); + if (pager != NULL) { + /* + * Use vm_object_lookup to gain a reference + * to the object and also to remove from the + * object cache. + */ + if (vm_object_lookup(pager) == NULL) + panic("swap_pager_alloc: bad object"); + return(pager); + } + } + /* + * Pager doesn't exist, allocate swap management resources + * and initialize. + */ + waitok = handle ? M_WAITOK : M_NOWAIT; + pager = (vm_pager_t)malloc(sizeof *pager, M_VMPAGER, waitok); + if (pager == NULL) + return(NULL); + swp = (sw_pager_t)malloc(sizeof *swp, M_VMPGDATA, waitok); + if (swp == NULL) { +#ifdef DEBUG + if (swpagerdebug & SDB_FAIL) + printf("swpg_alloc: swpager malloc failed\n"); +#endif + free((caddr_t)pager, M_VMPAGER); + return(NULL); + } + size = round_page(size); + for (swt = swtab; swt->st_osize; swt++) + if (size <= swt->st_osize) + break; +#ifdef DEBUG + swt->st_inuse++; + swt->st_usecnt++; +#endif + swp->sw_osize = size; + swp->sw_bsize = swt->st_bsize; + swp->sw_nblocks = (btodb(size) + swp->sw_bsize - 1) / swp->sw_bsize; + swp->sw_blocks = (sw_blk_t) + malloc(swp->sw_nblocks*sizeof(*swp->sw_blocks), + M_VMPGDATA, M_NOWAIT); + if (swp->sw_blocks == NULL) { + free((caddr_t)swp, M_VMPGDATA); + free((caddr_t)pager, M_VMPAGER); +#ifdef DEBUG + if (swpagerdebug & SDB_FAIL) + printf("swpg_alloc: sw_blocks malloc failed\n"); + swt->st_inuse--; + swt->st_usecnt--; +#endif + return(FALSE); + } + bzero((caddr_t)swp->sw_blocks, + swp->sw_nblocks * sizeof(*swp->sw_blocks)); + swp->sw_poip = 0; + if (handle) { + vm_object_t object; + + swp->sw_flags = SW_NAMED; + TAILQ_INSERT_TAIL(&swap_pager_list, pager, pg_list); + /* + * Consistant with other pagers: return with object + * referenced. Can't do this with handle == NULL + * since it might be the pageout daemon calling. + */ + object = vm_object_allocate(size); + vm_object_enter(object, pager); + vm_object_setpager(object, pager, 0, FALSE); + } else { + swp->sw_flags = 0; + pager->pg_list.tqe_next = NULL; + pager->pg_list.tqe_prev = NULL; + } + pager->pg_handle = handle; + pager->pg_ops = &swappagerops; + pager->pg_type = PG_SWAP; + pager->pg_flags = PG_CLUSTERPUT; + pager->pg_data = swp; + +#ifdef DEBUG + if (swpagerdebug & SDB_ALLOC) + printf("swpg_alloc: pg_data %x, %x of %x at %x\n", + swp, swp->sw_nblocks, swp->sw_bsize, swp->sw_blocks); +#endif + return(pager); +} + +static void +swap_pager_dealloc(pager) + vm_pager_t pager; +{ + register int i; + register sw_blk_t bp; + register sw_pager_t swp; + struct swtab *swt; + int s; + +#ifdef DEBUG + /* save panic time state */ + if ((swpagerdebug & SDB_ANOMPANIC) && panicstr) + return; + if (swpagerdebug & (SDB_FOLLOW|SDB_ALLOC)) + printf("swpg_dealloc(%x)\n", pager); +#endif + /* + * Remove from list right away so lookups will fail if we + * block for pageout completion. + */ + swp = (sw_pager_t) pager->pg_data; + if (swp->sw_flags & SW_NAMED) { + TAILQ_REMOVE(&swap_pager_list, pager, pg_list); + swp->sw_flags &= ~SW_NAMED; + } +#ifdef DEBUG + for (swt = swtab; swt->st_osize; swt++) + if (swp->sw_osize <= swt->st_osize) + break; + swt->st_inuse--; +#endif + + /* + * Wait for all pageouts to finish and remove + * all entries from cleaning list. + */ + s = splbio(); + while (swp->sw_poip) { + swp->sw_flags |= SW_WANTED; + (void) tsleep(swp, PVM, "swpgdealloc", 0); + } + splx(s); + swap_pager_clean(B_WRITE); + + /* + * Free left over swap blocks + */ + for (i = 0, bp = swp->sw_blocks; i < swp->sw_nblocks; i++, bp++) + if (bp->swb_block) { +#ifdef DEBUG + if (swpagerdebug & (SDB_ALLOCBLK|SDB_FULL)) + printf("swpg_dealloc: blk %x\n", + bp->swb_block); +#endif + rmfree(swapmap, swp->sw_bsize, bp->swb_block); + } + /* + * Free swap management resources + */ + free((caddr_t)swp->sw_blocks, M_VMPGDATA); + free((caddr_t)swp, M_VMPGDATA); + free((caddr_t)pager, M_VMPAGER); +} + +static int +swap_pager_getpage(pager, mlist, npages, sync) + vm_pager_t pager; + vm_page_t *mlist; + int npages; + boolean_t sync; +{ + register int rv; +#ifdef DIAGNOSTIC + vm_page_t m; + int i; +#endif + +#ifdef DEBUG + if (swpagerdebug & SDB_FOLLOW) + printf("swpg_getpage(%x, %x, %x, %x)\n", + pager, mlist, npages, sync); +#endif +#ifdef DIAGNOSTIC + for (i = 0; i < npages; i++) { + m = mlist[i]; + + if (m->flags & PG_FAULTING) + panic("swap_pager_getpage: page is already faulting"); + m->flags |= PG_FAULTING; + } +#endif + rv = swap_pager_io((sw_pager_t)pager->pg_data, mlist, npages, B_READ); +#ifdef DIAGNOSTIC + for (i = 0; i < npages; i++) { + m = mlist[i]; + + m->flags &= ~PG_FAULTING; + } +#endif + return(rv); +} + +static int +swap_pager_putpage(pager, mlist, npages, sync) + vm_pager_t pager; + vm_page_t *mlist; + int npages; + boolean_t sync; +{ + int flags; + +#ifdef DEBUG + if (swpagerdebug & SDB_FOLLOW) + printf("swpg_putpage(%x, %x, %x, %x)\n", + pager, mlist, npages, sync); +#endif + if (pager == NULL) { + swap_pager_clean(B_WRITE); + return (VM_PAGER_OK); /* ??? */ + } + flags = B_WRITE; + if (!sync) + flags |= B_ASYNC; + return(swap_pager_io((sw_pager_t)pager->pg_data, + mlist, npages, flags)); +} + +static boolean_t +swap_pager_haspage(pager, offset) + vm_pager_t pager; + vm_offset_t offset; +{ + register sw_pager_t swp; + register sw_blk_t swb; + int ix; + +#ifdef DEBUG + if (swpagerdebug & (SDB_FOLLOW|SDB_ALLOCBLK)) + printf("swpg_haspage(%x, %x) ", pager, offset); +#endif + swp = (sw_pager_t) pager->pg_data; + ix = offset / dbtob(swp->sw_bsize); + if (swp->sw_blocks == NULL || ix >= swp->sw_nblocks) { +#ifdef DEBUG + if (swpagerdebug & (SDB_FAIL|SDB_FOLLOW|SDB_ALLOCBLK)) + printf("swpg_haspage: %x bad offset %x, ix %x\n", + swp->sw_blocks, offset, ix); +#endif + return(FALSE); + } + swb = &swp->sw_blocks[ix]; + if (swb->swb_block) + ix = atop(offset % dbtob(swp->sw_bsize)); +#ifdef DEBUG + if (swpagerdebug & SDB_ALLOCBLK) + printf("%x blk %x+%x ", swp->sw_blocks, swb->swb_block, ix); + if (swpagerdebug & (SDB_FOLLOW|SDB_ALLOCBLK)) + printf("-> %c\n", + "FT"[swb->swb_block && (swb->swb_mask & (1 << ix))]); +#endif + if (swb->swb_block && (swb->swb_mask & (1 << ix))) + return(TRUE); + return(FALSE); +} + +static void +swap_pager_cluster(pager, offset, loffset, hoffset) + vm_pager_t pager; + vm_offset_t offset; + vm_offset_t *loffset; + vm_offset_t *hoffset; +{ + sw_pager_t swp; + register int bsize; + vm_offset_t loff, hoff; + +#ifdef DEBUG + if (swpagerdebug & (SDB_FOLLOW|SDB_CLUSTER)) + printf("swpg_cluster(%x, %x) ", pager, offset); +#endif + swp = (sw_pager_t) pager->pg_data; + bsize = dbtob(swp->sw_bsize); + if (bsize > swap_pager_maxcluster) + bsize = swap_pager_maxcluster; + + loff = offset - (offset % bsize); + if (loff >= swp->sw_osize) + panic("swap_pager_cluster: bad offset"); + + hoff = loff + bsize; + if (hoff > swp->sw_osize) + hoff = swp->sw_osize; + + *loffset = loff; + *hoffset = hoff; +#ifdef DEBUG + if (swpagerdebug & (SDB_FOLLOW|SDB_CLUSTER)) + printf("returns [%x-%x]\n", loff, hoff); +#endif +} + +/* + * Scaled down version of swap(). + * Assumes that PAGE_SIZE < MAXPHYS; i.e. only one operation needed. + * BOGUS: lower level IO routines expect a KVA so we have to map our + * provided physical page into the KVA to keep them happy. + */ +static int +swap_pager_io(swp, mlist, npages, flags) + register sw_pager_t swp; + vm_page_t *mlist; + int npages; + int flags; +{ + register struct buf *bp; + register sw_blk_t swb; + register int s; + int ix, mask; + boolean_t rv; + vm_offset_t kva, off; + swp_clean_t spc; + vm_page_t m; + +#ifdef DEBUG + /* save panic time state */ + if ((swpagerdebug & SDB_ANOMPANIC) && panicstr) + return (VM_PAGER_FAIL); /* XXX: correct return? */ + if (swpagerdebug & (SDB_FOLLOW|SDB_IO)) + printf("swpg_io(%x, %x, %x, %x)\n", swp, mlist, npages, flags); + if (flags & B_READ) { + if (flags & B_ASYNC) + panic("swap_pager_io: cannot do ASYNC reads"); + if (npages != 1) + panic("swap_pager_io: cannot do clustered reads"); + } +#endif + + /* + * First determine if the page exists in the pager if this is + * a sync read. This quickly handles cases where we are + * following shadow chains looking for the top level object + * with the page. + */ + m = *mlist; + off = m->offset + m->object->paging_offset; + ix = off / dbtob(swp->sw_bsize); + if (swp->sw_blocks == NULL || ix >= swp->sw_nblocks) { +#ifdef DEBUG + if ((flags & B_READ) == 0 && (swpagerdebug & SDB_ANOM)) { + printf("swap_pager_io: no swap block on write\n"); + return(VM_PAGER_BAD); + } +#endif + return(VM_PAGER_FAIL); + } + swb = &swp->sw_blocks[ix]; + off = off % dbtob(swp->sw_bsize); + if ((flags & B_READ) && + (swb->swb_block == 0 || (swb->swb_mask & (1 << atop(off))) == 0)) + return(VM_PAGER_FAIL); + + /* + * For reads (pageins) and synchronous writes, we clean up + * all completed async pageouts. + */ + if ((flags & B_ASYNC) == 0) { + s = splbio(); + swap_pager_clean(flags&B_READ); +#ifdef DEBUG + if (swpagerdebug & SDB_PARANOIA) + swap_pager_clean_check(mlist, npages, flags&B_READ); +#endif + splx(s); + } + /* + * For async writes (pageouts), we cleanup completed pageouts so + * that all available resources are freed. Also tells us if this + * page is already being cleaned. If it is, or no resources + * are available, we try again later. + */ + else { + swap_pager_clean(B_WRITE); +#ifdef DEBUG + if (swpagerdebug & SDB_PARANOIA) + swap_pager_clean_check(mlist, npages, B_WRITE); +#endif + if (swap_pager_free.tqh_first == NULL) { +#ifdef DEBUG + if (swpagerdebug & SDB_FAIL) + printf("%s: no available io headers\n", + "swap_pager_io"); +#endif + return(VM_PAGER_AGAIN); + } + } + + /* + * Allocate a swap block if necessary. + */ + if (swb->swb_block == 0) { + swb->swb_block = rmalloc(swapmap, swp->sw_bsize); + if (swb->swb_block == 0) { +#ifdef DEBUG + if (swpagerdebug & SDB_FAIL) + printf("swpg_io: rmalloc of %x failed\n", + swp->sw_bsize); +#endif + /* + * XXX this is technically a resource shortage that + * should return AGAIN, but the situation isn't likely + * to be remedied just by delaying a little while and + * trying again (the pageout daemon's current response + * to AGAIN) so we just return FAIL. + */ + return(VM_PAGER_FAIL); + } +#ifdef DEBUG + if (swpagerdebug & (SDB_FULL|SDB_ALLOCBLK)) + printf("swpg_io: %x alloc blk %x at ix %x\n", + swp->sw_blocks, swb->swb_block, ix); +#endif + } + + /* + * Allocate a kernel virtual address and initialize so that PTE + * is available for lower level IO drivers. + */ + kva = vm_pager_map_pages(mlist, npages, !(flags & B_ASYNC)); + if (kva == NULL) { +#ifdef DEBUG + if (swpagerdebug & SDB_FAIL) + printf("%s: no KVA space to map pages\n", + "swap_pager_io"); +#endif + return(VM_PAGER_AGAIN); + } + + /* + * Get a swap buffer header and initialize it. + */ + s = splbio(); + while (bswlist.b_actf == NULL) { +#ifdef DEBUG + if (swpagerdebug & SDB_IO) /* XXX what should this be? */ + printf("swap_pager_io: wait on swbuf for %x (%d)\n", + m, flags); +#endif + bswlist.b_flags |= B_WANTED; + tsleep((caddr_t)&bswlist, PSWP+1, "swpgiobuf", 0); + } + bp = bswlist.b_actf; + bswlist.b_actf = bp->b_actf; + splx(s); + bp->b_flags = B_BUSY | (flags & B_READ); + bp->b_proc = &proc0; /* XXX (but without B_PHYS set this is ok) */ + bp->b_data = (caddr_t)kva; + bp->b_blkno = swb->swb_block + btodb(off); + VHOLD(swapdev_vp); + bp->b_vp = swapdev_vp; + if (swapdev_vp->v_type == VBLK) + bp->b_dev = swapdev_vp->v_rdev; + bp->b_bcount = npages * PAGE_SIZE; + + /* + * For writes we set up additional buffer fields, record a pageout + * in progress and mark that these swap blocks are now allocated. + */ + if ((bp->b_flags & B_READ) == 0) { + bp->b_dirtyoff = 0; + bp->b_dirtyend = npages * PAGE_SIZE; + swapdev_vp->v_numoutput++; + s = splbio(); + swp->sw_poip++; + splx(s); + mask = (~(~0 << npages)) << atop(off); +#ifdef DEBUG + swap_pager_poip++; + if (swpagerdebug & SDB_WRITE) + printf("swpg_io: write: bp=%x swp=%x poip=%d\n", + bp, swp, swp->sw_poip); + if ((swpagerdebug & SDB_ALLOCBLK) && + (swb->swb_mask & mask) != mask) + printf("swpg_io: %x write %d pages at %x+%x\n", + swp->sw_blocks, npages, swb->swb_block, + atop(off)); + if (swpagerdebug & SDB_CLUSTER) + printf("swpg_io: off=%x, npg=%x, mask=%x, bmask=%x\n", + off, npages, mask, swb->swb_mask); +#endif + swb->swb_mask |= mask; + } + /* + * If this is an async write we set up still more buffer fields + * and place a "cleaning" entry on the inuse queue. + */ + if ((flags & (B_READ|B_ASYNC)) == B_ASYNC) { +#ifdef DEBUG + if (swap_pager_free.tqh_first == NULL) + panic("swpg_io: lost spc"); +#endif + spc = swap_pager_free.tqh_first; + TAILQ_REMOVE(&swap_pager_free, spc, spc_list); +#ifdef DEBUG + if (spc->spc_flags != SPC_FREE) + panic("swpg_io: bad free spc"); +#endif + spc->spc_flags = SPC_BUSY; + spc->spc_bp = bp; + spc->spc_swp = swp; + spc->spc_kva = kva; + /* + * Record the first page. This allows swap_pager_clean + * to efficiently handle the common case of a single page. + * For clusters, it allows us to locate the object easily + * and we then reconstruct the rest of the mlist from spc_kva. + */ + spc->spc_m = m; + spc->spc_npages = npages; + bp->b_flags |= B_CALL; + bp->b_iodone = swap_pager_iodone; + s = splbio(); + TAILQ_INSERT_TAIL(&swap_pager_inuse, spc, spc_list); + splx(s); + } + + /* + * Finally, start the IO operation. + * If it is async we are all done, otherwise we must wait for + * completion and cleanup afterwards. + */ +#ifdef DEBUG + if (swpagerdebug & SDB_IO) + printf("swpg_io: IO start: bp %x, db %x, va %x, pa %x\n", + bp, swb->swb_block+btodb(off), kva, VM_PAGE_TO_PHYS(m)); +#endif + VOP_STRATEGY(bp); + if ((flags & (B_READ|B_ASYNC)) == B_ASYNC) { +#ifdef DEBUG + if (swpagerdebug & SDB_IO) + printf("swpg_io: IO started: bp %x\n", bp); +#endif + return(VM_PAGER_PEND); + } + s = splbio(); +#ifdef DEBUG + if (flags & B_READ) + swap_pager_piip++; + else + swap_pager_poip++; +#endif + while ((bp->b_flags & B_DONE) == 0) + (void) tsleep(bp, PVM, "swpgio", 0); + if ((flags & B_READ) == 0) + --swp->sw_poip; +#ifdef DEBUG + if (flags & B_READ) + --swap_pager_piip; + else + --swap_pager_poip; +#endif + rv = (bp->b_flags & B_ERROR) ? VM_PAGER_ERROR : VM_PAGER_OK; + bp->b_flags &= ~(B_BUSY|B_WANTED|B_PHYS|B_PAGET|B_UAREA|B_DIRTY); + bp->b_actf = bswlist.b_actf; + bswlist.b_actf = bp; + if (bp->b_vp) + brelvp(bp); + if (bswlist.b_flags & B_WANTED) { + bswlist.b_flags &= ~B_WANTED; + wakeup(&bswlist); + } + if ((flags & B_READ) == 0 && rv == VM_PAGER_OK) { + m->flags |= PG_CLEAN; + pmap_clear_modify(VM_PAGE_TO_PHYS(m)); + } + splx(s); +#ifdef DEBUG + if (swpagerdebug & SDB_IO) + printf("swpg_io: IO done: bp %x, rv %d\n", bp, rv); + if ((swpagerdebug & SDB_FAIL) && rv == VM_PAGER_ERROR) + printf("swpg_io: IO error\n"); +#endif + vm_pager_unmap_pages(kva, npages); + return(rv); +} + +static void +swap_pager_clean(rw) + int rw; +{ + register swp_clean_t spc; + register int s, i; + vm_object_t object; + vm_page_t m; + +#ifdef DEBUG + /* save panic time state */ + if ((swpagerdebug & SDB_ANOMPANIC) && panicstr) + return; + if (swpagerdebug & SDB_FOLLOW) + printf("swpg_clean(%x)\n", rw); +#endif + + for (;;) { + /* + * Look up and removal from inuse list must be done + * at splbio() to avoid conflicts with swap_pager_iodone. + */ + s = splbio(); + for (spc = swap_pager_inuse.tqh_first; + spc != NULL; + spc = spc->spc_list.tqe_next) { + /* + * If the operation is done, remove it from the + * list and process it. + * + * XXX if we can't get the object lock we also + * leave it on the list and try again later. + * Is there something better we could do? + */ + if ((spc->spc_flags & SPC_DONE) && + vm_object_lock_try(spc->spc_m->object)) { + TAILQ_REMOVE(&swap_pager_inuse, spc, spc_list); + break; + } + } + splx(s); + + /* + * No operations done, thats all we can do for now. + */ + if (spc == NULL) + break; + + /* + * Found a completed operation so finish it off. + * Note: no longer at splbio since entry is off the list. + */ + m = spc->spc_m; + object = m->object; + + /* + * Process each page in the cluster. + * The first page is explicitly kept in the cleaning + * entry, others must be reconstructed from the KVA. + */ + for (i = 0; i < spc->spc_npages; i++) { + if (i) + m = vm_pager_atop(spc->spc_kva + ptoa(i)); + /* + * If no error mark as clean and inform the pmap + * system. If there was an error, mark as dirty + * so we will try again. + * + * XXX could get stuck doing this, should give up + * after awhile. + */ + if (spc->spc_flags & SPC_ERROR) { + printf("%s: clean of page %x failed\n", + "swap_pager_clean", + VM_PAGE_TO_PHYS(m)); + m->flags |= PG_LAUNDRY; + } else { + m->flags |= PG_CLEAN; + pmap_clear_modify(VM_PAGE_TO_PHYS(m)); + } + m->flags &= ~PG_BUSY; + PAGE_WAKEUP(m); + } + + /* + * Done with the object, decrement the paging count + * and unlock it. + */ + if (--object->paging_in_progress == 0) + wakeup(object); + vm_object_unlock(object); + + /* + * Free up KVM used and put the entry back on the list. + */ + vm_pager_unmap_pages(spc->spc_kva, spc->spc_npages); + spc->spc_flags = SPC_FREE; + TAILQ_INSERT_TAIL(&swap_pager_free, spc, spc_list); +#ifdef DEBUG + if (swpagerdebug & SDB_WRITE) + printf("swpg_clean: free spc %x\n", spc); +#endif + } +} + +#ifdef DEBUG +static void +swap_pager_clean_check(mlist, npages, rw) + vm_page_t *mlist; + int npages; + int rw; +{ + register swp_clean_t spc; + boolean_t bad; + int i, j, s; + vm_page_t m; + + if (panicstr) + return; + + bad = FALSE; + s = splbio(); + for (spc = swap_pager_inuse.tqh_first; + spc != NULL; + spc = spc->spc_list.tqe_next) { + for (j = 0; j < spc->spc_npages; j++) { + m = vm_pager_atop(spc->spc_kva + ptoa(j)); + for (i = 0; i < npages; i++) + if (m == mlist[i]) { + if (swpagerdebug & SDB_ANOM) + printf( + "swpg_clean_check: %s: page %x on list, flags %x\n", + rw == B_WRITE ? "write" : "read", mlist[i], spc->spc_flags); + bad = TRUE; + } + } + } + splx(s); + if (bad) + panic("swpg_clean_check"); +} +#endif + +static void +swap_pager_iodone(bp) + register struct buf *bp; +{ + register swp_clean_t spc; + daddr_t blk; + int s; + +#ifdef DEBUG + /* save panic time state */ + if ((swpagerdebug & SDB_ANOMPANIC) && panicstr) + return; + if (swpagerdebug & SDB_FOLLOW) + printf("swpg_iodone(%x)\n", bp); +#endif + s = splbio(); + for (spc = swap_pager_inuse.tqh_first; + spc != NULL; + spc = spc->spc_list.tqe_next) + if (spc->spc_bp == bp) + break; +#ifdef DEBUG + if (spc == NULL) + panic("swap_pager_iodone: bp not found"); +#endif + + spc->spc_flags &= ~SPC_BUSY; + spc->spc_flags |= SPC_DONE; + if (bp->b_flags & B_ERROR) + spc->spc_flags |= SPC_ERROR; + spc->spc_bp = NULL; + blk = bp->b_blkno; + +#ifdef DEBUG + --swap_pager_poip; + if (swpagerdebug & SDB_WRITE) + printf("swpg_iodone: bp=%x swp=%x flags=%x spc=%x poip=%x\n", + bp, spc->spc_swp, spc->spc_swp->sw_flags, + spc, spc->spc_swp->sw_poip); +#endif + + spc->spc_swp->sw_poip--; + if (spc->spc_swp->sw_flags & SW_WANTED) { + spc->spc_swp->sw_flags &= ~SW_WANTED; + wakeup(spc->spc_swp); + } + + bp->b_flags &= ~(B_BUSY|B_WANTED|B_PHYS|B_PAGET|B_UAREA|B_DIRTY); + bp->b_actf = bswlist.b_actf; + bswlist.b_actf = bp; + if (bp->b_vp) + brelvp(bp); + if (bswlist.b_flags & B_WANTED) { + bswlist.b_flags &= ~B_WANTED; + wakeup(&bswlist); + } + wakeup(&vm_pages_needed); + splx(s); +} diff --git a/sys/vm/swap_pager.h b/sys/vm/swap_pager.h new file mode 100644 index 00000000000..07b25ecbe8b --- /dev/null +++ b/sys/vm/swap_pager.h @@ -0,0 +1,92 @@ +/* $NetBSD: swap_pager.h,v 1.6 1994/06/29 06:47:49 cgd Exp $ */ + +/* + * Copyright (c) 1990 University of Utah. + * Copyright (c) 1991, 1993 + * The Regents of the University of California. All rights reserved. + * + * This code is derived from software contributed to Berkeley by + * the Systems Programming Group of the University of Utah Computer + * Science Department. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. All advertising materials mentioning features or use of this software + * must display the following acknowledgement: + * This product includes software developed by the University of + * California, Berkeley and its contributors. + * 4. Neither the name of the University nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * @(#)swap_pager.h 8.1 (Berkeley) 6/11/93 + */ + +#ifndef _SWAP_PAGER_ +#define _SWAP_PAGER_ 1 + +/* + * In the swap pager, the backing store for an object is organized as an + * array of some number of "swap blocks". A swap block consists of a bitmask + * and some number of contiguous DEV_BSIZE disk blocks. The minimum size + * of a swap block is: + * + * max(PAGE_SIZE, dmmin*DEV_BSIZE) [ 32k currently ] + * + * bytes (since the pager interface is page oriented), the maximum size is: + * + * min(#bits(swb_mask)*PAGE_SIZE, dmmax*DEV_BSIZE) [ 128k currently ] + * + * where dmmin and dmmax are left over from the old VM interface. The bitmask + * (swb_mask) is used by swap_pager_haspage() to determine if a particular + * page has actually been written; i.e. the pager copy of the page is valid. + * All swap blocks in the backing store of an object will be the same size. + * + * The reason for variable sized swap blocks is to reduce fragmentation of + * swap resources. Whenever possible we allocate smaller swap blocks to + * smaller objects. The swap block size is determined from a table of + * object-size vs. swap-block-size computed at boot time. + */ +typedef int sw_bm_t; /* pager bitmask */ + +struct swblock { + sw_bm_t swb_mask; /* bitmask of valid pages in this block */ + daddr_t swb_block; /* starting disk block for this block */ +}; +typedef struct swblock *sw_blk_t; + +/* + * Swap pager private data. + */ +struct swpager { + vm_size_t sw_osize; /* size of object we are backing (bytes) */ + int sw_bsize; /* size of swap blocks (DEV_BSIZE units) */ + int sw_nblocks;/* number of blocks in list (sw_blk_t units) */ + sw_blk_t sw_blocks; /* pointer to list of swap blocks */ + short sw_flags; /* flags */ + short sw_poip; /* pageouts in progress */ +}; +typedef struct swpager *sw_pager_t; + +#define SW_WANTED 0x01 +#define SW_NAMED 0x02 + +#endif /* _SWAP_PAGER_ */ diff --git a/sys/vm/vm.h b/sys/vm/vm.h new file mode 100644 index 00000000000..c23a14e1d16 --- /dev/null +++ b/sys/vm/vm.h @@ -0,0 +1,93 @@ +/* $NetBSD: vm.h,v 1.13 1994/06/29 06:47:52 cgd Exp $ */ + +/* + * Copyright (c) 1991, 1993 + * The Regents of the University of California. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. All advertising materials mentioning features or use of this software + * must display the following acknowledgement: + * This product includes software developed by the University of + * California, Berkeley and its contributors. + * 4. Neither the name of the University nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * @(#)vm.h 8.2 (Berkeley) 12/13/93 + */ + +#ifndef VM_H +#define VM_H + +typedef int vm_inherit_t; /* XXX: inheritance codes */ + +union vm_map_object; +typedef union vm_map_object vm_map_object_t; + +struct vm_map_entry; +typedef struct vm_map_entry *vm_map_entry_t; + +struct vm_map; +typedef struct vm_map *vm_map_t; + +struct vm_object; +typedef struct vm_object *vm_object_t; + +struct vm_page; +typedef struct vm_page *vm_page_t; + +struct pager_struct; +typedef struct pager_struct *vm_pager_t; + +#include <sys/vmmeter.h> +#include <sys/queue.h> +#include <vm/vm_param.h> +#include <vm/lock.h> +#include <vm/vm_prot.h> +#include <vm/vm_inherit.h> +#include <vm/vm_map.h> +#include <vm/vm_object.h> +#include <vm/pmap.h> +#include <vm/vm_extern.h> + +/* + * Shareable process virtual address space. + * May eventually be merged with vm_map. + * Several fields are temporary (text, data stuff). + */ +struct vmspace { + struct vm_map vm_map; /* VM address map */ + struct pmap vm_pmap; /* private physical map */ + int vm_refcnt; /* number of references */ + caddr_t vm_shm; /* SYS5 shared memory private data XXX */ +/* we copy from vm_startcopy to the end of the structure on fork */ +#define vm_startcopy vm_rssize + segsz_t vm_rssize; /* current resident set size in pages */ + segsz_t vm_swrss; /* resident set size before last swap */ + segsz_t vm_tsize; /* text size (pages) XXX */ + segsz_t vm_dsize; /* data size (pages) XXX */ + segsz_t vm_ssize; /* stack size (pages) */ + caddr_t vm_taddr; /* user virtual address of text XXX */ + caddr_t vm_daddr; /* user virtual address of data XXX */ + caddr_t vm_maxsaddr; /* user VA at max stack growth */ +}; +#endif /* VM_H */ diff --git a/sys/vm/vm_extern.h b/sys/vm/vm_extern.h new file mode 100644 index 00000000000..91e0417c78f --- /dev/null +++ b/sys/vm/vm_extern.h @@ -0,0 +1,126 @@ +/* $NetBSD: vm_extern.h,v 1.14 1995/09/27 20:30:17 thorpej Exp $ */ + +/*- + * Copyright (c) 1992, 1993 + * The Regents of the University of California. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. All advertising materials mentioning features or use of this software + * must display the following acknowledgement: + * This product includes software developed by the University of + * California, Berkeley and its contributors. + * 4. Neither the name of the University nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * @(#)vm_extern.h 8.2 (Berkeley) 1/12/94 + */ + +struct buf; +struct loadavg; +struct proc; +struct vmspace; +struct vmtotal; +struct mount; +struct vnode; + +#ifdef KGDB +void chgkprot __P((caddr_t, int, int)); +#endif + +#ifdef _KERNEL +#ifdef TYPEDEF_FOR_UAP +int compat_43_getpagesize __P((struct proc *p, void *, int *)); +int madvise __P((struct proc *, void *, int *)); +int mincore __P((struct proc *, void *, int *)); +int mprotect __P((struct proc *, void *, int *)); +int msync __P((struct proc *, void *, int *)); +int munmap __P((struct proc *, void *, int *)); +int obreak __P((struct proc *, void *, int *)); +int sbrk __P((struct proc *, void *, int *)); +int smmap __P((struct proc *, void *, int *)); +int sstk __P((struct proc *, void *, int *)); +#endif + +void assert_wait __P((void *, boolean_t)); +int grow __P((struct proc *, vm_offset_t)); +void iprintf __P((void (*)(const char *, ...), const char *, ...)); +int kernacc __P((caddr_t, int, int)); +int kinfo_loadavg __P((int, char *, int *, int, int *)); +int kinfo_meter __P((int, caddr_t, int *, int, int *)); +vm_offset_t kmem_alloc __P((vm_map_t, vm_size_t)); +vm_offset_t kmem_alloc_pageable __P((vm_map_t, vm_size_t)); +vm_offset_t kmem_alloc_wait __P((vm_map_t, vm_size_t)); +void kmem_free __P((vm_map_t, vm_offset_t, vm_size_t)); +void kmem_free_wakeup __P((vm_map_t, vm_offset_t, vm_size_t)); +void kmem_init __P((vm_offset_t, vm_offset_t)); +vm_offset_t kmem_malloc __P((vm_map_t, vm_size_t, boolean_t)); +vm_map_t kmem_suballoc __P((vm_map_t, vm_offset_t *, vm_offset_t *, + vm_size_t, boolean_t)); +void loadav __P((struct loadavg *)); +void munmapfd __P((struct proc *, int)); +int pager_cache __P((vm_object_t, boolean_t)); +void sched __P((void)); +int svm_allocate __P((struct proc *, void *, int *)); +int svm_deallocate __P((struct proc *, void *, int *)); +int svm_inherit __P((struct proc *, void *, int *)); +int svm_protect __P((struct proc *, void *, int *)); +void swapinit __P((void)); +void swapout __P((struct proc *)); +void swapout_threads __P((void)); +int swfree __P((struct proc *, int)); +void swstrategy __P((struct buf *)); +void thread_block __P((void)); +void thread_sleep __P((void *, simple_lock_t, boolean_t)); +void thread_wakeup __P((void *)); +int useracc __P((caddr_t, int, int)); +int vm_allocate __P((vm_map_t, + vm_offset_t *, vm_size_t, boolean_t)); +int vm_allocate_with_pager __P((vm_map_t, vm_offset_t *, + vm_size_t, boolean_t, vm_pager_t, vm_offset_t, boolean_t)); +int vm_deallocate __P((vm_map_t, vm_offset_t, vm_size_t)); +int vm_fault __P((vm_map_t, vm_offset_t, vm_prot_t, boolean_t)); +void vm_fault_copy_entry __P((vm_map_t, + vm_map_t, vm_map_entry_t, vm_map_entry_t)); +void vm_fault_unwire __P((vm_map_t, vm_offset_t, vm_offset_t)); +int vm_fault_wire __P((vm_map_t, vm_offset_t, vm_offset_t)); +int vm_fork __P((struct proc *, struct proc *, int)); +int vm_inherit __P((vm_map_t, + vm_offset_t, vm_size_t, vm_inherit_t)); +void vm_init_limits __P((struct proc *)); +void vm_mem_init __P((void)); +int vm_mmap __P((vm_map_t, vm_offset_t *, vm_size_t, + vm_prot_t, vm_prot_t, int, caddr_t, vm_offset_t)); +int vm_protect __P((vm_map_t, + vm_offset_t, vm_size_t, boolean_t, vm_prot_t)); +void vm_set_page_size __P((void)); +void vmmeter __P((void)); +struct vmspace *vmspace_alloc __P((vm_offset_t, vm_offset_t, int)); +struct vmspace *vmspace_fork __P((struct vmspace *)); +void vmspace_free __P((struct vmspace *)); +void vmtotal __P((struct vmtotal *)); +void vnode_pager_setsize __P((struct vnode *, u_long)); +void vnode_pager_umount __P((struct mount *)); +boolean_t vnode_pager_uncache __P((struct vnode *)); +void vslock __P((caddr_t, u_int)); +void vsunlock __P((caddr_t, u_int, int)); +#endif diff --git a/sys/vm/vm_fault.c b/sys/vm/vm_fault.c new file mode 100644 index 00000000000..4d9c8c5033e --- /dev/null +++ b/sys/vm/vm_fault.c @@ -0,0 +1,1036 @@ +/* $NetBSD: vm_fault.c,v 1.16 1994/09/07 20:25:07 mycroft Exp $ */ + +/* + * Copyright (c) 1991, 1993 + * The Regents of the University of California. All rights reserved. + * + * This code is derived from software contributed to Berkeley by + * The Mach Operating System project at Carnegie-Mellon University. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. All advertising materials mentioning features or use of this software + * must display the following acknowledgement: + * This product includes software developed by the University of + * California, Berkeley and its contributors. + * 4. Neither the name of the University nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * @(#)vm_fault.c 8.4 (Berkeley) 1/12/94 + * + * + * Copyright (c) 1987, 1990 Carnegie-Mellon University. + * All rights reserved. + * + * Authors: Avadis Tevanian, Jr., Michael Wayne Young + * + * Permission to use, copy, modify and distribute this software and + * its documentation is hereby granted, provided that both the copyright + * notice and this permission notice appear in all copies of the + * software, derivative works or modified versions, and any portions + * thereof, and that both notices appear in supporting documentation. + * + * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" + * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND + * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. + * + * Carnegie Mellon requests users of this software to return to + * + * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU + * School of Computer Science + * Carnegie Mellon University + * Pittsburgh PA 15213-3890 + * + * any improvements or extensions that they make and grant Carnegie the + * rights to redistribute these changes. + */ + +/* + * Page fault handling module. + */ + +#include <sys/param.h> +#include <sys/systm.h> + +#include <vm/vm.h> +#include <vm/vm_page.h> +#include <vm/vm_pageout.h> + +/* + * vm_fault: + * + * Handle a page fault occuring at the given address, + * requiring the given permissions, in the map specified. + * If successful, the page is inserted into the + * associated physical map. + * + * NOTE: the given address should be truncated to the + * proper page address. + * + * KERN_SUCCESS is returned if the page fault is handled; otherwise, + * a standard error specifying why the fault is fatal is returned. + * + * + * The map in question must be referenced, and remains so. + * Caller may hold no locks. + */ +int +vm_fault(map, vaddr, fault_type, change_wiring) + vm_map_t map; + vm_offset_t vaddr; + vm_prot_t fault_type; + boolean_t change_wiring; +{ + vm_object_t first_object; + vm_offset_t first_offset; + vm_map_entry_t entry; + register vm_object_t object; + register vm_offset_t offset; + register vm_page_t m; + vm_page_t first_m; + vm_prot_t prot; + int result; + boolean_t wired; + boolean_t su; + boolean_t lookup_still_valid; + boolean_t page_exists; + vm_page_t old_m; + vm_object_t next_object; + + cnt.v_faults++; /* needs lock XXX */ +/* + * Recovery actions + */ +#define FREE_PAGE(m) { \ + PAGE_WAKEUP(m); \ + vm_page_lock_queues(); \ + vm_page_free(m); \ + vm_page_unlock_queues(); \ +} + +#define RELEASE_PAGE(m) { \ + PAGE_WAKEUP(m); \ + vm_page_lock_queues(); \ + vm_page_activate(m); \ + vm_page_unlock_queues(); \ +} + +#define UNLOCK_MAP { \ + if (lookup_still_valid) { \ + vm_map_lookup_done(map, entry); \ + lookup_still_valid = FALSE; \ + } \ +} + +#define UNLOCK_THINGS { \ + object->paging_in_progress--; \ + vm_object_unlock(object); \ + if (object != first_object) { \ + vm_object_lock(first_object); \ + FREE_PAGE(first_m); \ + first_object->paging_in_progress--; \ + vm_object_unlock(first_object); \ + } \ + UNLOCK_MAP; \ +} + +#define UNLOCK_AND_DEALLOCATE { \ + UNLOCK_THINGS; \ + vm_object_deallocate(first_object); \ +} + + RetryFault: ; + + /* + * Find the backing store object and offset into + * it to begin the search. + */ + + if ((result = vm_map_lookup(&map, vaddr, fault_type, &entry, + &first_object, &first_offset, + &prot, &wired, &su)) != KERN_SUCCESS) { + return(result); + } + lookup_still_valid = TRUE; + + if (wired) + fault_type = prot; + + first_m = NULL; + + /* + * Make a reference to this object to + * prevent its disposal while we are messing with + * it. Once we have the reference, the map is free + * to be diddled. Since objects reference their + * shadows (and copies), they will stay around as well. + */ + + vm_object_lock(first_object); + + first_object->ref_count++; + first_object->paging_in_progress++; + + /* + * INVARIANTS (through entire routine): + * + * 1) At all times, we must either have the object + * lock or a busy page in some object to prevent + * some other thread from trying to bring in + * the same page. + * + * Note that we cannot hold any locks during the + * pager access or when waiting for memory, so + * we use a busy page then. + * + * Note also that we aren't as concerned about + * more than one thead attempting to pager_data_unlock + * the same page at once, so we don't hold the page + * as busy then, but do record the highest unlock + * value so far. [Unlock requests may also be delivered + * out of order.] + * + * 2) Once we have a busy page, we must remove it from + * the pageout queues, so that the pageout daemon + * will not grab it away. + * + * 3) To prevent another thread from racing us down the + * shadow chain and entering a new page in the top + * object before we do, we must keep a busy page in + * the top object while following the shadow chain. + * + * 4) We must increment paging_in_progress on any object + * for which we have a busy page, to prevent + * vm_object_collapse from removing the busy page + * without our noticing. + */ + + /* + * Search for the page at object/offset. + */ + + object = first_object; + offset = first_offset; + + /* + * See whether this page is resident + */ + + while (TRUE) { + m = vm_page_lookup(object, offset); + if (m != NULL) { + /* + * If the page is being brought in, + * wait for it and then retry. + */ + if (m->flags & PG_BUSY) { +#ifdef DOTHREADS + int wait_result; + + PAGE_ASSERT_WAIT(m, !change_wiring); + UNLOCK_THINGS; + thread_block(); + wait_result = current_thread()->wait_result; + vm_object_deallocate(first_object); + if (wait_result != THREAD_AWAKENED) + return(KERN_SUCCESS); + goto RetryFault; +#else + PAGE_ASSERT_WAIT(m, !change_wiring); + UNLOCK_THINGS; + cnt.v_intrans++; + thread_block(); + vm_object_deallocate(first_object); + goto RetryFault; +#endif + } + + /* + * Remove the page from the pageout daemon's + * reach while we play with it. + */ + + vm_page_lock_queues(); + if (m->flags & PG_INACTIVE) { + TAILQ_REMOVE(&vm_page_queue_inactive, m, pageq); + m->flags &= ~PG_INACTIVE; + cnt.v_inactive_count--; + cnt.v_reactivated++; + } + + if (m->flags & PG_ACTIVE) { + TAILQ_REMOVE(&vm_page_queue_active, m, pageq); + m->flags &= ~PG_ACTIVE; + cnt.v_active_count--; + } + vm_page_unlock_queues(); + + /* + * Mark page busy for other threads. + */ + m->flags |= PG_BUSY; + break; + } + + if (((object->pager != NULL) && + (!change_wiring || wired)) + || (object == first_object)) { + + /* + * Allocate a new page for this object/offset + * pair. + */ + + m = vm_page_alloc(object, offset); + + if (m == NULL) { + UNLOCK_AND_DEALLOCATE; + VM_WAIT; + goto RetryFault; + } + } + + if (object->pager != NULL && (!change_wiring || wired)) { + int rv; + + /* + * Now that we have a busy page, we can + * release the object lock. + */ + vm_object_unlock(object); + + /* + * Call the pager to retrieve the data, if any, + * after releasing the lock on the map. + */ + UNLOCK_MAP; + cnt.v_pageins++; + rv = vm_pager_get(object->pager, m, TRUE); + + /* + * Reaquire the object lock to preserve our + * invariant. + */ + vm_object_lock(object); + + /* + * Found the page. + * Leave it busy while we play with it. + */ + if (rv == VM_PAGER_OK) { + /* + * Relookup in case pager changed page. + * Pager is responsible for disposition + * of old page if moved. + */ + m = vm_page_lookup(object, offset); + + cnt.v_pgpgin++; + m->flags &= ~PG_FAKE; + m->flags |= PG_CLEAN; + pmap_clear_modify(VM_PAGE_TO_PHYS(m)); + break; + } + + /* + * IO error or page outside the range of the pager: + * cleanup and return an error. + */ + if (rv == VM_PAGER_ERROR || rv == VM_PAGER_BAD) { + FREE_PAGE(m); + UNLOCK_AND_DEALLOCATE; + return(KERN_PROTECTION_FAILURE); /* XXX */ + } + /* + * rv == VM_PAGER_FAIL: + * + * Page does not exist at this object/offset. + * Free the bogus page (waking up anyone waiting + * for it) and continue on to the next object. + * + * If this is the top-level object, we must + * leave the busy page to prevent another + * thread from rushing past us, and inserting + * the page in that object at the same time + * that we are. + */ + if (object != first_object) { + FREE_PAGE(m); + /* note that `m' is not used after this */ + } + } + + /* + * We get here if the object has no pager (or unwiring) + * or the pager doesn't have the page. + */ + if (object == first_object) + first_m = m; + + /* + * Move on to the next object. Lock the next + * object before unlocking the current one. + */ + + offset += object->shadow_offset; + next_object = object->shadow; + if (next_object == NULL) { + /* + * If there's no object left, fill the page + * in the top object with zeros. + */ + if (object != first_object) { + object->paging_in_progress--; + vm_object_unlock(object); + + object = first_object; + offset = first_offset; + m = first_m; + vm_object_lock(object); + } + first_m = NULL; + + vm_page_zero_fill(m); + cnt.v_zfod++; + m->flags &= ~PG_FAKE; + break; + } + else { + vm_object_lock(next_object); + if (object != first_object) + object->paging_in_progress--; + vm_object_unlock(object); + object = next_object; + object->paging_in_progress++; + } + } + + if ((m->flags & (PG_ACTIVE | PG_INACTIVE | PG_BUSY)) != PG_BUSY) + panic("vm_fault: active, inactive or !busy after main loop"); + + /* + * PAGE HAS BEEN FOUND. + * [Loop invariant still holds -- the object lock + * is held.] + */ + + old_m = m; /* save page that would be copied */ + + /* + * If the page is being written, but isn't + * already owned by the top-level object, + * we have to copy it into a new page owned + * by the top-level object. + */ + + if (object != first_object) { + /* + * We only really need to copy if we + * want to write it. + */ + + if (fault_type & VM_PROT_WRITE) { + + /* + * If we try to collapse first_object at this + * point, we may deadlock when we try to get + * the lock on an intermediate object (since we + * have the bottom object locked). We can't + * unlock the bottom object, because the page + * we found may move (by collapse) if we do. + * + * Instead, we first copy the page. Then, when + * we have no more use for the bottom object, + * we unlock it and try to collapse. + * + * Note that we copy the page even if we didn't + * need to... that's the breaks. + */ + + /* + * We already have an empty page in + * first_object - use it. + */ + + vm_page_copy(m, first_m); + first_m->flags &= ~PG_FAKE; + + /* + * If another map is truly sharing this + * page with us, we have to flush all + * uses of the original page, since we + * can't distinguish those which want the + * original from those which need the + * new copy. + * + * XXX If we know that only one map has + * access to this page, then we could + * avoid the pmap_page_protect() call. + */ + + vm_page_lock_queues(); + vm_page_deactivate(m); + pmap_page_protect(VM_PAGE_TO_PHYS(m), VM_PROT_NONE); + vm_page_unlock_queues(); + + /* + * We no longer need the old page or object. + */ + PAGE_WAKEUP(m); + object->paging_in_progress--; + vm_object_unlock(object); + + /* + * Only use the new page below... + */ + + cnt.v_cow_faults++; + m = first_m; + object = first_object; + offset = first_offset; + + /* + * Now that we've gotten the copy out of the + * way, let's try to collapse the top object. + */ + vm_object_lock(object); + /* + * But we have to play ugly games with + * paging_in_progress to do that... + */ + object->paging_in_progress--; + vm_object_collapse(object); + object->paging_in_progress++; + } + else { + prot &= ~VM_PROT_WRITE; + m->flags |= PG_COPYONWRITE; + } + } + + if (m->flags & (PG_ACTIVE|PG_INACTIVE)) + panic("vm_fault: active or inactive before copy object handling"); + + /* + * If the page is being written, but hasn't been + * copied to the copy-object, we have to copy it there. + */ + RetryCopy: + if (first_object->copy != NULL) { + vm_object_t copy_object = first_object->copy; + vm_offset_t copy_offset; + vm_page_t copy_m; + + /* + * We only need to copy if we want to write it. + */ + if ((fault_type & VM_PROT_WRITE) == 0) { + prot &= ~VM_PROT_WRITE; + m->flags |= PG_COPYONWRITE; + } + else { + /* + * Try to get the lock on the copy_object. + */ + if (!vm_object_lock_try(copy_object)) { + vm_object_unlock(object); + /* should spin a bit here... */ + vm_object_lock(object); + goto RetryCopy; + } + + /* + * Make another reference to the copy-object, + * to keep it from disappearing during the + * copy. + */ + copy_object->ref_count++; + + /* + * Does the page exist in the copy? + */ + copy_offset = first_offset + - copy_object->shadow_offset; + copy_m = vm_page_lookup(copy_object, copy_offset); + if (page_exists = (copy_m != NULL)) { + if (copy_m->flags & PG_BUSY) { +#ifdef DOTHREADS + int wait_result; + + /* + * If the page is being brought + * in, wait for it and then retry. + */ + PAGE_ASSERT_WAIT(copy_m, !change_wiring); + RELEASE_PAGE(m); + copy_object->ref_count--; + vm_object_unlock(copy_object); + UNLOCK_THINGS; + thread_block(); + wait_result = current_thread()->wait_result; + vm_object_deallocate(first_object); + if (wait_result != THREAD_AWAKENED) + return(KERN_SUCCESS); + goto RetryFault; +#else + /* + * If the page is being brought + * in, wait for it and then retry. + */ + PAGE_ASSERT_WAIT(copy_m, !change_wiring); + RELEASE_PAGE(m); + copy_object->ref_count--; + vm_object_unlock(copy_object); + UNLOCK_THINGS; + thread_block(); + vm_object_deallocate(first_object); + goto RetryFault; +#endif + } + } + + /* + * If the page is not in memory (in the object) + * and the object has a pager, we have to check + * if the pager has the data in secondary + * storage. + */ + if (!page_exists) { + + /* + * If we don't allocate a (blank) page + * here... another thread could try + * to page it in, allocate a page, and + * then block on the busy page in its + * shadow (first_object). Then we'd + * trip over the busy page after we + * found that the copy_object's pager + * doesn't have the page... + */ + copy_m = vm_page_alloc(copy_object, + copy_offset); + if (copy_m == NULL) { + /* + * Wait for a page, then retry. + */ + RELEASE_PAGE(m); + copy_object->ref_count--; + vm_object_unlock(copy_object); + UNLOCK_AND_DEALLOCATE; + VM_WAIT; + goto RetryFault; + } + + if (copy_object->pager != NULL) { + vm_object_unlock(object); + vm_object_unlock(copy_object); + UNLOCK_MAP; + + page_exists = vm_pager_has_page( + copy_object->pager, + (copy_offset + copy_object->paging_offset)); + + vm_object_lock(copy_object); + + /* + * Since the map is unlocked, someone + * else could have copied this object + * and put a different copy_object + * between the two. Or, the last + * reference to the copy-object (other + * than the one we have) may have + * disappeared - if that has happened, + * we don't need to make the copy. + */ + if (copy_object->shadow != object || + copy_object->ref_count == 1) { + /* + * Gaah... start over! + */ + FREE_PAGE(copy_m); + vm_object_unlock(copy_object); + vm_object_deallocate(copy_object); + /* may block */ + vm_object_lock(object); + goto RetryCopy; + } + vm_object_lock(object); + + if (page_exists) { + /* + * We didn't need the page + */ + FREE_PAGE(copy_m); + } + } + } + if (!page_exists) { + /* + * Must copy page into copy-object. + */ + vm_page_copy(m, copy_m); + copy_m->flags &= ~PG_FAKE; + + /* + * Things to remember: + * 1. The copied page must be marked 'dirty' + * so it will be paged out to the copy + * object. + * 2. If the old page was in use by any users + * of the copy-object, it must be removed + * from all pmaps. (We can't know which + * pmaps use it.) + */ + vm_page_lock_queues(); + pmap_page_protect(VM_PAGE_TO_PHYS(old_m), + VM_PROT_NONE); + copy_m->flags &= ~PG_CLEAN; + vm_page_activate(copy_m); /* XXX */ + vm_page_unlock_queues(); + + PAGE_WAKEUP(copy_m); + } + /* + * The reference count on copy_object must be + * at least 2: one for our extra reference, + * and at least one from the outside world + * (we checked that when we last locked + * copy_object). + */ + copy_object->ref_count--; + vm_object_unlock(copy_object); + m->flags &= ~PG_COPYONWRITE; + } + } + + if (m->flags & (PG_ACTIVE | PG_INACTIVE)) + panic("vm_fault: active or inactive before retrying lookup"); + + /* + * We must verify that the maps have not changed + * since our last lookup. + */ + + if (!lookup_still_valid) { + vm_object_t retry_object; + vm_offset_t retry_offset; + vm_prot_t retry_prot; + + /* + * Since map entries may be pageable, make sure we can + * take a page fault on them. + */ + vm_object_unlock(object); + + /* + * To avoid trying to write_lock the map while another + * thread has it read_locked (in vm_map_pageable), we + * do not try for write permission. If the page is + * still writable, we will get write permission. If it + * is not, or has been marked needs_copy, we enter the + * mapping without write permission, and will merely + * take another fault. + */ + result = vm_map_lookup(&map, vaddr, + fault_type & ~VM_PROT_WRITE, &entry, + &retry_object, &retry_offset, &retry_prot, + &wired, &su); + + vm_object_lock(object); + + /* + * If we don't need the page any longer, put it on the + * active list (the easiest thing to do here). If no + * one needs it, pageout will grab it eventually. + */ + + if (result != KERN_SUCCESS) { + RELEASE_PAGE(m); + UNLOCK_AND_DEALLOCATE; + return(result); + } + + lookup_still_valid = TRUE; + + if ((retry_object != first_object) || + (retry_offset != first_offset)) { + RELEASE_PAGE(m); + UNLOCK_AND_DEALLOCATE; + goto RetryFault; + } + + /* + * Check whether the protection has changed or the object + * has been copied while we left the map unlocked. + * Changing from read to write permission is OK - we leave + * the page write-protected, and catch the write fault. + * Changing from write to read permission means that we + * can't mark the page write-enabled after all. + */ + prot &= retry_prot; + if (m->flags & PG_COPYONWRITE) + prot &= ~VM_PROT_WRITE; + } + + /* + * (the various bits we're fiddling with here are locked by + * the object's lock) + */ + + /* XXX This distorts the meaning of the copy_on_write bit */ + + if (prot & VM_PROT_WRITE) + m->flags &= ~PG_COPYONWRITE; + + /* + * It's critically important that a wired-down page be faulted + * only once in each map for which it is wired. + */ + + if (m->flags & (PG_ACTIVE | PG_INACTIVE)) + panic("vm_fault: active or inactive before pmap_enter"); + + vm_object_unlock(object); + + /* + * Put this page into the physical map. + * We had to do the unlock above because pmap_enter + * may cause other faults. We don't put the + * page back on the active queue until later so + * that the page-out daemon won't find us (yet). + */ + + pmap_enter(map->pmap, vaddr, VM_PAGE_TO_PHYS(m), prot, wired); + + /* + * If the page is not wired down, then put it where the + * pageout daemon can find it. + */ + vm_object_lock(object); + vm_page_lock_queues(); + if (change_wiring) { + if (wired) + vm_page_wire(m); + else + vm_page_unwire(m); + } + else + vm_page_activate(m); + vm_page_unlock_queues(); + + /* + * Unlock everything, and return + */ + + PAGE_WAKEUP(m); + UNLOCK_AND_DEALLOCATE; + + return(KERN_SUCCESS); +} + +/* + * vm_fault_wire: + * + * Wire down a range of virtual addresses in a map. + */ +int +vm_fault_wire(map, start, end) + vm_map_t map; + vm_offset_t start, end; +{ + register vm_offset_t va; + register pmap_t pmap; + int rv; + + pmap = vm_map_pmap(map); + + /* + * Inform the physical mapping system that the + * range of addresses may not fault, so that + * page tables and such can be locked down as well. + */ + + pmap_pageable(pmap, start, end, FALSE); + + /* + * We simulate a fault to get the page and enter it + * in the physical map. + */ + + for (va = start; va < end; va += PAGE_SIZE) { + rv = vm_fault(map, va, VM_PROT_NONE, TRUE); + if (rv) { + if (va != start) + vm_fault_unwire(map, start, va); + return(rv); + } + } + return(KERN_SUCCESS); +} + + +/* + * vm_fault_unwire: + * + * Unwire a range of virtual addresses in a map. + */ +void +vm_fault_unwire(map, start, end) + vm_map_t map; + vm_offset_t start, end; +{ + + register vm_offset_t va, pa; + register pmap_t pmap; + + pmap = vm_map_pmap(map); + + /* + * Since the pages are wired down, we must be able to + * get their mappings from the physical map system. + */ + + vm_page_lock_queues(); + + for (va = start; va < end; va += PAGE_SIZE) { + pa = pmap_extract(pmap, va); + if (pa == (vm_offset_t) 0) { + panic("unwire: page not in pmap"); + } + pmap_change_wiring(pmap, va, FALSE); + vm_page_unwire(PHYS_TO_VM_PAGE(pa)); + } + vm_page_unlock_queues(); + + /* + * Inform the physical mapping system that the range + * of addresses may fault, so that page tables and + * such may be unwired themselves. + */ + + pmap_pageable(pmap, start, end, TRUE); + +} + +/* + * Routine: + * vm_fault_copy_entry + * Function: + * Copy all of the pages from a wired-down map entry to another. + * + * In/out conditions: + * The source and destination maps must be locked for write. + * The source map entry must be wired down (or be a sharing map + * entry corresponding to a main map entry that is wired down). + */ +void +vm_fault_copy_entry(dst_map, src_map, dst_entry, src_entry) + vm_map_t dst_map; + vm_map_t src_map; + vm_map_entry_t dst_entry; + vm_map_entry_t src_entry; +{ + + vm_object_t dst_object; + vm_object_t src_object; + vm_offset_t dst_offset; + vm_offset_t src_offset; + vm_prot_t prot; + vm_offset_t vaddr; + vm_page_t dst_m; + vm_page_t src_m; + +#ifdef lint + src_map++; +#endif + + src_object = src_entry->object.vm_object; + src_offset = src_entry->offset; + + /* + * Create the top-level object for the destination entry. + * (Doesn't actually shadow anything - we copy the pages + * directly.) + */ + dst_object = vm_object_allocate( + (vm_size_t) (dst_entry->end - dst_entry->start)); + + dst_entry->object.vm_object = dst_object; + dst_entry->offset = 0; + + prot = dst_entry->max_protection; + + /* + * Loop through all of the pages in the entry's range, copying + * each one from the source object (it should be there) to the + * destination object. + */ + for (vaddr = dst_entry->start, dst_offset = 0; + vaddr < dst_entry->end; + vaddr += PAGE_SIZE, dst_offset += PAGE_SIZE) { + + /* + * Allocate a page in the destination object + */ + vm_object_lock(dst_object); + do { + dst_m = vm_page_alloc(dst_object, dst_offset); + if (dst_m == NULL) { + vm_object_unlock(dst_object); + VM_WAIT; + vm_object_lock(dst_object); + } + } while (dst_m == NULL); + + /* + * Find the page in the source object, and copy it in. + * (Because the source is wired down, the page will be + * in memory.) + */ + vm_object_lock(src_object); + src_m = vm_page_lookup(src_object, dst_offset + src_offset); + if (src_m == NULL) + panic("vm_fault_copy_wired: page missing"); + + vm_page_copy(src_m, dst_m); + + /* + * Enter it in the pmap... + */ + vm_object_unlock(src_object); + vm_object_unlock(dst_object); + + pmap_enter(dst_map->pmap, vaddr, VM_PAGE_TO_PHYS(dst_m), + prot, FALSE); + + /* + * Mark it no longer busy, and put it on the active list. + */ + vm_object_lock(dst_object); + vm_page_lock_queues(); + vm_page_activate(dst_m); + vm_page_unlock_queues(); + PAGE_WAKEUP(dst_m); + vm_object_unlock(dst_object); + } + +} diff --git a/sys/vm/vm_glue.c b/sys/vm/vm_glue.c new file mode 100644 index 00000000000..d5c22223ad2 --- /dev/null +++ b/sys/vm/vm_glue.c @@ -0,0 +1,611 @@ +/* $NetBSD: vm_glue.c,v 1.47 1995/08/13 09:04:47 mycroft Exp $ */ + +/* + * Copyright (c) 1991, 1993 + * The Regents of the University of California. All rights reserved. + * + * This code is derived from software contributed to Berkeley by + * The Mach Operating System project at Carnegie-Mellon University. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. All advertising materials mentioning features or use of this software + * must display the following acknowledgement: + * This product includes software developed by the University of + * California, Berkeley and its contributors. + * 4. Neither the name of the University nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * @(#)vm_glue.c 8.6 (Berkeley) 1/5/94 + * + * + * Copyright (c) 1987, 1990 Carnegie-Mellon University. + * All rights reserved. + * + * Permission to use, copy, modify and distribute this software and + * its documentation is hereby granted, provided that both the copyright + * notice and this permission notice appear in all copies of the + * software, derivative works or modified versions, and any portions + * thereof, and that both notices appear in supporting documentation. + * + * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" + * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND + * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. + * + * Carnegie Mellon requests users of this software to return to + * + * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU + * School of Computer Science + * Carnegie Mellon University + * Pittsburgh PA 15213-3890 + * + * any improvements or extensions that they make and grant Carnegie the + * rights to redistribute these changes. + */ + +#include <sys/param.h> +#include <sys/systm.h> +#include <sys/proc.h> +#include <sys/resourcevar.h> +#include <sys/buf.h> +#include <sys/user.h> + +#include <vm/vm.h> +#include <vm/vm_page.h> +#include <vm/vm_kern.h> + +#include <machine/cpu.h> + +int avefree = 0; /* XXX */ +unsigned maxdmap = MAXDSIZ; /* XXX */ +unsigned maxsmap = MAXSSIZ; /* XXX */ +int readbuffers = 0; /* XXX allow kgdb to read kernel buffer pool */ + +int +kernacc(addr, len, rw) + caddr_t addr; + int len, rw; +{ + boolean_t rv; + vm_offset_t saddr, eaddr; + vm_prot_t prot = rw == B_READ ? VM_PROT_READ : VM_PROT_WRITE; + + saddr = trunc_page(addr); + eaddr = round_page(addr+len); + rv = vm_map_check_protection(kernel_map, saddr, eaddr, prot); + /* + * XXX there are still some things (e.g. the buffer cache) that + * are managed behind the VM system's back so even though an + * address is accessible in the mind of the VM system, there may + * not be physical pages where the VM thinks there is. This can + * lead to bogus allocation of pages in the kernel address space + * or worse, inconsistencies at the pmap level. We only worry + * about the buffer cache for now. + */ + if (!readbuffers && rv && (eaddr > (vm_offset_t)buffers && + saddr < (vm_offset_t)buffers + MAXBSIZE * nbuf)) + rv = FALSE; + return(rv == TRUE); +} + +int +useracc(addr, len, rw) + caddr_t addr; + int len, rw; +{ + boolean_t rv; + vm_prot_t prot = rw == B_READ ? VM_PROT_READ : VM_PROT_WRITE; + +#if defined(i386) || defined(pc532) + /* + * XXX - specially disallow access to user page tables - they are + * in the map. This is here until i386 & pc532 pmaps are fixed... + */ + if ((vm_offset_t) addr >= VM_MAXUSER_ADDRESS + || (vm_offset_t) addr + len > VM_MAXUSER_ADDRESS + || (vm_offset_t) addr + len <= (vm_offset_t) addr) + return (FALSE); +#endif + + rv = vm_map_check_protection(&curproc->p_vmspace->vm_map, + trunc_page(addr), round_page(addr+len), prot); + return(rv == TRUE); +} + +#ifdef KGDB +/* + * Change protections on kernel pages from addr to addr+len + * (presumably so debugger can plant a breakpoint). + * + * We force the protection change at the pmap level. If we were + * to use vm_map_protect a change to allow writing would be lazily- + * applied meaning we would still take a protection fault, something + * we really don't want to do. It would also fragment the kernel + * map unnecessarily. We cannot use pmap_protect since it also won't + * enforce a write-enable request. Using pmap_enter is the only way + * we can ensure the change takes place properly. + */ +void +chgkprot(addr, len, rw) + register caddr_t addr; + int len, rw; +{ + vm_prot_t prot; + vm_offset_t pa, sva, eva; + + prot = rw == B_READ ? VM_PROT_READ : VM_PROT_READ|VM_PROT_WRITE; + eva = round_page(addr + len); + for (sva = trunc_page(addr); sva < eva; sva += PAGE_SIZE) { + /* + * Extract physical address for the page. + * We use a cheezy hack to differentiate physical + * page 0 from an invalid mapping, not that it + * really matters... + */ + pa = pmap_extract(pmap_kernel(), sva|1); + if (pa == 0) + panic("chgkprot: invalid page"); + pmap_enter(pmap_kernel(), sva, pa&~1, prot, TRUE); + } +} +#endif + +void +vslock(addr, len) + caddr_t addr; + u_int len; +{ + vm_map_pageable(&curproc->p_vmspace->vm_map, trunc_page(addr), + round_page(addr+len), FALSE); +} + +void +vsunlock(addr, len, dirtied) + caddr_t addr; + u_int len; + int dirtied; +{ +#ifdef lint + dirtied++; +#endif + vm_map_pageable(&curproc->p_vmspace->vm_map, trunc_page(addr), + round_page(addr+len), TRUE); +} + +/* + * Implement fork's actions on an address space. + * Here we arrange for the address space to be copied or referenced, + * allocate a user struct (pcb and kernel stack), then call the + * machine-dependent layer to fill those in and make the new process + * ready to run. + * NOTE: the kernel stack may be at a different location in the child + * process, and thus addresses of automatic variables may be invalid + * after cpu_fork returns in the child process. We do nothing here + * after cpu_fork returns. + */ +int +vm_fork(p1, p2, isvfork) + register struct proc *p1, *p2; + int isvfork; +{ + register struct user *up; + vm_offset_t addr; + +#if defined(i386) || defined(pc532) + /* + * avoid copying any of the parent's pagetables or other per-process + * objects that reside in the map by marking all of them non-inheritable + */ + (void)vm_map_inherit(&p1->p_vmspace->vm_map, + VM_MAXUSER_ADDRESS, VM_MAX_ADDRESS, VM_INHERIT_NONE); +#endif + p2->p_vmspace = vmspace_fork(p1->p_vmspace); + +#ifdef SYSVSHM + if (p1->p_vmspace->vm_shm) + shmfork(p1, p2, isvfork); +#endif + +#if !defined(i386) && !defined(pc532) + /* + * Allocate a wired-down (for now) pcb and kernel stack for the process + */ + addr = kmem_alloc_pageable(kernel_map, USPACE); + if (addr == 0) + panic("vm_fork: no more kernel virtual memory"); + vm_map_pageable(kernel_map, addr, addr + USPACE, FALSE); +#else + /* + * XXX somehow, on 386, ocassionally pageout removes active, wired down + * kstack and pagetables, WITHOUT going thru vm_page_unwire! Why this + * appears to work is not yet clear, yet it does... + */ + addr = kmem_alloc(kernel_map, USPACE); + if (addr == 0) + panic("vm_fork: no more kernel virtual memory"); +#endif + up = (struct user *)addr; + p2->p_addr = up; + + /* + * p_stats and p_sigacts currently point at fields + * in the user struct but not at &u, instead at p_addr. + * Copy p_sigacts and parts of p_stats; zero the rest + * of p_stats (statistics). + */ + p2->p_stats = &up->u_stats; + p2->p_sigacts = &up->u_sigacts; + up->u_sigacts = *p1->p_sigacts; + bzero(&up->u_stats.pstat_startzero, + (unsigned) ((caddr_t)&up->u_stats.pstat_endzero - + (caddr_t)&up->u_stats.pstat_startzero)); + bcopy(&p1->p_stats->pstat_startcopy, &up->u_stats.pstat_startcopy, + ((caddr_t)&up->u_stats.pstat_endcopy - + (caddr_t)&up->u_stats.pstat_startcopy)); + +#if defined(i386) || defined(pc532) + { vm_offset_t addr = VM_MAXUSER_ADDRESS; struct vm_map *vp; + + /* ream out old pagetables and kernel stack */ + vp = &p2->p_vmspace->vm_map; + (void)vm_deallocate(vp, addr, VM_MAX_ADDRESS - addr); + (void)vm_allocate(vp, &addr, VM_MAX_ADDRESS - addr, FALSE); + (void)vm_map_inherit(vp, addr, VM_MAX_ADDRESS, VM_INHERIT_NONE); + } +#endif + /* + * cpu_fork will copy and update the kernel stack and pcb, + * and make the child ready to run. It marks the child + * so that it can return differently than the parent. + * It returns twice, once in the parent process and + * once in the child. + */ + return (cpu_fork(p1, p2)); +} + +/* + * Set default limits for VM system. + * Called for proc 0, and then inherited by all others. + */ +void +vm_init_limits(p) + register struct proc *p; +{ + + /* + * Set up the initial limits on process VM. + * Set the maximum resident set size to be all + * of (reasonably) available memory. This causes + * any single, large process to start random page + * replacement once it fills memory. + */ + p->p_rlimit[RLIMIT_STACK].rlim_cur = DFLSSIZ; + p->p_rlimit[RLIMIT_STACK].rlim_max = MAXSSIZ; + p->p_rlimit[RLIMIT_DATA].rlim_cur = DFLDSIZ; + p->p_rlimit[RLIMIT_DATA].rlim_max = MAXDSIZ; + p->p_rlimit[RLIMIT_RSS].rlim_cur = ptoa(cnt.v_free_count); +} + +#include <vm/vm_pageout.h> + +#ifdef DEBUG +int enableswap = 1; +int swapdebug = 0; +#define SDB_FOLLOW 1 +#define SDB_SWAPIN 2 +#define SDB_SWAPOUT 4 +#endif + +/* + * Swap in a process's u-area. + */ +void +swapin(p) + struct proc *p; +{ + vm_offset_t addr; + int s; + + addr = (vm_offset_t)p->p_addr; + vm_map_pageable(kernel_map, addr, addr + USPACE, FALSE); + /* + * Some architectures need to be notified when the + * user area has moved to new physical page(s) (e.g. + * see pmax/pmax/vm_machdep.c). + */ + cpu_swapin(p); + s = splstatclock(); + if (p->p_stat == SRUN) + setrunqueue(p); + p->p_flag |= P_INMEM; + splx(s); + p->p_swtime = 0; +} + +/* + * Brutally simple: + * 1. Attempt to swapin every swaped-out, runnable process in + * order of priority. + * 2. If not enough memory, wake the pageout daemon and let it + * clear some space. + */ +void +scheduler() +{ + register struct proc *p; + register int pri; + struct proc *pp; + int ppri; + +loop: +#ifdef DEBUG + while (!enableswap) + tsleep((caddr_t)&proc0, PVM, "noswap", 0); +#endif + pp = NULL; + ppri = INT_MIN; + for (p = allproc.lh_first; p != 0; p = p->p_list.le_next) { + if (p->p_stat == SRUN && (p->p_flag & P_INMEM) == 0) { + pri = p->p_swtime + p->p_slptime - p->p_nice * 8; + if (pri > ppri) { + pp = p; + ppri = pri; + } + } + } +#ifdef DEBUG + if (swapdebug & SDB_FOLLOW) + printf("scheduler: running, procp %x pri %d\n", pp, ppri); +#endif + /* + * Nothing to do, back to sleep + */ + if ((p = pp) == NULL) { + tsleep((caddr_t)&proc0, PVM, "scheduler", 0); + goto loop; + } + + /* + * We would like to bring someone in. + * This part is really bogus cuz we could deadlock on memory + * despite our feeble check. + */ + if (cnt.v_free_count > atop(USPACE)) { +#ifdef DEBUG + if (swapdebug & SDB_SWAPIN) + printf("swapin: pid %d(%s)@%x, pri %d free %d\n", + p->p_pid, p->p_comm, p->p_addr, + ppri, cnt.v_free_count); +#endif + swapin(p); + goto loop; + } + /* + * Not enough memory, jab the pageout daemon and wait til the + * coast is clear. + */ +#ifdef DEBUG + if (swapdebug & SDB_FOLLOW) + printf("scheduler: no room for pid %d(%s), free %d\n", + p->p_pid, p->p_comm, cnt.v_free_count); +#endif + (void) splhigh(); + VM_WAIT; + (void) spl0(); +#ifdef DEBUG + if (swapdebug & SDB_FOLLOW) + printf("scheduler: room again, free %d\n", cnt.v_free_count); +#endif + goto loop; +} + +#define swappable(p) \ + (((p)->p_flag & (P_SYSTEM | P_INMEM | P_WEXIT)) == P_INMEM && \ + (p)->p_holdcnt == 0) + +/* + * Swapout is driven by the pageout daemon. Very simple, we find eligible + * procs and unwire their u-areas. We try to always "swap" at least one + * process in case we need the room for a swapin. + * If any procs have been sleeping/stopped for at least maxslp seconds, + * they are swapped. Else, we swap the longest-sleeping or stopped process, + * if any, otherwise the longest-resident process. + */ +void +swapout_threads() +{ + register struct proc *p; + struct proc *outp, *outp2; + int outpri, outpri2; + int didswap = 0; + extern int maxslp; + +#ifdef DEBUG + if (!enableswap) + return; +#endif + outp = outp2 = NULL; + outpri = outpri2 = 0; + for (p = allproc.lh_first; p != 0; p = p->p_list.le_next) { + if (!swappable(p)) + continue; + switch (p->p_stat) { + case SRUN: + if (p->p_swtime > outpri2) { + outp2 = p; + outpri2 = p->p_swtime; + } + continue; + + case SSLEEP: + case SSTOP: + if (p->p_slptime >= maxslp) { + swapout(p); + didswap++; + } else if (p->p_slptime > outpri) { + outp = p; + outpri = p->p_slptime; + } + continue; + } + } + /* + * If we didn't get rid of any real duds, toss out the next most + * likely sleeping/stopped or running candidate. We only do this + * if we are real low on memory since we don't gain much by doing + * it (USPACE bytes). + */ + if (didswap == 0 && + cnt.v_free_count <= atop(round_page(USPACE))) { + if ((p = outp) == 0) + p = outp2; +#ifdef DEBUG + if (swapdebug & SDB_SWAPOUT) + printf("swapout_threads: no duds, try procp %x\n", p); +#endif + if (p) + swapout(p); + } +} + +void +swapout(p) + register struct proc *p; +{ + vm_offset_t addr; + vm_size_t size; + +#ifdef DEBUG + if (swapdebug & SDB_SWAPOUT) + printf("swapout: pid %d(%s)@%x, stat %x pri %d free %d\n", + p->p_pid, p->p_comm, p->p_addr, p->p_stat, + p->p_slptime, cnt.v_free_count); +#endif + + /* + * Do any machine-specific actions necessary before swapout. + * This can include saving floating point state, etc. + */ + cpu_swapout(p); + + /* + * Unwire the to-be-swapped process's user struct and kernel stack. + */ + addr = (vm_offset_t) p->p_addr; + size = round_page(USPACE); + vm_map_pageable(kernel_map, addr, addr+size, TRUE); + pmap_collect(vm_map_pmap(&p->p_vmspace->vm_map)); + + /* + * Mark it as (potentially) swapped out. + */ + (void) splhigh(); + p->p_flag &= ~P_INMEM; + if (p->p_stat == SRUN) + remrq(p); + (void) spl0(); + p->p_swtime = 0; +} + +/* + * The rest of these routines fake thread handling + */ + +void +assert_wait(event, ruptible) + void *event; + boolean_t ruptible; +{ +#ifdef lint + ruptible++; +#endif + curproc->p_thread = event; +} + +void +thread_block() +{ + int s = splhigh(); + + if (curproc->p_thread) + tsleep(curproc->p_thread, PVM, "thrd_block", 0); + splx(s); +} + +void +thread_sleep(event, lock, ruptible) + void *event; + simple_lock_t lock; + boolean_t ruptible; +{ + int s = splhigh(); + +#ifdef lint + ruptible++; +#endif + curproc->p_thread = event; + simple_unlock(lock); + if (curproc->p_thread) + tsleep(event, PVM, "thrd_sleep", 0); + splx(s); +} + +void +thread_wakeup(event) + void *event; +{ + int s = splhigh(); + + wakeup(event); + splx(s); +} + +/* + * DEBUG stuff + */ + +int indent = 0; + +#include <machine/stdarg.h> /* see subr_prf.c */ + +/*ARGSUSED2*/ +void +#if __STDC__ +iprintf(void (*pr)(const char *, ...), const char *fmt, ...) +#else +iprintf(pr, fmt /* , va_alist */) + void (*pr)(); + char *fmt; + /* va_dcl */ +#endif +{ + register int i; + va_list ap; + + for (i = indent; i >= 8; i -= 8) + (*pr)("\t"); + while (--i >= 0) + (*pr)(" "); + va_start(ap, fmt); + (*pr)("%r", fmt, ap); + va_end(ap); +} diff --git a/sys/vm/vm_inherit.h b/sys/vm/vm_inherit.h new file mode 100644 index 00000000000..23f21699763 --- /dev/null +++ b/sys/vm/vm_inherit.h @@ -0,0 +1,85 @@ +/* $NetBSD: vm_inherit.h,v 1.7 1994/06/29 06:47:58 cgd Exp $ */ + +/* + * Copyright (c) 1991, 1993 + * The Regents of the University of California. All rights reserved. + * + * This code is derived from software contributed to Berkeley by + * The Mach Operating System project at Carnegie-Mellon University. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. All advertising materials mentioning features or use of this software + * must display the following acknowledgement: + * This product includes software developed by the University of + * California, Berkeley and its contributors. + * 4. Neither the name of the University nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * @(#)vm_inherit.h 8.1 (Berkeley) 6/11/93 + * + * + * Copyright (c) 1987, 1990 Carnegie-Mellon University. + * All rights reserved. + * + * Authors: Avadis Tevanian, Jr., Michael Wayne Young + * + * Permission to use, copy, modify and distribute this software and + * its documentation is hereby granted, provided that both the copyright + * notice and this permission notice appear in all copies of the + * software, derivative works or modified versions, and any portions + * thereof, and that both notices appear in supporting documentation. + * + * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" + * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND + * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. + * + * Carnegie Mellon requests users of this software to return to + * + * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU + * School of Computer Science + * Carnegie Mellon University + * Pittsburgh PA 15213-3890 + * + * any improvements or extensions that they make and grant Carnegie the + * rights to redistribute these changes. + */ + +/* + * Virtual memory map inheritance definitions. + */ + +#ifndef _VM_INHERIT_ +#define _VM_INHERIT_ + +/* + * Enumeration of valid values for vm_inherit_t. + */ + +#define VM_INHERIT_SHARE ((vm_inherit_t) 0) /* share with child */ +#define VM_INHERIT_COPY ((vm_inherit_t) 1) /* copy into child */ +#define VM_INHERIT_NONE ((vm_inherit_t) 2) /* absent from child */ +#define VM_INHERIT_DONATE_COPY ((vm_inherit_t) 3) /* copy and delete */ + +#define VM_INHERIT_DEFAULT VM_INHERIT_COPY + +#endif /* _VM_INHERIT_ */ diff --git a/sys/vm/vm_init.c b/sys/vm/vm_init.c new file mode 100644 index 00000000000..14a539cee37 --- /dev/null +++ b/sys/vm/vm_init.c @@ -0,0 +1,122 @@ +/* $NetBSD: vm_init.c,v 1.9 1994/06/29 06:48:00 cgd Exp $ */ + +/* + * Copyright (c) 1991, 1993 + * The Regents of the University of California. All rights reserved. + * + * This code is derived from software contributed to Berkeley by + * The Mach Operating System project at Carnegie-Mellon University. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. All advertising materials mentioning features or use of this software + * must display the following acknowledgement: + * This product includes software developed by the University of + * California, Berkeley and its contributors. + * 4. Neither the name of the University nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * @(#)vm_init.c 8.1 (Berkeley) 6/11/93 + * + * + * Copyright (c) 1987, 1990 Carnegie-Mellon University. + * All rights reserved. + * + * Authors: Avadis Tevanian, Jr., Michael Wayne Young + * + * Permission to use, copy, modify and distribute this software and + * its documentation is hereby granted, provided that both the copyright + * notice and this permission notice appear in all copies of the + * software, derivative works or modified versions, and any portions + * thereof, and that both notices appear in supporting documentation. + * + * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" + * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND + * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. + * + * Carnegie Mellon requests users of this software to return to + * + * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU + * School of Computer Science + * Carnegie Mellon University + * Pittsburgh PA 15213-3890 + * + * any improvements or extensions that they make and grant Carnegie the + * rights to redistribute these changes. + */ + +/* + * Initialize the Virtual Memory subsystem. + */ + +#include <sys/param.h> + +#include <vm/vm.h> +#include <vm/vm_page.h> +#include <vm/vm_kern.h> + +/* + * vm_init initializes the virtual memory system. + * This is done only by the first cpu up. + * + * The start and end address of physical memory is passed in. + */ + +void vm_mem_init() +{ +#ifndef MACHINE_NONCONTIG + extern vm_offset_t avail_start, avail_end; + extern vm_offset_t virtual_avail, virtual_end; +#else + vm_offset_t start, end; +#endif + + /* + * Initializes resident memory structures. + * From here on, all physical memory is accounted for, + * and we use only virtual addresses. + */ + vm_set_page_size(); +#ifndef MACHINE_NONCONTIG + vm_page_startup(&avail_start, &avail_end); +#else + vm_page_bootstrap(&start, &end); +#endif + + /* + * Initialize other VM packages + */ +#ifndef MACHINE_NONCONTIG + vm_object_init(virtual_end - VM_MIN_KERNEL_ADDRESS); +#else + vm_object_init(end - VM_MIN_KERNEL_ADDRESS); +#endif + vm_map_startup(); +#ifndef MACHINE_NONCONTIG + kmem_init(virtual_avail, virtual_end); + pmap_init(avail_start, avail_end); +#else + kmem_init(start, end); + pmap_init(); +#endif + vm_pager_init(); +} diff --git a/sys/vm/vm_kern.c b/sys/vm/vm_kern.c new file mode 100644 index 00000000000..5b578fa3238 --- /dev/null +++ b/sys/vm/vm_kern.c @@ -0,0 +1,458 @@ +/* $NetBSD: vm_kern.c,v 1.17 1995/04/10 16:53:55 mycroft Exp $ */ + +/* + * Copyright (c) 1991, 1993 + * The Regents of the University of California. All rights reserved. + * + * This code is derived from software contributed to Berkeley by + * The Mach Operating System project at Carnegie-Mellon University. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. All advertising materials mentioning features or use of this software + * must display the following acknowledgement: + * This product includes software developed by the University of + * California, Berkeley and its contributors. + * 4. Neither the name of the University nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * @(#)vm_kern.c 8.3 (Berkeley) 1/12/94 + * + * + * Copyright (c) 1987, 1990 Carnegie-Mellon University. + * All rights reserved. + * + * Authors: Avadis Tevanian, Jr., Michael Wayne Young + * + * Permission to use, copy, modify and distribute this software and + * its documentation is hereby granted, provided that both the copyright + * notice and this permission notice appear in all copies of the + * software, derivative works or modified versions, and any portions + * thereof, and that both notices appear in supporting documentation. + * + * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" + * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND + * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. + * + * Carnegie Mellon requests users of this software to return to + * + * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU + * School of Computer Science + * Carnegie Mellon University + * Pittsburgh PA 15213-3890 + * + * any improvements or extensions that they make and grant Carnegie the + * rights to redistribute these changes. + */ + +/* + * Kernel memory management. + */ + +#include <sys/param.h> +#include <sys/systm.h> + +#include <vm/vm.h> +#include <vm/vm_page.h> +#include <vm/vm_pageout.h> +#include <vm/vm_kern.h> + +/* + * kmem_alloc_pageable: + * + * Allocate pageable memory to the kernel's address map. + * map must be "kernel_map" below. + */ +vm_offset_t +kmem_alloc_pageable(map, size) + vm_map_t map; + register vm_size_t size; +{ + vm_offset_t addr; + register int result; + +#if 0 + if (map != kernel_map) + panic("kmem_alloc_pageable: not called with kernel_map"); +#endif + + size = round_page(size); + + addr = vm_map_min(map); + result = vm_map_find(map, NULL, (vm_offset_t) 0, + &addr, size, TRUE); + if (result != KERN_SUCCESS) { + return(0); + } + + return(addr); +} + +/* + * Allocate wired-down memory in the kernel's address map + * or a submap. + */ +vm_offset_t +kmem_alloc(map, size) + register vm_map_t map; + register vm_size_t size; +{ + vm_offset_t addr; + register vm_offset_t offset; + extern vm_object_t kernel_object; + vm_offset_t i; + + size = round_page(size); + + /* + * Use the kernel object for wired-down kernel pages. + * Assume that no region of the kernel object is + * referenced more than once. + */ + + /* + * Locate sufficient space in the map. This will give us the + * final virtual address for the new memory, and thus will tell + * us the offset within the kernel map. + */ + vm_map_lock(map); + if (vm_map_findspace(map, 0, size, &addr)) { + vm_map_unlock(map); + return (0); + } + offset = addr - VM_MIN_KERNEL_ADDRESS; + vm_object_reference(kernel_object); + vm_map_insert(map, kernel_object, offset, addr, addr + size); + vm_map_unlock(map); + + /* + * Guarantee that there are pages already in this object + * before calling vm_map_pageable. This is to prevent the + * following scenario: + * + * 1) Threads have swapped out, so that there is a + * pager for the kernel_object. + * 2) The kmsg zone is empty, and so we are kmem_allocing + * a new page for it. + * 3) vm_map_pageable calls vm_fault; there is no page, + * but there is a pager, so we call + * pager_data_request. But the kmsg zone is empty, + * so we must kmem_alloc. + * 4) goto 1 + * 5) Even if the kmsg zone is not empty: when we get + * the data back from the pager, it will be (very + * stale) non-zero data. kmem_alloc is defined to + * return zero-filled memory. + * + * We're intentionally not activating the pages we allocate + * to prevent a race with page-out. vm_map_pageable will wire + * the pages. + */ + + vm_object_lock(kernel_object); + for (i = 0 ; i < size; i+= PAGE_SIZE) { + vm_page_t mem; + + while ((mem = vm_page_alloc(kernel_object, offset+i)) == NULL) { + vm_object_unlock(kernel_object); + VM_WAIT; + vm_object_lock(kernel_object); + } + vm_page_zero_fill(mem); + mem->flags &= ~PG_BUSY; + } + vm_object_unlock(kernel_object); + + /* + * And finally, mark the data as non-pageable. + */ + + (void) vm_map_pageable(map, (vm_offset_t) addr, addr + size, FALSE); + + /* + * Try to coalesce the map + */ + + vm_map_simplify(map, addr); + + return(addr); +} + +/* + * kmem_free: + * + * Release a region of kernel virtual memory allocated + * with kmem_alloc, and return the physical pages + * associated with that region. + */ +void +kmem_free(map, addr, size) + vm_map_t map; + register vm_offset_t addr; + vm_size_t size; +{ + (void) vm_map_remove(map, trunc_page(addr), round_page(addr + size)); +} + +/* + * kmem_suballoc: + * + * Allocates a map to manage a subrange + * of the kernel virtual address space. + * + * Arguments are as follows: + * + * parent Map to take range from + * size Size of range to find + * min, max Returned endpoints of map + * pageable Can the region be paged + */ +vm_map_t +kmem_suballoc(parent, min, max, size, pageable) + register vm_map_t parent; + vm_offset_t *min, *max; + register vm_size_t size; + boolean_t pageable; +{ + register int ret; + vm_map_t result; + + size = round_page(size); + + *min = (vm_offset_t) vm_map_min(parent); + ret = vm_map_find(parent, NULL, (vm_offset_t) 0, + min, size, TRUE); + if (ret != KERN_SUCCESS) { + printf("kmem_suballoc: bad status return of %d.\n", ret); + panic("kmem_suballoc"); + } + *max = *min + size; + pmap_reference(vm_map_pmap(parent)); + result = vm_map_create(vm_map_pmap(parent), *min, *max, pageable); + if (result == NULL) + panic("kmem_suballoc: cannot create submap"); + if ((ret = vm_map_submap(parent, *min, *max, result)) != KERN_SUCCESS) + panic("kmem_suballoc: unable to change range to submap"); + return(result); +} + +/* + * Allocate wired-down memory in the kernel's address map for the higher + * level kernel memory allocator (kern/kern_malloc.c). We cannot use + * kmem_alloc() because we may need to allocate memory at interrupt + * level where we cannot block (canwait == FALSE). + * + * This routine has its own private kernel submap (kmem_map) and object + * (kmem_object). This, combined with the fact that only malloc uses + * this routine, ensures that we will never block in map or object waits. + * + * Note that this still only works in a uni-processor environment and + * when called at splhigh(). + * + * We don't worry about expanding the map (adding entries) since entries + * for wired maps are statically allocated. + */ +vm_offset_t +kmem_malloc(map, size, canwait) + register vm_map_t map; + register vm_size_t size; + boolean_t canwait; +{ + register vm_offset_t offset, i; + vm_map_entry_t entry; + vm_offset_t addr; + vm_page_t m; + extern vm_object_t kmem_object; + + if (map != kmem_map && map != mb_map) + panic("kern_malloc_alloc: map != {kmem,mb}_map"); + + size = round_page(size); + addr = vm_map_min(map); + + /* + * Locate sufficient space in the map. This will give us the + * final virtual address for the new memory, and thus will tell + * us the offset within the kernel map. + */ + vm_map_lock(map); + if (vm_map_findspace(map, 0, size, &addr)) { + vm_map_unlock(map); + if (canwait) /* XXX should wait */ + panic("kmem_malloc: %s too small", + map == kmem_map ? "kmem_map" : "mb_map"); + return (0); + } + offset = addr - vm_map_min(kmem_map); + vm_object_reference(kmem_object); + vm_map_insert(map, kmem_object, offset, addr, addr + size); + + /* + * If we can wait, just mark the range as wired + * (will fault pages as necessary). + */ + if (canwait) { + vm_map_unlock(map); + (void) vm_map_pageable(map, (vm_offset_t) addr, addr + size, + FALSE); + vm_map_simplify(map, addr); + return(addr); + } + + /* + * If we cannot wait then we must allocate all memory up front, + * pulling it off the active queue to prevent pageout. + */ + vm_object_lock(kmem_object); + for (i = 0; i < size; i += PAGE_SIZE) { + m = vm_page_alloc(kmem_object, offset + i); + + /* + * Ran out of space, free everything up and return. + * Don't need to lock page queues here as we know + * that the pages we got aren't on any queues. + */ + if (m == NULL) { + while (i != 0) { + i -= PAGE_SIZE; + m = vm_page_lookup(kmem_object, offset + i); + vm_page_free(m); + } + vm_object_unlock(kmem_object); + vm_map_delete(map, addr, addr + size); + vm_map_unlock(map); + return(0); + } +#if 0 + vm_page_zero_fill(m); +#endif + m->flags &= ~PG_BUSY; + } + vm_object_unlock(kmem_object); + + /* + * Mark map entry as non-pageable. + * Assert: vm_map_insert() will never be able to extend the previous + * entry so there will be a new entry exactly corresponding to this + * address range and it will have wired_count == 0. + */ + if (!vm_map_lookup_entry(map, addr, &entry) || + entry->start != addr || entry->end != addr + size || + entry->wired_count) + panic("kmem_malloc: entry not found or misaligned"); + entry->wired_count++; + + /* + * Loop thru pages, entering them in the pmap. + * (We cannot add them to the wired count without + * wrapping the vm_page_queue_lock in splimp...) + */ + for (i = 0; i < size; i += PAGE_SIZE) { + vm_object_lock(kmem_object); + m = vm_page_lookup(kmem_object, offset + i); + vm_object_unlock(kmem_object); + pmap_enter(map->pmap, addr + i, VM_PAGE_TO_PHYS(m), + VM_PROT_DEFAULT, TRUE); + } + vm_map_unlock(map); + + vm_map_simplify(map, addr); + return(addr); +} + +/* + * kmem_alloc_wait + * + * Allocates pageable memory from a sub-map of the kernel. If the submap + * has no room, the caller sleeps waiting for more memory in the submap. + * + */ +vm_offset_t +kmem_alloc_wait(map, size) + vm_map_t map; + vm_size_t size; +{ + vm_offset_t addr; + + size = round_page(size); + + for (;;) { + /* + * To make this work for more than one map, + * use the map's lock to lock out sleepers/wakers. + */ + vm_map_lock(map); + if (vm_map_findspace(map, 0, size, &addr) == 0) + break; + /* no space now; see if we can ever get space */ + if (vm_map_max(map) - vm_map_min(map) < size) { + vm_map_unlock(map); + return (0); + } + assert_wait(map, TRUE); + vm_map_unlock(map); + thread_block(); + } + vm_map_insert(map, NULL, (vm_offset_t)0, addr, addr + size); + vm_map_unlock(map); + return (addr); +} + +/* + * kmem_free_wakeup + * + * Returns memory to a submap of the kernel, and wakes up any threads + * waiting for memory in that map. + */ +void +kmem_free_wakeup(map, addr, size) + vm_map_t map; + vm_offset_t addr; + vm_size_t size; +{ + vm_map_lock(map); + (void) vm_map_delete(map, trunc_page(addr), round_page(addr + size)); + thread_wakeup(map); + vm_map_unlock(map); +} + +/* + * Create the kernel map; insert a mapping covering kernel text, data, bss, + * and all space allocated thus far (`boostrap' data). The new map will thus + * map the range between VM_MIN_KERNEL_ADDRESS and `start' as allocated, and + * the range between `start' and `end' as free. + */ +void +kmem_init(start, end) + vm_offset_t start, end; +{ + register vm_map_t m; + + m = vm_map_create(pmap_kernel(), VM_MIN_KERNEL_ADDRESS, end, FALSE); + vm_map_lock(m); + /* N.B.: cannot use kgdb to debug, starting with this assignment ... */ + kernel_map = m; + (void) vm_map_insert(m, NULL, (vm_offset_t)0, + VM_MIN_KERNEL_ADDRESS, start); + /* ... and ending with the completion of the above `insert' */ + vm_map_unlock(m); +} diff --git a/sys/vm/vm_kern.h b/sys/vm/vm_kern.h new file mode 100644 index 00000000000..b4bf3cba04c --- /dev/null +++ b/sys/vm/vm_kern.h @@ -0,0 +1,74 @@ +/* $NetBSD: vm_kern.h,v 1.9 1994/06/29 06:48:03 cgd Exp $ */ + +/* + * Copyright (c) 1991, 1993 + * The Regents of the University of California. All rights reserved. + * + * This code is derived from software contributed to Berkeley by + * The Mach Operating System project at Carnegie-Mellon University. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. All advertising materials mentioning features or use of this software + * must display the following acknowledgement: + * This product includes software developed by the University of + * California, Berkeley and its contributors. + * 4. Neither the name of the University nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * @(#)vm_kern.h 8.1 (Berkeley) 6/11/93 + * + * + * Copyright (c) 1987, 1990 Carnegie-Mellon University. + * All rights reserved. + * + * Authors: Avadis Tevanian, Jr., Michael Wayne Young + * + * Permission to use, copy, modify and distribute this software and + * its documentation is hereby granted, provided that both the copyright + * notice and this permission notice appear in all copies of the + * software, derivative works or modified versions, and any portions + * thereof, and that both notices appear in supporting documentation. + * + * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" + * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND + * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. + * + * Carnegie Mellon requests users of this software to return to + * + * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU + * School of Computer Science + * Carnegie Mellon University + * Pittsburgh PA 15213-3890 + * + * any improvements or extensions that they make and grant Carnegie the + * rights to redistribute these changes. + */ + +/* Kernel memory management definitions. */ + +vm_map_t buffer_map; +vm_map_t exec_map; +vm_map_t kernel_map; +vm_map_t kmem_map; +vm_map_t mb_map; +vm_map_t phys_map; diff --git a/sys/vm/vm_map.c b/sys/vm/vm_map.c new file mode 100644 index 00000000000..98be7181420 --- /dev/null +++ b/sys/vm/vm_map.c @@ -0,0 +1,2657 @@ +/* $NetBSD: vm_map.c,v 1.21 1995/04/10 16:54:00 mycroft Exp $ */ + +/* + * Copyright (c) 1991, 1993 + * The Regents of the University of California. All rights reserved. + * + * This code is derived from software contributed to Berkeley by + * The Mach Operating System project at Carnegie-Mellon University. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. All advertising materials mentioning features or use of this software + * must display the following acknowledgement: + * This product includes software developed by the University of + * California, Berkeley and its contributors. + * 4. Neither the name of the University nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * @(#)vm_map.c 8.3 (Berkeley) 1/12/94 + * + * + * Copyright (c) 1987, 1990 Carnegie-Mellon University. + * All rights reserved. + * + * Authors: Avadis Tevanian, Jr., Michael Wayne Young + * + * Permission to use, copy, modify and distribute this software and + * its documentation is hereby granted, provided that both the copyright + * notice and this permission notice appear in all copies of the + * software, derivative works or modified versions, and any portions + * thereof, and that both notices appear in supporting documentation. + * + * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" + * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND + * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. + * + * Carnegie Mellon requests users of this software to return to + * + * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU + * School of Computer Science + * Carnegie Mellon University + * Pittsburgh PA 15213-3890 + * + * any improvements or extensions that they make and grant Carnegie the + * rights to redistribute these changes. + */ + +/* + * Virtual memory mapping module. + */ + +#include <sys/param.h> +#include <sys/systm.h> +#include <sys/malloc.h> + +#include <vm/vm.h> +#include <vm/vm_page.h> +#include <vm/vm_object.h> + +/* + * Virtual memory maps provide for the mapping, protection, + * and sharing of virtual memory objects. In addition, + * this module provides for an efficient virtual copy of + * memory from one map to another. + * + * Synchronization is required prior to most operations. + * + * Maps consist of an ordered doubly-linked list of simple + * entries; a single hint is used to speed up lookups. + * + * In order to properly represent the sharing of virtual + * memory regions among maps, the map structure is bi-level. + * Top-level ("address") maps refer to regions of sharable + * virtual memory. These regions are implemented as + * ("sharing") maps, which then refer to the actual virtual + * memory objects. When two address maps "share" memory, + * their top-level maps both have references to the same + * sharing map. When memory is virtual-copied from one + * address map to another, the references in the sharing + * maps are actually copied -- no copying occurs at the + * virtual memory object level. + * + * Since portions of maps are specified by start/end addreses, + * which may not align with existing map entries, all + * routines merely "clip" entries to these start/end values. + * [That is, an entry is split into two, bordering at a + * start or end value.] Note that these clippings may not + * always be necessary (as the two resulting entries are then + * not changed); however, the clipping is done for convenience. + * No attempt is currently made to "glue back together" two + * abutting entries. + * + * As mentioned above, virtual copy operations are performed + * by copying VM object references from one sharing map to + * another, and then marking both regions as copy-on-write. + * It is important to note that only one writeable reference + * to a VM object region exists in any map -- this means that + * shadow object creation can be delayed until a write operation + * occurs. + */ + +/* + * vm_map_startup: + * + * Initialize the vm_map module. Must be called before + * any other vm_map routines. + * + * Map and entry structures are allocated from the general + * purpose memory pool with some exceptions: + * + * - The kernel map and kmem submap are allocated statically. + * - Kernel map entries are allocated out of a static pool. + * + * These restrictions are necessary since malloc() uses the + * maps and requires map entries. + */ + +vm_offset_t kentry_data; +vm_size_t kentry_data_size; +vm_map_entry_t kentry_free; +vm_map_t kmap_free; + +static void _vm_map_clip_end __P((vm_map_t, vm_map_entry_t, vm_offset_t)); +static void _vm_map_clip_start __P((vm_map_t, vm_map_entry_t, vm_offset_t)); + +void +vm_map_startup() +{ + register int i; + register vm_map_entry_t mep; + vm_map_t mp; + + /* + * Static map structures for allocation before initialization of + * kernel map or kmem map. vm_map_create knows how to deal with them. + */ + kmap_free = mp = (vm_map_t) kentry_data; + i = MAX_KMAP; + while (--i > 0) { + mp->header.next = (vm_map_entry_t) (mp + 1); + mp++; + } + mp++->header.next = NULL; + + /* + * Form a free list of statically allocated kernel map entries + * with the rest. + */ + kentry_free = mep = (vm_map_entry_t) mp; + i = (kentry_data_size - MAX_KMAP * sizeof *mp) / sizeof *mep; + while (--i > 0) { + mep->next = mep + 1; + mep++; + } + mep->next = NULL; +} + +/* + * Allocate a vmspace structure, including a vm_map and pmap, + * and initialize those structures. The refcnt is set to 1. + * The remaining fields must be initialized by the caller. + */ +struct vmspace * +vmspace_alloc(min, max, pageable) + vm_offset_t min, max; + int pageable; +{ + register struct vmspace *vm; + + MALLOC(vm, struct vmspace *, sizeof(struct vmspace), M_VMMAP, M_WAITOK); + bzero(vm, (caddr_t) &vm->vm_startcopy - (caddr_t) vm); + vm_map_init(&vm->vm_map, min, max, pageable); + pmap_pinit(&vm->vm_pmap); + vm->vm_map.pmap = &vm->vm_pmap; /* XXX */ + vm->vm_refcnt = 1; + return (vm); +} + +void +vmspace_free(vm) + register struct vmspace *vm; +{ + + if (--vm->vm_refcnt == 0) { + /* + * Lock the map, to wait out all other references to it. + * Delete all of the mappings and pages they hold, + * then call the pmap module to reclaim anything left. + */ + vm_map_lock(&vm->vm_map); + (void) vm_map_delete(&vm->vm_map, vm->vm_map.min_offset, + vm->vm_map.max_offset); + pmap_release(&vm->vm_pmap); + FREE(vm, M_VMMAP); + } +} + +/* + * vm_map_create: + * + * Creates and returns a new empty VM map with + * the given physical map structure, and having + * the given lower and upper address bounds. + */ +vm_map_t +vm_map_create(pmap, min, max, pageable) + pmap_t pmap; + vm_offset_t min, max; + boolean_t pageable; +{ + register vm_map_t result; + extern vm_map_t kmem_map; + + if (kmem_map == NULL) { + result = kmap_free; + if (result == NULL) + panic("vm_map_create: out of maps"); + kmap_free = (vm_map_t) result->header.next; + } else + MALLOC(result, vm_map_t, sizeof(struct vm_map), + M_VMMAP, M_WAITOK); + + vm_map_init(result, min, max, pageable); + result->pmap = pmap; + return(result); +} + +/* + * Initialize an existing vm_map structure + * such as that in the vmspace structure. + * The pmap is set elsewhere. + */ +void +vm_map_init(map, min, max, pageable) + register struct vm_map *map; + vm_offset_t min, max; + boolean_t pageable; +{ + map->header.next = map->header.prev = &map->header; + map->nentries = 0; + map->size = 0; + map->ref_count = 1; + map->is_main_map = TRUE; + map->min_offset = min; + map->max_offset = max; + map->entries_pageable = pageable; + map->first_free = &map->header; + map->hint = &map->header; + map->timestamp = 0; + lock_init(&map->lock, TRUE); + simple_lock_init(&map->ref_lock); + simple_lock_init(&map->hint_lock); +} + +/* + * vm_map_entry_create: [ internal use only ] + * + * Allocates a VM map entry for insertion. + * No entry fields are filled in. This routine is + */ +vm_map_entry_t +vm_map_entry_create(map) + vm_map_t map; +{ + vm_map_entry_t entry; +#ifdef DEBUG + extern vm_map_t kernel_map, kmem_map, mb_map, pager_map; + boolean_t isspecial; + + isspecial = (map == kernel_map || map == kmem_map || + map == mb_map || map == pager_map); + if (isspecial && map->entries_pageable || + !isspecial && !map->entries_pageable) + panic("vm_map_entry_create: bogus map"); +#endif + if (map->entries_pageable) { + MALLOC(entry, vm_map_entry_t, sizeof(struct vm_map_entry), + M_VMMAPENT, M_WAITOK); + } else { + if (entry = kentry_free) + kentry_free = kentry_free->next; + } + if (entry == NULL) + panic("vm_map_entry_create: out of map entries"); + + return(entry); +} + +/* + * vm_map_entry_dispose: [ internal use only ] + * + * Inverse of vm_map_entry_create. + */ +void +vm_map_entry_dispose(map, entry) + vm_map_t map; + vm_map_entry_t entry; +{ +#ifdef DEBUG + extern vm_map_t kernel_map, kmem_map, mb_map, pager_map; + boolean_t isspecial; + + isspecial = (map == kernel_map || map == kmem_map || + map == mb_map || map == pager_map); + if (isspecial && map->entries_pageable || + !isspecial && !map->entries_pageable) + panic("vm_map_entry_dispose: bogus map"); +#endif + if (map->entries_pageable) { + FREE(entry, M_VMMAPENT); + } else { + entry->next = kentry_free; + kentry_free = entry; + } +} + +/* + * vm_map_entry_{un,}link: + * + * Insert/remove entries from maps. + */ +#define vm_map_entry_link(map, after_where, entry) \ + { \ + (map)->nentries++; \ + (entry)->prev = (after_where); \ + (entry)->next = (after_where)->next; \ + (entry)->prev->next = (entry); \ + (entry)->next->prev = (entry); \ + } +#define vm_map_entry_unlink(map, entry) \ + { \ + (map)->nentries--; \ + (entry)->next->prev = (entry)->prev; \ + (entry)->prev->next = (entry)->next; \ + } + +/* + * vm_map_reference: + * + * Creates another valid reference to the given map. + * + */ +void +vm_map_reference(map) + register vm_map_t map; +{ + if (map == NULL) + return; + + simple_lock(&map->ref_lock); + map->ref_count++; + simple_unlock(&map->ref_lock); +} + +/* + * vm_map_deallocate: + * + * Removes a reference from the specified map, + * destroying it if no references remain. + * The map should not be locked. + */ +void +vm_map_deallocate(map) + register vm_map_t map; +{ + register int c; + + if (map == NULL) + return; + + simple_lock(&map->ref_lock); + c = --map->ref_count; + simple_unlock(&map->ref_lock); + + if (c > 0) { + return; + } + + /* + * Lock the map, to wait out all other references + * to it. + */ + + vm_map_lock(map); + + (void) vm_map_delete(map, map->min_offset, map->max_offset); + + pmap_destroy(map->pmap); + + FREE(map, M_VMMAP); +} + +/* + * vm_map_insert: + * + * Inserts the given whole VM object into the target + * map at the specified address range. The object's + * size should match that of the address range. + * + * Requires that the map be locked, and leaves it so. + */ +int +vm_map_insert(map, object, offset, start, end) + vm_map_t map; + vm_object_t object; + vm_offset_t offset; + vm_offset_t start; + vm_offset_t end; +{ + register vm_map_entry_t new_entry; + register vm_map_entry_t prev_entry; + vm_map_entry_t temp_entry; + + /* + * Check that the start and end points are not bogus. + */ + + if ((start < map->min_offset) || (end > map->max_offset) || + (start >= end)) + return(KERN_INVALID_ADDRESS); + + /* + * Find the entry prior to the proposed + * starting address; if it's part of an + * existing entry, this range is bogus. + */ + + if (vm_map_lookup_entry(map, start, &temp_entry)) + return(KERN_NO_SPACE); + + prev_entry = temp_entry; + + /* + * Assert that the next entry doesn't overlap the + * end point. + */ + + if ((prev_entry->next != &map->header) && + (prev_entry->next->start < end)) + return(KERN_NO_SPACE); + + /* + * See if we can avoid creating a new entry by + * extending one of our neighbors. + */ + + if (object == NULL) { + if ((prev_entry != &map->header) && + (prev_entry->end == start) && + (map->is_main_map) && + (prev_entry->is_a_map == FALSE) && + (prev_entry->is_sub_map == FALSE) && + (prev_entry->inheritance == VM_INHERIT_DEFAULT) && + (prev_entry->protection == VM_PROT_DEFAULT) && + (prev_entry->max_protection == VM_PROT_DEFAULT) && + (prev_entry->wired_count == 0)) { + + if (vm_object_coalesce(prev_entry->object.vm_object, + NULL, + prev_entry->offset, + (vm_offset_t) 0, + (vm_size_t)(prev_entry->end + - prev_entry->start), + (vm_size_t)(end - prev_entry->end))) { + /* + * Coalesced the two objects - can extend + * the previous map entry to include the + * new range. + */ + map->size += (end - prev_entry->end); + prev_entry->end = end; + return(KERN_SUCCESS); + } + } + } + + /* + * Create a new entry + */ + + new_entry = vm_map_entry_create(map); + new_entry->start = start; + new_entry->end = end; + + new_entry->is_a_map = FALSE; + new_entry->is_sub_map = FALSE; + new_entry->object.vm_object = object; + new_entry->offset = offset; + + new_entry->copy_on_write = FALSE; + new_entry->needs_copy = FALSE; + + if (map->is_main_map) { + new_entry->inheritance = VM_INHERIT_DEFAULT; + new_entry->protection = VM_PROT_DEFAULT; + new_entry->max_protection = VM_PROT_DEFAULT; + new_entry->wired_count = 0; + } + + /* + * Insert the new entry into the list + */ + + vm_map_entry_link(map, prev_entry, new_entry); + map->size += new_entry->end - new_entry->start; + + /* + * Update the free space hint + */ + + if ((map->first_free == prev_entry) && (prev_entry->end >= new_entry->start)) + map->first_free = new_entry; + + return(KERN_SUCCESS); +} + +/* + * SAVE_HINT: + * + * Saves the specified entry as the hint for + * future lookups. Performs necessary interlocks. + */ +#define SAVE_HINT(map,value) \ + simple_lock(&(map)->hint_lock); \ + (map)->hint = (value); \ + simple_unlock(&(map)->hint_lock); + +/* + * vm_map_lookup_entry: [ internal use only ] + * + * Finds the map entry containing (or + * immediately preceding) the specified address + * in the given map; the entry is returned + * in the "entry" parameter. The boolean + * result indicates whether the address is + * actually contained in the map. + */ +boolean_t +vm_map_lookup_entry(map, address, entry) + register vm_map_t map; + register vm_offset_t address; + vm_map_entry_t *entry; /* OUT */ +{ + register vm_map_entry_t cur; + register vm_map_entry_t last; + + /* + * Start looking either from the head of the + * list, or from the hint. + */ + + simple_lock(&map->hint_lock); + cur = map->hint; + simple_unlock(&map->hint_lock); + + if (cur == &map->header) + cur = cur->next; + + if (address >= cur->start) { + /* + * Go from hint to end of list. + * + * But first, make a quick check to see if + * we are already looking at the entry we + * want (which is usually the case). + * Note also that we don't need to save the hint + * here... it is the same hint (unless we are + * at the header, in which case the hint didn't + * buy us anything anyway). + */ + last = &map->header; + if ((cur != last) && (cur->end > address)) { + *entry = cur; + return(TRUE); + } + } + else { + /* + * Go from start to hint, *inclusively* + */ + last = cur->next; + cur = map->header.next; + } + + /* + * Search linearly + */ + + while (cur != last) { + if (cur->end > address) { + if (address >= cur->start) { + /* + * Save this lookup for future + * hints, and return + */ + + *entry = cur; + SAVE_HINT(map, cur); + return(TRUE); + } + break; + } + cur = cur->next; + } + *entry = cur->prev; + SAVE_HINT(map, *entry); + return(FALSE); +} + +/* + * Find sufficient space for `length' bytes in the given map, starting at + * `start'. The map must be locked. Returns 0 on success, 1 on no space. + */ +int +vm_map_findspace(map, start, length, addr) + register vm_map_t map; + register vm_offset_t start; + vm_size_t length; + vm_offset_t *addr; +{ + register vm_map_entry_t entry, next; + register vm_offset_t end; + + if (start < map->min_offset) + start = map->min_offset; + if (start > map->max_offset) + return (1); + + /* + * Look for the first possible address; if there's already + * something at this address, we have to start after it. + */ + if (start == map->min_offset) { + if ((entry = map->first_free) != &map->header) + start = entry->end; + } else { + vm_map_entry_t tmp; + if (vm_map_lookup_entry(map, start, &tmp)) + start = tmp->end; + entry = tmp; + } + + /* + * Look through the rest of the map, trying to fit a new region in + * the gap between existing regions, or after the very last region. + */ + for (;; start = (entry = next)->end) { + /* + * Find the end of the proposed new region. Be sure we didn't + * go beyond the end of the map, or wrap around the address; + * if so, we lose. Otherwise, if this is the last entry, or + * if the proposed new region fits before the next entry, we + * win. + */ + end = start + length; + if (end > map->max_offset || end < start) + return (1); + next = entry->next; + if (next == &map->header || next->start >= end) + break; + } + SAVE_HINT(map, entry); + *addr = start; + return (0); +} + +/* + * vm_map_find finds an unallocated region in the target address + * map with the given length. The search is defined to be + * first-fit from the specified address; the region found is + * returned in the same parameter. + * + */ +int +vm_map_find(map, object, offset, addr, length, find_space) + vm_map_t map; + vm_object_t object; + vm_offset_t offset; + vm_offset_t *addr; /* IN/OUT */ + vm_size_t length; + boolean_t find_space; +{ + register vm_offset_t start; + int result; + + start = *addr; + vm_map_lock(map); + if (find_space) { + if (vm_map_findspace(map, start, length, addr)) { + vm_map_unlock(map); + return (KERN_NO_SPACE); + } + start = *addr; + } + result = vm_map_insert(map, object, offset, start, start + length); + vm_map_unlock(map); + return (result); +} + +/* + * vm_map_simplify_entry: [ internal use only ] + * + * Simplify the given map entry by: + * removing extra sharing maps + * [XXX maybe later] merging with a neighbor + */ +void +vm_map_simplify_entry(map, entry) + vm_map_t map; + vm_map_entry_t entry; +{ +#ifdef lint + map++; +#endif + + /* + * If this entry corresponds to a sharing map, then + * see if we can remove the level of indirection. + * If it's not a sharing map, then it points to + * a VM object, so see if we can merge with either + * of our neighbors. + */ + + if (entry->is_sub_map) + return; + if (entry->is_a_map) { +#if 0 + vm_map_t my_share_map; + int count; + + my_share_map = entry->object.share_map; + simple_lock(&my_share_map->ref_lock); + count = my_share_map->ref_count; + simple_unlock(&my_share_map->ref_lock); + + if (count == 1) { + /* Can move the region from + * entry->start to entry->end (+ entry->offset) + * in my_share_map into place of entry. + * Later. + */ + } +#endif + } + else { + /* + * Try to merge with our neighbors. + * + * Conditions for merge are: + * + * 1. entries are adjacent. + * 2. both entries point to objects + * with null pagers. + * + * If a merge is possible, we replace the two + * entries with a single entry, then merge + * the two objects into a single object. + * + * Now, all that is left to do is write the + * code! + */ + } +} + +/* + * vm_map_clip_start: [ internal use only ] + * + * Asserts that the given entry begins at or after + * the specified address; if necessary, + * it splits the entry into two. + */ +#define vm_map_clip_start(map, entry, startaddr) \ +{ \ + if (startaddr > entry->start) \ + _vm_map_clip_start(map, entry, startaddr); \ +} + +/* + * This routine is called only when it is known that + * the entry must be split. + */ +static void +_vm_map_clip_start(map, entry, start) + register vm_map_t map; + register vm_map_entry_t entry; + register vm_offset_t start; +{ + register vm_map_entry_t new_entry; + + /* + * See if we can simplify this entry first + */ + + vm_map_simplify_entry(map, entry); + + /* + * Split off the front portion -- + * note that we must insert the new + * entry BEFORE this one, so that + * this entry has the specified starting + * address. + */ + + new_entry = vm_map_entry_create(map); + *new_entry = *entry; + + new_entry->end = start; + entry->offset += (start - entry->start); + entry->start = start; + + vm_map_entry_link(map, entry->prev, new_entry); + + if (entry->is_a_map || entry->is_sub_map) + vm_map_reference(new_entry->object.share_map); + else + vm_object_reference(new_entry->object.vm_object); +} + +/* + * vm_map_clip_end: [ internal use only ] + * + * Asserts that the given entry ends at or before + * the specified address; if necessary, + * it splits the entry into two. + */ + +#define vm_map_clip_end(map, entry, endaddr) \ +{ \ + if (endaddr < entry->end) \ + _vm_map_clip_end(map, entry, endaddr); \ +} + +/* + * This routine is called only when it is known that + * the entry must be split. + */ +static void +_vm_map_clip_end(map, entry, end) + register vm_map_t map; + register vm_map_entry_t entry; + register vm_offset_t end; +{ + register vm_map_entry_t new_entry; + + /* + * Create a new entry and insert it + * AFTER the specified entry + */ + + new_entry = vm_map_entry_create(map); + *new_entry = *entry; + + new_entry->start = entry->end = end; + new_entry->offset += (end - entry->start); + + vm_map_entry_link(map, entry, new_entry); + + if (entry->is_a_map || entry->is_sub_map) + vm_map_reference(new_entry->object.share_map); + else + vm_object_reference(new_entry->object.vm_object); +} + +/* + * VM_MAP_RANGE_CHECK: [ internal use only ] + * + * Asserts that the starting and ending region + * addresses fall within the valid range of the map. + */ +#define VM_MAP_RANGE_CHECK(map, start, end) \ + { \ + if (start < vm_map_min(map)) \ + start = vm_map_min(map); \ + if (end > vm_map_max(map)) \ + end = vm_map_max(map); \ + if (start > end) \ + start = end; \ + } + +/* + * vm_map_submap: [ kernel use only ] + * + * Mark the given range as handled by a subordinate map. + * + * This range must have been created with vm_map_find, + * and no other operations may have been performed on this + * range prior to calling vm_map_submap. + * + * Only a limited number of operations can be performed + * within this rage after calling vm_map_submap: + * vm_fault + * [Don't try vm_map_copy!] + * + * To remove a submapping, one must first remove the + * range from the superior map, and then destroy the + * submap (if desired). [Better yet, don't try it.] + */ +int +vm_map_submap(map, start, end, submap) + register vm_map_t map; + register vm_offset_t start; + register vm_offset_t end; + vm_map_t submap; +{ + vm_map_entry_t entry; + register int result = KERN_INVALID_ARGUMENT; + + vm_map_lock(map); + + VM_MAP_RANGE_CHECK(map, start, end); + + if (vm_map_lookup_entry(map, start, &entry)) { + vm_map_clip_start(map, entry, start); + } + else + entry = entry->next; + + vm_map_clip_end(map, entry, end); + + if ((entry->start == start) && (entry->end == end) && + (!entry->is_a_map) && + (entry->object.vm_object == NULL) && + (!entry->copy_on_write)) { + entry->is_a_map = FALSE; + entry->is_sub_map = TRUE; + vm_map_reference(entry->object.sub_map = submap); + result = KERN_SUCCESS; + } + vm_map_unlock(map); + + return(result); +} + +/* + * vm_map_protect: + * + * Sets the protection of the specified address + * region in the target map. If "set_max" is + * specified, the maximum protection is to be set; + * otherwise, only the current protection is affected. + */ +int +vm_map_protect(map, start, end, new_prot, set_max) + register vm_map_t map; + register vm_offset_t start; + register vm_offset_t end; + register vm_prot_t new_prot; + register boolean_t set_max; +{ + register vm_map_entry_t current; + vm_map_entry_t entry; + + vm_map_lock(map); + + VM_MAP_RANGE_CHECK(map, start, end); + + if (vm_map_lookup_entry(map, start, &entry)) { + vm_map_clip_start(map, entry, start); + } + else + entry = entry->next; + + /* + * Make a first pass to check for protection + * violations. + */ + + current = entry; + while ((current != &map->header) && (current->start < end)) { + if (current->is_sub_map) + return(KERN_INVALID_ARGUMENT); + if ((new_prot & current->max_protection) != new_prot) { + vm_map_unlock(map); + return(KERN_PROTECTION_FAILURE); + } + + current = current->next; + } + + /* + * Go back and fix up protections. + * [Note that clipping is not necessary the second time.] + */ + + current = entry; + + while ((current != &map->header) && (current->start < end)) { + vm_prot_t old_prot; + + vm_map_clip_end(map, current, end); + + old_prot = current->protection; + if (set_max) + current->protection = + (current->max_protection = new_prot) & + old_prot; + else + current->protection = new_prot; + + /* + * Update physical map if necessary. + * Worry about copy-on-write here -- CHECK THIS XXX + */ + + if (current->protection != old_prot) { + +#define MASK(entry) ((entry)->copy_on_write ? ~VM_PROT_WRITE : \ + VM_PROT_ALL) +#define max(a,b) ((a) > (b) ? (a) : (b)) + + if (current->is_a_map) { + vm_map_entry_t share_entry; + vm_offset_t share_end; + + vm_map_lock(current->object.share_map); + (void) vm_map_lookup_entry( + current->object.share_map, + current->offset, + &share_entry); + share_end = current->offset + + (current->end - current->start); + while ((share_entry != + ¤t->object.share_map->header) && + (share_entry->start < share_end)) { + + pmap_protect(map->pmap, + (max(share_entry->start, + current->offset) - + current->offset + + current->start), + min(share_entry->end, + share_end) - + current->offset + + current->start, + current->protection & + MASK(share_entry)); + + share_entry = share_entry->next; + } + vm_map_unlock(current->object.share_map); + } + else + pmap_protect(map->pmap, current->start, + current->end, + current->protection & MASK(entry)); +#undef max +#undef MASK + } + current = current->next; + } + + vm_map_unlock(map); + return(KERN_SUCCESS); +} + +/* + * vm_map_inherit: + * + * Sets the inheritance of the specified address + * range in the target map. Inheritance + * affects how the map will be shared with + * child maps at the time of vm_map_fork. + */ +int +vm_map_inherit(map, start, end, new_inheritance) + register vm_map_t map; + register vm_offset_t start; + register vm_offset_t end; + register vm_inherit_t new_inheritance; +{ + register vm_map_entry_t entry; + vm_map_entry_t temp_entry; + + switch (new_inheritance) { + case VM_INHERIT_NONE: + case VM_INHERIT_COPY: + case VM_INHERIT_SHARE: + break; + default: + return(KERN_INVALID_ARGUMENT); + } + + vm_map_lock(map); + + VM_MAP_RANGE_CHECK(map, start, end); + + if (vm_map_lookup_entry(map, start, &temp_entry)) { + entry = temp_entry; + vm_map_clip_start(map, entry, start); + } + else + entry = temp_entry->next; + + while ((entry != &map->header) && (entry->start < end)) { + vm_map_clip_end(map, entry, end); + + entry->inheritance = new_inheritance; + + entry = entry->next; + } + + vm_map_unlock(map); + return(KERN_SUCCESS); +} + +/* + * vm_map_pageable: + * + * Sets the pageability of the specified address + * range in the target map. Regions specified + * as not pageable require locked-down physical + * memory and physical page maps. + * + * The map must not be locked, but a reference + * must remain to the map throughout the call. + */ +int +vm_map_pageable(map, start, end, new_pageable) + register vm_map_t map; + register vm_offset_t start; + register vm_offset_t end; + register boolean_t new_pageable; +{ + register vm_map_entry_t entry; + vm_map_entry_t start_entry; + register vm_offset_t failed; + int rv; + + vm_map_lock(map); + + VM_MAP_RANGE_CHECK(map, start, end); + + /* + * Only one pageability change may take place at one + * time, since vm_fault assumes it will be called + * only once for each wiring/unwiring. Therefore, we + * have to make sure we're actually changing the pageability + * for the entire region. We do so before making any changes. + */ + + if (vm_map_lookup_entry(map, start, &start_entry) == FALSE) { + vm_map_unlock(map); + return(KERN_INVALID_ADDRESS); + } + entry = start_entry; + + /* + * Actions are rather different for wiring and unwiring, + * so we have two separate cases. + */ + + if (new_pageable) { + + vm_map_clip_start(map, entry, start); + + /* + * Unwiring. First ensure that the range to be + * unwired is really wired down and that there + * are no holes. + */ + while ((entry != &map->header) && (entry->start < end)) { + + if (entry->wired_count == 0 || + (entry->end < end && + (entry->next == &map->header || + entry->next->start > entry->end))) { + vm_map_unlock(map); + return(KERN_INVALID_ARGUMENT); + } + entry = entry->next; + } + + /* + * Now decrement the wiring count for each region. + * If a region becomes completely unwired, + * unwire its physical pages and mappings. + */ + lock_set_recursive(&map->lock); + + entry = start_entry; + while ((entry != &map->header) && (entry->start < end)) { + vm_map_clip_end(map, entry, end); + + entry->wired_count--; + if (entry->wired_count == 0) + vm_fault_unwire(map, entry->start, entry->end); + + entry = entry->next; + } + lock_clear_recursive(&map->lock); + } + + else { + /* + * Wiring. We must do this in two passes: + * + * 1. Holding the write lock, we create any shadow + * or zero-fill objects that need to be created. + * Then we clip each map entry to the region to be + * wired and increment its wiring count. We + * create objects before clipping the map entries + * to avoid object proliferation. + * + * 2. We downgrade to a read lock, and call + * vm_fault_wire to fault in the pages for any + * newly wired area (wired_count is 1). + * + * Downgrading to a read lock for vm_fault_wire avoids + * a possible deadlock with another thread that may have + * faulted on one of the pages to be wired (it would mark + * the page busy, blocking us, then in turn block on the + * map lock that we hold). Because of problems in the + * recursive lock package, we cannot upgrade to a write + * lock in vm_map_lookup. Thus, any actions that require + * the write lock must be done beforehand. Because we + * keep the read lock on the map, the copy-on-write status + * of the entries we modify here cannot change. + */ + + /* + * Pass 1. + */ + while ((entry != &map->header) && (entry->start < end)) { + if (entry->wired_count == 0) { + + /* + * Perform actions of vm_map_lookup that need + * the write lock on the map: create a shadow + * object for a copy-on-write region, or an + * object for a zero-fill region. + * + * We don't have to do this for entries that + * point to sharing maps, because we won't hold + * the lock on the sharing map. + */ + if (!entry->is_a_map) { + if (entry->needs_copy && + ((entry->protection & VM_PROT_WRITE) != 0)) { + + vm_object_shadow(&entry->object.vm_object, + &entry->offset, + (vm_size_t)(entry->end + - entry->start)); + entry->needs_copy = FALSE; + } + else if (entry->object.vm_object == NULL) { + entry->object.vm_object = + vm_object_allocate((vm_size_t)(entry->end + - entry->start)); + entry->offset = (vm_offset_t)0; + } + } + } + vm_map_clip_start(map, entry, start); + vm_map_clip_end(map, entry, end); + entry->wired_count++; + + /* + * Check for holes + */ + if (entry->end < end && + (entry->next == &map->header || + entry->next->start > entry->end)) { + /* + * Found one. Object creation actions + * do not need to be undone, but the + * wired counts need to be restored. + */ + while (entry != &map->header && entry->end > start) { + entry->wired_count--; + entry = entry->prev; + } + vm_map_unlock(map); + return(KERN_INVALID_ARGUMENT); + } + entry = entry->next; + } + + /* + * Pass 2. + */ + + /* + * HACK HACK HACK HACK + * + * If we are wiring in the kernel map or a submap of it, + * unlock the map to avoid deadlocks. We trust that the + * kernel threads are well-behaved, and therefore will + * not do anything destructive to this region of the map + * while we have it unlocked. We cannot trust user threads + * to do the same. + * + * HACK HACK HACK HACK + */ + if (vm_map_pmap(map) == pmap_kernel()) { + vm_map_unlock(map); /* trust me ... */ + } + else { + lock_set_recursive(&map->lock); + lock_write_to_read(&map->lock); + } + + rv = 0; + entry = start_entry; + while (entry != &map->header && entry->start < end) { + /* + * If vm_fault_wire fails for any page we need to + * undo what has been done. We decrement the wiring + * count for those pages which have not yet been + * wired (now) and unwire those that have (later). + * + * XXX this violates the locking protocol on the map, + * needs to be fixed. + */ + if (rv) + entry->wired_count--; + else if (entry->wired_count == 1) { + rv = vm_fault_wire(map, entry->start, entry->end); + if (rv) { + failed = entry->start; + entry->wired_count--; + } + } + entry = entry->next; + } + + if (vm_map_pmap(map) == pmap_kernel()) { + vm_map_lock(map); + } + else { + lock_clear_recursive(&map->lock); + } + if (rv) { + vm_map_unlock(map); + (void) vm_map_pageable(map, start, failed, TRUE); + return(rv); + } + } + + vm_map_unlock(map); + + return(KERN_SUCCESS); +} + +/* + * vm_map_clean + * + * Push any dirty cached pages in the address range to their pager. + * If syncio is TRUE, dirty pages are written synchronously. + * If invalidate is TRUE, any cached pages are freed as well. + * + * Returns an error if any part of the specified range is not mapped. + */ +int +vm_map_clean(map, start, end, syncio, invalidate) + vm_map_t map; + vm_offset_t start; + vm_offset_t end; + boolean_t syncio; + boolean_t invalidate; +{ + register vm_map_entry_t current; + vm_map_entry_t entry; + vm_size_t size; + vm_object_t object; + vm_offset_t offset; + + vm_map_lock_read(map); + VM_MAP_RANGE_CHECK(map, start, end); + if (!vm_map_lookup_entry(map, start, &entry)) { + vm_map_unlock_read(map); + return(KERN_INVALID_ADDRESS); + } + + /* + * Make a first pass to check for holes. + */ + for (current = entry; current->start < end; current = current->next) { + if (current->is_sub_map) { + vm_map_unlock_read(map); + return(KERN_INVALID_ARGUMENT); + } + if (end > current->end && + (current->next == &map->header || + current->end != current->next->start)) { + vm_map_unlock_read(map); + return(KERN_INVALID_ADDRESS); + } + } + + /* + * Make a second pass, cleaning/uncaching pages from the indicated + * objects as we go. + */ + for (current = entry; current->start < end; current = current->next) { + offset = current->offset + (start - current->start); + size = (end <= current->end ? end : current->end) - start; + if (current->is_a_map) { + register vm_map_t smap; + vm_map_entry_t tentry; + vm_size_t tsize; + + smap = current->object.share_map; + vm_map_lock_read(smap); + (void) vm_map_lookup_entry(smap, offset, &tentry); + tsize = tentry->end - offset; + if (tsize < size) + size = tsize; + object = tentry->object.vm_object; + offset = tentry->offset + (offset - tentry->start); + vm_object_lock(object); + vm_map_unlock_read(smap); + } else { + object = current->object.vm_object; + vm_object_lock(object); + } + /* + * Flush pages if writing is allowed. + * XXX should we continue on an error? + */ + if ((current->protection & VM_PROT_WRITE) && + !vm_object_page_clean(object, offset, offset+size, + syncio, FALSE)) { + vm_object_unlock(object); + vm_map_unlock_read(map); + return(KERN_FAILURE); + } + if (invalidate) + vm_object_page_remove(object, offset, offset+size); + vm_object_unlock(object); + start += size; + } + + vm_map_unlock_read(map); + return(KERN_SUCCESS); +} + +/* + * vm_map_entry_unwire: [ internal use only ] + * + * Make the region specified by this entry pageable. + * + * The map in question should be locked. + * [This is the reason for this routine's existence.] + */ +void +vm_map_entry_unwire(map, entry) + vm_map_t map; + register vm_map_entry_t entry; +{ + vm_fault_unwire(map, entry->start, entry->end); + entry->wired_count = 0; +} + +/* + * vm_map_entry_delete: [ internal use only ] + * + * Deallocate the given entry from the target map. + */ +void +vm_map_entry_delete(map, entry) + register vm_map_t map; + register vm_map_entry_t entry; +{ + if (entry->wired_count != 0) + vm_map_entry_unwire(map, entry); + + vm_map_entry_unlink(map, entry); + map->size -= entry->end - entry->start; + + if (entry->is_a_map || entry->is_sub_map) + vm_map_deallocate(entry->object.share_map); + else + vm_object_deallocate(entry->object.vm_object); + + vm_map_entry_dispose(map, entry); +} + +/* + * vm_map_delete: [ internal use only ] + * + * Deallocates the given address range from the target + * map. + * + * When called with a sharing map, removes pages from + * that region from all physical maps. + */ +int +vm_map_delete(map, start, end) + register vm_map_t map; + vm_offset_t start; + register vm_offset_t end; +{ + register vm_map_entry_t entry; + vm_map_entry_t first_entry; + + /* + * Find the start of the region, and clip it + */ + + if (!vm_map_lookup_entry(map, start, &first_entry)) + entry = first_entry->next; + else { + entry = first_entry; + vm_map_clip_start(map, entry, start); + + /* + * Fix the lookup hint now, rather than each + * time though the loop. + */ + + SAVE_HINT(map, entry->prev); + } + + /* + * Save the free space hint + */ + + if (map->first_free->start >= start) + map->first_free = entry->prev; + + /* + * Step through all entries in this region + */ + + while ((entry != &map->header) && (entry->start < end)) { + vm_map_entry_t next; + register vm_offset_t s, e; + register vm_object_t object; + + vm_map_clip_end(map, entry, end); + + next = entry->next; + s = entry->start; + e = entry->end; + + /* + * Unwire before removing addresses from the pmap; + * otherwise, unwiring will put the entries back in + * the pmap. + */ + + object = entry->object.vm_object; + if (entry->wired_count != 0) + vm_map_entry_unwire(map, entry); + + /* + * If this is a sharing map, we must remove + * *all* references to this data, since we can't + * find all of the physical maps which are sharing + * it. + */ + + if (object == kernel_object || object == kmem_object) + vm_object_page_remove(object, entry->offset, + entry->offset + (e - s)); + else if (!map->is_main_map) + vm_object_pmap_remove(object, + entry->offset, + entry->offset + (e - s)); + else + pmap_remove(map->pmap, s, e); + + /* + * Delete the entry (which may delete the object) + * only after removing all pmap entries pointing + * to its pages. (Otherwise, its page frames may + * be reallocated, and any modify bits will be + * set in the wrong object!) + */ + + vm_map_entry_delete(map, entry); + entry = next; + } + return(KERN_SUCCESS); +} + +/* + * vm_map_remove: + * + * Remove the given address range from the target map. + * This is the exported form of vm_map_delete. + */ +int +vm_map_remove(map, start, end) + register vm_map_t map; + register vm_offset_t start; + register vm_offset_t end; +{ + register int result; + + vm_map_lock(map); + VM_MAP_RANGE_CHECK(map, start, end); + result = vm_map_delete(map, start, end); + vm_map_unlock(map); + + return(result); +} + +/* + * vm_map_check_protection: + * + * Assert that the target map allows the specified + * privilege on the entire address region given. + * The entire region must be allocated. + */ +boolean_t +vm_map_check_protection(map, start, end, protection) + register vm_map_t map; + register vm_offset_t start; + register vm_offset_t end; + register vm_prot_t protection; +{ + register vm_map_entry_t entry; + vm_map_entry_t tmp_entry; + + if (!vm_map_lookup_entry(map, start, &tmp_entry)) { + return(FALSE); + } + + entry = tmp_entry; + + while (start < end) { + if (entry == &map->header) { + return(FALSE); + } + + /* + * No holes allowed! + */ + + if (start < entry->start) { + return(FALSE); + } + + /* + * Check protection associated with entry. + */ + + if ((entry->protection & protection) != protection) { + return(FALSE); + } + + /* go to next entry */ + + start = entry->end; + entry = entry->next; + } + return(TRUE); +} + +/* + * vm_map_copy_entry: + * + * Copies the contents of the source entry to the destination + * entry. The entries *must* be aligned properly. + */ +void +vm_map_copy_entry(src_map, dst_map, src_entry, dst_entry) + vm_map_t src_map, dst_map; + register vm_map_entry_t src_entry, dst_entry; +{ + vm_object_t temp_object; + + if (src_entry->is_sub_map || dst_entry->is_sub_map) + return; + + if (dst_entry->object.vm_object != NULL && + (dst_entry->object.vm_object->flags & OBJ_INTERNAL) == 0) + printf("vm_map_copy_entry: copying over permanent data!\n"); + + /* + * If our destination map was wired down, + * unwire it now. + */ + + if (dst_entry->wired_count != 0) + vm_map_entry_unwire(dst_map, dst_entry); + + /* + * If we're dealing with a sharing map, we + * must remove the destination pages from + * all maps (since we cannot know which maps + * this sharing map belongs in). + */ + + if (dst_map->is_main_map) + pmap_remove(dst_map->pmap, dst_entry->start, dst_entry->end); + else + vm_object_pmap_remove(dst_entry->object.vm_object, + dst_entry->offset, + dst_entry->offset + + (dst_entry->end - dst_entry->start)); + + if (src_entry->wired_count == 0) { + + boolean_t src_needs_copy; + + /* + * If the source entry is marked needs_copy, + * it is already write-protected. + */ + if (!src_entry->needs_copy) { + + boolean_t su; + + /* + * If the source entry has only one mapping, + * we can just protect the virtual address + * range. + */ + if (!(su = src_map->is_main_map)) { + simple_lock(&src_map->ref_lock); + su = (src_map->ref_count == 1); + simple_unlock(&src_map->ref_lock); + } + + if (su) { + pmap_protect(src_map->pmap, + src_entry->start, + src_entry->end, + src_entry->protection & ~VM_PROT_WRITE); + } + else { + vm_object_pmap_copy(src_entry->object.vm_object, + src_entry->offset, + src_entry->offset + (src_entry->end + -src_entry->start)); + } + } + + /* + * Make a copy of the object. + */ + temp_object = dst_entry->object.vm_object; + vm_object_copy(src_entry->object.vm_object, + src_entry->offset, + (vm_size_t)(src_entry->end - + src_entry->start), + &dst_entry->object.vm_object, + &dst_entry->offset, + &src_needs_copy); + /* + * If we didn't get a copy-object now, mark the + * source map entry so that a shadow will be created + * to hold its changed pages. + */ + if (src_needs_copy) + src_entry->needs_copy = TRUE; + + /* + * The destination always needs to have a shadow + * created. + */ + dst_entry->needs_copy = TRUE; + + /* + * Mark the entries copy-on-write, so that write-enabling + * the entry won't make copy-on-write pages writable. + */ + src_entry->copy_on_write = TRUE; + dst_entry->copy_on_write = TRUE; + /* + * Get rid of the old object. + */ + vm_object_deallocate(temp_object); + + pmap_copy(dst_map->pmap, src_map->pmap, dst_entry->start, + dst_entry->end - dst_entry->start, src_entry->start); + } + else { + /* + * Of course, wired down pages can't be set copy-on-write. + * Cause wired pages to be copied into the new + * map by simulating faults (the new pages are + * pageable) + */ + vm_fault_copy_entry(dst_map, src_map, dst_entry, src_entry); + } +} + +/* + * vm_map_copy: + * + * Perform a virtual memory copy from the source + * address map/range to the destination map/range. + * + * If src_destroy or dst_alloc is requested, + * the source and destination regions should be + * disjoint, not only in the top-level map, but + * in the sharing maps as well. [The best way + * to guarantee this is to use a new intermediate + * map to make copies. This also reduces map + * fragmentation.] + */ +int +vm_map_copy(dst_map, src_map, + dst_addr, len, src_addr, + dst_alloc, src_destroy) + vm_map_t dst_map; + vm_map_t src_map; + vm_offset_t dst_addr; + vm_size_t len; + vm_offset_t src_addr; + boolean_t dst_alloc; + boolean_t src_destroy; +{ + register + vm_map_entry_t src_entry; + register + vm_map_entry_t dst_entry; + vm_map_entry_t tmp_entry; + vm_offset_t src_start; + vm_offset_t src_end; + vm_offset_t dst_start; + vm_offset_t dst_end; + vm_offset_t src_clip; + vm_offset_t dst_clip; + int result; + boolean_t old_src_destroy; + + /* + * XXX While we figure out why src_destroy screws up, + * we'll do it by explicitly vm_map_delete'ing at the end. + */ + + old_src_destroy = src_destroy; + src_destroy = FALSE; + + /* + * Compute start and end of region in both maps + */ + + src_start = src_addr; + src_end = src_start + len; + dst_start = dst_addr; + dst_end = dst_start + len; + + /* + * Check that the region can exist in both source + * and destination. + */ + + if ((dst_end < dst_start) || (src_end < src_start)) + return(KERN_NO_SPACE); + + /* + * Lock the maps in question -- we avoid deadlock + * by ordering lock acquisition by map value + */ + + if (src_map == dst_map) { + vm_map_lock(src_map); + } + else if ((long) src_map < (long) dst_map) { + vm_map_lock(src_map); + vm_map_lock(dst_map); + } else { + vm_map_lock(dst_map); + vm_map_lock(src_map); + } + + result = KERN_SUCCESS; + + /* + * Check protections... source must be completely readable and + * destination must be completely writable. [Note that if we're + * allocating the destination region, we don't have to worry + * about protection, but instead about whether the region + * exists.] + */ + + if (src_map->is_main_map && dst_map->is_main_map) { + if (!vm_map_check_protection(src_map, src_start, src_end, + VM_PROT_READ)) { + result = KERN_PROTECTION_FAILURE; + goto Return; + } + + if (dst_alloc) { + /* XXX Consider making this a vm_map_find instead */ + if ((result = vm_map_insert(dst_map, NULL, + (vm_offset_t) 0, dst_start, dst_end)) != KERN_SUCCESS) + goto Return; + } + else if (!vm_map_check_protection(dst_map, dst_start, dst_end, + VM_PROT_WRITE)) { + result = KERN_PROTECTION_FAILURE; + goto Return; + } + } + + /* + * Find the start entries and clip. + * + * Note that checking protection asserts that the + * lookup cannot fail. + * + * Also note that we wait to do the second lookup + * until we have done the first clip, as the clip + * may affect which entry we get! + */ + + (void) vm_map_lookup_entry(src_map, src_addr, &tmp_entry); + src_entry = tmp_entry; + vm_map_clip_start(src_map, src_entry, src_start); + + (void) vm_map_lookup_entry(dst_map, dst_addr, &tmp_entry); + dst_entry = tmp_entry; + vm_map_clip_start(dst_map, dst_entry, dst_start); + + /* + * If both source and destination entries are the same, + * retry the first lookup, as it may have changed. + */ + + if (src_entry == dst_entry) { + (void) vm_map_lookup_entry(src_map, src_addr, &tmp_entry); + src_entry = tmp_entry; + } + + /* + * If source and destination entries are still the same, + * a null copy is being performed. + */ + + if (src_entry == dst_entry) + goto Return; + + /* + * Go through entries until we get to the end of the + * region. + */ + + while (src_start < src_end) { + /* + * Clip the entries to the endpoint of the entire region. + */ + + vm_map_clip_end(src_map, src_entry, src_end); + vm_map_clip_end(dst_map, dst_entry, dst_end); + + /* + * Clip each entry to the endpoint of the other entry. + */ + + src_clip = src_entry->start + (dst_entry->end - dst_entry->start); + vm_map_clip_end(src_map, src_entry, src_clip); + + dst_clip = dst_entry->start + (src_entry->end - src_entry->start); + vm_map_clip_end(dst_map, dst_entry, dst_clip); + + /* + * Both entries now match in size and relative endpoints. + * + * If both entries refer to a VM object, we can + * deal with them now. + */ + + if (!src_entry->is_a_map && !dst_entry->is_a_map) { + vm_map_copy_entry(src_map, dst_map, src_entry, + dst_entry); + } + else { + register vm_map_t new_dst_map; + vm_offset_t new_dst_start; + vm_size_t new_size; + vm_map_t new_src_map; + vm_offset_t new_src_start; + + /* + * We have to follow at least one sharing map. + */ + + new_size = (dst_entry->end - dst_entry->start); + + if (src_entry->is_a_map) { + new_src_map = src_entry->object.share_map; + new_src_start = src_entry->offset; + } + else { + new_src_map = src_map; + new_src_start = src_entry->start; + lock_set_recursive(&src_map->lock); + } + + if (dst_entry->is_a_map) { + vm_offset_t new_dst_end; + + new_dst_map = dst_entry->object.share_map; + new_dst_start = dst_entry->offset; + + /* + * Since the destination sharing entries + * will be merely deallocated, we can + * do that now, and replace the region + * with a null object. [This prevents + * splitting the source map to match + * the form of the destination map.] + * Note that we can only do so if the + * source and destination do not overlap. + */ + + new_dst_end = new_dst_start + new_size; + + if (new_dst_map != new_src_map) { + vm_map_lock(new_dst_map); + (void) vm_map_delete(new_dst_map, + new_dst_start, + new_dst_end); + (void) vm_map_insert(new_dst_map, + NULL, + (vm_offset_t) 0, + new_dst_start, + new_dst_end); + vm_map_unlock(new_dst_map); + } + } + else { + new_dst_map = dst_map; + new_dst_start = dst_entry->start; + lock_set_recursive(&dst_map->lock); + } + + /* + * Recursively copy the sharing map. + */ + + (void) vm_map_copy(new_dst_map, new_src_map, + new_dst_start, new_size, new_src_start, + FALSE, FALSE); + + if (dst_map == new_dst_map) + lock_clear_recursive(&dst_map->lock); + if (src_map == new_src_map) + lock_clear_recursive(&src_map->lock); + } + + /* + * Update variables for next pass through the loop. + */ + + src_start = src_entry->end; + src_entry = src_entry->next; + dst_start = dst_entry->end; + dst_entry = dst_entry->next; + + /* + * If the source is to be destroyed, here is the + * place to do it. + */ + + if (src_destroy && src_map->is_main_map && + dst_map->is_main_map) + vm_map_entry_delete(src_map, src_entry->prev); + } + + /* + * Update the physical maps as appropriate + */ + + if (src_map->is_main_map && dst_map->is_main_map) { + if (src_destroy) + pmap_remove(src_map->pmap, src_addr, src_addr + len); + } + + /* + * Unlock the maps + */ + + Return: ; + + if (old_src_destroy) + vm_map_delete(src_map, src_addr, src_addr + len); + + vm_map_unlock(src_map); + if (src_map != dst_map) + vm_map_unlock(dst_map); + + return(result); +} + +/* + * vmspace_fork: + * Create a new process vmspace structure and vm_map + * based on those of an existing process. The new map + * is based on the old map, according to the inheritance + * values on the regions in that map. + * + * The source map must not be locked. + */ +struct vmspace * +vmspace_fork(vm1) + register struct vmspace *vm1; +{ + register struct vmspace *vm2; + vm_map_t old_map = &vm1->vm_map; + vm_map_t new_map; + vm_map_entry_t old_entry; + vm_map_entry_t new_entry; + pmap_t new_pmap; + + vm_map_lock(old_map); + + vm2 = vmspace_alloc(old_map->min_offset, old_map->max_offset, + old_map->entries_pageable); + bcopy(&vm1->vm_startcopy, &vm2->vm_startcopy, + (caddr_t) (vm1 + 1) - (caddr_t) &vm1->vm_startcopy); + new_pmap = &vm2->vm_pmap; /* XXX */ + new_map = &vm2->vm_map; /* XXX */ + + old_entry = old_map->header.next; + + while (old_entry != &old_map->header) { + if (old_entry->is_sub_map) + panic("vm_map_fork: encountered a submap"); + + switch (old_entry->inheritance) { + case VM_INHERIT_NONE: + break; + + case VM_INHERIT_SHARE: + /* + * If we don't already have a sharing map: + */ + + if (!old_entry->is_a_map) { + vm_map_t new_share_map; + vm_map_entry_t new_share_entry; + + /* + * Create a new sharing map + */ + + new_share_map = vm_map_create(NULL, + old_entry->start, + old_entry->end, + TRUE); + new_share_map->is_main_map = FALSE; + + /* + * Create the only sharing entry from the + * old task map entry. + */ + + new_share_entry = + vm_map_entry_create(new_share_map); + *new_share_entry = *old_entry; + new_share_entry->wired_count = 0; + + /* + * Insert the entry into the new sharing + * map + */ + + vm_map_entry_link(new_share_map, + new_share_map->header.prev, + new_share_entry); + + /* + * Fix up the task map entry to refer + * to the sharing map now. + */ + + old_entry->is_a_map = TRUE; + old_entry->object.share_map = new_share_map; + old_entry->offset = old_entry->start; + } + + /* + * Clone the entry, referencing the sharing map. + */ + + new_entry = vm_map_entry_create(new_map); + *new_entry = *old_entry; + new_entry->wired_count = 0; + vm_map_reference(new_entry->object.share_map); + + /* + * Insert the entry into the new map -- we + * know we're inserting at the end of the new + * map. + */ + + vm_map_entry_link(new_map, new_map->header.prev, + new_entry); + + /* + * Update the physical map + */ + + pmap_copy(new_map->pmap, old_map->pmap, + new_entry->start, + (old_entry->end - old_entry->start), + old_entry->start); + break; + + case VM_INHERIT_COPY: + /* + * Clone the entry and link into the map. + */ + + new_entry = vm_map_entry_create(new_map); + *new_entry = *old_entry; + new_entry->wired_count = 0; + new_entry->object.vm_object = NULL; + new_entry->is_a_map = FALSE; + vm_map_entry_link(new_map, new_map->header.prev, + new_entry); + if (old_entry->is_a_map) { + int check; + + check = vm_map_copy(new_map, + old_entry->object.share_map, + new_entry->start, + (vm_size_t)(new_entry->end - + new_entry->start), + old_entry->offset, + FALSE, FALSE); + if (check != KERN_SUCCESS) + printf("vm_map_fork: copy in share_map region failed\n"); + } + else { + vm_map_copy_entry(old_map, new_map, old_entry, + new_entry); + } + break; + } + old_entry = old_entry->next; + } + + new_map->size = old_map->size; + vm_map_unlock(old_map); + + return(vm2); +} + +/* + * vm_map_lookup: + * + * Finds the VM object, offset, and + * protection for a given virtual address in the + * specified map, assuming a page fault of the + * type specified. + * + * Leaves the map in question locked for read; return + * values are guaranteed until a vm_map_lookup_done + * call is performed. Note that the map argument + * is in/out; the returned map must be used in + * the call to vm_map_lookup_done. + * + * A handle (out_entry) is returned for use in + * vm_map_lookup_done, to make that fast. + * + * If a lookup is requested with "write protection" + * specified, the map may be changed to perform virtual + * copying operations, although the data referenced will + * remain the same. + */ +int +vm_map_lookup(var_map, vaddr, fault_type, out_entry, + object, offset, out_prot, wired, single_use) + vm_map_t *var_map; /* IN/OUT */ + register vm_offset_t vaddr; + register vm_prot_t fault_type; + + vm_map_entry_t *out_entry; /* OUT */ + vm_object_t *object; /* OUT */ + vm_offset_t *offset; /* OUT */ + vm_prot_t *out_prot; /* OUT */ + boolean_t *wired; /* OUT */ + boolean_t *single_use; /* OUT */ +{ + vm_map_t share_map; + vm_offset_t share_offset; + register vm_map_entry_t entry; + register vm_map_t map = *var_map; + register vm_prot_t prot; + register boolean_t su; + + RetryLookup: ; + + /* + * Lookup the faulting address. + */ + + vm_map_lock_read(map); + +#define RETURN(why) \ + { \ + vm_map_unlock_read(map); \ + return(why); \ + } + + /* + * If the map has an interesting hint, try it before calling + * full blown lookup routine. + */ + + simple_lock(&map->hint_lock); + entry = map->hint; + simple_unlock(&map->hint_lock); + + *out_entry = entry; + + if ((entry == &map->header) || + (vaddr < entry->start) || (vaddr >= entry->end)) { + vm_map_entry_t tmp_entry; + + /* + * Entry was either not a valid hint, or the vaddr + * was not contained in the entry, so do a full lookup. + */ + if (!vm_map_lookup_entry(map, vaddr, &tmp_entry)) + RETURN(KERN_INVALID_ADDRESS); + + entry = tmp_entry; + *out_entry = entry; + } + + /* + * Handle submaps. + */ + + if (entry->is_sub_map) { + vm_map_t old_map = map; + + *var_map = map = entry->object.sub_map; + vm_map_unlock_read(old_map); + goto RetryLookup; + } + + /* + * Check whether this task is allowed to have + * this page. + */ + + prot = entry->protection; + if ((fault_type & (prot)) != fault_type) + RETURN(KERN_PROTECTION_FAILURE); + + /* + * If this page is not pageable, we have to get + * it for all possible accesses. + */ + + if (*wired = (entry->wired_count != 0)) + prot = fault_type = entry->protection; + + /* + * If we don't already have a VM object, track + * it down. + */ + + if (su = !entry->is_a_map) { + share_map = map; + share_offset = vaddr; + } + else { + vm_map_entry_t share_entry; + + /* + * Compute the sharing map, and offset into it. + */ + + share_map = entry->object.share_map; + share_offset = (vaddr - entry->start) + entry->offset; + + /* + * Look for the backing store object and offset + */ + + vm_map_lock_read(share_map); + + if (!vm_map_lookup_entry(share_map, share_offset, + &share_entry)) { + vm_map_unlock_read(share_map); + RETURN(KERN_INVALID_ADDRESS); + } + entry = share_entry; + } + + /* + * If the entry was copy-on-write, we either ... + */ + + if (entry->needs_copy) { + /* + * If we want to write the page, we may as well + * handle that now since we've got the sharing + * map locked. + * + * If we don't need to write the page, we just + * demote the permissions allowed. + */ + + if (fault_type & VM_PROT_WRITE) { + /* + * Make a new object, and place it in the + * object chain. Note that no new references + * have appeared -- one just moved from the + * share map to the new object. + */ + + if (lock_read_to_write(&share_map->lock)) { + if (share_map != map) + vm_map_unlock_read(map); + goto RetryLookup; + } + + vm_object_shadow( + &entry->object.vm_object, + &entry->offset, + (vm_size_t) (entry->end - entry->start)); + + entry->needs_copy = FALSE; + + lock_write_to_read(&share_map->lock); + } + else { + /* + * We're attempting to read a copy-on-write + * page -- don't allow writes. + */ + + prot &= (~VM_PROT_WRITE); + } + } + + /* + * Create an object if necessary. + */ + if (entry->object.vm_object == NULL) { + + if (lock_read_to_write(&share_map->lock)) { + if (share_map != map) + vm_map_unlock_read(map); + goto RetryLookup; + } + + entry->object.vm_object = vm_object_allocate( + (vm_size_t)(entry->end - entry->start)); + entry->offset = 0; + lock_write_to_read(&share_map->lock); + } + + /* + * Return the object/offset from this entry. If the entry + * was copy-on-write or empty, it has been fixed up. + */ + + *offset = (share_offset - entry->start) + entry->offset; + *object = entry->object.vm_object; + + /* + * Return whether this is the only map sharing this data. + */ + + if (!su) { + simple_lock(&share_map->ref_lock); + su = (share_map->ref_count == 1); + simple_unlock(&share_map->ref_lock); + } + + *out_prot = prot; + *single_use = su; + + return(KERN_SUCCESS); + +#undef RETURN +} + +/* + * vm_map_lookup_done: + * + * Releases locks acquired by a vm_map_lookup + * (according to the handle returned by that lookup). + */ + +void +vm_map_lookup_done(map, entry) + register vm_map_t map; + vm_map_entry_t entry; +{ + /* + * If this entry references a map, unlock it first. + */ + + if (entry->is_a_map) + vm_map_unlock_read(entry->object.share_map); + + /* + * Unlock the main-level map + */ + + vm_map_unlock_read(map); +} + +/* + * Routine: vm_map_simplify + * Purpose: + * Attempt to simplify the map representation in + * the vicinity of the given starting address. + * Note: + * This routine is intended primarily to keep the + * kernel maps more compact -- they generally don't + * benefit from the "expand a map entry" technology + * at allocation time because the adjacent entry + * is often wired down. + */ +void +vm_map_simplify(map, start) + vm_map_t map; + vm_offset_t start; +{ + vm_map_entry_t this_entry; + vm_map_entry_t prev_entry; + + vm_map_lock(map); + if ( + (vm_map_lookup_entry(map, start, &this_entry)) && + ((prev_entry = this_entry->prev) != &map->header) && + + (prev_entry->end == start) && + (map->is_main_map) && + + (prev_entry->is_a_map == FALSE) && + (prev_entry->is_sub_map == FALSE) && + + (this_entry->is_a_map == FALSE) && + (this_entry->is_sub_map == FALSE) && + + (prev_entry->inheritance == this_entry->inheritance) && + (prev_entry->protection == this_entry->protection) && + (prev_entry->max_protection == this_entry->max_protection) && + (prev_entry->wired_count == this_entry->wired_count) && + + (prev_entry->copy_on_write == this_entry->copy_on_write) && + (prev_entry->needs_copy == this_entry->needs_copy) && + + (prev_entry->object.vm_object == this_entry->object.vm_object) && + ((prev_entry->offset + (prev_entry->end - prev_entry->start)) + == this_entry->offset) + ) { + if (map->first_free == this_entry) + map->first_free = prev_entry; + + SAVE_HINT(map, prev_entry); + vm_map_entry_unlink(map, this_entry); + prev_entry->end = this_entry->end; + vm_object_deallocate(this_entry->object.vm_object); + vm_map_entry_dispose(map, this_entry); + } + vm_map_unlock(map); +} + +/* + * vm_map_print: [ debug ] + */ +void +vm_map_print(map, full) + register vm_map_t map; + boolean_t full; +{ + extern void _vm_map_print(); + + _vm_map_print(map, full, printf); +} + +void +_vm_map_print(map, full, pr) + register vm_map_t map; + boolean_t full; + void (*pr) __P((const char *, ...)); +{ + register vm_map_entry_t entry; + extern int indent; + + iprintf(pr, "%s map 0x%lx: pmap=0x%lx,ref=%d,nentries=%d,version=%d\n", + (map->is_main_map ? "Task" : "Share"), + (long) map, (long) (map->pmap), map->ref_count, map->nentries, + map->timestamp); + + if (!full && indent) + return; + + indent += 2; + for (entry = map->header.next; entry != &map->header; + entry = entry->next) { + iprintf(pr, "map entry 0x%lx: start=0x%lx, end=0x%lx, ", + (long) entry, (long) entry->start, (long) entry->end); + if (map->is_main_map) { + static char *inheritance_name[4] = + { "share", "copy", "none", "donate_copy"}; + (*pr)("prot=%x/%x/%s, ", + entry->protection, + entry->max_protection, + inheritance_name[entry->inheritance]); + if (entry->wired_count != 0) + (*pr)("wired, "); + } + + if (entry->is_a_map || entry->is_sub_map) { + (*pr)("share=0x%lx, offset=0x%lx\n", + (long) entry->object.share_map, + (long) entry->offset); + if ((entry->prev == &map->header) || + (!entry->prev->is_a_map) || + (entry->prev->object.share_map != + entry->object.share_map)) { + indent += 2; + vm_map_print(entry->object.share_map, full); + indent -= 2; + } + + } + else { + (*pr)("object=0x%lx, offset=0x%lx", + (long) entry->object.vm_object, + (long) entry->offset); + if (entry->copy_on_write) + (*pr)(", copy (%s)", + entry->needs_copy ? "needed" : "done"); + (*pr)("\n"); + + if ((entry->prev == &map->header) || + (entry->prev->is_a_map) || + (entry->prev->object.vm_object != + entry->object.vm_object)) { + indent += 2; + _vm_object_print(entry->object.vm_object, + full, pr); + indent -= 2; + } + } + } + indent -= 2; +} diff --git a/sys/vm/vm_map.h b/sys/vm/vm_map.h new file mode 100644 index 00000000000..63c47111c4f --- /dev/null +++ b/sys/vm/vm_map.h @@ -0,0 +1,232 @@ +/* $NetBSD: vm_map.h,v 1.11 1995/03/26 20:39:10 jtc Exp $ */ + +/* + * Copyright (c) 1991, 1993 + * The Regents of the University of California. All rights reserved. + * + * This code is derived from software contributed to Berkeley by + * The Mach Operating System project at Carnegie-Mellon University. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. All advertising materials mentioning features or use of this software + * must display the following acknowledgement: + * This product includes software developed by the University of + * California, Berkeley and its contributors. + * 4. Neither the name of the University nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * @(#)vm_map.h 8.3 (Berkeley) 3/15/94 + * + * + * Copyright (c) 1987, 1990 Carnegie-Mellon University. + * All rights reserved. + * + * Authors: Avadis Tevanian, Jr., Michael Wayne Young + * + * Permission to use, copy, modify and distribute this software and + * its documentation is hereby granted, provided that both the copyright + * notice and this permission notice appear in all copies of the + * software, derivative works or modified versions, and any portions + * thereof, and that both notices appear in supporting documentation. + * + * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" + * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND + * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. + * + * Carnegie Mellon requests users of this software to return to + * + * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU + * School of Computer Science + * Carnegie Mellon University + * Pittsburgh PA 15213-3890 + * + * any improvements or extensions that they make and grant Carnegie the + * rights to redistribute these changes. + */ + +/* + * Virtual memory map module definitions. + */ + +#ifndef _VM_MAP_ +#define _VM_MAP_ + +/* + * Types defined: + * + * vm_map_t the high-level address map data structure. + * vm_map_entry_t an entry in an address map. + * vm_map_version_t a timestamp of a map, for use with vm_map_lookup + */ + +/* + * Objects which live in maps may be either VM objects, or + * another map (called a "sharing map") which denotes read-write + * sharing with other maps. + */ + +union vm_map_object { + struct vm_object *vm_object; /* object object */ + struct vm_map *share_map; /* share map */ + struct vm_map *sub_map; /* belongs to another map */ +}; + +/* + * Address map entries consist of start and end addresses, + * a VM object (or sharing map) and offset into that object, + * and user-exported inheritance and protection information. + * Also included is control information for virtual copy operations. + */ +struct vm_map_entry { + struct vm_map_entry *prev; /* previous entry */ + struct vm_map_entry *next; /* next entry */ + vm_offset_t start; /* start address */ + vm_offset_t end; /* end address */ + union vm_map_object object; /* object I point to */ + vm_offset_t offset; /* offset into object */ + boolean_t is_a_map; /* Is "object" a map? */ + boolean_t is_sub_map; /* Is "object" a submap? */ + /* Only in sharing maps: */ + boolean_t copy_on_write; /* is data copy-on-write */ + boolean_t needs_copy; /* does object need to be copied */ + /* Only in task maps: */ + vm_prot_t protection; /* protection code */ + vm_prot_t max_protection; /* maximum protection */ + vm_inherit_t inheritance; /* inheritance */ + int wired_count; /* can be paged if = 0 */ +}; + +/* + * Maps are doubly-linked lists of map entries, kept sorted + * by address. A single hint is provided to start + * searches again from the last successful search, + * insertion, or removal. + */ +struct vm_map { + struct pmap * pmap; /* Physical map */ + lock_data_t lock; /* Lock for map data */ + struct vm_map_entry header; /* List of entries */ + int nentries; /* Number of entries */ + vm_size_t size; /* virtual size */ + boolean_t is_main_map; /* Am I a main map? */ + int ref_count; /* Reference count */ + simple_lock_data_t ref_lock; /* Lock for ref_count field */ + vm_map_entry_t hint; /* hint for quick lookups */ + simple_lock_data_t hint_lock; /* lock for hint storage */ + vm_map_entry_t first_free; /* First free space hint */ + boolean_t entries_pageable; /* map entries pageable?? */ + unsigned int timestamp; /* Version number */ +#define min_offset header.start +#define max_offset header.end +}; + +/* + * Map versions are used to validate a previous lookup attempt. + * + * Since lookup operations may involve both a main map and + * a sharing map, it is necessary to have a timestamp from each. + * [If the main map timestamp has changed, the share_map and + * associated timestamp are no longer valid; the map version + * does not include a reference for the imbedded share_map.] + */ +typedef struct { + int main_timestamp; + vm_map_t share_map; + int share_timestamp; +} vm_map_version_t; + +/* + * Macros: vm_map_lock, etc. + * Function: + * Perform locking on the data portion of a map. + */ + +#define vm_map_lock(map) { \ + lock_write(&(map)->lock); \ + (map)->timestamp++; \ +} +#define vm_map_unlock(map) lock_write_done(&(map)->lock) +#define vm_map_lock_read(map) lock_read(&(map)->lock) +#define vm_map_unlock_read(map) lock_read_done(&(map)->lock) + +/* + * Functions implemented as macros + */ +#define vm_map_min(map) ((map)->min_offset) +#define vm_map_max(map) ((map)->max_offset) +#define vm_map_pmap(map) ((map)->pmap) + +/* XXX: number of kernel maps and entries to statically allocate */ +#define MAX_KMAP 10 +#define MAX_KMAPENT 500 + +#ifdef _KERNEL +boolean_t vm_map_check_protection __P((vm_map_t, + vm_offset_t, vm_offset_t, vm_prot_t)); +int vm_map_copy __P((vm_map_t, vm_map_t, vm_offset_t, + vm_size_t, vm_offset_t, boolean_t, boolean_t)); +void vm_map_copy_entry __P((vm_map_t, + vm_map_t, vm_map_entry_t, vm_map_entry_t)); +struct pmap; +vm_map_t vm_map_create __P((struct pmap *, + vm_offset_t, vm_offset_t, boolean_t)); +void vm_map_deallocate __P((vm_map_t)); +int vm_map_delete __P((vm_map_t, vm_offset_t, vm_offset_t)); +vm_map_entry_t vm_map_entry_create __P((vm_map_t)); +void vm_map_entry_delete __P((vm_map_t, vm_map_entry_t)); +void vm_map_entry_dispose __P((vm_map_t, vm_map_entry_t)); +void vm_map_entry_unwire __P((vm_map_t, vm_map_entry_t)); +int vm_map_find __P((vm_map_t, vm_object_t, + vm_offset_t, vm_offset_t *, vm_size_t, boolean_t)); +int vm_map_findspace __P((vm_map_t, + vm_offset_t, vm_size_t, vm_offset_t *)); +int vm_map_inherit __P((vm_map_t, + vm_offset_t, vm_offset_t, vm_inherit_t)); +void vm_map_init __P((struct vm_map *, + vm_offset_t, vm_offset_t, boolean_t)); +int vm_map_insert __P((vm_map_t, + vm_object_t, vm_offset_t, vm_offset_t, vm_offset_t)); +int vm_map_lookup __P((vm_map_t *, vm_offset_t, vm_prot_t, + vm_map_entry_t *, vm_object_t *, vm_offset_t *, vm_prot_t *, + boolean_t *, boolean_t *)); +void vm_map_lookup_done __P((vm_map_t, vm_map_entry_t)); +boolean_t vm_map_lookup_entry __P((vm_map_t, + vm_offset_t, vm_map_entry_t *)); +int vm_map_pageable __P((vm_map_t, + vm_offset_t, vm_offset_t, boolean_t)); +int vm_map_clean __P((vm_map_t, + vm_offset_t, vm_offset_t, boolean_t, boolean_t)); +void vm_map_print __P((vm_map_t, boolean_t)); +void _vm_map_print __P((vm_map_t, boolean_t, + void (*)(const char *, ...))); +int vm_map_protect __P((vm_map_t, + vm_offset_t, vm_offset_t, vm_prot_t, boolean_t)); +void vm_map_reference __P((vm_map_t)); +int vm_map_remove __P((vm_map_t, vm_offset_t, vm_offset_t)); +void vm_map_simplify __P((vm_map_t, vm_offset_t)); +void vm_map_simplify_entry __P((vm_map_t, vm_map_entry_t)); +void vm_map_startup __P((void)); +int vm_map_submap __P((vm_map_t, + vm_offset_t, vm_offset_t, vm_map_t)); +#endif +#endif /* _VM_MAP_ */ diff --git a/sys/vm/vm_meter.c b/sys/vm/vm_meter.c new file mode 100644 index 00000000000..bcaf79423c3 --- /dev/null +++ b/sys/vm/vm_meter.c @@ -0,0 +1,228 @@ +/* $NetBSD: vm_meter.c,v 1.17 1995/07/08 03:12:22 cgd Exp $ */ + +/* + * Copyright (c) 1982, 1986, 1989, 1993 + * The Regents of the University of California. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. All advertising materials mentioning features or use of this software + * must display the following acknowledgement: + * This product includes software developed by the University of + * California, Berkeley and its contributors. + * 4. Neither the name of the University nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * @(#)vm_meter.c 8.4 (Berkeley) 1/4/94 + */ + +#include <sys/param.h> +#include <sys/proc.h> +#include <sys/systm.h> +#include <sys/kernel.h> +#include <vm/vm.h> +#include <sys/sysctl.h> + +struct loadavg averunnable; /* load average, of runnable procs */ + +int maxslp = MAXSLP; +#ifndef MACHINE_NONCONTIG +int saferss = SAFERSS; +#endif /* MACHINE_NONCONTIG */ + +void +vmmeter() +{ + + if (time.tv_sec % 5 == 0) + loadav(&averunnable); + if (proc0.p_slptime > maxslp/2) + wakeup((caddr_t)&proc0); +} + +/* + * Constants for averages over 1, 5, and 15 minutes + * when sampling at 5 second intervals. + */ +fixpt_t cexp[3] = { + 0.9200444146293232 * FSCALE, /* exp(-1/12) */ + 0.9834714538216174 * FSCALE, /* exp(-1/60) */ + 0.9944598480048967 * FSCALE, /* exp(-1/180) */ +}; + +/* + * Compute a tenex style load average of a quantity on + * 1, 5 and 15 minute intervals. + */ +void +loadav(avg) + register struct loadavg *avg; +{ + register int i, nrun; + register struct proc *p; + + for (nrun = 0, p = allproc.lh_first; p != 0; p = p->p_list.le_next) { + switch (p->p_stat) { + case SSLEEP: + if (p->p_priority > PZERO || p->p_slptime > 1) + continue; + /* fall through */ + case SRUN: + case SIDL: + nrun++; + } + } + for (i = 0; i < 3; i++) + avg->ldavg[i] = (cexp[i] * avg->ldavg[i] + + nrun * FSCALE * (FSCALE - cexp[i])) >> FSHIFT; +} + +/* + * Attributes associated with virtual memory. + */ +vm_sysctl(name, namelen, oldp, oldlenp, newp, newlen, p) + int *name; + u_int namelen; + void *oldp; + size_t *oldlenp; + void *newp; + size_t newlen; + struct proc *p; +{ + struct vmtotal vmtotals; + + /* all sysctl names at this level are terminal */ + if (namelen != 1) + return (ENOTDIR); /* overloaded */ + + switch (name[0]) { + case VM_LOADAVG: + averunnable.fscale = FSCALE; + return (sysctl_rdstruct(oldp, oldlenp, newp, &averunnable, + sizeof(averunnable))); + case VM_METER: + vmtotal(&vmtotals); + return (sysctl_rdstruct(oldp, oldlenp, newp, &vmtotals, + sizeof(vmtotals))); + default: + return (EOPNOTSUPP); + } + /* NOTREACHED */ +} + +/* + * Calculate the current state of the system. + * Done on demand from getkerninfo(). + */ +void +vmtotal(totalp) + register struct vmtotal *totalp; +{ + register struct proc *p; + register vm_map_entry_t entry; + register vm_object_t object; + register vm_map_t map; + int paging; + + bzero(totalp, sizeof *totalp); + /* + * Mark all objects as inactive. + */ + simple_lock(&vm_object_list_lock); + for (object = vm_object_list.tqh_first; + object != NULL; + object = object->object_list.tqe_next) + object->flags &= ~OBJ_ACTIVE; + simple_unlock(&vm_object_list_lock); + /* + * Calculate process statistics. + */ + for (p = allproc.lh_first; p != 0; p = p->p_list.le_next) { + if (p->p_flag & P_SYSTEM) + continue; + switch (p->p_stat) { + case 0: + continue; + + case SSLEEP: + case SSTOP: + if (p->p_flag & P_INMEM) { + if (p->p_priority <= PZERO) + totalp->t_dw++; + else if (p->p_slptime < maxslp) + totalp->t_sl++; + } else if (p->p_slptime < maxslp) + totalp->t_sw++; + if (p->p_slptime >= maxslp) + continue; + break; + + case SRUN: + case SIDL: + if (p->p_flag & P_INMEM) + totalp->t_rq++; + else + totalp->t_sw++; + if (p->p_stat == SIDL) + continue; + break; + } + /* + * Note active objects. + */ + paging = 0; + for (map = &p->p_vmspace->vm_map, entry = map->header.next; + entry != &map->header; entry = entry->next) { + if (entry->is_a_map || entry->is_sub_map || + entry->object.vm_object == NULL) + continue; + entry->object.vm_object->flags |= OBJ_ACTIVE; + paging |= entry->object.vm_object->paging_in_progress; + } + if (paging) + totalp->t_pw++; + } + /* + * Calculate object memory usage statistics. + */ + simple_lock(&vm_object_list_lock); + for (object = vm_object_list.tqh_first; + object != NULL; + object = object->object_list.tqe_next) { + totalp->t_vm += num_pages(object->size); + totalp->t_rm += object->resident_page_count; + if (object->flags & OBJ_ACTIVE) { + totalp->t_avm += num_pages(object->size); + totalp->t_arm += object->resident_page_count; + } + if (object->ref_count > 1) { + /* shared object */ + totalp->t_vmshr += num_pages(object->size); + totalp->t_rmshr += object->resident_page_count; + if (object->flags & OBJ_ACTIVE) { + totalp->t_avmshr += num_pages(object->size); + totalp->t_armshr += object->resident_page_count; + } + } + } + totalp->t_free = cnt.v_free_count; +} diff --git a/sys/vm/vm_mmap.c b/sys/vm/vm_mmap.c new file mode 100644 index 00000000000..ea205439963 --- /dev/null +++ b/sys/vm/vm_mmap.c @@ -0,0 +1,981 @@ +/* $NetBSD: vm_mmap.c,v 1.42 1995/10/10 01:27:11 mycroft Exp $ */ + +/* + * Copyright (c) 1988 University of Utah. + * Copyright (c) 1991, 1993 + * The Regents of the University of California. All rights reserved. + * + * This code is derived from software contributed to Berkeley by + * the Systems Programming Group of the University of Utah Computer + * Science Department. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. All advertising materials mentioning features or use of this software + * must display the following acknowledgement: + * This product includes software developed by the University of + * California, Berkeley and its contributors. + * 4. Neither the name of the University nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * from: Utah $Hdr: vm_mmap.c 1.6 91/10/21$ + * + * @(#)vm_mmap.c 8.5 (Berkeley) 5/19/94 + */ + +/* + * Mapped file (mmap) interface to VM + */ + +#include <sys/param.h> +#include <sys/systm.h> +#include <sys/filedesc.h> +#include <sys/resourcevar.h> +#include <sys/proc.h> +#include <sys/vnode.h> +#include <sys/file.h> +#include <sys/mman.h> +#include <sys/conf.h> + +#include <sys/mount.h> +#include <sys/syscallargs.h> + +#include <miscfs/specfs/specdev.h> + +#include <vm/vm.h> +#include <vm/vm_pager.h> +#include <vm/vm_prot.h> + +#ifdef DEBUG +int mmapdebug = 0; +#define MDB_FOLLOW 0x01 +#define MDB_SYNC 0x02 +#define MDB_MAPIT 0x04 +#endif + +/* ARGSUSED */ +int +sys_sbrk(p, v, retval) + struct proc *p; + void *v; + register_t *retval; +{ + struct sys_sbrk_args /* { + syscallarg(int) incr; + } */ *uap = v; + + /* Not yet implemented */ + return (EOPNOTSUPP); +} + +/* ARGSUSED */ +int +sys_sstk(p, v, retval) + struct proc *p; + void *v; + register_t *retval; +{ + struct sys_sstk_args /* { + syscallarg(int) incr; + } */ *uap = v; + + /* Not yet implemented */ + return (EOPNOTSUPP); +} + +#if defined(COMPAT_43) || defined(COMPAT_SUNOS) || defined(COMPAT_OSF1) || \ + defined(COMPAT_FREEBSD) +/* ARGSUSED */ +int +compat_43_sys_getpagesize(p, v, retval) + struct proc *p; + void *v; + register_t *retval; +{ + + *retval = PAGE_SIZE; + return (0); +} +#endif /* COMPAT_43 || COMPAT_SUNOS || COMPAT_OSF1 || COMPAT_FREEBSD */ + +#if defined(COMPAT_43) || defined(COMPAT_FREEBSD) +int +compat_43_sys_mmap(p, v, retval) + struct proc *p; + void *v; + register_t *retval; +{ + register struct compat_43_sys_mmap_args /* { + syscallarg(caddr_t) addr; + syscallarg(size_t) len; + syscallarg(int) prot; + syscallarg(int) flags; + syscallarg(int) fd; + syscallarg(long) pos; + } */ *uap = v; + struct sys_mmap_args /* { + syscallarg(caddr_t) addr; + syscallarg(size_t) len; + syscallarg(int) prot; + syscallarg(int) flags; + syscallarg(int) fd; + syscallarg(long) pad; + syscallarg(off_t) pos; + } */ nargs; + static const char cvtbsdprot[8] = { + 0, + PROT_EXEC, + PROT_WRITE, + PROT_EXEC|PROT_WRITE, + PROT_READ, + PROT_EXEC|PROT_READ, + PROT_WRITE|PROT_READ, + PROT_EXEC|PROT_WRITE|PROT_READ, + }; +#define OMAP_ANON 0x0002 +#define OMAP_COPY 0x0020 +#define OMAP_SHARED 0x0010 +#define OMAP_FIXED 0x0100 +#define OMAP_INHERIT 0x0800 + + SCARG(&nargs, addr) = SCARG(uap, addr); + SCARG(&nargs, len) = SCARG(uap, len); + SCARG(&nargs, prot) = cvtbsdprot[SCARG(uap, prot)&0x7]; + SCARG(&nargs, flags) = 0; + if (SCARG(uap, flags) & OMAP_ANON) + SCARG(&nargs, flags) |= MAP_ANON; + if (SCARG(uap, flags) & OMAP_COPY) + SCARG(&nargs, flags) |= MAP_COPY; + if (SCARG(uap, flags) & OMAP_SHARED) + SCARG(&nargs, flags) |= MAP_SHARED; + else + SCARG(&nargs, flags) |= MAP_PRIVATE; + if (SCARG(uap, flags) & OMAP_FIXED) + SCARG(&nargs, flags) |= MAP_FIXED; + if (SCARG(uap, flags) & OMAP_INHERIT) + SCARG(&nargs, flags) |= MAP_INHERIT; + SCARG(&nargs, fd) = SCARG(uap, fd); + SCARG(&nargs, pos) = SCARG(uap, pos); + return (sys_mmap(p, &nargs, retval)); +} +#endif + +/* + * Memory Map (mmap) system call. Note that the file offset + * and address are allowed to be NOT page aligned, though if + * the MAP_FIXED flag it set, both must have the same remainder + * modulo the PAGE_SIZE (POSIX 1003.1b). If the address is not + * page-aligned, the actual mapping starts at trunc_page(addr) + * and the return value is adjusted up by the page offset. + */ +int +sys_mmap(p, v, retval) + struct proc *p; + void *v; + register_t *retval; +{ + register struct sys_mmap_args /* { + syscallarg(caddr_t) addr; + syscallarg(size_t) len; + syscallarg(int) prot; + syscallarg(int) flags; + syscallarg(int) fd; + syscallarg(long) pad; + syscallarg(off_t) pos; + } */ *uap = v; + register struct filedesc *fdp = p->p_fd; + register struct file *fp; + struct vnode *vp; + vm_offset_t addr, pos; + vm_size_t size, pageoff; + vm_prot_t prot, maxprot; + caddr_t handle; + int fd, flags, error; + + addr = (vm_offset_t) SCARG(uap, addr); + size = (vm_size_t) SCARG(uap, len); + prot = SCARG(uap, prot) & VM_PROT_ALL; + flags = SCARG(uap, flags); + fd = SCARG(uap, fd); + pos = (vm_offset_t) SCARG(uap, pos); + +#ifdef DEBUG + if (mmapdebug & MDB_FOLLOW) + printf("mmap(%d): addr %x len %x pro %x flg %x fd %d pos %x\n", + p->p_pid, addr, size, prot, flags, fd, pos); +#endif + + /* + * Align the file position to a page boundary, + * and save its page offset component. + */ + pageoff = (pos & PAGE_MASK); + pos -= pageoff; + + /* Adjust size for rounding (on both ends). */ + size += pageoff; /* low end... */ + size = (vm_size_t) round_page(size); /* hi end */ + + /* Do not allow mappings that cause address wrap... */ + if ((ssize_t)size < 0) + return (EINVAL); + + /* + * Check for illegal addresses. Watch out for address wrap... + * Note that VM_*_ADDRESS are not constants due to casts (argh). + */ + if (flags & MAP_FIXED) { + /* + * The specified address must have the same remainder + * as the file offset taken modulo PAGE_SIZE, so it + * should be aligned after adjustment by pageoff. + */ + addr -= pageoff; + if (addr & PAGE_MASK) + return (EINVAL); + /* Address range must be all in user VM space. */ + if (VM_MAXUSER_ADDRESS > 0 && addr + size > VM_MAXUSER_ADDRESS) + return (EINVAL); + if (VM_MIN_ADDRESS > 0 && addr < VM_MIN_ADDRESS) + return (EINVAL); + if (addr > addr + size) + return (EINVAL); + } + /* + * XXX for non-fixed mappings where no hint is provided or + * the hint would fall in the potential heap space, + * place it after the end of the largest possible heap. + * + * There should really be a pmap call to determine a reasonable + * location. (To avoid VA cache alias problems, for example!) + */ + else if (addr < round_page(p->p_vmspace->vm_daddr + MAXDSIZ)) + addr = round_page(p->p_vmspace->vm_daddr + MAXDSIZ); + + if ((flags & MAP_ANON) == 0) { + /* + * Mapping file, get fp for validation. + * Obtain vnode and make sure it is of appropriate type. + */ + if (((unsigned)fd) >= fdp->fd_nfiles || + (fp = fdp->fd_ofiles[fd]) == NULL) + return (EBADF); + if (fp->f_type != DTYPE_VNODE) + return (EINVAL); + vp = (struct vnode *)fp->f_data; + if (vp->v_type != VREG && vp->v_type != VCHR) + return (EINVAL); + /* + * XXX hack to handle use of /dev/zero to map anon + * memory (ala SunOS). + */ + if (vp->v_type == VCHR && iszerodev(vp->v_rdev)) { + flags |= MAP_ANON; + goto is_anon; + } + /* + * Ensure that file and memory protections are + * compatible. Note that we only worry about + * writability if mapping is shared; in this case, + * current and max prot are dictated by the open file. + * XXX use the vnode instead? Problem is: what + * credentials do we use for determination? + * What if proc does a setuid? + */ + maxprot = VM_PROT_EXECUTE; /* ??? */ + if (fp->f_flag & FREAD) + maxprot |= VM_PROT_READ; + else if (prot & PROT_READ) + return (EACCES); + if (flags & MAP_SHARED) { + if (fp->f_flag & FWRITE) + maxprot |= VM_PROT_WRITE; + else if (prot & PROT_WRITE) + return (EACCES); + } else + maxprot |= VM_PROT_WRITE; + handle = (caddr_t)vp; + } else { + /* + * (flags & MAP_ANON) == TRUE + * Mapping blank space is trivial. + */ + if (fd != -1) + return (EINVAL); + is_anon: + handle = NULL; + maxprot = VM_PROT_ALL; + pos = 0; + } + error = vm_mmap(&p->p_vmspace->vm_map, &addr, size, prot, maxprot, + flags, handle, pos); + if (error == 0) + *retval = (register_t)(addr + pageoff); + return (error); +} + +int +sys_msync(p, v, retval) + struct proc *p; + void *v; + register_t *retval; +{ + struct sys_msync_args /* { + syscallarg(caddr_t) addr; + syscallarg(size_t) len; + } */ *uap = v; + vm_offset_t addr; + vm_size_t size, pageoff; + vm_map_t map; + int rv; + boolean_t syncio, invalidate; + + addr = (vm_offset_t)SCARG(uap, addr); + size = (vm_size_t)SCARG(uap, len); +#ifdef DEBUG + if (mmapdebug & (MDB_FOLLOW|MDB_SYNC)) + printf("msync(%d): addr %x len %x\n", + p->p_pid, addr, size); +#endif + + /* + * Align the address to a page boundary, + * and adjust the size accordingly. + */ + pageoff = (addr & PAGE_MASK); + addr -= pageoff; + size += pageoff; + size = (vm_size_t) round_page(size); + + /* Disallow wrap-around. */ + if (addr + (int)size < addr) + return (EINVAL); + + map = &p->p_vmspace->vm_map; + /* + * XXX Gak! If size is zero we are supposed to sync "all modified + * pages with the region containing addr". Unfortunately, we + * don't really keep track of individual mmaps so we approximate + * by flushing the range of the map entry containing addr. + * This can be incorrect if the region splits or is coalesced + * with a neighbor. + */ + if (size == 0) { + vm_map_entry_t entry; + + vm_map_lock_read(map); + rv = vm_map_lookup_entry(map, addr, &entry); + vm_map_unlock_read(map); + if (rv == FALSE) + return (EINVAL); + addr = entry->start; + size = entry->end - entry->start; + } +#ifdef DEBUG + if (mmapdebug & MDB_SYNC) + printf("msync: cleaning/flushing address range [%x-%x)\n", + addr, addr+size); +#endif + /* + * Could pass this in as a third flag argument to implement + * Sun's MS_ASYNC. + */ + syncio = TRUE; + /* + * XXX bummer, gotta flush all cached pages to ensure + * consistency with the file system cache. Otherwise, we could + * pass this in to implement Sun's MS_INVALIDATE. + */ + invalidate = TRUE; + /* + * Clean the pages and interpret the return value. + */ + rv = vm_map_clean(map, addr, addr+size, syncio, invalidate); + switch (rv) { + case KERN_SUCCESS: + break; + case KERN_INVALID_ADDRESS: + return (EINVAL); /* Sun returns ENOMEM? */ + case KERN_FAILURE: + return (EIO); + default: + return (EINVAL); + } + return (0); +} + +int +sys_munmap(p, v, retval) + register struct proc *p; + void *v; + register_t *retval; +{ + register struct sys_munmap_args /* { + syscallarg(caddr_t) addr; + syscallarg(size_t) len; + } */ *uap = v; + vm_offset_t addr; + vm_size_t size, pageoff; + vm_map_t map; + + addr = (vm_offset_t) SCARG(uap, addr); + size = (vm_size_t) SCARG(uap, len); +#ifdef DEBUG + if (mmapdebug & MDB_FOLLOW) + printf("munmap(%d): addr %x len %x\n", + p->p_pid, addr, size); +#endif + + /* + * Align the address to a page boundary, + * and adjust the size accordingly. + */ + pageoff = (addr & PAGE_MASK); + addr -= pageoff; + size += pageoff; + size = (vm_size_t) round_page(size); + if ((int)size < 0) + return(EINVAL); + if (size == 0) + return(0); + /* + * Check for illegal addresses. Watch out for address wrap... + * Note that VM_*_ADDRESS are not constants due to casts (argh). + */ + if (VM_MAXUSER_ADDRESS > 0 && addr + size > VM_MAXUSER_ADDRESS) + return (EINVAL); + if (VM_MIN_ADDRESS > 0 && addr < VM_MIN_ADDRESS) + return (EINVAL); + if (addr > addr + size) + return (EINVAL); + map = &p->p_vmspace->vm_map; + /* + * Make sure entire range is allocated. + */ + if (!vm_map_check_protection(map, addr, addr + size, VM_PROT_NONE)) + return(EINVAL); + /* returns nothing but KERN_SUCCESS anyway */ + (void) vm_map_remove(map, addr, addr+size); + return(0); +} + +void +munmapfd(p, fd) + struct proc *p; + int fd; +{ +#ifdef DEBUG + if (mmapdebug & MDB_FOLLOW) + printf("munmapfd(%d): fd %d\n", p->p_pid, fd); +#endif + + /* + * XXX should vm_deallocate any regions mapped to this file + */ + p->p_fd->fd_ofileflags[fd] &= ~UF_MAPPED; +} + +int +sys_mprotect(p, v, retval) + struct proc *p; + void *v; + register_t *retval; +{ + struct sys_mprotect_args /* { + syscallarg(caddr_t) addr; + syscallarg(int) len; + syscallarg(int) prot; + } */ *uap = v; + vm_offset_t addr; + vm_size_t size, pageoff; + register vm_prot_t prot; + + addr = (vm_offset_t)SCARG(uap, addr); + size = (vm_size_t)SCARG(uap, len); + prot = SCARG(uap, prot) & VM_PROT_ALL; +#ifdef DEBUG + if (mmapdebug & MDB_FOLLOW) + printf("mprotect(%d): addr %x len %x prot %d\n", p->p_pid, + addr, size, prot); +#endif + /* + * Align the address to a page boundary, + * and adjust the size accordingly. + */ + pageoff = (addr & PAGE_MASK); + addr -= pageoff; + size += pageoff; + size = (vm_size_t) round_page(size); + if ((int)size < 0) + return(EINVAL); + + switch (vm_map_protect(&p->p_vmspace->vm_map, addr, addr+size, prot, + FALSE)) { + case KERN_SUCCESS: + return (0); + case KERN_PROTECTION_FAILURE: + return (EACCES); + } + return (EINVAL); +} + +/* ARGSUSED */ +int +sys_madvise(p, v, retval) + struct proc *p; + void *v; + register_t *retval; +{ + struct sys_madvise_args /* { + syscallarg(caddr_t) addr; + syscallarg(size_t) len; + syscallarg(int) behav; + } */ *uap = v; + + /* Not yet implemented */ + return (EOPNOTSUPP); +} + +/* ARGSUSED */ +int +sys_mincore(p, v, retval) + struct proc *p; + void *v; + register_t *retval; +{ + struct sys_mincore_args /* { + syscallarg(caddr_t) addr; + syscallarg(size_t) len; + syscallarg(char *) vec; + } */ *uap = v; + + /* Not yet implemented */ + return (EOPNOTSUPP); +} + +int +sys_mlock(p, v, retval) + struct proc *p; + void *v; + register_t *retval; +{ + struct sys_mlock_args /* { + syscallarg(caddr_t) addr; + syscallarg(size_t) len; + } */ *uap = v; + vm_offset_t addr; + vm_size_t size, pageoff; + int error; + extern int vm_page_max_wired; + + addr = (vm_offset_t)SCARG(uap, addr); + size = (vm_size_t)SCARG(uap, len); +#ifdef DEBUG + if (mmapdebug & MDB_FOLLOW) + printf("mlock(%d): addr %x len %x\n", + p->p_pid, addr, size); +#endif + /* + * Align the address to a page boundary, + * and adjust the size accordingly. + */ + pageoff = (addr & PAGE_MASK); + addr -= pageoff; + size += pageoff; + size = (vm_size_t) round_page(size); + + /* Disallow wrap-around. */ + if (addr + (int)size < addr) + return (EINVAL); + + if (atop(size) + cnt.v_wire_count > vm_page_max_wired) + return (EAGAIN); +#ifdef pmap_wired_count + if (size + ptoa(pmap_wired_count(vm_map_pmap(&p->p_vmspace->vm_map))) > + p->p_rlimit[RLIMIT_MEMLOCK].rlim_cur) + return (EAGAIN); +#else + if (error = suser(p->p_ucred, &p->p_acflag)) + return (error); +#endif + + error = vm_map_pageable(&p->p_vmspace->vm_map, addr, addr+size, FALSE); + return (error == KERN_SUCCESS ? 0 : ENOMEM); +} + +int +sys_munlock(p, v, retval) + struct proc *p; + void *v; + register_t *retval; +{ + struct sys_munlock_args /* { + syscallarg(caddr_t) addr; + syscallarg(size_t) len; + } */ *uap = v; + vm_offset_t addr; + vm_size_t size, pageoff; + int error; + + addr = (vm_offset_t)SCARG(uap, addr); + size = (vm_size_t)SCARG(uap, len); +#ifdef DEBUG + if (mmapdebug & MDB_FOLLOW) + printf("munlock(%d): addr %x len %x\n", + p->p_pid, addr, size); +#endif + /* + * Align the address to a page boundary, + * and adjust the size accordingly. + */ + pageoff = (addr & PAGE_MASK); + addr -= pageoff; + size += pageoff; + size = (vm_size_t) round_page(size); + + /* Disallow wrap-around. */ + if (addr + (int)size < addr) + return (EINVAL); + +#ifndef pmap_wired_count + if (error = suser(p->p_ucred, &p->p_acflag)) + return (error); +#endif + + error = vm_map_pageable(&p->p_vmspace->vm_map, addr, addr+size, TRUE); + return (error == KERN_SUCCESS ? 0 : ENOMEM); +} + +/* + * Internal version of mmap. + * Currently used by mmap, exec, and sys5 shared memory. + * Handle is either a vnode pointer or NULL for MAP_ANON. + * This (internal) interface requires the file offset to be + * page-aligned by the caller. (Also addr, if MAP_FIXED). + */ +int +vm_mmap(map, addr, size, prot, maxprot, flags, handle, foff) + register vm_map_t map; + register vm_offset_t *addr; + register vm_size_t size; + vm_prot_t prot, maxprot; + register int flags; + caddr_t handle; /* XXX should be vp */ + vm_offset_t foff; +{ + register vm_pager_t pager; + boolean_t fitit; + vm_object_t object; + struct vnode *vp = NULL; + int type; + int rv = KERN_SUCCESS; + + if (size == 0) + return (0); + + /* The file offset must be page aligned. */ + if (foff & PAGE_MASK) + return (EINVAL); + + if ((flags & MAP_FIXED) == 0) { + /* The address is just a hint */ + fitit = TRUE; + *addr = round_page(*addr); + } else { + /* + * Use the specified address exactly + * (but check alignment first). + */ + fitit = FALSE; + if (*addr & PAGE_MASK) + return (EINVAL); + (void)vm_deallocate(map, *addr, size); + } + + /* + * Lookup/allocate pager. All except an unnamed anonymous lookup + * gain a reference to ensure continued existance of the object. + * (XXX the exception is to appease the pageout daemon) + */ + if (flags & MAP_ANON) { + type = PG_DFLT; + foff = 0; + } else { + vp = (struct vnode *)handle; + if (vp->v_type == VCHR) { + type = PG_DEVICE; + handle = (caddr_t)(long)vp->v_rdev; + } else + type = PG_VNODE; + } + pager = vm_pager_allocate(type, handle, size, prot, foff); + if (pager == NULL) + return (type == PG_DEVICE ? EINVAL : ENOMEM); + /* + * Find object and release extra reference gained by lookup + */ + object = vm_object_lookup(pager); + vm_object_deallocate(object); + + /* + * Anonymous memory. + */ + if (flags & MAP_ANON) { + rv = vm_allocate_with_pager(map, addr, size, fitit, + pager, foff, TRUE); + if (rv != KERN_SUCCESS) { + if (handle == NULL) + vm_pager_deallocate(pager); + else + vm_object_deallocate(object); + goto out; + } + /* + * Don't cache anonymous objects. + * Loses the reference gained by vm_pager_allocate. + * Note that object will be NULL when handle == NULL, + * this is ok since vm_allocate_with_pager has made + * sure that these objects are uncached. + */ + (void) pager_cache(object, FALSE); +#ifdef DEBUG + if (mmapdebug & MDB_MAPIT) + printf("vm_mmap(%d): ANON *addr %x size %x pager %x\n", + curproc->p_pid, *addr, size, pager); +#endif + } + /* + * Must be a mapped file. + * Distinguish between character special and regular files. + */ + else if (vp->v_type == VCHR) { + rv = vm_allocate_with_pager(map, addr, size, fitit, + pager, foff, FALSE); + /* + * Uncache the object and lose the reference gained + * by vm_pager_allocate(). If the call to + * vm_allocate_with_pager() was sucessful, then we + * gained an additional reference ensuring the object + * will continue to exist. If the call failed then + * the deallocate call below will terminate the + * object which is fine. + */ + (void) pager_cache(object, FALSE); + if (rv != KERN_SUCCESS) + goto out; + } + /* + * A regular file + */ + else { +#ifdef DEBUG + if (object == NULL) + printf("vm_mmap: no object: vp %x, pager %x\n", + vp, pager); +#endif + /* + * Map it directly. + * Allows modifications to go out to the vnode. + */ + if (flags & MAP_SHARED) { + rv = vm_allocate_with_pager(map, addr, size, + fitit, pager, + foff, FALSE); + if (rv != KERN_SUCCESS) { + vm_object_deallocate(object); + goto out; + } + /* + * Don't cache the object. This is the easiest way + * of ensuring that data gets back to the filesystem + * because vnode_pager_deallocate() will fsync the + * vnode. pager_cache() will lose the extra ref. + */ + if (prot & VM_PROT_WRITE) + pager_cache(object, FALSE); + else + vm_object_deallocate(object); + } + /* + * Copy-on-write of file. Two flavors. + * MAP_COPY is true COW, you essentially get a snapshot of + * the region at the time of mapping. MAP_PRIVATE means only + * that your changes are not reflected back to the object. + * Changes made by others will be seen. + */ + else { + vm_map_t tmap; + vm_offset_t off; + + /* locate and allocate the target address space */ + if (fitit) { + /* + * We cannot call vm_map_find() because + * a proposed address may be vetoed by + * the pmap module. + * So we look for space ourselves, validate + * it and insert it into the map. + */ + vm_map_lock(map); + again: + if (vm_map_findspace(map, *addr, size, + addr) == 1) { + rv = KERN_NO_SPACE; + } else { + vm_object_prefer(object, foff, addr); + rv = vm_map_insert(map, NULL, + (vm_offset_t)0, + *addr, *addr+size); + if (rv == KERN_NO_SPACE) + /* + * Modified address didn't fit + * after all, the gap must + * have been to small. + */ + goto again; + } + vm_map_unlock(map); + } else { + rv = vm_map_find(map, NULL, (vm_offset_t)0, + addr, size, 0); + + /* + * Check against PMAP preferred address. If + * there's a mismatch, these pages should not + * be shared with others. <howto?> + */ + if (rv == KERN_SUCCESS) { + vm_offset_t paddr = *addr; + vm_object_prefer(object, foff, &paddr); + if (paddr != *addr) + printf("vm_mmap: pmap botch!\n"); + } + } + + if (rv != KERN_SUCCESS) { + vm_object_deallocate(object); + goto out; + } + tmap = vm_map_create(pmap_create(size), VM_MIN_ADDRESS, + VM_MIN_ADDRESS+size, TRUE); + off = VM_MIN_ADDRESS; + rv = vm_allocate_with_pager(tmap, &off, size, + TRUE, pager, + foff, FALSE); + if (rv != KERN_SUCCESS) { + vm_object_deallocate(object); + vm_map_deallocate(tmap); + goto out; + } + /* + * (XXX) + * MAP_PRIVATE implies that we see changes made by + * others. To ensure that we need to guarentee that + * no copy object is created (otherwise original + * pages would be pushed to the copy object and we + * would never see changes made by others). We + * totally sleeze it right now by marking the object + * internal temporarily. + */ + if ((flags & MAP_COPY) == 0) + object->flags |= OBJ_INTERNAL; + rv = vm_map_copy(map, tmap, *addr, size, off, + FALSE, FALSE); + object->flags &= ~OBJ_INTERNAL; + /* + * (XXX) + * My oh my, this only gets worse... + * Force creation of a shadow object so that + * vm_map_fork will do the right thing. + */ + if ((flags & MAP_COPY) == 0) { + vm_map_t tmap; + vm_map_entry_t tentry; + vm_object_t tobject; + vm_offset_t toffset; + vm_prot_t tprot; + boolean_t twired, tsu; + + tmap = map; + vm_map_lookup(&tmap, *addr, VM_PROT_WRITE, + &tentry, &tobject, &toffset, + &tprot, &twired, &tsu); + vm_map_lookup_done(tmap, tentry); + } + /* + * (XXX) + * Map copy code cannot detect sharing unless a + * sharing map is involved. So we cheat and write + * protect everything ourselves. + */ + vm_object_pmap_copy(object, foff, foff + size); + vm_object_deallocate(object); + vm_map_deallocate(tmap); + if (rv != KERN_SUCCESS) + goto out; + } +#ifdef DEBUG + if (mmapdebug & MDB_MAPIT) + printf("vm_mmap(%d): FILE *addr %x size %x pager %x\n", + curproc->p_pid, *addr, size, pager); +#endif + } + /* + * Correct protection (default is VM_PROT_ALL). + * If maxprot is different than prot, we must set both explicitly. + */ + rv = KERN_SUCCESS; + if (maxprot != VM_PROT_ALL) + rv = vm_map_protect(map, *addr, *addr+size, maxprot, TRUE); + if (rv == KERN_SUCCESS && prot != maxprot) + rv = vm_map_protect(map, *addr, *addr+size, prot, FALSE); + if (rv != KERN_SUCCESS) { + (void) vm_deallocate(map, *addr, size); + goto out; + } + /* + * Shared memory is also shared with children. + */ + if (flags & MAP_SHARED) { + rv = vm_map_inherit(map, *addr, *addr+size, VM_INHERIT_SHARE); + if (rv != KERN_SUCCESS) { + (void) vm_deallocate(map, *addr, size); + goto out; + } + } +out: +#ifdef DEBUG + if (mmapdebug & MDB_MAPIT) + printf("vm_mmap: rv %d\n", rv); +#endif + switch (rv) { + case KERN_SUCCESS: + return (0); + case KERN_INVALID_ADDRESS: + case KERN_NO_SPACE: + return (ENOMEM); + case KERN_PROTECTION_FAILURE: + return (EACCES); + default: + return (EINVAL); + } +} diff --git a/sys/vm/vm_object.c b/sys/vm/vm_object.c new file mode 100644 index 00000000000..2dc50bb47c8 --- /dev/null +++ b/sys/vm/vm_object.c @@ -0,0 +1,1502 @@ +/* $NetBSD: vm_object.c,v 1.29 1995/07/13 12:35:29 pk Exp $ */ + +/* + * Copyright (c) 1991, 1993 + * The Regents of the University of California. All rights reserved. + * + * This code is derived from software contributed to Berkeley by + * The Mach Operating System project at Carnegie-Mellon University. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. All advertising materials mentioning features or use of this software + * must display the following acknowledgement: + * This product includes software developed by the University of + * California, Berkeley and its contributors. + * 4. Neither the name of the University nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * @(#)vm_object.c 8.5 (Berkeley) 3/22/94 + * + * + * Copyright (c) 1987, 1990 Carnegie-Mellon University. + * All rights reserved. + * + * Authors: Avadis Tevanian, Jr., Michael Wayne Young + * + * Permission to use, copy, modify and distribute this software and + * its documentation is hereby granted, provided that both the copyright + * notice and this permission notice appear in all copies of the + * software, derivative works or modified versions, and any portions + * thereof, and that both notices appear in supporting documentation. + * + * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" + * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND + * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. + * + * Carnegie Mellon requests users of this software to return to + * + * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU + * School of Computer Science + * Carnegie Mellon University + * Pittsburgh PA 15213-3890 + * + * any improvements or extensions that they make and grant Carnegie the + * rights to redistribute these changes. + */ + +/* + * Virtual memory object module. + */ + +#include <sys/param.h> +#include <sys/systm.h> +#include <sys/malloc.h> + +#include <vm/vm.h> +#include <vm/vm_page.h> + +/* + * Virtual memory objects maintain the actual data + * associated with allocated virtual memory. A given + * page of memory exists within exactly one object. + * + * An object is only deallocated when all "references" + * are given up. Only one "reference" to a given + * region of an object should be writeable. + * + * Associated with each object is a list of all resident + * memory pages belonging to that object; this list is + * maintained by the "vm_page" module, and locked by the object's + * lock. + * + * Each object also records a "pager" routine which is + * used to retrieve (and store) pages to the proper backing + * storage. In addition, objects may be backed by other + * objects from which they were virtual-copied. + * + * The only items within the object structure which are + * modified after time of creation are: + * reference count locked by object's lock + * pager routine locked by object's lock + * + */ + +struct vm_object kernel_object_store; +struct vm_object kmem_object_store; + +#define VM_OBJECT_HASH_COUNT 157 + +int vm_cache_max = 100; /* can patch if necessary */ +struct vm_object_hash_head vm_object_hashtable[VM_OBJECT_HASH_COUNT]; + +long object_collapses = 0; +long object_bypasses = 0; + +static void _vm_object_allocate __P((vm_size_t, vm_object_t)); + +/* + * vm_object_init: + * + * Initialize the VM objects module. + */ +void +vm_object_init(size) + vm_size_t size; +{ + register int i; + + TAILQ_INIT(&vm_object_cached_list); + TAILQ_INIT(&vm_object_list); + vm_object_count = 0; + simple_lock_init(&vm_cache_lock); + simple_lock_init(&vm_object_list_lock); + + for (i = 0; i < VM_OBJECT_HASH_COUNT; i++) + TAILQ_INIT(&vm_object_hashtable[i]); + + kernel_object = &kernel_object_store; + _vm_object_allocate(size, kernel_object); + + kmem_object = &kmem_object_store; + _vm_object_allocate(VM_KMEM_SIZE + VM_MBUF_SIZE, kmem_object); +} + +/* + * vm_object_allocate: + * + * Returns a new object with the given size. + */ +vm_object_t +vm_object_allocate(size) + vm_size_t size; +{ + register vm_object_t result; + + result = (vm_object_t) + malloc((u_long)sizeof *result, M_VMOBJ, M_WAITOK); + + _vm_object_allocate(size, result); + + return(result); +} + +static void +_vm_object_allocate(size, object) + vm_size_t size; + register vm_object_t object; +{ + TAILQ_INIT(&object->memq); + vm_object_lock_init(object); + object->ref_count = 1; + object->resident_page_count = 0; + object->size = size; + object->flags = OBJ_INTERNAL; /* vm_allocate_with_pager will reset */ + object->paging_in_progress = 0; + object->copy = NULL; + + /* + * Object starts out read-write, with no pager. + */ + + object->pager = NULL; + object->paging_offset = 0; + object->shadow = NULL; + object->shadow_offset = (vm_offset_t) 0; + + simple_lock(&vm_object_list_lock); + TAILQ_INSERT_TAIL(&vm_object_list, object, object_list); + vm_object_count++; + cnt.v_nzfod += atop(size); + simple_unlock(&vm_object_list_lock); +} + +/* + * vm_object_reference: + * + * Gets another reference to the given object. + */ +void +vm_object_reference(object) + register vm_object_t object; +{ + if (object == NULL) + return; + + vm_object_lock(object); + object->ref_count++; + vm_object_unlock(object); +} + +/* + * vm_object_deallocate: + * + * Release a reference to the specified object, + * gained either through a vm_object_allocate + * or a vm_object_reference call. When all references + * are gone, storage associated with this object + * may be relinquished. + * + * No object may be locked. + */ +void +vm_object_deallocate(object) + register vm_object_t object; +{ + vm_object_t temp; + + while (object != NULL) { + + /* + * The cache holds a reference (uncounted) to + * the object; we must lock it before removing + * the object. + */ + + vm_object_cache_lock(); + + /* + * Lose the reference + */ + vm_object_lock(object); + if (--(object->ref_count) != 0) { + + /* + * If there are still references, then + * we are done. + */ + vm_object_unlock(object); + vm_object_cache_unlock(); + return; + } + + /* + * See if this object can persist. If so, enter + * it in the cache, then deactivate all of its + * pages. + */ + + if (object->flags & OBJ_CANPERSIST) { + + TAILQ_INSERT_TAIL(&vm_object_cached_list, object, + cached_list); + vm_object_cached++; + vm_object_cache_unlock(); + + vm_object_deactivate_pages(object); + vm_object_unlock(object); + + vm_object_cache_trim(); + return; + } + + /* + * Make sure no one can look us up now. + */ + vm_object_remove(object->pager); + vm_object_cache_unlock(); + + temp = object->shadow; + vm_object_terminate(object); + /* unlocks and deallocates object */ + object = temp; + } +} + + +/* + * vm_object_terminate actually destroys the specified object, freeing + * up all previously used resources. + * + * The object must be locked. + */ +void +vm_object_terminate(object) + register vm_object_t object; +{ + register vm_page_t p; + vm_object_t shadow_object; + + /* + * Detach the object from its shadow if we are the shadow's + * copy. + */ + if ((shadow_object = object->shadow) != NULL) { + vm_object_lock(shadow_object); + if (shadow_object->copy == object) + shadow_object->copy = NULL; +#if 0 + else if (shadow_object->copy != NULL) + panic("vm_object_terminate: copy/shadow inconsistency"); +#endif + vm_object_unlock(shadow_object); + } + + /* + * Wait until the pageout daemon is through with the object. + */ + while (object->paging_in_progress) { + vm_object_sleep(object, object, FALSE); + vm_object_lock(object); + } + + /* + * If not an internal object clean all the pages, removing them + * from paging queues as we go. + * + * XXX need to do something in the event of a cleaning error. + */ + if ((object->flags & OBJ_INTERNAL) == 0) { + (void) vm_object_page_clean(object, 0, 0, TRUE, TRUE); + vm_object_unlock(object); + } + + /* + * Now free the pages. + * For internal objects, this also removes them from paging queues. + */ + while ((p = object->memq.tqh_first) != NULL) { + VM_PAGE_CHECK(p); + vm_page_lock_queues(); + vm_page_free(p); + cnt.v_pfree++; + vm_page_unlock_queues(); + } + if ((object->flags & OBJ_INTERNAL) != 0) + vm_object_unlock(object); + + /* + * Let the pager know object is dead. + */ + if (object->pager != NULL) + vm_pager_deallocate(object->pager); + + simple_lock(&vm_object_list_lock); + TAILQ_REMOVE(&vm_object_list, object, object_list); + vm_object_count--; + simple_unlock(&vm_object_list_lock); + + /* + * Free the space for the object. + */ + free((caddr_t)object, M_VMOBJ); +} + +/* + * vm_object_page_clean + * + * Clean all dirty pages in the specified range of object. + * If syncio is TRUE, page cleaning is done synchronously. + * If de_queue is TRUE, pages are removed from any paging queue + * they were on, otherwise they are left on whatever queue they + * were on before the cleaning operation began. + * + * Odd semantics: if start == end, we clean everything. + * + * The object must be locked. + * + * Returns TRUE if all was well, FALSE if there was a pager error + * somewhere. We attempt to clean (and dequeue) all pages regardless + * of where an error occurs. + */ +boolean_t +vm_object_page_clean(object, start, end, syncio, de_queue) + register vm_object_t object; + register vm_offset_t start; + register vm_offset_t end; + boolean_t syncio; + boolean_t de_queue; +{ + register vm_page_t p; + int onqueue; + boolean_t noerror = TRUE; + + if (object == NULL) + return (TRUE); + + /* + * If it is an internal object and there is no pager, attempt to + * allocate one. Note that vm_object_collapse may relocate one + * from a collapsed object so we must recheck afterward. + */ + if ((object->flags & OBJ_INTERNAL) && object->pager == NULL) { + vm_object_collapse(object); + if (object->pager == NULL) { + vm_pager_t pager; + + vm_object_unlock(object); + pager = vm_pager_allocate(PG_DFLT, (caddr_t)0, + object->size, VM_PROT_ALL, + (vm_offset_t)0); + if (pager) + vm_object_setpager(object, pager, 0, FALSE); + vm_object_lock(object); + } + } + if (object->pager == NULL) + return (FALSE); + +again: + /* + * Wait until the pageout daemon is through with the object. + */ + while (object->paging_in_progress) { + vm_object_sleep(object, object, FALSE); + vm_object_lock(object); + } + /* + * Loop through the object page list cleaning as necessary. + */ + for (p = object->memq.tqh_first; p != NULL; p = p->listq.tqe_next) { + if ((start == end || p->offset >= start && p->offset < end) && + !(p->flags & PG_FICTITIOUS)) { + if ((p->flags & PG_CLEAN) && + pmap_is_modified(VM_PAGE_TO_PHYS(p))) + p->flags &= ~PG_CLEAN; + /* + * Remove the page from any paging queue. + * This needs to be done if either we have been + * explicitly asked to do so or it is about to + * be cleaned (see comment below). + */ + if (de_queue || !(p->flags & PG_CLEAN)) { + vm_page_lock_queues(); + if (p->flags & PG_ACTIVE) { + TAILQ_REMOVE(&vm_page_queue_active, + p, pageq); + p->flags &= ~PG_ACTIVE; + cnt.v_active_count--; + onqueue = 1; + } else if (p->flags & PG_INACTIVE) { + TAILQ_REMOVE(&vm_page_queue_inactive, + p, pageq); + p->flags &= ~PG_INACTIVE; + cnt.v_inactive_count--; + onqueue = -1; + } else + onqueue = 0; + vm_page_unlock_queues(); + } + /* + * To ensure the state of the page doesn't change + * during the clean operation we do two things. + * First we set the busy bit and write-protect all + * mappings to ensure that write accesses to the + * page block (in vm_fault). Second, we remove + * the page from any paging queue to foil the + * pageout daemon (vm_pageout_scan). + */ + pmap_page_protect(VM_PAGE_TO_PHYS(p), VM_PROT_READ); + if (!(p->flags & PG_CLEAN)) { + p->flags |= PG_BUSY; + object->paging_in_progress++; + vm_object_unlock(object); + /* + * XXX if put fails we mark the page as + * clean to avoid an infinite loop. + * Will loose changes to the page. + */ + if (vm_pager_put(object->pager, p, syncio)) { + printf("%s: pager_put error\n", + "vm_object_page_clean"); + p->flags |= PG_CLEAN; + noerror = FALSE; + } + vm_object_lock(object); + object->paging_in_progress--; + if (!de_queue && onqueue) { + vm_page_lock_queues(); + if (onqueue > 0) + vm_page_activate(p); + else + vm_page_deactivate(p); + vm_page_unlock_queues(); + } + p->flags &= ~PG_BUSY; + PAGE_WAKEUP(p); + goto again; + } + } + } + return (noerror); +} + +/* + * vm_object_deactivate_pages + * + * Deactivate all pages in the specified object. (Keep its pages + * in memory even though it is no longer referenced.) + * + * The object must be locked. + */ +void +vm_object_deactivate_pages(object) + register vm_object_t object; +{ + register vm_page_t p, next; + + for (p = object->memq.tqh_first; p != NULL; p = next) { + next = p->listq.tqe_next; + vm_page_lock_queues(); + if (p->flags & PG_ACTIVE) + vm_page_deactivate(p); + vm_page_unlock_queues(); + } +} + +/* + * Trim the object cache to size. + */ +void +vm_object_cache_trim() +{ + register vm_object_t object; + + vm_object_cache_lock(); + while (vm_object_cached > vm_cache_max) { + object = vm_object_cached_list.tqh_first; + vm_object_cache_unlock(); + + if (object != vm_object_lookup(object->pager)) + panic("vm_object_deactivate: I'm sooo confused."); + + pager_cache(object, FALSE); + + vm_object_cache_lock(); + } + vm_object_cache_unlock(); +} + +/* + * vm_object_pmap_copy: + * + * Makes all physical pages in the specified + * object range copy-on-write. No writeable + * references to these pages should remain. + * + * The object must *not* be locked. + */ +void +vm_object_pmap_copy(object, start, end) + register vm_object_t object; + register vm_offset_t start; + register vm_offset_t end; +{ + register vm_page_t p; + + if (object == NULL) + return; + + vm_object_lock(object); + for (p = object->memq.tqh_first; p != NULL; p = p->listq.tqe_next) { + if ((start <= p->offset) && (p->offset < end)) { + pmap_page_protect(VM_PAGE_TO_PHYS(p), VM_PROT_READ); + p->flags |= PG_COPYONWRITE; + } + } + vm_object_unlock(object); +} + +/* + * vm_object_pmap_remove: + * + * Removes all physical pages in the specified + * object range from all physical maps. + * + * The object must *not* be locked. + */ +void +vm_object_pmap_remove(object, start, end) + register vm_object_t object; + register vm_offset_t start; + register vm_offset_t end; +{ + register vm_page_t p; + + if (object == NULL) + return; + + vm_object_lock(object); + for (p = object->memq.tqh_first; p != NULL; p = p->listq.tqe_next) + if ((start <= p->offset) && (p->offset < end)) + pmap_page_protect(VM_PAGE_TO_PHYS(p), VM_PROT_NONE); + vm_object_unlock(object); +} + +/* + * vm_object_copy: + * + * Create a new object which is a copy of an existing + * object, and mark all of the pages in the existing + * object 'copy-on-write'. The new object has one reference. + * Returns the new object. + * + * May defer the copy until later if the object is not backed + * up by a non-default pager. + */ +void +vm_object_copy(src_object, src_offset, size, + dst_object, dst_offset, src_needs_copy) + register vm_object_t src_object; + vm_offset_t src_offset; + vm_size_t size; + vm_object_t *dst_object; /* OUT */ + vm_offset_t *dst_offset; /* OUT */ + boolean_t *src_needs_copy; /* OUT */ +{ + register vm_object_t new_copy; + register vm_object_t old_copy; + vm_offset_t new_start, new_end; + + register vm_page_t p; + + if (src_object == NULL) { + /* + * Nothing to copy + */ + *dst_object = NULL; + *dst_offset = 0; + *src_needs_copy = FALSE; + return; + } + + /* + * If the object's pager is null_pager or the + * default pager, we don't have to make a copy + * of it. Instead, we set the needs copy flag and + * make a shadow later. + */ + + vm_object_lock(src_object); + if (src_object->pager == NULL || + (src_object->flags & OBJ_INTERNAL)) { + + /* + * Make another reference to the object + */ + src_object->ref_count++; + + /* + * Mark all of the pages copy-on-write. + */ + for (p = src_object->memq.tqh_first; p; p = p->listq.tqe_next) + if (src_offset <= p->offset && + p->offset < src_offset + size) + p->flags |= PG_COPYONWRITE; + vm_object_unlock(src_object); + + *dst_object = src_object; + *dst_offset = src_offset; + + /* + * Must make a shadow when write is desired + */ + *src_needs_copy = TRUE; + return; + } + + /* + * Try to collapse the object before copying it. + */ + vm_object_collapse(src_object); + + /* + * If the object has a pager, the pager wants to + * see all of the changes. We need a copy-object + * for the changed pages. + * + * If there is a copy-object, and it is empty, + * no changes have been made to the object since the + * copy-object was made. We can use the same copy- + * object. + */ + + Retry1: + old_copy = src_object->copy; + if (old_copy != NULL) { + /* + * Try to get the locks (out of order) + */ + if (!vm_object_lock_try(old_copy)) { + vm_object_unlock(src_object); + + /* should spin a bit here... */ + vm_object_lock(src_object); + goto Retry1; + } + + if (old_copy->resident_page_count == 0 && + old_copy->pager == NULL) { + /* + * Return another reference to + * the existing copy-object. + */ + old_copy->ref_count++; + vm_object_unlock(old_copy); + vm_object_unlock(src_object); + *dst_object = old_copy; + *dst_offset = src_offset; + *src_needs_copy = FALSE; + return; + } + vm_object_unlock(old_copy); + } + vm_object_unlock(src_object); + + /* + * If the object has a pager, the pager wants + * to see all of the changes. We must make + * a copy-object and put the changed pages there. + * + * The copy-object is always made large enough to + * completely shadow the original object, since + * it may have several users who want to shadow + * the original object at different points. + */ + + new_copy = vm_object_allocate(src_object->size); + + Retry2: + vm_object_lock(src_object); + /* + * Copy object may have changed while we were unlocked + */ + old_copy = src_object->copy; + if (old_copy != NULL) { + /* + * Try to get the locks (out of order) + */ + if (!vm_object_lock_try(old_copy)) { + vm_object_unlock(src_object); + goto Retry2; + } + + /* + * Consistency check + */ + if (old_copy->shadow != src_object || + old_copy->shadow_offset != (vm_offset_t) 0) + panic("vm_object_copy: copy/shadow inconsistency"); + + /* + * Make the old copy-object shadow the new one. + * It will receive no more pages from the original + * object. + */ + + src_object->ref_count--; /* remove ref. from old_copy */ + old_copy->shadow = new_copy; + new_copy->ref_count++; /* locking not needed - we + have the only pointer */ + vm_object_unlock(old_copy); /* done with old_copy */ + } + + new_start = (vm_offset_t) 0; /* always shadow original at 0 */ + new_end = (vm_offset_t) new_copy->size; /* for the whole object */ + + /* + * Point the new copy at the existing object. + */ + + new_copy->shadow = src_object; + new_copy->shadow_offset = new_start; + src_object->ref_count++; + src_object->copy = new_copy; + + /* + * Mark all the affected pages of the existing object + * copy-on-write. + */ + for (p = src_object->memq.tqh_first; p != NULL; p = p->listq.tqe_next) + if ((new_start <= p->offset) && (p->offset < new_end)) + p->flags |= PG_COPYONWRITE; + + vm_object_unlock(src_object); + + *dst_object = new_copy; + *dst_offset = src_offset - new_start; + *src_needs_copy = FALSE; +} + +/* + * vm_object_shadow: + * + * Create a new object which is backed by the + * specified existing object range. The source + * object reference is deallocated. + * + * The new object and offset into that object + * are returned in the source parameters. + */ +void +vm_object_shadow(object, offset, length) + vm_object_t *object; /* IN/OUT */ + vm_offset_t *offset; /* IN/OUT */ + vm_size_t length; +{ + register vm_object_t source; + register vm_object_t result; + + source = *object; + + /* + * Allocate a new object with the given length + */ + + if ((result = vm_object_allocate(length)) == NULL) + panic("vm_object_shadow: no object for shadowing"); + + /* + * The new object shadows the source object, adding + * a reference to it. Our caller changes his reference + * to point to the new object, removing a reference to + * the source object. Net result: no change of reference + * count. + */ + result->shadow = source; + + /* + * Store the offset into the source object, + * and fix up the offset into the new object. + */ + + result->shadow_offset = *offset; + + /* + * Return the new things + */ + + *offset = 0; + *object = result; +} + +/* + * Set the specified object's pager to the specified pager. + */ +void +vm_object_setpager(object, pager, paging_offset, + read_only) + vm_object_t object; + vm_pager_t pager; + vm_offset_t paging_offset; + boolean_t read_only; +{ +#ifdef lint + read_only++; /* No longer used */ +#endif + + vm_object_lock(object); /* XXX ? */ + object->pager = pager; + object->paging_offset = paging_offset; + vm_object_unlock(object); /* XXX ? */ +} + +/* + * vm_object_hash hashes the pager/id pair. + */ + +#define vm_object_hash(pager) \ + (((unsigned long)pager)%VM_OBJECT_HASH_COUNT) + +/* + * vm_object_lookup looks in the object cache for an object with the + * specified pager and paging id. + */ +vm_object_t +vm_object_lookup(pager) + vm_pager_t pager; +{ + register vm_object_hash_entry_t entry; + vm_object_t object; + + vm_object_cache_lock(); + + for (entry = vm_object_hashtable[vm_object_hash(pager)].tqh_first; + entry != NULL; + entry = entry->hash_links.tqe_next) { + object = entry->object; + if (object->pager == pager) { + vm_object_lock(object); + if (object->ref_count == 0) { + TAILQ_REMOVE(&vm_object_cached_list, object, + cached_list); + vm_object_cached--; + } + object->ref_count++; + vm_object_unlock(object); + vm_object_cache_unlock(); + return(object); + } + } + + vm_object_cache_unlock(); + return(NULL); +} + +/* + * vm_object_enter enters the specified object/pager/id into + * the hash table. + */ + +void +vm_object_enter(object, pager) + vm_object_t object; + vm_pager_t pager; +{ + struct vm_object_hash_head *bucket; + register vm_object_hash_entry_t entry; + + /* + * We don't cache null objects, and we can't cache + * objects with the null pager. + */ + + if (object == NULL) + return; + if (pager == NULL) + return; + + bucket = &vm_object_hashtable[vm_object_hash(pager)]; + entry = (vm_object_hash_entry_t) + malloc((u_long)sizeof *entry, M_VMOBJHASH, M_WAITOK); + entry->object = object; + object->flags |= OBJ_CANPERSIST; + + vm_object_cache_lock(); + TAILQ_INSERT_TAIL(bucket, entry, hash_links); + vm_object_cache_unlock(); +} + +/* + * vm_object_remove: + * + * Remove the pager from the hash table. + * Note: This assumes that the object cache + * is locked. XXX this should be fixed + * by reorganizing vm_object_deallocate. + */ +void +vm_object_remove(pager) + register vm_pager_t pager; +{ + struct vm_object_hash_head *bucket; + register vm_object_hash_entry_t entry; + register vm_object_t object; + + bucket = &vm_object_hashtable[vm_object_hash(pager)]; + + for (entry = bucket->tqh_first; + entry != NULL; + entry = entry->hash_links.tqe_next) { + object = entry->object; + if (object->pager == pager) { + TAILQ_REMOVE(bucket, entry, hash_links); + free((caddr_t)entry, M_VMOBJHASH); + break; + } + } +} + +/* + * vm_object_cache_clear removes all objects from the cache. + * + */ +void +vm_object_cache_clear() +{ + register vm_object_t object; + + /* + * Remove each object in the cache by scanning down the + * list of cached objects. + */ + vm_object_cache_lock(); + while ((object = vm_object_cached_list.tqh_first) != NULL) { + vm_object_cache_unlock(); + + /* + * Note: it is important that we use vm_object_lookup + * to gain a reference, and not vm_object_reference, because + * the logic for removing an object from the cache lies in + * lookup. + */ + if (object != vm_object_lookup(object->pager)) + panic("vm_object_cache_clear: I'm sooo confused."); + pager_cache(object, FALSE); + + vm_object_cache_lock(); + } + vm_object_cache_unlock(); +} + +boolean_t vm_object_collapse_allowed = TRUE; +/* + * vm_object_collapse: + * + * Collapse an object with the object backing it. + * Pages in the backing object are moved into the + * parent, and the backing object is deallocated. + * + * Requires that the object be locked and the page + * queues be unlocked. + * + */ +void +vm_object_collapse(object) + register vm_object_t object; + +{ + register vm_object_t backing_object; + register vm_offset_t backing_offset; + register vm_size_t size; + register vm_offset_t new_offset; + register vm_page_t p, pp; + + if (!vm_object_collapse_allowed) + return; + + while (TRUE) { + /* + * Verify that the conditions are right for collapse: + * + * The object exists and no pages in it are currently + * being paged out (or have ever been paged out). + */ + if (object == NULL || + object->paging_in_progress != 0 || + object->pager != NULL) + return; + + /* + * There is a backing object, and + */ + + if ((backing_object = object->shadow) == NULL) + return; + + vm_object_lock(backing_object); + /* + * ... + * The backing object is not read_only, + * and no pages in the backing object are + * currently being paged out. + * The backing object is internal. + */ + + if ((backing_object->flags & OBJ_INTERNAL) == 0 || + backing_object->paging_in_progress != 0) { + vm_object_unlock(backing_object); + return; + } + + /* + * The backing object can't be a copy-object: + * the shadow_offset for the copy-object must stay + * as 0. Furthermore (for the 'we have all the + * pages' case), if we bypass backing_object and + * just shadow the next object in the chain, old + * pages from that object would then have to be copied + * BOTH into the (former) backing_object and into the + * parent object. + */ + if (backing_object->shadow != NULL && + backing_object->shadow->copy != NULL) { + vm_object_unlock(backing_object); + return; + } + + /* + * We know that we can either collapse the backing + * object (if the parent is the only reference to + * it) or (perhaps) remove the parent's reference + * to it. + */ + + backing_offset = object->shadow_offset; + size = object->size; + + /* + * If there is exactly one reference to the backing + * object, we can collapse it into the parent. + */ + + if (backing_object->ref_count == 1) { + + /* + * We can collapse the backing object. + * + * Move all in-memory pages from backing_object + * to the parent. Pages that have been paged out + * will be overwritten by any of the parent's + * pages that shadow them. + */ + + while ((p = backing_object->memq.tqh_first) != NULL) { + new_offset = (p->offset - backing_offset); + + /* + * If the parent has a page here, or if + * this page falls outside the parent, + * dispose of it. + * + * Otherwise, move it as planned. + */ + + if (p->offset < backing_offset || + new_offset >= size) { + vm_page_lock_queues(); + vm_page_free(p); + vm_page_unlock_queues(); + } else { + pp = vm_page_lookup(object, new_offset); + if (pp != NULL && !(pp->flags & PG_FAKE)) { + vm_page_lock_queues(); + vm_page_free(p); + vm_page_unlock_queues(); + } + else { + if (pp) { + /* may be someone waiting for it */ + PAGE_WAKEUP(pp); + vm_page_lock_queues(); + vm_page_free(pp); + vm_page_unlock_queues(); + } + vm_page_rename(p, object, new_offset); + } + } + } + + /* + * Move the pager from backing_object to object. + * + * XXX We're only using part of the paging space + * for keeps now... we ought to discard the + * unused portion. + */ + + if (backing_object->pager) { + object->pager = backing_object->pager; + object->paging_offset = backing_offset + + backing_object->paging_offset; + backing_object->pager = NULL; + } + + /* + * Object now shadows whatever backing_object did. + * Note that the reference to backing_object->shadow + * moves from within backing_object to within object. + */ + + object->shadow = backing_object->shadow; + object->shadow_offset += backing_object->shadow_offset; + if (object->shadow != NULL && + object->shadow->copy != NULL) { + panic("vm_object_collapse: we collapsed a copy-object!"); + } + /* + * Discard backing_object. + * + * Since the backing object has no pages, no + * pager left, and no object references within it, + * all that is necessary is to dispose of it. + */ + + vm_object_unlock(backing_object); + + simple_lock(&vm_object_list_lock); + TAILQ_REMOVE(&vm_object_list, backing_object, + object_list); + vm_object_count--; + simple_unlock(&vm_object_list_lock); + + free((caddr_t)backing_object, M_VMOBJ); + + object_collapses++; + } + else { + /* + * If all of the pages in the backing object are + * shadowed by the parent object, the parent + * object no longer has to shadow the backing + * object; it can shadow the next one in the + * chain. + * + * The backing object must not be paged out - we'd + * have to check all of the paged-out pages, as + * well. + */ + + if (backing_object->pager != NULL) { + vm_object_unlock(backing_object); + return; + } + + /* + * Should have a check for a 'small' number + * of pages here. + */ + + for (p = backing_object->memq.tqh_first; + p != NULL; + p = p->listq.tqe_next) { + new_offset = (p->offset - backing_offset); + + /* + * If the parent has a page here, or if + * this page falls outside the parent, + * keep going. + * + * Otherwise, the backing_object must be + * left in the chain. + */ + + if (p->offset >= backing_offset && + new_offset < size && + ((pp = vm_page_lookup(object, new_offset)) + == NULL || + (pp->flags & PG_FAKE))) { + /* + * Page still needed. + * Can't go any further. + */ + vm_object_unlock(backing_object); + return; + } + } + + /* + * Make the parent shadow the next object + * in the chain. Deallocating backing_object + * will not remove it, since its reference + * count is at least 2. + */ + + object->shadow = backing_object->shadow; + vm_object_reference(object->shadow); + object->shadow_offset += backing_object->shadow_offset; + + /* + * Backing object might have had a copy pointer + * to us. If it did, clear it. + */ + if (backing_object->copy == object) { + backing_object->copy = NULL; + } + + /* Drop the reference count on backing_object. + * Since its ref_count was at least 2, it + * will not vanish; so we don't need to call + * vm_object_deallocate. + */ + backing_object->ref_count--; + vm_object_unlock(backing_object); + + object_bypasses ++; + + } + + /* + * Try again with this object's new backing object. + */ + } +} + +/* + * vm_object_page_remove: [internal] + * + * Removes all physical pages in the specified + * object range from the object's list of pages. + * + * The object must be locked. + */ +void +vm_object_page_remove(object, start, end) + register vm_object_t object; + register vm_offset_t start; + register vm_offset_t end; +{ + register vm_page_t p, next; + + if (object == NULL) + return; + + for (p = object->memq.tqh_first; p != NULL; p = next) { + next = p->listq.tqe_next; + if ((start <= p->offset) && (p->offset < end)) { + pmap_page_protect(VM_PAGE_TO_PHYS(p), VM_PROT_NONE); + vm_page_lock_queues(); + vm_page_free(p); + vm_page_unlock_queues(); + } + } +} + +/* + * Routine: vm_object_coalesce + * Function: Coalesces two objects backing up adjoining + * regions of memory into a single object. + * + * returns TRUE if objects were combined. + * + * NOTE: Only works at the moment if the second object is NULL - + * if it's not, which object do we lock first? + * + * Parameters: + * prev_object First object to coalesce + * prev_offset Offset into prev_object + * next_object Second object into coalesce + * next_offset Offset into next_object + * + * prev_size Size of reference to prev_object + * next_size Size of reference to next_object + * + * Conditions: + * The object must *not* be locked. + */ +boolean_t +vm_object_coalesce(prev_object, next_object, + prev_offset, next_offset, + prev_size, next_size) + + register vm_object_t prev_object; + vm_object_t next_object; + vm_offset_t prev_offset, next_offset; + vm_size_t prev_size, next_size; +{ + vm_size_t newsize; + +#ifdef lint + next_offset++; +#endif + + if (next_object != NULL) { + return(FALSE); + } + + if (prev_object == NULL) { + return(TRUE); + } + + vm_object_lock(prev_object); + + /* + * Try to collapse the object first + */ + vm_object_collapse(prev_object); + + /* + * Can't coalesce if: + * . more than one reference + * . paged out + * . shadows another object + * . has a copy elsewhere + * (any of which mean that the pages not mapped to + * prev_entry may be in use anyway) + */ + + if (prev_object->ref_count > 1 || + prev_object->pager != NULL || + prev_object->shadow != NULL || + prev_object->copy != NULL) { + vm_object_unlock(prev_object); + return(FALSE); + } + + /* + * Remove any pages that may still be in the object from + * a previous deallocation. + */ + + vm_object_page_remove(prev_object, + prev_offset + prev_size, + prev_offset + prev_size + next_size); + + /* + * Extend the object if necessary. + */ + newsize = prev_offset + prev_size + next_size; + if (newsize > prev_object->size) + prev_object->size = newsize; + + vm_object_unlock(prev_object); + return(TRUE); +} + +/* + * vm_object_prefer: + * + * Return optimal virtual address for new mapping of this object. + * + * The object must *not* be locked. + */ +void +vm_object_prefer(object, offset, addr) + register vm_object_t object; + register vm_offset_t offset; + register vm_offset_t *addr; +{ + register vm_page_t p; + register vm_offset_t paddr; + + if (object == NULL) + return; + +#ifdef PMAP_PREFER + vm_object_lock(object); + /* + * Look for the first page that the pmap layer has something + * to say about. Since an object maps a contiguous range of + * virutal addresses, this will determine the preferred origin + * of the proposed mapping. + */ + for (p = object->memq.tqh_first; p != NULL; p = p->listq.tqe_next) { + if (p->flags & (PG_FAKE | PG_FICTITIOUS)) + continue; + paddr = PMAP_PREFER(VM_PAGE_TO_PHYS(p), *addr+p->offset-offset); + if (paddr == (vm_offset_t)-1) + continue; + *addr = paddr - (p->offset - offset); + break; + } + vm_object_unlock(object); +#endif +} +/* + * vm_object_print: [ debug ] + */ +void +vm_object_print(object, full) + vm_object_t object; + boolean_t full; +{ + extern void _vm_object_print(); + + _vm_object_print(object, full, printf); +} + +void +_vm_object_print(object, full, pr) + vm_object_t object; + boolean_t full; + void (*pr) __P((const char *, ...)); +{ + register vm_page_t p; + extern indent; + + register int count; + + if (object == NULL) + return; + + iprintf(pr, "Object 0x%lx: size=0x%lx, res=%d, ref=%d, ", + (long) object, (long) object->size, + object->resident_page_count, object->ref_count); + (*pr)("pager=0x%lx+0x%lx, shadow=(0x%lx)+0x%lx\n", + (long) object->pager, (long) object->paging_offset, + (long) object->shadow, (long) object->shadow_offset); + (*pr)("cache: next=0x%lx, prev=0x%lx\n", + (long)object->cached_list.tqe_next, + (long)object->cached_list.tqe_prev); + + if (!full) + return; + + indent += 2; + count = 0; + for (p = object->memq.tqh_first; p != NULL; p = p->listq.tqe_next) { + if (count == 0) + iprintf(pr, "memory:="); + else if (count == 6) { + (*pr)("\n"); + iprintf(pr, " ..."); + count = 0; + } else + (*pr)(","); + count++; + + (*pr)("(off=0x%x,page=0x%x)", p->offset, VM_PAGE_TO_PHYS(p)); + } + if (count != 0) + (*pr)("\n"); + indent -= 2; +} diff --git a/sys/vm/vm_object.h b/sys/vm/vm_object.h new file mode 100644 index 00000000000..fa12d563e6e --- /dev/null +++ b/sys/vm/vm_object.h @@ -0,0 +1,179 @@ +/* $NetBSD: vm_object.h,v 1.16 1995/03/29 22:10:28 briggs Exp $ */ + +/* + * Copyright (c) 1991, 1993 + * The Regents of the University of California. All rights reserved. + * + * This code is derived from software contributed to Berkeley by + * The Mach Operating System project at Carnegie-Mellon University. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. All advertising materials mentioning features or use of this software + * must display the following acknowledgement: + * This product includes software developed by the University of + * California, Berkeley and its contributors. + * 4. Neither the name of the University nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * @(#)vm_object.h 8.3 (Berkeley) 1/12/94 + * + * + * Copyright (c) 1987, 1990 Carnegie-Mellon University. + * All rights reserved. + * + * Authors: Avadis Tevanian, Jr., Michael Wayne Young + * + * Permission to use, copy, modify and distribute this software and + * its documentation is hereby granted, provided that both the copyright + * notice and this permission notice appear in all copies of the + * software, derivative works or modified versions, and any portions + * thereof, and that both notices appear in supporting documentation. + * + * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" + * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND + * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. + * + * Carnegie Mellon requests users of this software to return to + * + * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU + * School of Computer Science + * Carnegie Mellon University + * Pittsburgh PA 15213-3890 + * + * any improvements or extensions that they make and grant Carnegie the + * rights to redistribute these changes. + */ + +/* + * Virtual memory object module definitions. + */ + +#ifndef _VM_OBJECT_ +#define _VM_OBJECT_ + +#include <vm/vm_page.h> +#include <vm/vm_pager.h> + +/* + * Types defined: + * + * vm_object_t Virtual memory object. + */ + +struct vm_object { + struct pglist memq; /* Resident memory */ + TAILQ_ENTRY(vm_object) object_list; /* list of all objects */ + u_short flags; /* see below */ + u_short paging_in_progress; /* Paging (in or out) so + don't collapse or destroy */ + simple_lock_data_t Lock; /* Synchronization */ + int ref_count; /* How many refs?? */ + vm_size_t size; /* Object size */ + int resident_page_count; + /* number of resident pages */ + struct vm_object *copy; /* Object that holds copies of + my changed pages */ + vm_pager_t pager; /* Where to get data */ + vm_offset_t paging_offset; /* Offset into paging space */ + struct vm_object *shadow; /* My shadow */ + vm_offset_t shadow_offset; /* Offset in shadow */ + TAILQ_ENTRY(vm_object) cached_list; /* for persistence */ +}; +/* + * Flags + */ +#define OBJ_CANPERSIST 0x0001 /* allow to persist */ +#define OBJ_INTERNAL 0x0002 /* internally created object */ +#define OBJ_ACTIVE 0x0004 /* used to mark active objects */ + +TAILQ_HEAD(vm_object_hash_head, vm_object_hash_entry); + +struct vm_object_hash_entry { + TAILQ_ENTRY(vm_object_hash_entry) hash_links; /* hash chain links */ + vm_object_t object; /* object represented */ +}; + +typedef struct vm_object_hash_entry *vm_object_hash_entry_t; + +#ifdef _KERNEL +TAILQ_HEAD(object_q, vm_object); + +struct object_q vm_object_cached_list; /* list of objects persisting */ +int vm_object_cached; /* size of cached list */ +simple_lock_data_t vm_cache_lock; /* lock for object cache */ + +struct object_q vm_object_list; /* list of allocated objects */ +long vm_object_count; /* count of all objects */ +simple_lock_data_t vm_object_list_lock; + /* lock for object list and count */ + +vm_object_t kernel_object; /* the single kernel object */ +vm_object_t kmem_object; + +#define vm_object_cache_lock() simple_lock(&vm_cache_lock) +#define vm_object_cache_unlock() simple_unlock(&vm_cache_lock) +#endif /* _KERNEL */ + +#define vm_object_lock_init(object) simple_lock_init(&(object)->Lock) +#define vm_object_lock(object) simple_lock(&(object)->Lock) +#define vm_object_unlock(object) simple_unlock(&(object)->Lock) +#define vm_object_lock_try(object) simple_lock_try(&(object)->Lock) +#define vm_object_sleep(event, object, interruptible) \ + thread_sleep((event), &(object)->Lock, (interruptible)) + +#ifdef _KERNEL +vm_object_t vm_object_allocate __P((vm_size_t)); +void vm_object_cache_clear __P((void)); +void vm_object_cache_trim __P((void)); +boolean_t vm_object_coalesce __P((vm_object_t, vm_object_t, + vm_offset_t, vm_offset_t, vm_offset_t, vm_size_t)); +void vm_object_collapse __P((vm_object_t)); +void vm_object_copy __P((vm_object_t, vm_offset_t, vm_size_t, + vm_object_t *, vm_offset_t *, boolean_t *)); +void vm_object_deactivate_pages __P((vm_object_t)); +void vm_object_deallocate __P((vm_object_t)); +void vm_object_enter __P((vm_object_t, vm_pager_t)); +void vm_object_init __P((vm_size_t)); +vm_object_t vm_object_lookup __P((vm_pager_t)); +boolean_t vm_object_page_clean __P((vm_object_t, + vm_offset_t, vm_offset_t, boolean_t, boolean_t)); +void vm_object_page_remove __P((vm_object_t, + vm_offset_t, vm_offset_t)); +void vm_object_pmap_copy __P((vm_object_t, + vm_offset_t, vm_offset_t)); +void vm_object_pmap_remove __P((vm_object_t, + vm_offset_t, vm_offset_t)); +void vm_object_prefer __P((vm_object_t, + vm_offset_t, vm_offset_t *)); +void vm_object_print __P((vm_object_t, boolean_t)); +void _vm_object_print __P((vm_object_t, boolean_t, + void (*)(const char *, ...))); +void vm_object_reference __P((vm_object_t)); +void vm_object_remove __P((vm_pager_t)); +void vm_object_setpager __P((vm_object_t, + vm_pager_t, vm_offset_t, boolean_t)); +void vm_object_shadow __P((vm_object_t *, + vm_offset_t *, vm_size_t)); +void vm_object_terminate __P((vm_object_t)); +#endif +#endif /* _VM_OBJECT_ */ diff --git a/sys/vm/vm_page.c b/sys/vm/vm_page.c new file mode 100644 index 00000000000..5ecb20755e0 --- /dev/null +++ b/sys/vm/vm_page.c @@ -0,0 +1,980 @@ +/* $NetBSD: vm_page.c,v 1.26 1995/06/26 14:40:11 leo Exp $ */ + +/* + * Copyright (c) 1991, 1993 + * The Regents of the University of California. All rights reserved. + * + * This code is derived from software contributed to Berkeley by + * The Mach Operating System project at Carnegie-Mellon University. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. All advertising materials mentioning features or use of this software + * must display the following acknowledgement: + * This product includes software developed by the University of + * California, Berkeley and its contributors. + * 4. Neither the name of the University nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * @(#)vm_page.c 8.3 (Berkeley) 3/21/94 + * + * + * Copyright (c) 1987, 1990 Carnegie-Mellon University. + * All rights reserved. + * + * Authors: Avadis Tevanian, Jr., Michael Wayne Young + * + * Permission to use, copy, modify and distribute this software and + * its documentation is hereby granted, provided that both the copyright + * notice and this permission notice appear in all copies of the + * software, derivative works or modified versions, and any portions + * thereof, and that both notices appear in supporting documentation. + * + * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" + * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND + * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. + * + * Carnegie Mellon requests users of this software to return to + * + * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU + * School of Computer Science + * Carnegie Mellon University + * Pittsburgh PA 15213-3890 + * + * any improvements or extensions that they make and grant Carnegie the + * rights to redistribute these changes. + */ + +/* + * Resident memory management module. + */ + +#include <sys/param.h> +#include <sys/systm.h> + +#include <vm/vm.h> +#include <vm/vm_page.h> +#include <vm/vm_map.h> +#include <vm/vm_pageout.h> + +#include <machine/cpu.h> + +#ifdef MACHINE_NONCONTIG +/* + * These variables record the values returned by vm_page_bootstrap, + * for debugging purposes. The implementation of pmap_steal_memory + * and pmap_startup here also uses them internally. + */ +vm_offset_t virtual_space_start; +vm_offset_t virtual_space_end; +#endif /* MACHINE_NONCONTIG */ + +/* + * Associated with page of user-allocatable memory is a + * page structure. + */ + +struct pglist *vm_page_buckets; /* Array of buckets */ +int vm_page_bucket_count = 0; /* How big is array? */ +int vm_page_hash_mask; /* Mask for hash function */ +simple_lock_data_t bucket_lock; /* lock for all buckets XXX */ + +struct pglist vm_page_queue_free; +struct pglist vm_page_queue_active; +struct pglist vm_page_queue_inactive; +simple_lock_data_t vm_page_queue_lock; +simple_lock_data_t vm_page_queue_free_lock; + +/* has physical page allocation been initialized? */ +boolean_t vm_page_startup_initialized; + +vm_page_t vm_page_array; +#ifndef MACHINE_NONCONTIG +long first_page; +long last_page; +vm_offset_t first_phys_addr; +vm_offset_t last_phys_addr; +#else +u_long first_page; +int vm_page_count; +#endif /* MACHINE_NONCONTIG */ +vm_size_t page_mask; +int page_shift; + +/* + * vm_set_page_size: + * + * Sets the page size, perhaps based upon the memory + * size. Must be called before any use of page-size + * dependent functions. + * + * Sets page_shift and page_mask from cnt.v_page_size. + */ +void +vm_set_page_size() +{ + + if (cnt.v_page_size == 0) + cnt.v_page_size = DEFAULT_PAGE_SIZE; + page_mask = cnt.v_page_size - 1; + if ((page_mask & cnt.v_page_size) != 0) + panic("vm_set_page_size: page size not a power of two"); + for (page_shift = 0; ; page_shift++) + if ((1 << page_shift) == cnt.v_page_size) + break; +} + + +#ifdef MACHINE_NONCONTIG +/* + * vm_page_bootstrap: + * + * Initializes the resident memory module. + * + * Allocates memory for the page cells, and + * for the object/offset-to-page hash table headers. + * Each page cell is initialized and placed on the free list. + * Returns the range of available kernel virtual memory. + */ +void +vm_page_bootstrap(startp, endp) + vm_offset_t *startp; + vm_offset_t *endp; +{ + int i; + register struct pglist *bucket; + + extern vm_offset_t kentry_data; + extern vm_size_t kentry_data_size; + + + /* + * Initialize the locks + */ + + simple_lock_init(&vm_page_queue_free_lock); + simple_lock_init(&vm_page_queue_lock); + + /* + * Initialize the queue headers for the free queue, + * the active queue and the inactive queue. + */ + + TAILQ_INIT(&vm_page_queue_free); + TAILQ_INIT(&vm_page_queue_active); + TAILQ_INIT(&vm_page_queue_inactive); + + /* + * Pre-allocate maps and map entries that cannot be dynamically + * allocated via malloc(). The maps include the kernel_map and + * kmem_map which must be initialized before malloc() will + * work (obviously). Also could include pager maps which would + * be allocated before kmeminit. + * + * Allow some kernel map entries... this should be plenty + * since people shouldn't be cluttering up the kernel + * map (they should use their own maps). + */ + + kentry_data_size = MAX_KMAP * sizeof(struct vm_map) + + MAX_KMAPENT * sizeof(struct vm_map_entry); + kentry_data_size = round_page(kentry_data_size); + kentry_data = (vm_offset_t) pmap_steal_memory(kentry_data_size); + + /* + * Validate these zone addresses. + */ + + bzero((caddr_t) kentry_data, kentry_data_size); + + /* + * Allocate (and initialize) the virtual-to-physical + * table hash buckets. + * + * The number of buckets MUST BE a power of 2, and + * the actual value is the next power of 2 greater + * than the number of physical pages in the system. + * + * Note: + * This computation can be tweaked if desired. + */ + + if (vm_page_bucket_count == 0) { + unsigned int npages = pmap_free_pages(); + + vm_page_bucket_count = 1; + while (vm_page_bucket_count < npages) + vm_page_bucket_count <<= 1; + } + + vm_page_hash_mask = vm_page_bucket_count - 1; + + vm_page_buckets = (struct pglist *) + pmap_steal_memory(vm_page_bucket_count * sizeof(*vm_page_buckets)); + bucket = vm_page_buckets; + + for (i = vm_page_bucket_count; i--;) { + TAILQ_INIT(bucket); + bucket++; + } + + simple_lock_init(&bucket_lock); + + /* + * Machine-dependent code allocates the resident page table. + * It uses VM_PAGE_INIT to initialize the page frames. + * The code also returns to us the virtual space available + * to the kernel. We don't trust the pmap module + * to get the alignment right. + */ + + pmap_startup(&virtual_space_start, &virtual_space_end); + virtual_space_start = round_page(virtual_space_start); + virtual_space_end = trunc_page(virtual_space_end); + + *startp = virtual_space_start; + *endp = virtual_space_end; + + simple_lock_init(&vm_pages_needed_lock); +} + +#else /* MACHINE_NONCONTIG */ + +/* + * vm_page_startup: + * + * Initializes the resident memory module. + * + * Allocates memory for the page cells, and + * for the object/offset-to-page hash table headers. + * Each page cell is initialized and placed on the free list. + */ +void +vm_page_startup(start, end) + vm_offset_t *start; + vm_offset_t *end; +{ + register vm_page_t m; + register struct pglist *bucket; + vm_size_t npages; + int i; + vm_offset_t pa; + extern vm_offset_t kentry_data; + extern vm_size_t kentry_data_size; + + + /* + * Initialize the locks + */ + + simple_lock_init(&vm_page_queue_free_lock); + simple_lock_init(&vm_page_queue_lock); + + /* + * Initialize the queue headers for the free queue, + * the active queue and the inactive queue. + */ + + TAILQ_INIT(&vm_page_queue_free); + TAILQ_INIT(&vm_page_queue_active); + TAILQ_INIT(&vm_page_queue_inactive); + + /* + * Calculate the number of hash table buckets. + * + * The number of buckets MUST BE a power of 2, and + * the actual value is the next power of 2 greater + * than the number of physical pages in the system. + * + * Note: + * This computation can be tweaked if desired. + */ + + if (vm_page_bucket_count == 0) { + vm_page_bucket_count = 1; + while (vm_page_bucket_count < atop(*end - *start)) + vm_page_bucket_count <<= 1; + } + + vm_page_hash_mask = vm_page_bucket_count - 1; + + /* + * Allocate (and initialize) the hash table buckets. + */ + vm_page_buckets = (struct pglist *) + pmap_bootstrap_alloc(vm_page_bucket_count * sizeof(struct pglist)); + bucket = vm_page_buckets; + + for (i = vm_page_bucket_count; i--;) { + TAILQ_INIT(bucket); + bucket++; + } + + simple_lock_init(&bucket_lock); + + /* + * Truncate the remainder of physical memory to our page size. + */ + + *end = trunc_page(*end); + + /* + * Pre-allocate maps and map entries that cannot be dynamically + * allocated via malloc(). The maps include the kernel_map and + * kmem_map which must be initialized before malloc() will + * work (obviously). Also could include pager maps which would + * be allocated before kmeminit. + * + * Allow some kernel map entries... this should be plenty + * since people shouldn't be cluttering up the kernel + * map (they should use their own maps). + */ + + kentry_data_size = round_page(MAX_KMAP*sizeof(struct vm_map) + + MAX_KMAPENT*sizeof(struct vm_map_entry)); + kentry_data = (vm_offset_t) pmap_bootstrap_alloc(kentry_data_size); + + /* + * Compute the number of pages of memory that will be + * available for use (taking into account the overhead + * of a page structure per page). + */ + + cnt.v_free_count = npages = (*end - *start + sizeof(struct vm_page)) + / (PAGE_SIZE + sizeof(struct vm_page)); + + /* + * Record the extent of physical memory that the + * virtual memory system manages. + */ + + first_page = *start; + first_page += npages*sizeof(struct vm_page); + first_page = atop(round_page(first_page)); + last_page = first_page + npages - 1; + + first_phys_addr = ptoa(first_page); + last_phys_addr = ptoa(last_page) + PAGE_MASK; + + + /* + * Allocate and clear the mem entry structures. + */ + + m = vm_page_array = (vm_page_t) + pmap_bootstrap_alloc(npages * sizeof(struct vm_page)); + + /* + * Initialize the mem entry structures now, and + * put them in the free queue. + */ + + pa = first_phys_addr; + while (npages--) { + m->flags = 0; + m->object = NULL; + m->phys_addr = pa; + TAILQ_INSERT_TAIL(&vm_page_queue_free, m, pageq); + m++; + pa += PAGE_SIZE; + } + + /* + * Initialize vm_pages_needed lock here - don't wait for pageout + * daemon XXX + */ + simple_lock_init(&vm_pages_needed_lock); + + /* from now on, pmap_bootstrap_alloc can't be used */ + vm_page_startup_initialized = TRUE; +} +#endif /* MACHINE_NONCONTIG */ + +#if defined(MACHINE_NONCONTIG) && !defined(MACHINE_PAGES) +/* + * We implement pmap_steal_memory and pmap_startup with the help + * of two simpler functions, pmap_virtual_space and pmap_next_page. + */ +vm_offset_t +pmap_steal_memory(size) + vm_size_t size; +{ + vm_offset_t addr, vaddr, paddr; + +#ifdef i386 /* XXX i386 calls pmap_steal_memory before vm_mem_init() */ + if (cnt.v_page_size == 0) /* XXX */ + vm_set_page_size(); +#endif + + /* + * We round the size to an integer multiple. + */ + + size = (size + 3) &~ 3; /* XXX */ + + /* + * If this is the first call to pmap_steal_memory, + * we have to initialize ourself. + */ + + if (virtual_space_start == virtual_space_end) { + pmap_virtual_space(&virtual_space_start, &virtual_space_end); + + /* + * The initial values must be aligned properly, and + * we don't trust the pmap module to do it right. + */ + + virtual_space_start = round_page(virtual_space_start); + virtual_space_end = trunc_page(virtual_space_end); + } + + /* + * Allocate virtual memory for this request. + */ + + addr = virtual_space_start; + virtual_space_start += size; + + /* + * Allocate and map physical pages to back new virtual pages. + */ + + for (vaddr = round_page(addr); + vaddr < addr + size; + vaddr += PAGE_SIZE) { + if (!pmap_next_page(&paddr)) + panic("pmap_steal_memory"); + + /* + * XXX Logically, these mappings should be wired, + * but some pmap modules barf if they are. + */ + + pmap_enter(pmap_kernel(), vaddr, paddr, + VM_PROT_READ|VM_PROT_WRITE, FALSE); + } + + return addr; +} + +void +pmap_startup(startp, endp) + vm_offset_t *startp; + vm_offset_t *endp; +{ + unsigned int i, freepages; + vm_offset_t paddr; + + /* + * We calculate how many page frames we will have + * and then allocate the page structures in one chunk. + * The calculation is non-trivial. We want: + * + * vmpages > (freepages - (vmpages / sizeof(vm_page_t))) + * + * which, with some algebra, becomes: + * + * vmpages > (freepages * sizeof(...) / (1 + sizeof(...))) + * + * The value of vm_page_count need not be exact, but must be + * large enough so vm_page_array handles the index range. + */ + + freepages = pmap_free_pages(); + /* Fudge slightly to deal with truncation error. */ + freepages += 1; /* fudge */ + + vm_page_count = (PAGE_SIZE * freepages) / + (PAGE_SIZE + sizeof(*vm_page_array)); + + vm_page_array = (vm_page_t) + pmap_steal_memory(vm_page_count * sizeof(*vm_page_array)); + +#ifdef DIAGNOSTIC + /* + * Initialize everyting in case the holes are stepped in, + * and set PA to something that will cause a panic... + */ + for (i = 0; i < vm_page_count; i++) { + bzero(&vm_page_array[i], sizeof(*vm_page_array)); + vm_page_array[i].phys_addr = 0xdeadbeef; + } +#endif + + /* + * Initialize the page frames. + * Note that some page indices may not be usable + * when pmap_free_pages() counts pages in a hole. + */ + if (!pmap_next_page(&paddr)) + panic("pmap_startup: can't get first page"); + first_page = pmap_page_index(paddr); + i = 0; + for (;;) { + /* Initialize a page array element. */ + VM_PAGE_INIT(&vm_page_array[i], NULL, NULL); + vm_page_array[i].phys_addr = paddr; + vm_page_free(&vm_page_array[i]); + + /* Are there more physical pages? */ + if (!pmap_next_page(&paddr)) + break; + i = pmap_page_index(paddr) - first_page; + + /* Don't trust pmap_page_index()... */ + if (i < 0 || i >= vm_page_count) + panic("pmap_startup: bad i=0x%x", i); + } + + *startp = virtual_space_start; + *endp = virtual_space_end; +} +#endif /* MACHINE_NONCONTIG && !MACHINE_PAGES */ + +/* + * vm_page_hash: + * + * Distributes the object/offset key pair among hash buckets. + * + * NOTE: This macro depends on vm_page_bucket_count being a power of 2. + */ +#define vm_page_hash(object, offset) \ + (((unsigned long)object+(unsigned long)atop(offset))&vm_page_hash_mask) + +/* + * vm_page_insert: [ internal use only ] + * + * Inserts the given mem entry into the object/object-page + * table and object list. + * + * The object and page must be locked. + */ +void +vm_page_insert(mem, object, offset) + register vm_page_t mem; + register vm_object_t object; + register vm_offset_t offset; +{ + register struct pglist *bucket; + int spl; + + VM_PAGE_CHECK(mem); + + if (mem->flags & PG_TABLED) + panic("vm_page_insert: already inserted"); + + /* + * Record the object/offset pair in this page + */ + + mem->object = object; + mem->offset = offset; + + /* + * Insert it into the object_object/offset hash table + */ + + bucket = &vm_page_buckets[vm_page_hash(object, offset)]; + spl = splimp(); + simple_lock(&bucket_lock); + TAILQ_INSERT_TAIL(bucket, mem, hashq); + simple_unlock(&bucket_lock); + (void) splx(spl); + + /* + * Now link into the object's list of backed pages. + */ + + TAILQ_INSERT_TAIL(&object->memq, mem, listq); + mem->flags |= PG_TABLED; + + /* + * And show that the object has one more resident + * page. + */ + + object->resident_page_count++; +} + +/* + * vm_page_remove: [ internal use only ] + * NOTE: used by device pager as well -wfj + * + * Removes the given mem entry from the object/offset-page + * table and the object page list. + * + * The object and page must be locked. + */ +void +vm_page_remove(mem) + register vm_page_t mem; +{ + register struct pglist *bucket; + int spl; + + VM_PAGE_CHECK(mem); + +#ifdef DIAGNOSTIC + if (mem->flags & PG_FAULTING) + panic("vm_page_remove: page is faulting"); +#endif + + if (!(mem->flags & PG_TABLED)) + return; + + /* + * Remove from the object_object/offset hash table + */ + + bucket = &vm_page_buckets[vm_page_hash(mem->object, mem->offset)]; + spl = splimp(); + simple_lock(&bucket_lock); + TAILQ_REMOVE(bucket, mem, hashq); + simple_unlock(&bucket_lock); + (void) splx(spl); + + /* + * Now remove from the object's list of backed pages. + */ + + TAILQ_REMOVE(&mem->object->memq, mem, listq); + + /* + * And show that the object has one fewer resident + * page. + */ + + mem->object->resident_page_count--; + + mem->flags &= ~PG_TABLED; +} + +/* + * vm_page_lookup: + * + * Returns the page associated with the object/offset + * pair specified; if none is found, NULL is returned. + * + * The object must be locked. No side effects. + */ +vm_page_t +vm_page_lookup(object, offset) + register vm_object_t object; + register vm_offset_t offset; +{ + register vm_page_t mem; + register struct pglist *bucket; + int spl; + + /* + * Search the hash table for this object/offset pair + */ + + bucket = &vm_page_buckets[vm_page_hash(object, offset)]; + + spl = splimp(); + simple_lock(&bucket_lock); + for (mem = bucket->tqh_first; mem != NULL; mem = mem->hashq.tqe_next) { + VM_PAGE_CHECK(mem); + if ((mem->object == object) && (mem->offset == offset)) { + simple_unlock(&bucket_lock); + splx(spl); + return(mem); + } + } + + simple_unlock(&bucket_lock); + splx(spl); + return(NULL); +} + +/* + * vm_page_rename: + * + * Move the given memory entry from its + * current object to the specified target object/offset. + * + * The object must be locked. + */ +void +vm_page_rename(mem, new_object, new_offset) + register vm_page_t mem; + register vm_object_t new_object; + vm_offset_t new_offset; +{ + if (mem->object == new_object) + return; + + vm_page_lock_queues(); /* keep page from moving out from + under pageout daemon */ + vm_page_remove(mem); + vm_page_insert(mem, new_object, new_offset); + vm_page_unlock_queues(); +} + +/* + * vm_page_alloc: + * + * Allocate and return a memory cell associated + * with this VM object/offset pair. + * + * Object must be locked. + */ +vm_page_t +vm_page_alloc(object, offset) + vm_object_t object; + vm_offset_t offset; +{ + register vm_page_t mem; + int spl; + + spl = splimp(); /* XXX */ + simple_lock(&vm_page_queue_free_lock); + if (vm_page_queue_free.tqh_first == NULL) { + simple_unlock(&vm_page_queue_free_lock); + splx(spl); + return(NULL); + } + + mem = vm_page_queue_free.tqh_first; + TAILQ_REMOVE(&vm_page_queue_free, mem, pageq); + + cnt.v_free_count--; + simple_unlock(&vm_page_queue_free_lock); + splx(spl); + + VM_PAGE_INIT(mem, object, offset); + + /* + * Decide if we should poke the pageout daemon. + * We do this if the free count is less than the low + * water mark, or if the free count is less than the high + * water mark (but above the low water mark) and the inactive + * count is less than its target. + * + * We don't have the counts locked ... if they change a little, + * it doesn't really matter. + */ + + if (cnt.v_free_count < cnt.v_free_min || + (cnt.v_free_count < cnt.v_free_target && + cnt.v_inactive_count < cnt.v_inactive_target)) + thread_wakeup(&vm_pages_needed); + return (mem); +} + +/* + * vm_page_free: + * + * Returns the given page to the free list, + * disassociating it with any VM object. + * + * Object and page must be locked prior to entry. + */ +void +vm_page_free(mem) + register vm_page_t mem; +{ + vm_page_remove(mem); + if (mem->flags & PG_ACTIVE) { + TAILQ_REMOVE(&vm_page_queue_active, mem, pageq); + mem->flags &= ~PG_ACTIVE; + cnt.v_active_count--; + } + + if (mem->flags & PG_INACTIVE) { + TAILQ_REMOVE(&vm_page_queue_inactive, mem, pageq); + mem->flags &= ~PG_INACTIVE; + cnt.v_inactive_count--; + } + + if (!(mem->flags & PG_FICTITIOUS)) { + int spl; + + spl = splimp(); + simple_lock(&vm_page_queue_free_lock); + TAILQ_INSERT_TAIL(&vm_page_queue_free, mem, pageq); + + cnt.v_free_count++; + simple_unlock(&vm_page_queue_free_lock); + splx(spl); + } +} + +/* + * vm_page_wire: + * + * Mark this page as wired down by yet + * another map, removing it from paging queues + * as necessary. + * + * The page queues must be locked. + */ +void +vm_page_wire(mem) + register vm_page_t mem; +{ + VM_PAGE_CHECK(mem); + + if (mem->wire_count == 0) { + if (mem->flags & PG_ACTIVE) { + TAILQ_REMOVE(&vm_page_queue_active, mem, pageq); + cnt.v_active_count--; + mem->flags &= ~PG_ACTIVE; + } + if (mem->flags & PG_INACTIVE) { + TAILQ_REMOVE(&vm_page_queue_inactive, mem, pageq); + cnt.v_inactive_count--; + mem->flags &= ~PG_INACTIVE; + } + cnt.v_wire_count++; + } + mem->wire_count++; +} + +/* + * vm_page_unwire: + * + * Release one wiring of this page, potentially + * enabling it to be paged again. + * + * The page queues must be locked. + */ +void +vm_page_unwire(mem) + register vm_page_t mem; +{ + VM_PAGE_CHECK(mem); + + mem->wire_count--; + if (mem->wire_count == 0) { + TAILQ_INSERT_TAIL(&vm_page_queue_active, mem, pageq); + cnt.v_active_count++; + mem->flags |= PG_ACTIVE; + cnt.v_wire_count--; + } +} + +/* + * vm_page_deactivate: + * + * Returns the given page to the inactive list, + * indicating that no physical maps have access + * to this page. [Used by the physical mapping system.] + * + * The page queues must be locked. + */ +void +vm_page_deactivate(m) + register vm_page_t m; +{ + VM_PAGE_CHECK(m); + + /* + * Only move active pages -- ignore locked or already + * inactive ones. + */ + + if (m->flags & PG_ACTIVE) { + TAILQ_REMOVE(&vm_page_queue_active, m, pageq); + m->flags &= ~PG_ACTIVE; + cnt.v_active_count--; + goto deact; + } + if ((m->flags & PG_INACTIVE) == 0) { + deact: + TAILQ_INSERT_TAIL(&vm_page_queue_inactive, m, pageq); + m->flags |= PG_INACTIVE; + cnt.v_inactive_count++; + pmap_clear_reference(VM_PAGE_TO_PHYS(m)); + if (pmap_is_modified(VM_PAGE_TO_PHYS(m))) + m->flags &= ~PG_CLEAN; + if (m->flags & PG_CLEAN) + m->flags &= ~PG_LAUNDRY; + else + m->flags |= PG_LAUNDRY; + } +} + +/* + * vm_page_activate: + * + * Put the specified page on the active list (if appropriate). + * + * The page queues must be locked. + */ +void +vm_page_activate(m) + register vm_page_t m; +{ + VM_PAGE_CHECK(m); + + if (m->flags & PG_INACTIVE) { + TAILQ_REMOVE(&vm_page_queue_inactive, m, pageq); + m->flags &= ~PG_INACTIVE; + cnt.v_inactive_count--; + } + if (m->wire_count == 0) { + if (m->flags & PG_ACTIVE) + panic("vm_page_activate: already active"); + + TAILQ_INSERT_TAIL(&vm_page_queue_active, m, pageq); + m->flags |= PG_ACTIVE; + cnt.v_active_count++; + } +} + +/* + * vm_page_zero_fill: + * + * Zero-fill the specified page. + * Written as a standard pagein routine, to + * be used by the zero-fill object. + */ +boolean_t +vm_page_zero_fill(m) + vm_page_t m; +{ + VM_PAGE_CHECK(m); + + m->flags &= ~PG_CLEAN; + pmap_zero_page(VM_PAGE_TO_PHYS(m)); + return(TRUE); +} + +/* + * vm_page_copy: + * + * Copy one page to another + */ +void +vm_page_copy(src_m, dest_m) + vm_page_t src_m; + vm_page_t dest_m; +{ + VM_PAGE_CHECK(src_m); + VM_PAGE_CHECK(dest_m); + + dest_m->flags &= ~PG_CLEAN; + pmap_copy_page(VM_PAGE_TO_PHYS(src_m), VM_PAGE_TO_PHYS(dest_m)); +} diff --git a/sys/vm/vm_page.h b/sys/vm/vm_page.h new file mode 100644 index 00000000000..d4bdd132440 --- /dev/null +++ b/sys/vm/vm_page.h @@ -0,0 +1,297 @@ +/* $NetBSD: vm_page.h,v 1.18 1995/03/26 20:39:13 jtc Exp $ */ + +/* + * Copyright (c) 1991, 1993 + * The Regents of the University of California. All rights reserved. + * + * This code is derived from software contributed to Berkeley by + * The Mach Operating System project at Carnegie-Mellon University. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. All advertising materials mentioning features or use of this software + * must display the following acknowledgement: + * This product includes software developed by the University of + * California, Berkeley and its contributors. + * 4. Neither the name of the University nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * @(#)vm_page.h 7.3 (Berkeley) 4/21/91 + * + * + * Copyright (c) 1987, 1990 Carnegie-Mellon University. + * All rights reserved. + * + * Authors: Avadis Tevanian, Jr., Michael Wayne Young + * + * Permission to use, copy, modify and distribute this software and + * its documentation is hereby granted, provided that both the copyright + * notice and this permission notice appear in all copies of the + * software, derivative works or modified versions, and any portions + * thereof, and that both notices appear in supporting documentation. + * + * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" + * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND + * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. + * + * Carnegie Mellon requests users of this software to return to + * + * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU + * School of Computer Science + * Carnegie Mellon University + * Pittsburgh PA 15213-3890 + * + * any improvements or extensions that they make and grant Carnegie the + * rights to redistribute these changes. + */ + +/* + * Resident memory system definitions. + */ + +#ifndef _VM_PAGE_ +#define _VM_PAGE_ + +/* + * Management of resident (logical) pages. + * + * A small structure is kept for each resident + * page, indexed by page number. Each structure + * is an element of several lists: + * + * A hash table bucket used to quickly + * perform object/offset lookups + * + * A list of all pages for a given object, + * so they can be quickly deactivated at + * time of deallocation. + * + * An ordered list of pages due for pageout. + * + * In addition, the structure contains the object + * and offset to which this page belongs (for pageout), + * and sundry status bits. + * + * Fields in this structure are locked either by the lock on the + * object that the page belongs to (O) or by the lock on the page + * queues (P). + */ + +TAILQ_HEAD(pglist, vm_page); + +struct vm_page { + TAILQ_ENTRY(vm_page) pageq; /* queue info for FIFO + * queue or free list (P) */ + TAILQ_ENTRY(vm_page) hashq; /* hash table links (O)*/ + TAILQ_ENTRY(vm_page) listq; /* pages in same object (O)*/ + + vm_object_t object; /* which object am I in (O,P)*/ + vm_offset_t offset; /* offset into object (O,P) */ + + u_short wire_count; /* wired down maps refs (P) */ + u_short flags; /* see below */ + + vm_offset_t phys_addr; /* physical address of page */ +}; + +/* + * These are the flags defined for vm_page. + * + * Note: PG_FILLED and PG_DIRTY are added for the filesystems. + */ +#define PG_INACTIVE 0x0001 /* page is in inactive list (P) */ +#define PG_ACTIVE 0x0002 /* page is in active list (P) */ +#define PG_LAUNDRY 0x0004 /* page is being cleaned now (P)*/ +#define PG_CLEAN 0x0008 /* page has not been modified */ +#define PG_BUSY 0x0010 /* page is in transit (O) */ +#define PG_WANTED 0x0020 /* someone is waiting for page (O) */ +#define PG_TABLED 0x0040 /* page is in VP table (O) */ +#define PG_COPYONWRITE 0x0080 /* must copy page before changing (O) */ +#define PG_FICTITIOUS 0x0100 /* physical page doesn't exist (O) */ +#define PG_FAKE 0x0200 /* page is placeholder for pagein (O) */ +#define PG_FILLED 0x0400 /* client flag to set when filled */ +#define PG_DIRTY 0x0800 /* client flag to set when dirty */ +#define PG_FAULTING 0x2000 /* page is being faulted in */ +#define PG_PAGEROWNED 0x4000 /* DEBUG: async paging op in progress */ +#define PG_PTPAGE 0x8000 /* DEBUG: is a user page table page */ + +#if VM_PAGE_DEBUG +#ifndef MACHINE_NONCONTIG +#define VM_PAGE_CHECK(mem) { \ + if ((((unsigned int) mem) < ((unsigned int) &vm_page_array[0])) || \ + (((unsigned int) mem) > \ + ((unsigned int) &vm_page_array[last_page-first_page])) || \ + ((mem->flags & (PG_ACTIVE | PG_INACTIVE)) == \ + (PG_ACTIVE | PG_INACTIVE))) \ + panic("vm_page_check: not valid!"); \ +} +#else /* MACHINE_NONCONTIG */ +#define VM_PAGE_CHECK(mem) { \ + if ((((unsigned int) mem) < ((unsigned int) &vm_page_array[0])) || \ + (((unsigned int) mem) > \ + ((unsigned int) &vm_page_array[vm_page_count])) || \ + ((mem->flags & (PG_ACTIVE | PG_INACTIVE)) == \ + (PG_ACTIVE | PG_INACTIVE))) \ + panic("vm_page_check: not valid!"); \ +} +#endif /* MACHINE_NONCONTIG */ +#else /* VM_PAGE_DEBUG */ +#define VM_PAGE_CHECK(mem) +#endif /* VM_PAGE_DEBUG */ + +#ifdef _KERNEL +/* + * Each pageable resident page falls into one of three lists: + * + * free + * Available for allocation now. + * inactive + * Not referenced in any map, but still has an + * object/offset-page mapping, and may be dirty. + * This is the list of pages that should be + * paged out next. + * active + * A list of pages which have been placed in + * at least one physical map. This list is + * ordered, in LRU-like fashion. + */ + +extern +struct pglist vm_page_queue_free; /* memory free queue */ +extern +struct pglist vm_page_queue_active; /* active memory queue */ +extern +struct pglist vm_page_queue_inactive; /* inactive memory queue */ + +extern +vm_page_t vm_page_array; /* First resident page in table */ + +#ifndef MACHINE_NONCONTIG +extern +long first_page; /* first physical page number */ + /* ... represented in vm_page_array */ +extern +long last_page; /* last physical page number */ + /* ... represented in vm_page_array */ + /* [INCLUSIVE] */ +extern +vm_offset_t first_phys_addr; /* physical address for first_page */ +extern +vm_offset_t last_phys_addr; /* physical address for last_page */ +#else /* MACHINE_NONCONTIG */ +extern +u_long first_page; /* first physical page number */ +extern +int vm_page_count; /* How many pages do we manage? */ +#endif /* MACHINE_NONCONTIG */ + +#define VM_PAGE_TO_PHYS(entry) ((entry)->phys_addr) + +#ifndef MACHINE_NONCONTIG +#define IS_VM_PHYSADDR(pa) \ + ((pa) >= first_phys_addr && (pa) <= last_phys_addr) + +#define PHYS_TO_VM_PAGE(pa) \ + (&vm_page_array[atop(pa) - first_page ]) +#else +#define IS_VM_PHYSADDR(pa) \ + (pmap_page_index(pa) >= 0) + +#define PHYS_TO_VM_PAGE(pa) \ + (&vm_page_array[pmap_page_index(pa) - first_page]) +#endif /* MACHINE_NONCONTIG */ + +extern +simple_lock_data_t vm_page_queue_lock; /* lock on active and inactive + page queues */ +extern /* lock on free page queue */ +simple_lock_data_t vm_page_queue_free_lock; + +/* + * Functions implemented as macros + */ + +#define PAGE_ASSERT_WAIT(m, interruptible) { \ + (m)->flags |= PG_WANTED; \ + assert_wait((m), (interruptible)); \ + } + +#define PAGE_WAKEUP(m) { \ + (m)->flags &= ~PG_BUSY; \ + if ((m)->flags & PG_WANTED) { \ + (m)->flags &= ~PG_WANTED; \ + thread_wakeup((m)); \ + } \ + } + +#define vm_page_lock_queues() simple_lock(&vm_page_queue_lock) +#define vm_page_unlock_queues() simple_unlock(&vm_page_queue_lock) + +#define vm_page_set_modified(m) { (m)->flags &= ~PG_CLEAN; } + +#ifndef MACHINE_NONCONTIG +#define VM_PAGE_INIT(mem, obj, offset) { \ + (mem)->flags = PG_BUSY | PG_CLEAN | PG_FAKE; \ + vm_page_insert((mem), (obj), (offset)); \ + (mem)->wire_count = 0; \ +} +#else /* MACHINE_NONCONTIG */ +#define VM_PAGE_INIT(mem, obj, offset) { \ + (mem)->flags = PG_BUSY | PG_CLEAN | PG_FAKE; \ + if (obj) \ + vm_page_insert((mem), (obj), (offset)); \ + else \ + (mem)->object = NULL; \ + (mem)->wire_count = 0; \ +} +#endif /* MACHINE_NONCONTIG */ + +/* XXX what is this here for? */ +void vm_set_page_size __P((void)); + +/* XXX probably should be elsewhere. */ +#ifdef MACHINE_NONCONTIG +vm_offset_t pmap_steal_memory __P((vm_size_t)); +void pmap_startup __P((vm_offset_t *, vm_offset_t *)); +#endif + +void vm_page_activate __P((vm_page_t)); +vm_page_t vm_page_alloc __P((vm_object_t, vm_offset_t)); +#ifdef MACHINE_NONCONTIG +void vm_page_bootstrap __P((vm_offset_t *, vm_offset_t *)); +#endif +void vm_page_copy __P((vm_page_t, vm_page_t)); +void vm_page_deactivate __P((vm_page_t)); +void vm_page_free __P((vm_page_t)); +void vm_page_insert __P((vm_page_t, vm_object_t, vm_offset_t)); +vm_page_t vm_page_lookup __P((vm_object_t, vm_offset_t)); +void vm_page_remove __P((vm_page_t)); +void vm_page_rename __P((vm_page_t, vm_object_t, vm_offset_t)); +#ifndef MACHINE_NONCONTIG +void vm_page_startup __P((vm_offset_t *, vm_offset_t *)); +#endif +void vm_page_unwire __P((vm_page_t)); +void vm_page_wire __P((vm_page_t)); +boolean_t vm_page_zero_fill __P((vm_page_t)); + +#endif /* _KERNEL */ +#endif /* !_VM_PAGE_ */ diff --git a/sys/vm/vm_pageout.c b/sys/vm/vm_pageout.c new file mode 100644 index 00000000000..d4456e6009a --- /dev/null +++ b/sys/vm/vm_pageout.c @@ -0,0 +1,571 @@ +/* $NetBSD: vm_pageout.c,v 1.22 1995/06/28 02:58:51 cgd Exp $ */ + +/* + * Copyright (c) 1991, 1993 + * The Regents of the University of California. All rights reserved. + * + * This code is derived from software contributed to Berkeley by + * The Mach Operating System project at Carnegie-Mellon University. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. All advertising materials mentioning features or use of this software + * must display the following acknowledgement: + * This product includes software developed by the University of + * California, Berkeley and its contributors. + * 4. Neither the name of the University nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * @(#)vm_pageout.c 8.5 (Berkeley) 2/14/94 + * + * + * Copyright (c) 1987, 1990 Carnegie-Mellon University. + * All rights reserved. + * + * Authors: Avadis Tevanian, Jr., Michael Wayne Young + * + * Permission to use, copy, modify and distribute this software and + * its documentation is hereby granted, provided that both the copyright + * notice and this permission notice appear in all copies of the + * software, derivative works or modified versions, and any portions + * thereof, and that both notices appear in supporting documentation. + * + * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" + * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND + * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. + * + * Carnegie Mellon requests users of this software to return to + * + * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU + * School of Computer Science + * Carnegie Mellon University + * Pittsburgh PA 15213-3890 + * + * any improvements or extensions that they make and grant Carnegie the + * rights to redistribute these changes. + */ + +/* + * The proverbial page-out daemon. + */ + +#include <sys/param.h> + +#include <vm/vm.h> +#include <vm/vm_page.h> +#include <vm/vm_pageout.h> + +#ifndef VM_PAGE_FREE_MIN +#define VM_PAGE_FREE_MIN (cnt.v_free_count / 20) +#endif + +#ifndef VM_PAGE_FREE_TARGET +#define VM_PAGE_FREE_TARGET ((cnt.v_free_min * 4) / 3) +#endif + +int vm_page_free_min_min = 16 * 1024; +int vm_page_free_min_max = 256 * 1024; + +int vm_pages_needed; /* Event on which pageout daemon sleeps */ + +int vm_page_max_wired = 0; /* XXX max # of wired pages system-wide */ + +#ifdef CLUSTERED_PAGEOUT +#define MAXPOCLUSTER (MAXPHYS/NBPG) /* XXX */ +int doclustered_pageout = 1; +#endif + +/* + * vm_pageout_scan does the dirty work for the pageout daemon. + */ +void +vm_pageout_scan() +{ + register vm_page_t m, next; + register int page_shortage; + register int s; + register int pages_freed; + int free; + vm_object_t object; + + /* + * Only continue when we want more pages to be "free" + */ + + cnt.v_rev++; + + s = splimp(); + simple_lock(&vm_page_queue_free_lock); + free = cnt.v_free_count; + simple_unlock(&vm_page_queue_free_lock); + splx(s); + +#ifndef __SWAP_BROKEN /* XXX */ + if (free < cnt.v_free_target) { + swapout_threads(); + + /* + * Be sure the pmap system is updated so + * we can scan the inactive queue. + */ + + pmap_update(); + } +#endif /* XXX */ + + /* + * Acquire the resident page system lock, + * as we may be changing what's resident quite a bit. + */ + vm_page_lock_queues(); + + /* + * Start scanning the inactive queue for pages we can free. + * We keep scanning until we have enough free pages or + * we have scanned through the entire queue. If we + * encounter dirty pages, we start cleaning them. + */ + + pages_freed = 0; + for (m = vm_page_queue_inactive.tqh_first; m != NULL; m = next) { + s = splimp(); + simple_lock(&vm_page_queue_free_lock); + free = cnt.v_free_count; + simple_unlock(&vm_page_queue_free_lock); + splx(s); + if (free >= cnt.v_free_target) + break; + + cnt.v_scan++; + next = m->pageq.tqe_next; + + /* + * If the page has been referenced, move it back to the + * active queue. + */ + if (pmap_is_referenced(VM_PAGE_TO_PHYS(m))) { + vm_page_activate(m); + cnt.v_reactivated++; + continue; + } + + /* + * If the page is clean, free it up. + */ + if (m->flags & PG_CLEAN) { + object = m->object; + if (vm_object_lock_try(object)) { + pmap_page_protect(VM_PAGE_TO_PHYS(m), + VM_PROT_NONE); + vm_page_free(m); + pages_freed++; + cnt.v_dfree++; + vm_object_unlock(object); + } + continue; + } + + /* + * If the page is dirty but already being washed, skip it. + */ + if ((m->flags & PG_LAUNDRY) == 0) + continue; + + /* + * Otherwise the page is dirty and still in the laundry, + * so we start the cleaning operation and remove it from + * the laundry. + */ + object = m->object; + if (!vm_object_lock_try(object)) + continue; + cnt.v_pageouts++; +#ifdef CLUSTERED_PAGEOUT + if (object->pager && + vm_pager_cancluster(object->pager, PG_CLUSTERPUT)) + vm_pageout_cluster(m, object); + else +#endif + vm_pageout_page(m, object); + thread_wakeup(object); + vm_object_unlock(object); + /* + * Former next page may no longer even be on the inactive + * queue (due to potential blocking in the pager with the + * queues unlocked). If it isn't, we just start over. + */ + if (next && (next->flags & PG_INACTIVE) == 0) + next = vm_page_queue_inactive.tqh_first; + } + + /* + * Compute the page shortage. If we are still very low on memory + * be sure that we will move a minimal amount of pages from active + * to inactive. + */ + + page_shortage = cnt.v_inactive_target - cnt.v_inactive_count; + if (page_shortage <= 0 && pages_freed == 0) + page_shortage = 1; + + while (page_shortage > 0) { + /* + * Move some more pages from active to inactive. + */ + + if ((m = vm_page_queue_active.tqh_first) == NULL) + break; + vm_page_deactivate(m); + page_shortage--; + } + + vm_page_unlock_queues(); +} + +/* + * Called with object and page queues locked. + * If reactivate is TRUE, a pager error causes the page to be + * put back on the active queue, ow it is left on the inactive queue. + */ +void +vm_pageout_page(m, object) + vm_page_t m; + vm_object_t object; +{ + vm_pager_t pager; + int pageout_status; + + /* + * We set the busy bit to cause potential page faults on + * this page to block. + * + * We also set pageout-in-progress to keep the object from + * disappearing during pageout. This guarantees that the + * page won't move from the inactive queue. (However, any + * other page on the inactive queue may move!) + */ + pmap_page_protect(VM_PAGE_TO_PHYS(m), VM_PROT_NONE); + m->flags |= PG_BUSY; + + /* + * Try to collapse the object before making a pager for it. + * We must unlock the page queues first. + */ + vm_page_unlock_queues(); + if (object->pager == NULL) + vm_object_collapse(object); + + object->paging_in_progress++; + vm_object_unlock(object); + + /* + * Do a wakeup here in case the following operations block. + */ + thread_wakeup(&cnt.v_free_count); + + /* + * If there is no pager for the page, use the default pager. + * If there is no place to put the page at the moment, + * leave it in the laundry and hope that there will be + * paging space later. + */ + if ((pager = object->pager) == NULL) { + pager = vm_pager_allocate(PG_DFLT, (caddr_t)0, object->size, + VM_PROT_ALL, (vm_offset_t)0); + if (pager != NULL) + vm_object_setpager(object, pager, 0, FALSE); + } + pageout_status = pager ? vm_pager_put(pager, m, FALSE) : VM_PAGER_FAIL; + vm_object_lock(object); + vm_page_lock_queues(); + + switch (pageout_status) { + case VM_PAGER_OK: + case VM_PAGER_PEND: + cnt.v_pgpgout++; + m->flags &= ~PG_LAUNDRY; + break; + case VM_PAGER_BAD: + /* + * Page outside of range of object. Right now we + * essentially lose the changes by pretending it + * worked. + * + * XXX dubious, what should we do? + */ + m->flags &= ~PG_LAUNDRY; + m->flags |= PG_CLEAN; + pmap_clear_modify(VM_PAGE_TO_PHYS(m)); + break; + case VM_PAGER_AGAIN: + { + extern int lbolt; + + /* + * FAIL on a write is interpreted to mean a resource + * shortage, so we put pause for awhile and try again. + * XXX could get stuck here. + */ + (void) tsleep((caddr_t)&lbolt, PZERO|PCATCH, "pageout", 0); + break; + } + case VM_PAGER_FAIL: + case VM_PAGER_ERROR: + /* + * If page couldn't be paged out, then reactivate + * the page so it doesn't clog the inactive list. + * (We will try paging out it again later). + */ + vm_page_activate(m); + cnt.v_reactivated++; + break; + } + + pmap_clear_reference(VM_PAGE_TO_PHYS(m)); + + /* + * If the operation is still going, leave the page busy + * to block all other accesses. Also, leave the paging + * in progress indicator set so that we don't attempt an + * object collapse. + */ + if (pageout_status != VM_PAGER_PEND) { + m->flags &= ~PG_BUSY; + PAGE_WAKEUP(m); + object->paging_in_progress--; + } +} + +#ifdef CLUSTERED_PAGEOUT +#define PAGEOUTABLE(p) \ + ((((p)->flags & (PG_INACTIVE|PG_CLEAN|PG_LAUNDRY)) == \ + (PG_INACTIVE|PG_LAUNDRY)) && !pmap_is_referenced(VM_PAGE_TO_PHYS(p))) + +/* + * Attempt to pageout as many contiguous (to ``m'') dirty pages as possible + * from ``object''. Using information returned from the pager, we assemble + * a sorted list of contiguous dirty pages and feed them to the pager in one + * chunk. Called with paging queues and object locked. Also, object must + * already have a pager. + */ +void +vm_pageout_cluster(m, object) + vm_page_t m; + vm_object_t object; +{ + vm_offset_t offset, loff, hoff; + vm_page_t plist[MAXPOCLUSTER], *plistp, p; + int postatus, ix, count; + + /* + * Determine the range of pages that can be part of a cluster + * for this object/offset. If it is only our single page, just + * do it normally. + */ + vm_pager_cluster(object->pager, m->offset, &loff, &hoff); + if (hoff - loff == PAGE_SIZE) { + vm_pageout_page(m, object); + return; + } + + plistp = plist; + + /* + * Target page is always part of the cluster. + */ + pmap_page_protect(VM_PAGE_TO_PHYS(m), VM_PROT_NONE); + m->flags |= PG_BUSY; + plistp[atop(m->offset - loff)] = m; + count = 1; + + /* + * Backup from the given page til we find one not fulfilling + * the pageout criteria or we hit the lower bound for the + * cluster. For each page determined to be part of the + * cluster, unmap it and busy it out so it won't change. + */ + ix = atop(m->offset - loff); + offset = m->offset; + while (offset > loff && count < MAXPOCLUSTER-1) { + p = vm_page_lookup(object, offset - PAGE_SIZE); + if (p == NULL || !PAGEOUTABLE(p)) + break; + pmap_page_protect(VM_PAGE_TO_PHYS(p), VM_PROT_NONE); + p->flags |= PG_BUSY; + plistp[--ix] = p; + offset -= PAGE_SIZE; + count++; + } + plistp += atop(offset - loff); + loff = offset; + + /* + * Now do the same moving forward from the target. + */ + ix = atop(m->offset - loff) + 1; + offset = m->offset + PAGE_SIZE; + while (offset < hoff && count < MAXPOCLUSTER) { + p = vm_page_lookup(object, offset); + if (p == NULL || !PAGEOUTABLE(p)) + break; + pmap_page_protect(VM_PAGE_TO_PHYS(p), VM_PROT_NONE); + p->flags |= PG_BUSY; + plistp[ix++] = p; + offset += PAGE_SIZE; + count++; + } + hoff = offset; + + /* + * Pageout the page. + * Unlock everything and do a wakeup prior to the pager call + * in case it blocks. + */ + vm_page_unlock_queues(); + object->paging_in_progress++; + vm_object_unlock(object); +again: + thread_wakeup(&cnt.v_free_count); + postatus = vm_pager_put_pages(object->pager, plistp, count, FALSE); + /* + * XXX rethink this + */ + if (postatus == VM_PAGER_AGAIN) { + extern int lbolt; + + (void) tsleep((caddr_t)&lbolt, PZERO|PCATCH, "pageout", 0); + goto again; + } else if (postatus == VM_PAGER_BAD) + panic("vm_pageout_cluster: VM_PAGER_BAD"); + vm_object_lock(object); + vm_page_lock_queues(); + + /* + * Loop through the affected pages, reflecting the outcome of + * the operation. + */ + for (ix = 0; ix < count; ix++) { + p = *plistp++; + switch (postatus) { + case VM_PAGER_OK: + case VM_PAGER_PEND: + cnt.v_pgpgout++; + p->flags &= ~PG_LAUNDRY; + break; + case VM_PAGER_FAIL: + case VM_PAGER_ERROR: + /* + * Pageout failed, reactivate the target page so it + * doesn't clog the inactive list. Other pages are + * left as they are. + */ + if (p == m) { + vm_page_activate(p); + cnt.v_reactivated++; + } + break; + } + pmap_clear_reference(VM_PAGE_TO_PHYS(p)); + /* + * If the operation is still going, leave the page busy + * to block all other accesses. + */ + if (postatus != VM_PAGER_PEND) { + p->flags &= ~PG_BUSY; + PAGE_WAKEUP(p); + + } + } + /* + * If the operation is still going, leave the paging in progress + * indicator set so that we don't attempt an object collapse. + */ + if (postatus != VM_PAGER_PEND) + object->paging_in_progress--; + +} +#endif + +/* + * vm_pageout is the high level pageout daemon. + */ + +void +vm_pageout() +{ + (void) spl0(); + + /* + * Initialize some paging parameters. + */ + + if (cnt.v_free_min == 0) { + cnt.v_free_min = VM_PAGE_FREE_MIN; + vm_page_free_min_min /= cnt.v_page_size; + vm_page_free_min_max /= cnt.v_page_size; + if (cnt.v_free_min < vm_page_free_min_min) + cnt.v_free_min = vm_page_free_min_min; + if (cnt.v_free_min > vm_page_free_min_max) + cnt.v_free_min = vm_page_free_min_max; + } + + if (cnt.v_free_target == 0) + cnt.v_free_target = VM_PAGE_FREE_TARGET; + + if (cnt.v_free_target <= cnt.v_free_min) + cnt.v_free_target = cnt.v_free_min + 1; + + /* XXX does not really belong here */ + if (vm_page_max_wired == 0) + vm_page_max_wired = cnt.v_free_count / 3; + + /* + * The pageout daemon is never done, so loop + * forever. + */ + + simple_lock(&vm_pages_needed_lock); + while (TRUE) { + thread_sleep(&vm_pages_needed, &vm_pages_needed_lock, FALSE); + /* + * Compute the inactive target for this scan. + * We need to keep a reasonable amount of memory in the + * inactive list to better simulate LRU behavior. + */ + cnt.v_inactive_target = + (cnt.v_active_count + cnt.v_inactive_count) / 3; + if (cnt.v_inactive_target <= cnt.v_free_target) + cnt.v_inactive_target = cnt.v_free_target + 1; + + /* + * Only make a scan if we are likely to do something. + * Otherwise we might have been awakened by a pager + * to clean up async pageouts. + */ + if (cnt.v_free_count < cnt.v_free_target || + cnt.v_inactive_count < cnt.v_inactive_target) + vm_pageout_scan(); + vm_pager_sync(); + simple_lock(&vm_pages_needed_lock); + thread_wakeup(&cnt.v_free_count); + } +} diff --git a/sys/vm/vm_pageout.h b/sys/vm/vm_pageout.h new file mode 100644 index 00000000000..649f2852b7f --- /dev/null +++ b/sys/vm/vm_pageout.h @@ -0,0 +1,98 @@ +/* $NetBSD: vm_pageout.h,v 1.11 1995/03/26 20:39:14 jtc Exp $ */ + +/* + * Copyright (c) 1991, 1993 + * The Regents of the University of California. All rights reserved. + * + * This code is derived from software contributed to Berkeley by + * The Mach Operating System project at Carnegie-Mellon University. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. All advertising materials mentioning features or use of this software + * must display the following acknowledgement: + * This product includes software developed by the University of + * California, Berkeley and its contributors. + * 4. Neither the name of the University nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * @(#)vm_pageout.h 8.2 (Berkeley) 1/12/94 + * + * + * Copyright (c) 1987, 1990 Carnegie-Mellon University. + * All rights reserved. + * + * Author: Avadis Tevanian, Jr. + * + * Permission to use, copy, modify and distribute this software and + * its documentation is hereby granted, provided that both the copyright + * notice and this permission notice appear in all copies of the + * software, derivative works or modified versions, and any portions + * thereof, and that both notices appear in supporting documentation. + * + * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" + * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND + * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. + * + * Carnegie Mellon requests users of this software to return to + * + * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU + * School of Computer Science + * Carnegie Mellon University + * Pittsburgh PA 15213-3890 + * + * any improvements or extensions that they make and grant Carnegie the + * rights to redistribute these changes. + */ + +/* + * Header file for pageout daemon. + */ + +/* + * Exported data structures. + */ + +extern int vm_pages_needed; /* should be some "event" structure */ +simple_lock_data_t vm_pages_needed_lock; + + +/* + * Exported routines. + */ + +/* + * Signal pageout-daemon and wait for it. + */ + +#define VM_WAIT { \ + simple_lock(&vm_pages_needed_lock); \ + thread_wakeup(&vm_pages_needed); \ + thread_sleep(&cnt.v_free_count, \ + &vm_pages_needed_lock, FALSE); \ + } +#ifdef _KERNEL +void vm_pageout __P((void)); +void vm_pageout_scan __P((void)); +void vm_pageout_page __P((vm_page_t, vm_object_t)); +void vm_pageout_cluster __P((vm_page_t, vm_object_t)); +#endif diff --git a/sys/vm/vm_pager.c b/sys/vm/vm_pager.c new file mode 100644 index 00000000000..d832daec2fa --- /dev/null +++ b/sys/vm/vm_pager.c @@ -0,0 +1,403 @@ +/* $NetBSD: vm_pager.c,v 1.18 1995/05/07 19:27:30 cgd Exp $ */ + +/* + * Copyright (c) 1991, 1993 + * The Regents of the University of California. All rights reserved. + * + * This code is derived from software contributed to Berkeley by + * The Mach Operating System project at Carnegie-Mellon University. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. All advertising materials mentioning features or use of this software + * must display the following acknowledgement: + * This product includes software developed by the University of + * California, Berkeley and its contributors. + * 4. Neither the name of the University nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * @(#)vm_pager.c 8.6 (Berkeley) 1/12/94 + * + * + * Copyright (c) 1987, 1990 Carnegie-Mellon University. + * All rights reserved. + * + * Authors: Avadis Tevanian, Jr., Michael Wayne Young + * + * Permission to use, copy, modify and distribute this software and + * its documentation is hereby granted, provided that both the copyright + * notice and this permission notice appear in all copies of the + * software, derivative works or modified versions, and any portions + * thereof, and that both notices appear in supporting documentation. + * + * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" + * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND + * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. + * + * Carnegie Mellon requests users of this software to return to + * + * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU + * School of Computer Science + * Carnegie Mellon University + * Pittsburgh PA 15213-3890 + * + * any improvements or extensions that they make and grant Carnegie the + * rights to redistribute these changes. + */ + +/* + * Paging space routine stubs. Emulates a matchmaker-like interface + * for builtin pagers. + */ + +#include <sys/param.h> +#include <sys/systm.h> +#include <sys/malloc.h> + +#include <vm/vm.h> +#include <vm/vm_page.h> +#include <vm/vm_kern.h> + +#ifdef SWAPPAGER +extern struct pagerops swappagerops; +#endif + +#ifdef VNODEPAGER +extern struct pagerops vnodepagerops; +#endif + +#ifdef DEVPAGER +extern struct pagerops devicepagerops; +#endif + +struct pagerops *pagertab[] = { +#ifdef SWAPPAGER + &swappagerops, /* PG_SWAP */ +#else + NULL, +#endif +#ifdef VNODEPAGER + &vnodepagerops, /* PG_VNODE */ +#else + NULL, +#endif +#ifdef DEVPAGER + &devicepagerops, /* PG_DEV */ +#else + NULL, +#endif +}; +int npagers = sizeof (pagertab) / sizeof (pagertab[0]); + +struct pagerops *dfltpagerops = NULL; /* default pager */ + +/* + * Kernel address space for mapping pages. + * Used by pagers where KVAs are needed for IO. + * + * XXX needs to be large enough to support the number of pending async + * cleaning requests (NPENDINGIO == 64) * the maximum swap cluster size + * (MAXPHYS == 64k) if you want to get the most efficiency. + */ +#define PAGER_MAP_SIZE (4 * 1024 * 1024) + +vm_map_t pager_map; +boolean_t pager_map_wanted; +vm_offset_t pager_sva, pager_eva; + +void +vm_pager_init() +{ + struct pagerops **pgops; + + /* + * Allocate a kernel submap for tracking get/put page mappings + */ + pager_map = kmem_suballoc(kernel_map, &pager_sva, &pager_eva, + PAGER_MAP_SIZE, FALSE); + /* + * Initialize known pagers + */ + for (pgops = pagertab; pgops < &pagertab[npagers]; pgops++) + if (*pgops) + (*(*pgops)->pgo_init)(); + if (dfltpagerops == NULL) + panic("no default pager"); +} + +/* + * Allocate an instance of a pager of the given type. + * Size, protection and offset parameters are passed in for pagers that + * need to perform page-level validation (e.g. the device pager). + */ +vm_pager_t +vm_pager_allocate(type, handle, size, prot, off) + int type; + caddr_t handle; + vm_size_t size; + vm_prot_t prot; + vm_offset_t off; +{ + struct pagerops *ops; + + ops = (type == PG_DFLT) ? dfltpagerops : pagertab[type]; + if (ops) + return ((*ops->pgo_alloc)(handle, size, prot, off)); + return (NULL); +} + +void +vm_pager_deallocate(pager) + vm_pager_t pager; +{ + if (pager == NULL) + panic("vm_pager_deallocate: null pager"); + + (*pager->pg_ops->pgo_dealloc)(pager); +} + +int +vm_pager_get_pages(pager, mlist, npages, sync) + vm_pager_t pager; + vm_page_t *mlist; + int npages; + boolean_t sync; +{ + int rv; + + if (pager == NULL) { + rv = VM_PAGER_OK; + while (npages--) + if (!vm_page_zero_fill(*mlist)) { + rv = VM_PAGER_FAIL; + break; + } else + mlist++; + return (rv); + } + return ((*pager->pg_ops->pgo_getpages)(pager, mlist, npages, sync)); +} + +int +vm_pager_put_pages(pager, mlist, npages, sync) + vm_pager_t pager; + vm_page_t *mlist; + int npages; + boolean_t sync; +{ + if (pager == NULL) + panic("vm_pager_put_pages: null pager"); + return ((*pager->pg_ops->pgo_putpages)(pager, mlist, npages, sync)); +} + +/* XXX compatibility*/ +int +vm_pager_get(pager, m, sync) + vm_pager_t pager; + vm_page_t m; + boolean_t sync; +{ + return vm_pager_get_pages(pager, &m, 1, sync); +} + +/* XXX compatibility*/ +int +vm_pager_put(pager, m, sync) + vm_pager_t pager; + vm_page_t m; + boolean_t sync; +{ + return vm_pager_put_pages(pager, &m, 1, sync); +} + +boolean_t +vm_pager_has_page(pager, offset) + vm_pager_t pager; + vm_offset_t offset; +{ + if (pager == NULL) + panic("vm_pager_has_page: null pager"); + return ((*pager->pg_ops->pgo_haspage)(pager, offset)); +} + +/* + * Called by pageout daemon before going back to sleep. + * Gives pagers a chance to clean up any completed async pageing operations. + */ +void +vm_pager_sync() +{ + struct pagerops **pgops; + + for (pgops = pagertab; pgops < &pagertab[npagers]; pgops++) + if (*pgops) + (*(*pgops)->pgo_putpages)(NULL, NULL, 0, FALSE); +} + +void +vm_pager_cluster(pager, offset, loff, hoff) + vm_pager_t pager; + vm_offset_t offset; + vm_offset_t *loff; + vm_offset_t *hoff; +{ + if (pager == NULL) + panic("vm_pager_cluster: null pager"); + ((*pager->pg_ops->pgo_cluster)(pager, offset, loff, hoff)); +} + +void +vm_pager_clusternull(pager, offset, loff, hoff) + vm_pager_t pager; + vm_offset_t offset; + vm_offset_t *loff; + vm_offset_t *hoff; +{ + panic("vm_pager_nullcluster called"); +} + +vm_offset_t +vm_pager_map_pages(mlist, npages, canwait) + vm_page_t *mlist; + int npages; + boolean_t canwait; +{ + vm_offset_t kva, va; + vm_size_t size; + vm_page_t m; + + /* + * Allocate space in the pager map, if none available return 0. + * This is basically an expansion of kmem_alloc_wait with optional + * blocking on no space. + */ + size = npages * PAGE_SIZE; + vm_map_lock(pager_map); + while (vm_map_findspace(pager_map, 0, size, &kva)) { + if (!canwait) { + vm_map_unlock(pager_map); + return (0); + } + pager_map_wanted = TRUE; + vm_map_unlock(pager_map); + (void) tsleep(pager_map, PVM, "pager_map", 0); + vm_map_lock(pager_map); + } + vm_map_insert(pager_map, NULL, 0, kva, kva + size); + vm_map_unlock(pager_map); + + for (va = kva; npages--; va += PAGE_SIZE) { + m = *mlist++; +#ifdef DEBUG + if ((m->flags & PG_BUSY) == 0) + panic("vm_pager_map_pages: page not busy"); + if (m->flags & PG_PAGEROWNED) + panic("vm_pager_map_pages: page already in pager"); +#endif +#ifdef DEBUG + m->flags |= PG_PAGEROWNED; +#endif + pmap_enter(vm_map_pmap(pager_map), va, VM_PAGE_TO_PHYS(m), + VM_PROT_DEFAULT, TRUE); + } + return (kva); +} + +void +vm_pager_unmap_pages(kva, npages) + vm_offset_t kva; + int npages; +{ + vm_size_t size = npages * PAGE_SIZE; + +#ifdef DEBUG + vm_offset_t va; + vm_page_t m; + int np = npages; + + for (va = kva; np--; va += PAGE_SIZE) { + m = vm_pager_atop(va); + if (m->flags & PG_PAGEROWNED) + m->flags &= ~PG_PAGEROWNED; + else + printf("vm_pager_unmap_pages: %x(%x/%x) not owned\n", + m, va, VM_PAGE_TO_PHYS(m)); + } +#endif + pmap_remove(vm_map_pmap(pager_map), kva, kva + size); + vm_map_lock(pager_map); + (void) vm_map_delete(pager_map, kva, kva + size); + if (pager_map_wanted) + wakeup(pager_map); + vm_map_unlock(pager_map); +} + +vm_page_t +vm_pager_atop(kva) + vm_offset_t kva; +{ + vm_offset_t pa; + + pa = pmap_extract(vm_map_pmap(pager_map), kva); + if (pa == 0) + panic("vm_pager_atop"); + return (PHYS_TO_VM_PAGE(pa)); +} + +vm_pager_t +vm_pager_lookup(pglist, handle) + register struct pagerlst *pglist; + caddr_t handle; +{ + register vm_pager_t pager; + + for (pager = pglist->tqh_first; pager; pager = pager->pg_list.tqe_next) + if (pager->pg_handle == handle) + return (pager); + return (NULL); +} + +/* + * This routine gains a reference to the object. + * Explicit deallocation is necessary. + */ +int +pager_cache(object, should_cache) + vm_object_t object; + boolean_t should_cache; +{ + if (object == NULL) + return (KERN_INVALID_ARGUMENT); + + vm_object_cache_lock(); + vm_object_lock(object); + if (should_cache) + object->flags |= OBJ_CANPERSIST; + else + object->flags &= ~OBJ_CANPERSIST; + vm_object_unlock(object); + vm_object_cache_unlock(); + + vm_object_deallocate(object); + + return (KERN_SUCCESS); +} diff --git a/sys/vm/vm_pager.h b/sys/vm/vm_pager.h new file mode 100644 index 00000000000..49f5c0782a3 --- /dev/null +++ b/sys/vm/vm_pager.h @@ -0,0 +1,139 @@ +/* $NetBSD: vm_pager.h,v 1.10 1995/03/26 20:39:15 jtc Exp $ */ + +/* + * Copyright (c) 1990 University of Utah. + * Copyright (c) 1991, 1993 + * The Regents of the University of California. All rights reserved. + * + * This code is derived from software contributed to Berkeley by + * the Systems Programming Group of the University of Utah Computer + * Science Department. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. All advertising materials mentioning features or use of this software + * must display the following acknowledgement: + * This product includes software developed by the University of + * California, Berkeley and its contributors. + * 4. Neither the name of the University nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * @(#)vm_pager.h 8.4 (Berkeley) 1/12/94 + */ + +/* + * Pager routine interface definition. + * For BSD we use a cleaner version of the internal pager interface. + */ + +#ifndef _VM_PAGER_ +#define _VM_PAGER_ + +TAILQ_HEAD(pagerlst, pager_struct); + +struct pager_struct { + TAILQ_ENTRY(pager_struct) pg_list; /* links for list management */ + caddr_t pg_handle; /* ext. handle (vp, dev, fp) */ + int pg_type; /* type of pager */ + int pg_flags; /* flags */ + struct pagerops *pg_ops; /* pager operations */ + void *pg_data; /* private pager data */ +}; + +/* pager types */ +#define PG_DFLT -1 +#define PG_SWAP 0 +#define PG_VNODE 1 +#define PG_DEVICE 2 + +/* flags */ +#define PG_CLUSTERGET 1 +#define PG_CLUSTERPUT 2 + +struct pagerops { + void (*pgo_init) /* Initialize pager. */ + __P((void)); + vm_pager_t (*pgo_alloc) /* Allocate pager. */ + __P((caddr_t, vm_size_t, vm_prot_t, vm_offset_t)); + void (*pgo_dealloc) /* Disassociate. */ + __P((vm_pager_t)); + int (*pgo_getpages) /* Get (read) page. */ + __P((vm_pager_t, vm_page_t *, int, boolean_t)); + int (*pgo_putpages) /* Put (write) page. */ + __P((vm_pager_t, vm_page_t *, int, boolean_t)); + boolean_t (*pgo_haspage) /* Does pager have page? */ + __P((vm_pager_t, vm_offset_t)); + void (*pgo_cluster) /* Return range of cluster. */ + __P((vm_pager_t, vm_offset_t, + vm_offset_t *, vm_offset_t *)); +}; + +/* + * get/put return values + * OK operation was successful + * BAD specified data was out of the accepted range + * FAIL specified data was in range, but doesn't exist + * PEND operations was initiated but not completed + * ERROR error while accessing data that is in range and exists + * AGAIN temporary resource shortage prevented operation from happening + */ +#define VM_PAGER_OK 0 +#define VM_PAGER_BAD 1 +#define VM_PAGER_FAIL 2 +#define VM_PAGER_PEND 3 +#define VM_PAGER_ERROR 4 +#define VM_PAGER_AGAIN 5 + +#ifdef _KERNEL +extern struct pagerops *dfltpagerops; + +vm_pager_t vm_pager_allocate + __P((int, caddr_t, vm_size_t, vm_prot_t, vm_offset_t)); +vm_page_t vm_pager_atop __P((vm_offset_t)); +void vm_pager_cluster + __P((vm_pager_t, vm_offset_t, + vm_offset_t *, vm_offset_t *)); +void vm_pager_clusternull + __P((vm_pager_t, vm_offset_t, + vm_offset_t *, vm_offset_t *)); +void vm_pager_deallocate __P((vm_pager_t)); +int vm_pager_get_pages + __P((vm_pager_t, vm_page_t *, int, boolean_t)); +boolean_t vm_pager_has_page __P((vm_pager_t, vm_offset_t)); +void vm_pager_init __P((void)); +vm_pager_t vm_pager_lookup __P((struct pagerlst *, caddr_t)); +vm_offset_t vm_pager_map_pages __P((vm_page_t *, int, boolean_t)); +int vm_pager_put_pages + __P((vm_pager_t, vm_page_t *, int, boolean_t)); +void vm_pager_sync __P((void)); +void vm_pager_unmap_pages __P((vm_offset_t, int)); + +#define vm_pager_cancluster(p, b) ((p)->pg_flags & (b)) + +/* + * XXX compat with old interface + */ +int vm_pager_get __P((vm_pager_t, vm_page_t, boolean_t)); +int vm_pager_put __P((vm_pager_t, vm_page_t, boolean_t)); +#endif + +#endif /* _VM_PAGER_ */ diff --git a/sys/vm/vm_param.h b/sys/vm/vm_param.h new file mode 100644 index 00000000000..4f63abc747a --- /dev/null +++ b/sys/vm/vm_param.h @@ -0,0 +1,162 @@ +/* $NetBSD: vm_param.h,v 1.12 1995/03/26 20:39:16 jtc Exp $ */ + +/* + * Copyright (c) 1991, 1993 + * The Regents of the University of California. All rights reserved. + * + * This code is derived from software contributed to Berkeley by + * The Mach Operating System project at Carnegie-Mellon University. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. All advertising materials mentioning features or use of this software + * must display the following acknowledgement: + * This product includes software developed by the University of + * California, Berkeley and its contributors. + * 4. Neither the name of the University nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * @(#)vm_param.h 8.1 (Berkeley) 6/11/93 + * + * + * Copyright (c) 1987, 1990 Carnegie-Mellon University. + * All rights reserved. + * + * Authors: Avadis Tevanian, Jr., Michael Wayne Young + * + * Permission to use, copy, modify and distribute this software and + * its documentation is hereby granted, provided that both the copyright + * notice and this permission notice appear in all copies of the + * software, derivative works or modified versions, and any portions + * thereof, and that both notices appear in supporting documentation. + * + * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" + * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND + * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. + * + * Carnegie Mellon requests users of this software to return to + * + * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU + * School of Computer Science + * Carnegie Mellon University + * Pittsburgh PA 15213-3890 + * + * any improvements or extensions that they make and grant Carnegie the + * rights to redistribute these changes. + */ + +/* + * Machine independent virtual memory parameters. + */ + +#ifndef _VM_PARAM_ +#define _VM_PARAM_ + +#include <machine/vmparam.h> + +/* + * This belongs in types.h, but breaks too many existing programs. + */ +typedef int boolean_t; +#define TRUE 1 +#define FALSE 0 + +/* + * The machine independent pages are refered to as PAGES. A page + * is some number of hardware pages, depending on the target machine. + */ +#define DEFAULT_PAGE_SIZE 4096 + +/* + * All references to the size of a page should be done with PAGE_SIZE + * or PAGE_SHIFT. The fact they are variables is hidden here so that + * we can easily make them constant if we so desire. + */ +#define PAGE_SIZE cnt.v_page_size /* size of page */ +#define PAGE_MASK page_mask /* size of page - 1 */ +#define PAGE_SHIFT page_shift /* bits to shift for pages */ +#ifdef _KERNEL +extern vm_size_t page_mask; +extern int page_shift; +#endif + +/* + * CTL_VM identifiers + */ +#define VM_METER 1 /* struct vmmeter */ +#define VM_LOADAVG 2 /* struct loadavg */ +#define VM_MAXID 3 /* number of valid vm ids */ + +#define CTL_VM_NAMES { \ + { 0, 0 }, \ + { "vmmeter", CTLTYPE_STRUCT }, \ + { "loadavg", CTLTYPE_STRUCT }, \ +} + +/* + * Return values from the VM routines. + */ +#define KERN_SUCCESS 0 +#define KERN_INVALID_ADDRESS 1 +#define KERN_PROTECTION_FAILURE 2 +#define KERN_NO_SPACE 3 +#define KERN_INVALID_ARGUMENT 4 +#define KERN_FAILURE 5 +#define KERN_RESOURCE_SHORTAGE 6 +#define KERN_NOT_RECEIVER 7 +#define KERN_NO_ACCESS 8 + +#ifndef ASSEMBLER +/* + * Convert addresses to pages and vice versa. + * No rounding is used. + */ +#ifdef _KERNEL +#define atop(x) (((unsigned long)(x)) >> PAGE_SHIFT) +#define ptoa(x) ((vm_offset_t)((x) << PAGE_SHIFT)) + +/* + * Round off or truncate to the nearest page. These will work + * for either addresses or counts (i.e., 1 byte rounds to 1 page). + */ +#define round_page(x) \ + ((vm_offset_t)((((vm_offset_t)(x)) + PAGE_MASK) & ~PAGE_MASK)) +#define trunc_page(x) \ + ((vm_offset_t)(((vm_offset_t)(x)) & ~PAGE_MASK)) +#define num_pages(x) \ + ((vm_offset_t)((((vm_offset_t)(x)) + PAGE_MASK) >> PAGE_SHIFT)) + +extern vm_size_t mem_size; /* size of physical memory (bytes) */ +extern vm_offset_t first_addr; /* first physical page */ +extern vm_offset_t last_addr; /* last physical page */ + +#else +/* out-of-kernel versions of round_page and trunc_page */ +#define round_page(x) \ + ((((vm_offset_t)(x) + (vm_page_size - 1)) / vm_page_size) * \ + vm_page_size) +#define trunc_page(x) \ + ((((vm_offset_t)(x)) / vm_page_size) * vm_page_size) + +#endif /* _KERNEL */ +#endif /* ASSEMBLER */ +#endif /* _VM_PARAM_ */ diff --git a/sys/vm/vm_prot.h b/sys/vm/vm_prot.h new file mode 100644 index 00000000000..60734f1a618 --- /dev/null +++ b/sys/vm/vm_prot.h @@ -0,0 +1,104 @@ +/* $NetBSD: vm_prot.h,v 1.6 1994/06/29 06:48:42 cgd Exp $ */ + +/* + * Copyright (c) 1991, 1993 + * The Regents of the University of California. All rights reserved. + * + * This code is derived from software contributed to Berkeley by + * The Mach Operating System project at Carnegie-Mellon University. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. All advertising materials mentioning features or use of this software + * must display the following acknowledgement: + * This product includes software developed by the University of + * California, Berkeley and its contributors. + * 4. Neither the name of the University nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * @(#)vm_prot.h 8.1 (Berkeley) 6/11/93 + * + * + * Copyright (c) 1987, 1990 Carnegie-Mellon University. + * All rights reserved. + * + * Authors: Avadis Tevanian, Jr., Michael Wayne Young + * + * Permission to use, copy, modify and distribute this software and + * its documentation is hereby granted, provided that both the copyright + * notice and this permission notice appear in all copies of the + * software, derivative works or modified versions, and any portions + * thereof, and that both notices appear in supporting documentation. + * + * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" + * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND + * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. + * + * Carnegie Mellon requests users of this software to return to + * + * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU + * School of Computer Science + * Carnegie Mellon University + * Pittsburgh PA 15213-3890 + * + * any improvements or extensions that they make and grant Carnegie the + * rights to redistribute these changes. + */ + +/* + * Virtual memory protection definitions. + */ + +#ifndef _VM_PROT_ +#define _VM_PROT_ + +/* + * Types defined: + * + * vm_prot_t VM protection values. + */ + +typedef int vm_prot_t; + +/* + * Protection values, defined as bits within the vm_prot_t type + */ + +#define VM_PROT_NONE ((vm_prot_t) 0x00) + +#define VM_PROT_READ ((vm_prot_t) 0x01) /* read permission */ +#define VM_PROT_WRITE ((vm_prot_t) 0x02) /* write permission */ +#define VM_PROT_EXECUTE ((vm_prot_t) 0x04) /* execute permission */ + +/* + * The default protection for newly-created virtual memory + */ + +#define VM_PROT_DEFAULT (VM_PROT_READ|VM_PROT_WRITE|VM_PROT_EXECUTE) + +/* + * The maximum privileges possible, for parameter checking. + */ + +#define VM_PROT_ALL (VM_PROT_READ|VM_PROT_WRITE|VM_PROT_EXECUTE) + +#endif /* _VM_PROT_ */ diff --git a/sys/vm/vm_swap.c b/sys/vm/vm_swap.c new file mode 100644 index 00000000000..41958bd6d85 --- /dev/null +++ b/sys/vm/vm_swap.c @@ -0,0 +1,480 @@ +/* $NetBSD: vm_swap.c,v 1.31 1995/10/07 06:29:02 mycroft Exp $ */ + +/* + * Copyright (c) 1982, 1986, 1989, 1993 + * The Regents of the University of California. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. All advertising materials mentioning features or use of this software + * must display the following acknowledgement: + * This product includes software developed by the University of + * California, Berkeley and its contributors. + * 4. Neither the name of the University nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * @(#)vm_swap.c 8.5 (Berkeley) 2/17/94 + */ + +#include <sys/param.h> +#include <sys/systm.h> +#include <sys/buf.h> +#include <sys/conf.h> +#include <sys/proc.h> +#include <sys/namei.h> +#include <sys/dmap.h> /* XXX */ +#include <sys/vnode.h> +#include <sys/map.h> +#include <sys/file.h> + +#include <sys/mount.h> +#include <sys/syscallargs.h> + +#include <miscfs/specfs/specdev.h> + +/* + * Indirect driver for multi-controller paging. + */ + +int nswap, nswdev; +#ifdef SEQSWAP +int niswdev; /* number of interleaved swap devices */ +int niswap; /* size of interleaved swap area */ +#endif + +/* + * Set up swap devices. + * Initialize linked list of free swap + * headers. These do not actually point + * to buffers, but rather to pages that + * are being swapped in and out. + */ +void +swapinit() +{ + register int i; + register struct buf *sp = swbuf; + register struct proc *p = &proc0; /* XXX */ + struct swdevt *swp; + int error; + + /* + * Count swap devices, and adjust total swap space available. + * Some of the space will not be countable until later (dynamically + * configurable devices) and some of the counted space will not be + * available until a swapon() system call is issued, both usually + * happen when the system goes multi-user. + * + * If using NFS for swap, swdevt[0] will already be bdevvp'd. XXX + */ +#ifdef SEQSWAP + nswdev = niswdev = 0; + nswap = niswap = 0; + /* + * All interleaved devices must come first + */ + for (swp = swdevt; swp->sw_dev != NODEV || swp->sw_vp != NULL; swp++) { + if (swp->sw_flags & SW_SEQUENTIAL) + break; + niswdev++; + if (swp->sw_nblks > niswap) + niswap = swp->sw_nblks; + } + niswap = roundup(niswap, dmmax); + niswap *= niswdev; + if (swdevt[0].sw_vp == NULL && + bdevvp(swdevt[0].sw_dev, &swdevt[0].sw_vp)) + panic("swapvp"); + /* + * The remainder must be sequential + */ + for ( ; swp->sw_dev != NODEV; swp++) { + if ((swp->sw_flags & SW_SEQUENTIAL) == 0) + panic("binit: mis-ordered swap devices"); + nswdev++; + if (swp->sw_nblks > 0) { + if (swp->sw_nblks % dmmax) + swp->sw_nblks -= (swp->sw_nblks % dmmax); + nswap += swp->sw_nblks; + } + } + nswdev += niswdev; + if (nswdev == 0) + panic("swapinit"); + nswap += niswap; +#else + nswdev = 0; + nswap = 0; + for (swp = swdevt; swp->sw_dev != NODEV || swp->sw_vp != NULL; swp++) { + nswdev++; + if (swp->sw_nblks > nswap) + nswap = swp->sw_nblks; + } + if (nswdev == 0) + panic("swapinit"); + if (nswdev > 1) + nswap = ((nswap + dmmax - 1) / dmmax) * dmmax; + nswap *= nswdev; + if (swdevt[0].sw_vp == NULL && + bdevvp(swdevt[0].sw_dev, &swdevt[0].sw_vp)) + panic("swapvp"); +#endif + if (nswap == 0) + printf("WARNING: no swap space found\n"); + else if ((error = swfree(p, 0)) == ENXIO) + printf("WARNING: primary swap device not configured\n"); + else if (error) { + printf("swfree errno %d\n", error); /* XXX */ + panic("swapinit swfree 0"); + } + + /* + * Now set up swap buffer headers. + */ + bswlist.b_actf = sp; + for (i = 0; i < nswbuf - 1; i++, sp++) { + sp->b_actf = sp + 1; + sp->b_rcred = sp->b_wcred = p->p_ucred; + sp->b_vnbufs.le_next = NOLIST; + } + sp->b_rcred = sp->b_wcred = p->p_ucred; + sp->b_vnbufs.le_next = NOLIST; + sp->b_actf = NULL; +} + +void +swstrategy(bp) + register struct buf *bp; +{ + int sz, off, seg, index; + register struct swdevt *sp; + struct vnode *vp; + + sz = howmany(bp->b_bcount, DEV_BSIZE); + if (bp->b_blkno + sz > nswap) { + bp->b_error = EINVAL; + bp->b_flags |= B_ERROR; + biodone(bp); + return; + } + if (nswdev > 1) { +#ifdef SEQSWAP + if (bp->b_blkno < niswap) { + if (niswdev > 1) { + off = bp->b_blkno % dmmax; + if (off+sz > dmmax) { + bp->b_error = EINVAL; + bp->b_flags |= B_ERROR; + biodone(bp); + return; + } + seg = bp->b_blkno / dmmax; + index = seg % niswdev; + seg /= niswdev; + bp->b_blkno = seg*dmmax + off; + } else + index = 0; + } else { + register struct swdevt *swp; + + bp->b_blkno -= niswap; + for (index = niswdev, swp = &swdevt[niswdev]; + swp->sw_dev != NODEV; + swp++, index++) { + if (bp->b_blkno < swp->sw_nblks) + break; + bp->b_blkno -= swp->sw_nblks; + } + if (swp->sw_dev == NODEV || + bp->b_blkno+sz > swp->sw_nblks) { + bp->b_error = swp->sw_dev == NODEV ? + ENODEV : EINVAL; + bp->b_flags |= B_ERROR; + biodone(bp); + return; + } + } +#else + off = bp->b_blkno % dmmax; + if (off+sz > dmmax) { + bp->b_error = EINVAL; + bp->b_flags |= B_ERROR; + biodone(bp); + return; + } + seg = bp->b_blkno / dmmax; + index = seg % nswdev; + seg /= nswdev; + bp->b_blkno = seg*dmmax + off; +#endif + } else + index = 0; + sp = &swdevt[index]; + if (sp->sw_vp == NULL) { + bp->b_error = ENODEV; + bp->b_flags |= B_ERROR; + biodone(bp); + return; + } + if ((bp->b_dev = sp->sw_dev) == NODEV && sp->sw_vp->v_type != VREG) + panic("swstrategy"); + VHOLD(sp->sw_vp); + if ((bp->b_flags & B_READ) == 0) { + if (vp = bp->b_vp) { + vp->v_numoutput--; + if ((vp->v_flag & VBWAIT) && vp->v_numoutput <= 0) { + vp->v_flag &= ~VBWAIT; + wakeup((caddr_t)&vp->v_numoutput); + } + } + sp->sw_vp->v_numoutput++; + } + if (bp->b_vp != NULL) + brelvp(bp); + bp->b_vp = sp->sw_vp; + VOP_STRATEGY(bp); +} + +int +swread(dev, uio) + dev_t dev; + struct uio *uio; +{ + + return (physio(swstrategy, NULL, dev, B_READ, minphys, uio)); +} + +int +swwrite(dev, uio) + dev_t dev; + struct uio *uio; +{ + + return (physio(swstrategy, NULL, dev, B_WRITE, minphys, uio)); +} + +/* + * System call swapon(name) enables swapping on device name, + * which must be in the swdevsw. Return EBUSY + * if already swapping on this device. + */ +/* ARGSUSED */ +int +sys_swapon(p, v, retval) + struct proc *p; + void *v; + register_t *retval; +{ + struct sys_swapon_args /* { + syscallarg(char *) name; + } */ *uap = v; + register struct vnode *vp; + register struct swdevt *sp; + dev_t dev; + int error; + struct nameidata nd; + + if (error = suser(p->p_ucred, &p->p_acflag)) + return (error); + NDINIT(&nd, LOOKUP, FOLLOW, UIO_USERSPACE, SCARG(uap, name), p); + if (error = namei(&nd)) + return (error); + vp = nd.ni_vp; + if (vp->v_type != VBLK) { + vrele(vp); + return (ENOTBLK); + } + dev = (dev_t)vp->v_rdev; + if (major(dev) >= nblkdev) { + vrele(vp); + return (ENXIO); + } + for (sp = &swdevt[0]; sp->sw_dev != NODEV; sp++) { + if (sp->sw_dev == dev) { + if (sp->sw_flags & SW_FREED) { + vrele(vp); + return (EBUSY); + } + sp->sw_vp = vp; + if (error = swfree(p, sp - swdevt)) { + vrele(vp); + return (error); + } + return (0); + } +#ifdef SEQSWAP + /* + * If we have reached a non-freed sequential device without + * finding what we are looking for, it is an error. + * That is because all interleaved devices must come first + * and sequential devices must be freed in order. + */ + if ((sp->sw_flags & (SW_SEQUENTIAL|SW_FREED)) == SW_SEQUENTIAL) + break; +#endif + } + vrele(vp); + return (EINVAL); +} + +/* + * Swfree(index) frees the index'th portion of the swap map. + * Each of the nswdev devices provides 1/nswdev'th of the swap + * space, which is laid out with blocks of dmmax pages circularly + * among the devices. + */ +int +swfree(p, index) + struct proc *p; + int index; +{ + register struct swdevt *sp; + register swblk_t vsbase; + register long blk; + struct vnode *vp; + register swblk_t dvbase; + register int nblks; + int error; + + sp = &swdevt[index]; + vp = sp->sw_vp; + /* If root on swap, then the skip open/close operations. */ + if (vp != rootvp) { + if (error = VOP_OPEN(vp, FREAD|FWRITE, p->p_ucred, p)) + return (error); + } + sp->sw_flags |= SW_FREED; + nblks = sp->sw_nblks; + /* + * Some devices may not exist til after boot time. + * If so, their nblk count will be 0. + */ + if (nblks <= 0) { + int perdev; + dev_t dev = sp->sw_dev; + + if (bdevsw[major(dev)].d_psize == 0 || + (nblks = (*bdevsw[major(dev)].d_psize)(dev)) == -1) { + if (vp != rootvp) + (void) VOP_CLOSE(vp, FREAD|FWRITE, p->p_ucred, p); + sp->sw_flags &= ~SW_FREED; + return (ENXIO); + } +#ifdef SEQSWAP + if (index < niswdev) { + perdev = niswap / niswdev; + if (nblks > perdev) + nblks = perdev; + } else { + if (nblks % dmmax) + nblks -= (nblks % dmmax); + nswap += nblks; + } +#else + perdev = nswap / nswdev; + if (nblks > perdev) + nblks = perdev; +#endif + sp->sw_nblks = nblks; + } + if (nblks == 0) { + if (vp != rootvp) + (void) VOP_CLOSE(vp, FREAD|FWRITE, p->p_ucred, p); + sp->sw_flags &= ~SW_FREED; + return (0); /* XXX error? */ + } +#ifdef SEQSWAP + if (sp->sw_flags & SW_SEQUENTIAL) { + register struct swdevt *swp; + + blk = niswap; + for (swp = &swdevt[niswdev]; swp != sp; swp++) + blk += swp->sw_nblks; + rmfree(swapmap, nblks, blk); + return (0); + } +#endif + for (dvbase = 0; dvbase < nblks; dvbase += dmmax) { + blk = nblks - dvbase; +#ifdef SEQSWAP + if ((vsbase = index*dmmax + dvbase*niswdev) >= niswap) + panic("swfree"); +#else + if ((vsbase = index*dmmax + dvbase*nswdev) >= nswap) + panic("swfree"); +#endif + if (blk > dmmax) + blk = dmmax; + if (vsbase == 0) { + /* + * First of all chunks... initialize the swapmap. + * Don't use the first cluster of the device + * in case it starts with a label or boot block. + */ + rminit(swapmap, blk - ctod(CLSIZE), + vsbase + ctod(CLSIZE), "swap", nswapmap); + } else if (dvbase == 0) { + /* + * Don't use the first cluster of the device + * in case it starts with a label or boot block. + */ + rmfree(swapmap, blk - ctod(CLSIZE), + vsbase + ctod(CLSIZE)); + } else + rmfree(swapmap, blk, vsbase); + } + + /* + * Preserve the mini-root if appropriate: + * Note: this requires !SEQSWAP && nswdev==1 + * + * A mini-root gets copied into the front of the swap + * and we run over top of the swap area just long + * enough for us to do a mkfs and restor of the real + * root (sure beats rewriting standalone restor). + */ + if (vp == rootvp) { + struct mount *mp; + struct statfs *sp; + long firstblk; + int rootblks; + +#ifdef MINIROOTSIZE + rootblks = MINIROOTSIZE; +#else + /* Get size from root FS (mountroot did statfs) */ + mp = rootvnode->v_mount; + sp = &mp->mnt_stat; + rootblks = sp->f_blocks * (sp->f_bsize / DEV_BSIZE); +#endif + if (rootblks > nblks) + panic("swfree miniroot size"); + /* First ctod(CLSIZE) blocks are not in the map. */ + firstblk = rmalloc(swapmap, rootblks - ctod(CLSIZE)); + if (firstblk != ctod(CLSIZE)) + panic("swfree miniroot save"); + printf("Preserved %d blocks of miniroot leaving %d pages of swap\n", + rootblks, dtoc(nblks - rootblks)); + } + + return (0); +} diff --git a/sys/vm/vm_unix.c b/sys/vm/vm_unix.c new file mode 100644 index 00000000000..40ae7e564ae --- /dev/null +++ b/sys/vm/vm_unix.c @@ -0,0 +1,235 @@ +/* $NetBSD: vm_unix.c,v 1.18 1995/10/07 06:29:04 mycroft Exp $ */ + +/* + * Copyright (c) 1988 University of Utah. + * Copyright (c) 1991, 1993 + * The Regents of the University of California. All rights reserved. + * + * This code is derived from software contributed to Berkeley by + * the Systems Programming Group of the University of Utah Computer + * Science Department. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. All advertising materials mentioning features or use of this software + * must display the following acknowledgement: + * This product includes software developed by the University of + * California, Berkeley and its contributors. + * 4. Neither the name of the University nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * from: Utah $Hdr: vm_unix.c 1.1 89/11/07$ + * + * @(#)vm_unix.c 8.1 (Berkeley) 6/11/93 + */ + +/* + * Traditional sbrk/grow interface to VM + */ +#include <sys/param.h> +#include <sys/systm.h> +#include <sys/proc.h> +#include <sys/resourcevar.h> +#include <sys/vnode.h> +#include <sys/core.h> + +#include <sys/mount.h> +#include <sys/syscallargs.h> + +#include <vm/vm.h> + +/* ARGSUSED */ +int +sys_obreak(p, v, retval) + struct proc *p; + void *v; + register_t *retval; +{ + struct sys_obreak_args /* { + syscallarg(char *) nsize; + } */ *uap = v; + register struct vmspace *vm = p->p_vmspace; + vm_offset_t new, old; + int rv; + register int diff; + + old = (vm_offset_t)vm->vm_daddr; + new = round_page(SCARG(uap, nsize)); + if ((int)(new - old) > p->p_rlimit[RLIMIT_DATA].rlim_cur) + return(ENOMEM); + old = round_page(old + ctob(vm->vm_dsize)); + diff = new - old; + if (diff > 0) { + rv = vm_allocate(&vm->vm_map, &old, diff, FALSE); + if (rv != KERN_SUCCESS) { + uprintf("sbrk: grow failed, return = %d\n", rv); + return(ENOMEM); + } + vm->vm_dsize += btoc(diff); + } else if (diff < 0) { + diff = -diff; + rv = vm_deallocate(&vm->vm_map, new, diff); + if (rv != KERN_SUCCESS) { + uprintf("sbrk: shrink failed, return = %d\n", rv); + return(ENOMEM); + } + vm->vm_dsize -= btoc(diff); + } + return(0); +} + +/* + * Enlarge the "stack segment" to include the specified + * stack pointer for the process. + */ +int +grow(p, sp) + struct proc *p; + vm_offset_t sp; +{ + register struct vmspace *vm = p->p_vmspace; + register int si; + + /* + * For user defined stacks (from sendsig). + */ + if (sp < (vm_offset_t)vm->vm_maxsaddr) + return (0); + /* + * For common case of already allocated (from trap). + */ + if (sp >= USRSTACK - ctob(vm->vm_ssize)) + return (1); + /* + * Really need to check vs limit and increment stack size if ok. + */ + si = clrnd(btoc(USRSTACK-sp) - vm->vm_ssize); + if (vm->vm_ssize + si > btoc(p->p_rlimit[RLIMIT_STACK].rlim_cur)) + return (0); + vm->vm_ssize += si; + return (1); +} + +/* ARGSUSED */ +int +sys_ovadvise(p, v, retval) + struct proc *p; + void *v; + register_t *retval; +{ + struct sys_ovadvise_args /* { + syscallarg(int) anom; + } */ *uap = v; + + return (EINVAL); +} + +int +vm_coredump(p, vp, cred, chdr) + struct proc *p; + struct vnode *vp; + struct ucred *cred; + struct core *chdr; +{ + register struct vmspace *vm = p->p_vmspace; + register vm_map_t map = &vm->vm_map; + register vm_map_entry_t entry; + vm_offset_t start, end; + struct coreseg cseg; + off_t offset; + int flag, error = 0; + + if (!map->is_main_map) { +#ifdef DEBUG + uprintf( + "vm_coredump: %s map 0x%lx: pmap=0x%lx,ref=%d,nentries=%d,version=%d\n", + (map->is_main_map ? "Task" : "Share"), + (long)map, (long)(map->pmap), + map->ref_count, map->nentries, + map->timestamp); +#endif + return EIO; + } + + offset = chdr->c_hdrsize + chdr->c_seghdrsize + chdr->c_cpusize; + + for (entry = map->header.next; entry != &map->header; + entry = entry->next) { + + if (entry->is_a_map || entry->is_sub_map) { +#ifdef DEBUG + uprintf( + "vm_coredump: entry: share=0x%lx, offset=0x%lx\n", + (long) entry->object.share_map, + (long) entry->offset); +#endif + continue; + } + + if (!(entry->protection & VM_PROT_WRITE)) + continue; + + start = entry->start; + end = entry->end; + + if (start >= VM_MAXUSER_ADDRESS) + continue; + + if (end > VM_MAXUSER_ADDRESS) + end = VM_MAXUSER_ADDRESS; + + if (start >= (vm_offset_t)vm->vm_maxsaddr) { + flag = CORE_STACK; + start = trunc_page(USRSTACK - ctob(vm->vm_ssize)); + if (start >= end) + continue; + } else + flag = CORE_DATA; + + /* + * Set up a new core file segment. + */ + CORE_SETMAGIC(cseg, CORESEGMAGIC, CORE_GETMID(*chdr), flag); + cseg.c_addr = start; + cseg.c_size = end - start; + + error = vn_rdwr(UIO_WRITE, vp, + (caddr_t)&cseg, chdr->c_seghdrsize, + offset, UIO_SYSSPACE, + IO_NODELOCKED|IO_UNIT, cred, (int *) NULL, p); + if (error) + break; + + offset += chdr->c_seghdrsize; + error = vn_rdwr(UIO_WRITE, vp, + (caddr_t)cseg.c_addr, (int)cseg.c_size, + offset, UIO_USERSPACE, + IO_NODELOCKED|IO_UNIT, cred, (int *) NULL, p); + if (error) + break; + + offset += cseg.c_size; + chdr->c_nseg++; + } + + return error; +} diff --git a/sys/vm/vm_user.c b/sys/vm/vm_user.c new file mode 100644 index 00000000000..26d8730445e --- /dev/null +++ b/sys/vm/vm_user.c @@ -0,0 +1,318 @@ +/* $NetBSD: vm_user.c,v 1.11 1994/10/20 04:27:34 cgd Exp $ */ + +/* + * Copyright (c) 1991, 1993 + * The Regents of the University of California. All rights reserved. + * + * This code is derived from software contributed to Berkeley by + * The Mach Operating System project at Carnegie-Mellon University. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. All advertising materials mentioning features or use of this software + * must display the following acknowledgement: + * This product includes software developed by the University of + * California, Berkeley and its contributors. + * 4. Neither the name of the University nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * @(#)vm_user.c 8.2 (Berkeley) 1/12/94 + * + * + * Copyright (c) 1987, 1990 Carnegie-Mellon University. + * All rights reserved. + * + * Authors: Avadis Tevanian, Jr., Michael Wayne Young + * + * Permission to use, copy, modify and distribute this software and + * its documentation is hereby granted, provided that both the copyright + * notice and this permission notice appear in all copies of the + * software, derivative works or modified versions, and any portions + * thereof, and that both notices appear in supporting documentation. + * + * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" + * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND + * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. + * + * Carnegie Mellon requests users of this software to return to + * + * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU + * School of Computer Science + * Carnegie Mellon University + * Pittsburgh PA 15213-3890 + * + * any improvements or extensions that they make and grant Carnegie the + * rights to redistribute these changes. + */ + +/* + * User-exported virtual memory functions. + */ + +#include <sys/param.h> +#include <sys/systm.h> +#include <sys/proc.h> + +#include <vm/vm.h> + +simple_lock_data_t vm_alloc_lock; /* XXX */ + +#ifdef MACHVMCOMPAT +/* + * BSD style syscall interfaces to MACH calls + * All return MACH return values. + */ +struct svm_allocate_args { + vm_map_t map; + vm_offset_t *addr; + vm_size_t size; + boolean_t anywhere; +}; +/* ARGSUSED */ +int +svm_allocate(p, uap, retval) + struct proc *p; + struct svm_allocate_args *uap; + register_t *retval; +{ + vm_offset_t addr; + int rv; + + SCARG(uap, map) = p->p_map; /* XXX */ + + if (copyin((caddr_t)SCARG(uap, addr), (caddr_t)&addr, sizeof (addr))) + rv = KERN_INVALID_ARGUMENT; + else + rv = vm_allocate(SCARG(uap, map), &addr, SCARG(uap, size), + SCARG(uap, anywhere)); + if (rv == KERN_SUCCESS) { + if (copyout((caddr_t)&addr, (caddr_t)SCARG(uap, addr), + sizeof(addr))) + rv = KERN_INVALID_ARGUMENT; + } + return((int)rv); +} + +struct svm_deallocate_args { + vm_map_t map; + vm_offset_t addr; + vm_size_t size; +}; +/* ARGSUSED */ +int +svm_deallocate(p, uap, retval) + struct proc *p; + struct svm_deallocate_args *uap; + register_t *retval; +{ + int rv; + + SCARG(uap, map) = p->p_map; /* XXX */ + rv = vm_deallocate(SCARG(uap, map), SCARG(uap, addr), SCARG(uap, size)); + return((int)rv); +} + +struct svm_inherit_args { + vm_map_t map; + vm_offset_t addr; + vm_size_t size; + vm_inherit_t inherit; +}; +/* ARGSUSED */ +int +svm_inherit(p, uap, retval) + struct proc *p; + struct svm_inherit_args *uap; + register_t *retval; +{ + int rv; + + SCARG(uap, map) = p->p_map; /* XXX */ + rv = vm_inherit(SCARG(uap, map), SCARG(uap, addr), SCARG(uap, size), + SCARG(uap, inherit)); + return((int)rv); +} + +struct svm_protect_args { + vm_map_t map; + vm_offset_t addr; + vm_size_t size; + boolean_t setmax; + vm_prot_t prot; +}; +/* ARGSUSED */ +int +svm_protect(p, uap, retval) + struct proc *p; + struct svm_protect_args *uap; + register_t *retval; +{ + int rv; + + SCARG(uap, map) = p->p_map; /* XXX */ + rv = vm_protect(SCARG(uap, map), SCARG(uap, addr), SCARG(uap, size), + SCARG(uap, setmax), SCARG(uap, prot)); + return((int)rv); +} + +/* + * vm_inherit sets the inheritence of the specified range in the + * specified map. + */ +int +vm_inherit(map, start, size, new_inheritance) + register vm_map_t map; + vm_offset_t start; + vm_size_t size; + vm_inherit_t new_inheritance; +{ + if (map == NULL) + return(KERN_INVALID_ARGUMENT); + + return(vm_map_inherit(map, trunc_page(start), round_page(start+size), new_inheritance)); +} + +/* + * vm_protect sets the protection of the specified range in the + * specified map. + */ + +int +vm_protect(map, start, size, set_maximum, new_protection) + register vm_map_t map; + vm_offset_t start; + vm_size_t size; + boolean_t set_maximum; + vm_prot_t new_protection; +{ + if (map == NULL) + return(KERN_INVALID_ARGUMENT); + + return(vm_map_protect(map, trunc_page(start), round_page(start+size), new_protection, set_maximum)); +} +#endif + +/* + * vm_allocate allocates "zero fill" memory in the specfied + * map. + */ +int +vm_allocate(map, addr, size, anywhere) + register vm_map_t map; + register vm_offset_t *addr; + register vm_size_t size; + boolean_t anywhere; +{ + int result; + + if (map == NULL) + return(KERN_INVALID_ARGUMENT); + if (size == 0) { + *addr = 0; + return(KERN_SUCCESS); + } + + if (anywhere) + *addr = vm_map_min(map); + else + *addr = trunc_page(*addr); + size = round_page(size); + + result = vm_map_find(map, NULL, (vm_offset_t) 0, addr, size, anywhere); + + return(result); +} + +/* + * vm_deallocate deallocates the specified range of addresses in the + * specified address map. + */ +int +vm_deallocate(map, start, size) + register vm_map_t map; + vm_offset_t start; + vm_size_t size; +{ + if (map == NULL) + return(KERN_INVALID_ARGUMENT); + + if (size == (vm_offset_t) 0) + return(KERN_SUCCESS); + + return(vm_map_remove(map, trunc_page(start), round_page(start+size))); +} + +/* + * Similar to vm_allocate but assigns an explicit pager. + */ +int +vm_allocate_with_pager(map, addr, size, anywhere, pager, poffset, internal) + register vm_map_t map; + register vm_offset_t *addr; + register vm_size_t size; + boolean_t anywhere; + vm_pager_t pager; + vm_offset_t poffset; + boolean_t internal; +{ + register vm_object_t object; + register int result; + + if (map == NULL) + return(KERN_INVALID_ARGUMENT); + + *addr = trunc_page(*addr); + size = round_page(size); + + /* + * Lookup the pager/paging-space in the object cache. + * If it's not there, then create a new object and cache + * it. + */ + object = vm_object_lookup(pager); + cnt.v_lookups++; + if (object == NULL) { + object = vm_object_allocate(size); + /* + * From Mike Hibler: "unnamed anonymous objects should never + * be on the hash list ... For now you can just change + * vm_allocate_with_pager to not do vm_object_enter if this + * is an internal object ..." + */ + if (!internal) + vm_object_enter(object, pager); + } else + cnt.v_hits++; + if (internal) + object->flags |= OBJ_INTERNAL; + else { + object->flags &= ~OBJ_INTERNAL; + cnt.v_nzfod -= atop(size); + } + + result = vm_map_find(map, object, poffset, addr, size, anywhere); + if (result != KERN_SUCCESS) + vm_object_deallocate(object); + else if (pager != NULL) + vm_object_setpager(object, pager, (vm_offset_t) 0, TRUE); + return(result); +} diff --git a/sys/vm/vnode_pager.c b/sys/vm/vnode_pager.c new file mode 100644 index 00000000000..b80754f1005 --- /dev/null +++ b/sys/vm/vnode_pager.c @@ -0,0 +1,594 @@ +/* $NetBSD: vnode_pager.c,v 1.16 1994/06/29 06:48:47 cgd Exp $ */ + +/* + * Copyright (c) 1990 University of Utah. + * Copyright (c) 1991, 1993 + * The Regents of the University of California. All rights reserved. + * + * This code is derived from software contributed to Berkeley by + * the Systems Programming Group of the University of Utah Computer + * Science Department. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. All advertising materials mentioning features or use of this software + * must display the following acknowledgement: + * This product includes software developed by the University of + * California, Berkeley and its contributors. + * 4. Neither the name of the University nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * @(#)vnode_pager.c 8.8 (Berkeley) 2/13/94 + */ + +/* + * Page to/from files (vnodes). + * + * TODO: + * pageouts + * fix credential use (uses current process credentials now) + */ + +#include <sys/param.h> +#include <sys/systm.h> +#include <sys/proc.h> +#include <sys/malloc.h> +#include <sys/vnode.h> +#include <sys/uio.h> +#include <sys/mount.h> + +#include <vm/vm.h> +#include <vm/vm_page.h> +#include <vm/vnode_pager.h> + +struct pagerlst vnode_pager_list; /* list of managed vnodes */ + +#ifdef DEBUG +int vpagerdebug = 0x00; +#define VDB_FOLLOW 0x01 +#define VDB_INIT 0x02 +#define VDB_IO 0x04 +#define VDB_FAIL 0x08 +#define VDB_ALLOC 0x10 +#define VDB_SIZE 0x20 +#endif + +static vm_pager_t vnode_pager_alloc + __P((caddr_t, vm_size_t, vm_prot_t, vm_offset_t)); +static void vnode_pager_cluster + __P((vm_pager_t, vm_offset_t, + vm_offset_t *, vm_offset_t *)); +static void vnode_pager_dealloc __P((vm_pager_t)); +static int vnode_pager_getpage + __P((vm_pager_t, vm_page_t *, int, boolean_t)); +static boolean_t vnode_pager_haspage __P((vm_pager_t, vm_offset_t)); +static void vnode_pager_init __P((void)); +static int vnode_pager_io + __P((vn_pager_t, vm_page_t *, int, + boolean_t, enum uio_rw)); +static boolean_t vnode_pager_putpage + __P((vm_pager_t, vm_page_t *, int, boolean_t)); + +struct pagerops vnodepagerops = { + vnode_pager_init, + vnode_pager_alloc, + vnode_pager_dealloc, + vnode_pager_getpage, + vnode_pager_putpage, + vnode_pager_haspage, + vnode_pager_cluster +}; + +static void +vnode_pager_init() +{ +#ifdef DEBUG + if (vpagerdebug & VDB_FOLLOW) + printf("vnode_pager_init()\n"); +#endif + TAILQ_INIT(&vnode_pager_list); +} + +/* + * Allocate (or lookup) pager for a vnode. + * Handle is a vnode pointer. + */ +static vm_pager_t +vnode_pager_alloc(handle, size, prot, foff) + caddr_t handle; + vm_size_t size; + vm_prot_t prot; + vm_offset_t foff; +{ + register vm_pager_t pager; + register vn_pager_t vnp; + vm_object_t object; + struct vattr vattr; + struct vnode *vp; + struct proc *p = curproc; /* XXX */ + +#ifdef DEBUG + if (vpagerdebug & (VDB_FOLLOW|VDB_ALLOC)) + printf("vnode_pager_alloc(%x, %x, %x)\n", handle, size, prot); +#endif + /* + * Pageout to vnode, no can do yet. + */ + if (handle == NULL) + return(NULL); + + /* + * Vnodes keep a pointer to any associated pager so no need to + * lookup with vm_pager_lookup. + */ + vp = (struct vnode *)handle; + pager = (vm_pager_t)vp->v_vmdata; + if (pager == NULL) { + /* + * Allocate pager structures + */ + pager = (vm_pager_t)malloc(sizeof *pager, M_VMPAGER, M_WAITOK); + if (pager == NULL) + return(NULL); + vnp = (vn_pager_t)malloc(sizeof *vnp, M_VMPGDATA, M_WAITOK); + if (vnp == NULL) { + free((caddr_t)pager, M_VMPAGER); + return(NULL); + } + /* + * And an object of the appropriate size + */ + if (VOP_GETATTR(vp, &vattr, p->p_ucred, p) == 0) { + object = vm_object_allocate(round_page(vattr.va_size)); + vm_object_enter(object, pager); + vm_object_setpager(object, pager, 0, TRUE); + } else { + free((caddr_t)vnp, M_VMPGDATA); + free((caddr_t)pager, M_VMPAGER); + return(NULL); + } + /* + * Hold a reference to the vnode and initialize pager data. + */ + VREF(vp); + vnp->vnp_flags = 0; + vnp->vnp_vp = vp; + vnp->vnp_size = vattr.va_size; + TAILQ_INSERT_TAIL(&vnode_pager_list, pager, pg_list); + pager->pg_handle = handle; + pager->pg_type = PG_VNODE; + pager->pg_flags = 0; + pager->pg_ops = &vnodepagerops; + pager->pg_data = vnp; + vp->v_vmdata = (caddr_t)pager; + } else { + /* + * vm_object_lookup() will remove the object from the + * cache if found and also gain a reference to the object. + */ + object = vm_object_lookup(pager); +#ifdef DEBUG + vnp = (vn_pager_t)pager->pg_data; +#endif + } +#ifdef DEBUG + if (vpagerdebug & VDB_ALLOC) + printf("vnode_pager_setup: vp %x sz %x pager %x object %x\n", + vp, vnp->vnp_size, pager, object); +#endif + return(pager); +} + +static void +vnode_pager_dealloc(pager) + vm_pager_t pager; +{ + register vn_pager_t vnp = (vn_pager_t)pager->pg_data; + register struct vnode *vp; +#ifdef NOTDEF + struct proc *p = curproc; /* XXX */ +#endif + +#ifdef DEBUG + if (vpagerdebug & VDB_FOLLOW) + printf("vnode_pager_dealloc(%x)\n", pager); +#endif + if (vp = vnp->vnp_vp) { + vp->v_vmdata = NULL; + vp->v_flag &= ~VTEXT; +#if NOTDEF + /* can hang if done at reboot on NFS FS */ + (void) VOP_FSYNC(vp, p->p_ucred, p); +#endif + vrele(vp); + } + TAILQ_REMOVE(&vnode_pager_list, pager, pg_list); + free((caddr_t)vnp, M_VMPGDATA); + free((caddr_t)pager, M_VMPAGER); +} + +static int +vnode_pager_getpage(pager, mlist, npages, sync) + vm_pager_t pager; + vm_page_t *mlist; + int npages; + boolean_t sync; +{ + +#ifdef DEBUG + if (vpagerdebug & VDB_FOLLOW) + printf("vnode_pager_getpage(%x, %x, %x, %x)\n", + pager, mlist, npages, sync); +#endif + return(vnode_pager_io((vn_pager_t)pager->pg_data, + mlist, npages, sync, UIO_READ)); +} + +static boolean_t +vnode_pager_putpage(pager, mlist, npages, sync) + vm_pager_t pager; + vm_page_t *mlist; + int npages; + boolean_t sync; +{ + int err; + +#ifdef DEBUG + if (vpagerdebug & VDB_FOLLOW) + printf("vnode_pager_putpage(%x, %x, %x, %x)\n", + pager, mlist, npages, sync); +#endif + if (pager == NULL) + return (FALSE); /* ??? */ + err = vnode_pager_io((vn_pager_t)pager->pg_data, + mlist, npages, sync, UIO_WRITE); + /* + * If the operation was successful, mark the pages clean. + */ + if (err == VM_PAGER_OK) { + while (npages--) { + (*mlist)->flags |= PG_CLEAN; + pmap_clear_modify(VM_PAGE_TO_PHYS(*mlist)); + mlist++; + } + } + return(err); +} + +static boolean_t +vnode_pager_haspage(pager, offset) + vm_pager_t pager; + vm_offset_t offset; +{ + register vn_pager_t vnp = (vn_pager_t)pager->pg_data; + daddr_t bn; + int err; + +#ifdef DEBUG + if (vpagerdebug & VDB_FOLLOW) + printf("vnode_pager_haspage(%x, %x)\n", pager, offset); +#endif + + /* + * Offset beyond end of file, do not have the page + * Lock the vnode first to make sure we have the most recent + * version of the size. + */ + VOP_LOCK(vnp->vnp_vp); + if (offset >= vnp->vnp_size) { + VOP_UNLOCK(vnp->vnp_vp); +#ifdef DEBUG + if (vpagerdebug & (VDB_FAIL|VDB_SIZE)) + printf("vnode_pager_haspage: pg %x, off %x, size %x\n", + pager, offset, vnp->vnp_size); +#endif + return(FALSE); + } + + /* + * Read the index to find the disk block to read + * from. If there is no block, report that we don't + * have this data. + * + * Assumes that the vnode has whole page or nothing. + */ + err = VOP_BMAP(vnp->vnp_vp, + offset / vnp->vnp_vp->v_mount->mnt_stat.f_iosize, + (struct vnode **)0, &bn, NULL); + VOP_UNLOCK(vnp->vnp_vp); + if (err) { +#ifdef DEBUG + if (vpagerdebug & VDB_FAIL) + printf("vnode_pager_haspage: BMAP err %d, pg %x, off %x\n", + err, pager, offset); +#endif + return(TRUE); + } + return((long)bn < 0 ? FALSE : TRUE); +} + +static void +vnode_pager_cluster(pager, offset, loffset, hoffset) + vm_pager_t pager; + vm_offset_t offset; + vm_offset_t *loffset; + vm_offset_t *hoffset; +{ + vn_pager_t vnp = (vn_pager_t)pager->pg_data; + vm_offset_t loff, hoff; + +#ifdef DEBUG + if (vpagerdebug & VDB_FOLLOW) + printf("vnode_pager_cluster(%x, %x) ", pager, offset); +#endif + loff = offset; + if (loff >= vnp->vnp_size) + panic("vnode_pager_cluster: bad offset"); + /* + * XXX could use VOP_BMAP to get maxcontig value + */ + hoff = loff + MAXBSIZE; + if (hoff > round_page(vnp->vnp_size)) + hoff = round_page(vnp->vnp_size); + + *loffset = loff; + *hoffset = hoff; +#ifdef DEBUG + if (vpagerdebug & VDB_FOLLOW) + printf("returns [%x-%x]\n", loff, hoff); +#endif +} + +/* + * (XXX) + * Lets the VM system know about a change in size for a file. + * If this vnode is mapped into some address space (i.e. we have a pager + * for it) we adjust our own internal size and flush any cached pages in + * the associated object that are affected by the size change. + * + * Note: this routine may be invoked as a result of a pager put + * operation (possibly at object termination time), so we must be careful. + */ +void +vnode_pager_setsize(vp, nsize) + struct vnode *vp; + u_long nsize; +{ + register vn_pager_t vnp; + register vm_object_t object; + vm_pager_t pager; + + /* + * Not a mapped vnode + */ + if (vp == NULL || vp->v_type != VREG || vp->v_vmdata == NULL) + return; + /* + * Hasn't changed size + */ + pager = (vm_pager_t)vp->v_vmdata; + vnp = (vn_pager_t)pager->pg_data; + if (nsize == vnp->vnp_size) + return; + /* + * No object. + * This can happen during object termination since + * vm_object_page_clean is called after the object + * has been removed from the hash table, and clean + * may cause vnode write operations which can wind + * up back here. + */ + object = vm_object_lookup(pager); + if (object == NULL) + return; + +#ifdef DEBUG + if (vpagerdebug & (VDB_FOLLOW|VDB_SIZE)) + printf("vnode_pager_setsize: vp %x obj %x osz %d nsz %d\n", + vp, object, vnp->vnp_size, nsize); +#endif + /* + * File has shrunk. + * Toss any cached pages beyond the new EOF. + */ + if (nsize < vnp->vnp_size) { + vm_object_lock(object); + vm_object_page_remove(object, + (vm_offset_t)nsize, vnp->vnp_size); + vm_object_unlock(object); + } + vnp->vnp_size = (vm_offset_t)nsize; + vm_object_deallocate(object); +} + +void +vnode_pager_umount(mp) + register struct mount *mp; +{ + register vm_pager_t pager, npager; + struct vnode *vp; + + for (pager = vnode_pager_list.tqh_first; pager != NULL; pager = npager){ + /* + * Save the next pointer now since uncaching may + * terminate the object and render pager invalid + */ + npager = pager->pg_list.tqe_next; + vp = ((vn_pager_t)pager->pg_data)->vnp_vp; + if (mp == (struct mount *)0 || vp->v_mount == mp) { + VOP_LOCK(vp); + (void) vnode_pager_uncache(vp); + VOP_UNLOCK(vp); + } + } +} + +/* + * Remove vnode associated object from the object cache. + * + * XXX unlock the vnode if it is currently locked. + * We must do this since uncaching the object may result in its + * destruction which may initiate paging activity which may necessitate + * re-locking the vnode. + */ +boolean_t +vnode_pager_uncache(vp) + register struct vnode *vp; +{ + register vm_object_t object; + boolean_t uncached; + vm_pager_t pager; + + /* + * Not a mapped vnode + */ + pager = (vm_pager_t)vp->v_vmdata; + if (pager == NULL) + return (TRUE); +#ifdef DEBUG + if (!VOP_ISLOCKED(vp)) { +#ifdef NFSCLIENT + extern int (**nfsv2_vnodeop_p)(); + extern int (**spec_nfsv2nodeop_p)(); +#ifdef FIFO + extern int (**fifo_nfsv2nodeop_p)(); +#endif + + if (vp->v_op != nfsv2_vnodeop_p + && vp->v_op != spec_nfsv2nodeop_p +#ifdef FIFO + && vp->v_op != fifo_nfsv2nodeop_p +#endif + ) + +#endif + panic("vnode_pager_uncache: vnode not locked!"); + } +#endif + /* + * Must use vm_object_lookup() as it actually removes + * the object from the cache list. + */ + object = vm_object_lookup(pager); + if (object) { + uncached = (object->ref_count <= 1); + VOP_UNLOCK(vp); + pager_cache(object, FALSE); + VOP_LOCK(vp); + } else + uncached = TRUE; + return(uncached); +} + +static int +vnode_pager_io(vnp, mlist, npages, sync, rw) + register vn_pager_t vnp; + vm_page_t *mlist; + int npages; + boolean_t sync; + enum uio_rw rw; +{ + struct uio auio; + struct iovec aiov; + vm_offset_t kva, foff; + int error, size; + struct proc *p = curproc; /* XXX */ + + /* XXX */ + vm_page_t m; + if (npages != 1) + panic("vnode_pager_io: cannot handle multiple pages"); + m = *mlist; + /* XXX */ + +#ifdef DEBUG + if (vpagerdebug & VDB_FOLLOW) + printf("vnode_pager_io(%x, %x, %c): vnode %x\n", + vnp, m, rw == UIO_READ ? 'R' : 'W', vnp->vnp_vp); +#endif + foff = m->offset + m->object->paging_offset; + /* + * Allocate a kernel virtual address and initialize so that + * we can use VOP_READ/WRITE routines. + */ + kva = vm_pager_map_pages(mlist, npages, sync); + if (kva == NULL) + return(VM_PAGER_AGAIN); + /* + * After all of the potentially blocking operations have been + * performed, we can do the size checks: + * read beyond EOF (returns error) + * short read + */ + VOP_LOCK(vnp->vnp_vp); + if (foff >= vnp->vnp_size) { + VOP_UNLOCK(vnp->vnp_vp); + vm_pager_unmap_pages(kva, npages); +#ifdef DEBUG + if (vpagerdebug & VDB_SIZE) + printf("vnode_pager_io: vp %x, off %d size %d\n", + vnp->vnp_vp, foff, vnp->vnp_size); +#endif + return(VM_PAGER_BAD); + } + if (foff + PAGE_SIZE > vnp->vnp_size) + size = vnp->vnp_size - foff; + else + size = PAGE_SIZE; + aiov.iov_base = (caddr_t)kva; + aiov.iov_len = size; + auio.uio_iov = &aiov; + auio.uio_iovcnt = 1; + auio.uio_offset = foff; + auio.uio_segflg = UIO_SYSSPACE; + auio.uio_rw = rw; + auio.uio_resid = size; + auio.uio_procp = (struct proc *)0; +#ifdef DEBUG + if (vpagerdebug & VDB_IO) + printf("vnode_pager_io: vp %x kva %x foff %x size %x", + vnp->vnp_vp, kva, foff, size); +#endif + if (rw == UIO_READ) + error = VOP_READ(vnp->vnp_vp, &auio, 0, p->p_ucred); + else + error = VOP_WRITE(vnp->vnp_vp, &auio, 0, p->p_ucred); + VOP_UNLOCK(vnp->vnp_vp); +#ifdef DEBUG + if (vpagerdebug & VDB_IO) { + if (error || auio.uio_resid) + printf(" returns error %x, resid %x", + error, auio.uio_resid); + printf("\n"); + } +#endif + if (!error) { + register int count = size - auio.uio_resid; + + if (count == 0) + error = EINVAL; + else if (count != PAGE_SIZE && rw == UIO_READ) + bzero((void *)(kva + count), PAGE_SIZE - count); + } + vm_pager_unmap_pages(kva, npages); + return (error ? VM_PAGER_ERROR : VM_PAGER_OK); +} diff --git a/sys/vm/vnode_pager.h b/sys/vm/vnode_pager.h new file mode 100644 index 00000000000..4f903f4db82 --- /dev/null +++ b/sys/vm/vnode_pager.h @@ -0,0 +1,61 @@ +/* $NetBSD: vnode_pager.h,v 1.6 1994/06/29 06:48:49 cgd Exp $ */ + +/* + * Copyright (c) 1990 University of Utah. + * Copyright (c) 1991, 1993 + * The Regents of the University of California. All rights reserved. + * + * This code is derived from software contributed to Berkeley by + * the Systems Programming Group of the University of Utah Computer + * Science Department. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. All advertising materials mentioning features or use of this software + * must display the following acknowledgement: + * This product includes software developed by the University of + * California, Berkeley and its contributors. + * 4. Neither the name of the University nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * @(#)vnode_pager.h 8.1 (Berkeley) 6/11/93 + */ + +#ifndef _VNODE_PAGER_ +#define _VNODE_PAGER_ 1 + +/* + * VNODE pager private data. + */ +struct vnpager { + int vnp_flags; /* flags */ + struct vnode *vnp_vp; /* vnode */ + vm_size_t vnp_size; /* vnode current size */ +}; +typedef struct vnpager *vn_pager_t; + +#define VN_PAGER_NULL ((vn_pager_t)0) + +#define VNP_PAGING 0x01 /* vnode used for pageout */ +#define VNP_CACHED 0x02 /* vnode is cached */ + +#endif /* _VNODE_PAGER_ */ |