diff options
Diffstat (limited to 'sys')
| -rw-r--r-- | sys/arch/amd64/include/cpu.h | 5 | ||||
| -rw-r--r-- | sys/arch/arm64/include/cpu.h | 5 | ||||
| -rw-r--r-- | sys/arch/i386/include/cpu.h | 5 | ||||
| -rw-r--r-- | sys/uvm/uvm_page.c | 20 | ||||
| -rw-r--r-- | sys/uvm/uvm_pdaemon.c | 4 | ||||
| -rw-r--r-- | sys/uvm/uvm_percpu.h | 48 | ||||
| -rw-r--r-- | sys/uvm/uvm_pmemrange.c | 155 | ||||
| -rw-r--r-- | sys/uvm/uvm_pmemrange.h | 6 | ||||
| -rw-r--r-- | sys/uvm/uvmexp.h | 8 |
9 files changed, 232 insertions, 24 deletions
diff --git a/sys/arch/amd64/include/cpu.h b/sys/arch/amd64/include/cpu.h index 7bd9a54fa0f..a6377384ea5 100644 --- a/sys/arch/amd64/include/cpu.h +++ b/sys/arch/amd64/include/cpu.h @@ -1,4 +1,4 @@ -/* $OpenBSD: cpu.h,v 1.165 2024/04/14 09:59:04 kettenis Exp $ */ +/* $OpenBSD: cpu.h,v 1.166 2024/04/17 13:12:58 mpi Exp $ */ /* $NetBSD: cpu.h,v 1.1 2003/04/26 18:39:39 fvdl Exp $ */ /*- @@ -53,6 +53,7 @@ #include <sys/sched.h> #include <sys/sensors.h> #include <sys/srp.h> +#include <uvm/uvm_percpu.h> #ifdef _KERNEL @@ -210,6 +211,8 @@ struct cpu_info { #ifdef MULTIPROCESSOR struct srp_hazard ci_srp_hazards[SRP_HAZARD_NUM]; +#define __HAVE_UVM_PERCPU + struct uvm_pmr_cache ci_uvm; /* [o] page cache */ #endif struct ksensordev ci_sensordev; diff --git a/sys/arch/arm64/include/cpu.h b/sys/arch/arm64/include/cpu.h index d0521a33ab3..ee820bb65b0 100644 --- a/sys/arch/arm64/include/cpu.h +++ b/sys/arch/arm64/include/cpu.h @@ -1,4 +1,4 @@ -/* $OpenBSD: cpu.h,v 1.43 2024/02/25 19:15:50 cheloha Exp $ */ +/* $OpenBSD: cpu.h,v 1.44 2024/04/17 13:12:58 mpi Exp $ */ /* * Copyright (c) 2016 Dale Rahn <drahn@dalerahn.com> * @@ -108,6 +108,7 @@ void arm32_vector_init(vaddr_t, int); #include <sys/device.h> #include <sys/sched.h> #include <sys/srp.h> +#include <uvm/uvm_percpu.h> struct cpu_info { struct device *ci_dev; /* Device corresponding to this CPU */ @@ -161,6 +162,8 @@ struct cpu_info { #ifdef MULTIPROCESSOR struct srp_hazard ci_srp_hazards[SRP_HAZARD_NUM]; +#define __HAVE_UVM_PERCPU + struct uvm_pmr_cache ci_uvm; volatile int ci_flags; volatile int ci_ddb_paused; diff --git a/sys/arch/i386/include/cpu.h b/sys/arch/i386/include/cpu.h index ef7db2af795..9ad7163dccc 100644 --- a/sys/arch/i386/include/cpu.h +++ b/sys/arch/i386/include/cpu.h @@ -1,4 +1,4 @@ -/* $OpenBSD: cpu.h,v 1.185 2024/02/25 19:15:50 cheloha Exp $ */ +/* $OpenBSD: cpu.h,v 1.186 2024/04/17 13:12:58 mpi Exp $ */ /* $NetBSD: cpu.h,v 1.35 1996/05/05 19:29:26 christos Exp $ */ /*- @@ -69,6 +69,7 @@ #include <sys/sched.h> #include <sys/sensors.h> #include <sys/srp.h> +#include <uvm/uvm_percpu.h> struct intrsource; @@ -99,6 +100,8 @@ struct cpu_info { #if defined(MULTIPROCESSOR) struct srp_hazard ci_srp_hazards[SRP_HAZARD_NUM]; +#define __HAVE_UVM_PERCPU + struct uvm_pmr_cache ci_uvm; #endif /* diff --git a/sys/uvm/uvm_page.c b/sys/uvm/uvm_page.c index b9e69522d46..5958c77a0ae 100644 --- a/sys/uvm/uvm_page.c +++ b/sys/uvm/uvm_page.c @@ -1,4 +1,4 @@ -/* $OpenBSD: uvm_page.c,v 1.174 2024/02/13 10:16:28 miod Exp $ */ +/* $OpenBSD: uvm_page.c,v 1.175 2024/04/17 13:12:58 mpi Exp $ */ /* $NetBSD: uvm_page.c,v 1.44 2000/11/27 08:40:04 chs Exp $ */ /* @@ -877,13 +877,11 @@ uvm_pagerealloc_multi(struct uvm_object *obj, voff_t off, vsize_t size, * => only one of obj or anon can be non-null * => caller must activate/deactivate page if it is not wired. */ - struct vm_page * uvm_pagealloc(struct uvm_object *obj, voff_t off, struct vm_anon *anon, int flags) { - struct vm_page *pg; - struct pglist pgl; + struct vm_page *pg = NULL; int pmr_flags; KASSERT(obj == NULL || anon == NULL); @@ -906,13 +904,10 @@ uvm_pagealloc(struct uvm_object *obj, voff_t off, struct vm_anon *anon, if (flags & UVM_PGA_ZERO) pmr_flags |= UVM_PLA_ZERO; - TAILQ_INIT(&pgl); - if (uvm_pmr_getpages(1, 0, 0, 1, 0, 1, pmr_flags, &pgl) != 0) - goto fail; - - pg = TAILQ_FIRST(&pgl); - KASSERT(pg != NULL && TAILQ_NEXT(pg, pageq) == NULL); + pg = uvm_pmr_cache_get(pmr_flags); + if (pg == NULL) + return NULL; uvm_pagealloc_pg(pg, obj, off, anon); KASSERT((pg->pg_flags & PG_DEV) == 0); if (flags & UVM_PGA_ZERO) @@ -921,9 +916,6 @@ uvm_pagealloc(struct uvm_object *obj, voff_t off, struct vm_anon *anon, atomic_setbits_int(&pg->pg_flags, PG_CLEAN); return pg; - -fail: - return NULL; } /* @@ -1025,7 +1017,7 @@ void uvm_pagefree(struct vm_page *pg) { uvm_pageclean(pg); - uvm_pmr_freepages(pg, 1); + uvm_pmr_cache_put(pg); } /* diff --git a/sys/uvm/uvm_pdaemon.c b/sys/uvm/uvm_pdaemon.c index 58a47c37fb0..a8c4c32a3cd 100644 --- a/sys/uvm/uvm_pdaemon.c +++ b/sys/uvm/uvm_pdaemon.c @@ -1,4 +1,4 @@ -/* $OpenBSD: uvm_pdaemon.c,v 1.111 2024/04/10 15:26:18 mpi Exp $ */ +/* $OpenBSD: uvm_pdaemon.c,v 1.112 2024/04/17 13:12:58 mpi Exp $ */ /* $NetBSD: uvm_pdaemon.c,v 1.23 2000/08/20 10:24:14 bjh21 Exp $ */ /* @@ -262,6 +262,8 @@ uvm_pageout(void *arg) #if NDRM > 0 drmbackoff(size * 2); #endif + uvm_pmr_cache_drain(); + /* * scan if needed */ diff --git a/sys/uvm/uvm_percpu.h b/sys/uvm/uvm_percpu.h new file mode 100644 index 00000000000..bbf6897e40b --- /dev/null +++ b/sys/uvm/uvm_percpu.h @@ -0,0 +1,48 @@ +/* $OpenBSD: uvm_percpu.h,v 1.1 2024/04/17 13:12:58 mpi Exp $ */ + +/* + * Copyright (c) 2024 Martin Pieuchot <mpi@openbsd.org> + * + * Permission to use, copy, modify, and distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ + +#ifndef _UVM_UVM_PCPU_H_ +#define _UVM_UVM_PCPU_H_ + +struct vm_page; + +/* + * The number of pages per magazine should be large enough to get rid of the + * contention in the pmemrange allocator during concurrent page faults and + * small enough to limit fragmentation. + */ +#define UVM_PMR_CACHEMAGSZ 8 + +/* + * Magazine + */ +struct uvm_pmr_cache_item { + struct vm_page *upci_pages[UVM_PMR_CACHEMAGSZ]; + int upci_npages; /* # of pages in magazine */ +}; + +/* + * Per-CPU cache of physical pages. + */ +struct uvm_pmr_cache { + struct uvm_pmr_cache_item upc_magz[2]; /* magazines */ + int upc_actv; /* index of active magazine */ + +}; + +#endif /* _UVM_UVM_PCPU_H_ */ diff --git a/sys/uvm/uvm_pmemrange.c b/sys/uvm/uvm_pmemrange.c index 128772d90ed..8cd16969217 100644 --- a/sys/uvm/uvm_pmemrange.c +++ b/sys/uvm/uvm_pmemrange.c @@ -1,6 +1,7 @@ -/* $OpenBSD: uvm_pmemrange.c,v 1.63 2023/04/10 04:21:20 jsg Exp $ */ +/* $OpenBSD: uvm_pmemrange.c,v 1.64 2024/04/17 13:12:58 mpi Exp $ */ /* + * Copyright (c) 2024 Martin Pieuchot <mpi@openbsd.org> * Copyright (c) 2009, 2010 Ariane van der Steldt <ariane@stack.nl> * * Permission to use, copy, modify, and distribute this software for any @@ -1262,6 +1263,28 @@ out: } /* + * Acquire a single page. + * + * flags: UVM_PLA_* flags + * result: returned page. + */ +struct vm_page * +uvm_pmr_getone(int flags) +{ + struct vm_page *pg; + struct pglist pgl; + + TAILQ_INIT(&pgl); + if (uvm_pmr_getpages(1, 0, 0, 1, 0, 1, flags, &pgl) != 0) + return NULL; + + pg = TAILQ_FIRST(&pgl); + KASSERT(pg != NULL && TAILQ_NEXT(pg, pageq) == NULL); + + return pg; +} + +/* * Free a number of contig pages (invoked by uvm_page_init). */ void @@ -2190,3 +2213,133 @@ uvm_pagezero_thread(void *arg) yield(); } } + +#if defined(MULTIPROCESSOR) && defined(__HAVE_UVM_PERCPU) +int +uvm_pmr_cache_alloc(struct uvm_pmr_cache_item *upci) +{ + struct vm_page *pg; + struct pglist pgl; + int flags = UVM_PLA_NOWAIT|UVM_PLA_NOWAKE; + int npages = UVM_PMR_CACHEMAGSZ; + + KASSERT(upci->upci_npages == 0); + + TAILQ_INIT(&pgl); + if (uvm_pmr_getpages(npages, 0, 0, 1, 0, npages, flags, &pgl)) + return -1; + + while ((pg = TAILQ_FIRST(&pgl)) != NULL) { + TAILQ_REMOVE(&pgl, pg, pageq); + upci->upci_pages[upci->upci_npages] = pg; + upci->upci_npages++; + } + atomic_add_int(&uvmexp.percpucaches, npages); + + return 0; +} + +struct vm_page * +uvm_pmr_cache_get(int flags) +{ + struct uvm_pmr_cache *upc = &curcpu()->ci_uvm; + struct uvm_pmr_cache_item *upci; + struct vm_page *pg; + + upci = &upc->upc_magz[upc->upc_actv]; + if (upci->upci_npages == 0) { + unsigned int prev; + + prev = (upc->upc_actv == 0) ? 1 : 0; + upci = &upc->upc_magz[prev]; + if (upci->upci_npages == 0) { + atomic_inc_int(&uvmexp.pcpmiss); + if (uvm_pmr_cache_alloc(upci)) + return uvm_pmr_getone(flags); + } + /* Swap magazines */ + upc->upc_actv = prev; + } else { + atomic_inc_int(&uvmexp.pcphit); + } + + atomic_dec_int(&uvmexp.percpucaches); + upci->upci_npages--; + pg = upci->upci_pages[upci->upci_npages]; + + if (flags & UVM_PLA_ZERO) + uvm_pagezero(pg); + + return pg; +} + +void +uvm_pmr_cache_free(struct uvm_pmr_cache_item *upci) +{ + struct pglist pgl; + unsigned int i; + + TAILQ_INIT(&pgl); + for (i = 0; i < upci->upci_npages; i++) + TAILQ_INSERT_TAIL(&pgl, upci->upci_pages[i], pageq); + + uvm_pmr_freepageq(&pgl); + + atomic_sub_int(&uvmexp.percpucaches, upci->upci_npages); + upci->upci_npages = 0; + memset(upci->upci_pages, 0, sizeof(upci->upci_pages)); +} + +void +uvm_pmr_cache_put(struct vm_page *pg) +{ + struct uvm_pmr_cache *upc = &curcpu()->ci_uvm; + struct uvm_pmr_cache_item *upci; + + upci = &upc->upc_magz[upc->upc_actv]; + if (upci->upci_npages >= UVM_PMR_CACHEMAGSZ) { + unsigned int prev; + + prev = (upc->upc_actv == 0) ? 1 : 0; + upci = &upc->upc_magz[prev]; + if (upci->upci_npages > 0) + uvm_pmr_cache_free(upci); + + /* Swap magazines */ + upc->upc_actv = prev; + KASSERT(upci->upci_npages == 0); + } + + upci->upci_pages[upci->upci_npages] = pg; + upci->upci_npages++; + atomic_inc_int(&uvmexp.percpucaches); +} + +void +uvm_pmr_cache_drain(void) +{ + struct uvm_pmr_cache *upc = &curcpu()->ci_uvm; + + uvm_pmr_cache_free(&upc->upc_magz[0]); + uvm_pmr_cache_free(&upc->upc_magz[1]); +} + +#else /* !(MULTIPROCESSOR && __HAVE_UVM_PERCPU) */ + +struct vm_page * +uvm_pmr_cache_get(int flags) +{ + return uvm_pmr_getone(flags); +} + +void +uvm_pmr_cache_put(struct vm_page *pg) +{ + uvm_pmr_freepages(pg, 1); +} + +void +uvm_pmr_cache_drain(void) +{ +} +#endif diff --git a/sys/uvm/uvm_pmemrange.h b/sys/uvm/uvm_pmemrange.h index 923807c2196..3369085a2df 100644 --- a/sys/uvm/uvm_pmemrange.h +++ b/sys/uvm/uvm_pmemrange.h @@ -1,4 +1,4 @@ -/* $OpenBSD: uvm_pmemrange.h,v 1.14 2016/09/16 02:47:09 dlg Exp $ */ +/* $OpenBSD: uvm_pmemrange.h,v 1.15 2024/04/17 13:12:58 mpi Exp $ */ /* * Copyright (c) 2009 Ariane van der Steldt <ariane@stack.nl> @@ -147,5 +147,9 @@ void uvm_pmr_remove(struct uvm_pmemrange *, struct vm_page *uvm_pmr_extract_range(struct uvm_pmemrange *, struct vm_page *, paddr_t, paddr_t, struct pglist *); +struct vm_page *uvm_pmr_cache_get(int); +void uvm_pmr_cache_put(struct vm_page *); +void uvm_pmr_cache_drain(void); + #endif /* _UVM_UVM_PMEMRANGE_H_ */ diff --git a/sys/uvm/uvmexp.h b/sys/uvm/uvmexp.h index 5b8b182396a..dc4994fa7d2 100644 --- a/sys/uvm/uvmexp.h +++ b/sys/uvm/uvmexp.h @@ -1,4 +1,4 @@ -/* $OpenBSD: uvmexp.h,v 1.12 2024/03/24 10:29:35 mpi Exp $ */ +/* $OpenBSD: uvmexp.h,v 1.13 2024/04/17 13:12:58 mpi Exp $ */ #ifndef _UVM_UVMEXP_ #define _UVM_UVMEXP_ @@ -66,7 +66,7 @@ struct uvmexp { int zeropages; /* [F] number of zero'd pages */ int reserve_pagedaemon; /* [I] # of pages reserved for pagedaemon */ int reserve_kernel; /* [I] # of pages reserved for kernel */ - int unused01; /* formerly anonpages */ + int percpucaches; /* [a] # of pages in per-CPU caches */ int vnodepages; /* XXX # of pages used by vnode page cache */ int vtextpages; /* XXX # of pages used by vtext vnodes */ @@ -101,8 +101,8 @@ struct uvmexp { int syscalls; /* system calls */ int pageins; /* [p] pagein operation count */ /* pageouts are in pdpageouts below */ - int unused07; /* formerly obsolete_swapins */ - int unused08; /* formerly obsolete_swapouts */ + int pcphit; /* [a] # of pagealloc from per-CPU cache */ + int pcpmiss; /* [a] # of times a per-CPU cache was empty */ int pgswapin; /* pages swapped in */ int pgswapout; /* pages swapped out */ int forks; /* forks */ |