summaryrefslogtreecommitdiff
path: root/sys
diff options
context:
space:
mode:
Diffstat (limited to 'sys')
-rw-r--r--sys/arch/amd64/amd64/pmap.c4
-rw-r--r--sys/arch/arm/arm/pmap.c4
-rw-r--r--sys/kern/kern_fork.c4
-rw-r--r--sys/kern/kern_sched.c379
-rw-r--r--sys/kern/kern_synch.c3
-rw-r--r--sys/kern/sched_bsd.c10
-rw-r--r--sys/sys/proc.h24
-rw-r--r--sys/sys/sched.h21
-rw-r--r--sys/uvm/uvm_meter.c23
-rw-r--r--sys/uvm/uvm_page.c4
10 files changed, 416 insertions, 60 deletions
diff --git a/sys/arch/amd64/amd64/pmap.c b/sys/arch/amd64/amd64/pmap.c
index 6acd8a1fc3c..6d92e73acb0 100644
--- a/sys/arch/amd64/amd64/pmap.c
+++ b/sys/arch/amd64/amd64/pmap.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: pmap.c,v 1.37 2009/02/16 20:26:58 kurt Exp $ */
+/* $OpenBSD: pmap.c,v 1.38 2009/03/23 13:25:11 art Exp $ */
/* $NetBSD: pmap.c,v 1.3 2003/05/08 18:13:13 thorpej Exp $ */
/*
@@ -1322,7 +1322,7 @@ pmap_pageidlezero(struct vm_page *pg)
* with uncached mappings.
*/
for (i = 0, ptr = (long *) va; i < PAGE_SIZE / sizeof(long); i++) {
- if (!sched_is_idle()) {
+ if (!curcpu_is_idle()) {
/*
* A process has become ready. Abort now,
diff --git a/sys/arch/arm/arm/pmap.c b/sys/arch/arm/arm/pmap.c
index 3a8ecd90059..b47279ea4ec 100644
--- a/sys/arch/arm/arm/pmap.c
+++ b/sys/arch/arm/arm/pmap.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: pmap.c,v 1.19 2008/10/28 20:16:58 drahn Exp $ */
+/* $OpenBSD: pmap.c,v 1.20 2009/03/23 13:25:11 art Exp $ */
/* $NetBSD: pmap.c,v 1.147 2004/01/18 13:03:50 scw Exp $ */
/*
@@ -3327,7 +3327,7 @@ pmap_pageidlezero(struct vm_page *pg)
for (i = 0, ptr = (int *)cdstp;
i < (PAGE_SIZE / sizeof(int)); i++) {
- if (!sched_is_idle()) {
+ if (!curcpu_is_idle()) {
/*
* A process has become ready. Abort now,
* so we don't keep it waiting while we
diff --git a/sys/kern/kern_fork.c b/sys/kern/kern_fork.c
index 76d868fe0a4..10dd9a5f8eb 100644
--- a/sys/kern/kern_fork.c
+++ b/sys/kern/kern_fork.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: kern_fork.c,v 1.101 2008/11/11 02:13:14 tedu Exp $ */
+/* $OpenBSD: kern_fork.c,v 1.102 2009/03/23 13:25:11 art Exp $ */
/* $NetBSD: kern_fork.c,v 1.29 1996/02/09 18:59:34 christos Exp $ */
/*
@@ -260,8 +260,6 @@ fork1(struct proc *p1, int exitsig, int flags, void *stack, size_t stacksize,
timeout_set(&p2->p_sleep_to, endtsleep, p2);
timeout_set(&p2->p_realit_to, realitexpire, p2);
- p2->p_cpu = p1->p_cpu;
-
/*
* Duplicate sub-structures as needed.
* Increase reference counts on shared objects.
diff --git a/sys/kern/kern_sched.c b/sys/kern/kern_sched.c
index 79bc376314a..1b921159985 100644
--- a/sys/kern/kern_sched.c
+++ b/sys/kern/kern_sched.c
@@ -1,6 +1,6 @@
-/* $OpenBSD: kern_sched.c,v 1.8 2008/11/06 19:49:13 deraadt Exp $ */
+/* $OpenBSD: kern_sched.c,v 1.9 2009/03/23 13:25:11 art Exp $ */
/*
- * Copyright (c) 2007 Artur Grabowski <art@openbsd.org>
+ * Copyright (c) 2007, 2008 Artur Grabowski <art@openbsd.org>
*
* Permission to use, copy, modify, and distribute this software for any
* purpose with or without fee is hereby granted, provided that the above
@@ -24,12 +24,27 @@
#include <sys/resourcevar.h>
#include <sys/signalvar.h>
#include <sys/mutex.h>
+#include <machine/atomic.h>
#include <uvm/uvm_extern.h>
+#include <sys/malloc.h>
+
+
void sched_kthreads_create(void *);
void sched_idle(void *);
+int sched_proc_to_cpu_cost(struct cpu_info *ci, struct proc *p);
+struct proc *sched_steal_proc(struct cpu_info *);
+
+/*
+ * To help choosing which cpu should run which process we keep track
+ * of cpus which are currently idle and which cpus have processes
+ * queued.
+ */
+struct cpuset sched_idle_cpus;
+struct cpuset sched_queued_cpus;
+
/*
* A few notes about cpu_switchto that is implemented in MD code.
*
@@ -55,12 +70,22 @@ void
sched_init_cpu(struct cpu_info *ci)
{
struct schedstate_percpu *spc = &ci->ci_schedstate;
+ int i;
+
+ for (i = 0; i < SCHED_NQS; i++)
+ TAILQ_INIT(&spc->spc_qs[i]);
spc->spc_idleproc = NULL;
kthread_create_deferred(sched_kthreads_create, ci);
LIST_INIT(&spc->spc_deadproc);
+
+ /*
+ * Slight hack here until the cpuset code handles cpu_info
+ * structures.
+ */
+ cpuset_init_cpu(ci);
}
void
@@ -79,27 +104,31 @@ sched_kthreads_create(void *v)
void
sched_idle(void *v)
{
+ struct schedstate_percpu *spc;
struct proc *p = curproc;
struct cpu_info *ci = v;
int s;
KERNEL_PROC_UNLOCK(p);
+ spc = &ci->ci_schedstate;
+
/*
* First time we enter here, we're not supposed to idle,
* just go away for a while.
*/
SCHED_LOCK(s);
+ cpuset_add(&sched_idle_cpus, ci);
p->p_stat = SSLEEP;
mi_switch();
+ cpuset_del(&sched_idle_cpus, ci);
SCHED_UNLOCK(s);
- while (1) {
- KASSERT(ci == curcpu());
- KASSERT(curproc == ci->ci_schedstate.spc_idleproc);
+ KASSERT(ci == curcpu());
+ KASSERT(curproc == spc->spc_idleproc);
- while (!sched_is_idle()) {
- struct schedstate_percpu *spc = &ci->ci_schedstate;
+ while (1) {
+ while (!curcpu_is_idle()) {
struct proc *dead;
SCHED_LOCK(s);
@@ -115,10 +144,12 @@ sched_idle(void *v)
splassert(IPL_NONE);
+ cpuset_add(&sched_idle_cpus, ci);
cpu_idle_enter();
- while (sched_is_idle())
+ while (spc->spc_whichqs == 0)
cpu_idle_cycle();
cpu_idle_leave();
+ cpuset_del(&sched_idle_cpus, ci);
}
}
@@ -160,22 +191,10 @@ sched_exit(struct proc *p)
/*
* Run queue management.
- *
- * The run queue management is just like before, except that it's with
- * a bit more modern queue handling.
*/
-
-TAILQ_HEAD(prochead, proc) sched_qs[NQS];
-volatile int sched_whichqs;
-
void
sched_init_runqueues(void)
{
- int i;
-
- for (i = 0; i < NQS; i++)
- TAILQ_INIT(&sched_qs[i]);
-
#ifdef MULTIPROCESSOR
__mp_lock_init(&sched_lock);
#endif
@@ -184,37 +203,56 @@ sched_init_runqueues(void)
void
setrunqueue(struct proc *p)
{
+ struct schedstate_percpu *spc;
int queue = p->p_priority >> 2;
SCHED_ASSERT_LOCKED();
+ sched_choosecpu(p);
+ spc = &p->p_cpu->ci_schedstate;
+ spc->spc_nrun++;
+
+ TAILQ_INSERT_TAIL(&spc->spc_qs[queue], p, p_runq);
+ spc->spc_whichqs |= (1 << queue);
+ cpuset_add(&sched_queued_cpus, p->p_cpu);
- TAILQ_INSERT_TAIL(&sched_qs[queue], p, p_runq);
- sched_whichqs |= (1 << queue);
+ if (p->p_cpu != curcpu())
+ cpu_unidle(p->p_cpu);
}
void
remrunqueue(struct proc *p)
{
+ struct schedstate_percpu *spc;
int queue = p->p_priority >> 2;
SCHED_ASSERT_LOCKED();
-
- TAILQ_REMOVE(&sched_qs[queue], p, p_runq);
- if (TAILQ_EMPTY(&sched_qs[queue]))
- sched_whichqs &= ~(1 << queue);
+ spc = &p->p_cpu->ci_schedstate;
+ spc->spc_nrun--;
+
+ TAILQ_REMOVE(&spc->spc_qs[queue], p, p_runq);
+ if (TAILQ_EMPTY(&spc->spc_qs[queue])) {
+ spc->spc_whichqs &= ~(1 << queue);
+ if (spc->spc_whichqs == 0)
+ cpuset_del(&sched_queued_cpus, p->p_cpu);
+ }
}
struct proc *
sched_chooseproc(void)
{
+ struct schedstate_percpu *spc = &curcpu()->ci_schedstate;
struct proc *p;
int queue;
SCHED_ASSERT_LOCKED();
again:
- if (sched_is_idle()) {
- p = curcpu()->ci_schedstate.spc_idleproc;
+ if (spc->spc_whichqs) {
+ queue = ffs(spc->spc_whichqs) - 1;
+ p = TAILQ_FIRST(&spc->spc_qs[queue]);
+ remrunqueue(p);
+ } else if ((p = sched_steal_proc(curcpu())) == NULL) {
+ p = spc->spc_idleproc;
if (p == NULL) {
int s;
/*
@@ -232,13 +270,284 @@ again:
}
KASSERT(p);
p->p_stat = SRUN;
- } else {
- queue = ffs(sched_whichqs) - 1;
- p = TAILQ_FIRST(&sched_qs[queue]);
- TAILQ_REMOVE(&sched_qs[queue], p, p_runq);
- if (TAILQ_EMPTY(&sched_qs[queue]))
- sched_whichqs &= ~(1 << queue);
- }
+ }
return (p);
}
+
+uint64_t sched_nmigrations;
+uint64_t sched_noidle;
+uint64_t sched_stolen;
+
+uint64_t sched_choose;
+uint64_t sched_wasidle;
+uint64_t sched_nomigrations;
+
+void
+sched_choosecpu(struct proc *p)
+{
+ struct cpu_info *choice = NULL;
+ int last_cost = INT_MAX;
+ struct cpu_info *ci;
+ struct cpuset set;
+
+ sched_choose++;
+
+ /*
+ * The simplest case. Our cpu of choice was idle. This happens
+ * when we were sleeping and something woke us up.
+ *
+ * We also need to check sched_queued_cpus to make sure that
+ * we're not thundering herding one cpu that hasn't managed to
+ * get out of the idle loop yet.
+ */
+ if (p->p_cpu && cpuset_isset(&sched_idle_cpus, p->p_cpu) &&
+ !cpuset_isset(&sched_queued_cpus, p->p_cpu)) {
+ sched_wasidle++;
+ return;
+ }
+
+#if 0
+
+ /* Most likely, this is broken. don't do it. */
+ /*
+ * Second case. (shouldn't be necessary in the future)
+ * If our cpu is not idle, but has nothing else queued (which
+ * means that we are curproc and roundrobin asks us to reschedule).
+ */
+ if (p->p_cpu && p->p_cpu->ci_schedstate.spc_nrun == 0)
+ return;
+#endif
+
+ /*
+ * Look at all cpus that are currently idle. Pick the cheapest of
+ * those.
+ */
+ cpuset_copy(&set, &sched_idle_cpus);
+ while ((ci = cpuset_first(&set)) != NULL) {
+ int cost = sched_proc_to_cpu_cost(ci, p);
+
+ if (choice == NULL || cost < last_cost) {
+ choice = ci;
+ last_cost = cost;
+ }
+ cpuset_del(&set, ci);
+ }
+
+ /*
+ * All cpus are busy. Pick one.
+ */
+ if (choice == NULL) {
+ CPU_INFO_ITERATOR cii;
+
+ sched_noidle++;
+
+ /*
+ * Not curproc, pick the cpu with the lowest cost to switch to.
+ */
+ CPU_INFO_FOREACH(cii, ci) {
+ int cost = sched_proc_to_cpu_cost(ci, p);
+
+ if (choice == NULL || cost < last_cost) {
+ choice = ci;
+ last_cost = cost;
+ }
+ }
+ }
+
+ KASSERT(choice);
+
+ if (p->p_cpu && p->p_cpu != choice)
+ sched_nmigrations++;
+ else if (p->p_cpu != NULL)
+ sched_nomigrations++;
+
+ p->p_cpu = choice;
+}
+
+/*
+ * Attempt to steal a proc from some cpu.
+ */
+struct proc *
+sched_steal_proc(struct cpu_info *self)
+{
+ struct schedstate_percpu *spc;
+ struct proc *best = NULL;
+ int bestcost = INT_MAX;
+ struct cpu_info *ci;
+ struct cpuset set;
+
+ cpuset_copy(&set, &sched_queued_cpus);
+
+ while ((ci = cpuset_first(&set)) != NULL) {
+ struct proc *p;
+ int cost;
+
+ cpuset_del(&set, ci);
+
+ spc = &ci->ci_schedstate;
+
+ p = TAILQ_FIRST(&spc->spc_qs[ffs(spc->spc_whichqs) - 1]);
+ KASSERT(p);
+ cost = sched_proc_to_cpu_cost(self, p);
+
+ if (best == NULL || cost < bestcost) {
+ best = p;
+ bestcost = cost;
+ }
+ }
+ if (best == NULL)
+ return (NULL);
+
+ spc = &best->p_cpu->ci_schedstate;
+ remrunqueue(best);
+ best->p_cpu = self;
+
+ sched_stolen++;
+
+ return (best);
+}
+
+/*
+ * Base 2 logarithm of an int. returns 0 for 0 (yeye, I know).
+ */
+static int
+log2(unsigned int i)
+{
+ int ret = 0;
+
+ while (i >>= 1)
+ ret++;
+
+ return (ret);
+}
+
+/*
+ * Calculate the cost of moving the proc to this cpu.
+ *
+ * What we want is some guesstimate of how much "performance" it will
+ * cost us to move the proc here. Not just for caches and TLBs and NUMA
+ * memory, but also for the proc itself. A highly loaded cpu might not
+ * be the best candidate for this proc since it won't get run.
+ *
+ * Just total guesstimates for now.
+ */
+
+int sched_cost_load = 1;
+int sched_cost_priority = 1;
+int sched_cost_runnable = 3;
+int sched_cost_resident = 1;
+
+int
+sched_proc_to_cpu_cost(struct cpu_info *ci, struct proc *p)
+{
+ struct schedstate_percpu *spc;
+ int l2resident = 0;
+ int cost;
+
+ spc = &ci->ci_schedstate;
+
+ cost = 0;
+
+ /*
+ * First, account for the priority of the proc we want to move.
+ * More willing to move, the lower the priority of the destination
+ * and the higher the priority of the proc.
+ */
+ if (!cpuset_isset(&sched_idle_cpus, ci)) {
+ cost += (p->p_priority - spc->spc_curpriority) *
+ sched_cost_priority;
+ cost += sched_cost_runnable;
+ }
+ if (cpuset_isset(&sched_queued_cpus, ci)) {
+ cost += spc->spc_nrun * sched_cost_runnable;
+ }
+
+ /*
+ * Higher load on the destination means we don't want to go there.
+ */
+ cost += ((sched_cost_load * spc->spc_ldavg) >> FSHIFT);
+
+ /*
+ * If the proc is on this cpu already, lower the cost by how much
+ * it has been running and an estimate of its footprint.
+ */
+ if (p->p_cpu == ci && p->p_slptime == 0) {
+ l2resident =
+ log2(pmap_resident_count(p->p_vmspace->vm_map.pmap));
+ cost -= l2resident * sched_cost_resident;
+ }
+
+ return (cost);
+}
+
+/*
+ * Functions to manipulate cpu sets.
+ */
+struct cpu_info *cpuset_infos[MAXCPUS];
+static struct cpuset cpuset_all;
+
+void
+cpuset_init_cpu(struct cpu_info *ci)
+{
+ cpuset_add(&cpuset_all, ci);
+ cpuset_infos[CPU_INFO_UNIT(ci)] = ci;
+}
+
+void
+cpuset_clear(struct cpuset *cs)
+{
+ memset(cs, 0, sizeof(*cs));
+}
+
+/*
+ * XXX - implement it on SP architectures too
+ */
+#ifndef CPU_INFO_UNIT
+#define CPU_INFO_UNIT 0
+#endif
+
+void
+cpuset_add(struct cpuset *cs, struct cpu_info *ci)
+{
+ unsigned int num = CPU_INFO_UNIT(ci);
+ atomic_setbits_int(&cs->cs_set[num/32], (1 << (num % 32)));
+}
+
+void
+cpuset_del(struct cpuset *cs, struct cpu_info *ci)
+{
+ unsigned int num = CPU_INFO_UNIT(ci);
+ atomic_clearbits_int(&cs->cs_set[num/32], (1 << (num % 32)));
+}
+
+int
+cpuset_isset(struct cpuset *cs, struct cpu_info *ci)
+{
+ unsigned int num = CPU_INFO_UNIT(ci);
+ return (cs->cs_set[num/32] & (1 << (num % 32)));
+}
+
+void
+cpuset_add_all(struct cpuset *cs)
+{
+ cpuset_copy(cs, &cpuset_all);
+}
+
+void
+cpuset_copy(struct cpuset *to, struct cpuset *from)
+{
+ memcpy(to, from, sizeof(*to));
+}
+
+struct cpu_info *
+cpuset_first(struct cpuset *cs)
+{
+ int i;
+
+ for (i = 0; i < CPUSET_ASIZE(ncpus); i++)
+ if (cs->cs_set[i])
+ return (cpuset_infos[i * 32 + ffs(cs->cs_set[i]) - 1]);
+
+ return (NULL);
+}
diff --git a/sys/kern/kern_synch.c b/sys/kern/kern_synch.c
index 650b35e722f..449f93c7b5a 100644
--- a/sys/kern/kern_synch.c
+++ b/sys/kern/kern_synch.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: kern_synch.c,v 1.87 2008/09/10 12:30:40 blambert Exp $ */
+/* $OpenBSD: kern_synch.c,v 1.88 2009/03/23 13:25:11 art Exp $ */
/* $NetBSD: kern_synch.c,v 1.37 1996/04/22 01:38:37 christos Exp $ */
/*
@@ -381,7 +381,6 @@ wakeup_n(void *ident, int n)
* resched_proc().
*/
setrunqueue(p);
- KASSERT(p->p_cpu != NULL);
need_resched(p->p_cpu);
/* END INLINE EXPANSION */
diff --git a/sys/kern/sched_bsd.c b/sys/kern/sched_bsd.c
index e5db94d2aaa..53c0902dc49 100644
--- a/sys/kern/sched_bsd.c
+++ b/sys/kern/sched_bsd.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: sched_bsd.c,v 1.19 2008/11/06 22:11:36 art Exp $ */
+/* $OpenBSD: sched_bsd.c,v 1.20 2009/03/23 13:25:11 art Exp $ */
/* $NetBSD: kern_synch.c,v 1.37 1996/04/22 01:38:37 christos Exp $ */
/*-
@@ -97,7 +97,7 @@ roundrobin(struct cpu_info *ci)
spc->spc_rrticks = rrticks_init;
- if (curproc != NULL) {
+ if (ci->ci_curproc != NULL) {
s = splstatclock();
if (spc->spc_schedflags & SPCF_SEENRR) {
/*
@@ -112,7 +112,8 @@ roundrobin(struct cpu_info *ci)
splx(s);
}
- need_resched(curcpu());
+ if (spc->spc_nrun)
+ need_resched(ci);
}
/*
@@ -257,7 +258,8 @@ schedcpu(void *arg)
resetpriority(p);
if (p->p_priority >= PUSER) {
if (p->p_stat == SRUN &&
- (p->p_priority / PPQ) != (p->p_usrpri / PPQ)) {
+ (p->p_priority / SCHED_PPQ) !=
+ (p->p_usrpri / SCHED_PPQ)) {
remrunqueue(p);
p->p_priority = p->p_usrpri;
setrunqueue(p);
diff --git a/sys/sys/proc.h b/sys/sys/proc.h
index aeec114cec4..8ecbabd2947 100644
--- a/sys/sys/proc.h
+++ b/sys/sys/proc.h
@@ -1,4 +1,4 @@
-/* $OpenBSD: proc.h,v 1.112 2009/03/05 19:52:24 kettenis Exp $ */
+/* $OpenBSD: proc.h,v 1.113 2009/03/23 13:25:11 art Exp $ */
/* $NetBSD: proc.h,v 1.44 1996/04/22 01:23:21 christos Exp $ */
/*-
@@ -468,6 +468,28 @@ void proc_trampoline_mp(void); /* XXX */
int proc_isunder(struct proc *, struct proc *);
+/*
+ * functions to handle sets of cpus.
+ *
+ * For now we keep the cpus in ints so that we can use the generic
+ * atomic ops.
+ */
+#define CPUSET_ASIZE(x) (((x) - 1)/32 + 1)
+#define CPUSET_SSIZE CPUSET_ASIZE(MAXCPUS)
+struct cpuset {
+ int cs_set[CPUSET_SSIZE];
+};
+
+void cpuset_init_cpu(struct cpu_info *);
+
+void cpuset_clear(struct cpuset *);
+void cpuset_add(struct cpuset *, struct cpu_info *);
+void cpuset_del(struct cpuset *, struct cpu_info *);
+int cpuset_isset(struct cpuset *, struct cpu_info *);
+void cpuset_add_all(struct cpuset *);
+void cpuset_copy(struct cpuset *to, struct cpuset *from);
+struct cpu_info *cpuset_first(struct cpuset *);
+
#endif /* _KERNEL */
#endif /* !_SYS_PROC_H_ */
diff --git a/sys/sys/sched.h b/sys/sys/sched.h
index 1e2f9bc5847..151adc9af37 100644
--- a/sys/sys/sched.h
+++ b/sys/sys/sched.h
@@ -1,4 +1,4 @@
-/* $OpenBSD: sched.h,v 1.19 2008/06/26 05:42:20 ray Exp $ */
+/* $OpenBSD: sched.h,v 1.20 2009/03/23 13:25:11 art Exp $ */
/* $NetBSD: sched.h,v 1.2 1999/02/28 18:14:58 ross Exp $ */
/*-
@@ -87,6 +87,8 @@
#define CP_IDLE 4
#define CPUSTATES 5
+#define SCHED_NQS 32 /* 32 run queues. */
+
/*
* Per-CPU scheduler state.
* XXX - expose to userland for now.
@@ -101,6 +103,13 @@ struct schedstate_percpu {
int spc_pscnt; /* prof/stat counter */
int spc_psdiv; /* prof/stat divisor */
struct proc *spc_idleproc; /* idle proc for this cpu */
+
+ u_int spc_nrun; /* procs on the run queues */
+ fixpt_t spc_ldavg; /* shortest load avg. for this cpu */
+
+ TAILQ_HEAD(prochead, proc) spc_qs[SCHED_NQS];
+ volatile uint32_t spc_whichqs;
+
#ifdef notyet
struct proc *spc_reaper; /* dead proc reaper */
#endif
@@ -114,11 +123,9 @@ struct schedstate_percpu {
#define SPCF_SHOULDYIELD 0x0002 /* process should yield the CPU */
#define SPCF_SWITCHCLEAR (SPCF_SEENRR|SPCF_SHOULDYIELD)
-
-#define NQS 32 /* 32 run queues. */
-#define PPQ (128 / NQS) /* priorities per queue */
+#define SCHED_PPQ (128 / SCHED_NQS) /* priorities per queue */
#define NICE_WEIGHT 2 /* priorities per nice level */
-#define ESTCPULIM(e) min((e), NICE_WEIGHT * PRIO_MAX - PPQ)
+#define ESTCPULIM(e) min((e), NICE_WEIGHT * PRIO_MAX - SCHED_PPQ)
extern int schedhz; /* ideally: 16 */
extern int rrticks_init; /* ticks per roundrobin() */
@@ -133,12 +140,12 @@ void sched_exit(struct proc *);
void mi_switch(void);
void cpu_switchto(struct proc *, struct proc *);
struct proc *sched_chooseproc(void);
+void sched_choosecpu(struct proc *);
void cpu_idle_enter(void);
void cpu_idle_cycle(void);
void cpu_idle_leave(void);
-extern volatile int sched_whichqs;
-#define sched_is_idle() (sched_whichqs == 0)
+#define curcpu_is_idle() (curcpu()->ci_schedstate.spc_whichqs == 0)
void sched_init_runqueues(void);
void setrunqueue(struct proc *);
diff --git a/sys/uvm/uvm_meter.c b/sys/uvm/uvm_meter.c
index 3c10fff7847..7a77b187e60 100644
--- a/sys/uvm/uvm_meter.c
+++ b/sys/uvm/uvm_meter.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: uvm_meter.c,v 1.25 2009/03/20 15:19:04 oga Exp $ */
+/* $OpenBSD: uvm_meter.c,v 1.26 2009/03/23 13:25:11 art Exp $ */
/* $NetBSD: uvm_meter.c,v 1.21 2001/07/14 06:36:03 matt Exp $ */
/*
@@ -97,10 +97,15 @@ uvm_meter(void)
static void
uvm_loadav(struct loadavg *avg)
{
+ CPU_INFO_ITERATOR cii;
+ struct cpu_info *ci;
int i, nrun;
struct proc *p;
+ int nrun_cpu[MAXCPUS];
nrun = 0;
+ memset(nrun_cpu, 0, sizeof(nrun_cpu));
+
LIST_FOREACH(p, &allproc, p_list) {
switch (p->p_stat) {
case SSLEEP:
@@ -113,11 +118,25 @@ uvm_loadav(struct loadavg *avg)
continue;
case SIDL:
nrun++;
+ if (p->p_cpu)
+ nrun_cpu[CPU_INFO_UNIT(p->p_cpu)]++;
}
}
- for (i = 0; i < 3; i++)
+
+ for (i = 0; i < 3; i++) {
avg->ldavg[i] = (cexp[i] * avg->ldavg[i] +
nrun * FSCALE * (FSCALE - cexp[i])) >> FSHIFT;
+ }
+
+ CPU_INFO_FOREACH(cii, ci) {
+ struct schedstate_percpu *spc = &ci->ci_schedstate;
+
+ if (nrun_cpu[CPU_INFO_UNIT(ci)] == 0)
+ continue;
+ spc->spc_ldavg = (cexp[0] * spc->spc_ldavg +
+ nrun_cpu[CPU_INFO_UNIT(ci)] * FSCALE *
+ (FSCALE - cexp[0])) >> FSHIFT;
+ }
}
/*
diff --git a/sys/uvm/uvm_page.c b/sys/uvm/uvm_page.c
index 8a90c987693..3c16e1a5893 100644
--- a/sys/uvm/uvm_page.c
+++ b/sys/uvm/uvm_page.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: uvm_page.c,v 1.67 2008/07/02 15:21:33 art Exp $ */
+/* $OpenBSD: uvm_page.c,v 1.68 2009/03/23 13:25:11 art Exp $ */
/* $NetBSD: uvm_page.c,v 1.44 2000/11/27 08:40:04 chs Exp $ */
/*
@@ -1400,5 +1400,5 @@ uvm_pageidlezero()
uvmexp.free++;
uvmexp.zeropages++;
uvm_unlock_fpageq();
- } while (sched_is_idle());
+ } while (curcpu_is_idle());
}