Split `p_priority' into `p_runpri' and `p_slppri'.

Using different fields to remember in which runqueue or sleepqueue
threads currently are will make it easier to split the SCHED_LOCK().

With this change, the (potentially boosted) sleeping priority is no
longer overwriting the thread priority.  This let us get rids of the
logic required to synchronize `p_priority' with `p_usrpri'.

Tested by many, ok visa@

10 files changed, 44 insertions, 40 deletions

diff --git a/sys/arch/sparc64/sparc64/db_interface.c b/sys/arch/sparc64/sparc64/db_interface.c
index 60c05777b97..4ec7a1d4115 100644
--- a/sys/arch/sparc64/sparc64/db_interface.c
+++ b/sys/arch/sparc64/sparc64/db_interface.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: db_interface.c,v 1.54 2019/11/07 14:44:53 mpi Exp $	*/
+/*	$OpenBSD: db_interface.c,v 1.55 2020/01/30 08:51:27 mpi Exp $	*/
 /*	$NetBSD: db_interface.c,v 1.61 2001/07/31 06:55:47 eeh Exp $ */
 
 /*
@@ -958,10 +958,10 @@ db_proc_cmd(addr, have_addr, count, modif)
 		return;
 	}
 	db_printf("process %p:", p);
-	db_printf("pid:%d vmspace:%p pmap:%p ctx:%x wchan:%p pri:%d upri:%d\n",
+	db_printf("pid:%d vmspace:%p pmap:%p ctx:%x wchan:%p rpri:%d upri:%d\n",
 	    p->p_p->ps_pid, p->p_vmspace, p->p_vmspace->vm_map.pmap,
 	    p->p_vmspace->vm_map.pmap->pm_ctx,
-	    p->p_wchan, p->p_priority, p->p_usrpri);
+	    p->p_wchan, p->p_runpri, p->p_usrpri);
 	db_printf("maxsaddr:%p ssiz:%dpg or %llxB\n",
 	    p->p_vmspace->vm_maxsaddr, p->p_vmspace->vm_ssize,
 	    (unsigned long long)ptoa(p->p_vmspace->vm_ssize));
diff --git a/sys/dev/pci/drm/i915/intel_breadcrumbs.c b/sys/dev/pci/drm/i915/intel_breadcrumbs.c
index 0ed0ab35d9a..ee379dfe847 100644
--- a/sys/dev/pci/drm/i915/intel_breadcrumbs.c
+++ b/sys/dev/pci/drm/i915/intel_breadcrumbs.c
@@ -451,7 +451,7 @@ static bool __intel_engine_add_wait(struct intel_engine_cs *engine,
 #ifdef __linux__
 			if (wait->tsk->prio > to_wait(parent)->tsk->prio) {
 #else
-			if (wait->tsk->p_priority > to_wait(parent)->tsk->p_priority) {
+			if (wait->tsk->p_usrpri > to_wait(parent)->tsk->p_usrpri) {
 #endif
 				p = &parent->rb_right;
 				first = false;
@@ -538,7 +538,7 @@ static inline bool chain_wakeup(struct rb_node *rb, int priority)
 #else
 static inline bool chain_wakeup(struct rb_node *rb, int priority)
 {
-	return rb && to_wait(rb)->tsk->p_priority <= priority;
+	return rb && to_wait(rb)->tsk->p_usrpri <= priority;
 }
 #endif
 
@@ -558,7 +558,7 @@ static inline int wakeup_priority(struct intel_breadcrumbs *b,
 	if (p == b->signaler)
 		return INT_MIN;
 	else
-		return p->p_priority;
+		return p->p_usrpri;
 }
 #endif
 
diff --git a/sys/kern/kern_fork.c b/sys/kern/kern_fork.c
index 1f74865ed6a..6308e037832 100644
--- a/sys/kern/kern_fork.c
+++ b/sys/kern/kern_fork.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: kern_fork.c,v 1.221 2020/01/21 15:20:47 visa Exp $	*/
+/*	$OpenBSD: kern_fork.c,v 1.222 2020/01/30 08:51:27 mpi Exp $	*/
 /*	$NetBSD: kern_fork.c,v 1.29 1996/02/09 18:59:34 christos Exp $	*/
 
 /*
@@ -148,6 +148,7 @@ thread_new(struct proc *parent, vaddr_t uaddr)
 
 	p = pool_get(&proc_pool, PR_WAITOK);
 	p->p_stat = SIDL;			/* protect against others */
+	p->p_runpri = 0;
 	p->p_flag = 0;
 
 	/*
@@ -314,7 +315,7 @@ fork_thread_start(struct proc *p, struct proc *parent, int flags)
 
 	SCHED_LOCK(s);
 	ci = sched_choosecpu_fork(parent, flags);
-	setrunqueue(ci, p, p->p_priority);
+	setrunqueue(ci, p, p->p_usrpri);
 	SCHED_UNLOCK(s);
 }
 
diff --git a/sys/kern/kern_proc.c b/sys/kern/kern_proc.c
index 934cf455ba1..8ef2a143949 100644
--- a/sys/kern/kern_proc.c
+++ b/sys/kern/kern_proc.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: kern_proc.c,v 1.85 2018/11/12 15:09:17 visa Exp $	*/
+/*	$OpenBSD: kern_proc.c,v 1.86 2020/01/30 08:51:27 mpi Exp $	*/
 /*	$NetBSD: kern_proc.c,v 1.14 1996/02/09 18:59:41 christos Exp $	*/
 
 /*
@@ -476,7 +476,7 @@ proc_printit(struct proc *p, const char *modif,
 	(*pr)("    flags process=%b proc=%b\n",
 	    p->p_p->ps_flags, PS_BITS, p->p_flag, P_BITS);
 	(*pr)("    pri=%u, usrpri=%u, nice=%d\n",
-	    p->p_priority, p->p_usrpri, p->p_p->ps_nice);
+	    p->p_runpri, p->p_usrpri, p->p_p->ps_nice);
 	(*pr)("    forw=%p, list=%p,%p\n",
 	    TAILQ_NEXT(p, p_runq), p->p_list.le_next, p->p_list.le_prev);
 	(*pr)("    process=%p user=%p, vmspace=%p\n",
diff --git a/sys/kern/kern_sched.c b/sys/kern/kern_sched.c
index e25993099ee..ea893c49a47 100644
--- a/sys/kern/kern_sched.c
+++ b/sys/kern/kern_sched.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: kern_sched.c,v 1.63 2020/01/21 16:16:23 mpi Exp $	*/
+/*	$OpenBSD: kern_sched.c,v 1.64 2020/01/30 08:51:27 mpi Exp $	*/
 /*
  * Copyright (c) 2007, 2008 Artur Grabowski <art@openbsd.org>
  *
@@ -258,7 +258,7 @@ setrunqueue(struct cpu_info *ci, struct proc *p, uint8_t prio)
 
 	p->p_cpu = ci;
 	p->p_stat = SRUN;
-	p->p_priority = prio;
+	p->p_runpri = prio;
 
 	spc = &p->p_cpu->ci_schedstate;
 	spc->spc_nrun++;
@@ -279,7 +279,7 @@ void
 remrunqueue(struct proc *p)
 {
 	struct schedstate_percpu *spc;
-	int queue = p->p_priority >> 2;
+	int queue = p->p_runpri >> 2;
 
 	SCHED_ASSERT_LOCKED();
 	spc = &p->p_cpu->ci_schedstate;
@@ -309,7 +309,7 @@ sched_chooseproc(void)
 			for (queue = 0; queue < SCHED_NQS; queue++) {
 				while ((p = TAILQ_FIRST(&spc->spc_qs[queue]))) {
 					remrunqueue(p);
-					setrunqueue(NULL, p, p->p_priority);
+					setrunqueue(NULL, p, p->p_runpri);
 					if (p->p_cpu == curcpu()) {
 						KASSERT(p->p_flag & P_CPUPEG);
 						goto again;
@@ -581,7 +581,7 @@ sched_proc_to_cpu_cost(struct cpu_info *ci, struct proc *p)
 	 * and the higher the priority of the proc.
 	 */
 	if (!cpuset_isset(&sched_idle_cpus, ci)) {
-		cost += (p->p_priority - spc->spc_curpriority) *
+		cost += (p->p_usrpri - spc->spc_curpriority) *
 		    sched_cost_priority;
 		cost += sched_cost_runnable;
 	}
diff --git a/sys/kern/kern_sig.c b/sys/kern/kern_sig.c
index 172848428b7..f91e344ed8f 100644
--- a/sys/kern/kern_sig.c
+++ b/sys/kern/kern_sig.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: kern_sig.c,v 1.242 2020/01/16 16:35:03 mpi Exp $	*/
+/*	$OpenBSD: kern_sig.c,v 1.243 2020/01/30 08:51:27 mpi Exp $	*/
 /*	$NetBSD: kern_sig.c,v 1.54 1996/04/22 01:38:32 christos Exp $	*/
 
 /*
@@ -1130,8 +1130,8 @@ runfast:
 	/*
 	 * Raise priority to at least PUSER.
 	 */
-	if (p->p_priority > PUSER)
-		p->p_priority = PUSER;
+	if (p->p_usrpri > PUSER)
+		p->p_usrpri = PUSER;
 run:
 	setrunnable(p);
 out:
@@ -1886,7 +1886,7 @@ userret(struct proc *p)
 
 	WITNESS_WARN(WARN_PANIC, NULL, "userret: returning");
 
-	p->p_cpu->ci_schedstate.spc_curpriority = p->p_priority = p->p_usrpri;
+	p->p_cpu->ci_schedstate.spc_curpriority = p->p_usrpri;
 }
 
 int
diff --git a/sys/kern/kern_synch.c b/sys/kern/kern_synch.c
index f23fdfffcda..2f1a754aaf3 100644
--- a/sys/kern/kern_synch.c
+++ b/sys/kern/kern_synch.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: kern_synch.c,v 1.161 2020/01/24 15:58:33 cheloha Exp $	*/
+/*	$OpenBSD: kern_synch.c,v 1.162 2020/01/30 08:51:27 mpi Exp $	*/
 /*	$NetBSD: kern_synch.c,v 1.37 1996/04/22 01:38:37 christos Exp $	*/
 
 /*
@@ -393,7 +393,7 @@ sleep_setup(struct sleep_state *sls, const volatile void *ident, int prio,
 	p->p_wchan = ident;
 	p->p_wmesg = wmesg;
 	p->p_slptime = 0;
-	p->p_priority = prio & PRIMASK;
+	p->p_slppri = prio & PRIMASK;
 	TAILQ_INSERT_TAIL(&slpque[LOOKUP(ident)], p, p_runq);
 }
 
@@ -624,7 +624,7 @@ sys_sched_yield(struct proc *p, void *v, register_t *retval)
 	 */
 	newprio = p->p_usrpri;
 	TAILQ_FOREACH(q, &p->p_p->ps_threads, p_thr_link)
-		newprio = max(newprio, q->p_priority);
+		newprio = max(newprio, q->p_runpri);
 	setrunqueue(p->p_cpu, p, newprio);
 	p->p_ru.ru_nvcsw++;
 	mi_switch();
diff --git a/sys/kern/sched_bsd.c b/sys/kern/sched_bsd.c
index 9172bc7e24e..7806cc42c6b 100644
--- a/sys/kern/sched_bsd.c
+++ b/sys/kern/sched_bsd.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: sched_bsd.c,v 1.61 2020/01/21 16:16:23 mpi Exp $	*/
+/*	$OpenBSD: sched_bsd.c,v 1.62 2020/01/30 08:51:27 mpi Exp $	*/
 /*	$NetBSD: kern_synch.c,v 1.37 1996/04/22 01:38:37 christos Exp $	*/
 
 /*-
@@ -255,14 +255,10 @@ schedcpu(void *arg)
 		newcpu = (u_int) decay_cpu(loadfac, p->p_estcpu);
 		setpriority(p, newcpu, p->p_p->ps_nice);
 
-		if (p->p_priority >= PUSER) {
-			if (p->p_stat == SRUN &&
-			    (p->p_priority / SCHED_PPQ) !=
-			    (p->p_usrpri / SCHED_PPQ)) {
-				remrunqueue(p);
-				setrunqueue(p->p_cpu, p, p->p_usrpri);
-			} else
-				p->p_priority = p->p_usrpri;
+		if (p->p_stat == SRUN &&
+		    (p->p_runpri / SCHED_PPQ) != (p->p_usrpri / SCHED_PPQ)) {
+			remrunqueue(p);
+			setrunqueue(p->p_cpu, p, p->p_usrpri);
 		}
 		SCHED_UNLOCK(s);
 	}
@@ -449,6 +445,7 @@ void
 setrunnable(struct proc *p)
 {
 	struct process *pr = p->p_p;
+	u_char prio;
 
 	SCHED_ASSERT_LOCKED();
 
@@ -467,11 +464,15 @@ setrunnable(struct proc *p)
 		 */
 		if ((pr->ps_flags & PS_TRACED) != 0 && pr->ps_xsig != 0)
 			atomic_setbits_int(&p->p_siglist, sigmask(pr->ps_xsig));
+		prio = p->p_usrpri;
+		unsleep(p);
+		break;
 	case SSLEEP:
+		prio = p->p_slppri;
 		unsleep(p);		/* e.g. when sending signals */
 		break;
 	}
-	setrunqueue(NULL, p, p->p_priority);
+	setrunqueue(NULL, p, prio);
 	if (p->p_slptime > 1) {
 		uint32_t newcpu;
 
@@ -524,8 +525,6 @@ schedclock(struct proc *p)
 	SCHED_LOCK(s);
 	newcpu = ESTCPULIM(p->p_estcpu + 1);
 	setpriority(p, newcpu, p->p_p->ps_nice);
-	if (p->p_priority >= PUSER)
-		p->p_priority = p->p_usrpri;
 	SCHED_UNLOCK(s);
 }
 
diff --git a/sys/sys/proc.h b/sys/sys/proc.h
index 0648c02cfd2..162292d0a97 100644
--- a/sys/sys/proc.h
+++ b/sys/sys/proc.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: proc.h,v 1.286 2020/01/21 15:20:47 visa Exp $	*/
+/*	$OpenBSD: proc.h,v 1.287 2020/01/30 08:51:27 mpi Exp $	*/
 /*	$NetBSD: proc.h,v 1.44 1996/04/22 01:23:21 christos Exp $	*/
 
 /*-
@@ -346,7 +346,7 @@ struct proc {
 	int	p_flag;			/* P_* flags. */
 	u_char	p_spare;		/* unused */
 	char	p_stat;			/* [s] S* process status. */
-	char	p_pad1[1];
+	u_char	p_runpri;		/* [s] Runqueue priority */
 	u_char	p_descfd;		/* if not 255, fdesc permits this fd */
 
 	pid_t	p_tid;			/* Thread identifier. */
@@ -385,8 +385,8 @@ struct proc {
 #define	p_startcopy	p_sigmask
 	sigset_t p_sigmask;	/* Current signal mask. */
 
-	u_char	p_priority;	/* [s] Process priority. */
-	u_char	p_usrpri;	/* [s] User-prio based on p_estcpu & ps_nice. */
+	u_char	p_slppri;		/* [s] Sleeping priority */
+	u_char	p_usrpri;	/* [s] Priority based on p_estcpu & ps_nice */
 	u_int	p_estcpu;		/* [s] Time averaged val of p_cpticks */
 	int	p_pledge_syscall;	/* Cache of current syscall */
 
diff --git a/sys/sys/sysctl.h b/sys/sys/sysctl.h
index 37ebb1f6345..c3b877aad04 100644
--- a/sys/sys/sysctl.h
+++ b/sys/sys/sysctl.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: sysctl.h,v 1.200 2020/01/24 14:00:32 mpi Exp $	*/
+/*	$OpenBSD: sysctl.h,v 1.201 2020/01/30 08:51:27 mpi Exp $	*/
 /*	$NetBSD: sysctl.h,v 1.16 1996/04/09 20:55:36 cgd Exp $	*/
 
 /*
@@ -561,6 +561,10 @@ struct kinfo_vmentry {
 #define PR_UNLOCK(pr)	/* nothing */
 #endif
 
+#define _getcompatprio(_p)						\
+	((_p)->p_stat == SRUN ? (_p)->p_runpri : 			\
+	    ((_p)->p_stat == SSLEEP) ? (_p)->p_slppri : (_p)->p_usrpri)
+
 #define PTRTOINT64(_x)	((u_int64_t)(u_long)(_x))
 
 #define FILL_KPROC(kp, copy_str, p, pr, uc, pg, paddr, \
@@ -659,7 +663,7 @@ do {									\
 		(kp)->p_stat = (p)->p_stat;				\
 		(kp)->p_slptime = (p)->p_slptime;			\
 		(kp)->p_holdcnt = 1;					\
-		(kp)->p_priority = (p)->p_priority;			\
+		(kp)->p_priority = _getcompatprio(p);			\
 		(kp)->p_usrpri = (p)->p_usrpri;				\
 		if ((p)->p_wchan && (p)->p_wmesg)			\
 			copy_str((kp)->p_wmesg, (p)->p_wmesg,		\