A second approach at fixing the telnet localhost & problem

(but I tend to call it ssh localhost & now when telnetd is
history).  This is more localized patch, but leaves us with
a recursive lock for protecting scheduling and signal state.
Better care is taken to actually be symmetric over mi_switch.
Also, the dolock cruft in psignal can go with this solution.
Better test runs by more people for longer time has been
carried out compared to the c2k5 patch.

Long term the current mess with interruptible sleep, the
default action on stop signals and wakeup interactions need
to be revisited.  ok deraadt@, art@

6 files changed, 48 insertions, 64 deletions

diff --git a/sys/kern/kern_sig.c b/sys/kern/kern_sig.c
index a8092d7bcb1..9d46d3aa405 100644
--- a/sys/kern/kern_sig.c
+++ b/sys/kern/kern_sig.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: kern_sig.c,v 1.75 2005/05/29 03:20:41 deraadt Exp $	*/
+/*	$OpenBSD: kern_sig.c,v 1.76 2005/06/17 22:33:34 niklas Exp $	*/
 /*	$NetBSD: kern_sig.c,v 1.54 1996/04/22 01:38:32 christos Exp $	*/
 
 /*
@@ -805,14 +805,11 @@ trapsignal(p, signum, code, type, sigval)
  *     regardless of the signal action (eg, blocked or ignored).
  *
  * Other ignored signals are discarded immediately.
- *
- * XXXSMP: Invoked as psignal() or sched_psignal().
  */
 void
-psignal1(p, signum, dolock)
+psignal(p, signum)
 	register struct proc *p;
 	register int signum;
-	int dolock;		/* XXXSMP: works, but icky */
 {
 	register int s, prop;
 	register sig_t action;
@@ -821,12 +818,6 @@ psignal1(p, signum, dolock)
 #ifdef DIAGNOSTIC
 	if ((u_int)signum >= NSIG || signum == 0)
 		panic("psignal signal number");
-
-	/* XXXSMP: works, but icky */
-	if (dolock)
-		SCHED_ASSERT_UNLOCKED();
-	else
-		SCHED_ASSERT_LOCKED();
 #endif
 
 	/* Ignore signal if we are exiting */
@@ -890,9 +881,8 @@ psignal1(p, signum, dolock)
 	 */
 	if (action == SIG_HOLD && ((prop & SA_CONT) == 0 || p->p_stat != SSTOP))
 		return;
-	/* XXXSMP: works, but icky */
-	if (dolock)
-		SCHED_LOCK(s);
+
+	SCHED_LOCK(s);
 
 	switch (p->p_stat) {
 
@@ -935,11 +925,7 @@ psignal1(p, signum, dolock)
 			p->p_siglist &= ~mask;
 			p->p_xstat = signum;
 			if ((p->p_pptr->p_flag & P_NOCLDSTOP) == 0)
-				/*
-				 * XXXSMP: recursive call; don't lock
-				 * the second time around.
-				 */
-				sched_psignal(p->p_pptr, SIGCHLD);
+				psignal(p->p_pptr, SIGCHLD);
 			proc_stop(p);
 			goto out;
 		}
@@ -1027,9 +1013,7 @@ runfast:
 run:
 	setrunnable(p);
 out:
-	/* XXXSMP: works, but icky */
-	if (dolock)
-		SCHED_UNLOCK(s);
+	SCHED_UNLOCK(s);
 }
 
 /*
@@ -1090,8 +1074,7 @@ issignal(struct proc *p)
 				proc_stop(p);
 				mi_switch();
 			}
-			SCHED_ASSERT_UNLOCKED();
-			splx(s);
+			SCHED_UNLOCK(s);
 
 			/*
 			 * If we are no longer being traced, or the parent
@@ -1153,8 +1136,7 @@ issignal(struct proc *p)
 				SCHED_LOCK(s);
 				proc_stop(p);
 				mi_switch();
-				SCHED_ASSERT_UNLOCKED();
-				splx(s);
+				SCHED_UNLOCK(s);
 				break;
 			} else if (prop & SA_IGNORE) {
 				/*
diff --git a/sys/kern/kern_synch.c b/sys/kern/kern_synch.c
index 16fc2d6f8ae..3f6232c282f 100644
--- a/sys/kern/kern_synch.c
+++ b/sys/kern/kern_synch.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: kern_synch.c,v 1.63 2005/05/29 03:20:41 deraadt Exp $	*/
+/*	$OpenBSD: kern_synch.c,v 1.64 2005/06/17 22:33:34 niklas Exp $	*/
 /*	$NetBSD: kern_synch.c,v 1.37 1996/04/22 01:38:37 christos Exp $	*/
 
 /*-
@@ -174,12 +174,10 @@ ltsleep(ident, priority, wmesg, timo, interlock)
 			if (p->p_wchan)
 				unsleep(p);
 			p->p_stat = SONPROC;
-			SCHED_UNLOCK(s);
 			goto resume;
 		}
 		if (p->p_wchan == 0) {
 			catch = 0;
-			SCHED_UNLOCK(s);
 			goto resume;
 		}
 	} else
@@ -193,14 +191,9 @@ ltsleep(ident, priority, wmesg, timo, interlock)
 	__asm(".globl bpendtsleep\nbpendtsleep:");
 #endif
 
-	SCHED_ASSERT_UNLOCKED();
-	/*
-	 * Note! this splx belongs to the SCHED_LOCK(s) above, mi_switch
-	 * releases the scheduler lock, but does not lower the spl.
-	 */
-	splx(s);
-
 resume:
+	SCHED_UNLOCK(s);
+
 #ifdef __HAVE_CPUINFO
 	p->p_cpu->ci_schedstate.spc_curpriority = p->p_usrpri;
 #else
diff --git a/sys/kern/sched_bsd.c b/sys/kern/sched_bsd.c
index 854ff074573..4f6d0747461 100644
--- a/sys/kern/sched_bsd.c
+++ b/sys/kern/sched_bsd.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: sched_bsd.c,v 1.4 2005/05/29 03:20:41 deraadt Exp $	*/
+/*	$OpenBSD: sched_bsd.c,v 1.5 2005/06/17 22:33:34 niklas Exp $	*/
 /*	$NetBSD: kern_synch.c,v 1.37 1996/04/22 01:38:37 christos Exp $	*/
 
 /*-
@@ -381,8 +381,7 @@ yield()
 	setrunqueue(p);
 	p->p_stats->p_ru.ru_nvcsw++;
 	mi_switch();
-	SCHED_ASSERT_UNLOCKED();
-	splx(s);
+	SCHED_UNLOCK(s);
 }
 
 /*
@@ -410,8 +409,7 @@ preempt(newp)
 	setrunqueue(p);
 	p->p_stats->p_ru.ru_nivcsw++;
 	mi_switch();
-	SCHED_ASSERT_UNLOCKED();
-	splx(s);
+	SCHED_UNLOCK(s);
 }
 
 
@@ -426,6 +424,7 @@ mi_switch()
 	struct timeval tv;
 #if defined(MULTIPROCESSOR)
 	int hold_count;
+	int sched_count;
 #endif
 #ifdef __HAVE_CPUINFO
 	struct schedstate_percpu *spc = &p->p_cpu->ci_schedstate;
@@ -439,13 +438,10 @@ mi_switch()
 	 * The scheduler lock is still held until cpu_switch()
 	 * selects a new process and removes it from the run queue.
 	 */
+	sched_count = __mp_release_all_but_one(&sched_lock);
 	if (p->p_flag & P_BIGLOCK)
-#ifdef notyet
-		hold_count = spinlock_release_all(&kernel_lock);
-#else
 		hold_count = __mp_release_all(&kernel_lock);
 #endif
-#endif
 
 	/*
 	 * Compute the amount of time during which the current
@@ -532,14 +528,11 @@ mi_switch()
 	/*
 	 * Reacquire the kernel_lock now.  We do this after we've
 	 * released the scheduler lock to avoid deadlock, and before
-	 * we reacquire the interlock.
+	 * we reacquire the interlock and the scheduler lock.
 	 */
 	if (p->p_flag & P_BIGLOCK)
-#ifdef notyet
-		spinlock_acquire_count(&kernel_lock, hold_count);
-#else
 		__mp_acquire_count(&kernel_lock, hold_count);
-#endif
+	__mp_acquire_count(&sched_lock, sched_count + 1);
 #endif
 }
 
diff --git a/sys/sys/mplock.h b/sys/sys/mplock.h
index 3d17ede906a..0357c4ef48b 100644
--- a/sys/sys/mplock.h
+++ b/sys/sys/mplock.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: mplock.h,v 1.7 2005/05/29 03:20:42 deraadt Exp $	*/
+/*	$OpenBSD: mplock.h,v 1.8 2005/06/17 22:33:34 niklas Exp $	*/
 
 /*
  * Copyright (c) 2004 Niklas Hallqvist.  All rights reserved.
@@ -170,19 +170,37 @@ __mp_release_all(struct __mp_lock *lock) {
 	return (rv);
 }
 
+static __inline int
+__mp_release_all_but_one(struct __mp_lock *lock) {
+	int s = spllock();
+	int rv = lock->mpl_count - 1;
+
+#ifdef MP_LOCKDEBUG
+	if (lock->mpl_count == 0 || lock->mpl_cpu == LK_NOCPU) {
+		db_printf(
+		    "__mp_release_all_but_one(0x%x): releasing not locked lock\n",
+		    lock);
+		Debugger();
+	}
+#endif
+
+	lock->mpl_count = 1;
+	splx(s);
+	return (rv);
+}
+
 static __inline void
 __mp_acquire_count(struct __mp_lock *lock, int count) {
 	int s = spllock();
 
-	__cpu_simple_lock(&lock->mpl_lock);
-	lock->mpl_cpu = cpu_number();
-	lock->mpl_count = count;
+	while (count--)
+		__mp_lock(lock);
 	splx(s);
 }
 
 static __inline int
 __mp_lock_held(struct __mp_lock *lock) {
-	return lock->mpl_count;
+	return lock->mpl_count && lock->mpl_cpu == cpu_number();
 }
 
 extern struct __mp_lock kernel_lock;
diff --git a/sys/sys/sched.h b/sys/sys/sched.h
index 7be5d0161ab..1256bece474 100644
--- a/sys/sys/sched.h
+++ b/sys/sys/sched.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: sched.h,v 1.12 2005/05/29 03:20:42 deraadt Exp $	*/
+/*	$OpenBSD: sched.h,v 1.13 2005/06/17 22:33:34 niklas Exp $	*/
 /* $NetBSD: sched.h,v 1.2 1999/02/28 18:14:58 ross Exp $ */
 
 /*-
@@ -158,8 +158,8 @@ void roundrobin(struct cpu_info *);
 
 extern struct simplelock sched_lock;
 
-#define	SCHED_ASSERT_LOCKED()	LOCK_ASSERT(simple_lock_held(&sched_lock))
-#define	SCHED_ASSERT_UNLOCKED()	LOCK_ASSERT(simple_lock_held(&sched_lock) == 0)
+#define	SCHED_ASSERT_LOCKED()	KASSERT(simple_lock_held(&sched_lock))
+#define	SCHED_ASSERT_UNLOCKED()	KASSERT(simple_lock_held(&sched_lock) == 0)
 
 #define	SCHED_LOCK(s)							\
 do {									\
@@ -177,8 +177,8 @@ do {									\
 
 extern struct __mp_lock sched_lock;
 
-#define	SCHED_ASSERT_LOCKED()	LOCK_ASSERT(__mp_lock_held(&sched_lock))
-#define	SCHED_ASSERT_UNLOCKED()	LOCK_ASSERT(__mp_lock_held(&sched_lock) == 0)
+#define	SCHED_ASSERT_LOCKED()	KASSERT(__mp_lock_held(&sched_lock))
+#define	SCHED_ASSERT_UNLOCKED()	KASSERT(__mp_lock_held(&sched_lock) == 0)
 
 #define	SCHED_LOCK(s)							\
 do {									\
diff --git a/sys/sys/signalvar.h b/sys/sys/signalvar.h
index fbc8a7286fe..8618c280032 100644
--- a/sys/sys/signalvar.h
+++ b/sys/sys/signalvar.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: signalvar.h,v 1.14 2005/05/29 03:20:43 deraadt Exp $	*/
+/*	$OpenBSD: signalvar.h,v 1.15 2005/06/17 22:33:34 niklas Exp $	*/
 /*	$NetBSD: signalvar.h,v 1.17 1996/04/22 01:23:31 christos Exp $	*/
 
 /*
@@ -159,9 +159,7 @@ void	csignal(pid_t pgid, int signum, uid_t uid, uid_t euid);
 int	issignal(struct proc *p);
 void	pgsignal(struct pgrp *pgrp, int sig, int checkctty);
 void	postsig(int sig);
-void	psignal1(struct proc *p, int sig, int dolock);
-#define	psignal(p, sig)		psignal1((p), (sig), 1)
-#define	sched_psignal(p, sig)	psignal1((p), (sig), 0)
+void	psignal(struct proc *p, int sig);
 void	siginit(struct proc *p);
 void	trapsignal(struct proc *p, int sig, u_long code, int type,
 	    union sigval val);