This patch is mortly art's work and was done *a year* ago. Art wants to thank

everyone for the prompt review and ok of this work ;-)  Yeah, that includes me
too, or maybe especially me.  I am sorry.

Change the sched_lock to a mutex. This fixes, among other things, the infamous
"telnet localhost &" problem.  The real bug in that case was that the sched_lock
which is by design a non-recursive lock, was recursively acquired, and not
enough releases made us hold the lock in the idle loop, blocking scheduling
on the other processors.  Some of the other processors would hold the biglock though,
which made it impossible for cpu 0 to enter the kernel...  A nice deadlock.
Let me just say debugging this for days just to realize that it was all fixed
in an old diff noone ever ok'd was somewhat of an anti-climax.

This diff also changes splsched to be correct for all our architectures.

25 files changed, 173 insertions, 348 deletions

diff --git a/sys/arch/amd64/amd64/ioapic.c b/sys/arch/amd64/amd64/ioapic.c
index 93a53155c65..21c4b38ddab 100644
--- a/sys/arch/amd64/amd64/ioapic.c
+++ b/sys/arch/amd64/amd64/ioapic.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: ioapic.c,v 1.3 2004/06/27 16:17:50 deraadt Exp $	*/
+/*	$OpenBSD: ioapic.c,v 1.4 2005/05/25 23:17:47 niklas Exp $	*/
 /* 	$NetBSD: ioapic.c,v 1.6 2003/05/15 13:30:31 fvdl Exp $	*/
 
 /*-
@@ -127,14 +127,14 @@ ioapic_lock(struct ioapic_softc *sc)
 
 	flags = read_psl();
 	disable_intr();
-	SIMPLE_LOCK(&sc->sc_pic.pic_lock);
+	mtx_enter(&sc->sc_pic.pic_mutex);
 	return flags;
 }
 
 static __inline void
 ioapic_unlock(struct ioapic_softc *sc, u_long flags)
 {
-	SIMPLE_UNLOCK(&sc->sc_pic.pic_lock);
+	mtx_leave(&sc->sc_pic.pic_mutex);
 	write_psl(flags);
 }
 
@@ -294,7 +294,7 @@ ioapic_attach(struct device *parent, struct device *self, void *aux)
 	sc->sc_data = (volatile u_int32_t *)(bh + IOAPIC_DATA);	
 
 	sc->sc_pic.pic_type = PIC_IOAPIC;
-	SIMPLE_LOCK_INIT(&sc->sc_pic.pic_lock);
+	mtx_init(&sc->sc_pic.pic_mutex, IPL_NONE);
 	sc->sc_pic.pic_hwmask = ioapic_hwmask;
 	sc->sc_pic.pic_hwunmask = ioapic_hwunmask;
 	sc->sc_pic.pic_addroute = ioapic_addroute;
diff --git a/sys/arch/amd64/amd64/locore.S b/sys/arch/amd64/amd64/locore.S
index 0518a692380..5b7aefe62ac 100644
--- a/sys/arch/amd64/amd64/locore.S
+++ b/sys/arch/amd64/amd64/locore.S
@@ -1,4 +1,4 @@
-/*	$OpenBSD: locore.S,v 1.16 2005/01/06 20:15:47 martin Exp $	*/
+/*	$OpenBSD: locore.S,v 1.17 2005/05/25 23:17:47 niklas Exp $	*/
 /*	$NetBSD: locore.S,v 1.13 2004/03/25 18:33:17 drochner Exp $	*/
 
 /*
@@ -846,9 +846,8 @@ idle_start:
 	cmpl	$0,_C_LABEL(whichqs)(%rip)
 	jz	idle_loop
 idle_exit:
-	movl	$IPL_HIGH,CPUVAR(ILEVEL)
 	sti
-#if defined(MULTIPROCESSOR) || defined(LOCKDEBUG)       
+#if defined(MULTIPROCESSOR)
 	call	_C_LABEL(sched_lock_idle)
 #endif
 switch_search:
diff --git a/sys/arch/amd64/amd64/trap.c b/sys/arch/amd64/amd64/trap.c
index 79c5f7c0869..0bc830b1c7b 100644
--- a/sys/arch/amd64/amd64/trap.c
+++ b/sys/arch/amd64/amd64/trap.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: trap.c,v 1.4 2004/12/06 20:12:23 miod Exp $	*/
+/*	$OpenBSD: trap.c,v 1.5 2005/05/25 23:17:47 niklas Exp $	*/
 /*	$NetBSD: trap.c,v 1.2 2003/05/04 23:51:56 fvdl Exp $	*/
 
 /*-
@@ -386,10 +386,6 @@ copyfault:
 	case T_PAGEFLT:			/* allow page faults in kernel mode */
 		if (p == NULL)
 			goto we_re_toast;
-#ifdef LOCKDEBUG
-		if (simple_lock_held(&sched_lock))
-			goto we_re_toast;
-#endif
 #ifdef MULTIPROCESSOR
 		if ((p->p_flag & P_BIGLOCK) == 0)
 			goto we_re_toast;
diff --git a/sys/arch/amd64/include/pic.h b/sys/arch/amd64/include/pic.h
index b6a222b2780..0ef3a8fa4ca 100644
--- a/sys/arch/amd64/include/pic.h
+++ b/sys/arch/amd64/include/pic.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: pic.h,v 1.2 2004/06/25 11:03:28 art Exp $	*/
+/*	$OpenBSD: pic.h,v 1.3 2005/05/25 23:17:47 niklas Exp $	*/
 /*	$NetBSD: pic.h,v 1.1 2003/02/26 21:26:11 fvdl Exp $	*/
 
 #ifndef _X86_PIC_H
@@ -20,7 +20,7 @@ struct pic {
 	struct device pic_dev;
         int pic_type;
 #ifdef MULTIPROCESSOR
-	struct SIMPLE_LOCK pic_lock;
+	struct mutex pic_mutex;
 #endif
         void (*pic_hwmask)(struct pic *, int);
         void (*pic_hwunmask)(struct pic *, int);
diff --git a/sys/arch/hppa64/include/cpu.h b/sys/arch/hppa64/include/cpu.h
index 8de00e76eb8..e4054861dab 100644
--- a/sys/arch/hppa64/include/cpu.h
+++ b/sys/arch/hppa64/include/cpu.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: cpu.h,v 1.2 2005/04/19 15:29:48 mickey Exp $	*/
+/*	$OpenBSD: cpu.h,v 1.3 2005/05/25 23:17:47 niklas Exp $	*/
 
 /*
  * Copyright (c) 2005 Michael Shalayeff
@@ -83,8 +83,8 @@
 #define	IPL_VM		7
 #define	IPL_AUDIO	8
 #define	IPL_CLOCK	9
-#define	IPL_SCHED	9
 #define	IPL_STATCLOCK	10
+#define	IPL_SCHED	IPL_STATCLOCK
 #define	IPL_HIGH	11
 
 #define	NIPL		12
diff --git a/sys/arch/i386/i386/locore.s b/sys/arch/i386/i386/locore.s
index a067a9b3ca9..df652a97c19 100644
--- a/sys/arch/i386/i386/locore.s
+++ b/sys/arch/i386/i386/locore.s
@@ -1,4 +1,4 @@
-/*	$OpenBSD: locore.s,v 1.87 2005/05/25 22:50:25 beck Exp $	*/
+/*	$OpenBSD: locore.s,v 1.88 2005/05/25 23:17:47 niklas Exp $	*/
 /*	$NetBSD: locore.s,v 1.145 1996/05/03 19:41:19 christos Exp $	*/
 
 /*-
@@ -1616,7 +1616,6 @@ ENTRY(idle_loop)
 	jmp	_C_LABEL(idle_loop)
 
 ENTRY(idle_exit)
-	movl	$IPL_HIGH,CPL		# splhigh
 	sti
 #if defined(MULTIPROCESSOR) || defined(LOCKDEBUG)	
 	call	_C_LABEL(sched_lock_idle)
diff --git a/sys/arch/i386/i386/trap.c b/sys/arch/i386/i386/trap.c
index e0aeb203bfb..98d21021725 100644
--- a/sys/arch/i386/i386/trap.c
+++ b/sys/arch/i386/i386/trap.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: trap.c,v 1.67 2004/12/06 20:12:24 miod Exp $	*/
+/*	$OpenBSD: trap.c,v 1.68 2005/05/25 23:17:47 niklas Exp $	*/
 /*	$NetBSD: trap.c,v 1.95 1996/05/05 06:50:02 mycroft Exp $	*/
 
 /*-
@@ -451,15 +451,6 @@ trap(frame)
 	case T_PAGEFLT:			/* allow page faults in kernel mode */
 		if (p == 0 || p->p_addr == 0)
 			goto we_re_toast;
-#ifdef LOCKDEBUG
-		/* If we page-fault while in scheduler, we're doomed. */
-#ifdef notyet
-		if (simple_lock_held(&sched_lock))
-#else
-		if (__mp_lock_held(&sched_lock))
-#endif
-			goto we_re_toast;
-#endif
 
 		pcb = &p->p_addr->u_pcb;
 #if 0
diff --git a/sys/arch/i386/include/intrdefs.h b/sys/arch/i386/include/intrdefs.h
index c2c998b76b3..2cf218fc765 100644
--- a/sys/arch/i386/include/intrdefs.h
+++ b/sys/arch/i386/include/intrdefs.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: intrdefs.h,v 1.2 2004/06/13 21:49:16 niklas Exp $	*/
+/*	$OpenBSD: intrdefs.h,v 1.3 2005/05/25 23:17:47 niklas Exp $	*/
 /*	$NetBSD: intrdefs.h,v 1.2 2003/05/04 22:01:56 fvdl Exp $	*/
 
 #ifndef _i386_INTRDEFS_H
@@ -66,8 +66,8 @@
 #define IPL_IMP		IPL_VM		/* XXX - should not be here. */
 #define	IPL_AUDIO	MAKEIPL(7)	/* audio */
 #define	IPL_CLOCK	MAKEIPL(8)	/* clock */
-#define	IPL_SCHED	IPL_CLOCK
 #define	IPL_STATCLOCK	MAKEIPL(9)	/* statclock */
+#define	IPL_SCHED	IPL_STATCLOCK
 #define	IPL_HIGH	MAKEIPL(9)	/* everything */
 #define	IPL_IPI		MAKEIPL(10)	/* interprocessor interrupt */
 
diff --git a/sys/arch/mips64/mips64/interrupt.c b/sys/arch/mips64/mips64/interrupt.c
index 391c6ea26ce..95ea6c0ef8f 100644
--- a/sys/arch/mips64/mips64/interrupt.c
+++ b/sys/arch/mips64/mips64/interrupt.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: interrupt.c,v 1.11 2005/01/31 21:35:50 grange Exp $ */
+/*	$OpenBSD: interrupt.c,v 1.12 2005/05/25 23:17:47 niklas Exp $ */
 
 /*
  * Copyright (c) 2001-2004 Opsycon AB  (www.opsycon.se / www.opsycon.com)
@@ -282,21 +282,10 @@ softintr()
 		ADDUPROF(p);
 	}
 	if (want_resched) {
-		int s;
-
 		/*
-		 * Since we are curproc, clock will normally just change
-		 * our priority without moving us from one queue to another
-		 * (since the running process is not on a queue.)
-		 * If that happened after we put ourselves on the run queue
-		 * but before we switched, we might not be on the queue
-		 * indicated by our priority.
+		 * We're being preempted.
 		 */
-		s = splstatclock();
-		setrunqueue(p);
-		p->p_stats->p_ru.ru_nivcsw++;
-		mi_switch();
-		splx(s);
+		preempt(NULL);
 		while ((sig = CURSIG(p)) != 0)
 			postsig(sig);
 	}
diff --git a/sys/arch/sparc64/include/psl.h b/sys/arch/sparc64/include/psl.h
index 32a36b71ec2..633a97467a9 100644
--- a/sys/arch/sparc64/include/psl.h
+++ b/sys/arch/sparc64/include/psl.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: psl.h,v 1.13 2005/04/19 15:29:48 mickey Exp $	*/
+/*	$OpenBSD: psl.h,v 1.14 2005/05/25 23:17:47 niklas Exp $	*/
 /*	$NetBSD: psl.h,v 1.20 2001/04/13 23:30:05 thorpej Exp $ */
 
 /*
@@ -90,7 +90,7 @@
 #define PIL_SER		12
 #define PIL_STATCLOCK	14
 #define PIL_HIGH	15
-#define PIL_SCHED	PIL_CLOCK
+#define PIL_SCHED	PIL_STATCLOCK
 #define PIL_LOCK	PIL_HIGH
 
 /* 
diff --git a/sys/kern/kern_fork.c b/sys/kern/kern_fork.c
index ed404db4a0c..05ba79f3399 100644
--- a/sys/kern/kern_fork.c
+++ b/sys/kern/kern_fork.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: kern_fork.c,v 1.74 2005/02/24 04:44:25 tedu Exp $	*/
+/*	$OpenBSD: kern_fork.c,v 1.75 2005/05/25 23:17:47 niklas Exp $	*/
 /*	$NetBSD: kern_fork.c,v 1.29 1996/02/09 18:59:34 christos Exp $	*/
 
 /*
@@ -341,9 +341,9 @@ fork1(struct proc *p1, int exitsig, int flags, void *stack, size_t stacksize,
 	/*
 	 * Make child runnable, set start time, and add to run queue.
 	 */
-	SCHED_LOCK(s);
  	getmicrotime(&p2->p_stats->p_start);
 	p2->p_acflag = AFORK;
+	SCHED_LOCK(s);
 	p2->p_stat = SRUN;
 	setrunqueue(p2);
 	SCHED_UNLOCK(s);
diff --git a/sys/kern/kern_lock.c b/sys/kern/kern_lock.c
index 4a56f9e6388..d36c6d14ec6 100644
--- a/sys/kern/kern_lock.c
+++ b/sys/kern/kern_lock.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: kern_lock.c,v 1.17 2004/11/01 06:39:39 marius Exp $	*/
+/*	$OpenBSD: kern_lock.c,v 1.18 2005/05/25 23:17:47 niklas Exp $	*/
 
 /* 
  * Copyright (c) 1995
@@ -1121,18 +1121,6 @@ simple_lock_freecheck(void *start, void *end)
 	splx(s);
  }
 
-/*
- * We must be holding exactly one lock: the sched_lock.
- */
-
-#ifdef notyet
-void
-simple_lock_switchcheck(void)
-{
-
-	simple_lock_only_held(&sched_lock, "switching");
-}
-#endif
 
 void
 simple_lock_only_held(volatile struct simplelock *lp, const char *where)
@@ -1172,60 +1160,6 @@ simple_lock_only_held(volatile struct simplelock *lp, const char *where)
  * so that they show up in profiles.
  */
 
-/*
- * XXX Instead of using struct lock for the kernel lock and thus requiring us
- * XXX to implement simplelocks, causing all sorts of fine-grained locks all
- * XXX over our tree getting activated consuming both time and potentially
- * XXX introducing locking protocol bugs.
- */
-#ifdef notyet
-
-struct lock kernel_lock; 
-
-void
-_kernel_lock_init(void)
-{
-	spinlockinit(&kernel_lock, "klock", 0);
-}
-
-/*
- * Acquire/release the kernel lock.  Intended for use in the scheduler
- * and the lower half of the kernel.
- */
-void
-_kernel_lock(int flag)
-{
-	SCHED_ASSERT_UNLOCKED();
-	spinlockmgr(&kernel_lock, flag, 0);
-}
-
-void
-_kernel_unlock(void)
-{
-	spinlockmgr(&kernel_lock, LK_RELEASE, 0);
-}
-
-/*
- * Acquire/release the kernel_lock on behalf of a process.  Intended for
- * use in the top half of the kernel.
- */
-void
-_kernel_proc_lock(struct proc *p)
-{
-	SCHED_ASSERT_UNLOCKED();
-	spinlockmgr(&kernel_lock, LK_EXCLUSIVE, 0);
-	p->p_flag |= P_BIGLOCK;
-}
-
-void
-_kernel_proc_unlock(struct proc *p)
-{
-	p->p_flag &= ~P_BIGLOCK;
-	spinlockmgr(&kernel_lock, LK_RELEASE, 0);
-}
-
-#else
-
 struct __mp_lock kernel_lock; 
 
 void
@@ -1272,8 +1206,6 @@ _kernel_proc_unlock(struct proc *p)
 	__mp_unlock(&kernel_lock);
 }
 
-#endif
-
 #ifdef MP_LOCKDEBUG
 /* CPU-dependent timing, needs this to be settable from ddb. */
 int __mp_lock_spinout = 200000000;
diff --git a/sys/kern/kern_resource.c b/sys/kern/kern_resource.c
index 3d5759bad59..d770a3858af 100644
--- a/sys/kern/kern_resource.c
+++ b/sys/kern/kern_resource.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: kern_resource.c,v 1.28 2004/12/26 21:22:13 miod Exp $	*/
+/*	$OpenBSD: kern_resource.c,v 1.29 2005/05/25 23:17:47 niklas Exp $	*/
 /*	$NetBSD: kern_resource.c,v 1.38 1996/10/23 07:19:38 matthias Exp $	*/
 
 /*-
@@ -199,7 +199,7 @@ donice(curp, chgp, n)
 		return (EACCES);
 	chgp->p_nice = n;
 	SCHED_LOCK(s);
-	(void)resetpriority(chgp);
+	resetpriority(chgp);
 	SCHED_UNLOCK(s);
 	return (0);
 }
diff --git a/sys/kern/kern_sig.c b/sys/kern/kern_sig.c
index 7a0142d92dc..d228b218db6 100644
--- a/sys/kern/kern_sig.c
+++ b/sys/kern/kern_sig.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: kern_sig.c,v 1.73 2004/12/26 21:22:13 miod Exp $	*/
+/*	$OpenBSD: kern_sig.c,v 1.74 2005/05/25 23:17:47 niklas Exp $	*/
 /*	$NetBSD: kern_sig.c,v 1.54 1996/04/22 01:38:32 christos Exp $	*/
 
 /*
@@ -805,29 +805,19 @@ trapsignal(p, signum, code, type, sigval)
  *     regardless of the signal action (eg, blocked or ignored).
  *
  * Other ignored signals are discarded immediately.
- *
- * XXXSMP: Invoked as psignal() or sched_psignal().
  */
 void
-psignal1(p, signum, dolock)
-	register struct proc *p;
-	register int signum;
-	int dolock;		/* XXXSMP: works, but icky */
+psignal(struct proc *p, int signum)
 {
-	register int s, prop;
-	register sig_t action;
+	int s, prop;
+	sig_t action;
 	int mask;
 
 #ifdef DIAGNOSTIC
 	if ((u_int)signum >= NSIG || signum == 0)
 		panic("psignal signal number");
-
-	/* XXXSMP: works, but icky */
-	if (dolock)
-		SCHED_ASSERT_UNLOCKED();
-	else
-		SCHED_ASSERT_LOCKED();
 #endif
+	SCHED_ASSERT_UNLOCKED();
 
 	/* Ignore signal if we are exiting */
 	if (p->p_flag & P_WEXIT)
@@ -890,10 +880,8 @@ psignal1(p, signum, dolock)
 	 */
 	if (action == SIG_HOLD && ((prop & SA_CONT) == 0 || p->p_stat != SSTOP))
 		return;
-	/* XXXSMP: works, but icky */
-	if (dolock)
-		SCHED_LOCK(s);
 
+	SCHED_LOCK(s);
 	switch (p->p_stat) {
 
 	case SSLEEP:
@@ -934,12 +922,11 @@ psignal1(p, signum, dolock)
 				goto out;
 			p->p_siglist &= ~mask;
 			p->p_xstat = signum;
-			if ((p->p_pptr->p_flag & P_NOCLDSTOP) == 0)
-				/*
-				 * XXXSMP: recursive call; don't lock
-				 * the second time around.
-				 */
-				sched_psignal(p->p_pptr, SIGCHLD);
+			if ((p->p_pptr->p_flag & P_NOCLDSTOP) == 0) {
+				SCHED_UNLOCK(s);
+				psignal(p->p_pptr, SIGCHLD);
+				SCHED_LOCK(s);
+			}
 			proc_stop(p);
 			goto out;
 		}
@@ -976,7 +963,7 @@ psignal1(p, signum, dolock)
 			 * Otherwise, process goes back to sleep state.
 			 */
 			p->p_flag |= P_CONTINUED;
-			wakeup(p->p_pptr);
+			sched_wakeup(p->p_pptr);
 			if (action == SIG_DFL)
 				p->p_siglist &= ~mask;
 			if (action == SIG_CATCH)
@@ -1027,9 +1014,7 @@ runfast:
 run:
 	setrunnable(p);
 out:
-	/* XXXSMP: works, but icky */
-	if (dolock)
-		SCHED_UNLOCK(s);
+	SCHED_UNLOCK(s);
 }
 
 /*
@@ -1074,24 +1059,23 @@ issignal(struct proc *p)
 			 */
 			p->p_xstat = signum;
 
-			SCHED_LOCK(s);	/* protect mi_switch */
 			if (p->p_flag & P_FSTRACE) {
 #ifdef	PROCFS
+				SCHED_LOCK(s);
 				/* procfs debugging */
 				p->p_stat = SSTOP;
-				wakeup(p);
-				mi_switch();
+				sched_wakeup(p);
+				mi_switch(s);
 #else
 				panic("procfs debugging");
 #endif
 			} else {
 				/* ptrace debugging */
 				psignal(p->p_pptr, SIGCHLD);
+				SCHED_LOCK(s);
 				proc_stop(p);
-				mi_switch();
+				mi_switch(s);
 			}
-			SCHED_ASSERT_UNLOCKED();
-			splx(s);
 
 			/*
 			 * If we are no longer being traced, or the parent
@@ -1152,9 +1136,7 @@ issignal(struct proc *p)
 					psignal(p->p_pptr, SIGCHLD);
 				SCHED_LOCK(s);
 				proc_stop(p);
-				mi_switch();
-				SCHED_ASSERT_UNLOCKED();
-				splx(s);
+				mi_switch(s);
 				break;
 			} else if (prop & SA_IGNORE) {
 				/*
@@ -1198,16 +1180,13 @@ keep:
  * on the run queue.
  */
 void
-proc_stop(p)
-	struct proc *p;
+proc_stop(struct proc *p)
 {
-#ifdef MULTIPROCESSOR
 	SCHED_ASSERT_LOCKED();
-#endif
 
 	p->p_stat = SSTOP;
 	p->p_flag &= ~P_WAITED;
-	wakeup(p->p_pptr);
+	sched_wakeup(p->p_pptr);
 }
 
 /*
diff --git a/sys/kern/kern_synch.c b/sys/kern/kern_synch.c
index fa0ff867b3c..9d887083455 100644
--- a/sys/kern/kern_synch.c
+++ b/sys/kern/kern_synch.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: kern_synch.c,v 1.61 2004/07/29 06:25:45 tedu Exp $	*/
+/*	$OpenBSD: kern_synch.c,v 1.62 2005/05/25 23:17:47 niklas Exp $	*/
 /*	$NetBSD: kern_synch.c,v 1.37 1996/04/22 01:38:37 christos Exp $	*/
 
 /*-
@@ -145,8 +145,12 @@ ltsleep(ident, priority, wmesg, timo, interlock)
 	else
 		*qp->sq_tailp = p;
 	*(qp->sq_tailp = &p->p_forw) = 0;
+
+	p->p_stat = SSLEEP;
+
 	if (timo)
 		timeout_add(&p->p_sleep_to, timo);
+
 	/*
 	 * We can now release the interlock; the scheduler_slock
 	 * is held, so a thread can't get in to do wakeup() before
@@ -170,13 +174,16 @@ ltsleep(ident, priority, wmesg, timo, interlock)
 	 */
 	if (catch) {
 		p->p_flag |= P_SINTR;
+		SCHED_UNLOCK(s);	/* XXX - must unlock for CURSIG */
 		if ((sig = CURSIG(p)) != 0) {
+			SCHED_LOCK(s);
 			if (p->p_wchan)
 				unsleep(p);
 			p->p_stat = SONPROC;
 			SCHED_UNLOCK(s);
 			goto resume;
 		}
+		SCHED_LOCK(s);
 		if (p->p_wchan == 0) {
 			catch = 0;
 			SCHED_UNLOCK(s);
@@ -184,22 +191,14 @@ ltsleep(ident, priority, wmesg, timo, interlock)
 		}
 	} else
 		sig = 0;
-	p->p_stat = SSLEEP;
 	p->p_stats->p_ru.ru_nvcsw++;
 	SCHED_ASSERT_LOCKED();
-	mi_switch();
+	mi_switch(s);
 #ifdef	DDB
 	/* handy breakpoint location after process "wakes" */
 	__asm(".globl bpendtsleep\nbpendtsleep:");
 #endif
 
-	SCHED_ASSERT_UNLOCKED();
-	/*
-	 * Note! this splx belongs to the SCHED_LOCK(s) above, mi_switch
-	 * releases the scheduler lock, but does not lower the spl.
-	 */
-	splx(s);
-
 resume:
 #ifdef __HAVE_CPUINFO
 	p->p_cpu->ci_schedstate.spc_curpriority = p->p_usrpri;
@@ -270,20 +269,13 @@ endtsleep(arg)
  * Remove a process from its wait queue
  */
 void
-unsleep(p)
-	register struct proc *p;
+unsleep(struct proc *p)
 {
-	register struct slpque *qp;
-	register struct proc **hp;
-#if 0
-	int s;
+	struct slpque *qp;
+	struct proc **hp;
+
+	SCHED_ASSERT_LOCKED();
 
-	/*
-	 * XXX we cannot do recursive SCHED_LOCKing yet.  All callers lock
-	 * anyhow.
-	 */
-	SCHED_LOCK(s);
-#endif
 	if (p->p_wchan) {
 		hp = &(qp = &slpque[LOOKUP(p->p_wchan)])->sq_head;
 		while (*hp != p)
@@ -293,24 +285,39 @@ unsleep(p)
 			qp->sq_tailp = hp;
 		p->p_wchan = 0;
 	}
-#if 0
+}
+
+void
+wakeup(void *ident)
+{
+	int s;
+
+	SCHED_LOCK(s);
+	sched_wakeup(ident);
+	SCHED_UNLOCK(s);
+}
+
+void
+wakeup_n(void *ident, int n)
+{
+	int s;
+
+	SCHED_LOCK(s);
+	sched_wakeup_n(ident, n);
 	SCHED_UNLOCK(s);
-#endif
 }
 
 /*
  * Make all processes sleeping on the specified identifier runnable.
  */
 void
-wakeup_n(ident, n)
-	void *ident;
-	int n;
+sched_wakeup_n(void *ident, int n)
 {
 	struct slpque *qp;
 	struct proc *p, **q;
-	int s;
 
-	SCHED_LOCK(s);
+	SCHED_ASSERT_LOCKED();
+
 	qp = &slpque[LOOKUP(ident)];
 restart:
 	for (q = &qp->sq_head; (p = *q) != NULL; ) {
@@ -349,7 +356,7 @@ restart:
 					need_resched(0);
 #endif
 				} else {
-					wakeup((caddr_t)&proc0);
+					sched_wakeup((caddr_t)&proc0);
 				}
 				/* END INLINE EXPANSION */
 
@@ -361,12 +368,4 @@ restart:
 		} else
 			q = &p->p_forw;
 	}
-	SCHED_UNLOCK(s);
-}
-
-void
-wakeup(chan)
-	void *chan;
-{
-	wakeup_n(chan, -1);
 }
diff --git a/sys/kern/kern_time.c b/sys/kern/kern_time.c
index 567341a2f85..d5e29b55362 100644
--- a/sys/kern/kern_time.c
+++ b/sys/kern/kern_time.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: kern_time.c,v 1.45 2004/07/28 17:15:12 tholo Exp $	*/
+/*	$OpenBSD: kern_time.c,v 1.46 2005/05/25 23:17:47 niklas Exp $	*/
 /*	$NetBSD: kern_time.c,v 1.20 1996/02/18 11:57:06 fvdl Exp $	*/
 
 /*
@@ -560,8 +560,12 @@ sys_setitimer(p, v, retval)
 		}
 		p->p_realtimer = aitv;
 	} else {
+		int s;
+
 		itimerround(&aitv.it_interval);
+		s = splclock();
 		p->p_stats->p_timer[SCARG(uap, which)] = aitv;
+		splx(s);
 	}
 
 	return (0);
diff --git a/sys/kern/sched_bsd.c b/sys/kern/sched_bsd.c
index 523e240a4b1..ff39ea3d1ad 100644
--- a/sys/kern/sched_bsd.c
+++ b/sys/kern/sched_bsd.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: sched_bsd.c,v 1.1 2004/07/29 06:25:45 tedu Exp $	*/
+/*	$OpenBSD: sched_bsd.c,v 1.2 2005/05/25 23:17:47 niklas Exp $	*/
 /*	$NetBSD: kern_synch.c,v 1.37 1996/04/22 01:38:37 christos Exp $	*/
 
 /*-
@@ -65,7 +65,9 @@ int	rrticks_init;		/* # of hardclock ticks per roundrobin() */
 int whichqs;			/* Bit mask summary of non-empty Q's. */
 struct prochd qs[NQS];
 
-struct SIMPLELOCK sched_lock;
+#ifdef MULTIPROCESSOR
+struct mutex sched_mutex = MUTEX_INITIALIZER(IPL_SCHED);
+#endif
 
 void scheduler_start(void);
 
@@ -260,7 +262,7 @@ schedcpu(arg)
 	struct timeout *to = (struct timeout *)arg;
 	fixpt_t loadfac = loadfactor(averunnable.ldavg[0]);
 	struct proc *p;
-	int s;
+	int s, t;
 	unsigned int newcpu;
 	int phz;
 
@@ -274,6 +276,7 @@ schedcpu(arg)
 	KASSERT(phz);
 
 	for (p = LIST_FIRST(&allproc); p != 0; p = LIST_NEXT(p, p_list)) {
+		SCHED_LOCK(s);
 		/*
 		 * Increment time in/out of memory and sleep time
 		 * (if sleeping).  We ignore overflow; with 16-bit int's
@@ -287,9 +290,11 @@ schedcpu(arg)
 		 * If the process has slept the entire second,
 		 * stop recalculating its priority until it wakes up.
 		 */
-		if (p->p_slptime > 1)
+		if (p->p_slptime > 1) {
+			SCHED_UNLOCK(s);
 			continue;
-		s = splstatclock();	/* prevent state changes */
+		}
+		t = splstatclock();
 		/*
 		 * p_pctcpu is only for ps.
 		 */
@@ -305,8 +310,7 @@ schedcpu(arg)
 		p->p_cpticks = 0;
 		newcpu = (u_int) decay_cpu(loadfac, p->p_estcpu);
 		p->p_estcpu = newcpu;
-		splx(s);
-		SCHED_LOCK(s);
+		splx(t);
 		resetpriority(p);
 		if (p->p_priority >= PUSER) {
 			if ((p != curproc) &&
@@ -355,13 +359,13 @@ updatepri(p)
 void
 sched_unlock_idle(void)
 {
-	SIMPLE_UNLOCK(&sched_lock);
+	mtx_leave(&sched_mutex);
 }
 
 void
 sched_lock_idle(void)
 {
-	SIMPLE_LOCK(&sched_lock);
+	mtx_enter(&sched_mutex);
 }
 #endif /* MULTIPROCESSOR || LOCKDEBUG */
 
@@ -379,9 +383,7 @@ yield()
 	p->p_priority = p->p_usrpri;
 	setrunqueue(p);
 	p->p_stats->p_ru.ru_nvcsw++;
-	mi_switch();
-	SCHED_ASSERT_UNLOCKED();
-	splx(s);
+	mi_switch(s);
 }
 
 /*
@@ -408,19 +410,17 @@ preempt(newp)
 	p->p_stat = SRUN;
 	setrunqueue(p);
 	p->p_stats->p_ru.ru_nivcsw++;
-	mi_switch();
-	SCHED_ASSERT_UNLOCKED();
-	splx(s);
+	mi_switch(s);
 }
 
 
 /*
- * Must be called at splstatclock() or higher.
+ * Must be called at splsched() or higher.
  */
 void
-mi_switch()
+mi_switch(int s)
 {
-	struct proc *p = curproc;	/* XXX */
+	struct proc *p = curproc;
 	struct rlimit *rlim;
 	struct timeval tv;
 #if defined(MULTIPROCESSOR)
@@ -432,20 +432,6 @@ mi_switch()
 
 	SCHED_ASSERT_LOCKED();
 
-#if defined(MULTIPROCESSOR)
-	/*
-	 * Release the kernel_lock, as we are about to yield the CPU.
-	 * The scheduler lock is still held until cpu_switch()
-	 * selects a new process and removes it from the run queue.
-	 */
-	if (p->p_flag & P_BIGLOCK)
-#ifdef notyet
-		hold_count = spinlock_release_all(&kernel_lock);
-#else
-		hold_count = __mp_release_all(&kernel_lock);
-#endif
-#endif
-
 	/*
 	 * Compute the amount of time during which the current
 	 * process was running, and add that to its total so far.
@@ -483,6 +469,7 @@ mi_switch()
 	 */
 	rlim = &p->p_rlimit[RLIMIT_CPU];
 	if ((rlim_t)p->p_rtime.tv_sec >= rlim->rlim_cur) {
+		SCHED_UNLOCK(s);
 		if ((rlim_t)p->p_rtime.tv_sec >= rlim->rlim_max) {
 			psignal(p, SIGKILL);
 		} else {
@@ -490,8 +477,23 @@ mi_switch()
 			if (rlim->rlim_cur < rlim->rlim_max)
 				rlim->rlim_cur += 5;
 		}
+		SCHED_LOCK(s);
 	}
 
+#if defined(MULTIPROCESSOR)
+	/*
+	 * Release the kernel_lock, as we are about to yield the CPU.
+	 * The scheduler lock is still held until cpu_switch()
+	 * selects a new process and removes it from the run queue.
+	 */
+	if (p->p_flag & P_BIGLOCK)
+#ifdef notyet
+		hold_count = spinlock_release_all(&kernel_lock);
+#else
+		hold_count = __mp_release_all(&kernel_lock);
+#endif
+#endif
+
 	/*
 	 * Process is about to yield the CPU; clear the appropriate
 	 * scheduling flags.
@@ -534,12 +536,9 @@ mi_switch()
 	 * we reacquire the interlock.
 	 */
 	if (p->p_flag & P_BIGLOCK)
-#ifdef notyet
-		spinlock_acquire_count(&kernel_lock, hold_count);
-#else
 		__mp_acquire_count(&kernel_lock, hold_count);
 #endif
-#endif
+	splx(s);
 }
 
 /*
@@ -553,7 +552,6 @@ rqinit()
 
 	for (i = 0; i < NQS; i++)
 		qs[i].ph_link = qs[i].ph_rlink = (struct proc *)&qs[i];
-	SIMPLE_LOCK_INIT(&sched_lock);
 }
 
 static __inline void
@@ -601,8 +599,7 @@ resched_proc(struct proc *p, u_char pri)
  * and awakening the swapper if it isn't in memory.
  */
 void
-setrunnable(p)
-	register struct proc *p;
+setrunnable(struct proc *p)
 {
 	SCHED_ASSERT_LOCKED();
 
@@ -634,7 +631,7 @@ setrunnable(p)
 		updatepri(p);
 	p->p_slptime = 0;
 	if ((p->p_flag & P_INMEM) == 0)
-		wakeup((caddr_t)&proc0);
+		sched_wakeup((caddr_t)&proc0);
 	else
 		resched_proc(p, p->p_priority);
 }
@@ -679,10 +676,10 @@ schedclock(p)
 {
 	int s;
 
-	p->p_estcpu = ESTCPULIM(p->p_estcpu + 1);
 	SCHED_LOCK(s);
+	p->p_estcpu = ESTCPULIM(p->p_estcpu + 1);
 	resetpriority(p);
-	SCHED_UNLOCK(s);
 	if (p->p_priority >= PUSER)
 		p->p_priority = p->p_usrpri;
+	SCHED_UNLOCK(s);
 }
diff --git a/sys/kern/vfs_sync.c b/sys/kern/vfs_sync.c
index e9113ca4ea1..549962182f6 100644
--- a/sys/kern/vfs_sync.c
+++ b/sys/kern/vfs_sync.c
@@ -1,4 +1,4 @@
-/*       $OpenBSD: vfs_sync.c,v 1.29 2004/10/29 11:51:49 pedro Exp $  */
+/*       $OpenBSD: vfs_sync.c,v 1.30 2005/05/25 23:17:47 niklas Exp $  */
 
 /*
  *  Portions of this code are:
@@ -50,6 +50,7 @@
 #include <sys/malloc.h>
 
 #include <sys/kernel.h>
+#include <sys/sched.h>
 
 #ifdef FFS_SOFTUPDATES
 int   softdep_process_worklist(struct mount *);
@@ -244,10 +245,10 @@ speedup_syncer()
 {
 	int s;
 
-	s = splhigh();
+	SCHED_LOCK(s);
 	if (syncerproc && syncerproc->p_wchan == &lbolt)
 		setrunnable(syncerproc);
-	splx(s);
+	SCHED_UNLOCK(s);
 	if (rushjob < syncdelay / 2) {
 		rushjob += 1;
 		stat_rush_requests += 1;
diff --git a/sys/miscfs/procfs/procfs_ctl.c b/sys/miscfs/procfs/procfs_ctl.c
index c38270f3c33..7a1c92b0bb3 100644
--- a/sys/miscfs/procfs/procfs_ctl.c
+++ b/sys/miscfs/procfs/procfs_ctl.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: procfs_ctl.c,v 1.13 2004/05/20 18:32:38 tedu Exp $	*/
+/*	$OpenBSD: procfs_ctl.c,v 1.14 2005/05/25 23:17:47 niklas Exp $	*/
 /*	$NetBSD: procfs_ctl.c,v 1.14 1996/02/09 22:40:48 christos Exp $	*/
 
 /*
@@ -48,6 +48,7 @@
 #include <sys/resourcevar.h>
 #include <sys/signalvar.h>
 #include <sys/ptrace.h>
+#include <sys/sched.h>
 #include <miscfs/procfs/procfs.h>
 
 /*
@@ -110,6 +111,7 @@ procfs_control(curp, p, op)
 	int op;
 {
 	int error;
+	int s;
 
 	/*
 	 * Attach - attaches the target process for debugging
@@ -248,8 +250,10 @@ procfs_control(curp, p, op)
 #endif
 	}
 
+	SCHED_LOCK(s);
 	if (p->p_stat == SSTOP)
 		setrunnable(p);
+	SCHED_UNLOCK(s);
 	return (0);
 }
 #endif
@@ -265,6 +269,7 @@ procfs_doctl(curp, p, pfs, uio)
 	int error;
 	char msg[PROCFS_CTLLEN+1];
 	const vfs_namemap_t *nm;
+	int s;
 
 	if (uio->uio_rw != UIO_WRITE)
 		return (EOPNOTSUPP);
@@ -297,7 +302,9 @@ procfs_doctl(curp, p, pfs, uio)
 			if (TRACE_WAIT_P(curp, p)) {
 				p->p_xstat = nm->nm_val;
 				FIX_SSTEP(p);
+				SCHED_LOCK(s);
 				setrunnable(p);
+				SCHED_UNLOCK(s);
 			} else {
 				psignal(p, nm->nm_val);
 			}
diff --git a/sys/sys/lock.h b/sys/sys/lock.h
index 5da1c1bfd73..31c1614ed0e 100644
--- a/sys/sys/lock.h
+++ b/sys/sys/lock.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: lock.h,v 1.12 2004/06/13 21:49:28 niklas Exp $	*/
+/*	$OpenBSD: lock.h,v 1.13 2005/05/25 23:17:47 niklas Exp $	*/
 
 /* 
  * Copyright (c) 1995
@@ -239,37 +239,4 @@ void	spinlock_acquire_count(__volatile struct lock *, int);
 #define LOCK_ASSERT(x)	/* nothing */
 #endif
 
-#if defined(MULTIPROCESSOR)
-/*
- * XXX Instead of using struct lock for the kernel lock and thus requiring us
- * XXX to implement simplelocks, causing all sorts of fine-grained locks all
- * XXX over our tree getting activated consuming both time and potentially
- * XXX introducing locking protocol bugs.
- */
-#ifdef notyet
-
-extern struct lock kernel_lock;
-
-/*
- * XXX Simplelock macros used at "trusted" places.
- */
-#define SIMPLELOCK		simplelock
-#define SIMPLE_LOCK_INIT	simple_lock_init
-#define SIMPLE_LOCK		simple_lock
-#define SIMPLE_UNLOCK		simple_unlock
-
-#endif
-
-#else
-
-/*
- * XXX Simplelock macros used at "trusted" places.
- */
-#define SIMPLELOCK		simplelock
-#define SIMPLE_LOCK_INIT	simple_lock_init
-#define SIMPLE_LOCK		simple_lock
-#define SIMPLE_UNLOCK		simple_unlock
-
-#endif
-
 #endif /* !_LOCK_H_ */
diff --git a/sys/sys/mplock.h b/sys/sys/mplock.h
index 247d3f06fa9..1df8d7ef5f4 100644
--- a/sys/sys/mplock.h
+++ b/sys/sys/mplock.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: mplock.h,v 1.4 2004/07/22 15:42:11 art Exp $	*/
+/*	$OpenBSD: mplock.h,v 1.5 2005/05/25 23:17:47 niklas Exp $	*/
 
 /*
  * Copyright (c) 2004 Niklas Hallqvist.  All rights reserved.
@@ -45,14 +45,6 @@ static __inline int __mp_release_all(struct __mp_lock *);
 static __inline void __mp_acquire_count(struct __mp_lock *, int);
 static __inline int __mp_lock_held(struct __mp_lock *);
 
-/*
- * XXX Simplelocks macros used at "trusted" places.
- */
-#define SIMPLELOCK		__mp_lock
-#define SIMPLE_LOCK_INIT	__mp_lock_init
-#define SIMPLE_LOCK		__mp_lock
-#define SIMPLE_UNLOCK		__mp_unlock
-
 static __inline void
 __mp_lock_init(struct __mp_lock *lock)
 {
diff --git a/sys/sys/proc.h b/sys/sys/proc.h
index d7a789b17e1..e22d922e262 100644
--- a/sys/sys/proc.h
+++ b/sys/sys/proc.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: proc.h,v 1.77 2005/03/10 17:26:10 tedu Exp $	*/
+/*	$OpenBSD: proc.h,v 1.78 2005/05/25 23:17:47 niklas Exp $	*/
 /*	$NetBSD: proc.h,v 1.44 1996/04/22 01:23:21 christos Exp $	*/
 
 /*-
@@ -408,7 +408,7 @@ int	inferior(struct proc *p);
 int	leavepgrp(struct proc *p);
 void	yield(void);
 void	preempt(struct proc *);
-void	mi_switch(void);
+void	mi_switch(int);
 void	pgdelete(struct pgrp *pgrp);
 void	procinit(void);
 #if !defined(remrunqueue)
@@ -424,6 +424,8 @@ int	ltsleep(void *chan, int pri, const char *wmesg, int timo,
 	    volatile struct simplelock *);
 #define tsleep(chan, pri, wmesg, timo) ltsleep(chan, pri, wmesg, timo, NULL)
 void	unsleep(struct proc *);
+void	sched_wakeup_n(void *, int);
+#define sched_wakeup(c) sched_wakeup_n((c), -1)
 void    wakeup_n(void *chan, int);
 void    wakeup(void *chan);
 #define wakeup_one(c) wakeup_n((c), 1)
diff --git a/sys/sys/sched.h b/sys/sys/sched.h
index 2bf36105626..6bac76e9e33 100644
--- a/sys/sys/sched.h
+++ b/sys/sys/sched.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: sched.h,v 1.10 2004/06/22 01:16:50 art Exp $	*/
+/*	$OpenBSD: sched.h,v 1.11 2005/05/25 23:17:47 niklas Exp $	*/
 /* $NetBSD: sched.h,v 1.2 1999/02/28 18:14:58 ross Exp $ */
 
 /*-
@@ -145,59 +145,31 @@ void roundrobin(struct cpu_info *);
 #define IPL_SCHED IPL_HIGH
 #endif
 
-#if defined(MULTIPROCESSOR) || defined(LOCKDEBUG)
-#include <sys/lock.h>
+#if defined(MULTIPROCESSOR)
+#include <sys/mutex.h>
 
-/*
- * XXX Instead of using struct lock for the kernel lock and thus requiring us
- * XXX to implement simplelocks, causing all sorts of fine-grained locks all
- * XXX over our tree getting activated consuming both time and potentially
- * XXX introducing locking protocol bugs.
- */
-#ifdef notyet
-
-extern struct simplelock sched_lock;
-
-#define	SCHED_ASSERT_LOCKED()	LOCK_ASSERT(simple_lock_held(&sched_lock))
-#define	SCHED_ASSERT_UNLOCKED()	LOCK_ASSERT(simple_lock_held(&sched_lock) == 0)
-
-#define	SCHED_LOCK(s)							\
-do {									\
-	s = splsched();							\
-	simple_lock(&sched_lock);					\
-} while (/* CONSTCOND */ 0)
-
-#define	SCHED_UNLOCK(s)							\
-do {									\
-	simple_unlock(&sched_lock);					\
-	splx(s);							\
-} while (/* CONSTCOND */ 0)
-
-#else
+extern struct mutex sched_mutex;
 
-extern struct __mp_lock sched_lock;
+#define	SCHED_ASSERT_LOCKED()	MUTEX_ASSERT_LOCKED(&sched_mutex)
+#define	SCHED_ASSERT_UNLOCKED()	MUTEX_ASSERT_UNLOCKED(&sched_mutex)
 
-#define	SCHED_ASSERT_LOCKED()	LOCK_ASSERT(__mp_lock_held(&sched_lock))
-#define	SCHED_ASSERT_UNLOCKED()	LOCK_ASSERT(__mp_lock_held(&sched_lock) == 0)
-
-#define	SCHED_LOCK(s)							\
-do {									\
-	s = splsched();							\
-	__mp_lock(&sched_lock);						\
-} while (/* CONSTCOND */ 0)
-
-#define	SCHED_UNLOCK(s)							\
-do {									\
-	__mp_unlock(&sched_lock);					\
-	splx(s);							\
-} while (/* CONSTCOND */ 0)
+/*
+ * We need this MUTEX_OLDIPL hack to be able to properly restore the old
+ * ipl after all the ipl juggling in cpu_switch.
+ */
+#define SCHED_LOCK(s) do {						\
+	mtx_enter(&sched_mutex);					\
+	s = MUTEX_OLDIPL(&sched_mutex);					\
+} while (0)
 
-#endif
+#define SCHED_UNLOCK(s) do {						\
+	mtx_leave(&sched_mutex);					\
+} while (0)
 
 void	sched_lock_idle(void);
 void	sched_unlock_idle(void);
 
-#else /* ! MULTIPROCESSOR || LOCKDEBUG */
+#else /* ! MULTIPROCESSOR */
 
 #define	SCHED_ASSERT_LOCKED()		splassert(IPL_SCHED);
 #define	SCHED_ASSERT_UNLOCKED()		/* nothing */
@@ -205,7 +177,7 @@ void	sched_unlock_idle(void);
 #define	SCHED_LOCK(s)			s = splsched()
 #define	SCHED_UNLOCK(s)			splx(s)
 
-#endif /* MULTIPROCESSOR || LOCKDEBUG */
+#endif /* MULTIPROCESSOR */
 
 #endif	/* _KERNEL */
 #endif	/* _SYS_SCHED_H_ */
diff --git a/sys/sys/signalvar.h b/sys/sys/signalvar.h
index 9db5681a21d..27bcd0b10c7 100644
--- a/sys/sys/signalvar.h
+++ b/sys/sys/signalvar.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: signalvar.h,v 1.12 2004/06/13 21:49:28 niklas Exp $	*/
+/*	$OpenBSD: signalvar.h,v 1.13 2005/05/25 23:17:47 niklas Exp $	*/
 /*	$NetBSD: signalvar.h,v 1.17 1996/04/22 01:23:31 christos Exp $	*/
 
 /*
@@ -159,9 +159,7 @@ void	csignal(pid_t pgid, int signum, uid_t uid, uid_t euid);
 int	issignal(struct proc *p);
 void	pgsignal(struct pgrp *pgrp, int sig, int checkctty);
 void	postsig(int sig);
-void	psignal1(struct proc *p, int sig, int dolock);
-#define	psignal(p, sig)		psignal1((p), (sig), 1)
-#define	sched_psignal(p, sig)	psignal1((p), (sig), 0)
+void	psignal(struct proc *p, int sig);
 void	siginit(struct proc *p);
 void	trapsignal(struct proc *p, int sig, u_long code, int type,
 	    union sigval val);
diff --git a/sys/uvm/uvm_glue.c b/sys/uvm/uvm_glue.c
index aeb129d7646..c2e331eb087 100644
--- a/sys/uvm/uvm_glue.c
+++ b/sys/uvm/uvm_glue.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: uvm_glue.c,v 1.39 2004/02/23 06:19:32 drahn Exp $	*/
+/*	$OpenBSD: uvm_glue.c,v 1.40 2005/05/25 23:17:47 niklas Exp $	*/
 /*	$NetBSD: uvm_glue.c,v 1.44 2001/02/06 19:54:44 eeh Exp $	*/
 
 /* 
@@ -80,6 +80,7 @@
 #ifdef SYSVSHM
 #include <sys/shm.h>
 #endif
+#include <sys/sched.h>
 
 #include <uvm/uvm.h>
 
@@ -380,13 +381,13 @@ uvm_swapin(p)
 	 * moved to new physical page(s) (e.g.  see mips/mips/vm_machdep.c).
 	 */
 	cpu_swapin(p);
-	s = splstatclock();
+	SCHED_LOCK(s);
 	if (p->p_stat == SRUN)
 		setrunqueue(p);
 	p->p_flag |= P_INMEM;
 	p->p_flag &= ~P_SWAPIN;
-	splx(s);
 	p->p_swtime = 0;
+	SCHED_UNLOCK(s);
 	++uvmexp.swapins;
 }
 
@@ -577,16 +578,16 @@ uvm_swapout(p)
 	/*
 	 * Mark it as (potentially) swapped out.
 	 */
-	s = splstatclock();
+	SCHED_LOCK(s);
 	if (!(p->p_flag & P_INMEM)) {
-		splx(s);
+		SCHED_UNLOCK(s);
 		return;
 	}
 	p->p_flag &= ~P_INMEM;
 	if (p->p_stat == SRUN)
 		remrunqueue(p);
-	splx(s);
 	p->p_swtime = 0;
+	SCHED_UNLOCK(s);
 	++uvmexp.swapouts;
 
 	/*