Like i386 and amd64 - make the __mp_lock not spin at splhigh.

help from & ok art@

4 files changed, 201 insertions, 160 deletions

diff --git a/sys/arch/sparc64/conf/files.sparc64 b/sys/arch/sparc64/conf/files.sparc64
index 4b51481f49b..15135a6365e 100644
--- a/sys/arch/sparc64/conf/files.sparc64
+++ b/sys/arch/sparc64/conf/files.sparc64
@@ -1,4 +1,4 @@
-#	$OpenBSD: files.sparc64,v 1.91 2007/10/20 18:54:42 kettenis Exp $
+#	$OpenBSD: files.sparc64,v 1.92 2007/11/27 23:29:57 kettenis Exp $
 #	$NetBSD: files.sparc64,v 1.50 2001/08/10 20:53:50 eeh Exp $
 
 # maxpartitions must be first item in files.${ARCH}
@@ -243,6 +243,7 @@ file	arch/sparc64/sparc64/ipifuncs.c		multiprocessor
 file	arch/sparc64/sparc64/kgdb_machdep.c	kgdb
 # sparc64/sparc64/locore.s is handled specially in the makefile,
 # because it must come first in the "ld" command line.
+file	arch/sparc64/sparc64/lock_machdep.c	multiprocessor
 file	arch/sparc64/sparc64/machdep.c
 file	arch/sparc64/sparc64/mem.c
 file	arch/sparc64/sparc64/mutex.S
diff --git a/sys/arch/sparc64/include/atomic.h b/sys/arch/sparc64/include/atomic.h
index 519b5babefc..f0a06e01693 100644
--- a/sys/arch/sparc64/include/atomic.h
+++ b/sys/arch/sparc64/include/atomic.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: atomic.h,v 1.3 2007/03/13 08:34:03 art Exp $	*/
+/*	$OpenBSD: atomic.h,v 1.4 2007/11/27 23:29:57 kettenis Exp $	*/
 /*
  * Copyright (c) 2007 Artur Grabowski <art@openbsd.org>
  *
@@ -21,7 +21,7 @@
 #if defined(_KERNEL)
 
 static __inline unsigned int
-sparc64_casa(volatile unsigned int *uip, unsigned int expect, unsigned int new)
+sparc64_cas(volatile unsigned int *uip, unsigned int expect, unsigned int new)
 {
 	__asm __volatile("casa [%2] %3, %4, %0"
 	    : "+r" (new), "=m" (*uip)
@@ -30,6 +30,17 @@ sparc64_casa(volatile unsigned int *uip, unsigned int expect, unsigned int new)
 	return (new);
 }
 
+static __inline unsigned long
+sparc64_casx(volatile unsigned long *uip, unsigned long expect,
+    unsigned long new)
+{
+	__asm __volatile("casxa [%2] %3, %4, %0"
+	    : "+r" (new), "=m" (*uip)
+	    : "r" (uip), "n" (ASI_N), "r" (expect), "m" (*uip));
+
+	return (new);
+}
+
 static __inline void
 atomic_setbits_int(volatile unsigned int *uip, unsigned int v)
 {
@@ -38,7 +49,7 @@ atomic_setbits_int(volatile unsigned int *uip, unsigned int v)
 	r = *uip;
 	do {
 		e = r;
-		r = sparc64_casa(uip, e, e | v);
+		r = sparc64_cas(uip, e, e | v);
 	} while (r != e);
 }
 
@@ -50,7 +61,7 @@ atomic_clearbits_int(volatile unsigned int *uip, unsigned int v)
 	r = *uip;
 	do {
 		e = r;
-		r = sparc64_casa(uip, e, e & ~v);
+		r = sparc64_cas(uip, e, e & ~v);
 	} while (r != e);
 }
 
diff --git a/sys/arch/sparc64/include/mplock.h b/sys/arch/sparc64/include/mplock.h
index 8f87c895aae..aa133e36686 100644
--- a/sys/arch/sparc64/include/mplock.h
+++ b/sys/arch/sparc64/include/mplock.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: mplock.h,v 1.1 2007/11/26 17:15:29 art Exp $	*/
+/*	$OpenBSD: mplock.h,v 1.2 2007/11/27 23:29:57 kettenis Exp $	*/
 
 /*
  * Copyright (c) 2004 Niklas Hallqvist.  All rights reserved.
@@ -27,165 +27,21 @@
 #ifndef _MACHINE_MPLOCK_H_
 #define _MACHINE_MPLOCK_H_
 
-/*
- * Really simple spinlock implementation with recursive capabilities.
- * Correctness is paramount, no fancyness allowed.
- */
-
 struct __mp_lock {
-	__cpu_simple_lock_t mpl_lock;
-	cpuid_t	mpl_cpu;
-	int	mpl_count;
+	volatile struct cpu_info *mpl_cpu;
+	volatile long		mpl_count;
 };
 
-static __inline void
-__mp_lock_init(struct __mp_lock *lock)
-{
-	__cpu_simple_lock_init(&lock->mpl_lock);
-	lock->mpl_cpu = LK_NOCPU;
-	lock->mpl_count = 0;
-}
-
-#if defined(MP_LOCKDEBUG)
-#ifndef DDB
-#error "MP_LOCKDEBUG requires DDB"
-#endif
-
-extern void Debugger(void);
-extern int db_printf(const char *, ...)
-    __attribute__((__format__(__kprintf__,1,2)));
-
-/* CPU-dependent timing, needs this to be settable from ddb. */
-extern int __mp_lock_spinout;
-#endif
-
-static __inline void
-__mp_lock(struct __mp_lock *lock)
-{
-	int s = spllock();
-
-	if (lock->mpl_cpu != cpu_number()) {
-#ifndef MP_LOCKDEBUG
-		__cpu_simple_lock(&lock->mpl_lock);
-#else
-		{
-			int got_it;
-			do {
-				int ticks = __mp_lock_spinout;
-
-				do {
-					got_it = __cpu_simple_lock_try(
-					    &lock->mpl_lock);
-				} while (!got_it && ticks-- > 0);
-				if (!got_it) {
- 					db_printf(
-					    "__mp_lock(0x%x): lock spun out",
-					    lock);
-					Debugger();
-				}
-			} while (!got_it);
-		}
-#endif
-		lock->mpl_cpu = cpu_number();
-	}
-	lock->mpl_count++;
-	splx(s);
-}
-
-/*
- * Try to acquire the lock, if another cpu has it, fill it in the
- * call-by-reference cpu parameter.  Return true if acquired.
- */
-static __inline int
-__mp_lock_try(struct __mp_lock *lock, cpuid_t *cpu)
-{
-	int s = spllock();
-
-	if (lock->mpl_cpu != cpu_number()) {
-		if (!__cpu_simple_lock_try(&lock->mpl_lock)) {
-			*cpu = lock->mpl_cpu;
-			splx(s);
-			return 0;
-		}
-		lock->mpl_cpu = cpu_number();
-	}
-	lock->mpl_count++;
-	splx(s);
-	return 1;
-}
-
-static __inline void
-__mp_unlock(struct __mp_lock *lock)
-{
-	int s = spllock();
-
-#ifdef MP_LOCKDEBUG
-	if (lock->mpl_count == 0 || lock->mpl_cpu == LK_NOCPU) {
-		db_printf("__mp_unlock(0x%x): releasing not locked lock\n",
-		    lock);
-		Debugger();
-	}
-#endif
-
-	if (--lock->mpl_count == 0) {
-		lock->mpl_cpu = LK_NOCPU;
-		__cpu_simple_unlock(&lock->mpl_lock);
-	}
-	splx(s);
-}
-
-static __inline int
-__mp_release_all(struct __mp_lock *lock) {
-	int s = spllock();
-	int rv = lock->mpl_count;
+#ifndef _LOCORE
 
-#ifdef MP_LOCKDEBUG
-	if (lock->mpl_count == 0 || lock->mpl_cpu == LK_NOCPU) {
-		db_printf(
-		    "__mp_release_all(0x%x): releasing not locked lock\n",
-		    lock);
-		Debugger();
-	}
-#endif
-
-	lock->mpl_cpu = LK_NOCPU;
-	lock->mpl_count = 0;
-	__cpu_simple_unlock(&lock->mpl_lock);
-	splx(s);
-	return (rv);
-}
+void __mp_lock_init(struct __mp_lock *);
+void __mp_lock(struct __mp_lock *);
+void __mp_unlock(struct __mp_lock *);
+int __mp_release_all(struct __mp_lock *);
+int __mp_release_all_but_one(struct __mp_lock *);
+void __mp_acquire_count(struct __mp_lock *, int);
+int __mp_lock_held(struct __mp_lock *);
 
-static __inline int
-__mp_release_all_but_one(struct __mp_lock *lock) {
-	int s = spllock();
-	int rv = lock->mpl_count - 1;
-
-#ifdef MP_LOCKDEBUG
-	if (lock->mpl_count == 0 || lock->mpl_cpu == LK_NOCPU) {
-		db_printf(
-		    "__mp_release_all_but_one(0x%x): releasing not locked lock\n",
-		    lock);
-		Debugger();
-	}
 #endif
 
-	lock->mpl_count = 1;
-	splx(s);
-	return (rv);
-}
-
-static __inline void
-__mp_acquire_count(struct __mp_lock *lock, int count) {
-	int s = spllock();
-
-	while (count--)
-		__mp_lock(lock);
-	splx(s);
-}
-
-static __inline int
-__mp_lock_held(struct __mp_lock *lock) {
-	return lock->mpl_count && lock->mpl_cpu == cpu_number();
-}
-
 #endif /* !_MACHINE_MPLOCK_H */
diff --git a/sys/arch/sparc64/sparc64/lock_machdep.c b/sys/arch/sparc64/sparc64/lock_machdep.c
new file mode 100644
index 00000000000..7577eda10c1
--- /dev/null
+++ b/sys/arch/sparc64/sparc64/lock_machdep.c
@@ -0,0 +1,173 @@
+/*	$OpenBSD: lock_machdep.c,v 1.1 2007/11/27 23:29:57 kettenis Exp $	*/
+
+/*
+ * Copyright (c) 2007 Artur Grabowski <art@openbsd.org>
+ *
+ * Permission to use, copy, modify, and distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+
+#include <sys/param.h>
+#include <sys/lock.h>
+#include <sys/systm.h>
+
+#include <machine/atomic.h>
+#include <machine/lock.h>
+#include <machine/psl.h>
+
+#include <ddb/db_output.h>
+
+void
+__mp_lock_init(struct __mp_lock *lock)
+{
+	lock->mpl_cpu = NULL;
+	lock->mpl_count = 0;
+}
+
+#if defined(MP_LOCKDEBUG)
+#ifndef DDB
+#error "MP_LOCKDEBUG requires DDB"
+#endif
+
+/* CPU-dependent timing, needs this to be settable from ddb. */
+extern int __mp_lock_spinout;
+#endif
+
+#define SPINLOCK_SPIN_HOOK	/**/
+
+static __inline void
+__mp_lock_spin(struct __mp_lock *mpl)
+{
+#ifndef MP_LOCKDEBUG
+	while (mpl->mpl_count != 0)
+		SPINLOCK_SPIN_HOOK;
+#else
+	int ticks = __mp_lock_spinout;
+
+	while (mpl->mpl_count != 0 && ticks-- > 0)
+		SPINLOCK_SPIN_HOOK;
+
+	if (ticks == 0) {
+		db_printf("__mp_lock(0x%x): lock spun out", mpl);
+		Debugger();
+	}
+#endif
+}
+
+void
+__mp_lock(struct __mp_lock *mpl)
+{
+	/*
+	 * Please notice that mpl_count gets incremented twice for the
+	 * first lock. This is on purpose. The way we release the lock
+	 * in mp_unlock is to decrement the mpl_count and then check if
+	 * the lock should be released. Since mpl_count is what we're
+	 * spinning on, decrementing it in mpl_unlock to 0 means that
+	 * we can't clear mpl_cpu, because we're no longer holding the
+	 * lock. In theory mpl_cpu doesn't need to be cleared, but it's
+	 * safer to clear it and besides, setting mpl_count to 2 on the
+	 * first lock makes most of this code much simpler.
+	 */
+
+	while (1) {
+		u_int64_t s;
+
+		s = intr_disable();
+		if (sparc64_casx(&mpl->mpl_count, 0, 1) == 0) {
+			sparc_membar(LoadLoad | LoadStore);
+			mpl->mpl_cpu = curcpu();
+		}
+
+		if (mpl->mpl_cpu == curcpu()) {
+			mpl->mpl_count++;
+			intr_restore(s);
+			break;
+		}
+		intr_restore(s);
+
+		__mp_lock_spin(mpl);
+	}
+}
+
+void
+__mp_unlock(struct __mp_lock *mpl)
+{
+	u_int64_t s;
+
+#ifdef MP_LOCKDEBUG
+	if (mpl->mpl_cpu != curcpu()) {
+		db_printf("__mp_unlock(%p): not held lock\n", mpl);
+		Debugger();
+	}
+#endif
+
+	s = intr_disable();
+	if (--mpl->mpl_count == 1) {
+		mpl->mpl_cpu = NULL;
+		sparc_membar(StoreStore | LoadStore);
+		mpl->mpl_count = 0;
+	}
+	intr_restore(s);
+}
+
+int
+__mp_release_all(struct __mp_lock *mpl)
+{
+	int rv = mpl->mpl_count - 1;
+	u_int64_t s;
+
+#ifdef MP_LOCKDEBUG
+	if (mpl->mpl_cpu != curcpu()) {
+		db_printf("__mp_release_all(%p): not held lock\n", mpl);
+		Debugger();
+	}
+#endif
+
+	s = intr_disable();
+	mpl->mpl_cpu = NULL;
+	sparc_membar(StoreStore | LoadStore);
+	mpl->mpl_count = 0;
+	intr_restore(s);
+
+	return (rv);
+}
+
+int
+__mp_release_all_but_one(struct __mp_lock *mpl)
+{
+	int rv = mpl->mpl_count - 2;
+
+#ifdef MP_LOCKDEBUG
+	if (mpl->mpl_cpu != curcpu()) {
+		db_printf("__mp_release_all_but_one(%p): not held lock\n", mpl);
+		Debugger();
+	}
+#endif
+
+	mpl->mpl_count = 2;
+
+	return (rv);
+}
+
+void
+__mp_acquire_count(struct __mp_lock *mpl, int count)
+{
+	while (count--)
+		__mp_lock(mpl);
+}
+
+int
+__mp_lock_held(struct __mp_lock *mpl)
+{
+	return mpl->mpl_cpu == curcpu();
+}