Add a machine-independent implementation for the mplock.

This reduces code duplication and makes it easier to instrument
lock primitives.

The MI mplock uses the ticket lock code that has been in use
on amd64, i386 and sparc64. These are the architectures that now
switch to the MI code.

The lock_machdep.c files are unhooked from the build but not
removed yet, in case something goes wrong.

OK mpi@, kettenis@

1 files changed, 173 insertions, 9 deletions

diff --git a/sys/kern/kern_lock.c b/sys/kern/kern_lock.c
index 3bda382d569..3b6adb16c93 100644
--- a/sys/kern/kern_lock.c
+++ b/sys/kern/kern_lock.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: kern_lock.c,v 1.50 2017/10/09 08:16:13 mpi Exp $	*/
+/*	$OpenBSD: kern_lock.c,v 1.51 2017/10/17 14:25:35 visa Exp $	*/
 
 /* 
  * Copyright (c) 1995
@@ -36,24 +36,33 @@
  */
 
 #include <sys/param.h>
-#include <sys/lock.h>
 #include <sys/systm.h>
 #include <sys/sched.h>
+#include <sys/atomic.h>
 #include <sys/witness.h>
 
+#include <ddb/db_output.h>
+
 #ifdef MP_LOCKDEBUG
+#ifndef DDB
+#error "MP_LOCKDEBUG requires DDB"
+#endif
+
 /* CPU-dependent timing, this needs to be settable from ddb. */
 int __mp_lock_spinout = 200000000;
+#endif /* MP_LOCKDEBUG */
+
+#if defined(MULTIPROCESSOR) || defined(WITNESS)
+struct __mp_lock kernel_lock;
 #endif
 
-#if defined(MULTIPROCESSOR)
+#ifdef MULTIPROCESSOR
+
 /*
  * Functions for manipulating the kernel_lock.  We put them here
  * so that they show up in profiles.
  */
 
-struct __mp_lock kernel_lock;
-
 void
 _kernel_lock_init(void)
 {
@@ -90,10 +99,35 @@ _kernel_lock_held(void)
 	return (__mp_lock_held(&kernel_lock));
 }
 
-#ifdef WITNESS
+#ifdef __USE_MI_MPLOCK
+
+/* Ticket lock implementation */
+/*
+ * Copyright (c) 2014 David Gwynne <dlg@openbsd.org>
+ *
+ * Permission to use, copy, modify, and distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#include <machine/cpu.h>
+
 void
-_mp_lock_init(struct __mp_lock *mpl, struct lock_type *type)
+___mp_lock_init(struct __mp_lock *mpl, struct lock_type *type)
 {
+	memset(mpl->mpl_cpus, 0, sizeof(mpl->mpl_cpus));
+	mpl->mpl_users = 0;
+	mpl->mpl_ticket = 1;
+
+#ifdef WITNESS
 	mpl->mpl_lock_obj.lo_name = type->lt_name;
 	mpl->mpl_lock_obj.lo_type = type;
 	if (mpl == &kernel_lock)
@@ -103,9 +137,139 @@ _mp_lock_init(struct __mp_lock *mpl, struct lock_type *type)
 		mpl->mpl_lock_obj.lo_flags = LO_WITNESS | LO_INITIALIZED |
 		    LO_RECURSABLE | (LO_CLASS_SCHED_LOCK << LO_CLASSSHIFT);
 	WITNESS_INIT(&mpl->mpl_lock_obj, type);
+#endif
+}
+
+static __inline void
+__mp_lock_spin(struct __mp_lock *mpl, u_int me)
+{
+#ifndef MP_LOCKDEBUG
+	while (mpl->mpl_ticket != me)
+		CPU_BUSY_CYCLE();
+#else
+	int nticks = __mp_lock_spinout;
+
+	while (mpl->mpl_ticket != me) {
+		CPU_BUSY_CYCLE();
+
+		if (--nticks <= 0) {
+			db_printf("__mp_lock(%p): lock spun out", mpl);
+			db_enter();
+			nticks = __mp_lock_spinout;
+		}
+	}
+#endif
+}
+
+void
+___mp_lock(struct __mp_lock *mpl LOCK_FL_VARS)
+{
+	struct __mp_lock_cpu *cpu = &mpl->mpl_cpus[cpu_number()];
+	unsigned long s;
+
+#ifdef WITNESS
+	if (!__mp_lock_held(mpl))
+		WITNESS_CHECKORDER(&mpl->mpl_lock_obj,
+		    LOP_EXCLUSIVE | LOP_NEWORDER, file, line, NULL);
+#endif
+
+	s = intr_disable();
+	if (cpu->mplc_depth++ == 0)
+		cpu->mplc_ticket = atomic_inc_int_nv(&mpl->mpl_users);
+	intr_restore(s);
+
+	__mp_lock_spin(mpl, cpu->mplc_ticket);
+	membar_enter_after_atomic();
+
+	WITNESS_LOCK(&mpl->mpl_lock_obj, LOP_EXCLUSIVE, file, line);
+}
+
+void
+___mp_unlock(struct __mp_lock *mpl LOCK_FL_VARS)
+{
+	struct __mp_lock_cpu *cpu = &mpl->mpl_cpus[cpu_number()];
+	unsigned long s;
+
+#ifdef MP_LOCKDEBUG
+	if (!__mp_lock_held(mpl)) {
+		db_printf("__mp_unlock(%p): not held lock\n", mpl);
+		db_enter();
+	}
+#endif
+
+	WITNESS_UNLOCK(&mpl->mpl_lock_obj, LOP_EXCLUSIVE, file, line);
+
+	s = intr_disable();
+	if (--cpu->mplc_depth == 0) {
+		membar_exit();
+		mpl->mpl_ticket++;
+	}
+	intr_restore(s);
+}
+
+int
+___mp_release_all(struct __mp_lock *mpl LOCK_FL_VARS)
+{
+	struct __mp_lock_cpu *cpu = &mpl->mpl_cpus[cpu_number()];
+	unsigned long s;
+	int rv;
+#ifdef WITNESS
+	int i;
+#endif
+
+	s = intr_disable();
+	rv = cpu->mplc_depth;
+#ifdef WITNESS
+	for (i = 0; i < rv; i++)
+		WITNESS_UNLOCK(&mpl->mpl_lock_obj, LOP_EXCLUSIVE, file, line);
+#endif
+	cpu->mplc_depth = 0;
+	membar_exit();
+	mpl->mpl_ticket++;
+	intr_restore(s);
+
+	return (rv);
+}
+
+int
+___mp_release_all_but_one(struct __mp_lock *mpl LOCK_FL_VARS)
+{
+	struct __mp_lock_cpu *cpu = &mpl->mpl_cpus[cpu_number()];
+	int rv = cpu->mplc_depth - 1;
+#ifdef WITNESS
+	int i;
+
+	for (i = 0; i < rv; i++)
+		WITNESS_UNLOCK(&mpl->mpl_lock_obj, LOP_EXCLUSIVE, file, line);
+#endif
 
-	___mp_lock_init(mpl);
+#ifdef MP_LOCKDEBUG
+	if (!__mp_lock_held(mpl)) {
+		db_printf("__mp_release_all_but_one(%p): not held lock\n", mpl);
+		db_enter();
+	}
+#endif
+
+	cpu->mplc_depth = 1;
+
+	return (rv);
 }
-#endif /* WITNESS */
+
+void
+___mp_acquire_count(struct __mp_lock *mpl, int count LOCK_FL_VARS)
+{
+	while (count--)
+		___mp_lock(mpl LOCK_FL_ARGS);
+}
+
+int
+__mp_lock_held(struct __mp_lock *mpl)
+{
+	struct __mp_lock_cpu *cpu = &mpl->mpl_cpus[cpu_number()];
+
+	return (cpu->mplc_ticket == mpl->mpl_ticket && cpu->mplc_depth > 0);
+}
+
+#endif /* __USE_MI_MPLOCK */
 
 #endif /* MULTIPROCESSOR */