Rewrite m88k mutex code as a slight variation of the MI mutex code.

This will make mutex spinning time visible in top(1), and also might
improve stability.

The major change in this is that the old assembly code acquires
mutexes with an atomic exchange operation, but releases them with a
regular store, but the new code always uses atomic exchange
operations.

The mutex.h changes to the macros conform to <sys/mutex.h> to be able
to reset the system while in ddb.

Suggested from Miod Vallat, tested by me.  The stability in heavy load
is greatly improved in my case.

4 files changed, 181 insertions, 260 deletions

diff --git a/sys/arch/m88k/conf/files.m88k b/sys/arch/m88k/conf/files.m88k
index 1db60c9bc66..892fe7856d3 100644
--- a/sys/arch/m88k/conf/files.m88k
+++ b/sys/arch/m88k/conf/files.m88k
@@ -1,4 +1,4 @@
-#	$OpenBSD: files.m88k,v 1.30 2018/03/12 11:17:06 aoyama Exp $
+#	$OpenBSD: files.m88k,v 1.31 2020/05/26 11:55:10 aoyama Exp $
 
 file	arch/m88k/m88k/atomic.S				multiprocessor
 file	arch/m88k/m88k/db_disasm.c			ddb
@@ -14,7 +14,7 @@ file	arch/m88k/m88k/m88110_mmu.S			m88110
 file	arch/m88k/m88k/m8820x_machdep.c			m88100
 file	arch/m88k/m88k/m88k_machdep.c
 file	arch/m88k/m88k/mem.c
-file	arch/m88k/m88k/mutex.S
+file	arch/m88k/m88k/mutex.c
 file	arch/m88k/m88k/pmap.c
 file	arch/m88k/m88k/process.S
 file	arch/m88k/m88k/process_machdep.c
diff --git a/sys/arch/m88k/include/mutex.h b/sys/arch/m88k/include/mutex.h
index 2656398b590..5752a427f42 100644
--- a/sys/arch/m88k/include/mutex.h
+++ b/sys/arch/m88k/include/mutex.h
@@ -1,6 +1,6 @@
 #ifndef _M88K_MUTEX_H_
 #define _M88K_MUTEX_H_
-/*	$OpenBSD: mutex.h,v 1.7 2018/01/13 15:18:11 mpi Exp $	*/
+/*	$OpenBSD: mutex.h,v 1.8 2020/05/26 11:55:10 aoyama Exp $	*/
 
 /*
  * Copyright (c) 2005, Miodrag Vallat.
@@ -30,7 +30,7 @@
 #include <sys/_lock.h>
 
 struct mutex {
-	volatile int mtx_lock;	/* mutex.S relies upon this field being first */
+	volatile int mtx_lock;
 	int mtx_wantipl;
 	int mtx_oldipl;
 	volatile void *mtx_owner;
@@ -68,12 +68,12 @@ void __mtx_init(struct mutex *, int);
 #ifdef DIAGNOSTIC
 
 #define MUTEX_ASSERT_LOCKED(mtx) do {					\
-	if ((mtx)->mtx_owner != curcpu())				\
+	if (((mtx)->mtx_owner != curcpu()) && !(panicstr || db_active))	\
 		panic("mutex %p not held in %s", (mtx), __func__);	\
 } while (0)
 
 #define MUTEX_ASSERT_UNLOCKED(mtx) do {					\
-	if ((mtx)->mtx_owner == curcpu())				\
+	if (((mtx)->mtx_owner == curcpu()) && !(panicstr || db_active))	\
 		panic("mutex %p held in %s", (mtx), __func__);		\
 } while (0)
 
diff --git a/sys/arch/m88k/m88k/mutex.S b/sys/arch/m88k/m88k/mutex.S
deleted file mode 100644
index 090a8007e2d..00000000000
--- a/sys/arch/m88k/m88k/mutex.S
+++ /dev/null
@@ -1,254 +0,0 @@
-/*	$OpenBSD: mutex.S,v 1.17 2019/04/23 13:35:12 visa Exp $	*/
-
-/*
- * Copyright (c) 2005, Miodrag Vallat.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
- * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
- * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
- * DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT,
- * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
- * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
- * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
- * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
- * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- */
-
-#include "assym.h"
-#if MTX_LOCK != 0
-#error	Lack of Humppa in mutex code
-#endif
-
-#include <machine/asm.h>
-
-/*
- * void mtx_init(struct mutex *mtx, int ipl)
- */
-ENTRY(__mtx_init)
-	st	%r0, %r2, MTX_LOCK	/* mtx->mtx_lock = 0 */
-	st	%r0, %r2, MTX_OLDIPL	/* mtx->mtx_oldipl = IPL_NONE */
-	st	%r3, %r2, MTX_WANTIPL	/* mtx->mtx_wantipl = ipl */
-	jmp.n	%r1
-	 st	%r0, %r2, MTX_OWNER	/* mtx->mtx_owner = NULL */
-
-/*
- * void mtx_enter(struct mutex *mtx)
- */
-ENTRY(mtx_enter)
-	subu	%r31, %r31, 8
-	st	%r1,  %r31, 4		/* save return address */
-
-#ifdef MULTIPROCESSOR
-
-	st	%r2,  %r31, 0		/* save mtx */
-enter_again:
-	ld	%r2,  %r2,  MTX_WANTIPL
-	bcnd	eq0,  %r2,  1f
-	bsr	_C_LABEL(splraise)	/* splraise(mtx->mtx_wantipl) */
-1:
-	ld	%r4,  %r31, 0
-	or	%r3,  %r0,  1
-	xmem	%r3,  %r4,  %r0		/* attempt to claim the lock, old */
-	bcnd	ne0,  %r3,  enter_failed /* mtx->mtx_lock is 0 if successful */
-
-	ldcr	%r3,  CPU
-	st	%r2,  %r4,  MTX_OLDIPL	/* save into mtx_oldipl */
-	st	%r3,  %r4,  MTX_OWNER	/* mtx->mtx_owner = curcpu() */
-#ifdef DIAGNOSTIC
-	ld	%r2,  %r3,  CI_MUTEX_LEVEL
-	addu	%r2,  %r2,  1		/* curcpu()->ci_mutex_level++ */
-	st	%r2,  %r3,  CI_MUTEX_LEVEL
-#endif
-
-	ld	%r1,  %r31, 4
-	jmp.n	%r1
-	 addu	%r31, %r31, 8
-
-enter_failed:	/* the lock is not ours... */
-	ld	%r3,  %r4,  MTX_WANTIPL
-	bcnd	eq0,  %r3,  2f
-	bcnd	ne0,  %r2,  1f		/* splx(oldipl) */
-	bsr.n	_C_LABEL(spl0)
-	 addu	%r1,  %r1,  2f - . - 4
-1:
-	bsr	_C_LABEL(setipl)
-2:
-	ld	%r2,  %r31, 0		/* restore mtx */
-enter_spin:
-#ifdef DIAGNOSTIC
-	ldcr	%r3,  CPU
-	ld	%r4,  %r2,  MTX_OWNER
-	cmp	%r5,  %r3,  %r4
-	bcnd	eq0,  %r5,  enter_panic
-#endif
-	ld	%r3,  %r2,  MTX_LOCK
-	bcnd	eq0,  %r3,  enter_again
-	br	enter_spin
-
-#ifdef DIAGNOSTIC
-enter_panic:
-	or.u	%r2,  %r0,  %hi16(9f)
-	bsr.n	_C_LABEL(panic)
-	 or	%r2,  %r2,  %lo16(9f)
-
-	.data
-9:
-	.string	"mtx_enter: humpaan itsekseni"
-#endif
-
-#else	/* MULTIPROCESSOR */
-
-	st	%r2,  %r31, 0		/* save mtx */
-	ld	%r2,  %r2,  MTX_WANTIPL
-	bcnd	eq0,  %r2,  1f
-	bsr	_C_LABEL(splraise)	/* splraise(mtx->mtx_wantipl) */
-1:
-	ld	%r4,  %r31, 0
-	ldcr	%r3,  CPU
-	st	%r3,  %r4,  MTX_LOCK	/* locked! */
-
-	st	%r2,  %r4,  MTX_OLDIPL	/* save into mtx_oldipl */
-
-#ifdef DIAGNOSTIC
-	/* necessary for MUTEX_ASSERT_LOCKED */
-	st	%r3,  %r4,  MTX_OWNER	/* mtx->mtx_owner = curcpu() */
-
-	ld	%r2,  %r3,  CI_MUTEX_LEVEL
-	addu	%r2,  %r2,  1		/* curcpu()->ci_mutex_level++ */
-	st	%r2,  %r3,  CI_MUTEX_LEVEL
-#endif
-
-	ld	%r1,  %r31, 4
-	jmp.n	%r1
-	 addu	%r31, %r31, 8
-
-#endif	/* MULTIPROCESSOR */
-	
-/*
- * int mtx_enter_try(struct mutex *mtx)
- */
-ENTRY(mtx_enter_try)
-	subu	%r31, %r31, 8
-	st	%r1,  %r31, 4		/* save return address */
-
-#ifdef MULTIPROCESSOR
-
-	st	%r2,  %r31, 0		/* save mtx */
-enter_try_again:
-	ld	%r2,  %r2,  MTX_WANTIPL
-	bcnd	eq0,  %r2,  1f
-	bsr	_C_LABEL(splraise)	/* splraise(mtx->mtx_wantipl) */
-1:
-	ld	%r4,  %r31, 0
-	or	%r3,  %r0,  1
-	xmem	%r3,  %r4,  %r0		/* attempt to claim the lock, old */
-	bcnd	ne0,  %r3,  enter_try_failed /* mtx->mtx_lock is 0 if successful*/
-
-	ldcr	%r3,  CPU
-	st	%r2,  %r4,  MTX_OLDIPL	/* save into mtx_oldipl */
-	st	%r3,  %r4,  MTX_OWNER	/* mtx->mtx_owner = curcpu() */
-#ifdef DIAGNOSTIC
-	ld	%r2,  %r3,  CI_MUTEX_LEVEL
-	addu	%r2,  %r2,  1		/* curcpu()->ci_mutex_level++ */
-	st	%r2,  %r3,  CI_MUTEX_LEVEL
-#endif
-
-	ld	%r1,  %r31, 4
-	or	%r2,  %r0,  1		/* return nonzero */
-	jmp.n	%r1
-	 addu	%r31, %r31, 8
-
-enter_try_failed:	/* the lock is not ours... */
-	ld	%r3,  %r4,  MTX_WANTIPL
-	bcnd	eq0,  %r3,  2f
-	bcnd	ne0,  %r2,  1f		/* splx(oldipl) */
-	bsr.n	_C_LABEL(spl0)
-	 addu	%r1,  %r1,  2f - . - 4
-1:
-	bsr	_C_LABEL(setipl)
-2:
-#ifdef DIAGNOSTIC
-	ld	%r2,  %r31, 0		/* restore mtx */
-	ldcr	%r3,  CPU
-	ld	%r4,  %r2,  MTX_OWNER
-	cmp	%r5,  %r3,  %r4
-	bcnd	eq0,  %r5,  enter_try_panic
-#endif
-	or	%r2,  %r0,  %r0		/* return zero */
-	jmp.n	%r1
-	 addu	%r31, %r31, 8
-
-#ifdef DIAGNOSTIC
-enter_try_panic:
-	or.u	%r2,  %r0,  %hi16(9f)
-	bsr.n	_C_LABEL(panic)
-	 or	%r2,  %r2,  %lo16(9f)
-
-	.data
-9:
-	.string	"mtx_enter_try: humpaan itsekseni"
-#endif
-
-#else	/* MULTIPROCESSOR */
-
-	st	%r2,  %r31, 0		/* save mtx */
-	ld	%r2,  %r2,  MTX_WANTIPL
-	bcnd	eq0,  %r2,  1f
-	bsr	_C_LABEL(splraise)	/* splraise(mtx->mtx_wantipl) */
-1:
-	ld	%r4,  %r31, 0
-	ldcr	%r3,  CPU
-	st	%r3,  %r4,  MTX_LOCK	/* locked! */
-
-	st	%r2,  %r4,  MTX_OLDIPL	/* save into mtx_oldipl */
-
-#ifdef DIAGNOSTIC
-	/* necessary for MUTEX_ASSERT_LOCKED */
-	st	%r3,  %r4,  MTX_OWNER	/* mtx->mtx_owner = curcpu() */
-
-	ld	%r2,  %r3,  CI_MUTEX_LEVEL
-	addu	%r2,  %r2,  1		/* curcpu()->ci_mutex_level++ */
-	st	%r2,  %r3,  CI_MUTEX_LEVEL
-#endif
-
-	ld	%r1,  %r31, 4
-	or	%r2,  %r0,  1		/* return nonzero */
-	jmp.n	%r1
-	 addu	%r31, %r31, 8
-
-#endif	/* MULTIPROCESSOR */
-	
-/*
- * void mtx_leave(struct mutex *mtx)
- */
-ENTRY(mtx_leave)
-	ld	%r3,  %r2,  MTX_OLDIPL
-	ld	%r4,  %r2,  MTX_WANTIPL
-#ifdef DIAGNOSTIC
-	ld	%r5,  %r2,  MTX_OWNER
-	ld	%r6,  %r5,  CI_MUTEX_LEVEL
-	subu	%r6,  %r6,  1		/* curcpu()->ci_mutex_level++ */
-	st	%r6,  %r5,  CI_MUTEX_LEVEL
-#endif
-	st	%r0,  %r2,  MTX_OWNER		/* mtx->mtx_owner = NULL */
-	bcnd.n	eq0,  %r4,  2f
-	 st	%r0,  %r2,  MTX_LOCK		/* mtx->mtx_lock = 0 */
-
-	bcnd	ne0, %r3,  1f			/* splx(mtx->mtx_oldipl) */
-	br	_C_LABEL(spl0)
-1:
-	br.n	_C_LABEL(setipl)
-	 or	%r2,  %r3,  %r0
-2:
-	jmp	%r1
diff --git a/sys/arch/m88k/m88k/mutex.c b/sys/arch/m88k/m88k/mutex.c
new file mode 100644
index 00000000000..ff04d069900
--- /dev/null
+++ b/sys/arch/m88k/m88k/mutex.c
@@ -0,0 +1,175 @@
+/*	$OpenBSD: mutex.c,v 1.1 2020/05/26 11:55:10 aoyama Exp $	*/
+
+/*
+ * Copyright (c) 2020 Miodrag Vallat
+ * Copyright (c) 2017 Visa Hankala
+ * Copyright (c) 2014 David Gwynne <dlg@openbsd.org>
+ * Copyright (c) 2004 Artur Grabowski <art@openbsd.org>
+ *
+ * Permission to use, copy, modify, and distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/sched.h>
+#include <sys/atomic.h>
+#include <sys/witness.h>
+#include <sys/mutex.h>
+
+#include <ddb/db_output.h>
+
+#ifdef MP_LOCKDEBUG
+extern int __mp_lock_spinout;	/* kern_lock.c */
+#endif /* MP_LOCKDEBUG */
+
+static inline int
+atomic_swap(volatile int *lockptr, int new)
+{
+        int old = new;
+        asm volatile
+            ("xmem %0, %2, %%r0" : "+r"(old), "+m"(*lockptr) : "r"(lockptr));
+	return old;
+}
+
+void
+__mtx_init(struct mutex *mtx, int wantipl)
+{
+	mtx->mtx_lock = 0;
+	mtx->mtx_owner = NULL;
+	mtx->mtx_wantipl = wantipl;
+	mtx->mtx_oldipl = IPL_NONE;
+}
+
+#ifdef MULTIPROCESSOR
+void
+mtx_enter(struct mutex *mtx)
+{
+	struct schedstate_percpu *spc = &curcpu()->ci_schedstate;
+#ifdef MP_LOCKDEBUG
+	int nticks = __mp_lock_spinout;
+#endif
+
+	WITNESS_CHECKORDER(MUTEX_LOCK_OBJECT(mtx),
+	    LOP_EXCLUSIVE | LOP_NEWORDER, NULL);
+
+	spc->spc_spinning++;
+	while (mtx_enter_try(mtx) == 0) {
+		CPU_BUSY_CYCLE();
+
+#ifdef MP_LOCKDEBUG
+		if (--nticks == 0) {
+			db_printf("%s: %p lock spun out\n", __func__, mtx);
+			db_enter();
+			nticks = __mp_lock_spinout;
+		}
+#endif
+	}
+	spc->spc_spinning--;
+}
+
+int
+mtx_enter_try(struct mutex *mtx)
+{
+	struct cpu_info *ci = curcpu();
+	int s;
+
+	/* Avoid deadlocks after panic or in DDB */
+	if (panicstr || db_active)
+		return (1);
+
+	if (mtx->mtx_wantipl != IPL_NONE)
+		s = splraise(mtx->mtx_wantipl);
+
+	if (atomic_swap(&mtx->mtx_lock, 1) == 0) {
+		mtx->mtx_owner = ci;
+		membar_enter_after_atomic();
+		if (mtx->mtx_wantipl != IPL_NONE)
+			mtx->mtx_oldipl = s;
+#ifdef DIAGNOSTIC
+		ci->ci_mutex_level++;
+#endif
+		WITNESS_LOCK(MUTEX_LOCK_OBJECT(mtx), LOP_EXCLUSIVE);
+		return (1);
+	}
+
+#ifdef DIAGNOSTIC
+	if (__predict_false(mtx->mtx_owner == ci))
+		panic("mtx %p: locking against myself", mtx);
+#endif
+	if (mtx->mtx_wantipl != IPL_NONE)
+		splx(s);
+
+	return (0);
+}
+#else
+void
+mtx_enter(struct mutex *mtx)
+{
+	struct cpu_info *ci = curcpu();
+
+	/* Avoid deadlocks after panic or in DDB */
+	if (panicstr || db_active)
+		return;
+
+	WITNESS_CHECKORDER(MUTEX_LOCK_OBJECT(mtx),
+	    LOP_EXCLUSIVE | LOP_NEWORDER, NULL);
+
+#ifdef DIAGNOSTIC
+	if (__predict_false(mtx->mtx_owner == ci))
+		panic("mtx %p: locking against myself", mtx);
+#endif
+
+	if (mtx->mtx_wantipl != IPL_NONE)
+		mtx->mtx_oldipl = splraise(mtx->mtx_wantipl);
+
+	mtx->mtx_owner = ci;
+
+#ifdef DIAGNOSTIC
+	ci->ci_mutex_level++;
+#endif
+	WITNESS_LOCK(MUTEX_LOCK_OBJECT(mtx), LOP_EXCLUSIVE);
+}
+
+int
+mtx_enter_try(struct mutex *mtx)
+{
+	mtx_enter(mtx);
+	return (1);
+}
+#endif
+
+void
+mtx_leave(struct mutex *mtx)
+{
+	int s;
+
+	/* Avoid deadlocks after panic or in DDB */
+	if (panicstr || db_active)
+		return;
+
+	MUTEX_ASSERT_LOCKED(mtx);
+	WITNESS_UNLOCK(MUTEX_LOCK_OBJECT(mtx), LOP_EXCLUSIVE);
+
+#ifdef DIAGNOSTIC
+	curcpu()->ci_mutex_level--;
+#endif
+
+	s = mtx->mtx_oldipl;
+#ifdef MULTIPROCESSOR
+	membar_exit();
+#endif
+	mtx->mtx_owner = NULL;
+	(void)atomic_swap(&mtx->mtx_lock, 0);
+	if (mtx->mtx_wantipl != IPL_NONE)
+		splx(s);
+}