atomic_{set,clear}bits_int were not safe enough on 88110 systems, as they

can be interrupted by NMI; move the SMP version of these routines from
inlines to a separate file (kernel text shrinks 20KB...).

Since the implementation for 88110 becomes really hairy, the pre-main() code
is responsible for copying the appropriate code over for kernels configured
for both 88100 and 88110 cpus, to avoid having to choose the atomicity
strategy at runtime. Hairy, I said.

This gets GENERIC.MP run much further on 197DP. Not enough to reach multiuser
mode, but boots up to starting sshd and then panics.

4 files changed, 224 insertions, 30 deletions

diff --git a/sys/arch/m88k/conf/files.m88k b/sys/arch/m88k/conf/files.m88k
index eeb094edbd4..780cb3a40da 100644
--- a/sys/arch/m88k/conf/files.m88k
+++ b/sys/arch/m88k/conf/files.m88k
@@ -1,5 +1,6 @@
-#	$OpenBSD: files.m88k,v 1.21 2008/04/25 14:51:35 jmc Exp $
+#	$OpenBSD: files.m88k,v 1.22 2009/02/20 20:40:00 miod Exp $
 
+file	arch/m88k/m88k/atomic.S				multiprocessor
 file	arch/m88k/m88k/db_disasm.c			ddb
 file	arch/m88k/m88k/db_interface.c			ddb
 file	arch/m88k/m88k/db_sstep.c			ddb
diff --git a/sys/arch/m88k/include/atomic.h b/sys/arch/m88k/include/atomic.h
index 874764cd729..6dba1393f5a 100644
--- a/sys/arch/m88k/include/atomic.h
+++ b/sys/arch/m88k/include/atomic.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: atomic.h,v 1.5 2007/12/05 22:09:13 miod Exp $	*/
+/*	$OpenBSD: atomic.h,v 1.6 2009/02/20 20:40:01 miod Exp $	*/
 
 /* Public Domain */
 
@@ -7,14 +7,17 @@
 
 #if defined(_KERNEL)
 
+#ifdef MULTIPROCESSOR
+
+/* actual implementation is hairy, see atomic.S */
+void	atomic_setbits_int(__volatile unsigned int *, unsigned int);
+void	atomic_clearbits_int(__volatile unsigned int *, unsigned int);
+
+#else
+
 #include <machine/asm_macro.h>
-#include <machine/lock.h>
 #include <machine/psl.h>
 
-#ifdef MULTIPROCESSOR
-extern __cpu_simple_lock_t __atomic_lock;
-#endif
-
 static __inline void
 atomic_setbits_int(__volatile unsigned int *uip, unsigned int v)
 {
@@ -22,13 +25,7 @@ atomic_setbits_int(__volatile unsigned int *uip, unsigned int v)
 
 	psr = get_psr();
 	set_psr(psr | PSR_IND);
-#ifdef MULTIPROCESSOR
-	__cpu_simple_lock(&__atomic_lock);
-#endif
 	*uip |= v;
-#ifdef MULTIPROCESSOR
-	__cpu_simple_unlock(&__atomic_lock);
-#endif
 	set_psr(psr);
 }
 
@@ -39,15 +36,11 @@ atomic_clearbits_int(__volatile unsigned int *uip, unsigned int v)
 
 	psr = get_psr();
 	set_psr(psr | PSR_IND);
-#ifdef MULTIPROCESSOR
-	__cpu_simple_lock(&__atomic_lock);
-#endif
 	*uip &= ~v;
-#ifdef MULTIPROCESSOR
-	__cpu_simple_unlock(&__atomic_lock);
-#endif
 	set_psr(psr);
 }
 
+#endif	/* MULTIPROCESSOR */
+
 #endif /* defined(_KERNEL) */
 #endif /* __M88K_ATOMIC_H__ */
diff --git a/sys/arch/m88k/m88k/atomic.S b/sys/arch/m88k/m88k/atomic.S
new file mode 100644
index 00000000000..4832fac314a
--- /dev/null
+++ b/sys/arch/m88k/m88k/atomic.S
@@ -0,0 +1,168 @@
+/*	$OpenBSD: atomic.S,v 1.1 2009/02/20 20:40:01 miod Exp $	*/
+
+/*
+ * Copyright (c) 2009 Miodrag Vallat.
+ *
+ * Permission to use, copy, modify, and distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#include <machine/asm.h>
+
+	.data
+
+/*
+ * A __cpu_simplelock_t used to provide the inter-processor interlock.
+ */
+ASLOCAL(__atomic_interlock)
+	.word	0
+
+	.text
+
+/*
+ * Register usage in this file:
+ *
+ * r2 data address
+ * r3 bits to set or clear
+ * r4 return address
+ * r5 scratch
+ * r6 interlock address
+ * r7 psr upon entry
+ * r8 active psr
+ */
+
+ENTRY(atomic_setbits_int)
+	or	r4, r1, r0		/* save return address */
+	bsr	_C_LABEL(__atomic_lock)
+
+	ld	r5, r2, r0
+	or	r5, r5, r3
+	st	r5, r2, r0
+
+	br	_C_LABEL(__atomic_unlock)
+
+ENTRY(atomic_clearbits_int)
+	or	r4, r1, r0		/* save return address */
+	bsr	_C_LABEL(__atomic_lock)
+
+	ld	r5, r2, r0
+	or	r5, r5, r3
+	xor	r5, r5, r3		/* r5 &= ~r3 */
+	st	r5, r2, r0
+
+	br	_C_LABEL(__atomic_unlock)
+
+GLOBAL(__atomic_lock)
+
+/*
+ * If running a kernel with support for both 88100 and 88110 compiled-in
+ * on a 88100 machine, the 88100 code (shorter) will be copied over in
+ * vector_init().
+ */
+
+#ifdef M88110
+ASLOCAL(__atomic_lock_88110)
+	/*
+	 * This is the 88110 version: disable shadowing and interrupts,
+	 * then grab the interlock.
+	 */
+
+	ldcr	r7, PSR
+	set	r8, r7, 1<PSR_INTERRUPT_DISABLE_BIT>
+	set	r8, r8, 1<PSR_SHADOW_FREEZE_BIT>
+	stcr	r8, PSR
+	FLUSH_PIPELINE
+
+	or.u	r6, r0, hi16(_ASM_LABEL(__atomic_interlock))
+	or	r6, r6, lo16(_ASM_LABEL(__atomic_interlock))
+	or	r5, r0, 1	/* __SIMPLELOCK_LOCKED */
+1:
+	xmem	r5, r6, r0
+	bcnd	eq0, r5, 3f
+2:
+	ld	r5, r6, r0
+	bcnd	eq0, r5, 1b
+	br	2b
+3:
+	jmp	r1
+#endif
+
+#ifdef M88100
+GLOBAL(__atomic_lock_88100)
+	/*
+	 * This is the 88100 version: disable interrupts, then grab
+	 * the interlock.
+	 */
+
+	ldcr	r7, PSR
+	set	r8, r7, 1<PSR_INTERRUPT_DISABLE_BIT>
+	stcr	r8, PSR
+	FLUSH_PIPELINE
+
+	or.u	r6, r0, hi16(_ASM_LABEL(__atomic_interlock))
+	or	r6, r6, lo16(_ASM_LABEL(__atomic_interlock))
+	or	r5, r0, 1	/* __SIMPLELOCK_LOCKED */
+1:
+	xmem	r5, r6, r0
+	bcnd	eq0, r5, 3f
+2:
+	ld	r5, r6, r0
+	bcnd	eq0, r5, 1b
+	br	2b
+3:
+	jmp	r1
+GLOBAL(__atomic_lock_88100_end)
+#endif
+
+GLOBAL(__atomic_unlock)
+
+/*
+ * If running a kernel with support for both 88100 and 88110 compiled-in
+ * on a 88100 machine, the 88100 code (shorter) will be copied over in
+ * vector_init().
+ */
+
+#ifdef M88110
+ASLOCAL(__atomic_unlock_88110)
+	/*
+	 * This is the 88110 version: release the interlock, set up
+	 * exception registers to return to our caller with initial
+	 * psr restored.
+	 */
+
+	st	r0, r6, r0	/* release interlock */
+
+	stcr	r4, EXIP	/* return address */
+	stcr	r7, EPSR	/* original PSR */
+
+	/*
+	 * No need to workaround errata #18 (see m88110_user_rte in
+	 * eh_common.S), as we are not returning to user mode.
+	 */
+	RTE
+#endif
+
+#ifdef M88100
+GLOBAL(__atomic_unlock_88100)
+	/*
+	 * This is the 88100 version: release the interlock,
+	 * restore psr and return to the caller.
+	 */
+
+	st	r0, r6, r0	/* release interlock */
+
+	stcr	r7, PSR
+	FLUSH_PIPELINE
+	
+	jmp	r4
+GLOBAL(__atomic_unlock_88100_end)
+#endif
diff --git a/sys/arch/m88k/m88k/m88k_machdep.c b/sys/arch/m88k/m88k/m88k_machdep.c
index 6be580fac06..5e3da058053 100644
--- a/sys/arch/m88k/m88k/m88k_machdep.c
+++ b/sys/arch/m88k/m88k/m88k_machdep.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: m88k_machdep.c,v 1.45 2009/02/18 21:07:59 miod Exp $	*/
+/*	$OpenBSD: m88k_machdep.c,v 1.46 2009/02/20 20:40:01 miod Exp $	*/
 /*
  * Copyright (c) 1998, 1999, 2000, 2001 Steve Murphree, Jr.
  * Copyright (c) 1996 Nivas Madhur
@@ -93,7 +93,6 @@ void	vector_init(m88k_exception_vector_area *, u_int32_t *);
 
 #ifdef MULTIPROCESSOR
 cpuid_t	master_cpu;
-__cpu_simple_lock_t __atomic_lock = __SIMPLELOCK_UNLOCKED;
 __cpu_simple_lock_t cmmu_cpu_lock = __SIMPLELOCK_UNLOCKED;
 #endif
 
@@ -438,17 +437,17 @@ spl0()
 /*
  * vector_init(vector, vector_init_list)
  *
- * This routine sets up the m88k vector table for the running processor.
+ * This routine sets up the m88k vector table for the running processor,
+ * as well as the atomic operation routines for multiprocessor kernels.
  * This is the first C code to run, before anything is initialized.
  *
- * It fills the exception vectors page. I would add an extra four bytes
- * to the page pointed to by the vbr, since the 88100 may execute the
- * first instruction of the next trap handler, as documented in its
- * Errata. Processing trap #511 would then fall into the next page,
- * unless the address computation wraps, or software traps can not trigger
- * the issue - the Errata does not provide more detail. And since the
- * MVME BUG does not add an extra NOP after their VBR page, I'll assume this
- * is safe for now -- miod
+ * I would add an extra four bytes to the exception vectors page pointed
+ * to by the vbr, since the 88100 may execute the first instruction of the
+ * next trap handler, as documented in its Errata. Processing trap #511
+ * would then fall into the next page, unless the address computation wraps,
+ * or software traps can not trigger the issue - the Errata does not provide
+ * more detail. And since the MVME BUG does not add an extra NOP after their
+ * VBR page, I'll assume this is safe for now -- miod
  */
 void
 vector_init(m88k_exception_vector_area *vbr, u_int32_t *vector_init_list)
@@ -513,6 +512,39 @@ vector_init(m88k_exception_vector_area *vbr, u_int32_t *vector_init_list)
 		break;
 #endif
 	}
+
+#ifdef MULTIPROCESSOR
+	/*
+	 * Setting up the proper atomic operation code is not really
+	 * related to vector initialization, but is crucial enough to
+	 * be worth doing right now, rather than too late in the C code.
+	 *
+	 * This is only necessary for SMP kernels with 88100 and 88110
+	 * support compiled-in, which happen to run on 88100.
+	 */
+#if defined(M88100) && defined(M88110)
+	if (cputyp == CPU_88100) {
+		extern uint32_t __atomic_lock[];
+		extern uint32_t __atomic_lock_88100[], __atomic_lock_88100_end[];
+		extern uint32_t __atomic_unlock[];
+		extern uint32_t __atomic_unlock_88100[], __atomic_unlock_88100_end[];
+
+		uint32_t *s, *e, *d;
+
+		d = __atomic_lock;
+		s = __atomic_lock_88100;
+		e = __atomic_lock_88100_end;
+		while (s != e)
+				*d++ = *s++;
+
+		d = __atomic_unlock;
+		s = __atomic_unlock_88100;
+		e = __atomic_unlock_88100_end;
+		while (s != e)
+				*d++ = *s++;
+	}
+#endif	/* M88100 && M88110 */
+#endif	/* MULTIPROCESSOR */
 }
 
 #ifdef MULTIPROCESSOR