Work in progress SMP code; mvme88k boards can spin up secondary CPUs,

kernel boots single user. Still a lot of polishing and bugfixing to do.

7 files changed, 352 insertions, 303 deletions

diff --git a/sys/arch/mvme88k/dev/bugio.c b/sys/arch/mvme88k/dev/bugio.c
index 3d00a93deca..0cdc177c4e2 100644
--- a/sys/arch/mvme88k/dev/bugio.c
+++ b/sys/arch/mvme88k/dev/bugio.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: bugio.c,v 1.15 2005/10/13 19:43:43 miod Exp $ */
+/*	$OpenBSD: bugio.c,v 1.16 2005/12/11 21:45:31 miod Exp $ */
 /*  Copyright (c) 1998 Steve Murphree, Jr. */
 
 #include <sys/param.h>
@@ -8,7 +8,7 @@
 #include <machine/bugio.h>
 #include <machine/prom.h>
 
-register_t ossr0, ossr1, ossr2, ossr3;	/* XXX ought to be per-cpu */
+register_t ossr0, ossr1, ossr2, ossr3;
 register_t bugsr3;
 
 unsigned long bugvec[2], sysbugvec[2];
@@ -16,6 +16,16 @@ unsigned long bugvec[2], sysbugvec[2];
 void bug_vector(void);
 void sysbug_vector(void);
 
+#ifdef MULTIPROCESSOR
+#include <sys/lock.h>
+__cpu_simple_lock_t bug_lock = __SIMPLELOCK_UNLOCKED;
+#define	BUG_LOCK()	__cpu_simple_lock(&bug_lock)
+#define	BUG_UNLOCK()	__cpu_simple_unlock(&bug_lock)
+#else
+#define	BUG_LOCK()	do { } while (0)
+#define	BUG_UNLOCK()	do { } while (0)
+#endif
+
 #define MVMEPROM_CALL(x)						\
 	__asm__ __volatile__ ("or r9,r0," __STRING(x));			\
 	__asm__ __volatile__ ("tb0 0,r0,496" :::			\
@@ -44,6 +54,7 @@ sysbug_vector()
 
 #define	BUGCTXT()							\
 {									\
+	BUG_LOCK();							\
 	disable_interrupt(psr);			/* paranoia */		\
 	bug_vector();							\
 	__asm__ __volatile__ ("ldcr %0, cr17" : "=r" (ossr0));		\
@@ -64,6 +75,7 @@ sysbug_vector()
 	__asm__ __volatile__ ("stcr %0, cr20" :: "r"(ossr3));		\
 	sysbug_vector();						\
 	set_psr(psr);							\
+	BUG_UNLOCK();							\
 }
 
 static void
@@ -123,7 +135,7 @@ buginstat(void)
 	MVMEPROM_CALL(MVMEPROM_INSTAT);
 	__asm__ __volatile__  ("or %0,r0,r2" : "=r" (ret));
 	OSCTXT();
-	return (ret & 0x4 ? 0 : 1);
+	return ((ret & 0x08) >> 3);
 }
 
 void
@@ -180,3 +192,26 @@ bugdiskrd(struct mvmeprom_dskio *dio)
 	MVMEPROM_CALL(MVMEPROM_DSKRD);
 	OSCTXT();
 }
+
+#ifdef MULTIPROCESSOR
+
+/*
+ * Ask the BUG to start a particular cpu at our provided address.
+ */
+int
+spin_cpu(cpuid_t cpu, vaddr_t address)
+{
+	u_int psr;
+	int ret;
+
+	BUGCTXT();
+	__asm__ __volatile__ ("or r2, r0, %0" : : "r" (cpu));
+	__asm__ __volatile__ ("or r3, r0, %0" : : "r" (address));
+	MVMEPROM_CALL(MVMEPROM_FORKMPU);
+	__asm__ __volatile__ ("or %0,r0,r2" : "=r" (ret));
+	OSCTXT();
+
+	return (ret);
+}
+
+#endif	/* MULTIPROCESSOR */
diff --git a/sys/arch/mvme88k/dev/mainbus.c b/sys/arch/mvme88k/dev/mainbus.c
index 31feeb475a6..2fd42641548 100644
--- a/sys/arch/mvme88k/dev/mainbus.c
+++ b/sys/arch/mvme88k/dev/mainbus.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: mainbus.c,v 1.17 2005/11/25 22:14:31 miod Exp $ */
+/*	$OpenBSD: mainbus.c,v 1.18 2005/12/11 21:45:31 miod Exp $ */
 /*
  * Copyright (c) 1998 Steve Murphree, Jr.
  * Copyright (c) 2004, Miodrag Vallat.
@@ -276,7 +276,7 @@ mainbus_attach(parent, self, args)
 	printf(": %s\n", cpu_model);
 
 	/*
-	 * Display cpu/mmu details. Only for the master CPU so far.
+	 * Display cpu/mmu details for the main processor.
 	 */
 	cpu_configuration_print(1);
 
diff --git a/sys/arch/mvme88k/include/bugio.h b/sys/arch/mvme88k/include/bugio.h
index cdc62d71d66..4401834e8e3 100644
--- a/sys/arch/mvme88k/include/bugio.h
+++ b/sys/arch/mvme88k/include/bugio.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: bugio.h,v 1.14 2004/01/04 01:14:06 miod Exp $ */
+/*	$OpenBSD: bugio.h,v 1.15 2005/12/11 21:45:31 miod Exp $ */
 
 #ifndef __MACHINE_BUGIO_H__
 #define __MACHINE_BUGIO_H__
@@ -16,5 +16,6 @@ void bugrtcrd(struct mvmeprom_time *);
 void bugreturn(void);
 void bugbrdid(struct mvmeprom_brdid *);
 void bugdiskrd(struct mvmeprom_dskio *);
+int spin_cpu(cpuid_t, vaddr_t);
 
 #endif /* __MACHINE_BUGIO_H__ */
diff --git a/sys/arch/mvme88k/include/prom.h b/sys/arch/mvme88k/include/prom.h
index 62b97c577b2..a392d5cd117 100644
--- a/sys/arch/mvme88k/include/prom.h
+++ b/sys/arch/mvme88k/include/prom.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: prom.h,v 1.14 2004/08/02 14:38:40 miod Exp $ */
+/*	$OpenBSD: prom.h,v 1.15 2005/12/11 21:45:31 miod Exp $ */
 /*
  * Copyright (c) 1998 Steve Murphree, Jr.
  * Copyright (c) 1996 Nivas Madhur
@@ -70,6 +70,10 @@
 #define ENVIRONTYPE_NETBOOT	4
 #define ENVIRONTYPE_MEMSIZE	5
 
+#define	FORKMPU_NOT_IDLE	-1
+#define	FORKMPU_BAD_ADDRESS	-2
+#define	FORKMPU_NO_MPU		-3
+
 #ifndef LOCORE
 struct mvmeprom_netctrl {
 	u_char	ctrl;
diff --git a/sys/arch/mvme88k/mvme88k/locore.S b/sys/arch/mvme88k/mvme88k/locore.S
index 6fabd59e1c2..cce93535602 100644
--- a/sys/arch/mvme88k/mvme88k/locore.S
+++ b/sys/arch/mvme88k/mvme88k/locore.S
@@ -1,5 +1,6 @@
-/*	$OpenBSD: locore.S,v 1.45 2005/12/03 18:20:51 miod Exp $	*/
+/*	$OpenBSD: locore.S,v 1.46 2005/12/11 21:45:31 miod Exp $	*/
 /*
+ * Copyright (c) 2005, Miodrag Vallat.
  * Copyright (c) 1998 Steve Murphree, Jr.
  * Copyright (c) 1996 Nivas Madhur
  * All rights reserved.
@@ -60,13 +61,11 @@
 #include "assym.h"
 #include "ksyms.h"
 
-#include <sys/errno.h>
-
 #include <machine/asm.h>
-#include <machine/trap.h>
 #include <machine/m88100.h>
-#include <machine/psl.h>
 #include <machine/param.h>
+#include <machine/psl.h>
+#include <machine/trap.h>
 #include <machine/vmparam.h>
 
 /*
@@ -79,68 +78,29 @@
 	text
 
 GLOBAL(kernelstart)
-GLOBAL(kernel_text)
-GLOBAL(start)
 ASGLOBAL(start)
-	br	_C_LABEL(start_text)
-	br	_C_LABEL(start_text)
-	br	_C_LABEL(start_text)
-	br	_C_LABEL(start_text)
-
-GLOBAL(doboot)
 	/*
-	 * Try hitting the SRST bit in VMEchip2 to reset the system.
+	 * A few identical jump instructions to make sure the pipeline is
+	 * in a good state. Probably overkill, but it's cheap.
 	 */
-#ifdef MVME188
-	/* check if it's a mvme188 */
-	or.u	r4, r0, hi16(_C_LABEL(brdtyp))
-	ld	r3, r4, lo16(_C_LABEL(brdtyp))
-	cmp	r4, r3, BRD_188
-	bb1	ne, r4, 1f
-	bsr	_C_LABEL(m188_reset)
-	br	8f
-1:
-#endif	/* MVME188 */
-	or.u	r3, r0, 0xfff4
-	ld	r4, r3, 0x0060		/* read offset (LCSR + 0x60) */
-	bb0.n	30, r4, 1f		/* if not SYSCON, don't SYSRST */
-	 set	r4, r4, 1<23>		/* set SYSRST bit - bit 23 */
-	st	r4, r3, 0x0060		/* and store it back  */
-1:
-	ld	r4, r3, 0x0104		/* try local reset, then */
-	set	r4, r4, 1<7>
-	st	r4, r3, 0x0104
+	br	_ASM_LABEL(main_start)
+	br	_ASM_LABEL(main_start)
+	br	_ASM_LABEL(main_start)
+	br	_ASM_LABEL(main_start)
 
 	/*
-	 * We will be here if the reset above failed. In this case,
-	 * we will try to return to bug.
-	 *
-	 * Switch to interrupt stack and call _doboot to take care
-	 * going to BUG. Need to do this since _doboot turns off the
-	 * the MMU and we need to be on a 1-to-1 mapped stack so that
-	 * further calls don't get data access exceptions.
+	 * Startup code for main processor.
 	 */
-
-	/* Should we use idle_u instead? XXX nivas */
-8:
-	or.u	r31, r0,  hi16(_ASM_LABEL(intstack_end))
-	or	r31, r31, lo16(_ASM_LABEL(intstack_end))
-	clr	r31, r31, 3<0>	/* round down to 8-byte boundary */
-
-	bsr	_C_LABEL(_doboot)
-	/*NOTREACHED*/
-
-	/* This is the *real* start upon poweron or reset */
-GLOBAL(start_text)
+ASLOCAL(main_start)
 	/*
-	 * Args passed by boot loader
-	 * 	r2 howto
-	 *	r3 boot controller address
-	 *	r4 esym
-	 *	r5 start of mini
-	 *	r6 end miniroot
-	 *	r7 ((Clun << 8) ; Dlun & FF) -> bootdev
-	 *	r8 cpu type (0x187, 0x188, 0x197)
+	 * Save the arguments passed by the boot loader
+	 * 	r2 boot flags
+	 *	r3 boot controller physical address
+	 *	r4 esym (if applicable)
+	 *	r5 start of miniroot (unused)
+	 *	r6 end of miniroot (unused)
+	 *	r7 ((Clun << 8) | Dlun): encoded bootdev
+	 *	r8 board type (0x187, 0x188, 0x197)
 	 */
 /*
  * (*entry)(flag, bugargs.ctrl_addr, cp, kernel.smini,kernel.emini,
@@ -161,7 +121,7 @@ GLOBAL(start_text)
 	or.u	r13, r0,  hi16(_C_LABEL(brdtyp))
 	st	r8,  r13, lo16(_C_LABEL(brdtyp))
 
-	/* set _cputyp */
+	/* set cputyp */
 	cmp	r2,  r8, BRD_197	/* r8 contains brdtyp */
 	bb1	ne, r2, 1f	/* if it's a '197, CPU is 88110 */
 	or.u	r13, r0,  hi16(CPU_88110)
@@ -177,10 +137,7 @@ GLOBAL(start_text)
 	/*
 	 * CPU Initialization
 	 *
-	 * Every CPU starts from here..
-	 * (well, from 'start' above, which just jumps here).
-	 *
-	 * I use r11 and r22 here 'cause they're easy to not
+	 * I use r11 and r22 here because they're easy to not
 	 * get mixed up -- r10, for example, looks too similar
 	 * to r0 when not being careful....
 	 *
@@ -233,89 +190,25 @@ GLOBAL(start_text)
 	FLUSH_PIPELINE
 	stcr	r0,  VBR	/* set Vector Base Register to 0, ALWAYS! */
 	FLUSH_PIPELINE
-	cmp	r2, r8, CPU_88110	/* r8 contains cputyp */
-	bb1	eq, r2, _ASM_LABEL(master_start) /* if it's a '197, skip */
-
-#if 0
-	/* clear BSS. Boot loader might have already done this... */
-	or.u	r2, r0, hi16(_C_LABEL(edata))
-	or	r2, r2, lo16(_C_LABEL(edata))
-	or.u	r4, r0, hi16(_C_LABEL(end))
-	or	r4, r4, lo16(_C_LABEL(end))
-	bsr.n	_bzero		/* bzero(edata, end-edata) */
-	 subu	r3, r4, r2
-#endif
-	/*
-	 * First time to count how many CPUs to attach
-	 */
-	or.u	r11, r0,  hi16(_ASM_LABEL(initialized_cpu_lock))
-	or	r11, r11, lo16(_ASM_LABEL(initialized_cpu_lock))
-1:
-	FLUSH_PIPELINE
-	or	r22, r0,  1
-	xmem	r22, r11, r0	/* If r22 gets 0, we have the lock.. */
-	bcnd	eq0, r22, 3f	/* ..but if not, we must wait */
-2:
-	/* just watch the lock until it looks clear */
-	ld	r22, r11, r0
-	bcnd	eq0, r22, 1b
-	br	2b		/* looks clear -- try to grab */
-3:
-	FLUSH_PIPELINE
-	or.u	r11, r0,  hi16(_ASM_LABEL(initialized_cpus))
-	ld	r22, r11, lo16(_ASM_LABEL(initialized_cpus))
-	add	r23, r22, 1
-	st	r23, r11, lo16(_ASM_LABEL(initialized_cpus))
 
-	or.u	r11, r0,  hi16(_ASM_LABEL(initialized_cpu_lock))
-	st	r0,  r11, lo16(_ASM_LABEL(initialized_cpu_lock))
+#ifdef MULTIPROCESSOR
 	/*
-	 * Now we view with any other processors to see who's the master.
-	 * We first try to obtain a lock to see who's allowed
-	 * to check/set the master lock.
+	 * Have curcpu() point at the dummy cpuinfo structure,
+	 * so that cpu_number() does not dereference random memory.
+	 * This is necessary for early spl usage, despite the fact that
+	 * interrupts are disabled...
 	 */
-	or.u	r11, r0,  hi16(_ASM_LABEL(inter_processor_lock))
-	or	r11, r11, lo16(_ASM_LABEL(inter_processor_lock))
-1:
-	FLUSH_PIPELINE
-	or	r22, r0,  1
-	xmem	r22, r11, r0		/* If r22 gets 0, we have the lock.. */
-	bcnd	eq0, r22, 4f		/* ..but if not, we must wait */
-2:
-	/* just watch the lock until it looks clear */
-	ld	r22, r11, r0
-	bcnd	ne0, r22, 2b
-	/* since we can be here with caches off, add a few nops to
-	   keep the bus from getting overloaded */
-	or	r2, r0, lo16(1000)
-3:
-	subu	r2, r2, 1
-	bcnd	eq0, r2, 3b
-	br	1b			/* looks clear -- try to grab */
-4:
-	/* now try to grab the master_processor_chosen prize */
-	FLUSH_PIPELINE
-	or.u	r11, r0,  hi16(_ASM_LABEL(master_processor_chosen))
-	or	r11, r11, lo16(_ASM_LABEL(master_processor_chosen))
-	or	r22, r0,  1
-	xmem	r22, r11, r0
+	or.u	r11, r0,  hi16(_ASM_LABEL(dummy_cpu))
+	or	r11, r11, lo16(_ASM_LABEL(dummy_cpu))
+	stcr	r11, CPU
 
 	/*
-	 * If r22 is not clear we're a slave,
-	 * otherwise we're first and the master.
-	 *
-	 * Note that we haven't released the interprocessor lock....
-	 * We'll do that when we're ready for another CPU to go.
-	 * (if we're the master, we'll do that in master_start below.
-	 *  if we're a slave, we'll do it in slave_start below).
+	 * MVME BUG idles all secondary MPUs upon startup, so at this point
+	 * we do not have to compete with them.
 	 */
-	bcnd	ne0, r22, _ASM_LABEL(slave_start)
-	/* fall through to master start if that's appropriate */
+#endif	/* MULTIPROCESSOR */
 
-ASLOCAL(master_start)
-	/*
-	 * Switch to interrupt stack
-	 */
+	/* Switch to interrupt stack */
 	or.u	r31, r0,  hi16(_ASM_LABEL(intstack_end))
 	or	r31, r31, lo16(_ASM_LABEL(intstack_end))
 
@@ -349,10 +242,10 @@ ASLOCAL(master_start)
 	 */
 	ldcr	r10, CPU
 	ld	r31, r10, CI_CURPCB
-	addu	r31, r31, USIZE
 
 	/* call main() - no arguments although main() still defines one */
-	bsr	_C_LABEL(main)
+	bsr.n	_C_LABEL(main)
+	 addu	r31, r31, USIZE
 
 	or.u	r2, r0, hi16(_ASM_LABEL(main_panic))
 	bsr.n	_C_LABEL(panic)
@@ -365,44 +258,145 @@ ASLOCAL(main_panic)
 	text
 	.align	8
 
-/*
- *	slave CPUs starts here
- */
-ASLOCAL(slave_start)
+#ifdef MULTIPROCESSOR
+
 	/*
-	 * While holding the inter_processor_lock, the slave cpu can use
-	 * the slavestack to call slave_pre_main and determine its cpu number.
-	 * After that, however, it should switch over to the interrupt stack
-	 * associated with its cpu.
+	 * Startup code for secondary processors.
+	 * Some of these initializations are very close to main_start; refer
+	 * to the comments there for details.
 	 */
-
-	/* r31 <-- slavestack */
+GLOBAL(secondary_start)
 	or.u	r31, r0,  hi16(_ASM_LABEL(slavestack_end))
 	or	r31, r31, lo16(_ASM_LABEL(slavestack_end))
 
-	bsr.n	_C_LABEL(slave_pre_main)	/* set cpu number */
-	 clr	r31, r31, 3<0>	/* round down to 8-byte boundary */
+	or.u	r13, r0,  hi16(_C_LABEL(cputyp))
+	ld	r8,  r13, lo16(_C_LABEL(cputyp))
+
+	cmp	r2, r8, CPU_88110
+	bb1	eq, r2, 1f
+	stcr	r0, SSBR
+1:
+	stcr	r0, SR1
+
+	set	r11, r0,  1<PSR_SUPERVISOR_MODE_BIT>
+	set	r11, r11, 1<PSR_INTERRUPT_DISABLE_BIT>
+	set	r11, r11, 1<PSR_GRAPHICS_DISABLE_BIT>
+	/*
+	 * XXX On 88110 processors, force serial instruction execution for now.
+	 * Situation where OoO would break will be hopefully taken care of in
+	 * the near future -- miod
+	 */
+#if 0
+	clr	r11, r11, 1<PSR_SERIAL_MODE_BIT>
+#else
+	set	r11, r11, 1<PSR_SERIAL_MODE_BIT>
+#endif
+	set	r11, r11, 1<PSR_SERIALIZE_BIT>
+	stcr	r11, PSR
+	FLUSH_PIPELINE
+	stcr	r0,  VBR	/* set Vector Base Register to 0, ALWAYS! */
+	FLUSH_PIPELINE
 
-	bsr	_C_LABEL(get_slave_stack)
-	addu	r31, r2, INTSTACK_SIZE
+	/*
+	 * Have curcpu() point at the dummy cpuinfo structure,
+	 * so that cpu_number() does not dereference random memory.
+	 * This is necessary for early spl usage, despite the fact that
+	 * interrupts are disabled...
+	 */
+	or.u	r11, r0,  hi16(_ASM_LABEL(dummy_cpu))
+	or	r11, r11, lo16(_ASM_LABEL(dummy_cpu))
+	stcr	r11, CPU
 
 	/*
-	 * SR1 now contains our cpu number. We can now release the
-	 * inter_processor_lock, as we are done with the slavestack.
-	 * We also have an interrupt stack
+	 * Since there may be more than one secondary MPU, compete with them
+	 * to initialize safely.
+	 */
+	or.u	r11, r0,  hi16(_C_LABEL(cpu_mutex))
+	or	r11, r11, lo16(_C_LABEL(cpu_mutex))
+1:
+	or	r22, r0,  1
+	xmem	r22, r11, r0		/* If r22 gets 0, we have the lock... */
+	bcnd	eq0, r22, 4f		/* ...but if not, we must wait */
+2:
+	/* just watch the lock until it looks clear */
+	ld	r22, r11, r0
+	bcnd	eq0, r22, 1b
+	/* since we can be here with caches off, add a few nops to
+	   keep the bus from getting overloaded */
+	or	r2,  r0,  lo16(1000)
+3:
+	subu	r2,  r2,  1
+	bcnd	ne0, r2,  3b
+	br	1b
+4:
+	
+	/*
+	 * While holding the cpu_mutex, the secondary cpu can use the slavestack
+	 * to call secondary_pre_main() to determine its cpu number.
+	 * After that, however, it should allocate its own stack and switch
+	 * to it.
 	 */
 
-	or.u	r10, r0, hi16(_ASM_LABEL(inter_processor_lock))
-	st	r0, r10, lo16(_ASM_LABEL(inter_processor_lock))
+	bsr	_C_LABEL(secondary_pre_main)	/* set cpu number */
 
-	br	_C_LABEL(slave_main)	/* does not return */
+	ldcr	r2, CPU
+	ld	r3, r2, CI_IDLE_PCB
+	bsr.n	_C_LABEL(secondary_main)
+	 addu	r31, r3, USIZE			/* switch to idle stack */
 
-GLOBAL(spin_cpu)
-	or.u	r3,   r0,  hi16(_C_LABEL(start_text))
-	or	r3,   r3,  lo16(_C_LABEL(start_text))
-	or	r9,   r0,  0x100		/* .FORKMPU */
-	tb0	0,    r0,  0x200-16		/* call 188Bug */
-	jmp	r1
+	/*
+	 * At this point, the CPU has been correctly initialized and has
+	 * identified itself on the console.
+	 * All it needs now is to jump to the idle loop and wait for work to
+	 * be offered.
+	 */
+	br	_ASM_LABEL(cpu_switch_idle)
+
+#endif	/* MULTIPROCESSOR */
+
+/*
+ * Reset code.
+ * Should be rewritten in C eventually.
+ */
+GLOBAL(doboot)
+#ifdef MVME188
+	/* check if it's a mvme188 */
+	or.u	r4, r0, hi16(_C_LABEL(brdtyp))
+	ld	r3, r4, lo16(_C_LABEL(brdtyp))
+	cmp	r4, r3, BRD_188
+	bb1	ne, r4, 1f
+	bsr	_C_LABEL(m188_reset)
+	br	8f
+1:
+#endif	/* MVME188 */
+	/*
+	 * Try hitting the SRST bit in VMEchip2 to reset the system.
+	 */
+	or.u	r3, r0, 0xfff4
+	ld	r4, r3, 0x0060		/* read offset (LCSR + 0x60) */
+	bb0.n	30, r4, 1f		/* if not SYSCON, don't SYSRST */
+	 set	r4, r4, 1<23>		/* set SYSRST bit - bit 23 */
+	st	r4, r3, 0x0060		/* and store it back  */
+1:
+	ld	r4, r3, 0x0104		/* try local reset, then */
+	set	r4, r4, 1<7>
+	st	r4, r3, 0x0104
+
+	/*
+	 * We will be here if the reset above failed. In this case,
+	 * we will try to return to bug.
+	 *
+	 * Switch to interrupt stack and call _doboot to take care
+	 * going to BUG. Need to do this since _doboot turns off the
+	 * the MMU and we need to be on a 1-to-1 mapped stack so that
+	 * further calls don't get data access exceptions.
+	 */
+
+8:
+	or.u	r31, r0,  hi16(_ASM_LABEL(intstack_end))
+	bsr.n	_C_LABEL(_doboot)
+	 or	r31, r31, lo16(_ASM_LABEL(intstack_end))
+	/*NOTREACHED*/
 
 /*
  * MVME188 specific support routines
@@ -441,17 +435,14 @@ GLOBAL(kernel_sdt)		/* SDT (segment descriptor table */
 ASGLOBAL(intstack)
 	space	USIZE
 ASGLOBAL(intstack_end)
-ASGLOBAL(slavestack)
+
+#ifdef MULTIPROCESSOR
 	space	PAGE_SIZE	/* 4K, small, interim stack */
-ASGLOBAL(slavestack_end)
+ASLOCAL(slavestack_end)
+#endif
 
 /*
- * When a process exits and its u. area goes away, we set curpcb to point
- * to this `u.', leaving us with something to use for an interrupt stack,
- * and letting all the register save code have a pcb_uw to examine.
- * This is also carefully arranged (to come just before u0, so that
- * process 0's kernel stack can quietly overrun into it during bootup, if
- * we feel like doing that).
+ * Main processor's idle pcb and stack.
  * Should be page aligned.
  */
 	.align	PAGE_SIZE
@@ -460,8 +451,7 @@ GLOBAL(idle_u)
 
 /*
  * Process 0's u.
- *
- * This must be page aligned
+ * Should be page aligned.
  */
 	.align	PAGE_SIZE
 ASLOCAL(u0)
@@ -469,20 +459,14 @@ ASLOCAL(u0)
 GLOBAL(proc0paddr)
 	word	_ASM_LABEL(u0)	/*  KVA of proc0 uarea */
 
-GLOBAL(ret_addr)
-	word 0
-/* XMEM spin lock -- to count CPUs */
-ASLOCAL(initialized_cpu_lock)
-	word 0
-/* CPU counter to initialize */
-ASLOCAL(initialized_cpus)
-	word 0
-/* The first processor that XMEMs this becomes the master */
-ASLOCAL(master_processor_chosen)
-	word 0
-/* XMEM spin lock -- controls access to master_processor_chosen */
-ASLOCAL(inter_processor_lock)
-	word 0
+#ifdef MULTIPROCESSOR
+/* Dummy cpuinfo structure, for cpu_number() to work early. */
+ASLOCAL(dummy_cpu)
+	word	1	/* ci_alive */
+	word	0	/* ci_curproc */
+	word	0	/* ci_curpcb */
+	word	0	/* ci_cpuid */
+#endif	/* MULTIPROCESSOR */
 
 #if defined(DDB) || NKSYMS > 0
 GLOBAL(esym)
diff --git a/sys/arch/mvme88k/mvme88k/m88110.c b/sys/arch/mvme88k/mvme88k/m88110.c
index 18ddd7806a6..227b2f26665 100644
--- a/sys/arch/mvme88k/mvme88k/m88110.c
+++ b/sys/arch/mvme88k/mvme88k/m88110.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: m88110.c,v 1.34 2005/12/04 15:00:26 miod Exp $	*/
+/*	$OpenBSD: m88110.c,v 1.35 2005/12/11 21:45:31 miod Exp $	*/
 /*
  * Copyright (c) 1998 Steve Murphree, Jr.
  * All rights reserved.
@@ -62,6 +62,7 @@
 
 #include <uvm/uvm_extern.h>
 
+#include <machine/asm_macro.h>
 #include <machine/cmmu.h>
 #include <machine/cpu.h>
 #include <machine/lock.h>
@@ -71,21 +72,24 @@
 #include <machine/psl.h>
 #include <machine/trap.h>
 
-cpuid_t m88110_init(void);
-void m88110_setup_board_config(void);
-void m88110_cpu_configuration_print(int);
-void m88110_shutdown(void);
-cpuid_t m88110_cpu_number(void);
-void m88110_set_sapr(cpuid_t, apr_t);
-void m88110_set_uapr(apr_t);
-void m88110_flush_tlb(cpuid_t, unsigned, vaddr_t, u_int);
-void m88110_flush_cache(cpuid_t, paddr_t, psize_t);
-void m88110_flush_inst_cache(cpuid_t, paddr_t, psize_t);
-void m88110_flush_data_cache(cpuid_t, paddr_t, psize_t);
-int m88110_dma_cachectl(pmap_t, vaddr_t, vsize_t, int);
-int m88110_dma_cachectl_pa(paddr_t, psize_t, int);
-
-/* This is the function table for the mc88110 built-in CMMUs */
+#include <mvme88k/dev/busswreg.h>
+
+cpuid_t	m88110_init(void);
+void	m88110_setup_board_config(void);
+void	m88110_cpu_configuration_print(int);
+void	m88110_shutdown(void);
+cpuid_t	m88110_cpu_number(void);
+void	m88110_set_sapr(cpuid_t, apr_t);
+void	m88110_set_uapr(apr_t);
+void	m88110_flush_tlb(cpuid_t, unsigned, vaddr_t, u_int);
+void	m88110_flush_cache(cpuid_t, paddr_t, psize_t);
+void	m88110_flush_inst_cache(cpuid_t, paddr_t, psize_t);
+void	m88110_flush_data_cache(cpuid_t, paddr_t, psize_t);
+int	m88110_dma_cachectl(pmap_t, vaddr_t, vsize_t, int);
+int	m88110_dma_cachectl_pa(paddr_t, psize_t, int);
+void	m88110_initialize_cpu(cpuid_t);
+
+/* This is the function table for the MC88110 built-in CMMUs */
 struct cmmu_p cmmu88110 = {
         m88110_init,
 	m88110_setup_board_config,
@@ -100,6 +104,9 @@ struct cmmu_p cmmu88110 = {
 	m88110_flush_data_cache,
 	m88110_dma_cachectl,
 	m88110_dma_cachectl_pa,
+#ifdef MULTIPROCESSOR
+	m88110_initialize_cpu,
+#endif
 };
 
 void patc_clear(void);
@@ -126,8 +133,11 @@ patc_clear(void)
 void
 m88110_setup_board_config(void)
 {
-	/* we could print something here... */
+#ifdef MULTIPROCESSOR
+	max_cpus = 2;
+#else
 	max_cpus = 1;
+#endif
 }
 
 /*
@@ -142,13 +152,6 @@ m88110_cpu_configuration_print(int master)
 	int proctype = (pid & PID_ARN) >> ARN_SHIFT;
 	int procvers = (pid & PID_VN) >> VN_SHIFT;
 	int cpu = cpu_number();
-	static __cpu_simple_lock_t print_lock;
-
-	CMMU_LOCK;
-	if (master)
-		__cpu_simple_lock_init(&print_lock);
-
-	__cpu_simple_lock(&print_lock);
 
 	printf("cpu%d: ", cpu);
 	switch (proctype) {
@@ -163,9 +166,6 @@ m88110_cpu_configuration_print(int master)
 		break;
 	}
 	printf("\n");
-
-	__cpu_simple_unlock(&print_lock);
-        CMMU_UNLOCK;
 }
 
 /*
@@ -174,6 +174,16 @@ m88110_cpu_configuration_print(int master)
 cpuid_t
 m88110_init(void)
 {
+	cpuid_t cpu;
+
+	cpu = m88110_cpu_number();
+	m88110_initialize_cpu(cpu);
+	return (cpu);
+}
+
+void
+m88110_initialize_cpu(cpuid_t cpu)
+{
 	int i;
 
 	/* clear BATCs */
@@ -212,8 +222,6 @@ m88110_init(void)
 
 	set_isr(0);
 	set_dsr(0);
-
-	return (m88110_cpu_number());
 }
 
 /*
@@ -222,22 +230,16 @@ m88110_init(void)
 void
 m88110_shutdown(void)
 {
-#if 0
-	CMMU_LOCK;
-        CMMU_UNLOCK;
-#endif
 }
 
-/*
- * Find out the CPU number from accessing CMMU
- * Better be at splhigh, or even better, with interrupts
- * disabled.
- */
-
 cpuid_t
 m88110_cpu_number(void)
 {
-	return (0);	/* XXXSMP - need to tell DP processors apart */
+	u_int16_t gcsr;
+
+	gcsr = *(volatile u_int16_t *)(BS_BASE + BS_GCSR);
+
+	return ((gcsr & BS_GCSR_CPUID) != 0 ? 1 : 0);
 }
 
 void
diff --git a/sys/arch/mvme88k/mvme88k/machdep.c b/sys/arch/mvme88k/mvme88k/machdep.c
index 66a7f57ef1f..71d5df45598 100644
--- a/sys/arch/mvme88k/mvme88k/machdep.c
+++ b/sys/arch/mvme88k/mvme88k/machdep.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: machdep.c,v 1.175 2005/12/11 21:36:06 miod Exp $	*/
+/* $OpenBSD: machdep.c,v 1.176 2005/12/11 21:45:31 miod Exp $	*/
 /*
  * Copyright (c) 1998, 1999, 2000, 2001 Steve Murphree, Jr.
  * Copyright (c) 1996 Nivas Madhur
@@ -97,13 +97,12 @@ void	consinit(void);
 void	dumpconf(void);
 void	dumpsys(void);
 int	getcpuspeed(struct mvmeprom_brdid *);
-vaddr_t	get_slave_stack(void);
 void	identifycpu(void);
 void	mvme_bootstrap(void);
 void	savectx(struct pcb *);
+void	secondary_main(void);
+void	secondary_pre_main(void);
 void	setupiackvectors(void);
-void	slave_pre_main(void);
-int	slave_main(void);
 void	vector_init(m88k_exception_vector_area *, unsigned *);
 void	_doboot(void);
 
@@ -154,6 +153,10 @@ vaddr_t iomapbase;
 struct extent *iomap_extent;
 struct vm_map *iomap_map;
 
+#ifdef MULTIPROCESSOR
+__cpu_simple_lock_t cpu_mutex = __SIMPLELOCK_UNLOCKED;
+#endif
+
 /*
  * Declare these as initialized data so we can patch them.
  */
@@ -433,7 +436,7 @@ cpu_startup()
 		 * "base" pages for the rest.
 		 */
 		curbuf = (vaddr_t)buffers + (i * MAXBSIZE);
-		curbufsize = PAGE_SIZE * ((i < residual) ? (base+1) : base);
+		curbufsize = PAGE_SIZE * ((i < residual) ? (base + 1) : base);
 
 		while (curbufsize) {
 			pg = uvm_pagealloc(NULL, 0, NULL, 0);
@@ -806,44 +809,65 @@ setupiackvectors()
 	}
 }
 
-/* gets an interrupt stack for slave processors */
-vaddr_t
-get_slave_stack()
+#ifdef MULTIPROCESSOR
+
+/*
+ * Secondary CPU early initialization routine.
+ * Determine CPU number and set it, then allocate the idle pcb (and stack).
+ *
+ * Running on a minimal stack here, with interrupts disabled; do nothing fancy.
+ */
+void
+secondary_pre_main()
 {
-	vaddr_t addr;
+	struct cpu_info *ci;
 
-	addr = (vaddr_t)uvm_km_zalloc(kernel_map, INTSTACK_SIZE);
+	set_cpu_number(cmmu_cpu_number()); /* Determine cpu number by CMMU */
+	ci = curcpu();
+	ci->ci_curproc = &proc0;
 
-	if (addr == NULL)
-		panic("Cannot allocate slave stack for cpu %d",
-		    cpu_number());
+	splhigh();
 
-	return addr;
+	/*
+	 * Setup CMMUs and translation tables (shared with the master cpu).
+	 */
+	pmap_bootstrap_cpu(ci->ci_cpuid);
+
+	/*
+	 * Allocate UPAGES contiguous pages for the idle PCB and stack.
+	 */
+	ci->ci_idle_pcb = (struct pcb *)uvm_km_zalloc(kernel_map, USPACE);
+	if (ci->ci_idle_pcb == NULL) {
+		printf("cpu%d: unable to allocate idle stack\n", ci->ci_cpuid);
+	}
 }
 
 /*
- * Slave CPU pre-main routine.
- * Determine CPU number and set it.
+ * Further secondary CPU initialization.
  *
- * Running on an interrupt stack here; do nothing fancy.
+ * We are now running on our idle stack, with proper page tables.
+ * There is nothing to do but display some details about the CPU and its CMMUs.
  */
 void
-slave_pre_main()
+secondary_main()
 {
-	set_cpu_number(cmmu_cpu_number()); /* Determine cpu number by CMMU */
-	splhigh();
-	set_psr(get_psr() & ~PSR_IND);
-}
+	struct cpu_info *ci = curcpu();
 
-/* dummy main routine for slave processors */
-int
-slave_main()
-{
-	printf("slave CPU%d started\n", cpu_number());
-	while (1); /* spin forever */
-	return 0;
+	cpu_configuration_print(0);
+	__cpu_simple_unlock(&cpu_mutex);
+
+	microuptime(&ci->ci_schedstate.spc_runtime);
+	ci->ci_curproc = NULL;
+
+	/*
+	 * Upon return, the secondary cpu bootstrap code in locore will
+	 * enter the idle loop, waiting for some food to process on this
+	 * processor.
+	 */
 }
 
+#endif	/* MULTIPROCESSOR */
+
 /*
  * Search for the first available interrupt vector in the range start, end.
  * This should really only be used by VME devices.
@@ -1062,39 +1086,6 @@ mvme_bootstrap()
 	curproc = &proc0;
 	curpcb = &proc0paddr->u_pcb;
 
-	/*
-	 * If we have more than one CPU, mention which one is the master.
-	 * We will also want to spin up slave CPUs on the long run...
-	 */
-	switch (brdtyp) {
-#ifdef MVME188
-	case BRD_188:
-		printf("CPU%d is master CPU\n", master_cpu);
-
-#if 0
-		int i;
-		for (i = 0; i < MAX_CPUS; i++) {
-			if (!spin_cpu(i))
-				printf("CPU%d started\n", i);
-		}
-#endif
-		break;
-#endif
-#ifdef MVME197
-	case BRD_197:
-		/*
-		 * In the 197DP case, mention which CPU is the master
-		 * there too...
-		 * XXX TBD
-		 */
-		break;
-#endif
-#ifdef MVME187
-	default:
-		break;
-#endif
-	}
-
 	avail_start = first_addr;
 	avail_end = last_addr;
 
@@ -1126,6 +1117,38 @@ mvme_bootstrap()
 #endif
 }
 
+#ifdef MULTIPROCESSOR
+void
+cpu_boot_secondary_processors()
+{
+	cpuid_t cpu;
+	int rc;
+	extern void secondary_start(void);
+
+	switch (brdtyp) {
+#if defined(MVME188) || defined(MVME197)
+#ifdef MVME188
+	case BRD_188:
+#endif
+#ifdef MVME197
+	case BRD_197:
+#endif
+		for (cpu = 0; cpu < max_cpus; cpu++) {
+			if (cpu != curcpu()->ci_cpuid) {
+				rc = spin_cpu(cpu, (vaddr_t)secondary_start);
+				if (rc != 0 && rc != FORKMPU_NO_MPU)
+					printf("cpu%d: spin_cpu error %d\n",
+					    cpu, rc);
+			}
+		}
+		break;
+#endif
+	default:
+		break;
+	}
+}
+#endif
+
 /*
  * Boot console routines:
  * Enables printing of boot messages before consinit().
@@ -1169,7 +1192,7 @@ bootcnputc(dev, c)
 #define NO_OP 		0xf4005800	/* "or r0, r0, r0" */
 
 #define BRANCH(FROM, TO) \
-	(EMPTY_BR | ((unsigned)(TO) - (unsigned)(FROM)) >> 2)
+	(EMPTY_BR | ((vaddr_t)(TO) - (vaddr_t)(FROM)) >> 2)
 
 #define SET_VECTOR(NUM, VALUE) \
 	do { \