Work in progress SMP code; mvme88k boards can spin up secondary CPUs,

kernel boots single user. Still a lot of polishing and bugfixing to do.

17 files changed, 492 insertions, 372 deletions

diff --git a/sys/arch/m88k/include/cmmu.h b/sys/arch/m88k/include/cmmu.h
index c23a2f24b38..4a09689093a 100644
--- a/sys/arch/m88k/include/cmmu.h
+++ b/sys/arch/m88k/include/cmmu.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: cmmu.h,v 1.15 2005/12/04 15:00:25 miod Exp $ */
+/*	$OpenBSD: cmmu.h,v 1.16 2005/12/11 21:45:28 miod Exp $ */
 /*
  * Mach Operating System
  * Copyright (c) 1993-1992 Carnegie Mellon University
@@ -67,6 +67,9 @@ struct cmmu_p {
 	void (*flush_data_cache)(cpuid_t, paddr_t, psize_t);
 	int (*dma_cachectl)(pmap_t, vaddr_t, vsize_t, int);
 	int (*dma_cachectl_pa)(paddr_t, psize_t, int);
+#ifdef MULTIPROCESSOR
+	void (*initialize_cpu)(cpuid_t);
+#endif
 };
 
 extern struct cmmu_p *cmmu;
@@ -84,6 +87,7 @@ extern struct cmmu_p *cmmu;
 #define	cmmu_flush_data_cache(a, b, c)	(cmmu->flush_data_cache)(a, b, c)
 #define	dma_cachectl(a, b, c, d)	(cmmu->dma_cachectl)(a, b, c, d)
 #define	dma_cachectl_pa(a, b, c)	(cmmu->dma_cachectl_pa)(a, b, c)
+#define	cmmu_initialize_cpu(a)		(cmmu->initialize_cpu)(a)
 
 /*
  * dma_cachectl() modes
diff --git a/sys/arch/m88k/include/cpu.h b/sys/arch/m88k/include/cpu.h
index 161c7179dd5..bf975efb6ab 100644
--- a/sys/arch/m88k/include/cpu.h
+++ b/sys/arch/m88k/include/cpu.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: cpu.h,v 1.11 2005/12/11 21:36:06 miod Exp $ */
+/*	$OpenBSD: cpu.h,v 1.12 2005/12/11 21:45:28 miod Exp $ */
 /*
  * Copyright (c) 1996 Nivas Madhur
  * Copyright (c) 1992, 1993
@@ -95,7 +95,10 @@ struct cpu_info {
 
 	u_long	ci_spin_locks;			/* spin locks counter */
 
-	/* XXX ddb state? */
+	volatile int ci_ddb_state;		/* ddb status */
+#define	CI_DDB_RUNNING	0
+#define	CI_DDB_ENTERDDB	1
+#define	CI_DDB_INDDB	2
 };
 
 extern cpuid_t master_cpu;
diff --git a/sys/arch/m88k/include/pmap.h b/sys/arch/m88k/include/pmap.h
index e37a19c4c10..4585c8f66a3 100644
--- a/sys/arch/m88k/include/pmap.h
+++ b/sys/arch/m88k/include/pmap.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: pmap.h,v 1.8 2005/12/03 19:06:08 miod Exp $	*/
+/*	$OpenBSD: pmap.h,v 1.9 2005/12/11 21:45:28 miod Exp $	*/
 /*
  * Mach Operating System
  * Copyright (c) 1991 Carnegie Mellon University
@@ -63,6 +63,7 @@ extern	caddr_t		vmmap;
 #define	pmap_clear_reference(pg)	pmap_unsetbit(pg, PG_U)
 
 void	pmap_bootstrap(vaddr_t);
+void	pmap_bootstrap_cpu(cpuid_t);
 void	pmap_cache_ctrl(pmap_t, vaddr_t, vaddr_t, u_int);
 void	pmap_proc_iflush(struct proc *, vaddr_t, vsize_t);
 #define pmap_unuse_final(p)		/* nothing */
diff --git a/sys/arch/m88k/m88k/db_interface.c b/sys/arch/m88k/m88k/db_interface.c
index cc856fd0def..b5f78a06d4e 100644
--- a/sys/arch/m88k/m88k/db_interface.c
+++ b/sys/arch/m88k/m88k/db_interface.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: db_interface.c,v 1.3 2005/12/04 12:20:19 miod Exp $	*/
+/*	$OpenBSD: db_interface.c,v 1.4 2005/12/11 21:45:30 miod Exp $	*/
 /*
  * Mach Operating System
  * Copyright (c) 1993-1991 Carnegie Mellon University
@@ -75,6 +75,13 @@ void	m88k_db_cmmucfg(db_expr_t, int, db_expr_t, char *);
 
 db_regs_t ddb_regs;
 
+#ifdef MULTIPROCESSOR
+#include <sys/mplock.h>
+struct __mp_lock ddb_mp_lock;
+
+void	m88k_db_cpu_cmd(db_expr_t, int, db_expr_t, char *);
+#endif
+
 /*
  * If you really feel like understanding the following procedure and
  * macros, see pages 6-22 to 6-30 (Section 6.7.3) of
@@ -399,6 +406,12 @@ m88k_db_trap(type, frame)
 		}
 	}
 
+#ifdef MULTIPROCESSOR
+	curcpu()->ci_ddb_state = CI_DDB_ENTERDDB;
+	__mp_lock(&ddb_mp_lock);
+	curcpu()->ci_ddb_state = CI_DDB_INDDB;
+#endif
+
 	ddb_regs = frame->tf_regs;
 
 	cnpollc(TRUE);
@@ -406,6 +419,11 @@ m88k_db_trap(type, frame)
 	cnpollc(FALSE);
 
 	frame->tf_regs = ddb_regs;
+
+#ifdef MULTIPROCESSOR
+	curcpu()->ci_ddb_state = CI_DDB_RUNNING;
+	__mp_release_all(&ddb_mp_lock);
+#endif
 }
 
 extern const char *trap_type[];
@@ -598,11 +616,45 @@ m88k_db_frame_search(addr, have_addr, count, modif)
 	db_printf("(Walked back until 0x%x)\n",addr);
 }
 
+#ifdef MULTIPROCESSOR
+
+void
+m88k_db_cpu_cmd(db_expr_t addr, int have_addr, db_expr_t count, char *modif)
+{
+	cpuid_t cpu;
+	struct cpu_info *ci;
+
+	CPU_INFO_FOREACH(cpu, ci) {
+		db_printf("%c%4d: ", (cpu == cpu_number()) ? '*' : ' ',
+		    CPU_INFO_UNIT(ci));
+		switch (ci->ci_ddb_state) {
+		case CI_DDB_RUNNING:
+			db_printf("running\n");
+			break;
+		case CI_DDB_ENTERDDB:
+			db_printf("entering ddb\n");
+			break;
+		case CI_DDB_INDDB:
+			db_printf("ddb\n");
+			break;
+		default:
+			db_printf("? (%d)\n",
+			    ci->ci_ddb_state);
+			break;
+		}
+	}
+}
+
+#endif	/* MULTIPROCESSOR */
+
 /************************/
 /* COMMAND TABLE / INIT */
 /************************/
 
 struct db_command db_machine_cmds[] = {
+#ifdef MULTIPROCESSOR
+	{ "cpu",	m88k_db_cpu_cmd,	0,	NULL },
+#endif
 	{ "frame",	m88k_db_print_frame,	0,	NULL },
 	{ "regs",	m88k_db_registers,	0,	NULL },
 	{ "searchframe",m88k_db_frame_search,	0,	NULL },
@@ -617,4 +669,7 @@ void
 db_machine_init()
 {
 	db_machine_commands_install(db_machine_cmds);
+#ifdef MULTIPROCESSOR
+	__mp_lock_init(&ddb_mp_lock);
+#endif
 }
diff --git a/sys/arch/m88k/m88k/eh_common.S b/sys/arch/m88k/m88k/eh_common.S
index ab1290617ad..868707dae99 100644
--- a/sys/arch/m88k/m88k/eh_common.S
+++ b/sys/arch/m88k/m88k/eh_common.S
@@ -1,4 +1,4 @@
-/*   $OpenBSD: eh_common.S,v 1.15 2005/12/03 18:20:50 miod Exp $	*/
+/*   $OpenBSD: eh_common.S,v 1.16 2005/12/11 21:45:30 miod Exp $	*/
 /*
  * Mach Operating System
  * Copyright (c) 1993-1991 Carnegie Mellon University
@@ -222,9 +222,9 @@
  *	in a rather precarious state and so special cautions must
  * 	be taken.
  */
-#define FLAG_IGNORE_DATA_EXCEPTION	5
-#define FLAG_ENABLING_FPU		7
-#define FLAG_FROM_KERNEL		8
+#define FLAG_IGNORE_DATA_EXCEPTION	0
+#define FLAG_ENABLING_FPU		1
+#define FLAG_FROM_KERNEL		2
 
 /* REGister OFFset into the E.F. (exception frame) */
 #define REG_OFF(reg_num)  ((reg_num) * 4) /* (num * sizeof(register_t))  */
@@ -1591,6 +1591,9 @@ ASLOCAL(m88100_fpu_enable)
  */
 
 ENTRY(proc_trampoline)
+#ifdef MULTIPROCESSOR
+	bsr	_C_LABEL(proc_trampoline_mp)
+#endif
 	ld	r1, r31, 0		/* load func */
 	ld	r2, r31, 4 		/* load proc pointer */
 	jsr.n	r1
diff --git a/sys/arch/m88k/m88k/m8820x_machdep.c b/sys/arch/m88k/m88k/m8820x_machdep.c
index 34310fe0f26..8d25d48b9d7 100644
--- a/sys/arch/m88k/m88k/m8820x_machdep.c
+++ b/sys/arch/m88k/m88k/m8820x_machdep.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: m8820x_machdep.c,v 1.17 2005/12/10 22:31:38 miod Exp $	*/
+/*	$OpenBSD: m8820x_machdep.c,v 1.18 2005/12/11 21:45:30 miod Exp $	*/
 /*
  * Copyright (c) 2004, Miodrag Vallat.
  *
@@ -121,7 +121,10 @@ struct cmmu_p cmmu8820x = {
 	m8820x_flush_inst_cache,
 	m8820x_flush_data_cache,
 	m8820x_dma_cachectl,
-	m8820x_dma_cachectl_pa
+	m8820x_dma_cachectl_pa,
+#ifdef MULTIPROCESSOR
+	m8820x_initialize_cpu,
+#endif
 };
 
 /*
@@ -247,16 +250,10 @@ m8820x_cpu_configuration_print(int main)
 	int proctype = (pid & PID_ARN) >> ARN_SHIFT;
 	int procvers = (pid & PID_VN) >> VN_SHIFT;
 	int mmu, cnt, cpu = cpu_number();
-	static __cpu_simple_lock_t print_lock;
 #ifdef M88200_HAS_SPLIT_ADDRESS
 	int aline, abit, amask;
 #endif
 
-	if (main)
-		__cpu_simple_lock_init(&print_lock);
-
-	__cpu_simple_lock(&print_lock);
-
 	printf("cpu%d: ", cpu);
 	switch (proctype) {
 	default:
@@ -266,8 +263,8 @@ m8820x_cpu_configuration_print(int main)
 	case ARN_88100:
 		printf("M88100 rev 0x%x", procvers);
 #ifdef MULTIPROCESSOR
-		if (max_cpus > 1)
-			printf(", %s", master ? "master" : "slave");
+		if (main == 0)
+			printf(", secondary");
 #endif
 		printf(", %d CMMU", 1 << cmmu_shift);
 
@@ -329,8 +326,6 @@ m8820x_cpu_configuration_print(int main)
 		}
 	}
 #endif
-
-	__cpu_simple_unlock(&print_lock);
 }
 
 /*
@@ -390,8 +385,10 @@ m8820x_initialize_cpu(cpuid_t cpu)
 		 */
 		sctr = cmmu->cmmu_regs[CMMU_SCTR] &
 		    ~(CMMU_SCTR_PE | CMMU_SCTR_SE | CMMU_SCTR_PR);
+#ifdef MULTIPROCESSOR
 		if (max_cpus > 1)
 			sctr |= CMMU_SCTR_SE;
+#endif
 		cmmu->cmmu_regs[CMMU_SCTR] = sctr;
 
 		cmmu->cmmu_regs[CMMU_SAPR] = cmmu->cmmu_regs[CMMU_UAPR] = apr;
diff --git a/sys/arch/m88k/m88k/pmap.c b/sys/arch/m88k/m88k/pmap.c
index d549eafbc37..2c987e1bc3f 100644
--- a/sys/arch/m88k/m88k/pmap.c
+++ b/sys/arch/m88k/m88k/pmap.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: pmap.c,v 1.20 2005/12/03 19:06:11 miod Exp $	*/
+/*	$OpenBSD: pmap.c,v 1.21 2005/12/11 21:45:30 miod Exp $	*/
 /*
  * Copyright (c) 2001-2004, Miodrag Vallat
  * Copyright (c) 1998-2001 Steve Murphree, Jr.
@@ -55,6 +55,7 @@
 
 #include <machine/asm_macro.h>
 #include <machine/cmmu.h>
+#include <machine/cpu.h>
 #include <machine/lock.h>
 #include <machine/pmap_table.h>
 
@@ -157,14 +158,14 @@ vaddr_t kmapva = 0;
  */
 static void flush_atc_entry(long, vaddr_t, boolean_t);
 pt_entry_t *pmap_expand_kmap(vaddr_t, vm_prot_t, int);
-void pmap_remove_pte(pmap_t, vaddr_t, pt_entry_t *);
-void pmap_remove_range(pmap_t, vaddr_t, vaddr_t);
-void pmap_expand(pmap_t, vaddr_t);
-void pmap_release(pmap_t);
-vaddr_t pmap_map(vaddr_t, paddr_t, paddr_t, vm_prot_t, u_int);
+void	pmap_remove_pte(pmap_t, vaddr_t, pt_entry_t *);
+void	pmap_remove_range(pmap_t, vaddr_t, vaddr_t);
+void	pmap_expand(pmap_t, vaddr_t);
+void	pmap_release(pmap_t);
+vaddr_t	pmap_map(vaddr_t, paddr_t, paddr_t, vm_prot_t, u_int);
 pt_entry_t *pmap_pte(pmap_t, vaddr_t);
-void pmap_remove_all(struct vm_page *);
-void pmap_changebit(struct vm_page *, int, int);
+void	pmap_remove_all(struct vm_page *);
+void	pmap_changebit(struct vm_page *, int, int);
 boolean_t pmap_unsetbit(struct vm_page *, int);
 boolean_t pmap_testbit(struct vm_page *, int);
 
@@ -739,6 +740,25 @@ pmap_bootstrap(vaddr_t load_start)
 #endif	/* MULTIPROCESSOR */
 }
 
+#ifdef MULTIPROCESSOR
+void
+pmap_bootstrap_cpu(cpuid_t cpu)
+{
+	if (cpu != master_cpu) {
+		cmmu_initialize_cpu(cpu);
+	} else {
+		cmmu_flush_tlb(cpu, TRUE, VM_MIN_KERNEL_ADDRESS,
+		    btoc(VM_MAX_KERNEL_ADDRESS - VM_MIN_KERNEL_ADDRESS));
+	}
+	/* Load supervisor pointer to segment table. */
+	cmmu_set_sapr(cpu, kernel_pmap->pm_apr);
+#ifdef DEBUG
+	printf("cpu%d: running virtual\n", cpu);
+#endif
+	SETBIT_CPUSET(cpu, &kernel_pmap->pm_cpus);
+}
+#endif
+
 /*
  * Routine:	PMAP_INIT
  *
@@ -1021,7 +1041,6 @@ pmap_destroy(pmap_t pmap)
 void
 pmap_reference(pmap_t pmap)
 {
-
 	PMAP_LOCK(pmap);
 	pmap->pm_count++;
 	PMAP_UNLOCK(pmap);
@@ -1270,7 +1289,6 @@ pmap_remove(pmap_t pmap, vaddr_t s, vaddr_t e)
  *	pv lists
  *
  * Calls:
- *	__cpu_simple_lock
  *	pmap_pte
  *	pool_put
  *
@@ -2211,7 +2229,6 @@ next:
  *	pv lists
  *
  * Calls:
- *	simple_lock, simple_unlock
  *	pmap_pte
  *
  * If the attribute list for the given page has the bit, this routine
diff --git a/sys/arch/m88k/m88k/process.S b/sys/arch/m88k/m88k/process.S
index 62eb1540fff..7773baf15d2 100644
--- a/sys/arch/m88k/m88k/process.S
+++ b/sys/arch/m88k/m88k/process.S
@@ -1,4 +1,4 @@
-/*	$OpenBSD: process.S,v 1.12 2005/12/07 07:39:35 miod Exp $	*/
+/*	$OpenBSD: process.S,v 1.13 2005/12/11 21:45:30 miod Exp $	*/
 /*
  * Copyright (c) 1996 Nivas Madhur
  * All rights reserved.
@@ -121,16 +121,7 @@ ENTRY(cpu_switch)
 	bsr.n	_C_LABEL(pmap_deactivate)
 	 st	r0,  r11, CI_CURPROC		/* curproc = NULL */
 
-#ifdef MULTIPROCESSOR
-ASGLOBAL(cpu_switch_search)
-#else
 ASLOCAL(cpu_switch_search)
-#endif
-
-#if defined(MULTIPROCESSOR) || defined(LOCKDEBUG)
-	bsr	_C_LABEL(sched_unlock_idle)
-#endif
-
 	/*
 	 * This is the start of the idle loop. Find the highest-priority
 	 * queue that isn't empty, then take the first proc from that queue.
@@ -139,47 +130,49 @@ ASLOCAL(cpu_switch_search)
 	ld	r7, r7, lo16(_C_LABEL(whichqs))
 	bcnd	ne0, r7, _ASM_LABEL(cpu_switch_found)
 
-ASLOCAL(cpu_switch_idle)
+#if defined(MULTIPROCESSOR) || defined(LOCKDEBUG)
+	bsr	_C_LABEL(sched_unlock_idle)
+#endif
+
+#ifdef MULTIPROCESSOR
+ASGLOBAL(cpu_switch_idle)
+#endif
 	/*
 	 * There were no runnable processes. Enable all interrupts and
 	 * busy-wait for this to change.
-	 * Note that, besides doing spl0(), this will actually enable
+	 * Note that, besides doing setipl(IPL_NONE), this will actually enable
 	 * interrupts in the psr. Bootstrap of secondary processors
 	 * relies upon this.
 	 */
-	bsr.n	_C_LABEL(setipl)
-	 or	r2, r0, IPL_NONE
-
 	ldcr	r2, PSR
-	bb0.n	PSR_INTERRUPT_DISABLE_BIT, r2, 2f
-	 or.u	r7, r0, hi16(_C_LABEL(whichqs))
+	bb0	PSR_INTERRUPT_DISABLE_BIT, r2, 2f
 	clr	r2, r2, 1<PSR_INTERRUPT_DISABLE_BIT>
 	stcr	r2, PSR
 	FLUSH_PIPELINE
 2:
+	bsr.n	_C_LABEL(setipl)
+	 or	r2, r0, IPL_NONE
+
+	or.u	r7, r0, hi16(_C_LABEL(whichqs))
 	ld	r7, r7, lo16(_C_LABEL(whichqs))
-	bcnd	eq0, r7, _ASM_LABEL(cpu_switch_idle)
+	bcnd	eq0, r7, 2b
 	/* XXX run fancy things here, such as page zeroing... */
 
-ASLOCAL(cpu_switch_found)
-	bsr.n	_C_LABEL(setipl)		/* disable interrupts */
-	 or	r2, r0, IPL_HIGH
-
 #if defined(MULTIPROCESSOR) || defined(LOCKDEBUG)
 	bsr	_C_LABEL(sched_lock_idle)
 #endif
 
+ASLOCAL(cpu_switch_found)
+	bsr.n	_C_LABEL(setipl)		/* disable interrupts */
+	 or	r2, r0, IPL_HIGH
+
 	/*
 	 * An interrupt could have occured between the last whichqs check
 	 * and the call to setipl(). Check again that whichqs is nonzero.
 	 */
 	or.u	r7, r0, hi16(_C_LABEL(whichqs))	/* reload whichqs */
 	ld	r7, r7, lo16(_C_LABEL(whichqs))
-#if defined(MULTIPROCESSOR) || defined(LOCKDEBUG)
 	bcnd	eq0, r7, _ASM_LABEL(cpu_switch_search)
-#else
-	bcnd	eq0, r7, _ASM_LABEL(cpu_switch_idle)
-#endif
 
 	/* XXX use ff1, like powerpc... needs *runqueue() adjustments */
 	xor	r6, r6, r6		/* set r6 to 0 */
@@ -240,10 +233,6 @@ ASLOCAL(cpu_switch_found)
 	bsr.n	_C_LABEL(pmap_activate)
 	 or	r2, r0, r9
 
-#if defined(MULTIPROCESSOR) || defined(LOCKDEBUG)
-	bsr	_C_LABEL(sched_unlock_idle)
-#endif
-
 	ldcr	r10,  CPU
 	ld	r10,  r10, CI_CURPCB
 
@@ -273,8 +262,11 @@ ASLOCAL(cpu_switch_found)
 	ld	r1,  r14, PCB_PC
 	ld	r30, r14, PCB_R30	/* restore frame pointer & stack */
 	ld	r31, r14, PCB_SP
-
+#if defined(MULTIPROCESSOR) || defined(LOCKDEBUG)
+	br.n	_C_LABEL(sched_unlock_idle)
+#else
 	jmp.n	r1
+#endif
 	 ld	r14, r14, PCB_R14
 
 /*
diff --git a/sys/arch/m88k/m88k/trap.c b/sys/arch/m88k/m88k/trap.c
index 206ef090931..52ffff4ef31 100644
--- a/sys/arch/m88k/m88k/trap.c
+++ b/sys/arch/m88k/m88k/trap.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: trap.c,v 1.28 2005/12/11 21:30:30 miod Exp $	*/
+/*	$OpenBSD: trap.c,v 1.29 2005/12/11 21:45:30 miod Exp $	*/
 /*
  * Copyright (c) 2004, Miodrag Vallat.
  * Copyright (c) 1998 Steve Murphree, Jr.
@@ -1109,9 +1109,9 @@ __dead void
 error_fatal(struct trapframe *frame)
 {
 	if (frame->tf_vector == 0)
-		printf("\nReset Exception\n");
+		printf("\nCPU %d Reset Exception\n", cpu_number());
 	else
-		printf("\nError Exception\n");
+		printf("\nCPU %d Error Exception\n", cpu_number());
 
 #ifdef DDB
 	regdump((struct trapframe*)frame);
diff --git a/sys/arch/m88k/m88k/vm_machdep.c b/sys/arch/m88k/m88k/vm_machdep.c
index 53f5f181f5d..901b047e8ac 100644
--- a/sys/arch/m88k/m88k/vm_machdep.c
+++ b/sys/arch/m88k/m88k/vm_machdep.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: vm_machdep.c,v 1.9 2005/12/08 07:02:36 miod Exp $	*/
+/*	$OpenBSD: vm_machdep.c,v 1.10 2005/12/11 21:45:30 miod Exp $	*/
 
 /*
  * Copyright (c) 1998 Steve Murphree, Jr.
@@ -146,10 +146,9 @@ cpu_fork(p1, p2, stack, stacksize, func, arg)
 void
 cpu_exit(struct proc *p)
 {
-	pmap_deactivate(p);
-
 	splhigh();
 
+	pmap_deactivate(p);
 	uvmexp.swtch++;
 	switch_exit(p);
 	/* NOTREACHED */
diff --git a/sys/arch/mvme88k/dev/bugio.c b/sys/arch/mvme88k/dev/bugio.c
index 3d00a93deca..0cdc177c4e2 100644
--- a/sys/arch/mvme88k/dev/bugio.c
+++ b/sys/arch/mvme88k/dev/bugio.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: bugio.c,v 1.15 2005/10/13 19:43:43 miod Exp $ */
+/*	$OpenBSD: bugio.c,v 1.16 2005/12/11 21:45:31 miod Exp $ */
 /*  Copyright (c) 1998 Steve Murphree, Jr. */
 
 #include <sys/param.h>
@@ -8,7 +8,7 @@
 #include <machine/bugio.h>
 #include <machine/prom.h>
 
-register_t ossr0, ossr1, ossr2, ossr3;	/* XXX ought to be per-cpu */
+register_t ossr0, ossr1, ossr2, ossr3;
 register_t bugsr3;
 
 unsigned long bugvec[2], sysbugvec[2];
@@ -16,6 +16,16 @@ unsigned long bugvec[2], sysbugvec[2];
 void bug_vector(void);
 void sysbug_vector(void);
 
+#ifdef MULTIPROCESSOR
+#include <sys/lock.h>
+__cpu_simple_lock_t bug_lock = __SIMPLELOCK_UNLOCKED;
+#define	BUG_LOCK()	__cpu_simple_lock(&bug_lock)
+#define	BUG_UNLOCK()	__cpu_simple_unlock(&bug_lock)
+#else
+#define	BUG_LOCK()	do { } while (0)
+#define	BUG_UNLOCK()	do { } while (0)
+#endif
+
 #define MVMEPROM_CALL(x)						\
 	__asm__ __volatile__ ("or r9,r0," __STRING(x));			\
 	__asm__ __volatile__ ("tb0 0,r0,496" :::			\
@@ -44,6 +54,7 @@ sysbug_vector()
 
 #define	BUGCTXT()							\
 {									\
+	BUG_LOCK();							\
 	disable_interrupt(psr);			/* paranoia */		\
 	bug_vector();							\
 	__asm__ __volatile__ ("ldcr %0, cr17" : "=r" (ossr0));		\
@@ -64,6 +75,7 @@ sysbug_vector()
 	__asm__ __volatile__ ("stcr %0, cr20" :: "r"(ossr3));		\
 	sysbug_vector();						\
 	set_psr(psr);							\
+	BUG_UNLOCK();							\
 }
 
 static void
@@ -123,7 +135,7 @@ buginstat(void)
 	MVMEPROM_CALL(MVMEPROM_INSTAT);
 	__asm__ __volatile__  ("or %0,r0,r2" : "=r" (ret));
 	OSCTXT();
-	return (ret & 0x4 ? 0 : 1);
+	return ((ret & 0x08) >> 3);
 }
 
 void
@@ -180,3 +192,26 @@ bugdiskrd(struct mvmeprom_dskio *dio)
 	MVMEPROM_CALL(MVMEPROM_DSKRD);
 	OSCTXT();
 }
+
+#ifdef MULTIPROCESSOR
+
+/*
+ * Ask the BUG to start a particular cpu at our provided address.
+ */
+int
+spin_cpu(cpuid_t cpu, vaddr_t address)
+{
+	u_int psr;
+	int ret;
+
+	BUGCTXT();
+	__asm__ __volatile__ ("or r2, r0, %0" : : "r" (cpu));
+	__asm__ __volatile__ ("or r3, r0, %0" : : "r" (address));
+	MVMEPROM_CALL(MVMEPROM_FORKMPU);
+	__asm__ __volatile__ ("or %0,r0,r2" : "=r" (ret));
+	OSCTXT();
+
+	return (ret);
+}
+
+#endif	/* MULTIPROCESSOR */
diff --git a/sys/arch/mvme88k/dev/mainbus.c b/sys/arch/mvme88k/dev/mainbus.c
index 31feeb475a6..2fd42641548 100644
--- a/sys/arch/mvme88k/dev/mainbus.c
+++ b/sys/arch/mvme88k/dev/mainbus.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: mainbus.c,v 1.17 2005/11/25 22:14:31 miod Exp $ */
+/*	$OpenBSD: mainbus.c,v 1.18 2005/12/11 21:45:31 miod Exp $ */
 /*
  * Copyright (c) 1998 Steve Murphree, Jr.
  * Copyright (c) 2004, Miodrag Vallat.
@@ -276,7 +276,7 @@ mainbus_attach(parent, self, args)
 	printf(": %s\n", cpu_model);
 
 	/*
-	 * Display cpu/mmu details. Only for the master CPU so far.
+	 * Display cpu/mmu details for the main processor.
 	 */
 	cpu_configuration_print(1);
 
diff --git a/sys/arch/mvme88k/include/bugio.h b/sys/arch/mvme88k/include/bugio.h
index cdc62d71d66..4401834e8e3 100644
--- a/sys/arch/mvme88k/include/bugio.h
+++ b/sys/arch/mvme88k/include/bugio.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: bugio.h,v 1.14 2004/01/04 01:14:06 miod Exp $ */
+/*	$OpenBSD: bugio.h,v 1.15 2005/12/11 21:45:31 miod Exp $ */
 
 #ifndef __MACHINE_BUGIO_H__
 #define __MACHINE_BUGIO_H__
@@ -16,5 +16,6 @@ void bugrtcrd(struct mvmeprom_time *);
 void bugreturn(void);
 void bugbrdid(struct mvmeprom_brdid *);
 void bugdiskrd(struct mvmeprom_dskio *);
+int spin_cpu(cpuid_t, vaddr_t);
 
 #endif /* __MACHINE_BUGIO_H__ */
diff --git a/sys/arch/mvme88k/include/prom.h b/sys/arch/mvme88k/include/prom.h
index 62b97c577b2..a392d5cd117 100644
--- a/sys/arch/mvme88k/include/prom.h
+++ b/sys/arch/mvme88k/include/prom.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: prom.h,v 1.14 2004/08/02 14:38:40 miod Exp $ */
+/*	$OpenBSD: prom.h,v 1.15 2005/12/11 21:45:31 miod Exp $ */
 /*
  * Copyright (c) 1998 Steve Murphree, Jr.
  * Copyright (c) 1996 Nivas Madhur
@@ -70,6 +70,10 @@
 #define ENVIRONTYPE_NETBOOT	4
 #define ENVIRONTYPE_MEMSIZE	5
 
+#define	FORKMPU_NOT_IDLE	-1
+#define	FORKMPU_BAD_ADDRESS	-2
+#define	FORKMPU_NO_MPU		-3
+
 #ifndef LOCORE
 struct mvmeprom_netctrl {
 	u_char	ctrl;
diff --git a/sys/arch/mvme88k/mvme88k/locore.S b/sys/arch/mvme88k/mvme88k/locore.S
index 6fabd59e1c2..cce93535602 100644
--- a/sys/arch/mvme88k/mvme88k/locore.S
+++ b/sys/arch/mvme88k/mvme88k/locore.S
@@ -1,5 +1,6 @@
-/*	$OpenBSD: locore.S,v 1.45 2005/12/03 18:20:51 miod Exp $	*/
+/*	$OpenBSD: locore.S,v 1.46 2005/12/11 21:45:31 miod Exp $	*/
 /*
+ * Copyright (c) 2005, Miodrag Vallat.
  * Copyright (c) 1998 Steve Murphree, Jr.
  * Copyright (c) 1996 Nivas Madhur
  * All rights reserved.
@@ -60,13 +61,11 @@
 #include "assym.h"
 #include "ksyms.h"
 
-#include <sys/errno.h>
-
 #include <machine/asm.h>
-#include <machine/trap.h>
 #include <machine/m88100.h>
-#include <machine/psl.h>
 #include <machine/param.h>
+#include <machine/psl.h>
+#include <machine/trap.h>
 #include <machine/vmparam.h>
 
 /*
@@ -79,68 +78,29 @@
 	text
 
 GLOBAL(kernelstart)
-GLOBAL(kernel_text)
-GLOBAL(start)
 ASGLOBAL(start)
-	br	_C_LABEL(start_text)
-	br	_C_LABEL(start_text)
-	br	_C_LABEL(start_text)
-	br	_C_LABEL(start_text)
-
-GLOBAL(doboot)
 	/*
-	 * Try hitting the SRST bit in VMEchip2 to reset the system.
+	 * A few identical jump instructions to make sure the pipeline is
+	 * in a good state. Probably overkill, but it's cheap.
 	 */
-#ifdef MVME188
-	/* check if it's a mvme188 */
-	or.u	r4, r0, hi16(_C_LABEL(brdtyp))
-	ld	r3, r4, lo16(_C_LABEL(brdtyp))
-	cmp	r4, r3, BRD_188
-	bb1	ne, r4, 1f
-	bsr	_C_LABEL(m188_reset)
-	br	8f
-1:
-#endif	/* MVME188 */
-	or.u	r3, r0, 0xfff4
-	ld	r4, r3, 0x0060		/* read offset (LCSR + 0x60) */
-	bb0.n	30, r4, 1f		/* if not SYSCON, don't SYSRST */
-	 set	r4, r4, 1<23>		/* set SYSRST bit - bit 23 */
-	st	r4, r3, 0x0060		/* and store it back  */
-1:
-	ld	r4, r3, 0x0104		/* try local reset, then */
-	set	r4, r4, 1<7>
-	st	r4, r3, 0x0104
+	br	_ASM_LABEL(main_start)
+	br	_ASM_LABEL(main_start)
+	br	_ASM_LABEL(main_start)
+	br	_ASM_LABEL(main_start)
 
 	/*
-	 * We will be here if the reset above failed. In this case,
-	 * we will try to return to bug.
-	 *
-	 * Switch to interrupt stack and call _doboot to take care
-	 * going to BUG. Need to do this since _doboot turns off the
-	 * the MMU and we need to be on a 1-to-1 mapped stack so that
-	 * further calls don't get data access exceptions.
+	 * Startup code for main processor.
 	 */
-
-	/* Should we use idle_u instead? XXX nivas */
-8:
-	or.u	r31, r0,  hi16(_ASM_LABEL(intstack_end))
-	or	r31, r31, lo16(_ASM_LABEL(intstack_end))
-	clr	r31, r31, 3<0>	/* round down to 8-byte boundary */
-
-	bsr	_C_LABEL(_doboot)
-	/*NOTREACHED*/
-
-	/* This is the *real* start upon poweron or reset */
-GLOBAL(start_text)
+ASLOCAL(main_start)
 	/*
-	 * Args passed by boot loader
-	 * 	r2 howto
-	 *	r3 boot controller address
-	 *	r4 esym
-	 *	r5 start of mini
-	 *	r6 end miniroot
-	 *	r7 ((Clun << 8) ; Dlun & FF) -> bootdev
-	 *	r8 cpu type (0x187, 0x188, 0x197)
+	 * Save the arguments passed by the boot loader
+	 * 	r2 boot flags
+	 *	r3 boot controller physical address
+	 *	r4 esym (if applicable)
+	 *	r5 start of miniroot (unused)
+	 *	r6 end of miniroot (unused)
+	 *	r7 ((Clun << 8) | Dlun): encoded bootdev
+	 *	r8 board type (0x187, 0x188, 0x197)
 	 */
 /*
  * (*entry)(flag, bugargs.ctrl_addr, cp, kernel.smini,kernel.emini,
@@ -161,7 +121,7 @@ GLOBAL(start_text)
 	or.u	r13, r0,  hi16(_C_LABEL(brdtyp))
 	st	r8,  r13, lo16(_C_LABEL(brdtyp))
 
-	/* set _cputyp */
+	/* set cputyp */
 	cmp	r2,  r8, BRD_197	/* r8 contains brdtyp */
 	bb1	ne, r2, 1f	/* if it's a '197, CPU is 88110 */
 	or.u	r13, r0,  hi16(CPU_88110)
@@ -177,10 +137,7 @@ GLOBAL(start_text)
 	/*
 	 * CPU Initialization
 	 *
-	 * Every CPU starts from here..
-	 * (well, from 'start' above, which just jumps here).
-	 *
-	 * I use r11 and r22 here 'cause they're easy to not
+	 * I use r11 and r22 here because they're easy to not
 	 * get mixed up -- r10, for example, looks too similar
 	 * to r0 when not being careful....
 	 *
@@ -233,89 +190,25 @@ GLOBAL(start_text)
 	FLUSH_PIPELINE
 	stcr	r0,  VBR	/* set Vector Base Register to 0, ALWAYS! */
 	FLUSH_PIPELINE
-	cmp	r2, r8, CPU_88110	/* r8 contains cputyp */
-	bb1	eq, r2, _ASM_LABEL(master_start) /* if it's a '197, skip */
-
-#if 0
-	/* clear BSS. Boot loader might have already done this... */
-	or.u	r2, r0, hi16(_C_LABEL(edata))
-	or	r2, r2, lo16(_C_LABEL(edata))
-	or.u	r4, r0, hi16(_C_LABEL(end))
-	or	r4, r4, lo16(_C_LABEL(end))
-	bsr.n	_bzero		/* bzero(edata, end-edata) */
-	 subu	r3, r4, r2
-#endif
-	/*
-	 * First time to count how many CPUs to attach
-	 */
-	or.u	r11, r0,  hi16(_ASM_LABEL(initialized_cpu_lock))
-	or	r11, r11, lo16(_ASM_LABEL(initialized_cpu_lock))
-1:
-	FLUSH_PIPELINE
-	or	r22, r0,  1
-	xmem	r22, r11, r0	/* If r22 gets 0, we have the lock.. */
-	bcnd	eq0, r22, 3f	/* ..but if not, we must wait */
-2:
-	/* just watch the lock until it looks clear */
-	ld	r22, r11, r0
-	bcnd	eq0, r22, 1b
-	br	2b		/* looks clear -- try to grab */
-3:
-	FLUSH_PIPELINE
-	or.u	r11, r0,  hi16(_ASM_LABEL(initialized_cpus))
-	ld	r22, r11, lo16(_ASM_LABEL(initialized_cpus))
-	add	r23, r22, 1
-	st	r23, r11, lo16(_ASM_LABEL(initialized_cpus))
 
-	or.u	r11, r0,  hi16(_ASM_LABEL(initialized_cpu_lock))
-	st	r0,  r11, lo16(_ASM_LABEL(initialized_cpu_lock))
+#ifdef MULTIPROCESSOR
 	/*
-	 * Now we view with any other processors to see who's the master.
-	 * We first try to obtain a lock to see who's allowed
-	 * to check/set the master lock.
+	 * Have curcpu() point at the dummy cpuinfo structure,
+	 * so that cpu_number() does not dereference random memory.
+	 * This is necessary for early spl usage, despite the fact that
+	 * interrupts are disabled...
 	 */
-	or.u	r11, r0,  hi16(_ASM_LABEL(inter_processor_lock))
-	or	r11, r11, lo16(_ASM_LABEL(inter_processor_lock))
-1:
-	FLUSH_PIPELINE
-	or	r22, r0,  1
-	xmem	r22, r11, r0		/* If r22 gets 0, we have the lock.. */
-	bcnd	eq0, r22, 4f		/* ..but if not, we must wait */
-2:
-	/* just watch the lock until it looks clear */
-	ld	r22, r11, r0
-	bcnd	ne0, r22, 2b
-	/* since we can be here with caches off, add a few nops to
-	   keep the bus from getting overloaded */
-	or	r2, r0, lo16(1000)
-3:
-	subu	r2, r2, 1
-	bcnd	eq0, r2, 3b
-	br	1b			/* looks clear -- try to grab */
-4:
-	/* now try to grab the master_processor_chosen prize */
-	FLUSH_PIPELINE
-	or.u	r11, r0,  hi16(_ASM_LABEL(master_processor_chosen))
-	or	r11, r11, lo16(_ASM_LABEL(master_processor_chosen))
-	or	r22, r0,  1
-	xmem	r22, r11, r0
+	or.u	r11, r0,  hi16(_ASM_LABEL(dummy_cpu))
+	or	r11, r11, lo16(_ASM_LABEL(dummy_cpu))
+	stcr	r11, CPU
 
 	/*
-	 * If r22 is not clear we're a slave,
-	 * otherwise we're first and the master.
-	 *
-	 * Note that we haven't released the interprocessor lock....
-	 * We'll do that when we're ready for another CPU to go.
-	 * (if we're the master, we'll do that in master_start below.
-	 *  if we're a slave, we'll do it in slave_start below).
+	 * MVME BUG idles all secondary MPUs upon startup, so at this point
+	 * we do not have to compete with them.
 	 */
-	bcnd	ne0, r22, _ASM_LABEL(slave_start)
-	/* fall through to master start if that's appropriate */
+#endif	/* MULTIPROCESSOR */
 
-ASLOCAL(master_start)
-	/*
-	 * Switch to interrupt stack
-	 */
+	/* Switch to interrupt stack */
 	or.u	r31, r0,  hi16(_ASM_LABEL(intstack_end))
 	or	r31, r31, lo16(_ASM_LABEL(intstack_end))
 
@@ -349,10 +242,10 @@ ASLOCAL(master_start)
 	 */
 	ldcr	r10, CPU
 	ld	r31, r10, CI_CURPCB
-	addu	r31, r31, USIZE
 
 	/* call main() - no arguments although main() still defines one */
-	bsr	_C_LABEL(main)
+	bsr.n	_C_LABEL(main)
+	 addu	r31, r31, USIZE
 
 	or.u	r2, r0, hi16(_ASM_LABEL(main_panic))
 	bsr.n	_C_LABEL(panic)
@@ -365,44 +258,145 @@ ASLOCAL(main_panic)
 	text
 	.align	8
 
-/*
- *	slave CPUs starts here
- */
-ASLOCAL(slave_start)
+#ifdef MULTIPROCESSOR
+
 	/*
-	 * While holding the inter_processor_lock, the slave cpu can use
-	 * the slavestack to call slave_pre_main and determine its cpu number.
-	 * After that, however, it should switch over to the interrupt stack
-	 * associated with its cpu.
+	 * Startup code for secondary processors.
+	 * Some of these initializations are very close to main_start; refer
+	 * to the comments there for details.
 	 */
-
-	/* r31 <-- slavestack */
+GLOBAL(secondary_start)
 	or.u	r31, r0,  hi16(_ASM_LABEL(slavestack_end))
 	or	r31, r31, lo16(_ASM_LABEL(slavestack_end))
 
-	bsr.n	_C_LABEL(slave_pre_main)	/* set cpu number */
-	 clr	r31, r31, 3<0>	/* round down to 8-byte boundary */
+	or.u	r13, r0,  hi16(_C_LABEL(cputyp))
+	ld	r8,  r13, lo16(_C_LABEL(cputyp))
+
+	cmp	r2, r8, CPU_88110
+	bb1	eq, r2, 1f
+	stcr	r0, SSBR
+1:
+	stcr	r0, SR1
+
+	set	r11, r0,  1<PSR_SUPERVISOR_MODE_BIT>
+	set	r11, r11, 1<PSR_INTERRUPT_DISABLE_BIT>
+	set	r11, r11, 1<PSR_GRAPHICS_DISABLE_BIT>
+	/*
+	 * XXX On 88110 processors, force serial instruction execution for now.
+	 * Situation where OoO would break will be hopefully taken care of in
+	 * the near future -- miod
+	 */
+#if 0
+	clr	r11, r11, 1<PSR_SERIAL_MODE_BIT>
+#else
+	set	r11, r11, 1<PSR_SERIAL_MODE_BIT>
+#endif
+	set	r11, r11, 1<PSR_SERIALIZE_BIT>
+	stcr	r11, PSR
+	FLUSH_PIPELINE
+	stcr	r0,  VBR	/* set Vector Base Register to 0, ALWAYS! */
+	FLUSH_PIPELINE
 
-	bsr	_C_LABEL(get_slave_stack)
-	addu	r31, r2, INTSTACK_SIZE
+	/*
+	 * Have curcpu() point at the dummy cpuinfo structure,
+	 * so that cpu_number() does not dereference random memory.
+	 * This is necessary for early spl usage, despite the fact that
+	 * interrupts are disabled...
+	 */
+	or.u	r11, r0,  hi16(_ASM_LABEL(dummy_cpu))
+	or	r11, r11, lo16(_ASM_LABEL(dummy_cpu))
+	stcr	r11, CPU
 
 	/*
-	 * SR1 now contains our cpu number. We can now release the
-	 * inter_processor_lock, as we are done with the slavestack.
-	 * We also have an interrupt stack
+	 * Since there may be more than one secondary MPU, compete with them
+	 * to initialize safely.
+	 */
+	or.u	r11, r0,  hi16(_C_LABEL(cpu_mutex))
+	or	r11, r11, lo16(_C_LABEL(cpu_mutex))
+1:
+	or	r22, r0,  1
+	xmem	r22, r11, r0		/* If r22 gets 0, we have the lock... */
+	bcnd	eq0, r22, 4f		/* ...but if not, we must wait */
+2:
+	/* just watch the lock until it looks clear */
+	ld	r22, r11, r0
+	bcnd	eq0, r22, 1b
+	/* since we can be here with caches off, add a few nops to
+	   keep the bus from getting overloaded */
+	or	r2,  r0,  lo16(1000)
+3:
+	subu	r2,  r2,  1
+	bcnd	ne0, r2,  3b
+	br	1b
+4:
+	
+	/*
+	 * While holding the cpu_mutex, the secondary cpu can use the slavestack
+	 * to call secondary_pre_main() to determine its cpu number.
+	 * After that, however, it should allocate its own stack and switch
+	 * to it.
 	 */
 
-	or.u	r10, r0, hi16(_ASM_LABEL(inter_processor_lock))
-	st	r0, r10, lo16(_ASM_LABEL(inter_processor_lock))
+	bsr	_C_LABEL(secondary_pre_main)	/* set cpu number */
 
-	br	_C_LABEL(slave_main)	/* does not return */
+	ldcr	r2, CPU
+	ld	r3, r2, CI_IDLE_PCB
+	bsr.n	_C_LABEL(secondary_main)
+	 addu	r31, r3, USIZE			/* switch to idle stack */
 
-GLOBAL(spin_cpu)
-	or.u	r3,   r0,  hi16(_C_LABEL(start_text))
-	or	r3,   r3,  lo16(_C_LABEL(start_text))
-	or	r9,   r0,  0x100		/* .FORKMPU */
-	tb0	0,    r0,  0x200-16		/* call 188Bug */
-	jmp	r1
+	/*
+	 * At this point, the CPU has been correctly initialized and has
+	 * identified itself on the console.
+	 * All it needs now is to jump to the idle loop and wait for work to
+	 * be offered.
+	 */
+	br	_ASM_LABEL(cpu_switch_idle)
+
+#endif	/* MULTIPROCESSOR */
+
+/*
+ * Reset code.
+ * Should be rewritten in C eventually.
+ */
+GLOBAL(doboot)
+#ifdef MVME188
+	/* check if it's a mvme188 */
+	or.u	r4, r0, hi16(_C_LABEL(brdtyp))
+	ld	r3, r4, lo16(_C_LABEL(brdtyp))
+	cmp	r4, r3, BRD_188
+	bb1	ne, r4, 1f
+	bsr	_C_LABEL(m188_reset)
+	br	8f
+1:
+#endif	/* MVME188 */
+	/*
+	 * Try hitting the SRST bit in VMEchip2 to reset the system.
+	 */
+	or.u	r3, r0, 0xfff4
+	ld	r4, r3, 0x0060		/* read offset (LCSR + 0x60) */
+	bb0.n	30, r4, 1f		/* if not SYSCON, don't SYSRST */
+	 set	r4, r4, 1<23>		/* set SYSRST bit - bit 23 */
+	st	r4, r3, 0x0060		/* and store it back  */
+1:
+	ld	r4, r3, 0x0104		/* try local reset, then */
+	set	r4, r4, 1<7>
+	st	r4, r3, 0x0104
+
+	/*
+	 * We will be here if the reset above failed. In this case,
+	 * we will try to return to bug.
+	 *
+	 * Switch to interrupt stack and call _doboot to take care
+	 * going to BUG. Need to do this since _doboot turns off the
+	 * the MMU and we need to be on a 1-to-1 mapped stack so that
+	 * further calls don't get data access exceptions.
+	 */
+
+8:
+	or.u	r31, r0,  hi16(_ASM_LABEL(intstack_end))
+	bsr.n	_C_LABEL(_doboot)
+	 or	r31, r31, lo16(_ASM_LABEL(intstack_end))
+	/*NOTREACHED*/
 
 /*
  * MVME188 specific support routines
@@ -441,17 +435,14 @@ GLOBAL(kernel_sdt)		/* SDT (segment descriptor table */
 ASGLOBAL(intstack)
 	space	USIZE
 ASGLOBAL(intstack_end)
-ASGLOBAL(slavestack)
+
+#ifdef MULTIPROCESSOR
 	space	PAGE_SIZE	/* 4K, small, interim stack */
-ASGLOBAL(slavestack_end)
+ASLOCAL(slavestack_end)
+#endif
 
 /*
- * When a process exits and its u. area goes away, we set curpcb to point
- * to this `u.', leaving us with something to use for an interrupt stack,
- * and letting all the register save code have a pcb_uw to examine.
- * This is also carefully arranged (to come just before u0, so that
- * process 0's kernel stack can quietly overrun into it during bootup, if
- * we feel like doing that).
+ * Main processor's idle pcb and stack.
  * Should be page aligned.
  */
 	.align	PAGE_SIZE
@@ -460,8 +451,7 @@ GLOBAL(idle_u)
 
 /*
  * Process 0's u.
- *
- * This must be page aligned
+ * Should be page aligned.
  */
 	.align	PAGE_SIZE
 ASLOCAL(u0)
@@ -469,20 +459,14 @@ ASLOCAL(u0)
 GLOBAL(proc0paddr)
 	word	_ASM_LABEL(u0)	/*  KVA of proc0 uarea */
 
-GLOBAL(ret_addr)
-	word 0
-/* XMEM spin lock -- to count CPUs */
-ASLOCAL(initialized_cpu_lock)
-	word 0
-/* CPU counter to initialize */
-ASLOCAL(initialized_cpus)
-	word 0
-/* The first processor that XMEMs this becomes the master */
-ASLOCAL(master_processor_chosen)
-	word 0
-/* XMEM spin lock -- controls access to master_processor_chosen */
-ASLOCAL(inter_processor_lock)
-	word 0
+#ifdef MULTIPROCESSOR
+/* Dummy cpuinfo structure, for cpu_number() to work early. */
+ASLOCAL(dummy_cpu)
+	word	1	/* ci_alive */
+	word	0	/* ci_curproc */
+	word	0	/* ci_curpcb */
+	word	0	/* ci_cpuid */
+#endif	/* MULTIPROCESSOR */
 
 #if defined(DDB) || NKSYMS > 0
 GLOBAL(esym)
diff --git a/sys/arch/mvme88k/mvme88k/m88110.c b/sys/arch/mvme88k/mvme88k/m88110.c
index 18ddd7806a6..227b2f26665 100644
--- a/sys/arch/mvme88k/mvme88k/m88110.c
+++ b/sys/arch/mvme88k/mvme88k/m88110.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: m88110.c,v 1.34 2005/12/04 15:00:26 miod Exp $	*/
+/*	$OpenBSD: m88110.c,v 1.35 2005/12/11 21:45:31 miod Exp $	*/
 /*
  * Copyright (c) 1998 Steve Murphree, Jr.
  * All rights reserved.
@@ -62,6 +62,7 @@
 
 #include <uvm/uvm_extern.h>
 
+#include <machine/asm_macro.h>
 #include <machine/cmmu.h>
 #include <machine/cpu.h>
 #include <machine/lock.h>
@@ -71,21 +72,24 @@
 #include <machine/psl.h>
 #include <machine/trap.h>
 
-cpuid_t m88110_init(void);
-void m88110_setup_board_config(void);
-void m88110_cpu_configuration_print(int);
-void m88110_shutdown(void);
-cpuid_t m88110_cpu_number(void);
-void m88110_set_sapr(cpuid_t, apr_t);
-void m88110_set_uapr(apr_t);
-void m88110_flush_tlb(cpuid_t, unsigned, vaddr_t, u_int);
-void m88110_flush_cache(cpuid_t, paddr_t, psize_t);
-void m88110_flush_inst_cache(cpuid_t, paddr_t, psize_t);
-void m88110_flush_data_cache(cpuid_t, paddr_t, psize_t);
-int m88110_dma_cachectl(pmap_t, vaddr_t, vsize_t, int);
-int m88110_dma_cachectl_pa(paddr_t, psize_t, int);
-
-/* This is the function table for the mc88110 built-in CMMUs */
+#include <mvme88k/dev/busswreg.h>
+
+cpuid_t	m88110_init(void);
+void	m88110_setup_board_config(void);
+void	m88110_cpu_configuration_print(int);
+void	m88110_shutdown(void);
+cpuid_t	m88110_cpu_number(void);
+void	m88110_set_sapr(cpuid_t, apr_t);
+void	m88110_set_uapr(apr_t);
+void	m88110_flush_tlb(cpuid_t, unsigned, vaddr_t, u_int);
+void	m88110_flush_cache(cpuid_t, paddr_t, psize_t);
+void	m88110_flush_inst_cache(cpuid_t, paddr_t, psize_t);
+void	m88110_flush_data_cache(cpuid_t, paddr_t, psize_t);
+int	m88110_dma_cachectl(pmap_t, vaddr_t, vsize_t, int);
+int	m88110_dma_cachectl_pa(paddr_t, psize_t, int);
+void	m88110_initialize_cpu(cpuid_t);
+
+/* This is the function table for the MC88110 built-in CMMUs */
 struct cmmu_p cmmu88110 = {
         m88110_init,
 	m88110_setup_board_config,
@@ -100,6 +104,9 @@ struct cmmu_p cmmu88110 = {
 	m88110_flush_data_cache,
 	m88110_dma_cachectl,
 	m88110_dma_cachectl_pa,
+#ifdef MULTIPROCESSOR
+	m88110_initialize_cpu,
+#endif
 };
 
 void patc_clear(void);
@@ -126,8 +133,11 @@ patc_clear(void)
 void
 m88110_setup_board_config(void)
 {
-	/* we could print something here... */
+#ifdef MULTIPROCESSOR
+	max_cpus = 2;
+#else
 	max_cpus = 1;
+#endif
 }
 
 /*
@@ -142,13 +152,6 @@ m88110_cpu_configuration_print(int master)
 	int proctype = (pid & PID_ARN) >> ARN_SHIFT;
 	int procvers = (pid & PID_VN) >> VN_SHIFT;
 	int cpu = cpu_number();
-	static __cpu_simple_lock_t print_lock;
-
-	CMMU_LOCK;
-	if (master)
-		__cpu_simple_lock_init(&print_lock);
-
-	__cpu_simple_lock(&print_lock);
 
 	printf("cpu%d: ", cpu);
 	switch (proctype) {
@@ -163,9 +166,6 @@ m88110_cpu_configuration_print(int master)
 		break;
 	}
 	printf("\n");
-
-	__cpu_simple_unlock(&print_lock);
-        CMMU_UNLOCK;
 }
 
 /*
@@ -174,6 +174,16 @@ m88110_cpu_configuration_print(int master)
 cpuid_t
 m88110_init(void)
 {
+	cpuid_t cpu;
+
+	cpu = m88110_cpu_number();
+	m88110_initialize_cpu(cpu);
+	return (cpu);
+}
+
+void
+m88110_initialize_cpu(cpuid_t cpu)
+{
 	int i;
 
 	/* clear BATCs */
@@ -212,8 +222,6 @@ m88110_init(void)
 
 	set_isr(0);
 	set_dsr(0);
-
-	return (m88110_cpu_number());
 }
 
 /*
@@ -222,22 +230,16 @@ m88110_init(void)
 void
 m88110_shutdown(void)
 {
-#if 0
-	CMMU_LOCK;
-        CMMU_UNLOCK;
-#endif
 }
 
-/*
- * Find out the CPU number from accessing CMMU
- * Better be at splhigh, or even better, with interrupts
- * disabled.
- */
-
 cpuid_t
 m88110_cpu_number(void)
 {
-	return (0);	/* XXXSMP - need to tell DP processors apart */
+	u_int16_t gcsr;
+
+	gcsr = *(volatile u_int16_t *)(BS_BASE + BS_GCSR);
+
+	return ((gcsr & BS_GCSR_CPUID) != 0 ? 1 : 0);
 }
 
 void
diff --git a/sys/arch/mvme88k/mvme88k/machdep.c b/sys/arch/mvme88k/mvme88k/machdep.c
index 66a7f57ef1f..71d5df45598 100644
--- a/sys/arch/mvme88k/mvme88k/machdep.c
+++ b/sys/arch/mvme88k/mvme88k/machdep.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: machdep.c,v 1.175 2005/12/11 21:36:06 miod Exp $	*/
+/* $OpenBSD: machdep.c,v 1.176 2005/12/11 21:45:31 miod Exp $	*/
 /*
  * Copyright (c) 1998, 1999, 2000, 2001 Steve Murphree, Jr.
  * Copyright (c) 1996 Nivas Madhur
@@ -97,13 +97,12 @@ void	consinit(void);
 void	dumpconf(void);
 void	dumpsys(void);
 int	getcpuspeed(struct mvmeprom_brdid *);
-vaddr_t	get_slave_stack(void);
 void	identifycpu(void);
 void	mvme_bootstrap(void);
 void	savectx(struct pcb *);
+void	secondary_main(void);
+void	secondary_pre_main(void);
 void	setupiackvectors(void);
-void	slave_pre_main(void);
-int	slave_main(void);
 void	vector_init(m88k_exception_vector_area *, unsigned *);
 void	_doboot(void);
 
@@ -154,6 +153,10 @@ vaddr_t iomapbase;
 struct extent *iomap_extent;
 struct vm_map *iomap_map;
 
+#ifdef MULTIPROCESSOR
+__cpu_simple_lock_t cpu_mutex = __SIMPLELOCK_UNLOCKED;
+#endif
+
 /*
  * Declare these as initialized data so we can patch them.
  */
@@ -433,7 +436,7 @@ cpu_startup()
 		 * "base" pages for the rest.
 		 */
 		curbuf = (vaddr_t)buffers + (i * MAXBSIZE);
-		curbufsize = PAGE_SIZE * ((i < residual) ? (base+1) : base);
+		curbufsize = PAGE_SIZE * ((i < residual) ? (base + 1) : base);
 
 		while (curbufsize) {
 			pg = uvm_pagealloc(NULL, 0, NULL, 0);
@@ -806,44 +809,65 @@ setupiackvectors()
 	}
 }
 
-/* gets an interrupt stack for slave processors */
-vaddr_t
-get_slave_stack()
+#ifdef MULTIPROCESSOR
+
+/*
+ * Secondary CPU early initialization routine.
+ * Determine CPU number and set it, then allocate the idle pcb (and stack).
+ *
+ * Running on a minimal stack here, with interrupts disabled; do nothing fancy.
+ */
+void
+secondary_pre_main()
 {
-	vaddr_t addr;
+	struct cpu_info *ci;
 
-	addr = (vaddr_t)uvm_km_zalloc(kernel_map, INTSTACK_SIZE);
+	set_cpu_number(cmmu_cpu_number()); /* Determine cpu number by CMMU */
+	ci = curcpu();
+	ci->ci_curproc = &proc0;
 
-	if (addr == NULL)
-		panic("Cannot allocate slave stack for cpu %d",
-		    cpu_number());
+	splhigh();
 
-	return addr;
+	/*
+	 * Setup CMMUs and translation tables (shared with the master cpu).
+	 */
+	pmap_bootstrap_cpu(ci->ci_cpuid);
+
+	/*
+	 * Allocate UPAGES contiguous pages for the idle PCB and stack.
+	 */
+	ci->ci_idle_pcb = (struct pcb *)uvm_km_zalloc(kernel_map, USPACE);
+	if (ci->ci_idle_pcb == NULL) {
+		printf("cpu%d: unable to allocate idle stack\n", ci->ci_cpuid);
+	}
 }
 
 /*
- * Slave CPU pre-main routine.
- * Determine CPU number and set it.
+ * Further secondary CPU initialization.
  *
- * Running on an interrupt stack here; do nothing fancy.
+ * We are now running on our idle stack, with proper page tables.
+ * There is nothing to do but display some details about the CPU and its CMMUs.
  */
 void
-slave_pre_main()
+secondary_main()
 {
-	set_cpu_number(cmmu_cpu_number()); /* Determine cpu number by CMMU */
-	splhigh();
-	set_psr(get_psr() & ~PSR_IND);
-}
+	struct cpu_info *ci = curcpu();
 
-/* dummy main routine for slave processors */
-int
-slave_main()
-{
-	printf("slave CPU%d started\n", cpu_number());
-	while (1); /* spin forever */
-	return 0;
+	cpu_configuration_print(0);
+	__cpu_simple_unlock(&cpu_mutex);
+
+	microuptime(&ci->ci_schedstate.spc_runtime);
+	ci->ci_curproc = NULL;
+
+	/*
+	 * Upon return, the secondary cpu bootstrap code in locore will
+	 * enter the idle loop, waiting for some food to process on this
+	 * processor.
+	 */
 }
 
+#endif	/* MULTIPROCESSOR */
+
 /*
  * Search for the first available interrupt vector in the range start, end.
  * This should really only be used by VME devices.
@@ -1062,39 +1086,6 @@ mvme_bootstrap()
 	curproc = &proc0;
 	curpcb = &proc0paddr->u_pcb;
 
-	/*
-	 * If we have more than one CPU, mention which one is the master.
-	 * We will also want to spin up slave CPUs on the long run...
-	 */
-	switch (brdtyp) {
-#ifdef MVME188
-	case BRD_188:
-		printf("CPU%d is master CPU\n", master_cpu);
-
-#if 0
-		int i;
-		for (i = 0; i < MAX_CPUS; i++) {
-			if (!spin_cpu(i))
-				printf("CPU%d started\n", i);
-		}
-#endif
-		break;
-#endif
-#ifdef MVME197
-	case BRD_197:
-		/*
-		 * In the 197DP case, mention which CPU is the master
-		 * there too...
-		 * XXX TBD
-		 */
-		break;
-#endif
-#ifdef MVME187
-	default:
-		break;
-#endif
-	}
-
 	avail_start = first_addr;
 	avail_end = last_addr;
 
@@ -1126,6 +1117,38 @@ mvme_bootstrap()
 #endif
 }
 
+#ifdef MULTIPROCESSOR
+void
+cpu_boot_secondary_processors()
+{
+	cpuid_t cpu;
+	int rc;
+	extern void secondary_start(void);
+
+	switch (brdtyp) {
+#if defined(MVME188) || defined(MVME197)
+#ifdef MVME188
+	case BRD_188:
+#endif
+#ifdef MVME197
+	case BRD_197:
+#endif
+		for (cpu = 0; cpu < max_cpus; cpu++) {
+			if (cpu != curcpu()->ci_cpuid) {
+				rc = spin_cpu(cpu, (vaddr_t)secondary_start);
+				if (rc != 0 && rc != FORKMPU_NO_MPU)
+					printf("cpu%d: spin_cpu error %d\n",
+					    cpu, rc);
+			}
+		}
+		break;
+#endif
+	default:
+		break;
+	}
+}
+#endif
+
 /*
  * Boot console routines:
  * Enables printing of boot messages before consinit().
@@ -1169,7 +1192,7 @@ bootcnputc(dev, c)
 #define NO_OP 		0xf4005800	/* "or r0, r0, r0" */
 
 #define BRANCH(FROM, TO) \
-	(EMPTY_BR | ((unsigned)(TO) - (unsigned)(FROM)) >> 2)
+	(EMPTY_BR | ((vaddr_t)(TO) - (vaddr_t)(FROM)) >> 2)
 
 #define SET_VECTOR(NUM, VALUE) \
 	do { \