Rework secondary processor initialization. cmmu initialization is now

performed much earlier in the processor startup.
No visible change, paves the way for the much important diff three commits
from here.

12 files changed, 83 insertions, 78 deletions

diff --git a/sys/arch/aviion/aviion/autoconf.c b/sys/arch/aviion/aviion/autoconf.c
index 971277223c1..083ca8ae3cd 100644
--- a/sys/arch/aviion/aviion/autoconf.c
+++ b/sys/arch/aviion/aviion/autoconf.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: autoconf.c,v 1.12 2010/11/18 21:13:19 miod Exp $	*/
+/*	$OpenBSD: autoconf.c,v 1.13 2011/10/09 17:01:32 miod Exp $	*/
 /*
  * Copyright (c) 1998 Steve Murphree, Jr.
  * Copyright (c) 1996 Nivas Madhur
@@ -85,12 +85,15 @@ cpu_configure()
 	if (config_rootfound("mainbus", "mainbus") == 0)
 		panic("no mainbus found");
 
+	/* NO PROM CALLS FROM NOW ON */
+
+	cold = 0;
+
 	/*
 	 * Turn external interrupts on.
 	 */
 	set_psr(get_psr() & ~PSR_IND);
 	spl0();
-	cold = 0;
 }
 
 void
diff --git a/sys/arch/aviion/aviion/locore.S b/sys/arch/aviion/aviion/locore.S
index ef139810fb7..28ed5b28887 100644
--- a/sys/arch/aviion/aviion/locore.S
+++ b/sys/arch/aviion/aviion/locore.S
@@ -1,4 +1,4 @@
-/*	$OpenBSD: locore.S,v 1.14 2010/12/23 20:05:07 miod Exp $	*/
+/*	$OpenBSD: locore.S,v 1.15 2011/10/09 17:01:32 miod Exp $	*/
 /*
  * Copyright (c) 2005, Miodrag Vallat.
  * Copyright (c) 1998 Steve Murphree, Jr.
@@ -191,17 +191,6 @@ ASLOCAL(main_start)
 	bsr.n	_C_LABEL(main)
 	 addu	r31, r31, USPACE
 
-	or.u	r2, r0, hi16(_ASM_LABEL(main_panic))
-	bsr.n	_C_LABEL(panic)
-	 or	r2, r2, lo16(_ASM_LABEL(main_panic))
-
-	data
-	.align	4
-ASLOCAL(main_panic)
-	string	"main() returned\0"
-	text
-	.align	8
-
 #ifdef MULTIPROCESSOR
 
 	/*
@@ -221,11 +210,11 @@ GLOBAL(secondary_start)
 	xmem	r22, r11, r0	/* if r22 becomes zero, we own the lock... */
 	bcnd	eq0, r22, 4f	/* ... but if not, we must wait */
 2:
-	/* just watch the lock until it looks clear */
+	/* just watch the lock until it clears */
 	ld	r22, r11, r0
 	bcnd	eq0, r22, 1b
 	/* wait a bit to avoid overloading the bus */
-	or.u	r2,  r0,  1
+	or.u	r2,  r0,  100
 3:
 	subu	r2,  r2,  1
 	bcnd	ne0, r2,  3b
@@ -234,9 +223,9 @@ GLOBAL(secondary_start)
 
 	/*
 	 * We are now running free with cpu_hatch_mutex held; other
-	 * secondary processors are waiting for the lock, and the main
-	 * processor is waiting for us to decrease the hatch counter,
-	 * which we'll do in secondary_main() later.
+	 * secondary processors (if any) are waiting for the lock,
+	 * and the main processor is waiting for us to decrease the
+	 * hatch counter, which we'll do later in secondary_main().
 	 */
 
 	or.u	r31, r0,  hi16(_ASM_LABEL(slavestack_end))
@@ -254,6 +243,7 @@ GLOBAL(secondary_start)
 	 */
 	or.u	r11, r0,  hi16(_ASM_LABEL(dummy_cpu))
 	or	r11, r11, lo16(_ASM_LABEL(dummy_cpu))
+	st	r0,  r11, CI_FLAGS			/* reset CIF_PRIMARY */
 	stcr	r11, CPU
 
 	/*
diff --git a/sys/arch/aviion/aviion/machdep.c b/sys/arch/aviion/aviion/machdep.c
index f8b079414eb..7a4940f59aa 100644
--- a/sys/arch/aviion/aviion/machdep.c
+++ b/sys/arch/aviion/aviion/machdep.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: machdep.c,v 1.45 2011/06/26 22:39:59 deraadt Exp $	*/
+/*	$OpenBSD: machdep.c,v 1.46 2011/10/09 17:01:32 miod Exp $	*/
 /*
  * Copyright (c) 2007 Miodrag Vallat.
  *
@@ -517,7 +517,17 @@ secondary_pre_main()
 	struct cpu_info *ci;
 	vaddr_t init_stack;
 
-	set_cpu_number(cmmu_cpu_number()); /* Determine cpu number by CMMU */
+	/*
+	 * Invoke the CMMU initialization routine as early as possible,
+	 * so that we do not risk any memory writes to be lost during
+	 * cache setup.
+	 */
+	cmmu_initialize_cpu(cmmu_cpu_number());
+
+	/*
+	 * Now initialize your cpu_info structure.
+	 */
+	set_cpu_number(cmmu_cpu_number());
 	ci = curcpu();
 	ci->ci_curproc = &proc0;
 	platform->smp_setup(ci);
@@ -525,7 +535,7 @@ secondary_pre_main()
 	splhigh();
 
 	/*
-	 * Setup CMMUs and translation tables (shared with the master cpu).
+	 * Enable MMU on this processor.
 	 */
 	pmap_bootstrap_cpu(ci->ci_cpuid);
 
@@ -566,7 +576,6 @@ secondary_main()
 	microuptime(&ci->ci_schedstate.spc_runtime);
 	ci->ci_curproc = NULL;
 	ci->ci_randseed = random();
-	SET(ci->ci_flags, CIF_ALIVE);
 
 	/*
 	 * Release cpu_hatch_mutex to let other secondary processors
@@ -582,6 +591,9 @@ secondary_main()
 	spl0();
 	SCHED_LOCK(s);
 	set_psr(get_psr() & ~PSR_IND);
+
+	SET(ci->ci_flags, CIF_ALIVE);
+
 	cpu_switchto(NULL, sched_chooseproc());
 }
 
@@ -776,22 +788,24 @@ cpu_hatch_secondary_processors()
 	if (platform->send_ipi == NULL)
 		return;
 
-	for (cpu = 0; cpu < ncpusfound; cpu++) {
+	for (cpu = 0; cpu < MAX_CPUS; cpu++) {
 		if (cpu != ci->ci_cpuid) {
+			hatch_pending_count++;
 			rc = scm_jpstart(cpu, (vaddr_t)secondary_start);
 			switch (rc) {
 			case JPSTART_OK:
-				hatch_pending_count++;
-				break;
-			case JPSTART_NO_JP:
 				break;
 			case JPSTART_SINGLE_JP:
 				/* this should never happen, but just in case */
+				hatch_pending_count = 0;
 				ncpusfound = 1;
 				return;
 			default:
 				printf("CPU%d failed to start, error %d\n",
 				    cpu, rc);
+				/* FALLTHROUGH */
+			case JPSTART_NO_JP:
+				hatch_pending_count--;
 				break;
 			}
 		}
@@ -806,7 +820,7 @@ cpu_setup_secondary_processors()
 {
 	__cpu_simple_unlock(&cpu_hatch_mutex);
 	while (hatch_pending_count != 0)
-		delay(100000);
+		delay(10000);	/* 10ms */
 }
 
 /*
diff --git a/sys/arch/aviion/dev/mainbus.c b/sys/arch/aviion/dev/mainbus.c
index 0c8047ce701..58a60c4ad21 100644
--- a/sys/arch/aviion/dev/mainbus.c
+++ b/sys/arch/aviion/dev/mainbus.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: mainbus.c,v 1.6 2010/04/24 18:46:55 miod Exp $ */
+/*	$OpenBSD: mainbus.c,v 1.7 2011/10/09 17:01:34 miod Exp $ */
 /*
  * Copyright (c) 1998 Steve Murphree, Jr.
  * Copyright (c) 2004, Miodrag Vallat.
@@ -141,8 +141,8 @@ mainbus_attach(struct device *parent, struct device *self, void *args)
 
 #ifdef MULTIPROCESSOR
 	/*
-	 * Let secondary processor initialize further and print their
-	 * configuration information now.
+	 * Let secondary processors initialize further and print
+	 * their configuration information now.
 	 */
 	cpu_setup_secondary_processors();
 #endif
diff --git a/sys/arch/m88k/m88k/genassym.cf b/sys/arch/m88k/m88k/genassym.cf
index c8a34ae279f..270e36d06b6 100644
--- a/sys/arch/m88k/m88k/genassym.cf
+++ b/sys/arch/m88k/m88k/genassym.cf
@@ -1,4 +1,4 @@
-#	$OpenBSD: genassym.cf,v 1.22 2010/09/28 20:27:55 miod Exp $
+#	$OpenBSD: genassym.cf,v 1.23 2011/10/09 17:01:34 miod Exp $
 #
 # Copyright (c) 1982, 1990 The Regents of the University of California.
 # All rights reserved.
@@ -28,7 +28,7 @@
 # SUCH DAMAGE.
 #
 #	@(#)genassym.c	7.8 (Berkeley) 5/7/91
-#	$Id: genassym.cf,v 1.22 2010/09/28 20:27:55 miod Exp $
+#	$Id: genassym.cf,v 1.23 2011/10/09 17:01:34 miod Exp $
 #
 
 include <sys/param.h>
@@ -61,6 +61,7 @@ export	SONPROC
 
 # cpu fields
 struct	cpu_info
+member	ci_flags
 member	ci_curproc
 member	ci_curpcb
 member	ci_curpmap
diff --git a/sys/arch/m88k/m88k/m8820x_machdep.c b/sys/arch/m88k/m88k/m8820x_machdep.c
index 7e8f58a925d..757ee00e060 100644
--- a/sys/arch/m88k/m88k/m8820x_machdep.c
+++ b/sys/arch/m88k/m88k/m8820x_machdep.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: m8820x_machdep.c,v 1.47 2011/01/05 22:16:16 miod Exp $	*/
+/*	$OpenBSD: m8820x_machdep.c,v 1.48 2011/10/09 17:01:34 miod Exp $	*/
 /*
  * Copyright (c) 2004, 2007, 2010, 2011, Miodrag Vallat.
  *
@@ -396,8 +396,7 @@ m8820x_initialize_cpu(cpuid_t cpu)
 	int cssp, type;
 	apr_t apr;
 
-	apr = ((0x00000 << PG_BITS) | CACHE_WT | CACHE_GLOBAL | CACHE_INH) &
-	    ~APR_V;
+	apr = ((0x00000 << PG_BITS) | CACHE_GLOBAL | CACHE_INH) & ~APR_V;
 
 	cmmu = m8820x_cmmu + (cpu << cmmu_shift);
 
@@ -480,11 +479,22 @@ m8820x_initialize_cpu(cpuid_t cpu)
 
 	/*
 	 * Enable instruction cache.
-	 * Data cache will be enabled later.
 	 */
 	apr &= ~CACHE_INH;
 	m8820x_cmmu_set_reg(CMMU_SAPR, apr, MODE_VAL, cpu, INST_CMMU);
 
+	/*
+	 * Data cache will be enabled at pmap_bootstrap_cpu() time,
+	 * because the PROM won't likely expect its work area in memory
+	 * to be cached. On at least aviion, starting secondary processors
+	 * returns an error code although the processor has correctly spun
+	 * up, if the PROM work area is cached.
+	 */
+#ifdef dont_do_this_at_home
+	apr |= CACHE_WT;
+	m8820x_cmmu_set_reg(CMMU_SAPR, apr, MODE_VAL, cpu, DATA_CMMU);
+#endif
+
 	ci->ci_zeropage = m8820x_zeropage;
 	ci->ci_copypage = m8820x_copypage;
 }
diff --git a/sys/arch/m88k/m88k/pmap.c b/sys/arch/m88k/m88k/pmap.c
index a20bd5cebe3..25cf380ce07 100644
--- a/sys/arch/m88k/m88k/pmap.c
+++ b/sys/arch/m88k/m88k/pmap.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: pmap.c,v 1.63 2011/01/05 22:20:22 miod Exp $	*/
+/*	$OpenBSD: pmap.c,v 1.64 2011/10/09 17:01:34 miod Exp $	*/
 
 /*
  * Copyright (c) 2001-2004, 2010, Miodrag Vallat.
@@ -763,12 +763,6 @@ pmap_bootstrap(paddr_t s_rom, paddr_t e_rom)
 void
 pmap_bootstrap_cpu(cpuid_t cpu)
 {
-	/* Invalidate entire kernel TLB and get ready for address translation */
-#ifdef MULTIPROCESSOR
-	if (cpu != master_cpu)
-		cmmu_initialize_cpu(cpu);
-#endif
-
 	/* Load supervisor pointer to segment table. */
 	cmmu_set_sapr(pmap_kernel()->pm_apr);
 #ifdef PMAPDEBUG
diff --git a/sys/arch/mvme88k/dev/mainbus.c b/sys/arch/mvme88k/dev/mainbus.c
index 1d805faeeb1..71cfcc1e01d 100644
--- a/sys/arch/mvme88k/dev/mainbus.c
+++ b/sys/arch/mvme88k/dev/mainbus.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: mainbus.c,v 1.26 2011/04/07 15:30:15 miod Exp $ */
+/*	$OpenBSD: mainbus.c,v 1.27 2011/10/09 17:01:34 miod Exp $ */
 /*
  * Copyright (c) 1998 Steve Murphree, Jr.
  * Copyright (c) 2004, Miodrag Vallat.
@@ -287,7 +287,6 @@ mainbus_scan(struct device *parent, void *child, void *args)
 void
 mainbus_attach(struct device *parent, struct device *self, void *args)
 {
-	extern void cpu_hatch_secondary_processors(void *);
 	extern char cpu_model[];
 
 	printf(": %s\n", cpu_model);
@@ -317,17 +316,5 @@ mainbus_attach(struct device *parent, struct device *self, void *args)
 	if (bs_extent == NULL)
 		panic("unable to allocate bus_space extent");
 
-#ifdef MULTIPROCESSOR
-	/*
-	 * Spin up the other processors, but do not give them work to
-	 * do yet.
-	 * On MVME188 boards, the system hangs if secondary processors
-	 * try to issue BUG calls (i.e. when printing their information
-	 * on console), so we postpone this to the end of autoconf.
-	 */
-	if (brdtyp != BRD_188)
-		cpu_hatch_secondary_processors(NULL);
-#endif
-
 	(void)config_search(mainbus_scan, self, args);
 }
diff --git a/sys/arch/mvme88k/mvme88k/autoconf.c b/sys/arch/mvme88k/mvme88k/autoconf.c
index a1497190dad..100bfe8d1c8 100644
--- a/sys/arch/mvme88k/mvme88k/autoconf.c
+++ b/sys/arch/mvme88k/mvme88k/autoconf.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: autoconf.c,v 1.47 2010/12/23 20:05:08 miod Exp $	*/
+/*	$OpenBSD: autoconf.c,v 1.48 2011/10/09 17:01:34 miod Exp $	*/
 /*
  * Copyright (c) 1998 Steve Murphree, Jr.
  * Copyright (c) 1996 Nivas Madhur
@@ -91,13 +91,9 @@ cpu_configure()
 #ifdef MULTIPROCESSOR
 	/*
 	 * Spin up the other processors, but do not give them work to
-	 * do yet. This is normally done when attaching mainbus, but
-	 * on MVME188 boards, the system hangs if secondary processors
-	 * try to issue BUG calls (i.e. when printing their information
-	 * on console), so this has been postponed until now.
+	 * do yet.
 	 */
-	if (brdtyp == BRD_188)
-		cpu_hatch_secondary_processors(NULL);
+	cpu_hatch_secondary_processors(NULL);
 #endif
 
 	/* NO BUG CALLS FROM NOW ON */
diff --git a/sys/arch/mvme88k/mvme88k/locore.S b/sys/arch/mvme88k/mvme88k/locore.S
index 6551081a01c..d43d1aaea27 100644
--- a/sys/arch/mvme88k/mvme88k/locore.S
+++ b/sys/arch/mvme88k/mvme88k/locore.S
@@ -1,4 +1,4 @@
-/*	$OpenBSD: locore.S,v 1.67 2010/12/23 20:05:08 miod Exp $	*/
+/*	$OpenBSD: locore.S,v 1.68 2011/10/09 17:01:34 miod Exp $	*/
 /*
  * Copyright (c) 2005, Miodrag Vallat.
  * Copyright (c) 1998 Steve Murphree, Jr.
@@ -115,7 +115,6 @@ ASGLOBAL(start)
 	NOP
 #endif
 
-
 	/*
 	 * Startup code for main processor.
 	 */
@@ -230,10 +229,10 @@ GLOBAL(secondary_start)
 	bsr.n	_ASM_LABEL(setup_psr)
 	 or	r31, r31, lo16(_ASM_LABEL(slavestack_end))
 
-	/*
-	 * We expect secondary processors to start running with the same 
-	 * VBR value as the primary one.
-	 */
+	or.u	r3,  r0,  hi16(_C_LABEL(kernel_vbr))
+	ld	r2,  r3,  lo16(_C_LABEL(kernel_vbr))
+	stcr	r2,  VBR
+	FLUSH_PIPELINE
 
 	/*
 	 * Have curcpu() point at the dummy cpuinfo structure,
@@ -243,6 +242,7 @@ GLOBAL(secondary_start)
 	 */
 	or.u	r11, r0,  hi16(_ASM_LABEL(dummy_cpu))
 	or	r11, r11, lo16(_ASM_LABEL(dummy_cpu))
+	st	r0,  r11, CI_FLAGS		/* reset CIF_PRIMARY */
 	stcr	r11, CPU
 
 	/*
diff --git a/sys/arch/mvme88k/mvme88k/m8820x.c b/sys/arch/mvme88k/mvme88k/m8820x.c
index eb21476ba08..eeb821fd1c6 100644
--- a/sys/arch/mvme88k/mvme88k/m8820x.c
+++ b/sys/arch/mvme88k/mvme88k/m8820x.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: m8820x.c,v 1.50 2009/04/19 17:56:13 miod Exp $	*/
+/*	$OpenBSD: m8820x.c,v 1.51 2011/10/09 17:01:34 miod Exp $	*/
 /*
  * Copyright (c) 2004, Miodrag Vallat.
  *
@@ -84,10 +84,10 @@ m8820x_setup_board_config()
 {
 	extern u_int32_t pfsr_save[];
 	int num;
-	int vme188_config;
 	u_int32_t *m8820x_pfsr;
 #ifdef MVME188
 	u_int32_t whoami;
+	int vme188_config;
 	struct m8820x_cmmu *cmmu;
 	int cmmu_num;
 #endif
@@ -96,8 +96,10 @@ m8820x_setup_board_config()
 #ifdef MVME187
 	case BRD_187:
 	case BRD_8120:
+#ifdef MVME188
 		/* There is no WHOAMI reg on MVME187 - fake it... */
 		vme188_config = 0x0a;
+#endif
 		m8820x_cmmu[0].cmmu_regs = (void *)SBC_CMMU_I;
 		m8820x_cmmu[1].cmmu_regs = (void *)SBC_CMMU_D;
 		ncpusfound = 1;
diff --git a/sys/arch/mvme88k/mvme88k/machdep.c b/sys/arch/mvme88k/mvme88k/machdep.c
index 878d4f239a3..b9f8a00027e 100644
--- a/sys/arch/mvme88k/mvme88k/machdep.c
+++ b/sys/arch/mvme88k/mvme88k/machdep.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: machdep.c,v 1.244 2011/06/26 22:40:00 deraadt Exp $	*/
+/* $OpenBSD: machdep.c,v 1.245 2011/10/09 17:01:34 miod Exp $	*/
 /*
  * Copyright (c) 1998, 1999, 2000, 2001 Steve Murphree, Jr.
  * Copyright (c) 1996 Nivas Madhur
@@ -624,7 +624,17 @@ secondary_pre_main()
 	struct cpu_info *ci;
 	vaddr_t init_stack;
 
-	set_cpu_number(cmmu_cpu_number()); /* Determine cpu number by CMMU */
+	/*
+	 * Invoke the CMMU initialization routine as early as possible,
+	 * so that we do not risk any memory writes to be lost during
+	 * cache setup.
+	 */
+	cmmu_initialize_cpu(cmmu_cpu_number());
+
+	/*
+	 * Now initialize your cpu_info structure.
+	 */
+	set_cpu_number(cmmu_cpu_number());
 	ci = curcpu();
 	ci->ci_curproc = &proc0;
 	(*md_smp_setup)(ci);
@@ -632,7 +642,7 @@ secondary_pre_main()
 	splhigh();
 
 	/*
-	 * Setup CMMUs and translation tables (shared with the master cpu).
+	 * Enable MMU on this processor.
 	 */
 	pmap_bootstrap_cpu(ci->ci_cpuid);
 
@@ -676,8 +686,6 @@ secondary_main()
 	__cpu_simple_lock(&cpu_boot_mutex);
 	__cpu_simple_unlock(&cpu_boot_mutex);
 
-	set_vbr(kernel_vbr);
-
 	spl0();
 	SCHED_LOCK(s);
 	set_psr(get_psr() & ~PSR_IND);