Rework secondary processor initialization. cmmu initialization is now

performed much earlier in the processor startup.
No visible change, paves the way for the much important diff three commits
from here.

4 files changed, 38 insertions, 31 deletions

diff --git a/sys/arch/aviion/aviion/autoconf.c b/sys/arch/aviion/aviion/autoconf.c
index 971277223c1..083ca8ae3cd 100644
--- a/sys/arch/aviion/aviion/autoconf.c
+++ b/sys/arch/aviion/aviion/autoconf.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: autoconf.c,v 1.12 2010/11/18 21:13:19 miod Exp $	*/
+/*	$OpenBSD: autoconf.c,v 1.13 2011/10/09 17:01:32 miod Exp $	*/
 /*
  * Copyright (c) 1998 Steve Murphree, Jr.
  * Copyright (c) 1996 Nivas Madhur
@@ -85,12 +85,15 @@ cpu_configure()
 	if (config_rootfound("mainbus", "mainbus") == 0)
 		panic("no mainbus found");
 
+	/* NO PROM CALLS FROM NOW ON */
+
+	cold = 0;
+
 	/*
 	 * Turn external interrupts on.
 	 */
 	set_psr(get_psr() & ~PSR_IND);
 	spl0();
-	cold = 0;
 }
 
 void
diff --git a/sys/arch/aviion/aviion/locore.S b/sys/arch/aviion/aviion/locore.S
index ef139810fb7..28ed5b28887 100644
--- a/sys/arch/aviion/aviion/locore.S
+++ b/sys/arch/aviion/aviion/locore.S
@@ -1,4 +1,4 @@
-/*	$OpenBSD: locore.S,v 1.14 2010/12/23 20:05:07 miod Exp $	*/
+/*	$OpenBSD: locore.S,v 1.15 2011/10/09 17:01:32 miod Exp $	*/
 /*
  * Copyright (c) 2005, Miodrag Vallat.
  * Copyright (c) 1998 Steve Murphree, Jr.
@@ -191,17 +191,6 @@ ASLOCAL(main_start)
 	bsr.n	_C_LABEL(main)
 	 addu	r31, r31, USPACE
 
-	or.u	r2, r0, hi16(_ASM_LABEL(main_panic))
-	bsr.n	_C_LABEL(panic)
-	 or	r2, r2, lo16(_ASM_LABEL(main_panic))
-
-	data
-	.align	4
-ASLOCAL(main_panic)
-	string	"main() returned\0"
-	text
-	.align	8
-
 #ifdef MULTIPROCESSOR
 
 	/*
@@ -221,11 +210,11 @@ GLOBAL(secondary_start)
 	xmem	r22, r11, r0	/* if r22 becomes zero, we own the lock... */
 	bcnd	eq0, r22, 4f	/* ... but if not, we must wait */
 2:
-	/* just watch the lock until it looks clear */
+	/* just watch the lock until it clears */
 	ld	r22, r11, r0
 	bcnd	eq0, r22, 1b
 	/* wait a bit to avoid overloading the bus */
-	or.u	r2,  r0,  1
+	or.u	r2,  r0,  100
 3:
 	subu	r2,  r2,  1
 	bcnd	ne0, r2,  3b
@@ -234,9 +223,9 @@ GLOBAL(secondary_start)
 
 	/*
 	 * We are now running free with cpu_hatch_mutex held; other
-	 * secondary processors are waiting for the lock, and the main
-	 * processor is waiting for us to decrease the hatch counter,
-	 * which we'll do in secondary_main() later.
+	 * secondary processors (if any) are waiting for the lock,
+	 * and the main processor is waiting for us to decrease the
+	 * hatch counter, which we'll do later in secondary_main().
 	 */
 
 	or.u	r31, r0,  hi16(_ASM_LABEL(slavestack_end))
@@ -254,6 +243,7 @@ GLOBAL(secondary_start)
 	 */
 	or.u	r11, r0,  hi16(_ASM_LABEL(dummy_cpu))
 	or	r11, r11, lo16(_ASM_LABEL(dummy_cpu))
+	st	r0,  r11, CI_FLAGS			/* reset CIF_PRIMARY */
 	stcr	r11, CPU
 
 	/*
diff --git a/sys/arch/aviion/aviion/machdep.c b/sys/arch/aviion/aviion/machdep.c
index f8b079414eb..7a4940f59aa 100644
--- a/sys/arch/aviion/aviion/machdep.c
+++ b/sys/arch/aviion/aviion/machdep.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: machdep.c,v 1.45 2011/06/26 22:39:59 deraadt Exp $	*/
+/*	$OpenBSD: machdep.c,v 1.46 2011/10/09 17:01:32 miod Exp $	*/
 /*
  * Copyright (c) 2007 Miodrag Vallat.
  *
@@ -517,7 +517,17 @@ secondary_pre_main()
 	struct cpu_info *ci;
 	vaddr_t init_stack;
 
-	set_cpu_number(cmmu_cpu_number()); /* Determine cpu number by CMMU */
+	/*
+	 * Invoke the CMMU initialization routine as early as possible,
+	 * so that we do not risk any memory writes to be lost during
+	 * cache setup.
+	 */
+	cmmu_initialize_cpu(cmmu_cpu_number());
+
+	/*
+	 * Now initialize your cpu_info structure.
+	 */
+	set_cpu_number(cmmu_cpu_number());
 	ci = curcpu();
 	ci->ci_curproc = &proc0;
 	platform->smp_setup(ci);
@@ -525,7 +535,7 @@ secondary_pre_main()
 	splhigh();
 
 	/*
-	 * Setup CMMUs and translation tables (shared with the master cpu).
+	 * Enable MMU on this processor.
 	 */
 	pmap_bootstrap_cpu(ci->ci_cpuid);
 
@@ -566,7 +576,6 @@ secondary_main()
 	microuptime(&ci->ci_schedstate.spc_runtime);
 	ci->ci_curproc = NULL;
 	ci->ci_randseed = random();
-	SET(ci->ci_flags, CIF_ALIVE);
 
 	/*
 	 * Release cpu_hatch_mutex to let other secondary processors
@@ -582,6 +591,9 @@ secondary_main()
 	spl0();
 	SCHED_LOCK(s);
 	set_psr(get_psr() & ~PSR_IND);
+
+	SET(ci->ci_flags, CIF_ALIVE);
+
 	cpu_switchto(NULL, sched_chooseproc());
 }
 
@@ -776,22 +788,24 @@ cpu_hatch_secondary_processors()
 	if (platform->send_ipi == NULL)
 		return;
 
-	for (cpu = 0; cpu < ncpusfound; cpu++) {
+	for (cpu = 0; cpu < MAX_CPUS; cpu++) {
 		if (cpu != ci->ci_cpuid) {
+			hatch_pending_count++;
 			rc = scm_jpstart(cpu, (vaddr_t)secondary_start);
 			switch (rc) {
 			case JPSTART_OK:
-				hatch_pending_count++;
-				break;
-			case JPSTART_NO_JP:
 				break;
 			case JPSTART_SINGLE_JP:
 				/* this should never happen, but just in case */
+				hatch_pending_count = 0;
 				ncpusfound = 1;
 				return;
 			default:
 				printf("CPU%d failed to start, error %d\n",
 				    cpu, rc);
+				/* FALLTHROUGH */
+			case JPSTART_NO_JP:
+				hatch_pending_count--;
 				break;
 			}
 		}
@@ -806,7 +820,7 @@ cpu_setup_secondary_processors()
 {
 	__cpu_simple_unlock(&cpu_hatch_mutex);
 	while (hatch_pending_count != 0)
-		delay(100000);
+		delay(10000);	/* 10ms */
 }
 
 /*
diff --git a/sys/arch/aviion/dev/mainbus.c b/sys/arch/aviion/dev/mainbus.c
index 0c8047ce701..58a60c4ad21 100644
--- a/sys/arch/aviion/dev/mainbus.c
+++ b/sys/arch/aviion/dev/mainbus.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: mainbus.c,v 1.6 2010/04/24 18:46:55 miod Exp $ */
+/*	$OpenBSD: mainbus.c,v 1.7 2011/10/09 17:01:34 miod Exp $ */
 /*
  * Copyright (c) 1998 Steve Murphree, Jr.
  * Copyright (c) 2004, Miodrag Vallat.
@@ -141,8 +141,8 @@ mainbus_attach(struct device *parent, struct device *self, void *args)
 
 #ifdef MULTIPROCESSOR
 	/*
-	 * Let secondary processor initialize further and print their
-	 * configuration information now.
+	 * Let secondary processors initialize further and print
+	 * their configuration information now.
 	 */
 	cpu_setup_secondary_processors();
 #endif