Rework secondary processor initialization, in order to allocate their idle

stack from the main processor, instead of from the secondary processors
themselves. This used to work until recent churn in uvm require the kernel
lock to be taken, and cpu_configure() runs with the kernel lock held by the
boot processor.

Allocating the idle stack on the secondary processors themselves was choosen
early on, because there is no easy way, from the boot processor, to know the
assocation between physical cpu number (ci_cpuid) and logical cpu number
(the order they are initialized and reported in), especially on luna88k where
there is no way to get secondary processors parked by the prom.

Regression noticed by aoyama@. Verified to work on 2 and 4 cpu luna88k systems
and 2 cpu aviion systems.

2 files changed, 30 insertions, 23 deletions

diff --git a/sys/arch/aviion/aviion/locore.S b/sys/arch/aviion/aviion/locore.S
index 4dffeaf315b..27d7e0c9a95 100644
--- a/sys/arch/aviion/aviion/locore.S
+++ b/sys/arch/aviion/aviion/locore.S
@@ -1,4 +1,4 @@
-/*	$OpenBSD: locore.S,v 1.20 2013/10/10 21:24:59 miod Exp $	*/
+/*	$OpenBSD: locore.S,v 1.21 2015/02/25 17:41:22 miod Exp $	*/
 /*
  * Copyright (c) 2005, Miodrag Vallat.
  * Copyright (c) 1998 Steve Murphree, Jr.
@@ -289,19 +289,16 @@ GLOBAL(secondary_start)
 	stcr	%r11, CPU
 
 	/*
-	 * While holding the cpu_mutex, the secondary cpu can use the slavestack
-	 * to call secondary_pre_main() to determine its cpu number.
-	 * After that, however, it should allocate its own stack and switch
-	 * to it.
+	 * While holding the cpu_mutex, the secondary cpu can use the
+	 * slavestack to call secondary_pre_main() to determine its cpu
+	 * number. That function will also return the proper stack to
+	 * use and we'll switch to it.
 	 */
 
 	bsr	_C_LABEL(secondary_pre_main)	/* set cpu number */
 
-	ldcr	%r1,  CPU
-	st	%r2,  %r1, CI_CURPCB	/* save stack as curpcb for traps */
-
 	bsr.n	_C_LABEL(secondary_main)
-	 addu	%r31, %r2, USPACE			/* switch to startup stack */
+	 addu	%r31, %r2, USPACE		/* switch to idle stack */
 
 	/*
 	 * Dummy mp_atomic_begin() and mp_atomic_end() routine, so that
diff --git a/sys/arch/aviion/aviion/machdep.c b/sys/arch/aviion/aviion/machdep.c
index c2dd086a65c..11086a3ccd4 100644
--- a/sys/arch/aviion/aviion/machdep.c
+++ b/sys/arch/aviion/aviion/machdep.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: machdep.c,v 1.73 2014/12/10 15:29:53 mikeb Exp $	*/
+/*	$OpenBSD: machdep.c,v 1.74 2015/02/25 17:41:22 miod Exp $	*/
 /*
  * Copyright (c) 2007 Miodrag Vallat.
  *
@@ -122,7 +122,7 @@ void	dumpconf(void);
 void	dumpsys(void);
 void	savectx(struct pcb *);
 void	secondary_main(void);
-vaddr_t	secondary_pre_main(void);
+void   *secondary_pre_main(void);
 
 extern void bootstack(void);
 
@@ -132,9 +132,10 @@ struct vm_map *exec_map = NULL;
 struct vm_map *phys_map = NULL;
 
 #ifdef MULTIPROCESSOR
-u_int	hatch_pending_count = 0;
 __cpu_simple_lock_t cpu_hatch_mutex = __SIMPLELOCK_LOCKED;
 __cpu_simple_lock_t cpu_boot_mutex = __SIMPLELOCK_LOCKED;
+unsigned int hatch_pending_count;
+unsigned int hatch_mask;
 #endif
 
 struct uvm_constraint_range  dma_constraint = { 0x0, (paddr_t)-1 };
@@ -501,15 +502,14 @@ abort:
 
 /*
  * Secondary CPU early initialization routine.
- * Determine CPU number and set it, then allocate its startup stack.
+ * Determine CPU number and set it, then return the startup stack.
  *
  * Running on a minimal stack here, with interrupts disabled; do nothing fancy.
  */
-vaddr_t
+void *
 secondary_pre_main()
 {
 	struct cpu_info *ci;
-	vaddr_t init_stack;
 
 	/*
 	 * Invoke the CMMU initialization routine as early as possible,
@@ -533,13 +533,8 @@ secondary_pre_main()
 	 */
 	pmap_bootstrap_cpu(ci->ci_cpuid);
 
-	/*
-	 * Allocate UPAGES contiguous pages for the startup stack.
-	 */
-	init_stack = uvm_km_zalloc(kernel_map, USPACE);
-	if (init_stack == (vaddr_t)NULL) {
-		printf("cpu%d: unable to allocate startup stack\n",
-		    ci->ci_cpuid);
+	if (ci->ci_curpcb == NULL) {
+		printf("cpu%d: unable to get startup stack\n", ci->ci_cpuid);
 		/*
 		 * Release cpu_hatch_mutex to let other secondary processors
 		 * have a chance to run.
@@ -548,7 +543,7 @@ secondary_pre_main()
 		for (;;) ;
 	}
 
-	return (init_stack);
+	return ci->ci_curpcb;
 }
 
 /*
@@ -782,6 +777,7 @@ cpu_hatch_secondary_processors()
 			rc = scm_jpstart(cpu, (vaddr_t)secondary_start);
 			switch (rc) {
 			case JPSTART_OK:
+				hatch_mask |= 1U << cpu;
 				break;
 			case JPSTART_SINGLE_JP:
 				/* this should never happen, but just in case */
@@ -806,6 +802,20 @@ cpu_hatch_secondary_processors()
 void
 cpu_setup_secondary_processors()
 {
+	cpuid_t cpu;
+
+	/*
+	 * Allocate UPAGES contiguous pages for the idle stack of every
+	 * running secondary processor.
+	 */
+	for (cpu = 0; cpu < MAX_CPUS; cpu++) {
+		if ((hatch_mask & (1U << cpu)) == 0)
+			continue;
+
+		m88k_cpus[cpu].ci_curpcb =
+		    (void *)uvm_km_zalloc(kernel_map, USPACE);
+	}
+
 	__cpu_simple_unlock(&cpu_hatch_mutex);
 	while (hatch_pending_count != 0)
 		delay(10000);	/* 10ms */