Extend struct platform with MP control functions, and add bootstrap code

for secondary CPUs.

Discussed with miod@

6 files changed, 155 insertions, 6 deletions

diff --git a/sys/arch/loongson/dev/mainbus.c b/sys/arch/loongson/dev/mainbus.c
index 87501a88c2e..390a12d6ea1 100644
--- a/sys/arch/loongson/dev/mainbus.c
+++ b/sys/arch/loongson/dev/mainbus.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: mainbus.c,v 1.9 2016/11/17 14:41:21 visa Exp $ */
+/*	$OpenBSD: mainbus.c,v 1.10 2017/01/19 15:09:04 visa Exp $ */
 
 /*
  * Copyright (c) 2001-2003 Opsycon AB  (www.opsycon.se / www.opsycon.com)
@@ -67,6 +67,11 @@ mainbus_attach(struct device *parent, struct device *self, void *aux)
 	caa.caa_hw = &bootcpu_hwinfo;
 	config_found(self, &caa, mainbus_print);
 
+#ifdef MULTIPROCESSOR
+	if (sys_platform->config_secondary_cpus != NULL)
+		sys_platform->config_secondary_cpus(self, mainbus_print);
+#endif
+
 	caa.caa_maa.maa_name = "bonito";
 	config_found(self, &caa.caa_maa, mainbus_print);
 
diff --git a/sys/arch/loongson/include/autoconf.h b/sys/arch/loongson/include/autoconf.h
index c7d20447921..3de86536b7b 100644
--- a/sys/arch/loongson/include/autoconf.h
+++ b/sys/arch/loongson/include/autoconf.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: autoconf.h,v 1.13 2016/12/11 07:57:14 visa Exp $ */
+/*	$OpenBSD: autoconf.h,v 1.14 2017/01/19 15:09:04 visa Exp $ */
 
 /*
  * Copyright (c) 2001-2003 Opsycon AB  (www.opsycon.se / www.opsycon.com)
@@ -74,6 +74,17 @@ struct platform {
 	void				(*reset)(void);
 	int				(*suspend)(void);
 	int				(*resume)(void);
+
+#ifdef MULTIPROCESSOR
+	void				(*config_secondary_cpus)(
+					    struct device *, cfprint_t);
+	void				(*boot_secondary_cpu)(
+					    struct cpu_info *);
+	int				(*ipi_establish)(int (*)(void *),
+					    cpuid_t);
+	void				(*ipi_set)(cpuid_t);
+	void				(*ipi_clear)(cpuid_t);
+#endif /* MULTIPROCESSOR */
 };
 
 #define LOONGSON_MAXCPUS	16
diff --git a/sys/arch/loongson/include/cpu.h b/sys/arch/loongson/include/cpu.h
index 6120b24e787..d1bb9892eb1 100644
--- a/sys/arch/loongson/include/cpu.h
+++ b/sys/arch/loongson/include/cpu.h
@@ -1,4 +1,4 @@
-/* $OpenBSD: cpu.h,v 1.6 2016/01/05 05:27:54 visa Exp $ */
+/* $OpenBSD: cpu.h,v 1.7 2017/01/19 15:09:04 visa Exp $ */
 /*-
  * Copyright (c) 1992, 1993
  *	The Regents of the University of California.  All rights reserved.
@@ -43,6 +43,18 @@
 
 #ifdef	_KERNEL
 
+#if defined(MULTIPROCESSOR) && !defined(_LOCORE)
+struct cpu_info;
+struct cpu_info *hw_getcurcpu(void);
+void hw_setcurcpu(struct cpu_info *);
+void hw_cpu_boot_secondary(struct cpu_info *);
+void hw_cpu_hatch(struct cpu_info *);
+void hw_cpu_spinup_trampoline(struct cpu_info *);
+int hw_ipi_intr_establish(int (*)(void *), u_long);
+void hw_ipi_intr_set(u_long);
+void hw_ipi_intr_clear(u_long);
+#endif	/* MULTIPROCESSOR && !_LOCORE */
+
 #if defined(CPU_LOONGSON2) && !defined(CPU_LOONGSON3)
 #define	Mips_SyncCache(ci)			\
 	Loongson2_SyncCache((ci))
diff --git a/sys/arch/loongson/include/intr.h b/sys/arch/loongson/include/intr.h
index 949e33f6b0e..f0aa4b8f9cd 100644
--- a/sys/arch/loongson/include/intr.h
+++ b/sys/arch/loongson/include/intr.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: intr.h,v 1.10 2016/11/17 14:41:21 visa Exp $ */
+/*	$OpenBSD: intr.h,v 1.11 2017/01/19 15:09:04 visa Exp $ */
 
 /*
  * Copyright (c) 2001-2004 Opsycon AB  (www.opsycon.se / www.opsycon.com)
@@ -189,6 +189,11 @@ void	set_intr(int, uint32_t, uint32_t(*)(uint32_t, struct trapframe *));
 uint32_t updateimask(uint32_t);
 void	dosoftint(void);
 
+#ifdef MULTIPROCESSOR
+extern uint32_t ipi_mask;
+#define ENABLEIPI() updateimask(~ipi_mask)
+#endif
+
 struct pic {
 	void	(*pic_eoi)(int);
 	void	(*pic_mask)(int);
diff --git a/sys/arch/loongson/loongson/locore.S b/sys/arch/loongson/loongson/locore.S
index b2248fd73cb..8fde8bf9ae0 100644
--- a/sys/arch/loongson/loongson/locore.S
+++ b/sys/arch/loongson/loongson/locore.S
@@ -1,4 +1,4 @@
-/*	$OpenBSD: locore.S,v 1.6 2016/11/18 15:38:14 visa Exp $ */
+/*	$OpenBSD: locore.S,v 1.7 2017/01/19 15:09:04 visa Exp $ */
 
 /*
  * Copyright (c) 2001-2004 Opsycon AB  (www.opsycon.se / www.opsycon.com)
@@ -96,6 +96,24 @@ END(hibernate_resume_machdep)					/* } */
 
 #ifdef MULTIPROCESSOR
 
+LEAF(hw_cpu_spinup_trampoline, 0)
+	/* Disable interrupts. */
+	mfc0    v0, COP_0_STATUS_REG
+	li	v1, ~SR_INT_ENAB
+	and	v0, v1
+	mtc0    v0, COP_0_STATUS_REG
+
+	/* Clear any pending soft interrupts. */
+	mtc0	zero, COP_0_CAUSE_REG
+
+	LA	t0, cpu_spinup_a0
+	ld	a0, 0(t0)
+	LA	t0, cpu_spinup_sp
+	ld	sp, 0(t0)
+	jal	hw_cpu_hatch
+	 LA	gp, _gp
+END(hw_cpu_spinup_trampoline)
+
 LEAF(hw_getcurcpu, 0)
 	GET_CPU_INFO(v0, v1)
 	jr	ra
diff --git a/sys/arch/loongson/loongson/machdep.c b/sys/arch/loongson/loongson/machdep.c
index 6fa97451063..7aa6efac71b 100644
--- a/sys/arch/loongson/loongson/machdep.c
+++ b/sys/arch/loongson/loongson/machdep.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: machdep.c,v 1.71 2017/01/06 15:37:44 fcambus Exp $ */
+/*	$OpenBSD: machdep.c,v 1.72 2017/01/19 15:09:04 visa Exp $ */
 
 /*
  * Copyright (c) 2009, 2010, 2014 Miodrag Vallat.
@@ -118,6 +118,12 @@ int	nnodes = 1;		/* Number of NUMA nodes, only on 3A. */
 struct	user *proc0paddr;
 int	lid_suspend = 1;
 
+#ifdef MULTIPROCESSOR
+uint64_t cpu_spinup_a0;
+uint64_t cpu_spinup_sp;
+uint32_t ipi_mask;
+#endif
+
 const struct platform *sys_platform;
 struct cpu_hwinfo bootcpu_hwinfo;
 uint loongson_cpumask = 1;
@@ -512,6 +518,13 @@ mips_init(uint64_t argc, uint64_t argv, uint64_t envp, uint64_t cv,
 	extern char *hw_vendor, *hw_prod;
 	extern void xtlb_miss;
 
+#ifdef MULTIPROCESSOR
+	/*
+	 * Set curcpu address on primary processor.
+	 */
+	setcurcpu(&cpu_info_primary);
+#endif
+
 	/*
 	 * Make sure we can access the extended address space.
 	 * This is not necessary on real hardware, but some emulators
@@ -1231,3 +1244,88 @@ pmoncnputc(dev_t dev, int c)
 	else
 		pmon_printf("%c", c);
 }
+
+#ifdef MULTIPROCESSOR
+
+void
+hw_cpu_hatch(struct cpu_info *ci)
+{
+	int s;
+
+	/*
+	 * Set curcpu address on this processor.
+	 */
+	setcurcpu(ci);
+
+	tlb_init(ci->ci_hw.tlbsize);
+	tlb_set_pid(0);
+
+	/*
+	 * Make sure we can access the extended address space.
+	 */
+	setsr(getsr() | SR_KX | SR_UX);
+
+	/*
+	 * Turn off bootstrap exception vectors.
+	 */
+	setsr(getsr() & ~SR_BOOT_EXC_VEC);
+
+	/*
+	 * Clear out the I and D caches.
+	 */
+	switch (loongson_ver) {
+#ifdef CPU_LOONGSON3
+	case 0x3a:
+	case 0x3b:
+		Loongson3_ConfigCache(ci);
+		Loongson3_SyncCache(ci);
+		break;
+#endif
+	default:
+		panic("%s: unhandled Loongson version %x\n", __func__,
+		    loongson_ver);
+	}
+
+	(*md_startclock)(ci);
+
+	mips64_ipi_init();
+
+	ncpus++;
+	cpuset_add(&cpus_running, ci);
+
+	spl0();
+	(void)updateimask(0);
+
+	SCHED_LOCK(s);
+	cpu_switchto(NULL, sched_chooseproc());
+}
+
+void
+hw_cpu_boot_secondary(struct cpu_info *ci)
+{
+	sys_platform->boot_secondary_cpu(ci);
+}
+
+int
+hw_ipi_intr_establish(int (*func)(void *), u_long cpuid)
+{
+	if (sys_platform->ipi_establish != NULL)
+		return sys_platform->ipi_establish(func, cpuid);
+	else
+		return 0;
+}
+
+void
+hw_ipi_intr_set(u_long cpuid)
+{
+	sys_platform->ipi_set(cpuid);
+}
+
+void
+hw_ipi_intr_clear(u_long cpuid)
+{
+	if (sys_platform->ipi_clear != NULL)
+		sys_platform->ipi_clear(cpuid);
+}
+
+#endif /* MULTIPROCESSOR */