Make MULTIPROCESSOR kernels work on sun4v. Won't gracefully halt, powerdown

or reboot yet, but that will (hopefully) be fixed in the near future.

6 files changed, 212 insertions, 35 deletions

diff --git a/sys/arch/sparc64/include/cpu.h b/sys/arch/sparc64/include/cpu.h
index a80575ebd1b..27fc8491636 100644
--- a/sys/arch/sparc64/include/cpu.h
+++ b/sys/arch/sparc64/include/cpu.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: cpu.h,v 1.58 2008/03/23 23:46:21 kettenis Exp $	*/
+/*	$OpenBSD: cpu.h,v 1.59 2008/03/31 22:14:01 kettenis Exp $	*/
 /*	$NetBSD: cpu.h,v 1.28 2001/06/14 22:56:58 thorpej Exp $ */
 
 /*
@@ -157,7 +157,18 @@ extern struct cpu_info *cpus;
 #ifdef MULTIPROCESSOR
 
 #define	cpu_number()	(curcpu()->ci_number)
-#define	curcpu()	(((struct cpu_info *)CPUINFO_VA)->ci_self)
+
+extern __inline struct cpu_info *curcpu(void);
+extern __inline struct cpu_info *
+curcpu(void)
+{
+	if (CPU_ISSUN4V) {
+		__asm __volatile("nop" : : : "memory");
+		return (struct cpu_info *)ldxa(0, ASI_SCRATCHPAD);
+	}
+
+	return (((struct cpu_info *)CPUINFO_VA)->ci_self);
+}
 
 #define CPU_IS_PRIMARY(ci)	((ci)->ci_number == 0)
 #define CPU_INFO_ITERATOR	int
diff --git a/sys/arch/sparc64/sparc64/autoconf.c b/sys/arch/sparc64/sparc64/autoconf.c
index ed64488df53..605a5f0960c 100644
--- a/sys/arch/sparc64/sparc64/autoconf.c
+++ b/sys/arch/sparc64/sparc64/autoconf.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: autoconf.c,v 1.84 2008/03/30 12:30:01 kettenis Exp $	*/
+/*	$OpenBSD: autoconf.c,v 1.85 2008/03/31 22:14:01 kettenis Exp $	*/
 /*	$NetBSD: autoconf.c,v 1.51 2001/07/24 19:32:11 eeh Exp $ */
 
 /*
@@ -329,6 +329,24 @@ bootstrap(nctx)
 		insn = 0x94102003; 		/* mov MAP_ITLB|MAP_DTLB, %o2 */
 		((u_int32_t *)sp_tlb_flush_ctx)[1] = insn;
 
+#ifdef MULTIPROCESSOR
+	{
+		struct sun4v_patch {
+			u_int32_t addr;
+			u_int32_t insn;
+		};
+
+		extern struct sun4v_patch sun4v_mp_patch;
+		extern struct sun4v_patch sun4v_mp_patch_end;
+		struct sun4v_patch *p;
+
+		for (p = &sun4v_mp_patch; p < &sun4v_mp_patch_end; p++) {
+			*(u_int32_t *)(vaddr_t)p->addr = p->insn;
+			flush((void *)(vaddr_t)p->addr);
+		}
+	}
+#endif
+
 		cacheinfo.c_dcache_flush_page = no_dcache_flush_page;
 	}
 #endif
diff --git a/sys/arch/sparc64/sparc64/ipifuncs.c b/sys/arch/sparc64/sparc64/ipifuncs.c
index cddbcac4d8f..b5117888453 100644
--- a/sys/arch/sparc64/sparc64/ipifuncs.c
+++ b/sys/arch/sparc64/sparc64/ipifuncs.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: ipifuncs.c,v 1.7 2008/03/15 22:05:51 kettenis Exp $	*/
+/*	$OpenBSD: ipifuncs.c,v 1.8 2008/03/31 22:14:01 kettenis Exp $	*/
 /*	$NetBSD: ipifuncs.c,v 1.8 2006/10/07 18:11:36 rjs Exp $ */
 
 /*-
@@ -40,6 +40,7 @@
 
 #include <machine/cpu.h>
 #include <machine/ctlreg.h>
+#include <machine/hypervisor.h>
 #include <machine/pmap.h>
 #include <machine/sparc64.h>
 
@@ -49,11 +50,18 @@ extern int db_active;
 
 #define	sparc64_ipi_sleep()	delay(1000)
 
+void	sun4u_send_ipi(int, void (*)(void), u_int64_t, u_int64_t);
+void	sun4u_broadcast_ipi(void (*)(void), u_int64_t, u_int64_t);
+void	sun4v_send_ipi(int, void (*)(void), u_int64_t, u_int64_t);
+void	sun4v_broadcast_ipi(void (*)(void), u_int64_t, u_int64_t);
+
 /*
  * These are the "function" entry points in locore.s to handle IPI's.
  */
-void	ipi_tlb_page_demap(void);
-void	ipi_tlb_context_demap(void);
+void	sun4u_ipi_tlb_page_demap(void);
+void	sun4u_ipi_tlb_context_demap(void);
+void	sun4v_ipi_tlb_page_demap(void);
+void	sun4v_ipi_tlb_context_demap(void);
 void	ipi_softint(void);
 
 /*
@@ -62,6 +70,15 @@ void	ipi_softint(void);
 void
 sparc64_send_ipi(int itid, void (*func)(void), u_int64_t arg0, u_int64_t arg1)
 {
+	if (CPU_ISSUN4V)
+		sun4v_send_ipi(itid, func, arg0, arg1);
+	else
+		sun4u_send_ipi(itid, func, arg0, arg1);
+}
+
+void
+sun4u_send_ipi(int itid, void (*func)(void), u_int64_t arg0, u_int64_t arg1)
+{
 	int i, j, shift = 0;
 
 	KASSERT((u_int64_t)func > MAXINTNUM);
@@ -112,12 +129,56 @@ sparc64_send_ipi(int itid, void (*func)(void), u_int64_t arg0, u_int64_t arg1)
 #endif
 }
 
+void
+sun4v_send_ipi(int itid, void (*func)(void), u_int64_t arg0, u_int64_t arg1)
+{
+	struct cpu_info *ci = curcpu();
+	int err, i;
+
+	stha(ci->ci_cpuset, ASI_PHYS_CACHED, itid);
+	stxa(ci->ci_mondo, ASI_PHYS_CACHED, (vaddr_t)func);
+	stxa(ci->ci_mondo + 8, ASI_PHYS_CACHED, arg0);
+	stxa(ci->ci_mondo + 16, ASI_PHYS_CACHED, arg1);
+
+	for (i = 0; i < SPARC64_IPI_RETRIES; i++) {
+		err = hv_cpu_mondo_send(1, ci->ci_cpuset, ci->ci_mondo);
+		if (err != H_EWOULDBLOCK)
+			break;
+		delay(10);
+	}
+	if (err != H_EOK)
+		panic("Unable to send mondo %lx to cpu %d: %d", func, itid, err);
+}
+
 /*
  * Broadcast an IPI to all but ourselves.
  */
 void
 sparc64_broadcast_ipi(void (*func)(void), u_int64_t arg0, u_int64_t arg1)
 {
+	if (CPU_ISSUN4V)
+		sun4v_broadcast_ipi(func, arg0, arg1);
+	else
+		sun4u_broadcast_ipi(func, arg0, arg1);
+}
+
+void
+sun4u_broadcast_ipi(void (*func)(void), u_int64_t arg0, u_int64_t arg1)
+{
+	struct cpu_info *ci;
+
+	for (ci = cpus; ci != NULL; ci = ci->ci_next) {
+		if (ci->ci_number == cpu_number())
+			continue;
+		if ((ci->ci_flags & CPUF_RUNNING) == 0)
+			continue;
+		sun4u_send_ipi(ci->ci_itid, func, arg0, arg1);
+	}
+}
+
+void
+sun4v_broadcast_ipi(void (*func)(void), u_int64_t arg0, u_int64_t arg1)
+{
 	struct cpu_info *ci;
 
 	for (ci = cpus; ci != NULL; ci = ci->ci_next) {
@@ -125,7 +186,7 @@ sparc64_broadcast_ipi(void (*func)(void), u_int64_t arg0, u_int64_t arg1)
 			continue;
 		if ((ci->ci_flags & CPUF_RUNNING) == 0)
 			continue;
-		sparc64_send_ipi(ci->ci_itid, func, arg0, arg1);
+		sun4v_send_ipi(ci->ci_itid, func, arg0, arg1);
 	}
 }
 
@@ -137,7 +198,10 @@ smp_tlb_flush_pte(vaddr_t va, int ctx)
 	if (db_active)
 		return;
 
-	sparc64_broadcast_ipi(ipi_tlb_page_demap, va, ctx);
+	if (CPU_ISSUN4V)
+		sun4v_broadcast_ipi(sun4v_ipi_tlb_page_demap, va, ctx);
+	else
+		sun4u_broadcast_ipi(sun4u_ipi_tlb_page_demap, va, ctx);
 }
 
 void
@@ -148,14 +212,22 @@ smp_tlb_flush_ctx(int ctx)
 	if (db_active)
 		return;
 
-	sparc64_broadcast_ipi(ipi_tlb_context_demap, ctx, 0);
+	if (CPU_ISSUN4V)
+		sun4v_broadcast_ipi(sun4v_ipi_tlb_context_demap, ctx, 0);
+	else
+		sun4u_broadcast_ipi(sun4u_ipi_tlb_context_demap, ctx, 0);
 }
 
 void
 smp_signotify(struct proc *p)
 {
+	struct cpu_info *ci = p->p_cpu;
+
 	if (db_active)
 		return;
 
-	sparc64_send_ipi(p->p_cpu->ci_itid, ipi_softint, 1 << IPL_NONE, 0UL);
+	if (CPU_ISSUN4V)
+		sun4v_send_ipi(ci->ci_itid, ipi_softint, 1 << IPL_NONE, 0UL);
+	else
+		sun4u_send_ipi(ci->ci_itid, ipi_softint, 1 << IPL_NONE, 0UL);
 }
diff --git a/sys/arch/sparc64/sparc64/locore.s b/sys/arch/sparc64/sparc64/locore.s
index a7caaae7b79..fb1709bc8ac 100644
--- a/sys/arch/sparc64/sparc64/locore.s
+++ b/sys/arch/sparc64/sparc64/locore.s
@@ -1,4 +1,4 @@
-/*	$OpenBSD: locore.s,v 1.128 2008/03/30 13:39:53 kettenis Exp $	*/
+/*	$OpenBSD: locore.s,v 1.129 2008/03/31 22:14:01 kettenis Exp $	*/
 /*	$NetBSD: locore.s,v 1.137 2001/08/13 06:10:10 jdolecek Exp $	*/
 
 /*
@@ -96,8 +96,19 @@
 #define NOTREACHED
 #endif	/* 1 */
 
+#ifdef MULTIPROCESSOR
+	.section	.sun4v_mp_patch, "ax"
+	.globl _C_LABEL(sun4v_mp_patch)
+_C_LABEL(sun4v_mp_patch):
+	.previous
+#endif
+
 #define GET_CPUINFO_VA(ci) \
-	set CPUINFO_VA, ci
+999:	set	CPUINFO_VA, ci			;\
+	.section	.sun4v_mp_patch, "ax"	;\
+	.word	999b				;\
+	ldxa	[%g0] ASI_SCRATCHPAD, ci	;\
+	.previous
 
 #define GET_CPCB(pcb) \
 	GET_CPUINFO_VA(pcb)			;\
@@ -110,12 +121,13 @@
 #ifdef SUN4V
 
 #define GET_CPUINFO_PA(ci) \
-	nop					;\
-	ldxa	[%g0] ASI_SCRATCHPAD, ci
+	mov	8, ci				;\
+	ldxa	[ci] ASI_SCRATCHPAD, ci
 
 #define GET_MMFSA(mmfsa) \
-	GET_CPUINFO_VA(mmfsa)			;\
-	ldx	[mmfsa + CI_MMFSA], mmfsa
+	GET_CPUINFO_PA(mmfsa)			;\
+	add	mmfsa, CI_MMFSA, mmfsa		;\
+	ldxa	[mmfsa] ASI_PHYS_CACHED, mmfsa
 
 #endif
 
@@ -1485,10 +1497,9 @@ intr_setup_msg:
 	.macro	INTR_SETUP stackspace, label
 	rdpr	%wstate, %g7			! Find if we're from user mode
 
-	sethi	%hi(EINTSTACK-BIAS), %g6
+	GET_CPUINFO_VA(%g6)
 	sethi	%hi(EINTSTACK-INTSTACK), %g4
-
-	or	%g6, %lo(EINTSTACK-BIAS), %g6	! Base of interrupt stack
+	sub	%g6, BIAS, %g6			! Base of interrupt stack
 	dec	%g4				! Make it into a mask
 
 	sub	%g6, %sp, %g1			! Offset from interrupt stack
@@ -3624,9 +3635,9 @@ gl0_1:	nop
  *	%g4 = tt == T_AST
  */
 softtrap:
-	sethi	%hi(EINTSTACK-BIAS), %g5
+	GET_CPUINFO_VA(%g5)
 	sethi	%hi(EINTSTACK-INTSTACK), %g7
-	or	%g5, %lo(EINTSTACK-BIAS), %g5
+	sub	%g5, BIAS, %g5
 	dec	%g7
 
 	sub	%g5, %sp, %g5
@@ -3977,7 +3988,7 @@ sun4v_dev_mondo:
 #endif
 
 #ifdef MULTIPROCESSOR
-ENTRY(ipi_tlb_page_demap)
+ENTRY(sun4u_ipi_tlb_page_demap)
 	rdpr	%pstate, %g1
 	andn	%g1, PSTATE_IE, %g2
 	wrpr	%g2, %pstate				! disable interrupts
@@ -4008,7 +4019,7 @@ ENTRY(ipi_tlb_page_demap)
 	ba,a	ret_from_intr_vector
 	 nop
 
-ENTRY(ipi_tlb_context_demap)
+ENTRY(sun4u_ipi_tlb_context_demap)
 	rdpr	%pstate, %g1
 	andn	%g1, PSTATE_IE, %g2
 	wrpr	%g2, %pstate				! disable interrupts
@@ -4038,6 +4049,25 @@ ENTRY(ipi_tlb_context_demap)
 	ba,a	ret_from_intr_vector
 	 nop
 
+#ifdef SUN4V
+ENTRY(sun4v_ipi_tlb_page_demap)
+	mov	%o0, %g1
+	mov	%o1, %g2
+	mov	%o2, %g4
+	mov	%g3, %o0
+	mov	%g5, %o1
+	mov	MAP_DTLB|MAP_ITLB, %o2
+	ta	MMU_UNMAP_ADDR
+	mov	%g1, %o0
+	mov	%g2, %o1
+	mov	%g4, %o2
+
+	retry
+
+ENTRY(sun4v_ipi_tlb_context_demap)
+	NOTREACHED
+#endif
+
 ENTRY(ipi_save_fpstate)
 	GET_CPUINFO_VA(%g1)
 	ldx	[%g1 + CI_FPPROC], %g2
@@ -4045,8 +4075,10 @@ ENTRY(ipi_save_fpstate)
 	bne,pn	%xcc, 3f
 
 	 mov	CTX_SECONDARY, %g2
+ctxid_9:
 	ldxa	[%g2] ASI_DMMU, %g6
 	membar	#LoadStore
+ctxid_10:
 	stxa	%g0, [%g2] ASI_DMMU
 	membar	#Sync
 
@@ -4088,6 +4120,7 @@ ENTRY(ipi_save_fpstate)
 
 	stx	%g0, [%g1 + CI_FPPROC]	! fpproc = NULL
 	mov	CTX_SECONDARY, %g2
+ctxid_11:
 	stxa	%g6, [%g2] ASI_DMMU
 	membar	#Sync
 3:
@@ -4800,8 +4833,7 @@ _C_LABEL(cpu_initialize):
 	bne,pt	%icc, 1f
 	 nop
 	set	_C_LABEL(trapbase_sun4v), %l1
-	GET_CPUINFO_VA(%o1)
-	ldx	[%o1 + CI_MMFSA], %o1
+	GET_MMFSA(%o1)
 1:
 #endif
 	call	_C_LABEL(prom_set_trap_table)	! Now we should be running 100% from our handlers
@@ -9103,5 +9135,20 @@ _C_LABEL(ctxid_start):
 	.xword	ctxid_6
 	.xword	ctxid_7
 	.xword	ctxid_8
-	.xword	0
+#ifdef MULTIPROCESSOR
+	.xword	ctxid_9
+	.xword	ctxid_10
+	.xword	ctxid_11
+#endif
+	.xword	0	
+#endif
+
+	/* XXX This is in mutex.S for now */
+#if 0
+#ifdef MULTIPROCESSOR
+	.section	.sun4v_mp_patch, "ax"
+	.globl _C_LABEL(sun4v_mp_patch_end)
+_C_LABEL(sun4v_mp_patch_end):
+	.previous
+#endif
 #endif
diff --git a/sys/arch/sparc64/sparc64/mutex.S b/sys/arch/sparc64/sparc64/mutex.S
index 70f7e146041..2243f54e855 100644
--- a/sys/arch/sparc64/sparc64/mutex.S
+++ b/sys/arch/sparc64/sparc64/mutex.S
@@ -1,4 +1,4 @@
-/*	$OpenBSD: mutex.S,v 1.1 2007/08/25 18:36:47 kettenis Exp $	*/
+/*	$OpenBSD: mutex.S,v 1.2 2008/03/31 22:14:01 kettenis Exp $	*/
 
 /*
  * Copyright (c) 2007 Mark Kettenis
@@ -25,6 +25,26 @@
 #include <machine/asm.h>
 #include <machine/param.h>
 
+#ifdef MULTIPROCESSOR
+
+#include <machine/ctlreg.h>
+
+#define GET_CURCPU(ci) \
+	sethi	%hi(CPUINFO_VA + CI_SELF), ci		;\
+999:	ldx	[ci + %lo(CPUINFO_VA + CI_SELF)], ci	;\
+	.section	.sun4v_mp_patch, "ax"		;\
+	.word	999b					;\
+	ldxa	[%g0] ASI_SCRATCHPAD, ci		;\
+	.previous
+
+#else
+
+#define GET_CURCPU(ci) \
+	set	CPUINFO_VA, ci
+
+#endif
+
+
 ENTRY(mtx_init)
 	stx	%g0, [%o0 + MTX_OWNER]
 	stw	%o1, [%o0 + MTX_WANTIPL]
@@ -32,13 +52,8 @@ ENTRY(mtx_init)
 	 stw	%g0, [%o0 + MTX_OLDIPL]
 
 ENTRY(mtx_enter)
-#ifdef MULTIPROCESSOR
-	sethi	%hi(CPUINFO_VA+CI_SELF), %g1
-	ldx	[%g1 + %lo(CPUINFO_VA+CI_SELF)], %g1
-#else
-	set	CPUINFO_VA, %g1
-#endif
 	rdpr	%pil, %g4
+	GET_CURCPU(%g1)
 1:
 	ld	[%o0 + MTX_WANTIPL], %g5
 	cmp	%g4, %g5
@@ -73,3 +88,13 @@ ENTRY(mtx_leave)
 	stx	%g0, [%o0 + MTX_OWNER]
 	retl
 	 wrpr	%g1, %pil
+
+/*
+ *	XXX Move this back to locore.s.
+ */
+#ifdef MULTIPROCESSOR
+	.section	.sun4v_mp_patch, "ax"
+	.globl _C_LABEL(sun4v_mp_patch_end)
+_C_LABEL(sun4v_mp_patch_end):
+	.previous
+#endif
diff --git a/sys/arch/sparc64/sparc64/pmap.c b/sys/arch/sparc64/sparc64/pmap.c
index 02d651c643a..e4f4c871e99 100644
--- a/sys/arch/sparc64/sparc64/pmap.c
+++ b/sys/arch/sparc64/sparc64/pmap.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: pmap.c,v 1.56 2008/03/30 12:30:01 kettenis Exp $	*/
+/*	$OpenBSD: pmap.c,v 1.57 2008/03/31 22:14:01 kettenis Exp $	*/
 /*	$NetBSD: pmap.c,v 1.107 2001/08/31 16:47:41 eeh Exp $	*/
 #undef	NO_VCACHE /* Don't forget the locked TLB in dostart */
 /*
@@ -1556,9 +1556,13 @@ sun4v_bootstrap_cpu(paddr_t intstack)
 	err = hv_mmu_map_perm_addr(INTSTACK, data, MAP_DTLB);
 	if (err != H_EOK)
 		prom_printf("err: %d\r\n", err);
+#else
+	pa = intstack + (CPUINFO_VA - INTSTACK);
+	pa += offsetof(struct cpu_info, ci_self);
+	stxa(0, ASI_SCRATCHPAD, ldxa(pa, ASI_PHYS_CACHED));
 #endif
 
-	stxa(0, ASI_SCRATCHPAD, intstack + (CPUINFO_VA - INTSTACK));
+	stxa(8, ASI_SCRATCHPAD, intstack + (CPUINFO_VA - INTSTACK));
 
 	err = hv_mmu_tsb_ctx0(1, (paddr_t)tsb_desc + kdatap - kdata);
 	if (err != H_EOK)