Make lazy fpu context switching work for MULTIPROCESSOR kernels. Tested by

many.

ok deraadt@

4 files changed, 120 insertions, 17 deletions

diff --git a/sys/arch/sparc64/include/cpu.h b/sys/arch/sparc64/include/cpu.h
index c9da79d2235..b888013280d 100644
--- a/sys/arch/sparc64/include/cpu.h
+++ b/sys/arch/sparc64/include/cpu.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: cpu.h,v 1.47 2007/10/16 19:13:53 kettenis Exp $	*/
+/*	$OpenBSD: cpu.h,v 1.48 2007/10/16 19:22:49 kettenis Exp $	*/
 /*	$NetBSD: cpu.h,v 1.28 2001/06/14 22:56:58 thorpej Exp $ */
 
 /*
@@ -225,6 +225,7 @@ int	statintr(void *);	/* level 14 (statclock) interrupt code */
 struct fpstate64;
 void	savefpstate(struct fpstate64 *);
 void	loadfpstate(struct fpstate64 *);
+void	clearfpstate(void);
 u_int64_t	probeget(paddr_t, int, int);
 #define	 write_all_windows() __asm __volatile("flushw" : : )
 #define	 write_user_windows() __asm __volatile("flushw" : : )
@@ -240,6 +241,8 @@ void	switchtoctx(int);
 /* trap.c */
 void	pmap_unuse_final(struct proc *);
 int	rwindow_save(struct proc *);
+/* vm_machdep.c */
+void	save_and_clear_fpstate(struct proc *);
 /* amd7930intr.s */
 void	amd7930_trap(void);
 /* cons.c */
diff --git a/sys/arch/sparc64/sparc64/locore.s b/sys/arch/sparc64/sparc64/locore.s
index 2456810258e..db824e1d941 100644
--- a/sys/arch/sparc64/sparc64/locore.s
+++ b/sys/arch/sparc64/sparc64/locore.s
@@ -1,4 +1,4 @@
-/*	$OpenBSD: locore.s,v 1.84 2007/10/10 15:53:53 art Exp $	*/
+/*	$OpenBSD: locore.s,v 1.85 2007/10/16 19:22:49 kettenis Exp $	*/
 /*	$NetBSD: locore.s,v 1.137 2001/08/13 06:10:10 jdolecek Exp $	*/
 
 /*
@@ -3351,6 +3351,34 @@ ENTRY(ipi_tlb_context_demap)
 	wrpr	%g1, %pstate
 	ba,a	ret_from_intr_vector
 	 nop
+
+ENTRY(ipi_save_fpstate)
+	save	%sp, -CC64FSZ, %sp
+	mov	CTX_PRIMARY, %o2
+	ldxa	[%o2] ASI_DMMU, %g7
+	membar	#LoadStore
+	stxa	%g0, [%o2] ASI_DMMU
+	membar	#Sync
+	sethi	%hi(FPPROC), %o0
+	ldx	[%o0 + %lo(FPPROC)], %o0
+	call	savefpstate
+	 ldx	[%o0 + P_FPSTATE], %o0
+	sethi	%hi(FPPROC), %o0
+	stx	%g0, [%o0 + %lo(FPPROC)]	! fpproc = NULL
+	mov	CTX_PRIMARY, %o2
+	stxa	%g7, [%o2] ASI_DMMU
+	membar	#Sync
+	ba	ret_from_intr_vector
+	 restore
+
+ENTRY(ipi_drop_fpstate)
+	rdpr	%pstate, %g1
+	wr	%g0, FPRS_FEF, %fprs
+	or	%g1, PSTATE_PEF, %g1
+	wrpr	%g1, 0, %pstate
+	sethi	%hi(FPPROC), %g1
+	ba	ret_from_intr_vector
+	 stx	%g0, [%g1 + %lo(FPPROC)]	! fpproc = NULL
 #endif
 
 /*
@@ -8552,6 +8580,18 @@ Lback_mopb:
 
 
 /*
+ * clearfpstate()
+ *
+ * Drops the current fpu state, without saving it.
+ */
+ENTRY(clearfpstate)
+	rdpr	%pstate, %o1		! enable FPU
+	wr	%g0, FPRS_FEF, %fprs
+	or	%o1, PSTATE_PEF, %o1
+	retl
+	 wrpr	%o1, 0, %pstate
+
+/*
  * savefpstate(f) struct fpstate *f;
  *
  * Store the current FPU state.  The first `st %fsr' may cause a trap;
diff --git a/sys/arch/sparc64/sparc64/trap.c b/sys/arch/sparc64/sparc64/trap.c
index 48d07c2246e..a9821cca035 100644
--- a/sys/arch/sparc64/sparc64/trap.c
+++ b/sys/arch/sparc64/sparc64/trap.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: trap.c,v 1.55 2007/10/15 09:24:31 fgsch Exp $	*/
+/*	$OpenBSD: trap.c,v 1.56 2007/10/16 19:22:49 kettenis Exp $	*/
 /*	$NetBSD: trap.c,v 1.73 2001/08/09 01:03:01 eeh Exp $ */
 
 /*
@@ -424,9 +424,12 @@ trap(tf, type, pc, tstate)
 
 			if (CLKF_INTR((struct clockframe *)tf) || !curproc)
 				newfpproc = &proc0;
-			else
+			else {
 				newfpproc = curproc;
-
+				/* force other cpus to give up this fpstate */
+				if (newfpproc->p_md.md_fpstate)
+					save_and_clear_fpstate(newfpproc);
+			}
 			if (fpproc != newfpproc) {
 				if (fpproc != NULL) {
 					/* someone else had it, maybe? */
@@ -571,6 +574,8 @@ badtrap:
 			break;
 		}
 		if (fpproc != p) {		/* we do not have it */
+			/* but maybe another CPU has it? */
+			save_and_clear_fpstate(p);
 			if (fpproc != NULL)	/* someone else had it */
 				savefpstate(fpproc->p_md.md_fpstate);
 			loadfpstate(fs);
diff --git a/sys/arch/sparc64/sparc64/vm_machdep.c b/sys/arch/sparc64/sparc64/vm_machdep.c
index ac50d35e829..a9a8508d960 100644
--- a/sys/arch/sparc64/sparc64/vm_machdep.c
+++ b/sys/arch/sparc64/sparc64/vm_machdep.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: vm_machdep.c,v 1.17 2007/10/13 07:18:32 miod Exp $	*/
+/*	$OpenBSD: vm_machdep.c,v 1.18 2007/10/16 19:22:49 kettenis Exp $	*/
 /*	$NetBSD: vm_machdep.c,v 1.38 2001/06/30 00:02:20 eeh Exp $ */
 
 /*
@@ -227,11 +227,8 @@ cpu_fork(p1, p2, stack, stacksize, func, arg)
 	opcb->lastcall = NULL;
 #endif
 	bcopy((caddr_t)opcb, (caddr_t)npcb, sizeof(struct pcb));
-       	if (p1->p_md.md_fpstate) {
-		if (p1 == fpproc) {
-			savefpstate(p1->p_md.md_fpstate);
-			fpproc = NULL;
-		}
+	if (p1->p_md.md_fpstate) {
+		save_and_clear_fpstate(p1);
 		p2->p_md.md_fpstate = malloc(sizeof(struct fpstate64),
 		    M_SUBPROC, M_WAITOK);
 		bcopy(p1->p_md.md_fpstate, p2->p_md.md_fpstate,
@@ -308,6 +305,45 @@ cpu_fork(p1, p2, stack, stacksize, func, arg)
 }
 
 /*
+ * These are the "function" entry points in locore.s to handle IPI's.
+ */
+void	ipi_save_fpstate(void);
+void	ipi_drop_fpstate(void);
+
+void
+save_and_clear_fpstate(struct proc *p)
+{
+#ifdef MULTIPROCESSOR
+	struct cpu_info *ci;
+#endif
+
+	if (p == fpproc) {
+		savefpstate(p->p_md.md_fpstate);
+		fpproc = NULL;
+		return;
+	}
+#ifdef MULTIPROCESSOR
+	for (ci = cpus; ci != NULL; ci = ci->ci_next) {
+		int spincount = 0;
+
+		if (ci == curcpu())
+			continue;
+		if (ci->ci_fpproc != p)
+			continue;
+		sparc64_send_ipi(ci->ci_upaid, ipi_save_fpstate, 0, 0);
+		while(ci->ci_fpproc == p) {
+			spincount++;
+			if (spincount > 10000000) {
+				panic("ipi_save_fpstate didn't");
+			}
+			sparc_membar(Sync);
+		}
+		break;
+	}
+#endif
+}
+
+/*
  * cpu_exit is called as the last action during exit.
  *
  * We clean up a little and then call sched_exit() with the old proc
@@ -318,13 +354,35 @@ void
 cpu_exit(struct proc *p)
 {
 	register struct fpstate64 *fs;
+#ifdef MULTIPROCESSOR
+	struct cpu_info *ci;
+	int found = 0;
+#endif
 
 	if ((fs = p->p_md.md_fpstate) != NULL) {
 		if (p == fpproc) {
-			savefpstate(fs);
+			clearfpstate();
 			fpproc = NULL;
+#ifdef MULTIPROCESSOR
+			found = 1;
+#endif
+		}
+#ifdef MULTIPROCESSOR
+		if (!found) {
+			/* check if anyone else has this proc as fpproc */
+			for (ci = cpus; ci != NULL; ci = ci->ci_next) {
+				if (ci == curcpu())
+					continue;
+				if (ci->ci_fpproc != p)
+					continue;
+				sparc64_send_ipi(ci->ci_upaid,
+				    ipi_drop_fpstate, 0, 0);
+				break;
+			}
 		}
-		free((void *)fs, M_SUBPROC);
+#endif
+
+		free(fs, M_SUBPROC);
 	}
 
 	pmap_deactivate(p);
@@ -354,10 +412,7 @@ cpu_coredump(p, vp, cred, chdr)
 	md_core.md_tf = *p->p_md.md_tf;
 	md_core.md_wcookie = p->p_addr->u_pcb.pcb_wcookie;
 	if (p->p_md.md_fpstate) {
-		if (p == fpproc) {
-			savefpstate(p->p_md.md_fpstate);
-			fpproc = NULL;
-		}
+		save_and_clear_fpstate(p);
 		md_core.md_fpstate = *p->p_md.md_fpstate;
 	} else
 		bzero((caddr_t)&md_core.md_fpstate,