Overhaul fpu context save/restore code, making it more similar to what we do

on i386 and amd64.  Don't let IPIs in when saving fpu context by disabling
interrupts.

6 files changed, 65 insertions, 76 deletions

diff --git a/sys/arch/sparc64/include/cpu.h b/sys/arch/sparc64/include/cpu.h
index 5b3d461fe5c..07f20f77a86 100644
--- a/sys/arch/sparc64/include/cpu.h
+++ b/sys/arch/sparc64/include/cpu.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: cpu.h,v 1.51 2007/10/21 21:00:38 kettenis Exp $	*/
+/*	$OpenBSD: cpu.h,v 1.52 2007/10/31 22:46:52 kettenis Exp $	*/
 /*	$NetBSD: cpu.h,v 1.28 2001/06/14 22:56:58 thorpej Exp $ */
 
 /*
@@ -248,7 +248,8 @@ void	switchtoctx(int);
 void	pmap_unuse_final(struct proc *);
 int	rwindow_save(struct proc *);
 /* vm_machdep.c */
-void	save_and_clear_fpstate(struct proc *);
+void	fpusave_cpu(struct cpu_info *, int);
+void	fpusave_proc(struct proc *, int);
 /* amd7930intr.s */
 void	amd7930_trap(void);
 /* cons.c */
diff --git a/sys/arch/sparc64/sparc64/emul.c b/sys/arch/sparc64/sparc64/emul.c
index 4b31f6050fe..23b56b39059 100644
--- a/sys/arch/sparc64/sparc64/emul.c
+++ b/sys/arch/sparc64/sparc64/emul.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: emul.c,v 1.13 2007/09/09 10:17:33 kettenis Exp $	*/
+/*	$OpenBSD: emul.c,v 1.14 2007/10/31 22:46:52 kettenis Exp $	*/
 /*	$NetBSD: emul.c,v 1.8 2001/06/29 23:58:40 eeh Exp $	*/
 
 /*-
@@ -323,10 +323,7 @@ fixalign(p, tf)
 
 	if (op.bits.st) {
 		if (op.bits.fl) {
-			if (p == fpproc) {
-				savefpstate(p->p_md.md_fpstate);
-				fpproc = NULL;
-			}
+			fpusave_proc(p, 1);
 
 			error = readfpreg(p, code.i_op3.i_rd, &data.i[0]);
 			if (error)
@@ -533,19 +530,15 @@ emul_qf(int32_t insv, struct proc *p, union sigval sv, struct trapframe *tf)
 
 	fs = p->p_md.md_fpstate;
 	if (fs == NULL) {
+		KERNEL_PROC_LOCK(p);
 		/* don't currently have an fpu context, get one */
 		fs = malloc(sizeof(*fs), M_SUBPROC, M_WAITOK);
 		*fs = initfpstate;
 		fs->fs_qsize = 0;
 		p->p_md.md_fpstate = fs;
-	}
-	if (fpproc != p) {
-		/* make this process the current holder of the fpu */
-		if (fpproc != NULL)
-			savefpstate(fpproc->p_md.md_fpstate);
-		fpproc = p;
-	}
-	tf->tf_tstate |= TSTATE_PEF;
+		KERNEL_PROC_UNLOCK(p);
+	} else
+		fpusave_proc(p, 1);
 
 	/* Ok, try to do the actual operation (finally) */
 	if (isload) {
@@ -553,11 +546,9 @@ emul_qf(int32_t insv, struct proc *p, union sigval sv, struct trapframe *tf)
 		if (err != 0 && (asi & 2) == 0)
 			goto segv;
 		if (err == 0) {
-			savefpstate(fs);
 			if (asi & 8)
 				swap_quad(buf);
 			bcopy(buf, &fs->fs_regs[freg], sizeof(buf));
-			loadfpstate(fs);
 		}
 	} else {
 		bcopy(&fs->fs_regs[freg], buf, sizeof(buf));
diff --git a/sys/arch/sparc64/sparc64/machdep.c b/sys/arch/sparc64/sparc64/machdep.c
index 0e168761167..0f57eb17dd2 100644
--- a/sys/arch/sparc64/sparc64/machdep.c
+++ b/sys/arch/sparc64/sparc64/machdep.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: machdep.c,v 1.100 2007/10/20 16:54:52 miod Exp $	*/
+/*	$OpenBSD: machdep.c,v 1.101 2007/10/31 22:46:52 kettenis Exp $	*/
 /*	$NetBSD: machdep.c,v 1.108 2001/07/24 19:30:14 eeh Exp $ */
 
 /*-
@@ -322,7 +322,6 @@ setregs(p, pack, stack, retval)
 	register_t *retval;
 {
 	struct trapframe64 *tf = p->p_md.md_tf;
-	struct fpstate64 *fs;
 	int64_t tstate;
 	int pstate = PSTATE_USER;
 	Elf_Ehdr *eh = pack->ep_hdr;
@@ -379,14 +378,14 @@ setregs(p, pack, stack, retval)
 	tstate = (ASI_PRIMARY_NO_FAULT<<TSTATE_ASI_SHIFT) |
 		((pstate)<<TSTATE_PSTATE_SHIFT) | 
 		(tf->tf_tstate & TSTATE_CWP);
-	if ((fs = p->p_md.md_fpstate) != NULL) {
+	if (p->p_md.md_fpstate != NULL) {
 		/*
 		 * We hold an FPU state.  If we own *the* FPU chip state
 		 * we must get rid of it, and the only way to do that is
 		 * to save it.  In any case, get rid of our FPU state.
 		 */
-		save_and_clear_fpstate(p);
-		free((void *)fs, M_SUBPROC);
+		fpusave_proc(p, 0);
+		free(p->p_md.md_fpstate, M_SUBPROC);
 		p->p_md.md_fpstate = NULL;
 	}
 	bzero((caddr_t)tf, sizeof *tf);
diff --git a/sys/arch/sparc64/sparc64/process_machdep.c b/sys/arch/sparc64/sparc64/process_machdep.c
index 67fe4037810..d57c0887ebd 100644
--- a/sys/arch/sparc64/sparc64/process_machdep.c
+++ b/sys/arch/sparc64/sparc64/process_machdep.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: process_machdep.c,v 1.10 2005/12/31 23:50:39 kettenis Exp $	*/
+/*	$OpenBSD: process_machdep.c,v 1.11 2007/10/31 22:46:52 kettenis Exp $	*/
 /*	$NetBSD: process_machdep.c,v 1.10 2000/09/26 22:05:50 eeh Exp $ */
 
 /*
@@ -124,8 +124,7 @@ process_read_fpregs(p, regs)
 
 	/* NOTE: struct fpreg == struct fpstate */
 	if (p->p_md.md_fpstate) {
-		if (p == fpproc)
-			savefpstate(p->p_md.md_fpstate);
+		fpusave_proc(p, 1);
 		statep = p->p_md.md_fpstate;
 	}
 
@@ -214,13 +213,8 @@ process_write_fpregs(p, regs)
 	if (p->p_md.md_fpstate == NULL) {
 		p->p_md.md_fpstate = malloc(sizeof(struct fpstate64),
 		    M_SUBPROC, M_WAITOK);
-	}
-
-	if (p == fpproc) {
-		/* Release the fpu. */
-		savefpstate(p->p_md.md_fpstate);
-		fpproc = NULL;
-	}
+	} else
+		fpusave_proc(p, 1);
 
 	if (!(curproc->p_flag & P_32)) {
 		/* 64-bit mode -- copy in fregs */
diff --git a/sys/arch/sparc64/sparc64/trap.c b/sys/arch/sparc64/sparc64/trap.c
index a9821cca035..20378b731f1 100644
--- a/sys/arch/sparc64/sparc64/trap.c
+++ b/sys/arch/sparc64/sparc64/trap.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: trap.c,v 1.56 2007/10/16 19:22:49 kettenis Exp $	*/
+/*	$OpenBSD: trap.c,v 1.57 2007/10/31 22:46:52 kettenis Exp $	*/
 /*	$NetBSD: trap.c,v 1.73 2001/08/09 01:03:01 eeh Exp $ */
 
 /*
@@ -381,6 +381,7 @@ trap(tf, type, pc, tstate)
 	struct proc *p;
 	struct pcb *pcb;
 	int pstate = (tstate>>TSTATE_PSTATE_SHIFT);
+	u_int64_t s;
 	int64_t n;
 	union sigval sv;
 
@@ -428,14 +429,17 @@ trap(tf, type, pc, tstate)
 				newfpproc = curproc;
 				/* force other cpus to give up this fpstate */
 				if (newfpproc->p_md.md_fpstate)
-					save_and_clear_fpstate(newfpproc);
+					fpusave_proc(newfpproc, 1);
 			}
 			if (fpproc != newfpproc) {
+				s = intr_disable();
 				if (fpproc != NULL) {
 					/* someone else had it, maybe? */
 					savefpstate(fpproc->p_md.md_fpstate);
 					fpproc = NULL;
 				}
+				intr_restore(s);
+
 				/* If we have an allocated fpstate, load it */
 				if (newfpproc->p_md.md_fpstate != 0) {
 					fpproc = newfpproc;
@@ -575,11 +579,13 @@ badtrap:
 		}
 		if (fpproc != p) {		/* we do not have it */
 			/* but maybe another CPU has it? */
-			save_and_clear_fpstate(p);
+			fpusave_proc(p, 1);
+			s = intr_disable();
 			if (fpproc != NULL)	/* someone else had it */
 				savefpstate(fpproc->p_md.md_fpstate);
 			loadfpstate(fs);
 			fpproc = p;		/* now we do have it */
+			intr_restore(s);
 			uvmexp.fpswtch++;
 		}
 		tf->tf_tstate |= (PSTATE_PEF<<TSTATE_PSTATE_SHIFT);
@@ -671,8 +677,10 @@ badtrap:
 		 */
 		if (p != fpproc)
 			panic("fpe without being the FP user");
+		s = intr_disable();
 		savefpstate(p->p_md.md_fpstate);
 		fpproc = NULL;
+		intr_restore(s);
 		/* tf->tf_psr &= ~PSR_EF; */	/* share_fpu will do this */
 		if (type == T_FP_OTHER && p->p_md.md_fpstate->fs_qsize == 0) {
 			/*
diff --git a/sys/arch/sparc64/sparc64/vm_machdep.c b/sys/arch/sparc64/sparc64/vm_machdep.c
index acd7a169909..560f13bdba3 100644
--- a/sys/arch/sparc64/sparc64/vm_machdep.c
+++ b/sys/arch/sparc64/sparc64/vm_machdep.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: vm_machdep.c,v 1.19 2007/10/17 19:41:36 kettenis Exp $	*/
+/*	$OpenBSD: vm_machdep.c,v 1.20 2007/10/31 22:46:52 kettenis Exp $	*/
 /*	$NetBSD: vm_machdep.c,v 1.38 2001/06/30 00:02:20 eeh Exp $ */
 
 /*
@@ -228,7 +228,7 @@ cpu_fork(p1, p2, stack, stacksize, func, arg)
 #endif
 	bcopy((caddr_t)opcb, (caddr_t)npcb, sizeof(struct pcb));
 	if (p1->p_md.md_fpstate) {
-		save_and_clear_fpstate(p1);
+		fpusave_proc(p1, 1);
 		p2->p_md.md_fpstate = malloc(sizeof(struct fpstate64),
 		    M_SUBPROC, M_WAITOK);
 		bcopy(p1->p_md.md_fpstate, p2->p_md.md_fpstate,
@@ -309,18 +309,37 @@ void	ipi_save_fpstate(void);
 void	ipi_drop_fpstate(void);
 
 void
-save_and_clear_fpstate(struct proc *p)
+fpusave_cpu(struct cpu_info *ci, int save)
 {
-#ifdef MULTIPROCESSOR
-	struct cpu_info *ci;
-#endif
+	struct proc *p;
+
+	KDASSERT(ci == curcpu());
 
-	if (p == fpproc) {
+	p = ci->ci_fpproc;
+	if (p == NULL)
+		return;
+
+	if (save)
 		savefpstate(p->p_md.md_fpstate);
-		fpproc = NULL;
+	else
+		clearfpstate();
+
+	ci->ci_fpproc = NULL;
+}
+
+void
+fpusave_proc(struct proc *p, int save)
+{
+	struct cpu_info *ci = curcpu();
+
+#ifdef MULTIPROCESSOR
+	if (p == ci->ci_fpproc) {
+		u_int64_t s = intr_disable();
+		fpusave_cpu(ci, save);
+		intr_restore(s);
 		return;
 	}
-#ifdef MULTIPROCESSOR
+
 	for (ci = cpus; ci != NULL; ci = ci->ci_next) {
 		int spincount = 0;
 
@@ -328,7 +347,8 @@ save_and_clear_fpstate(struct proc *p)
 			continue;
 		if (ci->ci_fpproc != p)
 			continue;
-		sparc64_send_ipi(ci->ci_upaid, ipi_save_fpstate, 0, 0);
+		sparc64_send_ipi(ci->ci_upaid,
+		    save ? ipi_save_fpstate : ipi_drop_fpstate, 0, 0);
 		while(ci->ci_fpproc == p) {
 			spincount++;
 			if (spincount > 10000000) {
@@ -338,6 +358,9 @@ save_and_clear_fpstate(struct proc *p)
 		}
 		break;
 	}
+#else
+	if (p == ci->ci_fpproc)
+		fpusave_cpu(ci, save);
 #endif
 }
 
@@ -351,36 +374,9 @@ save_and_clear_fpstate(struct proc *p)
 void
 cpu_exit(struct proc *p)
 {
-	register struct fpstate64 *fs;
-#ifdef MULTIPROCESSOR
-	struct cpu_info *ci;
-	int found = 0;
-#endif
-
-	if ((fs = p->p_md.md_fpstate) != NULL) {
-		if (p == fpproc) {
-			clearfpstate();
-			fpproc = NULL;
-#ifdef MULTIPROCESSOR
-			found = 1;
-#endif
-		}
-#ifdef MULTIPROCESSOR
-		if (!found) {
-			/* check if anyone else has this proc as fpproc */
-			for (ci = cpus; ci != NULL; ci = ci->ci_next) {
-				if (ci == curcpu())
-					continue;
-				if (ci->ci_fpproc != p)
-					continue;
-				sparc64_send_ipi(ci->ci_upaid,
-				    ipi_drop_fpstate, 0, 0);
-				break;
-			}
-		}
-#endif
-
-		free(fs, M_SUBPROC);
+	if (p->p_md.md_fpstate != NULL) {
+		fpusave_proc(p, 0);
+		free(p->p_md.md_fpstate, M_SUBPROC);
 	}
 
 	pmap_deactivate(p);
@@ -410,7 +406,7 @@ cpu_coredump(p, vp, cred, chdr)
 	md_core.md_tf = *p->p_md.md_tf;
 	md_core.md_wcookie = p->p_addr->u_pcb.pcb_wcookie;
 	if (p->p_md.md_fpstate) {
-		save_and_clear_fpstate(p);
+		fpusave_proc(p, 1);
 		md_core.md_fpstate = *p->p_md.md_fpstate;
 	} else
 		bzero((caddr_t)&md_core.md_fpstate,