Move the FPU state out of 'struct pcb' like we did for hppa since it causes

evil problems with non-equivalent aliases.

10 files changed, 245 insertions, 111 deletions

diff --git a/sys/arch/hppa64/hppa64/fpu.c b/sys/arch/hppa64/hppa64/fpu.c
new file mode 100644
index 00000000000..0cae9abd694
--- /dev/null
+++ b/sys/arch/hppa64/hppa64/fpu.c
@@ -0,0 +1,126 @@
+/*	$OpenBSD: fpu.c,v 1.1 2011/04/16 22:02:32 kettenis Exp $	*/
+
+/*
+ * Copyright (c) 2010 Joel Sing <jsing@openbsd.org>
+ *
+ * Permission to use, copy, modify, and distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/proc.h>
+#include <sys/user.h>
+
+#include <machine/cpu.h>
+#include <machine/cpufunc.h>
+#include <machine/fpu.h>
+#include <machine/intr.h>
+#include <machine/pcb.h>
+#include <machine/reg.h>
+
+__inline void fpu_proc(struct proc *, int);
+
+void
+fpu_proc_flush(struct proc *p)
+{
+	fpu_proc(p, 0);
+}
+
+void
+fpu_proc_save(struct proc *p)
+{
+	fpu_proc(p, 1);
+}
+
+__inline void
+fpu_proc(struct proc *p, int save)
+{
+	struct cpu_info *ci = curcpu();
+	struct hppa_fpstate *hfp;
+	struct cpu_info *fpuci;
+#ifdef MULTIPROCESSOR
+	int s;
+#endif
+
+	hfp = (struct hppa_fpstate *)p->p_md.md_regs->tf_cr30;
+	fpuci = (struct cpu_info *)hfp->hfp_cpu;
+
+	if (fpuci == NULL)
+		return;
+
+#ifdef MULTIPROCESSOR
+	if (fpuci != ci) {
+
+		if (hppa_ipi_send(fpuci, HPPA_IPI_FPU_SAVE))
+			panic("FPU shootdown failed!");
+
+		/*
+		 * The sync is essential here since the volatile on hfp_cpu
+		 * is ignored by gcc. Without this we will deadlock since
+		 * hfp_cpu is never reloaded within the loop.
+		 */
+		while (hfp->hfp_cpu != NULL)
+			asm volatile ("sync" ::: "memory");
+
+	} else if (p->p_md.md_regs->tf_cr30 == ci->ci_fpu_state) {
+
+		s = splipi();
+		fpu_cpu_save(save);
+		splx(s);
+
+	}
+#else
+	if (p->p_md.md_regs->tf_cr30 == ci->ci_fpu_state)
+		fpu_cpu_save(save);
+#endif
+}
+
+/*
+ * Save or flush FPU state - note that this must be called at IPL IPI when
+ * running on a MULTIPROCESSOR kernel.
+ */
+void
+fpu_cpu_save(int save)
+{
+	struct cpu_info *ci = curcpu();
+	struct hppa_fpstate *hfp;
+	struct cpu_info *fpuci;
+	extern u_int fpu_enable;
+
+#ifdef MULTIPROCESSOR
+	splassert(IPL_IPI);
+#endif
+
+	if (ci->ci_fpu_state == 0)
+		return;
+
+	hfp = (struct hppa_fpstate *)ci->ci_fpu_state;
+	fpuci = (struct cpu_info *)hfp->hfp_cpu;
+
+#ifdef DIAGNOSTIC
+	if (fpuci != ci)
+		panic("FPU context is not on this CPU (%p != %p)",
+		    ci, hfp->hfp_cpu);
+#endif
+
+	if (save) {
+		mtctl(fpu_enable, CR_CCR);
+		fpu_save((paddr_t)&hfp->hfp_regs);
+		mtctl(0, CR_CCR);
+	} else
+		fpu_exit();
+
+	hfp->hfp_cpu = NULL;
+	ci->ci_fpu_state = 0;
+	asm volatile ("sync" ::: "memory");
+}
diff --git a/sys/arch/hppa64/hppa64/genassym.cf b/sys/arch/hppa64/hppa64/genassym.cf
index 322c1f1c2f6..851dcff2407 100644
--- a/sys/arch/hppa64/hppa64/genassym.cf
+++ b/sys/arch/hppa64/hppa64/genassym.cf
@@ -1,4 +1,4 @@
-#	$OpenBSD: genassym.cf,v 1.9 2011/04/14 19:34:55 kettenis Exp $
+#	$OpenBSD: genassym.cf,v 1.10 2011/04/16 22:02:32 kettenis Exp $
 
 #
 # Copyright (c) 1982, 1990, 1993
@@ -121,16 +121,21 @@ struct	user
 member	u_pcb
 
 struct	pcb
-member	pcb_fpregs
+member	pcb_fpstate
 member	pcb_onfault
 member	pcb_ksp
 member	pcb_space
 
 struct	cpu_info
-member	ci_trap_save
+member	ci_curproc
 member	ci_psw
 member	ci_cpl
-member	ci_curproc
+member	ci_trap_save
+member	ci_fpu_state
+
+struct	hppa_fpstate
+member	hfp_regs
+member	hfp_cpu
 
 struct	uvm
 member	page_idle_zero
diff --git a/sys/arch/hppa64/hppa64/locore.S b/sys/arch/hppa64/hppa64/locore.S
index 0d5637c7adf..4ee88b7e45e 100644
--- a/sys/arch/hppa64/hppa64/locore.S
+++ b/sys/arch/hppa64/hppa64/locore.S
@@ -1,4 +1,4 @@
-/*	$OpenBSD: locore.S,v 1.23 2011/04/14 19:37:07 kettenis Exp $	*/
+/*	$OpenBSD: locore.S,v 1.24 2011/04/16 22:02:32 kettenis Exp $	*/
 
 /*
  * Copyright (c) 2005 Michael Shalayeff
@@ -38,6 +38,7 @@
 	.import esym, data
 	.import proc0, data
 	.import proc0paddr, data
+	.import proc0fpstate, data
 
 #define	EMRG_STACKSIZE	(1*PAGE_SIZE)
 #define	FPEMU_STACKSIZE	(1*PAGE_SIZE)
@@ -201,7 +202,6 @@ $start_zero_tf
 	std	%arg3, R%proc0paddr(%r1)
 
 	ldo	PAGE_SIZE(%arg3), %sp
-	mtctl	%arg3, %cr30
 	std	%r0, U_PCB+PCB_ONFAULT(%arg3)
 	std	%r0, U_PCB+PCB_SPACE(%arg3)
 
@@ -213,7 +213,10 @@ $start_zero_tf
 
 	ldil	TFF_LAST, %r1
 	std	%r1, TF_FLAGS(%arg2)
-	std	%arg3, TF_CR30(%arg2)
+	ldil	L%proc0fpstate, %r1
+	ldo	R%proc0fpstate(%r1), %r1
+	std	%r1, TF_CR30(%arg2)
+	mtctl	%r1, %cr30
 
 	mfctl	%cr24, %r1
 	ldw	CI_PSW(%r1), arg7
@@ -919,7 +922,7 @@ TLABEL(all_virt)
 
 	mfctl	%cr30, %arg2
 	mfctl	%cr27, %arg3		/* XXX */
-	std,ma	%arg2, 8(%sr3,%sp)	/* pa(u) */
+	std,ma	%arg2, 8(%sr3,%sp)
 	std,ma	%arg3, 8(%sr3,%sp)	/* user curthread */
 
 	mfctl	%sar, %arg2
@@ -1044,8 +1047,8 @@ ENTRY(TLABEL(excpt),0)
 	copy	%rp, %r1
 	copy	%arg0, %r8
 	mfctl	%cr30, %r9
-#if (PCB_FPREGS+U_PCB) != 0
-	ldo	PCB_FPREGS+U_PCB(%r9), %r9
+#if FP_REGS != 0
+	ldo	HFP_REGS(%r9), %r9
 #endif
 	.import	fpu_save, code
 	.call
@@ -1151,17 +1154,17 @@ $fpusw_set
 	depdi	3, 37, 2, %r1
 	mtctl	%r1, %cr10	/* ccr */
 
-	ldil	L%fpu_curpcb, %r16
+	mfctl	%cr24, %r16
 	mfctl	%cr30, %r9
-	ldd	R%fpu_curpcb(%r16), %r16
+	ldd	CI_FPU_STATE(%r16), %r16
 
 	comb,=,n %r16, %r0, $fpusw_nosave
 	comb,=,n %r16, %r9, $fpusw_done
 
 	copy	%arg0, %r17
 	copy	%rp, %r1
-#if (PCB_FPREGS+U_PCB) != 0
-	ldo	PCB_FPREGS+U_PCB(%r16), %r16
+#if HFP_REGS != 0
+	ldo	HFP_REGS(%r16), %r16
 #endif
 	.import	fpu_save, code
 	.call
@@ -1175,7 +1178,7 @@ $fpusw_nosave
 	.import	umvexp, data
 	ldil	L%(uvmexp+FPSWTCH), %r1
 	ldw	R%(uvmexp+FPSWTCH)(%r1), %r16
-	ldo	31*8+PCB_FPREGS+U_PCB(%r9), %r17
+	ldo	31*8+HFP_REGS(%r9), %r17
 	ldo	1(%r16), %r16
 	stw	%r16, R%(uvmexp+FPSWTCH)(%r1)
 
@@ -1212,8 +1215,10 @@ $fpusw_nosave
 	fldds,ma -8(%r17), %fr1
 	fldds     0(%r17), %fr0	/* fr0 must be restored last */
 
-	ldil	L%fpu_curpcb, %r1
-	std	%r17, R%fpu_curpcb(%r1)
+	mfctl	%cr24, %r1
+	std	%r9, CI_FPU_STATE(%r1)
+	std	%r1, HFP_CPU(%r9)
+	sync
 
 $fpusw_done
 	rfir
diff --git a/sys/arch/hppa64/hppa64/machdep.c b/sys/arch/hppa64/hppa64/machdep.c
index 86928fd834c..bb74672a866 100644
--- a/sys/arch/hppa64/hppa64/machdep.c
+++ b/sys/arch/hppa64/hppa64/machdep.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: machdep.c,v 1.34 2011/04/15 04:52:39 guenther Exp $	*/
+/*	$OpenBSD: machdep.c,v 1.35 2011/04/16 22:02:32 kettenis Exp $	*/
 
 /*
  * Copyright (c) 2005 Michael Shalayeff
@@ -57,6 +57,7 @@
 #include <machine/reg.h>
 #include <machine/autoconf.h>
 #include <machine/kcore.h>
+#include <machine/fpu.h>
 
 #ifdef DDB
 #include <machine/db_machdep.h>
@@ -128,6 +129,8 @@ int	physmem, resvmem, resvphysmem, esym;
 struct user *proc0paddr;
 long mem_ex_storage[EXTENT_FIXED_STORAGE_SIZE(32) / sizeof(long)];
 struct extent *hppa_ex;
+struct pool hppa_fppl;
+struct hppa_fpstate proc0fpstate;
 struct consdev *cn_tab;
 
 struct vm_map *exec_map = NULL;
@@ -295,7 +298,10 @@ TODO hpmc/toc/pfr
 	ptlball();
 	ficacheall();
 	fdcacheall();
-printf("out\n");
+
+	proc0paddr->u_pcb.pcb_fpstate = &proc0fpstate;
+	pool_init(&hppa_fppl, sizeof(struct hppa_fpstate), 16, 0, 0,
+	    "hppafp", NULL);
 }
 
 void
@@ -791,7 +797,6 @@ void
 setregs(struct proc *p, struct exec_package *pack, u_long stack,
     register_t *retval)
 {
-	extern paddr_t fpu_curpcb;	/* from locore.S */
 	struct trapframe *tf = p->p_md.md_regs;
 	struct pcb *pcb = &p->p_addr->u_pcb;
 	register_t zero;
@@ -813,15 +818,12 @@ setregs(struct proc *p, struct exec_package *pack, u_long stack,
 	copyout(&zero, (caddr_t)(stack + HPPA_FRAME_RP), sizeof(register_t));
 
 	/* reset any of the pending FPU exceptions */
-	if (tf->tf_cr30 == fpu_curpcb) {
-		fpu_exit();
-		fpu_curpcb = 0;
-	}
-	pcb->pcb_fpregs[0] = ((u_int64_t)HPPA_FPU_INIT) << 32;
-	pcb->pcb_fpregs[1] = 0;
-	pcb->pcb_fpregs[2] = 0;
-	pcb->pcb_fpregs[3] = 0;
-	fdcache(HPPA_SID_KERNEL, (vaddr_t)pcb->pcb_fpregs, 8 * 4);
+	fpu_proc_flush(p);
+	pcb->pcb_fpstate->hfp_regs.fpr_regs[0] =
+	    ((u_int64_t)HPPA_FPU_INIT) << 32;
+	pcb->pcb_fpstate->hfp_regs.fpr_regs[1] = 0;
+	pcb->pcb_fpstate->hfp_regs.fpr_regs[2] = 0;
+	pcb->pcb_fpstate->hfp_regs.fpr_regs[3] = 0;
 
 	retval[1] = 0;
 }
@@ -833,8 +835,6 @@ void
 sendsig(sig_t catcher, int sig, int mask, u_long code, int type,
     union sigval val)
 {
-	extern paddr_t fpu_curpcb;	/* from locore.S */
-	extern u_int fpu_enable;
 	struct proc *p = curproc;
 	struct trapframe *tf = p->p_md.md_regs;
 	struct sigacts *psp = p->p_sigacts;
@@ -851,13 +851,8 @@ sendsig(sig_t catcher, int sig, int mask, u_long code, int type,
 		    p->p_comm, p->p_pid, sig, catcher);
 #endif
 
-	/* flush the FPU ctx first */
-	if (tf->tf_cr30 == fpu_curpcb) {
-		mtctl(fpu_enable, CR_CCR);
-		fpu_save(fpu_curpcb);
-		/* fpu_curpcb = 0; only needed if fpregs are preset */
-		mtctl(0, CR_CCR);
-	}
+	/* Save the FPU context first. */
+	fpu_proc_save(p);
 
 	ksc.sc_onstack = p->p_sigstk.ss_flags & SS_ONSTACK;
 
@@ -891,7 +886,7 @@ sendsig(sig_t catcher, int sig, int mask, u_long code, int type,
 	ksc.sc_pcoqt = tf->tf_iioq[1];
 	bcopy(tf, &ksc.sc_regs[0], 32*8);
 	ksc.sc_regs[0] = tf->tf_sar;
-	bcopy(p->p_addr->u_pcb.pcb_fpregs, ksc.sc_fpregs,
+	bcopy(&p->p_addr->u_pcb.pcb_fpstate->hfp_regs, ksc.sc_fpregs,
 	    sizeof(ksc.sc_fpregs));
 
 	sss += HPPA_FRAME_SIZE;
@@ -935,7 +930,6 @@ sendsig(sig_t catcher, int sig, int mask, u_long code, int type,
 int
 sys_sigreturn(struct proc *p, void *v, register_t *retval)
 {
-	extern paddr_t fpu_curpcb;	/* from locore.S */
 	struct sys_sigreturn_args /* {
 		syscallarg(struct sigcontext *) sigcntxp;
 	} */ *uap = v;
@@ -952,10 +946,7 @@ sys_sigreturn(struct proc *p, void *v, register_t *retval)
 #endif
 
 	/* flush the FPU ctx first */
-	if (tf->tf_cr30 == fpu_curpcb) {
-		fpu_exit();
-		fpu_curpcb = 0;
-	}
+	fpu_proc_flush(p);
 
 	if ((error = copyin((caddr_t)scp, (caddr_t)&ksc, sizeof ksc)))
 		return (error);
@@ -974,9 +965,7 @@ sys_sigreturn(struct proc *p, void *v, register_t *retval)
 	tf->tf_sar = ksc.sc_regs[0];
 	ksc.sc_regs[0] = tf->tf_flags;
 	bcopy(&ksc.sc_regs[0], tf, 32*8);
-	bcopy(ksc.sc_fpregs, p->p_addr->u_pcb.pcb_fpregs,
-	    sizeof(ksc.sc_fpregs));
-	fdcache(HPPA_SID_KERNEL, (vaddr_t)p->p_addr->u_pcb.pcb_fpregs,
+	bcopy(ksc.sc_fpregs, &p->p_addr->u_pcb.pcb_fpstate->hfp_regs,
 	    sizeof(ksc.sc_fpregs));
 
 	tf->tf_iioq[0] = ksc.sc_pcoqh;
diff --git a/sys/arch/hppa64/hppa64/process_machdep.c b/sys/arch/hppa64/hppa64/process_machdep.c
index 564af417d85..e285c308d4b 100644
--- a/sys/arch/hppa64/hppa64/process_machdep.c
+++ b/sys/arch/hppa64/hppa64/process_machdep.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: process_machdep.c,v 1.1 2005/04/01 10:40:47 mickey Exp $	*/
+/*	$OpenBSD: process_machdep.c,v 1.2 2011/04/16 22:02:32 kettenis Exp $	*/
 
 /*
  * Copyright (c) 2005 Michael Shalayeff
@@ -40,19 +40,11 @@ process_read_regs(p, regs)
 }
 
 int
-process_read_fpregs(p, fpregs)
-	struct proc *p;
-	struct fpreg *fpregs;
+process_read_fpregs(struct proc *p, struct fpreg *fpregs)
 {
-	extern paddr_t fpu_curpcb;
-	extern u_int fpu_enable;
+	fpu_proc_save(p);
 
-	if (p->p_md.md_regs->tf_cr30 == fpu_curpcb) {
-		mtctl(fpu_enable, CR_CCR);
-		fpu_save((vaddr_t)p->p_addr->u_pcb.pcb_fpregs);
-		mtctl(0, CR_CCR);
-	}
-	bcopy(p->p_addr->u_pcb.pcb_fpregs, fpregs, 32*8);
+	bcopy(&p->p_addr->u_pcb.pcb_fpstate->hfp_regs, fpregs, 32 * 8);
 
 	return (0);
 }
@@ -75,18 +67,11 @@ process_write_regs(p, regs)
 }
 
 int
-process_write_fpregs(p, fpregs)
-	struct proc *p;
-	struct fpreg *fpregs;
+process_write_fpregs(struct proc *p, struct fpreg *fpregs)
 {
-	extern paddr_t fpu_curpcb;
-
-	if (p->p_md.md_regs->tf_cr30 == fpu_curpcb) {
-		fpu_exit();
-		fpu_curpcb = 0;
-	}
+	fpu_proc_flush(p);
 
-	bcopy(fpregs, p->p_addr->u_pcb.pcb_fpregs, 32 * 8);
+	bcopy(fpregs, &p->p_addr->u_pcb.pcb_fpstate->hfp_regs, 32 * 8);
 
 	return (0);
 }
diff --git a/sys/arch/hppa64/hppa64/trap.c b/sys/arch/hppa64/hppa64/trap.c
index d04013fcbfb..262f942f9db 100644
--- a/sys/arch/hppa64/hppa64/trap.c
+++ b/sys/arch/hppa64/hppa64/trap.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: trap.c,v 1.18 2011/04/07 13:13:01 jsing Exp $	*/
+/*	$OpenBSD: trap.c,v 1.19 2011/04/16 22:02:32 kettenis Exp $	*/
 
 /*
  * Copyright (c) 2005 Michael Shalayeff
@@ -283,10 +283,14 @@ trap(int type, struct trapframe *frame)
 		break;
 
 	case T_EXCEPTION | T_USER: {
-		u_int64_t *fpp = (u_int64_t *)frame->tf_cr30;
+		struct hppa_fpstate *hfp;
+		u_int64_t *fpp;
 		u_int32_t *pex;
 		int i, flt;
 
+		hfp = (struct hppa_fpstate *)frame->tf_cr30;
+		fpp = (u_int64_t *)&hfp->hfp_regs;
+
 		pex = (u_int32_t *)&fpp[0];
 		for (i = 0, pex++; i < 7 && !*pex; i++, pex++);
 		flt = 0;
diff --git a/sys/arch/hppa64/hppa64/vm_machdep.c b/sys/arch/hppa64/hppa64/vm_machdep.c
index d1c75369ab3..75cc29443eb 100644
--- a/sys/arch/hppa64/hppa64/vm_machdep.c
+++ b/sys/arch/hppa64/hppa64/vm_machdep.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: vm_machdep.c,v 1.11 2011/04/14 19:34:55 kettenis Exp $	*/
+/*	$OpenBSD: vm_machdep.c,v 1.12 2011/04/16 22:02:32 kettenis Exp $	*/
 
 /*
  * Copyright (c) 2005 Michael Shalayeff
@@ -29,6 +29,7 @@
 #include <sys/ptrace.h>
 #include <sys/exec.h>
 #include <sys/core.h>
+#include <sys/pool.h>
 
 #include <machine/psl.h>
 #include <machine/pmap.h>
@@ -36,6 +37,7 @@
 
 #include <uvm/uvm.h>
 
+extern struct pool hppa_fppl;
 
 /*
  * Dump the machine specific header information at the start of a core dump.
@@ -89,48 +91,37 @@ cpu_fork(p1, p2, stack, stacksize, func, arg)
 	void (*func)(void *);
 	void *arg;
 {
-	extern paddr_t fpu_curpcb;	/* from locore.S */
 	extern register_t switch_tramp_p;
-	extern u_int fpu_enable;
 
 	struct pcb *pcbp;
 	struct trapframe *tf;
 	register_t sp, osp;
-	paddr_t pa;
 
 #ifdef DIAGNOSTIC
 	if (round_page(sizeof(struct user) + sizeof(*tf)) > PAGE_SIZE)
 		panic("USPACE too small for user");
 #endif
-	if (p1->p_md.md_regs->tf_cr30 == fpu_curpcb) {
-		mtctl(fpu_enable, CR_CCR);
-		fpu_save(fpu_curpcb);
-		mtctl(0, CR_CCR);
-	}
+	fpu_proc_save(p1);
 
 	pcbp = &p2->p_addr->u_pcb;
 	bcopy(&p1->p_addr->u_pcb, pcbp, sizeof(*pcbp));
 	/* space is cached for the copy{in,out}'s pleasure */
 	pcbp->pcb_space = p2->p_vmspace->vm_map.pmap->pm_space;
+	pcbp->pcb_fpstate = pool_get(&hppa_fppl, PR_WAITOK);
+	*pcbp->pcb_fpstate = *p1->p_addr->u_pcb.pcb_fpstate;
 	/* reset any of the pending FPU exceptions from parent */
-	pcbp->pcb_fpregs[0] = HPPA_FPU_FORK(pcbp->pcb_fpregs[0]);
-	pcbp->pcb_fpregs[1] = 0;
-	pcbp->pcb_fpregs[2] = 0;
-	pcbp->pcb_fpregs[3] = 0;
-	fdcache(HPPA_SID_KERNEL, (vaddr_t)&pcbp->pcb_fpregs[0], 8 * 4);
+	pcbp->pcb_fpstate->hfp_regs.fpr_regs[0] =
+	    HPPA_FPU_FORK(pcbp->pcb_fpstate->hfp_regs.fpr_regs[0]);
+	pcbp->pcb_fpstate->hfp_regs.fpr_regs[1] = 0;
+	pcbp->pcb_fpstate->hfp_regs.fpr_regs[2] = 0;
+	pcbp->pcb_fpstate->hfp_regs.fpr_regs[3] = 0;
 
 	sp = (register_t)p2->p_addr + PAGE_SIZE;
 	p2->p_md.md_regs = tf = (struct trapframe *)sp;
 	sp += sizeof(struct trapframe);
 	bcopy(p1->p_md.md_regs, tf, sizeof(*tf));
 
-	/*
-	 * Stash the physical for the pcb of U for later perusal
-	 */
-	if (!pmap_extract(pmap_kernel(), (vaddr_t)p2->p_addr, &pa))
-		panic("pmap_extract(%p) failed", p2->p_addr);
-
-	tf->tf_cr30 = pa;
+	tf->tf_cr30 = (paddr_t)pcbp->pcb_fpstate;
 
 	tf->tf_sr0 = tf->tf_sr1 = tf->tf_sr2 = tf->tf_sr3 =
 	tf->tf_sr4 = tf->tf_sr5 = tf->tf_sr6 =
@@ -165,20 +156,16 @@ cpu_fork(p1, p2, stack, stacksize, func, arg)
 	*(register_t*)(sp - HPPA_FRAME_SIZE + 8) = KERNMODE(func);
 	*(register_t*)(sp - HPPA_FRAME_SIZE + 16) = 0;	/* cpl */
 	pcbp->pcb_ksp = sp;
-	fdcache(HPPA_SID_KERNEL, (vaddr_t)p2->p_addr, sp - (vaddr_t)p2->p_addr);
 }
 
 void
-cpu_exit(p)
-	struct proc *p;
+cpu_exit(struct proc *p)
 {
-	extern paddr_t fpu_curpcb;	/* from locore.S */
-	struct trapframe *tf = p->p_md.md_regs;
+	struct pcb *pcb = &p->p_addr->u_pcb;
 
-	if (fpu_curpcb == tf->tf_cr30) {
-		fpu_exit();
-		fpu_curpcb = 0;
-	}
+	fpu_proc_flush(p);
+
+	pool_put(&hppa_fppl, pcb->pcb_fpstate);
 
 	pmap_deactivate(p);
 	sched_exit(p);
diff --git a/sys/arch/hppa64/include/cpu.h b/sys/arch/hppa64/include/cpu.h
index 948f2fb968c..a452ddd7790 100644
--- a/sys/arch/hppa64/include/cpu.h
+++ b/sys/arch/hppa64/include/cpu.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: cpu.h,v 1.23 2011/04/07 13:13:01 jsing Exp $	*/
+/*	$OpenBSD: cpu.h,v 1.24 2011/04/16 22:02:32 kettenis Exp $	*/
 
 /*
  * Copyright (c) 2005 Michael Shalayeff
@@ -134,8 +134,8 @@ struct cpu_info {
 	struct proc	*ci_curproc;
 	struct pcb	*ci_cpcb;
 	struct cpu_info	*ci_next;
+	paddr_t		ci_fpu_state;		/* Process FPU state. */
 
-	struct proc	*ci_fpproc;
 	int		ci_number;
 	struct schedstate_percpu ci_schedstate;	/* scheduler state */
 	u_int32_t	ci_randseed;
diff --git a/sys/arch/hppa64/include/fpu.h b/sys/arch/hppa64/include/fpu.h
new file mode 100644
index 00000000000..4e85b01699e
--- /dev/null
+++ b/sys/arch/hppa64/include/fpu.h
@@ -0,0 +1,36 @@
+/*	$OpenBSD: fpu.h,v 1.1 2011/04/16 22:02:32 kettenis Exp $	*/
+
+/*
+ * Copyright (c) 2010 Joel Sing <jsing@openbsd.org>
+ *
+ * Permission to use, copy, modify, and distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#ifndef _MACHINE_FPU_H_
+#define _MACHINE_FPU_H_
+
+#include <machine/cpu.h>
+#include <machine/reg.h>
+
+struct hppa_fpstate {
+	struct fpreg hfp_regs;
+	volatile struct cpu_info *hfp_cpu;	/* CPU which FPU state is on. */
+};
+
+struct proc;
+
+void	fpu_proc_flush(struct proc *);
+void	fpu_proc_save(struct proc *);
+void	fpu_cpu_save(int);
+
+#endif /* _MACHINE_FPU_H_ */
diff --git a/sys/arch/hppa64/include/pcb.h b/sys/arch/hppa64/include/pcb.h
index 2c8c9d514c8..54f2d89d7cc 100644
--- a/sys/arch/hppa64/include/pcb.h
+++ b/sys/arch/hppa64/include/pcb.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: pcb.h,v 1.2 2011/04/14 19:34:55 kettenis Exp $	*/
+/*	$OpenBSD: pcb.h,v 1.3 2011/04/16 22:02:32 kettenis Exp $	*/
 
 /*
  * Copyright (c) 2005 Michael Shalayeff
@@ -20,18 +20,15 @@
 #ifndef _MACHINE_PCB_H_
 #define _MACHINE_PCB_H_
 
+#include <machine/fpu.h>
 #include <machine/reg.h>
 
 struct pcb {
-	u_int64_t pcb_fpregs[HPPA_NFPREGS+1];	/* not in the trapframe */
-	u_int64_t pcb_onfault;		/* SW copy fault handler */
+	struct hppa_fpstate *pcb_fpstate;	/* not in the trapframe */
+
 	u_int64_t pcb_ksp;		/* kernel sp for ctxsw */
+	u_int64_t pcb_onfault;		/* SW copy fault handler */
 	pa_space_t pcb_space;		/* copy pmap_space, for asm's sake */
-
-#if 0	/* imaginary part that is after user but in the same page */
-	u_int32_t pcb_pad[53+768];
-	u_int64_t pcb_frame[64];	/* the very end */
-#endif
 };
 
 struct md_coredump {