Add support for per-rthread base-offset for the %fs selector on amd64.

Add pcb_fsbase to the PCB for tracking what the value for the thread
is, and ci_cur_fsbase to struct cpu_info for tracking the CPU's current
value for FS.base, then on return to user-space, skip the setting if the
CPU has the right value already.  Non-threaded processes without TLS leave
FS.base zero, which can be conveniently optimized: setting %fs zeros
FS.base for fewer cycles than wrmsr.

ok kettenis@

9 files changed, 95 insertions, 30 deletions

diff --git a/sys/arch/amd64/amd64/cpu.c b/sys/arch/amd64/amd64/cpu.c
index 6f05b311123..512f930ddfa 100644
--- a/sys/arch/amd64/amd64/cpu.c
+++ b/sys/arch/amd64/amd64/cpu.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: cpu.c,v 1.41 2011/03/18 03:10:47 guenther Exp $	*/
+/*	$OpenBSD: cpu.c,v 1.42 2011/04/05 21:14:00 guenther Exp $	*/
 /* $NetBSD: cpu.c,v 1.1 2003/04/26 18:39:26 fvdl Exp $ */
 
 /*-
@@ -657,6 +657,7 @@ cpu_init_msrs(struct cpu_info *ci)
 	wrmsr(MSR_CSTAR, (uint64_t)Xsyscall32);
 	wrmsr(MSR_SFMASK, PSL_NT|PSL_T|PSL_I|PSL_C);
 
+	ci->ci_cur_fsbase = 0;
 	wrmsr(MSR_FSBASE, 0);
 	wrmsr(MSR_GSBASE, (u_int64_t)ci);
 	wrmsr(MSR_KERNELGSBASE, 0);
diff --git a/sys/arch/amd64/amd64/genassym.cf b/sys/arch/amd64/amd64/genassym.cf
index b882b230349..fa89ea7d9f5 100644
--- a/sys/arch/amd64/amd64/genassym.cf
+++ b/sys/arch/amd64/amd64/genassym.cf
@@ -1,4 +1,4 @@
-#	$OpenBSD: genassym.cf,v 1.25 2010/11/13 04:16:42 guenther Exp $
+#	$OpenBSD: genassym.cf,v 1.26 2011/04/05 21:14:00 guenther Exp $
 #	Written by Artur Grabowski art@openbsd.org, Public Domain
 
 include <sys/param.h>
@@ -85,6 +85,7 @@ member	pcb_cr3
 member	pcb_rsp
 member	pcb_rbp
 member	pcb_kstack
+member	pcb_fsbase
 member	pcb_onfault
 member	pcb_fpcpu
 member	pcb_pmap
@@ -98,6 +99,7 @@ member	tss_rsp0
 
 struct	cpu_info
 member	CPU_INFO_SCRATCH	ci_scratch
+member	CPU_INFO_CUR_FSBASE	ci_cur_fsbase
 member  CPU_INFO_SELF		ci_self
 member  CPU_INFO_CPUID		ci_cpuid
 member  CPU_INFO_APICID		ci_apicid
diff --git a/sys/arch/amd64/amd64/locore.S b/sys/arch/amd64/amd64/locore.S
index f7016f9b4d3..408252316c6 100644
--- a/sys/arch/amd64/amd64/locore.S
+++ b/sys/arch/amd64/amd64/locore.S
@@ -1,4 +1,4 @@
-/*	$OpenBSD: locore.S,v 1.44 2010/12/04 05:20:18 guenther Exp $	*/
+/*	$OpenBSD: locore.S,v 1.45 2011/04/05 21:14:00 guenther Exp $	*/
 /*	$NetBSD: locore.S,v 1.13 2004/03/25 18:33:17 drochner Exp $	*/
 
 /*
@@ -962,14 +962,8 @@ syscall_return:
 	/*
 	 * XXX interrupts off longer than they should be here.
 	 */
-	cli
-	swapgs
-	movw	TF_ES(%rsp),%es
-	movw	TF_FS(%rsp),%fs
-	movw	TF_GS(%rsp),%gs
+	INTR_RESTORE_SELECTORS
 	INTR_RESTORE_GPRS
-	movw	$(GSEL(GUDATA_SEL, SEL_UPL)),%r11
-	movw	%r11,%ds
 	addq	$48,%rsp
 	popq	%rcx	/* return rip */
 	addq	$8,%rsp
@@ -1049,16 +1043,11 @@ iret_return:
 #endif /* DIAGNOSTIC */
 	.globl	intr_fast_exit
 intr_fast_exit:
-	INTR_RESTORE_GPRS
-	testq	$SEL_UPL,56(%rsp)
+	testq	$SEL_UPL,TF_CS(%rsp)
 	je	5f
-	cli
-	swapgs
-	movw	0(%rsp),%gs
-	movw	8(%rsp),%fs
-	movw	16(%rsp),%es
-	movw	24(%rsp),%ds
-5:	addq	$48,%rsp
+	INTR_RESTORE_SELECTORS
+5:	INTR_RESTORE_GPRS
+	addq	$48,%rsp
 	.globl	_C_LABEL(doreti_iret)
 _C_LABEL(doreti_iret):
 	iretq
diff --git a/sys/arch/amd64/amd64/machdep.c b/sys/arch/amd64/amd64/machdep.c
index 13249809fd3..700fe911de6 100644
--- a/sys/arch/amd64/amd64/machdep.c
+++ b/sys/arch/amd64/amd64/machdep.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: machdep.c,v 1.134 2011/04/02 18:16:50 oga Exp $	*/
+/*	$OpenBSD: machdep.c,v 1.135 2011/04/05 21:14:00 guenther Exp $	*/
 /*	$NetBSD: machdep.c,v 1.3 2003/05/07 22:58:18 fvdl Exp $	*/
 
 /*-
@@ -368,6 +368,7 @@ x86_64_proc0_tss_ldt_init(void)
 
 	cpu_info_primary.ci_curpcb = pcb = &proc0.p_addr->u_pcb;
 	pcb->pcb_cr0 = rcr0();
+	pcb->pcb_fsbase = 0;
 	pcb->pcb_kstack = (u_int64_t)proc0.p_addr + USPACE - 16;
 	proc0.p_md.md_regs = (struct trapframe *)pcb->pcb_kstack - 1;
 
@@ -999,6 +1000,7 @@ setregs(struct proc *p, struct exec_package *pack, u_long stack,
 	if (p->p_addr->u_pcb.pcb_fpcpu != NULL)
 		fpusave_proc(p, 0);
 	p->p_md.md_flags &= ~MDP_USEDFPU;
+	p->p_addr->u_pcb.pcb_fsbase = 0;
 
 	tf = p->p_md.md_regs;
 	tf->tf_ds = GSEL(GUDATA_SEL, SEL_UPL);
diff --git a/sys/arch/amd64/amd64/sys_machdep.c b/sys/arch/amd64/amd64/sys_machdep.c
index 7679f27b3b9..fdc50132d7a 100644
--- a/sys/arch/amd64/amd64/sys_machdep.c
+++ b/sys/arch/amd64/amd64/sys_machdep.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: sys_machdep.c,v 1.9 2011/03/18 03:10:47 guenther Exp $	*/
+/*	$OpenBSD: sys_machdep.c,v 1.10 2011/04/05 21:14:00 guenther Exp $	*/
 /*	$NetBSD: sys_machdep.c,v 1.1 2003/04/26 18:39:32 fvdl Exp $	*/
 
 /*-
@@ -136,6 +136,29 @@ amd64_set_ioperm(struct proc *p, void *args, register_t *retval)
 #endif
 
 int
+amd64_get_fsbase(struct proc *p, void *args)
+{
+	return copyout(&p->p_addr->u_pcb.pcb_fsbase, args,
+	    sizeof(p->p_addr->u_pcb.pcb_fsbase));
+}
+
+int
+amd64_set_fsbase(struct proc *p, void *args)
+{
+	int error;
+	uint64_t base;
+
+	if ((error = copyin(args, &base, sizeof(base))) != 0)
+		return (error);
+
+	if (base >= VM_MAXUSER_ADDRESS)
+		return (EINVAL);
+
+	p->p_addr->u_pcb.pcb_fsbase = base;
+	return 0;
+}
+
+int
 sys_sysarch(struct proc *p, void *v, register_t *retval)
 {
 	struct sys_sysarch_args /* {
@@ -172,6 +195,15 @@ sys_sysarch(struct proc *p, void *v, register_t *retval)
 		error = pmc_read(p, SCARG(uap, parms), retval);
 		break;
 #endif
+
+	case AMD64_GET_FSBASE: 
+		error = amd64_get_fsbase(p, SCARG(uap, parms));
+		break;
+
+	case AMD64_SET_FSBASE: 
+		error = amd64_set_fsbase(p, SCARG(uap, parms));
+		break;
+
 	default:
 		error = EINVAL;
 		break;
diff --git a/sys/arch/amd64/include/cpu.h b/sys/arch/amd64/include/cpu.h
index 260212f10a9..53e063ae551 100644
--- a/sys/arch/amd64/include/cpu.h
+++ b/sys/arch/amd64/include/cpu.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: cpu.h,v 1.63 2011/03/23 16:54:34 pirofti Exp $	*/
+/*	$OpenBSD: cpu.h,v 1.64 2011/04/05 21:14:00 guenther Exp $	*/
 /*	$NetBSD: cpu.h,v 1.1 2003/04/26 18:39:39 fvdl Exp $	*/
 
 /*-
@@ -72,6 +72,7 @@ struct cpu_info {
 	u_int32_t ci_randseed;
 
 	u_int64_t ci_scratch;
+	u_int64_t ci_cur_fsbase;
 
 	struct proc *ci_fpcurproc;
 	struct proc *ci_fpsaveproc;
diff --git a/sys/arch/amd64/include/frameasm.h b/sys/arch/amd64/include/frameasm.h
index 5e1fa6145e1..7a82c061d51 100644
--- a/sys/arch/amd64/include/frameasm.h
+++ b/sys/arch/amd64/include/frameasm.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: frameasm.h,v 1.2 2010/09/28 03:53:14 guenther Exp $	*/
+/*	$OpenBSD: frameasm.h,v 1.3 2011/04/05 21:14:00 guenther Exp $	*/
 /*	$NetBSD: frameasm.h,v 1.1 2003/04/26 18:39:40 fvdl Exp $	*/
 
 #ifndef _AMD64_MACHINE_FRAMEASM_H
@@ -72,6 +72,37 @@
 	pushq	%r11			; \
 	pushq	%r13			;
 
+/*
+ * Restore %ds, %es, %fs, and %gs, dealing with the FS.base MSR for
+ * %fs and doing the cli/swapgs for %gs.  Uses %rax, %rcx, and %rdx
+ */
+#define INTR_RESTORE_SELECTORS						\
+	movq	CPUVAR(CURPCB),%rdx	/* for below */			; \
+	/* %es and %ds */						  \
+	movw	TF_ES(%rsp),%es						; \
+	movw	$(GSEL(GUDATA_SEL, SEL_UPL)),%ax			; \
+	movw	%ax,%ds							; \
+	/* Make sure both %fs and FS.base are the desired values */	  \
+	movq	PCB_FSBASE(%rdx),%rax					; \
+	cmpq	$0,%rax							; \
+	jne	96f							; \
+	movw	TF_FS(%rsp),%fs	/* zero FS.base by setting %fs */	; \
+	jmp	98f							; \
+96:	cmpq	CPUVAR(CUR_FSBASE),%rax					; \
+	jne	97f							; \
+	movw	%fs,%cx		/* FS.base same, how about %fs? */	; \
+	cmpw	TF_FS(%rsp),%cx						; \
+	je	99f							; \
+97:	movw	TF_FS(%rsp),%fs		/* set them both */		; \
+	movq	%rax,%rdx						; \
+	shrq	$32,%rdx						; \
+	movl	$MSR_FSBASE,%ecx					; \
+	wrmsr								; \
+98:	movq	%rax,CPUVAR(CUR_FSBASE)					; \
+99:	cli		/* %fs done, so swapgs and do %gs */		; \
+	swapgs								; \
+	movw	TF_GS(%rsp),%gs
+
 
 #define CHECK_ASTPENDING(reg)	movq	CPUVAR(CURPROC),reg		; \
 				cmpq	$0, reg				; \
diff --git a/sys/arch/amd64/include/pcb.h b/sys/arch/amd64/include/pcb.h
index 3a7ced70e88..387fc6aa699 100644
--- a/sys/arch/amd64/include/pcb.h
+++ b/sys/arch/amd64/include/pcb.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: pcb.h,v 1.8 2011/03/23 16:54:34 pirofti Exp $	*/
+/*	$OpenBSD: pcb.h,v 1.9 2011/04/05 21:14:00 guenther Exp $	*/
 /*	$NetBSD: pcb.h,v 1.1 2003/04/26 18:39:45 fvdl Exp $	*/
 
 /*-
@@ -86,6 +86,7 @@ struct pcb {
 	u_int64_t	pcb_rsp;
 	u_int64_t	pcb_rbp;
 	u_int64_t	pcb_kstack;	/* kernel stack address */
+	u_int64_t	pcb_fsbase;	/* per-thread offset: %fs */
 	caddr_t	pcb_onfault;		/* copyin/out fault recovery */
 	struct	cpu_info *pcb_fpcpu;	/* cpu holding our fp state. */
 	struct	pmap *pcb_pmap;		/* back pointer to our pmap */
diff --git a/sys/arch/amd64/include/sysarch.h b/sys/arch/amd64/include/sysarch.h
index 92e50db5848..a8594974747 100644
--- a/sys/arch/amd64/include/sysarch.h
+++ b/sys/arch/amd64/include/sysarch.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: sysarch.h,v 1.7 2011/03/23 16:54:34 pirofti Exp $	*/
+/*	$OpenBSD: sysarch.h,v 1.8 2011/04/05 21:14:00 guenther Exp $	*/
 /*	$NetBSD: sysarch.h,v 1.1 2003/04/26 18:39:48 fvdl Exp $	*/
 
 #ifndef _MACHINE_SYSARCH_H_
@@ -7,13 +7,15 @@
 /*
  * Architecture specific syscalls (amd64)
  */
-#define	AMD64_IOPL	2
+#define	AMD64_IOPL		2
 #define	AMD64_GET_IOPERM	3
 #define	AMD64_SET_IOPERM	4
-#define	AMD64_VM86	5
-#define	AMD64_PMC_INFO	8
-#define	AMD64_PMC_STARTSTOP 9
-#define	AMD64_PMC_READ	10
+#define	AMD64_VM86		5
+#define	AMD64_PMC_INFO		8
+#define	AMD64_PMC_STARTSTOP	9
+#define	AMD64_PMC_READ		10
+#define	AMD64_GET_FSBASE	11
+#define	AMD64_SET_FSBASE	12
 
 struct amd64_iopl_args {
 	int iopl;
@@ -63,6 +65,8 @@ struct amd64_pmc_read_args {
 
 #ifdef _KERNEL
 int amd64_iopl(struct proc *, void *, register_t *);
+int amd64_set_fsbase(struct proc *, void *);
+int amd64_get_fsbase(struct proc *, void *);
 #else
 int amd64_iopl(int);
 int amd64_get_ioperm(u_long *);
@@ -70,6 +74,8 @@ int amd64_set_ioperm(u_long *);
 int amd64_pmc_info(struct amd64_pmc_info_args *);
 int amd64_pmc_startstop(struct amd64_pmc_startstop_args *);
 int amd64_pmc_read(struct amd64_pmc_read_args *);
+int amd64_set_fsbase(void *);
+int amd64_get_fsbase(void **);
 int sysarch(int, void *);
 #endif