Apply the retpoline transformation to indirect jumps in the raw ASM

ok mlarkin@ mortimer@ deraadt@

4 files changed, 50 insertions, 39 deletions

diff --git a/sys/arch/amd64/amd64/locore.S b/sys/arch/amd64/amd64/locore.S
index 7cea1392a31..3f1e2c3700c 100644
--- a/sys/arch/amd64/amd64/locore.S
+++ b/sys/arch/amd64/amd64/locore.S
@@ -1,4 +1,4 @@
-/*	$OpenBSD: locore.S,v 1.98 2018/06/05 06:39:10 guenther Exp $	*/
+/*	$OpenBSD: locore.S,v 1.99 2018/06/07 04:09:35 guenther Exp $	*/
 /*	$NetBSD: locore.S,v 1.13 2004/03/25 18:33:17 drochner Exp $	*/
 
 /*
@@ -247,8 +247,7 @@ gdt64_end:
 	.section .rodata
 	.globl	_C_LABEL(sigcode)
 _C_LABEL(sigcode):
-	call	*%rax
-
+	call	1f
 	movq	%rsp,%rdi
 	pushq	%rdi			/* fake return address */
 	movq	$SYS_sigreturn,%rax
@@ -257,6 +256,8 @@ _C_LABEL(sigcode):
 _C_LABEL(sigcoderet):
 	movq	$SYS_exit,%rax
 	syscall
+	.align	16,0xcc
+1:	JMP_RETPOLINE(rax)
 	.globl	_C_LABEL(esigcode)
 _C_LABEL(esigcode):
 
@@ -490,30 +491,27 @@ switch_restored:
 ENTRY(cpu_idle_enter)
 	movq	_C_LABEL(cpu_idle_enter_fcn),%rax
 	cmpq	$0,%rax
-	je	1f
-	jmpq	*%rax
-1:
+	jne	retpoline_rax
 	ret
 
+ENTRY(cpu_idle_leave)
+	movq	_C_LABEL(cpu_idle_leave_fcn),%rax
+	cmpq	$0,%rax
+	jne	retpoline_rax
+	ret
+
+/* placed here for correct static branch prediction in cpu_idle_* */
+NENTRY(retpoline_rax)
+	JMP_RETPOLINE(rax)
+
 ENTRY(cpu_idle_cycle)
 	movq	_C_LABEL(cpu_idle_cycle_fcn),%rax
 	cmpq	$0,%rax
-	je	1f
-	call	*%rax
-	ret
-1:
+	jne	retpoline_rax
 	sti
 	hlt
 	ret
 
-ENTRY(cpu_idle_leave)
-	movq	_C_LABEL(cpu_idle_leave_fcn),%rax
-	cmpq	$0,%rax
-	je	1f
-	jmpq	*%rax
-1:
-	ret
-
 	.globl	_C_LABEL(panic)
 
 #ifdef DIAGNOSTIC
@@ -720,7 +718,8 @@ NENTRY(proc_trampoline)
 #endif
 	movl	$IPL_NONE,CPUVAR(ILEVEL)
 	movq	%r13,%rdi
-	call	*%r12
+	movq	%r12,%rax
+	call	retpoline_rax
 	movq	CPUVAR(CURPROC),%r14
 	jmp	.Lsyscall_check_asts
 
diff --git a/sys/arch/amd64/amd64/spl.S b/sys/arch/amd64/amd64/spl.S
index 1fee60d110c..0c2baffd532 100644
--- a/sys/arch/amd64/amd64/spl.S
+++ b/sys/arch/amd64/amd64/spl.S
@@ -1,4 +1,4 @@
-/*	$OpenBSD: spl.S,v 1.13 2018/06/05 05:04:31 guenther Exp $	*/
+/*	$OpenBSD: spl.S,v 1.14 2018/06/07 04:09:35 guenther Exp $	*/
 /*	$NetBSD: spl.S,v 1.3 2004/06/28 09:13:11 fvdl Exp $	*/
 
 /*
@@ -99,7 +99,7 @@ _C_LABEL(splx):
 	ret
 #endif /* PROF || GPROF */
 #endif
-	
+
 /*
  * Process pending interrupts.
  *
@@ -124,17 +124,17 @@ KIDTVEC(spllower)
 	movq	CPUVAR(IUNMASK)(,%rax,8),%rax
 	cli
 	andq	CPUVAR(IPENDING),%rax		# any non-masked bits left?
-	jz	2f
-	bsrq	%rax,%rax
-	btrq	%rax,CPUVAR(IPENDING)
-	movq	CPUVAR(ISOURCES)(,%rax,8),%rax
-	jmp	*IS_RECURSE(%rax)
-2:
+	jnz	2f
 	movl	%ebx,CPUVAR(ILEVEL)
 	sti
 	popq	%r13
 	popq	%rbx
 	ret
+2:	bsrq	%rax,%rax
+	btrq	%rax,CPUVAR(IPENDING)
+	movq	CPUVAR(ISOURCES)(,%rax,8),%rax
+	movq    IS_RECURSE(%rax),%rax
+	jmp     retpoline_rax
 
 /*
  * Handle return from interrupt after device handler finishes.
@@ -151,13 +151,13 @@ KIDTVEC(doreti)
 	movq	CPUVAR(IUNMASK)(,%rax,8),%rax
 	cli
 	andq	CPUVAR(IPENDING),%rax
-	jz	2f
-	bsrq    %rax,%rax               # slow, but not worth optimizing
-	btrq    %rax,CPUVAR(IPENDING)
-	movq	CPUVAR(ISOURCES)(,%rax, 8),%rax
-	jmp	*IS_RESUME(%rax)
-2:	/* Check for ASTs on exit to user mode. */
+	jnz	2f
 	movl	%ebx,CPUVAR(ILEVEL)
 	testb	$SEL_RPL,TF_CS(%rsp)
-	jnz	intr_user_exit
+	jnz	intr_user_exit		# Check for ASTs on exit to user mode
 	INTRFASTEXIT
+2:	bsrq    %rax,%rax               # slow, but not worth optimizing
+	btrq    %rax,CPUVAR(IPENDING)
+	movq	CPUVAR(ISOURCES)(,%rax, 8),%rax
+	movq    IS_RESUME(%rax),%rax
+	jmp     retpoline_rax
diff --git a/sys/arch/amd64/amd64/vector.S b/sys/arch/amd64/amd64/vector.S
index a5cb2d5e835..f8e5f2f7d0e 100644
--- a/sys/arch/amd64/amd64/vector.S
+++ b/sys/arch/amd64/amd64/vector.S
@@ -1,4 +1,4 @@
-/*	$OpenBSD: vector.S,v 1.60 2018/06/05 06:39:10 guenther Exp $	*/
+/*	$OpenBSD: vector.S,v 1.61 2018/06/07 04:09:35 guenther Exp $	*/
 /*	$NetBSD: vector.S,v 1.5 2004/06/28 09:13:11 fvdl Exp $	*/
 
 /*
@@ -1130,6 +1130,9 @@ _C_LABEL(ioapic_level_stubs):
 /*
  * Soft interrupt handlers
  */
+NENTRY(retpoline_r13)
+	JMP_RETPOLINE(r13)
+
 KIDTVEC(softtty)
 	movl	$IPL_SOFTTTY, CPUVAR(ILEVEL)
 	sti
@@ -1137,7 +1140,7 @@ KIDTVEC(softtty)
 	movl	$X86_SOFTINTR_SOFTTTY,%edi
 	call	_C_LABEL(softintr_dispatch)
 	decl	CPUVAR(IDEPTH)
-	jmp	*%r13
+	jmp	retpoline_r13
 
 KIDTVEC(softnet)
 	movl	$IPL_SOFTNET, CPUVAR(ILEVEL)
@@ -1146,7 +1149,7 @@ KIDTVEC(softnet)
 	movl	$X86_SOFTINTR_SOFTNET,%edi
 	call	_C_LABEL(softintr_dispatch)
 	decl	CPUVAR(IDEPTH)
-	jmp	*%r13
+	jmp	retpoline_r13
 
 KIDTVEC(softclock)
 	movl	$IPL_SOFTCLOCK, CPUVAR(ILEVEL)
@@ -1155,4 +1158,4 @@ KIDTVEC(softclock)
 	movl	$X86_SOFTINTR_SOFTCLOCK,%edi
 	call	_C_LABEL(softintr_dispatch)
 	decl	CPUVAR(IDEPTH)
-	jmp	*%r13
+	jmp	retpoline_r13
diff --git a/sys/arch/amd64/include/asm.h b/sys/arch/amd64/include/asm.h
index cd3922b4b26..f3b7a249534 100644
--- a/sys/arch/amd64/include/asm.h
+++ b/sys/arch/amd64/include/asm.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: asm.h,v 1.9 2018/02/21 19:24:15 guenther Exp $	*/
+/*	$OpenBSD: asm.h,v 1.10 2018/06/07 04:09:35 guenther Exp $	*/
 /*	$NetBSD: asm.h,v 1.2 2003/05/02 18:05:47 yamt Exp $	*/
 
 /*-
@@ -120,4 +120,13 @@
 	.stabs __STRING(sym),1,0,0,0
 #endif /* __STDC__ */
 
+/* generic retpoline ("return trampoline") generator */
+#define	JMP_RETPOLINE(reg)		\
+		call	69f		; \
+	68:	pause			; \
+		jmp	68b		; \
+		.align	16,0xcc		; \
+	69:	mov	%reg,(%rsp)	; \
+		ret
+
 #endif /* !_MACHINE_ASM_H_ */