On CPUs with eIBRS ("enhanced Indirect Branch Restricted Speculation")

or IBT enabled the kernel, the hardware should the attacks which
retpolines were created to prevent.  In those cases, retpolines
should be a net negative for security as they are an indirect branch
gadget.  They're also slower.
 * use -mretpoline-external-thunk to give us control of the code
   used for indirect branches
 * default to using a retpoline as before, but marks it and the
   other ASM kernel retpolines for code patching
 * if the CPU has eIBRS, then enable it
 * if the CPU has eIBRS *or* IBT, then codepatch the three different
   retpolines to just indirect jumps

make clean && make config required after this

ok kettenis@

5 files changed, 57 insertions, 11 deletions

diff --git a/sys/arch/amd64/amd64/cpu.c b/sys/arch/amd64/amd64/cpu.c
index 542d961d050..60220bff4f5 100644
--- a/sys/arch/amd64/amd64/cpu.c
+++ b/sys/arch/amd64/amd64/cpu.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: cpu.c,v 1.174 2023/07/28 06:36:16 guenther Exp $	*/
+/*	$OpenBSD: cpu.c,v 1.175 2023/07/31 04:01:07 guenther Exp $	*/
 /* $NetBSD: cpu.c,v 1.1 2003/04/26 18:39:26 fvdl Exp $ */
 
 /*-
@@ -187,11 +187,7 @@ replacemeltdown(void)
 {
 	static int replacedone = 0;
 	struct cpu_info *ci = &cpu_info_primary;
-	int swapgs_vuln = 0, s;
-
-	if (replacedone)
-		return;
-	replacedone = 1;
+	int swapgs_vuln = 0, ibrs = 0, s;
 
 	if (strcmp(cpu_vendor, "GenuineIntel") == 0) {
 		int family = ci->ci_family;
@@ -208,9 +204,39 @@ replacemeltdown(void)
 			/* KnightsLanding */
 			swapgs_vuln = 0;
 		}
+		if ((ci->ci_feature_sefflags_edx & SEFF0EDX_ARCH_CAP) &&
+		    (rdmsr(MSR_ARCH_CAPABILITIES) & ARCH_CAP_IBRS_ALL)) {
+			ibrs = 2;
+		} else if (ci->ci_feature_sefflags_edx & SEFF0EDX_IBRS) {
+			ibrs = 1;
+		}
+        } else if (strcmp(cpu_vendor, "AuthenticAMD") == 0 &&
+            ci->ci_pnfeatset >= 0x80000008) {
+		if (ci->ci_feature_amdspec_ebx & CPUIDEBX_IBRS_ALWAYSON) {
+			ibrs = 2;
+		} else if ((ci->ci_feature_amdspec_ebx & CPUIDEBX_IBRS) &&
+		    (ci->ci_feature_amdspec_ebx & CPUIDEBX_IBRS_PREF)) {
+			ibrs = 1;
+		}
 	}
 
+	/* Enhanced IBRS: turn it on once on each CPU and don't touch again */
+	if (ibrs == 2)
+		wrmsr(MSR_SPEC_CTRL, SPEC_CTRL_IBRS);
+
+	if (replacedone)
+		return;
+	replacedone = 1;
+
 	s = splhigh();
+	if (ibrs == 2 || (ci->ci_feature_sefflags_edx & SEFF0EDX_IBT)) {
+		extern const char _jmprax, _jmpr11, _jmpr13;
+		extern const short _jmprax_len, _jmpr11_len, _jmpr13_len;
+		codepatch_replace(CPTAG_RETPOLINE_RAX, &_jmprax, _jmprax_len);
+		codepatch_replace(CPTAG_RETPOLINE_R11, &_jmpr11, _jmpr11_len);
+		codepatch_replace(CPTAG_RETPOLINE_R13, &_jmpr13, _jmpr13_len);
+	}
+
 	if (!cpu_meltdown)
 		codepatch_nop(CPTAG_MELTDOWN_NOP);
 	else {
diff --git a/sys/arch/amd64/amd64/locore.S b/sys/arch/amd64/amd64/locore.S
index 9024948b899..27938da9f28 100644
--- a/sys/arch/amd64/amd64/locore.S
+++ b/sys/arch/amd64/amd64/locore.S
@@ -1,4 +1,4 @@
-/*	$OpenBSD: locore.S,v 1.139 2023/07/28 06:18:35 guenther Exp $	*/
+/*	$OpenBSD: locore.S,v 1.140 2023/07/31 04:01:07 guenther Exp $	*/
 /*	$NetBSD: locore.S,v 1.13 2004/03/25 18:33:17 drochner Exp $	*/
 
 /*
@@ -474,7 +474,16 @@ bogus_proc_pmap:
 END(cpu_switchto)
 
 NENTRY(retpoline_rax)
+	CODEPATCH_START
 	JMP_RETPOLINE(rax)
+	CODEPATCH_END(CPTAG_RETPOLINE_RAX)
+END(retpoline_rax)
+
+NENTRY(__x86_indirect_thunk_r11)
+	CODEPATCH_START
+	JMP_RETPOLINE(r11)
+	CODEPATCH_END(CPTAG_RETPOLINE_R11)
+END(__x86_indirect_thunk_r11)
 
 ENTRY(cpu_idle_cycle_hlt)
 	RETGUARD_SETUP(cpu_idle_cycle_hlt, r11)
diff --git a/sys/arch/amd64/amd64/vector.S b/sys/arch/amd64/amd64/vector.S
index d3b31c76198..af268a5295e 100644
--- a/sys/arch/amd64/amd64/vector.S
+++ b/sys/arch/amd64/amd64/vector.S
@@ -1,4 +1,4 @@
-/*	$OpenBSD: vector.S,v 1.93 2023/07/27 00:30:07 guenther Exp $	*/
+/*	$OpenBSD: vector.S,v 1.94 2023/07/31 04:01:07 guenther Exp $	*/
 /*	$NetBSD: vector.S,v 1.5 2004/06/28 09:13:11 fvdl Exp $	*/
 
 /*
@@ -1269,7 +1269,9 @@ END(ioapic_level_stubs)
  * Soft interrupt handlers
  */
 NENTRY(retpoline_r13)
+	CODEPATCH_START
 	JMP_RETPOLINE(r13)
+	CODEPATCH_END(CPTAG_RETPOLINE_R13)
 END(retpoline_r13)
 
 KIDTVEC(softtty)
@@ -1280,7 +1282,9 @@ KIDTVEC(softtty)
 	movl	$X86_SOFTINTR_SOFTTTY,%edi
 	call	softintr_dispatch
 	decl	CPUVAR(IDEPTH)
+	CODEPATCH_START
 	jmp	retpoline_r13
+	CODEPATCH_END(CPTAG_RETPOLINE_R13)
 END(Xsofttty)
 
 KIDTVEC(softnet)
@@ -1291,7 +1295,9 @@ KIDTVEC(softnet)
 	movl	$X86_SOFTINTR_SOFTNET,%edi
 	call	softintr_dispatch
 	decl	CPUVAR(IDEPTH)
+	CODEPATCH_START
 	jmp	retpoline_r13
+	CODEPATCH_END(CPTAG_RETPOLINE_R13)
 END(Xsoftnet)
 
 KIDTVEC(softclock)
@@ -1302,5 +1308,7 @@ KIDTVEC(softclock)
 	movl	$X86_SOFTINTR_SOFTCLOCK,%edi
 	call	softintr_dispatch
 	decl	CPUVAR(IDEPTH)
+	CODEPATCH_START
 	jmp	retpoline_r13
+	CODEPATCH_END(CPTAG_RETPOLINE_R13)
 END(Xsoftclock)
diff --git a/sys/arch/amd64/conf/Makefile.amd64 b/sys/arch/amd64/conf/Makefile.amd64
index 7759b875d7f..6f760b174bb 100644
--- a/sys/arch/amd64/conf/Makefile.amd64
+++ b/sys/arch/amd64/conf/Makefile.amd64
@@ -1,4 +1,4 @@
-#	$OpenBSD: Makefile.amd64,v 1.132 2023/04/21 13:24:20 bluhm Exp $
+#	$OpenBSD: Makefile.amd64,v 1.133 2023/07/31 04:01:07 guenther Exp $
 
 # For instructions on building kernels consult the config(8) and options(4)
 # manual pages.
@@ -72,7 +72,7 @@ COPTIMIZE=	-Oz
 CMACHFLAGS+=	-mno-retpoline -fcf-protection=none
 .endif
 .else
-CMACHFLAGS+=	-fcf-protection=branch
+CMACHFLAGS+=	-mretpoline-external-thunk -fcf-protection=branch
 .endif
 .if ${COMPILER_VERSION:Mclang}
 NO_INTEGR_AS=	-no-integrated-as
diff --git a/sys/arch/amd64/include/codepatch.h b/sys/arch/amd64/include/codepatch.h
index 6401dab9f2f..6f2bdbe184e 100644
--- a/sys/arch/amd64/include/codepatch.h
+++ b/sys/arch/amd64/include/codepatch.h
@@ -1,4 +1,4 @@
-/*      $OpenBSD: codepatch.h,v 1.17 2023/07/31 01:33:57 guenther Exp $    */
+/*      $OpenBSD: codepatch.h,v 1.18 2023/07/31 04:01:07 guenther Exp $    */
 /*
  * Copyright (c) 2014-2015 Stefan Fritsch <sf@sfritsch.de>
  *
@@ -66,6 +66,9 @@ void codepatch_disable(void);
 #define CPTAG_FENCE_SWAPGS_MIS_TAKEN	11
 #define CPTAG_FENCE_NO_SAFE_SMAP	12
 #define CPTAG_XRSTORS			13
+#define CPTAG_RETPOLINE_RAX		14
+#define CPTAG_RETPOLINE_R11		15
+#define CPTAG_RETPOLINE_R13		16
 
 /*
  * stac/clac SMAP instructions have lfence like semantics.  Let's