SMT (Simultanious Multi Threading) implementations typically share

TLBs and L1 caches between threads.  This can make cache timing
attacks a lot easier and we strongly suspect that this will make
several spectre-class bugs exploitable.  Especially on Intel's SMT
implementation which is better known as Hypter-threading.  We really
should not run different security domains on different processor
threads of the same core.  Unfortunately changing our scheduler to
take this into account is far from trivial.  Since many modern
machines no longer provide the ability to disable Hyper-threading in
the BIOS setup, provide a way to disable the use of additional
processor threads in our scheduler.  And since we suspect there are
serious risks, we disable them by default.  This can be controlled
through a new hw.smt sysctl.  For now this only works on Intel CPUs
when running OpenBSD/amd64.  But we're planning to extend this feature
to CPUs from other vendors and other hardware architectures.

Note that SMT doesn't necessarily have a posive effect on performance;
it highly depends on the workload.  In all likelyhood it will actually
slow down most workloads if you have a CPU with more than two cores.

ok deraadt@

6 files changed, 67 insertions, 10 deletions

diff --git a/sys/arch/amd64/amd64/cpu.c b/sys/arch/amd64/amd64/cpu.c
index 3d9dcb7083b..12489c05a7e 100644
--- a/sys/arch/amd64/amd64/cpu.c
+++ b/sys/arch/amd64/amd64/cpu.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: cpu.c,v 1.122 2018/06/07 04:07:28 guenther Exp $	*/
+/*	$OpenBSD: cpu.c,v 1.123 2018/06/19 19:29:52 kettenis Exp $	*/
 /* $NetBSD: cpu.c,v 1.1 2003/04/26 18:39:26 fvdl Exp $ */
 
 /*-
@@ -467,8 +467,8 @@ cpu_attach(struct device *parent, struct device *self, void *aux)
 
 #if defined(MULTIPROCESSOR)
 		cpu_intr_init(ci);
-		sched_init_cpu(ci);
 		cpu_start_secondary(ci);
+		sched_init_cpu(ci);
 		ncpus++;
 		if (ci->ci_flags & CPUF_PRESENT) {
 			ci->ci_next = cpu_info_list->ci_next;
diff --git a/sys/arch/amd64/include/cpu.h b/sys/arch/amd64/include/cpu.h
index b9520284548..0c38a2ea7d9 100644
--- a/sys/arch/amd64/include/cpu.h
+++ b/sys/arch/amd64/include/cpu.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: cpu.h,v 1.123 2018/06/05 06:39:11 guenther Exp $	*/
+/*	$OpenBSD: cpu.h,v 1.124 2018/06/19 19:29:52 kettenis Exp $	*/
 /*	$NetBSD: cpu.h,v 1.1 2003/04/26 18:39:39 fvdl Exp $	*/
 
 /*-
@@ -152,7 +152,7 @@ struct cpu_info {
 
 	int		ci_inatomic;
 
-#define ARCH_HAVE_CPU_TOPOLOGY
+#define __HAVE_CPU_TOPOLOGY
 	u_int32_t	ci_smt_id;
 	u_int32_t	ci_core_id;
 	u_int32_t	ci_pkg_id;
diff --git a/sys/kern/kern_sched.c b/sys/kern/kern_sched.c
index cf45011b853..f566795dc60 100644
--- a/sys/kern/kern_sched.c
+++ b/sys/kern/kern_sched.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: kern_sched.c,v 1.47 2017/12/14 23:21:04 dlg Exp $	*/
+/*	$OpenBSD: kern_sched.c,v 1.48 2018/06/19 19:29:52 kettenis Exp $	*/
 /*
  * Copyright (c) 2007, 2008 Artur Grabowski <art@openbsd.org>
  *
@@ -56,6 +56,8 @@ uint64_t sched_wasidle;		/* Times we came out of idle */
 struct taskq *sbartq;
 #endif
 
+int sched_smt;
+
 /*
  * A few notes about cpu_switchto that is implemented in MD code.
  *
@@ -97,6 +99,11 @@ sched_init_cpu(struct cpu_info *ci)
 	 * structures.
 	 */
 	cpuset_init_cpu(ci);
+
+#ifdef __HAVE_CPU_TOPOLOGY
+	if (!sched_smt && ci->ci_smt_id > 0)
+		return;
+#endif
 	cpuset_add(&sched_all_cpus, ci);
 }
 
@@ -615,9 +622,13 @@ sched_start_secondary_cpus(void)
 
 		if (CPU_IS_PRIMARY(ci))
 			continue;
-		cpuset_add(&sched_all_cpus, ci);
 		atomic_clearbits_int(&spc->spc_schedflags,
 		    SPCF_SHOULDHALT | SPCF_HALTED);
+#ifdef __HAVE_CPU_TOPOLOGY
+		if (!sched_smt && ci->ci_smt_id > 0)
+			continue;
+#endif
+		cpuset_add(&sched_all_cpus, ci);
 	}
 }
 
@@ -793,3 +804,42 @@ cpuset_complement(struct cpuset *to, struct cpuset *a, struct cpuset *b)
 	for (i = 0; i < CPUSET_ASIZE(ncpus); i++)
 		to->cs_set[i] = b->cs_set[i] & ~a->cs_set[i];
 }
+
+#ifdef __HAVE_CPU_TOPOLOGY
+
+#include <sys/sysctl.h>
+
+int
+sysctl_hwsmt(void *oldp, size_t *oldlenp, void *newp, size_t newlen)
+{
+	CPU_INFO_ITERATOR cii;
+	struct cpu_info *ci;
+	int err, newsmt;
+
+	newsmt = sched_smt;
+	err = sysctl_int(oldp, oldlenp, newp, newlen, &newsmt);
+	if (err)
+		return err;
+	if (newsmt > 1)
+		newsmt = 1;
+	if (newsmt < 0)
+		newsmt = 0;
+	if (newsmt == sched_smt)
+		return 0;
+
+	sched_smt = newsmt;
+	CPU_INFO_FOREACH(cii, ci) {
+		if (CPU_IS_PRIMARY(ci))
+			continue;
+		if (ci->ci_smt_id == 0)
+			continue;
+		if (sched_smt)
+			cpuset_add(&sched_all_cpus, ci);
+		else
+			cpuset_del(&sched_all_cpus, ci);
+	}
+
+	return 0;
+}
+
+#endif
diff --git a/sys/kern/kern_sysctl.c b/sys/kern/kern_sysctl.c
index 43433d18d6b..3b4c0071bd7 100644
--- a/sys/kern/kern_sysctl.c
+++ b/sys/kern/kern_sysctl.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: kern_sysctl.c,v 1.341 2018/06/02 16:38:21 bluhm Exp $	*/
+/*	$OpenBSD: kern_sysctl.c,v 1.342 2018/06/19 19:29:52 kettenis Exp $	*/
 /*	$NetBSD: kern_sysctl.c,v 1.17 1996/05/20 17:49:05 mrg Exp $	*/
 
 /*-
@@ -782,6 +782,10 @@ hw_sysctl(int *name, u_int namelen, void *oldp, size_t *oldlenp, void *newp,
 			    allowpowerdown));
 		return (sysctl_int(oldp, oldlenp, newp, newlen,
 		    &allowpowerdown));
+#ifdef __HAVE_CPU_TOPOLOGY
+	case HW_SMT:
+		return (sysctl_hwsmt(oldp, oldlenp, newp, newlen));
+#endif
 	default:
 		return (EOPNOTSUPP);
 	}
diff --git a/sys/sys/sched.h b/sys/sys/sched.h
index a8b39ec11f0..0ce5332db07 100644
--- a/sys/sys/sched.h
+++ b/sys/sys/sched.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: sched.h,v 1.45 2018/05/14 12:31:21 mpi Exp $	*/
+/*	$OpenBSD: sched.h,v 1.46 2018/06/19 19:29:52 kettenis Exp $	*/
 /* $NetBSD: sched.h,v 1.2 1999/02/28 18:14:58 ross Exp $ */
 
 /*-
@@ -152,6 +152,7 @@ void sched_barrier(struct cpu_info *ci);
 
 int sysctl_hwsetperf(void *, size_t *, void *, size_t);
 int sysctl_hwperfpolicy(void *, size_t *, void *, size_t);
+int sysctl_hwsmt(void *, size_t *, void *, size_t);
 
 #ifdef MULTIPROCESSOR
 void sched_start_secondary_cpus(void);
diff --git a/sys/sys/sysctl.h b/sys/sys/sysctl.h
index 3618891b06c..25c35da5cac 100644
--- a/sys/sys/sysctl.h
+++ b/sys/sys/sysctl.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: sysctl.h,v 1.177 2018/05/26 10:16:14 ratchov Exp $	*/
+/*	$OpenBSD: sysctl.h,v 1.178 2018/06/19 19:29:52 kettenis Exp $	*/
 /*	$NetBSD: sysctl.h,v 1.16 1996/04/09 20:55:36 cgd Exp $	*/
 
 /*
@@ -863,7 +863,8 @@ struct kinfo_file {
 #define	HW_NCPUFOUND		21	/* int: number of cpus found*/
 #define	HW_ALLOWPOWERDOWN	22	/* allow power button shutdown */
 #define	HW_PERFPOLICY		23	/* set performance policy */
-#define	HW_MAXID		24	/* number of valid hw ids */
+#define	HW_SMT			24	/* int: enable SMT/HT/CMT */
+#define	HW_MAXID		25	/* number of valid hw ids */
 
 #define	CTL_HW_NAMES { \
 	{ 0, 0 }, \
@@ -890,6 +891,7 @@ struct kinfo_file {
 	{ "ncpufound", CTLTYPE_INT }, \
 	{ "allowpowerdown", CTLTYPE_INT }, \
 	{ "perfpolicy", CTLTYPE_STRING }, \
+	{ "smt", CTLTYPE_INT }, \
 }
 
 /*