Fix pctr(4) issues with MP and suspend:

 - use an IPI to notify other CPUs toi update CR4 and the MSRs
 - use the cpu(4) resume callback to restore the pctr(4) settings after
   suspend/hibernate

ok kettenis@ deraadt@

5 files changed, 97 insertions, 28 deletions

diff --git a/sys/arch/amd64/amd64/cpu.c b/sys/arch/amd64/amd64/cpu.c
index 593c1eda34e..0ae5c4c8ddd 100644
--- a/sys/arch/amd64/amd64/cpu.c
+++ b/sys/arch/amd64/amd64/cpu.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: cpu.c,v 1.132 2019/01/20 23:07:51 guenther Exp $	*/
+/*	$OpenBSD: cpu.c,v 1.133 2019/03/25 18:48:12 guenther Exp $	*/
 /* $NetBSD: cpu.c,v 1.1 2003/04/26 18:39:26 fvdl Exp $ */
 
 /*-
@@ -67,6 +67,7 @@
 #include "lapic.h"
 #include "ioapic.h"
 #include "vmm.h"
+#include "pctr.h"
 #include "pvbus.h"
 
 #include <sys/param.h>
@@ -105,6 +106,10 @@
 #include <machine/i82093var.h>
 #endif
 
+#if NPCTR > 0
+#include <machine/pctr.h>
+#endif
+
 #if NPVBUS > 0
 #include <dev/pv/pvvar.h>
 #endif
@@ -990,6 +995,9 @@ cpu_activate(struct device *self, int act)
 	case DVACT_RESUME:
 		if (sc->sc_info->ci_cpuid == 0)
 			rdrand(NULL);
+#if NPCTR > 0
+		pctr_resume(sc->sc_info);
+#endif
 		break;
 	}
 
diff --git a/sys/arch/amd64/amd64/ipifuncs.c b/sys/arch/amd64/amd64/ipifuncs.c
index c5e4b27cd36..a8cb3363700 100644
--- a/sys/arch/amd64/amd64/ipifuncs.c
+++ b/sys/arch/amd64/amd64/ipifuncs.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: ipifuncs.c,v 1.32 2018/07/27 21:11:31 kettenis Exp $	*/
+/*	$OpenBSD: ipifuncs.c,v 1.33 2019/03/25 18:48:12 guenther Exp $	*/
 /*	$NetBSD: ipifuncs.c,v 1.1 2003/04/26 18:39:28 fvdl Exp $ */
 
 /*-
@@ -71,6 +71,14 @@ void x86_64_ipi_halt_realmode(struct cpu_info *);
 extern void hibernate_drop_to_real_mode(void);
 #endif /* HIBERNATE */
 
+#include "pctr.h"
+#if NPCTR > 0
+#include <machine/pctr.h>
+#define x86_64_ipi_reload_pctr pctr_reload
+#else
+#define x86_64_ipi_reload_pctr NULL
+#endif
+
 #ifdef MTRR
 void x86_64_ipi_reload_mtrr(struct cpu_info *);
 #else
@@ -83,7 +91,7 @@ void (*ipifunc[X86_NIPI])(struct cpu_info *) =
 	x86_64_ipi_nop,
 	NULL,
 	NULL,
-	NULL,
+	x86_64_ipi_reload_pctr,
 	x86_64_ipi_reload_mtrr,
 	x86_setperf_ipi,
 #ifdef DDB
diff --git a/sys/arch/amd64/amd64/pctr.c b/sys/arch/amd64/amd64/pctr.c
index 7484b1ffd04..22b78568b3b 100644
--- a/sys/arch/amd64/amd64/pctr.c
+++ b/sys/arch/amd64/amd64/pctr.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: pctr.c,v 1.8 2017/09/08 05:36:51 deraadt Exp $	*/
+/*	$OpenBSD: pctr.c,v 1.9 2019/03/25 18:48:12 guenther Exp $	*/
 
 /*
  * Copyright (c) 2007 Mike Belopuhov
@@ -29,8 +29,10 @@
 #include <sys/errno.h>
 #include <sys/fcntl.h>
 #include <sys/ioccom.h>
+#include <sys/mutex.h>
 #include <sys/systm.h>
 
+#include <machine/intr.h>
 #include <machine/pctr.h>
 #include <machine/cpu.h>
 #include <machine/specialreg.h>
@@ -48,8 +50,12 @@ int			pctr_isamd;
 int			pctr_isintel;
 uint32_t		pctr_intel_cap;
 
+struct mutex		pctr_conf_lock = MUTEX_INITIALIZER(IPL_HIGH);
+uint32_t		pctr_fn[PCTR_NUM];
+
 static void		pctrrd(struct pctrst *);
-static int		pctrsel(int, u_int32_t, u_int32_t);
+static int		pctrsel(int, uint32_t, uint32_t);
+static void		pctr_enable(struct cpu_info *);
 
 static void
 pctrrd(struct pctrst *st)
@@ -80,7 +86,11 @@ pctrattach(int num)
 		pctr_isintel = (strcmp(cpu_vendor, "GenuineIntel") == 0);
 		CPUID(0xa, pctr_intel_cap, dummy, dummy, dummy);
 	}
+}
 
+void
+pctr_enable(struct cpu_info *ci)
+{
 	if (usepctr) {
 		/* Enable RDTSC and RDPMC instructions from user-level. */
 		__asm volatile("movq %%cr4,%%rax\n"
@@ -114,9 +124,9 @@ pctrclose(dev_t dev, int oflags, int devtype, struct proc *p)
 }
 
 static int
-pctrsel(int fflag, u_int32_t cmd, u_int32_t fn)
+pctrsel(int fflag, uint32_t cmd, uint32_t fn)
 {
-	int msrsel, msrval;
+	int msrsel, msrval, changed;
 
 	cmd -= PCIOCS0;
 	if (pctr_isamd) {
@@ -136,9 +146,22 @@ pctrsel(int fflag, u_int32_t cmd, u_int32_t fn)
 	if (fn & 0x380000)
 		return (EINVAL);
 
-	wrmsr(msrval, 0);
-	wrmsr(msrsel, fn);
-	wrmsr(msrval, 0);
+	if (fn != 0)
+		pctr_enable(curcpu());
+
+	mtx_enter(&pctr_conf_lock);
+	changed = fn != pctr_fn[cmd];
+	if (changed) {
+		pctr_fn[cmd] = fn;
+		wrmsr(msrval, 0);
+		wrmsr(msrsel, fn);
+		wrmsr(msrval, 0);
+	}
+	mtx_leave(&pctr_conf_lock);
+#ifdef MULTIPROCESSOR
+	if (changed)
+		x86_broadcast_ipi(X86_IPI_PCTR);
+#endif
 
 	return (0);
 }
@@ -168,3 +191,46 @@ pctrioctl(dev_t dev, u_long cmd, caddr_t data, int fflag, struct proc *p)
 		return (EINVAL);
 	}
 }
+
+void
+pctr_reload(struct cpu_info *ci)
+{
+	int num, i, msrsel, msrval, anyset;
+	uint32_t fn;
+
+	if (pctr_isamd) {
+		num = PCTR_AMD_NUM;
+		msrsel = MSR_K7_EVNTSEL0;
+		msrval = MSR_K7_PERFCTR0;
+	} else {
+		num = PCTR_INTEL_NUM;
+		msrsel = MSR_EVNTSEL0;
+		msrval = MSR_PERFCTR0;
+	}
+
+	anyset = 0;
+	mtx_enter(&pctr_conf_lock);
+	for (i = 0; i < num; i++) {
+		/* only update the ones that don't match */
+		/* XXX generation numbers for zeroing? */
+		fn = rdmsr(msrsel + i);
+		if (fn != pctr_fn[i]) {
+			wrmsr(msrval + i, 0);
+			wrmsr(msrsel + i, pctr_fn[i]);
+			wrmsr(msrval + i, 0);
+		}
+		if (fn)
+			anyset = 1;
+	}
+	mtx_leave(&pctr_conf_lock);
+
+	if (! anyset)
+		pctr_enable(curcpu());
+}
+
+void
+pctr_resume(struct cpu_info *ci)
+{
+	if (usepctr)
+		pctr_reload(ci);
+}
diff --git a/sys/arch/amd64/include/intrdefs.h b/sys/arch/amd64/include/intrdefs.h
index cecb7061123..93c30ebd59c 100644
--- a/sys/arch/amd64/include/intrdefs.h
+++ b/sys/arch/amd64/include/intrdefs.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: intrdefs.h,v 1.19 2019/03/25 18:45:27 guenther Exp $	*/
+/*	$OpenBSD: intrdefs.h,v 1.20 2019/03/25 18:48:12 guenther Exp $	*/
 /*	$NetBSD: intrdefs.h,v 1.2 2003/05/04 22:01:56 fvdl Exp $	*/
 
 #ifndef _AMD64_INTRDEFS_H
@@ -76,7 +76,7 @@
 
 #define X86_IPI_HALT			0x00000001
 #define X86_IPI_NOP			0x00000002
-#define X86_IPI_TLB			0x00000010
+#define X86_IPI_PCTR			0x00000010
 #define X86_IPI_MTRR			0x00000020
 #define X86_IPI_SETPERF			0x00000040
 #define X86_IPI_DDB			0x00000080
diff --git a/sys/arch/amd64/include/pctr.h b/sys/arch/amd64/include/pctr.h
index 65defd7d32d..5a0a14b40dc 100644
--- a/sys/arch/amd64/include/pctr.h
+++ b/sys/arch/amd64/include/pctr.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: pctr.h,v 1.5 2014/03/29 18:09:28 guenther Exp $	*/
+/*	$OpenBSD: pctr.h,v 1.6 2019/03/25 18:48:12 guenther Exp $	*/
 
 /*
  * Pentium performance counter driver for OpenBSD.
@@ -69,25 +69,12 @@ struct pctrst {
 
 #ifdef _KERNEL
 
-#define rdmsr(msr)						\
-({								\
-	u_int32_t hi, lo;					\
-	__asm volatile("rdmsr"					\
-	     : "=d" (hi), "=a" (lo) : "c" (msr));		\
-	((u_int64_t)hi << 32) | (u_int64_t) lo;			\
-})
-
-#define wrmsr(msr, v)						\
-({								\
-	__asm volatile("wrmsr" :				\
-	    : "a" ((u_int64_t)v & 0xffffffff),			\
-	      "d" ((u_int64_t)v >> 32), "c" (msr));		\
-})
-
 void	pctrattach(int);
 int	pctropen(dev_t, int, int, struct proc *);
 int	pctrclose(dev_t, int, int, struct proc *);
 int	pctrioctl(dev_t, u_long, caddr_t, int, struct proc *);
+void	pctr_reload(struct cpu_info *);
+void	pctr_resume(struct cpu_info *);
 
 #endif /* _KERNEL */
 #endif /* ! _MACHINE_PCTR_H_ */