Reload mtrr state on all CPUs after updates. Seems to speed up X on MP

systems, at least with Intel graphics.

ok marco@, deraadt@

6 files changed, 92 insertions, 60 deletions

diff --git a/sys/arch/amd64/amd64/amd64_mem.c b/sys/arch/amd64/amd64/amd64_mem.c
index 7ee4bb35ed7..2ee77dee8c1 100644
--- a/sys/arch/amd64/amd64/amd64_mem.c
+++ b/sys/arch/amd64/amd64/amd64_mem.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: amd64_mem.c,v 1.1 2008/06/11 09:22:38 phessler Exp $ */
+/* $OpenBSD: amd64_mem.c,v 1.2 2009/11/29 17:11:30 kettenis Exp $ */
 /*-
  * Copyright (c) 1999 Michael Smith <msmith@freebsd.org>
  * All rights reserved.
@@ -34,6 +34,7 @@
 #include <sys/memrange.h>
 
 #include <machine/cpufunc.h>
+#include <machine/intr.h>
 #include <machine/specialreg.h>
 
 /*
@@ -62,36 +63,35 @@ char *mem_owner_bios = "BIOS";
 
 void	amd64_mrinit(struct mem_range_softc *sc);
 int	amd64_mrset(struct mem_range_softc *sc,
-			   struct mem_range_desc *mrd,
-			   int *arg);
+	    struct mem_range_desc *mrd, int *arg);
 void	amd64_mrAPinit(struct mem_range_softc *sc);
+void	amd64_mrreload(struct mem_range_softc *sc);
 
 struct mem_range_ops amd64_mrops = {
 	amd64_mrinit,
 	amd64_mrset,
-	amd64_mrAPinit
+	amd64_mrAPinit,
+	amd64_mrreload
 };
 
 /* XXX for AP startup hook */
 u_int64_t	mtrrcap, mtrrdef;
 
 struct mem_range_desc	*mem_range_match(struct mem_range_softc *sc,
-						      struct mem_range_desc *mrd);
+			     struct mem_range_desc *mrd);
 void			 amd64_mrfetch(struct mem_range_softc *sc);
 int			 amd64_mtrrtype(u_int64_t flags);
 int			 amd64_mrt2mtrr(u_int64_t flags, int oldval);
 int			 amd64_mtrr2mrt(int val);
 int			 amd64_mtrrconflict(u_int64_t flag1, u_int64_t flag2);
 void			 amd64_mrstore(struct mem_range_softc *sc);
-void			 amd64_mrstoreone(void *arg);
+void			 amd64_mrstoreone(struct mem_range_softc *sc);
 struct mem_range_desc	*amd64_mtrrfixsearch(struct mem_range_softc *sc,
-						    u_int64_t addr);
+			     u_int64_t addr);
 int			 amd64_mrsetlow(struct mem_range_softc *sc,
-					      struct mem_range_desc *mrd,
-					      int *arg);
+			     struct mem_range_desc *mrd, int *arg);
 int			 amd64_mrsetvariable(struct mem_range_softc *sc,
-						   struct mem_range_desc *mrd,
-						   int *arg);
+			     struct mem_range_desc *mrd, int *arg);
 
 /* AMD64 MTRR type to memory range type conversion */
 int amd64_mtrrtomrt[] = {
@@ -264,7 +264,10 @@ void
 amd64_mrstore(struct mem_range_softc *sc)
 {
 	disable_intr();				/* disable interrupts */
-	amd64_mrstoreone((void *)sc);
+#ifdef MULTIPROCESSOR
+	x86_broadcast_ipi(X86_IPI_MTRR);
+#endif
+	amd64_mrstoreone(sc);
 	enable_intr();
 }
 
@@ -274,9 +277,8 @@ amd64_mrstore(struct mem_range_softc *sc)
  * just stuffing one entry; this is simpler (but slower, of course).
  */
 void
-amd64_mrstoreone(void *arg)
+amd64_mrstoreone(struct mem_range_softc *sc)
 {
-	struct mem_range_softc 	*sc = (struct mem_range_softc *)arg;
 	struct mem_range_desc	*mrd;
 	u_int64_t		 omsrv, msrv;
 	int			 i, j, msr;
@@ -595,7 +597,15 @@ amd64_mrinit(struct mem_range_softc *sc)
 void
 amd64_mrAPinit(struct mem_range_softc *sc)
 {
-	amd64_mrstoreone((void *)sc); /* set MTRRs to match BSP */
+	amd64_mrstoreone(sc); /* set MTRRs to match BSP */
 	wrmsr(MSR_MTRRdefType, mtrrdef); /* set MTRR behaviour to match BSP */
 }
 
+void
+amd64_mrreload(struct mem_range_softc *sc)
+{
+	disable_intr();				/* disable interrupts */
+	amd64_mrstoreone(sc); /* set MTRRs to match BSP */
+	enable_intr();
+}
+
diff --git a/sys/arch/amd64/amd64/ipifuncs.c b/sys/arch/amd64/amd64/ipifuncs.c
index b7eaaa6f81d..abe50ba74cb 100644
--- a/sys/arch/amd64/amd64/ipifuncs.c
+++ b/sys/arch/amd64/amd64/ipifuncs.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: ipifuncs.c,v 1.9 2008/06/26 05:42:09 ray Exp $	*/
+/*	$OpenBSD: ipifuncs.c,v 1.10 2009/11/29 17:11:30 kettenis Exp $	*/
 /*	$NetBSD: ipifuncs.c,v 1.1 2003/04/26 18:39:28 fvdl Exp $ */
 
 /*-
@@ -38,8 +38,11 @@
  * Interprocessor interrupt handlers.
  */
 
+#include "mtrr.h"
+
 #include <sys/param.h>
 #include <sys/device.h>
+#include <sys/memrange.h>
 #include <sys/systm.h>
 
 #include <uvm/uvm_extern.h>
@@ -57,17 +60,16 @@
 #include <ddb/db_output.h>
 #include <machine/db_machdep.h>
 
+void x86_64_ipi_nop(struct cpu_info *);
 void x86_64_ipi_halt(struct cpu_info *);
 
 void x86_64_ipi_synch_fpu(struct cpu_info *);
 void x86_64_ipi_flush_fpu(struct cpu_info *);
 
-void x86_64_ipi_nop(struct cpu_info *);
-
-#ifdef MTRR
-void x86_64_reload_mtrr(struct cpu_info *);
+#if NMTRR > 0
+void x86_64_ipi_reload_mtrr(struct cpu_info *);
 #else
-#define x86_64_reload_mtrr NULL
+#define x86_64_ipi_reload_mtrr NULL
 #endif
 
 void (*ipifunc[X86_NIPI])(struct cpu_info *) =
@@ -77,7 +79,7 @@ void (*ipifunc[X86_NIPI])(struct cpu_info *) =
 	x86_64_ipi_flush_fpu,
 	x86_64_ipi_synch_fpu,
 	NULL,
-	x86_64_reload_mtrr,
+	x86_64_ipi_reload_mtrr,
 	gdt_reload_cpu,
 #ifdef DDB
 	x86_ipi_db,
@@ -115,17 +117,11 @@ x86_64_ipi_synch_fpu(struct cpu_info *ci)
 	fpusave_cpu(ci, 1);
 }
 
-#ifdef MTRR
-
-/*
- * mtrr_reload_cpu() is a macro in mtrr.h which picks the appropriate
- * function to use..
- */
-
+#if NMTRR > 0
 void
-x86_64_reload_mtrr(struct cpu_info *ci)
+x86_64_ipi_reload_mtrr(struct cpu_info *ci)
 {
-	if (mtrr_funcs != NULL)
-		mtrr_reload_cpu(ci);
+	if (mem_range_softc.mr_op != NULL)
+		mem_range_softc.mr_op->reload(&mem_range_softc);
 }
 #endif
diff --git a/sys/arch/i386/i386/i686_mem.c b/sys/arch/i386/i386/i686_mem.c
index 0f746bd30e7..42bc25698e0 100644
--- a/sys/arch/i386/i386/i686_mem.c
+++ b/sys/arch/i386/i386/i686_mem.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: i686_mem.c,v 1.10 2007/09/07 15:00:19 art Exp $ */
+/* $OpenBSD: i686_mem.c,v 1.11 2009/11/29 17:11:30 kettenis Exp $ */
 /*-
  * Copyright (c) 1999 Michael Smith <msmith@freebsd.org>
  * All rights reserved.
@@ -34,6 +34,7 @@
 #include <sys/memrange.h>
 
 #include <machine/cpufunc.h>
+#include <machine/intr.h>
 #include <machine/specialreg.h>
 
 /*
@@ -62,36 +63,35 @@ char *mem_owner_bios = "BIOS";
 
 void	i686_mrinit(struct mem_range_softc *sc);
 int	i686_mrset(struct mem_range_softc *sc,
-			   struct mem_range_desc *mrd,
-			   int *arg);
+	    struct mem_range_desc *mrd, int *arg);
 void	i686_mrAPinit(struct mem_range_softc *sc);
+void	i686_mrreload(struct mem_range_softc *sc);
 
 struct mem_range_ops i686_mrops = {
 	i686_mrinit,
 	i686_mrset,
-	i686_mrAPinit
+	i686_mrAPinit,
+	i686_mrreload
 };
 
 /* XXX for AP startup hook */
 u_int64_t	mtrrcap, mtrrdef;
 
 struct mem_range_desc	*mem_range_match(struct mem_range_softc *sc,
-						      struct mem_range_desc *mrd);
+			     struct mem_range_desc *mrd);
 void			 i686_mrfetch(struct mem_range_softc *sc);
 int			 i686_mtrrtype(int flags);
 int			 i686_mrt2mtrr(int flags, int oldval);
 int			 i686_mtrr2mrt(int val);
 int			 i686_mtrrconflict(int flag1, int flag2);
 void			 i686_mrstore(struct mem_range_softc *sc);
-void			 i686_mrstoreone(void *arg);
+void			 i686_mrstoreone(struct mem_range_softc *sc);
 struct mem_range_desc	*i686_mtrrfixsearch(struct mem_range_softc *sc,
-						    u_int64_t addr);
+			     u_int64_t addr);
 int			 i686_mrsetlow(struct mem_range_softc *sc,
-					      struct mem_range_desc *mrd,
-					      int *arg);
+			     struct mem_range_desc *mrd, int *arg);
 int			 i686_mrsetvariable(struct mem_range_softc *sc,
-						   struct mem_range_desc *mrd,
-						   int *arg);
+			     struct mem_range_desc *mrd, int *arg);
 
 /* i686 MTRR type to memory range type conversion */
 int i686_mtrrtomrt[] = {
@@ -264,7 +264,10 @@ void
 i686_mrstore(struct mem_range_softc *sc)
 {
 	disable_intr();				/* disable interrupts */
-	i686_mrstoreone((void *)sc);
+#ifdef MULTIPROCESSOR
+	i386_broadcast_ipi(I386_IPI_MTRR);
+#endif
+	i686_mrstoreone(sc);
 	enable_intr();
 }
 
@@ -274,9 +277,8 @@ i686_mrstore(struct mem_range_softc *sc)
  * just stuffing one entry; this is simpler (but slower, of course).
  */
 void
-i686_mrstoreone(void *arg)
+i686_mrstoreone(struct mem_range_softc *sc)
 {
-	struct mem_range_softc 	*sc = (struct mem_range_softc *)arg;
 	struct mem_range_desc	*mrd;
 	u_int64_t		 omsrv, msrv;
 	int			 i, j, msr;
@@ -595,7 +597,14 @@ i686_mrinit(struct mem_range_softc *sc)
 void
 i686_mrAPinit(struct mem_range_softc *sc)
 {
-	i686_mrstoreone((void *)sc); /* set MTRRs to match BSP */
+	i686_mrstoreone(sc); /* set MTRRs to match BSP */
 	wrmsr(MSR_MTRRdefType, mtrrdef); /* set MTRR behaviour to match BSP */
 }
 
+void
+i686_mrreload(struct mem_range_softc *sc)
+{
+	disable_intr();				/* disable interrupts */
+	i686_mrstoreone(sc); /* set MTRRs to match BSP */
+	enable_intr();
+}
diff --git a/sys/arch/i386/i386/ipifuncs.c b/sys/arch/i386/i386/ipifuncs.c
index 6bf5012a107..aeba0d7df00 100644
--- a/sys/arch/i386/i386/ipifuncs.c
+++ b/sys/arch/i386/i386/ipifuncs.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: ipifuncs.c,v 1.13 2008/06/26 05:42:10 ray Exp $	*/
+/*	$OpenBSD: ipifuncs.c,v 1.14 2009/11/29 17:11:30 kettenis Exp $	*/
 /* $NetBSD: ipifuncs.c,v 1.1.2.3 2000/06/26 02:04:06 sommerfeld Exp $ */
 
 /*-
@@ -32,19 +32,22 @@
  * POSSIBILITY OF SUCH DAMAGE.
  */
 
-
 #include <sys/cdefs.h>			/* RCS ID & Copyright macro defns */
 
 /*
  * Interprocessor interrupt handlers.
  */
 
+#include "mtrr.h"
 #include "npx.h"
 
 #include <sys/param.h>
 #include <sys/device.h>
+#include <sys/memrange.h>
 #include <sys/systm.h>
 
+#include <uvm/uvm_extern.h>
+
 #include <machine/cpufunc.h>
 #include <machine/cpuvar.h>
 #include <machine/intr.h>
@@ -52,19 +55,22 @@
 #include <machine/i82093var.h>
 #include <machine/db_machdep.h>
 
-#include <uvm/uvm_extern.h>
-
+void i386_ipi_nop(struct cpu_info *);
 void i386_ipi_halt(struct cpu_info *);
 
 #if NNPX > 0
 void i386_ipi_synch_fpu(struct cpu_info *);
 void i386_ipi_flush_fpu(struct cpu_info *);
 #else
-#define i386_ipi_synch_fpu 0
-#define i386_ipi_flush_fpu 0
+#define i386_ipi_synch_fpu NULL
+#define i386_ipi_flush_fpu NULL
 #endif
 
-void i386_ipi_nop(struct cpu_info *);
+#if NMTRR > 0
+void i386_ipi_reload_mtrr(struct cpu_info *);
+#else
+#define i386_ipi_reload_mtrr 0
+#endif
 
 void (*ipifunc[I386_NIPI])(struct cpu_info *) =
 {
@@ -72,17 +78,16 @@ void (*ipifunc[I386_NIPI])(struct cpu_info *) =
 	i386_ipi_nop,
 	i386_ipi_flush_fpu,
 	i386_ipi_synch_fpu,
+	i386_ipi_reload_mtrr,
 #if 0
-	i386_reload_mtrr,
 	gdt_reload_cpu,
 #else
-	0,
-	0,
+	NULL,
 #endif
 #ifdef DDB
 	i386_ipi_db,
 #else
-	0,
+	NULL,
 #endif
 	i386_setperf_ipi,
 };
@@ -117,6 +122,15 @@ i386_ipi_synch_fpu(struct cpu_info *ci)
 }
 #endif
 
+#if NMTRR > 0
+void
+i386_ipi_reload_mtrr(struct cpu_info *ci)
+{
+	if (mem_range_softc.mr_op != NULL)
+		mem_range_softc.mr_op->reload(&mem_range_softc);
+}
+#endif
+
 void
 i386_spurious(void)
 {
diff --git a/sys/arch/i386/i386/k6_mem.c b/sys/arch/i386/i386/k6_mem.c
index af8e4e38d42..591ee2f4266 100644
--- a/sys/arch/i386/i386/k6_mem.c
+++ b/sys/arch/i386/i386/k6_mem.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: k6_mem.c,v 1.9 2009/08/15 00:34:44 jsg Exp $ */
+/* $OpenBSD: k6_mem.c,v 1.10 2009/11/29 17:11:30 kettenis Exp $ */
 /*-
  * Copyright (c) 1999 Brian Fundakowski Feldman
  * All rights reserved.
@@ -67,6 +67,7 @@ static __inline int k6_mrmake(struct mem_range_desc *, u_int32_t *);
 struct mem_range_ops k6_mrops = {
 	k6_mrinit,
 	k6_mrset,
+	NULL,
 	NULL
 };
 
diff --git a/sys/sys/memrange.h b/sys/sys/memrange.h
index fb5a273ff8a..f330d89a34f 100644
--- a/sys/sys/memrange.h
+++ b/sys/sys/memrange.h
@@ -1,4 +1,4 @@
-/* $OpenBSD: memrange.h,v 1.4 2002/10/14 21:01:01 matthieu Exp $ */
+/* $OpenBSD: memrange.h,v 1.5 2009/11/29 17:11:30 kettenis Exp $ */
 /*-
  * Copyright (c) 1999 Michael Smith <msmith@freebsd.org>
  * All rights reserved.
@@ -74,6 +74,7 @@ struct mem_range_ops
 	void	(*init)(struct mem_range_softc *sc);
 	int	(*set)(struct mem_range_softc *sc, struct mem_range_desc *mrd, int *arg);
     	void	(*initAP)(struct mem_range_softc *sc);
+	void	(*reload)(struct mem_range_softc *sc);
 };
 
 struct mem_range_softc 
@@ -90,6 +91,7 @@ __BEGIN_DECLS
 extern int mem_range_attr_get(struct mem_range_desc *mrd, int *arg);
 extern int mem_range_attr_set(struct mem_range_desc *mrd, int *arg);
 extern void mem_range_AP_init(void);
+extern void mem_range_reload(void);
 __END_DECLS
 #endif