Fix kernel profiling on MP systems by using per-CPU buffers and teach

kgmon(8) to deal with them, this time without public header changes.

Previously various CPUs were iterating over the same global buffer at
the same time to modify it and never ended.

This diff includes some ideas submited by Thor Simon to NetBSD via miod@.

ok deraadt@, mikeb@, haesbaert@

17 files changed, 150 insertions, 58 deletions

diff --git a/sys/arch/alpha/include/cpu.h b/sys/arch/alpha/include/cpu.h
index 6ce34749606..d43c510d202 100644
--- a/sys/arch/alpha/include/cpu.h
+++ b/sys/arch/alpha/include/cpu.h
@@ -1,4 +1,4 @@
-/* $OpenBSD: cpu.h,v 1.48 2013/02/12 08:06:22 mpi Exp $ */
+/* $OpenBSD: cpu.h,v 1.49 2013/03/12 09:37:16 mpi Exp $ */
 /* $NetBSD: cpu.h,v 1.45 2000/08/21 02:03:12 thorpej Exp $ */
 
 /*-
@@ -203,6 +203,9 @@ struct cpu_info {
 	u_long ci_ipis;			/* interprocessor interrupts pending */
 #endif
 	u_int32_t ci_randseed;
+#ifdef GPROF
+	struct gmonparam *ci_gmon;
+#endif
 };
 
 #define	CPUF_PRIMARY	0x01		/* CPU is primary CPU */
diff --git a/sys/arch/amd64/include/cpu.h b/sys/arch/amd64/include/cpu.h
index 61e9889d71f..c9977c895bb 100644
--- a/sys/arch/amd64/include/cpu.h
+++ b/sys/arch/amd64/include/cpu.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: cpu.h,v 1.78 2013/02/12 08:06:22 mpi Exp $	*/
+/*	$OpenBSD: cpu.h,v 1.79 2013/03/12 09:37:16 mpi Exp $	*/
 /*	$NetBSD: cpu.h,v 1.1 2003/04/26 18:39:39 fvdl Exp $	*/
 
 /*-
@@ -128,6 +128,9 @@ struct cpu_info {
 
 	struct ksensordev	ci_sensordev;
 	struct ksensor		ci_sensor;
+#ifdef GPROF
+	struct gmonparam	*ci_gmon;
+#endif
 };
 
 #define CPUF_BSP	0x0001		/* CPU is the original BSP */
diff --git a/sys/arch/arm/include/cpu.h b/sys/arch/arm/include/cpu.h
index 7e1ca74122c..5da7d8333bc 100644
--- a/sys/arch/arm/include/cpu.h
+++ b/sys/arch/arm/include/cpu.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: cpu.h,v 1.33 2013/02/12 08:06:22 mpi Exp $	*/
+/*	$OpenBSD: cpu.h,v 1.34 2013/03/12 09:37:16 mpi Exp $	*/
 /*	$NetBSD: cpu.h,v 1.34 2003/06/23 11:01:08 martin Exp $	*/
 
 /*
@@ -198,6 +198,9 @@ struct cpu_info {
 
 	uint32_t ci_cpl;
 	uint32_t ci_ipending;
+#ifdef GPROF
+	struct gmonparam *ci_gmon;
+#endif
 };
 
 #ifndef MULTIPROCESSOR
diff --git a/sys/arch/hppa/include/cpu.h b/sys/arch/hppa/include/cpu.h
index b477fc6b15a..536ae85f1fb 100644
--- a/sys/arch/hppa/include/cpu.h
+++ b/sys/arch/hppa/include/cpu.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: cpu.h,v 1.83 2013/02/12 08:06:22 mpi Exp $	*/
+/*	$OpenBSD: cpu.h,v 1.84 2013/03/12 09:37:16 mpi Exp $	*/
 
 /*
  * Copyright (c) 2000-2004 Michael Shalayeff
@@ -103,6 +103,9 @@ struct cpu_info {
 #ifdef DIAGNOSTIC
 	int		ci_mutex_level;
 #endif
+#ifdef GPROF
+	struct gmonparam *ci_gmon;
+#endif
 } __attribute__((__aligned__(64)));
 
 #define		CPUF_RUNNING	0x0001		/* CPU is running. */
diff --git a/sys/arch/hppa64/include/cpu.h b/sys/arch/hppa64/include/cpu.h
index 509a8a21b14..193a1bc7588 100644
--- a/sys/arch/hppa64/include/cpu.h
+++ b/sys/arch/hppa64/include/cpu.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: cpu.h,v 1.30 2013/02/12 08:06:22 mpi Exp $	*/
+/*	$OpenBSD: cpu.h,v 1.31 2013/03/12 09:37:16 mpi Exp $	*/
 
 /*
  * Copyright (c) 2005 Michael Shalayeff
@@ -109,6 +109,9 @@ struct cpu_info {
 #ifdef DIAGNOSTIC
 	int		ci_mutex_level;
 #endif
+#ifdef GPROF
+	struct gmonparam *ci_gmon;
+#endif
 };
 
 struct cpu_info *curcpu(void);
diff --git a/sys/arch/i386/include/cpu.h b/sys/arch/i386/include/cpu.h
index 2fe58c1d2a5..99a7448a763 100644
--- a/sys/arch/i386/include/cpu.h
+++ b/sys/arch/i386/include/cpu.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: cpu.h,v 1.127 2013/02/12 08:06:22 mpi Exp $	*/
+/*	$OpenBSD: cpu.h,v 1.128 2013/03/12 09:37:16 mpi Exp $	*/
 /*	$NetBSD: cpu.h,v 1.35 1996/05/05 19:29:26 christos Exp $	*/
 
 /*-
@@ -148,6 +148,9 @@ struct cpu_info {
 
 	struct ksensordev	ci_sensordev;
 	struct ksensor		ci_sensor;
+#ifdef GPROF
+	struct gmonparam	*ci_gmon;
+#endif
 };
 
 /*
diff --git a/sys/arch/m68k/include/cpu.h b/sys/arch/m68k/include/cpu.h
index bac5eedc662..02c99c399a7 100644
--- a/sys/arch/m68k/include/cpu.h
+++ b/sys/arch/m68k/include/cpu.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: cpu.h,v 1.28 2013/02/12 08:06:22 mpi Exp $	*/
+/*	$OpenBSD: cpu.h,v 1.29 2013/03/12 09:37:16 mpi Exp $	*/
 /*	$NetBSD: cpu.h,v 1.3 1997/02/02 06:56:57 thorpej Exp $	*/
 
 /*
@@ -59,6 +59,9 @@ struct cpu_info {
 #ifdef DIAGNOSTIC
 	int	ci_mutex_level;
 #endif
+#ifdef GPROF
+	struct gmonparam *ci_gmon;
+#endif
 };
 
 extern struct cpu_info cpu_info_store;
diff --git a/sys/arch/m88k/include/cpu.h b/sys/arch/m88k/include/cpu.h
index f5055a5463e..a9c7089d92d 100644
--- a/sys/arch/m88k/include/cpu.h
+++ b/sys/arch/m88k/include/cpu.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: cpu.h,v 1.58 2013/02/12 08:06:22 mpi Exp $ */
+/*	$OpenBSD: cpu.h,v 1.59 2013/03/12 09:37:16 mpi Exp $ */
 /*
  * Copyright (c) 1996 Nivas Madhur
  * Copyright (c) 1992, 1993
@@ -172,6 +172,9 @@ struct cpu_info {
 #ifdef DIAGNOSTIC
 	int	ci_mutex_level;
 #endif
+#ifdef GPROF
+	struct gmonparam *ci_gmon;
+#endif
 };
 
 extern cpuid_t master_cpu;
diff --git a/sys/arch/mips64/include/cpu.h b/sys/arch/mips64/include/cpu.h
index c7ba15de925..7c036339265 100644
--- a/sys/arch/mips64/include/cpu.h
+++ b/sys/arch/mips64/include/cpu.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: cpu.h,v 1.93 2013/02/12 08:06:22 mpi Exp $	*/
+/*	$OpenBSD: cpu.h,v 1.94 2013/03/12 09:37:16 mpi Exp $	*/
 
 /*-
  * Copyright (c) 1992, 1993
@@ -198,6 +198,9 @@ struct cpu_info {
 #ifdef DIAGNOSTIC
 	int	ci_mutex_level;
 #endif
+#ifdef GPROF
+	struct gmonparam *ci_gmon;
+#endif
 };
 
 #define	CPUF_PRIMARY	0x01		/* CPU is primary CPU */
diff --git a/sys/arch/powerpc/include/cpu.h b/sys/arch/powerpc/include/cpu.h
index a9561d6e820..3f47bdc7953 100644
--- a/sys/arch/powerpc/include/cpu.h
+++ b/sys/arch/powerpc/include/cpu.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: cpu.h,v 1.50 2013/02/12 08:06:22 mpi Exp $	*/
+/*	$OpenBSD: cpu.h,v 1.51 2013/03/12 09:37:16 mpi Exp $	*/
 /*	$NetBSD: cpu.h,v 1.1 1996/09/30 16:34:21 ws Exp $	*/
 
 /*
@@ -85,6 +85,9 @@ struct cpu_info {
 #ifdef DIAGNOSTIC
 	int	ci_mutex_level;
 #endif
+#ifdef GPROF
+	struct gmonparam *ci_gmon;
+#endif
 };
 
 static __inline struct cpu_info *
diff --git a/sys/arch/sh/include/cpu.h b/sys/arch/sh/include/cpu.h
index e313d701d22..31959429bd7 100644
--- a/sys/arch/sh/include/cpu.h
+++ b/sys/arch/sh/include/cpu.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: cpu.h,v 1.24 2013/02/12 08:06:22 mpi Exp $	*/
+/*	$OpenBSD: cpu.h,v 1.25 2013/03/12 09:37:16 mpi Exp $	*/
 /*	$NetBSD: cpu.h,v 1.41 2006/01/21 04:24:12 uwe Exp $	*/
 
 /*-
@@ -65,6 +65,9 @@ struct cpu_info {
 #ifdef DIAGNOSTIC
 	int	ci_mutex_level;
 #endif
+#ifdef GPROF
+	struct gmonparam *ci_gmon;
+#endif
 };
 
 extern struct cpu_info cpu_info_store;
diff --git a/sys/arch/sparc64/include/cpu.h b/sys/arch/sparc64/include/cpu.h
index 7b5e4fee0a0..e1bca2f7fa1 100644
--- a/sys/arch/sparc64/include/cpu.h
+++ b/sys/arch/sparc64/include/cpu.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: cpu.h,v 1.81 2013/02/12 08:06:22 mpi Exp $	*/
+/*	$OpenBSD: cpu.h,v 1.82 2013/03/12 09:37:16 mpi Exp $	*/
 /*	$NetBSD: cpu.h,v 1.28 2001/06/14 22:56:58 thorpej Exp $ */
 
 /*
@@ -156,6 +156,9 @@ struct cpu_info {
 #ifdef DIAGNOSTIC
 	int	ci_mutex_level;
 #endif
+#ifdef GPROF
+	struct gmonparam *ci_gmon;
+#endif
 };
 
 #define CPUF_RUNNING	0x0001		/* CPU is running */
diff --git a/sys/arch/vax/include/cpu.h b/sys/arch/vax/include/cpu.h
index 509556a3358..6284fd9e230 100644
--- a/sys/arch/vax/include/cpu.h
+++ b/sys/arch/vax/include/cpu.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: cpu.h,v 1.44 2013/02/12 08:06:22 mpi Exp $	*/
+/*	$OpenBSD: cpu.h,v 1.45 2013/03/12 09:37:16 mpi Exp $	*/
 /*	$NetBSD: cpu.h,v 1.41 1999/10/21 20:01:36 ragge Exp $	*/
 
 /*
@@ -55,6 +55,9 @@ struct cpu_info {
 #ifdef DIAGNOSTIC
 	int	ci_mutex_level;
 #endif
+#ifdef GPROF
+	struct gmonparam *ci_gmon;
+#endif
 };
 
 extern struct cpu_info cpu_info_store;
diff --git a/sys/kern/kern_clock.c b/sys/kern/kern_clock.c
index d7d51646c2c..7aec79d3e69 100644
--- a/sys/kern/kern_clock.c
+++ b/sys/kern/kern_clock.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: kern_clock.c,v 1.78 2013/02/12 08:06:22 mpi Exp $	*/
+/*	$OpenBSD: kern_clock.c,v 1.79 2013/03/12 09:37:16 mpi Exp $	*/
 /*	$NetBSD: kern_clock.c,v 1.34 1996/06/09 04:51:03 briggs Exp $	*/
 
 /*-
@@ -421,8 +421,8 @@ statclock(struct clockframe *frame)
 		/*
 		 * Kernel statistics are just like addupc_intr, only easier.
 		 */
-		g = &_gmonparam;
-		if (g->state == GMON_PROF_ON) {
+		g = ci->ci_gmon;
+		if (g != NULL && g->state == GMON_PROF_ON) {
 			i = CLKF_PC(frame) - g->lowpc;
 			if (i < g->textsize) {
 				i /= HISTFRACTION * sizeof(*g->kcount);
diff --git a/sys/kern/subr_prof.c b/sys/kern/subr_prof.c
index 3786cb11df8..5289ab7f8b4 100644
--- a/sys/kern/subr_prof.c
+++ b/sys/kern/subr_prof.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: subr_prof.c,v 1.23 2013/02/12 08:06:22 mpi Exp $	*/
+/*	$OpenBSD: subr_prof.c,v 1.24 2013/03/12 09:37:16 mpi Exp $	*/
 /*	$NetBSD: subr_prof.c,v 1.12 1996/04/22 01:38:50 christos Exp $	*/
 
 /*-
@@ -49,49 +49,72 @@
 #include <uvm/uvm_extern.h>
 
 /*
- * Froms is actually a bunch of unsigned shorts indexing tos
+ * Flag to prevent CPUs from executing the mcount() monitor function
+ * until we're sure they are in a sane state.
  */
-struct gmonparam _gmonparam = { GMON_PROF_OFF };
+int gmoninit = 0;
 
 extern char etext[];
 
-
 void
 kmstartup(void)
 {
+	CPU_INFO_ITERATOR cii;
+	struct cpu_info *ci;
+	struct gmonparam *p;
+	u_long lowpc, highpc, textsize;
+	u_long kcountsize, fromssize, tossize;
+	long tolimit;
 	char *cp;
-	struct gmonparam *p = &_gmonparam;
 	int size;
 
 	/*
 	 * Round lowpc and highpc to multiples of the density we're using
 	 * so the rest of the scaling (here and in gprof) stays in ints.
 	 */
-	p->lowpc = ROUNDDOWN(KERNBASE, HISTFRACTION * sizeof(HISTCOUNTER));
-	p->highpc = ROUNDUP((u_long)etext, HISTFRACTION * sizeof(HISTCOUNTER));
-	p->textsize = p->highpc - p->lowpc;
+	lowpc = ROUNDDOWN(KERNBASE, HISTFRACTION * sizeof(HISTCOUNTER));
+	highpc = ROUNDUP((u_long)etext, HISTFRACTION * sizeof(HISTCOUNTER));
+	textsize = highpc - lowpc;
 	printf("Profiling kernel, textsize=%ld [%lx..%lx]\n",
-	       p->textsize, p->lowpc, p->highpc);
-	p->kcountsize = p->textsize / HISTFRACTION;
-	p->hashfraction = HASHFRACTION;
-	p->fromssize = p->textsize / HASHFRACTION;
-	p->tolimit = p->textsize * ARCDENSITY / 100;
-	if (p->tolimit < MINARCS)
-		p->tolimit = MINARCS;
-	else if (p->tolimit > MAXARCS)
-		p->tolimit = MAXARCS;
-	p->tossize = p->tolimit * sizeof(struct tostruct);
-	size = p->kcountsize + p->fromssize + p->tossize;
-	cp = (char *)uvm_km_zalloc(kernel_map, round_page(size));
-	if (cp == 0) {
-		printf("No memory for profiling.\n");
-		return;
+	    textsize, lowpc, highpc);
+	kcountsize = textsize / HISTFRACTION;
+	fromssize = textsize / HASHFRACTION;
+	tolimit = textsize * ARCDENSITY / 100;
+	if (tolimit < MINARCS)
+		tolimit = MINARCS;
+	else if (tolimit > MAXARCS)
+		tolimit = MAXARCS;
+	tossize = tolimit * sizeof(struct tostruct);
+	size = sizeof(*p) + kcountsize + fromssize + tossize;
+
+	/* Allocate and initialize one profiling buffer per CPU. */
+	CPU_INFO_FOREACH(cii, ci) {
+		cp = km_alloc(round_page(size), &kv_any, &kp_zero, &kd_nowait);
+		if (cp == NULL) {
+			printf("No memory for profiling.\n");
+			return;
+		}
+
+		p = (struct gmonparam *)cp;
+		cp += sizeof(*p);
+		p->tos = (struct tostruct *)cp;
+		cp += tossize;
+		p->kcount = (u_short *)cp;
+		cp += kcountsize;
+		p->froms = (u_short *)cp;
+
+		p->state = GMON_PROF_OFF;
+		p->lowpc = lowpc;
+		p->highpc = highpc;
+		p->textsize = textsize;
+		p->hashfraction = HASHFRACTION;
+		p->kcountsize = kcountsize;
+		p->fromssize = fromssize;
+		p->tolimit = tolimit;
+		p->tossize = tossize;
+
+		ci->ci_gmon = p;
 	}
-	p->tos = (struct tostruct *)cp;
-	cp += p->tossize;
-	p->kcount = (u_short *)cp;
-	cp += p->kcountsize;
-	p->froms = (u_short *)cp;
 }
 
 /*
@@ -101,14 +124,32 @@ int
 sysctl_doprof(int *name, u_int namelen, void *oldp, size_t *oldlenp, void *newp,
     size_t newlen)
 {
-	struct gmonparam *gp = &_gmonparam;
-	int error;
+	CPU_INFO_ITERATOR cii;
+	struct cpu_info *ci;
+	struct gmonparam *gp = NULL;
+	int error, cpuid, op;
 
-	/* all sysctl names at this level are terminal */
-	if (namelen != 1)
+	/* all sysctl names at this level are name and field */
+	if (namelen != 2)
 		return (ENOTDIR);		/* overloaded */
 
-	switch (name[0]) {
+	op = name[0];
+	cpuid = name[1];
+
+	CPU_INFO_FOREACH(cii, ci) {
+		if (cpuid == CPU_INFO_UNIT(ci)) {
+			gp = ci->ci_gmon;
+			break;
+		}
+	}
+
+	if (gp == NULL)
+		return (EOPNOTSUPP);
+
+	/* Assume that if we're here it is safe to execute profiling. */
+	gmoninit = 1;
+
+	switch (op) {
 	case GPROF_STATE:
 		error = sysctl_int(oldp, oldlenp, newp, newlen, &gp->state);
 		if (error)
diff --git a/sys/lib/libkern/mcount.c b/sys/lib/libkern/mcount.c
index c66314cefe0..684ddadb9aa 100644
--- a/sys/lib/libkern/mcount.c
+++ b/sys/lib/libkern/mcount.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: mcount.c,v 1.12 2013/02/12 08:06:22 mpi Exp $	*/
+/*	$OpenBSD: mcount.c,v 1.13 2013/03/12 09:37:16 mpi Exp $	*/
 /*	$NetBSD: mcount.c,v 1.3.6.1 1996/06/12 04:23:01 cgd Exp $	*/
 
 /*-
@@ -43,13 +43,10 @@
  * _mcount updates data structures that represent traversals of the
  * program's call graph edges.  frompc and selfpc are the return
  * address and function address that represents the given call graph edge.
- * 
- * Note: the original BSD code used the same variable (frompcindex) for
- * both frompcindex and frompc.  Any reasonable, modern compiler will
- * perform this optimization.
  */
 _MCOUNT_DECL(u_long frompc, u_long selfpc) __used;
-_MCOUNT_DECL(u_long frompc, u_long selfpc)	/* _mcount; may be static, inline, etc */
+/* _mcount; may be static, inline, etc */
+_MCOUNT_DECL(u_long frompc, u_long selfpc)
 {
 	u_short *frompcindex;
 	struct tostruct *top, *prevtop;
@@ -57,9 +54,21 @@ _MCOUNT_DECL(u_long frompc, u_long selfpc)	/* _mcount; may be static, inline, et
 	long toindex;
 #ifdef _KERNEL
 	int s;
-#endif
 
+	/*
+	 * Do not profile execution if memory for the current CPU
+	 * desciptor and profiling buffers has not yet been allocated
+	 * or if the CPU we are running on has not yet set its trap
+	 * handler.
+	 */
+	if (gmoninit == 0)
+		return;
+
+	if ((p = curcpu()->ci_gmon) == NULL)
+		return;
+#else
 	p = &_gmonparam;
+#endif
 	/*
 	 * check that we are profiling
 	 * and that we aren't recursively invoked.
@@ -156,7 +165,6 @@ _MCOUNT_DECL(u_long frompc, u_long selfpc)	/* _mcount; may be static, inline, et
 			*frompcindex = toindex;
 			goto done;
 		}
-		
 	}
 done:
 #ifdef _KERNEL
diff --git a/sys/sys/gmon.h b/sys/sys/gmon.h
index 192b7c7ebbc..303d756a327 100644
--- a/sys/sys/gmon.h
+++ b/sys/sys/gmon.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: gmon.h,v 1.6 2013/02/12 08:06:22 mpi Exp $	*/
+/*	$OpenBSD: gmon.h,v 1.7 2013/03/12 09:37:16 mpi Exp $	*/
 /*	$NetBSD: gmon.h,v 1.5 1996/04/09 20:55:30 cgd Exp $	*/
 
 /*-
@@ -137,7 +137,11 @@ struct gmonparam {
 	u_long		textsize;
 	u_long		hashfraction;
 };
+#ifdef _KERNEL
+extern int gmoninit;		/* Is the kernel ready for beeing profiled? */
+#else
 extern struct gmonparam _gmonparam;
+#endif
 
 /*
  * Possible states of profiling.