Fix kernel profiling on MP systems by using per-CPU buffer. Previously

various CPUs were iterating over the same global buffer at the same
time to modify it and never ended.

This diff includes some ideas submited by Thor Simon to NetBSD via miod@.

ok mikeb@, haesbaert@

17 files changed, 154 insertions, 65 deletions

diff --git a/sys/arch/alpha/include/cpu.h b/sys/arch/alpha/include/cpu.h
index 689df5d258a..523dcfbcdea 100644
--- a/sys/arch/alpha/include/cpu.h
+++ b/sys/arch/alpha/include/cpu.h
@@ -1,4 +1,4 @@
-/* $OpenBSD: cpu.h,v 1.46 2012/12/02 07:03:30 guenther Exp $ */
+/* $OpenBSD: cpu.h,v 1.47 2013/02/11 17:05:25 mpi Exp $ */
 /* $NetBSD: cpu.h,v 1.45 2000/08/21 02:03:12 thorpej Exp $ */
 
 /*-
@@ -203,6 +203,9 @@ struct cpu_info {
 	u_long ci_ipis;			/* interprocessor interrupts pending */
 #endif
 	u_int32_t ci_randseed;
+#ifdef GPROF
+	struct gmonparam *ci_gmon;
+#endif
 };
 
 #define	CPUF_PRIMARY	0x01		/* CPU is primary CPU */
diff --git a/sys/arch/amd64/include/cpu.h b/sys/arch/amd64/include/cpu.h
index 8c86a9e1f55..9baaf42dc8c 100644
--- a/sys/arch/amd64/include/cpu.h
+++ b/sys/arch/amd64/include/cpu.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: cpu.h,v 1.76 2012/12/02 07:03:31 guenther Exp $	*/
+/*	$OpenBSD: cpu.h,v 1.77 2013/02/11 17:05:25 mpi Exp $	*/
 /*	$NetBSD: cpu.h,v 1.1 2003/04/26 18:39:39 fvdl Exp $	*/
 
 /*-
@@ -128,6 +128,9 @@ struct cpu_info {
 
 	struct ksensordev	ci_sensordev;
 	struct ksensor		ci_sensor;
+#ifdef GPROF
+	struct gmonparam	*ci_gmon;
+#endif
 };
 
 #define CPUF_BSP	0x0001		/* CPU is the original BSP */
diff --git a/sys/arch/arm/include/cpu.h b/sys/arch/arm/include/cpu.h
index 155a2a2a57e..c82ce63009d 100644
--- a/sys/arch/arm/include/cpu.h
+++ b/sys/arch/arm/include/cpu.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: cpu.h,v 1.31 2012/12/02 07:03:31 guenther Exp $	*/
+/*	$OpenBSD: cpu.h,v 1.32 2013/02/11 17:05:25 mpi Exp $	*/
 /*	$NetBSD: cpu.h,v 1.34 2003/06/23 11:01:08 martin Exp $	*/
 
 /*
@@ -198,6 +198,9 @@ struct cpu_info {
 
 	uint32_t ci_cpl;
 	uint32_t ci_ipending;
+#ifdef GPROF
+	struct gmonparam *ci_gmon;
+#endif
 };
 
 #ifndef MULTIPROCESSOR
diff --git a/sys/arch/hppa/include/cpu.h b/sys/arch/hppa/include/cpu.h
index c74038cd926..203ec4e3675 100644
--- a/sys/arch/hppa/include/cpu.h
+++ b/sys/arch/hppa/include/cpu.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: cpu.h,v 1.81 2012/12/02 07:03:31 guenther Exp $	*/
+/*	$OpenBSD: cpu.h,v 1.82 2013/02/11 17:05:25 mpi Exp $	*/
 
 /*
  * Copyright (c) 2000-2004 Michael Shalayeff
@@ -103,6 +103,9 @@ struct cpu_info {
 #ifdef DIAGNOSTIC
 	int		ci_mutex_level;
 #endif
+#ifdef GPROF
+	struct gmonparam *ci_gmon;
+#endif
 } __attribute__((__aligned__(64)));
 
 #define		CPUF_RUNNING	0x0001		/* CPU is running. */
diff --git a/sys/arch/hppa64/include/cpu.h b/sys/arch/hppa64/include/cpu.h
index 66ff94f697c..211a3ded9ab 100644
--- a/sys/arch/hppa64/include/cpu.h
+++ b/sys/arch/hppa64/include/cpu.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: cpu.h,v 1.28 2012/12/02 07:03:31 guenther Exp $	*/
+/*	$OpenBSD: cpu.h,v 1.29 2013/02/11 17:05:25 mpi Exp $	*/
 
 /*
  * Copyright (c) 2005 Michael Shalayeff
@@ -109,6 +109,9 @@ struct cpu_info {
 #ifdef DIAGNOSTIC
 	int		ci_mutex_level;
 #endif
+#ifdef GPROF
+	struct gmonparam *ci_gmon;
+#endif
 };
 
 struct cpu_info *curcpu(void);
diff --git a/sys/arch/i386/include/cpu.h b/sys/arch/i386/include/cpu.h
index a822caa3fc5..d68a669ac4e 100644
--- a/sys/arch/i386/include/cpu.h
+++ b/sys/arch/i386/include/cpu.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: cpu.h,v 1.125 2012/12/02 07:03:31 guenther Exp $	*/
+/*	$OpenBSD: cpu.h,v 1.126 2013/02/11 17:05:25 mpi Exp $	*/
 /*	$NetBSD: cpu.h,v 1.35 1996/05/05 19:29:26 christos Exp $	*/
 
 /*-
@@ -148,6 +148,9 @@ struct cpu_info {
 
 	struct ksensordev	ci_sensordev;
 	struct ksensor		ci_sensor;
+#ifdef GPROF
+	struct gmonparam	*ci_gmon;
+#endif
 };
 
 /*
diff --git a/sys/arch/m68k/include/cpu.h b/sys/arch/m68k/include/cpu.h
index 4a3df3f5514..723e39c1693 100644
--- a/sys/arch/m68k/include/cpu.h
+++ b/sys/arch/m68k/include/cpu.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: cpu.h,v 1.26 2012/12/02 07:03:31 guenther Exp $	*/
+/*	$OpenBSD: cpu.h,v 1.27 2013/02/11 17:05:25 mpi Exp $	*/
 /*	$NetBSD: cpu.h,v 1.3 1997/02/02 06:56:57 thorpej Exp $	*/
 
 /*
@@ -59,6 +59,9 @@ struct cpu_info {
 #ifdef DIAGNOSTIC
 	int	ci_mutex_level;
 #endif
+#ifdef GPROF
+	struct gmonparam *ci_gmon;
+#endif
 };
 
 extern struct cpu_info cpu_info_store;
diff --git a/sys/arch/m88k/include/cpu.h b/sys/arch/m88k/include/cpu.h
index 8bda7b55b9d..4da10bede57 100644
--- a/sys/arch/m88k/include/cpu.h
+++ b/sys/arch/m88k/include/cpu.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: cpu.h,v 1.56 2013/01/05 11:20:56 miod Exp $ */
+/*	$OpenBSD: cpu.h,v 1.57 2013/02/11 17:05:25 mpi Exp $ */
 /*
  * Copyright (c) 1996 Nivas Madhur
  * Copyright (c) 1992, 1993
@@ -172,6 +172,9 @@ struct cpu_info {
 #ifdef DIAGNOSTIC
 	int	ci_mutex_level;
 #endif
+#ifdef GPROF
+	struct gmonparam *ci_gmon;
+#endif
 };
 
 extern cpuid_t master_cpu;
diff --git a/sys/arch/mips64/include/cpu.h b/sys/arch/mips64/include/cpu.h
index daa8d49dd63..1bb6fabc04b 100644
--- a/sys/arch/mips64/include/cpu.h
+++ b/sys/arch/mips64/include/cpu.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: cpu.h,v 1.91 2012/12/02 07:03:31 guenther Exp $	*/
+/*	$OpenBSD: cpu.h,v 1.92 2013/02/11 17:05:25 mpi Exp $	*/
 
 /*-
  * Copyright (c) 1992, 1993
@@ -198,6 +198,9 @@ struct cpu_info {
 #ifdef DIAGNOSTIC
 	int	ci_mutex_level;
 #endif
+#ifdef GPROF
+	struct gmonparam *ci_gmon;
+#endif
 };
 
 #define	CPUF_PRIMARY	0x01		/* CPU is primary CPU */
diff --git a/sys/arch/powerpc/include/cpu.h b/sys/arch/powerpc/include/cpu.h
index d4237733cd8..7715195bde4 100644
--- a/sys/arch/powerpc/include/cpu.h
+++ b/sys/arch/powerpc/include/cpu.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: cpu.h,v 1.48 2012/12/08 12:49:00 mpi Exp $	*/
+/*	$OpenBSD: cpu.h,v 1.49 2013/02/11 17:05:25 mpi Exp $	*/
 /*	$NetBSD: cpu.h,v 1.1 1996/09/30 16:34:21 ws Exp $	*/
 
 /*
@@ -85,6 +85,9 @@ struct cpu_info {
 #ifdef DIAGNOSTIC
 	int	ci_mutex_level;
 #endif
+#ifdef GPROF
+	struct gmonparam *ci_gmon;
+#endif
 };
 
 static __inline struct cpu_info *
diff --git a/sys/arch/sh/include/cpu.h b/sys/arch/sh/include/cpu.h
index 021ab7203ff..ce6a69da2b6 100644
--- a/sys/arch/sh/include/cpu.h
+++ b/sys/arch/sh/include/cpu.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: cpu.h,v 1.22 2012/12/02 07:03:31 guenther Exp $	*/
+/*	$OpenBSD: cpu.h,v 1.23 2013/02/11 17:05:25 mpi Exp $	*/
 /*	$NetBSD: cpu.h,v 1.41 2006/01/21 04:24:12 uwe Exp $	*/
 
 /*-
@@ -65,6 +65,9 @@ struct cpu_info {
 #ifdef DIAGNOSTIC
 	int	ci_mutex_level;
 #endif
+#ifdef GPROF
+	struct gmonparam *ci_gmon;
+#endif
 };
 
 extern struct cpu_info cpu_info_store;
diff --git a/sys/arch/sparc64/include/cpu.h b/sys/arch/sparc64/include/cpu.h
index 0ca59053ace..b9e6115098a 100644
--- a/sys/arch/sparc64/include/cpu.h
+++ b/sys/arch/sparc64/include/cpu.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: cpu.h,v 1.79 2012/12/02 07:03:31 guenther Exp $	*/
+/*	$OpenBSD: cpu.h,v 1.80 2013/02/11 17:05:25 mpi Exp $	*/
 /*	$NetBSD: cpu.h,v 1.28 2001/06/14 22:56:58 thorpej Exp $ */
 
 /*
@@ -156,6 +156,9 @@ struct cpu_info {
 #ifdef DIAGNOSTIC
 	int	ci_mutex_level;
 #endif
+#ifdef GPROF
+	struct gmonparam *ci_gmon;
+#endif
 };
 
 #define CPUF_RUNNING	0x0001		/* CPU is running */
diff --git a/sys/arch/vax/include/cpu.h b/sys/arch/vax/include/cpu.h
index a9e076ab2ea..f8f6ba281c2 100644
--- a/sys/arch/vax/include/cpu.h
+++ b/sys/arch/vax/include/cpu.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: cpu.h,v 1.42 2012/12/05 23:20:15 deraadt Exp $	*/
+/*	$OpenBSD: cpu.h,v 1.43 2013/02/11 17:05:25 mpi Exp $	*/
 /*	$NetBSD: cpu.h,v 1.41 1999/10/21 20:01:36 ragge Exp $	*/
 
 /*
@@ -55,6 +55,9 @@ struct cpu_info {
 #ifdef DIAGNOSTIC
 	int	ci_mutex_level;
 #endif
+#ifdef GPROF
+	struct gmonparam *ci_gmon;
+#endif
 };
 
 extern struct cpu_info cpu_info_store;
diff --git a/sys/kern/kern_clock.c b/sys/kern/kern_clock.c
index 26ad86213c0..57220881f03 100644
--- a/sys/kern/kern_clock.c
+++ b/sys/kern/kern_clock.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: kern_clock.c,v 1.76 2012/11/05 19:39:34 miod Exp $	*/
+/*	$OpenBSD: kern_clock.c,v 1.77 2013/02/11 17:05:25 mpi Exp $	*/
 /*	$NetBSD: kern_clock.c,v 1.34 1996/06/09 04:51:03 briggs Exp $	*/
 
 /*-
@@ -421,8 +421,8 @@ statclock(struct clockframe *frame)
 		/*
 		 * Kernel statistics are just like addupc_intr, only easier.
 		 */
-		g = &_gmonparam;
-		if (g->state == GMON_PROF_ON) {
+		g = ci->ci_gmon;
+		if (g != NULL && g->state == GMON_PROF_ON) {
 			i = CLKF_PC(frame) - g->lowpc;
 			if (i < g->textsize) {
 				i /= HISTFRACTION * sizeof(*g->kcount);
diff --git a/sys/kern/subr_prof.c b/sys/kern/subr_prof.c
index 04e5003b1f6..a7eff46dc2f 100644
--- a/sys/kern/subr_prof.c
+++ b/sys/kern/subr_prof.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: subr_prof.c,v 1.21 2012/08/02 03:18:48 guenther Exp $	*/
+/*	$OpenBSD: subr_prof.c,v 1.22 2013/02/11 17:05:25 mpi Exp $	*/
 /*	$NetBSD: subr_prof.c,v 1.12 1996/04/22 01:38:50 christos Exp $	*/
 
 /*-
@@ -49,49 +49,75 @@
 #include <uvm/uvm_extern.h>
 
 /*
- * Froms is actually a bunch of unsigned shorts indexing tos
+ * Protect CPUs from executing profiling while they are not yet in a
+ * sane state.
  */
-struct gmonparam _gmonparam = { GMON_PROF_OFF };
+int gmoninit = 0;
 
 extern char etext[];
 
+#define ROUNDDOWN(x,y)	(((x)/(y))*(y))
+#define ROUNDUP(x,y)	((((x)+(y)-1)/(y))*(y))
 
 void
 kmstartup(void)
 {
+	CPU_INFO_ITERATOR cii;
+	struct cpu_info *ci;
+	struct gmonparam *p;
+	u_long lowpc, highpc, textsize;
+	u_long kcountsize, fromssize, tossize;
+	long tolimit;
 	char *cp;
-	struct gmonparam *p = &_gmonparam;
 	int size;
 
 	/*
 	 * Round lowpc and highpc to multiples of the density we're using
 	 * so the rest of the scaling (here and in gprof) stays in ints.
 	 */
-	p->lowpc = ROUNDDOWN(KERNBASE, HISTFRACTION * sizeof(HISTCOUNTER));
-	p->highpc = ROUNDUP((u_long)etext, HISTFRACTION * sizeof(HISTCOUNTER));
-	p->textsize = p->highpc - p->lowpc;
+	lowpc = ROUNDDOWN(KERNBASE, HISTFRACTION * sizeof(HISTCOUNTER));
+	highpc = ROUNDUP((u_long)etext, HISTFRACTION * sizeof(HISTCOUNTER));
+	textsize = highpc - lowpc;
 	printf("Profiling kernel, textsize=%ld [%lx..%lx]\n",
-	       p->textsize, p->lowpc, p->highpc);
-	p->kcountsize = p->textsize / HISTFRACTION;
-	p->hashfraction = HASHFRACTION;
-	p->fromssize = p->textsize / HASHFRACTION;
-	p->tolimit = p->textsize * ARCDENSITY / 100;
-	if (p->tolimit < MINARCS)
-		p->tolimit = MINARCS;
-	else if (p->tolimit > MAXARCS)
-		p->tolimit = MAXARCS;
-	p->tossize = p->tolimit * sizeof(struct tostruct);
-	size = p->kcountsize + p->fromssize + p->tossize;
-	cp = (char *)uvm_km_zalloc(kernel_map, round_page(size));
-	if (cp == 0) {
-		printf("No memory for profiling.\n");
-		return;
+	    textsize, lowpc, highpc);
+	kcountsize = textsize / HISTFRACTION;
+	fromssize = textsize / HASHFRACTION;
+	tolimit = textsize * ARCDENSITY / 100;
+	if (tolimit < MINARCS)
+		tolimit = MINARCS;
+	else if (tolimit > MAXARCS)
+		tolimit = MAXARCS;
+	tossize = tolimit * sizeof(struct tostruct);
+	size = sizeof(*p) + kcountsize + fromssize + tossize;
+
+	/* Allocate and initialize one profiling buffer per CPU. */
+	CPU_INFO_FOREACH(cii, ci) {
+		cp = km_alloc(round_page(size), &kv_any, &kp_zero, &kd_nowait);
+		if (cp == NULL) {
+			printf("No memory for profiling.\n");
+			return;
+		}
+
+		p = (struct gmonparam *)cp;
+		cp += sizeof(*p);
+		p->tos = (struct tostruct *)cp;
+		cp += tossize;
+		p->kcount = (u_short *)cp;
+		cp += kcountsize;
+		p->froms = (u_short *)cp;
+
+		p->state = GMON_PROF_OFF;
+		p->lowpc = lowpc;
+		p->highpc = highpc;
+		p->textsize = textsize;
+		p->hashfraction = HASHFRACTION;
+		p->kcountsize = kcountsize;
+		p->fromssize = fromssize;
+		p->tolimit = tolimit;
+		p->tossize = tossize;
+
+		ci->ci_gmon = p;
 	}
-	p->tos = (struct tostruct *)cp;
-	cp += p->tossize;
-	p->kcount = (u_short *)cp;
-	cp += p->kcountsize;
-	p->froms = (u_short *)cp;
 }
 
 /*
@@ -101,14 +127,32 @@ int
 sysctl_doprof(int *name, u_int namelen, void *oldp, size_t *oldlenp, void *newp,
     size_t newlen)
 {
-	struct gmonparam *gp = &_gmonparam;
-	int error;
+	CPU_INFO_ITERATOR cii;
+	struct cpu_info *ci;
+	struct gmonparam *gp = NULL;
+	int error, cpuid, op;
 
-	/* all sysctl names at this level are terminal */
-	if (namelen != 1)
+	/* all sysctl names at this level are name and field */
+	if (namelen != 2)
 		return (ENOTDIR);		/* overloaded */
 
-	switch (name[0]) {
+	op = name[0];
+	cpuid = name[1];
+
+	CPU_INFO_FOREACH(cii, ci) {
+		if (cpuid == CPU_INFO_UNIT(ci)) {
+			gp = ci->ci_gmon;
+			break;
+		}
+	}
+
+	if (gp == NULL)
+		return (EOPNOTSUPP);
+
+	/* Assume that if we're here it is safe to execute profiling. */
+	gmoninit = 1;
+
+	switch (op) {
 	case GPROF_STATE:
 		error = sysctl_int(oldp, oldlenp, newp, newlen, &gp->state);
 		if (error)
diff --git a/sys/lib/libkern/mcount.c b/sys/lib/libkern/mcount.c
index b6421c3cc3d..bcb5c1286bd 100644
--- a/sys/lib/libkern/mcount.c
+++ b/sys/lib/libkern/mcount.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: mcount.c,v 1.10 2013/01/16 23:38:14 dlg Exp $	*/
+/*	$OpenBSD: mcount.c,v 1.11 2013/02/11 17:05:25 mpi Exp $	*/
 /*	$NetBSD: mcount.c,v 1.3.6.1 1996/06/12 04:23:01 cgd Exp $	*/
 
 /*-
@@ -43,13 +43,10 @@
  * _mcount updates data structures that represent traversals of the
  * program's call graph edges.  frompc and selfpc are the return
  * address and function address that represents the given call graph edge.
- * 
- * Note: the original BSD code used the same variable (frompcindex) for
- * both frompcindex and frompc.  Any reasonable, modern compiler will
- * perform this optimization.
  */
 _MCOUNT_DECL(u_long frompc, u_long selfpc) __used;
-_MCOUNT_DECL(u_long frompc, u_long selfpc)	/* _mcount; may be static, inline, etc */
+/* _mcount; may be static, inline, etc */
+_MCOUNT_DECL(u_long frompc, u_long selfpc)
 {
 	u_short *frompcindex;
 	struct tostruct *top, *prevtop;
@@ -57,9 +54,21 @@ _MCOUNT_DECL(u_long frompc, u_long selfpc)	/* _mcount; may be static, inline, et
 	long toindex;
 #ifdef _KERNEL
 	int s;
-#endif
 
+	/*
+	 * Do not profile execution if memory for the current CPU
+	 * desciptor and profiling buffers has not yet been allocated
+	 * or if the CPU we are running on has not yet set its trap
+	 * handler.
+	 */
+	if (gmoninit == 0)
+		return;
+
+	if ((p = curcpu()->ci_gmon) == NULL)
+		return;
+#else
 	p = &_gmonparam;
+#endif
 	/*
 	 * check that we are profiling
 	 * and that we aren't recursively invoked.
@@ -156,7 +165,6 @@ _MCOUNT_DECL(u_long frompc, u_long selfpc)	/* _mcount; may be static, inline, et
 			*frompcindex = toindex;
 			goto done;
 		}
-		
 	}
 done:
 #ifdef _KERNEL
diff --git a/sys/sys/gmon.h b/sys/sys/gmon.h
index c817ce35977..0eaa36490d4 100644
--- a/sys/sys/gmon.h
+++ b/sys/sys/gmon.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: gmon.h,v 1.4 2003/06/02 23:28:21 millert Exp $	*/
+/*	$OpenBSD: gmon.h,v 1.5 2013/02/11 17:05:25 mpi Exp $	*/
 /*	$NetBSD: gmon.h,v 1.5 1996/04/09 20:55:30 cgd Exp $	*/
 
 /*-
@@ -115,12 +115,6 @@ struct rawarc {
 };
 
 /*
- * general rounding functions.
- */
-#define ROUNDDOWN(x,y)	(((x)/(y))*(y))
-#define ROUNDUP(x,y)	((((x)+(y)-1)/(y))*(y))
-
-/*
  * The profiling data structures are housed in this structure.
  */
 struct gmonparam {
@@ -137,15 +131,19 @@ struct gmonparam {
 	u_long		textsize;
 	u_long		hashfraction;
 };
+#ifdef _KERNEL
+extern int gmoninit;		/* Is the kernel ready for beeing profiled? */
+#else
 extern struct gmonparam _gmonparam;
+#endif
 
 /*
  * Possible states of profiling.
  */
-#define	GMON_PROF_ON	0
+#define	GMON_PROF_OFF	0
 #define	GMON_PROF_BUSY	1
 #define	GMON_PROF_ERROR	2
-#define	GMON_PROF_OFF	3
+#define	GMON_PROF_ON	3
 
 /*
  * Sysctl definitions for extracting profiling information from the kernel.