Add new KERN_CPUSTATS sysctl(2) so we can identify offline CPUs.

Because of hw.smt we need a way to determine whether a given CPU is "online"
or "offline" from userspace.  KERN_CPTIME2 is an array, and so cannot be
cleanly extended for this purpose, so add a new sysctl(2) KERN_CPUSTATS
with an extensible struct.  At the moment it's just KERN_CPTIME2 with a
flags member, but it can grow as needed.

KERN_CPUSTATS appears to have been defined by BSDi long ago, but there are
few (if any) packages in the wild still using the symbol so breakage in ports
should be near zero.  No other system inherited the symbol from BSDi, either.

Then, use the new sysctl(2) in systat(1) and top(1):

  - systat(1) draws placeholder marks ('-') instead of percentages for
    offline CPUs in the cpu view.

  - systat(1) omits offline CPU ticks when drawing the "big bar" in
    the vmstat view.  The upshot is that the bar isn't half idle when
    half your logical CPUs are disabled.

  - top(1) does not draw lines for offline CPUs; if CPUs toggle on or
    offline in interactive mode we redraw the display to expand/reduce
    space for the new/missing CPUs.  This is consistent with what some
    top(1) implementations do on Linux.

  - top(1) omits offline CPUs from the totals when CPU totals are
    combined into a single line (the '-1' flag).

Originally prompted by deraadt@.  Discussed endlessly with deraadt@,
ketennis@, and sthen@.  Tested by jmc@ and jca@.  Earlier versions also
discussed with jca@.  Earlier versions tested by jmc@, tb@, and many
others.

docs ok jmc@, kernel bits ok ketennis@, everything ok sthen@,
"Is your stuff in yet?" deraadt@

5 files changed, 59 insertions, 6 deletions

diff --git a/sys/kern/kern_pledge.c b/sys/kern/kern_pledge.c
index 60b4b1ef7ac..8f495a3c245 100644
--- a/sys/kern/kern_pledge.c
+++ b/sys/kern/kern_pledge.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: kern_pledge.c,v 1.244 2018/11/06 07:49:38 otto Exp $	*/
+/*	$OpenBSD: kern_pledge.c,v 1.245 2018/11/17 23:10:08 cheloha Exp $	*/
 
 /*
  * Copyright (c) 2015 Nicholas Marriott <nicm@openbsd.org>
@@ -855,6 +855,9 @@ pledge_sysctl(struct proc *p, int miblen, int *mib, void *new)
 		if (miblen == 3 &&			/* kern.cptime2 */
 		    mib[0] == CTL_KERN && mib[1] == KERN_CPTIME2)
 			return (0);
+		if (miblen == 3 &&			/* kern.cpustats */
+		    mib[0] == CTL_KERN && mib[1] == KERN_CPUSTATS)
+			return (0);
 	}
 
 	if ((p->p_p->ps_pledge & PLEDGE_PS)) {
diff --git a/sys/kern/kern_sched.c b/sys/kern/kern_sched.c
index e11744184c3..552a2fd85c0 100644
--- a/sys/kern/kern_sched.c
+++ b/sys/kern/kern_sched.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: kern_sched.c,v 1.53 2018/10/05 18:56:57 cheloha Exp $	*/
+/*	$OpenBSD: kern_sched.c,v 1.54 2018/11/17 23:10:08 cheloha Exp $	*/
 /*
  * Copyright (c) 2007, 2008 Artur Grabowski <art@openbsd.org>
  *
@@ -832,6 +832,12 @@ sysctl_hwncpuonline(void)
 	return cpuset_cardinality(&sched_all_cpus);
 }
 
+int
+cpu_is_online(struct cpu_info *ci)
+{
+	return cpuset_isset(&sched_all_cpus, ci);
+}
+
 #ifdef __HAVE_CPU_TOPOLOGY
 
 #include <sys/sysctl.h>
diff --git a/sys/kern/kern_sysctl.c b/sys/kern/kern_sysctl.c
index 3d6fc62f7e7..31d89a5777f 100644
--- a/sys/kern/kern_sysctl.c
+++ b/sys/kern/kern_sysctl.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: kern_sysctl.c,v 1.350 2018/10/05 18:56:57 cheloha Exp $	*/
+/*	$OpenBSD: kern_sysctl.c,v 1.351 2018/11/17 23:10:08 cheloha Exp $	*/
 /*	$NetBSD: kern_sysctl.c,v 1.17 1996/05/20 17:49:05 mrg Exp $	*/
 
 /*-
@@ -142,6 +142,7 @@ int sysctl_cptime2(int *, u_int, void *, size_t *, void *, size_t);
 #if NAUDIO > 0
 int sysctl_audio(int *, u_int, void *, size_t *, void *, size_t);
 #endif
+int sysctl_cpustats(int *, u_int, void *, size_t *, void *, size_t);
 
 void fill_file(struct kinfo_file *, struct file *, struct filedesc *, int,
     struct vnode *, struct process *, struct proc *, struct socket *, int);
@@ -311,6 +312,7 @@ kern_sysctl(int *name, u_int namelen, void *oldp, size_t *oldlenp, void *newp,
 		case KERN_CPTIME2:
 		case KERN_FILE:
 		case KERN_AUDIO:
+		case KERN_CPUSTATS:
 			break;
 		default:
 			return (ENOTDIR);	/* overloaded */
@@ -669,6 +671,9 @@ kern_sysctl(int *name, u_int namelen, void *oldp, size_t *oldlenp, void *newp,
 		return (sysctl_audio(name + 1, namelen - 1, oldp, oldlenp,
 		    newp, newlen));
 #endif
+	case KERN_CPUSTATS:
+		return (sysctl_cpustats(name + 1, namelen - 1, oldp, oldlenp,
+		    newp, newlen));
 	default:
 		return (EOPNOTSUPP);
 	}
@@ -2402,3 +2407,32 @@ sysctl_audio(int *name, u_int namelen, void *oldp, size_t *oldlenp,
 	return (sysctl_int(oldp, oldlenp, newp, newlen, &audio_record_enable));
 }
 #endif
+
+int
+sysctl_cpustats(int *name, u_int namelen, void *oldp, size_t *oldlenp,
+    void *newp, size_t newlen)
+{
+	CPU_INFO_ITERATOR cii;
+	struct cpustats cs;
+	struct cpu_info *ci;
+	int found = 0;
+
+	if (namelen != 1)
+		return (ENOTDIR);
+
+	CPU_INFO_FOREACH(cii, ci) {
+		if (name[0] == CPU_INFO_UNIT(ci)) {
+			found = 1;
+			break;
+		}
+	}
+	if (!found)
+		return (ENOENT);
+
+	memcpy(&cs.cs_time, &ci->ci_schedstate.spc_cp_time, sizeof(cs.cs_time));
+	cs.cs_flags = 0;
+	if (cpu_is_online(ci))
+		cs.cs_flags |= CPUSTATS_ONLINE;
+
+	return (sysctl_rdstruct(oldp, oldlenp, newp, &cs, sizeof(cs)));
+}
diff --git a/sys/sys/sched.h b/sys/sys/sched.h
index 103f1feaad0..baddb0669cf 100644
--- a/sys/sys/sched.h
+++ b/sys/sys/sched.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: sched.h,v 1.49 2018/10/05 18:56:57 cheloha Exp $	*/
+/*	$OpenBSD: sched.h,v 1.50 2018/11/17 23:10:08 cheloha Exp $	*/
 /* $NetBSD: sched.h,v 1.2 1999/02/28 18:14:58 ross Exp $ */
 
 /*-
@@ -113,6 +113,13 @@ struct schedstate_percpu {
 	volatile u_int spc_spinning;	/* this cpu is currently spinning */
 };
 
+struct cpustats {
+	uint64_t	cs_time[CPUSTATES];	/* CPU state statistics */
+	uint64_t	cs_flags;		/* see below */
+};
+
+#define CPUSTATS_ONLINE		0x0001	/* CPU is schedulable */
+
 #ifdef	_KERNEL
 
 /* spc_flags */
@@ -161,6 +168,7 @@ void sched_stop_secondary_cpus(void);
 #endif
 
 #define cpu_is_idle(ci)	((ci)->ci_schedstate.spc_whichqs == 0)
+int	cpu_is_online(struct cpu_info *);
 
 void sched_init_runqueues(void);
 void setrunqueue(struct proc *);
diff --git a/sys/sys/sysctl.h b/sys/sys/sysctl.h
index 7544d630e92..1acf1cab1a2 100644
--- a/sys/sys/sysctl.h
+++ b/sys/sys/sysctl.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: sysctl.h,v 1.179 2018/07/12 01:23:38 cheloha Exp $	*/
+/*	$OpenBSD: sysctl.h,v 1.180 2018/11/17 23:10:08 cheloha Exp $	*/
 /*	$NetBSD: sysctl.h,v 1.16 1996/04/09 20:55:36 cgd Exp $	*/
 
 /*
@@ -185,7 +185,8 @@ struct ctlname {
 #define	KERN_CONSBUFSIZE	82	/* int: console message buffer size */
 #define	KERN_CONSBUF		83	/* console message buffer */
 #define	KERN_AUDIO		84	/* struct: audio properties */
-#define	KERN_MAXID		85	/* number of valid kern ids */
+#define	KERN_CPUSTATS		85	/* struct: cpu statistics */
+#define	KERN_MAXID		86	/* number of valid kern ids */
 
 #define	CTL_KERN_NAMES { \
 	{ 0, 0 }, \
@@ -273,6 +274,7 @@ struct ctlname {
 	{ "gap", 0 }, \
 	{ "gap", 0 }, \
 	{ "audio", CTLTYPE_STRUCT }, \
+	{ "cpustats", CTLTYPE_STRUCT }, \
 }
 
 /*