KERN_CPTIME2: set ENODEV if the CPU is offline.

This lets userspace distinguish between idle CPUs and those that are
not schedulable because hw.smt=0.

A subsequent commit probably needs to add documentation for this
to sysctl.2 (and perhaps elsewhere) after the dust settles.

Also included here are changes to systat(1) and top(1) that account
for the ENODEV case and adjust behavior accordingly:

 - systat(1)'s cpu view prints placeholder marks ('-') instead of
   percentages for each state if the given CPU is offline.

 - systat(1)'s vmstat view checks for offline CPUs when computing the
   machine state total and excludes them, so the CPU usage graph
   only represents the states for online CPUs.

 - top(1) does not draw CPU rows for offline CPUs when the view is
   redrawn.  If CPUs "go offline", percentages for each state are
   replaced by placeholder marks ('-'); the view will need to be
   redrawn to remove these rows.  If CPUs "go online" the view will
   need to be redrawn to show these new CPUs.  In "combined CPU" mode,
   the count and the state totals only represent online CPUs.

Ports using KERN_CPTIME2 will need to be updated.  The changes
described above to make systat(1) and top(1) aware of the ENODEV
case *and* gracefully handle a changing HW_NCPUONLINE while the
application is running are not necessarily appropriate for each
and every port.

The changes described above are so extensive in part to demonstrate
one way a program *might* be made robust to changing CPU availability.
In particular, changing hw.smt after boot is an extremely rare event,
and this needs to be weighed when updating ports.

The logic needed to account for the KERN_CPTIME2 ENODEV case is
very roughly:

	if (sysctl(...) == -1) {
		if (errno != ENODEV) {
			/* Actual error occurred. */
		} else {
			/* CPU is offline. */
		}
	} else {
		/* CPU is online and CPU states were set by sysctl(2). */
	}

Prompted by deraadt@.  Basic idea for ENODEV from kettenis@.  Discussed at
length with kettenis@.  Additional testing by tb@.

No complaints from hackers@ after a week.

ok kettenis@, "I think you should commit [now]" deraadt@

1 files changed, 26 insertions, 7 deletions

diff --git a/usr.bin/systat/vmstat.c b/usr.bin/systat/vmstat.c
index 53922d90196..b59e43941fc 100644
--- a/usr.bin/systat/vmstat.c
+++ b/usr.bin/systat/vmstat.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: vmstat.c,v 1.86 2018/06/22 14:22:06 krw Exp $	*/
+/*	$OpenBSD: vmstat.c,v 1.87 2018/09/26 17:23:13 cheloha Exp $	*/
 /*	$NetBSD: vmstat.c,v 1.5 1996/05/10 23:16:40 thorpej Exp $	*/
 
 /*-
@@ -98,6 +98,7 @@ static	time_t t;
 static	double etime;
 static	float hertz;
 static	int nintr;
+static	int ncpu;
 static	long *intrloc;
 static	char **intrname;
 static	int ipktsrow;
@@ -200,6 +201,12 @@ initvmstat(void)
 			return (-1);
 	}
 
+	mib[0] = CTL_HW;
+	mib[1] = HW_NCPU;
+	size = sizeof(ncpu);
+	if (sysctl(mib, 2, &ncpu, &size, NULL, 0) < 0)
+		return (-1);
+
 	allocinfo(&s);
 	allocinfo(&s1);
 	allocinfo(&s2);
@@ -592,11 +599,12 @@ putfloat(double f, int l, int c, int w, int d, int nz)
 static void
 getinfo(struct Info *si)
 {
-	static int cp_time_mib[] = { CTL_KERN, KERN_CPTIME };
+	static int cp_time2_mib[3] = { CTL_KERN, KERN_CPTIME2, 0 };
 	static int nchstats_mib[2] = { CTL_KERN, KERN_NCHSTATS };
 	static int uvmexp_mib[2] = { CTL_VM, VM_UVMEXP };
 	static int vmtotal_mib[2] = { CTL_VM, VM_METER };
-	int mib[4], i;
+	int mib[4], cpu, i;
+	long cpu_time[CPUSTATES];
 	size_t size;
 
 	dkreadstats();
@@ -612,10 +620,21 @@ getinfo(struct Info *si)
 		}
 	}
 
-	size = sizeof(si->time);
-	if (sysctl(cp_time_mib, 2, &si->time, &size, NULL, 0) < 0) {
-		error("Can't get KERN_CPTIME: %s\n", strerror(errno));
-		memset(&si->time, 0, sizeof(si->time));
+	memset(&si->time, 0, sizeof(si->time));
+	for (cpu = 0; cpu < ncpu; cpu++) {
+		cp_time2_mib[2] = cpu;
+		size = sizeof(cpu_time);
+		if (sysctl(cp_time2_mib, 3, &cpu_time, &size, NULL, 0) < 0) {
+			if (errno != ENODEV) {
+				error("Can't get KERN_CPTIME2: %s\n",
+				    strerror(errno));
+				memset(&si->time, 0, sizeof(si->time));
+				break;
+			}
+			continue;	/* ignore offline CPUs */
+		}
+		for (i = 0; i < nitems(si->time); i++)
+			si->time[i] += cpu_time[i];
 	}
 
 	size = sizeof(si->nchstats);