On amd64 the bootup and resume paths for SP/BP/SP were not performing

msr/pat, clock-startup, firmware upgrade, fpu initialization, and cpu
soft-state initialization in the same order.  That kind of chaos is
crazy and fragile. Improve the acpi resume call sequence and unify order
of the operations.

7 files changed, 80 insertions, 86 deletions

diff --git a/sys/arch/amd64/amd64/acpi_machdep.c b/sys/arch/amd64/amd64/acpi_machdep.c
index e3ae0e3b5d4..ed133adf1ac 100644
--- a/sys/arch/amd64/amd64/acpi_machdep.c
+++ b/sys/arch/amd64/amd64/acpi_machdep.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: acpi_machdep.c,v 1.87 2019/08/19 21:22:26 deraadt Exp $	*/
+/*	$OpenBSD: acpi_machdep.c,v 1.88 2019/08/27 22:39:51 deraadt Exp $	*/
 /*
  * Copyright (c) 2005 Thorsten Lockert <tholo@sigmasoft.com>
  *
@@ -375,37 +375,6 @@ acpi_sleep_clocks(struct acpi_softc *sc, int state)
 }
 
 /*
- * We repair the interrupt hardware so that any events which occur
- * will cause the least number of unexpected side effects.  We re-start
- * the clocks early because we will soon run AML whigh might do DELAY.
- */ 
-void
-acpi_resume_clocks(struct acpi_softc *sc)
-{
-	cpu_init_msrs(&cpu_info_primary);
-
-#if NISA > 0
-	i8259_default_setup();
-#endif
-	intr_calculatemasks(curcpu());
-
-#if NIOAPIC > 0
-	ioapic_enable();
-#endif
-
-#if NLAPIC > 0
-	lapic_enable();
-	if (initclock_func == lapic_initclocks)
-		lapic_startclock();
-	lapic_set_lvt();
-#endif
-
-	i8254_startclock();
-	if (initclock_func == i8254_initclocks)
-		rtcstart();		/* in i8254 mode, rtc is profclock */
-}
-
-/*
  * This function may not have local variables due to a bug between
  * acpi_savecpu() and the resume path.
  */
@@ -478,13 +447,42 @@ acpi_sleep_cpu(struct acpi_softc *sc, int state)
 	return (0);
 }
 
+/*
+ * First repair the interrupt hardware so that any events which occur
+ * will cause the least number of unexpected side effects.  We re-start
+ * the clocks early because we will soon run AML whigh might do DELAY.
+ * Then PM, and then further system/CPU work for the BSP cpu.
+ */ 
 void
-acpi_resume_cpu(struct acpi_softc *sc)
+acpi_resume_cpu(struct acpi_softc *sc, int state)
 {
-	fpuinit(&cpu_info_primary);
+	cpu_init_msrs(&cpu_info_primary);
+
+#if NISA > 0
+	i8259_default_setup();
+#endif
+	intr_calculatemasks(curcpu());
+
+#if NIOAPIC > 0
+	ioapic_enable();
+#endif
+
+#if NLAPIC > 0
+	lapic_enable();
+	if (initclock_func == lapic_initclocks)
+		lapic_startclock();
+	lapic_set_lvt();
+#endif
+
+	i8254_startclock();
+	if (initclock_func == i8254_initclocks)
+		rtcstart();		/* in i8254 mode, rtc is profclock */
+
+	acpi_resume_pm(sc, state);
 
-	cpu_init(&cpu_info_primary);
 	cpu_ucode_apply(&cpu_info_primary);
+	fpuinit(&cpu_info_primary);
+	cpu_init(&cpu_info_primary);
 
 	/* Re-initialise memory range handling on BSP */
 	if (mem_range_softc.mr_op != NULL)
diff --git a/sys/arch/amd64/amd64/cpu.c b/sys/arch/amd64/amd64/cpu.c
index b1078f77baf..c25f96ab1d2 100644
--- a/sys/arch/amd64/amd64/cpu.c
+++ b/sys/arch/amd64/amd64/cpu.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: cpu.c,v 1.139 2019/08/09 15:20:04 pirofti Exp $	*/
+/*	$OpenBSD: cpu.c,v 1.140 2019/08/27 22:39:51 deraadt Exp $	*/
 /* $NetBSD: cpu.c,v 1.1 2003/04/26 18:39:26 fvdl Exp $ */
 
 /*-
@@ -637,6 +637,7 @@ cpu_attach(struct device *parent, struct device *self, void *aux)
 		mem_range_attach();
 #endif /* MTRR */
 		cpu_init(ci);
+		/* XXX SP fpuinit(ci) is done earlier */
 		cpu_init_mwait(sc);
 		break;
 
@@ -644,14 +645,10 @@ cpu_attach(struct device *parent, struct device *self, void *aux)
 		printf("apid %d (boot processor)\n", caa->cpu_apicid);
 		ci->ci_flags |= CPUF_PRESENT | CPUF_BSP | CPUF_PRIMARY;
 		cpu_intr_init(ci);
-#ifndef SMALL_KERNEL
-		cpu_ucode_apply(ci);
-#endif
 		identifycpu(ci);
 #ifdef MTRR
 		mem_range_attach();
 #endif /* MTRR */
-		cpu_init(ci);
 
 #if NLAPIC > 0
 		/*
@@ -660,6 +657,9 @@ cpu_attach(struct device *parent, struct device *self, void *aux)
 		lapic_enable();
 		lapic_calibrate_timer(ci);
 #endif
+		cpu_init(ci);
+		/* XXX BP fpuinit(ci) is done earlier */
+
 #if NIOAPIC > 0
 		ioapic_bsp_id = caa->cpu_apicid;
 #endif
diff --git a/sys/arch/amd64/amd64/machdep.c b/sys/arch/amd64/amd64/machdep.c
index eed79f4b32a..e14bfe64e76 100644
--- a/sys/arch/amd64/amd64/machdep.c
+++ b/sys/arch/amd64/amd64/machdep.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: machdep.c,v 1.257 2019/05/12 22:23:38 guenther Exp $	*/
+/*	$OpenBSD: machdep.c,v 1.258 2019/08/27 22:39:51 deraadt Exp $	*/
 /*	$NetBSD: machdep.c,v 1.3 2003/05/07 22:58:18 fvdl Exp $	*/
 
 /*-
@@ -1719,6 +1719,10 @@ init_x86_64(paddr_t first_avail)
 	cpu_init_idt();
 
 	intr_default_setup();
+
+#ifndef SMALL_KERNEL
+	cpu_ucode_apply(&cpu_info_primary);
+#endif
 	fpuinit(&cpu_info_primary);
 
 	softintr_init();
diff --git a/sys/arch/arm64/arm64/acpi_machdep.c b/sys/arch/arm64/arm64/acpi_machdep.c
index 3aa24a97e4b..5343fe30934 100644
--- a/sys/arch/arm64/arm64/acpi_machdep.c
+++ b/sys/arch/arm64/arm64/acpi_machdep.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: acpi_machdep.c,v 1.2 2019/01/19 20:45:06 tedu Exp $	*/
+/*	$OpenBSD: acpi_machdep.c,v 1.3 2019/08/27 22:39:53 deraadt Exp $	*/
 /*
  * Copyright (c) 2018 Mark Kettenis
  *
@@ -160,11 +160,6 @@ acpi_sleep_clocks(struct acpi_softc *sc, int state)
 {
 }
 
-void
-acpi_resume_clocks(struct acpi_softc *sc)
-{
-}
-
 int
 acpi_sleep_cpu(struct acpi_softc *sc, int state)
 {
@@ -172,7 +167,7 @@ acpi_sleep_cpu(struct acpi_softc *sc, int state)
 }
 
 void
-acpi_resume_cpu(struct acpi_softc *sc)
+acpi_resume_cpu(struct acpi_softc *sc, int state)
 {
 }
 
diff --git a/sys/arch/i386/i386/acpi_machdep.c b/sys/arch/i386/i386/acpi_machdep.c
index bdc97fa8288..2fd866d968c 100644
--- a/sys/arch/i386/i386/acpi_machdep.c
+++ b/sys/arch/i386/i386/acpi_machdep.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: acpi_machdep.c,v 1.70 2018/08/23 14:47:52 jsg Exp $	*/
+/*	$OpenBSD: acpi_machdep.c,v 1.71 2019/08/27 22:39:53 deraadt Exp $	*/
 /*
  * Copyright (c) 2005 Thorsten Lockert <tholo@sigmasoft.com>
  *
@@ -338,35 +338,6 @@ acpi_sleep_clocks(struct acpi_softc *sc, int state)
 }
 
 /*
- * Start the clocks early because AML will be executed next
- * which might do DELAY.
- */ 
-void
-acpi_resume_clocks(struct acpi_softc *sc)
-{
-#if NISA > 0
-	isa_defaultirq();
-#endif
-	intr_calculatemasks();
-
-#if NIOAPIC > 0
-	ioapic_enable();
-#endif
-
-#if NLAPIC > 0
-	lapic_tpr = save_lapic_tpr;
-	lapic_enable();
-	if (initclock_func == lapic_initclocks)
-		lapic_startclock();
-	lapic_set_lvt();
-#endif
-
-	i8254_startclock();
-	if (initclock_func == i8254_initclocks)
-		rtcstart();		/* in i8254 mode, rtc is profclock */
-}
- 
-/*
  * This function may not have local variables due to a bug between
  * acpi_savecpu() and the resume path.
  */
@@ -439,9 +410,38 @@ acpi_sleep_cpu(struct acpi_softc *sc, int state)
 	return (0);
 }
 
+/*
+ * First repair the interrupt hardware so that any events which occur
+ * will cause the least number of unexpected side effects.  We re-start
+ * the clocks early because we will soon run AML whigh might do DELAY.
+ * Then PM, and then further system/CPU work for the BSP cpu.
+ */ 
 void
-acpi_resume_cpu(struct acpi_softc *sc)
+acpi_resume_cpu(struct acpi_softc *sc, int state)
 {
+#if NISA > 0
+	isa_defaultirq();
+#endif
+	intr_calculatemasks();
+
+#if NIOAPIC > 0
+	ioapic_enable();
+#endif
+
+#if NLAPIC > 0
+	lapic_tpr = save_lapic_tpr;
+	lapic_enable();
+	if (initclock_func == lapic_initclocks)
+		lapic_startclock();
+	lapic_set_lvt();
+#endif
+
+	i8254_startclock();
+	if (initclock_func == i8254_initclocks)
+		rtcstart();		/* in i8254 mode, rtc is profclock */
+
+	acpi_resume_pm(sc, state);
+
 	npxinit(&cpu_info_primary);
 
 	cpu_init(&cpu_info_primary);
diff --git a/sys/dev/acpi/acpi.c b/sys/dev/acpi/acpi.c
index f6259ae7391..620c5fab27e 100644
--- a/sys/dev/acpi/acpi.c
+++ b/sys/dev/acpi/acpi.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: acpi.c,v 1.371 2019/07/02 21:17:24 jcs Exp $ */
+/* $OpenBSD: acpi.c,v 1.372 2019/08/27 22:39:53 deraadt Exp $ */
 /*
  * Copyright (c) 2005 Thorsten Lockert <tholo@sigmasoft.com>
  * Copyright (c) 2005 Jordan Hargrave <jordan@openbsd.org>
@@ -2686,9 +2686,7 @@ acpi_sleep_state(struct acpi_softc *sc, int sleepmode)
 	}
 #endif /* HIBERNATE */
 
-	acpi_resume_clocks(sc);		/* AML may need clocks */
-	acpi_resume_pm(sc, state);
-	acpi_resume_cpu(sc);
+	acpi_resume_cpu(sc, state);
 
 fail_pts:
 	config_suspend_all(DVACT_RESUME);
diff --git a/sys/dev/acpi/acpivar.h b/sys/dev/acpi/acpivar.h
index 74ca71ea593..0fe5aeb9659 100644
--- a/sys/dev/acpi/acpivar.h
+++ b/sys/dev/acpi/acpivar.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: acpivar.h,v 1.102 2019/07/02 21:17:24 jcs Exp $	*/
+/*	$OpenBSD: acpivar.h,v 1.103 2019/08/27 22:39:53 deraadt Exp $	*/
 /*
  * Copyright (c) 2005 Thorsten Lockert <tholo@sigmasoft.com>
  *
@@ -331,8 +331,7 @@ int	 acpi_sleep_cpu(struct acpi_softc *, int);
 void	 acpi_sleep_mp(void);
 void	 acpi_sleep_pm(struct acpi_softc *, int);
 void	 acpi_resume_pm(struct acpi_softc *, int);
-void	 acpi_resume_clocks(struct acpi_softc *);
-void	 acpi_resume_cpu(struct acpi_softc *);
+void	 acpi_resume_cpu(struct acpi_softc *, int);
 void	 acpi_resume_mp(void);
 void	 acpi_sleep_walk(struct acpi_softc *, int);