Finish the last missing piece for the i386 meltdown fix:

- handle protection fault on iret properly
- handle NMI
- actually enable U-K in pmap_switch()
from hshoexer@; input guenther@; OK mlarkin@ deraadt@

11 files changed, 302 insertions, 70 deletions

diff --git a/sys/arch/i386/i386/cpu.c b/sys/arch/i386/i386/cpu.c
index 4cb39e6308e..3d54c4719b3 100644
--- a/sys/arch/i386/i386/cpu.c
+++ b/sys/arch/i386/i386/cpu.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: cpu.c,v 1.92 2018/05/28 20:52:44 bluhm Exp $	*/
+/*	$OpenBSD: cpu.c,v 1.93 2018/06/22 13:21:14 bluhm Exp $	*/
 /* $NetBSD: cpu.c,v 1.1.2.7 2000/06/26 02:04:05 sommerfeld Exp $ */
 
 /*-
@@ -131,6 +131,8 @@ int     cpu_activate(struct device *, int);
 void	patinit(struct cpu_info *ci);
 void	cpu_idle_mwait_cycle(void);
 void	cpu_init_mwait(struct cpu_softc *);
+void	cpu_init_tss(struct i386tss *, void *, void *);
+void	cpu_update_nmi_cr3(vaddr_t);
 #if NVMM > 0
 void	cpu_init_vmm(struct cpu_info *ci);
 #endif /* NVMM > 0 */
@@ -250,6 +252,7 @@ cpu_attach(struct device *parent, struct device *self, void *aux)
 		ci = &cif->cif_cpu;
 #ifdef MULTIPROCESSOR
 		ci->ci_tss = &cif->cif_tss;
+		ci->ci_nmi_tss = &cif->cif_nmi_tss;
 		ci->ci_gdt = (void *)&cif->cif_gdt;
 		cpu_enter_pages(cif);
 		if (cpu_info[cpunum] != NULL)
@@ -558,6 +561,7 @@ cpu_enter_pages(struct cpu_info_full *cif)
 {
 	vaddr_t	va;
 	paddr_t pa;
+	extern void Xnmi(void);
 
 	/* The TSS + GDT need to be readable */
 	va = (vaddr_t)&cif->cif_tss;
@@ -573,15 +577,23 @@ cpu_enter_pages(struct cpu_info_full *cif)
 	DPRINTF("%s: entered t.stack page at va 0x%08x pa 0x%08x\n", __func__,
 	    (uint32_t)va, (uint32_t)pa);
 
-	cif->cif_tss.tss_ss0 = GSEL(GDATA_SEL, SEL_KPL);
+	/* Setup trampoline stack in TSS */
 	cif->cif_tss.tss_esp0 = va + sizeof(cif->cif_tramp_stack) - 16;
+	cif->cif_tss.tss_ss0 = GSEL(GDATA_SEL, SEL_KPL);
 	DPRINTF("%s: cif_tss.tss_esp0 = 0x%08x\n", __func__,
 	    (uint32_t)cif->cif_tss.tss_esp0);
 	cif->cif_cpu.ci_intr_esp = cif->cif_tss.tss_esp0 -
 	    sizeof(struct trampframe);
 
+	/* Setup NMI stack in NMI TSS */
+	va = (vaddr_t)&cif->cif_nmi_stack + sizeof(cif->cif_nmi_stack);
+	cpu_init_tss(&cif->cif_nmi_tss, (void *)va, Xnmi);
+	DPRINTF("%s: cif_nmi_tss.tss_esp0 = 0x%08x\n", __func__,
+	    (uint32_t)cif->cif_nmi_tss.tss_esp0);
+
 	/* empty iomap */
 	cif->cif_tss.tss_ioopt = sizeof(cif->cif_tss) << 16;
+	cif->cif_nmi_tss.tss_ioopt = sizeof(cif->cif_nmi_tss) << 16;
 }
 
 #ifdef MULTIPROCESSOR
@@ -865,3 +877,29 @@ cpu_init_mwait(struct cpu_softc *sc)
 		cpu_idle_cycle_fcn = &cpu_idle_mwait_cycle;
 }
 
+void
+cpu_init_tss(struct i386tss *tss, void *stack, void *func)
+{
+	memset(tss, 0, sizeof *tss);
+	tss->tss_esp0 = tss->tss_esp = (int)((char *)stack - 16);
+	tss->tss_ss0 = tss->tss_ss = GSEL(GDATA_SEL, SEL_KPL);
+	tss->tss_cs = GSEL(GCODE_SEL, SEL_KPL);
+	tss->tss_ds = tss->tss_es = tss->tss_ss = GSEL(GDATA_SEL, SEL_KPL);
+	tss->tss_fs = GSEL(GCPU_SEL, SEL_KPL);
+	tss->tss_gs = GSEL(GNULL_SEL, SEL_KPL);
+	tss->tss_ldt = GSEL(GNULL_SEL, SEL_KPL);
+	tss->tss_cr3 = pmap_kernel()->pm_pdirpa;
+	/* PSL_I not set -> no IRQs after task switch */
+	tss->tss_eflags = PSL_MBO;
+	tss->tss_eip = (int)func;
+}
+
+void
+cpu_update_nmi_cr3(vaddr_t cr3)
+{
+	CPU_INFO_ITERATOR cii;
+	struct cpu_info *ci;
+
+	CPU_INFO_FOREACH(cii, ci)
+		ci->ci_nmi_tss->tss_cr3 = cr3;
+}
diff --git a/sys/arch/i386/i386/gdt.c b/sys/arch/i386/i386/gdt.c
index 095019655a2..9482aa4cb5f 100644
--- a/sys/arch/i386/i386/gdt.c
+++ b/sys/arch/i386/i386/gdt.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: gdt.c,v 1.42 2018/05/28 20:52:44 bluhm Exp $	*/
+/*	$OpenBSD: gdt.c,v 1.43 2018/06/22 13:21:14 bluhm Exp $	*/
 /*	$NetBSD: gdt.c,v 1.28 2002/12/14 09:38:50 junyoung Exp $	*/
 
 /*-
@@ -115,6 +115,8 @@ gdt_init_cpu(struct cpu_info *ci)
 
 	setsegment(&ci->ci_gdt[GTSS_SEL].sd, ci->ci_tss,
 	    sizeof(*ci->ci_tss)-1, SDT_SYS386TSS, SEL_KPL, 0, 0);
+	setsegment(&ci->ci_gdt[GNMITSS_SEL].sd, ci->ci_nmi_tss,
+	    sizeof(*ci->ci_nmi_tss)-1, SDT_SYS386TSS, SEL_KPL, 0, 0);
 
 	setregion(&region, ci->ci_gdt, GDT_SIZE - 1);
 	lgdt(&region);
diff --git a/sys/arch/i386/i386/genassym.cf b/sys/arch/i386/i386/genassym.cf
index 5e0806de7d5..2b6872f35d1 100644
--- a/sys/arch/i386/i386/genassym.cf
+++ b/sys/arch/i386/i386/genassym.cf
@@ -1,4 +1,4 @@
-#	$OpenBSD: genassym.cf,v 1.46 2018/04/11 15:44:08 bluhm Exp $
+#	$OpenBSD: genassym.cf,v 1.47 2018/06/22 13:21:14 bluhm Exp $
 #
 # Copyright (c) 1982, 1990 The Regents of the University of California.
 # All rights reserved.
@@ -220,4 +220,19 @@ member pm_pdirpa_intel
 
 struct i386tss
 member	tss_esp0
-
+member	tss_ss
+member	tss_esp
+member	tss_eflags
+member	tss_cs
+member	tss_eip
+member	tss_eax
+member	tss_ecx
+member	tss_edx
+member	tss_ebx
+member	tss_ebp
+member	tss_esi
+member	tss_edi
+member	tss_ds
+member	tss_es
+member	tss_fs
+member	tss_gs
diff --git a/sys/arch/i386/i386/locore.s b/sys/arch/i386/i386/locore.s
index 6f3c2278203..44fd3f867d8 100644
--- a/sys/arch/i386/i386/locore.s
+++ b/sys/arch/i386/i386/locore.s
@@ -1,4 +1,4 @@
-/*	$OpenBSD: locore.s,v 1.188 2018/06/15 17:58:41 bluhm Exp $	*/
+/*	$OpenBSD: locore.s,v 1.189 2018/06/22 13:21:14 bluhm Exp $	*/
 /*	$NetBSD: locore.s,v 1.145 1996/05/03 19:41:19 christos Exp $	*/
 
 /*-
@@ -103,7 +103,7 @@
 #ifdef VM86
 #define SAVE_VM86	\
 	testl	$PSL_VM,TRF_EFLAGS(%ebp)	; \
-	jz	98f				; \
+	jz	102f				; \
 	movl	TRF_VM86_ES(%ebp),%eax		; \
 	movl	%eax,IRF_VM86_ES(%esp)		; \
 	movl	TRF_VM86_DS(%ebp),%eax		; \
@@ -112,11 +112,11 @@
 	movl	%eax,IRF_VM86_FS(%esp)		; \
 	movl	TRF_VM86_GS(%ebp),%eax		; \
 	movl	%eax,IRF_VM86_GS(%esp)		; \
-98:	;
+	102:
 
 #define RESTORE_VM86	\
 	testl	$PSL_VM,TRF_EFLAGS(%ebp)	; \
-	jz	99f				; \
+	jz	101f				; \
 	movl	TRF_VM86_ES(%esp),%eax		; \
 	movl	%eax,TRF_VM86_ES(%ebp)		; \
 	movl	TRF_VM86_DS(%esp),%eax		; \
@@ -125,7 +125,7 @@
 	movl	%eax,TRF_VM86_FS(%ebp)		; \
 	movl	TRF_VM86_GS(%esp),%eax		; \
 	movl	%eax,TRF_VM86_GS(%ebp)		; \
-99:	;
+	101:
 
 #else
 
@@ -137,6 +137,32 @@
 /*
  * These are used on interrupt or trap entry or exit.
  */
+#define INTR_COPY_FROM_TRAMP_STACK	\
+	movl	TRF_SS(%ebp),%eax	; \
+	movl	%eax,IRF_SS(%esp)	; \
+	movl	TRF_ESP(%ebp),%eax	; \
+	movl	%eax,IRF_ESP(%esp)	; \
+	movl	TRF_EFLAGS(%ebp),%eax	; \
+	movl	%eax,IRF_EFLAGS(%esp)	; \
+	movl	TRF_CS(%ebp),%eax	; \
+	movl	%eax,IRF_CS(%esp)	; \
+	movl	TRF_EIP(%ebp),%eax	; \
+	movl	%eax,IRF_EIP(%esp)	; \
+	movl	TRF_ERR(%ebp),%eax	; \
+	movl	%eax,IRF_ERR(%esp)	; \
+	movl	TRF_TRAPNO(%ebp),%eax	; \
+	movl	%eax,IRF_TRAPNO(%esp)	; \
+	SAVE_VM86
+
+#define INTR_ENABLE_U_PLUS_K	\
+	movl	$GSEL(GCPU_SEL, SEL_KPL),%eax	; \
+	movw	%ax,%fs			; \
+	movl	CPUVAR(KERN_CR3),%eax	; \
+	testl	%eax,%eax		; \
+	jz	100f			; \
+	movl	%eax,%cr3		; \
+	100:
+
 #define INTRENTRY_LABEL(label)	X##label##_untramp
 #define	INTRENTRY(label) \
 	/* we have an iretframe */	; \
@@ -147,40 +173,21 @@
 	pushl	%ebp			; \
 	pushl	%eax			; \
 	pushl	%fs			; \
-	movl	$GSEL(GCPU_SEL, SEL_KPL),%eax	; \
-	movw	%ax,%fs			; \
-	movl	CPUVAR(KERN_CR3),%eax	; \
-	testl	%eax,%eax		; \
-	jz	97f			; \
-	movl	%eax,%cr3		; \
-	jmp	97f			; \
+	INTR_ENABLE_U_PLUS_K		; \
+	jmp	99f			; \
 	.text				; \
 	.global INTRENTRY_LABEL(label) ; \
 INTRENTRY_LABEL(label):	/* from kernel */	; \
 	jmp	98f			; \
 	/* from user space, build trampframe */	; \
-97:	movl	CPUVAR(KERN_ESP),%eax	; \
+99:	movl	CPUVAR(KERN_ESP),%eax	; \
 	pushl	%eax			; \
 	pushl	$0xdeadbeef		; \
 	movl	%esp,%ebp		; \
 	movl	%eax,%esp		; \
 	subl	$SIZEOF_IRETFRAME,%esp	; \
 	/* we have a trampframe, copy to iretframe on kernel stack */	; \
-	movl	TRF_SS(%ebp),%eax	; \
-	movl	%eax,IRF_SS(%esp)	; \
-	movl	TRF_ESP(%ebp),%eax	; \
-	movl	%eax,IRF_ESP(%esp)	; \
-	movl	TRF_EFLAGS(%ebp),%eax	; \
-	movl	%eax,IRF_EFLAGS(%esp)	; \
-	movl	TRF_CS(%ebp),%eax	; \
-	movl	%eax,IRF_CS(%esp)	; \
-	movl	TRF_EIP(%ebp),%eax	; \
-	movl	%eax,IRF_EIP(%esp)	; \
-	movl	TRF_ERR(%ebp),%eax	; \
-	movl	%eax,IRF_ERR(%esp)	; \
-	movl	TRF_TRAPNO(%ebp),%eax	; \
-	movl	%eax,IRF_TRAPNO(%esp)	; \
-	SAVE_VM86			; \
+	INTR_COPY_FROM_TRAMP_STACK	; \
 	movl	TRF_FS(%ebp),%eax	; \
 	movw	%ax,%fs			; \
 	movl	TRF_EAX(%ebp),%eax	; \
@@ -985,8 +992,109 @@ IDTVEC(dbg)
 	movl	%eax,%dr6
 	popl	%eax
 	TRAP(T_TRCTRAP)
+
 IDTVEC(nmi)
-	ZTRAP(T_NMI)
+	/*
+	 * we came through a task gate; now U+K of the idle thread is
+	 * enabled; NMIs are blocked until next iret; IRQs are disabled;
+	 * all segment descriptors are useable
+	 *
+	 * first of all, switch back to the U+K we were actually running
+	 * on before
+	 */
+	movl	CPUVAR(CURPMAP),%eax
+	movl	PM_PDIRPA(%eax),%eax
+	movl	%eax,%cr3
+
+	/*
+	 * when we came from within the kernel, iret will not
+	 * switch back to the stack we came from but will keep
+	 * running on the NMI stack. in that case we switch
+	 * manually back to the stack we were running on and
+	 * build the iretframe there.
+	 */
+
+	/* was there a ring transition? */
+	movl	CPUVAR(TSS),%eax
+	testb	$SEL_RPL,TSS_CS(%eax)
+	jne	1f
+
+	/*
+	 * no ring transition, switch back to original stack, build
+	 * frame from state saved in TSS.
+	 */
+	movl	TSS_ESP(%eax),%esp
+	subl	$12,%esp
+	movl	TSS_EFLAGS(%eax),%ebx
+	movl	%ebx,8(%esp)
+	movl	TSS_CS(%eax),%ebx
+	movl	%ebx,4(%esp)
+	movl	TSS_EIP(%eax),%ebx
+	movl	%ebx,0(%esp)
+	pushl	$0
+	pushl	$T_NMI
+	jmp	2f
+
+	/*
+	 * ring transition, stay on stack, build frame from state
+	 * saved in TSS.
+	 */
+1:	subl	$20,%esp
+	pushl	$0
+	pushl	$T_NMI
+	movl	TSS_SS(%eax),%ebx
+	movl	%ebx,IRF_SS(%esp)
+	movl	TSS_ESP(%eax),%ebx
+	movl	%ebx,IRF_ESP(%esp)
+	movl	TSS_EFLAGS(%eax),%ebx
+	movl	%ebx,IRF_EFLAGS(%esp)
+	movl	TSS_CS(%eax),%ebx
+	movl	%ebx,IRF_CS(%esp)
+	movl	TSS_EIP(%eax),%ebx
+	movl	%ebx,IRF_EIP(%esp)
+
+	/* clear PSL_NT */
+2:	pushfl
+	popl	%eax
+	andl	$~PSL_NT,%eax
+	pushl	%eax
+	popfl
+
+	/* clear CR0_TS XXX hshoexer: needed? */
+	movl	%cr0,%eax
+	andl	$~CR0_TS,%eax
+	movl	%eax,%cr0
+
+	/* unbusy descriptors and reload common TSS */
+	movl	CPUVAR(GDT),%eax
+	movl	$GSEL(GNMITSS_SEL, SEL_KPL),%ebx
+	andl	$~0x200,4-SEL_KPL(%eax,%ebx,1)
+	movl	$GSEL(GTSS_SEL, SEL_KPL),%ebx
+	andl	$~0x200,4-SEL_KPL(%eax,%ebx,1)
+	ltr	%bx
+
+	/* load GPRs and segment registers with saved values from common TSS */
+	movl	CPUVAR(TSS),%eax
+	movl	TSS_ECX(%eax),%ecx
+	movl	TSS_EDX(%eax),%edx
+	movl	TSS_ESI(%eax),%esi
+	movl	TSS_EDI(%eax),%edi
+	movl	TSS_EBP(%eax),%ebp
+	movw	TSS_FS(%eax),%fs
+	movw	TSS_GS(%eax),%gs
+	movw	TSS_ES(%eax),%es
+	/* saved %ds might be invalid, thus push now and pop later */
+	movl	TSS_DS(%eax),%ebx
+	pushl	%ebx
+	movl	TSS_EBX(%eax),%ebx
+	movl	TSS_EAX(%eax),%eax
+	popl	%ds
+
+	/*
+	 * we can now proceed and save everything on the stack as
+	 * if no task switch had happend.
+	 */
+	jmp alltraps
 IDTVEC(bpt)
 	ZTRAP(T_BPTFLT)
 IDTVEC(ofl)
@@ -1024,8 +1132,68 @@ IDTVEC(missing)
 	TRAP(T_SEGNPFLT)
 IDTVEC(stk)
 	TRAP(T_STKFLT)
+
 IDTVEC(prot)
-	TRAP(T_PROTFLT)
+	pushl	$T_PROTFLT
+	/* If iret faults, we'll get a trap at doreti_iret+3 with CPL == 0. */
+	pushl	%eax
+	leal	_C_LABEL(doreti_iret+3),%eax
+	cmpl	%eax,12(%esp)	/* over %eax, trapno and err to %eip */
+	popl	%eax
+	jne	97f
+	pushl	%ebp
+	pushl	%eax
+	pushl	%fs
+	INTR_ENABLE_U_PLUS_K
+	/*
+	 * we have an iretframe on trampoline stack, above it the
+	 * remainder of the original iretframe iret faulted on.
+	 */
+	movl	CPUVAR(KERN_ESP),%eax
+	pushl	%eax
+	pushl	$0xdeadbeef
+	/*
+	 * now we have a trampframe on trampoline stack, above it the
+	 * remainder of the original iretframe iret faulted on.
+	 */
+	movl	%esp,%ebp
+	movl	%eax,%esp
+	subl	$SIZEOF_IRETFRAME+(5*4),%esp
+	/* copy to iretframe on kernel stack */
+	movl	TRF_EFLAGS(%ebp),%eax
+	movl	%eax,IRF_EFLAGS(%esp)
+	movl	TRF_CS(%ebp),%eax
+	movl	%eax,IRF_CS(%esp)
+	movl	TRF_EIP(%ebp),%eax
+	movl	%eax,IRF_EIP(%esp)
+	movl	TRF_ERR(%ebp),%eax
+	movl	%eax,IRF_ERR(%esp)
+	movl	TRF_TRAPNO(%ebp),%eax
+	movl	%eax,IRF_TRAPNO(%esp)
+	/* copy remainder of faulted iretframe */
+	movl	40(%ebp),%eax		/* eip */
+	movl	%eax,20(%esp)
+	movl	44(%ebp),%eax		/* cs */
+	movl	%eax,24(%esp)
+	movl	48(%ebp),%eax		/* eflags */
+	movl	%eax,28(%esp)
+	movl	52(%ebp),%eax		/* esp */
+	movl	%eax,32(%esp)
+	movl	56(%ebp),%eax		/* ss */
+	movl	%eax,36(%esp)
+	movl	TRF_FS(%ebp),%eax
+	movw	%ax,%fs
+	movl	TRF_EAX(%ebp),%eax
+	movl	TRF_EBP(%ebp),%ebp
+	/*
+	 * we have an iretframe on kernel stack, above it the
+	 * remainder of the original iretframe iret faulted on.
+	 * for INTRENTRY(prot) it looks like the fault happend
+	 * on the kernel stack
+	 */
+97:	INTRENTRY(prot)
+	sti
+	jmp	calltrap
 IDTVEC(f00f_redirect)
 	pushl	$T_PAGEFLT
 	INTRENTRY(f00f_redirect)
diff --git a/sys/arch/i386/i386/machdep.c b/sys/arch/i386/i386/machdep.c
index 02ed064740a..bf8c2aacb9a 100644
--- a/sys/arch/i386/i386/machdep.c
+++ b/sys/arch/i386/i386/machdep.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: machdep.c,v 1.617 2018/05/28 20:52:44 bluhm Exp $	*/
+/*	$OpenBSD: machdep.c,v 1.618 2018/06/22 13:21:14 bluhm Exp $	*/
 /*	$NetBSD: machdep.c,v 1.214 1996/11/10 03:16:17 thorpej Exp $	*/
 
 /*-
@@ -3102,6 +3102,7 @@ init386(paddr_t first_avail)
 	cpu_info_primary.ci_self = &cpu_info_primary;
 	cpu_info_primary.ci_curpcb = &proc0.p_addr->u_pcb;
 	cpu_info_primary.ci_tss = &cpu_info_full_primary.cif_tss;
+	cpu_info_primary.ci_nmi_tss = &cpu_info_full_primary.cif_nmi_tss;
 	cpu_info_primary.ci_gdt = (void *)&cpu_info_full_primary.cif_gdt;
 
 	/* make bootstrap gdt gates and memory segments */
@@ -3122,13 +3123,16 @@ init386(paddr_t first_avail)
 	setsegment(&cpu_info_primary.ci_gdt[GUGS_SEL].sd, 0,
 	    atop(VM_MAXUSER_ADDRESS) - 1, SDT_MEMRWA, SEL_UPL, 1, 1);
 	setsegment(&cpu_info_primary.ci_gdt[GTSS_SEL].sd,
-	    cpu_info_primary.ci_tss, sizeof(cpu_info_primary.ci_tss)-1,
+	    cpu_info_primary.ci_tss, sizeof(struct i386tss)-1,
+	    SDT_SYS386TSS, SEL_KPL, 0, 0);
+	setsegment(&cpu_info_primary.ci_gdt[GNMITSS_SEL].sd,
+	    cpu_info_primary.ci_nmi_tss, sizeof(struct i386tss)-1,
 	    SDT_SYS386TSS, SEL_KPL, 0, 0);
 
 	/* exceptions */
 	setgate(&idt[  0], &IDTVEC(div),     0, SDT_SYS386IGT, SEL_KPL, GCODE_SEL);
 	setgate(&idt[  1], &IDTVEC(dbg),     0, SDT_SYS386IGT, SEL_KPL, GCODE_SEL);
-	setgate(&idt[  2], &IDTVEC(nmi),     0, SDT_SYS386IGT, SEL_KPL, GCODE_SEL);
+	setgate(&idt[  2], NULL,             0, SDT_SYSTASKGT, SEL_KPL, GNMITSS_SEL);
 	setgate(&idt[  3], &IDTVEC(bpt),     0, SDT_SYS386IGT, SEL_UPL, GCODE_SEL);
 	setgate(&idt[  4], &IDTVEC(ofl),     0, SDT_SYS386IGT, SEL_UPL, GCODE_SEL);
 	setgate(&idt[  5], &IDTVEC(bnd),     0, SDT_SYS386IGT, SEL_KPL, GCODE_SEL);
diff --git a/sys/arch/i386/i386/pmap.c b/sys/arch/i386/i386/pmap.c
index ed2c99c8aad..e4a639400fa 100644
--- a/sys/arch/i386/i386/pmap.c
+++ b/sys/arch/i386/i386/pmap.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: pmap.c,v 1.202 2018/05/28 20:52:44 bluhm Exp $	*/
+/*	$OpenBSD: pmap.c,v 1.203 2018/06/22 13:21:14 bluhm Exp $	*/
 /*	$NetBSD: pmap.c,v 1.91 2000/06/02 17:46:37 thorpej Exp $	*/
 
 /*
@@ -1526,11 +1526,7 @@ pmap_switch(struct proc *o, struct proc *p)
 	 */
 	if (pmap->pm_pdirpa_intel) {
 		self->ci_kern_cr3 = pmap->pm_pdirpa;
-#if 0		/* XXX hshoexer:  Do not unmap kernel, yet */
 		self->ci_user_cr3 = pmap->pm_pdirpa_intel;
-#else
-		self->ci_user_cr3 = pmap->pm_pdirpa;
-#endif
 	}
 
 	/*
diff --git a/sys/arch/i386/i386/pmapae.c b/sys/arch/i386/i386/pmapae.c
index 20575eda6f9..4ea8707d776 100644
--- a/sys/arch/i386/i386/pmapae.c
+++ b/sys/arch/i386/i386/pmapae.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: pmapae.c,v 1.55 2018/05/28 20:52:44 bluhm Exp $	*/
+/*	$OpenBSD: pmapae.c,v 1.56 2018/06/22 13:21:14 bluhm Exp $	*/
 
 /*
  * Copyright (c) 2006-2008 Michael Shalayeff
@@ -749,6 +749,9 @@ pmap_bootstrap_pae(void)
 		    (vaddr_t)kpm - KERNBASE;
 		kpm->pm_pdirsize = 4 * NBPG;
 
+		/* Reset cr3 for NMI task switch */
+		cpu_update_nmi_cr3(kpm->pm_pdirpa);
+
 		DPRINTF("%s: pm_pdir 0x%x pm_pdirpa 0x%x pm_pdirsize %d\n",
 		    __func__, (uint32_t)kpm->pm_pdir, (uint32_t)kpm->pm_pdirpa,
 		    kpm->pm_pdirsize);
diff --git a/sys/arch/i386/include/cpu.h b/sys/arch/i386/include/cpu.h
index 351c9be6a7c..4d03c9e1e36 100644
--- a/sys/arch/i386/include/cpu.h
+++ b/sys/arch/i386/include/cpu.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: cpu.h,v 1.162 2018/04/11 15:44:08 bluhm Exp $	*/
+/*	$OpenBSD: cpu.h,v 1.163 2018/06/22 13:21:14 bluhm Exp $	*/
 /*	$NetBSD: cpu.h,v 1.35 1996/05/05 19:29:26 christos Exp $	*/
 
 /*-
@@ -184,6 +184,7 @@ struct cpu_info {
 
 	union descriptor *ci_gdt;
 	struct i386tss	*ci_tss;
+	struct i386tss	*ci_nmi_tss;
 
 	volatile int ci_ddb_paused;	/* paused due to other proc in ddb */
 #define CI_DDB_RUNNING		0
@@ -399,6 +400,7 @@ extern int cpu_apmi_edx;
 /* cpu.c */
 extern u_int cpu_mwait_size;
 extern u_int cpu_mwait_states;
+extern void cpu_update_nmi_cr3(vaddr_t);
 
 /* machdep.c */
 extern int cpu_apmhalt;
diff --git a/sys/arch/i386/include/cpu_full.h b/sys/arch/i386/include/cpu_full.h
index da2dee89e72..c7375af16c1 100644
--- a/sys/arch/i386/include/cpu_full.h
+++ b/sys/arch/i386/include/cpu_full.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: cpu_full.h,v 1.2 2018/05/28 20:52:44 bluhm Exp $	*/
+/*	$OpenBSD: cpu_full.h,v 1.3 2018/06/22 13:21:14 bluhm Exp $	*/
 /*
  * Copyright (c) 2018 Philip Guenther <guenther@openbsd.org>
  * Copyright (c) 2018 Hans-Joerg Hoexer <hshoexer@genua.de>
@@ -28,16 +28,19 @@ struct cpu_info_full {
 	union {
 		struct {
 			struct i386tss		uu_tss;
+			struct i386tss		uu_nmi_tss;
 			union descriptor	uu_gdt[NGDT];
 		} u_tssgdt;
 	char			u_align[PAGE_SIZE];
 	} cif_TSS_RO;
 #define cif_tss cif_TSS_RO.u_tssgdt.uu_tss
+#define cif_nmi_tss cif_TSS_RO.u_tssgdt.uu_nmi_tss
 #define cif_gdt cif_TSS_RO.u_tssgdt.uu_gdt
 
 	/* start of page mapped kRW in u-k */
-	uint32_t cif_tramp_stack[(PAGE_SIZE
+	uint32_t cif_tramp_stack[(PAGE_SIZE / 4
 	    - offsetof(struct cpu_info, ci_PAGEALIGN)) / sizeof(uint32_t)];
+	uint32_t cif_nmi_stack[(3 * PAGE_SIZE / 4) / sizeof(uint32_t)];
 
 	/*
 	 * Beginning of this hangs over into the kRW page; rest is
@@ -47,7 +50,7 @@ struct cpu_info_full {
 } __aligned(PAGE_SIZE);
 
 /* tss, align shim, and gdt must fit in a page */
-CTASSERT(_ALIGN(sizeof(struct i386tss)) +
+CTASSERT(_ALIGN(2 * sizeof(struct i386tss)) +
 	sizeof(struct segment_descriptor) * NGDT < PAGE_SIZE);
 
 /* verify expected alignment */
diff --git a/sys/arch/i386/include/segments.h b/sys/arch/i386/include/segments.h
index 22e9ed9feaf..141bc12e262 100644
--- a/sys/arch/i386/include/segments.h
+++ b/sys/arch/i386/include/segments.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: segments.h,v 1.24 2018/03/31 13:45:03 bluhm Exp $	*/
+/*	$OpenBSD: segments.h,v 1.25 2018/06/22 13:21:14 bluhm Exp $	*/
 /*	$NetBSD: segments.h,v 1.23 1996/02/01 22:31:03 mycroft Exp $	*/
 
 /*-
@@ -222,8 +222,9 @@ void idt_vec_free(int);
 #define	GUFS_SEL	11	/* User per-thread (%fs) descriptor */
 #define	GUGS_SEL	12	/* User per-thread (%gs) descriptor */
 #define GTSS_SEL	13	/* common TSS */
-#define	GBIOS32_SEL	14	/* spare slot for 32 bit BIOS calls */
-#define	NGDT		15
+#define GNMITSS_SEL	14	/* NMI TSS */
+#define	GBIOS32_SEL	15	/* spare slot for 32 bit BIOS calls */
+#define	NGDT		16
 
 #define GDT_SIZE	(NGDT << 3)
 
diff --git a/sys/arch/i386/include/tss.h b/sys/arch/i386/include/tss.h
index 35cfb6f81fd..20ff25dd341 100644
--- a/sys/arch/i386/include/tss.h
+++ b/sys/arch/i386/include/tss.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: tss.h,v 1.11 2018/03/31 13:45:03 bluhm Exp $	*/
+/*	$OpenBSD: tss.h,v 1.12 2018/06/22 13:21:14 bluhm Exp $	*/
 /*	$NetBSD: tss.h,v 1.6 1995/10/11 04:20:28 mycroft Exp $	*/
 
 /*-
@@ -50,24 +50,24 @@ struct i386tss {
 	int	__tss_ss1;
 	int	__tss_esp2;
 	int	__tss_ss2;
-	int	__tss_cr3;
-	int	__tss_eip;
-	int	__tss_eflags;
-	int	__tss_eax; 
-	int	__tss_ecx; 
-	int	__tss_edx; 
-	int	__tss_ebx; 
-	int	__tss_esp;
-	int	__tss_ebp;
-	int	__tss_esi; 
-	int	__tss_edi; 
-	int	__tss_es;
-	int	__tss_cs;
-	int	__tss_ss;
-	int	__tss_ds;
-	int	__tss_fs;
-	int	__tss_gs;
-	int	__tss_ldt;
+	int	tss_cr3;
+	int	tss_eip;
+	int	tss_eflags;
+	int	tss_eax;
+	int	tss_ecx;
+	int	tss_edx;
+	int	tss_ebx;
+	int	tss_esp;
+	int	tss_ebp;
+	int	tss_esi;
+	int	tss_edi;
+	int	tss_es;
+	int	tss_cs;
+	int	tss_ss;
+	int	tss_ds;
+	int	tss_fs;
+	int	tss_gs;
+	int	tss_ldt;
 	int	tss_ioopt;	/* options and I/O permission map offset */
 };