Just like on amd64. Instead of keeping a bitmap of which cpus a pmap

is active on, save a curpmap pointer in cpu_info. This lets us simplify
a few things and do lazy context switching from a user process to a
kernel thread. There's a new IPI introduced for forcing a cr3 reload
when we're tearing down a dead pmap.

kettenis@ ok (after I polished a few minor things)

10 files changed, 162 insertions, 183 deletions

diff --git a/sys/arch/i386/i386/apicvec.s b/sys/arch/i386/i386/apicvec.s
index 62de5ffadd3..bef1d855c14 100644
--- a/sys/arch/i386/i386/apicvec.s
+++ b/sys/arch/i386/i386/apicvec.s
@@ -1,4 +1,4 @@
-/* $OpenBSD: apicvec.s,v 1.16 2008/06/26 05:42:10 ray Exp $ */
+/* $OpenBSD: apicvec.s,v 1.17 2009/06/03 00:49:12 art Exp $ */
 /* $NetBSD: apicvec.s,v 1.1.2.2 2000/02/21 21:54:01 sommerfeld Exp $ */
 
 /*-
@@ -126,6 +126,29 @@ XINTR(ipi_invlrange):
 	popl	%eax
 	iret
 
+	.globl	XINTR(ipi_reloadcr3)
+	.p2align 4,0x90
+XINTR(ipi_reloadcr3):
+	pushl	%eax
+	pushl	%ds
+	movl	$GSEL(GDATA_SEL, SEL_KPL), %eax
+	movl	%eax, %ds
+
+	ioapic_asm_ack()
+
+	movl	CPUVAR(CURPCB), %eax
+	movl	PCB_PMAP(%eax), %eax
+	movl	%eax, CPUVAR(CURPMAP)
+	movl	PM_PDIRPA(%eax), %eax
+	movl	%eax, %cr3
+
+	lock
+	decl	tlb_shoot_wait
+
+	popl	%ds
+	popl	%eax
+	iret
+
 #endif
 
 	/*
diff --git a/sys/arch/i386/i386/cpu.c b/sys/arch/i386/i386/cpu.c
index fc8e0e4919c..b586ab0cc42 100644
--- a/sys/arch/i386/i386/cpu.c
+++ b/sys/arch/i386/i386/cpu.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: cpu.c,v 1.34 2009/01/20 20:21:03 mlarkin Exp $	*/
+/*	$OpenBSD: cpu.c,v 1.35 2009/06/03 00:49:12 art Exp $	*/
 /* $NetBSD: cpu.c,v 1.1.2.7 2000/06/26 02:04:05 sommerfeld Exp $ */
 
 /*-
@@ -244,6 +244,7 @@ cpu_attach(struct device *parent, struct device *self, void *aux)
 
 	cpu_default_ldt(ci);	/* Use the `global' ldt until one alloc'd */
 #endif
+	ci->ci_curpmap = pmap_kernel();
 
 	/* further PCB init done later. */
 
@@ -557,6 +558,8 @@ mp_cpu_start(struct cpu_info *ci)
 	dwordptr[0] = 0;
 	dwordptr[1] = MP_TRAMPOLINE >> 4;
 
+	pmap_activate(curproc);
+
 	pmap_kenter_pa(0, 0, VM_PROT_READ|VM_PROT_WRITE);
 	memcpy((u_int8_t *)0x467, dwordptr, 4);
 	pmap_kremove(0, PAGE_SIZE);
diff --git a/sys/arch/i386/i386/genassym.cf b/sys/arch/i386/i386/genassym.cf
index 211a677dfa5..6506af35feb 100644
--- a/sys/arch/i386/i386/genassym.cf
+++ b/sys/arch/i386/i386/genassym.cf
@@ -1,4 +1,4 @@
-#	$OpenBSD: genassym.cf,v 1.28 2007/10/10 15:53:51 art Exp $
+#	$OpenBSD: genassym.cf,v 1.29 2009/06/03 00:49:12 art Exp $
 #
 # Copyright (c) 1982, 1990 The Regents of the University of California.
 # All rights reserved.
@@ -114,6 +114,7 @@ member	pcb_ldt_sel
 member	pcb_onfault
 member	pcb_fpcpu
 member	pcb_flags
+member	pcb_pmap
 export	PCB_SAVECTX
 
 # frame definitions
@@ -206,6 +207,10 @@ define	CPU_INFO_IUNMASK	offsetof(struct cpu_info, ci_iunmask)
 define	CPU_INFO_ILEVEL		offsetof(struct cpu_info, ci_ilevel)
 define	CPU_INFO_IDEPTH		offsetof(struct cpu_info, ci_idepth)
 define	CPU_INFO_ISOURCES	offsetof(struct cpu_info, ci_isources)
+define	CPU_INFO_CURPMAP	offsetof(struct cpu_info, ci_curpmap)
+
+struct pmap
+member pm_pdirpa
 
 define	SIZEOF_CPU_INFO		sizeof(struct cpu_info)
 
diff --git a/sys/arch/i386/i386/lapic.c b/sys/arch/i386/i386/lapic.c
index c193d26faac..426971751f3 100644
--- a/sys/arch/i386/i386/lapic.c
+++ b/sys/arch/i386/i386/lapic.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: lapic.c,v 1.24 2008/08/13 15:46:21 art Exp $	*/
+/*	$OpenBSD: lapic.c,v 1.25 2009/06/03 00:49:12 art Exp $	*/
 /* $NetBSD: lapic.c,v 1.1.2.8 2000/02/23 06:10:50 sommerfeld Exp $ */
 
 /*-
@@ -203,6 +203,7 @@ lapic_boot_init(paddr_t lapic_base)
 	idt_vec_set(LAPIC_IPI_INVLTLB, Xintripi_invltlb);
 	idt_vec_set(LAPIC_IPI_INVLPG, Xintripi_invlpg);
 	idt_vec_set(LAPIC_IPI_INVLRANGE, Xintripi_invlrange);
+	idt_vec_set(LAPIC_IPI_RELOADCR3, Xintripi_reloadcr3);
 #endif
 	idt_vec_set(LAPIC_SPURIOUS_VECTOR, Xintrspurious);
 	idt_vec_set(LAPIC_TIMER_VECTOR, Xintrltimer);
diff --git a/sys/arch/i386/i386/locore.s b/sys/arch/i386/i386/locore.s
index 9e2b80afeb4..e2c9eaae415 100644
--- a/sys/arch/i386/i386/locore.s
+++ b/sys/arch/i386/i386/locore.s
@@ -1,4 +1,4 @@
-/*	$OpenBSD: locore.s,v 1.126 2009/06/03 00:41:48 weingart Exp $	*/
+/*	$OpenBSD: locore.s,v 1.127 2009/06/03 00:49:12 art Exp $	*/
 /*	$NetBSD: locore.s,v 1.145 1996/05/03 19:41:19 christos Exp $	*/
 
 /*-
@@ -85,9 +85,6 @@
 #define	GET_CURPCB(reg)					\
 	movl	CPUVAR(CURPCB), reg
 
-#define	SET_CURPCB(reg)					\
-	movl	reg, CPUVAR(CURPCB)
-
 #define	CHECK_ASTPENDING(treg)				\
 	movl 	CPUVAR(CURPROC),treg		;	\
 	cmpl	$0, treg			;	\
@@ -1270,49 +1267,28 @@ ENTRY(cpu_switchto)
 	testl	%esi,%esi
 	jz	switch_exited
 
-	/*
-	 * Save old context.
-	 *
-	 * Registers:
-	 *   %eax, %ecx - scratch
-	 *   %esi - old process, then old pcb
-	 *   %edi - new process
-	 */
-
-	pushl	%esi
-	call	_C_LABEL(pmap_deactivate)
-	addl	$4,%esp
-
-	movl	P_ADDR(%esi),%esi
-
-	/* Save stack pointers. */
-	movl	%esp,PCB_ESP(%esi)
-	movl	%ebp,PCB_EBP(%esi)
+	/* Save old stack pointers. */
+	movl	P_ADDR(%esi),%ebx
+	movl	%esp,PCB_ESP(%ebx)
+	movl	%ebp,PCB_EBP(%ebx)
 
 switch_exited:
-	/*
-	 * Third phase: restore saved context.
-	 *
-	 * Registers:
-	 *   %eax, %ecx, %edx - scratch
-	 *   %esi - new pcb
-	 *   %edi - new process
-	 */
+	/* Restore saved context. */
 
 	/* No interrupts while loading new state. */
 	cli
 
 	/* Record new process. */
-	movl	CPUVAR(SELF), %ebx
 	movl	%edi, CPUVAR(CURPROC)
 	movb	$SONPROC, P_STAT(%edi)
-	movl	%ebx, P_CPU(%edi)
-
-	movl	P_ADDR(%edi),%esi
 
 	/* Restore stack pointers. */
-	movl	PCB_ESP(%esi),%esp
-	movl	PCB_EBP(%esi),%ebp
+	movl	P_ADDR(%edi),%ebx
+	movl	PCB_ESP(%ebx),%esp
+	movl	PCB_EBP(%ebx),%ebp
+
+	/* Record new pcb. */
+	movl	%ebx, CPUVAR(CURPCB)
 
 	/*
 	 * Activate the address space.  The pcb copy of %cr3 and the
@@ -1320,9 +1296,10 @@ switch_exited:
 	 * curproc they'll both be reloaded into the CPU.
 	 */
 	pushl	%edi
-	call	_C_LABEL(pmap_activate)
-	addl	$4,%esp
-	
+	pushl	%esi
+	call	_C_LABEL(pmap_switch)
+	addl	$8,%esp
+
 	/* Load TSS info. */
 	movl	CPUVAR(GDT),%eax
 	movl	P_MD_TSS_SEL(%edi),%edx
@@ -1332,23 +1309,20 @@ switch_exited:
 	ltr	%dx
 
 	/* Restore cr0 (including FPU state). */
-	movl	PCB_CR0(%esi),%ecx
+	movl	PCB_CR0(%ebx),%ecx
 #ifdef MULTIPROCESSOR
 	/*
 	 * If our floating point registers are on a different CPU,
 	 * clear CR0_TS so we'll trap rather than reuse bogus state.
 	 */
-	movl	CPUVAR(SELF), %ebx
-	cmpl	PCB_FPCPU(%esi),%ebx
+	movl	CPUVAR(SELF), %esi
+	cmpl	PCB_FPCPU(%ebx), %esi
 	jz	1f
 	orl	$CR0_TS,%ecx
 1:	
 #endif	
 	movl	%ecx,%cr0
 
-	/* Record new pcb. */
-	SET_CURPCB(%esi)
-
 	/* Interrupts are okay again. */
 	sti
 
diff --git a/sys/arch/i386/i386/pmap.c b/sys/arch/i386/i386/pmap.c
index 80392600813..116579d1448 100644
--- a/sys/arch/i386/i386/pmap.c
+++ b/sys/arch/i386/i386/pmap.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: pmap.c,v 1.138 2009/06/02 23:00:19 oga Exp $	*/
+/*	$OpenBSD: pmap.c,v 1.139 2009/06/03 00:49:12 art Exp $	*/
 /*	$NetBSD: pmap.c,v 1.91 2000/06/02 17:46:37 thorpej Exp $	*/
 
 /*
@@ -337,8 +337,6 @@ void		 pmap_free_pvpage(void);
 struct vm_page	*pmap_get_ptp(struct pmap *, int, boolean_t);
 void		 pmap_drop_ptp(struct pmap *, vaddr_t, struct vm_page *,
     pt_entry_t *);
-boolean_t	 pmap_is_curpmap(struct pmap *);
-boolean_t	 pmap_is_active(struct pmap *, int);
 void		 pmap_sync_flags_pte(struct vm_page *, u_long);
 pt_entry_t	*pmap_map_ptes(struct pmap *);
 struct pv_entry	*pmap_remove_pv(struct vm_page *, struct pmap *, vaddr_t);
@@ -355,13 +353,12 @@ vaddr_t		 pmap_tmpmap_pa(paddr_t);
 pt_entry_t	*pmap_tmpmap_pvepte(struct pv_entry *);
 void		 pmap_tmpunmap_pa(void);
 void		 pmap_tmpunmap_pvepte(struct pv_entry *);
-void		 pmap_apte_flush(struct pmap *);
+void		 pmap_apte_flush(void);
 void		pmap_unmap_ptes(struct pmap *);
 void		pmap_exec_account(struct pmap *, vaddr_t, pt_entry_t,
 		    pt_entry_t);
 
 void			pmap_pinit(pmap_t);
-void			pmap_release(pmap_t);
 
 void			pmap_zero_phys(paddr_t);
 
@@ -372,30 +369,20 @@ void	setcslimit(struct pmap *, struct trapframe *, struct pcb *, vaddr_t);
  */
 
 /*
- * pmap_is_curpmap: is this pmap the one currently loaded [in %cr3]?
- *		of course the kernel is always loaded
+ * pmap_is_active: is this pmap loaded into the specified processor's %cr3?
  */
 
-boolean_t
-pmap_is_curpmap(pmap)
-	struct pmap *pmap;
+static __inline boolean_t
+pmap_is_active(struct pmap *pmap, struct cpu_info *ci)
 {
-	return((pmap == pmap_kernel()) ||
-	       (pmap->pm_pdirpa == (paddr_t) rcr3()));
-}
 
-/*
- * pmap_is_active: is this pmap loaded into the specified processor's %cr3?
- */
+	return (pmap == pmap_kernel() || ci->ci_curpmap == pmap);
+}
 
-boolean_t
-pmap_is_active(pmap, cpu_id)
-	struct pmap *pmap;
-	int cpu_id;
+static __inline boolean_t
+pmap_is_curpmap(struct pmap *pmap)
 {
-
-	return (pmap == pmap_kernel() ||
-	    (pmap->pm_cpus & (1U << cpu_id)) != 0);
+	return (pmap_is_active(pmap, curcpu()));
 }
 
 static __inline u_int
@@ -456,7 +443,7 @@ pmap_tmpunmap_pa()
 	if (!pmap_valid_entry(*ptpte))
 		panic("pmap_tmpunmap_pa: our pte invalid?");
 #endif
-	*ptpte = 0;		/* zap! */
+	*ptpte = 0;
 	pmap_update_pg((vaddr_t)ptpva);
 #ifdef MULTIPROCESSOR
 	/*
@@ -502,7 +489,7 @@ pmap_tmpunmap_pvepte(struct pv_entry *pve)
 }
 
 void
-pmap_apte_flush(struct pmap *pmap)
+pmap_apte_flush(void)
 {
 	pmap_tlb_shoottlb();
 	pmap_tlb_shootwait();
@@ -550,7 +537,7 @@ pmap_map_ptes(struct pmap *pmap)
 		*APDP_PDE = (pd_entry_t) (pmap->pm_pdirpa | PG_RW | PG_V |
 		    PG_U | PG_M);
 		if (pmap_valid_entry(opde))
-			pmap_apte_flush(curpcb->pcb_pmap);
+			pmap_apte_flush();
 	}
 	return(APTE_BASE);
 }
@@ -570,7 +557,7 @@ pmap_unmap_ptes(struct pmap *pmap)
 	} else {
 #if defined(MULTIPROCESSOR)
 		*APDP_PDE = 0;
-		pmap_apte_flush(curpcb->pcb_pmap);
+		pmap_apte_flush();
 #endif
 		simple_unlock(&pmap->pm_obj.vmobjlock);
 		simple_unlock(&curpcb->pcb_pmap->pm_obj.vmobjlock);
@@ -706,7 +693,7 @@ pmap_kenter_pa(vaddr_t va, paddr_t pa, vm_prot_t prot)
 	if (va >= (vaddr_t)NBPD)
 		npte |= pmap_pg_g;
 
-	opte = i386_atomic_testset_ul(pte, npte); /* zap! */
+	opte = i386_atomic_testset_ul(pte, npte);
 	if (pmap_valid_entry(opte)) {
 		/* NB. - this should not happen. */
 		pmap_tlb_shootpage(pmap_kernel(), va);
@@ -944,7 +931,7 @@ pmap_prealloc_lowmem_ptp(paddr_t ptppa)
 	pte = vtopte(ptpva);
 	npte = ptppa | PG_RW | PG_V | PG_U | PG_M;
 
-	i386_atomic_testset_ul(pte, npte); /* zap! */
+	i386_atomic_testset_ul(pte, npte);
 
 	/* make sure it is clean before using */
 	memset((void *)ptpva, 0, NBPG);
@@ -1426,7 +1413,7 @@ pmap_drop_ptp(struct pmap *pm, vaddr_t va, struct vm_page *ptp,
     pt_entry_t *ptes)
 {
 	i386_atomic_testset_ul(&pm->pm_pdir[pdei(va)], 0);
-	pmap_tlb_shootpage(curpcb->pcb_pmap, ((vaddr_t)ptes) + ptp->offset);
+	pmap_tlb_shootpage(curcpu()->ci_curpmap, ((vaddr_t)ptes) + ptp->offset);
 #ifdef MULTIPROCESSOR
 	/*
 	 * Always shoot down the other pmap's
@@ -1482,7 +1469,6 @@ pmap_pinit(struct pmap *pmap)
 	pmap->pm_ptphint = NULL;
 	pmap->pm_hiexec = 0;
 	pmap->pm_flags = 0;
-	pmap->pm_cpus = 0;
 
 	setsegment(&pmap->pm_codeseg, 0, atop(I386_MAX_EXE_ADDR) - 1,
 	    SDT_MEMERA, SEL_UPL, 1, 1);
@@ -1531,72 +1517,25 @@ pmap_pinit(struct pmap *pmap)
 void
 pmap_destroy(struct pmap *pmap)
 {
+	struct vm_page *pg;
 	int refs;
 
-	/*
-	 * drop reference count
-	 */
-
-	simple_lock(&pmap->pm_obj.vmobjlock);
 	refs = --pmap->pm_obj.uo_refs;
-	simple_unlock(&pmap->pm_obj.vmobjlock);
 	if (refs > 0)
 		return;
 
-	/*
-	 * reference count is zero, free pmap resources and then free pmap.
-	 */
-
-	pmap_release(pmap);
-	pool_put(&pmap_pmap_pool, pmap);
-}
-
-/*
- * pmap_release: release all resources held by a pmap
- *
- * => if pmap is still referenced it should be locked
- * => XXX: we currently don't expect any busy PTPs because we don't
- *    allow anything to map them (except for the kernel's private
- *    recursive mapping) or make them busy.
- */
-
-void
-pmap_release(struct pmap *pmap)
-{
-	struct vm_page *pg;
-
-	/*
-	 * remove it from global list of pmaps
-	 */
+	pmap_tlb_droppmap(pmap);	
 
 	simple_lock(&pmaps_lock);
 	LIST_REMOVE(pmap, pm_list);
 	simple_unlock(&pmaps_lock);
 
-	/*
-	 * Before we free the pmap just make sure it's not cached anywhere.
-	 */
-	tlbflushg();
-
-	/*
-	 * free any remaining PTPs
-	 */
-
+	/* Free any remaining PTPs. */
 	while ((pg = RB_ROOT(&pmap->pm_obj.memt)) != NULL) {
-#ifdef DIAGNOSTIC
-		if (pg->pg_flags & PG_BUSY)
-			panic("pmap_release: busy page table page");
-#endif
-		/* pmap_page_protect?  currently no need for it. */
-
 		pg->wire_count = 0;
 		uvm_pagefree(pg);
 	}
 
-	/*
-	 * MULTIPROCESSOR -- no need to flush out of other processors'
-	 * APTE space because we do that in pmap_unmap_ptes().
-	 */
 	uvm_km_free(kernel_map, (vaddr_t)pmap->pm_pdir, NBPG);
 	pmap->pm_pdir = NULL;
 
@@ -1614,8 +1553,10 @@ pmap_release(struct pmap *pmap)
 			    pmap->pm_ldt_len * sizeof(union descriptor));
 	}
 #endif
+	pool_put(&pmap_pmap_pool, pmap);
 }
 
+
 /*
  *	Add a reference to the specified pmap.
  */
@@ -1703,20 +1644,27 @@ pmap_ldt_cleanup(struct proc *p)
 }
 #endif /* USER_LDT */
 
-/*
- * pmap_activate: activate a process' pmap (fill in %cr3 and LDT info)
- *
- * => called from cpu_switchto()
- * => if proc is the curproc, then load it into the MMU
- */
-
 void
 pmap_activate(struct proc *p)
 {
+	KASSERT(curproc == p);
+	KASSERT(&p->p_addr->u_pcb == curpcb);
+	pmap_switch(NULL, p);
+}
+
+int nlazy_cr3_hit;
+int nlazy_cr3;
+
+void
+pmap_switch(struct proc *o, struct proc *p)
+{
 	struct pcb *pcb = &p->p_addr->u_pcb;
-	struct pmap *pmap = p->p_vmspace->vm_map.pmap;
+	struct pmap *pmap, *opmap;
 	struct cpu_info *self = curcpu();
 
+	pmap = p->p_vmspace->vm_map.pmap;
+	opmap = self->ci_curpmap;
+
 	pcb->pcb_pmap = pmap;
 	/* Get the LDT that this process will actually use */
 #ifdef MULTIPROCESSOR
@@ -1726,47 +1674,32 @@ pmap_activate(struct proc *p)
 #endif
 	pcb->pcb_ldt_sel = pmap->pm_ldt_sel;
 	pcb->pcb_cr3 = pmap->pm_pdirpa;
-	if (p == curproc) {
-		/*
-		 * Set the correct descriptor value (i.e. with the
-		 * correct code segment X limit) in the GDT.
-		 */
-		self->ci_gdt[GUCODE_SEL].sd = pmap->pm_codeseg;
-
-		lcr3(pcb->pcb_cr3);
-		lldt(pcb->pcb_ldt_sel);
 
-		/*
-		 * mark the pmap in use by this processor.
-		 */
-		i386_atomic_setbits_l(&pmap->pm_cpus, (1U << cpu_number()));
+	if (opmap == pmap) {
+		if (pmap != pmap_kernel())
+			nlazy_cr3_hit++;
+	} else if (o != NULL && pmap == pmap_kernel()) {
+		nlazy_cr3++;
+	} else {
+		curcpu()->ci_curpmap = pmap;
+		lcr3(pmap->pm_pdirpa);
 	}
-}
 
-/*
- * pmap_deactivate: deactivate a process' pmap
- */
+	/*
+	 * Set the correct descriptor value (i.e. with the
+	 * correct code segment X limit) in the GDT.
+	 */
+	self->ci_gdt[GUCODE_SEL].sd = pmap->pm_codeseg;
+
+	lldt(pcb->pcb_ldt_sel);
+}
 
 void
 pmap_deactivate(struct proc *p)
 {
-	struct pmap *pmap = p->p_vmspace->vm_map.pmap;
-
-	/*
-	 * mark the pmap no longer in use by this processor.
-	 */
-	i386_atomic_clearbits_l(&pmap->pm_cpus, (1U << cpu_number()));
 }
 
 /*
- * end of lifecycle functions
- */
-
-/*
- * some misc. functions
- */
-
-/*
  * pmap_extract: extract a PA for the given VA
  */
 
@@ -1832,7 +1765,7 @@ pmap_zero_phys(paddr_t pa)
 	*zpte = (pa & PG_FRAME) | PG_V | PG_RW;	/* map in */
 	pmap_update_pg((vaddr_t)zerova);	/* flush TLB */
 	pagezero(zerova, PAGE_SIZE);		/* zero */
-	*zpte = 0;				/* zap! */
+	*zpte = 0;
 }
 
 /*
@@ -1856,7 +1789,7 @@ pmap_zero_page_uncached(paddr_t pa)
 	*zpte = (pa & PG_FRAME) | PG_V | PG_RW | PG_N;	/* map in */
 	pmap_update_pg((vaddr_t)zerova);		/* flush TLB */
 	pagezero(zerova, PAGE_SIZE);				/* zero */
-	*zpte = 0;					/* zap! */
+	*zpte = 0;
 
 	return (TRUE);
 }
@@ -1887,7 +1820,7 @@ pmap_copy_page(struct vm_page *srcpg, struct vm_page *dstpg)
 	*dpte = (dstpa & PG_FRAME) | PG_V | PG_RW;
 	pmap_update_2pg((vaddr_t)csrcva, (vaddr_t)cdstva);
 	bcopy(csrcva, cdstva, PAGE_SIZE);
-	*spte = *dpte = 0;			/* zap! */
+	*spte = *dpte = 0;
 	pmap_update_2pg((vaddr_t)csrcva, (vaddr_t)cdstva);
 }
 
@@ -1933,7 +1866,7 @@ pmap_remove_ptes(struct pmap *pmap, struct vm_page *ptp, vaddr_t ptpva,
 		if ((flags & PMAP_REMOVE_SKIPWIRED) && (*pte & PG_W))
 			continue;
 
-		/* atomically save the old PTE and zap! it */
+		/* atomically save the old PTE and zero it */
 		opte = i386_atomic_testset_ul(pte, 0);
 
 		if (opte & PG_W)
@@ -2795,7 +2728,7 @@ pmap_tlb_shootpage(struct pmap *pm, vaddr_t va)
 	int mask = 0;
 
 	CPU_INFO_FOREACH(cii, ci) {
-		if (ci == self || !pmap_is_active(pm, ci->ci_cpuid) ||
+		if (ci == self || !pmap_is_active(pm, ci) ||
 		    !(ci->ci_flags & CPUF_RUNNING))
 			continue;
 		mask |= 1 << ci->ci_cpuid;
@@ -2833,7 +2766,7 @@ pmap_tlb_shootrange(struct pmap *pm, vaddr_t sva, vaddr_t eva)
 	vaddr_t va;
 
 	CPU_INFO_FOREACH(cii, ci) {
-		if (ci == self || !pmap_is_active(pm, ci->ci_cpuid) ||
+		if (ci == self || !pmap_is_active(pm, ci) ||
 		    !(ci->ci_flags & CPUF_RUNNING))
 			continue;
 		mask |= 1 << ci->ci_cpuid;
@@ -2899,6 +2832,45 @@ pmap_tlb_shoottlb(void)
 }
 
 void
+pmap_tlb_droppmap(struct pmap *pm)
+{
+	struct cpu_info *ci, *self = curcpu();
+	CPU_INFO_ITERATOR cii;
+	int wait = 0;
+	int mask = 0;
+
+	CPU_INFO_FOREACH(cii, ci) {
+		if (ci == self || !(ci->ci_flags & CPUF_RUNNING) ||
+		    ci->ci_curpmap != pm)
+			continue;
+		mask |= 1 << ci->ci_cpuid;
+		wait++;
+	}
+
+	if (wait) {
+		int s = splvm();
+
+		while (i486_atomic_cas_int(&tlb_shoot_wait, 0, wait) != 0) {
+			while (tlb_shoot_wait != 0)
+				SPINLOCK_SPIN_HOOK;
+		}
+
+		CPU_INFO_FOREACH(cii, ci) {
+			if ((mask & 1 << ci->ci_cpuid) == 0)
+				continue;
+			if (i386_fast_ipi(ci, LAPIC_IPI_RELOADCR3) != 0)
+				panic("pmap_tlb_droppmap: ipi failed");
+		}
+		splx(s);
+	}
+
+	if (self->ci_curpmap == pm)
+		pmap_activate(curproc);
+
+	pmap_tlb_shootwait();
+}
+
+void
 pmap_tlb_shootwait(void)
 {
 	while (tlb_shoot_wait != 0)
diff --git a/sys/arch/i386/i386/vm_machdep.c b/sys/arch/i386/i386/vm_machdep.c
index 1d43864a04c..5711b9d63e1 100644
--- a/sys/arch/i386/i386/vm_machdep.c
+++ b/sys/arch/i386/i386/vm_machdep.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: vm_machdep.c,v 1.57 2009/03/26 17:24:33 oga Exp $	*/
+/*	$OpenBSD: vm_machdep.c,v 1.58 2009/06/03 00:49:12 art Exp $	*/
 /*	$NetBSD: vm_machdep.c,v 1.61 1996/05/03 19:42:35 christos Exp $	*/
 
 /*-
@@ -135,7 +135,6 @@ cpu_exit(struct proc *p)
 		npxsave_proc(p, 0);
 #endif
 
-	pmap_deactivate(p);
 	tss_free(p->p_md.md_tss_sel);
 	sched_exit(p);
 }
diff --git a/sys/arch/i386/include/cpu.h b/sys/arch/i386/include/cpu.h
index c9115640e46..1380944ca1c 100644
--- a/sys/arch/i386/include/cpu.h
+++ b/sys/arch/i386/include/cpu.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: cpu.h,v 1.107 2009/02/16 15:44:25 jsg Exp $	*/
+/*	$OpenBSD: cpu.h,v 1.108 2009/06/03 00:49:12 art Exp $	*/
 /*	$NetBSD: cpu.h,v 1.35 1996/05/05 19:29:26 christos Exp $	*/
 
 /*-
@@ -94,11 +94,10 @@ struct cpu_info {
 	struct proc *ci_fpcurproc;	/* current owner of the FPU */
 	int ci_fpsaving;		/* save in progress */
 
-	volatile u_int32_t ci_tlb_ipi_mask;
-
 	struct pcb *ci_curpcb;		/* VA of current HW PCB */
 	struct pcb *ci_idle_pcb;	/* VA of current PCB */
 	int ci_idle_tss_sel;		/* TSS selector of idle PCB */
+	struct pmap *ci_curpmap;
 
 	struct intrsource *ci_isources[MAX_INTR_SOURCES];
 	u_int32_t	ci_ipending;
diff --git a/sys/arch/i386/include/i82489var.h b/sys/arch/i386/include/i82489var.h
index 4ecce7c3e62..6af9d3ed971 100644
--- a/sys/arch/i386/include/i82489var.h
+++ b/sys/arch/i386/include/i82489var.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: i82489var.h,v 1.9 2008/06/26 05:42:10 ray Exp $	*/
+/*	$OpenBSD: i82489var.h,v 1.10 2009/06/03 00:49:12 art Exp $	*/
 /*	$NetBSD: i82489var.h,v 1.1.2.2 2000/02/21 18:46:14 sommerfeld Exp $	*/
 
 /*-
@@ -103,10 +103,12 @@ extern void Xintrltimer(void);
 #define LAPIC_IPI_INVLTLB		(LAPIC_IPI_OFFSET + 0)
 #define LAPIC_IPI_INVLPG		(LAPIC_IPI_OFFSET + 1)
 #define LAPIC_IPI_INVLRANGE		(LAPIC_IPI_OFFSET + 2)
+#define LAPIC_IPI_RELOADCR3		(LAPIC_IPI_OFFSET + 3)
 
 extern void Xintripi_invltlb(void);
 extern void Xintripi_invlpg(void);
 extern void Xintripi_invlrange(void);
+extern void Xintripi_reloadcr3(void);
 
 extern void Xintrsoftclock(void);
 extern void Xintrsoftnet(void);
diff --git a/sys/arch/i386/include/pmap.h b/sys/arch/i386/include/pmap.h
index 89af1a42440..6b989e0f59e 100644
--- a/sys/arch/i386/include/pmap.h
+++ b/sys/arch/i386/include/pmap.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: pmap.h,v 1.51 2009/02/05 01:13:21 oga Exp $	*/
+/*	$OpenBSD: pmap.h,v 1.52 2009/06/03 00:49:12 art Exp $	*/
 /*	$NetBSD: pmap.h,v 1.44 2000/04/24 17:18:18 thorpej Exp $	*/
 
 /*
@@ -272,7 +272,6 @@ struct pmap {
 	union descriptor *pm_ldt;	/* user-set LDT */
 	int pm_ldt_len;			/* number of LDT entries */
 	int pm_ldt_sel;			/* LDT selector */
-	uint32_t pm_cpus;		/* mask of CPUs using map */
 };
 
 /* pm_flags */
@@ -384,6 +383,7 @@ void		pmap_write_protect(struct pmap *, vaddr_t,
 				vaddr_t, vm_prot_t);
 int		pmap_exec_fixup(struct vm_map *, struct trapframe *,
 		    struct pcb *);
+void		pmap_switch(struct proc *, struct proc *);
 
 vaddr_t reserve_dumppages(vaddr_t); /* XXX: not a pmap fn */
 
@@ -391,6 +391,7 @@ void	pmap_tlb_shootpage(struct pmap *, vaddr_t);
 void	pmap_tlb_shootrange(struct pmap *, vaddr_t, vaddr_t);
 void	pmap_tlb_shoottlb(void);
 #ifdef MULTIPROCESSOR
+void	pmap_tlb_droppmap(struct pmap *);
 void	pmap_tlb_shootwait(void);
 #else
 #define pmap_tlb_shootwait()