Bring back the curpmap change. It was missing a reload of the pmap on

curcpu when we were freeing a pmap. Tested and working for a few weeks
now, but I was a bit too busy to commit it earlier.

10 files changed, 150 insertions, 128 deletions

diff --git a/sys/arch/amd64/amd64/cpu.c b/sys/arch/amd64/amd64/cpu.c
index c9a747bb94b..d53d04e152c 100644
--- a/sys/arch/amd64/amd64/cpu.c
+++ b/sys/arch/amd64/amd64/cpu.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: cpu.c,v 1.24 2009/04/27 17:48:21 deraadt Exp $	*/
+/*	$OpenBSD: cpu.c,v 1.25 2009/05/28 09:05:33 art Exp $	*/
 /* $NetBSD: cpu.c,v 1.1 2003/04/26 18:39:26 fvdl Exp $ */
 
 /*-
@@ -294,6 +294,7 @@ cpu_attach(struct device *parent, struct device *self, void *aux)
 	pcb->pcb_cr0 = rcr0();
 	pcb->pcb_cr3 = pcb->pcb_pmap->pm_pdirpa;
 #endif
+	ci->ci_curpmap = pmap_kernel();
 
 	/* further PCB init done later. */
 
diff --git a/sys/arch/amd64/amd64/genassym.cf b/sys/arch/amd64/amd64/genassym.cf
index 9817549e5e6..7b223a6b66d 100644
--- a/sys/arch/amd64/amd64/genassym.cf
+++ b/sys/arch/amd64/amd64/genassym.cf
@@ -1,4 +1,4 @@
-#	$OpenBSD: genassym.cf,v 1.17 2009/04/27 17:48:21 deraadt Exp $
+#	$OpenBSD: genassym.cf,v 1.18 2009/05/28 09:05:33 art Exp $
 #	Written by Artur Grabowski art@openbsd.org, Public Domain
 
 include <sys/param.h>
@@ -90,9 +90,10 @@ member	pcb_cr0
 member	pcb_ldt_sel
 member	pcb_onfault
 member	pcb_fpcpu
+member	pcb_pmap
 
 struct	cpu_info
-member	CPU_INFO_SCRATCH	ci_scratch
+member	CPU_INFO_SCRATCH        ci_scratch
 member  CPU_INFO_SELF		ci_self
 member  CPU_INFO_RESCHED	ci_want_resched
 member  CPU_INFO_CURPROC	ci_curproc
@@ -105,6 +106,10 @@ member	CPU_INFO_ISOURCES	ci_isources
 member	CPU_INFO_IPENDING	ci_ipending
 member	CPU_INFO_IUNMASK	ci_iunmask
 member	CPU_INFO_GDT		ci_gdt
+member	CPU_INFO_CURPMAP	ci_curpmap
+
+struct	pmap
+member	pm_pdirpa
 
 struct	intrsource
 member	is_recurse
diff --git a/sys/arch/amd64/amd64/lapic.c b/sys/arch/amd64/amd64/lapic.c
index be5b6d8be6d..a07552829a6 100644
--- a/sys/arch/amd64/amd64/lapic.c
+++ b/sys/arch/amd64/amd64/lapic.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: lapic.c,v 1.17 2009/04/27 17:48:22 deraadt Exp $	*/
+/*	$OpenBSD: lapic.c,v 1.18 2009/05/28 09:05:33 art Exp $	*/
 /* $NetBSD: lapic.c,v 1.2 2003/05/08 01:04:35 fvdl Exp $ */
 
 /*-
@@ -234,6 +234,8 @@ lapic_boot_init(paddr_t lapic_base)
 	idt_vec_set(LAPIC_IPI_INVLPG, Xipi_invlpg);
 	idt_allocmap[LAPIC_IPI_INVLRANGE] = 1;
 	idt_vec_set(LAPIC_IPI_INVLRANGE, Xipi_invlrange);
+	idt_allocmap[LAPIC_IPI_RELOADCR3] = 1;
+	idt_vec_set(LAPIC_IPI_RELOADCR3, Xipi_reloadcr3);
 #endif
 	idt_allocmap[LAPIC_SPURIOUS_VECTOR] = 1;
 	idt_vec_set(LAPIC_SPURIOUS_VECTOR, Xintrspurious);
diff --git a/sys/arch/amd64/amd64/locore.S b/sys/arch/amd64/amd64/locore.S
index 242444a2a1a..694d16314ed 100644
--- a/sys/arch/amd64/amd64/locore.S
+++ b/sys/arch/amd64/amd64/locore.S
@@ -1,4 +1,4 @@
-/*	$OpenBSD: locore.S,v 1.34 2009/04/27 17:48:22 deraadt Exp $	*/
+/*	$OpenBSD: locore.S,v 1.35 2009/05/28 09:05:33 art Exp $	*/
 /*	$NetBSD: locore.S,v 1.13 2004/03/25 18:33:17 drochner Exp $	*/
 
 /*
@@ -132,20 +132,10 @@
 
 #include <machine/asm.h>
 
-#define SET_CURPROC(proc,cpu)			\
-	movq	CPUVAR(SELF),cpu	;	\
-	movq	proc,CPUVAR(CURPROC)      ;	\
-	movq	cpu,P_CPU(proc)
-
-#define GET_CURPCB(reg)			movq	CPUVAR(CURPCB),reg      
-#define SET_CURPCB(reg)			movq	reg,CPUVAR(CURPCB)
-
-
 /* XXX temporary kluge; these should not be here */
 /* Get definitions for IOM_BEGIN, IOM_END, and IOM_SIZE */
 #include <dev/isa/isareg.h>
 
-
 /*
  * Initialization
  */
@@ -726,43 +716,29 @@ ENTRY(cpu_switchto)
 	pushq	%r14
 	pushq	%r15
 
-	movq	%rdi, %r13
-	movq	%rsi, %r12
-
 #ifdef DIAGNOSTIC
 	xorq	%rax,%rax
-	cmpq	%rax,P_WCHAN(%r12)
+	cmpq	%rax,P_WCHAN(%rsi)
 	jne	_C_LABEL(switch_error2)
-	cmpb	$SRUN,P_STAT(%r12)
+	cmpb	$SRUN,P_STAT(%rsi)
 	jne	_C_LABEL(switch_error3)
 #endif
 
+	/* No interrupts while loading new state. */
+	cli
+
 	/* Record new proc. */
-	movb	$SONPROC,P_STAT(%r12)	# p->p_stat = SONPROC
-	SET_CURPROC(%r12,%rcx)
+	movb	$SONPROC,P_STAT(%rsi)	# p->p_stat = SONPROC
+	movq	%rsi, CPUVAR(CURPROC)
 
-	/* If old proc exited, don't bother. */
-	testq	%r13,%r13
+	/* If old proc exited, don't bother saving state. */
+	testq	%rdi,%rdi
 	jz	switch_exited
 
-	/*
-	 * Save old context.
-	 *
-	 * Registers:
-	 *   %rax, %rcx - scratch
-	 *   %r13 - old proc, then old pcb
-	 *   %r12 - new proc
-	 */
-
-	movq	%r13,%rdi
-	call	pmap_deactivate
-
-	movq	P_ADDR(%r13),%r13
-
 	/* Save stack pointers. */
+	movq	P_ADDR(%rdi),%r13
 	movq	%rsp,PCB_RSP(%r13)
 	movq	%rbp,PCB_RBP(%r13)
-
 switch_exited:
 	/*
 	 * Restore saved context.
@@ -770,41 +746,26 @@ switch_exited:
 	 * Registers:
 	 *   %rax, %rcx, %rdx - scratch
 	 *   %r13 - new pcb
-	 *   %r12 - new process
 	 */
 
-	/* No interrupts while loading new state. */
-	cli
-	movq	P_ADDR(%r12),%r13
-
 	/* Restore stack pointers. */
+	movq	P_ADDR(%rsi),%r13
 	movq	PCB_RSP(%r13),%rsp
 	movq	PCB_RBP(%r13),%rbp
 
-#if 0
-	/* Don't bother with the rest if switching to a system process. */
-	testl	$P_SYSTEM,P_FLAG(%r12)
-	jnz	switch_restored
-#endif
+	movq	%r13, CPUVAR(CURPCB)
 
 	/* Load TSS info. */
-#ifdef MULTIPROCESSOR
 	movq    CPUVAR(GDT),%rax
-#else   
-	movq	_C_LABEL(gdtstore)(%rip),%rax
-#endif
-	movl	P_MD_TSS_SEL(%r12),%edx
+	movl	P_MD_TSS_SEL(%rsi),%edx
 
 	/* Switch TSS. Reset "task busy" flag before */
 	andl	$~0x0200,4(%rax,%rdx, 1)
 	ltr	%dx
 
-	movq	%r12,%rdi
-	call	_C_LABEL(pmap_activate)
+	call	_C_LABEL(pmap_switch)
+	/* %rsi and %rdi no longer valid */
 
-#if 0
-switch_restored:
-#endif
 	/* Restore cr0 (including FPU state). */
 	movl	PCB_CR0(%r13),%ecx
 #ifdef MULTIPROCESSOR
@@ -816,13 +777,9 @@ switch_restored:
 #endif
 	movq	%rcx,%cr0
 
-	SET_CURPCB(%r13)
-
 	/* Interrupts are okay again. */
 	sti
 
-switch_return:
-
 	popq	%r15
 	popq	%r14
 	popq	%r13
diff --git a/sys/arch/amd64/amd64/pmap.c b/sys/arch/amd64/amd64/pmap.c
index 467543dac21..4bd4ba51f9f 100644
--- a/sys/arch/amd64/amd64/pmap.c
+++ b/sys/arch/amd64/amd64/pmap.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: pmap.c,v 1.41 2009/04/27 17:48:22 deraadt Exp $	*/
+/*	$OpenBSD: pmap.c,v 1.42 2009/05/28 09:05:33 art Exp $	*/
 /*	$NetBSD: pmap.c,v 1.3 2003/05/08 18:13:13 thorpej Exp $	*/
 
 /*
@@ -313,7 +313,7 @@ struct vm_page *pmap_find_ptp(struct pmap *, vaddr_t, paddr_t, int);
 void pmap_free_ptp(struct pmap *, struct vm_page *,
     vaddr_t, pt_entry_t *, pd_entry_t **, struct pg_to_free *);
 void pmap_freepage(struct pmap *, struct vm_page *, int, struct pg_to_free *);
-static boolean_t pmap_is_active(struct pmap *, int);
+static boolean_t pmap_is_active(struct pmap *, struct cpu_info *);
 void pmap_map_ptes(struct pmap *, pt_entry_t **, pd_entry_t ***);
 struct pv_entry *pmap_remove_pv(struct vm_page *, struct pmap *, vaddr_t);
 void pmap_do_remove(struct pmap *, vaddr_t, vaddr_t, int);
@@ -328,7 +328,7 @@ void pmap_unmap_ptes(struct pmap *);
 boolean_t pmap_get_physpage(vaddr_t, int, paddr_t *);
 boolean_t pmap_pdes_valid(vaddr_t, pd_entry_t **, pd_entry_t *);
 void pmap_alloc_level(pd_entry_t **, vaddr_t, int, long *);
-void pmap_apte_flush(struct pmap *pmap);
+void pmap_apte_flush(void);
 
 void pmap_sync_flags_pte(struct vm_page *, u_long);
 
@@ -337,28 +337,27 @@ void pmap_sync_flags_pte(struct vm_page *, u_long);
  */
 
 /*
- * pmap_is_curpmap: is this pmap the one currently loaded [in %cr3]?
- *		of course the kernel is always loaded
+ * pmap_is_active: is this pmap loaded into the specified processor's %cr3?
  */
 
 static __inline boolean_t
-pmap_is_curpmap(struct pmap *pmap)
+pmap_is_active(struct pmap *pmap, struct cpu_info *ci)
 {
-	return((pmap == pmap_kernel()) ||
-	       (pmap->pm_pdirpa == (paddr_t) rcr3()));
+	return (pmap == pmap_kernel() || ci->ci_curpmap == pmap);
 }
 
 /*
- * pmap_is_active: is this pmap loaded into the specified processor's %cr3?
+ * pmap_is_curpmap: is this pmap the one currently loaded [in %cr3]?
+ *		of course the kernel is always loaded
  */
 
 static __inline boolean_t
-pmap_is_active(struct pmap *pmap, int cpu_id)
+pmap_is_curpmap(struct pmap *pmap)
 {
-	return (pmap == pmap_kernel() ||
-	    (pmap->pm_cpus & (1U << cpu_id)) != 0);
+	return (pmap_is_active(pmap, curcpu()));
 }
 
+
 static __inline u_int
 pmap_pte2flags(u_long pte)
 {
@@ -375,7 +374,7 @@ pmap_sync_flags_pte(struct vm_page *pg, u_long pte)
 }
 
 void
-pmap_apte_flush(struct pmap *pmap)
+pmap_apte_flush(void)
 {
 	pmap_tlb_shoottlb();
 	pmap_tlb_shootwait();
@@ -406,7 +405,7 @@ pmap_map_ptes(struct pmap *pmap, pt_entry_t **ptepp, pd_entry_t ***pdeppp)
 		npde = (pd_entry_t) (pmap->pm_pdirpa | PG_RW | PG_V);
 		*APDP_PDE = npde;
 		if (pmap_valid_entry(opde))
-			pmap_apte_flush(curpcb->pcb_pmap);
+			pmap_apte_flush();
 	}
 	*ptepp = APTE_BASE;
 	*pdeppp = alternate_pdes;
@@ -420,7 +419,7 @@ pmap_unmap_ptes(struct pmap *pmap)
 
 #if defined(MULTIPROCESSOR)
 	*APDP_PDE = 0;
-	pmap_apte_flush(curpcb->pcb_pmap);
+	pmap_apte_flush();
 #endif
 	COUNT(apdp_pde_unmap);
 }
@@ -854,7 +853,7 @@ pmap_free_ptp(struct pmap *pmap, struct vm_page *ptp, vaddr_t va,
 		opde = pmap_pte_set(&pdes[level - 1][index], 0);
 		invaladdr = level == 1 ? (vaddr_t)ptes :
 		    (vaddr_t)pdes[level - 2];
-		pmap_tlb_shootpage(curpcb->pcb_pmap,
+		pmap_tlb_shootpage(curcpu()->ci_curpmap,
 		    invaladdr + index * PAGE_SIZE);
 #if defined(MULTIPROCESSOR)
 		invaladdr = level == 1 ? (vaddr_t)PTE_BASE :
@@ -1081,6 +1080,8 @@ pmap_destroy(struct pmap *pmap)
 	 * reference count is zero, free pmap resources and then free pmap.
 	 */
 
+	/* Make sure it's not used by some other cpu. */
+	pmap_tlb_droppmap(pmap);
 	/*
 	 * remove it from global list of pmaps
 	 */
@@ -1099,13 +1100,7 @@ pmap_destroy(struct pmap *pmap)
 		}
 	}
 
-	/*
-	 * MULTIPROCESSOR -- no need to flush out of other processors'
-	 * APTE space because we do that in pmap_unmap_ptes().
-	 */
-	/* XXX: need to flush it out of other processor's APTE space? */
 	pool_put(&pmap_pdp_pool, pmap->pm_pdir);
-
 	pool_put(&pmap_pmap_pool, pmap);
 }
 
@@ -1121,29 +1116,14 @@ pmap_reference(struct pmap *pmap)
 
 /*
  * pmap_activate: activate a process' pmap (fill in %cr3 and LDT info)
- *
- * => called from cpu_switch()
- * => if p is the curproc, then load it into the MMU
  */
-
 void
 pmap_activate(struct proc *p)
 {
-	struct pcb *pcb = &p->p_addr->u_pcb;
-	struct pmap *pmap = p->p_vmspace->vm_map.pmap;
-
-	pcb->pcb_pmap = pmap;
-	pcb->pcb_ldt_sel = pmap->pm_ldt_sel;
-	pcb->pcb_cr3 = pmap->pm_pdirpa;
-	if (p == curproc)
-		lcr3(pcb->pcb_cr3);
-	if (pcb == curpcb)
-		lldt(pcb->pcb_ldt_sel);
+	KASSERT(p == curproc);
+	KASSERT(&p->p_addr->u_pcb == curpcb);
 
-	/*
-	 * mark the pmap in use by this processor.
-	 */
-	x86_atomic_setbits_ul(&pmap->pm_cpus, (1U << cpu_number()));
+	pmap_switch(NULL, p);
 }
 
 /*
@@ -1153,13 +1133,41 @@ pmap_activate(struct proc *p)
 void
 pmap_deactivate(struct proc *p)
 {
-	struct pmap *pmap = p->p_vmspace->vm_map.pmap;
+}
+
+u_int64_t nlazy_cr3;
+u_int64_t nlazy_cr3_hit;
+
+void
+pmap_switch(struct proc *o, struct proc *n)
+{
+	struct pmap *npmap, *opmap;
+	struct pcb *npcb;
+
+	npmap = n->p_vmspace->vm_map.pmap;
+
+	npcb = &n->p_addr->u_pcb;
+	npcb->pcb_pmap = npmap;
+	npcb->pcb_ldt_sel = npmap->pm_ldt_sel;
+	npcb->pcb_cr3 = npmap->pm_pdirpa;
+
+	opmap = curcpu()->ci_curpmap;
 
 	/*
-	 * mark the pmap no longer in use by this processor. 
+	 * Don't reload cr3 if we're switching to the same pmap or
+	 * when we're not exiting and switching to kernel pmap.
 	 */
-	x86_atomic_clearbits_ul(&pmap->pm_cpus, (1U << cpu_number()));
+	if (opmap == npmap) {
+		if (npmap != pmap_kernel())
+			nlazy_cr3_hit++;
+	} else if (o != NULL && npmap == pmap_kernel()) {
+		nlazy_cr3++;
+	} else {
+		curcpu()->ci_curpmap = npmap;
+		lcr3(npmap->pm_pdirpa);
+	}
 
+	lldt(npcb->pcb_ldt_sel);
 }
 
 /*
@@ -2457,7 +2465,7 @@ pmap_tlb_shootpage(struct pmap *pm, vaddr_t va)
 	int mask = 0;
 
 	CPU_INFO_FOREACH(cii, ci) {
-		if (ci == self || !pmap_is_active(pm, ci->ci_cpuid) ||
+		if (ci == self || !pmap_is_active(pm, ci) ||
 		    !(ci->ci_flags & CPUF_RUNNING))
 			continue;
 		mask |= 1 << ci->ci_cpuid;
@@ -2495,7 +2503,7 @@ pmap_tlb_shootrange(struct pmap *pm, vaddr_t sva, vaddr_t eva)
 	vaddr_t va;
 
 	CPU_INFO_FOREACH(cii, ci) {
-		if (ci == self || !pmap_is_active(pm, ci->ci_cpuid) ||
+		if (ci == self || !pmap_is_active(pm, ci) ||
 		    !(ci->ci_flags & CPUF_RUNNING))
 			continue;
 		mask |= 1 << ci->ci_cpuid;
@@ -2561,6 +2569,42 @@ pmap_tlb_shoottlb(void)
 }
 
 void
+pmap_tlb_droppmap(struct pmap *pm)
+{
+	struct cpu_info *ci, *self = curcpu();
+	CPU_INFO_ITERATOR cii;
+	long wait = 0;
+	int mask = 0;
+
+	CPU_INFO_FOREACH(cii, ci) {
+		if (ci == self || !(ci->ci_flags & CPUF_RUNNING) ||
+		    ci->ci_curpmap != pm)
+			continue;
+		mask |= 1 << ci->ci_cpuid;
+		wait++;
+	}
+
+	if (wait) {
+		int s = splvm();
+
+		while (x86_atomic_cas_ul(&tlb_shoot_wait, 0, wait) != 0) {
+			while (tlb_shoot_wait != 0)
+				SPINLOCK_SPIN_HOOK;
+		}
+
+		CPU_INFO_FOREACH(cii, ci) {
+			if ((mask & 1 << ci->ci_cpuid) == 0)
+				continue;
+			if (x86_fast_ipi(ci, LAPIC_IPI_RELOADCR3) != 0)
+				panic("pmap_tlb_shoottlb: ipi failed");
+		}
+		splx(s);
+	}
+
+	pmap_activate(curproc);
+}
+
+void
 pmap_tlb_shootwait(void)
 {
 	while (tlb_shoot_wait != 0)
diff --git a/sys/arch/amd64/amd64/vector.S b/sys/arch/amd64/amd64/vector.S
index 133fef7330c..6ad9d2f92c6 100644
--- a/sys/arch/amd64/amd64/vector.S
+++ b/sys/arch/amd64/amd64/vector.S
@@ -1,4 +1,4 @@
-/*	$OpenBSD: vector.S,v 1.18 2009/04/27 17:48:22 deraadt Exp $	*/
+/*	$OpenBSD: vector.S,v 1.19 2009/05/28 09:05:33 art Exp $	*/
 /*	$NetBSD: vector.S,v 1.5 2004/06/28 09:13:11 fvdl Exp $	*/
 
 /*
@@ -347,6 +347,23 @@ IDTVEC(ipi_invlrange)
 	popq	%rax
 	iretq
 
+IDTVEC(ipi_reloadcr3)
+	pushq	%rax
+
+	ioapic_asm_ack()
+
+	movq	CPUVAR(CURPCB), %rax
+	movq	PCB_PMAP(%rax), %rax
+	movq	%rax, CPUVAR(CURPMAP)
+	movq	PM_PDIRPA(%rax), %rax
+	movq	%rax, %cr3
+
+	lock
+	decq	tlb_shoot_wait
+
+	popq	%rax
+	iretq
+
 #endif /* MULTIPROCESSOR */
 	
 	/*
diff --git a/sys/arch/amd64/amd64/vm_machdep.c b/sys/arch/amd64/amd64/vm_machdep.c
index 5eba51d5c8d..0351ff860b6 100644
--- a/sys/arch/amd64/amd64/vm_machdep.c
+++ b/sys/arch/amd64/amd64/vm_machdep.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: vm_machdep.c,v 1.18 2009/04/27 17:48:22 deraadt Exp $	*/
+/*	$OpenBSD: vm_machdep.c,v 1.19 2009/05/28 09:05:33 art Exp $	*/
 /*	$NetBSD: vm_machdep.c,v 1.1 2003/04/26 18:39:33 fvdl Exp $	*/
 
 /*-
@@ -106,10 +106,9 @@ cpu_fork(struct proc *p1, struct proc *p2, void *stack, size_t stacksize,
 #endif
 	*pcb = p1->p_addr->u_pcb;
 
-	/*
-	 * Activate the address space.  Note this will refresh pcb_ldt_sel.
-	 */
-	pmap_activate(p2);
+	pcb->pcb_pmap = p2->p_vmspace->vm_map.pmap;
+	pcb->pcb_ldt_sel = p2->p_vmspace->vm_map.pmap->pm_ldt_sel;
+	pcb->pcb_cr3 = p2->p_vmspace->vm_map.pmap->pm_pdirpa;
 
 	/* Fix up the TSS. */
 	pcb->pcb_tss.tss_rsp0 = (u_int64_t)p2->p_addr + USPACE - 16;
@@ -159,7 +158,7 @@ cpu_exit(struct proc *p)
 	if (p->p_md.md_flags & MDP_USEDMTRR)
 		mtrr_clean(p);
 
-	pmap_deactivate(p);
+	pmap_tlb_droppmap(p->p_vmspace->vm_map.pmap);
 	tss_free(p->p_md.md_tss_sel);
 	sched_exit(p);
 }
diff --git a/sys/arch/amd64/include/cpu.h b/sys/arch/amd64/include/cpu.h
index 30567b2d792..599cbf4449b 100644
--- a/sys/arch/amd64/include/cpu.h
+++ b/sys/arch/amd64/include/cpu.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: cpu.h,v 1.43 2009/04/27 17:48:25 deraadt Exp $	*/
+/*	$OpenBSD: cpu.h,v 1.44 2009/05/28 09:05:33 art Exp $	*/
 /*	$NetBSD: cpu.h,v 1.1 2003/04/26 18:39:39 fvdl Exp $	*/
 
 /*-
@@ -69,8 +69,6 @@ struct cpu_info {
 	struct simplelock ci_slock;
 	u_int ci_cpuid;
 	u_int ci_apicid;
-	u_long ci_spin_locks;
-	u_long ci_simple_locks;
 	u_int32_t ci_randseed;
 
 	u_int64_t ci_scratch;
@@ -78,12 +76,12 @@ struct cpu_info {
 	struct proc *ci_fpcurproc;
 	int ci_fpsaving;
 
-	volatile u_int32_t ci_tlb_ipi_mask;
-
 	struct pcb *ci_curpcb;
 	struct pcb *ci_idle_pcb;
 	int ci_idle_tss_sel;
 
+	struct pmap *ci_curpmap;
+
 	struct intrsource *ci_isources[MAX_INTR_SOURCES];
 	u_int32_t	ci_ipending;
 	int		ci_ilevel;
@@ -91,7 +89,6 @@ struct cpu_info {
 	u_int32_t	ci_imask[NIPL];
 	u_int32_t	ci_iunmask[NIPL];
 
-	paddr_t 	ci_idle_pcb_paddr;
 	u_int		ci_flags;
 	u_int32_t	ci_ipis;
 
@@ -110,11 +107,6 @@ struct cpu_info {
 
 	struct x86_cache_info ci_cinfo[CAI_COUNT];
 
-	struct timeval 	ci_cc_time;
-	int64_t		ci_cc_cc;
-	int64_t		ci_cc_ms_delta;
-	int64_t		ci_cc_denom;
-
 	char		*ci_gdt;
 
 	volatile int	ci_ddb_paused;
diff --git a/sys/arch/amd64/include/i82489var.h b/sys/arch/amd64/include/i82489var.h
index 0cfbe6771c3..67f61beff9e 100644
--- a/sys/arch/amd64/include/i82489var.h
+++ b/sys/arch/amd64/include/i82489var.h
@@ -86,10 +86,12 @@ extern void Xresume_lapic_ipi(void);
 #define LAPIC_IPI_INVLTLB			(LAPIC_IPI_OFFSET + 0)
 #define LAPIC_IPI_INVLPG			(LAPIC_IPI_OFFSET + 1)
 #define LAPIC_IPI_INVLRANGE			(LAPIC_IPI_OFFSET + 2)
+#define LAPIC_IPI_RELOADCR3			(LAPIC_IPI_OFFSET + 3)
 
 extern void Xipi_invltlb(void);
 extern void Xipi_invlpg(void);
 extern void Xipi_invlrange(void);
+extern void Xipi_reloadcr3(void);
 
 /*
  * Vector used for local apic timer interrupts.
diff --git a/sys/arch/amd64/include/pmap.h b/sys/arch/amd64/include/pmap.h
index 4ebdfa62e8f..7d2f97a5d63 100644
--- a/sys/arch/amd64/include/pmap.h
+++ b/sys/arch/amd64/include/pmap.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: pmap.h,v 1.23 2009/04/27 17:48:25 deraadt Exp $	*/
+/*	$OpenBSD: pmap.h,v 1.24 2009/05/28 09:05:33 art Exp $	*/
 /*	$NetBSD: pmap.h,v 1.1 2003/04/26 18:39:46 fvdl Exp $	*/
 
 /*
@@ -325,7 +325,6 @@ struct pmap {
 	union descriptor *pm_ldt;	/* user-set LDT */
 	int pm_ldt_len;			/* number of LDT entries */
 	int pm_ldt_sel;			/* LDT selector */
-	u_int32_t pm_cpus;		/* mask of CPUs using pmap */
 };
 
 /*
@@ -416,6 +415,8 @@ static void	pmap_update_pg(vaddr_t);
 static void	pmap_update_2pg(vaddr_t,vaddr_t);
 void		pmap_write_protect(struct pmap *, vaddr_t,
 				vaddr_t, vm_prot_t);
+void 		pmap_switch(struct proc *, struct proc *);
+
 
 vaddr_t reserve_dumppages(vaddr_t); /* XXX: not a pmap fn */
 
@@ -423,8 +424,10 @@ void	pmap_tlb_shootpage(struct pmap *, vaddr_t);
 void	pmap_tlb_shootrange(struct pmap *, vaddr_t, vaddr_t);
 void	pmap_tlb_shoottlb(void);
 #ifdef MULTIPROCESSOR
+void	pmap_tlb_droppmap(struct pmap *);
 void	pmap_tlb_shootwait(void);
 #else
+#define	pmap_tlb_droppmap(pm)
 #define	pmap_tlb_shootwait()
 #endif