Change the old slow and complicated TLB shootdown code to new, fast and

simple. This is basically the same code as on i386 and basically the same
performance improvements.

This change also includes code to delay the freeing of ptps until they
have been properly shot.

in snaps for a week, no problems reported.

4 files changed, 35 insertions, 6 deletions

diff --git a/sys/arch/amd64/include/atomic.h b/sys/arch/amd64/include/atomic.h
index db184d78122..1df5a74ef2c 100644
--- a/sys/arch/amd64/include/atomic.h
+++ b/sys/arch/amd64/include/atomic.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: atomic.h,v 1.5 2007/02/19 17:18:42 deraadt Exp $	*/
+/*	$OpenBSD: atomic.h,v 1.6 2007/05/25 16:22:11 art Exp $	*/
 /*	$NetBSD: atomic.h,v 1.1 2003/04/26 18:39:37 fvdl Exp $	*/
 
 /*
@@ -90,6 +90,16 @@ x86_atomic_clearbits_u32(volatile u_int32_t *ptr, u_int32_t bits)
 	__asm __volatile(LOCK " andl %1,%0" :  "=m" (*ptr) : "ir" (~bits));
 }
 
+static __inline u_long
+x86_atomic_cas_ul(volatile u_long *ptr, u_long expect, u_long set)
+{
+	u_long res;
+
+	__asm volatile(LOCK " cmpxchgq %2, %1" : "=a" (res), "=m" (*ptr)
+	    : "r" (set), "a" (expect), "m" (*ptr) : "memory");
+
+	return (res);
+}
 
 /*
  * XXX XXX XXX
diff --git a/sys/arch/amd64/include/i82489var.h b/sys/arch/amd64/include/i82489var.h
index ebcf439f2df..1ae48071abb 100644
--- a/sys/arch/amd64/include/i82489var.h
+++ b/sys/arch/amd64/include/i82489var.h
@@ -87,6 +87,18 @@ extern void Xresume_lapic_ipi(void);
 #define LAPIC_IPI_VECTOR			0xe0
 
 /*
+ * We take 0xf0-0xfe for fast IPI handlers.
+ */
+#define LAPIC_IPI_OFFSET			0xf0
+#define LAPIC_IPI_INVLTLB			(LAPIC_IPI_OFFSET + 0)
+#define LAPIC_IPI_INVLPG			(LAPIC_IPI_OFFSET + 1)
+#define LAPIC_IPI_INVLRANGE			(LAPIC_IPI_OFFSET + 2)
+
+extern void Xipi_invltlb(void);
+extern void Xipi_invlpg(void);
+extern void Xipi_invlrange(void);
+
+/*
  * Vector used for local apic timer interrupts.
  */
 
diff --git a/sys/arch/amd64/include/intr.h b/sys/arch/amd64/include/intr.h
index a658b1c525d..17bbb6a115b 100644
--- a/sys/arch/amd64/include/intr.h
+++ b/sys/arch/amd64/include/intr.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: intr.h,v 1.10 2007/05/16 19:37:06 thib Exp $	*/
+/*	$OpenBSD: intr.h,v 1.11 2007/05/25 16:22:11 art Exp $	*/
 /*	$NetBSD: intr.h,v 1.2 2003/05/04 22:01:56 fvdl Exp $	*/
 
 /*-
@@ -223,6 +223,7 @@ void intr_printconfig(void);
 
 #ifdef MULTIPROCESSOR
 int x86_send_ipi(struct cpu_info *, int);
+int x86_fast_ipi(struct cpu_info *, int);
 void x86_broadcast_ipi(int);
 void x86_multicast_ipi(int, int);
 void x86_ipi_handler(void);
diff --git a/sys/arch/amd64/include/pmap.h b/sys/arch/amd64/include/pmap.h
index 6a53ba4384a..b06e2528580 100644
--- a/sys/arch/amd64/include/pmap.h
+++ b/sys/arch/amd64/include/pmap.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: pmap.h,v 1.11 2007/05/15 16:38:33 art Exp $	*/
+/*	$OpenBSD: pmap.h,v 1.12 2007/05/25 16:22:11 art Exp $	*/
 /*	$NetBSD: pmap.h,v 1.1 2003/04/26 18:39:46 fvdl Exp $	*/
 
 /*
@@ -412,9 +412,15 @@ void		pmap_write_protect(struct pmap *, vaddr_t,
 
 vaddr_t reserve_dumppages(vaddr_t); /* XXX: not a pmap fn */
 
-void	pmap_tlb_shootdown(pmap_t, vaddr_t, pt_entry_t, int32_t *);
-void	pmap_tlb_shootnow(int32_t);
-void	pmap_do_tlb_shootdown(struct cpu_info *);
+void	pmap_tlb_shootpage(struct pmap *, vaddr_t);
+void	pmap_tlb_shootrange(struct pmap *, vaddr_t, vaddr_t);
+void	pmap_tlb_shoottlb(void);
+#ifdef MULTIPROCESSOR
+void	pmap_tlb_shootwait(void);
+#else
+#define	pmap_tlb_shootwait()
+#endif
+
 void	pmap_prealloc_lowmem_ptps(void);
 
 void	pagezero(vaddr_t);