Replace the overdesigned and overcomplicated tlb shootdown code with

very simple and dumb fast tlb IPI handlers that have in the order of
the same amount of instructions as the old code had function calls.

All TLB shootdowns are reorganized so that we always shoot the,
without looking at PG_U and when we're shooting a range (primarily in
pmap_remove), we shoot the range when there are 32 or less pages in
it, otherwise we just nuke the whole TLB (this might need tweaking if
someone is interested in micro-optimization). The IPIs are not handled
through the normal interrupt vectoring code, they are not blockable
and they only shoot one page or a range of pages or the whole tlb.

This gives a 15% reduction in system time on my dual-core laptop
during a kernel compile and an 18% reduction in real time on a quad
machine doing bulk ports build.

Tested by many, in snaps for a week, no slowdowns reported (although not
everyone is seeing such huge wins).

4 files changed, 35 insertions, 7 deletions

diff --git a/sys/arch/i386/include/atomic.h b/sys/arch/i386/include/atomic.h
index 44a7be7f52f..35ea910c8fa 100644
--- a/sys/arch/i386/include/atomic.h
+++ b/sys/arch/i386/include/atomic.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: atomic.h,v 1.5 2007/02/19 17:18:42 deraadt Exp $	*/
+/*	$OpenBSD: atomic.h,v 1.6 2007/05/25 15:55:27 art Exp $	*/
 /* $NetBSD: atomic.h,v 1.1.2.2 2000/02/21 18:54:07 sommerfeld Exp $ */
 
 /*-
@@ -92,6 +92,20 @@ i386_atomic_clearbits_l(volatile u_int32_t *ptr, unsigned long bits)
 	__asm __volatile(LOCK " andl %1,%0" :  "=m" (*ptr) : "ir" (bits));
 }
 
+/*
+ * cas = compare and set
+ */
+static __inline int
+i486_atomic_cas_int(volatile u_int *ptr, u_int expect, u_int set)
+{
+	int res;
+
+	__asm volatile(LOCK " cmpxchgl %2, %1" : "=a" (res), "=m" (*ptr)
+	     : "r" (set), "a" (expect), "m" (*ptr) : "memory");
+
+	return (res);
+}
+
 #define atomic_setbits_int i386_atomic_setbits_l
 #define atomic_clearbits_int i386_atomic_clearbits_l
 
diff --git a/sys/arch/i386/include/i82489var.h b/sys/arch/i386/include/i82489var.h
index 653641bf713..0fe445e41fe 100644
--- a/sys/arch/i386/include/i82489var.h
+++ b/sys/arch/i386/include/i82489var.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: i82489var.h,v 1.4 2007/04/12 20:22:58 art Exp $	*/
+/*	$OpenBSD: i82489var.h,v 1.5 2007/05/25 15:55:27 art Exp $	*/
 /*	$NetBSD: i82489var.h,v 1.1.2.2 2000/02/21 18:46:14 sommerfeld Exp $	*/
 
 /*-
@@ -109,6 +109,14 @@ extern void Xintrltimer(void);
  */
 #define LAPIC_IPI_OFFSET		0xf0
 #define LAPIC_IPI_AST			(LAPIC_IPI_OFFSET + 0)
+#define LAPIC_IPI_INVLTLB		(LAPIC_IPI_OFFSET + 1)
+#define LAPIC_IPI_INVLPG		(LAPIC_IPI_OFFSET + 2)
+#define LAPIC_IPI_INVLRANGE		(LAPIC_IPI_OFFSET + 3)
+
+extern void Xintripi_ast(void);
+extern void Xintripi_invltlb(void);
+extern void Xintripi_invlpg(void);
+extern void Xintripi_invlrange(void);
 
 extern void Xintrsoftclock(void);
 extern void Xintrsoftnet(void);
diff --git a/sys/arch/i386/include/intr.h b/sys/arch/i386/include/intr.h
index eeeb74a605e..0d69c57277a 100644
--- a/sys/arch/i386/include/intr.h
+++ b/sys/arch/i386/include/intr.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: intr.h,v 1.31 2007/05/16 19:37:06 thib Exp $	*/
+/*	$OpenBSD: intr.h,v 1.32 2007/05/25 15:55:27 art Exp $	*/
 /*	$NetBSD: intr.h,v 1.5 1996/05/13 06:11:28 mycroft Exp $	*/
 
 /*
@@ -137,6 +137,7 @@ struct cpu_info;
 
 #ifdef MULTIPROCESSOR
 int i386_send_ipi(struct cpu_info *, int);
+int i386_fast_ipi(struct cpu_info *, int);
 void i386_broadcast_ipi(int);
 void i386_multicast_ipi(int, int);
 void i386_ipi_handler(void);
diff --git a/sys/arch/i386/include/pmap.h b/sys/arch/i386/include/pmap.h
index 6520a9bbebe..9f0ed360a1a 100644
--- a/sys/arch/i386/include/pmap.h
+++ b/sys/arch/i386/include/pmap.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: pmap.h,v 1.45 2007/04/26 11:31:52 art Exp $	*/
+/*	$OpenBSD: pmap.h,v 1.46 2007/05/25 15:55:27 art Exp $	*/
 /*	$NetBSD: pmap.h,v 1.44 2000/04/24 17:18:18 thorpej Exp $	*/
 
 /*
@@ -383,9 +383,14 @@ int		pmap_exec_fixup(struct vm_map *, struct trapframe *,
 
 vaddr_t reserve_dumppages(vaddr_t); /* XXX: not a pmap fn */
 
-void	pmap_tlb_shootdown(pmap_t, vaddr_t, pt_entry_t, int32_t *);
-void	pmap_tlb_shootnow(int32_t);
-void	pmap_do_tlb_shootdown(struct cpu_info *);
+void	pmap_tlb_shootpage(struct pmap *, vaddr_t);
+void	pmap_tlb_shootrange(struct pmap *, vaddr_t, vaddr_t);
+void	pmap_tlb_shoottlb(void);
+#ifdef MULTIPROCESSOR
+void	pmap_tlb_shootwait(void);
+#else
+#define pmap_tlb_shootwait()
+#endif
 
 #define PMAP_GROWKERNEL		/* turn on pmap_growkernel interface */