Faster signal delivery on i386/MP.

We need to poke the other CPU so that it processes the AST immediately
and doesn't wait for the next interrupt or syscall.

Since IPIs really shouldn't process ASTs, we need to trigger a soft
interrupt on the destination CPU to process the AST. But since we can't
send soft interrupts to other CPUs, we send an IPI, that triggers a soft
interrupt that in turn processes the AST.

Also, this marks the beginning of moving to slightly better IPI mechanism
of short and optimized IPIs instead of the large and complicated IPI
infrastructure we're using now.

tested by many, ok tholo@

9 files changed, 106 insertions, 32 deletions

diff --git a/sys/arch/i386/i386/apicvec.s b/sys/arch/i386/i386/apicvec.s
index aa6ee91a1fb..d6422e9ebf3 100644
--- a/sys/arch/i386/i386/apicvec.s
+++ b/sys/arch/i386/i386/apicvec.s
@@ -1,4 +1,4 @@
-/* $OpenBSD: apicvec.s,v 1.8 2006/11/15 14:40:50 mickey Exp $ */
+/* $OpenBSD: apicvec.s,v 1.9 2007/04/12 20:22:58 art Exp $ */
 /* $NetBSD: apicvec.s,v 1.1.2.2 2000/02/21 21:54:01 sommerfeld Exp $ */
 
 /*-
@@ -66,6 +66,26 @@ XINTR(ipi):
 	cli
 	popl	CPL
 	INTRFASTEXIT
+
+	.globl XINTR(ipi_ast)
+XINTR(ipi_ast):
+	pushl	%eax
+	pushl	%ds
+	movl	$GSEL(GDATA_SEL, SEL_KPL), %eax
+	movl	%eax, %ds
+
+	ioapic_asm_ack()
+
+	movl	$IPL_SOFTAST, %eax
+	orl	$(1 << SIR_AST), _C_LABEL(ipending)
+
+	orl	$(LAPIC_DLMODE_FIXED|LAPIC_LVL_ASSERT|LAPIC_DEST_SELF), %eax
+	movl	%eax, _C_LABEL(local_apic) + LAPIC_ICRLO
+
+	movl	_C_LABEL(local_apic) + LAPIC_ID, %eax
+	popl	%ds
+	popl	%eax
+	iret
 #endif
 
 	/*
@@ -94,7 +114,7 @@ XINTR(ltimer):
 #endif
 	jmp	_C_LABEL(Xdoreti)
 
-	.globl	XINTR(softclock), XINTR(softnet), XINTR(softtty)
+	.globl	XINTR(softclock), XINTR(softnet), XINTR(softtty), XINTR(softast)
 XINTR(softclock):
 	pushl	$0
 	pushl	$T_ASTFLT
@@ -162,6 +182,18 @@ XINTR(softtty):
 #endif
 	jmp	_C_LABEL(Xdoreti)
 
+XINTR(softast):
+	pushl	$0
+	pushl	$T_ASTFLT
+	INTRENTRY
+	MAKE_FRAME
+	pushl	CPL
+	movl	$IPL_SOFTAST,CPL
+	andl	$~(1<<SIR_AST),_C_LABEL(ipending)
+	ioapic_asm_ack()
+	sti
+	jmp	_C_LABEL(Xdoreti)
+
 #if NIOAPIC > 0
 
 #define voidop(num)
diff --git a/sys/arch/i386/i386/lapic.c b/sys/arch/i386/i386/lapic.c
index 3c05f8f3aec..2d8d07b0872 100644
--- a/sys/arch/i386/i386/lapic.c
+++ b/sys/arch/i386/i386/lapic.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: lapic.c,v 1.14 2007/03/19 09:29:33 art Exp $	*/
+/*	$OpenBSD: lapic.c,v 1.15 2007/04/12 20:22:58 art Exp $	*/
 /* $NetBSD: lapic.c,v 1.1.2.8 2000/02/23 06:10:50 sommerfeld Exp $ */
 
 /*-
@@ -122,6 +122,7 @@ lapic_set_softvectors()
 	idt_vec_set(LAPIC_SOFTCLOCK_VECTOR, Xintrsoftclock);
 	idt_vec_set(LAPIC_SOFTNET_VECTOR, Xintrsoftnet);
 	idt_vec_set(LAPIC_SOFTTTY_VECTOR, Xintrsofttty);
+	idt_vec_set(LAPIC_SOFTAST_VECTOR, Xintrsoftast);
 }
 
 void
@@ -176,9 +177,9 @@ lapic_set_lvt()
  * Initialize fixed idt vectors for use by local apic.
  */
 void
-lapic_boot_init(lapic_base)
-	paddr_t lapic_base;
+lapic_boot_init(paddr_t lapic_base)
 {
+	extern void Xintripi_ast(void);
 	static int clk_irq = 0;
 	static int ipi_irq = 0;
 
@@ -186,6 +187,7 @@ lapic_boot_init(lapic_base)
 
 #ifdef MULTIPROCESSOR
 	idt_vec_set(LAPIC_IPI_VECTOR, Xintripi);
+	idt_vec_set(LAPIC_IPI_AST, Xintripi_ast);
 #endif
 	idt_vec_set(LAPIC_SPURIOUS_VECTOR, Xintrspurious);
 	idt_vec_set(LAPIC_TIMER_VECTOR, Xintrltimer);
diff --git a/sys/arch/i386/i386/machdep.c b/sys/arch/i386/i386/machdep.c
index 765ed409e9c..78445594e39 100644
--- a/sys/arch/i386/i386/machdep.c
+++ b/sys/arch/i386/i386/machdep.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: machdep.c,v 1.381 2007/04/03 10:14:47 art Exp $	*/
+/*	$OpenBSD: machdep.c,v 1.382 2007/04/12 20:22:58 art Exp $	*/
 /*	$NetBSD: machdep.c,v 1.214 1996/11/10 03:16:17 thorpej Exp $	*/
 
 /*-
@@ -2196,6 +2196,29 @@ ibcs2_sendsig(sig_t catcher, int sig, int mask, u_long code, int type,
 #endif
 
 /*
+ * To send an AST to a process on another cpu we send an IPI to that cpu,
+ * the IPI schedules a special soft interrupt (that does nothing) and then
+ * returns through the normal interrupt return path which in turn handles
+ * the AST.
+ *
+ * The IPI can't handle the AST because it usually requires grabbing the
+ * biglock and we can't afford spinning in the IPI handler with interrupts
+ * unlocked (so that we take further IPIs and grow our stack until it
+ * overflows).
+ */
+void
+aston(struct proc *p)
+{
+#ifdef MULTIPROCESSOR
+	if (i386_atomic_testset_i(&p->p_md.md_astpending, 1) == 0 &&
+	    p->p_cpu != curcpu())
+		i386_ipi(LAPIC_IPI_AST, p->p_cpu->ci_cpuid, LAPIC_DLMODE_FIXED);
+#else
+	p->p_md.md_astpending = 1;
+#endif
+}
+
+/*
  * Send an interrupt to process.
  *
  * Stack is set up to allow sigcode stored
diff --git a/sys/arch/i386/i386/vector.s b/sys/arch/i386/i386/vector.s
index d8c84c8d8d5..0d49e54b00d 100644
--- a/sys/arch/i386/i386/vector.s
+++ b/sys/arch/i386/i386/vector.s
@@ -1,4 +1,4 @@
-/*	$OpenBSD: vector.s,v 1.9 2007/02/09 19:49:47 tom Exp $	*/
+/*	$OpenBSD: vector.s,v 1.10 2007/04/12 20:22:58 art Exp $	*/
 /*	$NetBSD: vector.s,v 1.32 1996/01/07 21:29:47 mycroft Exp $	*/
 
 /*
@@ -241,9 +241,9 @@ IDTVEC(resume)
 	.long	_C_LABEL(Xresume_legacy12), _C_LABEL(Xresume_legacy13)
 	.long	_C_LABEL(Xresume_legacy14), _C_LABEL(Xresume_legacy15)
 	/* for soft interrupts */
-	.long	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
-	.long	_C_LABEL(Xsofttty), _C_LABEL(Xsoftnet)
-	.long	_C_LABEL(Xsoftclock)
+	.long	0, 0, 0, 0, 0, 0, 0, 0, 0, 0
+	.long	_C_LABEL(Xsoftast), _C_LABEL(Xsofttty) 
+	.long	_C_LABEL(Xsoftnet), _C_LABEL(Xsoftclock)
 	.long	0, 0
 /* fake interrupts to resume from splx() */
 IDTVEC(recurse)
@@ -256,7 +256,7 @@ IDTVEC(recurse)
 	.long	_C_LABEL(Xrecurse_legacy12), _C_LABEL(Xrecurse_legacy13)
 	.long	_C_LABEL(Xrecurse_legacy14), _C_LABEL(Xrecurse_legacy15)
 	/* for soft interrupts */
-	.long	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
-	.long	_C_LABEL(Xsofttty), _C_LABEL(Xsoftnet)
-	.long	_C_LABEL(Xsoftclock)
+	.long	0, 0, 0, 0, 0, 0, 0, 0, 0, 0
+	.long	_C_LABEL(Xsoftast), _C_LABEL(Xsofttty) 
+	.long	_C_LABEL(Xsoftnet), _C_LABEL(Xsoftclock)
 	.long	0, 0
diff --git a/sys/arch/i386/include/cpu.h b/sys/arch/i386/include/cpu.h
index de3390846aa..af74850d454 100644
--- a/sys/arch/i386/include/cpu.h
+++ b/sys/arch/i386/include/cpu.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: cpu.h,v 1.90 2007/04/03 10:14:47 art Exp $	*/
+/*	$OpenBSD: cpu.h,v 1.91 2007/04/12 20:22:58 art Exp $	*/
 /*	$NetBSD: cpu.h,v 1.35 1996/05/05 19:29:26 christos Exp $	*/
 
 /*-
@@ -217,7 +217,7 @@ extern void need_resched(struct cpu_info *);
  */
 #define	PROC_PC(p)		((p)->p_md.md_regs->tf_eip)
 
-#define aston(p)		((p)->p_md.md_astpending = 1)
+void aston(struct proc *);
 
 /*
  * Give a profiling tick to the current process when the user profiling
@@ -230,7 +230,7 @@ extern void need_resched(struct cpu_info *);
  * Notify the current process (p) that it has a signal pending,
  * process as soon as possible.
  */
-#define	signotify(p)		aston(p)
+#define signotify(p)		aston(p)
 
 /*
  * We need a machine-independent name for this.
diff --git a/sys/arch/i386/include/i82489var.h b/sys/arch/i386/include/i82489var.h
index 8bdf4eda158..653641bf713 100644
--- a/sys/arch/i386/include/i82489var.h
+++ b/sys/arch/i386/include/i82489var.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: i82489var.h,v 1.3 2006/03/29 15:02:27 mickey Exp $	*/
+/*	$OpenBSD: i82489var.h,v 1.4 2007/04/12 20:22:58 art Exp $	*/
 /*	$NetBSD: i82489var.h,v 1.1.2.2 2000/02/21 18:46:14 sommerfeld Exp $	*/
 
 /*-
@@ -102,10 +102,18 @@ extern void Xintrltimer(void);
 #define LAPIC_SOFTCLOCK_VECTOR		IPL_SOFTCLOCK
 #define LAPIC_SOFTNET_VECTOR		IPL_SOFTNET
 #define LAPIC_SOFTTTY_VECTOR		IPL_SOFTTTY
+#define LAPIC_SOFTAST_VECTOR		IPL_SOFTAST
+
+/*
+ * Special IPI vectors. We can use IDT 0xf0 - 0xff for this.
+ */
+#define LAPIC_IPI_OFFSET		0xf0
+#define LAPIC_IPI_AST			(LAPIC_IPI_OFFSET + 0)
 
 extern void Xintrsoftclock(void);
 extern void Xintrsoftnet(void);
 extern void Xintrsofttty(void);
+extern void Xintrsoftast(void);
 
 extern void (*apichandler[])(void);
 
diff --git a/sys/arch/i386/include/intrdefs.h b/sys/arch/i386/include/intrdefs.h
index 0bc73ad39f3..4f5083a79fe 100644
--- a/sys/arch/i386/include/intrdefs.h
+++ b/sys/arch/i386/include/intrdefs.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: intrdefs.h,v 1.6 2006/03/12 02:04:16 brad Exp $	*/
+/*	$OpenBSD: intrdefs.h,v 1.7 2007/04/12 20:22:58 art Exp $	*/
 /*	$NetBSD: intrdefs.h,v 1.2 2003/05/04 22:01:56 fvdl Exp $	*/
 
 #ifndef _i386_INTRDEFS_H
@@ -56,19 +56,20 @@
  * The level numbers are picked to fit into APIC vector priorities.
  */
 #define	IPL_NONE	0		/* nothing */
-#define	IPL_SOFTCLOCK	MAKEIPL(0)	/* timeouts */
-#define	IPL_SOFTNET	MAKEIPL(1)	/* protocol stacks */
-#define	IPL_BIO		MAKEIPL(2)	/* block I/O */
-#define	IPL_NET		MAKEIPL(3)	/* network */
-#define	IPL_SOFTTTY	MAKEIPL(4)	/* delayed terminal handling */
-#define	IPL_TTY		MAKEIPL(5)	/* terminal */
-#define	IPL_VM		MAKEIPL(6)	/* memory allocation */
-#define	IPL_AUDIO	MAKEIPL(7)	/* audio */
-#define	IPL_CLOCK	MAKEIPL(8)	/* clock */
-#define	IPL_STATCLOCK	MAKEIPL(9)	/* statclock */
+#define IPL_SOFTAST	MAKEIPL(0)	/* AST */
+#define	IPL_SOFTCLOCK	MAKEIPL(1)	/* timeouts */
+#define	IPL_SOFTNET	MAKEIPL(2)	/* protocol stacks */
+#define	IPL_BIO		MAKEIPL(3)	/* block I/O */
+#define	IPL_NET		MAKEIPL(4)	/* network */
+#define	IPL_SOFTTTY	MAKEIPL(5)	/* delayed terminal handling */
+#define	IPL_TTY		MAKEIPL(6)	/* terminal */
+#define	IPL_VM		MAKEIPL(7)	/* memory allocation */
+#define	IPL_AUDIO	MAKEIPL(8)	/* audio */
+#define	IPL_CLOCK	MAKEIPL(9)	/* clock */
+#define	IPL_STATCLOCK	MAKEIPL(10)	/* statclock */
 #define	IPL_SCHED	IPL_STATCLOCK
-#define	IPL_HIGH	MAKEIPL(9)	/* everything */
-#define	IPL_IPI		MAKEIPL(10)	/* interprocessor interrupt */
+#define	IPL_HIGH	MAKEIPL(11)	/* everything */
+#define	IPL_IPI		MAKEIPL(12)	/* interprocessor interrupt */
 
 /* Interrupt sharing types. */
 #define	IST_NONE	0	/* none */
@@ -87,6 +88,7 @@
 #define	SIR_CLOCK	29
 #define	SIR_NET		28
 #define	SIR_TTY		27
+#define SIR_AST		26
 
 
 /*
diff --git a/sys/arch/i386/isa/icu.s b/sys/arch/i386/isa/icu.s
index 743c2dc7c21..ff1693ec67a 100644
--- a/sys/arch/i386/isa/icu.s
+++ b/sys/arch/i386/isa/icu.s
@@ -1,4 +1,4 @@
-/*	$OpenBSD: icu.s,v 1.23 2005/12/13 16:14:49 aaron Exp $	*/
+/*	$OpenBSD: icu.s,v 1.24 2007/04/12 20:22:58 art Exp $	*/
 /*	$NetBSD: icu.s,v 1.45 1996/01/07 03:59:34 mycroft Exp $	*/
 
 /*-
@@ -123,6 +123,12 @@ IDTVEC(doreti)
 
 #include "pccom.h"
 
+IDTVEC(softast)
+	movl	$IPL_SOFTAST,%eax
+	movl	%eax,CPL
+	sti
+	jmp	*%esi
+
 IDTVEC(softtty)
 #if NPCCOM > 0
 	movl	$IPL_SOFTTTY,%eax
diff --git a/sys/arch/i386/isa/isa_machdep.c b/sys/arch/i386/isa/isa_machdep.c
index e1010530402..9dce5c2c7aa 100644
--- a/sys/arch/i386/isa/isa_machdep.c
+++ b/sys/arch/i386/isa/isa_machdep.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: isa_machdep.c,v 1.58 2007/03/22 19:31:59 miod Exp $	*/
+/*	$OpenBSD: isa_machdep.c,v 1.59 2007/04/12 20:22:58 art Exp $	*/
 /*	$NetBSD: isa_machdep.c,v 1.22 1997/06/12 23:57:32 thorpej Exp $	*/
 
 /*-
@@ -333,6 +333,7 @@ intr_calculatemasks(void)
 	/*
 	 * Initialize soft interrupt masks to block themselves.
 	 */
+	IMASK(IPL_SOFTAST) |= 1 << SIR_AST;
 	IMASK(IPL_SOFTCLOCK) |= 1 << SIR_CLOCK;
 	IMASK(IPL_SOFTNET) |= 1 << SIR_NET;
 	IMASK(IPL_SOFTTTY) |= 1 << SIR_TTY;