Split AST handling from trap() into ast() and get rid of T_ASTFLT.

Don't skip the AST check when returning from *fork() in the child.
Make sure to count interrupts even when they're deferred or stray.

testing by krw@, and then many via snapshots

6 files changed, 62 insertions, 65 deletions

diff --git a/sys/arch/amd64/amd64/db_trace.c b/sys/arch/amd64/amd64/db_trace.c
index 58f96e4cfb3..ef8863443c6 100644
--- a/sys/arch/amd64/amd64/db_trace.c
+++ b/sys/arch/amd64/amd64/db_trace.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: db_trace.c,v 1.12 2015/05/18 19:59:27 guenther Exp $	*/
+/*	$OpenBSD: db_trace.c,v 1.13 2015/06/28 01:16:28 guenther Exp $	*/
 /*	$NetBSD: db_trace.c,v 1.1 2003/04/26 18:39:27 fvdl Exp $	*/
 
 /* 
@@ -110,6 +110,7 @@ struct x86_64_frame {
 #define	TRAP		1
 #define	SYSCALL		2
 #define	INTERRUPT	3
+#define	AST		4
 
 db_addr_t	db_trap_symbol_value = 0;
 db_addr_t	db_syscall_symbol_value = 0;
@@ -179,6 +180,9 @@ db_nextframe(struct x86_64_frame **fp, db_addr_t *ip, long *argp, int is_trap,
 		case TRAP:
 			(*pr)("--- trap (number %d) ---\n", tf->tf_trapno);
 			break;
+		case AST:
+			(*pr)("--- ast ---\n");
+			break;
 		case SYSCALL:
 			(*pr)("--- syscall (number %ld) ---\n", tf->tf_rax);
 			break;
@@ -267,6 +271,8 @@ db_stack_trace_print(db_expr_t addr, boolean_t have_addr, db_expr_t count,
 		if (INKERNEL(frame) && name) {
 			if (!strcmp(name, "trap")) {
 				is_trap = TRAP;
+			} else if (!strcmp(name, "ast")) {
+				is_trap = AST;
 			} else if (!strcmp(name, "syscall")) {
 				is_trap = SYSCALL;
 			} else if (name[0] == 'X') {
diff --git a/sys/arch/amd64/amd64/locore.S b/sys/arch/amd64/amd64/locore.S
index c2e4d4f9508..7f4b28a9b78 100644
--- a/sys/arch/amd64/amd64/locore.S
+++ b/sys/arch/amd64/amd64/locore.S
@@ -1,4 +1,4 @@
-/*	$OpenBSD: locore.S,v 1.66 2015/06/23 14:19:21 bluhm Exp $	*/
+/*	$OpenBSD: locore.S,v 1.67 2015/06/28 01:16:28 guenther Exp $	*/
 /*	$NetBSD: locore.S,v 1.13 2004/03/25 18:33:17 drochner Exp $	*/
 
 /*
@@ -1039,34 +1039,34 @@ IDTVEC(syscall)
 	movq	$(GSEL(GUCODE_SEL, SEL_UPL)), TF_CS(%rsp)
 	movq	%rcx,TF_RIP(%rsp)
 	movq	$2,TF_ERR(%rsp)		/* ignored */
-	movq	$T_ASTFLT, TF_TRAPNO(%rsp)
 
 	movq	CPUVAR(CURPROC),%r14
 	movq	%rsp,P_MD_REGS(%r14)	# save pointer to frame
 	andl	$~MDP_IRET,P_MD_FLAGS(%r14)
 	movq	%rsp,%rdi
 	call	_C_LABEL(syscall)
-1:	/* Check for ASTs on exit to user mode. */
+
+.Lsyscall_check_asts:
+	/* Check for ASTs on exit to user mode. */
 	cli
 	CHECK_ASTPENDING(%r11)
 	je	2f
-	/* Always returning to user mode here. */
 	CLEAR_ASTPENDING(%r11)
 	sti
-	/* Pushed T_ASTFLT into tf_trapno on entry. */
 	movq	%rsp,%rdi
-	call	_C_LABEL(trap)
-	jmp	1b
+	call	_C_LABEL(ast)
+	jmp	.Lsyscall_check_asts
+
 2:
-	sti
-	testl	$MDP_IRET, P_MD_FLAGS(%r14)
-	jne	iret_return;
-syscall_return:
 #ifdef DIAGNOSTIC
 	cmpl	$IPL_NONE,CPUVAR(ILEVEL)
-	jne	3f
+	jne	.Lsyscall_spl_not_lowered
 #endif /* DIAGNOSTIC */
 
+	/* Could registers have been changed that require an iretq? */
+	testl	$MDP_IRET, P_MD_FLAGS(%r14)
+	jne	intr_fast_exit
+
 	movq	TF_RDI(%rsp),%rdi
 	movq	TF_RSI(%rsp),%rsi
 	movq	TF_R8(%rsp),%r8
@@ -1090,7 +1090,8 @@ syscall_return:
 	sysretq
 
 #ifdef DIAGNOSTIC
-3:	movabsq	$4f, %rdi
+.Lsyscall_spl_not_lowered:
+	movabsq	$4f, %rdi
 	movl	TF_RAX(%rsp),%esi
 	movl	TF_RDI(%rsp),%edx
 	movl	%ebx,%ecx
@@ -1101,7 +1102,7 @@ syscall_return:
 	int	$3
 #endif /* DDB */
 	movl	$IPL_NONE,CPUVAR(ILEVEL)
-	jmp	1b
+	jmp	.Lsyscall_check_asts
 4:	.asciz	"WARNING: SPL NOT LOWERED ON SYSCALL %d %d EXIT %x %x\n"
 #endif
 
@@ -1123,20 +1124,14 @@ NENTRY(child_trampoline)
 	movl	$IPL_NONE,CPUVAR(ILEVEL)
 	movq	%r13,%rdi
 	call	*%r12
-	jmp	syscall_return
+	movq	CPUVAR(CURPROC),%r14
+	jmp	.Lsyscall_check_asts
 
 
 /*
  * Return via iretq, for real interrupts and signal returns
  */
-iret_return:
-1:
-#ifdef DIAGNOSTIC
-	cmpl	$IPL_NONE,CPUVAR(ILEVEL)
-	jne	3f
-#endif /* DIAGNOSTIC */
-	.globl	intr_fast_exit
-intr_fast_exit:
+NENTRY(intr_fast_exit)
 	movq	TF_RDI(%rsp),%rdi
 	movq	TF_RSI(%rsp),%rsi
 	movq	TF_R8(%rsp),%r8
@@ -1164,19 +1159,6 @@ intr_fast_exit:
 _C_LABEL(doreti_iret):
 	iretq
 
-#ifdef DIAGNOSTIC
-3:	sti
-	movabsq	$4f, %rdi
-	xorq	%rax,%rax
-	call	_C_LABEL(printf)
-#ifdef DDB
-	int	$3
-#endif /* DDB */
-	movl	$IPL_NONE,CPUVAR(ILEVEL)
-	jmp	2b
-4:	.asciz	"WARNING: SPL NOT LOWERED ON SYSCALL EXIT\n"
-#endif /* DIAGNOSTIC */
-
 
 ENTRY(pagezero)
 	movq    $-PAGE_SIZE,%rdx
diff --git a/sys/arch/amd64/amd64/spl.S b/sys/arch/amd64/amd64/spl.S
index 33c5f9d1d8c..b296221f79b 100644
--- a/sys/arch/amd64/amd64/spl.S
+++ b/sys/arch/amd64/amd64/spl.S
@@ -1,4 +1,4 @@
-/*	$OpenBSD: spl.S,v 1.8 2011/06/16 19:46:39 kettenis Exp $	*/
+/*	$OpenBSD: spl.S,v 1.9 2015/06/28 01:16:28 guenther Exp $	*/
 /*	$NetBSD: spl.S,v 1.3 2004/06/28 09:13:11 fvdl Exp $	*/
 
 /*
@@ -166,10 +166,8 @@ IDTVEC(doreti)
 	jz	3f
 4:	CLEAR_ASTPENDING(%r11)
 	sti
-	movl	$T_ASTFLT,TF_TRAPNO(%rsp)	/* XXX undo later.. */
-	/* Pushed T_ASTFLT into tf_trapno on entry. */
 	movq	%rsp, %rdi
-	call	_C_LABEL(trap)
+	call	_C_LABEL(ast)
 	cli
 	jmp	5b
 3: 	INTRFASTEXIT
diff --git a/sys/arch/amd64/amd64/trap.c b/sys/arch/amd64/amd64/trap.c
index 3c452797354..2fcd708de08 100644
--- a/sys/arch/amd64/amd64/trap.c
+++ b/sys/arch/amd64/amd64/trap.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: trap.c,v 1.45 2015/04/18 05:14:05 guenther Exp $	*/
+/*	$OpenBSD: trap.c,v 1.46 2015/06/28 01:16:28 guenther Exp $	*/
 /*	$NetBSD: trap.c,v 1.2 2003/05/04 23:51:56 fvdl Exp $	*/
 
 /*-
@@ -98,13 +98,14 @@
 #endif
 
 void trap(struct trapframe *);
+void ast(struct trapframe *);
 void syscall(struct trapframe *);
 
 const char *trap_type[] = {
 	"privileged instruction fault",		/*  0 T_PRIVINFLT */
 	"breakpoint trap",			/*  1 T_BPTFLT */
 	"arithmetic trap",			/*  2 T_ARITHTRAP */
-	"asynchronous system trap",		/*  3 T_ASTFLT */
+	"reserved trap",			/*  3 T_RESERVED */
 	"protection fault",			/*  4 T_PROTFLT */
 	"trace trap",				/*  5 T_TRCTRAP */
 	"page fault",				/*  6 T_PAGEFLT */
@@ -121,7 +122,6 @@ const char *trap_type[] = {
 	"stack fault",				/* 17 T_STKFLT */
 	"machine check",			/* 18 T_MCA */
 	"SSE FP exception",			/* 19 T_XMM */
-	"reserved trap",			/* 20 T_RESERVED */
 };
 int	trap_types = nitems(trap_type);
 
@@ -142,7 +142,6 @@ static void frame_dump(struct trapframe *);
  * routines that prepare a suitable stack frame, and restore this
  * frame after the exception has been processed.
  */
-/*ARGSUSED*/
 void
 trap(struct trapframe *frame)
 {
@@ -278,12 +277,6 @@ copyfault:
 		trapsignal(p, SIGILL, type & ~T_USER, ILL_COPROC, sv);
 		KERNEL_UNLOCK();
 		goto out;
-
-	case T_ASTFLT|T_USER:		/* Allow process switch */
-		uvmexp.softs++;
-		mi_ast(p, curcpu()->ci_want_resched);
-		goto out;
-
 	case T_BOUND|T_USER:
 		sv.sival_ptr = (void *)frame->tf_rip;
 		KERNEL_LOCK();
@@ -480,6 +473,27 @@ frame_dump(struct trapframe *tf)
 }
 #endif
 
+
+/*
+ * ast(frame):
+ *	AST handler.  This is called from assembly language stubs when
+ *	returning to userspace after a syscall or interrupt.
+ */
+void
+ast(struct trapframe *frame)
+{
+	struct proc *p = curproc;
+
+	uvmexp.traps++;
+	KASSERT(!KERNELMODE(frame->tf_cs, frame->tf_rflags));
+	p->p_md.md_regs = frame;
+	refreshcreds(p);
+	uvmexp.softs++;
+	mi_ast(p, curcpu()->ci_want_resched);
+	userret(p);
+}
+
+
 /*
  * syscall(frame):
  *	System call request from POSIX system call gate interface to kernel.
diff --git a/sys/arch/amd64/amd64/vector.S b/sys/arch/amd64/amd64/vector.S
index 9e1d24408b6..d681958aaa9 100644
--- a/sys/arch/amd64/amd64/vector.S
+++ b/sys/arch/amd64/amd64/vector.S
@@ -1,4 +1,4 @@
-/*	$OpenBSD: vector.S,v 1.38 2015/05/18 19:59:27 guenther Exp $	*/
+/*	$OpenBSD: vector.S,v 1.39 2015/06/28 01:16:29 guenther Exp $	*/
 /*	$NetBSD: vector.S,v 1.5 2004/06/28 09:13:11 fvdl Exp $	*/
 
 /*
@@ -260,9 +260,8 @@ calltrap:
 	jz	1f
 5:	CLEAR_ASTPENDING(%r11)
 	sti
-	movl	$T_ASTFLT,TF_TRAPNO(%rsp)
 	movq	%rsp, %rdi
-	call	_C_LABEL(trap)
+	call	_C_LABEL(ast)
 	jmp	2b
 #ifndef DIAGNOSTIC
 1:	INTRFASTEXIT
@@ -326,12 +325,12 @@ _C_LABEL(x2apic_eoi):
 IDTVEC(recurse_lapic_ipi)
 	INTR_RECURSE_HWFRAME
 	pushq	$0		
-	pushq	$T_ASTFLT
+	subq	$8,%rsp			/* unused __if_trapno */
 	INTRENTRY		
 	jmp	1f
 IDTVEC(intr_lapic_ipi)
 	pushq	$0		
-	pushq	$T_ASTFLT
+	subq	$8,%rsp			/* unused __if_trapno */
 	INTRENTRY		
 	CODEPATCH_START
 	movl	$0,_C_LABEL(local_apic)+LAPIC_EOI
@@ -410,12 +409,12 @@ IDTVEC(ipi_invlrange)
 IDTVEC(recurse_lapic_ltimer)
 	INTR_RECURSE_HWFRAME
 	pushq	$0		
-	pushq	$T_ASTFLT
+	subq	$8,%rsp			/* unused __if_trapno */
 	INTRENTRY		
 	jmp	1f
 IDTVEC(intr_lapic_ltimer)
 	pushq	$0		
-	pushq	$T_ASTFLT
+	subq	$8,%rsp			/* unused __if_trapno */
 	INTRENTRY		
 	CODEPATCH_START
 	movl	$0,_C_LABEL(local_apic)+LAPIC_EOI
@@ -452,8 +451,7 @@ IDTVEC(resume_lapic_ltimer)
 #define	INTRSTUB(name, num, early_ack, late_ack, mask, unmask, level_mask) \
 IDTVEC(recurse_##name##num)						;\
 	INTR_RECURSE_HWFRAME						;\
-	subq	$8,%rsp							;\
-	pushq	$T_ASTFLT		/* trap # for doing ASTs */	;\
+	subq	$16,%rsp		/* space for __if_{trapno,err} */;\
 	INTRENTRY							;\
 IDTVEC(resume_##name##num)						\
 	movq	$IREENT_MAGIC,TF_ERR(%rsp)				;\
@@ -463,18 +461,18 @@ IDTVEC(resume_##name##num)						\
 	jmp	1f							;\
 IDTVEC(intr_##name##num)						;\
 	pushq	$0			/* dummy error code */		;\
-	pushq	$T_ASTFLT		/* trap # for doing ASTs */	;\
+	subq	$8,%rsp			/* unused __if_trapno */	;\
 	INTRENTRY							;\
 	movq	CPUVAR(ISOURCES) + (num) * 8, %r14			;\
 	mask(num)			/* mask it in hardware */	;\
 	early_ack(num)			/* and allow other intrs */	;\
+	incl	_C_LABEL(uvmexp)+V_INTR	/* statistical info */		;\
 	testq	%r14,%r14						;\
 	jz	9f			/* stray */			;\
 	movl	IS_MAXLEVEL(%r14),%ebx					;\
 	movl	CPUVAR(ILEVEL),%r13d					;\
 	cmpl	%ebx,%r13d						;\
 	jae	10f			/* currently masked; hold it */	;\
-	incl	_C_LABEL(uvmexp)+V_INTR	/* statistical info */		;\
 1:									\
 	pushq	%r13							;\
 	movl	%ebx,CPUVAR(ILEVEL)					;\
diff --git a/sys/arch/amd64/include/trap.h b/sys/arch/amd64/include/trap.h
index 32d30d5409f..067f8242dbf 100644
--- a/sys/arch/amd64/include/trap.h
+++ b/sys/arch/amd64/include/trap.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: trap.h,v 1.2 2005/12/13 00:18:19 jsg Exp $	*/
+/*	$OpenBSD: trap.h,v 1.3 2015/06/28 01:16:29 guenther Exp $	*/
 /*	$NetBSD: trap.h,v 1.4 1994/10/27 04:16:30 cgd Exp $	*/
 
 /*-
@@ -43,7 +43,7 @@
 #define	T_PRIVINFLT	 0	/* privileged instruction */
 #define	T_BPTFLT	 1	/* breakpoint trap */
 #define	T_ARITHTRAP	 2	/* arithmetic trap */
-#define	T_ASTFLT	 3	/* asynchronous system trap */
+#define	T_RESERVED	 3	/* reserved fault base */
 #define	T_PROTFLT	 4	/* protection fault */
 #define	T_TRCTRAP	 5	/* trace trap */
 #define	T_PAGEFLT	 6	/* page fault */
@@ -60,7 +60,6 @@
 #define	T_STKFLT	17	/* stack fault */
 #define	T_MCA		18	/* machine check ([P]Pro) */
 #define	T_XMM		19	/* SSE FP exception */
-#define	T_RESERVED	20	/* reserved fault base */
 
 /* Trap's coming from user mode */
 #define	T_USER	0x100