Extend single_thread_set() mode with additional flag attributes.

The mode can now be or-ed with SINGLE_DEEP or SINGLE_NOWAIT to alter
the behaviour of single_thread_set(). This allows explicit control
of the SINGLE_DEEP behaviour.

If SINGLE_DEEP is set the deep flag is passed to the initial check call
and by that the check will error out instead of suspending (SINGLE_UNWIND)
or exiting (SINGLE_EXIT). The SINGLE_DEEP flag is required in calls to
single_thread_set() outside of userret. E.g. at the start of sys_execve
because the proc is not allowed to call exit1() in that location.

SINGLE_NOWAIT skips the wait at the end of single_thread_set() and therefor
returns BEFORE all threads have been parked. Currently this is only used by
the ptrace code and should not be used anywhere else. Not waiting for all
threads to settle is asking for trouble.

This solves an issue by using SINGLE_UNWIND in the coredump case where
the code should actually exit in case another thread crashed moments earlier.
Also the SINGLE_UNWIND in pledge_fail() is now marked SINGLE_DEEP since
the call to pledge_fail() is for sure not at the kernel boundary.

OK mpi@

5 files changed, 26 insertions, 23 deletions

diff --git a/sys/kern/kern_exec.c b/sys/kern/kern_exec.c
index 3f3112b018f..a85010da5ae 100644
--- a/sys/kern/kern_exec.c
+++ b/sys/kern/kern_exec.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: kern_exec.c,v 1.250 2023/07/10 03:31:57 guenther Exp $	*/
+/*	$OpenBSD: kern_exec.c,v 1.251 2023/09/29 12:47:34 claudio Exp $	*/
 /*	$NetBSD: kern_exec.c,v 1.75 1996/02/09 18:59:28 christos Exp $	*/
 
 /*-
@@ -284,7 +284,7 @@ sys_execve(struct proc *p, void *v, register_t *retval)
 	}
 
 	/* get other threads to stop */
-	if ((error = single_thread_set(p, SINGLE_UNWIND, 1)))
+	if ((error = single_thread_set(p, SINGLE_UNWIND | SINGLE_DEEP)))
 		return (error);
 
 	/*
@@ -444,7 +444,7 @@ sys_execve(struct proc *p, void *v, register_t *retval)
 	 * we're committed: any further errors will kill the process, so
 	 * kill the other threads now.
 	 */
-	single_thread_set(p, SINGLE_EXIT, 1);
+	single_thread_set(p, SINGLE_EXIT);
 
 	/*
 	 * Prepare vmspace for remapping. Note that uvmspace_exec can replace
diff --git a/sys/kern/kern_exit.c b/sys/kern/kern_exit.c
index f57d48415ef..ce6aea2aa1b 100644
--- a/sys/kern/kern_exit.c
+++ b/sys/kern/kern_exit.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: kern_exit.c,v 1.216 2023/09/21 13:49:25 claudio Exp $	*/
+/*	$OpenBSD: kern_exit.c,v 1.217 2023/09/29 12:47:34 claudio Exp $	*/
 /*	$NetBSD: kern_exit.c,v 1.39 1996/04/22 01:38:25 christos Exp $	*/
 
 /*
@@ -131,7 +131,7 @@ exit1(struct proc *p, int xexit, int xsig, int flags)
 	} else {
 		/* nope, multi-threaded */
 		if (flags == EXIT_NORMAL)
-			single_thread_set(p, SINGLE_EXIT, 1);
+			single_thread_set(p, SINGLE_EXIT);
 		else if (flags == EXIT_THREAD)
 			single_thread_check(p, 0);
 	}
diff --git a/sys/kern/kern_pledge.c b/sys/kern/kern_pledge.c
index 7f2446e6318..2099db42f89 100644
--- a/sys/kern/kern_pledge.c
+++ b/sys/kern/kern_pledge.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: kern_pledge.c,v 1.308 2023/09/19 10:43:33 claudio Exp $	*/
+/*	$OpenBSD: kern_pledge.c,v 1.309 2023/09/29 12:47:34 claudio Exp $	*/
 
 /*
  * Copyright (c) 2015 Nicholas Marriott <nicm@openbsd.org>
@@ -578,9 +578,9 @@ pledge_fail(struct proc *p, int error, uint64_t code)
 	    p->p_p->ps_comm, p->p_p->ps_pid, codes, p->p_pledge_syscall);
 	p->p_p->ps_acflag |= APLEDGE;
 
-	/* Stop threads immediately, because this process is suspect */
+	/* Try to stop threads immediately, because this process is suspect */
 	if (P_HASSIBLING(p))
-		single_thread_set(p, SINGLE_UNWIND, 1);
+		single_thread_set(p, SINGLE_UNWIND | SINGLE_DEEP);
 
 	/* Send uncatchable SIGABRT for coredump */
 	sigabort(p);
diff --git a/sys/kern/kern_sig.c b/sys/kern/kern_sig.c
index 2bffc053225..89a703af7b2 100644
--- a/sys/kern/kern_sig.c
+++ b/sys/kern/kern_sig.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: kern_sig.c,v 1.318 2023/09/19 10:43:33 claudio Exp $	*/
+/*	$OpenBSD: kern_sig.c,v 1.319 2023/09/29 12:47:34 claudio Exp $	*/
 /*	$NetBSD: kern_sig.c,v 1.54 1996/04/22 01:38:32 christos Exp $	*/
 
 /*
@@ -840,7 +840,7 @@ trapsignal(struct proc *p, int signum, u_long trapno, int code,
 		    signum != SIGKILL && (p->p_sigmask & mask) != 0) {
 			int s;
 
-			single_thread_set(p, SINGLE_SUSPEND, 0);
+			single_thread_set(p, SINGLE_SUSPEND | SINGLE_NOWAIT);
 			pr->ps_xsig = signum;
 
 			SCHED_LOCK(s);
@@ -1290,7 +1290,7 @@ cursig(struct proc *p, struct sigctx *sctx)
 		 */
 		if (((pr->ps_flags & (PS_TRACED | PS_PPWAIT)) == PS_TRACED) &&
 		    signum != SIGKILL) {
-			single_thread_set(p, SINGLE_SUSPEND, 0);
+			single_thread_set(p, SINGLE_SUSPEND | SINGLE_NOWAIT);
 			pr->ps_xsig = signum;
 
 			SCHED_LOCK(s);
@@ -1559,7 +1559,7 @@ sigexit(struct proc *p, int signum)
 
 		/* if there are other threads, pause them */
 		if (P_HASSIBLING(p))
-			single_thread_set(p, SINGLE_UNWIND, 1);
+			single_thread_set(p, SINGLE_UNWIND);
 
 		if (coredump(p) == 0)
 			signum |= WCOREFLAG;
@@ -2066,16 +2066,16 @@ single_thread_check(struct proc *p, int deep)
  *  - SINGLE_EXIT: unwind to kernel boundary and exit
  */
 int
-single_thread_set(struct proc *p, enum single_thread_mode mode, int wait)
+single_thread_set(struct proc *p, int flags)
 {
 	struct process *pr = p->p_p;
 	struct proc *q;
-	int error, s;
+	int error, s, mode = flags & SINGLE_MASK;
 
 	KASSERT(curproc == p);
 
 	SCHED_LOCK(s);
-	error = single_thread_check_locked(p, (mode == SINGLE_UNWIND), s);
+	error = single_thread_check_locked(p, flags & SINGLE_DEEP, s);
 	if (error) {
 		SCHED_UNLOCK(s);
 		return error;
@@ -2146,7 +2146,7 @@ single_thread_set(struct proc *p, enum single_thread_mode mode, int wait)
 	}
 	SCHED_UNLOCK(s);
 
-	if (wait)
+	if ((flags & SINGLE_NOWAIT) == 0)
 		single_thread_wait(pr, 1);
 
 	return 0;
diff --git a/sys/sys/proc.h b/sys/sys/proc.h
index 34104386f6f..6ed699d9bc0 100644
--- a/sys/sys/proc.h
+++ b/sys/sys/proc.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: proc.h,v 1.351 2023/09/13 14:25:49 claudio Exp $	*/
+/*	$OpenBSD: proc.h,v 1.352 2023/09/29 12:47:34 claudio Exp $	*/
 /*	$NetBSD: proc.h,v 1.44 1996/04/22 01:23:21 christos Exp $	*/
 
 /*-
@@ -571,12 +571,15 @@ refreshcreds(struct proc *p)
 		dorefreshcreds(pr, p);
 }
 
-enum single_thread_mode {
-	SINGLE_SUSPEND,		/* other threads to stop wherever they are */
-	SINGLE_UNWIND,		/* other threads to unwind and stop */
-	SINGLE_EXIT		/* other threads to unwind and then exit */
-};
-int	single_thread_set(struct proc *, enum single_thread_mode, int);
+#define	SINGLE_SUSPEND	0x01	/* other threads to stop wherever they are */
+#define	SINGLE_UNWIND	0x02	/* other threads to unwind and stop */
+#define	SINGLE_EXIT	0x03	/* other threads to unwind and then exit */
+#define	SINGLE_MASK	0x0f
+/* extra flags for single_thread_set */
+#define	SINGLE_DEEP	0x10	/* call is in deep */
+#define	SINGLE_NOWAIT	0x20	/* do not wait for other threads to stop */
+
+int	single_thread_set(struct proc *, int);
 int	single_thread_wait(struct process *, int);
 void	single_thread_clear(struct proc *, int);
 int	single_thread_check(struct proc *, int);