- do not call pthread_atfork(3) handlers when a multithreaded program

calls vfork(2). "untested, but looks OK" marc@
- document vfork(2), popen(3) and system(3) don't call atfork handlers
in multithreaded programs. okay jmc@

5 files changed, 68 insertions, 33 deletions

diff --git a/lib/libc/gen/popen.3 b/lib/libc/gen/popen.3
index f3ce3b60ca4..df1a8759a6f 100644
--- a/lib/libc/gen/popen.3
+++ b/lib/libc/gen/popen.3
@@ -1,4 +1,4 @@
-.\"	$OpenBSD: popen.3,v 1.15 2007/05/31 19:19:28 jmc Exp $
+.\"	$OpenBSD: popen.3,v 1.16 2008/04/04 19:30:41 kurt Exp $
 .\"
 .\" Copyright (c) 1991, 1993
 .\"	The Regents of the University of California.  All rights reserved.
@@ -27,7 +27,7 @@
 .\" OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 .\" SUCH DAMAGE.
 .\"
-.Dd $Mdocdate: May 31 2007 $
+.Dd $Mdocdate: April 4 2008 $
 .Dt POPEN 3
 .Os
 .Sh NAME
@@ -94,6 +94,10 @@ the command's standard input is the same as that of the process that called
 Note that
 .Fn popen
 output streams are fully buffered by default.
+In addition, fork handlers established using
+.Xr pthread_atfork 3
+are not called when a multithreaded program calls
+.Fn popen .
 .Pp
 The
 .Fn pclose
diff --git a/lib/libc/stdlib/system.3 b/lib/libc/stdlib/system.3
index 338d3bfa835..a49e595bc0a 100644
--- a/lib/libc/stdlib/system.3
+++ b/lib/libc/stdlib/system.3
@@ -29,9 +29,9 @@
 .\" OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 .\" SUCH DAMAGE.
 .\"
-.\"     $OpenBSD: system.3,v 1.10 2007/05/31 19:19:32 jmc Exp $
+.\"     $OpenBSD: system.3,v 1.11 2008/04/04 19:30:41 kurt Exp $
 .\"
-.Dd $Mdocdate: May 31 2007 $
+.Dd $Mdocdate: April 4 2008 $
 .Dt SYSTEM 3
 .Os
 .Sh NAME
@@ -66,6 +66,11 @@ Otherwise,
 .Fn system
 returns the termination status of the shell in the format specified by
 .Xr waitpid 2 .
+.Pp
+Note that fork handlers established using
+.Xr pthread_atfork 3
+are not called when a multithreaded program calls
+.Fn system .
 .Sh RETURN VALUES
 If a child process cannot be created, or the termination status of
 the shell cannot be obtained,
diff --git a/lib/libc/sys/vfork.2 b/lib/libc/sys/vfork.2
index c1b4b6ec798..ec0e5a75639 100644
--- a/lib/libc/sys/vfork.2
+++ b/lib/libc/sys/vfork.2
@@ -1,4 +1,4 @@
-.\"	$OpenBSD: vfork.2,v 1.15 2007/05/31 19:19:34 jmc Exp $
+.\"	$OpenBSD: vfork.2,v 1.16 2008/04/04 19:30:41 kurt Exp $
 .\"	$NetBSD: vfork.2,v 1.8 1997/07/10 07:54:13 mikel Exp $
 .\"
 .\" Copyright (c) 1980, 1991, 1993
@@ -30,7 +30,7 @@
 .\"
 .\"     @(#)vfork.2	8.1 (Berkeley) 6/4/93
 .\"
-.Dd $Mdocdate: May 31 2007 $
+.Dd $Mdocdate: April 4 2008 $
 .Dt VFORK 2
 .Os
 .Sh NAME
@@ -62,6 +62,10 @@ in that the parent is suspended until the child makes a call to
 or an exit (either by a call to
 .Xr _exit 2
 or abnormally).
+In addition, fork handlers established using
+.Xr pthread_atfork 3
+are not called when a multithreaded program calls
+.Fn vfork .
 .Pp
 .Fn vfork
 returns 0 in the child's context and (later) the PID of the child in
diff --git a/lib/libpthread/uthread/uthread_fork.c b/lib/libpthread/uthread/uthread_fork.c
index 5234845a207..4eceda4c86c 100644
--- a/lib/libpthread/uthread/uthread_fork.c
+++ b/lib/libpthread/uthread/uthread_fork.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: uthread_fork.c,v 1.18 2007/11/20 19:35:37 deraadt Exp $	*/
+/*	$OpenBSD: uthread_fork.c,v 1.19 2008/04/04 19:30:41 kurt Exp $	*/
 /*
  * Copyright (c) 1995-1998 John Birrell <jb@cimlogic.com.au>
  * All rights reserved.
@@ -46,17 +46,8 @@ pid_t _dofork(int vfork);
 pid_t
 fork(void)
 {
-	return (_dofork(0));
-}
-
-pid_t
-_dofork(int vfork)
-{
-	struct pthread	*curthread = _get_curthread();
+	pid_t pid;
 	struct pthread_atfork *af;
-	int             i, flags;
-	pid_t           ret;
-	pthread_t	pthread;
 
 	/*
 	 * Defer signals to protect the scheduling queues from access
@@ -72,15 +63,40 @@ _dofork(int vfork)
 			af->prepare();
 	}
 
-	/* Fork a new process: */
-	if ((ret = (vfork ? _thread_sys_vfork() : _thread_sys_fork())) != 0) {
+	if ((pid = _dofork(0)) == 0) {
+		/* Run down atfork child handlers. */
+		TAILQ_FOREACH(af, &_atfork_list, qe) {
+			if (af->child != NULL)
+				af->child();
+		}
+		_mutex_reinit(&_atfork_mutex);
+	} else {
 		/* Run down atfork parent handlers. */
 		TAILQ_FOREACH(af, &_atfork_list, qe) {
 			if (af->parent != NULL)
 				af->parent();
 		}
 		pthread_mutex_unlock(&_atfork_mutex);
-	} else {
+	}
+
+	/*
+	 * Undefer and handle pending signals, yielding if necessary:
+	 */
+	_thread_kern_sig_undefer();
+
+	return(pid);
+}
+
+pid_t
+_dofork(int vfork)
+{
+	struct pthread	*curthread = _get_curthread();
+	int             i, flags;
+	pid_t           ret;
+	pthread_t	pthread;
+
+	/* Fork a new process and reset child's state */
+	if ((ret = (vfork ? _thread_sys_vfork() : _thread_sys_fork())) == 0) {
 		/* Close the pthread kernel pipe: */
 		_thread_sys_close(_thread_kern_pipe[0]);
 		_thread_sys_close(_thread_kern_pipe[1]);
@@ -210,19 +226,8 @@ _dofork(int vfork)
 				}
 			}
 		}
-		/* Run down atfork child handlers. */
-		TAILQ_FOREACH(af, &_atfork_list, qe) {
-			if (af->child != NULL)
-				af->child();
-		}
-		_mutex_reinit(&_atfork_mutex);
 	}
 
-	/*
-	 * Undefer and handle pending signals, yielding if necessary:
-	 */
-	_thread_kern_sig_undefer();
-
 	/* Return the process ID: */
 	return (ret);
 }
diff --git a/lib/libpthread/uthread/uthread_vfork.c b/lib/libpthread/uthread/uthread_vfork.c
index 10f8586536a..545bcdc96ba 100644
--- a/lib/libpthread/uthread/uthread_vfork.c
+++ b/lib/libpthread/uthread/uthread_vfork.c
@@ -1,12 +1,29 @@
-/*	$OpenBSD: uthread_vfork.c,v 1.3 2007/11/20 19:35:37 deraadt Exp $	*/
+/*	$OpenBSD: uthread_vfork.c,v 1.4 2008/04/04 19:30:41 kurt Exp $	*/
 #include <unistd.h>
 #ifdef _THREAD_SAFE
+#include <pthread.h>
+#include "pthread_private.h"
 
 pid_t	_dofork(int vfork);
 
 pid_t
 vfork(void)
 {
-	return (_dofork(1));
+	pid_t pid;
+
+	/*
+	 * Defer signals to protect the scheduling queues from access
+	 * by the signal handler:
+	 */
+	_thread_kern_sig_defer();
+
+	pid = _dofork(1);
+
+	/*
+	 * Undefer and handle pending signals, yielding if necessary:
+	 */
+	_thread_kern_sig_undefer();
+
+	return (pid);
 }
 #endif