Change threxit() to take a pointer to a pid_t to zero out from the

kernel so that librthread can detect when a thread is completely
done with its stack without need a kqueue.  The dying thread moves
itself to a GC list, other threads scan the GC list on pthread_create()
and pthread_join() and free the stack and handle once the thread's
thread id is zeroed.

"get it in" deraadt@, tedu@, cheers by others

5 files changed, 43 insertions, 102 deletions

diff --git a/lib/librthread/Makefile b/lib/librthread/Makefile
index 14f821f7781..5d2d31d4d20 100644
--- a/lib/librthread/Makefile
+++ b/lib/librthread/Makefile
@@ -1,4 +1,4 @@
-#	$OpenBSD: Makefile,v 1.13 2009/10/21 16:05:48 guenther Exp $
+#	$OpenBSD: Makefile,v 1.14 2009/11/27 19:42:24 guenther Exp $
 
 LIB=rthread
 WANTLINT=
@@ -12,7 +12,7 @@ CFLAGS+=-I${LIBCSRCDIR}/include
 .PATH: ${.CURDIR}/arch/${MACHINE_ARCH}
 SRCS= rthread.c rthread_attr.c rthread_sched.c rthread_sync.c rthread_tls.c \
 	rthread_sig.c rthread_np.c rthread_debug.c rthread_stack.c \
-	rthread_reaper.c rthread_libc.c rthread_fork.c rthread_file.c
+	rthread_libc.c rthread_fork.c rthread_file.c
 
 OBJS+= _atomic_lock.o rfork_thread.o
 
diff --git a/lib/librthread/rthread.c b/lib/librthread/rthread.c
index a1a76ac3fb4..8b9a4be6648 100644
--- a/lib/librthread/rthread.c
+++ b/lib/librthread/rthread.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: rthread.c,v 1.40 2009/02/20 01:24:05 tedu Exp $ */
+/*	$OpenBSD: rthread.c,v 1.41 2009/11/27 19:42:24 guenther Exp $ */
 /*
  * Copyright (c) 2004,2005 Ted Unangst <tedu@openbsd.org>
  * All Rights Reserved.
@@ -47,6 +47,9 @@ static int concurrency_level;	/* not used */
 int _threads_ready;
 struct listhead _thread_list = LIST_HEAD_INITIALIZER(_thread_list);
 _spinlock_lock_t _thread_lock = _SPINLOCK_UNLOCKED;
+static struct pthread_queue _thread_gc_list
+    = TAILQ_HEAD_INITIALIZER(_thread_gc_list);
+static _spinlock_lock_t _thread_gc_lock = _SPINLOCK_UNLOCKED;
 struct pthread _initial_thread;
 
 int rfork_thread(int, void *, void (*)(void *), void *);
@@ -96,16 +99,6 @@ _rthread_start(void *v)
 	pthread_exit(retval);
 }
 
-int
-_rthread_open_kqueue(void)
-{
-	_rthread_kq = kqueue();
-	if (_rthread_kq == -1)
-		return 1;
-	fcntl(_rthread_kq, F_SETFD, FD_CLOEXEC);
-	return 0;
-}
-
 static int
 _rthread_init(void)
 {
@@ -118,8 +111,6 @@ _rthread_init(void)
 	thread->flags_lock = _SPINLOCK_UNLOCKED;
 	strlcpy(thread->name, "Main process", sizeof(thread->name));
 	LIST_INSERT_HEAD(&_thread_list, thread, threads);
-	if (_rthread_open_kqueue())
-		return (errno);
 	_rthread_debug_init();
 	_rthread_debug(1, "rthread init\n");
 	_threads_ready = 1;
@@ -146,10 +137,18 @@ static void
 _rthread_free(pthread_t thread)
 {
 	/* catch wrongdoers for the moment */
+	/* initial_thread.tid must remain valid */
 	if (thread != &_initial_thread) {
-		/* initial_thread.tid must remain valid */
+		struct stack *stack = thread->stack;
+		pid_t tid = thread->tid;
+
+		/* catch wrongdoers for the moment */
 		memset(thread, 0xd0, sizeof(*thread));
-		free(thread);
+		thread->stack = stack;
+		thread->tid = tid;
+		_spinlock(&_thread_gc_lock);
+		TAILQ_INSERT_TAIL(&_thread_gc_list, thread, waiting);
+		_spinunlock(&_thread_gc_lock);
 	}
 }
 
@@ -188,6 +187,26 @@ pthread_self(void)
 	return (thread);
 }
 
+static void
+_rthread_reaper(void)
+{
+	pthread_t thread;
+
+restart:_spinlock(&_thread_gc_lock);
+	TAILQ_FOREACH(thread, &_thread_gc_list, waiting) {
+		if (thread->tid != 0)
+			continue;
+		TAILQ_REMOVE(&_thread_gc_list, thread, waiting);
+		_spinunlock(&_thread_gc_lock);
+		_rthread_debug(3, "rthread reaping %p stack %p\n",
+		    (void *)thread, (void *)thread->stack);
+		_rthread_free_stack(thread->stack);
+		free(thread);
+		goto restart;
+	}
+	_spinunlock(&_thread_gc_lock);
+}
+
 void
 pthread_exit(void *retval)
 {
@@ -197,7 +216,7 @@ pthread_exit(void *retval)
 	pthread_t thread = pthread_self();
 
 	thread->retval = retval;
-	
+
 	for (clfn = thread->cleanup_fns; clfn; ) {
 		struct rthread_cleanup_fn *oclfn = clfn;
 		clfn = clfn->next;
@@ -218,12 +237,7 @@ pthread_exit(void *retval)
 		_sem_post(&thread->donesem);
 	}
 
-	/* reap before adding self, we don't want to disappear too soon */
-	_rthread_reaper();
-	if (tid != _initial_thread.tid)
-		_rthread_add_to_reaper(tid, stack);
-
-	threxit(0);
+	threxit(&thread->tid);
 	for(;;);
 }
 
@@ -286,6 +300,8 @@ pthread_create(pthread_t *threadp, const pthread_attr_t *attr,
 		if ((rc = _rthread_init()))
 		    return (rc);
 
+	_rthread_reaper();
+
 	thread = calloc(1, sizeof(*thread));
 	if (!thread)
 		return (errno);
diff --git a/lib/librthread/rthread.h b/lib/librthread/rthread.h
index c7b0fb5a833..29102eb6ce7 100644
--- a/lib/librthread/rthread.h
+++ b/lib/librthread/rthread.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: rthread.h,v 1.22 2009/10/21 16:05:48 guenther Exp $ */
+/*	$OpenBSD: rthread.h,v 1.23 2009/11/27 19:42:24 guenther Exp $ */
 /*
  * Copyright (c) 2004,2005 Ted Unangst <tedu@openbsd.org>
  * All Rights Reserved.
@@ -138,7 +138,6 @@ extern int _threads_ready;
 extern LIST_HEAD(listhead, pthread) _thread_list;
 extern struct pthread _initial_thread;
 extern _spinlock_lock_t _thread_lock;
-extern int _rthread_kq;
 
 void	_spinlock(_spinlock_lock_t *);
 void	_spinunlock(_spinlock_lock_t *);
@@ -154,9 +153,6 @@ void	_rthread_tls_destructors(pthread_t);
 void	_rthread_debug(int, const char *, ...)
 		__attribute__((__format__ (printf, 2, 3)));
 void	_rthread_debug_init(void);
-void	_rthread_add_to_reaper(pid_t, struct stack *);
-void 	_rthread_reaper(void);
-int	_rthread_open_kqueue(void);
 #if defined(__ELF__) && defined(PIC)
 void	_rthread_dl_lock(int what);
 void	_rthread_bind_lock(int);
@@ -169,7 +165,7 @@ int	_atomic_lock(register volatile _spinlock_lock_t *);
 
 /* syscalls */
 int getthrid(void);
-void threxit(int);
+void threxit(pid_t *);
 int thrsleep(void *, int, void *);
 int thrwakeup(void *, int n);
 int sched_yield(void);
diff --git a/lib/librthread/rthread_fork.c b/lib/librthread/rthread_fork.c
index fe51c5de31e..b5a8afc2dde 100644
--- a/lib/librthread/rthread_fork.c
+++ b/lib/librthread/rthread_fork.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: rthread_fork.c,v 1.2 2009/10/21 15:32:01 guenther Exp $ */
+/*	$OpenBSD: rthread_fork.c,v 1.3 2009/11/27 19:42:24 guenther Exp $ */
 
 /*
  * Copyright (c) 2008 Kurt Miller <kurt@openbsd.org>
@@ -108,9 +108,6 @@ _dofork(int is_vfork)
 #endif
 
 	if (newid == 0) {
-		if (_rthread_open_kqueue())
-			_exit(126);	/* XXX */
-
 		/* update this thread's structure */
 		me->tid = getthrid();
 		me->donesem.lock = _SPINLOCK_UNLOCKED;
diff --git a/lib/librthread/rthread_reaper.c b/lib/librthread/rthread_reaper.c
deleted file mode 100644
index 14a37f192b9..00000000000
--- a/lib/librthread/rthread_reaper.c
+++ /dev/null
@@ -1,68 +0,0 @@
-/*	$OpenBSD: rthread_reaper.c,v 1.3 2007/04/10 17:39:21 tedu Exp $	*/
-/*
- * Copyright (c) 2006 Otto Moerbeek <otto@drijf.net>
- *
- * Permission to use, copy, modify, and distribute this software for any
- * purpose with or without fee is hereby granted, provided that the above
- * copyright notice and this permission notice appear in all copies.
- *
- * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
- * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
- * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
- * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
- * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
- * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
- * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
- */
-
-#include <sys/types.h>
-#include <sys/event.h>
-
-#include <errno.h>
-
-#include <machine/spinlock.h>
-#include <pthread.h>
-
-#include "rthread.h"
-
-int _rthread_kq;
-
-void
-_rthread_add_to_reaper(pid_t t, struct stack *s)
-{
-	struct kevent kc;
-	int n;
-
-	_rthread_debug(1, "Adding %d to reaper\n", t);
-	EV_SET(&kc, t, EVFILT_PROC, EV_ADD|EV_CLEAR, NOTE_EXIT, 0, s);
-	n = kevent(_rthread_kq, &kc, 1, NULL, 0, NULL);
-	if (n == -1)
-		_rthread_debug(0, "_rthread_add_to_reaper(): kevent %d\n", errno);
-}
-
-/* ARGSUSED */
-void
-_rthread_reaper(void)
-{
-	struct kevent ke;
-	int  n;
-	struct timespec t;
-
-	t.tv_sec = 0;
-	t.tv_nsec = 0;
-
-	for (;;) {
-		n = kevent(_rthread_kq, NULL, 0, &ke, 1, &t);
-		if (n == -1)
-			_rthread_debug(0, "_rthread_reaper(): kevent %d\n", errno);
-		else if (n == 0)
-			break;
-		else {
-			_rthread_debug(1, "_rthread_reaper(): %d died\n",
-			    ke.ident);
-			/* XXX check error conditions */
-			_rthread_free_stack(ke.udata);
-		}
-
-	}
-}