Cleanup struct pthread and stack after thread exits. This version does

not use a separate reaper thread. ok tedu@ marc@

4 files changed, 148 insertions, 23 deletions

diff --git a/lib/librthread/Makefile b/lib/librthread/Makefile
index 24a0fa53fdc..e8438e2823c 100644
--- a/lib/librthread/Makefile
+++ b/lib/librthread/Makefile
@@ -1,4 +1,4 @@
-#	$OpenBSD: Makefile,v 1.7 2006/01/01 19:32:30 marc Exp $
+#	$OpenBSD: Makefile,v 1.8 2006/01/04 19:48:52 otto Exp $
 
 LIB=rthread
 WANTLINT=
@@ -7,7 +7,8 @@ CFLAGS+=-Wall -g -Werror -Wshadow
 
 .PATH: ${.CURDIR}/arch/${MACHINE_ARCH}
 SRCS= rthread.c rthread_attr.c rthread_sched.c rthread_sync.c rthread_tls.c \
-	rthread_sig.c rthread_np.c rthread_debug.c rthread_stack.c
+	rthread_sig.c rthread_np.c rthread_debug.c rthread_stack.c \
+	rthread_reaper.c
 
 OBJS+= _atomic_lock.o rfork_thread.o
 
diff --git a/lib/librthread/rthread.c b/lib/librthread/rthread.c
index eb93ec7185c..975a35845ef 100644
--- a/lib/librthread/rthread.c
+++ b/lib/librthread/rthread.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: rthread.c,v 1.27 2006/01/04 08:48:01 marc Exp $ */
+/*	$OpenBSD: rthread.c,v 1.28 2006/01/04 19:48:52 otto Exp $ */
 /*
  * Copyright (c) 2004,2005 Ted Unangst <tedu@openbsd.org>
  * All Rights Reserved.
@@ -21,6 +21,7 @@
  */
 
 #include <sys/param.h>
+#include <sys/event.h>
 #include <sys/mman.h>
 #include <sys/wait.h>
 
@@ -103,14 +104,43 @@ _rthread_init(void)
 	thread->tid = getthrid();
 	thread->donesem.lock = _SPINLOCK_UNLOCKED;
 	thread->flags |= THREAD_CANCEL_ENABLE|THREAD_CANCEL_DEFERRED;
+	thread->flags_lock = _SPINLOCK_UNLOCKED;
 	strlcpy(thread->name, "Main process", sizeof(thread->name));
 	LIST_INSERT_HEAD(&_thread_list, thread, threads);
+	_rthread_kq = kqueue();
+	if (_rthread_kq == -1)
+		return (errno);
 	_threads_ready = 1;
 	__isthreaded = 1;
 
 	return (0);
 }
 
+static void
+_rthread_free(pthread_t thread)
+{
+	/* catch wrongdoers for the moment */
+	memset(thread, 0xd0, sizeof(*thread));
+	if (thread != &_initial_thread)
+		free(thread);
+}
+
+static void
+_rthread_setflag(pthread_t thread, int flag)
+{
+	_spinlock(&thread->flags_lock);
+	thread->flags |= flag;
+	_spinunlock(&thread->flags_lock);
+}
+
+static void
+_rthread_clearflag(pthread_t thread, int flag)
+{
+	_spinlock(&thread->flags_lock);
+	thread->flags &= ~flag;
+	_spinunlock(&thread->flags_lock);
+}
+
 /*
  * real pthread functions
  */
@@ -134,12 +164,12 @@ void
 pthread_exit(void *retval)
 {
 	struct rthread_cleanup_fn *clfn;
+	pid_t tid;
+	struct stack *stack;
 	pthread_t thread = pthread_self();
 
 	thread->retval = retval;
-	thread->flags |= THREAD_DONE;
 	
-	_sem_post(&thread->donesem);
 	for (clfn = thread->cleanup_fns; clfn; ) {
 		struct rthread_cleanup_fn *oclfn = clfn;
 		clfn = clfn->next;
@@ -150,10 +180,20 @@ pthread_exit(void *retval)
 	_spinlock(&_thread_lock);
 	LIST_REMOVE(thread, threads);
 	_spinunlock(&_thread_lock);
-#if 0
+
+	_sem_post(&thread->donesem);
+
+	stack = thread->stack;
+	tid = thread->tid;
 	if (thread->flags & THREAD_DETACHED)
-		free(thread);
-#endif
+		_rthread_free(thread);
+	else
+		_rthread_setflag(thread, THREAD_DONE);
+
+	if (tid != _initial_thread.tid)
+		_rthread_add_to_reaper(tid, stack);
+
+	_rthread_reaper();
 	threxit(0);
 	for(;;);
 }
@@ -171,22 +211,34 @@ pthread_join(pthread_t thread, void **retval)
 			*retval = thread->retval;
 		e = 0;
 	}
+	/* We should be the last having a ref to this thread, but
+	 * someone stupid or evil might haved detached it;
+	 * in that case the thread will cleanup itself */
+	if ((thread->flags & THREAD_DETACHED) == 0)
+		_rthread_free(thread);
 
+	_rthread_reaper();
 	return (e);
 }
 
 int
 pthread_detach(pthread_t thread)
 {
-	_spinlock(&_thread_lock);
-#if 0
-	if (thread->flags & THREAD_DONE)
-		free(thread);
-	else
-#endif
+	int rc = 0;
+
+	_spinlock(&thread->flags_lock);
+	if (thread->flags & THREAD_DETACHED) {
+		rc = EINVAL;
+		_spinunlock(&thread->flags_lock);
+	} else if (thread->flags & THREAD_DONE) {
+		_spinunlock(&thread->flags_lock);
+		_rthread_free(thread);
+	} else {
 		thread->flags |= THREAD_DETACHED;
-	_spinunlock(&_thread_lock);
-	return (0);
+		_spinunlock(&thread->flags_lock);
+	}
+	_rthread_reaper();
+	return (rc);
 }
 
 int
@@ -206,6 +258,7 @@ pthread_create(pthread_t *threadp, const pthread_attr_t *attr,
 		return (errno);
 	memset(thread, 0, sizeof(*thread));
 	thread->donesem.lock = _SPINLOCK_UNLOCKED;
+	thread->flags_lock = _SPINLOCK_UNLOCKED;
 	thread->fn = start_routine;
 	thread->arg = arg;
 	if (attr)
@@ -254,7 +307,7 @@ fail2:
 	LIST_REMOVE(thread, threads);
 fail1:
 	_spinunlock(&_thread_lock);
-	free(thread);
+	_rthread_free(thread);
 
 	return (rc);
 }
@@ -275,7 +328,7 @@ int
 pthread_cancel(pthread_t thread)
 {
 
-	thread->flags |= THREAD_CANCELLED;
+	_rthread_setflag(thread, THREAD_CANCELLED);
 	return (0);
 }
 
@@ -297,10 +350,10 @@ pthread_setcancelstate(int state, int *oldstatep)
 	oldstate = self->flags & THREAD_CANCEL_ENABLE ?
 	    PTHREAD_CANCEL_ENABLE : PTHREAD_CANCEL_DISABLE;
 	if (state == PTHREAD_CANCEL_ENABLE) {
-		self->flags |= THREAD_CANCEL_ENABLE;
+		_rthread_setflag(self, THREAD_CANCEL_ENABLE);
 		pthread_testcancel();
 	} else if (state == PTHREAD_CANCEL_DISABLE) {
-		self->flags &= ~THREAD_CANCEL_ENABLE;
+		_rthread_clearflag(self, THREAD_CANCEL_ENABLE);
 	} else {
 		return (EINVAL);
 	}
@@ -319,10 +372,10 @@ pthread_setcanceltype(int type, int *oldtypep)
 	oldtype = self->flags & THREAD_CANCEL_DEFERRED ?
 	    PTHREAD_CANCEL_DEFERRED : PTHREAD_CANCEL_ASYNCHRONOUS;
 	if (type == PTHREAD_CANCEL_DEFERRED) {
-		self->flags |= THREAD_CANCEL_DEFERRED;
+		_rthread_setflag(self, THREAD_CANCEL_DEFERRED);
 		pthread_testcancel();
 	} else if (type == PTHREAD_CANCEL_ASYNCHRONOUS) {
-		self->flags &= ~THREAD_CANCEL_DEFERRED;
+		_rthread_clearflag(self, THREAD_CANCEL_DEFERRED);
 	} else {
 		return (EINVAL);
 	}
diff --git a/lib/librthread/rthread.h b/lib/librthread/rthread.h
index 9a312ef64bc..04b8d04e8a9 100644
--- a/lib/librthread/rthread.h
+++ b/lib/librthread/rthread.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: rthread.h,v 1.15 2006/01/01 19:32:30 marc Exp $ */
+/*	$OpenBSD: rthread.h,v 1.16 2006/01/04 19:48:52 otto Exp $ */
 /*
  * Copyright (c) 2004,2005 Ted Unangst <tedu@openbsd.org>
  * All Rights Reserved.
@@ -107,6 +107,7 @@ struct pthread {
 	struct semaphore donesem;
 	pid_t tid;
 	unsigned int flags;
+	_spinlock_lock_t flags_lock;
 	void *retval;
 	void *(*fn)(void *);
 	void *arg;
@@ -130,6 +131,7 @@ extern int _threads_ready;
 extern LIST_HEAD(listhead, pthread) _thread_list;
 extern struct pthread _initial_thread;
 extern _spinlock_lock_t _thread_lock;
+extern int _rthread_kq;
 
 void	_spinlock(_spinlock_lock_t *);
 void	_spinunlock(_spinlock_lock_t *);
@@ -144,6 +146,9 @@ void	_rthread_free_stack(struct stack *);
 void	_rthread_tls_destructors(pthread_t);
 void	_rthread_debug(int, const char *, ...)
 		__attribute__((__format__ (printf, 2, 3)));
+void	_rthread_add_to_reaper(pid_t, struct stack *);
+void 	_rthread_reaper(void);
+
 
 void	_thread_dump_info(void);
 
diff --git a/lib/librthread/rthread_reaper.c b/lib/librthread/rthread_reaper.c
new file mode 100644
index 00000000000..8c9bf64c4aa
--- /dev/null
+++ b/lib/librthread/rthread_reaper.c
@@ -0,0 +1,66 @@
+/*	$OpenBSD: rthread_reaper.c,v 1.1 2006/01/04 19:48:52 otto Exp $	*/
+/*
+ * Copyright (c) 2006 Otto Moerbeek <otto@drijf.net>
+ *
+ * Permission to use, copy, modify, and distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#include <sys/types.h>
+#include <sys/event.h>
+
+#include <machine/spinlock.h>
+#include <pthread.h>
+
+#include "rthread.h"
+
+int _rthread_kq;
+
+void
+_rthread_add_to_reaper(pid_t t, struct stack *s)
+{
+	struct kevent kc;
+	int n;
+
+	_rthread_debug(1, "Adding %d to reaper\n", t);
+	EV_SET(&kc, t, EVFILT_PROC, EV_ADD|EV_CLEAR, NOTE_EXIT, 0, s);
+	n = kevent(_rthread_kq, &kc, 1, NULL, 0, NULL);
+	if (n)
+		_rthread_debug(0, "_rthread_add_to_reaper(): kevent %d", n);
+}
+
+/* ARGSUSED */
+void
+_rthread_reaper(void)
+{
+	struct kevent ke;
+	int  n;
+	struct timespec t;
+
+	t.tv_sec = 0;
+	t.tv_nsec = 0;
+
+	for (;;) {
+		n = kevent(_rthread_kq, NULL, 0, &ke, 1, &t);
+		if (n < 0)
+			_rthread_debug(0, "_rthread_reaper(): kevent %d", n);
+		else if (n == 0)
+			break;
+		else {
+			_rthread_debug(1, "_rthread_reaper(): %d died\n",
+			    ke.ident);
+			/* XXX check error conditions */
+			_rthread_free_stack(ke.udata);
+		}
+
+	}
+}