Move mutex, condvar, and thread-specific data routes, pthread_once, and

pthread_exit from libpthread to libc, along with low-level bits to
support them.  Major bump to both libc and libpthread.

Requested by libressl team.  Ports testing by naddy@
ok kettenis@

12 files changed, 161 insertions, 863 deletions

diff --git a/lib/libc/thread/Makefile.inc b/lib/libc/thread/Makefile.inc
index 8c0cecf92d7..b409f578e3e 100644
--- a/lib/libc/thread/Makefile.inc
+++ b/lib/libc/thread/Makefile.inc
@@ -1,5 +1,36 @@
-#	$OpenBSD: Makefile.inc,v 1.10 2016/05/07 19:05:22 guenther Exp $
+#	$OpenBSD: Makefile.inc,v 1.11 2017/09/05 02:40:54 guenther Exp $
 
 .PATH: ${LIBCSRCDIR}/thread
 
 SRCS+=	callbacks.c atfork.c
+
+# threads infrastructure
+SRCS+=	rthread.c \
+	rthread_condattr.c \
+	rthread_debug.c \
+	rthread_file.c \
+	rthread_libc.c \
+	rthread_once.c \
+	rthread_tls.c \
+
+notyet= rthread_condattr_clock.c \
+	rthread_equal.c \
+	rthread_exit.c \
+	spinlock.c \
+	spinlocktry.c
+
+.if ${MACHINE_ARCH} == "amd64" || ${MACHINE_ARCH} == "i386" || \
+    ${MACHINE_ARCH} == "mips64" || ${MACHINE_ARCH} == "mips64el"
+CFLAGS+= -DFUTEX
+SRCS+=	rthread_mutex.c \
+	rthread_cond.c
+.else
+SRCS+=	rthread_sync.c
+.endif
+
+.if defined(NOPIC)
+CFLAGS+=-DNO_PIC
+.endif
+
+OBJS+=	_atomic_lock.o
+
diff --git a/lib/libc/thread/callbacks.c b/lib/libc/thread/callbacks.c
index c76811dd246..e38cf205a54 100644
--- a/lib/libc/thread/callbacks.c
+++ b/lib/libc/thread/callbacks.c
@@ -14,12 +14,20 @@
  * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
  */
 
+#include <pthread.h>
 #include <signal.h>
 #include <stdio.h>
 #include <string.h>
 #include <unistd.h>
 
 #include "thread_private.h"
+#include "rthread_cb.h"
+
+static __dead void
+_thread_canceled(void)
+{
+	pthread_exit(PTHREAD_CANCELED);
+}
 
 void
 _thread_set_callbacks(const struct thread_callbacks *cb, size_t len)
@@ -37,6 +45,30 @@ _thread_set_callbacks(const struct thread_callbacks *cb, size_t len)
 	if (sigprocmask(SIG_BLOCK, &allmask, &omask) == 0) {
 		/* mprotect RW */
 		memcpy(&_thread_cb, cb, sizeof(_thread_cb));
+
+		/*
+		 * These are supplied by libc, but only enabled
+		 * here when we actually need to prep for doing MT.
+		 */
+		_thread_cb.tc_canceled		= _thread_canceled;
+		_thread_cb.tc_flockfile		= _thread_flockfile;
+		_thread_cb.tc_ftrylockfile	= _thread_ftrylockfile;
+		_thread_cb.tc_funlockfile	= _thread_funlockfile;
+		_thread_cb.tc_malloc_lock	= _thread_malloc_lock;
+		_thread_cb.tc_malloc_unlock	= _thread_malloc_unlock;
+		_thread_cb.tc_atexit_lock	= _thread_atexit_lock;
+		_thread_cb.tc_atexit_unlock	= _thread_atexit_unlock;
+		_thread_cb.tc_atfork_lock	= _thread_atfork_lock;
+		_thread_cb.tc_atfork_unlock	= _thread_atfork_unlock;
+		_thread_cb.tc_arc4_lock		= _thread_arc4_lock;
+		_thread_cb.tc_arc4_unlock	= _thread_arc4_unlock;
+		_thread_cb.tc_mutex_lock	= _thread_mutex_lock;
+		_thread_cb.tc_mutex_unlock	= _thread_mutex_unlock;
+		_thread_cb.tc_mutex_destroy	= _thread_mutex_destroy;
+		_thread_cb.tc_tag_lock		= _thread_tag_lock;
+		_thread_cb.tc_tag_unlock	= _thread_tag_unlock;
+		_thread_cb.tc_tag_storage	= _thread_tag_storage;
+
 		/* mprotect RO | LOCKPERM | NOUNMAP */
 		sigprocmask(SIG_SETMASK, &omask, NULL);
 	}
diff --git a/lib/libc/thread/rthread.c b/lib/libc/thread/rthread.c
index 97947e515df..0f19338b086 100644
--- a/lib/libc/thread/rthread.c
+++ b/lib/libc/thread/rthread.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: rthread.c,v 1.3 2017/08/15 07:06:29 guenther Exp $ */
+/*	$OpenBSD: rthread.c,v 1.4 2017/09/05 02:40:54 guenther Exp $ */
 /*
  * Copyright (c) 2004,2005 Ted Unangst <tedu@openbsd.org>
  * All Rights Reserved.
@@ -16,71 +16,25 @@
  * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
  */
 /*
- * The heart of rthreads.  Basic functions like creating and joining
- * threads.
+ * The infrastructure of rthreads
  */
 
-#include <sys/types.h>
-#ifndef NO_PIC
-#include <sys/exec_elf.h>
-#pragma weak _DYNAMIC
-#endif
-
-#include <dlfcn.h>
-#include <errno.h>
 #include <pthread.h>
-#include <signal.h>
-#include <stdio.h>
 #include <stdlib.h>
-#include <string.h>
 #include <tib.h>
 #include <unistd.h>
 
-#include "cancel.h"		/* in libc/include */
-#include "thread_private.h"
 #include "rthread.h"
-#include "rthread_cb.h"
-
-/*
- * Call nonstandard functions via names in the reserved namespace:
- *	dlctl() -> _dlctl()
- *	getthrid -> _thread_sys_getthrid
- */
-typeof(dlctl) dlctl asm("_dlctl") __attribute__((weak));
-REDIRECT_SYSCALL(getthrid);
-
-/* weak stub to be overriden by ld.so */
-int	dlctl(void *handle, int cmd, void *data) { return 0; }
 
-/*
- * libc's signal wrappers hide SIGTHR; we need to call the real syscall
- * stubs _thread_sys_* directly.
- */
-REDIRECT_SYSCALL(sigaction);
-REDIRECT_SYSCALL(sigprocmask);
-REDIRECT_SYSCALL(thrkill);
+#define RTHREAD_ENV_DEBUG	"RTHREAD_DEBUG"
 
-static int concurrency_level;	/* not used */
+int _rthread_debug_level;
 
-int _threads_ready;
-int _post_threaded;
-size_t _thread_pagesize;
-struct listhead _thread_list = LIST_HEAD_INITIALIZER(_thread_list);
-_atomic_lock_t _thread_lock = _SPINLOCK_UNLOCKED;
-static struct pthread_queue _thread_gc_list
-    = TAILQ_HEAD_INITIALIZER(_thread_gc_list);
-static _atomic_lock_t _thread_gc_lock = _SPINLOCK_UNLOCKED;
-static struct pthread _initial_thread;
+static int _threads_inited;
 
-struct pthread_attr _rthread_attr_default = {
-	.stack_addr			= NULL,
-	.stack_size			= RTHREAD_STACK_SIZE_DEF,
-/*	.guard_size		set in _rthread_init */
-	.detach_state			= PTHREAD_CREATE_JOINABLE,
-	.contention_scope		= PTHREAD_SCOPE_SYSTEM,
-	.sched_policy			= SCHED_OTHER,
-	.sched_param = { .sched_priority = 0 },
-	.sched_inherit			= PTHREAD_INHERIT_SCHED,
+struct pthread _initial_thread = {
+	.flags_lock = _SPINLOCK_UNLOCKED,
+	.name = "Original thread",
 };
 
 /*
@@ -92,6 +46,7 @@ _spinlock(volatile _atomic_lock_t *lock)
 	while (_atomic_lock(lock))
 		sched_yield();
 }
+DEF_STRONG(_spinlock);
 
 int
 _spinlocktry(volatile _atomic_lock_t *lock)
@@ -104,82 +59,16 @@ _spinunlock(volatile _atomic_lock_t *lock)
 {
 	*lock = _ATOMIC_LOCK_UNLOCKED;
 }
+DEF_STRONG(_spinunlock);
 
 static void
-_rthread_start(void *v)
-{
-	pthread_t thread = v;
-	void *retval;
-
-	retval = thread->fn(thread->arg);
-	pthread_exit(retval);
-}
-
-static void
-sigthr_handler(__unused int sig)
-{
-	struct tib *tib = TIB_GET();
-	pthread_t self = tib->tib_thread;
-
-	/*
-	 * Do nothing unless
-	 * 1) pthread_cancel() has been called on this thread,
-	 * 2) cancelation is enabled for it, and
-	 * 3) we're not already in cancelation processing
-	 */
-	if (!tib->tib_canceled || tib->tib_cantcancel)
-		return;
-
-	/*
-	 * If delaying cancels inside complex ops (pthread_cond_wait,
-	 * pthread_join, etc), just mark that this has happened to
-	 * prevent a race with going to sleep
-	 */
-	if (tib->tib_cancel_point & CANCEL_POINT_DELAYED) {
-		self->delayed_cancel = 1;
-		return;
-	}
-
-	/*
-	 * otherwise, if in a cancel point or async cancels are
-	 * enabled, then exit
-	 */
-	if (tib->tib_cancel_point ||
-	    (tib->tib_thread_flags & TIB_THREAD_ASYNC_CANCEL))
-		pthread_exit(PTHREAD_CANCELED);
-}
-
-
-/*
- * A few basic callbacks for libc.  The first couple are only used
- * on archs where there isn't a fast TCB_GET()
- */
-#ifndef TCB_HAVE_MD_GET
-static int *
-multi_threaded_errnoptr(void)
-{
-        return (&TIB_GET()->tib_errno);
-}
-
-static void *
-multi_threaded_tcb(void)
-{
-	return (TCB_GET());
-}
-#endif /* TCB_HAVE_MD_GET */
-
-void
-_thread_canceled(void)
-{
-	pthread_exit(PTHREAD_CANCELED);
-}
-
-void
 _rthread_init(void)
 {
 	pthread_t thread = &_initial_thread;
 	struct tib *tib;
-	struct sigaction sa;
+
+	if (_threads_inited)
+		return;
 
 	tib = TIB_GET();
 	tib->tib_thread = thread;
@@ -187,76 +76,24 @@ _rthread_init(void)
 
 	thread->donesem.lock = _SPINLOCK_UNLOCKED;
 	tib->tib_thread_flags = TIB_THREAD_INITIAL_STACK;
-	thread->flags_lock = _SPINLOCK_UNLOCKED;
-	strlcpy(thread->name, "Main process", sizeof(thread->name));
-	LIST_INSERT_HEAD(&_thread_list, thread, threads);
-	_rthread_debug_init();
-
-	_thread_pagesize = (size_t)sysconf(_SC_PAGESIZE);
-	_rthread_attr_default.guard_size = _thread_pagesize;
-	thread->attr = _rthread_attr_default;
-
-	/* get libc to start using our callbacks */
-	{
-		struct thread_callbacks cb = { 0 };
-
-#ifndef TCB_HAVE_MD_GET
-		cb.tc_errnoptr		= multi_threaded_errnoptr;
-		cb.tc_tcb		= multi_threaded_tcb;
-#endif
-		cb.tc_canceled		= _thread_canceled;
-		cb.tc_flockfile		= _thread_flockfile;
-		cb.tc_ftrylockfile	= _thread_ftrylockfile;
-		cb.tc_funlockfile	= _thread_funlockfile;
-		cb.tc_malloc_lock	= _thread_malloc_lock;
-		cb.tc_malloc_unlock	= _thread_malloc_unlock;
-		cb.tc_atexit_lock	= _thread_atexit_lock;
-		cb.tc_atexit_unlock	= _thread_atexit_unlock;
-		cb.tc_atfork_lock	= _thread_atfork_lock;
-		cb.tc_atfork_unlock	= _thread_atfork_unlock;
-		cb.tc_arc4_lock		= _thread_arc4_lock;
-		cb.tc_arc4_unlock	= _thread_arc4_unlock;
-		cb.tc_mutex_lock	= _thread_mutex_lock;
-		cb.tc_mutex_unlock	= _thread_mutex_unlock;
-		cb.tc_mutex_destroy	= _thread_mutex_destroy;
-		cb.tc_tag_lock		= _thread_tag_lock;
-		cb.tc_tag_unlock	= _thread_tag_unlock;
-		cb.tc_tag_storage	= _thread_tag_storage;
-		cb.tc_fork		= _thread_fork;
-		cb.tc_vfork		= _thread_vfork;
-		_thread_set_callbacks(&cb, sizeof(cb));
-	}
-
-#ifndef NO_PIC
-	if (_DYNAMIC) {
-		dlctl(NULL, DL_SETTHREADLCK, _rthread_dl_lock);
-	}
-#endif
 
 	/*
-	 * Set the handler on the signal used for cancelation and
-	 * suspension, and make sure it's unblocked
+	 * Set the debug level from an environment string.
+	 * Bogus values are silently ignored.
 	 */
-	memset(&sa, 0, sizeof(sa));
-	sigemptyset(&sa.sa_mask);
-	sa.sa_handler = sigthr_handler;
-	sigaction(SIGTHR, &sa, NULL);
-	sigaddset(&sa.sa_mask, SIGTHR);
-	sigprocmask(SIG_UNBLOCK, &sa.sa_mask, NULL);
-
-	_threads_ready = 1;
+	if (! issetugid()) {
+		char *envp = getenv(RTHREAD_ENV_DEBUG);
 
-	_malloc_init(1);
+		if (envp != NULL) {
+			char *rem;
 
-	_rthread_debug(1, "rthread init\n");
-}
+			_rthread_debug_level = (int) strtol(envp, &rem, 0);
+			if (*rem != '\0' || _rthread_debug_level < 0)
+				_rthread_debug_level = 0;
+		}
+	}
 
-static void
-_rthread_free(pthread_t thread)
-{
-	_spinlock(&_thread_gc_lock);
-	TAILQ_INSERT_TAIL(&_thread_gc_list, thread, waiting);
-	_spinunlock(&_thread_gc_lock);
+	_threads_inited = 1;
 }
 
 /*
@@ -265,51 +102,21 @@ _rthread_free(pthread_t thread)
 pthread_t
 pthread_self(void)
 {
-	if (!_threads_ready)
+	if (__predict_false(!_threads_inited))
 		_rthread_init();
 
-	return (TIB_GET()->tib_thread);
+	return TIB_GET()->tib_thread;
 }
 DEF_STRONG(pthread_self);
 
-static void
-_rthread_reaper(void)
-{
-	pthread_t thread;
-
-restart:
-	_spinlock(&_thread_gc_lock);
-	TAILQ_FOREACH(thread, &_thread_gc_list, waiting) {
-		if (thread->tib->tib_tid != 0)
-			continue;
-		TAILQ_REMOVE(&_thread_gc_list, thread, waiting);
-		_spinunlock(&_thread_gc_lock);
-		if (thread != &_initial_thread) {
-			_rthread_debug(3, "rthread reaping %p stack %p\n",
-			    (void *)thread, (void *)thread->stack);
-			_rthread_free_stack(thread->stack);
-			_dl_free_tib(thread->tib, sizeof(*thread));
-		} else {
-			/* initial thread isn't part of TIB allocation */
-			_rthread_debug(3, "rthread reaping %p (initial)\n",
-			    (void *)thread);
-			_dl_free_tib(thread->tib, 0);
-		}
-		goto restart;
-	}
-	_spinunlock(&_thread_gc_lock);
-}
-
 void
 pthread_exit(void *retval)
 {
 	struct rthread_cleanup_fn *clfn;
-	struct tib *tib = TIB_GET();
-	pthread_t thread;
+	struct tib *tib;
+	pthread_t thread = pthread_self();
 
-	if (!_threads_ready)
-		_rthread_init();
-	thread = tib->tib_thread;
+	tib = thread->tib;
 
 	if (tib->tib_cantcancel & CANCEL_DYING) {
 		/*
@@ -331,19 +138,9 @@ pthread_exit(void *retval)
 		free(oclfn);
 	}
 	_rthread_tls_destructors(thread);
-	_spinlock(&_thread_lock);
-	LIST_REMOVE(thread, threads);
-	_spinunlock(&_thread_lock);
 
-	_spinlock(&thread->flags_lock);
-	if (thread->flags & THREAD_DETACHED) {
-		_spinunlock(&thread->flags_lock);
-		_rthread_free(thread);
-	} else {
-		thread->flags |= THREAD_DONE;
-		_spinunlock(&thread->flags_lock);
-		_sem_post(&thread->donesem);
-	}
+	if (_thread_cb.tc_thread_release != NULL)
+		_thread_cb.tc_thread_release(thread);
 
 	__threxit(&tib->tib_tid);
 	for(;;);
@@ -351,341 +148,8 @@ pthread_exit(void *retval)
 DEF_STRONG(pthread_exit);
 
 int
-pthread_join(pthread_t thread, void **retval)
-{
-	int e;
-	struct tib *tib = TIB_GET();
-	pthread_t self;
-	PREP_CANCEL_POINT(tib);
-
-	if (_post_threaded) {
-#define GREATSCOTT "great scott! serious repercussions on future events!\n"
-		write(2, GREATSCOTT, sizeof(GREATSCOTT) - 1);
-		abort();
-	}
-	if (!_threads_ready)
-		_rthread_init();
-	self = tib->tib_thread;
-
-	e = 0;
-	ENTER_DELAYED_CANCEL_POINT(tib, self);
-	if (thread == NULL)
-		e = EINVAL;
-	else if (thread == self)
-		e = EDEADLK;
-	else if (thread->flags & THREAD_DETACHED)
-		e = EINVAL;
-	else if ((e = _sem_wait(&thread->donesem, 0, NULL,
-	    &self->delayed_cancel)) == 0) {
-		if (retval)
-			*retval = thread->retval;
-
-		/*
-		 * We should be the last having a ref to this thread,
-		 * but someone stupid or evil might haved detached it;
-		 * in that case the thread will clean up itself
-		 */
-		if ((thread->flags & THREAD_DETACHED) == 0)
-			_rthread_free(thread);
-	}
-
-	LEAVE_CANCEL_POINT_INNER(tib, e);
-	_rthread_reaper();
-	return (e);
-}
-
-int
-pthread_detach(pthread_t thread)
-{
-	int rc = 0;
-
-	_spinlock(&thread->flags_lock);
-	if (thread->flags & THREAD_DETACHED) {
-		rc = EINVAL;
-		_spinunlock(&thread->flags_lock);
-	} else if (thread->flags & THREAD_DONE) {
-		_spinunlock(&thread->flags_lock);
-		_rthread_free(thread);
-	} else {
-		thread->flags |= THREAD_DETACHED;
-		_spinunlock(&thread->flags_lock);
-	}
-	_rthread_reaper();
-	return (rc);
-}
-
-int
-pthread_create(pthread_t *threadp, const pthread_attr_t *attr,
-    void *(*start_routine)(void *), void *arg)
-{
-	extern int __isthreaded;
-	struct tib *tib;
-	pthread_t thread;
-	struct __tfork param;
-	int rc;
-
-	if (!_threads_ready)
-		_rthread_init();
-
-	_rthread_reaper();
-
-	tib = _dl_allocate_tib(sizeof(*thread));
-	if (tib == NULL)
-		return (ENOMEM);
-	thread = tib->tib_thread;
-	memset(thread, 0, sizeof(*thread));
-	thread->tib = tib;
-	thread->donesem.lock = _SPINLOCK_UNLOCKED;
-	thread->flags_lock = _SPINLOCK_UNLOCKED;
-	thread->fn = start_routine;
-	thread->arg = arg;
-	tib->tib_tid = -1;
-
-	thread->attr = attr != NULL ? *(*attr) : _rthread_attr_default;
-	if (thread->attr.sched_inherit == PTHREAD_INHERIT_SCHED) {
-		pthread_t self = pthread_self();
-
-		thread->attr.sched_policy = self->attr.sched_policy;
-		thread->attr.sched_param = self->attr.sched_param;
-	}
-	if (thread->attr.detach_state == PTHREAD_CREATE_DETACHED)
-		thread->flags |= THREAD_DETACHED;
-
-	thread->stack = _rthread_alloc_stack(thread);
-	if (!thread->stack) {
-		rc = errno;
-		goto fail1;
-	}
-
-	param.tf_tcb = TIB_TO_TCB(tib);
-	param.tf_tid = &tib->tib_tid;
-	param.tf_stack = thread->stack->sp;
-
-	_spinlock(&_thread_lock);
-	LIST_INSERT_HEAD(&_thread_list, thread, threads);
-	_spinunlock(&_thread_lock);
-
-	/* we're going to be multi-threaded real soon now */
-	__isthreaded = 1;
-	rc = __tfork_thread(&param, sizeof(param), _rthread_start, thread);
-	if (rc != -1) {
-		/* success */
-		*threadp = thread;
-		return (0);
-	}
-		
-	rc = errno;
-
-	_spinlock(&_thread_lock);
-	LIST_REMOVE(thread, threads);
-	_spinunlock(&_thread_lock);
-	_rthread_free_stack(thread->stack);
-fail1:
-	_dl_free_tib(tib, sizeof(*thread));
-
-	return (rc);
-}
-
-int
-pthread_kill(pthread_t thread, int sig)
-{
-	struct tib *tib = thread->tib;
-
-	if (sig == SIGTHR)
-		return (EINVAL);
-	if (thrkill(tib->tib_tid, sig, TIB_TO_TCB(tib)))
-		return (errno);
-	return (0);
-}
-
-int
 pthread_equal(pthread_t t1, pthread_t t2)
 {
 	return (t1 == t2);
 }
 
-int
-pthread_cancel(pthread_t thread)
-{
-	struct tib *tib = thread->tib;
-	pid_t tid = tib->tib_tid;
-
-	if (tib->tib_canceled == 0 && tid != 0 &&
-	    (tib->tib_cantcancel & CANCEL_DYING) == 0) {
-		tib->tib_canceled = 1;
-
-		if ((tib->tib_cantcancel & CANCEL_DISABLED) == 0) {
-			thrkill(tid, SIGTHR, TIB_TO_TCB(tib));
-			return (0);
-		}
-	}
-	return (0);
-}
-
-void
-pthread_testcancel(void)
-{
-	struct tib *tib = TIB_GET();
-
-	if (tib->tib_canceled && (tib->tib_cantcancel & CANCEL_DISABLED) == 0)
-		pthread_exit(PTHREAD_CANCELED);
-}
-
-int
-pthread_setcancelstate(int state, int *oldstatep)
-{
-	struct tib *tib = TIB_GET();
-	int oldstate;
-
-	oldstate = tib->tib_cantcancel & CANCEL_DISABLED ?
-	    PTHREAD_CANCEL_DISABLE : PTHREAD_CANCEL_ENABLE;
-	if (state == PTHREAD_CANCEL_ENABLE) {
-		tib->tib_cantcancel &= ~CANCEL_DISABLED;
-	} else if (state == PTHREAD_CANCEL_DISABLE) {
-		tib->tib_cantcancel |= CANCEL_DISABLED;
-	} else {
-		return (EINVAL);
-	}
-	if (oldstatep)
-		*oldstatep = oldstate;
-
-	return (0);
-}
-DEF_STRONG(pthread_setcancelstate);
-
-int
-pthread_setcanceltype(int type, int *oldtypep)
-{
-	struct tib *tib = TIB_GET();
-	int oldtype;
-
-	oldtype = tib->tib_thread_flags & TIB_THREAD_ASYNC_CANCEL ?
-	    PTHREAD_CANCEL_ASYNCHRONOUS : PTHREAD_CANCEL_DEFERRED;
-	if (type == PTHREAD_CANCEL_DEFERRED) {
-		tib->tib_thread_flags &=~ TIB_THREAD_ASYNC_CANCEL;
-	} else if (type == PTHREAD_CANCEL_ASYNCHRONOUS) {
-		tib->tib_thread_flags |= TIB_THREAD_ASYNC_CANCEL;
-	} else {
-		return (EINVAL);
-	}
-	if (oldtypep)
-		*oldtypep = oldtype;
-
-	return (0);
-}
-
-void
-pthread_cleanup_push(void (*fn)(void *), void *arg)
-{
-	struct rthread_cleanup_fn *clfn;
-	pthread_t self = pthread_self();
-
-	clfn = calloc(1, sizeof(*clfn));
-	if (!clfn)
-		return;
-	clfn->fn = fn;
-	clfn->arg = arg;
-	clfn->next = self->cleanup_fns;
-	self->cleanup_fns = clfn;
-}
-
-void
-pthread_cleanup_pop(int execute)
-{
-	struct rthread_cleanup_fn *clfn;
-	pthread_t self = pthread_self();
-
-	clfn = self->cleanup_fns;
-	if (clfn) {
-		self->cleanup_fns = clfn->next;
-		if (execute)
-			clfn->fn(clfn->arg);
-		free(clfn);
-	}
-}
-
-int
-pthread_getconcurrency(void)
-{
-	return (concurrency_level);
-}
-
-int
-pthread_setconcurrency(int new_level)
-{
-	if (new_level < 0)
-		return (EINVAL);
-	concurrency_level = new_level;
-	return (0);
-}
-
-/*
- * compat debug stuff
- */
-void
-_thread_dump_info(void)
-{
-	pthread_t thread;
-
-	_spinlock(&_thread_lock);
-	LIST_FOREACH(thread, &_thread_list, threads)
-		printf("thread %d flags 0x%x name %s\n", thread->tib->tib_tid,
-		    thread->tib->tib_thread_flags, thread->name);
-	_spinunlock(&_thread_lock);
-}
-
-#ifndef NO_PIC
-/*
- * _rthread_dl_lock() provides the locking for dlopen(), dlclose(), and
- * the function called via atexit() to invoke all destructors.  The latter
- * two call shared-object destructors, which may need to call dlclose(),
- * so this lock needs to permit recursive locking.
- * The specific code here was extracted from _rthread_mutex_lock() and
- * pthread_mutex_unlock() and simplified to use the static variables.
- */
-void
-_rthread_dl_lock(int what)
-{
-	static _atomic_lock_t lock = _SPINLOCK_UNLOCKED;
-	static pthread_t owner = NULL;
-	static struct pthread_queue lockers = TAILQ_HEAD_INITIALIZER(lockers);
-	static int count = 0;
-
-	if (what == 0) {
-		pthread_t self = pthread_self();
-
-		/* lock, possibly recursive */
-		_spinlock(&lock);
-		if (owner == NULL) {
-			owner = self;
-		} else if (owner != self) {
-			TAILQ_INSERT_TAIL(&lockers, self, waiting);
-			while (owner != self) {
-				__thrsleep(self, 0, NULL, &lock, NULL);
-				_spinlock(&lock);
-			}
-		}
-		count++;
-		_spinunlock(&lock);
-	} else if (what == 1) {
-		/* unlock, possibly recursive */
-		if (--count == 0) {
-			pthread_t next;
-
-			_spinlock(&lock);
-			owner = next = TAILQ_FIRST(&lockers);
-			if (next != NULL)
-				TAILQ_REMOVE(&lockers, next, waiting);
-			_spinunlock(&lock);
-			if (next != NULL)
-				__thrwakeup(next, 1);
-		}
-	} else {
-		/* reinit: used in child after fork to clear the queue */
-		lock = _SPINLOCK_UNLOCKED;
-		if (--count == 0)
-			owner = NULL;
-		TAILQ_INIT(&lockers);
-	}
-}
-#endif
diff --git a/lib/libc/thread/rthread.h b/lib/libc/thread/rthread.h
index 4bae5fd4e1d..fa603c1a792 100644
--- a/lib/libc/thread/rthread.h
+++ b/lib/libc/thread/rthread.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: rthread.h,v 1.1 2017/08/15 06:13:24 guenther Exp $ */
+/*	$OpenBSD: rthread.h,v 1.2 2017/09/05 02:40:54 guenther Exp $ */
 /*
  * Copyright (c) 2004,2005 Ted Unangst <tedu@openbsd.org>
  * All Rights Reserved.
@@ -15,222 +15,27 @@
  * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
  * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
  */
-/*
- * Private data structures that back up the typedefs in pthread.h.
- * Since only the thread library cares about their size or arrangement,
- * it should be possible to switch libraries without relinking.
- *
- * Do not reorder _atomic_lock_t and sem_t variables in the structs.
- * This is due to alignment requirements of certain arches like hppa.
- * The current requirement is 16 bytes.
- *
- * THE MACHINE DEPENDENT CERROR CODE HAS HARD CODED OFFSETS INTO PTHREAD_T!
- */
-
-#include <sys/queue.h>
-#include <semaphore.h>
-#include <machine/spinlock.h>
-
-#ifdef __LP64__
-#define RTHREAD_STACK_SIZE_DEF (512 * 1024)
-#else
-#define RTHREAD_STACK_SIZE_DEF (256 * 1024)
-#endif
-
-#define	_SPINLOCK_UNLOCKED _ATOMIC_LOCK_UNLOCKED
-
-struct stack {
-	SLIST_ENTRY(stack)	link;	/* link for free default stacks */
-	void	*sp;			/* machine stack pointer */
-	void	*base;			/* bottom of allocated area */
-	size_t	guardsize;		/* size of PROT_NONE zone or */
-					/* ==1 if application alloced */
-	size_t	len;			/* total size of allocated stack */
-};
-
-struct __sem {
-	_atomic_lock_t lock;
-	volatile int waitcount;
-	volatile int value;
-	int shared;
-};
-
-TAILQ_HEAD(pthread_queue, pthread);
-
-#ifdef FUTEX
-
-struct pthread_mutex {
-	volatile unsigned int lock;
-	int type;
-	pthread_t owner;
-	int count;
-	int prioceiling;
-};
-
-struct pthread_cond {
-	volatile unsigned int seq;
-	clockid_t clock;
-	struct pthread_mutex *mutex;
-};
-
-#else
-
-struct pthread_mutex {
-	_atomic_lock_t lock;
-	struct pthread_queue lockers;
-	int type;
-	pthread_t owner;
-	int count;
-	int prioceiling;
-};
-
-struct pthread_cond {
-	_atomic_lock_t lock;
-	struct pthread_queue waiters;
-	struct pthread_mutex *mutex;
-	clockid_t clock;
-};
-#endif /* FUTEX */
-
-struct pthread_mutex_attr {
-	int ma_type;
-	int ma_protocol;
-	int ma_prioceiling;
-};
 
-struct pthread_cond_attr {
-	clockid_t ca_clock;
-};
+#ifndef _RTHREAD_H_
+#define _RTHREAD_H_
 
-struct pthread_rwlock {
-	_atomic_lock_t lock;
-	pthread_t owner;
-	struct pthread_queue writers;
-	int readers;
-};
-
-struct pthread_rwlockattr {
-	int pshared;
-};
-
-struct pthread_attr {
-	void *stack_addr;
-	size_t stack_size;
-	size_t guard_size;
-	int detach_state;
-	int contention_scope;
-	int sched_policy;
-	struct sched_param sched_param;
-	int sched_inherit;
-};
-
-#define	PTHREAD_MIN_PRIORITY	0
-#define	PTHREAD_MAX_PRIORITY	31
-
-struct rthread_key {
-	int used;
-	void (*destructor)(void *);
-};
-
-struct rthread_storage {
-	int keyid;
-	struct rthread_storage *next;
-	void *data;
-};
-
-struct rthread_cleanup_fn {
-	void (*fn)(void *);
-	void *arg;
-	struct rthread_cleanup_fn *next;
-};
-
-struct pthread_barrier {
-	pthread_mutex_t mutex;
-	pthread_cond_t cond;
-	int threshold;
-	int in;
-	int out;
-	int generation;
-};
-
-struct pthread_barrierattr {
-	int pshared;
-};
-
-struct pthread_spinlock {
-	_atomic_lock_t lock;
-	pthread_t owner;
-};
-
-struct tib;
-struct pthread {
-	struct __sem donesem;
-	unsigned int flags;
-	_atomic_lock_t flags_lock;
-	struct tib *tib;
-	void *retval;
-	void *(*fn)(void *);
-	void *arg;
-	char name[32];
-	struct stack *stack;
-	LIST_ENTRY(pthread) threads;
-	TAILQ_ENTRY(pthread) waiting;
-	pthread_cond_t blocking_cond;
-	struct pthread_attr attr;
-	struct rthread_storage *local_storage;
-	struct rthread_cleanup_fn *cleanup_fns;
-	int myerrno;
-
-	/* cancel received in a delayed cancel block? */
-	int delayed_cancel;
-};
-/* flags in pthread->flags */
-#define	THREAD_DONE		0x001
-#define	THREAD_DETACHED		0x002
-
-/* flags in tib->tib_thread_flags */
-#define	TIB_THREAD_ASYNC_CANCEL		0x001
-#define	TIB_THREAD_INITIAL_STACK	0x002	/* has stack from exec */
-
-#define ENTER_DELAYED_CANCEL_POINT(tib, self)				\
-	(self)->delayed_cancel = 0;					\
-	ENTER_CANCEL_POINT_INNER(tib, 1, 1)
-
-#define	ROUND_TO_PAGE(size) \
-	(((size) + (_thread_pagesize - 1)) & ~(_thread_pagesize - 1))
+#include "thread_private.h"
 
 __BEGIN_HIDDEN_DECLS
-void	_spinlock(volatile _atomic_lock_t *);
-int	_spinlocktry(volatile _atomic_lock_t *);
-void	_spinunlock(volatile _atomic_lock_t *);
-int	_sem_wait(sem_t, int, const struct timespec *, int *);
-int	_sem_post(sem_t);
-
-void	_rthread_init(void);
-struct stack *_rthread_alloc_stack(pthread_t);
-void	_rthread_free_stack(struct stack *);
 void	_rthread_tls_destructors(pthread_t);
-void	_rthread_debug(int, const char *, ...)
-		__attribute__((__format__ (printf, 2, 3)));
-void	_rthread_debug_init(void);
-#ifndef NO_PIC
-void	_rthread_dl_lock(int what);
-#endif
-void	_thread_malloc_reinit(void);
 
-extern int _threads_ready;
-extern size_t _thread_pagesize;
-extern LIST_HEAD(listhead, pthread) _thread_list;
-extern _atomic_lock_t _thread_lock;
-extern struct pthread_attr _rthread_attr_default;
+extern int _rthread_debug_level;
+extern struct pthread _initial_thread;
 __END_HIDDEN_DECLS
 
-void	_thread_dump_info(void);
+PROTO_NORMAL(__threxit);
+PROTO_NORMAL(__thrsigdivert);
+PROTO_NORMAL(__thrsleep);
+PROTO_NORMAL(__thrwakeup);
+
+PROTO_NORMAL(_spinlock);
+PROTO_STD_DEPRECATED(_spinlocktry);
+PROTO_NORMAL(_spinunlock);
+PROTO_NORMAL(_rthread_debug);
 
-/* syscalls not declared in system headers */
-#define REDIRECT_SYSCALL(x)		typeof(x) x asm("_thread_sys_"#x)
-void	__threxit(pid_t *);
-int	__thrsleep(const volatile void *, clockid_t, const struct timespec *,
-	    volatile void *, const int *);
-int	__thrwakeup(const volatile void *, int n);
-int	__thrsigdivert(sigset_t, siginfo_t *, const struct timespec *);
+#endif /* _RTHREAD_H_ */
diff --git a/lib/libc/thread/rthread_cb.h b/lib/libc/thread/rthread_cb.h
index 4865e2ec4a2..3e8604c4ed4 100644
--- a/lib/libc/thread/rthread_cb.h
+++ b/lib/libc/thread/rthread_cb.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: rthread_cb.h,v 1.1 2017/08/15 06:13:24 guenther Exp $ */
+/*	$OpenBSD: rthread_cb.h,v 1.2 2017/09/05 02:40:54 guenther Exp $ */
 /*
  * Copyright (c) 2016 Philip Guenther <guenther@openbsd.org>
  * All Rights Reserved.
@@ -19,8 +19,6 @@
 #include <stdio.h>
 
 __BEGIN_HIDDEN_DECLS
-pid_t	_thread_fork(void);
-pid_t	_thread_vfork(void);
 void	_thread_flockfile(FILE *);
 int	_thread_ftrylockfile(FILE *);
 void	_thread_funlockfile(FILE *);
diff --git a/lib/libc/thread/rthread_cond.c b/lib/libc/thread/rthread_cond.c
index fb69dd2cb0a..3fd9bfcd56e 100644
--- a/lib/libc/thread/rthread_cond.c
+++ b/lib/libc/thread/rthread_cond.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: rthread_cond.c,v 1.3 2017/08/15 07:06:29 guenther Exp $ */
+/*	$OpenBSD: rthread_cond.c,v 1.4 2017/09/05 02:40:54 guenther Exp $ */
 /*
  * Copyright (c) 2017 Martin Pieuchot <mpi@openbsd.org>
  * Copyright (c) 2012 Philip Guenther <guenther@openbsd.org>
@@ -68,7 +68,6 @@ pthread_cond_destroy(pthread_cond_t *condp)
 
 	return (0);
 }
-DEF_STRONG(pthread_cond_destroy);
 
 int
 _rthread_cond_timedwait(pthread_cond_t cond, pthread_mutex_t *mutexp,
@@ -166,7 +165,6 @@ pthread_cond_wait(pthread_cond_t *condp, pthread_mutex_t *mutexp)
 	cond = *condp;
 	return (_rthread_cond_timedwait(cond, mutexp, NULL));
 }
-DEF_STRONG(pthread_cond_wait);
 
 int
 pthread_cond_signal(pthread_cond_t *condp)
@@ -187,7 +185,6 @@ pthread_cond_signal(pthread_cond_t *condp)
 
 	return (0);
 }
-DEF_STRONG(pthread_cond_signal);
 
 int
 pthread_cond_broadcast(pthread_cond_t *condp)
@@ -212,4 +209,3 @@ pthread_cond_broadcast(pthread_cond_t *condp)
 
 	return (0);
 }
-DEF_STRONG(pthread_cond_broadcast);
diff --git a/lib/libc/thread/rthread_condattr.c b/lib/libc/thread/rthread_condattr.c
index 3a5d8772b05..be3efc33e6a 100644
--- a/lib/libc/thread/rthread_condattr.c
+++ b/lib/libc/thread/rthread_condattr.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: rthread_condattr.c,v 1.2 2017/08/15 06:38:41 guenther Exp $ */
+/*	$OpenBSD: rthread_condattr.c,v 1.3 2017/09/05 02:40:54 guenther Exp $ */
 /*
  * Copyright (c) 2004,2005 Ted Unangst <tedu@openbsd.org>
  * Copyright (c) 2012 Philip Guenther <guenther@openbsd.org>
@@ -20,12 +20,9 @@
  * Condition Variable Attributes
  */
 
-#include <assert.h>
 #include <errno.h>
 #include <pthread.h>
 #include <stdlib.h>
-#include <string.h>
-#include <unistd.h>
 
 #include "rthread.h"
 
diff --git a/lib/libc/thread/rthread_debug.c b/lib/libc/thread/rthread_debug.c
index 18d3a8567d6..67dca29e054 100644
--- a/lib/libc/thread/rthread_debug.c
+++ b/lib/libc/thread/rthread_debug.c
@@ -1,27 +1,14 @@
-/* $OpenBSD: rthread_debug.c,v 1.2 2017/08/15 06:38:41 guenther Exp $ */
+/* $OpenBSD: rthread_debug.c,v 1.3 2017/09/05 02:40:54 guenther Exp $ */
 
 /* PUBLIC DOMAIN: No Rights Reserved. Marco S Hyman <marc@snafu.org> */
 
 #include <pthread.h>
 #include <stdarg.h>
 #include <stdio.h>
-#include <stdlib.h>
 #include <unistd.h>
 
 #include "rthread.h"
 
-REDIRECT_SYSCALL(issetugid);
-
-int _rthread_debug_level;
-
-/*
- * Note: messages truncated at 255 characters.   Could use vasprintf,
- * but don't want to use malloc here so the function can be used
- * in signal handlers.
- */
-#define MAX_MSG_LEN 256
-#define RTHREAD_ENV_DEBUG	"RTHREAD_DEBUG"
-
 /*
  * format and send output to stderr if the given "level" is less than or
  * equal to the current debug level.   Messages with a level <= 0 will
@@ -30,47 +17,12 @@ int _rthread_debug_level;
 void
 _rthread_debug(int level, const char *fmt, ...)
 {
-	char msg[MAX_MSG_LEN];
-	char *p;
-	int cnt;
-	ssize_t c;
-
 	if (_rthread_debug_level >= level) {
 		va_list ap;
 		va_start(ap, fmt);
-		cnt = vsnprintf(msg, MAX_MSG_LEN, fmt, ap);
+		vdprintf(STDERR_FILENO, fmt, ap);
 		va_end(ap);
-		if (cnt > MAX_MSG_LEN - 1)
-			cnt = MAX_MSG_LEN - 1;
-		p = msg;
-		do {
-			c = write(STDERR_FILENO, p, cnt);
-			if (c == -1)
-				break;
-			if (c != cnt)
-				sched_yield();
-			p += c;
-			cnt -= c;
-		} while (cnt > 0);
 	}
 }
+DEF_STRONG(_rthread_debug);
 
-/*
- * set the debug level from an environment string.  Bogus values are
- * silently ignored.
- */
-void
-_rthread_debug_init(void)
-{
-	char *envp;
-	char *rem;
-
-	if (issetugid())
-		return;
-	envp = getenv(RTHREAD_ENV_DEBUG);
-	if (envp) {
-		_rthread_debug_level = (int) strtol(envp, &rem, 0);
-		if (*rem || _rthread_debug_level < 0)
-			_rthread_debug_level = 0;
-	}
-}
diff --git a/lib/libc/thread/rthread_libc.c b/lib/libc/thread/rthread_libc.c
index 406fc9f939e..b3fcf22cad5 100644
--- a/lib/libc/thread/rthread_libc.c
+++ b/lib/libc/thread/rthread_libc.c
@@ -1,15 +1,11 @@
-/* $OpenBSD: rthread_libc.c,v 1.1 2017/08/15 06:13:24 guenther Exp $ */
+/* $OpenBSD: rthread_libc.c,v 1.2 2017/09/05 02:40:54 guenther Exp $ */
 
 /* PUBLIC DOMAIN: No Rights Reserved. Marco S Hyman <marc@snafu.org> */
 
-#include <sys/time.h>
-
 #include <pthread.h>
 #include <stdlib.h>
 #include <string.h>
 
-#include "thread_private.h"	/* in libc/include */
-
 #include "rthread.h"
 #include "rthread_cb.h"
 
@@ -195,7 +191,7 @@ _thread_malloc_unlock(int i)
 	pthread_mutex_unlock(&malloc_mutex[i]);
 }
 
-void
+static void
 _thread_malloc_reinit(void)
 {
 	int i;
@@ -260,3 +256,28 @@ _thread_arc4_unlock(void)
 {
 	_spinunlock(&arc4_lock);
 }
+
+pid_t
+_thread_dofork(pid_t (*sys_fork)(void))
+{
+	int i;
+	pid_t newid;
+
+	_thread_atexit_lock();
+	for (i = 0; i < _MALLOC_MUTEXES; i++)
+		_thread_malloc_lock(i);
+	_thread_arc4_lock();
+
+	newid = sys_fork();
+
+	_thread_arc4_unlock();
+	if (newid == 0)
+		_thread_malloc_reinit();
+	else
+		for (i = 0; i < _MALLOC_MUTEXES; i++)
+			_thread_malloc_unlock(i);
+	_thread_atexit_unlock();
+
+	return newid;
+}
+
diff --git a/lib/libc/thread/rthread_sync.c b/lib/libc/thread/rthread_sync.c
index 6f6bda4e6a4..91ce55cbcf9 100644
--- a/lib/libc/thread/rthread_sync.c
+++ b/lib/libc/thread/rthread_sync.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: rthread_sync.c,v 1.3 2017/08/15 07:06:29 guenther Exp $ */
+/*	$OpenBSD: rthread_sync.c,v 1.4 2017/09/05 02:40:54 guenther Exp $ */
 /*
  * Copyright (c) 2004,2005 Ted Unangst <tedu@openbsd.org>
  * Copyright (c) 2012 Philip Guenther <guenther@openbsd.org>
@@ -280,7 +280,6 @@ pthread_cond_destroy(pthread_cond_t *condp)
 
 	return (0);
 }
-DEF_STRONG(pthread_cond_destroy);
 
 int
 pthread_cond_timedwait(pthread_cond_t *condp, pthread_mutex_t *mutexp,
@@ -573,7 +572,6 @@ pthread_cond_wait(pthread_cond_t *condp, pthread_mutex_t *mutexp)
 
 	return (0);
 }
-DEF_STRONG(pthread_cond_wait);
 
 
 int
@@ -623,7 +621,6 @@ pthread_cond_signal(pthread_cond_t *condp)
 
 	return (0);
 }
-DEF_STRONG(pthread_cond_signal);
 
 int
 pthread_cond_broadcast(pthread_cond_t *condp)
@@ -689,4 +686,3 @@ pthread_cond_broadcast(pthread_cond_t *condp)
 
 	return (0);
 }
-DEF_STRONG(pthread_cond_broadcast);
diff --git a/lib/libc/thread/rthread_tls.c b/lib/libc/thread/rthread_tls.c
index ba92ca1ff30..58b39bb7df7 100644
--- a/lib/libc/thread/rthread_tls.c
+++ b/lib/libc/thread/rthread_tls.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: rthread_tls.c,v 1.3 2017/08/15 07:06:29 guenther Exp $ */
+/*	$OpenBSD: rthread_tls.c,v 1.4 2017/09/05 02:40:54 guenther Exp $ */
 /*
  * Copyright (c) 2004,2005 Ted Unangst <tedu@openbsd.org>
  * All Rights Reserved.
@@ -23,8 +23,17 @@
 #include <pthread.h>
 #include <stdlib.h>
 
+#include <pthread.h>
+#include <stdlib.h>
+
 #include "rthread.h"
 
+
+struct rthread_key {
+	int used;
+	void (*destructor)(void *);
+};
+
 static struct rthread_key rkeys[PTHREAD_KEYS_MAX];
 static _atomic_lock_t rkeyslock = _SPINLOCK_UNLOCKED;
 
@@ -61,7 +70,6 @@ DEF_STRONG(pthread_key_create);
 int
 pthread_key_delete(pthread_key_t key)
 {
-	pthread_t thread;
 	struct rthread_storage *rs;
 	int rv = 0;
 
@@ -76,14 +84,14 @@ pthread_key_delete(pthread_key_t key)
 
 	rkeys[key].used = 0;
 	rkeys[key].destructor = NULL;
-	_spinlock(&_thread_lock);
-	LIST_FOREACH(thread, &_thread_list, threads) {
-		for (rs = thread->local_storage; rs; rs = rs->next) {
+	if (_thread_cb.tc_thread_key_zero != NULL)
+		_thread_cb.tc_thread_key_zero(key);
+	else {
+		for (rs = _initial_thread.local_storage; rs; rs = rs->next) {
 			if (rs->keyid == key)
 				rs->data = NULL;
 		}
 	}
-	_spinunlock(&_thread_lock);
 
 out:
 	_spinunlock(&rkeyslock);
diff --git a/lib/libc/thread/synch.h b/lib/libc/thread/synch.h
index 0ac8b3419b1..8ab379530e8 100644
--- a/lib/libc/thread/synch.h
+++ b/lib/libc/thread/synch.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: synch.h,v 1.1 2017/08/15 06:13:24 guenther Exp $ */
+/*	$OpenBSD: synch.h,v 1.2 2017/09/05 02:40:54 guenther Exp $ */
 /*
  * Copyright (c) 2017 Martin Pieuchot
  *
@@ -19,8 +19,6 @@
 #include <sys/time.h>
 #include <sys/futex.h>
 
-REDIRECT_SYSCALL(futex);
-
 static inline int
 _wake(volatile uint32_t *p, int n)
 {