Reimplement mutexes, condvars, and rwlocks to eliminate bugs,

particularly the "consume the signal you just sent" hang, and putting
the wait queues in userspace.

Do cancellation handling in pthread_cond_*wait(), pthread_join(),
and sem_wait().

Add __ prefix to thr{sleep,wakeup,exit,sigdivert}() syscalls; add
'abort" argument to thrsleep to close cancellation race; make
thr{sleep,wakeup} return errno values via *retval to avoid touching
userspace errno.

8 files changed, 111 insertions, 75 deletions

diff --git a/sys/kern/init_sysent.c b/sys/kern/init_sysent.c
index 215f213454e..be822ad146e 100644
--- a/sys/kern/init_sysent.c
+++ b/sys/kern/init_sysent.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: init_sysent.c,v 1.131 2011/10/15 23:36:31 guenther Exp $	*/
+/*	$OpenBSD: init_sysent.c,v 1.132 2012/01/17 02:34:18 guenther Exp $	*/
 
 /*
  * System call switch table.
@@ -731,14 +731,14 @@ struct sysent sysent[] = {
 	    sys_sched_yield },			/* 298 = sched_yield */
 	{ 0, 0, SY_NOLOCK | 0,
 	    sys_getthrid },			/* 299 = getthrid */
-	{ 4, s(struct sys_thrsleep_args), 0,
-	    sys_thrsleep },			/* 300 = thrsleep */
-	{ 2, s(struct sys_thrwakeup_args), 0,
-	    sys_thrwakeup },			/* 301 = thrwakeup */
-	{ 1, s(struct sys_threxit_args), 0,
-	    sys_threxit },			/* 302 = threxit */
-	{ 3, s(struct sys_thrsigdivert_args), 0,
-	    sys_thrsigdivert },			/* 303 = thrsigdivert */
+	{ 5, s(struct sys___thrsleep_args), 0,
+	    sys___thrsleep },			/* 300 = __thrsleep */
+	{ 2, s(struct sys___thrwakeup_args), 0,
+	    sys___thrwakeup },			/* 301 = __thrwakeup */
+	{ 1, s(struct sys___threxit_args), 0,
+	    sys___threxit },			/* 302 = __threxit */
+	{ 3, s(struct sys___thrsigdivert_args), 0,
+	    sys___thrsigdivert },		/* 303 = __thrsigdivert */
 	{ 2, s(struct sys___getcwd_args), 0,
 	    sys___getcwd },			/* 304 = __getcwd */
 	{ 2, s(struct sys_adjfreq_args), 0,
diff --git a/sys/kern/kern_exit.c b/sys/kern/kern_exit.c
index ecd63cf0fda..50e9e0e3cf9 100644
--- a/sys/kern/kern_exit.c
+++ b/sys/kern/kern_exit.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: kern_exit.c,v 1.105 2011/12/14 07:32:16 guenther Exp $	*/
+/*	$OpenBSD: kern_exit.c,v 1.106 2012/01/17 02:34:18 guenther Exp $	*/
 /*	$NetBSD: kern_exit.c,v 1.39 1996/04/22 01:38:25 christos Exp $	*/
 
 /*
@@ -91,9 +91,9 @@ sys_exit(struct proc *p, void *v, register_t *retval)
 }
 
 int
-sys_threxit(struct proc *p, void *v, register_t *retval)
+sys___threxit(struct proc *p, void *v, register_t *retval)
 {
-	struct sys_threxit_args /* {
+	struct sys___threxit_args /* {
 		syscallarg(pid_t *) notdead;
 	} */ *uap = v;
 
diff --git a/sys/kern/kern_sig.c b/sys/kern/kern_sig.c
index 0775a1edf52..859e1ff75aa 100644
--- a/sys/kern/kern_sig.c
+++ b/sys/kern/kern_sig.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: kern_sig.c,v 1.131 2011/12/11 19:42:28 guenther Exp $	*/
+/*	$OpenBSD: kern_sig.c,v 1.132 2012/01/17 02:34:18 guenther Exp $	*/
 /*	$NetBSD: kern_sig.c,v 1.54 1996/04/22 01:38:32 christos Exp $	*/
 
 /*
@@ -1503,9 +1503,9 @@ sys_nosys(struct proc *p, void *v, register_t *retval)
 }
 
 int
-sys_thrsigdivert(struct proc *p, void *v, register_t *retval)
+sys___thrsigdivert(struct proc *p, void *v, register_t *retval)
 {
-	struct sys_thrsigdivert_args /* {
+	struct sys___thrsigdivert_args /* {
 		syscallarg(sigset_t) sigmask;
 		syscallarg(siginfo_t *) info;
 		syscallarg(const struct timespec *) timeout;
diff --git a/sys/kern/kern_synch.c b/sys/kern/kern_synch.c
index 4b711711c02..6b7f80ab747 100644
--- a/sys/kern/kern_synch.c
+++ b/sys/kern/kern_synch.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: kern_synch.c,v 1.98 2011/12/11 19:42:28 guenther Exp $	*/
+/*	$OpenBSD: kern_synch.c,v 1.99 2012/01/17 02:34:18 guenther Exp $	*/
 /*	$NetBSD: kern_synch.c,v 1.37 1996/04/22 01:38:37 christos Exp $	*/
 
 /*
@@ -404,37 +404,49 @@ sys_sched_yield(struct proc *p, void *v, register_t *retval)
 }
 
 int
-sys_thrsleep(struct proc *p, void *v, register_t *revtal)
+sys___thrsleep(struct proc *p, void *v, register_t *retval)
 {
-	struct sys_thrsleep_args /* {
-		syscallarg(void *) ident;
+	struct sys___thrsleep_args /* {
+		syscallarg(const volatile void *) ident;
 		syscallarg(clockid_t) clock_id;
 		syscallarg(struct timespec *) tp;
 		syscallarg(void *) lock;
+		syscallarg(const int *) abort;
 	} */ *uap = v;
 	long ident = (long)SCARG(uap, ident);
 	_spinlock_lock_t *lock = SCARG(uap, lock);
 	static _spinlock_lock_t unlocked = _SPINLOCK_UNLOCKED;
 	long long to_ticks = 0;
-	int error;
+	int abort, error;
 
-	if (!rthreads_enabled)
-		return (ENOTSUP);
+	if (!rthreads_enabled) {
+		*retval = ENOTSUP;
+		return (0);
+	}
+	if (ident == 0) {
+		*retval = EINVAL;
+		return (0);
+	}
 	if (SCARG(uap, tp) != NULL) {
 		struct timespec now, ats;
 
-		if ((error = copyin(SCARG(uap, tp), &ats, sizeof(ats))) != 0 ||
-		    (error = clock_gettime(p, SCARG(uap, clock_id), &now)) != 0)
-			return (error);
+		if ((error = copyin(SCARG(uap, tp), &ats, sizeof(ats))) ||
+		    (error = clock_gettime(p, SCARG(uap, clock_id), &now))) {
+			*retval = error;
+			return (0);
+		}
 
 		if (timespeccmp(&ats, &now, <)) {
 			/* already passed: still do the unlock */
 			if (lock) {
 				if ((error = copyout(&unlocked, lock,
-				    sizeof(unlocked))) != 0)
-					return (error);
+				    sizeof(unlocked))) != 0) {
+					*retval = error;
+					return (0);
+				}
 			}
-			return (EWOULDBLOCK);
+			*retval = EWOULDBLOCK;
+			return (0);
 		}
 
 		timespecsub(&ats, &now, &ats);
@@ -450,23 +462,41 @@ sys_thrsleep(struct proc *p, void *v, register_t *revtal)
 
 	if (lock) {
 		if ((error = copyout(&unlocked, lock, sizeof(unlocked))) != 0)
-			return (error);
+			goto out;
 	}
-	error = tsleep(&p->p_thrslpid, PUSER | PCATCH, "thrsleep",
-	    (int)to_ticks);
+
+	if (SCARG(uap, abort) != NULL) {
+		if ((error = copyin(SCARG(uap, abort), &abort,
+		    sizeof(abort))) != 0)
+			goto out;
+		if (abort) {
+			error = EINTR;
+			goto out;
+		}
+	}
+
+	if (p->p_thrslpid == 0)
+		error = 0;
+	else
+		error = tsleep(&p->p_thrslpid, PUSER | PCATCH, "thrsleep",
+		    (int)to_ticks);
+
+out:
+	p->p_thrslpid = 0;
 
 	if (error == ERESTART)
 		error = EINTR;
 
-	return (error);
+	*retval = error;
+	return (0);
 
 }
 
 int
-sys_thrwakeup(struct proc *p, void *v, register_t *retval)
+sys___thrwakeup(struct proc *p, void *v, register_t *retval)
 {
-	struct sys_thrwakeup_args /* {
-		syscallarg(void *) ident;
+	struct sys___thrwakeup_args /* {
+		syscallarg(const volatile void *) ident;
 		syscallarg(int) n;
 	} */ *uap = v;
 	long ident = (long)SCARG(uap, ident);
@@ -475,17 +505,20 @@ sys_thrwakeup(struct proc *p, void *v, register_t *retval)
 	int found = 0;
 
 	if (!rthreads_enabled)
-		return (ENOTSUP);
-	TAILQ_FOREACH(q, &p->p_p->ps_threads, p_thr_link) {
-		if (q->p_thrslpid == ident) {
-			wakeup_one(&q->p_thrslpid);
-			q->p_thrslpid = 0;
-			if (++found == n)
-				return (0);
+		*retval = ENOTSUP;
+	else if (ident == 0)
+		*retval = EINVAL;
+	else {
+		TAILQ_FOREACH(q, &p->p_p->ps_threads, p_thr_link) {
+			if (q->p_thrslpid == ident) {
+				wakeup_one(&q->p_thrslpid);
+				q->p_thrslpid = 0;
+				if (++found == n)
+					break;
+			}
 		}
+		*retval = found ? 0 : ESRCH;
 	}
-	if (!found)
-		return (ESRCH);
 
 	return (0);
 }
diff --git a/sys/kern/syscalls.c b/sys/kern/syscalls.c
index 29bad4c5909..a7880b9ef64 100644
--- a/sys/kern/syscalls.c
+++ b/sys/kern/syscalls.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: syscalls.c,v 1.132 2011/10/15 23:36:31 guenther Exp $	*/
+/*	$OpenBSD: syscalls.c,v 1.133 2012/01/17 02:34:18 guenther Exp $	*/
 
 /*
  * System call names.
@@ -382,10 +382,10 @@ char *syscallnames[] = {
 #endif
 	"sched_yield",			/* 298 = sched_yield */
 	"getthrid",			/* 299 = getthrid */
-	"thrsleep",			/* 300 = thrsleep */
-	"thrwakeup",			/* 301 = thrwakeup */
-	"threxit",			/* 302 = threxit */
-	"thrsigdivert",			/* 303 = thrsigdivert */
+	"__thrsleep",			/* 300 = __thrsleep */
+	"__thrwakeup",			/* 301 = __thrwakeup */
+	"__threxit",			/* 302 = __threxit */
+	"__thrsigdivert",			/* 303 = __thrsigdivert */
 	"__getcwd",			/* 304 = __getcwd */
 	"adjfreq",			/* 305 = adjfreq */
 	"getfsstat",			/* 306 = getfsstat */
diff --git a/sys/kern/syscalls.master b/sys/kern/syscalls.master
index c8011d97d42..af289aab619 100644
--- a/sys/kern/syscalls.master
+++ b/sys/kern/syscalls.master
@@ -1,4 +1,4 @@
-;	$OpenBSD: syscalls.master,v 1.119 2011/10/15 23:35:29 guenther Exp $
+;	$OpenBSD: syscalls.master,v 1.120 2012/01/17 02:34:18 guenther Exp $
 ;	$NetBSD: syscalls.master,v 1.32 1996/04/23 10:24:21 mycroft Exp $
 
 ;	@(#)syscalls.master	8.2 (Berkeley) 1/13/94
@@ -524,11 +524,13 @@
 #endif
 298	STD NOLOCK	{ int sys_sched_yield(void); }
 299	STD NOLOCK	{ pid_t sys_getthrid(void); }
-300	STD		{ int sys_thrsleep(void *ident, clockid_t clock_id, \
-			    const struct timespec *tp, void *lock); }
-301	STD		{ int sys_thrwakeup(void *ident, int n); }
-302	STD		{ void sys_threxit(pid_t *notdead); }
-303	STD		{ int sys_thrsigdivert(sigset_t sigmask, \
+300	STD		{ int sys___thrsleep(const volatile void *ident, \
+			    clockid_t clock_id, const struct timespec *tp, \
+			    void *lock, const int *abort); }
+301	STD		{ int sys___thrwakeup(const volatile void *ident, \
+			    int n); }
+302	STD		{ void sys___threxit(pid_t *notdead); }
+303	STD		{ int sys___thrsigdivert(sigset_t sigmask, \
 			    siginfo_t *info, const struct timespec *timeout); }
 304	STD		{ int sys___getcwd(char *buf, size_t len); }
 305	STD		{ int sys_adjfreq(const int64_t *freq, \
diff --git a/sys/sys/syscall.h b/sys/sys/syscall.h
index 0145f63cc6b..ccd89b97435 100644
--- a/sys/sys/syscall.h
+++ b/sys/sys/syscall.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: syscall.h,v 1.131 2011/10/15 23:36:31 guenther Exp $	*/
+/*	$OpenBSD: syscall.h,v 1.132 2012/01/17 02:34:18 guenther Exp $	*/
 
 /*
  * System call numbers.
@@ -589,17 +589,17 @@
 /* syscall: "getthrid" ret: "pid_t" args: */
 #define	SYS_getthrid	299
 
-/* syscall: "thrsleep" ret: "int" args: "void *" "clockid_t" "const struct timespec *" "void *" */
-#define	SYS_thrsleep	300
+/* syscall: "__thrsleep" ret: "int" args: "const volatile void *" "clockid_t" "const struct timespec *" "void *" "const int *" */
+#define	SYS___thrsleep	300
 
-/* syscall: "thrwakeup" ret: "int" args: "void *" "int" */
-#define	SYS_thrwakeup	301
+/* syscall: "__thrwakeup" ret: "int" args: "const volatile void *" "int" */
+#define	SYS___thrwakeup	301
 
-/* syscall: "threxit" ret: "void" args: "pid_t *" */
-#define	SYS_threxit	302
+/* syscall: "__threxit" ret: "void" args: "pid_t *" */
+#define	SYS___threxit	302
 
-/* syscall: "thrsigdivert" ret: "int" args: "sigset_t" "siginfo_t *" "const struct timespec *" */
-#define	SYS_thrsigdivert	303
+/* syscall: "__thrsigdivert" ret: "int" args: "sigset_t" "siginfo_t *" "const struct timespec *" */
+#define	SYS___thrsigdivert	303
 
 /* syscall: "__getcwd" ret: "int" args: "char *" "size_t" */
 #define	SYS___getcwd	304
diff --git a/sys/sys/syscallargs.h b/sys/sys/syscallargs.h
index 5e3077f901a..f43f66d46f3 100644
--- a/sys/sys/syscallargs.h
+++ b/sys/sys/syscallargs.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: syscallargs.h,v 1.133 2011/10/15 23:36:31 guenther Exp $	*/
+/*	$OpenBSD: syscallargs.h,v 1.134 2012/01/17 02:34:18 guenther Exp $	*/
 
 /*
  * System call argument lists.
@@ -880,23 +880,24 @@ struct sys_msgctl_args {
 	syscallarg(struct msqid_ds *) buf;
 };
 
-struct sys_thrsleep_args {
-	syscallarg(void *) ident;
+struct sys___thrsleep_args {
+	syscallarg(const volatile void *) ident;
 	syscallarg(clockid_t) clock_id;
 	syscallarg(const struct timespec *) tp;
 	syscallarg(void *) lock;
+	syscallarg(const int *) abort;
 };
 
-struct sys_thrwakeup_args {
-	syscallarg(void *) ident;
+struct sys___thrwakeup_args {
+	syscallarg(const volatile void *) ident;
 	syscallarg(int) n;
 };
 
-struct sys_threxit_args {
+struct sys___threxit_args {
 	syscallarg(pid_t *) notdead;
 };
 
-struct sys_thrsigdivert_args {
+struct sys___thrsigdivert_args {
 	syscallarg(sigset_t) sigmask;
 	syscallarg(siginfo_t *) info;
 	syscallarg(const struct timespec *) timeout;
@@ -1279,10 +1280,10 @@ int	sys_msgctl(struct proc *, void *, register_t *);
 #endif
 int	sys_sched_yield(struct proc *, void *, register_t *);
 int	sys_getthrid(struct proc *, void *, register_t *);
-int	sys_thrsleep(struct proc *, void *, register_t *);
-int	sys_thrwakeup(struct proc *, void *, register_t *);
-int	sys_threxit(struct proc *, void *, register_t *);
-int	sys_thrsigdivert(struct proc *, void *, register_t *);
+int	sys___thrsleep(struct proc *, void *, register_t *);
+int	sys___thrwakeup(struct proc *, void *, register_t *);
+int	sys___threxit(struct proc *, void *, register_t *);
+int	sys___thrsigdivert(struct proc *, void *, register_t *);
 int	sys___getcwd(struct proc *, void *, register_t *);
 int	sys_adjfreq(struct proc *, void *, register_t *);
 int	sys_getfsstat(struct proc *, void *, register_t *);