Enable the NET_LOCK(), take 2.

Recursions are currently known and marked a XXXSMP.

Please report any assert to bugs@

11 files changed, 92 insertions, 24 deletions

diff --git a/sys/kern/sched_bsd.c b/sys/kern/sched_bsd.c
index 8b318df5996..fa349bafaab 100644
--- a/sys/kern/sched_bsd.c
+++ b/sys/kern/sched_bsd.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: sched_bsd.c,v 1.43 2016/03/09 13:38:50 mpi Exp $	*/
+/*	$OpenBSD: sched_bsd.c,v 1.44 2017/01/25 06:15:50 mpi Exp $	*/
 /*	$NetBSD: kern_synch.c,v 1.37 1996/04/22 01:38:37 christos Exp $	*/
 
 /*-
@@ -297,6 +297,8 @@ yield(void)
 	struct proc *p = curproc;
 	int s;
 
+	NET_ASSERT_UNLOCKED();
+
 	SCHED_LOCK(s);
 	p->p_priority = p->p_usrpri;
 	p->p_stat = SRUN;
diff --git a/sys/kern/uipc_socket.c b/sys/kern/uipc_socket.c
index 245210e594a..06a1340e457 100644
--- a/sys/kern/uipc_socket.c
+++ b/sys/kern/uipc_socket.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: uipc_socket.c,v 1.171 2016/12/29 12:12:43 mpi Exp $	*/
+/*	$OpenBSD: uipc_socket.c,v 1.172 2017/01/25 06:15:50 mpi Exp $	*/
 /*	$NetBSD: uipc_socket.c,v 1.21 1996/02/04 02:17:52 christos Exp $	*/
 
 /*
@@ -256,7 +256,7 @@ soclose(struct socket *so)
 			    (so->so_state & SS_NBIO))
 				goto drop;
 			while (so->so_state & SS_ISCONNECTED) {
-				error = tsleep(&so->so_timeo,
+				error = rwsleep(&so->so_timeo, &netlock,
 				    PSOCK | PCATCH, "netcls",
 				    so->so_linger * hz);
 				if (error)
@@ -614,8 +614,8 @@ sbsync(struct sockbuf *sb, struct mbuf *nextrecord)
  * must begin with an address if the protocol so specifies,
  * followed by an optional mbuf or mbufs containing ancillary data,
  * and then zero or more mbufs of data.
- * In order to avoid blocking network for the entire time here, we splx()
- * and release NET_LOCK() while doing the actual copy to user space.
+ * In order to avoid blocking network for the entire time here, we release
+ * the NET_LOCK() while doing the actual copy to user space.
  * Although the sockbuf is locked, new data may still be appended,
  * and thus we must maintain consistency of the sockbuf during that time.
  *
@@ -800,9 +800,13 @@ dontblock:
 			if (controlp) {
 				if (pr->pr_domain->dom_externalize &&
 				    mtod(cm, struct cmsghdr *)->cmsg_type ==
-				    SCM_RIGHTS)
-				   error = (*pr->pr_domain->dom_externalize)(cm,
-				       controllen, flags);
+				    SCM_RIGHTS) {
+					NET_UNLOCK(s);
+					error =
+					    (*pr->pr_domain->dom_externalize)
+					    (cm, controllen, flags);
+					NET_LOCK(s);
+				}
 				*controlp = cm;
 			} else {
 				/*
@@ -1039,7 +1043,7 @@ sorflush(struct socket *so)
 	struct sockbuf asb;
 
 	sb->sb_flags |= SB_NOINTR;
-	(void) sblock(sb, M_WAITOK, NULL);
+	(void) sblock(sb, M_WAITOK, &netlock);
 	socantrcvmore(so);
 	sbunlock(sb);
 	asb = *sb;
@@ -1528,7 +1532,10 @@ sorwakeup(struct socket *so)
 #endif
 	sowakeup(so, &so->so_rcv);
 	if (so->so_upcall) {
+		/* XXXSMP breaks atomicity */
+		rw_exit_write(&netlock);
 		(*(so->so_upcall))(so, so->so_upcallarg, M_DONTWAIT);
+		rw_enter_write(&netlock);
 	}
 }
 
diff --git a/sys/kern/uipc_socket2.c b/sys/kern/uipc_socket2.c
index 1ac5f515bc5..b991120eb7a 100644
--- a/sys/kern/uipc_socket2.c
+++ b/sys/kern/uipc_socket2.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: uipc_socket2.c,v 1.70 2016/12/29 12:12:43 mpi Exp $	*/
+/*	$OpenBSD: uipc_socket2.c,v 1.71 2017/01/25 06:15:50 mpi Exp $	*/
 /*	$NetBSD: uipc_socket2.c,v 1.11 1996/02/04 02:17:55 christos Exp $	*/
 
 /*
@@ -276,7 +276,7 @@ sbwait(struct sockbuf *sb)
 	NET_ASSERT_LOCKED();
 
 	sb->sb_flagsintr |= SB_WAIT;
-	return (tsleep(&sb->sb_cc,
+	return (rwsleep(&sb->sb_cc, &netlock,
 	    (sb->sb_flags & SB_NOINTR) ? PSOCK : PSOCK | PCATCH, "netio",
 	    sb->sb_timeo));
 }
diff --git a/sys/kern/uipc_syscalls.c b/sys/kern/uipc_syscalls.c
index 3efaa23ac0e..482abd5e845 100644
--- a/sys/kern/uipc_syscalls.c
+++ b/sys/kern/uipc_syscalls.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: uipc_syscalls.c,v 1.145 2017/01/24 05:44:09 mpi Exp $	*/
+/*	$OpenBSD: uipc_syscalls.c,v 1.146 2017/01/25 06:15:50 mpi Exp $	*/
 /*	$NetBSD: uipc_syscalls.c,v 1.19 1996/02/09 19:00:48 christos Exp $	*/
 
 /*
@@ -307,7 +307,7 @@ redo:
 			head->so_error = ECONNABORTED;
 			break;
 		}
-		error = tsleep(&head->so_timeo, PSOCK | PCATCH,
+		error = rwsleep(&head->so_timeo, &netlock, PSOCK | PCATCH,
 		    "netcon", 0);
 		if (error)
 			goto out;
@@ -425,7 +425,7 @@ sys_connect(struct proc *p, void *v, register_t *retval)
 	}
 	NET_LOCK(s);
 	while ((so->so_state & SS_ISCONNECTING) && so->so_error == 0) {
-		error = tsleep(&so->so_timeo, PSOCK | PCATCH,
+		error = rwsleep(&so->so_timeo, &netlock, PSOCK | PCATCH,
 		    "netcon2", 0);
 		if (error) {
 			if (error == EINTR || error == ERESTART)
diff --git a/sys/kern/uipc_usrreq.c b/sys/kern/uipc_usrreq.c
index 4589f903a24..40c9b99bdb2 100644
--- a/sys/kern/uipc_usrreq.c
+++ b/sys/kern/uipc_usrreq.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: uipc_usrreq.c,v 1.111 2017/01/24 04:09:59 deraadt Exp $	*/
+/*	$OpenBSD: uipc_usrreq.c,v 1.112 2017/01/25 06:15:50 mpi Exp $	*/
 /*	$NetBSD: uipc_usrreq.c,v 1.18 1996/02/09 19:00:50 christos Exp $	*/
 
 /*
@@ -141,7 +141,10 @@ uipc_usrreq(struct socket *so, int req, struct mbuf *m, struct mbuf *nam,
 		break;
 
 	case PRU_BIND:
+		/* XXXSMP breaks atomicity */
+		rw_exit_write(&netlock);
 		error = unp_bind(unp, nam, p);
+		rw_enter_write(&netlock);
 		break;
 
 	case PRU_LISTEN:
@@ -150,7 +153,10 @@ uipc_usrreq(struct socket *so, int req, struct mbuf *m, struct mbuf *nam,
 		break;
 
 	case PRU_CONNECT:
+		/* XXXSMP breaks atomicity */
+		rw_exit_write(&netlock);
 		error = unp_connect(so, nam, p);
+		rw_enter_write(&netlock);
 		break;
 
 	case PRU_CONNECT2:
@@ -218,7 +224,10 @@ uipc_usrreq(struct socket *so, int req, struct mbuf *m, struct mbuf *nam,
 					error = EISCONN;
 					break;
 				}
+				/* XXXSMP breaks atomicity */
+				rw_exit_write(&netlock);
 				error = unp_connect(so, nam, p);
+				rw_enter_write(&netlock);
 				if (error)
 					break;
 			} else {
@@ -495,6 +504,8 @@ unp_connect(struct socket *so, struct mbuf *nam, struct proc *p)
 	struct nameidata nd;
 	int error, s;
 
+	NET_ASSERT_UNLOCKED();
+
 	if (soun->sun_family != AF_UNIX)
 		return (EAFNOSUPPORT);
 
diff --git a/sys/net/if.c b/sys/net/if.c
index b94659bf6f6..8d6721a7d6d 100644
--- a/sys/net/if.c
+++ b/sys/net/if.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: if.c,v 1.480 2017/01/24 10:08:30 krw Exp $	*/
+/*	$OpenBSD: if.c,v 1.481 2017/01/25 06:15:50 mpi Exp $	*/
 /*	$NetBSD: if.c,v 1.35 1996/05/07 05:26:04 thorpej Exp $	*/
 
 /*
@@ -230,6 +230,12 @@ struct taskq	*softnettq;
 struct task if_input_task_locked = TASK_INITIALIZER(if_netisr, NULL);
 
 /*
+ * Serialize socket operations to ensure no new sleeping points
+ * are introduced in IP output paths.
+ */
+struct rwlock netlock = RWLOCK_INITIALIZER("netlock");
+
+/*
  * Network interface utility routines.
  */
 void
@@ -1146,7 +1152,10 @@ if_clone_create(const char *name, int rdomain)
 	if (ifunit(name) != NULL)
 		return (EEXIST);
 
+	/* XXXSMP breaks atomicity */
+	rw_exit_write(&netlock);
 	ret = (*ifc->ifc_create)(ifc, unit);
+	rw_enter_write(&netlock);
 
 	if (ret != 0 || (ifp = ifunit(name)) == NULL)
 		return (ret);
@@ -1188,7 +1197,10 @@ if_clone_destroy(const char *name)
 		splx(s);
 	}
 
+	/* XXXSMP breaks atomicity */
+	rw_exit_write(&netlock);
 	ret = (*ifc->ifc_destroy)(ifp);
+	rw_enter_write(&netlock);
 
 	return (ret);
 }
diff --git a/sys/net/if_pflow.c b/sys/net/if_pflow.c
index 4be1dab0f9b..e6aec01e913 100644
--- a/sys/net/if_pflow.c
+++ b/sys/net/if_pflow.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: if_pflow.c,v 1.71 2017/01/24 10:08:30 krw Exp $	*/
+/*	$OpenBSD: if_pflow.c,v 1.72 2017/01/25 06:15:50 mpi Exp $	*/
 
 /*
  * Copyright (c) 2011 Florian Obser <florian@narrans.de>
@@ -463,9 +463,12 @@ pflowioctl(struct ifnet *ifp, u_long cmd, caddr_t data)
 			sc->sc_gcounter=pflowstats.pflow_flows;
 			/* send templates on startup */
 			if (sc->sc_version == PFLOW_PROTO_10) {
+				/* XXXSMP breaks atomicity */
+				rw_exit_write(&netlock);
 				s = splnet();
 				pflow_sendout_ipfix_tmpl(sc);
 				splx(s);
+				rw_enter_write(&netlock);
 			}
 		} else
 			ifp->if_flags &= ~IFF_RUNNING;
@@ -505,6 +508,8 @@ pflowioctl(struct ifnet *ifp, u_long cmd, caddr_t data)
 		    sizeof(pflowr))))
 			return (error);
 
+		/* XXXSMP breaks atomicity */
+		rw_exit_write(&netlock);
 		s = splnet();
 		error = pflow_set(sc, &pflowr);
 		splx(s);
@@ -522,6 +527,7 @@ pflowioctl(struct ifnet *ifp, u_long cmd, caddr_t data)
 		} else
 			ifp->if_flags &= ~IFF_RUNNING;
 
+		rw_enter_write(&netlock);
 		break;
 
 	default:
diff --git a/sys/net/pf.c b/sys/net/pf.c
index 98578708bab..e54e4493bcc 100644
--- a/sys/net/pf.c
+++ b/sys/net/pf.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: pf.c,v 1.1010 2017/01/09 14:47:13 mpi Exp $ */
+/*	$OpenBSD: pf.c,v 1.1011 2017/01/25 06:15:50 mpi Exp $ */
 
 /*
  * Copyright (c) 2001 Daniel Hartmeier
@@ -1305,8 +1305,12 @@ pf_remove_state(struct pf_state *cur)
 	}
 	RB_REMOVE(pf_state_tree_id, &tree_id, cur);
 #if NPFLOW > 0
-	if (cur->state_flags & PFSTATE_PFLOW)
+	if (cur->state_flags & PFSTATE_PFLOW) {
+		/* XXXSMP breaks atomicity */
+		rw_exit_write(&netlock);
 		export_pflow(cur);
+		rw_enter_write(&netlock);
+	}
 #endif	/* NPFLOW > 0 */
 #if NPFSYNC > 0
 	pfsync_delete_state(cur);
diff --git a/sys/sys/filedesc.h b/sys/sys/filedesc.h
index 4e8767df955..3c9b6b88aaf 100644
--- a/sys/sys/filedesc.h
+++ b/sys/sys/filedesc.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: filedesc.h,v 1.32 2017/01/24 04:09:59 deraadt Exp $	*/
+/*	$OpenBSD: filedesc.h,v 1.33 2017/01/25 06:15:50 mpi Exp $	*/
 /*	$NetBSD: filedesc.h,v 1.14 1996/04/09 20:55:28 cgd Exp $	*/
 
 /*
@@ -139,7 +139,7 @@ struct file *fd_getfile_mode(struct filedesc *, int, int);
 int	closef(struct file *, struct proc *);
 int	getsock(struct proc *, int, struct file **);
 
-#define	fdplock(fdp)	rw_enter_write(&(fdp)->fd_lock)
+#define	fdplock(fdp)	do { NET_ASSERT_UNLOCKED(); rw_enter_write(&(fdp)->fd_lock); } while (0)
 #define	fdpunlock(fdp)	rw_exit_write(&(fdp)->fd_lock)
 #define	fdpassertlocked(fdp)	rw_assert_wrlock(&(fdp)->fd_lock)
 #endif
diff --git a/sys/sys/systm.h b/sys/sys/systm.h
index 99855110bc6..7604f47b178 100644
--- a/sys/sys/systm.h
+++ b/sys/sys/systm.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: systm.h,v 1.122 2017/01/24 08:09:05 kettenis Exp $	*/
+/*	$OpenBSD: systm.h,v 1.123 2017/01/25 06:15:50 mpi Exp $	*/
 /*	$NetBSD: systm.h,v 1.50 1996/06/09 04:55:09 briggs Exp $	*/
 
 /*-
@@ -291,21 +291,35 @@ int	uiomove(void *, size_t, struct uio *);
 
 #if defined(_KERNEL)
 
+#include <sys/rwlock.h>
+
+extern struct rwlock netlock;
+
 #define	NET_LOCK(s)							\
 do {									\
+	rw_enter_write(&netlock);					\
 	s = splsoftnet();						\
 } while (0)
 
-#define	NET_UNLOCK(s)							\
+#define	NET_UNLOCK(s)						\
 do {									\
 	splx(s);							\
+	rw_exit_write(&netlock);					\
 } while (0)
 
 #define	NET_ASSERT_LOCKED()						\
 do {									\
+	if (rw_status(&netlock) != RW_WRITE)				\
+		splassert_fail(RW_WRITE, rw_status(&netlock), __func__);\
 	splsoftassert(IPL_SOFTNET);					\
 } while (0)
 
+#define	NET_ASSERT_UNLOCKED()						\
+do {									\
+	if (rw_status(&netlock) == RW_WRITE)				\
+		splassert_fail(0, rw_status(&netlock), __func__);	\
+} while (0)
+
 __returns_twice int	setjmp(label_t *);
 __dead void	longjmp(label_t *);
 #endif
diff --git a/sys/uvm/uvm_vnode.c b/sys/uvm/uvm_vnode.c
index 2abfac2f14d..e6725d6fc32 100644
--- a/sys/uvm/uvm_vnode.c
+++ b/sys/uvm/uvm_vnode.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: uvm_vnode.c,v 1.93 2016/09/16 02:35:42 dlg Exp $	*/
+/*	$OpenBSD: uvm_vnode.c,v 1.94 2017/01/25 06:15:51 mpi Exp $	*/
 /*	$NetBSD: uvm_vnode.c,v 1.36 2000/11/24 20:34:01 chs Exp $	*/
 
 /*
@@ -1176,6 +1176,15 @@ uvn_io(struct uvm_vnode *uvn, vm_page_t *pps, int npages, int flags, int rw)
 		result = vn_lock(vn, LK_EXCLUSIVE | LK_RECURSEFAIL, curproc);
 
 	if (result == 0) {
+		int netlocked = (rw_status(&netlock) == RW_WRITE);
+
+		/*
+		 * This process may already have the NET_LOCK(), if we
+		 * faulted in copyin() or copyout() in the network stack.
+		 */
+		if (netlocked)
+			rw_exit_write(&netlock);
+
 		/* NOTE: vnode now locked! */
 		if (rw == UIO_READ)
 			result = VOP_READ(vn, &uio, 0, curproc->p_ucred);
@@ -1184,6 +1193,9 @@ uvn_io(struct uvm_vnode *uvn, vm_page_t *pps, int npages, int flags, int rw)
 			    (flags & PGO_PDFREECLUST) ? IO_NOCACHE : 0,
 			    curproc->p_ucred);
 
+		if (netlocked)
+			rw_enter_write(&netlock);
+
 		if ((uvn->u_flags & UVM_VNODE_VNISLOCKED) == 0)
 			VOP_UNLOCK(vn, curproc);
 	}