Use shared netlock in soreceive(). The UDP and IP divert layer

provide locking of the PCB.  If that is possible, use shared instead
of exclusive netlock in soreceive().  The PCB mutex provides a per
socket lock against multiple soreceive() running in parallel.
Release and regrab both locks in sosleep_nsec().
OK mvs@

10 files changed, 147 insertions, 19 deletions

diff --git a/sys/kern/uipc_socket.c b/sys/kern/uipc_socket.c
index 52a3fda1358..05e6eb58249 100644
--- a/sys/kern/uipc_socket.c
+++ b/sys/kern/uipc_socket.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: uipc_socket.c,v 1.288 2022/09/04 09:04:27 bluhm Exp $	*/
+/*	$OpenBSD: uipc_socket.c,v 1.289 2022/09/05 14:56:08 bluhm Exp $	*/
 /*	$NetBSD: uipc_socket.c,v 1.21 1996/02/04 02:17:52 christos Exp $	*/
 
 /*
@@ -822,10 +822,10 @@ bad:
 	if (mp)
 		*mp = NULL;
 
-	solock(so);
+	solock_shared(so);
 restart:
 	if ((error = sblock(so, &so->so_rcv, SBLOCKWAIT(flags))) != 0) {
-		sounlock(so);
+		sounlock_shared(so);
 		return (error);
 	}
 
@@ -893,7 +893,7 @@ restart:
 		sbunlock(so, &so->so_rcv);
 		error = sbwait(so, &so->so_rcv);
 		if (error) {
-			sounlock(so);
+			sounlock_shared(so);
 			return (error);
 		}
 		goto restart;
@@ -962,11 +962,11 @@ dontblock:
 			sbsync(&so->so_rcv, nextrecord);
 			if (controlp) {
 				if (pr->pr_domain->dom_externalize) {
-					sounlock(so);
+					sounlock_shared(so);
 					error =
 					    (*pr->pr_domain->dom_externalize)
 					    (cm, controllen, flags);
-					solock(so);
+					solock_shared(so);
 				}
 				*controlp = cm;
 			} else {
@@ -1040,9 +1040,9 @@ dontblock:
 			SBLASTRECORDCHK(&so->so_rcv, "soreceive uiomove");
 			SBLASTMBUFCHK(&so->so_rcv, "soreceive uiomove");
 			resid = uio->uio_resid;
-			sounlock(so);
+			sounlock_shared(so);
 			uio_error = uiomove(mtod(m, caddr_t) + moff, len, uio);
-			solock(so);
+			solock_shared(so);
 			if (uio_error)
 				uio->uio_resid = resid - len;
 		} else
@@ -1126,7 +1126,7 @@ dontblock:
 			error = sbwait(so, &so->so_rcv);
 			if (error) {
 				sbunlock(so, &so->so_rcv);
-				sounlock(so);
+				sounlock_shared(so);
 				return (0);
 			}
 			if ((m = so->so_rcv.sb_mb) != NULL)
@@ -1171,7 +1171,7 @@ dontblock:
 		*flagsp |= flags;
 release:
 	sbunlock(so, &so->so_rcv);
-	sounlock(so);
+	sounlock_shared(so);
 	return (error);
 }
 
diff --git a/sys/kern/uipc_socket2.c b/sys/kern/uipc_socket2.c
index 21e3a965038..3658537a02d 100644
--- a/sys/kern/uipc_socket2.c
+++ b/sys/kern/uipc_socket2.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: uipc_socket2.c,v 1.127 2022/08/13 21:01:46 mvs Exp $	*/
+/*	$OpenBSD: uipc_socket2.c,v 1.128 2022/09/05 14:56:09 bluhm Exp $	*/
 /*	$NetBSD: uipc_socket2.c,v 1.11 1996/02/04 02:17:55 christos Exp $	*/
 
 /*
@@ -360,6 +360,24 @@ solock(struct socket *so)
 	}
 }
 
+void
+solock_shared(struct socket *so)
+{
+	switch (so->so_proto->pr_domain->dom_family) {
+	case PF_INET:
+	case PF_INET6:
+		if (so->so_proto->pr_usrreqs->pru_lock != NULL) {
+			NET_LOCK_SHARED();
+			pru_lock(so);
+		} else
+			NET_LOCK();
+		break;
+	default:
+		rw_enter_write(&so->so_lock);
+		break;
+	}
+}
+
 int
 solock_persocket(struct socket *so)
 {
@@ -403,6 +421,24 @@ sounlock(struct socket *so)
 }
 
 void
+sounlock_shared(struct socket *so)
+{
+	switch (so->so_proto->pr_domain->dom_family) {
+	case PF_INET:
+	case PF_INET6:
+		if (so->so_proto->pr_usrreqs->pru_unlock != NULL) {
+			pru_unlock(so);
+			NET_UNLOCK_SHARED();
+		} else
+			NET_UNLOCK();
+		break;
+	default:
+		rw_exit_write(&so->so_lock);
+		break;
+	}
+}
+
+void
 soassertlocked(struct socket *so)
 {
 	switch (so->so_proto->pr_domain->dom_family) {
@@ -425,7 +461,15 @@ sosleep_nsec(struct socket *so, void *ident, int prio, const char *wmesg,
 	switch (so->so_proto->pr_domain->dom_family) {
 	case PF_INET:
 	case PF_INET6:
+		if (so->so_proto->pr_usrreqs->pru_unlock != NULL &&
+		    rw_status(&netlock) == RW_READ) {
+			pru_unlock(so);
+		}
 		ret = rwsleep_nsec(ident, &netlock, prio, wmesg, nsecs);
+		if (so->so_proto->pr_usrreqs->pru_lock != NULL &&
+		    rw_status(&netlock) == RW_READ) {
+			pru_lock(so);
+		}
 		break;
 	default:
 		ret = rwsleep_nsec(ident, &so->so_lock, prio, wmesg, nsecs);
diff --git a/sys/netinet/ip_divert.c b/sys/netinet/ip_divert.c
index 361d00c4ea4..1cdad66c298 100644
--- a/sys/netinet/ip_divert.c
+++ b/sys/netinet/ip_divert.c
@@ -1,4 +1,4 @@
-/*      $OpenBSD: ip_divert.c,v 1.86 2022/09/03 22:43:38 mvs Exp $ */
+/*      $OpenBSD: ip_divert.c,v 1.87 2022/09/05 14:56:09 bluhm Exp $ */
 
 /*
  * Copyright (c) 2009 Michele Marchetto <michele@openbsd.org>
@@ -65,6 +65,8 @@ const struct sysctl_bounded_args divertctl_vars[] = {
 const struct pr_usrreqs divert_usrreqs = {
 	.pru_attach	= divert_attach,
 	.pru_detach	= divert_detach,
+	.pru_lock	= divert_lock,
+	.pru_unlock	= divert_unlock,
 	.pru_bind	= divert_bind,
 	.pru_shutdown	= divert_shutdown,
 	.pru_send	= divert_send,
@@ -288,6 +290,24 @@ divert_detach(struct socket *so)
 	return (0);
 }
 
+void
+divert_lock(struct socket *so)
+{
+	struct inpcb *inp = sotoinpcb(so);
+
+	NET_ASSERT_LOCKED();
+	mtx_enter(&inp->inp_mtx);
+}
+
+void
+divert_unlock(struct socket *so)
+{
+	struct inpcb *inp = sotoinpcb(so);
+
+	NET_ASSERT_LOCKED();
+	mtx_leave(&inp->inp_mtx);
+}
+
 int
 divert_bind(struct socket *so, struct mbuf *addr, struct proc *p)
 {
diff --git a/sys/netinet/ip_divert.h b/sys/netinet/ip_divert.h
index d5e4dabf631..5055a8f0a89 100644
--- a/sys/netinet/ip_divert.h
+++ b/sys/netinet/ip_divert.h
@@ -1,4 +1,4 @@
-/*      $OpenBSD: ip_divert.h,v 1.21 2022/09/03 22:43:38 mvs Exp $ */
+/*      $OpenBSD: ip_divert.h,v 1.22 2022/09/05 14:56:09 bluhm Exp $ */
 
 /*
  * Copyright (c) 2009 Michele Marchetto <michele@openbsd.org>
@@ -72,6 +72,8 @@ void	 divert_packet(struct mbuf *, int, u_int16_t);
 int	 divert_sysctl(int *, u_int, void *, size_t *, void *, size_t);
 int	 divert_attach(struct socket *, int);
 int	 divert_detach(struct socket *);
+void	 divert_lock(struct socket *);
+void	 divert_unlock(struct socket *);
 int	 divert_bind(struct socket *, struct mbuf *, struct proc *);
 int	 divert_shutdown(struct socket *);
 int	 divert_send(struct socket *, struct mbuf *, struct mbuf *,
diff --git a/sys/netinet/udp_usrreq.c b/sys/netinet/udp_usrreq.c
index 21440ab3a2c..a019b2c236d 100644
--- a/sys/netinet/udp_usrreq.c
+++ b/sys/netinet/udp_usrreq.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: udp_usrreq.c,v 1.301 2022/09/03 22:43:38 mvs Exp $	*/
+/*	$OpenBSD: udp_usrreq.c,v 1.302 2022/09/05 14:56:09 bluhm Exp $	*/
 /*	$NetBSD: udp_usrreq.c,v 1.28 1996/03/16 23:54:03 christos Exp $	*/
 
 /*
@@ -125,6 +125,8 @@ u_int	udp_recvspace = 40 * (1024 + sizeof(struct sockaddr_in));
 const struct pr_usrreqs udp_usrreqs = {
 	.pru_attach	= udp_attach,
 	.pru_detach	= udp_detach,
+	.pru_lock	= udp_lock,
+	.pru_unlock	= udp_unlock,
 	.pru_bind	= udp_bind,
 	.pru_connect	= udp_connect,
 	.pru_disconnect	= udp_disconnect,
@@ -140,6 +142,8 @@ const struct pr_usrreqs udp_usrreqs = {
 const struct pr_usrreqs udp6_usrreqs = {
 	.pru_attach	= udp_attach,
 	.pru_detach	= udp_detach,
+	.pru_lock	= udp_lock,
+	.pru_unlock	= udp_unlock,
 	.pru_bind	= udp_bind,
 	.pru_connect	= udp_connect,
 	.pru_disconnect	= udp_disconnect,
@@ -1112,6 +1116,24 @@ udp_detach(struct socket *so)
 	return (0);
 }
 
+void
+udp_lock(struct socket *so)
+{
+	struct inpcb *inp = sotoinpcb(so);
+
+	NET_ASSERT_LOCKED();
+	mtx_enter(&inp->inp_mtx);
+}
+
+void
+udp_unlock(struct socket *so)
+{
+	struct inpcb *inp = sotoinpcb(so);
+
+	NET_ASSERT_LOCKED();
+	mtx_leave(&inp->inp_mtx);
+}
+
 int
 udp_bind(struct socket *so, struct mbuf *addr, struct proc *p)
 {
diff --git a/sys/netinet/udp_var.h b/sys/netinet/udp_var.h
index 79fdfeed540..c086640cf6d 100644
--- a/sys/netinet/udp_var.h
+++ b/sys/netinet/udp_var.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: udp_var.h,v 1.46 2022/09/03 22:43:38 mvs Exp $	*/
+/*	$OpenBSD: udp_var.h,v 1.47 2022/09/05 14:56:09 bluhm Exp $	*/
 /*	$NetBSD: udp_var.h,v 1.12 1996/02/13 23:44:41 christos Exp $	*/
 
 /*
@@ -145,6 +145,8 @@ int	 udp6_output(struct inpcb *, struct mbuf *, struct mbuf *,
 int	 udp_sysctl(int *, u_int, void *, size_t *, void *, size_t);
 int	 udp_attach(struct socket *, int);
 int	 udp_detach(struct socket *);
+void	 udp_lock(struct socket *);
+void	 udp_unlock(struct socket *);
 int	 udp_bind(struct socket *, struct mbuf *, struct proc *);
 int	 udp_connect(struct socket *, struct mbuf *);
 int	 udp_disconnect(struct socket *);
diff --git a/sys/netinet6/ip6_divert.c b/sys/netinet6/ip6_divert.c
index 706d5b9f3b9..29b3afb4642 100644
--- a/sys/netinet6/ip6_divert.c
+++ b/sys/netinet6/ip6_divert.c
@@ -1,4 +1,4 @@
-/*      $OpenBSD: ip6_divert.c,v 1.85 2022/09/03 22:43:38 mvs Exp $ */
+/*      $OpenBSD: ip6_divert.c,v 1.86 2022/09/05 14:56:09 bluhm Exp $ */
 
 /*
  * Copyright (c) 2009 Michele Marchetto <michele@openbsd.org>
@@ -66,6 +66,8 @@ const struct sysctl_bounded_args divert6ctl_vars[] = {
 const struct pr_usrreqs divert6_usrreqs = {
 	.pru_attach	= divert6_attach,
 	.pru_detach	= divert6_detach,
+	.pru_lock	= divert6_lock,
+	.pru_unlock	= divert6_unlock,
 	.pru_bind	= divert6_bind,
 	.pru_shutdown	= divert6_shutdown,
 	.pru_send	= divert6_send,
@@ -295,6 +297,24 @@ divert6_detach(struct socket *so)
 	return (0);
 }
 
+void
+divert6_lock(struct socket *so)
+{
+	struct inpcb *inp = sotoinpcb(so);
+
+	NET_ASSERT_LOCKED();
+	mtx_enter(&inp->inp_mtx);
+}
+
+void
+divert6_unlock(struct socket *so)
+{
+	struct inpcb *inp = sotoinpcb(so);
+
+	NET_ASSERT_LOCKED();
+	mtx_leave(&inp->inp_mtx);
+}
+
 int
 divert6_bind(struct socket *so, struct mbuf *addr, struct proc *p)
 {
diff --git a/sys/netinet6/ip6_divert.h b/sys/netinet6/ip6_divert.h
index d76c123e15c..df0fbcf7677 100644
--- a/sys/netinet6/ip6_divert.h
+++ b/sys/netinet6/ip6_divert.h
@@ -1,4 +1,4 @@
-/*      $OpenBSD: ip6_divert.h,v 1.19 2022/09/03 22:43:38 mvs Exp $ */
+/*      $OpenBSD: ip6_divert.h,v 1.20 2022/09/05 14:56:09 bluhm Exp $ */
 
 /*
  * Copyright (c) 2009 Michele Marchetto <michele@openbsd.org>
@@ -72,6 +72,8 @@ void	 divert6_packet(struct mbuf *, int, u_int16_t);
 int	 divert6_sysctl(int *, u_int, void *, size_t *, void *, size_t);
 int	 divert6_attach(struct socket *, int);
 int	 divert6_detach(struct socket *);
+void	 divert6_lock(struct socket *);
+void	 divert6_unlock(struct socket *);
 int	 divert6_bind(struct socket *, struct mbuf *, struct proc *);
 int	 divert6_shutdown(struct socket *);
 int	 divert6_send(struct socket *, struct mbuf *, struct mbuf *,
diff --git a/sys/sys/protosw.h b/sys/sys/protosw.h
index eb7f67e7cb1..34eef0b441b 100644
--- a/sys/sys/protosw.h
+++ b/sys/sys/protosw.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: protosw.h,v 1.54 2022/09/03 22:43:39 mvs Exp $	*/
+/*	$OpenBSD: protosw.h,v 1.55 2022/09/05 14:56:09 bluhm Exp $	*/
 /*	$NetBSD: protosw.h,v 1.10 1996/04/09 20:55:32 cgd Exp $	*/
 
 /*-
@@ -64,6 +64,8 @@ struct ifnet;
 struct pr_usrreqs {
 	int	(*pru_attach)(struct socket *, int);
 	int	(*pru_detach)(struct socket *);
+	void	(*pru_lock)(struct socket *);
+	void	(*pru_unlock)(struct socket *);
 	int	(*pru_bind)(struct socket *, struct mbuf *, struct proc *);
 	int	(*pru_listen)(struct socket *);
 	int	(*pru_connect)(struct socket *, struct mbuf *);
@@ -276,6 +278,18 @@ pru_detach(struct socket *so)
 	return (*so->so_proto->pr_usrreqs->pru_detach)(so);
 }
 
+static inline void
+pru_lock(struct socket *so)
+{
+	(*so->so_proto->pr_usrreqs->pru_lock)(so);
+}
+
+static inline void
+pru_unlock(struct socket *so)
+{
+	(*so->so_proto->pr_usrreqs->pru_unlock)(so);
+}
+
 static inline int
 pru_bind(struct socket *so, struct mbuf *nam, struct proc *p)
 {
diff --git a/sys/sys/socketvar.h b/sys/sys/socketvar.h
index d21e9798325..16f586b19e0 100644
--- a/sys/sys/socketvar.h
+++ b/sys/sys/socketvar.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: socketvar.h,v 1.109 2022/09/01 05:31:49 jsg Exp $	*/
+/*	$OpenBSD: socketvar.h,v 1.110 2022/09/05 14:56:09 bluhm Exp $	*/
 /*	$NetBSD: socketvar.h,v 1.18 1996/02/09 18:25:38 christos Exp $	*/
 
 /*-
@@ -348,9 +348,11 @@ int	sockargs(struct mbuf **, const void *, size_t, int);
 
 int	sosleep_nsec(struct socket *, void *, int, const char *, uint64_t);
 void	solock(struct socket *);
+void	solock_shared(struct socket *);
 int	solock_persocket(struct socket *);
 void	solock_pair(struct socket *, struct socket *);
 void	sounlock(struct socket *);
+void	sounlock_shared(struct socket *);
 
 int	sendit(struct proc *, int, struct msghdr *, int, register_t *);
 int	recvit(struct proc *, int, struct msghdr *, caddr_t, register_t *);