Rework socket buffers locking for shared netlock.

Shared netlock is not sufficient to call so{r,w}wakeup(). The following
sowakeup() modifies `sb_flags' and knote(9) stuff. Unfortunately, we
can't call so{r,w}wakeup() with `inp_mtx' mutex(9) because sowakeup()
also calls pgsigio() which grabs kernel lock.

However, `so*_filtops' callbacks only perform read-only access to the
socket stuff, so it is enough to hold shared netlock only, but the klist
stuff needs to be protected.

This diff introduces `sb_mtx' mutex(9) to protect sockbuf. This time
`sb_mtx' used to protect only `sb_flags' and `sb_klist'.

Now we have soassertlocked_readonly() and soassertlocked(). The first
one is happy if only shared netlock is held, meanwhile the second wants
`so_lock' or pru_lock() be held together with shared netlock.

To keep soassertlocked*() assertions soft, we need to know mutex(9)
state, so new mtx_owned() macro was introduces. Also, the new optional
(*pru_locked)() handler brings the state of pru_lock().

Tests and ok from bluhm.

6 files changed, 37 insertions, 6 deletions

diff --git a/sys/netinet/ip_divert.c b/sys/netinet/ip_divert.c
index b21569e7205..593bfcfbc11 100644
--- a/sys/netinet/ip_divert.c
+++ b/sys/netinet/ip_divert.c
@@ -1,4 +1,4 @@
-/*      $OpenBSD: ip_divert.c,v 1.92 2023/09/16 09:33:27 mpi Exp $ */
+/*      $OpenBSD: ip_divert.c,v 1.93 2024/02/03 22:50:09 mvs Exp $ */
 
 /*
  * Copyright (c) 2009 Michele Marchetto <michele@openbsd.org>
@@ -67,6 +67,7 @@ const struct pr_usrreqs divert_usrreqs = {
 	.pru_detach	= divert_detach,
 	.pru_lock	= divert_lock,
 	.pru_unlock	= divert_unlock,
+	.pru_locked	= divert_locked,
 	.pru_bind	= divert_bind,
 	.pru_shutdown	= divert_shutdown,
 	.pru_send	= divert_send,
@@ -314,6 +315,14 @@ divert_unlock(struct socket *so)
 }
 
 int
+divert_locked(struct socket *so)
+{
+	struct inpcb *inp = sotoinpcb(so);
+
+	return mtx_owned(&inp->inp_mtx);
+}
+
+int
 divert_bind(struct socket *so, struct mbuf *addr, struct proc *p)
 {
 	struct inpcb *inp = sotoinpcb(so);
diff --git a/sys/netinet/ip_divert.h b/sys/netinet/ip_divert.h
index 14188641599..2c09d517559 100644
--- a/sys/netinet/ip_divert.h
+++ b/sys/netinet/ip_divert.h
@@ -1,4 +1,4 @@
-/*      $OpenBSD: ip_divert.h,v 1.24 2022/10/17 14:49:02 mvs Exp $ */
+/*      $OpenBSD: ip_divert.h,v 1.25 2024/02/03 22:50:09 mvs Exp $ */
 
 /*
  * Copyright (c) 2009 Michele Marchetto <michele@openbsd.org>
@@ -74,6 +74,7 @@ int	 divert_attach(struct socket *, int, int);
 int	 divert_detach(struct socket *);
 void	 divert_lock(struct socket *);
 void	 divert_unlock(struct socket *);
+int	 divert_locked(struct socket *);
 int	 divert_bind(struct socket *, struct mbuf *, struct proc *);
 int	 divert_shutdown(struct socket *);
 int	 divert_send(struct socket *, struct mbuf *, struct mbuf *,
diff --git a/sys/netinet/ip_var.h b/sys/netinet/ip_var.h
index 789796633e3..663671d1a26 100644
--- a/sys/netinet/ip_var.h
+++ b/sys/netinet/ip_var.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: ip_var.h,v 1.110 2023/11/26 22:08:10 bluhm Exp $	*/
+/*	$OpenBSD: ip_var.h,v 1.111 2024/02/03 22:50:09 mvs Exp $	*/
 /*	$NetBSD: ip_var.h,v 1.16 1996/02/13 23:43:20 christos Exp $	*/
 
 /*
@@ -260,6 +260,7 @@ int	 rip_attach(struct socket *, int, int);
 int	 rip_detach(struct socket *);
 void	 rip_lock(struct socket *);
 void	 rip_unlock(struct socket *);
+int	 rip_locked(struct socket *);
 int	 rip_bind(struct socket *, struct mbuf *, struct proc *);
 int	 rip_connect(struct socket *, struct mbuf *);
 int	 rip_disconnect(struct socket *);
diff --git a/sys/netinet/raw_ip.c b/sys/netinet/raw_ip.c
index ae9ddb52185..5f8047c77e7 100644
--- a/sys/netinet/raw_ip.c
+++ b/sys/netinet/raw_ip.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: raw_ip.c,v 1.154 2024/01/21 01:17:20 bluhm Exp $	*/
+/*	$OpenBSD: raw_ip.c,v 1.155 2024/02/03 22:50:09 mvs Exp $	*/
 /*	$NetBSD: raw_ip.c,v 1.25 1996/02/18 18:58:33 christos Exp $	*/
 
 /*
@@ -108,6 +108,7 @@ const struct pr_usrreqs rip_usrreqs = {
 	.pru_detach	= rip_detach,
 	.pru_lock	= rip_lock,
 	.pru_unlock	= rip_unlock,
+	.pru_locked	= rip_locked,
 	.pru_bind	= rip_bind,
 	.pru_connect	= rip_connect,
 	.pru_disconnect	= rip_disconnect,
@@ -525,6 +526,14 @@ rip_unlock(struct socket *so)
 }
 
 int
+rip_locked(struct socket *so)
+{
+	struct inpcb *inp = sotoinpcb(so);
+
+	return mtx_owned(&inp->inp_mtx);
+}
+
+int
 rip_bind(struct socket *so, struct mbuf *nam, struct proc *p)
 {
 	struct inpcb *inp = sotoinpcb(so);
diff --git a/sys/netinet/udp_usrreq.c b/sys/netinet/udp_usrreq.c
index 6a58bb91671..18b0ae983bc 100644
--- a/sys/netinet/udp_usrreq.c
+++ b/sys/netinet/udp_usrreq.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: udp_usrreq.c,v 1.316 2024/01/28 20:34:25 bluhm Exp $	*/
+/*	$OpenBSD: udp_usrreq.c,v 1.317 2024/02/03 22:50:09 mvs Exp $	*/
 /*	$NetBSD: udp_usrreq.c,v 1.28 1996/03/16 23:54:03 christos Exp $	*/
 
 /*
@@ -127,6 +127,7 @@ const struct pr_usrreqs udp_usrreqs = {
 	.pru_detach	= udp_detach,
 	.pru_lock	= udp_lock,
 	.pru_unlock	= udp_unlock,
+	.pru_locked	= udp_locked,
 	.pru_bind	= udp_bind,
 	.pru_connect	= udp_connect,
 	.pru_disconnect	= udp_disconnect,
@@ -143,6 +144,7 @@ const struct pr_usrreqs udp6_usrreqs = {
 	.pru_detach	= udp_detach,
 	.pru_lock	= udp_lock,
 	.pru_unlock	= udp_unlock,
+	.pru_locked	= udp_locked,
 	.pru_bind	= udp_bind,
 	.pru_connect	= udp_connect,
 	.pru_disconnect	= udp_disconnect,
@@ -1157,6 +1159,14 @@ udp_unlock(struct socket *so)
 }
 
 int
+udp_locked(struct socket *so)
+{
+	struct inpcb *inp = sotoinpcb(so);
+
+	return mtx_owned(&inp->inp_mtx);
+}
+
+int
 udp_bind(struct socket *so, struct mbuf *addr, struct proc *p)
 {
 	struct inpcb *inp = sotoinpcb(so);
diff --git a/sys/netinet/udp_var.h b/sys/netinet/udp_var.h
index 80df3318fe4..5731cf4ef1a 100644
--- a/sys/netinet/udp_var.h
+++ b/sys/netinet/udp_var.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: udp_var.h,v 1.50 2024/01/10 16:44:30 bluhm Exp $	*/
+/*	$OpenBSD: udp_var.h,v 1.51 2024/02/03 22:50:09 mvs Exp $	*/
 /*	$NetBSD: udp_var.h,v 1.12 1996/02/13 23:44:41 christos Exp $	*/
 
 /*
@@ -147,6 +147,7 @@ int	 udp_attach(struct socket *, int, int);
 int	 udp_detach(struct socket *);
 void	 udp_lock(struct socket *);
 void	 udp_unlock(struct socket *);
+int	 udp_locked(struct socket *);
 int	 udp_bind(struct socket *, struct mbuf *, struct proc *);
 int	 udp_connect(struct socket *, struct mbuf *);
 int	 udp_disconnect(struct socket *);