This time, socket's buffer lock requires solock() to be held. As a part of

socket buffers standalone locking work, move socket state bits which
represent its buffers state to per buffer state. Introduce `sb_state' and
turn SS_CANTSENDMORE to SBS_CANTSENDMORE. This bit will be processed on
`so_snd' buffer only.

Move SS_CANTRCVMORE and SS_RCVATMARK bits with separate diff to make
review easier and exclude possible so_rcv/so_snd mistypes.

Also, don't adjust the remaining SS_* bits right now.

ok millert@

7 files changed, 32 insertions, 26 deletions

diff --git a/sys/kern/sys_socket.c b/sys/kern/sys_socket.c
index a74125459be..e27577f9532 100644
--- a/sys/kern/sys_socket.c
+++ b/sys/kern/sys_socket.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: sys_socket.c,v 1.56 2022/11/19 14:26:39 kn Exp $	*/
+/*	$OpenBSD: sys_socket.c,v 1.57 2022/12/11 21:19:08 mvs Exp $	*/
 /*	$NetBSD: sys_socket.c,v 1.13 1995/08/12 23:59:09 mycroft Exp $	*/
 
 /*
@@ -151,7 +151,7 @@ soo_stat(struct file *fp, struct stat *ub, struct proc *p)
 	solock(so);
 	if ((so->so_state & SS_CANTRCVMORE) == 0 || so->so_rcv.sb_cc != 0)
 		ub->st_mode |= S_IRUSR | S_IRGRP | S_IROTH;
-	if ((so->so_state & SS_CANTSENDMORE) == 0)
+	if ((so->so_snd.sb_state & SBS_CANTSENDMORE) == 0)
 		ub->st_mode |= S_IWUSR | S_IWGRP | S_IWOTH;
 	ub->st_uid = so->so_euid;
 	ub->st_gid = so->so_egid;
diff --git a/sys/kern/uipc_socket.c b/sys/kern/uipc_socket.c
index 92612453418..e3f52c0bb4c 100644
--- a/sys/kern/uipc_socket.c
+++ b/sys/kern/uipc_socket.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: uipc_socket.c,v 1.291 2022/11/28 21:39:28 mvs Exp $	*/
+/*	$OpenBSD: uipc_socket.c,v 1.292 2022/12/11 21:19:08 mvs Exp $	*/
 /*	$NetBSD: uipc_socket.c,v 1.21 1996/02/04 02:17:52 christos Exp $	*/
 
 /*
@@ -580,7 +580,7 @@ restart:
 		goto out;
 	so->so_state |= SS_ISSENDING;
 	do {
-		if (so->so_state & SS_CANTSENDMORE)
+		if (so->so_snd.sb_state & SBS_CANTSENDMORE)
 			snderr(EPIPE);
 		if (so->so_error) {
 			error = so->so_error;
@@ -1465,7 +1465,7 @@ somove(struct socket *so, int wait)
 		error = so->so_error;
 		goto release;
 	}
-	if (sosp->so_state & SS_CANTSENDMORE) {
+	if (sosp->so_snd.sb_state & SBS_CANTSENDMORE) {
 		error = EPIPE;
 		goto release;
 	}
@@ -1659,7 +1659,8 @@ somove(struct socket *so, int wait)
 			if (o) {
 				error = pru_send(sosp, m, NULL, NULL);
 				if (error) {
-					if (sosp->so_state & SS_CANTSENDMORE)
+					if (sosp->so_snd.sb_state &
+					    SBS_CANTSENDMORE)
 						error = EPIPE;
 					m_freem(o);
 					goto release;
@@ -1676,7 +1677,7 @@ somove(struct socket *so, int wait)
 			*mtod(o, caddr_t) = *mtod(m, caddr_t);
 			error = pru_sendoob(sosp, o, NULL, NULL);
 			if (error) {
-				if (sosp->so_state & SS_CANTSENDMORE)
+				if (sosp->so_snd.sb_state & SBS_CANTSENDMORE)
 					error = EPIPE;
 				m_freem(m);
 				goto release;
@@ -1697,7 +1698,7 @@ somove(struct socket *so, int wait)
 		sosp->so_state &= ~SS_ISSENDING;
 	error = pru_send(sosp, m, NULL, NULL);
 	if (error) {
-		if (sosp->so_state & SS_CANTSENDMORE)
+		if (sosp->so_snd.sb_state & SBS_CANTSENDMORE)
 			error = EPIPE;
 		goto release;
 	}
@@ -1714,7 +1715,8 @@ somove(struct socket *so, int wait)
 	if (error)
 		so->so_error = error;
 	if (((so->so_state & SS_CANTRCVMORE) && so->so_rcv.sb_cc == 0) ||
-	    (sosp->so_state & SS_CANTSENDMORE) || maxreached || error) {
+	    (sosp->so_snd.sb_state & SBS_CANTSENDMORE) ||
+	    maxreached || error) {
 		sounsplice(so, sosp, 0);
 		return (0);
 	}
@@ -1839,7 +1841,7 @@ sosetopt(struct socket *so, int level, int optname, struct mbuf *m)
 			switch (optname) {
 
 			case SO_SNDBUF:
-				if (so->so_state & SS_CANTSENDMORE)
+				if (so->so_snd.sb_state & SBS_CANTSENDMORE)
 					return (EINVAL);
 				if (sbcheckreserve(cnt, so->so_snd.sb_wat) ||
 				    sbreserve(so, &so->so_snd, cnt))
@@ -2185,7 +2187,7 @@ filt_sowrite(struct knote *kn, long hint)
 	soassertlocked(so);
 
 	kn->kn_data = sbspace(so, &so->so_snd);
-	if (so->so_state & SS_CANTSENDMORE) {
+	if (so->so_snd.sb_state & SBS_CANTSENDMORE) {
 		kn->kn_flags |= EV_EOF;
 		if (kn->kn_flags & __EV_POLL) {
 			if (so->so_state & SS_ISDISCONNECTED)
diff --git a/sys/kern/uipc_socket2.c b/sys/kern/uipc_socket2.c
index d8b39e44c69..1ffce4625a4 100644
--- a/sys/kern/uipc_socket2.c
+++ b/sys/kern/uipc_socket2.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: uipc_socket2.c,v 1.129 2022/10/03 16:43:52 bluhm Exp $	*/
+/*	$OpenBSD: uipc_socket2.c,v 1.130 2022/12/11 21:19:08 mvs Exp $	*/
 /*	$NetBSD: uipc_socket2.c,v 1.11 1996/02/04 02:17:55 christos Exp $	*/
 
 /*
@@ -142,7 +142,8 @@ soisdisconnecting(struct socket *so)
 {
 	soassertlocked(so);
 	so->so_state &= ~SS_ISCONNECTING;
-	so->so_state |= (SS_ISDISCONNECTING|SS_CANTRCVMORE|SS_CANTSENDMORE);
+	so->so_state |= (SS_ISDISCONNECTING|SS_CANTRCVMORE);
+	so->so_snd.sb_state |= SBS_CANTSENDMORE;
 	wakeup(&so->so_timeo);
 	sowwakeup(so);
 	sorwakeup(so);
@@ -153,7 +154,8 @@ soisdisconnected(struct socket *so)
 {
 	soassertlocked(so);
 	so->so_state &= ~(SS_ISCONNECTING|SS_ISCONNECTED|SS_ISDISCONNECTING);
-	so->so_state |= (SS_CANTRCVMORE|SS_CANTSENDMORE|SS_ISDISCONNECTED);
+	so->so_state |= (SS_CANTRCVMORE|SS_ISDISCONNECTED);
+	so->so_snd.sb_state |= SBS_CANTSENDMORE;
 	wakeup(&so->so_timeo);
 	sowwakeup(so);
 	sorwakeup(so);
@@ -334,7 +336,7 @@ void
 socantsendmore(struct socket *so)
 {
 	soassertlocked(so);
-	so->so_state |= SS_CANTSENDMORE;
+	so->so_snd.sb_state |= SBS_CANTSENDMORE;
 	sowwakeup(so);
 }
 
diff --git a/sys/kern/uipc_usrreq.c b/sys/kern/uipc_usrreq.c
index 45267dd2071..d83eb7c25ce 100644
--- a/sys/kern/uipc_usrreq.c
+++ b/sys/kern/uipc_usrreq.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: uipc_usrreq.c,v 1.195 2022/12/05 23:18:37 deraadt Exp $	*/
+/*	$OpenBSD: uipc_usrreq.c,v 1.196 2022/12/11 21:19:08 mvs Exp $	*/
 /*	$NetBSD: uipc_usrreq.c,v 1.18 1996/02/09 19:00:50 christos Exp $	*/
 
 /*
@@ -509,7 +509,7 @@ uipc_send(struct socket *so, struct mbuf *m, struct mbuf *nam,
 			goto out;
 	}
 
-	if (so->so_state & SS_CANTSENDMORE) {
+	if (so->so_snd.sb_state & SBS_CANTSENDMORE) {
 		error = EPIPE;
 		goto dispose;
 	}
diff --git a/sys/miscfs/fifofs/fifo_vnops.c b/sys/miscfs/fifofs/fifo_vnops.c
index d4f5df4d2c7..72dcf0165fb 100644
--- a/sys/miscfs/fifofs/fifo_vnops.c
+++ b/sys/miscfs/fifofs/fifo_vnops.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: fifo_vnops.c,v 1.96 2022/07/01 09:56:17 mvs Exp $	*/
+/*	$OpenBSD: fifo_vnops.c,v 1.97 2022/12/11 21:19:08 mvs Exp $	*/
 /*	$NetBSD: fifo_vnops.c,v 1.18 1996/03/16 23:52:42 christos Exp $	*/
 
 /*
@@ -174,7 +174,7 @@ fifo_open(void *v)
 		}
 		fip->fi_readers = fip->fi_writers = 0;
 		solock(wso);
-		wso->so_state |= SS_CANTSENDMORE;
+		wso->so_snd.sb_state |= SBS_CANTSENDMORE;
 		wso->so_snd.sb_lowat = PIPE_BUF;
 		sounlock(wso);
 	} else {
@@ -185,7 +185,7 @@ fifo_open(void *v)
 		fip->fi_readers++;
 		if (fip->fi_readers == 1) {
 			solock(wso);
-			wso->so_state &= ~SS_CANTSENDMORE;
+			wso->so_snd.sb_state &= ~SBS_CANTSENDMORE;
 			sounlock(wso);
 			if (fip->fi_writers > 0)
 				wakeup(&fip->fi_writers);
@@ -559,7 +559,7 @@ filt_fifowrite(struct knote *kn, long hint)
 	soassertlocked(so);
 
 	kn->kn_data = sbspace(so, &so->so_snd);
-	if (so->so_state & SS_CANTSENDMORE) {
+	if (so->so_snd.sb_state & SBS_CANTSENDMORE) {
 		kn->kn_flags |= EV_EOF;
 		rv = 1;
 	} else {
diff --git a/sys/netinet/tcp_usrreq.c b/sys/netinet/tcp_usrreq.c
index 2595877d5dd..51d5a148ee1 100644
--- a/sys/netinet/tcp_usrreq.c
+++ b/sys/netinet/tcp_usrreq.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: tcp_usrreq.c,v 1.212 2022/11/09 15:01:24 claudio Exp $	*/
+/*	$OpenBSD: tcp_usrreq.c,v 1.213 2022/12/11 21:19:08 mvs Exp $	*/
 /*	$NetBSD: tcp_usrreq.c,v 1.20 1996/02/13 23:44:16 christos Exp $	*/
 
 /*
@@ -773,7 +773,7 @@ tcp_shutdown(struct socket *so)
 		ostate = tp->t_state;
 	}
 
-	if (so->so_state & SS_CANTSENDMORE)
+	if (so->so_snd.sb_state & SBS_CANTSENDMORE)
 		goto out;
 
 	socantsendmore(so);
diff --git a/sys/sys/socketvar.h b/sys/sys/socketvar.h
index 3e3f8757e7b..7a883027176 100644
--- a/sys/sys/socketvar.h
+++ b/sys/sys/socketvar.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: socketvar.h,v 1.112 2022/11/26 17:52:35 mvs Exp $	*/
+/*	$OpenBSD: socketvar.h,v 1.113 2022/12/11 21:19:08 mvs Exp $	*/
 /*	$NetBSD: socketvar.h,v 1.18 1996/02/09 18:25:38 christos Exp $	*/
 
 /*-
@@ -121,6 +121,7 @@ struct socket {
 		short	sb_flags;	/* flags, see below */
 /* End area that is zeroed on flush. */
 #define	sb_endzero	sb_flags
+		short	sb_state;	/* state, see below */
 		uint64_t sb_timeo_nsecs;/* timeout for read/write */
 		struct	selinfo sb_sel;	/* process selecting read/write */
 	} so_rcv, so_snd;
@@ -132,6 +133,8 @@ struct socket {
 #define	SB_SPLICE	0x20		/* buffer is splice source or drain */
 #define	SB_NOINTR	0x40		/* operations not interruptible */
 
+#define SBS_CANTSENDMORE 0x01		/* can't send more data to peer */
+
 	void	(*so_upcall)(struct socket *so, caddr_t arg, int waitf);
 	caddr_t	so_upcallarg;		/* Arg for above */
 	uid_t	so_euid, so_ruid;	/* who opened the socket */
@@ -146,7 +149,6 @@ struct socket {
 #define	SS_ISCONNECTED		0x002	/* socket connected to a peer */
 #define	SS_ISCONNECTING		0x004	/* in process of connecting to peer */
 #define	SS_ISDISCONNECTING	0x008	/* in process of disconnecting */
-#define	SS_CANTSENDMORE		0x010	/* can't send more data to peer */
 #define	SS_CANTRCVMORE		0x020	/* can't receive more data from peer */
 #define	SS_RCVATMARK		0x040	/* at mark on input */
 #define	SS_ISDISCONNECTED	0x800	/* socket disconnected from peer */
@@ -237,7 +239,7 @@ sowriteable(struct socket *so)
 	return ((sbspace(so, &so->so_snd) >= so->so_snd.sb_lowat &&
 	    ((so->so_state & SS_ISCONNECTED) ||
 	    (so->so_proto->pr_flags & PR_CONNREQUIRED)==0)) ||
-	    (so->so_state & SS_CANTSENDMORE) || so->so_error);
+	    (so->so_snd.sb_state & SBS_CANTSENDMORE) || so->so_error);
 }
 
 /* adjust counters in sb reflecting allocation of m */