Introduce per-sockbuf `sb_state' to use it with SS_CANTSENDMORE.

This time, socket's buffer lock requires solock() to be held. As a part of
socket buffers standalone locking work, move socket state bits which
represent its buffers state to per buffer state.

Opposing the previous reverted diff, the SS_CANTSENDMORE definition left
as is, but it used only with `sb_state'. `sb_state' ored with original
`so_state' when socket's data exported to the userland, so the ABI kept as
it was.

Inputs from deraadt@.

ok bluhm@

8 files changed, 38 insertions, 27 deletions

diff --git a/sys/kern/kern_sysctl.c b/sys/kern/kern_sysctl.c
index 7bd657691c8..94ae40f2523 100644
--- a/sys/kern/kern_sysctl.c
+++ b/sys/kern/kern_sysctl.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: kern_sysctl.c,v 1.409 2023/01/14 01:04:55 cheloha Exp $	*/
+/*	$OpenBSD: kern_sysctl.c,v 1.410 2023/01/21 11:23:23 mvs Exp $	*/
 /*	$NetBSD: kern_sysctl.c,v 1.17 1996/05/20 17:49:05 mrg Exp $	*/
 
 /*-
@@ -1184,7 +1184,7 @@ fill_file(struct kinfo_file *kf, struct file *fp, struct filedesc *fdp,
 		}
 
 		kf->so_type = so->so_type;
-		kf->so_state = so->so_state;
+		kf->so_state = so->so_state | so->so_snd.sb_state;
 		if (show_pointers)
 			kf->so_pcb = PTRTOINT64(so->so_pcb);
 		else
diff --git a/sys/kern/sys_socket.c b/sys/kern/sys_socket.c
index 70c3b4d932a..ddf1befbb08 100644
--- a/sys/kern/sys_socket.c
+++ b/sys/kern/sys_socket.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: sys_socket.c,v 1.58 2022/12/12 08:30:22 tb Exp $	*/
+/*	$OpenBSD: sys_socket.c,v 1.59 2023/01/21 11:23:23 mvs Exp $	*/
 /*	$NetBSD: sys_socket.c,v 1.13 1995/08/12 23:59:09 mycroft Exp $	*/
 
 /*
@@ -151,7 +151,7 @@ soo_stat(struct file *fp, struct stat *ub, struct proc *p)
 	solock(so);
 	if ((so->so_state & SS_CANTRCVMORE) == 0 || so->so_rcv.sb_cc != 0)
 		ub->st_mode |= S_IRUSR | S_IRGRP | S_IROTH;
-	if ((so->so_state & SS_CANTSENDMORE) == 0)
+	if ((so->so_snd.sb_state & SS_CANTSENDMORE) == 0)
 		ub->st_mode |= S_IWUSR | S_IWGRP | S_IWOTH;
 	ub->st_uid = so->so_euid;
 	ub->st_gid = so->so_egid;
diff --git a/sys/kern/uipc_socket.c b/sys/kern/uipc_socket.c
index 821fddf466d..8b3d00834dc 100644
--- a/sys/kern/uipc_socket.c
+++ b/sys/kern/uipc_socket.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: uipc_socket.c,v 1.293 2022/12/12 08:30:22 tb Exp $	*/
+/*	$OpenBSD: uipc_socket.c,v 1.294 2023/01/21 11:23:23 mvs Exp $	*/
 /*	$NetBSD: uipc_socket.c,v 1.21 1996/02/04 02:17:52 christos Exp $	*/
 
 /*
@@ -580,7 +580,7 @@ restart:
 		goto out;
 	so->so_state |= SS_ISSENDING;
 	do {
-		if (so->so_state & SS_CANTSENDMORE)
+		if (so->so_snd.sb_state & SS_CANTSENDMORE)
 			snderr(EPIPE);
 		if (so->so_error) {
 			error = so->so_error;
@@ -1465,7 +1465,7 @@ somove(struct socket *so, int wait)
 		error = so->so_error;
 		goto release;
 	}
-	if (sosp->so_state & SS_CANTSENDMORE) {
+	if (sosp->so_snd.sb_state & SS_CANTSENDMORE) {
 		error = EPIPE;
 		goto release;
 	}
@@ -1659,7 +1659,8 @@ somove(struct socket *so, int wait)
 			if (o) {
 				error = pru_send(sosp, m, NULL, NULL);
 				if (error) {
-					if (sosp->so_state & SS_CANTSENDMORE)
+					if (sosp->so_snd.sb_state &
+					    SS_CANTSENDMORE)
 						error = EPIPE;
 					m_freem(o);
 					goto release;
@@ -1676,7 +1677,7 @@ somove(struct socket *so, int wait)
 			*mtod(o, caddr_t) = *mtod(m, caddr_t);
 			error = pru_sendoob(sosp, o, NULL, NULL);
 			if (error) {
-				if (sosp->so_state & SS_CANTSENDMORE)
+				if (sosp->so_snd.sb_state & SS_CANTSENDMORE)
 					error = EPIPE;
 				m_freem(m);
 				goto release;
@@ -1697,7 +1698,7 @@ somove(struct socket *so, int wait)
 		sosp->so_state &= ~SS_ISSENDING;
 	error = pru_send(sosp, m, NULL, NULL);
 	if (error) {
-		if (sosp->so_state & SS_CANTSENDMORE)
+		if (sosp->so_snd.sb_state & SS_CANTSENDMORE)
 			error = EPIPE;
 		goto release;
 	}
@@ -1714,7 +1715,8 @@ somove(struct socket *so, int wait)
 	if (error)
 		so->so_error = error;
 	if (((so->so_state & SS_CANTRCVMORE) && so->so_rcv.sb_cc == 0) ||
-	    (sosp->so_state & SS_CANTSENDMORE) || maxreached || error) {
+	    (sosp->so_snd.sb_state & SS_CANTSENDMORE) ||
+	    maxreached || error) {
 		sounsplice(so, sosp, 0);
 		return (0);
 	}
@@ -1839,7 +1841,7 @@ sosetopt(struct socket *so, int level, int optname, struct mbuf *m)
 			switch (optname) {
 
 			case SO_SNDBUF:
-				if (so->so_state & SS_CANTSENDMORE)
+				if (so->so_snd.sb_state & SS_CANTSENDMORE)
 					return (EINVAL);
 				if (sbcheckreserve(cnt, so->so_snd.sb_wat) ||
 				    sbreserve(so, &so->so_snd, cnt))
@@ -2185,7 +2187,7 @@ filt_sowrite(struct knote *kn, long hint)
 	soassertlocked(so);
 
 	kn->kn_data = sbspace(so, &so->so_snd);
-	if (so->so_state & SS_CANTSENDMORE) {
+	if (so->so_snd.sb_state & SS_CANTSENDMORE) {
 		kn->kn_flags |= EV_EOF;
 		if (kn->kn_flags & __EV_POLL) {
 			if (so->so_state & SS_ISDISCONNECTED)
diff --git a/sys/kern/uipc_socket2.c b/sys/kern/uipc_socket2.c
index 1d2bf57df07..3fb722bc2b2 100644
--- a/sys/kern/uipc_socket2.c
+++ b/sys/kern/uipc_socket2.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: uipc_socket2.c,v 1.131 2022/12/12 08:30:22 tb Exp $	*/
+/*	$OpenBSD: uipc_socket2.c,v 1.132 2023/01/21 11:23:23 mvs Exp $	*/
 /*	$NetBSD: uipc_socket2.c,v 1.11 1996/02/04 02:17:55 christos Exp $	*/
 
 /*
@@ -142,7 +142,8 @@ soisdisconnecting(struct socket *so)
 {
 	soassertlocked(so);
 	so->so_state &= ~SS_ISCONNECTING;
-	so->so_state |= (SS_ISDISCONNECTING|SS_CANTRCVMORE|SS_CANTSENDMORE);
+	so->so_state |= (SS_ISDISCONNECTING|SS_CANTRCVMORE);
+	so->so_snd.sb_state |= SS_CANTSENDMORE;
 	wakeup(&so->so_timeo);
 	sowwakeup(so);
 	sorwakeup(so);
@@ -153,7 +154,8 @@ soisdisconnected(struct socket *so)
 {
 	soassertlocked(so);
 	so->so_state &= ~(SS_ISCONNECTING|SS_ISCONNECTED|SS_ISDISCONNECTING);
-	so->so_state |= (SS_CANTRCVMORE|SS_CANTSENDMORE|SS_ISDISCONNECTED);
+	so->so_state |= (SS_CANTRCVMORE|SS_ISDISCONNECTED);
+	so->so_snd.sb_state |= SS_CANTSENDMORE;
 	wakeup(&so->so_timeo);
 	sowwakeup(so);
 	sorwakeup(so);
@@ -334,7 +336,7 @@ void
 socantsendmore(struct socket *so)
 {
 	soassertlocked(so);
-	so->so_state |= SS_CANTSENDMORE;
+	so->so_snd.sb_state |= SS_CANTSENDMORE;
 	sowwakeup(so);
 }
 
diff --git a/sys/kern/uipc_usrreq.c b/sys/kern/uipc_usrreq.c
index e5922a91524..b7d9d633d64 100644
--- a/sys/kern/uipc_usrreq.c
+++ b/sys/kern/uipc_usrreq.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: uipc_usrreq.c,v 1.197 2022/12/12 08:30:22 tb Exp $	*/
+/*	$OpenBSD: uipc_usrreq.c,v 1.198 2023/01/21 11:23:23 mvs Exp $	*/
 /*	$NetBSD: uipc_usrreq.c,v 1.18 1996/02/09 19:00:50 christos Exp $	*/
 
 /*
@@ -509,7 +509,7 @@ uipc_send(struct socket *so, struct mbuf *m, struct mbuf *nam,
 			goto out;
 	}
 
-	if (so->so_state & SS_CANTSENDMORE) {
+	if (so->so_snd.sb_state & SS_CANTSENDMORE) {
 		error = EPIPE;
 		goto dispose;
 	}
diff --git a/sys/miscfs/fifofs/fifo_vnops.c b/sys/miscfs/fifofs/fifo_vnops.c
index 33039a4cd65..3c60b67fcfe 100644
--- a/sys/miscfs/fifofs/fifo_vnops.c
+++ b/sys/miscfs/fifofs/fifo_vnops.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: fifo_vnops.c,v 1.98 2022/12/12 08:30:22 tb Exp $	*/
+/*	$OpenBSD: fifo_vnops.c,v 1.99 2023/01/21 11:23:23 mvs Exp $	*/
 /*	$NetBSD: fifo_vnops.c,v 1.18 1996/03/16 23:52:42 christos Exp $	*/
 
 /*
@@ -174,7 +174,7 @@ fifo_open(void *v)
 		}
 		fip->fi_readers = fip->fi_writers = 0;
 		solock(wso);
-		wso->so_state |= SS_CANTSENDMORE;
+		wso->so_snd.sb_state |= SS_CANTSENDMORE;
 		wso->so_snd.sb_lowat = PIPE_BUF;
 		sounlock(wso);
 	} else {
@@ -185,7 +185,7 @@ fifo_open(void *v)
 		fip->fi_readers++;
 		if (fip->fi_readers == 1) {
 			solock(wso);
-			wso->so_state &= ~SS_CANTSENDMORE;
+			wso->so_snd.sb_state &= ~SS_CANTSENDMORE;
 			sounlock(wso);
 			if (fip->fi_writers > 0)
 				wakeup(&fip->fi_writers);
@@ -559,7 +559,7 @@ filt_fifowrite(struct knote *kn, long hint)
 	soassertlocked(so);
 
 	kn->kn_data = sbspace(so, &so->so_snd);
-	if (so->so_state & SS_CANTSENDMORE) {
+	if (so->so_snd.sb_state & SS_CANTSENDMORE) {
 		kn->kn_flags |= EV_EOF;
 		rv = 1;
 	} else {
diff --git a/sys/netinet/tcp_usrreq.c b/sys/netinet/tcp_usrreq.c
index 4f06bc5d3b9..9ce6f056dc8 100644
--- a/sys/netinet/tcp_usrreq.c
+++ b/sys/netinet/tcp_usrreq.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: tcp_usrreq.c,v 1.214 2022/12/12 08:30:22 tb Exp $	*/
+/*	$OpenBSD: tcp_usrreq.c,v 1.215 2023/01/21 11:23:24 mvs Exp $	*/
 /*	$NetBSD: tcp_usrreq.c,v 1.20 1996/02/13 23:44:16 christos Exp $	*/
 
 /*
@@ -773,7 +773,7 @@ tcp_shutdown(struct socket *so)
 		ostate = tp->t_state;
 	}
 
-	if (so->so_state & SS_CANTSENDMORE)
+	if (so->so_snd.sb_state & SS_CANTSENDMORE)
 		goto out;
 
 	socantsendmore(so);
diff --git a/sys/sys/socketvar.h b/sys/sys/socketvar.h
index 8696eca9f26..659a2265410 100644
--- a/sys/sys/socketvar.h
+++ b/sys/sys/socketvar.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: socketvar.h,v 1.114 2022/12/12 08:30:22 tb Exp $	*/
+/*	$OpenBSD: socketvar.h,v 1.115 2023/01/21 11:23:24 mvs Exp $	*/
 /*	$NetBSD: socketvar.h,v 1.18 1996/02/09 18:25:38 christos Exp $	*/
 
 /*-
@@ -121,6 +121,7 @@ struct socket {
 		short	sb_flags;	/* flags, see below */
 /* End area that is zeroed on flush. */
 #define	sb_endzero	sb_flags
+		short	sb_state;	/* socket state on sockbuf */
 		uint64_t sb_timeo_nsecs;/* timeout for read/write */
 		struct	selinfo sb_sel;	/* process selecting read/write */
 	} so_rcv, so_snd;
@@ -141,7 +142,13 @@ struct socket {
 
 /*
  * Socket state bits.
+ *
+ * NOTE: The following states should be used with corresponding socket's
+ * buffer `sb_state' only:
+ *
+ *	SS_CANTSENDMORE		with `so_snd' 
  */
+
 #define	SS_NOFDREF		0x001	/* no file table ref any more */
 #define	SS_ISCONNECTED		0x002	/* socket connected to a peer */
 #define	SS_ISCONNECTING		0x004	/* in process of connecting to peer */
@@ -237,7 +244,7 @@ sowriteable(struct socket *so)
 	return ((sbspace(so, &so->so_snd) >= so->so_snd.sb_lowat &&
 	    ((so->so_state & SS_ISCONNECTED) ||
 	    (so->so_proto->pr_flags & PR_CONNREQUIRED)==0)) ||
-	    (so->so_state & SS_CANTSENDMORE) || so->so_error);
+	    (so->so_snd.sb_state & SS_CANTSENDMORE) || so->so_error);
 }
 
 /* adjust counters in sb reflecting allocation of m */