Move solock() down to sosetopt(). A part of standalone sblock() work.

This movement required because buffers related SO_SND* and SO_RCV*
socket options should be protected with sblock(). However, standalone
sblock() has different lock order with solock() and `so_snd' and
`so_rcv' buffers. At least sblock() for `so_snd' buffer will always be
taken before solock() in the sosend() path.

The (*pr_ctloutput)() call was removed from the SOL_SOCKET level 'else'
branch. Except the SO_RTABLE case where it handled in the special way,
this is null op call.

For SO_SND* and SO_RCV* cases solock() will be replaced by sblock() in
the future.

Feedback from bluhm

Tested by bluhm naddy

ok bluhm

3 files changed, 17 insertions, 19 deletions

diff --git a/sys/nfs/krpc_subr.c b/sys/nfs/krpc_subr.c
index eb35b5f6c1d..fec8ae712e8 100644
--- a/sys/nfs/krpc_subr.c
+++ b/sys/nfs/krpc_subr.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: krpc_subr.c,v 1.37 2022/06/06 14:45:41 claudio Exp $	*/
+/*	$OpenBSD: krpc_subr.c,v 1.38 2023/08/03 09:49:09 mvs Exp $	*/
 /*	$NetBSD: krpc_subr.c,v 1.12.4.1 1996/06/07 00:52:26 cgd Exp $	*/
 
 /*
@@ -239,9 +239,7 @@ krpc_call(struct sockaddr_in *sa, u_int prog, u_int vers, u_int func,
 	tv.tv_usec = 0;
 	memcpy(mtod(m, struct timeval *), &tv, sizeof tv);
 	m->m_len = sizeof(tv);
-	solock(so);
 	error = sosetopt(so, SOL_SOCKET, SO_RCVTIMEO, m);
-	sounlock(so);
 	m_freem(m);
 	if (error)
 		goto out;
@@ -255,9 +253,7 @@ krpc_call(struct sockaddr_in *sa, u_int prog, u_int vers, u_int func,
 		on = mtod(m, int32_t *);
 		m->m_len = sizeof(*on);
 		*on = 1;
-		solock(so);
 		error = sosetopt(so, SOL_SOCKET, SO_BROADCAST, m);
-		sounlock(so);
 		m_freem(m);
 		if (error)
 			goto out;
@@ -272,9 +268,7 @@ krpc_call(struct sockaddr_in *sa, u_int prog, u_int vers, u_int func,
 	mopt->m_len = sizeof(int);
 	ip = mtod(mopt, int *);
 	*ip = IP_PORTRANGE_LOW;
-	solock(so);
 	error = sosetopt(so, IPPROTO_IP, IP_PORTRANGE, mopt);
-	sounlock(so);
 	m_freem(mopt);
 	if (error)
 		goto out;
@@ -299,9 +293,7 @@ krpc_call(struct sockaddr_in *sa, u_int prog, u_int vers, u_int func,
 	mopt->m_len = sizeof(int);
 	ip = mtod(mopt, int *);
 	*ip = IP_PORTRANGE_DEFAULT;
-	solock(so);
 	error = sosetopt(so, IPPROTO_IP, IP_PORTRANGE, mopt);
-	sounlock(so);
 	m_freem(mopt);
 	if (error)
 		goto out;
diff --git a/sys/nfs/nfs_socket.c b/sys/nfs/nfs_socket.c
index 5488a501040..80dfaa699c9 100644
--- a/sys/nfs/nfs_socket.c
+++ b/sys/nfs/nfs_socket.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: nfs_socket.c,v 1.143 2022/08/13 21:01:46 mvs Exp $	*/
+/*	$OpenBSD: nfs_socket.c,v 1.144 2023/08/03 09:49:09 mvs Exp $	*/
 /*	$NetBSD: nfs_socket.c,v 1.27 1996/04/15 20:20:00 thorpej Exp $	*/
 
 /*
@@ -258,7 +258,6 @@ nfs_connect(struct nfsmount *nmp, struct nfsreq *rep)
 		MGET(nam, M_WAIT, MT_SONAME);
 
 	so = nmp->nm_so;
-	solock(so);
 	nmp->nm_soflags = so->so_proto->pr_flags;
 
 	/*
@@ -282,7 +281,9 @@ nfs_connect(struct nfsmount *nmp, struct nfsreq *rep)
 		sin->sin_family = AF_INET;
 		sin->sin_addr.s_addr = INADDR_ANY;
 		sin->sin_port = htons(0);
+		solock(so);
 		error = sobind(so, nam, &proc0);
+		sounlock(so);
 		if (error)
 			goto bad;
 
@@ -294,6 +295,7 @@ nfs_connect(struct nfsmount *nmp, struct nfsreq *rep)
 			goto bad;
 	}
 
+	solock(so);
 	/*
 	 * Protocols that do not require connections may be optionally left
 	 * unconnected for servers that reply from a port other than NFS_PORT.
@@ -301,12 +303,12 @@ nfs_connect(struct nfsmount *nmp, struct nfsreq *rep)
 	if (nmp->nm_flag & NFSMNT_NOCONN) {
 		if (nmp->nm_soflags & PR_CONNREQUIRED) {
 			error = ENOTCONN;
-			goto bad;
+			goto bad_locked;
 		}
 	} else {
 		error = soconnect(so, nmp->nm_nam);
 		if (error)
-			goto bad;
+			goto bad_locked;
 
 		/*
 		 * Wait for the connection to complete. Cribbed from the
@@ -320,13 +322,13 @@ nfs_connect(struct nfsmount *nmp, struct nfsreq *rep)
 			    so->so_error == 0 && rep &&
 			    (error = nfs_sigintr(nmp, rep, rep->r_procp)) != 0){
 				so->so_state &= ~SS_ISCONNECTING;
-				goto bad;
+				goto bad_locked;
 			}
 		}
 		if (so->so_error) {
 			error = so->so_error;
 			so->so_error = 0;
-			goto bad;
+			goto bad_locked;
 		}
 	}
 	/*
@@ -338,6 +340,7 @@ nfs_connect(struct nfsmount *nmp, struct nfsreq *rep)
 		so->so_snd.sb_timeo_nsecs = SEC_TO_NSEC(5);
 	else
 		so->so_snd.sb_timeo_nsecs = INFSLP;
+	sounlock(so);
 	if (nmp->nm_sotype == SOCK_DGRAM) {
 		sndreserve = nmp->nm_wsize + NFS_MAXPKTHDR;
 		rcvreserve = (max(nmp->nm_rsize, nmp->nm_readdirsize) +
@@ -360,9 +363,10 @@ nfs_connect(struct nfsmount *nmp, struct nfsreq *rep)
 	} else {
 		panic("%s: nm_sotype %d", __func__, nmp->nm_sotype);
 	}
+	solock(so);
 	error = soreserve(so, sndreserve, rcvreserve);
 	if (error)
-		goto bad;
+		goto bad_locked;
 	so->so_rcv.sb_flags |= SB_NOINTR;
 	so->so_snd.sb_flags |= SB_NOINTR;
 	sounlock(so);
@@ -377,8 +381,9 @@ nfs_connect(struct nfsmount *nmp, struct nfsreq *rep)
 	nmp->nm_timeouts = 0;
 	return (0);
 
-bad:
+bad_locked:
 	sounlock(so);
+bad:
 
 	m_freem(mopt);
 	m_freem(nam);
diff --git a/sys/nfs/nfs_syscalls.c b/sys/nfs/nfs_syscalls.c
index b9eecb0430b..45ff763af20 100644
--- a/sys/nfs/nfs_syscalls.c
+++ b/sys/nfs/nfs_syscalls.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: nfs_syscalls.c,v 1.118 2022/06/06 14:45:41 claudio Exp $	*/
+/*	$OpenBSD: nfs_syscalls.c,v 1.119 2023/08/03 09:49:09 mvs Exp $	*/
 /*	$NetBSD: nfs_syscalls.c,v 1.19 1996/02/18 11:53:52 fvdl Exp $	*/
 
 /*
@@ -249,8 +249,8 @@ nfssvc_addsock(struct file *fp, struct mbuf *mynam)
 		siz = NFS_MAXPACKET;
 	solock(so);
 	error = soreserve(so, siz, siz); 
+	sounlock(so);
 	if (error) {
-		sounlock(so);
 		m_freem(mynam);
 		return (error);
 	}
@@ -275,6 +275,7 @@ nfssvc_addsock(struct file *fp, struct mbuf *mynam)
 		sosetopt(so, IPPROTO_TCP, TCP_NODELAY, m);
 		m_freem(m);
 	}
+	solock(so);
 	so->so_rcv.sb_flags &= ~SB_NOINTR;
 	so->so_rcv.sb_timeo_nsecs = INFSLP;
 	so->so_snd.sb_flags &= ~SB_NOINTR;