Header netinet/in_pcb.h includes sys/mutex.h now. Recommit mutex

for PCB tables.  It does not break userland build anymore.

pf_socket_lookup() calls in_pcbhashlookup() in the PCB layer.  To
run pf in parallel, make parts of the stack MP safe.  Protect the
list and hashes in the PCB tables with a mutex.
Note that the protocol notify functions may call pf via tcp_output().
As the pf lock is a sleeping rw_lock, we must not hold a mutex.  To
solve this for now, collect these PCBs in inp_notify list and protect
it with exclusive netlock.
OK sashan@

1 files changed, 51 insertions, 11 deletions

diff --git a/sys/netinet/in_pcb.c b/sys/netinet/in_pcb.c
index 26908d9b5fb..a2060c16fd6 100644
--- a/sys/netinet/in_pcb.c
+++ b/sys/netinet/in_pcb.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: in_pcb.c,v 1.262 2022/03/21 03:51:09 dlg Exp $	*/
+/*	$OpenBSD: in_pcb.c,v 1.263 2022/03/21 09:12:34 bluhm Exp $	*/
 /*	$NetBSD: in_pcb.c,v 1.25 1996/02/13 23:41:53 christos Exp $	*/
 
 /*
@@ -120,7 +120,8 @@ struct baddynamicports baddynamicports;
 struct baddynamicports rootonlyports;
 struct pool inpcb_pool;
 
-int in_pcbresize (struct inpcbtable *, int);
+void	in_pcbrehash_locked(struct inpcb *);
+int	in_pcbresize(struct inpcbtable *, int);
 
 #define	INPCBHASH_LOADFACTOR(_x)	(((_x) * 3) / 4)
 
@@ -173,7 +174,7 @@ in_pcblhash(struct inpcbtable *table, int rdom, u_short lport)
 void
 in_pcbinit(struct inpcbtable *table, int hashsize)
 {
-
+	mtx_init(&table->inpt_mtx, IPL_SOFTNET);
 	TAILQ_INIT(&table->inpt_queue);
 	table->inpt_hashtbl = hashinit(hashsize, M_PCB, M_WAITOK,
 	    &table->inpt_mask);
@@ -252,6 +253,7 @@ in_pcballoc(struct socket *so, struct inpcbtable *table)
 	inp->inp_cksum6 = -1;
 #endif /* INET6 */
 
+	mtx_enter(&table->inpt_mtx);
 	if (table->inpt_count++ > INPCBHASH_LOADFACTOR(table->inpt_size))
 		(void)in_pcbresize(table, table->inpt_size * 2);
 	TAILQ_INSERT_HEAD(&table->inpt_queue, inp, inp_queue);
@@ -268,6 +270,8 @@ in_pcballoc(struct socket *so, struct inpcbtable *table)
 		    &inp->inp_faddr, inp->inp_fport,
 		    &inp->inp_laddr, inp->inp_lport);
 	LIST_INSERT_HEAD(head, inp, inp_hash);
+	mtx_leave(&table->inpt_mtx);
+
 	so->so_pcb = inp;
 
 	return (0);
@@ -556,6 +560,7 @@ void
 in_pcbdetach(struct inpcb *inp)
 {
 	struct socket *so = inp->inp_socket;
+	struct inpcbtable *table = inp->inp_table;
 
 	NET_ASSERT_LOCKED();
 
@@ -585,10 +590,13 @@ in_pcbdetach(struct inpcb *inp)
 		pf_inp_unlink(inp);
 	}
 #endif
+	mtx_enter(&table->inpt_mtx);
 	LIST_REMOVE(inp, inp_lhash);
 	LIST_REMOVE(inp, inp_hash);
-	TAILQ_REMOVE(&inp->inp_table->inpt_queue, inp, inp_queue);
-	inp->inp_table->inpt_count--;
+	TAILQ_REMOVE(&table->inpt_queue, inp, inp_queue);
+	table->inpt_count--;
+	mtx_leave(&table->inpt_mtx);
+
 	in_pcbunref(inp);
 }
 
@@ -661,20 +669,25 @@ void
 in_pcbnotifyall(struct inpcbtable *table, struct sockaddr *dst, u_int rtable,
     int errno, void (*notify)(struct inpcb *, int))
 {
-	struct inpcb *inp, *ninp;
+	SIMPLEQ_HEAD(, inpcb) inpcblist;
+	struct inpcb *inp;
 	struct in_addr faddr;
 	u_int rdomain;
 
-	NET_ASSERT_LOCKED();
+	NET_ASSERT_WLOCKED();
 
 	if (dst->sa_family != AF_INET)
 		return;
 	faddr = satosin(dst)->sin_addr;
 	if (faddr.s_addr == INADDR_ANY)
 		return;
+	if (notify == NULL)
+		return;
 
+	SIMPLEQ_INIT(&inpcblist);
 	rdomain = rtable_l2(rtable);
-	TAILQ_FOREACH_SAFE(inp, &table->inpt_queue, inp_queue, ninp) {
+	mtx_enter(&table->inpt_mtx);
+	TAILQ_FOREACH(inp, &table->inpt_queue, inp_queue) {
 #ifdef INET6
 		if (inp->inp_flags & INP_IPV6)
 			continue;
@@ -684,8 +697,15 @@ in_pcbnotifyall(struct inpcbtable *table, struct sockaddr *dst, u_int rtable,
 		    inp->inp_socket == NULL) {
 			continue;
 		}
-		if (notify)
-			(*notify)(inp, errno);
+		in_pcbref(inp);
+		SIMPLEQ_INSERT_TAIL(&inpcblist, inp, inp_notify);
+	}
+	mtx_leave(&table->inpt_mtx);
+
+	while ((inp = SIMPLEQ_FIRST(&inpcblist)) != NULL) {
+		SIMPLEQ_REMOVE_HEAD(&inpcblist, inp_notify);
+		(*notify)(inp, errno);
+		in_pcbunref(inp);
 	}
 }
 
@@ -759,6 +779,7 @@ in_pcblookup_local(struct inpcbtable *table, void *laddrp, u_int lport_arg,
 	u_int rdomain;
 
 	rdomain = rtable_l2(rtable);
+	mtx_enter(&table->inpt_mtx);
 	head = in_pcblhash(table, rdomain, lport);
 	LIST_FOREACH(inp, head, inp_lhash) {
 		if (rtable_l2(inp->inp_rtableid) != rdomain)
@@ -809,6 +830,8 @@ in_pcblookup_local(struct inpcbtable *table, void *laddrp, u_int lport_arg,
 				break;
 		}
 	}
+	mtx_leave(&table->inpt_mtx);
+
 	return (match);
 }
 
@@ -979,9 +1002,20 @@ void
 in_pcbrehash(struct inpcb *inp)
 {
 	struct inpcbtable *table = inp->inp_table;
+
+	mtx_enter(&table->inpt_mtx);
+	in_pcbrehash_locked(inp);
+	mtx_leave(&table->inpt_mtx);
+}
+
+void
+in_pcbrehash_locked(struct inpcb *inp)
+{
+	struct inpcbtable *table = inp->inp_table;
 	struct inpcbhead *head;
 
 	NET_ASSERT_LOCKED();
+	MUTEX_ASSERT_LOCKED(&table->inpt_mtx);
 
 	LIST_REMOVE(inp, inp_lhash);
 	head = in_pcblhash(table, inp->inp_rtableid, inp->inp_lport);
@@ -1008,6 +1042,8 @@ in_pcbresize(struct inpcbtable *table, int hashsize)
 	void *nhashtbl, *nlhashtbl, *ohashtbl, *olhashtbl;
 	struct inpcb *inp;
 
+	MUTEX_ASSERT_LOCKED(&table->inpt_mtx);
+
 	ohashtbl = table->inpt_hashtbl;
 	olhashtbl = table->inpt_lhashtbl;
 	osize = table->inpt_size;
@@ -1029,7 +1065,7 @@ in_pcbresize(struct inpcbtable *table, int hashsize)
 	arc4random_buf(&table->inpt_lkey, sizeof(table->inpt_lkey));
 
 	TAILQ_FOREACH(inp, &table->inpt_queue, inp_queue) {
-		in_pcbrehash(inp);
+		in_pcbrehash_locked(inp);
 	}
 	hashfree(ohashtbl, osize, M_PCB);
 	hashfree(olhashtbl, osize, M_PCB);
@@ -1060,6 +1096,7 @@ in_pcbhashlookup(struct inpcbtable *table, struct in_addr faddr,
 	u_int rdomain;
 
 	rdomain = rtable_l2(rtable);
+	mtx_enter(&table->inpt_mtx);
 	head = in_pcbhash(table, rdomain, &faddr, fport, &laddr, lport);
 	LIST_FOREACH(inp, head, inp_hash) {
 #ifdef INET6
@@ -1082,6 +1119,7 @@ in_pcbhashlookup(struct inpcbtable *table, struct in_addr faddr,
 			break;
 		}
 	}
+	mtx_leave(&table->inpt_mtx);
 #ifdef DIAGNOSTIC
 	if (inp == NULL && in_pcbnotifymiss) {
 		printf("%s: faddr=%08x fport=%d laddr=%08x lport=%d rdom=%u\n",
@@ -1143,6 +1181,7 @@ in_pcblookup_listen(struct inpcbtable *table, struct in_addr laddr,
 #endif
 
 	rdomain = rtable_l2(rtable);
+	mtx_enter(&table->inpt_mtx);
 	head = in_pcbhash(table, rdomain, &zeroin_addr, 0, key1, lport);
 	LIST_FOREACH(inp, head, inp_hash) {
 #ifdef INET6
@@ -1179,6 +1218,7 @@ in_pcblookup_listen(struct inpcbtable *table, struct in_addr laddr,
 		LIST_REMOVE(inp, inp_hash);
 		LIST_INSERT_HEAD(head, inp, inp_hash);
 	}
+	mtx_leave(&table->inpt_mtx);
 #ifdef DIAGNOSTIC
 	if (inp == NULL && in_pcbnotifymiss) {
 		printf("%s: laddr=%08x lport=%d rdom=%u\n",