As a step towards per inpcb or socket locks, remove the net lock

for netstat -a.  Introduce a global mutex that protects the tables
and hashes for the internet PCBs.  To detect detached PCB, set its
inp_socket field to NULL.  This has to be protected by a per PCB
mutex.  The protocol pointer has to be protected by the mutex as
netstat uses it.
Always take the kernel lock in in_pcbnotifyall() and in6_pcbnotify()
before the table mutex to avoid lock ordering problems in the notify
functions.
OK visa@

1 files changed, 17 insertions, 2 deletions

diff --git a/sys/netinet6/in6_pcb.c b/sys/netinet6/in6_pcb.c
index 4dfc75bb246..2f56401b029 100644
--- a/sys/netinet6/in6_pcb.c
+++ b/sys/netinet6/in6_pcb.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: in6_pcb.c,v 1.106 2018/09/11 21:04:03 bluhm Exp $	*/
+/*	$OpenBSD: in6_pcb.c,v 1.107 2018/09/20 18:59:10 bluhm Exp $	*/
 
 /*
  * Copyright (C) 1995, 1996, 1997, and 1998 WIDE Project.
@@ -411,6 +411,8 @@ in6_pcbnotify(struct inpcbtable *table, struct sockaddr_in6 *dst,
 	errno = inet6ctlerrmap[cmd];
 
 	rdomain = rtable_l2(rtable);
+	KERNEL_LOCK();
+	mtx_enter(&inpcbtable_mtx);
 	TAILQ_FOREACH_SAFE(inp, &table->inpt_queue, inp_queue, ninp) {
 		if ((inp->inp_flags & INP_IPV6) == 0)
 			continue;
@@ -474,7 +476,7 @@ in6_pcbnotify(struct inpcbtable *table, struct sockaddr_in6 *dst,
 		else if (!IN6_ARE_ADDR_EQUAL(&inp->inp_faddr6,
 					     &dst->sin6_addr) ||
 			 rtable_l2(inp->inp_rtableid) != rdomain ||
-			 inp->inp_socket == 0 ||
+			 inp->inp_socket == NULL ||
 			 (lport && inp->inp_lport != lport) ||
 			 (!IN6_IS_ADDR_UNSPECIFIED(&sa6_src.sin6_addr) &&
 			  !IN6_ARE_ADDR_EQUAL(&inp->inp_laddr6,
@@ -484,9 +486,18 @@ in6_pcbnotify(struct inpcbtable *table, struct sockaddr_in6 *dst,
 		}
 	  do_notify:
 		nmatch++;
+		/*
+		 * The notify functions may grab the kernel lock.  Sometimes
+		 * we already hold the kernel lock when we acquire the pcb
+		 * mutex.  So do an extra kernel lock before the mutex outside
+		 * of this loop.  XXXSMP
+		 */
 		if (notify)
 			(*notify)(inp, errno);
 	}
+	mtx_leave(&inpcbtable_mtx);
+	KERNEL_UNLOCK();
+
 	return (nmatch);
 }
 
@@ -501,6 +512,7 @@ in6_pcbhashlookup(struct inpcbtable *table, const struct in6_addr *faddr,
 	u_int rdomain;
 
 	rdomain = rtable_l2(rtable);
+	mtx_enter(&inpcbtable_mtx);
 	head = in6_pcbhash(table, rdomain, faddr, fport, laddr, lport);
 	LIST_FOREACH(inp, head, inp_hash) {
 		if (!(inp->inp_flags & INP_IPV6))
@@ -521,6 +533,7 @@ in6_pcbhashlookup(struct inpcbtable *table, const struct in6_addr *faddr,
 			break;
 		}
 	}
+	mtx_leave(&inpcbtable_mtx);
 #ifdef DIAGNOSTIC
 	if (inp == NULL && in_pcbnotifymiss) {
 		printf("%s: faddr= fport=%d laddr= lport=%d rdom=%u\n",
@@ -571,6 +584,7 @@ in6_pcblookup_listen(struct inpcbtable *table, struct in6_addr *laddr,
 #endif
 
 	rdomain = rtable_l2(rtable);
+	mtx_enter(&inpcbtable_mtx);
 	head = in6_pcbhash(table, rdomain, &zeroin6_addr, 0, key1, lport);
 	LIST_FOREACH(inp, head, inp_hash) {
 		if (!(inp->inp_flags & INP_IPV6))
@@ -603,6 +617,7 @@ in6_pcblookup_listen(struct inpcbtable *table, struct in6_addr *laddr,
 		LIST_REMOVE(inp, inp_hash);
 		LIST_INSERT_HEAD(head, inp, inp_hash);
 	}
+	mtx_leave(&inpcbtable_mtx);
 #ifdef DIAGNOSTIC
 	if (inp == NULL && in_pcbnotifymiss) {
 		printf("%s: laddr= lport=%d rdom=%u\n",