Revert the inpcb table mutex commit. It triggers a witness panic

in raw IP delivery and UDP broadcast loops.  There inpcbtable_mtx
is held and sorwakeup() is called within the loop.  As sowakeup()
grabs the kernel lock, we have a lock ordering problem.
found by Hrvoje Popovski; OK deraadt@ mpi@

3 files changed, 4 insertions, 26 deletions

diff --git a/sys/netinet6/in6_pcb.c b/sys/netinet6/in6_pcb.c
index 2f56401b029..0100c9e28e5 100644
--- a/sys/netinet6/in6_pcb.c
+++ b/sys/netinet6/in6_pcb.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: in6_pcb.c,v 1.107 2018/09/20 18:59:10 bluhm Exp $	*/
+/*	$OpenBSD: in6_pcb.c,v 1.108 2018/10/04 17:33:41 bluhm Exp $	*/
 
 /*
  * Copyright (C) 1995, 1996, 1997, and 1998 WIDE Project.
@@ -411,8 +411,6 @@ in6_pcbnotify(struct inpcbtable *table, struct sockaddr_in6 *dst,
 	errno = inet6ctlerrmap[cmd];
 
 	rdomain = rtable_l2(rtable);
-	KERNEL_LOCK();
-	mtx_enter(&inpcbtable_mtx);
 	TAILQ_FOREACH_SAFE(inp, &table->inpt_queue, inp_queue, ninp) {
 		if ((inp->inp_flags & INP_IPV6) == 0)
 			continue;
@@ -486,18 +484,9 @@ in6_pcbnotify(struct inpcbtable *table, struct sockaddr_in6 *dst,
 		}
 	  do_notify:
 		nmatch++;
-		/*
-		 * The notify functions may grab the kernel lock.  Sometimes
-		 * we already hold the kernel lock when we acquire the pcb
-		 * mutex.  So do an extra kernel lock before the mutex outside
-		 * of this loop.  XXXSMP
-		 */
 		if (notify)
 			(*notify)(inp, errno);
 	}
-	mtx_leave(&inpcbtable_mtx);
-	KERNEL_UNLOCK();
-
 	return (nmatch);
 }
 
@@ -512,7 +501,6 @@ in6_pcbhashlookup(struct inpcbtable *table, const struct in6_addr *faddr,
 	u_int rdomain;
 
 	rdomain = rtable_l2(rtable);
-	mtx_enter(&inpcbtable_mtx);
 	head = in6_pcbhash(table, rdomain, faddr, fport, laddr, lport);
 	LIST_FOREACH(inp, head, inp_hash) {
 		if (!(inp->inp_flags & INP_IPV6))
@@ -533,7 +521,6 @@ in6_pcbhashlookup(struct inpcbtable *table, const struct in6_addr *faddr,
 			break;
 		}
 	}
-	mtx_leave(&inpcbtable_mtx);
 #ifdef DIAGNOSTIC
 	if (inp == NULL && in_pcbnotifymiss) {
 		printf("%s: faddr= fport=%d laddr= lport=%d rdom=%u\n",
@@ -584,7 +571,6 @@ in6_pcblookup_listen(struct inpcbtable *table, struct in6_addr *laddr,
 #endif
 
 	rdomain = rtable_l2(rtable);
-	mtx_enter(&inpcbtable_mtx);
 	head = in6_pcbhash(table, rdomain, &zeroin6_addr, 0, key1, lport);
 	LIST_FOREACH(inp, head, inp_hash) {
 		if (!(inp->inp_flags & INP_IPV6))
@@ -617,7 +603,6 @@ in6_pcblookup_listen(struct inpcbtable *table, struct in6_addr *laddr,
 		LIST_REMOVE(inp, inp_hash);
 		LIST_INSERT_HEAD(head, inp, inp_hash);
 	}
-	mtx_leave(&inpcbtable_mtx);
 #ifdef DIAGNOSTIC
 	if (inp == NULL && in_pcbnotifymiss) {
 		printf("%s: laddr= lport=%d rdom=%u\n",
diff --git a/sys/netinet6/ip6_divert.c b/sys/netinet6/ip6_divert.c
index 1c9933f2365..2c60fe91f84 100644
--- a/sys/netinet6/ip6_divert.c
+++ b/sys/netinet6/ip6_divert.c
@@ -1,4 +1,4 @@
-/*      $OpenBSD: ip6_divert.c,v 1.57 2018/09/20 18:59:10 bluhm Exp $ */
+/*      $OpenBSD: ip6_divert.c,v 1.58 2018/10/04 17:33:41 bluhm Exp $ */
 
 /*
  * Copyright (c) 2009 Michele Marchetto <michele@openbsd.org>
@@ -190,12 +190,10 @@ divert6_packet(struct mbuf *m, int dir, u_int16_t divert_port)
 		return (0);
 	}
 
-	mtx_enter(&inpcbtable_mtx);
 	TAILQ_FOREACH(inp, &divb6table.inpt_queue, inp_queue) {
 		if (inp->inp_lport == divert_port)
 			break;
 	}
-	mtx_leave(&inpcbtable_mtx);
 
 	memset(&addr, 0, sizeof(addr));
 	addr.sin6_family = AF_INET6;
diff --git a/sys/netinet6/raw_ip6.c b/sys/netinet6/raw_ip6.c
index b3b140fda82..ea5429e4e1a 100644
--- a/sys/netinet6/raw_ip6.c
+++ b/sys/netinet6/raw_ip6.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: raw_ip6.c,v 1.131 2018/09/20 18:59:10 bluhm Exp $	*/
+/*	$OpenBSD: raw_ip6.c,v 1.132 2018/10/04 17:33:41 bluhm Exp $	*/
 /*	$KAME: raw_ip6.c,v 1.69 2001/03/04 15:55:44 itojun Exp $	*/
 
 /*
@@ -158,7 +158,6 @@ rip6_input(struct mbuf **mp, int *offp, int proto, int af)
 	}
 #endif
 	NET_ASSERT_LOCKED();
-	mtx_enter(&inpcbtable_mtx);
 	TAILQ_FOREACH(in6p, &rawin6pcbtable.inpt_queue, inp_queue) {
 		if (in6p->inp_socket->so_state & SS_CANTRCVMORE)
 			continue;
@@ -178,10 +177,8 @@ rip6_input(struct mbuf **mp, int *offp, int proto, int af)
 
 			IP6_EXTHDR_GET(icmp6, struct icmp6_hdr *, m, *offp,
 			    sizeof(*icmp6));
-			if (icmp6 == NULL) {
-				mtx_leave(&inpcbtable_mtx);
+			if (icmp6 == NULL)
 				return IPPROTO_DONE;
-			}
 			if (ICMP6_FILTER_WILLBLOCK(icmp6->icmp6_type,
 			    in6p->inp_icmp6filt))
 				continue;
@@ -215,8 +212,6 @@ rip6_input(struct mbuf **mp, int *offp, int proto, int af)
 		}
 		last = in6p;
 	}
-	mtx_leave(&inpcbtable_mtx);
-
 	if (last) {
 		if (last->inp_flags & IN6P_CONTROLOPTS)
 			ip6_savecontrol(last, m, &opts);