Move bridge_broadcast and subsequently all IPsec SPD lookup code out

of the IPL_NET.  pf_test should be no longer called under IPL_NET as
well.  The problem became evident after the related issue was brought
up by David Hill <dhill at mindcry ! org>.

With input from and OK mpi.  Tested by David and me.

7 files changed, 23 insertions, 87 deletions

diff --git a/sys/net/if_bridge.c b/sys/net/if_bridge.c
index 41d7b678e82..d96166f4c41 100644
--- a/sys/net/if_bridge.c
+++ b/sys/net/if_bridge.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: if_bridge.c,v 1.211 2013/06/26 09:12:39 henning Exp $	*/
+/*	$OpenBSD: if_bridge.c,v 1.212 2013/07/31 15:41:51 mikeb Exp $	*/
 
 /*
  * Copyright (c) 1999, 2000 Jason L. Wright (jason@thought.net)
@@ -971,8 +971,6 @@ bridge_output(struct ifnet *ifp, struct mbuf *m, struct sockaddr *sa,
 	eh = mtod(m, struct ether_header *);
 	dst = (struct ether_addr *)&eh->ether_dhost[0];
 
-	s = splnet();
-
 	/*
 	 * If bridge is down, but original output interface is up,
 	 * go ahead and send out that interface.  Otherwise the packet
@@ -1009,7 +1007,6 @@ bridge_output(struct ifnet *ifp, struct mbuf *m, struct sockaddr *sa,
 		    NULL)) != NULL) {
 			ipsp_skipcrypto_unmark((struct tdb_ident *)(mtag + 1));
 			m_freem(m);
-			splx(s);
 			return (0);
 		}
 #endif /* IPSEC */
@@ -1076,13 +1073,14 @@ bridge_output(struct ifnet *ifp, struct mbuf *m, struct sockaddr *sa,
 				mc = m1;
 			}
 
+			s = splnet();
 			error = bridge_ifenqueue(sc, dst_if, mc);
+			splx(s);
 			if (error)
 				continue;
 		}
 		if (!used)
 			m_freem(m);
-		splx(s);
 		return (0);
 	}
 
@@ -1090,9 +1088,9 @@ sendunicast:
 	bridge_span(sc, NULL, m);
 	if ((dst_if->if_flags & IFF_RUNNING) == 0) {
 		m_freem(m);
-		splx(s);
 		return (ENETDOWN);
 	}
+	s = splnet();
 	bridge_ifenqueue(sc, dst_if, m);
 	splx(s);
 	return (0);
@@ -1253,9 +1251,7 @@ bridgeintr_frame(struct bridge_softc *sc, struct mbuf *m)
 	 */
 	if ((m->m_flags & (M_BCAST | M_MCAST)) || dst_if == NULL) {
 		sc->sc_if.if_imcasts++;
-		s = splnet();
 		bridge_broadcast(sc, src_if, &eh, m);
-		splx(s);
 		return;
 	}
 
@@ -1498,9 +1494,7 @@ bridge_broadcast(struct bridge_softc *sc, struct ifnet *ifp,
 	struct bridge_iflist *p;
 	struct mbuf *mc;
 	struct ifnet *dst_if;
-	int len, used = 0;
-
-	splassert(IPL_NET);
+	int len, s, used = 0;
 
 	TAILQ_FOREACH(p, &sc->sc_iflist, next) {
 		/*
@@ -1589,7 +1583,9 @@ bridge_broadcast(struct bridge_softc *sc, struct ifnet *ifp,
 		if ((len - ETHER_HDR_LEN) > dst_if->if_mtu)
 			bridge_fragment(sc, dst_if, eh, mc);
 		else {
+			s = splnet();
 			bridge_ifenqueue(sc, dst_if, mc);
+			splx(s);
 		}
 	}
 
@@ -1645,7 +1641,7 @@ bridge_span(struct bridge_softc *sc, struct ether_header *eh,
 	struct bridge_iflist *p;
 	struct ifnet *ifp;
 	struct mbuf *mc, *m;
-	int error;
+	int s, error;
 
 	if (TAILQ_EMPTY(&sc->sc_spanlist))
 		return;
@@ -1681,7 +1677,9 @@ bridge_span(struct bridge_softc *sc, struct ether_header *eh,
 			continue;
 		}
 
+		s = splnet();
 		error = bridge_ifenqueue(sc, ifp, mc);
+		splx(s);
 		if (error)
 			continue;
 	}
diff --git a/sys/netinet/ip_input.c b/sys/netinet/ip_input.c
index 664afbfce34..da49a5ba33e 100644
--- a/sys/netinet/ip_input.c
+++ b/sys/netinet/ip_input.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: ip_input.c,v 1.214 2013/07/04 08:22:19 mpi Exp $	*/
+/*	$OpenBSD: ip_input.c,v 1.215 2013/07/31 15:41:51 mikeb Exp $	*/
 /*	$NetBSD: ip_input.c,v 1.30 1996/03/16 23:53:58 christos Exp $	*/
 
 /*
@@ -245,7 +245,7 @@ ipv4_input(struct mbuf *m)
 	int hlen, len;
 	in_addr_t pfrdr = 0;
 #ifdef IPSEC
-	int error, s;
+	int error;
 	struct tdb *tdb;
 	struct tdb_ident *tdbi;
 	struct m_tag *mtag;
@@ -454,7 +454,6 @@ ipv4_input(struct mbuf *m)
 		 * inner-most IPsec SA used.
 		 */
 		mtag = m_tag_find(m, PACKET_TAG_IPSEC_IN_DONE, NULL);
-                s = splnet();
 		if (mtag != NULL) {
 			tdbi = (struct tdb_ident *)(mtag + 1);
 			tdb = gettdb(tdbi->rdomain, tdbi->spi,
@@ -463,7 +462,6 @@ ipv4_input(struct mbuf *m)
 			tdb = NULL;
 	        ipsp_spd_lookup(m, AF_INET, hlen, &error,
 		    IPSP_DIRECTION_IN, tdb, NULL, 0);
-                splx(s);
 
 		/* Error or otherwise drop-packet indication */
 		if (error) {
@@ -497,7 +495,7 @@ ip_ours(struct mbuf *m)
 	struct ipqent *ipqe;
 	int mff, hlen;
 #ifdef IPSEC
-	int error, s;
+	int error;
 	struct tdb *tdb;
 	struct tdb_ident *tdbi;
 	struct m_tag *mtag;
@@ -639,7 +637,6 @@ found:
 	 * that's needed in the real world (who uses bundles anyway ?).
 	 */
 	mtag = m_tag_find(m, PACKET_TAG_IPSEC_IN_DONE, NULL);
-        s = splnet();
 	if (mtag) {
 		tdbi = (struct tdb_ident *)(mtag + 1);
 	        tdb = gettdb(tdbi->rdomain, tdbi->spi, &tdbi->dst,
@@ -648,7 +645,6 @@ found:
 		tdb = NULL;
 	ipsp_spd_lookup(m, AF_INET, hlen, &error, IPSP_DIRECTION_IN,
 	    tdb, NULL, 0);
-        splx(s);
 
 	/* Error or otherwise drop-packet indication. */
 	if (error) {
diff --git a/sys/netinet/ip_output.c b/sys/netinet/ip_output.c
index e0e6b7d73f3..7d236446019 100644
--- a/sys/netinet/ip_output.c
+++ b/sys/netinet/ip_output.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: ip_output.c,v 1.243 2013/07/04 19:10:40 sf Exp $	*/
+/*	$OpenBSD: ip_output.c,v 1.244 2013/07/31 15:41:52 mikeb Exp $	*/
 /*	$NetBSD: ip_output.c,v 1.28 1996/02/13 23:43:07 christos Exp $	*/
 
 /*
@@ -109,7 +109,6 @@ ip_output(struct mbuf *m0, ...)
 	struct inpcb *inp;
 	struct tdb *tdb;
 	u_int32_t ipsecflowinfo;
-	int s;
 #if NPF > 0
 	struct ifnet *encif;
 #endif
@@ -256,12 +255,6 @@ reroute:
 	if (!ipsec_in_use && inp == NULL)
 		goto done_spd;
 
-	/*
-	 * splnet is chosen over splsoftnet because we are not allowed to
-	 * lower the level, and udp_output calls us in splnet().
-	 */
-	s = splnet();
-
 	/* Do we have any pending SAs to apply ? */
 	mtag = m_tag_find(m, PACKET_TAG_IPSEC_PENDING_TDB, NULL);
 	if (mtag != NULL) {
@@ -282,8 +275,6 @@ reroute:
 		    IPSP_DIRECTION_OUT, NULL, inp, ipsecflowinfo);
 
 	if (tdb == NULL) {
-		splx(s);
-
 		if (error == 0) {
 			/*
 			 * No IPsec processing required, we'll just send the
@@ -318,7 +309,6 @@ reroute:
 			    tdbi->rdomain == tdb->tdb_rdomain &&
 			    !bcmp(&tdbi->dst, &tdb->tdb_dst,
 			    sizeof(union sockaddr_union))) {
-				splx(s);
 				sproto = 0; /* mark as no-IPsec-needed */
 				goto done_spd;
 			}
@@ -328,7 +318,6 @@ reroute:
 		bcopy(&tdb->tdb_dst, &sdst, sizeof(sdst));
 		sspi = tdb->tdb_spi;
 		sproto = tdb->tdb_sproto;
-		splx(s);
 
 		/*
 		 * If it needs TCP/UDP hardware-checksumming, do the
@@ -575,14 +564,11 @@ sendit:
 	 * Check if the packet needs encapsulation.
 	 */
 	if (sproto != 0) {
-		s = splnet();
-
 		tdb = gettdb(rtable_l2(m->m_pkthdr.rdomain),
 		    sspi, &sdst, sproto);
 		if (tdb == NULL) {
 			DPRINTF(("ip_output: unknown TDB"));
 			error = EHOSTUNREACH;
-			splx(s);
 			m_freem(m);
 			goto done;
 		}
@@ -595,12 +581,10 @@ sendit:
 		    tdb->tdb_tap)) == NULL ||
 		    pf_test(AF_INET, PF_OUT, encif, &m, NULL) != PF_PASS) {
 			error = EACCES;
-			splx(s);
 			m_freem(m);
 			goto done;
 		}
 		if (m == NULL) {
-			splx(s);
 			goto done;
 		}
 		ip = mtod(m, struct ip *);
@@ -627,7 +611,6 @@ sendit:
 			    (tdb->tdb_dst.sin.sin_addr.s_addr ==
 			    ip->ip_dst.s_addr);
 			icmp_mtu = tdb->tdb_mtu;
-			splx(s);
 
 			/* Find a host route to store the mtu in */
 			if (ro != NULL)
@@ -667,7 +650,6 @@ sendit:
 
 		/* Callee frees mbuf */
 		error = ipsp_process_packet(m, tdb, AF_INET, 0);
-		splx(s);
 		return error;  /* Nothing more to be done */
 	}
 
diff --git a/sys/netinet/tcp_input.c b/sys/netinet/tcp_input.c
index 4e39e7f1f6f..734ae46c916 100644
--- a/sys/netinet/tcp_input.c
+++ b/sys/netinet/tcp_input.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: tcp_input.c,v 1.265 2013/07/01 10:53:52 bluhm Exp $	*/
+/*	$OpenBSD: tcp_input.c,v 1.266 2013/07/31 15:41:52 mikeb Exp $	*/
 /*	$NetBSD: tcp_input.c,v 1.23 1996/02/13 23:43:44 christos Exp $	*/
 
 /*
@@ -378,7 +378,7 @@ tcp_input(struct mbuf *m, ...)
 	struct m_tag *mtag;
 	struct tdb_ident *tdbi;
 	struct tdb *tdb;
-	int error, s;
+	int error;
 #endif /* IPSEC */
 	int af;
 #ifdef TCP_ECN
@@ -886,7 +886,6 @@ findpcb:
 #ifdef IPSEC
 	/* Find most recent IPsec tag */
 	mtag = m_tag_find(m, PACKET_TAG_IPSEC_IN_DONE, NULL);
-        s = splnet();
 	if (mtag != NULL) {
 		tdbi = (struct tdb_ident *)(mtag + 1);
 	        tdb = gettdb(tdbi->rdomain, tdbi->spi,
@@ -897,7 +896,6 @@ findpcb:
 	    tdb, inp, 0);
 	if (error) {
 		tcpstat.tcps_rcvnosec++;
-		splx(s);
 		goto drop;
 	}
 
@@ -909,7 +907,6 @@ findpcb:
 				inp->inp_ipo = ipsec_add_policy(inp, af,
 				    IPSP_DIRECTION_OUT);
 				if (inp->inp_ipo == NULL) {
-					splx(s);
 					goto drop;
 				}
 			}
@@ -936,7 +933,6 @@ findpcb:
 			inp->inp_tdb_in = NULL;
 		}
 	}
-        splx(s);
 #endif /* IPSEC */
 
 	/*
@@ -969,7 +965,7 @@ findpcb:
 
 		/* subtract out the tcp timestamp modulator */
 		opti.ts_ecr -= tp->ts_modulate;
-                                                     
+
 		/* make sure ts_ecr is sensible */
 		rtt_test = tcp_now - opti.ts_ecr;
 		if (rtt_test < 0 || rtt_test > TCP_RTT_MAX)
diff --git a/sys/netinet/udp_usrreq.c b/sys/netinet/udp_usrreq.c
index 2b5623f1430..89bd8b8be3c 100644
--- a/sys/netinet/udp_usrreq.c
+++ b/sys/netinet/udp_usrreq.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: udp_usrreq.c,v 1.164 2013/06/09 22:03:06 yasuoka Exp $	*/
+/*	$OpenBSD: udp_usrreq.c,v 1.165 2013/07/31 15:41:52 mikeb Exp $	*/
 /*	$NetBSD: udp_usrreq.c,v 1.28 1996/03/16 23:54:03 christos Exp $	*/
 
 /*
@@ -178,7 +178,7 @@ udp_input(struct mbuf *m, ...)
 	struct m_tag *mtag;
 	struct tdb_ident *tdbi;
 	struct tdb *tdb;
-	int error, s;
+	int error;
 	u_int32_t ipsecflowinfo = 0;
 #endif /* IPSEC */
 
@@ -600,7 +600,6 @@ udp_input(struct mbuf *m, ...)
 
 #ifdef IPSEC
 	mtag = m_tag_find(m, PACKET_TAG_IPSEC_IN_DONE, NULL);
-	s = splnet();
 	if (mtag != NULL) {
 		tdbi = (struct tdb_ident *)(mtag + 1);
 		tdb = gettdb(tdbi->rdomain, tdbi->spi,
@@ -611,7 +610,6 @@ udp_input(struct mbuf *m, ...)
 	    IPSP_DIRECTION_IN, tdb, inp, 0);
 	if (error) {
 		udpstat.udps_nosec++;
-		splx(s);
 		goto bad;
 	}
 
@@ -624,7 +622,6 @@ udp_input(struct mbuf *m, ...)
 				inp->inp_ipo = ipsec_add_policy(inp,
 				    srcsa.sa.sa_family, IPSP_DIRECTION_OUT);
 				if (inp->inp_ipo == NULL) {
-					splx(s);
 					goto bad;
 				}
 			}
@@ -655,7 +652,6 @@ udp_input(struct mbuf *m, ...)
 	if (tdb)
 		ipsecflowinfo = tdb->tdb_spi;
 
-	splx(s);
 #endif /*IPSEC */
 
 	opts = NULL;
diff --git a/sys/netinet6/ip6_forward.c b/sys/netinet6/ip6_forward.c
index 4e2e45918da..1cd52dbe254 100644
--- a/sys/netinet6/ip6_forward.c
+++ b/sys/netinet6/ip6_forward.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: ip6_forward.c,v 1.60 2013/07/04 19:10:41 sf Exp $	*/
+/*	$OpenBSD: ip6_forward.c,v 1.61 2013/07/31 15:41:52 mikeb Exp $	*/
 /*	$KAME: ip6_forward.c,v 1.75 2001/06/29 12:42:13 jinmei Exp $	*/
 
 /*
@@ -100,7 +100,6 @@ ip6_forward(struct mbuf *m, int srcrt)
 	struct tdb_ident *tdbi;
 	u_int32_t sspi;
 	struct tdb *tdb;
-	int s;
 #if NPF > 0
 	struct ifnet *encif;
 #endif
@@ -148,8 +147,6 @@ reroute:
 	if (!ipsec_in_use)
 		goto done_spd;
 
-	s = splnet();
-
 	/*
 	 * Check if there was an outgoing SA bound to the flow
 	 * from a transport protocol.
@@ -174,8 +171,6 @@ reroute:
 		    &error, IPSP_DIRECTION_OUT, NULL, NULL, 0);
 
 	if (tdb == NULL) {
-	        splx(s);
-
 		if (error == 0) {
 		        /*
 			 * No IPsec processing required, we'll just send the
@@ -209,7 +204,6 @@ reroute:
 			    tdbi->rdomain == tdb->tdb_rdomain &&
 			    !bcmp(&tdbi->dst, &tdb->tdb_dst,
 			    sizeof(union sockaddr_union))) {
-				splx(s);
 				sproto = 0; /* mark as no-IPsec-needed */
 				goto done_spd;
 			}
@@ -219,7 +213,6 @@ reroute:
 	        bcopy(&tdb->tdb_dst, &sdst, sizeof(sdst));
 		sspi = tdb->tdb_spi;
 		sproto = tdb->tdb_sproto;
-	        splx(s);
 	}
 
 	/* Fall through to the routing/multicast handling code */
@@ -337,12 +330,9 @@ reroute:
 	 * PMTU notification.  is it okay?
 	 */
 	if (sproto != 0) {
-		s = splnet();
-
 		tdb = gettdb(rtable_l2(m->m_pkthdr.rdomain),
 		    sspi, &sdst, sproto);
 		if (tdb == NULL) {
-			splx(s);
 			error = EHOSTUNREACH;
 			m_freem(m);
 			goto senderr;	/*XXX*/
@@ -352,15 +342,12 @@ reroute:
 		if ((encif = enc_getif(tdb->tdb_rdomain,
 		    tdb->tdb_tap)) == NULL ||
 		    pf_test(AF_INET6, PF_FWD, encif, &m, NULL) != PF_PASS) {
-			splx(s);
 			error = EHOSTUNREACH;
 			m_freem(m);
 			goto senderr;
 		}
-		if (m == NULL) {
-			splx(s);
+		if (m == NULL)
 			goto senderr;
-		}
 		ip6 = mtod(m, struct ip6_hdr *);
 		/*
 		 * PF_TAG_REROUTE handling or not...
@@ -376,7 +363,6 @@ reroute:
 
 		/* Callee frees mbuf */
 		error = ipsp_process_packet(m, tdb, AF_INET6, 0);
-		splx(s);
 		m_freem(mcopy);
 		goto freert;
 	}
diff --git a/sys/netinet6/ip6_output.c b/sys/netinet6/ip6_output.c
index d9a11ce4165..4df3fcea865 100644
--- a/sys/netinet6/ip6_output.c
+++ b/sys/netinet6/ip6_output.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: ip6_output.c,v 1.142 2013/07/04 19:10:41 sf Exp $	*/
+/*	$OpenBSD: ip6_output.c,v 1.143 2013/07/31 15:41:52 mikeb Exp $	*/
 /*	$KAME: ip6_output.c,v 1.172 2001/03/25 09:55:56 itojun Exp $	*/
 
 /*
@@ -173,7 +173,6 @@ ip6_output(struct mbuf *m0, struct ip6_pktopts *opt, struct route_in6 *ro,
 	struct tdb_ident *tdbi;
 	u_int32_t sspi;
 	struct tdb *tdb;
-	int s;
 #if NPF > 0
 	struct ifnet *encif;
 #endif
@@ -216,12 +215,6 @@ ip6_output(struct mbuf *m0, struct ip6_pktopts *opt, struct route_in6 *ro,
 		goto done_spd;
 
 	/*
-	 * splnet is chosen over splsoftnet because we are not allowed to
-	 * lower the level, and udp6_output calls us in splnet(). XXX check
-	 */
-	s = splnet();
-
-	/*
 	 * Check if there was an outgoing SA bound to the flow
 	 * from a transport protocol.
 	 */
@@ -245,8 +238,6 @@ ip6_output(struct mbuf *m0, struct ip6_pktopts *opt, struct route_in6 *ro,
 		    &error, IPSP_DIRECTION_OUT, NULL, inp, 0);
 
 	if (tdb == NULL) {
-	        splx(s);
-
 		if (error == 0) {
 		        /*
 			 * No IPsec processing required, we'll just send the
@@ -280,7 +271,6 @@ ip6_output(struct mbuf *m0, struct ip6_pktopts *opt, struct route_in6 *ro,
 			    tdbi->rdomain == tdb->tdb_rdomain &&
 			    !bcmp(&tdbi->dst, &tdb->tdb_dst,
 			    sizeof(union sockaddr_union))) {
-				splx(s);
 				sproto = 0; /* mark as no-IPsec-needed */
 				goto done_spd;
 			}
@@ -290,7 +280,6 @@ ip6_output(struct mbuf *m0, struct ip6_pktopts *opt, struct route_in6 *ro,
 	        bcopy(&tdb->tdb_dst, &sdst, sizeof(sdst));
 		sspi = tdb->tdb_spi;
 		sproto = tdb->tdb_sproto;
-	        splx(s);
 	}
 
 	/* Fall through to the routing/multicast handling code */
@@ -497,8 +486,6 @@ reroute:
 	 * ipsp_process_packet will never come back to here.
 	 */
 	if (sproto != 0) {
-	        s = splnet();
-
 		/*
 		 * XXX what should we do if ip6_hlim == 0 and the
 		 * packet gets tunneled?
@@ -507,7 +494,6 @@ reroute:
 		tdb = gettdb(rtable_l2(m->m_pkthdr.rdomain),
 		    sspi, &sdst, sproto);
 		if (tdb == NULL) {
-			splx(s);
 			error = EHOSTUNREACH;
 			m_freem(m);
 			goto done;
@@ -517,15 +503,12 @@ reroute:
 		if ((encif = enc_getif(tdb->tdb_rdomain,
 		    tdb->tdb_tap)) == NULL ||
 		    pf_test(AF_INET6, PF_OUT, encif, &m, NULL) != PF_PASS) {
-			splx(s);
 			error = EHOSTUNREACH;
 			m_freem(m);
 			goto done;
 		}
-		if (m == NULL) {
-			splx(s);
+		if (m == NULL)
 			goto done;
-		}
 		ip6 = mtod(m, struct ip6_hdr *);
 		/*
 		 * PF_TAG_REROUTE handling or not...
@@ -547,7 +530,6 @@ reroute:
 		 */
 		error = ipsp_process_packet(m, tdb, AF_INET6,
 		    exthdrs.ip6e_rthdr ? 1 : 0);
-		splx(s);
 
 		return error;  /* Nothing more to be done */
 	}