Kill ip6_forward_rt reducing differences between v4 and v6.

A single forwarding cache is not the answer.  The answer is 42... err PF!

ok bluhm@

5 files changed, 42 insertions, 87 deletions

diff --git a/sys/net/pf_norm.c b/sys/net/pf_norm.c
index 9c999829de8..450c151c4ee 100644
--- a/sys/net/pf_norm.c
+++ b/sys/net/pf_norm.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: pf_norm.c,v 1.189 2016/08/17 03:24:12 procter Exp $ */
+/*	$OpenBSD: pf_norm.c,v 1.190 2016/08/24 09:41:12 mpi Exp $ */
 
 /*
  * Copyright 2001 Niels Provos <provos@citi.umich.edu>
@@ -759,7 +759,7 @@ pf_refragment6(struct mbuf **m0, struct m_tag *mtag, struct sockaddr_in6 *dst,
 		m->m_pkthdr.pf.flags |= PF_TAG_REFRAGMENTED;
 		if (error == 0) {
 			if (ifp == NULL) {
-				ip6_forward(m, 0);
+				ip6_forward(m, NULL, 0);
 			} else if ((u_long)m->m_pkthdr.len <= ifp->if_mtu) {
 				ifp->if_output(ifp, m, sin6tosa(dst), rt);
 			} else {
diff --git a/sys/netinet6/frag6.c b/sys/netinet6/frag6.c
index 9f281150e5e..39de30cac21 100644
--- a/sys/netinet6/frag6.c
+++ b/sys/netinet6/frag6.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: frag6.c,v 1.68 2016/08/22 10:33:22 mpi Exp $	*/
+/*	$OpenBSD: frag6.c,v 1.69 2016/08/24 09:41:12 mpi Exp $	*/
 /*	$KAME: frag6.c,v 1.40 2002/05/27 21:40:31 itojun Exp $	*/
 
 /*
@@ -596,9 +596,9 @@ void
 frag6_slowtimo(void)
 {
 	struct ip6q *q6, *nq6;
-	int s = splsoftnet();
-	extern struct route_in6 ip6_forward_rt;
+	int s;
 
+	s = splsoftnet();
 	IP6Q_LOCK();
 	TAILQ_FOREACH_SAFE(q6, &frag6_queue, ip6q_queue, nq6)
 		if (--q6->ip6q_ttl == 0) {
@@ -617,17 +617,6 @@ frag6_slowtimo(void)
 		frag6_freef(TAILQ_LAST(&frag6_queue, ip6q_head));
 	}
 	IP6Q_UNLOCK();
-
-	/*
-	 * Routing changes might produce a better route than we last used;
-	 * make sure we notice eventually, even if forwarding only for one
-	 * destination and the cache is never replaced.
-	 */
-	if (ip6_forward_rt.ro_rt) {
-		rtfree(ip6_forward_rt.ro_rt);
-		ip6_forward_rt.ro_rt = NULL;
-	}
-
 	splx(s);
 }
 
diff --git a/sys/netinet6/ip6_forward.c b/sys/netinet6/ip6_forward.c
index 809d379e22b..09e7ac29dbe 100644
--- a/sys/netinet6/ip6_forward.c
+++ b/sys/netinet6/ip6_forward.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: ip6_forward.c,v 1.91 2016/06/15 11:49:34 mpi Exp $	*/
+/*	$OpenBSD: ip6_forward.c,v 1.92 2016/08/24 09:41:12 mpi Exp $	*/
 /*	$KAME: ip6_forward.c,v 1.75 2001/06/29 12:42:13 jinmei Exp $	*/
 
 /*
@@ -68,8 +68,6 @@
 #include <netinet/tcp.h>
 #endif
 
-struct	route_in6 ip6_forward_rt;
-
 /*
  * Forward a packet.  If some error occurs return the sender
  * an icmp packet.  Note we can't always generate a meaningful
@@ -84,18 +82,16 @@ struct	route_in6 ip6_forward_rt;
  */
 
 void
-ip6_forward(struct mbuf *m, int srcrt)
+ip6_forward(struct mbuf *m, struct rtentry *rt, int srcrt)
 {
 	struct ip6_hdr *ip6 = mtod(m, struct ip6_hdr *);
-	struct sockaddr_in6 *dst;
-	struct rtentry *rt;
+	struct sockaddr_in6 *dst, sin6;
 	struct ifnet *ifp = NULL;
 	int error = 0, type = 0, code = 0;
 	struct mbuf *mcopy = NULL;
 #ifdef IPSEC
 	struct tdb *tdb = NULL;
 #endif /* IPSEC */
-	u_int rtableid = 0;
 	char src6[INET6_ADDRSTRLEN], dst6[INET6_ADDRSTRLEN];
 
 	/*
@@ -120,13 +116,13 @@ ip6_forward(struct mbuf *m, int srcrt)
 			    m->m_pkthdr.ph_ifidx);
 		}
 		m_freem(m);
-		return;
+		goto out;
 	}
 
 	if (ip6->ip6_hlim <= IPV6_HLIMDEC) {
 		icmp6_error(m, ICMP6_TIME_EXCEEDED,
 				ICMP6_TIME_EXCEED_TRANSIT, 0);
-		return;
+		goto out;
 	}
 	ip6->ip6_hlim -= IPV6_HLIMDEC;
 
@@ -164,40 +160,26 @@ reroute:
 	}
 #endif /* IPSEC */
 
-#if NPF > 0
-	rtableid = m->m_pkthdr.ph_rtableid;
-#endif
-
-	dst = &ip6_forward_rt.ro_dst;
-	if (!rtisvalid(ip6_forward_rt.ro_rt) ||
-	   ISSET(ip6_forward_rt.ro_rt->rt_flags, RTF_MPATH) ||
-	   !IN6_ARE_ADDR_EQUAL(&ip6->ip6_dst, &dst->sin6_addr) ||
-	   ip6_forward_rt.ro_tableid != rtableid) {
-		if (ip6_forward_rt.ro_rt) {
-			rtfree(ip6_forward_rt.ro_rt);
-			ip6_forward_rt.ro_rt = NULL;
-		}
-		bzero(dst, sizeof(*dst));
-		dst->sin6_len = sizeof(struct sockaddr_in6);
-		dst->sin6_family = AF_INET6;
-		dst->sin6_addr = ip6->ip6_dst;
-		ip6_forward_rt.ro_tableid = rtableid;
-		ip6_forward_rt.ro_rt = rtalloc_mpath(
-		    sin6tosa(&ip6_forward_rt.ro_dst),
-		    &ip6->ip6_src.s6_addr32[0],
-		    ip6_forward_rt.ro_tableid);
-
-		if (ip6_forward_rt.ro_rt == NULL) {
+	dst = &sin6;
+	memset(dst, 0, sizeof(*dst));
+	dst->sin6_len = sizeof(struct sockaddr_in6);
+	dst->sin6_family = AF_INET6;
+	dst->sin6_addr = ip6->ip6_dst;
+
+	if (!rtisvalid(rt)) {
+		rtfree(rt);
+		rt = rtalloc_mpath(sin6tosa(dst), &ip6->ip6_src.s6_addr32[0],
+		    m->m_pkthdr.ph_rtableid);
+		if (rt == NULL) {
 			ip6stat.ip6s_noroute++;
 			if (mcopy) {
 				icmp6_error(mcopy, ICMP6_DST_UNREACH,
 					    ICMP6_DST_UNREACH_NOROUTE, 0);
 			}
 			m_freem(m);
-			return;
+			goto out;
 		}
 	}
-	rt = ip6_forward_rt.ro_rt;
 
 	/*
 	 * Scope check: if a packet can't be delivered to its destination
@@ -262,7 +244,7 @@ reroute:
 	    ip6_sendredirects &&
 	    (rt->rt_flags & (RTF_DYNAMIC|RTF_MODIFIED)) == 0) {
 		if ((ifp->if_flags & IFF_POINTOPOINT) &&
-		    nd6_is_addr_neighbor(&ip6_forward_rt.ro_dst, ifp)) {
+		    nd6_is_addr_neighbor(&sin6, ifp)) {
 			/*
 			 * If the incoming interface is equal to the outgoing
 			 * one, the link attached to the interface is
@@ -381,5 +363,6 @@ senderr:
 freecopy:
 	m_freem(mcopy);
 out:
+	rtfree(rt);
 	if_put(ifp);
 }
diff --git a/sys/netinet6/ip6_input.c b/sys/netinet6/ip6_input.c
index 0a8cca23376..9ac2555e632 100644
--- a/sys/netinet6/ip6_input.c
+++ b/sys/netinet6/ip6_input.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: ip6_input.c,v 1.167 2016/07/19 15:57:13 phessler Exp $	*/
+/*	$OpenBSD: ip6_input.c,v 1.168 2016/08/24 09:41:12 mpi Exp $	*/
 /*	$KAME: ip6_input.c,v 1.188 2001/03/29 05:34:31 itojun Exp $	*/
 
 /*
@@ -184,13 +184,13 @@ ip6intr(void)
 		ip6_input(m);
 }
 
-extern struct	route_in6 ip6_forward_rt;
-
 void
 ip6_input(struct mbuf *m)
 {
 	struct ifnet *ifp;
 	struct ip6_hdr *ip6;
+	struct sockaddr_in6 sin6;
+	struct rtentry *rt = NULL;
 	int off, nest;
 	u_int16_t src_scope, dst_scope;
 	int nxt, ours = 0;
@@ -414,43 +414,23 @@ ip6_input(struct mbuf *m)
 		goto hbhcheck;
 	}
 
+
 	/*
 	 *  Unicast check
 	 */
-	if (rtisvalid(ip6_forward_rt.ro_rt) &&
-	    !ISSET(ip6_forward_rt.ro_rt->rt_flags, RTF_MPATH) &&
-	    IN6_ARE_ADDR_EQUAL(&ip6->ip6_dst,
-			       &ip6_forward_rt.ro_dst.sin6_addr) &&
-	    m->m_pkthdr.ph_rtableid == ip6_forward_rt.ro_tableid)
-		ip6stat.ip6s_forward_cachehit++;
-	else {
-		if (ip6_forward_rt.ro_rt) {
-			/* route is down or destination is different */
-			ip6stat.ip6s_forward_cachemiss++;
-			rtfree(ip6_forward_rt.ro_rt);
-			ip6_forward_rt.ro_rt = NULL;
-		}
-
-		bzero(&ip6_forward_rt.ro_dst, sizeof(struct sockaddr_in6));
-		ip6_forward_rt.ro_dst.sin6_len = sizeof(struct sockaddr_in6);
-		ip6_forward_rt.ro_dst.sin6_family = AF_INET6;
-		ip6_forward_rt.ro_dst.sin6_addr = ip6->ip6_dst;
-		ip6_forward_rt.ro_tableid = m->m_pkthdr.ph_rtableid;
-
-		ip6_forward_rt.ro_rt = rtalloc_mpath(
-		    sin6tosa(&ip6_forward_rt.ro_dst),
-		    &ip6->ip6_src.s6_addr32[0],
-		    ip6_forward_rt.ro_tableid);
-	}
+	memset(&sin6, 0, sizeof(struct sockaddr_in6));
+	sin6.sin6_len = sizeof(struct sockaddr_in6);
+	sin6.sin6_family = AF_INET6;
+	sin6.sin6_addr = ip6->ip6_dst;
+	rt = rtalloc_mpath(sin6tosa(&sin6), &ip6->ip6_src.s6_addr32[0],
+	    m->m_pkthdr.ph_rtableid);
 
 	/*
 	 * Accept the packet if the route to the destination is marked
 	 * as local.
 	 */
-	if (rtisvalid(ip6_forward_rt.ro_rt) &&
-	    ISSET(ip6_forward_rt.ro_rt->rt_flags, RTF_LOCAL)) {
-		struct in6_ifaddr *ia6 =
-			ifatoia6(ip6_forward_rt.ro_rt->rt_ifa);
+	if (rtisvalid(rt) && ISSET(rt->rt_flags, RTF_LOCAL)) {
+		struct in6_ifaddr *ia6 = ifatoia6(rt->rt_ifa);
 		if (ia6->ia6_flags & IN6_IFF_ANYCAST)
 			m->m_flags |= M_ACAST;
 		/*
@@ -493,6 +473,7 @@ ip6_input(struct mbuf *m)
   hbhcheck:
 
 	if (ip6_hbhchcheck(m, &off, &nxt, &ours)) {
+		rtfree(rt);
 		if_put(ifp);
 		return;	/* m have already been freed */
 	}
@@ -522,7 +503,7 @@ ip6_input(struct mbuf *m)
 		if (!ours)
 			goto bad;
 	} else if (!ours) {
-		ip6_forward(m, srcrt);
+		ip6_forward(m, rt, srcrt);
 		if_put(ifp);
 		return;
 	}
@@ -566,9 +547,11 @@ ip6_input(struct mbuf *m)
 
 		nxt = (*inet6sw[ip6_protox[nxt]].pr_input)(&m, &off, nxt);
 	}
+	rtfree(rt);
 	if_put(ifp);
 	return;
  bad:
+	rtfree(rt);
 	if_put(ifp);
 	m_freem(m);
 }
diff --git a/sys/netinet6/ip6_var.h b/sys/netinet6/ip6_var.h
index 4cd10bcdc9a..0278b75bea4 100644
--- a/sys/netinet6/ip6_var.h
+++ b/sys/netinet6/ip6_var.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: ip6_var.h,v 1.63 2016/08/04 20:46:24 vgross Exp $	*/
+/*	$OpenBSD: ip6_var.h,v 1.64 2016/08/24 09:41:12 mpi Exp $	*/
 /*	$KAME: ip6_var.h,v 1.33 2000/06/11 14:59:20 jinmei Exp $	*/
 
 /*
@@ -258,7 +258,7 @@ int	ip6_process_hopopts(struct mbuf *, u_int8_t *, int, u_int32_t *,
 void	ip6_savecontrol(struct inpcb *, struct mbuf *, struct mbuf **);
 int	ip6_sysctl(int *, u_int, void *, size_t *, void *, size_t);
 
-void	ip6_forward(struct mbuf *, int);
+void	ip6_forward(struct mbuf *, struct rtentry *, int);
 
 void	ip6_mloopback(struct ifnet *, struct mbuf *, struct sockaddr_in6 *);
 int	ip6_output(struct mbuf *, struct ip6_pktopts *, struct route_in6 *, int,