Allow multiple connected/interface routes to exist at the same time.

Use the existing multipath code. Switch away from using the ifa address
when making the cloning route and instead put a dummy sockaddr_dl route
in. With this it is possible to use the same network on multiple interfaces
at the same time. So if wireless and ethernet share the same network
the system will use the wired connection as long as there is link.
Still missing is builtin proxy-arp for the other interface IPs to allow
hitless failover.
OK mpi@

6 files changed, 63 insertions, 170 deletions

diff --git a/sys/net/route.c b/sys/net/route.c
index a3d660a3613..d2f8260d27a 100644
--- a/sys/net/route.c
+++ b/sys/net/route.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: route.c,v 1.209 2015/04/20 09:12:57 mpi Exp $	*/
+/*	$OpenBSD: route.c,v 1.210 2015/05/15 12:00:57 claudio Exp $	*/
 /*	$NetBSD: route.c,v 1.14 1996/02/13 22:00:46 christos Exp $	*/
 
 /*
@@ -554,6 +554,16 @@ rtdeletemsg(struct rtentry *rt, u_int tableid)
 	return (error);
 }
 
+static inline int
+rtequal(struct rtentry *a, struct rtentry *b)
+{
+	if (memcmp(rt_key(a), rt_key(b), rt_key(a)->sa_len) == 0 &&
+	    memcmp(rt_mask(a), rt_mask(b), rt_mask(a)->sa_len) == 0)
+		return 1;
+	else
+		return 0;
+}
+
 int
 rtflushclone1(struct radix_node *rn, void *arg, u_int id)
 {
@@ -561,7 +571,8 @@ rtflushclone1(struct radix_node *rn, void *arg, u_int id)
 
 	rt = (struct rtentry *)rn;
 	parent = (struct rtentry *)arg;
-	if ((rt->rt_flags & RTF_CLONED) != 0 && rt->rt_parent == parent)
+	if ((rt->rt_flags & RTF_CLONED) != 0 && (rt->rt_parent == parent ||
+	    rtequal(rt->rt_parent, parent)))
 		rtdeletemsg(rt, id);
 	return 0;
 }
@@ -1106,16 +1117,20 @@ rt_ifa_add(struct ifaddr *ifa, int flags, struct sockaddr *dst)
 {
 	struct rtentry		*rt, *nrt = NULL;
 	struct sockaddr_rtlabel	 sa_rl;
+	struct sockaddr_dl	 sa_dl = { sizeof(sa_dl), AF_LINK };
 	struct rt_addrinfo	 info;
 	u_short			 rtableid = ifa->ifa_ifp->if_rdomain;
-	u_int8_t		 prio = RTP_CONNECTED;
+	u_int8_t		 prio = ifa->ifa_ifp->if_priority + RTP_STATIC;
 	int			 error;
 
+	sa_dl.sdl_type = ifa->ifa_ifp->if_type;
+	sa_dl.sdl_index = ifa->ifa_ifp->if_index;
+
 	memset(&info, 0, sizeof(info));
 	info.rti_ifa = ifa;
-	info.rti_flags = flags;
+	info.rti_flags = flags | RTF_MPATH;
 	info.rti_info[RTAX_DST] = dst;
-	info.rti_info[RTAX_GATEWAY] = ifa->ifa_addr;
+	info.rti_info[RTAX_GATEWAY] = (struct sockaddr *)&sa_dl;
 	info.rti_info[RTAX_LABEL] =
 	    rtlabel_id2sa(ifa->ifa_ifp->if_rtlabelid, &sa_rl);
 
@@ -1170,8 +1185,9 @@ rt_ifa_del(struct ifaddr *ifa, int flags, struct sockaddr *dst)
 	struct sockaddr		*deldst;
 	struct rt_addrinfo	 info;
 	struct sockaddr_rtlabel	 sa_rl;
+	struct sockaddr_dl	 sa_dl = { sizeof(sa_dl), AF_LINK };
 	u_short			 rtableid = ifa->ifa_ifp->if_rdomain;
-	u_int8_t		 prio = RTP_CONNECTED;
+	u_int8_t		 prio = ifa->ifa_ifp->if_priority + RTP_STATIC;
 	int			 error;
 
 #ifdef MPLS
@@ -1202,10 +1218,14 @@ rt_ifa_del(struct ifaddr *ifa, int flags, struct sockaddr *dst)
 		}
 	}
 
+	sa_dl.sdl_type = ifa->ifa_ifp->if_type;
+	sa_dl.sdl_index = ifa->ifa_ifp->if_index;
+
 	memset(&info, 0, sizeof(info));
 	info.rti_ifa = ifa;
 	info.rti_flags = flags;
 	info.rti_info[RTAX_DST] = dst;
+	info.rti_info[RTAX_GATEWAY] = (struct sockaddr *)&sa_dl;
 	info.rti_info[RTAX_LABEL] =
 	    rtlabel_id2sa(ifa->ifa_ifp->if_rtlabelid, &sa_rl);
 
@@ -1710,6 +1730,15 @@ rt_if_linkstate_change(struct radix_node *rn, void *arg, u_int id)
 			}
 		} else {
 			if (rt->rt_flags & RTF_UP) {
+				/*
+				 * Remove cloned routes (mainly arp) to
+				 * down interfaces so we have a chance to
+				 * clone a new route from a better source.
+				 */
+				if (rt->rt_flags & RTF_CLONED) {
+					rtdeletemsg(rt, id);
+					return (0);
+				}
 				/* take route down */
 				rt->rt_flags &= ~RTF_UP;
 				rn_mpath_reprio(rn, rt->rt_priority | RTP_DOWN);
diff --git a/sys/netinet/if_ether.c b/sys/netinet/if_ether.c
index b7e582e009a..be83797b409 100644
--- a/sys/netinet/if_ether.c
+++ b/sys/netinet/if_ether.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: if_ether.c,v 1.151 2015/04/22 04:12:22 jsg Exp $	*/
+/*	$OpenBSD: if_ether.c,v 1.152 2015/05/15 12:00:57 claudio Exp $	*/
 /*	$NetBSD: if_ether.c,v 1.31 1996/05/11 12:59:58 mycroft Exp $	*/
 
 /*
@@ -121,8 +121,6 @@ void	db_print_llinfo(caddr_t);
 int	db_show_radix_node(struct radix_node *, void *, u_int);
 #endif
 
-static const struct sockaddr_dl null_sdl = { sizeof(null_sdl), AF_LINK };
-
 /*
  * Timeout routine.  Age arp_tab entries periodically.
  */
@@ -190,14 +188,6 @@ arp_rtrequest(int req, struct rtentry *rt)
 		if (rt->rt_flags & RTF_CLONING ||
 		    ((rt->rt_flags & (RTF_LLINFO | RTF_LOCAL)) && !la)) {
 			/*
-			 * Case 1: This route should come from a route to iface.
-			 */
-			rt_setgate(rt, (struct sockaddr *)&null_sdl,
-			    ifp->if_rdomain);
-			gate = rt->rt_gateway;
-			SDL(gate)->sdl_type = ifp->if_type;
-			SDL(gate)->sdl_index = ifp->if_index;
-			/*
 			 * Give this route an expiration time, even though
 			 * it's a "permanent" route, so that routes cloned
 			 * from it do not need their expiration time set.
@@ -261,10 +251,6 @@ arp_rtrequest(int req, struct rtentry *rt)
 		}
 		if (ifa) {
 			rt->rt_expire = 0;
-			SDL(gate)->sdl_alen = ETHER_ADDR_LEN;
-			memcpy(LLADDR(SDL(gate)),
-			    ((struct arpcom *)ifp)->ac_enaddr, ETHER_ADDR_LEN);
-
 			/*
 			 * XXX Since lo0 is in the default rdomain we
 			 * should not (ab)use it for any route related
diff --git a/sys/netinet/in.c b/sys/netinet/in.c
index 974c61a0971..91636b9390b 100644
--- a/sys/netinet/in.c
+++ b/sys/netinet/in.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: in.c,v 1.115 2015/01/12 13:51:45 mpi Exp $	*/
+/*	$OpenBSD: in.c,v 1.116 2015/05/15 12:00:57 claudio Exp $	*/
 /*	$NetBSD: in.c,v 1.26 1996/02/13 23:41:39 christos Exp $	*/
 
 /*
@@ -93,8 +93,6 @@ int in_lifaddr_ioctl(struct socket *, u_long, caddr_t,
 	struct ifnet *);
 
 void in_purgeaddr(struct ifaddr *);
-int in_addprefix(struct in_ifaddr *);
-int in_scrubprefix(struct in_ifaddr *);
 int in_addhost(struct in_ifaddr *, struct sockaddr_in *);
 int in_scrubhost(struct in_ifaddr *, struct sockaddr_in *);
 int in_insert_prefix(struct in_ifaddr *);
@@ -590,7 +588,8 @@ in_ifscrub(struct ifnet *ifp, struct in_ifaddr *ia)
 	if (ISSET(ifp->if_flags, IFF_POINTOPOINT))
 		in_scrubhost(ia, &ia->ia_dstaddr);
 	else if (!ISSET(ifp->if_flags, IFF_LOOPBACK))
-		in_scrubprefix(ia);
+		if (ia->ia_flags & IFA_ROUTE)
+			in_remove_prefix(ia);
 }
 
 /*
@@ -669,7 +668,7 @@ in_ifinit(struct ifnet *ifp, struct in_ifaddr *ia, struct sockaddr_in *sin,
 			goto out;
 		error = in_addhost(ia, &ia->ia_dstaddr);
 	} else if (!ISSET(ifp->if_flags, IFF_LOOPBACK)) {
-		error = in_addprefix(ia);
+		error = in_insert_prefix(ia);
 	}
 
 	/*
@@ -762,125 +761,6 @@ in_remove_prefix(struct in_ifaddr *ia)
 }
 
 /*
- * add a route to prefix ("connected route" in cisco terminology).
- * does nothing if there's some interface address with the same prefix already.
- */
-int
-in_addprefix(struct in_ifaddr *ia0)
-{
-	struct ifnet *ifp;
-	struct ifaddr *ifa;
-	struct in_ifaddr *ia;
-	struct in_addr prefix, mask, p, m;
-
-	prefix = ia0->ia_addr.sin_addr;
-	mask = ia0->ia_sockmask.sin_addr;
-	prefix.s_addr &= mask.s_addr;
-
-	TAILQ_FOREACH(ifp, &ifnet, if_list) {
-		if (ifp->if_flags & (IFF_LOOPBACK|IFF_POINTOPOINT))
-			continue;
-
-		if (ifp->if_rdomain != ia0->ia_ifp->if_rdomain)
-			continue;
-
-		TAILQ_FOREACH(ifa, &ifp->if_addrlist, ifa_list) {
-			if (ifa->ifa_addr->sa_family != AF_INET)
-				continue;
-
-			ia = ifatoia(ifa);
-
-			if ((ia->ia_flags & IFA_ROUTE) == 0)
-				continue;
-
-			p = ia->ia_addr.sin_addr;
-			m = ia->ia_sockmask.sin_addr;
-			p.s_addr &= m.s_addr;
-
-			if (prefix.s_addr != p.s_addr ||
-			    mask.s_addr != m.s_addr)
-				continue;
-
-#if NCARP > 0
-			/* move to a real interface instead of carp interface */
-			if (ia->ia_ifp->if_type == IFT_CARP &&
-			    ia0->ia_ifp->if_type != IFT_CARP) {
-				in_remove_prefix(ia);
-				break;
-			}
-#endif
-			/*
-			 * If we got a matching prefix route inserted by other
-			 * interface address, we don't need to bother
-			 */
-			return (0);
-		}
-	}
-
-	/*
-	 * noone seem to have prefix route.  insert it.
-	 */
-	return in_insert_prefix(ia0);
-}
-
-/*
- * remove a route to prefix ("connected route" in cisco terminology).
- * re-installs the route by using another interface address, if there's one
- * with the same prefix (otherwise we lose the route mistakenly).
- */
-int
-in_scrubprefix(struct in_ifaddr *ia0)
-{
-	struct ifnet *ifp;
-	struct ifaddr *ifa;
-	struct in_ifaddr *ia;
-	struct in_addr prefix, mask, p, m;
-
-	if ((ia0->ia_flags & IFA_ROUTE) == 0)
-		return 0;
-
-	prefix = ia0->ia_addr.sin_addr;
-	mask = ia0->ia_sockmask.sin_addr;
-	prefix.s_addr &= mask.s_addr;
-
-	TAILQ_FOREACH(ifp, &ifnet, if_list) {
-		if (ifp->if_flags & (IFF_LOOPBACK|IFF_POINTOPOINT))
-			continue;
-
-		if (ifp->if_rdomain != ia0->ia_ifp->if_rdomain)
-			continue;
-
-		TAILQ_FOREACH(ifa, &ifp->if_addrlist, ifa_list) {
-			if (ifa->ifa_addr->sa_family != AF_INET)
-				continue;
-
-			ia = ifatoia(ifa);
-
-			if ((ia->ia_flags & IFA_ROUTE) != 0)
-				continue;
-
-			p = ia->ia_addr.sin_addr;
-			m = ia->ia_sockmask.sin_addr;
-			p.s_addr &= m.s_addr;
-
-			if (prefix.s_addr != p.s_addr ||
-			    mask.s_addr != m.s_addr)
-				continue;
-
-			/* Move IFA_ROUTE to the matching prefix route. */
-			in_remove_prefix(ia0);
-			return (in_insert_prefix(ia));
-		}
-	}
-
-	/*
-	 * noone seem to have prefix route.  remove it.
-	 */
-	in_remove_prefix(ia0);
-	return 0;
-}
-
-/*
  * Return 1 if the address is a local broadcast address.
  */
 int
diff --git a/sys/netinet6/in6.c b/sys/netinet6/in6.c
index 479220748c8..951476450be 100644
--- a/sys/netinet6/in6.c
+++ b/sys/netinet6/in6.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: in6.c,v 1.156 2015/04/20 09:07:42 mpi Exp $	*/
+/*	$OpenBSD: in6.c,v 1.157 2015/05/15 12:00:57 claudio Exp $	*/
 /*	$KAME: in6.c,v 1.372 2004/06/14 08:14:21 itojun Exp $	*/
 
 /*
@@ -78,6 +78,7 @@
 #include <sys/syslog.h>
 
 #include <net/if.h>
+#include <net/if_dl.h>
 #include <net/if_types.h>
 #include <net/route.h>
 
@@ -827,6 +828,10 @@ in6_update_ifa(struct ifnet *ifp, struct in6_aliasreq *ifra,
 
 		/* join solicited multicast addr for new host id */
 		struct sockaddr_in6 llsol;
+		struct sockaddr_dl sa_dl = { sizeof(sa_dl), AF_LINK };
+
+		sa_dl.sdl_type = ifp->if_type;
+		sa_dl.sdl_index = ifp->if_index;
 
 		bzero(&llsol, sizeof(llsol));
 		llsol.sin6_family = AF_INET6;
@@ -887,7 +892,7 @@ in6_update_ifa(struct ifnet *ifp, struct in6_aliasreq *ifra,
 
 			bzero(&info, sizeof(info));
 			info.rti_info[RTAX_DST] = sin6tosa(&mltaddr);
-			info.rti_info[RTAX_GATEWAY] = sin6tosa(&ia6->ia_addr);
+			info.rti_info[RTAX_GATEWAY] = (struct sockaddr *)&sa_dl;
 			info.rti_info[RTAX_NETMASK] = sin6tosa(&mltmask);
 			info.rti_info[RTAX_IFA] = sin6tosa(&ia6->ia_addr);
 			/* XXX: we need RTF_CLONING to fake nd6_rtrequest */
@@ -956,7 +961,7 @@ in6_update_ifa(struct ifnet *ifp, struct in6_aliasreq *ifra,
 
 			bzero(&info, sizeof(info));
 			info.rti_info[RTAX_DST] = sin6tosa(&mltaddr);
-			info.rti_info[RTAX_GATEWAY] = sin6tosa(&ia6->ia_addr);
+			info.rti_info[RTAX_GATEWAY] = (struct sockaddr *)&sa_dl;
 			info.rti_info[RTAX_NETMASK] = sin6tosa(&mltmask);
 			info.rti_info[RTAX_IFA] = sin6tosa(&ia6->ia_addr);
 			info.rti_flags = RTF_UP | RTF_CLONING;
diff --git a/sys/netinet6/nd6.c b/sys/netinet6/nd6.c
index 971a8948a1a..cea57b8c5de 100644
--- a/sys/netinet6/nd6.c
+++ b/sys/netinet6/nd6.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: nd6.c,v 1.135 2015/04/27 14:51:44 mpi Exp $	*/
+/*	$OpenBSD: nd6.c,v 1.136 2015/05/15 12:00:57 claudio Exp $	*/
 /*	$KAME: nd6.c,v 1.280 2002/06/08 19:52:07 itojun Exp $	*/
 
 /*
@@ -651,6 +651,7 @@ nd6_lookup(struct in6_addr *addr6, int create, struct ifnet *ifp,
 	}
 	if (!rt) {
 		if (create && ifp) {
+			struct sockaddr_dl sa_dl = { sizeof(sa_dl), AF_LINK };
 			struct rt_addrinfo info;
 			int e;
 
@@ -666,6 +667,9 @@ nd6_lookup(struct in6_addr *addr6, int create, struct ifnet *ifp,
 			if (ifa == NULL)
 				return (NULL);
 
+			sa_dl.sdl_type = ifp->if_type;
+			sa_dl.sdl_index = ifp->if_index;
+
 			/*
 			 * Create a new route.  RTF_LLINFO is necessary
 			 * to create a Neighbor Cache entry for the
@@ -675,7 +679,7 @@ nd6_lookup(struct in6_addr *addr6, int create, struct ifnet *ifp,
 			bzero(&info, sizeof(info));
 			info.rti_flags = RTF_UP | RTF_HOST | RTF_LLINFO;
 			info.rti_info[RTAX_DST] = sin6tosa(&sin6);
-			info.rti_info[RTAX_GATEWAY] = ifa->ifa_addr;
+			info.rti_info[RTAX_GATEWAY] = (struct sockaddr *)&sa_dl;
 			if ((e = rtrequest1(RTM_ADD, &info, RTP_CONNECTED,
 			    &rt, rtableid)) != 0) {
 #if 0
@@ -940,7 +944,6 @@ nd6_rtrequest(int req, struct rtentry *rt)
 {
 	struct sockaddr *gate = rt->rt_gateway;
 	struct llinfo_nd6 *ln = (struct llinfo_nd6 *)rt->rt_llinfo;
-	static struct sockaddr_dl null_sdl = {sizeof(null_sdl), AF_LINK};
 	struct ifnet *ifp = rt->rt_ifp;
 	struct ifaddr *ifa;
 	struct nd_defrouter *dr;
@@ -999,17 +1002,6 @@ nd6_rtrequest(int req, struct rtentry *rt)
 		 */
 		if ((rt->rt_flags & RTF_CLONING) ||
 		    ((rt->rt_flags & (RTF_LLINFO | RTF_LOCAL)) && !ln)) {
-			/*
-			 * Case 1: This route should come from a route to
-			 * interface (RTF_CLONING case) or the route should be
-			 * treated as on-link but is currently not
-			 * (RTF_LLINFO && !ln case).
-			 */
-			rt_setgate(rt, (struct sockaddr *)&null_sdl,
-			    ifp->if_rdomain);
-			gate = rt->rt_gateway;
-			SDL(gate)->sdl_type = ifp->if_type;
-			SDL(gate)->sdl_index = ifp->if_index;
 			if (ln)
 				nd6_llinfo_settimer(ln, 0);
 			if ((rt->rt_flags & RTF_CLONING) != 0)
@@ -1045,7 +1037,7 @@ nd6_rtrequest(int req, struct rtentry *rt)
 		/* FALLTHROUGH */
 	case RTM_RESOLVE:
 		if (gate->sa_family != AF_LINK ||
-		    gate->sa_len < sizeof(null_sdl)) {
+		    gate->sa_len < sizeof(struct sockaddr_dl)) {
 			log(LOG_DEBUG, "%s: bad gateway value: %s\n",
 			    __func__, ifp->if_xname);
 			break;
@@ -1127,14 +1119,9 @@ nd6_rtrequest(int req, struct rtentry *rt)
 		ifa = &in6ifa_ifpwithaddr(ifp,
 		    &satosin6(rt_key(rt))->sin6_addr)->ia_ifa;
 		if (ifa) {
-			caddr_t macp = nd6_ifptomac(ifp);
 			nd6_llinfo_settimer(ln, -1);
 			ln->ln_state = ND6_LLINFO_REACHABLE;
 			ln->ln_byhint = 0;
-			if (macp) {
-				memcpy(LLADDR(SDL(gate)), macp, ifp->if_addrlen);
-				SDL(gate)->sdl_alen = ifp->if_addrlen;
-			}
 
 			/*
 			 * XXX Since lo0 is in the default rdomain we
diff --git a/sys/netinet6/nd6_rtr.c b/sys/netinet6/nd6_rtr.c
index 8d6637fbb8e..50797c5004a 100644
--- a/sys/netinet6/nd6_rtr.c
+++ b/sys/netinet6/nd6_rtr.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: nd6_rtr.c,v 1.102 2015/04/27 14:51:44 mpi Exp $	*/
+/*	$OpenBSD: nd6_rtr.c,v 1.103 2015/05/15 12:00:57 claudio Exp $	*/
 /*	$KAME: nd6_rtr.c,v 1.97 2001/02/07 11:09:13 itojun Exp $	*/
 
 /*
@@ -45,6 +45,7 @@
 #include <sys/queue.h>
 
 #include <net/if.h>
+#include <net/if_dl.h>
 #include <net/if_var.h>
 #include <net/if_types.h>
 #include <net/route.h>
@@ -1651,6 +1652,7 @@ nd6_prefix_onlink(struct nd_prefix *pr)
 	struct ifaddr *ifa;
 	struct ifnet *ifp = pr->ndpr_ifp;
 	struct sockaddr_in6 mask6;
+	struct sockaddr_dl sa_dl = { sizeof(sa_dl), AF_LINK };
 	struct nd_prefix *opr;
 	u_long rtflags;
 	int error = 0;
@@ -1723,6 +1725,10 @@ nd6_prefix_onlink(struct nd_prefix *pr)
 	bzero(&mask6, sizeof(mask6));
 	mask6.sin6_len = sizeof(mask6);
 	mask6.sin6_addr = pr->ndpr_mask;
+
+	sa_dl.sdl_type = ifp->if_type;
+	sa_dl.sdl_index = ifp->if_index;
+
 	/* rtrequest1() will probably set RTF_UP, but we're not sure. */
 	rtflags = RTF_UP;
 	if (nd6_need_cache(ifp))
@@ -1733,7 +1739,7 @@ nd6_prefix_onlink(struct nd_prefix *pr)
 	bzero(&info, sizeof(info));
 	info.rti_flags = rtflags;
 	info.rti_info[RTAX_DST] = sin6tosa(&pr->ndpr_prefix);
-	info.rti_info[RTAX_GATEWAY] = ifa->ifa_addr;
+	info.rti_info[RTAX_GATEWAY] = (struct sockaddr *)&sa_dl;
 	info.rti_info[RTAX_NETMASK] = sin6tosa(&mask6);
 
 	error = rtrequest1(RTM_ADD, &info, RTP_CONNECTED, &rt, ifp->if_rdomain);