Make the ``rt_gwroute'' pointer of RTF_GATEWAY entries immutable.

This means that no protection is needed to guarantee that the next hop
route wont be modified by CPU1 while CPU0 is dereferencing it in a L2
resolution functions.

While here also fix an ``ifa'' leak resulting in RTF_GATEWAY being always
invalid.

dlg@ likes it, inputs and ok bluhm@

5 files changed, 243 insertions, 112 deletions

diff --git a/sys/net/route.c b/sys/net/route.c
index 1758bb3ac77..eed062d2b42 100644
--- a/sys/net/route.c
+++ b/sys/net/route.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: route.c,v 1.315 2016/08/19 07:12:54 mpi Exp $	*/
+/*	$OpenBSD: route.c,v 1.316 2016/08/22 16:01:52 mpi Exp $	*/
 /*	$NetBSD: route.c,v 1.14 1996/02/13 22:00:46 christos Exp $	*/
 
 /*
@@ -153,7 +153,9 @@ struct pool		rtentry_pool;	/* pool for rtentry structures */
 struct pool		rttimer_pool;	/* pool for rttimer structures */
 
 void	rt_timer_init(void);
-void	rt_setgwroute(struct rtentry *, u_int);
+int	rt_setgwroute(struct rtentry *, u_int);
+void	rt_putgwroute(struct rtentry *);
+int	rt_fixgwroute(struct rtentry *, void *, unsigned int);
 int	rtflushclone1(struct rtentry *, void *, u_int);
 void	rtflushclone(unsigned int, struct rtentry *);
 int	rt_if_remove_rtdelete(struct rtentry *, void *, u_int);
@@ -204,21 +206,20 @@ rtisvalid(struct rtentry *rt)
 	if (rt == NULL)
 		return (0);
 
-#ifdef DIAGNOSTIC
-	if (ISSET(rt->rt_flags, RTF_GATEWAY) && (rt->rt_gwroute != NULL) &&
-	    ISSET(rt->rt_gwroute->rt_flags, RTF_GATEWAY))
-	    	panic("next hop must be directly reachable");
-#endif
-
-	if ((rt->rt_flags & RTF_UP) == 0)
+	if (!ISSET(rt->rt_flags, RTF_UP))
 		return (0);
 
 	/* Routes attached to stale ifas should be freed. */
 	if (rt->rt_ifa == NULL || rt->rt_ifa->ifa_ifp == NULL)
 		return (0);
 
-	if (ISSET(rt->rt_flags, RTF_GATEWAY) && !rtisvalid(rt->rt_gwroute))
-		return (0);
+#ifdef DIAGNOSTIC
+	if (ISSET(rt->rt_flags, RTF_GATEWAY)) {
+		KASSERT(rt->rt_gwroute != NULL);
+	    	KASSERT(ISSET(rt->rt_gwroute->rt_flags, RTF_UP));
+	    	KASSERT(!ISSET(rt->rt_gwroute->rt_flags, RTF_GATEWAY));
+	}
+#endif /* DIAGNOSTIC */
 
 	return (1);
 }
@@ -269,8 +270,6 @@ rt_match(struct sockaddr *dst, uint32_t *src, int flags, unsigned int tableid)
 	return (rt);
 }
 
-struct rtentry *_rtalloc(struct sockaddr *, uint32_t *, int, unsigned int);
-
 #ifndef SMALL_KERNEL
 /*
  * Originated from bridge_hash() in if_bridge.c
@@ -351,16 +350,10 @@ rt_hash(struct rtentry *rt, struct sockaddr *dst, uint32_t *src)
 struct rtentry *
 rtalloc_mpath(struct sockaddr *dst, uint32_t *src, unsigned int rtableid)
 {
-	return (_rtalloc(dst, src, RT_RESOLVE, rtableid));
+	return (rt_match(dst, src, RT_RESOLVE, rtableid));
 }
 #endif /* SMALL_KERNEL */
 
-struct rtentry *
-rtalloc(struct sockaddr *dst, int flags, unsigned int rtableid)
-{
-	return (_rtalloc(dst, NULL, flags, rtableid));
-}
-
 /*
  * Look in the routing table for the best matching entry for
  * ``dst''.
@@ -369,44 +362,35 @@ rtalloc(struct sockaddr *dst, int flags, unsigned int rtableid)
  * longer valid, try to cache it.
  */
 struct rtentry *
-_rtalloc(struct sockaddr *dst, uint32_t *src, int flags, unsigned int rtableid)
+rtalloc(struct sockaddr *dst, int flags, unsigned int rtableid)
 {
-	struct rtentry *rt;
-
-	rt = rt_match(dst, src, flags, rtableid);
-
-	/* No match or route to host?  We're done. */
-	if (rt == NULL || !ISSET(rt->rt_flags, RTF_GATEWAY))
-		return (rt);
-
-	/* Nothing to do if the next hop is valid. */
-	if (rtisvalid(rt->rt_gwroute))
-		return (rt);
-
-	rt_setgwroute(rt, rtableid);
-
-	return (rt);
+	return (rt_match(dst, NULL, flags, rtableid));
 }
 
-void
+/*
+ * Cache the route entry corresponding to a reachable next hop in
+ * the gateway entry ``rt''.
+ */
+int
 rt_setgwroute(struct rtentry *rt, u_int rtableid)
 {
 	struct rtentry *nhrt;
 
-	rtfree(rt->rt_gwroute);
-	rt->rt_gwroute = NULL;
+	KERNEL_ASSERT_LOCKED();
 
-	/*
-	 * If we cannot find a valid next hop, return the route
-	 * with a gateway.
-	 *
-	 * XXX Some dragons hiding in the tree certainly depends on
-	 * this behavior.  But it is safe since rt_checkgate() wont
-	 * allow us to us this route later on.
-	 */
+	KASSERT(ISSET(rt->rt_flags, RTF_GATEWAY));
+	KASSERT(rt->rt_gwroute == NULL);
+
+	/* If we cannot find a valid next hop bail. */
 	nhrt = rt_match(rt->rt_gateway, NULL, RT_RESOLVE, rtable_l2(rtableid));
 	if (nhrt == NULL)
-		return;
+		return (ENOENT);
+
+	/* Next hop entry must be on the same interface. */
+	if (nhrt->rt_ifidx != rt->rt_ifidx) {
+		rtfree(nhrt);
+		return (EHOSTUNREACH);
+	}
 
 	/*
 	 * Next hop must be reachable, this also prevents rtentry
@@ -414,13 +398,7 @@ rt_setgwroute(struct rtentry *rt, u_int rtableid)
 	 */
 	if (ISSET(nhrt->rt_flags, RTF_CLONING|RTF_GATEWAY)) {
 		rtfree(nhrt);
-		return;
-	}
-
-	/* Next hop entry must be UP and on the same interface. */
-	if (!ISSET(nhrt->rt_flags, RTF_UP) || nhrt->rt_ifidx != rt->rt_ifidx) {
-		rtfree(nhrt);
-		return;
+		return (ELOOP);
 	}
 
 	/*
@@ -431,10 +409,76 @@ rt_setgwroute(struct rtentry *rt, u_int rtableid)
 		rt->rt_mtu = nhrt->rt_mtu;
 
 	/*
-	 * Do not return the cached next-hop route, rt_checkgate() will
-	 * do the magic for us.
+	 * To avoid reference counting problems when writting link-layer
+	 * addresses in an outgoing packet, we ensure that the lifetime
+	 * of a cached entry is greater that the bigger lifetime of the
+	 * gateway entries it is pointed by.
 	 */
+	nhrt->rt_flags |= RTF_CACHED;
+	nhrt->rt_cachecnt++;
+
 	rt->rt_gwroute = nhrt;
+
+	return (0);
+}
+
+/*
+ * Invalidate the cached route entry of the gateway entry ``rt''.
+ */
+void
+rt_putgwroute(struct rtentry *rt)
+{
+	struct rtentry *nhrt = rt->rt_gwroute;
+
+	KERNEL_ASSERT_LOCKED();
+
+	if (!ISSET(rt->rt_flags, RTF_GATEWAY) || nhrt == NULL)
+		return;
+
+	KASSERT(ISSET(nhrt->rt_flags, RTF_CACHED));
+	KASSERT(nhrt->rt_cachecnt > 0);
+
+	--nhrt->rt_cachecnt;
+	if (nhrt->rt_cachecnt == 0)
+		nhrt->rt_flags &= ~RTF_CACHED;
+
+	rtfree(rt->rt_gwroute);
+	rt->rt_gwroute = NULL;
+}
+
+/*
+ * Refresh cached entries of RTF_GATEWAY routes for a given interface.
+ *
+ * This clever logic is necessary to try to fix routes linked to stale
+ * ifas.
+ */
+int
+rt_fixgwroute(struct rtentry *rt, void *arg, unsigned int id)
+{
+	struct ifnet *ifp = arg;
+
+	KERNEL_ASSERT_LOCKED();
+
+	if (rt->rt_ifidx != ifp->if_index || !ISSET(rt->rt_flags, RTF_GATEWAY))
+		return (0);
+
+	/*
+	 * If the gateway route is not stale, its associated cached
+	 * is also not stale.
+	 */
+	if (rt->rt_ifa->ifa_ifp != NULL)
+		return (0);
+
+	/* If we can fix the cached next hop entry, we can fix the ifa. */
+	if (rt_setgate(rt, rt->rt_gateway, ifp->if_rdomain) == 0) {
+		struct ifaddr *ifa = rt->rt_gwroute->rt_ifa;
+
+		ifafree(rt->rt_ifa);
+		ifa->ifa_refcnt++;
+		rt->rt_ifa = ifa;
+	}
+
+	return (0);
 }
 
 void
@@ -891,8 +935,7 @@ rtrequest_delete(struct rt_addrinfo *info, u_int8_t prio, struct ifnet *ifp,
 	if ((rt->rt_flags & RTF_CLONING) != 0)
 		rtflushclone(tableid, rt);
 
-	rtfree(rt->rt_gwroute);
-	rt->rt_gwroute = NULL;
+	rt_putgwroute(rt);
 
 	rtfree(rt->rt_parent);
 	rt->rt_parent = NULL;
@@ -1101,7 +1144,7 @@ rtrequest(int req, struct rt_addrinfo *info, u_int8_t prio,
 		    tableid))) {
 			ifafree(ifa);
 			rtfree(rt->rt_parent);
-			rtfree(rt->rt_gwroute);
+			rt_putgwroute(rt);
 			free(rt->rt_gateway, M_RTABLE, 0);
 			free(ndst, M_RTABLE, dlen);
 			pool_put(&rtentry_pool, rt);
@@ -1131,7 +1174,7 @@ rtrequest(int req, struct rt_addrinfo *info, u_int8_t prio,
 		if (error != 0) {
 			ifafree(ifa);
 			rtfree(rt->rt_parent);
-			rtfree(rt->rt_gwroute);
+			rt_putgwroute(rt);
 			free(rt->rt_gateway, M_RTABLE, 0);
 			free(ndst, M_RTABLE, dlen);
 			pool_put(&rtentry_pool, rt);
@@ -1172,33 +1215,27 @@ rt_setgate(struct rtentry *rt, struct sockaddr *gate, u_int rtableid)
 	}
 	memmove(rt->rt_gateway, gate, glen);
 
-	if (ISSET(rt->rt_flags, RTF_GATEWAY))
-		rt_setgwroute(rt, rtableid);
+	if (ISSET(rt->rt_flags, RTF_GATEWAY)) {
+		rt_putgwroute(rt);
+		return (rt_setgwroute(rt, rtableid));
+	}
 
 	return (0);
 }
 
-int
-rt_checkgate(struct rtentry *rt, struct rtentry **rtp)
+/*
+ * Return the route entry containing the next hop link-layer
+ * address corresponding to ``rt''.
+ */
+struct rtentry *
+rt_getll(struct rtentry *rt)
 {
-	struct rtentry *rt0;
-
-	KASSERT(rt != NULL);
-
-	rt0 = rt;
-
-	if (rt->rt_flags & RTF_GATEWAY) {
-		if (rt->rt_gwroute == NULL)
-			return (EHOSTUNREACH);
-		rt = rt->rt_gwroute;
+	if (ISSET(rt->rt_flags, RTF_GATEWAY)) {
+		KASSERT(rt->rt_gwroute != NULL);
+		return (rt->rt_gwroute);
 	}
 
-	if (rt->rt_flags & RTF_REJECT)
-		if (rt->rt_expire == 0 || time_uptime < rt->rt_expire)
-			return (rt == rt0 ? EHOSTDOWN : EHOSTUNREACH);
-
-	*rtp = rt;
-	return (0);
+	return (rt);
 }
 
 void
@@ -1262,6 +1299,8 @@ rt_ifa_add(struct ifaddr *ifa, int flags, struct sockaddr *dst)
 
 	error = rtrequest(RTM_ADD, &info, prio, &rt, rtableid);
 	if (error == 0) {
+		unsigned int i;
+
 		/*
 		 * A local route is created for every address configured
 		 * on an interface, so use this information to notify
@@ -1271,6 +1310,18 @@ rt_ifa_add(struct ifaddr *ifa, int flags, struct sockaddr *dst)
 			rt_sendaddrmsg(rt, RTM_NEWADDR, ifa);
 		rt_sendmsg(rt, RTM_ADD, rtableid);
 		rtfree(rt);
+
+		/*
+		 * Userland inserted routes stay in the table even
+		 * if their corresponding ``ifa'' is no longer valid.
+		 *
+		 * Try to fix the stale RTF_GATEWAY entries in case
+		 * their gateway match the newly inserted route.
+		 */
+		for (i = 0; i <= RT_TABLEID_MAX; i++) {
+			rtable_walk(i, ifa->ifa_addr->sa_family,
+			    rt_fixgwroute, ifp);
+		}
 	}
 	return (error);
 }
@@ -1790,7 +1841,8 @@ rt_if_linkstate_change(struct rtentry *rt, void *arg, u_int id)
 			 * from down interfaces so we have a chance to get
 			 * new routes from a better source.
 			 */
-			if (ISSET(rt->rt_flags, RTF_CLONED|RTF_DYNAMIC)) {
+			if (ISSET(rt->rt_flags, RTF_CLONED|RTF_DYNAMIC) &&
+			    !ISSET(rt->rt_flags, RTF_CACHED)) {
 				int error;
 
 				if ((error = rtdeletemsg(rt, ifp, id)))
diff --git a/sys/net/route.h b/sys/net/route.h
index 0e3eaf56107..425341e9465 100644
--- a/sys/net/route.h
+++ b/sys/net/route.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: route.h,v 1.141 2016/07/13 08:40:46 mpi Exp $	*/
+/*	$OpenBSD: route.h,v 1.142 2016/08/22 16:01:52 mpi Exp $	*/
 /*	$NetBSD: route.h,v 1.9 1996/02/13 22:00:49 christos Exp $	*/
 
 /*
@@ -103,7 +103,12 @@ struct rtentry {
 	struct ifaddr	*rt_ifa;	/* the answer: interface addr to use */
 	caddr_t		 rt_llinfo;	/* pointer to link level info cache or
 					   to an MPLS structure */ 
-	struct rtentry	*rt_gwroute;	/* implied entry for gatewayed routes */
+	union {
+		struct rtentry	*_nh;	/* implied entry for gatewayed routes */
+		unsigned int	 _ref;	/* # gatewayed caching this route */
+	} RT_gw;
+#define rt_gwroute	 RT_gw._nh
+#define rt_cachecnt	 RT_gw._ref
 	struct rtentry	*rt_parent;	/* If cloned, parent of this route. */
 	LIST_HEAD(, rttimer) rt_timer;  /* queue of timeouts for misc funcs */
 	struct rt_kmetrics rt_rmx;	/* metrics used by rx'ing protocols */
@@ -139,6 +144,7 @@ struct rtentry {
 #define RTF_ANNOUNCE	RTF_PROTO2	/* announce L2 entry */
 #define RTF_PROTO1	0x8000		/* protocol specific routing flag */
 #define RTF_CLONED	0x10000		/* this is a cloned route */
+#define RTF_CACHED	0x20000		/* cached by a RTF_GATEWAY entry */
 #define RTF_MPATH	0x40000		/* multipath route or operation */
 #define RTF_MPLS	0x100000	/* MPLS additional infos */
 #define RTF_LOCAL	0x200000	/* route to a local address */
@@ -227,6 +233,7 @@ struct rt_msghdr {
 #define RTM_IFINFO	0xe	/* iface going up/down etc. */
 #define RTM_IFANNOUNCE	0xf	/* iface arrival/departure */
 #define RTM_DESYNC	0x10	/* route socket buffer overflow */
+#define RTM_INVALIDATE	0x11	/* Invalidate cache of L2 route */
 
 #define RTV_MTU		0x1	/* init or lock _mtu */
 #define RTV_HOPCOUNT	0x2	/* init or lock _hopcount */
@@ -363,7 +370,7 @@ void	 rt_sendmsg(struct rtentry *, int, u_int);
 void	 rt_sendaddrmsg(struct rtentry *, int, struct ifaddr *);
 void	 rt_missmsg(int, struct rt_addrinfo *, int, uint8_t, u_int, int, u_int);
 int	 rt_setgate(struct rtentry *, struct sockaddr *, u_int);
-int	 rt_checkgate(struct rtentry *, struct rtentry **);
+struct rtentry *rt_getll(struct rtentry *);
 void	 rt_setmetrics(u_long, const struct rt_metrics *, struct rt_kmetrics *);
 void	 rt_getmetrics(const struct rt_kmetrics *, struct rt_metrics *);
 
diff --git a/sys/net/rtsock.c b/sys/net/rtsock.c
index af1147fde81..6c79d2e9b71 100644
--- a/sys/net/rtsock.c
+++ b/sys/net/rtsock.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: rtsock.c,v 1.194 2016/07/11 13:06:31 bluhm Exp $	*/
+/*	$OpenBSD: rtsock.c,v 1.195 2016/08/22 16:01:52 mpi Exp $	*/
 /*	$NetBSD: rtsock.c,v 1.18 1996/03/29 00:32:10 cgd Exp $	*/
 
 /*
@@ -99,6 +99,7 @@ struct walkarg {
 int	route_ctloutput(int, struct socket *, int, int, struct mbuf **);
 void	route_input(struct mbuf *m0, ...);
 int	route_arp_conflict(struct rt_addrinfo *, unsigned int);
+int	route_cleargateway(struct rtentry *, void *, unsigned int);
 
 struct mbuf	*rt_msg1(int, struct rt_addrinfo *);
 int		 rt_msg2(int, int, struct rt_addrinfo *, caddr_t,
@@ -556,7 +557,7 @@ route_output(struct mbuf *m, ...)
 
 	/* make sure that kernel-only bits are not set */
 	rtm->rtm_priority &= RTP_MASK;
-	rtm->rtm_flags &= ~(RTF_DONE|RTF_CLONED);
+	rtm->rtm_flags &= ~(RTF_DONE|RTF_CLONED|RTF_CACHED);
 	rtm->rtm_fmask &= RTF_FMASK;
 
 	if (rtm->rtm_priority != 0) {
@@ -615,6 +616,33 @@ route_output(struct mbuf *m, ...)
 		}
 		break;
 	case RTM_DELETE:
+		if (!rtable_exists(tableid)) {
+			error = EAFNOSUPPORT;
+			goto flush;
+		}
+
+		rt = rtable_lookup(tableid, info.rti_info[RTAX_DST],
+		    info.rti_info[RTAX_NETMASK], info.rti_info[RTAX_GATEWAY],
+		    prio);
+
+		/*
+		 * Invalidate the cache of automagically created and
+		 * referenced L2 entries to make sure that ``rt_gwroute''
+		 * pointer stays valid for other CPUs.
+		 */
+		if ((rt != NULL) && (ISSET(rt->rt_flags, RTF_CACHED))) {
+			ifp = if_get(rt->rt_ifidx);
+			KASSERT(ifp != NULL);
+			ifp->if_rtrequest(ifp, RTM_INVALIDATE, rt);
+			if_put(ifp);
+			/* Reset the MTU of the gateway route. */
+			rtable_walk(tableid, rt_key(rt)->sa_family,
+			    route_cleargateway, rt);
+			goto report;
+		}
+		rtfree(rt);
+		rt = NULL;
+
 		error = rtrequest(RTM_DELETE, &info, prio, &rt, tableid);
 		if (error == 0)
 			goto report;
@@ -746,12 +774,6 @@ report:
 				if ((error = rt_getifa(&info, tableid)) != 0)
 					goto flush;
 				ifa = info.rti_ifa;
-			}
-			if (info.rti_info[RTAX_GATEWAY] != NULL && (error =
-			    rt_setgate(rt, info.rti_info[RTAX_GATEWAY],
-			    tableid)))
-				goto flush;
-			if (ifa) {
 				if (rt->rt_ifa != ifa) {
 					ifp = if_get(rt->rt_ifidx);
 					KASSERT(ifp != NULL);
@@ -769,6 +791,10 @@ report:
 #endif
 				}
 			}
+			if (info.rti_info[RTAX_GATEWAY] != NULL && (error =
+			    rt_setgate(rt, info.rti_info[RTAX_GATEWAY],
+			    tableid)))
+				goto flush;
 #ifdef MPLS
 			if ((rtm->rtm_flags & RTF_MPLS) &&
 			    info.rti_info[RTAX_SRC] != NULL) {
@@ -890,6 +916,18 @@ fail:
 	return (error);
 }
 
+int
+route_cleargateway(struct rtentry *rt, void *arg, unsigned int rtableid)
+{
+	struct rtentry *nhrt = arg;
+
+	if (ISSET(rt->rt_flags, RTF_GATEWAY) && rt->rt_gwroute == nhrt &&
+	    !ISSET(rt->rt_locks, RTV_MTU))
+                rt->rt_mtu = 0;
+
+	return (0);
+}
+
 /*
  * Check if the user request to insert an ARP entry does not conflict
  * with existing ones.
diff --git a/sys/netinet/if_ether.c b/sys/netinet/if_ether.c
index 1c4839d8c73..cab6f574d10 100644
--- a/sys/netinet/if_ether.c
+++ b/sys/netinet/if_ether.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: if_ether.c,v 1.220 2016/07/14 14:01:40 mpi Exp $	*/
+/*	$OpenBSD: if_ether.c,v 1.221 2016/08/22 16:01:52 mpi Exp $	*/
 /*	$NetBSD: if_ether.c,v 1.31 1996/05/11 12:59:58 mycroft Exp $	*/
 
 /*
@@ -78,6 +78,7 @@ int	arpt_prune = (5 * 60);	/* walk list every 5 minutes */
 int	arpt_keep = (20 * 60);	/* once resolved, cache for 20 minutes */
 int	arpt_down = 20;		/* once declared down, don't send for 20 secs */
 
+void arpinvalidate(struct rtentry *);
 void arptfree(struct rtentry *);
 void arptimer(void *);
 struct rtentry *arplookup(struct in_addr *, int, int, unsigned int);
@@ -215,6 +216,12 @@ arp_rtrequest(struct ifnet *ifp, int req, struct rtentry *rt)
 		rt->rt_flags &= ~RTF_LLINFO;
 		la_hold_total -= ml_purge(&la->la_ml);
 		pool_put(&arp_pool, la);
+		break;
+
+	case RTM_INVALIDATE:
+		if (!ISSET(rt->rt_flags, RTF_LOCAL))
+			arpinvalidate(rt);
+		break;
 	}
 }
 
@@ -306,7 +313,6 @@ arpresolve(struct ifnet *ifp, struct rtentry *rt0, struct mbuf *m,
 	struct sockaddr_dl *sdl;
 	struct rtentry *rt = NULL;
 	char addr[INET_ADDRSTRLEN];
-	int error;
 
 	if (m->m_flags & M_BCAST) {	/* broadcast */
 		memcpy(desten, etherbroadcastaddr, sizeof(etherbroadcastaddr));
@@ -317,10 +323,12 @@ arpresolve(struct ifnet *ifp, struct rtentry *rt0, struct mbuf *m,
 		return (0);
 	}
 
-	error = rt_checkgate(rt0, &rt);
-	if (error) {
+	rt = rt_getll(rt0);
+
+	if (ISSET(rt->rt_flags, RTF_REJECT) &&
+	    (rt->rt_expire == 0 || time_uptime < rt->rt_expire)) {
 		m_freem(m);
-		return (error);
+		return (rt == rt0 ? EHOSTDOWN : EHOSTUNREACH);
 	}
 
 	if (!ISSET(rt->rt_flags, RTF_LLINFO)) {
@@ -667,23 +675,31 @@ arpcache(struct ifnet *ifp, struct ether_arp *ea, struct rtentry *rt)
 
 	return (0);
 }
+
+void
+arpinvalidate(struct rtentry *rt)
+{
+	struct llinfo_arp *la = (struct llinfo_arp *)rt->rt_llinfo;
+	struct sockaddr_dl *sdl = satosdl(rt->rt_gateway);
+
+	la_hold_total -= ml_purge(&la->la_ml);
+	sdl->sdl_alen = 0;
+	la->la_asked = 0;
+}
+
 /*
  * Free an arp entry.
  */
 void
 arptfree(struct rtentry *rt)
 {
-	struct llinfo_arp *la = (struct llinfo_arp *)rt->rt_llinfo;
-	struct sockaddr_dl *sdl = satosdl(rt->rt_gateway);
 	struct ifnet *ifp;
 
-	ifp = if_get(rt->rt_ifidx);
-	if ((sdl != NULL) && (sdl->sdl_family == AF_LINK)) {
-		sdl->sdl_alen = 0;
-		la->la_asked = 0;
-	}
+	arpinvalidate(rt);
 
-	if (!ISSET(rt->rt_flags, RTF_STATIC))
+	ifp = if_get(rt->rt_ifidx);
+	KASSERT(ifp != NULL);
+	if (!ISSET(rt->rt_flags, RTF_STATIC|RTF_CACHED))
 		rtdeletemsg(rt, ifp, ifp->if_rdomain);
 	if_put(ifp);
 }
diff --git a/sys/netinet6/nd6.c b/sys/netinet6/nd6.c
index 66676cc879b..6faced72f2d 100644
--- a/sys/netinet6/nd6.c
+++ b/sys/netinet6/nd6.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: nd6.c,v 1.189 2016/08/22 10:33:22 mpi Exp $	*/
+/*	$OpenBSD: nd6.c,v 1.190 2016/08/22 16:01:52 mpi Exp $	*/
 /*	$KAME: nd6.c,v 1.280 2002/06/08 19:52:07 itojun Exp $	*/
 
 /*
@@ -93,6 +93,7 @@ struct nd_prhead nd_prefix = { 0 };
 int nd6_recalc_reachtm_interval = ND6_RECALC_REACHTM_INTERVAL;
 
 void nd6_slowtimo(void *);
+void nd6_invalidate(struct rtentry *);
 struct llinfo_nd6 *nd6_free(struct rtentry *, int);
 void nd6_llinfo_timer(void *);
 
@@ -711,6 +712,17 @@ nd6_is_addr_neighbor(struct sockaddr_in6 *addr, struct ifnet *ifp)
 	return (0);
 }
 
+void
+nd6_invalidate(struct rtentry *rt)
+{
+	struct llinfo_nd6 *ln = (struct llinfo_nd6 *)rt->rt_llinfo;
+
+	m_freem(ln->ln_hold);
+	ln->ln_hold = NULL;
+	ln->ln_state = ND6_LLINFO_INCOMPLETE;
+	ln->ln_asked = 0;
+}
+
 /*
  * Free an nd6 llinfo entry.
  * Since the function would cause significant changes in the kernel, DO NOT
@@ -814,7 +826,7 @@ nd6_free(struct rtentry *rt, int gc)
 	 * caches, and disable the route entry not to be used in already
 	 * cached routes.
 	 */
-	if (!ISSET(rt->rt_flags, RTF_STATIC))
+	if (!ISSET(rt->rt_flags, RTF_STATIC|RTF_CACHED))
 		rtdeletemsg(rt, ifp, ifp->if_rdomain);
 	splx(s);
 
@@ -1097,6 +1109,11 @@ nd6_rtrequest(struct ifnet *ifp, int req, struct rtentry *rt)
 		rt->rt_flags &= ~RTF_LLINFO;
 		m_freem(ln->ln_hold);
 		pool_put(&nd6_pool, ln);
+		break;
+
+	case RTM_INVALIDATE:
+		nd6_invalidate(rt);
+		break;
 	}
 }
 
@@ -1495,17 +1512,18 @@ nd6_resolve(struct ifnet *ifp, struct rtentry *rt0, struct mbuf *m,
 	struct sockaddr_dl *sdl;
 	struct rtentry *rt;
 	struct llinfo_nd6 *ln = NULL;
-	int error;
 
 	if (m->m_flags & M_MCAST) {
 		ETHER_MAP_IPV6_MULTICAST(&satosin6(dst)->sin6_addr, desten);
 		return (0);
 	}
 
-	error = rt_checkgate(rt0, &rt);
-	if (error) {
+	rt = rt_getll(rt0);
+
+	if (ISSET(rt->rt_flags, RTF_REJECT) &&
+	    (rt->rt_expire == 0 || time_uptime < rt->rt_expire)) {
 		m_freem(m);
-		return (error);
+		return (rt == rt0 ? EHOSTDOWN : EHOSTUNREACH);
 	}
 
 	/*