Initial support for routing domains. This allows to bind interfaces to

alternate routing table and separate them from other interfaces in distinct
routing tables. The same network can now be used in any doamin at the same
time without causing conflicts.
This diff is mostly mechanical and adds the necessary rdomain checks accross
net and netinet. L2 and IPv4 are mostly covered still missing pf and IPv6.
input and tested by jsg@, phessler@ and reyk@. "put it in" deraadt@

8 files changed, 188 insertions, 84 deletions

diff --git a/sys/net/if.c b/sys/net/if.c
index 061b8772190..8dd2fd9582b 100644
--- a/sys/net/if.c
+++ b/sys/net/if.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: if.c,v 1.193 2009/06/04 19:07:21 henning Exp $	*/
+/*	$OpenBSD: if.c,v 1.194 2009/06/05 00:05:21 claudio Exp $	*/
 /*	$NetBSD: if.c,v 1.35 1996/05/07 05:26:04 thorpej Exp $	*/
 
 /*
@@ -878,7 +878,7 @@ if_congestion_clear(void *arg)
  */
 /*ARGSUSED*/
 struct ifaddr *
-ifa_ifwithaddr(struct sockaddr *addr)
+ifa_ifwithaddr(struct sockaddr *addr, u_int rdomain)
 {
 	struct ifnet *ifp;
 	struct ifaddr *ifa;
@@ -887,6 +887,8 @@ ifa_ifwithaddr(struct sockaddr *addr)
 	(bcmp((caddr_t)(a1), (caddr_t)(a2),	\
 	((struct sockaddr *)(a1))->sa_len) == 0)
 	TAILQ_FOREACH(ifp, &ifnet, if_list) {
+	    if (ifp->if_rdomain != rdomain)
+		continue;
 	    TAILQ_FOREACH(ifa, &ifp->if_addrlist, ifa_list) {
 		if (ifa->ifa_addr->sa_family != addr->sa_family)
 			continue;
@@ -906,20 +908,22 @@ ifa_ifwithaddr(struct sockaddr *addr)
  */
 /*ARGSUSED*/
 struct ifaddr *
-ifa_ifwithdstaddr(struct sockaddr *addr)
+ifa_ifwithdstaddr(struct sockaddr *addr, u_int rdomain)
 {
 	struct ifnet *ifp;
 	struct ifaddr *ifa;
 
 	TAILQ_FOREACH(ifp, &ifnet, if_list) {
-	    if (ifp->if_flags & IFF_POINTOPOINT)
-		TAILQ_FOREACH(ifa, &ifp->if_addrlist, ifa_list) {
-			if (ifa->ifa_addr->sa_family != addr->sa_family ||
-			    ifa->ifa_dstaddr == NULL)
-				continue;
-			if (equal(addr, ifa->ifa_dstaddr))
-				return (ifa);
-		}
+		if (ifp->if_rdomain != rdomain)
+			continue;
+		if (ifp->if_flags & IFF_POINTOPOINT)
+			TAILQ_FOREACH(ifa, &ifp->if_addrlist, ifa_list) {
+				if (ifa->ifa_addr->sa_family !=
+				    addr->sa_family || ifa->ifa_dstaddr == NULL)
+					continue;
+				if (equal(addr, ifa->ifa_dstaddr))
+					return (ifa);
+			}
 	}
 	return (NULL);
 }
@@ -929,7 +933,7 @@ ifa_ifwithdstaddr(struct sockaddr *addr)
  * is most specific found.
  */
 struct ifaddr *
-ifa_ifwithnet(struct sockaddr *addr)
+ifa_ifwithnet(struct sockaddr *addr, u_int rdomain)
 {
 	struct ifnet *ifp;
 	struct ifaddr *ifa;
@@ -944,6 +948,8 @@ ifa_ifwithnet(struct sockaddr *addr)
 			return (ifnet_addrs[sdl->sdl_index]);
 	}
 	TAILQ_FOREACH(ifp, &ifnet, if_list) {
+		if (ifp->if_rdomain != rdomain)
+			continue;
 		TAILQ_FOREACH(ifa, &ifp->if_addrlist, ifa_list) {
 			char *cp, *cp2, *cp3;
 
@@ -972,12 +978,14 @@ ifa_ifwithnet(struct sockaddr *addr)
  * Find an interface using a specific address family
  */
 struct ifaddr *
-ifa_ifwithaf(int af)
+ifa_ifwithaf(int af, u_int rdomain)
 {
 	struct ifnet *ifp;
 	struct ifaddr *ifa;
 
 	TAILQ_FOREACH(ifp, &ifnet, if_list) {
+		if (ifp->if_rdomain != rdomain)
+			continue;
 		TAILQ_FOREACH(ifa, &ifp->if_addrlist, ifa_list) {
 			if (ifa->ifa_addr->sa_family == af)
 				return (ifa);
@@ -1202,7 +1210,7 @@ ifioctl(struct socket *so, u_long cmd, caddr_t data, struct proc *p)
 {
 	struct ifnet *ifp;
 	struct ifreq *ifr;
-	struct ifaddr *ifa;
+	struct ifaddr *ifa, *nifa;
 	struct sockaddr_dl *sdl;
 	struct ifgroupreq *ifgr;
 	char ifdescrbuf[IFDESCRSIZE];
@@ -1403,6 +1411,59 @@ ifioctl(struct socket *so, u_long cmd, caddr_t data, struct proc *p)
 		ifp->if_priority = ifr->ifr_metric;
 		break;
 
+	case SIOCGIFRTABLEID:
+		ifr->ifr_rdomainid = ifp->if_rdomain;
+		break;
+
+	case SIOCSIFRTABLEID:
+		if ((error = suser(p, 0)) != 0)
+			return (error);
+		if (ifr->ifr_rdomainid < 0 ||
+		    ifr->ifr_rdomainid > RT_TABLEID_MAX)
+			return (EINVAL);
+		/* remove all routing entries when switching domains */
+		/* XXX hell this is ugly */
+		if (ifr->ifr_rdomainid != ifp->if_rdomain) {
+			rt_if_remove(ifp);
+#ifdef INET
+			rti_delete(ifp);
+#if NETHER > 0
+			myip_ifp = NULL;
+#endif
+#ifdef MROUTING
+			vif_delete(ifp);
+#endif
+#endif
+#ifdef INET6
+			in6_ifdetach(ifp);
+#endif
+#ifdef INET
+			for (ifa = TAILQ_FIRST(&ifp->if_addrlist); ifa != NULL;
+			    ifa = nifa) {
+				nifa = TAILQ_NEXT(ifa, ifa_list);
+
+				/* only remove AF_INET */
+				if (ifa->ifa_addr->sa_family != AF_INET)
+					continue;
+
+				TAILQ_REMOVE(&in_ifaddr,
+				    (struct in_ifaddr *)ifa, ia_list);
+				TAILQ_REMOVE(&ifp->if_addrlist, ifa, ifa_list);
+				ifa->ifa_ifp = NULL;
+				IFAFREE(ifa);
+			}
+#endif
+		}
+
+		/* make sure that the routing table exists */
+		if (!rtable_exists(ifr->ifr_rdomainid)) {
+			if (rtable_add(ifr->ifr_rdomainid) == -1)
+				panic("rtinit: rtable_add");
+		}
+
+		ifp->if_rdomain = ifr->ifr_rdomainid;
+		break;
+
 	case SIOCAIFGROUP:
 		if ((error = suser(p, 0)))
 			return (error);
diff --git a/sys/net/if.h b/sys/net/if.h
index 94885445c40..65e40ef621f 100644
--- a/sys/net/if.h
+++ b/sys/net/if.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: if.h,v 1.105 2009/06/04 19:07:21 henning Exp $	*/
+/*	$OpenBSD: if.h,v 1.106 2009/06/05 00:05:21 claudio Exp $	*/
 /*	$NetBSD: if.h,v 1.23 1996/05/07 02:40:27 thorpej Exp $	*/
 
 /*
@@ -215,6 +215,7 @@ struct ifnet {				/* and the entries */
 	struct	if_data if_data;	/* stats and other data about if */
 	u_int32_t if_hardmtu;		/* maximum MTU device supports */
 	int	if_capabilities;	/* interface capabilities */
+	u_int	if_rdomain;		/* routing instance */
 	char	if_description[IFDESCRSIZE]; /* interface description */
 	u_short	if_rtlabelid;		/* next route label */
 	u_int8_t if_priority;
@@ -588,6 +589,7 @@ struct	ifreq {
 #define	ifr_metric	ifr_ifru.ifru_metric	/* metric */
 #define	ifr_mtu		ifr_ifru.ifru_metric	/* mtu (overload) */
 #define	ifr_media	ifr_ifru.ifru_metric	/* media options (overload) */
+#define	ifr_rdomainid	ifr_ifru.ifru_metric	/* VRF instance (overload) */
 #define	ifr_data	ifr_ifru.ifru_data	/* for use by interface */
 };
 
@@ -804,12 +806,12 @@ void	if_group_routechange(struct sockaddr *, struct sockaddr *);
 struct	ifnet *ifunit(const char *);
 void	if_start(struct ifnet *);
 
-struct	ifaddr *ifa_ifwithaddr(struct sockaddr *);
-struct	ifaddr *ifa_ifwithaf(int);
-struct	ifaddr *ifa_ifwithdstaddr(struct sockaddr *);
-struct	ifaddr *ifa_ifwithnet(struct sockaddr *);
+struct	ifaddr *ifa_ifwithaddr(struct sockaddr *, u_int);
+struct	ifaddr *ifa_ifwithaf(int, u_int);
+struct	ifaddr *ifa_ifwithdstaddr(struct sockaddr *, u_int);
+struct	ifaddr *ifa_ifwithnet(struct sockaddr *, u_int);
 struct	ifaddr *ifa_ifwithroute(int, struct sockaddr *,
-					struct sockaddr *);
+					struct sockaddr *, u_int);
 struct	ifaddr *ifaof_ifpforaddr(struct sockaddr *, struct ifnet *);
 void	ifafree(struct ifaddr *);
 void	link_rtrequest(int, struct rtentry *, struct rt_addrinfo *);
diff --git a/sys/net/if_ethersubr.c b/sys/net/if_ethersubr.c
index 5a6ce60cc0d..d0166e2e84a 100644
--- a/sys/net/if_ethersubr.c
+++ b/sys/net/if_ethersubr.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: if_ethersubr.c,v 1.132 2009/03/05 19:47:05 michele Exp $	*/
+/*	$OpenBSD: if_ethersubr.c,v 1.133 2009/06/05 00:05:21 claudio Exp $	*/
 /*	$NetBSD: if_ethersubr.c,v 1.19 1996/05/07 02:40:30 thorpej Exp $	*/
 
 /*
@@ -230,6 +230,15 @@ ether_output(ifp0, m0, dst, rt0)
 	short mflags;
 	struct ifnet *ifp = ifp0;
 
+#ifdef DIAGNOSTIC
+	if (ifp->if_rdomain != m->m_pkthdr.rdomain) {
+		printf("%s: trying to send packet on wrong domain. "
+		    "%d vs. %d, AF %d\n", ifp->if_xname, ifp->if_rdomain,
+		    m->m_pkthdr.rdomain, dst->sa_family);
+		senderr(ENETDOWN);
+	}
+#endif
+
 #if NTRUNK > 0
 	if (ifp->if_type == IFT_IEEE8023ADLAG)
 		senderr(EBUSY);
@@ -241,7 +250,8 @@ ether_output(ifp0, m0, dst, rt0)
 
 		/* loop back if this is going to the carp interface */
 		if (dst != NULL && LINK_STATE_IS_UP(ifp0->if_link_state) &&
-		    (ifa = ifa_ifwithaddr(dst)) != NULL &&
+		    /* XXX why ifa_ifwithaddr() and not ifaof_ifpforaddr() */
+		    (ifa = ifa_ifwithaddr(dst, ifp->if_rdomain)) != NULL &&
 		    ifa->ifa_ifp == ifp0)
 			return (looutput(ifp0, m, dst, rt0));
 
@@ -258,7 +268,8 @@ ether_output(ifp0, m0, dst, rt0)
 		senderr(ENETDOWN);
 	if ((rt = rt0) != NULL) {
 		if ((rt->rt_flags & RTF_UP) == 0) {
-			if ((rt0 = rt = rtalloc1(dst, 1, 0)) != NULL)
+			if ((rt0 = rt = rtalloc1(dst, 1,
+			    m->m_pkthdr.pf.rtableid)) != NULL)
 				rt->rt_refcnt--;
 			else
 				senderr(EHOSTUNREACH);
@@ -274,7 +285,8 @@ ether_output(ifp0, m0, dst, rt0)
 				goto lookup;
 			if (((rt = rt->rt_gwroute)->rt_flags & RTF_UP) == 0) {
 				rtfree(rt); rt = rt0;
-			lookup: rt->rt_gwroute = rtalloc1(rt->rt_gateway, 1, 0);
+			lookup: rt->rt_gwroute = rtalloc1(rt->rt_gateway, 1,
+			    ifp->if_rdomain);
 				if ((rt = rt->rt_gwroute) == 0)
 					senderr(EHOSTUNREACH);
 			}
@@ -532,6 +544,9 @@ ether_input(ifp0, eh, m)
 
 	m_cluncount(m, 1);
 
+	/* mark incomming routing domain */
+	m->m_pkthdr.rdomain = ifp->if_rdomain;
+
 	if (eh == NULL) {
 		eh = mtod(m, struct ether_header *);
 		m_adj(m, ETHER_HDR_LEN);
diff --git a/sys/net/if_fddisubr.c b/sys/net/if_fddisubr.c
index e33adfb9a0a..a43177f5089 100644
--- a/sys/net/if_fddisubr.c
+++ b/sys/net/if_fddisubr.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: if_fddisubr.c,v 1.54 2008/05/07 13:45:35 dlg Exp $	*/
+/*	$OpenBSD: if_fddisubr.c,v 1.55 2009/06/05 00:05:21 claudio Exp $	*/
 /*	$NetBSD: if_fddisubr.c,v 1.5 1996/05/07 23:20:21 christos Exp $	*/
 
 /*
@@ -154,7 +154,7 @@ fddi_output(ifp0, m0, dst, rt0)
 
 		/* loop back if this is going to the carp interface */
 		if (dst != NULL && LINK_STATE_IS_UP(ifp0->if_link_state) &&
-		    (ifa = ifa_ifwithaddr(dst)) != NULL &&
+		    (ifa = ifa_ifwithaddr(dst, ifp->if_rdomain)) != NULL &&
 		    ifa->ifa_ifp == ifp0)
 			return (looutput(ifp0, m, dst, rt0));
 
diff --git a/sys/net/pf.c b/sys/net/pf.c
index 2c68a807512..b3da5a1ca67 100644
--- a/sys/net/pf.c
+++ b/sys/net/pf.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: pf.c,v 1.648 2009/05/18 20:37:13 bluhm Exp $ */
+/*	$OpenBSD: pf.c,v 1.649 2009/06/05 00:05:21 claudio Exp $ */
 
 /*
  * Copyright (c) 2001 Daniel Hartmeier
@@ -153,8 +153,8 @@ void			 pf_send_tcp(const struct pf_rule *, sa_family_t,
 			    const struct pf_addr *, const struct pf_addr *,
 			    u_int16_t, u_int16_t, u_int32_t, u_int32_t,
 			    u_int8_t, u_int16_t, u_int16_t, u_int8_t, int,
-			    u_int16_t, struct ether_header *, struct ifnet *,
-			    int);
+			    u_int16_t, u_short, struct ether_header *,
+			    struct ifnet *, int);
 void			 pf_send_icmp(struct mbuf *, u_int8_t, u_int8_t,
 			    sa_family_t, struct pf_rule *, int);
 void			 pf_detach_state(struct pf_state *);
@@ -1093,6 +1093,10 @@ pf_src_tree_remove_state(struct pf_state *s)
 void
 pf_unlink_state(struct pf_state *cur)
 {
+	/*
+	 * XXX XXX XXX state needs to know routing domain so that states
+	 * XXX XXX XXX can not float between domain. May simplify other code.
+	 */
 	splsoftassert(IPL_SOFTNET);
 
 	if (cur->src.state == PF_TCPS_PROXY_DST) {
@@ -1103,7 +1107,7 @@ pf_unlink_state(struct pf_state *cur)
 		    cur->key[PF_SK_WIRE]->port[1],
 		    cur->key[PF_SK_WIRE]->port[0],
 		    cur->src.seqhi, cur->src.seqlo + 1,
-		    TH_RST|TH_ACK, 0, 0, 0, 1, cur->tag, NULL, NULL,
+		    TH_RST|TH_ACK, 0, 0, 0, 1, cur->tag, 0, NULL, NULL,
 		    cur->rtableid);
 	}
 	RB_REMOVE(pf_state_tree_id, &tree_id, cur);
@@ -1878,7 +1882,8 @@ pf_send_tcp(const struct pf_rule *r, sa_family_t af,
     const struct pf_addr *saddr, const struct pf_addr *daddr,
     u_int16_t sport, u_int16_t dport, u_int32_t seq, u_int32_t ack,
     u_int8_t flags, u_int16_t win, u_int16_t mss, u_int8_t ttl, int tag,
-    u_int16_t rtag, struct ether_header *eh, struct ifnet *ifp, int rtableid)
+    u_int16_t rtag, u_short rdom, struct ether_header *eh, struct ifnet *ifp,
+    int rtableid)
 {
 	struct mbuf	*m;
 	int		 len, tlen;
@@ -1916,6 +1921,7 @@ pf_send_tcp(const struct pf_rule *r, sa_family_t af,
 	if (tag)
 		m->m_pkthdr.pf.flags |= PF_TAG_GENERATED;
 	m->m_pkthdr.pf.tag = rtag;
+	m->m_pkthdr.rdomain = rdom;
 
 	if (rtableid >= 0)
 		m->m_pkthdr.pf.rtableid = rtableid;
@@ -2040,6 +2046,7 @@ pf_send_icmp(struct mbuf *m, u_int8_t type, u_int8_t code, sa_family_t af,
 		return;
 
 	m0->m_pkthdr.pf.flags |= PF_TAG_GENERATED;
+	m0->m_pkthdr.rdomain = m->m_pkthdr.rdomain;
 
 	if (rtableid >= 0)
 		m0->m_pkthdr.pf.rtableid = rtableid;
@@ -2402,10 +2409,11 @@ pf_socket_lookup(int direction, struct pf_pdesc *pd)
 	switch (pd->af) {
 #ifdef INET
 	case AF_INET:
-		inp = in_pcbhashlookup(tb, saddr->v4, sport, daddr->v4, dport);
+		inp = in_pcbhashlookup(tb, saddr->v4, sport, daddr->v4, dport,
+		    /* XXX */ 0);
 		if (inp == NULL) {
 			inp = in_pcblookup_listen(tb, daddr->v4, dport, 0,
-			    NULL);
+			    NULL, 0);
 			if (inp == NULL)
 				return (-1);
 		}
@@ -2526,6 +2534,7 @@ pf_calc_mss(struct pf_addr *addr, sa_family_t af, u_int16_t offer)
 	int			 hlen;
 	u_int16_t		 mss = tcp_mssdflt;
 
+	/* XXX needs to know about routing domain or actually rtableid */
 	switch (af) {
 #ifdef INET
 	case AF_INET:
@@ -3018,8 +3027,8 @@ pf_test_rule(struct pf_rule **rm, struct pf_state **sm, int direction,
 				pf_send_tcp(r, af, pd->dst,
 				    pd->src, th->th_dport, th->th_sport,
 				    ntohl(th->th_ack), ack, TH_RST|TH_ACK, 0, 0,
-				    r->return_ttl, 1, 0, pd->eh, kif->pfik_ifp,
-				    act.rtableid);
+				    r->return_ttl, 1, 0, m->m_pkthdr.rdomain,
+				    pd->eh, kif->pfik_ifp, act.rtableid);
 			}
 		} else if (pd->proto != IPPROTO_ICMP && af == AF_INET &&
 		    r->return_icmp)
@@ -3280,8 +3289,7 @@ pf_create_state(struct pf_rule *r, struct pf_rule *nr, struct pf_rule *a,
 		s->src.mss = mss;
 		pf_send_tcp(r, pd->af, pd->dst, pd->src, th->th_dport,
 		    th->th_sport, s->src.seqhi, ntohl(th->th_seq) + 1,
-		    TH_SYN|TH_ACK, 0, s->src.mss, 0, 1, 0, NULL, NULL,
-		    act->rtableid);
+		    TH_SYN|TH_ACK, 0, s->src.mss, 0, 1, 0, /* XXX */ 0, NULL, NULL, act->rtableid);
 		REASON_SET(&reason, PFRES_SYNPROXY);
 		return (PF_SYNPROXY_DROP);
 	}
@@ -3694,7 +3702,8 @@ pf_tcp_track_full(struct pf_state_peer *src, struct pf_state_peer *dst,
 				    th->th_sport, ntohl(th->th_ack), 0,
 				    TH_RST, 0, 0,
 				    (*state)->rule.ptr->return_ttl, 1, 0,
-				    pd->eh, kif->pfik_ifp, (*state)->rtableid);
+				    m->m_pkthdr.rdomain, pd->eh,
+				    kif->pfik_ifp, (*state)->rtableid);
 			src->seqlo = 0;
 			src->seqhi = 1;
 			src->max_win = 1;
@@ -3846,7 +3855,7 @@ pf_test_state_tcp(struct pf_state **state, int direction, struct pfi_kif *kif,
 			    pd->src, th->th_dport, th->th_sport,
 			    (*state)->src.seqhi, ntohl(th->th_seq) + 1,
 			    TH_SYN|TH_ACK, 0, (*state)->src.mss, 0, 1,
-			    0, NULL, NULL, (*state)->rtableid);
+			    0, /* XXX */ 0, NULL, NULL, (*state)->rtableid);
 			REASON_SET(reason, PFRES_SYNPROXY);
 			return (PF_SYNPROXY_DROP);
 		} else if (!(th->th_flags & TH_ACK) ||
@@ -3876,8 +3885,8 @@ pf_test_state_tcp(struct pf_state **state, int direction, struct pfi_kif *kif,
 			    &sk->addr[pd->sidx], &sk->addr[pd->didx],
 			    sk->port[pd->sidx], sk->port[pd->didx],
 			    (*state)->dst.seqhi, 0, TH_SYN, 0,
-			    (*state)->src.mss, 0, 0, (*state)->tag, NULL, NULL,
-			    (*state)->rtableid);
+			    (*state)->src.mss, 0, 0, (*state)->tag, /* XXX */ 0,
+			    NULL, NULL, (*state)->rtableid);
 			REASON_SET(reason, PFRES_SYNPROXY);
 			return (PF_SYNPROXY_DROP);
 		} else if (((th->th_flags & (TH_SYN|TH_ACK)) !=
@@ -3892,13 +3901,14 @@ pf_test_state_tcp(struct pf_state **state, int direction, struct pfi_kif *kif,
 			    pd->src, th->th_dport, th->th_sport,
 			    ntohl(th->th_ack), ntohl(th->th_seq) + 1,
 			    TH_ACK, (*state)->src.max_win, 0, 0, 0,
-			    (*state)->tag, NULL, NULL, (*state)->rtableid);
+			    (*state)->tag, /* XXX */ 0, NULL, NULL,
+			    (*state)->rtableid);
 			pf_send_tcp((*state)->rule.ptr, pd->af,
 			    &sk->addr[pd->sidx], &sk->addr[pd->didx],
 			    sk->port[pd->sidx], sk->port[pd->didx],
 			    (*state)->src.seqhi + 1, (*state)->src.seqlo + 1,
 			    TH_ACK, (*state)->dst.max_win, 0, 0, 1,
-			    0, NULL, NULL, (*state)->rtableid);
+			    0, /* XXX */ 0, NULL, NULL, (*state)->rtableid);
 			(*state)->src.seqdiff = (*state)->dst.seqhi -
 			    (*state)->src.seqlo;
 			(*state)->dst.seqdiff = (*state)->src.seqhi -
@@ -4827,6 +4837,7 @@ pf_routable(struct pf_addr *addr, sa_family_t af, struct pfi_kif *kif)
 	struct rtentry		*rt;
 	struct ifnet		*ifp;
 
+	/* XXX needs to know about routing domain or actually rtableid */
 	check_mpath = 0;
 	bzero(&ro, sizeof(ro));
 	switch (af) {
@@ -4908,6 +4919,7 @@ pf_rtlabel_match(struct pf_addr *addr, sa_family_t af, struct pf_addr_wrap *aw)
 #endif
 	int			 ret = 0;
 
+	/* XXX needs to know about routing domain or actually rtableid */
 	bzero(&ro, sizeof(ro));
 	switch (af) {
 	case AF_INET:
diff --git a/sys/net/route.c b/sys/net/route.c
index c09effaf253..d2acd43bd47 100644
--- a/sys/net/route.c
+++ b/sys/net/route.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: route.c,v 1.108 2009/05/31 04:07:03 claudio Exp $	*/
+/*	$OpenBSD: route.c,v 1.109 2009/06/05 00:05:22 claudio Exp $	*/
 /*	$NetBSD: route.c,v 1.14 1996/02/13 22:00:46 christos Exp $	*/
 
 /*
@@ -251,7 +251,7 @@ rtable_exists(u_int id)	/* verify table with that ID exists */
 	if (id > rtbl_id_max)
 		return (0);
 
-	if (rt_tables[id] == NULL)	/* should not happen */
+	if (rt_tables[id] == NULL)
 		return (0);
 
 	return (1);
@@ -450,7 +450,7 @@ ifafree(struct ifaddr *ifa)
 void
 rtredirect(struct sockaddr *dst, struct sockaddr *gateway,
     struct sockaddr *netmask, int flags, struct sockaddr *src,
-    struct rtentry **rtp)
+    struct rtentry **rtp, u_int rdomain)
 {
 	struct rtentry		*rt;
 	int			 error = 0;
@@ -462,12 +462,12 @@ rtredirect(struct sockaddr *dst, struct sockaddr *gateway,
 	splsoftassert(IPL_SOFTNET);
 
 	/* verify the gateway is directly reachable */
-	if ((ifa = ifa_ifwithnet(gateway)) == NULL) {
+	if ((ifa = ifa_ifwithnet(gateway, rdomain)) == NULL) {
 		error = ENETUNREACH;
 		goto out;
 	}
 	ifp = ifa->ifa_ifp;
-	rt = rtalloc1(dst, 0, 0);
+	rt = rtalloc1(dst, 0, rdomain);
 	/*
 	 * If the redirect isn't from our current router for this dst,
 	 * it's either old or wrong.  If it redirects us to ourselves,
@@ -480,7 +480,7 @@ rtredirect(struct sockaddr *dst, struct sockaddr *gateway,
 	if (!(flags & RTF_DONE) && rt &&
 	     (!equal(src, rt->rt_gateway) || rt->rt_ifa != ifa))
 		error = EINVAL;
-	else if (ifa_ifwithaddr(gateway) != NULL)
+	else if (ifa_ifwithaddr(gateway, rdomain) != NULL)
 		error = EHOSTUNREACH;
 	if (error)
 		goto done;
@@ -513,7 +513,8 @@ create:
 			info.rti_ifa = ifa;
 			info.rti_flags = flags;
 			rt = NULL;
-			error = rtrequest1(RTM_ADD, &info, RTP_DEFAULT, &rt, 0);
+			error = rtrequest1(RTM_ADD, &info, RTP_DEFAULT, &rt,
+			    rdomain);
 			if (rt != NULL)
 				flags = rt->rt_flags;
 			stat = &rtstat.rts_dynamic;
@@ -525,7 +526,7 @@ create:
 			rt->rt_flags |= RTF_MODIFIED;
 			flags |= RTF_MODIFIED;
 			stat = &rtstat.rts_newgateway;
-			rt_setgate(rt, rt_key(rt), gateway, 0);
+			rt_setgate(rt, rt_key(rt), gateway, rdomain);
 		}
 	} else
 		error = EHOSTUNREACH;
@@ -546,7 +547,7 @@ out:
 	info.rti_info[RTAX_GATEWAY] = gateway;
 	info.rti_info[RTAX_NETMASK] = netmask;
 	info.rti_info[RTAX_AUTHOR] = src;
-	rt_missmsg(RTM_REDIRECT, &info, flags, ifp, error, 0);
+	rt_missmsg(RTM_REDIRECT, &info, flags, ifp, error, rdomain);
 }
 
 /*
@@ -614,7 +615,8 @@ rtioctl(u_long req, caddr_t data, struct proc *p)
 }
 
 struct ifaddr *
-ifa_ifwithroute(int flags, struct sockaddr *dst, struct sockaddr *gateway)
+ifa_ifwithroute(int flags, struct sockaddr *dst, struct sockaddr *gateway,
+    u_int rdomain)
 {
 	struct ifaddr	*ifa;
 
@@ -639,21 +641,21 @@ ifa_ifwithroute(int flags, struct sockaddr *dst, struct sockaddr *gateway)
 		 */
 		ifa = NULL;
 		if (flags & RTF_HOST)
-			ifa = ifa_ifwithdstaddr(dst);
+			ifa = ifa_ifwithdstaddr(dst, rdomain);
 		if (ifa == NULL)
-			ifa = ifa_ifwithaddr(gateway);
+			ifa = ifa_ifwithaddr(gateway, rdomain);
 	} else {
 		/*
 		 * If we are adding a route to a remote net
 		 * or host, the gateway may still be on the
 		 * other end of a pt to pt link.
 		 */
-		ifa = ifa_ifwithdstaddr(gateway);
+		ifa = ifa_ifwithdstaddr(gateway, rdomain);
 	}
 	if (ifa == NULL)
-		ifa = ifa_ifwithnet(gateway);
+		ifa = ifa_ifwithnet(gateway, rdomain);
 	if (ifa == NULL) {
-		struct rtentry	*rt = rtalloc1(gateway, 0, 0);
+		struct rtentry	*rt = rtalloc1(gateway, 0, rdomain);
 		if (rt == NULL)
 			return (NULL);
 		rt->rt_refcnt--;
@@ -676,7 +678,7 @@ ifa_ifwithroute(int flags, struct sockaddr *dst, struct sockaddr *gateway)
 #define ROUNDUP(a) (a>0 ? (1 + (((a) - 1) | (sizeof(long) - 1))) : sizeof(long))
 
 int
-rt_getifa(struct rt_addrinfo *info)
+rt_getifa(struct rt_addrinfo *info, u_int rdom)
 {
 	struct ifaddr	*ifa;
 	int		 error = 0;
@@ -687,12 +689,12 @@ rt_getifa(struct rt_addrinfo *info)
 	 */
 	if (info->rti_ifp == NULL && info->rti_info[RTAX_IFP] != NULL
 	    && info->rti_info[RTAX_IFP]->sa_family == AF_LINK &&
-	    (ifa = ifa_ifwithnet((struct sockaddr *)info->rti_info[RTAX_IFP]))
-	    != NULL)
+	    (ifa = ifa_ifwithnet((struct sockaddr *)info->rti_info[RTAX_IFP],
+	    rdom)) != NULL)
 		info->rti_ifp = ifa->ifa_ifp;
 
 	if (info->rti_ifa == NULL && info->rti_info[RTAX_IFA] != NULL)
-		info->rti_ifa = ifa_ifwithaddr(info->rti_info[RTAX_IFA]);
+		info->rti_ifa = ifa_ifwithaddr(info->rti_info[RTAX_IFA], rdom);
 
 	if (info->rti_ifa == NULL) {
 		struct sockaddr	*sa;
@@ -707,10 +709,11 @@ rt_getifa(struct rt_addrinfo *info)
 		    info->rti_info[RTAX_GATEWAY] != NULL)
 			info->rti_ifa = ifa_ifwithroute(info->rti_flags,
 			    info->rti_info[RTAX_DST],
-			    info->rti_info[RTAX_GATEWAY]);
+			    info->rti_info[RTAX_GATEWAY],
+			    rdom);
 		else if (sa != NULL)
 			info->rti_ifa = ifa_ifwithroute(info->rti_flags,
-			    sa, sa);
+			    sa, sa, rdom);
 	}
 	if ((ifa = info->rti_ifa) != NULL) {
 		if (info->rti_ifp == NULL)
@@ -820,7 +823,8 @@ rtrequest1(int req, struct rt_addrinfo *info, u_int8_t prio,
 		goto makeroute;
 
 	case RTM_ADD:
-		if (info->rti_ifa == 0 && (error = rt_getifa(info)))
+		if (info->rti_ifa == 0 && (error = rt_getifa(info,
+		    /* XXX wrong because only rdomains allowed */ tableid)))
 			senderr(error);
 		ifa = info->rti_ifa;
 makeroute:
@@ -1057,6 +1061,7 @@ rtinit(struct ifaddr *ifa, int cmd, int flags)
 	int			 error;
 	struct rt_addrinfo	 info;
 	struct sockaddr_rtlabel	 sa_rl;
+	u_short			 rtableid = ifa->ifa_ifp->if_rdomain;
 
 	dst = flags & RTF_HOST ? ifa->ifa_dstaddr : ifa->ifa_addr;
 	if (cmd == RTM_DELETE) {
@@ -1068,7 +1073,7 @@ rtinit(struct ifaddr *ifa, int cmd, int flags)
 			rt_maskedcopy(dst, deldst, ifa->ifa_netmask);
 			dst = deldst;
 		}
-		if ((rt = rtalloc1(dst, 0, 0)) != NULL) {
+		if ((rt = rtalloc1(dst, 0, rtableid)) != NULL) {
 			rt->rt_refcnt--;
 			if (rt->rt_ifa != ifa) {
 				if (m != NULL)
@@ -1094,7 +1099,7 @@ rtinit(struct ifaddr *ifa, int cmd, int flags)
 	 * change it to meet bsdi4 behavior.
 	 */
 	info.rti_info[RTAX_NETMASK] = ifa->ifa_netmask;
-	error = rtrequest1(cmd, &info, RTP_CONNECTED, &nrt, 0);
+	error = rtrequest1(cmd, &info, RTP_CONNECTED, &nrt, rtableid);
 	if (cmd == RTM_DELETE && error == 0 && (rt = nrt) != NULL) {
 		rt_newaddrmsg(cmd, ifa, error, nrt);
 		if (rt->rt_refcnt <= 0) {
@@ -1289,6 +1294,8 @@ rt_timer_add(struct rtentry *rt, void (*func)(struct rtentry *,
 struct radix_node_head *
 rt_gettable(sa_family_t af, u_int id)
 {
+	if (id > rtbl_id_max)
+		return (NULL);
 	return (rt_tables[id] ? rt_tables[id][af2rtafidx[af]] : NULL);
 }
 
@@ -1431,13 +1438,17 @@ void
 rt_if_remove(struct ifnet *ifp)
 {
 	int			 i;
+	u_int			 tid;
 	struct radix_node_head	*rnh;
 
-	for (i = 1; i <= AF_MAX; i++)
-		if ((rnh = rt_gettable(i, 0)) != NULL)
-			while ((*rnh->rnh_walktree)(rnh,
-			    rt_if_remove_rtdelete, ifp) == EAGAIN)
-				;	/* nothing */
+	for (tid = 0; tid <= rtbl_id_max; tid++) {
+		for (i = 1; i <= AF_MAX; i++) {
+			if ((rnh = rt_gettable(i, tid)) != NULL)
+				while ((*rnh->rnh_walktree)(rnh,
+				    rt_if_remove_rtdelete, ifp) == EAGAIN)
+					;	/* nothing */
+		}
+	}
 }
 
 /*
@@ -1455,7 +1466,7 @@ rt_if_remove_rtdelete(struct radix_node *rn, void *vifp)
 	if (rt->rt_ifp == ifp) {
 		int	cloning = (rt->rt_flags & RTF_CLONING);
 
-		if (rtdeletemsg(rt, 0) == 0 && cloning)
+		if (rtdeletemsg(rt, ifp->if_rdomain /* XXX wrong */) == 0 && cloning)
 			return (EAGAIN);
 	}
 
diff --git a/sys/net/route.h b/sys/net/route.h
index 728ce54a6d6..d79c3d93883 100644
--- a/sys/net/route.h
+++ b/sys/net/route.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: route.h,v 1.62 2009/05/26 08:29:44 reyk Exp $	*/
+/*	$OpenBSD: route.h,v 1.63 2009/06/05 00:05:22 claudio Exp $	*/
 /*	$NetBSD: route.h,v 1.9 1996/02/13 22:00:49 christos Exp $	*/
 
 /*
@@ -351,8 +351,6 @@ struct sockaddr_rtlabel {
 	char		sr_label[RTLABEL_LEN];
 };
 
-#define	RT_TABLEID_MAX	255
-
 #ifdef _KERNEL
 const char	*rtlabel_id2name(u_int16_t);
 u_int16_t	 rtlabel_name2id(char *);
@@ -416,12 +414,12 @@ void	 rtalloc_noclone(struct route *, int);
 struct rtentry *
 	 rtalloc2(struct sockaddr *, int, int);
 void	 rtfree(struct rtentry *);
-int	 rt_getifa(struct rt_addrinfo *);
+int	 rt_getifa(struct rt_addrinfo *, u_int);
 int	 rtinit(struct ifaddr *, int, int);
 int	 rtioctl(u_long, caddr_t, struct proc *);
 void	 rtredirect(struct sockaddr *, struct sockaddr *,
 			 struct sockaddr *, int, struct sockaddr *,
-			 struct rtentry **);
+			 struct rtentry **, u_int);
 int	 rtrequest1(int, struct rt_addrinfo *, u_int8_t, struct rtentry **,
 	     u_int);
 void	 rt_if_remove(struct ifnet *);
diff --git a/sys/net/rtsock.c b/sys/net/rtsock.c
index 536debd3bb4..f1615869a99 100644
--- a/sys/net/rtsock.c
+++ b/sys/net/rtsock.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: rtsock.c,v 1.87 2009/05/31 18:00:54 claudio Exp $	*/
+/*	$OpenBSD: rtsock.c,v 1.88 2009/06/05 00:05:22 claudio Exp $	*/
 /*	$NetBSD: rtsock.c,v 1.18 1996/03/29 00:32:10 cgd Exp $	*/
 
 /*
@@ -620,19 +620,22 @@ report:
 			 * flags may also be different; ifp may be specified
 			 * by ll sockaddr when protocol address is ambiguous
 			 */
-			if ((error = rt_getifa(&info)) != 0)
+			if ((error = rt_getifa(&info,
+			    /* XXX wrong, only rdomain */ tableid)) != 0)
 				goto flush;
 			if (gate && rt_setgate(rt, rt_key(rt), gate, tableid)) {
 				error = EDQUOT;
 				goto flush;
 			}
-			if (ifpaddr && (ifa = ifa_ifwithnet(ifpaddr)) &&
+			if (ifpaddr && (ifa = ifa_ifwithnet(ifpaddr,
+			    /* XXX again rtable vs. rdomain */ tableid)) &&
 			    (ifp = ifa->ifa_ifp) && (ifaaddr || gate))
 				ifa = ifaof_ifpforaddr(ifaaddr ? ifaaddr : gate,
-							ifp);
-			else if ((ifaaddr && (ifa = ifa_ifwithaddr(ifaaddr))) ||
+				    ifp);
+			else if ((ifaaddr && (ifa = ifa_ifwithaddr(ifaaddr,
+			    /* XXX one more time */ tableid))) ||
 			    (gate && (ifa = ifa_ifwithroute(rt->rt_flags,
-			    rt_key(rt), gate))))
+			    rt_key(rt), gate, /* XXX again */ tableid))))
 				ifp = ifa->ifa_ifp;
 			if (ifa) {
 				struct ifaddr *oifa = rt->rt_ifa;
@@ -1062,6 +1065,7 @@ rt_newaddrmsg(int cmd, struct ifaddr *ifa, int error, struct rtentry *rt)
 			ifam->ifam_metric = ifa->ifa_metric;
 			ifam->ifam_flags = ifa->ifa_flags;
 			ifam->ifam_addrs = info.rti_addrs;
+			ifam->ifam_tableid = ifp->if_rdomain;
 		}
 		if ((cmd == RTM_ADD && pass == 2) ||
 		    (cmd == RTM_DELETE && pass == 1)) {
@@ -1079,6 +1083,7 @@ rt_newaddrmsg(int cmd, struct ifaddr *ifa, int error, struct rtentry *rt)
 			rtm->rtm_flags |= rt->rt_flags;
 			rtm->rtm_errno = error;
 			rtm->rtm_addrs = info.rti_addrs;
+			rtm->rtm_tableid = ifp->if_rdomain;
 		}
 		if (sa == NULL)
 			route_proto.sp_protocol = 0;