Initialize the routing table before domains.

The routing table is not an optional component of the network stack
and initializing it inside the "routing domain" requires some ugly
introspection in the domain interface.

This put the rtable* layer at the same level of the if* level.  These
two subsystem are organized around the two global data structure used
in the network stack:

- the global &ifnet list, to be used in process context only, and
- the routing table which can be read in interrupt context.

This change makes the rtable_* layer domain-aware and extends the
"struct domain" such that INET, INET6 and MPLS can specify the length
of the binary key used in lookups.  This allows us to keep, or move
towards, AF-free route and rtable layers.

While here stop the madness and pass the size of the maximum key length
in *byte* to rn_inithead0().

ok claudio@, mikeb@

15 files changed, 221 insertions, 236 deletions

diff --git a/sys/kern/init_main.c b/sys/kern/init_main.c
index e2505ea549d..3a919105aa9 100644
--- a/sys/kern/init_main.c
+++ b/sys/kern/init_main.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: init_main.c,v 1.244 2015/08/30 10:39:16 mpi Exp $	*/
+/*	$OpenBSD: init_main.c,v 1.245 2015/10/07 10:50:35 mpi Exp $	*/
 /*	$NetBSD: init_main.c,v 1.84.4.1 1996/06/02 09:08:06 mrg Exp $	*/
 
 /*
@@ -86,6 +86,7 @@
 #include <ufs/ufs/quota.h>
 
 #include <net/if.h>
+#include <net/rtable.h>
 #include <net/netisr.h>
 
 #if defined(CRYPTO)
@@ -390,7 +391,9 @@ main(void *framep)
 	crypto_init();
 	swcr_init();
 #endif /* CRYPTO */
-	
+
+	rtable_init();
+
 	/*
 	 * Initialize protocols.  Block reception of incoming packets
 	 * until everything is ready.
diff --git a/sys/net/art.c b/sys/net/art.c
index 5efaf81312b..74f13a0b39e 100644
--- a/sys/net/art.c
+++ b/sys/net/art.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: art.c,v 1.3 2015/08/20 12:51:10 mpi Exp $ */
+/*	$OpenBSD: art.c,v 1.4 2015/10/07 10:50:35 mpi Exp $ */
 
 /*
  * Copyright (c) 2015 Martin Pieuchot
@@ -78,21 +78,19 @@ int			 art_table_walk(struct art_table *,
 /*
  * Per routing table initialization API function.
  */
-int
-art_attach(void **head, int off)
+struct art_root *
+art_attach(unsigned int rtableid, int off)
 {
 	struct art_root		*ar;
 	int			 i;
 
-	if (*head)
-		return (1);
 	ar = malloc(sizeof(*ar), M_RTABLE, M_NOWAIT|M_ZERO);
 	if (ar == NULL)
-		return (0);
+		return (NULL);
 
 	/* XXX using the offset is a hack. */
 	switch (off) {
-	case 32: /* AF_INET && AF_MPLS */
+	case 4: /* AF_INET && AF_MPLS */
 		ar->ar_alen = 32;
 		ar->ar_nlvl = 3;
 		ar->ar_bits[0] = 16;
@@ -100,7 +98,7 @@ art_attach(void **head, int off)
 		ar->ar_bits[2] = 8;
 		break;
 #ifdef INET6
-	case 64: /* AF_INET6 */
+	case 8: /* AF_INET6 */
 		ar->ar_alen = 128;
 		ar->ar_nlvl = 16;
 		for (i = 0; i < ar->ar_nlvl; i++)
@@ -110,19 +108,19 @@ art_attach(void **head, int off)
 	default:
 		printf("%s: unknown offset %d\n", __func__, off);
 		free(ar, M_RTABLE, sizeof(*ar));
-		return (0);
+		return (NULL);
 	}
 
-	ar->ar_off = off / 8;
+	ar->ar_off = off;
 
 	ar->ar_root = art_table_get(ar, NULL, -1);
 	if (ar->ar_root == NULL) {
 		free(ar, M_RTABLE, sizeof(*ar));
-		return (0);
+		return (NULL);
 	}
+	ar->ar_rtableid = rtableid;
 
-	*head = ar;
-	return (1);
+	return (ar);
 }
 
 /*
diff --git a/sys/net/art.h b/sys/net/art.h
index 473de44cea7..0c97e96ef84 100644
--- a/sys/net/art.h
+++ b/sys/net/art.h
@@ -1,4 +1,4 @@
-/* $OpenBSD: art.h,v 1.2 2015/08/20 12:51:10 mpi Exp $ */
+/* $OpenBSD: art.h,v 1.3 2015/10/07 10:50:35 mpi Exp $ */
 
 /*
  * Copyright (c) 2015 Martin Pieuchot
@@ -50,7 +50,7 @@ struct art_node {
 };
 
 void		 art_init(void);
-int		 art_attach(void **, int);
+struct art_root	*art_attach(unsigned int, int);
 struct art_node *art_insert(struct art_root *, struct art_node *, uint8_t *,
 		     int);
 struct art_node *art_delete(struct art_root *, struct art_node *, uint8_t *,
diff --git a/sys/net/pfkey.c b/sys/net/pfkey.c
index 7428983e595..3e9b1dd9f8f 100644
--- a/sys/net/pfkey.c
+++ b/sys/net/pfkey.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: pfkey.c,v 1.27 2015/09/04 08:43:39 mpi Exp $	*/
+/*	$OpenBSD: pfkey.c,v 1.28 2015/10/07 10:50:35 mpi Exp $	*/
 
 /*
  *	@(#)COPYRIGHT	1.1 (NRL) 17 January 1995
@@ -269,8 +269,6 @@ struct domain pfkeydomain = {
 	NULL, /* dispose */
 	NULL, /* protosw */
 	NULL, /* protoswNPROTOSW */
-	NULL, /* dom_rtattach */
-	16 /* rtoffset */
 };
 
 static struct protosw pfkey_protosw_template = {
diff --git a/sys/net/pipex.c b/sys/net/pipex.c
index 2ae22f1ce29..b67d62415ba 100644
--- a/sys/net/pipex.c
+++ b/sys/net/pipex.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: pipex.c,v 1.82 2015/10/05 06:51:50 yasuoka Exp $	*/
+/*	$OpenBSD: pipex.c,v 1.83 2015/10/07 10:50:35 mpi Exp $	*/
 
 /*-
  * Copyright (c) 2009 Internet Initiative Japan Inc.
@@ -161,13 +161,13 @@ pipex_iface_init(struct pipex_iface_context *pipex_iface, struct ifnet *ifp)
 	if (!pipex_rd_head4_initialized) {
 		pipex_rd_head4_initialized++;
 		if (!rn_inithead0(&pipex_rd_head4,
-		    offsetof(struct sockaddr_in, sin_addr) * NBBY))
+		    offsetof(struct sockaddr_in, sin_addr)))
 			panic("rn_inithead0() failed on pipex_init()");
 	}
 	if (!pipex_rd_head6_initialized) {
 		pipex_rd_head6_initialized++;
 		if (!rn_inithead0(&pipex_rd_head6,
-		    offsetof(struct sockaddr_in6, sin6_addr) *NBBY))
+		    offsetof(struct sockaddr_in6, sin6_addr)))
 			panic("rn_inithead0() failed on pipex_init()");
 	}
 	splx(s);
diff --git a/sys/net/radix.c b/sys/net/radix.c
index 24fa00d3885..cffbc4a340c 100644
--- a/sys/net/radix.c
+++ b/sys/net/radix.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: radix.c,v 1.48 2015/09/04 08:43:39 mpi Exp $	*/
+/*	$OpenBSD: radix.c,v 1.49 2015/10/07 10:50:35 mpi Exp $	*/
 /*	$NetBSD: radix.c,v 1.20 2003/08/07 16:32:56 agc Exp $	*/
 
 /*
@@ -1185,9 +1185,10 @@ rn_inithead(void **head, int off)
 }
 
 int
-rn_inithead0(struct radix_node_head *rnh, int off)
+rn_inithead0(struct radix_node_head *rnh, int offset)
 {
 	struct radix_node *t, *tt, *ttt;
+	int off = offset * NBBY;
 
 	memset(rnh, 0, sizeof(*rnh));
 	t = rn_newpair(rn_zeros, off, rnh->rnh_nodes);
diff --git a/sys/net/route.c b/sys/net/route.c
index ab115ae3644..c993196e623 100644
--- a/sys/net/route.c
+++ b/sys/net/route.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: route.c,v 1.248 2015/10/07 08:58:01 mpi Exp $	*/
+/*	$OpenBSD: route.c,v 1.249 2015/10/07 10:50:35 mpi Exp $	*/
 /*	$NetBSD: route.c,v 1.14 1996/02/13 22:00:46 christos Exp $	*/
 
 /*
@@ -137,22 +137,18 @@
 #endif
 
 /* Give some jitter to hash, to avoid synchronization between routers. */
-static uint32_t		   rt_hashjitter;
+static uint32_t		rt_hashjitter;
 
-struct	rtstat		   rtstat;
-void			***rt_tables;
-u_int8_t		   af2rtafidx[AF_MAX+1];
-u_int8_t		   rtafidx_max;
-u_int			   rtbl_id_max = 0;
-u_int			  *rt_tab2dom;	/* rt table to domain lookup table */
+extern void	     ***rtables;
+extern unsigned int	rtables_id_max;
 
+struct rtstat		rtstat;
 int			rttrash;	/* routes not in table but not freed */
 
 struct pool		rtentry_pool;	/* pool for rtentry structures */
 struct pool		rttimer_pool;	/* pool for rttimer structures */
 
 void	rt_timer_init(void);
-int	rtable_alloc(void ***, u_int);
 int	rtflushclone1(struct rtentry *, void *, u_int);
 void	rtflushclone(unsigned int, struct rtentry *);
 int	rt_if_remove_rtdelete(struct rtentry *, void *, u_int);
@@ -177,65 +173,12 @@ struct rt_label {
 
 TAILQ_HEAD(rt_labels, rt_label)	rt_labels = TAILQ_HEAD_INITIALIZER(rt_labels);
 
-int
-rtable_alloc(void ***table, u_int id)
-{
-	void		**p;
-	struct domain	 *dom;
-	int		  i;
-
-	if ((p = mallocarray(rtafidx_max + 1, sizeof(void *), M_RTABLE,
-	    M_NOWAIT|M_ZERO)) == NULL)
-		return (ENOMEM);
-
-	/* 2nd pass: attach */
-	for (i = 0; (dom = domains[i]) != NULL; i++) {
-		if (dom->dom_rtattach)
-			dom->dom_rtattach(&p[af2rtafidx[dom->dom_family]],
-			    dom->dom_rtoffset);
-	}
-
-	for (i = 0; i < rtafidx_max; i++)
-		rtable_setid(p, id, i);
-
-	*table = (void **)p;
-
-	return (0);
-}
-
 void
 route_init(void)
 {
-	struct domain	*dom;
-	unsigned int	 keylen;
-	int		 i;
-
 	pool_init(&rtentry_pool, sizeof(struct rtentry), 0, 0, 0, "rtentry",
 	    NULL);
 
-	/*
-	 * Compute the maximum supported key length in case the routing
-	 * table backend needs it.
-	 */
-	keylen = sizeof(struct sockaddr_in);
-#ifdef INET6
-	keylen = max(keylen, (sizeof(struct sockaddr_in6)));
-#endif
-#ifdef MPLS
-	keylen = max(keylen, (sizeof(struct sockaddr_mpls)));
-#endif
-
-	rtable_init(keylen);
-
-	bzero(af2rtafidx, sizeof(af2rtafidx));
-	rtafidx_max = 1;	/* must have NULL at index 0, so start at 1 */
-
-	/* find out how many tables to allocate */
-	for (i = 0; (dom = domains[i]) != NULL; i++) {
-		if (dom->dom_rtattach)
-			af2rtafidx[dom->dom_family] = rtafidx_max++;
-	}
-
 	while (rt_hashjitter == 0)
 		rt_hashjitter = arc4random();
 
@@ -243,89 +186,6 @@ route_init(void)
 		panic("route_init rtable_add");
 }
 
-int
-rtable_add(u_int id)
-{
-	void	*p, *q;
-
-	splsoftassert(IPL_SOFTNET);
-
-	if (id > RT_TABLEID_MAX)
-		return (EINVAL);
-
-	if (id == 0 || id > rtbl_id_max) {
-		size_t	newlen;
-		size_t	newlen2;
-
-		if ((p = mallocarray(id + 1, sizeof(void *), M_RTABLE,
-		    M_NOWAIT|M_ZERO)) == NULL)
-			return (ENOMEM);
-		newlen = sizeof(void *) * (id+1);
-		if ((q = mallocarray(id + 1, sizeof(u_int), M_RTABLE,
-		    M_NOWAIT|M_ZERO)) == NULL) {
-			free(p, M_RTABLE, newlen);
-			return (ENOMEM);
-		}
-		newlen2 = sizeof(u_int) * (id+1);
-		if (rt_tables) {
-			bcopy(rt_tables, p, sizeof(void *) * (rtbl_id_max+1));
-			bcopy(rt_tab2dom, q, sizeof(u_int) * (rtbl_id_max+1));
-			free(rt_tables, M_RTABLE, 0);
-			free(rt_tab2dom, M_RTABLE, 0);
-		}
-		rt_tables = p;
-		rt_tab2dom = q;
-		rtbl_id_max = id;
-	}
-
-	if (rt_tables[id] != NULL)	/* already exists */
-		return (EEXIST);
-
-	rt_tab2dom[id] = 0;	/* use main table/domain by default */
-	return (rtable_alloc(&rt_tables[id], id));
-}
-
-void *
-rtable_get(u_int id, sa_family_t af)
-{
-	if (id > rtbl_id_max)
-		return (NULL);
-	return (rt_tables[id] ? rt_tables[id][af2rtafidx[af]] : NULL);
-}
-
-u_int
-rtable_l2(u_int id)
-{
-	if (id > rtbl_id_max)
-		return (0);
-	return (rt_tab2dom[id]);
-}
-
-void
-rtable_l2set(u_int id, u_int parent)
-{
-	splsoftassert(IPL_SOFTNET);
-
-	if (!rtable_exists(id) || !rtable_exists(parent))
-		return;
-	rt_tab2dom[id] = parent;
-}
-
-int
-rtable_exists(u_int id)	/* verify table with that ID exists */
-{
-	if (id > RT_TABLEID_MAX)
-		return (0);
-
-	if (id > rtbl_id_max)
-		return (0);
-
-	if (rt_tables[id] == NULL)
-		return (0);
-
-	return (1);
-}
-
 /*
  * Returns 1 if the (cached) ``rt'' entry is still valid, 0 otherwise.
  */
@@ -1751,7 +1611,7 @@ rt_if_remove(struct ifnet *ifp)
 	int			 i;
 	u_int			 tid;
 
-	for (tid = 0; tid <= rtbl_id_max; tid++) {
+	for (tid = 0; tid <= rtables_id_max; tid++) {
 		/* skip rtables that are not in the rdomain of the ifp */
 		if (rtable_l2(tid) != ifp->if_rdomain)
 			continue;
@@ -1796,10 +1656,10 @@ rt_if_track(struct ifnet *ifp)
 	int i;
 	u_int tid;
 
-	if (rt_tables == NULL)
+	if (rtables == NULL)
 		return;
 
-	for (tid = 0; tid <= rtbl_id_max; tid++) {
+	for (tid = 0; tid <= rtables_id_max; tid++) {
 		/* skip rtables that are not in the rdomain of the ifp */
 		if (rtable_l2(tid) != ifp->if_rdomain)
 			continue;
diff --git a/sys/net/route.h b/sys/net/route.h
index f1adf5c2ac7..1b698fb97f1 100644
--- a/sys/net/route.h
+++ b/sys/net/route.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: route.h,v 1.114 2015/10/03 18:57:11 renato Exp $	*/
+/*	$OpenBSD: route.h,v 1.115 2015/10/07 10:50:35 mpi Exp $	*/
 /*	$NetBSD: route.h,v 1.9 1996/02/13 22:00:49 christos Exp $	*/
 
 /*
@@ -346,10 +346,6 @@ extern const struct sockaddr_rtin rt_defmask4;
 struct	mbuf;
 struct	socket;
 void	 route_init(void);
-int	 rtable_add(u_int);
-u_int	 rtable_l2(u_int);
-void	 rtable_l2set(u_int, u_int);
-int	 rtable_exists(u_int);
 
 int	 route_output(struct mbuf *, ...);
 int	 route_usrreq(struct socket *, int, struct mbuf *,
diff --git a/sys/net/rtable.c b/sys/net/rtable.c
index 0b335c35cb0..80e7b9edca3 100644
--- a/sys/net/rtable.c
+++ b/sys/net/rtable.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: rtable.c,v 1.9 2015/10/07 08:43:36 mpi Exp $ */
+/*	$OpenBSD: rtable.c,v 1.10 2015/10/07 10:50:35 mpi Exp $ */
 
 /*
  * Copyright (c) 2014-2015 Martin Pieuchot
@@ -22,31 +22,172 @@
 #include <sys/malloc.h>
 #include <sys/pool.h>
 #include <sys/queue.h>
+#include <sys/domain.h>
 
 #include <net/rtable.h>
 #include <net/route.h>
 
-#ifndef ART
+uint8_t		   af2idx[AF_MAX+1];	/* To only allocate supported AF */
+uint8_t		   af2idx_max = 1;	/* Must have NULL at index 0 */
+
+void		***rtables;		/* Array of routing tables */
+unsigned int	   rtables_id_max = 0;
+unsigned int	  *rtables2dom;		/* rtable to domain lookup table */
+
+void		   rtable_init_backend(unsigned int);
+int		   rtable_attach(unsigned int, sa_family_t, int);
+void		   rtable_set(unsigned int, sa_family_t, void *);
+
+void
+rtable_init(void)
+{
+	struct domain	*dp;
+	unsigned int	 keylen = 0;
+	int		 i;
+
+	memset(af2idx, 0, sizeof(af2idx));
+
+	/*
+	 * Compute the maximum supported key length in case the routing
+	 * table backend needs it.
+	 */
+	for (i = 0; (dp = domains[i]) != NULL; i++) {
+		if (dp->dom_rtoffset)
+			af2idx[dp->dom_family] = af2idx_max++;
+		if (dp->dom_rtkeylen > keylen)
+			keylen = dp->dom_rtkeylen;
+	}
+
+	rtable_init_backend(keylen);
+}
+
+int
+rtable_add(unsigned int id)
+{
+	struct domain	*dp;
+	void		*p, *q;
+	int		 i, rv = 0;
+
+	if (id > RT_TABLEID_MAX)
+		return (EINVAL);
+
+	if (id == 0 || id > rtables_id_max) {
+		if ((p = mallocarray(id + 1, sizeof(void *), M_RTABLE,
+		    M_NOWAIT|M_ZERO)) == NULL)
+			return (ENOMEM);
+
+		if ((q = mallocarray(id + 1, sizeof(unsigned int), M_RTABLE,
+		    M_NOWAIT|M_ZERO)) == NULL) {
+			free(p, M_RTABLE, (id + 1) * sizeof(void *));
+			return (ENOMEM);
+		}
+		if (rtables) {
+			memcpy(p, rtables, (rtables_id_max+1) * sizeof(void *));
+			free(rtables, M_RTABLE,
+			    (rtables_id_max+1) * sizeof(void *));
+
+			memcpy(q, rtables2dom,
+			    (rtables_id_max+1) * sizeof(unsigned int));
+			free(rtables2dom, M_RTABLE,
+			    (rtables_id_max+1) * sizeof(unsigned int));
+		}
+		rtables = p;
+		rtables2dom = q;
+		rtables_id_max = id;
+	}
+
+	if (rtables[id] != NULL)	/* already exists */
+		return (EEXIST);
+
+	rtables2dom[id] = 0;	/* use main table/domain by default */
+	rtables[id] = mallocarray(af2idx_max + 1, sizeof(void *), M_RTABLE,
+	    M_NOWAIT|M_ZERO);
+	if (rtables[id] == NULL)
+		return (ENOMEM);
+
+	/* Per domain initialization. */
+	for (i = 0; (dp = domains[i]) != NULL; i++) {
+		if (dp->dom_rtoffset == 0)
+			continue;
+		rv |= rtable_attach(id, dp->dom_family, dp->dom_rtoffset);
+	}
+
+	return (rv);
+}
+void *
+rtable_get(unsigned int rtableid, sa_family_t af)
+{
+	if (rtableid > rtables_id_max)
+		return (NULL);
+	return (rtables[rtableid] ? rtables[rtableid][af2idx[af]] : NULL);
+}
+
+void
+rtable_set(unsigned int rtableid, sa_family_t af, void *p)
+{
+	if (rtableid > rtables_id_max)
+		return;
+
+	if (rtables[rtableid])
+		rtables[rtableid][af2idx[af]] = p;
+}
+
+int
+rtable_exists(unsigned int rtableid)
+{
+	if (rtableid > rtables_id_max)
+		return (0);
+
+	if (rtables[rtableid] == NULL)
+		return (0);
+
+	return (1);
+}
+
+unsigned int
+rtable_l2(unsigned int rtableid)
+{
+	if (rtableid > rtables_id_max)
+		return (0);
+
+	return (rtables2dom[rtableid]);
+}
+
+void
+rtable_l2set(unsigned int rtableid, unsigned int parent)
+{
+	if (!rtable_exists(rtableid) || !rtable_exists(parent))
+		return;
 
+	rtables2dom[rtableid] = parent;
+}
+
+#ifndef ART
 void
-rtable_init(unsigned int keylen)
+rtable_init_backend(unsigned int keylen)
 {
 	rn_init(keylen); /* initialize all zeroes, all ones, mask table */
 }
 
 int
-rtable_attach(void **head, int off)
+rtable_attach(unsigned int rtableid, sa_family_t af, int off)
 {
-	int rv;
+	struct radix_node_head *rnh;
+	int rv = 1;
 
-	rv = rn_inithead(head, off);
+	rnh = rtable_get(rtableid, af);
+	if (rnh != NULL)
+		return (EEXIST);
 
+	if (rn_inithead((void **)&rnh, off)) {
 #ifndef SMALL_KERNEL
-	if (rv == 1) {
-		struct radix_node_head *rnh = (struct radix_node_head *)*head;
 		rnh->rnh_multipath = 1;
-	}
 #endif /* SMALL_KERNEL */
+		rnh->rnh_rtableid = rtableid;
+		rv = 0;
+	}
+
+	rtable_set(rtableid, af, rnh);
 
 	return (rv);
 }
@@ -137,24 +278,11 @@ rtable_delete(unsigned int rtableid, struct sockaddr *dst,
 }
 
 int
-rtable_setid(void **p, unsigned int rtableid, sa_family_t af)
-{
-	struct radix_node_head **rnh = (struct radix_node_head **)p;
-
-	if (rnh == NULL || rnh[af] == NULL)
-		return (EINVAL);
-
-	rnh[af]->rnh_rtableid = rtableid;
-
-	return (0);
-}
-
-int
 rtable_walk(unsigned int rtableid, sa_family_t af,
     int (*func)(struct rtentry *, void *, unsigned int), void *arg)
 {
 	struct radix_node_head	*rnh;
-	int (*f)(struct radix_node *, void *, u_int) = (void *)func;
+	int (*f)(struct radix_node *, void *, unsigned int) = (void *)func;
 
 	rnh = rtable_get(rtableid, af);
 	if (rnh == NULL)
@@ -259,15 +387,27 @@ static inline int	 satoplen(struct art_root *, struct sockaddr *);
 static inline uint8_t	*satoaddr(struct art_root *, struct sockaddr *);
 
 void
-rtable_init(unsigned int keylen)
+rtable_init_backend(unsigned int keylen)
 {
 	pool_init(&an_pool, sizeof(struct art_node), 0, 0, 0, "art node", NULL);
 }
 
 int
-rtable_attach(void **head, int off)
+rtable_attach(unsigned int rtableid, sa_family_t af, int off)
 {
-	return (art_attach(head, off));
+	struct art_root			*ar;
+
+	ar = rtable_get(rtableid, af);
+	if (ar != NULL)
+		return (EEXIST);
+
+	ar = art_attach(rtableid, off);
+	if (ar == NULL)
+		return (ENOMEM);
+
+	rtable_set(rtableid, af, ar);
+
+	return (0);
 }
 
 struct rtentry *
@@ -488,19 +628,6 @@ rtable_delete(unsigned int rtableid, struct sockaddr *dst,
 	return (0);
 }
 
-int
-rtable_setid(void **p, unsigned int rtableid, sa_family_t af)
-{
-	struct art_root			**ar = (struct art_root **)p;
-
-	if (ar == NULL || ar[af] == NULL)
-		return (EINVAL);
-
-	ar[af]->ar_rtableid = rtableid;
-
-	return (0);
-}
-
 struct rtable_walk_cookie {
 	int		(*rwc_func)(struct rtentry *, void *, unsigned int);
 	void		 *rwc_arg;
diff --git a/sys/net/rtable.h b/sys/net/rtable.h
index f1f40eb86f8..802ff820890 100644
--- a/sys/net/rtable.h
+++ b/sys/net/rtable.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: rtable.h,v 1.4 2015/09/28 08:36:24 mpi Exp $ */
+/*	$OpenBSD: rtable.h,v 1.5 2015/10/07 10:50:35 mpi Exp $ */
 
 /*
  * Copyright (c) 2014-2015 Martin Pieuchot
@@ -49,8 +49,7 @@
 
 #endif /* ART */
 
-void		 rtable_init(unsigned int);
-int		 rtable_attach(void **, int);
+void		 rtable_init(void);
 struct rtentry	*rtable_lookup(unsigned int, struct sockaddr *,
 		     struct sockaddr *);
 struct rtentry	*rtable_match(unsigned int, struct sockaddr *);
@@ -59,8 +58,11 @@ int		 rtable_insert(unsigned int, struct sockaddr *,
 int		 rtable_delete(unsigned int, struct sockaddr *,
 		     struct sockaddr *, uint8_t, struct rtentry *);
 
-int		 rtable_setid(void **, unsigned int, sa_family_t);
+int		 rtable_exists(unsigned int);
+int		 rtable_add(unsigned int);
 void		*rtable_get(unsigned int, sa_family_t);
+unsigned int	 rtable_l2(unsigned int);
+void		 rtable_l2set(unsigned int, unsigned int);
 int		 rtable_walk(unsigned int, sa_family_t,
 		     int (*)(struct rtentry *, void *, unsigned int), void *);
 
diff --git a/sys/netinet/in_proto.c b/sys/netinet/in_proto.c
index 8196ffd0d4a..aea062d16f7 100644
--- a/sys/netinet/in_proto.c
+++ b/sys/netinet/in_proto.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: in_proto.c,v 1.67 2015/09/28 08:32:05 mpi Exp $	*/
+/*	$OpenBSD: in_proto.c,v 1.68 2015/10/07 10:50:35 mpi Exp $	*/
 /*	$NetBSD: in_proto.c,v 1.14 1996/02/18 18:58:32 christos Exp $	*/
 
 /*
@@ -310,5 +310,5 @@ struct protosw inetsw[] = {
 struct domain inetdomain =
     { AF_INET, "internet", 0, 0, 0,
       inetsw, &inetsw[nitems(inetsw)],
-      rtable_attach,
-      32 };
+      sizeof(struct sockaddr_in),
+      offsetof(struct sockaddr_in, sin_addr) };
diff --git a/sys/netinet/ip_spd.c b/sys/netinet/ip_spd.c
index 39226c874d6..60ac0b2a332 100644
--- a/sys/netinet/ip_spd.c
+++ b/sys/netinet/ip_spd.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: ip_spd.c,v 1.87 2015/09/10 17:52:05 claudio Exp $ */
+/* $OpenBSD: ip_spd.c,v 1.88 2015/10/07 10:50:35 mpi Exp $ */
 /*
  * The author of this code is Angelos D. Keromytis (angelos@cis.upenn.edu)
  *
@@ -83,7 +83,6 @@ spd_table_get(unsigned int rtableid)
 struct radix_node_head *
 spd_table_add(unsigned int rtableid)
 {
-	extern struct domain pfkeydomain;
 	struct radix_node_head *rnh = NULL;
 	unsigned int rdomain;
 	void *p;
@@ -103,7 +102,8 @@ spd_table_add(unsigned int rtableid)
 	}
 
 	if (spd_tables[rdomain] == NULL) {
-		if (rn_inithead((void **)&rnh, pfkeydomain.dom_rtoffset) == 0)
+		if (rn_inithead((void **)&rnh,
+		    offsetof(struct sockaddr_encap, sen_type)) == 0)
 			rnh = NULL;
 		spd_tables[rdomain] = rnh;
 	}
diff --git a/sys/netinet6/in6_proto.c b/sys/netinet6/in6_proto.c
index ecf704ffdd3..2e02e30b21c 100644
--- a/sys/netinet6/in6_proto.c
+++ b/sys/netinet6/in6_proto.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: in6_proto.c,v 1.81 2015/09/28 08:32:05 mpi Exp $	*/
+/*	$OpenBSD: in6_proto.c,v 1.82 2015/10/07 10:50:35 mpi Exp $	*/
 /*	$KAME: in6_proto.c,v 1.66 2000/10/10 15:35:47 itojun Exp $	*/
 
 /*
@@ -246,8 +246,8 @@ struct domain inet6domain =
     { AF_INET6, "internet6", 0, 0, 0,
       (struct protosw *)inet6sw,
       (struct protosw *)&inet6sw[nitems(inet6sw)],
-      rtable_attach,
-      offsetof(struct sockaddr_in6, sin6_addr) << 3,
+      sizeof(struct sockaddr_in6),
+      offsetof(struct sockaddr_in6, sin6_addr),
       in6_domifattach, in6_domifdetach, };
 
 /*
diff --git a/sys/netmpls/mpls_proto.c b/sys/netmpls/mpls_proto.c
index 1bba66cf7d4..c485c56381a 100644
--- a/sys/netmpls/mpls_proto.c
+++ b/sys/netmpls/mpls_proto.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: mpls_proto.c,v 1.12 2015/09/04 08:43:39 mpi Exp $	*/
+/*	$OpenBSD: mpls_proto.c,v 1.13 2015/10/07 10:50:35 mpi Exp $	*/
 
 /*
  * Copyright (C) 1999, 2000 and 2001 AYAME Project, WIDE Project.
@@ -68,6 +68,6 @@ struct domain mplsdomain = {
 	AF_MPLS, "mpls", mpls_init, 0, 0,
 	mplssw,
 	&mplssw[nitems(mplssw)],
-	rtable_attach,
-	offsetof(struct sockaddr_mpls, smpls_label) << 3
+	sizeof(struct sockaddr_mpls),
+	offsetof(struct sockaddr_mpls, smpls_label)
 };
diff --git a/sys/sys/domain.h b/sys/sys/domain.h
index 84f1463ab8a..843cc0a93be 100644
--- a/sys/sys/domain.h
+++ b/sys/sys/domain.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: domain.h,v 1.15 2015/09/04 08:43:39 mpi Exp $	*/
+/*	$OpenBSD: domain.h,v 1.16 2015/10/07 10:50:35 mpi Exp $	*/
 /*	$NetBSD: domain.h,v 1.10 1996/02/09 18:25:07 christos Exp $	*/
 
 /*
@@ -57,8 +57,8 @@ struct	domain {
 	void	(*dom_dispose)(struct mbuf *);
 	struct	protosw *dom_protosw, *dom_protoswNPROTOSW;
 					/* initialize routing table */
-	int	(*dom_rtattach)(void **, int);
-	int	dom_rtoffset;		/* an arg to rtattach, in bits */
+	unsigned int	dom_rtkeylen;	/* maximum size of the key */
+	unsigned int	dom_rtoffset;	/* offset of the key, in bytes */
 	void	*(*dom_ifattach)(struct ifnet *);
 	void	(*dom_ifdetach)(struct ifnet *, void *);
 					/* af-dependent data on ifnet */