add new 'router' functionality to dynamically add or remove routes

based on health check results, using the existing table syntax.  this
allows to maintain multiple (uplink) gateways to implement link
balancing or WAN link failover if no routing protocol or other
keepalive method is available.  works fine with or without
net.inet.ip.multipath enabled.

ok pyr@, jmc@ for manpages

7 files changed, 571 insertions, 19 deletions

diff --git a/usr.sbin/relayd/Makefile b/usr.sbin/relayd/Makefile
index 3015ba6dc37..77d2b0a4c86 100644
--- a/usr.sbin/relayd/Makefile
+++ b/usr.sbin/relayd/Makefile
@@ -1,10 +1,10 @@
-#	$OpenBSD: Makefile,v 1.18 2008/07/09 17:16:51 reyk Exp $
+#	$OpenBSD: Makefile,v 1.19 2009/08/13 13:51:21 reyk Exp $
 
 PROG=		relayd
 SRCS=		parse.y log.c control.c buffer.c imsg.c ssl.c ssl_privsep.c \
-		relayd.c pfe.c pfe_filter.c hce.c relay.c relay_udp.c \
-		carp.c check_icmp.c check_tcp.c check_script.c name2id.c \
-		snmp.c shuffle.c
+		relayd.c pfe.c pfe_filter.c pfe_route.c hce.c relay.c \
+		relay_udp.c carp.c check_icmp.c check_tcp.c check_script.c \
+		name2id.c snmp.c shuffle.c
 MAN=		relayd.8 relayd.conf.5
 
 LDADD=		-levent -lssl -lcrypto
diff --git a/usr.sbin/relayd/parse.y b/usr.sbin/relayd/parse.y
index 0d2beb9a694..31dcdba1c90 100644
--- a/usr.sbin/relayd/parse.y
+++ b/usr.sbin/relayd/parse.y
@@ -1,4 +1,4 @@
-/*	$OpenBSD: parse.y,v 1.140 2009/08/07 11:10:23 reyk Exp $	*/
+/*	$OpenBSD: parse.y,v 1.141 2009/08/13 13:51:21 reyk Exp $	*/
 
 /*
  * Copyright (c) 2007, 2008 Reyk Floeter <reyk@openbsd.org>
@@ -89,6 +89,8 @@ objid_t			 last_table_id = 0;
 objid_t			 last_host_id = 0;
 objid_t			 last_relay_id = 0;
 objid_t			 last_proto_id = 0;
+objid_t			 last_rt_id = 0;
+objid_t			 last_nr_id = 0;
 
 static struct rdr	*rdr = NULL;
 static struct table	*table = NULL;
@@ -97,6 +99,7 @@ static struct host	*hst = NULL;
 struct relaylist	 relays;
 static struct protocol	*proto = NULL;
 static struct protonode	 node;
+static struct router	*router = NULL;
 static u_int16_t	 label = 0;
 static in_port_t	 tableport = 0;
 static int		 nodedirection;
@@ -142,8 +145,8 @@ typedef struct {
 %token	NODELAY NOTHING ON PARENT PATH PORT PREFORK PROTO
 %token	QUERYSTR REAL REDIRECT RELAY REMOVE REQUEST RESPONSE RETRY
 %token	RETURN ROUNDROBIN ROUTE SACK SCRIPT SEND SESSION SOCKET
-%token	SSL STICKYADDR STYLE TABLE TAG TCP TIMEOUT TO
-%token	TRANSPARENT TRAP UPDATES URL VIRTUAL WITH TTL
+%token	SSL STICKYADDR STYLE TABLE TAG TCP TIMEOUT TO ROUTER RTLABEL
+%token	TRANSPARENT TRAP UPDATES URL VIRTUAL WITH TTL RTABLE
 %token	<v.string>	STRING
 %token  <v.number>	NUMBER
 %type	<v.string>	hostname interface table
@@ -169,6 +172,7 @@ grammar		: /* empty */
 		| grammar tabledef '\n'
 		| grammar relay '\n'
 		| grammar proto '\n'
+		| grammar router '\n'
 		| grammar error '\n'		{ file->errors++; }
 		;
 
@@ -1374,6 +1378,135 @@ dstmode		: /* empty */		{ $$ = RELAY_DSTMODE_DEFAULT; }
 		| HASH			{ $$ = RELAY_DSTMODE_HASH; }
 		;
 
+router		: ROUTER STRING		{
+			struct router *rt = NULL;
+
+			conf->sc_flags |= F_NEEDRT;
+			TAILQ_FOREACH(rt, conf->sc_rts, rt_entry)
+				if (!strcmp(rt->rt_conf.name, $2))
+					break;
+			if (rt != NULL) {
+				yyerror("router %s defined twice", $2);
+				free($2);
+				YYERROR;
+			}
+
+			if ((rt = calloc(1, sizeof (*rt))) == NULL)
+				fatal("out of memory");
+
+			if (strlcpy(rt->rt_conf.name, $2,
+			    sizeof(rt->rt_conf.name)) >=
+			    sizeof(rt->rt_conf.name)) {
+				yyerror("router name truncated");
+				free(rt);
+				YYERROR;
+			}
+			free($2);
+			rt->rt_conf.id = ++last_rt_id;
+			if (last_rt_id == INT_MAX) {
+				yyerror("too many routers defined");
+				free(rt);
+				YYERROR;
+			}
+			TAILQ_INIT(&rt->rt_netroutes);
+			router = rt;
+
+			tableport = -1;
+		} '{' optnl routeopts_l '}'	{
+			if (!router->rt_conf.nroutes) {
+				yyerror("router %s without routes",
+				    router->rt_conf.name);
+				free(router);
+				router = NULL;
+				YYERROR;
+			}
+
+			conf->sc_routercount++;
+			TAILQ_INSERT_TAIL(conf->sc_rts, router, rt_entry);
+			router = NULL;
+
+			tableport = 0;
+		}
+		;
+
+routeopts_l	: routeopts_l routeoptsl nl
+		| routeoptsl optnl
+		;
+
+routeoptsl	: ROUTE address '/' NUMBER {
+			struct netroute	*nr;
+
+			if (router->rt_af == AF_UNSPEC)
+				router->rt_af = $2.ss.ss_family;
+			else if (router->rt_af != $2.ss.ss_family) {
+				yyerror("router %s address family mismatch",
+				    router->rt_conf.name);
+				YYERROR;
+			}
+
+			if ((router->rt_af == AF_INET && ($4 > 32 || $4 < 0)) ||
+			    (router->rt_af == AF_INET6 && ($4 > 128 || $4 < 0))) {
+				yyerror("invalid prefixlen %d", $4);
+				YYERROR;
+			}
+
+			if ((nr = calloc(1, sizeof(*nr))) == NULL)
+				fatal("out of memory");
+
+			nr->nr_conf.id = ++last_nr_id;
+			if (last_nr_id == INT_MAX) {
+				yyerror("too many routes defined");
+				free(nr);
+				YYERROR;
+			}
+			nr->nr_conf.prefixlen = $4;
+			nr->nr_conf.routerid = router->rt_conf.id;
+			nr->nr_router = router;
+			bcopy(&$2.ss, &nr->nr_conf.ss, sizeof($2.ss));
+
+			router->rt_conf.nroutes++;
+			conf->sc_routecount++;
+			TAILQ_INSERT_TAIL(&router->rt_netroutes, nr, nr_entry);
+			TAILQ_INSERT_TAIL(conf->sc_routes, nr, nr_route);
+		}
+		| FORWARD TO tablespec {
+			if (router->rt_gwtable) {
+				yyerror("router %s table already specified",
+				    router->rt_conf.name);
+				purge_table(conf->sc_tables, $3);
+				YYERROR;
+			}
+			router->rt_gwtable = $3;
+			router->rt_gwtable->conf.flags |= F_USED;
+			router->rt_conf.gwtable = $3->conf.id;
+			router->rt_conf.gwport = $3->conf.port;
+		}
+		| RTABLE NUMBER {
+			if (router->rt_conf.rtable) {
+				yyerror("router %s rtable already specified",
+				    router->rt_conf.name);
+				YYERROR;
+			}
+			if ($2 < 0 || $2 > RT_TABLEID_MAX) {
+				yyerror("invalid rtable id %d", $2);
+				YYERROR;
+			}
+			router->rt_conf.rtable = $2;
+		}
+		| RTLABEL STRING {
+			if (strlcpy(router->rt_conf.label, $2,
+			    sizeof(router->rt_conf.label)) >=
+			    sizeof(router->rt_conf.label)) {
+				yyerror("route label truncated");
+				free($2);
+				YYERROR;
+			}
+			free($2);
+		}
+		| DISABLE		{ rlay->rl_conf.flags |= F_DISABLE; }
+		| include
+		;
+
 dstaf		: /* empty */		{
 			rlay->rl_conf.dstaf.ss_family = AF_UNSPEC;
 		}
@@ -1611,6 +1744,9 @@ lookup(char *s)
 		{ "return",		RETURN },
 		{ "roundrobin",		ROUNDROBIN },
 		{ "route",		ROUTE },
+		{ "router",		ROUTER },
+		{ "rtable",		RTABLE },
+		{ "rtlabel",		RTLABEL },
 		{ "sack",		SACK },
 		{ "script",		SCRIPT },
 		{ "send",		SEND },
@@ -1856,7 +1992,7 @@ nodigits:
 	(isalnum(x) || (ispunct(x) && x != '(' && x != ')' && \
 	x != '{' && x != '}' && x != '<' && x != '>' && \
 	x != '!' && x != '=' && x != '#' && \
-	x != ','))
+	x != ',' && x != '/'))
 
 	if (isalnum(c) || c == ':' || c == '_') {
 		do {
@@ -1960,6 +2096,8 @@ parse_config(const char *filename, int opts)
 	    (conf->sc_tables = calloc(1, sizeof(*conf->sc_tables))) == NULL ||
 	    (conf->sc_relays = calloc(1, sizeof(*conf->sc_relays))) == NULL ||
 	    (conf->sc_protos = calloc(1, sizeof(*conf->sc_protos))) == NULL ||
+	    (conf->sc_routes = calloc(1, sizeof(*conf->sc_routes))) == NULL ||
+	    (conf->sc_rts = calloc(1, sizeof(*conf->sc_rts))) == NULL ||
 	    (conf->sc_rdrs = calloc(1, sizeof(*conf->sc_rdrs))) == NULL) {
 		if (conf != NULL) {
 			if (conf->sc_tables != NULL)
@@ -1970,6 +2108,8 @@ parse_config(const char *filename, int opts)
 				free(conf->sc_protos);
 			if (conf->sc_rdrs != NULL)
 				free(conf->sc_rdrs);
+			if (conf->sc_rts != NULL)
+				free(conf->sc_rts);
 			free(conf);
 		}
 		log_warn("cannot allocate memory");
@@ -1978,17 +2118,20 @@ parse_config(const char *filename, int opts)
 
 	errors = 0;
 	last_host_id = last_table_id = last_rdr_id = last_proto_id =
-	    last_relay_id = 0;
+	    last_relay_id = last_rt_id = last_nr_id = 0;
 
 	rdr = NULL;
 	table = NULL;
 	rlay = NULL;
 	proto = NULL;
+	router = NULL;
 
 	TAILQ_INIT(conf->sc_rdrs);
 	TAILQ_INIT(conf->sc_tables);
 	TAILQ_INIT(conf->sc_protos);
 	TAILQ_INIT(conf->sc_relays);
+	TAILQ_INIT(conf->sc_rts);
+	TAILQ_INIT(conf->sc_routes);
 
 	memset(&conf->sc_empty_table, 0, sizeof(conf->sc_empty_table));
 	conf->sc_empty_table.conf.id = EMPTY_TABLE;
@@ -2042,8 +2185,10 @@ parse_config(const char *filename, int opts)
 		}
 	}
 
-	if (TAILQ_EMPTY(conf->sc_rdrs) && TAILQ_EMPTY(conf->sc_relays)) {
-		log_warnx("no redirections, nothing to do");
+	if (TAILQ_EMPTY(conf->sc_rdrs) &&
+	    TAILQ_EMPTY(conf->sc_relays) &&
+	    TAILQ_EMPTY(conf->sc_rts)) {
+		log_warnx("no actions, nothing to do");
 		errors++;
 	}
 
diff --git a/usr.sbin/relayd/pfe.c b/usr.sbin/relayd/pfe.c
index d6e7ac1c2cf..d5db13f5b03 100644
--- a/usr.sbin/relayd/pfe.c
+++ b/usr.sbin/relayd/pfe.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: pfe.c,v 1.61 2009/08/07 11:21:53 reyk Exp $	*/
+/*	$OpenBSD: pfe.c,v 1.62 2009/08/13 13:51:21 reyk Exp $	*/
 
 /*
  * Copyright (c) 2006 Pierre-Yves Ritschard <pyr@openbsd.org>
@@ -860,6 +860,7 @@ pfe_sync(void)
 	struct ctl_id		 id;
 	struct imsg		 imsg;
 	struct ctl_demote	 demote;
+	struct router		*rt;
 
 	bzero(&id, sizeof(id));
 	bzero(&imsg, sizeof(imsg));
@@ -914,6 +915,14 @@ pfe_sync(void)
 		}
 	}
 
+	TAILQ_FOREACH(rt, env->sc_rts, rt_entry) {
+		rt->rt_conf.flags &= ~(F_BACKUP);
+		rt->rt_conf.flags &= ~(F_DOWN);
+
+		if ((rt->rt_gwtable->conf.flags & F_CHANGED))
+			sync_routes(env, rt);
+	}
+
 	TAILQ_FOREACH(table, env->sc_tables, entry) {
 		/*
 		 * clean up change flag.
diff --git a/usr.sbin/relayd/pfe_route.c b/usr.sbin/relayd/pfe_route.c
new file mode 100644
index 00000000000..1189b4dee6f
--- /dev/null
+++ b/usr.sbin/relayd/pfe_route.c
@@ -0,0 +1,240 @@
+/*	$OpenBSD: pfe_route.c,v 1.1 2009/08/13 13:51:21 reyk Exp $	*/
+
+/*
+ * Copyright (c) 2009 Reyk Floeter <reyk@openbsd.org>
+ *
+ * Permission to use, copy, modify, and distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#include <sys/types.h>
+#include <sys/queue.h>
+#include <sys/socket.h>
+
+#include <net/if.h>
+#include <netinet/in.h>
+#include <arpa/inet.h>
+#include <net/route.h>
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <event.h>
+#include <string.h>
+#include <errno.h>
+
+#include <openssl/ssl.h>
+
+#include "relayd.h"
+
+extern struct imsgev	*iev_main;
+
+struct relay_rtmsg {
+	struct rt_msghdr	rm_hdr;
+	union {
+		struct {
+			struct sockaddr_in	rm_dst;
+			struct sockaddr_in	rm_gateway;
+			struct sockaddr_in	rm_netmask;
+			struct sockaddr_rtlabel	rm_label;
+		}		 u4;
+		struct {
+			struct sockaddr_in6	rm_dst;
+			struct sockaddr_in6	rm_gateway;
+			struct sockaddr_in6	rm_netmask;
+			struct sockaddr_rtlabel	rm_label;
+		}		 u6;
+	}			 rm_u;
+};
+
+void
+init_routes(struct relayd *env)
+{
+	u_int	 rtfilter;
+
+	if (!(env->sc_flags & F_NEEDRT))
+		return;
+
+	if ((env->sc_rtsock = socket(AF_ROUTE, SOCK_RAW, 0)) == -1)
+		fatal("init_routes: failed to open routing socket");
+
+	rtfilter = ROUTE_FILTER(0);
+	if (setsockopt(env->sc_rtsock, AF_ROUTE, ROUTE_MSGFILTER,
+	    &rtfilter, sizeof(rtfilter)) == -1)
+		fatal("init_routes: ROUTE_MSGFILTER");
+}
+
+void
+sync_routes(struct relayd *env, struct router *rt)
+{
+	struct netroute		*nr;
+	struct host		*host;
+	char			 buf[MAXHOSTNAMELEN];
+	struct ctl_netroute	 crt;
+
+	if (!(env->sc_flags & F_NEEDRT))
+		return;
+
+	TAILQ_FOREACH(nr, &rt->rt_netroutes, nr_entry) {
+		print_host(&nr->nr_conf.ss, buf, sizeof(buf));
+		TAILQ_FOREACH(host, &rt->rt_gwtable->hosts, entry) {
+			if (host->up == HOST_UNKNOWN)
+				continue;
+
+			log_debug("sync_routes: "
+			    "router %s route %s/%d gateway %s %s",
+			    rt->rt_conf.name, buf, nr->nr_conf.prefixlen,
+			    host->conf.name,
+			    HOST_ISUP(host->up) ? "up" : "down");
+
+			crt.id = nr->nr_conf.id;
+			crt.hostid = host->conf.id;
+			crt.up = host->up;
+
+			imsg_compose_event(iev_main, IMSG_RTMSG,
+			    0, 0, -1, &crt, sizeof(crt));
+		}
+	}
+}
+
+int
+pfe_route(struct relayd *env, struct ctl_netroute *crt)
+{
+	struct relay_rtmsg		 rm;
+	struct sockaddr_rtlabel		 sr;
+	struct sockaddr_storage		*gw;
+	struct sockaddr_in		*s4;
+	struct sockaddr_in6		*s6;
+	size_t				 len = 0;
+	struct netroute			 *nr;
+	struct host			*host;
+	char				*gwname;
+	int				 i = 0;
+
+	if ((nr = route_find(env, crt->id)) == NULL ||
+	    (host = host_find(env, crt->hostid)) == NULL) {
+		log_debug("pfe_route: invalid host or route id");
+		return (-1);
+	}
+
+	gw = &host->conf.ss;
+	gwname = host->conf.name;
+
+	bzero(&rm, sizeof(rm));
+	bzero(&sr, sizeof(sr));
+
+	rm.rm_hdr.rtm_msglen = len;
+	rm.rm_hdr.rtm_version = RTM_VERSION;
+	rm.rm_hdr.rtm_type = HOST_ISUP(crt->up) ? RTM_ADD : RTM_DELETE;
+	rm.rm_hdr.rtm_flags = RTF_STATIC | RTF_GATEWAY | RTF_MPATH;
+	rm.rm_hdr.rtm_seq = env->sc_rtseq++;
+	rm.rm_hdr.rtm_addrs = RTA_DST | RTA_GATEWAY;
+	rm.rm_hdr.rtm_tableid = nr->nr_router->rt_conf.rtable;
+
+	if (strlen(nr->nr_router->rt_conf.label)) {
+		rm.rm_hdr.rtm_addrs |= RTA_LABEL;
+		sr.sr_len = sizeof(sr);
+		if (snprintf(sr.sr_label, sizeof(sr.sr_label),
+		    "%s", nr->nr_router->rt_conf.label) == -1)
+			goto bad;
+	}
+
+	if (nr->nr_conf.ss.ss_family == AF_INET) {
+		rm.rm_hdr.rtm_msglen = len =
+		    sizeof(rm.rm_hdr) + sizeof(rm.rm_u.u4);
+
+		bcopy(&sr, &rm.rm_u.u4.rm_label, sizeof(sr));
+
+		s4 = &rm.rm_u.u4.rm_dst;
+		s4->sin_family = AF_INET;
+		s4->sin_len = sizeof(rm.rm_u.u4.rm_dst);
+		s4->sin_addr.s_addr =
+		    ((struct sockaddr_in *)&nr->nr_conf.ss)->sin_addr.s_addr;
+
+		s4 = &rm.rm_u.u4.rm_gateway;
+		s4->sin_family = AF_INET;
+		s4->sin_len = sizeof(rm.rm_u.u4.rm_gateway);
+		s4->sin_addr.s_addr =
+		    ((struct sockaddr_in *)gw)->sin_addr.s_addr;
+
+		rm.rm_hdr.rtm_addrs |= RTA_NETMASK;
+		s4 = &rm.rm_u.u4.rm_netmask;
+		s4->sin_family = AF_INET;
+		s4->sin_len = sizeof(rm.rm_u.u4.rm_netmask);
+		if (nr->nr_conf.prefixlen)
+			s4->sin_addr.s_addr =
+			    htonl(0xffffffff << (32 - nr->nr_conf.prefixlen));
+		else if (nr->nr_conf.prefixlen < 0)
+			rm.rm_hdr.rtm_flags |= RTF_HOST;
+	} else if (nr->nr_conf.ss.ss_family == AF_INET6) {
+		rm.rm_hdr.rtm_msglen = len =
+		    sizeof(rm.rm_hdr) + sizeof(rm.rm_u.u6);
+
+		bcopy(&sr, &rm.rm_u.u6.rm_label, sizeof(sr));
+
+		s6 = &rm.rm_u.u6.rm_dst;
+		bcopy(((struct sockaddr_in6 *)&nr->nr_conf.ss),
+		    s6, sizeof(*s6));
+		s6->sin6_family = AF_INET6;
+		s6->sin6_len = sizeof(*s6);
+
+		s6 = &rm.rm_u.u6.rm_gateway;
+		bcopy(((struct sockaddr_in6 *)gw), s6, sizeof(*s6));
+		s6->sin6_family = AF_INET6;
+		s6->sin6_len = sizeof(*s6);
+
+		rm.rm_hdr.rtm_addrs |= RTA_NETMASK;
+		s6 = &rm.rm_u.u6.rm_netmask;
+		s6->sin6_family = AF_INET6;
+		s6->sin6_len = sizeof(*s6);
+		if (nr->nr_conf.prefixlen) {
+			for (i = 0; i < nr->nr_conf.prefixlen / 8; i++)
+				s6->sin6_addr.s6_addr[i] = 0xff;
+			i = nr->nr_conf.prefixlen % 8;
+			if (i)
+				s6->sin6_addr.s6_addr[nr->nr_conf.prefixlen
+				    / 8] = 0xff00 >> i;
+		} else if (nr->nr_conf.prefixlen < 0)
+			rm.rm_hdr.rtm_flags |= RTF_HOST;
+	} else
+		fatal("pfe_route: invalid address family");
+
+ retry:
+	if (write(env->sc_rtsock, &rm, len) == -1) {
+		switch (errno) {
+		case EEXIST:
+		case ESRCH:
+			if (rm.rm_hdr.rtm_type == RTM_ADD) {
+				rm.rm_hdr.rtm_type = RTM_CHANGE;
+				goto retry;
+			} else if (rm.rm_hdr.rtm_type == RTM_DELETE) {
+				/* Ignore */
+				break;
+			}
+			/* FALLTHROUGH */
+		default:
+			goto bad;
+		}
+	}
+
+	log_debug("pfe_route: gateway %s %s", gwname,
+	    HOST_ISUP(crt->up) ? "added" : "deleted");
+
+	return (0);
+
+ bad:
+	log_debug("pfe_route: failed to %s gateway %s: %d %s",
+	    HOST_ISUP(crt->up) ? "add" : "delete", gwname,
+	    errno, strerror(errno));
+
+	return (-1);
+}
diff --git a/usr.sbin/relayd/relayd.c b/usr.sbin/relayd/relayd.c
index 503493a63de..68684bb9b5d 100644
--- a/usr.sbin/relayd/relayd.c
+++ b/usr.sbin/relayd/relayd.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: relayd.c,v 1.91 2009/08/07 11:21:53 reyk Exp $	*/
+/*	$OpenBSD: relayd.c,v 1.92 2009/08/13 13:51:21 reyk Exp $	*/
 
 /*
  * Copyright (c) 2007, 2008 Reyk Floeter <reyk@openbsd.org>
@@ -270,7 +270,9 @@ main(int argc, char *argv[])
 	imsg_init(&iev_pfe->ibuf, pipe_parent2pfe[0]);
 	imsg_init(&iev_hce->ibuf, pipe_parent2hce[0]);
 	iev_pfe->handler = main_dispatch_pfe;
+	iev_pfe->data = env;
 	iev_hce->handler = main_dispatch_hce;
+	iev_hce->data = env;
 
 	for (c = 0; c < env->sc_prefork_relay; c++) {
 		iev = &iev_relay[c];
@@ -295,6 +297,8 @@ main(int argc, char *argv[])
 	if (env->sc_flags & F_DEMOTE)
 		carp_demote_reset(env->sc_demote_group, 0);
 
+	init_routes(env);
+
 	event_dispatch();
 
 	return (0);
@@ -611,9 +615,12 @@ main_dispatch_pfe(int fd, short event, void *ptr)
 	struct imsg		 imsg;
 	ssize_t			 n;
 	struct ctl_demote	 demote;
+	struct ctl_netroute	 crt;
+	struct relayd		*env;
 
 	iev = ptr;
 	ibuf = &iev->ibuf;
+	env = (struct relayd *)iev->data;
 
 	if (event & EV_READ) {
 		if ((n = imsg_read(ibuf)) == -1)
@@ -646,6 +653,14 @@ main_dispatch_pfe(int fd, short event, void *ptr)
 			memcpy(&demote, imsg.data, sizeof(demote));
 			carp_demote_set(demote.group, demote.level);
 			break;
+		case IMSG_RTMSG:
+			if (imsg.hdr.len - IMSG_HEADER_SIZE !=
+			    sizeof(crt))
+				fatalx("main_dispatch_pfe: "
+				    "invalid size of rtmsg request");
+			memcpy(&crt, imsg.data, sizeof(crt));
+			pfe_route(env, &crt);
+			break;
 		case IMSG_CTL_RELOAD:
 			/*
 			 * so far we only get here if no L7 (relay) is done.
@@ -849,6 +864,17 @@ session_find(struct relayd *env, objid_t id)
 	return (NULL);
 }
 
+struct netroute *
+route_find(struct relayd *env, objid_t id)
+{
+	struct netroute	*nr;
+
+	TAILQ_FOREACH(nr, env->sc_routes, nr_route)
+		if (nr->nr_conf.id == id)
+			return (nr);
+	return (NULL);
+}
+
 struct host *
 host_findbyname(struct relayd *env, const char *name)
 {
diff --git a/usr.sbin/relayd/relayd.conf.5 b/usr.sbin/relayd/relayd.conf.5
index b053c8b8cfa..30288391e2f 100644
--- a/usr.sbin/relayd/relayd.conf.5
+++ b/usr.sbin/relayd/relayd.conf.5
@@ -1,4 +1,4 @@
-.\"	$OpenBSD: relayd.conf.5,v 1.108 2009/08/07 11:10:23 reyk Exp $
+.\"	$OpenBSD: relayd.conf.5,v 1.109 2009/08/13 13:51:21 reyk Exp $
 .\"
 .\" Copyright (c) 2006, 2007 Reyk Floeter <reyk@openbsd.org>
 .\" Copyright (c) 2006, 2007 Pierre-Yves Ritschard <pyr@openbsd.org>
@@ -15,7 +15,7 @@
 .\" ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
 .\" OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 .\"
-.Dd $Mdocdate: August 7 2009 $
+.Dd $Mdocdate: August 13 2009 $
 .Dt RELAYD.CONF 5
 .Os
 .Sh NAME
@@ -43,8 +43,8 @@ Table definitions describe a list of hosts,
 in a similar fashion to
 .Xr pf 4
 tables.
-They are used for relay and redirection target selection with the
-described options and health checking on the host they contain.
+They are used for relay, redirection, and router target selection with
+the described options and health checking on the host they contain.
 .It Sy Redirections
 Redirections are translated to
 .Xr pf 4
@@ -55,6 +55,9 @@ Relays allow application layer load balancing, SSL acceleration, and
 general purpose TCP proxying on layer 7.
 .It Sy Protocols
 Protocols are predefined protocol handlers and settings for relays.
+.It Sy Routers
+Routers are used to insert routes with health-checked gateways for
+(WAN) link balancing.
 .El
 .Pp
 Within the sections,
@@ -1035,6 +1038,64 @@ connection.
 This will affect the TCP window size.
 .El
 .El
+.Sh ROUTERS
+Routers represent routing table entries in the kernel forwarding
+database, see
+.Xr route 4 ,
+and a table of associated gateways.
+They are used to dynamically insert or remove routes with gateways
+based on their availability and health-check results.
+A router can include multiple network statements and a single forward
+statement with a table of one or more gateways.
+All entries in a single router directive must match the same address
+family, either IPv4 or IPv6.
+.Pp
+The kernel supports multipath routing when multiple gateways exist to
+the same destination address.
+The multipath routing behaviour can be changed globally using the
+.Xr sysctl 8
+variables
+.Va net.inet.ip.multipath
+and
+.Va net.inet6.ip6.multipath .
+With the default setting of 0,
+the first route selected will be used for subsequent packets to that
+destination regardless of source.
+Setting it to 1 will enable load balancing based on the packet source
+address across gateways; multiple routes with the same priority are
+used equally.
+The kernel will also check the link state of the related network
+interface and try a different route if it is not active.
+.Pp
+The configuration directives that are valid in the
+.Ic routers
+context are described below:
+.Bl -tag -width Ds
+.It Xo
+.Ic forward to
+.Aq Ar table
+.Ic port Ar number
+.Ar options ...
+.Xc
+Specify the table of target gateways to be used; see the
+.Sx TABLES
+section above for information about table options.
+This entry is mandatory and must be specified once.
+.It Xo
+.Ic route
+.Ar address Ns Li / Ns Ar prefix
+.Xc
+Specify the network address and prefix length of a route destination
+that is reachable via the active gateways.
+This entry must be specified at least once in a router directive.
+.It Ic rtable Ar id
+Add the routes to the kernel routing table with the specified
+.Ar id .
+.It Ic rtlabel Ar label
+Add the routes with the specified
+.Ar label
+to the kernel routing table.
+.El
 .Sh FILES
 .Bl -tag -width "/etc/ssl/private/address.keyXX" -compact
 .It Pa /etc/relayd.conf
@@ -1143,6 +1204,16 @@ relay "sshforward" {
 	forward to shell.example.com port 22
 }
 .Ed
+.Pp
+The next simple router configuration example can be used to run
+redundant, health-checked WAN links:
+.Bd -literal -offset indent
+table \*(Ltgateways\*(Gt { $gw1 ip ttl 1, $gw2 ip ttl 1 }
+router "uplinks" {
+	route 0.0.0.0/0
+	forward to \*(Ltgateways\*(Gt check icmp
+}
+.Ed
 .Sh SEE ALSO
 .Xr relayctl 8 ,
 .Xr relayd 8 ,
diff --git a/usr.sbin/relayd/relayd.h b/usr.sbin/relayd/relayd.h
index 7660b8a825c..39513f95f7f 100644
--- a/usr.sbin/relayd/relayd.h
+++ b/usr.sbin/relayd/relayd.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: relayd.h,v 1.129 2009/08/07 11:21:53 reyk Exp $	*/
+/*	$OpenBSD: relayd.h,v 1.130 2009/08/13 13:51:21 reyk Exp $	*/
 
 /*
  * Copyright (c) 2006, 2007 Pierre-Yves Ritschard <pyr@openbsd.org>
@@ -34,6 +34,7 @@
 #define EMPTY_ID		UINT_MAX
 #define TABLE_NAME_SIZE		64
 #define	TAG_NAME_SIZE		64
+#define	RT_LABEL_SIZE		32
 #define SRV_NAME_SIZE		64
 #define MAX_NAME_SIZE		64
 #define SRV_MAX_VIRTS		16
@@ -97,6 +98,12 @@ struct ctl_demote {
 	int		 level;
 };
 
+struct ctl_netroute {
+	objid_t		 id;
+	objid_t		 hostid;
+	int		 up;
+};
+
 struct ctl_icmp_event {
 	struct relayd		*env;
 	int			 s;
@@ -238,6 +245,7 @@ TAILQ_HEAD(addresslist, address);
 #define F_NEEDPF		0x00080000
 #define F_PORT			0x00100000
 #define F_SSLCLIENT		0x00200000
+#define F_NEEDRT		0x00400000
 
 enum forwardmode {
 	FWD_NORMAL		= 0,
@@ -575,6 +583,46 @@ enum dstmode {
 };
 #define RELAY_DSTMODE_DEFAULT		RELAY_DSTMODE_ROUNDROBIN
 
+struct router;
+struct netroute_config {
+	objid_t			 id;
+	struct sockaddr_storage	 ss;
+	int			 prefixlen;
+	objid_t			 routerid;
+};
+
+struct netroute {
+	TAILQ_ENTRY(netroute)	 nr_entry;
+	TAILQ_ENTRY(netroute)	 nr_route;
+
+	struct netroute_config	 nr_conf;
+
+	struct router		*nr_router;
+};
+TAILQ_HEAD(netroutelist, netroute);
+
+struct router_config {
+	objid_t			 id;
+	u_int32_t		 flags;
+	char			 name[MAXHOSTNAMELEN];
+	char			 label[RT_LABEL_SIZE];
+	int			 nroutes;
+	objid_t			 gwtable;
+	in_port_t		 gwport;
+	int			 rtable;
+};
+
+struct router {
+	TAILQ_ENTRY(router)	 rt_entry;
+	struct router_config	 rt_conf;
+
+	struct table		*rt_gwtable;
+	struct netroutelist	 rt_netroutes;
+
+	int			 rt_af;
+};
+TAILQ_HEAD(routerlist, router);
+
 enum {
 	PROC_MAIN,
 	PROC_PFE,
@@ -587,10 +635,14 @@ struct relayd {
 	u_int32_t		 sc_flags;
 	const char		*sc_confpath;
 	struct pfdata		*sc_pf;
+	int			 sc_rtsock;
+	int			 sc_rtseq;
 	int			 sc_tablecount;
 	int			 sc_rdrcount;
 	int			 sc_protocount;
 	int			 sc_relaycount;
+	int			 sc_routercount;
+	int			 sc_routecount;
 	struct timeval		 sc_interval;
 	struct timeval		 sc_timeout;
 	struct table		 sc_empty_table;
@@ -600,6 +652,8 @@ struct relayd {
 	struct rdrlist		*sc_rdrs;
 	struct protolist	*sc_protos;
 	struct relaylist	*sc_relays;
+	struct routerlist	*sc_rts;
+	struct netroutelist	*sc_routes;
 	u_int16_t		 sc_prefork_relay;
 	char			 sc_demote_group[IFNAMSIZ];
 	u_int16_t		 sc_id;
@@ -705,7 +759,8 @@ enum imsg_type {
 	IMSG_RECONF_END,
 	IMSG_SCRIPT,
 	IMSG_SNMPSOCK,
-	IMSG_BINDANY
+	IMSG_BINDANY,
+	IMSG_RTMSG		/* from pfe to parent */
 };
 
 /* control.c */
@@ -756,6 +811,11 @@ int	 natlook(struct relayd *, struct ctl_natlook *);
 u_int64_t
 	 check_table(struct relayd *, struct rdr *, struct table *);
 
+/* pfe_route.c */
+void	 init_routes(struct relayd *);
+void	 sync_routes(struct relayd *, struct router *);
+int	 pfe_route(struct relayd *, struct ctl_netroute *);
+
 /* hce.c */
 pid_t	 hce(struct relayd *, int [2], int [2], int [RELAY_MAXPROC][2],
 	    int [2], int [RELAY_MAXPROC][2]);
@@ -814,6 +874,7 @@ int	 ssl_ctx_load_verify_memory(SSL_CTX *, char *, off_t);
 struct host	*host_find(struct relayd *, objid_t);
 struct table	*table_find(struct relayd *, objid_t);
 struct rdr	*rdr_find(struct relayd *, objid_t);
+struct netroute	*route_find(struct relayd *, objid_t);
 struct host	*host_findbyname(struct relayd *, const char *);
 struct table	*table_findbyname(struct relayd *, const char *);
 struct table	*table_findbyconf(struct relayd *, struct table *);