Support more than one relay backup table. Instead of duplicating the

code for main and backup table all over the place, turn the relay
tables into a list attached to the relay.  This improves the code and
allows some other tricks with multiple tables later.

8 files changed, 234 insertions, 124 deletions

diff --git a/usr.sbin/relayd/config.c b/usr.sbin/relayd/config.c
index eb41bc57c15..016f26cd911 100644
--- a/usr.sbin/relayd/config.c
+++ b/usr.sbin/relayd/config.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: config.c,v 1.3 2012/05/08 15:10:15 benno Exp $	*/
+/*	$OpenBSD: config.c,v 1.4 2012/10/03 08:33:31 reyk Exp $	*/
 
 /*
  * Copyright (c) 2011 Reyk Floeter <reyk@openbsd.org>
@@ -792,11 +792,13 @@ config_getprotonode(struct relayd *env, struct imsg *imsg)
 int
 config_setrelay(struct relayd *env, struct relay *rlay)
 {
-	struct privsep	*ps = env->sc_ps;
-	int		 id;
-	int		 fd, n, m;
-	struct iovec	 iov[4];
-	size_t		 c;
+	struct privsep		*ps = env->sc_ps;
+	struct ctl_relaytable	 crt;
+	struct relay_table	*rlt;
+	int			 id;
+	int			 fd, n, m;
+	struct iovec		 iov[4];
+	size_t			 c;
 
 	/* opens listening sockets etc. */
 	if (relay_privinit(rlay) == -1)
@@ -840,6 +842,20 @@ config_setrelay(struct relayd *env, struct relay *rlay)
 			proc_composev_imsg(ps, id, -1, IMSG_CFG_RELAY, -1,
 			    iov, c);
 		}
+
+		/* Now send the tables associated to this relay */
+		TAILQ_FOREACH(rlt, &rlay->rl_tables, rlt_entry) {
+			crt.id = rlt->rlt_table->conf.id;
+			crt.relayid = rlay->rl_conf.id;
+			crt.mode = rlt->rlt_mode;
+
+			c = 0;
+			iov[c].iov_base = &crt;
+			iov[c++].iov_len = sizeof(crt);
+
+			proc_composev_imsg(ps, id, -1,
+			    IMSG_CFG_RELAY_TABLE, -1, iov, c);
+		}
 	}
 
 	close(rlay->rl_s);
@@ -875,21 +891,6 @@ config_getrelay(struct relayd *env, struct imsg *imsg)
 		}
 	}
 
-	if (rlay->rl_conf.dsttable != EMPTY_ID &&
-	    (rlay->rl_dsttable = table_find(env,
-	    rlay->rl_conf.dsttable)) == NULL) {
-		log_debug("%s: unknown table", __func__);
-		goto fail;
-	}
-
-	rlay->rl_backuptable = &env->sc_empty_table;
-	if (rlay->rl_conf.backuptable != EMPTY_ID &&
-	    (rlay->rl_backuptable = table_find(env,
-	    rlay->rl_conf.backuptable)) == NULL) {
-		log_debug("%s: unknown backup table", __func__);
-		goto fail;
-	}
-
 	if ((u_int)(IMSG_DATA_SIZE(imsg) - s) <
 	    (rlay->rl_conf.ssl_cert_len +
 	    rlay->rl_conf.ssl_key_len +
@@ -917,6 +918,7 @@ config_getrelay(struct relayd *env, struct imsg *imsg)
 		s += rlay->rl_conf.ssl_ca_len;
 	}
 
+	TAILQ_INIT(&rlay->rl_tables);
 	TAILQ_INSERT_TAIL(env->sc_relays, rlay, rl_entry);
 
 	env->sc_relaycount++;
@@ -938,3 +940,47 @@ config_getrelay(struct relayd *env, struct imsg *imsg)
 	free(rlay);
 	return (-1);
 }
+
+int
+config_getrelaytable(struct relayd *env, struct imsg *imsg)
+{
+	struct relay_table	*rlt = NULL;
+	struct ctl_relaytable	 crt;
+	struct relay		*rlay;
+	struct table		*table;
+	u_int8_t		*p = imsg->data;
+	size_t			 s;
+
+	IMSG_SIZE_CHECK(imsg, &crt);
+	memcpy(&crt, p, sizeof(crt));
+	s = sizeof(crt);
+
+	if ((rlay = relay_find(env, crt.relayid)) == NULL) {
+		log_debug("%s: unknown relay", __func__);
+		goto fail;
+	}
+
+	if ((table = table_find(env, crt.id)) == NULL) {
+		log_debug("%s: unknown table", __func__);
+		goto fail;
+	}
+
+	if ((rlt = calloc(1, sizeof(*rlt))) == NULL)
+		goto fail;
+
+	rlt->rlt_table = table;
+	rlt->rlt_mode = crt.mode;
+
+	TAILQ_INSERT_TAIL(&rlay->rl_tables, rlt, rlt_entry);
+
+	DPRINTF("%s: %s %d received relay table %s for relay %s", __func__,
+	    env->sc_ps->ps_title[privsep_process], env->sc_ps->ps_instance,
+	    table->conf.name, rlay->rl_conf.name);
+
+	return (0);
+
+ fail:
+	if (rlt != NULL)
+		free(rlt);
+	return (-1);
+}
diff --git a/usr.sbin/relayd/parse.y b/usr.sbin/relayd/parse.y
index a7be65c5272..119e8193115 100644
--- a/usr.sbin/relayd/parse.y
+++ b/usr.sbin/relayd/parse.y
@@ -1,4 +1,4 @@
-/*	$OpenBSD: parse.y,v 1.164 2012/05/29 23:46:50 benno Exp $	*/
+/*	$OpenBSD: parse.y,v 1.165 2012/10/03 08:33:31 reyk Exp $	*/
 
 /*
  * Copyright (c) 2007-2011 Reyk Floeter <reyk@openbsd.org>
@@ -108,6 +108,7 @@ static struct router	*router = NULL;
 static u_int16_t	 label = 0;
 static in_port_t	 tableport = 0;
 static int		 nodedirection;
+static int		 dstmode;
 
 struct address	*host_v4(const char *);
 struct address	*host_v6(const char *);
@@ -576,6 +577,7 @@ tabledef	: TABLE table		{
 			    sizeof(struct timeval));
 			TAILQ_INIT(&tb->hosts);
 			table = tb;
+			dstmode = RELAY_DSTMODE_DEFAULT;
 		} tabledefopts_l 	{
 			if (TAILQ_EMPTY(&table->hosts)) {
 				yyerror("table %s has no hosts",
@@ -679,8 +681,7 @@ tableopts	: CHECK tablecheck
 				}
 				/* FALLTHROUGH */
 			case RELAY_DSTMODE_ROUNDROBIN:
-				if (rlay != NULL)
-					rlay->rl_conf.dstmode = $2;
+				dstmode = $2;
 				break;
 			}
 		}
@@ -1223,14 +1224,14 @@ relay		: RELAY STRING	{
 			r->rl_conf.timeout.tv_sec = RELAY_TIMEOUT;
 			r->rl_proto = NULL;
 			r->rl_conf.proto = EMPTY_ID;
-			r->rl_conf.dsttable = EMPTY_ID;
-			r->rl_conf.dstmode = RELAY_DSTMODE_DEFAULT;
 			r->rl_conf.dstretry = 0;
+			TAILQ_INIT(&r->rl_tables);
 			if (last_relay_id == INT_MAX) {
 				yyerror("too many relays defined");
 				free(r);
 				YYERROR;
 			}
+			dstmode = RELAY_DSTMODE_DEFAULT;
 			rlay = r;
 		} '{' optnl relayopts_l '}'	{
 			struct relay	*r;
@@ -1248,16 +1249,11 @@ relay		: RELAY STRING	{
 			}
 			if ((rlay->rl_conf.flags & (F_NATLOOK|F_DIVERT)) == 0 &&
 			    rlay->rl_conf.dstss.ss_family == AF_UNSPEC &&
-			    rlay->rl_conf.dsttable == EMPTY_ID) {
+			    TAILQ_EMPTY(&rlay->rl_tables)) {
 				yyerror("relay %s has no target, rdr, "
 				    "or table", rlay->rl_conf.name);
 				YYERROR;
 			}
-			if (rlay->rl_backuptable == NULL) {
-				rlay->rl_conf.backuptable =
-				    conf->sc_empty_table.conf.id;
-				rlay->rl_backuptable = &conf->sc_empty_table;
-			}
 			if (rlay->rl_conf.proto == EMPTY_ID) {
 				rlay->rl_proto = &conf->sc_proto_default;
 				rlay->rl_conf.proto = conf->sc_proto_default.id;
@@ -1415,20 +1411,21 @@ forwardspec	: STRING port retry	{
 			rlay->rl_conf.dstretry = $2;
 		}
 		| tablespec	{
-			if (rlay->rl_backuptable) {
-				yyerror("only one backup table is allowed");
+			struct relay_table	*rlt;
+
+			if ((rlt = calloc(1, sizeof(*rlt))) == NULL) {
+				yyerror("failed to allocate table reference");
 				YYERROR;
 			}
-			if (rlay->rl_dsttable) {
-				rlay->rl_backuptable = $1;
-				rlay->rl_backuptable->conf.flags |= F_USED;
-				rlay->rl_conf.backuptable = $1->conf.id;
-			} else {
-				rlay->rl_dsttable = $1;
-				rlay->rl_dsttable->conf.flags |= F_USED;
-				rlay->rl_conf.dsttable = $1->conf.id;
-				rlay->rl_conf.dstport = $1->conf.port;
-			}
+
+			rlt->rlt_table = $1;
+			rlt->rlt_table->conf.flags |= F_USED;
+			rlt->rlt_mode = dstmode;
+			rlt->rlt_flags = F_USED;
+			if (!TAILQ_EMPTY(&rlay->rl_tables))
+				rlt->rlt_flags |= F_BACKUP;
+
+			TAILQ_INSERT_TAIL(&rlay->rl_tables, rlt, rlt_entry);
 		}
 		;
 
@@ -2211,9 +2208,10 @@ parse_config(const char *filename, struct relayd *x_conf)
 int
 load_config(const char *filename, struct relayd *x_conf)
 {
-	struct sym	*sym, *next;
-	struct table	*nexttb;
-	struct host	*h, *ph;
+	struct sym		*sym, *next;
+	struct table		*nexttb;
+	struct host		*h, *ph;
+	struct relay_table	*rlt;
 
 	conf = x_conf;
 	conf->sc_flags = 0;
@@ -2266,6 +2264,10 @@ load_config(const char *filename, struct relayd *x_conf)
 	/* Cleanup relay list to inherit */
 	while ((rlay = TAILQ_FIRST(&relays)) != NULL) {
 		TAILQ_REMOVE(&relays, rlay, rl_entry);
+		while ((rlt = TAILQ_FIRST(&rlay->rl_tables))) {
+			TAILQ_REMOVE(&rlay->rl_tables, rlt, rlt_entry);
+			free(rlt);
+		}
 		free(rlay);
 	}
 
@@ -2717,6 +2719,7 @@ struct relay *
 relay_inherit(struct relay *ra, struct relay *rb)
 {
 	struct relay_config	 rc;
+	struct relay_table	*rta, *rtb;
 
 	bcopy(&rb->rl_conf, &rc, sizeof(rc));
 	bcopy(ra, rb, sizeof(*rb));
@@ -2725,6 +2728,7 @@ relay_inherit(struct relay *ra, struct relay *rb)
 	rb->rl_conf.port = rc.port;
 	rb->rl_conf.flags =
 	    (ra->rl_conf.flags & ~F_SSL) | (rc.flags & F_SSL);
+	TAILQ_INIT(&rb->rl_tables);
 
 	rb->rl_conf.id = ++last_relay_id;
 	if (last_relay_id == INT_MAX) {
@@ -2750,6 +2754,17 @@ relay_inherit(struct relay *ra, struct relay *rb)
 		goto err;
 	}
 
+	TAILQ_FOREACH(rta, &ra->rl_tables, rlt_entry) {
+		if ((rtb = calloc(1, sizeof(*rtb))) == NULL) {
+			yyerror("cannot allocate relay table");
+			goto err;
+		}
+		rtb->rlt_table = rta->rlt_table;
+		rtb->rlt_mode = rta->rlt_mode;
+
+		TAILQ_INSERT_TAIL(&rb->rl_tables, rtb, rlt_entry);
+	}
+
 	conf->sc_relaycount++;
 	SPLAY_INIT(&rlay->rl_sessions);
 	TAILQ_INSERT_TAIL(conf->sc_relays, rb, rl_entry);
@@ -2757,6 +2772,10 @@ relay_inherit(struct relay *ra, struct relay *rb)
 	return (rb);
 
  err:
+	while ((rtb = TAILQ_FIRST(&rb->rl_tables))) {
+		TAILQ_REMOVE(&rb->rl_tables, rtb, rlt_entry);
+		free(rtb);
+	}
 	free(rb);
 	return (NULL);
 }
diff --git a/usr.sbin/relayd/pfe.c b/usr.sbin/relayd/pfe.c
index c69bdf24894..a678c9de8eb 100644
--- a/usr.sbin/relayd/pfe.c
+++ b/usr.sbin/relayd/pfe.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: pfe.c,v 1.72 2012/01/21 13:40:48 camield Exp $	*/
+/*	$OpenBSD: pfe.c,v 1.73 2012/10/03 08:33:31 reyk Exp $	*/
 
 /*
  * Copyright (c) 2006 Pierre-Yves Ritschard <pyr@openbsd.org>
@@ -199,6 +199,9 @@ pfe_dispatch_parent(int fd, struct privsep_proc *p, struct imsg *imsg)
 	case IMSG_CFG_RELAY:
 		config_getrelay(env, imsg);
 		break;
+	case IMSG_CFG_RELAY_TABLE:
+		config_getrelaytable(env, imsg);
+		break;
 	case IMSG_CFG_DONE:
 		config_getcfg(env, imsg);
 		init_filter(env, imsg->fd);
@@ -291,11 +294,12 @@ pfe_dispatch_relay(int fd, struct privsep_proc *p, struct imsg *imsg)
 void
 show(struct ctl_conn *c)
 {
-	struct rdr	*rdr;
-	struct host	*host;
-	struct relay	*rlay;
-	struct router	*rt;
-	struct netroute	*nr;
+	struct rdr		*rdr;
+	struct host		*host;
+	struct relay		*rlay;
+	struct router		*rt;
+	struct netroute		*nr;
+	struct relay_table	*rlt;
 
 	if (env->sc_rdrs == NULL)
 		goto relays;
@@ -334,23 +338,16 @@ relays:
 		imsg_compose_event(&c->iev, IMSG_CTL_RELAY_STATS, 0, 0, -1,
 		    &rlay->rl_stats, sizeof(rlay->rl_stats));
 
-		if (rlay->rl_dsttable == NULL)
-			continue;
-		imsg_compose_event(&c->iev, IMSG_CTL_TABLE, 0, 0, -1,
-		    rlay->rl_dsttable, sizeof(*rlay->rl_dsttable));
-		if (!(rlay->rl_dsttable->conf.flags & F_DISABLE))
-			TAILQ_FOREACH(host, &rlay->rl_dsttable->hosts, entry)
-				imsg_compose_event(&c->iev, IMSG_CTL_HOST,
-				    0, 0, -1, host, sizeof(*host));
-
-		if (rlay->rl_conf.backuptable == EMPTY_TABLE)
-			continue;
-		imsg_compose_event(&c->iev, IMSG_CTL_TABLE, 0, 0, -1,
-		    rlay->rl_backuptable, sizeof(*rlay->rl_backuptable));
-		if (!(rlay->rl_backuptable->conf.flags & F_DISABLE))
-			TAILQ_FOREACH(host, &rlay->rl_backuptable->hosts, entry)
-				imsg_compose_event(&c->iev, IMSG_CTL_HOST,
-				    0, 0, -1, host, sizeof(*host));
+		TAILQ_FOREACH(rlt, &rlay->rl_tables, rlt_entry) {
+			imsg_compose_event(&c->iev, IMSG_CTL_TABLE, 0, 0, -1,
+			    rlt->rlt_table, sizeof(*rlt->rlt_table));
+			if (!(rlt->rlt_table->conf.flags & F_DISABLE))
+				TAILQ_FOREACH(host,
+				    &rlt->rlt_table->hosts, entry)
+					imsg_compose_event(&c->iev,
+					    IMSG_CTL_HOST, 0, 0, -1,
+					    host, sizeof(*host));
+		}
 	}
 
 routers:
diff --git a/usr.sbin/relayd/relay.c b/usr.sbin/relayd/relay.c
index 79ff596d66f..47d279c3457 100644
--- a/usr.sbin/relayd/relay.c
+++ b/usr.sbin/relayd/relay.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: relay.c,v 1.153 2012/09/21 09:56:27 benno Exp $	*/
+/*	$OpenBSD: relay.c,v 1.154 2012/10/03 08:33:31 reyk Exp $	*/
 
 /*
  * Copyright (c) 2006 - 2012 Reyk Floeter <reyk@openbsd.org>
@@ -374,40 +374,41 @@ relay_statistics(int fd, short events, void *arg)
 void
 relay_launch(void)
 {
-	void		(*callback)(int, short, void *);
-	struct relay	*rlay;
-	struct host	*host;
+	void			(*callback)(int, short, void *);
+	struct relay		*rlay;
+	struct host		*host;
+	struct relay_table	*rlt;
 
 	TAILQ_FOREACH(rlay, env->sc_relays, rl_entry) {
 		if ((rlay->rl_conf.flags & (F_SSL|F_SSLCLIENT)) &&
 		    (rlay->rl_ssl_ctx = relay_ssl_ctx_create(rlay)) == NULL)
 			fatal("relay_init: failed to create SSL context");
 
-		if (rlay->rl_dsttable != NULL) {
-			switch (rlay->rl_conf.dstmode) {
+		TAILQ_FOREACH(rlt, &rlay->rl_tables, rlt_entry) {
+			switch (rlt->rlt_mode) {
 			case RELAY_DSTMODE_ROUNDROBIN:
-				rlay->rl_dstkey = 0;
+				rlt->rlt_key = 0;
 				break;
 			case RELAY_DSTMODE_LOADBALANCE:
 			case RELAY_DSTMODE_HASH:
-				rlay->rl_dstkey =
+				rlt->rlt_key =
 				    hash32_str(rlay->rl_conf.name, HASHINIT);
-				rlay->rl_dstkey =
-				    hash32_str(rlay->rl_dsttable->conf.name,
-				    rlay->rl_dstkey);
+				rlt->rlt_key =
+				    hash32_str(rlt->rlt_table->conf.name,
+				    rlt->rlt_key);
 				break;
 			}
-			rlay->rl_dstnhosts = 0;
-			TAILQ_FOREACH(host, &rlay->rl_dsttable->hosts, entry) {
-				if (rlay->rl_dstnhosts >= RELAY_MAXHOSTS)
+			rlt->rlt_nhosts = 0;
+			TAILQ_FOREACH(host, &rlt->rlt_table->hosts, entry) {
+				if (rlt->rlt_nhosts >= RELAY_MAXHOSTS)
 					fatal("relay_init: "
 					    "too many hosts in table");
-				host->idx = rlay->rl_dstnhosts;
-				rlay->rl_dsthost[rlay->rl_dstnhosts++] = host;
+				host->idx = rlt->rlt_nhosts;
+				rlt->rlt_host[rlt->rlt_nhosts++] = host;
 			}
 			log_info("adding %d hosts from table %s%s",
-			    rlay->rl_dstnhosts, rlay->rl_dsttable->conf.name,
-			    rlay->rl_dsttable->conf.check ? "" : " (no check)");
+			    rlt->rlt_nhosts, rlt->rlt_table->conf.name,
+			    rlt->rlt_table->conf.check ? "" : " (no check)");
 		}
 
 		switch (rlay->rl_proto->type) {
@@ -976,7 +977,6 @@ relay_accept(int fd, short event, void *arg)
 	con->se_id = ++relay_conid;
 	con->se_relayid = rlay->rl_conf.id;
 	con->se_pid = getpid();
-	con->se_hashkey = rlay->rl_dstkey;
 	con->se_in.tree = &proto->request_tree;
 	con->se_out.tree = &proto->response_tree;
 	con->se_in.dir = RELAY_DIR_REQUEST;
@@ -1104,22 +1104,46 @@ relay_from_table(struct rsession *con)
 {
 	struct relay		*rlay = (struct relay *)con->se_relay;
 	struct host		*host;
-	struct table		*table = rlay->rl_dsttable;
+	struct relay_table	*rlt = NULL;
+	struct table		*table = NULL;
 	u_int32_t		 p = con->se_hashkey;
 	int			 idx = 0;
 
-	if (table->conf.check && !table->up && !rlay->rl_backuptable->up) {
-		log_debug("%s: no active hosts", __func__);
+	/* the table is already selected */
+	if (con->se_table != NULL) {
+		rlt = con->se_table;
+		table = rlt->rlt_table;
+		if (table->conf.check && !table->up)
+			table = NULL;
+		goto gottable;
+	}
+
+	/* otherwise grep the first active table */
+	TAILQ_FOREACH(rlt, &rlay->rl_tables, rlt_entry) {
+		table = rlt->rlt_table;
+		if ((rlt->rlt_flags & F_USED == 0) ||
+		    (table->conf.check && !table->up))
+			table = NULL;
+		else
+			break;
+	}
+
+ gottable:
+	if (table == NULL) {
+		log_debug("%s: session %d: no active hosts",
+		    __func__, con->se_id);
 		return (-1);
-	} else if (!table->up && rlay->rl_backuptable->up) {
-		table = rlay->rl_backuptable;
+	}
+	if (!con->se_hashkeyset) {
+		p = con->se_hashkey = rlt->rlt_key;
+		con->se_hashkeyset = 1;
 	}
 
-	switch (rlay->rl_conf.dstmode) {
+	switch (rlt->rlt_mode) {
 	case RELAY_DSTMODE_ROUNDROBIN:
-		if ((int)rlay->rl_dstkey >= rlay->rl_dstnhosts)
-			rlay->rl_dstkey = 0;
-		idx = (int)rlay->rl_dstkey;
+		if ((int)rlt->rlt_key >= rlt->rlt_nhosts)
+			rlt->rlt_key = 0;
+		idx = (int)rlt->rlt_key;
 		break;
 	case RELAY_DSTMODE_LOADBALANCE:
 		p = relay_hash_addr(&con->se_in.ss, p);
@@ -1128,14 +1152,15 @@ relay_from_table(struct rsession *con)
 		p = relay_hash_addr(&rlay->rl_conf.ss, p);
 		p = hash32_buf(&rlay->rl_conf.port,
 		    sizeof(rlay->rl_conf.port), p);
-		if ((idx = p % rlay->rl_dstnhosts) >= RELAY_MAXHOSTS)
+		if ((idx = p % rlt->rlt_nhosts) >= RELAY_MAXHOSTS)
 			return (-1);
 	}
-	host = rlay->rl_dsthost[idx];
-	DPRINTF("%s: host %s, p 0x%08x, idx %d", __func__,
-	    host->conf.name, p, idx);
+	host = rlt->rlt_host[idx];
+	DPRINTF("%s: session %d: table %s host %s, p 0x%08x, idx %d",
+	    __func__, con->se_id, table->conf.name, host->conf.name, p, idx);
 	while (host != NULL) {
-		DPRINTF("%s: host %s", __func__, host->conf.name);
+		DPRINTF("%s: session %d: host %s", __func__,
+		    con->se_id, host->conf.name);
 		if (!table->conf.check || host->up == HOST_UP)
 			goto found;
 		host = TAILQ_NEXT(host, entry);
@@ -1150,8 +1175,8 @@ relay_from_table(struct rsession *con)
 	fatalx("relay_from_table: no active hosts, desynchronized");
 
  found:
-	if (rlay->rl_conf.dstmode == RELAY_DSTMODE_ROUNDROBIN)
-		rlay->rl_dstkey = host->idx + 1;
+	if (rlt->rlt_mode == RELAY_DSTMODE_ROUNDROBIN)
+		rlt->rlt_key = host->idx + 1;
 	con->se_retry = host->conf.retry;
 	con->se_out.port = table->conf.port;
 	bcopy(&host->conf.ss, &con->se_out.ss, sizeof(con->se_out.ss));
@@ -1171,7 +1196,7 @@ relay_natlook(int fd, short event, void *arg)
 
 	if (con->se_out.ss.ss_family == AF_UNSPEC && cnl->in == -1 &&
 	    rlay->rl_conf.dstss.ss_family == AF_UNSPEC &&
-	    rlay->rl_dsttable == NULL) {
+	    TAILQ_EMPTY(&rlay->rl_tables)) {
 		relay_close(con, "session NAT lookup failed");
 		return;
 	}
@@ -1355,7 +1380,7 @@ relay_connect(struct rsession *con)
 	if (gettimeofday(&con->se_tv_start, NULL) == -1)
 		return (-1);
 
-	if (rlay->rl_dsttable != NULL) {
+	if (!TAILQ_EMPTY(&rlay->rl_tables)) {
 		if (relay_from_table(con) != 0)
 			return (-1);
 	} else if (con->se_out.ss.ss_family == AF_UNSPEC) {
@@ -1691,6 +1716,9 @@ relay_dispatch_parent(int fd, struct privsep_proc *p, struct imsg *imsg)
 	case IMSG_CFG_RELAY:
 		config_getrelay(env, imsg);
 		break;
+	case IMSG_CFG_RELAY_TABLE:
+		config_getrelaytable(env, imsg);
+		break;
 	case IMSG_CFG_DONE:
 		config_getcfg(env, imsg);
 		break;
diff --git a/usr.sbin/relayd/relay_udp.c b/usr.sbin/relayd/relay_udp.c
index 3fe658a3f0b..3f760139ac4 100644
--- a/usr.sbin/relayd/relay_udp.c
+++ b/usr.sbin/relayd/relay_udp.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: relay_udp.c,v 1.24 2011/05/09 12:08:47 reyk Exp $	*/
+/*	$OpenBSD: relay_udp.c,v 1.25 2012/10/03 08:33:31 reyk Exp $	*/
 
 /*
  * Copyright (c) 2007, 2008 Reyk Floeter <reyk@openbsd.org>
@@ -249,7 +249,6 @@ relay_udp_server(int fd, short sig, void *arg)
 	con->se_in.con = con;
 	con->se_out.con = con;
 	con->se_relay = rlay;
-	con->se_hashkey = rlay->rl_dstkey;
 	con->se_id = ++relay_conid;
 	con->se_in.tree = &proto->request_tree;
 	con->se_out.tree = &proto->response_tree;
@@ -474,7 +473,7 @@ relay_dns_request(struct rsession *con)
 	if (gettimeofday(&con->se_tv_start, NULL) == -1)
 		return (-1);
 
-	if (rlay->rl_dsttable != NULL) {
+	if (!TAILQ_EMPTY(&rlay->rl_tables)) {
 		if (relay_from_table(con) != 0)
 			return (-1);
 	} else if (con->se_out.ss.ss_family == AF_UNSPEC) {
diff --git a/usr.sbin/relayd/relayd.c b/usr.sbin/relayd/relayd.c
index e8c2c5aba1f..ccc37788912 100644
--- a/usr.sbin/relayd/relayd.c
+++ b/usr.sbin/relayd/relayd.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: relayd.c,v 1.109 2012/09/21 09:56:27 benno Exp $	*/
+/*	$OpenBSD: relayd.c,v 1.110 2012/10/03 08:33:31 reyk Exp $	*/
 
 /*
  * Copyright (c) 2007, 2008 Reyk Floeter <reyk@openbsd.org>
@@ -564,6 +564,7 @@ void
 purge_relay(struct relayd *env, struct relay *rlay)
 {
 	struct rsession		*con;
+	struct relay_table	*rlt;
 
 	/* shutdown and remove relay */
 	if (event_initialized(&rlay->rl_ev))
@@ -591,6 +592,11 @@ purge_relay(struct relayd *env, struct relay *rlay)
 	if (rlay->rl_ssl_ca != NULL)
 		free(rlay->rl_ssl_ca);
 
+	while ((rlt = TAILQ_FIRST(&rlay->rl_tables))) {
+		TAILQ_REMOVE(&rlay->rl_tables, rlt, rlt_entry);
+		free(rlt);
+	}
+
 	free(rlay);
 }
 
diff --git a/usr.sbin/relayd/relayd.conf.5 b/usr.sbin/relayd/relayd.conf.5
index 3d1ec51bd73..1dce6f56e48 100644
--- a/usr.sbin/relayd/relayd.conf.5
+++ b/usr.sbin/relayd/relayd.conf.5
@@ -1,4 +1,4 @@
-.\"	$OpenBSD: relayd.conf.5,v 1.129 2012/08/24 20:13:03 jmc Exp $
+.\"	$OpenBSD: relayd.conf.5,v 1.130 2012/10/03 08:33:31 reyk Exp $
 .\"
 .\" Copyright (c) 2006, 2007 Reyk Floeter <reyk@openbsd.org>
 .\" Copyright (c) 2006, 2007 Pierre-Yves Ritschard <pyr@openbsd.org>
@@ -15,7 +15,7 @@
 .\" ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
 .\" OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 .\"
-.Dd $Mdocdate: August 24 2012 $
+.Dd $Mdocdate: October 3 2012 $
 .Dt RELAYD.CONF 5
 .Os
 .Sh NAME
@@ -596,8 +596,9 @@ Like the previous directive, but connect to a host from the specified
 table; see the
 .Sx TABLES
 section above for information about table options.
-This directive can be specified twice \(en the second entry will be used
-as the backup table if all hosts in the main table are down.
+This directive can be specified multiple times \(en subsequent entries
+will be used as the backup table if all hosts in the previous table
+are down.
 At least one entry for the main table is mandatory.
 .It Xo
 .Ic forward to
diff --git a/usr.sbin/relayd/relayd.h b/usr.sbin/relayd/relayd.h
index 5791f54287e..6b91f98814e 100644
--- a/usr.sbin/relayd/relayd.h
+++ b/usr.sbin/relayd/relayd.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: relayd.h,v 1.158 2012/09/21 09:56:27 benno Exp $	*/
+/*	$OpenBSD: relayd.h,v 1.159 2012/10/03 08:33:31 reyk Exp $	*/
 
 /*
  * Copyright (c) 2006 - 2012 Reyk Floeter <reyk@openbsd.org>
@@ -108,6 +108,12 @@ struct ctl_id {
 	char		 name[MAX_NAME_SIZE];
 };
 
+struct ctl_relaytable {
+	objid_t		 id;
+	objid_t		 relayid;
+	int		 mode;
+};
+
 struct ctl_script {
 	objid_t		 host;
 	int		 retval;
@@ -430,6 +436,8 @@ struct rsession {
 	struct ctl_relay_event		 se_out;
 	void				*se_priv;
 	u_int32_t			 se_hashkey;
+	int				 se_hashkeyset;
+	struct relay_table		*se_table;
 	struct event			 se_ev;
 	struct timeval			 se_timeout;
 	struct timeval			 se_tv_start;
@@ -580,6 +588,17 @@ struct protocol {
 };
 TAILQ_HEAD(protolist, protocol);
 
+struct relay_table {
+	struct table		*rlt_table;
+	u_int32_t		 rlt_flags;
+	int			 rlt_mode;
+	u_int32_t		 rlt_key;
+	struct host		*rlt_host[RELAY_MAXHOSTS];
+	int			 rlt_nhosts;
+	TAILQ_ENTRY(relay_table) rlt_entry;
+};
+TAILQ_HEAD(relaytables, relay_table);
+
 struct relay_config {
 	objid_t			 id;
 	u_int32_t		 flags;
@@ -588,10 +607,7 @@ struct relay_config {
 	char			 ifname[IFNAMSIZ];
 	in_port_t		 port;
 	in_port_t		 dstport;
-	int			 dstmode;
 	int			 dstretry;
-	objid_t			 dsttable;
-	objid_t			 backuptable;
 	struct sockaddr_storage	 ss;
 	struct sockaddr_storage	 dstss;
 	struct sockaddr_storage	 dstaf;
@@ -614,11 +630,7 @@ struct relay {
 	int			 rl_dsts;
 	struct bufferevent	*rl_dstbev;
 
-	struct table		*rl_dsttable;
-	struct table		*rl_backuptable;
-	u_int32_t		 rl_dstkey;
-	struct host		*rl_dsthost[RELAY_MAXHOSTS];
-	int			 rl_dstnhosts;
+	struct relaytables	 rl_tables;
 
 	struct event		 rl_ev;
 	struct event		 rl_evt;
@@ -789,6 +801,7 @@ enum imsg_type {
 	IMSG_CFG_PROTO,
 	IMSG_CFG_PROTONODE,
 	IMSG_CFG_RELAY,
+	IMSG_CFG_RELAY_TABLE,
 	IMSG_CFG_DONE
 };
 
@@ -1148,5 +1161,6 @@ int	 config_setprotonode(struct relayd *, enum privsep_procid,
 int	 config_getprotonode(struct relayd *, struct imsg *);
 int	 config_setrelay(struct relayd *env, struct relay *);
 int	 config_getrelay(struct relayd *, struct imsg *);
+int	 config_getrelaytable(struct relayd *, struct imsg *);
 
 #endif /* _RELAYD_H */