add an alternative "route to" mode to relayd redirections which maps

to pf route-to instead of the default rdr. it is a first steps towards
support for "direct server return" (dsr), an asynchronous mode where
the load balanced servers send the replies to a different gateway like
a l3 switch/router to handle higher amounts of return traffic.
because the state handling in pf isn't optimal for this case yet, it
just sees half of the TCP connection, the sessions are forced to time
out after fixed number of seconds.

discussed with many, thought about in the onsen

5 files changed, 97 insertions, 30 deletions

diff --git a/usr.sbin/relayd/parse.y b/usr.sbin/relayd/parse.y
index 85970a84a46..e4a49edcab0 100644
--- a/usr.sbin/relayd/parse.y
+++ b/usr.sbin/relayd/parse.y
@@ -1,4 +1,4 @@
-/*	$OpenBSD: parse.y,v 1.112 2008/05/06 12:58:00 reyk Exp $	*/
+/*	$OpenBSD: parse.y,v 1.113 2008/05/07 01:49:29 reyk Exp $	*/
 
 /*
  * Copyright (c) 2007, 2008 Reyk Floeter <reyk@openbsd.org>
@@ -130,12 +130,12 @@ typedef struct {
 %token	ON PATH PORT PREFORK PROTO QUERYSTR REAL REDIRECT RELAY REMOVE TRAP
 %token	REQUEST RESPONSE RETRY RETURN ROUNDROBIN SACK SCRIPT SEND SESSION
 %token	SOCKET SSL STICKYADDR STYLE TABLE TAG TCP TIMEOUT TO UPDATES URL
-%token	VIRTUAL WITH ERROR
+%token	VIRTUAL WITH ERROR ROUTE
 %token	<v.string>	STRING
 %token  <v.number>	NUMBER
 %type	<v.string>	interface hostname table
 %type	<v.number>	port http_type loglevel sslcache optssl mark
-%type	<v.number>	proto_type dstmode retry log flag direction
+%type	<v.number>	proto_type dstmode retry log flag direction forwardmode
 %type	<v.host>	host
 %type	<v.tv>		timeout
 %type	<v.digest>	digest
@@ -337,6 +337,7 @@ rdr		: REDIRECT STRING	{
 			}
 			free($2);
 			srv->conf.id = last_rdr_id++;
+			srv->conf.timeout.tv_sec = RELAY_TIMEOUT;
 			if (last_rdr_id == INT_MAX) {
 				yyerror("too many redirections defined");
 				free(srv);
@@ -378,7 +379,20 @@ rdropts_l	: rdropts_l rdroptsl nl
 		| rdroptsl optnl
 		;
 
-rdroptsl	: FORWARD TO tablespec		{
+rdroptsl	: forwardmode TO tablespec interface	{
+			if ($4 != NULL) {
+				strlcpy($3->conf.ifname, $4,
+				    sizeof($3->conf.ifname));
+				free($4);
+				if (($1 & F_ROUTE) == 0) {
+					yyerror("superfluous interface");
+					YYERROR;
+				}
+			} else if ($1 & F_ROUTE) {
+				yyerror("missing interface to route to");
+				YYERROR;
+			}
+
 			if ($3->conf.check == CHECK_NOCHECK) {
 				yyerror("table %s has no check", $3->conf.name);
 				purge_table(conf->sc_tables, $3);
@@ -397,7 +411,7 @@ rdroptsl	: FORWARD TO tablespec		{
 				rdr->conf.table_id = $3->conf.id;
 			}
 			$3->conf.rdrid = rdr->conf.id;
-			$3->conf.flags |= F_USED;
+			$3->conf.flags |= F_USED | $1;
 		}
 		| LISTEN ON STRING port interface {
 			if (host($3, &rdr->virts,
@@ -426,9 +440,19 @@ rdroptsl	: FORWARD TO tablespec		{
 			}
 			free($2);
 		}
+		| SESSION TIMEOUT NUMBER		{
+			if ((rdr->conf.timeout.tv_sec = $3) < 0) {
+				yyerror("invalid timeout: %d", $3);
+				YYERROR;
+			}
+		}
 		| include
 		;
 
+forwardmode	: FORWARD		{ $$ = 0; }
+		| ROUTE			{ $$ = F_ROUTE; }
+		;
+
 table		: '<' STRING '>'	{
 			conf->sc_flags |= F_NEEDPF;
 			if (strlen($2) >= TABLE_NAME_SIZE) {
@@ -1344,6 +1368,7 @@ lookup(char *s)
 		{ "retry",		RETRY },
 		{ "return",		RETURN },
 		{ "roundrobin",		ROUNDROBIN },
+		{ "route",		ROUTE },
 		{ "sack",		SACK },
 		{ "script",		SCRIPT },
 		{ "send",		SEND },
diff --git a/usr.sbin/relayd/pfe_filter.c b/usr.sbin/relayd/pfe_filter.c
index 514a78140f9..81d0444510d 100644
--- a/usr.sbin/relayd/pfe_filter.c
+++ b/usr.sbin/relayd/pfe_filter.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: pfe_filter.c,v 1.25 2008/05/06 11:52:49 reyk Exp $	*/
+/*	$OpenBSD: pfe_filter.c,v 1.26 2008/05/07 01:49:29 reyk Exp $	*/
 
 /*
  * Copyright (c) 2006 Pierre-Yves Ritschard <pyr@openbsd.org>
@@ -41,8 +41,9 @@
 struct pfdata {
 	int			 dev;
 	struct pf_anchor	*anchor;
-	struct pfioc_trans	 pft;
-	struct pfioc_trans_e	 pfte;
+	struct pfioc_trans	 pft[PF_RULESET_MAX];
+	struct pfioc_trans_e	 pfte[PF_RULESET_MAX];
+	u_int8_t		 pfused;
 };
 
 int	 transaction_init(struct relayd *, const char *);
@@ -273,24 +274,35 @@ flush_table(struct relayd *env, struct rdr *rdr)
 int
 transaction_init(struct relayd *env, const char *anchor)
 {
-	env->sc_pf->pft.size = 1;
-	env->sc_pf->pft.esize = sizeof env->sc_pf->pfte;
-	env->sc_pf->pft.array = &env->sc_pf->pfte;
-
-	memset(&env->sc_pf->pfte, 0, sizeof env->sc_pf->pfte);
-	(void)strlcpy(env->sc_pf->pfte.anchor, anchor, PF_ANCHOR_NAME_SIZE);
-	env->sc_pf->pfte.rs_num = PF_RULESET_RDR;
-
-	if (ioctl(env->sc_pf->dev, DIOCXBEGIN, &env->sc_pf->pft) == -1)
-		return (-1);
+	int i;
+
+	for (i = 0; i < PF_RULESET_MAX; i++) {
+		env->sc_pf->pft[i].size = 1;
+		env->sc_pf->pft[i].esize = sizeof(env->sc_pf->pfte[i]);
+		env->sc_pf->pft[i].array = &env->sc_pf->pfte[i];
+	
+		bzero(&env->sc_pf->pfte[i], sizeof(env->sc_pf->pfte[i]));
+		(void)strlcpy(env->sc_pf->pfte[i].anchor,
+		    anchor, PF_ANCHOR_NAME_SIZE);
+		env->sc_pf->pfte[i].rs_num = i;
+
+		if (ioctl(env->sc_pf->dev, DIOCXBEGIN,
+		    &env->sc_pf->pft[i]) == -1)
+			return (-1);
+	}
 	return (0);
 }
 
 int
 transaction_commit(struct relayd *env)
 {
-	if (ioctl(env->sc_pf->dev, DIOCXCOMMIT, &env->sc_pf->pft) == -1)
-		return (-1);
+	int i;
+
+	for (i = 0; i < PF_RULESET_MAX; i++) {
+		if (ioctl(env->sc_pf->dev, DIOCXCOMMIT,
+		    &env->sc_pf->pft[i]) == -1)
+			return (-1);
+	}
 	return (0);
 }
 
@@ -303,6 +315,8 @@ sync_ruleset(struct relayd *env, struct rdr *rdr, int enable)
 	struct sockaddr_in6	*sain6;
 	struct address		*address;
 	char			 anchor[PF_ANCHOR_NAME_SIZE];
+	int			 rs;
+	struct table		*t = rdr->table;
 
 	if (!(env->sc_flags & F_NEEDPF))
 		return;
@@ -333,7 +347,25 @@ sync_ruleset(struct relayd *env, struct rdr *rdr, int enable)
 		memset(&pio, 0, sizeof(pio));
 		(void)strlcpy(rio.anchor, anchor, sizeof(rio.anchor));
 
-		rio.ticket = env->sc_pf->pfte.ticket;
+		if ((t->conf.flags & F_ROUTE) == 0) {
+			/* traditional redirection in the rdr-anchor */
+			rs = PF_RULESET_RDR;
+			rio.rule.action = PF_RDR;
+		} else {
+			/* re-route with pf for DSR (direct server return) */
+			rs = PF_RULESET_FILTER;
+			rio.rule.action = PF_PASS;
+			rio.rule.rt = PF_ROUTETO;
+			rio.rule.direction = PF_IN;
+			rio.rule.quick = 1; /* force first match */
+
+			/* XXX This should use a loose pf state handling */
+			rio.rule.keep_state = PF_STATE_NORMAL;
+			rio.rule.timeout[PFTM_TCP_OPENING] =
+			    rdr->conf.timeout.tv_sec;
+		}
+
+		rio.ticket = env->sc_pf->pfte[rs].ticket;
 		if (ioctl(env->sc_pf->dev, DIOCBEGINADDRS, &pio) == -1)
 			fatal("sync_ruleset: cannot initialise address pool");
 
@@ -345,7 +377,7 @@ sync_ruleset(struct relayd *env, struct rdr *rdr, int enable)
 		rio.rule.dst.port_op = PF_OP_EQ;
 		rio.rule.dst.port[0] = address->port;
 		rio.rule.rtableid = -1; /* stay in the main routing table */
-		rio.rule.action = PF_RDR;
+
 		if (strlen(rdr->conf.tag))
 			(void)strlcpy(rio.rule.tagname, rdr->conf.tag,
 			    sizeof(rio.rule.tagname));
@@ -359,7 +391,6 @@ sync_ruleset(struct relayd *env, struct rdr *rdr, int enable)
 			rio.rule.dst.addr.v.a.addr.addr32[0] =
 			    sain->sin_addr.s_addr;
 			rio.rule.dst.addr.v.a.mask.addr32[0] = 0xffffffff;
-
 		} else {
 			sain6 = (struct sockaddr_in6 *)&address->ss;
 
@@ -370,6 +401,9 @@ sync_ruleset(struct relayd *env, struct rdr *rdr, int enable)
 		}
 
 		pio.addr.addr.type = PF_ADDR_TABLE;
+		if (strlen(t->conf.ifname))
+			(void)strlcpy(pio.addr.ifname, t->conf.ifname,
+			    sizeof(pio.addr.ifname));
 		if (strlcpy(pio.addr.addr.v.tblname, rdr->conf.name,
 		    sizeof(pio.addr.addr.v.tblname)) >=
 		    sizeof(pio.addr.addr.v.tblname))
@@ -385,7 +419,8 @@ sync_ruleset(struct relayd *env, struct rdr *rdr, int enable)
 
 		if (ioctl(env->sc_pf->dev, DIOCADDRULE, &rio) == -1)
 			fatal("cannot add rule");
-		log_debug("sync_ruleset: rule added");
+		log_debug("sync_ruleset: rule added to %sanchor \"%s\"",
+		    rdr->table->conf.flags & F_ROUTE ? "" : "rdr-", anchor);
 	}
 	if (transaction_commit(env) == -1)
 		log_warn("sync_ruleset: add rules transaction failed");
diff --git a/usr.sbin/relayd/relayd.8 b/usr.sbin/relayd/relayd.8
index 2ef4940fbac..90082b6428b 100644
--- a/usr.sbin/relayd/relayd.8
+++ b/usr.sbin/relayd/relayd.8
@@ -1,4 +1,4 @@
-.\"	$OpenBSD: relayd.8,v 1.16 2007/12/12 14:55:12 jmc Exp $
+.\"	$OpenBSD: relayd.8,v 1.17 2008/05/07 01:49:29 reyk Exp $
 .\"
 .\" Copyright (c) 2006 Pierre-Yves Ritschard <pyr@openbsd.org>
 .\"
@@ -14,7 +14,7 @@
 .\" ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
 .\" OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 .\"
-.Dd $Mdocdate: December 12 2007 $
+.Dd $Mdocdate: May 7 2008 $
 .Dt RELAYD 8
 .Os
 .Sh NAME
@@ -49,10 +49,11 @@ To allow
 .Nm
 to properly set up
 .Xr pf 4
-rules, the following line is required in the NAT section of
+rules, the following lines are required in the NAT and filter sections of
 .Xr pf.conf 5 :
 .Bd -literal -offset indent
 rdr-anchor "relayd/*"
+anchor "relayd/*"
 .Ed
 .Pp
 Layer 7 relaying happens at the application level and is
diff --git a/usr.sbin/relayd/relayd.conf.5 b/usr.sbin/relayd/relayd.conf.5
index c7a9c6c43d9..687d10fe6e3 100644
--- a/usr.sbin/relayd/relayd.conf.5
+++ b/usr.sbin/relayd/relayd.conf.5
@@ -1,4 +1,4 @@
-.\"	$OpenBSD: relayd.conf.5,v 1.83 2008/05/06 16:23:52 jmc Exp $
+.\"	$OpenBSD: relayd.conf.5,v 1.84 2008/05/07 01:49:29 reyk Exp $
 .\"
 .\" Copyright (c) 2006, 2007 Reyk Floeter <reyk@openbsd.org>
 .\" Copyright (c) 2006, 2007 Pierre-Yves Ritschard <pyr@openbsd.org>
@@ -15,7 +15,7 @@
 .\" ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
 .\" OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 .\"
-.Dd $Mdocdate: May 6 2008 $
+.Dd $Mdocdate: May 7 2008 $
 .Dt RELAYD.CONF 5
 .Os
 .Sh NAME
@@ -428,6 +428,9 @@ for an rdr rule in
 .Xr pf.conf 5 .
 It will ensure that multiple connections from the same source are
 mapped to the same redirection address.
+.It Ic session timeout Ar seconds
+Specify the timeout in seconds for routed session states.
+The default timeout is 600 seconds (10 minutes).
 .It Ic tag Ar name
 Automatically tag packets passing through the
 .Xr pf 4
diff --git a/usr.sbin/relayd/relayd.h b/usr.sbin/relayd/relayd.h
index 50da2c8603b..e6cd963a3bf 100644
--- a/usr.sbin/relayd/relayd.h
+++ b/usr.sbin/relayd/relayd.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: relayd.h,v 1.100 2008/05/06 06:09:48 pyr Exp $	*/
+/*	$OpenBSD: relayd.h,v 1.101 2008/05/07 01:49:29 reyk Exp $	*/
 
 /*
  * Copyright (c) 2006, 2007 Pierre-Yves Ritschard <pyr@openbsd.org>
@@ -319,6 +319,7 @@ TAILQ_HEAD(addresslist, address);
 #define F_RETURN		0x00020000
 #define F_TRAP			0x00040000
 #define F_NEEDPF		0x00080000
+#define F_ROUTE			0x00100000
 
 struct host_config {
 	objid_t			 id;
@@ -362,6 +363,7 @@ struct table_config {
 	u_int32_t		 flags;
 	int			 check;
 	char			 demote_group[IFNAMSIZ];
+	char			 ifname[IFNAMSIZ];
 	struct timeval		 timeout;
 	in_port_t		 port;
 	int			 retcode;
@@ -403,6 +405,7 @@ struct rdr_config {
 	objid_t			 backup_id;
 	char			 name[SRV_NAME_SIZE];
 	char			 tag[TAG_NAME_SIZE];
+	struct timeval		 timeout;
 };
 
 struct rdr {