Enable TCP socket splicing for HTTP persistent connection and chunked

transfer encoding.  This speeds up relayd for more protocol modes
by zero-copy TCP forwarding.
OK reyk@ benno@

3 files changed, 82 insertions, 12 deletions

diff --git a/usr.sbin/relayd/relay.c b/usr.sbin/relayd/relay.c
index b7df8059d5c..541d34a1f44 100644
--- a/usr.sbin/relayd/relay.c
+++ b/usr.sbin/relayd/relay.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: relay.c,v 1.162 2013/02/05 21:36:33 bluhm Exp $	*/
+/*	$OpenBSD: relay.c,v 1.163 2013/03/09 14:43:06 bluhm Exp $	*/
 
 /*
  * Copyright (c) 2006 - 2012 Reyk Floeter <reyk@openbsd.org>
@@ -70,9 +70,6 @@ void		 relay_input(struct rsession *);
 
 u_int32_t	 relay_hash_addr(struct sockaddr_storage *, u_int32_t);
 
-int		 relay_splice(struct ctl_relay_event *);
-int		 relay_splicelen(struct ctl_relay_event *);
-
 SSL_CTX		*relay_ssl_ctx_create(struct relay *);
 void		 relay_ssl_transaction(struct rsession *,
 		    struct ctl_relay_event *);
@@ -743,10 +740,19 @@ relay_write(struct bufferevent *bev, void *arg)
 {
 	struct ctl_relay_event	*cre = arg;
 	struct rsession		*con = cre->con;
+
 	if (gettimeofday(&con->se_tv_last, NULL) == -1)
-		con->se_done = 1;
+		goto fail;
 	if (con->se_done)
-		relay_close(con, "last write (done)");
+		goto done;
+	if (relay_splice(cre->dst) == -1)
+		goto fail;
+	return;
+ done:
+	relay_close(con, "last write (done)");
+	return;
+ fail:
+	relay_close(con, strerror(errno));
 }
 
 void
@@ -824,11 +830,31 @@ relay_splice(struct ctl_relay_event *cre)
 	    (proto->tcpflags & TCPFLAG_NSPLICE))
 		return (0);
 
-	if (cre->bev->readcb != relay_read)
+	if (cre->splicelen >= 0)
+		return (0);
+
+	/* still not connected */
+	if (cre->bev == NULL || cre->dst->bev == NULL)
+		return (0);
+
+	if (! (cre->toread == TOREAD_UNLIMITED || cre->toread > 0)) {
+		DPRINTF("%s: session %d: splice dir %d, nothing to read %lld",
+		    __func__, con->se_id, cre->dir, cre->toread);
 		return (0);
+	}
+
+	/* do not splice before buffers have not been completely flushed */
+	if (EVBUFFER_LENGTH(cre->bev->input) ||
+	    EVBUFFER_LENGTH(cre->dst->bev->output)) {
+		DPRINTF("%s: session %d: splice dir %d, dirty buffer",
+		    __func__, con->se_id, cre->dir);
+		bufferevent_disable(cre->bev, EV_READ);
+		return (0);
+	}
 
 	bzero(&sp, sizeof(sp));
 	sp.sp_fd = cre->dst->s;
+	sp.sp_max = cre->toread > 0 ? cre->toread : 0;
 	sp.sp_idle = rlay->rl_conf.timeout;
 	if (setsockopt(cre->s, SOL_SOCKET, SO_SPLICE, &sp, sizeof(sp)) == -1) {
 		log_debug("%s: session %d: splice dir %d failed: %s",
@@ -836,9 +862,12 @@ relay_splice(struct ctl_relay_event *cre)
 		return (-1);
 	}
 	cre->splicelen = 0;
-	DPRINTF("%s: session %d: splice dir %d successful",
-	    __func__, con->se_id, cre->dir);
-	return (1);
+	bufferevent_enable(cre->bev, EV_READ);
+
+	DPRINTF("%s: session %d: splice dir %d, maximum %lld, successful",
+	    __func__, con->se_id, cre->dir, cre->toread);
+
+	return (0);
 }
 
 int
@@ -848,16 +877,40 @@ relay_splicelen(struct ctl_relay_event *cre)
 	off_t			 len;
 	socklen_t		 optlen;
 
+	if (cre->splicelen < 0)
+		return (0);
+
 	optlen = sizeof(len);
 	if (getsockopt(cre->s, SOL_SOCKET, SO_SPLICE, &len, &optlen) == -1) {
 		log_debug("%s: session %d: splice dir %d get length failed: %s",
 		    __func__, con->se_id, cre->dir, strerror(errno));
 		return (-1);
 	}
+
+	DPRINTF("%s: session %d: splice dir %d, length %lld",
+	    __func__, con->se_id, cre->dir, len);
+
 	if (len > cre->splicelen) {
+		if (gettimeofday(&con->se_tv_last, NULL) == -1)
+			return (-1);
 		cre->splicelen = len;
 		return (1);
 	}
+
+	return (0);
+}
+
+int
+relay_spliceadjust(struct ctl_relay_event *cre)
+{
+	if (cre->splicelen < 0)
+		return (0);
+	if (relay_splicelen(cre) == -1)
+		return (-1);
+	if (cre->splicelen > 0 && cre->toread > 0)
+		cre->toread -= cre->splicelen;
+	cre->splicelen = -1;
+
 	return (0);
 }
 
@@ -900,10 +953,18 @@ relay_error(struct bufferevent *bev, short error, void *arg)
 				break;
 			}
 		}
+		if (relay_spliceadjust(cre) == -1)
+			goto fail;
 		if (relay_splice(cre) == -1)
 			goto fail;
 		return;
 	}
+	if (error & EVBUFFER_ERROR && errno == EFBIG) {
+		if (relay_spliceadjust(cre) == -1)
+			goto fail;
+		bufferevent_enable(cre->bev, EV_READ);
+		return;
+	}
 	if (error & (EVBUFFER_READ|EVBUFFER_WRITE|EVBUFFER_EOF)) {
 		bufferevent_disable(bev, EV_READ|EV_WRITE);
 
diff --git a/usr.sbin/relayd/relay_http.c b/usr.sbin/relayd/relay_http.c
index 4a8b3a4fbc9..cfbf7cb13fa 100644
--- a/usr.sbin/relayd/relay_http.c
+++ b/usr.sbin/relayd/relay_http.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: relay_http.c,v 1.9 2013/02/15 12:15:12 bluhm Exp $	*/
+/*	$OpenBSD: relay_http.c,v 1.10 2013/03/09 14:43:06 bluhm Exp $	*/
 
 /*
  * Copyright (c) 2006 - 2012 Reyk Floeter <reyk@openbsd.org>
@@ -375,6 +375,8 @@ relay_read_http(struct bufferevent *bev, void *arg)
 	if (EVBUFFER_LENGTH(src) && bev->readcb != relay_read_http)
 		bev->readcb(bev, arg);
 	bufferevent_enable(bev, EV_READ);
+	if (relay_splice(cre) == -1)
+		relay_close(con, strerror(errno));
 	return;
  fail:
 	relay_abort_http(con, 500, strerror(errno), 0);
@@ -398,6 +400,8 @@ relay_read_httpcontent(struct bufferevent *bev, void *arg)
 	    __func__, cre->dir, size, cre->toread);
 	if (!size)
 		return;
+	if (relay_spliceadjust(cre) == -1)
+		goto fail;
 
 	if (cre->toread > 0) {
 		/* Read content data */
@@ -449,6 +453,8 @@ relay_read_httpchunks(struct bufferevent *bev, void *arg)
 	    __func__, cre->dir, size, cre->toread);
 	if (!size)
 		return;
+	if (relay_spliceadjust(cre) == -1)
+		goto fail;
 
 	if (cre->toread > 0) {
 		/* Read chunk data */
diff --git a/usr.sbin/relayd/relayd.h b/usr.sbin/relayd/relayd.h
index 7c4d22dbc75..9a502648b44 100644
--- a/usr.sbin/relayd/relayd.h
+++ b/usr.sbin/relayd/relayd.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: relayd.h,v 1.165 2013/03/04 08:41:32 sthen Exp $	*/
+/*	$OpenBSD: relayd.h,v 1.166 2013/03/09 14:43:06 bluhm Exp $	*/
 
 /*
  * Copyright (c) 2006 - 2012 Reyk Floeter <reyk@openbsd.org>
@@ -995,6 +995,9 @@ int	 relay_cmp_af(struct sockaddr_storage *,
 	    struct sockaddr_storage *);
 void	 relay_write(struct bufferevent *, void *);
 void	 relay_read(struct bufferevent *, void *);
+int	 relay_splice(struct ctl_relay_event *);
+int	 relay_splicelen(struct ctl_relay_event *);
+int	 relay_spliceadjust(struct ctl_relay_event *);
 void	 relay_error(struct bufferevent *, short, void *);
 int	 relay_lognode(struct rsession *,
 	    struct protonode *, struct protonode *, char *, size_t);