file descriptor accounting for relays: track how many connections to

backend servers are unopened and reserve fds for them.
ok reyk@, "don't wait" deraadt@

3 files changed, 165 insertions, 18 deletions

diff --git a/usr.sbin/relayd/relay.c b/usr.sbin/relayd/relay.c
index c3ba78464c2..79ff596d66f 100644
--- a/usr.sbin/relayd/relay.c
+++ b/usr.sbin/relayd/relay.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: relay.c,v 1.152 2012/09/20 12:30:20 reyk Exp $	*/
+/*	$OpenBSD: relay.c,v 1.153 2012/09/21 09:56:27 benno Exp $	*/
 
 /*
  * Copyright (c) 2006 - 2012 Reyk Floeter <reyk@openbsd.org>
@@ -77,6 +77,7 @@ SSL_CTX		*relay_ssl_ctx_create(struct relay *);
 void		 relay_ssl_transaction(struct rsession *,
 		    struct ctl_relay_event *);
 void		 relay_ssl_accept(int, short, void *);
+void		 relay_connect_retry(int, short, void *);
 void		 relay_ssl_connect(int, short, void *);
 void		 relay_ssl_connected(struct ctl_relay_event *);
 void		 relay_ssl_readcb(int, short, void *);
@@ -88,7 +89,8 @@ static __inline int
 extern void	 bufferevent_read_pressure_cb(struct evbuffer *, size_t,
 		    size_t, void *);
 
-volatile sig_atomic_t relay_sessions;
+volatile int relay_sessions;
+volatile int relay_inflight = 0;
 objid_t relay_conid;
 
 static struct relayd		*env = NULL;
@@ -934,7 +936,8 @@ relay_accept(int fd, short event, void *arg)
 		return;
 
 	slen = sizeof(ss);
-	if ((s = accept(fd, (struct sockaddr *)&ss, (socklen_t *)&slen)) == -1) {
+	if ((s = accept_reserve(fd, (struct sockaddr *)&ss,
+	    (socklen_t *)&slen, FD_RESERVE, &relay_inflight)) == -1) {
 		/*
 		 * Pause accept if we are out of file descriptors, or
 		 * libevent will haunt us here too.
@@ -944,6 +947,8 @@ relay_accept(int fd, short event, void *arg)
 
 			event_del(&rlay->rl_ev);
 			evtimer_add(&rlay->rl_evt, &evtpause);
+			log_debug("%s: deferring connections",__func__,
+			    relay_inflight);
 		}
 		return;
 	}
@@ -1065,6 +1070,13 @@ relay_accept(int fd, short event, void *arg)
 		close(s);
 		if (con != NULL)
 			free(con);
+		/* 
+		 * the session struct was not completly set up, but still
+		 * counted as an inflight session. account for this.
+		 */
+		relay_inflight--;
+		log_debug("%s: inflight decremented, now %d",
+		    __func__, relay_inflight);
 	}
 }
 
@@ -1245,17 +1257,101 @@ relay_bindany(int fd, short event, void *arg)
 		relay_close(con, "bindany failed, invalid socket");
 		return;
 	}
-
 	if (relay_connect(con) == -1)
 		relay_close(con, "session failed");
 }
 
+void
+relay_connect_retry(int fd, short sig, void *arg)
+{
+	struct timeval	 evtpause = { 1, 0 };
+	struct rsession	*con = (struct rsession *)arg;
+	struct relay	*rlay = (struct relay *)con->se_relay;
+	int		 bnds = -1;
+
+	if (relay_inflight < 1)
+		fatalx("relay_connect_retry: no connection in flight");
+
+	DPRINTF("%s: retry %d of %d, inflight: %d",__func__,
+	    con->se_retrycount, con->se_retry, relay_inflight);
+
+	if (sig != EV_TIMEOUT)
+		fatalx("relay_connect_retry: called without timeout");
+
+	evtimer_del(&con->se_inflightevt);
+
+        /*
+	 * XXX we might want to check if the inbound socket is still
+	 * available: client could have closed it while we were waiting?
+	 */
+
+	DPRINTF("%s: got EV_TIMEOUT", __func__);
+
+	if (getdtablecount() + FD_RESERVE +
+	    relay_inflight > getdtablesize()) {
+		if (con->se_retrycount < RELAY_OUTOF_FD_RETRIES) {
+			evtimer_add(&con->se_inflightevt, &evtpause);
+			return;
+		}
+		/* we waited for RELAY_OUTOF_FD_RETRIES seconds, give up */
+		event_add(&rlay->rl_ev, NULL);
+		relay_abort_http(con, 504, "connection timed out", 0);		
+		return;
+	}
+
+	if (rlay->rl_conf.fwdmode == FWD_TRANS) {
+		/* con->se_bnds cannot be unset */
+		bnds = con->se_bnds;
+	}
+
+ retry:
+	if ((con->se_out.s = relay_socket_connect(&con->se_out.ss,
+	    con->se_out.port, rlay->rl_proto, bnds)) == -1) {
+		log_debug("%s: session %d: "
+		    "forward failed: %s, %s", __func__,
+		    con->se_id, strerror(errno),
+		    con->se_retry ? "next retry" : "last retry");
+
+		con->se_retrycount++;
+
+		if ((errno == ENFILE || errno == EMFILE) && 
+		    (con->se_retrycount < con->se_retry)) {
+			event_del(&rlay->rl_ev);
+			evtimer_add(&con->se_inflightevt, &evtpause);
+			evtimer_add(&rlay->rl_evt, &evtpause);
+			return;
+		} else if (con->se_retrycount < con->se_retry)
+			goto retry;
+		event_add(&rlay->rl_ev, NULL);
+		relay_abort_http(con, 504, "connect failed", 0);
+		return;
+	}
+
+	relay_inflight--;
+	DPRINTF("%s: inflight decremented, now %d",__func__, relay_inflight);
+
+	event_add(&rlay->rl_ev, NULL);
+
+	if (errno == EINPROGRESS)
+		event_again(&con->se_ev, con->se_out.s, EV_WRITE|EV_TIMEOUT,
+		    relay_connected, &con->se_tv_start, &rlay->rl_conf.timeout,
+		    con);
+	else
+		relay_connected(con->se_out.s, EV_WRITE, con);
+
+	return;
+}
+
 int
 relay_connect(struct rsession *con)
 {
 	struct relay	*rlay = (struct relay *)con->se_relay;
+	struct timeval	 evtpause = { 1, 0 };
 	int		 bnds = -1, ret;
 
+	if (relay_inflight < 1)
+		fatalx("relay_connect: no connection in flight");
+
 	if (gettimeofday(&con->se_tv_start, NULL) == -1)
 		return (-1);
 
@@ -1295,18 +1391,33 @@ relay_connect(struct rsession *con)
  retry:
 	if ((con->se_out.s = relay_socket_connect(&con->se_out.ss,
 	    con->se_out.port, rlay->rl_proto, bnds)) == -1) {
-		if (con->se_retry) {
-			con->se_retry--;
-			log_debug("%s: session %d: "
-			    "forward failed: %s, %s", __func__,
-			    con->se_id, strerror(errno),
-			    con->se_retry ? "next retry" : "last retry");
-			goto retry;
+		if (errno == ENFILE || errno == EMFILE) {
+			log_debug("%s: session %d: forward failed: %s",
+			    __func__, con->se_id, strerror(errno));
+			evtimer_set(&con->se_inflightevt, relay_connect_retry,
+			    con);
+			event_del(&rlay->rl_ev);
+			evtimer_add(&con->se_inflightevt, &evtpause);
+			evtimer_add(&rlay->rl_evt, &evtpause);
+			return (0);
+		} else {
+			if (con->se_retry) {
+				con->se_retry--;
+				log_debug("%s: session %d: "
+				    "forward failed: %s, %s", __func__,
+				    con->se_id, strerror(errno),
+				    con->se_retry ? "next retry" : "last retry");
+				goto retry;
+			}
+			log_debug("%s: session %d: forward failed: %s", __func__,
+			    con->se_id, strerror(errno));
+			return (-1);
 		}
-		log_debug("%s: session %d: forward failed: %s", __func__,
-		    con->se_id, strerror(errno));
-		return (-1);
-	}
+	}	
+
+	relay_inflight--;
+	DPRINTF("%s: inflight decremented, now %d",__func__,
+	    relay_inflight);
 
 	if (errno == EINPROGRESS)
 		event_again(&con->se_ev, con->se_out.s, EV_WRITE|EV_TIMEOUT,
@@ -1361,8 +1472,18 @@ relay_close(struct rsession *con, const char *msg)
 			SSL_shutdown(con->se_in.ssl);
 		SSL_free(con->se_in.ssl);
 	}
-	if (con->se_in.s != -1)
+	if (con->se_in.s != -1) {
 		close(con->se_in.s);
+		if (con->se_out.s == -1) {
+			/* 
+			 * the output was never connected,
+			 * thus this was an inflight session.
+			 */
+			relay_inflight--;
+			log_debug("%s: sessions inflight decremented, now %d",
+			    __func__, relay_inflight);
+		}
+	}
 	if (con->se_in.path != NULL)
 		free(con->se_in.path);
 	if (con->se_in.buf != NULL)
diff --git a/usr.sbin/relayd/relayd.c b/usr.sbin/relayd/relayd.c
index b369e2549de..e8c2c5aba1f 100644
--- a/usr.sbin/relayd/relayd.c
+++ b/usr.sbin/relayd/relayd.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: relayd.c,v 1.108 2012/05/08 15:10:15 benno Exp $	*/
+/*	$OpenBSD: relayd.c,v 1.109 2012/09/21 09:56:27 benno Exp $	*/
 
 /*
  * Copyright (c) 2007, 2008 Reyk Floeter <reyk@openbsd.org>
@@ -1220,3 +1220,22 @@ get_data(u_int8_t *ptr, size_t len)
 
 	return (data);
 }
+
+int
+accept_reserve(int sockfd, struct sockaddr *addr, socklen_t *addrlen,
+    int reserve, volatile int *counter)
+{
+	int ret;
+	if (getdtablecount() + reserve +
+	    *counter >= getdtablesize()) {
+		errno = EMFILE;
+		return -1;
+	}
+
+	if ((ret = accept(sockfd, addr, addrlen)) > -1) {
+		(*counter)++;
+		log_debug("%s: inflight incremented, now %d",__func__,
+		    *counter);
+	}
+	return ret;
+}
diff --git a/usr.sbin/relayd/relayd.h b/usr.sbin/relayd/relayd.h
index 0667258ae35..5791f54287e 100644
--- a/usr.sbin/relayd/relayd.h
+++ b/usr.sbin/relayd/relayd.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: relayd.h,v 1.157 2012/09/20 12:30:20 reyk Exp $	*/
+/*	$OpenBSD: relayd.h,v 1.158 2012/09/21 09:56:27 benno Exp $	*/
 
 /*
  * Copyright (c) 2006 - 2012 Reyk Floeter <reyk@openbsd.org>
@@ -48,6 +48,8 @@
 #define MAX_NAME_SIZE		64
 #define SRV_MAX_VIRTS		16
 
+#define FD_RESERVE		5
+
 #define RELAY_MAX_SESSIONS	1024
 #define RELAY_TIMEOUT		600
 #define RELAY_CACHESIZE		-1	/* use default size */
@@ -57,6 +59,7 @@
 #define RELAY_STATINTERVAL	60
 #define RELAY_BACKLOG		10
 #define RELAY_MAXLOOKUPLEVELS	5
+#define RELAY_OUTOF_FD_RETRIES	5 
 
 #define CONFIG_RELOAD		0x00
 #define CONFIG_TABLES		0x01
@@ -431,8 +434,10 @@ struct rsession {
 	struct timeval			 se_timeout;
 	struct timeval			 se_tv_start;
 	struct timeval			 se_tv_last;
+	struct event			 se_inflightevt;
 	int				 se_done;
 	int				 se_retry;
+	int				 se_retrycount;
 	u_int16_t			 se_mark;
 	struct evbuffer			*se_log;
 	struct relay			*se_relay;
@@ -1060,6 +1065,8 @@ int		 imsg_compose_event(struct imsgev *, u_int16_t, u_int32_t,
 void		 socket_rlimit(int);
 char		*get_string(u_int8_t *, size_t);
 void		*get_data(u_int8_t *, size_t);
+int		 accept_reserve(int sockfd, struct sockaddr *addr,
+			socklen_t *addrlen, int reserve, volatile int *);
 
 /* carp.c */
 int	 carp_demote_init(char *, int);