Do rate limiting of accept() when under pressure, like in other recent

daemons.  Light testing by some relayd users; let me know if issues
develop.

3 files changed, 70 insertions, 20 deletions

diff --git a/usr.sbin/relayd/control.c b/usr.sbin/relayd/control.c
index cdb4f0b063d..5cc6e60e04f 100644
--- a/usr.sbin/relayd/control.c
+++ b/usr.sbin/relayd/control.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: control.c,v 1.40 2011/05/20 09:43:53 reyk Exp $	*/
+/*	$OpenBSD: control.c,v 1.41 2012/04/11 08:25:26 deraadt Exp $	*/
 
 /*
  * Copyright (c) 2003, 2004 Henning Brauer <henning@openbsd.org>
@@ -41,7 +41,7 @@
 struct ctl_connlist ctl_conns;
 
 void		 control_accept(int, short, void *);
-void		 control_close(int);
+void		 control_close(int, struct control_sock *);
 
 int
 control_init(struct privsep *ps, struct control_sock *cs)
@@ -115,9 +115,10 @@ control_listen(struct control_sock *cs)
 		return (-1);
 	}
 
-	event_set(&cs->cs_ev, cs->cs_fd, EV_READ | EV_PERSIST,
-	    control_accept, cs->cs_env);
+	event_set(&cs->cs_ev, cs->cs_fd, EV_READ,
+	    control_accept, cs);
 	event_add(&cs->cs_ev, NULL);
+	evtimer_set(&cs->cs_evt, control_accept, cs);
 
 	return (0);
 }
@@ -127,6 +128,9 @@ control_cleanup(struct control_sock *cs)
 {
 	if (cs->cs_name == NULL)
 		return;
+	event_del(&cs->cs_ev);
+	if (evtimer_pending(&cs->cs_evt, NULL))
+		event_del(&cs->cs_evt);
 	(void)unlink(cs->cs_name);
 }
 
@@ -138,12 +142,25 @@ control_accept(int listenfd, short event, void *arg)
 	socklen_t		 len;
 	struct sockaddr_un	 sun;
 	struct ctl_conn		*c;
-	struct relayd		*env = arg;
+	struct control_sock	*cs = arg;
+
+	event_add(&cs->cs_ev, NULL);
+	if ((event & EV_TIMEOUT))
+		return;
 
 	len = sizeof(sun);
 	if ((connfd = accept(listenfd,
 	    (struct sockaddr *)&sun, &len)) == -1) {
-		if (errno != EWOULDBLOCK && errno != EINTR)
+		/*
+		 * Pause accept if we are out of file descriptors, or
+		 * libevent will haunt us here too.
+		 */
+		if (errno == ENFILE || errno == EMFILE) {
+			struct timeval evtpause = { 1, 0 };
+
+			event_del(&cs->cs_ev);
+			evtimer_add(&cs->cs_evt, &evtpause);
+		} else if (errno != EWOULDBLOCK && errno != EINTR)
 			log_warn("%s: accept", __func__);
 		return;
 	}
@@ -160,7 +177,7 @@ control_accept(int listenfd, short event, void *arg)
 	c->iev.handler = control_dispatch_imsg;
 	c->iev.events = EV_READ;
 	event_set(&c->iev.ev, c->iev.ibuf.fd, c->iev.events,
-	    c->iev.handler, env);
+	    c->iev.handler, cs);
 	event_add(&c->iev.ev, NULL);
 
 	TAILQ_INSERT_TAIL(&ctl_conns, c, entry);
@@ -179,7 +196,7 @@ control_connbyfd(int fd)
 }
 
 void
-control_close(int fd)
+control_close(int fd, struct control_sock *cs)
 {
 	struct ctl_conn	*c;
 
@@ -193,6 +210,13 @@ control_close(int fd)
 
 	event_del(&c->iev.ev);
 	close(c->iev.ibuf.fd);
+
+	/* Some file descriptors are available again. */
+	if (evtimer_pending(&cs->cs_evt, NULL)) {
+		evtimer_del(&cs->cs_evt);
+		event_add(&cs->cs_ev, NULL);
+	}
+
 	free(c);
 }
 
@@ -200,12 +224,13 @@ control_close(int fd)
 void
 control_dispatch_imsg(int fd, short event, void *arg)
 {
+	struct control_sock	*cs = (struct control_sock *)arg;
 	struct ctl_conn		*c;
 	struct imsg		 imsg;
 	struct ctl_id		 id;
 	int			 n;
 	int			 verbose;
-	struct relayd		*env = arg;
+	struct relayd		*env = cs->cs_env;
 
 	if ((c = control_connbyfd(fd)) == NULL) {
 		log_warn("%s: fd %d not found", __func__, fd);
@@ -214,21 +239,21 @@ control_dispatch_imsg(int fd, short event, void *arg)
 
 	if (event & EV_READ) {
 		if ((n = imsg_read(&c->iev.ibuf)) == -1 || n == 0) {
-			control_close(fd);
+			control_close(fd, cs);
 			return;
 		}
 	}
 
 	if (event & EV_WRITE) {
 		if (msgbuf_write(&c->iev.ibuf.w) < 0) {
-			control_close(fd);
+			control_close(fd, cs);
 			return;
 		}
 	}
 
 	for (;;) {
 		if ((n = imsg_get(&c->iev.ibuf, &imsg)) == -1) {
-			control_close(fd);
+			control_close(fd, cs);
 			return;
 		}
 
@@ -239,7 +264,7 @@ control_dispatch_imsg(int fd, short event, void *arg)
 			log_debug("%s: unexpected imsg %d",
 			    __func__, imsg.hdr.type);
 			imsg_free(&imsg);
-			control_close(fd);
+			control_close(fd, cs);
 			return;
 		}
 
diff --git a/usr.sbin/relayd/relay.c b/usr.sbin/relayd/relay.c
index 010134ec1d2..695c7dd06eb 100644
--- a/usr.sbin/relayd/relay.c
+++ b/usr.sbin/relayd/relay.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: relay.c,v 1.145 2012/03/24 14:48:18 sthen Exp $	*/
+/*	$OpenBSD: relay.c,v 1.146 2012/04/11 08:25:26 deraadt Exp $	*/
 
 /*
  * Copyright (c) 2006, 2007, 2008 Reyk Floeter <reyk@openbsd.org>
@@ -469,9 +469,10 @@ relay_launch(void)
 		else
 			callback = relay_accept;
 
-		event_set(&rlay->rl_ev, rlay->rl_s, EV_READ|EV_PERSIST,
+		event_set(&rlay->rl_ev, rlay->rl_s, EV_READ,
 		    callback, rlay);
 		event_add(&rlay->rl_ev, NULL);
+		evtimer_set(&rlay->rl_evt, callback, rlay);
 	}
 }
 
@@ -1945,7 +1946,7 @@ relay_error(struct bufferevent *bev, short error, void *arg)
 }
 
 void
-relay_accept(int fd, short sig, void *arg)
+relay_accept(int fd, short event, void *arg)
 {
 	struct relay *rlay = (struct relay *)arg;
 	struct protocol *proto = rlay->rl_proto;
@@ -1956,10 +1957,24 @@ relay_accept(int fd, short sig, void *arg)
 	struct sockaddr_storage ss;
 	int s = -1;
 
-	slen = sizeof(ss);
-	if ((s = accept(fd, (struct sockaddr *)&ss, (socklen_t *)&slen)) == -1)
+	event_add(&rlay->rl_ev, NULL);
+	if ((event & EV_TIMEOUT))
 		return;
 
+	slen = sizeof(ss);
+	if ((s = accept(fd, (struct sockaddr *)&ss, (socklen_t *)&slen)) == -1) {
+		/*
+		 * Pause accept if we are out of file descriptors, or
+		 * libevent will haunt us here too.
+		 */
+		if (errno == ENFILE || errno == EMFILE) {
+			struct timeval evtpause = { 1, 0 };
+
+			event_del(&rlay->rl_ev);
+			evtimer_add(&rlay->rl_evt, &evtpause);
+		}
+		return;
+	}
 	if (relay_sessions >= RELAY_MAX_SESSIONS ||
 	    rlay->rl_conf.flags & F_DISABLE)
 		goto err;
@@ -2393,8 +2408,16 @@ relay_close(struct rsession *con, const char *msg)
 			SSL_shutdown(con->se_out.ssl);
 		SSL_free(con->se_out.ssl);
 	}
-	if (con->se_out.s != -1)
+	if (con->se_out.s != -1) {
 		close(con->se_out.s);
+
+		/* Some file descriptors are available again. */
+		if (evtimer_pending(&rlay->rl_evt, NULL)) {
+			evtimer_del(&rlay->rl_evt);
+			event_add(&rlay->rl_ev, NULL);
+		}
+	}
+
 	if (con->se_out.path != NULL)
 		free(con->se_out.path);
 	if (con->se_out.buf != NULL)
diff --git a/usr.sbin/relayd/relayd.h b/usr.sbin/relayd/relayd.h
index 35db7e1dad1..96bcc8476a8 100644
--- a/usr.sbin/relayd/relayd.h
+++ b/usr.sbin/relayd/relayd.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: relayd.h,v 1.152 2012/01/21 13:40:48 camield Exp $	*/
+/*	$OpenBSD: relayd.h,v 1.153 2012/04/11 08:25:26 deraadt Exp $	*/
 
 /*
  * Copyright (c) 2006, 2007 Pierre-Yves Ritschard <pyr@openbsd.org>
@@ -607,6 +607,7 @@ struct relay {
 	int			 rl_dstnhosts;
 
 	struct event		 rl_ev;
+	struct event		 rl_evt;
 
 	SSL_CTX			*rl_ssl_ctx;
 	char			*rl_ssl_cert;
@@ -677,6 +678,7 @@ struct ctl_netroute {
 struct control_sock {
 	const char	*cs_name;
 	struct event	 cs_ev;
+	struct event	 cs_evt;
 	int		 cs_fd;
 	int		 cs_restricted;
 	void		*cs_env;