move to a dynamically allocated struct pollfd array.

we used a ststic one with OPEN_MAX entries, which is a rather arbitary limit
as OPEN_MAX is _not_ the max # of open fds we can have, but just a default
for that setting.
in the same move we have to allocate the peer_l array, basically there
for pfd-index to peer pointers to prevent peer list scans all time,
dynamiccaly to. we overallocate a little and use that reserve until we
have to realloc again later to prevent reallocs for every single control
connection or a single flapping peer.
help & ok claudio

3 files changed, 109 insertions, 54 deletions

diff --git a/usr.sbin/bgpd/control.c b/usr.sbin/bgpd/control.c
index a680938e26d..2c9e9ebe234 100644
--- a/usr.sbin/bgpd/control.c
+++ b/usr.sbin/bgpd/control.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: control.c,v 1.31 2004/05/21 11:48:56 claudio Exp $ */
+/*	$OpenBSD: control.c,v 1.32 2004/06/09 13:01:44 henning Exp $ */
 
 /*
  * Copyright (c) 2003, 2004 Henning Brauer <henning@openbsd.org>
@@ -36,6 +36,7 @@ struct {
 
 struct ctl_conn	*control_connbyfd(int);
 struct ctl_conn	*control_connbypid(pid_t);
+int		 control_close(int);
 
 int
 control_init(void)
@@ -104,7 +105,7 @@ control_cleanup(void)
 	unlink(SOCKET_NAME);
 }
 
-void
+int
 control_accept(int listenfd)
 {
 	int			 connfd;
@@ -117,19 +118,21 @@ control_accept(int listenfd)
 	    (struct sockaddr *)&sun, &len)) == -1) {
 		if (errno != EWOULDBLOCK && errno != EINTR)
 			log_warn("session_control_accept");
-		return;
+		return (0);
 	}
 
 	session_socket_blockmode(connfd, BM_NONBLOCK);
 
 	if ((ctl_conn = malloc(sizeof(struct ctl_conn))) == NULL) {
 		log_warn("session_control_accept");
-		return;
+		return (0);
 	}
 
 	imsg_init(&ctl_conn->ibuf, connfd);
 
 	TAILQ_INSERT_TAIL(&ctl_conns, ctl_conn, entries);
+
+	return (1);
 }
 
 struct ctl_conn *
@@ -156,14 +159,14 @@ control_connbypid(pid_t pid)
 	return (c);
 }
 
-void
+int
 control_close(int fd)
 {
 	struct ctl_conn	*c;
 
 	if ((c = control_connbyfd(fd)) == NULL) {
 		log_warn("control_close: fd %d: not found", fd);
-		return;
+		return (0);
 	}
 
 	msgbuf_clear(&c->ibuf.w);
@@ -171,10 +174,12 @@ control_close(int fd)
 
 	close(c->ibuf.fd);
 	free(c);
+
+	return (1);
 }
 
 int
-control_dispatch_msg(struct pollfd *pfd)
+control_dispatch_msg(struct pollfd *pfd, u_int *ctl_cnt)
 {
 	struct imsg		 imsg;
 	struct ctl_conn		*c;
@@ -189,7 +194,7 @@ control_dispatch_msg(struct pollfd *pfd)
 
 	if (pfd->revents & POLLOUT)
 		if (msgbuf_write(&c->ibuf.w) < 0) {
-			control_close(pfd->fd);
+			*ctl_cnt -= control_close(pfd->fd);
 			return (1);
 		}
 
@@ -197,13 +202,13 @@ control_dispatch_msg(struct pollfd *pfd)
 		return (0);
 
 	if (imsg_read(&c->ibuf) <= 0) {
-		control_close(pfd->fd);
+		*ctl_cnt -= control_close(pfd->fd);
 		return (1);
 	}
 
 	for (;;) {
 		if ((n = imsg_get(&c->ibuf, &imsg)) == -1) {
-			control_close(pfd->fd);
+			*ctl_cnt -= control_close(pfd->fd);
 			return (1);
 		}
 
diff --git a/usr.sbin/bgpd/session.c b/usr.sbin/bgpd/session.c
index 9d581aedee6..eee00b97f3d 100644
--- a/usr.sbin/bgpd/session.c
+++ b/usr.sbin/bgpd/session.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: session.c,v 1.173 2004/06/06 17:38:10 henning Exp $ */
+/*	$OpenBSD: session.c,v 1.174 2004/06/09 13:01:44 henning Exp $ */
 
 /*
  * Copyright (c) 2003, 2004 Henning Brauer <henning@openbsd.org>
@@ -50,7 +50,7 @@
 #define PFD_LISTENERS_START	3
 
 void	session_sighdlr(int);
-int	setup_listeners(void);
+int	setup_listeners(u_int *);
 void	init_conf(struct bgpd_config *);
 void	init_peer(struct peer *);
 int	timer_due(time_t);
@@ -75,7 +75,7 @@ int	parse_refresh(struct peer *);
 int	parse_notification(struct peer *);
 int	parse_keepalive(struct peer *);
 int	parse_capabilities(struct peer *, u_char *, u_int16_t);
-void	session_dispatch_imsg(struct imsgbuf *, int);
+void	session_dispatch_imsg(struct imsgbuf *, int, u_int *);
 void	session_up(struct peer *);
 void	session_down(struct peer *);
 
@@ -108,10 +108,11 @@ session_sighdlr(int sig)
 }
 
 int
-setup_listeners(void)
+setup_listeners(u_int *la_cnt)
 {
 	int			 opt;
 	struct listen_addr	*la;
+	u_int			 cnt = 0;
 
 	if (TAILQ_EMPTY(conf->listen_addrs)) {
 		if ((la = calloc(1, sizeof(struct listen_addr))) == NULL)
@@ -137,6 +138,7 @@ setup_listeners(void)
 
 	TAILQ_FOREACH(la, conf->listen_addrs, entry) {
 		la->reconf = RECONF_NONE;
+		cnt++;
 
 		if (la->fd != -1)
 			continue;
@@ -178,6 +180,8 @@ setup_listeners(void)
 		    log_sockaddr((struct sockaddr *)&la->sa));
 	}
 
+	*la_cnt = cnt;
+
 	return (0);
 }
 
@@ -189,15 +193,18 @@ session_main(struct bgpd_config *config, struct peer *cpeers,
 	int			 nfds, i, j, timeout, idx_peers, idx_listeners;
 	pid_t			 pid;
 	time_t			 nextaction;
+	u_int			 pfd_elms = 0, peer_l_elms = 0, new_cnt;
+	u_int			 listener_cnt, peer_cnt, ctl_cnt;
 	struct passwd		*pw;
-	struct peer		*p, *peer_l[OPEN_MAX], *last, *next;
+	struct peer		*p, **peer_l = NULL, *last, *next;
 	struct network		*net;
 	struct mrt		*m;
 	struct mrt_config	*mrt;
 	struct filter_rule	*r;
-	struct pollfd		 pfd[OPEN_MAX];
+	struct pollfd		*pfd = NULL;
 	struct ctl_conn		*ctl_conn;
 	struct listen_addr	*la;
+	void			*newp;
 	short			 events;
 
 	conf = config;
@@ -227,7 +234,8 @@ session_main(struct bgpd_config *config, struct peer *cpeers,
 	setproctitle("session engine");
 	bgpd_process = PROC_SE;
 
-	setup_listeners();
+	listener_cnt = 0;
+	setup_listeners(&listener_cnt);
 
 	if (pfkey_init(&sysdep) == -1)
 		fatalx("pfkey setup failed");
@@ -251,6 +259,8 @@ session_main(struct bgpd_config *config, struct peer *cpeers,
 	TAILQ_INIT(&ctl_conns);
 	csock = control_listen();
 	LIST_INIT(&mrt_l);
+	peer_cnt = 0;
+	ctl_cnt = 0;
 
 	/* filter rules are not used in the SE */
 	while ((r = TAILQ_FIRST(rules)) != NULL) {
@@ -272,36 +282,16 @@ session_main(struct bgpd_config *config, struct peer *cpeers,
 	}
 
 	while (session_quit == 0) {
-		bzero(&pfd, sizeof(pfd));
-		pfd[PFD_PIPE_MAIN].fd = ibuf_main.fd;
-		pfd[PFD_PIPE_MAIN].events = POLLIN;
-		if (ibuf_main.w.queued > 0)
-			pfd[PFD_PIPE_MAIN].events |= POLLOUT;
-		pfd[PFD_PIPE_ROUTE].fd = ibuf_rde.fd;
-		pfd[PFD_PIPE_ROUTE].events = POLLIN;
-		if (ibuf_rde.w.queued > 0)
-			pfd[PFD_PIPE_ROUTE].events |= POLLOUT;
-		pfd[PFD_SOCK_CTL].fd = csock;
-		pfd[PFD_SOCK_CTL].events = POLLIN;
-
-		nextaction = time(NULL) + 240;	/* loop every 240s at least */
-		i = PFD_LISTENERS_START;
-
-		TAILQ_FOREACH(la, conf->listen_addrs, entry) {
-			pfd[i].fd = la->fd;
-			pfd[i].events = POLLIN;
-			i++;
-		}
-
-		idx_listeners = i;
-
+		/* check for peers to be initialized or deleted */
 		last = NULL;
 		for (p = peers; p != NULL; p = next) {
 			next = p->next;
 			if (!pending_reconf) {
-				/* needs init? */
-				if (p->state == STATE_NONE)
+				/* new peer that needs init? */
+				if (p->state == STATE_NONE) {
 					init_peer(p);
+					peer_cnt++;
+				}
 
 				/* reinit due? */
 				if (p->conf.reconf_action == RECONF_REINIT) {
@@ -318,12 +308,68 @@ session_main(struct bgpd_config *config, struct peer *cpeers,
 					else
 						peers = next;
 					free(p);
+					peer_cnt--;
 					continue;
 				}
 				p->conf.reconf_action = RECONF_NONE;
 			}
 			last = p;
+		}
+
+		if (peer_cnt > peer_l_elms ||
+		    peer_cnt + PEER_L_RESERVE < peer_l_elms) {
+			if ((newp = realloc(peer_l, sizeof(struct peer *) *
+			    peer_cnt + PEER_L_RESERVE)) == NULL) {
+				/* panic for now  */
+				log_warn("could not resize peer_l from %u -> %u"
+				    " entries", peer_l_elms,
+				    peer_cnt + PEER_L_RESERVE);
+				fatalx("exiting");
+			}
+			peer_l = newp;
+			peer_l_elms = peer_cnt + PEER_L_RESERVE;
+		}
+
+		new_cnt =
+		    PFD_LISTENERS_START + listener_cnt + peer_cnt + ctl_cnt;
+		if (new_cnt > pfd_elms ||
+		    new_cnt + PFD_RESERVE < pfd_elms) {
+			if ((newp = realloc(pfd, sizeof(struct pollfd) *
+			    (new_cnt + PFD_RESERVE))) == NULL) {
+				/* panic for now  */
+				log_warn("could not resize pfd from %u -> %u"
+				    " entries", pfd_elms,
+				    new_cnt + PFD_RESERVE);
+				fatalx("exiting");
+			}
+			pfd = newp;
+			pfd_elms = new_cnt + PFD_RESERVE;
+		}
+
+		bzero(pfd, sizeof(struct pollfd) * pfd_elms);
+		pfd[PFD_PIPE_MAIN].fd = ibuf_main.fd;
+		pfd[PFD_PIPE_MAIN].events = POLLIN;
+		if (ibuf_main.w.queued > 0)
+			pfd[PFD_PIPE_MAIN].events |= POLLOUT;
+		pfd[PFD_PIPE_ROUTE].fd = ibuf_rde.fd;
+		pfd[PFD_PIPE_ROUTE].events = POLLIN;
+		if (ibuf_rde.w.queued > 0)
+			pfd[PFD_PIPE_ROUTE].events |= POLLOUT;
+		pfd[PFD_SOCK_CTL].fd = csock;
+		pfd[PFD_SOCK_CTL].events = POLLIN;
+
+		nextaction = time(NULL) + 240;	/* loop every 240s at least */
+		i = PFD_LISTENERS_START;
+
+		TAILQ_FOREACH(la, conf->listen_addrs, entry) {
+			pfd[i].fd = la->fd;
+			pfd[i].events = POLLIN;
+			i++;
+		}
+
+		idx_listeners = i;
 
+		for (p = peers; p != NULL; p = p->next) {
 			/* check timers */
 			if (timer_due(p->HoldTimer))
 				bgp_fsm(p, EVNT_TIMER_HOLDTIME);
@@ -368,7 +414,7 @@ session_main(struct bgpd_config *config, struct peer *cpeers,
 			if (p->fd != -1 && events != 0) {
 				pfd[i].fd = p->fd;
 				pfd[i].events = events;
-				peer_l[i] = p;
+				peer_l[i - idx_listeners] = p;
 				i++;
 			}
 		}
@@ -396,7 +442,8 @@ session_main(struct bgpd_config *config, struct peer *cpeers,
 
 		if (nfds > 0 && pfd[PFD_PIPE_MAIN].revents & POLLIN) {
 			nfds--;
-			session_dispatch_imsg(&ibuf_main, PFD_PIPE_MAIN);
+			session_dispatch_imsg(&ibuf_main, PFD_PIPE_MAIN,
+			    &listener_cnt);
 		}
 
 		if (nfds > 0 && pfd[PFD_PIPE_ROUTE].revents & POLLOUT)
@@ -405,12 +452,13 @@ session_main(struct bgpd_config *config, struct peer *cpeers,
 
 		if (nfds > 0 && pfd[PFD_PIPE_ROUTE].revents & POLLIN) {
 			nfds--;
-			session_dispatch_imsg(&ibuf_rde, PFD_PIPE_ROUTE);
+			session_dispatch_imsg(&ibuf_rde, PFD_PIPE_ROUTE,
+			    &listener_cnt);
 		}
 
 		if (nfds > 0 && pfd[PFD_SOCK_CTL].revents & POLLIN) {
 			nfds--;
-			control_accept(csock);
+			ctl_cnt += control_accept(csock);
 		}
 
 		for (j = PFD_LISTENERS_START; nfds > 0 && j < idx_listeners;
@@ -421,10 +469,11 @@ session_main(struct bgpd_config *config, struct peer *cpeers,
 			}
 
 		for (; nfds > 0 && j < idx_peers; j++)
-			nfds -= session_dispatch_msg(&pfd[j], peer_l[j]);
+			nfds -= session_dispatch_msg(&pfd[j],
+			    peer_l[j - idx_listeners]);
 
 		for (; nfds > 0 && j < i; j++)
-			nfds -= control_dispatch_msg(&pfd[j]);
+			nfds -= control_dispatch_msg(&pfd[j], &ctl_cnt);
 	}
 
 	while ((p = peers) != NULL) {
@@ -1947,7 +1996,7 @@ parse_capabilities(struct peer *peer, u_char *d, u_int16_t dlen)
 }
 
 void
-session_dispatch_imsg(struct imsgbuf *ibuf, int idx)
+session_dispatch_imsg(struct imsgbuf *ibuf, int idx, u_int *listener_cnt)
 {
 	struct imsg		 imsg;
 	struct mrt_config	 xmrt;
@@ -2082,7 +2131,7 @@ session_dispatch_imsg(struct imsgbuf *ibuf, int idx)
 				    entry);
 			}
 
-			setup_listeners();
+			setup_listeners(listener_cnt);
 			free(nconf->listen_addrs);
 			free(nconf);
 			nconf = NULL;
diff --git a/usr.sbin/bgpd/session.h b/usr.sbin/bgpd/session.h
index ce140a7a23a..d224198c37a 100644
--- a/usr.sbin/bgpd/session.h
+++ b/usr.sbin/bgpd/session.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: session.h,v 1.54 2004/06/06 17:38:10 henning Exp $ */
+/*	$OpenBSD: session.h,v 1.55 2004/06/09 13:01:44 henning Exp $ */
 
 /*
  * Copyright (c) 2003, 2004 Henning Brauer <henning@openbsd.org>
@@ -33,6 +33,8 @@
 #define	MSGSIZE_UPDATE_MIN		23
 #define	MSGSIZE_KEEPALIVE		MSGSIZE_HEADER
 #define MSGSIZE_RREFRESH		MSGSIZE_HEADER + 4
+#define	PFD_RESERVE			5
+#define	PEER_L_RESERVE			2
 
 enum session_state {
 	STATE_NONE,
@@ -216,9 +218,8 @@ int	 rde_main(struct bgpd_config *, struct network_head *,
 /* control.c */
 int	control_listen(void);
 void	control_shutdown(void);
-int	control_dispatch_msg(struct pollfd *);
-void	control_accept(int);
-void	control_close(int);
+int	control_dispatch_msg(struct pollfd *, u_int *);
+int	control_accept(int);
 
 /* pfkey.c */
 int	pfkey_establish(struct peer *);