Add tests that create and delete cloned routes during connect(2).

3 files changed, 246 insertions, 40 deletions

diff --git a/regress/sys/netinet/bindconnect/Makefile b/regress/sys/netinet/bindconnect/Makefile
index 2884042f645..258a6a3f8c7 100644
--- a/regress/sys/netinet/bindconnect/Makefile
+++ b/regress/sys/netinet/bindconnect/Makefile
@@ -1,4 +1,4 @@
-#	$OpenBSD: Makefile,v 1.1 2023/12/06 14:41:52 bluhm Exp $
+#	$OpenBSD: Makefile,v 1.2 2023/12/06 22:57:14 bluhm Exp $
 
 PROG=		bindconnect
 LDADD=		-lpthread
@@ -7,22 +7,63 @@ WARNINGS=	yes
 
 CLEANFILES=	ktrace.out
 
-${REGRESS_TARGETS}: ${PROG}
+LOCAL_NET ?=
+
+REGRESS_ROOT_TARGETS +=	setup-maxfiles run-100000 run-localnet-connect-delete
+
+REGRESS_SETUP_ONCE +=	setup-maxfiles
+setup-maxfiles:
+	[[ $$(sysctl -n kern.maxfiles) -ge 110000 ]] || \
+	    ${SUDO} sysctl kern.maxfiles=110000
+
+REGRESS_SETUP +=	${PROG}
+
+.if ! empty(LOCAL_NET)
+REGRESS_CLEANUP +=	cleanup-delete
+.endif
+cleanup-delete:
+	-${SUDO} time ./${PROG} -s 0 -o 0 -b 0 -c 0 -d 1 -N ${LOCAL_NET} -t 1
 
 REGRESS_TARGETS +=	run-default
 run-default:
-	${SUDO} time ${KTRACE} ./${PROG}
+	time ${KTRACE} ./${PROG}
 
 REGRESS_TARGETS +=	run-bind
 run-bind:
-	${SUDO} time ${KTRACE} ./${PROG} -n 10 -s 2 -o 1 -b 5 -c 0
+	time ${KTRACE} ./${PROG} -n 16 -s 2 -o 1 -b 5 -c 0
 
 REGRESS_TARGETS +=	run-connect
 run-connect:
-	${SUDO} time ${KTRACE} ./${PROG} -n 10 -s 2 -o 1 -b 0 -c 5
+	time ${KTRACE} ./${PROG} -n 16 -s 2 -o 1 -b 0 -c 5
 
 REGRESS_TARGETS +=	run-bind-connect
 run-bind-connect:
-	${SUDO} time ${KTRACE} ./${PROG} -n 10 -s 2 -o 1 -b 3 -c 3
+	time ${KTRACE} ./${PROG} -n 16 -s 2 -o 1 -b 3 -c 3
+
+REGRESS_TARGETS +=	run-100000
+run-100000:
+	${SUDO} time ${KTRACE} ./${PROG} -n 100000 -s 2 -o 1 -b 3 -c 3
+
+REGRESS_TARGETS +=	run-reuseport
+run-reuseport:
+	time ${KTRACE} ./${PROG} -n 16 -s 2 -o 1 -b 3 -c 3 -r
+
+.if empty(LOCAL_NET)
+REGRESS_SKIP_TARGETS +=	run-localnet-connect run-localnet-bind-connect \
+			run-localnet-connect-delete
+.endif
+
+REGRESS_TARGETS +=	run-localnet-connect
+run-localnet-connect:
+	time ${KTRACE} ./${PROG} -n 16 -s 2 -o 1 -c 5 -N ${LOCAL_NET}
+
+REGRESS_TARGETS +=	run-localnet-bind-connect
+run-localnet-bind-connect:
+	time ${KTRACE} ./${PROG} -n 16 -s 2 -o 1 -b 3 -c 3 -N ${LOCAL_NET}
+
+REGRESS_TARGETS +=	run-localnet-connect-delete
+run-localnet-connect-delete:
+	${SUDO} time ${KTRACE} \
+	    ./${PROG} -n 16 -s 2 -o 1 -b 0 -c 5 -d 3 -N ${LOCAL_NET}
 
 .include <bsd.regress.mk>
diff --git a/regress/sys/netinet/bindconnect/README b/regress/sys/netinet/bindconnect/README
index b8a00ad2f7d..0c168e63e10 100644
--- a/regress/sys/netinet/bindconnect/README
+++ b/regress/sys/netinet/bindconnect/README
@@ -1,12 +1,16 @@
 Stress test bind(2) and connect(2) system calls in OpenBSD regress.
 
-bindconnect [-b bind] [-c connect] [-n num] [-o close] [-s socket] [-t time]
-    -b bind     threads binding sockets, default 1
-    -c connect  threads connecting sockets, default 1
-    -n num      number of file descriptors, default 100
-    -o close    threads closing sockets, default 1
-    -s socket   threads creating sockets, default 1
-    -t time     run time in seconds, default 10
+bindconnect [-r] [-b bind] [-c connect] [-d delroute]
+[-N addr/net] [-n num] [-o close] [-s socket] [-t time]
+    -b bind      threads binding sockets, default 1
+    -c connect   threads connecting sockets, default 1
+    -d delroute  threads deleting cloned routes, default 0
+    -N addr/net  connect to any address within network
+    -n num       number of file descriptors, default 128
+    -o close     threads closing sockets, default 1
+    -r           set reuse port socket option
+    -s socket    threads creating sockets, default 1
+    -t time      run time in seconds, default 10
 
 Separate threads are started to run socket(2), close(2), bind(2),
 and connect(2) system calls concurrently.  The number of sockets
@@ -15,4 +19,9 @@ system calls operate on random file descriptors.  By setting the
 number of threads for each system call and the number of available
 file descriptors, the focus for the stress test can be changed.
 
-Currently only IPv4 UDP sockets with 127.0.0.1 are supported.
+Currently only IPv4 UDP sockets are supported.  Per default the
+address to bind and connect is 127.0.0.1.  LOCAL_NET environment
+variable allows to bind on a local address and connect to all
+directly attached hosts.  This triggers creation of cloned routes
+during source address selection.  To stress test routing table,
+these routes can be deleted in another thread.
diff --git a/regress/sys/netinet/bindconnect/bindconnect.c b/regress/sys/netinet/bindconnect/bindconnect.c
index e94f80987ff..19966d9bbb0 100644
--- a/regress/sys/netinet/bindconnect/bindconnect.c
+++ b/regress/sys/netinet/bindconnect/bindconnect.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: bindconnect.c,v 1.1 2023/12/06 14:41:52 bluhm Exp $	*/
+/*	$OpenBSD: bindconnect.c,v 1.2 2023/12/06 22:57:14 bluhm Exp $	*/
 
 /*
  * Copyright (c) 2023 Alexander Bluhm <bluhm@openbsd.org>
@@ -19,7 +19,9 @@
 #include <sys/resource.h>
 #include <sys/socket.h>
 
+#include <net/route.h>
 #include <netinet/in.h>
+#include <arpa/inet.h>
 
 #include <err.h>
 #include <errno.h>
@@ -32,23 +34,31 @@
 #define MAX(a, b)	((a) > (b) ? (a) : (b))
 
 int fd_base;
-unsigned int fd_num = 100;
+unsigned int fd_num = 128;
 unsigned int run_time = 10;
-unsigned int socket_num = 1, close_num = 1, bind_num = 1, connect_num = 1;
+unsigned int socket_num = 1, close_num = 1, bind_num = 1, connect_num = 1,
+    delroute_num = 0;
+int reuse_port = 0;
+struct in_addr addr, mask;
+int prefix = -1, route_sock = -1;
 
 static void __dead
 usage(void)
 {
 	fprintf(stderr,
-	    "bindconnect [-b bind] [-c connect] [-n num] [-o close]\n"
-	    "[-s socket] [-t time]\n"
-	    "    -b bind     threads binding sockets, default %u\n"
-	    "    -c connect  threads connecting sockets, default %u\n"
-	    "    -n num      number of file descriptors, default %u\n"
-	    "    -o close    threads closing sockets, default %u\n"
-	    "    -s socket   threads creating sockets, default %u\n"
-	    "    -t time     run time in seconds, default %u\n",
-	    bind_num, connect_num, fd_num, close_num, socket_num, run_time);
+	    "bindconnect [-r] [-b bind] [-c connect] [-d delroute]\n"
+	    "[-N addr/net] [-n num] [-o close] [-s socket] [-t time]\n"
+	    "    -b bind      threads binding sockets, default %u\n"
+	    "    -c connect   threads connecting sockets, default %u\n"
+	    "    -d delroute  threads deleting cloned routes, default %u\n"
+	    "    -N addr/net  connect to any address within network\n"
+	    "    -n num       number of file descriptors, default %u\n"
+	    "    -o close     threads closing sockets, default %u\n"
+	    "    -r           set reuse port socket option\n"
+	    "    -s socket    threads creating sockets, default %u\n"
+	    "    -t time      run time in seconds, default %u\n",
+	    bind_num, connect_num, delroute_num, fd_num, close_num, socket_num,
+	    run_time);
 	exit(2);
 }
 
@@ -58,14 +68,30 @@ sintosa(struct sockaddr_in *sin)
 	return ((struct sockaddr *)(sin));
 }
 
+static void
+in_prefixlen2mask(struct in_addr *maskp, int plen)
+{
+	if (plen == 0)
+		maskp->s_addr = 0;
+	else
+		maskp->s_addr = htonl(0xffffffff << (32 - plen));
+}
+
 static void *
 thread_socket(void *arg)
 {
 	volatile int *run = arg;
 	unsigned long count;
+	int fd;
 
 	for (count = 0; *run; count++) {
-		socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP);
+		int opt;
+
+		fd = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP);
+		if (fd < 0 || !reuse_port)
+			continue;
+		opt = 1;
+		setsockopt(fd, SOL_SOCKET, SO_REUSEPORT, &opt, sizeof(opt));
 	}
 
 	return (void *)count;
@@ -99,6 +125,9 @@ thread_bind(void *arg)
 	sin.sin_family = AF_INET;
 	sin.sin_addr.s_addr = htonl(INADDR_LOOPBACK);
 
+	if (prefix >= 0)
+		sin.sin_addr = addr;
+
 	for (count = 0; *run; count++) {
 		fd = fd_base + arc4random_uniform(fd_num);
 		bind(fd, sintosa(&sin), sizeof(sin));
@@ -119,31 +148,92 @@ thread_connect(void *arg)
 	sin.sin_len = sizeof(sin);
 	sin.sin_family = AF_INET;
 	sin.sin_addr.s_addr = htonl(INADDR_LOOPBACK);
-	sin.sin_port = arc4random();
+
+	if (prefix >= 0)
+		sin.sin_addr = addr;
 
 	for (count = 0; *run; count++) {
 		fd = fd_base + arc4random_uniform(fd_num);
+		if (prefix >=0 && prefix != 32) {
+			sin.sin_addr.s_addr &= mask.s_addr;
+			sin.sin_addr.s_addr |= ~mask.s_addr & arc4random();
+		}
+		sin.sin_port = arc4random();
 		connect(fd, sintosa(&sin), sizeof(sin));
 	}
 
 	return (void *)count;
 }
 
+static void *
+thread_delroute(void *arg)
+{
+	volatile int *run = arg;
+	unsigned long count;
+	int seq = 0;
+	struct {
+		struct rt_msghdr	m_rtm;
+		char			m_space[512];
+	} m_rtmsg;
+	struct sockaddr_in sin;
+
+#define rtm \
+	m_rtmsg.m_rtm
+#define ROUNDUP(a) \
+	((a) > 0 ? (1 + (((a) - 1) | (sizeof(long) - 1))) : sizeof(long))
+#define ADVANCE(x, n) \
+	(x += ROUNDUP((n)->sa_len))
+#define NEXTADDR(w, sa)				\
+	if (rtm.rtm_addrs & (w)) {		\
+		int l = ROUNDUP(sa->sa_len);	\
+		memcpy(cp, sa, l);		\
+		cp += l;			\
+	}
+
+	memset(&m_rtmsg, 0, sizeof(m_rtmsg));
+	rtm.rtm_type = RTM_DELETE;
+	rtm.rtm_flags = RTF_HOST;
+	rtm.rtm_version = RTM_VERSION;
+	rtm.rtm_addrs = RTA_DST;
+	rtm.rtm_priority = 0;  /* XXX */
+	rtm.rtm_hdrlen = sizeof(rtm);
+
+	memset(&sin, 0, sizeof(sin));
+	sin.sin_len = sizeof(sin);
+	sin.sin_family = AF_INET;
+	sin.sin_addr = addr;
+
+	for (count = 0; *run; count++) {
+		char *cp = m_rtmsg.m_space;
+
+		rtm.rtm_seq = ++seq;
+		sin.sin_addr.s_addr &= mask.s_addr;
+		sin.sin_addr.s_addr |= ~mask.s_addr & arc4random();
+		NEXTADDR(RTA_DST, sintosa(&sin));
+		rtm.rtm_msglen = cp - (char *)&m_rtmsg;
+		write(route_sock, &m_rtmsg, rtm.rtm_msglen);
+	}
+
+#undef rtm
+#undef ROUNDUP
+#undef ADVANCE
+#undef NEXTADDR
+
+	return (void *)count;
+}
+
 int
 main(int argc, char *argv[])
 {
 	struct rlimit rlim;
-	pthread_t *tsocket, *tclose, *tbind, *tconnect;
-	const char *errstr;
+	pthread_t *tsocket, *tclose, *tbind, *tconnect, *tdelroute;
+	const char *errstr, *addr_net = NULL;
 	int ch, run;
 	unsigned int n;
-	unsigned long socket_count, close_count, bind_count, connect_count;
+	unsigned long socket_count, close_count, bind_count, connect_count,
+	    delroute_count;
 
-	fd_base = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP);
-	if (fd_base < 0)
-		err(1, "socket fd_base");
-
-	while ((ch = getopt(argc, argv, "b:c:n:o:s:t:")) != -1) {
+	while ((ch = getopt(argc, argv, "b:c:d:N:n:o:rs:t:")) != -1) {
 		switch (ch) {
 		case 'b':
 			bind_num = strtonum(optarg, 0, UINT_MAX, &errstr);
@@ -155,9 +245,16 @@ main(int argc, char *argv[])
 			if (errstr != NULL)
 				errx(1, "connect is %s: %s", errstr, optarg);
 			break;
+		case 'd':
+			delroute_num = strtonum(optarg, 0, UINT_MAX, &errstr);
+			if (errstr != NULL)
+				errx(1, "delroute is %s: %s", errstr, optarg);
+			break;
+		case 'N':
+			addr_net = optarg;
+			break;
 		case 'n':
-			fd_num = strtonum(optarg, 1, INT_MAX - fd_base,
-			    &errstr);
+			fd_num = strtonum(optarg, 1, INT_MAX, &errstr);
 			if (errstr != NULL)
 				errx(1, "num is %s: %s", errstr, optarg);
 			break;
@@ -166,6 +263,9 @@ main(int argc, char *argv[])
 			if (errstr != NULL)
 				errx(1, "close is %s: %s", errstr, optarg);
 			break;
+		case 'r':
+			reuse_port = 1;
+			break;
 		case 's':
 			socket_num = strtonum(optarg, 0, UINT_MAX, &errstr);
 			if (errstr != NULL)
@@ -185,6 +285,27 @@ main(int argc, char *argv[])
 	if (argc > 0)
 		usage();
 
+	if (addr_net != NULL) {
+		prefix = inet_net_pton(AF_INET, addr_net, &addr, sizeof(addr));
+		if (prefix < 0)
+			err(1, "inet_net_pton %s", addr_net);
+		in_prefixlen2mask(&mask, prefix);
+	}
+	if (delroute_num > 0) {
+		if (prefix < 0 || prefix == 32)
+			errx(1, "delroute %u needs addr/net", delroute_num);
+		route_sock = socket(AF_ROUTE, SOCK_RAW, AF_INET);
+		if (route_sock < 0)
+			err(1, "socket route");
+		if (shutdown(route_sock, SHUT_RD) < 0)
+			err(1, "shutdown read route");
+	}
+
+	fd_base = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP);
+	if (fd_base < 0)
+		err(1, "socket fd_base");
+	if (fd_base > INT_MAX - (int)fd_num)
+		err(1, "fd base %d and num %u overflow", fd_base, fd_num);
 	if (closefrom(fd_base) < 0)
 		err(1, "closefrom %d", fd_base);
 
@@ -196,6 +317,7 @@ main(int argc, char *argv[])
 		err(1, "setrlimit %llu", rlim.rlim_cur);
 
 	run = 1;
+
 	tsocket = calloc(socket_num, sizeof(pthread_t));
 	if (tsocket == NULL)
 		err(1, "tsocket");
@@ -204,6 +326,7 @@ main(int argc, char *argv[])
 		if (errno)
 			err(1, "pthread_create socket %u", n);
 	}
+
 	tclose = calloc(close_num, sizeof(pthread_t));
 	if (tclose == NULL)
 		err(1, "tclose");
@@ -212,6 +335,7 @@ main(int argc, char *argv[])
 		if (errno)
 			err(1, "pthread_create close %u", n);
 	}
+
 	tbind = calloc(bind_num, sizeof(pthread_t));
 	if (tbind == NULL)
 		err(1, "tbind");
@@ -220,6 +344,7 @@ main(int argc, char *argv[])
 		if (errno)
 			err(1, "pthread_create bind %u", n);
 	}
+
 	tconnect = calloc(connect_num, sizeof(pthread_t));
 	if (tconnect == NULL)
 		err(1, "tconnect");
@@ -230,6 +355,16 @@ main(int argc, char *argv[])
 			err(1, "pthread_create connect %u", n);
 	}
 
+	tdelroute = calloc(delroute_num, sizeof(pthread_t));
+	if (tdelroute == NULL)
+		err(1, "tdelroute");
+	for (n = 0; n < delroute_num; n++) {
+		errno = pthread_create(&tdelroute[n], NULL, thread_delroute,
+		    &run);
+		if (errno)
+			err(1, "pthread_create delroute %u", n);
+	}
+
 	if (run_time > 0) {
 		if (sleep(run_time) < 0)
 			err(1, "sleep %u", run_time);
@@ -245,6 +380,8 @@ main(int argc, char *argv[])
 			err(1, "pthread_join socket %u", n);
 		socket_count += count;
 	}
+	free(tsocket);
+
 	close_count = 0;
 	for (n = 0; n < close_num; n++) {
 		unsigned long count;
@@ -254,6 +391,8 @@ main(int argc, char *argv[])
 			err(1, "pthread_join close %u", n);
 		close_count += count;
 	}
+	free(tclose);
+
 	bind_count = 0;
 	for (n = 0; n < bind_num; n++) {
 		unsigned long count;
@@ -263,6 +402,8 @@ main(int argc, char *argv[])
 			err(1, "pthread_join bind %u", n);
 		bind_count += count;
 	}
+	free(tbind);
+
 	connect_count = 0;
 	for (n = 0; n < connect_num; n++) {
 		unsigned long count;
@@ -272,8 +413,23 @@ main(int argc, char *argv[])
 			err(1, "pthread_join connect %u", n);
 		connect_count += count;
 	}
-	printf("count: socket %lu, close %lu, bind %lu, connect %lu\n",
-	    socket_count, close_count, bind_count, connect_count);
+	free(tconnect);
+
+	delroute_count = 0;
+	for (n = 0; n < delroute_num; n++) {
+		unsigned long count;
+
+		errno = pthread_join(tdelroute[n], (void **)&count);
+		if (errno)
+			err(1, "pthread_join delroute %u", n);
+		delroute_count += count;
+	}
+	free(tdelroute);
+
+	printf("count: socket %lu, close %lu, bind %lu, connect %lu, "
+	    "delroute %lu\n",
+	    socket_count, close_count, bind_count, connect_count,
+	    delroute_count);
 
 	return 0;
 }