Update to nsd 4.4.0

tested by sthen and me
OK sthen

30 files changed, 418 insertions, 105 deletions

diff --git a/usr.sbin/nsd/Makefile.in b/usr.sbin/nsd/Makefile.in
index 8aa40269f2a..e28fc47cd32 100644
--- a/usr.sbin/nsd/Makefile.in
+++ b/usr.sbin/nsd/Makefile.in
@@ -586,7 +586,7 @@ cutest_udb.o: $(srcdir)/tpkg/cutest/cutest_udb.c config.h $(srcdir)/tpkg/cutest/
 cutest_udbrad.o: $(srcdir)/tpkg/cutest/cutest_udbrad.c config.h \
  $(srcdir)/tpkg/cutest/cutest.h $(srcdir)/udbradtree.h $(srcdir)/udb.h
 cutest_util.o: $(srcdir)/tpkg/cutest/cutest_util.c config.h $(srcdir)/tpkg/cutest/cutest.h \
- $(srcdir)/region-allocator.h $(srcdir)/util.h
+ $(srcdir)/region-allocator.h $(srcdir)/util.h $(srcdir)/xfrd-tcp.h
 qtest.o: $(srcdir)/tpkg/cutest/qtest.c config.h $(srcdir)/tpkg/cutest/qtest.h $(srcdir)/buffer.h \
  $(srcdir)/region-allocator.h $(srcdir)/util.h $(srcdir)/query.h $(srcdir)/namedb.h $(srcdir)/dname.h $(srcdir)/buffer.h $(srcdir)/dns.h \
  $(srcdir)/radtree.h $(srcdir)/rbtree.h $(srcdir)/nsd.h $(srcdir)/edns.h $(srcdir)/packet.h $(srcdir)/tsig.h $(srcdir)/namedb.h $(srcdir)/util.h $(srcdir)/nsec3.h \
diff --git a/usr.sbin/nsd/configlexer.lex b/usr.sbin/nsd/configlexer.lex
index d5fcd58b7f6..a168834067a 100644
--- a/usr.sbin/nsd/configlexer.lex
+++ b/usr.sbin/nsd/configlexer.lex
@@ -297,6 +297,8 @@ tls-cert-bundle{COLON}	{ LEXOUT(("v(%s) ", yytext)); return VAR_TLS_CERT_BUNDLE;
 answer-cookie{COLON}	{ LEXOUT(("v(%s) ", yytext)); return VAR_ANSWER_COOKIE;}
 cookie-secret{COLON}	{ LEXOUT(("v(%s) ", yytext)); return VAR_COOKIE_SECRET;}
 cookie-secret-file{COLON}	{ LEXOUT(("v(%s) ", yytext)); return VAR_COOKIE_SECRET_FILE;}
+xfrd-tcp-max{COLON}	{ LEXOUT(("v(%s) ", yytext)); return VAR_XFRD_TCP_MAX;}
+xfrd-tcp-pipeline{COLON}	{ LEXOUT(("v(%s) ", yytext)); return VAR_XFRD_TCP_PIPELINE;}
 {NEWLINE}		{ LEXOUT(("NL\n")); cfg_parser->line++;}
 
 servers={UNQUOTEDLETTER}*	{
diff --git a/usr.sbin/nsd/configparser.y b/usr.sbin/nsd/configparser.y
index 70e54cf21f3..0815a0a16af 100644
--- a/usr.sbin/nsd/configparser.y
+++ b/usr.sbin/nsd/configparser.y
@@ -119,6 +119,8 @@ static int parse_range(const char *str, long long *low, long long *high);
 %token VAR_XFRD_CPU_AFFINITY
 %token <llng> VAR_SERVER_CPU_AFFINITY
 %token VAR_DROP_UPDATES
+%token VAR_XFRD_TCP_MAX
+%token VAR_XFRD_TCP_PIPELINE
 
 /* dnstap */
 %token VAR_DNSTAP
@@ -454,6 +456,10 @@ server_option:
     { cfg_parser->opt->cookie_secret = region_strdup(cfg_parser->opt->region, $2); }
   | VAR_COOKIE_SECRET_FILE STRING
     { cfg_parser->opt->cookie_secret_file = region_strdup(cfg_parser->opt->region, $2); }
+  | VAR_XFRD_TCP_MAX number
+    { cfg_parser->opt->xfrd_tcp_max = (int)$2; }
+  | VAR_XFRD_TCP_PIPELINE number
+    { cfg_parser->opt->xfrd_tcp_pipeline = (int)$2; }
   | VAR_CPU_AFFINITY cpus
     {
       cfg_parser->opt->cpu_affinity = $2;
diff --git a/usr.sbin/nsd/configure b/usr.sbin/nsd/configure
index ca313c85794..0a5938d017b 100644
--- a/usr.sbin/nsd/configure
+++ b/usr.sbin/nsd/configure
@@ -1,6 +1,6 @@
 #! /bin/sh
 # Guess values for system-dependent variables and create Makefiles.
-# Generated by GNU Autoconf 2.69 for NSD 4.3.9.
+# Generated by GNU Autoconf 2.69 for NSD 4.4.0.
 #
 # Report bugs to <nsd-bugs@nlnetlabs.nl>.
 #
@@ -580,8 +580,8 @@ MAKEFLAGS=
 # Identity of this package.
 PACKAGE_NAME='NSD'
 PACKAGE_TARNAME='nsd'
-PACKAGE_VERSION='4.3.9'
-PACKAGE_STRING='NSD 4.3.9'
+PACKAGE_VERSION='4.4.0'
+PACKAGE_STRING='NSD 4.4.0'
 PACKAGE_BUGREPORT='nsd-bugs@nlnetlabs.nl'
 PACKAGE_URL=''
 
@@ -1328,7 +1328,7 @@ if test "$ac_init_help" = "long"; then
   # Omit some internal or obsolete options to make the list less imposing.
   # This message is too long to be a string in the A/UX 3.1 sh.
   cat <<_ACEOF
-\`configure' configures NSD 4.3.9 to adapt to many kinds of systems.
+\`configure' configures NSD 4.4.0 to adapt to many kinds of systems.
 
 Usage: $0 [OPTION]... [VAR=VALUE]...
 
@@ -1390,7 +1390,7 @@ fi
 
 if test -n "$ac_init_help"; then
   case $ac_init_help in
-     short | recursive ) echo "Configuration of NSD 4.3.9:";;
+     short | recursive ) echo "Configuration of NSD 4.4.0:";;
    esac
   cat <<\_ACEOF
 
@@ -1563,7 +1563,7 @@ fi
 test -n "$ac_init_help" && exit $ac_status
 if $ac_init_version; then
   cat <<\_ACEOF
-NSD configure 4.3.9
+NSD configure 4.4.0
 generated by GNU Autoconf 2.69
 
 Copyright (C) 2012 Free Software Foundation, Inc.
@@ -2272,7 +2272,7 @@ cat >config.log <<_ACEOF
 This file contains any messages produced by compilers while
 running configure, to aid debugging if configure makes a mistake.
 
-It was created by NSD $as_me 4.3.9, which was
+It was created by NSD $as_me 4.4.0, which was
 generated by GNU Autoconf 2.69.  Invocation command line was
 
   $ $0 $@
@@ -11155,7 +11155,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1
 # report actual input values of CONFIG_FILES etc. instead of their
 # values after options handling.
 ac_log="
-This file was extended by NSD $as_me 4.3.9, which was
+This file was extended by NSD $as_me 4.4.0, which was
 generated by GNU Autoconf 2.69.  Invocation command line was
 
   CONFIG_FILES    = $CONFIG_FILES
@@ -11217,7 +11217,7 @@ _ACEOF
 cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1
 ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`"
 ac_cs_version="\\
-NSD config.status 4.3.9
+NSD config.status 4.4.0
 configured by $0, generated by GNU Autoconf 2.69,
   with options \\"\$ac_cs_config\\"
 
diff --git a/usr.sbin/nsd/configure.ac b/usr.sbin/nsd/configure.ac
index 94a2e1ee38d..979b9ecda56 100644
--- a/usr.sbin/nsd/configure.ac
+++ b/usr.sbin/nsd/configure.ac
@@ -5,7 +5,7 @@ dnl
 sinclude(acx_nlnetlabs.m4)
 sinclude(dnstap/dnstap.m4)
 
-AC_INIT([NSD],[4.3.9],[nsd-bugs@nlnetlabs.nl])
+AC_INIT([NSD],[4.4.0],[nsd-bugs@nlnetlabs.nl])
 AC_CONFIG_HEADERS([config.h])
 
 #
diff --git a/usr.sbin/nsd/dbaccess.c b/usr.sbin/nsd/dbaccess.c
index 4291451a250..4849481f483 100644
--- a/usr.sbin/nsd/dbaccess.c
+++ b/usr.sbin/nsd/dbaccess.c
@@ -491,7 +491,7 @@ namedb_open (const char* filename, struct nsd_options* opt)
 #endif /* HAVE_MMAP */
 }
 
-/** the the file mtime stat (or nonexist or error) */
+/** get the file mtime stat (or nonexist or error) */
 int
 file_get_mtime(const char* file, struct timespec* mtime, int* nonexist)
 {
diff --git a/usr.sbin/nsd/doc/ChangeLog b/usr.sbin/nsd/doc/ChangeLog
index 0075ef70e2a..24b6e09c6c1 100644
--- a/usr.sbin/nsd/doc/ChangeLog
+++ b/usr.sbin/nsd/doc/ChangeLog
@@ -1,9 +1,54 @@
+10 February 2022: Wouter
+	- Tag for 4.4.0rc1 release.
+
+9 February 2022: Wouter
+	- Fix unit tests for nds-control-setup exit code and the
+	  xfrd-tcp-max default.
+
+7 February 2022: Wouter
+	- Merge #207 Sync nsd-control-setup with unbound-control-setup to
+	  generate certificates with SANs.
+
+28 January 2022: Wouter
+	- Fix #206: build with --without-ssl fails.
+
+27 January 2022: Wouter
+	- current code branch continues as version 4.4.0, because of added
+	  feature.
+
+26 January 2022: Wouter
+	- Merge #193: Lower memory usage of the XFRD process by default.
+	  Instead of preallocating all elements, they are allocated when used.
+	  There are options for managing the memory usage, defaults are the
+	  same as before. xfrd-tcp-max sets the number of sockets for tcp
+	  connections that xfrd can make to download zone contents. And
+	  xfrd-tcp-pipeline the number of simultaneous transfers over the
+	  same connection.
+
+12 January 2022: Wouter
+	- Fix to document nsd-checkzone -p in the man page for nsd-checkzone.
+
+7 January 2022: Wouter
+	- Fix to change file mode before changing file owner for the
+	  nsd-control unix socket file.
+
+3 January 2022: Wouter
+	- Merge #204 from jonathangray: correct some spelling mistakes.
+
+15 December 2021: Wouter
+	- Fix #200: nsd-checkzone succeeds even with incorrect serial in SOA
+	  record.
+
 2 December 2021: Wouter
 	- Fix socket_partitioning unit test for FreeBSD.
 	- Fix SVCB test to work around older dig with drill.
+	- Fix unit test to not syslog setlogin failures.
 
 1 December 2021: Wouter
 	- Set up for branch for 4.3.9 release.
+	  This became release 4.3.9 on 9 Dec 2021 and included the changes
+	  until the SVCB fix on 2 dec 2021, but not the setlogin fix.
+	  The main branch continues as 4.3.10.
 	- Fix unit tests for new answer-cookie default.
 
 30 November 2021: Wouter
diff --git a/usr.sbin/nsd/doc/RELNOTES b/usr.sbin/nsd/doc/RELNOTES
index 176d762de21..aa649184046 100644
--- a/usr.sbin/nsd/doc/RELNOTES
+++ b/usr.sbin/nsd/doc/RELNOTES
@@ -1,5 +1,28 @@
 NSD RELEASE NOTES
 
+4.4.0
+================
+FEATURES:
+	- Merge #193: Lower memory usage of the XFRD process by default.
+	  Instead of preallocating all elements, they are allocated when used.
+	  There are options for managing the memory usage, defaults are the
+	  same as before. xfrd-tcp-max sets the number of sockets for tcp
+	  connections that xfrd can make to download zone contents. And
+	  xfrd-tcp-pipeline the number of simultaneous transfers over the
+	  same connection.
+BUG FIXES:
+	- Fix #200: nsd-checkzone succeeds even with incorrect serial in SOA
+	  record.
+	- Merge #204 from jonathangray: correct some spelling mistakes.
+	- Fix to change file mode before changing file owner for the
+	  nsd-control unix socket file.
+	- Fix to document nsd-checkzone -p in the man page for nsd-checkzone.
+	- Fix #206: build with --without-ssl fails.
+	- Merge #207 Sync nsd-control-setup with unbound-control-setup to
+	  generate certificates with SANs.
+	- Fix unit tests for nds-control-setup exit code and the
+	  xfrd-tcp-max default.
+
 4.3.9
 ================
 BUG FIXES:
diff --git a/usr.sbin/nsd/edns.c b/usr.sbin/nsd/edns.c
index c7fc39d9d01..478ec681df5 100644
--- a/usr.sbin/nsd/edns.c
+++ b/usr.sbin/nsd/edns.c
@@ -21,6 +21,12 @@
 #include "nsd.h"
 #include "query.h"
 
+#if !defined(HAVE_SSL) || !defined(HAVE_CRYPTO_MEMCMP)
+/* we need fixed time compare, pull it in from tsig.c */
+#define CRYPTO_memcmp memcmp_fixedtime
+int memcmp_fixedtime(const void *s1, const void *s2, size_t n);
+#endif
+
 void
 edns_init_data(edns_data_type *data, uint16_t max_length)
 {
diff --git a/usr.sbin/nsd/lookup3.c b/usr.sbin/nsd/lookup3.c
index 9602ffa374b..9b47d156ca9 100644
--- a/usr.sbin/nsd/lookup3.c
+++ b/usr.sbin/nsd/lookup3.c
@@ -872,7 +872,7 @@ void driver2()
     {
       for (j=0; j<8; ++j)   /*------------------------ for each input bit, */
       {
-	for (m=1; m<8; ++m) /*------------ for serveral possible initvals, */
+	for (m=1; m<8; ++m) /*------------ for several possible initvals, */
 	{
 	  for (l=0; l<HASHSTATE; ++l)
 	    e[l]=f[l]=g[l]=h[l]=x[l]=y[l]=~((uint32_t)0);
diff --git a/usr.sbin/nsd/nsd-checkconf.8.in b/usr.sbin/nsd/nsd-checkconf.8.in
index d97e0217ccf..60fb0693f40 100644
--- a/usr.sbin/nsd/nsd-checkconf.8.in
+++ b/usr.sbin/nsd/nsd-checkconf.8.in
@@ -1,4 +1,4 @@
-.TH "nsd\-checkconf" "8" "Dec  9, 2021" "NLnet Labs" "nsd 4.3.9"
+.TH "nsd\-checkconf" "8" "Feb 17, 2022" "NLnet Labs" "nsd 4.4.0"
 .\" Copyright (c) 2001\-2008, NLnet Labs. All rights reserved.
 .\" See LICENSE for the license.
 .SH "NAME"
diff --git a/usr.sbin/nsd/nsd-checkconf.c b/usr.sbin/nsd/nsd-checkconf.c
index 342b308fa08..b9c947288d0 100644
--- a/usr.sbin/nsd/nsd-checkconf.c
+++ b/usr.sbin/nsd/nsd-checkconf.c
@@ -433,6 +433,8 @@ config_print_zone(nsd_options_type* opt, const char* k, int s, const char *o,
 		SERV_GET_INT(tcp_timeout, o);
 		SERV_GET_INT(tcp_mss, o);
 		SERV_GET_INT(outgoing_tcp_mss, o);
+		SERV_GET_INT(xfrd_tcp_max, o);
+		SERV_GET_INT(xfrd_tcp_pipeline, o);
 		SERV_GET_INT(ipv4_edns_size, o);
 		SERV_GET_INT(ipv6_edns_size, o);
 		SERV_GET_INT(statistics, o);
@@ -581,6 +583,8 @@ config_test_print_server(nsd_options_type* opt)
 	printf("\ttcp-timeout: %d\n", opt->tcp_timeout);
 	printf("\ttcp-mss: %d\n", opt->tcp_mss);
 	printf("\toutgoing-tcp-mss: %d\n", opt->outgoing_tcp_mss);
+	printf("\txfrd-tcp-max: %d\n", opt->xfrd_tcp_max);
+	printf("\txfrd-tcp-pipeline: %d\n", opt->xfrd_tcp_pipeline);
 	printf("\tipv4-edns-size: %d\n", (int) opt->ipv4_edns_size);
 	printf("\tipv6-edns-size: %d\n", (int) opt->ipv6_edns_size);
 	print_string_var("pidfile:", opt->pidfile);
diff --git a/usr.sbin/nsd/nsd-checkzone.8.in b/usr.sbin/nsd/nsd-checkzone.8.in
index c43a34aadeb..bc2cb47d1a5 100644
--- a/usr.sbin/nsd/nsd-checkzone.8.in
+++ b/usr.sbin/nsd/nsd-checkzone.8.in
@@ -1,4 +1,4 @@
-.TH "nsd\-checkzone" "8" "Dec  9, 2021" "NLnet Labs" "nsd 4.3.9"
+.TH "nsd\-checkzone" "8" "Feb 17, 2022" "NLnet Labs" "nsd 4.4.0"
 .\" Copyright (c) 2014, NLnet Labs. All rights reserved.
 .\" See LICENSE for the license.
 .SH "NAME"
@@ -25,6 +25,12 @@ The name of the zone to check, eg. "example.com".
 .TP
 .I zonefile
 The file to read, eg. "zones/example.com.zone.signed".
+.TP
+.B \-p
+Print the zone contents to stdout if the zone is ok. This prints the
+contents as it has been parsed, not literally a copy of the input, but
+as printed by the formatting routines in NSD, much like the nsd-control
+command write does.
 .SH "SEE ALSO"
 \fInsd\fR(8), \fInsd-checkconf\fR(8)
 .SH "AUTHORS"
diff --git a/usr.sbin/nsd/nsd-control-setup.sh.in b/usr.sbin/nsd/nsd-control-setup.sh.in
index ab9b1eaf4c5..d1cc038111b 100644
--- a/usr.sbin/nsd/nsd-control-setup.sh.in
+++ b/usr.sbin/nsd/nsd-control-setup.sh.in
@@ -43,7 +43,7 @@ SERVERNAME=nsd
 CLIENTNAME=nsd-control
 
 # validity period for certificates
-DAYS=3650
+DAYS=7200
 
 # size of keys in bits
 BITS=3072
@@ -86,9 +86,7 @@ fatal() {
 usage() {
     cat <<EOF
 usage: $0 OPTIONS
-
 OPTIONS
-
 -d <dir>  used directory to store keys and certificates (default: $DESTDIR)
 -h        show help notice
 -r        recreate certificates
@@ -99,7 +97,7 @@ OPTIND=1
 while getopts 'd:hr' arg; do
     case "$arg" in
       d) DESTDIR="$OPTARG" ;;
-      h) usage; exit 0 ;;
+      h) usage; exit 1 ;;
       r) RECREATE=1 ;;
       ?) fatal "'$arg' unknown option" ;;
     esac
@@ -122,13 +120,19 @@ if [ ! -f "$SVR_BASE.key" ]; then
 fi
 
 cat >server.cnf <<EOF
+[req]
 default_bits=$BITS
 default_md=$HASH
 prompt=no
 distinguished_name=req_distinguished_name
-
+x509_extensions=v3_ca
 [req_distinguished_name]
 commonName=$SERVERNAME
+[v3_ca]
+subjectKeyIdentifier=hash
+authorityKeyIdentifier=keyid:always,issuer:always
+basicConstraints=critical,CA:TRUE,pathlen:0
+subjectAltName=DNS:$SERVERNAME
 EOF
 
 [ -f server.cnf ] || fatal "cannot create openssl configuration"
@@ -159,9 +163,12 @@ default_bits=$BITS
 default_md=$HASH
 prompt=no
 distinguished_name=req_distinguished_name
-
+req_extensions=v3_req
 [req_distinguished_name]
 commonName=$CLIENTNAME
+[v3_req]
+basicConstraints=critical,CA:FALSE
+subjectAltName=DNS:$CLIENTNAME
 EOF
 
 [ -f client.cnf ] || fatal "cannot create openssl configuration"
@@ -183,6 +190,8 @@ if [ ! -f "$CTL_BASE.pem" -o $RECREATE -eq 1 ]; then
                   -CAkey "$SVR_BASE.key" \
                   -CAcreateserial \
                   -$HASH \
+                  -extfile client.cnf \
+                  -extensions v3_req \
                   -out "$CTL_BASE.pem"
 
     [ ! -f "CTL_BASE.pem" ] || fatal "cannot create signed client certificate"
@@ -199,7 +208,6 @@ cleanup
 
 echo "Setup success. Certificates created. Enable in nsd.conf file to use"
 
-
 # create trusted usage pem
 # openssl x509 -in $CTL_BASE.pem -addtrust clientAuth -out $CTL_BASE"_trust.pem"
 
diff --git a/usr.sbin/nsd/nsd-control.8.in b/usr.sbin/nsd/nsd-control.8.in
index 6051468b4d1..8242eeb8332 100644
--- a/usr.sbin/nsd/nsd-control.8.in
+++ b/usr.sbin/nsd/nsd-control.8.in
@@ -1,4 +1,4 @@
-.TH "nsd\-control" "8" "Dec  9, 2021" "NLnet Labs" "nsd 4.3.9"
+.TH "nsd\-control" "8" "Feb 17, 2022" "NLnet Labs" "nsd 4.4.0"
 .\" Copyright (c) 2011, NLnet Labs. All rights reserved.
 .\" See LICENSE for the license.
 .SH "NAME"
diff --git a/usr.sbin/nsd/nsd.8.in b/usr.sbin/nsd/nsd.8.in
index 48682e90eb8..9edbf09b85c 100644
--- a/usr.sbin/nsd/nsd.8.in
+++ b/usr.sbin/nsd/nsd.8.in
@@ -1,9 +1,9 @@
-.TH "NSD" "8" "Dec  9, 2021" "NLnet Labs" "NSD 4.3.9"
+.TH "NSD" "8" "Feb 17, 2022" "NLnet Labs" "NSD 4.4.0"
 .\" Copyright (c) 2001\-2008, NLnet Labs. All rights reserved.
 .\" See LICENSE for the license.
 .SH "NAME"
 .B nsd
-\- Name Server Daemon (NSD) version 4.3.9.
+\- Name Server Daemon (NSD) version 4.4.0.
 .SH "SYNOPSIS"
 .B nsd
 .RB [ \-4 ] 
diff --git a/usr.sbin/nsd/nsd.conf.5.in b/usr.sbin/nsd/nsd.conf.5.in
index cf05f082c85..f1299c1db25 100644
--- a/usr.sbin/nsd/nsd.conf.5.in
+++ b/usr.sbin/nsd/nsd.conf.5.in
@@ -1,4 +1,4 @@
-.TH "nsd.conf" "5" "Dec  9, 2021" "NLnet Labs" "nsd 4.3.9"
+.TH "nsd.conf" "5" "Feb 17, 2022" "NLnet Labs" "nsd 4.4.0"
 .\" Copyright (c) 2001\-2008, NLnet Labs. All rights reserved.
 .\" See LICENSE for the license.
 .SH "NAME"
@@ -298,13 +298,29 @@ negotiation between server and client.
 .TP
 .B outgoing\-tcp\-mss:\fR <number>
 Maximum segment size (MSS) of TCP socket for outgoing XFR request
-to other namesevers. Value lower than
+to other nameservers. Value lower than
 common MSS on Ethernet (1220 for example) will
 address path MTU problem.
 Note that not all platform supports socket option to set MSS (TCP_MAXSEG).
 Default is system default MSS determined by interface MTU and
 negotiation between NSD and other servers.
 .TP
+.B xfrd\-tcp\-max:\fR <number>
+Number of sockets for xfrd to use for outgoing zone transfers. Default 128.
+Increase it to allow more zone transfer sockets, like to 256.
+To save memory, this can be lowered, set it lower together with some other
+settings to have reduced memory footprint for NSD. xfrd\-tcp\-max: 32
+and xfrd\-tcp\-pipeline: 128 and rrl\-size: 1000
+.IP
+This reduces memory footprint, other memory usage is caused mainly by
+the server\-count setting, the number of server processes, and the
+tcp\-count setting, which keeps buffers per server process, and by the
+size of the zone data.
+.TP
+.B xfrd\-tcp\-pipeline:\fR <number>
+Number of simultaneous outgoing zone transfers that are possible on the
+tcp sockets of xfrd. Max is 65536, default is 128.
+.TP
 .B ipv4\-edns\-size:\fR <number>
 Preferred EDNS buffer size for IPv4.  Default 1232.
 .TP
@@ -493,7 +509,7 @@ With the value 0 the rate is unlimited.
 .\" rrlend
 .TP
 .B answer\-cookie:\fR <yes or no>
-Enable to answer to requests containig DNS Cookies as specified in RFC7873.
+Enable to answer to requests containing DNS Cookies as specified in RFC7873.
 Default is yes.
 .TP
 .B cookie\-secret:\fR <128 bit hex string>
diff --git a/usr.sbin/nsd/nsd.conf.sample.in b/usr.sbin/nsd/nsd.conf.sample.in
index 9758c37b0c0..0e006bafb4c 100644
--- a/usr.sbin/nsd/nsd.conf.sample.in
+++ b/usr.sbin/nsd/nsd.conf.sample.in
@@ -174,6 +174,15 @@ server:
 	# Default is 0, system default MSS.
 	# outgoing-tcp-mss: 0
 
+	# reduce these settings to save memory for NSD, to about
+	# xfrd-tcp-max: 32 and xfrd-tcp-pipeline: 128, also rrl-size: 1000
+	# other memory is determined by server-count, tcp-count and zone data
+	# max number of sockets used for outgoing zone transfers.
+	# Increase this to allow more sockets for zone transfers.
+	# xfrd-tcp-max: 128
+	# max number of simultaneous outgoing zone transfers over one socket.
+	# xfrd-tcp-pipeline: 128
+
 	# Preferred EDNS buffer size for IPv4.
 	# ipv4-edns-size: 1232
 
@@ -378,7 +387,7 @@ remote-control:
 	#request-xfr: 192.0.2.2 the_tsig_key_name
 	#request-xfr: 192.0.2.2 the_tsig_key_name the_tls_auth_name
 	# Attention: You cannot use UDP and AXFR together. AXFR is always over
-	# TCP. If you use UDP, we higly recommend you to deploy TSIG.
+	# TCP. If you use UDP, we highly recommend you to deploy TSIG.
 	# Allow AXFR fallback if the master does not support IXFR. Default
 	# is yes.
 	#allow-axfr-fallback: yes
diff --git a/usr.sbin/nsd/options.c b/usr.sbin/nsd/options.c
index d8fe022b412..c684698b175 100644
--- a/usr.sbin/nsd/options.c
+++ b/usr.sbin/nsd/options.c
@@ -91,6 +91,8 @@ nsd_options_create(region_type* region)
 	opt->port = UDP_PORT;
 /* deprecated?	opt->port = TCP_PORT; */
 	opt->reuseport = 0;
+	opt->xfrd_tcp_max = 128;
+	opt->xfrd_tcp_pipeline = 128;
 	opt->statistics = 0;
 	opt->chroot = 0;
 	opt->username = USER;
diff --git a/usr.sbin/nsd/options.h b/usr.sbin/nsd/options.h
index f1c57aea2bf..54357c863a6 100644
--- a/usr.sbin/nsd/options.h
+++ b/usr.sbin/nsd/options.h
@@ -112,6 +112,10 @@ struct nsd_options {
 	int minimal_responses;
 	int refuse_any;
 	int reuseport;
+	/* max number of xfrd tcp sockets */
+	int xfrd_tcp_max;
+	/* max number of simultaneous requests on xfrd tcp socket */
+	int xfrd_tcp_pipeline;
 
 	/* private key file for TLS */
 	char* tls_service_key;
diff --git a/usr.sbin/nsd/remote.c b/usr.sbin/nsd/remote.c
index 81ec53f5eb8..c94a1d90b1b 100644
--- a/usr.sbin/nsd/remote.c
+++ b/usr.sbin/nsd/remote.c
@@ -436,6 +436,9 @@ add_open(struct daemon_remote* rc, struct nsd_options* cfg, const char* ip,
 		 */
 		if(fd != -1) {
 #ifdef HAVE_CHOWN
+			if(chmod(ip, (mode_t)(S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP)) == -1) {
+				VERBOSITY(3, (LOG_INFO, "cannot chmod control socket %s: %s", ip, strerror(errno)));
+			}
 			if (cfg->username && cfg->username[0] &&
 				nsd.uid != (uid_t)-1) {
 				if(chown(ip, nsd.uid, nsd.gid) == -1)
@@ -443,9 +446,6 @@ add_open(struct daemon_remote* rc, struct nsd_options* cfg, const char* ip,
 					  (unsigned)nsd.uid, (unsigned)nsd.gid,
 					  ip, strerror(errno)));
 			}
-			if(chmod(ip, (mode_t)(S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP)) == -1) {
-				VERBOSITY(3, (LOG_INFO, "cannot chmod control socket %s: %s", ip, strerror(errno)));
-			}
 #else
 			(void)cfg;
 #endif
diff --git a/usr.sbin/nsd/rrl.h b/usr.sbin/nsd/rrl.h
index 8dbc3184d1a..b08d65f7861 100644
--- a/usr.sbin/nsd/rrl.h
+++ b/usr.sbin/nsd/rrl.h
@@ -51,7 +51,7 @@ void rrl_mmap_init(int numch, size_t numbuck, size_t lm, size_t wlm, size_t sm,
  */
 void rrl_init(size_t ch);
 
-/** deinit (for this child server processs) */
+/** deinit (for this child server process) */
 void rrl_deinit(size_t ch);
 
 /** deinit mmaps for n children */
diff --git a/usr.sbin/nsd/server.c b/usr.sbin/nsd/server.c
index f7a2f53c60a..1e29ccc92cc 100644
--- a/usr.sbin/nsd/server.c
+++ b/usr.sbin/nsd/server.c
@@ -367,7 +367,7 @@ static void report_tcp_fastopen_config() {
 	}
 	if (!(tcp_fastopen_value & TCP_FASTOPEN_SERVER_BIT_MASK)) {
 		log_msg(LOG_WARNING, "Error: TCP Fast Open support is available and configured in NSD by default.\n");
-		log_msg(LOG_WARNING, "However the kernel paramenters are not configured to support TCP_FASTOPEN in server mode.\n");
+		log_msg(LOG_WARNING, "However the kernel parameters are not configured to support TCP_FASTOPEN in server mode.\n");
 		log_msg(LOG_WARNING, "To enable TFO use the command:");
 		log_msg(LOG_WARNING, "  'sudo sysctl -w net.ipv4.tcp_fastopen=2' for pure server mode or\n");
 		log_msg(LOG_WARNING, "  'sudo sysctl -w net.ipv4.tcp_fastopen=3' for both client and server mode\n");
diff --git a/usr.sbin/nsd/udb.c b/usr.sbin/nsd/udb.c
index fb129b82100..1131fb43e79 100644
--- a/usr.sbin/nsd/udb.c
+++ b/usr.sbin/nsd/udb.c
@@ -1149,7 +1149,7 @@ grow_extra_check(udb_alloc* alloc, uint64_t ge)
 	return ge;
 }
 
-/** see if free space is enogh to warrant shrink (while file is open) */
+/** see if free space is enough to warrant shrink (while file is open) */
 static int
 enough_free(udb_alloc* alloc)
 {
diff --git a/usr.sbin/nsd/udb.h b/usr.sbin/nsd/udb.h
index 3ee98648b15..b3f495d9b38 100644
--- a/usr.sbin/nsd/udb.h
+++ b/usr.sbin/nsd/udb.h
@@ -198,7 +198,7 @@ struct udb_base {
 	udb_ptr** ram_hash;
 	/** size of the current udb_ptr hashtable array */
 	size_t ram_size;
-	/** mask for the curren udb_ptr hashtable lookups */
+	/** mask for the current udb_ptr hashtable lookups */
 	int ram_mask;
 	/** number of ptrs in ram, used to decide when to grow */
 	size_t ram_num;
diff --git a/usr.sbin/nsd/util.c b/usr.sbin/nsd/util.c
index 45954236309..7afe34e6518 100644
--- a/usr.sbin/nsd/util.c
+++ b/usr.sbin/nsd/util.c
@@ -529,7 +529,7 @@ strtoserial(const char* nptr, const char** endptr)
 			i += (**endptr - '0');
 			break;
 		default:
-			break;
+			return 0;
 		}
 	}
 	serial += i;
diff --git a/usr.sbin/nsd/xfrd-tcp.c b/usr.sbin/nsd/xfrd-tcp.c
index a71c1955cfb..df9caafa79d 100644
--- a/usr.sbin/nsd/xfrd-tcp.c
+++ b/usr.sbin/nsd/xfrd-tcp.c
@@ -156,12 +156,15 @@ xfrd_pipe_cmp(const void* a, const void* b)
 	return (uintptr_t)x < (uintptr_t)y ? -1 : 1;
 }
 
-struct xfrd_tcp_set* xfrd_tcp_set_create(struct region* region, const char *tls_cert_bundle)
+struct xfrd_tcp_set* xfrd_tcp_set_create(struct region* region, const char *tls_cert_bundle, int tcp_max, int tcp_pipeline)
 {
 	int i;
 	struct xfrd_tcp_set* tcp_set = region_alloc(region,
 		sizeof(struct xfrd_tcp_set));
 	memset(tcp_set, 0, sizeof(struct xfrd_tcp_set));
+	tcp_set->tcp_state = NULL;
+	tcp_set->tcp_max = tcp_max;
+	tcp_set->tcp_pipeline = tcp_pipeline;
 	tcp_set->tcp_count = 0;
 	tcp_set->tcp_waiting_first = 0;
 	tcp_set->tcp_waiting_last = 0;
@@ -180,27 +183,191 @@ struct xfrd_tcp_set* xfrd_tcp_set_create(struct region* region, const char *tls_
 	(void)tls_cert_bundle;
 	log_msg(LOG_INFO, "xfrd: No TLS 1.3 support - XFR-over-TLS not available");
 #endif
-	for(i=0; i<XFRD_MAX_TCP; i++)
-		tcp_set->tcp_state[i] = xfrd_tcp_pipeline_create(region);
+	tcp_set->tcp_state = region_alloc(region,
+		sizeof(*tcp_set->tcp_state)*tcp_set->tcp_max);
+	for(i=0; i<tcp_set->tcp_max; i++)
+		tcp_set->tcp_state[i] = xfrd_tcp_pipeline_create(region,
+			tcp_pipeline);
 	tcp_set->pipetree = rbtree_create(region, &xfrd_pipe_cmp);
 	return tcp_set;
 }
 
+static int pipeline_id_compare(const void* x, const void* y)
+{
+	struct xfrd_tcp_pipeline_id* a = (struct xfrd_tcp_pipeline_id*)x;
+	struct xfrd_tcp_pipeline_id* b = (struct xfrd_tcp_pipeline_id*)y;
+	if(a->id < b->id)
+		return -1;
+	if(a->id > b->id)
+		return 1;
+	return 0;
+}
+
+void pick_id_values(uint16_t* array, int num, int max)
+{
+	uint8_t inserted[65536];
+	int j, done;
+	if(num == 65536) {
+		/* all of them, loop and insert */
+		int i;
+		for(i=0; i<num; i++)
+			array[i] = (uint16_t)i;
+		return;
+	}
+	assert(max <= 65536);
+	/* This uses the Robert Floyd sampling algorithm */
+	/* keep track if values are already inserted, using the bitmap
+	 * in insert array */
+	memset(inserted, 0, sizeof(inserted[0])*max);
+	done=0;
+	for(j = max-num; j<max; j++) {
+		/* random generate creates from 0..arg-1 */
+		int t;
+		if(j+1 <= 1)
+			t = 0;
+		else	t = random_generate(j+1);
+		if(!inserted[t]) {
+			array[done++]=t;
+			inserted[t] = 1;
+		} else {
+			array[done++]=j;
+			inserted[j] = 1;
+		}
+	}
+}
+
+static void
+clear_pipeline_entry(struct xfrd_tcp_pipeline* tp, rbnode_type* node)
+{
+	struct xfrd_tcp_pipeline_id *n;
+	if(node == NULL || node == RBTREE_NULL)
+		return;
+	clear_pipeline_entry(tp, node->left);
+	node->left = NULL;
+	clear_pipeline_entry(tp, node->right);
+	node->right = NULL;
+	/* move the node into the free list */
+	n = (struct xfrd_tcp_pipeline_id*)node;
+	n->next_free = tp->pipe_id_free_list;
+	tp->pipe_id_free_list = n;
+}
+
+static void
+xfrd_tcp_pipeline_cleanup(struct xfrd_tcp_pipeline* tp)
+{
+	/* move entries into free list */
+	clear_pipeline_entry(tp, tp->zone_per_id->root);
+	/* clear the tree */
+	tp->zone_per_id->count = 0;
+	tp->zone_per_id->root = RBTREE_NULL;
+}
+
+static void
+xfrd_tcp_pipeline_init(struct xfrd_tcp_pipeline* tp)
+{
+	tp->key.node.key = tp;
+	tp->key.num_unused = tp->pipe_num;
+	tp->key.num_skip = 0;
+	tp->tcp_send_first = NULL;
+	tp->tcp_send_last = NULL;
+	xfrd_tcp_pipeline_cleanup(tp);
+	pick_id_values(tp->unused, tp->pipe_num, 65536);
+}
+
 struct xfrd_tcp_pipeline*
-xfrd_tcp_pipeline_create(region_type* region)
+xfrd_tcp_pipeline_create(region_type* region, int tcp_pipeline)
 {
 	int i;
 	struct xfrd_tcp_pipeline* tp = (struct xfrd_tcp_pipeline*)
 		region_alloc_zero(region, sizeof(*tp));
-	tp->key.num_unused = ID_PIPE_NUM;
-	assert(sizeof(tp->unused)/sizeof(tp->unused[0]) == ID_PIPE_NUM);
-	for(i=0; i<ID_PIPE_NUM; i++)
-		tp->unused[i] = (uint16_t)i;
+	if(tcp_pipeline < 0)
+		tcp_pipeline = 0;
+	if(tcp_pipeline > 65536)
+		tcp_pipeline = 65536; /* max 16 bit ID numbers */
+	tp->pipe_num = tcp_pipeline;
+	tp->key.num_unused = tp->pipe_num;
+	tp->zone_per_id = rbtree_create(region, &pipeline_id_compare);
+	tp->pipe_id_free_list = NULL;
+	for(i=0; i<tp->pipe_num; i++) {
+		struct xfrd_tcp_pipeline_id* n = (struct xfrd_tcp_pipeline_id*)
+			region_alloc_zero(region, sizeof(*n));
+		n->next_free = tp->pipe_id_free_list;
+		tp->pipe_id_free_list = n;
+	}
+	tp->unused = (uint16_t*)region_alloc_zero(region,
+		sizeof(tp->unused[0])*tp->pipe_num);
 	tp->tcp_r = xfrd_tcp_create(region, QIOBUFSZ);
 	tp->tcp_w = xfrd_tcp_create(region, 512);
+	xfrd_tcp_pipeline_init(tp);
 	return tp;
 }
 
+static struct xfrd_zone*
+xfrd_tcp_pipeline_lookup_id(struct xfrd_tcp_pipeline* tp, uint16_t id)
+{
+	struct xfrd_tcp_pipeline_id key;
+	rbnode_type* n;
+	memset(&key, 0, sizeof(key));
+	key.node.key = &key;
+	key.id = id;
+	n = rbtree_search(tp->zone_per_id, &key);
+	if(n && n != RBTREE_NULL) {
+		return ((struct xfrd_tcp_pipeline_id*)n)->zone;
+	}
+	return NULL;
+}
+
+static void
+xfrd_tcp_pipeline_insert_id(struct xfrd_tcp_pipeline* tp, uint16_t id,
+	struct xfrd_zone* zone)
+{
+	struct xfrd_tcp_pipeline_id* n;
+	/* because there are tp->pipe_num preallocated entries, and we have
+	 * only tp->pipe_num id values, the list cannot be empty now. */
+	assert(tp->pipe_id_free_list != NULL);
+	/* pick up next free xfrd_tcp_pipeline_id node */
+	n = tp->pipe_id_free_list;
+	tp->pipe_id_free_list = n->next_free;
+	n->next_free = NULL;
+	memset(&n->node, 0, sizeof(n->node));
+	n->node.key = n;
+	n->id = id;
+	n->zone = zone;
+	rbtree_insert(tp->zone_per_id, &n->node);
+}
+
+static void
+xfrd_tcp_pipeline_remove_id(struct xfrd_tcp_pipeline* tp, uint16_t id)
+{
+	struct xfrd_tcp_pipeline_id key;
+	rbnode_type* node;
+	memset(&key, 0, sizeof(key));
+	key.node.key = &key;
+	key.id = id;
+	node = rbtree_delete(tp->zone_per_id, &key);
+	if(node && node != RBTREE_NULL) {
+		struct xfrd_tcp_pipeline_id* n =
+			(struct xfrd_tcp_pipeline_id*)node;
+		n->next_free = tp->pipe_id_free_list;
+		tp->pipe_id_free_list = n;
+	}
+}
+
+static void
+xfrd_tcp_pipeline_skip_id(struct xfrd_tcp_pipeline* tp, uint16_t id)
+{
+	struct xfrd_tcp_pipeline_id key;
+	rbnode_type* n;
+	memset(&key, 0, sizeof(key));
+	key.node.key = &key;
+	key.id = id;
+	n = rbtree_search(tp->zone_per_id, &key);
+	if(n && n != RBTREE_NULL) {
+		struct xfrd_tcp_pipeline_id* zid = (struct xfrd_tcp_pipeline_id*)n;
+		zid->zone = TCP_NULL_SKIP;
+	}
+}
+
 void
 xfrd_setup_packet(buffer_type* packet,
 	uint16_t type, uint16_t klass, const dname_type* dname, uint16_t qid)
@@ -346,7 +513,7 @@ pipeline_find(struct xfrd_tcp_set* set, xfrd_zone_type* zone)
 	struct xfrd_tcp_pipeline_key k, *key=&k;
 	key->node.key = key;
 	key->ip_len = xfrd_acl_sockaddr_to(zone->master, &key->ip);
-	key->num_unused = ID_PIPE_NUM;
+	key->num_unused = set->tcp_pipeline;
 	/* lookup existing tcp transfer to the master with highest unused */
 	if(rbtree_find_less_equal(set->pipetree, key, &sme)) {
 		/* exact match, strange, fully unused tcp cannot be open */
@@ -407,11 +574,12 @@ tcp_pipe_sendlist_popfirst(struct xfrd_tcp_pipeline* tp, xfrd_zone_type* zone)
 
 /* remove zone from tcp pipe ID map */
 static void
-tcp_pipe_id_remove(struct xfrd_tcp_pipeline* tp, xfrd_zone_type* zone)
+tcp_pipe_id_remove(struct xfrd_tcp_pipeline* tp, xfrd_zone_type* zone,
+	int alsotree)
 {
-	assert(tp->key.num_unused < ID_PIPE_NUM && tp->key.num_unused >= 0);
-	assert(tp->id[zone->query_id] == zone);
-	tp->id[zone->query_id] = NULL;
+	assert(tp->key.num_unused < tp->pipe_num && tp->key.num_unused >= 0);
+	if(alsotree)
+		xfrd_tcp_pipeline_remove_id(tp, zone->query_id);
 	tp->unused[tp->key.num_unused] = zone->query_id;
 	/* must remove and re-add for sort order in tree */
 	(void)rbtree_delete(xfrd->tcp_set->pipetree, &tp->key.node);
@@ -423,22 +591,25 @@ tcp_pipe_id_remove(struct xfrd_tcp_pipeline* tp, xfrd_zone_type* zone)
 static void
 xfrd_tcp_pipe_stop(struct xfrd_tcp_pipeline* tp)
 {
-	int i, conn = -1;
-	assert(tp->key.num_unused < ID_PIPE_NUM); /* at least one 'in-use' */
-	assert(ID_PIPE_NUM - tp->key.num_unused > tp->key.num_skip); /* at least one 'nonskip' */
+	struct xfrd_tcp_pipeline_id* zid;
+	int conn = -1;
+	assert(tp->key.num_unused < tp->pipe_num); /* at least one 'in-use' */
+	assert(tp->pipe_num - tp->key.num_unused > tp->key.num_skip); /* at least one 'nonskip' */
 	/* need to retry for all the zones connected to it */
 	/* these could use different lists and go to a different nextmaster*/
-	for(i=0; i<ID_PIPE_NUM; i++) {
-		if(tp->id[i] && tp->id[i] != TCP_NULL_SKIP) {
-			xfrd_zone_type* zone = tp->id[i];
+	RBTREE_FOR(zid, struct xfrd_tcp_pipeline_id*, tp->zone_per_id) {
+		xfrd_zone_type* zone = zid->zone;
+		if(zone && zone != TCP_NULL_SKIP) {
+			assert(zone->query_id == zid->id);
 			conn = zone->tcp_conn;
 			zone->tcp_conn = -1;
 			zone->tcp_waiting = 0;
 			tcp_pipe_sendlist_remove(tp, zone);
-			tcp_pipe_id_remove(tp, zone);
+			tcp_pipe_id_remove(tp, zone, 0);
 			xfrd_set_refresh_now(zone);
 		}
 	}
+	xfrd_tcp_pipeline_cleanup(tp);
 	assert(conn != -1);
 	/* now release the entire tcp pipe */
 	xfrd_tcp_pipe_release(xfrd->tcp_set, tp, conn);
@@ -508,7 +679,7 @@ pipeline_setup_new_zone(struct xfrd_tcp_set* set, struct xfrd_tcp_pipeline* tp,
 	idx = random_generate(tp->key.num_unused);
 	zone->query_id = tp->unused[idx];
 	tp->unused[idx] = tp->unused[tp->key.num_unused-1];
-	tp->id[zone->query_id] = zone;
+	xfrd_tcp_pipeline_insert_id(tp, zone->query_id, zone);
 	/* decrement unused counter, and fixup tree */
 	(void)rbtree_delete(set->pipetree, &tp->key.node);
 	tp->key.num_unused--;
@@ -538,12 +709,12 @@ xfrd_tcp_obtain(struct xfrd_tcp_set* set, xfrd_zone_type* zone)
 	assert(zone->tcp_conn == -1);
 	assert(zone->tcp_waiting == 0);
 
-	if(set->tcp_count < XFRD_MAX_TCP) {
+	if(set->tcp_count < set->tcp_max) {
 		int i;
 		assert(!set->tcp_waiting_first);
 		set->tcp_count ++;
 		/* find a free tcp_buffer */
-		for(i=0; i<XFRD_MAX_TCP; i++) {
+		for(i=0; i<set->tcp_max; i++) {
 			if(set->tcp_state[i]->tcp_r->fd == -1) {
 				zone->tcp_conn = i;
 				break;
@@ -568,15 +739,7 @@ xfrd_tcp_obtain(struct xfrd_tcp_set* set, xfrd_zone_type* zone)
 			return;
 		}
 		/* ip and ip_len set by tcp_open */
-		tp->key.node.key = tp;
-		tp->key.num_unused = ID_PIPE_NUM;
-		tp->key.num_skip = 0;
-		tp->tcp_send_first = NULL;
-		tp->tcp_send_last = NULL;
-		memset(tp->id, 0, sizeof(tp->id));
-		for(i=0; i<ID_PIPE_NUM; i++) {
-			tp->unused[i] = i;
-		}
+		xfrd_tcp_pipeline_init(tp);
 
 		/* insert into tree */
 		(void)rbtree_insert(set->pipetree, &tp->key.node);
@@ -590,7 +753,7 @@ xfrd_tcp_obtain(struct xfrd_tcp_set* set, xfrd_zone_type* zone)
 		int i;
 		if(zone->zone_handler.ev_fd != -1)
 			xfrd_udp_release(zone);
-		for(i=0; i<XFRD_MAX_TCP; i++) {
+		for(i=0; i<set->tcp_max; i++) {
 			if(set->tcp_state[i] == tp)
 				zone->tcp_conn = i;
 		}
@@ -602,7 +765,7 @@ xfrd_tcp_obtain(struct xfrd_tcp_set* set, xfrd_zone_type* zone)
 
 	/* wait, at end of line */
 	DEBUG(DEBUG_XFRD,2, (LOG_INFO, "xfrd: max number of tcp "
-		"connections (%d) reached.", XFRD_MAX_TCP));
+		"connections (%d) reached.", set->tcp_max));
 	zone->tcp_waiting_next = 0;
 	zone->tcp_waiting_prev = set->tcp_waiting_last;
 	zone->tcp_waiting = 1;
@@ -1312,7 +1475,7 @@ xfrd_tcp_read(struct xfrd_tcp_pipeline* tp)
 		tcp_conn_ready_for_reading(tcp);
 		return;
 	}
-	zone = tp->id[ID(tcp->packet)];
+	zone = xfrd_tcp_pipeline_lookup_id(tp, ID(tcp->packet));
 	if(!zone || zone == TCP_NULL_SKIP) {
 		/* no zone for this id? skip it */
 		DEBUG(DEBUG_XFRD,1, (LOG_INFO,
@@ -1333,7 +1496,7 @@ xfrd_tcp_read(struct xfrd_tcp_pipeline* tp)
 			break;
 		case xfrd_packet_newlease:
 			/* set to skip if more packets with this ID */
-			tp->id[zone->query_id] = TCP_NULL_SKIP;
+			xfrd_tcp_pipeline_skip_id(tp, zone->query_id);
 			tp->key.num_skip++;
 			/* fall through to remove zone from tp */
 			/* fallthrough */
@@ -1356,7 +1519,7 @@ xfrd_tcp_read(struct xfrd_tcp_pipeline* tp)
 		case xfrd_packet_tcp:
 		default:
 			/* set to skip if more packets with this ID */
-			tp->id[zone->query_id] = TCP_NULL_SKIP;
+			xfrd_tcp_pipeline_skip_id(tp, zone->query_id);
 			tp->key.num_skip++;
 			xfrd_tcp_release(xfrd->tcp_set, zone);
 			/* query next server */
@@ -1380,8 +1543,8 @@ xfrd_tcp_release(struct xfrd_tcp_set* set, xfrd_zone_type* zone)
 	/* remove from tcp_send list */
 	tcp_pipe_sendlist_remove(tp, zone);
 	/* remove it from the ID list */
-	if(tp->id[zone->query_id] != TCP_NULL_SKIP)
-		tcp_pipe_id_remove(tp, zone);
+	if(xfrd_tcp_pipeline_lookup_id(tp, zone->query_id) != TCP_NULL_SKIP)
+		tcp_pipe_id_remove(tp, zone, 1);
 	DEBUG(DEBUG_XFRD,1, (LOG_INFO, "xfrd: released tcp pipe now %d unused",
 		tp->key.num_unused));
 	/* if pipe was full, but no more, then see if waiting element is
@@ -1410,7 +1573,7 @@ xfrd_tcp_release(struct xfrd_tcp_set* set, xfrd_zone_type* zone)
 	}
 
 	/* if all unused, or only skipped leftover, close the pipeline */
-	if(tp->key.num_unused >= ID_PIPE_NUM || tp->key.num_skip >= ID_PIPE_NUM - tp->key.num_unused)
+	if(tp->key.num_unused >= tp->pipe_num || tp->key.num_skip >= tp->pipe_num - tp->key.num_unused)
 		xfrd_tcp_pipe_release(set, tp, conn);
 }
 
@@ -1446,9 +1609,7 @@ xfrd_tcp_pipe_release(struct xfrd_tcp_set* set, struct xfrd_tcp_pipeline* tp,
 	/* a waiting zone can use the free tcp slot (to another server) */
 	/* if that zone fails to set-up or connect, we try to start the next
 	 * waiting zone in the list */
-	while(set->tcp_count == XFRD_MAX_TCP && set->tcp_waiting_first) {
-		int i;
-
+	while(set->tcp_count == set->tcp_max && set->tcp_waiting_first) {
 		/* pop first waiting process */
 		xfrd_zone_type* zone = set->tcp_waiting_first;
 		/* start it */
@@ -1467,15 +1628,7 @@ xfrd_tcp_pipe_release(struct xfrd_tcp_set* set, struct xfrd_tcp_pipeline* tp,
 		}
 		/* re-init this tcppipe */
 		/* ip and ip_len set by tcp_open */
-		tp->key.node.key = tp;
-		tp->key.num_unused = ID_PIPE_NUM;
-		tp->key.num_skip = 0;
-		tp->tcp_send_first = NULL;
-		tp->tcp_send_last = NULL;
-		memset(tp->id, 0, sizeof(tp->id));
-		for(i=0; i<ID_PIPE_NUM; i++) {
-			tp->unused[i] = i;
-		}
+		xfrd_tcp_pipeline_init(tp);
 
 		/* insert into tree */
 		(void)rbtree_insert(set->pipetree, &tp->key.node);
diff --git a/usr.sbin/nsd/xfrd-tcp.h b/usr.sbin/nsd/xfrd-tcp.h
index 60d3e23965b..ab20cf7bf24 100644
--- a/usr.sbin/nsd/xfrd-tcp.h
+++ b/usr.sbin/nsd/xfrd-tcp.h
@@ -31,8 +31,12 @@ typedef struct xfrd_tcp_set xfrd_tcp_set_type;
  * A set of xfrd tcp connections.
  */
 struct xfrd_tcp_set {
-	/* tcp connections, each has packet and read/wr state */
-	struct xfrd_tcp_pipeline *tcp_state[XFRD_MAX_TCP];
+	/* tcp connections, array, each has packet and read/wr state */
+	struct xfrd_tcp_pipeline **tcp_state;
+	/* max number of tcp connections, size of tcp_state array */
+	int tcp_max;
+	/* max number of simultaneous connections on a tcp_pipeline */
+	int tcp_pipeline;
 	/* number of TCP connections in use. */
 	int tcp_count;
 	/* TCP timeout. */
@@ -74,8 +78,21 @@ struct xfrd_tcp {
 /* use illegal pointer value to denote skipped ID number.
  * if this does not work, we can allocate with malloc */
 #define TCP_NULL_SKIP ((struct xfrd_zone*)-1)
-/* the number of ID values (16 bits) for a pipeline */
-#define ID_PIPE_NUM 65536
+
+/**
+ * The per-id zone pointers, with TCP_NULL_SKIP or a zone pointer for the
+ * ID value.
+ */
+struct xfrd_tcp_pipeline_id {
+	/** rbtree node as first member, this is the key. */
+	rbnode_type node;
+	/** the ID of this member */
+	uint16_t id;
+	/** zone pointer or TCP_NULL_SKIP */
+	struct xfrd_zone* zone;
+	/** next free in free list */
+	struct xfrd_tcp_pipeline_id* next_free;
+};
 
 /**
  * The tcp pipeline key structure. By ip_len, ip, num_unused and unique by
@@ -138,23 +155,31 @@ struct xfrd_tcp_pipeline {
 	/* list of queries that want to send, first to get write event,
 	 * if NULL, no write event interest */
 	struct xfrd_zone* tcp_send_first, *tcp_send_last;
-	/* the unused and id arrays must be last in the structure */
-	/* per-ID number the queries that have this ID number, every
+
+	/* size of the id and unused arrays. */
+	int pipe_num;
+	/* list of free xfrd_tcp_pipeline_id nodes, these are not in the
+	 * zone_per_id tree. preallocated at pipe_num amount. */
+	struct xfrd_tcp_pipeline_id* pipe_id_free_list;
+	/* The xfrd_zone pointers, per id number.
+	 * The key is struct xfrd_tcp_pipeline_id.
+	 * per-ID number the queries that have this ID number, every
 	 * query owns one ID numbers (until it is done). NULL: unused
 	 * When a query is done but not all answer-packets have been
 	 * consumed for that ID number, the rest is skipped, this
 	 * is denoted with the pointer-value TCP_NULL_SKIP, the ids that
 	 * are skipped are not on the unused list.  They may be
 	 * removed once the last answer packet is skipped.
-	 * ID_PIPE_NUM-num_unused values in the id array are nonNULL (either
+	 * pipe_num-num_unused values are in the tree (either
 	 * a zone pointer or SKIP) */
-	struct xfrd_zone* id[ID_PIPE_NUM];
-	/* unused ID numbers; the first part of the array contains the IDs */
-	uint16_t unused[ID_PIPE_NUM];
+	rbtree_type* zone_per_id;
+	/* Array of uint16_t, with ID values.
+	 * unused ID numbers; the first part of the array contains the IDs */
+	uint16_t* unused;
 };
 
 /* create set of tcp connections */
-struct xfrd_tcp_set* xfrd_tcp_set_create(struct region* region, const char *tls_cert_bundle);
+struct xfrd_tcp_set* xfrd_tcp_set_create(struct region* region, const char *tls_cert_bundle, int tcp_max, int tcp_pipeline);
 
 /* init tcp state */
 struct xfrd_tcp* xfrd_tcp_create(struct region* region, size_t bufsize);
@@ -219,6 +244,9 @@ socklen_t xfrd_acl_sockaddr_frm(struct acl_options* acl,
 #endif /* INET6 */
 
 /* create pipeline tcp structure */
-struct xfrd_tcp_pipeline* xfrd_tcp_pipeline_create(region_type* region);
+struct xfrd_tcp_pipeline* xfrd_tcp_pipeline_create(region_type* region,
+	int tcp_pipeline);
+/* pick num uint16_t values, from 0..max-1, store in array */
+void pick_id_values(uint16_t* array, int num, int max);
 
 #endif /* XFRD_TCP_H */
diff --git a/usr.sbin/nsd/xfrd.c b/usr.sbin/nsd/xfrd.c
index c40e8ca57a5..a9e1558ea72 100644
--- a/usr.sbin/nsd/xfrd.c
+++ b/usr.sbin/nsd/xfrd.c
@@ -196,7 +196,7 @@ xfrd_init(int socket, struct nsd* nsd, int shortsoa, int reload_active,
 	daemon_remote_attach(xfrd->nsd->rc, xfrd);
 #endif
 
-	xfrd->tcp_set = xfrd_tcp_set_create(xfrd->region, nsd->options->tls_cert_bundle);
+	xfrd->tcp_set = xfrd_tcp_set_create(xfrd->region, nsd->options->tls_cert_bundle, nsd->options->xfrd_tcp_max, nsd->options->xfrd_tcp_pipeline);
 	xfrd->tcp_set->tcp_timeout = nsd->tcp_timeout;
 #if !defined(HAVE_ARC4RANDOM) && !defined(HAVE_GETRANDOM)
 	srandom((unsigned long) getpid() * (unsigned long) time(NULL));
diff --git a/usr.sbin/nsd/xfrd.h b/usr.sbin/nsd/xfrd.h
index a3e68ffa135..b4edf66f4be 100644
--- a/usr.sbin/nsd/xfrd.h
+++ b/usr.sbin/nsd/xfrd.h
@@ -239,9 +239,10 @@ enum xfrd_packet_result {
    And it should be below FD_SETSIZE, to be able to select() on replies.
    Note that also some sockets are used for writing the ixfr.db, xfrd.state
    files and for the pipes to the main parent process.
+
+   For xfrd_tcp_max, 128 is the default number of TCP AXFR/IXFR concurrent
+   connections. Each entry has 64Kb buffer preallocated.
 */
-#define XFRD_MAX_TCP 128 /* max number of TCP AXFR/IXFR concurrent connections.*/
-			/* Each entry has 64Kb buffer preallocated.*/
 #define XFRD_MAX_UDP 128 /* max number of UDP sockets at a time for IXFR */
 #define XFRD_MAX_UDP_NOTIFY 128 /* max concurrent UDP sockets for NOTIFY */
 
@@ -364,7 +365,7 @@ void xfrd_deactivate_zone(xfrd_zone_type* z);
 /*
  * Make a new request to next master server.
  * uses next_master if set (and a fresh set of rounds).
- * otherwised, starts new round of requests if none started already.
+ * otherwise, starts new round of requests if none started already.
  * starts next round of requests if at last master.
  * if too many rounds of requests, sets timer for next retry.
  */