Merge in some commits from upstream..

- Fix that failure to add tcp to tcp base does not leak the socket.
- Fixes for wildcard addition and deletion, speedup for some cases.
- Fix that queries for noname CH TXT are REFUSED instead of nodata.
- Fix #616: retry xfer for zones with no content after command.
- Fix that expired zones stay expired after a server restart.
- RFC 7344: CDS and CDNSKEY (read in).

ok sthen@

13 files changed, 2018 insertions, 1003 deletions

diff --git a/usr.sbin/nsd/difffile.c b/usr.sbin/nsd/difffile.c
index c11f5e7faaf..c9c8026369e 100644
--- a/usr.sbin/nsd/difffile.c
+++ b/usr.sbin/nsd/difffile.c
@@ -7,28 +7,33 @@
  *
  */
 
-#include <config.h>
+#include "config.h"
 #include <assert.h>
 #include <string.h>
 #include <unistd.h>
 #include <stdlib.h>
 #include <errno.h>
 #include "difffile.h"
+#include "xfrd-disk.h"
 #include "util.h"
 #include "packet.h"
 #include "rdata.h"
+#include "udb.h"
+#include "udbzone.h"
+#include "nsec3.h"
+#include "nsd.h"
+#include "rrl.h"
 
 static int
-write_32(FILE *out, uint32_t val)
+write_64(FILE *out, uint64_t val)
 {
-	val = htonl(val);
 	return write_data(out, &val, sizeof(val));
 }
 
 static int
-write_16(FILE *out, uint16_t val)
+write_32(FILE *out, uint32_t val)
 {
-	val = htons(val);
+	val = htonl(val);
 	return write_data(out, &val, sizeof(val));
 }
 
@@ -48,143 +53,117 @@ write_str(FILE *out, const char* str)
 }
 
 void
-diff_write_packet(const char* zone, uint32_t new_serial, uint16_t id,
-	uint32_t seq_nr, uint8_t* data, size_t len, nsd_options_t* opt)
+diff_write_packet(const char* zone, const char* pat, uint32_t old_serial,
+	uint32_t new_serial, uint32_t seq_nr, uint8_t* data, size_t len,
+	struct nsd* nsd, uint64_t filenumber)
 {
-	const char* filename = opt->difffile;
-	struct timeval tv;
-	FILE *df;
-	uint32_t file_len = sizeof(uint32_t) + strlen(zone) +
-		sizeof(new_serial) + sizeof(id) + sizeof(seq_nr) + len;
-
-	if (gettimeofday(&tv, NULL) != 0) {
-		log_msg(LOG_ERR, "could not set timestamp for %s: %s",
-			filename, strerror(errno));
+	FILE* df = xfrd_open_xfrfile(nsd, filenumber, seq_nr?"a":"w");
+	if(!df) {
+		log_msg(LOG_ERR, "could not open transfer %s file %lld: %s",
+			zone, (long long)filenumber, strerror(errno));
 		return;
 	}
 
-	df = fopen(filename, "a");
-	if(!df) {
-		log_msg(LOG_ERR, "could not open file %s for append: %s",
-			filename, strerror(errno));
-		return;
+	/* if first part, first write the header */
+	if(seq_nr == 0) {
+		struct timeval tv;
+		if (gettimeofday(&tv, NULL) != 0) {
+			log_msg(LOG_ERR, "could not get timestamp for %s: %s",
+				zone, strerror(errno));
+		}
+		if(!write_32(df, DIFF_PART_XFRF) ||
+			!write_8(df, 0) /* notcommitted(yet) */ ||
+			!write_32(df, 0) /* numberofparts when done */ ||
+			!write_64(df, (uint64_t) tv.tv_sec) ||
+			!write_32(df, (uint32_t) tv.tv_usec) ||
+			!write_32(df, old_serial) ||
+			!write_32(df, new_serial) ||
+			!write_64(df, (uint64_t) tv.tv_sec) ||
+			!write_32(df, (uint32_t) tv.tv_usec) ||
+			!write_str(df, zone) ||
+			!write_str(df, pat)) {
+			log_msg(LOG_ERR, "could not write transfer %s file %lld: %s",
+				zone, (long long)filenumber, strerror(errno));
+			fclose(df);
+			return;
+		}
 	}
 
-	if(!write_32(df, DIFF_PART_IXFR) ||
-		!write_32(df, (uint32_t) tv.tv_sec) ||
-		!write_32(df, (uint32_t) tv.tv_usec) ||
-		!write_32(df, file_len) ||
-		!write_str(df, zone) ||
-		!write_32(df, new_serial) ||
-		!write_16(df, id) ||
-		!write_32(df, seq_nr) ||
+	if(!write_32(df, DIFF_PART_XXFR) ||
+		!write_32(df, len) ||
 		!write_data(df, data, len) ||
-		!write_32(df, file_len))
+		!write_32(df, len))
 	{
-		log_msg(LOG_ERR, "could not write to file %s: %s",
-			filename, strerror(errno));
+		log_msg(LOG_ERR, "could not write transfer %s file %lld: %s",
+			zone, (long long)filenumber, strerror(errno));
 	}
-	fflush(df);
 	fclose(df);
 }
 
 void
-diff_write_commit(const char* zone, uint32_t old_serial,
-	uint32_t new_serial, uint16_t id, uint32_t num_parts,
-	uint8_t commit, const char* log_str, nsd_options_t* opt)
+diff_write_commit(const char* zone, uint32_t old_serial, uint32_t new_serial,
+	uint32_t num_parts, uint8_t commit, const char* log_str,
+	struct nsd* nsd, uint64_t filenumber)
 {
-	const char* filename = opt->difffile;
 	struct timeval tv;
-	FILE *df;
-	uint32_t len;
+	FILE* df;
 
 	if (gettimeofday(&tv, NULL) != 0) {
 		log_msg(LOG_ERR, "could not set timestamp for %s: %s",
-			filename, strerror(errno));
-		return;
+			zone, strerror(errno));
 	}
 
-	df = fopen(filename, "a");
+	/* overwrite the first part of the file with 'committed = 1', 
+	 * as well as the end_time and number of parts.
+	 * also write old_serial and new_serial, so that a bad file mixup
+	 * will result in unusable serial numbers. */
+
+	df = xfrd_open_xfrfile(nsd, filenumber, "r+");
 	if(!df) {
-		log_msg(LOG_ERR, "could not open file %s for append: %s",
-			filename, strerror(errno));
+		log_msg(LOG_ERR, "could not open transfer %s file %lld: %s",
+			zone, (long long)filenumber, strerror(errno));
 		return;
 	}
-
-	len = strlen(zone) + sizeof(len) + sizeof(old_serial) +
-		sizeof(new_serial) + sizeof(id) + sizeof(num_parts) +
-		sizeof(commit) + strlen(log_str) + sizeof(len);
-
-	if(!write_32(df, DIFF_PART_SURE) ||
-		!write_32(df, (uint32_t) tv.tv_sec) ||
+	if(!write_32(df, DIFF_PART_XFRF) ||
+		!write_8(df, commit) /* committed */ ||
+		!write_32(df, num_parts) ||
+		!write_64(df, (uint64_t) tv.tv_sec) ||
 		!write_32(df, (uint32_t) tv.tv_usec) ||
-		!write_32(df, len) ||
-		!write_str(df, zone) ||
 		!write_32(df, old_serial) ||
-		!write_32(df, new_serial) ||
-		!write_16(df, id) ||
-		!write_32(df, num_parts) ||
-		!write_8(df, commit) ||
-		!write_str(df, log_str) ||
-		!write_32(df, len))
+		!write_32(df, new_serial))
 	{
-		log_msg(LOG_ERR, "could not write to file %s: %s",
-			filename, strerror(errno));
+		log_msg(LOG_ERR, "could not write transfer %s file %lld: %s",
+			zone, (long long)filenumber, strerror(errno));
+		fclose(df);
+		return;
+	}
+
+	/* append the log_str to the end of the file */
+	if(fseek(df, 0, SEEK_END) == -1) {
+		log_msg(LOG_ERR, "could not fseek transfer %s file %lld: %s",
+			zone, (long long)filenumber, strerror(errno));
+		fclose(df);
+		return;
+	}
+	if(!write_str(df, log_str)) {
+		log_msg(LOG_ERR, "could not write transfer %s file %lld: %s",
+			zone, (long long)filenumber, strerror(errno));
+		fclose(df);
+		return;
+
 	}
 	fflush(df);
 	fclose(df);
 }
 
-/*
- * Checksum to signal no data change occured (for example, by a
- * zonec run.
- */
 int
-db_crc_different(namedb_type* db)
+diff_read_64(FILE *in, uint64_t* result)
 {
-	FILE *fd = fopen(db->filename, "r");
-	uint32_t crc_file;
-	char buf[NAMEDB_MAGIC_SIZE];
-	if(fd == NULL) {
-		log_msg(LOG_ERR, "unable to load %s: %s",
-			db->filename, strerror(errno));
-		return -1;
-	}
-
-	/* seek to position of CRC, check it and magic no */
-	if(fseeko(fd, db->crc_pos, SEEK_SET)==-1) {
-		log_msg(LOG_ERR, "unable to fseeko %s: %s. db changed?",
-			db->filename, strerror(errno));
-		fclose(fd);
-		return -1;
-	}
-
-	if(fread(&crc_file, sizeof(crc_file), 1, fd) != 1) {
-		if(!feof(fd))
-			log_msg(LOG_ERR, "could not read %s CRC: %s. "
-				"db changed?", db->filename, strerror(errno));
-		fclose(fd);
-		return -1;
-	}
-	crc_file = ntohl(crc_file);
-
-	if(fread(buf, sizeof(char), sizeof(buf), fd) != sizeof(buf)) {
-		if(!feof(fd))
-			log_msg(LOG_ERR, "could not read %s magic: %s. "
-				"db changed?", db->filename, strerror(errno));
-		fclose(fd);
-		return -1;
-	}
-	if(memcmp(buf, NAMEDB_MAGIC, NAMEDB_MAGIC_SIZE) != 0) {
-		fclose(fd);
-		return -1;
-	}
-
-	fclose(fd);
-
-	if(db->crc == crc_file)
+	if (fread(result, sizeof(*result), 1, in) == 1) {
+		return 1;
+	} else {
 		return 0;
-	return 1;
+	}
 }
 
 int
@@ -199,17 +178,6 @@ diff_read_32(FILE *in, uint32_t* result)
 }
 
 int
-diff_read_16(FILE *in, uint16_t* result)
-{
-        if (fread(result, sizeof(*result), 1, in) == 1) {
-                *result = ntohs(*result);
-                return 1;
-        } else {
-                return 0;
-        }
-}
-
-int
 diff_read_8(FILE *in, uint8_t* result)
 {
         if (fread(result, sizeof(*result), 1, in) == 1) {
@@ -259,7 +227,7 @@ has_data_below(domain_type* top)
 	assert(d != NULL);
 	/* in the canonical ordering subdomains are after this name */
 	d = domain_next(d);
-	while(d != NULL && dname_is_subdomain(domain_dname(d), domain_dname(top))) {
+	while(d != NULL && domain_is_subdomain(d, top)) {
 		if(d->is_existing)
 			return 1;
 		d = domain_next(d);
@@ -267,6 +235,34 @@ has_data_below(domain_type* top)
 	return 0;
 }
 
+/** check if domain with 0 rrsets has become empty (nonexist) */
+static domain_type*
+rrset_zero_nonexist_check(domain_type* domain, domain_type* ce)
+{
+	/* is the node now an empty node (completely deleted) */
+	if(domain->rrsets == 0) {
+		/* if there is no data below it, it becomes non existing.
+		   also empty nonterminals above it become nonexisting */
+		/* check for data below this node. */
+		if(!has_data_below(domain)) {
+			/* nonexist this domain and all parent empty nonterminals */
+			domain_type* p = domain;
+			while(p != NULL && p->rrsets == 0) {
+				if(p == ce || has_data_below(p))
+					return p;
+				p->is_existing = 0;
+				/* fixup wildcard child of parent */
+				if(p->parent &&
+					p->parent->wildcard_child_closest_match == p)
+					p->parent->wildcard_child_closest_match = domain_previous_existing_child(p);
+				p = p->parent;
+			}
+		}
+	}
+	return NULL;
+}
+
+/** remove rrset.  Adjusts zone params.  Does not remove domain */
 static void
 rrset_delete(namedb_type* db, domain_type* domain, rrset_type* rrset)
 {
@@ -283,98 +279,355 @@ rrset_delete(namedb_type* db, domain_type* domain, rrset_type* rrset)
 	*pp = rrset->next;
 
 	DEBUG(DEBUG_XFRD,2, (LOG_INFO, "delete rrset of %s type %s",
-		dname_to_string(domain_dname(domain),0),
+		domain_to_string(domain),
 		rrtype_to_string(rrset_rrtype(rrset))));
 
 	/* is this a SOA rrset ? */
 	if(rrset->zone->soa_rrset == rrset) {
 		rrset->zone->soa_rrset = 0;
-		rrset->zone->updated = 1;
 	}
 	if(rrset->zone->ns_rrset == rrset) {
 		rrset->zone->ns_rrset = 0;
 	}
-#ifdef DNSSEC
 	if(domain == rrset->zone->apex && rrset_rrtype(rrset) == TYPE_RRSIG) {
 		for (i = 0; i < rrset->rr_count; ++i) {
-			if (rr_rrsig_type_covered(&rrset->rrs[i]) == TYPE_SOA) {
+			if(rr_rrsig_type_covered(&rrset->rrs[i])==TYPE_DNSKEY) {
 				rrset->zone->is_secure = 0;
 				break;
 			}
 		}
 	}
-#endif
 	/* recycle the memory space of the rrset */
 	for (i = 0; i < rrset->rr_count; ++i)
 		add_rdata_to_recyclebin(db, &rrset->rrs[i]);
 	region_recycle(db->region, rrset->rrs,
 		sizeof(rr_type) * rrset->rr_count);
-	region_recycle(db->region, rrset, sizeof(rrset_type));
-
-	/* is the node now an empty node (completely deleted) */
-	if(domain->rrsets == 0) {
-		/* if there is no data below it, it becomes non existing.
-		   also empty nonterminals above it become nonexisting */
-		/* check for data below this node. */
-		if(!has_data_below(domain)) {
-			/* nonexist this domain and all parent empty nonterminals */
-			domain_type* p = domain;
-			while(p != NULL && p->rrsets == 0) {
-				p->is_existing = 0;
-				p = p->parent;
-			}
-		}
-	}
 	rrset->rr_count = 0;
+	region_recycle(db->region, rrset, sizeof(rrset_type));
 }
 
 static int
-rdatas_equal(rdata_atom_type *a, rdata_atom_type *b, int num, uint16_t type)
+rdatas_equal(rdata_atom_type *a, rdata_atom_type *b, int num, uint16_t type,
+	int* rdnum, char** reason)
 {
-	int k;
-	for(k = 0; k < num; k++)
+	int k, start, end;
+	start = 0;
+	end = num;
+	/**
+	 * SOA RDATA comparisons in XFR are more lenient,
+	 * only serial rdata is checked.
+	 **/
+	if (type == TYPE_SOA) {
+		start = 2;
+		end = 3;
+	}
+	for(k = start; k < end; k++)
 	{
 		if(rdata_atom_is_domain(type, k)) {
 			if(dname_compare(domain_dname(a[k].domain),
-				domain_dname(b[k].domain))!=0)
+				domain_dname(b[k].domain))!=0) {
+				*rdnum = k;
+				*reason = "dname data";
+				return 0;
+			}
+		} else if(rdata_atom_is_literal_domain(type, k)) {
+			/* literal dname, but compare case insensitive */
+			if(a[k].data[0] != b[k].data[0]) {
+				*rdnum = k;
+				*reason = "literal dname len";
+				return 0; /* uncompressed len must be equal*/
+			}
+			if(!dname_equal_nocase((uint8_t*)(a[k].data+1),
+				(uint8_t*)(b[k].data+1), a[k].data[0])) {
+				*rdnum = k;
+				*reason = "literal dname data";
 				return 0;
+			}
 		} else {
 			/* check length */
-			if(a[k].data[0] != b[k].data[0])
+			if(a[k].data[0] != b[k].data[0]) {
+				*rdnum = k;
+				*reason = "rdata len";
 				return 0;
+			}
 			/* check data */
-			if(memcmp(a[k].data+1, b[k].data+1, a[k].data[0])!=0)
+			if(memcmp(a[k].data+1, b[k].data+1, a[k].data[0])!=0) {
+				*rdnum = k;
+				*reason = "rdata data";
 				return 0;
+			}
 		}
 	}
 	return 1;
 }
 
-static int
-find_rr_num(rrset_type* rrset,
-	uint16_t type, uint16_t klass,
+static void
+debug_find_rr_num(rrset_type* rrset, uint16_t type, uint16_t klass,
 	rdata_atom_type *rdatas, ssize_t rdata_num)
 {
-	int i;
+	int i, rd;
+	char* reason = "";
+
+	for(i=0; i < rrset->rr_count; ++i) {
+		if (rrset->rrs[i].type != type) {
+			log_msg(LOG_WARNING, "diff: RR <%s, %s> does not match "
+				"RR num %d type %s",
+				dname_to_string(rrset->rrs[i].owner->dname,0),
+				rrtype_to_string(type),	i,
+				rrtype_to_string(rrset->rrs[i].type));
+		}
+		if (rrset->rrs[i].klass != klass) {
+			log_msg(LOG_WARNING, "diff: RR <%s, %s> class %d "
+				"does not match RR num %d class %d",
+				dname_to_string(rrset->rrs[i].owner->dname,0),
+				rrtype_to_string(type),
+				klass, i,
+				rrset->rrs[i].klass);
+		}
+		if (rrset->rrs[i].rdata_count != rdata_num) {
+			log_msg(LOG_WARNING, "diff: RR <%s, %s> rdlen %u "
+				"does not match RR num %d rdlen %d",
+				dname_to_string(rrset->rrs[i].owner->dname,0),
+				rrtype_to_string(type),
+				(unsigned) rdata_num, i,
+				(unsigned) rrset->rrs[i].rdata_count);
+		}
+		if (!rdatas_equal(rdatas, rrset->rrs[i].rdatas, rdata_num, type,
+			&rd, &reason)) {
+			log_msg(LOG_WARNING, "diff: RR <%s, %s> rdata element "
+				"%d differs from RR num %d rdata (%s)",
+				dname_to_string(rrset->rrs[i].owner->dname,0),
+				rrtype_to_string(type),
+				rd, i, reason);
+		}
+	}
+}
+
+static int
+find_rr_num(rrset_type* rrset, uint16_t type, uint16_t klass,
+	rdata_atom_type *rdatas, ssize_t rdata_num, int add)
+{
+	int i, rd;
+	char* reason;
 
 	for(i=0; i < rrset->rr_count; ++i) {
 		if(rrset->rrs[i].type == type &&
 		   rrset->rrs[i].klass == klass &&
 		   rrset->rrs[i].rdata_count == rdata_num &&
-		   rdatas_equal(rdatas, rrset->rrs[i].rdatas, rdata_num, type))
+		   rdatas_equal(rdatas, rrset->rrs[i].rdatas, rdata_num, type,
+			&rd, &reason))
 		{
 			return i;
 		}
 	}
-
+        /* this is odd. Log why rr cannot be found. */
+	if (!add) {
+		debug_find_rr_num(rrset, type, klass, rdatas, rdata_num);
+	}
 	return -1;
 }
 
-static int
+#ifdef NSEC3
+/* see if nsec3 deletion triggers need action */
+static void
+nsec3_delete_rr_trigger(namedb_type* db, rr_type* rr, zone_type* zone,
+	udb_ptr* udbz)
+{
+	/* the RR has not actually been deleted yet, so we can inspect it */
+	if(!zone->nsec3_param)
+		return;
+	/* see if the domain was an NSEC3-domain in the chain, but no longer */
+	if(rr->type == TYPE_NSEC3 && rr->owner->nsec3 &&
+		rr->owner->nsec3->nsec3_node.key &&
+		nsec3_rr_uses_params(rr, zone) &&
+		nsec3_in_chain_count(rr->owner, zone) <= 1) {
+		domain_type* prev = nsec3_chain_find_prev(zone, rr->owner);
+		/* remove from prehash because no longer an NSEC3 domain */
+		if(domain_is_prehash(db->domains, rr->owner))
+			prehash_del(db->domains, rr->owner);
+		/* fixup the last in the zone */
+		if(rr->owner == zone->nsec3_last)
+			zone->nsec3_last = prev;
+		/* unlink from the nsec3tree */
+		zone_del_domain_in_hash_tree(zone->nsec3tree,
+			&rr->owner->nsec3->nsec3_node);
+		/* add previous NSEC3 to the prehash list */
+		if(prev && prev != rr->owner)
+			prehash_add(db->domains, prev);
+		else	nsec3_clear_precompile(db, zone);
+		/* this domain becomes ordinary data domain: done later */
+	}
+	/* see if the rr was NSEC3PARAM that we were using */
+	else if(rr->type == TYPE_NSEC3PARAM && rr == zone->nsec3_param) {
+		/* clear trees, wipe hashes, wipe precompile */
+		nsec3_clear_precompile(db, zone);
+		/* pick up new nsec3param (from udb, or avoid deleted rr) */
+		nsec3_find_zone_param(db, zone, udbz, rr);
+		/* if no more NSEC3, done */
+		if(!zone->nsec3_param)
+			return;
+		nsec3_precompile_newparam(db, zone);
+	}
+}
+
+/* see if nsec3 prehash can be removed with new rrset content */
+static void
+nsec3_rrsets_changed_remove_prehash(domain_type* domain, zone_type* zone)
+{
+	/* deletion of rrset already done, we can check if conditions apply */
+	/* see if the domain is no longer precompiled */
+	/* it has a hash_node, but no longer fulfills conditions */
+	if(nsec3_domain_part_of_zone(domain, zone) && domain->nsec3 &&
+		domain->nsec3->hash_node.key &&
+		!nsec3_condition_hash(domain, zone)) {
+		/* remove precompile */
+		domain->nsec3->nsec3_cover = NULL;
+		domain->nsec3->nsec3_wcard_child_cover = NULL;
+		domain->nsec3->nsec3_is_exact = 0;
+		/* remove it from the hash tree */
+		zone_del_domain_in_hash_tree(zone->hashtree,
+			&domain->nsec3->hash_node);
+		zone_del_domain_in_hash_tree(zone->wchashtree,
+			&domain->nsec3->wchash_node);
+	}
+	if(domain != zone->apex && domain->nsec3 &&
+		domain->nsec3->dshash_node.key &&
+		(!domain->parent || nsec3_domain_part_of_zone(domain->parent, zone)) &&
+		!nsec3_condition_dshash(domain, zone)) {
+		/* remove precompile */
+		domain->nsec3->nsec3_ds_parent_cover = NULL;
+		domain->nsec3->nsec3_ds_parent_is_exact = 0;
+		/* remove it from the hash tree */
+		zone_del_domain_in_hash_tree(zone->dshashtree,
+			&domain->nsec3->dshash_node);
+	}
+}
+
+/* see if domain needs to get precompiled info */
+static void
+nsec3_rrsets_changed_add_prehash(namedb_type* db, domain_type* domain,
+	zone_type* zone)
+{
+	if(!zone->nsec3_param)
+		return;
+	if((!domain->nsec3 || !domain->nsec3->hash_node.key)
+		&& nsec3_condition_hash(domain, zone)) {
+		region_type* tmpregion = region_create(xalloc, free);
+		nsec3_precompile_domain(db, domain, zone, tmpregion);
+		region_destroy(tmpregion);
+	}
+	if((!domain->nsec3 || !domain->nsec3->dshash_node.key)
+		&& nsec3_condition_dshash(domain, zone)) {
+		nsec3_precompile_domain_ds(db, domain, zone);
+	}
+}
+
+/* see if nsec3 rrset-deletion triggers need action */
+static void
+nsec3_delete_rrset_trigger(namedb_type* db, domain_type* domain,
+	zone_type* zone, uint16_t type)
+{
+	if(!zone->nsec3_param)
+		return;
+	nsec3_rrsets_changed_remove_prehash(domain, zone);
+	/* for type nsec3, or a delegation, the domain may have become a
+	 * 'normal' domain with its remaining data now */
+	if(type == TYPE_NSEC3 || type == TYPE_NS || type == TYPE_DS)
+		nsec3_rrsets_changed_add_prehash(db, domain, zone);
+	/* for type DNAME or a delegation, obscured data may be revealed */
+	if(type == TYPE_NS || type == TYPE_DS || type == TYPE_DNAME) {
+		/* walk over subdomains and check them each */
+		domain_type *d;
+		for(d=domain_next(domain); d && domain_is_subdomain(d, domain);
+			d=domain_next(d)) {
+			nsec3_rrsets_changed_add_prehash(db, d, zone);
+		}
+	}
+}
+
+/* see if nsec3 addition triggers need action */
+static void
+nsec3_add_rr_trigger(namedb_type* db, rr_type* rr, zone_type* zone,
+	udb_ptr* udbz)
+{
+	/* the RR has been added in full, also to UDB (and thus NSEC3PARAM 
+	 * in the udb has been adjusted) */
+	if(zone->nsec3_param && rr->type == TYPE_NSEC3 &&
+		(!rr->owner->nsec3 || !rr->owner->nsec3->nsec3_node.key)
+		&& nsec3_rr_uses_params(rr, zone)) {
+		/* added NSEC3 into the chain */
+		nsec3_precompile_nsec3rr(db, rr->owner, zone);
+		/* the domain has become an NSEC3-domain, if it was precompiled
+		 * previously, remove that, neatly done in routine above */
+		nsec3_rrsets_changed_remove_prehash(rr->owner, zone);
+		/* set this NSEC3 to prehash */
+		prehash_add(db->domains, rr->owner);
+	} else if(!zone->nsec3_param && rr->type == TYPE_NSEC3PARAM) {
+		/* see if this means NSEC3 chain can be used */
+		nsec3_find_zone_param(db, zone, udbz, NULL);
+		if(!zone->nsec3_param)
+			return;
+		nsec3_zone_trees_create(db->region, zone);
+		nsec3_precompile_newparam(db, zone);
+	}
+}
+
+/* see if nsec3 rrset-addition triggers need action */
+static void
+nsec3_add_rrset_trigger(namedb_type* db, domain_type* domain, zone_type* zone,
+	uint16_t type)
+{
+	/* the rrset has been added so we can inspect it */
+	if(!zone->nsec3_param)
+		return;
+	/* because the rrset is added we can check conditions easily.
+	 * check if domain needs to become precompiled now */
+	nsec3_rrsets_changed_add_prehash(db, domain, zone);
+	/* if a delegation, it changes from normal name to unhashed referral */
+	if(type == TYPE_NS || type == TYPE_DS) {
+		nsec3_rrsets_changed_remove_prehash(domain, zone);
+	}
+	/* if delegation or DNAME added, then some RRs may get obscured */
+	if(type == TYPE_NS || type == TYPE_DS || type == TYPE_DNAME) {
+		/* walk over subdomains and check them each */
+		domain_type *d;
+		for(d=domain_next(domain); d && domain_is_subdomain(d, domain);
+			d=domain_next(d)) {
+			nsec3_rrsets_changed_remove_prehash(d, zone);
+		}
+	}
+}
+#endif /* NSEC3 */
+
+/* fixup usage lower for domain names in the rdata */
+static void
+rr_lower_usage(namedb_type* db, rr_type* rr)
+{
+	unsigned i;
+	for(i=0; i<rr->rdata_count; i++) {
+		if(rdata_atom_is_domain(rr->type, i)) {
+			assert(rdata_atom_domain(rr->rdatas[i])->usage > 0);
+			rdata_atom_domain(rr->rdatas[i])->usage --;
+			if(rdata_atom_domain(rr->rdatas[i])->usage == 0)
+				domain_table_deldomain(db,
+					rdata_atom_domain(rr->rdatas[i]));
+		}
+	}
+}
+
+static void
+rrset_lower_usage(namedb_type* db, rrset_type* rrset)
+{
+	unsigned i;
+	for(i=0; i<rrset->rr_count; i++)
+		rr_lower_usage(db, &rrset->rrs[i]);
+}
+
+int
 delete_RR(namedb_type* db, const dname_type* dname,
 	uint16_t type, uint16_t klass,
 	buffer_type* packet, size_t rdatalen, zone_type *zone,
-	region_type* temp_region)
+	region_type* temp_region, udb_ptr* udbz, int* softfail)
 {
 	domain_type *domain;
 	rrset_type *rrset;
@@ -383,6 +636,7 @@ delete_RR(namedb_type* db, const dname_type* dname,
 		log_msg(LOG_WARNING, "diff: domain %s does not exist",
 			dname_to_string(dname,0));
 		buffer_skip(packet, rdatalen);
+		*softfail = 1;
 		return 1; /* not fatal error */
 	}
 	rrset = domain_find_rrset(domain, zone, type);
@@ -390,6 +644,7 @@ delete_RR(namedb_type* db, const dname_type* dname,
 		log_msg(LOG_WARNING, "diff: rrset %s does not exist",
 			dname_to_string(dname,0));
 		buffer_skip(packet, rdatalen);
+		*softfail = 1;
 		return 1; /* not fatal error */
 	} else {
 		/* find the RR in the rrset */
@@ -408,15 +663,37 @@ delete_RR(namedb_type* db, const dname_type* dname,
 				dname_to_string(dname,0));
 			return 0;
 		}
-		rrnum = find_rr_num(rrset, type, klass, rdatas, rdata_num);
+		rrnum = find_rr_num(rrset, type, klass, rdatas, rdata_num, 0);
+		if(rrnum == -1 && type == TYPE_SOA && domain == zone->apex
+			&& rrset->rr_count != 0)
+			rrnum = 0; /* replace existing SOA if no match */
 		if(rrnum == -1) {
-			log_msg(LOG_WARNING, "diff: RR %s does not exist",
-				dname_to_string(dname,0));
+			log_msg(LOG_WARNING, "diff: RR <%s, %s> does not exist",
+				dname_to_string(dname,0), rrtype_to_string(type));
+			*softfail = 1;
 			return 1; /* not fatal error */
 		}
+		/* delete the normalized RR from the udb */
+		if(db->udb)
+			udb_del_rr(db->udb, udbz, &rrset->rrs[rrnum]);
+#ifdef NSEC3
+		/* process triggers for RR deletions */
+		nsec3_delete_rr_trigger(db, &rrset->rrs[rrnum], zone, udbz);
+#endif
+		/* lower usage (possibly deleting other domains, and thus
+		 * invalidating the current RR's domain pointers) */
+		rr_lower_usage(db, &rrset->rrs[rrnum]);
 		if(rrset->rr_count == 1) {
 			/* delete entire rrset */
 			rrset_delete(db, domain, rrset);
+			/* check if domain is now nonexisting (or parents) */
+			rrset_zero_nonexist_check(domain, NULL);
+#ifdef NSEC3
+			/* cleanup nsec3 */
+			nsec3_delete_rrset_trigger(db, domain, zone, type);
+#endif
+			/* see if the domain can be deleted (and inspect parents) */
+			domain_table_deldomain(db, domain);
 		} else {
 			/* swap out the bad RR and decrease the count */
 			rr_type* rrs_orig = rrset->rrs;
@@ -433,16 +710,41 @@ delete_RR(namedb_type* db, const dname_type* dname,
 			}
 			region_recycle(db->region, rrs_orig,
 				sizeof(rr_type) * rrset->rr_count);
+#ifdef NSEC3
+			if(type == TYPE_NSEC3PARAM && zone->nsec3_param) {
+				/* fixup nsec3_param pointer to same RR */
+				assert(zone->nsec3_param >= rrs_orig &&
+					zone->nsec3_param <=
+					rrs_orig+rrset->rr_count);
+				/* last moved to rrnum, others at same index*/
+				if(zone->nsec3_param == &rrs_orig[
+					rrset->rr_count-1])
+					zone->nsec3_param = &rrset->rrs[rrnum];
+				else
+					zone->nsec3_param =
+						(void*)zone->nsec3_param
+						-(void*)rrs_orig +
+						(void*)rrset->rrs;
+			}
+#endif /* NSEC3 */
 			rrset->rr_count --;
+#ifdef NSEC3
+			/* for type nsec3, the domain may have become a
+			 * 'normal' domain with its remaining data now */
+			if(type == TYPE_NSEC3)
+				nsec3_rrsets_changed_add_prehash(db, domain,
+					zone);
+#endif /* NSEC3 */
 		}
 	}
 	return 1;
 }
 
-static int
+int
 add_RR(namedb_type* db, const dname_type* dname,
 	uint16_t type, uint16_t klass, uint32_t ttl,
-	buffer_type* packet, size_t rdatalen, zone_type *zone)
+	buffer_type* packet, size_t rdatalen, zone_type *zone, udb_ptr* udbz,
+	int* softfail)
 {
 	domain_type* domain;
 	rrset_type* rrset;
@@ -450,6 +752,7 @@ add_RR(namedb_type* db, const dname_type* dname,
 	rr_type *rrs_old;
 	ssize_t rdata_num;
 	int rrnum;
+	int rrset_added = 0;
 	domain = domain_table_find(db->domains, dname);
 	if(!domain) {
 		/* create the domain */
@@ -467,6 +770,7 @@ add_RR(namedb_type* db, const dname_type* dname,
 		rrset->rrs = 0;
 		rrset->rr_count = 0;
 		domain_add_rrset(domain, rrset);
+		rrset_added = 1;
 	}
 
 	/* dnames in rdata are normalized, conform RFC 4035,
@@ -479,11 +783,12 @@ add_RR(namedb_type* db, const dname_type* dname,
 			dname_to_string(dname,0));
 		return 0;
 	}
-	rrnum = find_rr_num(rrset, type, klass, rdatas, rdata_num);
+	rrnum = find_rr_num(rrset, type, klass, rdatas, rdata_num, 1);
 	if(rrnum != -1) {
-		DEBUG(DEBUG_XFRD, 2, (LOG_ERR, "diff: RR %s already exists",
-			dname_to_string(dname,0)));
+		DEBUG(DEBUG_XFRD, 2, (LOG_ERR, "diff: RR <%s, %s> already exists",
+			dname_to_string(dname,0), rrtype_to_string(type)));
 		/* ignore already existing RR: lenient accepting of messages */
+		*softfail = 1;
 		return 1;
 	}
 
@@ -509,157 +814,151 @@ add_RR(namedb_type* db, const dname_type* dname,
 
 	/* see if it is a SOA */
 	if(domain == zone->apex) {
-		if(type == TYPE_SOA) {
-			uint32_t soa_minimum;
-			zone->soa_rrset = rrset;
-			zone->updated = 1;
-			/* BUG #103 tweaked SOA ttl value */
-			if(zone->soa_nx_rrset == 0) {
-				zone->soa_nx_rrset = region_alloc(db->region,
-					sizeof(rrset_type));
-				if(!zone->soa_nx_rrset) {
-					log_msg(LOG_ERR, "out of memory, %s:%d",
-						__FILE__, __LINE__);
-					exit(1);
-				}
-				zone->soa_nx_rrset->rr_count = 1;
-				zone->soa_nx_rrset->next = 0;
-				zone->soa_nx_rrset->zone = zone;
-				zone->soa_nx_rrset->rrs = region_alloc(db->region,
-					sizeof(rr_type));
-				if(!zone->soa_nx_rrset->rrs) {
-					log_msg(LOG_ERR, "out of memory, %s:%d",
-						__FILE__, __LINE__);
-					exit(1);
-				}
-			}
-			memcpy(zone->soa_nx_rrset->rrs, rrset->rrs, sizeof(rr_type));
-			memcpy(&soa_minimum, rdata_atom_data(rrset->rrs->rdatas[6]),
-				rdata_atom_size(rrset->rrs->rdatas[6]));
-			if (rrset->rrs->ttl > ntohl(soa_minimum)) {
-				rrset->zone->soa_nx_rrset->rrs[0].ttl = ntohl(soa_minimum);
-			}
+		apex_rrset_checks(db, rrset, domain);
+#ifdef NSEC3
+		if(type == TYPE_NSEC3PARAM && zone->nsec3_param) {
+			/* the pointer just changed, fix it up to point
+			 * to the same record */
+			assert(zone->nsec3_param >= rrs_old &&
+				zone->nsec3_param < rrs_old+rrset->rr_count);
+			/* in this order to make sure no overflow/underflow*/
+			zone->nsec3_param = (void*)zone->nsec3_param - 
+				(void*)rrs_old + (void*)rrset->rrs;
 		}
-		if(type == TYPE_NS) {
-			zone->ns_rrset = rrset;
+#endif /* NSEC3 */
+	}
+
+	/* write the just-normalized RR to the udb */
+	if(db->udb) {
+		if(!udb_write_rr(db->udb, udbz, &rrset->rrs[rrset->rr_count - 1])) {
+			log_msg(LOG_ERR, "could not add RR to nsd.db, disk-space?");
+			return 0;
 		}
-#ifdef DNSSEC
-		if(type == TYPE_RRSIG) {
-			int i;
-			for (i = 0; i < rrset->rr_count; ++i) {
-				if (rr_rrsig_type_covered(&rrset->rrs[i]) == TYPE_SOA) {
-					zone->is_secure = 1;
-					break;
-				}
-			}
+	}
+#ifdef NSEC3
+	if(rrset_added) {
+		domain_type* p = domain->parent;
+		nsec3_add_rrset_trigger(db, domain, zone, type);
+		/* go up and process (possibly created) empty nonterminals, 
+		 * until we hit the apex or root */
+		while(p && p->rrsets == NULL && !p->is_apex) {
+			nsec3_rrsets_changed_add_prehash(db, p, zone);
+			p = p->parent;
 		}
-#endif
 	}
+	nsec3_add_rr_trigger(db, &rrset->rrs[rrset->rr_count - 1], zone, udbz);
+#endif /* NSEC3 */
 	return 1;
 }
 
 static zone_type*
-find_zone(namedb_type* db, const dname_type* zone_name, nsd_options_t* opt,
-	size_t child_count)
+find_or_create_zone(namedb_type* db, const dname_type* zone_name,
+	nsd_options_t* opt, const char* zstr, const char* patname)
 {
-	domain_type *domain;
 	zone_type* zone;
-	domain = domain_table_find(db->domains, zone_name);
-	if(!domain) {
-		DEBUG(DEBUG_XFRD,1, (LOG_INFO, "xfr: creating domain %s",
-			dname_to_string(zone_name,0)));
-		/* create the zone and domain of apex (zone has config options) */
-		domain = domain_table_insert(db->domains, zone_name);
-	} else {
-		/* O(1) if SOA exists */
-		zone = domain_find_zone(domain);
-		/* if domain was empty (no rrsets, empty zone) search in zonelist */
-		/* check apex to make sure we don't find a parent zone */
-		if(!zone || zone->apex != domain)
-			zone = namedb_find_zone(db, domain);
-		if(zone) {
-			assert(zone->apex == domain);
-			return zone;
+	zone_options_t* zopt;
+	zone = namedb_find_zone(db, zone_name);
+	if(zone) {
+		return zone;
+	}
+	zopt = zone_options_find(opt, zone_name);
+	if(!zopt) {
+		/* if _implicit_ then insert as _part_of_config */
+		if(strncmp(patname, PATTERN_IMPLICIT_MARKER,
+			strlen(PATTERN_IMPLICIT_MARKER)) == 0) {
+			zopt = zone_options_create(opt->region);
+			if(!zopt) return 0;
+			zopt->part_of_config = 1;
+			zopt->name = region_strdup(opt->region, zstr);
+			zopt->pattern = pattern_options_find(opt, patname);
+			if(!zopt->name || !zopt->pattern) return 0;
+			if(!nsd_options_insert_zone(opt, zopt)) {
+				log_msg(LOG_ERR, "bad domain name or duplicate zone '%s' "
+					"pattern %s", zstr, patname);
+			}
+		} else {
+			/* create zone : presumably already added to zonelist
+			 * by xfrd, who wrote the AXFR or IXFR to disk, so we only
+			 * need to add it to our config.
+			 * This process does not need linesize and offset zonelist */
+			zopt = zone_list_zone_insert(opt, zstr, patname, 0, 0);
+			if(!zopt)
+				return 0;
 		}
 	}
-	/* create the zone */
-	DEBUG(DEBUG_XFRD,1, (LOG_INFO, "xfr: creating zone_type %s",
-		dname_to_string(zone_name,0)));
-	zone = (zone_type *) region_alloc(db->region, sizeof(zone_type));
-	if(!zone) {
-		log_msg(LOG_ERR, "out of memory, %s:%d", __FILE__, __LINE__);
-		exit(1);
-	}
-	zone->next = db->zones;
-	db->zones = zone;
-	db->zone_count++;
-	zone->apex = domain;
-	zone->soa_rrset = 0;
-	zone->soa_nx_rrset = 0;
-	zone->ns_rrset = 0;
-#ifdef NSEC3
-	zone->nsec3_soa_rr = NULL;
-	zone->nsec3_last = NULL;
-#endif
-	zone->dirty = region_alloc(db->region, sizeof(uint8_t)*child_count);
-	if(!zone->dirty) {
-		log_msg(LOG_ERR, "out of memory, %s:%d", __FILE__, __LINE__);
-		exit(1);
-	}
-	memset(zone->dirty, 0, sizeof(uint8_t)*child_count);
-	zone->opts = zone_options_find(opt, domain_dname(zone->apex));
-	if(!zone->opts) {
-		log_msg(LOG_ERR, "xfr: zone %s not in config.",
-			dname_to_string(zone_name,0));
-		return 0;
-	}
-	zone->number = db->zone_count;
-	zone->is_secure = 0;
-	zone->updated = 1;
-	zone->is_ok = 0;
+	zone = namedb_zone_create(db, zone_name, zopt);
 	return zone;
 }
 
-static void
+void
 delete_zone_rrs(namedb_type* db, zone_type* zone)
 {
 	rrset_type *rrset;
-	domain_type *domain = zone->apex;
+	domain_type *domain = zone->apex, *next;
+	int nonexist_check = 0;
 	/* go through entire tree below the zone apex (incl subzones) */
-	while(domain && dname_is_subdomain(
-		domain_dname(domain), domain_dname(zone->apex)))
+	while(domain && domain_is_subdomain(domain, zone->apex))
 	{
 		DEBUG(DEBUG_XFRD,2, (LOG_INFO, "delete zone visit %s",
-			dname_to_string(domain_dname(domain),0)));
+			domain_to_string(domain)));
 		/* delete all rrsets of the zone */
 		while((rrset = domain_find_any_rrset(domain, zone))) {
+			/* lower usage can delete other domains */
+			rrset_lower_usage(db, rrset);
+			/* rrset del does not delete our domain(yet) */
 			rrset_delete(db, domain, rrset);
+			/* no rrset_zero_nonexist_check, do that later */
+			if(domain->rrsets == 0)
+				nonexist_check = 1;
+		}
+		/* the delete upcoming could delete parents, but nothing next
+		 * or after the domain so store next ptr */
+		next = domain_next(domain);
+		/* see if the domain can be deleted (and inspect parents) */
+		domain_table_deldomain(db, domain);
+		domain = next;
+	}
+
+	/* check if data deletions have created nonexisting domain entries,
+	 * but after deleting domains so the checks are faster */
+	if(nonexist_check) {
+		domain_type* ce = NULL; /* for speeding up has_data_below */
+		DEBUG(DEBUG_XFRD, 1, (LOG_INFO, "axfrdel: zero rrset check"));
+		domain = zone->apex;
+		while(domain && domain_is_subdomain(domain, zone->apex))
+		{
+			/* the interesting domains should be existing==1
+			 * and rrsets==0, speeding up out processing of
+			 * sub-zones, since we only spuriously check empty
+			 * nonterminals */
+			if(domain->is_existing)
+				ce = rrset_zero_nonexist_check(domain, ce);
+			domain = domain_next(domain);
 		}
-		domain = domain_next(domain);
 	}
 
 	DEBUG(DEBUG_XFRD, 1, (LOG_INFO, "axfrdel: recyclebin holds %lu bytes",
 		(unsigned long) region_get_recycle_size(db->region)));
 #ifndef NDEBUG
-	if(nsd_debug_level >= 1)
+	if(nsd_debug_level >= 2)
 		region_log_stats(db->region);
 #endif
 
 	assert(zone->soa_rrset == 0);
-	/* keep zone->soa_nx_rrset alloced */
+	/* keep zone->soa_nx_rrset alloced: it is reused */
 	assert(zone->ns_rrset == 0);
 	assert(zone->is_secure == 0);
-	assert(zone->updated == 1);
 }
 
+/* return value 0: syntaxerror,badIXFR, 1:OK, 2:done_and_skip_it */
 static int
-apply_ixfr(namedb_type* db, FILE *in, const off_t* startpos,
-	const char* zone, uint32_t serialno, nsd_options_t* opt,
-	uint16_t id, uint32_t seq_nr, uint32_t seq_total,
+apply_ixfr(namedb_type* db, FILE *in, const char* zone, uint32_t serialno,
+	nsd_options_t* opt, uint32_t seq_nr, uint32_t seq_total,
 	int* is_axfr, int* delete_mode, int* rr_count,
-	size_t child_count)
+	udb_ptr* udbz, struct zone** zone_res, const char* patname, int* bytes,
+	int* softfail)
 {
-	uint32_t filelen, msglen, pkttype, timestamp[2];
+	uint32_t msglen, checklen, pkttype;
 	int qcount, ancount, counter;
 	buffer_type* packet;
 	region_type* region;
@@ -667,35 +966,24 @@ apply_ixfr(namedb_type* db, FILE *in, const off_t* startpos,
 	uint16_t rrlen;
 	const dname_type *dname_zone, *dname;
 	zone_type* zone_db;
-	char file_zone_name[3072];
-	uint32_t file_serial, file_seq_nr;
-	uint16_t file_id;
-	off_t mempos;
-
-	memmove(&mempos, startpos, sizeof(off_t));
-	if(fseeko(in, mempos, SEEK_SET) == -1) {
-		log_msg(LOG_INFO, "could not fseeko: %s.", strerror(errno));
-		return 0;
-	}
-	/* read ixfr packet RRs and apply to in memory db */
 
-	if(!diff_read_32(in, &pkttype) || pkttype != DIFF_PART_IXFR) {
+	/* note that errors could not really happen due to format of the
+	 * packet since xfrd has checked all dnames and RRs before commit,
+	 * this is why the errors are fatal (exit process), it must be
+	 * something internal or a bad disk or something. */
+
+	/* read ixfr packet RRs and apply to in memory db */
+	if(!diff_read_32(in, &pkttype) || pkttype != DIFF_PART_XXFR) {
 		log_msg(LOG_ERR, "could not read type or wrong type");
 		return 0;
 	}
-	if(!diff_read_32(in, &timestamp[0]) ||
-	   !diff_read_32(in, &timestamp[1])) {
-		log_msg(LOG_ERR, "could not read timestamp");
-		return 0;
-	}
 
-	if(!diff_read_32(in, &filelen)) {
+	if(!diff_read_32(in, &msglen)) {
 		log_msg(LOG_ERR, "could not read len");
 		return 0;
 	}
 
-	/* read header */
-	if(filelen < QHEADERSZ + sizeof(uint32_t)*3 + sizeof(uint16_t)) {
+	if(msglen < QHEADERSZ) {
 		log_msg(LOG_ERR, "msg too short");
 		return 0;
 	}
@@ -705,34 +993,7 @@ apply_ixfr(namedb_type* db, FILE *in, const off_t* startpos,
 		log_msg(LOG_ERR, "out of memory");
 		return 0;
 	}
-
-	if(!diff_read_str(in, file_zone_name, sizeof(file_zone_name)) ||
-		!diff_read_32(in, &file_serial) ||
-		!diff_read_16(in, &file_id) ||
-		!diff_read_32(in, &file_seq_nr))
-	{
-		log_msg(LOG_ERR, "could not part data");
-		region_destroy(region);
-		return 0;
-	}
-
-	if(strcmp(file_zone_name, zone) != 0 || serialno != file_serial ||
-		id != file_id || seq_nr != file_seq_nr) {
-		log_msg(LOG_ERR, "internal error: reading part with changed id");
-		region_destroy(region);
-		return 0;
-	}
-	msglen = filelen - sizeof(uint32_t)*3 - sizeof(uint16_t)
-		- strlen(file_zone_name);
 	packet = buffer_create(region, QIOBUFSZ);
-	dname_zone = dname_parse(region, zone);
-	zone_db = find_zone(db, dname_zone, opt, child_count);
-	if(!zone_db) {
-		log_msg(LOG_ERR, "no zone exists");
-		region_destroy(region);
-		return 0;
-	}
-
 	if(msglen > QIOBUFSZ) {
 		log_msg(LOG_ERR, "msg too long");
 		region_destroy(region);
@@ -746,6 +1007,23 @@ apply_ixfr(namedb_type* db, FILE *in, const off_t* startpos,
 	}
 	buffer_set_limit(packet, msglen);
 
+	/* see if check on data fails: checks that we are not reading
+	 * random garbage */
+	if(!diff_read_32(in, &checklen) || checklen != msglen) {
+		log_msg(LOG_ERR, "transfer part has incorrect checkvalue");
+		return 0;
+	}
+	*bytes += msglen;
+
+	dname_zone = dname_parse(region, zone);
+	zone_db = find_or_create_zone(db, dname_zone, opt, zone, patname);
+	if(!zone_db) {
+		log_msg(LOG_ERR, "could not create zone %s %s", zone, patname);
+		region_destroy(region);
+		return 0;
+	}
+	*zone_res = zone_db;
+
 	/* only answer section is really used, question, additional and
 	   authority section RRs are skipped */
 	qcount = QDCOUNT(packet);
@@ -801,8 +1079,8 @@ apply_ixfr(namedb_type* db, FILE *in, const off_t* startpos,
 		}
 		if(buffer_read_u32(packet) != serialno) {
 			buffer_skip(packet, -4);
-			log_msg(LOG_ERR, "SOA serial %d different from commit %d",
-				buffer_read_u32(packet), serialno);
+			log_msg(LOG_ERR, "SOA serial %u different from commit %u",
+				(unsigned)buffer_read_u32(packet), (unsigned)serialno);
 			region_destroy(region);
 			return 0;
 		}
@@ -811,7 +1089,6 @@ apply_ixfr(namedb_type* db, FILE *in, const off_t* startpos,
 		*rr_count = 1;
 		*is_axfr = 0;
 		*delete_mode = 0;
-
 		DEBUG(DEBUG_XFRD,2, (LOG_INFO, "diff: %s start count %d, ax %d, delmode %d",
 			dname_to_string(dname_zone, 0), *rr_count, *is_axfr, *delete_mode));
 	}
@@ -847,7 +1124,16 @@ apply_ixfr(namedb_type* db, FILE *in, const off_t* startpos,
 
 		if(*rr_count == 1 && type != TYPE_SOA) {
 			/* second RR: if not SOA: this is an AXFR; delete all zone contents */
+#ifdef NSEC3
+			nsec3_hash_tree_clear(zone_db);
+#endif
 			delete_zone_rrs(db, zone_db);
+			if(db->udb)
+				udb_zone_clear(db->udb, udbz);
+#ifdef NSEC3
+			nsec3_clear_precompile(db, zone_db);
+			zone_db->nsec3_param = NULL;
+#endif /* NSEC3 */
 			/* add everything else (incl end SOA) */
 			*delete_mode = 0;
 			*is_axfr = 1;
@@ -869,10 +1155,30 @@ apply_ixfr(namedb_type* db, FILE *in, const off_t* startpos,
 			thisserial = buffer_read_u32(packet);
 			if(thisserial == serialno) {
 				/* AXFR */
+#ifdef NSEC3
+				nsec3_hash_tree_clear(zone_db);
+#endif
 				delete_zone_rrs(db, zone_db);
+				if(db->udb)
+					udb_zone_clear(db->udb, udbz);
+#ifdef NSEC3
+				nsec3_clear_precompile(db, zone_db);
+				zone_db->nsec3_param = NULL;
+#endif /* NSEC3 */
 				*delete_mode = 0;
 				*is_axfr = 1;
 			}
+			/* must have stuff in memory for a successful IXFR,
+			 * the serial number of the SOA has been checked
+			 * previously (by check_for_bad_serial) if it exists */
+			if(!*is_axfr && !domain_find_rrset(zone_db->apex,
+				zone_db, TYPE_SOA)) {
+				log_msg(LOG_ERR, "%s SOA serial %u is not "
+					"in memory, skip IXFR", zone, serialno);
+				region_destroy(region);
+				/* break out and stop the IXFR, ignore it */
+				return 2;
+			}
 			buffer_set_position(packet, bufpos);
 		}
 		if(type == TYPE_SOA && !*is_axfr) {
@@ -899,7 +1205,7 @@ apply_ixfr(namedb_type* db, FILE *in, const off_t* startpos,
 				continue; /* do not delete final SOA RR for IXFR */
 			}
 			if(!delete_RR(db, dname, type, klass, packet,
-				rrlen, zone_db, region)) {
+				rrlen, zone_db, region, udbz, softfail)) {
 				region_destroy(region);
 				return 0;
 			}
@@ -908,7 +1214,7 @@ apply_ixfr(namedb_type* db, FILE *in, const off_t* startpos,
 		{
 			/* add this rr */
 			if(!add_RR(db, dname, type, klass, ttl, packet,
-				rrlen, zone_db)) {
+				rrlen, zone_db, udbz, softfail)) {
 				region_destroy(region);
 				return 0;
 			}
@@ -928,7 +1234,7 @@ check_for_bad_serial(namedb_type* db, const char* zone_str, uint32_t old_serial)
 	zone_type* zone = 0;
 	domain = domain_table_find(db->domains, zone_name);
 	if(domain)
-		zone = domain_find_zone(domain);
+		zone = domain_find_zone(db, domain);
 	if(zone && zone->apex == domain && zone->soa_rrset && old_serial)
 	{
 		uint32_t memserial;
@@ -943,548 +1249,796 @@ check_for_bad_serial(namedb_type* db, const char* zone_str, uint32_t old_serial)
 	return 0;
 }
 
-/* for multiple tcp packets use a data structure that has
- * a rbtree (zone_names) with for each zone:
- * 	has a rbtree by sequence number
- *		with inside a serial_number and ID (for checking only)
- *		and contains a off_t to the IXFR packet in the file.
- * so when you get a commit for a zone, get zone obj, find sequence,
- * then check if you have all sequence numbers available. Apply all packets.
- */
-struct diff_read_data {
-	/* rbtree of struct diff_zone*/
-	rbtree_t* zones;
-	/* region for allocation */
-	region_type* region;
-};
-struct diff_zone {
-	/* key is dname of zone */
-	rbnode_t node;
-	/* rbtree of struct diff_xfrpart */
-	rbtree_t* parts;
-};
-struct diff_xfrpart {
-	/* key is sequence number */
-	rbnode_t node;
-	uint32_t seq_nr;
-	uint32_t new_serial;
-	uint16_t id;
-	off_t file_pos;
-};
-
-static struct diff_read_data*
-diff_read_data_create()
-{
-	region_type* region = region_create(xalloc, free);
-	struct diff_read_data* data = (struct diff_read_data*)
-		region_alloc(region, sizeof(struct diff_read_data));
-	if(!data) {
-		log_msg(LOG_ERR, "out of memory, %s:%d", __FILE__, __LINE__);
-		exit(1);
-	}
-	data->region = region;
-	data->zones = rbtree_create(region,
-		(int (*)(const void *, const void *)) dname_compare);
-	return data;
-}
-
-static struct diff_zone*
-diff_read_find_zone(struct diff_read_data* data, const char* name)
-{
-	const dname_type* dname = dname_parse(data->region, name);
-	struct diff_zone* zp = (struct diff_zone*)
-		rbtree_search(data->zones, dname);
-	return zp;
-}
-
-static int intcompf(const void* a, const void* b)
-{
-	if(*(uint32_t*)a < *(uint32_t*)b)
-		return -1;
-	if(*(uint32_t*)a > *(uint32_t*)b)
-		return +1;
-	return 0;
-}
-
-static struct diff_zone*
-diff_read_insert_zone(struct diff_read_data* data, const char* name)
-{
-	const dname_type* dname = dname_parse(data->region, name);
-	struct diff_zone* zp = region_alloc(data->region,
-		sizeof(struct diff_zone));
-	if(!zp) {
-		log_msg(LOG_ERR, "out of memory, %s:%d", __FILE__, __LINE__);
-		exit(1);
-	}
-	zp->node = *RBTREE_NULL;
-	zp->node.key = dname;
-	zp->parts = rbtree_create(data->region, intcompf);
-	rbtree_insert(data->zones, (rbnode_t*)zp);
-	return zp;
-}
-
-static struct diff_xfrpart*
-diff_read_find_part(struct diff_zone* zp, uint32_t seq_nr)
-{
-	struct diff_xfrpart* xp = (struct diff_xfrpart*)
-		rbtree_search(zp->parts, &seq_nr);
-	return xp;
-}
-
-static struct diff_xfrpart*
-diff_read_insert_part(struct diff_read_data* data,
-	struct diff_zone* zp, uint32_t seq_nr)
-{
-	struct diff_xfrpart* xp = region_alloc(data->region,
-		sizeof(struct diff_xfrpart));
-	if(!xp) {
-		log_msg(LOG_ERR, "out of memory, %s:%d", __FILE__, __LINE__);
-		exit(1);
-	}
-	xp->node = *RBTREE_NULL;
-	xp->node.key = &xp->seq_nr;
-	xp->seq_nr = seq_nr;
-	rbtree_insert(zp->parts, (rbnode_t*)xp);
-	return xp;
-}
-
-/* mark commit as rollback and close inputfile, fatal exits */
-static void
-mark_and_exit(nsd_options_t* opt, FILE* f, off_t commitpos, const char* desc)
-{
-	const char* filename = opt->difffile;
-	fclose(f);
-	if(!(f = fopen(filename, "r+"))) {
-		log_msg(LOG_ERR, "mark xfr, failed to re-open difffile %s: %s",
-			filename, strerror(errno));
-	} else if(fseeko(f, commitpos, SEEK_SET) == -1) {
-		log_msg(LOG_INFO, "could not fseeko: %s.", strerror(errno));
-		fclose(f);
-	} else {
-		uint8_t c = 0;
-		fwrite(&c, sizeof(c), 1, f);
-		fclose(f);
-		log_msg(LOG_ERR, "marked xfr as failed: %s", desc);
-		log_msg(LOG_ERR, "marked xfr so that next reload can succeed");
-	}
-	exit(1);
-}
-
 static int
-read_sure_part(namedb_type* db, FILE *in, nsd_options_t* opt,
-	struct diff_read_data* data, struct diff_log** log,
-	size_t child_count)
+apply_ixfr_for_zone(nsd_type* nsd, zone_type* zonedb, FILE* in,
+	nsd_options_t* opt, udb_base* taskudb, udb_ptr* last_task,
+	uint32_t xfrfilenr)
 {
 	char zone_buf[3072];
 	char log_buf[5120];
-	uint32_t old_serial, new_serial, num_parts;
-	uint16_t id;
+	char patname_buf[2048];
+
+	uint32_t old_serial, new_serial, num_parts, type;
+	uint64_t time_end_0, time_start_0;
+	uint32_t time_end_1, time_start_1;
 	uint8_t committed;
-	struct diff_zone *zp;
 	uint32_t i;
-	int have_all_parts = 1;
-	struct diff_log* thislog = 0;
-	off_t commitpos;
+	int num_bytes = 0;
 
 	/* read zone name and serial */
-	if(!diff_read_str(in, zone_buf, sizeof(zone_buf)) ||
-		!diff_read_32(in, &old_serial) ||
-		!diff_read_32(in, &new_serial) ||
-		!diff_read_16(in, &id) ||
-		!diff_read_32(in, &num_parts)) {
-		log_msg(LOG_ERR, "diff file bad commit part");
+	if(!diff_read_32(in, &type)) {
+		log_msg(LOG_ERR, "diff file too short");
 		return 0;
 	}
-	commitpos = ftello(in); /* position of commit byte */
-	if(commitpos == -1) {
-		log_msg(LOG_INFO, "could not ftello: %s.", strerror(errno));
+	if(type != DIFF_PART_XFRF) {
+		log_msg(LOG_ERR, "xfr file has wrong format");
 		return 0;
+
 	}
+	/* committed and num_parts are first because they need to be
+	 * updated once the rest is written.  The log buf is not certain
+	 * until its done, so at end of file.  The patname is in case a
+	 * new zone is created, we know what the options-pattern is */
 	if(!diff_read_8(in, &committed) ||
-		!diff_read_str(in, log_buf, sizeof(log_buf)) )
-	{
+		!diff_read_32(in, &num_parts) ||
+		!diff_read_64(in, &time_end_0) ||
+		!diff_read_32(in, &time_end_1) ||
+		!diff_read_32(in, &old_serial) ||
+		!diff_read_32(in, &new_serial) ||
+		!diff_read_64(in, &time_start_0) ||
+		!diff_read_32(in, &time_start_1) ||
+		!diff_read_str(in, zone_buf, sizeof(zone_buf)) ||
+		!diff_read_str(in, patname_buf, sizeof(patname_buf))) {
 		log_msg(LOG_ERR, "diff file bad commit part");
 		return 0;
 	}
 
-	if(log) {
-		thislog = (struct diff_log*)region_alloc(db->region, sizeof(struct diff_log));
-		if(!thislog) {
-			log_msg(LOG_ERR, "out of memory, %s:%d", __FILE__, __LINE__);
-			exit(1);
-		}
-		thislog->zone_name = region_strdup(db->region, zone_buf);
-		thislog->comment = region_strdup(db->region, log_buf);
-		thislog->error = 0;
-		thislog->next = *log;
-		*log = thislog;
-	}
-
 	/* has been read in completely */
-	zp = diff_read_find_zone(data, zone_buf);
-	if(!zp) {
-		log_msg(LOG_ERR, "diff file commit without IXFR");
-		if(thislog)
-			thislog->error = "error no IXFR parts";
-		return 1;
+	if(strcmp(zone_buf, dname_to_string(zonedb->apex->dname,0)) != 0) {
+		log_msg(LOG_ERR, "file %s does not match task %s",
+			zone_buf, dname_to_string(zonedb->apex->dname,0));
+		return 0;
 	}
-	if(committed && check_for_bad_serial(db, zone_buf, old_serial)) {
-		DEBUG(DEBUG_XFRD,1, (LOG_ERR,
-			"skipping diff file commit with bad serial"));
-		zp->parts->root = RBTREE_NULL;
-		zp->parts->count = 0;
-		if(thislog)
-			thislog->error = "error bad serial";
-		return 1;
+	if(!committed) {
+		log_msg(LOG_ERR, "diff file %s was not committed", zone_buf);
+		return 0;
 	}
-	for(i=0; i<num_parts; i++) {
-		struct diff_xfrpart *xp = diff_read_find_part(zp, i);
-		if(!xp || xp->id != id || xp->new_serial != new_serial) {
-			have_all_parts = 0;
-		}
+	if(num_parts == 0) {
+		log_msg(LOG_ERR, "diff file %s was not completed", zone_buf);
+		return 0;
 	}
-	if(!have_all_parts) {
+	if(check_for_bad_serial(nsd->db, zone_buf, old_serial)) {
 		DEBUG(DEBUG_XFRD,1, (LOG_ERR,
-			"skipping diff file commit without all parts"));
-		if(thislog)
-			thislog->error = "error missing parts";
+			"skipping diff file commit with bad serial"));
+		return 1;
 	}
 
-	if(committed && have_all_parts)
+	if(committed)
 	{
-		int is_axfr=0, delete_mode=0, rr_count=0;
-		off_t resume_pos;
-
-		DEBUG(DEBUG_XFRD,1, (LOG_INFO, "processing xfr: %s", log_buf));
-		resume_pos = ftello(in);
-		if(resume_pos == -1) {
-			log_msg(LOG_INFO, "could not ftello: %s.", strerror(errno));
-			return 0;
+		int is_axfr=0, delete_mode=0, rr_count=0, softfail=0;
+		const dname_type* apex = zonedb->apex->dname;
+		udb_ptr z;
+
+		DEBUG(DEBUG_XFRD,1, (LOG_INFO, "processing xfr: %s", zone_buf));
+		if(nsd->db->udb) {
+			if(udb_base_get_userflags(nsd->db->udb) != 0) {
+				log_msg(LOG_ERR, "database corrupted, cannot update");
+				xfrd_unlink_xfrfile(nsd, xfrfilenr);
+				exit(1);
+			}
+			/* all parts were checked by xfrd before commit */
+			if(!udb_zone_search(nsd->db->udb, &z, dname_name(apex),
+				apex->name_size)) {
+				/* create it */
+				if(!udb_zone_create(nsd->db->udb, &z, dname_name(apex),
+					apex->name_size)) {
+					/* out of disk space perhaps */
+					log_msg(LOG_ERR, "could not udb_create_zone "
+						"%s, disk space full?", log_buf);
+					return 0;
+				}
+			}
+			/* set the udb dirty until we are finished applying changes */
+			udb_base_set_userflags(nsd->db->udb, 1);
 		}
+		/* read and apply all of the parts */
 		for(i=0; i<num_parts; i++) {
-			struct diff_xfrpart *xp = diff_read_find_part(zp, i);
+			int ret;
 			DEBUG(DEBUG_XFRD,2, (LOG_INFO, "processing xfr: apply part %d", (int)i));
-			if(!apply_ixfr(db, in, &xp->file_pos, zone_buf, new_serial, opt,
-				id, xp->seq_nr, num_parts, &is_axfr, &delete_mode,
-				&rr_count, child_count)) {
-				log_msg(LOG_ERR, "bad ixfr packet part %d in %s", (int)i,
-					opt->difffile);
-				mark_and_exit(opt, in, commitpos, log_buf);
+			ret = apply_ixfr(nsd->db, in, zone_buf, new_serial, opt,
+				i, num_parts, &is_axfr, &delete_mode,
+				&rr_count, (nsd->db->udb?&z:NULL), &zonedb,
+				patname_buf, &num_bytes, &softfail);
+			if(ret == 0) {
+				log_msg(LOG_ERR, "bad ixfr packet part %d in diff file for %s", (int)i, zone_buf);
+				xfrd_unlink_xfrfile(nsd, xfrfilenr);
+				/* the udb is still dirty, it is bad */
+				exit(1);
+			} else if(ret == 2) {
+				break;
 			}
 		}
-		if(fseeko(in, resume_pos, SEEK_SET) == -1) {
-			log_msg(LOG_INFO, "could not fseeko: %s.", strerror(errno));
-			return 0;
+		if(nsd->db->udb)
+			udb_base_set_userflags(nsd->db->udb, 0);
+		/* read the final log_str: but do not fail on it */
+		if(!diff_read_str(in, log_buf, sizeof(log_buf))) {
+			log_msg(LOG_ERR, "could not read log for transfer %s",
+				zone_buf);
+			snprintf(log_buf, sizeof(log_buf), "error reading log");
+		}
+#ifdef NSEC3
+		if(zonedb) prehash_zone(nsd->db, zonedb);
+#endif /* NSEC3 */
+		zonedb->is_changed = 1;
+		if(nsd->db->udb) {
+			ZONE(&z)->is_changed = 1;
+			ZONE(&z)->mtime = time_end_0;
+			udb_zone_set_log_str(nsd->db->udb, &z, log_buf);
+			udb_zone_set_file_str(nsd->db->udb, &z, NULL);
+			udb_ptr_unlink(&z, nsd->db->udb);
+		} else {
+			zonedb->mtime = time_end_0;
+			if(zonedb->logstr)
+				region_recycle(nsd->db->region, zonedb->logstr,
+					strlen(zonedb->logstr)+1);
+			zonedb->logstr = region_strdup(nsd->db->region, log_buf);
+			if(zonedb->filename)
+				region_recycle(nsd->db->region, zonedb->filename,
+					strlen(zonedb->filename)+1);
+			zonedb->filename = NULL;
+		}
+		if(softfail && taskudb && !is_axfr) {
+			log_msg(LOG_ERR, "Failed to apply IXFR cleanly "
+				"(deletes nonexistent RRs, adds existing RRs). "
+				"Zone %s contents is different from master, "
+				"starting AXFR. Transfer %s", zone_buf, log_buf);
+			/* add/del failures in IXFR, get an AXFR */
+			task_new_soainfo(taskudb, last_task, zonedb, 1);
+		} else {
+			if(taskudb)
+				task_new_soainfo(taskudb, last_task, zonedb, 0);
+		}
+
+		if(1 <= verbosity) {
+			double elapsed = (double)(time_end_0 - time_start_0)+
+				(double)((double)time_end_1
+				-(double)time_start_1) / 1000000.0;
+			VERBOSITY(2, (LOG_INFO, "zone %s %s of %d bytes in %g seconds",
+				zone_buf, log_buf, num_bytes, elapsed));
 		}
 	}
 	else {
 	 	DEBUG(DEBUG_XFRD,1, (LOG_INFO, "skipping xfr: %s", log_buf));
 	}
-
-	/* clean out the parts for the zone after the commit/rollback */
-	zp->parts->root = RBTREE_NULL;
-	zp->parts->count = 0;
 	return 1;
 }
 
+struct udb_base* task_file_create(const char* file)
+{
+        return udb_base_create_new(file, &namedb_walkfunc, NULL);
+}
+
 static int
-store_ixfr_data(FILE *in, uint32_t len, struct diff_read_data* data, off_t* startpos)
-{
-	char zone_name[3072];
-	struct diff_zone* zp;
-	struct diff_xfrpart* xp;
-	uint32_t new_serial, seq;
-	uint16_t id;
-	if(!diff_read_str(in, zone_name, sizeof(zone_name)) ||
-		!diff_read_32(in, &new_serial) ||
-		!diff_read_16(in, &id) ||
-		!diff_read_32(in, &seq)) {
-		log_msg(LOG_INFO, "could not read ixfr store info: file format error");
+task_create_new_elem(struct udb_base* udb, udb_ptr* last, udb_ptr* e,
+	size_t sz, const dname_type* zname)
+{
+	if(!udb_ptr_alloc_space(e, udb, udb_chunk_type_task, sz)) {
 		return 0;
 	}
-	len -= sizeof(uint32_t)*3 + sizeof(uint16_t) + strlen(zone_name);
-	if(fseeko(in, len, SEEK_CUR) == -1)
-		log_msg(LOG_INFO, "fseek failed: %s", strerror(errno));
-	/* store the info */
-	zp = diff_read_find_zone(data, zone_name);
-	if(!zp)
-		zp = diff_read_insert_zone(data, zone_name);
-	xp = diff_read_find_part(zp, seq);
-	if(xp) {
-		log_msg(LOG_INFO, "discarding partial xfr part: %s %d", zone_name, seq);
-		/* overwrite with newer value (which probably relates to next commit) */
+	if(udb_ptr_is_null(last)) {
+		udb_base_set_userdata(udb, e->data);
+	} else {
+		udb_rptr_set_ptr(&TASKLIST(last)->next, udb, e);
 	}
-	else {
-		xp = diff_read_insert_part(data, zp, seq);
+	udb_ptr_set_ptr(last, udb, e);
+
+	/* fill in tasklist item */
+	udb_rel_ptr_init(&TASKLIST(e)->next);
+	TASKLIST(e)->size = sz;
+	TASKLIST(e)->oldserial = 0;
+	TASKLIST(e)->newserial = 0;
+	TASKLIST(e)->yesno = 0;
+
+	if(zname) {
+		memmove(TASKLIST(e)->zname, zname, dname_total_size(zname));
 	}
-	xp->new_serial = new_serial;
-	xp->id = id;
-	memmove(&xp->file_pos, startpos, sizeof(off_t));
 	return 1;
 }
 
-static int
-read_process_part(namedb_type* db, FILE *in, uint32_t type,
-	nsd_options_t* opt, struct diff_read_data* data,
-	struct diff_log** log, size_t child_count, off_t* startpos)
+void task_new_soainfo(struct udb_base* udb, udb_ptr* last, struct zone* z,
+	int gone)
 {
-	uint32_t len, len2;
+	/* calculate size */
+	udb_ptr e;
+	size_t sz;
+	const dname_type* apex, *ns, *em;
+	if(!z || !z->apex || !domain_dname(z->apex))
+		return; /* safety check */
+
+	DEBUG(DEBUG_IPC,1, (LOG_INFO, "nsd: add soa info for zone %s",
+		domain_to_string(z->apex)));
+	apex = domain_dname(z->apex);
+	sz = sizeof(struct task_list_d) + dname_total_size(apex);
+	if(z->soa_rrset && !gone) {
+		ns = domain_dname(rdata_atom_domain(
+			z->soa_rrset->rrs[0].rdatas[0]));
+		em = domain_dname(rdata_atom_domain(
+			z->soa_rrset->rrs[0].rdatas[1]));
+		sz += sizeof(uint32_t)*6 + sizeof(uint8_t)*2
+			+ ns->name_size + em->name_size;
+	} else {
+		ns = 0;
+		em = 0;
+	}
 
-	/* read length */
-	if(!diff_read_32(in, &len))
-		return 1;
-	/* read content */
-	if(type == DIFF_PART_IXFR) {
-		DEBUG(DEBUG_XFRD,2, (LOG_INFO, "part IXFR len %d", len));
-		if(!store_ixfr_data(in, len, data, startpos))
-			return 0;
+	/* create new task_list item */
+	if(!task_create_new_elem(udb, last, &e, sz, apex)) {
+		log_msg(LOG_ERR, "tasklist: out of space, cannot add SOAINFO");
+		return;
 	}
-	else if(type == DIFF_PART_SURE) {
-		DEBUG(DEBUG_XFRD,2, (LOG_INFO, "part SURE len %d", len));
-		if(!read_sure_part(db, in, opt, data, log, child_count))
-			return 0;
-	} else {
-		DEBUG(DEBUG_XFRD,1, (LOG_INFO, "unknown part %x len %d", type, len));
-		return 0;
+	TASKLIST(&e)->task_type = task_soa_info;
+
+	if(z->soa_rrset && !gone) {
+		uint32_t ttl = htonl(z->soa_rrset->rrs[0].ttl);
+		uint8_t* p = (uint8_t*)TASKLIST(&e)->zname;
+		p += dname_total_size(apex);
+		memmove(p, &ttl, sizeof(uint32_t));
+		p += sizeof(uint32_t);
+		memmove(p, &ns->name_size, sizeof(uint8_t));
+		p += sizeof(uint8_t);
+		memmove(p, dname_name(ns), ns->name_size);
+		p += ns->name_size;
+		memmove(p, &em->name_size, sizeof(uint8_t));
+		p += sizeof(uint8_t);
+		memmove(p, dname_name(em), em->name_size);
+		p += em->name_size;
+		memmove(p, rdata_atom_data(z->soa_rrset->rrs[0].rdatas[2]),
+			sizeof(uint32_t));
+		p += sizeof(uint32_t);
+		memmove(p, rdata_atom_data(z->soa_rrset->rrs[0].rdatas[3]),
+			sizeof(uint32_t));
+		p += sizeof(uint32_t);
+		memmove(p, rdata_atom_data(z->soa_rrset->rrs[0].rdatas[4]),
+			sizeof(uint32_t));
+		p += sizeof(uint32_t);
+		memmove(p, rdata_atom_data(z->soa_rrset->rrs[0].rdatas[5]),
+			sizeof(uint32_t));
+		p += sizeof(uint32_t);
+		memmove(p, rdata_atom_data(z->soa_rrset->rrs[0].rdatas[6]),
+			sizeof(uint32_t));
 	}
-	/* read length */
-	if(!diff_read_32(in, &len2))
-		return 1; /* short read is OK */
-	/* verify length */
-	if(len != len2)
-		return 0; /* bad data is wrong */
-	return 1;
+	udb_ptr_unlink(&e, udb);
 }
 
-/*
- * Finds smallest offset in data structs
- * returns 0 if no offsets in the data structs.
- */
-static int
-find_smallest_offset(struct diff_read_data* data, off_t* offset)
+void task_process_sync(struct udb_base* taskudb)
 {
-	int found_any = 0;
-	struct diff_zone* dz;
-	struct diff_xfrpart* dx;
-	off_t mem_offset, mem_fpos;
+	/* need to sync before other process uses the mmap? */
+	DEBUG(DEBUG_IPC,1, (LOG_INFO, "task procsync %s size %d",
+		taskudb->fname, (int)taskudb->base_size));
+	(void)taskudb;
+}
 
-	if(!data || !data->zones)
-		return 0;
-	RBTREE_FOR(dz, struct diff_zone*, data->zones)
-	{
-		if(!dz->parts)
-			continue;
-		RBTREE_FOR(dx, struct diff_xfrpart*, dz->parts)
-		{
-			memmove(&mem_fpos, &dx->file_pos, sizeof(off_t));
+void task_remap(struct udb_base* taskudb)
+{
+	DEBUG(DEBUG_IPC,1, (LOG_INFO, "task remap %s size %d",
+		taskudb->fname, (int)taskudb->glob_data->fsize));
+	udb_base_remap_process(taskudb);
+}
 
-			if(found_any) {
-				memmove(&mem_offset, offset, sizeof(off_t));
+void task_clear(struct udb_base* taskudb)
+{
+	udb_ptr t, n;
+	udb_ptr_new(&t, taskudb, udb_base_get_userdata(taskudb));
+	udb_base_set_userdata(taskudb, 0);
+	udb_ptr_init(&n, taskudb);
+	while(!udb_ptr_is_null(&t)) {
+		udb_ptr_set_rptr(&n, taskudb, &TASKLIST(&t)->next);
+		udb_rptr_zero(&TASKLIST(&t)->next, taskudb);
+		udb_ptr_free_space(&t, taskudb, TASKLIST(&t)->size);
+		udb_ptr_set_ptr(&t, taskudb, &n);
+	}
+	udb_ptr_unlink(&t, taskudb);
+	udb_ptr_unlink(&n, taskudb);
+}
 
-				if(mem_fpos < mem_offset)
-					memmove(offset, &mem_fpos, sizeof(off_t));
-			} else {
-				found_any = 1;
-				memmove(offset, &mem_fpos, sizeof(off_t));
-			}
-		}
+void task_new_expire(struct udb_base* udb, udb_ptr* last,
+	const struct dname* z, int expired)
+{
+	udb_ptr e;
+	if(!z) return;
+	DEBUG(DEBUG_IPC,1, (LOG_INFO, "add expire info for zone %s",
+		dname_to_string(z,NULL)));
+	if(!task_create_new_elem(udb, last, &e, sizeof(struct task_list_d)+
+		dname_total_size(z), z)) {
+		log_msg(LOG_ERR, "tasklist: out of space, cannot add expire");
+		return;
 	}
+	TASKLIST(&e)->task_type = task_expire;
+	TASKLIST(&e)->yesno = expired;
+	udb_ptr_unlink(&e, udb);
+}
 
-	return found_any;
+void task_new_check_zonefiles(udb_base* udb, udb_ptr* last,
+	const dname_type* zone)
+{
+	udb_ptr e;
+	DEBUG(DEBUG_IPC,1, (LOG_INFO, "add task checkzonefiles"));
+	if(!task_create_new_elem(udb, last, &e, sizeof(struct task_list_d) +
+		(zone?dname_total_size(zone):0), zone)) {
+		log_msg(LOG_ERR, "tasklist: out of space, cannot add check_zones");
+		return;
+	}
+	TASKLIST(&e)->task_type = task_check_zonefiles;
+	TASKLIST(&e)->yesno = (zone!=NULL);
+	udb_ptr_unlink(&e, udb);
 }
 
-int
-diff_read_file(namedb_type* db, nsd_options_t* opt, struct diff_log** log,
+void task_new_write_zonefiles(udb_base* udb, udb_ptr* last,
+	const dname_type* zone)
+{
+	udb_ptr e;
+	DEBUG(DEBUG_IPC,1, (LOG_INFO, "add task writezonefiles"));
+	if(!task_create_new_elem(udb, last, &e, sizeof(struct task_list_d) +
+		(zone?dname_total_size(zone):0), zone)) {
+		log_msg(LOG_ERR, "tasklist: out of space, cannot add writezones");
+		return;
+	}
+	TASKLIST(&e)->task_type = task_write_zonefiles;
+	TASKLIST(&e)->yesno = (zone!=NULL);
+	udb_ptr_unlink(&e, udb);
+}
+
+void task_new_set_verbosity(udb_base* udb, udb_ptr* last, int v)
+{
+	udb_ptr e;
+	DEBUG(DEBUG_IPC,1, (LOG_INFO, "add task set_verbosity"));
+	if(!task_create_new_elem(udb, last, &e, sizeof(struct task_list_d),
+		NULL)) {
+		log_msg(LOG_ERR, "tasklist: out of space, cannot add set_v");
+		return;
+	}
+	TASKLIST(&e)->task_type = task_set_verbosity;
+	TASKLIST(&e)->yesno = v;
+	udb_ptr_unlink(&e, udb);
+}
+
+#ifdef BIND8_STATS
+void* task_new_stat_info(udb_base* udb, udb_ptr* last, struct nsdst* stat,
 	size_t child_count)
 {
-	const char* filename = opt->difffile;
-	FILE *df;
-	uint32_t type, timestamp[2], curr_timestamp[2];
-	struct diff_read_data* data = diff_read_data_create();
-	off_t startpos;
+	void* p;
+	udb_ptr e;
+	DEBUG(DEBUG_IPC,1, (LOG_INFO, "add task stat_info"));
+	if(!task_create_new_elem(udb, last, &e, sizeof(struct task_list_d)+
+		sizeof(*stat) + sizeof(stc_t)*child_count, NULL)) {
+		log_msg(LOG_ERR, "tasklist: out of space, cannot add stati");
+		return NULL;
+	}
+	TASKLIST(&e)->task_type = task_stat_info;
+	p = TASKLIST(&e)->zname;
+	memcpy(p, stat, sizeof(*stat));
+	udb_ptr_unlink(&e, udb);
+	return p + sizeof(*stat);
+}
+#endif /* BIND8_STATS */
 
-	df = fopen(filename, "r");
-	if(!df) {
-		DEBUG(DEBUG_XFRD,1, (LOG_INFO, "could not open file %s for reading: %s",
-			filename, strerror(errno)));
-		region_destroy(data->region);
-		return 1;
+void
+task_new_add_zone(udb_base* udb, udb_ptr* last, const char* zone,
+	const char* pattern)
+{
+	size_t zlen = strlen(zone);
+	size_t plen = strlen(pattern);
+	void *p;
+	udb_ptr e;
+	DEBUG(DEBUG_IPC,1, (LOG_INFO, "add task addzone %s %s", zone, pattern));
+	if(!task_create_new_elem(udb, last, &e, sizeof(struct task_list_d)+
+		zlen + 1 + plen + 1, NULL)) {
+		log_msg(LOG_ERR, "tasklist: out of space, cannot add addz");
+		return;
 	}
+	TASKLIST(&e)->task_type = task_add_zone;
+	p = TASKLIST(&e)->zname;
+	memcpy(p, zone, zlen+1);
+	memmove(p+zlen+1, pattern, plen+1);
+	udb_ptr_unlink(&e, udb);
+}
 
-	/* check timestamp */
-	curr_timestamp[0] = (uint32_t) db->diff_timestamp.tv_sec;
-	curr_timestamp[1] = (uint32_t) db->diff_timestamp.tv_usec;
+void
+task_new_del_zone(udb_base* udb, udb_ptr* last, const dname_type* dname)
+{
+	udb_ptr e;
+	DEBUG(DEBUG_IPC,1, (LOG_INFO, "add task delzone %s", dname_to_string(dname, 0)));
+	if(!task_create_new_elem(udb, last, &e, sizeof(struct task_list_d)
+		+dname_total_size(dname), dname)) {
+		log_msg(LOG_ERR, "tasklist: out of space, cannot add delz");
+		return;
+	}
+	TASKLIST(&e)->task_type = task_del_zone;
+	udb_ptr_unlink(&e, udb);
+}
 
-	if(!diff_read_32(df, &type)) {
-		DEBUG(DEBUG_XFRD,1, (LOG_INFO, "difffile %s is empty",
-			filename));
-		db->diff_skip = 0;
-		db->diff_pos = 0;
+void task_new_add_key(udb_base* udb, udb_ptr* last, key_options_t* key)
+{
+	char* p;
+	udb_ptr e;
+	assert(key->name && key->algorithm && key->secret);
+	DEBUG(DEBUG_IPC,1, (LOG_INFO, "add task addkey"));
+	if(!task_create_new_elem(udb, last, &e, sizeof(struct task_list_d)
+		+strlen(key->name)+1+strlen(key->algorithm)+1+
+		strlen(key->secret)+1, NULL)) {
+		log_msg(LOG_ERR, "tasklist: out of space, cannot add addk");
+		return;
 	}
-	else if (!diff_read_32(df, &timestamp[0]) ||
-		 !diff_read_32(df, &timestamp[1])) {
-		log_msg(LOG_ERR, "difffile %s bad first part: no timestamp",
-			filename);
-		region_destroy(data->region);
-		return 0;
+	TASKLIST(&e)->task_type = task_add_key;
+	p = (char*)TASKLIST(&e)->zname;
+	memmove(p, key->name, strlen(key->name)+1);
+	p+=strlen(key->name)+1;
+	memmove(p, key->algorithm, strlen(key->algorithm)+1);
+	p+=strlen(key->algorithm)+1;
+	memmove(p, key->secret, strlen(key->secret)+1);
+	udb_ptr_unlink(&e, udb);
+}
+
+void task_new_del_key(udb_base* udb, udb_ptr* last, const char* name)
+{
+	char* p;
+	udb_ptr e;
+	DEBUG(DEBUG_IPC,1, (LOG_INFO, "add task delkey"));
+	if(!task_create_new_elem(udb, last, &e, sizeof(struct task_list_d)
+		+strlen(name)+1, NULL)) {
+		log_msg(LOG_ERR, "tasklist: out of space, cannot add delk");
+		return;
 	}
-	else if (curr_timestamp[0] != timestamp[0] ||
-		 curr_timestamp[1] != timestamp[1]) {
-		/* new timestamp, no skipping */
-		db->diff_timestamp.tv_sec = (time_t) timestamp[0];
-		db->diff_timestamp.tv_usec = (suseconds_t) timestamp[1];
-
-		if (db->diff_skip) {
-			DEBUG(DEBUG_XFRD,1, (LOG_INFO, "new timestamp on "
-				"difffile %s, restoring diff_skip and diff_pos "
-				"[old timestamp: %u.%u; new timestamp: %u.%u]",
-				filename, curr_timestamp[0], curr_timestamp[1],
-				timestamp[0], timestamp[1]));
-			db->diff_skip = 0;
-			db->diff_pos = 0;
-		}
+	TASKLIST(&e)->task_type = task_del_key;
+	p = (char*)TASKLIST(&e)->zname;
+	memmove(p, name, strlen(name)+1);
+	udb_ptr_unlink(&e, udb);
+}
+
+void task_new_add_pattern(udb_base* udb, udb_ptr* last, pattern_options_t* p)
+{
+	region_type* temp;
+	buffer_type* buffer;
+	udb_ptr e;
+	DEBUG(DEBUG_IPC,1, (LOG_INFO, "add task addpattern %s", p->pname));
+	temp = region_create(xalloc, free);
+	buffer = buffer_create(temp, 4096);
+	pattern_options_marshal(buffer, p);
+	buffer_flip(buffer);
+	if(!task_create_new_elem(udb, last, &e, sizeof(struct task_list_d)
+		+ buffer_limit(buffer), NULL)) {
+		log_msg(LOG_ERR, "tasklist: out of space, cannot add addp");
+		region_destroy(temp);
+		return;
 	}
+	TASKLIST(&e)->task_type = task_add_pattern;
+	TASKLIST(&e)->yesno = buffer_limit(buffer);
+	memmove(TASKLIST(&e)->zname, buffer_begin(buffer),
+		buffer_limit(buffer));
+	udb_ptr_unlink(&e, udb);
+	region_destroy(temp);
+}
 
-	/* Always seek, to diff_pos or to beginning of the file. */
-	if (fseeko(df, 0, SEEK_SET)==-1) {
-		log_msg(LOG_INFO, "could not fseeko file %s: %s.", filename,
-				strerror(errno));
-		region_destroy(data->region);
-		return 0;
+void task_new_del_pattern(udb_base* udb, udb_ptr* last, const char* name)
+{
+	char* p;
+	udb_ptr e;
+	DEBUG(DEBUG_IPC,1, (LOG_INFO, "add task delpattern %s", name));
+	if(!task_create_new_elem(udb, last, &e, sizeof(struct task_list_d)
+		+strlen(name)+1, NULL)) {
+		log_msg(LOG_ERR, "tasklist: out of space, cannot add delp");
+		return;
 	}
-	if(db->diff_skip) {
-		DEBUG(DEBUG_XFRD,1, (LOG_INFO, "skip diff file"));
-		if(fseeko(df, db->diff_pos, SEEK_SET)==-1) {
-			log_msg(LOG_INFO, "could not fseeko file %s: %s. "
-					  "Reread from start.", filename,
-				strerror(errno));
-		}
+	TASKLIST(&e)->task_type = task_del_pattern;
+	p = (char*)TASKLIST(&e)->zname;
+	memmove(p, name, strlen(name)+1);
+	udb_ptr_unlink(&e, udb);
+}
+
+void task_new_opt_change(udb_base* udb, udb_ptr* last, nsd_options_t* opt)
+{
+	udb_ptr e;
+	DEBUG(DEBUG_IPC,1, (LOG_INFO, "add task opt_change"));
+	if(!task_create_new_elem(udb, last, &e, sizeof(struct task_list_d),
+		NULL)) {
+		log_msg(LOG_ERR, "tasklist: out of space, cannot add o_c");
+		return;
 	}
+	TASKLIST(&e)->task_type = task_opt_change;
+#ifdef RATELIMIT
+	TASKLIST(&e)->oldserial = opt->rrl_ratelimit;
+	TASKLIST(&e)->newserial = opt->rrl_whitelist_ratelimit;
+	TASKLIST(&e)->yesno = (uint64_t) opt->rrl_slip;
+#else
+	(void)opt;
+#endif
+	udb_ptr_unlink(&e, udb);
+}
 
-	startpos = ftello(df);
-	if(startpos == -1) {
-		log_msg(LOG_INFO, "could not ftello: %s.", strerror(errno));
-		region_destroy(data->region);
+int
+task_new_apply_xfr(udb_base* udb, udb_ptr* last, const dname_type* dname,
+	uint32_t old_serial, uint32_t new_serial, uint64_t filenumber)
+{
+	udb_ptr e;
+	DEBUG(DEBUG_IPC,1, (LOG_INFO, "add task apply_xfr"));
+	if(!task_create_new_elem(udb, last, &e, sizeof(struct task_list_d)
+		+dname_total_size(dname), dname)) {
+		log_msg(LOG_ERR, "tasklist: out of space, cannot add applyxfr");
 		return 0;
 	}
+	TASKLIST(&e)->oldserial = old_serial;
+	TASKLIST(&e)->newserial = new_serial;
+	TASKLIST(&e)->yesno = filenumber;
+	TASKLIST(&e)->task_type = task_apply_xfr;
+	udb_ptr_unlink(&e, udb);
+	return 1;
+}
 
-	DEBUG(DEBUG_XFRD,1, (LOG_INFO, "start of diff file read at pos %u",
-		(uint32_t) db->diff_pos));
-	while(diff_read_32(df, &type))
-	{
-		DEBUG(DEBUG_XFRD,2, (LOG_INFO, "iter loop"));
-
-		/* read timestamp */
-		if(!diff_read_32(df, &timestamp[0]) ||
-			!diff_read_32(df, &timestamp[1])) {
-			log_msg(LOG_INFO, "could not read timestamp: %s.",
-				strerror(errno));
-			region_destroy(data->region);
-			return 0;
-		}
+void
+task_process_expire(namedb_type* db, struct task_list_d* task)
+{
+	uint8_t ok;
+	zone_type* z = namedb_find_zone(db, task->zname);
+	assert(task->task_type == task_expire);
+	if(!z) {
+		DEBUG(DEBUG_IPC, 1, (LOG_WARNING, "zone %s %s but not in zonetree",
+			dname_to_string(task->zname, NULL),
+			task->yesno?"expired":"unexpired"));
+		return;
+	}
+	DEBUG(DEBUG_IPC,1, (LOG_INFO, "xfrd: expire task zone %s %s",
+		dname_to_string(task->zname,0),
+		task->yesno?"expired":"unexpired"));
+	/* find zone, set expire flag */
+	ok = !task->yesno;
+	/* only update zone->is_ok if needed to minimize copy-on-write
+	 * of memory pages shared after fork() */
+	if(ok && !z->is_ok)
+		z->is_ok = 1;
+	else if(!ok && z->is_ok)
+		z->is_ok = 0;
+}
 
-		if(!read_process_part(db, df, type, opt, data, log,
-			child_count, &startpos))
-		{
-			log_msg(LOG_INFO, "error processing diff file");
-			region_destroy(data->region);
-			return 0;
-		}
-		startpos = ftello(df);
-		if(startpos == -1) {
-			log_msg(LOG_INFO, "could not ftello: %s.", strerror(errno));
-			region_destroy(data->region);
-			return 0;
-		}
+static void
+task_process_set_verbosity(struct task_list_d* task)
+{
+	DEBUG(DEBUG_IPC,1, (LOG_INFO, "verbosity task %d", (int)task->yesno));
+	verbosity = task->yesno;
+}
+
+static void
+task_process_checkzones(struct nsd* nsd, udb_base* udb, udb_ptr* last_task,
+	struct task_list_d* task)
+{
+	/* on SIGHUP check if zone-text-files changed and if so,
+	 * reread.  When from xfrd-reload, no need to fstat the files */
+	if(task->yesno) {
+		zone_options_t* zo = zone_options_find(nsd->options,
+			task->zname);
+		if(zo)
+			namedb_check_zonefile(nsd, udb, last_task, zo);
+	} else {
+		/* check all zones */
+		namedb_check_zonefiles(nsd, nsd->options, udb, last_task);
 	}
-	DEBUG(DEBUG_XFRD,1, (LOG_INFO, "end of diff file read"));
+}
 
-	if(find_smallest_offset(data, &db->diff_pos)) {
-		/* can skip to the first unused element */
-		DEBUG(DEBUG_XFRD,2, (LOG_INFO, "next time skip diff file"));
-		db->diff_skip = 1;
+static void
+task_process_writezones(struct nsd* nsd, struct task_list_d* task)
+{
+	if(task->yesno) {
+		zone_options_t* zo = zone_options_find(nsd->options,
+			task->zname);
+		if(zo)
+			namedb_write_zonefile(nsd, zo);
 	} else {
-		/* all processed, can skip to here next time */
-		DEBUG(DEBUG_XFRD,2, (LOG_INFO, "next time skip diff file"));
-		db->diff_skip = 1;
-		db->diff_pos = ftello(df);
-		if(db->diff_pos == -1) {
-			log_msg(LOG_INFO, "could not ftello: %s.",
-				strerror(errno));
-			db->diff_skip = 0;
-		}
+		namedb_write_zonefiles(nsd, nsd->options);
 	}
+}
 
-	region_destroy(data->region);
-	fclose(df);
-	return 1;
+static void
+task_process_add_zone(struct nsd* nsd, udb_base* udb, udb_ptr* last_task,
+	struct task_list_d* task)
+{
+	zone_type* z;
+	const dname_type* zdname;
+	const char* zname = (const char*)task->zname;
+	const char* pname = zname + strlen(zname)+1;
+	DEBUG(DEBUG_IPC,1, (LOG_INFO, "addzone task %s %s", zname, pname));
+	zdname = dname_parse(nsd->db->region, zname);
+	if(!zdname) {
+		log_msg(LOG_ERR, "can not parse zone name %s", zname);
+		return;
+	}
+	/* create zone */
+	z = find_or_create_zone(nsd->db, zdname, nsd->options, zname, pname);
+	if(!z) {
+		region_recycle(nsd->db->region, (void*)zdname,
+			dname_total_size(zdname));
+		log_msg(LOG_ERR, "can not add zone %s %s", zname, pname);
+		return;
+	}
+	/* if zone is empty, attempt to read the zonefile from disk (if any) */
+	if(!z->soa_rrset && z->opts->pattern->zonefile) {
+		namedb_read_zonefile(nsd, z, udb, last_task);
+	}
 }
 
-static int diff_broken(FILE *df, off_t* break_pos)
+static void
+task_process_del_zone(struct nsd* nsd, struct task_list_d* task)
 {
-	uint32_t type, len, len2;
-	*break_pos = ftello(df);
+	zone_type* zone;
+	zone_options_t* zopt;
+	DEBUG(DEBUG_IPC,1, (LOG_INFO, "delzone task %s", dname_to_string(
+		task->zname, NULL)));
+	zone = namedb_find_zone(nsd->db, task->zname);
+	if(!zone)
+		return;
 
-	/* try to read and validate parts of the file */
-	while(diff_read_32(df, &type)) /* cannot read type is no error, normal EOF */
-	{
-		/* check type */
-		if(type != DIFF_PART_IXFR && type != DIFF_PART_SURE)
-			return 1;
-		/* check length */
-		if(!diff_read_32(df, &len))
-			return 1; /* EOF inside the part is error */
-		if(fseeko(df, len, SEEK_CUR) == -1)
-		{
-			log_msg(LOG_INFO, "fseeko failed: %s", strerror(errno));
-			return 1;
+#ifdef NSEC3
+	nsec3_hash_tree_clear(zone);
+#endif
+	delete_zone_rrs(nsd->db, zone);
+	if(nsd->db->udb) {
+		udb_ptr udbz;
+		if(udb_zone_search(nsd->db->udb, &udbz, dname_name(task->zname),
+			task->zname->name_size)) {
+			udb_zone_delete(nsd->db->udb, &udbz);
+			udb_ptr_unlink(&udbz, nsd->db->udb);
 		}
-		/* fseek clears EOF flag, but try reading length value,
-		   if EOF, the part is truncated */
-		if(!diff_read_32(df, &len2))
-			return 1;
-		if(len != len2)
-			return 1; /* bad part, lengths must agree */
-		/* this part is ok */
-		*break_pos = ftello(df);
 	}
-	return 0;
+#ifdef NSEC3
+	nsec3_clear_precompile(nsd->db, zone);
+	zone->nsec3_param = NULL;
+#endif /* NSEC3 */
+
+	/* remove from zonetree, apex, soa */
+	zopt = zone->opts;
+	namedb_zone_delete(nsd->db, zone);
+	/* remove from options (zone_list already edited by xfrd) */
+	zone_options_delete(nsd->options, zopt);
 }
 
-void diff_snip_garbage(namedb_type* db, nsd_options_t* opt)
+static void
+task_process_add_key(struct nsd* nsd, struct task_list_d* task)
 {
-	off_t break_pos;
-	const char* filename = opt->difffile;
-	FILE *df;
+	key_options_t key;
+	key.name = (char*)task->zname;
+	DEBUG(DEBUG_IPC,1, (LOG_INFO, "addkey task %s", key.name));
+	key.algorithm = key.name + strlen(key.name)+1;
+	key.secret = key.algorithm + strlen(key.algorithm)+1;
+	key_options_add_modify(nsd->options, &key);
+	memset(key.secret, 0xdd, strlen(key.secret)); /* wipe secret */
+}
 
-	/* open file here and keep open, so it cannot change under our nose */
-	df = fopen(filename, "r+");
-	if(!df) {
-		DEBUG(DEBUG_XFRD,1, (LOG_INFO, "could not open file %s for garbage collecting: %s",
-			filename, strerror(errno)));
+static void
+task_process_del_key(struct nsd* nsd, struct task_list_d* task)
+{
+	char* name = (char*)task->zname;
+	DEBUG(DEBUG_IPC,1, (LOG_INFO, "delkey task %s", name));
+	/* this is reload and nothing is using the TSIG key right now */
+	key_options_remove(nsd->options, name);
+}
+
+static void
+task_process_add_pattern(struct nsd* nsd, struct task_list_d* task)
+{
+	region_type* temp = region_create(xalloc, free);
+	buffer_type buffer;
+	pattern_options_t *pat;
+	buffer_create_from(&buffer, task->zname, task->yesno);
+	pat = pattern_options_unmarshal(temp, &buffer);
+	DEBUG(DEBUG_IPC,1, (LOG_INFO, "addpattern task %s", pat->pname));
+	pattern_options_add_modify(nsd->options, pat);
+	region_destroy(temp);
+}
+
+static void
+task_process_del_pattern(struct nsd* nsd, struct task_list_d* task)
+{
+	char* name = (char*)task->zname;
+	DEBUG(DEBUG_IPC,1, (LOG_INFO, "delpattern task %s", name));
+	pattern_options_remove(nsd->options, name);
+}
+
+static void
+task_process_opt_change(struct nsd* nsd, struct task_list_d* task)
+{
+	DEBUG(DEBUG_IPC,1, (LOG_INFO, "optchange task"));
+#ifdef RATELIMIT
+	nsd->options->rrl_ratelimit = task->oldserial;
+	nsd->options->rrl_whitelist_ratelimit = task->newserial;
+	nsd->options->rrl_slip = task->yesno;
+	rrl_set_limit(nsd->options->rrl_ratelimit, nsd->options->rrl_whitelist_ratelimit,
+		nsd->options->rrl_slip);
+#else
+	(void)nsd; (void)task;
+#endif
+}
+
+static void
+task_process_apply_xfr(struct nsd* nsd, udb_base* udb, udb_ptr *last_task,
+	udb_ptr* task)
+{
+	/* we have to use an udb_ptr task here, because the apply_xfr procedure
+	 * appends soa_info which may remap and change the pointer. */
+	zone_type* zone;
+	FILE* df;
+	DEBUG(DEBUG_IPC,1, (LOG_INFO, "applyxfr task %s", dname_to_string(
+		TASKLIST(task)->zname, NULL)));
+	zone = namedb_find_zone(nsd->db, TASKLIST(task)->zname);
+	if(!zone) {
+		/* assume the zone has been deleted and a zone transfer was
+		 * still waiting to be processed */
+		xfrd_unlink_xfrfile(nsd, TASKLIST(task)->yesno);
 		return;
 	}
-	/* and skip into file, since nsd does not read anything before the pos */
-	if(db->diff_skip) {
-		DEBUG(DEBUG_XFRD,1, (LOG_INFO, "garbage collect skip diff file"));
-		if(fseeko(df, db->diff_pos, SEEK_SET)==-1) {
-			log_msg(LOG_INFO, "could not fseeko file %s: %s.",
-				filename, strerror(errno));
-			fclose(df);
-			return;
-		}
+	/* apply the XFR */
+	/* oldserial, newserial, yesno is filenumber */
+	df = xfrd_open_xfrfile(nsd, TASKLIST(task)->yesno, "r");
+	if(!df) {
+		/* could not open file to update */
+		/* there is no reply to xfrd failed-update,
+		 * because xfrd has a scan for apply-failures. */
+		xfrd_unlink_xfrfile(nsd, TASKLIST(task)->yesno);
+		return;
 	}
-
-	/* detect break point */
-	if(diff_broken(df, &break_pos))
-	{
-		/* snip off at break_pos */
-		DEBUG(DEBUG_XFRD,1, (LOG_INFO, "snipping off trailing partial part of %s",
-			filename));
-		if(ftruncate(fileno(df), break_pos) == -1)
-			log_msg(LOG_ERR, "ftruncate %s failed: %s",
-				filename, strerror(errno));
+	/* read and apply zone transfer */
+	if(!apply_ixfr_for_zone(nsd, zone, df, nsd->options, udb,
+		last_task, TASKLIST(task)->yesno)) {
+		/* there is no reply to xfrd failed-update,
+		 * because xfrd has a scan for apply-failures. */
 	}
 
 	fclose(df);
+	xfrd_unlink_xfrfile(nsd, TASKLIST(task)->yesno);
+}
+
+
+void task_process_in_reload(struct nsd* nsd, udb_base* udb, udb_ptr *last_task,
+        udb_ptr* task)
+{
+	switch(TASKLIST(task)->task_type) {
+	case task_expire:
+		task_process_expire(nsd->db, TASKLIST(task));
+		break;
+	case task_check_zonefiles:
+		task_process_checkzones(nsd, udb, last_task, TASKLIST(task));
+		break;
+	case task_write_zonefiles:
+		task_process_writezones(nsd, TASKLIST(task));
+		break;
+	case task_set_verbosity:
+		task_process_set_verbosity(TASKLIST(task));
+		break;
+	case task_add_zone:
+		task_process_add_zone(nsd, udb, last_task, TASKLIST(task));
+		break;
+	case task_del_zone:
+		task_process_del_zone(nsd, TASKLIST(task));
+		break;
+	case task_add_key:
+		task_process_add_key(nsd, TASKLIST(task));
+		break;
+	case task_del_key:
+		task_process_del_key(nsd, TASKLIST(task));
+		break;
+	case task_add_pattern:
+		task_process_add_pattern(nsd, TASKLIST(task));
+		break;
+	case task_del_pattern:
+		task_process_del_pattern(nsd, TASKLIST(task));
+		break;
+	case task_opt_change:
+		task_process_opt_change(nsd, TASKLIST(task));
+		break;
+	case task_apply_xfr:
+		task_process_apply_xfr(nsd, udb, last_task, task);
+		break;
+	default:
+		log_msg(LOG_WARNING, "unhandled task in reload type %d",
+			(int)TASKLIST(task)->task_type);
+		break;
+	}
+	udb_ptr_free_space(task, udb, TASKLIST(task)->size);
 }
diff --git a/usr.sbin/nsd/dns.c b/usr.sbin/nsd/dns.c
index a54fc9871c0..5d80f8a11ba 100644
--- a/usr.sbin/nsd/dns.c
+++ b/usr.sbin/nsd/dns.c
@@ -289,9 +289,14 @@ static rrtype_descriptor_type rrtype_descriptors[(RRTYPE_DESCRIPTORS_LENGTH+1)]
 	/* 58 - TALINK */
 	{ 58, NULL, T_UTYPE, 1, 1, { RDATA_WF_BINARY }, { RDATA_ZF_UNKNOWN } },
 	/* 59 - CDS */
-	{ 59, NULL, T_UTYPE, 1, 1, { RDATA_WF_BINARY }, { RDATA_ZF_UNKNOWN } },
-	/* 60 */
-	{ 60, NULL, T_UTYPE, 1, 1, { RDATA_WF_BINARY }, { RDATA_ZF_UNKNOWN } },
+	{ TYPE_CDS, "CDS", T_CDS, 4, 4,
+	  { RDATA_WF_SHORT, RDATA_WF_BYTE, RDATA_WF_BYTE, RDATA_WF_BINARY },
+	  { RDATA_ZF_SHORT, RDATA_ZF_ALGORITHM, RDATA_ZF_BYTE, RDATA_ZF_HEX } },
+	/* 60 - CDNSKEY */
+	{ TYPE_CDNSKEY, "CDNSKEY", T_CDNSKEY, 4, 4,
+	  { RDATA_WF_SHORT, RDATA_WF_BYTE, RDATA_WF_BYTE, RDATA_WF_BINARY },
+	  { RDATA_ZF_SHORT, RDATA_ZF_BYTE, RDATA_ZF_ALGORITHM,
+	    RDATA_ZF_BASE64 } },
 	/* 61 */
 	{ 61, NULL, T_UTYPE, 1, 1, { RDATA_WF_BINARY }, { RDATA_ZF_UNKNOWN } },
 	/* 62 */
diff --git a/usr.sbin/nsd/dns.h b/usr.sbin/nsd/dns.h
index 2b0734764ec..077f00d2f27 100644
--- a/usr.sbin/nsd/dns.h
+++ b/usr.sbin/nsd/dns.h
@@ -136,6 +136,8 @@ typedef enum nsd_rc nsd_rc_type;
 #define TYPE_NSEC3	50	/* NSEC3, secure denial, prevents zonewalking */
 #define TYPE_NSEC3PARAM 51	/* NSEC3PARAM at zone apex nsec3 parameters */
 #define TYPE_TLSA	52	/* RFC 6698 */
+#define TYPE_CDS	59	/* RFC 7344 */
+#define TYPE_CDNSKEY	60	/* RFC 7344 */
 
 #define TYPE_SPF        99      /* RFC 4408 */
 
diff --git a/usr.sbin/nsd/namedb.c b/usr.sbin/nsd/namedb.c
index 82801fb545a..b30e96b01c7 100644
--- a/usr.sbin/nsd/namedb.c
+++ b/usr.sbin/nsd/namedb.c
@@ -7,7 +7,7 @@
  *
  */
 
-#include <config.h>
+#include "config.h"
 
 #include <sys/types.h>
 
@@ -18,12 +18,12 @@
 #include <string.h>
 
 #include "namedb.h"
-
+#include "nsec3.h"
 
 static domain_type *
-allocate_domain_info(domain_table_type *table,
-		     const dname_type *dname,
-		     domain_type *parent)
+allocate_domain_info(domain_table_type* table,
+		     const dname_type* dname,
+		     domain_type* parent)
 {
 	domain_type *result;
 
@@ -33,62 +33,312 @@ allocate_domain_info(domain_table_type *table,
 
 	result = (domain_type *) region_alloc(table->region,
 					      sizeof(domain_type));
-	result->node.key = dname_partial_copy(
+	result->dname = dname_partial_copy(
 		table->region, dname, domain_dname(parent)->label_count + 1);
 	result->parent = parent;
 	result->wildcard_child_closest_match = result;
 	result->rrsets = NULL;
-	result->number = 0;
+	result->usage = 0;
 #ifdef NSEC3
-	result->nsec3_cover = NULL;
-	result->nsec3_wcard_child_cover = NULL;
-	result->nsec3_ds_parent_cover = NULL;
-	result->nsec3_lookup = NULL;
-	result->nsec3_is_exact = 0;
-	result->nsec3_ds_parent_is_exact = 0;
+	result->nsec3 = NULL;
 #endif
 	result->is_existing = 0;
 	result->is_apex = 0;
+	assert(table->numlist_last); /* it exists because root exists */
+	/* push this domain at the end of the numlist */
+	result->number = table->numlist_last->number+1;
+	result->numlist_next = NULL;
+	result->numlist_prev = table->numlist_last;
+	table->numlist_last->numlist_next = result;
+	table->numlist_last = result;
 
 	return result;
 }
 
+#ifdef NSEC3
+void
+allocate_domain_nsec3(domain_table_type* table, domain_type* result)
+{
+	if(result->nsec3)
+		return;
+	result->nsec3 = (struct nsec3_domain_data*) region_alloc(table->region,
+		sizeof(struct nsec3_domain_data));
+	result->nsec3->nsec3_cover = NULL;
+	result->nsec3->nsec3_wcard_child_cover = NULL;
+	result->nsec3->nsec3_ds_parent_cover = NULL;
+	result->nsec3->nsec3_is_exact = 0;
+	result->nsec3->nsec3_ds_parent_is_exact = 0;
+	result->nsec3->have_nsec3_hash = 0;
+	result->nsec3->have_nsec3_wc_hash = 0;
+	result->nsec3->have_nsec3_ds_parent_hash = 0;
+	result->nsec3->prehash_prev = NULL;
+	result->nsec3->prehash_next = NULL;
+	result->nsec3->nsec3_node.key = NULL;
+	result->nsec3->hash_node.key = NULL;
+	result->nsec3->wchash_node.key = NULL;
+	result->nsec3->dshash_node.key = NULL;
+}
+#endif /* NSEC3 */
+
+/** make the domain last in the numlist, changes numbers of domains */
+static void
+numlist_make_last(domain_table_type* table, domain_type* domain)
+{
+	size_t sw;
+	domain_type* last = table->numlist_last;
+	if(domain == last)
+		return;
+	/* swap numbers with the last element */
+	sw = domain->number;
+	domain->number = last->number;
+	last->number = sw;
+	/* swap list position with the last element */
+	assert(domain->numlist_next);
+	assert(last->numlist_prev);
+	if(domain->numlist_next != last) {
+		/* case 1: there are nodes between domain .. last */
+		domain_type* span_start = domain->numlist_next;
+		domain_type* span_end = last->numlist_prev;
+		/* these assignments walk the new list from start to end */
+		if(domain->numlist_prev)
+			domain->numlist_prev->numlist_next = last;
+		last->numlist_prev = domain->numlist_prev;
+		last->numlist_next = span_start;
+		span_start->numlist_prev = last;
+		span_end->numlist_next = domain;
+		domain->numlist_prev = span_end;
+		domain->numlist_next = NULL;
+	} else {
+		/* case 2: domain and last are neighbors */
+		/* these assignments walk the new list from start to end */
+		if(domain->numlist_prev)
+			domain->numlist_prev->numlist_next = last;
+		last->numlist_prev = domain->numlist_prev;
+		last->numlist_next = domain;
+		domain->numlist_prev = last;
+		domain->numlist_next = NULL;
+	}
+	table->numlist_last = domain;
+}
+
+/** pop the biggest domain off the numlist */
+static domain_type*
+numlist_pop_last(domain_table_type* table)
+{
+	domain_type* d = table->numlist_last;
+	table->numlist_last = table->numlist_last->numlist_prev;
+	if(table->numlist_last)
+		table->numlist_last->numlist_next = NULL;
+	return d;
+}
+
+/** see if a domain is eligible to be deleted, and thus is not used */
+static int
+domain_can_be_deleted(domain_type* domain)
+{
+	domain_type* n;
+	/* it has data or it has usage, do not delete it */
+	if(domain->rrsets) return 0;
+	if(domain->usage) return 0;
+	n = domain_next(domain);
+	/* it has children domains, do not delete it */
+	if(n && domain_is_subdomain(n, domain))
+		return 0;
+	return 1;
+}
+
+#ifdef NSEC3
+/** see if domain is on the prehash list */
+int domain_is_prehash(domain_table_type* table, domain_type* domain)
+{
+	if(domain->nsec3
+		&& (domain->nsec3->prehash_prev || domain->nsec3->prehash_next))
+		return 1;
+	return (table->prehash_list == domain);
+}
+
+/** remove domain node from NSEC3 tree in hash space */
+void
+zone_del_domain_in_hash_tree(rbtree_t* tree, rbnode_t* node)
+{
+	if(!node->key)
+		return;
+	rbtree_delete(tree, node->key);
+	/* note that domain is no longer in the tree */
+	node->key = NULL;
+}
+
+/** clear the prehash list */
+void prehash_clear(domain_table_type* table)
+{
+	domain_type* d = table->prehash_list, *n;
+	while(d) {
+		n = d->nsec3->prehash_next;
+		d->nsec3->prehash_prev = NULL;
+		d->nsec3->prehash_next = NULL;
+		d = n;
+	}
+	table->prehash_list = NULL;
+}
+
+/** add domain to prehash list */
+void
+prehash_add(domain_table_type* table, domain_type* domain)
+{
+	if(domain_is_prehash(table, domain))
+		return;
+	allocate_domain_nsec3(table, domain);
+	domain->nsec3->prehash_next = table->prehash_list;
+	if(table->prehash_list)
+		table->prehash_list->nsec3->prehash_prev = domain;
+	table->prehash_list = domain;
+}
+
+/** remove domain from prehash list */
+void
+prehash_del(domain_table_type* table, domain_type* domain)
+{
+	if(domain->nsec3->prehash_next)
+		domain->nsec3->prehash_next->nsec3->prehash_prev =
+			domain->nsec3->prehash_prev;
+	if(domain->nsec3->prehash_prev)
+		domain->nsec3->prehash_prev->nsec3->prehash_next =
+			domain->nsec3->prehash_next;
+	else	table->prehash_list = domain->nsec3->prehash_next;
+	domain->nsec3->prehash_next = NULL;
+	domain->nsec3->prehash_prev = NULL;
+}
+#endif /* NSEC3 */
+
+/** perform domain name deletion */
+static void
+do_deldomain(namedb_type* db, domain_type* domain)
+{
+	assert(domain && domain->parent); /* exists and not root */
+	/* first adjust the number list so that domain is the last one */
+	numlist_make_last(db->domains, domain);
+	/* pop off the domain from the number list */
+	(void)numlist_pop_last(db->domains);
+
+#ifdef NSEC3
+	/* if on prehash list, remove from prehash */
+	if(domain_is_prehash(db->domains, domain))
+		prehash_del(db->domains, domain);
+
+	/* see if nsec3-nodes are used */
+	if(domain->nsec3) {
+		if(domain->nsec3->nsec3_node.key)
+			zone_del_domain_in_hash_tree(nsec3_tree_zone(db, domain)
+				->nsec3tree, &domain->nsec3->nsec3_node);
+		if(domain->nsec3->hash_node.key)
+			zone_del_domain_in_hash_tree(nsec3_tree_zone(db, domain)
+				->hashtree, &domain->nsec3->hash_node);
+		if(domain->nsec3->wchash_node.key)
+			zone_del_domain_in_hash_tree(nsec3_tree_zone(db, domain)
+				->wchashtree, &domain->nsec3->wchash_node);
+		if(domain->nsec3->dshash_node.key)
+			zone_del_domain_in_hash_tree(nsec3_tree_dszone(db, domain)
+				->dshashtree, &domain->nsec3->dshash_node);
+		region_recycle(db->domains->region, domain->nsec3,
+			sizeof(struct nsec3_domain_data));
+	}
+#endif /* NSEC3 */
+
+	/* see if this domain is someones wildcard-child-closest-match,
+	 * which can only be the parent, and then it should use the
+	 * one-smaller than this domain as closest-match. */
+	if(domain->parent->wildcard_child_closest_match == domain)
+		domain->parent->wildcard_child_closest_match =
+			domain_previous_existing_child(domain);
+
+	/* actual removal */
+	radix_delete(db->domains->nametree, domain->rnode);
+	region_recycle(db->domains->region, (dname_type*)domain->dname,
+		dname_total_size(domain->dname));
+	region_recycle(db->domains->region, domain, sizeof(domain_type));
+}
+
+void
+domain_table_deldomain(namedb_type* db, domain_type* domain)
+{
+	while(domain_can_be_deleted(domain)) {
+		/* delete it */
+		do_deldomain(db, domain);
+		/* test parent */
+		domain = domain->parent;
+	}
+}
+
+/** clear hash tree */
+void
+hash_tree_clear(rbtree_t* tree)
+{
+	rbnode_t* n;
+	if(!tree) return;
+
+	/* note that elements are no longer in the tree */
+	for(n=rbtree_first(tree); n!=RBTREE_NULL; n=rbtree_next(n)) {
+		n->key = NULL;
+	}
+	tree->count = 0;
+	tree->root = RBTREE_NULL;
+}
+
+void hash_tree_delete(region_type* region, rbtree_t* tree)
+{
+	region_recycle(region, tree, sizeof(rbtree_t));
+}
+
+/** add domain nsec3 node to hashedspace tree */
+void zone_add_domain_in_hash_tree(region_type* region, rbtree_t** tree,
+	int (*cmpf)(const void*, const void*),
+	domain_type* domain, rbnode_t* node)
+{
+	if(!*tree)
+		*tree = rbtree_create(region, cmpf);
+	memset(node, 0, sizeof(rbnode_t));
+	node->key = domain;
+	rbtree_insert(*tree, node);
+}
+
 domain_table_type *
-domain_table_create(region_type *region)
+domain_table_create(region_type* region)
 {
-	const dname_type *origin;
-	domain_table_type *result;
-	domain_type *root;
+	const dname_type* origin;
+	domain_table_type* result;
+	domain_type* root;
 
 	assert(region);
 
 	origin = dname_make(region, (uint8_t *) "", 0);
 
 	root = (domain_type *) region_alloc(region, sizeof(domain_type));
-	root->node.key = origin;
+	root->dname = origin;
 	root->parent = NULL;
 	root->wildcard_child_closest_match = root;
 	root->rrsets = NULL;
 	root->number = 1; /* 0 is used for after header */
+	root->usage = 1; /* do not delete root, ever */
 	root->is_existing = 0;
 	root->is_apex = 0;
+	root->numlist_prev = NULL;
+	root->numlist_next = NULL;
 #ifdef NSEC3
-	root->nsec3_is_exact = 0;
-	root->nsec3_ds_parent_is_exact = 0;
-	root->nsec3_cover = NULL;
-	root->nsec3_wcard_child_cover = NULL;
-	root->nsec3_ds_parent_cover = NULL;
-	root->nsec3_lookup = NULL;
+	root->nsec3 = NULL;
 #endif
 
 	result = (domain_table_type *) region_alloc(region,
 						    sizeof(domain_table_type));
 	result->region = region;
-	result->names_to_domains = rbtree_create(
-		region, (int (*)(const void *, const void *)) dname_compare);
-	rbtree_insert(result->names_to_domains, (rbnode_t *) root);
+	result->nametree = radix_tree_create(region);
+	root->rnode = radname_insert(result->nametree, dname_name(root->dname),
+		root->dname->name_size, root);
 
 	result->root = root;
+	result->numlist_last = root;
+#ifdef NSEC3
+	result->prehash_list = NULL;
+#endif
 
 	return result;
 }
@@ -107,7 +357,9 @@ domain_table_search(domain_table_type *table,
 	assert(closest_match);
 	assert(closest_encloser);
 
-	exact = rbtree_find_less_equal(table->names_to_domains, dname, (rbnode_t **) closest_match);
+	exact = radname_find_less_equal(table->nametree, dname_name(dname),
+		dname->name_size, (struct radnode**)closest_match);
+	*closest_match = (domain_type*)((*(struct radnode**)closest_match)->elem);
 	assert(*closest_match);
 
 	*closest_encloser = *closest_match;
@@ -127,11 +379,11 @@ domain_table_search(domain_table_type *table,
 }
 
 domain_type *
-domain_table_find(domain_table_type *table,
-		  const dname_type *dname)
+domain_table_find(domain_table_type* table,
+		  const dname_type* dname)
 {
-	domain_type *closest_match;
-	domain_type *closest_encloser;
+	domain_type* closest_match;
+	domain_type* closest_encloser;
 	int exact;
 
 	exact = domain_table_search(
@@ -141,12 +393,12 @@ domain_table_find(domain_table_type *table,
 
 
 domain_type *
-domain_table_insert(domain_table_type *table,
-		    const dname_type  *dname)
+domain_table_insert(domain_table_type* table,
+		    const dname_type* dname)
 {
-	domain_type *closest_match;
-	domain_type *closest_encloser;
-	domain_type *result;
+	domain_type* closest_match;
+	domain_type* closest_encloser;
+	domain_type* result;
 	int exact;
 
 	assert(table);
@@ -164,8 +416,9 @@ domain_table_insert(domain_table_type *table,
 			result = allocate_domain_info(table,
 						      dname,
 						      closest_encloser);
-			rbtree_insert(table->names_to_domains, (rbnode_t *) result);
-			result->number = table->names_to_domains->count;
+			result->rnode = radname_insert(table->nametree,
+				dname_name(result->dname),
+				result->dname->name_size, result);
 
 			/*
 			 * If the newly added domain name is larger
@@ -191,26 +444,32 @@ domain_table_insert(domain_table_type *table,
 }
 
 int
-domain_table_iterate(domain_table_type *table,
+domain_table_iterate(domain_table_type* table,
 		    domain_table_iterator_type iterator,
-		    void *user_data)
+		    void* user_data)
 {
-	const void *dname;
-	void *node;
 	int error = 0;
-
-	assert(table);
-
-	RBTREE_WALK(table->names_to_domains, dname, node) {
-		error += iterator((domain_type *) node, user_data);
+	struct radnode* n;
+	for(n = radix_first(table->nametree); n; n = radix_next(n)) {
+		error += iterator((domain_type*)n->elem, user_data);
 	}
-
 	return error;
 }
 
+domain_type *domain_previous_existing_child(domain_type* domain)
+{
+	domain_type* parent = domain->parent;
+	domain = domain_previous(domain);
+	while(domain && !domain->is_existing) {
+		if(domain == parent) /* do not walk back above parent */
+			return parent;
+		domain = domain_previous(domain);
+	}
+	return domain;
+}
 
 void
-domain_add_rrset(domain_type *domain, rrset_type *rrset)
+domain_add_rrset(domain_type* domain, rrset_type* rrset)
 {
 #if 0 	/* fast */
 	rrset->next = domain->rrsets;
@@ -226,15 +485,24 @@ domain_add_rrset(domain_type *domain, rrset_type *rrset)
 
 	while (domain && !domain->is_existing) {
 		domain->is_existing = 1;
+		/* does this name in existance update the parent's
+		 * wildcard closest match? */
+		if(domain->parent
+		   && label_compare(dname_name(domain_dname(domain)),
+			(const uint8_t *) "\001*") <= 0
+		   && dname_compare(domain_dname(domain),
+		   	domain_dname(domain->parent->wildcard_child_closest_match)) > 0) {
+			domain->parent->wildcard_child_closest_match = domain;
+		}
 		domain = domain->parent;
 	}
 }
 
 
 rrset_type *
-domain_find_rrset(domain_type *domain, zone_type *zone, uint16_t type)
+domain_find_rrset(domain_type* domain, zone_type* zone, uint16_t type)
 {
-	rrset_type *result = domain->rrsets;
+	rrset_type* result = domain->rrsets;
 
 	while (result) {
 		if (result->zone == zone && rrset_rrtype(result) == type) {
@@ -246,9 +514,9 @@ domain_find_rrset(domain_type *domain, zone_type *zone, uint16_t type)
 }
 
 rrset_type *
-domain_find_any_rrset(domain_type *domain, zone_type *zone)
+domain_find_any_rrset(domain_type* domain, zone_type* zone)
 {
-	rrset_type *result = domain->rrsets;
+	rrset_type* result = domain->rrsets;
 
 	while (result) {
 		if (result->zone == zone) {
@@ -260,14 +528,17 @@ domain_find_any_rrset(domain_type *domain, zone_type *zone)
 }
 
 zone_type *
-domain_find_zone(domain_type *domain)
+domain_find_zone(namedb_type* db, domain_type* domain)
 {
-	rrset_type *rrset;
+	rrset_type* rrset;
 	while (domain) {
-		for (rrset = domain->rrsets; rrset; rrset = rrset->next) {
-			if (rrset_rrtype(rrset) == TYPE_SOA) {
-				return rrset->zone;
+		if(domain->is_apex) {
+			for (rrset = domain->rrsets; rrset; rrset = rrset->next) {
+				if (rrset_rrtype(rrset) == TYPE_SOA) {
+					return rrset->zone;
+				}
 			}
+			return namedb_find_zone(db, domain_dname(domain));
 		}
 		domain = domain->parent;
 	}
@@ -275,9 +546,9 @@ domain_find_zone(domain_type *domain)
 }
 
 zone_type *
-domain_find_parent_zone(zone_type *zone)
+domain_find_parent_zone(zone_type* zone)
 {
-	rrset_type *rrset;
+	rrset_type* rrset;
 
 	assert(zone);
 
@@ -290,7 +561,7 @@ domain_find_parent_zone(zone_type *zone)
 }
 
 domain_type *
-domain_find_ns_rrsets(domain_type *domain, zone_type *zone, rrset_type **ns)
+domain_find_ns_rrsets(domain_type* domain, zone_type* zone, rrset_type **ns)
 {
 	while (domain && domain != zone->apex) {
 		*ns = domain_find_rrset(domain, zone, TYPE_NS);
@@ -304,18 +575,18 @@ domain_find_ns_rrsets(domain_type *domain, zone_type *zone, rrset_type **ns)
 }
 
 int
-domain_is_glue(domain_type *domain, zone_type *zone)
+domain_is_glue(domain_type* domain, zone_type* zone)
 {
-	rrset_type *unused;
-	domain_type *ns_domain = domain_find_ns_rrsets(domain, zone, &unused);
+	rrset_type* unused;
+	domain_type* ns_domain = domain_find_ns_rrsets(domain, zone, &unused);
 	return (ns_domain != NULL &&
 		domain_find_rrset(ns_domain, zone, TYPE_SOA) == NULL);
 }
 
 domain_type *
-domain_wildcard_child(domain_type *domain)
+domain_wildcard_child(domain_type* domain)
 {
-	domain_type *wildcard_child;
+	domain_type* wildcard_child;
 
 	assert(domain);
 	assert(domain->wildcard_child_closest_match);
@@ -331,14 +602,14 @@ domain_wildcard_child(domain_type *domain)
 }
 
 int
-zone_is_secure(zone_type *zone)
+zone_is_secure(zone_type* zone)
 {
 	assert(zone);
 	return zone->is_secure;
 }
 
 uint16_t
-rr_rrsig_type_covered(rr_type *rr)
+rr_rrsig_type_covered(rr_type* rr)
 {
 	assert(rr->type == TYPE_RRSIG);
 	assert(rr->rdata_count > 0);
@@ -348,20 +619,16 @@ rr_rrsig_type_covered(rr_type *rr)
 }
 
 zone_type *
-namedb_find_zone(namedb_type *db, domain_type *domain)
+namedb_find_zone(namedb_type* db, const dname_type* dname)
 {
-	zone_type *zone;
-
-	for (zone = db->zones; zone; zone = zone->next) {
-		if (zone->apex == domain)
-			break;
-	}
-
-	return zone;
+	struct radnode* n = radname_search(db->zonetree, dname_name(dname),
+		dname->name_size);
+	if(n) return (zone_type*)n->elem;
+	return NULL;
 }
 
 rrset_type *
-domain_find_non_cname_rrset(domain_type *domain, zone_type *zone)
+domain_find_non_cname_rrset(domain_type* domain, zone_type* zone)
 {
 	/* find any rrset type that is not allowed next to a CNAME */
 	/* nothing is allowed next to a CNAME, except RRSIG, NSEC, NSEC3 */
@@ -381,3 +648,13 @@ domain_find_non_cname_rrset(domain_type *domain, zone_type *zone)
 	}
 	return NULL;
 }
+
+int
+namedb_lookup(struct namedb* db,
+	      const dname_type* dname,
+	      domain_type     **closest_match,
+	      domain_type     **closest_encloser)
+{
+	return domain_table_search(
+		db->domains, dname, closest_match, closest_encloser);
+}
diff --git a/usr.sbin/nsd/namedb.h b/usr.sbin/nsd/namedb.h
index c22a69034c8..34cf9e288bd 100644
--- a/usr.sbin/nsd/namedb.h
+++ b/usr.sbin/nsd/namedb.h
@@ -14,12 +14,13 @@
 
 #include "dname.h"
 #include "dns.h"
+#include "radtree.h"
 #include "rbtree.h"
 struct zone_options;
 struct nsd_options;
-
-#define	NAMEDB_MAGIC		"NSDdbV07"
-#define	NAMEDB_MAGIC_SIZE	8
+struct udb_base;
+struct udb_ptr;
+struct nsd;
 
 typedef union rdata_atom rdata_atom_type;
 typedef struct rrset rrset_type;
@@ -31,75 +32,111 @@ typedef struct rr rr_type;
 typedef struct domain_table domain_table_type;
 typedef struct domain domain_type;
 typedef struct zone zone_type;
+typedef struct namedb namedb_type;
 
 struct domain_table
 {
-	region_type *region;
-	rbtree_t      *names_to_domains;
-	domain_type *root;
+	region_type* region;
+	struct radtree *nametree;
+	domain_type* root;
+	/* ptr to biggest domain.number and last in list.
+	 * the root is the lowest and first in the list. */
+	domain_type *numlist_last;
+#ifdef NSEC3
+	/* the prehash list, start of the list */
+	domain_type* prehash_list;
+#endif /* NSEC3 */
 };
 
-struct domain
-{
-	rbnode_t     node;
-	domain_type *parent;
-	domain_type *wildcard_child_closest_match;
-	rrset_type  *rrsets;
 #ifdef NSEC3
+struct nsec3_domain_data {
 	/* (if nsec3 chain complete) always the covering nsec3 record */
-	domain_type *nsec3_cover;
+	domain_type* nsec3_cover;
 	/* the nsec3 that covers the wildcard child of this domain. */
-	domain_type *nsec3_wcard_child_cover;
+	domain_type* nsec3_wcard_child_cover;
 	/* for the DS case we must answer on the parent side of zone cut */
-	domain_type *nsec3_ds_parent_cover;
-	/* the NSEC3 domain that has a hash-base32 <= than this dname. */
-	/* or NULL (no smaller one within this zone)
-	 * this variable is used to look up the NSEC3 record that matches
-	 * or covers a given b64-encoded-hash-string domain name.
-	 * The result of the lookup is stored in the *_cover variables.
-	 * The variable makes it possible to perform a rbtree lookup for
-	 * a name, then take this 'jump' to the previous element that contains
-	 * an NSEC3 record, with hopefully the correct parameters. */
-	domain_type *nsec3_lookup;
+	domain_type* nsec3_ds_parent_cover;
+	/* NSEC3 domains to prehash, prev and next on the list or cleared */
+	domain_type* prehash_prev, *prehash_next;
+	/* entry in the nsec3tree (for NSEC3s in the chain in use) */
+	rbnode_t nsec3_node;
+	/* entry in the hashtree (for precompiled domains) */
+	rbnode_t hash_node;
+	/* entry in the wchashtree (the wildcard precompile) */
+	rbnode_t wchash_node;
+	/* entry in the dshashtree (the parent ds precompile) */
+	rbnode_t dshash_node;
+
+	/* nsec3 hash */
+	uint8_t nsec3_hash[NSEC3_HASH_LEN];
+	/* nsec3 hash of wildcard before this name */
+	uint8_t nsec3_wc_hash[NSEC3_HASH_LEN];
+	/* parent-side DS hash */
+	uint8_t nsec3_ds_parent_hash[NSEC3_HASH_LEN];
+	/* if the nsec3 has is available */
+	unsigned  have_nsec3_hash : 1;
+	unsigned  have_nsec3_wc_hash : 1;
+	unsigned  have_nsec3_ds_parent_hash : 1;
+	/* if the domain has an NSEC3 for it, use cover ptr to get it. */
+	unsigned     nsec3_is_exact : 1;
+	/* same but on parent side */
+	unsigned     nsec3_ds_parent_is_exact : 1;
+};
+#endif /* NSEC3 */
+
+struct domain
+{
+	struct radnode* rnode;
+	const dname_type* dname;
+	domain_type* parent;
+	domain_type* wildcard_child_closest_match;
+	rrset_type* rrsets;
+#ifdef NSEC3
+	struct nsec3_domain_data* nsec3;
 #endif
-	uint32_t     number; /* Unique domain name number.  */
+	/* double-linked list sorted by domain.number */
+	domain_type* numlist_prev, *numlist_next;
+	size_t     number; /* Unique domain name number.  */
+	size_t     usage; /* number of ptrs to this from RRs(in rdata) and
+			     from zone-apex pointers, also the root has one
+			     more to make sure it cannot be deleted. */
 
 	/*
 	 * This domain name exists (see wildcard clarification draft).
 	 */
 	unsigned     is_existing : 1;
 	unsigned     is_apex : 1;
-#ifdef NSEC3
-	/* if the domain has an NSEC3 for it, use cover ptr to get it. */
-	unsigned     nsec3_is_exact : 1;
-	/* same but on parent side */
-	unsigned     nsec3_ds_parent_is_exact : 1;
-#endif
 };
 
 struct zone
 {
-	zone_type   *next;
-	domain_type *apex;
-	rrset_type  *soa_rrset;
-	rrset_type  *soa_nx_rrset; /* see bug #103 */
-	rrset_type  *ns_rrset;
+	struct radnode *node; /* this entry in zonetree */
+	domain_type* apex;
+	rrset_type*  soa_rrset;
+	rrset_type*  soa_nx_rrset; /* see bug #103 */
+	rrset_type*  ns_rrset;
 #ifdef NSEC3
-	rr_type	    *nsec3_soa_rr; /* rrset with SOA bit set */
-	domain_type *nsec3_last; /* last domain with nsec3, wraps */
+	rr_type* nsec3_param; /* NSEC3PARAM RR of chain in use or NULL */
+	domain_type* nsec3_last; /* last domain with nsec3, wraps */
+	/* in these trees, the root contains an elem ptr to the radtree* */
+	rbtree_t* nsec3tree; /* tree with relevant NSEC3 domains */
+	rbtree_t* hashtree; /* tree, hashed NSEC3precompiled domains */
+	rbtree_t* wchashtree; /* tree, wildcard hashed domains */
+	rbtree_t* dshashtree; /* tree, ds-parent-hash domains */
 #endif
-	struct zone_options *opts;
-	uint32_t     number;
-	uint8_t*     dirty; /* array of dirty-flags, per child */
+	struct zone_options* opts;
+	char*        filename; /* set if read from file, which file */
+	char*        logstr; /* set for zone xfer, the log string */
+	time_t       mtime; /* time of last modification */
 	unsigned     is_secure : 1; /* zone uses DNSSEC */
-	unsigned     updated : 1; /* zone SOA was updated */
 	unsigned     is_ok : 1; /* zone has not expired. */
+	unsigned     is_changed : 1; /* zone was changed by AXFR */
 };
 
 /* a RR in DNS */
 struct rr {
-	domain_type     *owner;
-	rdata_atom_type *rdatas;
+	domain_type*     owner;
+	rdata_atom_type* rdatas;
 	uint32_t         ttl;
 	uint16_t         type;
 	uint16_t         klass;
@@ -112,9 +149,9 @@ struct rr {
  */
 struct rrset
 {
-	rrset_type *next;
-	zone_type  *zone;
-	rr_type    *rrs;
+	rrset_type* next;
+	zone_type*  zone;
+	rr_type*    rrs;
 	uint16_t    rr_count;
 };
 
@@ -126,10 +163,10 @@ struct rrset
 union rdata_atom
 {
 	/* RDATA_WF_COMPRESSED_DNAME, RDATA_WF_UNCOMPRESSED_DNAME */
-	domain_type *domain;
+	domain_type* domain;
 
 	/* Default. */
-	uint16_t    *data;
+	uint16_t*    data;
 };
 
 /*
@@ -140,8 +177,8 @@ domain_table_type *domain_table_create(region_type *region);
 /*
  * Search the domain table for a match and the closest encloser.
  */
-int domain_table_search(domain_table_type *table,
-			const dname_type  *dname,
+int domain_table_search(domain_table_type* table,
+			const dname_type* dname,
 			domain_type      **closest_match,
 			domain_type      **closest_encloser);
 
@@ -150,17 +187,17 @@ int domain_table_search(domain_table_type *table,
  * root domain).
  */
 static inline uint32_t
-domain_table_count(domain_table_type *table)
+domain_table_count(domain_table_type* table)
 {
-	return table->names_to_domains->count;
+	return table->nametree->count;
 }
 
 /*
  * Find the specified dname in the domain_table.  NULL is returned if
  * there is no exact match.
  */
-domain_type *domain_table_find(domain_table_type *table,
-			       const dname_type  *dname);
+domain_type* domain_table_find(domain_table_type* table,
+			       const dname_type* dname);
 
 /*
  * Insert a domain name in the domain table.  If the domain name is
@@ -172,6 +209,17 @@ domain_type *domain_table_find(domain_table_type *table,
 domain_type *domain_table_insert(domain_table_type *table,
 				 const dname_type  *dname);
 
+/* put domain into nsec3 hash space tree */
+void zone_add_domain_in_hash_tree(region_type* region, rbtree_t** tree,
+	int (*cmpf)(const void*, const void*), domain_type* domain,
+	rbnode_t* node);
+void zone_del_domain_in_hash_tree(rbtree_t* tree, rbnode_t* node);
+void hash_tree_clear(rbtree_t* tree);
+void hash_tree_delete(region_type* region, rbtree_t* tree);
+void prehash_clear(domain_table_type* table);
+void prehash_add(domain_table_type* table, domain_type* domain);
+void prehash_del(domain_table_type* table, domain_type* domain);
+int domain_is_prehash(domain_table_type* table, domain_type* domain);
 
 /*
  * Iterate over all the domain names in the domain tree.
@@ -179,77 +227,80 @@ domain_type *domain_table_insert(domain_table_type *table,
 typedef int (*domain_table_iterator_type)(domain_type *node,
 					   void *user_data);
 
-int domain_table_iterate(domain_table_type *table,
+int domain_table_iterate(domain_table_type* table,
 			  domain_table_iterator_type iterator,
-			  void *user_data);
+			  void* user_data);
 
 /*
  * Add an RRset to the specified domain.  Updates the is_existing flag
  * as required.
  */
-void domain_add_rrset(domain_type *domain, rrset_type *rrset);
+void domain_add_rrset(domain_type* domain, rrset_type* rrset);
 
-rrset_type *domain_find_rrset(domain_type *domain, zone_type *zone, uint16_t type);
-rrset_type *domain_find_any_rrset(domain_type *domain, zone_type *zone);
+rrset_type* domain_find_rrset(domain_type* domain, zone_type* zone, uint16_t type);
+rrset_type* domain_find_any_rrset(domain_type* domain, zone_type* zone);
 
-zone_type *domain_find_zone(domain_type *domain);
-zone_type *domain_find_parent_zone(zone_type *zone);
+zone_type* domain_find_zone(namedb_type* db, domain_type* domain);
+zone_type* domain_find_parent_zone(zone_type* zone);
 
-domain_type *domain_find_ns_rrsets(domain_type *domain, zone_type *zone, rrset_type **ns);
+domain_type* domain_find_ns_rrsets(domain_type* domain, zone_type* zone, rrset_type **ns);
 
-int domain_is_glue(domain_type *domain, zone_type *zone);
+int domain_is_glue(domain_type* domain, zone_type* zone);
 
-rrset_type *domain_find_non_cname_rrset(domain_type *domain, zone_type *zone);
+rrset_type* domain_find_non_cname_rrset(domain_type* domain, zone_type* zone);
 
-domain_type *domain_wildcard_child(domain_type *domain);
+domain_type* domain_wildcard_child(domain_type* domain);
+domain_type *domain_previous_existing_child(domain_type* domain);
 
-int zone_is_secure(zone_type *zone);
+int zone_is_secure(zone_type* zone);
 
 static inline const dname_type *
-domain_dname(domain_type *domain)
+domain_dname(domain_type* domain)
 {
-	return (const dname_type *) domain->node.key;
+	return domain->dname;
 }
 
 static inline domain_type *
-domain_previous(domain_type *domain)
+domain_previous(domain_type* domain)
 {
-	rbnode_t *prev = rbtree_previous((rbnode_t *) domain);
-	return prev == RBTREE_NULL ? NULL : (domain_type *) prev;
+	struct radnode* prev = radix_prev(domain->rnode);
+	return prev == NULL ? NULL : (domain_type*)prev->elem;
 }
 
 static inline domain_type *
-domain_next(domain_type *domain)
+domain_next(domain_type* domain)
 {
-	rbnode_t *next = rbtree_next((rbnode_t *) domain);
-	return next == RBTREE_NULL ? NULL : (domain_type *) next;
+	struct radnode* next = radix_next(domain->rnode);
+	return next == NULL ? NULL : (domain_type*)next->elem;
 }
 
+/* easy comparison for subdomain, true if d1 is subdomain of d2. */
+static inline int domain_is_subdomain(domain_type* d1, domain_type* d2)
+{ return dname_is_subdomain(domain_dname(d1), domain_dname(d2)); }
+/* easy printout, to static buffer of dname_to_string, fqdn. */
+static inline const char* domain_to_string(domain_type* domain)
+{ return dname_to_string(domain_dname(domain), NULL); }
+
 /*
  * The type covered by the signature in the specified RRSIG RR.
  */
-uint16_t rr_rrsig_type_covered(rr_type *rr);
+uint16_t rr_rrsig_type_covered(rr_type* rr);
 
-typedef struct namedb namedb_type;
 struct namedb
 {
-	region_type       *region;
-	domain_table_type *domains;
-	zone_type         *zones;
-	size_t	  	  zone_count;
-	char              *filename;
-	FILE              *fd;
+	region_type*       region;
+	domain_table_type* domains;
+	struct radtree*    zonetree;
+	struct udb_base*   udb;
 	/* the timestamp on the ixfr.db file */
 	struct timeval	  diff_timestamp;
-	/* the CRC on the nsd.db file and position of CRC in the db file */
-	uint32_t	  crc;
-	off_t		  crc_pos;
 	/* if diff_skip=1, diff_pos contains the nsd.diff place to continue */
 	uint8_t		  diff_skip;
 	off_t		  diff_pos;
 };
 
 static inline int rdata_atom_is_domain(uint16_t type, size_t index);
+static inline int rdata_atom_is_literal_domain(uint16_t type, size_t index);
 
 static inline domain_type *
 rdata_atom_domain(rdata_atom_type atom)
@@ -270,26 +321,49 @@ rdata_atom_data(rdata_atom_type atom)
 }
 
 
+/* Find the zone for the specified dname in DB. */
+zone_type *namedb_find_zone(namedb_type *db, const dname_type *dname);
 /*
- * Find the zone for the specified DOMAIN in DB.
+ * Delete a domain name from the domain table.  Removes dname_info node.
+ * Only deletes if usage is 0, has no rrsets and no children.  Checks parents
+ * for deletion as well.  Adjusts numberlist(domain.number), and 
+ * wcard_child closest match.
  */
-zone_type *namedb_find_zone(namedb_type *db, domain_type *domain);
-
-/* dbcreate.c */
-struct namedb *namedb_new(const char *filename);
-int namedb_save(struct namedb *db);
-void namedb_discard(struct namedb *db);
+void domain_table_deldomain(namedb_type* db, domain_type* domain);
 
 
+/** dbcreate.c */
+int udb_write_rr(struct udb_base* udb, struct udb_ptr* z, rr_type* rr);
+void udb_del_rr(struct udb_base* udb, struct udb_ptr* z, rr_type* rr);
+int write_zone_to_udb(struct udb_base* udb, zone_type* zone, time_t mtime,
+	const char* file_str);
+/** marshal rdata into buffer, must be MAX_RDLENGTH in size */
+size_t rr_marshal_rdata(rr_type* rr, uint8_t* rdata, size_t sz);
 /* dbaccess.c */
-int namedb_lookup (struct namedb    *db,
-		   const dname_type *dname,
+int namedb_lookup (struct namedb* db,
+		   const dname_type* dname,
 		   domain_type     **closest_match,
 		   domain_type     **closest_encloser);
 /* pass number of children (to alloc in dirty array */
-struct namedb *namedb_open(const char *filename, struct nsd_options* opt,
-	size_t num_children);
-void namedb_close(struct namedb *db);
+struct namedb *namedb_open(const char *filename, struct nsd_options* opt);
+void namedb_close_udb(struct namedb* db);
+void namedb_close(struct namedb* db);
+void namedb_check_zonefiles(struct nsd* nsd, struct nsd_options* opt,
+	struct udb_base* taskudb, struct udb_ptr* last_task);
+void namedb_check_zonefile(struct nsd* nsd, struct udb_base* taskudb,
+	struct udb_ptr* last_task, struct zone_options* zo);
+/** zone one zonefile into memory and revert on parse error, write to udb */
+void namedb_read_zonefile(struct nsd* nsd, struct zone* zone,
+	struct udb_base* taskudb, struct udb_ptr* last_task);
+void apex_rrset_checks(struct namedb* db, rrset_type* rrset,
+	domain_type* domain);
+zone_type* namedb_zone_create(namedb_type* db, const dname_type* dname,
+        struct zone_options* zopt);
+void namedb_zone_delete(namedb_type* db, zone_type* zone);
+void namedb_write_zonefile(struct nsd* nsd, struct zone_options* zopt);
+void namedb_write_zonefiles(struct nsd* nsd, struct nsd_options* options);
+int create_dirs(const char* path);
+void allocate_domain_nsec3(domain_table_type *table, domain_type *result);
 
 static inline int
 rdata_atom_is_domain(uint16_t type, size_t index)
@@ -301,6 +375,15 @@ rdata_atom_is_domain(uint16_t type, size_t index)
 		    || descriptor->wireformat[index] == RDATA_WF_UNCOMPRESSED_DNAME));
 }
 
+static inline int
+rdata_atom_is_literal_domain(uint16_t type, size_t index)
+{
+	const rrtype_descriptor_type *descriptor
+		= rrtype_descriptor_by_type(type);
+	return (index < descriptor->maximum
+		&& (descriptor->wireformat[index] == RDATA_WF_LITERAL_DNAME));
+}
+
 static inline rdata_wireformat_type
 rdata_atom_wireformat_type(uint16_t type, size_t index)
 {
@@ -311,7 +394,7 @@ rdata_atom_wireformat_type(uint16_t type, size_t index)
 }
 
 static inline uint16_t
-rrset_rrtype(rrset_type *rrset)
+rrset_rrtype(rrset_type* rrset)
 {
 	assert(rrset);
 	assert(rrset->rr_count > 0);
@@ -319,12 +402,11 @@ rrset_rrtype(rrset_type *rrset)
 }
 
 static inline uint16_t
-rrset_rrclass(rrset_type *rrset)
+rrset_rrclass(rrset_type* rrset)
 {
 	assert(rrset);
 	assert(rrset->rr_count > 0);
 	return rrset->rrs[0].klass;
 }
 
-
 #endif
diff --git a/usr.sbin/nsd/nsd-control.8.in b/usr.sbin/nsd/nsd-control.8.in
index bf610f1097b..f5aa2cbe4e4 100644
--- a/usr.sbin/nsd/nsd-control.8.in
+++ b/usr.sbin/nsd/nsd-control.8.in
@@ -1,8 +1,7 @@
-.TH "nsd\-control" "8" "Oct 29, 2013" "NLnet Labs" "nsd 4.0.0"
+.TH "nsd\-control" "8" "Sep  4, 2014" "NLnet Labs" "nsd 4.1.0"
 .\" Copyright (c) 2011, NLnet Labs. All rights reserved.
 .\" See LICENSE for the license.
 .SH "NAME"
-.LP
 .B nsd\-control,
 .B nsd\-control\-setup
 \- NSD remote server control utility.
@@ -109,7 +108,9 @@ Attempt to update slave zones that are hosted on this server by contacting
 the masters.  The masters are configured via 'request\-xfr:' lists.
 If a zone is given, that zone is updated.  Usually NSD receives a NOTIFY
 from the masters (configured via 'allow\-notify:' acl list) that a new zone
-serial has to be transferred.
+serial has to be transferred.  For zones with no content, NSD may have backed
+off from asking often because the masters did not respond, but this command
+will reset the backoff to its initial timeout, for frequent retries.
 .TP
 .B force_transfer [<zone>]
 Force update slave zones that are hosted on this server.  Even if the
@@ -121,6 +122,12 @@ increases, use the 'transfer' command.
 Print state of the zone, the serial numbers and since when they have
 been acquired.  Also prints the notify action (to which server), and
 zone transfer (and from which master) if there is activity right now.
+The state of the zone is printed as: 'master' (master zones), 'ok' (slave
+zone is up\-to\-date), 'expired' (slave zone has expired), 'refreshing' (slave
+zone has transfers active).  The serial numbers printed are
+the 'served\-serial' (currently active), the 'commit\-serial' (is in reload),
+the 'notified\-serial' (got notify, busy fetching the data).  The serial
+numbers are only printed if such a serial number is available.
 .TP
 .B serverpid
 Prints the PID of the server process.  This is used for statistics (and
diff --git a/usr.sbin/nsd/query.c b/usr.sbin/nsd/query.c
index d5afe789cf0..f8cb1b496c0 100644
--- a/usr.sbin/nsd/query.c
+++ b/usr.sbin/nsd/query.c
@@ -528,6 +528,8 @@ answer_chaos(struct nsd *nsd, query_type *q)
 			} else {
 				RCODE_SET(q->packet, RCODE_REFUSE);
 			}
+		} else {
+			RCODE_SET(q->packet, RCODE_REFUSE);
 		}
 		break;
 	default:
diff --git a/usr.sbin/nsd/remote.c b/usr.sbin/nsd/remote.c
index 759136c9645..82b49990216 100644
--- a/usr.sbin/nsd/remote.c
+++ b/usr.sbin/nsd/remote.c
@@ -567,10 +567,16 @@ remote_accept_callback(int fd, short event, void* arg)
 
 	event_set(&n->c, newfd, EV_PERSIST|EV_TIMEOUT|EV_READ,
 		remote_control_callback, n);
-	if(event_base_set(xfrd->event_base, &n->c) != 0)
+	if(event_base_set(xfrd->event_base, &n->c) != 0) {
 		log_msg(LOG_ERR, "remote_accept: cannot set event_base");
-	if(event_add(&n->c, &n->tval) != 0)
+		free(n);
+		goto close_exit;
+	}
+	if(event_add(&n->c, &n->tval) != 0) {
 		log_msg(LOG_ERR, "remote_accept: cannot add event");
+		free(n);
+		goto close_exit;
+	}
 	n->event_added = 1;
 
 	if(2 <= verbosity) {
diff --git a/usr.sbin/nsd/server.c b/usr.sbin/nsd/server.c
index 22cfff7897c..0c8ca29c754 100644
--- a/usr.sbin/nsd/server.c
+++ b/usr.sbin/nsd/server.c
@@ -2522,10 +2522,18 @@ handle_tcp_accept(int fd, short event, void* arg)
 
 	event_set(&tcp_data->event, s, EV_PERSIST | EV_READ | EV_TIMEOUT,
 		handle_tcp_reading, tcp_data);
-	if(event_base_set(data->event.ev_base, &tcp_data->event) != 0)
-		log_msg(LOG_ERR, "cannot set tcp event base");
-	if(event_add(&tcp_data->event, &timeout) != 0)
+	if(event_base_set(data->event.ev_base, &tcp_data->event) != 0) {
 		log_msg(LOG_ERR, "cannot set tcp event base");
+		close(s);
+		region_destroy(tcp_region);
+		return;
+	}
+	if(event_add(&tcp_data->event, &timeout) != 0) {
+		log_msg(LOG_ERR, "cannot add tcp to event base");
+		close(s);
+		region_destroy(tcp_region);
+		return;
+	}
 
 	/*
 	 * Keep track of the total number of TCP handlers installed so
diff --git a/usr.sbin/nsd/xfrd-disk.c b/usr.sbin/nsd/xfrd-disk.c
index a095bbfacf9..a2f3e8af01d 100644
--- a/usr.sbin/nsd/xfrd-disk.c
+++ b/usr.sbin/nsd/xfrd-disk.c
@@ -308,6 +308,10 @@ xfrd_read_state(struct xfrd_state* xfrd)
 		 * contents trumps the contents of this cache */
 		/* zone->soa_disk_acquired = soa_disk_acquired_read; */
 		zone->soa_notified_acquired = soa_notified_acquired_read;
+		if (zone->state == xfrd_zone_expired)
+		{
+			xfrd_send_expire_notification(zone);
+		}
 		xfrd_handle_incoming_soa(zone, &incoming_soa, incoming_acquired);
 	}
 
diff --git a/usr.sbin/nsd/xfrd.c b/usr.sbin/nsd/xfrd.c
index 0ea9ea651d3..54b9650d458 100644
--- a/usr.sbin/nsd/xfrd.c
+++ b/usr.sbin/nsd/xfrd.c
@@ -76,8 +76,6 @@ static void xfrd_handle_reload(int fd, short event, void* arg);
 /* handle child timeout */
 static void xfrd_handle_child_timer(int fd, short event, void* arg);
 
-/* send expiry notifications to nsd */
-static void xfrd_send_expire_notification(xfrd_zone_t* zone);
 /* send ixfr request, returns fd of connection to read on */
 static int xfrd_send_ixfr_request_udp(xfrd_zone_t* zone);
 /* obtain udp socket slot */
@@ -1147,7 +1145,7 @@ xfrd_handle_incoming_soa(xfrd_zone_t* zone,
 	xfrd_send_notify(xfrd->notify_zones, zone->apex, &zone->soa_nsd);
 }
 
-static void
+void
 xfrd_send_expire_notification(xfrd_zone_t* zone)
 {
 	task_new_expire(xfrd->nsd->task[xfrd->nsd->mytask], xfrd->last_task,
@@ -2087,6 +2085,11 @@ xfrd_handle_notify_and_start_xfr(xfrd_zone_t* zone, xfrd_soa_t* soa)
 			!zone->tcp_waiting && !zone->udp_waiting) {
 			xfrd_set_refresh_now(zone);
 		}
+		/* zones with no content start expbackoff again; this is also
+		 * for nsd-control started transfer commands, and also when
+		 * the master apparantly sends notifies (is back up) */
+		if(zone->soa_disk_acquired == 0)
+			zone->fresh_xfr_timeout = XFRD_TRANSFER_TIMEOUT_START;
 	}
 }
 
diff --git a/usr.sbin/nsd/xfrd.h b/usr.sbin/nsd/xfrd.h
index 9b34322b67b..e29a0d52e97 100644
--- a/usr.sbin/nsd/xfrd.h
+++ b/usr.sbin/nsd/xfrd.h
@@ -10,8 +10,17 @@
 #ifndef XFRD_H
 #define XFRD_H
 
-#include <config.h>
-#include "netio.h"
+#ifndef USE_MINI_EVENT
+#  ifdef HAVE_EVENT_H
+#    include <event.h>
+#  else
+#    include <event2/event.h>
+#    include "event2/event_struct.h"
+#    include "event2/event_compat.h"
+#  endif
+#else
+#  include "mini_event.h"
+#endif
 #include "rbtree.h"
 #include "namedb.h"
 #include "options.h"
@@ -24,6 +33,7 @@ struct buffer;
 struct xfrd_tcp;
 struct xfrd_tcp_set;
 struct notify_zone_t;
+struct udb_ptr;
 typedef struct xfrd_state xfrd_state_t;
 typedef struct xfrd_zone xfrd_zone_t;
 typedef struct xfrd_soa xfrd_soa_t;
@@ -35,41 +45,55 @@ struct xfrd_state {
 	/* time when daemon was last started */
 	time_t xfrd_start_time;
 	struct region* region;
-	netio_type* netio;
+	struct event_base* event_base;
 	struct nsd* nsd;
 
 	struct xfrd_tcp_set* tcp_set;
 	/* packet buffer for udp packets */
 	struct buffer* packet;
-	/* udp waiting list */
+	/* udp waiting list, double linked list */
 	struct xfrd_zone *udp_waiting_first, *udp_waiting_last;
 	/* number of udp sockets (for sending queries) in use */
 	size_t udp_use_num;
+	/* activated waiting list, double linked list */
+	struct xfrd_zone *activated_first;
 
 	/* current time is cached */
 	uint8_t got_time;
 	time_t current_time;
 
+	/* counter for xfr file numbers */
+	uint64_t xfrfilenumber;
+
 	/* timer for NSD reload */
-	struct timespec reload_timeout;
-	netio_handler_type reload_handler;
+	struct timeval reload_timeout;
+	struct event reload_handler;
+	int reload_added;
 	/* last reload must have caught all zone updates before this time */
 	time_t reload_cmd_last_sent;
 	uint8_t can_send_reload;
+	pid_t reload_pid;
+	/* timeout for lost sigchild and reaping children */
+	struct event child_timer;
+	int child_timer_added;
+
+	/* timeout event for zonefiles_write events */
+	struct event write_timer;
+	/* set to 1 if zones have received xfrs since the last write_timer */
+	int write_zonefile_needed;
 
 	/* communication channel with server_main */
-	netio_handler_type ipc_handler;
-	uint8_t ipc_is_soa;
-	uint8_t parent_soa_info_pass;
+	struct event ipc_handler;
+	int ipc_handler_flags;
 	struct xfrd_tcp *ipc_conn;
 	struct buffer* ipc_pass;
 	/* sending ipc to server_main */
-	struct xfrd_tcp *ipc_conn_write;
+	uint8_t need_to_send_shutdown;
 	uint8_t need_to_send_reload;
+	uint8_t need_to_send_stats;
 	uint8_t need_to_send_quit;
-	uint8_t sending_zone_state;
 	uint8_t	ipc_send_blocked;
-	stack_type* dirty_zones; /* stack of xfrd_zone* */
+	struct udb_ptr* last_task;
 
 	/* xfrd shutdown flag */
 	uint8_t shutdown;
@@ -138,10 +162,6 @@ struct xfrd_zone {
 		xfrd_zone_expired
 	} state;
 
-	/* if state is dirty it needs to be sent to server_main.
-	 * it is also on the dirty_stack. Not saved on disk. */
-	uint8_t dirty;
-
 	/* master to try to transfer from, number for persistence */
 	acl_options_t* master;
 	int master_num;
@@ -152,8 +172,10 @@ struct xfrd_zone {
 	int fresh_xfr_timeout;
 
 	/* handler for timeouts */
-	struct timespec timeout;
-	netio_handler_type zone_handler;
+	struct timeval timeout;
+	struct event zone_handler;
+	int zone_handler_flags;
+	int event_added;
 
 	/* tcp connection zone is using, or -1 */
 	int tcp_conn;
@@ -161,10 +183,22 @@ struct xfrd_zone {
 	uint8_t tcp_waiting;
 	/* next zone in waiting list */
 	xfrd_zone_t* tcp_waiting_next;
+	xfrd_zone_t* tcp_waiting_prev;
+	/* zone is in its tcp send queue */
+	uint8_t in_tcp_send;
+	/* next zone in tcp send queue */
+	xfrd_zone_t* tcp_send_next;
+	xfrd_zone_t* tcp_send_prev;
 	/* zone is waiting for a udp connection (tcp is preferred) */
 	uint8_t udp_waiting;
 	/* next zone in waiting list for UDP */
 	xfrd_zone_t* udp_waiting_next;
+	xfrd_zone_t* udp_waiting_prev;
+	/* zone has been activated to run now (after the other events
+	 * but before blocking in select again) */
+	uint8_t is_activated;
+	xfrd_zone_t* activated_next;
+	xfrd_zone_t* activated_prev;
 
 	/* xfr message handling data */
 	/* query id */
@@ -173,13 +207,14 @@ struct xfrd_zone {
 	uint32_t msg_old_serial, msg_new_serial; /* host byte order */
 	size_t msg_rr_count;
 	uint8_t msg_is_ixfr; /* 1:IXFR detected. 2:middle IXFR SOA seen. */
-#ifdef TSIG
 	tsig_record_type tsig; /* tsig state for IXFR/AXFR */
-#endif
+	uint64_t xfrfilenumber; /* identifier for file to store xfr into,
+				valid if msg_seq_nr nonzero */
 };
 
 enum xfrd_packet_result {
 	xfrd_packet_bad, /* drop the packet/connection */
+	xfrd_packet_drop, /* drop the connection, but not report bad */
 	xfrd_packet_more, /* more packets to follow on tcp */
 	xfrd_packet_notimpl, /* server responded with NOTIMPL or FORMATERR */
 	xfrd_packet_tcp, /* try tcp connection */
@@ -195,18 +230,25 @@ enum xfrd_packet_result {
    Note that also some sockets are used for writing the ixfr.db, xfrd.state
    files and for the pipes to the main parent process.
 */
-#define XFRD_MAX_TCP 50 /* max number of TCP AXFR/IXFR concurrent connections.*/
+#define XFRD_MAX_TCP 32 /* max number of TCP AXFR/IXFR concurrent connections.*/
 			/* Each entry has 64Kb buffer preallocated.*/
-#define XFRD_MAX_UDP 100 /* max number of UDP sockets at a time for IXFR */
-#define XFRD_MAX_UDP_NOTIFY 50 /* max concurrent UDP sockets for NOTIFY */
+#define XFRD_MAX_UDP 64 /* max number of UDP sockets at a time for IXFR */
+#define XFRD_MAX_UDP_NOTIFY 64 /* max concurrent UDP sockets for NOTIFY */
 
 extern xfrd_state_t* xfrd;
 
 /* start xfrd, new start. Pass socket to server_main. */
-void xfrd_init(int socket, struct nsd* nsd);
+void xfrd_init(int socket, struct nsd* nsd, int shortsoa, int reload_active,
+	pid_t nsd_pid);
+
+/* add new slave zone, dname(from zone_opt) and given options */
+void xfrd_init_slave_zone(xfrd_state_t* xfrd, zone_options_t* zone_opt);
+
+/* delete slave zone */
+void xfrd_del_slave_zone(xfrd_state_t* xfrd, const dname_type* dname);
 
 /* get the current time epoch. Cached for speed. */
-time_t xfrd_time();
+time_t xfrd_time(void);
 
 /*
  * Handle final received packet from network.
@@ -221,6 +263,8 @@ void xfrd_set_timer(xfrd_zone_t* zone, time_t t);
 void xfrd_set_refresh_now(xfrd_zone_t* zone);
 /* unset the timer - no more timeouts, for when zone is queued */
 void xfrd_unset_timer(xfrd_zone_t* zone);
+/* remove the 'refresh now', remove it from the activated list */
+void xfrd_deactivate_zone(xfrd_zone_t* z);
 
 /*
  * Make a new request to next master server.
@@ -250,15 +294,13 @@ void xfrd_udp_release(xfrd_zone_t* zone);
 /*
  * Get a static buffer for temporary use (to build a packet).
  */
-struct buffer* xfrd_get_temp_buffer();
+struct buffer* xfrd_get_temp_buffer(void);
 
 /*
  * TSIG sign outgoing request. Call if acl has a key.
  */
-#ifdef TSIG
 void xfrd_tsig_sign_request(buffer_type* packet, struct tsig_record* tsig,
         acl_options_t* acl);
-#endif
 
 /* handle incoming soa information (NSD is running it, time acquired=guess).
    Pass soa=NULL,acquired=now if NSD has nothing loaded for the zone
@@ -266,30 +308,49 @@ void xfrd_tsig_sign_request(buffer_type* packet, struct tsig_record* tsig,
 void xfrd_handle_incoming_soa(xfrd_zone_t* zone, xfrd_soa_t* soa,
 	time_t acquired);
 /* handle a packet passed along ipc route. acl is the one that accepted
-   the packet. The packet is the network blob received. */
-void xfrd_handle_passed_packet(buffer_type* packet, int acl_num);
-
-/* send expiry notify for all zones to nsd (sets all dirty). */
-void xfrd_send_expy_all_zones();
+   the packet. The packet is the network blob received. acl_xfr is 
+   provide-xfr acl matching notify sender or -1 */
+void xfrd_handle_passed_packet(buffer_type* packet,
+	int acl_num, int acl_xfr);
 
 /* try to reopen the logfile. */
-void xfrd_reopen_logfile();
+void xfrd_reopen_logfile(void);
+
+/* free namedb for xfrd usage */
+void xfrd_free_namedb(struct nsd* nsd);
 
 /* copy SOA info from rr to soa struct. */
 void xfrd_copy_soa(xfrd_soa_t* soa, rr_type* rr);
 
 /* check for failed updates - it is assumed that now the reload has
    finished, and all zone SOAs have been sent. */
-void xfrd_check_failed_updates();
+void xfrd_check_failed_updates(void);
 
 /*
  * Prepare zones for a reload, this sets the times on the zones to be
  * before the current time, so the reload happens after.
  */
-void xfrd_prepare_zones_for_reload();
+void xfrd_prepare_zones_for_reload(void);
 
 /* Bind a local interface to a socket descriptor, return 1 on success */
 int xfrd_bind_local_interface(int sockd, acl_options_t* ifc,
 	acl_options_t* acl, int tcp);
 
+/* process results and soa info from reload */
+void xfrd_process_task_result(xfrd_state_t* xfrd, struct udb_base* taskudb);
+
+/* set to reload right away (for user controlled reload events) */
+void xfrd_set_reload_now(xfrd_state_t* xfrd);
+
+/* send expiry notifications to nsd */
+void xfrd_send_expire_notification(xfrd_zone_t* zone);
+
+/* handle incoming notify (soa or NULL) and start zone xfr if necessary */
+void xfrd_handle_notify_and_start_xfr(xfrd_zone_t* zone, xfrd_soa_t* soa);
+
+/* handle zone timeout, event */
+void xfrd_handle_zone(int fd, short event, void* arg);
+
+const char* xfrd_pretty_time(time_t v);
+
 #endif /* XFRD_H */
diff --git a/usr.sbin/nsd/zparser.y b/usr.sbin/nsd/zparser.y
index 4855784f7e2..f916a74d936 100644
--- a/usr.sbin/nsd/zparser.y
+++ b/usr.sbin/nsd/zparser.y
@@ -67,7 +67,7 @@ nsec3_add_params(const char* hash_algo_str, const char* flag_str,
 %token <type> T_OPT T_APL T_UINFO T_UID T_GID T_UNSPEC T_TKEY T_TSIG T_IXFR
 %token <type> T_AXFR T_MAILB T_MAILA T_DS T_DLV T_SSHFP T_RRSIG T_NSEC T_DNSKEY
 %token <type> T_SPF T_NSEC3 T_IPSECKEY T_DHCID T_NSEC3PARAM T_TLSA
-%token <type> T_NID T_L32 T_L64 T_LP T_EUI48 T_EUI64 T_CAA
+%token <type> T_NID T_L32 T_L64 T_LP T_EUI48 T_EUI64 T_CAA T_CDS T_CDNSKEY
 
 /* other tokens */
 %token	       DOLLAR_TTL DOLLAR_ORIGIN NL SP
@@ -612,6 +612,10 @@ type_and_rdata:
     |	T_EUI64 sp rdata_unknown { $$ = $1; parse_unknown_rdata($1, $3); }
     |	T_CAA sp rdata_caa
     |	T_CAA sp rdata_unknown { $$ = $1; parse_unknown_rdata($1, $3); }
+    |	T_CDS sp rdata_ds
+    |	T_CDS sp rdata_unknown { $$ = $1; parse_unknown_rdata($1, $3); }
+    |	T_CDNSKEY sp rdata_dnskey
+    |	T_CDNSKEY sp rdata_unknown { $$ = $1; parse_unknown_rdata($1, $3); }
     |	T_UTYPE sp rdata_unknown { $$ = $1; parse_unknown_rdata($1, $3); }
     |	STR error NL
     {