Introduce a validated cache which holds all the files that have

successfully been verified by rpki-client.

With this the rsync and rrdp directories are more of a temporary storage
location. New files are downloaded there and then moved to the valid
directory at the end. In -n mode only the valid directory is looked at with
the exception of the ta directory holding the trust anchors.
A file can now be in two different locations so adjust all the code paths
that open files to check both locations.
One nice side-effect of this is that the RRDP handling in the main process
got simplified. There is no longer the need for temporary RRDP directories.
OK tb@

5 files changed, 410 insertions, 284 deletions

diff --git a/usr.sbin/rpki-client/extern.h b/usr.sbin/rpki-client/extern.h
index aa683901e0d..7c0fbbc4a87 100644
--- a/usr.sbin/rpki-client/extern.h
+++ b/usr.sbin/rpki-client/extern.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: extern.h,v 1.103 2022/01/13 13:46:03 claudio Exp $ */
+/*	$OpenBSD: extern.h,v 1.104 2022/01/14 15:00:23 claudio Exp $ */
 /*
  * Copyright (c) 2019 Kristaps Dzonsons <kristaps@bsd.lv>
  *
@@ -341,7 +341,7 @@ enum publish_type {
 struct entity {
 	TAILQ_ENTRY(entity) entries;
 	char		*path;		/* path relative to repository */
-	char		*file;		/* filename */
+	char		*file;		/* filename or valid repo path */
 	unsigned char	*data;		/* optional data blob */
 	size_t		 datasz; 	/* length of optional data blob */
 	unsigned int	 repoid;	/* repository identifier */
@@ -380,6 +380,7 @@ struct stats {
 	size_t	 vrps; /* total number of vrps */
 	size_t	 uniqs; /* number of unique vrps */
 	size_t	 del_files; /* number of files removed in cleanup */
+	size_t	 extra_files; /* number of superfluous files */
 	size_t	 del_dirs; /* number of directories removed in cleanup */
 	size_t	 brks; /* number of BGPsec Router Key (BRK) certificates */
 	struct timeval	elapsed_time;
@@ -506,7 +507,7 @@ void		 rrdp_clear(unsigned int);
 void		 rrdp_save_state(unsigned int, struct rrdp_session *);
 int		 rrdp_handle_file(unsigned int, enum publish_type, char *,
 		    char *, size_t, char *, size_t);
-char		*repo_basedir(const struct repo *);
+char		*repo_basedir(const struct repo *, int);
 unsigned int	 repo_id(const struct repo *);
 const char	*repo_uri(const struct repo *);
 struct repo	*ta_lookup(int, struct tal *);
@@ -520,7 +521,8 @@ void		 rsync_finish(unsigned int, int);
 void		 http_finish(unsigned int, enum http_result, const char *);
 void		 rrdp_finish(unsigned int, int);
 
-void		 rsync_fetch(unsigned int, const char *, const char *);
+void		 rsync_fetch(unsigned int, const char *, const char *,
+		    const char *);
 void		 http_fetch(unsigned int, const char *, const char *, int);
 void		 rrdp_fetch(unsigned int, const char *, const char *,
 		    struct rrdp_session *);
diff --git a/usr.sbin/rpki-client/main.c b/usr.sbin/rpki-client/main.c
index e8084bc745f..1a0df6ec433 100644
--- a/usr.sbin/rpki-client/main.c
+++ b/usr.sbin/rpki-client/main.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: main.c,v 1.175 2022/01/13 13:18:41 claudio Exp $ */
+/*	$OpenBSD: main.c,v 1.176 2022/01/14 15:00:23 claudio Exp $ */
 /*
  * Copyright (c) 2021 Claudio Jeker <claudio@openbsd.org>
  * Copyright (c) 2019 Kristaps Dzonsons <kristaps@bsd.lv>
@@ -151,20 +151,22 @@ entity_write_repo(struct repo *rp)
 	struct ibuf *b;
 	enum rtype type = RTYPE_REPO;
 	unsigned int repoid;
-	char *path;
+	char *path, *altpath;
 	int talid = 0;
 
 	repoid = repo_id(rp);
-	path = repo_basedir(rp);
+	path = repo_basedir(rp, 0);
+	altpath = repo_basedir(rp, 1);
 	b = io_new_buffer();
 	io_simple_buffer(b, &type, sizeof(type));
 	io_simple_buffer(b, &repoid, sizeof(repoid));
 	io_simple_buffer(b, &talid, sizeof(talid));
 	io_str_buffer(b, path);
-	io_str_buffer(b, NULL);
+	io_str_buffer(b, altpath);
 	io_buf_buffer(b, NULL, 0);
 	io_close_buffer(&procq, b);
 	free(path);
+	free(altpath);
 }
 
 /*
@@ -254,14 +256,15 @@ rrdp_fetch(unsigned int id, const char *uri, const char *local,
  * Request a repository sync via rsync URI to directory local.
  */
 void
-rsync_fetch(unsigned int id, const char *uri, const char *local)
+rsync_fetch(unsigned int id, const char *uri, const char *local,
+    const char *base)
 {
 	struct ibuf	*b;
 
 	b = io_new_buffer();
 	io_simple_buffer(b, &id, sizeof(id));
 	io_str_buffer(b, local);
-	io_str_buffer(b, NULL);
+	io_str_buffer(b, base);
 	io_str_buffer(b, uri);
 	io_close_buffer(&rsyncq, b);
 }
@@ -1246,8 +1249,8 @@ main(int argc, char *argv[])
 	logx("Certificate revocation lists: %zu", stats.crls);
 	logx("Ghostbuster records: %zu", stats.gbrs);
 	logx("Repositories: %zu", stats.repos);
-	logx("Cleanup: removed %zu files, %zu directories",
-	    stats.del_files, stats.del_dirs);
+	logx("Cleanup: removed %zu files, %zu directories, %zu superfluous",
+	    stats.del_files, stats.del_dirs, stats.extra_files);
 	logx("VRP Entries: %zu (%zu unique)", stats.vrps, stats.uniqs);
 
 	/* Memory cleanup. */
diff --git a/usr.sbin/rpki-client/output-json.c b/usr.sbin/rpki-client/output-json.c
index 116a35d88ad..2703134a097 100644
--- a/usr.sbin/rpki-client/output-json.c
+++ b/usr.sbin/rpki-client/output-json.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: output-json.c,v 1.22 2021/11/04 11:32:55 claudio Exp $ */
+/*	$OpenBSD: output-json.c,v 1.23 2022/01/14 15:00:23 claudio Exp $ */
 /*
  * Copyright (c) 2019 Claudio Jeker <claudio@openbsd.org>
  *
@@ -78,6 +78,7 @@ outputheader_json(FILE *out, struct stats *st)
 	    "\t\t\"vrps\": %zu,\n"
 	    "\t\t\"uniquevrps\": %zu,\n"
 	    "\t\t\"cachedir_del_files\": %zu,\n"
+	    "\t\t\"cachedir_superfluous_files\": %zu,\n"
 	    "\t\t\"cachedir_del_dirs\": %zu\n"
 	    "\t},\n\n",
 	    st->mfts, st->mfts_fail, st->mfts_stale,
@@ -85,7 +86,7 @@ outputheader_json(FILE *out, struct stats *st)
 	    st->gbrs,
 	    st->repos,
 	    st->vrps, st->uniqs,
-	    st->del_files, st->del_dirs) < 0)
+	    st->del_files, st->extra_files, st->del_dirs) < 0)
 		return -1;
 	return 0;
 }
diff --git a/usr.sbin/rpki-client/parser.c b/usr.sbin/rpki-client/parser.c
index 73e757cdfd3..878c0d1923f 100644
--- a/usr.sbin/rpki-client/parser.c
+++ b/usr.sbin/rpki-client/parser.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: parser.c,v 1.36 2022/01/13 14:58:21 claudio Exp $ */
+/*	$OpenBSD: parser.c,v 1.37 2022/01/14 15:00:23 claudio Exp $ */
 /*
  * Copyright (c) 2019 Claudio Jeker <claudio@openbsd.org>
  * Copyright (c) 2019 Kristaps Dzonsons <kristaps@bsd.lv>
@@ -50,6 +50,7 @@ static struct crl_tree	 crlt = RB_INITIALIZER(&crlt);
 struct parse_repo {
 	RB_ENTRY(parse_repo)	 entry;
 	char			*path;
+	char			*validpath;
 	unsigned int		 id;
 };
 
@@ -72,20 +73,75 @@ repo_get(unsigned int id)
 }
 
 static void
-repo_add(unsigned int id, char *path)
+repo_add(unsigned int id, char *path, char *validpath)
 {
 	struct parse_repo *rp;
 
-	if ((rp = malloc(sizeof(*rp))) == NULL)
+	if ((rp = calloc(1, sizeof(*rp))) == NULL)
 		err(1, NULL);
 	rp->id = id;
-	if ((rp->path = strdup(path)) == NULL)
-		err(1, NULL);
+	if (path != NULL)
+		if ((rp->path = strdup(path)) == NULL)
+			err(1, NULL);
+	if (validpath != NULL)
+		if ((rp->validpath = strdup(validpath)) == NULL)
+			err(1, NULL);
 
 	if (RB_INSERT(repo_tree, &repos, rp) != NULL)
 		errx(1, "repository already added: id %d, %s", id, path);
 }
 
+/*
+ * Build access path to file based on repoid, path and file values.
+ * If wantalt == 1 the function can return NULL, if wantalt == 0 it
+ * can not fail.
+ */
+static char *
+parse_filepath(unsigned int repoid, const char *path, const char *file,
+    int wantalt)
+{
+	struct parse_repo	*rp;
+	char			*fn, *repopath;
+
+	/* build file path based on repoid, entity path and filename */
+	rp = repo_get(repoid);
+	if (rp == NULL) {
+		/* no repo so no alternative path. */
+		if (wantalt)
+			return NULL;
+
+		if (path == NULL) {
+			if ((fn = strdup(file)) == NULL)
+				err(1, NULL);
+		} else {
+			if (asprintf(&fn, "%s/%s", path, file) == -1)
+				err(1, NULL);
+		}
+	} else {
+		if (wantalt || rp->path == NULL)
+			repopath = rp->validpath;
+		else
+			repopath = rp->path;
+
+		if (repopath == NULL)
+			return NULL;
+
+		if (path == NULL) {
+			if (asprintf(&fn, "%s/%s", repopath, file) == -1)
+				err(1, NULL);
+		} else {
+			if (asprintf(&fn, "%s/%s/%s", repopath, path,
+			    file) == -1)
+				err(1, NULL);
+		}
+	}
+	return fn;
+}
+
+/*
+ * Callback for X509_verify_cert() to handle critical extensions in old
+ * LibreSSL libraries or OpenSSL libs without RFC3779 support.
+ */
 static int
 verify_cb(int ok, X509_STORE_CTX *store_ctx)
 {
@@ -229,13 +285,8 @@ int
 mft_check(const char *fn, struct mft *p)
 {
 	size_t	i;
-	int	fd, rc = 1;
-	char	*cp, *h, *path = NULL;
-
-	/* Check hash of file now, but first build path for it */
-	cp = strrchr(fn, '/');
-	assert(cp != NULL);
-	assert(cp - fn < INT_MAX);
+	int	fd, try, rc = 1;
+	char	*h, *path;
 
 	for (i = 0; i < p->filesz; i++) {
 		const struct mftfile *m = &p->files[i];
@@ -246,15 +297,24 @@ mft_check(const char *fn, struct mft *p)
 			free(h);
 			continue;
 		}
-		if (asprintf(&path, "%.*s/%s", (int)(cp - fn), fn,
-		    m->file) == -1)
-			err(1, NULL);
-		fd = open(path, O_RDONLY);
+
+		fd = -1;
+		try = 0;
+		path = NULL;
+		do {
+			free(path);
+			if ((path = parse_filepath(p->repoid, p->path, m->file,
+			    try++)) == NULL)
+				break;
+			fd = open(path, O_RDONLY);
+		} while (fd == -1 && try < 2);
+
+		free(path);
+
 		if (!valid_filehash(fd, m->hash, sizeof(m->hash))) {
 			warnx("%s: bad message digest for %s", fn, m->file);
 			rc = 0;
 		}
-		free(path);
 	}
 
 	return rc;
@@ -631,33 +691,40 @@ build_crls(const struct crl *crl, STACK_OF(X509_CRL) **crls)
 }
 
 static char *
-parse_filepath(struct entity *entp)
+parse_load_file(struct entity *entp, unsigned char **f, size_t *flen)
 {
-	struct parse_repo	*rp;
-	char			*file;
+	char *file, *nfile;
 
-	/* build file path based on repoid, entity path and filename */
-	rp = repo_get(entp->repoid);
-	if (rp == NULL) {
-		if (entp->path == NULL) {
-			if ((file = strdup(entp->file)) == NULL)
-					err(1, NULL);
-		} else {
-			if (asprintf(&file, "%s/%s", entp->path,
-			    entp->file) == -1)
-				err(1, NULL);
-		}
-	} else {
-		if (entp->path == NULL) {
-			if (asprintf(&file, "%s/%s", rp->path,
-			    entp->file) == -1)
-				err(1, NULL);
-		} else {
-			if (asprintf(&file, "%s/%s/%s", rp->path,
-			    entp->path, entp->file) == -1)
-				err(1, NULL);
-		}
+	file = parse_filepath(entp->repoid, entp->path, entp->file, 0);
+
+	/* TAL files include the data already */
+	if (entp->type == RTYPE_TAL) {
+		*f = NULL;
+		*flen = 0;
+		return file;
 	}
+
+	*f = load_file(file, flen);
+	if (*f != NULL)
+		return file;
+
+	if (errno != ENOENT)
+		goto fail;
+
+	/* try alternate file location */
+	nfile = parse_filepath(entp->repoid, entp->path, entp->file, 1);
+	if (nfile == NULL)
+		goto fail;
+
+	free(file);
+	file = nfile;
+
+	*f = load_file(file, flen);
+	if (*f != NULL)
+		return file;
+
+fail:
+	warn("parse file %s", file);
 	return file;
 }
 
@@ -680,20 +747,14 @@ parse_entity(struct entityq *q, struct msgbuf *msgq)
 
 		/* handle RTYPE_REPO first */
 		if (entp->type == RTYPE_REPO) {
-			repo_add(entp->repoid, entp->path);
+			repo_add(entp->repoid, entp->path, entp->file);
 			entity_free(entp);
 			continue;
 		}
 
-		f = NULL;
-		file = parse_filepath(entp);
-		if (entp->type != RTYPE_TAL) {
-			f = load_file(file, &flen);
-			if (f == NULL)
-				warn("%s", file);
-		}
+		file = parse_load_file(entp, &f, &flen);
 
-		/* pass back at least type and filename */
+		/* pass back at least type, repoid and filename */
 		b = io_new_buffer();
 		io_simple_buffer(b, &entp->type, sizeof(entp->type));
 		io_str_buffer(b, file);
diff --git a/usr.sbin/rpki-client/repo.c b/usr.sbin/rpki-client/repo.c
index fb05e375791..773290dc055 100644
--- a/usr.sbin/rpki-client/repo.c
+++ b/usr.sbin/rpki-client/repo.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: repo.c,v 1.24 2022/01/13 14:57:02 claudio Exp $ */
+/*	$OpenBSD: repo.c,v 1.25 2022/01/14 15:00:23 claudio Exp $ */
 /*
  * Copyright (c) 2021 Claudio Jeker <claudio@openbsd.org>
  * Copyright (c) 2019 Kristaps Dzonsons <kristaps@bsd.lv>
@@ -58,8 +58,6 @@ struct rrdprepo {
 	SLIST_ENTRY(rrdprepo)	 entry;
 	char			*notifyuri;
 	char			*basedir;
-	char			*temp;
-	struct filepath_tree	 added;
 	struct filepath_tree	 deleted;
 	unsigned int		 id;
 	enum repo_state		 state;
@@ -92,6 +90,7 @@ struct repo {
 	SLIST_ENTRY(repo)	 entry;
 	char			*repouri;
 	char			*notifyuri;
+	char			*basedir;
 	const struct rrdprepo	*rrdp;
 	const struct rsyncrepo	*rsync;
 	const struct tarepo	*ta;
@@ -105,7 +104,7 @@ static SLIST_HEAD(, repo)	repos = SLIST_HEAD_INITIALIZER(repos);
 /* counter for unique repo id */
 unsigned int		repoid;
 
-static struct rsyncrepo	*rsync_get(const char *, int);
+static struct rsyncrepo	*rsync_get(const char *, const char *);
 static void		 remove_contents(char *);
 
 /*
@@ -168,26 +167,6 @@ filepath_exists(struct filepath_tree *tree, char *file)
 }
 
 /*
- * Return true if a filepath entry exists that starts with path.
- */
-static int
-filepath_dir_exists(struct filepath_tree *tree, char *path)
-{
-	struct filepath needle;
-	struct filepath *res;
-
-	needle.file = path;
-	res = RB_NFIND(filepath_tree, tree, &needle);
-	while (res != NULL && strstr(res->file, path) == res->file) {
-		/* make sure that filepath actually is in that path */
-		if (res->file[strlen(path)] == '/')
-			return 1;
-		res = RB_NEXT(filepath_tree, tree, res);
-	}
-	return 0;
-}
-
-/*
  * Remove entry from tree and free it.
  */
 static void
@@ -284,7 +263,8 @@ repo_state(struct repo *rp)
 		return rp->rsync->state;
 	if (rp->rrdp)
 		return rp->rrdp->state;
-	errx(1, "%s: bad repo", rp->repouri);
+	/* No backend so sync is by definition done. */
+	return REPO_DONE;
 }
 
 /*
@@ -305,7 +285,8 @@ repo_done(const void *vp, int ok)
 			if (!ok) {
 				/* try to fall back to rsync */
 				rp->rrdp = NULL;
-				rp->rsync = rsync_get(rp->repouri, 0);
+				rp->rsync = rsync_get(rp->repouri,
+				    rp->basedir);
 				/* need to check if it was already loaded */
 				if (repo_state(rp) != REPO_LOADING)
 					entityq_flush(&rp->queue, rp);
@@ -362,7 +343,7 @@ ta_fetch(struct tarepo *tr)
 		 * Create destination location.
 		 * Build up the tree to this point.
 		 */
-		rsync_fetch(tr->id, tr->uri[tr->uriidx], tr->basedir);
+		rsync_fetch(tr->id, tr->uri[tr->uriidx], tr->basedir, NULL);
 	} else {
 		int fd;
 
@@ -387,9 +368,6 @@ ta_get(struct tal *tal)
 
 	/* no need to look for possible other repo */
 
-	if (tal->urisz == 0)
-		errx(1, "TAL %s has no URI", tal->descr);
-
 	if ((tr = calloc(1, sizeof(*tr))) == NULL)
 		err(1, NULL);
 	tr->id = ++repoid;
@@ -405,17 +383,7 @@ ta_get(struct tal *tal)
 	tal->urisz = 0;
 	tal->uri = NULL;
 
-	if (noop) {
-		tr->state = REPO_DONE;
-		logx("ta/%s: using cache", tr->descr);
-		repo_done(tr, 0);
-	} else {
-		/* try to create base directory */
-		if (mkpath(tr->basedir) == -1)
-			warn("mkpath %s", tr->basedir);
-
-		ta_fetch(tr);
-	}
+	ta_fetch(tr);
 
 	return tr;
 }
@@ -447,7 +415,7 @@ ta_free(void)
 }
 
 static struct rsyncrepo *
-rsync_get(const char *uri, int nofetch)
+rsync_get(const char *uri, const char *validdir)
 {
 	struct rsyncrepo *rr;
 	char *repo;
@@ -470,22 +438,16 @@ rsync_get(const char *uri, int nofetch)
 	rr->repouri = repo;
 	rr->basedir = repo_dir(repo, "rsync", 0);
 
-	if (noop || nofetch) {
-		rr->state = REPO_DONE;
-		logx("%s: using cache", rr->basedir);
-		repo_done(rr, 0);
-	} else {
-		/* create base directory */
-		if (mkpath(rr->basedir) == -1) {
-			warn("mkpath %s", rr->basedir);
-			rsync_finish(rr->id, 0);
-			return rr;
-		}
-
-		logx("%s: pulling from %s", rr->basedir, rr->repouri);
-		rsync_fetch(rr->id, rr->repouri, rr->basedir);
+	/* create base directory */
+	if (mkpath(rr->basedir) == -1) {
+		warn("mkpath %s", rr->basedir);
+		rsync_finish(rr->id, 0);
+		return rr;
 	}
 
+	logx("%s: pulling from %s", rr->basedir, rr->repouri);
+	rsync_fetch(rr->id, rr->repouri, rr->basedir, validdir);
+
 	return rr;
 }
 
@@ -513,25 +475,19 @@ rsync_free(void)
 	}
 }
 
-static int rrdprepo_fetch(struct rrdprepo *);
-
 /*
  * Build local file name base on the URI and the rrdprepo info.
  */
 static char *
-rrdp_filename(const struct rrdprepo *rr, const char *uri, int temp)
+rrdp_filename(const struct rrdprepo *rr, const char *uri, int valid)
 {
 	char *nfile;
-	char *dir = rr->basedir;
-
-	if (temp)
-		dir = rr->temp;
-
-	if (!valid_uri(uri, strlen(uri), "rsync://")) {
-		warnx("%s: bad URI %s", rr->basedir, uri);
-		return NULL;
-	}
+	const char *dir = rr->basedir;
 
+	if (valid)
+		dir = "valid";
+	if (!valid_uri(uri, strlen(uri), "rsync://"))
+		errx(1, "%s: bad URI %s", rr->basedir, uri);
 	uri += strlen("rsync://");	/* skip proto */
 	if (asprintf(&nfile, "%s/%s", dir, uri) == -1)
 		err(1, NULL);
@@ -575,28 +531,46 @@ rrdp_free(void)
 
 		free(rr->notifyuri);
 		free(rr->basedir);
-		free(rr->temp);
 
-		filepath_free(&rr->added);
 		filepath_free(&rr->deleted);
 
 		free(rr);
 	}
 }
 
-static struct rrdprepo *
-rrdp_basedir(const char *dir)
+/*
+ * Check if a directory is an active rrdp repository.
+ * Returns 1 if found else 0.
+ */
+static int
+rrdp_is_active(const char *dir)
 {
 	struct rrdprepo *rr;
 
 	SLIST_FOREACH(rr, &rrdprepos, entry)
-		if (strcmp(dir, rr->basedir) == 0) {
-			if (rr->state == REPO_FAILED)
-				return NULL;
-			return rr;
-		}
+		if (strcmp(dir, rr->basedir) == 0)
+			return rr->state != REPO_FAILED;
 
-	return NULL;
+	return 0;
+}
+
+/*
+ * Check if the URI is actually covered by one of the repositories
+ * that depend on this RRDP repository.
+ * Returns 1 if the URI is valid, 0 if no repouri matches the URI.
+ */
+static int
+rrdp_uri_valid(struct rrdprepo *rr, const char *uri)
+{
+	struct repo *rp;
+
+	SLIST_FOREACH(rp, &repos, entry) {
+		if (rp->rrdp != rr)
+			continue;
+		if (strncmp(uri, rp->repouri, strlen(rp->repouri)) == 0)
+			return 1;
+	}
+	return 0;
 }
 
 /*
@@ -740,8 +714,9 @@ fail:
 }
 
 static struct rrdprepo *
-rrdp_get(const char *uri, int nofetch)
+rrdp_get(const char *uri)
 {
+	struct rrdp_session state = { 0 };
 	struct rrdprepo *rr;
 
 	SLIST_FOREACH(rr, &rrdprepos, entry)
@@ -761,28 +736,24 @@ rrdp_get(const char *uri, int nofetch)
 		err(1, NULL);
 	rr->basedir = repo_dir(uri, "rrdp", 1);
 
-	RB_INIT(&rr->added);
 	RB_INIT(&rr->deleted);
 
-	if (noop || nofetch) {
-		rr->state = REPO_DONE;
-		logx("%s: using cache", rr->notifyuri);
-		repo_done(rr, 0);
-	} else {
-		/* create base directory */
-		if (mkpath(rr->basedir) == -1) {
-			warn("mkpath %s", rr->basedir);
-			rrdp_finish(rr->id, 0);
-			return rr;
-		}
-		if (rrdprepo_fetch(rr) == -1) {
-			rrdp_finish(rr->id, 0);
-			return rr;
-		}
 
-		logx("%s: pulling from %s", rr->notifyuri, "network");
+	/* create base directory */
+	if (mkpath(rr->basedir) == -1) {
+		warn("mkpath %s", rr->basedir);
+		rrdp_finish(rr->id, 0);
+		return rr;
 	}
 
+	/* parse state and start the sync */
+	rrdp_parse_state(rr, &state);
+	rrdp_fetch(rr->id, rr->notifyuri, rr->notifyuri, &state);
+	free(state.session_id);
+	free(state.last_mod);
+
+	logx("%s: pulling from %s", rr->notifyuri, "network");
+
 	return rr;
 }
 
@@ -800,7 +771,6 @@ rrdp_clear(unsigned int id)
 
 	/* remove rrdp repository contents */
 	remove_contents(rr->basedir);
-	remove_contents(rr->temp);
 }
 
 /*
@@ -816,8 +786,8 @@ rrdp_handle_file(unsigned int id, enum publish_type pt, char *uri,
 	struct rrdprepo *rr;
 	struct filepath *fp;
 	ssize_t s;
-	char *fn;
-	int fd = -1;
+	char *fn = NULL;
+	int fd = -1, try = 0;
 
 	rr = rrdp_find(id);
 	if (rr == NULL)
@@ -825,21 +795,20 @@ rrdp_handle_file(unsigned int id, enum publish_type pt, char *uri,
 	if (rr->state == REPO_FAILED)
 		return -1;
 
+	/* check hash of original file for updates and deletes */
 	if (pt == PUB_UPD || pt == PUB_DEL) {
 		if (filepath_exists(&rr->deleted, uri)) {
 			warnx("%s: already deleted", uri);
 			return 0;
 		}
-		fp = filepath_find(&rr->added, uri);
-		if (fp == NULL) {
-			if ((fn = rrdp_filename(rr, uri, 0)) == NULL)
-				return 0;
-		} else {
-			filepath_put(&rr->added, fp);
-			if ((fn = rrdp_filename(rr, uri, 1)) == NULL)
+		/* try to open file first in rrdp then in valid repo */
+		do {
+			free(fn);
+			if ((fn = rrdp_filename(rr, uri, try++)) == NULL)
 				return 0;
-		}
-		fd = open(fn, O_RDONLY);
+			fd = open(fn, O_RDONLY);
+		} while (fd == -1 && try < 2);
+
 		if (!valid_filehash(fd, hash, hlen)) {
 			warnx("%s: bad file digest for %s", rr->notifyuri, fn);
 			free(fn);
@@ -848,6 +817,7 @@ rrdp_handle_file(unsigned int id, enum publish_type pt, char *uri,
 		free(fn);
 	}
 
+	/* write new content or mark uri as deleted. */
 	if (pt == PUB_DEL) {
 		filepath_add(&rr->deleted, uri);
 	} else {
@@ -855,8 +825,8 @@ rrdp_handle_file(unsigned int id, enum publish_type pt, char *uri,
 		if (fp != NULL)
 			filepath_put(&rr->deleted, fp);
 
-		/* add new file to temp dir */
-		if ((fn = rrdp_filename(rr, uri, 1)) == NULL)
+		/* add new file to rrdp dir */
+		if ((fn = rrdp_filename(rr, uri, 0)) == NULL)
 			return 0;
 
 		if (repo_mkpath(fn) == -1)
@@ -876,7 +846,6 @@ rrdp_handle_file(unsigned int id, enum publish_type pt, char *uri,
 		if ((size_t)s != dlen)	/* impossible */
 			errx(1, "short write %s", fn);
 		free(fn);
-		filepath_add(&rr->added, uri);
 	}
 
 	return 1;
@@ -890,66 +859,6 @@ fail:
 }
 
 /*
- * Initiate a RRDP sync, create the required temporary directory and
- * parse a possible state file before sending the request to the RRDP process.
- */
-static int
-rrdprepo_fetch(struct rrdprepo *rr)
-{
-	struct rrdp_session state = { 0 };
-
-	if (asprintf(&rr->temp, "%s.XXXXXXXX", rr->basedir) == -1)
-		err(1, NULL);
-	if (mkdtemp(rr->temp) == NULL) {
-		warn("mkdtemp %s", rr->temp);
-		return -1;
-	}
-
-	rrdp_parse_state(rr, &state);
-	rrdp_fetch(rr->id, rr->notifyuri, rr->notifyuri, &state);
-
-	free(state.session_id);
-	free(state.last_mod);
-
-	return 0;
-}
-
-static int
-rrdp_merge_repo(struct rrdprepo *rr)
-{
-	struct filepath *fp, *nfp;
-	char *fn, *rfn;
-
-	RB_FOREACH_SAFE(fp, filepath_tree, &rr->added, nfp) {
-		fn = rrdp_filename(rr, fp->file, 1);
-		rfn = rrdp_filename(rr, fp->file, 0);
-
-		if (fn == NULL || rfn == NULL)
-			errx(1, "bad filepath");	/* should not happen */
-
-		if (repo_mkpath(rfn) == -1) {
-			goto fail;
-		}
-
-		if (rename(fn, rfn) == -1) {
-			warn("rename %s", rfn);
-			goto fail;
-		}
-
-		free(rfn);
-		free(fn);
-		filepath_put(&rr->added, fp);
-	}
-
-	return 1;
-
-fail:
-	free(rfn);
-	free(fn);
-	return 0;
-}
-
-/*
  * RSYNC sync finished, either with or without success.
  */
 void
@@ -993,6 +902,8 @@ rsync_finish(unsigned int id, int ok)
 		    rr->basedir);
 		stats.rsync_fails++;
 		rr->state = REPO_FAILED;
+		/* clear rsync repo since it failed */
+		remove_contents(rr->basedir);
 	}
 
 	repo_done(rr, ok);
@@ -1013,19 +924,20 @@ rrdp_finish(unsigned int id, int ok)
 	if (rr->state != REPO_LOADING)
 		return;
 
-	if (ok && rrdp_merge_repo(rr)) {
+	if (ok) {
 		logx("%s: loaded from network", rr->notifyuri);
 		stats.rrdp_repos++;
 		rr->state = REPO_DONE;
-		repo_done(rr, ok);
 	} else {
 		logx("%s: load from network failed, fallback to rsync",
 		    rr->notifyuri);
 		stats.rrdp_fails++;
 		rr->state = REPO_FAILED;
-		remove_contents(rr->temp);
-		repo_done(rr, 0);
+		/* clear the RRDP repo since it failed */
+		remove_contents(rr->basedir);
 	}
+
+	repo_done(rr, ok);
 }
 
 /*
@@ -1082,6 +994,9 @@ ta_lookup(int id, struct tal *tal)
 {
 	struct repo	*rp;
 
+	if (tal->urisz == 0)
+		errx(1, "TAL %s has no URI", tal->descr);
+
 	/* Look up in repository table. (Lookup should actually fail here) */
 	SLIST_FOREACH(rp, &repos, entry) {
 		if (strcmp(rp->repouri, tal->descr) == 0)
@@ -1089,11 +1004,24 @@ ta_lookup(int id, struct tal *tal)
 	}
 
 	rp = repo_alloc(id);
+	rp->basedir = repo_dir(tal->descr, "ta", 0);
 	if ((rp->repouri = strdup(tal->descr)) == NULL)
 		err(1, NULL);
 
+	/* try to create base directory */
+	if (mkpath(rp->basedir) == -1)
+		warn("mkpath %s", rp->basedir);
+
+	/* check if sync disabled ... */
+	if (noop) {
+		logx("ta/%s: using cache", rp->repouri);
+		entityq_flush(&rp->queue, rp);
+		return rp;
+	}
+
 	rp->ta = ta_get(tal);
 
+	/* need to check if it was already loaded */
 	if (repo_state(rp) != REPO_LOADING)
 		entityq_flush(&rp->queue, rp);
 
@@ -1130,6 +1058,7 @@ repo_lookup(int talid, const char *uri, const char *notify)
 	}
 
 	rp = repo_alloc(talid);
+	rp->basedir = repo_dir(repouri, "valid", 0);
 	rp->repouri = repouri;
 	if (notify != NULL)
 		if ((rp->notifyuri = strdup(notify)) == NULL)
@@ -1141,12 +1070,24 @@ repo_lookup(int talid, const char *uri, const char *notify)
 		nofetch = 1;
 	}
 
-	/* try RRDP first if available */
+	/* try to create base directory */
+	if (mkpath(rp->basedir) == -1)
+		warn("mkpath %s", rp->basedir);
+
+	/* check if sync disabled ... */
+	if (noop || nofetch) {
+		logx("%s: using cache", rp->basedir);
+		entityq_flush(&rp->queue, rp);
+		return rp;
+	}
+
+	/* ... else try RRDP first if available then rsync */
 	if (notify != NULL)
-		rp->rrdp = rrdp_get(notify, nofetch);
+		rp->rrdp = rrdp_get(notify);
 	if (rp->rrdp == NULL)
-		rp->rsync = rsync_get(uri, nofetch);
+		rp->rsync = rsync_get(uri, rp->basedir);
 
+	/* need to check if it was already loaded */
 	if (repo_state(rp) != REPO_LOADING)
 		entityq_flush(&rp->queue, rp);
 
@@ -1169,24 +1110,29 @@ repo_byid(unsigned int id)
 }
 
 /*
- * Return the repository base directory.
+ * Return the repository base or alternate directory.
  * Returned string must be freed by caller.
  */
 char *
-repo_basedir(const struct repo *rp)
+repo_basedir(const struct repo *rp, int wantvalid)
 {
-	char *path;
+	char *path = NULL;
 
-	if (rp->ta) {
-		if ((path = strdup(rp->ta->basedir)) == NULL)
-			err(1, NULL);
-	} else if (rp->rsync) {
-		if ((path = strdup(rp->rsync->basedir)) == NULL)
+	if (!wantvalid) {
+		if (rp->ta) {
+			if ((path = strdup(rp->ta->basedir)) == NULL)
+				err(1, NULL);
+		} else if (rp->rsync) {
+			if ((path = strdup(rp->rsync->basedir)) == NULL)
+				err(1, NULL);
+		} else if (rp->rrdp) {
+			path = rrdp_filename(rp->rrdp, rp->repouri, 0);
+		} else
+			path = NULL;	/* only valid repo available */
+	} else if (rp->basedir != NULL) {
+		if ((path = strdup(rp->basedir)) == NULL)
 			err(1, NULL);
-	} else if (rp->rrdp) {
-		path = rrdp_filename(rp->rrdp, rp->repouri, 0);
-	} else
-		errx(1, "%s: bad repo", rp->repouri);
+	}
 
 	return path;
 }
@@ -1289,64 +1235,173 @@ add_to_del(char **del, size_t *dsz, char *file)
 	return del;
 }
 
+/*
+ * Delayed delete of files from RRDP. Since RRDP has no security built-in
+ * this code needs to check if this RRDP repository is actually allowed to
+ * remove the file referenced by the URI.
+ */
 static char **
-repo_rrdp_cleanup(struct filepath_tree *tree, struct rrdprepo *rr,
-    char **del, size_t *delsz)
+repo_cleanup_rrdp(struct filepath_tree *tree, char **del, size_t *delsz)
 {
+	struct rrdprepo *rr;
 	struct filepath *fp, *nfp;
 	char *fn;
-
-	RB_FOREACH_SAFE(fp, filepath_tree, &rr->deleted, nfp) {
-		fn = rrdp_filename(rr, fp->file, 0);
-		/* temp dir will be cleaned up by repo_cleanup() */
-
-		if (fn == NULL)
-			errx(1, "bad filepath");	/* should not happen */
-
-		if (!filepath_exists(tree, fn))
+	
+	SLIST_FOREACH(rr, &rrdprepos, entry) {
+		RB_FOREACH_SAFE(fp, filepath_tree, &rr->deleted, nfp) {
+			if (!rrdp_uri_valid(rr, fp->file)) {
+				warnx("%s: external URI %s", rr->notifyuri,
+				    fp->file);
+				filepath_put(&rr->deleted, fp);
+				continue;
+			}
+			/* try to remove file from rrdp repo ... */
+			fn = rrdp_filename(rr, fp->file, 0);
 			del = add_to_del(del, delsz, fn);
-		else
-			warnx("%s: referenced file supposed to be deleted", fn);
+			free(fn);
 
-		free(fn);
-		filepath_put(&rr->deleted, fp);
+			/* ... and from the valid repository if unused. */
+			fn = rrdp_filename(rr, fp->file, 1);
+			if (!filepath_exists(tree, fn))
+				del = add_to_del(del, delsz, fn);
+			else
+				warnx("%s: referenced file supposed to be "
+				    "deleted", fn);
+
+			free(fn);
+			filepath_put(&rr->deleted, fp);
+		}
 	}
 
 	return del;
 }
 
+/*
+ * All files in tree are valid and should be moved to the valid repository
+ * if not already there. Rename the files to the new path and readd the
+ * filepath entry with the new path if successful.
+ */
+static void
+repo_move_valid(struct filepath_tree *tree)
+{
+	struct filepath *fp, *nfp;
+	size_t rsyncsz = strlen("rsync/");
+	size_t rrdpsz = strlen("rrdp/");
+	char *fn, *base;
+
+	RB_FOREACH_SAFE(fp, filepath_tree, tree, nfp) {
+		if (strncmp(fp->file, "rsync/", rsyncsz) != 0 &&
+		    strncmp(fp->file, "rrdp/", rrdpsz) != 0)
+			continue; /* not a temporary file path */
+
+		if (strncmp(fp->file, "rsync/", rsyncsz) == 0) {
+			if (asprintf(&fn, "valid/%s", fp->file + rsyncsz) == -1)
+				err(1, NULL);
+		} else {
+			base = strchr(fp->file + rrdpsz, '/');
+			assert(base != NULL);
+			if (asprintf(&fn, "valid/%s", base + 1) == -1)
+				err(1, NULL);
+		}
+
+		if (repo_mkpath(fn) == -1) {
+			free(fn);
+			continue;
+		}
+
+		if (rename(fp->file, fn) == -1) {
+			warn("rename %s", fn);
+			free(fn);
+			continue;
+		}
+
+		/* switch filepath node to new path */
+		RB_REMOVE(filepath_tree, tree, fp);
+		free(fp->file);
+		fp->file = fn;
+		if (RB_INSERT(filepath_tree, tree, fp) != NULL)
+			errx(1, "both possibilities of file present");
+	}
+}
+
+#define	BASE_DIR	(void *)0x62617365
+#define	RSYNC_DIR	(void *)0x73796e63
+#define	RRDP_DIR	(void *)0x52524450
+
 void
 repo_cleanup(struct filepath_tree *tree)
 {
 	size_t i, cnt, delsz = 0, dirsz = 0;
 	char **del = NULL, **dir = NULL;
-	char *argv[4] = { "ta", "rsync", "rrdp", NULL };
-	struct rrdprepo *rr;
+	char *argv[5] = { "ta", "rsync", "rrdp", "valid", NULL };
 	FTS *fts;
 	FTSENT *e;
 
+	/* first move temp files which have been used to valid dir */
+	repo_move_valid(tree);
+	/* then delete files requested by rrdp */ 
+	del = repo_cleanup_rrdp(tree, del, &delsz);
+
 	if ((fts = fts_open(argv, FTS_PHYSICAL | FTS_NOSTAT, NULL)) == NULL)
 		err(1, "fts_open");
 	errno = 0;
 	while ((e = fts_read(fts)) != NULL) {
 		switch (e->fts_info) {
 		case FTS_NSOK:
-			if (!filepath_exists(tree, e->fts_path))
+			/* handle rrdp .state files explicitly */
+			if (e->fts_parent->fts_pointer == RRDP_DIR &&
+			    e->fts_level == 2 &&
+			    strcmp(e->fts_name, ".state") == 0) {
+				e->fts_parent->fts_number++;
+			} else if (filepath_exists(tree, e->fts_path)) {
+				e->fts_parent->fts_number++;
+			} else if (e->fts_parent->fts_pointer == RRDP_DIR) {
+				/* can't delete these extra files */
+				e->fts_parent->fts_number++;
+				stats.extra_files++;
+				if (verbose > 1)
+					logx("superfluous %s", e->fts_path);
+			} else {
+				if (e->fts_parent->fts_pointer == RSYNC_DIR) {
+					/* no need to keep rsync files */
+					stats.extra_files++;
+					if (verbose > 1)
+						logx("superfluous %s",
+						    e->fts_path);
+				}
 				del = add_to_del(del, &delsz,
 				    e->fts_path);
+			}
 			break;
 		case FTS_D:
-			/* special cleanup for rrdp directories */
-			if ((rr = rrdp_basedir(e->fts_path)) != NULL) {
-				del = repo_rrdp_cleanup(tree, rr, del, &delsz);
-				if (fts_set(fts, e, FTS_SKIP) == -1)
-					err(1, "fts_set");
+			if (e->fts_level == FTS_ROOTLEVEL) {
+				if (strcmp("rsync", e->fts_path) == 0)
+					e->fts_pointer = RSYNC_DIR;
+				else if (strcmp("rrdp", e->fts_path) == 0)
+					e->fts_pointer = RRDP_DIR;
+				else
+					e->fts_pointer = BASE_DIR;
+			} else
+				e->fts_pointer = e->fts_parent->fts_pointer;
+
+			/*
+			 * special handling for rrdp directories,
+			 * clear them if they are not used anymore but
+			 * only if rrdp is active.
+			 */
+			if (e->fts_pointer == RRDP_DIR && !noop && rrdpon &&
+			    e->fts_level == 1) {
+				if (!rrdp_is_active(e->fts_path))
+					e->fts_pointer = NULL;
 			}
 			break;
 		case FTS_DP:
-			if (!filepath_dir_exists(tree, e->fts_path))
-				dir = add_to_del(dir, &dirsz,
-				    e->fts_path);
+			if (e->fts_level == FTS_ROOTLEVEL)
+				break;
+			if (e->fts_number == 0)
+				dir = add_to_del(dir, &dirsz, e->fts_path);
+
+			e->fts_parent->fts_number += e->fts_number;
 			break;
 		case FTS_SL:
 		case FTS_SLNONE:
@@ -1356,8 +1411,7 @@ repo_cleanup(struct filepath_tree *tree)
 		case FTS_NS:
 		case FTS_ERR:
 			if (e->fts_errno == ENOENT &&
-			    (strcmp(e->fts_path, "rsync") == 0 ||
-			    strcmp(e->fts_path, "rrdp") == 0))
+			    e->fts_level == FTS_ROOTLEVEL)
 				continue;
 			warnx("fts_read %s: %s", e->fts_path,
 			    strerror(e->fts_errno));
@@ -1388,7 +1442,7 @@ repo_cleanup(struct filepath_tree *tree)
 		free(del[i]);
 	}
 	free(del);
-	stats.del_files = cnt;
+	stats.del_files += cnt;
 
 	cnt = 0;
 	for (i = 0; i < dirsz; i++) {
@@ -1399,7 +1453,7 @@ repo_cleanup(struct filepath_tree *tree)
 		free(dir[i]);
 	}
 	free(dir);
-	stats.del_dirs = cnt;
+	stats.del_dirs += cnt;
 }
 
 void
@@ -1410,6 +1464,8 @@ repo_free(void)
 	while ((rp = SLIST_FIRST(&repos)) != NULL) {
 		SLIST_REMOVE_HEAD(&repos, entry);
 		free(rp->repouri);
+		free(rp->notifyuri);
+		free(rp->basedir);
 		free(rp);
 	}
 
@@ -1418,6 +1474,9 @@ repo_free(void)
 	rsync_free();
 }
 
+/*
+ * Remove all files and directories under base but do not remove base itself.
+ */
 static void
 remove_contents(char *base)
 {