Stop using rsync --delete when syncing up with the CA repos. Instead

use the files referenced in the manifests to build up a list of files
to keep and remove anything that is not in the list after doing the
full computation.
OK job@ benno@

2 files changed, 139 insertions, 5 deletions

diff --git a/usr.sbin/rpki-client/extern.h b/usr.sbin/rpki-client/extern.h
index cfbc5a1dfb9..3b0b81ae7c1 100644
--- a/usr.sbin/rpki-client/extern.h
+++ b/usr.sbin/rpki-client/extern.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: extern.h,v 1.29 2020/04/30 13:46:39 deraadt Exp $ */
+/*	$OpenBSD: extern.h,v 1.30 2020/06/24 14:39:21 claudio Exp $ */
 /*
  * Copyright (c) 2019 Kristaps Dzonsons <kristaps@bsd.lv>
  *
@@ -263,6 +263,7 @@ struct	stats {
 	size_t	 crls; /* revocation lists */
 	size_t	 vrps; /* total number of vrps */
 	size_t	 uniqs; /* number of unique vrps */
+	size_t	 del_files; /* number of files removed in cleanup */
 	char	*talnames;
 	struct timeval	elapsed_time;
 	struct timeval	user_time;
diff --git a/usr.sbin/rpki-client/main.c b/usr.sbin/rpki-client/main.c
index d19de6e60f1..31f2cb27c45 100644
--- a/usr.sbin/rpki-client/main.c
+++ b/usr.sbin/rpki-client/main.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: main.c,v 1.70 2020/05/18 08:46:39 claudio Exp $ */
+/*	$OpenBSD: main.c,v 1.71 2020/06/24 14:39:21 claudio Exp $ */
 /*
  * Copyright (c) 2019 Kristaps Dzonsons <kristaps@bsd.lv>
  *
@@ -128,6 +128,24 @@ struct	entity {
 TAILQ_HEAD(entityq, entity);
 
 /*
+ * Database of all file path accessed during a run.
+ */
+struct filepath {
+	RB_ENTRY(filepath)	entry;
+	char			*file;
+};
+
+static inline int
+filepathcmp(struct filepath *a, struct filepath *b)
+{
+	return strcasecmp(a->file, b->file);
+}
+
+RB_HEAD(filepath_tree, filepath);
+RB_PROTOTYPE(filepath_tree, filepath, entry, filepathcmp);
+struct filepath_tree  fpt = RB_INITIALIZER(&fpt);
+
+/*
  * Mark that our subprocesses will never return.
  */
 static void	proc_parser(int, int) __attribute__((noreturn));
@@ -160,6 +178,37 @@ logx(const char *fmt, ...)
 }
 
 /*
+ * Functions to lookup which files have been accessed during computation.
+ */
+static void
+filepath_add(char *file)
+{
+	struct filepath *fp;
+
+	if ((fp = malloc(sizeof(*fp))) == NULL)
+		err(1, NULL);
+	if ((fp->file = strdup(file)) == NULL)
+		err(1, NULL);
+
+	if (RB_INSERT(filepath_tree, &fpt, fp) != NULL) {
+		/* already in the tree */
+		free(fp->file);
+		free(fp);
+	}
+}
+
+static int
+filepath_exists(char *file)
+{
+	struct filepath needle;
+
+	needle.file = file;
+	return RB_FIND(filepath_tree, &fpt, &needle) != NULL;
+}
+
+RB_GENERATE(filepath_tree, filepath, entry, filepathcmp);
+
+/*
  * Resolve the media type of a resource by looking at its suffice.
  * Returns the type of RTYPE_EOF if not found.
  */
@@ -374,6 +423,7 @@ entityq_add(int fd, struct entityq *q, char *file, enum rtype type,
 		if ((p->descr = strdup(descr)) == NULL)
 			err(1, "strdup");
 
+	filepath_add(file);
 	TAILQ_INSERT_TAIL(q, p, entries);
 
 	/*
@@ -724,7 +774,6 @@ proc_rsync(char *prog, char *bind_addr, int fd, int noop)
 			i = 0;
 			args[i++] = (char *)prog;
 			args[i++] = "-rt";
-			args[i++] = "--delete";
 			if (bind_addr != NULL) {
 				args[i++] = "--address";
 				args[i++] = (char *)bind_addr;
@@ -1365,6 +1414,88 @@ tal_load_default(const char *tals[], size_t max)
 	return (s);
 }
 
+static char **
+add_to_del(char **del, size_t *dsz, char *file)
+{
+	size_t i = *dsz;
+
+	del = reallocarray(del, i + 1, sizeof(*del));
+	if (del == NULL)
+		err(1, "reallocarray");
+	del[i] = strdup(file);
+	if (del[i] == NULL)
+		err(1, "strdup");
+	*dsz = i + 1;
+	return del;
+}
+
+static size_t
+repo_cleanup(const char *cachedir, struct repotab *rt)
+{
+	size_t i, delsz = 0;
+	char *argv[2], **del = NULL;
+	FTS *fts;
+	FTSENT *e;
+
+	/* change working directory to the cache directory */
+	if (chdir(cachedir) == -1)
+		err(1, "%s: chdir", cachedir);
+
+	for (i = 0; i < rt->reposz; i++) {
+		if (asprintf(&argv[0], "%s/%s", rt->repos[i].host,
+		    rt->repos[i].module) == -1)
+			err(1, NULL);
+		argv[1] = NULL;
+		if ((fts = fts_open(argv, FTS_PHYSICAL | FTS_NOSTAT,
+		    NULL)) == NULL)
+			err(1, "fts_open");
+		errno = 0;
+		while ((e = fts_read(fts)) != NULL) {
+			switch (e->fts_info) {
+			case FTS_NSOK:
+				if (!filepath_exists(e->fts_path))
+					del = add_to_del(del, &delsz,
+					    e->fts_path);
+				break;
+			case FTS_D:
+			case FTS_DP:
+				/* TODO empty directory pruning */
+				break;
+			case FTS_SL:
+			case FTS_SLNONE:
+				warnx("symlink %s", e->fts_path);
+				del = add_to_del(del, &delsz, e->fts_path);
+				break;
+			case FTS_NS:
+			case FTS_ERR:
+				warnc(e->fts_errno, "fts_read %s", e->fts_path);
+				break;
+			default:
+				warnx("unhandled[%x] %s", e->fts_info,
+				    e->fts_path);
+				break;
+			}
+
+			errno = 0;
+		}
+		if (errno)
+			err(1, "fts_read");
+		if (fts_close(fts) == -1)
+			err(1, "fts_close");
+	}
+
+	for (i = 0; i < delsz; i++) {
+		if (unlink(del[i]) == -1)
+			warn("unlink %s", del[i]);
+		if (verbose > 1)
+			logx("deleted %s", del[i]);
+		free(del[i]);
+	}
+	free(del);
+
+	return delsz;
+}
+
 int
 main(int argc, char *argv[])
 {
@@ -1620,7 +1751,7 @@ main(int argc, char *argv[])
 			ent = entityq_next(proc, &q);
 			entity_process(proc, rsync, &stats,
 			    &q, ent, &rt, &eid, &v);
-			if (verbose > 1)
+			if (verbose > 2)
 				fprintf(stderr, "%s\n", ent->uri);
 			entity_free(ent);
 		}
@@ -1666,6 +1797,8 @@ main(int argc, char *argv[])
 	if (outputfiles(&v, &stats))
 		rc = 1;
 
+	stats.del_files = repo_cleanup(cachedir, &rt);
+
 	logx("Route Origin Authorizations: %zu (%zu failed parse, %zu invalid)",
 	    stats.roas, stats.roas_fail, stats.roas_invalid);
 	logx("Certificates: %zu (%zu failed parse, %zu invalid)",
@@ -1675,10 +1808,10 @@ main(int argc, char *argv[])
 	    stats.mfts, stats.mfts_fail, stats.mfts_stale);
 	logx("Certificate revocation lists: %zu", stats.crls);
 	logx("Repositories: %zu", stats.repos);
+	logx("Files removed: %zu", stats.del_files);
 	logx("VRP Entries: %zu (%zu unique)", stats.vrps, stats.uniqs);
 
 	/* Memory cleanup. */
-
 	for (i = 0; i < rt.reposz; i++) {
 		free(rt.repos[i].host);
 		free(rt.repos[i].module);