/* $OpenBSD: repo.c,v 1.70 2024/11/13 12:51:04 tb Exp $ */ /* * Copyright (c) 2021 Claudio Jeker * Copyright (c) 2019 Kristaps Dzonsons * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above * copyright notice and this permission notice appear in all copies. * * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "extern.h" extern struct stats stats; extern int rrdpon; extern int repo_timeout; extern time_t deadline; int nofetch; enum repo_state { REPO_LOADING = 0, REPO_DONE = 1, REPO_FAILED = -1, }; /* * A ta, rsync or rrdp repository. * Depending on what is needed the generic repository is backed by * a ta, rsync or rrdp repository. Multiple repositories can use the * same backend. */ struct rrdprepo { SLIST_ENTRY(rrdprepo) entry; char *notifyuri; char *basedir; struct filepath_tree deleted; unsigned int id; enum repo_state state; time_t last_reset; }; static SLIST_HEAD(, rrdprepo) rrdprepos = SLIST_HEAD_INITIALIZER(rrdprepos); struct rsyncrepo { SLIST_ENTRY(rsyncrepo) entry; char *repouri; char *basedir; unsigned int id; enum repo_state state; }; static SLIST_HEAD(, rsyncrepo) rsyncrepos = SLIST_HEAD_INITIALIZER(rsyncrepos); struct tarepo { SLIST_ENTRY(tarepo) entry; char *descr; char *basedir; char **uri; size_t num_uris; size_t uriidx; unsigned int id; enum repo_state state; }; static SLIST_HEAD(, tarepo) tarepos = SLIST_HEAD_INITIALIZER(tarepos); struct repo { SLIST_ENTRY(repo) entry; char *repouri; char *notifyuri; char *basedir; const struct rrdprepo *rrdp; const struct rsyncrepo *rsync; const struct tarepo *ta; struct entityq queue; /* files waiting for repo */ struct repotalstats stats[TALSZ_MAX]; struct repostats repostats; struct timespec start_time; time_t alarm; /* sync timeout */ int talid; int stats_used[TALSZ_MAX]; unsigned int id; /* identifier */ }; static SLIST_HEAD(, repo) repos = SLIST_HEAD_INITIALIZER(repos); /* counter for unique repo id */ unsigned int repoid; static struct rsyncrepo *rsync_get(const char *, const char *); static void remove_contents(char *); /* * Database of all file path accessed during a run. */ struct filepath { RB_ENTRY(filepath) entry; char *file; time_t mtime; unsigned int talmask; }; static inline int filepathcmp(const struct filepath *a, const struct filepath *b) { return strcmp(a->file, b->file); } RB_PROTOTYPE(filepath_tree, filepath, entry, filepathcmp); /* * Functions to lookup which files have been accessed during computation. */ int filepath_add(struct filepath_tree *tree, char *file, int id, time_t mtime, int ok) { struct filepath *fp, *rfp; CTASSERT(TALSZ_MAX < 8 * sizeof(fp->talmask)); assert(id >= 0 && id < 8 * (int)sizeof(fp->talmask)); if ((fp = calloc(1, sizeof(*fp))) == NULL) err(1, NULL); if ((fp->file = strdup(file)) == NULL) err(1, NULL); fp->mtime = mtime; if ((rfp = RB_INSERT(filepath_tree, tree, fp)) != NULL) { /* already in the tree */ free(fp->file); free(fp); if (rfp->talmask & (1 << id)) return 0; fp = rfp; } if (ok) fp->talmask |= (1 << id); return 1; } /* * Lookup a file path in the tree and return the object if found or NULL. */ static struct filepath * filepath_find(struct filepath_tree *tree, char *file) { struct filepath needle = { .file = file }; return RB_FIND(filepath_tree, tree, &needle); } /* * Returns true if file exists in the tree. */ static int filepath_exists(struct filepath_tree *tree, char *file) { return filepath_find(tree, file) != NULL; } /* * Returns true if file exists and the id bit is set and ok flag is true. */ int filepath_valid(struct filepath_tree *tree, char *file, int id) { struct filepath *fp; if ((fp = filepath_find(tree, file)) == NULL) return 0; return (fp->talmask & (1 << id)) != 0; } /* * Remove entry from tree and free it. */ static void filepath_put(struct filepath_tree *tree, struct filepath *fp) { RB_REMOVE(filepath_tree, tree, fp); free((void *)fp->file); free(fp); } /* * Free all elements of a filepath tree. */ static void filepath_free(struct filepath_tree *tree) { struct filepath *fp, *nfp; RB_FOREACH_SAFE(fp, filepath_tree, tree, nfp) filepath_put(tree, fp); } RB_GENERATE(filepath_tree, filepath, entry, filepathcmp); /* * Function to hash a string into a unique directory name. * Returned hash needs to be freed. */ static char * hash_dir(const char *uri) { unsigned char m[SHA256_DIGEST_LENGTH]; SHA256(uri, strlen(uri), m); return hex_encode(m, sizeof(m)); } /* * Function to build the directory name based on URI and a directory * as prefix. Skip the proto:// in URI but keep everything else. */ static char * repo_dir(const char *uri, const char *dir, int hash) { const char *local; char *out, *hdir = NULL; if (hash) { local = hdir = hash_dir(uri); } else { local = strchr(uri, ':'); if (local != NULL) local += strlen("://"); else local = uri; } if (dir == NULL) { if ((out = strdup(local)) == NULL) err(1, NULL); } else { if (asprintf(&out, "%s/%s", dir, local) == -1) err(1, NULL); } free(hdir); return out; } /* * Function to create all missing directories to a path. * This functions alters the path temporarily. */ static int repo_mkpath(int fd, char *file) { char *slash; /* build directory hierarchy */ slash = strrchr(file, '/'); assert(slash != NULL); *slash = '\0'; if (mkpathat(fd, file) == -1) { warn("mkpath %s", file); return -1; } *slash = '/'; return 0; } /* * Return the state of a repository. */ static enum repo_state repo_state(const struct repo *rp) { if (rp->ta) return rp->ta->state; if (rp->rsync) return rp->rsync->state; if (rp->rrdp) return rp->rrdp->state; /* No backend so sync is by definition done. */ return REPO_DONE; } /* * Function called once a repository is done with the sync. Either * successfully or after failure. */ static void repo_done(const void *vp, int ok) { struct repo *rp; struct timespec flush_time; SLIST_FOREACH(rp, &repos, entry) { if (vp != rp->ta && vp != rp->rsync && vp != rp->rrdp) continue; /* for rrdp try to fall back to rsync */ if (vp == rp->rrdp && !ok && !nofetch) { rp->rrdp = NULL; rp->rsync = rsync_get(rp->repouri, rp->basedir); /* need to check if it was already loaded */ if (repo_state(rp) == REPO_LOADING) continue; } entityq_flush(&rp->queue, rp); clock_gettime(CLOCK_MONOTONIC, &flush_time); timespecsub(&flush_time, &rp->start_time, &rp->repostats.sync_time); } } /* * Build TA file name based on the repo info. * If temp is set add Xs for mkostemp. */ static char * ta_filename(const struct tarepo *tr) { const char *file; char *nfile; /* does not matter which URI, all end with same filename */ file = strrchr(tr->uri[0], '/'); assert(file); if (asprintf(&nfile, "%s%s", tr->basedir, file) == -1) err(1, NULL); return nfile; } static void ta_fetch(struct tarepo *tr) { if (!rrdpon) { for (; tr->uriidx < tr->num_uris; tr->uriidx++) { if (strncasecmp(tr->uri[tr->uriidx], RSYNC_PROTO, RSYNC_PROTO_LEN) == 0) break; } } if (tr->uriidx >= tr->num_uris) { tr->state = REPO_FAILED; logx("ta/%s: fallback to cache", tr->descr); repo_done(tr, 0); return; } logx("ta/%s: pulling from %s", tr->descr, tr->uri[tr->uriidx]); if (strncasecmp(tr->uri[tr->uriidx], RSYNC_PROTO, RSYNC_PROTO_LEN) == 0) { /* * Create destination location. * Build up the tree to this point. */ rsync_fetch(tr->id, tr->uri[tr->uriidx], tr->basedir, NULL); } else { char *temp; int fd; temp = ta_filename(tr); fd = open(temp, O_WRONLY | O_CREAT | O_TRUNC | O_NOFOLLOW | O_CLOEXEC, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH); if (fd == -1) { warn("open: %s", temp); free(temp); http_finish(tr->id, HTTP_FAILED, NULL); return; } free(temp); http_fetch(tr->id, tr->uri[tr->uriidx], NULL, fd); } } static struct tarepo * ta_get(struct tal *tal) { struct tarepo *tr; /* no need to look for possible other repo */ if ((tr = calloc(1, sizeof(*tr))) == NULL) err(1, NULL); tr->id = ++repoid; SLIST_INSERT_HEAD(&tarepos, tr, entry); if ((tr->descr = strdup(tal->descr)) == NULL) err(1, NULL); tr->basedir = repo_dir(tal->descr, ".ta", 0); /* create base directory */ if (mkpath(tr->basedir) == -1) { warn("mkpath %s", tr->basedir); tr->state = REPO_FAILED; repo_done(tr, 0); return tr; } /* steal URI information from TAL */ tr->num_uris = tal->num_uris; tr->uri = tal->uri; tal->num_uris = 0; tal->uri = NULL; ta_fetch(tr); return tr; } static struct tarepo * ta_find(unsigned int id) { struct tarepo *tr; SLIST_FOREACH(tr, &tarepos, entry) if (id == tr->id) break; return tr; } static void ta_free(void) { struct tarepo *tr; while ((tr = SLIST_FIRST(&tarepos)) != NULL) { SLIST_REMOVE_HEAD(&tarepos, entry); free(tr->descr); free(tr->basedir); free(tr->uri); free(tr); } } static struct rsyncrepo * rsync_get(const char *uri, const char *validdir) { struct rsyncrepo *rr; char *repo; if ((repo = rsync_base_uri(uri)) == NULL) errx(1, "bad caRepository URI: %s", uri); SLIST_FOREACH(rr, &rsyncrepos, entry) if (strcmp(rr->repouri, repo) == 0) { free(repo); return rr; } if ((rr = calloc(1, sizeof(*rr))) == NULL) err(1, NULL); rr->id = ++repoid; SLIST_INSERT_HEAD(&rsyncrepos, rr, entry); rr->repouri = repo; rr->basedir = repo_dir(repo, ".rsync", 0); /* create base directory */ if (mkpath(rr->basedir) == -1) { warn("mkpath %s", rr->basedir); rsync_finish(rr->id, 0); return rr; } logx("%s: pulling from %s", rr->basedir, rr->repouri); rsync_fetch(rr->id, rr->repouri, rr->basedir, validdir); return rr; } static struct rsyncrepo * rsync_find(unsigned int id) { struct rsyncrepo *rr; SLIST_FOREACH(rr, &rsyncrepos, entry) if (id == rr->id) break; return rr; } static void rsync_free(void) { struct rsyncrepo *rr; while ((rr = SLIST_FIRST(&rsyncrepos)) != NULL) { SLIST_REMOVE_HEAD(&rsyncrepos, entry); free(rr->repouri); free(rr->basedir); free(rr); } } /* * Build local file name base on the URI and the rrdprepo info. */ static char * rrdp_filename(const struct rrdprepo *rr, const char *uri, int valid) { char *nfile; const char *dir = rr->basedir; if (!valid_uri(uri, strlen(uri), RSYNC_PROTO)) errx(1, "%s: bad URI %s", rr->basedir, uri); uri += RSYNC_PROTO_LEN; /* skip proto */ if (valid) { if ((nfile = strdup(uri)) == NULL) err(1, NULL); } else { if (asprintf(&nfile, "%s/%s", dir, uri) == -1) err(1, NULL); } return nfile; } /* * Build RRDP state file name based on the repo info. * If temp is set add Xs for mkostemp. */ static char * rrdp_state_filename(const struct rrdprepo *rr, int temp) { char *nfile; if (asprintf(&nfile, "%s/.state%s", rr->basedir, temp ? ".XXXXXXXX" : "") == -1) err(1, NULL); return nfile; } static struct rrdprepo * rrdp_find(unsigned int id) { struct rrdprepo *rr; SLIST_FOREACH(rr, &rrdprepos, entry) if (id == rr->id) break; return rr; } static void rrdp_free(void) { struct rrdprepo *rr; while ((rr = SLIST_FIRST(&rrdprepos)) != NULL) { SLIST_REMOVE_HEAD(&rrdprepos, entry); free(rr->notifyuri); free(rr->basedir); filepath_free(&rr->deleted); free(rr); } } /* * Check if a directory is an active rrdp repository. * Returns 1 if found else 0. */ static struct repo * repo_rrdp_bypath(const char *dir) { struct repo *rp; SLIST_FOREACH(rp, &repos, entry) { if (rp->rrdp == NULL) continue; if (strcmp(dir, rp->rrdp->basedir) == 0) return rp; } return NULL; } /* * Check if the URI is actually covered by one of the repositories * that depend on this RRDP repository. * Returns 1 if the URI is valid, 0 if no repouri matches the URI. */ static int rrdp_uri_valid(struct rrdprepo *rr, const char *uri) { struct repo *rp; SLIST_FOREACH(rp, &repos, entry) { if (rp->rrdp != rr) continue; if (strncmp(uri, rp->repouri, strlen(rp->repouri)) == 0) return 1; } return 0; } /* * Allocate and insert a new repository. */ static struct repo * repo_alloc(int talid) { struct repo *rp; if ((rp = calloc(1, sizeof(*rp))) == NULL) err(1, NULL); rp->id = ++repoid; rp->talid = talid; rp->alarm = getmonotime() + repo_timeout; TAILQ_INIT(&rp->queue); SLIST_INSERT_HEAD(&repos, rp, entry); clock_gettime(CLOCK_MONOTONIC, &rp->start_time); stats.repos++; return rp; } /* * Parse the RRDP state file if it exists and set the session struct * based on that information. */ static struct rrdp_session * rrdp_session_parse(struct rrdprepo *rr) { FILE *f; struct rrdp_session *state; int fd, ln = 0, deltacnt = 0; const char *errstr; char *line = NULL, *file; size_t i, len = 0; ssize_t n; time_t now, weeks; now = time(NULL); if ((state = calloc(1, sizeof(*state))) == NULL) err(1, NULL); file = rrdp_state_filename(rr, 0); if ((fd = open(file, O_RDONLY)) == -1) { if (errno != ENOENT) warn("%s: open state file", rr->basedir); free(file); rr->last_reset = now; return state; } free(file); f = fdopen(fd, "r"); if (f == NULL) err(1, "fdopen"); while ((n = getline(&line, &len, f)) != -1) { if (line[n - 1] == '\n') line[n - 1] = '\0'; switch (ln) { case 0: if ((state->session_id = strdup(line)) == NULL) err(1, NULL); break; case 1: state->serial = strtonum(line, 1, LLONG_MAX, &errstr); if (errstr) { warnx("%s: state file: serial is %s: %s", rr->basedir, errstr, line); goto reset; } break; case 2: rr->last_reset = strtonum(line, 1, LLONG_MAX, &errstr); if (errstr) { warnx("%s: state file: last_reset is %s: %s", rr->basedir, errstr, line); goto reset; } break; case 3: if (strcmp(line, "-") == 0) break; if ((state->last_mod = strdup(line)) == NULL) err(1, NULL); break; default: if (deltacnt >= MAX_RRDP_DELTAS) { warnx("%s: state file: too many deltas: %d", rr->basedir, deltacnt); goto reset; } if ((state->deltas[deltacnt++] = strdup(line)) == NULL) err(1, NULL); break; } ln++; } if (ferror(f)) { warn("%s: error reading state file", rr->basedir); goto reset; } /* check if it's time for reinitialization */ weeks = (now - rr->last_reset) / (86400 * 7); if (now <= rr->last_reset || weeks > RRDP_RANDOM_REINIT_MAX) { warnx("%s: reinitializing", rr->notifyuri); goto reset; } if (arc4random_uniform(1U << RRDP_RANDOM_REINIT_MAX) < (1U << weeks)) { warnx("%s: reinitializing", rr->notifyuri); goto reset; } fclose(f); free(line); return state; reset: fclose(f); free(line); free(state->session_id); free(state->last_mod); for (i = 0; i < sizeof(state->deltas) / sizeof(state->deltas[0]); i++) free(state->deltas[i]); memset(state, 0, sizeof(*state)); rr->last_reset = now; return state; } /* * Carefully write the RRDP session state file back. */ void rrdp_session_save(unsigned int id, struct rrdp_session *state) { struct rrdprepo *rr; char *temp, *file; FILE *f = NULL; int fd, i; rr = rrdp_find(id); if (rr == NULL) errx(1, "non-existent rrdp repo %u", id); file = rrdp_state_filename(rr, 0); temp = rrdp_state_filename(rr, 1); if ((fd = mkostemp(temp, O_CLOEXEC)) == -1) goto fail; (void)fchmod(fd, 0644); f = fdopen(fd, "w"); if (f == NULL) err(1, "fdopen"); /* write session state file out */ if (fprintf(f, "%s\n%lld\n%lld\n", state->session_id, state->serial, (long long)rr->last_reset) < 0) goto fail; if (state->last_mod != NULL) { if (fprintf(f, "%s\n", state->last_mod) < 0) goto fail; } else { if (fprintf(f, "-\n") < 0) goto fail; } for (i = 0; i < MAX_RRDP_DELTAS && state->deltas[i] != NULL; i++) { if (fprintf(f, "%s\n", state->deltas[i]) < 0) goto fail; } if (fclose(f) != 0) { f = NULL; goto fail; } if (rename(temp, file) == -1) { warn("%s: rename %s to %s", rr->basedir, temp, file); unlink(temp); } free(temp); free(file); return; fail: warn("%s: save state to %s", rr->basedir, temp); if (f != NULL) fclose(f); unlink(temp); free(temp); free(file); } /* * Free an rrdp_session pointer. Safe to call with NULL. */ void rrdp_session_free(struct rrdp_session *s) { size_t i; if (s == NULL) return; free(s->session_id); free(s->last_mod); for (i = 0; i < sizeof(s->deltas) / sizeof(s->deltas[0]); i++) free(s->deltas[i]); free(s); } void rrdp_session_buffer(struct ibuf *b, const struct rrdp_session *s) { size_t i; io_str_buffer(b, s->session_id); io_simple_buffer(b, &s->serial, sizeof(s->serial)); io_str_buffer(b, s->last_mod); for (i = 0; i < sizeof(s->deltas) / sizeof(s->deltas[0]); i++) io_str_buffer(b, s->deltas[i]); } struct rrdp_session * rrdp_session_read(struct ibuf *b) { struct rrdp_session *s; size_t i; if ((s = calloc(1, sizeof(*s))) == NULL) err(1, NULL); io_read_str(b, &s->session_id); io_read_buf(b, &s->serial, sizeof(s->serial)); io_read_str(b, &s->last_mod); for (i = 0; i < sizeof(s->deltas) / sizeof(s->deltas[0]); i++) io_read_str(b, &s->deltas[i]); return s; } static struct rrdprepo * rrdp_get(const char *uri) { struct rrdp_session *state; struct rrdprepo *rr; SLIST_FOREACH(rr, &rrdprepos, entry) if (strcmp(rr->notifyuri, uri) == 0) { if (rr->state == REPO_FAILED) return NULL; return rr; } if ((rr = calloc(1, sizeof(*rr))) == NULL) err(1, NULL); rr->id = ++repoid; SLIST_INSERT_HEAD(&rrdprepos, rr, entry); if ((rr->notifyuri = strdup(uri)) == NULL) err(1, NULL); rr->basedir = repo_dir(uri, ".rrdp", 1); RB_INIT(&rr->deleted); /* create base directory */ if (mkpath(rr->basedir) == -1) { warn("mkpath %s", rr->basedir); rrdp_finish(rr->id, 0); return rr; } /* parse state and start the sync */ state = rrdp_session_parse(rr); rrdp_fetch(rr->id, rr->notifyuri, rr->notifyuri, state); rrdp_session_free(state); logx("%s: pulling from %s", rr->notifyuri, "network"); return rr; } /* * Remove RRDP repo and start over. */ void rrdp_clear(unsigned int id) { struct rrdprepo *rr; rr = rrdp_find(id); if (rr == NULL) errx(1, "non-existent rrdp repo %u", id); /* remove rrdp repository contents */ remove_contents(rr->basedir); } /* * Write a file into the temporary RRDP dir but only after checking * its hash (if required). The function also makes sure that the file * tracking is properly adjusted. * Returns 1 on success, 0 if the repo is corrupt, -1 on IO error */ int rrdp_handle_file(unsigned int id, enum publish_type pt, char *uri, char *hash, size_t hlen, char *data, size_t dlen) { struct rrdprepo *rr; struct filepath *fp; ssize_t s; char *fn = NULL; int fd = -1, try = 0, deleted = 0; int flags; rr = rrdp_find(id); if (rr == NULL) errx(1, "non-existent rrdp repo %u", id); if (rr->state == REPO_FAILED) return -1; /* check hash of original file for updates and deletes */ if (pt == PUB_UPD || pt == PUB_DEL) { if (filepath_exists(&rr->deleted, uri)) { warnx("%s: already deleted", uri); return 0; } /* try to open file first in rrdp then in valid repo */ do { free(fn); if ((fn = rrdp_filename(rr, uri, try++)) == NULL) return 0; fd = open(fn, O_RDONLY); } while (fd == -1 && try < 2); if (!valid_filehash(fd, hash, hlen)) { warnx("%s: bad file digest for %s", rr->notifyuri, fn); free(fn); return 0; } free(fn); } /* write new content or mark uri as deleted. */ if (pt == PUB_DEL) { filepath_add(&rr->deleted, uri, 0, 0, 1); } else { fp = filepath_find(&rr->deleted, uri); if (fp != NULL) { filepath_put(&rr->deleted, fp); deleted = 1; } /* add new file to rrdp dir */ if ((fn = rrdp_filename(rr, uri, 0)) == NULL) return 0; if (repo_mkpath(AT_FDCWD, fn) == -1) goto fail; flags = O_WRONLY|O_CREAT|O_TRUNC; if (pt == PUB_ADD && !deleted) flags |= O_EXCL; fd = open(fn, flags, 0644); if (fd == -1) { if (errno == EEXIST) { warnx("%s: duplicate publish element for %s", rr->notifyuri, fn); free(fn); return 0; } warn("open %s", fn); goto fail; } if ((s = write(fd, data, dlen)) == -1) { warn("write %s", fn); goto fail; } close(fd); if ((size_t)s != dlen) /* impossible */ errx(1, "short write %s", fn); free(fn); } return 1; fail: rr->state = REPO_FAILED; if (fd != -1) close(fd); free(fn); return -1; } /* * RSYNC sync finished, either with or without success. */ void rsync_finish(unsigned int id, int ok) { struct rsyncrepo *rr; struct tarepo *tr; tr = ta_find(id); if (tr != NULL) { /* repository changed state already, ignore request */ if (tr->state != REPO_LOADING) return; if (ok) { logx("ta/%s: loaded from network", tr->descr); stats.rsync_repos++; tr->state = REPO_DONE; repo_done(tr, 1); } else { warnx("ta/%s: load from network failed", tr->descr); stats.rsync_fails++; tr->uriidx++; ta_fetch(tr); } return; } rr = rsync_find(id); if (rr == NULL) errx(1, "unknown rsync repo %u", id); /* repository changed state already, ignore request */ if (rr->state != REPO_LOADING) return; if (ok) { logx("%s: loaded from network", rr->basedir); stats.rsync_repos++; rr->state = REPO_DONE; } else { warnx("%s: load from network failed, fallback to cache", rr->basedir); stats.rsync_fails++; rr->state = REPO_FAILED; /* clear rsync repo since it failed */ remove_contents(rr->basedir); } repo_done(rr, ok); } /* * RRDP sync finished, either with or without success. */ void rrdp_finish(unsigned int id, int ok) { struct rrdprepo *rr; rr = rrdp_find(id); if (rr == NULL) errx(1, "unknown RRDP repo %u", id); /* repository changed state already, ignore request */ if (rr->state != REPO_LOADING) return; if (ok) { logx("%s: loaded from network", rr->notifyuri); stats.rrdp_repos++; rr->state = REPO_DONE; } else { warnx("%s: load from network failed, fallback to %s", rr->notifyuri, nofetch ? "cache" : "rsync"); stats.rrdp_fails++; rr->state = REPO_FAILED; /* clear the RRDP repo since it failed */ remove_contents(rr->basedir); /* also clear the list of deleted files */ filepath_free(&rr->deleted); } repo_done(rr, ok); } /* * Handle responses from the http process. For TA file, either rename * or delete the temporary file. For RRDP requests relay the request * over to the rrdp process. */ void http_finish(unsigned int id, enum http_result res, const char *last_mod) { struct tarepo *tr; tr = ta_find(id); if (tr == NULL) { /* not a TA fetch therefore RRDP */ rrdp_http_done(id, res, last_mod); return; } /* repository changed state already, ignore request */ if (tr->state != REPO_LOADING) return; /* Move downloaded TA file into place, or unlink on failure. */ if (res == HTTP_OK) { logx("ta/%s: loaded from network", tr->descr); tr->state = REPO_DONE; stats.http_repos++; repo_done(tr, 1); } else { remove_contents(tr->basedir); tr->uriidx++; warnx("ta/%s: load from network failed", tr->descr); ta_fetch(tr); } } /* * Look up a trust anchor, queueing it for download if not found. */ struct repo * ta_lookup(int id, struct tal *tal) { struct repo *rp; if (tal->num_uris == 0) errx(1, "TAL %s has no URI", tal->descr); /* Look up in repository table. (Lookup should actually fail here) */ SLIST_FOREACH(rp, &repos, entry) { if (strcmp(rp->repouri, tal->uri[0]) == 0) return rp; } rp = repo_alloc(id); rp->basedir = repo_dir(tal->descr, "ta", 0); if ((rp->repouri = strdup(tal->uri[0])) == NULL) err(1, NULL); /* check if sync disabled ... */ if (noop) { logx("%s: using cache", rp->basedir); entityq_flush(&rp->queue, rp); return rp; } /* try to create base directory */ if (mkpath(rp->basedir) == -1) warn("mkpath %s", rp->basedir); rp->ta = ta_get(tal); /* need to check if it was already loaded */ if (repo_state(rp) != REPO_LOADING) entityq_flush(&rp->queue, rp); return rp; } /* * Look up a repository, queueing it for discovery if not found. */ struct repo * repo_lookup(int talid, const char *uri, const char *notify) { struct repo *rp; char *repouri; if ((repouri = rsync_base_uri(uri)) == NULL) errx(1, "bad caRepository URI: %s", uri); /* Look up in repository table. */ SLIST_FOREACH(rp, &repos, entry) { if (strcmp(rp->repouri, repouri) != 0) continue; if (rp->notifyuri != NULL) { if (notify == NULL) continue; if (strcmp(rp->notifyuri, notify) != 0) continue; } else if (notify != NULL) continue; /* found matching repo */ free(repouri); return rp; } rp = repo_alloc(talid); rp->basedir = repo_dir(repouri, NULL, 0); rp->repouri = repouri; if (notify != NULL) if ((rp->notifyuri = strdup(notify)) == NULL) err(1, NULL); if (++talrepocnt[talid] >= MAX_REPO_PER_TAL) { if (talrepocnt[talid] == MAX_REPO_PER_TAL) warnx("too many repositories under %s", tals[talid]); nofetch = 1; } /* check if sync disabled ... */ if (noop || nofetch) { logx("%s: using cache", rp->basedir); entityq_flush(&rp->queue, rp); return rp; } /* try to create base directory */ if (mkpath(rp->basedir) == -1) warn("mkpath %s", rp->basedir); /* ... else try RRDP first if available then rsync */ if (notify != NULL) rp->rrdp = rrdp_get(notify); if (rp->rrdp == NULL) rp->rsync = rsync_get(uri, rp->basedir); /* need to check if it was already loaded */ if (repo_state(rp) != REPO_LOADING) entityq_flush(&rp->queue, rp); return rp; } /* * Find repository by identifier. */ struct repo * repo_byid(unsigned int id) { struct repo *rp; SLIST_FOREACH(rp, &repos, entry) { if (rp->id == id) return rp; } return NULL; } /* * Find repository by base path. */ static struct repo * repo_bypath(const char *path) { struct repo *rp; SLIST_FOREACH(rp, &repos, entry) { if (strcmp(rp->basedir, path) == 0) return rp; } return NULL; } /* * Return the repository base or alternate directory. * Returned string must be freed by caller. */ char * repo_basedir(const struct repo *rp, int wantvalid) { char *path = NULL; if (!wantvalid) { if (rp->ta) { if ((path = strdup(rp->ta->basedir)) == NULL) err(1, NULL); } else if (rp->rsync) { if ((path = strdup(rp->rsync->basedir)) == NULL) err(1, NULL); } else if (rp->rrdp) { path = rrdp_filename(rp->rrdp, rp->repouri, 0); } else path = NULL; /* only valid repo available */ } else if (rp->basedir != NULL) { if ((path = strdup(rp->basedir)) == NULL) err(1, NULL); } return path; } /* * Return the repository identifier. */ unsigned int repo_id(const struct repo *rp) { return rp->id; } /* * Return the repository URI. */ const char * repo_uri(const struct repo *rp) { return rp->repouri; } /* * Return the repository URI. */ void repo_fetch_uris(const struct repo *rp, const char **carepo, const char **notifyuri) { *carepo = rp->repouri; *notifyuri = rp->notifyuri; } /* * Return 1 if repository is synced else 0. */ int repo_synced(const struct repo *rp) { if (repo_state(rp) == REPO_DONE && !(rp->rrdp == NULL && rp->rsync == NULL && rp->ta == NULL)) return 1; return 0; } /* * Return the protocol string "rrdp", "rsync", "https" which was used to sync. * Result is only correct if repository was properly synced. */ const char * repo_proto(const struct repo *rp) { if (rp->ta != NULL) { const struct tarepo *tr = rp->ta; if (tr->uriidx < tr->num_uris && strncasecmp(tr->uri[tr->uriidx], RSYNC_PROTO, RSYNC_PROTO_LEN) == 0) return "rsync"; else return "https"; } if (rp->rrdp != NULL) return "rrdp"; return "rsync"; } /* * Return the repository tal ID. */ int repo_talid(const struct repo *rp) { return rp->talid; } int repo_queued(struct repo *rp, struct entity *p) { if (repo_state(rp) == REPO_LOADING) { TAILQ_INSERT_TAIL(&rp->queue, p, entries); return 1; } return 0; } static void repo_fail(struct repo *rp) { /* reset the alarm since code may fallback to rsync */ rp->alarm = getmonotime() + repo_timeout; if (rp->ta) http_finish(rp->ta->id, HTTP_FAILED, NULL); else if (rp->rsync) rsync_finish(rp->rsync->id, 0); else if (rp->rrdp) rrdp_finish(rp->rrdp->id, 0); else errx(1, "%s: bad repo", rp->repouri); } static void repo_abort(struct repo *rp) { /* reset the alarm */ rp->alarm = getmonotime() + repo_timeout; if (rp->rsync) { warnx("%s: synchronisation timeout", rp->repouri); rsync_abort(rp->rsync->id); } else if (rp->rrdp) { warnx("%s: synchronisation timeout", rp->notifyuri); rrdp_abort(rp->rrdp->id); } repo_fail(rp); } int repo_check_timeout(int timeout) { struct repo *rp; time_t now; int diff; now = getmonotime(); /* check against our runtime deadline first */ if (deadline != 0) { if (deadline <= now) { warnx("deadline reached, giving up on repository sync"); nofetch = 1; /* clear deadline since nofetch is set */ deadline = 0; /* increase now enough so that all pending repos fail */ now += repo_timeout; } else { diff = deadline - now; diff *= 1000; if (timeout == INFTIM || diff < timeout) timeout = diff; } } /* Look up in repository table. (Lookup should actually fail here) */ SLIST_FOREACH(rp, &repos, entry) { if (repo_state(rp) == REPO_LOADING) { if (rp->alarm <= now) repo_abort(rp); else { diff = rp->alarm - now; diff *= 1000; if (timeout == INFTIM || diff < timeout) timeout = diff; } } } return timeout; } /* * Update repo-specific stats when files are going to be moved * from DIR_TEMP to DIR_VALID. */ void repostats_new_files_inc(struct repo *rp, const char *file) { if (strncmp(file, ".rsync/", strlen(".rsync/")) == 0 || strncmp(file, ".rrdp/", strlen(".rrdp/")) == 0) rp->repostats.new_files++; } /* * Update stats object of repository depending on rtype and subtype. */ void repo_stat_inc(struct repo *rp, int talid, enum rtype type, enum stype subtype) { if (rp == NULL) return; rp->stats_used[talid] = 1; switch (type) { case RTYPE_CER: if (subtype == STYPE_OK) rp->stats[talid].certs++; if (subtype == STYPE_FAIL) rp->stats[talid].certs_fail++; if (subtype == STYPE_BGPSEC) { rp->stats[talid].certs--; rp->stats[talid].brks++; } break; case RTYPE_MFT: if (subtype == STYPE_OK) rp->stats[talid].mfts++; if (subtype == STYPE_FAIL) rp->stats[talid].mfts_fail++; if (subtype == STYPE_SEQNUM_GAP) rp->stats[talid].mfts_gap++; break; case RTYPE_ROA: switch (subtype) { case STYPE_OK: rp->stats[talid].roas++; break; case STYPE_FAIL: rp->stats[talid].roas_fail++; break; case STYPE_INVALID: rp->stats[talid].roas_invalid++; break; case STYPE_TOTAL: rp->stats[talid].vrps++; break; case STYPE_UNIQUE: rp->stats[talid].vrps_uniqs++; break; case STYPE_DEC_UNIQUE: rp->stats[talid].vrps_uniqs--; break; default: break; } break; case RTYPE_ASPA: switch (subtype) { case STYPE_OK: rp->stats[talid].aspas++; break; case STYPE_FAIL: rp->stats[talid].aspas_fail++; break; case STYPE_INVALID: rp->stats[talid].aspas_invalid++; break; case STYPE_TOTAL: rp->stats[talid].vaps++; break; case STYPE_UNIQUE: rp->stats[talid].vaps_uniqs++; break; case STYPE_DEC_UNIQUE: rp->stats[talid].vaps_uniqs--; break; case STYPE_PROVIDERS: rp->stats[talid].vaps_pas++; break; case STYPE_OVERFLOW: rp->stats[talid].vaps_overflowed++; break; default: break; } break; case RTYPE_SPL: switch (subtype) { case STYPE_OK: rp->stats[talid].spls++; break; case STYPE_FAIL: rp->stats[talid].spls_fail++; break; case STYPE_INVALID: rp->stats[talid].spls_invalid++; break; case STYPE_TOTAL: rp->stats[talid].vsps++; break; case STYPE_UNIQUE: rp->stats[talid].vsps_uniqs++; break; case STYPE_DEC_UNIQUE: rp->stats[talid].vsps_uniqs--; break; default: break; } break; case RTYPE_CRL: rp->stats[talid].crls++; break; case RTYPE_GBR: rp->stats[talid].gbrs++; break; case RTYPE_TAK: rp->stats[talid].taks++; break; default: break; } } void repo_tal_stats_collect(void (*cb)(const struct repo *, const struct repotalstats *, void *), int talid, void *arg) { struct repo *rp; SLIST_FOREACH(rp, &repos, entry) { if (rp->stats_used[talid]) cb(rp, &rp->stats[talid], arg); } } void repo_stats_collect(void (*cb)(const struct repo *, const struct repostats *, void *), void *arg) { struct repo *rp; SLIST_FOREACH(rp, &repos, entry) cb(rp, &rp->repostats, arg); } /* * Delayed delete of files from RRDP. Since RRDP has no security built-in * this code needs to check if this RRDP repository is actually allowed to * remove the file referenced by the URI. */ static void repo_cleanup_rrdp(struct filepath_tree *tree) { struct repo *rp; struct rrdprepo *rr; struct filepath *fp, *nfp; char *fn; SLIST_FOREACH(rp, &repos, entry) { if (rp->rrdp == NULL) continue; rr = (struct rrdprepo *)rp->rrdp; RB_FOREACH_SAFE(fp, filepath_tree, &rr->deleted, nfp) { if (!rrdp_uri_valid(rr, fp->file)) { warnx("%s: external URI %s", rr->notifyuri, fp->file); filepath_put(&rr->deleted, fp); continue; } /* try to remove file from rrdp repo ... */ fn = rrdp_filename(rr, fp->file, 0); if (unlink(fn) == -1) { if (errno != ENOENT) warn("unlink %s", fn); } else { if (verbose > 1) logx("deleted %s", fn); rp->repostats.del_files++; } free(fn); /* ... and from the valid repository if unused. */ fn = rrdp_filename(rr, fp->file, 1); if (!filepath_exists(tree, fn)) { if (unlink(fn) == -1) { if (errno != ENOENT) warn("unlink %s", fn); } else { if (verbose > 1) logx("deleted %s", fn); rp->repostats.del_files++; } } else warnx("%s: referenced file supposed to be " "deleted", fn); free(fn); filepath_put(&rr->deleted, fp); } } } /* * All files in tree are valid and should be moved to the valid repository * if not already there. Rename the files to the new path and readd the * filepath entry with the new path if successful. */ static void repo_move_valid(struct filepath_tree *tree) { struct filepath *fp, *nfp, *ofp; size_t rsyncsz = strlen(".rsync/"); size_t rrdpsz = strlen(".rrdp/"); size_t tasz = strlen(".ta/"); char *fn, *base; RB_FOREACH_SAFE(fp, filepath_tree, tree, nfp) { if (strncmp(fp->file, ".rsync/", rsyncsz) != 0 && strncmp(fp->file, ".rrdp/", rrdpsz) != 0 && strncmp(fp->file, ".ta/", tasz) != 0) continue; /* not a temporary file path */ if (strncmp(fp->file, ".rsync/", rsyncsz) == 0) { fn = fp->file + rsyncsz; } else if (strncmp(fp->file, ".ta/", tasz) == 0) { fn = fp->file + 1; /* just skip the '.' */ } else { base = strchr(fp->file + rrdpsz, '/'); assert(base != NULL); fn = base + 1; /* * Adjust file last modification time in order to * minimize RSYNC synchronization load after transport * failover. * While serializing RRDP datastructures to disk, set * the last modified timestamp to the CMS signing-time, * the X.509 notBefore, or CRL lastUpdate timestamp. */ if (fp->mtime != 0) { int ret; struct timespec ts[2]; ts[0].tv_nsec = UTIME_OMIT; ts[1].tv_sec = fp->mtime; ts[1].tv_nsec = 0; ret = utimensat(AT_FDCWD, fp->file, ts, 0); if (ret == -1) { warn("utimensat %s", fp->file); continue; } } } if (repo_mkpath(AT_FDCWD, fn) == -1) continue; /* switch filepath node to new path */ RB_REMOVE(filepath_tree, tree, fp); base = fp->file; if ((fp->file = strdup(fn)) == NULL) err(1, NULL); again: if ((ofp = RB_INSERT(filepath_tree, tree, fp)) != NULL) { if (ofp->talmask == 0) { /* conflicting path is not valid, drop it */ filepath_put(tree, ofp); goto again; } if (fp->talmask != 0) { warnx("%s: file already present in " "validated cache", fp->file); } free(fp->file); free(fp); free(base); continue; } if (rename(base, fp->file) == -1) warn("rename to %s", fp->file); free(base); } } struct fts_state { enum { BASE_DIR, RSYNC_DIR, TA_DIR, RRDP_DIR } type; struct repo *rp; } fts_state; static const struct rrdprepo * repo_is_rrdp(struct repo *rp) { /* check for special pointers first these are not a repository */ if (rp != NULL && rp->rrdp != NULL) return rp->rrdp->state == REPO_DONE ? rp->rrdp : NULL; return NULL; } static inline char * skip_dotslash(char *in) { if (memcmp(in, "./", 2) == 0) return in + 2; return in; } static void repo_cleanup_entry(FTSENT *e, struct filepath_tree *tree, int cachefd) { const struct rrdprepo *rr; char *path; path = skip_dotslash(e->fts_path); switch (e->fts_info) { case FTS_NSOK: if (filepath_exists(tree, path)) { e->fts_parent->fts_number++; break; } if (fts_state.type == RRDP_DIR && fts_state.rp != NULL) { e->fts_parent->fts_number++; /* handle rrdp .state files explicitly */ if (e->fts_level == 3 && strcmp(e->fts_name, ".state") == 0) break; /* can't delete these extra files */ fts_state.rp->repostats.extra_files++; if (verbose > 1) logx("superfluous %s", path); break; } rr = repo_is_rrdp(fts_state.rp); if (rr != NULL) { struct stat st; char *fn; if (asprintf(&fn, "%s/%s", rr->basedir, path) == -1) err(1, NULL); /* * If the file exists in the rrdp dir * that file is newer and needs to be kept * so unlink this file instead of moving * it over the file in the rrdp dir. */ if (fstatat(cachefd, fn, &st, 0) == 0 && S_ISREG(st.st_mode)) { free(fn); goto unlink; } if (repo_mkpath(cachefd, fn) == 0) { if (renameat(AT_FDCWD, e->fts_accpath, cachefd, fn) == -1) warn("rename %s to %s", path, fn); else if (verbose > 1) logx("moved %s", path); fts_state.rp->repostats.extra_files++; } free(fn); } else { unlink: if (unlink(e->fts_accpath) == -1) { warn("unlink %s", path); } else if (fts_state.type == RSYNC_DIR || fts_state.type == TA_DIR) { /* no need to keep rsync or ta files */ if (verbose > 1) logx("deleted superfluous %s", path); if (fts_state.rp != NULL) fts_state.rp->repostats.del_extra_files++; else stats.repo_stats.del_extra_files++; } else { if (verbose > 1) logx("deleted %s", path); if (fts_state.rp != NULL) fts_state.rp->repostats.del_files++; else stats.repo_stats.del_files++; } } break; case FTS_D: if (e->fts_level == FTS_ROOTLEVEL) { fts_state.type = BASE_DIR; fts_state.rp = NULL; } if (e->fts_level == 1) { /* rpki.example.org or .rrdp / .rsync / .ta */ if (strcmp(".rsync", e->fts_name) == 0) fts_state.type = RSYNC_DIR; else if (strcmp(".ta", e->fts_name) == 0) fts_state.type = TA_DIR; else if (strcmp(".rrdp", e->fts_name) == 0) fts_state.type = RRDP_DIR; else fts_state.type = BASE_DIR; fts_state.rp = NULL; } if (e->fts_level == 2) { /* rpki.example.org/repository or .rrdp/hashdir */ if (fts_state.type == BASE_DIR) fts_state.rp = repo_bypath(path); if (fts_state.type == TA_DIR) fts_state.rp = repo_bypath(path + 1); /* * special handling for rrdp directories, * clear them if they are not used anymore but * only if rrdp is active. * Look them up just using the hash. */ if (fts_state.type == RRDP_DIR) fts_state.rp = repo_rrdp_bypath(path); } if (e->fts_level == 3 && fts_state.type == RSYNC_DIR) { /* .rsync/rpki.example.org/repository */ fts_state.rp = repo_bypath(path + strlen(".rsync/")); } break; case FTS_DP: if (e->fts_level == FTS_ROOTLEVEL) break; if (e->fts_level == 1) { /* do not remove .rsync and .rrdp */ fts_state.rp = NULL; if (fts_state.type == RRDP_DIR || fts_state.type == RSYNC_DIR || fts_state.type == TA_DIR) break; } e->fts_parent->fts_number += e->fts_number; if (e->fts_number == 0) { if (rmdir(e->fts_accpath) == -1) warn("rmdir %s", path); if (fts_state.rp != NULL) fts_state.rp->repostats.del_dirs++; else stats.repo_stats.del_dirs++; } break; case FTS_SL: case FTS_SLNONE: warnx("symlink %s", path); if (unlink(e->fts_accpath) == -1) warn("unlink %s", path); stats.repo_stats.del_extra_files++; break; case FTS_NS: case FTS_ERR: if (e->fts_errno == ENOENT && e->fts_level == FTS_ROOTLEVEL) break; warnx("fts_read %s: %s", path, strerror(e->fts_errno)); break; default: warnx("fts_read %s: unhandled[%x]", path, e->fts_info); break; } } void repo_cleanup(struct filepath_tree *tree, int cachefd) { char *argv[2] = { ".", NULL }; FTS *fts; FTSENT *e; /* first move temp files which have been used to valid dir */ repo_move_valid(tree); /* then delete files requested by rrdp */ repo_cleanup_rrdp(tree); if ((fts = fts_open(argv, FTS_PHYSICAL | FTS_NOSTAT, NULL)) == NULL) err(1, "fts_open"); errno = 0; while ((e = fts_read(fts)) != NULL) { repo_cleanup_entry(e, tree, cachefd); errno = 0; } if (errno) err(1, "fts_read"); if (fts_close(fts) == -1) err(1, "fts_close"); } void repo_free(void) { struct repo *rp; while ((rp = SLIST_FIRST(&repos)) != NULL) { SLIST_REMOVE_HEAD(&repos, entry); free(rp->repouri); free(rp->notifyuri); free(rp->basedir); free(rp); } ta_free(); rrdp_free(); rsync_free(); } /* * Remove all files and directories under base. * Do not remove base directory itself and the .state file. */ static void remove_contents(char *base) { char *argv[2] = { base, NULL }; FTS *fts; FTSENT *e; if ((fts = fts_open(argv, FTS_PHYSICAL | FTS_NOSTAT, NULL)) == NULL) err(1, "fts_open"); errno = 0; while ((e = fts_read(fts)) != NULL) { switch (e->fts_info) { case FTS_NSOK: case FTS_SL: case FTS_SLNONE: if (e->fts_level == 1 && strcmp(e->fts_name, ".state") == 0) break; if (unlink(e->fts_accpath) == -1) warn("unlink %s", e->fts_path); break; case FTS_D: break; case FTS_DP: /* keep root directory */ if (e->fts_level == FTS_ROOTLEVEL) break; if (rmdir(e->fts_accpath) == -1) warn("rmdir %s", e->fts_path); break; case FTS_NS: case FTS_ERR: warnx("fts_read %s: %s", e->fts_path, strerror(e->fts_errno)); break; default: warnx("unhandled[%x] %s", e->fts_info, e->fts_path); break; } errno = 0; } if (errno) err(1, "fts_read"); if (fts_close(fts) == -1) err(1, "fts_close"); }