Introduce an openssl(1) certhash command.

This is effectively a reimplementation of the functionality provided by
the previously removed c_rehash Perl script. The c_rehash script had a
number of known issues, including the fact that it needs to run openssl(1)
multiple times and that it starts by removing all symlinks before
putting them back, creating atomicity issues/race conditions, even when
nothing has changed.

certhash is self-contained and is intended to be stable - no changes
should be made unless something has actually changed. This means it can
be run regularly in a production environment without causing certificate
lookup failures.

Further testing and improvements will happen in tree.

Discussed with tedu@

3 files changed, 685 insertions, 9 deletions

diff --git a/usr.bin/openssl/Makefile b/usr.bin/openssl/Makefile
index 1619163a136..04a24c8c59b 100644
--- a/usr.bin/openssl/Makefile
+++ b/usr.bin/openssl/Makefile
@@ -1,4 +1,4 @@
-#	$OpenBSD: Makefile,v 1.4 2014/12/03 22:16:02 bcook Exp $
+#	$OpenBSD: Makefile,v 1.5 2015/02/10 15:29:34 jsing Exp $
 
 PROG=	openssl
 LDADD=	-lssl -lcrypto
@@ -17,12 +17,12 @@ CFLAGS+= -Wunused
 
 CFLAGS+= -DLIBRESSL_INTERNAL
 
-SRCS=	apps.c apps_posix.c asn1pars.c ca.c ciphers.c cms.c crl.c crl2p7.c \
-	dgst.c dh.c dhparam.c dsa.c dsaparam.c ec.c ecparam.c enc.c engine.c \
-	errstr.c gendh.c gendsa.c genpkey.c genrsa.c nseq.c ocsp.c openssl.c \
-	passwd.c pkcs12.c pkcs7.c pkcs8.c pkey.c pkeyparam.c pkeyutl.c prime.c \
-	rand.c req.c rsa.c rsautl.c s_cb.c s_client.c s_server.c s_socket.c \
-	s_time.c sess_id.c smime.c speed.c spkac.c ts.c verify.c version.c \
-	x509.c
+SRCS=	apps.c apps_posix.c asn1pars.c ca.c certhash.c ciphers.c cms.c crl.c \
+	crl2p7.c dgst.c dh.c dhparam.c dsa.c dsaparam.c ec.c ecparam.c enc.c \
+	engine.c errstr.c gendh.c gendsa.c genpkey.c genrsa.c nseq.c ocsp.c \
+	openssl.c passwd.c pkcs12.c pkcs7.c pkcs8.c pkey.c pkeyparam.c \
+	pkeyutl.c prime.c rand.c req.c rsa.c rsautl.c s_cb.c s_client.c \
+	s_server.c s_socket.c s_time.c sess_id.c smime.c speed.c spkac.c ts.c \
+	verify.c version.c x509.c
 
 .include <bsd.prog.mk>
diff --git a/usr.bin/openssl/certhash.c b/usr.bin/openssl/certhash.c
new file mode 100644
index 00000000000..39e8324ea0b
--- /dev/null
+++ b/usr.bin/openssl/certhash.c
@@ -0,0 +1,674 @@
+/*
+ * Copyright (c) 2014, 2015 Joel Sing <jsing@openbsd.org>
+ *
+ * Permission to use, copy, modify, and distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#include <sys/param.h>
+#include <sys/types.h>
+#include <sys/limits.h>
+#include <sys/stat.h>
+
+#include <errno.h>
+#include <dirent.h>
+#include <fcntl.h>
+#include <stdio.h>
+#include <string.h>
+#include <unistd.h>
+
+#include <openssl/bio.h>
+#include <openssl/evp.h>
+#include <openssl/pem.h>
+#include <openssl/x509.h>
+
+#include "apps.h"
+
+static struct {
+	int dryrun;
+	int verbose;
+} certhash_config;
+
+struct option certhash_options[] = {
+	{
+		.name = "n",
+		.desc = "Perform a dry-run - do not make any changes",
+		.type = OPTION_FLAG,
+		.opt.flag = &certhash_config.dryrun,
+	},
+	{
+		.name = "v",
+		.desc = "Verbose",
+		.type = OPTION_FLAG,
+		.opt.flag = &certhash_config.verbose,
+	},
+	{ NULL },
+};
+
+struct hashinfo {
+	char *filename;
+	char *target;
+	unsigned long hash;
+	unsigned int index;
+	unsigned char fingerprint[EVP_MAX_MD_SIZE];
+	int is_crl;
+	int is_dup;
+	int exists;
+	int changed;
+	struct hashinfo *reference;
+	struct hashinfo *next;
+};
+
+static struct hashinfo *
+hashinfo(const char *filename, unsigned long hash, unsigned char *fingerprint)
+{
+	struct hashinfo *hi;
+
+	if ((hi = calloc(1, sizeof(*hi))) == NULL)
+		return (NULL);
+	if (filename != NULL) {
+		if ((hi->filename = strdup(filename)) == NULL) {
+			free(hi);
+			return (NULL);
+		}
+	}
+	hi->hash = hash;
+	if (fingerprint != NULL)
+		memcpy(hi->fingerprint, fingerprint, sizeof(hi->fingerprint));
+
+	return (hi);
+}
+
+static void
+hashinfo_free(struct hashinfo *hi)
+{
+	free(hi->filename);
+	free(hi->target);
+	free(hi);
+}
+
+#ifdef DEBUG
+static void
+hashinfo_print(struct hashinfo *hi)
+{
+	int i;
+
+	printf("hashinfo %s %08lx %u %i\n", hi->filename, hi->hash,
+	    hi->index, hi->is_crl);
+	for (i = 0; i < (int)EVP_MAX_MD_SIZE; i++) {
+		printf("%02X%c", hi->fingerprint[i],
+		    (i + 1 == (int)EVP_MAX_MD_SIZE) ? '\n' : ':');
+	}
+}
+#endif
+
+static int
+hashinfo_compare(const void *a, const void *b)
+{
+	struct hashinfo *hia = *(struct hashinfo **)a;
+	struct hashinfo *hib = *(struct hashinfo **)b;
+	int rv;
+
+	rv = hia->hash - hib->hash;
+	if (rv != 0)
+		return (rv);
+	rv = bcmp(hia->fingerprint, hib->fingerprint, sizeof(hia->fingerprint));
+	if (rv != 0)
+		return (rv);
+	return strcmp(hia->filename, hib->filename);
+}
+
+static struct hashinfo *
+hashinfo_chain(struct hashinfo *head, struct hashinfo *entry)
+{
+	struct hashinfo *hi = head;
+
+	if (hi == NULL)
+		return (entry);
+	while (hi->next != NULL)
+		hi = hi->next;
+	hi->next = entry;
+
+	return (head);
+}
+
+static void
+hashinfo_chain_free(struct hashinfo *hi)
+{
+	struct hashinfo *next;
+
+	while (hi != NULL) {
+		next = hi->next;
+		hashinfo_free(hi);
+		hi = next;
+	}
+}
+
+static size_t
+hashinfo_chain_length(struct hashinfo *hi)
+{
+	int len = 0;
+
+	while (hi != NULL) {
+		len++;
+		hi = hi->next;
+	}
+	return (len);
+}
+
+static int
+hashinfo_chain_sort(struct hashinfo **head)
+{
+	struct hashinfo **list, *entry;
+	size_t len;
+	int i;
+
+	if (*head == NULL)
+		return (0);
+
+	len = hashinfo_chain_length(*head);
+	if ((list = reallocarray(NULL, len, sizeof(struct hashinfo *))) == NULL)
+		return (-1);
+
+	for (entry = *head, i = 0; entry != NULL; entry = entry->next, i++)
+		list[i] = entry;
+	qsort(list, len, sizeof(struct hashinfo *), hashinfo_compare);
+
+	*head = entry = list[0];
+	for (i = 1; i < len; i++) {
+		entry->next = list[i];
+		entry = list[i];
+	}
+	entry->next = NULL;
+
+	return (0);
+}
+
+static char *
+hashinfo_linkname(struct hashinfo *hi)
+{
+	char *filename;
+
+	if (asprintf(&filename, "%08lx.%s%u", hi->hash,
+	    (hi->is_crl ? "r" : ""), hi->index) == -1)
+		return (NULL);
+
+	return (filename);
+}
+
+static int
+filename_is_hash(const char *filename)
+{
+	const char *p = filename;
+
+	while ((*p >= '0' && *p <= '9') || (*p >= 'a' && *p <= 'f'))
+		p++;
+	if (*p++ != '.')
+		return (0);
+	if (*p == 'r')		/* CRL format. */
+		p++;
+	while (*p >= '0' && *p <= '9')
+		p++;
+	if (*p != '\0')
+		return (0);
+
+	return (1);
+}
+
+static int
+filename_is_pem(const char *filename)
+{
+	const char *q, *p = filename;
+
+	if ((q = strchr(p, '\0')) == NULL)
+		return (0);
+	if ((q - p) < 4)
+		return (0);
+	if (strncmp((q - 4), ".pem", 4) != 0)
+		return (0);
+
+	return (1);
+}
+
+static struct hashinfo *
+hashinfo_from_linkname(const char *linkname, const char *target)
+{
+	struct hashinfo *hi = NULL;
+	const char *errstr;
+	char *l, *p, *ep;
+	long long val;
+
+	if ((l = strdup(linkname)) == NULL)
+		goto err;
+	if ((p = strchr(l, '.')) == NULL)
+		goto err;
+	*p++ = '\0';
+
+	if ((hi = hashinfo(linkname, 0, NULL)) == NULL)
+		goto err;
+	if ((hi->target = strdup(target)) == NULL)
+		goto err;
+
+	errno = 0;
+	val = strtoll(l, &ep, 16);
+	if (l[0] == '\0' || *ep != '\0')
+		goto err;
+	if (errno == ERANGE && (val == LONG_MAX || val == LONG_MIN))
+		goto err;
+	if (val < 0 || val > ULONG_MAX)
+		goto err;
+	hi->hash = (unsigned long)val;
+
+	if (*p == 'r') {
+		hi->is_crl = 1;
+		p++;
+	}
+
+	val = strtonum(p, 0, 0xffffffff, &errstr);
+	if (errstr != NULL)
+		goto err;
+
+	hi->index = (unsigned int)val;
+
+	goto done;
+
+err:
+	hashinfo_free(hi);
+	hi = NULL;
+
+done:
+	free(l);
+
+	return (hi);
+}
+
+static struct hashinfo *
+certhash_cert(BIO *bio, const char *filename)
+{
+	unsigned char fingerprint[EVP_MAX_MD_SIZE];
+	struct hashinfo *hi = NULL;
+	const EVP_MD *digest;
+	X509 *cert = NULL;
+	unsigned long hash;
+	unsigned int len;
+
+	if ((cert = PEM_read_bio_X509(bio, NULL, NULL, NULL)) == NULL)
+		goto err;
+
+	hash = X509_subject_name_hash(cert);
+
+	digest = EVP_sha256();
+	if (X509_digest(cert, digest, fingerprint, &len) != 1) {
+		fprintf(stderr, "out of memory\n");
+		goto err;
+	}
+
+	hi = hashinfo(filename, hash, fingerprint);
+
+err:
+	X509_free(cert);
+
+	return (hi);
+}
+
+static struct hashinfo *
+certhash_crl(BIO *bio, const char *filename)
+{
+	unsigned char fingerprint[EVP_MAX_MD_SIZE];
+	struct hashinfo *hi = NULL;
+	const EVP_MD *digest;
+	X509_CRL *crl = NULL;
+	unsigned long hash;
+	unsigned int len;
+
+	if ((crl = PEM_read_bio_X509_CRL(bio, NULL, NULL, NULL)) == NULL)
+		return (NULL);
+
+	hash = X509_NAME_hash(X509_CRL_get_issuer(crl));
+
+	digest = EVP_sha256();
+	if (X509_CRL_digest(crl, digest, fingerprint, &len) != 1) {
+		fprintf(stderr, "out of memory\n");
+		goto err;
+	}
+
+	hi = hashinfo(filename, hash, fingerprint);
+
+err:
+	X509_CRL_free(crl);
+
+	return (hi);
+}
+
+static int
+certhash_addlink(struct hashinfo **links, struct hashinfo *hi)
+{
+	struct hashinfo *link = NULL;
+
+	if ((link = hashinfo(NULL, hi->hash, hi->fingerprint)) == NULL)
+		goto err;
+
+printf("hi->is_crl = %i\n", hi->is_crl);
+	if ((link->filename = hashinfo_linkname(hi)) == NULL)
+		goto err;
+printf("filename = %s\n", link->filename);
+
+	link->reference = hi;
+	link->changed = 1;
+	*links = hashinfo_chain(*links, link);
+	hi->reference = link;
+
+	return (0);
+
+err:
+	hashinfo_free(link);
+	return (-1);
+}
+
+static void
+certhash_findlink(struct hashinfo *links, struct hashinfo *hi)
+{
+	struct hashinfo *link;
+
+	for (link = links; link != NULL; link = link->next) {
+		if (link->is_crl == hi->is_crl &&
+		    link->hash == hi->hash &&
+		    link->index == hi->index &&
+		    link->reference == NULL) {
+			link->reference = hi;
+			if (link->target == NULL ||
+			    strcmp(link->target, hi->filename) != 0)
+				link->changed = 1;
+			hi->reference = link;
+			break;
+		}
+	}
+}
+
+static void
+certhash_index(struct hashinfo *head, const char *name)
+{
+	struct hashinfo *last, *entry;
+	int index = 0;
+
+	last = NULL;
+	for (entry = head; entry != NULL; entry = entry->next) {
+		if (last != NULL) {
+			if (entry->hash == last->hash) {
+				if (bcmp(entry->fingerprint, last->fingerprint,
+				    sizeof(entry->fingerprint)) == 0) {
+					fprintf(stderr, "WARNING: duplicate %s "
+					    "in %s (using %s), ignoring...\n",
+					    name, entry->filename,
+					    last->filename);
+					entry->is_dup = 1;
+					continue;
+				}
+				index++;
+			} else {
+				index = 0;
+			}
+		}
+		entry->index = index;
+		last = entry;
+	}
+}
+
+static int
+certhash_merge(struct hashinfo **links, struct hashinfo **certs,
+    struct hashinfo **crls)
+{
+	struct hashinfo *cert, *crl;
+
+	/* Pass 1 - sort and index entries. */
+	if (hashinfo_chain_sort(certs) == -1)
+		return (-1);
+	if (hashinfo_chain_sort(crls) == -1)
+		return (-1);
+	certhash_index(*certs, "certificate");
+	certhash_index(*crls, "CRL");
+
+	/* Pass 2 - map to existing links. */
+	for (cert = *certs; cert != NULL; cert = cert->next) {
+		if (cert->is_dup == 1)
+			continue;
+		certhash_findlink(*links, cert);
+	}
+	for (crl = *crls; crl != NULL; crl = crl->next) {
+		if (crl->is_dup == 1)
+			continue;
+		certhash_findlink(*links, crl);
+	}
+
+	/* Pass 3 - determine missing links. */
+	for (cert = *certs; cert != NULL; cert = cert->next) {
+		if (cert->is_dup == 1 || cert->reference != NULL)
+			continue;
+		if (certhash_addlink(links, cert) == -1)
+			return (-1);
+	}
+	for (crl = *crls; crl != NULL; crl = crl->next) {
+		if (crl->is_dup == 1 || crl->reference != NULL)
+			continue;
+		if (certhash_addlink(links, crl) == -1)
+			return (-1);
+	}
+
+	return (0);
+}
+
+static int
+certhash_link(int dfd, struct dirent *dep, struct hashinfo **links)
+{
+	struct hashinfo *hi = NULL;
+	char target[MAXPATHLEN];
+	struct stat sb;
+	int n;
+
+	if (fstatat(dfd, dep->d_name, &sb, AT_SYMLINK_NOFOLLOW) == -1) {
+		fprintf(stderr, "failed to stat %s\n", dep->d_name);
+		return (-1);
+	}
+	if (!S_ISLNK(sb.st_mode))
+		return (0);
+
+	n = readlinkat(dfd, dep->d_name, target, sizeof(target));
+	if (n == -1) {
+		fprintf(stderr, "failed to readlink %s\n", dep->d_name);
+		return (-1);
+	}
+	target[n] = '\0';
+
+	hi = hashinfo_from_linkname(dep->d_name, target);
+	if (hi == NULL) {
+		fprintf(stderr, "failed to get hash info %s\n", dep->d_name);
+		return (-1);
+	}
+	hi->exists = 1;
+	*links = hashinfo_chain(*links, hi);
+
+	return (0);
+}
+
+static int
+certhash_file(int dfd, struct dirent *dep, struct hashinfo **certs,
+    struct hashinfo **crls)
+{
+	struct hashinfo *hi = NULL;
+	int has_cert, has_crl;
+	int ffd, ret = -1;
+	BIO *bio = NULL;
+	FILE *f;
+
+	has_cert = has_crl = 0;
+
+	if ((ffd = openat(dfd, dep->d_name, O_RDONLY)) == -1) {
+		fprintf(stderr, "failed to open %s\n", dep->d_name);
+		goto err;
+	}
+	if ((f = fdopen(ffd, "r")) == NULL) {
+		fprintf(stderr, "failed to fdopen %s\n", dep->d_name);
+		goto err;
+	}
+	if ((bio = BIO_new_fp(f, BIO_CLOSE)) == NULL) {
+		fprintf(stderr, "failed to create bio\n");
+		goto err;
+	}
+
+	if ((hi = certhash_cert(bio, dep->d_name)) != NULL) {
+		has_cert = 1;
+		*certs = hashinfo_chain(*certs, hi);
+	}
+
+	if (BIO_reset(bio) != 0) {
+		fprintf(stderr, "BIO_reset failed\n");
+		goto err;
+	}
+
+	if ((hi = certhash_crl(bio, dep->d_name)) != NULL) {
+		has_crl = hi->is_crl = 1;
+		*crls = hashinfo_chain(*crls, hi);
+	}
+
+	if (!has_cert && !has_crl)
+		fprintf(stderr, "PEM file %s does not contain a certificate "
+		    "or CRL, ignoring...\n", dep->d_name);
+
+	ret = 0;
+
+err:
+	BIO_free(bio);
+	if (ffd != -1)
+		close(ffd);
+
+	return (ret);
+}
+
+static int
+certhash_directory(const char *path)
+{
+	struct hashinfo *links = NULL, *certs = NULL, *crls = NULL, *link;
+	int dfd = -1, ret = 0;
+	struct dirent *dep;
+	DIR *dip = NULL;
+
+	if ((dfd = open(path, O_DIRECTORY)) == -1) {
+		fprintf(stderr, "failed to open directory %s\n", path);
+		goto err;
+	}
+	if ((dip = fdopendir(dfd)) == NULL) {
+		fprintf(stderr, "failed to open directory %s\n", path);
+		goto err;
+	}
+
+	if (certhash_config.verbose)
+		fprintf(stdout, "scanning directory %s\n", path);
+
+	/* Create lists of existing hash links, certs and CRLs. */
+	while ((dep = readdir(dip)) != NULL) {
+		if (filename_is_hash(dep->d_name)) {
+			if (certhash_link(dfd, dep, &links) == -1)
+				goto err;
+		}
+		if (filename_is_pem(dep->d_name)) {
+			if (certhash_file(dfd, dep, &certs, &crls) == -1)
+				goto err;
+		}
+	}
+
+	if (certhash_merge(&links, &certs, &crls) == -1) {
+		fprintf(stderr, "certhash merge failed\n");
+		goto err;
+	}
+
+	/* Remove spurious links. */
+	for (link = links; link != NULL; link = link->next) {
+		if (link->exists == 0 ||
+		    (link->reference != NULL && link->changed == 0))
+			continue;
+		if (certhash_config.verbose)
+			fprintf(stdout, "%s link %s -> %s\n",
+			    (certhash_config.dryrun ? "would remove" :
+				"removing"), link->filename, link->target);
+		if (certhash_config.dryrun)
+			continue;
+		if (unlinkat(dfd, link->filename, 0) == -1) {
+			fprintf(stderr, "failed to remove link %s\n",
+			    link->filename);
+			goto err;
+		}
+	}
+
+	/* Create missing links. */
+	for (link = links; link != NULL; link = link->next) {
+		if (link->exists == 1 && link->changed == 0)
+			continue;
+		if (certhash_config.verbose)
+			fprintf(stdout, "%s link %s -> %s\n",
+			    (certhash_config.dryrun ? "would create" :
+				"creating"), link->filename,
+			    link->reference->filename);
+		if (certhash_config.dryrun)
+			continue;
+		if (symlinkat(link->reference->filename, dfd,
+		    link->filename) == -1) {
+			fprintf(stderr, "failed to create link %s -> %s\n",
+			    link->filename, link->reference->filename);
+			goto err;
+		}
+	}
+
+	goto done;
+
+err:
+	ret = 1;
+
+done:
+	hashinfo_chain_free(certs);
+	hashinfo_chain_free(crls);
+	hashinfo_chain_free(links);
+
+	if (dip != NULL)
+		closedir(dip);
+	else if (dfd != -1)
+		close(dfd);
+
+	return (ret);
+}
+
+static void
+certhash_usage(void)
+{
+	fprintf(stderr, "usage: certhash [-nv] dir ...\n");
+	options_usage(certhash_options);
+}
+
+int certhash_main(int argc, char **argv);
+
+int
+certhash_main(int argc, char **argv)
+{
+	int argsused;
+	int i, ret = 0;
+
+	memset(&certhash_config, 0, sizeof(certhash_config));
+
+	if (options_parse(argc, argv, certhash_options, NULL, &argsused) != 0) {
+                certhash_usage();
+                return (1);
+        }
+
+	for (i = argsused; i < argc; i++)
+		ret |= certhash_directory(argv[i]);
+
+	return (ret);
+}
diff --git a/usr.bin/openssl/progs.h b/usr.bin/openssl/progs.h
index 6f957c6f7c9..e1494e1147f 100644
--- a/usr.bin/openssl/progs.h
+++ b/usr.bin/openssl/progs.h
@@ -1,8 +1,9 @@
-/* $OpenBSD: progs.h,v 1.1 2014/08/26 17:47:25 jsing Exp $ */
+/* $OpenBSD: progs.h,v 1.2 2015/02/10 15:29:34 jsing Exp $ */
 /* Public domain */
 
 extern int asn1parse_main(int argc, char *argv[]);
 extern int ca_main(int argc, char *argv[]);
+extern int certhash_main(int argc, char *argv[]);
 extern int ciphers_main(int argc, char *argv[]);
 extern int cms_main(int argc, char *argv[]);
 extern int crl2pkcs7_main(int argc, char *argv[]);
@@ -66,6 +67,7 @@ FUNCTION functions[] = {
 	/* General functions. */
 	{ FUNC_TYPE_GENERAL, "asn1parse", asn1parse_main },
 	{ FUNC_TYPE_GENERAL, "ca", ca_main },
+	{ FUNC_TYPE_GENERAL, "certhash", certhash_main },
 	{ FUNC_TYPE_GENERAL, "ciphers", ciphers_main },
 #ifndef OPENSSL_NO_CMS
 	{ FUNC_TYPE_GENERAL, "cms", cms_main },