Add bn_printf(), a replacement for ASN1_bn_print()

ASN1_bn_print() will be removed in an upcoming bump. This adds an internal
API that covers the same functionality but doesn't require that the caller
pass in a sufficiently large scratch space that ASN1_bn_print() may or may
not use. In addition, this takes a format string, which allows us to ditch
some extra dances.

ok jsing

3 files changed, 154 insertions, 2 deletions

diff --git a/lib/libcrypto/Makefile b/lib/libcrypto/Makefile
index 6e41dae162a..f40ef656b74 100644
--- a/lib/libcrypto/Makefile
+++ b/lib/libcrypto/Makefile
@@ -1,4 +1,4 @@
-# $OpenBSD: Makefile,v 1.135 2023/07/05 12:31:14 tb Exp $
+# $OpenBSD: Makefile,v 1.136 2023/07/06 14:37:39 tb Exp $
 
 LIB=	crypto
 LIBREBUILD=y
@@ -195,6 +195,7 @@ SRCS+= bn_mont.c
 SRCS+= bn_mul.c
 SRCS+= bn_prime.c
 SRCS+= bn_primitives.c
+SRCS+= bn_print.c
 SRCS+= bn_rand.c
 SRCS+= bn_recp.c
 SRCS+= bn_shift.c
diff --git a/lib/libcrypto/bn/bn_local.h b/lib/libcrypto/bn/bn_local.h
index 17f5447becf..86aa9722754 100644
--- a/lib/libcrypto/bn/bn_local.h
+++ b/lib/libcrypto/bn/bn_local.h
@@ -1,4 +1,4 @@
-/* $OpenBSD: bn_local.h,v 1.24 2023/06/24 16:01:43 jsing Exp $ */
+/* $OpenBSD: bn_local.h,v 1.25 2023/07/06 14:37:39 tb Exp $ */
 /* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com)
  * All rights reserved.
  *
@@ -323,5 +323,9 @@ int bn_is_perfect_square(int *out_perfect, const BIGNUM *n, BN_CTX *ctx);
 
 int bn_is_prime_bpsw(int *is_prime, const BIGNUM *n, BN_CTX *ctx, size_t rounds);
 
+int bn_printf(BIO *bio, const BIGNUM *bn, int indent, const char *fmt, ...)
+    __attribute__((__format__ (printf, 4, 5)))
+    __attribute__((__nonnull__ (4)));
+
 __END_HIDDEN_DECLS
 #endif /* !HEADER_BN_LOCAL_H */
diff --git a/lib/libcrypto/bn/bn_print.c b/lib/libcrypto/bn/bn_print.c
new file mode 100644
index 00000000000..466aeb3d646
--- /dev/null
+++ b/lib/libcrypto/bn/bn_print.c
@@ -0,0 +1,147 @@
+/*	$OpenBSD: bn_print.c,v 1.40 2023/07/06 14:37:39 tb Exp $ */
+
+/*
+ * Copyright (c) 2023 Theo Buehler <tb@openbsd.org>
+ *
+ * Permission to use, copy, modify, and distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#include <ctype.h>
+#include <limits.h>
+#include <stdarg.h>
+#include <stdint.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include <openssl/bio.h>
+#include <openssl/bn.h>
+
+#include "bytestring.h"
+
+static int
+bn_print_zero(BIO *bio, const BIGNUM *bn)
+{
+	if (!BN_is_zero(bn))
+		return 0;
+	if (BIO_printf(bio, " 0\n") <= 0)
+		return 0;
+	return 1;
+}
+
+static int
+bn_print_word(BIO *bio, const BIGNUM *bn)
+{
+	BN_ULONG word;
+	const char *neg = "";
+
+	if (BN_is_zero(bn) || BN_num_bytes(bn) > BN_BYTES)
+		return 0;
+
+	if (BN_is_negative(bn))
+		neg = "-";
+
+	word = BN_get_word(bn);
+	if (BIO_printf(bio, " %s%lu (%s0x%lx)\n", neg, word, neg, word) <= 0)
+		return 0;
+
+	return 1;
+}
+
+static int
+bn_print_bignum(BIO *bio, const BIGNUM *bn, int indent)
+{
+	CBS cbs;
+	char *hex = NULL;
+	size_t hex_len = 0;
+	size_t octets = 0;
+	uint8_t hi, lo;
+	const char *sep = ":";
+	int ret = 0;
+
+	if (BN_num_bytes(bn) <= BN_BYTES)
+		goto err;
+
+	/* Secondary indent is 4 spaces, capped at 128. */
+	if (indent > INT_MAX - 4)
+		goto err;
+	indent += 4;
+	if (indent > 128)
+		indent = 128;
+	if (indent < 0)
+		indent = 0;
+
+	if ((hex = BN_bn2hex(bn)) == NULL)
+		goto err;
+	hex_len = strlen(hex);
+
+	CBS_init(&cbs, hex, hex_len);
+
+	if (BN_is_negative(bn)) {
+		if (BIO_printf(bio, " (Negative)") <= 0)
+			goto err;
+		if (!CBS_skip(&cbs, 1))
+			goto err;
+	}
+
+	while (CBS_len(&cbs) > 0) {
+		if (!CBS_get_u8(&cbs, &hi))
+			goto err;
+		if (!CBS_get_u8(&cbs, &lo))
+			goto err;
+		if (octets++ % 15 == 0) {
+			if (BIO_printf(bio, "\n%*s", indent, "") <= 0)
+				goto err;
+		}
+		if (CBS_len(&cbs) == 0)
+			sep = "";
+		if (BIO_printf(bio, "%c%c%s", tolower(hi), tolower(lo), sep) <= 0)
+			goto err;
+	}
+
+	if (BIO_printf(bio, "\n") <= 0)
+		goto err;
+
+	ret = 1;
+
+ err:
+	freezero(hex, hex_len);
+
+	return ret;
+}
+
+int
+bn_printf(BIO *bio, const BIGNUM *bn, int indent, const char *fmt, ...)
+{
+	va_list ap;
+	int rv;
+
+	if (bn == NULL)
+		return 1;
+
+	if (!BIO_indent(bio, indent, 128))
+		return 0;
+
+	va_start(ap, fmt);
+	rv = BIO_vprintf(bio, fmt, ap);
+	va_end(ap);
+	if (rv < 0)
+		return 0;
+
+	if (BN_is_zero(bn))
+		return bn_print_zero(bio, bn);
+
+	if (BN_num_bytes(bn) <= BN_BYTES)
+		return bn_print_word(bio, bn);
+
+	return bn_print_bignum(bio, bn, indent);
+}