Refactor BN_bn2hex()

Various outputting functions are variants of BN_bn2hex(). They do not
want a sign or they display the BIGNUM at nibble granularity instead
of byte granularity. So add this functionality to an internal variant
of BN_bn2hex().

with/ok jsing

2 files changed, 57 insertions, 8 deletions

diff --git a/lib/libcrypto/bn/bn_convert.c b/lib/libcrypto/bn/bn_convert.c
index cb0a23e0108..788e90cc8d8 100644
--- a/lib/libcrypto/bn/bn_convert.c
+++ b/lib/libcrypto/bn/bn_convert.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: bn_convert.c,v 1.13 2023/07/08 12:21:58 beck Exp $ */
+/* $OpenBSD: bn_convert.c,v 1.14 2023/07/09 18:27:22 tb Exp $ */
 /* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com)
  * All rights reserved.
  *
@@ -497,20 +497,27 @@ BN_dec2bn(BIGNUM **bnp, const char *s)
 }
 LCRYPTO_ALIAS(BN_dec2bn);
 
-char *
-BN_bn2hex(const BIGNUM *bn)
+static int
+bn_bn2hex_internal(const BIGNUM *bn, int include_sign, int nibbles_only,
+    char **out, size_t *out_len)
 {
 	int started = 0;
 	uint8_t *s = NULL;
-	size_t s_len;
+	size_t s_len = 0;
 	BN_ULONG v, w;
 	int i, j;
 	CBB cbb;
+	CBS cbs;
+	uint8_t nul;
+	int ret = 0;
+
+	*out = NULL;
+	*out_len = 0;
 
 	if (!CBB_init(&cbb, 0))
 		goto err;
 
-	if (BN_is_negative(bn)) {
+	if (BN_is_negative(bn) && include_sign) {
 		if (!CBB_add_u8(&cbb, '-'))
 			goto err;
 	}
@@ -524,8 +531,10 @@ BN_bn2hex(const BIGNUM *bn)
 			v = (w >> j) & 0xff;
 			if (!started && v == 0)
 				continue;
-			if (!CBB_add_u8(&cbb, hex_digits[v >> 4]))
-				goto err;
+			if (started || !nibbles_only || (v >> 4) != 0) {
+				if (!CBB_add_u8(&cbb, hex_digits[v >> 4]))
+					goto err;
+			}
 			if (!CBB_add_u8(&cbb, hex_digits[v & 0xf]))
 				goto err;
 			started = 1;
@@ -536,8 +545,45 @@ BN_bn2hex(const BIGNUM *bn)
 	if (!CBB_finish(&cbb, &s, &s_len))
 		goto err;
 
+	/* The length of a C string does not include the terminating NUL. */
+	CBS_init(&cbs, s, s_len);
+	if (!CBS_get_last_u8(&cbs, &nul))
+		goto err;
+
+	*out = (char *)CBS_data(&cbs);
+	*out_len = CBS_len(&cbs);
+	s = NULL;
+	s_len = 0;
+
+	ret = 1;
+
  err:
 	CBB_cleanup(&cbb);
+	freezero(s, s_len);
+
+	return ret;
+}
+
+int
+bn_bn2hex_nosign(const BIGNUM *bn, char **out, size_t *out_len)
+{
+	return bn_bn2hex_internal(bn, 0, 0, out, out_len);
+}
+
+int
+bn_bn2hex_nibbles(const BIGNUM *bn, char **out, size_t *out_len)
+{
+	return bn_bn2hex_internal(bn, 1, 1, out, out_len);
+}
+
+char *
+BN_bn2hex(const BIGNUM *bn)
+{
+	char *s;
+	size_t s_len;
+
+	if (!bn_bn2hex_internal(bn, 1, 0, &s, &s_len))
+		return NULL;
 
 	return s;
 }
diff --git a/lib/libcrypto/bn/bn_local.h b/lib/libcrypto/bn/bn_local.h
index 86aa9722754..a8d40fbcc8b 100644
--- a/lib/libcrypto/bn/bn_local.h
+++ b/lib/libcrypto/bn/bn_local.h
@@ -1,4 +1,4 @@
-/* $OpenBSD: bn_local.h,v 1.25 2023/07/06 14:37:39 tb Exp $ */
+/* $OpenBSD: bn_local.h,v 1.26 2023/07/09 18:27:22 tb Exp $ */
 /* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com)
  * All rights reserved.
  *
@@ -327,5 +327,8 @@ int bn_printf(BIO *bio, const BIGNUM *bn, int indent, const char *fmt, ...)
     __attribute__((__format__ (printf, 4, 5)))
     __attribute__((__nonnull__ (4)));
 
+int bn_bn2hex_nosign(const BIGNUM *bn, char **out, size_t *out_len);
+int bn_bn2hex_nibbles(const BIGNUM *bn, char **out, size_t *out_len);
+
 __END_HIDDEN_DECLS
 #endif /* !HEADER_BN_LOCAL_H */