Mop up remaining uses of ASN1_bn_print()

This removes lots of silly buffers and will allow us to make this API
go away.

ok jsing

4 files changed, 38 insertions, 140 deletions

diff --git a/lib/libcrypto/dh/dh_ameth.c b/lib/libcrypto/dh/dh_ameth.c
index 3de0bb93333..61d3d143972 100644
--- a/lib/libcrypto/dh/dh_ameth.c
+++ b/lib/libcrypto/dh/dh_ameth.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: dh_ameth.c,v 1.28 2023/04/17 05:57:17 tb Exp $ */
+/* $OpenBSD: dh_ameth.c,v 1.29 2023/07/07 06:59:18 tb Exp $ */
 /* Written by Dr Stephen N Henson (steve@openssl.org) for the OpenSSL
  * project 2006.
  */
@@ -65,6 +65,7 @@
 #include <openssl/x509.h>
 
 #include "asn1_local.h"
+#include "bn_local.h"
 #include "dh_local.h"
 #include "evp_local.h"
 
@@ -280,17 +281,6 @@ err:
 	return 0;
 }
 
-static void
-update_buflen(const BIGNUM *b, size_t *pbuflen)
-{
-	size_t i;
-
-	if (!b)
-		return;
-	if (*pbuflen < (i = (size_t)BN_num_bytes(b)))
-		*pbuflen = i;
-}
-
 static int
 dh_param_decode(EVP_PKEY *pkey, const unsigned char **pder, int derlen)
 {
@@ -313,9 +303,7 @@ dh_param_encode(const EVP_PKEY *pkey, unsigned char **pder)
 static int
 do_dh_print(BIO *bp, const DH *x, int indent, ASN1_PCTX *ctx, int ptype)
 {
-	unsigned char *m = NULL;
 	int reason = ERR_R_BUF_LIB, ret = 0;
-	size_t buf_len = 0;
 	const char *ktype = NULL;
 	BIGNUM *priv_key, *pub_key;
 
@@ -329,17 +317,6 @@ do_dh_print(BIO *bp, const DH *x, int indent, ASN1_PCTX *ctx, int ptype)
 	else
 		pub_key = NULL;
 
-	update_buflen(x->p, &buf_len);
-
-	if (buf_len == 0) {
-		reason = ERR_R_PASSED_NULL_PARAMETER;
-		goto err;
-	}
-
-	update_buflen(x->g, &buf_len);
-	update_buflen(pub_key, &buf_len);
-	update_buflen(priv_key, &buf_len);
-
 	if (ptype == 2)
 		ktype = "PKCS#3 DH Private-Key";
 	else if (ptype == 1)
@@ -347,9 +324,8 @@ do_dh_print(BIO *bp, const DH *x, int indent, ASN1_PCTX *ctx, int ptype)
 	else
 		ktype = "PKCS#3 DH Parameters";
 
-	m= malloc(buf_len + 10);
-	if (m == NULL) {
-		reason = ERR_R_MALLOC_FAILURE;
+	if (x->p == NULL) {
+		reason = ERR_R_PASSED_NULL_PARAMETER;
 		goto err;
 	}
 
@@ -359,14 +335,14 @@ do_dh_print(BIO *bp, const DH *x, int indent, ASN1_PCTX *ctx, int ptype)
 		goto err;
 	indent += 4;
 
-	if (!ASN1_bn_print(bp, "private-key:", priv_key, m, indent))
+	if (!bn_printf(bp, priv_key, indent, "private-key:"))
 		goto err;
-	if (!ASN1_bn_print(bp, "public-key:", pub_key, m, indent))
+	if (!bn_printf(bp, pub_key, indent, "public-key:"))
 		goto err;
 
-	if (!ASN1_bn_print(bp, "prime:", x->p, m, indent))
+	if (!bn_printf(bp, x->p, indent, "prime:"))
 		goto err;
-	if (!ASN1_bn_print(bp, "generator:", x->g, m, indent))
+	if (!bn_printf(bp, x->g, indent, "generator:"))
 		goto err;
 	if (x->length != 0) {
 		if (!BIO_indent(bp, indent, 128))
@@ -378,10 +354,9 @@ do_dh_print(BIO *bp, const DH *x, int indent, ASN1_PCTX *ctx, int ptype)
 
 	ret = 1;
 	if (0) {
-err:
+ err:
 		DHerror(reason);
 	}
-	free(m);
 	return(ret);
 }
 
diff --git a/lib/libcrypto/dsa/dsa_ameth.c b/lib/libcrypto/dsa/dsa_ameth.c
index f282caae06b..5a0c3116aad 100644
--- a/lib/libcrypto/dsa/dsa_ameth.c
+++ b/lib/libcrypto/dsa/dsa_ameth.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: dsa_ameth.c,v 1.42 2023/03/04 21:42:49 tb Exp $ */
+/* $OpenBSD: dsa_ameth.c,v 1.43 2023/07/07 06:59:18 tb Exp $ */
 /* Written by Dr Stephen N Henson (steve@openssl.org) for the OpenSSL
  * project 2006.
  */
@@ -385,25 +385,12 @@ int_dsa_free(EVP_PKEY *pkey)
 	DSA_free(pkey->pkey.dsa);
 }
 
-static void
-update_buflen(const BIGNUM *b, size_t *pbuflen)
-{
-	size_t i;
-
-	if (!b)
-		return;
-	if (*pbuflen < (i = (size_t)BN_num_bytes(b)))
-		*pbuflen = i;
-}
-
 static int
 do_dsa_print(BIO *bp, const DSA *x, int off, int ptype)
 {
-	unsigned char *m = NULL;
-	int ret = 0;
-	size_t buf_len = 0;
 	const char *ktype = NULL;
 	const BIGNUM *priv_key, *pub_key;
+	int ret = 0;
 
 	if (ptype == 2)
 		priv_key = x->priv_key;
@@ -422,18 +409,6 @@ do_dsa_print(BIO *bp, const DSA *x, int off, int ptype)
 	else
 		ktype = "DSA-Parameters";
 
-	update_buflen(x->p, &buf_len);
-	update_buflen(x->q, &buf_len);
-	update_buflen(x->g, &buf_len);
-	update_buflen(priv_key, &buf_len);
-	update_buflen(pub_key, &buf_len);
-
-	m = malloc(buf_len + 10);
-	if (m == NULL) {
-		DSAerror(ERR_R_MALLOC_FAILURE);
-		goto err;
-	}
-
 	if (priv_key) {
 		if (!BIO_indent(bp, off, 128))
 			goto err;
@@ -442,19 +417,20 @@ do_dsa_print(BIO *bp, const DSA *x, int off, int ptype)
 			goto err;
 	}
 
-	if (!ASN1_bn_print(bp, "priv:", priv_key, m, off))
+	if (!bn_printf(bp, priv_key, off, "priv:"))
 		goto err;
-	if (!ASN1_bn_print(bp, "pub: ", pub_key, m, off))
+	if (!bn_printf(bp, pub_key, off, "pub: "))
 		goto err;
-	if (!ASN1_bn_print(bp, "P:   ", x->p, m, off))
+	if (!bn_printf(bp, x->p, off, "P:   "))
 		goto err;
-	if (!ASN1_bn_print(bp, "Q:   ", x->q, m, off))
+	if (!bn_printf(bp, x->q, off, "Q:   "))
 		goto err;
-	if (!ASN1_bn_print(bp, "G:   ", x->g, m, off))
+	if (!bn_printf(bp, x->g, off, "G:   "))
 		goto err;
+
 	ret = 1;
-err:
-	free(m);
+
+ err:
 	return ret;
 }
 
@@ -594,27 +570,16 @@ dsa_sig_print(BIO *bp, const X509_ALGOR *sigalg, const ASN1_STRING *sig,
 	dsa_sig = d2i_DSA_SIG(NULL, &p, sig->length);
 	if (dsa_sig) {
 		int rv = 0;
-		size_t buf_len = 0;
-		unsigned char *m = NULL;
-
-		update_buflen(dsa_sig->r, &buf_len);
-		update_buflen(dsa_sig->s, &buf_len);
-		m = malloc(buf_len + 10);
-		if (m == NULL) {
-			DSAerror(ERR_R_MALLOC_FAILURE);
-			goto err;
-		}
 
 		if (BIO_write(bp, "\n", 1) != 1)
 			goto err;
 
-		if (!ASN1_bn_print(bp, "r:   ", dsa_sig->r, m, indent))
+		if (!bn_printf(bp, dsa_sig->r, indent, "r:   "))
 			goto err;
-		if (!ASN1_bn_print(bp, "s:   ", dsa_sig->s, m, indent))
+		if (!bn_printf(bp, dsa_sig->s, indent, "s:   "))
 			goto err;
 		rv = 1;
-err:
-		free(m);
+ err:
 		DSA_SIG_free(dsa_sig);
 		return rv;
 	}
diff --git a/lib/libcrypto/ec/ec_ameth.c b/lib/libcrypto/ec/ec_ameth.c
index 8676ace9d86..49ae80494d8 100644
--- a/lib/libcrypto/ec/ec_ameth.c
+++ b/lib/libcrypto/ec/ec_ameth.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: ec_ameth.c,v 1.40 2023/07/03 09:25:44 tb Exp $ */
+/* $OpenBSD: ec_ameth.c,v 1.41 2023/07/07 06:59:18 tb Exp $ */
 /* Written by Dr Stephen N Henson (steve@openssl.org) for the OpenSSL
  * project 2006.
  */
@@ -427,9 +427,7 @@ int_ec_free(EVP_PKEY *pkey)
 static int
 do_EC_KEY_print(BIO *bp, const EC_KEY *x, int off, int ktype)
 {
-	unsigned char *buffer = NULL;
 	const char *ecstr;
-	size_t buf_len = 0, i;
 	int ret = 0, reason = ERR_R_BIO_LIB;
 	BIGNUM *pub_key = NULL;
 	BN_CTX *ctx = NULL;
@@ -454,24 +452,13 @@ do_EC_KEY_print(BIO *bp, const EC_KEY *x, int off, int ktype)
 				reason = ERR_R_EC_LIB;
 				goto err;
 			}
-			if (pub_key)
-				buf_len = (size_t) BN_num_bytes(pub_key);
 		}
 	}
 	if (ktype == 2) {
 		priv_key = EC_KEY_get0_private_key(x);
-		if (priv_key && (i = (size_t) BN_num_bytes(priv_key)) > buf_len)
-			buf_len = i;
 	} else
 		priv_key = NULL;
 
-	if (ktype > 0) {
-		buf_len += 10;
-		if ((buffer = malloc(buf_len)) == NULL) {
-			reason = ERR_R_MALLOC_FAILURE;
-			goto err;
-		}
-	}
 	if (ktype == 2)
 		ecstr = "Private-Key";
 	else if (ktype == 1)
@@ -485,19 +472,21 @@ do_EC_KEY_print(BIO *bp, const EC_KEY *x, int off, int ktype)
 	    EC_GROUP_order_bits(group)) <= 0)
 		goto err;
 
-	if (!ASN1_bn_print(bp, "priv:", priv_key, buffer, off))
+	if (!bn_printf(bp, priv_key, off, "priv:"))
 		goto err;
-	if (!ASN1_bn_print(bp, "pub: ", pub_key, buffer, off))
+	if (!bn_printf(bp, pub_key, off, "pub: "))
 		goto err;
 	if (!ECPKParameters_print(bp, group, off))
 		goto err;
+
 	ret = 1;
+
  err:
 	if (!ret)
 		ECerror(reason);
 	BN_free(pub_key);
 	BN_CTX_free(ctx);
-	free(buffer);
+
 	return (ret);
 }
 
diff --git a/lib/libcrypto/rsa/rsa_ameth.c b/lib/libcrypto/rsa/rsa_ameth.c
index 1cf2069a183..825a9f44470 100644
--- a/lib/libcrypto/rsa/rsa_ameth.c
+++ b/lib/libcrypto/rsa/rsa_ameth.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: rsa_ameth.c,v 1.29 2023/05/19 17:31:20 tb Exp $ */
+/* $OpenBSD: rsa_ameth.c,v 1.30 2023/07/07 06:59:18 tb Exp $ */
 /* Written by Dr Stephen N Henson (steve@openssl.org) for the OpenSSL
  * project 2006.
  */
@@ -68,6 +68,7 @@
 #include <openssl/x509.h>
 
 #include "asn1_local.h"
+#include "bn_local.h"
 #include "cryptlib.h"
 #include "evp_local.h"
 #include "rsa_local.h"
@@ -408,44 +409,13 @@ rsa_pss_param_print(BIO *bp, int pss_key, RSA_PSS_PARAMS *pss, int indent)
 
 }
 
-static void
-update_buflen(const BIGNUM *b, size_t *pbuflen)
-{
-	size_t i;
-
-	if (!b)
-		return;
-	if (*pbuflen < (i = (size_t)BN_num_bytes(b)))
-		*pbuflen = i;
-}
-
 static int
 pkey_rsa_print(BIO *bp, const EVP_PKEY *pkey, int off, int priv)
 {
 	const RSA *x = pkey->pkey.rsa;
-	unsigned char *m = NULL;
 	char *str;
 	const char *s;
 	int ret = 0, mod_len = 0;
-	size_t buf_len = 0;
-
-	update_buflen(x->n, &buf_len);
-	update_buflen(x->e, &buf_len);
-
-	if (priv) {
-		update_buflen(x->d, &buf_len);
-		update_buflen(x->p, &buf_len);
-		update_buflen(x->q, &buf_len);
-		update_buflen(x->dmp1, &buf_len);
-		update_buflen(x->dmq1, &buf_len);
-		update_buflen(x->iqmp, &buf_len);
-	}
-
-	m = malloc(buf_len + 10);
-	if (m == NULL) {
-		RSAerror(ERR_R_MALLOC_FAILURE);
-		goto err;
-	}
 
 	if (x->n != NULL)
 		mod_len = BN_num_bits(x->n);
@@ -467,29 +437,28 @@ pkey_rsa_print(BIO *bp, const EVP_PKEY *pkey, int off, int priv)
 		str = "Modulus:";
 		s = "Exponent:";
 	}
-	if (!ASN1_bn_print(bp, str, x->n, m, off))
+	if (!bn_printf(bp, x->n, off, "%s", str))
 		goto err;
-	if (!ASN1_bn_print(bp, s, x->e, m, off))
+	if (!bn_printf(bp, x->e, off, "%s", s))
 		goto err;
 	if (priv) {
-		if (!ASN1_bn_print(bp, "privateExponent:", x->d, m, off))
+		if (!bn_printf(bp, x->d, off, "privateExponent:"))
 			goto err;
-		if (!ASN1_bn_print(bp, "prime1:", x->p, m, off))
+		if (!bn_printf(bp, x->p, off, "prime1:"))
 			goto err;
-		if (!ASN1_bn_print(bp, "prime2:", x->q, m, off))
+		if (!bn_printf(bp, x->q, off, "prime2:"))
 			goto err;
-		if (!ASN1_bn_print(bp, "exponent1:", x->dmp1, m, off))
+		if (!bn_printf(bp, x->dmp1, off, "exponent1:"))
 			goto err;
-		if (!ASN1_bn_print(bp, "exponent2:", x->dmq1, m, off))
+		if (!bn_printf(bp, x->dmq1, off, "exponent2:"))
 			goto err;
-		if (!ASN1_bn_print(bp, "coefficient:", x->iqmp, m, off))
+		if (!bn_printf(bp, x->iqmp, off, "coefficient:"))
 			goto err;
 	}
 	if (pkey_is_pss(pkey) && !rsa_pss_param_print(bp, 1, x->pss, off))
 		goto err;
 	ret = 1;
  err:
-	free(m);
 	return ret;
 }