Rework asn1_item_flags_i2d()

Flip the logic of NULL checks on out and *out to unindent, use calloc()
instead of malloc() and check on assign. Also drop the newly added len2
again, it isn't needed.

ok jsing

1 files changed, 20 insertions, 19 deletions

diff --git a/lib/libcrypto/asn1/tasn_enc.c b/lib/libcrypto/asn1/tasn_enc.c
index 1b9cfa1a0e8..6e0524c39fd 100644
--- a/lib/libcrypto/asn1/tasn_enc.c
+++ b/lib/libcrypto/asn1/tasn_enc.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: tasn_enc.c,v 1.28 2023/03/06 08:08:31 tb Exp $ */
+/* $OpenBSD: tasn_enc.c,v 1.29 2023/03/06 12:00:27 tb Exp $ */
 /* Written by Dr Stephen N Henson (steve@openssl.org) for the OpenSSL
  * project 2000.
  */
@@ -106,27 +106,28 @@ static int
 asn1_item_flags_i2d(ASN1_VALUE *val, unsigned char **out, const ASN1_ITEM *it,
     int flags)
 {
-	if (out && !*out) {
-		unsigned char *p, *buf;
-		int len, len2;
-		len = ASN1_item_ex_i2d(&val, NULL, it, -1, flags);
-		if (len <= 0)
-			return len;
-		buf = malloc(len);
-		if (!buf)
-			return -1;
-		p = buf;
-		len2 = ASN1_item_ex_i2d(&val, &p, it, -1, flags);
-		if (len2 != len) {
-			freezero(buf, len);
-			ASN1error(ASN1_R_LENGTH_ERROR);
-			return -1;
-		}
-		*out = buf;
+	unsigned char *p, *buf;
+	int len;
+
+	if (out == NULL || *out != NULL)
+		return ASN1_item_ex_i2d(&val, out, it, -1, flags);
+
+	if ((len = ASN1_item_ex_i2d(&val, NULL, it, -1, flags)) <= 0)
 		return len;
+
+	if ((buf = calloc(1, len)) == NULL)
+		return -1;
+
+	p = buf;
+	if (ASN1_item_ex_i2d(&val, &p, it, -1, flags) != len) {
+		freezero(buf, len);
+		ASN1error(ASN1_R_LENGTH_ERROR);
+		return -1;
 	}
 
-	return ASN1_item_ex_i2d(&val, out, it, -1, flags);
+	*out = buf;
+
+	return len;
 }
 
 /* Encode an item, taking care of IMPLICIT tagging (if any).