Convert EVP_EncodeUpdate() to return an int to allow for error

checking. Matches our documented behavior.

Based on OpenSSL commit c5ebfcab713a82a1d46a51c8c2668c419425b387

tested in a bulk by sthen
ok jsing

2 files changed, 10 insertions, 8 deletions

diff --git a/lib/libcrypto/evp/encode.c b/lib/libcrypto/evp/encode.c
index 1097a7c9039..07cfd7f2bc4 100644
--- a/lib/libcrypto/evp/encode.c
+++ b/lib/libcrypto/evp/encode.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: encode.c,v 1.24 2016/05/04 15:05:13 tedu Exp $ */
+/* $OpenBSD: encode.c,v 1.25 2018/08/24 19:45:11 tb Exp $ */
 /* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com)
  * All rights reserved.
  *
@@ -120,7 +120,7 @@ EVP_EncodeInit(EVP_ENCODE_CTX *ctx)
 	ctx->line_num = 0;
 }
 
-void
+int
 EVP_EncodeUpdate(EVP_ENCODE_CTX *ctx, unsigned char *out, int *outl,
     const unsigned char *in, int inl)
 {
@@ -128,13 +128,13 @@ EVP_EncodeUpdate(EVP_ENCODE_CTX *ctx, unsigned char *out, int *outl,
 	size_t total = 0;
 
 	*outl = 0;
-	if (inl == 0)
-		return;
+	if (inl <= 0)
+		return 0;
 	OPENSSL_assert(ctx->length <= (int)sizeof(ctx->enc_data));
 	if (ctx->length - ctx->num > inl) {
 		memcpy(&(ctx->enc_data[ctx->num]), in, inl);
 		ctx->num += inl;
-		return;
+		return 1;
 	}
 	if (ctx->num != 0) {
 		i = ctx->length - ctx->num;
@@ -160,12 +160,14 @@ EVP_EncodeUpdate(EVP_ENCODE_CTX *ctx, unsigned char *out, int *outl,
 	if (total > INT_MAX) {
 		/* Too much output data! */
 		*outl = 0;
-		return;
+		return 0;
 	}
 	if (inl != 0)
 		memcpy(&(ctx->enc_data[0]), in, inl);
 	ctx->num = inl;
 	*outl = total;
+
+	return 1;
 }
 
 void
diff --git a/lib/libcrypto/evp/evp.h b/lib/libcrypto/evp/evp.h
index b607be7ae5b..0ec0431bc7d 100644
--- a/lib/libcrypto/evp/evp.h
+++ b/lib/libcrypto/evp/evp.h
@@ -1,4 +1,4 @@
-/* $OpenBSD: evp.h,v 1.66 2018/08/24 19:36:52 tb Exp $ */
+/* $OpenBSD: evp.h,v 1.67 2018/08/24 19:45:11 tb Exp $ */
 /* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com)
  * All rights reserved.
  *
@@ -631,7 +631,7 @@ int EVP_SealInit(EVP_CIPHER_CTX *ctx, const EVP_CIPHER *type,
 int EVP_SealFinal(EVP_CIPHER_CTX *ctx, unsigned char *out, int *outl);
 
 void EVP_EncodeInit(EVP_ENCODE_CTX *ctx);
-void EVP_EncodeUpdate(EVP_ENCODE_CTX *ctx, unsigned char *out, int *outl,
+int EVP_EncodeUpdate(EVP_ENCODE_CTX *ctx, unsigned char *out, int *outl,
     const unsigned char *in, int inl);
 void EVP_EncodeFinal(EVP_ENCODE_CTX *ctx, unsigned char *out, int *outl);
 int EVP_EncodeBlock(unsigned char *t, const unsigned char *f, int n);