fix for integer overflow in encode and encrypt update functions.

additionally, in EncodeUpdate, if the amount written would overflow,
return 0 instead to prevent bugs in the caller.
CVE-2016-2105 and CVE-2016-2106 from openssl.

2 files changed, 11 insertions, 6 deletions

diff --git a/lib/libcrypto/evp/encode.c b/lib/libcrypto/evp/encode.c
index 637870cef6b..1097a7c9039 100644
--- a/lib/libcrypto/evp/encode.c
+++ b/lib/libcrypto/evp/encode.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: encode.c,v 1.23 2016/05/04 14:53:29 tedu Exp $ */
+/* $OpenBSD: encode.c,v 1.24 2016/05/04 15:05:13 tedu Exp $ */
 /* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com)
  * All rights reserved.
  *
@@ -125,13 +125,13 @@ EVP_EncodeUpdate(EVP_ENCODE_CTX *ctx, unsigned char *out, int *outl,
     const unsigned char *in, int inl)
 {
 	int i, j;
-	unsigned int total = 0;
+	size_t total = 0;
 
 	*outl = 0;
 	if (inl == 0)
 		return;
 	OPENSSL_assert(ctx->length <= (int)sizeof(ctx->enc_data));
-	if ((ctx->num + inl) < ctx->length) {
+	if (ctx->length - ctx->num > inl) {
 		memcpy(&(ctx->enc_data[ctx->num]), in, inl);
 		ctx->num += inl;
 		return;
@@ -148,7 +148,7 @@ EVP_EncodeUpdate(EVP_ENCODE_CTX *ctx, unsigned char *out, int *outl,
 		*out = '\0';
 		total = j + 1;
 	}
-	while (inl >= ctx->length) {
+	while (inl >= ctx->length && total <= INT_MAX) {
 		j = EVP_EncodeBlock(out, in, ctx->length);
 		in += ctx->length;
 		inl -= ctx->length;
@@ -157,6 +157,11 @@ EVP_EncodeUpdate(EVP_ENCODE_CTX *ctx, unsigned char *out, int *outl,
 		*out = '\0';
 		total += j + 1;
 	}
+	if (total > INT_MAX) {
+		/* Too much output data! */
+		*outl = 0;
+		return;
+	}
 	if (inl != 0)
 		memcpy(&(ctx->enc_data[0]), in, inl);
 	ctx->num = inl;
diff --git a/lib/libcrypto/evp/evp_enc.c b/lib/libcrypto/evp/evp_enc.c
index a4e369f6228..556908fd106 100644
--- a/lib/libcrypto/evp/evp_enc.c
+++ b/lib/libcrypto/evp/evp_enc.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: evp_enc.c,v 1.29 2016/05/04 14:53:29 tedu Exp $ */
+/* $OpenBSD: evp_enc.c,v 1.30 2016/05/04 15:05:13 tedu Exp $ */
 /* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com)
  * All rights reserved.
  *
@@ -334,7 +334,7 @@ EVP_EncryptUpdate(EVP_CIPHER_CTX *ctx, unsigned char *out, int *outl,
 		return 0;
 	}
 	if (i != 0) {
-		if (i + inl < bl) {
+		if (bl - i > inl) {
 			memcpy(&(ctx->buf[i]), in, inl);
 			ctx->buf_len += inl;
 			*outl = 0;