Add bounds checks for various EVP cipher implementations.

The EVP cipher API uses size_t, however a number of the underlying
implementations use long in their API. This means that an input with
size > LONG_MAX will go negative.

Found by Coverity, hiding under a large pile of macros.

ok tb@

6 files changed, 98 insertions, 7 deletions

diff --git a/lib/libcrypto/evp/e_bf.c b/lib/libcrypto/evp/e_bf.c
index ab6dc4f7de9..4122f701dad 100644
--- a/lib/libcrypto/evp/e_bf.c
+++ b/lib/libcrypto/evp/e_bf.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: e_bf.c,v 1.11 2022/09/04 13:55:39 jsing Exp $ */
+/* $OpenBSD: e_bf.c,v 1.12 2022/09/04 15:45:25 jsing Exp $ */
 /* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com)
  * All rights reserved.
  *
@@ -56,6 +56,7 @@
  * [including the GNU Public Licence.]
  */
 
+#include <limits.h>
 #include <stdio.h>
 
 #include <openssl/opensslconf.h>
@@ -85,6 +86,9 @@ bf_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *key,
 static int
 bf_cbc_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, const unsigned char *in, size_t inl)
 {
+	if (inl > LONG_MAX)
+		return 0;
+
 	while (inl >= EVP_MAXCHUNK) {
 		BF_cbc_encrypt(in, out, (long)EVP_MAXCHUNK, &((EVP_BF_KEY *)ctx->cipher_data)->ks, ctx->iv, ctx->encrypt);
 		inl -= EVP_MAXCHUNK;
@@ -103,6 +107,9 @@ bf_cfb64_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, const unsigned char *in
 {
 	size_t chunk = EVP_MAXCHUNK;
 
+	if (inl > LONG_MAX)
+		return 0;
+
 	if (inl < chunk)
 		chunk = inl;
 
@@ -123,6 +130,9 @@ bf_ecb_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, const unsigned char *in,
 {
 	size_t i, bl;
 
+	if (inl > LONG_MAX)
+		return 0;
+
 	bl = ctx->cipher->block_size;
 
 	if (inl < bl)
@@ -139,6 +149,9 @@ bf_ecb_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, const unsigned char *in,
 static int
 bf_ofb_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, const unsigned char *in, size_t inl)
 {
+	if (inl > LONG_MAX)
+		return 0;
+
 	while (inl >= EVP_MAXCHUNK) {
 		BF_ofb64_encrypt(in, out, (long)EVP_MAXCHUNK, &((EVP_BF_KEY *)ctx->cipher_data)->ks, ctx->iv, &ctx->num);
 		inl -= EVP_MAXCHUNK;
diff --git a/lib/libcrypto/evp/e_cast.c b/lib/libcrypto/evp/e_cast.c
index d6f1b1d1a04..e654962c756 100644
--- a/lib/libcrypto/evp/e_cast.c
+++ b/lib/libcrypto/evp/e_cast.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: e_cast.c,v 1.10 2022/09/04 13:55:39 jsing Exp $ */
+/* $OpenBSD: e_cast.c,v 1.11 2022/09/04 15:45:25 jsing Exp $ */
 /* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com)
  * All rights reserved.
  *
@@ -56,6 +56,7 @@
  * [including the GNU Public Licence.]
  */
 
+#include <limits.h>
 #include <stdio.h>
 
 #include <openssl/opensslconf.h>
@@ -85,6 +86,9 @@ cast_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *key,
 static int
 cast5_cbc_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, const unsigned char *in, size_t inl)
 {
+	if (inl > LONG_MAX)
+		return 0;
+
 	while (inl >= EVP_MAXCHUNK) {
 		CAST_cbc_encrypt(in, out, (long)EVP_MAXCHUNK, &((EVP_CAST_KEY *)ctx->cipher_data)->ks, ctx->iv, ctx->encrypt);
 		inl -= EVP_MAXCHUNK;
@@ -103,6 +107,9 @@ cast5_cfb64_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, const unsigned char
 {
 	size_t chunk = EVP_MAXCHUNK;
 
+	if (inl > LONG_MAX)
+		return 0;
+
 	if (inl < chunk)
 		chunk = inl;
 
@@ -123,6 +130,9 @@ cast5_ecb_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, const unsigned char *i
 {
 	size_t i, bl;
 
+	if (inl > LONG_MAX)
+		return 0;
+
 	bl = ctx->cipher->block_size;
 
 	if (inl < bl)
@@ -139,6 +149,9 @@ cast5_ecb_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, const unsigned char *i
 static int
 cast5_ofb_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, const unsigned char *in, size_t inl)
 {
+	if (inl > LONG_MAX)
+		return 0;
+
 	while (inl >= EVP_MAXCHUNK) {
 		CAST_ofb64_encrypt(in, out, (long)EVP_MAXCHUNK, &((EVP_CAST_KEY *)ctx->cipher_data)->ks, ctx->iv, &ctx->num);
 		inl -= EVP_MAXCHUNK;
diff --git a/lib/libcrypto/evp/e_des.c b/lib/libcrypto/evp/e_des.c
index bf037591be8..9205128cf40 100644
--- a/lib/libcrypto/evp/e_des.c
+++ b/lib/libcrypto/evp/e_des.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: e_des.c,v 1.17 2022/09/04 13:17:18 jsing Exp $ */
+/* $OpenBSD: e_des.c,v 1.18 2022/09/04 15:45:25 jsing Exp $ */
 /* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com)
  * All rights reserved.
  *
@@ -56,6 +56,7 @@
  * [including the GNU Public Licence.]
  */
 
+#include <limits.h>
 #include <stdio.h>
 
 #include <openssl/opensslconf.h>
@@ -98,6 +99,9 @@ des_ecb_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out,
 {
 	size_t i, bl;
 
+	if (inl > LONG_MAX)
+		return 0;
+
 	bl = ctx->cipher->block_size;
 
 	if (inl < bl)
@@ -108,6 +112,7 @@ des_ecb_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out,
 	for (i = 0; i <= inl; i += bl)
 		DES_ecb_encrypt((DES_cblock *)(in + i), (DES_cblock *)(out + i),
 		    ctx->cipher_data, ctx->encrypt);
+
 	return 1;
 }
 
@@ -115,6 +120,9 @@ static int
 des_ofb_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out,
     const unsigned char *in, size_t inl)
 {
+	if (inl > LONG_MAX)
+		return 0;
+
 	while (inl >= EVP_MAXCHUNK) {
 		DES_ofb64_encrypt(in, out, (long)EVP_MAXCHUNK, ctx->cipher_data,
 		    (DES_cblock *)ctx->iv, &ctx->num);
@@ -132,6 +140,9 @@ static int
 des_cbc_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out,
     const unsigned char *in, size_t inl)
 {
+	if (inl > LONG_MAX)
+		return 0;
+
 	while (inl >= EVP_MAXCHUNK) {
 		DES_ncbc_encrypt(in, out, (long)EVP_MAXCHUNK, ctx->cipher_data,
 		    (DES_cblock *)ctx->iv, ctx->encrypt);
@@ -149,6 +160,9 @@ static int
 des_cfb64_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out,
     const unsigned char *in, size_t inl)
 {
+	if (inl > LONG_MAX)
+		return 0;
+
 	while (inl >= EVP_MAXCHUNK) {
 		DES_cfb64_encrypt(in, out, (long)EVP_MAXCHUNK, ctx->cipher_data,
 		    (DES_cblock *)ctx->iv, &ctx->num, ctx->encrypt);
@@ -171,6 +185,9 @@ des_cfb1_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out,
 	size_t n, chunk = EVP_MAXCHUNK/8;
 	unsigned char c[1], d[1];
 
+	if (inl > LONG_MAX)
+		return 0;
+
 	if (inl < chunk)
 		chunk = inl;
 
@@ -197,6 +214,9 @@ static int
 des_cfb8_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out,
     const unsigned char *in, size_t inl)
 {
+	if (inl > LONG_MAX)
+		return 0;
+
 	while (inl >= EVP_MAXCHUNK) {
 		DES_cfb_encrypt(in, out, 8, (long)EVP_MAXCHUNK,
 		    ctx->cipher_data, (DES_cblock *)ctx->iv, ctx->encrypt);
diff --git a/lib/libcrypto/evp/e_des3.c b/lib/libcrypto/evp/e_des3.c
index e9d7f56809f..1171a53b743 100644
--- a/lib/libcrypto/evp/e_des3.c
+++ b/lib/libcrypto/evp/e_des3.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: e_des3.c,v 1.23 2022/09/04 13:17:18 jsing Exp $ */
+/* $OpenBSD: e_des3.c,v 1.24 2022/09/04 15:45:25 jsing Exp $ */
 /* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com)
  * All rights reserved.
  *
@@ -56,6 +56,7 @@
  * [including the GNU Public Licence.]
  */
 
+#include <limits.h>
 #include <stdio.h>
 #include <string.h>
 
@@ -129,6 +130,9 @@ des_ede_ecb_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out,
 {
 	size_t i, bl;
 
+	if (inl > LONG_MAX)
+		return 0;
+
 	bl = ctx->cipher->block_size;
 
 	if (inl < bl)
@@ -146,6 +150,9 @@ static int
 des_ede_ofb_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out,
     const unsigned char *in, size_t inl)
 {
+	if (inl > LONG_MAX)
+		return 0;
+
 	while (inl >= EVP_MAXCHUNK) {
 		DES_ede3_ofb64_encrypt(in, out, (long)EVP_MAXCHUNK,
 		    &data(ctx)->ks1, &data(ctx)->ks2, &data(ctx)->ks3,
@@ -166,6 +173,9 @@ static int
 des_ede_cbc_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out,
     const unsigned char *in, size_t inl)
 {
+	if (inl > LONG_MAX)
+		return 0;
+
 	while (inl >= EVP_MAXCHUNK) {
 		DES_ede3_cbc_encrypt(in, out, (long)EVP_MAXCHUNK,
 		    &data(ctx)->ks1, &data(ctx)->ks2, &data(ctx)->ks3,
@@ -185,6 +195,9 @@ static int
 des_ede_cfb64_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out,
     const unsigned char *in, size_t inl)
 {
+	if (inl > LONG_MAX)
+		return 0;
+
 	while (inl >= EVP_MAXCHUNK) {
 		DES_ede3_cfb64_encrypt(in, out, (long)EVP_MAXCHUNK,
 		    &data(ctx)->ks1, &data(ctx)->ks2, &data(ctx)->ks3,
@@ -208,6 +221,10 @@ des_ede3_cfb1_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out,
 {
 	size_t n;
 	unsigned char c[1], d[1];
+
+	if (inl > LONG_MAX)
+		return 0;
+
 	if (!(ctx->flags & EVP_CIPH_FLAG_LENGTH_BITS))
 		inl *= 8;
 
@@ -227,6 +244,9 @@ static int
 des_ede3_cfb8_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out,
     const unsigned char *in, size_t inl)
 {
+	if (inl > LONG_MAX)
+		return 0;
+
 	while (inl >= EVP_MAXCHUNK) {
 		DES_ede3_cfb_encrypt(in, out, 8, (long)EVP_MAXCHUNK,
 		    &data(ctx)->ks1, &data(ctx)->ks2, &data(ctx)->ks3,
diff --git a/lib/libcrypto/evp/e_idea.c b/lib/libcrypto/evp/e_idea.c
index c25f0318715..c7f2b30a44f 100644
--- a/lib/libcrypto/evp/e_idea.c
+++ b/lib/libcrypto/evp/e_idea.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: e_idea.c,v 1.14 2022/09/04 13:55:39 jsing Exp $ */
+/* $OpenBSD: e_idea.c,v 1.15 2022/09/04 15:45:25 jsing Exp $ */
 /* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com)
  * All rights reserved.
  *
@@ -56,6 +56,7 @@
  * [including the GNU Public Licence.]
  */
 
+#include <limits.h>
 #include <stdio.h>
 #include <string.h>
 
@@ -102,6 +103,9 @@ idea_ecb_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out,
 {
 	size_t i, bl;
 
+	if (inl > LONG_MAX)
+		return 0;
+
 	bl = ctx->cipher->block_size;
 
 	if (inl < bl)
@@ -121,6 +125,9 @@ typedef struct {
 static int
 idea_cbc_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, const unsigned char *in, size_t inl)
 {
+	if (inl > LONG_MAX)
+		return 0;
+
 	while (inl >= EVP_MAXCHUNK) {
 		idea_cbc_encrypt(in, out, (long)EVP_MAXCHUNK, &((EVP_IDEA_KEY *)ctx->cipher_data)->ks, ctx->iv, ctx->encrypt);
 		inl -= EVP_MAXCHUNK;
@@ -137,6 +144,9 @@ idea_cbc_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, const unsigned char *in
 static int
 idea_ofb_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, const unsigned char *in, size_t inl)
 {
+	if (inl > LONG_MAX)
+		return 0;
+
 	while (inl >= EVP_MAXCHUNK) {
 		idea_ofb64_encrypt(in, out, (long)EVP_MAXCHUNK, &((EVP_IDEA_KEY *)ctx->cipher_data)->ks, ctx->iv, &ctx->num);
 		inl -= EVP_MAXCHUNK;
@@ -155,6 +165,9 @@ idea_cfb64_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, const unsigned char *
 {
 	size_t chunk = EVP_MAXCHUNK;
 
+	if (inl > LONG_MAX)
+		return 0;
+
 	if (inl < chunk)
 		chunk = inl;
 
@@ -170,7 +183,6 @@ idea_cfb64_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, const unsigned char *
 	return 1;
 }
 
-
 static const EVP_CIPHER idea_cbc = {
 	.nid = NID_idea_cbc,
 	.block_size = 8,
diff --git a/lib/libcrypto/evp/e_rc2.c b/lib/libcrypto/evp/e_rc2.c
index 6567e75b0ca..72e582d5e01 100644
--- a/lib/libcrypto/evp/e_rc2.c
+++ b/lib/libcrypto/evp/e_rc2.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: e_rc2.c,v 1.16 2022/09/04 13:55:39 jsing Exp $ */
+/* $OpenBSD: e_rc2.c,v 1.17 2022/09/04 15:45:25 jsing Exp $ */
 /* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com)
  * All rights reserved.
  *
@@ -56,6 +56,7 @@
  * [including the GNU Public Licence.]
  */
 
+#include <limits.h>
 #include <stdio.h>
 
 #include <openssl/opensslconf.h>
@@ -87,6 +88,9 @@ typedef struct {
 static int
 rc2_cbc_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, const unsigned char *in, size_t inl)
 {
+	if (inl > LONG_MAX)
+		return 0;
+
 	while (inl >= EVP_MAXCHUNK) {
 		RC2_cbc_encrypt(in, out, (long)EVP_MAXCHUNK, &((EVP_RC2_KEY *)ctx->cipher_data)->ks, ctx->iv, ctx->encrypt);
 		inl -= EVP_MAXCHUNK;
@@ -105,6 +109,9 @@ rc2_cfb64_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, const unsigned char *i
 {
 	size_t chunk = EVP_MAXCHUNK;
 
+	if (inl > LONG_MAX)
+		return 0;
+
 	if (inl < chunk)
 		chunk = inl;
 
@@ -125,6 +132,9 @@ rc2_ecb_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, const unsigned char *in,
 {
 	size_t i, bl;
 
+	if (inl > LONG_MAX)
+		return 0;
+
 	bl = ctx->cipher->block_size;
 
 	if (inl < bl)
@@ -141,6 +151,9 @@ rc2_ecb_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, const unsigned char *in,
 static int
 rc2_ofb_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, const unsigned char *in, size_t inl)
 {
+	if (inl > LONG_MAX)
+		return 0;
+
 	while (inl >= EVP_MAXCHUNK) {
 		RC2_ofb64_encrypt(in, out, (long)EVP_MAXCHUNK, &((EVP_RC2_KEY *)ctx->cipher_data)->ks, ctx->iv, &ctx->num);
 		inl -= EVP_MAXCHUNK;