Move towards making RSA OAEP functions handle arbitrary message digests.

Based on OpenSSL 1.1.1.

ok tb@, inoguchi@ (on an earlier/larger diff)

1 files changed, 59 insertions, 53 deletions

diff --git a/lib/libcrypto/rsa/rsa_oaep.c b/lib/libcrypto/rsa/rsa_oaep.c
index c463a885d27..2b902f44b42 100644
--- a/lib/libcrypto/rsa/rsa_oaep.c
+++ b/lib/libcrypto/rsa/rsa_oaep.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: rsa_oaep.c,v 1.29 2018/08/19 20:17:20 tb Exp $ */
+/* $OpenBSD: rsa_oaep.c,v 1.30 2019/10/03 17:45:27 jsing Exp $ */
 /* Written by Ulf Moeller. This software is distributed on an "AS IS"
    basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. */
 
@@ -32,63 +32,68 @@
 #include <openssl/rsa.h>
 #include <openssl/sha.h>
 
-static int MGF1(unsigned char *mask, long len, const unsigned char *seed,
-    long seedlen);
-
 int
 RSA_padding_add_PKCS1_OAEP(unsigned char *to, int tlen,
     const unsigned char *from, int flen, const unsigned char *param, int plen)
 {
 	int i, emlen = tlen - 1;
 	unsigned char *db, *seed;
-	unsigned char *dbmask, seedmask[SHA_DIGEST_LENGTH];
+	unsigned char *dbmask = NULL;
+	unsigned char seedmask[EVP_MAX_MD_SIZE];
+	const EVP_MD *md, *mgf1md;
+	int mdlen, dbmask_len = 0;
+	int rv = 0;
+
+	md = EVP_sha1();
+	mgf1md = EVP_sha1();
+
+	if ((mdlen = EVP_MD_size(md)) <= 0)
+		goto err;
 
-	if (flen > emlen - 2 * SHA_DIGEST_LENGTH - 1) {
+	if (flen > emlen - 2 * mdlen - 1) {
 		RSAerror(RSA_R_DATA_TOO_LARGE_FOR_KEY_SIZE);
-		return 0;
+		goto err;
 	}
 
-	if (emlen < 2 * SHA_DIGEST_LENGTH + 1) {
+	if (emlen < 2 * mdlen + 1) {
 		RSAerror(RSA_R_KEY_SIZE_TOO_SMALL);
-		return 0;
+		goto err;
 	}
 
 	to[0] = 0;
 	seed = to + 1;
-	db = to + SHA_DIGEST_LENGTH + 1;
-
-	if (!EVP_Digest((void *)param, plen, db, NULL, EVP_sha1(), NULL))
-		return 0;
-	memset(db + SHA_DIGEST_LENGTH, 0,
-	    emlen - flen - 2 * SHA_DIGEST_LENGTH - 1);
-	db[emlen - flen - SHA_DIGEST_LENGTH - 1] = 0x01;
-	memcpy(db + emlen - flen - SHA_DIGEST_LENGTH, from, flen);
-	arc4random_buf(seed, SHA_DIGEST_LENGTH);
-
-	dbmask = malloc(emlen - SHA_DIGEST_LENGTH);
-	if (dbmask == NULL) {
+	db = to + mdlen + 1;
+
+	if (!EVP_Digest((void *)param, plen, db, NULL, md, NULL))
+		goto err;
+
+	memset(db + mdlen, 0, emlen - flen - 2 * mdlen - 1);
+	db[emlen - flen - mdlen - 1] = 0x01;
+	memcpy(db + emlen - flen - mdlen, from, flen);
+	arc4random_buf(seed, mdlen);
+
+	dbmask_len = emlen - mdlen;
+	if ((dbmask = malloc(dbmask_len)) == NULL) {
 		RSAerror(ERR_R_MALLOC_FAILURE);
-		return 0;
+		goto err;
 	}
 
-	if (MGF1(dbmask, emlen - SHA_DIGEST_LENGTH, seed,
-	    SHA_DIGEST_LENGTH) < 0) {
-		free(dbmask);
-		return 0;
-	}
-	for (i = 0; i < emlen - SHA_DIGEST_LENGTH; i++)
+	if (PKCS1_MGF1(dbmask, dbmask_len, seed, mdlen, mgf1md) < 0)
+		goto err;
+	for (i = 0; i < dbmask_len; i++)
 		db[i] ^= dbmask[i];
-
-	if (MGF1(seedmask, SHA_DIGEST_LENGTH, db,
-	    emlen - SHA_DIGEST_LENGTH) < 0) {
-		free(dbmask);
-		return 0;
-	}
-	for (i = 0; i < SHA_DIGEST_LENGTH; i++)
+	if (PKCS1_MGF1(seedmask, mdlen, db, dbmask_len, mgf1md) < 0)
+		goto err;
+	for (i = 0; i < mdlen; i++)
 		seed[i] ^= seedmask[i];
 
-	free(dbmask);
-	return 1;
+	rv = 1;
+
+ err:
+	explicit_bzero(seedmask, sizeof(seedmask));
+	freezero(dbmask, dbmask_len);
+
+	return rv;
 }
 
 int
@@ -103,8 +108,16 @@ RSA_padding_check_PKCS1_OAEP(unsigned char *to, int tlen,
 	unsigned char seed[SHA_DIGEST_LENGTH], phash[SHA_DIGEST_LENGTH];
 	unsigned char *padded_from;
 	int bad = 0;
+	const EVP_MD *md, *mgf1md;
+	int mdlen;
 
-	if (--num < 2 * SHA_DIGEST_LENGTH + 1)
+	md = EVP_sha1();
+	mgf1md = EVP_sha1();
+
+	if ((mdlen = EVP_MD_size(md)) <= 0)
+		goto err;
+
+	if (--num < 2 * mdlen + 1)
 		/*
 		 * 'num' is the length of the modulus, i.e. does not depend
 		 * on the particular ciphertext.
@@ -125,7 +138,7 @@ RSA_padding_check_PKCS1_OAEP(unsigned char *to, int tlen,
 		flen = num; /* don't overflow the memcpy to padded_from */
 	}
 
-	dblen = num - SHA_DIGEST_LENGTH;
+	dblen = num - mdlen;
 	if ((db = malloc(dblen + num)) == NULL) {
 		RSAerror(ERR_R_MALLOC_FAILURE);
 		return -1;
@@ -139,25 +152,24 @@ RSA_padding_check_PKCS1_OAEP(unsigned char *to, int tlen,
 	memset(padded_from, 0, lzero);
 	memcpy(padded_from + lzero, from, flen);
 
-	maskeddb = padded_from + SHA_DIGEST_LENGTH;
+	maskeddb = padded_from + mdlen;
 
-	if (MGF1(seed, SHA_DIGEST_LENGTH, maskeddb, dblen))
+	if (PKCS1_MGF1(seed, mdlen, maskeddb, dblen, mgf1md))
 		goto err;
-	for (i = 0; i < SHA_DIGEST_LENGTH; i++)
+	for (i = 0; i < mdlen; i++)
 		seed[i] ^= padded_from[i];
-
-	if (MGF1(db, dblen, seed, SHA_DIGEST_LENGTH))
+	if (PKCS1_MGF1(db, dblen, seed, mdlen, mgf1md))
 		goto err;
 	for (i = 0; i < dblen; i++)
 		db[i] ^= maskeddb[i];
 
-	if (!EVP_Digest((void *)param, plen, phash, NULL, EVP_sha1(), NULL))
+	if (!EVP_Digest((void *)param, plen, phash, NULL, md, NULL))
 		goto err;
 
-	if (timingsafe_memcmp(db, phash, SHA_DIGEST_LENGTH) != 0 || bad)
+	if (timingsafe_memcmp(db, phash, mdlen) != 0 || bad)
 		goto decoding_err;
 	else {
-		for (i = SHA_DIGEST_LENGTH; i < dblen; i++)
+		for (i = mdlen; i < dblen; i++)
 			if (db[i] != 0x00)
 				break;
 		if (i == dblen || db[i] != 0x01)
@@ -227,10 +239,4 @@ PKCS1_MGF1(unsigned char *mask, long len, const unsigned char *seed,
 	EVP_MD_CTX_cleanup(&c);
 	return rv;
 }
-
-static int
-MGF1(unsigned char *mask, long len, const unsigned char *seed, long seedlen)
-{
-	return PKCS1_MGF1(mask, len, seed, seedlen, EVP_sha1());
-}
 #endif