Add error handling for EVP_DigestInit_ex().

A few EVP_DigestInit_ex() calls were left alone since reporting an
error would change the public API.

Changed internal ssl3_cbc_digest_record() to return a value due to the above
change.  It will also now set md_out_size=0 on failure.

This is based on part of BoringSSL's commit to fix malloc crashes:
https://boringssl.googlesource.com/boringssl/+/69a01608f33ab6fe2c3485d94aef1fe9eacf5364

ok miod@

8 files changed, 58 insertions, 33 deletions

diff --git a/lib/libssl/src/ssl/d1_srvr.c b/lib/libssl/src/ssl/d1_srvr.c
index dee182f5416..057d92109cd 100644
--- a/lib/libssl/src/ssl/d1_srvr.c
+++ b/lib/libssl/src/ssl/d1_srvr.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: d1_srvr.c,v 1.45 2014/12/14 15:30:50 jsing Exp $ */
+/* $OpenBSD: d1_srvr.c,v 1.46 2014/12/15 00:46:53 doug Exp $ */
 /*
  * DTLS implementation written by Nagendra Modadugu
  * (nagendra@cs.stanford.edu) for the OpenSSL project 2005.
@@ -1213,8 +1213,9 @@ dtls1_send_server_key_exchange(SSL *s)
 				q = md_buf;
 				j = 0;
 				for (num = 2; num > 0; num--) {
-					EVP_DigestInit_ex(&md_ctx, (num == 2)
-					    ? s->ctx->md5 : s->ctx->sha1, NULL);
+					if (!EVP_DigestInit_ex(&md_ctx, (num == 2)
+					    ? s->ctx->md5 : s->ctx->sha1, NULL))
+						goto err;
 					EVP_DigestUpdate(&md_ctx,
 					    &(s->s3->client_random[0]),
 					    SSL3_RANDOM_SIZE);
diff --git a/lib/libssl/src/ssl/s3_cbc.c b/lib/libssl/src/ssl/s3_cbc.c
index 74bd4b47c8a..fd4781b64cc 100644
--- a/lib/libssl/src/ssl/s3_cbc.c
+++ b/lib/libssl/src/ssl/s3_cbc.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: s3_cbc.c,v 1.8 2014/07/10 08:51:14 tedu Exp $ */
+/* $OpenBSD: s3_cbc.c,v 1.9 2014/12/15 00:46:53 doug Exp $ */
 /* ====================================================================
  * Copyright (c) 2012 The OpenSSL Project.  All rights reserved.
  *
@@ -416,7 +416,8 @@ ssl3_cbc_record_digest_supported(const EVP_MD_CTX *ctx)
  * functions, above, we know that data_plus_mac_size is large enough to contain
  * a padding byte and MAC. (If the padding was invalid, it might contain the
  * padding too. ) */
-void ssl3_cbc_digest_record(const EVP_MD_CTX *ctx, unsigned char* md_out,
+int
+ssl3_cbc_digest_record(const EVP_MD_CTX *ctx, unsigned char* md_out,
     size_t* md_out_size, const unsigned char header[13],
     const unsigned char *data, size_t data_plus_mac_size,
     size_t data_plus_mac_plus_padding_size, const unsigned char *mac_secret,
@@ -497,8 +498,8 @@ void ssl3_cbc_digest_record(const EVP_MD_CTX *ctx, unsigned char* md_out,
 		 * supported. */
 		OPENSSL_assert(0);
 		if (md_out_size)
-			*md_out_size = -1;
-		return;
+			*md_out_size = 0;
+		return 0;
 	}
 
 	OPENSSL_assert(md_length_size <= MAX_HASH_BIT_COUNT_BYTES);
@@ -675,7 +676,10 @@ void ssl3_cbc_digest_record(const EVP_MD_CTX *ctx, unsigned char* md_out,
 	}
 
 	EVP_MD_CTX_init(&md_ctx);
-	EVP_DigestInit_ex(&md_ctx, ctx->digest, NULL /* engine */);
+	if (!EVP_DigestInit_ex(&md_ctx, ctx->digest, NULL /* engine */)) {
+		EVP_MD_CTX_cleanup(&md_ctx);
+		return 0;
+	}
 	if (is_sslv3) {
 		/* We repurpose |hmac_pad| to contain the SSLv3 pad2 block. */
 		memset(hmac_pad, 0x5c, sslv3_pad_length);
@@ -695,4 +699,6 @@ void ssl3_cbc_digest_record(const EVP_MD_CTX *ctx, unsigned char* md_out,
 	if (md_out_size)
 		*md_out_size = md_out_size_u;
 	EVP_MD_CTX_cleanup(&md_ctx);
+
+	return 1;
 }
diff --git a/lib/libssl/src/ssl/s3_clnt.c b/lib/libssl/src/ssl/s3_clnt.c
index 47b68245334..d1f2e05eb8f 100644
--- a/lib/libssl/src/ssl/s3_clnt.c
+++ b/lib/libssl/src/ssl/s3_clnt.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: s3_clnt.c,v 1.102 2014/12/14 16:19:38 jsing Exp $ */
+/* $OpenBSD: s3_clnt.c,v 1.103 2014/12/15 00:46:53 doug Exp $ */
 /* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com)
  * All rights reserved.
  *
@@ -1439,9 +1439,12 @@ ssl3_get_key_exchange(SSL *s)
 			j = 0;
 			q = md_buf;
 			for (num = 2; num > 0; num--) {
-				EVP_DigestInit_ex(&md_ctx,
+				if (!EVP_DigestInit_ex(&md_ctx,
 				    (num == 2) ?  s->ctx->md5 : s->ctx->sha1,
-				    NULL);
+				    NULL)) {
+					al = SSL_AD_INTERNAL_ERROR;
+					goto f_err;
+				}
 				EVP_DigestUpdate(&md_ctx,
 				    s->s3->client_random,
 				    SSL3_RANDOM_SIZE);
@@ -2245,7 +2248,8 @@ ssl3_send_client_key_exchange(SSL *s)
 				nid = NID_id_GostR3411_94;
 			else
 				nid = NID_id_tc26_gost3411_2012_256;
-			EVP_DigestInit(ukm_hash, EVP_get_digestbynid(nid));
+			if (!EVP_DigestInit(ukm_hash, EVP_get_digestbynid(nid)))
+				goto err;
 			EVP_DigestUpdate(ukm_hash,
 			    s->s3->client_random, SSL3_RANDOM_SIZE);
 			EVP_DigestUpdate(ukm_hash,
diff --git a/lib/libssl/src/ssl/s3_enc.c b/lib/libssl/src/ssl/s3_enc.c
index ec7df59f3b8..0c7cda3c60b 100644
--- a/lib/libssl/src/ssl/s3_enc.c
+++ b/lib/libssl/src/ssl/s3_enc.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: s3_enc.c,v 1.57 2014/12/10 15:43:31 jsing Exp $ */
+/* $OpenBSD: s3_enc.c,v 1.58 2014/12/15 00:46:53 doug Exp $ */
 /* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com)
  * All rights reserved.
  *
@@ -187,7 +187,8 @@ ssl3_generate_key_block(SSL *s, unsigned char *km, int num)
 		for (j = 0; j < k; j++)
 			buf[j] = c;
 		c++;
-		EVP_DigestInit_ex(&s1, EVP_sha1(), NULL);
+		if (!EVP_DigestInit_ex(&s1, EVP_sha1(), NULL))
+			return 0;
 		EVP_DigestUpdate(&s1, buf, k);
 		EVP_DigestUpdate(&s1, s->session->master_key,
 		    s->session->master_key_length);
@@ -195,7 +196,8 @@ ssl3_generate_key_block(SSL *s, unsigned char *km, int num)
 		EVP_DigestUpdate(&s1, s->s3->client_random, SSL3_RANDOM_SIZE);
 		EVP_DigestFinal_ex(&s1, smd, NULL);
 
-		EVP_DigestInit_ex(&m5, EVP_md5(), NULL);
+		if (!EVP_DigestInit_ex(&m5, EVP_md5(), NULL))
+			return 0;
 		EVP_DigestUpdate(&m5, s->session->master_key,
 		    s->session->master_key_length);
 		EVP_DigestUpdate(&m5, smd, SHA_DIGEST_LENGTH);
@@ -547,8 +549,10 @@ ssl3_digest_cached_records(SSL *s)
 				return 0;
 			}
 			if (!EVP_DigestInit_ex(s->s3->handshake_dgst[i],
-			    md, NULL))
+			    md, NULL)) {
+				EVP_MD_CTX_destroy(s->s3->handshake_dgst[i]);
 				return 0;
+			}
 			if (!EVP_DigestUpdate(s->s3->handshake_dgst[i], hdata,
 			    hdatalen))
 				return 0;
@@ -625,7 +629,8 @@ ssl3_handshake_mac(SSL *s, int md_nid, const char *sender, int len,
 	EVP_DigestUpdate(&ctx, ssl3_pad_1, npad);
 	EVP_DigestFinal_ex(&ctx, md_buf, &i);
 
-	EVP_DigestInit_ex(&ctx, EVP_MD_CTX_md(&ctx), NULL);
+	if (!EVP_DigestInit_ex(&ctx, EVP_MD_CTX_md(&ctx), NULL))
+		return 0;
 	EVP_DigestUpdate(&ctx, s->session->master_key,
 	    s->session->master_key_length);
 	EVP_DigestUpdate(&ctx, ssl3_pad_2, npad);
@@ -697,9 +702,10 @@ n_ssl3_mac(SSL *ssl, unsigned char *md, int send)
 		header[j++] = rec->length >> 8;
 		header[j++] = rec->length & 0xff;
 
-		ssl3_cbc_digest_record(hash, md, &md_size, header, rec->input,
-		    rec->length + md_size, orig_len, mac_sec, md_size,
-		    1 /* is SSLv3 */);
+		if (!ssl3_cbc_digest_record(hash, md, &md_size, header,
+		    rec->input, rec->length + md_size, orig_len, mac_sec,
+		    md_size, 1 /* is SSLv3 */))
+			return (-1);
 	} else {
 		unsigned int md_size_u;
 		/* Chop the digest off the end :-) */
@@ -757,14 +763,16 @@ ssl3_generate_master_secret(SSL *s, unsigned char *out, unsigned char *p,
 
 	EVP_MD_CTX_init(&ctx);
 	for (i = 0; i < 3; i++) {
-		EVP_DigestInit_ex(&ctx, s->ctx->sha1, NULL);
+		if (!EVP_DigestInit_ex(&ctx, s->ctx->sha1, NULL))
+			return 0;
 		EVP_DigestUpdate(&ctx, salt[i], strlen((const char *)salt[i]));
 		EVP_DigestUpdate(&ctx, p, len);
 		EVP_DigestUpdate(&ctx, s->s3->client_random, SSL3_RANDOM_SIZE);
 		EVP_DigestUpdate(&ctx, s->s3->server_random, SSL3_RANDOM_SIZE);
 		EVP_DigestFinal_ex(&ctx, buf, &n);
 
-		EVP_DigestInit_ex(&ctx, s->ctx->md5, NULL);
+		if (!EVP_DigestInit_ex(&ctx, s->ctx->md5, NULL))
+			return 0;
 		EVP_DigestUpdate(&ctx, p, len);
 		EVP_DigestUpdate(&ctx, buf, n);
 		EVP_DigestFinal_ex(&ctx, out, &n);
diff --git a/lib/libssl/src/ssl/s3_srvr.c b/lib/libssl/src/ssl/s3_srvr.c
index 783b1df782b..5e4a605c605 100644
--- a/lib/libssl/src/ssl/s3_srvr.c
+++ b/lib/libssl/src/ssl/s3_srvr.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: s3_srvr.c,v 1.94 2014/12/14 14:34:43 jsing Exp $ */
+/* $OpenBSD: s3_srvr.c,v 1.95 2014/12/15 00:46:53 doug Exp $ */
 /* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com)
  * All rights reserved.
  *
@@ -1612,9 +1612,10 @@ ssl3_send_server_key_exchange(SSL *s)
 				q = md_buf;
 				j = 0;
 				for (num = 2; num > 0; num--) {
-					EVP_DigestInit_ex(&md_ctx,
+					if (!EVP_DigestInit_ex(&md_ctx,
 					    (num == 2) ? s->ctx->md5 :
-					    s->ctx->sha1, NULL);
+					    s->ctx->sha1, NULL))
+						goto err;
 					EVP_DigestUpdate(&md_ctx,
 					    s->s3->client_random,
 					    SSL3_RANDOM_SIZE);
diff --git a/lib/libssl/src/ssl/ssl_lib.c b/lib/libssl/src/ssl/ssl_lib.c
index e809ff0bc00..8dbd4a3f392 100644
--- a/lib/libssl/src/ssl/ssl_lib.c
+++ b/lib/libssl/src/ssl/ssl_lib.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: ssl_lib.c,v 1.93 2014/12/14 14:34:43 jsing Exp $ */
+/* $OpenBSD: ssl_lib.c,v 1.94 2014/12/15 00:46:53 doug Exp $ */
 /* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com)
  * All rights reserved.
  *
@@ -3033,8 +3033,12 @@ ssl_replace_hash(EVP_MD_CTX **hash, const EVP_MD *md)
 {
 	ssl_clear_hash_ctx(hash);
 	*hash = EVP_MD_CTX_create();
-	if (*hash != NULL && md != NULL)
-		EVP_DigestInit_ex(*hash, md, NULL);
+	if (*hash != NULL && md != NULL) {
+		if (!EVP_DigestInit_ex(*hash, md, NULL)) {
+			ssl_clear_hash_ctx(hash);
+			return (NULL);
+		}
+	}
 	return (*hash);
 }
 
diff --git a/lib/libssl/src/ssl/ssl_locl.h b/lib/libssl/src/ssl/ssl_locl.h
index 97e32de3801..3312aebaada 100644
--- a/lib/libssl/src/ssl/ssl_locl.h
+++ b/lib/libssl/src/ssl/ssl_locl.h
@@ -1,4 +1,4 @@
-/* $OpenBSD: ssl_locl.h,v 1.83 2014/12/14 16:19:38 jsing Exp $ */
+/* $OpenBSD: ssl_locl.h,v 1.84 2014/12/15 00:46:53 doug Exp $ */
 /* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com)
  * All rights reserved.
  *
@@ -865,7 +865,7 @@ int ssl3_cbc_remove_padding(const SSL *s, SSL3_RECORD *rec,
 int tls1_cbc_remove_padding(const SSL *s, SSL3_RECORD *rec,
     unsigned block_size, unsigned mac_size);
 char ssl3_cbc_record_digest_supported(const EVP_MD_CTX *ctx);
-void ssl3_cbc_digest_record(const EVP_MD_CTX *ctx, unsigned char *md_out,
+int ssl3_cbc_digest_record(const EVP_MD_CTX *ctx, unsigned char *md_out,
     size_t *md_out_size, const unsigned char header[13],
     const unsigned char *data, size_t data_plus_mac_size,
     size_t data_plus_mac_plus_padding_size, const unsigned char *mac_secret,
diff --git a/lib/libssl/src/ssl/t1_enc.c b/lib/libssl/src/ssl/t1_enc.c
index 4aae344696b..3b7e625db33 100644
--- a/lib/libssl/src/ssl/t1_enc.c
+++ b/lib/libssl/src/ssl/t1_enc.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: t1_enc.c,v 1.74 2014/12/14 15:30:50 jsing Exp $ */
+/* $OpenBSD: t1_enc.c,v 1.75 2014/12/15 00:46:53 doug Exp $ */
 /* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com)
  * All rights reserved.
  *
@@ -1054,12 +1054,13 @@ tls1_mac(SSL *ssl, unsigned char *md, int send)
 		 * timing-side channel information about how many blocks of
 		 * data we are hashing because that gives an attacker a
 		 * timing-oracle. */
-		ssl3_cbc_digest_record(mac_ctx,
+		if (!ssl3_cbc_digest_record(mac_ctx,
 		    md, &md_size, header, rec->input,
 		    rec->length + md_size, orig_len,
 		    ssl->s3->read_mac_secret,
 		    ssl->s3->read_mac_secret_size,
-		    0 /* not SSLv3 */);
+		    0 /* not SSLv3 */))
+			return -1;
 	} else {
 		EVP_DigestSignUpdate(mac_ctx, header, sizeof(header));
 		EVP_DigestSignUpdate(mac_ctx, rec->input, rec->length);