Convert signatures and verifcation to use the EVP_DigestXXX api

to allow for adding PSS, Nuke the now unneejded guard around the PSS
algorithms in the sigalgs table
ok jsing@ tb@

3 files changed, 93 insertions, 45 deletions

diff --git a/lib/libssl/ssl_clnt.c b/lib/libssl/ssl_clnt.c
index ac2cddacf9a..298e4b7ff89 100644
--- a/lib/libssl/ssl_clnt.c
+++ b/lib/libssl/ssl_clnt.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: ssl_clnt.c,v 1.41 2018/11/10 01:19:09 beck Exp $ */
+/* $OpenBSD: ssl_clnt.c,v 1.42 2018/11/11 02:03:23 beck Exp $ */
 /* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com)
  * All rights reserved.
  *
@@ -1508,15 +1508,21 @@ ssl3_get_server_key_exchange(SSL *s)
 
 	/* if it was signed, check the signature */
 	if (pkey != NULL) {
+		EVP_PKEY_CTX *pctx;
+		const struct ssl_sigalg *sigalg;
+
 		if (SSL_USE_SIGALGS(s)) {
-			const struct ssl_sigalg *sigalg;
 			uint16_t sigalg_value;
 
 			if (!CBS_get_u16(&cbs, &sigalg_value))
 				goto truncated;
 			if ((sigalg = ssl_sigalg(sigalg_value, tls12_sigalgs,
-			    tls12_sigalgs_len)) == NULL ||
-			    (md = sigalg->md()) == NULL) {
+				    tls12_sigalgs_len)) == NULL) {
+				SSLerror(s, SSL_R_UNKNOWN_DIGEST);
+				al = SSL_AD_DECODE_ERROR;
+				goto f_err;
+			}
+			if ((md = sigalg->md()) == NULL) {
 				SSLerror(s, SSL_R_UNKNOWN_DIGEST);
 				al = SSL_AD_DECODE_ERROR;
 				goto f_err;
@@ -1527,10 +1533,15 @@ ssl3_get_server_key_exchange(SSL *s)
 				goto f_err;
 			}
 		} else if (pkey->type == EVP_PKEY_RSA) {
-			md = EVP_md5_sha1();
+			sigalg = ssl_sigalg_lookup(SIGALG_RSA_PKCS1_SHA1);
+		} else if (pkey->type == EVP_PKEY_EC) {
+			sigalg = ssl_sigalg_lookup(SIGALG_ECDSA_SHA1);
 		} else {
-			md = EVP_sha1();
+			SSLerror(s, SSL_R_UNKNOWN_PKEY_TYPE);
+			al = SSL_AD_DECODE_ERROR;
+			goto f_err;
 		}
+		md = sigalg->md();
 
 		if (!CBS_get_u16_length_prefixed(&cbs, &signature))
 			goto truncated;
@@ -1540,18 +1551,18 @@ ssl3_get_server_key_exchange(SSL *s)
 			goto f_err;
 		}
 
-		if (!EVP_VerifyInit_ex(&md_ctx, md, NULL))
+		if (!EVP_DigestVerifyInit(&md_ctx, &pctx, md, NULL, pkey))
 			goto err;
-		if (!EVP_VerifyUpdate(&md_ctx, s->s3->client_random,
+		if (!EVP_DigestVerifyUpdate(&md_ctx, s->s3->client_random,
 		    SSL3_RANDOM_SIZE))
 			goto err;
-		if (!EVP_VerifyUpdate(&md_ctx, s->s3->server_random,
+		if (!EVP_DigestVerifyUpdate(&md_ctx, s->s3->server_random,
 		    SSL3_RANDOM_SIZE))
 			goto err;
-		if (!EVP_VerifyUpdate(&md_ctx, param, param_len))
+		if (!EVP_DigestVerifyUpdate(&md_ctx, param, param_len))
 			goto err;
-		if (EVP_VerifyFinal(&md_ctx, CBS_data(&signature),
-		    CBS_len(&signature), pkey) <= 0) {
+		if (EVP_DigestVerifyFinal(&md_ctx, CBS_data(&signature),
+		    CBS_len(&signature)) <= 0) {
 			al = SSL_AD_DECRYPT_ERROR;
 			SSLerror(s, SSL_R_BAD_SIGNATURE);
 			goto f_err;
@@ -2363,13 +2374,15 @@ ssl3_send_client_verify(SSL *s)
 	CBB cbb, cert_verify, cbb_signature;
 	unsigned char data[MD5_DIGEST_LENGTH + SHA_DIGEST_LENGTH];
 	unsigned char *signature = NULL;
-	unsigned int signature_len;
+	unsigned int signature_len = 0;
 	const unsigned char *hdata;
 	size_t hdatalen;
 	EVP_PKEY_CTX *pctx = NULL;
 	EVP_PKEY *pkey;
 	EVP_MD_CTX mctx;
 	const EVP_MD *md;
+	size_t siglen;
+
 
 	EVP_MD_CTX_init(&mctx);
 
@@ -2379,12 +2392,12 @@ ssl3_send_client_verify(SSL *s)
 		if (!ssl3_handshake_msg_start(s, &cbb, &cert_verify,
 		    SSL3_MT_CERTIFICATE_VERIFY))
 			goto err;
-
 		/*
 		 * Create context from key and test if sha1 is allowed as
 		 * digest.
 		 */
 		pkey = s->cert->key->privatekey;
+		md = s->cert->key->sigalg->md();
 		pctx = EVP_PKEY_CTX_new(pkey, NULL);
 		EVP_PKEY_sign_init(pctx);
 
@@ -2392,37 +2405,50 @@ ssl3_send_client_verify(SSL *s)
 		if (EVP_PKEY_CTX_set_signature_md(pctx, EVP_sha1()) <= 0)
 			ERR_clear_error();
 
-		if ((signature = calloc(1, EVP_PKEY_size(pkey))) == NULL)
-			goto err;
-
 		if (!SSL_USE_SIGALGS(s)) {
 			tls1_transcript_free(s);
 			if (!tls1_handshake_hash_value(s, data, sizeof(data),
 			    NULL))
 				goto err;
 		}
-
 		/*
 		 * For TLS v1.2 send signature algorithm and signature
 		 * using agreed digest and cached handshake records.
 		 */
 		if (SSL_USE_SIGALGS(s)) {
-			md = s->cert->key->sigalg->md();
+			EVP_PKEY_CTX *pctx;
 			if (!tls1_transcript_data(s, &hdata, &hdatalen) ||
 			    !CBB_add_u16(&cert_verify,
-				s->cert->key->sigalg->value)) {
+			    s->cert->key->sigalg->value)) {
 				SSLerror(s, ERR_R_INTERNAL_ERROR);
 				goto err;
 			}
-			if (!EVP_SignInit_ex(&mctx, md, NULL) ||
-			    !EVP_SignUpdate(&mctx, hdata, hdatalen) ||
-			    !EVP_SignFinal(&mctx, signature, &signature_len,
-				pkey)) {
+			if (!EVP_DigestSignInit(&mctx, &pctx, md, NULL, pkey)) {
 				SSLerror(s, ERR_R_EVP_LIB);
 				goto err;
 			}
+			if (!EVP_DigestSignUpdate(&mctx, hdata, hdatalen)) {
+				SSLerror(s, ERR_R_EVP_LIB);
+				goto err;
+			}
+			if (!EVP_DigestSignFinal(&mctx, NULL, &siglen) ||
+			    siglen == 0) {
+				SSLerror(s, ERR_R_EVP_LIB);
+				goto err;
+			}
+			if ((signature = calloc(1, siglen)) == NULL) {
+				SSLerror(s, ERR_R_MALLOC_FAILURE);
+				goto err;
+			}
+			if (!EVP_DigestSignFinal(&mctx, signature, &siglen)) {
+				SSLerror(s, ERR_R_EVP_LIB);
+				goto err;
+			}
+			signature_len = siglen; /* XXX */
 			tls1_transcript_free(s);
 		} else if (pkey->type == EVP_PKEY_RSA) {
+			if ((signature = calloc(1, EVP_PKEY_size(pkey))) == NULL)
+				goto err;
 			if (RSA_sign(NID_md5_sha1, data,
 			    MD5_DIGEST_LENGTH + SHA_DIGEST_LENGTH, signature,
 			    &signature_len, pkey->pkey.rsa) <= 0 ) {
@@ -2430,6 +2456,8 @@ ssl3_send_client_verify(SSL *s)
 				goto err;
 			}
 		} else if (pkey->type == EVP_PKEY_EC) {
+			if ((signature = calloc(1, EVP_PKEY_size(pkey))) == NULL)
+				goto err;
 			if (!ECDSA_sign(pkey->save_type,
 			    &data[MD5_DIGEST_LENGTH], SHA_DIGEST_LENGTH,
 			    signature, &signature_len, pkey->pkey.ec)) {
diff --git a/lib/libssl/ssl_sigalgs.c b/lib/libssl/ssl_sigalgs.c
index cee3f0bf6de..5dc261810bf 100644
--- a/lib/libssl/ssl_sigalgs.c
+++ b/lib/libssl/ssl_sigalgs.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: ssl_sigalgs.c,v 1.5 2018/11/10 08:42:39 beck Exp $ */
+/* $OpenBSD: ssl_sigalgs.c,v 1.6 2018/11/11 02:03:23 beck Exp $ */
 /*
  * Copyright (c) 2018, Bob Beck <beck@openbsd.org>
  *
@@ -83,7 +83,6 @@ const struct ssl_sigalg sigalgs[] = {
 		.pkey_idx = SSL_PKEY_GOST01,
 	},
 #endif
-#ifdef LIBRESSL_HAS_TLS1_3
 	{
 		.value = SIGALG_RSA_PSS_RSAE_SHA256,
 		.md = EVP_sha256,
@@ -126,7 +125,6 @@ const struct ssl_sigalg sigalgs[] = {
 		.pkey_idx = SSL_PKEY_RSA_SIGN,
 		.flags = SIGALG_FLAG_RSA_PSS,
 	},
-#endif
 	{
 		.value = SIGALG_RSA_PKCS1_SHA224,
 		.md = EVP_sha224,
diff --git a/lib/libssl/ssl_srvr.c b/lib/libssl/ssl_srvr.c
index 587a538060a..f1b8a494688 100644
--- a/lib/libssl/ssl_srvr.c
+++ b/lib/libssl/ssl_srvr.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: ssl_srvr.c,v 1.55 2018/11/10 01:19:09 beck Exp $ */
+/* $OpenBSD: ssl_srvr.c,v 1.56 2018/11/11 02:03:23 beck Exp $ */
 /* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com)
  * All rights reserved.
  *
@@ -345,7 +345,7 @@ ssl3_accept(SSL *s)
 					D1I(s)->handshake_write_seq = 1;
 					D1I(s)->next_handshake_write_seq = 1;
 					goto end;
-				}				
+				}
 			} else {
 				if (s->internal->rwstate != SSL_X509_LOOKUP) {
 					ret = ssl3_get_client_hello(s);
@@ -1485,12 +1485,13 @@ ssl3_send_server_key_exchange(SSL *s)
 	CBB cbb, cbb_params, cbb_signature, server_kex;
 	const struct ssl_sigalg *sigalg = NULL;
 	unsigned char *signature = NULL;
-	unsigned int signature_len;
+	size_t signature_len = 0;
 	unsigned char *params = NULL;
 	size_t params_len;
 	const EVP_MD *md = NULL;
 	unsigned long type;
 	EVP_MD_CTX md_ctx;
+	EVP_PKEY_CTX *pctx;
 	EVP_PKEY *pkey;
 	int al;
 
@@ -1544,21 +1545,34 @@ ssl3_send_server_key_exchange(SSL *s)
 				}
 			}
 
-			if ((signature = calloc(1, EVP_PKEY_size(pkey))) == NULL)
+			if (!EVP_DigestSignInit(&md_ctx, &pctx, md, NULL, pkey)) {
+				SSLerror(s, ERR_R_EVP_LIB);
 				goto err;
-
-			if (!EVP_SignInit_ex(&md_ctx, md, NULL))
+			}
+			if (!EVP_DigestSignUpdate(&md_ctx, s->s3->client_random,
+			    SSL3_RANDOM_SIZE)) {
+				SSLerror(s, ERR_R_EVP_LIB);
 				goto err;
-			if (!EVP_SignUpdate(&md_ctx, s->s3->client_random,
-			    SSL3_RANDOM_SIZE))
+			}
+			if (!EVP_DigestSignUpdate(&md_ctx, s->s3->server_random,
+			    SSL3_RANDOM_SIZE)) {
+				SSLerror(s, ERR_R_EVP_LIB);
+				goto err;
+			}
+			if (!EVP_DigestSignUpdate(&md_ctx, params, params_len)) {
+				SSLerror(s, ERR_R_EVP_LIB);
 				goto err;
-			if (!EVP_SignUpdate(&md_ctx, s->s3->server_random,
-			    SSL3_RANDOM_SIZE))
+			}
+			if (!EVP_DigestSignFinal(&md_ctx, NULL, &signature_len) ||
+			    !signature_len) {
+				SSLerror(s, ERR_R_EVP_LIB);
 				goto err;
-			if (!EVP_SignUpdate(&md_ctx, params, params_len))
+			}
+			if ((signature = calloc(1, signature_len)) == NULL) {
+				SSLerror(s, ERR_R_MALLOC_FAILURE);
 				goto err;
-			if (!EVP_SignFinal(&md_ctx, signature, &signature_len,
-			    pkey)) {
+			}
+			if (!EVP_DigestSignFinal(&md_ctx, signature, &signature_len)) {
 				SSLerror(s, ERR_R_EVP_LIB);
 				goto err;
 			}
@@ -2071,6 +2085,7 @@ int
 ssl3_get_cert_verify(SSL *s)
 {
 	CBS cbs, signature;
+	const struct ssl_sigalg *sigalg;
 	const EVP_MD *md = NULL;
 	EVP_PKEY *pkey = NULL;
 	X509 *peer = NULL;
@@ -2135,14 +2150,16 @@ ssl3_get_cert_verify(SSL *s)
 	 * If key is GOST and n is exactly 64, it is a bare
 	 * signature without length field.
 	 */
+	/* This hack is awful and needs to die in fire */
 	if ((pkey->type == NID_id_GostR3410_94 ||
 	     pkey->type == NID_id_GostR3410_2001) && CBS_len(&cbs) == 64) {
+		if (SSL_USE_SIGALGS(s))
+			goto truncated;
 		CBS_dup(&cbs, &signature);
 		if (!CBS_skip(&cbs, CBS_len(&cbs)))
 			goto err;
 	} else {
 		if (SSL_USE_SIGALGS(s)) {
-			const struct ssl_sigalg *sigalg;
 			uint16_t sigalg_value;
 
 			if (!CBS_get_u16(&cbs, &sigalg_value))
@@ -2175,19 +2192,24 @@ ssl3_get_cert_verify(SSL *s)
 	}
 
 	if (SSL_USE_SIGALGS(s)) {
+		EVP_PKEY_CTX *pctx;
 		if (!tls1_transcript_data(s, &hdata, &hdatalen)) {
 			SSLerror(s, ERR_R_INTERNAL_ERROR);
 			al = SSL_AD_INTERNAL_ERROR;
 			goto f_err;
 		}
-		if (!EVP_VerifyInit_ex(&mctx, md, NULL) ||
-		    !EVP_VerifyUpdate(&mctx, hdata, hdatalen)) {
+		if (!EVP_DigestVerifyInit(&mctx, &pctx, md, NULL, pkey)) {
+			SSLerror(s, ERR_R_EVP_LIB);
+			al = SSL_AD_INTERNAL_ERROR;
+			goto f_err;
+		}
+		if (!EVP_DigestVerifyUpdate(&mctx, hdata, hdatalen)) {
 			SSLerror(s, ERR_R_EVP_LIB);
 			al = SSL_AD_INTERNAL_ERROR;
 			goto f_err;
 		}
-		if (EVP_VerifyFinal(&mctx, CBS_data(&signature),
-		    CBS_len(&signature), pkey) <= 0) {
+		if (EVP_DigestVerifyFinal(&mctx, CBS_data(&signature),
+		    CBS_len(&signature)) <= 0) {
 			al = SSL_AD_DECRYPT_ERROR;
 			SSLerror(s, SSL_R_BAD_SIGNATURE);
 			goto f_err;