Add check function to verify that pkey is usable with a sigalg.

Include check for appropriate RSA key size when used with PSS.
ok tb@

4 files changed, 24 insertions, 7 deletions

diff --git a/lib/libssl/ssl_clnt.c b/lib/libssl/ssl_clnt.c
index 9f8d999ff19..20944179947 100644
--- a/lib/libssl/ssl_clnt.c
+++ b/lib/libssl/ssl_clnt.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: ssl_clnt.c,v 1.43 2018/11/11 02:22:34 beck Exp $ */
+/* $OpenBSD: ssl_clnt.c,v 1.44 2018/11/11 21:54:47 beck Exp $ */
 /* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com)
  * All rights reserved.
  *
@@ -1527,7 +1527,7 @@ ssl3_get_server_key_exchange(SSL *s)
 				al = SSL_AD_DECODE_ERROR;
 				goto f_err;
 			}
-			if (sigalg->key_type != pkey->type) {
+			if (!ssl_sigalg_pkey_ok(sigalg, pkey)) {
 				SSLerror(s, SSL_R_WRONG_SIGNATURE_TYPE);
 				al = SSL_AD_DECODE_ERROR;
 				goto f_err;
diff --git a/lib/libssl/ssl_sigalgs.c b/lib/libssl/ssl_sigalgs.c
index 5dc261810bf..a6c5a4e9d8d 100644
--- a/lib/libssl/ssl_sigalgs.c
+++ b/lib/libssl/ssl_sigalgs.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: ssl_sigalgs.c,v 1.6 2018/11/11 02:03:23 beck Exp $ */
+/* $OpenBSD: ssl_sigalgs.c,v 1.7 2018/11/11 21:54:47 beck Exp $ */
 /*
  * Copyright (c) 2018, Bob Beck <beck@openbsd.org>
  *
@@ -225,3 +225,19 @@ ssl_sigalgs_build(CBB *cbb, uint16_t *values, size_t len)
 	}
 	return 1;
 }
+
+int
+ssl_sigalg_pkey_ok(const struct ssl_sigalg *sigalg, EVP_PKEY *pkey)
+{
+	if (sigalg->key_type == pkey->type) {
+		if (!(sigalg->flags & SIGALG_FLAG_RSA_PSS))
+			return 1;
+		/*
+		 * RSA keys for PSS need to be at least
+		 * as big as twice the size of the hash + 2
+		 */
+		if (EVP_PKEY_size(pkey) < (2 * EVP_MD_size(sigalg->md()) + 2))
+			return 1;
+	}
+	return 0;
+}
diff --git a/lib/libssl/ssl_sigalgs.h b/lib/libssl/ssl_sigalgs.h
index f6fb5e2ad60..1bce6e8ee3d 100644
--- a/lib/libssl/ssl_sigalgs.h
+++ b/lib/libssl/ssl_sigalgs.h
@@ -1,4 +1,4 @@
-/* $OpenBSD: ssl_sigalgs.h,v 1.6 2018/11/10 08:42:39 beck Exp $ */
+/* $OpenBSD: ssl_sigalgs.h,v 1.7 2018/11/11 21:54:47 beck Exp $ */
 /*
  * Copyright (c) 2018, Bob Beck <beck@openbsd.org>
  *
@@ -73,6 +73,7 @@ const struct ssl_sigalg *ssl_sigalg_lookup(uint16_t sigalg);
 const struct ssl_sigalg *ssl_sigalg(uint16_t sigalg, uint16_t *values, size_t len);
 int ssl_sigalgs_build(CBB *cbb, uint16_t *values, size_t len);
 int ssl_sigalg_pkey_check(uint16_t sigalg, EVP_PKEY *pk);
+int ssl_sigalg_pkey_ok(const struct ssl_sigalg *sigalg, EVP_PKEY *pkey);
 
 __END_HIDDEN_DECLS
 
diff --git a/lib/libssl/ssl_srvr.c b/lib/libssl/ssl_srvr.c
index e09817e2d02..27024be856a 100644
--- a/lib/libssl/ssl_srvr.c
+++ b/lib/libssl/ssl_srvr.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: ssl_srvr.c,v 1.59 2018/11/11 07:57:44 bcook Exp $ */
+/* $OpenBSD: ssl_srvr.c,v 1.60 2018/11/11 21:54:47 beck Exp $ */
 /* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com)
  * All rights reserved.
  *
@@ -2179,7 +2179,7 @@ ssl3_get_cert_verify(SSL *s)
 				al = SSL_AD_DECODE_ERROR;
 				goto f_err;
 			}
-			if (sigalg->key_type != pkey->type) {
+			if (!ssl_sigalg_pkey_ok(sigalg, pkey)) {
 				SSLerror(s, SSL_R_WRONG_SIGNATURE_TYPE);
 				al = SSL_AD_DECODE_ERROR;
 				goto f_err;
@@ -2216,7 +2216,7 @@ ssl3_get_cert_verify(SSL *s)
 		    (pctx, RSA_PKCS1_PSS_PADDING) ||
 		    !EVP_PKEY_CTX_set_rsa_pss_saltlen(pctx, -1))) {
 			al = SSL_AD_INTERNAL_ERROR;
-			goto err;
+			goto f_err;
 		}
 		if (!EVP_DigestVerifyUpdate(&mctx, hdata, hdatalen)) {
 			SSLerror(s, ERR_R_EVP_LIB);