Fix BN_is_prime_* calls in libcrypto, the API returns -1 on error.

From BoringSSL's commit 53409ee3d7595ed37da472bc73b010cd2c8a5ffd
by David Benjamin.

ok djm, jsing

3 files changed, 35 insertions, 18 deletions

diff --git a/lib/libcrypto/bn/bn_x931p.c b/lib/libcrypto/bn/bn_x931p.c
index 45b61c91280..55ca21c08c3 100644
--- a/lib/libcrypto/bn/bn_x931p.c
+++ b/lib/libcrypto/bn/bn_x931p.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: bn_x931p.c,v 1.10 2017/01/25 06:15:44 beck Exp $ */
+/* $OpenBSD: bn_x931p.c,v 1.11 2019/01/20 01:56:59 tb Exp $ */
 /* Written by Dr Stephen N Henson (steve@openssl.org) for the OpenSSL
  * project 2005.
  */
@@ -71,7 +71,7 @@
 static int
 bn_x931_derive_pi(BIGNUM *pi, const BIGNUM *Xpi, BN_CTX *ctx, BN_GENCB *cb)
 {
-	int i = 0;
+	int i = 0, is_prime;
 
 	if (!BN_copy(pi, Xpi))
 		return 0;
@@ -81,7 +81,10 @@ bn_x931_derive_pi(BIGNUM *pi, const BIGNUM *Xpi, BN_CTX *ctx, BN_GENCB *cb)
 		i++;
 		BN_GENCB_call(cb, 0, i);
 		/* NB 27 MR is specificed in X9.31 */
-		if (BN_is_prime_fasttest_ex(pi, 27, ctx, 1, cb))
+		is_prime = BN_is_prime_fasttest_ex(pi, 27, ctx, 1, cb);
+		if (is_prime < 0)
+			return 0;
+		if (is_prime == 1)
 			break;
 		if (!BN_add_word(pi, 2))
 			return 0;
@@ -173,13 +176,20 @@ BN_X931_derive_prime_ex(BIGNUM *p, BIGNUM *p1, BIGNUM *p2, const BIGNUM *Xp,
 			goto err;
 		if (!BN_gcd_ct(t, pm1, e, ctx))
 			goto err;
-		if (BN_is_one(t)
-		/* X9.31 specifies 8 MR and 1 Lucas test or any prime test
-		 * offering similar or better guarantees 50 MR is considerably
-		 * better.
-		 */
-		    && BN_is_prime_fasttest_ex(p, 50, ctx, 1, cb))
-			break;
+		if (BN_is_one(t)) {
+			int r;
+
+			/*
+			 * X9.31 specifies 8 MR and 1 Lucas test or any prime
+			 * test offering similar or better guarantees 50 MR
+			 * is considerably better.
+			 */
+			r = BN_is_prime_fasttest_ex(p, 50, ctx, 1, cb);
+			if (r < 0)
+				goto err;
+			if (r == 1)
+				break;
+		}
 		if (!BN_add(p, p, p1p2))
 			goto err;
 	}
diff --git a/lib/libcrypto/dh/dh_check.c b/lib/libcrypto/dh/dh_check.c
index a6010f0a6dc..a8227d31ca6 100644
--- a/lib/libcrypto/dh/dh_check.c
+++ b/lib/libcrypto/dh/dh_check.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: dh_check.c,v 1.16 2016/07/05 02:54:35 bcook Exp $ */
+/* $OpenBSD: dh_check.c,v 1.17 2019/01/20 01:56:59 tb Exp $ */
 /* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com)
  * All rights reserved.
  *
@@ -74,7 +74,7 @@
 int
 DH_check(const DH *dh, int *ret)
 {
-	int ok = 0;
+	int is_prime, ok = 0;
 	BN_CTX *ctx = NULL;
 	BN_ULONG l;
 	BIGNUM *q = NULL;
@@ -102,16 +102,23 @@ DH_check(const DH *dh, int *ret)
 	} else
 		*ret |= DH_UNABLE_TO_CHECK_GENERATOR;
 
-	if (!BN_is_prime_ex(dh->p, BN_prime_checks, ctx, NULL))
+	is_prime = BN_is_prime_ex(dh->p, BN_prime_checks, ctx, NULL);
+	if (is_prime < 0)
+		goto err;
+	if (is_prime == 0)
 		*ret |= DH_CHECK_P_NOT_PRIME;
 	else {
 		if (!BN_rshift1(q, dh->p))
 			goto err;
-		if (!BN_is_prime_ex(q, BN_prime_checks, ctx, NULL))
+		is_prime = BN_is_prime_ex(q, BN_prime_checks, ctx, NULL);
+		if (is_prime < 0)
+			goto err;
+		if (is_prime == 0)
 			*ret |= DH_CHECK_P_NOT_SAFE_PRIME;
 	}
 	ok = 1;
-err:
+
+ err:
 	BN_CTX_free(ctx);
 	BN_free(q);
 	return ok;
diff --git a/lib/libcrypto/dsa/dsa_ameth.c b/lib/libcrypto/dsa/dsa_ameth.c
index 26d81eed7b1..85ef234bb9f 100644
--- a/lib/libcrypto/dsa/dsa_ameth.c
+++ b/lib/libcrypto/dsa/dsa_ameth.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: dsa_ameth.c,v 1.26 2018/08/24 20:22:15 tb Exp $ */
+/* $OpenBSD: dsa_ameth.c,v 1.27 2019/01/20 01:56:59 tb Exp $ */
 /* Written by Dr Stephen N Henson (steve@openssl.org) for the OpenSSL
  * project 2006.
  */
@@ -515,7 +515,7 @@ old_dsa_priv_decode(EVP_PKEY *pkey, const unsigned char **pder, int derlen)
 	 * Check that q is not a composite number.
 	 */
 
-	if (BN_is_prime_ex(dsa->q, BN_prime_checks, ctx, NULL) == 0) {
+	if (BN_is_prime_ex(dsa->q, BN_prime_checks, ctx, NULL) <= 0) {
 		DSAerror(DSA_R_BAD_Q_VALUE);
 		goto err;
 	}
@@ -525,7 +525,7 @@ old_dsa_priv_decode(EVP_PKEY *pkey, const unsigned char **pder, int derlen)
 	EVP_PKEY_assign_DSA(pkey, dsa);
 	return 1;
 
-err:
+ err:
 	BN_CTX_free(ctx);
 	DSA_free(dsa);
 	return 0;