Reject too small bits value in BN_generate_prime_ex(), so that it does not risk

becoming negative in probable_prime_dh_safe(). Reported by Franck Denis who
noticed `openssl gendh 0' would segfault.
Fix adapted from OpenSSL RT#2701.

ok beck@ jsing@

3 files changed, 22 insertions, 4 deletions

diff --git a/lib/libcrypto/bn/bn.h b/lib/libcrypto/bn/bn.h
index 33c6162874a..2c648ba2ee3 100644
--- a/lib/libcrypto/bn/bn.h
+++ b/lib/libcrypto/bn/bn.h
@@ -1,4 +1,4 @@
-/* $OpenBSD: bn.h,v 1.27 2015/10/16 12:41:29 beck Exp $ */
+/* $OpenBSD: bn.h,v 1.28 2015/10/21 19:02:22 miod Exp $ */
 /* Copyright (C) 1995-1997 Eric Young (eay@cryptsoft.com)
  * All rights reserved.
  *
@@ -768,6 +768,7 @@ void ERR_load_BN_strings(void);
 #define BN_F_BN_DIV_RECP				 130
 #define BN_F_BN_EXP					 123
 #define BN_F_BN_EXPAND2					 108
+#define BN_F_BN_GENERATE_PRIME_EX			 140
 #define BN_F_BN_EXPAND_INTERNAL				 120
 #define BN_F_BN_GF2M_MOD				 131
 #define BN_F_BN_GF2M_MOD_EXP				 132
@@ -797,6 +798,7 @@ void ERR_load_BN_strings(void);
 #define BN_R_ARG2_LT_ARG3				 100
 #define BN_R_BAD_RECIPROCAL				 101
 #define BN_R_BIGNUM_TOO_LONG				 114
+#define BN_R_BITS_TOO_SMALL				 117
 #define BN_R_CALLED_WITH_EVEN_MODULUS			 102
 #define BN_R_DIV_BY_ZERO				 103
 #define BN_R_ENCODING_ERROR				 104
diff --git a/lib/libcrypto/bn/bn_err.c b/lib/libcrypto/bn/bn_err.c
index 5a0f359d86c..149e58eafcb 100644
--- a/lib/libcrypto/bn/bn_err.c
+++ b/lib/libcrypto/bn/bn_err.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: bn_err.c,v 1.12 2014/07/10 22:45:56 jsing Exp $ */
+/* $OpenBSD: bn_err.c,v 1.13 2015/10/21 19:02:22 miod Exp $ */
 /* ====================================================================
  * Copyright (c) 1999-2007 The OpenSSL Project.  All rights reserved.
  *
@@ -89,6 +89,7 @@ static ERR_STRING_DATA BN_str_functs[]= {
 	{ERR_FUNC(BN_F_BN_EXP),	"BN_exp"},
 	{ERR_FUNC(BN_F_BN_EXPAND2),	"bn_expand2"},
 	{ERR_FUNC(BN_F_BN_EXPAND_INTERNAL),	"BN_EXPAND_INTERNAL"},
+	{ERR_FUNC(BN_F_BN_GENERATE_PRIME_EX),	"BN_generate_prime_ex"},
 	{ERR_FUNC(BN_F_BN_GF2M_MOD),	"BN_GF2m_mod"},
 	{ERR_FUNC(BN_F_BN_GF2M_MOD_EXP),	"BN_GF2m_mod_exp"},
 	{ERR_FUNC(BN_F_BN_GF2M_MOD_MUL),	"BN_GF2m_mod_mul"},
@@ -119,6 +120,7 @@ static ERR_STRING_DATA BN_str_reasons[]= {
 	{ERR_REASON(BN_R_ARG2_LT_ARG3)           , "arg2 lt arg3"},
 	{ERR_REASON(BN_R_BAD_RECIPROCAL)         , "bad reciprocal"},
 	{ERR_REASON(BN_R_BIGNUM_TOO_LONG)        , "bignum too long"},
+	{ERR_REASON(BN_R_BITS_TOO_SMALL)         , "bits too small"},
 	{ERR_REASON(BN_R_CALLED_WITH_EVEN_MODULUS), "called with even modulus"},
 	{ERR_REASON(BN_R_DIV_BY_ZERO)            , "div by zero"},
 	{ERR_REASON(BN_R_ENCODING_ERROR)         , "encoding error"},
diff --git a/lib/libcrypto/bn/bn_prime.c b/lib/libcrypto/bn/bn_prime.c
index 02780d32e6a..b1aba663dfc 100644
--- a/lib/libcrypto/bn/bn_prime.c
+++ b/lib/libcrypto/bn/bn_prime.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: bn_prime.c,v 1.13 2015/02/09 15:49:22 jsing Exp $ */
+/* $OpenBSD: bn_prime.c,v 1.14 2015/10/21 19:02:22 miod Exp $ */
 /* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com)
  * All rights reserved.
  *
@@ -112,6 +112,8 @@
 #include <stdio.h>
 #include <time.h>
 
+#include <openssl/err.h>
+
 #include "bn_lcl.h"
 
 /* NB: these functions have been "upgraded", the deprecated versions (which are
@@ -164,7 +166,16 @@ BN_generate_prime_ex(BIGNUM *ret, int bits, int safe, const BIGNUM *add,
 	int found = 0;
 	int i, j, c1 = 0;
 	BN_CTX *ctx;
-	int checks = BN_prime_checks_for_size(bits);
+	int checks;
+
+	if (bits < 2 || (bits == 2 && safe)) {
+		/*
+		 * There are no prime numbers smaller than 2, and the smallest
+		 * safe prime (7) spans three bits.
+		 */
+		BNerr(BN_F_BN_GENERATE_PRIME_EX, BN_R_BITS_TOO_SMALL);
+		return 0;
+	}
 
 	ctx = BN_CTX_new();
 	if (ctx == NULL)
@@ -172,6 +183,9 @@ BN_generate_prime_ex(BIGNUM *ret, int bits, int safe, const BIGNUM *add,
 	BN_CTX_start(ctx);
 	if ((t = BN_CTX_get(ctx)) == NULL)
 		goto err;
+
+	checks = BN_prime_checks_for_size(bits);
+
 loop:
 	/* make a random number and set the top and bottom bits */
 	if (add == NULL) {