Disallow aliasing of return value and modulus

All the functions changed in this commit would silently misbehave if the
return value aliases the modulus, most of the time they would succeed and
return an incorrect result of 0 in that situation. This adjusts all the
functions in BN_mod.c, others and documentation will follow later.

Prompted by a bug report about BN_mod_inverse() by Guido Vranken.

ok jsing

1 files changed, 44 insertions, 1 deletions

diff --git a/lib/libcrypto/bn/bn_mod.c b/lib/libcrypto/bn/bn_mod.c
index 868ef5bc5bf..79766d00362 100644
--- a/lib/libcrypto/bn/bn_mod.c
+++ b/lib/libcrypto/bn/bn_mod.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: bn_mod.c,v 1.20 2023/03/27 10:21:23 tb Exp $ */
+/* $OpenBSD: bn_mod.c,v 1.21 2023/06/13 09:28:13 tb Exp $ */
 /* Includes code written by Lenka Fibikova <fibikova@exp-math.uni-essen.de>
  * for the OpenSSL project. */
 /* ====================================================================
@@ -136,6 +136,10 @@ BN_mod_nonct(BIGNUM *r, const BIGNUM *a, const BIGNUM *m, BN_CTX *ctx)
 int
 BN_nnmod(BIGNUM *r, const BIGNUM *a, const BIGNUM *m, BN_CTX *ctx)
 {
+	if (r == m) {
+		BNerror(BN_R_INVALID_ARGUMENT);
+		return 0;
+	}
 	if (!BN_mod_ct(r, a, m, ctx))
 		return 0;
 	if (BN_is_negative(r))
@@ -147,6 +151,10 @@ int
 BN_mod_add(BIGNUM *r, const BIGNUM *a, const BIGNUM *b, const BIGNUM *m,
     BN_CTX *ctx)
 {
+	if (r == m) {
+		BNerror(BN_R_INVALID_ARGUMENT);
+		return 0;
+	}
 	if (!BN_add(r, a, b))
 		return 0;
 	return BN_nnmod(r, r, m, ctx);
@@ -159,6 +167,10 @@ BN_mod_add(BIGNUM *r, const BIGNUM *a, const BIGNUM *b, const BIGNUM *m,
 int
 BN_mod_add_quick(BIGNUM *r, const BIGNUM *a, const BIGNUM *b, const BIGNUM *m)
 {
+	if (r == m) {
+		BNerror(BN_R_INVALID_ARGUMENT);
+		return 0;
+	}
 	if (!BN_uadd(r, a, b))
 		return 0;
 	if (BN_ucmp(r, m) >= 0)
@@ -170,6 +182,10 @@ int
 BN_mod_sub(BIGNUM *r, const BIGNUM *a, const BIGNUM *b, const BIGNUM *m,
     BN_CTX *ctx)
 {
+	if (r == m) {
+		BNerror(BN_R_INVALID_ARGUMENT);
+		return 0;
+	}
 	if (!BN_sub(r, a, b))
 		return 0;
 	return BN_nnmod(r, r, m, ctx);
@@ -182,6 +198,10 @@ BN_mod_sub(BIGNUM *r, const BIGNUM *a, const BIGNUM *b, const BIGNUM *m,
 int
 BN_mod_sub_quick(BIGNUM *r, const BIGNUM *a, const BIGNUM *b, const BIGNUM *m)
 {
+	if (r == m) {
+		BNerror(BN_R_INVALID_ARGUMENT);
+		return 0;
+	}
 	if (BN_ucmp(a, b) >= 0)
 		return BN_usub(r, a, b);
 	if (!BN_usub(r, b, a))
@@ -198,6 +218,11 @@ BN_mod_mul(BIGNUM *r, const BIGNUM *a, const BIGNUM *b, const BIGNUM *m,
 
 	BN_CTX_start(ctx);
 
+	if (r == m) {
+		BNerror(BN_R_INVALID_ARGUMENT);
+		goto err;
+	}
+
 	rr = r;
 	if (rr == a || rr == b)
 		rr = BN_CTX_get(ctx);
@@ -231,6 +256,10 @@ BN_mod_sqr(BIGNUM *r, const BIGNUM *a, const BIGNUM *m, BN_CTX *ctx)
 int
 BN_mod_lshift1(BIGNUM *r, const BIGNUM *a, const BIGNUM *m, BN_CTX *ctx)
 {
+	if (r == m) {
+		BNerror(BN_R_INVALID_ARGUMENT);
+		return 0;
+	}
 	if (!BN_lshift1(r, a))
 		return 0;
 	return BN_nnmod(r, r, m, ctx);
@@ -243,6 +272,10 @@ BN_mod_lshift1(BIGNUM *r, const BIGNUM *a, const BIGNUM *m, BN_CTX *ctx)
 int
 BN_mod_lshift1_quick(BIGNUM *r, const BIGNUM *a, const BIGNUM *m)
 {
+	if (r == m) {
+		BNerror(BN_R_INVALID_ARGUMENT);
+		return 0;
+	}
 	if (!BN_lshift1(r, a))
 		return 0;
 	if (BN_ucmp(r, m) >= 0)
@@ -258,6 +291,11 @@ BN_mod_lshift(BIGNUM *r, const BIGNUM *a, int n, const BIGNUM *m, BN_CTX *ctx)
 
 	BN_CTX_start(ctx);
 
+	if (r == m) {
+		BNerror(BN_R_INVALID_ARGUMENT);
+		goto err;
+	}
+
 	if (!BN_nnmod(r, a, m, ctx))
 		goto err;
 
@@ -288,6 +326,11 @@ BN_mod_lshift_quick(BIGNUM *r, const BIGNUM *a, int n, const BIGNUM *m)
 {
 	int max_shift;
 
+	if (r == m) {
+		BNerror(BN_R_INVALID_ARGUMENT);
+		return 0;
+	}
+
 	if (!bn_copy(r, a))
 		return 0;