From e2f314dc3ea961765f59724834f9aee19441a126 Mon Sep 17 00:00:00 2001 From: Joel Sing Date: Tue, 7 Mar 2023 06:28:37 +0000 Subject: Limit bn_mul_mont() usage to sizes less than or equal to 8192 bits. The assembly bn_mul_mont() implementations effectively use alloca() to allocate space for computation (at up to 8x the input size), without any limitation. This means that sufficiently large inputs lead to the stack being blown. Prevent this by using the C based implementation instead. Thanks to Jiayi Lin for reporting this to us. ok beck@ tb@ --- lib/libcrypto/bn/bn_mont.c | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/lib/libcrypto/bn/bn_mont.c b/lib/libcrypto/bn/bn_mont.c index e92ceae5f48..314d6837825 100644 --- a/lib/libcrypto/bn/bn_mont.c +++ b/lib/libcrypto/bn/bn_mont.c @@ -1,4 +1,4 @@ -/* $OpenBSD: bn_mont.c,v 1.50 2023/03/07 06:19:44 jsing Exp $ */ +/* $OpenBSD: bn_mont.c,v 1.51 2023/03/07 06:28:36 jsing Exp $ */ /* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com) * All rights reserved. * @@ -435,6 +435,14 @@ bn_mod_mul_montgomery(BIGNUM *r, const BIGNUM *a, const BIGNUM *b, if (mctx->N.top <= 1 || a->top != mctx->N.top || b->top != mctx->N.top) return bn_mod_mul_montgomery_simple(r, a, b, mctx, ctx); + /* + * Legacy bn_mul_mont() performs stack based allocation, without + * size limitation. Allowing a large size results in the stack + * being blown. + */ + if (mctx->N.top > (8 * 1024 / sizeof(BN_ULONG))) + return bn_montgomery_multiply(r, a, b, mctx, ctx); + if (!bn_wexpand(r, mctx->N.top)) return 0; -- cgit v1.2.3