Make RSA, RSA_PSS_PARAMS and RSA_METHOD opaque

Move the struct internals to rsa_locl.h and provide a missing
typedef in ossl_typ.h.

ok inoguchi jsing

3 files changed, 80 insertions, 83 deletions

diff --git a/lib/libcrypto/ossl_typ.h b/lib/libcrypto/ossl_typ.h
index 161bf86a3e3..4f94c4032e1 100644
--- a/lib/libcrypto/ossl_typ.h
+++ b/lib/libcrypto/ossl_typ.h
@@ -1,4 +1,4 @@
-/* $OpenBSD: ossl_typ.h,v 1.19 2022/01/14 08:21:12 tb Exp $ */
+/* $OpenBSD: ossl_typ.h,v 1.20 2022/01/14 08:34:39 tb Exp $ */
 /* ====================================================================
  * Copyright (c) 1998-2001 The OpenSSL Project.  All rights reserved.
  *
@@ -121,6 +121,7 @@ typedef struct dsa_method DSA_METHOD;
 
 typedef struct rsa_st RSA;
 typedef struct rsa_meth_st RSA_METHOD;
+typedef struct rsa_pss_params_st RSA_PSS_PARAMS;
 
 typedef struct rand_meth_st RAND_METHOD;
 
diff --git a/lib/libcrypto/rsa/rsa.h b/lib/libcrypto/rsa/rsa.h
index e01e6ba5535..d59fd03f765 100644
--- a/lib/libcrypto/rsa/rsa.h
+++ b/lib/libcrypto/rsa/rsa.h
@@ -1,4 +1,4 @@
-/* $OpenBSD: rsa.h,v 1.54 2022/01/14 08:12:31 tb Exp $ */
+/* $OpenBSD: rsa.h,v 1.55 2022/01/14 08:34:39 tb Exp $ */
 /* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com)
  * All rights reserved.
  *
@@ -80,11 +80,7 @@
 extern "C" {
 #endif
 
-/* Declared already in ossl_typ.h */
-/* typedef struct rsa_st RSA; */
-/* typedef struct rsa_meth_st RSA_METHOD; */
-
-typedef struct rsa_pss_params_st {
+struct rsa_pss_params_st {
 	X509_ALGOR *hashAlgorithm;
 	X509_ALGOR *maskGenAlgorithm;
 	ASN1_INTEGER *saltLength;
@@ -92,7 +88,7 @@ typedef struct rsa_pss_params_st {
 
 	/* Hash algorithm decoded from maskGenAlgorithm. */
 	X509_ALGOR *maskHash;
-} RSA_PSS_PARAMS;
+} /* RSA_PSS_PARAMS */;
 
 typedef struct rsa_oaep_params_st {
 	X509_ALGOR *hashFunc;
@@ -103,80 +99,6 @@ typedef struct rsa_oaep_params_st {
 	X509_ALGOR *maskHash;
 } RSA_OAEP_PARAMS;
 
-struct rsa_meth_st {
-	const char *name;
-	int (*rsa_pub_enc)(int flen, const unsigned char *from,
-	    unsigned char *to, RSA *rsa, int padding);
-	int (*rsa_pub_dec)(int flen, const unsigned char *from,
-	    unsigned char *to, RSA *rsa, int padding);
-	int (*rsa_priv_enc)(int flen, const unsigned char *from,
-	    unsigned char *to, RSA *rsa, int padding);
-	int (*rsa_priv_dec)(int flen, const unsigned char *from,
-	    unsigned char *to, RSA *rsa, int padding);
-	int (*rsa_mod_exp)(BIGNUM *r0, const BIGNUM *I, RSA *rsa,
-	    BN_CTX *ctx); /* Can be null */
-	int (*bn_mod_exp)(BIGNUM *r, const BIGNUM *a, const BIGNUM *p,
-	    const BIGNUM *m, BN_CTX *ctx, BN_MONT_CTX *m_ctx); /* Can be null */
-	int (*init)(RSA *rsa);		/* called at new */
-	int (*finish)(RSA *rsa);	/* called at free */
-	int flags;			/* RSA_METHOD_FLAG_* things */
-	char *app_data;			/* may be needed! */
-/* New sign and verify functions: some libraries don't allow arbitrary data
- * to be signed/verified: this allows them to be used. Note: for this to work
- * the RSA_public_decrypt() and RSA_private_encrypt() should *NOT* be used
- * RSA_sign(), RSA_verify() should be used instead. Note: for backwards
- * compatibility this functionality is only enabled if the RSA_FLAG_SIGN_VER
- * option is set in 'flags'.
- */
-	int (*rsa_sign)(int type, const unsigned char *m, unsigned int m_length,
-	    unsigned char *sigret, unsigned int *siglen, const RSA *rsa);
-	int (*rsa_verify)(int dtype, const unsigned char *m,
-	    unsigned int m_length, const unsigned char *sigbuf,
-	    unsigned int siglen, const RSA *rsa);
-/* If this callback is NULL, the builtin software RSA key-gen will be used. This
- * is for behavioural compatibility whilst the code gets rewired, but one day
- * it would be nice to assume there are no such things as "builtin software"
- * implementations. */
-	int (*rsa_keygen)(RSA *rsa, int bits, BIGNUM *e, BN_GENCB *cb);
-};
-
-struct rsa_st {
-	/* The first parameter is used to pickup errors where
-	 * this is passed instead of aEVP_PKEY, it is set to 0 */
-	int pad;
-	long version;
-	const RSA_METHOD *meth;
-
-	/* functional reference if 'meth' is ENGINE-provided */
-	ENGINE *engine;
-	BIGNUM *n;
-	BIGNUM *e;
-	BIGNUM *d;
-	BIGNUM *p;
-	BIGNUM *q;
-	BIGNUM *dmp1;
-	BIGNUM *dmq1;
-	BIGNUM *iqmp;
-
-	/* Parameter restrictions for PSS only keys. */
-	RSA_PSS_PARAMS *pss;
-
-	/* be careful using this if the RSA structure is shared */
-	CRYPTO_EX_DATA ex_data;
-	int references;
-	int flags;
-
-	/* Used to cache montgomery values */
-	BN_MONT_CTX *_method_mod_n;
-	BN_MONT_CTX *_method_mod_p;
-	BN_MONT_CTX *_method_mod_q;
-
-	/* all BIGNUM values are actually in the following data, if it is not
-	 * NULL */
-	BN_BLINDING *blinding;
-	BN_BLINDING *mt_blinding;
-};
-
 #ifndef OPENSSL_RSA_MAX_MODULUS_BITS
 # define OPENSSL_RSA_MAX_MODULUS_BITS	16384
 #endif
diff --git a/lib/libcrypto/rsa/rsa_locl.h b/lib/libcrypto/rsa/rsa_locl.h
index 7036449c363..9eae2b3a24b 100644
--- a/lib/libcrypto/rsa/rsa_locl.h
+++ b/lib/libcrypto/rsa/rsa_locl.h
@@ -1,4 +1,4 @@
-/* $OpenBSD: rsa_locl.h,v 1.11 2019/11/02 13:47:41 jsing Exp $ */
+/* $OpenBSD: rsa_locl.h,v 1.12 2022/01/14 08:34:39 tb Exp $ */
 
 __BEGIN_HIDDEN_DECLS
 
@@ -8,6 +8,80 @@ __BEGIN_HIDDEN_DECLS
 #define pkey_is_pss(pkey) (pkey->ameth->pkey_id == EVP_PKEY_RSA_PSS)
 #define pkey_ctx_is_pss(ctx) (ctx->pmeth->pkey_id == EVP_PKEY_RSA_PSS)
 
+struct rsa_meth_st {
+	const char *name;
+	int (*rsa_pub_enc)(int flen, const unsigned char *from,
+	    unsigned char *to, RSA *rsa, int padding);
+	int (*rsa_pub_dec)(int flen, const unsigned char *from,
+	    unsigned char *to, RSA *rsa, int padding);
+	int (*rsa_priv_enc)(int flen, const unsigned char *from,
+	    unsigned char *to, RSA *rsa, int padding);
+	int (*rsa_priv_dec)(int flen, const unsigned char *from,
+	    unsigned char *to, RSA *rsa, int padding);
+	int (*rsa_mod_exp)(BIGNUM *r0, const BIGNUM *I, RSA *rsa,
+	    BN_CTX *ctx); /* Can be null */
+	int (*bn_mod_exp)(BIGNUM *r, const BIGNUM *a, const BIGNUM *p,
+	    const BIGNUM *m, BN_CTX *ctx, BN_MONT_CTX *m_ctx); /* Can be null */
+	int (*init)(RSA *rsa);		/* called at new */
+	int (*finish)(RSA *rsa);	/* called at free */
+	int flags;			/* RSA_METHOD_FLAG_* things */
+	char *app_data;			/* may be needed! */
+/* New sign and verify functions: some libraries don't allow arbitrary data
+ * to be signed/verified: this allows them to be used. Note: for this to work
+ * the RSA_public_decrypt() and RSA_private_encrypt() should *NOT* be used
+ * RSA_sign(), RSA_verify() should be used instead. Note: for backwards
+ * compatibility this functionality is only enabled if the RSA_FLAG_SIGN_VER
+ * option is set in 'flags'.
+ */
+	int (*rsa_sign)(int type, const unsigned char *m, unsigned int m_length,
+	    unsigned char *sigret, unsigned int *siglen, const RSA *rsa);
+	int (*rsa_verify)(int dtype, const unsigned char *m,
+	    unsigned int m_length, const unsigned char *sigbuf,
+	    unsigned int siglen, const RSA *rsa);
+/* If this callback is NULL, the builtin software RSA key-gen will be used. This
+ * is for behavioural compatibility whilst the code gets rewired, but one day
+ * it would be nice to assume there are no such things as "builtin software"
+ * implementations. */
+	int (*rsa_keygen)(RSA *rsa, int bits, BIGNUM *e, BN_GENCB *cb);
+};
+
+struct rsa_st {
+	/* The first parameter is used to pickup errors where
+	 * this is passed instead of aEVP_PKEY, it is set to 0 */
+	int pad;
+	long version;
+	const RSA_METHOD *meth;
+
+	/* functional reference if 'meth' is ENGINE-provided */
+	ENGINE *engine;
+	BIGNUM *n;
+	BIGNUM *e;
+	BIGNUM *d;
+	BIGNUM *p;
+	BIGNUM *q;
+	BIGNUM *dmp1;
+	BIGNUM *dmq1;
+	BIGNUM *iqmp;
+
+	/* Parameter restrictions for PSS only keys. */
+	RSA_PSS_PARAMS *pss;
+
+	/* be careful using this if the RSA structure is shared */
+	CRYPTO_EX_DATA ex_data;
+	int references;
+	int flags;
+
+	/* Used to cache montgomery values */
+	BN_MONT_CTX *_method_mod_n;
+	BN_MONT_CTX *_method_mod_p;
+	BN_MONT_CTX *_method_mod_q;
+
+	/* all BIGNUM values are actually in the following data, if it is not
+	 * NULL */
+	BN_BLINDING *blinding;
+	BN_BLINDING *mt_blinding;
+};
+
 RSA_PSS_PARAMS *rsa_pss_params_create(const EVP_MD *sigmd, const EVP_MD *mgf1md,
     int saltlen);
 int rsa_pss_get_param(const RSA_PSS_PARAMS *pss, const EVP_MD **pmd,