Convert relayd for opaque RSA_METHOD

This is a mostly mechanical diff which will hopefully be superseded
soon by work in libtls.

ok jsing

1 files changed, 56 insertions, 57 deletions

diff --git a/usr.sbin/relayd/ca.c b/usr.sbin/relayd/ca.c
index 88bdc1eaa21..ec24eff4364 100644
--- a/usr.sbin/relayd/ca.c
+++ b/usr.sbin/relayd/ca.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: ca.c,v 1.37 2021/12/08 03:40:44 tb Exp $	*/
+/*	$OpenBSD: ca.c,v 1.38 2022/01/11 19:06:23 tb Exp $	*/
 
 /*
  * Copyright (c) 2014 Reyk Floeter <reyk@openbsd.org>
@@ -220,10 +220,10 @@ ca_dispatch_parent(int fd, struct privsep_proc *p, struct imsg *imsg)
 		config_getreset(env, imsg);
 		break;
 	default:
-		return (-1);
+		return -1;
 	}
 
-	return (0);
+	return 0;
 }
 
 int
@@ -290,10 +290,10 @@ ca_dispatch_relay(int fd, struct privsep_proc *p, struct imsg *imsg)
 		RSA_free(rsa);
 		break;
 	default:
-		return (-1);
+		return -1;
 	}
 
-	return (0);
+	return 0;
 }
 
 /*
@@ -301,23 +301,7 @@ ca_dispatch_relay(int fd, struct privsep_proc *p, struct imsg *imsg)
  */
 
 const RSA_METHOD *rsa_default = NULL;
-
-static RSA_METHOD rsae_method = {
-	"RSA privsep engine",
-	rsae_pub_enc,
-	rsae_pub_dec,
-	rsae_priv_enc,
-	rsae_priv_dec,
-	rsae_mod_exp,
-	rsae_bn_mod_exp,
-	rsae_init,
-	rsae_finish,
-	0,
-	NULL,
-	rsae_sign,
-	rsae_verify,
-	rsae_keygen
-};
+static RSA_METHOD *rsae_method;
 
 static int
 rsae_send_imsg(int flen, const u_char *from, u_char *to, RSA *rsa,
@@ -337,7 +321,7 @@ rsae_send_imsg(int flen, const u_char *from, u_char *to, RSA *rsa,
 	static u_int	 seq = 0;
 
 	if ((hash = RSA_get_ex_data(rsa, 0)) == NULL)
-		return (0);
+		return 0;
 
 	iev = proc_iev(ps, PROC_CA, ps->ps_instance);
 	ibuf = &iev->ibuf;
@@ -378,7 +362,7 @@ rsae_send_imsg(int flen, const u_char *from, u_char *to, RSA *rsa,
 			    __func__,
 			    cmd == IMSG_CA_PRIVENC ? "enc" : "dec",
 			    cko.cko_cookie);
-			return (-1);
+			return -1;
 		default:
 			break;
 		}
@@ -426,44 +410,42 @@ rsae_send_imsg(int flen, const u_char *from, u_char *to, RSA *rsa,
 	}
 	imsg_event_add(iev);
 
-	return (ret);
+	return ret;
 }
 
 int
 rsae_pub_enc(int flen,const u_char *from, u_char *to, RSA *rsa,int padding)
 {
 	DPRINTF("%s:%d", __func__, __LINE__);
-	return (rsa_default->rsa_pub_enc(flen, from, to, rsa, padding));
+	return RSA_meth_get_pub_enc(rsa_default)(flen, from, to, rsa, padding);
 }
 
 int
 rsae_pub_dec(int flen,const u_char *from, u_char *to, RSA *rsa,int padding)
 {
 	DPRINTF("%s:%d", __func__, __LINE__);
-	return (rsa_default->rsa_pub_dec(flen, from, to, rsa, padding));
+	return RSA_meth_get_pub_dec(rsa_default)(flen, from, to, rsa, padding);
 }
 
 int
 rsae_priv_enc(int flen, const u_char *from, u_char *to, RSA *rsa, int padding)
 {
 	DPRINTF("%s:%d", __func__, __LINE__);
-	return (rsae_send_imsg(flen, from, to, rsa, padding,
-	    IMSG_CA_PRIVENC));
+	return rsae_send_imsg(flen, from, to, rsa, padding, IMSG_CA_PRIVENC);
 }
 
 int
 rsae_priv_dec(int flen, const u_char *from, u_char *to, RSA *rsa, int padding)
 {
 	DPRINTF("%s:%d", __func__, __LINE__);
-	return (rsae_send_imsg(flen, from, to, rsa, padding,
-	    IMSG_CA_PRIVDEC));
+	return rsae_send_imsg(flen, from, to, rsa, padding, IMSG_CA_PRIVDEC);
 }
 
 int
 rsae_mod_exp(BIGNUM *r0, const BIGNUM *I, RSA *rsa, BN_CTX *ctx)
 {
 	DPRINTF("%s:%d", __func__, __LINE__);
-	return (rsa_default->rsa_mod_exp(r0, I, rsa, ctx));
+	return RSA_meth_get_mod_exp(rsa_default)(r0, I, rsa, ctx);
 }
 
 int
@@ -471,25 +453,25 @@ rsae_bn_mod_exp(BIGNUM *r, const BIGNUM *a, const BIGNUM *p,
     const BIGNUM *m, BN_CTX *ctx, BN_MONT_CTX *m_ctx)
 {
 	DPRINTF("%s:%d", __func__, __LINE__);
-	return (rsa_default->bn_mod_exp(r, a, p, m, ctx, m_ctx));
+	return RSA_meth_get_bn_mod_exp(rsa_default)(r, a, p, m, ctx, m_ctx);
 }
 
 int
 rsae_init(RSA *rsa)
 {
 	DPRINTF("%s:%d", __func__, __LINE__);
-	if (rsa_default->init == NULL)
-		return (1);
-	return (rsa_default->init(rsa));
+	if (RSA_meth_get_init(rsa_default) == NULL)
+		return 1;
+	return RSA_meth_get_init(rsa_default)(rsa);
 }
 
 int
 rsae_finish(RSA *rsa)
 {
 	DPRINTF("%s:%d", __func__, __LINE__);
-	if (rsa_default->finish == NULL)
-		return (1);
-	return (rsa_default->finish(rsa));
+	if (RSA_meth_get_finish(rsa_default) == NULL)
+		return 1;
+	return RSA_meth_get_finish(rsa_default)(rsa);
 }
 
 int
@@ -497,8 +479,8 @@ rsae_sign(int type, const u_char *m, u_int m_length, u_char *sigret,
     u_int *siglen, const RSA *rsa)
 {
 	DPRINTF("%s:%d", __func__, __LINE__);
-	return (rsa_default->rsa_sign(type, m, m_length,
-	    sigret, siglen, rsa));
+	return RSA_meth_get_sign(rsa_default)(type, m, m_length,
+	    sigret, siglen, rsa);
 }
 
 int
@@ -506,15 +488,15 @@ rsae_verify(int dtype, const u_char *m, u_int m_length, const u_char *sigbuf,
     u_int siglen, const RSA *rsa)
 {
 	DPRINTF("%s:%d", __func__, __LINE__);
-	return (rsa_default->rsa_verify(dtype, m, m_length,
-	    sigbuf, siglen, rsa));
+	return RSA_meth_get_verify(rsa_default)(dtype, m, m_length,
+	    sigbuf, siglen, rsa);
 }
 
 int
 rsae_keygen(RSA *rsa, int bits, BIGNUM *e, BN_GENCB *cb)
 {
 	DPRINTF("%s:%d", __func__, __LINE__);
-	return (rsa_default->rsa_keygen(rsa, bits, e, cb));
+	return RSA_meth_get_keygen(rsa_default)(rsa, bits, e, cb);
 }
 
 void
@@ -529,12 +511,27 @@ ca_engine_init(struct relayd *x_env)
 	if (rsa_default != NULL)
 		return;
 
+	if ((rsae_method = RSA_meth_new("RSA privsep engine", 0)) == NULL)
+		goto fail;
+
+	RSA_meth_set_pub_enc(rsae_method, rsae_pub_enc);
+	RSA_meth_set_pub_dec(rsae_method, rsae_pub_dec);
+	RSA_meth_set_priv_enc(rsae_method, rsae_priv_enc);
+	RSA_meth_set_priv_dec(rsae_method, rsae_priv_dec);
+	RSA_meth_set_mod_exp(rsae_method, rsae_mod_exp);
+	RSA_meth_set_bn_mod_exp(rsae_method, rsae_bn_mod_exp);
+	RSA_meth_set_init(rsae_method, rsae_init);
+	RSA_meth_set_finish(rsae_method, rsae_finish);
+	RSA_meth_set_sign(rsae_method, rsae_sign);
+	RSA_meth_set_verify(rsae_method, rsae_verify);
+	RSA_meth_set_keygen(rsae_method, rsae_keygen);
+
 	if ((e = ENGINE_get_default_RSA()) == NULL) {
 		if ((e = ENGINE_new()) == NULL) {
 			errstr = "ENGINE_new";
 			goto fail;
 		}
-		if (!ENGINE_set_name(e, rsae_method.name)) {
+		if (!ENGINE_set_name(e, RSA_meth_get0_name(rsae_method))) {
 			errstr = "ENGINE_set_name";
 			goto fail;
 		}
@@ -552,20 +549,21 @@ ca_engine_init(struct relayd *x_env)
 
 	log_debug("%s: using %s", __func__, name);
 
-	if (rsa_default->flags & RSA_FLAG_SIGN_VER)
+	if (RSA_meth_get_flags(rsa_default) & RSA_FLAG_SIGN_VER)
 		fatalx("unsupported RSA engine");
 
-	if (rsa_default->rsa_mod_exp == NULL)
-		rsae_method.rsa_mod_exp = NULL;
-	if (rsa_default->bn_mod_exp == NULL)
-		rsae_method.bn_mod_exp = NULL;
-	if (rsa_default->rsa_keygen == NULL)
-		rsae_method.rsa_keygen = NULL;
-	rsae_method.flags = rsa_default->flags |
-	    RSA_METHOD_FLAG_NO_CHECK;
-	rsae_method.app_data = rsa_default->app_data;
-
-	if (!ENGINE_set_RSA(e, &rsae_method)) {
+	if (RSA_meth_get_mod_exp(rsa_default) == NULL)
+		RSA_meth_set_mod_exp(rsae_method, NULL);
+	if (RSA_meth_get_bn_mod_exp(rsa_default) == NULL)
+		RSA_meth_set_bn_mod_exp(rsae_method, NULL);
+	if (RSA_meth_get_keygen(rsa_default) == NULL)
+		RSA_meth_set_keygen(rsae_method, NULL);
+	RSA_meth_set_flags(rsae_method,
+	    RSA_meth_get_flags(rsa_default) | RSA_METHOD_FLAG_NO_CHECK);
+	RSA_meth_set0_app_data(rsae_method,
+	    RSA_meth_get0_app_data(rsa_default));
+
+	if (!ENGINE_set_RSA(e, rsae_method)) {
 		errstr = "ENGINE_set_RSA";
 		goto fail;
 	}
@@ -577,5 +575,6 @@ ca_engine_init(struct relayd *x_env)
 	return;
 
  fail:
+	RSA_meth_free(rsae_method);
 	fatalx("%s: %s", __func__, errstr);
 }