Retire asynchronous crypto API as it is no longer required by any driver and

adds unnecessary complexity.  Dedicated crypto offloading devices are not common
anymore.  Modern CPU crypto acceleration works synchronously, eliminating the need
for callbacks.

Replace all occurrences of crypto_dispatch() with crypto_invoke(), which is
blocking and only returns after the operation has completed or an error occured.
Invoke callback functions directly from the consumer (e.g. IPsec, softraid)
instead of relying on the crypto driver to call crypto_done().

ok bluhm@ mvs@ patrick@

6 files changed, 182 insertions, 179 deletions

diff --git a/sys/netinet/ip_ah.c b/sys/netinet/ip_ah.c
index a10c2a7b7ba..e64026d6392 100644
--- a/sys/netinet/ip_ah.c
+++ b/sys/netinet/ip_ah.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: ip_ah.c,v 1.158 2021/10/22 15:44:20 bluhm Exp $ */
+/*	$OpenBSD: ip_ah.c,v 1.159 2021/10/23 15:42:35 tobhe Exp $ */
 /*
  * The authors of this code are John Ioannidis (ji@tla.org),
  * Angelos D. Keromytis (kermit@csd.uch.gr) and
@@ -531,7 +531,7 @@ ah_input(struct mbuf *m, struct tdb *tdb, int skip, int protoff)
 	struct tdb_crypto *tc = NULL;
 	u_int32_t btsx, esn;
 	u_int8_t hl;
-	int error, rplen;
+	int error, rplen, clen;
 	u_int64_t ibytes;
 #ifdef ENCDEBUG
 	char buf[INET6_ADDRSTRLEN];
@@ -686,9 +686,7 @@ ah_input(struct mbuf *m, struct tdb *tdb, int skip, int protoff)
 	crp->crp_ilen = m->m_pkthdr.len; /* Total input length. */
 	crp->crp_flags = CRYPTO_F_IMBUF | CRYPTO_F_MPSAFE;
 	crp->crp_buf = (caddr_t)m;
-	crp->crp_callback = ipsec_input_cb;
 	crp->crp_sid = tdb->tdb_cryptoid;
-	crp->crp_opaque = (caddr_t)tc;
 
 	/* These are passed as-is to the callback. */
 	tc->tc_skip = skip;
@@ -699,7 +697,34 @@ ah_input(struct mbuf *m, struct tdb *tdb, int skip, int protoff)
 	memcpy(&tc->tc_dst, &tdb->tdb_dst, sizeof(union sockaddr_union));
 	tc->tc_rpl = tdb->tdb_rpl;
 
-	crypto_dispatch(crp);
+	KERNEL_LOCK();
+	crypto_invoke(crp);
+	while (crp->crp_etype == EAGAIN) {
+		/* Reset the session ID */
+		if (tdb->tdb_cryptoid != 0)
+			tdb->tdb_cryptoid = crp->crp_sid;
+		crypto_invoke(crp);
+	}
+	KERNEL_UNLOCK();
+	if (crp->crp_etype) {
+		DPRINTF("crypto error %d", crp->crp_etype);
+		ipsecstat_inc(ipsec_noxform);
+		error = crp->crp_etype;
+		goto drop;
+	}
+
+	/* Length of data after processing */
+	clen = crp->crp_olen;
+
+	/* Release the crypto descriptors */
+	crypto_freereq(crp);
+
+	error = ah_input_cb(tdb, tc, m, clen);
+	if (error) {
+		ipsecstat_inc(ipsec_idrops);
+		tdb->tdb_idrops++;
+	}
+
 	return 0;
 
  drop:
@@ -893,7 +918,7 @@ ah_output(struct mbuf *m, struct tdb *tdb, int skip, int protoff)
 	struct cryptop *crp = NULL;
 	u_int64_t replay64;
 	u_int16_t iplen;
-	int error, rplen, roff;
+	int error, rplen, roff, ilen, olen;
 	u_int8_t prot;
 	struct ah *ah;
 #if NBPFILTER > 0
@@ -1133,9 +1158,7 @@ ah_output(struct mbuf *m, struct tdb *tdb, int skip, int protoff)
 	crp->crp_ilen = m->m_pkthdr.len; /* Total input length. */
 	crp->crp_flags = CRYPTO_F_IMBUF | CRYPTO_F_MPSAFE;
 	crp->crp_buf = (caddr_t)m;
-	crp->crp_callback = ipsec_output_cb;
 	crp->crp_sid = tdb->tdb_cryptoid;
-	crp->crp_opaque = (caddr_t)tc;
 
 	/* These are passed as-is to the callback. */
 	tc->tc_skip = skip;
@@ -1145,7 +1168,34 @@ ah_output(struct mbuf *m, struct tdb *tdb, int skip, int protoff)
 	tc->tc_rdomain = tdb->tdb_rdomain;
 	memcpy(&tc->tc_dst, &tdb->tdb_dst, sizeof(union sockaddr_union));
 
-	crypto_dispatch(crp);
+	KERNEL_LOCK();
+	crypto_invoke(crp);
+	while (crp->crp_etype == EAGAIN) {
+		/* Reset the session ID */
+		if (tdb->tdb_cryptoid != 0)
+			tdb->tdb_cryptoid = crp->crp_sid;
+		crypto_invoke(crp);
+	}
+	KERNEL_UNLOCK();
+	if (crp->crp_etype) {
+		DPRINTF("crypto error %d", crp->crp_etype);
+		ipsecstat_inc(ipsec_noxform);
+		error = crp->crp_etype;
+		goto drop;
+	}
+
+	ilen = crp->crp_ilen;
+	olen = crp->crp_olen;
+
+	/* Release the crypto descriptors */
+	crypto_freereq(crp);
+
+	error = ah_output_cb(tdb, tc, m, ilen, olen);
+	if (error) {
+		ipsecstat_inc(ipsec_odrops);
+		tdb->tdb_odrops++;
+	}
+
 	return 0;
 
  drop:
diff --git a/sys/netinet/ip_esp.c b/sys/netinet/ip_esp.c
index 1b96a2b942d..391cfe0f4f5 100644
--- a/sys/netinet/ip_esp.c
+++ b/sys/netinet/ip_esp.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: ip_esp.c,v 1.177 2021/10/22 15:44:20 bluhm Exp $ */
+/*	$OpenBSD: ip_esp.c,v 1.178 2021/10/23 15:42:35 tobhe Exp $ */
 /*
  * The authors of this code are John Ioannidis (ji@tla.org),
  * Angelos D. Keromytis (kermit@csd.uch.gr) and
@@ -347,7 +347,7 @@ esp_input(struct mbuf *m, struct tdb *tdb, int skip, int protoff)
 	struct cryptodesc *crde = NULL, *crda = NULL;
 	struct cryptop *crp = NULL;
 	struct tdb_crypto *tc = NULL;
-	int plen, alen, hlen, error;
+	int plen, alen, hlen, error, clen;
 	u_int32_t btsx, esn;
 #ifdef ENCDEBUG
 	char buf[INET6_ADDRSTRLEN];
@@ -498,9 +498,7 @@ esp_input(struct mbuf *m, struct tdb *tdb, int skip, int protoff)
 	crp->crp_ilen = m->m_pkthdr.len; /* Total input length */
 	crp->crp_flags = CRYPTO_F_IMBUF | CRYPTO_F_MPSAFE;
 	crp->crp_buf = (caddr_t)m;
-	crp->crp_callback = ipsec_input_cb;
 	crp->crp_sid = tdb->tdb_cryptoid;
-	crp->crp_opaque = (caddr_t)tc;
 
 	/* These are passed as-is to the callback */
 	tc->tc_skip = skip;
@@ -526,7 +524,33 @@ esp_input(struct mbuf *m, struct tdb *tdb, int skip, int protoff)
 			crde->crd_len = m->m_pkthdr.len - (skip + hlen + alen);
 	}
 
-	crypto_dispatch(crp);
+	KERNEL_LOCK();
+	crypto_invoke(crp);
+	while (crp->crp_etype == EAGAIN) {
+		/* Reset the session ID */
+		if (tdb->tdb_cryptoid != 0)
+			tdb->tdb_cryptoid = crp->crp_sid;
+		crypto_invoke(crp);
+	}
+	KERNEL_UNLOCK();
+	if (crp->crp_etype) {
+		DPRINTF("crypto error %d", crp->crp_etype);
+		ipsecstat_inc(ipsec_noxform);
+		error = crp->crp_etype;
+		goto drop;
+	}
+
+	clen = crp->crp_olen;
+
+	/* Release the crypto descriptors */
+	crypto_freereq(crp);
+
+	error = esp_input_cb(tdb, tc, m, clen);
+	if (error) {
+		ipsecstat_inc(ipsec_idrops);
+		tdb->tdb_idrops++;
+	}
+
 	return 0;
 
  drop:
@@ -742,7 +766,7 @@ esp_output(struct mbuf *m, struct tdb *tdb, int skip, int protoff)
 {
 	const struct enc_xform *espx = tdb->tdb_encalgxform;
 	const struct auth_hash *esph = tdb->tdb_authalgxform;
-	int ilen, hlen, rlen, padding, blks, alen, roff, error;
+	int ilen, olen, hlen, rlen, padding, blks, alen, roff, error;
 	u_int64_t replay64;
 	u_int32_t replay;
 	struct mbuf *mi, *mo = (struct mbuf *) NULL;
@@ -980,8 +1004,6 @@ esp_output(struct mbuf *m, struct tdb *tdb, int skip, int protoff)
 	crp->crp_ilen = m->m_pkthdr.len; /* Total input length. */
 	crp->crp_flags = CRYPTO_F_IMBUF | CRYPTO_F_MPSAFE;
 	crp->crp_buf = (caddr_t)m;
-	crp->crp_callback = ipsec_output_cb;
-	crp->crp_opaque = (caddr_t)tc;
 	crp->crp_sid = tdb->tdb_cryptoid;
 
 	if (esph) {
@@ -1010,7 +1032,34 @@ esp_output(struct mbuf *m, struct tdb *tdb, int skip, int protoff)
 			crda->crd_len = m->m_pkthdr.len - (skip + alen);
 	}
 
-	crypto_dispatch(crp);
+	KERNEL_LOCK();
+	crypto_invoke(crp);
+	while (crp->crp_etype == EAGAIN) {
+		/* Reset the session ID */
+		if (tdb->tdb_cryptoid != 0)
+			tdb->tdb_cryptoid = crp->crp_sid;
+		crypto_invoke(crp);
+	}
+	KERNEL_UNLOCK();
+	if (crp->crp_etype) {
+		DPRINTF("crypto error %d", crp->crp_etype);
+		ipsecstat_inc(ipsec_noxform);
+		error = crp->crp_etype;
+		goto drop;
+	}
+
+	ilen = crp->crp_ilen;
+	olen = crp->crp_olen;
+
+	/* Release the crypto descriptors */
+	crypto_freereq(crp);
+
+	error = esp_output_cb(tdb, tc, m, ilen, olen);
+	if (error) {
+		ipsecstat_inc(ipsec_odrops);
+		tdb->tdb_odrops++;
+	}
+
 	return 0;
 
  drop:
diff --git a/sys/netinet/ip_ipcomp.c b/sys/netinet/ip_ipcomp.c
index 902955c9d6e..1854df89a5d 100644
--- a/sys/netinet/ip_ipcomp.c
+++ b/sys/netinet/ip_ipcomp.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: ip_ipcomp.c,v 1.79 2021/10/22 15:44:20 bluhm Exp $ */
+/* $OpenBSD: ip_ipcomp.c,v 1.80 2021/10/23 15:42:35 tobhe Exp $ */
 
 /*
  * Copyright (c) 2001 Jean-Jacques Bernard-Gundol (jj@wabbitt.org)
@@ -135,7 +135,7 @@ ipcomp_input(struct mbuf *m, struct tdb *tdb, int skip, int protoff)
 {
 	const struct comp_algo *ipcompx = tdb->tdb_compalgxform;
 	struct tdb_crypto *tc;
-	int hlen;
+	int hlen, error, clen;
 
 	struct cryptodesc *crdc = NULL;
 	struct cryptop *crp;
@@ -172,9 +172,7 @@ ipcomp_input(struct mbuf *m, struct tdb *tdb, int skip, int protoff)
 	crp->crp_ilen = m->m_pkthdr.len - (skip + hlen);
 	crp->crp_flags = CRYPTO_F_IMBUF | CRYPTO_F_MPSAFE;
 	crp->crp_buf = (caddr_t)m;
-	crp->crp_callback = ipsec_input_cb;
 	crp->crp_sid = tdb->tdb_cryptoid;
-	crp->crp_opaque = (caddr_t)tc;
 
 	/* These are passed as-is to the callback */
 	tc->tc_skip = skip;
@@ -184,7 +182,35 @@ ipcomp_input(struct mbuf *m, struct tdb *tdb, int skip, int protoff)
 	tc->tc_rdomain = tdb->tdb_rdomain;
 	tc->tc_dst = tdb->tdb_dst;
 
-	crypto_dispatch(crp);
+	KERNEL_LOCK();
+	crypto_invoke(crp);
+	while (crp->crp_etype == EAGAIN) {
+		/* Reset the session ID */
+		if (tdb->tdb_cryptoid != 0)
+			tdb->tdb_cryptoid = crp->crp_sid;
+		crypto_invoke(crp);
+	}
+	KERNEL_UNLOCK();
+	if (crp->crp_etype) {
+		DPRINTF("crypto error %d", crp->crp_etype);
+		ipsecstat_inc(ipsec_noxform);
+		free(tc, M_XDATA, 0);
+		m_freem(m);
+		crypto_freereq(crp);
+		return crp->crp_etype;
+	}
+
+	clen = crp->crp_olen;
+
+	/* Release the crypto descriptors */
+	crypto_freereq(crp);
+
+	error = ipcomp_input_cb(tdb, tc, m, clen);
+	if (error) {
+		ipsecstat_inc(ipsec_idrops);
+		tdb->tdb_idrops++;
+	}
+
 	return 0;
 }
 
@@ -319,7 +345,7 @@ int
 ipcomp_output(struct mbuf *m, struct tdb *tdb, int skip, int protoff)
 {
 	const struct comp_algo *ipcompx = tdb->tdb_compalgxform;
-	int error, hlen;
+	int error, hlen, ilen, olen;
 	struct cryptodesc *crdc = NULL;
 	struct cryptop *crp = NULL;
 	struct tdb_crypto *tc;
@@ -474,11 +500,38 @@ ipcomp_output(struct mbuf *m, struct tdb *tdb, int skip, int protoff)
 	crp->crp_ilen = m->m_pkthdr.len;	/* Total input length */
 	crp->crp_flags = CRYPTO_F_IMBUF | CRYPTO_F_MPSAFE;
 	crp->crp_buf = (caddr_t)m;
-	crp->crp_callback = ipsec_output_cb;
-	crp->crp_opaque = (caddr_t)tc;
 	crp->crp_sid = tdb->tdb_cryptoid;
 
-	crypto_dispatch(crp);
+	KERNEL_LOCK();
+	crypto_invoke(crp);
+	while (crp->crp_etype == EAGAIN) {
+		/* Reset the session ID */
+		if (tdb->tdb_cryptoid != 0)
+			tdb->tdb_cryptoid = crp->crp_sid;
+		crypto_invoke(crp);
+	}
+	KERNEL_UNLOCK();
+	if (crp->crp_etype) {
+		DPRINTF("crypto error %d", crp->crp_etype);
+		ipsecstat_inc(ipsec_noxform);
+		free(tc, M_XDATA, 0);
+		m_freem(m);
+		crypto_freereq(crp);
+		return crp->crp_etype;
+	}
+
+	ilen = crp->crp_ilen;
+	olen = crp->crp_olen;
+
+	/* Release the crypto descriptors */
+	crypto_freereq(crp);
+
+	error = ipcomp_output_cb(tdb, tc, m, crp->crp_ilen, crp->crp_olen);
+	if (error) {
+		ipsecstat_inc(ipsec_odrops);
+		tdb->tdb_odrops++;
+	}
+
 	return 0;
 
  drop:
diff --git a/sys/netinet/ip_ipsp.h b/sys/netinet/ip_ipsp.h
index 9bb39465b57..55452ca25c5 100644
--- a/sys/netinet/ip_ipsp.h
+++ b/sys/netinet/ip_ipsp.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: ip_ipsp.h,v 1.210 2021/10/13 14:36:31 bluhm Exp $	*/
+/*	$OpenBSD: ip_ipsp.h,v 1.211 2021/10/23 15:42:35 tobhe Exp $	*/
 /*
  * The authors of this code are John Ioannidis (ji@tla.org),
  * Angelos D. Keromytis (kermit@csd.uch.gr),
@@ -648,8 +648,6 @@ void	ipsp_init(void);
 void	ipsec_init(void);
 int	ipsec_sysctl(int *, u_int, void *, size_t *, void *, size_t);
 int	ipsec_common_input(struct mbuf *, int, int, int, int, int);
-void	ipsec_input_cb(struct cryptop *);
-void	ipsec_output_cb(struct cryptop *);
 int	ipsec_common_input_cb(struct mbuf *, struct tdb *, int, int);
 int	ipsec_delete_policy(struct ipsec_policy *);
 ssize_t	ipsec_hdrsz(struct tdb *);
diff --git a/sys/netinet/ipsec_input.c b/sys/netinet/ipsec_input.c
index 2a9a378bde7..12b845087d5 100644
--- a/sys/netinet/ipsec_input.c
+++ b/sys/netinet/ipsec_input.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: ipsec_input.c,v 1.185 2021/10/22 15:44:20 bluhm Exp $	*/
+/*	$OpenBSD: ipsec_input.c,v 1.186 2021/10/23 15:42:35 tobhe Exp $	*/
 /*
  * The authors of this code are John Ioannidis (ji@tla.org),
  * Angelos D. Keromytis (kermit@csd.uch.gr) and
@@ -366,78 +366,6 @@ ipsec_common_input(struct mbuf *m, int skip, int protoff, int af, int sproto,
 	return error;
 }
 
-void
-ipsec_input_cb(struct cryptop *crp)
-{
-	struct tdb_crypto *tc = (struct tdb_crypto *) crp->crp_opaque;
-	struct mbuf *m = (struct mbuf *) crp->crp_buf;
-	struct tdb *tdb = NULL;
-	int clen, error;
-
-	NET_ASSERT_LOCKED();
-
-	if (m == NULL) {
-		DPRINTF("bogus returned buffer from crypto");
-		ipsecstat_inc(ipsec_crypto);
-		goto drop;
-	}
-
-	tdb = gettdb(tc->tc_rdomain, tc->tc_spi, &tc->tc_dst, tc->tc_proto);
-	if (tdb == NULL) {
-		DPRINTF("TDB is expired while in crypto");
-		ipsecstat_inc(ipsec_notdb);
-		goto drop;
-	}
-
-	/* Check for crypto errors */
-	if (crp->crp_etype) {
-		if (crp->crp_etype == EAGAIN) {
-			/* Reset the session ID */
-			if (tdb->tdb_cryptoid != 0)
-				tdb->tdb_cryptoid = crp->crp_sid;
-			crypto_dispatch(crp);
-			return;
-		}
-		DPRINTF("crypto error %d", crp->crp_etype);
-		ipsecstat_inc(ipsec_noxform);
-		goto drop;
-	}
-
-	/* Length of data after processing */
-	clen = crp->crp_olen;
-
-	/* Release the crypto descriptors */
-	crypto_freereq(crp);
-
-	switch (tdb->tdb_sproto) {
-	case IPPROTO_ESP:
-		error = esp_input_cb(tdb, tc, m, clen);
-		break;
-	case IPPROTO_AH:
-		error = ah_input_cb(tdb, tc, m, clen);
-		break;
-	case IPPROTO_IPCOMP:
-		error = ipcomp_input_cb(tdb, tc, m, clen);
-		break;
-	default:
-		panic("%s: unknown/unsupported security protocol %d",
-		    __func__, tdb->tdb_sproto);
-	}
-	if (error) {
-		ipsecstat_inc(ipsec_idrops);
-		tdb->tdb_idrops++;
-	}
-	return;
-
- drop:
-	m_freem(m);
-	free(tc, M_XDATA, 0);
-	crypto_freereq(crp);
-	ipsecstat_inc(ipsec_idrops);
-	if (tdb != NULL)
-		tdb->tdb_idrops++;
-}
-
 /*
  * IPsec input callback, called by the transform callback. Takes care of
  * filtering and other sanity checks on the processed packet.
diff --git a/sys/netinet/ipsec_output.c b/sys/netinet/ipsec_output.c
index ed11171b543..43b94b843db 100644
--- a/sys/netinet/ipsec_output.c
+++ b/sys/netinet/ipsec_output.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: ipsec_output.c,v 1.90 2021/10/22 15:44:20 bluhm Exp $ */
+/*	$OpenBSD: ipsec_output.c,v 1.91 2021/10/23 15:42:35 tobhe Exp $ */
 /*
  * The author of this code is Angelos D. Keromytis (angelos@cis.upenn.edu)
  *
@@ -378,81 +378,6 @@ ipsp_process_packet(struct mbuf *m, struct tdb *tdb, int af, int tunalready)
 }
 
 /*
- * IPsec output callback, called directly by the crypto driver.
- */
-void
-ipsec_output_cb(struct cryptop *crp)
-{
-	struct tdb_crypto *tc = (struct tdb_crypto *) crp->crp_opaque;
-	struct mbuf *m = (struct mbuf *) crp->crp_buf;
-	struct tdb *tdb = NULL;
-	int error, ilen, olen;
-
-	NET_ASSERT_LOCKED();
-
-	if (m == NULL) {
-		DPRINTF("bogus returned buffer from crypto");
-		ipsecstat_inc(ipsec_crypto);
-		goto drop;
-	}
-
-	tdb = gettdb(tc->tc_rdomain, tc->tc_spi, &tc->tc_dst, tc->tc_proto);
-	if (tdb == NULL) {
-		DPRINTF("TDB is expired while in crypto");
-		ipsecstat_inc(ipsec_notdb);
-		goto drop;
-	}
-
-	/* Check for crypto errors. */
-	if (crp->crp_etype) {
-		if (crp->crp_etype == EAGAIN) {
-			/* Reset the session ID */
-			if (tdb->tdb_cryptoid != 0)
-				tdb->tdb_cryptoid = crp->crp_sid;
-			crypto_dispatch(crp);
-			return;
-		}
-		DPRINTF("crypto error %d", crp->crp_etype);
-		ipsecstat_inc(ipsec_noxform);
-		goto drop;
-	}
-
-	olen = crp->crp_olen;
-	ilen = crp->crp_ilen;
-
-	/* Release crypto descriptors. */
-	crypto_freereq(crp);
-
-	switch (tdb->tdb_sproto) {
-	case IPPROTO_ESP:
-		error = esp_output_cb(tdb, tc, m, ilen, olen);
-		break;
-	case IPPROTO_AH:
-		error = ah_output_cb(tdb, tc, m, ilen, olen);
-		break;
-	case IPPROTO_IPCOMP:
-		error = ipcomp_output_cb(tdb, tc, m, ilen, olen);
-		break;
-	default:
-		panic("%s: unhandled security protocol %d",
-		    __func__, tdb->tdb_sproto);
-	}
-	if (error) {
-		ipsecstat_inc(ipsec_odrops);
-		tdb->tdb_odrops++;
-	}
-	return;
-
- drop:
-	m_freem(m);
-	free(tc, M_XDATA, 0);
-	crypto_freereq(crp);
-	ipsecstat_inc(ipsec_odrops);
-	if (tdb != NULL)
-		tdb->tdb_odrops++;
-}
-
-/*
  * Called by the IPsec output transform callbacks, to transmit the packet
  * or do further processing, as necessary.
  */