Do not queue crypto operations for IPsec. The packet entries in

task queues were unlimited and could overflow during havy traffic.
Even if we still use hardware drivers that sleep, softnet task
instead of soft interrupt can handle this now.  Without queues net
lock is inherited and kernel lock is only needed once per packet.
This results in less lock contention and faster IPsec.
Also protect tdb drop counters with net lock and avoid a leak in
crypto dispatch error handling.
intense testing Hrvoje Popovski; OK mpi@

2 files changed, 26 insertions, 9 deletions

diff --git a/sys/crypto/crypto.c b/sys/crypto/crypto.c
index 4f3f1f85509..bff6dd4a97c 100644
--- a/sys/crypto/crypto.c
+++ b/sys/crypto/crypto.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: crypto.c,v 1.84 2021/07/21 11:11:41 bluhm Exp $	*/
+/*	$OpenBSD: crypto.c,v 1.85 2021/07/26 21:27:56 bluhm Exp $	*/
 /*
  * The author of this code is Angelos D. Keromytis (angelos@cis.upenn.edu)
  *
@@ -387,23 +387,32 @@ crypto_unregister(u_int32_t driverid, int alg)
 int
 crypto_dispatch(struct cryptop *crp)
 {
-	struct taskq *tq = crypto_taskq;
-	int error = 0, s;
+	int error = 0, lock = 1, s;
 	u_int32_t hid;
 
 	s = splvm();
 	hid = (crp->crp_sid >> 32) & 0xffffffff;
 	if (hid < crypto_drivers_num) {
 		if (crypto_drivers[hid].cc_flags & CRYPTOCAP_F_MPSAFE)
-			tq = crypto_taskq_mpsafe;
+			lock = 0;
 	}
 	splx(s);
 
-	if ((crp->crp_flags & CRYPTO_F_NOQUEUE) == 0) {
+	/* XXXSMP crypto_invoke() is not MP safe */
+	lock = 1;
+
+	if (crp->crp_flags & CRYPTO_F_NOQUEUE) {
+		if (lock)
+			KERNEL_LOCK();
+		error = crypto_invoke(crp);
+		if (lock)
+			KERNEL_UNLOCK();
+	} else {
+		struct taskq *tq;
+
+		tq = lock ? crypto_taskq : crypto_taskq_mpsafe;
 		task_set(&crp->crp_task, (void (*))crypto_invoke, crp);
 		task_add(tq, &crp->crp_task);
-	} else {
-		error = crypto_invoke(crp);
 	}
 
 	return error;
@@ -424,6 +433,8 @@ crypto_invoke(struct cryptop *crp)
 	if (crp == NULL || crp->crp_callback == NULL)
 		return EINVAL;
 
+	KERNEL_ASSERT_LOCKED();
+
 	s = splvm();
 	if (crp->crp_ndesc < 1 || crypto_drivers == NULL) {
 		crp->crp_etype = EINVAL;
@@ -535,11 +546,16 @@ void
 crypto_done(struct cryptop *crp)
 {
 	crp->crp_flags |= CRYPTO_F_DONE;
+
 	if (crp->crp_flags & CRYPTO_F_NOQUEUE) {
 		/* not from the crypto queue, wakeup the userland process */
 		crp->crp_callback(crp);
 	} else {
+		struct taskq *tq;
+
+		tq = (crp->crp_flags & CRYPTO_F_MPSAFE) ?
+		    crypto_taskq_mpsafe : crypto_taskq;
 		task_set(&crp->crp_task, (void (*))crp->crp_callback, crp);
-		task_add(crypto_taskq, &crp->crp_task);
+		task_add(tq, &crp->crp_task);
 	}
 }
diff --git a/sys/crypto/cryptodev.h b/sys/crypto/cryptodev.h
index 42ef0c58e8c..10bb1a6f625 100644
--- a/sys/crypto/cryptodev.h
+++ b/sys/crypto/cryptodev.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: cryptodev.h,v 1.73 2021/07/09 20:43:28 mvs Exp $	*/
+/*	$OpenBSD: cryptodev.h,v 1.74 2021/07/26 21:27:56 bluhm Exp $	*/
 
 /*
  * The author of this code is Angelos D. Keromytis (angelos@cis.upenn.edu)
@@ -171,6 +171,7 @@ struct cryptop {
 
 #define CRYPTO_F_IMBUF	0x0001	/* Input/output are mbuf chains, otherwise contig */
 #define CRYPTO_F_IOV	0x0002	/* Input/output are uio */
+#define CRYPTO_F_MPSAFE	0x0004	/* Do not use kernel lock for callback */
 #define CRYPTO_F_NOQUEUE	0x0008	/* Don't use crypto queue/thread */
 #define CRYPTO_F_DONE	0x0010	/* request completed */