IPsec is not MP safe yet. To allow forwarding in parallel without

dirty hacks, it is better to protect IPsec input and output with
kernel lock.  Not much is lost as crypto needs the kernel lock
anyway.  From here we can refine the lock later.
Note that there is no kernel lock in the SPD lockup path.  Goal is
to keep that lock free to allow fast forwarding with non IPsec
traffic.
tested by Hrvoje Popovski; OK tobhe@

8 files changed, 20 insertions, 22 deletions

diff --git a/sys/net/if_bridge.c b/sys/net/if_bridge.c
index 9bc7c3e7522..68cb283945b 100644
--- a/sys/net/if_bridge.c
+++ b/sys/net/if_bridge.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: if_bridge.c,v 1.361 2021/12/03 17:18:34 bluhm Exp $	*/
+/*	$OpenBSD: if_bridge.c,v 1.362 2021/12/23 12:21:48 bluhm Exp $	*/
 
 /*
  * Copyright (c) 1999, 2000 Jason L. Wright (jason@thought.net)
@@ -1625,12 +1625,15 @@ bridge_ipsec(struct ifnet *ifp, struct ether_header *eh, int hassnap,
 			if ((af == AF_INET) &&
 			    ip_mtudisc && (ip->ip_off & htons(IP_DF)) &&
 			    tdb->tdb_mtu && ntohs(ip->ip_len) > tdb->tdb_mtu &&
-			    tdb->tdb_mtutimeout > gettime())
+			    tdb->tdb_mtutimeout > gettime()) {
 				bridge_send_icmp_err(ifp, eh, m,
 				    hassnap, llc, tdb->tdb_mtu,
 				    ICMP_UNREACH, ICMP_UNREACH_NEEDFRAG);
-			else
+			} else {
+				KERNEL_LOCK();
 				error = ipsp_process_packet(m, tdb, af, 0);
+				KERNEL_UNLOCK();
+			}
 			tdb_unref(tdb);
 			return (1);
 		} else
diff --git a/sys/netinet/ip_ah.c b/sys/netinet/ip_ah.c
index 54ca7658fc3..4cf51d26535 100644
--- a/sys/netinet/ip_ah.c
+++ b/sys/netinet/ip_ah.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: ip_ah.c,v 1.171 2021/12/20 17:09:18 tobhe Exp $ */
+/*	$OpenBSD: ip_ah.c,v 1.172 2021/12/23 12:21:48 bluhm Exp $ */
 /*
  * The authors of this code are John Ioannidis (ji@tla.org),
  * Angelos D. Keromytis (kermit@csd.uch.gr) and
@@ -687,13 +687,11 @@ ah_input(struct mbuf **mp, struct tdb *tdb, int skip, int protoff)
 	crp->crp_buf = (caddr_t)m;
 	crp->crp_sid = tdb->tdb_cryptoid;
 
-	KERNEL_LOCK();
 	while ((error = crypto_invoke(crp)) == EAGAIN) {
 		/* Reset the session ID */
 		if (tdb->tdb_cryptoid != 0)
 			tdb->tdb_cryptoid = crp->crp_sid;
 	}
-	KERNEL_UNLOCK();
 	if (error) {
 		DPRINTF("crypto error %d", error);
 		ipsecstat_inc(ipsec_noxform);
@@ -1112,13 +1110,11 @@ ah_output(struct mbuf *m, struct tdb *tdb, int skip, int protoff)
 	crp->crp_buf = (caddr_t)m;
 	crp->crp_sid = tdb->tdb_cryptoid;
 
-	KERNEL_LOCK();
 	while ((error = crypto_invoke(crp)) == EAGAIN) {
 		/* Reset the session ID */
 		if (tdb->tdb_cryptoid != 0)
 			tdb->tdb_cryptoid = crp->crp_sid;
 	}
-	KERNEL_UNLOCK();
 	if (error) {
 		DPRINTF("crypto error %d", error);
 		ipsecstat_inc(ipsec_noxform);
diff --git a/sys/netinet/ip_esp.c b/sys/netinet/ip_esp.c
index ee43d9e6398..a3e8e5f8689 100644
--- a/sys/netinet/ip_esp.c
+++ b/sys/netinet/ip_esp.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: ip_esp.c,v 1.191 2021/12/20 17:09:18 tobhe Exp $ */
+/*	$OpenBSD: ip_esp.c,v 1.192 2021/12/23 12:21:48 bluhm Exp $ */
 /*
  * The authors of this code are John Ioannidis (ji@tla.org),
  * Angelos D. Keromytis (kermit@csd.uch.gr) and
@@ -502,13 +502,11 @@ esp_input(struct mbuf **mp, struct tdb *tdb, int skip, int protoff)
 			crde->crd_len = plen;
 	}
 
-	KERNEL_LOCK();
 	while ((error = crypto_invoke(crp)) == EAGAIN) {
 		/* Reset the session ID */
 		if (tdb->tdb_cryptoid != 0)
 			tdb->tdb_cryptoid = crp->crp_sid;
 	}
-	KERNEL_UNLOCK();
 	if (error) {
 		DPRINTF("crypto error %d", error);
 		ipsecstat_inc(ipsec_noxform);
@@ -948,13 +946,11 @@ esp_output(struct mbuf *m, struct tdb *tdb, int skip, int protoff)
 			crda->crd_len = m->m_pkthdr.len - (skip + alen);
 	}
 
-	KERNEL_LOCK();
 	while ((error = crypto_invoke(crp)) == EAGAIN) {
 		/* Reset the session ID */
 		if (tdb->tdb_cryptoid != 0)
 			tdb->tdb_cryptoid = crp->crp_sid;
 	}
-	KERNEL_UNLOCK();
 	if (error) {
 		DPRINTF("crypto error %d", error);
 		ipsecstat_inc(ipsec_noxform);
diff --git a/sys/netinet/ip_ipcomp.c b/sys/netinet/ip_ipcomp.c
index 0026fe2da12..2221c7a44b7 100644
--- a/sys/netinet/ip_ipcomp.c
+++ b/sys/netinet/ip_ipcomp.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: ip_ipcomp.c,v 1.90 2021/12/20 15:59:09 mvs Exp $ */
+/* $OpenBSD: ip_ipcomp.c,v 1.91 2021/12/23 12:21:48 bluhm Exp $ */
 
 /*
  * Copyright (c) 2001 Jean-Jacques Bernard-Gundol (jj@wabbitt.org)
@@ -171,13 +171,11 @@ ipcomp_input(struct mbuf **mp, struct tdb *tdb, int skip, int protoff)
 	crp->crp_buf = (caddr_t)m;
 	crp->crp_sid = tdb->tdb_cryptoid;
 
-	KERNEL_LOCK();
 	while ((error = crypto_invoke(crp)) == EAGAIN) {
 		/* Reset the session ID */
 		if (tdb->tdb_cryptoid != 0)
 			tdb->tdb_cryptoid = crp->crp_sid;
 	}
-	KERNEL_UNLOCK();
 	if (error) {
 		DPRINTF("crypto error %d", error);
 		ipsecstat_inc(ipsec_noxform);
@@ -459,13 +457,11 @@ ipcomp_output(struct mbuf *m, struct tdb *tdb, int skip, int protoff)
 	crp->crp_buf = (caddr_t)m;
 	crp->crp_sid = tdb->tdb_cryptoid;
 
-	KERNEL_LOCK();
 	while ((error = crypto_invoke(crp)) == EAGAIN) {
 		/* Reset the session ID */
 		if (tdb->tdb_cryptoid != 0)
 			tdb->tdb_cryptoid = crp->crp_sid;
 	}
-	KERNEL_UNLOCK();
 	if (error) {
 		DPRINTF("crypto error %d", error);
 		ipsecstat_inc(ipsec_noxform);
diff --git a/sys/netinet/ip_output.c b/sys/netinet/ip_output.c
index de87ad3aac1..7d8353f707d 100644
--- a/sys/netinet/ip_output.c
+++ b/sys/netinet/ip_output.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: ip_output.c,v 1.378 2021/12/20 15:59:10 mvs Exp $	*/
+/*	$OpenBSD: ip_output.c,v 1.379 2021/12/23 12:21:48 bluhm Exp $	*/
 /*	$NetBSD: ip_output.c,v 1.28 1996/02/13 23:43:07 christos Exp $	*/
 
 /*
@@ -659,7 +659,9 @@ ip_output_ipsec_send(struct tdb *tdb, struct mbuf *m, struct route *ro, int fwd)
 	m->m_flags &= ~(M_MCAST | M_BCAST);
 
 	/* Callee frees mbuf */
+	KERNEL_LOCK();
 	error = ipsp_process_packet(m, tdb, AF_INET, 0);
+	KERNEL_UNLOCK();
 	if (error) {
 		ipsecstat_inc(ipsec_odrops);
 		tdbstat_inc(tdb, tdb_odrops);
diff --git a/sys/netinet/ipsec_input.c b/sys/netinet/ipsec_input.c
index 6bd3fa6df1a..ee7d0475122 100644
--- a/sys/netinet/ipsec_input.c
+++ b/sys/netinet/ipsec_input.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: ipsec_input.c,v 1.200 2021/12/22 13:37:46 tobhe Exp $	*/
+/*	$OpenBSD: ipsec_input.c,v 1.201 2021/12/23 12:21:48 bluhm Exp $	*/
 /*
  * The authors of this code are John Ioannidis (ji@tla.org),
  * Angelos D. Keromytis (kermit@csd.uch.gr) and
@@ -309,6 +309,7 @@ ipsec_common_input(struct mbuf **mp, int skip, int protoff, int af, int sproto,
 		goto drop;
 	}
 
+	KERNEL_LOCK();
 	/* Register first use, setup expiration timer. */
 	if (tdbp->tdb_first_use == 0) {
 		tdbp->tdb_first_use = gettime();
@@ -336,6 +337,7 @@ ipsec_common_input(struct mbuf **mp, int skip, int protoff, int af, int sproto,
 		tdbstat_inc(tdbp, tdb_idrops);
 	}
 	tdb_unref(tdbp);
+	KERNEL_UNLOCK();
 	return prot;
 
  drop:
diff --git a/sys/netinet/ipsec_output.c b/sys/netinet/ipsec_output.c
index 33b6244ec3b..58298d3ad91 100644
--- a/sys/netinet/ipsec_output.c
+++ b/sys/netinet/ipsec_output.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: ipsec_output.c,v 1.95 2021/12/20 15:59:10 mvs Exp $ */
+/*	$OpenBSD: ipsec_output.c,v 1.96 2021/12/23 12:21:48 bluhm Exp $ */
 /*
  * The author of this code is Angelos D. Keromytis (angelos@cis.upenn.edu)
  *
@@ -512,6 +512,7 @@ ipsp_process_done(struct mbuf *m, struct tdb *tdb)
 	/* If there's another (bundled) TDB to apply, do so. */
 	tdbo = tdb_ref(tdb->tdb_onext);
 	if (tdbo != NULL) {
+		KERNEL_ASSERT_LOCKED();
 		error = ipsp_process_packet(m, tdbo,
 		    tdb->tdb_dst.sa.sa_family, 0);
 		tdb_unref(tdbo);
diff --git a/sys/netinet6/ip6_output.c b/sys/netinet6/ip6_output.c
index c7f2fec20bb..9141f2e6652 100644
--- a/sys/netinet6/ip6_output.c
+++ b/sys/netinet6/ip6_output.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: ip6_output.c,v 1.264 2021/12/20 15:59:10 mvs Exp $	*/
+/*	$OpenBSD: ip6_output.c,v 1.265 2021/12/23 12:21:48 bluhm Exp $	*/
 /*	$KAME: ip6_output.c,v 1.172 2001/03/25 09:55:56 itojun Exp $	*/
 
 /*
@@ -2894,7 +2894,9 @@ ip6_output_ipsec_send(struct tdb *tdb, struct mbuf *m, struct route_in6 *ro,
 	m->m_flags &= ~(M_BCAST | M_MCAST);
 
 	/* Callee frees mbuf */
+	KERNEL_LOCK();
 	error = ipsp_process_packet(m, tdb, AF_INET6, tunalready);
+	KERNEL_UNLOCK();
 	if (error) {
 		ipsecstat_inc(ipsec_odrops);
 		tdbstat_inc(tdb, tdb_odrops);