Last changes before running IPsec w/o KERNEL_LOCK().

Put more NET_ASSERT_LOCK() and document which globals it protects.

Add a mutex for pfkeyv2 globals.

Convert ipsp_delete_acquire() to timeout_set_proc().

Tested by Hrvoje Popovski, ok bluhm@ visa@

3 files changed, 55 insertions, 15 deletions

diff --git a/sys/netinet/ip_ipsp.c b/sys/netinet/ip_ipsp.c
index 0dc50de6219..6fa11ee087d 100644
--- a/sys/netinet/ip_ipsp.c
+++ b/sys/netinet/ip_ipsp.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: ip_ipsp.c,v 1.227 2017/10/11 13:44:49 mpi Exp $	*/
+/*	$OpenBSD: ip_ipsp.c,v 1.228 2017/10/16 08:22:25 mpi Exp $	*/
 /*
  * The authors of this code are John Ioannidis (ji@tla.org),
  * Angelos D. Keromytis (kermit@csd.uch.gr),
@@ -87,16 +87,14 @@ int		tdb_hash(u_int, u_int32_t, union sockaddr_union *, u_int8_t);
 
 int ipsec_in_use = 0;
 u_int64_t ipsec_last_added = 0;
+int ipsec_ids_idle = 100;		/* keep free ids for 100s */
 
-struct ipsec_policy_head ipsec_policy_head =
-    TAILQ_HEAD_INITIALIZER(ipsec_policy_head);
-struct ipsec_acquire_head ipsec_acquire_head =
-    TAILQ_HEAD_INITIALIZER(ipsec_acquire_head);
-
+/* Protected by the NET_LOCK(). */
 u_int32_t ipsec_ids_next_flow = 1;	/* may not be zero */
-int ipsec_ids_idle = 100;		/* keep free ids for 100s */
 struct ipsec_ids_tree ipsec_ids_tree;
 struct ipsec_ids_flows ipsec_ids_flows;
+struct ipsec_policy_head ipsec_policy_head =
+    TAILQ_HEAD_INITIALIZER(ipsec_policy_head);
 
 void ipsp_ids_timeout(void *);
 static inline int ipsp_ids_cmp(const struct ipsec_ids *,
@@ -173,6 +171,7 @@ struct xformsw *xformswNXFORMSW = &xformsw[nitems(xformsw)];
 
 #define	TDB_HASHSIZE_INIT	32
 
+/* Protected by the NET_LOCK(). */
 static SIPHASH_KEY tdbkey;
 static struct tdb **tdbh = NULL;
 static struct tdb **tdbdst = NULL;
@@ -190,6 +189,8 @@ tdb_hash(u_int rdomain, u_int32_t spi, union sockaddr_union *dst,
 {
 	SIPHASH_CTX ctx;
 
+	NET_ASSERT_LOCKED();
+
 	SipHash24_Init(&ctx, &tdbkey);
 	SipHash24_Update(&ctx, &rdomain, sizeof(rdomain));
 	SipHash24_Update(&ctx, &spi, sizeof(spi));
@@ -336,6 +337,8 @@ gettdbbysrcdst(u_int rdomain, u_int32_t spi, union sockaddr_union *src,
 	struct tdb *tdbp;
 	union sockaddr_union su_null;
 
+	NET_ASSERT_LOCKED();
+
 	if (tdbsrc == NULL)
 		return (struct tdb *) NULL;
 
@@ -420,6 +423,8 @@ gettdbbydst(u_int rdomain, union sockaddr_union *dst, u_int8_t sproto,
 	u_int32_t hashval;
 	struct tdb *tdbp;
 
+	NET_ASSERT_LOCKED();
+
 	if (tdbdst == NULL)
 		return (struct tdb *) NULL;
 
@@ -451,6 +456,8 @@ gettdbbysrc(u_int rdomain, union sockaddr_union *src, u_int8_t sproto,
 	u_int32_t hashval;
 	struct tdb *tdbp;
 
+	NET_ASSERT_LOCKED();
+
 	if (tdbsrc == NULL)
 		return (struct tdb *) NULL;
 
@@ -788,6 +795,8 @@ tdb_alloc(u_int rdomain)
 {
 	struct tdb *tdbp;
 
+	NET_ASSERT_LOCKED();
+
 	tdbp = malloc(sizeof(*tdbp), M_TDB, M_WAITOK | M_ZERO);
 
 	TAILQ_INIT(&tdbp->tdb_policy_head);
@@ -812,6 +821,8 @@ tdb_free(struct tdb *tdbp)
 {
 	struct ipsec_policy *ipo;
 
+	NET_ASSERT_LOCKED();
+
 	if (tdbp->tdb_xform) {
 		(*(tdbp->tdb_xform->xf_zeroize))(tdbp);
 		tdbp->tdb_xform = NULL;
@@ -944,6 +955,8 @@ ipsp_ids_insert(struct ipsec_ids *ids)
 	struct ipsec_ids *found;
 	u_int32_t start_flow;
 
+	NET_ASSERT_LOCKED();
+
 	found = RBT_INSERT(ipsec_ids_tree, &ipsec_ids_tree, ids);
 	if (found) {
 		/* if refcount was zero, then timeout is running */
@@ -977,6 +990,8 @@ ipsp_ids_lookup(u_int32_t ipsecflowinfo)
 {
 	struct ipsec_ids	key;
 
+	NET_ASSERT_LOCKED();
+
 	key.id_flow = ipsecflowinfo;
 	return RBT_FIND(ipsec_ids_flows, &ipsec_ids_flows, &key);
 }
diff --git a/sys/netinet/ip_ipsp.h b/sys/netinet/ip_ipsp.h
index df145de2b64..cd6d58e9d8f 100644
--- a/sys/netinet/ip_ipsp.h
+++ b/sys/netinet/ip_ipsp.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: ip_ipsp.h,v 1.183 2017/06/26 09:08:00 patrick Exp $	*/
+/*	$OpenBSD: ip_ipsp.h,v 1.184 2017/10/16 08:22:25 mpi Exp $	*/
 /*
  * The authors of this code are John Ioannidis (ji@tla.org),
  * Angelos D. Keromytis (kermit@csd.uch.gr),
@@ -440,7 +440,6 @@ extern struct auth_hash auth_hash_hmac_ripemd_160_96;
 extern struct comp_algo comp_algo_deflate;
 
 extern TAILQ_HEAD(ipsec_policy_head, ipsec_policy) ipsec_policy_head;
-extern TAILQ_HEAD(ipsec_acquire_head, ipsec_acquire) ipsec_acquire_head;
 
 /* Misc. */
 #ifdef ENCDEBUG
diff --git a/sys/netinet/ip_spd.c b/sys/netinet/ip_spd.c
index 5d8b3a7b19e..8a26e990b6c 100644
--- a/sys/netinet/ip_spd.c
+++ b/sys/netinet/ip_spd.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: ip_spd.c,v 1.92 2017/04/06 14:25:18 dhill Exp $ */
+/* $OpenBSD: ip_spd.c,v 1.93 2017/10/16 08:22:25 mpi Exp $ */
 /*
  * The author of this code is Angelos D. Keromytis (angelos@cis.upenn.edu)
  *
@@ -45,7 +45,8 @@ int	ipsp_acquire_sa(struct ipsec_policy *, union sockaddr_union *,
 	    union sockaddr_union *, struct sockaddr_encap *, struct mbuf *);
 struct	ipsec_acquire *ipsp_pending_acquire(struct ipsec_policy *,
 	    union sockaddr_union *);
-void	ipsp_delete_acquire(void *);
+void	ipsp_delete_acquire_timo(void *);
+void	ipsp_delete_acquire(struct ipsec_acquire *);
 
 #ifdef ENCDEBUG
 #define	DPRINTF(x)	if (encdebug) printf x
@@ -56,16 +57,21 @@ void	ipsp_delete_acquire(void *);
 struct pool ipsec_policy_pool;
 struct pool ipsec_acquire_pool;
 int ipsec_policy_pool_initialized = 0;
-int ipsec_acquire_pool_initialized = 0;
 
+/* Protected by the NET_LOCK(). */
+int ipsec_acquire_pool_initialized = 0;
 struct radix_node_head **spd_tables;
 unsigned int spd_table_max;
+TAILQ_HEAD(ipsec_acquire_head, ipsec_acquire) ipsec_acquire_head =
+    TAILQ_HEAD_INITIALIZER(ipsec_acquire_head);
 
 struct radix_node_head *
 spd_table_get(unsigned int rtableid)
 {
 	unsigned int rdomain;
 
+	NET_ASSERT_LOCKED();
+
 	if (spd_tables == NULL)
 		return (NULL);
 
@@ -83,6 +89,8 @@ spd_table_add(unsigned int rtableid)
 	unsigned int rdomain;
 	void *p;
 
+	NET_ASSERT_LOCKED();
+
 	rdomain = rtable_l2(rtableid);
 	if (spd_tables == NULL || rdomain > spd_table_max) {
 		if ((p = mallocarray(rdomain + 1, sizeof(*rnh),
@@ -135,6 +143,8 @@ ipsp_spd_lookup(struct mbuf *m, int af, int hlen, int *error, int direction,
 	int signore = 0, dignore = 0;
 	u_int rdomain = rtable_l2(m->m_pkthdr.ph_rtableid);
 
+	NET_ASSERT_LOCKED();
+
 	/*
 	 * If there are no flows in place, there's no point
 	 * continuing with the SPD lookup.
@@ -587,6 +597,8 @@ ipsec_delete_policy(struct ipsec_policy *ipo)
 	struct radix_node *rn = (struct radix_node *)ipo;
 	int err = 0;
 
+	NET_ASSERT_LOCKED();
+
 	if (--ipo->ipo_ref_count > 0)
 		return 0;
 
@@ -614,13 +626,23 @@ ipsec_delete_policy(struct ipsec_policy *ipo)
 	return err;
 }
 
+void
+ipsp_delete_acquire_timo(void *v)
+{
+	struct ipsec_acquire *ipa = v;
+
+	NET_LOCK();
+	ipsp_delete_acquire(ipa);
+	NET_UNLOCK();
+}
+
 /*
  * Delete a pending IPsec acquire record.
  */
 void
-ipsp_delete_acquire(void *v)
+ipsp_delete_acquire(struct ipsec_acquire *ipa)
 {
-	struct ipsec_acquire *ipa = v;
+	NET_ASSERT_LOCKED();
 
 	timeout_del(&ipa->ipa_timeout);
 	TAILQ_REMOVE(&ipsec_acquire_head, ipa, ipa_next);
@@ -639,6 +661,8 @@ ipsp_pending_acquire(struct ipsec_policy *ipo, union sockaddr_union *gw)
 {
 	struct ipsec_acquire *ipa;
 
+	NET_ASSERT_LOCKED();
+
 	TAILQ_FOREACH (ipa, &ipo->ipo_acquires, ipa_ipo_next) {
 		if (!memcmp(gw, &ipa->ipa_addr, gw->sa.sa_len))
 			return ipa;
@@ -657,6 +681,8 @@ ipsp_acquire_sa(struct ipsec_policy *ipo, union sockaddr_union *gw,
 {
 	struct ipsec_acquire *ipa;
 
+	NET_ASSERT_LOCKED();
+
 	/* Check whether request has been made already. */
 	if ((ipa = ipsp_pending_acquire(ipo, gw)) != NULL)
 		return 0;
@@ -674,7 +700,7 @@ ipsp_acquire_sa(struct ipsec_policy *ipo, union sockaddr_union *gw,
 
 	ipa->ipa_addr = *gw;
 
-	timeout_set(&ipa->ipa_timeout, ipsp_delete_acquire, ipa);
+	timeout_set_proc(&ipa->ipa_timeout, ipsp_delete_acquire_timo, ipa);
 
 	ipa->ipa_info.sen_len = ipa->ipa_mask.sen_len = SENT_LEN;
 	ipa->ipa_info.sen_family = ipa->ipa_mask.sen_family = PF_KEY;