ip_send()/ip6_send() allow PF to send response packet in ipsoftnet task.

this avoids current recursion to pf_test() function. the change also
switches icmp_error()/icmp6_error() to use ip_send()/ip6_send() so
they are safe for PF.

The idea comes from Markus Friedl. bluhm, mikeb and mpi helped me
a lot to get it into shape.

OK bluhm@, mpi@

8 files changed, 87 insertions, 12 deletions

diff --git a/sys/net/if_var.h b/sys/net/if_var.h
index f4f7419fa20..b0dd01ae9e2 100644
--- a/sys/net/if_var.h
+++ b/sys/net/if_var.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: if_var.h,v 1.62 2015/12/03 16:27:32 mpi Exp $	*/
+/*	$OpenBSD: if_var.h,v 1.63 2015/12/03 21:11:53 sashan Exp $	*/
 /*	$NetBSD: if.h,v 1.23 1996/05/07 02:40:27 thorpej Exp $	*/
 
 /*
@@ -361,6 +361,7 @@ int		niq_enlist(struct niqueue *, struct mbuf_list *);
 
 extern struct ifnet_head ifnet;
 extern unsigned int lo0ifidx;
+extern struct taskq *softnettq;
 
 void	if_start(struct ifnet *);
 void	if_start_barrier(struct ifnet *);
diff --git a/sys/net/pf.c b/sys/net/pf.c
index c04d19c7a2b..ee13eccce1b 100644
--- a/sys/net/pf.c
+++ b/sys/net/pf.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: pf.c,v 1.956 2015/12/03 14:05:28 bluhm Exp $ */
+/*	$OpenBSD: pf.c,v 1.957 2015/12/03 21:11:53 sashan Exp $ */
 
 /*
  * Copyright (c) 2001 Daniel Hartmeier
@@ -2424,11 +2424,11 @@ pf_send_tcp(const struct pf_rule *r, sa_family_t af,
 
 	switch (af) {
 	case AF_INET:
-		ip_output(m, NULL, NULL, 0, NULL, NULL, 0);
+		ip_send(m);
 		break;
 #ifdef INET6
 	case AF_INET6:
-		ip6_output(m, NULL, NULL, 0, NULL, NULL);
+		ip6_send(m);
 		break;
 #endif /* INET6 */
 	}
diff --git a/sys/netinet/ip_icmp.c b/sys/netinet/ip_icmp.c
index eb5b3a9a4e0..f0b2d14f08d 100644
--- a/sys/netinet/ip_icmp.c
+++ b/sys/netinet/ip_icmp.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: ip_icmp.c,v 1.149 2015/12/02 16:35:53 bluhm Exp $	*/
+/*	$OpenBSD: ip_icmp.c,v 1.150 2015/12/03 21:11:53 sashan Exp $	*/
 /*	$NetBSD: ip_icmp.c,v 1.19 1996/02/13 23:42:22 christos Exp $	*/
 
 /*
@@ -854,7 +854,10 @@ icmp_send(struct mbuf *m, struct mbuf *opts)
 		printf("icmp_send dst %s src %s\n", dst, src);
 	}
 #endif
-	ip_output(m, opts, NULL, 0, NULL, NULL, 0);
+	if (opts != NULL)
+		m = ip_insertoptions(m, opts, &hlen);
+
+	ip_send(m);
 }
 
 u_int32_t
diff --git a/sys/netinet/ip_input.c b/sys/netinet/ip_input.c
index 69d655dc293..885e6fc4ed8 100644
--- a/sys/netinet/ip_input.c
+++ b/sys/netinet/ip_input.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: ip_input.c,v 1.264 2015/12/03 15:12:59 markus Exp $	*/
+/*	$OpenBSD: ip_input.c,v 1.265 2015/12/03 21:11:53 sashan Exp $	*/
 /*	$NetBSD: ip_input.c,v 1.30 1996/03/16 23:53:58 christos Exp $	*/
 
 /*
@@ -44,6 +44,7 @@
 #include <sys/socketvar.h>
 #include <sys/sysctl.h>
 #include <sys/pool.h>
+#include <sys/task.h>
 
 #include <net/if.h>
 #include <net/if_var.h>
@@ -121,6 +122,8 @@ struct pool ipq_pool;
 
 struct ipstat ipstat;
 
+static struct mbuf_queue	ipsend_mq;
+
 void	ip_ours(struct mbuf *);
 int	ip_dooptions(struct mbuf *, struct ifnet *);
 int	in_ouraddr(struct mbuf *, struct ifnet *, struct in_addr);
@@ -130,6 +133,8 @@ int	ip_input_ipsec_fwd_check(struct mbuf *, int);
 int	ip_input_ipsec_ours_check(struct mbuf *, int);
 #endif /* IPSEC */
 
+static void ip_send_dispatch(void *);
+static struct task ipsend_task = TASK_INITIALIZER(ip_send_dispatch, &ipsend_mq);
 /*
  * Used to save the IP options in case a protocol wants to respond
  * to an incoming packet over the same route if the packet got here
@@ -188,6 +193,8 @@ ip_init(void)
 	strlcpy(ipsec_def_enc, IPSEC_DEFAULT_DEF_ENC, sizeof(ipsec_def_enc));
 	strlcpy(ipsec_def_auth, IPSEC_DEFAULT_DEF_AUTH, sizeof(ipsec_def_auth));
 	strlcpy(ipsec_def_comp, IPSEC_DEFAULT_DEF_COMP, sizeof(ipsec_def_comp));
+
+	mq_init(&ipsend_mq, 64, IPL_SOFTNET);
 }
 
 struct	route ipforward_rt;
@@ -1752,3 +1759,27 @@ ip_savecontrol(struct inpcb *inp, struct mbuf **mp, struct ip *ip,
 	}
 }
 
+void
+ip_send_dispatch(void *xmq)
+{
+	struct mbuf_queue *mq = xmq;
+	struct mbuf *m;
+	struct mbuf_list ml;
+	int s;
+
+	mq_delist(mq, &ml);
+	KERNEL_LOCK();
+	s = splsoftnet();
+	while ((m = ml_dequeue(&ml)) != NULL) {
+		ip_output(m, NULL, NULL, 0, NULL, NULL, 0);
+	}
+	splx(s);
+	KERNEL_UNLOCK();
+}
+
+void
+ip_send(struct mbuf *m)
+{
+	mq_enqueue(&ipsend_mq, m);
+	task_add(softnettq, &ipsend_task);
+}
diff --git a/sys/netinet/ip_var.h b/sys/netinet/ip_var.h
index a0f5527f255..049e4a10f13 100644
--- a/sys/netinet/ip_var.h
+++ b/sys/netinet/ip_var.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: ip_var.h,v 1.60 2015/07/16 21:14:21 mpi Exp $	*/
+/*	$OpenBSD: ip_var.h,v 1.61 2015/12/03 21:11:53 sashan Exp $	*/
 /*	$NetBSD: ip_var.h,v 1.16 1996/02/13 23:43:20 christos Exp $	*/
 
 /*
@@ -180,6 +180,8 @@ void	 ip_freef(struct ipq *);
 void	 ip_freemoptions(struct ip_moptions *);
 int	 ip_getmoptions(int, struct ip_moptions *, struct mbuf **);
 void	 ip_init(void);
+struct mbuf*
+	 ip_insertoptions(struct mbuf *, struct mbuf *, int *);
 int	 ip_mforward(struct mbuf *, struct ifnet *);
 int	 ip_optcopy(struct ip *, struct ip *);
 int	 ip_output(struct mbuf *, struct mbuf *, struct route *, int,
@@ -191,6 +193,7 @@ struct in_ifaddr *
 	 ip_rtaddr(struct in_addr, u_int);
 u_int16_t
 	 ip_randomid(void);
+void	 ip_send(struct mbuf *);
 int	 ip_setmoptions(int, struct ip_moptions **, struct mbuf *, u_int);
 void	 ip_slowtimo(void);
 struct mbuf *
@@ -207,5 +210,6 @@ void	 rip_input(struct mbuf *, ...);
 int	 rip_output(struct mbuf *, ...);
 int	 rip_usrreq(struct socket *,
 	    int, struct mbuf *, struct mbuf *, struct mbuf *, struct proc *);
+
 #endif /* _KERNEL */
 #endif /* _NETINET_IP_VAR_H_ */
diff --git a/sys/netinet6/icmp6.c b/sys/netinet6/icmp6.c
index f0100428732..192837e6900 100644
--- a/sys/netinet6/icmp6.c
+++ b/sys/netinet6/icmp6.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: icmp6.c,v 1.181 2015/12/02 16:35:53 bluhm Exp $	*/
+/*	$OpenBSD: icmp6.c,v 1.182 2015/12/03 21:11:53 sashan Exp $	*/
 /*	$KAME: icmp6.c,v 1.217 2001/06/20 15:03:29 jinmei Exp $	*/
 
 /*
@@ -1302,7 +1302,7 @@ icmp6_reflect(struct mbuf *m, size_t off)
 #if NPF > 0
 	pf_pkt_addr_changed(m);
 #endif
-	ip6_output(m, NULL, NULL, IPV6_MINMTU, NULL, NULL);
+	ip6_send(m);
 	return;
 
  bad:
diff --git a/sys/netinet6/ip6_input.c b/sys/netinet6/ip6_input.c
index 3479082b849..08e015d103e 100644
--- a/sys/netinet6/ip6_input.c
+++ b/sys/netinet6/ip6_input.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: ip6_input.c,v 1.151 2015/11/11 10:23:23 mpi Exp $	*/
+/*	$OpenBSD: ip6_input.c,v 1.152 2015/12/03 21:11:54 sashan Exp $	*/
 /*	$KAME: ip6_input.c,v 1.188 2001/03/29 05:34:31 itojun Exp $	*/
 
 /*
@@ -77,6 +77,7 @@
 #include <sys/timeout.h>
 #include <sys/kernel.h>
 #include <sys/syslog.h>
+#include <sys/task.h>
 
 #include <net/if.h>
 #include <net/if_var.h>
@@ -89,6 +90,7 @@
 #include <netinet/ip.h>
 
 #include <netinet/in_pcb.h>
+#include <netinet/ip_var.h>
 #include <netinet6/in6_var.h>
 #include <netinet/ip6.h>
 #include <netinet6/ip6_var.h>
@@ -123,6 +125,12 @@ int ip6_check_rh0hdr(struct mbuf *, int *);
 int ip6_hopopts_input(u_int32_t *, u_int32_t *, struct mbuf **, int *);
 struct mbuf *ip6_pullexthdr(struct mbuf *, size_t, int);
 
+static struct mbuf_queue	ip6send_mq;
+
+static void ip6_send_dispatch(void *);
+static struct task ip6send_task =
+	TASK_INITIALIZER(ip6_send_dispatch, &ip6send_mq);
+
 /*
  * IP6 initialization: fill in IP6 protocol switch table.
  * All protocols not implemented in kernel go to raw IP6 protocol handler.
@@ -149,6 +157,8 @@ ip6_init(void)
 	nd6_init();
 	frag6_init();
 	ip6_init2((void *)0);
+
+	mq_init(&ip6send_mq, 64, IPL_SOFTNET);
 }
 
 void
@@ -1431,3 +1441,28 @@ ip6_sysctl(int *name, u_int namelen, void *oldp, size_t *oldlenp,
 	}
 	/* NOTREACHED */
 }
+
+void
+ip6_send_dispatch(void *xmq)
+{
+	struct mbuf_queue *mq = xmq;
+	struct mbuf *m;
+	struct mbuf_list ml;
+	int s;
+
+	mq_delist(mq, &ml);
+	KERNEL_LOCK();
+	s = splsoftnet();
+	while ((m = ml_dequeue(&ml)) != NULL) {
+		ip6_output(m, NULL, NULL, IPV6_MINMTU, NULL, NULL);
+	}
+	splx(s);
+	KERNEL_UNLOCK();
+}
+
+void
+ip6_send(struct mbuf *m)
+{
+	mq_enqueue(&ip6send_mq, m);
+	task_add(softnettq, &ip6send_task);
+}
diff --git a/sys/netinet6/ip6_var.h b/sys/netinet6/ip6_var.h
index 2af3a05d810..b7c99c5887b 100644
--- a/sys/netinet6/ip6_var.h
+++ b/sys/netinet6/ip6_var.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: ip6_var.h,v 1.56 2015/10/25 14:43:06 florian Exp $	*/
+/*	$OpenBSD: ip6_var.h,v 1.57 2015/12/03 21:11:54 sashan Exp $	*/
 /*	$KAME: ip6_var.h,v 1.33 2000/06/11 14:59:20 jinmei Exp $	*/
 
 /*
@@ -275,6 +275,7 @@ int	ip6_setpktopts(struct mbuf *, struct ip6_pktopts *,
 void	ip6_clearpktopts(struct ip6_pktopts *, int);
 void	ip6_randomid_init(void);
 u_int32_t ip6_randomid(void);
+void	ip6_send(struct mbuf *);
 
 int	route6_input(struct mbuf **, int *, int);