IPv6 reassembly on "scrub" directive.

caveats: (to be addressed soon)
- "scrub in" should queue fragments back into ip6intrq again, but
  somehow it does not happen - the packet is kept inside reass queue.
  need investigation
- ip6_forwarding path is not tested
- does not use red-black tree.  somehow red-black tree behaved badly
  and was not robust.  performance issue, the above one is more
  important.

good things:
- "scrub out" is perfectly ok
- i think now we can inspect upper-layer protocol fields (tcp port)
  even if ip6 packet is fragmented.
- reass queue will be cleaned up properly by timeout (60sec).  we might
  want to impose pool limit as well

4 files changed, 235 insertions, 46 deletions

diff --git a/sys/net/pf.c b/sys/net/pf.c
index ab5179596d2..9fdfbf36847 100644
--- a/sys/net/pf.c
+++ b/sys/net/pf.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: pf.c,v 1.454 2004/06/22 07:35:19 cedric Exp $ */
+/*	$OpenBSD: pf.c,v 1.455 2004/06/25 00:42:58 itojun Exp $ */
 
 /*
  * Copyright (c) 2001 Daniel Hartmeier
@@ -53,6 +53,7 @@
 #include <net/if_types.h>
 #include <net/bpf.h>
 #include <net/route.h>
+#include <net/netisr.h>
 
 #include <netinet/in.h>
 #include <netinet/in_var.h>
@@ -1300,7 +1301,7 @@ pf_send_tcp(const struct pf_rule *r, sa_family_t af,
 	if (m == NULL)
 		return;
 	if (tag) {
-		struct m_tag	*mtag;
+		struct m_tag *mtag;
 
 		mtag = m_tag_get(PACKET_TAG_PF_GENERATED, 0, M_NOWAIT);
 		if (mtag == NULL) {
@@ -5680,7 +5681,8 @@ pf_test6(int dir, struct ifnet *ifp, struct mbuf **m0,
 {
 	struct pfi_kif		*kif;
 	u_short			 action, reason = 0, log = 0;
-	struct mbuf		*m = *m0;
+	struct mbuf		*m = *m0, *n;
+	struct mbuf		*frag;
 	struct ip6_hdr		*h;
 	struct pf_rule		*a = NULL, *r = &pf_default_rule, *tr, *nr;
 	struct pf_state		*s = NULL;
@@ -5709,11 +5711,26 @@ pf_test6(int dir, struct ifnet *ifp, struct mbuf **m0,
 		goto done;
 	}
 
-	/* We do IP header normalization and packet reassembly here */
-	if (pf_normalize_ip6(m0, dir, kif, &reason, &pd) != PF_PASS) {
+	/*
+	 * We do IP header normalization and packet reassembly here.
+	 * due to KAME mbuf handling rule, pf_normalize_ip6 may lose mbuf,
+	 * so keep a copy here.
+	 */
+	frag = NULL;
+	n = m_copym(*m0, 0, M_COPYALL, M_DONTWAIT);
+	m = *m0;
+	if (pf_normalize_ip6(m0, dir, kif, &reason, &pd, &frag) != PF_PASS) {
 		action = PF_DROP;
+		if (!*m0)
+			*m0 = n;
+		else
+			m_freem(n);
 		goto done;
 	}
+	if (!*m0)
+		*m0 = n;
+	else
+		m_freem(n);
 	m = *m0;
 	h = mtod(m, struct ip6_hdr *);
 
@@ -5730,12 +5747,6 @@ pf_test6(int dir, struct ifnet *ifp, struct mbuf **m0,
 	pd.proto = h->ip6_nxt;
 	do {
 		switch (pd.proto) {
-		case IPPROTO_FRAGMENT:
-			action = pf_test_fragment(&r, dir, kif, m, h,
-			    &pd, &a, &ruleset);
-			if (action == PF_DROP)
-				REASON_SET(&reason, PFRES_FRAG);
-			goto done;
 		case IPPROTO_AH:
 		case IPPROTO_HOPOPTS:
 		case IPPROTO_ROUTING:
@@ -5760,6 +5771,7 @@ pf_test6(int dir, struct ifnet *ifp, struct mbuf **m0,
 			/* goto the next header */
 			break;
 		}
+		case IPPROTO_FRAGMENT:
 		default:
 			terminal++;
 			break;
@@ -5987,6 +5999,78 @@ done:
 		/* pf_route6 can free the mbuf causing *m0 to become NULL */
 		pf_route6(m0, r, dir, ifp, s);
 
+	/*
+	 * it's the tricky part - how to return value is different by caller
+	 */
+	switch (dir) {
+	case PF_IN:
+		if (frag) {
+			int s;
+			struct mbuf *q, *r;
+			struct m_tag *mtag;
+
+			if (action != PF_PASS) {
+				for (q = frag; q; q = r) {
+					r = q->m_nextpkt;
+					q->m_nextpkt = NULL;
+					m_freem(q);
+					q = NULL;
+				}
+				return (action);
+			}
+
+			for (q = frag; q; q = r) {
+				r = q->m_nextpkt;
+				q->m_nextpkt = NULL;
+
+				mtag = m_tag_get(PACKET_TAG_PF_FRAGCACHE,
+				    0, M_NOWAIT);
+				if (mtag == NULL) {
+					s = splimp();
+					IF_DROP(&ip6intrq);
+					splx(s);
+					m_freem(q);
+					q = NULL;
+					continue;
+				}
+				m_tag_prepend(q, mtag);
+
+				s = splimp();
+				IF_ENQUEUE(&ip6intrq, q);
+				q = NULL;
+				splx(s);
+			}
+			schednetisr(NETISR_IPV6);
+
+			m_freem(*m0);
+			*m0 = NULL;
+			return (PF_PASS);
+		} else
+			break;
+
+	case PF_OUT:
+		if (frag) {
+			struct mbuf *q, *r;
+			for (q = frag; q; q = r) {
+				r = q->m_nextpkt;
+				q->m_nextpkt = NULL;
+				m_freem(q);
+				q = NULL;
+			}
+		}
+		break;
+
+	case PF_FORWARD:
+		if (action == PF_PASS) {
+			if (frag) {
+				m_freem(*m0);
+				*m0 = NULL;
+				*m0 = frag;
+			} else
+				(*m0)->m_nextpkt = NULL;
+		}
+		break;
+	}
 	return (action);
 }
 #endif /* INET6 */
diff --git a/sys/net/pf_norm.c b/sys/net/pf_norm.c
index 9b76985ac4c..0386379fd99 100644
--- a/sys/net/pf_norm.c
+++ b/sys/net/pf_norm.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: pf_norm.c,v 1.90 2004/06/24 19:35:25 tholo Exp $ */
+/*	$OpenBSD: pf_norm.c,v 1.91 2004/06/25 00:42:58 itojun Exp $ */
 
 /*
  * Copyright 2001 Niels Provos <provos@citi.umich.edu>
@@ -56,6 +56,7 @@
 
 #ifdef INET6
 #include <netinet/ip6.h>
+#include <netinet6/ip6_var.h>
 #endif /* INET6 */
 
 #include <net/pfvar.h>
@@ -104,6 +105,25 @@ RB_HEAD(pf_frag_tree, pf_fragment)	pf_frag_tree, pf_cache_tree;
 RB_PROTOTYPE(pf_frag_tree, pf_fragment, fr_entry, pf_frag_compare);
 RB_GENERATE(pf_frag_tree, pf_fragment, fr_entry, pf_frag_compare);
 
+#ifdef INET6
+struct pf_fragment6 {
+	TAILQ_ENTRY(pf_fragment6) next;
+
+	/* packets received, connected by m->m_nextpkt */
+	struct mbuf *m;
+
+	/* search key */
+	struct in6_addr src, dst;
+	u_int32_t ident;
+
+	struct timeout lifetime;
+};
+
+TAILQ_HEAD(pf_frag_tree6, pf_fragment6) pf_frag_tree6;
+
+struct pool pf_fragment6_pl;
+#endif
+
 /* Private prototypes */
 void			 pf_ip2key(struct pf_fragment *, struct ip *);
 void			 pf_remove_fragment(struct pf_fragment *);
@@ -116,6 +136,13 @@ struct mbuf		*pf_fragcache(struct mbuf **, struct ip*,
 			    struct pf_fragment **, int, int, int *);
 int			 pf_normalize_tcpopt(struct pf_rule *, struct mbuf *,
 			    struct tcphdr *, int);
+#ifdef INET6
+void			 pf_ip2key6(struct pf_fragment6 *, struct ip6_hdr *,
+			    struct ip6_frag *);
+struct pf_fragment6	*pf_find_fragment6(u_int32_t, struct in6_addr *,
+			    struct in6_addr *);
+static void		 pf_frag6_expire(void *);
+#endif
 
 #define	DPFPRINTF(x) do {				\
 	if (pf_status.debug >= PF_DEBUG_MISC) {		\
@@ -143,6 +170,11 @@ pf_normalize_init(void)
 	pool_init(&pf_state_scrub_pl, sizeof(struct pf_state_scrub), 0, 0, 0,
 	    "pfstscr", NULL);
 
+#ifdef INET6
+	pool_init(&pf_fragment6_pl, sizeof(struct pf_fragment6), 0, 0, 0,
+	    "pffrag6", NULL);
+#endif
+
 	pool_sethiwat(&pf_frag_pl, PFFRAG_FRAG_HIWAT);
 	pool_sethardlimit(&pf_frent_pl, PFFRAG_FRENT_HIWAT, NULL, 0);
 	pool_sethardlimit(&pf_cache_pl, PFFRAG_FRCACHE_HIWAT, NULL, 0);
@@ -1026,11 +1058,24 @@ pf_normalize_ip(struct mbuf **m0, int dir, struct pfi_kif *kif, u_short *reason,
 }
 
 #ifdef INET6
+struct pf_fragment6 *
+pf_find_fragment6(u_int32_t ident, struct in6_addr *src, struct in6_addr *dst)
+{
+	struct pf_fragment6 *p;
+
+	TAILQ_FOREACH(p, &pf_frag_tree6, next) {
+		if (p->ident == ident && IN6_ARE_ADDR_EQUAL(&p->src, src) &&
+		    IN6_ARE_ADDR_EQUAL(&p->dst, dst))
+			return p;
+	}
+	return NULL;
+}
+
 int
 pf_normalize_ip6(struct mbuf **m0, int dir, struct pfi_kif *kif,
-    u_short *reason, struct pf_pdesc *pd)
+    u_short *reason, struct pf_pdesc *pd, struct mbuf **tree)
 {
-	struct mbuf		*m = *m0;
+	struct mbuf		*m = *m0, *n;
 	struct pf_rule		*r;
 	struct ip6_hdr		*h = mtod(m, struct ip6_hdr *);
 	int			 off;
@@ -1039,11 +1084,13 @@ pf_normalize_ip6(struct mbuf **m0, int dir, struct pfi_kif *kif,
 	struct ip6_opt_jumbo	 jumbo;
 	struct ip6_frag		 frag;
 	u_int32_t		 jumbolen = 0, plen;
-	u_int16_t		 fragoff = 0;
 	int			 optend;
 	int			 ooff;
 	u_int8_t		 proto;
+	int			 protooff;
 	int			 terminal;
+	int			 nxt;
+	struct pf_fragment6	*frag6;
 
 	r = TAILQ_FIRST(pf_main_ruleset.rules[PF_RULESET_SCRUB].active.ptr);
 	while (r != NULL) {
@@ -1080,10 +1127,20 @@ pf_normalize_ip6(struct mbuf **m0, int dir, struct pfi_kif *kif,
 
 	off = sizeof(struct ip6_hdr);
 	proto = h->ip6_nxt;
+	protooff = offsetof(struct ip6_hdr, ip6_nxt);
 	terminal = 0;
 	do {
 		switch (proto) {
 		case IPPROTO_FRAGMENT:
+			if (m_tag_find(m, PACKET_TAG_PF_FRAGCACHE, NULL)
+			    != NULL) {
+				/*
+				 * the fragment have already passed the
+				 * "scrub in".  no need to go to reass code
+				 */
+				terminal = 1;
+				break;
+			}
 			goto fragment;
 			break;
 		case IPPROTO_AH:
@@ -1097,6 +1154,7 @@ pf_normalize_ip6(struct mbuf **m0, int dir, struct pfi_kif *kif,
 			else
 				off += (ext.ip6e_len + 1) * 8;
 			proto = ext.ip6e_nxt;
+			protooff = off + offsetof(struct ip6_ext, ip6e_nxt);
 			break;
 		case IPPROTO_HOPOPTS:
 			if (!pf_pull_hdr(m, off, &ext, sizeof(ext), NULL,
@@ -1143,6 +1201,7 @@ pf_normalize_ip6(struct mbuf **m0, int dir, struct pfi_kif *kif,
 
 			off = optend;
 			proto = ext.ip6e_nxt;
+			protooff = off + offsetof(struct ip6_ext, ip6e_nxt);
 			break;
 		default:
 			terminal = 1;
@@ -1167,18 +1226,49 @@ pf_normalize_ip6(struct mbuf **m0, int dir, struct pfi_kif *kif,
 	return (PF_PASS);
 
  fragment:
-	if (ntohs(h->ip6_plen) == 0 || jumbolen)
-		goto drop;
+	/* jumbo payload packets cannot be fragmented */
 	plen = ntohs(h->ip6_plen);
+	if (plen == 0 || jumbolen)
+		goto drop;
+
+	m->m_nextpkt = NULL;
 
 	if (!pf_pull_hdr(m, off, &frag, sizeof(frag), NULL, NULL, AF_INET6))
 		goto shortpkt;
-	fragoff = ntohs(frag.ip6f_offlg & IP6F_OFF_MASK);
-	if (fragoff + (plen - off - sizeof(frag)) > IPV6_MAXPACKET)
-		goto badfrag;
 
-	/* do something about it */
-	/* remember to set pd->flags |= PFDESC_IP_REAS */
+	frag6 = pf_find_fragment6(frag.ip6f_ident, &h->ip6_src, &h->ip6_dst);
+	if (frag6 == NULL) {
+		/* fresh fragment id/src/dst tuple */
+		frag6 = pool_get(&pf_fragment6_pl, PR_NOWAIT);
+		bzero(frag6, sizeof(*frag6));
+		frag6->ident = frag.ip6f_ident;
+		frag6->src = h->ip6_src;
+		frag6->dst = h->ip6_dst;
+		timeout_set(&frag6->lifetime, pf_frag6_expire, frag6);
+		timeout_add(&frag6->lifetime, hz * 60);
+		TAILQ_INSERT_HEAD(&pf_frag_tree6, frag6, next);
+	} else
+		; /* i saw this id/src/dst tuple in the past */
+
+	/* to be passed to the caller as is */
+	n = m_copym(m, 0, M_COPYALL, M_DONTWAIT);
+	if (n) {
+		n->m_nextpkt = frag6->m;
+		frag6->m = n;
+	}
+	n = NULL;
+
+	nxt = frag6_input(m0, &off, IPPROTO_FRAGMENT);
+	if (nxt == IPPROTO_DONE) {
+		*m0 = m = NULL;
+		goto drop;
+	}
+
+	TAILQ_REMOVE(&pf_frag_tree6, frag6, next);
+	*tree = frag6->m;
+	timeout_del(&frag6->lifetime);
+	pool_put(&pf_fragment6_pl, frag6);
+
 	return (PF_PASS);
 
  shortpkt:
@@ -1192,12 +1282,22 @@ pf_normalize_ip6(struct mbuf **m0, int dir, struct pfi_kif *kif,
 	if (r != NULL && r->log)
 		PFLOG_PACKET(kif, h, m, AF_INET6, dir, *reason, r, NULL, NULL);
 	return (PF_DROP);
+}
 
- badfrag:
-	REASON_SET(reason, PFRES_FRAG);
-	if (r != NULL && r->log)
-		PFLOG_PACKET(kif, h, m, AF_INET6, dir, *reason, r, NULL, NULL);
-	return (PF_DROP);
+static void
+pf_frag6_expire(void *arg)
+{
+	struct pf_fragment6 *frag6 = (struct pf_fragment6 *)arg;
+	struct mbuf *n;
+
+	TAILQ_REMOVE(&pf_frag_tree6, frag6, next);
+
+	for (; frag6->m; frag6->m = n) {
+		n = frag6->m->m_nextpkt;
+		frag6->m->m_nextpkt = NULL;
+		m_freem(frag6->m);
+	}
+	pool_put(&pf_fragment6_pl, frag6);
 }
 #endif /* INET6 */
 
diff --git a/sys/net/pfvar.h b/sys/net/pfvar.h
index 23ff4358ff1..3bf60ae0229 100644
--- a/sys/net/pfvar.h
+++ b/sys/net/pfvar.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: pfvar.h,v 1.198 2004/06/21 19:26:01 mcbride Exp $ */
+/*	$OpenBSD: pfvar.h,v 1.199 2004/06/25 00:42:58 itojun Exp $ */
 
 /*
  * Copyright (c) 2001 Daniel Hartmeier
@@ -47,7 +47,7 @@ struct ip;
 #define	PF_TCPS_PROXY_SRC	((TCP_NSTATES)+0)
 #define	PF_TCPS_PROXY_DST	((TCP_NSTATES)+1)
 
-enum	{ PF_INOUT, PF_IN, PF_OUT };
+enum	{ PF_FORWARD, PF_IN, PF_OUT };
 enum	{ PF_LAN_EXT, PF_EXT_GWY, PF_ID };
 enum	{ PF_PASS, PF_DROP, PF_SCRUB, PF_NAT, PF_NONAT,
 	  PF_BINAT, PF_NOBINAT, PF_RDR, PF_NORDR, PF_SYNPROXY_DROP };
@@ -1367,7 +1367,7 @@ void	pf_normalize_init(void);
 int	pf_normalize_ip(struct mbuf **, int, struct pfi_kif *, u_short *,
 	    struct pf_pdesc *);
 int	pf_normalize_ip6(struct mbuf **, int, struct pfi_kif *, u_short *,
-	    struct pf_pdesc *);
+	    struct pf_pdesc *, struct mbuf **);
 int	pf_normalize_tcp(int, struct pfi_kif *, struct mbuf *, int, int, void *,
 	    struct pf_pdesc *);
 void	pf_normalize_tcp_cleanup(struct pf_state *);
diff --git a/sys/netinet6/ip6_forward.c b/sys/netinet6/ip6_forward.c
index eafe2b36e23..188455ad852 100644
--- a/sys/netinet6/ip6_forward.c
+++ b/sys/netinet6/ip6_forward.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: ip6_forward.c,v 1.31 2004/06/24 15:01:32 itojun Exp $	*/
+/*	$OpenBSD: ip6_forward.c,v 1.32 2004/06/25 00:42:58 itojun Exp $	*/
 /*	$KAME: ip6_forward.c,v 1.75 2001/06/29 12:42:13 jinmei Exp $	*/
 
 /*
@@ -429,28 +429,33 @@ ip6_forward(m, srcrt)
 		ip6->ip6_dst.s6_addr16[1] = 0;
 
 #if NPF > 0 
-	if (pf_test6(PF_OUT, rt->rt_ifp, &m, NULL) != PF_PASS) {
+	if (pf_test6(PF_FORWARD, rt->rt_ifp, &m, NULL) != PF_PASS) {
 		m_freem(m);
 		goto senderr;
 	}
 	if (m == NULL)
 		goto senderr;
-
-	ip6 = mtod(m, struct ip6_hdr *);
 #endif 
 
-	error = nd6_output(rt->rt_ifp, origifp, m, dst, rt);
-	if (error) {
-		in6_ifstat_inc(rt->rt_ifp, ifs6_out_discard);
-		ip6stat.ip6s_cantforward++;
-	} else {
-		ip6stat.ip6s_forward++;
-		in6_ifstat_inc(rt->rt_ifp, ifs6_out_forward);
-		if (type)
-			ip6stat.ip6s_redirectsent++;
-		else {
-			if (mcopy)
-				goto freecopy;
+#if NPF > 0
+	for (; m; m = m->m_nextpkt)
+#else
+	if (1)
+#endif
+	{
+		error = nd6_output(rt->rt_ifp, origifp, m, dst, rt);
+		if (error) {
+			in6_ifstat_inc(rt->rt_ifp, ifs6_out_discard);
+			ip6stat.ip6s_cantforward++;
+		} else {
+			ip6stat.ip6s_forward++;
+			in6_ifstat_inc(rt->rt_ifp, ifs6_out_forward);
+			if (type)
+				ip6stat.ip6s_redirectsent++;
+			else {
+				if (mcopy)
+					goto freecopy;
+			}
 		}
 	}