SCRUB(fragcache) to do gap tracking and overlap pruning of IPv4 fragments

without the memory overhead of the conventional defrag in SCRUB
ok dhartmei@, idea by deraadt@

4 files changed, 466 insertions, 62 deletions

diff --git a/sbin/pfctl/parse.y b/sbin/pfctl/parse.y
index 8fa6adc3b61..d91dced4147 100644
--- a/sbin/pfctl/parse.y
+++ b/sbin/pfctl/parse.y
@@ -1,4 +1,4 @@
-/*	$OpenBSD: parse.y,v 1.95 2002/06/11 02:12:37 dhartmei Exp $	*/
+/*	$OpenBSD: parse.y,v 1.96 2002/06/11 02:27:19 frantzen Exp $	*/
 
 /*
  * Copyright (c) 2001 Markus Friedl.  All rights reserved.
@@ -207,13 +207,13 @@ typedef struct {
 %token	RETURNRST RETURNICMP RETURNICMP6 PROTO INET INET6 ALL ANY ICMPTYPE
 %token	ICMP6TYPE CODE KEEP MODULATE STATE PORT RDR NAT BINAT ARROW NODF
 %token	MINTTL IPV6ADDR ERROR ALLOWOPTS FASTROUTE ROUTETO DUPTO NO LABEL
-%token	NOROUTE FRAGMENT USER GROUP MAXMSS MAXIMUM TTL
+%token	NOROUTE FRAGCACHE FRAGMENT USER GROUP MAXMSS MAXIMUM TTL
 %token	<v.string> STRING
 %token	<v.number> NUMBER
 %token	<v.i>	PORTUNARY PORTBINARY
 %type	<v.interface>	interface if_list if_item_not if_item
 %type	<v.number>	port icmptype icmp6type minttl uid gid maxmss
-%type	<v.i>	no dir log quick af nodf allowopts fragment
+%type	<v.i>	no dir log quick af nodf allowopts fragment fragcache
 %type	<v.b>	action flag flags blockspec
 %type	<v.range>	dport rport
 %type	<v.proto>	proto proto_list proto_item
@@ -253,7 +253,7 @@ varset		: STRING PORTUNARY STRING
 		}
 		;
 
-scrubrule	: SCRUB dir interface fromto nodf minttl maxmss
+scrubrule	: SCRUB fragcache dir interface fromto nodf minttl maxmss
 		{
 			struct pf_rule r;
 			
@@ -267,14 +267,16 @@ scrubrule	: SCRUB dir interface fromto nodf minttl maxmss
 			memset(&r, 0, sizeof(r));
 
 			r.action = PF_SCRUB;
-			r.direction = $2;
+			r.direction = $3;
 
-			if ($5)
-				r.rule_flag |= PFRULE_NODF;
+			if ($2)
+				r.rule_flag |= PFRULE_FRAGCACHE;
 			if ($6)
-				r.min_ttl = $6;
+				r.rule_flag |= PFRULE_NODF;
 			if ($7)
-				r.max_mss = $7;
+				r.min_ttl = $7;
+			if ($8)
+				r.max_mss = $8;
 
 			pfctl_add_rule(pf, &r);
 
@@ -431,6 +433,11 @@ blockspec	: /* empty */		{ $$.b2 = 0; $$.w = 0; }
 		}
 		;
 
+fragcache	: /* empty */		{ $$ = 0; }
+		| '(' FRAGCACHE ')'	{ $$ = PFRULE_FRAGCACHE; }
+		;
+
+
 dir		: IN			{ $$ = PF_IN; }
 		| OUT				{ $$ = PF_OUT; }
 		;
@@ -2038,6 +2045,7 @@ lookup(char *s)
 		{ "dup-to",	DUPTO},
 		{ "fastroute",	FASTROUTE},
 		{ "flags",	FLAGS},
+		{ "fragcache",	FRAGCACHE},
 		{ "fragment",	FRAGMENT},
 		{ "from",	FROM},
 		{ "group",	GROUP},
diff --git a/sbin/pfctl/pfctl_parser.c b/sbin/pfctl/pfctl_parser.c
index dae3e6f8f5e..0697ccd68ec 100644
--- a/sbin/pfctl/pfctl_parser.c
+++ b/sbin/pfctl/pfctl_parser.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: pfctl_parser.c,v 1.87 2002/06/11 02:12:37 dhartmei Exp $ */
+/*	$OpenBSD: pfctl_parser.c,v 1.88 2002/06/11 02:27:19 frantzen Exp $ */
 
 /*
  * Copyright (c) 2001 Daniel Hartmeier
@@ -648,8 +648,12 @@ print_rule(struct pf_rule *r)
 			else
 				printf(" ");
 		}
-	} else
-		printf("scrub ");
+	} else {
+		if ((r->rule_flag & PFRULE_FRAGCACHE) == 0)
+			printf("scrub ");
+		else
+			printf("scrub(fragcache) ");
+	}
 	if (r->direction == 0)
 		printf("in ");
 	else
diff --git a/sys/net/pf_norm.c b/sys/net/pf_norm.c
index c0ee3444aa3..ea959900626 100644
--- a/sys/net/pf_norm.c
+++ b/sys/net/pf_norm.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: pf_norm.c,v 1.31 2002/06/10 17:05:11 dhartmei Exp $ */
+/*	$OpenBSD: pf_norm.c,v 1.32 2002/06/11 02:27:19 frantzen Exp $ */
 
 /*
  * Copyright 2001 Niels Provos <provos@citi.umich.edu>
@@ -61,7 +61,15 @@ struct pf_frent {
 	struct mbuf *fr_m;
 };
 
+struct pf_frcache {
+	LIST_ENTRY(pf_frcache) fr_next;
+	uint16_t	fr_off;
+	uint16_t	fr_end;
+};
+
 #define PFFRAG_SEENLAST	0x0001		/* Seen the last fragment for this */
+#define PFFRAG_NOBUFFER	0x0002		/* Non-buffering fragment cache */
+#define BUFFER_FRAGMENTS(fr)	(!((fr)->fr_flags & PFFRAG_NOBUFFER))
 
 struct pf_fragment {
 	RB_ENTRY(pf_fragment) fr_entry;
@@ -73,14 +81,20 @@ struct pf_fragment {
 	u_int16_t	fr_id;		/* fragment id for reassemble */
 	u_int16_t	fr_max;		/* fragment data max */
 	u_int32_t	fr_timeout;
-	LIST_HEAD(pf_fragq, pf_frent) fr_queue;
+#define fr_queue	fr_u.fru_queue
+#define fr_cache	fr_u.fru_cache
+	union {
+		LIST_HEAD(pf_fragq, pf_frent) fru_queue;	/* buffering */
+		LIST_HEAD(pf_cacheq, pf_frcache) fru_cache;	/* non-buf */
+	} fr_u;
 };
 
 TAILQ_HEAD(pf_fragqueue, pf_fragment)	pf_fragqueue;
+TAILQ_HEAD(pf_cachequeue, pf_fragment)	pf_cachequeue;
 
 static __inline int	 pf_frag_compare(struct pf_fragment *,
 			    struct pf_fragment *);
-RB_HEAD(pf_frag_tree, pf_fragment)	pf_frag_tree;
+RB_HEAD(pf_frag_tree, pf_fragment)	pf_frag_tree, pf_cache_tree;
 RB_PROTOTYPE(pf_frag_tree, pf_fragment, fr_entry, pf_frag_compare);
 RB_GENERATE(pf_frag_tree, pf_fragment, fr_entry, pf_frag_compare);
 
@@ -89,9 +103,11 @@ void			 pf_ip2key(struct pf_fragment *, struct ip *);
 void			 pf_remove_fragment(struct pf_fragment *);
 void			 pf_flush_fragments(void);
 void			 pf_free_fragment(struct pf_fragment *);
-struct pf_fragment	*pf_find_fragment(struct ip *);
+struct pf_fragment	*pf_find_fragment(struct ip *, struct pf_frag_tree *);
 struct mbuf		*pf_reassemble(struct mbuf **, struct pf_fragment *,
 			    struct pf_frent *, int);
+struct mbuf		*pf_fragcache(struct mbuf **, struct ip*,
+			    struct pf_fragment *, int, int *);
 u_int16_t		 pf_cksum_fixup(u_int16_t, u_int16_t, u_int16_t);
 int			 pf_normalize_tcp(int, struct ifnet *, struct mbuf *,
 			    int, int, void *, struct pf_pdesc *);
@@ -118,8 +134,8 @@ int			 pf_normalize_tcpopt(struct pf_rule *, struct mbuf *,
 #endif
 
 /* Globals */
-struct pool		 pf_frent_pl, pf_frag_pl;
-int			 pf_nfrents;
+struct pool		 pf_frent_pl, pf_frag_pl, pf_cache_pl, pf_cent_pl;
+int			 pf_nfrents, pf_ncache;
 extern int		 pftm_frag;	/* Fragment expire timeout */
 
 void
@@ -129,11 +145,18 @@ pf_normalize_init(void)
 	    NULL);
 	pool_init(&pf_frag_pl, sizeof(struct pf_fragment), 0, 0, 0, "pffrag",
 	    NULL);
+	pool_init(&pf_cache_pl, sizeof(struct pf_fragment), 0, 0, 0,
+	    "pffrcache", NULL);
+	pool_init(&pf_cent_pl, sizeof(struct pf_frcache), 0, 0, 0, "pffrcent",
+	    NULL);
 
 	pool_sethiwat(&pf_frag_pl, PFFRAG_FRAG_HIWAT);
 	pool_sethardlimit(&pf_frent_pl, PFFRAG_FRENT_HIWAT, NULL, 0);
+	pool_sethardlimit(&pf_cache_pl, PFFRAG_FRCACHE_HIWAT, NULL, 0);
+	pool_sethardlimit(&pf_cent_pl, PFFRAG_FRCENT_HIWAT, NULL, 0);
 
 	TAILQ_INIT(&pf_fragqueue);
+	TAILQ_INIT(&pf_cachequeue);
 }
 
 static __inline int	
@@ -162,11 +185,23 @@ pf_purge_expired_fragments(void)
 	u_int32_t expire = time.tv_sec - pftm_frag;
 
 	while ((frag = TAILQ_LAST(&pf_fragqueue, pf_fragqueue)) != NULL) {
+		KASSERT(BUFFER_FRAGMENTS(frag));
+		if (frag->fr_timeout > expire)
+			break;
+
+		DPFPRINTF(("expiring %d(%p)\n", frag->fr_id, frag));
+		pf_free_fragment(frag);
+	}
+
+	while ((frag = TAILQ_LAST(&pf_cachequeue, pf_cachequeue)) != NULL) {
+		KASSERT(!BUFFER_FRAGMENTS(frag));
 		if (frag->fr_timeout > expire)
 			break;
 
 		DPFPRINTF(("expiring %d(%p)\n", frag->fr_id, frag));
 		pf_free_fragment(frag);
+		KASSERT(TAILQ_EMPTY(&pf_cachequeue) ||
+		    TAILQ_LAST(&pf_cachequeue, pf_cachequeue) != frag);
 	}
 }
 
@@ -178,17 +213,28 @@ void
 pf_flush_fragments(void)
 {
 	struct pf_fragment *frag;
-	int goal = pf_nfrents * 9 / 10;
+	int goal;
 
+	goal = pf_nfrents * 9 / 10;
 	DPFPRINTF(("trying to free > %d frents\n",
 		   pf_nfrents - goal));
-
 	while (goal < pf_nfrents) {
 		frag = TAILQ_LAST(&pf_fragqueue, pf_fragqueue);
 		if (frag == NULL)
 			break;
 		pf_free_fragment(frag);
 	}
+
+
+	goal = pf_ncache * 9 / 10;
+	DPFPRINTF(("trying to free > %d cache entries\n",
+		   pf_ncache - goal));
+	while (goal < pf_ncache) {
+		frag = TAILQ_LAST(&pf_cachequeue, pf_cachequeue);
+		if (frag == NULL)
+			break;
+		pf_free_fragment(frag);
+	}
 }
 
 /* Frees the fragments and all associated entries */
@@ -197,15 +243,30 @@ void
 pf_free_fragment(struct pf_fragment *frag)
 {
 	struct pf_frent *frent;
+	struct pf_frcache *frcache;
 
 	/* Free all fragments */
-	for (frent = LIST_FIRST(&frag->fr_queue); frent;
-	    frent = LIST_FIRST(&frag->fr_queue)) {
-		LIST_REMOVE(frent, fr_next);
+	if (BUFFER_FRAGMENTS(frag)) {
+		for (frent = LIST_FIRST(&frag->fr_queue); frent;
+		    frent = LIST_FIRST(&frag->fr_queue)) {
+			LIST_REMOVE(frent, fr_next);
+
+			m_freem(frent->fr_m);
+			pool_put(&pf_frent_pl, frent);
+			pf_nfrents--;
+		}
+	} else {
+		for (frcache = LIST_FIRST(&frag->fr_cache); frcache;
+		    frcache = LIST_FIRST(&frag->fr_cache)) {
+			LIST_REMOVE(frcache, fr_next);
 
-		m_freem(frent->fr_m);
-		pool_put(&pf_frent_pl, frent);
-		pf_nfrents--;
+			KASSERT(LIST_EMPTY(&frag->fr_cache) ||
+			    LIST_FIRST(&frag->fr_cache)->fr_off >
+			    frcache->fr_end);
+
+			pool_put(&pf_cent_pl, frcache);
+			pf_ncache--;
+		}
 	}
 
 	pf_remove_fragment(frag);
@@ -221,18 +282,24 @@ pf_ip2key(struct pf_fragment *key, struct ip *ip)
 }
 
 struct pf_fragment *
-pf_find_fragment(struct ip *ip)
+pf_find_fragment(struct ip *ip, struct pf_frag_tree *tree)
 {
 	struct pf_fragment key;
 	struct pf_fragment *frag;
 
 	pf_ip2key(&key, ip);
 
-	frag = RB_FIND(pf_frag_tree, &pf_frag_tree, &key);
+	frag = RB_FIND(pf_frag_tree, tree, &key);
 	if (frag != NULL) {
+		/* XXX Are we sure we want to update the timeout? */
 		frag->fr_timeout = time.tv_sec;
-		TAILQ_REMOVE(&pf_fragqueue, frag, frag_next);
-		TAILQ_INSERT_HEAD(&pf_fragqueue, frag, frag_next);
+		if (BUFFER_FRAGMENTS(frag)) {
+			TAILQ_REMOVE(&pf_fragqueue, frag, frag_next);
+			TAILQ_INSERT_HEAD(&pf_fragqueue, frag, frag_next);
+		} else {
+			TAILQ_REMOVE(&pf_cachequeue, frag, frag_next);
+			TAILQ_INSERT_HEAD(&pf_cachequeue, frag, frag_next);
+		}
 	}
 
 	return (frag);
@@ -243,9 +310,15 @@ pf_find_fragment(struct ip *ip)
 void
 pf_remove_fragment(struct pf_fragment *frag)
 {
-	RB_REMOVE(pf_frag_tree, &pf_frag_tree, frag);
-	TAILQ_REMOVE(&pf_fragqueue, frag, frag_next);
-	pool_put(&pf_frag_pl, frag);
+	if (BUFFER_FRAGMENTS(frag)) {
+		RB_REMOVE(pf_frag_tree, &pf_frag_tree, frag);
+		TAILQ_REMOVE(&pf_fragqueue, frag, frag_next);
+		pool_put(&pf_frag_pl, frag);
+	} else {
+		RB_REMOVE(pf_frag_tree, &pf_cache_tree, frag);
+		TAILQ_REMOVE(&pf_cachequeue, frag, frag_next);
+		pool_put(&pf_cache_pl, frag);
+	}
 }
 
 struct mbuf *
@@ -260,6 +333,8 @@ pf_reassemble(struct mbuf **m0, struct pf_fragment *frag,
 	u_int16_t off = ip->ip_off;
 	u_int16_t max = ip->ip_len + off;
 
+	KASSERT(frag == NULL || BUFFER_FRAGMENTS(frag));
+
 	/* Strip off ip header */
 	m->m_data += hlen;
 	m->m_len -= hlen;
@@ -433,18 +508,278 @@ pf_reassemble(struct mbuf **m0, struct pf_fragment *frag,
 	return (NULL);
 }
 
+struct mbuf *
+pf_fragcache(struct mbuf **m0, struct ip *h, struct pf_fragment *frag, int mff,
+    int *nomem)
+{
+	struct mbuf *m = *m0;
+	struct pf_frcache *frp, *fra, *cur = NULL;
+	int ip_len = h->ip_len - (h->ip_hl << 2);
+	u_int16_t off = h->ip_off << 3;
+	u_int16_t max = ip_len + off;
+
+	KASSERT(frag == NULL || !BUFFER_FRAGMENTS(frag));
+
+	/* Create a new range queue for this packet */
+	if (frag == NULL) {
+		frag = pool_get(&pf_cache_pl, PR_NOWAIT);
+		if (frag == NULL) {
+			pf_flush_fragments();
+			frag = pool_get(&pf_cache_pl, PR_NOWAIT);
+			if (frag == NULL)
+				goto no_mem;
+		}
+
+		/* Get an entry for the queue */
+		cur = pool_get(&pf_cent_pl, PR_NOWAIT);
+		if (cur == NULL) {
+			pool_put(&pf_cache_pl, frag);
+			goto no_mem;
+		}
+		pf_ncache++;
+
+		frag->fr_flags = PFFRAG_NOBUFFER;
+		frag->fr_max = 0;
+		frag->fr_src = h->ip_src;
+		frag->fr_dst = h->ip_dst;
+		frag->fr_p = h->ip_p;
+		frag->fr_id = h->ip_id;
+		frag->fr_timeout = time.tv_sec;
+
+		cur->fr_off = off;
+		cur->fr_end = max;
+		LIST_INIT(&frag->fr_cache);
+		LIST_INSERT_HEAD(&frag->fr_cache, cur, fr_next);
+
+		RB_INSERT(pf_frag_tree, &pf_cache_tree, frag);
+		TAILQ_INSERT_HEAD(&pf_cachequeue, frag, frag_next);
+
+		DPFPRINTF(("fragcache[%d]: new %d-%d\n", h->ip_id, off, max));
+
+		goto pass;
+	}
+
+	/*
+	 * Find a fragment after the current one:
+	 *  - off contains the real shifted offset.
+	 */
+	frp = NULL;
+	LIST_FOREACH(fra, &frag->fr_cache, fr_next) {
+		if (fra->fr_off > off)
+			break;
+		frp = fra;
+	}
+
+	KASSERT(frp != NULL || fra != NULL);
+
+	if (frp != NULL) {
+		int precut;
+
+		precut = frp->fr_end - off;
+		if (precut >= ip_len) {
+			/* Fragment is entirely a duplicate */
+			DPFPRINTF(("fragcache[%d]: dead (%d-%d) %d-%d\n",
+			    h->ip_id, frp->fr_off, frp->fr_end, off, max));
+			goto drop_fragment;
+		}
+		if (precut == 0) {
+			/* They are adjacent.  Fixup cache entry */
+			DPFPRINTF(("fragcache[%d]: adjacent (%d-%d) %d-%d\n",
+			    h->ip_id, frp->fr_off, frp->fr_end, off, max));
+			frp->fr_end = max;
+		} else if (precut > 0) {
+			/* The first part of this payload overlaps with a
+			 * fragment that has already been passed.
+			 * Need to trim off the first part of the payload.
+			 * But to do so easily, we need to create another
+			 * mbuf to throw the original header into.
+			 */
+
+			DPFPRINTF(("fragcache[%d]: chop %d (%d-%d) %d-%d\n",
+			    h->ip_id, precut, frp->fr_off, frp->fr_end, off,
+			    max));
+
+			/*
+			 * This is a very heavy way to trim the payload.
+			 * we could do it much faster by diddling mbuf
+			 * internals but that would be even less legible
+			 * than this mbuf magic.  For my next trick,
+			 * I'll pull a rabbit out of my laptop.
+			 */
+			*m0 = m_copym2(m, 0, h->ip_hl << 2, M_NOWAIT);
+			if (*m0 == NULL)
+				goto no_mem;
+			KASSERT((*m0)->m_next == NULL);
+			m_adj(m, precut + (h->ip_hl << 2));
+			m_cat(*m0, m);
+			m = *m0;
+			if (m->m_flags & M_PKTHDR) {
+				int plen = 0;
+				struct mbuf *t;
+				for (t = m; t; t = t->m_next)
+					plen += t->m_len;
+				m->m_pkthdr.len = plen;
+			}
+
+
+			h = mtod(m, struct ip *);
+
+			KASSERT(m->m_len == h->ip_len - precut);
+
+			off += precut;
+			h->ip_off += precut >> 3;
+			h->ip_len -= precut;
+			max -= precut;
+
+			/* Update the previous frag to encompas this one */
+			frp->fr_end = max;
+		} else {
+			/* There is a gap between fragments */
+
+			DPFPRINTF(("fragcache[%d]: gap %d (%d-%d) %d-%d\n",
+			    h->ip_id, -precut, frp->fr_off, frp->fr_end, off,
+			    max));
+
+			cur = pool_get(&pf_cent_pl, PR_NOWAIT);
+			if (cur == NULL)
+				goto no_mem;
+			pf_ncache++;
+
+			cur->fr_off = off;
+			cur->fr_end = max;
+			LIST_INSERT_AFTER(frp, cur, fr_next);
+		}
+	}
+
+	if (fra != NULL) {
+		int aftercut;
+		int merge = 0;
+
+		aftercut = max - fra->fr_off;
+		if (aftercut == 0) {
+			/* Adjacent fragments */
+			DPFPRINTF(("fragcache[%d]: adjacent %d-%d (%d-%d)\n",
+			    h->ip_id, off, max, fra->fr_off, fra->fr_end));
+			fra->fr_off = off;
+			merge = 1;
+		} else if (aftercut > 0) {
+			/* Need to chop off the tail of this fragment */
+			DPFPRINTF(("fragcache[%d]: chop %d %d-%d (%d-%d)\n",
+			    h->ip_id, aftercut, off, max, fra->fr_off,
+			    fra->fr_end));
+			fra->fr_off = off;
+
+			m_adj(m, -aftercut);
+			if (m->m_flags & M_PKTHDR) {
+				int plen = 0;
+				struct mbuf *t;
+				for (t = m; t; t = t->m_next)
+					plen += t->m_len;
+				m->m_pkthdr.len = plen;
+			}
+			h = mtod(m, struct ip *);
+			KASSERT(m->m_len == h->ip_len - aftercut);
+			h->ip_len -= aftercut;
+			max -= aftercut;
+			merge = 1;
+
+		} else {
+			/* There is a gap between fragments */
+			DPFPRINTF(("fragcache[%d]: gap %d %d-%d (%d-%d)\n",
+			    h->ip_id, -aftercut, off, max, fra->fr_off,
+			    fra->fr_end));
+
+			cur = pool_get(&pf_cent_pl, PR_NOWAIT);
+			if (cur == NULL)
+				goto no_mem;
+			pf_ncache++;
+
+			cur->fr_off = off;
+			cur->fr_end = max;
+			LIST_INSERT_BEFORE(fra, cur, fr_next);
+		}
+
+
+		/* Need to glue together two seperate fragment descriptors */
+		if (merge) {
+			if (cur && fra->fr_off <= cur->fr_end) {
+				/* Need to merge in a previous 'cur' */
+				DPFPRINTF(("fragcache[%d]: adjacent(merge %d-%d) %d-%d (%d-%d)\n",
+				    h->ip_id, cur->fr_off, cur->fr_end, off,
+				    max, fra->fr_off, fra->fr_end));
+				fra->fr_off = cur->fr_off;
+				LIST_REMOVE(cur, fr_next);
+				pool_put(&pf_cent_pl, cur);
+				pf_ncache--;
+				cur = NULL;
+
+			} else if (frp && fra->fr_off <= frp->fr_end) {
+				/* Need to merge in a modified 'frp' */
+				KASSERT(cur == NULL);
+				DPFPRINTF(("fragcache[%d]: adjacent(merge %d-%d) %d-%d (%d-%d)\n",
+				    h->ip_id, frp->fr_off, frp->fr_end, off,
+				    max, fra->fr_off, fra->fr_end));
+				fra->fr_off = frp->fr_off;
+				LIST_REMOVE(frp, fr_next);
+				pool_put(&pf_cent_pl, frp);
+				pf_ncache--;
+				frp = NULL;
+
+			}
+		}
+	}
+
+
+ pass:
+	/* Update maximum data size */
+	if (frag->fr_max < max)
+		frag->fr_max = max;
+
+	/* This is the last segment */
+	if (!mff)
+		frag->fr_flags |= PFFRAG_SEENLAST;
+
+	/* Check if we are completely reassembled */
+	if ((frag->fr_flags & PFFRAG_SEENLAST) &&
+	    LIST_FIRST(&frag->fr_cache)->fr_off == 0 &&
+	    LIST_FIRST(&frag->fr_cache)->fr_end == frag->fr_max) {
+		/* Remove from fragment queue */
+		DPFPRINTF(("fragcache[%d]: done 0-%d\n", h->ip_id,
+		    frag->fr_max));
+		pf_remove_fragment(frag);
+	}
+
+	return (m);
+
+ no_mem:
+	*nomem = 1;
+	/* FALLTHROUGH */
+
+ drop_fragment:
+
+	/* Still need to pay attention to !IP_MF */
+	if (!mff && frag)
+		frag->fr_flags |= PFFRAG_SEENLAST;
+
+	m_freem(m);
+	return (NULL);
+}
+
 int
 pf_normalize_ip(struct mbuf **m0, int dir, struct ifnet *ifp, u_short *reason)
 {
 	struct mbuf *m = *m0;
 	struct pf_rule *r;
 	struct pf_frent *frent;
-	struct pf_fragment *frag;
+	struct pf_fragment *frag = NULL;
 	struct ip *h = mtod(m, struct ip *);
 	int mff = (h->ip_off & IP_MF), hlen = h->ip_hl << 2;
 	u_int16_t fragoff = (h->ip_off & IP_OFFMASK) << 3;
 	u_int16_t max;
 
+	int ip_len;
+	int ip_off;
+
 	r = TAILQ_FIRST(pf_rules_active);
 	while (r != NULL) {
 		if (r->action != PF_SCRUB)
@@ -483,58 +818,96 @@ pf_normalize_ip(struct mbuf **m0, int dir, struct ifnet *ifp, u_short *reason)
 	if (!fragoff && !mff)
 		goto no_fragment;
 
-	/* Now we are dealing with a fragmented packet */
-	frag = pf_find_fragment(h);
-
 	/* This can not happen */
 	if (h->ip_off & IP_DF) {
 		DPFPRINTF(("IP_DF\n"));
 		goto bad;
 	}
 
-	h->ip_len -= hlen;
-	h->ip_off <<= 3;
+	ip_len = h->ip_len - hlen;
+	ip_off = h->ip_off << 3;
 
 	/* All fragments are 8 byte aligned */
-	if (mff && (h->ip_len & 0x7)) {
-		DPFPRINTF(("mff and %d\n", h->ip_len));
+	if (mff && (ip_len & 0x7)) {
+		DPFPRINTF(("mff and %d\n", ip_len));
 		goto bad;
 	}
 
-	max = fragoff + h->ip_len;
+	max = fragoff + ip_len;
 	/* Respect maximum length */
 	if (max > IP_MAXPACKET) {
 		DPFPRINTF(("max packet %d\n", max));
 		goto bad;
 	}
-	/* Check if we saw the last fragment already */
-	if (frag != NULL && (frag->fr_flags & PFFRAG_SEENLAST) &&
-	    max > frag->fr_max)
-		goto bad;
 
-	/* Get an entry for the fragment queue */
-	frent = pool_get(&pf_frent_pl, PR_NOWAIT);
-	if (frent == NULL) {
-		/* Try to clean up old fragments */
-		pf_flush_fragments();
+	if ((r->rule_flag & PFRULE_FRAGCACHE) == 0) {
+		/* Fully buffer all of the fragments */
+
+		h->ip_len = ip_len;	/* logic need muddled off/len */
+		h->ip_off = ip_off;
+		frag = pf_find_fragment(h, &pf_frag_tree);
+
+		/* Check if we saw the last fragment already */
+		if (frag != NULL && (frag->fr_flags & PFFRAG_SEENLAST) &&
+		    max > frag->fr_max)
+			goto bad;
+
+		/* Get an entry for the fragment queue */
 		frent = pool_get(&pf_frent_pl, PR_NOWAIT);
 		if (frent == NULL) {
 			REASON_SET(reason, PFRES_MEMORY);
 			return (PF_DROP);
 		}
-	}
-	pf_nfrents++;
-	frent->fr_ip = h;
-	frent->fr_m = m;
+		pf_nfrents++;
+		frent->fr_ip = h;
+		frent->fr_m = m;
+
+		/* Might return a completely reassembled mbuf, or NULL */
+		DPFPRINTF(("reass frag %d @ %d-%d\n", h->ip_id, fragoff, max));
+		*m0 = m = pf_reassemble(m0, frag, frent, mff);
+
+		if (m == NULL)
+			return (PF_DROP);
+
+		h = mtod(m, struct ip *);
+	} else {
+		/* non-buffering fragment cache */
+		int nomem = 0;
+
+		if (dir == PF_OUT) {
+			if (m_tag_find(m, PACKET_TAG_PF_FRAGCACHE, NULL) !=
+			    NULL) {
+				/* Already passed the fragment cache in the
+				 * input direction.  If we continued, it would
+				 * appear to be a dup and would be dropped.
+				 */
+				goto fragment_pass;
+			}
+		}
+
+		frag = pf_find_fragment(h, &pf_cache_tree);
 
-	/* Might return a completely reassembled mbuf, or NULL */
-	DPFPRINTF(("reass frag %d @ %d-%d\n", h->ip_id, fragoff, max));
-	*m0 = m = pf_reassemble(m0, frag, frent, mff);
+		/* Check if we saw the last fragment already */
+		if (frag != NULL && (frag->fr_flags & PFFRAG_SEENLAST) &&
+		    max > frag->fr_max)
+			goto bad;
 
-	if (m == NULL)
-		return (PF_DROP);
+		*m0 = m = pf_fragcache(m0, h, frag, mff, &nomem);
+		if (m == NULL) {
+			if (nomem)
+				goto no_mem;
+			goto drop;
+		}
 
-	h = mtod(m, struct ip *);
+		if (dir == PF_IN) {
+			struct m_tag *mtag;
+			mtag = m_tag_get(PACKET_TAG_PF_FRAGCACHE, 0, M_NOWAIT);
+			if (mtag == NULL)
+				goto no_mem;
+			m_tag_prepend(m, mtag);
+		}
+ 		goto fragment_pass;
+	}
 
  no_fragment:
 	if (dir != PF_OUT)
@@ -552,6 +925,22 @@ pf_normalize_ip(struct mbuf **m0, int dir, struct ifnet *ifp, u_short *reason)
 
 	return (PF_PASS);
 
+ fragment_pass:
+ 	if (dir != PF_OUT)
+		return (PF_PASS);
+
+	/* Enforce a minimum ttl, may cause endless packet loops */
+	if (r->min_ttl && h->ip_ttl < r->min_ttl)
+		h->ip_ttl = r->min_ttl;
+
+	return (PF_PASS);
+
+ no_mem:
+	REASON_SET(reason, PFRES_MEMORY);
+	if (r != NULL && r->log)
+		PFLOG_PACKET(ifp, h, m, AF_INET, dir, *reason, r);
+	return (PF_DROP);
+
  drop:
 	REASON_SET(reason, PFRES_NORM);
 	if (r != NULL && r->log)
diff --git a/sys/net/pfvar.h b/sys/net/pfvar.h
index 6db53b977ec..d5fa79b158b 100644
--- a/sys/net/pfvar.h
+++ b/sys/net/pfvar.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: pfvar.h,v 1.84 2002/06/11 02:12:37 dhartmei Exp $ */
+/*	$OpenBSD: pfvar.h,v 1.85 2002/06/11 02:27:19 frantzen Exp $ */
 
 /*
  * Copyright (c) 2001 Daniel Hartmeier
@@ -280,6 +280,7 @@ struct pf_rule {
 #define	PFRULE_RETURNRST	0x01
 #define	PFRULE_NODF		0x02
 #define	PFRULE_FRAGMENT		0x04
+#define	PFRULE_FRAGCACHE	0x10	/* non-buffering frag cache */
 
 struct pf_state_host {
 	struct pf_addr	addr;
@@ -462,6 +463,8 @@ struct pf_status {
 
 #define PFFRAG_FRENT_HIWAT	5000	/* Number of fragment entries */
 #define PFFRAG_FRAG_HIWAT	1000	/* Number of fragmented packets */
+#define PFFRAG_FRCENT_HIWAT	50000	/* Number of fragment cache entries */
+#define PFFRAG_FRCACHE_HIWAT	10000	/* Number of fragment descriptors */
 
 /*
  * ioctl parameter structures