Add initial support for pf state synchronization over the network.

Implemented as an in-kernel multicast IP protocol.

Turn it on like this:

# ifconfig pfsync0 up syncif fxp0

There is not yet any authentication on this protocol, so the syncif
must be on a trusted network. ie, a crossover cable between the two
firewalls.

NOTABLE CHANGES:
- A new index based on a unique (creatorid, stateid) tuple has been
  added to the state tree.
- Updates now appear on the pfsync(4) interface; multiple updates may
  be compressed into a single update.
- Applications which use bpf on pfsync(4) will need modification;
  packets on pfsync no longer contains regular pf_state structs,
  but pfsync_state structs which contain no pointers.

Much more to come.

ok deraadt@

7 files changed, 915 insertions, 105 deletions

diff --git a/sys/net/if_pfsync.c b/sys/net/if_pfsync.c
index fa0962fe279..cd413730112 100644
--- a/sys/net/if_pfsync.c
+++ b/sys/net/if_pfsync.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: if_pfsync.c,v 1.7 2003/11/08 19:51:38 dhartmei Exp $	*/
+/*	$OpenBSD: if_pfsync.c,v 1.8 2003/12/15 07:11:30 mcbride Exp $	*/
 
 /*
  * Copyright (c) 2002 Michael Shalayeff
@@ -30,6 +30,7 @@
 #include "pfsync.h"
 
 #include <sys/param.h>
+#include <sys/proc.h>
 #include <sys/systm.h>
 #include <sys/time.h>
 #include <sys/mbuf.h>
@@ -44,7 +45,10 @@
 
 #ifdef	INET
 #include <netinet/in.h>
+#include <netinet/in_systm.h>
 #include <netinet/in_var.h>
+#include <netinet/ip.h>
+#include <netinet/ip_var.h>
 #endif
 
 #ifdef INET6
@@ -68,28 +72,34 @@ int pfsyncdebug;
 #endif
 
 struct pfsync_softc pfsyncif;
+struct pfsyncstats pfsyncstats;
 
 void	pfsyncattach(int);
-void	pfsync_setmtu(struct pfsync_softc *sc, int);
+void	pfsync_setmtu(struct pfsync_softc *, int);
+int	pfsync_insert_net_state(struct pfsync_state *);
 int	pfsyncoutput(struct ifnet *, struct mbuf *, struct sockaddr *,
 	       struct rtentry *);
 int	pfsyncioctl(struct ifnet *, u_long, caddr_t);
 void	pfsyncstart(struct ifnet *);
 
-struct mbuf *pfsync_get_mbuf(struct pfsync_softc *sc, u_int8_t action);
+struct mbuf *pfsync_get_mbuf(struct pfsync_softc *, u_int8_t, void **);
 int	pfsync_sendout(struct pfsync_softc *sc);
-void	pfsync_timeout(void *v);
+void	pfsync_timeout(void *);
 
 extern int ifqmaxlen;
+extern struct timeval time;
 
 void
 pfsyncattach(int npfsync)
 {
 	struct ifnet *ifp;
 
+	bzero(&pfsyncif, sizeof(pfsyncif));
 	pfsyncif.sc_mbuf = NULL;
-	pfsyncif.sc_ptr = NULL;
-	pfsyncif.sc_count = 8;
+	pfsyncif.sc_mbuf_net = NULL;
+	pfsyncif.sc_sp.s = NULL;
+	pfsyncif.sc_sp_net.s = NULL;
+	pfsyncif.sc_maxupdates = 128;
 	ifp = &pfsyncif.sc_if;
 	strlcpy(ifp->if_xname, "pfsync0", sizeof ifp->if_xname);
 	ifp->if_softc = &pfsyncif;
@@ -133,6 +143,310 @@ pfsyncstart(struct ifnet *ifp)
 }
 
 int
+pfsync_insert_net_state(struct pfsync_state *sp)
+{
+	struct pf_state	*st = NULL;
+	struct pf_rule *r = NULL;
+	u_long secs;
+
+	if (sp->creatorid == 0 && pf_status.debug >= PF_DEBUG_MISC) {
+		printf("pfsync_insert_net_state: invalid creator id:"
+		    "id: %016llx creatorid: %08x\n",
+		    betoh64(sp->id), ntohl(sp->creatorid));
+		return (EINVAL);
+	}
+
+	/*
+	 * Just use the default rule until we have infrastructure to find the
+	 * best matching rule.
+	 */
+	r = &pf_default_rule;
+
+	if (!r->max_states || r->states < r->max_states)
+		st = pool_get(&pf_state_pl, PR_NOWAIT);
+	if (st == NULL)
+		return (ENOMEM);
+	bzero(st, sizeof(*st));
+
+	st->rule.ptr = r;
+	/* XXX get pointers to nat_rule and anchor */
+
+	/* fill in the rest of the state entry */
+	pf_state_host_ntoh(&sp->lan, &st->lan);
+	pf_state_host_ntoh(&sp->gwy, &st->gwy);
+	pf_state_host_ntoh(&sp->ext, &st->ext);
+
+	pf_state_peer_ntoh(&sp->src, &st->src);
+	pf_state_peer_ntoh(&sp->dst, &st->dst);
+
+	bcopy(&sp->rt_addr, &st->rt_addr, sizeof(st->rt_addr));
+	secs = time.tv_sec;
+	st->creation = secs + ntohl(sp->creation);
+
+	st->af = sp->af;
+	st->proto = sp->proto;
+	st->direction = sp->direction;
+	st->log = sp->log;
+	st->allow_opts = sp->allow_opts;
+
+	st->id = sp->id;
+	st->creatorid = sp->creatorid;
+	st->sync_flags = sp->sync_flags | PFSTATE_FROMSYNC;
+
+	secs = time.tv_sec;
+	if (sp->expire)
+		st->expire = 0;
+	else
+		st->expire = ntohl(sp->expire) + secs;
+
+	if (pf_insert_state(st)) {
+		pool_put(&pf_state_pl, st);
+		return (EINVAL);
+	}
+
+	return (0);
+}
+
+void
+pfsync_input(struct mbuf *m, ...)
+{
+	struct ip *ip = mtod(m, struct ip *);
+	struct pfsync_header *ph;
+	struct pfsync_softc *sc = &pfsyncif;
+	struct pf_state *st, key;
+	struct pfsync_state *sp;
+	struct pfsync_state_upd *up;
+	struct pfsync_state_del *dp;
+	struct pfsync_state_clr *cp;
+	struct mbuf *mp;
+	int iplen, action, error, i, s, count, offp;
+	u_long secs;
+
+	pfsyncstats.pfsyncs_ipackets++;
+
+	/* verify that we have a sync interface configured */
+	if (!sc->sc_sync_ifp)
+		goto done;
+
+	/* verify that the packet came in on the right interface */
+	if (sc->sc_sync_ifp != m->m_pkthdr.rcvif) {
+		pfsyncstats.pfsyncs_badif++;
+		goto done;
+	}
+
+	/* verify that the IP TTL is 255.  */
+	if (ip->ip_ttl != PFSYNC_DFLTTL) {
+		pfsyncstats.pfsyncs_badttl++;
+		goto done;
+	}
+
+	iplen = ip->ip_hl << 2;
+
+	if (m->m_pkthdr.len < iplen + sizeof(*ph)) {
+		pfsyncstats.pfsyncs_hdrops++;
+		goto done;
+	}
+
+	if (iplen + sizeof(*ph) > m->m_len) {
+		if ((m = m_pullup(m, iplen + sizeof(*ph))) == NULL) {
+			pfsyncstats.pfsyncs_hdrops++;
+			goto done;
+		}
+		ip = mtod(m, struct ip *);
+	}
+	ph = (void *)ip + iplen;
+
+	/* verify the version */
+	if (ph->version != PFSYNC_VERSION) {
+		pfsyncstats.pfsyncs_badver++;
+		goto done;
+	}
+
+	action = ph->action;
+	count = ph->count;
+
+	/* make sure it's a valid action code */
+	if (action >= PFSYNC_ACT_MAX) {
+		pfsyncstats.pfsyncs_badact++;
+		goto done;
+	}
+
+	switch (action) {
+	case PFSYNC_ACT_CLR: {
+		u_int32_t creatorid;
+		if ((mp = m_pulldown(m, iplen + sizeof(*ph),
+		    sizeof(*cp), &offp)) == NULL) {
+			pfsyncstats.pfsyncs_badlen++;
+			return;
+		}
+
+		s = splsoftnet();	
+		cp = (void *)((char *)mp->m_data + iplen + PFSYNC_HDRLEN); 
+		creatorid = cp->creatorid;
+
+                RB_FOREACH(st, pf_state_tree_ext_gwy, &tree_ext_gwy) {
+			if (st->creatorid == creatorid)
+                        	st->timeout = PFTM_PURGE;
+		}
+                pf_purge_expired_states();
+                splx(s);
+		break;
+	}
+	case PFSYNC_ACT_INS:
+		if ((mp = m_pulldown(m, iplen + sizeof(*ph),
+		    count * sizeof(*sp), &offp)) == NULL) {
+			pfsyncstats.pfsyncs_badlen++;
+			return;
+		}
+
+		s = splsoftnet();	
+		for (i = 0, sp = (void *)((char *)mp->m_data +
+		    iplen + PFSYNC_HDRLEN); i < count; i++, sp++) {
+			if ((error = pfsync_insert_net_state(sp))) {
+				if (error == ENOMEM) {
+					splx(s);
+                       			goto done;
+				}
+				continue;
+			}
+		}
+		splx(s);
+		break;
+
+	/*
+	 * It's not strictly necessary for us to support the "uncompressed"
+	 * update and delete actions, but it's relatively simple for us to do.
+	 */
+	case PFSYNC_ACT_UPD:
+		if ((mp = m_pulldown(m, iplen + sizeof(*ph),
+		    count * sizeof(*sp), &offp)) == NULL) {
+			pfsyncstats.pfsyncs_badlen++;
+			return;
+		}
+		
+		s = splsoftnet();	
+		for (i = 0, sp = (void *)((char *)mp->m_data +
+		    iplen + PFSYNC_HDRLEN); i < count; i++, sp++) {
+			key.id = sp->id;
+			key.creatorid = sp->creatorid;
+
+			st = pf_find_state(&key, PF_ID);
+			if (st == NULL) {
+				/* try to do an insert? */
+				pfsyncstats.pfsyncs_badstate++;
+				continue;
+			}
+			pf_state_peer_ntoh(&sp->src, &st->src);
+			pf_state_peer_ntoh(&sp->dst, &st->dst);
+			secs = time.tv_sec;
+			if (sp->expire)
+				st->expire = 0;
+			else
+				st->expire = ntohl(sp->expire) + secs;
+
+		}
+		splx(s);
+		break;
+	case PFSYNC_ACT_DEL:
+		if ((mp = m_pulldown(m, iplen + sizeof(*ph),
+		    count * sizeof(*sp), &offp)) == NULL) {
+			pfsyncstats.pfsyncs_badlen++;
+			return;
+		}
+		
+		s = splsoftnet();	
+		for (i = 0, sp = (void *)((char *)mp->m_data +
+		    iplen + PFSYNC_HDRLEN); i < count; i++, sp++) {
+			key.id = sp->id;
+			key.creatorid = sp->creatorid;
+
+			st = pf_find_state(&key, PF_ID);
+			if (st == NULL) {
+				pfsyncstats.pfsyncs_badstate++;
+				continue;
+			}
+			/*
+			 * XXX 
+			 * pf_purge_expired_states() is expensive,
+			 * we really want to purge the state directly.
+			 */
+			st->timeout = PFTM_PURGE;
+			st->sync_flags |= PFSTATE_FROMSYNC;
+		}
+		pf_purge_expired_states();
+		splx(s);
+		break;
+	case PFSYNC_ACT_UPD_C:
+		if ((mp = m_pulldown(m, iplen + sizeof(*ph),
+		    count * sizeof(*up), &offp)) == NULL) {
+			pfsyncstats.pfsyncs_badlen++;
+			return;
+		}
+		
+		s = splsoftnet();	
+		for (i = 0, up = (void *)((char *)mp->m_data +
+		    iplen + PFSYNC_HDRLEN); i < count; i++, up++) {
+			key.id = up->id;
+			key.creatorid = up->creatorid;
+
+			st = pf_find_state(&key, PF_ID);
+			if (st == NULL) {
+				/* send out a request for a full state? */
+				pfsyncstats.pfsyncs_badstate++;
+				continue;
+			}
+			pf_state_peer_ntoh(&up->src, &st->src);
+			pf_state_peer_ntoh(&up->dst, &st->dst);
+			secs = time.tv_sec;
+			if (up->expire)
+				st->expire = 0;
+			else
+				st->expire = ntohl(up->expire) + secs;
+
+		}
+		splx(s);
+		break;
+	case PFSYNC_ACT_DEL_C:
+		if ((mp = m_pulldown(m, iplen + sizeof(*ph),
+		    count * sizeof(*dp), &offp)) == NULL) {
+			pfsyncstats.pfsyncs_badlen++;
+			return;
+		}
+		
+		s = splsoftnet();	
+		for (i = 0, dp = (void *)((char *)mp->m_data +
+		    iplen + PFSYNC_HDRLEN); i < count; i++, dp++) {
+			key.id = dp->id;
+			key.creatorid = dp->creatorid;
+
+			st = pf_find_state(&key, PF_ID);
+			if (st == NULL) {
+				pfsyncstats.pfsyncs_badstate++;
+				continue;
+			}
+			/*
+			 * XXX 
+			 * pf_purge_expired_states() is expensive,
+			 * we really want to purge the state directly.
+			 */
+			st->timeout = PFTM_PURGE;
+			st->sync_flags |= PFSTATE_FROMSYNC;
+		}
+		pf_purge_expired_states();
+		splx(s);
+		break;
+	case PFSYNC_ACT_INS_F:
+	case PFSYNC_ACT_DEL_F:
+		/* not implemented */
+		break;
+	}
+	
+done:
+	if (m)
+		m_freem(m);
+}
+
+int
 pfsyncoutput(struct ifnet *ifp, struct mbuf *m, struct sockaddr *dst,
 	struct rtentry *rt)
 {
@@ -144,9 +458,13 @@ pfsyncoutput(struct ifnet *ifp, struct mbuf *m, struct sockaddr *dst,
 int
 pfsyncioctl(struct ifnet *ifp, u_long cmd, caddr_t data)
 {
+	struct proc *p = curproc;
 	struct pfsync_softc *sc = ifp->if_softc;
 	struct ifreq *ifr = (struct ifreq *)data;
-	int s;
+	struct ip_moptions *imo = &sc->sc_imo;
+	struct pfsyncreq pfsyncr;
+	struct ifnet    *sifp;
+	int s, error;
 
 	switch (cmd) {
 	case SIOCSIFADDR:
@@ -164,11 +482,72 @@ pfsyncioctl(struct ifnet *ifp, u_long cmd, caddr_t data)
 		if (ifr->ifr_mtu > MCLBYTES)
 			ifr->ifr_mtu = MCLBYTES;
 		s = splnet();
-		if (ifr->ifr_mtu < ifp->if_mtu)
+		if (ifr->ifr_mtu < ifp->if_mtu) 
 			pfsync_sendout(sc);
 		pfsync_setmtu(sc, ifr->ifr_mtu);
 		splx(s);
 		break;
+	case SIOCGETPFSYNC:
+		bzero(&pfsyncr, sizeof(pfsyncr));
+		if (sc->sc_sync_ifp) 
+			strlcpy(pfsyncr.pfsyncr_syncif,
+			    sc->sc_sync_ifp->if_xname, IFNAMSIZ);
+		pfsyncr.pfsyncr_maxupdates = sc->sc_maxupdates;
+                if ((error = copyout(&pfsyncr, ifr->ifr_data, sizeof(pfsyncr))))
+			return (error);
+		break;
+	case SIOCSETPFSYNC:
+		if ((error = suser(p, p->p_acflag)) != 0)
+			return (error);
+		if ((error = copyin(ifr->ifr_data, &pfsyncr, sizeof(pfsyncr))))
+			return (error);
+
+		if (pfsyncr.pfsyncr_maxupdates > 255)
+			return (EINVAL);
+		sc->sc_maxupdates = pfsyncr.pfsyncr_maxupdates;
+                
+		if (pfsyncr.pfsyncr_syncif[0] == 0) {
+			sc->sc_sync_ifp = NULL;
+			break;
+		}
+		if ((sifp = ifunit(pfsyncr.pfsyncr_syncif)) == NULL)
+			return (EINVAL);
+		else if (sifp == sc->sc_sync_ifp)
+			break;
+
+		s = splnet();
+		if (sifp->if_mtu < sc->sc_if.if_mtu ||
+		    (sc->sc_sync_ifp != NULL && 
+		    sifp->if_mtu < sc->sc_sync_ifp->if_mtu) ||
+		    sifp->if_mtu < MCLBYTES - sizeof(struct ip))
+			pfsync_sendout(sc);
+		sc->sc_sync_ifp = sifp;
+		
+		pfsync_setmtu(sc, sc->sc_if.if_mtu);
+		
+		if (imo->imo_num_memberships > 0) {
+			in_delmulti(imo->imo_membership[--imo->imo_num_memberships]);
+			imo->imo_multicast_ifp = NULL;
+		}
+
+		if (sc->sc_sync_ifp) {
+			struct in_addr addr;
+
+			addr.s_addr = INADDR_PFSYNC_GROUP;
+			if ((imo->imo_membership[0] =
+			    in_addmulti(&addr, sc->sc_sync_ifp)) == NULL) {
+				splx(s);
+				return (ENOBUFS);
+			}
+			imo->imo_num_memberships++;
+			imo->imo_multicast_ifp = sc->sc_sync_ifp;
+			imo->imo_multicast_ttl = PFSYNC_DFLTTL;
+			imo->imo_multicast_loop = 0;
+		}
+		splx(s);
+		
+		break;
+
 	default:
 		return (ENOTTY);
 	}
@@ -177,20 +556,25 @@ pfsyncioctl(struct ifnet *ifp, u_long cmd, caddr_t data)
 }
 
 void
-pfsync_setmtu(sc, mtu)
-	struct pfsync_softc *sc;
-	int mtu;
+pfsync_setmtu(struct pfsync_softc *sc, int mtu_req)
 {
-	sc->sc_count = (mtu - sizeof(struct pfsync_header)) /
-	    sizeof(struct pf_state);
+	int mtu;
+
+	if (sc->sc_sync_ifp && sc->sc_sync_ifp->if_mtu < mtu_req)
+		mtu = sc->sc_sync_ifp->if_mtu;
+	else
+		mtu = mtu_req; 
+
+	sc->sc_maxcount = (mtu - sizeof(struct pfsync_header)) /
+	    sizeof(struct pfsync_state);
+	if (sc->sc_maxcount > 254)
+	    sc->sc_maxcount = 254;
 	sc->sc_if.if_mtu = sizeof(struct pfsync_header) +
-	    sc->sc_count * sizeof(struct pf_state);
+	    sc->sc_maxcount * sizeof(struct pfsync_state);
 }
 
 struct mbuf *
-pfsync_get_mbuf(sc, action)
-	struct pfsync_softc *sc;
-	u_int8_t action;
+pfsync_get_mbuf(struct pfsync_softc *sc, u_int8_t action, void **sp)
 {
 	extern int hz;
 	struct pfsync_header *h;
@@ -203,7 +587,25 @@ pfsync_get_mbuf(sc, action)
 		return (NULL);
 	}
 
-	len = sc->sc_if.if_mtu;
+	switch (action) {
+	case PFSYNC_ACT_UPD_C:
+		len = (sc->sc_maxcount * sizeof(struct pfsync_state_upd))
+		    + sizeof(struct pfsync_header);
+		break;
+	case PFSYNC_ACT_DEL_C:
+		len = (sc->sc_maxcount * sizeof(struct pfsync_state_del))
+		    + sizeof(struct pfsync_header);
+		break;
+	case PFSYNC_ACT_CLR:
+		len = sizeof(struct pfsync_header) +
+		     sizeof(struct pfsync_state_clr);
+		break;
+	default:
+		len = (sc->sc_maxcount * sizeof(struct pfsync_state))
+		    + sizeof(struct pfsync_header);
+		break;
+	}
+
 	if (len > MHLEN) {
 		MCLGET(m, M_DONTWAIT);
 		if ((m->m_flags & M_EXT) == 0) {
@@ -211,119 +613,215 @@ pfsync_get_mbuf(sc, action)
 			sc->sc_if.if_oerrors++;
 			return (NULL);
 		}
-	}
-	m->m_pkthdr.rcvif = NULL;
-	m->m_pkthdr.len = m->m_len = len;
+		m->m_data += (MCLBYTES - len) &~ (sizeof(long) - 1);
+	} else 
+		MH_ALIGN(m, len);
 
+	m->m_pkthdr.rcvif = NULL;
+	m->m_pkthdr.len = m->m_len = sizeof(struct pfsync_header);
 	h = mtod(m, struct pfsync_header *);
 	h->version = PFSYNC_VERSION;
 	h->af = 0;
 	h->count = 0;
 	h->action = action;
 
-	sc->sc_mbuf = m;
-	sc->sc_ptr = (struct pf_state *)((char *)h + PFSYNC_HDRLEN);
+	*sp = (void *)((char *)h + PFSYNC_HDRLEN);
 	timeout_add(&sc->sc_tmo, hz);
-
 	return (m);
 }
 
 int
-pfsync_pack_state(action, st)
-	u_int8_t action;
-	struct pf_state *st;
+pfsync_pack_state(u_int8_t action, struct pf_state *st)
 {
-	extern struct timeval time;
 	struct ifnet *ifp = &pfsyncif.sc_if;
 	struct pfsync_softc *sc = ifp->if_softc;
-	struct pfsync_header *h;
-	struct pf_state *sp;
+	struct pfsync_header *h, *h_net;
+	struct pfsync_state *sp = NULL;
+	struct pfsync_state_upd *up = NULL;
+	struct pfsync_state_del *dp = NULL;
 	struct pf_rule *r;
-	struct mbuf *m;
 	u_long secs;
-	int s, ret;
+	int s, ret = 0;
+	u_int8_t i = 255, newaction = 0;
 
 	if (action >= PFSYNC_ACT_MAX)
 		return (EINVAL);
 
 	s = splnet();
-	m = sc->sc_mbuf;
-	if (m == NULL) {
-		if ((m = pfsync_get_mbuf(sc, action)) == NULL) {
+	if (sc->sc_mbuf == NULL) {
+		if ((sc->sc_mbuf = pfsync_get_mbuf(sc, action,
+		    (void *)&sc->sc_sp.s)) == NULL) {
 			splx(s);
 			return (ENOMEM);
 		}
-		h = mtod(m, struct pfsync_header *);
+		h = mtod(sc->sc_mbuf, struct pfsync_header *);
 	} else {
-		h = mtod(m, struct pfsync_header *);
+		h = mtod(sc->sc_mbuf, struct pfsync_header *);
 		if (h->action != action) {
 			pfsync_sendout(sc);
-			if ((m = pfsync_get_mbuf(sc, action)) == NULL) {
+			if ((sc->sc_mbuf = pfsync_get_mbuf(sc, action,
+			    (void *)&sc->sc_sp.s)) == NULL) {
 				splx(s);
 				return (ENOMEM);
 			}
-			h = mtod(m, struct pfsync_header *);
+			h = mtod(sc->sc_mbuf, struct pfsync_header *);
+		} else {
+			/*
+			 * If it's an update, look in the packet to see if
+			 * we already have an update for the state.
+			 */
+			if (action == PFSYNC_ACT_UPD && sc->sc_maxupdates) {
+				struct pfsync_state *usp =
+				    (void *)((char *)h + PFSYNC_HDRLEN);
+
+				for (i = 0; i < h->count; i++) {
+					if (usp->id == st->id &&
+					    usp->creatorid == st->creatorid) {
+						sp = usp;
+						break;
+					}
+					usp++;
+				}
+			}
 		}
 	}
 
-	sp = sc->sc_ptr++;
-	h->count++;
-	bzero(sp, sizeof(*sp));
+	secs = time.tv_sec;
 
-	bcopy(&st->lan, &sp->lan, sizeof(sp->lan));
-	bcopy(&st->gwy, &sp->gwy, sizeof(sp->gwy));
-	bcopy(&st->ext, &sp->ext, sizeof(sp->ext));
+	if (sp == NULL) {
+		/* not a "duplicate" update */
+		sp = sc->sc_sp.s++;
+		sc->sc_mbuf->m_pkthdr.len = 
+		    sc->sc_mbuf->m_len += sizeof(struct pfsync_state);
+		h->count++;
+		bzero(sp, sizeof(*sp));
+
+		sp->id = st->id;
+		sp->creatorid = st->creatorid;
+
+		pf_state_host_hton(&st->lan, &sp->lan);
+		pf_state_host_hton(&st->gwy, &sp->gwy);
+		pf_state_host_hton(&st->ext, &sp->ext);
+
+		bcopy(&st->rt_addr, &sp->rt_addr, sizeof(sp->rt_addr));
+
+		sp->creation = htonl(secs - st->creation);
+		sp->packets[0] = htonl(st->packets[0]);
+		sp->packets[1] = htonl(st->packets[1]);
+		sp->bytes[0] = htonl(st->bytes[0]);
+		sp->bytes[1] = htonl(st->bytes[1]);
+		if ((r = st->rule.ptr) == NULL)
+			sp->rule = htonl(-1);
+		else
+			sp->rule = htonl(r->nr);
+		if ((r = st->anchor.ptr) == NULL)
+			sp->anchor = htonl(-1);
+		else
+			sp->anchor = htonl(r->nr);
+		sp->af = st->af;
+		sp->proto = st->proto;
+		sp->direction = st->direction;
+		sp->log = st->log;
+		sp->allow_opts = st->allow_opts;
+
+		sp->sync_flags = st->sync_flags & PFSTATE_NOSYNC;
+	} else
+		sp->updates++;
 
 	pf_state_peer_hton(&st->src, &sp->src);
 	pf_state_peer_hton(&st->dst, &sp->dst);
 
-	bcopy(&st->rt_addr, &sp->rt_addr, sizeof(sp->rt_addr));
-	secs = time.tv_sec;
-	sp->creation = htonl(secs - st->creation);
 	if (st->expire <= secs)
 		sp->expire = htonl(0);
 	else
 		sp->expire = htonl(st->expire - secs);
-	sp->packets[0] = htonl(st->packets[0]);
-	sp->packets[1] = htonl(st->packets[1]);
-	sp->bytes[0] = htonl(st->bytes[0]);
-	sp->bytes[1] = htonl(st->bytes[1]);
-	if ((r = st->rule.ptr) == NULL)
-		sp->rule.nr = htonl(-1);
-	else
-		sp->rule.nr = htonl(r->nr);
-	if ((r = st->anchor.ptr) == NULL)
-		sp->anchor.nr = htonl(-1);
-	else
-		sp->anchor.nr = htonl(r->nr);
-	sp->af = st->af;
-	sp->proto = st->proto;
-	sp->direction = st->direction;
-	sp->log = st->log;
-	sp->allow_opts = st->allow_opts;
-
-	ret = 0;
-	if (h->count == sc->sc_count)
+
+	/* do we need to build "compressed" actions for network transfer? */
+	if (sc->sc_sync_ifp) {
+		switch (action) {
+		case PFSYNC_ACT_UPD:
+			newaction = PFSYNC_ACT_UPD_C;
+			break;
+		case PFSYNC_ACT_DEL:
+			newaction = PFSYNC_ACT_DEL_C;
+			break;
+		default:
+			/* by default we just send the uncompressed states */
+			break;
+		}
+	}
+		
+	if (newaction) {
+		if (sc->sc_mbuf_net == NULL) {
+			if ((sc->sc_mbuf_net = pfsync_get_mbuf(sc, newaction,
+			    (void *)&sc->sc_sp_net.s)) == NULL) {
+				splx(s);
+				return (ENOMEM);
+			}
+		}
+		h_net = mtod(sc->sc_mbuf_net, struct pfsync_header *);
+		
+		switch (newaction) {
+		case PFSYNC_ACT_UPD_C:
+			if (i < h->count) {
+				up = (void *)((char *)h_net +
+				    PFSYNC_HDRLEN + (i * sizeof(*up)));
+				up->updates++;
+			} else {
+				h_net->count++;
+				sc->sc_mbuf_net->m_pkthdr.len =
+				    sc->sc_mbuf_net->m_len += sizeof(*up);
+				up = sc->sc_sp_net.u++;
+
+				bzero(up, sizeof(*up));
+				up->id = st->id;
+				up->creatorid = st->creatorid;
+			}
+			up->expire = sp->expire;
+			up->src = sp->src;
+			up->dst = sp->dst;
+			break;
+		case PFSYNC_ACT_DEL_C:
+			sc->sc_mbuf_net->m_pkthdr.len =
+			    sc->sc_mbuf_net->m_len += sizeof(*dp);
+			dp = sc->sc_sp_net.d++;
+			h_net->count++;
+
+			bzero(dp, sizeof(*dp));
+			dp->id = st->id;
+			dp->creatorid = st->creatorid;
+			break;
+		}
+	}
+
+	if (h->count == sc->sc_maxcount ||
+	    (sc->sc_maxupdates && (sp->updates >= sc->sc_maxupdates)))
 		ret = pfsync_sendout(sc);
 
 	splx(s);
-	return (0);
+	return (ret);
 }
 
 int
-pfsync_clear_state(st)
-	struct pf_state *st;
+pfsync_clear_states(u_int32_t creatorid)
 {
 	struct ifnet *ifp = &pfsyncif.sc_if;
 	struct pfsync_softc *sc = ifp->if_softc;
-	struct mbuf *m = sc->sc_mbuf;
+	struct pfsync_state_clr *cp;
 	int s, ret;
 
 	s = splnet();
-	if (m == NULL && (m = pfsync_get_mbuf(sc, PFSYNC_ACT_CLR)) == NULL) {
+	if (sc->sc_mbuf != NULL) {
+		pfsync_sendout(sc);
+	}
+	if ((sc->sc_mbuf = pfsync_get_mbuf(sc, PFSYNC_ACT_CLR,
+	    (void *)&sc->sc_sp.c)) == NULL) {
 		splx(s);
 		return (ENOMEM);
 	}
+	sc->sc_mbuf->m_pkthdr.len = sc->sc_mbuf->m_len += sizeof(*cp);
+	cp = sc->sc_sp.c;
+	cp->creatorid = creatorid;
 
 	ret = (pfsync_sendout(sc));
 	splx(s);
@@ -350,14 +848,58 @@ pfsync_sendout(sc)
 
 	timeout_del(&sc->sc_tmo);
 	sc->sc_mbuf = NULL;
-	sc->sc_ptr = NULL;
+	sc->sc_sp.s = NULL;
 
 #if NBPFILTER > 0
 	if (ifp->if_bpf)
 		bpf_mtap(ifp->if_bpf, m);
 #endif
 
-	m_freem(m);
+
+	if (sc->sc_mbuf_net) {
+		m_freem(m);
+		m = sc->sc_mbuf_net;
+		sc->sc_mbuf_net = NULL;
+		sc->sc_sp_net.s = NULL;
+	}
+
+	if (sc->sc_sync_ifp) {
+		struct ip *ip;
+		struct ifaddr *ifa;
+		struct sockaddr sa;
+
+		M_PREPEND(m, sizeof(struct ip), M_DONTWAIT);
+		if (m == NULL) {
+			pfsyncstats.pfsyncs_onomem++;
+			return (0);
+		}
+		m->m_flags |= M_MCAST;
+		ip = mtod(m, struct ip *);
+		ip->ip_v = IPVERSION;
+		ip->ip_hl = sizeof(*ip) >> 2;
+		ip->ip_tos = IPTOS_LOWDELAY;
+		ip->ip_len = htons(m->m_pkthdr.len);
+		ip->ip_id = htons(ip_randomid());
+		ip->ip_off = htons(IP_DF);
+		ip->ip_ttl = PFSYNC_DFLTTL;
+		ip->ip_p = IPPROTO_PFSYNC;
+		ip->ip_sum = 0;
+
+		bzero(&sa, sizeof(sa));
+		sa.sa_family = AF_INET;
+		ifa = ifaof_ifpforaddr(&sa, sc->sc_sync_ifp);
+		if (ifa == NULL)
+			return (0);
+		ip->ip_src.s_addr = ifatoia(ifa)->ia_addr.sin_addr.s_addr;
+		ip->ip_dst.s_addr = INADDR_PFSYNC_GROUP;
+
+		pfsyncstats.pfsyncs_opackets++;
+
+		if (ip_output(m, NULL, NULL, IP_RAWOUTPUT, &sc->sc_imo, NULL))
+			pfsyncstats.pfsyncs_oerrors++;
+	} else {
+		m_freem(m);
+	}
 
 	return (0);
 }
diff --git a/sys/net/if_pfsync.h b/sys/net/if_pfsync.h
index cbeebcd9adf..6d6ad601dd0 100644
--- a/sys/net/if_pfsync.h
+++ b/sys/net/if_pfsync.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: if_pfsync.h,v 1.3 2003/11/08 00:45:34 mcbride Exp $	*/
+/*	$OpenBSD: if_pfsync.h,v 1.4 2003/12/15 07:11:30 mcbride Exp $	*/
 
 /*
  * Copyright (c) 2001 Michael Shalayeff
@@ -29,40 +29,172 @@
 #ifndef _NET_IF_PFSYNC_H_
 #define _NET_IF_PFSYNC_H_
 
+struct pfsync_state_scrub {
+	u_int16_t	pfss_flags;
+	u_int8_t	pfss_ttl;	/* stashed TTL		*/
+	u_int8_t	scrub_flag;
+	u_int32_t	pfss_ts_mod;	/* timestamp modulation	*/
+} __packed;
+
+struct pfsync_state_host {
+	struct pf_addr	addr;
+	u_int16_t	port;
+	u_int16_t	pad[3];
+} __packed;
+
+struct pfsync_state_peer {
+	struct pfsync_state_scrub scrub;	/* state is scrubbed	*/
+	u_int32_t	seqlo;		/* Max sequence number sent	*/
+	u_int32_t	seqhi;		/* Max the other end ACKd + win	*/
+	u_int32_t	seqdiff;	/* Sequence number modulator	*/
+	u_int16_t	max_win;	/* largest window (pre scaling)	*/
+	u_int16_t	mss;		/* Maximum segment size option	*/
+	u_int8_t	state;		/* active state level		*/
+	u_int8_t	wscale;		/* window scaling factor	*/
+	u_int8_t	scrub_flag;
+	u_int8_t	pad[5];
+} __packed;
+
+struct pfsync_state {
+	u_int64_t	 id;
+	struct pfsync_state_host lan;
+	struct pfsync_state_host gwy;
+	struct pfsync_state_host ext;
+	struct pfsync_state_peer src;
+	struct pfsync_state_peer dst;
+	struct pf_addr	 rt_addr;
+	u_int32_t	 rule;
+	u_int32_t	 anchor;
+	u_int32_t	 nat_rule;
+	u_int32_t	 creation;
+	u_int32_t	 expire;
+	u_int32_t	 packets[2];
+	u_int32_t	 bytes[2];
+	u_int32_t	 creatorid;
+	sa_family_t	 af;
+	u_int8_t	 proto;
+	u_int8_t	 direction;
+	u_int8_t	 log;
+	u_int8_t	 allow_opts;
+	u_int8_t	 timeout;
+	u_int8_t	 sync_flags;
+	u_int8_t	 updates;
+} __packed;
+
+struct pfsync_state_upd {
+	u_int64_t		id;
+	struct pfsync_state_peer	src;
+	struct pfsync_state_peer	dst;
+	u_int32_t		creatorid;
+	u_int32_t		expire;
+	u_int8_t		updates;
+	u_int8_t		pad[7];
+} __packed;
+
+struct pfsync_state_del {
+	u_int64_t		id;
+	u_int32_t		creatorid;
+	struct {
+		u_int8_t	state;
+	} src;
+	struct {
+		u_int8_t	state;
+	} dst;
+	u_int8_t		pad[2];
+} __packed;
+
+struct pfsync_state_clr {
+	u_int32_t		creatorid;
+	u_int32_t		pad;
+} __packed;
+
 #ifdef _KERNEL
+
+union sc_sp {	
+	struct pfsync_state	*s;	
+	struct pfsync_state_upd	*u;
+	struct pfsync_state_del	*d;
+	struct pfsync_state_clr	*c;
+};
+
 struct pfsync_softc {
-	struct ifnet	sc_if;
+	struct ifnet		 sc_if;
+	struct ifnet		*sc_sync_ifp;
 
-	struct timeout	sc_tmo;
-	struct mbuf	*sc_mbuf;	/* current cummulative mbuf */
-	struct pf_state	*sc_ptr;	/* current ongoing state */
-	int		 sc_count;	/* number of states in one mtu */
+	struct ip_moptions	 sc_imo;
+	struct timeout		 sc_tmo;
+	struct mbuf		*sc_mbuf;	/* current cummulative mbuf */
+	struct mbuf		*sc_mbuf_net;	/* current cummulative mbuf */
+	union sc_sp		 sc_sp;
+	union sc_sp		 sc_sp_net;			
+	int			 sc_maxcount;	/* number of states in mtu */
+	int			 sc_maxupdates;	/* number of updates/state */
 };
 #endif
 
+
 struct pfsync_header {
 	u_int8_t version;
 #define	PFSYNC_VERSION	1
 	u_int8_t af;
 	u_int8_t action;
-#define	PFSYNC_ACT_CLR	0
-#define	PFSYNC_ACT_INS	1
-#define	PFSYNC_ACT_UPD	2
-#define	PFSYNC_ACT_DEL	3
-#define	PFSYNC_ACT_MAX	4
+#define	PFSYNC_ACT_CLR		0	/* clear all states */
+#define	PFSYNC_ACT_INS		1	/* insert state */
+#define	PFSYNC_ACT_UPD		2	/* update state */
+#define	PFSYNC_ACT_DEL		3	/* delete state */	
+#define	PFSYNC_ACT_UPD_C	4	/* "compressed" state update */
+#define	PFSYNC_ACT_DEL_C	5	/* "compressed" state delete */
+#define	PFSYNC_ACT_INS_F	6	/* insert fragment */
+#define	PFSYNC_ACT_DEL_F	7	/* delete fragments */
+#define	PFSYNC_ACT_MAX		8
 	u_int8_t count;
-};
+} __packed;
 
 #define PFSYNC_HDRLEN	sizeof(struct pfsync_header)
 #define	PFSYNC_ACTIONS \
-	"CLR ST", "INS ST", "UPD ST", "DEL ST"
+	"CLR ST", "INS ST", "UPD ST", "DEL ST", \
+	"UPD ST COMP", "DEL ST COMP", "INS FR", "DEL FR"
+
+#define PFSYNC_DFLTTL             255
+
+struct pfsyncstats {
+	u_long	pfsyncs_ipackets;	/* total input packets, IPv4 */
+	u_long	pfsyncs_ipackets6;	/* total input packets, IPv6 */
+	u_long	pfsyncs_badif;		/* not the right interface */
+	u_long	pfsyncs_badttl;		/* TTL is not PFSYNC_DFLTTL */
+	u_long	pfsyncs_hdrops;		/* packets shorter than header */
+	u_long	pfsyncs_badver;		/* bad (incl unsupp) version */
+	u_long	pfsyncs_badact;		/* bad action */
+	u_long	pfsyncs_badlen;		/* data length does not match */
+	u_long	pfsyncs_badauth;	/* bad authentication */
+	u_long	pfsyncs_badstate;	/* insert/lookup failed */
+
+	u_long	pfsyncs_opackets;	/* total output packets, IPv4 */
+	u_long	pfsyncs_opackets6;	/* total output packets, IPv6 */
+	u_long	pfsyncs_onomem;		/* no memory for an mbuf for a send */
+	u_long	pfsyncs_oerrors;	/* ip output error */
+};
+
+/*
+ * Configuration structure for SIOCSETPFSYNC SIOCGETPFSYNC
+ */
+struct pfsyncreq {
+	char	pfsyncr_syncif[IFNAMSIZ];
+	int	pfsyncr_maxupdates;
+	int	pfsyncr_authlevel;
+};
+#define SIOCSETPFSYNC	_IOW('i', 247, struct ifreq)
+#define SIOCGETPFSYNC	_IOWR('i', 248, struct ifreq)
+
 
 #define pf_state_peer_hton(s,d) do {		\
 	(d)->seqlo = htonl((s)->seqlo);		\
 	(d)->seqhi = htonl((s)->seqhi);		\
 	(d)->seqdiff = htonl((s)->seqdiff);	\
 	(d)->max_win = htons((s)->max_win);	\
+	(d)->mss = htons((s)->mss);		\
 	(d)->state = (s)->state;		\
+	(d)->wscale = (s)->wscale;		\
 } while (0)
 
 #define pf_state_peer_ntoh(s,d) do {		\
@@ -70,23 +202,41 @@ struct pfsync_header {
 	(d)->seqhi = ntohl((s)->seqhi);		\
 	(d)->seqdiff = ntohl((s)->seqdiff);	\
 	(d)->max_win = ntohs((s)->max_win);	\
+	(d)->mss = ntohs((s)->mss);		\
 	(d)->state = (s)->state;		\
+	(d)->wscale = (s)->wscale;		\
+} while (0)
+
+#define pf_state_host_hton(s,d) do {				\
+	bcopy(&(s)->addr, &(d)->addr, sizeof((d)->addr));	\
+	(d)->port = htons((s)->port);				\
+} while (0)
+
+#define pf_state_host_ntoh(s,d) do {				\
+	bcopy(&(s)->addr, &(d)->addr, sizeof((d)->addr));	\
+	(d)->port = ntohs((s)->port);				\
 } while (0)
 
 #ifdef _KERNEL
-int pfsync_clear_state(struct pf_state *);
+void pfsync_input(struct mbuf *, ...);
+int pfsync_clear_states(u_int32_t);
 int pfsync_pack_state(u_int8_t, struct pf_state *);
 #define pfsync_insert_state(st)	do {			\
-	if (!(st->rule.ptr->rule_flag & PFRULE_NOSYNC))	\
+	if (st->rule.ptr->rule_flag & PFRULE_NOSYNC)	\
+		st->sync_flags |= PFSTATE_NOSYNC;	\
+	else if (!st->sync_flags)			\
 		pfsync_pack_state(PFSYNC_ACT_INS, (st));\
+	st->sync_flags &= ~PFSTATE_FROMSYNC;		\
 } while (0)
 #define pfsync_update_state(st) do {			\
-	if (!(st->rule.ptr->rule_flag & PFRULE_NOSYNC))	\
+	if (!st->sync_flags)				\
 		pfsync_pack_state(PFSYNC_ACT_UPD, (st));\
+	st->sync_flags &= ~PFSTATE_FROMSYNC;		\
 } while (0)
 #define pfsync_delete_state(st) do {			\
-	if (!(st->rule.ptr->rule_flag & PFRULE_NOSYNC))	\
+	if (!st->sync_flags)				\
 		pfsync_pack_state(PFSYNC_ACT_DEL, (st));\
+	st->sync_flags &= ~PFSTATE_FROMSYNC;		\
 } while (0)
 #endif
 
diff --git a/sys/net/pf.c b/sys/net/pf.c
index 0392850a2fa..14bdf9624d3 100644
--- a/sys/net/pf.c
+++ b/sys/net/pf.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: pf.c,v 1.408 2003/12/15 00:02:03 mcbride Exp $ */
+/*	$OpenBSD: pf.c,v 1.409 2003/12/15 07:11:30 mcbride Exp $ */
 
 /*
  * Copyright (c) 2001 Daniel Hartmeier
@@ -240,16 +240,21 @@ static __inline int pf_state_compare_lan_ext(struct pf_state *,
 	struct pf_state *);
 static __inline int pf_state_compare_ext_gwy(struct pf_state *,
 	struct pf_state *);
+static __inline int pf_state_compare_id(struct pf_state *,
+	struct pf_state *);
 
 struct pf_src_tree tree_src_tracking;
 struct pf_state_tree_lan_ext tree_lan_ext;
 struct pf_state_tree_ext_gwy tree_ext_gwy;
+struct pf_state_tree_id tree_id;
 
 RB_GENERATE(pf_src_tree, pf_src_node, entry, pf_src_compare);
 RB_GENERATE(pf_state_tree_lan_ext, pf_state,
     entry_lan_ext, pf_state_compare_lan_ext);
 RB_GENERATE(pf_state_tree_ext_gwy, pf_state,
     entry_ext_gwy, pf_state_compare_ext_gwy);
+RB_GENERATE(pf_state_tree_id, pf_state,
+    entry_id, pf_state_compare_id);
 
 static __inline int
 pf_src_compare(struct pf_src_node *a, struct pf_src_node *b)
@@ -431,6 +436,21 @@ pf_state_compare_ext_gwy(struct pf_state *a, struct pf_state *b)
 	return (0);
 }
 
+static __inline int
+pf_state_compare_id(struct pf_state *a, struct pf_state *b)
+{
+	if (a->id > b->id)
+		return (1);
+	if (a->id < b->id)
+		return (-1);
+	if (a->creatorid > b->creatorid)
+		return (1);
+	if (a->creatorid < b->creatorid)
+		return (-1);
+
+	return (0);
+}
+
 #ifdef INET6
 void
 pf_addrcpy(struct pf_addr *dst, struct pf_addr *src, sa_family_t af)
@@ -465,6 +485,9 @@ pf_find_state(struct pf_state *key, u_int8_t tree)
 	case PF_EXT_GWY:
 		s = RB_FIND(pf_state_tree_ext_gwy, &tree_ext_gwy, key);
 		break;
+	case PF_ID:
+		s = RB_FIND(pf_state_tree_id, &tree_id, key);
+		break;
 	default:
 		/* XXX should we just return NULL? */
 		panic("pf_find_state");
@@ -545,6 +568,8 @@ pf_insert_state(struct pf_state *state)
 			printf(" ext: ");
 			pf_print_host(&state->ext.addr, state->ext.port,
 			    state->af);
+			if (state->sync_flags & PFSTATE_FROMSYNC)
+				printf(" (from sync)");
 			printf("\n");
 		}
 		pf_src_tree_remove_state(state);
@@ -563,6 +588,8 @@ pf_insert_state(struct pf_state *state)
 			printf(" ext: ");
 			pf_print_host(&state->ext.addr, state->ext.port,
 			    state->af);
+			if (state->sync_flags & PFSTATE_FROMSYNC)
+				printf(" (from sync)");
 			printf("\n");
 		}
 		RB_REMOVE(pf_state_tree_lan_ext, &tree_lan_ext, state);
@@ -570,6 +597,24 @@ pf_insert_state(struct pf_state *state)
 		return (-1);
 	}
 
+	if (state->id == 0 && state->creatorid == 0) {
+		state->id = htobe64(pf_status.stateid++);
+		state->creatorid = pf_status.hostid;
+	}
+	if (RB_INSERT(pf_state_tree_id, &tree_id, state) != NULL) {
+		if (pf_status.debug >= PF_DEBUG_MISC) {
+			printf("pf: state insert failed: "
+			    "id: %016x creatorid: %08x",
+			    betoh64(state->id), ntohl(state->creatorid));
+			if (state->sync_flags & PFSTATE_FROMSYNC)
+				printf(" (from sync)");
+			printf("\n");
+		}
+		RB_REMOVE(pf_state_tree_lan_ext, &tree_lan_ext, state);
+		RB_REMOVE(pf_state_tree_ext_gwy, &tree_ext_gwy, state);
+		return (-1);
+	}
+
 	pf_status.fcounters[FCNT_STATE_INSERT]++;
 	pf_status.states++;
 #if NPFSYNC
@@ -692,6 +737,7 @@ pf_purge_expired_states(void)
 				    TH_RST|TH_ACK, 0, 0);
 			RB_REMOVE(pf_state_tree_ext_gwy, &tree_ext_gwy, cur);
 			RB_REMOVE(pf_state_tree_lan_ext, &tree_lan_ext, cur);
+			RB_REMOVE(pf_state_tree_id, &tree_id, cur);
 
 #if NPFSYNC
 			pfsync_delete_state(cur);
@@ -3869,8 +3915,8 @@ pf_test_state_tcp(struct pf_state **state, int direction, struct ifnet *ifp,
 	}
 
 	if (dst->scrub || src->scrub) {
-		if (pf_normalize_tcp_stateful(m, off, pd, reason, th, src, dst,
-		    &copyback))
+		if (pf_normalize_tcp_stateful(m, off, pd, reason, th,
+		    src, dst, &copyback))
 			return (PF_DROP);
 	}
 
@@ -5168,6 +5214,9 @@ pf_test(int dir, struct ifnet *ifp, struct mbuf **m0)
 		action = pf_test_state_tcp(&s, dir, ifp, m, off, h, &pd,
 		    &reason);
 		if (action == PF_PASS) {
+#if NPFSYNC
+			pfsync_update_state(s);
+#endif
 			r = s->rule.ptr;
 			a = s->anchor.ptr;
 			log = s->log;
@@ -5193,6 +5242,9 @@ pf_test(int dir, struct ifnet *ifp, struct mbuf **m0)
 		}
 		action = pf_test_state_udp(&s, dir, ifp, m, off, h, &pd);
 		if (action == PF_PASS) {
+#if NPFSYNC
+			pfsync_update_state(s);
+#endif
 			r = s->rule.ptr;
 			a = s->anchor.ptr;
 			log = s->log;
@@ -5218,6 +5270,10 @@ pf_test(int dir, struct ifnet *ifp, struct mbuf **m0)
 		}
 		action = pf_test_state_icmp(&s, dir, ifp, m, off, h, &pd);
 		if (action == PF_PASS) {
+#if NPFSYNC
+			pfsync_update_state(s);
+#endif
+
 			r = s->rule.ptr;
 			a = s->anchor.ptr;
 			log = s->log;
@@ -5230,6 +5286,9 @@ pf_test(int dir, struct ifnet *ifp, struct mbuf **m0)
 	default:
 		action = pf_test_state_other(&s, dir, ifp, &pd);
 		if (action == PF_PASS) {
+#if NPFSYNC
+			pfsync_update_state(s);
+#endif
 			r = s->rule.ptr;
 			a = s->anchor.ptr;
 			log = s->log;
@@ -5474,6 +5533,9 @@ pf_test6(int dir, struct ifnet *ifp, struct mbuf **m0)
 		action = pf_test_state_tcp(&s, dir, ifp, m, off, h, &pd,
 		    &reason);
 		if (action == PF_PASS) {
+#if NPFSYNC
+			pfsync_update_state(s);
+#endif
 			r = s->rule.ptr;
 			a = s->anchor.ptr;
 			log = s->log;
@@ -5499,6 +5561,9 @@ pf_test6(int dir, struct ifnet *ifp, struct mbuf **m0)
 		}
 		action = pf_test_state_udp(&s, dir, ifp, m, off, h, &pd);
 		if (action == PF_PASS) {
+#if NPFSYNC
+			pfsync_update_state(s);
+#endif
 			r = s->rule.ptr;
 			a = s->anchor.ptr;
 			log = s->log;
@@ -5525,6 +5590,9 @@ pf_test6(int dir, struct ifnet *ifp, struct mbuf **m0)
 		action = pf_test_state_icmp(&s, dir, ifp,
 		    m, off, h, &pd);
 		if (action == PF_PASS) {
+#if NPFSYNC
+			pfsync_update_state(s);
+#endif
 			r = s->rule.ptr;
 			a = s->anchor.ptr;
 			log = s->log;
diff --git a/sys/net/pf_ioctl.c b/sys/net/pf_ioctl.c
index bd1e7db5921..2d0493f9538 100644
--- a/sys/net/pf_ioctl.c
+++ b/sys/net/pf_ioctl.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: pf_ioctl.c,v 1.89 2003/12/15 00:02:04 mcbride Exp $ */
+/*	$OpenBSD: pf_ioctl.c,v 1.90 2003/12/15 07:11:30 mcbride Exp $ */
 
 /*
  * Copyright (c) 2001 Daniel Hartmeier
@@ -58,7 +58,9 @@
 #include <netinet/ip_var.h>
 #include <netinet/ip_icmp.h>
 
+#include <dev/rndvar.h>
 #include <net/pfvar.h>
+#include <net/if_pfsync.h>
 
 #ifdef INET6
 #include <netinet/ip6.h>
@@ -161,6 +163,10 @@ pfattach(int num)
 	pf_normalize_init();
 	bzero(&pf_status, sizeof(pf_status));
 	pf_status.debug = PF_DEBUG_URGENT;
+
+	/* XXX do our best to avoid a conflict */
+	pf_status.hostid = arc4random();
+	pf_status.stateid = 1; 	/* might want 0 for something special */
 }
 
 int
@@ -782,13 +788,17 @@ pfioctl(dev_t dev, u_long cmd, caddr_t addr, int flags, struct proc *p)
 		if (pf_status.running)
 			error = EEXIST;
 		else {
+			u_int64_t stateid = pf_status.stateid;
 			u_int32_t states = pf_status.states;
 			u_int32_t debug = pf_status.debug;
+			u_int32_t hostid = pf_status.hostid;
 			u_int32_t src_nodes = pf_status.src_nodes;
 			bzero(&pf_status, sizeof(struct pf_status));
 			pf_status.running = 1;
 			pf_status.states = states;
 			pf_status.debug = debug;
+			pf_status.stateid = stateid;
+			pf_status.hostid = hostid;
 			pf_status.states = src_nodes;
 			pf_status.since = time.tv_sec;
 			if (status_ifp != NULL)
@@ -1199,6 +1209,7 @@ pfioctl(dev_t dev, u_long cmd, caddr_t addr, int flags, struct proc *p)
 		pf_purge_expired_states();
 		pf_status.states = 0;
 		splx(s);
+		pfsync_clear_states(pf_status.hostid);
 		break;
 	}
 
@@ -2486,6 +2497,17 @@ pfioctl(dev_t dev, u_long cmd, caddr_t addr, int flags, struct proc *p)
 		break;
 	}
 
+	case DIOCSETHOSTID: {
+		u_int32_t	*hostid = (u_int32_t *)addr;
+
+		if (*hostid == 0) {
+			error = EINVAL;
+			goto fail;
+		}
+		pf_status.hostid = *hostid;
+		break;
+	}
+
 	case DIOCOSFPFLUSH:
 		s = splsoftnet();
 		pf_osfp_flush();
diff --git a/sys/net/pfvar.h b/sys/net/pfvar.h
index 4b4e4219a97..5789dcb5e11 100644
--- a/sys/net/pfvar.h
+++ b/sys/net/pfvar.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: pfvar.h,v 1.177 2003/12/15 00:02:04 mcbride Exp $ */
+/*	$OpenBSD: pfvar.h,v 1.178 2003/12/15 07:11:30 mcbride Exp $ */
 
 /*
  * Copyright (c) 2001 Daniel Hartmeier
@@ -47,7 +47,7 @@ struct ip;
 #define	PF_TCPS_PROXY_DST	((TCP_NSTATES)+1)
 
 enum	{ PF_INOUT, PF_IN, PF_OUT };
-enum	{ PF_LAN_EXT, PF_EXT_GWY };
+enum	{ PF_LAN_EXT, PF_EXT_GWY, PF_ID };
 enum	{ PF_PASS, PF_DROP, PF_SCRUB, PF_NAT, PF_NONAT,
 	  PF_BINAT, PF_NOBINAT, PF_RDR, PF_NORDR, PF_SYNPROXY_DROP };
 enum	{ PF_RULESET_SCRUB, PF_RULESET_FILTER, PF_RULESET_NAT,
@@ -576,8 +576,10 @@ struct pf_state_peer {
 };
 
 struct pf_state {
+	u_int64_t	 id;
 	RB_ENTRY(pf_state) entry_lan_ext;
 	RB_ENTRY(pf_state) entry_ext_gwy;
+	RB_ENTRY(pf_state) entry_id;
 	struct pf_state_host lan;
 	struct pf_state_host gwy;
 	struct pf_state_host ext;
@@ -594,13 +596,17 @@ struct pf_state {
 	u_int32_t	 expire;
 	u_int32_t	 packets[2];
 	u_int32_t	 bytes[2];
+	u_int32_t        creatorid;
 	sa_family_t	 af;
 	u_int8_t	 proto;
 	u_int8_t	 direction;
 	u_int8_t	 log;
 	u_int8_t	 allow_opts;
 	u_int8_t	 timeout;
-	u_int8_t	 pad[2];
+	u_int8_t	 sync_flags;
+#define	PFSTATE_NOSYNC	 0x01
+#define	PFSTATE_FROMSYNC 0x02
+	u_int8_t	 pad;
 };
 
 TAILQ_HEAD(pf_rulequeue, pf_rule);
@@ -844,11 +850,13 @@ struct pf_status {
 	u_int64_t	scounters[SCNT_MAX];
 	u_int64_t	pcounters[2][2][3];
 	u_int64_t	bcounters[2][2];
+	u_int64_t	stateid;
 	u_int32_t	running;
 	u_int32_t	states;
 	u_int32_t	src_nodes;
 	u_int32_t	since;
 	u_int32_t	debug;
+	u_int32_t	hostid;
 	char		ifname[IFNAMSIZ];
 };
 
@@ -1155,9 +1163,9 @@ struct pfioc_table {
 #define DIOCXBEGIN      _IOWR('D', 81, struct pfioc_trans)
 #define DIOCXCOMMIT     _IOWR('D', 82, struct pfioc_trans)
 #define DIOCXROLLBACK   _IOWR('D', 83, struct pfioc_trans)
-#define DIOCGETSRCNODES		_IOWR('D', 84, struct pfioc_src_nodes)
-#define DIOCCLRSRCNODES		_IO('D', 85)
-
+#define DIOCGETSRCNODES	_IOWR('D', 84, struct pfioc_src_nodes)
+#define DIOCCLRSRCNODES	_IO('D', 85)
+#define DIOCSETHOSTID	_IOWR('D', 86, u_int32_t)
 
 #ifdef _KERNEL
 RB_HEAD(pf_src_tree, pf_src_node);
@@ -1174,6 +1182,11 @@ RB_PROTOTYPE(pf_state_tree_ext_gwy, pf_state,
     entry_ext_gwy, pf_state_compare_ext_gwy);
 extern struct pf_state_tree_ext_gwy tree_ext_gwy;
 
+RB_HEAD(pf_state_tree_id, pf_state);
+RB_PROTOTYPE(pf_state_tree_id, pf_state,
+    entry_id, pf_state_compare_id);
+extern struct pf_state_tree_id tree_id;
+
 extern struct pf_anchorqueue		 pf_anchors;
 extern struct pf_ruleset		 pf_main_ruleset;
 TAILQ_HEAD(pf_poolqueue, pf_pool);
@@ -1217,7 +1230,7 @@ extern struct pf_ruleset	*pf_find_or_create_ruleset(char *, char *);
 extern void			 pf_remove_if_empty_ruleset(
 				    struct pf_ruleset *);
 
-extern struct ifnet		*status_ifp;
+extern struct ifnet		*status_ifp, *sync_ifp;
 extern struct pf_rule		 pf_default_rule;
 extern void			 pf_addrcpy(struct pf_addr *, struct pf_addr *,
 				    u_int8_t);
diff --git a/sys/netinet/in.h b/sys/netinet/in.h
index 03dabb85914..5d09134af38 100644
--- a/sys/netinet/in.h
+++ b/sys/netinet/in.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: in.h,v 1.58 2003/10/17 21:04:58 mcbride Exp $	*/
+/*	$OpenBSD: in.h,v 1.59 2003/12/15 07:11:30 mcbride Exp $	*/
 /*	$NetBSD: in.h,v 1.20 1996/02/13 23:41:47 christos Exp $	*/
 
 /*
@@ -73,6 +73,7 @@
 #define IPPROTO_PIM		103		/* Protocol indep. multicast */
 #define IPPROTO_IPCOMP		108		/* IP Payload Comp. Protocol */
 #define	IPPROTO_CARP		112		/* CARP */
+#define	IPPROTO_PFSYNC		136		/* PFSYNC */
 #define	IPPROTO_RAW		255		/* raw IP packet */
 
 #define	IPPROTO_MAX		256
@@ -200,6 +201,7 @@ struct in_addr {
 #define	INADDR_ALLHOSTS_GROUP	__IPADDR(0xe0000001)	/* 224.0.0.1 */
 #define	INADDR_ALLROUTERS_GROUP __IPADDR(0xe0000002)	/* 224.0.0.2 */
 #define	INADDR_CARP_GROUP	__IPADDR(0xe0000012)	/* 224.0.0.18 */
+#define	INADDR_PFSYNC_GROUP	__IPADDR(0xe0000088)	/* 224.0.0.136 */
 #define INADDR_MAX_LOCAL_GROUP	__IPADDR(0xe00000ff)	/* 224.0.0.255 */
 
 #define	IN_LOOPBACKNET		127			/* official! */
diff --git a/sys/netinet/in_proto.c b/sys/netinet/in_proto.c
index 2f83f8fcb65..06c535d0fb1 100644
--- a/sys/netinet/in_proto.c
+++ b/sys/netinet/in_proto.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: in_proto.c,v 1.37 2003/10/17 21:04:58 mcbride Exp $	*/
+/*	$OpenBSD: in_proto.c,v 1.38 2003/12/15 07:11:30 mcbride Exp $	*/
 /*	$NetBSD: in_proto.c,v 1.14 1996/02/18 18:58:32 christos Exp $	*/
 
 /*
@@ -181,6 +181,12 @@
 #include <netinet/ip_carp.h>
 #endif
 
+#include "pfsync.h"
+#if NPFSYNC > 0
+#include <net/pfvar.h>
+#include <net/if_pfsync.h>
+#endif
+
 extern	struct domain inetdomain;
 
 struct protosw inetsw[] = {
@@ -311,6 +317,13 @@ struct protosw inetsw[] = {
   0,		0,		0,		0,		carp_sysctl
 },
 #endif /* NCARP > 0 */
+#if NPFSYNC > 0
+{ SOCK_RAW,	&inetdomain,	IPPROTO_PFSYNC,	PR_ATOMIC|PR_ADDR,
+  pfsync_input,	rip_output,	0,		rip_ctloutput,
+  rip_usrreq,
+  0,		0,		0,		0,
+},
+#endif /* NPFSYNC > 0 */
 /* raw wildcard */
 { SOCK_RAW,	&inetdomain,	0,		PR_ATOMIC|PR_ADDR,
   rip_input,	rip_output,	0,		rip_ctloutput,