pfsync v5, mostly written at n2k9, but based on work done at n2k8.

WARNING: THIS BREAKS COMPATIBILITY WITH THE PREVIOUS VERSION OF PFSYNC

this is a new variant of the protocol and a large reworking of the
pfsync code to address some performance issues. the single largest
benefit comes from having multiple pfsync messages of different
types handled in a single packet. pfsyncs handling of pf states is
highly optimised now, along with packet parsing and construction.

huggz for beck@ for testing.
huge thanks to mcbride@ for his help during development and for
finding all the bugs during the initial tests.
thanks to peter sutton for letting me get credit for this work.

ok beck@ mcbride@ "good." deraadt@

1 files changed, 1298 insertions, 979 deletions

diff --git a/sys/net/if_pfsync.c b/sys/net/if_pfsync.c
index ae5216edb84..3cba30e64bd 100644
--- a/sys/net/if_pfsync.c
+++ b/sys/net/if_pfsync.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: if_pfsync.c,v 1.102 2008/12/21 23:41:26 dlg Exp $	*/
+/*	$OpenBSD: if_pfsync.c,v 1.103 2009/02/16 00:31:25 dlg Exp $	*/
 
 /*
  * Copyright (c) 2002 Michael Shalayeff
@@ -37,16 +37,17 @@
 #include <sys/timeout.h>
 #include <sys/kernel.h>
 #include <sys/sysctl.h>
+#include <sys/pool.h>
 
 #include <net/if.h>
 #include <net/if_types.h>
 #include <net/route.h>
 #include <net/bpf.h>
+#include <net/netisr.h>
 #include <netinet/in.h>
 #include <netinet/if_ether.h>
 #include <netinet/tcp.h>
 #include <netinet/tcp_seq.h>
-#include <sys/pool.h>
 
 #ifdef	INET
 #include <netinet/in_systm.h>
@@ -70,15 +71,132 @@
 #include "bpfilter.h"
 #include "pfsync.h"
 
-#define PFSYNC_MINMTU	\
-    (sizeof(struct pfsync_header) + sizeof(struct pf_state))
+#define PFSYNC_MINPKT ( \
+	sizeof(struct ip) + \
+	sizeof(struct pfsync_header) + \
+	sizeof(struct pfsync_subheader) + \
+	sizeof(struct pfsync_eof))
 
-#ifdef PFSYNCDEBUG
-#define DPRINTF(x)    do { if (pfsyncdebug) printf x ; } while (0)
-int pfsyncdebug;
-#else
-#define DPRINTF(x)
-#endif
+struct pfsync_pkt {
+	struct ip *ip;
+	struct in_addr src;
+	u_int8_t flags;
+};
+
+int	pfsync_input_hmac(struct mbuf *, int);
+
+int	pfsync_upd_tcp(struct pf_state *, struct pfsync_state_peer *,
+	    struct pfsync_state_peer *);
+
+int	pfsync_in_clr(struct pfsync_pkt *, struct mbuf *, int, int);
+int	pfsync_in_ins(struct pfsync_pkt *, struct mbuf *, int, int);
+int	pfsync_in_iack(struct pfsync_pkt *, struct mbuf *, int, int);
+int	pfsync_in_upd(struct pfsync_pkt *, struct mbuf *, int, int);
+int	pfsync_in_upd_c(struct pfsync_pkt *, struct mbuf *, int, int);
+int	pfsync_in_ureq(struct pfsync_pkt *, struct mbuf *, int, int);
+int	pfsync_in_del(struct pfsync_pkt *, struct mbuf *, int, int);
+int	pfsync_in_del_c(struct pfsync_pkt *, struct mbuf *, int, int);
+int	pfsync_in_bus(struct pfsync_pkt *, struct mbuf *, int, int);
+int	pfsync_in_tdb(struct pfsync_pkt *, struct mbuf *, int, int);
+int	pfsync_in_eof(struct pfsync_pkt *, struct mbuf *, int, int);
+
+int	pfsync_in_error(struct pfsync_pkt *, struct mbuf *, int, int);
+
+int	(*pfsync_acts[])(struct pfsync_pkt *, struct mbuf *, int, int) = {
+	pfsync_in_clr,			/* PFSYNC_ACT_CLR */
+	pfsync_in_ins,			/* PFSYNC_ACT_INS */
+	pfsync_in_iack,			/* PFSYNC_ACT_INS_ACK */
+	pfsync_in_upd,			/* PFSYNC_ACT_UPD */
+	pfsync_in_upd_c,		/* PFSYNC_ACT_UPD_C */
+	pfsync_in_ureq,			/* PFSYNC_ACT_UPD_REQ */
+	pfsync_in_del,			/* PFSYNC_ACT_DEL */
+	pfsync_in_del_c,		/* PFSYNC_ACT_DEL_C */
+	pfsync_in_error,		/* PFSYNC_ACT_INS_F */
+	pfsync_in_error,		/* PFSYNC_ACT_DEL_F */
+	pfsync_in_bus,			/* PFSYNC_ACT_BUS */
+	pfsync_in_tdb,			/* PFSYNC_ACT_TDB */
+	pfsync_in_eof			/* PFSYNC_ACT_EOF */
+};
+
+struct pfsync_q {
+	int		(*write)(struct pf_state *, struct mbuf *, int);
+	size_t		len;
+	u_int8_t	action;
+};
+
+/* we have one of these for every PFSYNC_S_ */
+int	pfsync_out_state(struct pf_state *, struct mbuf *, int);
+int	pfsync_out_iack(struct pf_state *, struct mbuf *, int);
+int	pfsync_out_upd_c(struct pf_state *, struct mbuf *, int);
+int	pfsync_out_del(struct pf_state *, struct mbuf *, int);
+
+struct pfsync_q pfsync_qs[] = {
+	{ pfsync_out_state, sizeof(struct pfsync_state),   PFSYNC_ACT_INS },
+	{ pfsync_out_iack,  sizeof(struct pfsync_ins_ack), PFSYNC_ACT_INS_ACK },
+	{ pfsync_out_state, sizeof(struct pfsync_state),   PFSYNC_ACT_UPD },
+	{ pfsync_out_upd_c, sizeof(struct pfsync_upd_c),   PFSYNC_ACT_UPD_C },
+	{ pfsync_out_del,   sizeof(struct pfsync_del_c),   PFSYNC_ACT_DEL_C }
+};
+
+void	pfsync_q_ins(struct pf_state *, int);
+void	pfsync_q_del(struct pf_state *);
+
+struct pfsync_upd_req_item {
+	TAILQ_ENTRY(pfsync_upd_req_item)	ur_entry;
+	struct pfsync_upd_req			ur_msg;
+};
+TAILQ_HEAD(pfsync_upd_reqs, pfsync_upd_req_item);
+
+struct pfsync_deferral {
+	TAILQ_ENTRY(pfsync_deferral)		 pd_entry;
+	struct pf_state				*pd_st;
+	struct mbuf				*pd_m;
+	struct timeout				 pd_tmo;
+};
+TAILQ_HEAD(pfsync_deferrals, pfsync_deferral);
+
+#define PFSYNC_PLSIZE	MAX(sizeof(struct pfsync_upd_req_item), \
+			    sizeof(struct pfsync_deferral))
+
+int	pfsync_out_tdb(struct tdb *, struct mbuf *, int);
+
+struct pfsync_softc {
+	struct ifnet		 sc_if;
+	struct ifnet		*sc_sync_if;
+
+	struct pool		 sc_pool;
+
+	struct ip_moptions	 sc_imo;
+
+	struct in_addr		 sc_sync_peer;
+	u_int8_t		 sc_maxupdates;
+
+	struct ip		 sc_template;
+
+	struct pf_state_queue	 sc_qs[PFSYNC_S_COUNT];
+	size_t			 sc_len;
+
+	struct pfsync_upd_reqs	 sc_upd_req_list;
+
+	struct pfsync_deferrals	 sc_deferrals;
+	u_int			 sc_deferred;
+
+	void			*sc_plus;
+	size_t			 sc_pluslen;
+
+	u_int32_t		 sc_ureq_sent;
+	int			 sc_bulk_tries;
+	struct timeout		 sc_bulkfail_tmo;
+
+	u_int32_t		 sc_ureq_received;
+	struct pf_state		*sc_bulk_next;
+	struct pf_state		*sc_bulk_last;
+	struct timeout		 sc_bulk_tmo;
+
+	TAILQ_HEAD(, tdb)	 sc_tdb_q;
+
+	struct timeout		 sc_tmo;
+};
 
 struct pfsync_softc	*pfsyncif = NULL;
 struct pfsyncstats	 pfsyncstats;
@@ -86,7 +204,6 @@ struct pfsyncstats	 pfsyncstats;
 void	pfsyncattach(int);
 int	pfsync_clone_create(struct if_clone *, int);
 int	pfsync_clone_destroy(struct ifnet *);
-void	pfsync_setmtu(struct pfsync_softc *, int);
 int	pfsync_alloc_scrub_memory(struct pfsync_state_peer *,
 	    struct pf_state_peer *);
 void	pfsync_update_net_tdb(struct pfsync_tdb *);
@@ -95,51 +212,31 @@ int	pfsyncoutput(struct ifnet *, struct mbuf *, struct sockaddr *,
 int	pfsyncioctl(struct ifnet *, u_long, caddr_t);
 void	pfsyncstart(struct ifnet *);
 
-struct mbuf *pfsync_get_mbuf(struct pfsync_softc *, u_int8_t, void **);
-int	pfsync_request_update(struct pfsync_state_upd *, struct in_addr *);
-int	pfsync_sendout(struct pfsync_softc *);
+struct mbuf *pfsync_if_dequeue(struct ifnet *);
+struct mbuf *pfsync_get_mbuf(struct pfsync_softc *);
+
+void	pfsync_deferred(struct pf_state *, int);
+void	pfsync_undefer(struct pfsync_deferral *, int);
+void	pfsync_defer_tmo(void *);
+
+void	pfsync_request_update(u_int32_t, u_int64_t);
+void	pfsync_update_state_req(struct pf_state *);
+
+void	pfsync_drop(struct pfsync_softc *);
+void	pfsync_sendout(void);
+void	pfsync_send_plus(void *, size_t);
 int	pfsync_tdb_sendout(struct pfsync_softc *);
 int	pfsync_sendout_mbuf(struct pfsync_softc *, struct mbuf *);
 void	pfsync_timeout(void *);
 void	pfsync_tdb_timeout(void *);
 void	pfsync_send_bus(struct pfsync_softc *, u_int8_t);
-void	pfsync_bulk_update(void *);
-void	pfsync_bulkfail(void *);
-
-struct pfsync_pkt {
-	struct ip *ip;
-	struct in_addr src;
-	u_int8_t flags;
-};
 
-void	pfsync4_input(struct pfsync_pkt *, struct mbuf *);
-
-int	pfsync4_in_clr(struct pfsync_pkt *, struct mbuf *, int, int);
-int	pfsync4_in_ins(struct pfsync_pkt *, struct mbuf *, int, int);
-int	pfsync4_in_upd(struct pfsync_pkt *, struct mbuf *, int, int);
-int	pfsync4_in_del(struct pfsync_pkt *, struct mbuf *, int, int);
-int	pfsync4_in_upd_c(struct pfsync_pkt *, struct mbuf *, int, int);
-int	pfsync4_in_del_c(struct pfsync_pkt *, struct mbuf *, int, int);
-int	pfsync4_in_ureq(struct pfsync_pkt *, struct mbuf *, int, int);
-int	pfsync4_in_bus(struct pfsync_pkt *, struct mbuf *, int, int);
-int	pfsync4_in_tdb_upd(struct pfsync_pkt *, struct mbuf *, int, int);
-
-int	pfsync4_in_error(struct pfsync_pkt *, struct mbuf *, int, int);
-
-int	(*pfsync4_acts[])(struct pfsync_pkt *, struct mbuf *, int, int) = {
-	pfsync4_in_clr,			/* PFSYNC_ACT_CLR */
-	pfsync4_in_ins,			/* PFSYNC_ACT_INS */
-	pfsync4_in_upd,			/* PFSYNC_ACT_UPD */
-	pfsync4_in_del,			/* PFSYNC_ACT_DEL */
-	pfsync4_in_upd_c,		/* PFSYNC_ACT_UPD_C */
-	pfsync4_in_del_c,		/* PFSYNC_ACT_DEL_C */
-	pfsync4_in_error,		/* PFSYNC_ACT_INS_F */
-	pfsync4_in_error,		/* PFSYNC_ACT_DEL_F */
-	pfsync4_in_ureq,		/* PFSYNC_ACT_UREQ */
-	pfsync4_in_bus,			/* PFSYNC_ACT_BUS */
-	pfsync4_in_tdb_upd		/* PFSYNC_ACT_TDB_UPD */
-};
+void	pfsync_bulk_start(void);
+void	pfsync_bulk_status(u_int8_t);
+void	pfsync_bulk_update(void *);
+void	pfsync_bulk_fail(void *);
 
+#define PFSYNC_MAX_BULKTRIES	12
 int	pfsync_sync_ok;
 
 struct if_clone	pfsync_cloner =
@@ -153,46 +250,50 @@ pfsyncattach(int npfsync)
 int
 pfsync_clone_create(struct if_clone *ifc, int unit)
 {
+	struct pfsync_softc *sc;
 	struct ifnet *ifp;
+	int q;
 
 	if (unit != 0)
 		return (EINVAL);
 
 	pfsync_sync_ok = 1;
-	if ((pfsyncif = malloc(sizeof(*pfsyncif), M_DEVBUF,
-	    M_NOWAIT|M_ZERO)) == NULL)
+
+	sc = malloc(sizeof(*pfsyncif), M_DEVBUF, M_NOWAIT | M_ZERO);
+	if (sc == NULL)
 		return (ENOMEM);
-	pfsyncif->sc_mbuf = NULL;
-	pfsyncif->sc_mbuf_net = NULL;
-	pfsyncif->sc_mbuf_tdb = NULL;
-	pfsyncif->sc_statep.s = NULL;
-	pfsyncif->sc_statep_net.s = NULL;
-	pfsyncif->sc_statep_tdb.t = NULL;
-	pfsyncif->sc_maxupdates = 128;
-	pfsyncif->sc_sync_peer.s_addr = INADDR_PFSYNC_GROUP;
-	pfsyncif->sc_sendaddr.s_addr = INADDR_PFSYNC_GROUP;
-	pfsyncif->sc_ureq_received = 0;
-	pfsyncif->sc_ureq_sent = 0;
-	pfsyncif->sc_bulk_send_next = NULL;
-	pfsyncif->sc_bulk_terminator = NULL;
-	pfsyncif->sc_imo.imo_membership = (struct in_multi **)malloc(
+
+	for (q = 0; q < PFSYNC_S_COUNT; q++)
+		TAILQ_INIT(&sc->sc_qs[q]);
+
+	pool_init(&sc->sc_pool, PFSYNC_PLSIZE, 0, 0, 0, "pfsync", NULL);
+	TAILQ_INIT(&sc->sc_upd_req_list);
+	TAILQ_INIT(&sc->sc_deferrals);
+	sc->sc_deferred = 0;
+
+	sc->sc_len = PFSYNC_MINPKT;
+	sc->sc_maxupdates = 128;
+
+	sc->sc_imo.imo_membership = (struct in_multi **)malloc(
 	    (sizeof(struct in_multi *) * IP_MIN_MEMBERSHIPS), M_IPMOPTS,
-	    M_WAITOK|M_ZERO);
-	pfsyncif->sc_imo.imo_max_memberships = IP_MIN_MEMBERSHIPS;
-	ifp = &pfsyncif->sc_if;
+	    M_WAITOK | M_ZERO);
+	sc->sc_imo.imo_max_memberships = IP_MIN_MEMBERSHIPS;
+
+	ifp = &sc->sc_if;
 	snprintf(ifp->if_xname, sizeof ifp->if_xname, "pfsync%d", unit);
-	ifp->if_softc = pfsyncif;
+	ifp->if_softc = sc;
 	ifp->if_ioctl = pfsyncioctl;
 	ifp->if_output = pfsyncoutput;
 	ifp->if_start = pfsyncstart;
 	ifp->if_type = IFT_PFSYNC;
 	ifp->if_snd.ifq_maxlen = ifqmaxlen;
-	ifp->if_hdrlen = PFSYNC_HDRLEN;
-	pfsync_setmtu(pfsyncif, ETHERMTU);
-	timeout_set(&pfsyncif->sc_tmo, pfsync_timeout, pfsyncif);
-	timeout_set(&pfsyncif->sc_tdb_tmo, pfsync_tdb_timeout, pfsyncif);
-	timeout_set(&pfsyncif->sc_bulk_tmo, pfsync_bulk_update, pfsyncif);
-	timeout_set(&pfsyncif->sc_bulkfail_tmo, pfsync_bulkfail, pfsyncif);
+	ifp->if_hdrlen = sizeof(struct pfsync_header);
+	ifp->if_mtu = 1500; /* XXX */
+	ifp->if_hardmtu = MCLBYTES; /* XXX */
+	timeout_set(&sc->sc_tmo, pfsync_timeout, sc);
+	timeout_set(&sc->sc_bulk_tmo, pfsync_bulk_update, sc);
+	timeout_set(&sc->sc_bulkfail_tmo, pfsync_bulk_fail, sc);
+
 	if_attach(ifp);
 	if_alloc_sadl(ifp);
 
@@ -201,9 +302,11 @@ pfsync_clone_create(struct if_clone *ifc, int unit)
 #endif
 
 #if NBPFILTER > 0
-	bpfattach(&pfsyncif->sc_if.if_bpf, ifp, DLT_PFSYNC, PFSYNC_HDRLEN);
+	bpfattach(&sc->sc_if.if_bpf, ifp, DLT_PFSYNC, PFSYNC_HDRLEN);
 #endif
 
+	pfsyncif = sc;
+
 	return (0);
 }
 
@@ -212,10 +315,8 @@ pfsync_clone_destroy(struct ifnet *ifp)
 {
 	struct pfsync_softc *sc = ifp->if_softc;
 
-	timeout_del(&sc->sc_tmo);
-	timeout_del(&sc->sc_tdb_tmo);
 	timeout_del(&sc->sc_bulk_tmo);
-	timeout_del(&sc->sc_bulkfail_tmo);
+	timeout_del(&sc->sc_tmo);
 #if NCARP > 0
 	if (!pfsync_sync_ok)
 		carp_group_demote_adj(&sc->sc_if, -1);
@@ -224,12 +325,34 @@ pfsync_clone_destroy(struct ifnet *ifp)
 	bpfdetach(ifp);
 #endif
 	if_detach(ifp);
-	free(pfsyncif->sc_imo.imo_membership, M_IPMOPTS);
-	free(pfsyncif, M_DEVBUF);
+
+	pfsync_drop(sc);
+
+	while (sc->sc_deferred > 0)
+		pfsync_undefer(TAILQ_FIRST(&sc->sc_deferrals), 0);
+
+	pool_destroy(&sc->sc_pool);
+	free(sc->sc_imo.imo_membership, M_IPMOPTS);
+	free(sc, M_DEVBUF);
+
 	pfsyncif = NULL;
+
 	return (0);
 }
 
+struct mbuf *
+pfsync_if_dequeue(struct ifnet *ifp)
+{
+	struct mbuf *m;
+	int s;
+
+	s = splnet();
+	IF_DEQUEUE(&ifp->if_snd, m);
+	splx(s);
+
+	return (m);
+}
+
 /*
  * Start output on the pfsync interface.
  */
@@ -237,18 +360,10 @@ void
 pfsyncstart(struct ifnet *ifp)
 {
 	struct mbuf *m;
-	int s;
 
-	for (;;) {
-		s = splnet();
+	while ((m = pfsync_if_dequeue(ifp)) != NULL) {
 		IF_DROP(&ifp->if_snd);
-		IF_DEQUEUE(&ifp->if_snd, m);
-		splx(s);
-
-		if (m == NULL)
-			return;
-		else
-			m_freem(m);
+		m_freem(m);
 	}
 }
 
@@ -423,8 +538,6 @@ pfsync_state_import(struct pfsync_state *sp, u_int8_t flags)
 	st->log = sp->log;
 	st->timeout = sp->timeout;
 	st->state_flags = sp->state_flags;
-	if (!(flags & PFSYNC_SI_IOCTL))
-		st->sync_flags = PFSTATE_FROMSYNC;
 
 	bcopy(sp->id, &st->id, sizeof(st->id));
 	st->creatorid = sp->creatorid;
@@ -436,19 +549,31 @@ pfsync_state_import(struct pfsync_state *sp, u_int8_t flags)
 	st->anchor.ptr = NULL;
 	st->rt_kif = NULL;
 
-	st->pfsync_time = 0;
-
+	st->pfsync_time = time_second;
+	st->sync_state = PFSYNC_S_NONE;
 
 	/* XXX when we have nat_rule/anchors, use STATE_INC_COUNTERS */
 	r->states_cur++;
 	r->states_tot++;
 
+	if (!ISSET(flags, PFSYNC_SI_IOCTL))
+		SET(st->state_flags, PFSTATE_NOSYNC);
+
 	if ((error = pf_state_insert(kif, skw, sks, st)) != 0) {
 		/* XXX when we have nat_rule/anchors, use STATE_DEC_COUNTERS */
 		r->states_cur--;
 		goto cleanup_state;
 	}
 
+	if (!ISSET(flags, PFSYNC_SI_IOCTL)) {
+		CLR(st->state_flags, PFSTATE_NOSYNC);
+		if (ISSET(st->state_flags, PFSTATE_ACK)) {
+			pfsync_q_ins(st, PFSYNC_S_IACK);
+			schednetisr(NETISR_PFSYNC);
+		}
+	}
+	CLR(st->state_flags, PFSTATE_ACK);
+
 	return (0);
 
  cleanup:
@@ -478,31 +603,33 @@ pfsync_input(struct mbuf *m, ...)
 	struct pfsync_pkt pkt;
 	struct ip *ip = mtod(m, struct ip *);
 	struct pfsync_header *ph;
+	struct pfsync_subheader subh;
 
 	int offset;
-	int action, count;
 	int rv;
 
 	pfsyncstats.pfsyncs_ipackets++;
 
 	/* verify that we have a sync interface configured */
-	if (!sc || !sc->sc_sync_ifp || !pf_status.running)
+	if (!sc || !sc->sc_sync_if || !pf_status.running)
 		goto done;
 
 	/* verify that the packet came in on the right interface */
-	if (sc->sc_sync_ifp != m->m_pkthdr.rcvif) {
+	if (sc->sc_sync_if != m->m_pkthdr.rcvif) {
 		pfsyncstats.pfsyncs_badif++;
 		goto done;
 	}
 
-	/* verify that the IP TTL is 255.  */
+	sc->sc_if.if_ipackets++;
+	sc->sc_if.if_ibytes += m->m_pkthdr.len;
+
+	/* verify that the IP TTL is 255. */
 	if (ip->ip_ttl != PFSYNC_DFLTTL) {
 		pfsyncstats.pfsyncs_badttl++;
 		goto done;
 	}
 
 	offset = ip->ip_hl << 2;
-
 	if (m->m_pkthdr.len < offset + sizeof(*ph)) {
 		pfsyncstats.pfsyncs_hdrops++;
 		goto done;
@@ -523,103 +650,121 @@ pfsync_input(struct mbuf *m, ...)
 		goto done;
 	}
 
-	action = ph->action;
-	count = ph->count;
-
-	/* make sure it's a valid action code */
-	if (action >= PFSYNC_ACT_MAX) {
-		pfsyncstats.pfsyncs_badact++;
+#if 0
+	if (pfsync_input_hmac(m, offset) != 0) {
+		/* XXX stats */
 		goto done;
 	}
+#endif
 
 	/* Cheaper to grab this now than having to mess with mbufs later */
 	pkt.ip = ip;
 	pkt.src = ip->ip_src;
 	pkt.flags = 0;
 
-	if (!bcmp(&ph->pf_chksum, &pf_status.pf_chksum, PF_MD5_DIGEST_LENGTH))
+	if (!bcmp(&ph->pfcksum, &pf_status.pf_chksum, PF_MD5_DIGEST_LENGTH))
 		pkt.flags |= PFSYNC_SI_CKSUM;
 
 	offset += sizeof(*ph);
-	rv = (*pfsync4_acts[action])(&pkt, m, offset, count);
-	if (rv == -1)
-		return;
+	for (;;) {
+		m_copydata(m, offset, sizeof(subh), (caddr_t)&subh);
+		offset += sizeof(subh);
+
+		if (subh.action >= PFSYNC_ACT_MAX) {
+			pfsyncstats.pfsyncs_badact++;
+			goto done;
+		}
+
+		rv = (*pfsync_acts[subh.action])(&pkt, m, offset,
+		    ntohs(subh.count));
+		if (rv == -1)
+			return;
+
+		offset += rv;
+	}
 
 done:
 	m_freem(m);
 }
 
 int
-pfsync4_in_clr(struct pfsync_pkt *pkt, struct mbuf *m, int offset, int count)
+pfsync_in_clr(struct pfsync_pkt *pkt, struct mbuf *m, int offset, int count)
 {
+	struct pfsync_clr *clr;
 	struct mbuf *mp;
-	int offp;
+	int len = sizeof(*clr) * count;
+	int i, offp;
 
-	struct pfsync_state_clr *cp;
-	struct pf_state *nexts;
+	struct pf_state *st, *nexts;
 	struct pf_state_key *sk, *nextsk;
-        struct pf_state_item *si;
-	struct pf_state *st;
-	struct pfi_kif *kif;
+	struct pf_state_item *si;
 	u_int32_t creatorid;
 	int s;
 
-	mp = m_pulldown(m, offset, sizeof(*cp), &offp);
+	mp = m_pulldown(m, offset, len, &offp);
 	if (mp == NULL) {
 		pfsyncstats.pfsyncs_badlen++;
 		return (-1);
 	}
-	cp = (struct pfsync_state_clr *)(mp->m_data + offp);
+	clr = (struct pfsync_clr *)(mp->m_data + offp);
 
 	s = splsoftnet();
-	if (cp->ifname[0] == '\0') {
-		for (st = RB_MIN(pf_state_tree_id, &tree_id); st; st = nexts) {
-			nexts = RB_NEXT(pf_state_tree_id, &tree_id, st);
-			if (st->creatorid == creatorid) {
-				st->sync_flags |= PFSTATE_FROMSYNC;
-				pf_unlink_state(st);
+	for (i = 0; i < count; i++) {
+		creatorid = clr[i].creatorid;
+
+		if (clr[i].ifname[0] == '\0') {
+			for (st = RB_MIN(pf_state_tree_id, &tree_id);
+			    st; st = nexts) {
+				nexts = RB_NEXT(pf_state_tree_id, &tree_id, st);
+				if (st->creatorid == creatorid) {
+					SET(st->state_flags, PFSTATE_NOSYNC);
+					pf_unlink_state(st);
+				}
 			}
-		}
-	} else if ((kif = pfi_kif_get(cp->ifname)) != NULL) {
-		/* XXX correct? */
-		for (sk = RB_MIN(pf_state_tree, &pf_statetbl);
-		    sk; sk = nextsk) {
-			nextsk = RB_NEXT(pf_state_tree, &pf_statetbl, sk);
-			TAILQ_FOREACH(si, &sk->states, entry) {
-				if (si->s->creatorid == creatorid &&
-				    si->s->kif == kif) {
-					si->s->sync_flags |= PFSTATE_FROMSYNC;
-					pf_unlink_state(si->s);
+		} else {
+			if (pfi_kif_get(clr[i].ifname) == NULL)
+				continue;
+
+			/* XXX correct? */
+			for (sk = RB_MIN(pf_state_tree, &pf_statetbl);
+			    sk; sk = nextsk) {
+				nextsk = RB_NEXT(pf_state_tree,
+				    &pf_statetbl, sk);
+				TAILQ_FOREACH(si, &sk->states, entry) {
+					if (si->s->creatorid == creatorid) {
+						SET(si->s->state_flags,
+						    PFSTATE_NOSYNC);
+						pf_unlink_state(si->s);
+					}
 				}
 			}
 		}
 	}
 	splx(s);
 
-	return (sizeof(*cp));
+	return (len);
 }
 
 int
-pfsync4_in_ins(struct pfsync_pkt *pkt, struct mbuf *m, int offset, int count)
+pfsync_in_ins(struct pfsync_pkt *pkt, struct mbuf *m, int offset, int count)
 {
-	struct pfsync_state *sp, *spa;
-	int s;
-
 	struct mbuf *mp;
-	int len = count * sizeof(*sp);
-	int offp;
-	int i;
+	struct pfsync_state *sa, *sp;
+	int len = sizeof(*sp) * count;
+	int i, offp;
+
+	int s;
 
 	mp = m_pulldown(m, offset, len, &offp);
 	if (mp == NULL) {
 		pfsyncstats.pfsyncs_badlen++;
 		return (-1);
 	}
-	spa = (struct pfsync_state *)(mp->m_data + offp);
+	sa = (struct pfsync_state *)(mp->m_data + offp);
 
 	s = splsoftnet();
 	for (i = 0; i < count; i++) {
-		sp = &spa[i];
+		sp = &sa[i];
 
 		/* check for invalid values */
 		if (sp->timeout >= PFTM_MAX ||
@@ -628,7 +773,7 @@ pfsync4_in_ins(struct pfsync_pkt *pkt, struct mbuf *m, int offset, int count)
 		    sp->direction > PF_OUT ||
 		    (sp->af != AF_INET && sp->af != AF_INET6)) {
 			if (pf_status.debug >= PF_DEBUG_MISC) {
-				printf("pfsync_input: PFSYNC_ACT_INS: "
+				printf("pfsync_input: PFSYNC5_ACT_INS: "
 				    "invalid value\n");
 			}
 			pfsyncstats.pfsyncs_badval++;
@@ -636,7 +781,7 @@ pfsync4_in_ins(struct pfsync_pkt *pkt, struct mbuf *m, int offset, int count)
 		}
 
 		if (pfsync_state_import(sp, pkt->flags) == ENOMEM) {
-			/* drop out */
+			/* drop out, but process the rest of the actions */
 			break;
 		}
 	}
@@ -646,34 +791,106 @@ pfsync4_in_ins(struct pfsync_pkt *pkt, struct mbuf *m, int offset, int count)
 }
 
 int
-pfsync4_in_upd(struct pfsync_pkt *pkt, struct mbuf *m, int offset, int count)
+pfsync_in_iack(struct pfsync_pkt *pkt, struct mbuf *m, int offset, int count)
 {
-	struct pfsync_softc *sc = pfsyncif;
-	struct pfsync_state *sp, *spa;
+	struct pfsync_ins_ack *ia, *iaa;
 	struct pf_state_cmp id_key;
-	struct pf_state_key *sk;
 	struct pf_state *st;
-	int stale, sfail;
-	int flags;
+
+	struct mbuf *mp;
+	int len = count * sizeof(*ia);
+	int offp, i;
 	int s;
 
+	mp = m_pulldown(m, offset, len, &offp);
+	if (mp == NULL) {
+		pfsyncstats.pfsyncs_badlen++;
+		return (-1);
+	}
+	iaa = (struct pfsync_ins_ack *)(mp->m_data + offp);
+
+	s = splsoftnet();
+	for (i = 0; i < count; i++) {
+		ia = &iaa[i];
+
+		bcopy(&ia->id, &id_key.id, sizeof(id_key.id));
+		id_key.creatorid = ia->creatorid;
+
+		st = pf_find_state_byid(&id_key);
+		if (st == NULL)
+			continue;
+
+		if (ISSET(st->state_flags, PFSTATE_ACK))
+			pfsync_deferred(st, 0);
+	}
+	splx(s);
+	/*
+	 * XXX this is not yet implemented, but we know the size of the
+	 * message so we can skip it.
+	 */
+
+	return (count * sizeof(struct pfsync_ins_ack));
+}
+
+int
+pfsync_upd_tcp(struct pf_state *st, struct pfsync_state_peer *src,
+    struct pfsync_state_peer *dst)
+{
+	int sfail = 0;
+
+	/*
+	 * The state should never go backwards except
+	 * for syn-proxy states.  Neither should the
+	 * sequence window slide backwards.
+	 */
+	if (st->src.state > src->state &&
+	    (st->src.state < PF_TCPS_PROXY_SRC ||
+	    src->state >= PF_TCPS_PROXY_SRC))
+		sfail = 1;
+	else if (SEQ_GT(st->src.seqlo, ntohl(src->seqlo)))
+		sfail = 3;
+	else if (st->dst.state > dst->state) {
+		/* There might still be useful
+		 * information about the src state here,
+		 * so import that part of the update,
+		 * then "fail" so we send the updated
+		 * state back to the peer who is missing
+		 * our what we know. */
+		pf_state_peer_ntoh(src, &st->src);
+		/* XXX do anything with timeouts? */
+		sfail = 7;
+	} else if (st->dst.state >= TCPS_SYN_SENT &&
+	    SEQ_GT(st->dst.seqlo, ntohl(dst->seqlo)))
+		sfail = 4;
+
+	return (sfail);
+}
+
+int
+pfsync_in_upd(struct pfsync_pkt *pkt, struct mbuf *m, int offset, int count)
+{
+//	struct pfsync_softc *sc = pfsyncif;
+	struct pfsync_state *sa, *sp;
+	struct pf_state_cmp id_key;
+	struct pf_state_key *sk;
+	struct pf_state *st;
+	int sfail;
+
 	struct mbuf *mp;
 	int len = count * sizeof(*sp);
-	int offp;
-	int i;
+	int offp, i;
+	int s;
 
 	mp = m_pulldown(m, offset, len, &offp);
 	if (mp == NULL) {
 		pfsyncstats.pfsyncs_badlen++;
 		return (-1);
 	}
-	spa = (struct pfsync_state *)(mp->m_data + offp);
+	sa = (struct pfsync_state *)(mp->m_data + offp);
 
 	s = splsoftnet();
 	for (i = 0; i < count; i++) {
-		sp = &spa[i];
-
-		flags = PFSYNC_FLAG_STALE;
+		sp = &sa[i];
 
 		/* check for invalid values */
 		if (sp->timeout >= PFTM_MAX ||
@@ -693,40 +910,19 @@ pfsync4_in_upd(struct pfsync_pkt *pkt, struct mbuf *m, int offset, int count)
 		st = pf_find_state_byid(&id_key);
 		if (st == NULL) {
 			/* insert the update */
-			if (pfsync_state_import(sp, flags))
+			if (pfsync_state_import(sp, 0))
 				pfsyncstats.pfsyncs_badstate++;
 			continue;
 		}
-		sk = st->key[PF_SK_WIRE];	/* XXX right one? */
 
+		if (ISSET(st->state_flags, PFSTATE_ACK))
+			pfsync_deferred(st, 1);
+
+		sk = st->key[PF_SK_WIRE];	/* XXX right one? */
 		sfail = 0;
-		if (sk->proto == IPPROTO_TCP) {
-			/*
-			 * The state should never go backwards except
-			 * for syn-proxy states.  Neither should the
-			 * sequence window slide backwards.
-			 */
-			if (st->src.state > sp->src.state &&
-			    (st->src.state < PF_TCPS_PROXY_SRC ||
-			    sp->src.state >= PF_TCPS_PROXY_SRC))
-				sfail = 1;
-			else if (SEQ_GT(st->src.seqlo, ntohl(sp->src.seqlo)))
-				sfail = 3;
-			else if (st->dst.state > sp->dst.state) {
-				/* There might still be useful
-				 * information about the src state here,
-				 * so import that part of the update,
-				 * then "fail" so we send the updated
-				 * state back to the peer who is missing
-				 * our what we know. */
-				pf_state_peer_ntoh(&sp->src, &st->src);
-				/* XXX do anything with timeouts? */
-				sfail = 7;
-				flags = 0;
-			} else if (st->dst.state >= TCPS_SYN_SENT &&
-			    SEQ_GT(st->dst.seqlo, ntohl(sp->dst.seqlo)))
-				sfail = 4;
-		} else {
+		if (sk->proto == IPPROTO_TCP)
+			sfail = pfsync_upd_tcp(st, &sp->src, &sp->dst);
+		else {
 			/*
 			 * Non-TCP protocol state machine always go
 			 * forwards
@@ -736,26 +932,19 @@ pfsync4_in_upd(struct pfsync_pkt *pkt, struct mbuf *m, int offset, int count)
 			else if (st->dst.state > sp->dst.state)
 				sfail = 6;
 		}
+
 		if (sfail) {
-			if (pf_status.debug >= PF_DEBUG_MISC)
-				printf("pfsync: %s stale update "
-				    "(%d) id: %016llx "
-				    "creatorid: %08x\n",
-				    (sfail < 7 ?  "ignoring"
-				     : "partial"), sfail,
-				    betoh64(st->id),
+			if (pf_status.debug >= PF_DEBUG_MISC) {
+				printf("pfsync: %s stale update (%d)"
+				    " id: %016llx creatorid: %08x\n",
+				    (sfail < 7 ?  "ignoring" : "partial"),
+				    sfail, betoh64(st->id),
 				    ntohl(st->creatorid));
+			}
 			pfsyncstats.pfsyncs_stale++;
 
-			if (!(sp->sync_flags & PFSTATE_STALE)) {
-				/* we have a better state, send it */
-				if (sc->sc_mbuf != NULL && !stale)
-					pfsync_sendout(sc);
-				stale++;
-				if (!st->sync_flags)
-					pfsync_pack_state( PFSYNC_ACT_UPD,
-					    st, flags);
-			}
+			pfsync_update_state(st);
+			schednetisr(NETISR_PFSYNC);
 			continue;
 		}
 		pfsync_alloc_scrub_memory(&sp->dst, &st->dst);
@@ -763,161 +952,92 @@ pfsync4_in_upd(struct pfsync_pkt *pkt, struct mbuf *m, int offset, int count)
 		pf_state_peer_ntoh(&sp->dst, &st->dst);
 		st->expire = ntohl(sp->expire) + time_second;
 		st->timeout = sp->timeout;
+		st->pfsync_time = time_second;
 	}
-	if (stale && sc->sc_mbuf != NULL)
-		pfsync_sendout(sc);
 	splx(s);
 
 	return (len);
 }
 
 int
-pfsync4_in_del(struct pfsync_pkt *pkt, struct mbuf *m, int offset, int count)
+pfsync_in_upd_c(struct pfsync_pkt *pkt, struct mbuf *m, int offset, int count)
 {
-	struct pfsync_state *sp, *spa;
+//	struct pfsync_softc *sc = pfsyncif;
+	struct pfsync_upd_c *ua, *up;
+	struct pf_state_key *sk;
 	struct pf_state_cmp id_key;
 	struct pf_state *st;
-	int s;
-
-	struct mbuf *mp;
-	int len = count * sizeof(*sp);
-	int offp;
-	int i;
-
-	mp = m_pulldown(m, offset, len, &offp);
-	if (mp == NULL) {
-		pfsyncstats.pfsyncs_badlen++;
-		return (-1);
-	}
-	spa = (struct pfsync_state *)(mp->m_data + offp);
-
-	s = splsoftnet();
-	for (i = 0; i < count; i++) {
-		sp = &spa[i];
-
-		bcopy(sp->id, &id_key.id, sizeof(id_key.id));
-		id_key.creatorid = sp->creatorid;
-
-		st = pf_find_state_byid(&id_key);
-		if (st == NULL) {
-			pfsyncstats.pfsyncs_badstate++;
-			continue;
-		}
-		st->sync_flags |= PFSTATE_FROMSYNC;
-		pf_unlink_state(st);
-	}
-	splx(s);
-
-	return (len);
-}
 
-int
-pfsync4_in_upd_c(struct pfsync_pkt *pkt, struct mbuf *m, int offset, int count)
-{
-	struct pfsync_softc *sc = pfsyncif;
-	struct pfsync_state_upd *up, *upa;
-	struct pf_state_cmp id_key;
-	struct pf_state_key *sk;
-	struct pf_state *st;
-	int stale, sfail;
-	int update_requested;
-	int s;
+	int len = count * sizeof(*up);
+	int sfail;
 
 	struct mbuf *mp;
-	int len = count * sizeof(*up);
-	int offp;
-	int i;
+	int offp, i;
+	int s;
 
 	mp = m_pulldown(m, offset, len, &offp);
 	if (mp == NULL) {
 		pfsyncstats.pfsyncs_badlen++;
 		return (-1);
 	}
-	upa = (struct pfsync_state_upd *)(mp->m_data + offp);
+	ua = (struct pfsync_upd_c *)(mp->m_data + offp);
 
-	s = splnet();
+	s = splsoftnet();
 	for (i = 0; i < count; i++) {
-		up = &upa[i];
+		up = &ua[i];
 
 		/* check for invalid values */
 		if (up->timeout >= PFTM_MAX ||
 		    up->src.state > PF_TCPS_PROXY_DST ||
 		    up->dst.state > PF_TCPS_PROXY_DST) {
-			if (pf_status.debug >= PF_DEBUG_MISC)
+			if (pf_status.debug >= PF_DEBUG_MISC) {
 				printf("pfsync_input: "
 				    "PFSYNC_ACT_UPD_C: "
 				    "invalid value\n");
+			}
 			pfsyncstats.pfsyncs_badval++;
 			continue;
 		}
 
-		bcopy(up->id, &id_key.id, sizeof(id_key.id));
+		bcopy(&up->id, &id_key.id, sizeof(id_key.id));
 		id_key.creatorid = up->creatorid;
 
 		st = pf_find_state_byid(&id_key);
 		if (st == NULL) {
 			/* We don't have this state. Ask for it. */
-			switch (pfsync_request_update(up, &pkt->src)) {
-			case 0:
-				update_requested = 1;
-				break;
-			case ENOMEM:
-				break;
-			default:
-				pfsyncstats.pfsyncs_badstate++;
-				break;
-			}
+			pfsync_request_update(id_key.creatorid, id_key.id);
 			continue;
 		}
 
+		if (ISSET(st->state_flags, PFSTATE_ACK))
+			pfsync_deferred(st, 1);
+
 		sk = st->key[PF_SK_WIRE]; /* XXX right one? */
 		sfail = 0;
-		if (sk->proto == IPPROTO_TCP) {
-			/*
-			 * The state should never go backwards except
-			 * for syn-proxy states.  Neither should the
-			 * sequence window slide backwards.
-			 */
-			if (st->src.state > up->src.state &&
-			    (st->src.state < PF_TCPS_PROXY_SRC ||
-			    up->src.state >= PF_TCPS_PROXY_SRC))
-				sfail = 1;
-			else if (st->dst.state > up->dst.state)
-				sfail = 2;
-			else if (SEQ_GT(st->src.seqlo, ntohl(up->src.seqlo)))
-				sfail = 3;
-			else if (st->dst.state >= TCPS_SYN_SENT &&
-			    SEQ_GT(st->dst.seqlo, ntohl(up->dst.seqlo)))
-				sfail = 4;
-		} else {
+		if (sk->proto == IPPROTO_TCP)
+			sfail = pfsync_upd_tcp(st, &up->src, &up->dst);
+		else {
 			/*
-			 * Non-TCP protocol state machine always go
-			 * forwards
+			 * Non-TCP protocol state machine always go forwards
 			 */
 			if (st->src.state > up->src.state)
 				sfail = 5;
 			else if (st->dst.state > up->dst.state)
 				sfail = 6;
 		}
+
 		if (sfail) {
-			if (pf_status.debug >= PF_DEBUG_MISC)
+			if (pf_status.debug >= PF_DEBUG_MISC) {
 				printf("pfsync: ignoring stale update "
 				    "(%d) id: %016llx "
 				    "creatorid: %08x\n", sfail,
 				    betoh64(st->id),
 				    ntohl(st->creatorid));
+			}
 			pfsyncstats.pfsyncs_stale++;
 
-			/* we have a better state, send it out */
-			if ((!stale || update_requested) &&
-			    sc->sc_mbuf != NULL) {
-				pfsync_sendout(sc);
-				update_requested = 0;
-			}
-			stale++;
-			if (!st->sync_flags)
-				pfsync_pack_state(PFSYNC_ACT_UPD, st,
-				    PFSYNC_FLAG_STALE);
+			pfsync_update_state(st);
+			schednetisr(NETISR_PFSYNC);
 			continue;
 		}
 		pfsync_alloc_scrub_memory(&up->dst, &st->dst);
@@ -925,47 +1045,87 @@ pfsync4_in_upd_c(struct pfsync_pkt *pkt, struct mbuf *m, int offset, int count)
 		pf_state_peer_ntoh(&up->dst, &st->dst);
 		st->expire = ntohl(up->expire) + time_second;
 		st->timeout = up->timeout;
+		st->pfsync_time = time_second;
 	}
-	if ((update_requested || stale) && sc->sc_mbuf)
-		pfsync_sendout(sc);
 	splx(s);
 
 	return (len);
 }
 
+int pfsync_req_del;
+
 int
-pfsync4_in_del_c(struct pfsync_pkt *pkt, struct mbuf *m, int offset, int count)
+pfsync_in_ureq(struct pfsync_pkt *pkt, struct mbuf *m, int offset, int count)
 {
-	struct pfsync_state_del *dp, *dpa;
+	struct pfsync_upd_req *ur, *ura;
+	struct mbuf *mp;
+	int len = count * sizeof(*ur);
+	int i, offp;
+
 	struct pf_state_cmp id_key;
 	struct pf_state *st;
-	int s;
 
+	mp = m_pulldown(m, offset, len, &offp);
+	if (mp == NULL) {
+		pfsyncstats.pfsyncs_badlen++;
+		return (-1);
+	}
+	ura = (struct pfsync_upd_req *)(mp->m_data + offp);
+
+	for (i = 0; i < count; i++) {
+		ur = &ura[i];
+
+		bcopy(&ur->id, &id_key.id, sizeof(id_key.id));
+		id_key.creatorid = ur->creatorid;
+
+		if (id_key.id == 0 && id_key.creatorid == 0)
+			pfsync_bulk_start();
+		else {
+			st = pf_find_state_byid(&id_key);
+			if (st == NULL) {
+				pfsyncstats.pfsyncs_badstate++;
+				continue;
+			}
+
+			if (st->timeout == PFTM_UNLINKED)
+				pfsync_req_del++;
+
+			pfsync_update_state_req(st);
+		}
+	}
+
+	return (len);
+}
+
+int
+pfsync_in_del(struct pfsync_pkt *pkt, struct mbuf *m, int offset, int count)
+{
 	struct mbuf *mp;
-	int len = count * sizeof(*dp);
-	int offp;
-	int i;
+	struct pfsync_state *sa, *sp;
+	struct pf_state_cmp id_key;
+	struct pf_state *st;
+	int len = count * sizeof(*sp);
+	int offp, i;
+	int s;
 
 	mp = m_pulldown(m, offset, len, &offp);
 	if (mp == NULL) {
 		pfsyncstats.pfsyncs_badlen++;
 		return (-1);
 	}
-	dpa = (struct pfsync_state_del *)(mp->m_data + offp);
+	sa = (struct pfsync_state *)(mp->m_data + offp);
 
 	s = splsoftnet();
 	for (i = 0; i < count; i++) {
-		dp = &dpa[i];
-
-		bcopy(dp->id, &id_key.id, sizeof(id_key.id));
-		id_key.creatorid = dp->creatorid;
+		bcopy(sp->id, &id_key.id, sizeof(id_key.id));
+		id_key.creatorid = sp->creatorid;
 
 		st = pf_find_state_byid(&id_key);
 		if (st == NULL) {
 			pfsyncstats.pfsyncs_badstate++;
 			continue;
 		}
-		st->sync_flags |= PFSTATE_FROMSYNC;
+		SET(st->state_flags, PFSTATE_NOSYNC);
 		pf_unlink_state(st);
 	}
 	splx(s);
@@ -974,101 +1134,75 @@ pfsync4_in_del_c(struct pfsync_pkt *pkt, struct mbuf *m, int offset, int count)
 }
 
 int
-pfsync4_in_error(struct pfsync_pkt *pkt, struct mbuf *m, int offset, int count)
-{
-	m_freem(m);
-	return (-1);
-}
-
-int
-pfsync4_in_ureq(struct pfsync_pkt *pkt, struct mbuf *m, int offset, int count)
+pfsync_in_del_c(struct pfsync_pkt *pkt, struct mbuf *m, int offset, int count)
 {
-	struct pfsync_softc *sc = pfsyncif;
-	struct pfsync_state_upd_req *rup, *rupa;
+	struct mbuf *mp;
+	struct pfsync_del_c *sa, *sp;
 	struct pf_state_cmp id_key;
 	struct pf_state *st;
+	int len = count * sizeof(*sp);
+	int offp, i;
 	int s;
 
-	struct mbuf *mp;
-	int len = count * sizeof(*rup);
-	int offp;
-	int i;
-
 	mp = m_pulldown(m, offset, len, &offp);
 	if (mp == NULL) {
 		pfsyncstats.pfsyncs_badlen++;
 		return (-1);
 	}
-	rupa = (struct pfsync_state_upd_req *)(mp->m_data + offp);
+	sa = (struct pfsync_del_c *)(mp->m_data + offp);
 
 	s = splsoftnet();
-	if (sc->sc_mbuf != NULL)
-		pfsync_sendout(sc);
-
 	for (i = 0; i < count; i++) {
-		    rup = &rupa[i];
+		sp = &sa[i];
 
-		bcopy(rup->id, &id_key.id, sizeof(id_key.id));
-		id_key.creatorid = rup->creatorid;
+		bcopy(&sp->id, &id_key.id, sizeof(id_key.id));
+		id_key.creatorid = sp->creatorid;
 
-		if (id_key.id == 0 && id_key.creatorid == 0) {
-			sc->sc_ureq_received = time_uptime;
-			if (sc->sc_bulk_send_next == NULL)
-				sc->sc_bulk_send_next =
-				    TAILQ_FIRST(&state_list);
-			sc->sc_bulk_terminator = sc->sc_bulk_send_next;
-			if (pf_status.debug >= PF_DEBUG_MISC)
-				printf("pfsync: received "
-				    "bulk update request\n");
-			pfsync_send_bus(sc, PFSYNC_BUS_START);
-			timeout_add_sec(&sc->sc_bulk_tmo, 1);
-		} else {
-			st = pf_find_state_byid(&id_key);
-			if (st == NULL) {
-				pfsyncstats.pfsyncs_badstate++;
-				continue;
-			}
-			if (!st->sync_flags)
-				pfsync_pack_state(PFSYNC_ACT_UPD, st, 0);
+		st = pf_find_state_byid(&id_key);
+		if (st == NULL) {
+			pfsyncstats.pfsyncs_badstate++;
+			continue;
 		}
-	}
 
-	if (sc->sc_mbuf != NULL)
-		pfsync_sendout(sc);
+		SET(st->state_flags, PFSTATE_NOSYNC);
+		pf_unlink_state(st);
+	}
 	splx(s);
 
 	return (len);
 }
 
 int
-pfsync4_in_bus(struct pfsync_pkt *pkt, struct mbuf *m, int offset, int count)
+pfsync_in_bus(struct pfsync_pkt *pkt, struct mbuf *m, int offset, int count)
 {
 	struct pfsync_softc *sc = pfsyncif;
-	struct pfsync_state_bus *bus;
-
+	struct pfsync_bus *bus;
 	struct mbuf *mp;
+	int len = count * sizeof(*bus);
 	int offp;
 
 	/* If we're not waiting for a bulk update, who cares. */
 	if (sc->sc_ureq_sent == 0)
-		return (sizeof(*bus));
+		return (len);
 
-	mp = m_pulldown(m, offset, sizeof(*bus), &offp);
+	mp = m_pulldown(m, offset, len, &offp);
 	if (mp == NULL) {
 		pfsyncstats.pfsyncs_badlen++;
 		return (-1);
 	}
-	bus = (struct pfsync_state_bus *)(mp->m_data + offp);
+	bus = (struct pfsync_bus *)(mp->m_data + offp);
 
 	switch (bus->status) {
 	case PFSYNC_BUS_START:
-		timeout_add(&sc->sc_bulkfail_tmo,
+		timeout_add_sec(&sc->sc_bulkfail_tmo, 5); /* XXX magic */
+#if XXX
 		    pf_pool_limits[PF_LIMIT_STATES].limit /
 		    (PFSYNC_BULKPACKETS * sc->sc_maxcount));
+#endif
 		if (pf_status.debug >= PF_DEBUG_MISC)
-			printf("pfsync: received bulk "
-			    "update start\n");
+			printf("pfsync: received bulk update start\n");
 		break;
+
 	case PFSYNC_BUS_END:
 		if (time_uptime - ntohl(bus->endtime) >=
 		    sc->sc_ureq_sent) {
@@ -1092,33 +1226,31 @@ pfsync4_in_bus(struct pfsync_pkt *pkt, struct mbuf *m, int offset, int count)
 		break;
 	}
 
-	return (sizeof(*bus));
+	return (len);
 }
 
 int
-pfsync4_in_tdb_upd(struct pfsync_pkt *pkt, struct mbuf *m, int offset,
-    int count)
+pfsync_in_tdb(struct pfsync_pkt *pkt, struct mbuf *m, int offset, int count)
 {
-	struct pfsync_tdb *pt;
-	int len = count * sizeof(*pt);
-
-#ifdef IPSEC
-	int s;
+	int len = count * sizeof(struct pfsync_tdb);
 
+#if 0 && defined(IPSEC)
+	struct pfsync_tdb *tp;
 	struct mbuf *mp;
 	int offp;
 	int i;
+	int s;
 
 	mp = m_pulldown(m, offset, len, &offp);
 	if (mp == NULL) {
 		pfsyncstats.pfsyncs_badlen++;
 		return (-1);
 	}
-	pt = (struct pfsync_tdb *)(mp->m_data + offp);
+	tp = (struct pfsync_tdb *)(mp->m_data + offp);
 
 	s = splsoftnet();
 	for (i = 0; i < count; i++)
-		pfsync_update_net_tdb(&pt[i]);
+		pfsync_update_net_tdb(&tp[i]); /* XXX */
 	splx(s);
 #endif
 
@@ -1126,6 +1258,27 @@ pfsync4_in_tdb_upd(struct pfsync_pkt *pkt, struct mbuf *m, int offset,
 }
 
 int
+pfsync_in_eof(struct pfsync_pkt *pkt, struct mbuf *m, int offset, int count)
+{
+	/* check if we are at the right place in the packet */
+	if (offset != m->m_pkthdr.len - sizeof(struct pfsync_eof))
+		pfsyncstats.pfsyncs_badact++;
+
+	/* we're done. free and let the caller return */
+	m_freem(m);
+	return (-1);
+}
+
+int
+pfsync_in_error(struct pfsync_pkt *pkt, struct mbuf *m, int offset, int count)
+{
+	pfsyncstats.pfsyncs_badact++;
+
+	m_freem(m);
+	return (-1);
+}
+
+int
 pfsyncoutput(struct ifnet *ifp, struct mbuf *m, struct sockaddr *dst,
 	struct rtentry *rt)
 {
@@ -1143,12 +1296,15 @@ pfsyncioctl(struct ifnet *ifp, u_long cmd, caddr_t data)
 	struct ip_moptions *imo = &sc->sc_imo;
 	struct pfsyncreq pfsyncr;
 	struct ifnet    *sifp;
+	struct ip *ip;
 	int s, error;
 
 	switch (cmd) {
+#if 0
 	case SIOCSIFADDR:
 	case SIOCAIFADDR:
 	case SIOCSIFDSTADDR:
+#endif
 	case SIOCSIFFLAGS:
 		if (ifp->if_flags & IFF_UP)
 			ifp->if_flags |= IFF_RUNNING;
@@ -1156,26 +1312,27 @@ pfsyncioctl(struct ifnet *ifp, u_long cmd, caddr_t data)
 			ifp->if_flags &= ~IFF_RUNNING;
 		break;
 	case SIOCSIFMTU:
-		if (ifr->ifr_mtu < PFSYNC_MINMTU)
+		if (ifr->ifr_mtu <= PFSYNC_MINPKT)
 			return (EINVAL);
-		if (ifr->ifr_mtu > MCLBYTES)
+		if (ifr->ifr_mtu > MCLBYTES) /* XXX could be bigger */
 			ifr->ifr_mtu = MCLBYTES;
-		s = splnet();
-		if (ifr->ifr_mtu < ifp->if_mtu)
-			pfsync_sendout(sc);
-		pfsync_setmtu(sc, ifr->ifr_mtu);
-		splx(s);
+		if (ifr->ifr_mtu < ifp->if_mtu) {
+			s = splnet();
+			pfsync_sendout();
+			splx(s);
+		}
+		ifp->if_mtu = ifr->ifr_mtu;
 		break;
 	case SIOCGETPFSYNC:
 		bzero(&pfsyncr, sizeof(pfsyncr));
-		if (sc->sc_sync_ifp)
+		if (sc->sc_sync_if) {
 			strlcpy(pfsyncr.pfsyncr_syncdev,
-			    sc->sc_sync_ifp->if_xname, IFNAMSIZ);
+			    sc->sc_sync_if->if_xname, IFNAMSIZ);
+		}
 		pfsyncr.pfsyncr_syncpeer = sc->sc_sync_peer;
 		pfsyncr.pfsyncr_maxupdates = sc->sc_maxupdates;
-		if ((error = copyout(&pfsyncr, ifr->ifr_data, sizeof(pfsyncr))))
-			return (error);
-		break;
+		return (copyout(&pfsyncr, ifr->ifr_data, sizeof(pfsyncr)));
+
 	case SIOCSETPFSYNC:
 		if ((error = suser(p, p->p_acflag)) != 0)
 			return (error);
@@ -1193,17 +1350,10 @@ pfsyncioctl(struct ifnet *ifp, u_long cmd, caddr_t data)
 		sc->sc_maxupdates = pfsyncr.pfsyncr_maxupdates;
 
 		if (pfsyncr.pfsyncr_syncdev[0] == 0) {
-			sc->sc_sync_ifp = NULL;
-			if (sc->sc_mbuf_net != NULL) {
-				/* Don't keep stale pfsync packets around. */
-				s = splnet();
-				m_freem(sc->sc_mbuf_net);
-				sc->sc_mbuf_net = NULL;
-				sc->sc_statep_net.s = NULL;
-				splx(s);
-			}
+			sc->sc_sync_if = NULL;
 			if (imo->imo_num_memberships > 0) {
-				in_delmulti(imo->imo_membership[--imo->imo_num_memberships]);
+				in_delmulti(imo->imo_membership[
+				    --imo->imo_num_memberships]);
 				imo->imo_multicast_ifp = NULL;
 			}
 			break;
@@ -1214,25 +1364,23 @@ pfsyncioctl(struct ifnet *ifp, u_long cmd, caddr_t data)
 
 		s = splnet();
 		if (sifp->if_mtu < sc->sc_if.if_mtu ||
-		    (sc->sc_sync_ifp != NULL &&
-		    sifp->if_mtu < sc->sc_sync_ifp->if_mtu) ||
+		    (sc->sc_sync_if != NULL &&
+		    sifp->if_mtu < sc->sc_sync_if->if_mtu) ||
 		    sifp->if_mtu < MCLBYTES - sizeof(struct ip))
-			pfsync_sendout(sc);
-		sc->sc_sync_ifp = sifp;
-
-		pfsync_setmtu(sc, sc->sc_if.if_mtu);
+			pfsync_sendout();
+		sc->sc_sync_if = sifp;
 
 		if (imo->imo_num_memberships > 0) {
 			in_delmulti(imo->imo_membership[--imo->imo_num_memberships]);
 			imo->imo_multicast_ifp = NULL;
 		}
 
-		if (sc->sc_sync_ifp &&
+		if (sc->sc_sync_if &&
 		    sc->sc_sync_peer.s_addr == INADDR_PFSYNC_GROUP) {
 			struct in_addr addr;
 
-			if (!(sc->sc_sync_ifp->if_flags & IFF_MULTICAST)) {
-				sc->sc_sync_ifp = NULL;
+			if (!(sc->sc_sync_if->if_flags & IFF_MULTICAST)) {
+				sc->sc_sync_if = NULL;
 				splx(s);
 				return (EADDRNOTAVAIL);
 			}
@@ -1240,19 +1388,31 @@ pfsyncioctl(struct ifnet *ifp, u_long cmd, caddr_t data)
 			addr.s_addr = INADDR_PFSYNC_GROUP;
 
 			if ((imo->imo_membership[0] =
-			    in_addmulti(&addr, sc->sc_sync_ifp)) == NULL) {
-				sc->sc_sync_ifp = NULL;
+			    in_addmulti(&addr, sc->sc_sync_if)) == NULL) {
+				sc->sc_sync_if = NULL;
 				splx(s);
 				return (ENOBUFS);
 			}
 			imo->imo_num_memberships++;
-			imo->imo_multicast_ifp = sc->sc_sync_ifp;
+			imo->imo_multicast_ifp = sc->sc_sync_if;
 			imo->imo_multicast_ttl = PFSYNC_DFLTTL;
 			imo->imo_multicast_loop = 0;
 		}
 
-		if (sc->sc_sync_ifp ||
-		    sc->sc_sendaddr.s_addr != INADDR_PFSYNC_GROUP) {
+		ip = &sc->sc_template;
+		bzero(ip, sizeof(*ip));
+		ip->ip_v = IPVERSION;
+		ip->ip_hl = sizeof(sc->sc_template) >> 2;
+		ip->ip_tos = IPTOS_LOWDELAY;
+		/* len and id are set later */
+		ip->ip_off = htons(IP_DF);
+		ip->ip_ttl = PFSYNC_DFLTTL;
+		ip->ip_p = IPPROTO_PFSYNC;
+		ip->ip_src.s_addr = INADDR_ANY;
+		ip->ip_dst.s_addr = sc->sc_sync_peer.s_addr;
+
+		if (sc->sc_sync_if) {
+#if 0
 			/* Request a full state table update. */
 			sc->sc_ureq_sent = time_uptime;
 #if NCARP > 0
@@ -1263,12 +1423,9 @@ pfsyncioctl(struct ifnet *ifp, u_long cmd, caddr_t data)
 			if (pf_status.debug >= PF_DEBUG_MISC)
 				printf("pfsync: requesting bulk update\n");
 			timeout_add_sec(&sc->sc_bulkfail_tmo, 5);
-			error = pfsync_request_update(NULL, NULL);
-			if (error == ENOMEM) {
-				splx(s);
-				return (ENOMEM);
-			}
-			pfsync_sendout(sc);
+			/* XXX bulks done this way? */
+			pfsync_request_update(0, 0);
+#endif
 		}
 		splx(s);
 
@@ -1281,691 +1438,686 @@ pfsyncioctl(struct ifnet *ifp, u_long cmd, caddr_t data)
 	return (0);
 }
 
-void
-pfsync_setmtu(struct pfsync_softc *sc, int mtu_req)
+int
+pfsync_out_state(struct pf_state *st, struct mbuf *m, int offset)
+{
+	struct pfsync_state *sp = (struct pfsync_state *)(m->m_data + offset);
+
+	pfsync_state_export(sp, st);
+
+	return (sizeof(*sp));
+}
+
+int
+pfsync_out_iack(struct pf_state *st, struct mbuf *m, int offset)
 {
-	int mtu;
+	struct pfsync_ins_ack *iack =
+	    (struct pfsync_ins_ack *)(m->m_data + offset);
+
+	iack->id = st->id;
+	iack->creatorid = st->creatorid;
 
-	if (sc->sc_sync_ifp && sc->sc_sync_ifp->if_mtu < mtu_req)
-		mtu = sc->sc_sync_ifp->if_mtu;
+	return (sizeof(*iack));
+}
+
+int
+pfsync_out_upd_c(struct pf_state *st, struct mbuf *m, int offset)
+{
+	struct pfsync_upd_c *up = (struct pfsync_upd_c *)(m->m_data + offset);
+
+	up->id = st->id;
+	pf_state_peer_hton(&st->src, &up->src);
+	pf_state_peer_hton(&st->dst, &up->dst);
+	up->creatorid = st->creatorid;
+
+	up->expire = pf_state_expires(st);
+	if (up->expire <= time_second)
+		up->expire = htonl(0);
 	else
-		mtu = mtu_req;
-
-	sc->sc_maxcount = (mtu - sizeof(struct pfsync_header)) /
-	    sizeof(struct pfsync_state);
-	if (sc->sc_maxcount > 254)
-	    sc->sc_maxcount = 254;
-	sc->sc_if.if_mtu = sizeof(struct pfsync_header) +
-	    sc->sc_maxcount * sizeof(struct pfsync_state);
+		up->expire = htonl(up->expire - time_second);
+	up->timeout = st->timeout;
+
+	bzero(up->_pad, sizeof(up->_pad)); /* XXX */
+
+	return (sizeof(*up));
 }
 
-struct mbuf *
-pfsync_get_mbuf(struct pfsync_softc *sc, u_int8_t action, void **sp)
+int
+pfsync_out_del(struct pf_state *st, struct mbuf *m, int offset)
 {
-	struct pfsync_header *h;
-	struct mbuf *m;
-	int len;
+	struct pfsync_del_c *dp = (struct pfsync_del_c *)(m->m_data + offset);
 
-	MGETHDR(m, M_DONTWAIT, MT_DATA);
-	if (m == NULL) {
-		sc->sc_if.if_oerrors++;
-		return (NULL);
+	dp->id = st->id;
+	dp->creatorid = st->creatorid;
+
+	SET(st->state_flags, PFSTATE_NOSYNC);
+
+	return (sizeof(*dp));
+}
+
+void
+pfsync_drop(struct pfsync_softc *sc)
+{
+	struct pf_state *st;
+	struct pfsync_upd_req_item *ur;
+	struct tdb *t;
+	int q;
+
+	for (q = 0; q < PFSYNC_S_COUNT; q++) {
+		if (TAILQ_EMPTY(&sc->sc_qs[q]))
+			continue;
+
+		TAILQ_FOREACH(st, &sc->sc_qs[q], sync_list) {
+#ifdef PFSYNC_DEBUG
+			KASSERT(st->sync_state == q);
+#endif
+			st->sync_state = PFSYNC_S_NONE;
+		}
+		TAILQ_INIT(&sc->sc_qs[q]);
 	}
 
-	switch (action) {
-	case PFSYNC_ACT_CLR:
-		len = sizeof(struct pfsync_header) +
-		    sizeof(struct pfsync_state_clr);
-		break;
-	case PFSYNC_ACT_UPD_C:
-		len = (sc->sc_maxcount * sizeof(struct pfsync_state_upd)) +
-		    sizeof(struct pfsync_header);
-		break;
-	case PFSYNC_ACT_DEL_C:
-		len = (sc->sc_maxcount * sizeof(struct pfsync_state_del)) +
-		    sizeof(struct pfsync_header);
-		break;
-	case PFSYNC_ACT_UREQ:
-		len = (sc->sc_maxcount * sizeof(struct pfsync_state_upd_req)) +
-		    sizeof(struct pfsync_header);
-		break;
-	case PFSYNC_ACT_BUS:
-		len = sizeof(struct pfsync_header) +
-		    sizeof(struct pfsync_state_bus);
-		break;
-	case PFSYNC_ACT_TDB_UPD:
-		len = (sc->sc_maxcount * sizeof(struct pfsync_tdb)) +
-		    sizeof(struct pfsync_header);
-		break;
-	default:
-		len = (sc->sc_maxcount * sizeof(struct pfsync_state)) +
-		    sizeof(struct pfsync_header);
-		break;
+	while ((ur = TAILQ_FIRST(&sc->sc_upd_req_list)) != NULL) {
+		TAILQ_REMOVE(&sc->sc_upd_req_list, ur, ur_entry);
+		pool_put(&sc->sc_pool, ur);
 	}
 
-	if (len > MHLEN) {
-		MCLGET(m, M_DONTWAIT);
-		if ((m->m_flags & M_EXT) == 0) {
-			m_free(m);
-			sc->sc_if.if_oerrors++;
-			return (NULL);
-		}
-		m->m_data += (MCLBYTES - len) &~ (sizeof(long) - 1);
-	} else
-		MH_ALIGN(m, len);
-
-	m->m_pkthdr.rcvif = NULL;
-	m->m_pkthdr.len = m->m_len = sizeof(struct pfsync_header);
-	h = mtod(m, struct pfsync_header *);
-	h->version = PFSYNC_VERSION;
-	h->af = 0;
-	h->count = 0;
-	h->action = action;
-	if (action != PFSYNC_ACT_TDB_UPD)
-		bcopy(&pf_status.pf_chksum, &h->pf_chksum,
-		    PF_MD5_DIGEST_LENGTH);
-
-	*sp = (void *)((char *)h + PFSYNC_HDRLEN);
-	if (action == PFSYNC_ACT_TDB_UPD)
-		timeout_add_sec(&sc->sc_tdb_tmo, 1);
-	else
-		timeout_add_sec(&sc->sc_tmo, 1);
-	return (m);
+	sc->sc_plus = NULL;
+
+	if (!TAILQ_EMPTY(&sc->sc_tdb_q)) {
+		TAILQ_FOREACH(t, &sc->sc_tdb_q, tdb_sync_entry)
+			CLR(t->tdb_flags, TDBF_PFSYNC);
+
+		TAILQ_INIT(&sc->sc_tdb_q);
+	}
+
+	sc->sc_len = PFSYNC_MINPKT;
 }
 
-int
-pfsync_pack_state(u_int8_t action, struct pf_state *st, int flags)
+void
+pfsync_sendout(void)
 {
-	struct ifnet *ifp = NULL;
 	struct pfsync_softc *sc = pfsyncif;
-	struct pfsync_header *h, *h_net;
-	struct pfsync_state *sp = NULL;
-	struct pfsync_state_upd *up = NULL;
-	struct pfsync_state_del *dp = NULL;
-	int s, ret = 0;
-	u_int8_t i = 255, newaction = 0;
+#if NBPFILTER > 0
+	struct ifnet *ifp = &sc->sc_if;
+#endif
+	struct mbuf *m;
+	struct ip *ip;
+	struct pfsync_header *ph;
+	struct pfsync_subheader *subh;
+	struct pf_state *st;
+	struct pfsync_upd_req_item *ur;
+	struct tdb *t;
 
-	if (sc == NULL)
-		return (0);
-	ifp = &sc->sc_if;
+	int offset;
+	int q, count = 0;
 
-	/*
-	 * If a packet falls in the forest and there's nobody around to
-	 * hear, does it make a sound?
-	 */
-	if (ifp->if_bpf == NULL && sc->sc_sync_ifp == NULL &&
-	    sc->sc_sync_peer.s_addr == INADDR_PFSYNC_GROUP) {
-		/* Don't leave any stale pfsync packets hanging around. */
-		if (sc->sc_mbuf != NULL) {
-			m_freem(sc->sc_mbuf);
-			sc->sc_mbuf = NULL;
-			sc->sc_statep.s = NULL;
+	splassert(IPL_NET);
+
+	if (sc == NULL || sc->sc_len == PFSYNC_MINPKT)
+		return;
+
+	MGETHDR(m, M_DONTWAIT, MT_DATA);
+	if (m == NULL) {
+		sc->sc_if.if_oerrors++;
+		pfsyncstats.pfsyncs_onomem++;
+		pfsync_drop(sc);
+		return;
+	}
+
+	if (max_linkhdr + sc->sc_len > MHLEN) {
+		MCLGETI(m, M_DONTWAIT, NULL, max_linkhdr + sc->sc_len);
+		if (!ISSET(m->m_flags, M_EXT)) {
+			m_free(m);
+			sc->sc_if.if_oerrors++;
+			pfsyncstats.pfsyncs_onomem++;
+			pfsync_drop(sc);
+			return;
 		}
-		return (0);
 	}
+	m->m_data += max_linkhdr;
+	m->m_len = m->m_pkthdr.len = sc->sc_len;
 
-	if (action >= PFSYNC_ACT_MAX)
-		return (EINVAL);
+	/* build the ip header */
+	ip = (struct ip *)m->m_data;
+	bcopy(&sc->sc_template, ip, sizeof(*ip));
+	offset = sizeof(*ip);
 
-	s = splnet();
-	if (sc->sc_mbuf == NULL) {
-		if ((sc->sc_mbuf = pfsync_get_mbuf(sc, action,
-		    (void *)&sc->sc_statep.s)) == NULL) {
-			splx(s);
-			return (ENOMEM);
-		}
-		h = mtod(sc->sc_mbuf, struct pfsync_header *);
-	} else {
-		h = mtod(sc->sc_mbuf, struct pfsync_header *);
-		if (h->action != action) {
-			pfsync_sendout(sc);
-			if ((sc->sc_mbuf = pfsync_get_mbuf(sc, action,
-			    (void *)&sc->sc_statep.s)) == NULL) {
-				splx(s);
-				return (ENOMEM);
-			}
-			h = mtod(sc->sc_mbuf, struct pfsync_header *);
-		} else {
-			/*
-			 * If it's an update, look in the packet to see if
-			 * we already have an update for the state.
-			 */
-			if (action == PFSYNC_ACT_UPD && sc->sc_maxupdates) {
-				struct pfsync_state *usp =
-				    (void *)((char *)h + PFSYNC_HDRLEN);
-
-				for (i = 0; i < h->count; i++) {
-					if (!memcmp(usp->id, &st->id,
-					    PFSYNC_ID_LEN) &&
-					    usp->creatorid == st->creatorid) {
-						sp = usp;
-						sp->updates++;
-						break;
-					}
-					usp++;
-				}
-			}
+	ip->ip_len = htons(m->m_pkthdr.len);
+	ip->ip_id = htons(ip_randomid());
+
+	/* build the pfsync header */
+	ph = (struct pfsync_header *)(m->m_data + offset);
+	bzero(ph, sizeof(*ph));
+	offset += sizeof(*ph);
+
+	ph->version = PFSYNC_VERSION;
+	ph->len = htons(sc->sc_len - sizeof(*ip));
+	bcopy(pf_status.pf_chksum, ph->pfcksum, PF_MD5_DIGEST_LENGTH);
+
+	/* walk the queues */
+	for (q = 0; q < PFSYNC_S_COUNT; q++) {
+		if (TAILQ_EMPTY(&sc->sc_qs[q]))
+			continue;
+
+		subh = (struct pfsync_subheader *)(m->m_data + offset);
+		offset += sizeof(*subh);
+
+		count = 0;
+		TAILQ_FOREACH(st, &sc->sc_qs[q], sync_list) {
+#ifdef PFSYNC_DEBUG
+			KASSERT(st->sync_state == q);
+#endif
+
+			offset += pfsync_qs[q].write(st, m, offset);
+			st->sync_state = PFSYNC_S_NONE;
+			count++;
 		}
+		TAILQ_INIT(&sc->sc_qs[q]);
+
+		bzero(subh, sizeof(*subh));
+		subh->action = pfsync_qs[q].action;
+		subh->count = htons(count);
 	}
 
-	st->pfsync_time = time_uptime;
+	if (!TAILQ_EMPTY(&sc->sc_upd_req_list)) {
+		subh = (struct pfsync_subheader *)(m->m_data + offset);
+		offset += sizeof(*subh);
 
-	if (sp == NULL) {
-		/* not a "duplicate" update */
-		i = 255;
-		sp = sc->sc_statep.s++;
-		sc->sc_mbuf->m_pkthdr.len =
-		    sc->sc_mbuf->m_len += sizeof(struct pfsync_state);
-		h->count++;
-		bzero(sp, sizeof(*sp));
+		count = 0;
+		while ((ur = TAILQ_FIRST(&sc->sc_upd_req_list)) != NULL) {
+			TAILQ_REMOVE(&sc->sc_upd_req_list, ur, ur_entry);
 
-		pfsync_state_export(sp, st);
+			bcopy(&ur->ur_msg, m->m_data + offset,
+			    sizeof(ur->ur_msg));
+			offset += sizeof(ur->ur_msg);
 
-		if (flags & PFSYNC_FLAG_STALE)
-			sp->sync_flags |= PFSTATE_STALE;
-	} else {
-		pf_state_peer_hton(&st->src, &sp->src);
-		pf_state_peer_hton(&st->dst, &sp->dst);
+			pool_put(&sc->sc_pool, ur);
 
-		if (st->expire <= time_second)
-			sp->expire = htonl(0);
-		else
-			sp->expire = htonl(st->expire - time_second);
+			count++;
+		}
+
+		bzero(subh, sizeof(*subh));
+		subh->action = PFSYNC_ACT_UPD_REQ;
+		subh->count = htons(count);
 	}
 
-	/* do we need to build "compressed" actions for network transfer? */
-	if (sc->sc_sync_ifp && flags & PFSYNC_FLAG_COMPRESS) {
-		switch (action) {
-		case PFSYNC_ACT_UPD:
-			newaction = PFSYNC_ACT_UPD_C;
-			break;
-		case PFSYNC_ACT_DEL:
-			newaction = PFSYNC_ACT_DEL_C;
-			break;
-		default:
-			/* by default we just send the uncompressed states */
-			break;
-		}
+	/* has someone built a custom region for us to add? */
+	if (sc->sc_plus != NULL) {
+		bcopy(sc->sc_plus, m->m_data + offset, sc->sc_pluslen);
+		offset += sc->sc_pluslen;
+
+		sc->sc_plus = NULL;
 	}
 
-	if (newaction) {
-		if (sc->sc_mbuf_net == NULL) {
-			if ((sc->sc_mbuf_net = pfsync_get_mbuf(sc, newaction,
-			    (void *)&sc->sc_statep_net.s)) == NULL) {
-				splx(s);
-				return (ENOMEM);
-			}
-		}
-		h_net = mtod(sc->sc_mbuf_net, struct pfsync_header *);
-
-		switch (newaction) {
-		case PFSYNC_ACT_UPD_C:
-			if (i != 255) {
-				up = (void *)((char *)h_net +
-				    PFSYNC_HDRLEN + (i * sizeof(*up)));
-				up->updates++;
-			} else {
-				h_net->count++;
-				sc->sc_mbuf_net->m_pkthdr.len =
-				    sc->sc_mbuf_net->m_len += sizeof(*up);
-				up = sc->sc_statep_net.u++;
-
-				bzero(up, sizeof(*up));
-				bcopy(&st->id, up->id, sizeof(up->id));
-				up->creatorid = st->creatorid;
-			}
-			up->timeout = st->timeout;
-			up->expire = sp->expire;
-			up->src = sp->src;
-			up->dst = sp->dst;
-			break;
-		case PFSYNC_ACT_DEL_C:
-			sc->sc_mbuf_net->m_pkthdr.len =
-			    sc->sc_mbuf_net->m_len += sizeof(*dp);
-			dp = sc->sc_statep_net.d++;
-			h_net->count++;
-
-			bzero(dp, sizeof(*dp));
-			bcopy(&st->id, dp->id, sizeof(dp->id));
-			dp->creatorid = st->creatorid;
-			break;
+	if (!TAILQ_EMPTY(&sc->sc_tdb_q)) {
+		subh = (struct pfsync_subheader *)(m->m_data + offset);
+		offset += sizeof(*subh);
+
+		count = 0;
+		TAILQ_FOREACH(t, &sc->sc_tdb_q, tdb_sync_entry) {
+			offset += pfsync_out_tdb(t, m, offset);
+			CLR(t->tdb_flags, TDBF_PFSYNC);
+
+			count++;
 		}
+		TAILQ_INIT(&sc->sc_tdb_q);
+
+		bzero(subh, sizeof(*subh));
+		subh->action = PFSYNC_ACT_TDB;
+		subh->count = htons(count);
 	}
 
-	if (h->count == sc->sc_maxcount ||
-	    (sc->sc_maxupdates && (sp->updates >= sc->sc_maxupdates)))
-		ret = pfsync_sendout(sc);
+	subh = (struct pfsync_subheader *)(m->m_data + offset);
+	offset += sizeof(*subh);
 
-	splx(s);
-	return (ret);
+	bzero(subh, sizeof(*subh));
+	subh->action = PFSYNC_ACT_EOF;
+	subh->count = htons(1);
+
+	/* XXX write checksum in EOF here */
+
+	/* we're done, let's put it on the wire */
+#if NBPFILTER > 0
+	if (ifp->if_bpf) {
+		m->m_data += sizeof(*ip);
+		m->m_len = m->m_pkthdr.len = sc->sc_len - sizeof(*ip);
+		bpf_mtap(ifp->if_bpf, m, BPF_DIRECTION_OUT);
+		m->m_data -= sizeof(*ip);
+		m->m_len = m->m_pkthdr.len = sc->sc_len;
+	}
+#endif
+	sc->sc_if.if_opackets++;
+	sc->sc_if.if_obytes += m->m_pkthdr.len;
+
+	if (ip_output(m, NULL, NULL, IP_RAWOUTPUT, &sc->sc_imo, NULL) == 0)
+		pfsyncstats.pfsyncs_opackets++;
+	else
+		pfsyncstats.pfsyncs_oerrors++;
+
+	/* start again */
+	sc->sc_len = PFSYNC_MINPKT;
 }
 
-/* This must be called in splnet() */
-int
-pfsync_request_update(struct pfsync_state_upd *up, struct in_addr *src)
+void
+pfsync_insert_state(struct pf_state *st)
 {
-	struct pfsync_header *h;
 	struct pfsync_softc *sc = pfsyncif;
-	struct pfsync_state_upd_req *rup;
-	int ret = 0;
 
-	if (sc == NULL)
-		return (0);
+	splassert(IPL_SOFTNET);
 
-	if (sc->sc_mbuf == NULL) {
-		if ((sc->sc_mbuf = pfsync_get_mbuf(sc, PFSYNC_ACT_UREQ,
-		    (void *)&sc->sc_statep.s)) == NULL)
-			return (ENOMEM);
-		h = mtod(sc->sc_mbuf, struct pfsync_header *);
-	} else {
-		h = mtod(sc->sc_mbuf, struct pfsync_header *);
-		if (h->action != PFSYNC_ACT_UREQ) {
-			pfsync_sendout(sc);
-			if ((sc->sc_mbuf = pfsync_get_mbuf(sc, PFSYNC_ACT_UREQ,
-			    (void *)&sc->sc_statep.s)) == NULL)
-				return (ENOMEM);
-			h = mtod(sc->sc_mbuf, struct pfsync_header *);
-		}
+	if (ISSET(st->rule.ptr->rule_flag, PFRULE_NOSYNC) ||
+	    st->key[PF_SK_WIRE]->proto == IPPROTO_PFSYNC) {
+		SET(st->state_flags, PFSTATE_NOSYNC);
+		return;
 	}
 
-	if (src != NULL)
-		sc->sc_sendaddr = *src;
-	sc->sc_mbuf->m_pkthdr.len = sc->sc_mbuf->m_len += sizeof(*rup);
-	h->count++;
-	rup = sc->sc_statep.r++;
-	bzero(rup, sizeof(*rup));
-	if (up != NULL) {
-		bcopy(up->id, rup->id, sizeof(rup->id));
-		rup->creatorid = up->creatorid;
-	}
+	if (sc == NULL || ISSET(st->state_flags, PFSTATE_NOSYNC))
+		return;
+
+#ifdef PFSYNC_DEBUG
+	KASSERT(st->sync_state == PFSYNC_S_NONE);
+#endif
+
+	if (sc->sc_len == PFSYNC_MINPKT)
+		timeout_add_sec(&sc->sc_tmo, 1);
 
-	if (h->count == sc->sc_maxcount)
-		ret = pfsync_sendout(sc);
+	pfsync_q_ins(st, PFSYNC_S_INS);
 
-	return (ret);
+	if (ISSET(st->state_flags, PFSTATE_ACK))
+		schednetisr(NETISR_PFSYNC);
+	else
+		st->sync_updates = 0;
 }
 
+int defer = 10;
+
 int
-pfsync_clear_states(u_int32_t creatorid, char *ifname)
+pfsync_defer(struct pf_state *st, struct mbuf *m)
 {
+	return (0);
+#ifdef notyet
 	struct pfsync_softc *sc = pfsyncif;
-	struct pfsync_state_clr *cp;
-	int s, ret;
+	struct pfsync_deferral *pd;
 
-	if (sc == NULL)
+	splassert(IPL_SOFTNET);
+
+	if (sc->sc_deferred >= 128)
+		pfsync_undefer(TAILQ_FIRST(&sc->sc_deferrals), 0);
+
+	pd = pool_get(&sc->sc_pool, M_NOWAIT);
+	if (pd == NULL)
 		return (0);
+	sc->sc_deferred++;
 
-	s = splnet();
-	if (sc->sc_mbuf != NULL)
-		pfsync_sendout(sc);
-	if ((sc->sc_mbuf = pfsync_get_mbuf(sc, PFSYNC_ACT_CLR,
-	    (void *)&sc->sc_statep.c)) == NULL) {
-		splx(s);
-		return (ENOMEM);
-	}
-	sc->sc_mbuf->m_pkthdr.len = sc->sc_mbuf->m_len += sizeof(*cp);
-	cp = sc->sc_statep.c;
-	cp->creatorid = creatorid;
-	if (ifname != NULL)
-		strlcpy(cp->ifname, ifname, IFNAMSIZ);
+	m->m_pkthdr.pf.flags |= PF_TAG_GENERATED;
+	SET(st->state_flags, PFSTATE_ACK);
 
-	ret = (pfsync_sendout(sc));
-	splx(s);
-	return (ret);
+	pd->pd_st = st;
+	pd->pd_m = m;
+
+	TAILQ_INSERT_TAIL(&sc->sc_deferrals, pd, pd_entry);
+	timeout_set(&pd->pd_tmo, pfsync_defer_tmo, pd);
+	timeout_add(&pd->pd_tmo, defer);
+
+	return (1);
+#endif
 }
 
 void
-pfsync_timeout(void *v)
+pfsync_undefer(struct pfsync_deferral *pd, int drop)
 {
-	struct pfsync_softc *sc = v;
+	struct pfsync_softc *sc = pfsyncif;
 	int s;
 
-	s = splnet();
-	pfsync_sendout(sc);
-	splx(s);
+	splassert(IPL_SOFTNET);
+
+	TAILQ_REMOVE(&sc->sc_deferrals, pd, pd_entry);
+	sc->sc_deferred--;
+
+	CLR(pd->pd_st->state_flags, PFSTATE_ACK);
+	timeout_del(&pd->pd_tmo); /* bah */
+	if (drop)
+		m_freem(pd->pd_m);
+	else {
+		s = splnet();
+		ip_output(pd->pd_m, (void *)NULL, (void *)NULL, 0,
+		    (void *)NULL, (void *)NULL);
+		splx(s);
+	}
+
+	pool_put(&sc->sc_pool, pd);
 }
 
 void
-pfsync_tdb_timeout(void *v)
+pfsync_defer_tmo(void *arg)
 {
-	struct pfsync_softc *sc = v;
 	int s;
 
-	s = splnet();
-	pfsync_tdb_sendout(sc);
+	s = splsoftnet();
+	pfsync_undefer(arg, 0);
 	splx(s);
 }
 
-/* This must be called in splnet() */
 void
-pfsync_send_bus(struct pfsync_softc *sc, u_int8_t status)
+pfsync_deferred(struct pf_state *st, int drop)
 {
-	struct pfsync_state_bus *bus;
-
-	if (sc->sc_mbuf != NULL)
-		pfsync_sendout(sc);
-
-	if (pfsync_sync_ok &&
-	    (sc->sc_mbuf = pfsync_get_mbuf(sc, PFSYNC_ACT_BUS,
-	    (void *)&sc->sc_statep.b)) != NULL) {
-		sc->sc_mbuf->m_pkthdr.len = sc->sc_mbuf->m_len += sizeof(*bus);
-		bus = sc->sc_statep.b;
-		bus->creatorid = pf_status.hostid;
-		bus->status = status;
-		bus->endtime = htonl(time_uptime - sc->sc_ureq_received);
-		pfsync_sendout(sc);
+	struct pfsync_softc *sc = pfsyncif;
+	struct pfsync_deferral *pd;
+
+	TAILQ_FOREACH(pd, &sc->sc_deferrals, pd_entry) {
+		 if (pd->pd_st == st) {
+			pfsync_undefer(pd, drop);
+			return;
+		}
 	}
+
+	panic("pfsync_send_deferred: unable to find deferred state");
 }
 
+u_int pfsync_upds = 0;
+
 void
-pfsync_bulk_update(void *v)
+pfsync_update_state(struct pf_state *st)
 {
-	struct pfsync_softc *sc = v;
-	int s, i = 0;
-	struct pf_state *state;
+	struct pfsync_softc *sc = pfsyncif;
+	int sync = 0;
 
-	s = splnet();
-	if (sc->sc_mbuf != NULL)
-		pfsync_sendout(sc);
+	splassert(IPL_SOFTNET);
 
-	/*
-	 * Grab at most PFSYNC_BULKPACKETS worth of states which have not
-	 * been sent since the latest request was made.
-	 */
-	state = sc->sc_bulk_send_next;
-	if (state)
-		do {
-			/* send state update if syncable and not already sent */
-			if (!state->sync_flags
-			    && state->timeout < PFTM_MAX
-			    && state->pfsync_time <= sc->sc_ureq_received) {
-				pfsync_pack_state(PFSYNC_ACT_UPD, state, 0);
-				i++;
-			}
+	if (sc == NULL)
+		return;
 
-			/* figure next state to send */
-			state = TAILQ_NEXT(state, entry_list);
-
-			/* wrap to start of list if we hit the end */
-			if (!state)
-				state = TAILQ_FIRST(&state_list);
-		} while (i < sc->sc_maxcount * PFSYNC_BULKPACKETS &&
-		    state != sc->sc_bulk_terminator);
-
-	if (!state || state == sc->sc_bulk_terminator) {
-		/* we're done */
-		pfsync_send_bus(sc, PFSYNC_BUS_END);
-		sc->sc_ureq_received = 0;
-		sc->sc_bulk_send_next = NULL;
-		sc->sc_bulk_terminator = NULL;
-		timeout_del(&sc->sc_bulk_tmo);
-		if (pf_status.debug >= PF_DEBUG_MISC)
-			printf("pfsync: bulk update complete\n");
-	} else {
-		/* look again for more in a bit */
-		timeout_add(&sc->sc_bulk_tmo, 1);
-		sc->sc_bulk_send_next = state;
+	if (ISSET(st->state_flags, PFSTATE_ACK))
+		pfsync_deferred(st, 0);
+	if (ISSET(st->state_flags, PFSTATE_NOSYNC)) {
+		if (st->sync_state != PFSYNC_S_NONE)
+			pfsync_q_del(st);
+		return;
+	}
+
+	if (sc->sc_len == PFSYNC_MINPKT)
+		timeout_add_sec(&sc->sc_tmo, 1);
+
+	switch (st->sync_state) {
+	case PFSYNC_S_UPD_C:
+	case PFSYNC_S_UPD:
+	case PFSYNC_S_INS:
+		/* we're already handling it */
+
+		st->sync_updates++;
+		if (st->sync_updates >= sc->sc_maxupdates)
+			sync = 1;
+		break;
+
+	case PFSYNC_S_IACK:
+		pfsync_q_del(st);
+	case PFSYNC_S_NONE:
+		pfsync_q_ins(st, PFSYNC_S_UPD_C);
+		st->sync_updates = 0;
+		break;
+
+	default:
+		panic("pfsync_update_state: unexpected sync state %d",
+		    st->sync_state);
+	}
+
+	if (sync || (time_second - st->pfsync_time) < 2) {
+		pfsync_upds++;
+		schednetisr(NETISR_PFSYNC);
 	}
-	if (sc->sc_mbuf != NULL)
-		pfsync_sendout(sc);
-	splx(s);
 }
 
 void
-pfsync_bulkfail(void *v)
+pfsync_request_update(u_int32_t creatorid, u_int64_t id)
 {
-	struct pfsync_softc *sc = v;
-	int s, error;
+	struct pfsync_softc *sc = pfsyncif;
+	struct pfsync_upd_req_item *item;
+	size_t nlen = sizeof(struct pfsync_upd_req);
+	int s;
 
-	if (sc->sc_bulk_tries++ < PFSYNC_MAX_BULKTRIES) {
-		/* Try again in a bit */
-		timeout_add_sec(&sc->sc_bulkfail_tmo, 5);
+	/*
+	 * this code does nothing to prevent multiple update requests for the
+	 * same state being generated.
+	 */
+
+	item = pool_get(&sc->sc_pool, PR_NOWAIT);
+	if (item == NULL) {
+		/* XXX stats */
+		return;
+	}
+
+	item->ur_msg.id = id;
+	item->ur_msg.creatorid = creatorid;
+
+	if (TAILQ_EMPTY(&sc->sc_upd_req_list))
+		nlen += sizeof(struct pfsync_subheader);
+
+	if (sc->sc_len + nlen > sc->sc_if.if_mtu) {
 		s = splnet();
-		error = pfsync_request_update(NULL, NULL);
-		if (error == ENOMEM) {
-			if (pf_status.debug >= PF_DEBUG_MISC)
-				printf("pfsync: cannot allocate mbufs for "
-				    "bulk update\n");
-		} else
-			pfsync_sendout(sc);
+		pfsync_sendout();
 		splx(s);
-	} else {
-		/* Pretend like the transfer was ok */
-		sc->sc_ureq_sent = 0;
-		sc->sc_bulk_tries = 0;
-#if NCARP > 0
-		if (!pfsync_sync_ok)
-			carp_group_demote_adj(&sc->sc_if, -1);
-#endif
-		pfsync_sync_ok = 1;
-		if (pf_status.debug >= PF_DEBUG_MISC)
-			printf("pfsync: failed to receive "
-			    "bulk update status\n");
-		timeout_del(&sc->sc_bulkfail_tmo);
+
+		nlen = sizeof(struct pfsync_subheader) +
+		    sizeof(struct pfsync_upd_req);
 	}
+
+	TAILQ_INSERT_TAIL(&sc->sc_upd_req_list, item, ur_entry);
+	sc->sc_len += nlen;
+
+	schednetisr(NETISR_PFSYNC);
 }
 
-/* This must be called in splnet() */
-int
-pfsync_sendout(struct pfsync_softc *sc)
+void
+pfsync_update_state_req(struct pf_state *st)
 {
-#if NBPFILTER > 0
-	struct ifnet *ifp = &sc->sc_if;
-#endif
-	struct mbuf *m;
+	struct pfsync_softc *sc = pfsyncif;
 
-	timeout_del(&sc->sc_tmo);
+	if (sc == NULL)
+		panic("pfsync_update_state_req: nonexistant instance");
 
-	if (sc->sc_mbuf == NULL)
-		return (0);
-	m = sc->sc_mbuf;
-	sc->sc_mbuf = NULL;
-	sc->sc_statep.s = NULL;
+	if (ISSET(st->state_flags, PFSTATE_NOSYNC)) {
+		if (st->sync_state != PFSYNC_S_NONE)
+			pfsync_q_del(st);
+		return;
+	}
 
-#if NBPFILTER > 0
-	if (ifp->if_bpf)
-		bpf_mtap(ifp->if_bpf, m, BPF_DIRECTION_OUT);
-#endif
+	switch (st->sync_state) {
+	case PFSYNC_S_UPD_C:
+	case PFSYNC_S_IACK:
+		pfsync_q_del(st);
+	case PFSYNC_S_NONE:
+		pfsync_q_ins(st, PFSYNC_S_UPD);
+		schednetisr(NETISR_PFSYNC);
+		return;
 
-	if (sc->sc_mbuf_net) {
-		m_freem(m);
-		m = sc->sc_mbuf_net;
-		sc->sc_mbuf_net = NULL;
-		sc->sc_statep_net.s = NULL;
-	}
+	case PFSYNC_S_INS:
+	case PFSYNC_S_UPD:
+	case PFSYNC_S_DEL:
+		/* we're already handling it */
+		return;
 
-	return pfsync_sendout_mbuf(sc, m);
+	default:
+		panic("pfsync_update_state_req: unexpected sync state %d",
+		    st->sync_state);
+	}
 }
 
-int
-pfsync_tdb_sendout(struct pfsync_softc *sc)
+void
+pfsync_delete_state(struct pf_state *st)
 {
-#if NBPFILTER > 0
-	struct ifnet *ifp = &sc->sc_if;
-#endif
-	struct mbuf *m;
+	struct pfsync_softc *sc = pfsyncif;
 
-	timeout_del(&sc->sc_tdb_tmo);
+	splassert(IPL_SOFTNET);
 
-	if (sc->sc_mbuf_tdb == NULL)
-		return (0);
-	m = sc->sc_mbuf_tdb;
-	sc->sc_mbuf_tdb = NULL;
-	sc->sc_statep_tdb.t = NULL;
+	if (sc == NULL)
+		return;
 
-#if NBPFILTER > 0
-	if (ifp->if_bpf)
-		bpf_mtap(ifp->if_bpf, m, BPF_DIRECTION_OUT);
-#endif
+	if (ISSET(st->state_flags, PFSTATE_ACK))
+		pfsync_deferred(st, 1);
+	if (ISSET(st->state_flags, PFSTATE_NOSYNC)) {
+		if (st->sync_state != PFSYNC_S_NONE)
+			pfsync_q_del(st);
+		return;
+	}
+
+	if (sc->sc_len == PFSYNC_MINPKT)
+		timeout_add_sec(&sc->sc_tmo, 1);
+
+	switch (st->sync_state) {
+	case PFSYNC_S_INS:
+		/* we never got to tell the world so just forget about it */
+		pfsync_q_del(st);
+		return;
+
+	case PFSYNC_S_UPD_C:
+	case PFSYNC_S_UPD:
+	case PFSYNC_S_IACK:
+		pfsync_q_del(st);
+		/* FALLTHROUGH to putting it on the del list */
 
-	return pfsync_sendout_mbuf(sc, m);
+	case PFSYNC_S_NONE:
+		pfsync_q_ins(st, PFSYNC_S_DEL);
+		return;
+
+	default:
+		panic("pfsync_delete_state: unexpected sync state %d",
+		    st->sync_state);
+	}
 }
 
-int
-pfsync_sendout_mbuf(struct pfsync_softc *sc, struct mbuf *m)
+void
+pfsync_clear_states(u_int32_t creatorid, const char *ifname)
 {
-	struct sockaddr sa;
-	struct ip *ip;
+	struct {
+		struct pfsync_subheader subh;
+		struct pfsync_clr clr;
+	} __packed r;
 
-	if (sc->sc_sync_ifp ||
-	    sc->sc_sync_peer.s_addr != INADDR_PFSYNC_GROUP) {
-		M_PREPEND(m, sizeof(struct ip), M_DONTWAIT);
-		if (m == NULL) {
-			pfsyncstats.pfsyncs_onomem++;
-			return (0);
-		}
-		ip = mtod(m, struct ip *);
-		ip->ip_v = IPVERSION;
-		ip->ip_hl = sizeof(*ip) >> 2;
-		ip->ip_tos = IPTOS_LOWDELAY;
-		ip->ip_len = htons(m->m_pkthdr.len);
-		ip->ip_id = htons(ip_randomid());
-		ip->ip_off = htons(IP_DF);
-		ip->ip_ttl = PFSYNC_DFLTTL;
-		ip->ip_p = IPPROTO_PFSYNC;
-		ip->ip_sum = 0;
+	struct pfsync_softc *sc = pfsyncif;
 
-		bzero(&sa, sizeof(sa));
-		ip->ip_src.s_addr = INADDR_ANY;
+	splassert(IPL_SOFTNET);
 
-		if (sc->sc_sendaddr.s_addr == INADDR_PFSYNC_GROUP)
-			m->m_flags |= M_MCAST;
-		ip->ip_dst = sc->sc_sendaddr;
-		sc->sc_sendaddr.s_addr = sc->sc_sync_peer.s_addr;
+	if (sc == NULL)
+		return;
 
-		pfsyncstats.pfsyncs_opackets++;
+	bzero(&r, sizeof(r));
 
-		if (ip_output(m, NULL, NULL, IP_RAWOUTPUT, &sc->sc_imo, NULL))
-			pfsyncstats.pfsyncs_oerrors++;
-	} else
-		m_freem(m);
+	r.subh.action = PFSYNC_ACT_CLR;
+	r.subh.count = htons(1);
 
-	return (0);
+	strlcpy(r.clr.ifname, ifname, sizeof(r.clr.ifname));
+	r.clr.creatorid = creatorid;
+
+	pfsync_send_plus(&r, sizeof(r));
 }
 
-#ifdef IPSEC
-/* Update an in-kernel tdb. Silently fail if no tdb is found. */
 void
-pfsync_update_net_tdb(struct pfsync_tdb *pt)
+pfsync_q_ins(struct pf_state *st, int q)
 {
-	struct tdb		*tdb;
-	int			 s;
-
-	/* check for invalid values */
-	if (ntohl(pt->spi) <= SPI_RESERVED_MAX ||
-	    (pt->dst.sa.sa_family != AF_INET &&
-	     pt->dst.sa.sa_family != AF_INET6))
-		goto bad;
-
-	s = spltdb();
-	tdb = gettdb(pt->spi, &pt->dst, pt->sproto);
-	if (tdb) {
-		pt->rpl = ntohl(pt->rpl);
-		pt->cur_bytes = betoh64(pt->cur_bytes);
-
-		/* Neither replay nor byte counter should ever decrease. */
-		if (pt->rpl < tdb->tdb_rpl ||
-		    pt->cur_bytes < tdb->tdb_cur_bytes) {
-			splx(s);
-			goto bad;
-		}
+	struct pfsync_softc *sc = pfsyncif;
+	size_t nlen = pfsync_qs[q].len;
+	int s;
+
+	KASSERT(st->sync_state == PFSYNC_S_NONE);
+
+#if 1 || defined(PFSYNC_DEBUG)
+	if (sc->sc_len < PFSYNC_MINPKT)
+		panic("pfsync pkt len is too low %d", sc->sc_len);
+#endif
+	if (TAILQ_EMPTY(&sc->sc_qs[q]))
+		nlen += sizeof(struct pfsync_subheader);
 
-		tdb->tdb_rpl = pt->rpl;
-		tdb->tdb_cur_bytes = pt->cur_bytes;
+	if (sc->sc_len + nlen > sc->sc_if.if_mtu) {
+		s = splnet();
+		pfsync_sendout();
+		splx(s);
+
+		nlen = sizeof(struct pfsync_subheader) + pfsync_qs[q].len;
 	}
-	splx(s);
-	return;
 
- bad:
-	if (pf_status.debug >= PF_DEBUG_MISC)
-		printf("pfsync_insert: PFSYNC_ACT_TDB_UPD: "
-		    "invalid value\n");
-	pfsyncstats.pfsyncs_badstate++;
-	return;
+	sc->sc_len += nlen;
+	TAILQ_INSERT_TAIL(&sc->sc_qs[q], st, sync_list);
+	st->sync_state = q;
 }
 
-/* One of our local tdbs have been updated, need to sync rpl with others */
-int
-pfsync_update_tdb(struct tdb *tdb, int output)
+void
+pfsync_q_del(struct pf_state *st)
 {
-	struct ifnet *ifp = NULL;
 	struct pfsync_softc *sc = pfsyncif;
-	struct pfsync_header *h;
-	struct pfsync_tdb *pt = NULL;
-	int s, i, ret;
+	int q = st->sync_state;
+
+	KASSERT(st->sync_state != PFSYNC_S_NONE);
+
+	sc->sc_len -= pfsync_qs[q].len;
+	TAILQ_REMOVE(&sc->sc_qs[q], st, sync_list);
+	st->sync_state = PFSYNC_S_NONE;
+
+	if (TAILQ_EMPTY(&sc->sc_qs[q]))
+		sc->sc_len -= sizeof(struct pfsync_subheader);
+}
+
+void
+pfsync_update_tdb(struct tdb *t, int output)
+{
+	struct pfsync_softc *sc = pfsyncif;
+	size_t nlen = sizeof(struct pfsync_tdb);
+	int s;
 
 	if (sc == NULL)
-		return (0);
+		return;
 
-	ifp = &sc->sc_if;
-	if (ifp->if_bpf == NULL && sc->sc_sync_ifp == NULL &&
-	    sc->sc_sync_peer.s_addr == INADDR_PFSYNC_GROUP) {
-		/* Don't leave any stale pfsync packets hanging around. */
-		if (sc->sc_mbuf_tdb != NULL) {
-			m_freem(sc->sc_mbuf_tdb);
-			sc->sc_mbuf_tdb = NULL;
-			sc->sc_statep_tdb.t = NULL;
-		}
-		return (0);
-	}
+	if (!ISSET(t->tdb_flags, TDBF_PFSYNC)) {
+		if (TAILQ_EMPTY(&sc->sc_tdb_q))
+			nlen += sizeof(struct pfsync_subheader);
 
-	s = splnet();
-	if (sc->sc_mbuf_tdb == NULL) {
-		if ((sc->sc_mbuf_tdb = pfsync_get_mbuf(sc, PFSYNC_ACT_TDB_UPD,
-		    (void *)&sc->sc_statep_tdb.t)) == NULL) {
+		if (sc->sc_len + nlen > sc->sc_if.if_mtu) {
+			s = splnet();
+			pfsync_sendout();
 			splx(s);
-			return (ENOMEM);
+
+			nlen = sizeof(struct pfsync_subheader) +
+			    sizeof(struct pfsync_tdb);
 		}
-		h = mtod(sc->sc_mbuf_tdb, struct pfsync_header *);
+
+		sc->sc_len += nlen;
+		TAILQ_INSERT_TAIL(&sc->sc_tdb_q, t, tdb_sync_entry);
+		SET(t->tdb_flags, TDBF_PFSYNC);
+		t->tdb_updates = 0;
 	} else {
-		h = mtod(sc->sc_mbuf_tdb, struct pfsync_header *);
-		if (h->action != PFSYNC_ACT_TDB_UPD) {
-			/*
-			 * XXX will never happen as long as there's
-			 * only one "TDB action".
-			 */
-			pfsync_tdb_sendout(sc);
-			sc->sc_mbuf_tdb = pfsync_get_mbuf(sc,
-			    PFSYNC_ACT_TDB_UPD, (void *)&sc->sc_statep_tdb.t);
-			if (sc->sc_mbuf_tdb == NULL) {
-				splx(s);
-				return (ENOMEM);
-			}
-			h = mtod(sc->sc_mbuf_tdb, struct pfsync_header *);
-		} else if (sc->sc_maxupdates) {
-			/*
-			 * If it's an update, look in the packet to see if
-			 * we already have an update for the state.
-			 */
-			struct pfsync_tdb *u =
-			    (void *)((char *)h + PFSYNC_HDRLEN);
-
-			for (i = 0; !pt && i < h->count; i++) {
-				if (tdb->tdb_spi == u->spi &&
-				    tdb->tdb_sproto == u->sproto &&
-				    !bcmp(&tdb->tdb_dst, &u->dst,
-				    SA_LEN(&u->dst.sa))) {
-					pt = u;
-					pt->updates++;
-				}
-				u++;
-			}
-		}
+		if (++t->tdb_updates >= sc->sc_maxupdates)
+			schednetisr(NETISR_PFSYNC);
 	}
 
-	if (pt == NULL) {
-		/* not a "duplicate" update */
-		pt = sc->sc_statep_tdb.t++;
-		sc->sc_mbuf_tdb->m_pkthdr.len =
-		    sc->sc_mbuf_tdb->m_len += sizeof(struct pfsync_tdb);
-		h->count++;
-		bzero(pt, sizeof(*pt));
-
-		pt->spi = tdb->tdb_spi;
-		memcpy(&pt->dst, &tdb->tdb_dst, sizeof pt->dst);
-		pt->sproto = tdb->tdb_sproto;
-	}
+	if (output)
+		SET(t->tdb_flags, TDBF_PFSYNC_RPL);
+	else
+		CLR(t->tdb_flags, TDBF_PFSYNC_RPL);
+}
+
+void
+pfsync_delete_tdb(struct tdb *t)
+{
+	struct pfsync_softc *sc = pfsyncif;
+
+	if (sc == NULL || !ISSET(t->tdb_flags, TDBF_PFSYNC))
+		return;
+
+	sc->sc_len -= sizeof(struct pfsync_tdb);
+	TAILQ_REMOVE(&sc->sc_tdb_q, t, tdb_sync_entry);
+	CLR(t->tdb_flags, TDBF_PFSYNC);
+
+	if (TAILQ_EMPTY(&sc->sc_tdb_q))
+		sc->sc_len -= sizeof(struct pfsync_subheader);
+}
+
+int
+pfsync_out_tdb(struct tdb *t, struct mbuf *m, int offset)
+{
+	struct pfsync_tdb *ut = (struct pfsync_tdb *)(m->m_data + offset);
 
+	bzero(ut, sizeof(*ut));
+	ut->spi = t->tdb_spi;
+	bcopy(&t->tdb_dst, &ut->dst, sizeof(ut->dst));
 	/*
 	 * When a failover happens, the master's rpl is probably above
 	 * what we see here (we may be up to a second late), so
@@ -1984,17 +2136,184 @@ pfsync_update_tdb(struct tdb *tdb, int output)
 	 * this edge case.
 	 */
 #define RPL_INCR 16384
-	pt->rpl = htonl(tdb->tdb_rpl + (output ? RPL_INCR : 0));
-	pt->cur_bytes = htobe64(tdb->tdb_cur_bytes);
+	ut->rpl = htonl(t->tdb_rpl + (ISSET(t->tdb_flags, TDBF_PFSYNC_RPL) ?
+	    RPL_INCR : 0));
+	ut->cur_bytes = htobe64(t->tdb_cur_bytes);
+	ut->sproto = t->tdb_sproto;
 
-	if (h->count == sc->sc_maxcount ||
-	    (sc->sc_maxupdates && (pt->updates >= sc->sc_maxupdates)))
-		ret = pfsync_tdb_sendout(sc);
+	return (sizeof(*ut));
+}
 
-	splx(s);
-	return (ret);
+void
+pfsync_bulk_start(void)
+{
+	struct pfsync_softc *sc = pfsyncif;
+
+	sc->sc_ureq_received = time_uptime;
+
+	if (sc->sc_bulk_next == NULL)
+		sc->sc_bulk_next = TAILQ_FIRST(&state_list);
+	sc->sc_bulk_last = sc->sc_bulk_next;
+
+	if (pf_status.debug >= PF_DEBUG_MISC)
+		printf("pfsync: received bulk update request\n");
+
+	pfsync_bulk_status(PFSYNC_BUS_START);
+	pfsync_bulk_update(sc);
+}
+
+void
+pfsync_bulk_update(void *arg)
+{
+	struct pfsync_softc *sc = arg;
+	struct pf_state *st = sc->sc_bulk_next;
+	int i = 0;
+
+	do {
+		if (st->sync_state == PFSYNC_S_NONE &&
+		    st->timeout < PFTM_MAX &&
+		    st->pfsync_time <= sc->sc_ureq_received) {
+			pfsync_update_state_req(st);
+			i++;
+		}
+
+		st = TAILQ_NEXT(st, entry_list);
+		if (st == NULL)
+			st = TAILQ_FIRST(&state_list);
+
+		if (i > 0 && TAILQ_EMPTY(&sc->sc_qs[PFSYNC_S_UPD])) {
+			sc->sc_bulk_next = st;
+			timeout_add(&sc->sc_bulk_tmo, 1);
+			return;
+		}
+	} while (st != sc->sc_bulk_last);
+
+	/* we're done */
+	sc->sc_bulk_next = NULL;
+	sc->sc_bulk_last = NULL;
+	pfsync_bulk_status(PFSYNC_BUS_END);
+}
+
+void
+pfsync_bulk_status(u_int8_t status)
+{
+	struct {
+		struct pfsync_subheader subh;
+		struct pfsync_bus bus;
+	} __packed r;
+
+	struct pfsync_softc *sc = pfsyncif;
+
+	bzero(&r, sizeof(r));
+
+	r.subh.action = PFSYNC_ACT_BUS;
+	r.subh.count = htons(1);
+
+	r.bus.creatorid = pf_status.hostid;
+	r.bus.endtime = htonl(time_uptime - sc->sc_ureq_received);
+	r.bus.status = status;
+
+	pfsync_send_plus(&r, sizeof(r));
 }
+
+void
+pfsync_bulk_fail(void *arg)
+{
+	struct pfsync_softc *sc = arg;
+
+	if (sc->sc_bulk_tries++ < PFSYNC_MAX_BULKTRIES) {
+		/* Try again */
+		timeout_add_sec(&sc->sc_bulkfail_tmo, 5);
+		pfsync_request_update(0, 0);
+	} else {
+		/* Pretend like the transfer was ok */
+		sc->sc_ureq_sent = 0;
+		sc->sc_bulk_tries = 0;
+#if NCARP > 0
+		if (!pfsync_sync_ok)
+			carp_group_demote_adj(&sc->sc_if, -1);
 #endif
+		pfsync_sync_ok = 1;
+		if (pf_status.debug >= PF_DEBUG_MISC)
+			printf("pfsync: failed to receive bulk update\n");
+	}
+}
+
+void
+pfsync_send_plus(void *plus, size_t pluslen)
+{
+	struct pfsync_softc *sc = pfsyncif;
+	int s;
+
+	if (sc->sc_len + pluslen > sc->sc_if.if_mtu) {
+		s = splnet();
+		pfsync_sendout();
+		splx(s);
+	}
+
+	sc->sc_plus = plus;
+	sc->sc_len += (sc->sc_pluslen = pluslen);
+
+	s = splnet();
+	pfsync_sendout();
+	splx(s);
+}
+
+int
+pfsync_up(void)
+{
+	struct pfsync_softc *sc = pfsyncif;
+
+	if (sc == NULL || !ISSET(sc->sc_if.if_flags, IFF_RUNNING))
+		return (0);
+
+	return (1);
+}
+
+int
+pfsync_state_in_use(struct pf_state *st)
+{
+	struct pfsync_softc *sc = pfsyncif;
+
+	if (sc == NULL)
+		return (0);
+
+	if (st->sync_state != PFSYNC_S_NONE)
+		return (1);
+
+	if (sc->sc_bulk_next == NULL && sc->sc_bulk_last == NULL)
+		return (0);
+
+	return (1);
+}
+
+u_int pfsync_ints;
+u_int pfsync_tmos;
+
+void
+pfsync_timeout(void *arg)
+{
+	int s;
+
+	pfsync_tmos++;
+
+	s = splnet();
+	pfsync_sendout();
+	splx(s);
+}
+
+/* this is a softnet/netisr handler */
+void
+pfsyncintr(void)
+{
+	int s;
+
+	pfsync_ints++;
+
+	s = splnet();
+	pfsync_sendout();
+	splx(s);
+}
 
 int
 pfsync_sysctl(int *name, u_int namelen, void *oldp, size_t *oldlenp, void *newp,