make ph_flowid in mbufs 16bits by storing whether it's set in csum_flags.

i've been wanting to do this for a while, and now that we've got
stoeplitz and it gives us 16 bits, it seems like the right time.

12 files changed, 51 insertions, 54 deletions

diff --git a/sys/dev/pci/if_mcx.c b/sys/dev/pci/if_mcx.c
index a4cb1858c53..211e8d61399 100644
--- a/sys/dev/pci/if_mcx.c
+++ b/sys/dev/pci/if_mcx.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: if_mcx.c,v 1.55 2020/06/16 02:23:40 jmatthew Exp $ */
+/*	$OpenBSD: if_mcx.c,v 1.56 2020/06/17 06:45:22 dlg Exp $ */
 
 /*
  * Copyright (c) 2017 David Gwynne <dlg@openbsd.org>
@@ -5816,8 +5816,8 @@ mcx_process_rx(struct mcx_softc *sc, struct mcx_cq_entry *cqe,
 	m->m_pkthdr.len = m->m_len = bemtoh32(&cqe->cq_byte_cnt);
 
 	if (cqe->cq_rx_hash_type) {
-		m->m_pkthdr.ph_flowid = M_FLOWID_VALID |
-		    betoh32(cqe->cq_rx_hash);
+		m->m_pkthdr.ph_flowid = betoh32(cqe->cq_rx_hash);
+		m->m_pkthdr.csum_flags |= M_FLOWID;
 	}
 
 	flags = bemtoh32(&cqe->cq_flags);
diff --git a/sys/net/fq_codel.c b/sys/net/fq_codel.c
index a5077be4cc2..9fac8819854 100644
--- a/sys/net/fq_codel.c
+++ b/sys/net/fq_codel.c
@@ -521,8 +521,8 @@ classify_flow(struct fqcodel *fqc, struct mbuf *m)
 {
 	unsigned int index;
 
-	if (m->m_pkthdr.ph_flowid & M_FLOWID_VALID)
-		index = (m->m_pkthdr.ph_flowid & M_FLOWID_MASK) % fqc->nflows;
+	if (m->m_pkthdr.csum_flags & M_FLOWID)
+		index = m->m_pkthdr.ph_flowid % fqc->nflows;
 	else
 		index = arc4random_uniform(fqc->nflows);
 
@@ -746,7 +746,7 @@ fqcodel_pf_addqueue(void *arg, struct pf_queuespec *qs)
 	struct ifnet *ifp = qs->kif->pfik_ifp;
 	struct fqcodel *fqc = arg;
 
-	if (qs->flowqueue.flows == 0 || qs->flowqueue.flows > M_FLOWID_MASK)
+	if (qs->flowqueue.flows == 0 || qs->flowqueue.flows > 0xffff)
 		return (EINVAL);
 
 	fqc->nflows = qs->flowqueue.flows;
diff --git a/sys/net/if.c b/sys/net/if.c
index ff7a03f7e0a..1cd42e48bc5 100644
--- a/sys/net/if.c
+++ b/sys/net/if.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: if.c,v 1.606 2020/05/29 04:42:25 deraadt Exp $	*/
+/*	$OpenBSD: if.c,v 1.607 2020/06/17 06:45:22 dlg Exp $	*/
 /*	$NetBSD: if.c,v 1.35 1996/05/07 05:26:04 thorpej Exp $	*/
 
 /*
@@ -803,8 +803,8 @@ if_output_local(struct ifnet *ifp, struct mbuf *m, sa_family_t af)
 	m->m_pkthdr.ph_ifidx = ifp->if_index;
 	m->m_pkthdr.ph_rtableid = ifp->if_rdomain;
 
-	if (ISSET(m->m_pkthdr.ph_flowid, M_FLOWID_VALID))
-		flow = m->m_pkthdr.ph_flowid & M_FLOWID_MASK;
+	if (ISSET(m->m_pkthdr.csum_flags, M_FLOWID))
+		flow = m->m_pkthdr.ph_flowid;
 
 	ifiq = ifp->if_iqs[flow % ifp->if_niqs];
 
diff --git a/sys/net/if_aggr.c b/sys/net/if_aggr.c
index dfbd2e1310b..5b45c3dcfe7 100644
--- a/sys/net/if_aggr.c
+++ b/sys/net/if_aggr.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: if_aggr.c,v 1.30 2020/06/02 00:58:09 jmatthew Exp $ */
+/*	$OpenBSD: if_aggr.c,v 1.31 2020/06/17 06:45:22 dlg Exp $ */
 
 /*
  * Copyright (c) 2019 The University of Queensland
@@ -660,7 +660,7 @@ aggr_transmit(struct aggr_softc *sc, const struct aggr_map *map, struct mbuf *m)
 	}
 #endif
 
-	if (ISSET(m->m_pkthdr.ph_flowid, M_FLOWID_VALID))
+	if (ISSET(m->m_pkthdr.csum_flags, M_FLOWID))
 		flow = m->m_pkthdr.ph_flowid;
 
 	ifp0 = map->m_ifp0s[flow % AGGR_MAX_PORTS];
@@ -765,10 +765,8 @@ aggr_input(struct ifnet *ifp0, struct mbuf *m, void *cookie)
 	if (__predict_false(!p->p_collecting))
 		goto drop;
 
-	if (!ISSET(m->m_pkthdr.ph_flowid, M_FLOWID_VALID)) {
-		m->m_pkthdr.ph_flowid = M_FLOWID_VALID |
-		    (ifp0->if_index ^ sc->sc_mix);
-	}
+	if (!ISSET(m->m_pkthdr.csum_flags, M_FLOWID))
+		m->m_pkthdr.ph_flowid = ifp0->if_index ^ sc->sc_mix;
 
 	if_vinput(ifp, m);
 
diff --git a/sys/net/if_gif.c b/sys/net/if_gif.c
index 07c11fbb40f..333b1ccb469 100644
--- a/sys/net/if_gif.c
+++ b/sys/net/if_gif.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: if_gif.c,v 1.128 2019/10/04 05:00:49 dlg Exp $	*/
+/*	$OpenBSD: if_gif.c,v 1.129 2020/06/17 06:45:22 dlg Exp $	*/
 /*	$KAME: if_gif.c,v 1.43 2001/02/20 08:51:07 itojun Exp $	*/
 
 /*
@@ -355,8 +355,8 @@ gif_send(struct gif_softc *sc, struct mbuf *m,
 			return (-1);
 
 		flow = otos << 20;
-		if (ISSET(m->m_pkthdr.ph_flowid, M_FLOWID_VALID))
-			flow |= m->m_pkthdr.ph_flowid & M_FLOWID_MASK;
+		if (ISSET(m->m_pkthdr.csum_flags, M_FLOWID))
+			flow |= m->m_pkthdr.ph_flowid;
 
 		ip6 = mtod(m, struct ip6_hdr *);
 		ip6->ip6_flow = htonl(flow);
diff --git a/sys/net/if_gre.c b/sys/net/if_gre.c
index 95b3082f672..0b3c3525d8e 100644
--- a/sys/net/if_gre.c
+++ b/sys/net/if_gre.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: if_gre.c,v 1.156 2020/04/12 11:56:52 mpi Exp $ */
+/*	$OpenBSD: if_gre.c,v 1.157 2020/06/17 06:45:22 dlg Exp $ */
 /*	$NetBSD: if_gre.c,v 1.9 1999/10/25 19:18:11 drochner Exp $ */
 
 /*
@@ -1167,8 +1167,9 @@ gre_input_key(struct mbuf **mp, int *offp, int type, int af, uint8_t otos,
 		return (IPPROTO_DONE);
 
 	if (tunnel->t_key_mask == GRE_KEY_ENTROPY) {
-		m->m_pkthdr.ph_flowid = M_FLOWID_VALID |
-		    (bemtoh32(&key->t_key) & ~GRE_KEY_ENTROPY);
+		SET(m->m_pkthdr.csum_flags, M_FLOWID);
+		m->m_pkthdr.ph_flowid =
+		    bemtoh32(&key->t_key) & ~GRE_KEY_ENTROPY;
 	}
 
 	rxprio = tunnel->t_rxhprio;
@@ -1326,8 +1327,9 @@ egre_input(const struct gre_tunnel *key, struct mbuf *m, int hlen, uint8_t otos)
 		return (0);
 
 	if (sc->sc_tunnel.t_key_mask == GRE_KEY_ENTROPY) {
-		m->m_pkthdr.ph_flowid = M_FLOWID_VALID |
-		    (bemtoh32(&key->t_key) & ~GRE_KEY_ENTROPY);
+		SET(m->m_pkthdr.csum_flags, M_FLOWID);
+		m->m_pkthdr.ph_flowid =
+		    bemtoh32(&key->t_key) & ~GRE_KEY_ENTROPY;
 	}
 
 	m->m_flags &= ~(M_MCAST|M_BCAST);
@@ -1577,8 +1579,8 @@ nvgre_input(const struct gre_tunnel *key, struct mbuf *m, int hlen,
 
 	nvgre_input_map(sc, key, mtod(m, struct ether_header *));
 
-	m->m_pkthdr.ph_flowid = M_FLOWID_VALID |
-	    (bemtoh32(&key->t_key) & ~GRE_KEY_ENTROPY);
+	SET(m->m_pkthdr.csum_flags, M_FLOWID);
+	m->m_pkthdr.ph_flowid = bemtoh32(&key->t_key) & ~GRE_KEY_ENTROPY;
 
 	gre_l2_prio(&sc->sc_tunnel, m, otos);
 
@@ -2132,9 +2134,9 @@ gre_encap_dst(const struct gre_tunnel *tunnel, const union gre_addr *dst,
 		gkh->gre_key = tunnel->t_key;
 
 		if (tunnel->t_key_mask == GRE_KEY_ENTROPY &&
-		    ISSET(m->m_pkthdr.ph_flowid, M_FLOWID_VALID)) {
+		    ISSET(m->m_pkthdr.csum_flags, M_FLOWID)) {
 			gkh->gre_key |= htonl(~GRE_KEY_ENTROPY &
-			    (m->m_pkthdr.ph_flowid & M_FLOWID_MASK));
+			    m->m_pkthdr.ph_flowid);
 		}
 	}
 
@@ -2179,8 +2181,8 @@ gre_encap_dst_ip(const struct gre_tunnel *tunnel, const union gre_addr *dst,
 			return (NULL);
 
 		ip6 = mtod(m, struct ip6_hdr *);
-		ip6->ip6_flow = ISSET(m->m_pkthdr.ph_flowid, M_FLOWID_VALID) ?
-		    htonl(m->m_pkthdr.ph_flowid & M_FLOWID_MASK) : 0;
+		ip6->ip6_flow = ISSET(m->m_pkthdr.csum_flags, M_FLOWID) ?
+		    htonl(m->m_pkthdr.ph_flowid) : 0;
 		ip6->ip6_vfc |= IPV6_VERSION;
 		ip6->ip6_flow |= htonl((uint32_t)tos << 20);
 		ip6->ip6_plen = htons(len);
diff --git a/sys/net/if_mpip.c b/sys/net/if_mpip.c
index f4f11781e82..5f82da2f94a 100644
--- a/sys/net/if_mpip.c
+++ b/sys/net/if_mpip.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: if_mpip.c,v 1.8 2019/06/26 08:13:13 claudio Exp $ */
+/*	$OpenBSD: if_mpip.c,v 1.9 2020/06/17 06:45:22 dlg Exp $ */
 
 /*
  * Copyright (c) 2015 Rafael Zalamena <rzalamena@openbsd.org>
@@ -496,7 +496,8 @@ mpip_input(struct mpip_softc *sc, struct mbuf *m)
 
 		label -= MPLS_LABEL_RESERVED_MAX + 1;
 		label ^= sc->sc_flow;
-		m->m_pkthdr.ph_flowid = M_FLOWID_VALID | label;
+		SET(m->m_pkthdr.csum_flags, M_FLOWID);
+		m->m_pkthdr.ph_flowid = label;
 
 		m_adj(m, sizeof(shim));
 	} else if (!MPLS_BOS_ISSET(shim))
@@ -777,8 +778,8 @@ mpip_start(struct ifnet *ifp)
 			if (m == NULL)
 				continue;
 
-			if (ISSET(m->m_pkthdr.ph_flowid, M_FLOWID_VALID))
-				flow = m->m_pkthdr.ph_flowid & M_FLOWID_MASK;
+			if (ISSET(m->m_pkthdr.csum_flags, M_FLOWID))
+				flow = m->m_pkthdr.ph_flowid;
 			flow ^= sc->sc_flow;
 			flow += MPLS_LABEL_RESERVED_MAX + 1;
 
diff --git a/sys/net/if_mpw.c b/sys/net/if_mpw.c
index ae3c18a04c0..be8bdda2247 100644
--- a/sys/net/if_mpw.c
+++ b/sys/net/if_mpw.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: if_mpw.c,v 1.55 2019/06/26 08:13:13 claudio Exp $ */
+/*	$OpenBSD: if_mpw.c,v 1.56 2020/06/17 06:45:22 dlg Exp $ */
 
 /*
  * Copyright (c) 2015 Rafael Zalamena <rzalamena@openbsd.org>
@@ -538,7 +538,8 @@ mpw_input(struct mpw_softc *sc, struct mbuf *m)
 
 		flow = MPLS_SHIM2LABEL(shim->shim_label);
 		flow ^= sc->sc_flow;
-		m->m_pkthdr.ph_flowid = M_FLOWID_VALID | flow;
+		SET(m->m_pkthdr.csum_flags, M_FLOWID);
+		m->m_pkthdr.ph_flowid = flow;
 	} else {
 		if (!MPLS_BOS_ISSET(shim->shim_label))
 			goto drop;
@@ -712,8 +713,8 @@ mpw_start(struct ifnet *ifp)
 			if (m0 == NULL)
 				continue;
 
-			if (ISSET(m->m_pkthdr.ph_flowid, M_FLOWID_VALID))
-				flow ^= m->m_pkthdr.ph_flowid & M_FLOWID_MASK;
+			if (ISSET(m->m_pkthdr.csum_flags, M_FLOWID))
+				flow ^= m->m_pkthdr.ph_flowid;
 
 			shim = mtod(m0, struct shim_hdr *);
 			shim->shim_label = htonl(1) & MPLS_TTL_MASK;
diff --git a/sys/net/if_trunk.c b/sys/net/if_trunk.c
index 8858702562d..4f0995ef7dd 100644
--- a/sys/net/if_trunk.c
+++ b/sys/net/if_trunk.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: if_trunk.c,v 1.145 2020/05/21 00:44:36 dlg Exp $	*/
+/*	$OpenBSD: if_trunk.c,v 1.146 2020/06/17 06:45:22 dlg Exp $	*/
 
 /*
  * Copyright (c) 2005, 2006, 2007 Reyk Floeter <reyk@openbsd.org>
@@ -1045,8 +1045,8 @@ trunk_hashmbuf(struct mbuf *m, SIPHASH_KEY *key)
 #endif
 	SIPHASH_CTX ctx;
 
-	if (m->m_pkthdr.ph_flowid & M_FLOWID_VALID)
-		return (m->m_pkthdr.ph_flowid & M_FLOWID_MASK);
+	if (m->m_pkthdr.csum_flags & M_FLOWID)
+		return (m->m_pkthdr.ph_flowid);
 
 	SipHash24_Init(&ctx, key);
 	off = sizeof(*eh);
diff --git a/sys/net/ifq.c b/sys/net/ifq.c
index fc6518d782d..2846d23ed03 100644
--- a/sys/net/ifq.c
+++ b/sys/net/ifq.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: ifq.c,v 1.39 2020/05/21 00:06:16 dlg Exp $ */
+/*	$OpenBSD: ifq.c,v 1.40 2020/06/17 06:45:22 dlg Exp $ */
 
 /*
  * Copyright (c) 2015 David Gwynne <dlg@openbsd.org>
@@ -695,8 +695,8 @@ priq_idx(unsigned int nqueues, const struct mbuf *m)
 {
 	unsigned int flow = 0;
 
-	if (ISSET(m->m_pkthdr.ph_flowid, M_FLOWID_VALID))
-		flow = m->m_pkthdr.ph_flowid & M_FLOWID_MASK;
+	if (ISSET(m->m_pkthdr.csum_flags, M_FLOWID))
+		flow = m->m_pkthdr.ph_flowid;
 
 	return (flow % nqueues);
 }
diff --git a/sys/net/pf.c b/sys/net/pf.c
index ebe339921fa..d31ef989743 100644
--- a/sys/net/pf.c
+++ b/sys/net/pf.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: pf.c,v 1.1091 2019/11/17 08:25:05 otto Exp $ */
+/*	$OpenBSD: pf.c,v 1.1092 2020/06/17 06:45:22 dlg Exp $ */
 
 /*
  * Copyright (c) 2001 Daniel Hartmeier
@@ -7177,10 +7177,8 @@ done:
 		pf_state_key_link_inpcb(s->key[PF_SK_STACK],
 		    pd.m->m_pkthdr.pf.inp);
 
-	if (s && (pd.m->m_pkthdr.ph_flowid & M_FLOWID_VALID) == 0) {
-		pd.m->m_pkthdr.ph_flowid = M_FLOWID_VALID |
-		    (M_FLOWID_MASK & bemtoh64(&s->id));
-	}
+	if (s && (pd.m->m_pkthdr.csum_flags & M_FLOWID) == 0)
+		pd.m->m_pkthdr.ph_flowid = bemtoh64(&s->id);
 
 	/*
 	 * connections redirected to loopback should not match sockets
diff --git a/sys/sys/mbuf.h b/sys/sys/mbuf.h
index a54b003754d..f63f7356f88 100644
--- a/sys/sys/mbuf.h
+++ b/sys/sys/mbuf.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: mbuf.h,v 1.246 2020/01/22 22:56:35 dlg Exp $	*/
+/*	$OpenBSD: mbuf.h,v 1.247 2020/06/17 06:45:22 dlg Exp $	*/
 /*	$NetBSD: mbuf.h,v 1.19 1996/02/09 18:25:14 christos Exp $	*/
 
 /*
@@ -227,13 +227,14 @@ struct mbuf {
 #define	M_ICMP_CSUM_IN_BAD	0x0800	/* ICMP/ICMPv6 checksum bad */
 #define	M_IPV6_DF_OUT		0x1000	/* don't fragment outgoing IPv6 */
 #define	M_TIMESTAMP		0x2000	/* ph_timestamp is set */
+#define	M_FLOWID		0x4000	/* ph_flowid is set */
 
 #ifdef _KERNEL
 #define MCS_BITS \
     ("\20\1IPV4_CSUM_OUT\2TCP_CSUM_OUT\3UDP_CSUM_OUT\4IPV4_CSUM_IN_OK" \
     "\5IPV4_CSUM_IN_BAD\6TCP_CSUM_IN_OK\7TCP_CSUM_IN_BAD\10UDP_CSUM_IN_OK" \
     "\11UDP_CSUM_IN_BAD\12ICMP_CSUM_OUT\13ICMP_CSUM_IN_OK\14ICMP_CSUM_IN_BAD" \
-    "\15IPV6_NODF_OUT" "\16TIMESTAMP")
+    "\15IPV6_NODF_OUT" "\16TIMESTAMP" "\17FLOWID")
 #endif
 
 /* mbuf types */
@@ -247,10 +248,6 @@ struct mbuf {
 #define	MT_OOBDATA	7	/* expedited data  */
 #define	MT_NTYPES	8
 
-/* flowid field */
-#define M_FLOWID_VALID	0x8000	/* is the flowid set */
-#define M_FLOWID_MASK	0x7fff	/* flow id to map to path */
-
 /* flags to m_get/MGET */
 #include <sys/malloc.h>
 #define	M_DONTWAIT	M_NOWAIT