17 files changed, 305 insertions, 160 deletions

diff --git a/sys/arch/octeon/dev/if_cnmac.c b/sys/arch/octeon/dev/if_cnmac.c
index 2c9257eac8b..67011eaf41d 100644
--- a/sys/arch/octeon/dev/if_cnmac.c
+++ b/sys/arch/octeon/dev/if_cnmac.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: if_cnmac.c,v 1.61 2016/11/05 05:14:18 visa Exp $	*/
+/*	$OpenBSD: if_cnmac.c,v 1.62 2017/01/24 03:57:34 dlg Exp $	*/
 
 /*
  * Copyright (c) 2007 Internet Initiative Japan, Inc.
@@ -138,7 +138,7 @@ int	octeon_eth_ioctl(struct ifnet *, u_long, caddr_t);
 void	octeon_eth_watchdog(struct ifnet *);
 int	octeon_eth_init(struct ifnet *);
 int	octeon_eth_stop(struct ifnet *, int);
-void	octeon_eth_start(struct ifnet *);
+void	octeon_eth_start(struct ifqueue *);
 
 int	octeon_eth_send_cmd(struct octeon_eth_softc *, uint64_t, uint64_t);
 uint64_t octeon_eth_send_makecmd_w1(int, paddr_t);
@@ -303,7 +303,7 @@ octeon_eth_attach(struct device *parent, struct device *self, void *aux)
 	ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
 	ifp->if_xflags = IFXF_MPSAFE;
 	ifp->if_ioctl = octeon_eth_ioctl;
-	ifp->if_start = octeon_eth_start;
+	ifp->if_qstart = octeon_eth_start;
 	ifp->if_watchdog = octeon_eth_watchdog;
 	ifp->if_hardmtu = OCTEON_ETH_MAX_MTU;
 	IFQ_SET_MAXLEN(&ifp->if_snd, max(GATHER_QUEUE_SIZE, IFQ_MAXLEN));
@@ -704,8 +704,6 @@ octeon_eth_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data)
 		error = 0;
 	}
 
-	if_start(ifp);
-
 	splx(s);
 	return (error);
 }
@@ -923,13 +921,14 @@ done:
 }
 
 void
-octeon_eth_start(struct ifnet *ifp)
+octeon_eth_start(struct ifqueue *ifq)
 {
+	struct ifnet *ifp = ifq->ifq_if;
 	struct octeon_eth_softc *sc = ifp->if_softc;
 	struct mbuf *m;
 
 	if (__predict_false(!cn30xxgmx_link_status(sc->sc_gmx_port))) {
-		ifq_purge(&ifp->if_snd);
+		ifq_purge(ifq);
 		return;
 	}
 
@@ -948,12 +947,12 @@ octeon_eth_start(struct ifnet *ifp)
 		 * and bail out.
 		 */
 		if (octeon_eth_send_queue_is_full(sc)) {
-			ifq_set_oactive(&ifp->if_snd);
+			ifq_set_oactive(ifq);
 			timeout_add(&sc->sc_tick_free_ch, 1);
 			return;
 		}
 
-		m = ifq_dequeue(&ifp->if_snd);
+		m = ifq_dequeue(ifq);
 		if (m == NULL)
 			return;
 
@@ -1320,6 +1319,7 @@ octeon_eth_free_task(void *arg)
 {
 	struct octeon_eth_softc *sc = arg;
 	struct ifnet *ifp = &sc->sc_arpcom.ac_if;
+	struct ifqueue *ifq = &ifp->if_snd;
 	int resched = 1;
 	int timeout;
 
@@ -1329,13 +1329,14 @@ octeon_eth_free_task(void *arg)
 		octeon_eth_send_queue_flush(sc);
 	}
 
-	if (ifq_is_oactive(&ifp->if_snd)) {
-		ifq_clr_oactive(&ifp->if_snd);
-		octeon_eth_start(ifp);
+	if (ifq_is_oactive(ifq)) {
+		ifq_clr_oactive(ifq);
+		octeon_eth_start(ifq);
 
-		if (ifq_is_oactive(&ifp->if_snd))
+		if (ifq_is_oactive(ifq)) {
 			/* The start routine did rescheduling already. */
 			resched = 0;
+		}
 	}
 
 	if (resched) {
diff --git a/sys/dev/ic/re.c b/sys/dev/ic/re.c
index 205d3569702..3af886547be 100644
--- a/sys/dev/ic/re.c
+++ b/sys/dev/ic/re.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: re.c,v 1.200 2017/01/22 10:17:38 dlg Exp $	*/
+/*	$OpenBSD: re.c,v 1.201 2017/01/24 03:57:34 dlg Exp $	*/
 /*	$FreeBSD: if_re.c,v 1.31 2004/09/04 07:54:05 ru Exp $	*/
 /*
  * Copyright (c) 1997, 1998-2003
@@ -161,7 +161,7 @@ int	re_tx_list_init(struct rl_softc *);
 int	re_rxeof(struct rl_softc *);
 int	re_txeof(struct rl_softc *);
 void	re_tick(void *);
-void	re_start(struct ifnet *);
+void	re_start(struct ifqueue *);
 void	re_txstart(void *);
 int	re_ioctl(struct ifnet *, u_long, caddr_t);
 void	re_watchdog(struct ifnet *);
@@ -1005,7 +1005,7 @@ re_attach(struct rl_softc *sc, const char *intrstr)
 	ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
 	ifp->if_xflags = IFXF_MPSAFE;
 	ifp->if_ioctl = re_ioctl;
-	ifp->if_start = re_start;
+	ifp->if_qstart = re_start;
 	ifp->if_watchdog = re_watchdog;
 	ifp->if_hardmtu = sc->rl_max_mtu;
 	IFQ_SET_MAXLEN(&ifp->if_snd, sc->rl_ldata.rl_tx_desc_cnt);
@@ -1776,8 +1776,9 @@ re_txstart(void *xsc)
  */
 
 void
-re_start(struct ifnet *ifp)
+re_start(struct ifqueue *ifq)
 {
+	struct ifnet	*ifp = ifq->ifq_if;
 	struct rl_softc	*sc = ifp->if_softc;
 	struct mbuf	*m;
 	unsigned int	idx;
@@ -1785,7 +1786,7 @@ re_start(struct ifnet *ifp)
 	int		post = 0;
 
 	if (!ISSET(sc->rl_flags, RL_FLAG_LINK)) {
-		IFQ_PURGE(&ifp->if_snd);
+		ifq_purge(ifq);
 		return;
 	}
 
@@ -1797,11 +1798,11 @@ re_start(struct ifnet *ifp)
 
 	for (;;) {
 		if (sc->rl_ldata.rl_tx_ndescs >= free + 2) {
-			ifq_set_oactive(&ifp->if_snd);
+			ifq_set_oactive(ifq);
 			break;
 		}
 
-		m = ifq_dequeue(&ifp->if_snd);
+		m = ifq_dequeue(ifq);
 		if (m == NULL)
 			break;
 
@@ -1831,7 +1832,7 @@ re_start(struct ifnet *ifp)
 
 	ifp->if_timer = 5;
 	sc->rl_ldata.rl_txq_prodidx = idx;
-	ifq_serialize(&ifp->if_snd, &sc->rl_start);
+	ifq_serialize(ifq, &sc->rl_start);
 }
 
 int
diff --git a/sys/dev/pci/if_bge.c b/sys/dev/pci/if_bge.c
index 5ccd8c1efe6..1d0ff59ebb9 100644
--- a/sys/dev/pci/if_bge.c
+++ b/sys/dev/pci/if_bge.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: if_bge.c,v 1.383 2017/01/22 10:17:38 dlg Exp $	*/
+/*	$OpenBSD: if_bge.c,v 1.384 2017/01/24 03:57:34 dlg Exp $	*/
 
 /*
  * Copyright (c) 2001 Wind River Systems
@@ -142,7 +142,7 @@ int bge_encap(struct bge_softc *, struct mbuf *, int *);
 int bge_compact_dma_runt(struct mbuf *);
 
 int bge_intr(void *);
-void bge_start(struct ifnet *);
+void bge_start(struct ifqueue *);
 int bge_ioctl(struct ifnet *, u_long, caddr_t);
 int bge_rxrinfo(struct bge_softc *, struct if_rxrinfo *);
 void bge_init(void *);
@@ -2996,7 +2996,7 @@ bge_attach(struct device *parent, struct device *self, void *aux)
 	ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
 	ifp->if_xflags = IFXF_MPSAFE;
 	ifp->if_ioctl = bge_ioctl;
-	ifp->if_start = bge_start;
+	ifp->if_qstart = bge_start;
 	ifp->if_watchdog = bge_watchdog;
 	IFQ_SET_MAXLEN(&ifp->if_snd, BGE_TX_RING_CNT - 1);
 
@@ -4116,14 +4116,15 @@ fail_unload:
  * to the mbuf data regions directly in the transmit descriptors.
  */
 void
-bge_start(struct ifnet *ifp)
+bge_start(struct ifqueue *ifq)
 {
+	struct ifnet *ifp = ifq->ifq_if;
 	struct bge_softc *sc = ifp->if_softc;
 	struct mbuf *m;
 	int txinc;
 
 	if (!BGE_STS_BIT(sc, BGE_STS_LINK)) {
-		IFQ_PURGE(&ifp->if_snd);
+		ifq_purge(ifq);
 		return;
 	}
 
@@ -4132,11 +4133,11 @@ bge_start(struct ifnet *ifp)
 		/* Check if we have enough free send BDs. */
 		if (sc->bge_txcnt + txinc + BGE_NTXSEG + 16 >=
 		    BGE_TX_RING_CNT) {
-			ifq_set_oactive(&ifp->if_snd);
+			ifq_set_oactive(ifq);
 			break;
 		}
 
-		IFQ_DEQUEUE(&ifp->if_snd, m);
+		m = ifq_dequeue(ifq);
 		if (m == NULL)
 			break;
 
diff --git a/sys/dev/pci/if_bnx.c b/sys/dev/pci/if_bnx.c
index f3e6ea52944..b054cb10407 100644
--- a/sys/dev/pci/if_bnx.c
+++ b/sys/dev/pci/if_bnx.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: if_bnx.c,v 1.123 2017/01/22 10:17:38 dlg Exp $	*/
+/*	$OpenBSD: if_bnx.c,v 1.124 2017/01/24 03:57:35 dlg Exp $	*/
 
 /*-
  * Copyright (c) 2006 Broadcom Corporation
@@ -366,7 +366,7 @@ void	bnx_free_tx_chain(struct bnx_softc *);
 void	bnx_rxrefill(void *);
 
 int	bnx_tx_encap(struct bnx_softc *, struct mbuf *, int *);
-void	bnx_start(struct ifnet *);
+void	bnx_start(struct ifqueue *);
 int	bnx_ioctl(struct ifnet *, u_long, caddr_t);
 void	bnx_watchdog(struct ifnet *);
 int	bnx_ifmedia_upd(struct ifnet *);
@@ -873,7 +873,7 @@ bnx_attachhook(struct device *self)
 	ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
 	ifp->if_xflags = IFXF_MPSAFE;
 	ifp->if_ioctl = bnx_ioctl;
-	ifp->if_start = bnx_start;
+	ifp->if_qstart = bnx_start;
 	ifp->if_watchdog = bnx_watchdog;
 	IFQ_SET_MAXLEN(&ifp->if_snd, USABLE_TX_BD - 1);
 	bcopy(sc->eaddr, sc->arpcom.ac_enaddr, ETHER_ADDR_LEN);
@@ -4865,15 +4865,16 @@ bnx_tx_encap(struct bnx_softc *sc, struct mbuf *m, int *used)
 /*   Nothing.                                                               */
 /****************************************************************************/
 void
-bnx_start(struct ifnet *ifp)
+bnx_start(struct ifqueue *ifq)
 {
+	struct ifnet		*ifp = ifq->ifq_if;
 	struct bnx_softc	*sc = ifp->if_softc;
 	struct mbuf		*m_head = NULL;
 	int			used;
 	u_int16_t		tx_prod, tx_chain_prod;
 
 	if (!sc->bnx_link) {
-		ifq_purge(&ifp->if_snd);
+		ifq_purge(ifq);
 		goto bnx_start_exit;
 	}
 
@@ -4895,11 +4896,11 @@ bnx_start(struct ifnet *ifp)
 			DBPRINT(sc, BNX_INFO_SEND, "TX chain is closed for "
 			    "business! Total tx_bd used = %d\n",
 			    sc->used_tx_bd + used);
-			ifq_set_oactive(&ifp->if_snd);
+			ifq_set_oactive(ifq);
 			break;
 		}
 
-		IFQ_DEQUEUE(&ifp->if_snd, m_head);
+		m_head = ifq_dequeue(ifq);
 		if (m_head == NULL)
 			break;
 
@@ -5149,11 +5150,8 @@ bnx_intr(void *xsc)
 
 		/* Start moving packets again */
 		if (ifp->if_flags & IFF_RUNNING &&
-		    !IFQ_IS_EMPTY(&ifp->if_snd)) {
-			KERNEL_LOCK();
-			bnx_start(ifp);
-			KERNEL_UNLOCK();
-		}
+		    !IFQ_IS_EMPTY(&ifp->if_snd))
+			ifq_start(&ifp->if_snd);
 	}
 
 out:
@@ -5486,8 +5484,8 @@ bnx_tick(void *xsc)
 	    IFM_SUBTYPE(mii->mii_media_active) != IFM_NONE) {
 		sc->bnx_link++;
 		/* Now that link is up, handle any outstanding TX traffic. */
-		if (!IFQ_IS_EMPTY(&ifp->if_snd))
-			bnx_start(ifp);
+		if (!ifq_empty(&ifp->if_snd))
+			ifq_start(&ifp->if_snd);
 	}
 
 bnx_tick_exit:
diff --git a/sys/dev/pci/if_em.c b/sys/dev/pci/if_em.c
index f900706f966..8676731af77 100644
--- a/sys/dev/pci/if_em.c
+++ b/sys/dev/pci/if_em.c
@@ -31,7 +31,7 @@ POSSIBILITY OF SUCH DAMAGE.
 
 ***************************************************************************/
 
-/* $OpenBSD: if_em.c,v 1.333 2017/01/22 10:17:38 dlg Exp $ */
+/* $OpenBSD: if_em.c,v 1.334 2017/01/24 03:57:35 dlg Exp $ */
 /* $FreeBSD: if_em.c,v 1.46 2004/09/29 18:28:28 mlaier Exp $ */
 
 #include <dev/pci/if_em.h>
@@ -206,7 +206,7 @@ void em_defer_attach(struct device*);
 int  em_detach(struct device *, int);
 int  em_activate(struct device *, int);
 int  em_intr(void *);
-void em_start(struct ifnet *);
+void em_start(struct ifqueue *);
 int  em_ioctl(struct ifnet *, u_long, caddr_t);
 void em_watchdog(struct ifnet *);
 void em_init(void *);
@@ -583,15 +583,16 @@ err_pci:
  **********************************************************************/
 
 void
-em_start(struct ifnet *ifp)
+em_start(struct ifqueue *ifq)
 {
+	struct ifnet *ifp = ifq->ifq_if;
 	struct em_softc *sc = ifp->if_softc;
 	u_int head, free, used;
 	struct mbuf *m;
 	int post = 0;
 
 	if (!sc->link_active) {
-		IFQ_PURGE(&ifp->if_snd);
+		ifq_purge(ifq);
 		return;
 	}
 
@@ -611,11 +612,11 @@ em_start(struct ifnet *ifp)
 	for (;;) {
 		/* use 2 because cksum setup can use an extra slot */
 		if (EM_MAX_SCATTER + 2 > free) {
-			ifq_set_oactive(&ifp->if_snd);
+			ifq_set_oactive(ifq);
 			break;
 		}
 
-		m = ifq_dequeue(&ifp->if_snd);
+		m = ifq_dequeue(ifq);
 		if (m == NULL)
 			break;
 
@@ -1870,7 +1871,7 @@ em_setup_interface(struct em_softc *sc)
 	ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
 	ifp->if_xflags = IFXF_MPSAFE;
 	ifp->if_ioctl = em_ioctl;
-	ifp->if_start = em_start;
+	ifp->if_qstart = em_start;
 	ifp->if_watchdog = em_watchdog;
 	ifp->if_hardmtu =
 		sc->hw.max_frame_size - ETHER_HDR_LEN - ETHER_CRC_LEN;
diff --git a/sys/dev/pci/if_ix.c b/sys/dev/pci/if_ix.c
index 72d5a3eaaaf..b24326f26e1 100644
--- a/sys/dev/pci/if_ix.c
+++ b/sys/dev/pci/if_ix.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: if_ix.c,v 1.149 2017/01/22 10:17:38 dlg Exp $	*/
+/*	$OpenBSD: if_ix.c,v 1.150 2017/01/24 03:57:35 dlg Exp $	*/
 
 /******************************************************************************
 
@@ -93,7 +93,7 @@ const struct pci_matchid ixgbe_devices[] = {
 int	ixgbe_probe(struct device *, void *, void *);
 void	ixgbe_attach(struct device *, struct device *, void *);
 int	ixgbe_detach(struct device *, int);
-void	ixgbe_start(struct ifnet *);
+void	ixgbe_start(struct ifqueue *);
 int	ixgbe_ioctl(struct ifnet *, u_long, caddr_t);
 int	ixgbe_rxrinfo(struct ix_softc *, struct if_rxrinfo *);
 void	ixgbe_watchdog(struct ifnet *);
@@ -379,14 +379,15 @@ ixgbe_detach(struct device *self, int flags)
  **********************************************************************/
 
 void
-ixgbe_start(struct ifnet * ifp)
+ixgbe_start(struct ifqueue *ifq)
 {
+	struct ifnet		*ifp = ifq->ifq_if;
 	struct ix_softc		*sc = ifp->if_softc;
 	struct tx_ring		*txr = sc->tx_rings;
 	struct mbuf  		*m_head;
 	int			 post = 0;
 
-	if (!(ifp->if_flags & IFF_RUNNING) || ifq_is_oactive(&ifp->if_snd))
+	if (!(ifp->if_flags & IFF_RUNNING) || ifq_is_oactive(ifq))
 		return;
 	if (!sc->link_up)
 		return;
@@ -398,11 +399,11 @@ ixgbe_start(struct ifnet * ifp)
 	for (;;) {
 		/* Check that we have the minimal number of TX descriptors. */
 		if (txr->tx_avail <= IXGBE_TX_OP_THRESHOLD) {
-			ifq_set_oactive(&ifp->if_snd);
+			ifq_set_oactive(ifq);
 			break;
 		}
 
-		m_head = ifq_dequeue(&ifp->if_snd);
+		m_head = ifq_dequeue(ifq);
 		if (m_head == NULL)
 			break;
 
@@ -1612,7 +1613,7 @@ ixgbe_setup_interface(struct ix_softc *sc)
 	ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
 	ifp->if_xflags = IFXF_MPSAFE;
 	ifp->if_ioctl = ixgbe_ioctl;
-	ifp->if_start = ixgbe_start;
+	ifp->if_qstart = ixgbe_start;
 	ifp->if_timer = 0;
 	ifp->if_watchdog = ixgbe_watchdog;
 	ifp->if_hardmtu = IXGBE_MAX_FRAME_SIZE -
diff --git a/sys/dev/pci/if_myx.c b/sys/dev/pci/if_myx.c
index 1d501028ffd..24a3c75e29d 100644
--- a/sys/dev/pci/if_myx.c
+++ b/sys/dev/pci/if_myx.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: if_myx.c,v 1.100 2017/01/22 10:17:38 dlg Exp $	*/
+/*	$OpenBSD: if_myx.c,v 1.101 2017/01/24 03:57:35 dlg Exp $	*/
 
 /*
  * Copyright (c) 2007 Reyk Floeter <reyk@openbsd.org>
@@ -201,7 +201,7 @@ void	 myx_up(struct myx_softc *);
 void	 myx_iff(struct myx_softc *);
 void	 myx_down(struct myx_softc *);
 
-void	 myx_start(struct ifnet *);
+void	 myx_start(struct ifqueue *);
 void	 myx_write_txd_tail(struct myx_softc *, struct myx_slot *, u_int8_t,
 	    u_int32_t, u_int);
 int	 myx_load_mbuf(struct myx_softc *, struct myx_slot *, struct mbuf *);
@@ -510,7 +510,7 @@ myx_attachhook(struct device *self)
 	ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
 	ifp->if_xflags = IFXF_MPSAFE;
 	ifp->if_ioctl = myx_ioctl;
-	ifp->if_start = myx_start;
+	ifp->if_qstart = myx_start;
 	ifp->if_watchdog = myx_watchdog;
 	ifp->if_hardmtu = MYX_RXBIG_SIZE;
 	strlcpy(ifp->if_xname, DEVNAME(sc), IFNAMSIZ);
@@ -1200,10 +1200,9 @@ myx_up(struct myx_softc *sc)
 		goto empty_rx_ring_big;
 	}
 
-	ifq_clr_oactive(&ifp->if_snd);
-	SET(ifp->if_flags, IFF_RUNNING);
 	myx_iff(sc);
-	if_start(ifp);
+	SET(ifp->if_flags, IFF_RUNNING);
+	ifq_restart(&ifp->if_snd);
 
 	return;
 
@@ -1422,8 +1421,9 @@ myx_write_txd_tail(struct myx_softc *sc, struct myx_slot *ms, u_int8_t flags,
 }
 
 void
-myx_start(struct ifnet *ifp)
+myx_start(struct ifqueue *ifq)
 {
+	struct ifnet			*ifp = ifq->ifq_if;
 	struct myx_tx_desc		txd;
 	struct myx_softc		*sc = ifp->if_softc;
 	struct myx_slot			*ms;
@@ -1448,11 +1448,11 @@ myx_start(struct ifnet *ifp)
 
 	for (;;) {
 		if (used + sc->sc_tx_nsegs + 1 > free) {
-			ifq_set_oactive(&ifp->if_snd);
+			ifq_set_oactive(ifq);
 			break;
 		}
 
-		IFQ_DEQUEUE(&ifp->if_snd, m);
+		m = ifq_dequeue(ifq);
 		if (m == NULL)
 			break;
 
diff --git a/sys/dev/pv/if_hvn.c b/sys/dev/pv/if_hvn.c
index 3fd0b1fb353..295fc8ae05f 100644
--- a/sys/dev/pv/if_hvn.c
+++ b/sys/dev/pv/if_hvn.c
@@ -179,7 +179,7 @@ void	hvn_media_status(struct ifnet *, struct ifmediareq *);
 int	hvn_iff(struct hvn_softc *);
 void	hvn_init(struct hvn_softc *);
 void	hvn_stop(struct hvn_softc *);
-void	hvn_start(struct ifnet *);
+void	hvn_start(struct ifqueue *);
 int	hvn_encap(struct hvn_softc *, struct mbuf *, struct hvn_tx_desc **);
 void	hvn_decap(struct hvn_softc *, struct hvn_tx_desc *);
 void	hvn_txeof(struct hvn_softc *, uint64_t);
@@ -266,7 +266,7 @@ hvn_attach(struct device *parent, struct device *self, void *aux)
 	ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
 	ifp->if_xflags = IFXF_MPSAFE;
 	ifp->if_ioctl = hvn_ioctl;
-	ifp->if_start = hvn_start;
+	ifp->if_qstart = hvn_start;
 	ifp->if_softc = sc;
 
 	ifp->if_capabilities = IFCAP_CSUM_IPv4 | IFCAP_CSUM_TCPv4 |
@@ -320,7 +320,7 @@ hvn_attach(struct device *parent, struct device *self, void *aux)
 	hvn_rx_ring_destroy(sc);
 	hvn_tx_ring_destroy(sc);
 	hvn_nvs_detach(sc);
-	if (ifp->if_start)
+	if (ifp->if_qstart)
 		if_detach(ifp);
 }
 
@@ -441,23 +441,21 @@ hvn_stop(struct hvn_softc *sc)
 }
 
 void
-hvn_start(struct ifnet *ifp)
+hvn_start(struct ifqueue *ifq)
 {
+	struct ifnet *ifp = ifq->ifq_if;
 	struct hvn_softc *sc = ifp->if_softc;
 	struct hvn_tx_desc *txd;
 	struct mbuf *m;
 
-	if (!(ifp->if_flags & IFF_RUNNING) || ifq_is_oactive(&ifp->if_snd))
-		return;
-
 	for (;;) {
 		if (!sc->sc_tx_avail) {
 			/* transient */
-			ifq_set_oactive(&ifp->if_snd);
+			ifq_set_oactive(ifq);
 			break;
 		}
 
-		m = ifq_dequeue(&ifp->if_snd);
+		m = ifq_dequeue(ifq);
 		if (m == NULL)
 			break;
 
diff --git a/sys/dev/pv/if_xnf.c b/sys/dev/pv/if_xnf.c
index 9838a8d645a..5ba44603661 100644
--- a/sys/dev/pv/if_xnf.c
+++ b/sys/dev/pv/if_xnf.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: if_xnf.c,v 1.47 2017/01/22 10:17:39 dlg Exp $	*/
+/*	$OpenBSD: if_xnf.c,v 1.48 2017/01/24 03:57:35 dlg Exp $	*/
 
 /*
  * Copyright (c) 2015, 2016 Mike Belopuhov
@@ -199,7 +199,7 @@ void	xnf_media_status(struct ifnet *, struct ifmediareq *);
 int	xnf_iff(struct xnf_softc *);
 void	xnf_init(struct xnf_softc *);
 void	xnf_stop(struct xnf_softc *);
-void	xnf_start(struct ifnet *);
+void	xnf_start(struct ifqueue *);
 int	xnf_encap(struct xnf_softc *, struct mbuf *, uint32_t *);
 void	xnf_intr(void *);
 void	xnf_watchdog(struct ifnet *);
@@ -292,7 +292,7 @@ xnf_attach(struct device *parent, struct device *self, void *aux)
 	ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
 	ifp->if_xflags = IFXF_MPSAFE;
 	ifp->if_ioctl = xnf_ioctl;
-	ifp->if_start = xnf_start;
+	ifp->if_qstart = xnf_start;
 	ifp->if_watchdog = xnf_watchdog;
 	ifp->if_softc = sc;
 
@@ -477,17 +477,15 @@ xnf_stop(struct xnf_softc *sc)
 }
 
 void
-xnf_start(struct ifnet *ifp)
+xnf_start(struct ifqueue *ifq)
 {
+	struct ifnet *ifp = ifq->ifq_if;
 	struct xnf_softc *sc = ifp->if_softc;
 	struct xnf_tx_ring *txr = sc->sc_tx_ring;
 	struct mbuf *m;
 	int pkts = 0;
 	uint32_t prod, oprod;
 
-	if (!(ifp->if_flags & IFF_RUNNING) || ifq_is_oactive(&ifp->if_snd))
-		return;
-
 	bus_dmamap_sync(sc->sc_dmat, sc->sc_tx_rmap, 0, 0,
 	    BUS_DMASYNC_POSTREAD);
 
@@ -497,10 +495,11 @@ xnf_start(struct ifnet *ifp)
 		if ((XNF_TX_DESC - (prod - sc->sc_tx_cons)) <
 		    sc->sc_tx_frags) {
 			/* transient */
-			ifq_set_oactive(&ifp->if_snd);
+			ifq_set_oactive(ifq);
 			break;
 		}
-		m = ifq_dequeue(&ifp->if_snd);
+
+		m = ifq_dequeue(ifq);
 		if (m == NULL)
 			break;
 
diff --git a/sys/net/hfsc.c b/sys/net/hfsc.c
index 447ca4f9cd0..f8e67722bb7 100644
--- a/sys/net/hfsc.c
+++ b/sys/net/hfsc.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: hfsc.c,v 1.34 2017/01/22 04:48:23 dlg Exp $	*/
+/*	$OpenBSD: hfsc.c,v 1.35 2017/01/24 03:57:35 dlg Exp $	*/
 
 /*
  * Copyright (c) 2012-2013 Henning Brauer <henning@openbsd.org>
@@ -259,20 +259,22 @@ struct pool	hfsc_class_pl, hfsc_internal_sc_pl;
  * ifqueue glue.
  */
 
-void		*hfsc_alloc(void *);
-void		 hfsc_free(void *);
+unsigned int	 hfsc_idx(unsigned int, const struct mbuf *);
 int		 hfsc_enq(struct ifqueue *, struct mbuf *);
 struct mbuf	*hfsc_deq_begin(struct ifqueue *, void **);
 void		 hfsc_deq_commit(struct ifqueue *, struct mbuf *, void *);
 void		 hfsc_purge(struct ifqueue *, struct mbuf_list *);
+void		*hfsc_alloc(unsigned int, void *);
+void		 hfsc_free(unsigned int, void *);
 
 const struct ifq_ops hfsc_ops = {
-	hfsc_alloc,
-	hfsc_free,
+	hfsc_idx,
 	hfsc_enq,
 	hfsc_deq_begin,
 	hfsc_deq_commit,
 	hfsc_purge,
+	hfsc_alloc,
+	hfsc_free,
 };
 
 const struct ifq_ops * const ifq_hfsc_ops = &hfsc_ops;
@@ -414,13 +416,26 @@ hfsc_pf_qstats(struct pf_queuespec *q, void *ubuf, int *nbytes)
 void
 hfsc_pf_free(struct hfsc_if *hif)
 {
-	hfsc_free(hif);
+	hfsc_free(0, hif);
+}
+
+unsigned int
+hfsc_idx(unsigned int nqueues, const struct mbuf *m)
+{
+	/*
+	 * hfsc can only function on a single ifq and the stack understands
+	 * this. when the first ifq on an interface is switched to hfsc,
+	 * this gets used to map all mbufs to the first and only ifq that
+	 * is set up for hfsc.
+	 */
+	return (0);
 }
 
 void *
-hfsc_alloc(void *q)
+hfsc_alloc(unsigned int idx, void *q)
 {
 	struct hfsc_if *hif = q;
+	KASSERT(idx == 0); /* when hfsc is enabled we only use the first ifq */
 	KASSERT(hif != NULL);
 
 	timeout_add(&hif->hif_defer, 1);
@@ -428,12 +443,13 @@ hfsc_alloc(void *q)
 }
 
 void
-hfsc_free(void *q)
+hfsc_free(unsigned int idx, void *q)
 {
 	struct hfsc_if *hif = q;
 	int i;
 
 	KERNEL_ASSERT_LOCKED();
+	KASSERT(idx == 0); /* when hfsc is enabled we only use the first ifq */
 
 	timeout_del(&hif->hif_defer);
 
@@ -758,18 +774,16 @@ void
 hfsc_deferred(void *arg)
 {
 	struct ifnet *ifp = arg;
+	struct ifqueue *ifq = &ifp->if_snd;
 	struct hfsc_if *hif;
-	int s;
 
 	KERNEL_ASSERT_LOCKED();
-	KASSERT(HFSC_ENABLED(&ifp->if_snd));
+	KASSERT(HFSC_ENABLED(ifq));
 
-	s = splnet();
-	if (!IFQ_IS_EMPTY(&ifp->if_snd))
-		if_start(ifp);
-	splx(s);
+	if (!ifq_empty(ifq))
+		(*ifp->if_qstart)(ifq);
 
-	hif = ifp->if_snd.ifq_q;
+	hif = ifq->ifq_q;
 
 	/* XXX HRTIMER nearest virtual/fit time is likely less than 1/HZ. */
 	timeout_add(&hif->hif_defer, 1);
diff --git a/sys/net/if.c b/sys/net/if.c
index b01f9a539d5..3e10955e1be 100644
--- a/sys/net/if.c
+++ b/sys/net/if.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: if.c,v 1.478 2017/01/23 11:37:29 mpi Exp $	*/
+/*	$OpenBSD: if.c,v 1.479 2017/01/24 03:57:35 dlg Exp $	*/
 /*	$NetBSD: if.c,v 1.35 1996/05/07 05:26:04 thorpej Exp $	*/
 
 /*
@@ -136,7 +136,7 @@ void	if_attach_common(struct ifnet *);
 int	if_setrdomain(struct ifnet *, int);
 void	if_slowtimo(void *);
 
-void	if_detached_start(struct ifnet *);
+void	if_detached_qstart(struct ifqueue *);
 int	if_detached_ioctl(struct ifnet *, u_long, caddr_t);
 
 int	if_getgroup(caddr_t, struct ifnet *);
@@ -161,7 +161,7 @@ void	if_netisr(void *);
 void	ifa_print_all(void);
 #endif
 
-void	if_start_locked(struct ifnet *);
+void	if_qstart_compat(struct ifqueue *);
 
 /*
  * interface index map
@@ -527,12 +527,53 @@ if_attach(struct ifnet *ifp)
 }
 
 void
+if_attach_queues(struct ifnet *ifp, unsigned int nqs)
+{
+	struct ifqueue **map;
+	struct ifqueue *ifq;
+	int i;
+
+	KASSERT(ifp->if_ifqs == ifp->if_snd.ifq_ifqs);
+	KASSERT(nqs != 0);
+
+	map = mallocarray(sizeof(*map), nqs, M_DEVBUF, M_WAITOK);
+
+	ifp->if_snd.ifq_softc = NULL;
+	map[0] = &ifp->if_snd;
+
+	for (i = 1; i < nqs; i++) {
+		ifq = malloc(sizeof(*ifq), M_DEVBUF, M_WAITOK|M_ZERO);
+		ifq_set_maxlen(ifq, ifp->if_snd.ifq_maxlen);
+		ifq_init(ifq, ifp, i);
+		map[i] = ifq;
+	}
+
+	ifp->if_ifqs = map;
+	ifp->if_nifqs = nqs;
+}
+
+void
 if_attach_common(struct ifnet *ifp)
 {
 	TAILQ_INIT(&ifp->if_addrlist);
 	TAILQ_INIT(&ifp->if_maddrlist);
 
-	ifq_init(&ifp->if_snd, ifp);
+	if (!ISSET(ifp->if_xflags, IFXF_MPSAFE)) {
+		KASSERTMSG(ifp->if_qstart == NULL,
+		    "%s: if_qstart set without MPSAFE set", ifp->if_xname);
+		ifp->if_qstart = if_qstart_compat;
+	} else {
+		KASSERTMSG(ifp->if_start == NULL,
+		    "%s: if_start set with MPSAFE set", ifp->if_xname);
+		KASSERTMSG(ifp->if_qstart != NULL,
+		    "%s: if_qstart not set with MPSAFE set", ifp->if_xname);
+	}
+
+	ifq_init(&ifp->if_snd, ifp, 0);
+
+	ifp->if_snd.ifq_ifqs[0] = &ifp->if_snd;
+	ifp->if_ifqs = ifp->if_snd.ifq_ifqs;
+	ifp->if_nifqs = 1;
 
 	ifp->if_addrhooks = malloc(sizeof(*ifp->if_addrhooks),
 	    M_TEMP, M_WAITOK);
@@ -560,22 +601,44 @@ if_attach_common(struct ifnet *ifp)
 }
 
 void
-if_start(struct ifnet *ifp)
+if_attach_ifq(struct ifnet *ifp, const struct ifq_ops *newops, void *args)
 {
-	if (ISSET(ifp->if_xflags, IFXF_MPSAFE))
-		ifq_start(&ifp->if_snd);
-	else
-		if_start_locked(ifp);
+	/*
+	 * only switch the ifq_ops on the first ifq on an interface.
+	 *
+	 * the only ifq_ops we provide priq and hfsc, and hfsc only
+	 * works on a single ifq. because the code uses the ifq_ops
+	 * on the first ifq (if_snd) to select a queue for an mbuf,
+	 * by switching only the first one we change both the algorithm
+	 * and force the routing of all new packets to it.
+	 */
+	ifq_attach(&ifp->if_snd, newops, args);
 }
 
 void
-if_start_locked(struct ifnet *ifp)
+if_start(struct ifnet *ifp)
 {
+	KASSERT(ifp->if_qstart == if_qstart_compat);
+	if_qstart_compat(&ifp->if_snd);
+}
+void
+if_qstart_compat(struct ifqueue *ifq)
+{
+	struct ifnet *ifp = ifq->ifq_if;
 	int s;
 
+	/*
+	 * the stack assumes that an interface can have multiple
+	 * transmit rings, but a lot of drivers are still written
+	 * so that interfaces and send rings have a 1:1 mapping.
+	 * this provides compatability between the stack and the older
+	 * drivers by translating from the only queue they have
+	 * (ifp->if_snd) back to the interface and calling if_start.
+ 	 */
+
 	KERNEL_LOCK();
 	s = splnet();
-	ifp->if_start(ifp);
+	(*ifp->if_start)(ifp);
 	splx(s);
 	KERNEL_UNLOCK();
 }
@@ -583,7 +646,9 @@ if_start_locked(struct ifnet *ifp)
 int
 if_enqueue(struct ifnet *ifp, struct mbuf *m)
 {
-	int error = 0;
+	unsigned int idx;
+	struct ifqueue *ifq;
+	int error;
 
 #if NBRIDGE > 0
 	if (ifp->if_bridgeport && (m->m_flags & M_PROTO1) == 0) {
@@ -599,14 +664,17 @@ if_enqueue(struct ifnet *ifp, struct mbuf *m)
 #endif	/* NPF > 0 */
 
 	/*
-	 * Queue message on interface, and start output if interface
-	 * not yet active.
+	 * use the operations on the first ifq to pick which of the array
+	 * gets this mbuf.
 	 */
-	IFQ_ENQUEUE(&ifp->if_snd, m, error);
+	idx = ifq_idx(&ifp->if_snd, ifp->if_nifqs, m);
+	ifq = ifp->if_ifqs[idx];
+
+	error = ifq_enqueue(ifq, m);
 	if (error)
 		return (error);
 
-	if_start(ifp);
+	ifq_start(ifq);
 
 	return (0);
 }
@@ -943,7 +1011,7 @@ if_detach(struct ifnet *ifp)
 	/* Other CPUs must not have a reference before we start destroying. */
 	if_idxmap_remove(ifp);
 
-	ifp->if_start = if_detached_start;
+	ifp->if_qstart = if_detached_qstart;
 	ifp->if_ioctl = if_detached_ioctl;
 	ifp->if_watchdog = NULL;
 
@@ -1017,7 +1085,16 @@ if_detach(struct ifnet *ifp)
 	splx(s2);
 	NET_UNLOCK(s);
 
-	ifq_destroy(&ifp->if_snd);
+	for (i = 0; i < ifp->if_nifqs; i++)
+		ifq_destroy(ifp->if_ifqs[i]);
+	if (ifp->if_ifqs != ifp->if_snd.ifq_ifqs) {
+		for (i = 1; i < ifp->if_nifqs; i++) {
+			free(ifp->if_ifqs[i], M_DEVBUF,
+			    sizeof(struct ifqueue));
+		}
+		free(ifp->if_ifqs, M_DEVBUF,
+		    sizeof(struct ifqueue *) * ifp->if_nifqs);
+	}
 }
 
 /*
@@ -2166,21 +2243,24 @@ ifconf(u_long cmd, caddr_t data)
 void
 if_getdata(struct ifnet *ifp, struct if_data *data)
 {
+	unsigned int i;
 	struct ifqueue *ifq;
 	uint64_t opackets = 0;
 	uint64_t obytes = 0;
 	uint64_t omcasts = 0;
 	uint64_t oqdrops = 0;
 
-	ifq = &ifp->if_snd;
+	for (i = 0; i < ifp->if_nifqs; i++) {
+		ifq = ifp->if_ifqs[i];
 
-	mtx_enter(&ifq->ifq_mtx);
-	opackets += ifq->ifq_packets;
-	obytes += ifq->ifq_bytes;
-	oqdrops += ifq->ifq_qdrops;
-	omcasts += ifq->ifq_mcasts;
-	/* ifq->ifq_errors */
-	mtx_leave(&ifq->ifq_mtx);
+		mtx_enter(&ifq->ifq_mtx);
+		opackets += ifq->ifq_packets;
+		obytes += ifq->ifq_bytes;
+		oqdrops += ifq->ifq_qdrops;
+		omcasts += ifq->ifq_mcasts;
+		mtx_leave(&ifq->ifq_mtx);
+		/* ifq->ifq_errors */
+	}
 
 	*data = ifp->if_data;
 	data->ifi_opackets += opackets;
@@ -2195,9 +2275,9 @@ if_getdata(struct ifnet *ifp, struct if_data *data)
  * fiddle with the if during detach.
  */
 void
-if_detached_start(struct ifnet *ifp)
+if_detached_qstart(struct ifqueue *ifq)
 {
-	IFQ_PURGE(&ifp->if_snd);
+	ifq_purge(ifq);
 }
 
 int
diff --git a/sys/net/if.h b/sys/net/if.h
index cbc25aaecb5..549f854993a 100644
--- a/sys/net/if.h
+++ b/sys/net/if.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: if.h,v 1.184 2017/01/23 11:37:29 mpi Exp $	*/
+/*	$OpenBSD: if.h,v 1.185 2017/01/24 03:57:35 dlg Exp $	*/
 /*	$NetBSD: if.h,v 1.23 1996/05/07 02:40:27 thorpej Exp $	*/
 
 /*
@@ -457,10 +457,13 @@ struct if_parent {
 #ifdef _KERNEL
 struct socket;
 struct ifnet;
+struct ifq_ops;
 
 void	if_alloc_sadl(struct ifnet *);
 void	if_free_sadl(struct ifnet *);
 void	if_attach(struct ifnet *);
+void	if_attach_queues(struct ifnet *, unsigned int);
+void	if_attach_ifq(struct ifnet *, const struct ifq_ops *, void *);
 void	if_attachtail(struct ifnet *);
 void	if_attachhead(struct ifnet *);
 void	if_deactivate(struct ifnet *);
diff --git a/sys/net/if_mpw.c b/sys/net/if_mpw.c
index cd51b2387e9..aaabf84498f 100644
--- a/sys/net/if_mpw.c
+++ b/sys/net/if_mpw.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: if_mpw.c,v 1.17 2017/01/23 11:37:29 mpi Exp $ */
+/*	$OpenBSD: if_mpw.c,v 1.18 2017/01/24 03:57:35 dlg Exp $ */
 
 /*
  * Copyright (c) 2015 Rafael Zalamena <rzalamena@openbsd.org>
@@ -333,7 +333,7 @@ mpw_output(struct ifnet *ifp, struct mbuf *m, struct sockaddr *sa,
 }
 
 #if NVLAN > 0
-extern void vlan_start(struct ifnet *ifp);
+extern void vlan_start(struct ifqueue *);
 
 /*
  * This routine handles VLAN tag reinsertion in packets flowing through
@@ -350,7 +350,7 @@ mpw_vlan_handle(struct mbuf *m, struct mpw_softc *sc)
 	uint16_t tag = 0;
  
 	ifp = if_get(m->m_pkthdr.ph_ifidx);
-	if (ifp != NULL && ifp->if_start == vlan_start &&
+	if (ifp != NULL && ifp->if_qstart == vlan_start &&
 	    ISSET(ifp->if_flags, IFF_RUNNING)) {
  		ifv = ifp->if_softc;
 		type = ifv->ifv_type;
diff --git a/sys/net/if_var.h b/sys/net/if_var.h
index 3059acef552..dab939a648c 100644
--- a/sys/net/if_var.h
+++ b/sys/net/if_var.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: if_var.h,v 1.79 2017/01/21 01:32:19 patrick Exp $	*/
+/*	$OpenBSD: if_var.h,v 1.80 2017/01/24 03:57:35 dlg Exp $	*/
 /*	$NetBSD: if.h,v 1.23 1996/05/07 02:40:27 thorpej Exp $	*/
 
 /*
@@ -157,7 +157,12 @@ struct ifnet {				/* and the entries */
 					/* timer routine */
 	void	(*if_watchdog)(struct ifnet *);
 	int	(*if_wol)(struct ifnet *, int);
-	struct	ifqueue if_snd;		/* output queue */
+
+	struct	ifqueue if_snd;		/* transmit queue */
+	struct	ifqueue **if_ifqs;	/* pointer to an array of sndqs */
+	void	(*if_qstart)(struct ifqueue *);
+	unsigned int if_nifqs;
+
 	struct sockaddr_dl *if_sadl;	/* pointer to our sockaddr_dl */
 
 	void	*if_afdata[AF_MAX];
diff --git a/sys/net/if_vlan.c b/sys/net/if_vlan.c
index 8ec731e89c3..a0166e6b8c7 100644
--- a/sys/net/if_vlan.c
+++ b/sys/net/if_vlan.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: if_vlan.c,v 1.169 2017/01/23 11:37:29 mpi Exp $	*/
+/*	$OpenBSD: if_vlan.c,v 1.170 2017/01/24 03:57:35 dlg Exp $	*/
 
 /*
  * Copyright 1998 Massachusetts Institute of Technology
@@ -85,7 +85,7 @@ int	vlan_clone_create(struct if_clone *, int);
 int	vlan_clone_destroy(struct ifnet *);
 
 int	vlan_input(struct ifnet *, struct mbuf *, void *);
-void	vlan_start(struct ifnet *ifp);
+void	vlan_start(struct ifqueue *ifq);
 int	vlan_ioctl(struct ifnet *ifp, u_long cmd, caddr_t addr);
 
 int	vlan_up(struct ifvlan *);
@@ -175,7 +175,7 @@ vlan_clone_create(struct if_clone *ifc, int unit)
 
 	ifp->if_flags = IFF_BROADCAST | IFF_MULTICAST;
 	ifp->if_xflags = IFXF_CLONED|IFXF_MPSAFE;
-	ifp->if_start = vlan_start;
+	ifp->if_qstart = vlan_start;
 	ifp->if_ioctl = vlan_ioctl;
 	ifp->if_hardmtu = 0xffff;
 	ifp->if_link_state = LINK_STATE_DOWN;
@@ -238,8 +238,9 @@ vlan_mplstunnel(int ifidx)
 }
 
 void
-vlan_start(struct ifnet *ifp)
+vlan_start(struct ifqueue *ifq)
 {
+	struct ifnet	*ifp = ifq->ifq_if;
 	struct ifvlan   *ifv;
 	struct ifnet	*ifp0;
 	struct mbuf	*m;
@@ -249,15 +250,11 @@ vlan_start(struct ifnet *ifp)
 	ifp0 = if_get(ifv->ifv_ifp0);
 	if (ifp0 == NULL || (ifp0->if_flags & (IFF_UP|IFF_RUNNING)) !=
 	    (IFF_UP|IFF_RUNNING)) {
-		ifq_purge(&ifp->if_snd);
+		ifq_purge(ifq);
 		goto leave;
 	}
 
-	for (;;) {
-		IFQ_DEQUEUE(&ifp->if_snd, m);
-		if (m == NULL)
-			break;
-
+	while ((m = ifq_dequeue(ifq)) != NULL) {
 #if NBPFILTER > 0
 		if (ifp->if_bpf)
 			bpf_mtap_ether(ifp->if_bpf, m, BPF_DIRECTION_OUT);
@@ -296,6 +293,7 @@ vlan_start(struct ifnet *ifp)
 
 		if (if_enqueue(ifp0, m)) {
 			ifp->if_oerrors++;
+			ifq->ifq_errors++;
 			continue;
 		}
 	}
diff --git a/sys/net/ifq.c b/sys/net/ifq.c
index e92d6f09490..40731637a2c 100644
--- a/sys/net/ifq.c
+++ b/sys/net/ifq.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: ifq.c,v 1.5 2017/01/20 03:48:03 dlg Exp $ */
+/*	$OpenBSD: ifq.c,v 1.6 2017/01/24 03:57:35 dlg Exp $ */
 
 /*
  * Copyright (c) 2015 David Gwynne <dlg@openbsd.org>
@@ -28,20 +28,23 @@
 /*
  * priq glue
  */
-void		*priq_alloc(void *);
-void		 priq_free(void *);
+unsigned int	 priq_idx(unsigned int, const struct mbuf *);
 int		 priq_enq(struct ifqueue *, struct mbuf *);
 struct mbuf	*priq_deq_begin(struct ifqueue *, void **);
 void		 priq_deq_commit(struct ifqueue *, struct mbuf *, void *);
 void		 priq_purge(struct ifqueue *, struct mbuf_list *);
 
+void		*priq_alloc(unsigned int, void *);
+void		 priq_free(unsigned int, void *);
+
 const struct ifq_ops priq_ops = {
-	priq_alloc,
-	priq_free,
+	priq_idx,
 	priq_enq,
 	priq_deq_begin,
 	priq_deq_commit,
 	priq_purge,
+	priq_alloc,
+	priq_free,
 };
 
 const struct ifq_ops * const ifq_priq_ops = &priq_ops;
@@ -119,7 +122,7 @@ ifq_start_task(void *p)
 	    ifq_empty(ifq) || ifq_is_oactive(ifq))
 		return;
 
-	ifp->if_start(ifp);
+	ifp->if_qstart(ifq);
 }
 
 void
@@ -129,7 +132,7 @@ ifq_restart_task(void *p)
 	struct ifnet *ifp = ifq->ifq_if;
 
 	ifq_clr_oactive(ifq);
-	ifp->if_start(ifp);
+	ifp->if_qstart(ifq);
 }
 
 void
@@ -167,19 +170,26 @@ ifq_barrier_task(void *p)
  */
 
 void
-ifq_init(struct ifqueue *ifq, struct ifnet *ifp)
+ifq_init(struct ifqueue *ifq, struct ifnet *ifp, unsigned int idx)
 {
 	ifq->ifq_if = ifp;
+	ifq->ifq_softc = NULL;
 
 	mtx_init(&ifq->ifq_mtx, IPL_NET);
 	ifq->ifq_qdrops = 0;
 
 	/* default to priq */
 	ifq->ifq_ops = &priq_ops;
-	ifq->ifq_q = priq_ops.ifqop_alloc(NULL);
+	ifq->ifq_q = priq_ops.ifqop_alloc(idx, NULL);
 
 	ifq->ifq_len = 0;
 
+	ifq->ifq_packets = 0;
+	ifq->ifq_bytes = 0;
+	ifq->ifq_qdrops = 0;
+	ifq->ifq_errors = 0;
+	ifq->ifq_mcasts = 0;
+
 	mtx_init(&ifq->ifq_task_mtx, IPL_NET);
 	TAILQ_INIT(&ifq->ifq_task_list);
 	ifq->ifq_serializer = NULL;
@@ -189,6 +199,8 @@ ifq_init(struct ifqueue *ifq, struct ifnet *ifp)
 
 	if (ifq->ifq_maxlen == 0)
 		ifq_set_maxlen(ifq, IFQ_MAXLEN);
+
+	ifq->ifq_idx = idx;
 }
 
 void
@@ -200,7 +212,7 @@ ifq_attach(struct ifqueue *ifq, const struct ifq_ops *newops, void *opsarg)
 	const struct ifq_ops *oldops;
 	void *newq, *oldq;
 
-	newq = newops->ifqop_alloc(opsarg);
+	newq = newops->ifqop_alloc(ifq->ifq_idx, opsarg);
 
 	mtx_enter(&ifq->ifq_mtx);
 	ifq->ifq_ops->ifqop_purge(ifq, &ml);
@@ -221,7 +233,7 @@ ifq_attach(struct ifqueue *ifq, const struct ifq_ops *newops, void *opsarg)
 	}
 	mtx_leave(&ifq->ifq_mtx);
 
-	oldops->ifqop_free(oldq);
+	oldops->ifqop_free(ifq->ifq_idx, oldq);
 
 	ml_purge(&free_ml);
 }
@@ -234,7 +246,7 @@ ifq_destroy(struct ifqueue *ifq)
 	/* don't need to lock because this is the last use of the ifq */
 
 	ifq->ifq_ops->ifqop_purge(ifq, &ml);
-	ifq->ifq_ops->ifqop_free(ifq->ifq_q);
+	ifq->ifq_ops->ifqop_free(ifq->ifq_idx, ifq->ifq_q);
 
 	ml_purge(&ml);
 }
@@ -368,14 +380,25 @@ ifq_q_leave(struct ifqueue *ifq, void *q)
  * priq implementation
  */
 
+unsigned int
+priq_idx(unsigned int nqueues, const struct mbuf *m)
+{
+	unsigned int flow = 0;
+
+	if (ISSET(m->m_pkthdr.ph_flowid, M_FLOWID_VALID))
+		flow = m->m_pkthdr.ph_flowid & M_FLOWID_MASK;
+
+	return (flow % nqueues);
+}
+
 void *
-priq_alloc(void *null)
+priq_alloc(unsigned int idx, void *null)
 {
 	return (malloc(sizeof(struct priq), M_DEVBUF, M_WAITOK | M_ZERO));
 }
 
 void
-priq_free(void *pq)
+priq_free(unsigned int idx, void *pq)
 {
 	free(pq, M_DEVBUF, sizeof(struct priq));
 }
diff --git a/sys/net/ifq.h b/sys/net/ifq.h
index 4dbef36090e..d159f80d0fe 100644
--- a/sys/net/ifq.h
+++ b/sys/net/ifq.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: ifq.h,v 1.7 2017/01/22 04:48:23 dlg Exp $ */
+/*	$OpenBSD: ifq.h,v 1.8 2017/01/24 03:57:35 dlg Exp $ */
 
 /*
  * Copyright (c) 2015 David Gwynne <dlg@openbsd.org>
@@ -25,6 +25,18 @@ struct ifq_ops;
 
 struct ifqueue {
 	struct ifnet		*ifq_if;
+	union {
+		void			*_ifq_softc;
+		/*
+		 * a rings sndq is found by looking up an array of pointers.
+		 * by default we only have one sndq and the default drivers
+		 * dont use ifq_softc, so we can borrow it for the map until
+		 * we need to allocate a proper map.
+		 */
+		struct ifqueue		*_ifq_ifqs[1];
+	} _ifq_ptr;
+#define ifq_softc		 _ifq_ptr._ifq_softc
+#define ifq_ifqs		 _ifq_ptr._ifq_ifqs
 
 	/* mbuf handling */
 	struct mutex		 ifq_mtx;
@@ -49,7 +61,9 @@ struct ifqueue {
 	struct task		 ifq_start;
 	struct task		 ifq_restart;
 
+	/* properties */
 	unsigned int		 ifq_maxlen;
+	unsigned int		 ifq_idx;
 };
 
 #ifdef _KERNEL
@@ -308,21 +322,23 @@ struct ifqueue {
  */
 
 struct ifq_ops {
-	void			*(*ifqop_alloc)(void *);
-	void			 (*ifqop_free)(void *);
+	unsigned int		 (*ifqop_idx)(unsigned int,
+				    const struct mbuf *);
 	int			 (*ifqop_enq)(struct ifqueue *, struct mbuf *);
 	struct mbuf		*(*ifqop_deq_begin)(struct ifqueue *, void **);
 	void			 (*ifqop_deq_commit)(struct ifqueue *,
 				    struct mbuf *, void *);
 	void			 (*ifqop_purge)(struct ifqueue *,
 				    struct mbuf_list *);
+	void			*(*ifqop_alloc)(unsigned int, void *);
+	void			 (*ifqop_free)(unsigned int, void *);
 };
 
 /*
  * Interface send queues.
  */
 
-void		 ifq_init(struct ifqueue *, struct ifnet *);
+void		 ifq_init(struct ifqueue *, struct ifnet *, unsigned int);
 void		 ifq_attach(struct ifqueue *, const struct ifq_ops *, void *);
 void		 ifq_destroy(struct ifqueue *);
 int		 ifq_enqueue_try(struct ifqueue *, struct mbuf *);
@@ -372,6 +388,12 @@ ifq_restart(struct ifqueue *ifq)
 	ifq_serialize(ifq, &ifq->ifq_restart);
 }
 
+static inline unsigned int
+ifq_idx(struct ifqueue *ifq, unsigned int nifqs, const struct mbuf *m)
+{
+	return ((*ifq->ifq_ops->ifqop_idx)(nifqs, m));
+}
+
 #define IFQ_ASSERT_SERIALIZED(_ifq)	KASSERT(ifq_is_serialized(_ifq))
 
 extern const struct ifq_ops * const ifq_priq_ops;