split if_enqueue up so drivers can replace ifq handling if needed

if_enqueue() still makes sure packets get handled by pf on the way
out, and seen by bridge if needed. however instead of falling through
to ifq mapping and output, it now calls a function pointer in the
ifnet struct. that pointer defaults to the ifq handling, but drivers
can override it to bypass ifq processing.

the most obvious users of the function pointer will be virtual
interfaces, eg, vlan(4). ifqs are good if you need to serialise
access to the thing that transmits packets (like hardware rings on
nics), or mitigate the number of times you do ring processing, but
neither of those things are desirable on vlan interfaces. ideally
vlan could transmit on any cpu without having packets serialised
by it's own ifq before being pushed down to an arbitrary number of
rings on the parent interface. bypassing ifqs means the driver can
push the vlan tag on concurrently and push down to the parent frmo
any cpu.

ok mpi@
no objection from claudio@

2 files changed, 28 insertions, 12 deletions

diff --git a/sys/net/if.c b/sys/net/if.c
index 12abcd4a012..6ead594ff76 100644
--- a/sys/net/if.c
+++ b/sys/net/if.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: if.c,v 1.570 2018/12/20 10:26:36 claudio Exp $	*/
+/*	$OpenBSD: if.c,v 1.571 2019/01/09 01:14:21 dlg Exp $	*/
 /*	$NetBSD: if.c,v 1.35 1996/05/07 05:26:04 thorpej Exp $	*/
 
 /*
@@ -632,6 +632,8 @@ if_attach_common(struct ifnet *ifp)
 
 	if (ifp->if_rtrequest == NULL)
 		ifp->if_rtrequest = if_rtrequest_dummy;
+	if (ifp->if_enqueue == NULL)
+		ifp->if_enqueue = if_enqueue_ifq;
 	ifp->if_llprio = IFQ_DEFPRIO;
 
 	SRPL_INIT(&ifp->if_inputs);
@@ -683,10 +685,6 @@ if_qstart_compat(struct ifqueue *ifq)
 int
 if_enqueue(struct ifnet *ifp, struct mbuf *m)
 {
-	unsigned int idx;
-	struct ifqueue *ifq;
-	int error;
-
 #if NPF > 0
 	if (m->m_pkthdr.pf.delay > 0)
 		return (pf_delay_pkt(m, ifp->if_index));
@@ -694,6 +692,8 @@ if_enqueue(struct ifnet *ifp, struct mbuf *m)
 
 #if NBRIDGE > 0
 	if (ifp->if_bridgeport && (m->m_flags & M_PROTO1) == 0) {
+		int error;
+
 		KERNEL_LOCK();
 		error = bridge_output(ifp, m, NULL, NULL);
 		KERNEL_UNLOCK();
@@ -705,12 +705,26 @@ if_enqueue(struct ifnet *ifp, struct mbuf *m)
 	pf_pkt_addr_changed(m);
 #endif	/* NPF > 0 */
 
-	/*
-	 * use the operations on the first ifq to pick which of the array
-	 * gets this mbuf.
-	 */
-	idx = ifq_idx(&ifp->if_snd, ifp->if_nifqs, m);
-	ifq = ifp->if_ifqs[idx];
+	return ((*ifp->if_enqueue)(ifp, m));
+}
+
+int
+if_enqueue_ifq(struct ifnet *ifp, struct mbuf *m)
+{
+	struct ifqueue *ifq = &ifp->if_snd;
+	int error;
+
+	if (ifp->if_nifqs > 1) {
+		unsigned int idx;
+
+		/*
+		 * use the operations on the first ifq to pick which of
+		 * the array gets this mbuf.
+		 */
+
+		idx = ifq_idx(&ifp->if_snd, ifp->if_nifqs, m);
+		ifq = ifp->if_ifqs[idx];
+	}
 
 	error = ifq_enqueue(ifq, m);
 	if (error)
diff --git a/sys/net/if_var.h b/sys/net/if_var.h
index 7ccbb9b2c30..8760bafeecf 100644
--- a/sys/net/if_var.h
+++ b/sys/net/if_var.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: if_var.h,v 1.93 2018/12/20 10:26:36 claudio Exp $	*/
+/*	$OpenBSD: if_var.h,v 1.94 2019/01/09 01:14:21 dlg Exp $	*/
 /*	$NetBSD: if.h,v 1.23 1996/05/07 02:40:27 thorpej Exp $	*/
 
 /*
@@ -162,6 +162,7 @@ struct ifnet {				/* and the entries */
 					/* link level output function */
 	int	(*if_ll_output)(struct ifnet *, struct mbuf *,
 		    struct sockaddr *, struct rtentry *);
+	int	(*if_enqueue)(struct ifnet *, struct mbuf *);
 	void	(*if_start)(struct ifnet *);	/* initiate output */
 	int	(*if_ioctl)(struct ifnet *, u_long, caddr_t); /* ioctl hook */
 	void	(*if_watchdog)(struct ifnet *);	/* timer routine */
@@ -331,6 +332,7 @@ extern struct ifnet_head ifnet;
 
 void	if_start(struct ifnet *);
 int	if_enqueue(struct ifnet *, struct mbuf *);
+int	if_enqueue_ifq(struct ifnet *, struct mbuf *);
 void	if_input(struct ifnet *, struct mbuf_list *);
 void	if_input_process(struct ifnet *, struct mbuf_list *);
 int	if_input_local(struct ifnet *, struct mbuf *, sa_family_t);