deprecate interface input handler lists, just use one input function.

the interface input handler lists were originally set up to help
us during the intial mpsafe network stack work. at the time not all
the virtual ethernet interfaces (vlan, svlan, bridge, trunk, etc)
were mpsafe, so we wanted a way to avoid them by default, and only
take the kernel lock hit when they were specifically enabled on the
interface. since then, they have been fixed up to be mpsafe.

i could leave the list in place, but it has some semantic problems.
because virtual interfaces filter packets based on the order they
were attached to the parent interface, you can get packets taken
away in surprising ways, especially when you reboot and netstart
does something different to what you did by hand. by hardcoding the
order that things like vlan and bridge get to look at packets, we
can document the behaviour and get consistency.

it also means we can get rid of a use of SRPs which were difficult
to replace with SMRs. the interface input handler list is an SRPL,
which we would like to deprecate. it turns out that you can sleep
during stack processing, which you're not supposed to do with SRPs
or SMRs, but SRPs are a lot more forgiving and it worked.

lastly, it turns out that this code is faster than the input list
handling, so lots of winning all around.

special thanks to hrvoje popovski and aaron bieber for testing.
this has been in snaps as part of a larger diff for over a week.

10 files changed, 139 insertions, 202 deletions

diff --git a/sys/dev/usb/if_umb.c b/sys/dev/usb/if_umb.c
index 698a754d6b7..ec5808badfa 100644
--- a/sys/dev/usb/if_umb.c
+++ b/sys/dev/usb/if_umb.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: if_umb.c,v 1.35 2020/07/10 13:26:41 patrick Exp $ */
+/*	$OpenBSD: if_umb.c,v 1.36 2020/07/22 02:16:02 dlg Exp $ */
 
 /*
  * Copyright (c) 2016 genua mbH
@@ -137,7 +137,7 @@ void		 umb_close_bulkpipes(struct umb_softc *);
 int		 umb_ioctl(struct ifnet *, u_long, caddr_t);
 int		 umb_output(struct ifnet *, struct mbuf *, struct sockaddr *,
 		    struct rtentry *);
-int		 umb_input(struct ifnet *, struct mbuf *, void *);
+void		 umb_input(struct ifnet *, struct mbuf *);
 void		 umb_start(struct ifnet *);
 void		 umb_rtrequest(struct ifnet *, int, struct rtentry *);
 void		 umb_watchdog(struct ifnet *);
@@ -522,9 +522,9 @@ umb_attach(struct device *parent, struct device *self, void *aux)
 	    sizeof (struct ncm_pointer16);
 	ifp->if_mtu = 1500;		/* use a common default */
 	ifp->if_hardmtu = sc->sc_maxpktlen;
+	ifp->if_input = umb_input;
 	ifp->if_output = umb_output;
 	if_attach(ifp);
-	if_ih_insert(ifp, umb_input, NULL);
 	if_alloc_sadl(ifp);
 	ifp->if_softc = sc;
 #if NBPFILTER > 0
@@ -575,7 +575,6 @@ umb_detach(struct device *self, int flags)
 		sc->sc_resp_buf = NULL;
 	}
 	if (ifp->if_softc != NULL) {
-		if_ih_remove(ifp, umb_input, NULL);
 		if_detach(ifp);
 	}
 
@@ -775,21 +774,21 @@ umb_output(struct ifnet *ifp, struct mbuf *m, struct sockaddr *dst,
 	return if_enqueue(ifp, m);
 }
 
-int
-umb_input(struct ifnet *ifp, struct mbuf *m, void *cookie)
+void
+umb_input(struct ifnet *ifp, struct mbuf *m)
 {
 	uint32_t af;
 
 	if ((ifp->if_flags & IFF_UP) == 0) {
 		m_freem(m);
-		return 1;
+		return;
 	}
 	if (m->m_pkthdr.len < sizeof (struct ip) + sizeof(af)) {
 		ifp->if_ierrors++;
 		DPRINTFN(4, "%s: dropping short packet (len %d)\n", __func__,
 		    m->m_pkthdr.len);
 		m_freem(m);
-		return 1;
+		return;
 	}
 	m->m_pkthdr.ph_rtableid = ifp->if_rdomain;
 
@@ -802,20 +801,19 @@ umb_input(struct ifnet *ifp, struct mbuf *m, void *cookie)
 	switch (af) {
 	case AF_INET:
 		ipv4_input(ifp, m);
-		return 1;
+		return;
 #ifdef INET6
 	case AF_INET6:
 		ipv6_input(ifp, m);
-		return 1;
+		return;
 #endif /* INET6 */
 	default:
 		ifp->if_ierrors++;
 		DPRINTFN(4, "%s: dropping packet with bad IP version (af %d)\n",
 		    __func__, af);
 		m_freem(m);
-		return 1;
+		return;
 	}
-	return 1;
 }
 
 static inline int
diff --git a/sys/net/if.c b/sys/net/if.c
index a7aacb0c6c8..51ec987c37d 100644
--- a/sys/net/if.c
+++ b/sys/net/if.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: if.c,v 1.614 2020/07/20 13:55:32 mvs Exp $	*/
+/*	$OpenBSD: if.c,v 1.615 2020/07/22 02:16:01 dlg Exp $	*/
 /*	$NetBSD: if.c,v 1.35 1996/05/07 05:26:04 thorpej Exp $	*/
 
 /*
@@ -633,8 +633,6 @@ if_attach_common(struct ifnet *ifp)
 	if (ifp->if_enqueue == NULL)
 		ifp->if_enqueue = if_enqueue_ifq;
 	ifp->if_llprio = IFQ_DEFPRIO;
-
-	SRPL_INIT(&ifp->if_inputs);
 }
 
 void
@@ -807,109 +805,6 @@ if_output_local(struct ifnet *ifp, struct mbuf *m, sa_family_t af)
 	return (ifiq_enqueue(ifiq, m) == 0 ? 0 : ENOBUFS);
 }
 
-struct ifih {
-	SRPL_ENTRY(ifih)	  ifih_next;
-	int			(*ifih_input)(struct ifnet *, struct mbuf *,
-				      void *);
-	void			 *ifih_cookie;
-	int			  ifih_refcnt;
-	struct refcnt		  ifih_srpcnt;
-};
-
-void	if_ih_ref(void *, void *);
-void	if_ih_unref(void *, void *);
-
-struct srpl_rc ifih_rc = SRPL_RC_INITIALIZER(if_ih_ref, if_ih_unref, NULL);
-
-void
-if_ih_insert(struct ifnet *ifp, int (*input)(struct ifnet *, struct mbuf *,
-    void *), void *cookie)
-{
-	struct ifih *ifih;
-
-	/* the kernel lock guarantees serialised modifications to if_inputs */
-	KERNEL_ASSERT_LOCKED();
-
-	SRPL_FOREACH_LOCKED(ifih, &ifp->if_inputs, ifih_next) {
-		if (ifih->ifih_input == input && ifih->ifih_cookie == cookie) {
-			ifih->ifih_refcnt++;
-			break;
-		}
-	}
-
-	if (ifih == NULL) {
-		ifih = malloc(sizeof(*ifih), M_DEVBUF, M_WAITOK);
-
-		ifih->ifih_input = input;
-		ifih->ifih_cookie = cookie;
-		ifih->ifih_refcnt = 1;
-		refcnt_init(&ifih->ifih_srpcnt);
-		SRPL_INSERT_HEAD_LOCKED(&ifih_rc, &ifp->if_inputs,
-		    ifih, ifih_next);
-	}
-}
-
-void
-if_ih_ref(void *null, void *i)
-{
-	struct ifih *ifih = i;
-
-	refcnt_take(&ifih->ifih_srpcnt);
-}
-
-void
-if_ih_unref(void *null, void *i)
-{
-	struct ifih *ifih = i;
-
-	refcnt_rele_wake(&ifih->ifih_srpcnt);
-}
-
-void
-if_ih_remove(struct ifnet *ifp, int (*input)(struct ifnet *, struct mbuf *,
-    void *), void *cookie)
-{
-	struct ifih *ifih;
-
-	/* the kernel lock guarantees serialised modifications to if_inputs */
-	KERNEL_ASSERT_LOCKED();
-
-	SRPL_FOREACH_LOCKED(ifih, &ifp->if_inputs, ifih_next) {
-		if (ifih->ifih_input == input && ifih->ifih_cookie == cookie)
-			break;
-	}
-
-	KASSERT(ifih != NULL);
-
-	if (--ifih->ifih_refcnt == 0) {
-		SRPL_REMOVE_LOCKED(&ifih_rc, &ifp->if_inputs, ifih,
-		    ifih, ifih_next);
-
-		refcnt_finalize(&ifih->ifih_srpcnt, "ifihrm");
-		free(ifih, M_DEVBUF, sizeof(*ifih));
-	}
-}
-
-static void
-if_ih_input(struct ifnet *ifp, struct mbuf *m)
-{
-	struct ifih *ifih;
-	struct srp_ref sr;
-
-	/*
-	 * Pass this mbuf to all input handlers of its
-	 * interface until it is consumed.
-	 */
-	SRPL_FOREACH(ifih, &sr, &ifp->if_inputs, ifih_next) {
-		if ((*ifih->ifih_input)(ifp, m, ifih->ifih_cookie))
-			break;
-	}
-	SRPL_LEAVE(&sr);
-
-	if (ifih == NULL)
-		m_freem(m);
-}
-
 void
 if_input_process(struct ifnet *ifp, struct mbuf_list *ml)
 {
@@ -935,7 +830,7 @@ if_input_process(struct ifnet *ifp, struct mbuf_list *ml)
 	 */
 	NET_LOCK();
 	while ((m = ml_dequeue(ml)) != NULL)
-		if_ih_input(ifp, m);
+		(*ifp->if_input)(ifp, m);
 	NET_UNLOCK();
 }
 
@@ -962,7 +857,7 @@ if_vinput(struct ifnet *ifp, struct mbuf *m)
 	}
 #endif
 
-	if_ih_input(ifp, m);
+	(*ifp->if_input)(ifp, m);
 }
 
 void
diff --git a/sys/net/if_aggr.c b/sys/net/if_aggr.c
index 1707230cd96..aaceaf6f737 100644
--- a/sys/net/if_aggr.c
+++ b/sys/net/if_aggr.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: if_aggr.c,v 1.32 2020/07/10 13:26:41 patrick Exp $ */
+/*	$OpenBSD: if_aggr.c,v 1.33 2020/07/22 02:16:01 dlg Exp $ */
 
 /*
  * Copyright (c) 2019 The University of Queensland
@@ -332,6 +332,7 @@ struct aggr_port {
 	uint32_t		 p_mtu;
 
 	int (*p_ioctl)(struct ifnet *, u_long, caddr_t);
+	void (*p_input)(struct ifnet *, struct mbuf *);
 	int (*p_output)(struct ifnet *, struct mbuf *, struct sockaddr *,
 	    struct rtentry *);
 
@@ -723,13 +724,14 @@ aggr_eh_is_slow(const struct ether_header *eh)
 	    eh->ether_type == htons(ETHERTYPE_SLOW));
 }
 
-static int
-aggr_input(struct ifnet *ifp0, struct mbuf *m, void *cookie)
+static void
+aggr_input(struct ifnet *ifp0, struct mbuf *m)
 {
-	struct ether_header *eh;
-	struct aggr_port *p = cookie;
+	struct arpcom *ac0 = (struct arpcom *)ifp0;
+	struct aggr_port *p = ac0->ac_trunkport;
 	struct aggr_softc *sc = p->p_aggr;
 	struct ifnet *ifp = &sc->sc_if;
+	struct ether_header *eh;
 	int hlen = sizeof(*eh);
 
 	if (!ISSET(ifp->if_flags, IFF_RUNNING))
@@ -745,7 +747,7 @@ aggr_input(struct ifnet *ifp0, struct mbuf *m, void *cookie)
 			m = m_pullup(m, hlen);
 			if (m == NULL) {
 				/* short++ */
-				return (1);
+				return;
 			}
 			eh = mtod(m, struct ether_header *);
 		}
@@ -756,7 +758,7 @@ aggr_input(struct ifnet *ifp0, struct mbuf *m, void *cookie)
 		case SLOWPROTOCOLS_SUBTYPE_LACP_MARKER:
 			if (mq_enqueue(&p->p_rxm_mq, m) == 0)
 				task_add(systq, &p->p_rxm_task);
-			return (1);
+			return;
 		default:
 			break;
 		}
@@ -770,11 +772,10 @@ aggr_input(struct ifnet *ifp0, struct mbuf *m, void *cookie)
 
 	if_vinput(ifp, m);
 
-	return (1);
+	return;
 
 drop:
 	m_freem(m);
-	return (1);
 }
 
 static int
@@ -1072,6 +1073,10 @@ aggr_add_port(struct aggr_softc *sc, const struct trunk_reqport *rp)
 	if (ifp0->if_type != IFT_ETHER)
 		return (EPROTONOSUPPORT);
 
+	error = ether_brport_isset(ifp0);
+	if (error != 0)
+		return (error);
+
 	if (ifp0->if_hardmtu < ifp->if_mtu)
 		return (ENOBUFS);
 
@@ -1103,6 +1108,7 @@ aggr_add_port(struct aggr_softc *sc, const struct trunk_reqport *rp)
 	CTASSERT(sizeof(p->p_lladdr) == sizeof(ac0->ac_enaddr));
 	memcpy(p->p_lladdr, ac0->ac_enaddr, sizeof(p->p_lladdr));
 	p->p_ioctl = ifp0->if_ioctl;
+	p->p_input = ifp0->if_input;
 	p->p_output = ifp0->if_output;
 
 	error = aggr_group(sc, p, SIOCADDMULTI);
@@ -1161,12 +1167,18 @@ aggr_add_port(struct aggr_softc *sc, const struct trunk_reqport *rp)
 
 	aggr_update_capabilities(sc);
 
+	/*
+         * use (and modification) of ifp->if_input and ac->ac_trunkport
+         * is protected by NET_LOCK.
+	 */
+
 	ac0->ac_trunkport = p;
+
 	/* make sure p is visible before handlers can run */
 	membar_producer();
 	ifp0->if_ioctl = aggr_p_ioctl;
+	ifp0->if_input = aggr_input;
 	ifp0->if_output = aggr_p_output;
-	if_ih_insert(ifp0, aggr_input, p);
 
 	aggr_mux(sc, p, LACP_MUX_E_BEGIN);
 	aggr_rxm(sc, p, LACP_RXM_E_BEGIN);
@@ -1385,10 +1397,13 @@ aggr_p_dtor(struct aggr_softc *sc, struct aggr_port *p, const char *op)
 	timeout_del(&p->p_current_while_timer);
 	timeout_del(&p->p_wait_while_timer);
 
-	if_ih_remove(ifp0, aggr_input, p);
+	/*
+         * use (and modification) of ifp->if_input and ac->ac_trunkport
+         * is protected by NET_LOCK.
+	 */
 
 	ac0->ac_trunkport = NULL;
-
+	ifp0->if_input = p->p_input;
 	ifp0->if_ioctl = p->p_ioctl;
 	ifp0->if_output = p->p_output;
 
diff --git a/sys/net/if_ethersubr.c b/sys/net/if_ethersubr.c
index 34282deda2a..964e27f313b 100644
--- a/sys/net/if_ethersubr.c
+++ b/sys/net/if_ethersubr.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: if_ethersubr.c,v 1.265 2020/07/22 01:50:39 dlg Exp $	*/
+/*	$OpenBSD: if_ethersubr.c,v 1.266 2020/07/22 02:16:01 dlg Exp $	*/
 /*	$NetBSD: if_ethersubr.c,v 1.19 1996/05/07 02:40:30 thorpej Exp $	*/
 
 /*
@@ -355,12 +355,22 @@ ether_output(struct ifnet *ifp, struct mbuf *m, struct sockaddr *dst,
 }
 
 /*
- * Process a received Ethernet packet;
- * the packet is in the mbuf chain m without
- * the ether header, which is provided separately.
+ * Process a received Ethernet packet.
+ *
+ * Ethernet input has several "phases" of filtering packets to
+ * support virtual/pseudo interfaces before actual layer 3 protocol
+ * handling.
+ *
+ * First phase:
+ *
+ * The first phase supports drivers that aggregate multiple Ethernet
+ * ports into a single logical interface, ie, aggr(4) and trunk(4).
+ * These drivers intercept packets by swapping out the if_input handler
+ * on the "port" interfaces to steal the packets before they get here
+ * to ether_input().
  */
-int
-ether_input(struct ifnet *ifp, struct mbuf *m, void *cookie)
+void
+ether_input(struct ifnet *ifp, struct mbuf *m)
 {
 	struct ether_header *eh;
 	void (*input)(struct ifnet *, struct mbuf *);
@@ -374,6 +384,8 @@ ether_input(struct ifnet *ifp, struct mbuf *m, void *cookie)
 		goto dropanyway;
 
 	/*
+	 * Second phase: service delimited packet filtering.
+	 *
 	 * Let vlan(4) and svlan(4) look at "service delimited"
 	 * packets. If a virtual interface does not exist to take
 	 * those packets, they're returned to ether_input() so a
@@ -388,13 +400,15 @@ ether_input(struct ifnet *ifp, struct mbuf *m, void *cookie)
 #if NVLAN > 0
 		m = vlan_input(ifp, m);
 		if (m == NULL)
-			return (1);
+			return;
 #endif /* NVLAN > 0 */
 
 		sdelim = 1;
 	}
 
 	/*
+	 * Third phase: bridge processing.
+	 *
 	 * Give the packet to a bridge interface, ie, bridge(4),
 	 * switch(4), or tpmr(4), if it is configured. A bridge
 	 * may take the packet and forward it to another port, or it
@@ -410,12 +424,14 @@ ether_input(struct ifnet *ifp, struct mbuf *m, void *cookie)
 		m = (*eb->eb_input)(ifp, m, eb->eb_port);
 		if (m == NULL) {
 			smr_read_leave();
-			return (1);
+			return;
 		}
 	}
 	smr_read_leave();
 
 	/*
+	 * Fourth phase: drop service delimited packets.
+	 *
 	 * If the packet has a tag, and a bridge didn't want it,
 	 * it's not for this port.
 	 */
@@ -423,9 +439,13 @@ ether_input(struct ifnet *ifp, struct mbuf *m, void *cookie)
 	if (sdelim)
 		goto dropanyway;
 
-	eh = mtod(m, struct ether_header *);
+	/*
+	 * Fifth phase: destination address check.
+	 *
+	 * Is the packet specifically addressed to this port?
+	 */
 
-	/* Is the packet for us? */
+	eh = mtod(m, struct ether_header *);
 	if (memcmp(ac->ac_enaddr, eh->ether_dhost, ETHER_ADDR_LEN) != 0) {
 #if NCARP > 0
 		/*
@@ -435,13 +455,15 @@ ether_input(struct ifnet *ifp, struct mbuf *m, void *cookie)
 		    !SRPL_EMPTY_LOCKED(&ifp->if_carp)) {
 			m = carp_input(ifp, m);
 			if (m == NULL)
-				return (1);
+				return;
 
 			eh = mtod(m, struct ether_header *);
 		}
 #endif
 
-		/* If not, it must be multicast or broadcast to go further */
+		/*
+		 * If not, it must be multicast or broadcast to go further.
+		 */
 		if (!ETHER_IS_MULTICAST(eh->ether_dhost))
 			goto dropanyway;
 
@@ -462,6 +484,13 @@ ether_input(struct ifnet *ifp, struct mbuf *m, void *cookie)
 		ifp->if_imcasts++;
 	}
 
+ 	/*
+	 * Sixth phase: protocol demux.
+	 *
+	 * At this point it is known that the packet is destined
+	 * for layer 3 protocol handling on the local port.
+ 	 */
+
 	switch (etype) {
 	case ETHERTYPE_IP:
 		input = ipv4_input;
@@ -498,7 +527,7 @@ ether_input(struct ifnet *ifp, struct mbuf *m, void *cookie)
 
 			if ((session = pipex_pppoe_lookup_session(m)) != NULL) {
 				pipex_pppoe_input(m, session);
-				return (1);
+				return;
 			}
 		}
 #endif
@@ -506,7 +535,7 @@ ether_input(struct ifnet *ifp, struct mbuf *m, void *cookie)
 			niq_enqueue(&pppoediscinq, m);
 		else
 			niq_enqueue(&pppoeinq, m);
-		return (1);
+		return;
 #endif
 #ifdef MPLS
 	case ETHERTYPE_MPLS:
@@ -517,7 +546,7 @@ ether_input(struct ifnet *ifp, struct mbuf *m, void *cookie)
 #if NBPE > 0
 	case ETHERTYPE_PBB:
 		bpe_input(ifp, m);
-		return (1);
+		return;
 #endif
 	default:
 		goto dropanyway;
@@ -525,10 +554,10 @@ ether_input(struct ifnet *ifp, struct mbuf *m, void *cookie)
 
 	m_adj(m, sizeof(*eh));
 	(*input)(ifp, m);
-	return (1);
+	return;
 dropanyway:
 	m_freem(m);
-	return (1);
+	return;
 }
 
 int
@@ -640,12 +669,11 @@ ether_ifattach(struct ifnet *ifp)
 	ifp->if_addrlen = ETHER_ADDR_LEN;
 	ifp->if_hdrlen = ETHER_HDR_LEN;
 	ifp->if_mtu = ETHERMTU;
+	ifp->if_input = ether_input;
 	if (ifp->if_output == NULL)
 		ifp->if_output = ether_output;
 	ifp->if_rtrequest = ether_rtrequest;
 
-	if_ih_insert(ifp, ether_input, NULL);
-
 	if (ifp->if_hardmtu == 0)
 		ifp->if_hardmtu = ETHERMTU;
 
@@ -666,10 +694,6 @@ ether_ifdetach(struct ifnet *ifp)
 	/* Undo pseudo-driver changes. */
 	if_deactivate(ifp);
 
-	if_ih_remove(ifp, ether_input, NULL);
-
-	KASSERT(SRPL_EMPTY_LOCKED(&ifp->if_inputs));
-
 	for (enm = LIST_FIRST(&ac->ac_multiaddrs);
 	    enm != NULL;
 	    enm = LIST_FIRST(&ac->ac_multiaddrs)) {
diff --git a/sys/net/if_loop.c b/sys/net/if_loop.c
index 1d3a8095b0e..2ba5b85570c 100644
--- a/sys/net/if_loop.c
+++ b/sys/net/if_loop.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: if_loop.c,v 1.90 2020/01/08 09:09:10 claudio Exp $	*/
+/*	$OpenBSD: if_loop.c,v 1.91 2020/07/22 02:16:01 dlg Exp $	*/
 /*	$NetBSD: if_loop.c,v 1.15 1996/05/07 02:40:33 thorpej Exp $	*/
 
 /*
@@ -143,7 +143,7 @@
 int	loioctl(struct ifnet *, u_long, caddr_t);
 void	loopattach(int);
 void	lortrequest(struct ifnet *, int, struct rtentry *);
-int	loinput(struct ifnet *, struct mbuf *, void *);
+void	loinput(struct ifnet *, struct mbuf *);
 int	looutput(struct ifnet *,
 	    struct mbuf *, struct sockaddr *, struct rtentry *);
 
@@ -175,6 +175,7 @@ loop_clone_create(struct if_clone *ifc, int unit)
 	ifp->if_xflags = IFXF_CLONED;
 	ifp->if_rtrequest = lortrequest;
 	ifp->if_ioctl = loioctl;
+	ifp->if_input = loinput;
 	ifp->if_output = looutput;
 	ifp->if_type = IFT_LOOP;
 	ifp->if_hdrlen = sizeof(u_int32_t);
@@ -188,7 +189,6 @@ loop_clone_create(struct if_clone *ifc, int unit)
 #if NBPFILTER > 0
 	bpfattach(&ifp->if_bpf, ifp, DLT_LOOP, sizeof(u_int32_t));
 #endif
-	if_ih_insert(ifp, loinput, NULL);
 	return (0);
 }
 
@@ -218,7 +218,6 @@ loop_clone_destroy(struct ifnet *ifp)
 		rdomain = ifp->if_rdomain;
 	}
 
-	if_ih_remove(ifp, loinput, NULL);
 	if_detach(ifp);
 
 	free(ifp, M_DEVBUF, sizeof(*ifp));
@@ -228,8 +227,8 @@ loop_clone_destroy(struct ifnet *ifp)
 	return (0);
 }
 
-int
-loinput(struct ifnet *ifp, struct mbuf *m, void *cookie)
+void
+loinput(struct ifnet *ifp, struct mbuf *m)
 {
 	int error;
 
@@ -239,8 +238,6 @@ loinput(struct ifnet *ifp, struct mbuf *m, void *cookie)
 	error = if_input_local(ifp, m, m->m_pkthdr.ph_family);
 	if (error)
 		ifp->if_ierrors++;
-
-	return (1);
 }
 
 int
diff --git a/sys/net/if_trunk.c b/sys/net/if_trunk.c
index a19e0e0fc88..22dce44cfaf 100644
--- a/sys/net/if_trunk.c
+++ b/sys/net/if_trunk.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: if_trunk.c,v 1.147 2020/07/10 13:22:22 patrick Exp $	*/
+/*	$OpenBSD: if_trunk.c,v 1.148 2020/07/22 02:16:02 dlg Exp $	*/
 
 /*
  * Copyright (c) 2005, 2006, 2007 Reyk Floeter <reyk@openbsd.org>
@@ -76,7 +76,7 @@ int	 trunk_ether_delmulti(struct trunk_softc *, struct ifreq *);
 void	 trunk_ether_purgemulti(struct trunk_softc *);
 int	 trunk_ether_cmdmulti(struct trunk_port *, u_long);
 int	 trunk_ioctl_allports(struct trunk_softc *, u_long, caddr_t);
-int	 trunk_input(struct ifnet *, struct mbuf *, void *);
+void	 trunk_input(struct ifnet *, struct mbuf *);
 void	 trunk_start(struct ifnet *);
 void	 trunk_init(struct ifnet *);
 void	 trunk_stop(struct ifnet *);
@@ -380,9 +380,11 @@ trunk_port_create(struct trunk_softc *tr, struct ifnet *ifp)
 	if (tr->tr_port_create != NULL)
 		error = (*tr->tr_port_create)(tp);
 
-	ac0->ac_trunkport = tp;
 	/* Change input handler of the physical interface. */
-	if_ih_insert(ifp, trunk_input, tp);
+	tp->tp_input = ifp->if_input;
+	NET_ASSERT_LOCKED();
+	ac0->ac_trunkport = tp;
+	ifp->if_input = trunk_input;
 
 	return (error);
 }
@@ -413,7 +415,8 @@ trunk_port_destroy(struct trunk_port *tp)
 	struct arpcom *ac0 = (struct arpcom *)ifp;
 
 	/* Restore previous input handler. */
-	if_ih_remove(ifp, trunk_input, tp);
+	NET_ASSERT_LOCKED();
+	ifp->if_input = tp->tp_input;
 	ac0->ac_trunkport = NULL;
 
 	/* Remove multicast addresses from this port */
@@ -649,15 +652,24 @@ trunk_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data)
 			error = EPROTONOSUPPORT;
 			break;
 		}
+
 		/*
+		 * Use of ifp->if_input and ac->ac_trunkport is
+		 * protected by NET_LOCK, but that may not be true
+		 * in the future. The below comment and code flow is
+		 * maintained to help in that future.
+		 *
 		 * Serialize modifications to the trunk and trunk
 		 * ports via the ifih SRP: detaching trunk_input
 		 * from the trunk port will require all currently
 		 * running trunk_input's on this port to finish
 		 * granting us an exclusive access to it.
 		 */
-		SLIST_FOREACH(tp, &tr->tr_ports, tp_entries)
-			if_ih_remove(tp->tp_if, trunk_input, tp);
+		NET_ASSERT_LOCKED();
+		SLIST_FOREACH(tp, &tr->tr_ports, tp_entries) {
+			/* if_ih_remove(tp->tp_if, trunk_input, tp); */
+			tp->tp_if->if_input = tp->tp_input;
+		}
 		if (tr->tr_proto != TRUNK_PROTO_NONE)
 			error = tr->tr_detach(tr);
 		if (error != 0)
@@ -671,9 +683,11 @@ trunk_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data)
 				tr->tr_proto = trunk_protos[i].ti_proto;
 				if (tr->tr_proto != TRUNK_PROTO_NONE)
 					error = trunk_protos[i].ti_attach(tr);
-				SLIST_FOREACH(tp, &tr->tr_ports, tp_entries)
-					if_ih_insert(tp->tp_if,
-					    trunk_input, tp);
+				SLIST_FOREACH(tp, &tr->tr_ports, tp_entries) {
+					/* if_ih_insert(tp->tp_if,
+					    trunk_input, tp); */
+					tp->tp_if->if_input = trunk_input;
+				}
 				/* Update trunk capabilities */
 				tr->tr_capabilities = trunk_capabilities(tr);
 				goto out;
@@ -1118,14 +1132,18 @@ trunk_stop(struct ifnet *ifp)
 		(*tr->tr_stop)(tr);
 }
 
-int
-trunk_input(struct ifnet *ifp, struct mbuf *m, void *cookie)
+void
+trunk_input(struct ifnet *ifp, struct mbuf *m)
 {
-	struct trunk_softc *tr;
+	struct arpcom *ac0 = (struct arpcom *)ifp;
 	struct trunk_port *tp;
+	struct trunk_softc *tr;
 	struct ifnet *trifp = NULL;
 	struct ether_header *eh;
 
+	if (m->m_len < sizeof(*eh))
+		goto bad;
+
 	eh = mtod(m, struct ether_header *);
 	if (ETHER_IS_MULTICAST(eh->ether_dhost))
 		ifp->if_imcasts++;
@@ -1134,7 +1152,7 @@ trunk_input(struct ifnet *ifp, struct mbuf *m, void *cookie)
 	if (ifp->if_type != IFT_IEEE8023ADLAG)
 		goto bad;
 
-	tp = (struct trunk_port *)cookie;
+	tp = (struct trunk_port *)ac0->ac_trunkport;
 	if ((tr = (struct trunk_softc *)tp->tp_trunk) == NULL)
 		goto bad;
 
@@ -1147,7 +1165,7 @@ trunk_input(struct ifnet *ifp, struct mbuf *m, void *cookie)
 		 * We stop here if the packet has been consumed
 		 * by the protocol routine.
 		 */
-		return (1);
+		return;
 	}
 
 	if ((trifp->if_flags & (IFF_UP|IFF_RUNNING)) != (IFF_UP|IFF_RUNNING))
@@ -1163,19 +1181,18 @@ trunk_input(struct ifnet *ifp, struct mbuf *m, void *cookie)
 		if (bcmp(&tr->tr_ac.ac_enaddr, eh->ether_dhost,
 		    ETHER_ADDR_LEN)) {
 			m_freem(m);
-			return (1);
+			return;
 		}
 	}
 
 
 	if_vinput(trifp, m);
-	return (1);
+	return;
 
  bad:
 	if (trifp != NULL)
 		trifp->if_ierrors++;
 	m_freem(m);
-	return (1);
 }
 
 int
diff --git a/sys/net/if_trunk.h b/sys/net/if_trunk.h
index 849f295a432..fbffffe5fc8 100644
--- a/sys/net/if_trunk.h
+++ b/sys/net/if_trunk.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: if_trunk.h,v 1.29 2019/11/07 07:36:32 dlg Exp $	*/
+/*	$OpenBSD: if_trunk.h,v 1.30 2020/07/22 02:16:02 dlg Exp $	*/
 
 /*
  * Copyright (c) 2005, 2006, 2007 Reyk Floeter <reyk@openbsd.org>
@@ -172,6 +172,7 @@ struct trunk_port {
 	int	(*tp_ioctl)(struct ifnet *, u_long, caddr_t);
 	int	(*tp_output)(struct ifnet *, struct mbuf *, struct sockaddr *,
 		    struct rtentry *);
+	void	(*tp_input)(struct ifnet *, struct mbuf *);
 
 	SLIST_ENTRY(trunk_port)		tp_entries;
 };
diff --git a/sys/net/if_tun.c b/sys/net/if_tun.c
index e94bcb0d31a..25bd417031d 100644
--- a/sys/net/if_tun.c
+++ b/sys/net/if_tun.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: if_tun.c,v 1.224 2020/07/10 13:26:42 patrick Exp $	*/
+/*	$OpenBSD: if_tun.c,v 1.225 2020/07/22 02:16:02 dlg Exp $	*/
 /*	$NetBSD: if_tun.c,v 1.24 1996/05/07 02:40:48 thorpej Exp $	*/
 
 /*
@@ -115,7 +115,7 @@ int	tun_dev_poll(dev_t, int, struct proc *);
 int	tun_dev_kqfilter(dev_t, struct knote *);
 
 int	tun_ioctl(struct ifnet *, u_long, caddr_t);
-int	tun_input(struct ifnet *, struct mbuf *, void *);
+void	tun_input(struct ifnet *, struct mbuf *);
 int	tun_output(struct ifnet *, struct mbuf *, struct sockaddr *,
 	    struct rtentry *);
 int	tun_enqueue(struct ifnet *, struct mbuf *);
@@ -240,6 +240,7 @@ tun_create(struct if_clone *ifc, int unit, int flags)
 	if_counters_alloc(ifp);
 
 	if ((flags & TUN_LAYER2) == 0) {
+		ifp->if_input = tun_input;
 		ifp->if_output = tun_output;
 		ifp->if_mtu = ETHERMTU;
 		ifp->if_flags = (IFF_POINTOPOINT|IFF_MULTICAST);
@@ -253,8 +254,6 @@ tun_create(struct if_clone *ifc, int unit, int flags)
 #if NBPFILTER > 0
 		bpfattach(&ifp->if_bpf, ifp, DLT_LOOP, sizeof(u_int32_t));
 #endif
-
-		if_ih_insert(ifp, tun_input, NULL);
 	} else {
 		sc->sc_flags |= TUN_LAYER2;
 		ether_fakeaddr(ifp);
@@ -320,9 +319,7 @@ tun_clone_destroy(struct ifnet *ifp)
 	klist_invalidate(&sc->sc_wsel.si_note);
 	splx(s);
 
-	if (!ISSET(sc->sc_flags, TUN_LAYER2))
-		if_ih_remove(ifp, tun_input, NULL);
-	else
+	if (ISSET(sc->sc_flags, TUN_LAYER2))
 		ether_ifdetach(ifp);
 
 	if_detach(ifp);
@@ -880,8 +877,8 @@ put:
 	return (error);
 }
 
-int
-tun_input(struct ifnet *ifp, struct mbuf *m0, void *cookie)
+void
+tun_input(struct ifnet *ifp, struct mbuf *m0)
 {
 	uint32_t		af;
 
@@ -909,8 +906,6 @@ tun_input(struct ifnet *ifp, struct mbuf *m0, void *cookie)
 		m_freem(m0);
 		break;
 	}
-
-	return (1);
 }
 
 /*
diff --git a/sys/net/if_var.h b/sys/net/if_var.h
index f95ce9912a8..a132b1a3801 100644
--- a/sys/net/if_var.h
+++ b/sys/net/if_var.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: if_var.h,v 1.109 2020/07/10 13:26:42 patrick Exp $	*/
+/*	$OpenBSD: if_var.h,v 1.110 2020/07/22 02:16:02 dlg Exp $	*/
 /*	$NetBSD: if.h,v 1.23 1996/05/07 02:40:27 thorpej Exp $	*/
 
 /*
@@ -159,7 +159,7 @@ struct ifnet {				/* and the entries */
 	struct	task if_linkstatetask;	/* [I] task to do route updates */
 
 	/* procedure handles */
-	SRPL_HEAD(, ifih) if_inputs;	/* [K] input routines (dequeue) */
+	void	(*if_input)(struct ifnet *, struct mbuf *);
 	int	(*if_output)(struct ifnet *, struct mbuf *, struct sockaddr *,
 		     struct rtentry *);	/* output routine (enqueue) */
 					/* link level output function */
@@ -346,11 +346,6 @@ void	ifa_del(struct ifnet *, struct ifaddr *);
 void	ifa_update_broadaddr(struct ifnet *, struct ifaddr *,
 	    struct sockaddr *);
 
-void	if_ih_insert(struct ifnet *, int (*)(struct ifnet *, struct mbuf *,
-	    void *), void *);
-void	if_ih_remove(struct ifnet *, int (*)(struct ifnet *, struct mbuf *,
-	    void *), void *);
-
 void	if_addrhook_add(struct ifnet *, struct task *);
 void	if_addrhook_del(struct ifnet *, struct task *);
 void	if_addrhooks_run(struct ifnet *);
diff --git a/sys/netinet/if_ether.h b/sys/netinet/if_ether.h
index 5ae7228509f..8ae1f2b3bce 100644
--- a/sys/netinet/if_ether.h
+++ b/sys/netinet/if_ether.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: if_ether.h,v 1.77 2020/07/22 00:29:00 dlg Exp $	*/
+/*	$OpenBSD: if_ether.h,v 1.78 2020/07/22 02:16:02 dlg Exp $	*/
 /*	$NetBSD: if_ether.h,v 1.22 1996/05/11 13:00:00 mycroft Exp $	*/
 
 /*
@@ -253,7 +253,7 @@ int	ether_multiaddr(struct sockaddr *, u_int8_t[], u_int8_t[]);
 void	ether_ifattach(struct ifnet *);
 void	ether_ifdetach(struct ifnet *);
 int	ether_ioctl(struct ifnet *, struct arpcom *, u_long, caddr_t);
-int	ether_input(struct ifnet *, struct mbuf *, void *);
+void	ether_input(struct ifnet *, struct mbuf *);
 int	ether_resolve(struct ifnet *, struct mbuf *, struct sockaddr *,
 	    struct rtentry *, struct ether_header *);
 struct mbuf *