ether_input() called with shared netlock, but pppoe(4) wants it to be

exclusive. Do the pppoe(4) input within netisr handler with exclusive
netlok held and remove kernel lock hack from ether_input().

This is the step back, but it makes ether_input() path better then it
is now.

Tested by Hrvoje Popovski.

ok bluhm@ claudio@

5 files changed, 43 insertions, 13 deletions

diff --git a/sys/net/if.c b/sys/net/if.c
index 0493f1b113a..19b2f3824c4 100644
--- a/sys/net/if.c
+++ b/sys/net/if.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: if.c,v 1.656 2022/06/28 09:41:24 jan Exp $	*/
+/*	$OpenBSD: if.c,v 1.657 2022/06/29 09:08:07 mvs Exp $	*/
 /*	$NetBSD: if.c,v 1.35 1996/05/07 05:26:04 thorpej Exp $	*/
 
 /*
@@ -68,6 +68,7 @@
 #include "pf.h"
 #include "pfsync.h"
 #include "ppp.h"
+#include "pppoe.h"
 #include "if_wg.h"
 
 #include <sys/param.h>
@@ -918,6 +919,10 @@ if_netisr(void *unused)
 		if (n & (1 << NETISR_PIPEX))
 			pipexintr();
 #endif
+#if NPPPOE > 0
+		if (n & (1 << NETISR_PPPOE))
+			pppoeintr();
+#endif
 		t |= n;
 	}
 
diff --git a/sys/net/if_ethersubr.c b/sys/net/if_ethersubr.c
index 5e9067b0676..3fb702c453f 100644
--- a/sys/net/if_ethersubr.c
+++ b/sys/net/if_ethersubr.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: if_ethersubr.c,v 1.283 2022/06/28 16:10:43 mvs Exp $	*/
+/*	$OpenBSD: if_ethersubr.c,v 1.284 2022/06/29 09:08:07 mvs Exp $	*/
 /*	$NetBSD: if_ethersubr.c,v 1.19 1996/05/07 02:40:30 thorpej Exp $	*/
 
 /*
@@ -545,12 +545,13 @@ ether_input(struct ifnet *ifp, struct mbuf *m)
 			}
 		}
 #endif
-		KERNEL_LOCK();
-		if (etype == ETHERTYPE_PPPOEDISC)
-			pppoe_disc_input(m);
-		else
-			pppoe_data_input(m);
-		KERNEL_UNLOCK();
+		if (etype == ETHERTYPE_PPPOEDISC) {
+			if (mq_enqueue(&pppoediscinq, m) == 0)
+				schednetisr(NETISR_PPPOE);
+		} else {
+			if (mq_enqueue(&pppoeinq, m) == 0)
+				schednetisr(NETISR_PPPOE);
+		}
 		return;
 #endif
 #ifdef MPLS
diff --git a/sys/net/if_pppoe.c b/sys/net/if_pppoe.c
index 1ada5efdd6c..300b805445f 100644
--- a/sys/net/if_pppoe.c
+++ b/sys/net/if_pppoe.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: if_pppoe.c,v 1.80 2022/05/14 09:46:15 tobhe Exp $ */
+/* $OpenBSD: if_pppoe.c,v 1.81 2022/06/29 09:08:07 mvs Exp $ */
 /* $NetBSD: if_pppoe.c,v 1.51 2003/11/28 08:56:48 keihan Exp $ */
 
 /*
@@ -144,6 +144,8 @@ struct pppoe_softc {
 };
 
 /* input routines */
+void pppoe_disc_input(struct mbuf *);
+void pppoe_data_input(struct mbuf *);
 static void pppoe_dispatch_disc_pkt(struct mbuf *);
 
 /* management routines */
@@ -181,6 +183,26 @@ int pppoe_clone_destroy(struct ifnet *);
 struct if_clone pppoe_cloner =
     IF_CLONE_INITIALIZER("pppoe", pppoe_clone_create, pppoe_clone_destroy);
 
+struct mbuf_queue pppoediscinq = MBUF_QUEUE_INITIALIZER(
+	IFQ_MAXLEN, IPL_SOFTNET);
+struct mbuf_queue pppoeinq = MBUF_QUEUE_INITIALIZER(
+	IFQ_MAXLEN, IPL_SOFTNET);
+
+void pppoeintr(void)
+{
+	struct mbuf_list ml;
+	struct mbuf *m;
+
+	NET_ASSERT_LOCKED();
+
+	mq_delist(&pppoediscinq, &ml);
+	while ((m = ml_dequeue(&ml)) != NULL)
+		pppoe_disc_input(m);
+
+	mq_delist(&pppoeinq, &ml);
+	while ((m = ml_dequeue(&ml)) != NULL)
+		pppoe_data_input(m);
+}
 
 void
 pppoeattach(int count)
diff --git a/sys/net/if_pppoe.h b/sys/net/if_pppoe.h
index eabb404b78b..997b17a5ddb 100644
--- a/sys/net/if_pppoe.h
+++ b/sys/net/if_pppoe.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: if_pppoe.h,v 1.7 2021/01/04 21:21:41 kn Exp $ */
+/*	$OpenBSD: if_pppoe.h,v 1.8 2022/06/29 09:08:07 mvs Exp $ */
 /*	$NetBSD: if_pppoe.h,v 1.5 2003/11/28 08:56:48 keihan Exp $ */
 
 /*
@@ -66,8 +66,8 @@ struct pppoeconnectionstate {
 
 #ifdef _KERNEL
 
-void pppoe_disc_input(struct mbuf *);
-void pppoe_data_input(struct mbuf *);
+extern struct mbuf_queue pppoediscinq;
+extern struct mbuf_queue pppoeinq;
 
 #endif /* _KERNEL */
 #endif /* _NET_IF_PPPOE_H_ */
diff --git a/sys/net/netisr.h b/sys/net/netisr.h
index 616296357a2..e74ad3c4707 100644
--- a/sys/net/netisr.h
+++ b/sys/net/netisr.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: netisr.h,v 1.57 2022/06/28 08:01:40 mvs Exp $	*/
+/*	$OpenBSD: netisr.h,v 1.58 2022/06/29 09:08:07 mvs Exp $	*/
 /*	$NetBSD: netisr.h,v 1.12 1995/08/12 23:59:24 mycroft Exp $	*/
 
 /*
@@ -48,6 +48,7 @@
 #define	NETISR_PIPEX	27		/* for pipex processing */
 #define	NETISR_PPP	28		/* for PPP processing */
 #define	NETISR_BRIDGE	29		/* for bridge processing */
+#define	NETISR_PPPOE	30		/* for pppoe processing */
 #define	NETISR_SWITCH	31		/* for switch dataplane */
 
 #ifndef _LOCORE
@@ -67,6 +68,7 @@ void	bridgeintr(void);
 void	switchintr(void);
 void	pfsyncintr(void);
 void	pipexintr(void);
+void	pppoeintr(void);
 
 #define	schednetisr(anisr)						\
 do {									\