split ether_output into resolution, encapsulation, and output functions

if if_output can be overridden on ethernet interfaces, it will allow
things like vlan to do it's packet encapsulation during output
before putting the packet directly on the underlying interface for
output.

this has two benefits. first, it can avoid having ether_output on
pseudo interfaces recurse, which makes profiling of the network
stack a lot clearer. secondly, and more importantly, it allows
pseudo ethernet interface packet encapsulation to by run concurrently
by the stack, rather than having packets unnecessarily serialied
by an ifq.

this diff just splits ether_output up, it doesnt have any interface
take advantage of it yet.

tweaks and ok claudio@

2 files changed, 103 insertions, 71 deletions

diff --git a/sys/net/if_ethersubr.c b/sys/net/if_ethersubr.c
index 76f6c3147e0..68a2b265b26 100644
--- a/sys/net/if_ethersubr.c
+++ b/sys/net/if_ethersubr.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: if_ethersubr.c,v 1.253 2018/03/13 01:31:48 dlg Exp $	*/
+/*	$OpenBSD: if_ethersubr.c,v 1.254 2018/12/11 01:27:08 dlg Exp $	*/
 /*	$NetBSD: if_ethersubr.c,v 1.19 1996/05/07 02:40:30 thorpej Exp $	*/
 
 /*
@@ -178,24 +178,18 @@ ether_rtrequest(struct ifnet *ifp, int req, struct rtentry *rt)
 		break;
 	}
 }
-/*
- * Ethernet output routine.
- * Encapsulate a packet of type family for the local net.
- * Assumes that ifp is actually pointer to arpcom structure.
- */
+
 int
-ether_output(struct ifnet *ifp, struct mbuf *m, struct sockaddr *dst,
-    struct rtentry *rt)
+ether_resolve(struct ifnet *ifp, struct mbuf *m, struct sockaddr *dst,
+    struct rtentry *rt, struct ether_header *eh)
 {
-	u_int16_t etype;
-	u_char edst[ETHER_ADDR_LEN];
-	u_char *esrc;
-	struct mbuf *mcopy = NULL;
-	struct ether_header *eh;
 	struct arpcom *ac = (struct arpcom *)ifp;
 	sa_family_t af = dst->sa_family;
 	int error = 0;
 
+	if (!ISSET(ifp->if_flags, IFF_RUNNING))
+		senderr(ENETDOWN);
+
 	KASSERT(rt != NULL || ISSET(m->m_flags, M_MCAST|M_BCAST) ||
 		af == AF_UNSPEC || af == pseudo_AF_HDRCMPLT);
 
@@ -207,28 +201,31 @@ ether_output(struct ifnet *ifp, struct mbuf *m, struct sockaddr *dst,
 	}
 #endif
 
-	esrc = ac->ac_enaddr;
-
-	if ((ifp->if_flags & (IFF_UP|IFF_RUNNING)) != (IFF_UP|IFF_RUNNING))
-		senderr(ENETDOWN);
-
 	switch (af) {
 	case AF_INET:
-		error = arpresolve(ifp, rt, m, dst, edst);
+		error = arpresolve(ifp, rt, m, dst, eh->ether_dhost);
 		if (error)
-			return (error == EAGAIN ? 0 : error);
+			return (error);
+		eh->ether_type = htons(ETHERTYPE_IP);
+
 		/* If broadcasting on a simplex interface, loopback a copy */
-		if ((m->m_flags & M_BCAST) && (ifp->if_flags & IFF_SIMPLEX) &&
-		    !m->m_pkthdr.pf.routed)
+		if (ISSET(m->m_flags, M_BCAST) &&
+		    ISSET(ifp->if_flags, IFF_SIMPLEX) &&
+		    !m->m_pkthdr.pf.routed) {
+			struct mbuf *mcopy;
+
+			/* XXX Should we input an unencrypted IPsec packet? */
 			mcopy = m_copym(m, 0, M_COPYALL, M_NOWAIT);
-		etype = htons(ETHERTYPE_IP);
+			if (mcopy != NULL)
+				if_input_local(ifp, mcopy, af);
+		}
 		break;
 #ifdef INET6
 	case AF_INET6:
-		error = nd6_resolve(ifp, rt, m, dst, edst);
+		error = nd6_resolve(ifp, rt, m, dst, eh->ether_dhost);
 		if (error)
-			return (error == EAGAIN ? 0 : error);
-		etype = htons(ETHERTYPE_IPV6);
+			return (error);
+		eh->ether_type = htons(ETHERTYPE_IPV6);
 		break;
 #endif
 #ifdef MPLS
@@ -242,72 +239,102 @@ ether_output(struct ifnet *ifp, struct mbuf *m, struct sockaddr *dst,
 			senderr(ENETUNREACH);
 
 		switch (dst->sa_family) {
-			case AF_LINK:
-				if (satosdl(dst)->sdl_alen < sizeof(edst))
-					senderr(EHOSTUNREACH);
-				memcpy(edst, LLADDR(satosdl(dst)),
-				    sizeof(edst));
-				break;
+		case AF_LINK:
+			if (satosdl(dst)->sdl_alen < sizeof(eh->ether_dhost))
+				senderr(EHOSTUNREACH);
+			memcpy(eh->ether_dhost, LLADDR(satosdl(dst)),
+			    sizeof(eh->ether_dhost));
+			break;
 #ifdef INET6
-			case AF_INET6:
-				error = nd6_resolve(ifp, rt, m, dst, edst);
-				if (error)
-					return (error == EAGAIN ? 0 : error);
-				break;
+		case AF_INET6:
+			error = nd6_resolve(ifp, rt, m, dst, eh->ether_dhost);
+			if (error)
+				return (error);
+			break;
 #endif
-			case AF_INET:
-			case AF_MPLS:
-				error = arpresolve(ifp, rt, m, dst, edst);
-				if (error)
-					return (error == EAGAIN ? 0 : error);
-				break;
-			default:
-				senderr(EHOSTUNREACH);
+		case AF_INET:
+		case AF_MPLS:
+			error = arpresolve(ifp, rt, m, dst, eh->ether_dhost);
+			if (error)
+				return (error);
+			break;
+		default:
+			senderr(EHOSTUNREACH);
 		}
 		/* XXX handling for simplex devices in case of M/BCAST ?? */
 		if (m->m_flags & (M_BCAST | M_MCAST))
-			etype = htons(ETHERTYPE_MPLS_MCAST);
+			eh->ether_type = htons(ETHERTYPE_MPLS_MCAST);
 		else
-			etype = htons(ETHERTYPE_MPLS);
+			eh->ether_type = htons(ETHERTYPE_MPLS);
 		break;
 #endif /* MPLS */
 	case pseudo_AF_HDRCMPLT:
-		eh = (struct ether_header *)dst->sa_data;
-		esrc = eh->ether_shost;
-		/* FALLTHROUGH */
+		/* take the whole header from the sa */
+		memcpy(eh, dst->sa_data, sizeof(*eh));
+		return (0);
 
 	case AF_UNSPEC:
-		eh = (struct ether_header *)dst->sa_data;
-		memcpy(edst, eh->ether_dhost, sizeof(edst));
-		/* AF_UNSPEC doesn't swap the byte order of the ether_type. */
-		etype = eh->ether_type;
+		/* take the dst and type from the sa, but get src below */
+		memcpy(eh, dst->sa_data, sizeof(*eh));
 		break;
 
 	default:
-		printf("%s: can't handle af%d\n", ifp->if_xname,
-			dst->sa_family);
+		printf("%s: can't handle af%d\n", ifp->if_xname, af);
 		senderr(EAFNOSUPPORT);
 	}
 
-	/* XXX Should we feed-back an unencrypted IPsec packet ? */
-	if (mcopy)
-		if_input_local(ifp, mcopy, dst->sa_family);
+	memcpy(eh->ether_shost, ac->ac_enaddr, sizeof(eh->ether_shost));
 
-	M_PREPEND(m, sizeof(struct ether_header) + ETHER_ALIGN, M_DONTWAIT);
-	if (m == NULL)
-		return (ENOBUFS);
-	m_adj(m, ETHER_ALIGN);
-	eh = mtod(m, struct ether_header *);
-	eh->ether_type = etype;
-	memcpy(eh->ether_dhost, edst, sizeof(eh->ether_dhost));
-	memcpy(eh->ether_shost, esrc, sizeof(eh->ether_shost));
+	return (0);
 
-	return (if_enqueue(ifp, m));
 bad:
 	m_freem(m);
 	return (error);
 }
 
+struct mbuf*
+ether_encap(struct ifnet *ifp, struct mbuf *m, struct sockaddr *dst,
+    struct rtentry *rt, int *errorp)
+{
+	struct ether_header eh;
+	int error;
+
+	error = ether_resolve(ifp, m, dst, rt, &eh);
+	switch (error) {
+	case 0:
+		break;
+	case EAGAIN:
+		error = 0;
+	default:
+		*errorp = error;
+		return (NULL);
+	}
+
+	m = m_prepend(m, ETHER_ALIGN + sizeof(eh), M_DONTWAIT);
+	if (m == NULL) {
+		*errorp = ENOBUFS;
+		return (NULL);
+	}
+
+	m_adj(m, ETHER_ALIGN);
+	memcpy(mtod(m, struct ether_header *), &eh, sizeof(eh));
+
+	return (m);
+}
+
+int
+ether_output(struct ifnet *ifp, struct mbuf *m, struct sockaddr *dst,
+    struct rtentry *rt)
+{
+	int error;
+
+	m = ether_encap(ifp, m, dst, rt, &error);
+	if (m == NULL)
+		return (error);
+
+	return (if_enqueue(ifp, m));
+}
+
 /*
  * Process a received Ethernet packet;
  * the packet is in the mbuf chain m without
diff --git a/sys/netinet/if_ether.h b/sys/netinet/if_ether.h
index fdff5c8c1cf..441d2696176 100644
--- a/sys/netinet/if_ether.h
+++ b/sys/netinet/if_ether.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: if_ether.h,v 1.73 2016/11/29 10:09:57 reyk Exp $	*/
+/*	$OpenBSD: if_ether.h,v 1.74 2018/12/11 01:27:08 dlg Exp $	*/
 /*	$NetBSD: if_ether.h,v 1.22 1996/05/11 13:00:00 mycroft Exp $	*/
 
 /*
@@ -240,8 +240,13 @@ void	ether_ifattach(struct ifnet *);
 void	ether_ifdetach(struct ifnet *);
 int	ether_ioctl(struct ifnet *, struct arpcom *, u_long, caddr_t);
 int	ether_input(struct ifnet *, struct mbuf *, void *);
-int	ether_output(struct ifnet *,
-	    struct mbuf *, struct sockaddr *, struct rtentry *);
+int	ether_resolve(struct ifnet *, struct mbuf *, struct sockaddr *,
+	    struct rtentry *, struct ether_header *);
+struct mbuf *
+	ether_encap(struct ifnet *, struct mbuf *, struct sockaddr *,
+	    struct rtentry *, int *);
+int	ether_output(struct ifnet *, struct mbuf *, struct sockaddr *,
+	    struct rtentry *);
 void	ether_rtrequest(struct ifnet *, int, struct rtentry *);
 char	*ether_sprintf(u_char *);