Use TSO offloading in ix(4).

With a lot of tweaks, improvements and testing from bluhm.

Thanks to Hrvoje Popovski from the University of Zagreb for
his great testing effort to make this happen.

ok bluhm

4 files changed, 58 insertions, 12 deletions

diff --git a/sys/dev/pci/if_ix.c b/sys/dev/pci/if_ix.c
index 8bf3ef49429..4119a2416dc 100644
--- a/sys/dev/pci/if_ix.c
+++ b/sys/dev/pci/if_ix.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: if_ix.c,v 1.194 2023/05/16 14:32:54 jan Exp $	*/
+/*	$OpenBSD: if_ix.c,v 1.195 2023/05/18 08:22:37 jan Exp $	*/
 
 /******************************************************************************
 
@@ -1924,6 +1924,7 @@ ixgbe_setup_interface(struct ix_softc *sc)
 	ifp->if_capabilities |= IFCAP_CSUM_TCPv6 | IFCAP_CSUM_UDPv6;
 	ifp->if_capabilities |= IFCAP_CSUM_IPv4;
 
+	ifp->if_capabilities |= IFCAP_TSOv4 | IFCAP_TSOv6;
 	if (sc->hw.mac.type != ixgbe_mac_82598EB)
 		ifp->if_capabilities |= IFCAP_LRO;
 
@@ -2344,6 +2345,7 @@ ixgbe_initialize_transmit_units(struct ix_softc *sc)
 	int		 i;
 	uint64_t	 tdba;
 	uint32_t	 txctrl;
+	uint32_t	 hlreg;
 
 	/* Setup the Base and Length of the Tx Descriptor Ring */
 
@@ -2405,6 +2407,11 @@ ixgbe_initialize_transmit_units(struct ix_softc *sc)
 		rttdcs &= ~IXGBE_RTTDCS_ARBDIS;
 		IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs);
 	}
+
+	/* Enable TCP/UDP padding when using TSO */
+	hlreg = IXGBE_READ_REG(hw, IXGBE_HLREG0);
+	hlreg |= IXGBE_HLREG0_TXPADEN;
+	IXGBE_WRITE_REG(hw, IXGBE_HLREG0, hlreg);
 }
 
 /*********************************************************************
@@ -2473,16 +2480,18 @@ ixgbe_free_transmit_buffers(struct tx_ring *txr)
  **********************************************************************/
 
 static inline int
-ixgbe_csum_offload(struct mbuf *mp, uint32_t *vlan_macip_lens,
-    uint32_t *type_tucmd_mlhl, uint32_t *olinfo_status)
+ixgbe_tx_offload(struct mbuf *mp, uint32_t *vlan_macip_lens,
+    uint32_t *type_tucmd_mlhl, uint32_t *olinfo_status, uint32_t *cmd_type_len,
+    uint32_t *mss_l4len_idx)
 {
 	struct ether_extracted ext;
 	int offload = 0;
-	uint32_t iphlen;
+	uint32_t ethlen, iphlen;
 
 	ether_extract_headers(mp, &ext);
+	ethlen = sizeof(*ext.eh);
 
-	*vlan_macip_lens |= (sizeof(*ext.eh) << IXGBE_ADVTXD_MACLEN_SHIFT);
+	*vlan_macip_lens |= (ethlen << IXGBE_ADVTXD_MACLEN_SHIFT);
 
 	if (ext.ip4) {
 		iphlen = ext.ip4->ip_hl << 2;
@@ -2500,6 +2509,8 @@ ixgbe_csum_offload(struct mbuf *mp, uint32_t *vlan_macip_lens,
 		*type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV6;
 #endif
 	} else {
+		if (mp->m_pkthdr.csum_flags & M_TCP_TSO)
+			tcpstat_inc(tcps_outbadtso);
 		return offload;
 	}
 
@@ -2519,6 +2530,31 @@ ixgbe_csum_offload(struct mbuf *mp, uint32_t *vlan_macip_lens,
 		}
 	}
 
+	if (mp->m_pkthdr.csum_flags & M_TCP_TSO) {
+		if (ext.tcp) {
+			uint32_t hdrlen, thlen, paylen, outlen;
+
+			thlen = ext.tcp->th_off << 2;
+
+			outlen = mp->m_pkthdr.ph_mss;
+			*mss_l4len_idx |= outlen << IXGBE_ADVTXD_MSS_SHIFT;
+			*mss_l4len_idx |= thlen << IXGBE_ADVTXD_L4LEN_SHIFT;
+
+			hdrlen = ethlen + iphlen + thlen;
+			paylen = mp->m_pkthdr.len - hdrlen;
+			CLR(*olinfo_status, IXGBE_ADVTXD_PAYLEN_MASK
+			    << IXGBE_ADVTXD_PAYLEN_SHIFT);
+			*olinfo_status |= paylen << IXGBE_ADVTXD_PAYLEN_SHIFT;
+
+			*cmd_type_len |= IXGBE_ADVTXD_DCMD_TSE;
+			offload = 1;
+
+			tcpstat_add(tcps_outpkttso,
+			    (paylen + outlen - 1) / outlen);
+		} else
+			tcpstat_inc(tcps_outbadtso);
+	}
+
 	return offload;
 }
 
@@ -2529,6 +2565,7 @@ ixgbe_tx_ctx_setup(struct tx_ring *txr, struct mbuf *mp,
 	struct ixgbe_adv_tx_context_desc *TXD;
 	struct ixgbe_tx_buf *tx_buffer;
 	uint32_t vlan_macip_lens = 0, type_tucmd_mlhl = 0;
+	uint32_t mss_l4len_idx = 0;
 	int	ctxd = txr->next_avail_desc;
 	int	offload = 0;
 
@@ -2544,8 +2581,8 @@ ixgbe_tx_ctx_setup(struct tx_ring *txr, struct mbuf *mp,
 	}
 #endif
 
-	offload |= ixgbe_csum_offload(mp, &vlan_macip_lens, &type_tucmd_mlhl,
-	    olinfo_status);
+	offload |= ixgbe_tx_offload(mp, &vlan_macip_lens, &type_tucmd_mlhl,
+	    olinfo_status, cmd_type_len, &mss_l4len_idx);
 
 	if (!offload)
 		return (0);
@@ -2559,7 +2596,7 @@ ixgbe_tx_ctx_setup(struct tx_ring *txr, struct mbuf *mp,
 	TXD->vlan_macip_lens = htole32(vlan_macip_lens);
 	TXD->type_tucmd_mlhl = htole32(type_tucmd_mlhl);
 	TXD->seqnum_seed = htole32(0);
-	TXD->mss_l4len_idx = htole32(0);
+	TXD->mss_l4len_idx = htole32(mss_l4len_idx);
 
 	tx_buffer->m_head = NULL;
 	tx_buffer->eop_index = -1;
@@ -2868,9 +2905,11 @@ ixgbe_initialize_receive_units(struct ix_softc *sc)
 	}
 	IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl);
 
-	/* Always enable jumbo frame reception */
 	hlreg = IXGBE_READ_REG(hw, IXGBE_HLREG0);
+	/* Always enable jumbo frame reception */
 	hlreg |= IXGBE_HLREG0_JUMBOEN;
+	/* Always enable CRC stripping */
+	hlreg |= IXGBE_HLREG0_RXCRCSTRP;
 	IXGBE_WRITE_REG(hw, IXGBE_HLREG0, hlreg);
 
 	if (ISSET(ifp->if_xflags, IFXF_LRO)) {
diff --git a/sys/dev/pci/ixgbe.h b/sys/dev/pci/ixgbe.h
index af6ebde1f9f..5fbfba93d50 100644
--- a/sys/dev/pci/ixgbe.h
+++ b/sys/dev/pci/ixgbe.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: ixgbe.h,v 1.33 2022/02/08 03:38:00 dlg Exp $	*/
+/*	$OpenBSD: ixgbe.h,v 1.34 2023/05/18 08:22:37 jan Exp $	*/
 
 /******************************************************************************
 
@@ -60,12 +60,16 @@
 
 #include <net/if.h>
 #include <net/if_media.h>
+#include <net/route.h>
 #include <net/toeplitz.h>
 
 #include <netinet/in.h>
 #include <netinet/if_ether.h>
 #include <netinet/ip.h>
 #include <netinet/ip6.h>
+#include <netinet/tcp.h>
+#include <netinet/tcp_timer.h>
+#include <netinet/tcp_var.h>
 
 #if NBPFILTER > 0
 #include <net/bpf.h>
diff --git a/sys/dev/pci/ixgbe_type.h b/sys/dev/pci/ixgbe_type.h
index 78ae1e3158b..4b4dcb737f2 100644
--- a/sys/dev/pci/ixgbe_type.h
+++ b/sys/dev/pci/ixgbe_type.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: ixgbe_type.h,v 1.36 2022/01/09 05:42:56 jsg Exp $	*/
+/*	$OpenBSD: ixgbe_type.h,v 1.37 2023/05/18 08:22:37 jan Exp $	*/
 
 /******************************************************************************
   SPDX-License-Identifier: BSD-3-Clause
@@ -3355,6 +3355,7 @@ struct ixgbe_adv_tx_context_desc {
 /* 1st&Last TSO-full iSCSI PDU */
 #define IXGBE_ADVTXD_POPTS_ISCO_FULL	0x00001800
 #define IXGBE_ADVTXD_POPTS_RSV		0x00002000 /* POPTS Reserved */
+#define IXGBE_ADVTXD_PAYLEN_MASK	0x0003FFFF /* Adv desc PAYLEN */
 #define IXGBE_ADVTXD_PAYLEN_SHIFT	14 /* Adv desc PAYLEN shift */
 #define IXGBE_ADVTXD_MACLEN_SHIFT	9  /* Adv ctxt desc mac len shift */
 #define IXGBE_ADVTXD_VLAN_SHIFT		16  /* Adv ctxt vlan tag shift */
diff --git a/sys/netinet/tcp_var.h b/sys/netinet/tcp_var.h
index fc64e11876c..0a9630d719f 100644
--- a/sys/netinet/tcp_var.h
+++ b/sys/netinet/tcp_var.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: tcp_var.h,v 1.165 2023/05/15 16:34:56 bluhm Exp $	*/
+/*	$OpenBSD: tcp_var.h,v 1.166 2023/05/18 08:22:37 jan Exp $	*/
 /*	$NetBSD: tcp_var.h,v 1.17 1996/02/13 23:44:24 christos Exp $	*/
 
 /*
@@ -688,6 +688,8 @@ extern	int tcp_syn_use_limit;   /* number of uses before reseeding hash */
 extern	struct syn_cache_set tcp_syn_cache[];
 extern	int tcp_syn_cache_active; /* active syn cache, may be 0 or 1 */
 
+struct tdb;
+
 void	 tcp_canceltimers(struct tcpcb *);
 struct tcpcb *
 	 tcp_close(struct tcpcb *);