Run the tx completion path without the kernel held. This makes the

"fast path" through the interrupt handler not grab the kernel lock anymore.
This removes the code that attempts to reclaim tx descriptors from em_start().
Keeping that code would have complicated the locking.  The need to reclaim
tx descriptors that way should have largely disappeared now that the interrupt
handler doesn't have to wait on the kernel lock.

ok mpi@
tested by many

2 files changed, 29 insertions, 47 deletions

diff --git a/sys/dev/pci/if_em.c b/sys/dev/pci/if_em.c
index d209b01365b..94d906b6572 100644
--- a/sys/dev/pci/if_em.c
+++ b/sys/dev/pci/if_em.c
@@ -31,7 +31,7 @@ POSSIBILITY OF SUCH DAMAGE.
 
 ***************************************************************************/
 
-/* $OpenBSD: if_em.c,v 1.305 2015/09/19 12:48:26 kettenis Exp $ */
+/* $OpenBSD: if_em.c,v 1.306 2015/09/30 11:25:08 kettenis Exp $ */
 /* $FreeBSD: if_em.c,v 1.46 2004/09/29 18:28:28 mlaier Exp $ */
 
 #include <dev/pci/if_em.h>
@@ -920,20 +920,15 @@ em_intr(void *arg)
 	if (reg_icr & E1000_ICR_RXO)
 		sc->rx_overruns++;
 
-	KERNEL_LOCK();
-
 	/* Link status change */
 	if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
+		KERNEL_LOCK();
 		sc->hw.get_link_status = 1;
 		em_check_for_link(&sc->hw);
 		em_update_link_status(sc);
+		KERNEL_UNLOCK();
 	}
 
-	if (ifp->if_flags & IFF_RUNNING && !IFQ_IS_EMPTY(&ifp->if_snd))
-		em_start(ifp);
-
-	KERNEL_UNLOCK();
-
 	if (refill && em_rxfill(sc)) {
 		/* Advance the Rx Queue #0 "Tail Pointer". */
 		E1000_WRITE_REG(&sc->hw, RDT, sc->last_rx_desc_filled);
@@ -1110,17 +1105,10 @@ em_encap(struct em_softc *sc, struct mbuf *m_head)
 	struct em_buffer   *tx_buffer, *tx_buffer_mapped;
 	struct em_tx_desc *current_tx_desc = NULL;
 
-	/*
-	 * Force a cleanup if number of TX descriptors
-	 * available hits the threshold
-	 */
-	if (sc->num_tx_desc_avail <= EM_TX_CLEANUP_THRESHOLD) {
-		em_txeof(sc);
-		/* Now do we at least have a minimal? */
-		if (sc->num_tx_desc_avail <= EM_TX_OP_THRESHOLD) {
-			sc->no_tx_desc_avail1++;
-			return (ENOBUFS);
-		}
+	/* Check that we have least the minimal number of TX descriptors. */
+	if (sc->num_tx_desc_avail <= EM_TX_OP_THRESHOLD) {
+		sc->no_tx_desc_avail1++;
+		return (ENOBUFS);
 	}
 
 	if (sc->hw.mac_type == em_82547) {
@@ -1224,9 +1212,9 @@ em_encap(struct em_softc *sc, struct mbuf *m_head)
 
 	sc->next_avail_tx_desc = i;
 	if (sc->pcix_82544)
-		sc->num_tx_desc_avail -= txd_used;
+		atomic_sub_int(&sc->num_tx_desc_avail, txd_used);
 	else
-		sc->num_tx_desc_avail -= map->dm_nsegs;
+		atomic_sub_int(&sc->num_tx_desc_avail, map->dm_nsegs);
 
 #if NVLAN > 0
 	/* Find out if we are in VLAN mode */
@@ -2393,7 +2381,7 @@ em_transmit_checksum_setup(struct em_softc *sc, struct mbuf *mp,
 	if (++curr_txd == sc->num_tx_desc)
 		curr_txd = 0;
 
-	sc->num_tx_desc_avail--;
+	atomic_dec_int(&sc->num_tx_desc_avail);
 	sc->next_avail_tx_desc = curr_txd;
 }
 
@@ -2407,7 +2395,7 @@ em_transmit_checksum_setup(struct em_softc *sc, struct mbuf *mp,
 void
 em_txeof(struct em_softc *sc)
 {
-	int first, last, done, num_avail;
+	int first, last, done, num_avail, free = 0;
 	struct em_buffer *tx_buffer;
 	struct em_tx_desc   *tx_desc, *eop_desc;
 	struct ifnet   *ifp = &sc->interface_data.ac_if;
@@ -2415,9 +2403,6 @@ em_txeof(struct em_softc *sc)
 	if (sc->num_tx_desc_avail == sc->num_tx_desc)
 		return;
 
-	KERNEL_LOCK();
-
-	num_avail = sc->num_tx_desc_avail;
 	first = sc->next_tx_to_clean;
 	tx_desc = &sc->tx_desc_base[first];
 	tx_buffer = &sc->tx_buffer_area[first];
@@ -2441,7 +2426,7 @@ em_txeof(struct em_softc *sc)
 		while (first != done) {
 			tx_desc->upper.data = 0;
 			tx_desc->lower.data = 0;
-			num_avail++;
+			free++;
 
 			if (tx_buffer->m_head != NULL) {
 				ifp->if_opackets++;
@@ -2481,25 +2466,23 @@ em_txeof(struct em_softc *sc)
 
 	sc->next_tx_to_clean = first;
 
-	/*
-	 * If we have enough room, clear IFF_OACTIVE to tell the stack
-	 * that it is OK to send packets.
-	 * If there are no pending descriptors, clear the timeout. Otherwise,
-	 * if some descriptors have been freed, restart the timeout.
-	 */
-	if (num_avail > EM_TX_CLEANUP_THRESHOLD)
-		ifp->if_flags &= ~IFF_OACTIVE;
+	num_avail = atomic_add_int_nv(&sc->num_tx_desc_avail, free);
 
 	/* All clean, turn off the timer */
 	if (num_avail == sc->num_tx_desc)
 		ifp->if_timer = 0;
-	/* Some cleaned, reset the timer */
-	else if (num_avail != sc->num_tx_desc_avail)
-		ifp->if_timer = EM_TX_TIMEOUT;
-
-	sc->num_tx_desc_avail = num_avail;
 
-	KERNEL_UNLOCK();
+	/*
+	 * If we have enough room, clear IFF_OACTIVE to tell the stack
+	 * that it is OK to send packets.
+	 */
+	if (ISSET(ifp->if_flags, IFF_OACTIVE) &&
+	    num_avail > EM_TX_OP_THRESHOLD) {
+		KERNEL_LOCK();
+		CLR(ifp->if_flags, IFF_OACTIVE);
+		em_start(ifp);
+		KERNEL_UNLOCK();
+	}
 }
 
 /*********************************************************************
diff --git a/sys/dev/pci/if_em.h b/sys/dev/pci/if_em.h
index d9149efb19d..7f0ab234add 100644
--- a/sys/dev/pci/if_em.h
+++ b/sys/dev/pci/if_em.h
@@ -32,7 +32,7 @@ POSSIBILITY OF SUCH DAMAGE.
 ***************************************************************************/
 
 /* $FreeBSD: if_em.h,v 1.26 2004/09/01 23:22:41 pdeuskar Exp $ */
-/* $OpenBSD: if_em.h,v 1.57 2015/09/19 12:48:26 kettenis Exp $ */
+/* $OpenBSD: if_em.h,v 1.58 2015/09/30 11:25:08 kettenis Exp $ */
 
 #ifndef _EM_H_DEFINED_
 #define _EM_H_DEFINED_
@@ -187,10 +187,9 @@ typedef int	boolean_t;
 #define EM_TX_TIMEOUT			5	/* set to 5 seconds */
 
 /*
- * These parameters control when the driver calls the routine to reclaim
- * transmit descriptors.
+ * Thise parameter controls the minimum number of available transmit
+ * descriptors needed before we attempt transmission of a packet.
  */
-#define EM_TX_CLEANUP_THRESHOLD		(sc->num_tx_desc / 8)
 #define EM_TX_OP_THRESHOLD		(sc->num_tx_desc / 32)
 
 /*
@@ -356,8 +355,8 @@ struct em_softc {
 	struct em_tx_desc	*tx_desc_base;
 	u_int32_t		next_avail_tx_desc;
 	u_int32_t		next_tx_to_clean;
-	volatile u_int16_t	num_tx_desc_avail;
-	u_int16_t		num_tx_desc;
+	volatile u_int32_t	num_tx_desc_avail;
+	u_int32_t		num_tx_desc;
 	u_int32_t		txd_cmd;
 	struct em_buffer	*tx_buffer_area;
 	bus_dma_tag_t		txtag;		/* dma tag for tx */