replace IFF_OACTIVE manipulation with mpsafe operations.

there are two things shared between the network stack and drivers
in the send path: the send queue and the IFF_OACTIVE flag. the send
queue is now protected by a mutex. this diff makes the oactive
functionality mpsafe too.

IFF_OACTIVE is part of if_flags. there are two problems with that.
firstly, if_flags is a short and we dont have any MI atomic operations
to manipulate a short. secondly, while we could make the IFF_OACTIVE
operates mpsafe, all changes to other flags would have to be made
safe at the same time, otherwise a read-modify-write cycle on their
updates could clobber the oactive change.

instead, this moves the oactive mark into struct ifqueue and provides
an API for changing it. there's ifq_set_oactive, ifq_clr_oactive,
and ifq_is_oactive. these are modelled on ifsq_set_oactive,
ifsq_clr_oactive, and ifsq_is_oactive in dragonflybsd.

this diff includes changes to all the drivers manipulating IFF_OACTIVE
to now use the ifsq_{set,clr_is}_oactive API too.

ok kettenis@ mpi@ jmatthew@ deraadt@

4 files changed, 23 insertions, 21 deletions

diff --git a/sys/arch/sparc/dev/be.c b/sys/arch/sparc/dev/be.c
index a45833e6708..966ea70bc3a 100644
--- a/sys/arch/sparc/dev/be.c
+++ b/sys/arch/sparc/dev/be.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: be.c,v 1.56 2015/11/24 17:11:38 mpi Exp $	*/
+/*	$OpenBSD: be.c,v 1.57 2015/11/25 03:09:58 dlg Exp $	*/
 
 /*
  * Copyright (c) 1998 Theo de Raadt and Jason L. Wright.
@@ -251,7 +251,7 @@ bestart(ifp)
 		be_tx_harvest(sc);
 	}
 
-	if ((ifp->if_flags & (IFF_RUNNING | IFF_OACTIVE)) != IFF_RUNNING)
+	if (!(ifp->if_flags & IFF_RUNNING) || ifq_is_oactive(&ifp->if_snd))
 		return;
 
 	bix = sc->sc_last_td;
@@ -288,7 +288,7 @@ bestart(ifp)
 			bix = 0;
 
 		if (++cnt == BE_TX_RING_SIZE) {
-			ifp->if_flags |= IFF_OACTIVE;
+			ifq_set_oactive(&ifp->if_snd);
 			break;
 		}
 	}
@@ -469,7 +469,7 @@ be_tx_harvest(sc)
 	if (sc->sc_no_td != cnt) {
 		sc->sc_first_td = bix;
 		sc->sc_no_td = cnt;
-		ifp->if_flags &= ~IFF_OACTIVE;
+		ifq_clr_oactive(&ifp->if_snd);
 	}
 
 	if (sc->sc_no_td < BE_TX_LOW_WATER) {
@@ -714,7 +714,7 @@ beinit(sc)
 	br->rx_cfg |= BE_BR_RXCFG_ENABLE;
 
 	ifp->if_flags |= IFF_RUNNING;
-	ifp->if_flags &= ~IFF_OACTIVE;
+	ifq_clr_oactive(&ifp->if_snd);
 	splx(s);
 
 	timeout_add_sec(&sc->sc_tick, 1);
diff --git a/sys/arch/sparc/dev/hme.c b/sys/arch/sparc/dev/hme.c
index a197caa59a3..217c8e8a642 100644
--- a/sys/arch/sparc/dev/hme.c
+++ b/sys/arch/sparc/dev/hme.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: hme.c,v 1.75 2015/11/24 17:11:38 mpi Exp $	*/
+/*	$OpenBSD: hme.c,v 1.76 2015/11/25 03:09:58 dlg Exp $	*/
 
 /*
  * Copyright (c) 1998 Jason L. Wright (jason@thought.net)
@@ -272,7 +272,7 @@ hmestart(ifp)
 	struct mbuf *m;
 	int bix, len;
 
-	if ((ifp->if_flags & (IFF_RUNNING | IFF_OACTIVE)) != IFF_RUNNING)
+	if (!(ifp->if_flags & IFF_RUNNING) || ifq_is_oactive(&ifp->if_snd))
 		return;
 
 	bix = sc->sc_last_td;
@@ -307,7 +307,7 @@ hmestart(ifp)
 			bix = 0;
 
 		if (++sc->sc_no_td == HME_TX_RING_SIZE) {
-			ifp->if_flags |= IFF_OACTIVE;
+			ifq_set_oactive(&ifp->if_snd);
 			break;
 		}
 	}
@@ -329,7 +329,8 @@ hmestop(sc)
 	/*
 	 * Mark the interface down and cancel the watchdog timer.
 	 */
-	ifp->if_flags &= ~(IFF_RUNNING | IFF_OACTIVE);
+	ifp->if_flags &= ~IFF_RUNNING;
+	ifq_clr_oactive(&ifp->if_snd);
 	ifp->if_timer = 0;
 
 	mii_down(&sc->sc_mii);
@@ -571,7 +572,7 @@ hmeinit(sc)
 	timeout_add_sec(&sc->sc_tick, 1);
 
 	ifp->if_flags |= IFF_RUNNING;
-	ifp->if_flags &= ~IFF_OACTIVE;
+	ifq_clr_oactive(&ifp->if_snd);
 }
 
 void
@@ -669,7 +670,7 @@ hme_tint(sc)
 		if (txd.tx_flags & HME_TXD_OWN)
 			break;
 
-		ifp->if_flags &= ~IFF_OACTIVE;
+		ifq_clr_oactive(&ifp->if_snd);
 		ifp->if_opackets++;
 
 		if (++bix == HME_TX_RING_SIZE)
diff --git a/sys/arch/sparc/dev/if_ie.c b/sys/arch/sparc/dev/if_ie.c
index 50bc9282f77..5430c53467b 100644
--- a/sys/arch/sparc/dev/if_ie.c
+++ b/sys/arch/sparc/dev/if_ie.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: if_ie.c,v 1.58 2015/11/24 17:11:38 mpi Exp $	*/
+/*	$OpenBSD: if_ie.c,v 1.59 2015/11/25 03:09:58 dlg Exp $	*/
 /*	$NetBSD: if_ie.c,v 1.33 1997/07/29 17:55:38 fair Exp $	*/
 
 /*-
@@ -819,7 +819,7 @@ ietint(sc)
 	int status;
 
 	sc->sc_arpcom.ac_if.if_timer = 0;
-	sc->sc_arpcom.ac_if.if_flags &= ~IFF_OACTIVE;
+	ifq_clr_oactive(&sc->sc_arpcom.ac_if.if_snd);
 
 	status = sc->xmit_cmds[sc->xctail]->ie_xmit_status;
 
@@ -1306,7 +1306,7 @@ iestart(ifp)
 		return;
 
 	if (sc->xmit_free == 0) {
-		ifp->if_flags |= IFF_OACTIVE;
+		ifq_set_oactive(&ifp->if_snd);
 		if (!sc->xmit_busy)
 			iexmit(sc);
 		return;
@@ -1406,7 +1406,8 @@ iereset(sc)
 	printf("%s: reset\n", sc->sc_dev.dv_xname);
 
 	/* Clear OACTIVE in case we're called from watchdog (frozen xmit). */
-	sc->sc_arpcom.ac_if.if_flags &= ~(IFF_UP | IFF_OACTIVE);
+	sc->sc_arpcom.ac_if.if_flags &= ~IFF_UP;
+	ifq_clr_oactive(&sc->sc_arpcom.ac_if.if_snd);
 	ieioctl(&sc->sc_arpcom.ac_if, SIOCSIFFLAGS, 0);
 
 	/*
diff --git a/sys/arch/sparc/dev/qe.c b/sys/arch/sparc/dev/qe.c
index 0da22b6e595..8251d544311 100644
--- a/sys/arch/sparc/dev/qe.c
+++ b/sys/arch/sparc/dev/qe.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: qe.c,v 1.46 2015/11/24 17:11:38 mpi Exp $	*/
+/*	$OpenBSD: qe.c,v 1.47 2015/11/25 03:09:58 dlg Exp $	*/
 
 /*
  * Copyright (c) 1998, 2000 Jason L. Wright.
@@ -192,7 +192,7 @@ qestart(ifp)
 	struct mbuf *m;
 	int bix, len;
 
-	if ((ifp->if_flags & (IFF_RUNNING | IFF_OACTIVE)) != IFF_RUNNING)
+	if (!(ifp->if_flags & IFF_RUNNING) || ifq_is_oactive(&ifp->if_snd))
 		return;
 
 	bix = sc->sc_last_td;
@@ -228,7 +228,7 @@ qestart(ifp)
 			bix = 0;
 
 		if (++sc->sc_no_td == QE_TX_RING_SIZE) {
-			ifp->if_flags |= IFF_OACTIVE;
+			ifq_set_oactive(&ifp->if_snd);
 			break;
 		}
 	}
@@ -355,8 +355,8 @@ qe_tint(sc)
 	 */
 	if (sc->sc_first_td != bix) {
 		sc->sc_first_td = bix;
-		if (ifp->if_flags & IFF_OACTIVE) {
-			ifp->if_flags &= ~IFF_OACTIVE;
+		if (ifq_is_oactive(&ifp->if_snd)) {
+			ifq_clr_oactive(&ifp->if_snd);
 			qestart(ifp);
 		}
 	}
@@ -717,7 +717,7 @@ qeinit(sc)
 	i = mr->mpc;	/* cleared on read */
 
 	ifp->if_flags |= IFF_RUNNING;
-	ifp->if_flags &= ~IFF_OACTIVE;
+	ifq_clr_oactive(&ifp->if_snd);
 
 	mr->maccc = QE_MR_MACCC_ENXMT | QE_MR_MACCC_ENRCV |
 	    ((ifp->if_flags & IFF_PROMISC) ? QE_MR_MACCC_PROM : 0);