mask and unmask the interrupt source in an intx specific intr handler.

it seems devices using levelled intx interrupts need to explicitely ack
interrupts by masking and unmasking the source around the completion
ring handling. without this completions can be lost, which in turn
causes long (permanent?) stalls in the block layer under heavy write
load.

ive experienced this problem with an intel nvme part that only has
intx and msix support. because we dont support msix yet we only
use intx on it. it appeared to lock up before this fix.

this has been tested on both that intel board and a samsung with msi.
this fix was based on work found in code by nonaka

3 files changed, 25 insertions, 7 deletions

diff --git a/sys/dev/ic/nvme.c b/sys/dev/ic/nvme.c
index fa021e42f6c..9ccff72a27f 100644
--- a/sys/dev/ic/nvme.c
+++ b/sys/dev/ic/nvme.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: nvme.c,v 1.50 2016/05/20 11:11:05 dlg Exp $ */
+/*	$OpenBSD: nvme.c,v 1.51 2016/10/25 06:20:41 dlg Exp $ */
 
 /*
  * Copyright (c) 2014 David Gwynne <dlg@openbsd.org>
@@ -1277,6 +1277,19 @@ nvme_intr(void *xsc)
 	return (rv);
 }
 
+int
+nvme_intr_intx(void *xsc)
+{
+	struct nvme_softc *sc = xsc;
+	int rv;
+
+	nvme_write4(sc, NVME_INTMS, 1);
+	rv = nvme_intr(sc);
+	nvme_write4(sc, NVME_INTMC, 1);
+
+	return (rv);
+}
+
 struct nvme_dmamem *
 nvme_dmamem_alloc(struct nvme_softc *sc, size_t size)
 {
diff --git a/sys/dev/ic/nvmevar.h b/sys/dev/ic/nvmevar.h
index f59de44967b..aced9feb6c3 100644
--- a/sys/dev/ic/nvmevar.h
+++ b/sys/dev/ic/nvmevar.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: nvmevar.h,v 1.8 2016/04/14 11:18:32 dlg Exp $ */
+/*	$OpenBSD: nvmevar.h,v 1.9 2016/10/25 06:20:41 dlg Exp $ */
 
 /*
  * Copyright (c) 2014 David Gwynne <dlg@openbsd.org>
@@ -102,5 +102,6 @@ struct nvme_softc {
 int	nvme_attach(struct nvme_softc *);
 int	nvme_activate(struct nvme_softc *, int);
 int	nvme_intr(void *);
+int	nvme_intr_intx(void *);
 
 #define DEVNAME(_sc) ((_sc)->sc_dev.dv_xname)
diff --git a/sys/dev/pci/nvme_pci.c b/sys/dev/pci/nvme_pci.c
index 1fe72591992..e40a09268d7 100644
--- a/sys/dev/pci/nvme_pci.c
+++ b/sys/dev/pci/nvme_pci.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: nvme_pci.c,v 1.3 2016/04/14 11:18:32 dlg Exp $ */
+/*	$OpenBSD: nvme_pci.c,v 1.4 2016/10/25 06:20:41 dlg Exp $ */
 
 /*
  * Copyright (c) 2014 David Gwynne <dlg@openbsd.org>
@@ -81,6 +81,7 @@ nvme_pci_attach(struct device *parent, struct device *self, void *aux)
 	struct pci_attach_args *pa = aux;
 	pcireg_t maptype;
 	pci_intr_handle_t ih;
+	int msi = 1;
 
 	psc->psc_pc = pa->pa_pc;
 	sc->sc_dmat = pa->pa_dmat;
@@ -92,13 +93,16 @@ nvme_pci_attach(struct device *parent, struct device *self, void *aux)
 		return;
 	}
 
-	if (pci_intr_map_msi(pa, &ih) != 0 && pci_intr_map(pa, &ih) != 0) {
-		printf(": unable to map interrupt\n");
-		goto unmap;
+	if (pci_intr_map_msi(pa, &ih) != 0) {
+		if (pci_intr_map(pa, &ih) != 0) {
+			printf(": unable to map interrupt\n");
+			goto unmap;
+		}
+		msi = 0;
 	}
 
 	sc->sc_ih = pci_intr_establish(pa->pa_pc, ih, IPL_BIO,
-	    nvme_intr, sc, DEVNAME(sc));
+	    msi ? nvme_intr : nvme_intr_intx, sc, DEVNAME(sc));
 	if (sc->sc_ih == NULL) {
 		printf(": unable to establish interrupt\n");
 		goto unmap;