Switch USB to use non-coherent buffers for data transfers. Since

the import in '99 all buffers allocated using usb_allocmem() have
been mapped COHERENT.  On some ARM SoCs, where the USB controller
is not coherent with the caches, this means the buffers were mapped
uncached.  This drastically reduces the performance, especially on
reads.

We already added cache syncs before and after USB transfers, but so
far those have essentially been no-ops.  Since other drivers make
use of the same allocation code, and those haven't been proven to
have correct syncs, we can't just remove the COHERENT flag.

This splits the allocation into coherent and non-coherent blocks.
All drivers who call usb_allocmem() themselves now pass a flag to
show they require coherent blocks.  Onced verified that they also
work fine without coherent, or once they have been refactored, we
can remove this again.

On a ure(4) connected to an i.MX8MQ, the receive performance is
10x as fast as before.  The Raspberry Pi's ethernet receive speed
doubled.

Debugged using dt(4)
Original diff from Marius Strobl
Feedback from kettenis@
Tested on an i.MX8MQ (arm64) by patrick@
Tested on a Raspberry Pi (arm64) by tobhe@
Tested on an ERL (octeon) by Janne Johansson
ok mpi@ gerhard@ tobhe@

9 files changed, 46 insertions, 16 deletions

diff --git a/sys/dev/usb/dwc2/dwc2.c b/sys/dev/usb/dwc2/dwc2.c
index 6f035467213..9f795500b02 100644
--- a/sys/dev/usb/dwc2/dwc2.c
+++ b/sys/dev/usb/dwc2/dwc2.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: dwc2.c,v 1.49 2019/11/27 11:16:59 mpi Exp $	*/
+/*	$OpenBSD: dwc2.c,v 1.50 2020/03/19 14:18:38 patrick Exp $	*/
 /*	$NetBSD: dwc2.c,v 1.32 2014/09/02 23:26:20 macallan Exp $	*/
 
 /*-
@@ -473,6 +473,7 @@ dwc2_open(struct usbd_pipe *pipe)
 	switch (xfertype) {
 	case UE_CONTROL:
 		pipe->methods = &dwc2_device_ctrl_methods;
+		dpipe->req_dma.flags |= USB_DMA_COHERENT;
 		err = usb_allocmem(&sc->sc_bus, sizeof(usb_device_request_t),
 		    0, &dpipe->req_dma);
 		if (err)
diff --git a/sys/dev/usb/dwc2/dwc2_hcd.c b/sys/dev/usb/dwc2/dwc2_hcd.c
index 7e5c91481d5..04018d6b67e 100644
--- a/sys/dev/usb/dwc2/dwc2_hcd.c
+++ b/sys/dev/usb/dwc2/dwc2_hcd.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: dwc2_hcd.c,v 1.20 2017/09/08 05:36:53 deraadt Exp $	*/
+/*	$OpenBSD: dwc2_hcd.c,v 1.21 2020/03/19 14:18:38 patrick Exp $	*/
 /*	$NetBSD: dwc2_hcd.c,v 1.15 2014/11/24 10:14:14 skrll Exp $	*/
 
 /*
@@ -679,6 +679,7 @@ STATIC int dwc2_hc_setup_align_buf(struct dwc2_hsotg *hsotg, struct dwc2_qh *qh,
 
 		qh->dw_align_buf = NULL;
 		qh->dw_align_buf_dma = 0;
+		qh->dw_align_buf_usbdma.flags |= USB_DMA_COHERENT;
 		err = usb_allocmem(&hsotg->hsotg_sc->sc_bus, buf_size, buf_size,
 				   &qh->dw_align_buf_usbdma);
 		if (!err) {
@@ -2267,6 +2268,7 @@ int dwc2_hcd_init(struct dwc2_hsotg *hsotg,
 	 */
 	hsotg->status_buf = NULL;
 	if (hsotg->core_params->dma_enable > 0) {
+		hsotg->status_buf_usbdma.flags |= USB_DMA_COHERENT;
 		retval = usb_allocmem(&hsotg->hsotg_sc->sc_bus,
 				      DWC2_HCD_STATUS_BUF_SIZE, 0,
 				      &hsotg->status_buf_usbdma);
diff --git a/sys/dev/usb/dwc2/dwc2_hcdddma.c b/sys/dev/usb/dwc2/dwc2_hcdddma.c
index d8584eed50d..3077517154a 100644
--- a/sys/dev/usb/dwc2/dwc2_hcdddma.c
+++ b/sys/dev/usb/dwc2/dwc2_hcdddma.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: dwc2_hcdddma.c,v 1.14 2017/09/08 05:36:53 deraadt Exp $	*/
+/*	$OpenBSD: dwc2_hcdddma.c,v 1.15 2020/03/19 14:18:38 patrick Exp $	*/
 /*	$NetBSD: dwc2_hcdddma.c,v 1.6 2014/04/03 06:34:58 skrll Exp $	*/
 
 /*
@@ -102,6 +102,7 @@ STATIC int dwc2_desc_list_alloc(struct dwc2_hsotg *hsotg, struct dwc2_qh *qh,
 	//KASSERT(!cpu_intr_p() && !cpu_softintr_p());
 
 	qh->desc_list = NULL;
+	qh->desc_list_usbdma.flags |= USB_DMA_COHERENT;
 	err = usb_allocmem(&hsotg->hsotg_sc->sc_bus,
 	    sizeof(struct dwc2_hcd_dma_desc) * dwc2_max_desc_num(qh), 0,
 	    &qh->desc_list_usbdma);
@@ -143,6 +144,7 @@ STATIC int dwc2_frame_list_alloc(struct dwc2_hsotg *hsotg, gfp_t mem_flags)
 
 	/* XXXNH - struct pool */
 	hsotg->frame_list = NULL;
+	hsotg->frame_list_usbdma.flags |= USB_DMA_COHERENT;
 	err = usb_allocmem(&hsotg->hsotg_sc->sc_bus, 4 * FRLISTEN_64_SIZE,
 	    0, &hsotg->frame_list_usbdma);
 
diff --git a/sys/dev/usb/ehci.c b/sys/dev/usb/ehci.c
index ddde2796409..c0db91ac106 100644
--- a/sys/dev/usb/ehci.c
+++ b/sys/dev/usb/ehci.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: ehci.c,v 1.206 2020/02/22 14:01:34 jasper Exp $ */
+/*	$OpenBSD: ehci.c,v 1.207 2020/03/19 14:18:38 patrick Exp $ */
 /*	$NetBSD: ehci.c,v 1.66 2004/06/30 03:11:56 mycroft Exp $	*/
 
 /*
@@ -355,6 +355,7 @@ ehci_init(struct ehci_softc *sc)
 	case 3:
 		return (USBD_IOERROR);
 	}
+	sc->sc_fldma.flags |= USB_DMA_COHERENT;
 	err = usb_allocmem(&sc->sc_bus, sc->sc_flsize * sizeof(ehci_link_t),
 	    EHCI_FLALIGN_ALIGN, &sc->sc_fldma);
 	if (err)
@@ -1469,6 +1470,7 @@ ehci_open(struct usbd_pipe *pipe)
 
 	switch (xfertype) {
 	case UE_CONTROL:
+		epipe->u.ctl.reqdma.flags |= USB_DMA_COHERENT;
 		err = usb_allocmem(&sc->sc_bus, sizeof(usb_device_request_t),
 		    0, &epipe->u.ctl.reqdma);
 		if (err) {
@@ -2257,6 +2259,7 @@ ehci_alloc_sqh(struct ehci_softc *sc)
 	s = splusb();
 	if (sc->sc_freeqhs == NULL) {
 		DPRINTFN(2, ("ehci_alloc_sqh: allocating chunk\n"));
+		dma.flags |= USB_DMA_COHERENT;
 		err = usb_allocmem(&sc->sc_bus, EHCI_SQH_SIZE * EHCI_SQH_CHUNK,
 		    EHCI_PAGE_SIZE, &dma);
 		if (err)
@@ -2305,6 +2308,7 @@ ehci_alloc_sqtd(struct ehci_softc *sc)
 	s = splusb();
 	if (sc->sc_freeqtds == NULL) {
 		DPRINTFN(2, ("ehci_alloc_sqtd: allocating chunk\n"));
+		dma.flags |= USB_DMA_COHERENT;
 		err = usb_allocmem(&sc->sc_bus, EHCI_SQTD_SIZE*EHCI_SQTD_CHUNK,
 		    EHCI_PAGE_SIZE, &dma);
 		if (err)
@@ -2533,6 +2537,7 @@ ehci_alloc_itd(struct ehci_softc *sc)
 	}
 
 	if (freeitd == NULL) {
+		dma.flags |= USB_DMA_COHERENT;
 		err = usb_allocmem(&sc->sc_bus, EHCI_ITD_SIZE * EHCI_ITD_CHUNK,
 		    EHCI_PAGE_SIZE, &dma);
 		if (err) {
diff --git a/sys/dev/usb/ohci.c b/sys/dev/usb/ohci.c
index ad3a9811a9c..ed11da7bfb4 100644
--- a/sys/dev/usb/ohci.c
+++ b/sys/dev/usb/ohci.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: ohci.c,v 1.158 2020/02/22 14:01:34 jasper Exp $ */
+/*	$OpenBSD: ohci.c,v 1.159 2020/03/19 14:18:38 patrick Exp $ */
 /*	$NetBSD: ohci.c,v 1.139 2003/02/22 05:24:16 tsutsui Exp $	*/
 /*	$FreeBSD: src/sys/dev/usb/ohci.c,v 1.22 1999/11/17 22:33:40 n_hibma Exp $	*/
 
@@ -394,6 +394,7 @@ ohci_alloc_sed(struct ohci_softc *sc)
 	s = splusb();
 	if (sc->sc_freeeds == NULL) {
 		DPRINTFN(2, ("ohci_alloc_sed: allocating chunk\n"));
+		dma.flags |= USB_DMA_COHERENT;
 		err = usb_allocmem(&sc->sc_bus, OHCI_SED_SIZE * OHCI_SED_CHUNK,
 			  OHCI_ED_ALIGN, &dma);
 		if (err)
@@ -439,6 +440,7 @@ ohci_alloc_std(struct ohci_softc *sc)
 	s = splusb();
 	if (sc->sc_freetds == NULL) {
 		DPRINTFN(2, ("ohci_alloc_std: allocating chunk\n"));
+		dma.flags |= USB_DMA_COHERENT;
 		err = usb_allocmem(&sc->sc_bus, OHCI_STD_SIZE * OHCI_STD_CHUNK,
 			  OHCI_TD_ALIGN, &dma);
 		if (err)
@@ -597,6 +599,7 @@ ohci_alloc_sitd(struct ohci_softc *sc)
 
 	if (sc->sc_freeitds == NULL) {
 		DPRINTFN(2, ("ohci_alloc_sitd: allocating chunk\n"));
+		dma.flags |= USB_DMA_COHERENT;
 		err = usb_allocmem(&sc->sc_bus, OHCI_SITD_SIZE * OHCI_SITD_CHUNK,
 			  OHCI_ITD_ALIGN, &dma);
 		if (err)
@@ -728,6 +731,7 @@ ohci_init(struct ohci_softc *sc)
 
 	/* XXX determine alignment by R/W */
 	/* Allocate the HCCA area. */
+	sc->sc_hccadma.flags |= USB_DMA_COHERENT;
 	err = usb_allocmem(&sc->sc_bus, OHCI_HCCA_SIZE,
 			 OHCI_HCCA_ALIGN, &sc->sc_hccadma);
 	if (err)
@@ -1931,6 +1935,7 @@ ohci_open(struct usbd_pipe *pipe)
 		switch (xfertype) {
 		case UE_CONTROL:
 			pipe->methods = &ohci_device_ctrl_methods;
+			opipe->u.ctl.reqdma.flags |= USB_DMA_COHERENT;
 			err = usb_allocmem(&sc->sc_bus,
 				  sizeof(usb_device_request_t),
 				  0, &opipe->u.ctl.reqdma);
diff --git a/sys/dev/usb/uhci.c b/sys/dev/usb/uhci.c
index 47e6d1678c0..db05f3c2ccc 100644
--- a/sys/dev/usb/uhci.c
+++ b/sys/dev/usb/uhci.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: uhci.c,v 1.149 2020/02/22 14:01:34 jasper Exp $	*/
+/*	$OpenBSD: uhci.c,v 1.150 2020/03/19 14:18:38 patrick Exp $	*/
 /*	$NetBSD: uhci.c,v 1.172 2003/02/23 04:19:26 simonb Exp $	*/
 /*	$FreeBSD: src/sys/dev/usb/uhci.c,v 1.33 1999/11/17 22:33:41 n_hibma Exp $	*/
 
@@ -377,6 +377,7 @@ uhci_init(struct uhci_softc *sc)
 	UWRITE1(sc, UHCI_SOF, sc->sc_saved_sof);
 
 	/* Allocate and initialize real frame array. */
+	sc->sc_dma.flags |= USB_DMA_COHERENT;
 	err = usb_allocmem(&sc->sc_bus,
 		  UHCI_FRAMELIST_COUNT * sizeof(uhci_physaddr_t),
 		  UHCI_FRAMELIST_ALIGN, &sc->sc_dma);
@@ -1414,6 +1415,7 @@ uhci_alloc_std(struct uhci_softc *sc)
 	s = splusb();
 	if (sc->sc_freetds == NULL) {
 		DPRINTFN(2,("uhci_alloc_std: allocating chunk\n"));
+		dma.flags |= USB_DMA_COHERENT;
 		err = usb_allocmem(&sc->sc_bus, UHCI_STD_SIZE * UHCI_STD_CHUNK,
 			  UHCI_TD_ALIGN, &dma);
 		if (err)
@@ -1468,6 +1470,7 @@ uhci_alloc_sqh(struct uhci_softc *sc)
 	s = splusb();
 	if (sc->sc_freeqhs == NULL) {
 		DPRINTFN(2, ("uhci_alloc_sqh: allocating chunk\n"));
+		dma.flags |= USB_DMA_COHERENT;
 		err = usb_allocmem(&sc->sc_bus, UHCI_SQH_SIZE * UHCI_SQH_CHUNK,
 			  UHCI_QH_ALIGN, &dma);
 		if (err)
@@ -2650,6 +2653,7 @@ uhci_open(struct usbd_pipe *pipe)
 				uhci_free_std(sc, upipe->u.ctl.setup);
 				goto bad;
 			}
+			upipe->u.ctl.reqdma.flags |= USB_DMA_COHERENT;
 			err = usb_allocmem(&sc->sc_bus,
 				  sizeof(usb_device_request_t),
 				  0, &upipe->u.ctl.reqdma);
diff --git a/sys/dev/usb/usb_mem.c b/sys/dev/usb/usb_mem.c
index c65906b43f4..d1e51e868ad 100644
--- a/sys/dev/usb/usb_mem.c
+++ b/sys/dev/usb/usb_mem.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: usb_mem.c,v 1.32 2018/12/05 17:41:23 gerhard Exp $ */
+/*	$OpenBSD: usb_mem.c,v 1.33 2020/03/19 14:18:38 patrick Exp $ */
 /*	$NetBSD: usb_mem.c,v 1.26 2003/02/01 06:23:40 thorpej Exp $	*/
 
 /*
@@ -72,7 +72,7 @@ struct usb_frag_dma {
 };
 
 usbd_status	usb_block_allocmem(bus_dma_tag_t, size_t, size_t,
-		    struct usb_dma_block **);
+		    struct usb_dma_block **, int);
 void		usb_block_freemem(struct usb_dma_block *);
 
 LIST_HEAD(, usb_dma_block) usb_blk_freelist =
@@ -84,7 +84,7 @@ LIST_HEAD(, usb_frag_dma) usb_frag_freelist =
 
 usbd_status
 usb_block_allocmem(bus_dma_tag_t tag, size_t size, size_t align,
-    struct usb_dma_block **dmap)
+    struct usb_dma_block **dmap, int coherent)
 {
 	int error;
         struct usb_dma_block *p;
@@ -96,7 +96,8 @@ usb_block_allocmem(bus_dma_tag_t tag, size_t size, size_t align,
 	s = splusb();
 	/* First check the free list. */
 	for (p = LIST_FIRST(&usb_blk_freelist); p; p = LIST_NEXT(p, next)) {
-		if (p->tag == tag && p->size >= size && p->align >= align) {
+		if (p->tag == tag && p->size >= size && p->align >= align &&
+		    p->coherent == coherent) {
 			LIST_REMOVE(p, next);
 			usb_blk_nfree--;
 			splx(s);
@@ -116,6 +117,7 @@ usb_block_allocmem(bus_dma_tag_t tag, size_t size, size_t align,
 	p->tag = tag;
 	p->size = size;
 	p->align = align;
+	p->coherent = coherent;
 	error = bus_dmamem_alloc(tag, p->size, align, 0,
 				 p->segs, nitems(p->segs),
 				 &p->nsegs, BUS_DMA_NOWAIT);
@@ -123,7 +125,8 @@ usb_block_allocmem(bus_dma_tag_t tag, size_t size, size_t align,
 		goto free0;
 
 	error = bus_dmamem_map(tag, p->segs, p->nsegs, p->size,
-			       &p->kaddr, BUS_DMA_NOWAIT|BUS_DMA_COHERENT);
+			       &p->kaddr, BUS_DMA_NOWAIT | (coherent ?
+			       BUS_DMA_COHERENT : 0));
 	if (error)
 		goto free1;
 
@@ -187,14 +190,18 @@ usb_allocmem(struct usbd_bus *bus, size_t size, size_t align, struct usb_dma *p)
 	usbd_status err;
 	struct usb_frag_dma *f;
 	struct usb_dma_block *b;
+	int coherent;
 	int i;
 	int s;
 
+	coherent = !!(p->flags & USB_DMA_COHERENT);
+
 	/* If the request is large then just use a full block. */
 	if (size > USB_MEM_SMALL || align > USB_MEM_SMALL) {
 		DPRINTFN(1, ("%s: large alloc %d\n", __func__, (int)size));
 		size = (size + USB_MEM_BLOCK - 1) & ~(USB_MEM_BLOCK - 1);
-		err = usb_block_allocmem(tag, size, align, &p->block);
+		err = usb_block_allocmem(tag, size, align, &p->block,
+		    coherent);
 		if (!err) {
 			p->block->frags = NULL;
 			p->offs = 0;
@@ -205,11 +212,12 @@ usb_allocmem(struct usbd_bus *bus, size_t size, size_t align, struct usb_dma *p)
 	s = splusb();
 	/* Check for free fragments. */
 	for (f = LIST_FIRST(&usb_frag_freelist); f; f = LIST_NEXT(f, next))
-		if (f->block->tag == tag)
+		if (f->block->tag == tag && f->block->coherent == coherent)
 			break;
 	if (f == NULL) {
 		DPRINTFN(1, ("usb_allocmem: adding fragments\n"));
-		err = usb_block_allocmem(tag, USB_MEM_BLOCK, USB_MEM_SMALL,&b);
+		err = usb_block_allocmem(tag, USB_MEM_BLOCK, USB_MEM_SMALL, &b,
+		    coherent);
 		if (err) {
 			splx(s);
 			return (err);
diff --git a/sys/dev/usb/usb_mem.h b/sys/dev/usb/usb_mem.h
index 1ae933f4bd9..f0e0d1c217d 100644
--- a/sys/dev/usb/usb_mem.h
+++ b/sys/dev/usb/usb_mem.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: usb_mem.h,v 1.15 2016/11/30 10:19:18 mpi Exp $ */
+/*	$OpenBSD: usb_mem.h,v 1.16 2020/03/19 14:18:38 patrick Exp $ */
 /*	$NetBSD: usb_mem.h,v 1.20 2003/05/03 18:11:42 wiz Exp $	*/
 /*	$FreeBSD: src/sys/dev/usb/usb_mem.h,v 1.9 1999/11/17 22:33:47 n_hibma Exp $	*/
 
@@ -40,6 +40,7 @@ struct usb_dma_block {
         caddr_t kaddr;
         bus_dma_segment_t segs[1];
         int nsegs;
+	int coherent;
         size_t size;
         size_t align;
 	struct usb_frag_dma *frags;
diff --git a/sys/dev/usb/usbdivar.h b/sys/dev/usb/usbdivar.h
index 6600c402979..448aab4dcb8 100644
--- a/sys/dev/usb/usbdivar.h
+++ b/sys/dev/usb/usbdivar.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: usbdivar.h,v 1.79 2018/11/27 14:56:09 mpi Exp $ */
+/*	$OpenBSD: usbdivar.h,v 1.80 2020/03/19 14:18:38 patrick Exp $ */
 /*	$NetBSD: usbdivar.h,v 1.70 2002/07/11 21:14:36 augustss Exp $	*/
 /*	$FreeBSD: src/sys/dev/usb/usbdivar.h,v 1.11 1999/11/17 22:33:51 n_hibma Exp $	*/
 
@@ -47,6 +47,8 @@ struct usb_dma_block;
 struct usb_dma {
 	struct usb_dma_block	*block;
 	u_int			 offs;
+	int			 flags;
+#define USB_DMA_COHERENT		(1 << 0)
 };
 
 struct usbd_xfer;