Reduce the per-packet allocation costs for crypto operations (cryptop)

by pre-allocating two cryptodesc objects and storing them in an array
instead of a linked list.  If more than two cryptodesc objects are
required use mallocarray to fetch them.  Adapt the drivers to the new
API.

This change results in one pool-get per ESP packet instead of three.
It also simplifies softraid crypto where more cryptodesc objects are
allocated than used.

From, with and ok markus@, ok bluhm@
"looks sane" mpi@

4 files changed, 35 insertions, 34 deletions

diff --git a/sys/dev/pci/hifn7751.c b/sys/dev/pci/hifn7751.c
index 3814bdcf545..6f55e6783e5 100644
--- a/sys/dev/pci/hifn7751.c
+++ b/sys/dev/pci/hifn7751.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: hifn7751.c,v 1.176 2017/01/19 10:20:29 jsg Exp $	*/
+/*	$OpenBSD: hifn7751.c,v 1.177 2017/02/07 17:25:46 patrick Exp $	*/
 
 /*
  * Invertex AEON / Hifn 7751 driver
@@ -1927,7 +1927,7 @@ hifn_process(struct cryptop *crp)
 	struct hifn_command *cmd = NULL;
 	int card, session, err = 0, ivlen;
 	struct hifn_softc *sc;
-	struct cryptodesc *crd1, *crd2, *maccrd, *enccrd;
+	struct cryptodesc *crd1, *crd2 = NULL, *maccrd, *enccrd;
 
 	if (crp == NULL || crp->crp_callback == NULL) {
 		hifnstats.hst_invalid++;
@@ -1969,12 +1969,13 @@ hifn_process(struct cryptop *crp)
 		goto errout;	/* XXX we don't handle contiguous buffers! */
 	}
 
-	crd1 = crp->crp_desc;
-	if (crd1 == NULL) {
+	if (crp->crp_ndesc < 1) {
 		err = EINVAL;
 		goto errout;
 	}
-	crd2 = crd1->crd_next;
+	crd1 = &crp->crp_desc[0];
+	if (crp->crp_ndesc >= 2)
+		crd2 = &crp->crp_desc[1];
 
 	if (crd2 == NULL) {
 		if (crd1->crd_alg == CRYPTO_MD5_HMAC ||
@@ -2291,9 +2292,11 @@ hifn_callback(struct hifn_softc *sc, struct hifn_command *cmd,
 			macbuf += sizeof(struct hifn_comp_result);
 		macbuf += sizeof(struct hifn_mac_result);
 
-		for (crd = crp->crp_desc; crd; crd = crd->crd_next) {
+		for (i = 0; i < crp->crp_ndesc; i++) {
 			int len;
 
+			crd = &crp->crp_desc[i];
+
 			if (crd->crd_alg == CRYPTO_MD5_HMAC ||
 			    crd->crd_alg == CRYPTO_SHA1_HMAC)
 				len = 12;
@@ -2326,7 +2329,7 @@ int
 hifn_compression(struct hifn_softc *sc, struct cryptop *crp,
     struct hifn_command *cmd)
 {
-	struct cryptodesc *crd = crp->crp_desc;
+	struct cryptodesc *crd = &crp->crp_desc[0];
 	int s, err = 0;
 
 	cmd->compcrd = crd;
diff --git a/sys/dev/pci/safe.c b/sys/dev/pci/safe.c
index 44bf9f0edc9..0a4c0de75a5 100644
--- a/sys/dev/pci/safe.c
+++ b/sys/dev/pci/safe.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: safe.c,v 1.41 2015/12/10 21:00:51 naddy Exp $	*/
+/*	$OpenBSD: safe.c,v 1.42 2017/02/07 17:25:46 patrick Exp $	*/
 
 /*-
  * Copyright (c) 2003 Sam Leffler, Errno Consulting
@@ -301,7 +301,7 @@ safe_process(struct cryptop *crp)
 {
 	int err = 0, i, nicealign, uniform, s;
 	struct safe_softc *sc;
-	struct cryptodesc *crd1, *crd2, *maccrd, *enccrd;
+	struct cryptodesc *crd1, *crd2 = NULL, *maccrd, *enccrd;
 	int bypass, oplen, ivsize, card;
 	int16_t coffset;
 	struct safe_session *ses;
@@ -360,13 +360,14 @@ safe_process(struct cryptop *crp)
 	sa = &re->re_sa;
 	ses = &sc->sc_sessions[re->re_sesn];
 
-	crd1 = crp->crp_desc;
-	if (crd1 == NULL) {
+	if (crp->crp_ndesc < 1) {
 		safestats.st_nodesc++;
 		err = EINVAL;
 		goto errout;
 	}
-	crd2 = crd1->crd_next;
+	crd1 = &crp->crp_desc[0];
+	if (crp->crp_ndesc >= 2)
+		crd2 = &crp->crp_desc[1];
 
 	cmd0 = SAFE_SA_CMD0_BASIC;		/* basic group operation */
 	cmd1 = 0;
@@ -1608,6 +1609,7 @@ safe_callback(struct safe_softc *sc, struct safe_ringentry *re)
 {
 	struct cryptop *crp = (struct cryptop *)re->re_crp;
 	struct cryptodesc *crd;
+	int i;
 
 	safestats.st_opackets++;
 	safestats.st_obytes += (re->re_dst_map == NULL) ?
@@ -1644,7 +1646,8 @@ safe_callback(struct safe_softc *sc, struct safe_ringentry *re)
 
 	if (re->re_flags & SAFE_QFLAGS_COPYOUTICV) {
 		/* copy out ICV result */
-		for (crd = crp->crp_desc; crd; crd = crd->crd_next) {
+		for (i = 0; i < crp->crp_ndesc; i++) {
+			crd = &crp->crp_desc[i];
 			if (!(crd->crd_alg == CRYPTO_MD5_HMAC ||
 			    crd->crd_alg == CRYPTO_SHA1_HMAC))
 				continue;
diff --git a/sys/dev/pci/ubsec.c b/sys/dev/pci/ubsec.c
index cbefa8576ca..70517670e7b 100644
--- a/sys/dev/pci/ubsec.c
+++ b/sys/dev/pci/ubsec.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: ubsec.c,v 1.162 2016/05/30 23:38:21 dlg Exp $	*/
+/*	$OpenBSD: ubsec.c,v 1.163 2017/02/07 17:25:46 patrick Exp $	*/
 
 /*
  * Copyright (c) 2000 Jason L. Wright (jason@thought.net)
@@ -776,7 +776,7 @@ ubsec_process(struct cryptop *crp)
 	struct ubsec_q *q = NULL;
 	int card, err = 0, i, j, s, nicealign;
 	struct ubsec_softc *sc;
-	struct cryptodesc *crd1, *crd2, *maccrd, *enccrd;
+	struct cryptodesc *crd1, *crd2 = NULL, *maccrd, *enccrd;
 	int encoffset = 0, macoffset = 0, cpskip, cpoffset;
 	int sskip, dskip, stheend, dtheend;
 	int16_t coffset;
@@ -835,12 +835,13 @@ ubsec_process(struct cryptop *crp)
 	dmap->d_dma->d_mcr.mcr_flags = 0;
 	q->q_crp = crp;
 
-	crd1 = crp->crp_desc;
-	if (crd1 == NULL) {
+	if (crp->crp_ndesc < 1) {
 		err = EINVAL;
 		goto errout;
 	}
-	crd2 = crd1->crd_next;
+	crd1 = &crp->crp_desc[0];
+	if (crp->crp_ndesc >= 2)
+		crd2 = &crp->crp_desc[1];
 
 	if (crd2 == NULL) {
 		if (crd1->crd_alg == CRYPTO_MD5_HMAC ||
@@ -1330,6 +1331,7 @@ ubsec_callback(struct ubsec_softc *sc, struct ubsec_q *q)
 	u_int8_t *ctx = (u_int8_t *)(dmap->d_alloc.dma_vaddr +
 		    offsetof(struct ubsec_dmachunk, d_ctx));
 	struct ubsec_pktctx_hdr *ph = (struct ubsec_pktctx_hdr *)ctx;
+	int i;
 
 	ubsecstats.hst_opackets++;
 	ubsecstats.hst_obytes += dmap->d_alloc.dma_size;
@@ -1355,7 +1357,8 @@ ubsec_callback(struct ubsec_softc *sc, struct ubsec_q *q)
 		crp->crp_buf = (caddr_t)q->q_dst_m;
 	}
 
-	for (crd = crp->crp_desc; crd; crd = crd->crd_next) {
+	for (i = 0; i < crp->crp_ndesc; i++) {
+		crd = &crp->crp_desc[i];
 		if (crd->crd_alg != CRYPTO_MD5_HMAC &&
 		    crd->crd_alg != CRYPTO_SHA1_HMAC)
 			continue;
diff --git a/sys/dev/softraid_crypto.c b/sys/dev/softraid_crypto.c
index 2dc4c6b2328..c419686997e 100644
--- a/sys/dev/softraid_crypto.c
+++ b/sys/dev/softraid_crypto.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: softraid_crypto.c,v 1.132 2017/02/07 15:10:48 bluhm Exp $ */
+/* $OpenBSD: softraid_crypto.c,v 1.133 2017/02/07 17:25:46 patrick Exp $ */
 /*
  * Copyright (c) 2007 Marco Peereboom <marco@peereboom.us>
  * Copyright (c) 2008 Hans-Joerg Hoexer <hshoexer@openbsd.org>
@@ -64,7 +64,6 @@ struct sr_crypto_wu {
 	struct uio			 cr_uio;
 	struct iovec			 cr_iov;
 	struct cryptop	 		*cr_crp;
-	struct cryptodesc		*cr_descs;
 	void				*cr_dmabuf;
 };
 
@@ -270,15 +269,11 @@ sr_crypto_prepare(struct sr_workunit *wu, int encrypt)
 
 	/*
 	 * We preallocated enough crypto descs for up to MAXPHYS of I/O.
-	 * Since there may be less than that we need to tweak the linked list
+	 * Since there may be less than that we need to tweak the amount
 	 * of crypto desc structures to be just long enough for our needs.
 	 */
-	crd = crwu->cr_descs;
-	for (i = 0; i < ((MAXPHYS >> DEV_BSHIFT) - n); i++) {
-		crd = crd->crd_next;
-		KASSERT(crd);
-	}
-	crwu->cr_crp->crp_desc = crd;
+	KASSERT(crwu->cr_crp->crp_ndescalloc >= n);
+	crwu->cr_crp->crp_ndesc = n;
 	flags = (encrypt ? CRD_F_ENCRYPT : 0) |
 	    CRD_F_IV_PRESENT | CRD_F_IV_EXPLICIT;
 
@@ -297,8 +292,8 @@ sr_crypto_prepare(struct sr_workunit *wu, int encrypt)
 	crwu->cr_crp->crp_alloctype = M_DEVBUF;
 	crwu->cr_crp->crp_flags = CRYPTO_F_IOV | CRYPTO_F_NOQUEUE;
 	crwu->cr_crp->crp_buf = &crwu->cr_uio;
-	for (i = 0, crd = crwu->cr_crp->crp_desc; crd;
-	    i++, blkno++, crd = crd->crd_next) {
+	for (i = 0; i < crwu->cr_crp->crp_ndesc; i++, blkno++) {
+		crd = &crwu->cr_crp->crp_desc[i];
 		crd->crd_skip = i << DEV_BSHIFT;
 		crd->crd_len = DEV_BSIZE;
 		crd->crd_inject = 0;
@@ -946,7 +941,6 @@ sr_crypto_alloc_resources(struct sr_discipline *sd)
 		crwu->cr_crp = crypto_getreq(MAXPHYS >> DEV_BSHIFT);
 		if (crwu->cr_crp == NULL)
 			return (ENOMEM);
-		crwu->cr_descs = crwu->cr_crp->crp_desc;
 	}
 
 	memset(&cri, 0, sizeof(cri));
@@ -1005,10 +999,8 @@ sr_crypto_free_resources(struct sr_discipline *sd)
 		crwu = (struct sr_crypto_wu *)wu;
 		if (crwu->cr_dmabuf)
 			dma_free(crwu->cr_dmabuf, MAXPHYS);
-		if (crwu->cr_crp) {
-			crwu->cr_crp->crp_desc = crwu->cr_descs;
+		if (crwu->cr_crp)
 			crypto_freereq(crwu->cr_crp);
-		}
 	}
 
 	sr_wu_free(sd);