Reduce the per-packet allocation costs for crypto operations (cryptop)

by pre-allocating two cryptodesc objects and storing them in an array
instead of a linked list.  If more than two cryptodesc objects are
required use mallocarray to fetch them.  Adapt the drivers to the new
API.

This change results in one pool-get per ESP packet instead of three.
It also simplifies softraid crypto where more cryptodesc objects are
allocated than used.

From, with and ok markus@, ok bluhm@
"looks sane" mpi@

3 files changed, 31 insertions, 32 deletions

diff --git a/sys/crypto/crypto.c b/sys/crypto/crypto.c
index 650eda28826..2ee1cb6d887 100644
--- a/sys/crypto/crypto.c
+++ b/sys/crypto/crypto.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: crypto.c,v 1.78 2016/09/19 18:09:40 tedu Exp $	*/
+/*	$OpenBSD: crypto.c,v 1.79 2017/02/07 17:25:46 patrick Exp $	*/
 /*
  * The author of this code is Angelos D. Keromytis (angelos@cis.upenn.edu)
  *
@@ -400,18 +400,17 @@ crypto_dispatch(struct cryptop *crp)
 int
 crypto_invoke(struct cryptop *crp)
 {
-	struct cryptodesc *crd;
 	u_int64_t nid;
 	u_int32_t hid;
 	int error;
-	int s;
+	int s, i;
 
 	/* Sanity checks. */
 	if (crp == NULL || crp->crp_callback == NULL)
 		return EINVAL;
 
 	s = splvm();
-	if (crp->crp_desc == NULL || crypto_drivers == NULL) {
+	if (crp->crp_ndesc < 1 || crypto_drivers == NULL) {
 		crp->crp_etype = EINVAL;
 		crypto_done(crp);
 		splx(s);
@@ -449,8 +448,9 @@ crypto_invoke(struct cryptop *crp)
 
  migrate:
 	/* Migrate session. */
-	for (crd = crp->crp_desc; crd->crd_next; crd = crd->crd_next)
-		crd->CRD_INI.cri_next = &(crd->crd_next->CRD_INI);
+	for (i = 0; i < crp->crp_ndesc - 1; i++)
+		crp->crp_desc[i].CRD_INI.cri_next = &crp->crp_desc[i+1].CRD_INI;
+	crp->crp_desc[crp->crp_ndesc].CRD_INI.cri_next = NULL;
 
 	if (crypto_newsession(&nid, &(crp->crp_desc->CRD_INI), 0) == 0)
 		crp->crp_sid = nid;
@@ -467,16 +467,12 @@ crypto_invoke(struct cryptop *crp)
 void
 crypto_freereq(struct cryptop *crp)
 {
-	struct cryptodesc *crd;
-
 	if (crp == NULL)
 		return;
 
-	while ((crd = crp->crp_desc) != NULL) {
-		crp->crp_desc = crd->crd_next;
-		pool_put(&cryptodesc_pool, crd);
-	}
-
+	if (crp->crp_ndescalloc > 2)
+		free(crp->crp_desc, M_CRYPTO_DATA,
+		    crp->crp_ndescalloc * sizeof(struct cryptodesc));
 	pool_put(&cryptop_pool, crp);
 }
 
@@ -486,22 +482,22 @@ crypto_freereq(struct cryptop *crp)
 struct cryptop *
 crypto_getreq(int num)
 {
-	struct cryptodesc *crd;
 	struct cryptop *crp;
-	
+
 	crp = pool_get(&cryptop_pool, PR_NOWAIT | PR_ZERO);
 	if (crp == NULL)
 		return NULL;
 
-	while (num--) {
-		crd = pool_get(&cryptodesc_pool, PR_NOWAIT | PR_ZERO);
-		if (crd == NULL) {
-			crypto_freereq(crp);
+	crp->crp_desc = crp->crp_sdesc;
+	crp->crp_ndescalloc = crp->crp_ndesc = num;
+
+	if (num > 2) {
+		crp->crp_desc = mallocarray(num, sizeof(struct cryptodesc),
+		    M_CRYPTO_DATA, M_NOWAIT | M_ZERO);
+		if (crp->crp_desc == NULL) {
+			pool_put(&cryptop_pool, crp);
 			return NULL;
 		}
-
-		crd->crd_next = crp->crp_desc;
-		crp->crp_desc = crd;
 	}
 
 	return crp;
@@ -515,8 +511,6 @@ crypto_init(void)
 
 	pool_init(&cryptop_pool, sizeof(struct cryptop), 0, IPL_VM, 0,
 	    "cryptop", NULL);
-	pool_init(&cryptodesc_pool, sizeof(struct cryptodesc), 0, IPL_VM, 0,
-	    "cryptodesc", NULL);
 }
 
 /*
diff --git a/sys/crypto/cryptodev.h b/sys/crypto/cryptodev.h
index bd915817ebd..c64f74ff663 100644
--- a/sys/crypto/cryptodev.h
+++ b/sys/crypto/cryptodev.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: cryptodev.h,v 1.69 2017/02/07 15:10:48 bluhm Exp $	*/
+/*	$OpenBSD: cryptodev.h,v 1.70 2017/02/07 17:25:46 patrick Exp $	*/
 
 /*
  * The author of this code is Angelos D. Keromytis (angelos@cis.upenn.edu)
@@ -147,8 +147,6 @@ struct cryptodesc {
 #define crd_rnd		CRD_INI.cri_rnd
 #define crd_alg		CRD_INI.cri_alg
 #define crd_klen	CRD_INI.cri_klen
-
-	struct cryptodesc *crd_next;
 };
 
 /* Structure describing complete operation */
@@ -179,7 +177,11 @@ struct cryptop {
 
 	void 		*crp_buf;	/* Data to be processed */
 	void 		*crp_opaque;	/* Opaque pointer, passed along */
-	struct cryptodesc *crp_desc;	/* Linked list of processing descriptors */
+
+	struct cryptodesc *crp_desc;	/* List of processing descriptors */
+	struct cryptodesc crp_sdesc[2];	/* Static array for small ops */
+	int		 crp_ndesc;	/* Amount of descriptors to use */
+	int		 crp_ndescalloc;/* Amount of descriptors allocated */
 
 	void (*crp_callback)(struct cryptop *); /* Callback function */
 
diff --git a/sys/crypto/cryptosoft.c b/sys/crypto/cryptosoft.c
index 06792eb0a9a..315210c5407 100644
--- a/sys/crypto/cryptosoft.c
+++ b/sys/crypto/cryptosoft.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: cryptosoft.c,v 1.81 2016/09/02 09:12:49 tom Exp $	*/
+/*	$OpenBSD: cryptosoft.c,v 1.82 2017/02/07 17:25:46 patrick Exp $	*/
 
 /*
  * The author of this code is Angelos D. Keromytis (angelos@cis.upenn.edu)
@@ -495,7 +495,8 @@ swcr_authenc(struct cryptop *crp)
 
 	ivlen = blksz = iskip = oskip = 0;
 
-	for (crd = crp->crp_desc; crd; crd = crd->crd_next) {
+	for (i = 0; i < crp->crp_ndesc; i++) {
+		crd = &crp->crp_desc[i];
 		for (sw = swcr_sessions[crp->crp_sid & 0xffffffff];
 		     sw && sw->sw_alg != crd->crd_alg;
 		     sw = sw->sw_next)
@@ -1020,12 +1021,13 @@ swcr_process(struct cryptop *crp)
 	struct swcr_data *sw;
 	u_int32_t lid;
 	int type;
+	int i;
 
 	/* Sanity check */
 	if (crp == NULL)
 		return EINVAL;
 
-	if (crp->crp_desc == NULL || crp->crp_buf == NULL) {
+	if (crp->crp_ndesc < 1 || crp->crp_buf == NULL) {
 		crp->crp_etype = EINVAL;
 		goto done;
 	}
@@ -1042,7 +1044,8 @@ swcr_process(struct cryptop *crp)
 		type = CRYPTO_BUF_IOV;
 
 	/* Go through crypto descriptors, processing as we go */
-	for (crd = crp->crp_desc; crd; crd = crd->crd_next) {
+	for (i = 0; i < crp->crp_ndesc; i++) {
+		crd = &crp->crp_desc[i];
 		/*
 		 * Find the crypto context.
 		 *