Fastpath for userland crypto requests. This change makes userland

crypto requests attempt to call the crypto driver directly to process
crypto layer requests, as opposed to queueing them in the kernel
crypto thread. If we can't use the crypto devices (i.e. they're busy)
we fall back to queueing the request up in the crypto thread as
before. This does allow for faster performance in some cases (smaller
requests, how small seems to be dependent on the card/cpu combination)
where context switching is a major issue in performance.
ok deraadt@ jason@

3 files changed, 62 insertions, 22 deletions

diff --git a/sys/crypto/crypto.c b/sys/crypto/crypto.c
index 427f2444f38..8087e86bcfe 100644
--- a/sys/crypto/crypto.c
+++ b/sys/crypto/crypto.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: crypto.c,v 1.43 2003/02/19 03:41:31 jason Exp $	*/
+/*	$OpenBSD: crypto.c,v 1.44 2003/06/03 15:28:06 beck Exp $	*/
 /*
  * The author of this code is Angelos D. Keromytis (angelos@cis.upenn.edu)
  *
@@ -686,18 +686,28 @@ crypto_thread(void)
 void
 crypto_done(struct cryptop *crp)
 {
-	int s = splimp();
+	int s;
 
-	crp->crp_next = NULL;
-	if (crp_ret_queue == NULL) {
-		crp_ret_queue = crp;
-		crp_ret_queue_tail = &(crp->crp_next);
-		splx(s);
-		wakeup((caddr_t) &crp_req_queue); /* Shared wait channel. */
+	if (crp->crp_flags & CRYPTO_F_NOQUEUE) {
+		/* not from the crypto queue, wakeup the userland
+		 * process 
+		 */
+		crp->crp_flags |= CRYPTO_F_DONE;
+		crp->crp_callback(crp);
 	} else {
-		*crp_ret_queue_tail = crp;
-		crp_ret_queue_tail = &(crp->crp_next);
-		splx(s);
+		s = splimp();
+		crp->crp_flags |= CRYPTO_F_DONE;
+		crp->crp_next = NULL;
+		if (crp_ret_queue == NULL) {
+			crp_ret_queue = crp;
+			crp_ret_queue_tail = &(crp->crp_next);
+			splx(s);
+			wakeup((caddr_t) &crp_req_queue); /* Shared wait channel. */
+		} else {
+			*crp_ret_queue_tail = crp;
+			crp_ret_queue_tail = &(crp->crp_next);
+			splx(s);
+		}
 	}
 }
 
diff --git a/sys/crypto/cryptodev.c b/sys/crypto/cryptodev.c
index 25b91cd5ba2..59830fd0a9f 100644
--- a/sys/crypto/cryptodev.c
+++ b/sys/crypto/cryptodev.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: cryptodev.c,v 1.56 2003/05/30 03:33:23 jason Exp $	*/
+/*	$OpenBSD: cryptodev.c,v 1.57 2003/06/03 15:28:06 beck Exp $	*/
 
 /*
  * Copyright (c) 2001 Theo de Raadt
@@ -52,6 +52,9 @@
 #include <crypto/cryptodev.h>
 #include <crypto/xform.h>
 
+extern struct cryptocap *crypto_drivers;
+extern int crypto_drivers_num;
+
 struct csession {
 	TAILQ_ENTRY(csession) next;
 	u_int64_t	sid;
@@ -297,9 +300,10 @@ bail:
 int
 cryptodev_op(struct csession *cse, struct crypt_op *cop, struct proc *p)
 {
-	struct cryptop *crp = NULL;
+	struct cryptop *crp= NULL;
 	struct cryptodesc *crde = NULL, *crda = NULL;
-	int i, error;
+	int i, s, error;
+	u_int32_t hid;
 
 	if (cop->len > 64*1024-4)
 		return (E2BIG);
@@ -367,7 +371,6 @@ cryptodev_op(struct csession *cse, struct crypt_op *cop, struct proc *p)
 	}
 
 	crp->crp_ilen = cop->len;
-	crp->crp_flags = CRYPTO_F_IOV;
 	crp->crp_buf = (caddr_t)&cse->uio;
 	crp->crp_callback = (int (*) (struct cryptop *)) cryptodev_cb;
 	crp->crp_sid = cse->sid;
@@ -403,22 +406,45 @@ cryptodev_op(struct csession *cse, struct crypt_op *cop, struct proc *p)
 		crp->crp_mac=cse->tmp_mac;
 	}
 
+	/* try the fast path first */
+	crp->crp_flags = CRYPTO_F_IOV | CRYPTO_F_NOQUEUE;
+	hid = (crp->crp_sid >> 32) & 0xffffffff;
+	if (hid >= crypto_drivers_num)
+		goto dispatch;
+	if (crypto_drivers[hid].cc_flags & CRYPTOCAP_F_SOFTWARE)
+		goto dispatch;
+	if (crypto_drivers[hid].cc_process == NULL)
+		goto dispatch;
+	error = crypto_drivers[hid].cc_process(crp);
+	if (error) {
+		/* clear error */
+		crp->crp_etype = 0;
+		goto dispatch;
+	}
+	goto processed;
+ dispatch:
+	crp->crp_flags = CRYPTO_F_IOV;
 	crypto_dispatch(crp);
-	error = tsleep(cse, PSOCK, "crydev", 0);
+ processed:
+	s = splnet();
+	while (!(crp->crp_flags & CRYPTO_F_DONE)) {
+		error = tsleep(cse, PSOCK, "crydev", 0);
+	}
+	splx(s);
 	if (error) {
 		/* XXX can this happen?  if so, how do we recover? */
 		goto bail;
 	}
 
+	if (cse->error) {
+		error = cse->error;
+		goto bail;
+	}
 	if (crp->crp_etype != 0) {
 		error = crp->crp_etype;
 		goto bail;
 	}
 
-	if (cse->error) {
-		error = cse->error;
-		goto bail;
-	}
 
 	if (cop->dst &&
 	    (error = copyout(cse->uio.uio_iov[0].iov_base, cop->dst, cop->len)))
@@ -444,8 +470,10 @@ cryptodev_cb(void *op)
 	struct csession *cse = (struct csession *)crp->crp_opaque;
 
 	cse->error = crp->crp_etype;
-	if (crp->crp_etype == EAGAIN)
+	if (crp->crp_etype == EAGAIN) {
+		crp->crp_flags = CRYPTO_F_IOV;
 		return crypto_dispatch(crp);
+	}
 	wakeup(cse);
 	return (0);
 }
diff --git a/sys/crypto/cryptodev.h b/sys/crypto/cryptodev.h
index 8cc9ccc1690..edac1105b57 100644
--- a/sys/crypto/cryptodev.h
+++ b/sys/crypto/cryptodev.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: cryptodev.h,v 1.37 2003/02/15 22:57:58 jason Exp $	*/
+/*	$OpenBSD: cryptodev.h,v 1.38 2003/06/03 15:28:06 beck Exp $	*/
 
 /*
  * The author of this code is Angelos D. Keromytis (angelos@cis.upenn.edu)
@@ -160,6 +160,8 @@ struct cryptop {
 #define CRYPTO_F_IMBUF	0x0001	/* Input/output are mbuf chains, otherwise contig */
 #define CRYPTO_F_IOV	0x0002	/* Input/output are uio */
 #define CRYPTO_F_REL	0x0004	/* Must return data in same place */
+#define CRYPTO_F_NOQUEUE	0x0008	/* Don't use crypto queue/thread */
+#define CRYPTO_F_DONE	0x0010	/* request completed */
 
 	caddr_t		crp_buf;	/* Data to be processed */
 	caddr_t		crp_opaque;	/* Opaque pointer, passed along */