Ensure each nfsiod can actually enqueue more than one asynchio - this mirrors

the accidental situation that used to happen when it leaked buffers and allowed
the syncer to do it, however this puts a limit on how much of the buffer cache
it is allowed to consume to a sensible amount - improves nfs write performance
since we don't have to do tons of them synch now.

Modifies the existing code to use wakeup_one instead of cruft, and now
all nfsiod's tsleep the same way.

ok thib@ art@

3 files changed, 36 insertions, 26 deletions

diff --git a/sys/nfs/nfs_bio.c b/sys/nfs/nfs_bio.c
index e71beddf1a8..929152321e0 100644
--- a/sys/nfs/nfs_bio.c
+++ b/sys/nfs/nfs_bio.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: nfs_bio.c,v 1.50 2008/06/12 19:14:15 thib Exp $	*/
+/*	$OpenBSD: nfs_bio.c,v 1.51 2008/06/14 19:33:58 beck Exp $	*/
 /*	$NetBSD: nfs_bio.c,v 1.25.4.2 1996/07/08 20:47:04 jtc Exp $	*/
 
 /*
@@ -60,6 +60,7 @@ extern struct proc *nfs_iodwant[NFS_MAXASYNCDAEMON];
 extern int nfs_numasync;
 extern struct nfsstats nfsstats;
 struct nfs_bufqhead nfs_bufq;
+uint32_t nfs_bufqmax, nfs_bufqlen;
 
 /*
  * Vnode op for read using bio
@@ -545,24 +546,22 @@ int
 nfs_asyncio(bp)
 	struct buf *bp;
 {
-	int i;
-
 	if (nfs_numasync == 0)
 		goto out;
 
-	for (i = 0; i < NFS_MAXASYNCDAEMON; i++) {
-	    if (nfs_iodwant[i]) {
-		if ((bp->b_flags & B_READ) == 0) {
-			bp->b_flags |= B_WRITEINPROG;
-		}
-	
-		TAILQ_INSERT_TAIL(&nfs_bufq, bp, b_freelist);
-		nfs_iodwant[i] = (struct proc *)0;
-		wakeup((caddr_t)&nfs_iodwant[i]);
-		return (0);
-	    }
+	if (nfs_bufqlen > nfs_bufqmax)
+		goto out; /* too many bufs in use, force sync */
+
+	if ((bp->b_flags & B_READ) == 0) {
+		bp->b_flags |= B_WRITEINPROG;
 	}
 
+	TAILQ_INSERT_TAIL(&nfs_bufq, bp, b_freelist);
+	nfs_bufqlen++;
+
+	wakeup_one(&nfs_bufq);
+	return (0);
+
 out:
 	nfsstats.forcedsync++;
 	return (EIO);
diff --git a/sys/nfs/nfs_syscalls.c b/sys/nfs/nfs_syscalls.c
index 5269b611f16..c1367013e58 100644
--- a/sys/nfs/nfs_syscalls.c
+++ b/sys/nfs/nfs_syscalls.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: nfs_syscalls.c,v 1.65 2008/06/13 22:19:35 blambert Exp $	*/
+/*	$OpenBSD: nfs_syscalls.c,v 1.66 2008/06/14 19:33:58 beck Exp $	*/
 /*	$NetBSD: nfs_syscalls.c,v 1.19 1996/02/18 11:53:52 fvdl Exp $	*/
 
 /*
@@ -678,33 +678,42 @@ nfssvc_iod(p)
 	struct buf *bp, *nbp;
 	int i, myiod;
 	struct vnode *vp;
-	int error = 0, s;
+	int error = 0, s, bufcount;
 
-	/*
-	 * Assign my position or return error if too many already running
-	 */
+	bufcount = 256;	/* XXX: Big enough? sysctl, constant ? */
+
+	/* Assign my position or return error if too many already running. */
 	myiod = -1;
-	for (i = 0; i < NFS_MAXASYNCDAEMON; i++)
+	for (i = 0; i < NFS_MAXASYNCDAEMON; i++) {
 		if (nfs_asyncdaemon[i] == NULL) {
 			myiod = i;
 			break;
 		}
+	}
 	if (myiod == -1)
 		return (EBUSY);
+
 	nfs_asyncdaemon[myiod] = p;
 	nfs_numasync++;
-	/*
-	 * Just loop around doin our stuff until SIGKILL
-	 */
+
+	/* Upper limit on how many bufs we'll queue up for this iod. */
+	if (nfs_bufqmax > bcstats.numbufs / 4) {
+		nfs_bufqmax = bcstats.numbufs / 4; /* limit to 1/4 of bufs */
+		bufcount = 0;
+	}
+
+	nfs_bufqmax += bufcount;
+
+	/* Just loop around doin our stuff until SIGKILL. */
 	for (;;) {
 	    while (TAILQ_FIRST(&nfs_bufq) == NULL && error == 0) {
-		nfs_iodwant[myiod] = p;
-		error = tsleep((caddr_t)&nfs_iodwant[myiod],
+		    error = tsleep(&nfs_bufq,
 			PWAIT | PCATCH, "nfsidl", 0);
 	    }
 	    while ((bp = TAILQ_FIRST(&nfs_bufq)) != NULL) {
 		/* Take one off the front of the list */
 		TAILQ_REMOVE(&nfs_bufq, bp, b_freelist);
+		nfs_bufqlen--;
 		if (bp->b_flags & B_READ)
 		    (void) nfs_doio(bp, NULL);
 		else do {
@@ -742,6 +751,7 @@ nfssvc_iod(p)
 	    if (error) {
 		nfs_asyncdaemon[myiod] = NULL;
 		nfs_numasync--;
+		nfs_bufqmax -= bufcount;
 		return (error);
 	    }
 	}
diff --git a/sys/nfs/nfsnode.h b/sys/nfs/nfsnode.h
index c20e0542ee1..61c58b9cea9 100644
--- a/sys/nfs/nfsnode.h
+++ b/sys/nfs/nfsnode.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: nfsnode.h,v 1.30 2008/06/10 22:59:09 thib Exp $	*/
+/*	$OpenBSD: nfsnode.h,v 1.31 2008/06/14 19:33:58 beck Exp $	*/
 /*	$NetBSD: nfsnode.h,v 1.16 1996/02/18 11:54:04 fvdl Exp $	*/
 
 /*
@@ -139,5 +139,6 @@ struct nfsnode {
  * Queue head for nfsiod's
  */
 extern TAILQ_HEAD(nfs_bufqhead, buf) nfs_bufq;
+extern uint32_t nfs_bufqlen, nfs_bufqmax;
 
 #endif		/* _NFS_NFSNODE_H_ */