Introduce bufq_quiesce(), which will block I/O ifrom getting on the queues,

and waits until all I/O currently on the queues has been completed.  To get
I/O going again, call bufq_restart().

To be used for suspend/resume.

Joint effort with thib@, tedu@; tested by mlarkin@, marco@

3 files changed, 125 insertions, 26 deletions

diff --git a/sys/kern/kern_bufq.c b/sys/kern/kern_bufq.c
index c0d0c3774bd..23f209abe25 100644
--- a/sys/kern/kern_bufq.c
+++ b/sys/kern/kern_bufq.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: kern_bufq.c,v 1.8 2010/06/27 22:05:28 thib Exp $	*/
+/*	$OpenBSD: kern_bufq.c,v 1.9 2010/06/29 18:52:20 kettenis Exp $	*/
 /*
  * Copyright (c) 2010 Thordur I. Bjornsson <thib@openbsd.org>
  *
@@ -26,15 +26,19 @@
 
 #include <sys/disklabel.h>
 
-struct buf *(*bufq_dequeue[BUFQ_HOWMANY])(struct bufq *, int) = {
+SLIST_HEAD(, bufq)	bufqs = SLIST_HEAD_INITIALIZER(&bufq);
+struct mutex		bufqs_mtx = MUTEX_INITIALIZER(IPL_BIO);
+int			bufqs_stop;
+
+struct buf *(*bufq_dequeuev[BUFQ_HOWMANY])(struct bufq *, int) = {
 	bufq_disksort_dequeue,
 	bufq_fifo_dequeue
 };
-void (*bufq_queue[BUFQ_HOWMANY])(struct bufq *, struct buf *) = {
+void (*bufq_queuev[BUFQ_HOWMANY])(struct bufq *, struct buf *) = {
 	bufq_disksort_queue,
 	bufq_fifo_queue
 };
-void (*bufq_requeue[BUFQ_HOWMANY])(struct bufq *, struct buf *) = {
+void (*bufq_requeuev[BUFQ_HOWMANY])(struct bufq *, struct buf *) = {
 	bufq_disksort_requeue,
 	bufq_fifo_requeue
 };
@@ -66,6 +70,10 @@ bufq_init(int type)
 
 	KASSERT(error == 0);
 
+	mtx_enter(&bufqs_mtx);
+	SLIST_INSERT_HEAD(&bufqs, bq, bufq_entries);
+	mtx_leave(&bufqs_mtx);
+
 	return (bq);
 }
 
@@ -77,10 +85,39 @@ bufq_destroy(struct bufq *bq)
 	if (bq->bufq_data != NULL)
 		free(bq->bufq_data, M_DEVBUF);
 
+	mtx_enter(&bufqs_mtx);
+	while (bufqs_stop) {
+		msleep(&bufqs_stop, &bufqs_mtx, PRIBIO, "bufqstop", 0);
+	}
+	SLIST_REMOVE(&bufqs, bq, bufq, bufq_entries);
+	mtx_leave(&bufqs_mtx);
+
 	free(bq, M_DEVBUF);
 }
 
 void
+bufq_queue(struct bufq *bq, struct buf *bp)
+{
+	mtx_enter(&bq->bufq_mtx);
+	while (bq->bufq_stop) {
+		msleep(&bq->bufq_stop, &bq->bufq_mtx, PRIBIO, "bufqstop", 0);
+	}
+	bufq_queuev[bq->bufq_type](bq, bp);
+	mtx_leave(&bq->bufq_mtx);
+}
+
+void
+bufq_requeue(struct bufq *bq, struct buf *bp)
+{
+	mtx_enter(&bq->bufq_mtx);
+	while (bq->bufq_stop) {
+		msleep(&bq->bufq_stop, &bq->bufq_mtx, PRIBIO, "bufqstop", 0);
+	}
+	bufq_requeuev[bq->bufq_type](bq, bp);
+	mtx_leave(&bq->bufq_mtx);
+}
+
+void
 bufq_drain(struct bufq *bq)
 {
 	struct buf	*bp;
@@ -96,15 +133,67 @@ bufq_drain(struct bufq *bq)
 }
 
 void
+bufq_done(struct bufq *bq, struct buf *bp)
+{
+	mtx_enter(&bq->bufq_mtx);
+	bq->bufq_outstanding--;
+	KASSERT(bq->bufq_outstanding >= 0);
+	if (bq->bufq_outstanding == 0)
+		wakeup(&bq->bufq_outstanding);
+	mtx_leave(&bq->bufq_mtx);
+	bp->b_bq = NULL;
+}
+
+void
+bufq_quiesce(void)
+{
+	struct bufq		*bq;
+
+restart:
+	mtx_enter(&bufqs_mtx);
+	bufqs_stop = 1;
+	SLIST_FOREACH(bq, &bufqs, bufq_entries) {
+		mtx_enter(&bq->bufq_mtx);
+		bq->bufq_stop = 1;
+		if (bq->bufq_outstanding) {
+			mtx_leave(&bufqs_mtx);
+			msleep(&bq->bufq_outstanding, &bq->bufq_mtx, PRIBIO,
+			    "bufqqui", 0);
+			mtx_leave(&bq->bufq_mtx);
+			goto restart;
+		}
+		mtx_leave(&bq->bufq_mtx);
+	}
+	mtx_leave(&bufqs_mtx);
+}
+
+void
+bufq_restart(void)
+{
+	struct bufq		*bq;
+
+	mtx_enter(&bufqs_mtx);
+	SLIST_FOREACH(bq, &bufqs, bufq_entries) {
+		mtx_enter(&bq->bufq_mtx);
+		bq->bufq_stop = 0;
+		wakeup(&bq->bufq_stop);
+		mtx_leave(&bq->bufq_mtx);
+	}
+	bufqs_stop = 0;
+	wakeup(&bufqs_stop);
+	mtx_leave(&bufqs_mtx);
+}
+
+void
 bufq_disksort_queue(struct bufq *bq, struct buf *bp)
 {
 	struct buf	*bufq;
 
 	bufq = (struct buf  *)bq->bufq_data;
 
-	mtx_enter(&bq->bufq_mtx);
+	bq->bufq_outstanding++;
+	bp->b_bq = bq;
 	disksort(bufq, bp);
-	mtx_leave(&bq->bufq_mtx);
 }
 
 void
@@ -114,12 +203,12 @@ bufq_disksort_requeue(struct bufq *bq, struct buf *bp)
 
 	bufq = (struct buf *)bq->bufq_data;
 
-	mtx_enter(&bq->bufq_mtx);
+	bq->bufq_outstanding++;
+	bp->b_bq = bq; 
 	bp->b_actf = bufq->b_actf;
 	bufq->b_actf = bp;
 	if (bp->b_actf == NULL)
 		bufq->b_actb = &bp->b_actf;
-	mtx_leave(&bq->bufq_mtx);
 }
 
 struct buf *
@@ -160,9 +249,9 @@ bufq_fifo_queue(struct bufq *bq, struct buf *bp)
 {
 	struct bufq_fifo_head	*head = bq->bufq_data;
 
-	mtx_enter(&bq->bufq_mtx);
+	bq->bufq_outstanding++;
+	bp->b_bq = bq;
 	TAILQ_INSERT_TAIL(head, bp, b_bufq.bufq_data_fifo.bqf_entries);
-	mtx_leave(&bq->bufq_mtx);
 }
 
 void
@@ -170,9 +259,9 @@ bufq_fifo_requeue(struct bufq *bq, struct buf *bp)
 {
 	struct bufq_fifo_head	*head = bq->bufq_data;
 
-	mtx_enter(&bq->bufq_mtx);
+	bq->bufq_outstanding++;
+	bp->b_bq = bq;;
 	TAILQ_INSERT_HEAD(head, bp, b_bufq.bufq_data_fifo.bqf_entries);
-	mtx_leave(&bq->bufq_mtx);
 }
 
 struct buf *
diff --git a/sys/kern/vfs_bio.c b/sys/kern/vfs_bio.c
index 93c404bb457..b242d58978c 100644
--- a/sys/kern/vfs_bio.c
+++ b/sys/kern/vfs_bio.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: vfs_bio.c,v 1.121 2010/02/05 12:24:32 jsing Exp $	*/
+/*	$OpenBSD: vfs_bio.c,v 1.122 2010/06/29 18:52:20 kettenis Exp $	*/
 /*	$NetBSD: vfs_bio.c,v 1.44 1996/06/11 11:15:36 pk Exp $	*/
 
 /*
@@ -1228,6 +1228,9 @@ biodone(struct buf *bp)
 			wakeup(bp);
 		}
 	}
+
+	if (bp->b_bq)
+		bufq_done(bp->b_bq, bp);
 }
 
 #ifdef DDB
diff --git a/sys/sys/buf.h b/sys/sys/buf.h
index 901933befca..b2b14244cec 100644
--- a/sys/sys/buf.h
+++ b/sys/sys/buf.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: buf.h,v 1.68 2010/05/26 16:38:20 thib Exp $	*/
+/*	$OpenBSD: buf.h,v 1.69 2010/06/29 18:52:20 kettenis Exp $	*/
 /*	$NetBSD: buf.h,v 1.25 1997/04/09 21:12:17 mycroft Exp $	*/
 
 /*
@@ -68,9 +68,12 @@ LIST_HEAD(workhead, worklist);
 #define BUFQ_HOWMANY	2
 
 struct bufq {
-	struct mutex	   bufq_mtx;
-	void		  *bufq_data;
-	int		   bufq_type;
+	SLIST_ENTRY(bufq)	 bufq_entries;
+	struct mutex	 	 bufq_mtx;
+	void			*bufq_data;
+	u_int			 bufq_outstanding;
+	int			 bufq_stop;
+	int			 bufq_type;
 };
 
 struct buf	*bufq_disksort_dequeue(struct bufq *, int);
@@ -96,22 +99,25 @@ union bufq_data {
 	struct bufq_fifo	bufq_data_fifo;
 };
 
-extern struct buf *(*bufq_dequeue[BUFQ_HOWMANY])(struct bufq *, int);
-extern void (*bufq_queue[BUFQ_HOWMANY])(struct bufq *, struct buf *);
-extern void (*bufq_requeue[BUFQ_HOWMANY])(struct bufq *, struct buf *);
+extern struct buf *(*bufq_dequeuev[BUFQ_HOWMANY])(struct bufq *, int);
+extern void (*bufq_queuev[BUFQ_HOWMANY])(struct bufq *, struct buf *);
+extern void (*bufq_requeuev[BUFQ_HOWMANY])(struct bufq *, struct buf *);
 
-#define	BUFQ_QUEUE(_bufq, _bp)		\
-	    bufq_queue[(_bufq)->bufq_type](_bufq, _bp)
-#define BUFQ_REQUEUE(_bufq, _bp)	\
-	    bufq_requeue[(_bufq)->bufq_type](_bufq, _bp)
+#define	BUFQ_QUEUE(_bufq, _bp)	 bufq_queue(_bufq, _bp)
+#define BUFQ_REQUEUE(_bufq, _bp) bufq_requeue(_bufq, _bp)
 #define	BUFQ_DEQUEUE(_bufq)		\
-	    bufq_dequeue[(_bufq)->bufq_type](_bufq, 0)
+	    bufq_dequeuev[(_bufq)->bufq_type](_bufq, 0)
 #define	BUFQ_PEEK(_bufq)		\
-	    bufq_dequeue[(_bufq)->bufq_type](_bufq, 1)
+	bufq_dequeuev[(_bufq)->bufq_type](_bufq, 1)
 
 struct bufq	*bufq_init(int);
+void		 bufq_queue(struct bufq *, struct buf *);
+void		 bufq_requeue(struct bufq *, struct buf *);
 void		 bufq_destroy(struct bufq *);
 void		 bufq_drain(struct bufq *);
+void		 bufq_done(struct bufq *, struct buf *);
+void		 bufq_quiesce(void);
+void		 bufq_restart(void);
 
 
 /*
@@ -152,6 +158,7 @@ struct buf {
 	TAILQ_ENTRY(buf) b_valist;	/* LRU of va to reuse. */
 
 	union	bufq_data b_bufq;
+	struct	bufq	  *b_bq;	/* What bufq this buf is on */
 
 	struct uvm_object *b_pobj;	/* Object containing the pages */
 	off_t	b_poffs;		/* Offset within object */