implement handling of group failover.

if a controller sends sense data back, the path driver can tell
mpath that its indicating failover which kicks off an iteration
over all the groups until one says its active. if no groups claim
to be active, a timeout fires the process off again after a second.

you can start controller handover on rdac (well, an md3200i is all
i had to test with, others might need more work) and everything
keeps going. ill try to get to emc and hds working when i can poke
hardware again.

4 files changed, 118 insertions, 35 deletions

diff --git a/sys/scsi/mpath.c b/sys/scsi/mpath.c
index ca0d8546d98..c3aef7a19f3 100644
--- a/sys/scsi/mpath.c
+++ b/sys/scsi/mpath.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: mpath.c,v 1.30 2013/08/26 10:13:17 dlg Exp $ */
+/*	$OpenBSD: mpath.c,v 1.31 2013/08/26 12:20:12 dlg Exp $ */
 
 /*
  * Copyright (c) 2009 David Gwynne <dlg@openbsd.org>
@@ -45,6 +45,7 @@ TAILQ_HEAD(mpath_paths, mpath_path);
 struct mpath_group {
 	TAILQ_ENTRY(mpath_group) g_entry;
 	struct mpath_paths	 g_paths;
+	struct mpath_dev	*g_dev;
 	u_int			 g_id;
 };
 TAILQ_HEAD(mpath_groups, mpath_group);
@@ -53,9 +54,14 @@ struct mpath_dev {
 	struct mutex		 d_mtx;
 
 	struct scsi_xfer_list	 d_xfers;
-	struct mpath_groups	 d_groups;
 	struct mpath_path	*d_next_path;
 
+	struct mpath_groups	 d_groups;
+
+	struct mpath_group	*d_failover_iter;
+	struct timeout		 d_failover_tmo;
+	u_int			 d_failover;
+
 	const struct mpath_ops	*d_ops;
 	struct devid		*d_id;
 };
@@ -89,6 +95,10 @@ int		mpath_probe(struct scsi_link *);
 struct mpath_path *mpath_next_path(struct mpath_dev *, int);
 void		mpath_done(struct scsi_xfer *);
 
+void		mpath_failover(struct mpath_dev *);
+void		mpath_failover_start(void *);
+void		mpath_failover_check(struct mpath_dev *);
+
 struct scsi_adapter mpath_switch = {
 	mpath_cmd,
 	scsi_minphys,
@@ -231,7 +241,7 @@ mpath_cmd(struct scsi_xfer *xs)
 void
 mpath_start(struct mpath_path *p, struct scsi_xfer *mxs)
 {
-	struct mpath_dev *d = p->p_dev;
+	struct mpath_dev *d = p->p_group->g_dev;
 	struct scsi_xfer *xs;
 	int addxsh = 0;
 
@@ -295,8 +305,27 @@ mpath_done(struct scsi_xfer *mxs)
 
 		if (p != NULL)
 			scsi_xsh_add(&p->p_xsh);
-
 		return;
+	case XS_SENSE:
+		switch (d->d_ops->op_checksense(mxs)) {
+		case MPATH_SENSE_FAILOVER:
+			mtx_enter(&d->d_mtx);
+			SIMPLEQ_INSERT_HEAD(&d->d_xfers, xs, xfer_list);
+			p = mpath_next_path(d, next);
+			mtx_leave(&d->d_mtx);
+
+			scsi_xs_put(mxs);
+
+			mpath_failover(d);
+			return;
+		case MPATH_SENSE_DECLINED:
+			break;
+#ifdef DIAGNOSTIC
+		default:
+			panic("unexpected return from checksense");
+#endif
+		}
+		break;
 	}
 
 	xs->error = mxs->error;
@@ -311,6 +340,64 @@ mpath_done(struct scsi_xfer *mxs)
 }
 
 void
+mpath_failover(struct mpath_dev *d)
+{
+	if (!scsi_sem_enter(&d->d_mtx, &d->d_failover))
+		return;
+
+	mpath_failover_start(d);
+}
+
+void
+mpath_failover_start(void *xd)
+{
+	struct mpath_dev *d = xd;
+
+	mtx_enter(&d->d_mtx);
+	d->d_failover_iter = TAILQ_FIRST(&d->d_groups);
+	mtx_leave(&d->d_mtx);
+
+	mpath_failover_check(d);
+}
+
+void
+mpath_failover_check(struct mpath_dev *d)
+{
+	struct mpath_group *g = d->d_failover_iter;
+	struct mpath_path *p;
+
+	if (g == NULL)
+		timeout_add_sec(&d->d_failover_tmo, 1);
+	else {
+		p = TAILQ_FIRST(&g->g_paths);
+		d->d_ops->op_status(p->p_link);
+	}
+}
+
+void
+mpath_path_status(struct mpath_path *p, int status)
+{
+	struct mpath_group *g = p->p_group;
+	struct mpath_dev *d = g->g_dev;
+
+	mtx_enter(&d->d_mtx);
+	if (status == MPATH_S_ACTIVE) {
+		TAILQ_REMOVE(&d->d_groups, g, g_entry);
+		TAILQ_INSERT_HEAD(&d->d_groups, g, g_entry);
+		d->d_next_path = p;
+	} else
+		d->d_failover_iter = TAILQ_NEXT(d->d_failover_iter, g_entry);
+	mtx_leave(&d->d_mtx);
+
+	if (status == MPATH_S_ACTIVE) {
+		scsi_xsh_add(&p->p_xsh);
+		if (!scsi_sem_leave(&d->d_mtx, &d->d_failover))
+			mpath_failover_start(d);
+	} else
+		mpath_failover_check(d);
+}
+
+void
 mpath_minphys(struct buf *bp, struct scsi_link *link)
 {
 	struct mpath_softc *sc = link->adapter_softc;
@@ -361,8 +448,8 @@ mpath_path_attach(struct mpath_path *p, u_int g_id, const struct mpath_ops *ops)
 #ifdef DIAGNOSTIC
 	if (p->p_link == NULL)
 		panic("mpath_path_attach: NULL link");
-	if (p->p_dev != NULL)
-		panic("mpath_path_attach: dev is not NULL");
+	if (p->p_group != NULL)
+		panic("mpath_path_attach: group is not NULL");
 #endif
 
 	for (target = 0; target < MPATH_BUSWIDTH; target++) {
@@ -393,6 +480,8 @@ mpath_path_attach(struct mpath_path *p, u_int g_id, const struct mpath_ops *ops)
 		d->d_id = devid_copy(link->id);
 		d->d_ops = ops;
 
+		timeout_set(&d->d_failover_tmo, mpath_failover_start, d);
+
 		sc->sc_devs[target] = d;
 		newdev = 1;
 	} else {
@@ -423,6 +512,7 @@ mpath_path_attach(struct mpath_path *p, u_int g_id, const struct mpath_ops *ops)
 		}
 
 		TAILQ_INIT(&g->g_paths);
+		g->g_dev = d;
 		g->g_id = g_id;
 
 		mtx_enter(&d->d_mtx);
@@ -430,18 +520,15 @@ mpath_path_attach(struct mpath_path *p, u_int g_id, const struct mpath_ops *ops)
 		mtx_leave(&d->d_mtx);
 	}
 
-	p->p_dev = d;
+	p->p_group = g;
 
 	mtx_enter(&d->d_mtx);
-
-	if (d->d_next_path == NULL)
-		d->d_next_path = p;
-
 	TAILQ_INSERT_TAIL(&g->g_paths, p, p_entry);
-
 	if (!SIMPLEQ_EMPTY(&d->d_xfers))
 		addxsh = 1;
 
+	if (d->d_next_path == NULL)
+		d->d_next_path = p;
 	mtx_leave(&d->d_mtx);
 
 	if (newdev)
@@ -455,46 +542,43 @@ mpath_path_attach(struct mpath_path *p, u_int g_id, const struct mpath_ops *ops)
 int
 mpath_path_detach(struct mpath_path *p)
 {
-	struct mpath_dev *d = p->p_dev;
-	struct mpath_group *g;
+	struct mpath_group *g = p->p_group;
+	struct mpath_dev *d = g->g_dev;
 	struct mpath_path *np = NULL;
 
 #ifdef DIAGNOSTIC
-	if (d == NULL)
+	if (g == NULL)
 		panic("mpath: detaching a path from a nonexistant bus");
 #endif
-	p->p_dev = NULL;
+	p->p_group = NULL;
 
 	mtx_enter(&d->d_mtx);
-	g = TAILQ_FIRST(&d->d_groups);
-
 	TAILQ_REMOVE(&g->g_paths, p, p_entry);
 	if (d->d_next_path == p)
 		d->d_next_path = TAILQ_FIRST(&g->g_paths);
 
-	if (TAILQ_EMPTY(&g->g_paths)) {
+	if (TAILQ_EMPTY(&g->g_paths))
 		TAILQ_REMOVE(&d->d_groups, g, g_entry);
-		free(g, M_DEVBUF);
-	}
+	else
+		g = NULL;
 
 	if (!SIMPLEQ_EMPTY(&d->d_xfers))
 		np = d->d_next_path;
 	mtx_leave(&d->d_mtx);
 
+	if (g != NULL)
+		free(g, M_DEVBUF);
+
 	scsi_xsh_del(&p->p_xsh);
 
-	if (np != NULL)
+	if (np == NULL)
+		mpath_failover(d);
+	else
 		scsi_xsh_add(&np->p_xsh);
 
 	return (0);
 }
 
-void
-mpath_path_status(struct mpath_path *p, int status)
-{
-
-}
-
 struct device *
 mpath_bootdv(struct device *dev)
 {
diff --git a/sys/scsi/mpath_rdac.c b/sys/scsi/mpath_rdac.c
index ac784fc1346..39b39691ece 100644
--- a/sys/scsi/mpath_rdac.c
+++ b/sys/scsi/mpath_rdac.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: mpath_rdac.c,v 1.15 2013/08/26 11:58:01 dlg Exp $ */
+/*	$OpenBSD: mpath_rdac.c,v 1.16 2013/08/26 12:20:12 dlg Exp $ */
 
 /*
  * Copyright (c) 2010 David Gwynne <dlg@openbsd.org>
@@ -257,7 +257,7 @@ rdac_activate(struct device *self, int act)
 	case DVACT_DEACTIVATE:
 		if (scsi_xsh_del(&sc->sc_xsh))
 			mpath_path_status(&sc->sc_path, MPATH_S_UNKNOWN);
-		if (sc->sc_path.p_dev != NULL)
+		if (sc->sc_path.p_group != NULL)
 			mpath_path_detach(&sc->sc_path);
 		break;
 	}
diff --git a/sys/scsi/mpath_sym.c b/sys/scsi/mpath_sym.c
index 54f15a2d744..b2f15630e3a 100644
--- a/sys/scsi/mpath_sym.c
+++ b/sys/scsi/mpath_sym.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: mpath_sym.c,v 1.14 2013/08/26 10:50:15 dlg Exp $ */
+/*	$OpenBSD: mpath_sym.c,v 1.15 2013/08/26 12:20:12 dlg Exp $ */
 
 /*
  * Copyright (c) 2010 David Gwynne <dlg@openbsd.org>
@@ -184,7 +184,7 @@ sym_activate(struct device *self, int act)
 	case DVACT_RESUME:
 		break;
 	case DVACT_DEACTIVATE:
-		if (sc->sc_path.p_dev != NULL)
+		if (sc->sc_path.p_group != NULL)
 			mpath_path_detach(&sc->sc_path);
 		break;
 	}
diff --git a/sys/scsi/mpathvar.h b/sys/scsi/mpathvar.h
index afb78afc823..224f60e5631 100644
--- a/sys/scsi/mpathvar.h
+++ b/sys/scsi/mpathvar.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: mpathvar.h,v 1.7 2013/08/26 10:13:17 dlg Exp $ */
+/*	$OpenBSD: mpathvar.h,v 1.8 2013/08/26 12:20:12 dlg Exp $ */
 
 /*
  * Copyright (c) 2010 David Gwynne <dlg@openbsd.org>
@@ -19,7 +19,6 @@
 #ifndef _SYS_SCSI_MPATH_H_
 #define _SYS_SCSI_MPATH_H_
 
-struct mpath_dev;
 struct mpath_group;
 
 struct mpath_ops {
@@ -48,7 +47,7 @@ struct mpath_path {
 
 	/* the following are private to mpath.c */
 	TAILQ_ENTRY(mpath_path)	 p_entry;
-	struct mpath_dev	*p_dev;
+	struct mpath_group	*p_group;
 	int			 p_state;
 };