From 615ee12ec399029b7e8a4272bc38a7ae68e02025 Mon Sep 17 00:00:00 2001 From: David Gwynne Date: Mon, 26 Aug 2013 12:20:13 +0000 Subject: implement handling of group failover. if a controller sends sense data back, the path driver can tell mpath that its indicating failover which kicks off an iteration over all the groups until one says its active. if no groups claim to be active, a timeout fires the process off again after a second. you can start controller handover on rdac (well, an md3200i is all i had to test with, others might need more work) and everything keeps going. ill try to get to emc and hds working when i can poke hardware again. --- sys/scsi/mpath.c | 140 ++++++++++++++++++++++++++++++++++++++++---------- sys/scsi/mpath_rdac.c | 4 +- sys/scsi/mpath_sym.c | 4 +- sys/scsi/mpathvar.h | 5 +- 4 files changed, 118 insertions(+), 35 deletions(-) (limited to 'sys/scsi') diff --git a/sys/scsi/mpath.c b/sys/scsi/mpath.c index ca0d8546d98..c3aef7a19f3 100644 --- a/sys/scsi/mpath.c +++ b/sys/scsi/mpath.c @@ -1,4 +1,4 @@ -/* $OpenBSD: mpath.c,v 1.30 2013/08/26 10:13:17 dlg Exp $ */ +/* $OpenBSD: mpath.c,v 1.31 2013/08/26 12:20:12 dlg Exp $ */ /* * Copyright (c) 2009 David Gwynne @@ -45,6 +45,7 @@ TAILQ_HEAD(mpath_paths, mpath_path); struct mpath_group { TAILQ_ENTRY(mpath_group) g_entry; struct mpath_paths g_paths; + struct mpath_dev *g_dev; u_int g_id; }; TAILQ_HEAD(mpath_groups, mpath_group); @@ -53,9 +54,14 @@ struct mpath_dev { struct mutex d_mtx; struct scsi_xfer_list d_xfers; - struct mpath_groups d_groups; struct mpath_path *d_next_path; + struct mpath_groups d_groups; + + struct mpath_group *d_failover_iter; + struct timeout d_failover_tmo; + u_int d_failover; + const struct mpath_ops *d_ops; struct devid *d_id; }; @@ -89,6 +95,10 @@ int mpath_probe(struct scsi_link *); struct mpath_path *mpath_next_path(struct mpath_dev *, int); void mpath_done(struct scsi_xfer *); +void mpath_failover(struct mpath_dev *); +void mpath_failover_start(void *); +void mpath_failover_check(struct mpath_dev *); + struct scsi_adapter mpath_switch = { mpath_cmd, scsi_minphys, @@ -231,7 +241,7 @@ mpath_cmd(struct scsi_xfer *xs) void mpath_start(struct mpath_path *p, struct scsi_xfer *mxs) { - struct mpath_dev *d = p->p_dev; + struct mpath_dev *d = p->p_group->g_dev; struct scsi_xfer *xs; int addxsh = 0; @@ -295,8 +305,27 @@ mpath_done(struct scsi_xfer *mxs) if (p != NULL) scsi_xsh_add(&p->p_xsh); - return; + case XS_SENSE: + switch (d->d_ops->op_checksense(mxs)) { + case MPATH_SENSE_FAILOVER: + mtx_enter(&d->d_mtx); + SIMPLEQ_INSERT_HEAD(&d->d_xfers, xs, xfer_list); + p = mpath_next_path(d, next); + mtx_leave(&d->d_mtx); + + scsi_xs_put(mxs); + + mpath_failover(d); + return; + case MPATH_SENSE_DECLINED: + break; +#ifdef DIAGNOSTIC + default: + panic("unexpected return from checksense"); +#endif + } + break; } xs->error = mxs->error; @@ -310,6 +339,64 @@ mpath_done(struct scsi_xfer *mxs) scsi_done(xs); } +void +mpath_failover(struct mpath_dev *d) +{ + if (!scsi_sem_enter(&d->d_mtx, &d->d_failover)) + return; + + mpath_failover_start(d); +} + +void +mpath_failover_start(void *xd) +{ + struct mpath_dev *d = xd; + + mtx_enter(&d->d_mtx); + d->d_failover_iter = TAILQ_FIRST(&d->d_groups); + mtx_leave(&d->d_mtx); + + mpath_failover_check(d); +} + +void +mpath_failover_check(struct mpath_dev *d) +{ + struct mpath_group *g = d->d_failover_iter; + struct mpath_path *p; + + if (g == NULL) + timeout_add_sec(&d->d_failover_tmo, 1); + else { + p = TAILQ_FIRST(&g->g_paths); + d->d_ops->op_status(p->p_link); + } +} + +void +mpath_path_status(struct mpath_path *p, int status) +{ + struct mpath_group *g = p->p_group; + struct mpath_dev *d = g->g_dev; + + mtx_enter(&d->d_mtx); + if (status == MPATH_S_ACTIVE) { + TAILQ_REMOVE(&d->d_groups, g, g_entry); + TAILQ_INSERT_HEAD(&d->d_groups, g, g_entry); + d->d_next_path = p; + } else + d->d_failover_iter = TAILQ_NEXT(d->d_failover_iter, g_entry); + mtx_leave(&d->d_mtx); + + if (status == MPATH_S_ACTIVE) { + scsi_xsh_add(&p->p_xsh); + if (!scsi_sem_leave(&d->d_mtx, &d->d_failover)) + mpath_failover_start(d); + } else + mpath_failover_check(d); +} + void mpath_minphys(struct buf *bp, struct scsi_link *link) { @@ -361,8 +448,8 @@ mpath_path_attach(struct mpath_path *p, u_int g_id, const struct mpath_ops *ops) #ifdef DIAGNOSTIC if (p->p_link == NULL) panic("mpath_path_attach: NULL link"); - if (p->p_dev != NULL) - panic("mpath_path_attach: dev is not NULL"); + if (p->p_group != NULL) + panic("mpath_path_attach: group is not NULL"); #endif for (target = 0; target < MPATH_BUSWIDTH; target++) { @@ -393,6 +480,8 @@ mpath_path_attach(struct mpath_path *p, u_int g_id, const struct mpath_ops *ops) d->d_id = devid_copy(link->id); d->d_ops = ops; + timeout_set(&d->d_failover_tmo, mpath_failover_start, d); + sc->sc_devs[target] = d; newdev = 1; } else { @@ -423,6 +512,7 @@ mpath_path_attach(struct mpath_path *p, u_int g_id, const struct mpath_ops *ops) } TAILQ_INIT(&g->g_paths); + g->g_dev = d; g->g_id = g_id; mtx_enter(&d->d_mtx); @@ -430,18 +520,15 @@ mpath_path_attach(struct mpath_path *p, u_int g_id, const struct mpath_ops *ops) mtx_leave(&d->d_mtx); } - p->p_dev = d; + p->p_group = g; mtx_enter(&d->d_mtx); - - if (d->d_next_path == NULL) - d->d_next_path = p; - TAILQ_INSERT_TAIL(&g->g_paths, p, p_entry); - if (!SIMPLEQ_EMPTY(&d->d_xfers)) addxsh = 1; + if (d->d_next_path == NULL) + d->d_next_path = p; mtx_leave(&d->d_mtx); if (newdev) @@ -455,46 +542,43 @@ mpath_path_attach(struct mpath_path *p, u_int g_id, const struct mpath_ops *ops) int mpath_path_detach(struct mpath_path *p) { - struct mpath_dev *d = p->p_dev; - struct mpath_group *g; + struct mpath_group *g = p->p_group; + struct mpath_dev *d = g->g_dev; struct mpath_path *np = NULL; #ifdef DIAGNOSTIC - if (d == NULL) + if (g == NULL) panic("mpath: detaching a path from a nonexistant bus"); #endif - p->p_dev = NULL; + p->p_group = NULL; mtx_enter(&d->d_mtx); - g = TAILQ_FIRST(&d->d_groups); - TAILQ_REMOVE(&g->g_paths, p, p_entry); if (d->d_next_path == p) d->d_next_path = TAILQ_FIRST(&g->g_paths); - if (TAILQ_EMPTY(&g->g_paths)) { + if (TAILQ_EMPTY(&g->g_paths)) TAILQ_REMOVE(&d->d_groups, g, g_entry); - free(g, M_DEVBUF); - } + else + g = NULL; if (!SIMPLEQ_EMPTY(&d->d_xfers)) np = d->d_next_path; mtx_leave(&d->d_mtx); + if (g != NULL) + free(g, M_DEVBUF); + scsi_xsh_del(&p->p_xsh); - if (np != NULL) + if (np == NULL) + mpath_failover(d); + else scsi_xsh_add(&np->p_xsh); return (0); } -void -mpath_path_status(struct mpath_path *p, int status) -{ - -} - struct device * mpath_bootdv(struct device *dev) { diff --git a/sys/scsi/mpath_rdac.c b/sys/scsi/mpath_rdac.c index ac784fc1346..39b39691ece 100644 --- a/sys/scsi/mpath_rdac.c +++ b/sys/scsi/mpath_rdac.c @@ -1,4 +1,4 @@ -/* $OpenBSD: mpath_rdac.c,v 1.15 2013/08/26 11:58:01 dlg Exp $ */ +/* $OpenBSD: mpath_rdac.c,v 1.16 2013/08/26 12:20:12 dlg Exp $ */ /* * Copyright (c) 2010 David Gwynne @@ -257,7 +257,7 @@ rdac_activate(struct device *self, int act) case DVACT_DEACTIVATE: if (scsi_xsh_del(&sc->sc_xsh)) mpath_path_status(&sc->sc_path, MPATH_S_UNKNOWN); - if (sc->sc_path.p_dev != NULL) + if (sc->sc_path.p_group != NULL) mpath_path_detach(&sc->sc_path); break; } diff --git a/sys/scsi/mpath_sym.c b/sys/scsi/mpath_sym.c index 54f15a2d744..b2f15630e3a 100644 --- a/sys/scsi/mpath_sym.c +++ b/sys/scsi/mpath_sym.c @@ -1,4 +1,4 @@ -/* $OpenBSD: mpath_sym.c,v 1.14 2013/08/26 10:50:15 dlg Exp $ */ +/* $OpenBSD: mpath_sym.c,v 1.15 2013/08/26 12:20:12 dlg Exp $ */ /* * Copyright (c) 2010 David Gwynne @@ -184,7 +184,7 @@ sym_activate(struct device *self, int act) case DVACT_RESUME: break; case DVACT_DEACTIVATE: - if (sc->sc_path.p_dev != NULL) + if (sc->sc_path.p_group != NULL) mpath_path_detach(&sc->sc_path); break; } diff --git a/sys/scsi/mpathvar.h b/sys/scsi/mpathvar.h index afb78afc823..224f60e5631 100644 --- a/sys/scsi/mpathvar.h +++ b/sys/scsi/mpathvar.h @@ -1,4 +1,4 @@ -/* $OpenBSD: mpathvar.h,v 1.7 2013/08/26 10:13:17 dlg Exp $ */ +/* $OpenBSD: mpathvar.h,v 1.8 2013/08/26 12:20:12 dlg Exp $ */ /* * Copyright (c) 2010 David Gwynne @@ -19,7 +19,6 @@ #ifndef _SYS_SCSI_MPATH_H_ #define _SYS_SCSI_MPATH_H_ -struct mpath_dev; struct mpath_group; struct mpath_ops { @@ -48,7 +47,7 @@ struct mpath_path { /* the following are private to mpath.c */ TAILQ_ENTRY(mpath_path) p_entry; - struct mpath_dev *p_dev; + struct mpath_group *p_group; int p_state; }; -- cgit v1.2.3