/* $OpenBSD: rf_map.c,v 1.1 1999/01/11 14:29:28 niklas Exp $ */ /* $NetBSD: rf_map.c,v 1.1 1998/11/13 04:20:31 oster Exp $ */ /* * Copyright (c) 1995 Carnegie-Mellon University. * All rights reserved. * * Author: Mark Holland * * Permission to use, copy, modify and distribute this software and * its documentation is hereby granted, provided that both the copyright * notice and this permission notice appear in all copies of the * software, derivative works or modified versions, and any portions * thereof, and that both notices appear in supporting documentation. * * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. * * Carnegie Mellon requests users of this software to return to * * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU * School of Computer Science * Carnegie Mellon University * Pittsburgh PA 15213-3890 * * any improvements or extensions that they make and grant Carnegie the * rights to redistribute these changes. */ /************************************************************************** * * map.c -- main code for mapping RAID addresses to physical disk addresses * **************************************************************************/ /* * : * Log: rf_map.c,v * Revision 1.53 1996/11/05 21:10:40 jimz * failed pda generalization * * Revision 1.52 1996/08/20 19:58:39 jimz * initialize numParityFailed and numQFailed to 0 in MarkFailuresInASMList * * Revision 1.51 1996/08/19 22:26:31 jimz * add Chang's bugfixes for double-disk failures in MarkFailuresInASMList * * Revision 1.50 1996/08/19 21:38:06 jimz * stripeOffset was uninitialized in CheckStripeForFailures * * Revision 1.49 1996/07/31 15:34:56 jimz * evenodd changes; bugfixes for double-degraded archs, generalize * some formerly PQ-only functions * * Revision 1.48 1996/07/27 23:36:08 jimz * Solaris port of simulator * * Revision 1.47 1996/07/22 19:52:16 jimz * switched node params to RF_DagParam_t, a union of * a 64-bit int and a void *, for better portability * attempted hpux port, but failed partway through for * lack of a single C compiler capable of compiling all * source files * * Revision 1.46 1996/06/10 12:50:57 jimz * Add counters to freelists to track number of allocations, frees, * grows, max size, etc. Adjust a couple sets of PRIME params based * on the results. * * Revision 1.45 1996/06/10 11:55:47 jimz * Straightened out some per-array/not-per-array distinctions, fixed * a couple bugs related to confusion. Added shutdown lists. Removed * layout shutdown function (now subsumed by shutdown lists). * * Revision 1.44 1996/06/09 02:36:46 jimz * lots of little crufty cleanup- fixup whitespace * issues, comment #ifdefs, improve typing in some * places (esp size-related) * * Revision 1.43 1996/06/07 21:33:04 jimz * begin using consistent types for sector numbers, * stripe numbers, row+col numbers, recon unit numbers * * Revision 1.42 1996/06/05 18:06:02 jimz * Major code cleanup. The Great Renaming is now done. * Better modularity. Better typing. Fixed a bunch of * synchronization bugs. Made a lot of global stuff * per-desc or per-array. Removed dead code. * * Revision 1.41 1996/06/03 23:28:26 jimz * more bugfixes * check in tree to sync for IPDS runs with current bugfixes * there still may be a problem with threads in the script test * getting I/Os stuck- not trivially reproducible (runs ~50 times * in a row without getting stuck) * * Revision 1.40 1996/05/31 22:26:54 jimz * fix a lot of mapping problems, memory allocation problems * found some weird lock issues, fixed 'em * more code cleanup * * Revision 1.39 1996/05/30 23:22:16 jimz * bugfixes of serialization, timing problems * more cleanup * * Revision 1.38 1996/05/30 11:29:41 jimz * Numerous bug fixes. Stripe lock release code disagreed with the taking code * about when stripes should be locked (I made it consistent: no parity, no lock) * There was a lot of extra serialization of I/Os which I've removed- a lot of * it was to calculate values for the cache code, which is no longer with us. * More types, function, macro cleanup. Added code to properly quiesce the array * on shutdown. Made a lot of stuff array-specific which was (bogusly) general * before. Fixed memory allocation, freeing bugs. * * Revision 1.37 1996/05/27 18:56:37 jimz * more code cleanup * better typing * compiles in all 3 environments * * Revision 1.36 1996/05/23 21:46:35 jimz * checkpoint in code cleanup (release prep) * lots of types, function names have been fixed * * Revision 1.35 1996/05/23 00:33:23 jimz * code cleanup: move all debug decls to rf_options.c, all extern * debug decls to rf_options.h, all debug vars preceded by rf_ * * Revision 1.34 1996/05/20 16:14:45 jimz * switch to rf_{mutex,cond}_{init,destroy} * * Revision 1.33 1996/05/18 19:51:34 jimz * major code cleanup- fix syntax, make some types consistent, * add prototypes, clean out dead code, et cetera * * Revision 1.32 1996/05/17 00:51:47 jimz * reformat for readability * * Revision 1.31 1996/05/16 23:06:26 jimz * convert asmhdr to use RF_FREELIST stuff * * Revision 1.30 1996/05/16 19:09:42 jimz * grow init asm freelist to 32 * * Revision 1.29 1996/05/16 15:27:55 jimz * prime freelist pumps for asm and pda lists * * Revision 1.28 1996/05/02 14:58:35 jimz * legibility cleanup * * Revision 1.27 1995/12/12 18:10:06 jimz * MIN -> RF_MIN, MAX -> RF_MAX, ASSERT -> RF_ASSERT * fix 80-column brain damage in comments * * Revision 1.26 1995/12/01 19:25:06 root * added copyright info * * Revision 1.25 1995/11/17 19:01:57 wvcii * added call to MapQ in two fault tolerant case * * Revision 1.24 1995/11/17 15:10:53 wvcii * fixed bug in ASMCheckStatus - ASSERT was using disk sector addresses * rather than raidAddress * * Revision 1.23 1995/07/26 03:26:51 robby * map the allocation and freeing routines for some stuff non-static * * Revision 1.22 1995/06/28 09:33:45 holland * bug fixes related to dist sparing and multiple-row arrays * * Revision 1.21 1995/06/28 04:51:08 holland * added some asserts against zero-length accesses * * Revision 1.20 1995/06/23 13:40:06 robby * updeated to prototypes in rf_layout.h * */ #include "rf_types.h" #include "rf_threadstuff.h" #include "rf_raid.h" #include "rf_general.h" #include "rf_map.h" #include "rf_freelist.h" #include "rf_shutdown.h" #include "rf_sys.h" static void rf_FreePDAList(RF_PhysDiskAddr_t *start, RF_PhysDiskAddr_t *end, int count); static void rf_FreeASMList(RF_AccessStripeMap_t *start, RF_AccessStripeMap_t *end, int count); /***************************************************************************************** * * MapAccess -- main 1st order mapping routine. * * Maps an access in the RAID address space to the corresponding set of physical disk * addresses. The result is returned as a list of AccessStripeMap structures, one per * stripe accessed. Each ASM structure contains a pointer to a list of PhysDiskAddr * structures, which describe the physical locations touched by the user access. Note * that this routine returns only static mapping information, i.e. the list of physical * addresses returned does not necessarily identify the set of physical locations that * will actually be read or written. * * The routine also maps the parity. The physical disk location returned always * indicates the entire parity unit, even when only a subset of it is being accessed. * This is because an access that is not stripe unit aligned but that spans a stripe * unit boundary may require access two distinct portions of the parity unit, and we * can't yet tell which portion(s) we'll actually need. We leave it up to the algorithm * selection code to decide what subset of the parity unit to access. * * Note that addresses in the RAID address space must always be maintained as * longs, instead of ints. * * This routine returns NULL if numBlocks is 0 * ****************************************************************************************/ RF_AccessStripeMapHeader_t *rf_MapAccess(raidPtr, raidAddress, numBlocks, buffer, remap) RF_Raid_t *raidPtr; RF_RaidAddr_t raidAddress; /* starting address in RAID address space */ RF_SectorCount_t numBlocks; /* number of blocks in RAID address space to access */ caddr_t buffer; /* buffer to supply/receive data */ int remap; /* 1 => remap addresses to spare space */ { RF_RaidLayout_t *layoutPtr = &(raidPtr->Layout); RF_AccessStripeMapHeader_t *asm_hdr = NULL; RF_AccessStripeMap_t *asm_list = NULL, *asm_p = NULL; int faultsTolerated = layoutPtr->map->faultsTolerated; RF_RaidAddr_t startAddress = raidAddress; /* we'll change raidAddress along the way */ RF_RaidAddr_t endAddress = raidAddress + numBlocks; RF_RaidDisk_t **disks = raidPtr->Disks; RF_PhysDiskAddr_t *pda_p, *pda_q; RF_StripeCount_t numStripes = 0; RF_RaidAddr_t stripeRealEndAddress, stripeEndAddress, nextStripeUnitAddress; RF_RaidAddr_t startAddrWithinStripe, lastRaidAddr; RF_StripeCount_t totStripes; RF_StripeNum_t stripeID, lastSID, SUID, lastSUID; RF_AccessStripeMap_t *asmList, *t_asm; RF_PhysDiskAddr_t *pdaList, *t_pda; /* allocate all the ASMs and PDAs up front */ lastRaidAddr = raidAddress + numBlocks - 1 ; stripeID = rf_RaidAddressToStripeID(layoutPtr, raidAddress); lastSID = rf_RaidAddressToStripeID(layoutPtr, lastRaidAddr); totStripes = lastSID - stripeID + 1; SUID = rf_RaidAddressToStripeUnitID(layoutPtr, raidAddress); lastSUID = rf_RaidAddressToStripeUnitID(layoutPtr, lastRaidAddr); asmList = rf_AllocASMList(totStripes); pdaList = rf_AllocPDAList(lastSUID - SUID + 1 + faultsTolerated * totStripes); /* may also need pda(s) per stripe for parity */ if (raidAddress+numBlocks > raidPtr->totalSectors) { RF_ERRORMSG1("Unable to map access because offset (%d) was invalid\n", (int)raidAddress); return(NULL); } if (rf_mapDebug) rf_PrintRaidAddressInfo(raidPtr, raidAddress, numBlocks); for (; raidAddress < endAddress; ) { /* make the next stripe structure */ RF_ASSERT(asmList); t_asm = asmList; asmList = asmList->next; bzero((char *)t_asm, sizeof(RF_AccessStripeMap_t)); if (!asm_p) asm_list = asm_p = t_asm; else { asm_p->next = t_asm; asm_p = asm_p->next; } numStripes++; /* map SUs from current location to the end of the stripe */ asm_p->stripeID = /*rf_RaidAddressToStripeID(layoutPtr, raidAddress)*/ stripeID++; stripeRealEndAddress = rf_RaidAddressOfNextStripeBoundary(layoutPtr, raidAddress); stripeEndAddress = RF_MIN(endAddress,stripeRealEndAddress ); asm_p->raidAddress = raidAddress; asm_p->endRaidAddress = stripeEndAddress; /* map each stripe unit in the stripe */ pda_p = NULL; startAddrWithinStripe = raidAddress; /* Raid addr of start of portion of access that is within this stripe */ for (; raidAddress < stripeEndAddress; ) { RF_ASSERT(pdaList); t_pda = pdaList; pdaList = pdaList->next; bzero((char *)t_pda, sizeof(RF_PhysDiskAddr_t)); if (!pda_p) asm_p->physInfo = pda_p = t_pda; else { pda_p->next = t_pda; pda_p = pda_p->next; } pda_p->type = RF_PDA_TYPE_DATA; (layoutPtr->map->MapSector)(raidPtr, raidAddress, &(pda_p->row), &(pda_p->col), &(pda_p->startSector), remap); /* mark any failures we find. failedPDA is don't-care if there is more than one failure */ pda_p->raidAddress = raidAddress; /* the RAID address corresponding to this physical disk address */ nextStripeUnitAddress = rf_RaidAddressOfNextStripeUnitBoundary(layoutPtr, raidAddress); pda_p->numSector = RF_MIN(endAddress, nextStripeUnitAddress) - raidAddress; RF_ASSERT(pda_p->numSector != 0); rf_ASMCheckStatus(raidPtr,pda_p,asm_p,disks,0); pda_p->bufPtr = buffer + rf_RaidAddressToByte(raidPtr, (raidAddress - startAddress)); asm_p->totalSectorsAccessed += pda_p->numSector; asm_p->numStripeUnitsAccessed++; asm_p->origRow = pda_p->row; /* redundant but harmless to do this in every loop iteration */ raidAddress = RF_MIN(endAddress, nextStripeUnitAddress); } /* Map the parity. At this stage, the startSector and numSector fields * for the parity unit are always set to indicate the entire parity unit. * We may modify this after mapping the data portion. */ switch (faultsTolerated) { case 0: break; case 1: /* single fault tolerant */ RF_ASSERT(pdaList); t_pda = pdaList; pdaList = pdaList->next; bzero((char *)t_pda, sizeof(RF_PhysDiskAddr_t)); pda_p = asm_p->parityInfo = t_pda; pda_p->type = RF_PDA_TYPE_PARITY; (layoutPtr->map->MapParity)(raidPtr, rf_RaidAddressOfPrevStripeUnitBoundary(layoutPtr, startAddrWithinStripe), &(pda_p->row), &(pda_p->col), &(pda_p->startSector), remap); pda_p->numSector = layoutPtr->sectorsPerStripeUnit; /* raidAddr may be needed to find unit to redirect to */ pda_p->raidAddress = rf_RaidAddressOfPrevStripeUnitBoundary(layoutPtr, startAddrWithinStripe); rf_ASMCheckStatus(raidPtr,pda_p,asm_p,disks,1); rf_ASMParityAdjust(asm_p->parityInfo,startAddrWithinStripe,endAddress,layoutPtr,asm_p); break; case 2: /* two fault tolerant */ RF_ASSERT(pdaList && pdaList->next); t_pda = pdaList; pdaList = pdaList->next; bzero((char *)t_pda, sizeof(RF_PhysDiskAddr_t)); pda_p = asm_p->parityInfo = t_pda; pda_p->type = RF_PDA_TYPE_PARITY; t_pda = pdaList; pdaList = pdaList->next; bzero((char *)t_pda, sizeof(RF_PhysDiskAddr_t)); pda_q = asm_p->qInfo = t_pda; pda_q->type = RF_PDA_TYPE_Q; (layoutPtr->map->MapParity)(raidPtr, rf_RaidAddressOfPrevStripeUnitBoundary(layoutPtr, startAddrWithinStripe), &(pda_p->row), &(pda_p->col), &(pda_p->startSector), remap); (layoutPtr->map->MapQ)(raidPtr, rf_RaidAddressOfPrevStripeUnitBoundary(layoutPtr, startAddrWithinStripe), &(pda_q->row), &(pda_q->col), &(pda_q->startSector), remap); pda_q->numSector = pda_p->numSector = layoutPtr->sectorsPerStripeUnit; /* raidAddr may be needed to find unit to redirect to */ pda_p->raidAddress = rf_RaidAddressOfPrevStripeUnitBoundary(layoutPtr, startAddrWithinStripe); pda_q->raidAddress = rf_RaidAddressOfPrevStripeUnitBoundary(layoutPtr, startAddrWithinStripe); /* failure mode stuff */ rf_ASMCheckStatus(raidPtr,pda_p,asm_p,disks,1); rf_ASMCheckStatus(raidPtr,pda_q,asm_p,disks,1); rf_ASMParityAdjust(asm_p->parityInfo,startAddrWithinStripe,endAddress,layoutPtr,asm_p); rf_ASMParityAdjust(asm_p->qInfo,startAddrWithinStripe,endAddress,layoutPtr,asm_p); break; } } RF_ASSERT(asmList == NULL && pdaList == NULL); /* make the header structure */ asm_hdr = rf_AllocAccessStripeMapHeader(); RF_ASSERT(numStripes == totStripes); asm_hdr->numStripes = numStripes; asm_hdr->stripeMap = asm_list; if (rf_mapDebug) rf_PrintAccessStripeMap(asm_hdr); return(asm_hdr); } /***************************************************************************************** * This routine walks through an ASM list and marks the PDAs that have failed. * It's called only when a disk failure causes an in-flight DAG to fail. * The parity may consist of two components, but we want to use only one failedPDA * pointer. Thus we set failedPDA to point to the first parity component, and rely * on the rest of the code to do the right thing with this. ****************************************************************************************/ void rf_MarkFailuresInASMList(raidPtr, asm_h) RF_Raid_t *raidPtr; RF_AccessStripeMapHeader_t *asm_h; { RF_RaidDisk_t **disks = raidPtr->Disks; RF_AccessStripeMap_t *asmap; RF_PhysDiskAddr_t *pda; for (asmap = asm_h->stripeMap; asmap; asmap = asmap->next) { asmap->numDataFailed = asmap->numParityFailed = asmap->numQFailed = 0; asmap->numFailedPDAs = 0; bzero((char *)asmap->failedPDAs, RF_MAX_FAILED_PDA*sizeof(RF_PhysDiskAddr_t *)); for (pda = asmap->physInfo; pda; pda=pda->next) { if (RF_DEAD_DISK(disks[pda->row][pda->col].status)) { printf("DEAD DISK BOGUSLY DETECTED!!\n"); asmap->numDataFailed++; asmap->failedPDAs[asmap->numFailedPDAs] = pda; asmap->numFailedPDAs++; } } pda = asmap->parityInfo; if (pda && RF_DEAD_DISK(disks[pda->row][pda->col].status)) { asmap->numParityFailed++; asmap->failedPDAs[asmap->numFailedPDAs] = pda; asmap->numFailedPDAs++; } pda = asmap->qInfo; if (pda && RF_DEAD_DISK(disks[pda->row][pda->col].status)) { asmap->numQFailed++; asmap->failedPDAs[asmap->numFailedPDAs] = pda; asmap->numFailedPDAs++; } } } /***************************************************************************************** * * DuplicateASM -- duplicates an ASM and returns the new one * ****************************************************************************************/ RF_AccessStripeMap_t *rf_DuplicateASM(asmap) RF_AccessStripeMap_t *asmap; { RF_AccessStripeMap_t *new_asm; RF_PhysDiskAddr_t *pda, *new_pda, *t_pda; new_pda = NULL; new_asm = rf_AllocAccessStripeMapComponent(); bcopy((char *)asmap, (char *)new_asm, sizeof(RF_AccessStripeMap_t)); new_asm->numFailedPDAs = 0; /* ??? */ new_asm->failedPDAs[0] = NULL; new_asm->physInfo = NULL; new_asm->parityInfo = NULL; new_asm->next = NULL; for (pda = asmap->physInfo; pda; pda=pda->next) { /* copy the physInfo list */ t_pda = rf_AllocPhysDiskAddr(); bcopy((char *)pda, (char *)t_pda, sizeof(RF_PhysDiskAddr_t)); t_pda->next = NULL; if (!new_asm->physInfo) {new_asm->physInfo = t_pda; new_pda = t_pda;} else {new_pda->next = t_pda; new_pda = new_pda->next;} if (pda == asmap->failedPDAs[0]) new_asm->failedPDAs[0] = t_pda; } for (pda = asmap->parityInfo; pda; pda=pda->next) { /* copy the parityInfo list */ t_pda = rf_AllocPhysDiskAddr(); bcopy((char *)pda, (char *)t_pda, sizeof(RF_PhysDiskAddr_t)); t_pda->next = NULL; if (!new_asm->parityInfo) {new_asm->parityInfo = t_pda; new_pda = t_pda;} else {new_pda->next = t_pda; new_pda = new_pda->next;} if (pda == asmap->failedPDAs[0]) new_asm->failedPDAs[0] = t_pda; } return(new_asm); } /***************************************************************************************** * * DuplicatePDA -- duplicates a PDA and returns the new one * ****************************************************************************************/ RF_PhysDiskAddr_t *rf_DuplicatePDA(pda) RF_PhysDiskAddr_t *pda; { RF_PhysDiskAddr_t *new; new = rf_AllocPhysDiskAddr(); bcopy((char *)pda, (char *)new, sizeof(RF_PhysDiskAddr_t)); return(new); } /***************************************************************************************** * * routines to allocate and free list elements. All allocation routines zero the * structure before returning it. * * FreePhysDiskAddr is static. It should never be called directly, because * FreeAccessStripeMap takes care of freeing the PhysDiskAddr list. * ****************************************************************************************/ static RF_FreeList_t *rf_asmhdr_freelist; #define RF_MAX_FREE_ASMHDR 128 #define RF_ASMHDR_INC 16 #define RF_ASMHDR_INITIAL 32 static RF_FreeList_t *rf_asm_freelist; #define RF_MAX_FREE_ASM 192 #define RF_ASM_INC 24 #define RF_ASM_INITIAL 64 static RF_FreeList_t *rf_pda_freelist; #define RF_MAX_FREE_PDA 192 #define RF_PDA_INC 24 #define RF_PDA_INITIAL 64 /* called at shutdown time. So far, all that is necessary is to release all the free lists */ static void rf_ShutdownMapModule(void *); static void rf_ShutdownMapModule(ignored) void *ignored; { RF_FREELIST_DESTROY(rf_asmhdr_freelist,next,(RF_AccessStripeMapHeader_t *)); RF_FREELIST_DESTROY(rf_pda_freelist,next,(RF_PhysDiskAddr_t *)); RF_FREELIST_DESTROY(rf_asm_freelist,next,(RF_AccessStripeMap_t *)); } int rf_ConfigureMapModule(listp) RF_ShutdownList_t **listp; { int rc; RF_FREELIST_CREATE(rf_asmhdr_freelist, RF_MAX_FREE_ASMHDR, RF_ASMHDR_INC, sizeof(RF_AccessStripeMapHeader_t)); if (rf_asmhdr_freelist == NULL) { return(ENOMEM); } RF_FREELIST_CREATE(rf_asm_freelist, RF_MAX_FREE_ASM, RF_ASM_INC, sizeof(RF_AccessStripeMap_t)); if (rf_asm_freelist == NULL) { RF_FREELIST_DESTROY(rf_asmhdr_freelist,next,(RF_AccessStripeMapHeader_t *)); return(ENOMEM); } RF_FREELIST_CREATE(rf_pda_freelist, RF_MAX_FREE_PDA, RF_PDA_INC, sizeof(RF_PhysDiskAddr_t)); if (rf_pda_freelist == NULL) { RF_FREELIST_DESTROY(rf_asmhdr_freelist,next,(RF_AccessStripeMapHeader_t *)); RF_FREELIST_DESTROY(rf_pda_freelist,next,(RF_PhysDiskAddr_t *)); return(ENOMEM); } rc = rf_ShutdownCreate(listp, rf_ShutdownMapModule, NULL); if (rc) { RF_ERRORMSG3("Unable to add to shutdown list file %s line %d rc=%d\n", __FILE__, __LINE__, rc); rf_ShutdownMapModule(NULL); return(rc); } RF_FREELIST_PRIME(rf_asmhdr_freelist, RF_ASMHDR_INITIAL,next, (RF_AccessStripeMapHeader_t *)); RF_FREELIST_PRIME(rf_asm_freelist, RF_ASM_INITIAL,next, (RF_AccessStripeMap_t *)); RF_FREELIST_PRIME(rf_pda_freelist, RF_PDA_INITIAL,next, (RF_PhysDiskAddr_t *)); return(0); } RF_AccessStripeMapHeader_t *rf_AllocAccessStripeMapHeader() { RF_AccessStripeMapHeader_t *p; RF_FREELIST_GET(rf_asmhdr_freelist,p,next,(RF_AccessStripeMapHeader_t *)); bzero((char *)p, sizeof(RF_AccessStripeMapHeader_t)); return(p); } void rf_FreeAccessStripeMapHeader(p) RF_AccessStripeMapHeader_t *p; { RF_FREELIST_FREE(rf_asmhdr_freelist,p,next); } RF_PhysDiskAddr_t *rf_AllocPhysDiskAddr() { RF_PhysDiskAddr_t *p; RF_FREELIST_GET(rf_pda_freelist,p,next,(RF_PhysDiskAddr_t *)); bzero((char *)p, sizeof(RF_PhysDiskAddr_t)); return(p); } /* allocates a list of PDAs, locking the free list only once * when we have to call calloc, we do it one component at a time to simplify * the process of freeing the list at program shutdown. This should not be * much of a performance hit, because it should be very infrequently executed. */ RF_PhysDiskAddr_t *rf_AllocPDAList(count) int count; { RF_PhysDiskAddr_t *p = NULL; RF_FREELIST_GET_N(rf_pda_freelist,p,next,(RF_PhysDiskAddr_t *),count); return(p); } void rf_FreePhysDiskAddr(p) RF_PhysDiskAddr_t *p; { RF_FREELIST_FREE(rf_pda_freelist,p,next); } static void rf_FreePDAList(l_start, l_end, count) RF_PhysDiskAddr_t *l_start, *l_end; /* pointers to start and end of list */ int count; /* number of elements in list */ { RF_FREELIST_FREE_N(rf_pda_freelist,l_start,next,(RF_PhysDiskAddr_t *),count); } RF_AccessStripeMap_t *rf_AllocAccessStripeMapComponent() { RF_AccessStripeMap_t *p; RF_FREELIST_GET(rf_asm_freelist,p,next,(RF_AccessStripeMap_t *)); bzero((char *)p, sizeof(RF_AccessStripeMap_t)); return(p); } /* this is essentially identical to AllocPDAList. I should combine the two. * when we have to call calloc, we do it one component at a time to simplify * the process of freeing the list at program shutdown. This should not be * much of a performance hit, because it should be very infrequently executed. */ RF_AccessStripeMap_t *rf_AllocASMList(count) int count; { RF_AccessStripeMap_t *p = NULL; RF_FREELIST_GET_N(rf_asm_freelist,p,next,(RF_AccessStripeMap_t *),count); return(p); } void rf_FreeAccessStripeMapComponent(p) RF_AccessStripeMap_t *p; { RF_FREELIST_FREE(rf_asm_freelist,p,next); } static void rf_FreeASMList(l_start, l_end, count) RF_AccessStripeMap_t *l_start, *l_end; int count; { RF_FREELIST_FREE_N(rf_asm_freelist,l_start,next,(RF_AccessStripeMap_t *),count); } void rf_FreeAccessStripeMap(hdr) RF_AccessStripeMapHeader_t *hdr; { RF_AccessStripeMap_t *p, *pt = NULL; RF_PhysDiskAddr_t *pdp, *trailer, *pdaList = NULL, *pdaEnd = NULL; int count = 0, t, asm_count = 0; for (p = hdr->stripeMap; p; p=p->next) { /* link the 3 pda lists into the accumulating pda list */ if (!pdaList) pdaList = p->qInfo; else pdaEnd->next = p->qInfo; for (trailer=NULL,pdp=p->qInfo; pdp; ) {trailer = pdp; pdp=pdp->next; count++;} if (trailer) pdaEnd = trailer; if (!pdaList) pdaList = p->parityInfo; else pdaEnd->next = p->parityInfo; for (trailer=NULL,pdp=p->parityInfo; pdp; ) {trailer = pdp; pdp=pdp->next; count++;} if (trailer) pdaEnd = trailer; if (!pdaList) pdaList = p->physInfo; else pdaEnd->next = p->physInfo; for (trailer=NULL,pdp=p->physInfo; pdp; ) {trailer = pdp; pdp=pdp->next; count++;} if (trailer) pdaEnd = trailer; pt = p; asm_count++; } /* debug only */ for (t=0,pdp=pdaList; pdp; pdp=pdp->next) t++; RF_ASSERT(t == count); if (pdaList) rf_FreePDAList(pdaList, pdaEnd, count); rf_FreeASMList(hdr->stripeMap, pt, asm_count); rf_FreeAccessStripeMapHeader(hdr); } /* We can't use the large write optimization if there are any failures in the stripe. * In the declustered layout, there is no way to immediately determine what disks * constitute a stripe, so we actually have to hunt through the stripe looking for failures. * The reason we map the parity instead of just using asm->parityInfo->col is because * the latter may have been already redirected to a spare drive, which would * mess up the computation of the stripe offset. * * ASSUMES AT MOST ONE FAILURE IN THE STRIPE. */ int rf_CheckStripeForFailures(raidPtr, asmap) RF_Raid_t *raidPtr; RF_AccessStripeMap_t *asmap; { RF_RowCol_t trow, tcol, prow, pcol, *diskids, row, i; RF_RaidLayout_t *layoutPtr = &raidPtr->Layout; RF_StripeCount_t stripeOffset; int numFailures; RF_RaidAddr_t sosAddr; RF_SectorNum_t diskOffset, poffset; RF_RowCol_t testrow; /* quick out in the fault-free case. */ RF_LOCK_MUTEX(raidPtr->mutex); numFailures = raidPtr->numFailures; RF_UNLOCK_MUTEX(raidPtr->mutex); if (numFailures == 0) return(0); sosAddr = rf_RaidAddressOfPrevStripeBoundary(layoutPtr, asmap->raidAddress); row = asmap->physInfo->row; (layoutPtr->map->IdentifyStripe)(raidPtr, asmap->raidAddress, &diskids, &testrow); (layoutPtr->map->MapParity)(raidPtr, asmap->raidAddress, &prow, &pcol, &poffset, 0); /* get pcol */ /* this need not be true if we've redirected the access to a spare in another row RF_ASSERT(row == testrow); */ stripeOffset = 0; for (i=0; inumDataCol+layoutPtr->numParityCol; i++) { if (diskids[i] != pcol) { if (RF_DEAD_DISK(raidPtr->Disks[testrow][diskids[i]].status)) { if (raidPtr->status[testrow] != rf_rs_reconstructing) return(1); RF_ASSERT(raidPtr->reconControl[testrow]->fcol == diskids[i]); layoutPtr->map->MapSector(raidPtr, sosAddr + stripeOffset * layoutPtr->sectorsPerStripeUnit, &trow, &tcol, &diskOffset, 0); RF_ASSERT( (trow == testrow) && (tcol == diskids[i]) ); if (!rf_CheckRUReconstructed(raidPtr->reconControl[testrow]->reconMap, diskOffset)) return(1); asmap->flags |= RF_ASM_REDIR_LARGE_WRITE; return(0); } stripeOffset++; } } return(0); } /* return the number of failed data units in the stripe. */ int rf_NumFailedDataUnitsInStripe(raidPtr, asmap) RF_Raid_t *raidPtr; RF_AccessStripeMap_t *asmap; { RF_RaidLayout_t *layoutPtr = &raidPtr->Layout; RF_RowCol_t trow, tcol, row, i; RF_SectorNum_t diskOffset; RF_RaidAddr_t sosAddr; int numFailures; /* quick out in the fault-free case. */ RF_LOCK_MUTEX(raidPtr->mutex); numFailures = raidPtr->numFailures; RF_UNLOCK_MUTEX(raidPtr->mutex); if (numFailures == 0) return(0); numFailures = 0; sosAddr = rf_RaidAddressOfPrevStripeBoundary(layoutPtr, asmap->raidAddress); row = asmap->physInfo->row; for (i=0; inumDataCol; i++) { (layoutPtr->map->MapSector)(raidPtr, sosAddr + i * layoutPtr->sectorsPerStripeUnit, &trow, &tcol, &diskOffset, 0); if (RF_DEAD_DISK(raidPtr->Disks[trow][tcol].status)) numFailures++; } return numFailures; } /***************************************************************************************** * * debug routines * ****************************************************************************************/ void rf_PrintAccessStripeMap(asm_h) RF_AccessStripeMapHeader_t *asm_h; { rf_PrintFullAccessStripeMap(asm_h, 0); } void rf_PrintFullAccessStripeMap(asm_h, prbuf) RF_AccessStripeMapHeader_t *asm_h; int prbuf; /* flag to print buffer pointers */ { int i; RF_AccessStripeMap_t *asmap = asm_h->stripeMap; RF_PhysDiskAddr_t *p; printf("%d stripes total\n", (int)asm_h->numStripes); for (; asmap; asmap = asmap->next) { /* printf("Num failures: %d\n",asmap->numDataFailed); */ /* printf("Num sectors: %d\n",(int)asmap->totalSectorsAccessed); */ printf("Stripe %d (%d sectors), failures: %d data, %d parity: ", (int) asmap->stripeID, (int) asmap->totalSectorsAccessed, (int) asmap->numDataFailed, (int) asmap->numParityFailed); if (asmap->parityInfo) { printf("Parity [r%d c%d s%d-%d", asmap->parityInfo->row, asmap->parityInfo->col, (int)asmap->parityInfo->startSector, (int)(asmap->parityInfo->startSector + asmap->parityInfo->numSector - 1)); if (prbuf) printf(" b0x%lx",(unsigned long) asmap->parityInfo->bufPtr); if (asmap->parityInfo->next) { printf(", r%d c%d s%d-%d", asmap->parityInfo->next->row, asmap->parityInfo->next->col, (int) asmap->parityInfo->next->startSector, (int)(asmap->parityInfo->next->startSector + asmap->parityInfo->next->numSector - 1)); if (prbuf) printf(" b0x%lx",(unsigned long) asmap->parityInfo->next->bufPtr); RF_ASSERT(asmap->parityInfo->next->next == NULL); } printf("]\n\t"); } for (i=0,p=asmap->physInfo; p; p=p->next,i++) { printf("SU r%d c%d s%d-%d ", p->row, p->col, (int)p->startSector, (int)(p->startSector + p->numSector - 1)); if (prbuf) printf("b0x%lx ", (unsigned long) p->bufPtr); if (i && !(i&1)) printf("\n\t"); } printf("\n"); p = asm_h->stripeMap->failedPDAs[0]; if (asm_h->stripeMap->numDataFailed + asm_h->stripeMap->numParityFailed > 1) printf("[multiple failures]\n"); else if (asm_h->stripeMap->numDataFailed + asm_h->stripeMap->numParityFailed > 0) printf("\t[Failed PDA: r%d c%d s%d-%d]\n",p->row, p->col, (int)p->startSector, (int)(p->startSector + p->numSector-1)); } } void rf_PrintRaidAddressInfo(raidPtr, raidAddr, numBlocks) RF_Raid_t *raidPtr; RF_RaidAddr_t raidAddr; RF_SectorCount_t numBlocks; { RF_RaidLayout_t *layoutPtr = &raidPtr->Layout; RF_RaidAddr_t ra, sosAddr = rf_RaidAddressOfPrevStripeBoundary(layoutPtr, raidAddr); printf("Raid addrs of SU boundaries from start of stripe to end of access:\n\t"); for (ra = sosAddr; ra <= raidAddr + numBlocks; ra += layoutPtr->sectorsPerStripeUnit) { printf("%d (0x%x), ",(int)ra, (int)ra); } printf("\n"); printf("Offset into stripe unit: %d (0x%x)\n", (int)(raidAddr % layoutPtr->sectorsPerStripeUnit), (int)(raidAddr % layoutPtr->sectorsPerStripeUnit)); } /* given a parity descriptor and the starting address within a stripe, range restrict the parity descriptor to touch only the correct stuff. */ void rf_ASMParityAdjust( RF_PhysDiskAddr_t *toAdjust, RF_StripeNum_t startAddrWithinStripe, RF_SectorNum_t endAddress, RF_RaidLayout_t *layoutPtr, RF_AccessStripeMap_t *asm_p) { RF_PhysDiskAddr_t *new_pda; /* when we're accessing only a portion of one stripe unit, we want the parity descriptor * to identify only the chunk of parity associated with the data. When the access spans * exactly one stripe unit boundary and is less than a stripe unit in size, it uses two disjoint * regions of the parity unit. When an access spans more than one stripe unit boundary, it * uses all of the parity unit. * * To better handle the case where stripe units are small, we may eventually want to change * the 2nd case so that if the SU size is below some threshold, we just read/write the whole * thing instead of breaking it up into two accesses. */ if (asm_p->numStripeUnitsAccessed == 1) { int x = (startAddrWithinStripe % layoutPtr->sectorsPerStripeUnit); toAdjust->startSector += x; toAdjust->raidAddress += x; toAdjust->numSector = asm_p->physInfo->numSector; RF_ASSERT(toAdjust->numSector != 0); } else if (asm_p->numStripeUnitsAccessed == 2 && asm_p->totalSectorsAccessed < layoutPtr->sectorsPerStripeUnit) { int x = (startAddrWithinStripe % layoutPtr->sectorsPerStripeUnit); /* create a second pda and copy the parity map info into it */ RF_ASSERT(toAdjust->next == NULL); new_pda = toAdjust->next = rf_AllocPhysDiskAddr(); *new_pda = *toAdjust; /* structure assignment */ new_pda->next = NULL; /* adjust the start sector & number of blocks for the first parity pda */ toAdjust->startSector += x; toAdjust->raidAddress += x; toAdjust->numSector = rf_RaidAddressOfNextStripeUnitBoundary(layoutPtr, startAddrWithinStripe) - startAddrWithinStripe; RF_ASSERT(toAdjust->numSector != 0); /* adjust the second pda */ new_pda->numSector = endAddress - rf_RaidAddressOfPrevStripeUnitBoundary(layoutPtr, endAddress); /*new_pda->raidAddress = rf_RaidAddressOfNextStripeUnitBoundary(layoutPtr, toAdjust->raidAddress);*/ RF_ASSERT(new_pda->numSector != 0); } } /* Check if a disk has been spared or failed. If spared, redirect the I/O. If it has been failed, record it in the asm pointer. Fourth arg is whether data or parity. */ void rf_ASMCheckStatus( RF_Raid_t *raidPtr, RF_PhysDiskAddr_t *pda_p, RF_AccessStripeMap_t *asm_p, RF_RaidDisk_t **disks, int parity) { RF_DiskStatus_t dstatus; RF_RowCol_t frow, fcol; dstatus = disks[pda_p->row][pda_p->col].status; if (dstatus == rf_ds_spared) { /* if the disk has been spared, redirect access to the spare */ frow = pda_p->row; fcol = pda_p->col; pda_p->row = disks[frow][fcol].spareRow; pda_p->col = disks[frow][fcol].spareCol; } else if (dstatus == rf_ds_dist_spared) { /* ditto if disk has been spared to dist spare space */ RF_RowCol_t or = pda_p->row, oc=pda_p->col; RF_SectorNum_t oo = pda_p->startSector; if (pda_p -> type == RF_PDA_TYPE_DATA) raidPtr->Layout.map->MapSector(raidPtr, pda_p->raidAddress, &pda_p->row, &pda_p->col, &pda_p->startSector, RF_REMAP); else raidPtr->Layout.map->MapParity(raidPtr, pda_p->raidAddress, &pda_p->row, &pda_p->col, &pda_p->startSector, RF_REMAP); if (rf_mapDebug) { printf("Redirected r %d c %d o %d -> r%d c %d o %d\n",or,oc,(int)oo, pda_p->row,pda_p->col,(int)pda_p->startSector); } } else if (RF_DEAD_DISK(dstatus)) { /* if the disk is inaccessible, mark the failure */ if (parity) asm_p->numParityFailed++; else { asm_p->numDataFailed++; #if 0 /* XXX Do we really want this spewing out on the console? GO */ printf("DATA_FAILED!\n"); #endif } asm_p->failedPDAs[asm_p->numFailedPDAs] = pda_p; asm_p->numFailedPDAs++; #if 0 switch (asm_p->numParityFailed + asm_p->numDataFailed) { case 1: asm_p->failedPDAs[0] = pda_p; break; case 2: asm_p->failedPDAs[1] = pda_p; default: break; } #endif } /* the redirected access should never span a stripe unit boundary */ RF_ASSERT(rf_RaidAddressToStripeUnitID(&raidPtr->Layout,pda_p->raidAddress) == rf_RaidAddressToStripeUnitID(&raidPtr->Layout,pda_p->raidAddress + pda_p->numSector -1)); RF_ASSERT(pda_p->col != -1); }