vdev_label.c revision 0430f8daa551890e0788d3fd28aef3be44cf8730
/*
* CDDL HEADER START
*
* The contents of this file are subject to the terms of the
* Common Development and Distribution License (the "License").
* You may not use this file except in compliance with the License.
*
* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
* See the License for the specific language governing permissions
* and limitations under the License.
*
* When distributing Covered Code, include this CDDL HEADER in each
* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
* If applicable, add the following below this CDDL HEADER, with the
* fields enclosed by brackets "[]" replaced with your own identifying
* information: Portions Copyright [yyyy] [name of copyright owner]
*
* CDDL HEADER END
*/
/*
* Copyright 2008 Sun Microsystems, Inc. All rights reserved.
* Use is subject to license terms.
*/
#pragma ident "%Z%%M% %I% %E% SMI"
/*
* Virtual Device Labels
* ---------------------
*
* The vdev label serves several distinct purposes:
*
* 1. Uniquely identify this device as part of a ZFS pool and confirm its
* identity within the pool.
*
* 2. Verify that all the devices given in a configuration are present
* within the pool.
*
* 3. Determine the uberblock for the pool.
*
* 4. In case of an import operation, determine the configuration of the
* toplevel vdev of which it is a part.
*
* 5. If an import operation cannot find all the devices in the pool,
* provide enough information to the administrator to determine which
* devices are missing.
*
* It is important to note that while the kernel is responsible for writing the
* label, it only consumes the information in the first three cases. The
* latter information is only consumed in userland when determining the
* configuration to import a pool.
*
*
* Label Organization
* ------------------
*
* Before describing the contents of the label, it's important to understand how
* the labels are written and updated with respect to the uberblock.
*
* When the pool configuration is altered, either because it was newly created
* or a device was added, we want to update all the labels such that we can deal
* with fatal failure at any point. To this end, each disk has two labels which
* are updated before and after the uberblock is synced. Assuming we have
* labels and an uberblock with the following transaction groups:
*
* L1 UB L2
* +------+ +------+ +------+
* | | | | | |
* | t10 | | t10 | | t10 |
* | | | | | |
* +------+ +------+ +------+
*
* In this stable state, the labels and the uberblock were all updated within
* the same transaction group (10). Each label is mirrored and checksummed, so
* that we can detect when we fail partway through writing the label.
*
* In order to identify which labels are valid, the labels are written in the
* following manner:
*
* 1. For each vdev, update 'L1' to the new label
* 2. Update the uberblock
* 3. For each vdev, update 'L2' to the new label
*
* Given arbitrary failure, we can determine the correct label to use based on
* the transaction group. If we fail after updating L1 but before updating the
* UB, we will notice that L1's transaction group is greater than the uberblock,
* so L2 must be valid. If we fail after writing the uberblock but before
* writing L2, we will notice that L2's transaction group is less than L1, and
* therefore L1 is valid.
*
* Another added complexity is that not every label is updated when the config
* is synced. If we add a single device, we do not want to have to re-write
* every label for every device in the pool. This means that both L1 and L2 may
* be older than the pool uberblock, because the necessary information is stored
* on another vdev.
*
*
* On-disk Format
* --------------
*
* The vdev label consists of two distinct parts, and is wrapped within the
* vdev_label_t structure. The label includes 8k of padding to permit legacy
* VTOC disk labels, but is otherwise ignored.
*
* The first half of the label is a packed nvlist which contains pool wide
* properties, per-vdev properties, and configuration information. It is
* described in more detail below.
*
* The latter half of the label consists of a redundant array of uberblocks.
* These uberblocks are updated whenever a transaction group is committed,
* or when the configuration is updated. When a pool is loaded, we scan each
* vdev for the 'best' uberblock.
*
*
* Configuration Information
* -------------------------
*
* The nvlist describing the pool and vdev contains the following elements:
*
* version ZFS on-disk version
* name Pool name
* state Pool state
* txg Transaction group in which this label was written
* pool_guid Unique identifier for this pool
* vdev_tree An nvlist describing vdev tree.
*
* Each leaf device label also contains the following:
*
* top_guid Unique ID for top-level vdev in which this is contained
* guid Unique ID for the leaf vdev
*
* The 'vs' configuration follows the format described in 'spa_config.c'.
*/
#include <sys/zfs_context.h>
#include <sys/spa_impl.h>
#include <sys/vdev_impl.h>
#include <sys/uberblock_impl.h>
#include <sys/metaslab.h>
/*
* Basic routines to read and write from a vdev label.
* Used throughout the rest of this file.
*/
{
}
/*
* Returns back the vdev label associated with the passed in offset.
*/
int
{
int l;
}
l = offset / sizeof (vdev_label_t);
return (l < VDEV_LABELS ? l : -1);
}
static void
{
B_TRUE));
}
static void
{
}
/*
* Generate the nvlist representing this vdev's config.
*/
nvlist_t *
{
== 0);
vd->vdev_devid) == 0);
vd->vdev_physpath) == 0);
if (vd->vdev_nparity != 0) {
VDEV_TYPE_RAIDZ) == 0);
/*
* Make sure someone hasn't managed to sneak a fancy new vdev
* into a crufty old storage pool.
*/
/*
* Note that we'll add the nparity tag even on storage pools
* that only support a single parity device -- older software
* will just ignore it.
*/
vd->vdev_nparity) == 0);
}
vd->vdev_wholedisk) == 0);
if (vd->vdev_not_present)
if (vd->vdev_isspare)
vd->vdev_ms_array) == 0);
vd->vdev_ms_shift) == 0);
vd->vdev_ashift) == 0);
vd->vdev_asize) == 0);
vd->vdev_islog) == 0);
}
if (getstats) {
}
int c;
KM_SLEEP);
for (c = 0; c < vd->vdev_children; c++)
for (c = 0; c < vd->vdev_children; c++)
nvlist_free(child[c]);
} else {
B_TRUE) == 0);
if (vd->vdev_faulted)
B_TRUE) == 0);
if (vd->vdev_degraded)
B_TRUE) == 0);
if (vd->vdev_removed)
B_TRUE) == 0);
if (vd->vdev_unspare)
B_TRUE) == 0);
}
return (nv);
}
nvlist_t *
{
int l;
if (!vdev_readable(vd))
return (NULL);
for (l = 0; l < VDEV_LABELS; l++) {
&config, 0) == 0)
break;
}
}
return (config);
}
/*
* Determine if a device is in use. The 'spare_guid' parameter will be filled
* in with the device guid if this spare is active elsewhere on the system.
*/
static boolean_t
{
if (spare_guid)
*spare_guid = 0ULL;
if (l2cache_guid)
*l2cache_guid = 0ULL;
/*
* Read the label, if any, and perform some basic sanity checks.
*/
return (B_FALSE);
&vdtxg);
&state) != 0 ||
&device_guid) != 0) {
return (B_FALSE);
}
&pool_guid) != 0 ||
&txg) != 0)) {
return (B_FALSE);
}
/*
* Check to see if this device indeed belongs to the pool it claims to
* be a part of. The only way this is allowed is if the device is a hot
* spare (which we check for later on).
*/
return (B_FALSE);
/*
* If the transaction group is zero, then this an initialized (but
* unused) label. This is only an error if the create transaction
* on-disk is the same as the one we're using now, in which case the
* user has attempted to add the same vdev multiple times in the same
* transaction.
*/
return (B_TRUE);
/*
* Check to see if this is a spare device. We do an explicit check for
* spa_has_spare() here because it may be on our pending list of spares
* to add. We also check if it is an l2cache device.
*/
if (spare_guid)
switch (reason) {
case VDEV_LABEL_CREATE:
case VDEV_LABEL_L2CACHE:
return (B_TRUE);
case VDEV_LABEL_REPLACE:
spare_pool != 0ULL);
case VDEV_LABEL_SPARE:
}
}
/*
* Check to see if this is an l2cache device.
*/
return (B_TRUE);
/*
* If the device is marked ACTIVE, then this device is in use by another
* pool on the system.
*/
return (state == POOL_STATE_ACTIVE);
}
/*
* Initialize a vdev label. We check to make sure each leaf device is not in
* use, and writable. We put down an initial label which we will later
* overwrite with a complete label. Note that it's important to do this
* sequentially, not in parallel, so that we catch cases of multiple use of the
* same leaf vdev in the vdev we're creating -- e.g. mirroring a disk with
* itself.
*/
int
{
int l, c, n;
char *buf;
int error;
for (c = 0; c < vd->vdev_children; c++)
return (error);
return (0);
/*
* Dead vdevs cannot be initialized.
*/
if (vdev_is_dead(vd))
return (EIO);
/*
* Determine if the vdev is in use.
*/
if (reason != VDEV_LABEL_REMOVE &&
return (EBUSY);
/*
* If this is a request to add or replace a spare or l2cache device
* that is in use elsewhere on the system, then we must update the
* guid (which was initialized to a random value) to reflect the
* actual GUID (which is shared between multiple pools).
*/
spare_guid != 0ULL) {
}
/*
* If this is a replacement, then we want to fallthrough to the
* rest of the code. If we're adding a spare, then it's already
* labeled appropriately and we can just return.
*/
if (reason == VDEV_LABEL_SPARE)
return (0);
}
l2cache_guid != 0ULL) {
}
/*
* If this is a replacement, then we want to fallthrough to the
* rest of the code. If we're adding an l2cache, then it's
* already labeled appropriately and we can just return.
*/
if (reason == VDEV_LABEL_L2CACHE)
return (0);
}
/*
* Initialize its label.
*/
/*
* Generate a label describing the pool and our top-level vdev.
* We mark it as being from txg 0 to indicate that it's not
* really part of an active pool just yet. The labels will
* be written again with a meaningful txg by spa_sync().
*/
if (reason == VDEV_LABEL_SPARE ||
/*
* For inactive hot spares, we generate a special label that
* identifies as a mutually shared hot spare. We write the
* label if we are adding a hot spare, or if we are removing an
* active hot spare (in which case we want to revert the
* labels).
*/
spa_version(spa)) == 0);
POOL_STATE_SPARE) == 0);
} else if (reason == VDEV_LABEL_L2CACHE ||
/*
* For level 2 ARC devices, add a special label.
*/
spa_version(spa)) == 0);
POOL_STATE_L2CACHE) == 0);
} else {
/*
* Add our creation time. This allows us to detect multiple
* vdev uses as described above, and automatically expires if we
* fail.
*/
crtxg) == 0);
}
if (error != 0) {
/* EFAULT means nvlist_pack ran out of room */
}
/*
* Initialize boot block header.
*/
/*
* Initialize uberblock template.
*/
/*
* Write everything in parallel.
*/
for (l = 0; l < VDEV_LABELS; l++) {
for (n = 0; n < VDEV_UBERBLOCK_COUNT(vd); n++) {
VDEV_UBERBLOCK_OFFSET(vd, n),
}
}
/*
* If this vdev hasn't been previously identified as a spare, then we
* mark it as such only if a) we are labeling it as a spare, or b) it
* exists as a spare elsewhere in the system. Do the same for
* level 2 ARC devices.
*/
(reason == VDEV_LABEL_SPARE ||
(reason == VDEV_LABEL_L2CACHE ||
return (error);
}
/*
* ==========================================================================
* ==========================================================================
*/
/*
* Consider the following situation: txg is safely synced to disk. We've
* written the first uberblock for txg + 1, and then we lose power. When we
* come back up, we fail to see the uberblock for txg + 1 because, say,
* it was on a mirrored device and the replica to which we wrote txg + 1
* is now offline. If we then make some changes and sync txg + 1, and then
* the missing replica comes back, then for a new seconds we'll have two
* conflicting uberblocks on disk with the same txg. The solution is simple:
* among uberblocks with equal txg, choose the one with the latest timestamp.
*/
static int
{
return (-1);
return (1);
return (-1);
return (1);
return (0);
}
static void
{
}
}
void
{
int l, c, n;
for (c = 0; c < vd->vdev_children; c++)
return;
if (vdev_is_dead(vd))
return;
for (l = 0; l < VDEV_LABELS; l++) {
for (n = 0; n < VDEV_UBERBLOCK_COUNT(vd); n++) {
VDEV_UBERBLOCK_OFFSET(vd, n),
}
}
}
/*
* On success, increment root zio's count of good writes.
* We only get credit for writes to known-visible vdevs; see spa_vdev_add().
*/
static void
{
}
/*
* Write the uberblock to all labels of all leaves of the specified vdev.
*/
static void
{
int l, c, n;
for (c = 0; c < vd->vdev_children; c++)
return;
if (vdev_is_dead(vd))
return;
for (l = 0; l < VDEV_LABELS; l++)
VDEV_UBERBLOCK_OFFSET(vd, n),
}
static void
{
if (*good_writes == 0)
}
int
{
int v;
uint64_t good_writes = 0;
/*
* If we've been asked to update all the vdevs then we change
* our flags to ZIO_FLAG_MUSTSUCCEED so that the pipeline can
* handle error should all update fail.
*/
io_flags &= ~ZIO_FLAG_CANFAIL;
/*
* We rely on the value of good_writes and the root I/O to determine
* how a complete failure is handled. In the event that the root is a
* ZIO_FLAG_MUSTSUCCED, then the pipeline will block this I/O if we
* were unable to update any uberblock. Once the I/O is blocked the
* pipeline will retry it when the error is cleared. Unfortunately,
* the pipeline does not have the complete I/O tree so it will be
* unable to retry the actual uberblock update. Instead we rely on
* the value of good_writes to return the failed status to the caller
* which will retry on error and thus resubmit the complete I/O
* tree.
*/
flags);
for (v = 0; v < svdcount; v++)
/*
* Flush the uberblocks to disk. This ensures that the odd labels
* are no longer needed (because the new uberblocks and the even
* labels are safely on disk), so it is safe to overwrite them.
*/
for (v = 0; v < svdcount; v++)
}
/*
* On success, increment the count of good writes for our top-level vdev.
*/
static void
{
}
/*
* If there weren't enough good writes, indicate failure to the parent.
*/
static void
{
if (*good_writes == 0)
}
/*
* We ignore errors for log and cache devices, simply free the private data.
*/
static void
{
}
/*
* Write all even or odd labels to all leaves of the specified vdev.
*/
static void
{
char *buf;
int c;
for (c = 0; c < vd->vdev_children; c++)
return;
if (vdev_is_dead(vd))
return;
/*
* Generate a label describing the top-level config to which we belong.
*/
for (; l < VDEV_LABELS; l += 2) {
sizeof (vdev_phys_t),
}
}
}
int
{
int error;
/*
* The root I/O for all label updates must succeed and we track
* the error returned back from the null I/O to determine if we
* need to reissue the I/O tree from scratch. If we are unable
* to update any leaf vdev associated with a dirty top-level vdev,
* then the pipeline will either suspend or panic when the root I/O
* is issued. If the error is cleared, then the pipleine will retry
* the root I/O. Unfortunately we've lost the entire I/O tree so we
* return back the original error to the caller and allow the caller
* to call use again so that we can build the I/O tree from scratch.
*/
KM_SLEEP);
good_writes, flags);
}
/*
* Flush the new labels to disk.
*/
return (error);
}
/*
* Sync the uberblock and any changes to the vdev configuration.
*
* The order of operations is carefully crafted to ensure that
* if the system panics or loses power at any time, the state on disk
* is still transactionally consistent. The in-line comments below
* describe the failure semantics at each stage.
*
* Moreover, vdev_config_sync() is designed to be idempotent: if it fails
* at any time, you can just call it again, and it will resume its work.
*/
void
{
/*
* If this isn't a resync due to I/O errors,
* and nothing changed in this transaction group,
* and the vdev configuration hasn't changed,
* then there's nothing to do.
*/
return;
return;
/*
* Flush the write cache of every disk that's been written to
* in this transaction group. This ensures that all blocks
* written in this txg will be committed to stable storage
* before any uberblock that references them.
*/
/*
* Sync out the even labels (L0, L2) for every dirty vdev. If the
* system dies in the middle of this process, that's OK: all of the
* even labels that made it to disk will be newer than any uberblock,
* and will therefore be considered invalid. The odd labels (L1, L3),
* which have not yet been touched, will still be valid. We flush
* the new labels to disk to ensure that all even-label updates
* are committed to stable storage before the uberblock update.
* Failure to update any of the labels will invoke the 'failmode'
* code path. Thus we must retry the entire I/O tree once the error
* is cleared and we ar resumed.
*/
;
/*
* Sync the uberblocks to all vdevs in svd[]. If we are unable
* to do so, then we attempt to sync out to all top-level vdevs.
* If the system dies in the middle of this step, there are two cases
* to consider, and the on-disk state is consistent either way:
*
* (1) If none of the new uberblocks made it to disk, then the
* previous uberblock will be the newest, and the odd labels
* (which had not yet been touched) will be valid with respect
* to that uberblock.
*
* (2) If one or more new uberblocks made it to disk, then they
* will be the newest, and the even labels (which had all
* been successfully committed) will be valid with respect
* to the new uberblocks.
*
* In addition, if we have failed to update all the uberblocks then
* we will follow the 'failmode' code path. We must retry the entire
* I/O tree if we are resumed.
*/
;
}
/*
* Sync out odd labels for every dirty vdev. If the system dies
* in the middle of this process, the even labels and the new
* uberblocks will suffice to open the pool. The next time
* the pool is opened, the first thing we'll do -- before any
* user data is modified -- is mark every vdev dirty so that
* all labels will be brought up to date. We flush the new labels
* to disk to ensure that all odd-label updates are committed to
* stable storage before the next transaction group begins.
* Failure to update any of the labels will invoke the 'failmode'
* code path. Thus we must retry the entire I/O tree once the error
* is cleared and we are resumed.
*/
;
}