vdev_label.c revision 91ebeef555ce7f899b6270a3c2df47b51f7ad59a
/*
* CDDL HEADER START
*
* The contents of this file are subject to the terms of the
* Common Development and Distribution License (the "License").
* You may not use this file except in compliance with the License.
*
* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
* See the License for the specific language governing permissions
* and limitations under the License.
*
* When distributing Covered Code, include this CDDL HEADER in each
* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
* If applicable, add the following below this CDDL HEADER, with the
* fields enclosed by brackets "[]" replaced with your own identifying
* information: Portions Copyright [yyyy] [name of copyright owner]
*
* CDDL HEADER END
*/
/*
* Copyright 2007 Sun Microsystems, Inc. All rights reserved.
* Use is subject to license terms.
*/
#pragma ident "%Z%%M% %I% %E% SMI"
/*
* Virtual Device Labels
* ---------------------
*
* The vdev label serves several distinct purposes:
*
* 1. Uniquely identify this device as part of a ZFS pool and confirm its
* identity within the pool.
*
* 2. Verify that all the devices given in a configuration are present
* within the pool.
*
* 3. Determine the uberblock for the pool.
*
* 4. In case of an import operation, determine the configuration of the
* toplevel vdev of which it is a part.
*
* 5. If an import operation cannot find all the devices in the pool,
* provide enough information to the administrator to determine which
* devices are missing.
*
* It is important to note that while the kernel is responsible for writing the
* label, it only consumes the information in the first three cases. The
* latter information is only consumed in userland when determining the
* configuration to import a pool.
*
*
* Label Organization
* ------------------
*
* Before describing the contents of the label, it's important to understand how
* the labels are written and updated with respect to the uberblock.
*
* When the pool configuration is altered, either because it was newly created
* or a device was added, we want to update all the labels such that we can deal
* with fatal failure at any point. To this end, each disk has two labels which
* are updated before and after the uberblock is synced. Assuming we have
* labels and an uberblock with the following transaction groups:
*
* L1 UB L2
* +------+ +------+ +------+
* | | | | | |
* | t10 | | t10 | | t10 |
* | | | | | |
* +------+ +------+ +------+
*
* In this stable state, the labels and the uberblock were all updated within
* the same transaction group (10). Each label is mirrored and checksummed, so
* that we can detect when we fail partway through writing the label.
*
* In order to identify which labels are valid, the labels are written in the
* following manner:
*
* 1. For each vdev, update 'L1' to the new label
* 2. Update the uberblock
* 3. For each vdev, update 'L2' to the new label
*
* Given arbitrary failure, we can determine the correct label to use based on
* the transaction group. If we fail after updating L1 but before updating the
* UB, we will notice that L1's transaction group is greater than the uberblock,
* so L2 must be valid. If we fail after writing the uberblock but before
* writing L2, we will notice that L2's transaction group is less than L1, and
* therefore L1 is valid.
*
* Another added complexity is that not every label is updated when the config
* is synced. If we add a single device, we do not want to have to re-write
* every label for every device in the pool. This means that both L1 and L2 may
* be older than the pool uberblock, because the necessary information is stored
* on another vdev.
*
*
* On-disk Format
* --------------
*
* The vdev label consists of two distinct parts, and is wrapped within the
* vdev_label_t structure. The label includes 8k of padding to permit legacy
* VTOC disk labels, but is otherwise ignored.
*
* The first half of the label is a packed nvlist which contains pool wide
* properties, per-vdev properties, and configuration information. It is
* described in more detail below.
*
* The latter half of the label consists of a redundant array of uberblocks.
* These uberblocks are updated whenever a transaction group is committed,
* or when the configuration is updated. When a pool is loaded, we scan each
* vdev for the 'best' uberblock.
*
*
* Configuration Information
* -------------------------
*
* The nvlist describing the pool and vdev contains the following elements:
*
* version ZFS on-disk version
* name Pool name
* state Pool state
* txg Transaction group in which this label was written
* pool_guid Unique identifier for this pool
* vdev_tree An nvlist describing vdev tree.
*
* Each leaf device label also contains the following:
*
* top_guid Unique ID for top-level vdev in which this is contained
* guid Unique ID for the leaf vdev
*
* The 'vs' configuration follows the format described in 'spa_config.c'.
*/
#include <sys/zfs_context.h>
#include <sys/spa_impl.h>
#include <sys/vdev_impl.h>
#include <sys/uberblock_impl.h>
#include <sys/metaslab.h>
/*
* Basic routines to read and write from a vdev label.
* Used throughout the rest of this file.
*/
{
}
static void
{
}
static void
{
}
/*
* Generate the nvlist representing this vdev's config.
*/
nvlist_t *
{
if (!isspare)
== 0);
vd->vdev_devid) == 0);
vd->vdev_physpath) == 0);
if (vd->vdev_nparity != 0) {
VDEV_TYPE_RAIDZ) == 0);
/*
* Make sure someone hasn't managed to sneak a fancy new vdev
* into a crufty old storage pool.
*/
/*
* Note that we'll add the nparity tag even on storage pools
* that only support a single parity device -- older software
* will just ignore it.
*/
vd->vdev_nparity) == 0);
}
vd->vdev_wholedisk) == 0);
if (vd->vdev_not_present)
if (vd->vdev_isspare)
vd->vdev_ms_array) == 0);
vd->vdev_ms_shift) == 0);
vd->vdev_ashift) == 0);
vd->vdev_asize) == 0);
vd->vdev_islog) == 0);
}
if (getstats) {
}
int c;
KM_SLEEP);
for (c = 0; c < vd->vdev_children; c++)
for (c = 0; c < vd->vdev_children; c++)
nvlist_free(child[c]);
} else {
B_TRUE) == 0);
if (vd->vdev_faulted)
B_TRUE) == 0);
if (vd->vdev_degraded)
B_TRUE) == 0);
if (vd->vdev_removed)
B_TRUE) == 0);
if (vd->vdev_unspare)
B_TRUE) == 0);
}
return (nv);
}
nvlist_t *
{
int l;
if (vdev_is_dead(vd))
return (NULL);
for (l = 0; l < VDEV_LABELS; l++) {
&config, 0) == 0)
break;
}
}
return (config);
}
/*
* Determine if a device is in use. The 'spare_guid' parameter will be filled
* in with the device guid if this spare is active elsewhere on the system.
*/
static boolean_t
{
if (spare_guid)
*spare_guid = 0ULL;
/*
* Read the label, if any, and perform some basic sanity checks.
*/
return (B_FALSE);
&vdtxg);
&state) != 0 ||
&device_guid) != 0) {
return (B_FALSE);
}
if (state != POOL_STATE_SPARE &&
&pool_guid) != 0 ||
&txg) != 0)) {
return (B_FALSE);
}
/*
* Check to see if this device indeed belongs to the pool it claims to
* be a part of. The only way this is allowed is if the device is a hot
* spare (which we check for later on).
*/
if (state != POOL_STATE_SPARE &&
return (B_FALSE);
/*
* If the transaction group is zero, then this an initialized (but
* unused) label. This is only an error if the create transaction
* on-disk is the same as the one we're using now, in which case the
* user has attempted to add the same vdev multiple times in the same
* transaction.
*/
return (B_TRUE);
/*
* Check to see if this is a spare device. We do an explicit check for
* spa_has_spare() here because it may be on our pending list of spares
* to add.
*/
if (spare_guid)
switch (reason) {
case VDEV_LABEL_CREATE:
return (B_TRUE);
case VDEV_LABEL_REPLACE:
spare_pool != 0ULL);
case VDEV_LABEL_SPARE:
}
}
/*
* If the device is marked ACTIVE, then this device is in use by another
* pool on the system.
*/
return (state == POOL_STATE_ACTIVE);
}
/*
* Initialize a vdev label. We check to make sure each leaf device is not in
* use, and writable. We put down an initial label which we will later
* overwrite with a complete label. Note that it's important to do this
* sequentially, not in parallel, so that we catch cases of multiple use of the
* same leaf vdev in the vdev we're creating -- e.g. mirroring a disk with
* itself.
*/
int
{
int l, c, n;
char *buf;
int error;
for (c = 0; c < vd->vdev_children; c++)
return (error);
return (0);
/*
* Dead vdevs cannot be initialized.
*/
if (vdev_is_dead(vd))
return (EIO);
/*
* Determine if the vdev is in use.
*/
if (reason != VDEV_LABEL_REMOVE &&
return (EBUSY);
/*
* If this is a request to add or replace a spare that is in use
* elsewhere on the system, then we must update the guid (which was
* initialized to a random value) to reflect the actual GUID (which is
* shared between multiple pools).
*/
}
/*
* If this is a replacement, then we want to fallthrough to the
* rest of the code. If we're adding a spare, then it's already
* labeled appropriately and we can just return.
*/
if (reason == VDEV_LABEL_SPARE)
return (0);
}
/*
* Initialize its label.
*/
/*
* Generate a label describing the pool and our top-level vdev.
* We mark it as being from txg 0 to indicate that it's not
* really part of an active pool just yet. The labels will
* be written again with a meaningful txg by spa_sync().
*/
if (reason == VDEV_LABEL_SPARE ||
/*
* For inactive hot spares, we generate a special label that
* identifies as a mutually shared hot spare. We write the
* label if we are adding a hot spare, or if we are removing an
* active hot spare (in which case we want to revert the
* labels).
*/
spa_version(spa)) == 0);
POOL_STATE_SPARE) == 0);
} else {
/*
* Add our creation time. This allows us to detect multiple
* vdev uses as described above, and automatically expires if we
* fail.
*/
crtxg) == 0);
}
if (error != 0) {
/* EFAULT means nvlist_pack ran out of room */
}
/*
* Initialize boot block header.
*/
/*
* Initialize uberblock template.
*/
/*
* Write everything in parallel.
*/
for (l = 0; l < VDEV_LABELS; l++) {
for (n = 0; n < VDEV_UBERBLOCK_COUNT(vd); n++) {
VDEV_UBERBLOCK_OFFSET(vd, n),
}
}
/*
* If this vdev hasn't been previously identified as a spare, then we
* mark it as such only if a) we are labeling it as a spare, or b) it
* exists as a spare elsewhere in the system.
*/
(reason == VDEV_LABEL_SPARE ||
return (error);
}
/*
* ==========================================================================
* ==========================================================================
*/
/*
* Consider the following situation: txg is safely synced to disk. We've
* written the first uberblock for txg + 1, and then we lose power. When we
* come back up, we fail to see the uberblock for txg + 1 because, say,
* it was on a mirrored device and the replica to which we wrote txg + 1
* is now offline. If we then make some changes and sync txg + 1, and then
* the missing replica comes back, then for a new seconds we'll have two
* conflicting uberblocks on disk with the same txg. The solution is simple:
* among uberblocks with equal txg, choose the one with the latest timestamp.
*/
static int
{
return (-1);
return (1);
return (-1);
return (1);
return (0);
}
static void
{
}
}
void
{
int l, c, n;
for (c = 0; c < vd->vdev_children; c++)
return;
if (vdev_is_dead(vd))
return;
for (l = 0; l < VDEV_LABELS; l++) {
for (n = 0; n < VDEV_UBERBLOCK_COUNT(vd); n++) {
VDEV_UBERBLOCK_OFFSET(vd, n),
}
}
}
/*
* Write the uberblock to both labels of all leaves of the specified vdev.
* We only get credit for writes to known-visible vdevs; see spa_vdev_add().
*/
static void
{
}
static void
{
int l, c, n;
for (c = 0; c < vd->vdev_children; c++)
return;
if (vdev_is_dead(vd))
return;
for (l = 0; l < VDEV_LABELS; l++)
VDEV_UBERBLOCK_OFFSET(vd, n),
}
static int
{
int error;
if (error && *good_writes != 0) {
error = 0;
}
/*
* It's possible to have no good writes and no error if every vdev is in
* the CANT_OPEN state.
*/
if (*good_writes == 0 && error == 0)
return (error);
}
/*
* Sync out an individual vdev.
*/
static void
{
}
static void
{
char *buf;
int c;
for (c = 0; c < vd->vdev_children; c++)
return;
if (vdev_is_dead(vd))
return;
/*
* Generate a label describing the top-level config to which we belong.
*/
}
static int
{
int error;
/*
* Recursively kick off writes to all labels.
*/
if (error && *good_writes != 0) {
error = 0;
}
if (*good_writes == 0 && error == 0)
/*
* Failure to write a label can be fatal for a
* top level vdev. We don't want this for slogs
* as we use the main pool if they go away.
*/
if (vd->vdev_islog)
error = 0;
return (error);
}
/*
* Sync the entire vdev configuration.
*
* The order of operations is carefully crafted to ensure that
* if the system panics or loses power at any time, the state on disk
* is still transactionally consistent. The in-line comments below
* describe the failure semantics at each stage.
*
* Moreover, it is designed to be idempotent: if spa_sync_labels() fails
* at any time, you can just call it again, and it will resume its work.
*/
int
{
int l, error;
/*
* If this isn't a resync due to I/O errors, and nothing changed
* in this transaction group, and the vdev configuration hasn't changed,
* then there's nothing to do.
*/
dprintf("nothing to sync in %s in txg %llu\n",
return (0);
}
return (0);
/*
* Flush the write cache of every disk that's been written to
* in this transaction group. This ensures that all blocks
* written in this txg will be committed to stable storage
* before any uberblock that references them.
*/
}
/*
* Sync out the even labels (L0, L2) for every dirty vdev. If the
* system dies in the middle of this process, that's OK: all of the
* even labels that made it to disk will be newer than any uberblock,
* and will therefore be considered invalid. The odd labels (L1, L3),
* which have not yet been touched, will still be valid.
*/
for (l = 0; l < VDEV_LABELS; l++) {
if (l & 1)
continue;
return (error);
}
}
/*
* Flush the new labels to disk. This ensures that all even-label
* updates are committed to stable storage before the uberblock update.
*/
}
/*
* Sync the uberblocks to all vdevs in the tree specified by uvd.
* If the system dies in the middle of this step, there are two cases
* to consider, and the on-disk state is consistent either way:
*
* (1) If none of the new uberblocks made it to disk, then the
* previous uberblock will be the newest, and the odd labels
* (which had not yet been touched) will be valid with respect
* to that uberblock.
*
* (2) If one or more new uberblocks made it to disk, then they
* will be the newest, and the even labels (which had all
* been successfully committed) will be valid with respect
* to the new uberblocks.
*/
return (error);
/*
* Flush the uberblocks to disk. This ensures that the odd labels
* are no longer needed (because the new uberblocks and the even
* labels are safely on disk), so it is safe to overwrite them.
*/
/*
* Sync out odd labels for every dirty vdev. If the system dies
* in the middle of this process, the even labels and the new
* uberblocks will suffice to open the pool. The next time
* the pool is opened, the first thing we'll do -- before any
* user data is modified -- is mark every vdev dirty so that
* all labels will be brought up to date.
*/
for (l = 0; l < VDEV_LABELS; l++) {
if ((l & 1) == 0)
continue;
return (error);
}
}
/*
* Flush the new labels to disk. This ensures that all odd-label
* updates are committed to stable storage before the next
* transaction group begins.
*/
}
return (0);
}