99653d4ee642c6528e88224f12409a5f23060994eschrock/*
99653d4ee642c6528e88224f12409a5f23060994eschrock * CDDL HEADER START
99653d4ee642c6528e88224f12409a5f23060994eschrock *
99653d4ee642c6528e88224f12409a5f23060994eschrock * The contents of this file are subject to the terms of the
99653d4ee642c6528e88224f12409a5f23060994eschrock * Common Development and Distribution License (the "License").
99653d4ee642c6528e88224f12409a5f23060994eschrock * You may not use this file except in compliance with the License.
99653d4ee642c6528e88224f12409a5f23060994eschrock *
99653d4ee642c6528e88224f12409a5f23060994eschrock * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
99653d4ee642c6528e88224f12409a5f23060994eschrock * or http://www.opensolaris.org/os/licensing.
99653d4ee642c6528e88224f12409a5f23060994eschrock * See the License for the specific language governing permissions
99653d4ee642c6528e88224f12409a5f23060994eschrock * and limitations under the License.
99653d4ee642c6528e88224f12409a5f23060994eschrock *
99653d4ee642c6528e88224f12409a5f23060994eschrock * When distributing Covered Code, include this CDDL HEADER in each
99653d4ee642c6528e88224f12409a5f23060994eschrock * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
99653d4ee642c6528e88224f12409a5f23060994eschrock * If applicable, add the following below this CDDL HEADER, with the
99653d4ee642c6528e88224f12409a5f23060994eschrock * fields enclosed by brackets "[]" replaced with your own identifying
99653d4ee642c6528e88224f12409a5f23060994eschrock * information: Portions Copyright [yyyy] [name of copyright owner]
99653d4ee642c6528e88224f12409a5f23060994eschrock *
99653d4ee642c6528e88224f12409a5f23060994eschrock * CDDL HEADER END
99653d4ee642c6528e88224f12409a5f23060994eschrock */
99653d4ee642c6528e88224f12409a5f23060994eschrock/*
2a417b235e35bd992c3c76d3eb89d35467286133Robert Johnston * Copyright (c) 2006, 2010, Oracle and/or its affiliates. All rights reserved.
99653d4ee642c6528e88224f12409a5f23060994eschrock */
99653d4ee642c6528e88224f12409a5f23060994eschrock
99653d4ee642c6528e88224f12409a5f23060994eschrock/*
99653d4ee642c6528e88224f12409a5f23060994eschrock * The ZFS retire agent is responsible for managing hot spares across all pools.
3d7072f8bd27709dba14f6fe336f149d25d9e207eschrock * When we see a device fault or a device removal, we try to open the associated
3d7072f8bd27709dba14f6fe336f149d25d9e207eschrock * pool and look for any hot spares. We iterate over any available hot spares
3d7072f8bd27709dba14f6fe336f149d25d9e207eschrock * and attempt a 'zpool replace' for each one.
3d7072f8bd27709dba14f6fe336f149d25d9e207eschrock *
3d7072f8bd27709dba14f6fe336f149d25d9e207eschrock * For vdevs diagnosed as faulty, the agent is also responsible for proactively
3d7072f8bd27709dba14f6fe336f149d25d9e207eschrock * marking the vdev FAULTY (for I/O errors) or DEGRADED (for checksum errors).
99653d4ee642c6528e88224f12409a5f23060994eschrock */
99653d4ee642c6528e88224f12409a5f23060994eschrock
99653d4ee642c6528e88224f12409a5f23060994eschrock#include <fm/fmd_api.h>
99653d4ee642c6528e88224f12409a5f23060994eschrock#include <sys/fs/zfs.h>
99653d4ee642c6528e88224f12409a5f23060994eschrock#include <sys/fm/protocol.h>
99653d4ee642c6528e88224f12409a5f23060994eschrock#include <sys/fm/fs/zfs.h>
99653d4ee642c6528e88224f12409a5f23060994eschrock#include <libzfs.h>
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock#include <fm/libtopo.h>
3d7072f8bd27709dba14f6fe336f149d25d9e207eschrock#include <string.h>
99653d4ee642c6528e88224f12409a5f23060994eschrock
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrocktypedef struct zfs_retire_repaired {
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock struct zfs_retire_repaired *zrr_next;
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock uint64_t zrr_pool;
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock uint64_t zrr_vdev;
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock} zfs_retire_repaired_t;
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrocktypedef struct zfs_retire_data {
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock libzfs_handle_t *zrd_hdl;
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock zfs_retire_repaired_t *zrd_repaired;
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock} zfs_retire_data_t;
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrockstatic void
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrockzfs_retire_clear_data(fmd_hdl_t *hdl, zfs_retire_data_t *zdp)
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock{
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock zfs_retire_repaired_t *zrp;
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock while ((zrp = zdp->zrd_repaired) != NULL) {
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock zdp->zrd_repaired = zrp->zrr_next;
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock fmd_hdl_free(hdl, zrp, sizeof (zfs_retire_repaired_t));
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock }
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock}
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock
99653d4ee642c6528e88224f12409a5f23060994eschrock/*
99653d4ee642c6528e88224f12409a5f23060994eschrock * Find a pool with a matching GUID.
99653d4ee642c6528e88224f12409a5f23060994eschrock */
99653d4ee642c6528e88224f12409a5f23060994eschrocktypedef struct find_cbdata {
99653d4ee642c6528e88224f12409a5f23060994eschrock uint64_t cb_guid;
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock const char *cb_fru;
99653d4ee642c6528e88224f12409a5f23060994eschrock zpool_handle_t *cb_zhp;
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock nvlist_t *cb_vdev;
99653d4ee642c6528e88224f12409a5f23060994eschrock} find_cbdata_t;
99653d4ee642c6528e88224f12409a5f23060994eschrock
99653d4ee642c6528e88224f12409a5f23060994eschrockstatic int
99653d4ee642c6528e88224f12409a5f23060994eschrockfind_pool(zpool_handle_t *zhp, void *data)
99653d4ee642c6528e88224f12409a5f23060994eschrock{
99653d4ee642c6528e88224f12409a5f23060994eschrock find_cbdata_t *cbp = data;
99653d4ee642c6528e88224f12409a5f23060994eschrock
990b4856d0eaada6f8140335733a1b1771ed2746lling if (cbp->cb_guid ==
990b4856d0eaada6f8140335733a1b1771ed2746lling zpool_get_prop_int(zhp, ZPOOL_PROP_GUID, NULL)) {
99653d4ee642c6528e88224f12409a5f23060994eschrock cbp->cb_zhp = zhp;
99653d4ee642c6528e88224f12409a5f23060994eschrock return (1);
99653d4ee642c6528e88224f12409a5f23060994eschrock }
99653d4ee642c6528e88224f12409a5f23060994eschrock
99653d4ee642c6528e88224f12409a5f23060994eschrock zpool_close(zhp);
99653d4ee642c6528e88224f12409a5f23060994eschrock return (0);
99653d4ee642c6528e88224f12409a5f23060994eschrock}
99653d4ee642c6528e88224f12409a5f23060994eschrock
99653d4ee642c6528e88224f12409a5f23060994eschrock/*
99653d4ee642c6528e88224f12409a5f23060994eschrock * Find a vdev within a tree with a matching GUID.
99653d4ee642c6528e88224f12409a5f23060994eschrock */
99653d4ee642c6528e88224f12409a5f23060994eschrockstatic nvlist_t *
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrockfind_vdev(libzfs_handle_t *zhdl, nvlist_t *nv, const char *search_fru,
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock uint64_t search_guid)
99653d4ee642c6528e88224f12409a5f23060994eschrock{
99653d4ee642c6528e88224f12409a5f23060994eschrock uint64_t guid;
99653d4ee642c6528e88224f12409a5f23060994eschrock nvlist_t **child;
99653d4ee642c6528e88224f12409a5f23060994eschrock uint_t c, children;
99653d4ee642c6528e88224f12409a5f23060994eschrock nvlist_t *ret;
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock char *fru;
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock if (search_fru != NULL) {
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock if (nvlist_lookup_string(nv, ZPOOL_CONFIG_FRU, &fru) == 0 &&
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock libzfs_fru_compare(zhdl, fru, search_fru))
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock return (nv);
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock } else {
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock if (nvlist_lookup_uint64(nv, ZPOOL_CONFIG_GUID, &guid) == 0 &&
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock guid == search_guid)
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock return (nv);
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock }
99653d4ee642c6528e88224f12409a5f23060994eschrock
99653d4ee642c6528e88224f12409a5f23060994eschrock if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_CHILDREN,
99653d4ee642c6528e88224f12409a5f23060994eschrock &child, &children) != 0)
99653d4ee642c6528e88224f12409a5f23060994eschrock return (NULL);
99653d4ee642c6528e88224f12409a5f23060994eschrock
99653d4ee642c6528e88224f12409a5f23060994eschrock for (c = 0; c < children; c++) {
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock if ((ret = find_vdev(zhdl, child[c], search_fru,
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock search_guid)) != NULL)
99653d4ee642c6528e88224f12409a5f23060994eschrock return (ret);
99653d4ee642c6528e88224f12409a5f23060994eschrock }
99653d4ee642c6528e88224f12409a5f23060994eschrock
c5904d138f3bdf0762dbf452a43d5a5c387ea6a8eschrock if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_L2CACHE,
c5904d138f3bdf0762dbf452a43d5a5c387ea6a8eschrock &child, &children) != 0)
c5904d138f3bdf0762dbf452a43d5a5c387ea6a8eschrock return (NULL);
c5904d138f3bdf0762dbf452a43d5a5c387ea6a8eschrock
c5904d138f3bdf0762dbf452a43d5a5c387ea6a8eschrock for (c = 0; c < children; c++) {
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock if ((ret = find_vdev(zhdl, child[c], search_fru,
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock search_guid)) != NULL)
c5904d138f3bdf0762dbf452a43d5a5c387ea6a8eschrock return (ret);
c5904d138f3bdf0762dbf452a43d5a5c387ea6a8eschrock }
c5904d138f3bdf0762dbf452a43d5a5c387ea6a8eschrock
99653d4ee642c6528e88224f12409a5f23060994eschrock return (NULL);
99653d4ee642c6528e88224f12409a5f23060994eschrock}
99653d4ee642c6528e88224f12409a5f23060994eschrock
3d7072f8bd27709dba14f6fe336f149d25d9e207eschrock/*
3d7072f8bd27709dba14f6fe336f149d25d9e207eschrock * Given a (pool, vdev) GUID pair, find the matching pool and vdev.
3d7072f8bd27709dba14f6fe336f149d25d9e207eschrock */
3d7072f8bd27709dba14f6fe336f149d25d9e207eschrockstatic zpool_handle_t *
3d7072f8bd27709dba14f6fe336f149d25d9e207eschrockfind_by_guid(libzfs_handle_t *zhdl, uint64_t pool_guid, uint64_t vdev_guid,
3d7072f8bd27709dba14f6fe336f149d25d9e207eschrock nvlist_t **vdevp)
3d7072f8bd27709dba14f6fe336f149d25d9e207eschrock{
3d7072f8bd27709dba14f6fe336f149d25d9e207eschrock find_cbdata_t cb;
3d7072f8bd27709dba14f6fe336f149d25d9e207eschrock zpool_handle_t *zhp;
3d7072f8bd27709dba14f6fe336f149d25d9e207eschrock nvlist_t *config, *nvroot;
3d7072f8bd27709dba14f6fe336f149d25d9e207eschrock
3d7072f8bd27709dba14f6fe336f149d25d9e207eschrock /*
3d7072f8bd27709dba14f6fe336f149d25d9e207eschrock * Find the corresponding pool and make sure the vdev still exists.
3d7072f8bd27709dba14f6fe336f149d25d9e207eschrock */
3d7072f8bd27709dba14f6fe336f149d25d9e207eschrock cb.cb_guid = pool_guid;
3d7072f8bd27709dba14f6fe336f149d25d9e207eschrock if (zpool_iter(zhdl, find_pool, &cb) != 1)
3d7072f8bd27709dba14f6fe336f149d25d9e207eschrock return (NULL);
3d7072f8bd27709dba14f6fe336f149d25d9e207eschrock
3d7072f8bd27709dba14f6fe336f149d25d9e207eschrock zhp = cb.cb_zhp;
3d7072f8bd27709dba14f6fe336f149d25d9e207eschrock config = zpool_get_config(zhp, NULL);
3d7072f8bd27709dba14f6fe336f149d25d9e207eschrock if (nvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE,
3d7072f8bd27709dba14f6fe336f149d25d9e207eschrock &nvroot) != 0) {
3d7072f8bd27709dba14f6fe336f149d25d9e207eschrock zpool_close(zhp);
3d7072f8bd27709dba14f6fe336f149d25d9e207eschrock return (NULL);
3d7072f8bd27709dba14f6fe336f149d25d9e207eschrock }
3d7072f8bd27709dba14f6fe336f149d25d9e207eschrock
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock if (vdev_guid != 0) {
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock if ((*vdevp = find_vdev(zhdl, nvroot, NULL,
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock vdev_guid)) == NULL) {
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock zpool_close(zhp);
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock return (NULL);
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock }
3d7072f8bd27709dba14f6fe336f149d25d9e207eschrock }
3d7072f8bd27709dba14f6fe336f149d25d9e207eschrock
3d7072f8bd27709dba14f6fe336f149d25d9e207eschrock return (zhp);
3d7072f8bd27709dba14f6fe336f149d25d9e207eschrock}
3d7072f8bd27709dba14f6fe336f149d25d9e207eschrock
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrockstatic int
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrocksearch_pool(zpool_handle_t *zhp, void *data)
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock{
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock find_cbdata_t *cbp = data;
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock nvlist_t *config;
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock nvlist_t *nvroot;
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock config = zpool_get_config(zhp, NULL);
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock if (nvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE,
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock &nvroot) != 0) {
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock zpool_close(zhp);
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock return (0);
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock }
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock if ((cbp->cb_vdev = find_vdev(zpool_get_handle(zhp), nvroot,
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock cbp->cb_fru, 0)) != NULL) {
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock cbp->cb_zhp = zhp;
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock return (1);
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock }
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock zpool_close(zhp);
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock return (0);
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock}
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock/*
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock * Given a FRU FMRI, find the matching pool and vdev.
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock */
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrockstatic zpool_handle_t *
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrockfind_by_fru(libzfs_handle_t *zhdl, const char *fru, nvlist_t **vdevp)
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock{
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock find_cbdata_t cb;
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock cb.cb_fru = fru;
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock cb.cb_zhp = NULL;
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock if (zpool_iter(zhdl, search_pool, &cb) != 1)
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock return (NULL);
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock *vdevp = cb.cb_vdev;
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock return (cb.cb_zhp);
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock}
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock
3d7072f8bd27709dba14f6fe336f149d25d9e207eschrock/*
3d7072f8bd27709dba14f6fe336f149d25d9e207eschrock * Given a vdev, attempt to replace it with every known spare until one
3d7072f8bd27709dba14f6fe336f149d25d9e207eschrock * succeeds.
3d7072f8bd27709dba14f6fe336f149d25d9e207eschrock */
3d7072f8bd27709dba14f6fe336f149d25d9e207eschrockstatic void
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrockreplace_with_spare(fmd_hdl_t *hdl, zpool_handle_t *zhp, nvlist_t *vdev)
3d7072f8bd27709dba14f6fe336f149d25d9e207eschrock{
3d7072f8bd27709dba14f6fe336f149d25d9e207eschrock nvlist_t *config, *nvroot, *replacement;
3d7072f8bd27709dba14f6fe336f149d25d9e207eschrock nvlist_t **spares;
3d7072f8bd27709dba14f6fe336f149d25d9e207eschrock uint_t s, nspares;
3d7072f8bd27709dba14f6fe336f149d25d9e207eschrock char *dev_name;
3d7072f8bd27709dba14f6fe336f149d25d9e207eschrock
3d7072f8bd27709dba14f6fe336f149d25d9e207eschrock config = zpool_get_config(zhp, NULL);
3d7072f8bd27709dba14f6fe336f149d25d9e207eschrock if (nvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE,
3d7072f8bd27709dba14f6fe336f149d25d9e207eschrock &nvroot) != 0)
3d7072f8bd27709dba14f6fe336f149d25d9e207eschrock return;
3d7072f8bd27709dba14f6fe336f149d25d9e207eschrock
3d7072f8bd27709dba14f6fe336f149d25d9e207eschrock /*
3d7072f8bd27709dba14f6fe336f149d25d9e207eschrock * Find out if there are any hot spares available in the pool.
3d7072f8bd27709dba14f6fe336f149d25d9e207eschrock */
3d7072f8bd27709dba14f6fe336f149d25d9e207eschrock if (nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_SPARES,
3d7072f8bd27709dba14f6fe336f149d25d9e207eschrock &spares, &nspares) != 0)
3d7072f8bd27709dba14f6fe336f149d25d9e207eschrock return;
3d7072f8bd27709dba14f6fe336f149d25d9e207eschrock
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock replacement = fmd_nvl_alloc(hdl, FMD_SLEEP);
3d7072f8bd27709dba14f6fe336f149d25d9e207eschrock
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock (void) nvlist_add_string(replacement, ZPOOL_CONFIG_TYPE,
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock VDEV_TYPE_ROOT);
3d7072f8bd27709dba14f6fe336f149d25d9e207eschrock
88ecc943b4eb72f7c4fbbd8435997b85ef171fc3George Wilson dev_name = zpool_vdev_name(NULL, zhp, vdev, B_FALSE);
3d7072f8bd27709dba14f6fe336f149d25d9e207eschrock
3d7072f8bd27709dba14f6fe336f149d25d9e207eschrock /*
3d7072f8bd27709dba14f6fe336f149d25d9e207eschrock * Try to replace each spare, ending when we successfully
3d7072f8bd27709dba14f6fe336f149d25d9e207eschrock * replace it.
3d7072f8bd27709dba14f6fe336f149d25d9e207eschrock */
3d7072f8bd27709dba14f6fe336f149d25d9e207eschrock for (s = 0; s < nspares; s++) {
3d7072f8bd27709dba14f6fe336f149d25d9e207eschrock char *spare_name;
3d7072f8bd27709dba14f6fe336f149d25d9e207eschrock
3d7072f8bd27709dba14f6fe336f149d25d9e207eschrock if (nvlist_lookup_string(spares[s], ZPOOL_CONFIG_PATH,
3d7072f8bd27709dba14f6fe336f149d25d9e207eschrock &spare_name) != 0)
3d7072f8bd27709dba14f6fe336f149d25d9e207eschrock continue;
3d7072f8bd27709dba14f6fe336f149d25d9e207eschrock
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock (void) nvlist_add_nvlist_array(replacement,
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock ZPOOL_CONFIG_CHILDREN, &spares[s], 1);
3d7072f8bd27709dba14f6fe336f149d25d9e207eschrock
3d7072f8bd27709dba14f6fe336f149d25d9e207eschrock if (zpool_vdev_attach(zhp, dev_name, spare_name,
3d7072f8bd27709dba14f6fe336f149d25d9e207eschrock replacement, B_TRUE) == 0)
3d7072f8bd27709dba14f6fe336f149d25d9e207eschrock break;
3d7072f8bd27709dba14f6fe336f149d25d9e207eschrock }
3d7072f8bd27709dba14f6fe336f149d25d9e207eschrock
3d7072f8bd27709dba14f6fe336f149d25d9e207eschrock free(dev_name);
3d7072f8bd27709dba14f6fe336f149d25d9e207eschrock nvlist_free(replacement);
3d7072f8bd27709dba14f6fe336f149d25d9e207eschrock}
3d7072f8bd27709dba14f6fe336f149d25d9e207eschrock
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock/*
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock * Repair this vdev if we had diagnosed a 'fault.fs.zfs.device' and
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock * ASRU is now usable. ZFS has found the device to be present and
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock * functioning.
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock */
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock/*ARGSUSED*/
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrockvoid
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrockzfs_vdev_repair(fmd_hdl_t *hdl, nvlist_t *nvl)
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock{
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock zfs_retire_data_t *zdp = fmd_hdl_getspecific(hdl);
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock zfs_retire_repaired_t *zrp;
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock uint64_t pool_guid, vdev_guid;
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock nvlist_t *asru;
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock if (nvlist_lookup_uint64(nvl, FM_EREPORT_PAYLOAD_ZFS_POOL_GUID,
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock &pool_guid) != 0 || nvlist_lookup_uint64(nvl,
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock FM_EREPORT_PAYLOAD_ZFS_VDEV_GUID, &vdev_guid) != 0)
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock return;
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock /*
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock * Before checking the state of the ASRU, go through and see if we've
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock * already made an attempt to repair this ASRU. This list is cleared
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock * whenever we receive any kind of list event, and is designed to
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock * prevent us from generating a feedback loop when we attempt repairs
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock * against a faulted pool. The problem is that checking the unusable
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock * state of the ASRU can involve opening the pool, which can post
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock * statechange events but otherwise leave the pool in the faulted
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock * state. This list allows us to detect when a statechange event is
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock * due to our own request.
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock */
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock for (zrp = zdp->zrd_repaired; zrp != NULL; zrp = zrp->zrr_next) {
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock if (zrp->zrr_pool == pool_guid &&
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock zrp->zrr_vdev == vdev_guid)
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock return;
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock }
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock asru = fmd_nvl_alloc(hdl, FMD_SLEEP);
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock (void) nvlist_add_uint8(asru, FM_VERSION, ZFS_SCHEME_VERSION0);
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock (void) nvlist_add_string(asru, FM_FMRI_SCHEME, FM_FMRI_SCHEME_ZFS);
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock (void) nvlist_add_uint64(asru, FM_FMRI_ZFS_POOL, pool_guid);
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock (void) nvlist_add_uint64(asru, FM_FMRI_ZFS_VDEV, vdev_guid);
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock /*
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock * We explicitly check for the unusable state here to make sure we
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock * aren't responding to a transient state change. As part of opening a
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock * vdev, it's possible to see the 'statechange' event, only to be
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock * followed by a vdev failure later. If we don't check the current
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock * state of the vdev (or pool) before marking it repaired, then we risk
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock * generating spurious repair events followed immediately by the same
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock * diagnosis.
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock *
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock * This assumes that the ZFS scheme code associated unusable (i.e.
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock * isolated) with its own definition of faulty state. In the case of a
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock * DEGRADED leaf vdev (due to checksum errors), this is not the case.
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock * This works, however, because the transient state change is not
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock * posted in this case. This could be made more explicit by not
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock * relying on the scheme's unusable callback and instead directly
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock * checking the vdev state, where we could correctly account for
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock * DEGRADED state.
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock */
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock if (!fmd_nvl_fmri_unusable(hdl, asru) && fmd_nvl_fmri_has_fault(hdl,
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock asru, FMD_HAS_FAULT_ASRU, NULL)) {
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock topo_hdl_t *thp;
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock char *fmri = NULL;
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock int err;
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock thp = fmd_hdl_topo_hold(hdl, TOPO_VERSION);
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock if (topo_fmri_nvl2str(thp, asru, &fmri, &err) == 0)
eb04386ee9658c0271a7af93e0595582e2235689Eric Schrock (void) fmd_repair_asru(hdl, fmri);
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock fmd_hdl_topo_rele(hdl, thp);
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock topo_hdl_strfree(thp, fmri);
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock }
2a417b235e35bd992c3c76d3eb89d35467286133Robert Johnston nvlist_free(asru);
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock zrp = fmd_hdl_alloc(hdl, sizeof (zfs_retire_repaired_t), FMD_SLEEP);
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock zrp->zrr_next = zdp->zrd_repaired;
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock zrp->zrr_pool = pool_guid;
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock zrp->zrr_vdev = vdev_guid;
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock zdp->zrd_repaired = zrp;
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock}
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock
99653d4ee642c6528e88224f12409a5f23060994eschrock/*ARGSUSED*/
99653d4ee642c6528e88224f12409a5f23060994eschrockstatic void
99653d4ee642c6528e88224f12409a5f23060994eschrockzfs_retire_recv(fmd_hdl_t *hdl, fmd_event_t *ep, nvlist_t *nvl,
99653d4ee642c6528e88224f12409a5f23060994eschrock const char *class)
99653d4ee642c6528e88224f12409a5f23060994eschrock{
99653d4ee642c6528e88224f12409a5f23060994eschrock uint64_t pool_guid, vdev_guid;
99653d4ee642c6528e88224f12409a5f23060994eschrock zpool_handle_t *zhp;
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock nvlist_t *resource, *fault, *fru;
3d7072f8bd27709dba14f6fe336f149d25d9e207eschrock nvlist_t **faults;
3d7072f8bd27709dba14f6fe336f149d25d9e207eschrock uint_t f, nfaults;
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock zfs_retire_data_t *zdp = fmd_hdl_getspecific(hdl);
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock libzfs_handle_t *zhdl = zdp->zrd_hdl;
3d7072f8bd27709dba14f6fe336f149d25d9e207eschrock boolean_t fault_device, degrade_device;
3d7072f8bd27709dba14f6fe336f149d25d9e207eschrock boolean_t is_repair;
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock char *scheme, *fmri;
3d7072f8bd27709dba14f6fe336f149d25d9e207eschrock nvlist_t *vdev;
25c6ff4b77fcddf4097ce78a8277275ca603b46cstephh char *uuid;
25c6ff4b77fcddf4097ce78a8277275ca603b46cstephh int repair_done = 0;
cbf75e67acb6c32a2f4884f28a839d59f7988d37Stephen Hanson boolean_t retire;
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock boolean_t is_disk;
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock vdev_aux_t aux;
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock topo_hdl_t *thp;
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock int err;
3d7072f8bd27709dba14f6fe336f149d25d9e207eschrock
3d7072f8bd27709dba14f6fe336f149d25d9e207eschrock /*
3d7072f8bd27709dba14f6fe336f149d25d9e207eschrock * If this is a resource notifying us of device removal, then simply
3d7072f8bd27709dba14f6fe336f149d25d9e207eschrock * check for an available spare and continue.
3d7072f8bd27709dba14f6fe336f149d25d9e207eschrock */
3d7072f8bd27709dba14f6fe336f149d25d9e207eschrock if (strcmp(class, "resource.fs.zfs.removed") == 0) {
3d7072f8bd27709dba14f6fe336f149d25d9e207eschrock if (nvlist_lookup_uint64(nvl, FM_EREPORT_PAYLOAD_ZFS_POOL_GUID,
3d7072f8bd27709dba14f6fe336f149d25d9e207eschrock &pool_guid) != 0 ||
3d7072f8bd27709dba14f6fe336f149d25d9e207eschrock nvlist_lookup_uint64(nvl, FM_EREPORT_PAYLOAD_ZFS_VDEV_GUID,
3d7072f8bd27709dba14f6fe336f149d25d9e207eschrock &vdev_guid) != 0)
3d7072f8bd27709dba14f6fe336f149d25d9e207eschrock return;
3d7072f8bd27709dba14f6fe336f149d25d9e207eschrock
3d7072f8bd27709dba14f6fe336f149d25d9e207eschrock if ((zhp = find_by_guid(zhdl, pool_guid, vdev_guid,
3d7072f8bd27709dba14f6fe336f149d25d9e207eschrock &vdev)) == NULL)
3d7072f8bd27709dba14f6fe336f149d25d9e207eschrock return;
3d7072f8bd27709dba14f6fe336f149d25d9e207eschrock
3d7072f8bd27709dba14f6fe336f149d25d9e207eschrock if (fmd_prop_get_int32(hdl, "spare_on_remove"))
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock replace_with_spare(hdl, zhp, vdev);
3d7072f8bd27709dba14f6fe336f149d25d9e207eschrock zpool_close(zhp);
3d7072f8bd27709dba14f6fe336f149d25d9e207eschrock return;
3d7072f8bd27709dba14f6fe336f149d25d9e207eschrock }
3d7072f8bd27709dba14f6fe336f149d25d9e207eschrock
cbf75e67acb6c32a2f4884f28a839d59f7988d37Stephen Hanson if (strcmp(class, FM_LIST_RESOLVED_CLASS) == 0)
cbf75e67acb6c32a2f4884f28a839d59f7988d37Stephen Hanson return;
cbf75e67acb6c32a2f4884f28a839d59f7988d37Stephen Hanson
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock if (strcmp(class, "resource.fs.zfs.statechange") == 0 ||
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock strcmp(class,
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock "resource.sysevent.EC_zfs.ESC_ZFS_vdev_remove") == 0) {
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock zfs_vdev_repair(hdl, nvl);
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock return;
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock }
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock zfs_retire_clear_data(hdl, zdp);
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock
25c6ff4b77fcddf4097ce78a8277275ca603b46cstephh if (strcmp(class, FM_LIST_REPAIRED_CLASS) == 0)
3d7072f8bd27709dba14f6fe336f149d25d9e207eschrock is_repair = B_TRUE;
3d7072f8bd27709dba14f6fe336f149d25d9e207eschrock else
3d7072f8bd27709dba14f6fe336f149d25d9e207eschrock is_repair = B_FALSE;
99653d4ee642c6528e88224f12409a5f23060994eschrock
99653d4ee642c6528e88224f12409a5f23060994eschrock /*
3d7072f8bd27709dba14f6fe336f149d25d9e207eschrock * We subscribe to zfs faults as well as all repair events.
99653d4ee642c6528e88224f12409a5f23060994eschrock */
99653d4ee642c6528e88224f12409a5f23060994eschrock if (nvlist_lookup_nvlist_array(nvl, FM_SUSPECT_FAULT_LIST,
99653d4ee642c6528e88224f12409a5f23060994eschrock &faults, &nfaults) != 0)
99653d4ee642c6528e88224f12409a5f23060994eschrock return;
99653d4ee642c6528e88224f12409a5f23060994eschrock
99653d4ee642c6528e88224f12409a5f23060994eschrock for (f = 0; f < nfaults; f++) {
3d7072f8bd27709dba14f6fe336f149d25d9e207eschrock fault = faults[f];
3d7072f8bd27709dba14f6fe336f149d25d9e207eschrock
3d7072f8bd27709dba14f6fe336f149d25d9e207eschrock fault_device = B_FALSE;
3d7072f8bd27709dba14f6fe336f149d25d9e207eschrock degrade_device = B_FALSE;
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock is_disk = B_FALSE;
99653d4ee642c6528e88224f12409a5f23060994eschrock
cbf75e67acb6c32a2f4884f28a839d59f7988d37Stephen Hanson if (nvlist_lookup_boolean_value(fault, FM_SUSPECT_RETIRE,
cbf75e67acb6c32a2f4884f28a839d59f7988d37Stephen Hanson &retire) == 0 && retire == 0)
cbf75e67acb6c32a2f4884f28a839d59f7988d37Stephen Hanson continue;
cbf75e67acb6c32a2f4884f28a839d59f7988d37Stephen Hanson
99653d4ee642c6528e88224f12409a5f23060994eschrock /*
3d7072f8bd27709dba14f6fe336f149d25d9e207eschrock * While we subscribe to fault.fs.zfs.*, we only take action
3d7072f8bd27709dba14f6fe336f149d25d9e207eschrock * for faults targeting a specific vdev (open failure or SERD
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock * failure). We also subscribe to fault.io.* events, so that
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock * faulty disks will be faulted in the ZFS configuration.
99653d4ee642c6528e88224f12409a5f23060994eschrock */
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock if (fmd_nvl_class_match(hdl, fault, "fault.fs.zfs.vdev.io")) {
3d7072f8bd27709dba14f6fe336f149d25d9e207eschrock fault_device = B_TRUE;
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock } else if (fmd_nvl_class_match(hdl, fault,
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock "fault.fs.zfs.vdev.checksum")) {
3d7072f8bd27709dba14f6fe336f149d25d9e207eschrock degrade_device = B_TRUE;
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock } else if (fmd_nvl_class_match(hdl, fault,
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock "fault.fs.zfs.device")) {
3d7072f8bd27709dba14f6fe336f149d25d9e207eschrock fault_device = B_FALSE;
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock } else if (fmd_nvl_class_match(hdl, fault, "fault.io.*")) {
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock is_disk = B_TRUE;
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock fault_device = B_TRUE;
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock } else {
99653d4ee642c6528e88224f12409a5f23060994eschrock continue;
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock }
99653d4ee642c6528e88224f12409a5f23060994eschrock
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock if (is_disk) {
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock /*
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock * This is a disk fault. Lookup the FRU, convert it to
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock * an FMRI string, and attempt to find a matching vdev.
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock */
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock if (nvlist_lookup_nvlist(fault, FM_FAULT_FRU,
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock &fru) != 0 ||
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock nvlist_lookup_string(fru, FM_FMRI_SCHEME,
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock &scheme) != 0)
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock continue;
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock if (strcmp(scheme, FM_FMRI_SCHEME_HC) != 0)
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock continue;
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock thp = fmd_hdl_topo_hold(hdl, TOPO_VERSION);
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock if (topo_fmri_nvl2str(thp, fru, &fmri, &err) != 0) {
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock fmd_hdl_topo_rele(hdl, thp);
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock continue;
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock }
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock zhp = find_by_fru(zhdl, fmri, &vdev);
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock topo_hdl_strfree(thp, fmri);
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock fmd_hdl_topo_rele(hdl, thp);
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock if (zhp == NULL)
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock continue;
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock (void) nvlist_lookup_uint64(vdev,
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock ZPOOL_CONFIG_GUID, &vdev_guid);
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock aux = VDEV_AUX_EXTERNAL;
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock } else {
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock /*
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock * This is a ZFS fault. Lookup the resource, and
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock * attempt to find the matching vdev.
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock */
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock if (nvlist_lookup_nvlist(fault, FM_FAULT_RESOURCE,
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock &resource) != 0 ||
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock nvlist_lookup_string(resource, FM_FMRI_SCHEME,
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock &scheme) != 0)
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock continue;
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock if (strcmp(scheme, FM_FMRI_SCHEME_ZFS) != 0)
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock continue;
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock if (nvlist_lookup_uint64(resource, FM_FMRI_ZFS_POOL,
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock &pool_guid) != 0)
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock continue;
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock if (nvlist_lookup_uint64(resource, FM_FMRI_ZFS_VDEV,
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock &vdev_guid) != 0) {
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock if (is_repair)
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock vdev_guid = 0;
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock else
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock continue;
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock }
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock if ((zhp = find_by_guid(zhdl, pool_guid, vdev_guid,
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock &vdev)) == NULL)
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock continue;
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock aux = VDEV_AUX_ERR_EXCEEDED;
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock }
99653d4ee642c6528e88224f12409a5f23060994eschrock
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock if (vdev_guid == 0) {
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock /*
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock * For pool-level repair events, clear the entire pool.
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock */
468c413a79615e77179e8d98f22a7e513a8135bdTim Haley (void) zpool_clear(zhp, NULL, NULL);
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock zpool_close(zhp);
99653d4ee642c6528e88224f12409a5f23060994eschrock continue;
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock }
99653d4ee642c6528e88224f12409a5f23060994eschrock
3d7072f8bd27709dba14f6fe336f149d25d9e207eschrock /*
3d7072f8bd27709dba14f6fe336f149d25d9e207eschrock * If this is a repair event, then mark the vdev as repaired and
3d7072f8bd27709dba14f6fe336f149d25d9e207eschrock * continue.
3d7072f8bd27709dba14f6fe336f149d25d9e207eschrock */
3d7072f8bd27709dba14f6fe336f149d25d9e207eschrock if (is_repair) {
25c6ff4b77fcddf4097ce78a8277275ca603b46cstephh repair_done = 1;
3d7072f8bd27709dba14f6fe336f149d25d9e207eschrock (void) zpool_vdev_clear(zhp, vdev_guid);
99653d4ee642c6528e88224f12409a5f23060994eschrock zpool_close(zhp);
99653d4ee642c6528e88224f12409a5f23060994eschrock continue;
99653d4ee642c6528e88224f12409a5f23060994eschrock }
99653d4ee642c6528e88224f12409a5f23060994eschrock
99653d4ee642c6528e88224f12409a5f23060994eschrock /*
3d7072f8bd27709dba14f6fe336f149d25d9e207eschrock * Actively fault the device if needed.
99653d4ee642c6528e88224f12409a5f23060994eschrock */
3d7072f8bd27709dba14f6fe336f149d25d9e207eschrock if (fault_device)
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock (void) zpool_vdev_fault(zhp, vdev_guid, aux);
3d7072f8bd27709dba14f6fe336f149d25d9e207eschrock if (degrade_device)
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock (void) zpool_vdev_degrade(zhp, vdev_guid, aux);
99653d4ee642c6528e88224f12409a5f23060994eschrock
3d7072f8bd27709dba14f6fe336f149d25d9e207eschrock /*
3d7072f8bd27709dba14f6fe336f149d25d9e207eschrock * Attempt to substitute a hot spare.
3d7072f8bd27709dba14f6fe336f149d25d9e207eschrock */
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock replace_with_spare(hdl, zhp, vdev);
99653d4ee642c6528e88224f12409a5f23060994eschrock zpool_close(zhp);
99653d4ee642c6528e88224f12409a5f23060994eschrock }
25c6ff4b77fcddf4097ce78a8277275ca603b46cstephh
25c6ff4b77fcddf4097ce78a8277275ca603b46cstephh if (strcmp(class, FM_LIST_REPAIRED_CLASS) == 0 && repair_done &&
25c6ff4b77fcddf4097ce78a8277275ca603b46cstephh nvlist_lookup_string(nvl, FM_SUSPECT_UUID, &uuid) == 0)
25c6ff4b77fcddf4097ce78a8277275ca603b46cstephh fmd_case_uuresolved(hdl, uuid);
99653d4ee642c6528e88224f12409a5f23060994eschrock}
99653d4ee642c6528e88224f12409a5f23060994eschrock
99653d4ee642c6528e88224f12409a5f23060994eschrockstatic const fmd_hdl_ops_t fmd_ops = {
99653d4ee642c6528e88224f12409a5f23060994eschrock zfs_retire_recv, /* fmdo_recv */
99653d4ee642c6528e88224f12409a5f23060994eschrock NULL, /* fmdo_timeout */
99653d4ee642c6528e88224f12409a5f23060994eschrock NULL, /* fmdo_close */
99653d4ee642c6528e88224f12409a5f23060994eschrock NULL, /* fmdo_stats */
99653d4ee642c6528e88224f12409a5f23060994eschrock NULL, /* fmdo_gc */
99653d4ee642c6528e88224f12409a5f23060994eschrock};
99653d4ee642c6528e88224f12409a5f23060994eschrock
99653d4ee642c6528e88224f12409a5f23060994eschrockstatic const fmd_prop_t fmd_props[] = {
3d7072f8bd27709dba14f6fe336f149d25d9e207eschrock { "spare_on_remove", FMD_TYPE_BOOL, "true" },
99653d4ee642c6528e88224f12409a5f23060994eschrock { NULL, 0, NULL }
99653d4ee642c6528e88224f12409a5f23060994eschrock};
99653d4ee642c6528e88224f12409a5f23060994eschrock
99653d4ee642c6528e88224f12409a5f23060994eschrockstatic const fmd_hdl_info_t fmd_info = {
99653d4ee642c6528e88224f12409a5f23060994eschrock "ZFS Retire Agent", "1.0", &fmd_ops, fmd_props
99653d4ee642c6528e88224f12409a5f23060994eschrock};
99653d4ee642c6528e88224f12409a5f23060994eschrock
99653d4ee642c6528e88224f12409a5f23060994eschrockvoid
99653d4ee642c6528e88224f12409a5f23060994eschrock_fmd_init(fmd_hdl_t *hdl)
99653d4ee642c6528e88224f12409a5f23060994eschrock{
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock zfs_retire_data_t *zdp;
99653d4ee642c6528e88224f12409a5f23060994eschrock libzfs_handle_t *zhdl;
99653d4ee642c6528e88224f12409a5f23060994eschrock
99653d4ee642c6528e88224f12409a5f23060994eschrock if ((zhdl = libzfs_init()) == NULL)
99653d4ee642c6528e88224f12409a5f23060994eschrock return;
99653d4ee642c6528e88224f12409a5f23060994eschrock
99653d4ee642c6528e88224f12409a5f23060994eschrock if (fmd_hdl_register(hdl, FMD_API_VERSION, &fmd_info) != 0) {
99653d4ee642c6528e88224f12409a5f23060994eschrock libzfs_fini(zhdl);
99653d4ee642c6528e88224f12409a5f23060994eschrock return;
99653d4ee642c6528e88224f12409a5f23060994eschrock }
99653d4ee642c6528e88224f12409a5f23060994eschrock
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock zdp = fmd_hdl_zalloc(hdl, sizeof (zfs_retire_data_t), FMD_SLEEP);
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock zdp->zrd_hdl = zhdl;
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock fmd_hdl_setspecific(hdl, zdp);
99653d4ee642c6528e88224f12409a5f23060994eschrock}
99653d4ee642c6528e88224f12409a5f23060994eschrock
99653d4ee642c6528e88224f12409a5f23060994eschrockvoid
99653d4ee642c6528e88224f12409a5f23060994eschrock_fmd_fini(fmd_hdl_t *hdl)
99653d4ee642c6528e88224f12409a5f23060994eschrock{
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock zfs_retire_data_t *zdp = fmd_hdl_getspecific(hdl);
99653d4ee642c6528e88224f12409a5f23060994eschrock
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock if (zdp != NULL) {
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock zfs_retire_clear_data(hdl, zdp);
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock libzfs_fini(zdp->zrd_hdl);
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock fmd_hdl_free(hdl, zdp, sizeof (zfs_retire_data_t));
069f55e237020c4a4907b235fc38fafc6442ce94Eric Schrock }
99653d4ee642c6528e88224f12409a5f23060994eschrock}