fs/ufs/ufs_panic.c

	ufs_panic.c revision d3d50737e566cade9a08d73d2af95105ac7cd960
/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*
 * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

#include <sys/types.h>
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/errno.h>
#include <sys/mode.h>
#include <sys/sysmacros.h>
#include <sys/cmn_err.h>
#include <sys/varargs.h>
#include <sys/time.h>
#include <sys/buf.h>
#include <sys/kmem.h>
#include <sys/t_lock.h>
#include <sys/poll.h>
#include <sys/debug.h>
#include <sys/cred.h>
#include <sys/lockfs.h>
#include <sys/fs/ufs_fs.h>
#include <sys/fs/ufs_inode.h>
#include <sys/fs/ufs_panic.h>
#include <sys/fs/ufs_lockfs.h>
#include <sys/fs/ufs_trans.h>
#include <sys/fs/ufs_mount.h>
#include <sys/fs/ufs_prot.h>
#include <sys/fs/ufs_bio.h>
#include <sys/pathname.h>
#include <sys/utsname.h>
#include <sys/conf.h>

/* handy */
#define abs(x)      ((x) < 0? -(x): (x))

#if defined(DEBUG)

#define DBGLVL_NONE 0x00000000
#define DBGLVL_MAJOR    0x00000100
#define DBGLVL_MINOR    0x00000200
#define DBGLVL_MINUTE   0x00000400
#define DBGLVL_TRIVIA   0x00000800
#define DBGLVL_HIDEOUS  0x00001000

#define DBGFLG_NONE     0x00000000
#define DBGFLG_NOPANIC      0x00000001
#define DBGFLG_LVLONLY      0x00000002
#define DBGFLG_FIXWOULDPANIC    0x00000004

#define DBGFLG_FLAGMASK     0x0000000F
#define DBGFLG_LEVELMASK    ~DBGFLG_FLAGMASK

#define DEBUG_FLAGS (ufs_fix_failure_dbg & DBGFLG_FLAGMASK)
#define DEBUG_LEVEL (ufs_fix_failure_dbg & DBGFLG_LEVELMASK)

unsigned int ufs_fix_failure_dbg =  DBGLVL_NONE | DBGFLG_NONE;

#define DCALL(dbg_level, call)                      \
    {                               \
        if (DEBUG_LEVEL != DBGLVL_NONE) {           \
            if (DEBUG_FLAGS & DBGFLG_LVLONLY) {     \
                if (DEBUG_LEVEL & dbg_level) {      \
                    call;               \
                }                   \
            } else {                    \
                if (dbg_level <= DEBUG_LEVEL) {     \
                    call;               \
                }                   \
            }                       \
        }                           \
    }

#define DPRINTF(dbg_level, msg)     DCALL(dbg_level, printf msg)

#define MAJOR(msg)          DPRINTF(DBGLVL_MAJOR, msg)
#define MINOR(msg)          DPRINTF(DBGLVL_MINOR, msg)
#define MINUTE(msg)         DPRINTF(DBGLVL_MINUTE, msg)
#define TRIVIA(msg)         DPRINTF(DBGLVL_TRIVIA, msg)
#define HIDEOUS(msg)            DPRINTF(DBGLVL_HIDEOUS, msg)

#else   /* !DEBUG */

#define DCALL(ignored_dbg_level, ignored_routine)
#define MAJOR(ignored)
#define MINOR(ignored)
#define MINUTE(ignored)
#define TRIVIA(ignored)
#define HIDEOUS(ignored)

#endif /* DEBUG */

#define NULLSTR(str)    (!(str) || *(str) == '\0'? "<null>" : (str))
#define NULSTRING   ""

/* somewhat arbitrary limits, in seconds */
/* all probably ought to be different, but these are convenient for debugging */
const time_t    UF_TOO_LONG     = 128;  /* max. wait for fsck start */

/* all of these are in units of seconds used for retry period while ... */
const time_t    UF_FIXSTART_PERIOD  = 16;   /* awaiting fsck start */
const time_t    UF_FIXPOLL_PERIOD   = 256;  /* awaiting fsck finish */
const time_t    UF_SHORT_ERROR_PERIOD   = 4;    /* after (lockfs) error */
const time_t    UF_LONG_ERROR_PERIOD    = 512;  /* after (lockfs) error */

#define NO_ERROR        0
#define LOCKFS_OLOCK        LOCKFS_MAXLOCK+1

const ulong_t   GB          = 1024 * 1024 * 1024;
const ulong_t   SecondsPerGig       = 1024; /* ~17 minutes (overestimate) */

/*
 * per filesystem flags
 */
const int   UFSFX_PANIC     = (UFSMNT_ONERROR_PANIC >> 4);
const int   UFSFX_LCKONLY       = (UFSMNT_ONERROR_LOCK >> 4);
const int   UFSFX_LCKUMOUNT     = (UFSMNT_ONERROR_UMOUNT >> 4);
const int   UFSFX_DEFAULT       = (UFSMNT_ONERROR_DEFAULT >> 4);
const int   UFSFX_REPAIR_START  = 0x10000000;

/* return protocols */

typedef enum triage_return_code {
    TRIAGE_DEAD = -1,
    TRIAGE_NO_SPIRIT,
    TRIAGE_ATTEND_TO
} triage_t;

typedef enum statefunc_return_code {
    SFRC_SUCCESS = 1,
    SFRC_FAIL = 0
} sfrc_t;

/* external references */
/* in ufs_thread.c */
extern int  ufs_thread_run(struct ufs_q *, callb_cpr_t *cprinfop);
extern int  ufs_checkaccton(vnode_t *);     /* in ufs_lockfs.c */
extern int  ufs_checkswapon(vnode_t *);     /* in ufs_lockfs.c */

extern struct pollhead      ufs_pollhd;     /* in ufs_vnops.c */

/* globals */
struct  ufs_q    ufs_fix;

/*
 * patchable constants:
 *   These are set in ufsfx_init() [called at modload]
 */
struct ufs_failure_tunable {
    long     uft_too_long;      /* limit repair startup time */
    long     uft_fixstart_period;   /* pre-repair start period */
    long     uft_fixpoll_period;    /* post-fsck start period */
    long     uft_short_err_period;  /* post-error short period */
    long     uft_long_err_period;   /* post-error long period */
} ufsfx_tune;

/* internal statistics of events */
struct uf_statistics {
    ulong_t     ufst_lock_violations;
    ulong_t     ufst_current_races;
    ulong_t     ufst_unmount_failures;
    ulong_t     ufst_num_fixed;
    ulong_t     ufst_num_failed;
    ulong_t     ufst_cpu_waste;
    time_t      ufst_last_start_tm;
    kmutex_t    ufst_mutex;
} uf_stats;

typedef enum state_action {
    UFA_ERROR = -1,     /* internal error */
    UFA_FOUND,      /* found uf in state */
    UFA_SET         /* change uf to state */
} ufsa_t;

/* state definition */
typedef struct uf_state_desc {
    int   ud_v;                 /* value */
    char     *ud_name;              /* name */
    sfrc_t  (*ud_sfp)(ufs_failure_t *, ufsa_t, ufs_failure_states_t);
                            /* per-state actions */
    ufs_failure_states_t      ud_prev;      /* valid prev. states */

    struct uf_state_desc_attr {
        unsigned    terminal:1; /* no action req. if found */
        unsigned    at_fail:1;  /* state set by thread */
                        /* encountering the error */
        unsigned    unused;
    } ud_attr;
} ufsd_t;

/*
 * forward references
 */

/* thread to watch for failures */
static void ufsfx_thread_fix_failures(void *);
static int  ufsfx_do_failure_q(void);
static void ufsfx_kill_fix_failure_thread(void *);

/* routines called when failure occurs */
static int       ufs_fault_v(vnode_t *, char *, va_list)
    __KVPRINTFLIKE(2);
static ufs_failure_t    *init_failure(vnode_t *, char *, va_list)
    __KVPRINTFLIKE(2);
static void      queue_failure(ufs_failure_t *);
/*PRINTFLIKE2*/
static void      real_panic(ufs_failure_t *, const char *, ...)
    __KPRINTFLIKE(2);
static void      real_panic_v(ufs_failure_t *, const char *, va_list)
    __KVPRINTFLIKE(2);
static triage_t      triage(vnode_t *);

/* routines called when failure record is acted upon */
static sfrc_t   set_state(ufs_failure_t *, ufs_failure_states_t);
static int  state_trans_valid(ufs_failure_states_t, ufs_failure_states_t);
static int  terminal_state(ufs_failure_states_t);

/* routines called when states entered/found */
static sfrc_t   sf_minimum(ufs_failure_t *, ufsa_t, ufs_failure_states_t);
static sfrc_t   sf_undef(ufs_failure_t *, ufsa_t, ufs_failure_states_t);
static sfrc_t   sf_init(ufs_failure_t *, ufsa_t, ufs_failure_states_t);
static sfrc_t   sf_queue(ufs_failure_t *, ufsa_t, ufs_failure_states_t);
static sfrc_t   sf_found_queue(ufs_failure_t *);
static sfrc_t   sf_nonterm_cmn(ufs_failure_t *, ufsa_t, ufs_failure_states_t);
static sfrc_t   sf_term_cmn(ufs_failure_t *, ufsa_t, ufs_failure_states_t);
static sfrc_t   sf_panic(ufs_failure_t *, ufsa_t, ufs_failure_states_t);
static sfrc_t   sf_set_trylck(ufs_failure_t *);
static sfrc_t   sf_set_locked(ufs_failure_t *);
static sfrc_t   sf_found_trylck(ufs_failure_t *);
static sfrc_t   sf_found_lock_fix_cmn(ufs_failure_t *, ufs_failure_states_t);
static sfrc_t   sf_found_umount(ufs_failure_t *);

/* support routines, called by sf_nonterm_cmn and sf_term_cmn */
static time_t   trylock_time_exceeded(ufs_failure_t *);
static void     pester_msg(ufs_failure_t *, int);
static int  get_lockfs_status(ufs_failure_t *, struct lockfs *);
static void     alloc_lockfs_comment(ufs_failure_t *, struct lockfs *);
static int  set_lockfs(ufs_failure_t *, struct lockfs *);
static int  lockfs_failure(ufs_failure_t *);
static int  lockfs_success(ufs_failure_t *);
static int  fsck_active(ufs_failure_t *);

/* low-level support routines */
static ufsd_t   *get_state_desc(ufs_failure_states_t);
static char *fs_name(ufs_failure_t *);

#if defined(DEBUG)
static char *state_name(ufs_failure_states_t);
static char *lock_name(struct lockfs *);
static char *err_name(int);
static char *act_name(ufsa_t);
static void  dump_uf_list(char *msg);
static void  dump_uf(ufs_failure_t *, int i);
#endif /* DEBUG */
/*
 *
 * State Transitions:
 *
 * normally:
 * if flagged to be locked but not unmounted:   (UFSMNT_ONERROR_LOCK)
 *  UNDEF -> INIT -> QUEUE -> TRYLCK -> LOCKED -> FIXING -> FIXED
 *
 * The only difference between these two is that the fsck must be started
 * manually.
 *
 * if flagged to be unmounted:          (UFSMNT_ONERROR_UMOUNT)
 *  UNDEF -> INIT -> QUEUE -> TRYLCK -> LOCKED -> UMOUNT -> NOTFIX
 *
 * if flagged to panic:             (UFSMNT_ONERROR_PANIC)
 *  UNDEF -> INIT -> PANIC
 *
 * if a secondary panic on a file system which has an active failure
 * record:
 *  UNDEF -> INIT -> QUEUE -> REPLICA
 *
 * UNDEF, INIT, QUEUE all are set in the context of the failing thread.
 * All other states (except possibly PANIC) are set in by the monitor
 * (lock) thread.
 *
 */

ufsd_t  state_desc[] =
{
    { UF_ILLEGAL,   "in an unknown state",  sf_minimum, UF_ILLEGAL,
                                { 0, 1, 0 } },
    { UF_UNDEF, "undefined",        sf_undef,   UF_UNDEF,
                                { 0, 1, 0 } },
    { UF_INIT,  "being initialized",    sf_init,    UF_UNDEF,
                                { 0, 1, 0 } },
    { UF_QUEUE, "queued",       sf_queue,   UF_INIT,
                                { 0, 1, 0 } },
    { UF_TRYLCK,    "trying to be locked",  sf_nonterm_cmn,
                        UF_QUEUE,   { 0, 0, 0 } },
    { UF_LOCKED,    "locked",       sf_nonterm_cmn,
                    UF_TRYLCK | UF_FIXING,  { 0, 0, 0 } },
    { UF_UMOUNT,    "being unmounted",  sf_nonterm_cmn,

#if defined(DEBUG)
                    UF_PANIC |
#endif /* DEBUG */
                    UF_TRYLCK | UF_LOCKED,  { 0, 0, 0 } },
    { UF_FIXING,    "being fixed",      sf_nonterm_cmn,
                        UF_LOCKED,  { 0, 0, 0 } },
    { UF_FIXED, "fixed",        sf_term_cmn,
                        UF_FIXING,  { 1, 0, 0 } },
    { UF_NOTFIX,    "not fixed",        sf_term_cmn,

#if defined(DEBUG)
                            UF_PANIC |
#endif /* DEBUG */

        UF_QUEUE | UF_TRYLCK | UF_LOCKED | UF_UMOUNT | UF_FIXING,
                                { 1, 0, 0 } },
    { UF_REPLICA,   "a replica",        sf_term_cmn,
                        UF_QUEUE,   { 1, 0, 0 } },
    { UF_PANIC, "panicking",        sf_panic,
        /* XXX make this narrower */    UF_ALLSTATES,   { 0, 0, 0 } },
    { UF_UNDEF, NULL,           ((sfrc_t (*)()) NULL),
                        UF_UNDEF,   { 0, 0, 0 } }
};

/* unified collection */
struct ufsfx_info {
    struct uf_statistics        *ufi_statp;
    struct ufs_failure_tunable  *ufi_tunep;
    ufsd_t              *ufi_statetab;
} uffsinfo;

#if defined(DEBUG)
struct action_description {
    ufsa_t   ad_v;
    char    *ad_name;
};

#define EUNK        (-1)

struct error_description {
    int  ed_errno;
    char    *ed_name;
} err_desc[] =
{
    { EUNK,     "<unexpected errno?>"   },
    { EINVAL,   "EINVAL"        },
    { EACCES,   "EACCES"        },
    { EPERM,    "EPERM"         },
    { EIO,      "EIO"           },
    { EDEADLK,  "EDEADLK"       },
    { EBUSY,    "EBUSY"         },
    { EAGAIN,   "EAGAIN"        },
    { ERESTART, "ERESTART"      },
    { ETIMEDOUT,    "ETIMEDOUT"     },
    { NO_ERROR, "Ok"            },
    { EUNK,     NULL            }
};

struct action_description act_desc[] =
{
    { UFA_ERROR,    "<unexpected action?>"  },
    { UFA_FOUND,    "\"found\"" },
    { UFA_SET,  "\"set\""   },
    { UFA_ERROR,    NULL            },
};

#define LOCKFS_BADLOCK  (-1)

struct lock_description {
    int  ld_type;
    char    *ld_name;
} lock_desc[] =
{
    { LOCKFS_BADLOCK,   "<unexpected lock?>"    },
    { LOCKFS_ULOCK,     "Unlock"        },
    { LOCKFS_ELOCK,     "Error Lock"        },
    { LOCKFS_HLOCK,     "Hard Lock"     },
    { LOCKFS_OLOCK,     "Old Lock"      },
    { LOCKFS_BADLOCK,   NULL            }
};

#endif /* DEBUG */

/*
 * ufs_fault, ufs_fault_v
 *
 *  called instead of cmn_err(CE_PANIC, ...) by ufs routines
 *  when a failure is detected to put the file system into an
 *  error state (if possible) or to devolve to a panic otherwise
 *
 * vnode is some vnode in this file system, used to find the way
 * to ufsvfs, vfsp etc.  Since a panic can be called from many
 * levels, the vnode is the most convenient hook to pass through.
 *
 */

/*PRINTFLIKE2*/
int
ufs_fault(vnode_t *vp, char *fmt, ...)
{
    va_list adx;
    int error;

    MINOR(("[ufs_fault"));

    va_start(adx, fmt);
    error = ufs_fault_v(vp, fmt, adx);
    va_end(adx);

    MINOR((": %s (%d)]\n", err_name(error), error));
    return (error);
}

const char *nullfmt = "<null format?>";

static int
ufs_fault_v(vnode_t *vp, char *fmt, va_list adx)
{
    ufs_failure_t       *new = NULL;
    ufsvfs_t        *ufsvfsp;
    triage_t         fix;
    int          err = ERESTART;
    int         need_vfslock;

    MINOR(("[ufs_fault_v"));

    if (fmt == NULL)
        fmt = (char *)nullfmt;

    fix = triage(vp);

    if (vp) {
        ufsvfsp = (struct ufsvfs *)vp->v_vfsp->vfs_data;

        /*
         * Something bad has happened. That is why we are here.
         *
         * In order for the bad thing to be recorded in the superblock
         * we need to write to the superblock directly.
         * In the case that logging is enabled the logging code
         * would normally intercept our write as a delta to the log,
         * thus we mark the filesystem FSBAD in any case.
         */
        need_vfslock = !MUTEX_HELD(&ufsvfsp->vfs_lock);

        if (need_vfslock) {
            mutex_enter(&ufsvfsp->vfs_lock);
        }

        ufsvfsp->vfs_fs->fs_clean = FSBAD;
        ASSERT(SEMA_HELD(&ufsvfsp->vfs_bufp->b_sem));
        ufsvfsp->vfs_bufp->b_flags &=
            ~(B_ASYNC | B_READ | B_DONE | B_ERROR | B_DELWRI);

        (void) bdev_strategy(ufsvfsp->vfs_bufp);
        (void) biowait(ufsvfsp->vfs_bufp);

        if (need_vfslock) {
            mutex_exit(&ufsvfsp->vfs_lock);
        }
    }

    switch (fix) {

    default:
    case TRIAGE_DEAD:
    case TRIAGE_NO_SPIRIT:

        real_panic_v(new, fmt, adx);
        /* LINTED: warning: logical expression always true: op "||" */
        ASSERT(DEBUG);
        err = EAGAIN;

#if defined(DEBUG)
        if (!(DEBUG_FLAGS & DBGFLG_FIXWOULDPANIC)) {
            break;
        }
        /* FALLTHROUGH */

#else
        break;

#endif /* DEBUG */

    case TRIAGE_ATTEND_TO:

        /* q thread not running yet? */
        if (mutex_tryenter(&ufs_fix.uq_mutex)) {
            if (!ufs_fix.uq_threadp) {
                mutex_exit(&ufs_fix.uq_mutex);
                ufs_thread_start(&ufs_fix,
                    ufsfx_thread_fix_failures, NULL);
                ufs_fix.uq_threadp->t_flag |= T_DONTBLOCK;
                mutex_enter(&ufs_fix.uq_mutex);
            } else {
                /*
                 * We got the lock but we are not the current
                 * threadp so we have to release the lock.
                 */
                mutex_exit(&ufs_fix.uq_mutex);
            }
        } else {
            MINOR((": fix failure thread already running "));
            /*
             * No need to log another failure as one is already
             * being logged.
             */
            break;
        }

        if (ufs_fix.uq_threadp && ufs_fix.uq_threadp == curthread) {
            mutex_exit(&ufs_fix.uq_mutex);
            cmn_err(CE_WARN, "ufs_fault_v: recursive ufs_fault");
        } else {
            /*
             * Must check if we actually still own the lock and
             * if so then release the lock and move on with life.
             */
            if (mutex_owner(&ufs_fix.uq_mutex) == curthread)
                mutex_exit(&ufs_fix.uq_mutex);
        }

        new = init_failure(vp, fmt, adx);
        if (new != NULL) {
            queue_failure(new);
            break;
        }
        real_panic_v(new, fmt, adx);
        break;

    }
    MINOR(("] "));
    return (err);
}

/*
 * triage()
 *
 *  Attempt to fix iff:
 *    - the system is not already panicking
 *    - this file system isn't explicitly marked not to be fixed
 *    - we can connect to the user-level daemon
 * These conditions are detectable later, but if we can determine
 * them in the failing threads context the core dump may be more
 * useful.
 *
 */

static triage_t
triage(vnode_t *vp)
{
    struct inode     *ip;
    int       need_unlock_vfs;
    int       fs_flags;

    MINUTE(("[triage"));

    if (panicstr) {
        MINUTE((
        ": already panicking: \"%s\" => TRIAGE_DEAD]\n", panicstr));
        return (TRIAGE_DEAD);
    }

    if (!vp || !(ip = VTOI(vp)) || !ip->i_ufsvfs) {
        MINUTE((
    ": vp, ip or ufsvfs is NULL; can't determine fs => TRIAGE_DEAD]\n"));
        return (TRIAGE_DEAD);
    }

    /* use tryenter and continue no matter what since we're panicky */
    need_unlock_vfs = !MUTEX_HELD(&ip->i_ufsvfs->vfs_lock);
    if (need_unlock_vfs)
        need_unlock_vfs = mutex_tryenter(&ip->i_ufsvfs->vfs_lock);

    fs_flags = ip->i_ufsvfs->vfs_fsfx.fx_flags;
    if (need_unlock_vfs)
        mutex_exit(&ip->i_ufsvfs->vfs_lock);

    if (fs_flags & UFSFX_PANIC) {
        MINUTE((
        ": filesystem marked \"panic\" => TRIAGE_NO_SPIRIT]\n"));
        return (TRIAGE_NO_SPIRIT);
    }

    if (ufs_checkaccton(vp) != 0) {
        MINUTE((
        ": filesystem would deadlock (accounting) => TRIAGE_DEAD]\n"));
        return (TRIAGE_DEAD);
    }

    if (ufs_checkswapon(vp) != 0) {
        MINUTE((
        ": filesystem would deadlock (swapping) => TRIAGE_DEAD]\n"));
        return (TRIAGE_DEAD);
    }

    MINUTE((": return TRIAGE_ATTEND_TO] "));
    return (TRIAGE_ATTEND_TO);
}

/*
 * init failure
 *
 * This routine allocates a failure struct and initializes
 * it's member elements.
 * Space is allocated for copies of dynamic identifying fs structures
 * passed in.  Without a much more segmented kernel architecture
 * this is as protected as we can make it (for now.)
 */
static ufs_failure_t *
init_failure(vnode_t *vp, char *fmt, va_list adx)
{
    ufs_failure_t   *new;
    struct inode    *ip;
    int      initialization_worked = 0;
    int      need_vfs_unlock;

    MINOR(("[init_failure"));

    new = kmem_zalloc(sizeof (ufs_failure_t), KM_NOSLEEP);
    if (!new) {
        MINOR((": kmem_zalloc failed]\n"));
        return (NULL);
    }

    /*
     * enough information to make a fix attempt possible?
     */
    if (!vp || !(ip = VTOI(vp)) || !ip->i_ufsvfs || !vp->v_vfsp ||
        !ip->i_ufsvfs->vfs_bufp || !ITOF(ip) || !fmt)
        goto errout;

    if (vp->v_type != VREG && vp->v_type != VDIR &&
        vp->v_type != VBLK && vp->v_type != VCHR &&
        vp->v_type != VLNK && vp->v_type != VFIFO &&
        vp->v_type != VSOCK)
        goto errout;

    if (ip->i_ufsvfs->vfs_root->v_type != VREG &&
        ip->i_ufsvfs->vfs_root->v_type != VDIR &&
        ip->i_ufsvfs->vfs_root->v_type != VBLK &&
        ip->i_ufsvfs->vfs_root->v_type != VCHR &&
        ip->i_ufsvfs->vfs_root->v_type != VLNK &&
        ip->i_ufsvfs->vfs_root->v_type != VFIFO &&
        ip->i_ufsvfs->vfs_root->v_type != VSOCK)
        goto errout;

    if ((ITOF(ip)->fs_magic != FS_MAGIC) &&
        (ITOF(ip)->fs_magic != MTB_UFS_MAGIC))
        goto errout;

    /* intialize values */

    (void) vsnprintf(new->uf_panic_str, LOCKFS_MAXCOMMENTLEN - 1, fmt, adx);

    new->uf_ufsvfsp = ip->i_ufsvfs;
    new->uf_vfsp    = ip->i_vfs;

    mutex_init(&new->uf_mutex, NULL, MUTEX_DEFAULT, NULL);
    need_vfs_unlock = !MUTEX_HELD(&ip->i_ufsvfs->vfs_lock);

    if (need_vfs_unlock) {
        if (!mutex_tryenter(&ip->i_ufsvfs->vfs_lock)) {
            /*
             * not much alternative here, but we're panicking
             * already, it couldn't be worse - so just
             * proceed optimistically and take note.
             */
            mutex_enter(&uf_stats.ufst_mutex);
            uf_stats.ufst_lock_violations++;
            mutex_exit(&uf_stats.ufst_mutex);
            MINOR((": couldn't get vfs lock"))
            need_vfs_unlock = 0;
        }
    }

    if (mutex_tryenter(&new->uf_mutex)) {
        initialization_worked = set_state(new, UF_INIT);
        mutex_exit(&new->uf_mutex);
    }

    if (need_vfs_unlock)
        mutex_exit(&ip->i_ufsvfs->vfs_lock);

    if (initialization_worked) {
        MINOR(("] "));
        return (new);
    }
    /* FALLTHROUGH */

errout:
    if (new)
        kmem_free(new, sizeof (ufs_failure_t));
    MINOR((": failed]\n"));
    return (NULL);
}

static void
queue_failure(ufs_failure_t *new)
{
    MINOR(("[queue_failure"));

    mutex_enter(&ufs_fix.uq_mutex);

    if (ufs_fix.uq_ufhead)
        insque(new, &ufs_fix.uq_ufhead);
    else
        ufs_fix.uq_ufhead = new;

    if (mutex_tryenter(&new->uf_mutex)) {
        (void) set_state(new, UF_QUEUE);
        mutex_exit(&new->uf_mutex);
    }

    mutex_enter(&uf_stats.ufst_mutex);      /* force wakeup */
    ufs_fix.uq_ne = ufs_fix.uq_lowat = uf_stats.ufst_num_failed;
    mutex_exit(&uf_stats.ufst_mutex);

    cv_broadcast(&ufs_fix.uq_cv);

    DCALL(DBGLVL_MAJOR, cmn_err(CE_WARN, new->uf_panic_str ?
        new->uf_panic_str : "queue_failure: NULL panic str?"));
    mutex_exit(&ufs_fix.uq_mutex);

    MINOR(("] "));
}

/*PRINTFLIKE2*/
static void
real_panic(ufs_failure_t *f, const char *fmt, ...)
{
    va_list adx;

    MINUTE(("[real_panic "));

    va_start(adx, fmt);
    real_panic_v(f, fmt, adx);
    va_end(adx);

    MINUTE((": return?!]\n"));
}

static void
real_panic_v(ufs_failure_t *f, const char *fmt, va_list adx)
{
    int seriousness = CE_PANIC;
    int need_unlock;

    MINUTE(("[real_panic_v "));

    if (f && f->uf_ufsvfsp)
        TRANS_SETERROR(f->uf_ufsvfsp);

#if defined(DEBUG)
    if (DEBUG_FLAGS & DBGFLG_NOPANIC) {
        seriousness = CE_WARN;
        cmn_err(CE_WARN, "real_panic: EWOULDPANIC\n");
    }
#endif /* DEBUG */

    delay(hz >> 1);         /* allow previous warnings to get out */

    if (!f && fmt)
        vcmn_err(seriousness, fmt, adx);
    else
        cmn_err(seriousness, f && f->uf_panic_str? f->uf_panic_str:
            "real_panic: <unknown panic?>");

    if (f) {
        need_unlock = !MUTEX_HELD(&f->uf_mutex);
        if (need_unlock) {
            mutex_enter(&f->uf_mutex);
        }

        f->uf_retry = -1;
        (void) set_state(f, UF_PANIC);

        if (need_unlock) {
            mutex_exit(&f->uf_mutex);
        }
    }
    MINUTE((": return?!]\n"));
}

/*
 * initializes ufs panic structs, locks, etc
 */
void
ufsfx_init(void)
{

    MINUTE(("[ufsfx_init"));

    /* patchable; unchanged while running, so no lock is needed */
    ufsfx_tune.uft_too_long     = UF_TOO_LONG;
    ufsfx_tune.uft_fixstart_period  = UF_FIXSTART_PERIOD;
    ufsfx_tune.uft_fixpoll_period   = UF_FIXPOLL_PERIOD;
    ufsfx_tune.uft_short_err_period = UF_SHORT_ERROR_PERIOD;
    ufsfx_tune.uft_long_err_period  = UF_LONG_ERROR_PERIOD;

    uffsinfo.ufi_statp  = &uf_stats;
    uffsinfo.ufi_tunep  = &ufsfx_tune;
    uffsinfo.ufi_statetab   = &state_desc[0];

    mutex_init(&uf_stats.ufst_mutex, NULL, MUTEX_DEFAULT, NULL);
    ufs_thread_init(&ufs_fix, /* maxne */ 1);

    MINUTE(("] "));
}

/*
 * initializes per-ufs values
 * returns 0 (ok) or errno
 */
int
ufsfx_mount(struct ufsvfs *ufsvfsp, int flags)
{
    MINUTE(("[ufsfx_mount (%d)", flags));
    /* don't check/need vfs_lock because it's still being initialized */

    ufsvfsp->vfs_fsfx.fx_flags = (flags & UFSMNT_ONERROR_FLGMASK) >> 4;

    MINUTE((": %s: fx_flags:%ld,",
        ufsvfsp->vfs_fs->fs_fsmnt, ufsvfsp->vfs_fsfx.fx_flags));
    /*
     *  onerror={panic ^ lock only ^ unmount}
     */

    if (ufsvfsp->vfs_fsfx.fx_flags & UFSFX_PANIC) {
        MINUTE((" PANIC"));

    } else if (ufsvfsp->vfs_fsfx.fx_flags & UFSFX_LCKONLY) {
        MINUTE((" LCKONLY"));

    } else if (ufsvfsp->vfs_fsfx.fx_flags & UFSFX_LCKUMOUNT) {
        MINUTE((" LCKUMOUNT"));

    } else {
        ufsvfsp->vfs_fsfx.fx_flags = UFSFX_DEFAULT;
        ASSERT(ufsvfsp->vfs_fsfx.fx_flags &
            (UFSMNT_ONERROR_FLGMASK >> 4));
        MINUTE((" DEFAULT"));
    }

    pollwakeup(&ufs_pollhd, POLLPRI);
    MINUTE(("]\n"));
    return (0);
}

/*
 * ufsfx_unmount
 *
 * called during unmount
 */
void
ufsfx_unmount(struct ufsvfs *ufsvfsp)
{
    ufs_failure_t   *f;
    int      must_unlock_list;

    MINUTE(("[ufsfx_unmount"));

    if (!ufsvfsp) {
        MINUTE((": no ufsvfsp]"));
        return;
    }

    if ((must_unlock_list = !MUTEX_HELD(&ufs_fix.uq_mutex)) != 0)
        mutex_enter(&ufs_fix.uq_mutex);

    for (f = ufs_fix.uq_ufhead; f; f = f->uf_next) {
        int must_unlock_failure;

        must_unlock_failure = !MUTEX_HELD(&f->uf_mutex);
        if (must_unlock_failure) {
            mutex_enter(&f->uf_mutex);
        }

        if (f->uf_ufsvfsp == ufsvfsp) {

            /*
             * if we owned the failure record lock, then this
             * is probably a fix failure-triggered unmount, so
             * the warning is not appropriate or needed
             */

            /* XXX if rebooting don't print this? */
            if (!terminal_state(f->uf_s) && must_unlock_failure) {
                cmn_err(CE_WARN,
                    "Unmounting %s while error-locked",
                    fs_name(f));
            }

            f->uf_ufsvfsp       = NULL;
            f->uf_vfs_ufsfxp    = NULL;
            f->uf_vfs_lockp     = NULL;
            f->uf_bp        = NULL;
            f->uf_vfsp      = NULL;
            f->uf_retry     = -1;
        }

        if (must_unlock_failure)
            mutex_exit(&f->uf_mutex);
    }
    if (must_unlock_list)
        mutex_exit(&ufs_fix.uq_mutex);

    pollwakeup(&ufs_pollhd, POLLPRI | POLLHUP);
    MINUTE(("] "));
}

/*
 * ufsfx_(un)lockfs
 *
 * provides hook from lockfs code so we can recognize unlock/relock
 *  This is called after it is certain that the (un)lock will succeed.
 */
void
ufsfx_unlockfs(struct ufsvfs *ufsvfsp)
{
    ufs_failure_t   *f;
    int      need_unlock;
    int      need_unlock_list;
    int      informed = 0;

    MINUTE(("[ufsfx_unlockfs"));

    if (!ufsvfsp)
        return;

    need_unlock_list = !MUTEX_HELD(&ufs_fix.uq_mutex);

    if (need_unlock_list)
        mutex_enter(&ufs_fix.uq_mutex);

    for (f = ufs_fix.uq_ufhead; f; f = f->uf_next) {

        need_unlock = !MUTEX_HELD(&f->uf_mutex);
        if (need_unlock)
            mutex_enter(&f->uf_mutex);

        if (f->uf_ufsvfsp == ufsvfsp && !terminal_state(f->uf_s)) {
            if (!(f->uf_s & UF_FIXING)) {
                /*
                 * This might happen if we don't notice that
                 * the fs gets marked FSFIX before it is
                 * marked FSCLEAN, as might occur if the
                 * the superblock was hammered directly.
                 */
                if (!informed) {
                    informed = 1;
                    cmn_err(CE_NOTE,
                        "Unlock of %s succeeded before "
                        "fs_clean marked FSFIX?",
                        fs_name(f));
                }

                /*
                 * pass through fixing state so
                 * transition protocol is satisfied
                 */
                if (!set_state(f, UF_FIXING)) {
                    MINUTE((": failed] "));
                }
            }

            if (!set_state(f, UF_FIXED)) {
                /* it's already fixed, so don't panic now */
                MINUTE((": failed] "));
            }
        }

        if (need_unlock)
            mutex_exit(&f->uf_mutex);
    }
    if (need_unlock_list)
        mutex_exit(&ufs_fix.uq_mutex);
    MINUTE(("] "));
}

void
ufsfx_lockfs(struct ufsvfs *ufsvfsp)
{
    ufs_failure_t   *f;
    int      need_unlock;
    int      need_unlock_list;

    MINUTE(("[ufsfx_lockfs"));

    if (!ufsvfsp)
        return;

    need_unlock_list = !MUTEX_HELD(&ufs_fix.uq_mutex);

    if (need_unlock_list)
        mutex_enter(&ufs_fix.uq_mutex);

    for (f = ufs_fix.uq_ufhead; f; f = f->uf_next) {

        need_unlock = !MUTEX_HELD(&f->uf_mutex);
        if (need_unlock)
            mutex_enter(&f->uf_mutex);

        if (f->uf_ufsvfsp == ufsvfsp && !terminal_state(f->uf_s) &&
            f->uf_s != UF_PANIC) {
            switch (f->uf_s) {

            default:
                cmn_err(CE_WARN,
                    "fs %s not in state "
                    "UF_TRYLCK, UF_LOCKED or UF_FIXING",
                    fs_name(f));
                break;

            case UF_TRYLCK:
                if (!set_state(f, UF_LOCKED)) {
                    MINUTE((": failed] "));
                }
                break;

            case UF_LOCKED:
                if (!set_state(f, UF_FIXING)) {
                    MINUTE((": failed] "));
                }
                break;

            case UF_FIXING:
                break;

            }
        }

        if (need_unlock)
            mutex_exit(&f->uf_mutex);
    }
    if (need_unlock_list)
        mutex_exit(&ufs_fix.uq_mutex);

    MINUTE(("] "));
}

/*
 * error lock, trigger fsck and unlock those fs with failures
 * blatantly copied from the hlock routine, although this routine
 * triggers differently in order to use uq_ne as meaningful data.
 */
/* ARGSUSED */
void
ufsfx_thread_fix_failures(void *ignored)
{
    int     retry;
    callb_cpr_t cprinfo;

    CALLB_CPR_INIT(&cprinfo, &ufs_fix.uq_mutex, callb_generic_cpr,
        "ufsfixfail");

    MINUTE(("[ufsfx_thread_fix_failures] "));

    for (;;) {
        /* sleep until there is work to do */

        mutex_enter(&ufs_fix.uq_mutex);
        (void) ufs_thread_run(&ufs_fix, &cprinfo);
        ufs_fix.uq_ne = 0;
        mutex_exit(&ufs_fix.uq_mutex);

        /* process failures on our q */
        do {
            retry = ufsfx_do_failure_q();
            if (retry) {
                mutex_enter(&ufs_fix.uq_mutex);
                CALLB_CPR_SAFE_BEGIN(&cprinfo);
                (void) cv_reltimedwait(&ufs_fix.uq_cv,
                    &ufs_fix.uq_mutex, (hz * retry),
                    TR_CLOCK_TICK);
                CALLB_CPR_SAFE_END(&cprinfo,
                    &ufs_fix.uq_mutex);
                mutex_exit(&ufs_fix.uq_mutex);
            }
        } while (retry);
    }
    /* NOTREACHED */
}


/*
 * watch for fix-on-panic work
 *
 * returns # of seconds to sleep before trying again
 * and zero if no retry is needed
 */

int
ufsfx_do_failure_q(void)
{
    ufs_failure_t   *f;
    long         retry = 1;
    ufsd_t      *s;

    MAJOR(("[ufsfx_do_failure_q"));
    DCALL(DBGLVL_HIDEOUS, dump_uf_list(NULL));

    if (!mutex_tryenter(&ufs_fix.uq_mutex))
        return (retry);

    retry = 0;
rescan_q:

    /*
     * walk down failure list
     *  depending on state of each failure, do whatever
     *  is appropriate to move it to the next state
     *  taking note of whether retry gets set
     *
     * retry protocol:
     * wakeup in shortest required time for any failure
     *   retry == 0; nothing more to do (terminal state)
     *   retry < 0; reprocess queue immediately, retry will
     *      be abs(retry) for the next cycle
     *   retry > 0; schedule wakeup for retry seconds
     */

    for (f = ufs_fix.uq_ufhead; f; f = f->uf_next) {

        if (!mutex_tryenter(&f->uf_mutex)) {
            retry = 1;
            continue;
        }
        s = get_state_desc(f->uf_s);

        MINOR((": found%s: %s, \"%s: %s\"\n",
            s->ud_attr.terminal ? " old" : "",
            fs_name(f), state_name(f->uf_s), f->uf_panic_str));

        if (s->ud_attr.terminal) {
            mutex_exit(&f->uf_mutex);
            continue;
        }

        if (s->ud_sfp)
            (*s->ud_sfp)(f, UFA_FOUND, f->uf_s);

        ASSERT(terminal_state(f->uf_s) || f->uf_retry != 0);

        if (f->uf_retry != 0) {
            if (retry > f->uf_retry || retry == 0)
                retry = f->uf_retry;
            if (f->uf_retry < 0)
                f->uf_retry = abs(f->uf_retry);
        }
        mutex_exit(&f->uf_mutex);
    }


    if (retry < 0) {
        retry = abs(retry);
        goto rescan_q;
    }

    mutex_exit(&ufs_fix.uq_mutex);

    DCALL(DBGLVL_HIDEOUS, dump_uf_list(NULL));
    MAJOR((": retry=%ld, good night]\n\n", retry));

    return (retry);
}

static void
pester_msg(ufs_failure_t *f, int seriousness)
{
    MINUTE(("[pester_msg"));
    ASSERT(f->uf_s & (UF_LOCKED | UF_FIXING));

    /*
     * XXX if seems too long for this fs, poke administrator
     * XXX to run fsck manually (and change retry time?)
     */
    cmn_err(seriousness, "Waiting for repair of %s to %s",
        fs_name(f), f->uf_s & UF_LOCKED ? "start" : "finish");
    MINUTE(("]"));
}

static time_t
trylock_time_exceeded(ufs_failure_t *f)
{
    time_t      toolong;
    extern time_t   time;

    MINUTE(("[trylock_time_exceeded"));
    ASSERT(MUTEX_HELD(&f->uf_mutex));

    toolong = (time_t)ufsfx_tune.uft_too_long + f->uf_entered_tm;
    if (time > toolong)
        cmn_err(CE_WARN, "error-lock timeout exceeded: %s", fs_name(f));

    MINUTE(("] "));
    return (time <= toolong? 0: time - toolong);
}

static int
get_lockfs_status(ufs_failure_t *f, struct lockfs *lfp)
{
    MINUTE(("[get_lockfs_status"));

    if (!f->uf_ufsvfsp) {
        MINUTE((": ufsvfsp is NULL]\n"));
        return (0);
    }

    ASSERT(MUTEX_HELD(&f->uf_mutex));
    ASSERT(MUTEX_NOT_HELD(f->uf_vfs_lockp));
    ASSERT(!vfs_lock_held(f->uf_vfsp));
    ASSERT(f->uf_ufsvfsp->vfs_root != NULL);

    f->uf_lf_err = ufs_fiolfss(f->uf_ufsvfsp->vfs_root, lfp);

    if (f->uf_lf_err) {
        f->uf_retry = ufsfx_tune.uft_short_err_period;
    }

    MINUTE(("] "));
    return (1);
}

static sfrc_t
set_state(ufs_failure_t *f, ufs_failure_states_t new_state)
{
    ufsd_t      *s;
    sfrc_t       sfrc = SFRC_FAIL;
    int      need_unlock;
    extern time_t    time;

    HIDEOUS(("[set_state: new state:%s", state_name(new_state)));
    ASSERT(f);
    ASSERT(MUTEX_HELD(&f->uf_mutex));

    /*
     * if someone else is panicking, just let panic sync proceed
     */
    if (panicstr) {
        (void) set_state(f, UF_NOTFIX);
        HIDEOUS((": state reset: not fixed] "));
        return (sfrc);
    }

    /*
     * bad state transition, an internal error
     */
    if (!state_trans_valid(f->uf_s, new_state)) {
        /* recursion */
        if (!(f->uf_s & UF_PANIC) && !(new_state & UF_PANIC))
            (void) set_state(f, UF_PANIC);
        MINOR((": state reset: transition failure (\"%s\"->\"%s\")] ",
            state_name(f->uf_s), state_name(new_state)));
        return (sfrc);
    }

    s = get_state_desc(new_state);

    need_unlock = !MUTEX_HELD(&ufs_fix.uq_mutex);
    if (need_unlock)
        mutex_enter(&ufs_fix.uq_mutex);

    if (s->ud_attr.at_fail && ufs_fix.uq_threadp &&
        curthread == ufs_fix.uq_threadp) {
        cmn_err(CE_WARN, "set_state: probable recursive panic of %s",
            fs_name(f));
    }
    if (need_unlock)
        mutex_exit(&ufs_fix.uq_mutex);

    /* NULL state functions always succeed */
    sfrc = !s->ud_sfp? SFRC_SUCCESS: (*s->ud_sfp)(f, UFA_SET, new_state);

    if (sfrc == SFRC_SUCCESS && f->uf_s != new_state) {
        f->uf_s = new_state;
        f->uf_entered_tm = time;
        f->uf_counter = 0;
    }

    HIDEOUS(("]\n"));
    return (sfrc);
}

static ufsd_t *
get_state_desc(ufs_failure_states_t state)
{
    ufsd_t *s;

    HIDEOUS(("[get_state_desc"));

    for (s = &state_desc[1]; s->ud_name != NULL; s++) {
        if (s->ud_v == state) {
            HIDEOUS(("] "));
            return (s);
        }
    }

    HIDEOUS(("] "));
    return (&state_desc[0]);    /* default */
}

static sfrc_t
sf_undef(ufs_failure_t *f, ufsa_t a, ufs_failure_states_t s)
{
    sfrc_t rc;

    TRIVIA(("[sf_undef, action is %s, state is %s\n",
        act_name(a), state_name(s)));
    ASSERT(s == UF_UNDEF);

    /* shouldn't find null failure records or ever set one */
    rc = set_state(f, UF_NOTFIX);

    TRIVIA(("] "));
    return (rc);
}


static sfrc_t
sf_init(
    ufs_failure_t   *f,
    ufsa_t   a,
    ufs_failure_states_t     s)
{
    sfrc_t      rc = SFRC_FAIL;
    extern time_t   time;

    TRIVIA(("[sf_init, action is %s", act_name(a)));
    ASSERT(s & UF_INIT);

    switch (a) {
    case UFA_SET:
        f->uf_begin_tm = time;
        f->uf_retry = 1;
        if (!f->uf_ufsvfsp) {
            (void) set_state(f, UF_PANIC);
            TRIVIA((": NULL ufsvfsp]\n"));
            return (rc);
        }
        /*
         * because we can call panic from many different levels,
         * we can't be sure that we've got the vfs_lock at this
         * point.  However, there's not much alternative and if
         * we don't (have the lock) the worst case is we'll just
         * panic again
         */
        f->uf_vfs_lockp     = &f->uf_ufsvfsp->vfs_lock;
        f->uf_vfs_ufsfxp    = &f->uf_ufsvfsp->vfs_fsfx;

        if (!f->uf_ufsvfsp->vfs_bufp) {
            (void) set_state(f, UF_PANIC);
            TRIVIA((": NULL vfs_bufp]\n"));
            return (rc);
        }
        f->uf_bp = f->uf_ufsvfsp->vfs_bufp;

        if (!f->uf_ufsvfsp->vfs_bufp->b_un.b_fs) {
            (void) set_state(f, UF_PANIC);
            TRIVIA((": NULL vfs_fs]\n"));
            return (rc);
        }

        /* vfs_fs = vfs_bufp->b_un.b_fs */
        bcopy(f->uf_ufsvfsp->vfs_fs->fs_fsmnt, f->uf_fsname, MAXMNTLEN);

        f->uf_lf.lf_lock  = LOCKFS_ELOCK;   /* primer */

        if (!f->uf_vfsp || f->uf_vfsp->vfs_dev == NODEV) {
            (void) set_state(f, UF_PANIC);
            TRIVIA((": NULL vfsp or vfs_dev == NODEV"));
            return (rc);
        }
        f->uf_dev = f->uf_vfsp->vfs_dev;

        rc = SFRC_SUCCESS;
        break;

    case UFA_FOUND:
    default:
        /* failures marked init shouldn't even be on the queue yet */
        rc = set_state(f, UF_QUEUE);
        TRIVIA((": found failure with state init]\n"));
    }

    TRIVIA(("] "));
    return (rc);
}

static sfrc_t
sf_queue(
    ufs_failure_t   *f,
    ufsa_t   a,
    ufs_failure_states_t     s)
{
    sfrc_t      rc = SFRC_FAIL;

    TRIVIA(("[sf_queue, action is %s", act_name(a)));
    ASSERT(s & UF_QUEUE);

    if (!f->uf_ufsvfsp) {
        TRIVIA((": NULL ufsvfsp]\n"));
        return (rc);
    }

    switch (a) {
    case UFA_FOUND:
        rc = sf_found_queue(f);
        break;

    case UFA_SET:

        ASSERT(MUTEX_HELD(&ufs_fix.uq_mutex));

        mutex_enter(&uf_stats.ufst_mutex);
        uf_stats.ufst_num_failed++;
        mutex_exit(&uf_stats.ufst_mutex);

        /*
         * if can't get the vfs lock, just wait until
         * UF_TRYLCK to set fx_current
         */
        if (mutex_tryenter(f->uf_vfs_lockp)) {
            f->uf_vfs_ufsfxp->fx_current = f;
            mutex_exit(f->uf_vfs_lockp);
        } else {
            mutex_enter(&uf_stats.ufst_mutex);
            uf_stats.ufst_current_races++;
            mutex_exit(&uf_stats.ufst_mutex);
        }

        f->uf_retry = 1;
        rc = SFRC_SUCCESS;
        TRIVIA(("] "));
        break;

    default:
        (void) set_state(f, UF_PANIC);
        TRIVIA((": failed] "));
    }

    return (rc);
}

static sfrc_t
sf_found_queue(ufs_failure_t *f)
{
    int     replica;
    sfrc_t      rc = SFRC_FAIL;

    TRIVIA(("[sf_found_queue"));

    /*
     * don't need to check for null ufsvfsp because
     * unmount must own list's ufs_fix.uq_mutex
     * to mark it null and we own that lock since
     * we got here.
     */

    ASSERT(MUTEX_HELD(&ufs_fix.uq_mutex));
    ASSERT(MUTEX_NOT_HELD(f->uf_vfs_lockp));

    if (!mutex_tryenter(f->uf_vfs_lockp)) {
        TRIVIA((": tryenter(vfslockp) failed; retry]\n"));
        f->uf_retry = 1;
        return (rc);
    }

    replica = f->uf_vfs_ufsfxp && f->uf_vfs_ufsfxp->fx_current != NULL &&
        f->uf_vfs_ufsfxp->fx_current != f &&
        !terminal_state(f->uf_vfs_ufsfxp->fx_current->uf_s);

    /*
     * copy general flags to this ufs_failure so we don't
     * need to refer back to the ufsvfs, or, more importantly,
     * don't need to keep acquiring (trying to acquire) vfs_lockp
     *
     * The most restrictive option wins:
     *  panic > errlock only > errlock+unmount > repair
     * XXX panic > elock > elock > elock+umount
     */
    if (f->uf_vfs_ufsfxp->fx_flags & UFSFX_PANIC) {
        if (!set_state(f, UF_PANIC)) {
            TRIVIA((": marked panic but was queued?"));
            real_panic(f, " ");
            /*NOTREACHED*/
        }
        mutex_exit(f->uf_vfs_lockp);
        return (rc);
    }
    f->uf_flags = f->uf_vfs_ufsfxp->fx_flags;

    if (replica) {
        if (!set_state(f, UF_REPLICA)) {
            f->uf_retry = 1;
            TRIVIA((": set to replica failed] "));
        } else {
            TRIVIA(("] "));
        }
        mutex_exit(f->uf_vfs_lockp);
        return (rc);
    }
    mutex_exit(f->uf_vfs_lockp);

    if (!set_state(f, UF_TRYLCK)) {
        TRIVIA((": failed] "));
    } else {
        rc = SFRC_SUCCESS;
    }
    return (rc);
}

static sfrc_t
sf_nonterm_cmn(ufs_failure_t *f, ufsa_t a, ufs_failure_states_t s)
{
    sfrc_t  rc = SFRC_FAIL;

    TRIVIA(("[sf_nonterm_cmn, action: %s, %s", act_name(a), state_name(s)));
    ASSERT(s & (UF_TRYLCK | UF_LOCKED | UF_UMOUNT | UF_FIXING));
    ASSERT(!terminal_state(s));

    if (!f->uf_ufsvfsp && !(f->uf_s & UF_UMOUNT)) {
        TRIVIA((": NULL ufsvfsp (state != UMOUNT)]\n"));
        (void) set_state(f, UF_NOTFIX);
        return (rc);
    }

    switch (a) {
    case UFA_SET:
        switch (s) {
        case UF_TRYLCK:
            ASSERT(MUTEX_NOT_HELD(f->uf_vfs_lockp));
            rc = sf_set_trylck(f);
            break;

        case UF_LOCKED:
            rc = sf_set_locked(f);
            break;

        case UF_FIXING:
            f->uf_flags |= UFSFX_REPAIR_START;
            f->uf_retry  = ufsfx_tune.uft_fixpoll_period;
            rc = SFRC_SUCCESS;
            break;

        case UF_UMOUNT:
            f->uf_retry = -ufsfx_tune.uft_short_err_period;
            rc = SFRC_SUCCESS;
            break;

        default:
            (void) set_state(f, UF_PANIC);
            TRIVIA((": failed] "));
        }
        break;

    case UFA_FOUND:

        switch (s) {
        case UF_TRYLCK:
            rc = sf_found_trylck(f);
            break;

        case UF_LOCKED:
        case UF_FIXING:
            rc = sf_found_lock_fix_cmn(f, s);
            break;

        case UF_UMOUNT:
            rc = sf_found_umount(f);
            break;

        default:
            (void) set_state(f, UF_PANIC);
            TRIVIA((": failed] "));
            break;
        }
        break;
    default:
        (void) set_state(f, UF_PANIC);
        TRIVIA((": failed] "));
        break;
    }

    TRIVIA(("] "));
    return (rc);
}

static sfrc_t
sf_set_trylck(ufs_failure_t *f)
{
    TRIVIA(("[sf_set_trylck"));

    if (!mutex_tryenter(f->uf_vfs_lockp)) {
        TRIVIA((": tryenter(vfslockp) failed; retry]\n"));
        f->uf_retry = 1;
        return (SFRC_FAIL);
    }

    if (!f->uf_vfs_ufsfxp->fx_current)
        f->uf_vfs_ufsfxp->fx_current = f;

    mutex_exit(f->uf_vfs_lockp);

    f->uf_lf.lf_flags = 0;
    f->uf_lf.lf_lock  = LOCKFS_ELOCK;
    f->uf_retry = -ufsfx_tune.uft_fixstart_period;
    TRIVIA(("] "));
    return (SFRC_SUCCESS);
}

static sfrc_t
sf_found_trylck(ufs_failure_t *f)
{
    struct lockfs lockfs_status;

    TRIVIA(("[sf_found_trylck"));

    if (trylock_time_exceeded(f) > 0) {
        (void) set_state(f, UF_PANIC);
        TRIVIA((": failed] "));
        return (SFRC_FAIL);
    }

    if (!get_lockfs_status(f, &lockfs_status)) {
        (void) set_state(f, UF_PANIC);
        TRIVIA((": failed] "));
        return (SFRC_FAIL);
    }

    if (f->uf_lf_err == NO_ERROR)
        f->uf_lf.lf_key = lockfs_status.lf_key;

    if (!set_lockfs(f, &lockfs_status)) {
        (void) set_state(f, UF_PANIC);
        TRIVIA((": failed] "));
        return (SFRC_FAIL);
    }
    TRIVIA(("] "));
    return (SFRC_SUCCESS);
}

static sfrc_t
sf_set_locked(ufs_failure_t *f)
{
    TRIVIA(("[sf_set_locked"));

    f->uf_retry = -ufsfx_tune.uft_fixstart_period;

#if defined(DEBUG)
    if (f->uf_flags & UFSFX_REPAIR_START)
        TRIVIA(("clearing UFSFX_REPAIR_START "));
#endif /* DEBUG */

    f->uf_flags &= ~UFSFX_REPAIR_START;

    if (f->uf_s & UF_TRYLCK) {
        cmn_err(CE_WARN, "Error-locked %s: \"%s\"",
            fs_name(f), f->uf_panic_str);

        if (f->uf_flags & UFSFX_LCKONLY)
            cmn_err(CE_WARN, "Manual repair of %s required",
                fs_name(f));
    }

    /*
     * just reset to current state
     */
#if defined(DEBUG)
    TRIVIA(("locked->locked "));
#endif /* DEBUG */

    TRIVIA(("] "));
    return (SFRC_SUCCESS);
}

static sfrc_t
sf_found_lock_fix_cmn(ufs_failure_t *f, ufs_failure_states_t s)
{
    time_t      toolong;
    extern time_t   time;
    struct buf  *bp         = NULL;
    struct fs   *dfs;
    time_t       concerned, anxious;
    sfrc_t       rc         = SFRC_FAIL;
    ulong_t      gb_size;

    TRIVIA(("[sf_found_lock_fix_cmn (\"%s\")", state_name(s)));

    if (s & UF_LOCKED) {
        ASSERT(MUTEX_HELD(&f->uf_mutex));

        toolong =
            time > (ufsfx_tune.uft_too_long + f->uf_entered_tm);
        TRIVIA(("%stoolong", !toolong? "not": ""));
        HIDEOUS((": time:%ld, too long:%ld, entered_tm:%ld ",
            time, ufsfx_tune.uft_too_long, f->uf_entered_tm));

        if (f->uf_flags & UFSFX_LCKUMOUNT) {
            if (set_state(f, UF_UMOUNT)) {
                TRIVIA(("] "));
                rc = SFRC_SUCCESS;
            } else {
                TRIVIA((": failed] "));
                f->uf_retry = 1;
            }
            return (rc);
        }
        if (!toolong) {
            rc = SFRC_SUCCESS;
        } else {
            if (!(f->uf_flags & UFSFX_REPAIR_START)) {
                cmn_err(CE_WARN, "%s repair of %s not started.",
                    (f->uf_flags & UFSFX_LCKONLY) ?
                    "Manual" : "Automatic", fs_name(f));

                f->uf_retry = ufsfx_tune.uft_long_err_period;
            } else {
                f->uf_retry = ufsfx_tune.uft_long_err_period;
                cmn_err(CE_WARN, "Repair of %s is not timely; "
                    "operator attention is required.",
                    fs_name(f));
            }
            TRIVIA(("] "));
            return (rc);
        }
    }

#if defined(DEBUG)
    else {
        ASSERT(s & UF_FIXING);
    }
#endif /* DEBUG */

    /*
     * get on disk superblock; force it to really
     * come from the disk
     */
    (void) bfinval(f->uf_dev, 0);
    bp = UFS_BREAD(f->uf_ufsvfsp, f->uf_dev, SBLOCK, SBSIZE);
    if (bp) {
        bp->b_flags |= (B_STALE | B_AGE);
        dfs = bp->b_un.b_fs;
    }

    if (!bp || (bp->b_flags & B_ERROR) || ((dfs->fs_magic != FS_MAGIC) &&
        (dfs->fs_magic != MTB_UFS_MAGIC))) {
        TRIVIA((": UFS_BREAD(SBLOCK) failed]\n"));
        f->uf_retry = 1;
        goto out;
    }

    /* fsck started but we haven't noticed yet? */
    if (!(s & UF_FIXING) && dfs->fs_clean == FSFIX) {
        if (!set_state(f, UF_FIXING)) {
            TRIVIA((": failed]\n"));
            f->uf_retry = 1;
            goto out;
        }
    }

    /* fsck started but didn't succeed? */
    if ((s & UF_FIXING) && ((dfs->fs_clean == FSBAD) || !fsck_active(f))) {
        TRIVIA((": fs_clean: %d", (int)dfs->fs_clean));
        (void) set_state(f, UF_LOCKED);
        cmn_err(CE_WARN, "%s: Manual repair is necessary.", fs_name(f));
        f->uf_retry = ufsfx_tune.uft_long_err_period;
        goto out;
    }

    gb_size = (dfs->fs_size * dfs->fs_bshift) / GB;
    toolong = (time_t)((gb_size == 0? 1: gb_size) * SecondsPerGig);

    /* fsck started but doesn't seem to be proceeding? */
    if ((s & UF_FIXING) && dfs->fs_clean == FSFIX) {
        if (time > f->uf_entered_tm + toolong) {

            cmn_err(CE_WARN,
                "Repair completion timeout exceeded on %s; "
                "manual fsck may be required", fs_name(f));
            f->uf_retry = ufsfx_tune.uft_long_err_period;
        }
    }

    concerned = f->uf_entered_tm + (toolong / 3);
    anxious = f->uf_entered_tm + ((2 * toolong) / 3);

    if (time > concerned)
        pester_msg(f, time > anxious? CE_WARN: CE_NOTE);

    TRIVIA(("] "));

out:
    if (bp)
        brelse(bp);

    return (rc);
}

static sfrc_t
sf_found_umount(ufs_failure_t *f)
{
    extern time_t    time;
    sfrc_t       rc         = SFRC_FAIL;
    struct vfs  *vfsp           = f->uf_vfsp;
    struct ufsvfs   *ufsvfsp        = f->uf_ufsvfsp;
    int      toolong        = 0;
    int      err            = 0;

    TRIVIA(("[sf_found_umount"));

    toolong = time > ufsfx_tune.uft_too_long + f->uf_entered_tm;
    if (toolong) {
        TRIVIA((": unmount time limit exceeded] "));
        goto out;
    }

    if (!vfsp || !ufsvfsp) {    /* trivial case */
        TRIVIA((": NULL vfsp and/or ufsvfsp, already unmounted?] "));
        goto out;
    }

    if (!ULOCKFS_IS_ELOCK(&ufsvfsp->vfs_ulockfs)) {
        TRIVIA((": !not error locked?"));
        err = EINVAL;
        goto out;
    }

    /* The vn_vfsunlock will be done in dounmount() [.../common/fs/vfs.c] */
    if (vn_vfswlock(vfsp->vfs_vnodecovered)) {
        TRIVIA((": couldn't lock coveredvp"));
        err = EBUSY;
        goto out;
    }

    if ((err = dounmount(vfsp, 0, kcred)) != 0) {

        /* take note, but not many alternatives here */
        mutex_enter(&uf_stats.ufst_mutex);
        uf_stats.ufst_unmount_failures++;
        mutex_exit(&uf_stats.ufst_mutex);

        TRIVIA((": unmount failed] "));
    } else {
        cmn_err(CE_NOTE, "unmounted error-locked %s", fs_name(f));
    }

out:
    if (toolong || (err != EBUSY && err != EAGAIN))
        rc = set_state(f, UF_NOTFIX);

    TRIVIA(("] "));
    return (rc);
}

static sfrc_t
sf_term_cmn(ufs_failure_t *f, ufsa_t a, ufs_failure_states_t s)
{
    extern time_t   time;
    sfrc_t      rc = SFRC_FAIL;

    TRIVIA(("[sf_term_cmn, action is %s, state is %s",
        act_name(a), state_name(s)));
    ASSERT(s & (UF_FIXED | UF_NOTFIX | UF_REPLICA));
    ASSERT(terminal_state(s));

    if (!f->uf_ufsvfsp && !(f->uf_s & (UF_UMOUNT | UF_NOTFIX))) {
        TRIVIA((": NULL ufsvfsp (state != UMOUNT | NOTFIX)]\n"));
        return (rc);
    }

    switch (a) {
    case UFA_SET:
        switch (s) {
        case UF_NOTFIX:
        case UF_FIXED:
        {
            int need_lock_vfs;

            if (f->uf_ufsvfsp && f->uf_vfs_lockp)
                need_lock_vfs = !MUTEX_HELD(f->uf_vfs_lockp);
            else
                need_lock_vfs = 0;

            if (need_lock_vfs && !mutex_tryenter(f->uf_vfs_lockp)) {
                TRIVIA((": tryenter(vfslockp) fail; retry]\n"));
                f->uf_retry = 1;
                break;
            }

            f->uf_end_tm = time;
            f->uf_lf.lf_lock = LOCKFS_OLOCK;
            f->uf_retry = 0;

            if (f->uf_vfs_ufsfxp)
                f->uf_vfs_ufsfxp->fx_current = NULL;

            if (need_lock_vfs)
                mutex_exit(f->uf_vfs_lockp);

            cmn_err(CE_NOTE, (s & UF_NOTFIX)? "Could not fix %s":
                "%s is now accessible", fs_name(f));

            if (s & UF_FIXED) {
                mutex_enter(&uf_stats.ufst_mutex);
                uf_stats.ufst_num_fixed++;
                mutex_exit(&uf_stats.ufst_mutex);
            }
            (void) timeout(ufsfx_kill_fix_failure_thread,
                (void *)(ufsfx_tune.uft_short_err_period * hz),
                ufsfx_tune.uft_short_err_period * hz);
            rc = SFRC_SUCCESS;
            break;
        }
        case UF_REPLICA:

            ASSERT(MUTEX_HELD(f->uf_vfs_lockp));

            /* not actually a replica? */
            if (f->uf_vfs_ufsfxp && f->uf_vfs_ufsfxp->fx_current &&
                f->uf_vfs_ufsfxp->fx_current != f &&
                !terminal_state(
                f->uf_vfs_ufsfxp->fx_current->uf_s)) {

                f->uf_orig = f->uf_vfs_ufsfxp->fx_current;
                f->uf_retry = 0;
                rc = SFRC_SUCCESS;
            } else {
                TRIVIA((": NULL fx_current]\n"));
                f->uf_retry = 1;
            }

            break;

        default:
            rc = set_state(f, UF_PANIC);
            TRIVIA((": failed] "));
            break;
        }
        break;

    case UFA_FOUND:
        /*
         * XXX de-allocate these after some period?
         * XXX or move to an historical list?
         * XXX or have an ioctl which reaps them?
         */
        /*
         * For now, since we don't expect lots of failures
         * to occur (to the point of memory shortages),
         * just punt
         */

        /* be sure we're not wasting cpu on old failures */
        if (f->uf_retry != 0) {
            mutex_enter(&uf_stats.ufst_mutex);
            uf_stats.ufst_cpu_waste++;
            mutex_exit(&uf_stats.ufst_mutex);
            f->uf_retry = 0;
        }
        rc = SFRC_SUCCESS;
        break;

    default:
        (void) set_state(f, UF_PANIC);
        TRIVIA((": failed] "));
        break;
    }

    TRIVIA(("] "));
    return (rc);
}

static sfrc_t
sf_panic(
    ufs_failure_t   *f,
    ufsa_t   a,
    ufs_failure_states_t     s)
{
    sfrc_t  rc = SFRC_FAIL;

    TRIVIA(("[sf_panic, action is %s, prev. state is %s",
        act_name(a), state_name(f->uf_s)));
    ASSERT(s & UF_PANIC);

    switch (a) {
    case UFA_SET:
        f->uf_retry = -ufsfx_tune.uft_short_err_period;
        rc = SFRC_SUCCESS;
        break;

    case UFA_FOUND:
    default:
        real_panic(f, " ");

        /* LINTED: warning: logical expression always true: op "||" */
        ASSERT(DEBUG);

        (void) set_state(f, UF_UMOUNT); /* XXX UF_NOTFIX? */

        break;
    }

    TRIVIA(("] "));
    return (rc);
}

/*
 * minimum state function
 */
static sfrc_t
sf_minimum(
    ufs_failure_t   *f,
    ufsa_t   a, /* LINTED argument unused in function: ignored */
    ufs_failure_states_t     ignored)
{
    sfrc_t rc = SFRC_FAIL;

    TRIVIA(("[sf_minimum, action is %s", act_name(a)));

    switch (a) {
    case UFA_SET:
        f->uf_retry = 0;
        /* FALLTHROUGH */

    case UFA_FOUND:
        rc = SFRC_SUCCESS;
        break;

    default:
        (void) set_state(f, UF_PANIC);
        TRIVIA((": failed] "));
        break;
    }

    TRIVIA(("] "));
    return (rc);
}

static int
state_trans_valid(ufs_failure_states_t from, ufs_failure_states_t to)
{
    ufsd_t  *s;
    int  valid;

    HIDEOUS(("[state_trans_valid"));

    if (from & to)
        return (1);

    s = get_state_desc(to);

    /*
     * extra test is necessary since we want UF_UNDEF = 0,
     * (to detect freshly allocated memory)
     * but can't check for that value with a bit test
     */
    valid = (to & UF_INIT)? from == s->ud_prev: from & s->ud_prev;

    HIDEOUS((": %svalid] ", valid? "": "in"));
    return (valid);
}

static int
terminal_state(ufs_failure_states_t state)
{
    ufsd_t  *s;

    HIDEOUS(("[terminal_state"));

    s = get_state_desc(state);

    HIDEOUS((": %sterminal] ", s->ud_attr.terminal? "": "not "));
    return ((int)s->ud_attr.terminal);
}

static void
alloc_lockfs_comment(ufs_failure_t *f, struct lockfs *lfp)
{
    MINUTE(("[alloc_lockfs_comment"));
    ASSERT(MUTEX_HELD(&f->uf_mutex));

    /*
     * ufs_fiolfs expects a kmem_alloc'ed comment;
     * it frees the comment if the lock fails
     * or else when the lock is unlocked.
     */

    f->uf_lf.lf_comment = kmem_zalloc(LOCKFS_MAXCOMMENTLEN, KM_NOSLEEP);
    if (f->uf_lf.lf_comment) {
        char    *from;
        size_t   len;

        /*
         * use panic string if there's no previous comment
         * or if we're setting the error lock
         */
        if ((LOCKFS_IS_ELOCK(&f->uf_lf) || !lfp->lf_comment ||
            lfp->lf_comlen <= 0)) {
            from = f->uf_panic_str;
            len = LOCKFS_MAXCOMMENTLEN;
        } else {
            from = lfp->lf_comment;
            len = lfp->lf_comlen;
        }

        bcopy(from, f->uf_lf.lf_comment, len);
        f->uf_lf.lf_comlen = len;

    } else {
        f->uf_lf.lf_comlen = 0;
    }
    MINUTE(("] "));
}

static int
set_lockfs(ufs_failure_t *f, struct lockfs *lfp)
{
    int (*handle_lockfs_rc)(ufs_failure_t *);
    int   rc;

    MINUTE(("[set_lockfs"));
    ASSERT(MUTEX_HELD(&f->uf_mutex));
    ASSERT(!vfs_lock_held(f->uf_vfsp));
    ASSERT(MUTEX_NOT_HELD(f->uf_vfs_lockp));

    if (!f->uf_ufsvfsp) {
        MINUTE((": ufsvfsp is NULL]\n"));
        return (0);
    }

    ASSERT(MUTEX_NOT_HELD(&f->uf_ufsvfsp->vfs_ulockfs.ul_lock));

    if (!f->uf_ufsvfsp->vfs_root) {
        MINUTE((": vfs_root is NULL]\n"));
        return (0);
    }

    alloc_lockfs_comment(f, lfp);
    f->uf_lf_err = 0;

    if (!LOCKFS_IS_ELOCK(lfp)) {
        lfp->lf_lock = f->uf_lf.lf_lock = LOCKFS_ELOCK;
        VN_HOLD(f->uf_ufsvfsp->vfs_root);
        f->uf_lf_err =
            ufs__fiolfs(f->uf_ufsvfsp->vfs_root,
            &f->uf_lf, /* from_user */ 0, /* from_log */ 0);
        VN_RELE(f->uf_ufsvfsp->vfs_root);
    }

    handle_lockfs_rc = f->uf_lf_err != 0? lockfs_failure: lockfs_success;
    rc = handle_lockfs_rc(f);

    MINUTE(("] "));
    return (rc);
}

static int
lockfs_failure(ufs_failure_t *f)
{
    int error;
    ufs_failure_states_t    s;

    TRIVIA(("[lockfs_failure"));
    ASSERT(MUTEX_HELD(&f->uf_mutex));

    if (!f->uf_ufsvfsp) {
        TRIVIA((": ufsvfsp is NULL]\n"));
        return (0);
    }

    error = f->uf_lf_err;
    switch (error) {
            /* non-transient errors: */
    case EACCES:    /* disk/in-core metadata reconciliation failed  */
    case EPERM: /* inode reconciliation failed; incore inode changed? */
    case EIO:   /* device is hard-locked or not responding */
    case EROFS: /* device is write-locked */
    case EDEADLK:   /* can't lockfs; deadlock would result; */
            /* Swapping or saving accounting records */
            /* onto this fs can cause this errno. */

        MINOR(("ufs_fiolfs(\"%s\") of %s failed: %s (%d)",
            fs_name(f), lock_name(&f->uf_lf),
            err_name(error), error));

        /*
         * if can't get lock, then fallback to panic, unless
         * unless unmount was requested (although unmount will
         * probably fail if the lock failed, so we'll panic
         * anyway
         */

        s = ((f->uf_flags & UFSFX_LCKUMOUNT) && error != EDEADLK) ?
            UF_UMOUNT: UF_PANIC;

        if (!set_state(f, s)) {
            real_panic(f, " ");
            /*NOTREACHED*/
            break;
        }
        break;


    case EBUSY:
    case EAGAIN:

        f->uf_retry = ufsfx_tune.uft_short_err_period;
        if (curthread->t_flag & T_DONTPEND) {
            curthread->t_flag &= ~T_DONTPEND;

        } else if (!(f->uf_s & (UF_LOCKED | UF_FIXING))) {
            ufs_failure_states_t state;
            /*
             * if we didn't know that the fix had started,
             * take note
             */
            state = error == EBUSY? UF_LOCKED: UF_FIXING;
            if (!set_state(f, state)) {
                TRIVIA((": failed] "));
                return (0);
            }
        }
        break;

    default:    /* some other non-fatal error */
        MINOR(("lockfs(\"%s\") of %s returned %s (%d)",
            lock_name(&f->uf_lf), fs_name(f),
            err_name(f->uf_lf_err), f->uf_lf_err));

        f->uf_retry = ufsfx_tune.uft_short_err_period;
        break;

    case EINVAL:    /* unmounted? */
        (void) set_state(f, UF_NOTFIX);
        break;
    }
    TRIVIA(("] "));
    return (1);
}

static int
lockfs_success(ufs_failure_t *f)
{
    TRIVIA(("[lockfs_success"));
    ASSERT(MUTEX_HELD(&f->uf_mutex));

    if (!f->uf_ufsvfsp) {
        TRIVIA((": ufsvfsp is NULL]\n"));
        return (0);
    }

    switch (f->uf_lf.lf_lock) {
    case LOCKFS_ELOCK:  /* error lock worked */

        if (!set_state(f, UF_LOCKED)) {
            TRIVIA((": failed] "));
            return (0);
        }
        break;

    case LOCKFS_ULOCK:          /* unlock worked */
        /*
         * how'd we get here?
         * This should be done from fsck's unlock,
         * not from this thread's context.
         */
        cmn_err(CE_WARN, "Unlocked error-lock of %s", fs_name(f));
        ufsfx_unlockfs(f->uf_ufsvfsp);
        break;

    default:
        if (!set_state(f, UF_NOTFIX)) {
            TRIVIA((": failed] "));
            return (0);
        }
        break;
    }
    TRIVIA(("] "));
    return (1);
}

/*
 * when fsck is running it puts its pid into the lockfs
 * comment structure, prefaced by PIDSTR
 */
const char *PIDSTR = "[pid:";
static int
fsck_active(ufs_failure_t *f)
{
    char        *cp;
    int      i, found, errlocked;
    size_t       comlen;
    const int    PIDSTRLEN = (int)strlen(PIDSTR);
    struct ulockfs  *ulp = &f->uf_ufsvfsp->vfs_ulockfs;

    TRIVIA(("[fsck_active"));

    ASSERT(f);
    ASSERT(f->uf_s & UF_FIXING);
    ASSERT(MUTEX_HELD(&f->uf_mutex));
    ASSERT(f->uf_ufsvfsp);
    ASSERT(MUTEX_NOT_HELD(f->uf_vfs_lockp));
    ASSERT(MUTEX_NOT_HELD(&ulp->ul_lock));

    mutex_enter(&ulp->ul_lock);
    cp = ulp->ul_lockfs.lf_comment;
    comlen = ulp->ul_lockfs.lf_comlen;
    errlocked = (int)ULOCKFS_IS_ELOCK(ulp);
    mutex_exit(&ulp->ul_lock);

    if (!cp || comlen == 0) {
        TRIVIA((": null comment or comlen <= 0, found:0]"));
        return (0);
    }

    for (found = i = 0; !found && i < (comlen - PIDSTRLEN); i++, cp++)
        found = strncmp(cp, PIDSTR, PIDSTRLEN) == 0;

    TRIVIA(("found:%d, is_elock:%d]", found, errlocked));
    return (errlocked & found);
}

static const char unknown_fs[]      = "<unknown fs>";
static const char null_failure[] = "<NULL ufs failure record; unknown fs>";
static const char mutated_vfs_bufp[]    = "<mutated vfs_bufp, unknown fs>";
static const char mutated_vfs_fs[]  = "<mutated vfs_fs, unknown fs>";

static char *
fs_name(ufs_failure_t *f)
{
    HIDEOUS(("[fs_name"));
    ASSERT(MUTEX_HELD(&f->uf_mutex));

    if (!f) {
        HIDEOUS((": failure ptr is NULL]\n"));
        return ((char *)null_failure);
    }

    if (f->uf_fsname[0] != '\0') {
        HIDEOUS((": return (uf_fsname)]\n"));
        return (f->uf_fsname);
    }

    if (MUTEX_HELD(f->uf_vfs_lockp)) {
        if (f->uf_bp != f->uf_ufsvfsp->vfs_bufp) {
            HIDEOUS((": vfs_bufp mutated from 0x%p to 0x%p\n",
                (void *)f->uf_bp, (void *)f->uf_ufsvfsp->vfs_bufp));
            return ((char *)mutated_vfs_bufp);
        }
        if (f->uf_fs != f->uf_ufsvfsp->vfs_fs) {
            HIDEOUS((": vfs_bufp mutated from 0x%p to 0x%p\n",
                (void *)f->uf_fs, (void *)f->uf_ufsvfsp->vfs_fs));
            return ((char *)mutated_vfs_fs);
        }
        if (f->uf_ufsvfsp && f->uf_bp && f->uf_fs &&
            *f->uf_fs->fs_fsmnt != '\0') {
            HIDEOUS((": return (fs_fsmnt)]\n"));
            return (f->uf_fs->fs_fsmnt);
        }
    }

    HIDEOUS((": unknown file system]\n"));
    return ((char *)unknown_fs);
}

#if defined(DEBUG)
static char *
lock_name(struct lockfs *lfp)
{
    struct lock_description *l;
    char            *lname;

    HIDEOUS(("[lock_name"));

    lname = lock_desc[0].ld_name;
    for (l = &lock_desc[1]; l->ld_name != NULL; l++) {
        if (lfp && lfp->lf_lock == l->ld_type) {
            lname = l->ld_name;
            break;
        }
    }
    HIDEOUS(("]"));
    return (lname);
}

static char *
state_name(ufs_failure_states_t state)
{
    ufsd_t  *s;

    HIDEOUS(("[state_name"));

    s = get_state_desc(state);

    HIDEOUS(("]"));
    return (s->ud_name);
}

static char *
err_name(int error)
{
    struct error_description *e;

    HIDEOUS(("[err_name"));

    for (e = &err_desc[1]; e->ed_name != NULL; e++) {
        if (error == e->ed_errno) {
            HIDEOUS(("]"));
            return (e->ed_name);
        }
    }
    HIDEOUS(("]"));
    return (err_desc[0].ed_name);
}

static char *
act_name(ufsa_t action)
{
    struct action_description *a;

    HIDEOUS(("[act_name"));

    for (a = &act_desc[1]; a->ad_name != NULL; a++) {
        if (action == a->ad_v) {
            HIDEOUS(("]"));
            return (a->ad_name);
        }
    }
    HIDEOUS(("]"));
    return (act_desc[0].ad_name);
}

/*
 * dump failure list
 */
static void
dump_uf_list(char *msg)
{
    ufs_failure_t   *f;
    int      i;
    int      list_was_locked = MUTEX_HELD(&ufs_fix.uq_mutex);

    if (!list_was_locked && !mutex_tryenter(&ufs_fix.uq_mutex)) {
        printf("dump_uf_list: couldn't get list lock\n");
        return;
    }

    if (msg) {
        printf("\n%s", msg);
    }
    printf("\ndump_uf_list:\n\tuq_lowat: %d, uq_ne: %d\n",
        ufs_fix.uq_lowat, ufs_fix.uq_ne);

    mutex_enter(&uf_stats.ufst_mutex);
    printf("\tuf_stats.current_races: %ld\n", uf_stats.ufst_current_races);
    printf("\tuf_stats.num_failed: %ld\n", uf_stats.ufst_num_failed);
    printf("\tuf_stats.num_fixed: %ld\n", uf_stats.ufst_num_fixed);
    printf("\tuf_stats.cpu_waste: %ld\n", uf_stats.ufst_cpu_waste);
    printf("\tuf_stats.lock_violations: %ld, unmount_failures: %ld\n",
        uf_stats.ufst_lock_violations, uf_stats.ufst_unmount_failures);
    mutex_exit(&uf_stats.ufst_mutex);

    for (f = ufs_fix.uq_ufhead, i = 1; f; f = f->uf_next, i++) {

        if (!mutex_tryenter(&f->uf_mutex)) {
            printf("%d.\t\"skipped - try enter failed\"\n", i);
            continue;
        }

        dump_uf(f, i);

        mutex_exit(&f->uf_mutex);
    }

    printf("\n");

    if (!list_was_locked)
        mutex_exit(&ufs_fix.uq_mutex);
}

static void
dump_uf(ufs_failure_t *f, int i)
{
    if (!f) {
        printf("dump_uf: NULL failure record\n");
        return;
    }

    printf("%d.\t\"%s\" is %s.\n",
        i, fs_name(f), state_name(f->uf_s));
    printf("\t\"%s\"\tAddr: 0x%p\n", f->uf_panic_str, (void *)f);
    printf("\tNext: 0x%p\t\tPrev: 0x%p\n",
        (void *)f->uf_next, (void *)f->uf_prev);

    if (f->uf_orig)
        printf("\tOriginal failure: 0x%p \"%s\"\n",
            (void *)f->uf_orig, f->uf_orig->uf_panic_str);

    printf("\tUfsvfs: 0x%p\t\tVfs_lockp: 0x%p\n",
        (void *)f->uf_ufsvfsp, (void *)f->uf_vfs_lockp);
    printf("\tVfs_fsfxp: 0x%p\n", (void *)f->uf_vfs_ufsfxp);
    printf("\tVfs_bufp: 0x%p", (void *)f->uf_bp);

    if (f->uf_bp)
        printf("\t\tVfs_fs: 0x%p\n", (void *)f->uf_fs);
    else
        printf("\n");

    printf("\tBegin: 0x%lx\tEntered: 0x%lx\tEnd: 0x%lx\n",
        f->uf_begin_tm, f->uf_entered_tm, f->uf_end_tm);

    printf("\tFlags: (%d) %s%s%s%s", f->uf_flags,
        f->uf_flags & UFSFX_LCKONLY?     "\"lock only\" "   : "",
        f->uf_flags & UFSFX_LCKUMOUNT?   "\"lock+unmount\" "    : "",
        f->uf_flags & UFSFX_REPAIR_START? "\"started repair\" " : "",
        f->uf_flags == 0?                "<none>"               : "");

    printf("\tRetry: %ld seconds\n", f->uf_retry);

    printf("\tLockfs:\ttype: %s\terror: %s (%d)\n",
        lock_name(&f->uf_lf), err_name(f->uf_lf_err), f->uf_lf_err);

}
#endif /* DEBUG */

/*
 * returns # of ufs_failures in a non-terminal state on queue
 * used to coordinate with hlock thread (see ufs_thread.c)
 * and to determine when the error lock thread may exit
 */

int
ufsfx_get_failure_qlen(void)
{
    ufs_failure_t   *f;
    ufsd_t      *s;
    int      qlen = 0;

    MINUTE(("[ufsfx_get_failure_qlen"));

    if (!mutex_tryenter(&ufs_fix.uq_mutex))
        return (-1);

    /*
     * walk down failure list
     */

    for (f = ufs_fix.uq_ufhead; f; f = f->uf_next) {

        if (!mutex_tryenter(&f->uf_mutex))
            continue;

        s = get_state_desc(f->uf_s);

        if (s->ud_attr.terminal) {
            mutex_exit(&f->uf_mutex);
            continue;
        }

        MINUTE((": found: %s, \"%s: %s\"\n",
            fs_name(f), state_name(f->uf_s), f->uf_panic_str));

        qlen++;
        mutex_exit(&f->uf_mutex);
    }

    mutex_exit(&ufs_fix.uq_mutex);

    MINUTE((": qlen=%d]\n", qlen));

    return (qlen);
}

/*
 * timeout routine
 *  called to shutdown fix failure thread and server daemon
 */
static void
ufsfx_kill_fix_failure_thread(void *arg)
{
    clock_t odelta = (clock_t)arg;
    int qlen;

    MAJOR(("[ufsfx_kill_fix_failure_thread"));

    qlen = ufsfx_get_failure_qlen();

    if (qlen < 0) {
        clock_t delta;

        delta = odelta << 1;
        if (delta <= 0)
            delta = INT_MAX;

        (void) timeout(ufsfx_kill_fix_failure_thread,
            (void *)delta, delta);
        MAJOR((": rescheduled"));

    } else if (qlen == 0) {
        ufs_thread_exit(&ufs_fix);
        MAJOR((": killed"));
    }
    /*
     * else
     *  let timeout expire
     */
    MAJOR(("]\n"));
}