fs/nfs/nfs4_client_debug.c

/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*
 *  Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
 *  Use is subject to license terms.
 */

#include <sys/cred.h>
#include <sys/kstat.h>
#include <sys/list.h>
#include <sys/systm.h>
#include <sys/vfs.h>
#include <sys/vnode.h>
#include <sys/cmn_err.h>

#include <nfs/nfs4_clnt.h>
#include <nfs/rnode4.h>

/*
 * Recovery kstats
 */
typedef struct rkstat {
    kstat_named_t   badhandle;
    kstat_named_t   badowner;
    kstat_named_t   clientid;
    kstat_named_t   dead_file;
    kstat_named_t   delay;
    kstat_named_t   fail_relock;
    kstat_named_t   file_diff;
    kstat_named_t   no_grace;
    kstat_named_t   not_responding;
    kstat_named_t   opens_changed;
    kstat_named_t   siglost;
    kstat_named_t   unexp_action;
    kstat_named_t   unexp_errno;
    kstat_named_t   unexp_status;
    kstat_named_t   wrongsec;
    kstat_named_t   lost_state_bad_op;
} rkstat_t;

static rkstat_t rkstat_template = {
    { "badhandle",      KSTAT_DATA_ULONG },
    { "badowner",       KSTAT_DATA_ULONG },
    { "clientid",       KSTAT_DATA_ULONG },
    { "dead_file",      KSTAT_DATA_ULONG },
    { "delay",      KSTAT_DATA_ULONG },
    { "fail_relock",    KSTAT_DATA_ULONG },
    { "file_diff",      KSTAT_DATA_ULONG },
    { "no_grace",       KSTAT_DATA_ULONG },
    { "not_responding", KSTAT_DATA_ULONG },
    { "opens_changed",  KSTAT_DATA_ULONG },
    { "siglost",        KSTAT_DATA_ULONG },
    { "unexp_action",   KSTAT_DATA_ULONG },
    { "unexp_errno",    KSTAT_DATA_ULONG },
    { "unexp_status",   KSTAT_DATA_ULONG },
    { "wrongsec",       KSTAT_DATA_ULONG },
    { "bad_op",     KSTAT_DATA_ULONG },
};

/* maximum number of messages allowed on the mi's mi_msg_list */
int nfs4_msg_max = NFS4_MSG_MAX;
#define DEFAULT_LEASE   180

/*
 * Sets the appropiate fields of "ep", given "id" and various parameters.
 * Assumes that ep's fields have been initialized to zero/null, except for
 * re_type and mount point info, which are already set.
 */
static void
set_event(nfs4_event_type_t id, nfs4_revent_t *ep, mntinfo4_t *mi,
    rnode4_t *rp1, rnode4_t *rp2, uint_t count, pid_t pid, nfsstat4 nfs4_error,
    char *server1, char *why, nfs4_tag_type_t tag1, nfs4_tag_type_t tag2,
    seqid4 seqid1, seqid4 seqid2)
{
    int len;

    switch (id) {
    case RE_BAD_SEQID:
        ep->re_mi = mi;

        /* bad seqid'd file <path/component name> */
        if (rp1 && rp1->r_svnode.sv_name)
            ep->re_char1 = fn_path(rp1->r_svnode.sv_name);
        else
            ep->re_char1 = NULL;
        ep->re_rp1 = rp1;

        /* for LOCK/LOCKU */
        ep->re_pid = pid;

        ep->re_stat4 = nfs4_error;
        ep->re_tag1 = tag1;
        ep->re_tag2 = tag2;
        ep->re_seqid1 = seqid1;
        ep->re_seqid2 = seqid2;
        break;
    case RE_BADHANDLE:
        ASSERT(rp1 != NULL);

        /* dead file <path/component name> */
        if (rp1->r_svnode.sv_name)
            ep->re_char1 = fn_path(rp1->r_svnode.sv_name);
        else
            ep->re_char1 = NULL;
        ep->re_rp1 = rp1;
        break;
    case RE_CLIENTID:
        ep->re_mi = mi;

        /* the error we failed with */
        ep->re_uint = count;
        ep->re_stat4 = nfs4_error;
        break;
    case RE_DEAD_FILE:
        ASSERT(rp1 != NULL);

        /* dead file <path/component name> */
        if (rp1->r_svnode.sv_name)
            ep->re_char1 = fn_path(rp1->r_svnode.sv_name);
        else
            ep->re_char1 = NULL;
        ep->re_rp1 = rp1;

        /* why the file got killed */
        if (why) {
            len = strlen(why);
            ep->re_char2 = kmem_alloc(len + 1, KM_SLEEP);
            bcopy(why, ep->re_char2, len);
            ep->re_char2[len] = '\0';
        } else
            ep->re_char2 = NULL;

        ep->re_stat4 = nfs4_error;
        break;
    case RE_END:
        /* first rnode */
        if (rp1 && rp1->r_svnode.sv_name)
            ep->re_char1 = fn_path(rp1->r_svnode.sv_name);
        else
            ep->re_char1 = NULL;
        ep->re_rp1 = rp1;

        /* second rnode */
        if (rp2 && rp2->r_svnode.sv_name)
            ep->re_char2 = fn_path(rp2->r_svnode.sv_name);
        else
            ep->re_char2 = NULL;
        ep->re_rp2 = rp2;

        ep->re_mi = mi;
        break;
    case RE_FAIL_RELOCK:
        ASSERT(rp1 != NULL);

        /* error on fail relock */
        ep->re_uint = count;

        /* process that failed */
        ep->re_pid = pid;

        /* nfs4 error */
        ep->re_stat4 = nfs4_error;

        /* file <path/component name> */
        if (rp1->r_svnode.sv_name)
            ep->re_char1 = fn_path(rp1->r_svnode.sv_name);
        else
            ep->re_char1 = NULL;
        ep->re_rp1 = rp1;
        break;
    case RE_FAIL_REMAP_LEN:
        /* length of returned filehandle */
        ep->re_uint = count;
        break;
    case RE_FAIL_REMAP_OP:
        break;
    case RE_FAILOVER:
        /* server we're failing over to (if not picking original) */
        if (server1 != NULL) {
            len = strlen(server1);
            ep->re_char1 = kmem_alloc(len + 1, KM_SLEEP);
            bcopy(server1, ep->re_char1, len);
            ep->re_char1[len] = '\0';
        } else {
            ep->re_char1 = NULL;
        }
        break;
    case RE_FILE_DIFF:
        ASSERT(rp1 != NULL);

        /* dead file <path/component name> */
        if (rp1->r_svnode.sv_name)
            ep->re_char1 = fn_path(rp1->r_svnode.sv_name);
        else
            ep->re_char1 = NULL;
        ep->re_rp1 = rp1;
        break;
    case RE_LOST_STATE:
        ep->re_uint = count;        /* op number */
        if (rp1 && rp1->r_svnode.sv_name)
            ep->re_char1 = fn_path(rp1->r_svnode.sv_name);
        else
            ep->re_char1 = NULL;
        ep->re_rp1 = rp1;
        if (rp2 && rp2->r_svnode.sv_name)
            ep->re_char2 = fn_path(rp2->r_svnode.sv_name);
        else
            ep->re_char2 = NULL;
        ep->re_rp2 = rp2;
        break;
    case RE_OPENS_CHANGED:
        ep->re_mi = mi;

        /* original number of open files */
        ep->re_uint = count;
        /* new number of open files */
        ep->re_pid = pid;
        break;
    case RE_SIGLOST:
    case RE_SIGLOST_NO_DUMP:
        ASSERT(rp1 != NULL);

        /* file <path/component name> */
        if (rp1->r_svnode.sv_name)
            ep->re_char1 = fn_path(rp1->r_svnode.sv_name);
        else
            ep->re_char1 = NULL;
        ep->re_rp1 = rp1;
        ep->re_pid = pid;
        ep->re_uint = count;
        ep->re_stat4 = nfs4_error;
        break;
    case RE_START:
        /* file <path/component name> */
        if (rp1 && rp1->r_svnode.sv_name)
            ep->re_char1 = fn_path(rp1->r_svnode.sv_name);
        else
            ep->re_char1 = NULL;
        ep->re_rp1 = rp1;

        /* file <path/component name> */
        if (rp2 && rp2->r_svnode.sv_name)
            ep->re_char2 = fn_path(rp2->r_svnode.sv_name);
        else
            ep->re_char2 = NULL;
        ep->re_rp2 = rp2;

        ep->re_mi = mi;
        ep->re_uint = count;
        break;
    case RE_UNEXPECTED_ACTION:
    case RE_UNEXPECTED_ERRNO:
        /* the error that is unexpected */
        ep->re_uint = count;
        break;
    case RE_UNEXPECTED_STATUS:
        /* nfsstat4 error */
        ep->re_stat4 = nfs4_error;
        break;
    case RE_WRONGSEC:
        /* the error we failed with */
        ep->re_uint = count;

        /* file <path/component name> */
        if (rp1 && rp1->r_svnode.sv_name)
            ep->re_char1 = fn_path(rp1->r_svnode.sv_name);
        else
            ep->re_char1 = NULL;
        ep->re_rp1 = rp1;

        /* file <path/component name> */
        if (rp2 && rp2->r_svnode.sv_name)
            ep->re_char2 = fn_path(rp2->r_svnode.sv_name);
        else
            ep->re_char2 = NULL;
        ep->re_rp2 = rp2;
        break;
    case RE_LOST_STATE_BAD_OP:
        ep->re_uint = count;    /* the unexpected op */
        ep->re_pid = pid;
        ep->re_rp1 = rp1;
        if (rp1 != NULL && rp1->r_svnode.sv_name != NULL)
            ep->re_char1 = fn_path(rp1->r_svnode.sv_name);
        ep->re_rp2 = rp2;
        if (rp2 != NULL && rp2->r_svnode.sv_name != NULL)
            ep->re_char2 = fn_path(rp2->r_svnode.sv_name);
        break;
    case RE_REFERRAL:
        /* server we're being referred to */
        if (server1 != NULL) {
            len = strlen(server1);
            ep->re_char1 = kmem_alloc(len + 1, KM_SLEEP);
            bcopy(server1, ep->re_char1, len);
            ep->re_char1[len] = '\0';
        } else {
            ep->re_char1 = NULL;
        }
        break;
    default:
        break;
    }
}

/*
 * Sets the appropiate fields of the 'fact' for this 'id'.
 */
static void
set_fact(nfs4_fact_type_t id, nfs4_rfact_t *fp, nfsstat4 stat4,
    nfs4_recov_t raction, nfs_opnum4 op, bool_t reboot, int error,
    vnode_t *vp)
{
    rnode4_t *rp1;

    switch (id) {
    case RF_BADOWNER:
        fp->rf_op = op;
        fp->rf_reboot = reboot;
        fp->rf_stat4 = stat4;
        break;
    case RF_RENEW_EXPIRED:
        break;
    case RF_ERR:
        fp->rf_op = op;
        fp->rf_reboot = reboot;
        fp->rf_stat4 = stat4;
        fp->rf_action = raction;
        fp->rf_error = error;
        break;
    case RF_SRV_OK:
        break;
    case RF_SRV_NOT_RESPOND:
        break;
    case RF_SRVS_OK:
        break;
    case RF_SRVS_NOT_RESPOND:
        gethrestime(&fp->rf_time);
        break;
    case RF_DELMAP_CB_ERR:
        fp->rf_op = op;
        fp->rf_stat4 = stat4;

        rp1 = VTOR4(vp);
        fp->rf_rp1 = rp1;
        if (rp1 && rp1->r_svnode.sv_name)
            fp->rf_char1 = fn_path(rp1->r_svnode.sv_name);
        else
            fp->rf_char1 = NULL;
        break;
    case RF_SENDQ_FULL:
        break;
    default:
        zcmn_err(getzoneid(), CE_NOTE, "illegal fact %d", id);
        break;
    }
}

/*
 * Returns 1 if the event/fact is of a successful communication
 * from the server; 0 otherwise.
 */
static int
successful_comm(nfs4_debug_msg_t *msgp)
{
    if (msgp->msg_type == RM_EVENT) {
        switch (msgp->rmsg_u.msg_event.re_type) {
        case RE_BAD_SEQID:
        case RE_BADHANDLE:
        case RE_FAIL_REMAP_LEN:
        case RE_FAIL_REMAP_OP:
        case RE_FILE_DIFF:
        case RE_START:
        case RE_UNEXPECTED_ACTION:
        case RE_UNEXPECTED_ERRNO:
        case RE_UNEXPECTED_STATUS:
        case RE_WRONGSEC:
            return (1);
        case RE_CLIENTID:
        case RE_DEAD_FILE:
        case RE_END:
        case RE_FAIL_RELOCK:
        case RE_FAILOVER:
        case RE_LOST_STATE:
        case RE_OPENS_CHANGED:
        case RE_SIGLOST:
        case RE_SIGLOST_NO_DUMP:
        case RE_LOST_STATE_BAD_OP:
        case RE_REFERRAL:
            /* placeholder */
            return (0);
        default:
            return (0);
        }
    } else {
        switch (msgp->rmsg_u.msg_fact.rf_type) {
        case RF_BADOWNER:
        case RF_ERR:
        case RF_RENEW_EXPIRED:
        case RF_SRV_OK:
        case RF_SRVS_OK:
        case RF_DELMAP_CB_ERR:
            return (1);
        case RF_SRV_NOT_RESPOND:
        case RF_SRVS_NOT_RESPOND:
        case RF_SENDQ_FULL:
            return (0);
        default:
            return (0);
        }
    }
}

/*
 * Iterate backwards through the mi's mi_msg_list to find the earliest
 * message that we should find relevant facts to investigate.
 */
static nfs4_debug_msg_t *
find_beginning(nfs4_debug_msg_t *first_msg, mntinfo4_t *mi)
{
    nfs4_debug_msg_t    *oldest_msg, *cur_msg;
    time_t          lease;

    ASSERT(mutex_owned(&mi->mi_msg_list_lock));
    if (mi->mi_lease_period > 0)
        lease = 2 * mi->mi_lease_period;
    else
        lease = DEFAULT_LEASE;

    oldest_msg = first_msg;
    cur_msg = list_prev(&mi->mi_msg_list, first_msg);
    while (cur_msg &&
        first_msg->msg_time.tv_sec - cur_msg->msg_time.tv_sec < lease) {
        oldest_msg = cur_msg;
        if ((cur_msg->msg_type == RM_FACT) &&
            (cur_msg->rmsg_u.msg_fact.rf_type == RF_SRV_OK)) {
            /* find where we lost contact with the server */
            while (cur_msg) {
                if ((cur_msg->msg_type == RM_FACT) &&
                    (cur_msg->rmsg_u.msg_fact.rf_type ==
                    RF_SRV_NOT_RESPOND))
                    break;
                oldest_msg = cur_msg;
                cur_msg = list_prev(&mi->mi_msg_list, cur_msg);
            }
            /*
             * Find the first successful message before
             * we lost contact with the server.
             */
            if (cur_msg) {
                cur_msg = list_prev(&mi->mi_msg_list, cur_msg);
                while (cur_msg && !successful_comm(cur_msg)) {
                    oldest_msg = cur_msg;
                    cur_msg = list_prev(&mi->mi_msg_list,
                        cur_msg);
                }
            }
            /*
             * If we're not at the dummy head pointer,
             * set the oldest and current message.
             */
            if (cur_msg) {
                first_msg = cur_msg;
                oldest_msg = cur_msg;
                cur_msg = list_prev(&mi->mi_msg_list, cur_msg);
            }
        } else
            cur_msg = list_prev(&mi->mi_msg_list, cur_msg);
    }

    return (oldest_msg);
}

/*
 * Returns 1 if facts have been found; 0 otherwise.
 */
static int
get_facts(nfs4_debug_msg_t *msgp, nfs4_rfact_t *ret_fp, char **mnt_pt,
mntinfo4_t *mi)
{
    nfs4_debug_msg_t    *cur_msg, *oldest_msg;
    nfs4_rfact_t        *cur_fp;
    int         found_a_fact = 0;
    int         len;

    cur_msg = msgp;

    /* find the oldest msg to search backwards to */
    oldest_msg = find_beginning(cur_msg, mi);
    ASSERT(oldest_msg != NULL);

    /*
     * Create a fact sheet by searching from our current message
     * backwards to the 'oldest_msg', recording facts along the way
     * until we found facts that have been inspected by another time.
     */
    while (cur_msg && cur_msg != list_prev(&mi->mi_msg_list, oldest_msg)) {
        if (cur_msg->msg_type != RM_FACT) {
            cur_msg = list_prev(&mi->mi_msg_list, cur_msg);
            continue;
        }

        cur_fp = &cur_msg->rmsg_u.msg_fact;
        /*
         * If this fact has already been looked at, then so
         * have all preceding facts.  Return Now.
         */
        if (cur_fp->rf_status == RFS_INSPECT)
            return (found_a_fact);

        cur_fp->rf_status = RFS_INSPECT;
        found_a_fact = 1;
        switch (cur_fp->rf_type) {
        case RF_BADOWNER:
            break;
        case RF_ERR:
            /*
             * Don't want to overwrite a fact that was
             * previously found during our current search.
             */
            if (!ret_fp->rf_reboot)
                ret_fp->rf_reboot = cur_fp->rf_reboot;
            if (!ret_fp->rf_stat4)
                ret_fp->rf_stat4 = cur_fp->rf_stat4;
            if (!ret_fp->rf_action)
                ret_fp->rf_action = cur_fp->rf_action;
            break;
        case RF_RENEW_EXPIRED:
            if (cur_msg->msg_mntpt && !(*mnt_pt)) {
                len = strlen(cur_msg->msg_mntpt) + 1;
                *mnt_pt = kmem_alloc(len, KM_SLEEP);
                bcopy(cur_msg->msg_mntpt, *mnt_pt, len);
            }
            break;
        case RF_SRV_OK:
            break;
        case RF_SRV_NOT_RESPOND:
            /*
             * Okay to overwrite this fact as
             * we want the earliest time.
             */
            ret_fp->rf_time = cur_fp->rf_time;
            break;
        case RF_SRVS_OK:
            break;
        case RF_SRVS_NOT_RESPOND:
            break;
        case RF_DELMAP_CB_ERR:
            break;
        case RF_SENDQ_FULL:
            break;
        default:
            zcmn_err(getzoneid(), CE_NOTE,
                "get facts: illegal fact %d", cur_fp->rf_type);
            break;
        }
        cur_msg = list_prev(&mi->mi_msg_list, cur_msg);
    }

    return (found_a_fact);
}

/*
 * Returns 1 if this fact is identical to the last fact recorded
 * (only checks for a match within the last 2 lease periods).
 */
static int
facts_same(nfs4_debug_msg_t *cur_msg, nfs4_debug_msg_t *new_msg,
    mntinfo4_t *mi)
{
    nfs4_rfact_t    *fp1, *fp2;
    int     lease, len;

    ASSERT(mutex_owned(&mi->mi_msg_list_lock));
    if (mi->mi_lease_period > 0)
        lease = 2 * mi->mi_lease_period;
    else
        lease = DEFAULT_LEASE;

    fp2 = &new_msg->rmsg_u.msg_fact;

    while (cur_msg &&
        new_msg->msg_time.tv_sec - cur_msg->msg_time.tv_sec < lease) {
        if (cur_msg->msg_type != RM_FACT) {
            cur_msg = list_prev(&mi->mi_msg_list, cur_msg);
            continue;
        }
        fp1 = &cur_msg->rmsg_u.msg_fact;
        if (fp1->rf_type != fp2->rf_type)
            return (0);

        /* now actually compare the facts */
        if (fp1->rf_action != fp2->rf_action)
            return (0);
        if (fp1->rf_stat4 != fp2->rf_stat4)
            return (0);
        if (fp1->rf_reboot != fp2->rf_reboot)
            return (0);
        if (fp1->rf_op != fp2->rf_op)
            return (0);
        if (fp1->rf_time.tv_sec != fp2->rf_time.tv_sec)
            return (0);
        if (fp1->rf_error != fp2->rf_error)
            return (0);
        if (fp1->rf_rp1 != fp2->rf_rp1)
            return (0);
        if (cur_msg->msg_srv != NULL) {
            if (new_msg->msg_srv == NULL)
                return (0);
            len = strlen(cur_msg->msg_srv);
            if (strncmp(cur_msg->msg_srv, new_msg->msg_srv,
                len) != 0)
                return (0);
        } else if (new_msg->msg_srv != NULL) {
            return (0);
        }
        if (cur_msg->msg_mntpt != NULL) {
            if (new_msg->msg_mntpt == NULL)
                return (0);
            len = strlen(cur_msg->msg_mntpt);
            if (strncmp(cur_msg->msg_mntpt, new_msg->msg_mntpt,
                len) != 0)
                return (0);
        } else if (new_msg->msg_mntpt != NULL) {
            return (0);
        }
        if (fp1->rf_char1 != NULL) {
            if (fp2->rf_char1 == NULL)
                return (0);
            len = strlen(fp1->rf_char1);
            if (strncmp(fp1->rf_char1, fp2->rf_char1, len) != 0)
                return (0);
        } else if (fp2->rf_char1 != NULL) {
            return (0);
        }
        return (1);
    }

    return (0);
}

/*
 * Returns 1 if these two messages are identical; 0 otherwise.
 */
static int
events_same(nfs4_debug_msg_t *cur_msg, nfs4_debug_msg_t *new_msg,
    mntinfo4_t *mi)
{
    nfs4_revent_t   *ep1, *ep2;
    int     len;

    /* find the last event, bypassing all facts */
    while (cur_msg && cur_msg->msg_type != RM_EVENT)
        cur_msg = list_prev(&mi->mi_msg_list, cur_msg);

    if (!cur_msg)
        return (0);

    if (cur_msg->msg_type != RM_EVENT)
        return (0);

    ep1 = &cur_msg->rmsg_u.msg_event;
    ep2 = &new_msg->rmsg_u.msg_event;
    if (ep1->re_type != ep2->re_type)
        return (0);

    /*
     * Since we zalloc the buffer, then the two nfs4_debug_msg's
     * must match up even if all the fields weren't filled in
     * the first place.
     */
    if (ep1->re_mi != ep2->re_mi)
        return (0);
    if (ep1->re_uint != ep2->re_uint)
        return (0);
    if (ep1->re_stat4 != ep2->re_stat4)
        return (0);
    if (ep1->re_pid != ep2->re_pid)
        return (0);
    if (ep1->re_rp1 != ep2->re_rp1)
        return (0);
    if (ep1->re_rp2 != ep2->re_rp2)
        return (0);
    if (ep1->re_tag1 != ep2->re_tag1)
        return (0);
    if (ep1->re_tag2 != ep2->re_tag2)
        return (0);
    if (ep1->re_seqid1 != ep2->re_seqid1)
        return (0);
    if (ep1->re_seqid2 != ep2->re_seqid2)
        return (0);

    if (cur_msg->msg_srv != NULL) {
        if (new_msg->msg_srv == NULL)
            return (0);
        len = strlen(cur_msg->msg_srv);
        if (strncmp(cur_msg->msg_srv, new_msg->msg_srv, len) != 0)
            return (0);
    } else if (new_msg->msg_srv != NULL) {
        return (0);
    }

    if (ep1->re_char1 != NULL) {
        if (ep2->re_char1 == NULL)
            return (0);
        len = strlen(ep1->re_char1);
        if (strncmp(ep1->re_char1, ep2->re_char1, len) != 0)
            return (0);
    } else if (ep2->re_char1 != NULL) {
        return (0);
    }

    if (ep1->re_char2 != NULL) {
        if (ep2->re_char2 == NULL)
            return (0);
        len = strlen(ep1->re_char2);
        if (strncmp(ep1->re_char2, ep2->re_char2, len) != 0)
            return (0);
    } else if (ep2->re_char2 != NULL) {
        return (0);
    }

    if (cur_msg->msg_mntpt != NULL) {
        if (new_msg->msg_mntpt == NULL)
            return (0);
        len = strlen(cur_msg->msg_mntpt);
        if (strncmp(cur_msg->msg_mntpt, cur_msg->msg_mntpt, len) != 0)
            return (0);
    } else if (new_msg->msg_mntpt != NULL) {
        return (0);
    }

    return (1);
}

/*
 * Free up a recovery event.
 */
static void
free_event(nfs4_revent_t *ep)
{
    int len;

    if (ep->re_char1) {
        len = strlen(ep->re_char1) + 1;
        kmem_free(ep->re_char1, len);
    }
    if (ep->re_char2) {
        len = strlen(ep->re_char2) + 1;
        kmem_free(ep->re_char2, len);
    }
}

/*
 * Free up a recovery fact.
 */
static void
free_fact(nfs4_rfact_t *fp)
{
    int len;

    if (fp->rf_char1) {
        len = strlen(fp->rf_char1) + 1;
        kmem_free(fp->rf_char1, len);
    }
}

/*
 * Free up the message.
 */
void
nfs4_free_msg(nfs4_debug_msg_t *msg)
{
    int len;

    if (msg->msg_type == RM_EVENT)
        free_event(&msg->rmsg_u.msg_event);
    else
        free_fact(&msg->rmsg_u.msg_fact);

    if (msg->msg_srv) {
        len = strlen(msg->msg_srv) + 1;
        kmem_free(msg->msg_srv, len);
    }

    if (msg->msg_mntpt) {
        len = strlen(msg->msg_mntpt) + 1;
        kmem_free(msg->msg_mntpt, len);
    }

    /* free up the data structure itself */
    kmem_free(msg, sizeof (*msg));
}

/*
 * Prints out the interesting facts for recovery events:
 * -DEAD_FILE
 * -SIGLOST(_NO_DUMP)
 */
static void
print_facts(nfs4_debug_msg_t *msg, mntinfo4_t *mi)
{
    nfs4_rfact_t *fp;
    char *mount_pt;
    int len;

    if (msg->rmsg_u.msg_event.re_type != RE_DEAD_FILE &&
        msg->rmsg_u.msg_event.re_type != RE_SIGLOST &&
        msg->rmsg_u.msg_event.re_type != RE_SIGLOST_NO_DUMP)
        return;

    fp = kmem_zalloc(sizeof (*fp), KM_SLEEP);
    mount_pt = NULL;

    if (get_facts(msg, fp, &mount_pt, mi)) {
        char    time[256];


        if (fp->rf_time.tv_sec)
            (void) snprintf(time, 256, "%ld",
                (gethrestime_sec() - fp->rf_time.tv_sec)/60);
        zcmn_err(mi->mi_zone->zone_id, CE_NOTE,
            "!NFS4 FACT SHEET: %s%s %s%s %s %s%s%s %s%s",
            fp->rf_action ? "\n Action: " : "",
            fp->rf_action ? nfs4_recov_action_to_str(fp->rf_action) :
            "",
            fp->rf_stat4 ? "\n NFS4 error: " : "",
            fp->rf_stat4 ? nfs4_stat_to_str(fp->rf_stat4) : "",
            fp->rf_reboot ? "\n Suspected server reboot. " : "",
            fp->rf_time.tv_sec ? "\n Server was down for " : "",
            fp->rf_time.tv_sec ? time : "",
            fp->rf_time.tv_sec ? " minutes." : "",
            mount_pt ? " \n Client's lease expired on mount " : "",
            mount_pt ? mount_pt : "");
    }

    if (mount_pt) {
        len = strlen(mount_pt) + 1;
        kmem_free(mount_pt, len);
    }

    /* free the fact struct itself */
    if (fp)
        kmem_free(fp, sizeof (*fp));
}

/*
 * Print an event message to /var/adm/messages
 * The last argument to this fuction dictates the repeat status
 * of the event. If set to 1, it means that we are dumping this
 * event and it will _never_ be printed after this time. Else if
 * set to 0 it will be printed again.
 */
static void
queue_print_event(nfs4_debug_msg_t *msg, mntinfo4_t *mi, int dump)
{
    nfs4_revent_t       *ep;
    zoneid_t        zoneid;

    ep = &msg->rmsg_u.msg_event;
    zoneid = mi->mi_zone->zone_id;

    switch (ep->re_type) {
    case RE_BAD_SEQID:
        zcmn_err(zoneid, CE_NOTE, "![NFS4][Server: %s][Mntpt: %s]"
            "Operation %s for file %s (rnode_pt 0x%p), pid %d using "
            "seqid %d got %s.  Last good seqid was %d for "
            "operation %s.",
            msg->msg_srv, msg->msg_mntpt,
            nfs4_ctags[ep->re_tag1].ct_str, ep->re_char1,
            (void *)ep->re_rp1, ep->re_pid, ep->re_seqid1,
            nfs4_stat_to_str(ep->re_stat4), ep->re_seqid2,
            nfs4_ctags[ep->re_tag2].ct_str);
        break;
    case RE_BADHANDLE:
        ASSERT(ep->re_rp1 != NULL);
        if (ep->re_char1 != NULL) {
            zcmn_err(zoneid, CE_NOTE,
                "![NFS4][Server: %s][Mntpt: %s]"
                "server %s said filehandle was "
                "invalid for file: %s (rnode_pt 0x%p) on mount %s",
                msg->msg_srv, msg->msg_mntpt, msg->msg_srv,
                ep->re_char1, (void *)ep->re_rp1, msg->msg_mntpt);
        } else {
            zcmn_err(zoneid, CE_NOTE,
                "![NFS4][Server: %s][Mntpt: %s]"
                "server %s said filehandle was "
                "invalid for file: (rnode_pt 0x%p) on mount %s"
                " for fh:", msg->msg_srv, msg->msg_mntpt,
                msg->msg_srv, (void *)ep->re_rp1, msg->msg_mntpt);
            sfh4_printfhandle(ep->re_rp1->r_fh);
        }
        break;
    case RE_CLIENTID:
        zcmn_err(zoneid, CE_NOTE, "![NFS4][Server: %s][Mntpt: %s]"
            "Can't recover clientid on mount point %s "
            "(mi 0x%p) due to error %d (%s), for server %s.  Marking "
            "file system as unusable.",
            msg->msg_srv, msg->msg_mntpt, msg->msg_mntpt,
            (void *)ep->re_mi, ep->re_uint,
            nfs4_stat_to_str(ep->re_stat4),
            msg->msg_srv);
        break;
    case RE_DEAD_FILE:
        zcmn_err(zoneid, CE_NOTE, "![NFS4][Server: %s][Mntpt: %s]"
            "File %s (rnode_pt: %p) was closed due to NFS "
            "recovery error on server %s(%s %s)", msg->msg_srv,
            msg->msg_mntpt, ep->re_char1, (void *)ep->re_rp1,
            msg->msg_srv, ep->re_char2 ? ep->re_char2 : "",
            ep->re_stat4 ? nfs4_stat_to_str(ep->re_stat4) : "");
        break;
    case RE_END:
        zcmn_err(zoneid, CE_NOTE, "![NFS4][Server: %s][Mntpt: %s]"
            "NFS Recovery done for mount %s (mi 0x%p) "
            "on server %s, rnode_pt1 %s (0x%p), "
            "rnode_pt2 %s (0x%p)", msg->msg_srv, msg->msg_mntpt,
            msg->msg_mntpt, (void *)ep->re_mi, msg->msg_srv,
            ep->re_char1, (void *)ep->re_rp1, ep->re_char2,
            (void *)ep->re_rp2);
        break;
    case RE_FAIL_RELOCK:
        zcmn_err(zoneid, CE_NOTE, "![NFS4][Server: %s][Mntpt: %s]"
            "Couldn't reclaim lock for pid %d for "
            "file %s (rnode_pt 0x%p) on (server %s): error %d",
            msg->msg_srv, msg->msg_mntpt, ep->re_pid, ep->re_char1,
            (void *)ep->re_rp1, msg->msg_srv,
            ep->re_uint ? ep->re_uint : ep->re_stat4);
        break;
    case RE_FAIL_REMAP_LEN:
        zcmn_err(zoneid, CE_NOTE, "![NFS4][Server: %s][Mntpt: %s]"
            "remap_lookup: server %s returned bad "
            "fhandle length (%d)", msg->msg_srv, msg->msg_mntpt,
            msg->msg_srv, ep->re_uint);
        break;
    case RE_FAIL_REMAP_OP:
        zcmn_err(zoneid, CE_NOTE, "![NFS4][Server: %s][Mntpt: %s]"
            "remap_lookup: didn't get expected OP_GETFH"
            " for server %s", msg->msg_srv, msg->msg_mntpt,
            msg->msg_srv);
        break;
    case RE_FAILOVER:
        if (ep->re_char1)
            zcmn_err(zoneid, CE_NOTE,
                "![NFS4][Server: %s][Mntpt: %s]"
                "failing over from %s to %s", msg->msg_srv,
                msg->msg_mntpt, msg->msg_srv, ep->re_char1);
        else
            zcmn_err(zoneid, CE_NOTE,
                "![NFS4][Server: %s][Mntpt: %s]"
                "NFS4: failing over: selecting "
                "original server %s", msg->msg_srv, msg->msg_mntpt,
                msg->msg_srv);
        break;
    case RE_FILE_DIFF:
        zcmn_err(zoneid, CE_NOTE, "![NFS4][Server: %s][Mntpt: %s]"
            "File %s (rnode_pt: %p) on server %s was closed "
            "and failed attempted failover since its is different than "
            "the original file", msg->msg_srv, msg->msg_mntpt,
            ep->re_char1, (void *)ep->re_rp1, msg->msg_srv);
        break;
    case RE_LOST_STATE:
        zcmn_err(zoneid, CE_NOTE, "![NFS4][Server: %s][Mntpt: %s]"
            "Lost %s request for fs %s, file %s (rnode_pt: 0x%p), "
            "dir %s (0x%p) for server %s", msg->msg_srv, msg->msg_mntpt,
            nfs4_op_to_str(ep->re_uint), msg->msg_mntpt,
            ep->re_char1, (void *)ep->re_rp1, ep->re_char2,
            (void *)ep->re_rp2, msg->msg_srv);
        break;
    case RE_OPENS_CHANGED:
        zcmn_err(zoneid, CE_NOTE, "![NFS4][Server: %s][Mntpt: %s]"
            "The number of open files to reopen changed "
            "for mount %s mi 0x%p (old %d, new %d) on server %s",
            msg->msg_srv, msg->msg_mntpt, msg->msg_mntpt,
            (void *)ep->re_mi, ep->re_uint, ep->re_pid, msg->msg_srv);
        break;
    case RE_SIGLOST:
    case RE_SIGLOST_NO_DUMP:
        if (ep->re_uint)
            zcmn_err(zoneid, CE_NOTE,
                "![NFS4][Server: %s][Mntpt: %s]"
                "Process %d lost its locks on "
                "file %s (rnode_pt: %p) due to NFS recovery error "
                "(%d) on server %s.", msg->msg_srv, msg->msg_mntpt,
                ep->re_pid, ep->re_char1, (void *)ep->re_rp1,
                ep->re_uint, msg->msg_srv);
        else
            zcmn_err(zoneid, CE_NOTE,
                "![NFS4][Server: %s][Mntpt: %s]"
                "Process %d lost its locks on "
                "file %s (rnode_pt: %p) due to NFS recovery error "
                "(%s) on server %s.", msg->msg_srv, msg->msg_mntpt,
                ep->re_pid, ep->re_char1, (void *)ep->re_rp1,
                nfs4_stat_to_str(ep->re_stat4), msg->msg_srv);
        break;
    case RE_START:
        zcmn_err(zoneid, CE_NOTE, "![NFS4][Server: %s][Mntpt: %s]"
            "NFS Starting recovery for mount %s "
            "(mi 0x%p mi_recovflags [0x%x]) on server %s, "
            "rnode_pt1 %s (0x%p), rnode_pt2 %s (0x%p)", msg->msg_srv,
            msg->msg_mntpt, msg->msg_mntpt, (void *)ep->re_mi,
            ep->re_uint, msg->msg_srv, ep->re_char1, (void *)ep->re_rp1,
            ep->re_char2, (void *)ep->re_rp2);
        break;
    case RE_UNEXPECTED_ACTION:
        zcmn_err(zoneid, CE_NOTE, "![NFS4][Server: %s][Mntpt: %s]"
            "NFS recovery: unexpected action (%s) on server %s",
            msg->msg_srv, msg->msg_mntpt,
            nfs4_recov_action_to_str(ep->re_uint), msg->msg_srv);
        break;
    case RE_UNEXPECTED_ERRNO:
        zcmn_err(zoneid, CE_NOTE, "![NFS4][Server: %s][Mntpt: %s]"
            "NFS recovery: unexpected errno (%d) on server %s",
            msg->msg_srv, msg->msg_mntpt, ep->re_uint, msg->msg_srv);
        break;
    case RE_UNEXPECTED_STATUS:
        zcmn_err(zoneid, CE_NOTE, "![NFS4][Server: %s][Mntpt: %s]"
            "NFS recovery: unexpected NFS status code (%s) "
            "on server %s", msg->msg_srv, msg->msg_mntpt,
            nfs4_stat_to_str(ep->re_stat4),
            msg->msg_srv);
        break;
    case RE_WRONGSEC:
        zcmn_err(zoneid, CE_NOTE, "![NFS4][Server: %s][Mntpt: %s]"
            "NFS can't recover from NFS4ERR_WRONGSEC."
            "  error %d for server %s: rnode_pt1 %s (0x%p)"
            " rnode_pt2 %s (0x%p)", msg->msg_srv, msg->msg_mntpt,
            ep->re_uint, msg->msg_srv, ep->re_char1, (void *)ep->re_rp1,
            ep->re_char2, (void *)ep->re_rp2);
        break;
    case RE_LOST_STATE_BAD_OP:
        zcmn_err(zoneid, CE_NOTE, "![NFS4][Server: %s][Mntpt: %s]"
            "NFS lost state with unrecognized op (%d)."
            "  fs %s, server %s, pid %d, file %s (rnode_pt: 0x%p), "
            "dir %s (0x%p)", msg->msg_srv, msg->msg_mntpt,
            ep->re_uint, msg->msg_mntpt, msg->msg_srv, ep->re_pid,
            ep->re_char1, (void *)ep->re_rp1, ep->re_char2,
            (void *)ep->re_rp2);
        break;
    case RE_REFERRAL:
        if (ep->re_char1)
            zcmn_err(zoneid, CE_NOTE,
                "![NFS4][Server: %s][Mntpt: %s]"
                "being referred from %s to %s", msg->msg_srv,
                msg->msg_mntpt, msg->msg_srv, ep->re_char1);
        else
            zcmn_err(zoneid, CE_NOTE,
                "![NFS4][Server: %s][Mntpt: %s]"
                "NFS4: being referred from %s to unknown server",
                msg->msg_srv, msg->msg_mntpt, msg->msg_srv);
        break;
    default:
        zcmn_err(zoneid, CE_WARN,
            "!queue_print_event: illegal event %d", ep->re_type);
        break;
    }

    print_facts(msg, mi);

    /*
     * If set this event will not be printed again and is considered
     * dumped.
     */
    if (dump)
        msg->msg_status = NFS4_MS_NO_DUMP;
}

/*
 * Print a fact message to /var/adm/messages
 */
static void
queue_print_fact(nfs4_debug_msg_t *msg, int dump)
{
    nfs4_rfact_t    *fp;
    zoneid_t    zoneid;

    fp = &msg->rmsg_u.msg_fact;
    zoneid = getzoneid();

    switch (fp->rf_type) {
    case RF_BADOWNER:
        zcmn_err(zoneid, CE_NOTE, "![NFS4][Server: %s][Mntpt: %s]"
            "NFSMAPID_DOMAIN does not match the server: %s domain\n"
            "Please check configuration", msg->msg_srv, msg->msg_mntpt,
            msg->msg_srv);
        break;
    case RF_ERR:
        if (fp->rf_error)
            zcmn_err(zoneid, CE_NOTE,
                "![NFS4][Server: %s][Mntpt: %s]NFS op %s got "
                "error %d causing recovery action %s.%s",
                msg->msg_srv, msg->msg_mntpt,
                nfs4_op_to_str(fp->rf_op), fp->rf_error,
                nfs4_recov_action_to_str(fp->rf_action),
                fp->rf_reboot ?
                "  Client also suspects that the server rebooted,"
                " or experienced a network partition." : "");
        else
            zcmn_err(zoneid, CE_NOTE,
                "![NFS4][Server: %s][Mntpt: %s]NFS op %s got "
                "error %s causing recovery action %s.%s",
                msg->msg_srv, msg->msg_mntpt,
                nfs4_op_to_str(fp->rf_op),
                nfs4_stat_to_str(fp->rf_stat4),
                nfs4_recov_action_to_str(fp->rf_action),
                fp->rf_reboot ?
                "  Client also suspects that the server rebooted,"
                " or experienced a network partition." : "");
        break;
    case RF_RENEW_EXPIRED:
        zcmn_err(zoneid, CE_NOTE, "![NFS4][Server: %s][Mntpt: %s]"
            "NFS4 renew thread detected client's "
            "lease has expired. Current open files/locks/IO may fail",
            msg->msg_srv, msg->msg_mntpt);
        break;
    case RF_SRV_NOT_RESPOND:
        zcmn_err(zoneid, CE_NOTE, "![NFS4][Server: %s][Mntpt: %s]"
            "NFS server %s not responding; still trying\n",
            msg->msg_srv, msg->msg_mntpt, msg->msg_srv);
        break;
    case RF_SRV_OK:
        zcmn_err(zoneid, CE_NOTE, "![NFS4][Server: %s][Mntpt: %s]"
            "NFS server %s ok", msg->msg_srv, msg->msg_mntpt,
            msg->msg_srv);
        break;
    case RF_SRVS_NOT_RESPOND:
        zcmn_err(zoneid, CE_NOTE, "![NFS4][Server: %s][Mntpt: %s]"
            "NFS servers %s not responding; still trying", msg->msg_srv,
            msg->msg_mntpt, msg->msg_srv);
        break;
    case RF_SRVS_OK:
        zcmn_err(zoneid, CE_NOTE, "![NFS4][Server: %s][Mntpt: %s]"
            "NFS servers %s ok", msg->msg_srv, msg->msg_mntpt,
            msg->msg_srv);
        break;
    case RF_DELMAP_CB_ERR:
        zcmn_err(zoneid, CE_NOTE, "![NFS4][Server: %s][Mntpt: %s]"
            "NFS op %s got error %s when executing delmap on file %s "
            "(rnode_pt 0x%p).",
            msg->msg_srv, msg->msg_mntpt, nfs4_op_to_str(fp->rf_op),
            nfs4_stat_to_str(fp->rf_stat4), fp->rf_char1,
            (void *)fp->rf_rp1);
        break;
    case RF_SENDQ_FULL:
        zcmn_err(zoneid, CE_NOTE, "![NFS4][Server: %s][Mntpt: %s]"
            "send queue to NFS server %s is full; still trying\n",
            msg->msg_srv, msg->msg_mntpt, msg->msg_srv);
        break;

    default:
        zcmn_err(zoneid, CE_WARN, "!queue_print_fact: illegal fact %d",
            fp->rf_type);
    }

    /*
     * If set this fact will not be printed again and is considered
     * dumped.
     */
    if (dump)
        msg->msg_status = NFS4_MS_NO_DUMP;
}

/*
 * Returns 1 if the entire queue should be dumped, 0 otherwise.
 */
static int
id_to_dump_queue(nfs4_event_type_t id)
{
    switch (id) {
    case RE_DEAD_FILE:
    case RE_SIGLOST:
    case RE_WRONGSEC:
    case RE_CLIENTID:
        return (1);
    default:
        return (0);
    }
}

/*
 * Returns 1 if the event (but not the entire queue) should be printed;
 * 0 otherwise.
 */
static int
id_to_dump_solo_event(nfs4_event_type_t id)
{
    switch (id) {
    case RE_BAD_SEQID:
    case RE_BADHANDLE:
    case RE_FAIL_REMAP_LEN:
    case RE_FAIL_REMAP_OP:
    case RE_FAILOVER:
    case RE_OPENS_CHANGED:
    case RE_SIGLOST_NO_DUMP:
    case RE_UNEXPECTED_ACTION:
    case RE_UNEXPECTED_ERRNO:
    case RE_UNEXPECTED_STATUS:
    case RE_LOST_STATE_BAD_OP:
    case RE_REFERRAL:
        return (1);
    default:
        return (0);
    }
}

/*
 * Returns 1 if the fact (but not the entire queue) should be printed;
 * 0 otherwise.
 */
static int
id_to_dump_solo_fact(nfs4_fact_type_t id)
{
    switch (id) {
    case RF_SRV_NOT_RESPOND:
    case RF_SRV_OK:
    case RF_SRVS_NOT_RESPOND:
    case RF_SRVS_OK:
    case RF_SENDQ_FULL:
        return (1);
    default:
        return (0);
    }
}

/*
 * Update a kernel stat
 */
static void
update_recov_kstats(nfs4_debug_msg_t *msg, mntinfo4_t *mi)
{
    rkstat_t    *rsp;

    if (!mi->mi_recov_ksp)
        return;

    rsp = (rkstat_t *)mi->mi_recov_ksp->ks_data;

    if (msg->msg_type == RM_EVENT) {
        switch (msg->rmsg_u.msg_event.re_type) {
        case RE_BADHANDLE:
            rsp->badhandle.value.ul++;
            break;
        case RE_CLIENTID:
            rsp->clientid.value.ul++;
            break;
        case RE_DEAD_FILE:
            rsp->dead_file.value.ul++;
            break;
        case RE_FAIL_RELOCK:
            rsp->fail_relock.value.ul++;
            break;
        case RE_FILE_DIFF:
            rsp->file_diff.value.ul++;
            break;
        case RE_OPENS_CHANGED:
            rsp->opens_changed.value.ul++;
            break;
        case RE_SIGLOST:
        case RE_SIGLOST_NO_DUMP:
            rsp->siglost.value.ul++;
            break;
        case RE_UNEXPECTED_ACTION:
            rsp->unexp_action.value.ul++;
            break;
        case RE_UNEXPECTED_ERRNO:
            rsp->unexp_errno.value.ul++;
            break;
        case RE_UNEXPECTED_STATUS:
            rsp->unexp_status.value.ul++;
            break;
        case RE_WRONGSEC:
            rsp->wrongsec.value.ul++;
            break;
        case RE_LOST_STATE_BAD_OP:
            rsp->lost_state_bad_op.value.ul++;
            break;
        default:
            break;
        }
    } else if (msg->msg_type == RM_FACT) {
        switch (msg->rmsg_u.msg_fact.rf_type) {
        case RF_BADOWNER:
            rsp->badowner.value.ul++;
            break;
        case RF_SRV_NOT_RESPOND:
            rsp->not_responding.value.ul++;
            break;
        default:
            break;
        }
    }
}

/*
 * Dump the mi's mi_msg_list of recovery messages.
 */
static void
dump_queue(mntinfo4_t *mi, nfs4_debug_msg_t *msg)
{
    nfs4_debug_msg_t *tmp_msg;

    ASSERT(mutex_owned(&mi->mi_msg_list_lock));

    /* update kstats */
    update_recov_kstats(msg, mi);

    /*
     * If we aren't supposed to dump the queue then see if we
     * should just print this single message, then return.
     */
    if (!id_to_dump_queue(msg->rmsg_u.msg_event.re_type)) {
        if (id_to_dump_solo_event(msg->rmsg_u.msg_event.re_type))
            queue_print_event(msg, mi, 0);
        return;
    }

    /*
     * Write all events/facts in the queue that haven't been
     * previously written to disk.
     */
    tmp_msg = list_head(&mi->mi_msg_list);
    while (tmp_msg) {
        if (tmp_msg->msg_status == NFS4_MS_DUMP) {
            if (tmp_msg->msg_type == RM_EVENT)
                queue_print_event(tmp_msg, mi, 1);
            else if (tmp_msg->msg_type == RM_FACT)
                queue_print_fact(tmp_msg, 1);
        }
        tmp_msg = list_next(&mi->mi_msg_list, tmp_msg);
    }
}

/*
 * Places the event into mi's debug recovery message queue.  Some of the
 * fields can be overloaded to be a generic value, depending on the event
 * type.  These include "count", "why".
 */
void
nfs4_queue_event(nfs4_event_type_t id, mntinfo4_t *mi, char *server1,
    uint_t count, vnode_t *vp1, vnode_t *vp2, nfsstat4 nfs4_error,
    char *why, pid_t pid, nfs4_tag_type_t tag1, nfs4_tag_type_t tag2,
    seqid4 seqid1, seqid4 seqid2)
{
    nfs4_debug_msg_t    *msg;
    nfs4_revent_t       *ep;
    char            *cur_srv;
    rnode4_t        *rp1 = NULL, *rp2 = NULL;
    refstr_t        *mntpt;

    ASSERT(mi != NULL);
    if (vp1)
        rp1 = VTOR4(vp1);
    if (vp2)
        rp2 = VTOR4(vp2);

    /*
     * Initialize the message with the relevant server/mount_pt/time
     * information. Also place the relevent event related info.
     */
    msg = kmem_zalloc(sizeof (*msg), KM_SLEEP);
    msg->msg_type = RM_EVENT;
    msg->msg_status = NFS4_MS_DUMP;
    ep = &msg->rmsg_u.msg_event;
    ep->re_type = id;
    gethrestime(&msg->msg_time);

    cur_srv = mi->mi_curr_serv->sv_hostname;
    msg->msg_srv = strdup(cur_srv);
    mntpt = vfs_getmntpoint(mi->mi_vfsp);
    msg->msg_mntpt = strdup(refstr_value(mntpt));
    refstr_rele(mntpt);

    set_event(id, ep, mi, rp1, rp2, count, pid, nfs4_error, server1,
        why, tag1, tag2, seqid1, seqid2);

    mutex_enter(&mi->mi_msg_list_lock);

    /* if this event is the same as the last event, drop it */
    if (events_same(list_tail(&mi->mi_msg_list), msg, mi)) {
        mutex_exit(&mi->mi_msg_list_lock);
        nfs4_free_msg(msg);
        return;
    }

    /* queue the message at the end of the list */
    list_insert_tail(&mi->mi_msg_list, msg);

    dump_queue(mi, msg);

    if (mi->mi_msg_count == nfs4_msg_max) {
        nfs4_debug_msg_t *rm_msg;

        /* remove the queue'd message at the front of the list */
        rm_msg = list_head(&mi->mi_msg_list);
        list_remove(&mi->mi_msg_list, rm_msg);
        mutex_exit(&mi->mi_msg_list_lock);
        nfs4_free_msg(rm_msg);
    } else {
        mi->mi_msg_count++;
        mutex_exit(&mi->mi_msg_list_lock);
    }
}

/*
 * Places the fact into mi's debug recovery messages queue.
 */
void
nfs4_queue_fact(nfs4_fact_type_t fid, mntinfo4_t *mi, nfsstat4 stat4,
    nfs4_recov_t raction, nfs_opnum4 op, bool_t reboot, char *srvname,
    int error, vnode_t *vp)
{
    nfs4_debug_msg_t    *msg;
    nfs4_rfact_t        *fp;
    char            *cur_srv;
    refstr_t        *mntpt;

    /*
     * Initialize the message with the relevant server/mount_pt/time
     * information. Also place the relevant fact related info.
     */
    msg = kmem_zalloc(sizeof (*msg), KM_SLEEP);
    msg->msg_type = RM_FACT;
    msg->msg_status = NFS4_MS_DUMP;
    gethrestime(&msg->msg_time);

    if (srvname)
        cur_srv = srvname;
    else
        cur_srv = mi->mi_curr_serv->sv_hostname;

    msg->msg_srv = strdup(cur_srv);
    mntpt = vfs_getmntpoint(mi->mi_vfsp);
    msg->msg_mntpt = strdup(refstr_value(mntpt));
    refstr_rele(mntpt);

    fp = &msg->rmsg_u.msg_fact;
    fp->rf_type = fid;
    fp->rf_status = RFS_NO_INSPECT;
    set_fact(fid, fp, stat4, raction, op, reboot, error, vp);

    update_recov_kstats(msg, mi);

    mutex_enter(&mi->mi_msg_list_lock);

    /* if this fact is the same as the last fact, drop it */
    if (facts_same(list_tail(&mi->mi_msg_list), msg, mi)) {
        mutex_exit(&mi->mi_msg_list_lock);
        nfs4_free_msg(msg);
        return;
    }

    /* queue the message at the end of the list */
    list_insert_tail(&mi->mi_msg_list, msg);

    if (id_to_dump_solo_fact(msg->rmsg_u.msg_fact.rf_type))
        queue_print_fact(msg, 0);

    if (mi->mi_msg_count == nfs4_msg_max) {
        nfs4_debug_msg_t *rm_msg;

        /* remove the queue'd message at the front of the list */
        rm_msg = list_head(&mi->mi_msg_list);
        list_remove(&mi->mi_msg_list, rm_msg);
        mutex_exit(&mi->mi_msg_list_lock);
        nfs4_free_msg(rm_msg);
    } else {
        mi->mi_msg_count++;
        mutex_exit(&mi->mi_msg_list_lock);
    }
}

/*
 * Initialize the 'mi_recov_kstat' kstat.
 */
void
nfs4_mnt_recov_kstat_init(vfs_t *vfsp)
{
    mntinfo4_t *mi = VFTOMI4(vfsp);
    kstat_t     *ksp;
    zoneid_t    zoneid = mi->mi_zone->zone_id;

    /*
     * Create the version specific kstats.
     *
     * PSARC 2001/697 Contract Private Interface
     * All nfs kstats are under SunMC contract
     * Please refer to the PSARC listed above and contact
     * SunMC before making any changes!
     *
     * Changes must be reviewed by Solaris File Sharing
     * Changes must be communicated to contract-2001-697@sun.com
     *
     */

    if ((ksp = kstat_create_zone("nfs", getminor(vfsp->vfs_dev),
        "mi_recov_kstat", "misc", KSTAT_TYPE_NAMED,
        sizeof (rkstat_t) / sizeof (kstat_named_t),
        KSTAT_FLAG_WRITABLE, zoneid)) == NULL) {
        mi->mi_recov_ksp = NULL;
        zcmn_err(GLOBAL_ZONEID, CE_NOTE,
            "!mi_recov_kstat for mi %p failed\n",
            (void *)mi);
        return;
    }
    if (zoneid != GLOBAL_ZONEID)
        kstat_zone_add(ksp, GLOBAL_ZONEID);
    mi->mi_recov_ksp = ksp;
    bcopy(&rkstat_template, ksp->ks_data, sizeof (rkstat_t));
    kstat_install(ksp);
}

/*
 * Increment the "delay" kstat.
 */
void
nfs4_mi_kstat_inc_delay(mntinfo4_t *mi)
{
    rkstat_t    *rsp;

    if (!mi->mi_recov_ksp)
        return;

    rsp = (rkstat_t *)mi->mi_recov_ksp->ks_data;
    rsp->delay.value.ul++;
}

/*
 * Increment the "no_grace" kstat.
 */
void
nfs4_mi_kstat_inc_no_grace(mntinfo4_t *mi)
{
    rkstat_t    *rsp;

    if (!mi->mi_recov_ksp)
        return;

    rsp = (rkstat_t *)mi->mi_recov_ksp->ks_data;
    rsp->no_grace.value.ul++;
}