sun4/os/x_call.c

	x_call.c revision 7c478bd95313f5f23a4c958a745db2134aa03244
/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License, Version 1.0 only
 * (the "License").  You may not use this file except in compliance
 * with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*
 * Copyright 2005 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

#pragma ident   "%Z%%M% %I% %E% SMI"

#include <sys/systm.h>
#include <sys/archsystm.h>
#include <sys/machsystm.h>
#include <sys/cpuvar.h>
#include <sys/intreg.h>
#include <sys/x_call.h>
#include <sys/cmn_err.h>
#include <sys/membar.h>
#include <sys/disp.h>
#include <sys/debug.h>
#include <sys/privregs.h>
#include <sys/xc_impl.h>
#include <sys/ivintr.h>
#include <sys/dmv.h>

#ifdef DEBUG
uint_t x_dstat[NCPU][XC_LOOP_EXIT+1];
uint_t x_rstat[NCPU][4];
#endif /* DEBUG */

static int xc_serv_inum;    /* software interrupt number for xc_serv() */
static int xc_loop_inum;    /* software interrupt number for xc_loop() */
kmutex_t xc_sys_mutex;      /* protect xcall session and xc_mbox */
int xc_spl_enter[NCPU];     /* protect sending x-call */
static int xc_holder = -1; /* the cpu who initiates xc_attention, 0 is valid */

/*
 * Mail box for handshaking and xcall request; protected by xc_sys_mutex
 */
static struct xc_mbox {
    xcfunc_t *xc_func;
    uint64_t xc_arg1;
    uint64_t xc_arg2;
    cpuset_t xc_cpuset;
    volatile uint_t xc_state;
} xc_mbox[NCPU];

uint64_t xc_tick_limit;     /* send_mondo() tick limit value */
uint64_t xc_tick_limit_scale = 1;   /* scale used to increase the limit */
uint64_t xc_tick_jump_limit;    /* send_mondo() irregular tick jump limit */

/* timeout value for xcalls to be received by the target CPU */
uint64_t xc_mondo_time_limit;

/* timeout value for xcall functions to be executed on the target CPU */
uint64_t xc_func_time_limit;

uint64_t xc_scale = 1;  /* scale used to calculate timeout limits */

uint_t sendmondo_in_recover;

/*
 * sending x-calls
 */
void    init_mondo(xcfunc_t *func, uint64_t arg1, uint64_t arg2);
void    send_one_mondo(int cpuid);
void    send_mondo_set(cpuset_t set);

/*
 * xc_init - initialize x-call related locks
 */
void
xc_init(void)
{
#ifdef DEBUG
    int pix;
#endif /* DEBUG */

    mutex_init(&xc_sys_mutex, NULL, MUTEX_SPIN,
        (void *)ipltospl(XCALL_PIL));

#ifdef DEBUG
    /* Initialize for all possible CPUs. */
    for (pix = 0; pix < NCPU; pix++) {
        XC_STAT_INIT(pix);
    }
#endif /* DEBUG */

    xc_serv_inum = add_softintr(XCALL_PIL, (softintrfunc)xc_serv, 0);
    xc_loop_inum = add_softintr(XCALL_PIL, (softintrfunc)xc_loop, 0);

    /*
     * Initialize the calibrated tick limit for send_mondo.
     * The value represents the maximum tick count to wait.
     */
    xc_tick_limit =
        ((uint64_t)sys_tick_freq * XC_SEND_MONDO_MSEC) / 1000;
    xc_tick_jump_limit = xc_tick_limit / 32;
    xc_tick_limit *= xc_tick_limit_scale;

    /*
     * Maximum number of loops to wait before timing out in xc_attention.
     */
    xc_mondo_time_limit = cpunodes[CPU->cpu_id].clock_freq * xc_scale;

    /*
     * Maximum number of loops to wait for a xcall function to be
     * executed on the target CPU.  Default to 10 times the value
     * of xc_mondo_time_limit.
     */
    xc_func_time_limit = xc_mondo_time_limit * 10;
}

/*
 * The following routines basically provide callers with two kinds of
 * inter-processor interrupt services:
 *  1. cross calls (x-calls) - requests are handled at target cpu's TL=0
 *  2. cross traps (c-traps) - requests are handled at target cpu's TL>0
 *
 * Although these routines protect the services from migrating to other cpus
 * "after" they are called, it is the caller's choice or responsibility to
 * prevent the cpu migration "before" calling them.
 *
 * X-call routines:
 *
 *  xc_one()  - send a request to one processor
 *  xc_some() - send a request to some processors
 *  xc_all()  - send a request to all processors
 *
 *  Their common parameters:
 *      func - a TL=0 handler address
 *      arg1 and arg2  - optional
 *
 *  The services provided by x-call routines allow callers
 *  to send a request to target cpus to execute a TL=0
 *  handler.
 *  The interface of the registers of the TL=0 handler:
 *      %o0: arg1
 *      %o1: arg2
 *
 * X-trap routines:
 *
 *  xt_one()  - send a request to one processor
 *  xt_some() - send a request to some processors
 *  xt_all()  - send a request to all processors
 *
 *  Their common parameters:
 *      func - a TL>0 handler address or an interrupt number
 *      arg1, arg2
 *             optional when "func" is an address;
 *             0        when "func" is an interrupt number
 *
 *  If the request of "func" is a kernel address, then
 *  the target cpu will execute the request of "func" with
 *  args at "TL>0" level.
 *  The interface of the registers of the TL>0 handler:
 *      %g1: arg1
 *      %g2: arg2
 *
 *  If the request of "func" is not a kernel address, then it has
 *  to be an assigned interrupt number through add_softintr().
 *  An interrupt number is an index to the interrupt vector table,
 *  which entry contains an interrupt handler address with its
 *  corresponding interrupt level and argument.
 *  The target cpu will arrange the request to be serviced according
 *  to its pre-registered information.
 *  args are assumed to be zeros in this case.
 *
 * In addition, callers are allowed to capture and release cpus by
 * calling the routines: xc_attention() and xc_dismissed().
 */

/*
 * xt_one - send a "x-trap" to a cpu
 */
void
xt_one(int cix, xcfunc_t *func, uint64_t arg1, uint64_t arg2)
{
    if (!CPU_IN_SET(cpu_ready_set, cix)) {
        return;
    }
    xt_one_unchecked(cix, func, arg1, arg2);
}

/*
 * xt_one_unchecked - send a "x-trap" to a cpu without checking for its
 * existance in cpu_ready_set
 */
void
xt_one_unchecked(int cix, xcfunc_t *func, uint64_t arg1, uint64_t arg2)
{
    int lcx;
    int opl;
    cpuset_t tset;

    /*
     * Make sure the function address will not be interpreted as a
     * dmv interrupt
     */
    ASSERT(!DMV_IS_DMV(func));

    /*
     * It's illegal to send software inums through the cross-trap
     * interface.
     */
    ASSERT((uintptr_t)func >= KERNELBASE);

    CPUSET_ZERO(tset);

    XC_SPL_ENTER(lcx, opl);         /* lcx set by the macro */

    CPUSET_ADD(tset, cix);

    if (cix == lcx) {
        /*
         * same cpu - use software fast trap
         */
        send_self_xcall(CPU, arg1, arg2, func);
        XC_STAT_INC(x_dstat[lcx][XT_ONE_SELF]);
        XC_TRACE(XT_ONE_SELF, &tset, func, arg1, arg2);
    } else {    /* other cpu - send a mondo to the target cpu */
        /*
         * other cpu - send a mondo to the target cpu
         */
        XC_TRACE(XT_ONE_OTHER, &tset, func, arg1, arg2);
        init_mondo(func, arg1, arg2);
        send_one_mondo(cix);
        XC_STAT_INC(x_dstat[lcx][XT_ONE_OTHER]);
    }
    XC_SPL_EXIT(lcx, opl);
}

/*
 * xt_some - send a "x-trap" to some cpus
 */
void
xt_some(cpuset_t cpuset, xcfunc_t *func, uint64_t arg1, uint64_t arg2)
{
    int lcx;
    int opl;
    cpuset_t xc_cpuset, tset;

    /*
     * Make sure the function address will not be interpreted as a
     * dmv interrupt
     */
    ASSERT(!DMV_IS_DMV(func));

    /*
     * It's illegal to send software inums through the cross-trap
     * interface.
     */
    ASSERT((uintptr_t)func >= KERNELBASE);

    CPUSET_ZERO(tset);

    XC_SPL_ENTER(lcx, opl);     /* lcx set by the macro */

    CPUSET_ADD(tset, lcx);

    /*
     * only send to the CPU_READY ones
     */
    xc_cpuset = cpu_ready_set;
    CPUSET_AND(xc_cpuset, cpuset);

    /*
     * send to nobody; just return
     */
    if (CPUSET_ISNULL(xc_cpuset)) {
        XC_SPL_EXIT(lcx, opl);
        return;
    }

    /*
     * don't send mondo to self
     */
    if (CPU_IN_SET(xc_cpuset, lcx)) {
        /*
         * same cpu - use software fast trap
         */
        send_self_xcall(CPU, arg1, arg2, func);
        XC_STAT_INC(x_dstat[lcx][XT_SOME_SELF]);
        XC_TRACE(XT_SOME_SELF, &tset, func, arg1, arg2);
        CPUSET_DEL(xc_cpuset, lcx);
        if (CPUSET_ISNULL(xc_cpuset)) {
            XC_SPL_EXIT(lcx, opl);
            return;
        }
    }
    XC_TRACE(XT_SOME_OTHER, &xc_cpuset, func, arg1, arg2);
    init_mondo(func, arg1, arg2);
    send_mondo_set(xc_cpuset);
    XC_STAT_INC(x_dstat[lcx][XT_SOME_OTHER]);

    XC_SPL_EXIT(lcx, opl);
}

/*
 * xt_all - send a "x-trap" to all cpus
 */
void
xt_all(xcfunc_t *func, uint64_t arg1, uint64_t arg2)
{
    int lcx;
    int opl;
    cpuset_t xc_cpuset, tset;

    /*
     * Make sure the function address will not be interpreted as a
     * dmv interrupt
     */
    ASSERT(!DMV_IS_DMV(func));

    /*
     * It's illegal to send software inums through the cross-trap
     * interface.
     */
    ASSERT((uintptr_t)func >= KERNELBASE);

    CPUSET_ZERO(tset);

    XC_SPL_ENTER(lcx, opl);     /* lcx set by the macro */

    CPUSET_ADD(tset, lcx);

    /*
     * same cpu - use software fast trap
     */
    if (CPU_IN_SET(cpu_ready_set, lcx))
        send_self_xcall(CPU, arg1, arg2, func);

    XC_TRACE(XT_ALL_OTHER, &cpu_ready_set, func, arg1, arg2);

    /*
     * don't send mondo to self
     */
    xc_cpuset = cpu_ready_set;
    CPUSET_DEL(xc_cpuset, lcx);

    if (CPUSET_ISNULL(xc_cpuset)) {
        XC_STAT_INC(x_dstat[lcx][XT_ALL_SELF]);
        XC_TRACE(XT_ALL_SELF, &tset, func, arg1, arg2);
        XC_SPL_EXIT(lcx, opl);
        return;
    }

    init_mondo(func, arg1, arg2);
    send_mondo_set(xc_cpuset);

    XC_STAT_INC(x_dstat[lcx][XT_ALL_OTHER]);
    XC_SPL_EXIT(lcx, opl);
}

/*
 * xc_one - send a "x-call" to a cpu
 */
void
xc_one(int cix, xcfunc_t *func, uint64_t arg1, uint64_t arg2)
{
    int lcx;
    int opl;
    uint64_t loop_cnt = 0;
    cpuset_t tset;
    int first_time = 1;

    /*
     * send to nobody; just return
     */
    if (!CPU_IN_SET(cpu_ready_set, cix))
        return;

    ASSERT((uintptr_t)func > KERNELBASE);
    ASSERT(((uintptr_t)func % PC_ALIGN) == 0);

    CPUSET_ZERO(tset);

    kpreempt_disable();

    XC_SPL_ENTER(lcx, opl);     /* lcx set by the macro */

    CPUSET_ADD(tset, cix);

    if (cix == lcx) {   /* same cpu just do it */
        XC_TRACE(XC_ONE_SELF, &tset, func, arg1, arg2);
        (*func)(arg1, arg2);
        XC_STAT_INC(x_dstat[lcx][XC_ONE_SELF]);
        XC_SPL_EXIT(lcx, opl);
        kpreempt_enable();
        return;
    }

    if (xc_holder == lcx) {     /* got the xc_sys_mutex already */
        ASSERT(MUTEX_HELD(&xc_sys_mutex));
        ASSERT(CPU_IN_SET(xc_mbox[lcx].xc_cpuset, lcx));
        ASSERT(CPU_IN_SET(xc_mbox[cix].xc_cpuset, cix));
        ASSERT(xc_mbox[cix].xc_state == XC_WAIT);
        XC_TRACE(XC_ONE_OTHER_H, &tset, func, arg1, arg2);

        /*
         * target processor's xc_loop should be waiting
         * for the work to do; just set up the xc_mbox
         */
        XC_SETUP(cix, func, arg1, arg2);
        membar_stld();

        while (xc_mbox[cix].xc_state != XC_WAIT) {
            if (loop_cnt++ > xc_func_time_limit) {
                if (sendmondo_in_recover) {
                    drv_usecwait(1);
                    loop_cnt = 0;
                    continue;
                }
                cmn_err(CE_PANIC, "xc_one() timeout, "
                    "xc_state[%d] != XC_WAIT", cix);
            }
        }
        XC_STAT_INC(x_dstat[lcx][XC_ONE_OTHER_H]);
        XC_SPL_EXIT(lcx, opl);
        kpreempt_enable();
        return;
    }

    /*
     * Avoid dead lock if someone has sent us a xc_loop request while
     * we are trying to grab xc_sys_mutex.
     */
    XC_SPL_EXIT(lcx, opl);

    /*
     * At this point, since we don't own xc_sys_mutex,
     * our pil shouldn't run at or above the XCALL_PIL.
     */
    ASSERT(getpil() < XCALL_PIL);

    /*
     * Since xc_holder is not owned by us, it could be that
     * no one owns it, or we are not informed to enter into
     * xc_loop(). In either case, we need to grab the
     * xc_sys_mutex before we write to the xc_mbox, and
     * we shouldn't release it until the request is finished.
     */

    mutex_enter(&xc_sys_mutex);
    xc_spl_enter[lcx] = 1;

    /*
     * Since we own xc_sys_mutex now, we are safe to
     * write to the xc_mobx.
     */
    ASSERT(xc_mbox[cix].xc_state == XC_IDLE);
    XC_TRACE(XC_ONE_OTHER, &tset, func, arg1, arg2);
    XC_SETUP(cix, func, arg1, arg2);
    init_mondo(setsoftint_tl1, xc_serv_inum, 0);
    send_one_mondo(cix);

    /* xc_serv does membar_stld */
    while (xc_mbox[cix].xc_state != XC_IDLE) {
        if (loop_cnt++ > xc_func_time_limit) {
            if (sendmondo_in_recover) {
                drv_usecwait(1);
                loop_cnt = 0;
                continue;
            }
            if (first_time) {
                XT_SYNC_ONE(cix);
                first_time = 0;
                loop_cnt = 0;
                continue;
            }
            cmn_err(CE_PANIC, "xc_one() timeout, "
                "xc_state[%d] != XC_IDLE", cix);
        }
    }
    xc_spl_enter[lcx] = 0;
    XC_STAT_INC(x_dstat[lcx][XC_ONE_OTHER]);
    mutex_exit(&xc_sys_mutex);

    kpreempt_enable();
}

/*
 * xc_some - send a "x-call" to some cpus; sending to self is excluded
 */
void
xc_some(cpuset_t cpuset, xcfunc_t *func, uint64_t arg1, uint64_t arg2)
{
    int lcx;
    int opl;
    cpuset_t xc_cpuset, tset;

    ASSERT((uintptr_t)func > KERNELBASE);
    ASSERT(((uintptr_t)func % PC_ALIGN) == 0);

    CPUSET_ZERO(tset);

    kpreempt_disable();
    XC_SPL_ENTER(lcx, opl);         /* lcx set by the macro */

    CPUSET_ADD(tset, lcx);

    /*
     * only send to the CPU_READY ones
     */
    xc_cpuset = cpu_ready_set;
    CPUSET_AND(xc_cpuset, cpuset);

    /*
     * send to nobody; just return
     */
    if (CPUSET_ISNULL(xc_cpuset)) {
        XC_SPL_EXIT(lcx, opl);
        kpreempt_enable();
        return;
    }

    if (CPU_IN_SET(xc_cpuset, lcx)) {
        /*
         * same cpu just do it
         */
        (*func)(arg1, arg2);
        CPUSET_DEL(xc_cpuset, lcx);
        if (CPUSET_ISNULL(xc_cpuset)) {
            XC_STAT_INC(x_dstat[lcx][XC_SOME_SELF]);
            XC_TRACE(XC_SOME_SELF, &tset, func, arg1, arg2);
            XC_SPL_EXIT(lcx, opl);
            kpreempt_enable();
            return;
        }
    }

    if (xc_holder == lcx) {     /* got the xc_sys_mutex already */
        cpuset_t mset = xc_mbox[lcx].xc_cpuset;

        CPUSET_AND(mset, cpuset);
        ASSERT(MUTEX_HELD(&xc_sys_mutex));
        ASSERT(CPUSET_ISEQUAL(mset, cpuset));
        SEND_MBOX_ONLY(xc_cpuset, func, arg1, arg2, lcx, XC_WAIT);
        WAIT_MBOX_DONE(xc_cpuset, lcx, XC_WAIT, 0);
        XC_STAT_INC(x_dstat[lcx][XC_SOME_OTHER_H]);
        XC_TRACE(XC_SOME_OTHER_H, &xc_cpuset, func, arg1, arg2);
        XC_SPL_EXIT(lcx, opl);
        kpreempt_enable();
        return;
    }

    /*
     * Avoid dead lock if someone has sent us a xc_loop request while
     * we are trying to grab xc_sys_mutex.
     */
    XC_SPL_EXIT(lcx, opl);

    /*
     * At this point, since we don't own xc_sys_mutex,
     * our pil shouldn't run at or above the XCALL_PIL.
     */
    ASSERT(getpil() < XCALL_PIL);

    /*
     * grab xc_sys_mutex before writing to the xc_mbox
     */
    mutex_enter(&xc_sys_mutex);
    xc_spl_enter[lcx] = 1;

    XC_TRACE(XC_SOME_OTHER, &xc_cpuset, func, arg1, arg2);
    init_mondo(setsoftint_tl1, xc_serv_inum, 0);
    SEND_MBOX_MONDO(xc_cpuset, func, arg1, arg2, XC_IDLE);
    WAIT_MBOX_DONE(xc_cpuset, lcx, XC_IDLE, 1);

    xc_spl_enter[lcx] = 0;
    XC_STAT_INC(x_dstat[lcx][XC_SOME_OTHER]);
    mutex_exit(&xc_sys_mutex);
    kpreempt_enable();
}

/*
 * xc_all - send a "x-call" to all cpus
 */
void
xc_all(xcfunc_t *func, uint64_t arg1, uint64_t arg2)
{
    int lcx;
    int opl;
    cpuset_t xc_cpuset, tset;

    ASSERT((uintptr_t)func > KERNELBASE);
    ASSERT(((uintptr_t)func % PC_ALIGN) == 0);

    CPUSET_ZERO(tset);

    kpreempt_disable();
    XC_SPL_ENTER(lcx, opl);         /* lcx set by the macro */

    CPUSET_ADD(tset, lcx);

    /*
     * same cpu just do it
     */
    (*func)(arg1, arg2);
    xc_cpuset = cpu_ready_set;
    CPUSET_DEL(xc_cpuset, lcx);

    if (CPUSET_ISNULL(xc_cpuset)) {
        XC_STAT_INC(x_dstat[lcx][XC_ALL_SELF]);
        XC_TRACE(XC_ALL_SELF, &tset, func, arg1, arg2);
        XC_SPL_EXIT(lcx, opl);
        kpreempt_enable();
        return;
    }

    if (xc_holder == lcx) {     /* got the xc_sys_mutex already */
        cpuset_t mset = xc_mbox[lcx].xc_cpuset;

        CPUSET_AND(mset, xc_cpuset);
        ASSERT(MUTEX_HELD(&xc_sys_mutex));
        ASSERT(CPUSET_ISEQUAL(mset, xc_cpuset));
        XC_TRACE(XC_ALL_OTHER_H, &xc_cpuset, func, arg1, arg2);
        SEND_MBOX_ONLY(xc_cpuset, func, arg1, arg2, lcx, XC_WAIT);
        WAIT_MBOX_DONE(xc_cpuset, lcx, XC_WAIT, 0);
        XC_STAT_INC(x_dstat[lcx][XC_ALL_OTHER_H]);
        XC_SPL_EXIT(lcx, opl);
        kpreempt_enable();
        return;
    }

    /*
     * Avoid dead lock if someone has sent us a xc_loop request while
     * we are trying to grab xc_sys_mutex.
     */
    XC_SPL_EXIT(lcx, opl);

    /*
     * At this point, since we don't own xc_sys_mutex,
     * our pil shouldn't run at or above the XCALL_PIL.
     */
    ASSERT(getpil() < XCALL_PIL);

    /*
     * grab xc_sys_mutex before writing to the xc_mbox
     */
    mutex_enter(&xc_sys_mutex);
    xc_spl_enter[lcx] = 1;

    XC_TRACE(XC_ALL_OTHER, &xc_cpuset, func, arg1, arg2);
    init_mondo(setsoftint_tl1, xc_serv_inum, 0);
    SEND_MBOX_MONDO(xc_cpuset, func, arg1, arg2, XC_IDLE);
    WAIT_MBOX_DONE(xc_cpuset, lcx, XC_IDLE, 1);

    xc_spl_enter[lcx] = 0;
    XC_STAT_INC(x_dstat[lcx][XC_ALL_OTHER]);
    mutex_exit(&xc_sys_mutex);
    kpreempt_enable();
}

/*
 * xc_attention - paired with xc_dismissed()
 *
 * xt_attention() holds the xc_sys_mutex and xc_dismissed() releases it
 * called when an initiator wants to capture some/all cpus for a critical
 * session.
 */
void
xc_attention(cpuset_t cpuset)
{
    int pix, lcx;
    cpuset_t xc_cpuset, tmpset;
    cpuset_t recv_cpuset;
    uint64_t loop_cnt = 0;
    int first_time = 1;

    CPUSET_ZERO(recv_cpuset);

    /*
     * don't migrate the cpu until xc_dismissed() is finished
     */
    ASSERT(getpil() < XCALL_PIL);
    mutex_enter(&xc_sys_mutex);
    lcx = (int)(CPU->cpu_id);
    ASSERT(x_dstat[lcx][XC_ATTENTION] ==
        x_dstat[lcx][XC_DISMISSED]);
    ASSERT(xc_holder == -1);
    xc_mbox[lcx].xc_cpuset = cpuset;
    xc_holder = lcx; /* no membar; only current cpu needs the right lcx */

    /*
     * only send to the CPU_READY ones
     */
    xc_cpuset = cpu_ready_set;
    CPUSET_AND(xc_cpuset, cpuset);

    /*
     * don't send mondo to self
     */
    CPUSET_DEL(xc_cpuset, lcx);

    XC_STAT_INC(x_dstat[lcx][XC_ATTENTION]);
    XC_TRACE(XC_ATTENTION, &xc_cpuset, NULL, NULL, NULL);

    if (CPUSET_ISNULL(xc_cpuset))
        return;

    xc_spl_enter[lcx] = 1;
    /*
     * inform the target processors to enter into xc_loop()
     */
    tmpset = xc_cpuset;
    init_mondo(setsoftint_tl1, xc_loop_inum, 0);
    for (pix = 0; pix < NCPU; pix++) {
        if (CPU_IN_SET(tmpset, pix)) {
            ASSERT(xc_mbox[pix].xc_state == XC_IDLE);
            xc_mbox[pix].xc_state = XC_ENTER;
            send_one_mondo(pix);
            CPUSET_DEL(tmpset, pix);
            if (CPUSET_ISNULL(tmpset)) {
                break;
            }
        }
    }
    xc_spl_enter[lcx] = 0;

    /*
     * make sure target processors have entered into xc_loop()
     */
    while (!CPUSET_ISEQUAL(recv_cpuset, xc_cpuset)) {
        tmpset = xc_cpuset;
        for (pix = 0; pix < NCPU; pix++) {
            if (CPU_IN_SET(tmpset, pix)) {
                /*
                 * membar_stld() is done in xc_loop
                 */
                if (xc_mbox[pix].xc_state == XC_WAIT) {
                    CPUSET_ADD(recv_cpuset, pix);
                }
                CPUSET_DEL(tmpset, pix);
                if (CPUSET_ISNULL(tmpset)) {
                    break;
                }
            }
        }
        if (loop_cnt++ > xc_mondo_time_limit) {
            if (sendmondo_in_recover) {
                drv_usecwait(1);
                loop_cnt = 0;
                continue;
            }
            if (first_time) {
                XT_SYNC_SOME(xc_cpuset);
                first_time = 0;
                loop_cnt = 0;
                continue;
            }
            cmn_err(CE_PANIC, "xc_attention() timeout");
        }
    }

    /*
     * xc_sys_mutex remains held until xc_dismissed() is finished
     */
}

/*
 * xc_dismissed - paired with xc_attention()
 *
 * Called after the critical session is finished.
 */
void
xc_dismissed(cpuset_t cpuset)
{
    int pix;
    int lcx = (int)(CPU->cpu_id);
    cpuset_t xc_cpuset, tmpset;
    cpuset_t recv_cpuset;
    uint64_t loop_cnt = 0;

    ASSERT(lcx == xc_holder);
    ASSERT(CPUSET_ISEQUAL(xc_mbox[lcx].xc_cpuset, cpuset));
    ASSERT(getpil() >= XCALL_PIL);
    CPUSET_ZERO(xc_mbox[lcx].xc_cpuset);
    CPUSET_ZERO(recv_cpuset);
    membar_stld();

    XC_STAT_INC(x_dstat[lcx][XC_DISMISSED]);
    ASSERT(x_dstat[lcx][XC_DISMISSED] == x_dstat[lcx][XC_ATTENTION]);

    /*
     * only send to the CPU_READY ones
     */
    xc_cpuset = cpu_ready_set;
    CPUSET_AND(xc_cpuset, cpuset);

    /*
     * exclude itself
     */
    CPUSET_DEL(xc_cpuset, lcx);
    XC_TRACE(XC_DISMISSED, &xc_cpuset, NULL, NULL, NULL);
    if (CPUSET_ISNULL(xc_cpuset)) {
        xc_holder = -1;
        mutex_exit(&xc_sys_mutex);
        return;
    }

    /*
     * inform other processors to get out of xc_loop()
     */
    tmpset = xc_cpuset;
    for (pix = 0; pix < NCPU; pix++) {
        if (CPU_IN_SET(tmpset, pix)) {
            xc_mbox[pix].xc_state = XC_EXIT;
            membar_stld();
            CPUSET_DEL(tmpset, pix);
            if (CPUSET_ISNULL(tmpset)) {
                break;
            }
        }
    }

    /*
     * make sure target processors have exited from xc_loop()
     */
    while (!CPUSET_ISEQUAL(recv_cpuset, xc_cpuset)) {
        tmpset = xc_cpuset;
        for (pix = 0; pix < NCPU; pix++) {
            if (CPU_IN_SET(tmpset, pix)) {
                /*
                 * membar_stld() is done in xc_loop
                 */
                if (xc_mbox[pix].xc_state == XC_IDLE) {
                    CPUSET_ADD(recv_cpuset, pix);
                }
                CPUSET_DEL(tmpset, pix);
                if (CPUSET_ISNULL(tmpset)) {
                    break;
                }
            }
        }
        if (loop_cnt++ > xc_func_time_limit) {
                if (sendmondo_in_recover) {
                    drv_usecwait(1);
                    loop_cnt = 0;
                    continue;
                }
            cmn_err(CE_PANIC, "xc_dismissed() timeout");
        }
    }
    xc_holder = -1;
    mutex_exit(&xc_sys_mutex);
}

/*
 * xc_serv - "x-call" handler at TL=0; serves only one x-call request
 * runs at XCALL_PIL level.
 */
uint_t
xc_serv(void)
{
    int lcx = (int)(CPU->cpu_id);
    struct xc_mbox *xmp;
    xcfunc_t *func;
    uint64_t arg1, arg2;
    cpuset_t tset;

    ASSERT(getpil() == XCALL_PIL);
    CPUSET_ZERO(tset);
    CPUSET_ADD(tset, lcx);
    flush_windows();
    xmp = &xc_mbox[lcx];
    ASSERT(lcx != xc_holder);
    ASSERT(xmp->xc_state == XC_DOIT);
    func = xmp->xc_func;
    XC_TRACE(XC_SERV, &tset, func, xmp->xc_arg1, xmp->xc_arg2);
    if (func != NULL) {
        arg1 = xmp->xc_arg1;
        arg2 = xmp->xc_arg2;
        (*func)(arg1, arg2);
    }
    XC_STAT_INC(x_rstat[lcx][XC_SERV]);
    XC_TRACE(XC_SERV, &tset, func, arg1, arg2);
    xmp->xc_state = XC_IDLE;
    membar_stld();
    return (1);
}

/*
 * if == 1, an xc_loop timeout will cause a panic
 * otherwise print a warning
 */
uint_t xc_loop_panic = 0;

/*
 * xc_loop - "x-call" handler at TL=0; capture the cpu for a critial
 * session, or serve multiple x-call requests runs at XCALL_PIL level.
 */
uint_t
xc_loop(void)
{
    int lcx = (int)(CPU->cpu_id);
    struct xc_mbox *xmp;
    xcfunc_t *func;
    uint64_t arg1, arg2;
    uint64_t loop_cnt = 0;
    cpuset_t tset;

    ASSERT(getpil() == XCALL_PIL);

    CPUSET_ZERO(tset);
    flush_windows();

    /*
     * Some one must have owned the xc_sys_mutex;
     * no further interrupt (at XCALL_PIL or below) can
     * be taken by this processor until xc_loop exits.
     *
     * The owner of xc_sys_mutex (or xc_holder) can expect
     * its xc/xt requests are handled as follows:
     *  xc requests use xc_mbox's handshaking for their services
     *  xt requests at TL>0 will be handled immediately
     *  xt requests at TL=0:
     *      if their handlers'pils are <= XCALL_PIL, then
     *          they will be handled after xc_loop exits
     *          (so, they probably should not be used)
     *      else they will be handled immediately
     *
     * For those who are not informed to enter xc_loop, if they
     * send xc/xt requests to this processor at this moment,
     * the requests will be handled as follows:
     *  xc requests will be handled after they grab xc_sys_mutex
     *  xt requests at TL>0 will be handled immediately
     *  xt requests at TL=0:
     *      if their handlers'pils are <= XCALL_PIL, then
     *          they will be handled after xc_loop exits
     *      else they will be handled immediately
     */
    xmp = &xc_mbox[lcx];
    ASSERT(lcx != xc_holder);
    ASSERT(xmp->xc_state == XC_ENTER);
    xmp->xc_state = XC_WAIT;
    CPUSET_ADD(tset, lcx);
    membar_stld();
    XC_STAT_INC(x_rstat[lcx][XC_LOOP]);
    XC_TRACE(XC_LOOP_ENTER, &tset, NULL, NULL, NULL);
    while (xmp->xc_state != XC_EXIT) {
        if (xmp->xc_state == XC_DOIT) {
            func = xmp->xc_func;
            arg1 = xmp->xc_arg1;
            arg2 = xmp->xc_arg2;
            XC_TRACE(XC_LOOP_DOIT, &tset, func, arg1, arg2);
            if (func != NULL)
                (*func)(arg1, arg2);
            xmp->xc_state = XC_WAIT;
            membar_stld();
            /*
             * reset the timeout counter
             * since some work was done
             */
            loop_cnt = 0;
        } else {
            /* patience is a virtue... */
            loop_cnt++;
        }

        if (loop_cnt > xc_func_time_limit) {
            if (sendmondo_in_recover) {
                drv_usecwait(1);
                loop_cnt = 0;
                continue;
            }
            cmn_err(xc_loop_panic ? CE_PANIC : CE_WARN,
                "xc_loop() timeout");
            /*
             * if the above displayed a warning,
             * reset the timeout counter and be patient
             */
            loop_cnt = 0;
        }
    }
    ASSERT(xmp->xc_state == XC_EXIT);
    ASSERT(xc_holder != -1);
    XC_TRACE(XC_LOOP_EXIT, &tset, NULL, NULL, NULL);
    xmp->xc_state = XC_IDLE;
    membar_stld();
    return (1);
}