i86xpv/os/evtchn.c

	evtchn.c revision a43153bf6cee5e5307b15ed4f77dafc5c56f333d
/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

#pragma ident   "%Z%%M% %I% %E% SMI"

/*
 * evtchn.c
 *
 * Communication via hypervisor event channels.
 *
 * Copyright (c) 2002-2005, K A Fraser
 *
 * This file may be distributed separately from the Linux kernel, or
 * incorporated into other software packages, subject to the following license:
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this source file (the "Software"), to deal in the Software without
 * restriction, including without limitation the rights to use, copy, modify,
 * merge, publish, distribute, sublicense, and/or sell copies of the Software,
 * and to permit persons to whom the Software is furnished to do so, subject to
 * the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
 * IN THE SOFTWARE.
 */

/* some parts derived from netbsd's hypervisor_machdep.c 1.2.2.2 */

/*
 *
 * Copyright (c) 2004 Christian Limpach.
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 * 3. This section intentionally left blank.
 * 4. The name of the author may not be used to endorse or promote products
 *    derived from this software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */
/*
 * Section 3 of the above license was updated in response to bug 6379571.
 */

#include <sys/types.h>
#include <sys/hypervisor.h>
#include <sys/machsystm.h>
#include <sys/mutex.h>
#include <sys/evtchn_impl.h>
#include <sys/ddi_impldefs.h>
#include <sys/avintr.h>
#include <sys/cpuvar.h>
#include <sys/smp_impldefs.h>
#include <sys/archsystm.h>
#include <sys/sysmacros.h>
#include <sys/cmn_err.h>
#include <sys/promif.h>
#include <sys/debug.h>
#include <sys/psm.h>
#include <sys/privregs.h>
#include <sys/trap.h>
#include <sys/atomic.h>
#include <sys/cpu.h>
#include <sys/psw.h>
#include <sys/traptrace.h>
#include <sys/stack.h>
#include <sys/x_call.h>
#include <xen/public/physdev.h>

/*
 * This file manages our association between hypervisor event channels and
 * Solaris's IRQs.  This is a one-to-one mapping, with the exception of
 * IPI IRQs, for which there is one event channel per CPU participating
 * in the IPI, and the clock VIRQ which also has an event channel per cpu
 * and the IRQ for /dev/xen/evtchn. The IRQ types are:
 *
 * IRQT_VIRQ:
 *  The hypervisor's standard virtual IRQ, used for the clock timer, for
 *  example.  This code allows any cpu to bind to one of these, although
 *  some are treated specially (i.e. VIRQ_DEBUG).
 *  Event channel binding is done via EVTCHNOP_bind_virq.
 *
 * IRQT_PIRQ:
 *  These associate a physical IRQ with an event channel via
 *  EVTCHNOP_bind_pirq.
 *
 * IRQT_IPI:
 *  A cross-call IRQ. Maps to "ncpus" event channels, each of which is
 *  bound to exactly one of the vcpus.  We do not currently support
 *  unbinding of IPIs (since Solaris doesn't need it). Uses
 *  EVTCHNOP_bind_ipi.
 *
 * IRQT_EVTCHN:
 *  A "normal" binding to an event channel, typically used by the frontend
 *      drivers to bind to the their backend event channel.
 *
 * IRQT_DEV_EVTCHN:
 *  This is a one-time IRQ used by /dev/xen/evtchn. Unlike other IRQs, we
 *  have a one-IRQ to many-evtchn mapping. We only track evtchn->irq for
 *  these event channels, which are managed via ec_irq_add/rm_evtchn().
 *  We enforce that IRQT_DEV_EVTCHN's representative evtchn (->ii_evtchn)
 *  is zero, and make any calls to irq_evtchn() an error, to prevent
 *  accidentally attempting to use the illegal evtchn 0.
 *
 * Suspend/resume
 *
 *  During a suspend/resume cycle, we need to tear down the event channels.
 *  All other mapping data is kept. The drivers will remove their own event
 *  channels via xendev on receiving a DDI_SUSPEND.  This leaves us with
 *  the IPIs and VIRQs, which we handle in ec_suspend() and ec_resume()
 *  below.
 *
 * CPU binding
 *
 *  When an event channel is bound to a CPU, we set a bit in a mask present
 *  in the machcpu (evt_affinity) to indicate that this CPU can accept this
 *  event channel.  For both IPIs and VIRQs, this binding is fixed at
 *  allocation time and we never modify it.  All other event channels are
 *  bound via the PSM either as part of add_avintr(), or interrupt
 *  redistribution (xen_psm_dis/enable_intr()) as a result of CPU
 *  offline/online.
 *
 * Locking
 *
 *  Updates are done holding the ec_lock.  The xen_callback_handler()
 *  routine reads the mapping data in a lockless fashion.  Additionally
 *  suspend takes ec_lock to prevent update races during a suspend/resume
 *  cycle.  The IPI info is also examined without the lock; this is OK
 *  since we only ever change IPI info during initial setup and resume.
 */

#define IRQ_IS_CPUPOKE(irq) (ipi_info[XC_CPUPOKE_PIL].mi_irq == (irq))

#define EVTCHN_MASKED(ev) \
    (HYPERVISOR_shared_info->evtchn_mask[(ev) >> EVTCHN_SHIFT] & \
    (1ul << ((ev) & ((1ul << EVTCHN_SHIFT) - 1))))

static short evtchn_to_irq[NR_EVENT_CHANNELS];
static cpuset_t evtchn_cpus[NR_EVENT_CHANNELS];
static int  evtchn_owner[NR_EVENT_CHANNELS];
#ifdef DEBUG
static kthread_t *evtchn_owner_thread[NR_EVENT_CHANNELS];
#endif

static irq_info_t irq_info[NR_IRQS];
static mec_info_t ipi_info[MAXIPL];
static mec_info_t virq_info[NR_VIRQS];
/*
 * Mailbox for communication with the evtchn device driver.
 * We rely on only cpu 0 servicing the event channels associated
 * with the driver.  i.e. all evtchn driver evtchns are bound to cpu 0.
 */
volatile int ec_dev_mbox;   /* mailbox for evtchn device driver */

/*
 * See the locking description above.
 */
kmutex_t ec_lock;

/*
 * Bitmap indicating which PIRQs require the hypervisor to be notified
 * on unmask.
 */
static unsigned long pirq_needs_eoi[NR_PIRQS / (sizeof (unsigned long) * NBBY)];

static int ec_debug_irq = INVALID_IRQ;
int ec_dev_irq = INVALID_IRQ;

int
xen_bind_virq(unsigned int virq, processorid_t cpu, int *port)
{
    evtchn_bind_virq_t bind;
    int err;

    bind.virq = virq;
    bind.vcpu = cpu;
    if ((err = HYPERVISOR_event_channel_op(EVTCHNOP_bind_virq, &bind)) == 0)
        *port = bind.port;
    else
        err = xen_xlate_errcode(err);
    return (err);
}

int
xen_bind_interdomain(int domid, int remote_port, int *port)
{
    evtchn_bind_interdomain_t bind;
    int err;

    bind.remote_dom  = domid;
    bind.remote_port = remote_port;
    if ((err = HYPERVISOR_event_channel_op(EVTCHNOP_bind_interdomain,
        &bind)) == 0)
        *port = bind.local_port;
    else
        err = xen_xlate_errcode(err);
    return (err);
}

int
xen_alloc_unbound_evtchn(int domid, int *evtchnp)
{
    evtchn_alloc_unbound_t alloc;
    int err;

    alloc.dom = DOMID_SELF;
    alloc.remote_dom = domid;

    if ((err = HYPERVISOR_event_channel_op(EVTCHNOP_alloc_unbound,
        &alloc)) == 0) {
        *evtchnp = alloc.port;
        /* ensure evtchn is masked till we're ready to use it */
        (void) ec_mask_evtchn(*evtchnp);
    } else {
        err = xen_xlate_errcode(err);
    }

    return (err);
}

static int
xen_close_evtchn(int evtchn)
{
    evtchn_close_t close;
    int err;

    close.port = evtchn;
    err = HYPERVISOR_event_channel_op(EVTCHNOP_close, &close);
    if (err)
        err = xen_xlate_errcode(err);
    return (err);
}

static int
xen_bind_ipi(processorid_t cpu)
{
    evtchn_bind_ipi_t bind;

    ASSERT(MUTEX_HELD(&ec_lock));

    bind.vcpu = cpu;
    if (HYPERVISOR_event_channel_op(EVTCHNOP_bind_ipi, &bind) != 0)
        panic("xen_bind_ipi() failed");
    return (bind.port);
}

/* Send future instances of this interrupt to other vcpu. */
static void
xen_bind_vcpu(int evtchn, int cpu)
{
    evtchn_bind_vcpu_t bind;

    ASSERT(MUTEX_HELD(&ec_lock));

    bind.port = evtchn;
    bind.vcpu = cpu;
    if (HYPERVISOR_event_channel_op(EVTCHNOP_bind_vcpu, &bind) != 0)
        panic("xen_bind_vcpu() failed");
}

static int
xen_bind_pirq(int pirq)
{
    evtchn_bind_pirq_t bind;
    int ret;

    bind.pirq = pirq;
    bind.flags = BIND_PIRQ__WILL_SHARE;
    if ((ret = HYPERVISOR_event_channel_op(EVTCHNOP_bind_pirq, &bind)) != 0)
        panic("xen_bind_pirq() failed (err %d)", ret);
    return (bind.port);
}

/* unmask an evtchn and send upcall to appropriate vcpu if pending bit is set */
static void
xen_evtchn_unmask(int evtchn)
{
    evtchn_unmask_t unmask;

    unmask.port = evtchn;
    if (HYPERVISOR_event_channel_op(EVTCHNOP_unmask, &unmask) != 0)
        panic("xen_evtchn_unmask() failed");
}

static void
update_evtchn_affinity(int evtchn)
{
    cpu_t *cp;
    struct xen_evt_data *cpe;

    ASSERT(evtchn_to_irq[evtchn] != INVALID_IRQ);
    ASSERT(MUTEX_HELD(&ec_lock));

    /*
     * Use lockless search of cpu_list, similar to mutex_vector_enter().
     */
    kpreempt_disable();
    cp = cpu_list;
    do {
        cpe = cp->cpu_m.mcpu_evt_pend;
        if (CPU_IN_SET(evtchn_cpus[evtchn], cp->cpu_id))
            SET_EVTCHN_BIT(evtchn, cpe->evt_affinity);
        else
            CLEAR_EVTCHN_BIT(evtchn, cpe->evt_affinity);
    } while ((cp = cp->cpu_next) != cpu_list);
    kpreempt_enable();
}

static void
bind_evtchn_to_cpuset(int evtchn, cpuset_t cpus)
{
    ASSERT(evtchn_to_irq[evtchn] != INVALID_IRQ);

    CPUSET_ZERO(evtchn_cpus[evtchn]);
    CPUSET_OR(evtchn_cpus[evtchn], cpus);
    update_evtchn_affinity(evtchn);
}

static void
clear_evtchn_affinity(int evtchn)
{
    CPUSET_ZERO(evtchn_cpus[evtchn]);
    update_evtchn_affinity(evtchn);
}

static void
alloc_irq_evtchn(int irq, int index, int evtchn, int cpu)
{
    irq_info_t *irqp = &irq_info[irq];

    switch (irqp->ii_type) {
    case IRQT_IPI:
        ipi_info[index].mi_evtchns[cpu] = evtchn;
        irqp->ii_u.index = index;
        break;
    case IRQT_VIRQ:
        virq_info[index].mi_evtchns[cpu] = evtchn;
        irqp->ii_u.index = index;
        break;
    default:
        irqp->ii_u.evtchn = evtchn;
        break;
    }

    evtchn_to_irq[evtchn] = irq;

    /*
     * If a CPU is not specified, we expect to bind it to a CPU later via
     * the PSM.
     */
    if (cpu != -1) {
        cpuset_t tcpus;
        CPUSET_ONLY(tcpus, cpu);
        bind_evtchn_to_cpuset(evtchn, tcpus);
    }
}

static int
alloc_irq(int type, int index, int evtchn, int cpu)
{
    int irq;
    irq_info_t *irqp;

    ASSERT(MUTEX_HELD(&ec_lock));
    ASSERT(type != IRQT_IPI || cpu != -1);

    for (irq = 0; irq < NR_IRQS; irq++) {
        if (irq_info[irq].ii_type == IRQT_UNBOUND)
            break;
    }

    if (irq == NR_IRQS)
        panic("No available IRQ to bind to: increase NR_IRQS!\n");

    irqp = &irq_info[irq];

    irqp->ii_type = type;
    /*
     * Set irq/has_handler field to zero which means handler not installed
     */
    irqp->ii_u2.has_handler = 0;

    alloc_irq_evtchn(irq, index, evtchn, cpu);
    return (irq);
}

static int
irq_evtchn(irq_info_t *irqp)
{
    int evtchn;

    ASSERT(irqp->ii_type != IRQT_DEV_EVTCHN);

    switch (irqp->ii_type) {
    case IRQT_IPI:
        ASSERT(irqp->ii_u.index != 0);
        evtchn = ipi_info[irqp->ii_u.index].mi_evtchns[CPU->cpu_id];
        break;
    case IRQT_VIRQ:
        evtchn = virq_info[irqp->ii_u.index].mi_evtchns[CPU->cpu_id];
        break;
    default:
        evtchn = irqp->ii_u.evtchn;
        break;
    }

    return (evtchn);
}

static void
unbind_evtchn(ushort_t *evtchnp)
{
    int err;

    ASSERT(MUTEX_HELD(&ec_lock));

    ASSERT(*evtchnp != 0);

    err = xen_close_evtchn(*evtchnp);
    ASSERT(err == 0);
    clear_evtchn_affinity(*evtchnp);
    evtchn_to_irq[*evtchnp] = INVALID_IRQ;
    *evtchnp = 0;
}

static void
pirq_unmask_notify(int pirq)
{
    struct physdev_eoi eoi;

    if (TEST_EVTCHN_BIT(pirq, &pirq_needs_eoi[0])) {
        eoi.irq = pirq;
        (void) HYPERVISOR_physdev_op(PHYSDEVOP_eoi, &eoi);
    }
}

static void
pirq_query_unmask(int pirq)
{
    struct physdev_irq_status_query irq_status;

    irq_status.irq = pirq;
    (void) HYPERVISOR_physdev_op(PHYSDEVOP_irq_status_query, &irq_status);
    CLEAR_EVTCHN_BIT(pirq, &pirq_needs_eoi[0]);
    if (irq_status.flags & XENIRQSTAT_needs_eoi)
        SET_EVTCHN_BIT(pirq, &pirq_needs_eoi[0]);
}

static void
end_pirq(int irq)
{
    int evtchn = irq_evtchn(&irq_info[irq]);

    ec_unmask_evtchn(evtchn);
    pirq_unmask_notify(IRQ_TO_PIRQ(irq));
}

/*
 * probe if a pirq is available to bind to, return 1 if available
 * else return 0.
 * Note that for debug versions of xen this probe may cause an in use IRQ
 * warning message from xen.
 */
int
ec_probe_pirq(int pirq)
{
    evtchn_bind_pirq_t bind;

    bind.pirq = pirq;
    bind.flags = 0;
    if (HYPERVISOR_event_channel_op(EVTCHNOP_bind_pirq, &bind) != 0) {
        return (0);
    } else {
        (void) xen_close_evtchn(bind.port);
        return (1);
    }
}

/*
 * Bind an event channel to a vcpu
 */
void
ec_bind_vcpu(int evtchn, int cpu)
{
    mutex_enter(&ec_lock);
    xen_bind_vcpu(evtchn, cpu);
    mutex_exit(&ec_lock);
}

/*
 * Set up a physical device irq to be associated with an event channel.
 */
void
ec_setup_pirq(int irq, int ipl, cpuset_t *cpusp)
{
    int evtchn;
    irq_info_t *irqp = &irq_info[irq];

    /*
     * Test if this PIRQ is already bound to an evtchn,
     * which means it is a shared IRQ and we don't want to
     * bind and do some initial setup that has already been
     * done for this irq on a previous trip through this code.
     */
    if (irqp->ii_u.evtchn == INVALID_EVTCHN) {
        evtchn = xen_bind_pirq(irq);

        pirq_query_unmask(IRQ_TO_PIRQ(irq));

        irqp->ii_type = IRQT_PIRQ;
        irqp->ii_u.evtchn = evtchn;

        evtchn_to_irq[evtchn] = irq;
        irqp->ii_u2.ipl = ipl;
        ec_set_irq_affinity(irq, *cpusp);
        ec_enable_irq(irq);
        pirq_unmask_notify(IRQ_TO_PIRQ(irq));
    } else {
        ASSERT(irqp->ii_u2.ipl != 0);
        cmn_err(CE_NOTE, "IRQ%d is shared", irq);
        if (ipl > irqp->ii_u2.ipl)
            irqp->ii_u2.ipl = ipl;
        *cpusp = evtchn_cpus[irqp->ii_u.evtchn];
    }
}

void
ec_unbind_irq(int irq)
{
    irq_info_t *irqp = &irq_info[irq];
    mec_info_t *virqp;
    int drop_lock = 0;
    int type, i;

    /*
     * Nasty, but we need this during suspend.
     */
    if (mutex_owner(&ec_lock) != curthread) {
        mutex_enter(&ec_lock);
        drop_lock = 1;
    }

    type = irqp->ii_type;

    ASSERT((type == IRQT_EVTCHN) || (type == IRQT_PIRQ) ||
        (type == IRQT_VIRQ));

    if ((type == IRQT_EVTCHN) || (type == IRQT_PIRQ)) {
        /* There's only one event channel associated with this irq */
        unbind_evtchn(&irqp->ii_u.evtchn);
    } else if (type == IRQT_VIRQ) {
        /*
         * Each cpu on the system can have it's own event channel
         * associated with a virq.  Unbind them all.
         */
        virqp = &virq_info[irqp->ii_u.index];
        for (i = 0; i < NCPU; i++) {
            if (virqp->mi_evtchns[i] != 0)
                unbind_evtchn(&virqp->mi_evtchns[i]);
        }
        /* Mark the virq structure as invalid. */
        virqp->mi_irq = INVALID_IRQ;
    }

    bzero(irqp, sizeof (*irqp));
    /* Re-reserve PIRQ. */
    if (type == IRQT_PIRQ)
        irqp->ii_type = IRQT_PIRQ;

    if (drop_lock)
        mutex_exit(&ec_lock);
}

/*
 * Rebind an event channel for delivery to a CPU.
 */
void
ec_set_irq_affinity(int irq, cpuset_t dest)
{
    int evtchn, tcpu;
    irq_info_t *irqp = &irq_info[irq];

    mutex_enter(&ec_lock);

    ASSERT(irq < NR_IRQS);
    ASSERT(irqp->ii_type != IRQT_UNBOUND);

    /*
     * Binding is done at allocation time for these types, so we should
     * never modify them.
     */
    if (irqp->ii_type == IRQT_IPI || irqp->ii_type == IRQT_VIRQ ||
        irqp->ii_type == IRQT_DEV_EVTCHN) {
        mutex_exit(&ec_lock);
        return;
    }

    CPUSET_FIND(dest, tcpu);
    ASSERT(tcpu != CPUSET_NOTINSET);

    evtchn = irq_evtchn(irqp);

    xen_bind_vcpu(evtchn, tcpu);

    bind_evtchn_to_cpuset(evtchn, dest);

    mutex_exit(&ec_lock);

    /*
     * Now send the new target processor a NOP IPI.
     * It will check for any pending interrupts, and so service any that
     * got delivered to the wrong processor by mistake.
     */
    if (ncpus > 1)
        poke_cpu(tcpu);
}

int
ec_set_irq_priority(int irq, int pri)
{
    irq_info_t *irqp;

    if (irq >= NR_IRQS)
        return (-1);

    irqp = &irq_info[irq];

    if (irqp->ii_type == IRQT_UNBOUND)
        return (-1);

    irqp->ii_u2.ipl = pri;

    return (0);
}

void
ec_clear_irq_priority(int irq)
{
    irq_info_t *irqp = &irq_info[irq];

    ASSERT(irq < NR_IRQS);
    ASSERT(irqp->ii_type != IRQT_UNBOUND);

    irqp->ii_u2.ipl = 0;
}

int
ec_bind_evtchn_to_irq(int evtchn)
{
    mutex_enter(&ec_lock);

    ASSERT(evtchn_to_irq[evtchn] == INVALID_IRQ);

    (void) alloc_irq(IRQT_EVTCHN, 0, evtchn, -1);

    mutex_exit(&ec_lock);
    return (evtchn_to_irq[evtchn]);
}

int
ec_bind_virq_to_irq(int virq, int cpu)
{
    int err;
    int evtchn;
    mec_info_t *virqp;

    virqp = &virq_info[virq];
    mutex_enter(&ec_lock);

    err = xen_bind_virq(virq, cpu, &evtchn);
    ASSERT(err == 0);

    ASSERT(evtchn_to_irq[evtchn] == INVALID_IRQ);

    if (virqp->mi_irq == INVALID_IRQ) {
        virqp->mi_irq = alloc_irq(IRQT_VIRQ, virq, evtchn, cpu);
    } else {
        alloc_irq_evtchn(virqp->mi_irq, virq, evtchn, cpu);
    }

    mutex_exit(&ec_lock);

    return (virqp->mi_irq);
}

int
ec_bind_ipi_to_irq(int ipl, int cpu)
{
    int evtchn;
    ulong_t flags;
    mec_info_t *ipip;

    mutex_enter(&ec_lock);

    ipip = &ipi_info[ipl];

    evtchn = xen_bind_ipi(cpu);

    ASSERT(evtchn_to_irq[evtchn] == INVALID_IRQ);

    if (ipip->mi_irq == INVALID_IRQ) {
        ipip->mi_irq = alloc_irq(IRQT_IPI, ipl, evtchn, cpu);
    } else {
        alloc_irq_evtchn(ipip->mi_irq, ipl, evtchn, cpu);
    }

    /*
     * Unmask the new evtchn so that it can be seen by the target cpu
     */
    flags = intr_clear();
    ec_unmask_evtchn(evtchn);
    intr_restore(flags);

    mutex_exit(&ec_lock);
    return (ipip->mi_irq);
}

/*
 * When bringing up a CPU, bind to all the IPIs that CPU0 bound.
 */
void
ec_bind_cpu_ipis(int cpu)
{
    int i;

    for (i = 0; i < MAXIPL; i++) {
        mec_info_t *ipip = &ipi_info[i];
        if (ipip->mi_irq == INVALID_IRQ)
            continue;

        (void) ec_bind_ipi_to_irq(i, cpu);
    }
}

/*
 * Can this IRQ be rebound to another CPU?
 */
int
ec_irq_rebindable(int irq)
{
    irq_info_t *irqp = &irq_info[irq];

    if (irqp->ii_u.evtchn == 0)
        return (0);

    return (irqp->ii_type == IRQT_EVTCHN || irqp->ii_type == IRQT_PIRQ);
}

/*
 * Should this IRQ be unbound from this CPU (which is being offlined) to
 * another?
 */
int
ec_irq_needs_rebind(int irq, int cpu)
{
    irq_info_t *irqp = &irq_info[irq];

    return (ec_irq_rebindable(irq) &&
        CPU_IN_SET(evtchn_cpus[irqp->ii_u.evtchn], cpu));
}

void
ec_send_ipi(int ipl, int cpu)
{
    mec_info_t *ipip = &ipi_info[ipl];

    ASSERT(ipip->mi_irq != INVALID_IRQ);

    ec_notify_via_evtchn(ipip->mi_evtchns[cpu]);
}

void
ec_try_ipi(int ipl, int cpu)
{
    mec_info_t *ipip = &ipi_info[ipl];

    if (ipip->mi_irq == INVALID_IRQ || ipip->mi_irq == 0)
        return;

    ec_notify_via_evtchn(ipip->mi_evtchns[cpu]);
}

void
ec_irq_add_evtchn(int irq, int evtchn)
{
    mutex_enter(&ec_lock);

    /*
     * See description of IRQT_DEV_EVTCHN above.
     */
    ASSERT(irq == ec_dev_irq);

    alloc_irq_evtchn(irq, 0, evtchn, 0);
    /*
     * We enforce that the representative event channel for IRQT_DEV_EVTCHN
     * is zero, so PSM operations on it have no effect.
     */
    irq_info[irq].ii_u.evtchn = 0;
    mutex_exit(&ec_lock);
}

void
ec_irq_rm_evtchn(int irq, int evtchn)
{
    ushort_t ec = evtchn;

    mutex_enter(&ec_lock);
    ASSERT(irq == ec_dev_irq);
    unbind_evtchn(&ec);
    mutex_exit(&ec_lock);
}

/*
 * Allocate an /dev/xen/evtchn IRQ.  See the big comment at the top
 * for an explanation.
 */
int
ec_dev_alloc_irq(void)
{
    int i;
    irq_info_t *irqp;

    for (i = 0; i < NR_IRQS; i++) {
        if (irq_info[i].ii_type == IRQT_UNBOUND)
            break;
    }

    ASSERT(i != NR_IRQS);

    irqp = &irq_info[i];
    irqp->ii_type = IRQT_DEV_EVTCHN;
    irqp->ii_u2.ipl = IPL_EVTCHN;
    /*
     * Force the evtchn to zero for the special evtchn device irq
     */
    irqp->ii_u.evtchn = 0;
    return (i);
}

void
ec_enable_irq(unsigned int irq)
{
    ulong_t flag;
    irq_info_t *irqp = &irq_info[irq];

    if (irqp->ii_type == IRQT_DEV_EVTCHN)
        return;

    flag = intr_clear();
    ec_unmask_evtchn(irq_evtchn(irqp));
    intr_restore(flag);
}

void
ec_disable_irq(unsigned int irq)
{
    irq_info_t *irqp = &irq_info[irq];

    if (irqp->ii_type == IRQT_DEV_EVTCHN)
        return;

    /*
     * Spin till we are the one to mask the evtchn
     * Ensures no one else can be servicing this evtchn.
     */
    while (!ec_mask_evtchn(irq_evtchn(irqp)))
        SMT_PAUSE();
}

static int
ec_evtchn_pending(uint_t ev)
{
    uint_t evi;
    shared_info_t *si = HYPERVISOR_shared_info;

    evi = ev >> EVTCHN_SHIFT;
    ev &= (1ul << EVTCHN_SHIFT) - 1;
    return ((si->evtchn_pending[evi] & (1ul << ev)) != 0);
}

int
ec_pending_irq(unsigned int irq)
{
    int evtchn = irq_evtchn(&irq_info[irq]);

    return (ec_evtchn_pending(evtchn));
}

void
ec_clear_irq(int irq)
{
    irq_info_t *irqp = &irq_info[irq];
    int evtchn;

    if (irqp->ii_type == IRQT_DEV_EVTCHN)
        return;

    ASSERT(irqp->ii_type != IRQT_UNBOUND);

    evtchn = irq_evtchn(irqp);

    ASSERT(EVTCHN_MASKED(evtchn));
    ec_clear_evtchn(evtchn);
}

void
ec_unmask_irq(int irq)
{
    ulong_t flags;
    irq_info_t *irqp = &irq_info[irq];

    flags = intr_clear();
    switch (irqp->ii_type) {
    case IRQT_PIRQ:
        end_pirq(irq);
        break;
    case IRQT_DEV_EVTCHN:
        break;
    default:
        ec_unmask_evtchn(irq_evtchn(irqp));
        break;
    }
    intr_restore(flags);
}

void
ec_try_unmask_irq(int irq)
{
    ulong_t flags;
    irq_info_t *irqp = &irq_info[irq];
    int evtchn;

    flags = intr_clear();
    switch (irqp->ii_type) {
    case IRQT_PIRQ:
        end_pirq(irq);
        break;
    case IRQT_DEV_EVTCHN:
        break;
    default:
        if ((evtchn = irq_evtchn(irqp)) != 0)
            ec_unmask_evtchn(evtchn);
        break;
    }
    intr_restore(flags);
}

/*
 * Poll until an event channel is ready or 'check_func' returns true.  This can
 * only be used in a situation where interrupts are masked, otherwise we have a
 * classic time-of-check vs. time-of-use race.
 */
void
ec_wait_on_evtchn(int evtchn, int (*check_func)(void *), void *arg)
{
    if (DOMAIN_IS_INITDOMAIN(xen_info)) {
        while (!check_func(arg))
            (void) HYPERVISOR_yield();
        return;
    }

    ASSERT(CPU->cpu_m.mcpu_vcpu_info->evtchn_upcall_mask != 0);

    for (;;) {
        evtchn_port_t ports[1];

        ports[0] = evtchn;

        ec_clear_evtchn(evtchn);

        if (check_func(arg))
            return;

        (void) HYPERVISOR_poll(ports, 1, 0);
    }
}

void
ec_wait_on_ipi(int ipl, int (*check_func)(void *), void *arg)
{
    mec_info_t *ipip = &ipi_info[ipl];

    if (ipip->mi_irq == INVALID_IRQ || ipip->mi_irq == 0)
        return;

    ec_wait_on_evtchn(ipip->mi_evtchns[CPU->cpu_id], check_func, arg);
}

void
ec_suspend(void)
{
    irq_info_t *irqp;
    ushort_t *evtchnp;
    int i;
    int c;

    ASSERT(MUTEX_HELD(&ec_lock));

    for (i = 0; i < MAXIPL; i++) {
        if (ipi_info[i].mi_irq == INVALID_IRQ)
            continue;

        for (c = 0; c < NCPU; c++) {
            if (cpu[c] == NULL)
                continue;

            if (CPU_IN_SET(cpu_suspend_lost_set, c))
                continue;

            evtchnp = &ipi_info[i].mi_evtchns[c];
            ASSERT(*evtchnp != 0);
            unbind_evtchn(evtchnp);
        }
    }

    for (i = 0; i < NR_VIRQS; i++) {
        if (virq_info[i].mi_irq == INVALID_IRQ)
            continue;

        /*
         * If we're sharing a single event channel across all CPUs, we
         * should only unbind once.
         */
        if (virq_info[i].mi_shared) {
            evtchnp = &virq_info[i].mi_evtchns[0];
            unbind_evtchn(evtchnp);
            for (c = 1; c < NCPU; c++)
                virq_info[i].mi_evtchns[c] = 0;
        } else {
            for (c = 0; c < NCPU; c++) {
                if (cpu[c] == NULL)
                    continue;

                evtchnp = &virq_info[i].mi_evtchns[c];
                if (*evtchnp != 0)
                    unbind_evtchn(evtchnp);
            }
        }
    }

    for (i = 0; i < NR_IRQS; i++) {
        irqp = &irq_info[i];

        switch (irqp->ii_type) {
        case IRQT_EVTCHN:
        case IRQT_DEV_EVTCHN:
            (void) HYPERVISOR_shutdown(SHUTDOWN_crash);
            break;
        case IRQT_PIRQ:
            if (irqp->ii_u.evtchn != 0)
                (void) HYPERVISOR_shutdown(SHUTDOWN_crash);
            break;
        default:
            break;
        }
    }
}

/*
 * The debug irq is special, we only have one evtchn and irq but we allow all
 * cpus to service it.  It's marked as shared and we propogate the event
 * channel into all CPUs by hand.
 */
static void
share_virq(mec_info_t *virqp)
{
    int evtchn = virqp->mi_evtchns[0];
    cpuset_t tset;
    int i;

    ASSERT(evtchn != 0);

    virqp->mi_shared = 1;

    for (i = 1; i < NCPU; i++)
        virqp->mi_evtchns[i] = evtchn;
    CPUSET_ALL(tset);
    bind_evtchn_to_cpuset(evtchn, tset);
}

static void
virq_resume(int virq)
{
    mec_info_t *virqp = &virq_info[virq];
    int evtchn;
    int i, err;

    for (i = 0; i < NCPU; i++) {
        cpuset_t tcpus;

        if (cpu[i] == NULL || CPU_IN_SET(cpu_suspend_lost_set, i))
            continue;

        err = xen_bind_virq(virq, i, &evtchn);
        ASSERT(err == 0);

        virqp->mi_evtchns[i] = evtchn;
        evtchn_to_irq[evtchn] = virqp->mi_irq;
        CPUSET_ONLY(tcpus, i);
        bind_evtchn_to_cpuset(evtchn, tcpus);
        ec_unmask_evtchn(evtchn);
        /*
         * only timer VIRQ is bound to all cpus
         */
        if (virq != VIRQ_TIMER)
            break;
    }

    if (virqp->mi_shared)
        share_virq(virqp);
}

static void
ipi_resume(int ipl)
{
    mec_info_t *ipip = &ipi_info[ipl];
    int i;

    for (i = 0; i < NCPU; i++) {
        cpuset_t tcpus;
        int evtchn;

        if (cpu[i] == NULL || CPU_IN_SET(cpu_suspend_lost_set, i))
            continue;

        evtchn = xen_bind_ipi(i);
        ipip->mi_evtchns[i] = evtchn;
        evtchn_to_irq[evtchn] = ipip->mi_irq;
        CPUSET_ONLY(tcpus, i);
        bind_evtchn_to_cpuset(evtchn, tcpus);
        ec_unmask_evtchn(evtchn);
    }
}

void
ec_resume(void)
{
    int i;

    /* New event-channel space is not 'live' yet. */
    for (i = 0; i < NR_EVENT_CHANNELS; i++)
        (void) ec_mask_evtchn(i);

    for (i = 0; i < MAXIPL; i++) {
        if (ipi_info[i].mi_irq == INVALID_IRQ)
            continue;
        ipi_resume(i);
    }

    for (i = 0; i < NR_VIRQS; i++) {
        if (virq_info[i].mi_irq == INVALID_IRQ)
            continue;
        virq_resume(i);
    }
}

void
ec_init(void)
{
    int i;
    mutex_init(&ec_lock, NULL, MUTEX_SPIN, (void *)ipltospl(SPL7));

    for (i = 0; i < NR_EVENT_CHANNELS; i++) {
        CPUSET_ZERO(evtchn_cpus[i]);
        evtchn_to_irq[i] = INVALID_IRQ;
        (void) ec_mask_evtchn(i);
    }

    for (i = 0; i < MAXIPL; i++)
        ipi_info[i].mi_irq = INVALID_IRQ;

    for (i = 0; i < NR_VIRQS; i++)
        virq_info[i].mi_irq = INVALID_IRQ;

    /*
     * Phys IRQ space is statically bound (1:1 mapping), grab the IRQs
     * now.
     */
    for (i = PIRQ_BASE; i < NR_PIRQS; i++) {
        irq_info[PIRQ_TO_IRQ(i)].ii_type = IRQT_PIRQ;
    }
}

void
ec_init_debug_irq()
{
    int irq;

    irq = ec_bind_virq_to_irq(VIRQ_DEBUG, 0);
    (void) add_avintr(NULL, IPL_DEBUG, (avfunc)xen_debug_handler,
        "debug", irq, NULL, NULL, NULL, NULL);

    mutex_enter(&ec_lock);
    share_virq(&virq_info[irq_info[irq].ii_u.index]);
    mutex_exit(&ec_lock);
    ec_debug_irq = irq;
}

#define UNBLOCKED_EVENTS(si, ix, cpe, cpu_id) \
    ((si)->evtchn_pending[ix] & ~(si)->evtchn_mask[ix] & \
        (cpe)->evt_affinity[ix])

/*
 * This is the entry point for processing events from xen
 *
 * (See the commentary associated with the shared_info_st structure
 * in hypervisor-if.h)
 *
 * Since the event channel mechanism doesn't really implement the
 * concept of priority like hardware interrupt controllers, we simulate
 * that in software here using the cpu priority field and the pending
 * interrupts field.  Events/interrupts that are not able to be serviced
 * now because they are at a lower priority than the current cpu priority
 * cause a level bit to be recorded in the pending interrupts word.  When
 * the priority is lowered (either by spl or interrupt exit code) the pending
 * levels are checked and an upcall is scheduled if there are events/interrupts
 * that have become deliverable.
 */
void
xen_callback_handler(struct regs *rp, trap_trace_rec_t *ttp)
{
    ulong_t pending_sels, pe, selbit;
    int i, j, port, pri, curpri, irq;
    uint16_t pending_ints;
    struct cpu *cpu = CPU;
    volatile shared_info_t *si = HYPERVISOR_shared_info;
    volatile vcpu_info_t *vci = cpu->cpu_m.mcpu_vcpu_info;
    volatile struct xen_evt_data *cpe = cpu->cpu_m.mcpu_evt_pend;
    volatile uint16_t *cpu_ipp = &cpu->cpu_m.mcpu_intr_pending;

    ASSERT(rp->r_trapno == T_AST && rp->r_err == 0);
    ASSERT(&si->vcpu_info[cpu->cpu_id] == vci);
    ASSERT_STACK_ALIGNED();

    vci->evtchn_upcall_pending = 0;

    /*
     * To expedite scanning of pending notifications, any 0->1
     * pending transition on an unmasked channel causes a
     * corresponding bit in evtchn_pending_sel to be set.
     * Each bit in the selector covers a 32-bit word in
     * the evtchn_pending[] array.
     */
    membar_enter();
    do {
        pending_sels = vci->evtchn_pending_sel;
    } while (atomic_cas_ulong((volatile ulong_t *)&vci->evtchn_pending_sel,
        pending_sels, 0) != pending_sels);

    pending_ints = *cpu_ipp;
    while ((i = ffs(pending_sels)) != 0) {
        i--;
        selbit = 1ul << i;
        pending_sels &= ~selbit;

        membar_enter();
        while ((pe = UNBLOCKED_EVENTS(si, i, cpe, cpu->cpu_id)) != 0) {
            j = ffs(pe) - 1;
            pe &= ~(1ul << j);

            port = (i << EVTCHN_SHIFT) + j;

            irq = evtchn_to_irq[port];

            /*
             * If no irq set, just ignore the event.
             * On e.g. netbsd they call evtchn_device_upcall(port)
             * We require the evtchn driver to install a handler
             * so there will be an irq associated with user mode
             * evtchns.
             */
            if (irq == INVALID_IRQ) {
                ec_clear_evtchn(port);
                continue;
            }

            /*
             * If there's no handler, it could be a poke, so just
             * accept the event and continue.
             */
            if (!irq_info[irq].ii_u2.has_handler) {
#ifdef TRAPTRACE
                ttp->ttr_ipl = 0xff;
                if (IRQ_IS_CPUPOKE(irq)) {
                    ttp->ttr_ipl = XC_CPUPOKE_PIL;
                    ttp->ttr_marker = TT_INTERRUPT;
                }
                ttp->ttr_pri = cpu->cpu_pri;
                ttp->ttr_spl = cpu->cpu_base_spl;
                ttp->ttr_vector = 0xff;
#endif /* TRAPTRACE */
                if (ec_mask_evtchn(port)) {
                    ec_clear_evtchn(port);
                    ec_unmask_evtchn(port);
                    continue;
                }
            }

            pri = irq_info[irq].ii_u2.ipl;

            /*
             * If we are the cpu that successfully masks
             * the event, then record it as a pending event
             * for this cpu to service
             */
            if (ec_mask_evtchn(port)) {
                if (ec_evtchn_pending(port)) {
                    cpe->pending_sel[pri] |= selbit;
                    cpe->pending_evts[pri][i] |= (1ul << j);
                    pending_ints |= 1 << pri;
                } else {
                    /*
                     * another cpu serviced this event
                     * before us, clear the mask.
                     */
                    ec_unmask_evtchn(port);
                }
            }
        }
    }
    *cpu_ipp = pending_ints;
    if (pending_ints == 0)
        return;
    /*
     * We have gathered all the pending events/interrupts,
     * go service all the ones we can from highest priority to lowest.
     * Note: This loop may not actually complete and service all
     * pending interrupts since one of the interrupt threads may
     * block and the pinned thread runs.  In that case, when we
     * exit the interrupt thread that blocked we will check for
     * any unserviced interrupts and re-post an upcall to process
     * any unserviced pending events.
     */
    curpri = cpu->cpu_pri;
    for (pri = bsrw_insn(*cpu_ipp); pri > curpri; pri--) {
        while ((pending_sels = cpe->pending_sel[pri]) != 0) {
            i = ffs(pending_sels) - 1;
            while ((pe = cpe->pending_evts[pri][i]) != 0) {
                j = ffs(pe) - 1;
                pe &= ~(1ul << j);
                cpe->pending_evts[pri][i] = pe;
                if (pe == 0) {
                    /*
                     * Must reload pending selector bits
                     * here as they could have changed on
                     * a previous trip around the inner loop
                     * while we were interrupt enabled
                     * in a interrupt service routine.
                     */
                    pending_sels = cpe->pending_sel[pri];
                    pending_sels &= ~(1ul << i);
                    cpe->pending_sel[pri] = pending_sels;
                    if (pending_sels == 0)
                        *cpu_ipp &= ~(1 << pri);
                }
                port = (i << EVTCHN_SHIFT) + j;
                irq = evtchn_to_irq[port];
                if (irq == INVALID_IRQ) {
                    /*
                     * No longer a handler for this event
                     * channel.  Clear the event and
                     * ignore it, unmask the event.
                     */
                    ec_clear_evtchn(port);
                    ec_unmask_evtchn(port);
                    continue;
                }
                if (irq == ec_dev_irq) {
                    volatile int *tptr = &ec_dev_mbox;

                    ASSERT(ec_dev_mbox == 0);
                    /*
                     * NOTE: this gross store thru a pointer
                     * is necessary because of a Sun C
                     * compiler bug that does not properly
                     * honor a volatile declaration.
                     * we really should just be able to say
                     *  ec_dev_mbox = port;
                     * here
                     */
                    *tptr = port;
                }
                /*
                 * Set up the regs struct to
                 * look like a normal hardware int
                 * and do normal interrupt handling.
                 */
                rp->r_trapno = irq;
                do_interrupt(rp, ttp);
                /*
                 * Check for cpu priority change
                 * Can happen if int thread blocks
                 */
                if (cpu->cpu_pri > curpri)
                    return;
            }
        }
    }
}

void
ec_unmask_evtchn(unsigned int ev)
{
    uint_t evi;
    volatile shared_info_t *si = HYPERVISOR_shared_info;
    volatile vcpu_info_t *vci = CPU->cpu_m.mcpu_vcpu_info;
    volatile ulong_t *ulp;

    ASSERT(!interrupts_enabled());
    /*
     * Check if we need to take slow path
     */
    if (!CPU_IN_SET(evtchn_cpus[ev], CPU->cpu_id)) {
        xen_evtchn_unmask(ev);
        return;
    }
    evi = ev >> EVTCHN_SHIFT;
    ev &= (1ul << EVTCHN_SHIFT) - 1;
    ulp = (volatile ulong_t *)&si->evtchn_mask[evi];
    atomic_and_ulong(ulp, ~(1ul << ev));
    /*
     * The following is basically the equivalent of
     * 'hw_resend_irq'. Just like a real IO-APIC we 'lose the
     * interrupt edge' if the channel is masked.
     * XXPV - slight race if upcall was about to be set, we may get
     * an extra upcall.
     */
    membar_enter();
    if (si->evtchn_pending[evi] & (1ul << ev)) {
        membar_consumer();
        ulp = (volatile ulong_t *)&vci->evtchn_pending_sel;
        if (!(*ulp & (1ul << evi))) {
            atomic_or_ulong(ulp, (1ul << evi));
        }
        vci->evtchn_upcall_pending = 1;
    }
}

/*
 * Set a bit in an evtchan mask word, return true if we are the cpu that
 * set the bit.
 */
int
ec_mask_evtchn(unsigned int ev)
{
    uint_t evi, evb;
    ulong_t new, old, bit;
    volatile shared_info_t *si = HYPERVISOR_shared_info;
    volatile ulong_t *maskp;
    int masked;

    kpreempt_disable();
    evi = ev >> EVTCHN_SHIFT;
    evb = ev & ((1ul << EVTCHN_SHIFT) - 1);
    bit = 1ul << evb;
    maskp = (volatile ulong_t *)&si->evtchn_mask[evi];
    do {
        old = si->evtchn_mask[evi];
        new = old | bit;
    } while (atomic_cas_ulong(maskp, old, new) != old);
    masked = (old & bit) == 0;
    if (masked) {
        evtchn_owner[ev] = CPU->cpu_id;
#ifdef DEBUG
        evtchn_owner_thread[ev] = curthread;
#endif
    }
    kpreempt_enable();
    return (masked);
}

void
ec_clear_evtchn(unsigned int ev)
{
    uint_t evi;
    shared_info_t *si = HYPERVISOR_shared_info;
    volatile ulong_t *pendp;

    evi = ev >> EVTCHN_SHIFT;
    ev &= (1ul << EVTCHN_SHIFT) - 1;
    pendp = (volatile ulong_t *)&si->evtchn_pending[evi];
    atomic_and_ulong(pendp, ~(1ul << ev));
}

void
ec_notify_via_evtchn(unsigned int port)
{
    evtchn_send_t send;

    ASSERT(port != INVALID_EVTCHN);

    send.port = port;
    (void) HYPERVISOR_event_channel_op(EVTCHNOP_send, &send);
}

int
ec_block_irq(int irq)
{
    irq_info_t *irqp = &irq_info[irq];
    int evtchn;


    evtchn = irq_evtchn(irqp);
    (void) ec_mask_evtchn(evtchn);
    return (evtchn_owner[evtchn]);
}

/*
 * Make a event that is pending for delivery on the current cpu  "go away"
 * without servicing the interrupt.
 */
void
ec_unpend_irq(int irq)
{
    irq_info_t *irqp = &irq_info[irq];
    int pri = irqp->ii_u2.ipl;
    ulong_t flags;
    uint_t evtchn, evi, bit;
    unsigned long pe, pending_sels;
    struct xen_evt_data *cpe;

    /*
     * The evtchn must be masked
     */
    evtchn = irq_evtchn(irqp);
    ASSERT(EVTCHN_MASKED(evtchn));
    evi = evtchn >> EVTCHN_SHIFT;
    bit = evtchn & (1ul << EVTCHN_SHIFT) - 1;
    flags = intr_clear();
    cpe = CPU->cpu_m.mcpu_evt_pend;
    pe = cpe->pending_evts[pri][evi] & ~(1ul << bit);
    cpe->pending_evts[pri][evi] = pe;
    if (pe == 0) {
        pending_sels = cpe->pending_sel[pri];
        pending_sels &= ~(1ul << evi);
        cpe->pending_sel[pri] = pending_sels;
        if (pending_sels == 0)
            CPU->cpu_m.mcpu_intr_pending &= ~(1 << pri);
    }
    intr_restore(flags);
}