sun4v/io/cnex.c

	cnex.c revision 0de66c941512d3ab0d15397815d8c9c552bcbf8d
/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*
 * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */


/*
 * Logical domain channel devices are devices implemented entirely
 * in software; cnex is the nexus for channel-devices. They use
 * the HV channel interfaces via the LDC transport module to send
 * and receive data and to register callbacks.
 */

#include <sys/types.h>
#include <sys/cmn_err.h>
#include <sys/conf.h>
#include <sys/ddi.h>
#include <sys/ddi_impldefs.h>
#include <sys/devops.h>
#include <sys/instance.h>
#include <sys/modctl.h>
#include <sys/open.h>
#include <sys/stat.h>
#include <sys/sunddi.h>
#include <sys/sunndi.h>
#include <sys/systm.h>
#include <sys/mkdev.h>
#include <sys/machsystm.h>
#include <sys/intreg.h>
#include <sys/intr.h>
#include <sys/ddi_intr_impl.h>
#include <sys/ivintr.h>
#include <sys/hypervisor_api.h>
#include <sys/ldc.h>
#include <sys/cnex.h>
#include <sys/mach_descrip.h>
#include <sys/hsvc.h>
#include <sys/sdt.h>

/*
 * Internal functions/information
 */
static struct cnex_intr_map cnex_class_to_intr[] = {
    {LDC_DEV_GENERIC,   PIL_3,   0},
    {LDC_DEV_BLK,       PIL_4,  10},
    {LDC_DEV_BLK_SVC,   PIL_3,  10},
    {LDC_DEV_NT,        PIL_6,  35},
    {LDC_DEV_NT_SVC,    PIL_4,  35},
    {LDC_DEV_SERIAL,    PIL_6,   0}
};
#define CNEX_MAX_DEVS (sizeof (cnex_class_to_intr) / \
                sizeof (cnex_class_to_intr[0]))

#define CNEX_TX_INTR_WEIGHT 0

#define SUN4V_REG_SPEC2CFG_HDL(x)   ((x >> 32) & ~(0xfull << 28))

static clock_t cnex_wait_usecs = 1000; /* wait time in usecs */
static int cnex_wait_retries = 3;
static void *cnex_state;

static uint_t cnex_intr_wrapper(caddr_t arg);
static dev_info_t *cnex_find_chan_dip(dev_info_t *dip, uint64_t chan_id,
    md_t *mdp, mde_cookie_t mde);

/*
 * Channel Interrupt Distribution
 *
 * In order to balance interrupts among available CPUs, we use
 * the intr_dist_cpuid_{add,remove}_device_weight() interface to
 * assign weights to channel interrupts. These weights, which are
 * defined in the cnex_intr_map structure, influence which CPU
 * is returned by intr_dist_cpuid() when called via the cnex
 * interrupt redistribution callback cnex_intr_redist().
 * Interrupts for VIO devclass channels are given more weight than
 * other interrupts because they are expected to occur more
 * frequently and have a larger impact on overall performance.
 * Transmit interrupts are given a zero weight because they are
 * not used.
 *
 * The interrupt weights influence the target CPU selection when
 * interrupts are redistributed and when they are added. However,
 * removal of interrupts can unbalance the distribution even if
 * they are removed in converse order--compared to the order they
 * are added. This can occur when interrupts are removed after
 * redistribution occurs.
 *
 * Channel interrupt weights affect interrupt-CPU distribution
 * relative to other weighted interrupts on the system. For VIO
 * devclass channels, values are chosen to match those used by
 * the PCI express nexus driver for net and storage devices.
 */
static void cnex_intr_redist(void *arg, int32_t weight_max, int32_t weight);
static int cnex_intr_new_cpu(cnex_soft_state_t *ssp, cnex_intr_t *iinfo);
static int cnex_intr_dis_wait(cnex_soft_state_t *ssp, cnex_intr_t *iinfo);
static int32_t cnex_class_weight(ldc_dev_t devclass);

/*
 * Debug info
 */
#ifdef DEBUG

/*
 * Print debug messages
 *
 * set cnexdbg to 0xf for enabling all msgs
 * 0x8 - Errors
 * 0x4 - Warnings
 * 0x2 - All debug messages
 * 0x1 - Minimal debug messages
 */

int cnexdbg = 0x8;

static void
cnexdebug(const char *fmt, ...)
{
    char buf[512];
    va_list ap;

    va_start(ap, fmt);
    (void) vsprintf(buf, fmt, ap);
    va_end(ap);

    cmn_err(CE_CONT, "%s\n", buf);
}

#define D1      \
if (cnexdbg & 0x01) \
    cnexdebug

#define D2      \
if (cnexdbg & 0x02) \
    cnexdebug

#define DWARN       \
if (cnexdbg & 0x04) \
    cnexdebug

#define DERR        \
if (cnexdbg & 0x08) \
    cnexdebug

#else

#define D1
#define D2
#define DWARN
#define DERR

#endif

/*
 * Config information
 */
static int cnex_attach(dev_info_t *, ddi_attach_cmd_t);
static int cnex_detach(dev_info_t *, ddi_detach_cmd_t);
static int cnex_open(dev_t *, int, int, cred_t *);
static int cnex_close(dev_t, int, int, cred_t *);
static int cnex_ioctl(dev_t, int, intptr_t, int, cred_t *, int *);
static int cnex_ctl(dev_info_t *, dev_info_t *, ddi_ctl_enum_t, void *,
    void *);

static struct bus_ops cnex_bus_ops = {
    BUSO_REV,
    nullbusmap,     /* bus_map */
    NULL,           /* bus_get_intrspec */
    NULL,           /* bus_add_intrspec */
    NULL,           /* bus_remove_intrspec */
    i_ddi_map_fault,    /* bus_map_fault */
    ddi_no_dma_map,     /* bus_dma_map */
    ddi_no_dma_allochdl,    /* bus_dma_allochdl */
    NULL,           /* bus_dma_freehdl */
    NULL,           /* bus_dma_bindhdl */
    NULL,           /* bus_dma_unbindhdl */
    NULL,           /* bus_dma_flush */
    NULL,           /* bus_dma_win */
    NULL,           /* bus_dma_ctl */
    cnex_ctl,       /* bus_ctl */
    ddi_bus_prop_op,    /* bus_prop_op */
    0,          /* bus_get_eventcookie */
    0,          /* bus_add_eventcall */
    0,          /* bus_remove_eventcall */
    0,          /* bus_post_event */
    NULL,           /* bus_intr_ctl */
    NULL,           /* bus_config */
    NULL,           /* bus_unconfig */
    NULL,           /* bus_fm_init */
    NULL,           /* bus_fm_fini */
    NULL,           /* bus_fm_access_enter */
    NULL,           /* bus_fm_access_exit */
    NULL,           /* bus_power */
    NULL            /* bus_intr_op */
};

static struct cb_ops cnex_cb_ops = {
    cnex_open,          /* open */
    cnex_close,         /* close */
    nodev,              /* strategy */
    nodev,              /* print */
    nodev,              /* dump */
    nodev,              /* read */
    nodev,              /* write */
    cnex_ioctl,         /* ioctl */
    nodev,              /* devmap */
    nodev,              /* mmap */
    nodev,              /* segmap */
    nochpoll,           /* poll */
    ddi_prop_op,            /* cb_prop_op */
    0,              /* streamtab  */
    D_MP | D_NEW | D_HOTPLUG    /* Driver compatibility flag */
};

static struct dev_ops cnex_ops = {
    DEVO_REV,       /* devo_rev, */
    0,          /* refcnt  */
    ddi_getinfo_1to1,   /* info */
    nulldev,        /* identify */
    nulldev,        /* probe */
    cnex_attach,        /* attach */
    cnex_detach,        /* detach */
    nodev,          /* reset */
    &cnex_cb_ops,       /* driver operations */
    &cnex_bus_ops,      /* bus operations */
    nulldev,        /* power */
    ddi_quiesce_not_needed,     /* quiesce */
};

/*
 * Module linkage information for the kernel.
 */
static struct modldrv modldrv = {
    &mod_driverops,
    "sun4v channel-devices nexus",
    &cnex_ops,
};

static struct modlinkage modlinkage = {
    MODREV_1, (void *)&modldrv, NULL
};

int
_init(void)
{
    int err;
    uint64_t majornum;
    uint64_t minornum;

    /*
     * Check HV intr group api versioning.
     * Note that cnex assumes interrupt cookies is
     * in version 1.0 of the intr group api.
     */
    if ((err = hsvc_version(HSVC_GROUP_INTR, &majornum, &minornum)) != 0) {
        cmn_err(CE_WARN, "cnex: failed to get intr api "
            "group versioning errno=%d", err);
        return (err);
    } else if ((majornum != 1) && (majornum != 2)) {
        cmn_err(CE_WARN, "cnex: unsupported intr api group: "
            "maj:0x%lx, min:0x%lx", majornum, minornum);
        return (ENOTSUP);
    }

    if ((err = ddi_soft_state_init(&cnex_state,
        sizeof (cnex_soft_state_t), 0)) != 0) {
        return (err);
    }
    if ((err = mod_install(&modlinkage)) != 0) {
        ddi_soft_state_fini(&cnex_state);
        return (err);
    }
    return (0);
}

int
_fini(void)
{
    int err;

    if ((err = mod_remove(&modlinkage)) != 0)
        return (err);
    ddi_soft_state_fini(&cnex_state);
    return (0);
}

int
_info(struct modinfo *modinfop)
{
    return (mod_info(&modlinkage, modinfop));
}

/*
 * Callback function invoked by the interrupt redistribution
 * framework. This will redirect interrupts at CPUs that are
 * currently available in the system.
 *
 * Note: any interrupts with weight greater than or equal to
 * weight_max must be redistributed when this callback is
 * invoked with (weight == weight_max) which will be once per
 * redistribution.
 */
/*ARGSUSED*/
static void
cnex_intr_redist(void *arg, int32_t weight_max, int32_t weight)
{
    cnex_ldc_t      *cldcp;
    cnex_soft_state_t   *cnex_ssp = arg;

    ASSERT(cnex_ssp != NULL);
    mutex_enter(&cnex_ssp->clist_lock);

    cldcp = cnex_ssp->clist;
    while (cldcp != NULL) {

        mutex_enter(&cldcp->lock);

        if (cldcp->tx.hdlr && (cldcp->tx.weight == weight ||
            (weight_max == weight && cldcp->tx.weight > weight))) {
            (void) cnex_intr_new_cpu(cnex_ssp, &cldcp->tx);
        }

        if (cldcp->rx.hdlr && (cldcp->rx.weight == weight ||
            (weight_max == weight && cldcp->rx.weight > weight))) {
            (void) cnex_intr_new_cpu(cnex_ssp, &cldcp->rx);
        }

        mutex_exit(&cldcp->lock);

        /* next channel */
        cldcp = cldcp->next;
    }

    mutex_exit(&cnex_ssp->clist_lock);
}

/*
 * Internal function to replace the CPU used by an interrupt
 * during interrupt redistribution.
 */
static int
cnex_intr_new_cpu(cnex_soft_state_t *ssp, cnex_intr_t *iinfo)
{
    int intr_state;
    int     rv;

    /* Determine if the interrupt is enabled */
    rv = hvldc_intr_getvalid(ssp->cfghdl, iinfo->ino, &intr_state);
    if (rv) {
        DWARN("cnex_intr_new_cpu: rx ino=0x%llx, can't get valid\n",
            iinfo->ino);
        return (rv);
    }

    /* If it is enabled, disable it */
    if (intr_state == HV_INTR_VALID) {
        rv = cnex_intr_dis_wait(ssp, iinfo);
        if (rv) {
            return (rv);
        }
    }

    /* Target the interrupt at a new CPU. */
    iinfo->cpuid = intr_dist_cpuid();
    (void) hvldc_intr_settarget(ssp->cfghdl, iinfo->ino, iinfo->cpuid);
    intr_dist_cpuid_add_device_weight(iinfo->cpuid, iinfo->dip,
        iinfo->weight);

    /* Re-enable the interrupt if it was enabled */
    if (intr_state == HV_INTR_VALID) {
        (void) hvldc_intr_setvalid(ssp->cfghdl, iinfo->ino,
            HV_INTR_VALID);
    }

    return (0);
}

/*
 * Internal function to disable an interrupt and wait
 * for any pending interrupts to finish.
 */
static int
cnex_intr_dis_wait(cnex_soft_state_t *ssp, cnex_intr_t *iinfo)
{
    int rv, intr_state, retries;

    /* disable interrupts */
    rv = hvldc_intr_setvalid(ssp->cfghdl, iinfo->ino, HV_INTR_NOTVALID);
    if (rv) {
        DWARN("cnex_intr_dis_wait: ino=0x%llx, can't set valid\n",
            iinfo->ino);
        return (ENXIO);
    }

    /*
     * Make a best effort to wait for pending interrupts
     * to finish. There is not much we can do if we timeout.
     */
    retries = 0;

    do {
        rv = hvldc_intr_getstate(ssp->cfghdl, iinfo->ino, &intr_state);
        if (rv) {
            DWARN("cnex_intr_dis_wait: ino=0x%llx, can't get "
                "state\n", iinfo->ino);
            return (ENXIO);
        }

        if (intr_state != HV_INTR_DELIVERED_STATE)
            break;

        drv_usecwait(cnex_wait_usecs);

    } while (!panicstr && ++retries <= cnex_wait_retries);

    return (0);
}

/*
 * Returns the interrupt weight to use for the specified devclass.
 */
static int32_t
cnex_class_weight(ldc_dev_t devclass)
{
    int idx;

    for (idx = 0; idx < CNEX_MAX_DEVS; idx++) {
        if (devclass == cnex_class_to_intr[idx].devclass) {
            return (cnex_class_to_intr[idx].weight);
        }
    }

    /*
     * If this code is reached, the specified devclass is
     * invalid. New devclasses should be added to
     * cnex_class_to_intr.
     */
    ASSERT(0);

    return (0);
}

/*
 * Exported interface to register a LDC endpoint with
 * the channel nexus
 */
static int
cnex_reg_chan(dev_info_t *dip, uint64_t id, ldc_dev_t devclass)
{
    int     idx;
    cnex_ldc_t  *cldcp;
    cnex_ldc_t  *new_cldcp;
    int     listsz, num_nodes, num_channels;
    md_t        *mdp = NULL;
    mde_cookie_t    rootnode, *listp = NULL;
    uint64_t    tmp_id;
    uint64_t    rxino = (uint64_t)-1;
    uint64_t    txino = (uint64_t)-1;
    cnex_soft_state_t *cnex_ssp;
    int     status, instance;
    dev_info_t  *chan_dip = NULL;

    /* Get device instance and structure */
    instance = ddi_get_instance(dip);
    cnex_ssp = ddi_get_soft_state(cnex_state, instance);

    /* Check to see if channel is already registered */
    mutex_enter(&cnex_ssp->clist_lock);
    cldcp = cnex_ssp->clist;
    while (cldcp) {
        if (cldcp->id == id) {
            DWARN("cnex_reg_chan: channel 0x%llx exists\n", id);
            mutex_exit(&cnex_ssp->clist_lock);
            return (EINVAL);
        }
        cldcp = cldcp->next;
    }
    mutex_exit(&cnex_ssp->clist_lock);

    /* Get the Tx/Rx inos from the MD */
    if ((mdp = md_get_handle()) == NULL) {
        DWARN("cnex_reg_chan: cannot init MD\n");
        return (ENXIO);
    }
    num_nodes = md_node_count(mdp);
    ASSERT(num_nodes > 0);

    listsz = num_nodes * sizeof (mde_cookie_t);
    listp = (mde_cookie_t *)kmem_zalloc(listsz, KM_SLEEP);

    rootnode = md_root_node(mdp);

    /* search for all channel_endpoint nodes */
    num_channels = md_scan_dag(mdp, rootnode,
        md_find_name(mdp, "channel-endpoint"),
        md_find_name(mdp, "fwd"), listp);
    if (num_channels <= 0) {
        DWARN("cnex_reg_chan: invalid channel id\n");
        kmem_free(listp, listsz);
        (void) md_fini_handle(mdp);
        return (EINVAL);
    }

    for (idx = 0; idx < num_channels; idx++) {

        /* Get the channel ID */
        status = md_get_prop_val(mdp, listp[idx], "id", &tmp_id);
        if (status) {
            DWARN("cnex_reg_chan: cannot read LDC ID\n");
            kmem_free(listp, listsz);
            (void) md_fini_handle(mdp);
            return (ENXIO);
        }
        if (tmp_id != id)
            continue;

        /* Get the Tx and Rx ino */
        status = md_get_prop_val(mdp, listp[idx], "tx-ino", &txino);
        if (status) {
            DWARN("cnex_reg_chan: cannot read Tx ino\n");
            kmem_free(listp, listsz);
            (void) md_fini_handle(mdp);
            return (ENXIO);
        }
        status = md_get_prop_val(mdp, listp[idx], "rx-ino", &rxino);
        if (status) {
            DWARN("cnex_reg_chan: cannot read Rx ino\n");
            kmem_free(listp, listsz);
            (void) md_fini_handle(mdp);
            return (ENXIO);
        }
        chan_dip = cnex_find_chan_dip(dip, id, mdp, listp[idx]);
        ASSERT(chan_dip != NULL);
    }
    kmem_free(listp, listsz);
    (void) md_fini_handle(mdp);

    /*
     * check to see if we looped through the list of channel IDs without
     * matching one (i.e. an 'ino' has not been initialised).
     */
    if ((rxino == -1) || (txino == -1)) {
        DERR("cnex_reg_chan: no ID matching '%llx' in MD\n", id);
        return (ENOENT);
    }

    /* Allocate a new channel structure */
    new_cldcp = kmem_zalloc(sizeof (*new_cldcp), KM_SLEEP);

    /* Initialize the channel */
    mutex_init(&new_cldcp->lock, NULL, MUTEX_DRIVER, NULL);

    new_cldcp->id = id;
    new_cldcp->tx.ino = txino;
    new_cldcp->rx.ino = rxino;
    new_cldcp->devclass = devclass;
    new_cldcp->tx.weight = CNEX_TX_INTR_WEIGHT;
    new_cldcp->rx.weight = cnex_class_weight(devclass);
    new_cldcp->dip = chan_dip;

    /*
     * Add channel to nexus channel list.
     * Check again to see if channel is already registered since
     * clist_lock was dropped above.
     */
    mutex_enter(&cnex_ssp->clist_lock);
    cldcp = cnex_ssp->clist;
    while (cldcp) {
        if (cldcp->id == id) {
            DWARN("cnex_reg_chan: channel 0x%llx exists\n", id);
            mutex_exit(&cnex_ssp->clist_lock);
            mutex_destroy(&new_cldcp->lock);
            kmem_free(new_cldcp, sizeof (*new_cldcp));
            return (EINVAL);
        }
        cldcp = cldcp->next;
    }
    new_cldcp->next = cnex_ssp->clist;
    cnex_ssp->clist = new_cldcp;
    mutex_exit(&cnex_ssp->clist_lock);

    return (0);
}

/*
 * Add Tx/Rx interrupt handler for the channel
 */
static int
cnex_add_intr(dev_info_t *dip, uint64_t id, cnex_intrtype_t itype,
    uint_t (*hdlr)(), caddr_t arg1, caddr_t arg2)
{
    int     rv, idx, pil;
    cnex_ldc_t  *cldcp;
    cnex_intr_t *iinfo;
    cnex_soft_state_t *cnex_ssp;
    int     instance;

    /* Get device instance and structure */
    instance = ddi_get_instance(dip);
    cnex_ssp = ddi_get_soft_state(cnex_state, instance);

    /* get channel info */
    mutex_enter(&cnex_ssp->clist_lock);
    cldcp = cnex_ssp->clist;
    while (cldcp) {
        if (cldcp->id == id)
            break;
        cldcp = cldcp->next;
    }
    if (cldcp == NULL) {
        DWARN("cnex_add_intr: channel 0x%llx does not exist\n", id);
        mutex_exit(&cnex_ssp->clist_lock);
        return (EINVAL);
    }
    mutex_exit(&cnex_ssp->clist_lock);

    /* get channel lock */
    mutex_enter(&cldcp->lock);

    /* get interrupt type */
    if (itype == CNEX_TX_INTR) {
        iinfo = &(cldcp->tx);
    } else if (itype == CNEX_RX_INTR) {
        iinfo = &(cldcp->rx);
    } else {
        DWARN("cnex_add_intr: invalid interrupt type\n", id);
        mutex_exit(&cldcp->lock);
        return (EINVAL);
    }

    /* check if a handler is already added */
    if (iinfo->hdlr != 0) {
        DWARN("cnex_add_intr: interrupt handler exists\n");
        mutex_exit(&cldcp->lock);
        return (EINVAL);
    }

    /* save interrupt handler info */
    iinfo->hdlr = hdlr;
    iinfo->arg1 = arg1;
    iinfo->arg2 = arg2;

    /* save data for DTrace probes used by intrstat(1m) */
    iinfo->dip = cldcp->dip;
    iinfo->id = cldcp->id;

    iinfo->icookie = MINVINTR_COOKIE + iinfo->ino;

    /*
     * Verify that the ino does not generate a cookie which
     * is outside the (MINVINTR_COOKIE, MAXIVNUM) range of the
     * system interrupt table.
     */
    if (iinfo->icookie >= MAXIVNUM || iinfo->icookie < MINVINTR_COOKIE) {
        DWARN("cnex_add_intr: invalid cookie %x ino %x\n",
            iinfo->icookie, iinfo->ino);
        mutex_exit(&cldcp->lock);
        return (EINVAL);
    }

    D1("cnex_add_intr: add hdlr, cfghdl=0x%llx, ino=0x%llx, "
        "cookie=0x%llx\n", cnex_ssp->cfghdl, iinfo->ino, iinfo->icookie);

    /* Pick a PIL on the basis of the channel's devclass */
    for (idx = 0, pil = PIL_3; idx < CNEX_MAX_DEVS; idx++) {
        if (cldcp->devclass == cnex_class_to_intr[idx].devclass) {
            pil = cnex_class_to_intr[idx].pil;
            break;
        }
    }

    /* add interrupt to solaris ivec table */
    if (add_ivintr(iinfo->icookie, pil, (intrfunc)cnex_intr_wrapper,
        (caddr_t)iinfo, NULL, NULL) != 0) {
        DWARN("cnex_add_intr: add_ivintr fail cookie %x ino %x\n",
            iinfo->icookie, iinfo->ino);
        mutex_exit(&cldcp->lock);
        return (EINVAL);
    }

    /* set the cookie in the HV */
    rv = hvldc_intr_setcookie(cnex_ssp->cfghdl, iinfo->ino, iinfo->icookie);

    /* pick next CPU in the domain for this channel */
    iinfo->cpuid = intr_dist_cpuid();

    /* set the target CPU and then enable interrupts */
    rv = hvldc_intr_settarget(cnex_ssp->cfghdl, iinfo->ino, iinfo->cpuid);
    if (rv) {
        DWARN("cnex_add_intr: ino=0x%llx, cannot set target cpu\n",
            iinfo->ino);
        goto hv_error;
    }
    rv = hvldc_intr_setstate(cnex_ssp->cfghdl, iinfo->ino,
        HV_INTR_IDLE_STATE);
    if (rv) {
        DWARN("cnex_add_intr: ino=0x%llx, cannot set state\n",
            iinfo->ino);
        goto hv_error;
    }
    rv = hvldc_intr_setvalid(cnex_ssp->cfghdl, iinfo->ino, HV_INTR_VALID);
    if (rv) {
        DWARN("cnex_add_intr: ino=0x%llx, cannot set valid\n",
            iinfo->ino);
        goto hv_error;
    }

    intr_dist_cpuid_add_device_weight(iinfo->cpuid, iinfo->dip,
        iinfo->weight);

    mutex_exit(&cldcp->lock);
    return (0);

hv_error:
    (void) rem_ivintr(iinfo->icookie, pil);
    mutex_exit(&cldcp->lock);
    return (ENXIO);
}


/*
 * Exported interface to unregister a LDC endpoint with
 * the channel nexus
 */
static int
cnex_unreg_chan(dev_info_t *dip, uint64_t id)
{
    cnex_ldc_t  *cldcp, *prev_cldcp;
    cnex_soft_state_t *cnex_ssp;
    int     instance;

    /* Get device instance and structure */
    instance = ddi_get_instance(dip);
    cnex_ssp = ddi_get_soft_state(cnex_state, instance);

    /* find and remove channel from list */
    mutex_enter(&cnex_ssp->clist_lock);
    prev_cldcp = NULL;
    cldcp = cnex_ssp->clist;
    while (cldcp) {
        if (cldcp->id == id)
            break;
        prev_cldcp = cldcp;
        cldcp = cldcp->next;
    }

    if (cldcp == 0) {
        DWARN("cnex_unreg_chan: invalid channel %d\n", id);
        mutex_exit(&cnex_ssp->clist_lock);
        return (EINVAL);
    }

    if (cldcp->tx.hdlr || cldcp->rx.hdlr) {
        DWARN("cnex_unreg_chan: handlers still exist: chan %lx\n", id);
        mutex_exit(&cnex_ssp->clist_lock);
        return (ENXIO);
    }

    if (prev_cldcp)
        prev_cldcp->next = cldcp->next;
    else
        cnex_ssp->clist = cldcp->next;

    mutex_exit(&cnex_ssp->clist_lock);

    /* destroy mutex */
    mutex_destroy(&cldcp->lock);

    /* free channel */
    kmem_free(cldcp, sizeof (*cldcp));

    return (0);
}

/*
 * Remove Tx/Rx interrupt handler for the channel
 */
static int
cnex_rem_intr(dev_info_t *dip, uint64_t id, cnex_intrtype_t itype)
{
    int         rv, idx, pil;
    cnex_ldc_t      *cldcp;
    cnex_intr_t     *iinfo;
    cnex_soft_state_t   *cnex_ssp;
    int         instance, istate;

    /* Get device instance and structure */
    instance = ddi_get_instance(dip);
    cnex_ssp = ddi_get_soft_state(cnex_state, instance);

    /* get channel info */
    mutex_enter(&cnex_ssp->clist_lock);
    cldcp = cnex_ssp->clist;
    while (cldcp) {
        if (cldcp->id == id)
            break;
        cldcp = cldcp->next;
    }
    if (cldcp == NULL) {
        DWARN("cnex_rem_intr: channel 0x%llx does not exist\n", id);
        mutex_exit(&cnex_ssp->clist_lock);
        return (EINVAL);
    }
    mutex_exit(&cnex_ssp->clist_lock);

    /* get rid of the channel intr handler */
    mutex_enter(&cldcp->lock);

    /* get interrupt type */
    if (itype == CNEX_TX_INTR) {
        iinfo = &(cldcp->tx);
    } else if (itype == CNEX_RX_INTR) {
        iinfo = &(cldcp->rx);
    } else {
        DWARN("cnex_rem_intr: invalid interrupt type\n");
        mutex_exit(&cldcp->lock);
        return (EINVAL);
    }

    D1("cnex_rem_intr: interrupt ino=0x%x\n", iinfo->ino);

    /* check if a handler is already added */
    if (iinfo->hdlr == 0) {
        DWARN("cnex_rem_intr: interrupt handler does not exist\n");
        mutex_exit(&cldcp->lock);
        return (EINVAL);
    }

    D1("cnex_rem_intr: set intr to invalid ino=0x%x\n", iinfo->ino);
    rv = hvldc_intr_setvalid(cnex_ssp->cfghdl,
        iinfo->ino, HV_INTR_NOTVALID);
    if (rv) {
        DWARN("cnex_rem_intr: cannot set valid ino=%x\n", iinfo->ino);
        mutex_exit(&cldcp->lock);
        return (ENXIO);
    }

    /*
     * Check if there are pending interrupts. If interrupts are
     * pending return EAGAIN.
     */
    rv = hvldc_intr_getstate(cnex_ssp->cfghdl, iinfo->ino, &istate);
    if (rv) {
        DWARN("cnex_rem_intr: ino=0x%llx, cannot get state\n",
            iinfo->ino);
        mutex_exit(&cldcp->lock);
        return (ENXIO);
    }

    /* if interrupts are still pending print warning */
    if (istate != HV_INTR_IDLE_STATE) {
        DWARN("cnex_rem_intr: cannot remove intr busy ino=%x\n",
            iinfo->ino);
        mutex_exit(&cldcp->lock);
        return (EAGAIN);
    }

    /* Pick a PIL on the basis of the channel's devclass */
    for (idx = 0, pil = PIL_3; idx < CNEX_MAX_DEVS; idx++) {
        if (cldcp->devclass == cnex_class_to_intr[idx].devclass) {
            pil = cnex_class_to_intr[idx].pil;
            break;
        }
    }

    intr_dist_cpuid_rem_device_weight(iinfo->cpuid, iinfo->dip);

    /* remove interrupt */
    (void) rem_ivintr(iinfo->icookie, pil);

    /* clear interrupt info */
    bzero(iinfo, sizeof (*iinfo));

    mutex_exit(&cldcp->lock);

    return (0);
}


/*
 * Clear pending Tx/Rx interrupt
 */
static int
cnex_clr_intr(dev_info_t *dip, uint64_t id, cnex_intrtype_t itype)
{
    int         rv;
    cnex_ldc_t      *cldcp;
    cnex_intr_t     *iinfo;
    cnex_soft_state_t   *cnex_ssp;
    int         instance;

    /* Get device instance and structure */
    instance = ddi_get_instance(dip);
    cnex_ssp = ddi_get_soft_state(cnex_state, instance);

    /* get channel info */
    mutex_enter(&cnex_ssp->clist_lock);
    cldcp = cnex_ssp->clist;
    while (cldcp) {
        if (cldcp->id == id)
            break;
        cldcp = cldcp->next;
    }
    if (cldcp == NULL) {
        DWARN("cnex_clr_intr: channel 0x%llx does not exist\n", id);
        mutex_exit(&cnex_ssp->clist_lock);
        return (EINVAL);
    }
    mutex_exit(&cnex_ssp->clist_lock);

    mutex_enter(&cldcp->lock);

    /* get interrupt type */
    if (itype == CNEX_TX_INTR) {
        iinfo = &(cldcp->tx);
    } else if (itype == CNEX_RX_INTR) {
        iinfo = &(cldcp->rx);
    } else {
        DWARN("cnex_clr_intr: invalid interrupt type\n");
        mutex_exit(&cldcp->lock);
        return (EINVAL);
    }

    D1("%s: interrupt ino=0x%x\n", __func__, iinfo->ino);

    /* check if a handler is already added */
    if (iinfo->hdlr == 0) {
        DWARN("cnex_clr_intr: interrupt handler does not exist\n");
        mutex_exit(&cldcp->lock);
        return (EINVAL);
    }

    rv = hvldc_intr_setstate(cnex_ssp->cfghdl, iinfo->ino,
        HV_INTR_IDLE_STATE);
    if (rv) {
        DWARN("cnex_clr_intr: cannot clear interrupt state\n");
        mutex_exit(&cldcp->lock);
        return (ENXIO);
    }

    mutex_exit(&cldcp->lock);

    return (0);
}

/*
 * Channel nexus interrupt handler wrapper
 */
static uint_t
cnex_intr_wrapper(caddr_t arg)
{
    int             res;
    uint_t          (*handler)();
    caddr_t         handler_arg1;
    caddr_t         handler_arg2;
    cnex_intr_t         *iinfo = (cnex_intr_t *)arg;

    ASSERT(iinfo != NULL);

    handler = iinfo->hdlr;
    handler_arg1 = iinfo->arg1;
    handler_arg2 = iinfo->arg2;

    /*
     * The 'interrupt__start' and 'interrupt__complete' probes
     * are provided to support 'intrstat' command. These probes
     * help monitor the interrupts on a per device basis only.
     * In order to provide the ability to monitor the
     * activity on a per channel basis, two additional
     * probes('channelintr__start','channelintr__complete')
     * are provided here.
     */
    DTRACE_PROBE4(channelintr__start, uint64_t, iinfo->id,
        cnex_intr_t *, iinfo, void *, handler, caddr_t, handler_arg1);

    DTRACE_PROBE4(interrupt__start, dev_info_t, iinfo->dip,
        void *, handler, caddr_t, handler_arg1, caddr_t, handler_arg2);

    D1("cnex_intr_wrapper:ino=0x%llx invoke client handler\n", iinfo->ino);
    res = (*handler)(handler_arg1, handler_arg2);

    DTRACE_PROBE4(interrupt__complete, dev_info_t, iinfo->dip,
        void *, handler, caddr_t, handler_arg1, int, res);

    DTRACE_PROBE4(channelintr__complete, uint64_t, iinfo->id,
        cnex_intr_t *, iinfo, void *, handler, caddr_t, handler_arg1);

    return (res);
}

/*ARGSUSED*/
static int
cnex_attach(dev_info_t *devi, ddi_attach_cmd_t cmd)
{
    int         rv, instance, reglen;
    cnex_regspec_t  *reg_p;
    ldc_cnex_t  cinfo;
    cnex_soft_state_t *cnex_ssp;

    switch (cmd) {
    case DDI_ATTACH:
        break;
    case DDI_RESUME:
        return (DDI_SUCCESS);
    default:
        return (DDI_FAILURE);
    }

    /*
     * Get the instance specific soft state structure.
     * Save the devi for this instance in the soft_state data.
     */
    instance = ddi_get_instance(devi);
    if (ddi_soft_state_zalloc(cnex_state, instance) != DDI_SUCCESS)
        return (DDI_FAILURE);
    cnex_ssp = ddi_get_soft_state(cnex_state, instance);

    cnex_ssp->devi = devi;
    cnex_ssp->clist = NULL;

    if (ddi_getlongprop(DDI_DEV_T_ANY, devi, DDI_PROP_DONTPASS,
        "reg", (caddr_t)&reg_p, &reglen) != DDI_SUCCESS) {
        return (DDI_FAILURE);
    }

    /* get the sun4v config handle for this device */
    cnex_ssp->cfghdl = SUN4V_REG_SPEC2CFG_HDL(reg_p->physaddr);
    kmem_free(reg_p, reglen);

    D1("cnex_attach: cfghdl=0x%llx\n", cnex_ssp->cfghdl);

    /* init channel list mutex */
    mutex_init(&cnex_ssp->clist_lock, NULL, MUTEX_DRIVER, NULL);

    /* Register with LDC module */
    cinfo.dip = devi;
    cinfo.reg_chan = cnex_reg_chan;
    cinfo.unreg_chan = cnex_unreg_chan;
    cinfo.add_intr = cnex_add_intr;
    cinfo.rem_intr = cnex_rem_intr;
    cinfo.clr_intr = cnex_clr_intr;

    /*
     * LDC register will fail if an nexus instance had already
     * registered with the LDC framework
     */
    rv = ldc_register(&cinfo);
    if (rv) {
        DWARN("cnex_attach: unable to register with LDC\n");
        ddi_soft_state_free(cnex_state, instance);
        mutex_destroy(&cnex_ssp->clist_lock);
        return (DDI_FAILURE);
    }

    if (ddi_create_minor_node(devi, "devctl", S_IFCHR, instance,
        DDI_NT_NEXUS, 0) != DDI_SUCCESS) {
        ddi_remove_minor_node(devi, NULL);
        ddi_soft_state_free(cnex_state, instance);
        mutex_destroy(&cnex_ssp->clist_lock);
        return (DDI_FAILURE);
    }

    /* Add interrupt redistribution callback. */
    intr_dist_add_weighted(cnex_intr_redist, cnex_ssp);

    ddi_report_dev(devi);
    return (DDI_SUCCESS);
}

/*ARGSUSED*/
static int
cnex_detach(dev_info_t *devi, ddi_detach_cmd_t cmd)
{
    int         instance;
    ldc_cnex_t  cinfo;
    cnex_soft_state_t *cnex_ssp;

    switch (cmd) {
    case DDI_DETACH:
        break;
    case DDI_SUSPEND:
        return (DDI_SUCCESS);
    default:
        return (DDI_FAILURE);
    }

    instance = ddi_get_instance(devi);
    cnex_ssp = ddi_get_soft_state(cnex_state, instance);

    /* check if there are any channels still registered */
    if (cnex_ssp->clist) {
        cmn_err(CE_WARN, "?cnex_dettach: channels registered %d\n",
            ddi_get_instance(devi));
        return (DDI_FAILURE);
    }

    /* Unregister with LDC module */
    cinfo.dip = devi;
    (void) ldc_unregister(&cinfo);

    /* Remove interrupt redistribution callback. */
    intr_dist_rem_weighted(cnex_intr_redist, cnex_ssp);

    /* destroy mutex */
    mutex_destroy(&cnex_ssp->clist_lock);

    /* free soft state structure */
    ddi_soft_state_free(cnex_state, instance);

    return (DDI_SUCCESS);
}

/*ARGSUSED*/
static int
cnex_open(dev_t *devp, int flags, int otyp, cred_t *credp)
{
    int instance;

    if (otyp != OTYP_CHR)
        return (EINVAL);

    instance = getminor(*devp);
    if (ddi_get_soft_state(cnex_state, instance) == NULL)
        return (ENXIO);

    return (0);
}

/*ARGSUSED*/
static int
cnex_close(dev_t dev, int flags, int otyp, cred_t *credp)
{
    int instance;

    if (otyp != OTYP_CHR)
        return (EINVAL);

    instance = getminor(dev);
    if (ddi_get_soft_state(cnex_state, instance) == NULL)
        return (ENXIO);

    return (0);
}

/*ARGSUSED*/
static int
cnex_ioctl(dev_t dev,
    int cmd, intptr_t arg, int mode, cred_t *cred_p, int *rval_p)
{
    int instance;
    cnex_soft_state_t *cnex_ssp;

    instance = getminor(dev);
    if ((cnex_ssp = ddi_get_soft_state(cnex_state, instance)) == NULL)
        return (ENXIO);
    ASSERT(cnex_ssp->devi);
    return (ndi_devctl_ioctl(cnex_ssp->devi, cmd, arg, mode, 0));
}

static int
cnex_ctl(dev_info_t *dip, dev_info_t *rdip, ddi_ctl_enum_t ctlop,
    void *arg, void *result)
{
    char        name[MAXNAMELEN];
    uint32_t    reglen;
    int     *cnex_regspec;

    switch (ctlop) {
    case DDI_CTLOPS_REPORTDEV:
        if (rdip == NULL)
            return (DDI_FAILURE);
        cmn_err(CE_CONT, "?channel-device: %s%d\n",
            ddi_driver_name(rdip), ddi_get_instance(rdip));
        return (DDI_SUCCESS);

    case DDI_CTLOPS_INITCHILD:
    {
        dev_info_t *child = (dev_info_t *)arg;

        if (ddi_prop_lookup_int_array(DDI_DEV_T_ANY, child,
            DDI_PROP_DONTPASS, "reg",
            &cnex_regspec, &reglen) != DDI_SUCCESS) {
            return (DDI_FAILURE);
        }

        (void) snprintf(name, sizeof (name), "%x", *cnex_regspec);
        ddi_set_name_addr(child, name);
        ddi_set_parent_data(child, NULL);
        ddi_prop_free(cnex_regspec);
        return (DDI_SUCCESS);
    }

    case DDI_CTLOPS_UNINITCHILD:
    {
        dev_info_t *child = (dev_info_t *)arg;

        NDI_CONFIG_DEBUG((CE_NOTE,
            "DDI_CTLOPS_UNINITCHILD(%s, instance=%d)",
            ddi_driver_name(child), DEVI(child)->devi_instance));

        ddi_set_name_addr(child, NULL);

        return (DDI_SUCCESS);
    }

    case DDI_CTLOPS_DMAPMAPC:
    case DDI_CTLOPS_REPORTINT:
    case DDI_CTLOPS_REGSIZE:
    case DDI_CTLOPS_NREGS:
    case DDI_CTLOPS_SIDDEV:
    case DDI_CTLOPS_SLAVEONLY:
    case DDI_CTLOPS_AFFINITY:
    case DDI_CTLOPS_POKE:
    case DDI_CTLOPS_PEEK:
        /*
         * These ops correspond to functions that "shouldn't" be called
         * by a channel-device driver.  So we whine when we're called.
         */
        cmn_err(CE_WARN, "%s%d: invalid op (%d) from %s%d\n",
            ddi_driver_name(dip), ddi_get_instance(dip), ctlop,
            ddi_driver_name(rdip), ddi_get_instance(rdip));
        return (DDI_FAILURE);

    case DDI_CTLOPS_ATTACH:
    case DDI_CTLOPS_BTOP:
    case DDI_CTLOPS_BTOPR:
    case DDI_CTLOPS_DETACH:
    case DDI_CTLOPS_DVMAPAGESIZE:
    case DDI_CTLOPS_IOMIN:
    case DDI_CTLOPS_POWER:
    case DDI_CTLOPS_PTOB:
    default:
        /*
         * Everything else (e.g. PTOB/BTOP/BTOPR requests) we pass up
         */
        return (ddi_ctlops(dip, rdip, ctlop, arg, result));
    }
}

/*
 * cnex_find_chan_dip -- Find the dip of a device that is corresponding
 *  to the specific channel. Below are the details on how the dip
 *  is derived.
 *
 *  - In the MD, the cfg-handle is expected to be unique for
 *    virtual-device nodes that have the same 'name' property value.
 *    This value is expected to be the same as that of "reg" property
 *    of the corresponding OBP device node.
 *
 *  - The value of the 'name' property of a virtual-device node
 *    in the MD is expected to be the same for the corresponding
 *    OBP device node.
 *
 *  - Find the virtual-device node corresponding to a channel-endpoint
 *    by walking backwards. Then obtain the values for the 'name' and
 *    'cfg-handle' properties.
 *
 *  - Walk all the children of the cnex, find a matching dip which
 *    has the same 'name' and 'reg' property values.
 *
 *  - The channels that have no corresponding device driver are
 *    treated as if they  correspond to the cnex driver,
 *    that is, return cnex dip for them. This means, the
 *    cnex acts as an umbrella device driver. Note, this is
 *    for 'intrstat' statistics purposes only. As a result of this,
 *    the 'intrstat' shows cnex as the device that is servicing the
 *    interrupts corresponding to these channels.
 *
 *    For now, only one such case is known, that is, the channels that
 *    are used by the "domain-services".
 */
static dev_info_t *
cnex_find_chan_dip(dev_info_t *dip, uint64_t chan_id,
    md_t *mdp, mde_cookie_t mde)
{
    int listsz;
    int num_nodes;
    int num_devs;
    uint64_t cfghdl;
    char *md_name;
    mde_cookie_t *listp;
    dev_info_t *cdip = NULL;

    num_nodes = md_node_count(mdp);
    ASSERT(num_nodes > 0);
    listsz = num_nodes * sizeof (mde_cookie_t);
    listp = (mde_cookie_t *)kmem_zalloc(listsz, KM_SLEEP);

    num_devs = md_scan_dag(mdp, mde, md_find_name(mdp, "virtual-device"),
        md_find_name(mdp, "back"), listp);
    ASSERT(num_devs <= 1);
    if (num_devs <= 0) {
        DWARN("cnex_find_chan_dip:channel(0x%llx): "
            "No virtual-device found\n", chan_id);
        goto fdip_exit;
    }
    if (md_get_prop_str(mdp, listp[0], "name", &md_name) != 0) {
        DWARN("cnex_find_chan_dip:channel(0x%llx): "
            "name property not found\n", chan_id);
        goto fdip_exit;
    }

    D1("cnex_find_chan_dip: channel(0x%llx): virtual-device "
        "name property value = %s\n", chan_id, md_name);

    if (md_get_prop_val(mdp, listp[0], "cfg-handle", &cfghdl) != 0) {
        DWARN("cnex_find_chan_dip:channel(0x%llx): virtual-device's "
            "cfg-handle property not found\n", chan_id);
        goto fdip_exit;
    }

    D1("cnex_find_chan_dip:channel(0x%llx): virtual-device cfg-handle "
        " property value = 0x%x\n", chan_id, cfghdl);

    for (cdip = ddi_get_child(dip); cdip != NULL;
        cdip = ddi_get_next_sibling(cdip)) {

        int *cnex_regspec;
        uint32_t reglen;
        char    *dev_name;

        if (ddi_prop_lookup_string(DDI_DEV_T_ANY, cdip,
            DDI_PROP_DONTPASS, "name",
            &dev_name) != DDI_PROP_SUCCESS) {
            DWARN("cnex_find_chan_dip: name property not"
                " found for dip(0x%p)\n", cdip);
            continue;
        }
        if (strcmp(md_name, dev_name) != 0) {
            ddi_prop_free(dev_name);
            continue;
        }
        ddi_prop_free(dev_name);
        if (ddi_prop_lookup_int_array(DDI_DEV_T_ANY, cdip,
            DDI_PROP_DONTPASS, "reg",
            &cnex_regspec, &reglen) != DDI_SUCCESS) {
            DWARN("cnex_find_chan_dip: reg property not"
                " found for dip(0x%p)\n", cdip);
            continue;
        }
        if (*cnex_regspec == cfghdl) {
            D1("cnex_find_chan_dip:channel(0x%llx): found "
                "dip(0x%p) drvname=%s\n", chan_id, cdip,
                ddi_driver_name(cdip));
            ddi_prop_free(cnex_regspec);
            break;
        }
        ddi_prop_free(cnex_regspec);
    }

fdip_exit:
    if (cdip == NULL) {
        /*
         * If a virtual-device node exists but no dip found,
         * then for now print a DEBUG error message only.
         */
        if (num_devs > 0) {
            DERR("cnex_find_chan_dip:channel(0x%llx): "
                "No device found\n", chan_id);
        }

        /* If no dip was found, return cnex device's dip. */
        cdip = dip;
    }

    kmem_free(listp, listsz);
    D1("cnex_find_chan_dip:channel(0x%llx): returning dip=0x%p\n",
        chan_id, cdip);
    return (cdip);
}

/* -------------------------------------------------------------------------- */