clients/rdsv3/rdsv3_af_thr.c

/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*
 * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved.
 */
#include <sys/ib/clients/rdsv3/ib.h>
#include <sys/ib/clients/rdsv3/rdsv3_af_thr_impl.h>
#include <sys/ib/clients/rdsv3/rdsv3_debug.h>

extern pri_t maxclsyspri;
extern kmutex_t cpu_lock;

int rdsv3_enable_snd_cq = 0;
int rdsv3_intr_line_up_mode = 0;
static kmutex_t rdsv3_cpuid_pool_lock;

void
rdsv3_af_init(dev_info_t *dip)
{
    int i;
    cpu_t *cp;
    int *msix;
    uint_t nmsix;
    extern int ncpus;

    mutex_init(&rdsv3_cpuid_pool_lock, NULL, MUTEX_DEFAULT, NULL);
    if (ncpus < RDSV3_CPUID_POOL_MAX)
        rdsv3_cpuid_pool_cnt = ncpus;
    else
        rdsv3_cpuid_pool_cnt = RDSV3_CPUID_POOL_MAX;

    /* hold cpu_lock before calling cpu_get and cpu_is_online */
    mutex_enter(&cpu_lock);
    for (i = 0; i < rdsv3_cpuid_pool_cnt; i++) {
        cp = cpu_get((processorid_t)i);
        if (cp == NULL || !cpu_is_online(cp))
            rdsv3_cpuid_pool[i] = RDSV3_CPUFLAGS_OFF;
        else
            rdsv3_cpuid_pool[i] = RDSV3_CPUFLAGS_ON;
    }
    mutex_exit(&cpu_lock);

    if (ddi_prop_lookup_int_array(DDI_DEV_T_ANY, dip, DDI_PROP_DONTPASS,
        "HcaMsix", (int **)&msix, &nmsix) == DDI_PROP_SUCCESS) {
        /* remove the hca MSI-x interrupt cpu's */
        for (i = 0; i < nmsix; i++) {
            rdsv3_cpuid_pool[msix[i]] |= RDSV3_CPUFLAGS_INTR;
            rdsv3_msix_pool[i] = msix[i];
        }
        rdsv3_msix_pool_cnt = nmsix;
        ddi_prop_free(msix);
    }
    rdsv3_enable_snd_cq = ddi_prop_get_int(DDI_DEV_T_ANY, dip,
        DDI_PROP_DONTPASS, "EnableSendCQ", 0);
    rdsv3_intr_line_up_mode = ddi_prop_get_int(DDI_DEV_T_ANY, dip,
        DDI_PROP_DONTPASS, "IntrLineUpMode", 0);
}

static void
rdsv3_af_cpu_assign(rdsv3_af_grp_t *hcagp)
{
    int i, j, k, idx;

    RDSV3_DPRINTF2("rdsv3_af_cpu_assign", "hcagp %p", hcagp);

    mutex_enter(&rdsv3_cpuid_pool_lock);
    for (i = 0; i < rdsv3_cpuid_pool_cnt; i++) {
        if (!(rdsv3_cpuid_pool[i] & (RDSV3_CPUFLAGS_UNAVAIL |
            RDSV3_CPUFLAGS_ASSIGNED | RDSV3_CPUFLAGS_HCA))) {
            rdsv3_cpuid_pool[i] |= RDSV3_CPUFLAGS_HCA;
            hcagp->g_hca_cpuid = i;
            break;
        }
        /* share an assigned cpu */
        for (j = 0; j < rdsv3_cpuid_pool_cnt; j++) {
            if (!(rdsv3_cpuid_pool[j] & (RDSV3_CPUFLAGS_UNAVAIL |
                RDSV3_CPUFLAGS_HCA))) {
                hcagp->g_hca_cpuid = j;
                break;
            }
        }
        /* if the code comes down here, cpu 0 will be used */
    }

    for (j = 0; j < RDSV3_AFT_CONN_CPU_POOL; j++) {
        /* initialize to be an out-of-bound cpuid, no binding */
        hcagp->g_conn_cpuid_pool[j] = rdsv3_cpuid_pool_cnt;
        for (i = 0; i < rdsv3_cpuid_pool_cnt; i++) {
            if (!(rdsv3_cpuid_pool[i] & (RDSV3_CPUFLAGS_UNAVAIL |
                RDSV3_CPUFLAGS_ASSIGNED | RDSV3_CPUFLAGS_HCA))) {
                rdsv3_cpuid_pool[i] |= RDSV3_CPUFLAGS_ASSIGNED;
                hcagp->g_conn_cpuid_pool[j] = i;
                break;
            }
        }
        if (i >= rdsv3_cpuid_pool_cnt)
            break;
    }
    if (j >= RDSV3_AFT_CONN_CPU_POOL) {
        mutex_exit(&rdsv3_cpuid_pool_lock);
        return;
    }
    /* avoid the primary group */
    for (k = 0, idx = 0; k < 2; k++) {
        /* search to the start of an hca group */
        for (i = idx; i < rdsv3_cpuid_pool_cnt; i++) {
            if (rdsv3_cpuid_pool[i] & RDSV3_CPUFLAGS_HCA) {
                idx = i + 1;
                break;
            }
        }
    }
    /* share an assigned cpu */
    for (; j < RDSV3_AFT_CONN_CPU_POOL; j++) {
        for (i = idx; i < rdsv3_cpuid_pool_cnt; i++) {
            if (!(rdsv3_cpuid_pool[i] & (RDSV3_CPUFLAGS_UNAVAIL |
                RDSV3_CPUFLAGS_HCA))) {
                hcagp->g_conn_cpuid_pool[j] = i;
                idx = i + 1;
                break;
            }
        }
    }
    mutex_exit(&rdsv3_cpuid_pool_lock);
}

rdsv3_af_grp_t *
rdsv3_af_grp_create(ibt_hca_hdl_t hca, uint64_t id)
{
    char name[128];
    ibt_cq_sched_attr_t cq_sched_attr;
    ibt_status_t status;
    rdsv3_af_grp_t *hcagp;
    uint64_t l_id = id;

    hcagp = kmem_zalloc(sizeof (*hcagp), KM_NOSLEEP);
    if (!hcagp)
        return (NULL);
    hcagp->g_hca_hdl = hca;

    rdsv3_af_cpu_assign(hcagp);
    return (hcagp);
}

void
rdsv3_af_grp_destroy(rdsv3_af_grp_t *hcagp)
{
    if (hcagp == NULL)
        return;

    kmem_free(hcagp, sizeof (*hcagp));
}

void
rdsv3_af_grp_draw(rdsv3_af_grp_t *hcagp)
{
    rdsv3_af_grp_t *l_hcagp = hcagp;
}

ibt_sched_hdl_t
rdsv3_af_grp_get_sched(rdsv3_af_grp_t *hcagp)
{
    return (hcagp->g_sched_hdl);
}

rdsv3_af_thr_t *
rdsv3_af_intr_thr_create(rdsv3_af_thr_drain_func_t fn, void *data, uint_t flag,
    rdsv3_af_grp_t *hcagp, ibt_cq_hdl_t ibt_cq_hdl)
{
    rdsv3_af_thr_t *ringp;
    processorid_t cpuid;

    if (ibt_cq_hdl == NULL)
        return (NULL);
    ringp = rdsv3_af_thr_create(fn, data, flag, hcagp);
    if (ringp == NULL)
        return (NULL);

    mutex_enter(&cpu_lock);
    if (hcagp->g_conn_cpuid_idx >= RDSV3_AFT_CONN_CPU_POOL)
        hcagp->g_conn_cpuid_idx = 0;
    cpuid =  hcagp->g_conn_cpuid_pool[hcagp->g_conn_cpuid_idx++];
    (void) rdsv3_af_thr_bind(ringp, cpuid);
    mutex_exit(&cpu_lock);

    if (ringp->aft_intr) {
        if (rdsv3_intr_line_up_mode) {
            (void) ddi_intr_set_affinity(ringp->aft_intr, cpuid);
        } else {
            (void) ddi_intr_set_affinity(ringp->aft_intr,
                rdsv3_msix_pool[0]);
        }
    }
    return (ringp);
}

rdsv3_af_thr_t *
rdsv3_af_thr_create(rdsv3_af_thr_drain_func_t fn, void *data, uint_t flag,
    rdsv3_af_grp_t *hcagp)
{
    rdsv3_af_thr_t *ringp;
    pri_t pri;
    uint_t l_flags = flag;
    rdsv3_af_grp_t *l_hcagp = hcagp;

    ringp = kmem_zalloc(sizeof (rdsv3_af_thr_t), KM_NOSLEEP);
    if (ringp == NULL)
        return (NULL);

    ringp->aft_grp = hcagp;
    mutex_init(&ringp->aft_lock, NULL, MUTEX_DEFAULT, NULL);
    cv_init(&ringp->aft_async, NULL, CV_DEFAULT, NULL);
    if (flag & SCQ_WRK_BIND_CPU)
        pri = maxclsyspri;
    else
        pri = maxclsyspri;
    ringp->aft_worker = thread_create(NULL, 0,
        rdsv3_af_thr_worker, ringp, 0, &p0, TS_RUN, pri);
    ringp->aft_data = data;
    ringp->aft_drain_func = (rdsv3_af_thr_drain_func_t)fn;

    /* set the bind CPU to -1 to indicate no thread affinity set */
    ringp->aft_cpuid = -1;
    ringp->aft_state = 0;
    ringp->aft_cflag = flag;

    if (flag & SCQ_BIND_CPU) {
        mutex_enter(&cpu_lock);
        if (flag & SCQ_HCA_BIND_CPU) {
            (void) rdsv3_af_thr_bind(ringp, hcagp->g_hca_cpuid);
        } else if (flag & SCQ_WRK_BIND_CPU) {
            (void) rdsv3_af_thr_bind(ringp, hcagp->g_hca_cpuid);
        }
        mutex_exit(&cpu_lock);
    }

    RDSV3_DPRINTF2("rdsv3_af_thr_create", "af_thr %p ic %p", ringp, data);
    return (ringp);
}

void
rdsv3_af_thr_destroy(rdsv3_af_thr_t *ringp)
{
    RDSV3_DPRINTF2("rdsv3_af_thr_destroy", "af_thr %p", ringp);

    /* wait until the af_thr has gone to sleep */
    mutex_enter(&ringp->aft_lock);
    while (ringp->aft_state & AFT_PROC) {
        mutex_exit(&ringp->aft_lock);
        delay(drv_usectohz(1000));
        mutex_enter(&ringp->aft_lock);
    }
    ringp->aft_state |= AFT_CONDEMNED;
    if (!(ringp->aft_state & AFT_PROC)) {
        cv_signal(&ringp->aft_async);
    }
    mutex_exit(&ringp->aft_lock);
}

void
rdsv3_af_thr_fire(rdsv3_af_thr_t *ringp)
{
    mutex_enter(&ringp->aft_lock);
    ringp->aft_state |= AFT_ARMED;
    if (!(ringp->aft_state & AFT_PROC)) {
        cv_signal(&ringp->aft_async);
    }
    mutex_exit(&ringp->aft_lock);
}

static void
rdsv3_af_thr_worker(rdsv3_af_thr_t *ringp)
{
    kmutex_t *lock = &ringp->aft_lock;
    kcondvar_t *async = &ringp->aft_async;
    callb_cpr_t cprinfo;

    RDSV3_DPRINTF4("rdsv3_af_thr_worker", "Enter af_thr %p", ringp);

    CALLB_CPR_INIT(&cprinfo, lock, callb_generic_cpr, "rdsv3_af_thr");
    mutex_enter(lock);
    for (;;) {
        while (!(ringp->aft_state & (AFT_ARMED | AFT_CONDEMNED))) {
            CALLB_CPR_SAFE_BEGIN(&cprinfo);
            cv_wait(async, lock);
            CALLB_CPR_SAFE_END(&cprinfo, lock);
        }
        ringp->aft_state &= ~AFT_ARMED;

        /*
         * Either we have work to do, or we have been asked to
         * shutdown
         */
        if (ringp->aft_state & AFT_CONDEMNED)
            goto done;
        ASSERT(!(ringp->aft_state & AFT_PROC));
        ringp->aft_state |= AFT_PROC;
        mutex_exit(&ringp->aft_lock);

        ringp->aft_drain_func(ringp->aft_data);

        mutex_enter(&ringp->aft_lock);
        ringp->aft_state &= ~AFT_PROC;
    }
done:
    CALLB_CPR_EXIT(&cprinfo);
    RDSV3_DPRINTF2("rdsv3_af_thr_worker", "Exit af_thr %p", ringp);
    cv_destroy(&ringp->aft_async);
    mutex_destroy(&ringp->aft_lock);
    kmem_free(ringp, sizeof (rdsv3_af_thr_t));
    thread_exit();
}


int rdsv3_af_thr_thread_bind = 1;

/*
 * Bind a soft ring worker thread to supplied CPU.
 */
cpu_t *
rdsv3_af_thr_bind(rdsv3_af_thr_t *ringp, processorid_t cpuid)
{
    cpu_t *cp;
    boolean_t clear = B_FALSE;

    ASSERT(MUTEX_HELD(&cpu_lock));

    if (rdsv3_af_thr_thread_bind == 0) {
        return (NULL);
    }

    cp = cpu_get(cpuid);
    if (cp == NULL || !cpu_is_online(cp))
        return (NULL);

    mutex_enter(&ringp->aft_lock);
    ringp->aft_state |= AFT_BOUND;
    if (ringp->aft_cpuid != -1)
        clear = B_TRUE;
    ringp->aft_cpuid = cpuid;
    mutex_exit(&ringp->aft_lock);

    if (clear)
        thread_affinity_clear(ringp->aft_worker);

    RDSV3_DPRINTF4("rdsv3_af_thr_bind", "Bound af_thr %p to cpu %d",
        ringp, cpuid);
    thread_affinity_set(ringp->aft_worker, cpuid);
    return (cp);
}

/*
 * Un Bind a soft ring worker thread.
 */
static void
rdsv3_af_thr_unbind(rdsv3_af_thr_t *ringp)
{
    ASSERT(MUTEX_HELD(&cpu_lock));

    mutex_enter(&ringp->aft_lock);
    if (!(ringp->aft_state & AFT_BOUND)) {
        ASSERT(ringp->aft_cpuid == -1);
        mutex_exit(&ringp->aft_lock);
        return;
    }

    ringp->aft_cpuid = -1;
    ringp->aft_state &= ~AFT_BOUND;
    thread_affinity_clear(ringp->aft_worker);
    mutex_exit(&ringp->aft_lock);
}