of/sol_ofs/sol_cma.c

/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved.
 */

/*
 * sol_cma is a part of sol_ofs misc module. This file
 * provides interfaces for supporting the communication
 * management API defined in "rdma_cm.h". In-Kernel
 * consumers of the "rdma_cm.h" API should link sol_ofs
 * misc module using :
 *  -N misc/sol_ofs
 * Solaris uCMA (sol_ucma) driver is the current consumer for
 * sol_cma.
 */

/* Standard driver includes */
#include <sys/types.h>
#include <sys/modctl.h>
#include <sys/errno.h>
#include <sys/stat.h>
#include <sys/ddi.h>
#include <sys/sunddi.h>
#include <sys/modctl.h>

#include <sys/ib/clients/of/ofed_kernel.h>
#include <sys/ib/clients/of/rdma/ib_addr.h>

#include <sys/ib/clients/of/sol_ofs/sol_cma.h>
#include <sys/ib/clients/of/sol_ofs/sol_kverb_impl.h>

/* Modload support */
static struct modlmisc sol_ofs_modmisc  = {
    &mod_miscops,
    "Solaris OFS Misc module"
};

struct modlinkage sol_ofs_modlinkage = {
    MODREV_1,
    (void *)&sol_ofs_modmisc,
    NULL
};

static ib_client_t  *sol_cma_ib_client;
sol_cma_glbl_listen_t   sol_cma_glbl_listen;
avl_tree_t      sol_cma_glbl_listen_tree;

static void     sol_cma_add_dev(struct ib_device *);
static void     sol_cma_rem_dev(struct ib_device *);

static llist_head_t sol_cma_dev_list = LLIST_HEAD_INIT(sol_cma_dev_list);
kmutex_t        sol_cma_dev_mutex;
kmutex_t        sol_cma_glob_mutex;

char    *sol_rdmacm_dbg_str = "sol_rdmacm";
char    *sol_ofs_dbg_str = "sol_ofs_mod";

/*
 * Local functions defines.
 */
int sol_cma_req_cmid_cmp(const void *p1, const void *p2);
int sol_cma_cmid_cmp(const void *p1, const void *p2);
int sol_cma_svc_cmp(const void *, const void *);

static struct rdma_cm_id *cma_alloc_chan(rdma_cm_event_handler,
    void *, enum rdma_port_space);
static void cma_set_chan_state(sol_cma_chan_t *, cma_chan_state_t);
static int cma_cas_chan_state(sol_cma_chan_t *, cma_chan_state_t,
    cma_chan_state_t);
static void cma_free_listen_list(struct rdma_cm_id *);
static void cma_destroy_id(struct rdma_cm_id *);
static void cma_handle_nomore_events(sol_cma_chan_t *);

extern void sol_ofs_dprintf_init();
extern void sol_ofs_dprintf_fini();

cma_chan_state_t cma_get_chan_state(sol_cma_chan_t *);
extern int ibcma_init_root_chan(sol_cma_chan_t *, sol_cma_glbl_listen_t *);
extern int ibcma_fini_root_chan(sol_cma_chan_t *);
extern void ibcma_copy_srv_hdl(sol_cma_chan_t *, sol_cma_glbl_listen_t *);
extern int ibcma_fini_ep_chan(sol_cma_chan_t *);
extern uint64_t ibcma_init_root_sid(sol_cma_chan_t *);
extern void rdma_ib_destroy_id(struct rdma_cm_id *);
extern int rdma_ib_bind_addr(struct rdma_cm_id *, struct sockaddr *);
extern int rdma_ib_resolve_addr(struct rdma_cm_id *, struct sockaddr *,
    struct sockaddr *, int);
extern int rdma_ib_resolve_route(struct rdma_cm_id *, int);
extern int rdma_ib_init_qp_attr(struct rdma_cm_id *, struct ib_qp_attr *,
    int *);
extern int rdma_ib_connect(struct rdma_cm_id *, struct rdma_conn_param *);
extern int rdma_ib_listen(struct rdma_cm_id *, int);
extern int rdma_ib_accept(struct rdma_cm_id *, struct rdma_conn_param *);
extern int rdma_ib_reject(struct rdma_cm_id *, const void *, uint8_t);
extern int rdma_ib_disconnect(struct rdma_cm_id *);
extern int rdma_ib_join_multicast(struct rdma_cm_id *, struct sockaddr *,
    void *);
extern void rdma_ib_leave_multicast(struct rdma_cm_id *, struct sockaddr *);

int
_init(void)
{
    int     err;

    sol_ofs_dprintf_init();
    SOL_OFS_DPRINTF_L5(sol_ofs_dbg_str, "_init()");

    mutex_init(&sol_cma_glob_mutex, NULL, MUTEX_DRIVER, NULL);
    mutex_init(&sol_cma_dev_mutex, NULL, MUTEX_DRIVER, NULL);
    avl_create(&sol_cma_glbl_listen_tree,
        sol_cma_svc_cmp, sizeof (sol_cma_glbl_listen_t),
        offsetof(sol_cma_glbl_listen_t, cma_listen_node));

    sol_cma_ib_client = kmem_zalloc(sizeof (ib_client_t), KM_NOSLEEP);
    if (!sol_cma_ib_client) {
        SOL_OFS_DPRINTF_L2(sol_ofs_dbg_str,
            "_init() - mem alloc failed");
        avl_destroy(&sol_cma_glbl_listen_tree);
        mutex_destroy(&sol_cma_dev_mutex);
        mutex_destroy(&sol_cma_glob_mutex);
        sol_ofs_dprintf_fini();
        return (ENOMEM);
    }

    sol_cma_ib_client->name = "sol_ofs";
    sol_cma_ib_client->add = sol_cma_add_dev;
    sol_cma_ib_client->remove = sol_cma_rem_dev;
    sol_cma_ib_client->dip = NULL;

    if ((err = ib_register_client(sol_cma_ib_client)) != 0) {
        SOL_OFS_DPRINTF_L2(sol_ofs_dbg_str,
            "_init() ib_register_client() failed with err %d",
            err);
        kmem_free(sol_cma_ib_client, sizeof (ib_client_t));
        avl_destroy(&sol_cma_glbl_listen_tree);
        mutex_destroy(&sol_cma_dev_mutex);
        mutex_destroy(&sol_cma_glob_mutex);
        sol_ofs_dprintf_fini();
        return (err);
    }

    if ((err = mod_install(&sol_ofs_modlinkage)) != 0) {
        SOL_OFS_DPRINTF_L2(sol_ofs_dbg_str,
            "_init() - mod_install() failed");
        ib_unregister_client(sol_cma_ib_client);
        kmem_free(sol_cma_ib_client, sizeof (ib_client_t));
        avl_destroy(&sol_cma_glbl_listen_tree);
        mutex_destroy(&sol_cma_dev_mutex);
        mutex_destroy(&sol_cma_glob_mutex);
        sol_ofs_dprintf_fini();
        return (err);
    }

    SOL_OFS_DPRINTF_L5(sol_ofs_dbg_str, "_init() - ret");
    return (err);
}

int
_fini(void)
{
    int     err;

    SOL_OFS_DPRINTF_L5(sol_ofs_dbg_str, "_fini()");

    if (avl_numnodes(&sol_cma_glbl_listen_tree)) {
        SOL_OFS_DPRINTF_L2(sol_ofs_dbg_str, "_fini - "
            "listen CMIDs still active");
        return (EBUSY);
    }
    if ((err = mod_remove(&sol_ofs_modlinkage)) != 0) {
        SOL_OFS_DPRINTF_L3(sol_ofs_dbg_str,
            "_fini: mod_remove failed");
        return (err);
    }

    ib_unregister_client(sol_cma_ib_client);
    kmem_free(sol_cma_ib_client, sizeof (ib_client_t));
    avl_destroy(&sol_cma_glbl_listen_tree);
    mutex_destroy(&sol_cma_dev_mutex);
    mutex_destroy(&sol_cma_glob_mutex);
    SOL_OFS_DPRINTF_L5(sol_ofs_dbg_str, "_fini() - ret");
    sol_ofs_dprintf_fini();
    return (err);
}

int
_info(struct modinfo *modinfop)
{
    return (mod_info(&sol_ofs_modlinkage, modinfop));
}

typedef struct cma_device {
    kmutex_t        cma_mutex;
    /* Ptr in the global sol_cma_dev_list */
    llist_head_t        cma_list;
    /* List of listeners for this device */
    genlist_t       cma_epchan_list;
    struct ib_device    *cma_device;
    uint_t          cma_ref_count;
    enum {
        SOL_CMA_DEV_ADDED,
        SOL_CMA_DEV_REM_IN_PROGRESS
    } cma_dev_state;
} cma_device_t;

static void
sol_cma_add_dev(struct ib_device *dev)
{
    cma_device_t    *new_device;

    new_device = kmem_zalloc(sizeof (cma_device_t), KM_NOSLEEP);
    if (!new_device) {
        SOL_OFS_DPRINTF_L2(sol_ofs_dbg_str, "sol_cma_add_dev() "
            "alloc failed!!");
        return;
    }
    mutex_init(&new_device->cma_mutex, NULL, MUTEX_DRIVER, NULL);
    llist_head_init(&new_device->cma_list, new_device);
    init_genlist(&new_device->cma_epchan_list);
    new_device->cma_device = dev;

    ib_set_client_data(dev, sol_cma_ib_client, new_device);

    mutex_enter(&sol_cma_dev_mutex);
    llist_add_tail(&new_device->cma_list, &sol_cma_dev_list);
    mutex_exit(&sol_cma_dev_mutex);
}

static void
sol_cma_rem_dev(struct ib_device *dev)
{
    cma_device_t    *rem_device;
    genlist_entry_t *entry;

    SOL_OFS_DPRINTF_L5(sol_ofs_dbg_str, "sol_rem_dev(%p)", dev);

    rem_device = (cma_device_t *)ib_get_client_data(dev, sol_cma_ib_client);
    if (!rem_device) {
        SOL_OFS_DPRINTF_L2(sol_ofs_dbg_str, "sol_cma_rem_dev() "
            "NULL cma_dev!!");
        return;
    }

    mutex_enter(&rem_device->cma_mutex);
    rem_device->cma_dev_state = SOL_CMA_DEV_REM_IN_PROGRESS;
    if (rem_device->cma_ref_count) {
        mutex_exit(&rem_device->cma_mutex);
        SOL_OFS_DPRINTF_L3(sol_ofs_dbg_str, "sol_cma_rem_dev() "
            "BUSY cma_dev!!");
        return;
    }
    entry = remove_genlist_head(&rem_device->cma_epchan_list);
    while (entry) {
        sol_cma_chan_t  *ep_chanp;

        ep_chanp = (sol_cma_chan_t *)entry->data;
        if (ibcma_fini_ep_chan(ep_chanp) == 0) {
            genlist_entry_t *entry1;
            sol_cma_chan_t  *root_chanp;

            ASSERT(ep_chanp->chan_listenp);
            entry1 = ep_chanp->chan_listenp->listen_ep_root_entry;
            root_chanp = (sol_cma_chan_t *)ep_chanp->listen_root;
            root_chanp->chan_listenp->listen_eps--;
            delete_genlist(&root_chanp->chan_listenp->listen_list,
                entry1);

            kmem_free(ep_chanp, sizeof (sol_cma_chan_t));
            kmem_free(entry, sizeof (genlist_entry_t));
        }

        entry = remove_genlist_head(&rem_device->cma_epchan_list);
    }
    mutex_exit(&rem_device->cma_mutex);

    mutex_enter(&sol_cma_dev_mutex);
    llist_del(&rem_device->cma_list);
    mutex_exit(&sol_cma_dev_mutex);

    kmem_free(rem_device, sizeof (cma_device_t));
}

struct ib_device *
sol_cma_acquire_device(ib_guid_t hca_guid)
{
    llist_head_t    *entry;
    cma_device_t    *cma_devp;

    mutex_enter(&sol_cma_dev_mutex);
    list_for_each(entry, &sol_cma_dev_list) {
        cma_devp = (cma_device_t *)entry->ptr;

        if (cma_devp->cma_device->node_guid != hca_guid)
            continue;

        mutex_enter(&cma_devp->cma_mutex);
        if (cma_devp->cma_dev_state == SOL_CMA_DEV_REM_IN_PROGRESS) {
            SOL_OFS_DPRINTF_L3(sol_ofs_dbg_str,
                "sol_cma_acquire_dev() - Device getting removed!!");
            mutex_exit(&cma_devp->cma_mutex);
            mutex_exit(&sol_cma_dev_mutex);
            return (NULL);
        }
        cma_devp->cma_ref_count++;
        mutex_exit(&cma_devp->cma_mutex);
        mutex_exit(&sol_cma_dev_mutex);
        return (cma_devp->cma_device);

    }
    mutex_exit(&sol_cma_dev_mutex);
    return (NULL);
}

static void
sol_cma_release_device(struct rdma_cm_id *id)
{
    ib_device_t *device = id->device;
    llist_head_t    *entry;
    cma_device_t    *cma_devp;

    mutex_enter(&sol_cma_dev_mutex);
    list_for_each(entry, &sol_cma_dev_list) {
        cma_devp = (cma_device_t *)entry->ptr;

        if (cma_devp->cma_device != device)
            continue;

        mutex_enter(&cma_devp->cma_mutex);
        cma_devp->cma_ref_count--;
        if (cma_devp->cma_dev_state == SOL_CMA_DEV_REM_IN_PROGRESS &&
            cma_devp->cma_ref_count == 0) {
            SOL_OFS_DPRINTF_L3(sol_ofs_dbg_str,
                "sol_cma_release_dev() - Device free removed!!");
            mutex_exit(&cma_devp->cma_mutex);
            llist_del(&cma_devp->cma_list);
            kmem_free(cma_devp, sizeof (cma_device_t));
            mutex_exit(&sol_cma_dev_mutex);
            return;
        }
        mutex_exit(&cma_devp->cma_mutex);
    }
    mutex_exit(&sol_cma_dev_mutex);
}

void
sol_cma_add_hca_list(sol_cma_chan_t *ep_chanp, ib_guid_t hca_guid)
{
    llist_head_t    *entry;
    cma_device_t    *cma_devp;

    SOL_OFS_DPRINTF_L5(sol_rdmacm_dbg_str, "add_hca_list(%p, %llx)",
        ep_chanp, hca_guid);
    mutex_enter(&sol_cma_dev_mutex);
    list_for_each(entry, &sol_cma_dev_list) {
        cma_devp = (cma_device_t *)entry->ptr;

        if ((cma_devp->cma_device)->node_guid != hca_guid)
            continue;

        mutex_enter(&cma_devp->cma_mutex);
        ep_chanp->chan_listenp->listen_ep_dev_entry =
            add_genlist(&cma_devp->cma_epchan_list,
            (uintptr_t)ep_chanp, NULL);
        ep_chanp->chan_listenp->listen_ep_device = cma_devp->cma_device;
        mutex_exit(&cma_devp->cma_mutex);
        mutex_exit(&sol_cma_dev_mutex);
        return;
    }
    mutex_exit(&sol_cma_dev_mutex);
    SOL_OFS_DPRINTF_L2(sol_rdmacm_dbg_str, "add_hca_list(%p, %llx): "
        "No matching HCA in list!!", ep_chanp, hca_guid);
}

/*
 * rdma_cm.h API functions.
 */
struct rdma_cm_id *
rdma_create_id(rdma_cm_event_handler evt_hdlr, void *context,
    enum rdma_port_space ps)
{
    struct rdma_cm_id   *rdma_idp;

    SOL_OFS_DPRINTF_L5(sol_rdmacm_dbg_str, "rdma_create_id(%p, %p, %x)",
        evt_hdlr, context, ps);

    if (ps != RDMA_PS_TCP && ps != RDMA_PS_UDP && ps != RDMA_PS_IPOIB) {
        SOL_OFS_DPRINTF_L2(sol_rdmacm_dbg_str,
            "rdma_create_id: unsupported protocol %x", ps);
        return (NULL);
    }

    rdma_idp = cma_alloc_chan(evt_hdlr, context, ps);
    SOL_OFS_DPRINTF_L5(sol_rdmacm_dbg_str,
        "rdma_create_id : ret %p", rdma_idp);

    return (rdma_idp);
}

void
rdma_map_id2clnthdl(struct rdma_cm_id *rdma_idp, void *ib_client_hdl,
    void *iw_client_hdl)
{
    sol_cma_chan_t  *chanp = (sol_cma_chan_t *)rdma_idp;

    SOL_OFS_DPRINTF_L5(sol_rdmacm_dbg_str,
        "rdma_map_id2clnthdl(%p, %p, %p)",
        rdma_idp, ib_client_hdl, iw_client_hdl);
    ASSERT(ib_client_hdl != NULL || iw_client_hdl != NULL);
    chanp->chan_ib_client_hdl = ib_client_hdl;
    chanp->chan_iw_client_hdl = iw_client_hdl;
}

void
rdma_map_id2qphdl(struct rdma_cm_id *rdma_idp, void *qp_hdl)
{
    sol_cma_chan_t  *chanp = (sol_cma_chan_t *)rdma_idp;

    ASSERT(rdma_idp);
    ASSERT(qp_hdl);
    SOL_OFS_DPRINTF_L5(sol_rdmacm_dbg_str, "rdma_mapid2qphdl(%p, %p)",
        rdma_idp, qp_hdl);
    chanp->chan_qp_hdl = qp_hdl;
}


void
rdma_destroy_id(struct rdma_cm_id *rdma_idp)
{
    sol_cma_chan_t      *chanp, *root_chanp;
    cma_chan_state_t    state;
    int         rc, is_root_cmid, do_wait, is_passive;

    SOL_OFS_DPRINTF_L5(sol_rdmacm_dbg_str, "rdma_destroy_id(%p)", rdma_idp);

    if (!rdma_idp)
        return;

    is_root_cmid = do_wait = is_passive = 0;

    chanp = (sol_cma_chan_t *)rdma_idp;
    root_chanp = (sol_cma_chan_t *)chanp->listen_root;
    SOL_OFS_DPRINTF_L5(sol_rdmacm_dbg_str, "rdma_destroy_id(%p), %p",
        rdma_idp, root_chanp);

    mutex_enter(&chanp->chan_mutex);
    chanp->chan_cmid_destroy_state |= SOL_CMA_CALLER_CMID_DESTROYED;

    /*
     * Wait in destroy of CMID when rdma_resolve_addr() / rdma_listen()
     * rdma_resolve_route() API is in progress.
     */
    while (chanp->chan_cmid_destroy_state & SOL_CMA_CALLER_API_PROGRESS)
        cv_wait(&chanp->chan_destroy_cv, &chanp->chan_mutex);

    /* Wait if Event is been notified to consumer */
    while (chanp->chan_cmid_destroy_state & SOL_CMA_CALLER_EVENT_PROGRESS)
        cv_wait(&chanp->chan_destroy_cv, &chanp->chan_mutex);

    if (rdma_idp->device)
        sol_cma_release_device(rdma_idp);

    if (chanp->chan_listenp && chanp->chan_listenp->listen_is_root)
        is_root_cmid = 1;
    if (root_chanp == NULL && is_root_cmid == 0)
        is_passive = 1;

    /*
     * Skip Active side handling for passive CMIDs and listen CMID
     * for which REQ CMIDs have not been created.
     */
    if (is_passive || (is_root_cmid && chanp->chan_req_state !=
        REQ_CMID_QUEUED)) {
        SOL_OFS_DPRINTF_L5(sol_rdmacm_dbg_str, "rdma_destroy_id: "
            "Skipping passive %p, %x, %x", chanp->chan_listenp,
            is_root_cmid, chanp->chan_req_state);
        goto skip_passive_handling;
    }

    /*
     * destroy_id() called for listening CMID and there are REQ
     * CMIDs not yet notified. Reject such CMIDs and decrement
     * the count.
     */
    if (is_root_cmid && chanp->chan_req_cnt) {
        sol_cma_chan_t  *req_cmid_chan, *next_chan;

        SOL_OFS_DPRINTF_L5(sol_rdmacm_dbg_str, "rdma_destroy_id: "
            "not notified handling");
        for (req_cmid_chan = (sol_cma_chan_t *)avl_first(
            &chanp->chan_req_avl_tree); req_cmid_chan &&
            chanp->chan_req_cnt; req_cmid_chan = next_chan) {
            next_chan = AVL_NEXT(
                &chanp->chan_req_avl_tree, req_cmid_chan);
            if (req_cmid_chan->chan_req_state ==
                REQ_CMID_NOTIFIED) {
                avl_remove(&chanp->chan_req_avl_tree,
                    req_cmid_chan);
                chanp->chan_req_cnt--;
                chanp->chan_req_total_cnt--;
                mutex_exit(&chanp->chan_mutex);
                mutex_enter(&req_cmid_chan->chan_mutex);
                req_cmid_chan->chan_req_state =
                    REQ_CMID_SERVER_NONE;
                if (rdma_idp->ps == RDMA_PS_TCP)
                    cma_set_chan_state(req_cmid_chan,
                        SOL_CMA_CHAN_DESTROY_PENDING);
                mutex_exit(&req_cmid_chan->chan_mutex);
                (void) rdma_disconnect(
                    (struct rdma_cm_id *)req_cmid_chan);
                mutex_enter(&chanp->chan_mutex);
                if (rdma_idp->ps == RDMA_PS_TCP) {
                    mutex_enter(
                        &req_cmid_chan->chan_mutex);
                    req_cmid_chan->listen_root =
                        rdma_idp;
                    mutex_exit(
                        &req_cmid_chan->chan_mutex);
                } else {
                    mutex_destroy(
                        &req_cmid_chan->chan_mutex);
                    cv_destroy(
                        &req_cmid_chan->chan_destroy_cv);
                    kmem_free(req_cmid_chan,
                        sizeof (sol_cma_chan_t));
                }
            }
        }
    }

    /*
     * destroy_id() called for :
     *  listening CMID and all REQ CMIDs destroy_id() called
     *  REQ CMID and 1 more REQ CMID not yet destroyed.
     * wait till the CMID is completly destroyed.
     */
    if (is_root_cmid && chanp->chan_req_total_cnt == 0) {
        SOL_OFS_DPRINTF_L5(sol_rdmacm_dbg_str, "rdma_destroy_id: "
            "root idp waiting");
        cma_set_chan_state(chanp, SOL_CMA_CHAN_DESTROY_WAIT);
        cv_wait(&chanp->chan_destroy_cv, &chanp->chan_mutex);
    }
    mutex_exit(&chanp->chan_mutex);

    if (root_chanp)
        mutex_enter(&root_chanp->chan_mutex);
    mutex_enter(&chanp->chan_mutex);
#ifdef  DEBUG
    SOL_OFS_DPRINTF_L5(sol_rdmacm_dbg_str, "rdma_destroy_id: "
        "root_idp %p, cnt %x, state %x", root_chanp,
        root_chanp ? root_chanp->chan_req_total_cnt : 0,
        root_chanp ? cma_get_chan_state(root_chanp) : 0);
#endif

    if (root_chanp && root_chanp->chan_req_total_cnt == 1 &&
        cma_get_chan_state(root_chanp) == SOL_CMA_CHAN_DESTROY_PENDING)
        do_wait = 1;
    if (root_chanp)
        mutex_exit(&root_chanp->chan_mutex);

skip_passive_handling :
    state = cma_get_chan_state(chanp);
    if (is_root_cmid == 0 && state != SOL_CMA_CHAN_DISCONNECT &&
        SOL_CMAID_CONNECTED(chanp)) {
        /*
         * A connected CM ID has not been disconnected.
         * Call rdma_disconnect() to disconnect it.
         */
        mutex_exit(&chanp->chan_mutex);
        rc = rdma_disconnect(rdma_idp);
        if (rc) {
            SOL_OFS_DPRINTF_L2(sol_rdmacm_dbg_str,
                "rdma_destroy_id(%p)- disconnect failed!!",
                rdma_idp);
            return;
        }
        mutex_enter(&chanp->chan_mutex);
        if (root_chanp && chanp->listen_root == NULL)
            chanp->listen_root = (struct rdma_cm_id *)root_chanp;
        SOL_OFS_DPRINTF_L5(sol_rdmacm_dbg_str,
            "rdma_destroy_id(chanp %p, connect %x, ps %x)",
            chanp, chanp->chan_connect_flag, rdma_idp->ps);
        if (SOL_CMAID_CONNECTED(chanp)) {
            if (do_wait) {
                cma_set_chan_state(chanp,
                    SOL_CMA_CHAN_DESTROY_WAIT);
                cv_wait(&chanp->chan_destroy_cv,
                    &chanp->chan_mutex);
                mutex_exit(&chanp->chan_mutex);
                cma_destroy_id(rdma_idp);
            } else {
                cma_set_chan_state(chanp,
                    SOL_CMA_CHAN_DESTROY_PENDING);
                mutex_exit(&chanp->chan_mutex);
            }
        } else {
            /*
             * No more callbacks are expected for this CMID.
             * Free this CMID.
             */
            mutex_exit(&chanp->chan_mutex);
            cma_destroy_id(rdma_idp);
        }
    } else if (is_root_cmid == 0 && state ==
        SOL_CMA_CHAN_DISCONNECT && SOL_CMAID_CONNECTED(chanp)) {
        /*
         * CM ID was connected and disconnect is process.
         * Free of this CM ID is done for the DISCONNECT
         * notification for this CMID.
         */
        cma_set_chan_state(chanp, SOL_CMA_CHAN_DESTROY_PENDING);
        mutex_exit(&chanp->chan_mutex);
    } else if (state != SOL_CMA_CHAN_DESTROY_PENDING) {
        /* CM ID, not connected, just free it. */
        mutex_exit(&chanp->chan_mutex);
        cma_destroy_id(rdma_idp);
    } else
        mutex_exit(&chanp->chan_mutex);

    SOL_OFS_DPRINTF_L5(sol_rdmacm_dbg_str, "rdma_destroy_id: ret");
}

/*
 * State transitions for Address resolution :
 *  Active Side (Client) :
 *  1. CREATE_ID-->BIND_ADDR-->RESOLVE_ADDR-->RESOLVE_ROUTE
 *
 *  Passive Side (Server) :
 *  2. CREATE_ID-->RESOLVE_ADDR-->RESOLVE_ROUTE
 *  IF_ADDR_ANY can be passed as local address in RESOLVE_ADDR
 */
int
rdma_bind_addr(struct rdma_cm_id *idp, struct sockaddr *addr)
{
    sol_cma_chan_t      *chanp;
    struct rdma_addr    *addrp;
    int         ret;

    ASSERT(idp);
    ASSERT(addr);
    chanp = (sol_cma_chan_t *)idp;
    addrp = &(idp->route.addr);
    SOL_OFS_DPRINTF_L5(sol_rdmacm_dbg_str, "rdma_bind_addr(%p, %p)",
        idp, addr);

    mutex_enter(&chanp->chan_mutex);
    ret = cma_cas_chan_state(chanp, SOL_CMA_CHAN_IDLE, SOL_CMA_CHAN_BOUND);
    if (ret) {
        mutex_exit(&chanp->chan_mutex);
        return (ret);
    }
    /* Copy the local address to rdma_id structure */
    bcopy((void *)addr, (void *)&(addrp->src_addr),
        sizeof (struct sockaddr));
    mutex_exit(&chanp->chan_mutex);

    /*
     * First call rdma_ib_bind_addr() to bind this address.
     * Next call rdma_iw_bind_addr() to bind this address.
     * For IF_ADDR_ANY, IB address is given priority over
     * iWARP.
     */
    if (chanp->chan_ib_client_hdl == NULL) {
        ofs_client_t    *ofs_clnt;

        ofs_clnt = (ofs_client_t *)sol_cma_ib_client->clnt_hdl;
        chanp->chan_ib_client_hdl = ofs_clnt->ibt_hdl;
    }
    if (chanp->chan_ib_client_hdl && rdma_ib_bind_addr(idp, addr) == 0) {
        SOL_OFS_DPRINTF_L5(sol_rdmacm_dbg_str,
            "rdma_bind_addr: ret IB @");
        return (0);
#ifdef  IWARP_SUPPORT
    } else if (chanp->chan_iw_client_hdl && rdma_iw_bind_addr(idp, addr)
        == 0) {
        SOL_OFS_DPRINTF_L5(sol_rdmacm_dbg_str,
            "rdma_bind_addr: ret iWARP @");
        return (0);
#endif  /* IWARP_SUPPORT */
    }

    mutex_enter(&chanp->chan_mutex);
    cma_set_chan_state(chanp, SOL_CMA_CHAN_IDLE);
    mutex_exit(&chanp->chan_mutex);
    SOL_OFS_DPRINTF_L4(sol_rdmacm_dbg_str, "rdma_bind_addr: ret failure!");
    return (EINVAL);
}

int
rdma_resolve_addr(struct rdma_cm_id *idp, struct sockaddr *src_addr,
    struct sockaddr *dst_addr, int timeout_ms)
{
    sol_cma_chan_t      *chanp;
    struct rdma_addr    *addrp;
    cma_chan_state_t    state;

    ASSERT(idp);
    chanp = (sol_cma_chan_t *)idp;
    addrp = &(idp->route.addr);
    SOL_OFS_DPRINTF_L5(sol_rdmacm_dbg_str, "rdma_resolve_addr(%p, %p, "
        "%p, %x)", idp, src_addr, dst_addr, timeout_ms);

    mutex_enter(&chanp->chan_mutex);
    state = cma_get_chan_state(chanp);
    if (state != SOL_CMA_CHAN_IDLE && state != SOL_CMA_CHAN_BOUND) {
        SOL_OFS_DPRINTF_L2(sol_rdmacm_dbg_str,
            "rdma_resolve_addr : invalid chan state %x", state);
        mutex_exit(&chanp->chan_mutex);
        return (EINVAL);
    }
    if (chanp->chan_cmid_destroy_state &
        SOL_CMA_CALLER_CMID_DESTROYED) {
        SOL_OFS_DPRINTF_L3(sol_rdmacm_dbg_str,
            "rdma_resolve_addr : CMID %p, destroy called", chanp);
        mutex_exit(&chanp->chan_mutex);
        return (EINVAL);
    }
    chanp->chan_cmid_destroy_state |= SOL_CMA_CALLER_API_PROGRESS;

    if (chanp->chan_xport_type == SOL_CMA_XPORT_NONE) {
        bcopy((void *)src_addr, (void *)&(addrp->src_addr),
            sizeof (struct sockaddr));
    }
    bcopy((void *)dst_addr, (void *)&(addrp->dst_addr),
        sizeof (struct sockaddr));
    mutex_exit(&chanp->chan_mutex);

    /*
     * First resolve this as an @ corresponding to IB fabric
     * if this fails, resolve this as an @ corresponding to iWARP
     */
    if (chanp->chan_ib_client_hdl == NULL) {
        ofs_client_t    *ofs_clnt;

        ofs_clnt = (ofs_client_t *)sol_cma_ib_client->clnt_hdl;
        chanp->chan_ib_client_hdl = ofs_clnt->ibt_hdl;
    }
    if (chanp->chan_ib_client_hdl && rdma_ib_resolve_addr(idp, src_addr,
        dst_addr, timeout_ms) == 0) {
        SOL_OFS_DPRINTF_L4(sol_rdmacm_dbg_str,
            "rdma_resolve_addr: ret IB @");
#ifdef IWARP_SUPPORT
    } else if (chanp->chan_iw_client_hdl && rdma_iw_resolve_addr(idp,
        src_addr, dst_addr, timeout_ms) == 0) {
        SOL_OFS_DPRINTF_L2(sol_rdmacm_dbg_str,
            "rdma_resolve_addr: ret iWARP @");
#endif  /* IWARP_SUPPORT */
    } else {
        SOL_OFS_DPRINTF_L2(sol_rdmacm_dbg_str,
            "rdma_resolve_addr: Invalid @");
        return (EINVAL);
    }
    SOL_OFS_DPRINTF_L5(sol_rdmacm_dbg_str, "rdma_resolve_addr: ret 0");
    return (0);
}

static void cma_generate_event_sync(struct rdma_cm_id *,
    enum rdma_cm_event_type, int, struct rdma_conn_param *,
    struct rdma_ud_param *);

void
cma_resolve_addr_callback(sol_cma_chan_t *chanp, int rc)
{
    enum rdma_cm_event_type event;

    mutex_enter(&chanp->chan_mutex);
    if (chanp->chan_cmid_destroy_state &
        SOL_CMA_CALLER_CMID_DESTROYED) {
        SOL_OFS_DPRINTF_L3(sol_rdmacm_dbg_str,
            "cma_resolve_addr : CMID %p, destroy called", chanp);
        chanp->chan_cmid_destroy_state &=
            ~SOL_CMA_CALLER_API_PROGRESS;
        cv_broadcast(&chanp->chan_destroy_cv);
        mutex_exit(&chanp->chan_mutex);
        return;
    }
    if (rc == 0) {
        cma_set_chan_state(chanp, SOL_CMA_CHAN_ADDR_RESLVD);
        event = RDMA_CM_EVENT_ADDR_RESOLVED;
    } else
        event = RDMA_CM_EVENT_ADDR_ERROR;

    /*
     * Generate RDMA_CM_EVENT_ADDR_RESOLVED event
     * This will result in RDMA_USER_CM_CMD_RESOLVE_ROUTE in
     * userland.
     */
    chanp->chan_cmid_destroy_state |= SOL_CMA_CALLER_EVENT_PROGRESS;
    mutex_exit(&chanp->chan_mutex);
    cma_generate_event_sync((struct rdma_cm_id *)chanp, event, 0,
        NULL, NULL);

    mutex_enter(&chanp->chan_mutex);
    chanp->chan_cmid_destroy_state &= ~SOL_CMA_CALLER_API_PROGRESS;
    if (chanp->chan_cmid_destroy_state & SOL_CMA_CALLER_CMID_DESTROYED)
        cv_broadcast(&chanp->chan_destroy_cv);
    mutex_exit(&chanp->chan_mutex);
}

int
rdma_resolve_route(struct rdma_cm_id *idp, int timeout_ms)
{
    sol_cma_chan_t      *chanp;

    ASSERT(idp);
    chanp = (sol_cma_chan_t *)idp;
    SOL_OFS_DPRINTF_L5(sol_rdmacm_dbg_str, "resolve_route(%p, %x)", idp,
        timeout_ms);

    mutex_enter(&chanp->chan_mutex);
    if (cma_cas_chan_state(chanp, SOL_CMA_CHAN_ADDR_RESLVD,
        SOL_CMA_CHAN_ROUTE_RESLVD) != 0) {
        mutex_exit(&chanp->chan_mutex);
        SOL_OFS_DPRINTF_L2(sol_rdmacm_dbg_str,
            "resolve_route: Invalid state");
        return (EINVAL);
    }
    if (chanp->chan_cmid_destroy_state &
        SOL_CMA_CALLER_CMID_DESTROYED) {
        SOL_OFS_DPRINTF_L3(sol_rdmacm_dbg_str,
            "rdma_resolve_route : CMID %p, destroy called", chanp);
        mutex_exit(&chanp->chan_mutex);
        return (EINVAL);
    }
    chanp->chan_cmid_destroy_state |= SOL_CMA_CALLER_API_PROGRESS;
    mutex_exit(&chanp->chan_mutex);

    /*
     * Generate RDMA_CM_EVENT_ROUTE_RESOLVED event
     * This will result in RDMA_USER_CM_CMD_RESOLVE_ROUTE in
     * userland
     */
    cma_generate_event(idp, RDMA_CM_EVENT_ROUTE_RESOLVED, 0,
        NULL, NULL);

    mutex_enter(&chanp->chan_mutex);
    chanp->chan_cmid_destroy_state &= ~SOL_CMA_CALLER_API_PROGRESS;
    if (chanp->chan_cmid_destroy_state & SOL_CMA_CALLER_CMID_DESTROYED)
        cv_broadcast(&chanp->chan_destroy_cv);
    mutex_exit(&chanp->chan_mutex);

    SOL_OFS_DPRINTF_L5(sol_rdmacm_dbg_str, "resolve_route: ret 0");
    return (0);
}

/*
 * Connect or Listen request should be send after Route is resolved
 *
 *  Active Side (Client) :
 *  1. (State ROUTE_RESOLVED)-->CONNECT-->ACCEPT/REJECT-->DISCONNECT
 *         -->DESTROY_ID-->close(9E)
 *  2. Same as (1), DESTROY_ID without DISCONNECT
 *  3. Same as (1), close(9e) without DESTROY_ID.
 *
 *  Passive Side (Server) :
 *  4. (State ROUTE_RESOLVED)-->LISTEN->DISCONNECT
 *      -->DESTROY_ID-->close(9E)
 *  5. Same as (4), DESTROY_ID without DISCONNECT
 *  6. Same as (4), close(9e) without DESTROY_ID.
 */
int
rdma_connect(struct rdma_cm_id *idp, struct rdma_conn_param *conn_param)
{
    sol_cma_chan_t      *chanp;
    int         ret = EINVAL;

    ASSERT(idp);
    chanp = (sol_cma_chan_t *)idp;
    SOL_OFS_DPRINTF_L5(sol_rdmacm_dbg_str, "rdma_connect(%p, %p)", idp,
        conn_param);

    mutex_enter(&chanp->chan_mutex);
    if (chanp->chan_xport_type == SOL_CMA_XPORT_NONE) {
        mutex_exit(&chanp->chan_mutex);
        SOL_OFS_DPRINTF_L2(sol_rdmacm_dbg_str,
            "rdma_connect, Invalid Xport");
        return (EINVAL);
    }
    if (cma_cas_chan_state(chanp, SOL_CMA_CHAN_ROUTE_RESLVD,
        SOL_CMA_CHAN_CONNECT)) {
        mutex_exit(&chanp->chan_mutex);
        SOL_OFS_DPRINTF_L2(sol_rdmacm_dbg_str,
            "rdma_connect, Invalid state");
        return (EINVAL);
    }

    if (chanp->chan_xport_type == SOL_CMA_XPORT_IB) {
        ret = rdma_ib_connect(idp, conn_param);
#ifdef  IWARP_SUPPORT
    } else if (chanp->chan_xport_type == SOL_CMA_XPORT_IWARP) {
        ret = rdma_iw_connect(idp, conn_param);
#endif  /* IWARP_SUPPORT */
    }
    mutex_exit(&chanp->chan_mutex);

    SOL_OFS_DPRINTF_L5(sol_rdmacm_dbg_str, "rdma_connect: ret %x", ret);
    return (ret);
}

static int cma_init_listen_root(sol_cma_chan_t *);
static void cma_fini_listen_root(sol_cma_chan_t *);

int
rdma_listen(struct rdma_cm_id *idp, int bklog)
{
    sol_cma_chan_t      *chanp;
    int         ret = 0;
    genlist_entry_t     *entry;
    cma_chan_state_t    state;

    ASSERT(idp);
    chanp = (sol_cma_chan_t *)idp;
    SOL_OFS_DPRINTF_L5(sol_rdmacm_dbg_str, "rdma_listen(%p, %x)",
        idp, bklog);

    mutex_enter(&chanp->chan_mutex);
    state = cma_get_chan_state(chanp);
    if (state == SOL_CMA_CHAN_IDLE) {
        mutex_exit(&chanp->chan_mutex);
        return (EINVAL);
    }
    cma_set_chan_state(chanp, SOL_CMA_CHAN_LISTEN);

    if (chanp->chan_cmid_destroy_state &
        SOL_CMA_CALLER_CMID_DESTROYED) {
        SOL_OFS_DPRINTF_L3(sol_rdmacm_dbg_str,
            "rdma_listen : CMID %p, destroy called", chanp);
        mutex_exit(&chanp->chan_mutex);
        return (EINVAL);
    }
    chanp->chan_cmid_destroy_state |= SOL_CMA_CALLER_API_PROGRESS;

    ASSERT(chanp->chan_listenp == NULL);

    chanp->chan_listenp = kmem_zalloc(sizeof (sol_cma_listen_info_t),
        KM_SLEEP);
    init_genlist(&(CHAN_LISTEN_LIST(chanp)));
    (chanp->chan_listenp)->listen_is_root = 1;
    ret = cma_init_listen_root(chanp);
    if (ret) {
        chanp->chan_listenp = NULL;
        mutex_exit(&chanp->chan_mutex);
        SOL_OFS_DPRINTF_L2(sol_rdmacm_dbg_str, "rdma_listen: "
            "cma_init_listen_root: failed");
        kmem_free(chanp->chan_listenp,
            sizeof (sol_cma_listen_info_t));
        return (EINVAL);
    }

    if (chanp->chan_xport_type == SOL_CMA_XPORT_NONE) {
        ibcma_append_listen_list(idp);
#ifdef IWARP_SUPPORT
        iwcma_append_listen_list(idp);
#endif
    } else if (chanp->chan_xport_type == SOL_CMA_XPORT_IB) {
        ibcma_append_listen_list(idp);
#ifdef  IWARP_SUPPORT
    } else if (chanp->chan_xport_type == SOL_CMA_XPORT_IWARP) {
        iwcma_append_listen_list(idp);
#endif  /* IWARP_SUPPORT */
    }

    if (genlist_empty(&(CHAN_LISTEN_LIST(chanp)))) {
        cma_fini_listen_root(chanp);
        kmem_free((void *)chanp->chan_listenp,
            sizeof (sol_cma_listen_info_t));
        chanp->chan_listenp = NULL;
        SOL_OFS_DPRINTF_L2(sol_rdmacm_dbg_str, "rdma_listen: "
            "No listeners");
        mutex_exit(&chanp->chan_mutex);
        return (0);
    }

    if (chanp->chan_cmid_destroy_state & SOL_CMA_CALLER_CMID_DESTROYED) {
        chanp->chan_cmid_destroy_state &=
            ~SOL_CMA_CALLER_API_PROGRESS;
        cv_broadcast(&chanp->chan_destroy_cv);
    }

    genlist_for_each(entry, &(CHAN_LISTEN_LIST(chanp))) {
        struct rdma_cm_id   *ep_idp;
        sol_cma_chan_t      *ep_chanp;

        ep_idp = (struct rdma_cm_id *)entry->data;
        ep_chanp = (sol_cma_chan_t *)ep_idp;
        if (ep_chanp->chan_xport_type == SOL_CMA_XPORT_IB)
            ret = rdma_ib_listen(ep_idp, bklog);
#ifdef IWARP_SUPPORT
        if (ep_chanp->chan_xport_type == SOL_CMA_XPORT_IWARP)
            ret = rdma_iw_listen(ep_idp, bklog);
#endif
        if (ret)
            break;
    }

    chanp->chan_cmid_destroy_state &= ~SOL_CMA_CALLER_API_PROGRESS;
    if (chanp->chan_cmid_destroy_state & SOL_CMA_CALLER_CMID_DESTROYED)
        cv_broadcast(&chanp->chan_destroy_cv);
    mutex_exit(&chanp->chan_mutex);

    SOL_OFS_DPRINTF_L5(sol_rdmacm_dbg_str, "rdma_listen: ret %x", ret);
    return (ret);
}

int
rdma_accept(struct rdma_cm_id *idp, struct rdma_conn_param *conn_param)
{
    struct rdma_cm_id   *root_idp;
    sol_cma_chan_t      *root_chanp, *chanp;
    int         ret = EINVAL;

    ASSERT(idp);
    chanp = (sol_cma_chan_t *)idp;
    SOL_OFS_DPRINTF_L5(sol_rdmacm_dbg_str, "rdma_accept(%p, %p)",
        idp, conn_param);

    mutex_enter(&chanp->chan_mutex);
    if (cma_cas_chan_state(chanp, SOL_CMA_CHAN_LISTEN,
        SOL_CMA_CHAN_ACCEPT) && cma_cas_chan_state(chanp,
        SOL_CMA_CHAN_CONNECT, SOL_CMA_CHAN_ACCEPT)) {
        SOL_OFS_DPRINTF_L2(sol_rdmacm_dbg_str,
            "rdma_accept, Invalid state");
        mutex_exit(&chanp->chan_mutex);
        return (EINVAL);
    }
    mutex_exit(&chanp->chan_mutex);

    root_idp = CHAN_LISTEN_ROOT(chanp);
    root_chanp = (sol_cma_chan_t *)root_idp;
    SOL_OFS_DPRINTF_L5(sol_rdmacm_dbg_str, "accept: root_idp %p",
        root_idp);

    /* For TCP, delete from REQ AVL & insert to ACPT AVL */
    if (root_idp && root_idp->ps == RDMA_PS_TCP) {
        void        *find_ret;
        avl_index_t where;

        SOL_OFS_DPRINTF_L5(sol_rdmacm_dbg_str, "accept: root_idp %p"
            "REQ AVL remove %p", root_chanp, idp);
        mutex_enter(&root_chanp->chan_mutex);
        mutex_enter(&chanp->chan_mutex);

        /*
         * This CMID has been deleted, maybe because of timeout.
         * Return EINVAL.
         */
        if (chanp->chan_req_state != REQ_CMID_NOTIFIED) {
            mutex_exit(&chanp->chan_mutex);
            mutex_exit(&root_chanp->chan_mutex);
            SOL_OFS_DPRINTF_L3(sol_rdmacm_dbg_str,
                "accept: root_idp %p chanp %p, not in REQ "
                "AVL tree",  root_chanp, chanp);
            return (EINVAL);
        }
        ASSERT(cma_get_req_idp(root_idp, chanp->chan_session_id));
        avl_remove(&root_chanp->chan_req_avl_tree, idp);


        SOL_OFS_DPRINTF_L5(sol_rdmacm_dbg_str,
            "Add to ACPT AVL of %p IDP, idp %p, qp_hdl %p",
            root_idp, idp, chanp->chan_qp_hdl);
        find_ret = avl_find(&root_chanp->chan_acpt_avl_tree,
            (void *)chanp->chan_qp_hdl, &where);
        if (find_ret) {
            chanp->chan_req_state = REQ_CMID_SERVER_NONE;
            mutex_exit(&chanp->chan_mutex);
            mutex_exit(&root_chanp->chan_mutex);
            SOL_OFS_DPRINTF_L2(sol_rdmacm_dbg_str,
                "DUPLICATE ENTRY in ACPT AVL : root %p, "
                "idp %p, qp_hdl %p",
                root_idp, idp, chanp->chan_qp_hdl);
            return (EINVAL);
        }
        avl_insert(&root_chanp->chan_acpt_avl_tree,
            (void *)idp, where);
        chanp->chan_req_state = REQ_CMID_ACCEPTED;
        mutex_exit(&chanp->chan_mutex);
        mutex_exit(&root_chanp->chan_mutex);
    }

    if (root_idp && IS_UDP_CMID(root_idp)) {
        cma_chan_state_t    chan_state;

        /*
         * Accepting the connect request, no more events for this
         * connection.
         */
        cma_handle_nomore_events(chanp);
        mutex_enter(&chanp->chan_mutex);
        chan_state = cma_get_chan_state(chanp);
        mutex_exit(&chanp->chan_mutex);
        /* If rdma_destroy_id() was called, destroy CMID */
        if (chan_state == SOL_CMA_CHAN_DESTROY_PENDING) {
            cma_destroy_id((struct rdma_cm_id *)chanp);
            return (EINVAL);
        }
    }

    if (chanp->chan_xport_type == SOL_CMA_XPORT_IB)
        ret = rdma_ib_accept(idp, conn_param);
#ifdef  IWARP_SUPPORT
    if (chanp->chan_xport_type == SOL_CMA_XPORT_IWARP)
        ret = rdma_iw_accept(idp, conn_param);
#endif  /* IWARP_SUPPORT */

    if (ret && root_idp && idp->ps == RDMA_PS_TCP) {
        void        *find_ret;
        avl_index_t where;

        SOL_OFS_DPRINTF_L5(sol_rdmacm_dbg_str,
            "Delete from REQ AVL of %p IDP, idp %p",
            root_idp, idp);
        mutex_enter(&root_chanp->chan_mutex);
        mutex_enter(&chanp->chan_mutex);
        if (chanp->chan_req_state == REQ_CMID_ACCEPTED) {
            ASSERT(cma_get_acpt_idp(root_idp,
                chanp->chan_qp_hdl));
            avl_remove(&root_chanp->chan_acpt_avl_tree,
                idp);
            find_ret = avl_find(&root_chanp->chan_req_avl_tree,
                (void *)chanp->chan_qp_hdl, &where);
            if (find_ret) {
                chanp->chan_req_state = REQ_CMID_SERVER_NONE;
                mutex_exit(&chanp->chan_mutex);
                mutex_exit(&root_chanp->chan_mutex);
                SOL_OFS_DPRINTF_L2(sol_rdmacm_dbg_str,
                    "DUPLICATE ENTRY in REQ AVL : root %p, "
                    "idp %p, session_id %p",
                    root_idp, idp, chanp->chan_session_id);
                return (EINVAL);
            }
            avl_insert(&root_chanp->chan_req_avl_tree, idp, where);
            chanp->chan_req_state = REQ_CMID_NOTIFIED;
        }
        mutex_exit(&chanp->chan_mutex);
        mutex_exit(&root_chanp->chan_mutex);
    }

    SOL_OFS_DPRINTF_L5(sol_rdmacm_dbg_str, "rdma_accept: ret %x", ret);
    return (ret);
}

int
rdma_notify(struct rdma_cm_id *idp, enum ib_event_type evt)
{
    sol_cma_chan_t      *chanp;

    ASSERT(idp);
    chanp = (sol_cma_chan_t *)idp;
    SOL_OFS_DPRINTF_L5(sol_rdmacm_dbg_str, "rdma_notify(%p, %x)", idp, evt);

    mutex_enter(&chanp->chan_mutex);
    if (cma_cas_chan_state(chanp, SOL_CMA_CHAN_ROUTE_RESLVD,
        SOL_CMA_CHAN_EVENT_NOTIFIED)) {
        SOL_OFS_DPRINTF_L2(sol_rdmacm_dbg_str,
            "rdma_notify, Invalid state");
        mutex_exit(&chanp->chan_mutex);
        return (EINVAL);
    }
    mutex_exit(&chanp->chan_mutex);

    SOL_OFS_DPRINTF_L5(sol_rdmacm_dbg_str, "rdma_notify: ret 0");
    return (0);
}

int
rdma_reject(struct rdma_cm_id *idp, const void *priv_data,
    uint8_t priv_data_len)
{
    struct rdma_cm_id   *root_idp;
    sol_cma_chan_t      *root_chanp, *chanp;
    int         ret = EINVAL;

    ASSERT(idp);
    chanp = (sol_cma_chan_t *)idp;
    root_idp = CHAN_LISTEN_ROOT(chanp);
    root_chanp = (sol_cma_chan_t *)root_idp;
    SOL_OFS_DPRINTF_L5(sol_rdmacm_dbg_str, "rdma_reject(%p, %p)", idp,
        priv_data, priv_data_len);

    mutex_enter(&chanp->chan_mutex);
    if (cma_cas_chan_state(chanp, SOL_CMA_CHAN_LISTEN,
        SOL_CMA_CHAN_REJECT)) {
        SOL_OFS_DPRINTF_L2(sol_rdmacm_dbg_str,
            "rdma_accept, Invalid state");
        mutex_exit(&chanp->chan_mutex);
        return (EINVAL);
    }
    mutex_exit(&chanp->chan_mutex);

    if (root_idp) {
        SOL_OFS_DPRINTF_L5(sol_rdmacm_dbg_str, "reject: root_idp %p"
            "REQ AVL remove %p", root_chanp, idp);

        /*
         * Remove from REQ AVL tree. If this CMID has been deleted,
         * it maybe because of timeout. Return EINVAL.
         */
        mutex_enter(&root_chanp->chan_mutex);
        mutex_enter(&chanp->chan_mutex);
        if (chanp->chan_req_state != REQ_CMID_NOTIFIED &&
            chanp->chan_req_state != REQ_CMID_QUEUED) {
            mutex_exit(&chanp->chan_mutex);
            mutex_exit(&root_chanp->chan_mutex);
            SOL_OFS_DPRINTF_L3(sol_rdmacm_dbg_str,
                "reject: root_idp %p chanp %p, not in REQ "
                "AVL tree",  root_chanp, chanp);
            return (EINVAL);
        }
        ASSERT(cma_get_req_idp(root_idp, chanp->chan_session_id));
        avl_remove(&root_chanp->chan_req_avl_tree, idp);
        chanp->chan_req_state = REQ_CMID_SERVER_NONE;
        mutex_exit(&chanp->chan_mutex);
        mutex_exit(&root_chanp->chan_mutex);
    }

    if (chanp->chan_xport_type == SOL_CMA_XPORT_IB)
        ret = rdma_ib_reject(idp, priv_data, priv_data_len);
#ifdef  IWARP_SUPPORT
    if (chanp->chan_xport_type == SOL_CMA_XPORT_IWARP)
        ret = rdma_iw_reject(idp, priv_data, priv_data_len);
#endif  /* IWARP_SUPPORT */


    if (!ret && root_idp) {
        cma_chan_state_t    chan_state;

        /*
         * Rejecting connect request, no more events for this
         * connection.
         */
        cma_handle_nomore_events(chanp);
        mutex_enter(&chanp->chan_mutex);
        chan_state = cma_get_chan_state(chanp);
        mutex_exit(&chanp->chan_mutex);
        /* If rdma_destroy_id() was called, destroy CMID */
        if (chan_state == SOL_CMA_CHAN_DESTROY_PENDING)
            cma_destroy_id((struct rdma_cm_id *)chanp);
    } else if (ret && root_idp) {
        avl_index_t where;

        SOL_OFS_DPRINTF_L5(sol_rdmacm_dbg_str,
            "reject fail: Add to Req AVL of %p IDP, idp %p,"
            "session_id %p", root_idp, idp,
            chanp->chan_session_id);
        mutex_enter(&root_chanp->chan_mutex);
        mutex_enter(&chanp->chan_mutex);
        if (chanp->chan_req_state == REQ_CMID_SERVER_NONE) {
            if (avl_find(&root_chanp->chan_req_avl_tree,
                (void *)chanp->chan_session_id, &where)) {
                mutex_exit(&chanp->chan_mutex);
                mutex_exit(&root_chanp->chan_mutex);
                SOL_OFS_DPRINTF_L2(sol_rdmacm_dbg_str,
                    "DUPLICATE ENTRY in REQ AVL : root %p, "
                    "idp %p, session_id %p",
                    root_idp, idp, chanp->chan_session_id);
                return (EINVAL);
            }
            avl_insert(&root_chanp->chan_req_avl_tree,
                (void *)idp, where);
            chanp->chan_req_state = REQ_CMID_NOTIFIED;
        }
        mutex_exit(&chanp->chan_mutex);
        mutex_exit(&root_chanp->chan_mutex);
    }

    SOL_OFS_DPRINTF_L5(sol_rdmacm_dbg_str, "rdma_reject: ret %x", ret);
    return (ret);
}

int
rdma_disconnect(struct rdma_cm_id *idp)
{
    sol_cma_chan_t      *chanp;
    int         ret = EINVAL;
    cma_chan_state_t    state;

    chanp = (sol_cma_chan_t *)idp;
    SOL_OFS_DPRINTF_L5(sol_rdmacm_dbg_str, "rdma_disconnect(%p)", idp);

    if (!idp)
        return (0);

    mutex_enter(&chanp->chan_mutex);
    if (!(SOL_CMAID_CONNECTED(chanp))) {
        SOL_OFS_DPRINTF_L3(sol_rdmacm_dbg_str,
            "rdma_disconnect(%p) - Not connected!!", idp);
        mutex_exit(&chanp->chan_mutex);
        return (EINVAL);
    }
    state = cma_get_chan_state(chanp);
    cma_set_chan_state(chanp, SOL_CMA_CHAN_DISCONNECT);
    mutex_exit(&chanp->chan_mutex);

    if (chanp->chan_xport_type == SOL_CMA_XPORT_IB) {
        ret = rdma_ib_disconnect(idp);
#ifdef  IWARP_SUPPORT
    } else if (chanp->chan_xport_type == SOL_CMA_XPORT_IWARP) {
        ret = rdma_iw_disconnect(idp);
#endif  /* IWARP_SUPPORT */
    }

    if (ret) {
        mutex_enter(&chanp->chan_mutex);
        cma_set_chan_state(chanp, state);
        mutex_exit(&chanp->chan_mutex);
        return (ret);
    }

    SOL_OFS_DPRINTF_L5(sol_rdmacm_dbg_str, "rdma_disconnect: ret %x", ret);
    return (ret);
}

int
rdma_init_qp_attr(struct rdma_cm_id *idp, struct ib_qp_attr *qpattr,
    int *qp_attr_mask)
{
    sol_cma_chan_t      *chanp;
    int         ret = EINVAL;

    ASSERT(idp);
    chanp = (sol_cma_chan_t *)idp;
    SOL_OFS_DPRINTF_L5(sol_rdmacm_dbg_str, "rdma_init_qp_attr(%p, %p, %p)",
        idp, qpattr, qp_attr_mask);

    if (chanp->chan_xport_type == SOL_CMA_XPORT_IB) {
        ret = rdma_ib_init_qp_attr(idp, qpattr, qp_attr_mask);
#ifdef  IWARP_SUPPORT
    } else if (chanp->chan_xport_type == SOL_CMA_XPORT_IWARP)
        ret = rdma_iw_init_qp_attr(idp, qpattr, qp_attr_mask);
#endif  /* IWARP_SUPPORT */
    } else {
        ret = EINVAL;
    }

    SOL_OFS_DPRINTF_L5(sol_rdmacm_dbg_str,
        "rdma_init_qp_attr: ret %x", ret);

    return (ret);
}

int
rdma_join_multicast(struct rdma_cm_id *idp, struct sockaddr *addr,
    void *context)
{
    sol_cma_chan_t      *chanp;
    int         ret = ENODEV;
    cma_chan_state_t    state;

    ASSERT(idp);
    chanp = (sol_cma_chan_t *)idp;
    SOL_OFS_DPRINTF_L5(sol_rdmacm_dbg_str,
        "rdma_join_multicast(%p, %p, %p)",
        idp, addr, context);

    mutex_enter(&chanp->chan_mutex);
    state = cma_get_chan_state(chanp);
    if (state != SOL_CMA_CHAN_BOUND &&
        state != SOL_CMA_CHAN_ROUTE_RESLVD &&
        state != SOL_CMA_CHAN_ADDR_RESLVD) {
        SOL_OFS_DPRINTF_L2(sol_rdmacm_dbg_str,
            "rdma_join_multicast, Invalid state");
        mutex_exit(&chanp->chan_mutex);
        return (EINVAL);
    }

    if (chanp->chan_xport_type == SOL_CMA_XPORT_IB)
        ret = rdma_ib_join_multicast(idp, addr, context);
#ifdef  IWARP_SUPPORT
    /* No support for Multicast on iWARP */
    else if (chanp->chan_xport_type == SOL_CMA_XPORT_IWARP)
        ret = ENOTSUP;
#endif  /* IWARP_SUPPORT */
    mutex_exit(&chanp->chan_mutex);

    SOL_OFS_DPRINTF_L5(sol_rdmacm_dbg_str,
        "rdma_join_multicast: ret %x", ret);
    return (ret);
}

void
rdma_leave_multicast(struct rdma_cm_id *idp, struct sockaddr *addr)
{
    sol_cma_chan_t      *chanp;
    cma_chan_state_t    state;

    ASSERT(idp);
    chanp = (sol_cma_chan_t *)idp;
    SOL_OFS_DPRINTF_L5(sol_rdmacm_dbg_str, "rdma_leave_multicast(%p, %p)",
        idp, addr);

    mutex_enter(&chanp->chan_mutex);
    state = cma_get_chan_state(chanp);
    if (state != SOL_CMA_CHAN_BOUND &&
        state != SOL_CMA_CHAN_ROUTE_RESLVD &&
        state != SOL_CMA_CHAN_ADDR_RESLVD) {
        SOL_OFS_DPRINTF_L2(sol_rdmacm_dbg_str,
            "rdma_leave_multicast, Invalid state");
        mutex_exit(&chanp->chan_mutex);
        return;
    }

    if (chanp->chan_xport_type == SOL_CMA_XPORT_IB)
        rdma_ib_leave_multicast(idp, addr);
#ifdef  IWARP_SUPPORT
    /* No support for Multicast on iWARP */
    else if (chanp->chan_xport_type == SOL_CMA_XPORT_IWARP)
        SOL_OFS_DPRINTF_L2(sol_rdmacm_dbg_str,
            "rdma_leave_multicast, iWARP");
#endif  /* IWARP_SUPPORT */
    mutex_exit(&chanp->chan_mutex);

    SOL_OFS_DPRINTF_L5(sol_rdmacm_dbg_str, "rdma_join_multicast: ret");
}

/*
 * Functions to compare to rdma_cm_id *, used by AVL tree
 * routines.
 */
int
sol_cma_req_cmid_cmp(const void *p1, const void *p2)
{
    sol_cma_chan_t      *chanp;

    chanp = (sol_cma_chan_t *)p2;
    if (chanp->chan_session_id > p1)
        return (+1);
    else if (chanp->chan_session_id < p1)
        return (-1);
    else
        return (0);
}

int
sol_cma_cmid_cmp(const void *p1, const void *p2)
{
    sol_cma_chan_t      *chanp;

    chanp = (sol_cma_chan_t *)p2;
    if (chanp->chan_qp_hdl > p1)
        return (+1);
    else if (chanp->chan_qp_hdl < p1)
        return (-1);
    else
        return (0);
}

/*
 * Function to compare two sol_cma_glbl_listen_t *, used by
 * AVL tree routines.
 */
int
sol_cma_svc_cmp(const void *p1, const void *p2)
{
    sol_cma_glbl_listen_t   *listenp;
    uint64_t        sid;

    sid = *(uint64_t *)p1;
    listenp = (sol_cma_glbl_listen_t *)p2;
    if (listenp->cma_listen_chan_sid > sid)
        return (+1);
    else if (listenp->cma_listen_chan_sid < sid)
        return (-1);
    else
        return (0);
}

static int
cma_init_listen_root(sol_cma_chan_t *chanp)
{
    sol_cma_glbl_listen_t   *cma_listenp;
    sol_cma_listen_info_t   *chan_listenp;
    int         rc = 0;
    avl_index_t     where = 0;
    uint64_t        listen_sid;

    ASSERT(chanp);
    ASSERT(chanp->chan_listenp);
    chan_listenp = chanp->chan_listenp;

    SOL_OFS_DPRINTF_L5(sol_rdmacm_dbg_str,
        "cma_init_listen_root(%p)", chanp);

    /*
     * First search for matching global listen_info for this SID.
     * If found with the same client handle, reuse the service
     * handle, if matching SID is found with different client
     * handle, return EINVAL.
     */
    listen_sid = ibcma_init_root_sid(chanp);
    SOL_OFS_DPRINTF_L5(sol_rdmacm_dbg_str,
        "cma_init_listen_root: search SID 0x%llx",
        listen_sid);

    mutex_enter(&sol_cma_glob_mutex);
    cma_listenp = avl_find(&sol_cma_glbl_listen_tree,
        (void *) &listen_sid, &where);
    if (cma_listenp && cma_listenp->cma_listen_clnt_hdl ==
        chanp->chan_ib_client_hdl) {
        SOL_OFS_DPRINTF_L5(sol_rdmacm_dbg_str,
            "cma_init_listen_root: matching listenp %p SID 0x%llx",
            cma_listenp, listen_sid);
        chan_listenp->listen_entry = add_genlist(
            &cma_listenp->cma_listen_chan_list,
            (uintptr_t)chanp, NULL);
        chan_listenp->chan_glbl_listen_info = cma_listenp;
        ibcma_copy_srv_hdl(chanp, cma_listenp);
        mutex_exit(&sol_cma_glob_mutex);
        return (0);
    } else if (cma_listenp) {
        SOL_OFS_DPRINTF_L2(sol_rdmacm_dbg_str,
            "cma_init_listen_root: listenp %p, SID 0x%llx match, "
            "client hdl prev %p, new %p mismatch",
            cma_listenp, listen_sid,
            cma_listenp->cma_listen_clnt_hdl,
            chanp->chan_ib_client_hdl);
        mutex_exit(&sol_cma_glob_mutex);
        return (EINVAL);
    }

    cma_listenp = kmem_zalloc(sizeof (sol_cma_glbl_listen_t), KM_SLEEP);
    init_genlist(&cma_listenp->cma_listen_chan_list);
    chan_listenp->listen_entry = add_genlist(
        &cma_listenp->cma_listen_chan_list, (uintptr_t)chanp, NULL);
    chan_listenp->chan_glbl_listen_info = cma_listenp;
    cma_listenp->cma_listen_clnt_hdl = chanp->chan_ib_client_hdl;
    cma_listenp->cma_listen_chan_sid = listen_sid;

    rc = ibcma_init_root_chan(chanp, cma_listenp);
    if (rc) {
        mutex_exit(&sol_cma_glob_mutex);
        SOL_OFS_DPRINTF_L2(sol_rdmacm_dbg_str,
            "cma_init_listen_root: ibcma_init_root_chan failed!!");
        delete_genlist(&cma_listenp->cma_listen_chan_list,
            chan_listenp->listen_entry);
        kmem_free(cma_listenp, sizeof (sol_cma_glbl_listen_t));
        return (rc);
    }
    avl_insert(&sol_cma_glbl_listen_tree, cma_listenp, where);
    mutex_exit(&sol_cma_glob_mutex);
    return (0);
}

static void
cma_fini_listen_root(sol_cma_chan_t *chanp)
{
    sol_cma_glbl_listen_t   *cma_listenp;
    sol_cma_listen_info_t   *chan_listenp;

    ASSERT(chanp);
    ASSERT(chanp->chan_listenp);
    SOL_OFS_DPRINTF_L5(sol_rdmacm_dbg_str, "cma_fini_listen_root(%p)",
        chanp);
    chan_listenp = chanp->chan_listenp;
    cma_listenp = chan_listenp->chan_glbl_listen_info;
    ASSERT(cma_listenp);
    mutex_enter(&sol_cma_glob_mutex);
    delete_genlist(&cma_listenp->cma_listen_chan_list,
        chan_listenp->listen_entry);
    if (genlist_empty(&cma_listenp->cma_listen_chan_list)) {
        if (ibcma_fini_root_chan(chanp) == 0) {
            avl_remove(&sol_cma_glbl_listen_tree,
                cma_listenp);
            kmem_free(cma_listenp,
                sizeof (sol_cma_glbl_listen_t));
        } else
            SOL_OFS_DPRINTF_L2(sol_rdmacm_dbg_str,
                "cma_fini_listen_root: "
                "ibcma_fini_root_chan failed");
    }

    mutex_exit(&sol_cma_glob_mutex);
}

typedef struct cma_event_async_arg {
    struct rdma_cm_id   *idp;
    enum rdma_cm_event_type event;
    int         status;
    union {
        struct rdma_conn_param  conn;
        struct rdma_ud_param    param;
    } un;
    struct rdma_conn_param  *conn_param;
    struct rdma_ud_param    *ud_paramp;
} cma_event_async_arg_t;

static void cma_generate_event_sync(struct rdma_cm_id *,
    enum rdma_cm_event_type, int, struct rdma_conn_param *,
    struct rdma_ud_param *);

void
cma_generate_event_thr(void *arg)
{
    cma_event_async_arg_t   *event_arg = (cma_event_async_arg_t *)arg;

    cma_generate_event_sync(event_arg->idp, event_arg->event,
        event_arg->status, event_arg->conn_param,
        event_arg->ud_paramp);

    if (event_arg->conn_param && event_arg->conn_param->private_data_len)
        kmem_free((void *)event_arg->conn_param->private_data,
            event_arg->conn_param->private_data_len);
    if (event_arg->ud_paramp && event_arg->ud_paramp->private_data_len)
        kmem_free((void *)event_arg->ud_paramp->private_data,
            event_arg->ud_paramp->private_data_len);
    kmem_free(arg, sizeof (cma_event_async_arg_t));
}

void
cma_generate_event(struct rdma_cm_id *idp, enum rdma_cm_event_type event,
    int status, struct rdma_conn_param *conn_param,
    struct rdma_ud_param *ud_paramp)
{
    cma_event_async_arg_t   *event_arg;
    sol_cma_chan_t      *chanp = (sol_cma_chan_t *)idp;

    /*
     * Set SOL_CMA_CALLER_EVENT_PROGRESS to indicate event
     * notification is in progress, so that races between
     * rdma_destroy_id() and event notification is taken care.
     *
     * If rdma_destroy_id() has been called for this CMID, call
     * cma_generate_event_sync() which skips notification to the
     * consumer and handles the event.
     */
    mutex_enter(&chanp->chan_mutex);
    chanp->chan_cmid_destroy_state |= SOL_CMA_CALLER_EVENT_PROGRESS;
    if (chanp->chan_cmid_destroy_state & SOL_CMA_CALLER_CMID_DESTROYED) {
        mutex_exit(&chanp->chan_mutex);
        cma_generate_event_sync(idp, event, status, conn_param,
            ud_paramp);
        return;
    }
    mutex_exit(&chanp->chan_mutex);

    event_arg = kmem_zalloc(sizeof (cma_event_async_arg_t), KM_SLEEP);
    event_arg->idp = idp;
    event_arg->event = event;
    event_arg->status = status;
    event_arg->conn_param = NULL;
    event_arg->ud_paramp = NULL;
    if (conn_param && conn_param->private_data_len) {
        bcopy(conn_param, &(event_arg->un.conn),
            sizeof (struct rdma_conn_param));
        event_arg->conn_param = &(event_arg->un.conn);
        event_arg->conn_param->private_data = kmem_zalloc(
            conn_param->private_data_len, KM_SLEEP);
        bcopy(conn_param->private_data,
            (void *)event_arg->conn_param->private_data,
            conn_param->private_data_len);
    } else if (conn_param && conn_param->private_data_len == 0) {
        bcopy(conn_param, &(event_arg->un.conn),
            sizeof (struct rdma_conn_param));
    } else if (ud_paramp) {
        bcopy(ud_paramp, &(event_arg->un.param),
            sizeof (struct rdma_ud_param));
        event_arg->ud_paramp = &(event_arg->un.param);
        if (ud_paramp->private_data_len) {
            event_arg->ud_paramp->private_data = kmem_zalloc(
                ud_paramp->private_data_len, KM_SLEEP);
            bcopy(ud_paramp->private_data,
                (void *)event_arg->ud_paramp->private_data,
                ud_paramp->private_data_len);
        } else if (ud_paramp->private_data) {
            event_arg->ud_paramp->private_data =
                ud_paramp->private_data;
        }
    }

    if (taskq_dispatch(system_taskq, cma_generate_event_thr,
        (void *)event_arg, TQ_SLEEP) == 0) {
        SOL_OFS_DPRINTF_L2(sol_rdmacm_dbg_str,
            "generate_event_async: taskq_dispatch() failed!!");
        mutex_enter(&chanp->chan_mutex);
        chanp->chan_cmid_destroy_state &=
            ~SOL_CMA_CALLER_EVENT_PROGRESS;
        if (chanp->chan_cmid_destroy_state &
            SOL_CMA_CALLER_CMID_DESTROYED)
            cv_broadcast(&chanp->chan_destroy_cv);
        mutex_exit(&chanp->chan_mutex);
    }
}

static void
cma_generate_event_sync(struct rdma_cm_id *idp, enum rdma_cm_event_type event,
    int status, struct rdma_conn_param *conn_param,
    struct rdma_ud_param *ud_paramp)
{
    struct rdma_cm_event    cm_event;
    sol_cma_chan_t      *chanp = (sol_cma_chan_t *)idp;
    struct rdma_cm_id   *root_idp = NULL;
    sol_cma_chan_t      *root_chanp;
    int         ret;
    cma_chan_state_t    chan_state;

    SOL_OFS_DPRINTF_L5(sol_rdmacm_dbg_str, "generate_event_sync(%p, %x, "
        "%x, %p, %p", idp, event, status, conn_param, ud_paramp);

    bzero(&cm_event, sizeof (cm_event));
    cm_event.event = event;
    cm_event.status = status;
    if (conn_param)
        bcopy((void *)conn_param, (void *)(&(cm_event.param.conn)),
            sizeof (struct rdma_conn_param));
    else if (ud_paramp)
        bcopy((void *)ud_paramp, (void *)(&(cm_event.param.ud)),
            sizeof (struct rdma_ud_param));

    /*
     * If the consumer has destroyed the context for this CMID -
     * do not notify, skip to handling the sol_ofs specific
     * handling of the event.
     */
    mutex_enter(&chanp->chan_mutex);
    if (chanp->chan_cmid_destroy_state & SOL_CMA_CALLER_CMID_DESTROYED) {
        mutex_exit(&chanp->chan_mutex);
        goto ofs_consume_event;
    }
    mutex_exit(&chanp->chan_mutex);

    root_idp = CHAN_LISTEN_ROOT(chanp);
    root_chanp = (sol_cma_chan_t *)root_idp;
    SOL_OFS_DPRINTF_L5(sol_rdmacm_dbg_str, "gen_event: root_idp %p",
        root_idp);

    if (event == RDMA_CM_EVENT_CONNECT_REQUEST) {
        /*
         * Update chan_req_state for the REQ CMID. Decrement
         * count of REQ CMIDs not notifed to consumer.
         */
        ASSERT(root_idp);
        mutex_enter(&root_chanp->chan_mutex);
        root_chanp->chan_req_cnt--;
#ifdef  DEBUG
        SOL_OFS_DPRINTF_L5(sol_rdmacm_dbg_str,
            "Dec req_cnt of %p IDP, idp %p, req_cnt %x",
            root_idp, idp, root_chanp->chan_req_cnt);
#endif
        mutex_exit(&root_chanp->chan_mutex);
    }

    /* Pass the event to the client */
    ret = (idp->event_handler) (idp, &cm_event);

    if (ret) {
        /*
         * If the consumer returned failure :
         *  CONNECT_REQUEST :
         *  1. rdma_disconnect() to disconnect connection.
         *  2. wakeup destroy, if destroy has been called
         *      for this CMID
         *  3. Destroy CMID if rdma_destroy has not been
         *      called.
         *  DISCONNECTED :
         *  1. call cma_handle_nomore_events() to cleanup
         *  Other Events :
         *  1. Client is expected to destroy the CMID.
         */
        if (event == RDMA_CM_EVENT_CONNECT_REQUEST) {
            SOL_OFS_DPRINTF_L4(sol_rdmacm_dbg_str,
                "cma_generate_event_async: consumer failed %d "
                "event", event);
            if (rdma_disconnect(idp)) {
                SOL_OFS_DPRINTF_L2(sol_rdmacm_dbg_str,
                    "generate_event_async: rdma_disconnect "
                    "failed");
            }
            mutex_enter(&chanp->chan_mutex);
            ASSERT(SOL_IS_SERVER_CMID(chanp));
            chanp->chan_req_state = REQ_CMID_SERVER_NONE;
            chanp->chan_cmid_destroy_state &=
                ~SOL_CMA_CALLER_EVENT_PROGRESS;
            if (chanp->chan_cmid_destroy_state &
                SOL_CMA_CALLER_CMID_DESTROYED) {
                cv_broadcast(&chanp->chan_destroy_cv);
                mutex_exit(&chanp->chan_mutex);
            } else {
                mutex_exit(&chanp->chan_mutex);
                rdma_destroy_id(idp);
            }
        } else if (event == RDMA_CM_EVENT_DISCONNECTED) {
            SOL_OFS_DPRINTF_L2(sol_rdmacm_dbg_str,
                "generate_event_async: consumer failed %d event",
                event);
            cma_handle_nomore_events(chanp);
            mutex_enter(&chanp->chan_mutex);
            chan_state = cma_get_chan_state(chanp);
            chanp->chan_cmid_destroy_state &=
                ~SOL_CMA_CALLER_EVENT_PROGRESS;
            if (chanp->chan_cmid_destroy_state &
                SOL_CMA_CALLER_CMID_DESTROYED) {
                cv_broadcast(&chanp->chan_destroy_cv);
                mutex_exit(&chanp->chan_mutex);
            } else if (chan_state == SOL_CMA_CHAN_DESTROY_PENDING) {
                /* rdma_destroy_id() called: destroy CMID */
                mutex_exit(&chanp->chan_mutex);
                cma_destroy_id((struct rdma_cm_id *)chanp);
            } else
                mutex_exit(&chanp->chan_mutex);
        } else {
            SOL_OFS_DPRINTF_L2(sol_rdmacm_dbg_str,
                "generate_event_async: consumer failed %d event",
                event);
        }

        return;
    }
ofs_consume_event:
    if (event == RDMA_CM_EVENT_DISCONNECTED) {
        cma_chan_state_t    chan_state;

        cma_handle_nomore_events(chanp);
        mutex_enter(&chanp->chan_mutex);
        chan_state = cma_get_chan_state(chanp);
        chanp->chan_cmid_destroy_state &=
            ~SOL_CMA_CALLER_EVENT_PROGRESS;
        if (chanp->chan_cmid_destroy_state &
            SOL_CMA_CALLER_CMID_DESTROYED) {
            cv_broadcast(&chanp->chan_destroy_cv);
            mutex_exit(&chanp->chan_mutex);
        } else if (chan_state == SOL_CMA_CHAN_DESTROY_PENDING) {
            /* If rdma_destroy_id() was called, destroy CMID */
            mutex_exit(&chanp->chan_mutex);
            cma_destroy_id((struct rdma_cm_id *)chanp);
        } else
            mutex_exit(&chanp->chan_mutex);
        return;
    } else if (IS_UDP_CMID(idp) && event == RDMA_CM_EVENT_UNREACHABLE) {
        /*
         * If rdma_destroy_id() was called, destroy CMID
         * If not chan_connect_flag/ chan_req_state has already been
         * set to indicate that it can be deleted.
         */
        mutex_enter(&chanp->chan_mutex);
        chan_state = cma_get_chan_state(chanp);
        chanp->chan_cmid_destroy_state &=
            ~SOL_CMA_CALLER_EVENT_PROGRESS;
        if (chanp->chan_cmid_destroy_state &
            SOL_CMA_CALLER_CMID_DESTROYED) {
            cv_broadcast(&chanp->chan_destroy_cv);
            mutex_exit(&chanp->chan_mutex);
        } else if (chan_state == SOL_CMA_CHAN_DESTROY_PENDING) {
            mutex_exit(&chanp->chan_mutex);
            cma_destroy_id(idp);
        } else
            mutex_exit(&chanp->chan_mutex);
        return;
    }

    mutex_enter(&chanp->chan_mutex);
    chanp->chan_cmid_destroy_state &= ~SOL_CMA_CALLER_EVENT_PROGRESS;
    if (chanp->chan_cmid_destroy_state & SOL_CMA_CALLER_CMID_DESTROYED)
        cv_broadcast(&chanp->chan_destroy_cv);
    mutex_exit(&chanp->chan_mutex);
}

/* Local Static functions */
static struct rdma_cm_id *
cma_alloc_chan(rdma_cm_event_handler evt_hdlr, void *context,
    enum rdma_port_space ps)
{
    struct rdma_cm_id   *rdma_idp;
    sol_cma_chan_t      *chanp;

    chanp = kmem_zalloc(sizeof (sol_cma_chan_t), KM_SLEEP);
    mutex_init(&chanp->chan_mutex, NULL, MUTEX_DRIVER, NULL);
    cv_init(&chanp->chan_destroy_cv, NULL, CV_DRIVER, NULL);
    rdma_idp = &(chanp->chan_rdma_cm);
    rdma_idp->context = context;
    rdma_idp->ps = ps;
    rdma_idp->event_handler = evt_hdlr;
    mutex_enter(&chanp->chan_mutex);
    cma_set_chan_state(chanp, SOL_CMA_CHAN_IDLE);
    avl_create(&chanp->chan_req_avl_tree, sol_cma_req_cmid_cmp,
        sizeof (sol_cma_chan_t),
        offsetof(sol_cma_chan_t, chan_req_avl_node));
    avl_create(&chanp->chan_acpt_avl_tree, sol_cma_cmid_cmp,
        sizeof (sol_cma_chan_t),
        offsetof(sol_cma_chan_t, chan_acpt_avl_node));
    mutex_exit(&chanp->chan_mutex);

    return (rdma_idp);
}

/* Change the state of sol_cma_chan_t */
static void
cma_set_chan_state(sol_cma_chan_t *chanp, cma_chan_state_t newstate)
{
    ASSERT(MUTEX_HELD(&chanp->chan_mutex));
    chanp->chan_state = newstate;
}

cma_chan_state_t
cma_get_chan_state(sol_cma_chan_t *chanp)
{
    ASSERT(MUTEX_HELD(&chanp->chan_mutex));
    return (chanp->chan_state);
}

/* Check & Swap the state of sol_ucma_chan_t */
static int
cma_cas_chan_state(sol_cma_chan_t *chanp, cma_chan_state_t prevstate,
    cma_chan_state_t newstate)
{
    int ret = 0;

    ASSERT(MUTEX_HELD(&chanp->chan_mutex));
    if (chanp->chan_state != prevstate)
        ret = -1;
    else
        chanp->chan_state = newstate;

    return (ret);
}

static void
cma_free_listen_list(struct rdma_cm_id *idp)
{
    genlist_entry_t *entry;
    sol_cma_chan_t  *chanp = (sol_cma_chan_t *)idp;

    SOL_OFS_DPRINTF_L5(sol_rdmacm_dbg_str, "cma_free_listen_list(%p)", idp);
    mutex_enter(&chanp->chan_mutex);
    entry = remove_genlist_head(&(CHAN_LISTEN_LIST(chanp)));
    mutex_exit(&chanp->chan_mutex);
    while (entry) {
        sol_cma_chan_t  *ep_chanp;

        ep_chanp = (sol_cma_chan_t *)entry->data;
        SOL_OFS_DPRINTF_L5(sol_rdmacm_dbg_str, "fini_ep_chan: %p",
            ep_chanp);
        if (ibcma_fini_ep_chan(ep_chanp) == 0) {
            genlist_entry_t     *entry1;
            struct ib_device    *device;
            cma_device_t        *cma_device;

            ASSERT(ep_chanp->chan_listenp);
            mutex_enter(&ep_chanp->chan_mutex);
            entry1 = ep_chanp->chan_listenp->listen_ep_dev_entry;
            device = ep_chanp->chan_listenp->listen_ep_device;
            ASSERT(device);
            cma_device = device->data;
            delete_genlist(&cma_device->cma_epchan_list,
                entry1);
            sol_cma_release_device(
                (struct rdma_cm_id *)ep_chanp);
            mutex_exit(&ep_chanp->chan_mutex);
            if (ep_chanp->chan_listenp)
                kmem_free(ep_chanp->chan_listenp,
                    sizeof (sol_cma_listen_info_t));

            mutex_destroy(&ep_chanp->chan_mutex);
            cv_destroy(&ep_chanp->chan_destroy_cv);
            kmem_free(ep_chanp, sizeof (sol_cma_chan_t));
            kmem_free(entry, sizeof (genlist_entry_t));
        }

        mutex_enter(&chanp->chan_mutex);
        entry = remove_genlist_head(&(CHAN_LISTEN_LIST(chanp)));
        mutex_exit(&chanp->chan_mutex);
    }
}

/*
 * Destroy a listening CMID when :
 *  a. All CONNECTION REQUEST recieved have been rejected
 *     or closed.
 *  b. No CONNECTION REQUEST recieved.
 * Do not destroy a listening CMID when :
 *  a. CONNECTION REQUEST has been recieved and not been
 *     accepted from the passive / server side.
 *  b. CONNECTION REQUEST has been recieved and has been
 *     accepted from the passive server side.
 *  Mark the listening CMID as destroy pending.
 *
 * For CMIDs created for rdma_connect() or created for a
 * CONNECT request, destroy the CMID only when :
 *       CONNECTION has been closed or rejected.
 *
 *       Mark the CMID as destroy pending.
 *
 * When a connection is rejected or closed :
 *  Check if flag indicates - destroy pending,
 *  cma_destroy_id() is called, this also does
 *
 *  If there is a listening CMID assosiated with it,
 *     call cma_destroy_if(listen_cmid);
 */
void
cma_destroy_id(struct rdma_cm_id *idp)
{
    sol_cma_chan_t      *chanp = (sol_cma_chan_t *)idp;
    cma_chan_state_t    state;
    ulong_t         acpt_nodes, req_nodes;

    mutex_enter(&chanp->chan_mutex);
    acpt_nodes = avl_numnodes(&chanp->chan_acpt_avl_tree);
    req_nodes = avl_numnodes(&chanp->chan_req_avl_tree);
    state = cma_get_chan_state(chanp);
    SOL_OFS_DPRINTF_L5(sol_rdmacm_dbg_str, "cma_destroy_id(%p)- "
        "est CMIDs %ld, req CMID %ld, listen_root %p, state %x, %x",
        idp, acpt_nodes, req_nodes, chanp->listen_root,
        state, chanp->chan_req_state);

    /*
     * If there are either REQ recieved or Established CMIDs just return.
     * rdma_destroy() for these CMIDs can be called by client later.
     */
    if (acpt_nodes || req_nodes) {
        cma_set_chan_state(chanp, SOL_CMA_CHAN_DESTROY_PENDING);
        mutex_exit(&chanp->chan_mutex);
        return;
    }
    cma_set_chan_state(chanp, SOL_CMA_CHAN_DESTROYING);
    avl_destroy(&chanp->chan_req_avl_tree);
    avl_destroy(&chanp->chan_acpt_avl_tree);

    mutex_exit(&chanp->chan_mutex);
    if (idp->route.path_rec) {
        kmem_free(idp->route.path_rec,
            sizeof (struct ib_sa_path_rec) * idp->route.num_paths);
        idp->route.path_rec = NULL;
    }

    switch (chanp->chan_xport_type) {
    case SOL_CMA_XPORT_NONE :
        break;
    case SOL_CMA_XPORT_IB :
        rdma_ib_destroy_id(idp);
        break;
#ifdef  IWARP_SUPPORT
    case SOL_CMA_XPORT_IWARP :
        rdma_iw_destroy_id(idp);
        break;
#endif  /* IWARP_SUPPORT */
    default :
        SOL_OFS_DPRINTF_L2(sol_rdmacm_dbg_str,
            "cma_destroy_id: Unsupported xport type %x",
            chanp->chan_xport_type);
        break;
    }

    /*
     * Flush out & Free all listeners wrt to this ID
     * No locking is required as this code is executed
     * all REQ CMIDs have been destroyed. listen_list
     * will therefore not be modified during this loop.
     */
    if (chanp->chan_listenp) {
        cma_free_listen_list(idp);
        cma_fini_listen_root(chanp);
        kmem_free((void *)chanp->chan_listenp,
            sizeof (sol_cma_listen_info_t));
        chanp->chan_listenp = NULL;
    }

    if (chanp->listen_root) {
        struct rdma_cm_id   *root_idp;
        sol_cma_chan_t      *root_chanp;

        root_idp = chanp->listen_root;
        root_chanp = (sol_cma_chan_t *)root_idp;
        mutex_enter(&root_chanp->chan_mutex);
        state = cma_get_chan_state(root_chanp);
        acpt_nodes = avl_numnodes(&root_chanp->chan_acpt_avl_tree);
        req_nodes = avl_numnodes(&root_chanp->chan_req_avl_tree);
        mutex_exit(&root_chanp->chan_mutex);
        SOL_OFS_DPRINTF_L5(sol_rdmacm_dbg_str, "cma_destroy_id(%p)-"
            " root idp %p, state %x, acpt_nodes %ld, req_nodes %ld",
            idp, root_idp, state, acpt_nodes, req_nodes);

        if (state == SOL_CMA_CHAN_DESTROY_PENDING &&
            req_nodes == 0UL && acpt_nodes == 0UL) {
            mutex_enter(&root_chanp->chan_mutex);
            root_chanp->chan_req_state = REQ_CMID_SERVER_NONE;
            mutex_exit(&root_chanp->chan_mutex);
            cma_destroy_id(root_idp);
        } else if (state == SOL_CMA_CHAN_DESTROY_WAIT &&
            req_nodes == 0UL && acpt_nodes == 0UL) {
            mutex_enter(&root_chanp->chan_mutex);
            cma_set_chan_state(root_chanp,
                SOL_CMA_CHAN_DESTROY_PENDING);
            root_chanp->chan_req_state = REQ_CMID_SERVER_NONE;
            cv_broadcast(&root_chanp->chan_destroy_cv);
            mutex_exit(&root_chanp->chan_mutex);
        }
    }

    mutex_destroy(&chanp->chan_mutex);
    cv_destroy(&chanp->chan_destroy_cv);
    kmem_free(chanp, sizeof (sol_cma_chan_t));
}

/*
 * Server TCP disconnect for an established channel.
 *  If destroy_id() has been called for the listening
 *  CMID and there are no more CMIDs with pending
 *  events corresponding to the listening CMID, free
 *  the listening CMID.
 *
 */
static void
cma_handle_nomore_events(sol_cma_chan_t *chanp)
{
    struct rdma_cm_id   *idp, *root_idp;
    sol_cma_chan_t      *root_chanp;
    cma_chan_state_t    state;
    ulong_t         req_nodes, acpt_nodes;

    idp = (struct rdma_cm_id *)chanp;
    root_idp = CHAN_LISTEN_ROOT(chanp);
    root_chanp = (sol_cma_chan_t *)root_idp;
    if (!root_chanp)
        return;

    mutex_enter(&root_chanp->chan_mutex);
    mutex_enter(&chanp->chan_mutex);
    CHAN_LISTEN_ROOT(chanp) = NULL;
    root_chanp->chan_req_total_cnt--;

    /*
     * Removal of CMID from the AVL trees should already have been done
     * by now. Below code mainly as a  safety net.
     */
    if (chanp->chan_req_state == REQ_CMID_ACCEPTED) {
        ASSERT(chanp->chan_qp_hdl);
        ASSERT(cma_get_acpt_idp(root_idp,
            chanp->chan_qp_hdl));
        avl_remove(&root_chanp->chan_acpt_avl_tree, idp);
        chanp->chan_req_state = REQ_CMID_SERVER_NONE;
    }
    if (REQ_CMID_IN_REQ_AVL_TREE(chanp)) {
        ASSERT(chanp->chan_session_id);
        ASSERT(cma_get_req_idp(root_idp,
            chanp->chan_session_id));
        avl_remove(&root_chanp->chan_req_avl_tree, idp);
        chanp->chan_req_state = REQ_CMID_SERVER_NONE;
    }

    state = cma_get_chan_state(root_chanp);
    req_nodes = avl_numnodes(&root_chanp->chan_req_avl_tree);
    acpt_nodes = avl_numnodes(&root_chanp->chan_acpt_avl_tree);
    mutex_exit(&chanp->chan_mutex);
    mutex_exit(&root_chanp->chan_mutex);
    if (state == SOL_CMA_CHAN_DESTROY_PENDING && req_nodes == 0UL &&
        acpt_nodes == 0UL)
        cma_destroy_id(root_idp);
}

extern int ib_modify_qp(struct ib_qp *, struct ib_qp_attr *, int);
extern int rdma_init_qp_attr(struct rdma_cm_id *, struct ib_qp_attr *,
    int *);

static int
cma_init_ud_qp(sol_cma_chan_t *chanp, struct ib_qp *qp)
{
    struct ib_qp_attr qp_attr;
    int qp_attr_mask, ret;

    qp_attr.qp_state = IB_QPS_INIT;
    ret = rdma_init_qp_attr(&chanp->chan_rdma_cm, &qp_attr, &qp_attr_mask);
    if (ret)
        return (ret);

    ret = ib_modify_qp(qp, &qp_attr, qp_attr_mask);
    if (ret)
        return (ret);

    qp_attr.qp_state = IB_QPS_RTR;
    ret = ib_modify_qp(qp, &qp_attr, IB_QP_STATE);
    if (ret)
        return (ret);

    qp_attr.qp_state = IB_QPS_RTS;
    qp_attr.sq_psn = 0;
    ret = ib_modify_qp(qp, &qp_attr, IB_QP_STATE | IB_QP_SQ_PSN);

    return (ret);
}

static int
cma_init_conn_qp(sol_cma_chan_t *chanp, struct ib_qp *qp)
{
    struct ib_qp_attr qp_attr;
    int qp_attr_mask, ret;

    qp_attr.qp_state = IB_QPS_INIT;
    ret = rdma_init_qp_attr(&chanp->chan_rdma_cm, &qp_attr, &qp_attr_mask);
    if (ret)
        return (ret);

    return (ib_modify_qp(qp, &qp_attr, qp_attr_mask));
}

static inline int
cma_is_ud_ps(enum rdma_port_space ps)
{
    return (ps == RDMA_PS_UDP || ps == RDMA_PS_IPOIB);
}

int
rdma_create_qp(struct rdma_cm_id *idp, struct ib_pd *pd,
    struct ib_qp_init_attr *qp_init_attr)
{
    sol_cma_chan_t  *chanp;
    struct ib_qp    *qp;
    int     ret;
    ofs_client_t    *dev_ofs_client;

    ASSERT(idp);
    chanp = (sol_cma_chan_t *)idp;
    if (idp->device->node_guid != pd->device->node_guid)
        return (-EINVAL);

    dev_ofs_client = (ofs_client_t *)pd->device->clnt_hdl;
    rdma_map_id2clnthdl(idp, dev_ofs_client->ibt_hdl, NULL);

    qp = ib_create_qp(pd, qp_init_attr);
    if ((uintptr_t)qp >= (uintptr_t)-0xFFF) {
        return ((intptr_t)qp);
    }
    rdma_map_id2qphdl(idp, (void *)qp->ibt_qp);

    if (cma_is_ud_ps(idp->ps)) {
        ret = cma_init_ud_qp(chanp, qp);
    } else {
        ret = cma_init_conn_qp(chanp, qp);
    }

    if (ret) {
        goto err;
    }

    idp->qp = qp;
    chanp->chan_qp_num = qp->qp_num;
    chanp->chan_is_srq = (qp->srq != NULL);
    return (0);
err:
    (void) ib_destroy_qp(qp);
    return (ret);
}

void
rdma_destroy_qp(struct rdma_cm_id *idp)
{
    ASSERT(idp);
    (void) ib_destroy_qp(idp->qp);
    idp->qp = NULL;
}