of/sol_uverbs/sol_uverbs_comp.c

	sol_uverbs_comp.c revision c39526b769298791ff5b0b6c5e761f49aabaeb4e
/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved.
 */


/*
 * NAME: sol_uverbs_comp.c
 *
 * OFED User Verbs Kernel completion queue/processing implementation
 *
 */
#include <sys/file.h>
#include <sys/vfs.h>
#include <sys/vnode.h>
#include <sys/errno.h>
#include <sys/cred.h>
#include <sys/uio.h>
#include <sys/semaphore.h>
#include <sys/ddi.h>

#include <sys/ib/clients/of/ofa_solaris.h>
#include <sys/ib/clients/of/sol_ofs/sol_ofs_common.h>
#include <sys/ib/ibtl/ibvti.h>
#include <sys/ib/clients/of/ofed_kernel.h>
#include <sys/ib/clients/of/sol_uverbs/sol_uverbs.h>
#include <sys/ib/clients/of/sol_uverbs/sol_uverbs_comp.h>
#include <sys/ib/clients/of/sol_uverbs/sol_uverbs_event.h>

extern char *sol_uverbs_dbg_str;

/*
 * Function:
 *      uverbs_convert_wc
 * Input:
 *      uctxt   - Pointer to the callers user context.
 *  ibt_wc  - Pointer to IBT work completion.
 * Output:
 *      ofed_wc - Pointer to hold converted OFED work completion.
 * Returns:
 *      None
 * Description:
 *      Convert and IBT work completion to an OFED work completion.
 */
/* ARGSUSED */
static void
uverbs_convert_wc(uverbs_uctxt_uobj_t *uctxt, ibt_wc_t *ibt_wc,
    struct ib_uverbs_wc *ofed_wc)
{
    ASSERT(uctxt != NULL);
    ASSERT(ibt_wc != NULL);
    ASSERT(ofed_wc != NULL);

    ofed_wc->wr_id  = ibt_wc->wc_id;

    switch (ibt_wc->wc_status) {

        case IBT_WC_SUCCESS:
            ofed_wc->status = IB_WC_SUCCESS;
            break;
        case IBT_WC_LOCAL_LEN_ERR:
            ofed_wc->status = IB_WC_LOC_LEN_ERR;
            break;
        case IBT_WC_LOCAL_CHAN_OP_ERR:
            ofed_wc->status = IB_WC_LOC_QP_OP_ERR;
            break;
        case IBT_WC_LOCAL_PROTECT_ERR:
            ofed_wc->status = IB_WC_LOC_PROT_ERR;
            break;
        case IBT_WC_WR_FLUSHED_ERR:
            ofed_wc->status = IB_WC_WR_FLUSH_ERR;
            break;
        case IBT_WC_MEM_WIN_BIND_ERR:
            ofed_wc->status = IB_WC_MW_BIND_ERR;
            break;
        case IBT_WC_BAD_RESPONSE_ERR:
            ofed_wc->status = IB_WC_BAD_RESP_ERR;
            break;
        case IBT_WC_LOCAL_ACCESS_ERR:
            ofed_wc->status = IB_WC_LOC_ACCESS_ERR;
            break;
        case IBT_WC_REMOTE_INVALID_REQ_ERR:
            ofed_wc->status = IB_WC_REM_INV_REQ_ERR;
            break;
        case IBT_WC_REMOTE_ACCESS_ERR:
            ofed_wc->status = IB_WC_REM_ACCESS_ERR;
            break;
        case IBT_WC_REMOTE_OP_ERR:
            ofed_wc->status = IB_WC_REM_OP_ERR;
            break;
        case IBT_WC_TRANS_TIMEOUT_ERR:
        case IBT_WC_RNR_NAK_TIMEOUT_ERR:
            ofed_wc->status = IB_WC_RESP_TIMEOUT_ERR;
            break;
        default:
            ofed_wc->status = IB_WC_FATAL_ERR;
            break;
    }

    switch (ibt_wc->wc_type) {

        case IBT_WRC_SEND:
            ofed_wc->opcode = IB_WC_SEND;
            break;
        case IBT_WRC_RDMAR:
            ofed_wc->opcode = IB_WC_RDMA_READ;
            break;
        case IBT_WRC_RDMAW:
            ofed_wc->opcode = IB_WC_RDMA_WRITE;
            break;
        case IBT_WRC_CSWAP:
            ofed_wc->opcode = IB_WC_COMP_SWAP;
            break;
        case IBT_WRC_FADD:
            ofed_wc->opcode = IB_WC_FETCH_ADD;
            break;
        case IBT_WRC_BIND:
            ofed_wc->opcode = IB_WC_BIND_MW;
            break;
        case IBT_WRC_RECV:
            ofed_wc->opcode = IB_WC_RECV;
            break;
        case IBT_WRC_RECV_RDMAWI:
            ofed_wc->opcode = IB_WC_RECV_RDMA_WITH_IMM;
            break;

        case IBT_WRC_FAST_REG_PMR:
        case IBT_WRC_LOCAL_INVALIDATE:
        default:
            ofed_wc->opcode = -1; /* (?) What to do here */
            break;
    }

    ofed_wc->vendor_err     = 0;
    ofed_wc->byte_len   = ibt_wc->wc_bytes_xfer;
    ofed_wc->imm_data   = ibt_wc->wc_immed_data;
    ofed_wc->qp_num     = ibt_wc->wc_local_qpn;
    ofed_wc->src_qp     = ibt_wc->wc_qpn;
    ofed_wc->wc_flags   = 0;

    if (ibt_wc->wc_flags & IBT_WC_GRH_PRESENT) {
        ofed_wc->wc_flags |= IB_WC_GRH;
    }

    if (ibt_wc->wc_flags & IBT_WC_IMMED_DATA_PRESENT) {
        ofed_wc->wc_flags |= IB_WC_WITH_IMM;
    }

    ofed_wc->pkey_index = ibt_wc->wc_pkey_ix;
    ofed_wc->slid       = ibt_wc->wc_slid;
    ofed_wc->sl     = ibt_wc->wc_sl;
    ofed_wc->dlid_path_bits = ibt_wc->wc_path_bits;
    ofed_wc->port_num   = 0;
    ofed_wc->reserved   = 0;
}

/*
 * Function:
 *      sol_uverbs_create_cq
 * Input:
 *  uctxt   - Pointer to the callers user context.
 *  buf     - Pointer to kernel buffer containing create CQ command.
 *  in_len  - Length in bytes of input command buffer.
 *  out_len - Length in bytes of output response buffer.
 * Output:
 *  The command output buffer is updated with command results.
 * Returns:
 *      DDI_SUCCESS on success, else error code.
 * Description:
 *      User verbs entry point to create a device CQ.
 */
/* ARGSUSED */
int
sol_uverbs_create_cq(uverbs_uctxt_uobj_t *uctxt, char *buf,
    int in_len, int out_len)
{
    struct ib_uverbs_create_cq  cmd;
    struct ib_uverbs_create_cq_resp resp;
    uverbs_ucq_uobj_t       *ucq;
    ibt_cq_attr_t           cq_attr;
    uint_t              real_size;
    int             rc;

    (void) memcpy(&cmd, buf, sizeof (cmd));
    (void) memset(&resp, 0, sizeof (resp));
    (void) memset(&cq_attr, 0, sizeof (cq_attr));

    cq_attr.cq_size   = cmd.cqe;
    cq_attr.cq_sched  = 0;
    cq_attr.cq_flags  = IBT_CQ_USER_MAP;

    SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, "create_cq: "
        "num_cqe=%d, sched=%x, flags=%x",
        cq_attr.cq_size, cq_attr.cq_sched, cq_attr.cq_flags);

    /*
     * To be consistent with OFED semantics, we fail a CQ that is being
     * created with 0 CQ entries.
     */
    if (!cmd.cqe) {
        SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str, "create_cq: 0 cqe");
        rc = EINVAL;
        goto err_out;
    }

    ucq = kmem_zalloc(sizeof (*ucq), KM_NOSLEEP);
    if (!ucq) {
        SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
            "create_cq: mem alloc failure");
        rc = ENOMEM;
        goto err_out;
    }
    sol_ofs_uobj_init(&ucq->uobj, cmd.user_handle,
        SOL_UVERBS_UCQ_UOBJ_TYPE);
    rw_enter(&ucq->uobj.uo_lock, RW_WRITER);
    SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str,
        "create_cq: ucq %p, comp_chan %d", ucq, cmd.comp_channel);

    /*
     * If a event completion channel was specified look it up and
     * assign the channel to the user CQ object.  Note that this
     * places a reference on the file itself.
     */
    if ((int)cmd.comp_channel > SOL_UVERBS_DRIVER_MAX_MINOR) {
        uverbs_uctxt_uobj_t *compl_uctxt = NULL;
        uverbs_ufile_uobj_t *ufile;

        SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, "create_cq: "
            "cmd.comp_chan %d", cmd.comp_channel);
        compl_uctxt = uverbs_uobj_get_uctxt_write(
            cmd.comp_channel - SOL_UVERBS_DRIVER_MAX_MINOR);
        if (!compl_uctxt) {
            SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
                "create_cq: Invalid comp channel %d",
                cmd.comp_channel);
            rc = EINVAL;
            goto chan_err;
        }
        if (compl_uctxt->uctxt_verbs_id != uctxt->uobj.uo_id +
            SOL_UVERBS_DRIVER_MAX_MINOR) {
            SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
                "create_cq: Invalid comp channel %d, "
                "verbs id %d mismatch",
                cmd.comp_channel,
                compl_uctxt->uctxt_verbs_id);
            rc = EINVAL;
            sol_ofs_uobj_put(&compl_uctxt->uobj);
            goto chan_err;
        }
        ufile = compl_uctxt->comp_evfile;
        ucq->comp_chan = ufile;
        rw_enter(&ufile->uobj.uo_lock, RW_WRITER);
        ufile->ufile_cq_cnt++;
        rw_exit(&ufile->uobj.uo_lock);
        sol_ofs_uobj_put(&compl_uctxt->uobj);

        SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, "create_cq: "
            "ucq %p, comp_chan %p", ucq, ucq->comp_chan);
    } else {
        ucq->comp_chan = NULL;
    }

    llist_head_init(&ucq->async_list, NULL);
    llist_head_init(&ucq->comp_list, NULL);

    rc = ibt_alloc_cq(uctxt->hca->hdl, &cq_attr, &ucq->cq, &real_size);

    if (rc != IBT_SUCCESS) {
        SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
            "create_cq: ibt_alloc_cq() (rc=%d)", rc);
        rc = sol_uverbs_ibt_to_kernel_status(rc);
        ucq->uobj.uo_uobj_sz = sizeof (uverbs_ucq_uobj_t);
        goto alloc_err;
    }

    ibt_set_cq_private(ucq->cq, ucq);

    SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str,
        "create_cq: ib_alloc_cq() (rc=%d), real_size=%d",
        rc, real_size);
    /*
     * Query underlying hardware for data used in mapping CQ back to user
     * space, we will return this information in the user verbs command
     * response.
     */
    rc = ibt_ci_data_out(uctxt->hca->hdl, IBT_CI_NO_FLAGS, IBT_HDL_CQ,
        (void *) ucq->cq,  &resp.drv_out, sizeof (resp.drv_out));
    if (rc != IBT_SUCCESS) {
        SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
            "create_cq: ibt_ci_data_out() rc=%d", rc);
        rc = EFAULT;
        ucq->uobj.uo_uobj_sz = sizeof (uverbs_ucq_uobj_t);
        goto err_cq_destroy;
    }

    if (sol_ofs_uobj_add(&uverbs_ucq_uo_tbl, &ucq->uobj) != 0) {
        SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
            "create_cq: User object add failed");
        rc = ENOMEM;
        goto err_cq_destroy;
    }

    SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str,
        "create_cq: ibt_ci_data_out: 0x%16llx 0x%16llx "
        "0x%16llx 0x%16llx", resp.drv_out[0], resp.drv_out[1],
        resp.drv_out[2], resp.drv_out[3]);

    resp.cqe    = real_size;
    resp.cq_handle  = ucq->uobj.uo_id;

#ifdef  _LP64
    rc = copyout((void*)&resp, (void*)cmd.response.r_laddr, sizeof (resp));
#else
    rc = copyout((void*)&resp, (void*)cmd.response.r_addr, sizeof (resp));
#endif
    if (rc != 0) {
        SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
            "create_cq: copyout failed %x", rc);
        rc = EFAULT;
        goto err_uo_delete;
    }

    ucq->uctxt = uctxt;

    mutex_enter(&uctxt->lock);
    ucq->list_entry = add_genlist(&uctxt->cq_list, (uintptr_t)ucq, uctxt);
    mutex_exit(&uctxt->lock);

    if (!ucq->list_entry) {
        SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
            "create_cq: Error adding ucq to cq_list");
        rc = ENOMEM;
        goto err_uo_delete;
    }

    if (ucq->comp_chan) {
        ibt_set_cq_handler(ucq->cq, sol_uverbs_comp_event_handler, ucq);
    }

    ucq->uobj.uo_live = 1;
    rw_exit(&ucq->uobj.uo_lock);

    return (DDI_SUCCESS);

err_uo_delete:
    /*
     * Need to set uo_live, so sol_ofs_uobj_remove() will
     * remove the object from the object table.
     */
    ucq->uobj.uo_live = 1;
    (void) sol_ofs_uobj_remove(&uverbs_ucq_uo_tbl, &ucq->uobj);

err_cq_destroy:
    (void) ibt_free_cq(ucq->cq);

alloc_err:
    if (ucq->comp_chan) {
        uverbs_release_ucq_channel(uctxt, ucq->comp_chan, ucq);
    }

chan_err:
    rw_exit(&ucq->uobj.uo_lock);
    sol_ofs_uobj_deref(&ucq->uobj, sol_ofs_uobj_free);

err_out:
    return (rc);
}

int
uverbs_ucq_free(uverbs_ucq_uobj_t *ucq, uverbs_uctxt_uobj_t *uctxt)
{
    int rc;

    rc = ibt_free_cq(ucq->cq);
    if (rc != IBT_SUCCESS) {
        SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
            "destroy_id: ibt_free_cq() rc=%d",
            rc);
        rc = sol_uverbs_ibt_to_kernel_status(rc);
        sol_ofs_uobj_put(&ucq->uobj);
        return (rc);
    }
    (void) sol_ofs_uobj_remove(&uverbs_ucq_uo_tbl, &ucq->uobj);
    sol_ofs_uobj_put(&ucq->uobj);

    if (ucq->list_entry) {
        mutex_enter(&uctxt->lock);
        delete_genlist(&uctxt->cq_list,  ucq->list_entry);
        mutex_exit(&uctxt->lock);
    }
    sol_ofs_uobj_deref(&ucq->uobj, sol_ofs_uobj_free);
    return (0);
}

/*
 * Function:
 *      sol_uverbs_destroy_cq
 * Input:
 *  uctxt   - Pointer to the callers user context.
 *  buf     - Pointer to kernel buffer containing a destroy CQ command.
 *  in_len  - Length in bytes of input command buffer.
 *  out_len - Length in bytes of output response buffer.
 * Output:
 *  The command output buffer is updated with command results.
 * Returns:
 *      DDI_SUCCESS on success, else error code.
 * Description:
 *      User verbs entry point to destroy a device CQ.
 */
/* ARGSUSED */
int
sol_uverbs_destroy_cq(uverbs_uctxt_uobj_t *uctxt, char *buf,
    int in_len, int out_len)
{
    struct ib_uverbs_destroy_cq     cmd;
    struct ib_uverbs_destroy_cq_resp    resp;
    uverbs_ucq_uobj_t           *ucq;
    int                 rc;

    (void) memcpy(&cmd, buf, sizeof (cmd));
    (void) memset(&resp, 0, sizeof (resp));

    SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str,
        "destroy_cq(cq_handle=%d)", cmd.cq_handle);

    ucq = uverbs_uobj_get_ucq_write(cmd.cq_handle);
    if (ucq == NULL) {
        SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
            "destroy_cq: Invalid handle: %d",
            cmd.cq_handle);
        rc = EINVAL;
        goto err_out;
    }

    uverbs_release_ucq_channel(uctxt, ucq->comp_chan, ucq);
    cmd.cq_handle           = 0;
    resp.comp_events_reported   = ucq->comp_events_reported;
    resp.async_events_reported  = ucq->async_events_reported;

    if (ucq->active_qp_cnt) {
        sol_ofs_uobj_put(&ucq->uobj);
        return (EBUSY);
    } else {
        rc = uverbs_ucq_free(ucq, uctxt);
        if (rc)
            goto err_out;
    }

#ifdef  _LP64
    rc = copyout((void*)&resp, (void*)cmd.response.r_laddr, sizeof (resp));
#else
    rc = copyout((void*)&resp, (void*)cmd.response.r_addr, sizeof (resp));
#endif
    if (rc != 0) {
        SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
            "destroy_cq: copuout failed %x", rc);
        rc = EFAULT;
        goto err_out;
    }

    return (DDI_SUCCESS);

err_out:
    return (rc);
}

/*
 * Function:
 *      sol_uverbs_resize_cq
 * Input:
 *  uctxt   - Pointer to the callers user context.
 *  buf     - Pointer to kernel buffer containing a resize CQ command.
 *  in_len  - Length in bytes of input command buffer.
 *  out_len - Length in bytes of output response buffer.
 * Output:
 *  The command output buffer is updated with command results.
 * Returns:
 *      DDI_SUCCESS on success, else error code.
 * Description:
 *      User verbs entry point to resize a device CQ.
 */
/* ARGSUSED */
int
sol_uverbs_resize_cq(uverbs_uctxt_uobj_t *uctxt, char *buf,
    int in_len, int out_len)
{
    struct ib_uverbs_resize_cq  cmd;
    struct ib_uverbs_resize_cq_resp resp;
    uverbs_ucq_uobj_t       *ucq;
    int             rc;
    int             resize_status;

    (void) memcpy(&cmd, buf, sizeof (cmd));
    (void) memset(&resp, 0, sizeof (resp));

    SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str,
        "resize_cq(cq_handle=%d)", cmd.cq_handle);

    ucq = uverbs_uobj_get_ucq_write(cmd.cq_handle);
    if (ucq == NULL) {
        SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
            "resize_cq: Invalid handle: %d",
            cmd.cq_handle);
        rc = EINVAL;
        goto err_out;
    }

    /*
     * If CQ resize fails, note the error but keep going.  In this case we
     * expect the old CQ size to be returned, and when we extract the
     * mapping data, we expect the offset and length are approrpriate for
     * the old CQ.
     */
    resize_status = ibt_resize_cq(ucq->cq, cmd.cqe, &resp.cqe);
    if (resize_status != IBT_SUCCESS) {
        SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
            "resize_cq: ibt_resize_cq() (rc=%d), using "
            "original CQ", rc);
    }

    sol_ofs_uobj_put(&ucq->uobj);

    rc = ibt_ci_data_out(uctxt->hca->hdl, IBT_CI_NO_FLAGS, IBT_HDL_CQ,
        (void *) ucq->cq,  &resp.drv_out, sizeof (resp.drv_out));
    if (rc != IBT_SUCCESS) {
        SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
            "resize_cq: Error in ibt_ci_data_out() "
            "(rc=%d)", rc);
        rc = EFAULT;
        goto err_out;
    }

    SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str,
        "resize_cq: ibt_ci_data_out: 0x%16llx 0x%16llx "
        "0x%16llx 0x%16llx", resp.drv_out[0], resp.drv_out[1],
        resp.drv_out[2], resp.drv_out[3]);

#ifdef  _LP64
    rc = copyout((void*)&resp, (void*)cmd.response.r_laddr, sizeof (resp));
#else
    rc = copyout((void*)&resp, (void*)cmd.response.r_addr, sizeof (resp));
#endif
    if (rc != 0) {
        SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
            "resize_cq: copyout %d", rc);
        rc = EFAULT;
        goto err_out;
    }

    return (resize_status);

err_out:
    return (rc);
}

/*
 * Function:
 *      sol_uverbs_req_notify_cq
 * Input:
 *  uctxt   - Pointer to the callers user context.
 *  buf     - Pointer to kernel buffer containing request notify
 *            command.
 *  in_len  - Length in bytes of input command buffer.
 *  out_len - Length in bytes of output response buffer.
 * Output:
 *  The command output buffer is updated with command results.
 * Returns:
 *      DDI_SUCCESS on success, else error code.
 * Description:
 *      User verbs entry point to request notification of CQ events.
 */
/* ARGSUSED */
int
sol_uverbs_req_notify_cq(uverbs_uctxt_uobj_t *uctxt, char *buf,
    int in_len, int out_len)
{
    struct ib_uverbs_req_notify_cq  cmd;
    ibt_cq_notify_flags_t       flag;
    uverbs_ucq_uobj_t       *ucq;
    int             rc;

    (void) memcpy(&cmd, buf, sizeof (cmd));

    SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str,
        "req_notify_cq(cq_handle=%d)", cmd.cq_handle);

    ucq = uverbs_uobj_get_ucq_read(cmd.cq_handle);
    if (ucq == NULL) {
        SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
            "req_notify_cq: List lookup failure");
        rc = EINVAL;
        goto err_out;
    }

    flag = IBT_NEXT_COMPLETION;

    if (cmd.solicited_only != 0) {
        flag = IBT_NEXT_SOLICITED;
    }

    rc = ibt_enable_cq_notify(ucq->cq, flag);
    if (rc != IBT_SUCCESS) {
        SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
            "req_notify_cq: ibt_enable_cq_notify() "
            "(rc=%d)", rc);
        rc = sol_uverbs_ibt_to_kernel_status(rc);
        goto err_put;
    }

    sol_ofs_uobj_put(&ucq->uobj);
    return (DDI_SUCCESS);

err_put:
    sol_ofs_uobj_put(&ucq->uobj);

err_out:
    return (rc);
}

/*
 * Function:
 *      sol_uverbs_poll_cq
 * Input:
 *  uctxt   - Pointer to the callers user context.
 *  buf     - Pointer to kernel buffer containing poll CQ command.
 *  in_len  - Length in bytes of input command buffer.
 *  out_len - Length in bytes of output response buffer.
 * Output:
 *  The command output buffer is updated with command results.
 * Returns:
 *      DDI_SUCCESS on success, else error code.
 * Description:
 *      User verbs entry point to poll a CQ for completion events.  Note that
 *  this is a non OS Bypass version, the CQ normally would be polled
 *  directly from the user space driver.
 */
/* ARGSUSED */
int
sol_uverbs_poll_cq(uverbs_uctxt_uobj_t *uctxt, char *buf,
    int in_len, int out_len)
{
    struct ib_uverbs_poll_cq    cmd;
    struct ib_uverbs_poll_cq_resp   resp;
    uverbs_ucq_uobj_t       *ucq;
    ibt_wc_t            *completions;
    struct ib_uverbs_wc     ofed_wc;
    int             rc;
    int             i;

    (void) memcpy(&cmd, buf, sizeof (cmd));
    (void) memset(&resp, 0, sizeof (resp));

#ifdef DEBUG
    SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str,
        "poll_cq(cq_handle=%d)", cmd.cq_handle);
#endif

    ucq = uverbs_uobj_get_ucq_read(cmd.cq_handle);
    if (ucq == NULL) {
        SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
            "poll_cq: List lookup failure");
        rc = EINVAL;
        goto err_find;
    }

    completions = (ibt_wc_t *)kmem_zalloc(sizeof (ibt_wc_t) * cmd.ne,
        KM_NOSLEEP);
    if (completions == NULL) {
        SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
            "poll_cq: Allocation Error");
        rc = ENOMEM;
        goto err_alloc;
    }

    rc = ibt_poll_cq(ucq->cq, completions, cmd.ne, &resp.count);
    if (rc != IBT_SUCCESS) {
        SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
            "poll_cq: ibt_poll_cq() (rc=%d)", rc);
        rc = sol_uverbs_ibt_to_kernel_status(rc);
        goto err_poll;
    }

#ifdef  _LP64
    rc = copyout((void*)&resp, (void*)cmd.response.r_laddr, sizeof (resp));
#else
    rc = copyout((void*)&resp, (void*)cmd.response.r_addr, sizeof (resp));
#endif
    if (rc != 0) {
        SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
            "poll_cq: copyout %x", rc);
        rc = EFAULT;
        goto err_poll;
    }

    for (i = 0; i < resp.count; i++) {
        (void) memset(&ofed_wc, 0, sizeof (ofed_wc));
        uverbs_convert_wc(uctxt, &completions[i], &ofed_wc);

#ifdef  _LP64
        rc = copyout((void*)&ofed_wc,
            (void *)cmd.response.r_laddr, sizeof (ofed_wc));
#else
        rc = copyout((void*)&ofed_wc,
            (void *)cmd.response.r_addr, sizeof (ofed_wc));
#endif
        if (rc != 0) {
            SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
                "poll_cq: copyout wc data %x", rc);
            rc = EFAULT;
            goto err_poll;
        }
    }

    kmem_free((void*)completions, sizeof (ibt_wc_t) * cmd.ne);
    sol_ofs_uobj_put(&ucq->uobj);

    return (DDI_SUCCESS);

err_poll:
    kmem_free((void *)completions, sizeof (ibt_wc_t) * cmd.ne);

err_alloc:
    sol_ofs_uobj_put(&ucq->uobj);

err_find:
    return (rc);
}

/*
 * Function:
 *      sol_uverbs_comp_event_handler
 * Input:
 *      ibt_cq  - Handle to the IBT CQ.
 *      arg     - Address of the associated Solaris User Verbs CQ
 *            object.
 * Output:
 *      None
 * Returns:
 *      None
 * Description:
 *  Solaris User Verbs completion channel IBT CQ callback.  Queue
 *  the completion event and wakeup/notify consumers that may be
 *  blocked waiting for the completion.
 */
/* ARGSUSED */
void
sol_uverbs_comp_event_handler(ibt_cq_hdl_t ibt_cq, void *arg)
{
    uverbs_ucq_uobj_t   *ucq = arg;
    uverbs_ufile_uobj_t *ufile;
    uverbs_event_t      *entry;

#ifdef DEBUG
    SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, "comp_evt_hdlr(%p, %p) - ",
        "ucq = %p, ucq->cq = %p, ucq->uctxt = %p, ucq->comp_chan =%p",
        ibt_cq, arg, ucq, ucq->cq, ucq->uctxt, ucq->comp_chan);
#endif

    if (!ucq || !ucq->comp_chan) {
        SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str, "comp_evt_hdlr "
            "ucq %p ucq->comp_chan %p", ucq, (ucq) ? ucq->comp_chan :
            NULL);
        return;
    }

    ufile = ucq->comp_chan;

    mutex_enter(&ufile->lock);
    if (!ufile->uctxt) {
        mutex_exit(&ufile->lock);
        SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str, "comp_evt_hdlr "
            "ufile->uctxt %p", ufile->uctxt);
        return;
    }

    entry = kmem_zalloc(sizeof (*entry), KM_NOSLEEP);
    if (!entry) {
        mutex_exit(&ufile->lock);
        SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str, "comp_evt_hdlr() "
            "memory allocation error");
        return;
    }

    entry->ev_desc.comp.cq_handle   = ucq->uobj.uo_user_handle;
    entry->ev_counter       = &ucq->comp_events_reported;

    /*
     * Add to list of entries associated with completion channel
     * and the list associated with the specific CQ.
     */
    llist_head_init(&entry->ev_list, entry);
    llist_head_init(&entry->ev_obj_list, entry);

#ifdef DEBUG
    SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, "comp_evt_hdlr() "
        "Add COMP entry->ev_list=%p, &entry->ev_obj_list, entry=%p",
        &entry->ev_list, &entry->ev_obj_list, entry);
#endif

    llist_add_tail(&entry->ev_list, &ufile->event_list);
    llist_add_tail(&entry->ev_obj_list, &ucq->comp_list);

    /* Do not notify users if sol_ucma has disabled CQ notify */
    if (ufile->ufile_notify_enabled ==
        SOL_UVERBS2UCMA_CQ_NOTIFY_DISABLE) {
        mutex_exit(&ufile->lock);
        return;
    }

    cv_signal(&ufile->poll_wait);
    mutex_exit(&ufile->lock);
    pollwakeup(&ufile->poll_head, POLLIN | POLLRDNORM);
}