udapl_tavor/tavor/dapl_tavor_ibtf_util.c

/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright 2010 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

#include "dapl.h"
#include "dapl_adapter_util.h"
#include "dapl_evd_util.h"
#include "dapl_cr_util.h"
#include "dapl_lmr_util.h"
#include "dapl_rmr_util.h"
#include "dapl_cookie.h"
#include "dapl_ring_buffer_util.h"
#include "dapl_vendor.h"
#include "dapl_tavor_ibtf_impl.h"

/* Function prototypes */
static DAT_RETURN dapli_ib_cq_resize_internal(DAPL_EVD *, DAT_COUNT);

/*
 * The following declarations/fn are to used by the base library
 * place holder for now
 */

int g_loopback_connection = 0;

/*
 * dapl_ib_cq_alloc
 *
 * Alloc a CQ
 *
 * Input:
 *  ia_handle       IA handle
 *  evd_ptr         pointer to EVD struct
 *  cno_ptr         pointer to CNO struct
 *  cqlen           minimum QLen
 *
 * Output:
 *  none
 *
 * Returns:
 *  DAT_SUCCESS
 *  DAT_INSUFFICIENT_RESOURCES
 *
 */
DAT_RETURN
dapls_ib_cq_alloc(
    IN  DAPL_IA     *ia_ptr,
    IN  DAPL_EVD        *evd_ptr,
    IN  DAPL_CNO        *cno_ptr,
    IN  DAT_COUNT       *cqlen)
{
    dapl_evd_create_t   create_msg;
    dapl_evd_free_t     free_msg;
    ib_cq_handle_t      cq_handle = IB_INVALID_HANDLE;
    int         ia_fd;
    int         hca_fd;
    int         retval;
    mlnx_umap_cq_data_out_t *mcq;

    /* cq handle is created even for non-cq type events */
    /* since cq handle is where the evd fd gets stored. */
    cq_handle = (ib_cq_handle_t)
        dapl_os_alloc(sizeof (struct dapls_ib_cq_handle));
    if (cq_handle == NULL) {
        dapl_dbg_log(DAPL_DBG_TYPE_ERR,
            "cq_alloc: evd_ptr 0x%p, cq_handle == NULL\n",
            evd_ptr);
        return (DAT_INSUFFICIENT_RESOURCES);
    }

    (void) dapl_os_memzero(cq_handle, sizeof (*cq_handle));

    /* get the hca information from ia_ptr */
    (void) dapl_os_memzero(&create_msg, sizeof (create_msg));
    create_msg.evd_flags = evd_ptr->evd_flags;
    create_msg.evd_cookie = (uintptr_t)evd_ptr;
    if (cno_ptr != NULL) {
        create_msg.evd_cno_hkey =
            (uint64_t)cno_ptr->ib_cno_handle;
    }
    if (evd_ptr->evd_flags & (DAT_EVD_DTO_FLAG | DAT_EVD_RMR_BIND_FLAG)) {
        create_msg.evd_cq_size = (uint32_t)*cqlen;
    }

    dapl_dbg_log(DAPL_DBG_TYPE_UTIL,
        "cq_alloc: evd 0x%p, flags 0x%x, cookie 0x%llx, hkey 0x%llx,\n"
        "          cno_hkey 0x%llx, cq_size %d\n", evd_ptr,
        create_msg.evd_flags, create_msg.evd_cookie, create_msg.evd_hkey,
        create_msg.evd_cno_hkey, create_msg.evd_cq_size);

    ia_fd = ia_ptr->hca_ptr->ib_hca_handle->ia_fd;
    hca_fd = ia_ptr->hca_ptr->ib_hca_handle->hca_fd;
    mcq = (mlnx_umap_cq_data_out_t *)create_msg.evd_cq_data_out;

    /* The next line is only needed for backward compatibility */
    mcq->mcq_rev = MLNX_UMAP_IF_VERSION;

    /* call into driver to allocate cq */
    retval = ioctl(ia_fd, DAPL_EVD_CREATE, &create_msg);
    if (retval != 0 || mcq->mcq_rev != MLNX_UMAP_IF_VERSION) {
        dapl_dbg_log(DAPL_DBG_TYPE_ERR,
            "cq_alloc: evd_create failed, %s\n", strerror(errno));
        dapl_os_free(cq_handle, sizeof (struct dapls_ib_cq_handle));
        return (dapls_convert_error(errno, retval));
    }
    (void) dapl_os_memzero(cq_handle, sizeof (struct dapls_ib_cq_handle));
    dapl_os_lock_init(&cq_handle->cq_wrid_wqhdr_lock);

    dapl_dbg_log(DAPL_DBG_TYPE_UTIL,
        "cq_alloc: created, evd 0x%p, hkey 0x%016llx\n\n", evd_ptr,
        create_msg.evd_hkey);

    cq_handle->evd_hkey = create_msg.evd_hkey;

    if (evd_ptr->evd_flags & (DAT_EVD_DTO_FLAG | DAT_EVD_RMR_BIND_FLAG)) {

        /*
         * allocate a hash table for wrid management, the key is
         * a combination of QPnumber and SEND/RECV type. This is
         * required only for evd which have a CQ mapped to
         * it.
         */
        if (DAT_SUCCESS != dapls_hash_create(DAPL_MED_HASHSIZE,
            DAT_FALSE, &cq_handle->cq_wrid_wqhdr_list)) {
            dapl_dbg_log(DAPL_DBG_TYPE_ERR,
                "cq_alloc: hash_create failed\n");
            dapl_os_free(cq_handle,
                sizeof (struct dapls_ib_cq_handle));
            return (DAT_INSUFFICIENT_RESOURCES |
                DAT_RESOURCE_MEMORY);
        }

        dapl_os_assert(create_msg.evd_cq_real_size > 0);

        /* In the case of Arbel or Hermon */
        if (mcq->mcq_polldbr_mapoffset != 0 ||
            mcq->mcq_polldbr_maplen != 0)
            cq_handle->cq_poll_dbp = dapls_ib_get_dbp(
                mcq->mcq_polldbr_maplen, hca_fd,
                mcq->mcq_polldbr_mapoffset,
                mcq->mcq_polldbr_offset);
        if (mcq->mcq_armdbr_mapoffset != 0 ||
            mcq->mcq_armdbr_maplen != 0)
            cq_handle->cq_arm_dbp = dapls_ib_get_dbp(
                mcq->mcq_armdbr_maplen, hca_fd,
                mcq->mcq_armdbr_mapoffset,
                mcq->mcq_armdbr_offset);

        cq_handle->cq_addr = (tavor_hw_cqe_t *)(void *) mmap64(
            (void *)0, mcq->mcq_maplen,
            (PROT_READ | PROT_WRITE), MAP_SHARED, hca_fd,
            mcq->mcq_mapoffset);

        if (cq_handle->cq_addr == MAP_FAILED ||
            cq_handle->cq_poll_dbp == MAP_FAILED ||
            cq_handle->cq_arm_dbp == MAP_FAILED) {
            free_msg.evf_hkey = cq_handle->evd_hkey;
            retval = ioctl(ia_fd, DAPL_EVD_FREE, &free_msg);
            if (retval != 0) {
                dapl_dbg_log(DAPL_DBG_TYPE_ERR,
                    "cq_alloc: EVD_FREE err:%s\n",
                    strerror(errno));
            }

            dapl_dbg_log(DAPL_DBG_TYPE_ERR,
                "cq_alloc: DAPL_CQ_ALLOC failed\n");
            /* free the hash table we created */
            (void) dapls_hash_free(cq_handle->cq_wrid_wqhdr_list);
            dapl_os_free(cq_handle,
                sizeof (struct dapls_ib_cq_handle));
            return (DAT_INSUFFICIENT_RESOURCES);
        }

        cq_handle->cq_map_offset = mcq->mcq_mapoffset;
        cq_handle->cq_map_len = mcq->mcq_maplen;
        cq_handle->cq_num = mcq->mcq_cqnum;
        /*
         * cq_size is the actual depth of the CQ which is 1 more
         * than what ibt_alloc_cq reports. However the application
         * can only use (cq_size - 1) entries.
         */
        cq_handle->cq_size = create_msg.evd_cq_real_size + 1;
        cq_handle->cq_cqesz = mcq->mcq_cqesz;
        cq_handle->cq_iauar = ia_ptr->hca_ptr->ib_hca_handle->ia_uar;
        *cqlen = create_msg.evd_cq_real_size;

        DAPL_INIT_CQ(ia_ptr)(cq_handle);
    }

    evd_ptr->ib_cq_handle = cq_handle;
    return (DAT_SUCCESS);
}


/*
 * dapl_ib_cq_resize
 *
 * Resize a CQ
 *
 * Input:
 *  evd_ptr         pointer to EVD struct
 *  cqlen           new length of the cq
 * Output:
 *  none
 *
 * Returns:
 *  DAT_SUCCESS
 *  DAT_INVALID_HANDLE
 *  DAT_INTERNAL_ERROR
 *  DAT_INSUFFICIENT_RESOURCES
 *
 */
DAT_RETURN
dapls_ib_cq_resize(
    IN  DAPL_EVD        *evd_ptr,
    IN  DAT_COUNT       cqlen)
{
    ib_cq_handle_t  cq_handle;
    DAT_RETURN  dat_status;

    dat_status = dapli_ib_cq_resize_internal(evd_ptr, cqlen);
    if (DAT_INSUFFICIENT_RESOURCES == dat_status) {
        cq_handle = evd_ptr->ib_cq_handle;
        /* attempt to resize back to the current size */
        dat_status = dapli_ib_cq_resize_internal(evd_ptr,
            cq_handle->cq_size - 1);
        if (DAT_SUCCESS != dat_status) {
            /*
             * XXX this is catastrophic need to post an event
             * to the async evd
             */
            return (DAT_INTERNAL_ERROR);
        }
    }

    return (dat_status);
}

/*
 * dapli_ib_cq_resize_internal
 *
 * An internal routine to resize a CQ.
 *
 * Input:
 *  evd_ptr         pointer to EVD struct
 *  cqlen           new length of the cq
 * Output:
 *  none
 *
 * Returns:
 *  DAT_SUCCESS
 *  DAT_INVALID_HANDLE
 *  DAT_INSUFFICIENT_RESOURCES
 *
 */
static DAT_RETURN
dapli_ib_cq_resize_internal(
    IN  DAPL_EVD        *evd_ptr,
    IN  DAT_COUNT       cqlen)
{
    ib_cq_handle_t      cq_handle;
    dapl_cq_resize_t    resize_msg;
    int         ia_fd;
    int         hca_fd;
    int         retval;
    mlnx_umap_cq_data_out_t *mcq;
    DAPL_HCA        *hca_ptr;
    dapls_hw_cqe_t      cq_addr;

    dapl_dbg_log(DAPL_DBG_TYPE_UTIL,
        "dapls_ib_cq_resize: evd 0x%p cq 0x%p "
        "evd_hkey 0x%016llx cqlen %d\n",
        evd_ptr, (void *)evd_ptr->ib_cq_handle,
        evd_ptr->ib_cq_handle->evd_hkey, cqlen);

    cq_handle = evd_ptr->ib_cq_handle;
    /*
     * Since CQs are created in powers of 2 with one non-usable slot,
     * its possible that the previously allocated CQ has sufficient
     * entries. If the current cq is big enough and it is mapped in
     * we are done.
     */
    if ((cqlen < cq_handle->cq_size) && (cq_handle->cq_addr)) {
        return (DAT_SUCCESS);
    }

    hca_ptr = evd_ptr->header.owner_ia->hca_ptr;

    /* unmap the CQ before resizing it */
    if (hca_ptr->hermon_resize_cq == 0) {
        if ((cq_handle->cq_addr) &&
            (munmap((char *)cq_handle->cq_addr,
            cq_handle->cq_map_len) < 0)) {
            dapl_dbg_log(DAPL_DBG_TYPE_ERR,
                "cq_resize: munmap(%p:0x%llx) failed(%d)\n",
                cq_handle->cq_addr, cq_handle->cq_map_len, errno);
            return (DAT_INVALID_HANDLE);
        }
        /* cq_addr is unmapped and no longer valid */
        cq_handle->cq_addr = NULL;
    }

    ia_fd = hca_ptr->ib_hca_handle->ia_fd;
    hca_fd = hca_ptr->ib_hca_handle->hca_fd;

    (void) dapl_os_memzero(&resize_msg, sizeof (resize_msg));
    mcq = (mlnx_umap_cq_data_out_t *)resize_msg.cqr_cq_data_out;
    resize_msg.cqr_evd_hkey = cq_handle->evd_hkey;
    resize_msg.cqr_cq_new_size = cqlen;

    /* The next line is only needed for backward compatibility */
    mcq->mcq_rev = MLNX_UMAP_IF_VERSION;
    retval = ioctl(ia_fd, DAPL_CQ_RESIZE, &resize_msg);
    if (retval != 0 || mcq->mcq_rev != MLNX_UMAP_IF_VERSION) {
        dapl_dbg_log(DAPL_DBG_TYPE_ERR,
            "dapls_ib_cq_resize: evd 0x%p, err: %s\n",
            evd_ptr, strerror(errno));
        if (errno == EINVAL) { /* Couldn't find evd for this cq */
            return (DAT_INVALID_HANDLE);
        } else { /* Need to retry resize with a smaller qlen */
            return (DAT_INSUFFICIENT_RESOURCES);
        }
    }

    dapl_os_assert(cq_handle->cq_num == mcq->mcq_cqnum);

    /* In the case of Arbel or Hermon */
    if (mcq->mcq_polldbr_mapoffset != 0 ||
        mcq->mcq_polldbr_maplen != 0)
        cq_handle->cq_poll_dbp = dapls_ib_get_dbp(
            mcq->mcq_polldbr_maplen, hca_fd,
            mcq->mcq_polldbr_mapoffset,
            mcq->mcq_polldbr_offset);
    if (mcq->mcq_armdbr_mapoffset != 0 ||
        mcq->mcq_armdbr_maplen != 0)
        cq_handle->cq_arm_dbp = dapls_ib_get_dbp(
            mcq->mcq_armdbr_maplen, hca_fd,
            mcq->mcq_armdbr_mapoffset,
            mcq->mcq_armdbr_offset);

    cq_addr = (tavor_hw_cqe_t *)(void *)mmap64((void *)0,
        mcq->mcq_maplen, (PROT_READ | PROT_WRITE),
        MAP_SHARED, hca_fd, mcq->mcq_mapoffset);

    if (cq_addr == MAP_FAILED ||
        cq_handle->cq_poll_dbp == MAP_FAILED ||
        cq_handle->cq_arm_dbp == MAP_FAILED) {
        if (hca_ptr->hermon_resize_cq == 0)
            cq_handle->cq_addr = NULL;
        dapl_dbg_log(DAPL_DBG_TYPE_ERR,
            "cq_resize: mmap failed(%d)\n", errno);
        /* Need to retry resize with a smaller qlen */
        return (DAT_INSUFFICIENT_RESOURCES);
    }

    if (hca_ptr->hermon_resize_cq == 0) {
        cq_handle->cq_addr = cq_addr;
        cq_handle->cq_map_offset = mcq->mcq_mapoffset;
        cq_handle->cq_map_len = mcq->mcq_maplen;
        cq_handle->cq_size = resize_msg.cqr_cq_real_size + 1;
        cq_handle->cq_cqesz = mcq->mcq_cqesz;
        /*
         * upon resize the old events are moved to the start of the CQ
         * hence we need to reset the consumer index too
         */
        cq_handle->cq_consindx = 0;
    } else {    /* Hermon */
        cq_handle->cq_resize_addr = cq_addr;
        cq_handle->cq_resize_map_offset = mcq->mcq_mapoffset;
        cq_handle->cq_resize_map_len = mcq->mcq_maplen;
        cq_handle->cq_resize_size = resize_msg.cqr_cq_real_size + 1;
        cq_handle->cq_resize_cqesz = mcq->mcq_cqesz;
    }

    return (DAT_SUCCESS);
}

/*
 * dapl_ib_cq_free
 *
 * Free a CQ
 *
 * Input:
 *  ia_handle       IA handle
 *  evd_ptr         pointer to EVD struct
 * Output:
 *  none
 *
 * Returns:
 *  DAT_SUCCESS
 *  DAT_INVALID_HANDLE
 *  DAT_INSUFFICIENT_RESOURCES
 *
 */
DAT_RETURN
dapls_ib_cq_free(
    IN  DAPL_IA     *ia_ptr,
    IN  DAPL_EVD        *evd_ptr)
{
    dapl_evd_free_t     args;
    int         retval;
    ib_cq_handle_t      cq_handle = evd_ptr->ib_cq_handle;

    dapl_dbg_log(DAPL_DBG_TYPE_UTIL,
        "dapls_ib_cq_free: evd 0x%p cq 0x%p hkey %016llx\n", evd_ptr,
        (void *)evd_ptr->ib_cq_handle, evd_ptr->ib_cq_handle->evd_hkey);

    /* If the cq was mmap'd unmap it before freeing it */
    if ((cq_handle->cq_addr) &&
        (munmap((char *)cq_handle->cq_addr, cq_handle->cq_map_len) < 0)) {
            dapl_dbg_log(DAPL_DBG_TYPE_ERR,
                "cq_free: (%p:0x%llx)\n", cq_handle->cq_addr,
                cq_handle->cq_map_len);
    }


    args.evf_hkey = cq_handle->evd_hkey;

    retval = ioctl(ia_ptr->hca_ptr->ib_hca_handle->ia_fd,
        DAPL_EVD_FREE, &args);
    if (retval != 0) {
        dapl_dbg_log(DAPL_DBG_TYPE_ERR,
            "dapls_ib_cq_free: evd 0x%p, err: %s\n",
            evd_ptr, strerror(errno));
        return (dapls_convert_error(errno, retval));
    }

    dapl_os_free(cq_handle, sizeof (struct dapls_ib_cq_handle));
    evd_ptr->ib_cq_handle = NULL;

    return (DAT_SUCCESS);
}

/*
 * dapl_set_cq_notify
 *
 * Set up CQ completion notifications
 *
 * Input:
 *  ia_handle       IA handle
 *  evd_ptr         pointer to EVD struct
 *
 * Output:
 *  none
 *
 * Returns:
 *  DAT_SUCCESS
 *  DAT_INVALID_HANDLE
 *  DAT_INSUFFICIENT_RESOURCES
 *
 */
/* ARGSUSED */
DAT_RETURN
dapls_set_cq_notify(
    IN  DAPL_IA     *ia_ptr,
    IN  DAPL_EVD        *evd_ptr)
{
    int         retval;
    ib_cq_handle_t      cq_handle = evd_ptr->ib_cq_handle;

    dapl_dbg_log(DAPL_DBG_TYPE_UTIL,
        "dapls_ib_cq_notify: evd 0x%p cq 0x%p\n", evd_ptr,
        (void *)cq_handle);

    retval = DAPL_NOTIFY(evd_ptr)(cq_handle, IB_NOTIFY_ON_NEXT_COMP, 0);

    return (retval);

}

/* ARGSUSED */
DAT_RETURN
dapls_set_cqN_notify(
    IN  DAPL_IA     *ia_ptr,
    IN  DAPL_EVD        *evd_ptr,
    IN  uint32_t        num_events)
{
    int         retval;
    ib_cq_handle_t      cq_handle = evd_ptr->ib_cq_handle;

    dapl_dbg_log(DAPL_DBG_TYPE_UTIL,
        "dapls_set_cqN_notify:evd %p cq %p num_events %d\n", evd_ptr,
        (void *)cq_handle, num_events);

    retval = DAPL_NOTIFY(evd_ptr)(cq_handle, IB_NOTIFY_ON_NEXT_NCOMP,
        num_events);

    return (retval);

}

/*
 * dapls_ib_cqd_create
 *
 * Set up CQ notification event thread
 *
 * Input:
 *  ia_handle       IA handle
 *
 * Output:
 *  none
 *
 * Returns:
 *  DAT_SUCCESS
 *  DAT_INVALID_HANDLE
 *  DAT_INSUFFICIENT_RESOURCES
 *
 */
/* ARGSUSED */
DAT_RETURN
dapls_ib_cqd_create(
    IN  DAPL_HCA        *hca_ptr)
{
    return (DAT_SUCCESS);
}


/*
 * dapl_cqd_destroy
 *
 * Destroy CQ notification event thread
 *
 * Input:
 *  ia_handle       IA handle
 *
 * Output:
 *  none
 *
 * Returns:
 *  DAT_SUCCESS
 *  DAT_INVALID_HANDLE
 *  DAT_INSUFFICIENT_RESOURCES
 *
 */
DAT_RETURN
dapls_ib_cqd_destroy(
    IN  DAPL_HCA        *hca_ptr)
{
    dapl_evd_free_t     args;
    ib_cq_handle_t      cq_handle;
    int         retval;

    if (hca_ptr->null_ib_cq_handle != IB_INVALID_HANDLE) {
        /* free up the dummy cq */
        cq_handle = hca_ptr->null_ib_cq_handle;
        dapl_dbg_log(DAPL_DBG_TYPE_UTIL,
            "dapls_ib_cqd_destroy: cq %p\n", (void *)cq_handle);

        args.evf_hkey = cq_handle->evd_hkey;

        retval = ioctl(hca_ptr->ib_hca_handle->ia_fd,
            DAPL_EVD_FREE, &args);
        if (retval != 0) {
            dapl_dbg_log(DAPL_DBG_TYPE_ERR,
                "dapls_ib_cqd_destroy: EVD_FREE err:%d errno:%d\n",
                retval, errno);
        }

        dapl_os_free(cq_handle, sizeof (struct dapls_ib_cq_handle));
        hca_ptr->null_ib_cq_handle = IB_INVALID_HANDLE;
    }

    return (DAT_SUCCESS);
}


/*
 * dapl_ib_pd_alloc
 *
 * Alloc a PD
 *
 * Input:
 *  ia_handle       IA handle
 *  PZ_ptr          pointer to PZEVD struct
 *
 * Output:
 *  none
 *
 * Returns:
 *  DAT_SUCCESS
 *  DAT_INSUFFICIENT_RESOURCES
 *
 */
DAT_RETURN
dapls_ib_pd_alloc(
    IN  DAPL_IA         *ia,
    IN  DAPL_PZ         *pz)
{
    struct dapls_ib_pd_handle *pd_p;
    dapl_pd_alloc_t args;
    int retval;

    pd_p = (struct dapls_ib_pd_handle *)dapl_os_alloc(sizeof (*pd_p));
    if (pd_p == NULL) {
        dapl_dbg_log(DAPL_DBG_TYPE_ERR,
            "pd_alloc: ia 0x%p, pz 0x%p, cannot allocate pd\n",
            ia, pz);
        return (DAT_INSUFFICIENT_RESOURCES);
    }
    retval = ioctl(ia->hca_ptr->ib_hca_handle->ia_fd,
        DAPL_PD_ALLOC, &args);
    if (retval != 0) {
        dapl_dbg_log(DAPL_DBG_TYPE_ERR,
            "pd_alloc: ia 0x%p, pz 0x%p, cannot create pd, "
            "err: %s\n", ia, pz, strerror(errno));
        dapl_os_free(pd_p, sizeof (*pd_p));
        return (dapls_convert_error(errno, retval));
    }

    pd_p->pd_hkey = args.pda_hkey;
    pz->pd_handle = pd_p;
    dapl_dbg_log(DAPL_DBG_TYPE_UTIL,
        "pd_alloc: successful, ia 0x%p, pz 0x%p, hkey %016llx\n",
        ia, pz, args.pda_hkey);

    return (DAT_SUCCESS);
}


/*
 * dapl_ib_pd_free
 *
 * Free a PD
 *
 * Input:
 *  ia_handle       IA handle
 *  PZ_ptr          pointer to PZ struct
 *
 * Output:
 *  none
 *
 * Returns:
 *  DAT_SUCCESS
 *  DAT_INSUFFICIENT_RESOURCES
 *
 */
DAT_RETURN
dapls_ib_pd_free(
    IN  DAPL_PZ         *pz)
{
    struct dapls_ib_pd_handle *pd_p;
    dapl_pd_free_t args;
    int retval;

    pd_p = (struct dapls_ib_pd_handle *)pz->pd_handle;
    args.pdf_hkey = pd_p->pd_hkey;

    retval = ioctl(pz->header.owner_ia->hca_ptr->ib_hca_handle->ia_fd,
        DAPL_PD_FREE, &args);
    if (retval != 0) {
        dapl_dbg_log(DAPL_DBG_TYPE_ERR,
            "pd_free: pz 0x%p, cannot free pd\n", pz);
        return (dapls_convert_error(errno, retval));
    }
    dapl_dbg_log(DAPL_DBG_TYPE_UTIL,
        "pd_free: pz 0x%p, hkey %016llx, freed\n", pz, pd_p->pd_hkey);
    dapl_os_free((void *)pd_p, sizeof (*pd_p));
    pz->pd_handle = NULL;
    return (DAT_SUCCESS);
}


/*
 * dapl_ib_mr_register
 *
 * Register a virtual memory region
 *
 * Input:
 *  ia_handle       IA handle
 *  lmr         pointer to dapl_lmr struct
 *  virt_addr       virtual address of beginning of mem region
 *  length          length of memory region
 *
 * Output:
 *  none
 *
 * Returns:
 *  DAT_SUCCESS
 *  DAT_INSUFFICIENT_RESOURCES
 *
 */
DAT_RETURN
dapls_ib_mr_register(
    IN  DAPL_IA         *ia,
    IN  DAPL_LMR        *lmr,
    IN  DAT_PVOID       virt_addr,
    IN  DAT_VLEN        length,
    IN  DAT_MEM_PRIV_FLAGS  privileges)
{
    dapl_mr_register_t  reg_msg;
    ib_mr_handle_t      mr_handle;
    DAPL_PZ *       pz_handle;
    int         ia_fd;
    int         retval;

    ia_fd = ia->hca_ptr->ib_hca_handle->ia_fd;
    mr_handle = dapl_os_alloc(sizeof (struct dapls_ib_mr_handle));
    if (mr_handle == NULL) {
        dapl_dbg_log(DAPL_DBG_TYPE_ERR,
            "mr_register: lmr 0x%p, ia 0x%p, "
            "cannot alloc mr_handle\n", lmr, ia);
        return (DAT_INSUFFICIENT_RESOURCES);
    }
    pz_handle = ((DAPL_PZ *)lmr->param.pz_handle);
    if (pz_handle == NULL) {
        dapl_dbg_log(DAPL_DBG_TYPE_ERR,
            "mr_register: lmr 0x%p, ia 0x%p, "
            "pz_handle == NULL!\n", lmr, ia);
        dapl_os_free(mr_handle, sizeof (struct dapls_ib_mr_handle));
        return (DAT_INVALID_PARAMETER);
    }
    reg_msg.mr_pd_hkey = pz_handle->pd_handle->pd_hkey;
    reg_msg.mr_vaddr = (ib_vaddr_t)(uintptr_t)virt_addr;
    reg_msg.mr_len = (ib_memlen_t)length;
    reg_msg.mr_flags = (ibt_mr_flags_t)
        dapl_lmr_convert_privileges(privileges);
    reg_msg.mr_flags |= IBT_MR_ENABLE_WINDOW_BIND;

    dapl_dbg_log(DAPL_DBG_TYPE_UTIL,
        "mr_register: lmr 0x%p, pd_hkey 0x%016llx, vaddr 0x%016llx, "
        "len %llu, flags 0x%x\n", lmr, reg_msg.mr_pd_hkey,
        reg_msg.mr_vaddr, reg_msg.mr_len, reg_msg.mr_flags);

    /* call into driver to allocate MR resource */
    retval = ioctl(ia_fd, DAPL_MR_REGISTER, &reg_msg);
    if (retval != 0) {
        dapl_dbg_log(DAPL_DBG_TYPE_ERR,
            "mr_register: lmr 0x%p, failed (%s)\n",
            lmr, strerror(errno));
        dapl_os_free(mr_handle, sizeof (struct dapls_ib_mr_handle));
        return (dapls_convert_error(errno, retval));
    }
    mr_handle->mr_hkey = reg_msg.mr_hkey;
    lmr->param.lmr_context = (DAT_LMR_CONTEXT)reg_msg.mr_lkey;
    lmr->param.rmr_context = (DAT_RMR_CONTEXT)reg_msg.mr_rkey;
    lmr->param.registered_address = reg_msg.mr_vaddr;
    lmr->param.registered_size = reg_msg.mr_len;
    lmr->mr_handle = mr_handle;

    dapl_dbg_log(DAPL_DBG_TYPE_UTIL,
        "mr_register: successful, lmr 0x%p, mr_hkey 0x%016llx, "
        "lmr_ctx 0x%08x\n\n", lmr, reg_msg.mr_hkey,
        reg_msg.mr_lkey);
    return (DAT_SUCCESS);
}

/*
 * dapl_ib_mr_register_shared
 *
 * Register a shared virtual memory region
 *
 * Input:
 *  ia_handle       IA handle
 *  lmr         pointer to dapl_lmr struct
 *  virt_addr       virtual address of beginning of mem region
 *  cookie          shared memory identifer
 *  length          length of memory region
 *
 * Output:
 *  none
 *
 * Returns:
 *  DAT_SUCCESS
 *  DAT_INSUFFICIENT_RESOURCES
 *
 */
DAT_RETURN
dapls_ib_mr_register_shared(
    IN  DAPL_IA         *ia,
    IN  DAPL_LMR        *lmr,
    IN  DAT_PVOID       virt_addr,
    IN  DAT_VLEN        length,
    IN  DAT_LMR_COOKIE  cookie,
    IN  DAT_MEM_PRIV_FLAGS  privileges)
{
    dapl_mr_register_shared_t   reg_msg;
    ib_mr_handle_t          mr_handle;
    DAPL_PZ             *pz_handle;
    int             ia_fd, i;
    int             retval;

    ia_fd = ia->hca_ptr->ib_hca_handle->ia_fd;
    mr_handle = dapl_os_alloc(sizeof (struct dapls_ib_mr_handle));
    if (mr_handle == NULL) {
        dapl_dbg_log(DAPL_DBG_TYPE_ERR,
            "mr_register_shared: lmr 0x%p, ia 0x%p, "
            "cannot alloc mr_handle\n", lmr, ia);
        return (DAT_INSUFFICIENT_RESOURCES);
    }
    pz_handle = ((DAPL_PZ *)lmr->param.pz_handle);
    if (pz_handle == NULL) {
        dapl_dbg_log(DAPL_DBG_TYPE_ERR,
            "mr_register_shared: lmr 0x%p, ia 0x%p, "
            "pz_handle == NULL!\n", lmr, ia);
        dapl_os_free(mr_handle, sizeof (struct dapls_ib_mr_handle));
        return (DAT_INVALID_PARAMETER);
    }
    reg_msg.mrs_pd_hkey = pz_handle->pd_handle->pd_hkey;
    reg_msg.mrs_vaddr = (ib_vaddr_t)(uintptr_t)virt_addr;
    reg_msg.mrs_len = (ib_memlen_t)length;
    reg_msg.mrs_flags = (ibt_mr_flags_t)
        dapl_lmr_convert_privileges(privileges);
    reg_msg.mrs_flags |= IBT_MR_ENABLE_WINDOW_BIND;
    /*CONSTCOND*/
    dapl_os_assert(DAT_LMR_COOKIE_SIZE == sizeof (reg_msg.mrs_shm_cookie));
    (void) dapl_os_memcpy((void *)&reg_msg.mrs_shm_cookie, (void *)cookie,
        DAT_LMR_COOKIE_SIZE);

    dapl_dbg_log(DAPL_DBG_TYPE_UTIL,
        "mr_register_shared: lmr 0x%p, pd_hkey 0x%016llx, "
        "vaddr 0x%016llx, len %llu, flags 0x%x\n",
        lmr, reg_msg.mrs_pd_hkey, reg_msg.mrs_vaddr, reg_msg.mrs_len,
        reg_msg.mrs_flags);

    dapl_dbg_log(DAPL_DBG_TYPE_UTIL,
        "mr_register_shared: cookie \n0x");
    for (i = 4; i >= 0; i--) {
        dapl_dbg_log(DAPL_DBG_TYPE_UTIL,
            "%016llx", reg_msg.mrs_shm_cookie.mc_uint_arr[i]);
    }
    dapl_dbg_log(DAPL_DBG_TYPE_UTIL, "\n");

    /* call into driver to allocate MR resource */
    retval = ioctl(ia_fd, DAPL_MR_REGISTER_SHARED, &reg_msg);
    if (retval != 0) {
        dapl_dbg_log(DAPL_DBG_TYPE_ERR,
            "mr_register_shared: lmr 0x%p, failed (%s)\n",
            lmr, strerror(errno));
        dapl_os_free(mr_handle, sizeof (struct dapls_ib_mr_handle));
        return (dapls_convert_error(errno, retval));
    }
    mr_handle->mr_hkey = reg_msg.mrs_hkey;
    lmr->param.lmr_context = (DAT_LMR_CONTEXT)reg_msg.mrs_lkey;
    lmr->param.rmr_context = (DAT_RMR_CONTEXT)reg_msg.mrs_rkey;
    lmr->param.registered_address = reg_msg.mrs_vaddr;
    lmr->param.registered_size = reg_msg.mrs_len;
    lmr->mr_handle = mr_handle;

    dapl_dbg_log(DAPL_DBG_TYPE_UTIL,
        "mr_register_shared: successful, lmr 0x%p, mr_hkey 0x%016llx, "
        "lmr_ctx 0x%08x\n\n", lmr, reg_msg.mrs_hkey,
        reg_msg.mrs_lkey);
    return (DAT_SUCCESS);
}

/*
 * dapl_ib_mr_deregister
 *
 * Free a memory region
 *
 * Input:
 *  lmr         pointer to dapl_lmr struct
 *
 * Output:
 *  none
 *
 * Returns:
 *  DAT_SUCCESS
 *  DAT_INSUFFICIENT_RESOURCES
 *
 */
DAT_RETURN
dapls_ib_mr_deregister(
    IN  DAPL_LMR        *lmr)
{
    dapl_mr_deregister_t args;
    int retval;

    args.mrd_hkey = lmr->mr_handle->mr_hkey;
    dapl_dbg_log(DAPL_DBG_TYPE_UTIL,
        "mr_deregister: lmr 0x%p, hkey 0x%016llx, lmr_ctx 0x%08x\n"
        "               vaddr 0x%016llx, len %llu, flags 0x%x\n",
        lmr, args.mrd_hkey, lmr->param.lmr_context,
        lmr->param.registered_address, lmr->param.registered_size,
        dapl_lmr_convert_privileges(lmr->param.mem_priv) |
        IBT_MR_ENABLE_WINDOW_BIND);

    /* call into driver to do MR deregister */
    retval = ioctl(lmr->header.owner_ia->hca_ptr->ib_hca_handle->ia_fd,
        DAPL_MR_DEREGISTER, &args);

    if (retval != 0) {
        dapl_dbg_log(DAPL_DBG_TYPE_ERR,
            "mr_deregister: lmr 0x%p, failed (%s)\n",
            lmr, strerror(errno));
        return (dapls_convert_error(errno, retval));
    }

    dapl_dbg_log(DAPL_DBG_TYPE_UTIL,
        "mr_deregister: successful\n\n");
    dapl_os_free(lmr->mr_handle, sizeof (struct dapls_ib_mr_handle));
    lmr->mr_handle = NULL;
    return (DAT_SUCCESS);
}


/*
 * dapl_ib_mr_register_lmr
 *
 * Register a memory region based on attributes of an existing one
 *
 * Input:
 *  ia_handle       IA handle
 *  lmr         pointer to dapl_lmr struct
 *  virt_addr       virtual address of beginning of mem region
 *  length          length of memory region
 *
 * Output:
 *  none
 *
 * Returns:
 *  DAT_SUCCESS
 *  DAT_INSUFFICIENT_RESOURCES
 *
 */
DAT_RETURN
dapls_ib_mr_register_lmr(
    IN  DAPL_IA             *ia,
    IN  DAPL_LMR            *lmr,
    IN  DAT_MEM_PRIV_FLAGS      privileges)
{
    dapl_mr_register_lmr_t      regl_msg;
    DAPL_LMR            *orig_lmr;
    struct dapls_ib_mr_handle   *orig_mr_handle;
    ib_mr_handle_t          mr_handle;
    int             ia_fd;
    int             retval;

    ia_fd = ia->hca_ptr->ib_hca_handle->ia_fd;
    mr_handle = dapl_os_alloc(sizeof (struct dapls_ib_mr_handle));
    if (mr_handle == NULL) {
        dapl_dbg_log(DAPL_DBG_TYPE_ERR,
            "mr_register_lmr: lmr 0x%p, ia 0x%p, "
            "cannot alloc mr_handle\n", lmr, ia);
        return (DAT_INSUFFICIENT_RESOURCES);
    }

    orig_lmr = (DAPL_LMR *)lmr->param.region_desc.for_lmr_handle;
    orig_mr_handle = (struct dapls_ib_mr_handle *)orig_lmr->mr_handle;
    regl_msg.mrl_orig_hkey = orig_mr_handle->mr_hkey;
    regl_msg.mrl_flags = (ibt_mr_flags_t)
        dapl_lmr_convert_privileges(privileges);
    regl_msg.mrl_flags |= IBT_MR_ENABLE_WINDOW_BIND;
    regl_msg.mrl_lkey = regl_msg.mrl_rkey = 0;

    dapl_dbg_log(DAPL_DBG_TYPE_UTIL,
        "mr_register_lmr: lmr 0x%p, hkey 0x%016llx, lmr_ctx 0x%08x\n"
        "                 vaddr 0x%016llx, len %llu, flags 0x%x\n",
        lmr, mr_handle->mr_hkey, lmr->param.lmr_context,
        orig_lmr->param.registered_address,
        orig_lmr->param.registered_size,
        dapl_lmr_convert_privileges(orig_lmr->param.mem_priv) |
        IBT_MR_ENABLE_WINDOW_BIND);


    /* call into driver to allocate MR resource */
    retval = ioctl(ia_fd, DAPL_MR_REGISTER_LMR, &regl_msg);
    if (retval != 0) {
        dapl_dbg_log(DAPL_DBG_TYPE_ERR,
            "mr_register_lmr: failed (%s), orig_hkey (%016llx)\n",
            strerror(errno), orig_mr_handle->mr_hkey);
        dapl_os_free(mr_handle, sizeof (struct dapls_ib_mr_handle));
        return (dapls_convert_error(errno, retval));
    }

    dapl_dbg_log(DAPL_DBG_TYPE_UTIL,
        "mr_registered_lmr: successful, lmr 0x%p, hkey 0x%016llx\n",
        lmr, regl_msg.mrl_hkey);

    mr_handle->mr_hkey = regl_msg.mrl_hkey;
    lmr->param.lmr_context = (DAT_LMR_CONTEXT)regl_msg.mrl_lkey;
    lmr->param.rmr_context = (DAT_RMR_CONTEXT)regl_msg.mrl_rkey;
    lmr->param.registered_address = orig_lmr->param.registered_address;
    lmr->param.registered_size = orig_lmr->param.registered_size;
    lmr->mr_handle = mr_handle;

    return (DAT_SUCCESS);
}


/*
 * dapls_ib_mw_alloc
 *
 * Bind a protection domain to a memory window
 *
 * Input:
 *  rmr         Initialized rmr to hold binding handles
 *
 * Output:
 *  none
 *
 * Returns:
 *  DAT_SUCCESS
 *  DAT_INSUFFICIENT_RESOURCES
 *
 */
DAT_RETURN
dapls_ib_mw_alloc(
    IN  DAPL_RMR    *rmr)
{
    DAPL_IA     *ia_hdl = (DAPL_IA *)rmr->param.ia_handle;
    DAPL_PZ     *pz_hdl = rmr->param.pz_handle;
    dapl_mw_alloc_t args;
    ib_mw_handle_t  mw_handle;
    int     ia_fd;
    int     retval;

    ia_fd = ((struct dapls_ib_hca_handle *)(ia_hdl->hca_ptr->
        ib_hca_handle))->ia_fd;

    mw_handle = dapl_os_alloc(sizeof (struct dapls_ib_mw_handle));
    if (mw_handle == NULL) {
        dapl_dbg_log(DAPL_DBG_TYPE_ERR,
            "mw_alloc: rmr 0x%p, cannot alloc mw_handle\n", rmr);
        return (DAT_INSUFFICIENT_RESOURCES);
    }
    args.mw_pd_hkey = ((struct dapls_ib_pd_handle *)
        (pz_hdl->pd_handle))->pd_hkey;

    dapl_dbg_log(DAPL_DBG_TYPE_UTIL,
        "mw_alloc: rmr 0x%p, pd_hkey 0x%016llx\n",
        rmr, args.mw_pd_hkey);

    retval = ioctl(ia_fd, DAPL_MW_ALLOC, &args);
    if (retval != 0) {
        dapl_dbg_log(DAPL_DBG_TYPE_ERR,
            "mw_alloc: rmr 0x%p, failed (%s)\n", rmr, strerror(errno));
        dapl_os_free(mw_handle, sizeof (struct dapls_ib_mr_handle));
        return (dapls_convert_error(errno, retval));
    }

    mw_handle->mw_hkey = args.mw_hkey;
    rmr->mw_handle = mw_handle;
    rmr->param.rmr_context = (DAT_RMR_CONTEXT) args.mw_rkey;

    dapl_dbg_log(DAPL_DBG_TYPE_UTIL,
        "mw_alloc: successful, rmr 0x%p, mw_hkey 0x%llx, "
        "rmr_ctx 0x%x\n\n", rmr, (uint64_t)args.mw_hkey,
        rmr->param.rmr_context);

    return (DAT_SUCCESS);
}


/*
 * dapls_ib_mw_free
 *
 * Release bindings of a protection domain to a memory window
 *
 * Input:
 *  rmr         Initialized rmr to hold binding handles
 *
 * Output:
 *  none
 *
 * Returns:
 *  DAT_SUCCESS
 *  DAT_INSUFFICIENT_RESOURCES
 *
 */
DAT_RETURN
dapls_ib_mw_free(
    IN  DAPL_RMR    *rmr)
{
    DAPL_IA     *ia_hdl = rmr->param.ia_handle;
    dapl_mw_free_t  args;
    int     ia_fd;
    int     retval;

    ia_fd = ((struct dapls_ib_hca_handle *)(ia_hdl->hca_ptr->
        ib_hca_handle))->ia_fd;

    args.mw_hkey = rmr->mw_handle->mw_hkey;

    dapl_dbg_log(DAPL_DBG_TYPE_UTIL,
        "mw_free: rmr 0x%p, mw_hkey 0x%016llx\n", rmr, args.mw_hkey);

    retval = ioctl(ia_fd, DAPL_MW_FREE, &args);
    if (retval != 0) {
        dapl_dbg_log(DAPL_DBG_TYPE_ERR,
            "mw_free: rmr 0x%p, failed (%s)\n", rmr, strerror(errno));
        return (dapls_convert_error(errno, retval));
    }

    dapl_dbg_log(DAPL_DBG_TYPE_UTIL, "mw_free: successful\n\n");
    dapl_os_free(rmr->mw_handle, sizeof (struct dapls_ib_mr_handle));
    rmr->mw_handle = NULL;

    return (DAT_SUCCESS);
}

/*
 * dapls_ib_mw_bind
 *
 * Bind a protection domain to a memory window
 *
 * Input:
 *  rmr         Initialized rmr to hold binding handles
 *
 * Output:
 *  none
 *
 * Returns:
 *  DAT_SUCCESS
 *  DAT_INSUFFICIENT_RESOURCES
 *
 */
DAT_RETURN
dapls_ib_mw_bind(
    IN  DAPL_RMR        *rmr,
    IN  DAT_LMR_CONTEXT lmr_context,
    IN  DAPL_EP     *ep,
    IN  DAPL_COOKIE     *cookie,
    IN  DAT_VADDR       virtual_address,
    IN  DAT_VLEN        length,
    IN  DAT_MEM_PRIV_FLAGS  mem_priv,
    IN  DAT_COMPLETION_FLAGS completion_flags)
{
    ibt_send_wr_t   wre;
    ibt_wr_bind_t   wrbind;
    boolean_t   suppress_notification;
    int     retval;

    if (length > 0) {
        wrbind.bind_flags = (ibt_bind_flags_t)
            (dapl_rmr_convert_privileges(mem_priv) |
            IBT_WR_BIND_ATOMIC);
    } else {
        wrbind.bind_flags = (ibt_bind_flags_t)NULL;
    }
    wrbind.bind_rkey = rmr->param.rmr_context;
    wrbind.bind_va = virtual_address;
    wrbind.bind_len = length;
    wrbind.bind_lkey = lmr_context;

    wre.wr_id = (ibt_wrid_t)(uintptr_t)cookie;
    /*
     * wre.wr_flags = (is_signaled) ? IBT_WR_SEND_SIGNAL :
     *   IBT_WR_NO_FLAGS;
     * Till we fix the chan alloc flags do the following -
     */
    /* Translate dapl flags */
    wre.wr_flags = (DAT_COMPLETION_BARRIER_FENCE_FLAG &
        completion_flags) ? IBT_WR_SEND_FENCE : 0;
    /* suppress completions */
    wre.wr_flags |= (DAT_COMPLETION_SUPPRESS_FLAG &
        completion_flags) ? 0 : IBT_WR_SEND_SIGNAL;

    wre.wr_trans = IBT_RC_SRV;
    wre.wr_opcode = IBT_WRC_BIND;
    wre.wr_nds = 0;
    wre.wr_sgl = NULL;
    wre.wr.rc.rcwr.bind = &wrbind;

    dapl_dbg_log(DAPL_DBG_TYPE_UTIL,
        "mw_bind: rmr 0x%p, wr_flags 0x%x, rkey 0x%x, bind_flags 0x%x\n"
        "         bind_va 0x%llx, bind_len 0x%llx, mem_priv 0x%x\n",
        rmr, wre.wr_flags, wrbind.bind_rkey, wrbind.bind_flags,
        wrbind.bind_va, wrbind.bind_len, mem_priv);

    if (ep->param.ep_attr.recv_completion_flags &
        DAT_COMPLETION_UNSIGNALLED_FLAG) {
        /* This flag is used to control notification of completions */
        suppress_notification = (completion_flags &
            DAT_COMPLETION_UNSIGNALLED_FLAG) ? B_TRUE : B_FALSE;
    } else {
        /*
         * The evd waiter will use threshold to control wakeups
         * Hence the event notification will be done via arming the
         * CQ so we do not need special notification generation
         * hence set suppression to true
         */
        suppress_notification = B_TRUE;
    }

    retval = DAPL_SEND(ep)(ep, &wre, suppress_notification);

    if (retval != 0) {
        dapl_dbg_log(DAPL_DBG_TYPE_ERR,
            "mw_bind: rmr 0x%p, failed (%s)\n", rmr, strerror(errno));
        return (dapls_convert_error(errno, retval));
    }

    dapl_dbg_log(DAPL_DBG_TYPE_UTIL,
        "mw_bind: new_rkey = 0x%x\n", wrbind.bind_rkey_out);
    rmr->param.rmr_context = (DAT_RMR_CONTEXT) wrbind.bind_rkey_out;

    return (DAT_SUCCESS);
}

/*
 * dapls_ib_mw_unbind
 *
 * Unbind a protection domain from a memory window
 *
 * Input:
 *  rmr         Initialized rmr to hold binding handles
 *
 * Output:
 *  none
 *
 * Returns:
 *  DAT_SUCCESS
 *  DAT_INSUFFICIENT_RESOURCES
 *
 */
DAT_RETURN
dapls_ib_mw_unbind(
    IN  DAPL_RMR        *rmr,
    IN  DAT_LMR_CONTEXT lmr_context,
    IN  DAPL_EP     *ep,
    IN  DAPL_COOKIE     *cookie,
    IN  DAT_COMPLETION_FLAGS completion_flags)
{
    DAT_RETURN retval;

    dapl_dbg_log(DAPL_DBG_TYPE_UTIL,
        "mw_unbind: rmr 0x%p, enter\n", rmr);

    retval = dapls_ib_mw_bind(rmr, lmr_context, ep, cookie,
        (DAT_VADDR)NULL, (DAT_VLEN)0, (DAT_MEM_PRIV_FLAGS)NULL,
        completion_flags);

    dapl_dbg_log(DAPL_DBG_TYPE_UTIL,
        "mw_unbind: rmr 0x%p, exit\n\n", rmr);

    return (retval);
}

/*
 * Processes async events and calls appropriate callbacks so that events
 * can be posted to the async evd.
 */
void
dapls_ib_async_callback(
    IN    DAPL_EVD        *async_evd,
    IN    ib_hca_handle_t     hca_handle,
    IN    ib_error_record_t   *event_ptr,
    IN    void        *context)
{
    DAPL_IA         *ia_ptr;
    DAPL_EP         *ep_ptr;
    DAPL_EVD        *evd_ptr;
    dapl_ib_async_event_t   *async_evp;

    ia_ptr = (DAPL_IA *)context;

    dapl_os_assert(event_ptr != NULL);
    async_evp = (dapl_ib_async_event_t *)event_ptr;

    switch (async_evp->ibae_type) {
    case IBT_ERROR_INVALID_REQUEST_CHAN:
    case IBT_ERROR_CATASTROPHIC_CHAN:
        /*
         * Walk the EPs to match this EP, then invoke the
         * routine when we have the EP we need
         */
        dapl_os_assert(!dapl_llist_is_empty(&ia_ptr->ep_list_head));
        dapl_os_lock(&ia_ptr->header.lock);

        ep_ptr = (DAPL_EP *)dapl_llist_next_entry(&ia_ptr->ep_list_head,
            NULL);
        while (ep_ptr != NULL) {
            if (ep_ptr ==
                (DAPL_EP *)(uintptr_t)async_evp->ibae_cookie) {
                break;
            }

            ep_ptr = (DAPL_EP *) dapl_llist_next_entry(
                &ia_ptr->ep_list_head,
                &ep_ptr->header.ia_list_entry);
        }

        dapl_os_unlock(&ia_ptr->header.lock);
        dapl_os_assert(ep_ptr != NULL);
        dapl_evd_qp_async_error_callback(hca_handle, NULL, event_ptr,
            (void *)ep_ptr);
        break;
    case IBT_ERROR_CQ:
        /*
         * Walk the EVDs to match this EVD, then invoke the
         * routine when we have the EVD we need
         */
        dapl_os_assert(!dapl_llist_is_empty(&ia_ptr->evd_list_head));
        dapl_os_lock(&ia_ptr->header.lock);

        evd_ptr = (DAPL_EVD *) dapl_llist_next_entry(
            &ia_ptr->evd_list_head, NULL);
        while (evd_ptr != NULL) {
            if (evd_ptr ==
                (DAPL_EVD *)(uintptr_t)async_evp->ibae_cookie) {
                break;
            }
            evd_ptr = (DAPL_EVD *)
                dapl_llist_next_entry(&ia_ptr->evd_list_head,
                &evd_ptr->header.ia_list_entry);
        }
        dapl_os_unlock(&ia_ptr->header.lock);
        dapl_os_assert(evd_ptr != NULL);
        dapl_evd_cq_async_error_callback(hca_handle, NULL, event_ptr,
            (void *)evd_ptr);
        break;
    case IBT_ERROR_PORT_DOWN:
    case IBT_ERROR_LOCAL_CATASTROPHIC:
        dapl_evd_un_async_error_callback(hca_handle, event_ptr,
            (void *)async_evd);
        break;
    default:
        /*
         * We are not interested in the following events
         * case IBT_EVENT_PATH_MIGRATED:
         * case IBT_EVENT_COM_EST:
         * case IBT_EVENT_SQD:
         * case IBT_ERROR_PATH_MIGRATE_REQ:
         * case IBT_EVENT_PORT_UP:
         */
        dapl_dbg_log(DAPL_DBG_TYPE_UTIL,
            "dapls_ib_async_callback: unhandled async code:%x\n",
            async_evp->ibae_type);
        break;
    }
}

/*
 * dapls_ib_setup_async_callback
 * The reference implementation calls this to register callbacks,
 * but since our model of polling for events is based on retrieving
 * events by the waiting thread itself this is a NOOP for us.
 */
/* ARGSUSED */
DAT_RETURN
dapls_ib_setup_async_callback(
    IN  DAPL_IA         *ia_ptr,
    IN  DAPL_ASYNC_HANDLER_TYPE handler_type,
    IN  unsigned int        *callback_handle,
    IN  ib_async_handler_t      callback,
    IN  void            *context)
{
    return (DAT_SUCCESS);
}

/*
 * dapls_ib_query_hca
 *
 * Set up an asynchronous callbacks of various kinds
 *
 * Input:
 *  hca_handl       hca handle
 *  ep_attr         attribute of the ep
 *
 * Output:
 *  none
 *
 * Returns:
 *  DAT_SUCCESS
 *  DAT_INVALID_PARAMETER
 */

/* these are just arbitrary values for now */

static DAT_RETURN
dapls_ib_query_ia(
    IN  dapl_hca_attr_t     *hca_attr,
    IN  DAPL_HCA            *hca_ptr,
    OUT DAT_IA_ATTR         *ia_attr)
{
    (void) dapl_os_memzero(ia_attr, sizeof (*ia_attr));

    (void) dapl_os_strcpy(ia_attr->adapter_name, VN_ADAPTER_NAME);

    (void) sprintf(ia_attr->vendor_name, "0x%08x:0x%08x",
        hca_attr->dhca_vendor_id, hca_attr->dhca_device_id);

    ia_attr->hardware_version_major = (DAT_UINT32)hca_attr->dhca_version_id;
    ia_attr->ia_address_ptr = (DAT_IA_ADDRESS_PTR)&hca_ptr->hca_address;
    ia_attr->max_eps = (DAT_COUNT)hca_attr->dhca_max_chans;
    ia_attr->max_dto_per_ep = (DAT_COUNT)hca_attr->dhca_max_chan_sz;
    ia_attr->max_rdma_read_per_ep_in = hca_attr->dhca_max_rdma_in_chan;
    ia_attr->max_rdma_read_per_ep_out = hca_attr->dhca_max_rdma_out_chan;
    ia_attr->max_evds = (DAT_COUNT)hca_attr->dhca_max_cq;
    ia_attr->max_evd_qlen = (DAT_COUNT)hca_attr->dhca_max_cq_sz;
    /* max_iov_segments_per_dto is for non-RDMA */
    ia_attr->max_iov_segments_per_dto = (DAT_COUNT)hca_attr->dhca_max_sgl;
    ia_attr->max_lmrs = (DAT_COUNT)hca_attr->dhca_max_memr;
    ia_attr->max_lmr_block_size = (DAT_VLEN)hca_attr->dhca_max_memr_len;
    ia_attr->max_lmr_virtual_address = (DAT_VADDR)DAPL_MAX_ADDRESS;
    ia_attr->max_pzs = (DAT_COUNT)hca_attr->dhca_max_pd;
    ia_attr->max_mtu_size = (DAT_VLEN)DAPL_IB_MAX_MESSAGE_SIZE;
    ia_attr->max_rdma_size = (DAT_VLEN)DAPL_IB_MAX_MESSAGE_SIZE;
    ia_attr->max_rmrs = (DAT_COUNT)hca_attr->dhca_max_mem_win;
    ia_attr->max_rmr_target_address = (DAT_VADDR)DAPL_MAX_ADDRESS;
    ia_attr->max_iov_segments_per_rdma_read =
        (DAT_COUNT)hca_attr->dhca_max_sgl;
    ia_attr->max_iov_segments_per_rdma_write =
        (DAT_COUNT)hca_attr->dhca_max_sgl;
    /* all instances of IA */
    ia_attr->max_rdma_read_in = hca_attr->dhca_max_rdma_in_chan *
        hca_attr->dhca_max_chans;
    ia_attr->max_rdma_read_out = hca_attr->dhca_max_rdma_out_chan *
        hca_attr->dhca_max_chans;
    ia_attr->max_rdma_read_per_ep_in_guaranteed = DAT_TRUE;
    ia_attr->max_rdma_read_per_ep_out_guaranteed = DAT_TRUE;
    ia_attr->max_srqs = (DAT_COUNT)hca_attr->dhca_max_srqs;
    ia_attr->max_ep_per_srq = ia_attr->max_eps;
    ia_attr->max_recv_per_srq = (DAT_COUNT)hca_attr->dhca_max_srqs_sz;

    dapl_dbg_log(DAPL_DBG_TYPE_UTIL, "IA Attributes:\n"
        "\tadapter_name %s\n "
        "\tvendor_name %s\n "
        "\thardware_version_major 0x%08x\n"
        "\tmax_eps %d\n"
        "\tmax_dto_per_ep %d\n"
        "\tmax_rdma_read_per_ep_in %d\n"
        "\tmax_rdma_read_per_ep_out %d\n"
        "\tmax_evds %d\n"
        "\tmax_evd_qlen %d\n"
        "\tmax_iov_segments_per_dto %d\n"
        "\tmax_lmrs %d\n"
        "\tmax_lmr_block_size 0x%016llx\n"
        "\tmax_lmr_virtual_address 0x%016llx\n"
        "\tmax_pzs %d\n"
        "\tmax_mtu_size 0x%016llx\n"
        "\tmax_rdma_size 0x%016llx\n"
        "\tmax_rmrs %d\n"
        "\tmax_rmr_target_address 0x%016llx\n"
        "\tmax_iov_segments_per_rdma_read %d\n"
        "\tmax_iov_segments_per_rdma_write %d\n"
        "\tmax_rdma_read_in %d\n"
        "\tmax_rdma_read_out %d\n"
        "\tmax_srqs %d\n"
        "\tmax_ep_per_srq %d\n"
        "\tmax_recv_per_srq %d\n"
        "\n",
        ia_attr->adapter_name,
        ia_attr->vendor_name,
        ia_attr->hardware_version_major,
        ia_attr->max_eps,
        ia_attr->max_dto_per_ep,
        ia_attr->max_rdma_read_per_ep_in,
        ia_attr->max_rdma_read_per_ep_out,
        ia_attr->max_evds,
        ia_attr->max_evd_qlen,
        ia_attr->max_iov_segments_per_dto,
        ia_attr->max_lmrs,
        ia_attr->max_lmr_block_size,
        ia_attr->max_lmr_virtual_address,
        ia_attr->max_pzs,
        ia_attr->max_mtu_size,
        ia_attr->max_rdma_size,
        ia_attr->max_rmrs,
        ia_attr->max_rmr_target_address,
        ia_attr->max_iov_segments_per_rdma_read,
        ia_attr->max_iov_segments_per_rdma_write,
        ia_attr->max_rdma_read_in,
        ia_attr->max_rdma_read_out,
        ia_attr->max_srqs,
        ia_attr->max_ep_per_srq,
        ia_attr->max_recv_per_srq);

    return (DAT_SUCCESS);
}

/* ARGSUSED */
static DAT_RETURN
dapls_ib_query_ep(
    IN  dapl_hca_attr_t     *hca_attr,
    IN  DAPL_HCA            *hca_ptr,
    OUT DAT_EP_ATTR         *ep_attr)
{
    (void) dapl_os_memzero(ep_attr, sizeof (*ep_attr));
    ep_attr->service_type = DAT_SERVICE_TYPE_RC;
    ep_attr->max_mtu_size = DAPL_IB_MAX_MESSAGE_SIZE;
    ep_attr->max_rdma_size = DAPL_IB_MAX_MESSAGE_SIZE;
    ep_attr->qos = DAT_QOS_BEST_EFFORT;
    ep_attr->max_recv_dtos = hca_attr->dhca_max_chan_sz;
    ep_attr->max_request_dtos = hca_attr->dhca_max_chan_sz;
    ep_attr->max_recv_iov = hca_attr->dhca_max_sgl;
    ep_attr->max_request_iov = hca_attr->dhca_max_sgl;
    ep_attr->request_completion_flags = DAT_COMPLETION_DEFAULT_FLAG;
    ep_attr->recv_completion_flags = DAT_COMPLETION_DEFAULT_FLAG;
    ep_attr->srq_soft_hw = DAT_HW_DEFAULT;
    return (DAT_SUCCESS);
}

static void
dapls_ib_query_srq(
    IN  dapl_hca_attr_t     *hca_attr,
    OUT DAT_SRQ_ATTR        *srq_attr)
{
    (void) dapl_os_memzero(srq_attr, sizeof (*srq_attr));
    srq_attr->max_recv_dtos = hca_attr->dhca_max_srqs_sz;
    srq_attr->max_recv_iov = hca_attr->dhca_max_srq_sgl;
    srq_attr->low_watermark = DAT_SRQ_LW_DEFAULT;
}

/* ARGSUSED */
DAT_RETURN
dapls_ib_query_hca(
    IN  DAPL_HCA            *hca_ptr,
    OUT DAT_IA_ATTR         *ia_attr,
    OUT DAT_EP_ATTR         *ep_attr,
    OUT DAT_SOCK_ADDR6      *ip_addr,
    OUT DAT_SRQ_ATTR        *srq_attr)
{
    dapl_ia_query_t args;
    int ia_fd, retval;

    if (hca_ptr == NULL) {
        return (DAT_INVALID_PARAMETER);
    }

    ia_fd = hca_ptr->ib_hca_handle->ia_fd;
    retval = ioctl(ia_fd, DAPL_IA_QUERY, &args);
    if (retval != 0) {
        return (dapls_convert_error(errno, retval));
    }

    if (ia_attr != NULL) {
        (void) dapls_ib_query_ia(&args.hca_attr, hca_ptr, ia_attr);
    }
    if (ep_attr != NULL) {
        (void) dapls_ib_query_ep(&args.hca_attr, hca_ptr, ep_attr);
    }
    if (srq_attr != NULL) {
        (void) dapls_ib_query_srq(&args.hca_attr, srq_attr);
    }
    if (ia_attr == NULL && ep_attr == NULL && srq_attr == NULL) {
        return (DAT_INVALID_PARAMETER);
    }
    return (DAT_SUCCESS);
}

void
dapls_ib_store_premature_events(
    IN ib_qp_handle_t   qp_ptr,
    IN ib_work_completion_t *cqe_ptr)
{
    ib_srq_handle_t srqp;
    int     head;

    if (qp_ptr->qp_srq_enabled) {
        /*
         * For QPs with SRQ attached store the premature event in the
         * SRQ's premature event list
         */
        srqp = qp_ptr->qp_srq;
        dapl_os_assert(srqp->srq_freepr_num_events > 0);
        head = srqp->srq_freepr_events[srqp->srq_freepr_head];
        /*
         * mark cqe as valid before storing it in the
         * premature events list
         */
        DAPL_SET_CQE_VALID(cqe_ptr);
        (void) dapl_os_memcpy(&(srqp->srq_premature_events[head]),
            cqe_ptr, sizeof (*cqe_ptr));
        srqp->srq_freepr_head = (srqp->srq_freepr_head + 1) %
            srqp->srq_wq_numwqe;
        srqp->srq_freepr_num_events--;
    } else {
        (void) dapl_os_memcpy(&(qp_ptr->qp_premature_events[
            qp_ptr->qp_num_premature_events]),
            cqe_ptr, sizeof (*cqe_ptr));
    }
    qp_ptr->qp_num_premature_events++;
}

void
dapls_ib_poll_premature_events(
    IN  DAPL_EP         *ep_ptr,
    OUT ib_work_completion_t    **cqe_ptr,
    OUT int             *nevents)
{
    ib_qp_handle_t qp = ep_ptr->qp_handle;

    if (qp->qp_srq_enabled) {
        *cqe_ptr = qp->qp_srq->srq_premature_events;
    } else {
        *cqe_ptr = qp->qp_premature_events;
    }

    *nevents = qp->qp_num_premature_events;
    qp->qp_num_premature_events = 0;
}

/*
 * Return the premature events to the free list after processing it
 * This function is called only for premature events on the SRQ
 */
void
dapls_ib_free_premature_events(
    IN  DAPL_EP *ep_ptr,
    IN  int     free_index)
{
    ib_qp_handle_t  qp_ptr;
    ib_srq_handle_t srq_ptr;
    int     tail;

    qp_ptr = ep_ptr->qp_handle;
    srq_ptr = qp_ptr->qp_srq;

    dapl_os_assert(qp_ptr->qp_srq_enabled);

    tail = srq_ptr->srq_freepr_tail;
    srq_ptr->srq_freepr_events[tail] = free_index;
    srq_ptr->srq_freepr_tail = (tail + 1) % srq_ptr->srq_wq_numwqe;
    srq_ptr->srq_freepr_num_events++;
    DAPL_SET_CQE_INVALID(&srq_ptr->srq_premature_events[free_index]);
}

/*
 * dapls_ib_get_async_event
 *
 * Translate an asynchronous event type to the DAT event.
 * Note that different providers have different sets of errors.
 *
 * Input:
 *  cause_ptr       provider event cause
 *
 * Output:
 *  async_event     DAT mapping of error
 *
 * Returns:
 *  DAT_SUCCESS
 *  DAT_NOT_IMPLEMENTED Caller is not interested this event
 */

DAT_RETURN dapls_ib_get_async_event(
    IN  ib_error_record_t       *cause_ptr,
    OUT DAT_EVENT_NUMBER        *async_event)
{
    ibt_async_code_t    code;
    DAT_RETURN      dat_status;

    dat_status = DAT_SUCCESS;
    code = (ibt_async_code_t)((dapl_ib_async_event_t *)cause_ptr->
        ibae_type);

    switch (code) {
    case IBT_ERROR_CQ:
    case IBT_ERROR_ACCESS_VIOLATION_CHAN:
    case IBT_ERROR_INVALID_REQUEST_CHAN:
        *async_event = DAT_ASYNC_ERROR_PROVIDER_INTERNAL_ERROR;
        break;
    /* CATASTROPHIC errors */
    case IBT_ERROR_CATASTROPHIC_CHAN:
    case IBT_ERROR_LOCAL_CATASTROPHIC:
    case IBT_ERROR_PORT_DOWN:
        *async_event = DAT_ASYNC_ERROR_IA_CATASTROPHIC;
        break;
    default:
        /*
         * Errors we are not interested in reporting:
         * IBT_EVENT_PATH_MIGRATED
         * IBT_ERROR_PATH_MIGRATE_REQ
         * IBT_EVENT_COM_EST
         * IBT_EVENT_SQD
         * IBT_EVENT_PORT_UP
         */
        dat_status = DAT_NOT_IMPLEMENTED;
    }
    return (dat_status);
}

DAT_RETURN
dapls_ib_event_poll(
    IN DAPL_EVD     *evd_ptr,
    IN uint64_t     timeout,
    IN uint_t       threshold,
    OUT dapl_ib_event_t *evp_ptr,
    OUT int         *num_events)
{
    dapl_event_poll_t   evp_msg;
    int         ia_fd;
    int         retval;

    *num_events = 0;
    ia_fd = evd_ptr->header.owner_ia->hca_ptr->ib_hca_handle->ia_fd;

    evp_msg.evp_evd_hkey = evd_ptr->ib_cq_handle->evd_hkey;
    evp_msg.evp_threshold = threshold;
    evp_msg.evp_timeout = timeout;
    evp_msg.evp_ep = evp_ptr;
    if (evp_ptr) {
        evp_msg.evp_num_ev =
            DAPL_MAX(evd_ptr->threshold, NUM_EVENTS_PER_POLL);
    } else {
        evp_msg.evp_num_ev = 0;
    }
    evp_msg.evp_num_polled = 0;

    dapl_dbg_log(DAPL_DBG_TYPE_UTIL,
        "event_poll: evd 0x%p, hkey 0x%llx, threshold %d,\n"
        "            timeout 0x%llx, evp_ptr 0x%p, num_ev %d\n",
        evd_ptr, evp_msg.evp_evd_hkey, evp_msg.evp_threshold,
        timeout, evp_ptr, evp_msg.evp_num_ev);

    /*
     * Poll the EVD and if there are no events then we wait in
     * the kernel.
     */
    retval = ioctl(ia_fd, DAPL_EVENT_POLL, &evp_msg);
    if (retval != 0) {
        dapl_dbg_log(DAPL_DBG_TYPE_EVD,
            "event_poll: evd 0x%p, retval %d err: %s\n",
            evd_ptr, retval, strerror(errno));
        *num_events = evp_msg.evp_num_polled;
        return (dapls_convert_error(errno, retval));
    }

    dapl_dbg_log(DAPL_DBG_TYPE_EVD,
        "dapls_ib_event_poll: evd %p nevents %d\n", evd_ptr,
        evp_msg.evp_num_polled);

    *num_events = evp_msg.evp_num_polled;

    return (DAT_SUCCESS);
}

DAT_RETURN
dapls_ib_event_wakeup(
    IN DAPL_EVD     *evd_ptr)
{
    dapl_event_wakeup_t evw_msg;
    int         ia_fd;
    int         retval;

    ia_fd = evd_ptr->header.owner_ia->hca_ptr->ib_hca_handle->ia_fd;

    evw_msg.evw_hkey = evd_ptr->ib_cq_handle->evd_hkey;

    dapl_dbg_log(DAPL_DBG_TYPE_UTIL,
        "event_wakeup: evd 0x%p, hkey 0x%llx\n",
        evd_ptr, evw_msg.evw_hkey);

    /*
     * Wakeup any thread waiting in the kernel on this EVD
     */
    retval = ioctl(ia_fd, DAPL_EVENT_WAKEUP, &evw_msg);
    if (retval != 0) {
        dapl_dbg_log(DAPL_DBG_TYPE_EVD,
            "event_wakeup: evd 0x%p, retval %d err: %s\n",
            evd_ptr, retval, strerror(errno));
        return (dapls_convert_error(errno, retval));
    }

    return (DAT_SUCCESS);
}

/*
 * dapls_ib_cq_peek is used by dapl_cno_wait(). After the CQ has been
 * inspected we arm the CQ if it was empty.
 *
 */
void dapls_ib_cq_peek(
    IN DAPL_EVD *evd_ptr,
    OUT int     *num_cqe)
{
    DAPL_IA     *ia_ptr;

    *num_cqe = 0;
    if (evd_ptr->evd_flags & (DAT_EVD_DTO_FLAG | DAT_EVD_RMR_BIND_FLAG)) {
        DAPL_PEEK(evd_ptr)(evd_ptr->ib_cq_handle, num_cqe);
        /* No events found in CQ arm it now */
        if (*num_cqe == 0) {
            ia_ptr = evd_ptr->header.owner_ia;
            (void) dapls_set_cq_notify(ia_ptr, evd_ptr);
            dapl_dbg_log(DAPL_DBG_TYPE_EVD,
                "dapls_ib_cq_peek: set_cq_notify\n");
        }
    }
}

/*
 * Modifies the CNO associated to an EVD
 */
DAT_RETURN dapls_ib_modify_cno(
    IN DAPL_EVD *evd_ptr,
    IN DAPL_CNO *cno_ptr)
{
    dapl_evd_modify_cno_t   evmc_msg;
    int         ia_fd;
    int         retval;

    ia_fd = evd_ptr->header.owner_ia->hca_ptr->ib_hca_handle->ia_fd;

    evmc_msg.evmc_hkey = evd_ptr->ib_cq_handle->evd_hkey;

    if (cno_ptr) {
        evmc_msg.evmc_cno_hkey = (uint64_t)cno_ptr->ib_cno_handle;
    } else {
        evmc_msg.evmc_cno_hkey = 0;
    }

    dapl_dbg_log(DAPL_DBG_TYPE_UTIL,
        "modify_cno: evd 0x%p, hkey 0x%llx, cno 0x%p, cno_hkey 0x%llx\n",
        evd_ptr, evmc_msg.evmc_hkey, cno_ptr, evmc_msg.evmc_cno_hkey);

    /*
     * modify CNO associated with the EVD
     */
    retval = ioctl(ia_fd, DAPL_EVD_MODIFY_CNO, &evmc_msg);
    if (retval != 0) {
        dapl_dbg_log(DAPL_DBG_TYPE_EVD,
            "modify_cno: evd 0x%p, cno %p retval %d err: %s\n",
            evd_ptr, cno_ptr, retval, strerror(errno));
        return (dapls_convert_error(errno, retval));
    }

    return (DAT_SUCCESS);
}

DAT_RETURN
dapls_ib_cno_wait(
    IN DAPL_CNO *cno_ptr,
    IN DAT_TIMEOUT  timeout,
    IN DAPL_EVD **evd_ptr_p)
{
    dapl_cno_wait_t     args;
    int         retval;

    args.cnw_hkey = (uint64_t)cno_ptr->ib_cno_handle;
    if (timeout == DAT_TIMEOUT_INFINITE) {
        args.cnw_timeout = UINT64_MAX;
    } else {
        args.cnw_timeout = (uint64_t)timeout & 0x00000000ffffffff;
    }

    dapl_dbg_log(DAPL_DBG_TYPE_UTIL,
        "cno_wait: cno 0x%p, hkey 0x%016llx, timeout 0x%016llx\n",
        cno_ptr, args.cnw_hkey, args.cnw_timeout);

    retval = ioctl(cno_ptr->header.owner_ia->hca_ptr->
        ib_hca_handle->ia_fd, DAPL_CNO_WAIT, &args);

    if (retval != 0) {
        *evd_ptr_p = (DAPL_EVD *)NULL;
        dapl_dbg_log(DAPL_DBG_TYPE_ERR,
            "cno_wait: cno 0x%p ioctl err: %s\n",
            cno_ptr, strerror(errno));
        return (dapls_convert_error(errno, retval));
    }

    *evd_ptr_p = (DAPL_EVD *)(uintptr_t)args.cnw_evd_cookie;
    dapl_dbg_log(DAPL_DBG_TYPE_UTIL,
        "cno_wait: woken up, cno 0x%p, evd 0x%p\n\n",
        cno_ptr, *evd_ptr_p);

    return (DAT_SUCCESS);
}

DAT_RETURN
dapls_ib_cno_alloc(
    IN DAPL_IA  *ia_ptr,
    IN DAPL_CNO *cno_ptr)
{
    dapl_cno_alloc_t    args;
    int         retval;

    if (cno_ptr->cno_wait_agent.instance_data != NULL ||
        cno_ptr->cno_wait_agent.proxy_agent_func != NULL) {
        dapl_dbg_log(DAPL_DBG_TYPE_ERR,
            "cno_alloc: cno 0x%p, wait_agent != NULL\n", cno_ptr);
        return (DAT_NOT_IMPLEMENTED);
    }

    retval = ioctl(ia_ptr->hca_ptr->ib_hca_handle->ia_fd,
        DAPL_CNO_ALLOC, &args);
    if (retval != 0) {
        dapl_dbg_log(DAPL_DBG_TYPE_ERR,
            "cno_alloc: cno 0x%p ioctl err: %s\n",
            cno_ptr, strerror(errno));
        return (dapls_convert_error(errno, retval));
    }

    cno_ptr->ib_cno_handle = (ib_cno_handle_t)args.cno_hkey;
    dapl_dbg_log(DAPL_DBG_TYPE_UTIL,
        "cno_alloc: cno 0x%p allocated, ia_ptr 0x%p, hkey 0x%016llx\n",
        cno_ptr, ia_ptr, args.cno_hkey);

    return (DAT_SUCCESS);
}

DAT_RETURN
dapls_ib_cno_free(
    IN DAPL_CNO *cno_ptr)
{
    dapl_cno_free_t     args;
    int         retval;

    args.cnf_hkey = (uint64_t)cno_ptr->ib_cno_handle;
    dapl_dbg_log(DAPL_DBG_TYPE_UTIL,
        "cno_free: cno 0x%p, hkey 0x%016llx\n",
        cno_ptr, args.cnf_hkey);

    retval = ioctl(cno_ptr->header.owner_ia->hca_ptr->
        ib_hca_handle->ia_fd, DAPL_CNO_FREE, &args);

    if (retval != 0) {
        dapl_dbg_log(DAPL_DBG_TYPE_ERR,
            "cno_free: cno 0x%p ioctl err: %s\n",
            cno_ptr, strerror(errno));
        return (dapls_convert_error(errno, retval));
    }

    dapl_dbg_log(DAPL_DBG_TYPE_UTIL,
        "cno_free: cno 0x%p freed\n", cno_ptr);

    return (DAT_SUCCESS);
}

DAT_RETURN
dapls_convert_error(int errnum, int retval)
{
    if (retval < 0) {
        switch (errnum) {
        case EINVAL:
            return (DAT_INVALID_PARAMETER);
        case ENOMEM:
            return (DAT_INSUFFICIENT_RESOURCES);
        case ETIME:
            return (DAT_TIMEOUT_EXPIRED);
        case EINTR:
            return (DAT_INTERRUPTED_CALL);
        case EFAULT:
            return (DAT_INTERNAL_ERROR);
        default:
            return (DAT_INTERNAL_ERROR);
        }
    } else {
        dapl_dbg_log(DAPL_DBG_TYPE_ERR,
            "ERROR: got IBTF error %d\n", retval);
        switch (retval) {
        case IBT_SERVICE_RECORDS_NOT_FOUND:
            /*
             * Connecting to a non-existant conn qual gets
             * us here
             */
            return (DAT_ERROR(DAT_INVALID_PARAMETER,
                DAT_INVALID_ADDRESS_UNREACHABLE));
        case IBT_INSUFF_RESOURCE:
        case IBT_INSUFF_KERNEL_RESOURCE:
            return (DAT_ERROR(DAT_INSUFFICIENT_RESOURCES, 0));
        case IBT_AR_NOT_REGISTERED:
            /*
             * forward ipaddr lookup failed
             */
            return (DAT_ERROR(DAT_INVALID_ADDRESS, 0));
        default:
            return (DAT_INTERNAL_ERROR);
        }
    }
}

typedef struct dapls_ib_dbp_page_s {
    uint32_t            *dbp_page_addr;
    uint64_t            dbp_mapoffset;
    struct dapls_ib_dbp_page_s  *next;
    int             fd;
} dapls_ib_dbp_page_t;

dapls_ib_dbp_page_t *dapls_ib_pagelist = NULL;

/* Function that returns a pointer to the specified doorbell entry */
uint32_t *dapls_ib_get_dbp(uint64_t maplen, int fd, uint64_t mapoffset,
    uint32_t offset)
{
    dapls_ib_dbp_page_t *new_page;
    dapls_ib_dbp_page_t *cur_page;

    dapl_os_lock(&dapls_ib_dbp_lock);
    /* Check to see if page already mapped for entry */
    for (cur_page = dapls_ib_pagelist; cur_page != NULL;
        cur_page = cur_page->next)
        if (cur_page->dbp_mapoffset == mapoffset &&
            cur_page->fd == fd) {
            dapl_os_unlock(&dapls_ib_dbp_lock);
            return ((uint32_t *)
                (offset + (uintptr_t)cur_page->dbp_page_addr));
        }

    /* If not, map a new page and prepend to pagelist */
    new_page = malloc(sizeof (dapls_ib_dbp_page_t));
    if (new_page == NULL) {
        dapl_os_unlock(&dapls_ib_dbp_lock);
        return (MAP_FAILED);
    }
    new_page->dbp_page_addr = (uint32_t *)(void *)mmap64((void *)0,
        maplen, (PROT_READ | PROT_WRITE), MAP_SHARED, fd, mapoffset);
    if (new_page->dbp_page_addr == MAP_FAILED) {
        free(new_page);
        dapl_os_unlock(&dapls_ib_dbp_lock);
        return (MAP_FAILED);
    }
    new_page->next = dapls_ib_pagelist;
    new_page->dbp_mapoffset = mapoffset;
    new_page->fd = fd;
    dapls_ib_pagelist = new_page;
    dapl_os_unlock(&dapls_ib_dbp_lock);
    return ((uint32_t *)(offset + (uintptr_t)new_page->dbp_page_addr));
}