dapl_tavor_ibtf_dto.c revision 9e39c5ba00a55fa05777cc94b148296af305e135
/*
* CDDL HEADER START
*
* The contents of this file are subject to the terms of the
* Common Development and Distribution License (the "License").
* You may not use this file except in compliance with the License.
*
* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
* or http://www.opensolaris.org/os/licensing.
* See the License for the specific language governing permissions
* and limitations under the License.
*
* When distributing Covered Code, include this CDDL HEADER in each
* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
* If applicable, add the following below this CDDL HEADER, with the
* fields enclosed by brackets "[]" replaced with your own identifying
* information: Portions Copyright [yyyy] [name of copyright owner]
*
* CDDL HEADER END
*/
/*
* Copyright 2008 Sun Microsystems, Inc. All rights reserved.
* Use is subject to license terms.
*/
#include "dapl.h"
#include "dapl_adapter_util.h"
#include "dapl_lmr_util.h"
#include "dapl_rmr_util.h"
#include "dapl_cookie.h"
#include "dapl_tavor_ibtf_impl.h"
/*
*
*
* MODULE: dapl_tavor_ibtf_dto.c
*
* PURPOSE: Utility routines for data transfer operations
*
*/
/*
* dapls_ib_post_recv
*
* Provider specific Post RECV function
*/
DAT_RETURN
dapls_ib_post_recv(
IN DAPL_EP *ep_ptr,
IN DAPL_COOKIE *cookie,
IN DAT_COUNT num_segments,
IN DAT_LMR_TRIPLET *local_iov,
IN DAT_COMPLETION_FLAGS completion_flags)
{
ibt_recv_wr_t pr_wr;
ibt_wr_ds_t pr_sgl_arr[DAPL_MAX_IOV];
ibt_wr_ds_t *pr_sgl;
boolean_t suppress_notification;
DAT_COUNT total_len;
int retval;
int i;
total_len = 0;
if (ep_ptr->qp_handle == NULL) {
dapl_dbg_log(DAPL_DBG_TYPE_EP, "dapls_ib_post_recv: "
"qp_handle == NULL\n");
return (DAT_INVALID_PARAMETER);
}
/* allocate scatter-gather list on the heap if its large */
if (num_segments > DAPL_MAX_IOV) {
pr_sgl = dapl_os_alloc(num_segments * sizeof (ibt_wr_ds_t));
if (NULL == pr_sgl) {
dapl_dbg_log(DAPL_DBG_TYPE_ERR,
"dapls_ib_post_recv: pr_sgl alloc failed");
return (DAT_INSUFFICIENT_RESOURCES);
}
} else {
pr_sgl = pr_sgl_arr;
}
for (i = 0; i < num_segments; i++) {
pr_sgl[i].ds_va = (ib_vaddr_t)local_iov[i].virtual_address;
pr_sgl[i].ds_key = (ibt_lkey_t)local_iov[i].lmr_context;
pr_sgl[i].ds_len = (ib_msglen_t)local_iov[i].segment_length;
total_len += pr_sgl[i].ds_len;
dapl_dbg_log(DAPL_DBG_TYPE_EP, "dapls_ib_post_recv: "
"i(%d) va(%p), lmrctxt(0x%x), len(%llu)\n", i,
pr_sgl[i].ds_va, pr_sgl[i].ds_key, pr_sgl[i].ds_len);
}
if (cookie != NULL) {
cookie->val.dto.size = total_len;
dapl_dbg_log(DAPL_DBG_TYPE_EP,
"dapls_ib_post_recv: dto_cookie(%p), num_seg(%d), "
"size(%d) hkey(%016llx)\n", cookie, num_segments,
cookie->val.dto.size, ep_ptr->qp_handle->ep_hkey);
}
pr_wr.wr_id = (ibt_wrid_t)(uintptr_t)cookie;
pr_wr.wr_nds = (uint32_t)num_segments;
if (num_segments > 0) {
pr_wr.wr_sgl = &pr_sgl[0];
} else {
pr_wr.wr_sgl = NULL;
}
if (ep_ptr->param.ep_attr.recv_completion_flags &
DAT_COMPLETION_UNSIGNALLED_FLAG) {
/* This flag is used to control notification of completions */
suppress_notification = (completion_flags &
DAT_COMPLETION_UNSIGNALLED_FLAG) ? B_TRUE : B_FALSE;
} else {
/*
* The evd waiter will use threshold to control wakeups
* Hence the event notification will be done via arming the
* CQ so we do not need special notification generation
* hence set suppression to true
*/
suppress_notification = B_TRUE;
}
retval = DAPL_RECV(ep_ptr)(ep_ptr, &pr_wr, suppress_notification);
if (retval != 0) {
dapl_dbg_log(DAPL_DBG_TYPE_EP,
"dapls_ib_post_recv: post_recv failed %s\n",
strerror(errno));
}
/* free the pr_sgl if we had allocated it */
if (num_segments > DAPL_MAX_IOV) {
dapl_os_free(pr_sgl, num_segments*sizeof (ibt_wr_ds_t));
}
return (retval);
}
/*
* dapls_ib_post_recv_one
*
* Provider specific Post RECV function
*/
DAT_RETURN
dapls_ib_post_recv_one(
IN DAPL_EP *ep_ptr,
IN DAPL_COOKIE *cookie,
IN DAT_LMR_TRIPLET *local_iov)
{
ibt_recv_wr_t pr_wr;
ibt_wr_ds_t pr_sgl;
boolean_t suppress_notification;
DAT_COUNT total_len;
int retval;
if (ep_ptr->qp_handle == NULL) {
dapl_dbg_log(DAPL_DBG_TYPE_EP, "dapls_ib_post_recv_one: "
"qp_handle == NULL\n");
return (DAT_INVALID_PARAMETER);
}
pr_sgl.ds_va = (ib_vaddr_t)local_iov->virtual_address;
pr_sgl.ds_key = (ibt_lkey_t)local_iov->lmr_context;
pr_sgl.ds_len = (ib_msglen_t)local_iov->segment_length;
total_len = pr_sgl.ds_len;
dapl_dbg_log(DAPL_DBG_TYPE_EP, "dapls_ib_post_recv_one: "
"va(%p), lmrctxt(0x%x), len(%llu)\n",
pr_sgl.ds_va, pr_sgl.ds_key, pr_sgl.ds_len);
if (cookie != NULL) {
cookie->val.dto.size = total_len;
dapl_dbg_log(DAPL_DBG_TYPE_EP,
"dapls_ib_post_recv_one: dto_cookie(%p), num_seg(1), "
"size(%d) hkey(%016llx)\n", cookie,
cookie->val.dto.size, ep_ptr->qp_handle->ep_hkey);
}
pr_wr.wr_id = (ibt_wrid_t)(uintptr_t)cookie;
pr_wr.wr_nds = 1;
pr_wr.wr_sgl = &pr_sgl;
if (ep_ptr->param.ep_attr.recv_completion_flags &
DAT_COMPLETION_UNSIGNALLED_FLAG) {
/* This flag is used to control notification of completions */
suppress_notification = B_FALSE;
} else {
/*
* The evd waiter will use threshold to control wakeups
* Hence the event notification will be done via arming the
* CQ so we do not need special notification generation
* hence set suppression to true
*/
suppress_notification = B_TRUE;
}
retval = DAPL_RECV(ep_ptr)(ep_ptr, &pr_wr, suppress_notification);
if (retval != 0) {
dapl_dbg_log(DAPL_DBG_TYPE_EP,
"dapls_ib_post_recv_one: post_recv failed %s\n",
strerror(errno));
}
return (retval);
}
/*
* dapls_ib_srq_post_recv
*
* Provider specific SRQ Post RECV function
*/
DAT_RETURN
dapls_ib_post_srq(
IN DAPL_SRQ *srq_ptr,
IN DAPL_COOKIE *cookie,
IN DAT_COUNT num_segments,
IN DAT_LMR_TRIPLET *local_iov)
{
ibt_recv_wr_t pr_wr;
ibt_wr_ds_t pr_sgl_arr[DAPL_MAX_IOV];
ibt_wr_ds_t *pr_sgl;
DAT_COUNT total_len;
int retval;
int i;
total_len = 0;
if (srq_ptr->srq_handle == NULL) {
dapl_dbg_log(DAPL_DBG_TYPE_EP, "dapls_ib_post_srq: "
"srq_handle == NULL\n");
return (DAT_INVALID_PARAMETER);
}
/* allocate scatter-gather list on the heap if its large */
if (num_segments > DAPL_MAX_IOV) {
pr_sgl = dapl_os_alloc(num_segments * sizeof (ibt_wr_ds_t));
if (NULL == pr_sgl) {
dapl_dbg_log(DAPL_DBG_TYPE_ERR,
"dapls_ib_post_srq: pr_sgl alloc failed");
return (DAT_INSUFFICIENT_RESOURCES);
}
} else {
pr_sgl = pr_sgl_arr;
}
for (i = 0; i < num_segments; i++) {
pr_sgl[i].ds_va = (ib_vaddr_t)local_iov[i].virtual_address;
pr_sgl[i].ds_key = (ibt_lkey_t)local_iov[i].lmr_context;
pr_sgl[i].ds_len = (ib_msglen_t)local_iov[i].segment_length;
total_len += pr_sgl[i].ds_len;
dapl_dbg_log(DAPL_DBG_TYPE_EP, "dapls_ib_post_srq: "
"i(%d) va(%p), lmrctxt(0x%x), len(%u)\n", i,
pr_sgl[i].ds_va, pr_sgl[i].ds_key, pr_sgl[i].ds_len);
}
if (cookie != NULL) {
cookie->val.dto.size = total_len;
dapl_dbg_log(DAPL_DBG_TYPE_EP,
"dapls_ib_post_srq: dto_cookie(%p), num_seg(%d), "
"size(%d) hkey(%016llx)\n", cookie, num_segments,
cookie->val.dto.size, srq_ptr->srq_handle->srq_hkey);
}
pr_wr.wr_id = (ibt_wrid_t)(uintptr_t)cookie;
pr_wr.wr_nds = (uint32_t)num_segments;
if (num_segments > 0) {
pr_wr.wr_sgl = &pr_sgl[0];
} else {
pr_wr.wr_sgl = NULL;
}
retval = DAPL_SRECV(srq_ptr)(srq_ptr, &pr_wr, B_TRUE);
if (retval != 0) {
dapl_dbg_log(DAPL_DBG_TYPE_EP,
"dapls_ib_post_srq: post_recv failed %s\n",
strerror(errno));
}
/* free the pr_sgl if we had allocated it */
if (num_segments > DAPL_MAX_IOV) {
dapl_os_free(pr_sgl, num_segments*sizeof (ibt_wr_ds_t));
}
return (retval);
}
/*
* dapls_ib_post_send
*
* Provider specific Post SEND function
*/
DAT_RETURN
dapls_ib_post_send(IN DAPL_EP *ep_ptr,
IN ib_send_op_type_t op_type,
IN DAPL_COOKIE *cookie,
IN DAT_COUNT num_segments,
IN DAT_LMR_TRIPLET *local_iov,
IN const DAT_RMR_TRIPLET *remote_iov,
IN DAT_COMPLETION_FLAGS completion_flags)
{
ibt_send_wr_t ps_wr;
ibt_wr_ds_t ps_sgl_arr[DAPL_MAX_IOV];
ibt_wr_ds_t *ps_sgl;
DAT_COUNT total_len;
boolean_t suppress_notification;
int retval;
int i;
total_len = 0;
retval = DAT_SUCCESS;
if (ep_ptr->qp_handle == NULL) {
dapl_dbg_log(DAPL_DBG_TYPE_EP, "dapls_ib_post_send: "
"qp_handle == NULL\n");
return (DAT_INVALID_PARAMETER);
}
/* allocate scatter-gather list on the heap if its large */
if (num_segments > DAPL_MAX_IOV) {
ps_sgl = dapl_os_alloc(num_segments * sizeof (ibt_wr_ds_t));
if (NULL == ps_sgl) {
dapl_dbg_log(DAPL_DBG_TYPE_ERR,
"dapls_ib_post_send: pr_sgl alloc failed");
return (DAT_INSUFFICIENT_RESOURCES);
}
} else {
ps_sgl = ps_sgl_arr;
}
for (i = 0; i < num_segments; i++) {
ps_sgl[i].ds_va = (ib_vaddr_t)local_iov[i].virtual_address;
ps_sgl[i].ds_key = (ibt_lkey_t)local_iov[i].lmr_context;
ps_sgl[i].ds_len = (ib_msglen_t)local_iov[i].segment_length;
total_len += ps_sgl[i].ds_len;
dapl_dbg_log(DAPL_DBG_TYPE_EP, "dapls_ib_post_send: "
"i(%d), va(0x%llx), lmrctxt(0x%x), len(%u)\n",
i, ps_sgl[i].ds_va, ps_sgl[i].ds_key, ps_sgl[i].ds_len);
}
if (cookie != NULL) {
cookie->val.dto.size = total_len;
dapl_dbg_log(DAPL_DBG_TYPE_EVD,
"dapls_ib_post_send: op_type(%d), cookie(%p) "
"num_seg(%d) size(%d) hkey(%016llx)\n", op_type,
cookie, num_segments, cookie->val.dto.size,
ep_ptr->qp_handle->ep_hkey);
}
ps_wr.wr_id = (ibt_wrid_t)(uintptr_t)cookie;
/* Translate dapl flags */
ps_wr.wr_flags = (DAT_COMPLETION_BARRIER_FENCE_FLAG &
completion_flags) ? IBT_WR_SEND_FENCE : 0;
/* suppress completions */
ps_wr.wr_flags |= (DAT_COMPLETION_SUPPRESS_FLAG &
completion_flags) ? 0 : IBT_WR_SEND_SIGNAL;
/* Solicited wait flag is valid only for post_send */
if (op_type == OP_SEND) {
ps_wr.wr_flags |= (DAT_COMPLETION_SOLICITED_WAIT_FLAG &
completion_flags) ? IBT_WR_SEND_SOLICIT : 0;
}
ps_wr.wr_opcode = (ibt_wrc_opcode_t)op_type;
ps_wr.wr_nds = (uint32_t)num_segments;
if (num_segments > 0) {
ps_wr.wr_sgl = &ps_sgl[0];
if (op_type == OP_RDMA_READ || op_type == OP_RDMA_WRITE) {
if (remote_iov == NULL) {
/* free the ps_sgl if we had allocated it */
if (num_segments > DAPL_MAX_IOV) {
dapl_os_free(ps_sgl,
num_segments*sizeof (ibt_wr_ds_t));
}
dapl_dbg_log(DAPL_DBG_TYPE_EP,
"dapls_ib_post_send: "
"remote_iov == NULL\n");
return (DAT_INVALID_PARAMETER);
}
if (remote_iov->segment_length != (DAT_VLEN)total_len) {
/* free the ps_sgl if we had allocated it */
if (num_segments > DAPL_MAX_IOV) {
dapl_os_free(ps_sgl,
num_segments*sizeof (ibt_wr_ds_t));
}
dapl_dbg_log(DAPL_DBG_TYPE_EP,
"dapls_ib_post_send: "
"remote_iov length(%llu != %llu)\n",
(DAT_VLEN)total_len,
remote_iov->segment_length);
return (DAT_LENGTH_ERROR);
}
ps_wr.wr.rc.rcwr.rdma.rdma_raddr =
(ib_vaddr_t)remote_iov->target_address;
ps_wr.wr.rc.rcwr.rdma.rdma_rkey =
(ibt_rkey_t)remote_iov->rmr_context;
dapl_dbg_log(DAPL_DBG_TYPE_EP,
"dapls_ib_post_send: remote_iov taddr(0x%llx), "
"rmr(0x%x)\n", remote_iov->target_address,
remote_iov->rmr_context);
}
} else {
ps_wr.wr_sgl = NULL;
}
if (ep_ptr->param.ep_attr.recv_completion_flags &
DAT_COMPLETION_UNSIGNALLED_FLAG) {
/* This flag is used to control notification of completions */
suppress_notification = (completion_flags &
DAT_COMPLETION_UNSIGNALLED_FLAG) ? B_TRUE : B_FALSE;
} else {
/*
* The evd waiter will use threshold to control wakeups
* Hence the event notification will be done via arming the
* CQ so we do not need special notification generation
* hence set suppression to true
*/
suppress_notification = B_TRUE;
}
retval = DAPL_SEND(ep_ptr)(ep_ptr, &ps_wr, suppress_notification);
if (retval != 0) {
dapl_dbg_log(DAPL_DBG_TYPE_EP,
"dapls_ib_post_send: post_send failed %d\n", retval);
}
/* free the pr_sgl if we had allocated it */
if (num_segments > DAPL_MAX_IOV) {
dapl_os_free(ps_sgl, num_segments*sizeof (ibt_wr_ds_t));
}
return (retval);
}
/*
* dapls_ib_post_send_one
*
* Provider specific Post SEND function - special case for the common case of
* sgl num_segments == 1 and completion_flags == DAT_COMPLETION_DEFAULT_FLAG.
*/
DAT_RETURN
dapls_ib_post_send_one(IN DAPL_EP *ep_ptr,
IN ib_send_op_type_t op_type,
IN DAPL_COOKIE *cookie,
IN DAT_LMR_TRIPLET *local_iov,
IN const DAT_RMR_TRIPLET *remote_iov)
{
ibt_send_wr_t ps_wr;
ibt_wr_ds_t ps_sgl;
boolean_t suppress_notification;
int retval;
if (ep_ptr->qp_handle == NULL) {
dapl_dbg_log(DAPL_DBG_TYPE_EP, "dapls_ib_post_send_one: "
"qp_handle == NULL\n");
return (DAT_INVALID_PARAMETER);
}
ps_sgl.ds_va = (ib_vaddr_t)local_iov[0].virtual_address;
ps_sgl.ds_key = (ibt_lkey_t)local_iov[0].lmr_context;
ps_sgl.ds_len = (ib_msglen_t)local_iov[0].segment_length;
dapl_dbg_log(DAPL_DBG_TYPE_EP, "dapls_ib_post_send_one: "
"i(%d), va(0x%llx), lmrctxt(0x%x), len(%u)\n",
0, ps_sgl.ds_va, ps_sgl.ds_key, ps_sgl.ds_len);
cookie->val.dto.size = ps_sgl.ds_len;
dapl_dbg_log(DAPL_DBG_TYPE_EVD,
"dapls_ib_post_send_one: op_type(%d), cookie(%p) "
"num_seg(%d) size(%d) hkey(%016llx)\n", op_type,
cookie, 1, cookie->val.dto.size,
ep_ptr->qp_handle->ep_hkey);
ps_wr.wr_id = (ibt_wrid_t)(uintptr_t)cookie;
/* suppress completions */
ps_wr.wr_flags = IBT_WR_SEND_SIGNAL;
ps_wr.wr_opcode = (ibt_wrc_opcode_t)op_type;
ps_wr.wr_nds = 1;
ps_wr.wr_sgl = &ps_sgl;
if (op_type == OP_RDMA_READ || op_type == OP_RDMA_WRITE) {
if (remote_iov == NULL) {
/* free the ps_sgl if we had allocated it */
dapl_dbg_log(DAPL_DBG_TYPE_EP,
"dapls_ib_post_send_one: "
"remote_iov == NULL\n");
return (DAT_INVALID_PARAMETER);
}
if (remote_iov->segment_length != (DAT_VLEN)ps_sgl.ds_len) {
dapl_dbg_log(DAPL_DBG_TYPE_EP,
"dapls_ib_post_send_one: "
"remote_iov length(%llu != %llu)\n",
(DAT_VLEN)ps_sgl.ds_len,
remote_iov->segment_length);
return (DAT_LENGTH_ERROR);
}
ps_wr.wr.rc.rcwr.rdma.rdma_raddr =
(ib_vaddr_t)remote_iov->target_address;
ps_wr.wr.rc.rcwr.rdma.rdma_rkey =
(ibt_rkey_t)remote_iov->rmr_context;
dapl_dbg_log(DAPL_DBG_TYPE_EP,
"dapls_ib_post_send_one: remote_iov taddr(0x%llx), "
"rmr(0x%x)\n", remote_iov->target_address,
remote_iov->rmr_context);
}
if (ep_ptr->param.ep_attr.recv_completion_flags &
DAT_COMPLETION_UNSIGNALLED_FLAG) {
/* This flag is used to control notification of completions */
suppress_notification = B_FALSE;
} else {
/*
* The evd waiter will use threshold to control wakeups
* Hence the event notification will be done via arming the
* CQ so we do not need special notification generation
* hence set suppression to true
*/
suppress_notification = B_TRUE;
}
retval = DAPL_SEND(ep_ptr)(ep_ptr, &ps_wr, suppress_notification);
if (retval != 0) {
dapl_dbg_log(DAPL_DBG_TYPE_EP,
"dapls_ib_post_send_one: post_send failed %d\n", retval);
}
return (retval);
}