9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * CDDL HEADER START
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * The contents of this file are subject to the terms of the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Common Development and Distribution License (the "License").
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * You may not use this file except in compliance with the License.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * See the License for the specific language governing permissions
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * and limitations under the License.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * When distributing Covered Code, include this CDDL HEADER in each
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * If applicable, add the following below this CDDL HEADER, with the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * fields enclosed by brackets "[]" replaced with your own identifying
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * information: Portions Copyright [yyyy] [name of copyright owner]
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * CDDL HEADER END
1ed53a3f65abecaadc1b967e341970ad0f6b2aebBill Taylor * Copyright 2010 Sun Microsystems, Inc. All rights reserved.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Use is subject to license terms.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor/* Function prototypes */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorextern DAT_RETURN dapls_tavor_wrid_init(ib_qp_handle_t);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorextern DAT_RETURN dapls_tavor_srq_wrid_init(ib_srq_handle_t);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorextern void dapls_tavor_srq_wrid_free(ib_srq_handle_t);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorextern DAT_BOOLEAN dapls_tavor_srq_wrid_resize(ib_srq_handle_t, uint32_t);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic DAT_RETURN dapli_ib_srq_add_ep(IN ib_srq_handle_t srq_ptr,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic void dapli_ib_srq_remove_ep(IN ib_srq_handle_t srq_ptr,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic DAT_RETURN dapli_ib_srq_resize_internal(IN DAPL_SRQ *srq_ptr,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * dapli_get_dto_cq
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Obtain the cq_handle for a DTO EVD. If the EVD is NULL, use the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * null_ib_cq_handle. If it hasn't been created yet, create it now in
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the HCA structure. It will be cleaned up in dapls_ib_cqd_destroy().
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * This is strictly internal to IB. DAPL allows a NULL DTO EVD handle,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * but IB does not. So we create a CQ under the hood and make sure
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * an error is generated if the user every tries to post, by
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * setting the WQ length to 0 in ep_create and/or ep_modify.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * A valid CQ handle
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor } else if (ia_ptr->hca_ptr->null_ib_cq_handle != IB_INVALID_HANDLE) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor dapl_os_alloc(sizeof (struct dapls_ib_cq_handle));
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor "dapli_get_dto_cq: cq malloc failed\n");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ia_ptr->hca_ptr->null_ib_cq_handle = IB_INVALID_HANDLE;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * create a fake a CQ, we don't bother to mmap this CQ
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * since nobody know about it to reap events from it.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor (void) dapl_os_memzero(&create_msg, sizeof (create_msg));
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor mcq = (mlnx_umap_cq_data_out_t *)create_msg.evd_cq_data_out;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* call into driver to allocate cq */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor retval = ioctl(ia_fd, DAPL_EVD_CREATE, &create_msg);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor "dapli_get_dto_cq: DAPL_EVD_CREATE failed\n");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor sizeof (struct dapls_ib_cq_handle));
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ia_ptr->hca_ptr->null_ib_cq_handle = IB_INVALID_HANDLE;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor sizeof (struct dapls_ib_cq_handle));
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor dapl_os_lock_init(&cq_handle->cq_wrid_wqhdr_lock);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor cq_handle->cq_size = create_msg.evd_cq_real_size;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor cq_handle->cq_iauar = ia_ptr->hca_ptr->ib_hca_handle->ia_uar;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor "dapli_get_dto_cq: cq 0x%p created, hkey 0x%016llx\n",
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* save this dummy CQ handle into the hca */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * dapl_ib_qp_alloc
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Alloc a QP
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * *ep_ptr pointer to EP INFO
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * ib_hca_handle provider HCA handle
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * ib_pd_handle provider protection domain handle
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * cq_recv provider recv CQ handle
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * cq_send provider send CQ handle
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * DAT_SUCCESS
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * DAT_INSUFFICIENT_RESOURCES
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* check parameters */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor "qp_alloc: hca_handle == NULL\n");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor hca_fd = ia_ptr->hca_ptr->ib_hca_handle->hca_fd;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor dapl_os_assert(ep_ptr->param.pz_handle != NULL);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor dapl_os_assert(ep_ptr->param.connect_evd_handle != NULL);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* fill in args for ep_create */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor (void) dapl_os_memzero(&ep_args, sizeof (ep_args));
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor mqp = (mlnx_umap_qp_data_out_t *)ep_args.ep_qp_data_out;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ep_args.ep_pd_hkey = pz_handle->pd_handle->pd_hkey;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor evd_handle = (DAPL_EVD *)ep_ptr->param.connect_evd_handle;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ep_args.ep_conn_evd_hkey = evd_handle->ib_cq_handle->evd_hkey;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ep_args.ep_ch_sizes.dcs_sq = ep_ptr->param.ep_attr.max_request_dtos;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ep_args.ep_ch_sizes.dcs_sq_sgl = ep_ptr->param.ep_attr.max_request_iov;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor sizeof (struct dapls_ib_qp_handle));
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor "qp_alloc: os_alloc failed\n");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor premev_size = ep_ptr->param.ep_attr.max_recv_dtos *
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor qp_p->qp_premature_events = (ib_work_completion_t *)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor "qp_alloc:alloc premature_events failed\n");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* premature events for EPs with SRQ sit on the SRQ */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ep_args.ep_srq_hkey = srq_p->srq_handle->srq_hkey;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * there are cases when ep_ptr is a dummy container ep, and the orig
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * ep pointer is passed in ep_ctx_ptr. eg - dapl_ep_modify does this.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * ep_cookie should be the actual ep pointer, not the dummy container
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * ep since the kernel returns this via events and the CM callback
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor "qp_alloc: ep_ptr 0x%p, pz 0x%p (0x%llx), rcv_evd 0x%p (0x%llx)\n"
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor " snd_evd 0x%p (0x%llx), conn_evd 0x%p (0x%llx)\n"
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor " srq_hdl 0x%p (0x%llx)\n"
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor " sq_sz %d, rq_sz %d, sq_sgl_sz %d, rq_sgl_sz %d\n",
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ep_ptr->param.recv_evd_handle, ep_args.ep_rcv_evd_hkey,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ep_ptr->param.request_evd_handle, ep_args.ep_snd_evd_hkey,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ep_ptr->param.connect_evd_handle, ep_args.ep_conn_evd_hkey,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ep_args.ep_ch_sizes.dcs_sq, ep_args.ep_ch_sizes.dcs_rq,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ep_args.ep_ch_sizes.dcs_sq_sgl, ep_args.ep_ch_sizes.dcs_rq_sgl);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* The next line is only needed for backward compatibility */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor retval = ioctl(ia_fd, DAPL_EP_CREATE, &ep_args);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor if (retval != 0 || mqp->mqp_rev != MLNX_UMAP_IF_VERSION) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor "qp_alloc: ep_create failed errno %d, retval %d\n",
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor dapl_os_free(qp_p->qp_premature_events, premev_size);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* In the case of Arbel or Hermon */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor if (mqp->mqp_sdbr_mapoffset != 0 || mqp->mqp_sdbr_maplen != 0)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor qp_p->qp_sq_dbp = dapls_ib_get_dbp(mqp->mqp_sdbr_maplen,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor hca_fd, mqp->mqp_sdbr_mapoffset, mqp->mqp_sdbr_offset);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor if (mqp->mqp_rdbr_mapoffset != 0 || mqp->mqp_rdbr_maplen != 0)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor qp_p->qp_rq_dbp = dapls_ib_get_dbp(mqp->mqp_rdbr_maplen,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor hca_fd, mqp->mqp_rdbr_mapoffset, mqp->mqp_rdbr_offset);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor qp_p->qp_addr = mmap64((void *)0, mqp->mqp_maplen,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor dapl_os_free(qp_p->qp_premature_events, premev_size);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor qp_p->qp_iauar = ia_ptr->hca_ptr->ib_hca_handle->ia_uar;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor qp_p->qp_ia_bf = ia_ptr->hca_ptr->ib_hca_handle->ia_bf;
1ed53a3f65abecaadc1b967e341970ad0f6b2aebBill Taylor qp_p->qp_ia_bf_toggle = ia_ptr->hca_ptr->ib_hca_handle->ia_bf_toggle;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor evd_handle = (DAPL_EVD *)ep_ptr->param.request_evd_handle;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor qp_p->qp_sq_buf = (caddr_t)(qp_p->qp_addr + mqp->mqp_sq_off);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor qp_p->qp_sq_sgl = ep_ptr->param.ep_attr.max_request_iov;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor qp_p->qp_sq_inline = ia_ptr->hca_ptr->max_inline_send;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor qp_p->qp_sq_headroom = mqp->mqp_sq_headroomwqes;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor evd_handle = (DAPL_EVD *)ep_ptr->param.recv_evd_handle;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor qp_p->qp_rq_buf = (caddr_t)(qp_p->qp_addr + mqp->mqp_rq_off);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor qp_p->qp_rq_sgl = ep_ptr->param.ep_attr.max_recv_iov;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor "qp_alloc: created, qp_sq_buf %p, qp_rq_buf %p\n",
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor "qp_alloc: created, sq numwqe %x wqesz %x, rq numwqe %x wqesz %x\n",
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor "qp_alloc: created, qp_sq_desc_addr %x, qp_rq_desc_addr %x\n",
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor "qp_alloc: created, ep_ptr 0x%p, ep_hkey 0x%016llx\n\n",
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Calculate the number of bits in max_rmrs - this is indirectly
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the max number of entried in the MPT table (defaults to 512K
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * but is configurable). This value is used while creating new
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * rkeys in bind processing (see dapl_tavor_hw.c).
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Stash this value in the qp handle, don't want to do this math
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * for every bind
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor mpt_mask = (uint32_t)ia_ptr->hca_ptr->ia_attr.max_rmrs - 1;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor for (i = 0; mpt_mask > 0; mpt_mask = (mpt_mask >> 1), i++)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* update the qp handle in the ep ptr */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * ibt_alloc_rc_channel transitions the qp state to INIT.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * hence we directly transition from UNATTACHED to INIT
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* insert ep into the SRQ's ep_table */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor dat_status = dapli_ib_srq_add_ep(srq_p->srq_handle,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor "qp_alloc: srq_add_ep failed ep_ptr 0x%p, 0x%x\n",
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor if (dapls_tavor_wrid_init(qp_p) != DAT_SUCCESS) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * dapls_ib_qp_free
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * *ep_ptr pointer to EP INFO
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * ib_hca_handle provider HCA handle
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylordapls_ib_qp_free(IN DAPL_IA *ia_ptr, IN DAPL_EP *ep_ptr)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ib_hca_handle_t ib_hca_handle = ia_ptr->hca_ptr->ib_hca_handle;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor (ep_ptr->qp_state != DAPL_QP_STATE_UNATTACHED)) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor if (munmap((void *)qp_p->qp_addr, qp_p->qp_map_len) < 0) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor retval = ioctl(ib_hca_handle->ia_fd, DAPL_EP_FREE, &args);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor "qp_free: ioctl errno = %d, retval = %d\n",
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor "qp_free: freed, ep_ptr 0x%p, ep_hkey 0x%016llx\n",
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor dapli_ib_srq_remove_ep(qp_p->qp_srq, qp_p->qp_num);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * dapl_ib_qp_modify
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Set the QP to the parameters specified in an EP_PARAM
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * We can't be sure what state the QP is in so we first obtain the state
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * from the driver. The EP_PARAM structure that is provided has been
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * sanitized such that only non-zero values are valid.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * ib_hca_handle HCA handle
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * qp_handle QP handle
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * ep_attr Sanitized EP Params
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * DAT_SUCCESS
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * DAT_INSUFFICIENT_RESOURCES
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * DAT_INVALID_PARAMETER
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylordapls_ib_qp_modify(IN DAPL_IA *ia_ptr, IN DAPL_EP *ep_ptr,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor "qp_modify: qp_handle == NULL\n");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor "qp_modify: hca_handle == NULL\n");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * NOTE: ep_attr->max_mtu_size indicates the maximum message
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * size, which is always 2GB for IB. Nothing to do with the IB
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * implementation, nothing to set up.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor if (ep_attr->max_rdma_size > DAPL_IB_MAX_MESSAGE_SIZE) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor (void) memset((void *)&epm_args, 0, sizeof (epm_args));
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * The following parameters are dealt by creating a new qp
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * in dapl_ep_modify.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * - max_recv_dtos
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * - max_request_dtos
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * - max_recv_iov
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * - max_request_iov
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor epm_args.epm_rdma_ra_in = ep_attr->max_rdma_read_in;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor epm_args.epm_rdma_ra_out = ep_attr->max_rdma_read_out;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor "qp_modify: ep_hkey = %016llx nothing to do\n",
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor retval = ioctl(ia_fd, DAPL_EP_MODIFY, &epm_args);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor "qp_modify: ioctl failed errno %d, retval %d\n",
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor "qp_modify: ep_hkey = %016llx\n", ep_ptr->qp_handle->ep_hkey);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Allocate the srq data structure as well as the kernel resource
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * corresponding to it.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylordapls_ib_srq_alloc(IN DAPL_IA *ia_ptr, IN DAPL_SRQ *srq_ptr)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* check parameters */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor "srq_alloc: hca_handle == NULL\n");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor return (DAT_ERROR(DAT_INVALID_PARAMETER, DAT_INVALID_ARG1));
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor hca_fd = ia_ptr->hca_ptr->ib_hca_handle->hca_fd;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor dapl_os_assert(srq_ptr->param.pz_handle != NULL);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* fill in args for srq_create */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor pz_handle = (DAPL_PZ *)srq_ptr->param.pz_handle;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ibsrq_p = (ib_srq_handle_t)dapl_os_alloc(sizeof (*ibsrq_p));
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor "srq_alloc: os_alloc failed\n");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor (void) dapl_os_memzero(ibsrq_p, sizeof (*ibsrq_p));
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor (void) dapl_os_memzero(&srqc_args, sizeof (srqc_args));
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor msrq = (mlnx_umap_srq_data_out_t *)srqc_args.srqc_data_out;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor srqc_args.srqc_pd_hkey = pz_handle->pd_handle->pd_hkey;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor srqc_args.srqc_sizes.srqs_sz = srq_ptr->param.max_recv_dtos;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor srqc_args.srqc_sizes.srqs_sgl = srq_ptr->param.max_recv_iov;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor "srq_alloc: srq_ptr 0x%p, pz 0x%p (0x%llx), srq_sz %d"
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor " srq_sgl %d\n",
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor srqc_args.srqc_sizes.srqs_sz, srqc_args.srqc_sizes.srqs_sgl);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* The next line is only needed for backward compatibility */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor retval = ioctl(ia_fd, DAPL_SRQ_CREATE, &srqc_args);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor if (retval != 0 || msrq->msrq_rev != MLNX_UMAP_IF_VERSION) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor "srq_alloc: srq_create failed errno %d, retval %d\n",
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* In the case of Arbel or Hermon */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor if (msrq->msrq_rdbr_mapoffset != 0 || msrq->msrq_rdbr_maplen != 0)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor msrq->msrq_rdbr_mapoffset, msrq->msrq_rdbr_offset);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor retval = ioctl(ia_fd, DAPL_SRQ_FREE, &srqf_args);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ibsrq_p->srq_iauar = ia_ptr->hca_ptr->ib_hca_handle->ia_uar;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* since 0 is a valid index, -1 indicates invalid value */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ibsrq_p->srq_wq_desc_addr = msrq->msrq_desc_addr;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ibsrq_p->srq_wq_sgl = srqc_args.srqc_real_sizes.srqs_sgl;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * update the srq handle in the srq ptr, this is needed since from
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * here on cleanup is done by calling dapls_ib_srq_free()
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor premev_size = ibsrq_p->srq_wq_numwqe * sizeof (ib_work_completion_t);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ibsrq_p->srq_premature_events = (ib_work_completion_t *)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor "srq_alloc: os_alloc premature_events failed\n");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor freeev_size = ibsrq_p->srq_wq_numwqe * sizeof (uint32_t);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ibsrq_p->srq_freepr_events = (uint32_t *)dapl_os_alloc(freeev_size);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor "srq_alloc: os_alloc freepr_events failed\n");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ibsrq_p->srq_freepr_num_events = ibsrq_p->srq_wq_numwqe;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* initialize the free list of premature events */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor for (i = 0; i < ibsrq_p->srq_freepr_num_events; i++) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * wc_res_hash field is used to mark entries in the premature
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * events list
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DAPL_SET_CQE_INVALID(&(ibsrq_p->srq_premature_events[i]));
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor "srq_alloc: created, srq_ptr 0x%p, srq_hkey 0x%016llx\n",
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor if (dapls_tavor_srq_wrid_init(ibsrq_p) != DAT_SUCCESS) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor "srq_alloc: wridlist alloc failed\n");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* allocate a hash table to to store EPs */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor retval = dapls_hash_create(DAPL_HASH_TABLE_DEFAULT_CAPACITY,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor dapl_dbg_log(DAPL_DBG_TYPE_ERR, "dapls_ib_srq_alloc hash "
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * SRQ Free routine
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylordapls_ib_srq_free(IN DAPL_IA *ia_handle, IN DAPL_SRQ *srq_ptr)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ib_srq_handle_t srq_handle = srq_ptr->srq_handle;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ib_hca_handle_t ib_hca_handle = ia_handle->hca_ptr->ib_hca_handle;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor return; /* nothing to do */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor if (munmap((void *)srq_handle->srq_addr, srq_handle->srq_map_len) < 0) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor retval = ioctl(ib_hca_handle->ia_fd, DAPL_SRQ_FREE, &srqf_args);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor "srq_free: ioctl errno = %d, retval = %d\n", errno, retval);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor "srq_free: freed, srq_ptr 0x%p, srq_hkey 0x%016llx\n",
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor (void) dapls_hash_free(srq_handle->srq_ep_table);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor srq_handle->srq_wq_numwqe * sizeof (ib_work_completion_t));
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Adds EP to a hashtable in SRQ
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylordapli_ib_srq_add_ep(IN ib_srq_handle_t srq_ptr, IN uint32_t qp_num,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor "srq_insert_ep:%p %p %llx\n", srq_ptr, htable, key);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor return (dapls_hash_insert(htable, key, ep_ptr));
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Removes an EP from the hashtable in SRQ
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylordapli_ib_srq_remove_ep(IN ib_srq_handle_t srq_ptr, IN uint32_t qp_num)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor retval = dapls_hash_remove(htable, key, (DAPL_HASH_DATA *)&epp);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor "srq_remove_ep(%d): %p %llx\n", retval, htable, key);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Lookup an EP from the hashtable in SRQ
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylordapls_ib_srq_lookup_ep(IN DAPL_SRQ *srq_ptr, IN ib_work_completion_t *cqe_ptr)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor retval = dapls_hash_search(htable, key, (DAPL_HASH_DATA *)&epp);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor "srq_lookup_ep(%x): %p %llx\n", retval, htable, key);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * dapl_ib_srq_resize
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Resize an SRQ
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * srq_ptr pointer to SRQ struct
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * srqlen new length of the SRQ
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * DAT_SUCCESS
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * DAT_INVALID_HANDLE
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * DAT_INTERNAL_ERROR
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * DAT_INSUFFICIENT_RESOURCES
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor dat_status = dapli_ib_srq_resize_internal(srq_ptr, srqlen);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor if (DAT_INSUFFICIENT_RESOURCES == DAT_GET_TYPE(dat_status)) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* attempt to resize back to the current size */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor dat_status = dapli_ib_srq_resize_internal(srq_ptr,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * XXX this is catastrophic need to post an event
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * to the async evd
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * dapli_ib_srq_resize_internal
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * An internal routine to resize a SRQ.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * srq_ptr pointer to SRQ struct
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * srqlen new length of the srq
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * DAT_SUCCESS
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * DAT_INVALID_HANDLE
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * DAT_INSUFFICIENT_RESOURCES
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor "dapls_ib_srq_resize: srq 0x%p srq_hdl 0x%p "
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor "srq_hkey 0x%016llx srqlen %d\n",
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Since SRQs are created in powers of 2 its possible that the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * previously allocated SRQ has sufficient entries. If the current
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * SRQ is big enough and it is mapped we are done.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor if ((srqlen < srq_handle->srq_wq_numwqe) && (srq_handle->srq_addr)) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* unmap the SRQ before resizing it */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor if ((srq_handle->srq_addr) && (munmap((char *)srq_handle->srq_addr,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor "srq_resize: munmap(%p:0x%llx) failed(%d)\n",
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor srq_handle->srq_addr, srq_handle->srq_map_len, errno);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor return (DAT_ERROR(DAT_INVALID_HANDLE, DAT_INVALID_HANDLE_SRQ));
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* srq_addr is unmapped and no longer valid */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ia_fd = srq_ptr->header.owner_ia->hca_ptr->ib_hca_handle->ia_fd;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor hca_fd = srq_ptr->header.owner_ia->hca_ptr->ib_hca_handle->hca_fd;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor (void) dapl_os_memzero(&resize_msg, sizeof (resize_msg));
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor msrq = (mlnx_umap_srq_data_out_t *)resize_msg.srqr_data_out;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* The next line is only needed for backward compatibility */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor retval = ioctl(ia_fd, DAPL_SRQ_RESIZE, &resize_msg);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor if (retval != 0 || msrq->msrq_rev != MLNX_UMAP_IF_VERSION) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor "dapls_ib_srq_resize: srq 0x%p, err: %s\n",
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor if (errno == EINVAL) { /* Couldn't find this srq */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor } else { /* Need to retry resize with a smaller qlen */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor dapl_os_assert(srq_handle->srq_num == msrq->msrq_srqnum);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* In the case of Arbel or Hermon */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Need to retry resize with a smaller qlen */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor old_freepr_size = srq_handle->srq_wq_numwqe * sizeof (uint32_t);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor old_premature_size = srq_handle->srq_wq_numwqe *
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor old_freepr_events = srq_handle->srq_freepr_events;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor old_premature_events = srq_handle->srq_premature_events;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor new_freepr_size = resize_msg.srqr_real_size * sizeof (uint32_t);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor new_premature_size = resize_msg.srqr_real_size *
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor new_freepr_events = (uint32_t *)dapl_os_alloc(new_freepr_size);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor new_premature_events = (ib_work_completion_t *)dapl_os_alloc(
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* copy valid premature events */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor for (i = 0; i < srq_handle->srq_wq_numwqe; i++) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor if (!DAPL_CQE_IS_VALID(&old_premature_events[i])) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor (void) dapl_os_memcpy(&new_premature_events[idx],
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor &old_premature_events[i], sizeof (ib_work_completion_t));
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor dapl_os_assert(srq_handle->srq_wq_numwqe - idx ==
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Initialize free events lists */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor for (i = 0; i < resize_msg.srqr_real_size - idx; i++) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor srq_handle->srq_freepr_events = new_freepr_events;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor srq_handle->srq_premature_events = new_premature_events;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor srq_handle->srq_freepr_num_events = resize_msg.srqr_real_size - idx;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* a full freepr list has tail at 0 */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor srq_handle->srq_freepr_tail = srq_handle->srq_freepr_num_events;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor old_freepr_size = old_freepr_size; /* pacify lint */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor dapl_os_free(old_freepr_events, old_freepr_size);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor old_premature_size = old_premature_size; /* pacify lint */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor dapl_os_free(old_premature_events, old_premature_size);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * update the srq fields,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * note: the srq_wq_lastwqeindex doesn't change since the old
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * work queue is copied as a whole into the new work queue.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor srq_handle->srq_map_offset = msrq->msrq_mapoffset;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor srq_handle->srq_wq_desc_addr = msrq->msrq_desc_addr;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor dapl_os_free(new_freepr_events, new_freepr_size);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor dapl_os_free(new_premature_events, new_premature_size);