9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * CDDL HEADER START
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * The contents of this file are subject to the terms of the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Common Development and Distribution License (the "License").
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * You may not use this file except in compliance with the License.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * See the License for the specific language governing permissions
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * and limitations under the License.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * When distributing Covered Code, include this CDDL HEADER in each
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * If applicable, add the following below this CDDL HEADER, with the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * fields enclosed by brackets "[]" replaced with your own identifying
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * information: Portions Copyright [yyyy] [name of copyright owner]
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * CDDL HEADER END
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Copyright 2009 Sun Microsystems, Inc. All rights reserved.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Use is subject to license terms.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Tavor Queue Pair Processing Routines
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Implements all the routines necessary for allocating, freeing, and
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * querying the Tavor queue pairs.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic int tavor_qp_create_qpn(tavor_state_t *state, tavor_qphdl_t qp,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic int tavor_qpn_avl_compare(const void *q, const void *e);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic int tavor_special_qp_rsrc_alloc(tavor_state_t *state,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ibt_sqp_type_t type, uint_t port, tavor_rsrc_t **qp_rsrc);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic int tavor_special_qp_rsrc_free(tavor_state_t *state, ibt_sqp_type_t type,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic void tavor_qp_sgl_to_logwqesz(tavor_state_t *state, uint_t num_sgl,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor tavor_qp_wq_type_t wq_type, uint_t *logwqesz, uint_t *max_sgl);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * tavor_qp_alloc()
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Context: Can be called only from user or kernel context.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylortavor_qp_alloc(tavor_state_t *state, tavor_qp_info_t *qpinfo,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*attr_p, *queuesz_p))
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Check the "options" flag. Currently this flag tells the driver
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * whether or not the QP's work queues should be come from normal
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * system memory or whether they should be allocated from DDR memory.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Extract the necessary info from the tavor_qp_info_t structure
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Determine whether QP is being allocated for userland access or
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * whether it is being allocated for kernel access. If the QP is
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * being allocated for userland access, then lookup the UAR doorbell
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * page number for the current process. Note: If this is not found
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * (e.g. if the process has not previously open()'d the Tavor driver),
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * then an error is returned.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor qp_is_umap = (attr_p->qp_alloc_flags & IBT_QP_USER_MAP) ? 1 : 0;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor status = tavor_umap_db_find(state->ts_instance, ddi_get_pid(),
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Set "status" and "errormsg" and goto failure */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TAVOR_TNF_FAIL(IBT_INVALID_PARAM, "failed UAR page");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor uarpg = ((tavor_rsrc_t *)(uintptr_t)value)->tr_indx;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Determine whether QP is being associated with an SRQ
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor qp_srq_en = (attr_p->qp_alloc_flags & IBT_QP_USES_SRQ) ? 1 : 0;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Check for valid SRQ handle pointers
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Set "status" and "errormsg" and goto failure */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor "invalid SRQ handle");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Check for valid QP service type (only UD/RC/UC supported)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor if (((type != IBT_UD_RQP) && (type != IBT_RC_RQP) &&
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Set "status" and "errormsg" and goto failure */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TAVOR_TNF_FAIL(IBT_QP_SRV_TYPE_INVALID, "invalid serv type");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Only RC is supported on an SRQ -- This is a Tavor hardware
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * limitation. Arbel native mode will not have this shortcoming.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Set "status" and "errormsg" and goto failure */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TAVOR_TNF_FAIL(IBT_INVALID_PARAM, "invalid serv type with SRQ");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Check for valid PD handle pointer
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Set "status" and "errormsg" and goto failure */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TAVOR_TNF_FAIL(IBT_PD_HDL_INVALID, "invalid PD handle");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * If on an SRQ, check to make sure the PD is the same
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor if (qp_srq_en && (pd->pd_pdnum != srq->srq_pdhdl->pd_pdnum)) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Set "status" and "errormsg" and goto failure */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TAVOR_TNF_FAIL(IBT_PD_HDL_INVALID, "invalid PD handle");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Increment the reference count on the protection domain (PD) */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Check for valid CQ handle pointers
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Set "status" and "errormsg" and goto failure */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TAVOR_TNF_FAIL(IBT_CQ_HDL_INVALID, "invalid CQ handle");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Increment the reference count on the CQs. One or both of these
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * could return error if we determine that the given CQ is already
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * being used with a special (SMI/GSI) QP.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor status = tavor_cq_refcnt_inc(sq_cq, TAVOR_CQ_IS_NORMAL);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Set "status" and "errormsg" and goto failure */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TAVOR_TNF_FAIL(IBT_CQ_HDL_INVALID, "invalid CQ handle");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor status = tavor_cq_refcnt_inc(rq_cq, TAVOR_CQ_IS_NORMAL);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Set "status" and "errormsg" and goto failure */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TAVOR_TNF_FAIL(IBT_CQ_HDL_INVALID, "invalid CQ handle");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Allocate an QP context entry. This will be filled in with all
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the necessary parameters to define the Queue Pair. Unlike
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * other Tavor hardware resources, ownership is not immediately
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * given to hardware in the final step here. Instead, we must
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * wait until the QP is later transitioned to the "Init" state before
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * passing the QP to hardware. If we fail here, we must undo all
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the reference count (CQ and PD).
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor status = tavor_rsrc_alloc(state, TAVOR_QPC, 1, sleepflag, &qpc);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Set "status" and "errormsg" and goto failure */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TAVOR_TNF_FAIL(IBT_INSUFF_RESOURCE, "failed QP context");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Allocate the software structure for tracking the queue pair
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * (i.e. the Tavor Queue Pair handle). If we fail here, we must
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * undo the reference counts and the previous resource allocation.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor status = tavor_rsrc_alloc(state, TAVOR_QPHDL, 1, sleepflag, &rsrc);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Set "status" and "errormsg" and goto failure */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TAVOR_TNF_FAIL(IBT_INSUFF_RESOURCE, "failed QP handle");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Calculate the QP number from QPC index. This routine handles
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * all of the operations necessary to keep track of used, unused,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * and released QP numbers.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Set "status" and "errormsg" and goto failure */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TAVOR_TNF_FAIL(IBT_INSUFF_RESOURCE, "failed QPN create");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * If this will be a user-mappable QP, then allocate an entry for
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the "userland resources database". This will later be added to
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the database (after all further QP operations are successful).
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * If we fail here, we must undo the reference counts and the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * previous resource allocation.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor umapdb = tavor_umap_db_alloc(state->ts_instance, qp->qp_qpnum,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor MLNX_UMAP_QPMEM_RSRC, (uint64_t)(uintptr_t)rsrc);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Set "status" and "errormsg" and goto failure */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TAVOR_TNF_FAIL(IBT_INSUFF_RESOURCE, "failed umap add");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * If this is an RC QP, then pre-allocate the maximum number of RDB
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * entries. This allows us to ensure that we can later cover all
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the resources needed by hardware for handling multiple incoming
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * RDMA Reads. Note: These resources are obviously not always
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * necessary. They are allocated here anyway. Someday maybe this
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * can be modified to allocate these on-the-fly (i.e. only if RDMA
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Read or Atomic operations are enabled) XXX
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * If we fail here, we have a bunch of resource and reference count
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * cleanup to do.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor max_rdb = state->ts_cfg_profile->cp_hca_max_rdma_in_qp;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor status = tavor_rsrc_alloc(state, TAVOR_RDB, max_rdb,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Set "status" and "errormsg" and goto failure */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TAVOR_TNF_FAIL(IBT_INSUFF_RESOURCE, "failed RDB");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Calculate offset (into DDR memory) of RDB entries */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor qp->qp_rdb_ddraddr = (uintptr_t)rsrc_pool->rsrc_ddr_offset +
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Calculate the appropriate size for the work queues.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Note: All Tavor QP work queues must be a power-of-2 in size. Also
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * they may not be any smaller than TAVOR_QP_MIN_SIZE. This step is
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * to round the requested size up to the next highest power-of-2
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor attr_p->qp_sizes.cs_sq = max(attr_p->qp_sizes.cs_sq, TAVOR_QP_MIN_SIZE);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor attr_p->qp_sizes.cs_rq = max(attr_p->qp_sizes.cs_rq, TAVOR_QP_MIN_SIZE);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor log_qp_sq_size = highbit(attr_p->qp_sizes.cs_sq);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor log_qp_rq_size = highbit(attr_p->qp_sizes.cs_rq);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Next we verify that the rounded-up size is valid (i.e. consistent
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * with the device limits and/or software-configured limits). If not,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * then obviously we have a lot of cleanup to do before returning.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor if ((log_qp_sq_size > state->ts_cfg_profile->cp_log_max_qp_sz) ||
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Set "status" and "errormsg" and goto failure */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TAVOR_TNF_FAIL(IBT_HCA_WR_EXCEEDED, "max QP size");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Next we verify that the requested number of SGL is valid (i.e.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * consistent with the device limits and/or software-configured
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * limits). If not, then obviously the same cleanup needs to be done.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor max_sgl = state->ts_cfg_profile->cp_wqe_real_max_sgl;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor (!qp_srq_en && (attr_p->qp_sizes.cs_rq_sgl > max_sgl))) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Set "status" and "errormsg" and goto failure */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TAVOR_TNF_FAIL(IBT_HCA_SGL_EXCEEDED, "max QP SGL");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Determine this QP's WQE sizes (for both the Send and Recv WQEs).
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * This will depend on the requested number of SGLs. Note: this
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * has the side-effect of also calculating the real number of SGLs
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * (for the calculated WQE size).
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * For QP's on an SRQ, we set these to 0.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor tavor_qp_sgl_to_logwqesz(state, attr_p->qp_sizes.cs_rq_sgl,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor tavor_qp_sgl_to_logwqesz(state, attr_p->qp_sizes.cs_sq_sgl,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TAVOR_QP_WQ_TYPE_SENDQ, &qp->qp_sq_log_wqesz, &qp->qp_sq_sgl);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Allocate the memory for QP work queues. Note: The location from
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * which we will allocate these work queues has been passed in
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * through the tavor_qp_options_t structure. Since Tavor work queues
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * are not allowed to cross a 32-bit (4GB) boundary, the alignment of
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the work queue memory is very important. We used to allocate
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * work queues (the combined receive and send queues) so that they
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * would be aligned on their combined size. That alignment guaranteed
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * that they would never cross the 4GB boundary (Tavor work queues
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * are on the order of MBs at maximum). Now we are able to relax
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * this alignment constraint by ensuring that the IB address assigned
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * to the queue memory (as a result of the tavor_mr_register() call)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * is offset from zero.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Previously, we had wanted to use the ddi_dma_mem_alloc() routine to
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * guarantee the alignment, but when attempting to use IOMMU bypass
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * mode we found that we were not allowed to specify any alignment
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * that was more restrictive than the system page size.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * So we avoided this constraint by passing two alignment values,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * one for the memory allocation itself and the other for the DMA
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * handle (for later bind). This used to cause more memory than
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * necessary to be allocated (in order to guarantee the more
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * restrictive alignment contraint). But be guaranteeing the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * zero-based IB virtual address for the queue, we are able to
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * conserve this memory.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Note: If QP is not user-mappable, then it may come from either
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * kernel system memory or from HCA-attached local DDR memory.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* QP on SRQ sets these to 0 */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor qp->qp_wqinfo.qa_alloc_align = max(sq_wqe_size, rq_wqe_size);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor qp->qp_wqinfo.qa_bind_align = max(sq_wqe_size, rq_wqe_size);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor qp->qp_wqinfo.qa_location = TAVOR_QUEUE_LOCATION_USERLAND;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor status = tavor_queue_alloc(state, &qp->qp_wqinfo, sleepflag);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Set "status" and "errormsg" and goto failure */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TAVOR_TNF_FAIL(IBT_INSUFF_RESOURCE, "failed work queue");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * If QP's on an SRQ, we set the rq_buf to NULL
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor rq_buf = (uint32_t *)((uintptr_t)sq_buf + sq_size);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor sq_buf = (uint32_t *)((uintptr_t)rq_buf + rq_size);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Register the memory for the QP work queues. The memory for the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * QP must be registered in the Tavor TPT tables. This gives us the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * LKey to specify in the QP context later. Note: The memory for
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Tavor work queues (both Send and Recv) must be contiguous and
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * registered as a single memory region. Note also: If the work
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * queue is to be allocated from DDR memory, then only a "bypass"
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * mapping is appropriate. And if the QP memory is user-mappable,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * then we force DDI_DMA_CONSISTENT mapping.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Also, in order to meet the alignment restriction, we pass the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * "mro_bind_override_addr" flag in the call to tavor_mr_register().
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * This guarantees that the resulting IB vaddr will be zero-based
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * (modulo the offset into the first page).
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * If we fail here, we still have the bunch of resource and reference
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * count cleanup to do.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor flag = (sleepflag == TAVOR_SLEEP) ? IBT_MR_SLEEP :
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor mr_attr.mr_vaddr = (uint64_t)(uintptr_t)qp->qp_wqinfo.qa_buf_aligned;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor mr_op.mro_bind_type = state->ts_cfg_profile->cp_iommu_bypass;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor if (wq_location == TAVOR_QUEUE_LOCATION_NORMAL) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor mr_op.mro_bind_dmahdl = qp->qp_wqinfo.qa_dmahdl;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor status = tavor_mr_register(state, pd, &mr_attr, &mr, &mr_op);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Set "status" and "errormsg" and goto failure */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TAVOR_TNF_FAIL(IBT_INSUFF_RESOURCE, "failed register mr");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Calculate the offset between the kernel virtual address space
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * and the IB virtual address space. This will be used when
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * posting work requests to properly initialize each WQE.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor qp_desc_off = (uint64_t)(uintptr_t)qp->qp_wqinfo.qa_buf_aligned -
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Fill in all the return arguments (if necessary). This includes
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * real work queue sizes, real SGLs, and QP number
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* QP on an SRQ set these to 0 */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Fill in the rest of the Tavor Queue Pair handle. We can update
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the following fields for use in further operations on the QP.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor qp->qp_sq_sigtype = (attr_p->qp_flags & IBT_WR_SIGNALED) ?
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TAVOR_QP_SQ_WR_SIGNALED : TAVOR_QP_SQ_ALL_SIGNALED;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* QP on an SRQ sets this to 0 */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * If this QP is to be associated with an SRQ, then set the SRQ handle
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * appropriately.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Determine if later ddi_dma_sync will be necessary */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor qp->qp_sync = TAVOR_QP_IS_SYNC_REQ(state, qp->qp_wqinfo);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Determine the QP service type */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Zero out the QP context */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Put QP handle in Tavor QPNum-to-QPHdl list. Then fill in the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * "qphdl" and return success
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * If this is a user-mappable QP, then we need to insert the previously
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * allocated entry into the "userland resources database". This will
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * allow for later lookup during devmap() (i.e. mmap()) calls.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * The following is cleanup for all possible failure cases in this routine
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Releasing the QPN will also free up the QPC context. Update
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the QPC context pointer to indicate this.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor tavor_qp_release_qpn(state, qp->qp_qpn_hdl, TAVOR_QPN_RELEASE);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TNF_PROBE_1(tavor_qp_alloc_fail, TAVOR_TNF_ERROR, "",
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * tavor_special_qp_alloc()
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Context: Can be called only from user or kernel context.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylortavor_special_qp_alloc(tavor_state_t *state, tavor_qp_info_t *qpinfo,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Check the "options" flag. Currently this flag tells the driver
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * whether or not the QP's work queues should be come from normal
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * system memory or whether they should be allocated from DDR memory.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Extract the necessary info from the tavor_qp_info_t structure
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Check for valid special QP type (only SMI & GSI supported)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor if ((type != IBT_SMI_SQP) && (type != IBT_GSI_SQP)) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Set "status" and "errormsg" and goto failure */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TAVOR_TNF_FAIL(IBT_QP_SPECIAL_TYPE_INVALID, "invalid QP type");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Check for valid port number
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Set "status" and "errormsg" and goto failure */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TAVOR_TNF_FAIL(IBT_HCA_PORT_INVALID, "invalid port num");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Check for valid PD handle pointer
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Set "status" and "errormsg" and goto failure */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TAVOR_TNF_FAIL(IBT_PD_HDL_INVALID, "invalid PD handle");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Increment the reference count on the PD */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Check for valid CQ handle pointers
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Set "status" and "errormsg" and goto failure */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TAVOR_TNF_FAIL(IBT_CQ_HDL_INVALID, "invalid CQ handle");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Increment the reference count on the CQs. One or both of these
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * could return error if we determine that the given CQ is already
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * being used with a non-special QP (i.e. a normal QP).
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor status = tavor_cq_refcnt_inc(sq_cq, TAVOR_CQ_IS_SPECIAL);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Set "status" and "errormsg" and goto failure */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TAVOR_TNF_FAIL(IBT_CQ_HDL_INVALID, "invalid CQ handle");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor status = tavor_cq_refcnt_inc(rq_cq, TAVOR_CQ_IS_SPECIAL);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Set "status" and "errormsg" and goto failure */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TAVOR_TNF_FAIL(IBT_CQ_HDL_INVALID, "invalid CQ handle");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Allocate the special QP resources. Essentially, this allocation
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * amounts to checking if the request special QP has already been
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * allocated. If successful, the QP context return is an actual
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * QP context that has been "aliased" to act as a special QP of the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * appropriate type (and for the appropriate port). Just as in
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * tavor_qp_alloc() above, ownership for this QP context is not
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * immediately given to hardware in the final step here. Instead, we
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * wait until the QP is later transitioned to the "Init" state before
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * passing the QP to hardware. If we fail here, we must undo all
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the reference count (CQ and PD).
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor status = tavor_special_qp_rsrc_alloc(state, type, port, &qpc);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Set "status" and "errormsg" and goto failure */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TAVOR_TNF_FAIL(status, "failed special QP rsrc");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Allocate the software structure for tracking the special queue
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * pair (i.e. the Tavor Queue Pair handle). If we fail here, we
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * must undo the reference counts and the previous resource allocation.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor status = tavor_rsrc_alloc(state, TAVOR_QPHDL, 1, sleepflag, &rsrc);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Set "status" and "errormsg" and goto failure */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TAVOR_TNF_FAIL(IBT_INSUFF_RESOURCE, "failed QP handle");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Actual QP number is a combination of the index of the QPC and
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the port number. This is because the special QP contexts must
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * be allocated two-at-a-time.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Calculate the appropriate size for the work queues.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Note: All Tavor QP work queues must be a power-of-2 in size. Also
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * they may not be any smaller than TAVOR_QP_MIN_SIZE. This step is
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * to round the requested size up to the next highest power-of-2
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor attr_p->qp_sizes.cs_sq = max(attr_p->qp_sizes.cs_sq, TAVOR_QP_MIN_SIZE);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor attr_p->qp_sizes.cs_rq = max(attr_p->qp_sizes.cs_rq, TAVOR_QP_MIN_SIZE);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor log_qp_sq_size = highbit(attr_p->qp_sizes.cs_sq);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor log_qp_rq_size = highbit(attr_p->qp_sizes.cs_rq);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Next we verify that the rounded-up size is valid (i.e. consistent
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * with the device limits and/or software-configured limits). If not,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * then obviously we have a bit of cleanup to do before returning.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor if ((log_qp_sq_size > state->ts_cfg_profile->cp_log_max_qp_sz) ||
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor (log_qp_rq_size > state->ts_cfg_profile->cp_log_max_qp_sz)) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Set "status" and "errormsg" and goto failure */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TAVOR_TNF_FAIL(IBT_HCA_WR_EXCEEDED, "max QP size");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Next we verify that the requested number of SGL is valid (i.e.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * consistent with the device limits and/or software-configured
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * limits). If not, then obviously the same cleanup needs to be done.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor max_sgl = state->ts_cfg_profile->cp_wqe_real_max_sgl;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Set "status" and "errormsg" and goto failure */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TAVOR_TNF_FAIL(IBT_HCA_SGL_EXCEEDED, "max QP SGL");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Determine this QP's WQE sizes (for both the Send and Recv WQEs).
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * This will depend on the requested number of SGLs. Note: this
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * has the side-effect of also calculating the real number of SGLs
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * (for the calculated WQE size).
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor tavor_qp_sgl_to_logwqesz(state, attr_p->qp_sizes.cs_rq_sgl,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TAVOR_QP_WQ_TYPE_RECVQ, &qp->qp_rq_log_wqesz, &qp->qp_rq_sgl);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor tavor_qp_sgl_to_logwqesz(state, attr_p->qp_sizes.cs_sq_sgl,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TAVOR_QP_WQ_TYPE_SENDMLX_QP0, &qp->qp_sq_log_wqesz,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor tavor_qp_sgl_to_logwqesz(state, attr_p->qp_sizes.cs_sq_sgl,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TAVOR_QP_WQ_TYPE_SENDMLX_QP1, &qp->qp_sq_log_wqesz,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Allocate the memory for QP work queues. Note: The location from
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * which we will allocate these work queues has been passed in
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * through the tavor_qp_options_t structure. Since Tavor work queues
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * are not allowed to cross a 32-bit (4GB) boundary, the alignment of
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the work queue memory is very important. We used to allocate
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * work queues (the combined receive and send queues) so that they
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * would be aligned on their combined size. That alignment guaranteed
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * that they would never cross the 4GB boundary (Tavor work queues
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * are on the order of MBs at maximum). Now we are able to relax
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * this alignment constraint by ensuring that the IB address assigned
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * to the queue memory (as a result of the tavor_mr_register() call)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * is offset from zero.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Previously, we had wanted to use the ddi_dma_mem_alloc() routine to
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * guarantee the alignment, but when attempting to use IOMMU bypass
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * mode we found that we were not allowed to specify any alignment
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * that was more restrictive than the system page size.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * So we avoided this constraint by passing two alignment values,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * one for the memory allocation itself and the other for the DMA
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * handle (for later bind). This used to cause more memory than
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * necessary to be allocated (in order to guarantee the more
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * restrictive alignment contraint). But be guaranteeing the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * zero-based IB virtual address for the queue, we are able to
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * conserve this memory.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor qp->qp_wqinfo.qa_alloc_align = max(sq_wqe_size, rq_wqe_size);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor qp->qp_wqinfo.qa_bind_align = max(sq_wqe_size, rq_wqe_size);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor status = tavor_queue_alloc(state, &qp->qp_wqinfo, sleepflag);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Set "status" and "errormsg" and goto failure */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TAVOR_TNF_FAIL(IBT_INSUFF_RESOURCE, "failed work queue");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor rq_buf = (uint32_t *)((uintptr_t)sq_buf + sq_size);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor sq_buf = (uint32_t *)((uintptr_t)rq_buf + rq_size);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Register the memory for the special QP work queues. The memory for
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the special QP must be registered in the Tavor TPT tables. This
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * gives us the LKey to specify in the QP context later. Note: The
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * memory for Tavor work queues (both Send and Recv) must be contiguous
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * and registered as a single memory region. Note also: If the work
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * queue is to be allocated from DDR memory, then only a "bypass"
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * mapping is appropriate.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Also, in order to meet the alignment restriction, we pass the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * "mro_bind_override_addr" flag in the call to tavor_mr_register().
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * This guarantees that the resulting IB vaddr will be zero-based
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * (modulo the offset into the first page).
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * If we fail here, we have a bunch of resource and reference count
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * cleanup to do.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor flag = (sleepflag == TAVOR_SLEEP) ? IBT_MR_SLEEP :
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor mr_attr.mr_vaddr = (uint64_t)(uintptr_t)qp->qp_wqinfo.qa_buf_aligned;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor if (wq_location == TAVOR_QUEUE_LOCATION_NORMAL) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor mr_op.mro_bind_type = state->ts_cfg_profile->cp_iommu_bypass;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor dma_xfer_mode = state->ts_cfg_profile->cp_streaming_consistent;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor mr_op.mro_bind_dmahdl = qp->qp_wqinfo.qa_dmahdl;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor status = tavor_mr_register(state, pd, &mr_attr, &mr, &mr_op);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Set "status" and "errormsg" and goto failure */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TAVOR_TNF_FAIL(IBT_INSUFF_RESOURCE, "failed register mr");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Calculate the offset between the kernel virtual address space
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * and the IB virtual address space. This will be used when
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * posting work requests to properly initialize each WQE.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor qp_desc_off = (uint64_t)(uintptr_t)qp->qp_wqinfo.qa_buf_aligned -
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Fill in all the return arguments (if necessary). This includes
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * real work queue sizes, real SGLs, and QP number (which will be
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * either zero or one, depending on the special QP type)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Fill in the rest of the Tavor Queue Pair handle. We can update
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the following fields for use in further operations on the QP.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor qp->qp_sq_sigtype = (attr_p->qp_flags & IBT_WR_SIGNALED) ?
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TAVOR_QP_SQ_WR_SIGNALED : TAVOR_QP_SQ_ALL_SIGNALED;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Determine if later ddi_dma_sync will be necessary */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor qp->qp_sync = TAVOR_QP_IS_SYNC_REQ(state, qp->qp_wqinfo);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* All special QPs are UD QP service type */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Zero out the QP context */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Put QP handle in Tavor QPNum-to-QPHdl list. Then fill in the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * "qphdl" and return success
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ASSERT(state->ts_qphdl[qpc->tr_indx + port] == NULL);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * The following is cleanup for all possible failure cases in this routine
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor if (tavor_special_qp_rsrc_free(state, type, port) != DDI_SUCCESS) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TAVOR_WARNING(state, "failed to free special QP rsrc");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TNF_PROBE_1(tavor_special_qp_alloc_fail, TAVOR_TNF_ERROR, "",
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * tavor_qp_free()
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * This function frees up the QP resources. Depending on the value
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * of the "free_qp_flags", the QP number may not be released until
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * a subsequent call to tavor_qp_release_qpn().
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Context: Can be called only from user or kernel context.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor/* ARGSUSED */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylortavor_qp_free(tavor_state_t *state, tavor_qphdl_t *qphdl,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ibc_free_qp_flags_t free_qp_flags, ibc_qpn_hdl_t *qpnh,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Pull all the necessary information from the Tavor Queue Pair
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * handle. This is necessary here because the resource for the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * QP handle is going to be freed up as part of this operation.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * If the QP is part of an MCG, then we fail the qp_free
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TAVOR_TNF_FAIL(ibc_get_ci_failure(0), "QP part of MCG on free");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * If the QP is not already in "Reset" state, then transition to
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * "Reset". This is necessary because software does not reclaim
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * ownership of the QP context until the QP is in the "Reset" state.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * If the ownership transfer fails for any reason, then it is an
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * indication that something (either in HW or SW) has gone seriously
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * wrong. So we print a warning message and return.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor if (tavor_qp_to_reset(state, qp) != DDI_SUCCESS) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TAVOR_WARNING(state, "failed to reset QP context");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Set "status" and "errormsg" and goto failure */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor "reset QP context");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Do any additional handling necessary for the transition
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * to the "Reset" state (e.g. update the WRID lists)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * If this was a user-mappable QP, then we need to remove its entry
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * from the "userland resources database". If it is also currently
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * mmap()'d out to a user process, then we need to call
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * devmap_devmem_remap() to remap the QP memory to an invalid mapping.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * We also need to invalidate the QP tracking information for the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * user mapping.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor status = tavor_umap_db_find(state->ts_instance, qp->qp_qpnum,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor MLNX_UMAP_QPMEM_RSRC, &value, TAVOR_UMAP_DB_REMOVE,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TAVOR_WARNING(state, "failed to find in database");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor "devmap_devmem_remap()");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Put NULL into the Tavor QPNum-to-QPHdl list. This will allow any
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * in-progress events to detect that the QP corresponding to this
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * number has been freed. Note: it does depend in whether we are
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * freeing a special QP or not.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Drop the QP lock
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * At this point the lock is no longer necessary. We cannot
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * protect from multiple simultaneous calls to free the same QP.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * In addition, since the QP lock is contained in the QP "software
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * handle" resource, which we will free (see below), it is
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * important that we have no further references to that memory.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Free the QP resources
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Start by deregistering and freeing the memory for work queues.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Next free any previously allocated context information
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * (depending on QP type)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Finally, decrement the necessary reference counts.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * If this fails for any reason, then it is an indication that
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * something (either in HW or SW) has gone seriously wrong. So we
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * print a warning message and return.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor status = tavor_mr_deregister(state, &mr, TAVOR_MR_DEREG_ALL,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TAVOR_WARNING(state, "failed to deregister QP memory");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Set "status" and "errormsg" and goto failure */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TAVOR_TNF_FAIL(ibc_get_ci_failure(0), "failed deregister mr");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Free the memory for the QP */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Free up the remainder of the QP resources. Note: we have a few
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * different resources to free up depending on whether the QP is a
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * special QP or not. As described above, if any of these fail for
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * any reason it is an indication that something (either in HW or SW)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * has gone seriously wrong. So we print a warning message and
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Free up resources for the special QP */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor status = tavor_special_qp_rsrc_free(state, type, port);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TAVOR_WARNING(state, "failed to free special QP rsrc");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Set "status" and "errormsg" and goto failure */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor "failed special QP rsrc");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Free up the RDB entries resource */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Check the flags and determine whether to release the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * QPN or not, based on their value.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Free the Tavor Queue Pair handle */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Decrement the reference counts on CQs, PD and SRQ (if needed) */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Set the qphdl pointer to NULL and return success */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TNF_PROBE_1(tavor_qp_free_fail, TAVOR_TNF_ERROR, "",
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * tavor_qp_query()
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Context: Can be called from interrupt or base context.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylortavor_qp_query(tavor_state_t *state, tavor_qphdl_t qp,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Grab the temporary QPC entry from QP software state
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Convert the current Tavor QP state to IBTF QP state */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor qp_state = IBT_STATE_RTR; /* Ready to Receive */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor qp_state = IBT_STATE_SQE; /* Send Queue Error */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TAVOR_TNF_ERROR, "", tnf_uint, qpstate, qp->qp_state);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* SRQ Hook. */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * The following QP information is always returned, regardless of
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the current QP state. Note: Some special handling is necessary
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * for calculating the QP number on special QP (QP0 and QP1).
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor attr_p->qp_qpn = (qp->qp_is_special == TAVOR_QP_SMI) ? 0 : 1;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * If QP is currently in the "Reset" state, then only the above are
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Post QUERY_QP command to firmware
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * We do a TAVOR_NOSLEEP here because we are holding the "qp_lock".
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Since we may be in the interrupt context (or subsequently raised
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * to interrupt level by priority inversion), we do not want to block
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * in this routine waiting for success.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor status = tavor_cmn_query_cmd_post(state, QUERY_QP, qp->qp_qpnum,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor qpc, sizeof (tavor_hw_qpc_t), TAVOR_CMD_NOSLEEP_SPIN);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor cmn_err(CE_CONT, "Tavor: QUERY_QP command failed: %08x\n",
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TNF_PROBE_1(tavor_qp_query_cmd_fail, TAVOR_TNF_ERROR, "",
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Fill in the additional QP info based on the QP's transport type.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Fill in the UD-specific info */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Fill in the RC-specific info */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Grab the path migration state information */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor if (qpc->pm_state == TAVOR_QP_PMSTATE_MIGRATED) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor } else if (qpc->pm_state == TAVOR_QP_PMSTATE_REARM) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Get the common primary address path fields */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor tavor_get_addr_path(state, qpc_path, &path_ptr->cep_adds_vect,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Fill in the additional primary address path fields */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Get the common alternate address path fields */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor &alt_path_ptr->cep_adds_vect, TAVOR_ADDRPATH_QP, qp);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Fill in the additional alternate address path fields */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor alt_path_ptr->cep_pkey_ix = qpc_alt_path->pkey_indx;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor alt_path_ptr->cep_hca_port_num = qpc_alt_path->portnum;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor alt_path_ptr->cep_timeout = qpc_alt_path->ack_timeout;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Get the RNR retry time from primary path */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Set the enable flags based on RDMA/Atomic enable bits */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor enable_flags |= ((qpc->rre == 0) ? 0 : IBT_CEP_RDMA_RD);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor enable_flags |= ((qpc->rwe == 0) ? 0 : IBT_CEP_RDMA_WR);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor enable_flags |= ((qpc->rae == 0) ? 0 : IBT_CEP_ATOMIC);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Fill in the UC-specific info */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Grab the path migration state information */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor if (qpc->pm_state == TAVOR_QP_PMSTATE_MIGRATED) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor } else if (qpc->pm_state == TAVOR_QP_PMSTATE_REARM) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Get the common primary address path fields */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor tavor_get_addr_path(state, qpc_path, &path_ptr->cep_adds_vect,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Fill in the additional primary address path fields */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Get the common alternate address path fields */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor &alt_path_ptr->cep_adds_vect, TAVOR_ADDRPATH_QP, qp);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Fill in the additional alternate address path fields */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor alt_path_ptr->cep_pkey_ix = qpc_alt_path->pkey_indx;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor alt_path_ptr->cep_hca_port_num = qpc_alt_path->portnum;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Set the enable flags based on RDMA enable bits (by
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * definition UC doesn't support Atomic or RDMA Read)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor enable_flags = ((qpc->rwe == 0) ? 0 : IBT_CEP_RDMA_WR);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TAVOR_WARNING(state, "unexpected QP transport type");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Under certain circumstances it is possible for the Tavor hardware
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * to transition to one of the error states without software directly
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * knowing about it. The QueryQP() call is the one place where we
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * have an opportunity to sample and update our view of the QP state.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * tavor_qp_create_qpn()
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Context: Can be called from interrupt or base context.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylortavor_qp_create_qpn(tavor_state_t *state, tavor_qphdl_t qp, tavor_rsrc_t *qpc)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Build a query (for the AVL tree lookup) and attempt to find
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * a previously added entry that has a matching QPC index. If
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * no matching entry is found, then allocate, initialize, and
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * add an entry to the AVL tree.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * If a matching entry is found, then increment its QPN counter
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * and reference counter.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor entry = (tavor_qpn_entry_t *)avl_find(&state->ts_qpn_avl,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Allocate and initialize a QPN entry, then insert
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * it into the AVL tree.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Make the AVL tree entry point to the QP context resource that
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * it will be responsible for tracking
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Setup the QP handle to point to the AVL tree entry. Then
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * generate the new QP number from the entry's QPN counter value
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * and the hardware's QP context table index.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor state->ts_cfg_profile->cp_log_num_qp) | qpc->tr_indx) &
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Increment the reference counter and QPN counter. The QPN
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * counter always indicates the next available number for use.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * tavor_qp_release_qpn()
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Context: Can be called only from user or kernel context.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylortavor_qp_release_qpn(tavor_state_t *state, tavor_qpn_entry_t *entry, int flags)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * If we are releasing the QP number here, then we decrement the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * reference count and check for zero references. If there are
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * zero references, then we free the QPC context (if it hadn't
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * already been freed during a TAVOR_QPN_FREE_ONLY free, i.e. for
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * reuse with another similar QP number) and remove the tracking
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * structure from the QP number AVL tree and free the structure.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * If we are not releasing the QP number here, then, as long as we
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * have not exhausted the usefulness of the QPC context (that is,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * re-used it too many times without the reference count having
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * gone to zero), we free up the QPC context for use by another
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * thread (which will use it to construct a different QP number
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * from the same QPC table index).
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * If the reference count is zero, then we free the QPC
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * context (if it hadn't already been freed in an early
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * step, e.g. TAVOR_QPN_FREE_ONLY) and remove/free the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * tracking structure from the QP number AVL tree.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * If the current entry has served it's useful
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * purpose (i.e. been reused the maximum allowable
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * number of times), then remove it from QP number
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * AVL tree and free it up.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Even if we are not freeing the QP number, that will not
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * always prevent us from releasing the QPC context. In fact,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * since the QPC context only forms part of the whole QPN,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * we want to free it up for use by other consumers. But
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * if the reference count is non-zero (which it will always
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * be when we are doing TAVOR_QPN_FREE_ONLY) and the counter
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * has reached its maximum value, then we cannot reuse the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * QPC context until the reference count eventually reaches
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * zero (in TAVOR_QPN_RELEASE, above).
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * tavor_qpn_db_compare()
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Context: Can be called from user or kernel context.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylortavor_qpn_avl_compare(const void *q, const void *e)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor return (-1);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor return (+1);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * tavor_qpn_avl_init()
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Context: Only called from attach() path context
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Initialize the lock used for QP number (QPN) AVL tree access */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor mutex_init(&state->ts_qpn_avl_lock, NULL, MUTEX_DRIVER,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Initialize the AVL tree for the QP number (QPN) storage */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor avl_create(&state->ts_qpn_avl, tavor_qpn_avl_compare,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * tavor_qpn_avl_fini()
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Context: Only called from attach() and/or detach() path contexts
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Empty all entries (if necessary) and destroy the AVL tree
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * that was used for QP number (QPN) tracking.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor while ((entry = (tavor_qpn_entry_t *)avl_destroy_nodes(
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Destroy the lock used for QP number (QPN) AVL tree access */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * tavor_qphdl_from_qpnum()
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Context: Can be called from interrupt or base context.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * This routine is important because changing the unconstrained
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * portion of the QP number is critical to the detection of a
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * potential race condition in the QP event handler code (i.e. the case
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * where a QP is freed and alloc'd again before an event for the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * "old" QP can be handled).
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * While this is not a perfect solution (not sure that one exists)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * it does help to mitigate the chance that this race condition will
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * cause us to deliver a "stale" event to the new QP owner. Note:
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * this solution does not scale well because the number of constrained
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * bits increases (and, hence, the number of unconstrained bits
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * decreases) as the number of supported QPs grows. For small and
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * intermediate values, it should hopefully provide sufficient
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * protection.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylortavor_qphdl_from_qpnum(tavor_state_t *state, uint_t qpnum)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Calculate the QP table index from the qpnum */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor qpmask = (1 << state->ts_cfg_profile->cp_log_num_qp) - 1;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * tavor_special_qp_rsrc_alloc
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Context: Can be called from interrupt or base context.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylortavor_special_qp_rsrc_alloc(tavor_state_t *state, ibt_sqp_type_t type,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Check here to see if the driver has been configured
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * to instruct the Tavor firmware to handle all incoming
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * SMP messages (i.e. messages sent to SMA). If so,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * then we will treat QP0 as if it has already been
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * allocated (for internal use). Otherwise, if we allow
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the allocation to happen, it will cause unexpected
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * behaviors (e.g. Tavor SMA becomes unresponsive).
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor if (state->ts_cfg_profile->cp_qp0_agents_in_fw != 0) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TNF_PROBE_0(tavor_special_qp0_alloc_already_in_fw,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * If this is the first QP0 allocation, then post
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * a CONF_SPECIAL_QP firmware command
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor if ((flags & TAVOR_SPECIAL_QP0_RSRC_MASK) == 0) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Now check (and, if necessary, modify) the flags to indicate
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * whether the allocation was successful
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * If this is the first QP1 allocation, then post
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * a CONF_SPECIAL_QP firmware command
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor if ((flags & TAVOR_SPECIAL_QP1_RSRC_MASK) == 0) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Now check (and, if necessary, modify) the flags to indicate
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * whether the allocation was successful
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * tavor_special_qp_rsrc_free
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Context: Can be called from interrupt or base context.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylortavor_special_qp_rsrc_free(tavor_state_t *state, ibt_sqp_type_t type,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * If this is the last QP0 free, then post a CONF_SPECIAL_QP
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * firmware command
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor if ((flags & TAVOR_SPECIAL_QP0_RSRC_MASK) == 0) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor status = tavor_conf_special_qp_cmd_post(state, 0,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * If this is the last QP1 free, then post a CONF_SPECIAL_QP
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * firmware command
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor if ((flags & TAVOR_SPECIAL_QP1_RSRC_MASK) == 0) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor status = tavor_conf_special_qp_cmd_post(state, 0,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * tavor_qp_sgl_to_logwqesz()
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Context: Can be called from interrupt or base context.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylortavor_qp_sgl_to_logwqesz(tavor_state_t *state, uint_t num_sgl,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor tavor_qp_wq_type_t wq_type, uint_t *logwqesz, uint_t *max_sgl)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Use requested maximum SGL to calculate max descriptor size
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * (while guaranteeing that the descriptor size is a
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * power-of-2 cachelines).
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor max_size = (TAVOR_QP_WQE_MLX_SND_HDRS + (num_sgl << 4));
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Make sure descriptor is at least the minimum size */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Calculate actual number of SGL (given WQE size) */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor actual_sgl = ((1 << log2) - TAVOR_QP_WQE_MLX_SND_HDRS) >> 4;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Same as above (except for Recv WQEs)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor max_size = (TAVOR_QP_WQE_MLX_RCV_HDRS + (num_sgl << 4));
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Make sure descriptor is at least the minimum size */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Calculate actual number of SGL (given WQE size) */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor actual_sgl = ((1 << log2) - TAVOR_QP_WQE_MLX_RCV_HDRS) >> 4;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Same as above (except for MLX transport WQEs). For these
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * WQEs we have to account for the space consumed by the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * "inline" packet headers. (This is smaller than for QP1
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * below because QP0 is not allowed to send packets with a GRH.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor max_size = (TAVOR_QP_WQE_MLX_QP0_HDRS + (num_sgl << 4));
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Make sure descriptor is at least the minimum size */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Calculate actual number of SGL (given WQE size) */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor actual_sgl = ((1 << log2) - TAVOR_QP_WQE_MLX_QP0_HDRS) >> 4;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Same as above. For these WQEs we again have to account for
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the space consumed by the "inline" packet headers. (This
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * is larger than for QP0 above because we have to account for
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the possibility of a GRH in each packet - and this
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * introduces an alignment issue that causes us to consume
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * an additional 8 bytes).
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor max_size = (TAVOR_QP_WQE_MLX_QP1_HDRS + (num_sgl << 4));
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Make sure descriptor is at least the minimum size */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Calculate actual number of SGL (given WQE size) */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor actual_sgl = ((1 << log2) - TAVOR_QP_WQE_MLX_QP1_HDRS) >> 4;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TAVOR_WARNING(state, "unexpected work queue type");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TNF_PROBE_0(tavor_qp_sgl_to_logwqesz_inv_wqtype_fail,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Fill in the return values */