9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * CDDL HEADER START
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * The contents of this file are subject to the terms of the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Common Development and Distribution License (the "License").
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * You may not use this file except in compliance with the License.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * See the License for the specific language governing permissions
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * and limitations under the License.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * When distributing Covered Code, include this CDDL HEADER in each
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * If applicable, add the following below this CDDL HEADER, with the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * fields enclosed by brackets "[]" replaced with your own identifying
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * information: Portions Copyright [yyyy] [name of copyright owner]
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * CDDL HEADER END
36e3c6951df011a6cdb6b112be41a14ce9d1cf20Pramod Gunjikar * Copyright 2009 Sun Microsystems, Inc. All rights reserved.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Use is subject to license terms.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Tavor Shared Receive Queue Processing Routines
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Implements all the routines necessary for allocating, freeing, querying,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * modifying and posting shared receive queues.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic void tavor_srq_sgl_to_logwqesz(tavor_state_t *state, uint_t num_sgl,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor tavor_qp_wq_type_t wq_type, uint_t *logwqesz, uint_t *max_sgl);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * tavor_srq_alloc()
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Context: Can be called only from user or kernel context.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylortavor_srq_alloc(tavor_state_t *state, tavor_srq_info_t *srqinfo,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Check the "options" flag. Currently this flag tells the driver
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * whether or not the SRQ's work queues should be come from normal
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * system memory or whether they should be allocated from DDR memory.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Extract the necessary info from the tavor_srq_info_t structure
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Determine whether SRQ is being allocated for userland access or
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * whether it is being allocated for kernel access. If the SRQ is
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * being allocated for userland access, then lookup the UAR doorbell
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * page number for the current process. Note: If this is not found
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * (e.g. if the process has not previously open()'d the Tavor driver),
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * then an error is returned.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor srq_is_umap = (flags & IBT_SRQ_USER_MAP) ? 1 : 0;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor status = tavor_umap_db_find(state->ts_instance, ddi_get_pid(),
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Set "status" and "errormsg" and goto failure */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TAVOR_TNF_FAIL(IBT_INVALID_PARAM, "failed UAR page");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor uarpg = ((tavor_rsrc_t *)(uintptr_t)value)->tr_indx;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Increase PD refcnt */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Allocate an SRQ context entry */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor status = tavor_rsrc_alloc(state, TAVOR_SRQC, 1, sleepflag, &srqc);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Set "status" and "errormsg" and goto failure */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TAVOR_TNF_FAIL(IBT_INSUFF_RESOURCE, "failed SRQ context");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Allocate the SRQ Handle entry */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor status = tavor_rsrc_alloc(state, TAVOR_SRQHDL, 1, sleepflag, &rsrc);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Set "status" and "errormsg" and goto failure */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TAVOR_TNF_FAIL(IBT_INSUFF_RESOURCE, "failed SRQ handle");
36e3c6951df011a6cdb6b112be41a14ce9d1cf20Pramod Gunjikar srq->srq_srqnum = srqc->tr_indx; /* just use index */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * If this will be a user-mappable SRQ, then allocate an entry for
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the "userland resources database". This will later be added to
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the database (after all further SRQ operations are successful).
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * If we fail here, we must undo the reference counts and the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * previous resource allocation.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor umapdb = tavor_umap_db_alloc(state->ts_instance,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Set "status" and "errormsg" and goto failure */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TAVOR_TNF_FAIL(IBT_INSUFF_RESOURCE, "failed umap add");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Calculate the appropriate size for the SRQ.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Note: All Tavor SRQs must be a power-of-2 in size. Also
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * they may not be any smaller than TAVOR_SRQ_MIN_SIZE. This step
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * is to round the requested size up to the next highest power-of-2
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor sizes->srq_wr_sz = max(sizes->srq_wr_sz, TAVOR_SRQ_MIN_SIZE);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Next we verify that the rounded-up size is valid (i.e. consistent
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * with the device limits and/or software-configured limits). If not,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * then obviously we have a lot of cleanup to do before returning.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor if (log_srq_size > state->ts_cfg_profile->cp_log_max_srq_sz) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Set "status" and "errormsg" and goto failure */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TAVOR_TNF_FAIL(IBT_HCA_WR_EXCEEDED, "max SRQ size");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Next we verify that the requested number of SGL is valid (i.e.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * consistent with the device limits and/or software-configured
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * limits). If not, then obviously the same cleanup needs to be done.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor max_sgl = state->ts_cfg_profile->cp_srq_max_sgl;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Set "status" and "errormsg" and goto failure */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TAVOR_TNF_FAIL(IBT_HCA_SGL_EXCEEDED, "max SRQ SGL");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Determine the SRQ's WQE sizes. This depends on the requested
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * number of SGLs. Note: This also has the side-effect of
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * calculating the real number of SGLs (for the calculated WQE size)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor tavor_srq_sgl_to_logwqesz(state, sizes->srq_sgl_sz,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Allocate the memory for SRQ work queues. Note: The location from
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * which we will allocate these work queues has been passed in through
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the tavor_qp_options_t structure. Since Tavor work queues are not
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * allowed to cross a 32-bit (4GB) boundary, the alignment of the work
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * queue memory is very important. We used to allocate work queues
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * (the combined receive and send queues) so that they would be aligned
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * on their combined size. That alignment guaranteed that they would
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * never cross the 4GB boundary (Tavor work queues are on the order of
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * MBs at maximum). Now we are able to relax this alignment constraint
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * by ensuring that the IB address assigned to the queue memory (as a
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * result of the tavor_mr_register() call) is offset from zero.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Previously, we had wanted to use the ddi_dma_mem_alloc() routine to
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * guarantee the alignment, but when attempting to use IOMMU bypass
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * mode we found that we were not allowed to specify any alignment that
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * was more restrictive than the system page size. So we avoided this
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * constraint by passing two alignment values, one for the memory
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * allocation itself and the other for the DMA handle (for later bind).
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * This used to cause more memory than necessary to be allocated (in
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * order to guarantee the more restrictive alignment contraint). But
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * be guaranteeing the zero-based IB virtual address for the queue, we
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * are able to conserve this memory.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Note: If SRQ is not user-mappable, then it may come from either
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * kernel system memory or from HCA-attached local DDR memory.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Note2: We align this queue on a pagesize boundary. This is required
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * to make sure that all the resulting IB addresses will start at 0, for
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * a zero-based queue. By making sure we are aligned on at least a
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * page, any offset we use into our queue will be the same as when we
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * perform tavor_srq_modify() operations later.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor srq->srq_wqinfo.qa_size = (1 << log_srq_size) * wqesz;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor srq->srq_wqinfo.qa_location = TAVOR_QUEUE_LOCATION_USERLAND;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor status = tavor_queue_alloc(state, &srq->srq_wqinfo, sleepflag);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Set "status" and "errormsg" and goto failure */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TAVOR_TNF_FAIL(IBT_INSUFF_RESOURCE, "failed srq");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor buf = (uint32_t *)srq->srq_wqinfo.qa_buf_aligned;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Register the memory for the SRQ work queues. The memory for the SRQ
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * must be registered in the Tavor TPT tables. This gives us the LKey
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * to specify in the SRQ context later. Note: If the work queue is to
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * be allocated from DDR memory, then only a "bypass" mapping is
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * appropriate. And if the SRQ memory is user-mappable, then we force
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * DDI_DMA_CONSISTENT mapping. Also, in order to meet the alignment
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * restriction, we pass the "mro_bind_override_addr" flag in the call
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * to tavor_mr_register(). This guarantees that the resulting IB vaddr
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * will be zero-based (modulo the offset into the first page). If we
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * fail here, we still have the bunch of resource and reference count
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * cleanup to do.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor flag = (sleepflag == TAVOR_SLEEP) ? IBT_MR_SLEEP :
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor mr_attr.mr_flags = flag | IBT_MR_ENABLE_LOCAL_WRITE;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor mr_op.mro_bind_type = state->ts_cfg_profile->cp_iommu_bypass;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor if (wq_location == TAVOR_QUEUE_LOCATION_NORMAL) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor mr_op.mro_bind_dmahdl = srq->srq_wqinfo.qa_dmahdl;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor status = tavor_mr_register(state, pd, &mr_attr, &mr, &mr_op);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Set "status" and "errormsg" and goto failure */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TAVOR_TNF_FAIL(IBT_INSUFF_RESOURCE, "failed register mr");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Calculate the offset between the kernel virtual address space
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * and the IB virtual address space. This will be used when
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * posting work requests to properly initialize each WQE.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor srq_desc_off = (uint64_t)(uintptr_t)srq->srq_wqinfo.qa_buf_aligned -
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Create WQL and Wridlist for use by this SRQ
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor srq->srq_wrid_wql = tavor_wrid_wql_create(state);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Set "status" and "errormsg" and goto failure */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TAVOR_TNF_FAIL(IBT_INSUFF_RESOURCE, "failed wql create");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*(srq->srq_wrid_wql)))
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor srq->srq_wridlist = tavor_wrid_get_list(1 << log_srq_size);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Set "status" and "errormsg" and goto failure */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TAVOR_TNF_FAIL(IBT_INSUFF_RESOURCE, "failed wridlist create");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*(srq->srq_wridlist)))
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Fill in all the return arguments (if necessary). This includes
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * real queue size and real SGLs.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Fill in the SRQC entry. This is the final step before passing
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * ownership of the SRQC entry to the Tavor hardware. We use all of
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the information collected/calculated above to fill in the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * requisite portions of the SRQC. Note: If this SRQ is going to be
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * used for userland access, then we need to set the UAR page number
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * appropriately (otherwise it's a "don't care")
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Write the SRQC entry to hardware. Lastly, we pass ownership of
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the entry to the hardware (using the Tavor SW2HW_SRQ firmware
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * command). Note: In general, this operation shouldn't fail. But
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * if it does, we have to undo everything we've done above before
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * returning error.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor status = tavor_cmn_ownership_cmd_post(state, SW2HW_SRQ, &srqc_entry,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor cmn_err(CE_CONT, "Tavor: SW2HW_SRQ command failed: %08x\n",
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Set "status" and "errormsg" and goto failure */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TAVOR_TNF_FAIL(IBT_FAILURE, "tavor SW2HW_SRQ command");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Fill in the rest of the Tavor SRQ handle. We can update
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the following fields for use in further operations on the SRQ.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor srq->srq_uarpg = (srq->srq_is_umap) ? uarpg : 0;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor srq->srq_real_sizes.srq_wr_sz = (1 << log_srq_size);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor srq->srq_real_sizes.srq_sgl_sz = srq->srq_wq_sgl;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Determine if later ddi_dma_sync will be necessary */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor srq->srq_sync = TAVOR_SRQ_IS_SYNC_REQ(state, srq->srq_wqinfo);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Put SRQ handle in Tavor SRQNum-to-SRQhdl list. Then fill in the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * "srqhdl" and return success
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ASSERT(state->ts_srqhdl[srqc->tr_indx] == NULL);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * If this is a user-mappable SRQ, then we need to insert the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * previously allocated entry into the "userland resources database".
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * This will allow for later lookup during devmap() (i.e. mmap())
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor tavor_wrid_list_srq_init(srq->srq_wridlist, srq, 0);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * The following is cleanup for all possible failure cases in this routine
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor kmem_free(srq->srq_wridlist->wl_wre, srq->srq_wridlist->wl_size *
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor kmem_free(srq->srq_wridlist, sizeof (tavor_wrid_list_hdr_t));
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor if (tavor_mr_deregister(state, &mr, TAVOR_MR_DEREG_ALL,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TAVOR_SLEEPFLAG_FOR_CONTEXT()) != DDI_SUCCESS) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TAVOR_WARNING(state, "failed to deregister SRQ memory");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TNF_PROBE_1(tavor_srq_alloc_fail, TAVOR_TNF_ERROR, "",
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * tavor_srq_free()
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Context: Can be called only from user or kernel context.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor/* ARGSUSED */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylortavor_srq_free(tavor_state_t *state, tavor_srqhdl_t *srqhdl, uint_t sleepflag)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Pull all the necessary information from the Tavor Shared Receive
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Queue handle. This is necessary here because the resource for the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * SRQ handle is going to be freed up as part of this operation.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * If there are work queues still associated with the SRQ, then return
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * an error. Otherwise, we will be holding the SRQ lock.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TNF_PROBE_1(tavor_srq_free_refcnt_fail, TAVOR_TNF_ERROR, "",
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * If this was a user-mappable SRQ, then we need to remove its entry
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * from the "userland resources database". If it is also currently
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * mmap()'d out to a user process, then we need to call
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * devmap_devmem_remap() to remap the SRQ memory to an invalid mapping.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * We also need to invalidate the SRQ tracking information for the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * user mapping.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor status = tavor_umap_db_find(state->ts_instance, srq->srq_srqnum,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor MLNX_UMAP_SRQMEM_RSRC, &value, TAVOR_UMAP_DB_REMOVE,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TAVOR_WARNING(state, "failed to find in database");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor "devmap_devmem_remap()");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Put NULL into the Tavor SRQNum-to-SRQHdl list. This will allow any
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * in-progress events to detect that the SRQ corresponding to this
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * number has been freed.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*srq->srq_wridlist));
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Reclaim SRQC entry from hardware (using the Tavor HW2SW_SRQ
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * firmware command). If the ownership transfer fails for any reason,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * then it is an indication that something (either in HW or SW) has
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * gone seriously wrong.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor status = tavor_cmn_ownership_cmd_post(state, HW2SW_SRQ, &srqc_entry,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TAVOR_WARNING(state, "failed to reclaim SRQC ownership");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor cmn_err(CE_CONT, "Tavor: HW2SW_SRQ command failed: %08x\n",
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Deregister the memory for the Shared Receive Queue. If this fails
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * for any reason, then it is an indication that something (either
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * in HW or SW) has gone seriously wrong. So we print a warning
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * message and return.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor status = tavor_mr_deregister(state, &mr, TAVOR_MR_DEREG_ALL,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TAVOR_WARNING(state, "failed to deregister SRQ memory");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TNF_PROBE_0(tavor_srq_free_dereg_mr_fail, TAVOR_TNF_ERROR, "");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Calculate the size and free the wridlist container */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor kmem_free(srq->srq_wridlist, sizeof (tavor_wrid_list_hdr_t));
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Release reference to WQL; If this is the last reference,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * this call also has the side effect of freeing up the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * 'srq_wrid_wql' memory.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Free the memory for the SRQ */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Free the Tavor SRQ Handle */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Free the SRQC entry resource */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Decrement the reference count on the protection domain (PD) */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Set the srqhdl pointer to NULL and return success */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * tavor_srq_modify()
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Context: Can be called only from user or kernel context.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylortavor_srq_modify(tavor_state_t *state, tavor_srqhdl_t srq, uint_t size,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Check the "inddr" flag. This flag tells the driver whether or not
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the SRQ's work queues should be come from normal system memory or
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * whether they should be allocated from DDR memory.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor wq_location = state->ts_cfg_profile->cp_srq_wq_inddr;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * If size requested is larger than device capability, return
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Insufficient Resources
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor max_srq_size = (1 << state->ts_cfg_profile->cp_log_max_srq_sz);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TNF_PROBE_0(tavor_srq_modify_size_larger_than_maxsize,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Calculate the appropriate size for the SRQ.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Note: All Tavor SRQs must be a power-of-2 in size. Also
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * they may not be any smaller than TAVOR_SRQ_MIN_SIZE. This step
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * is to round the requested size up to the next highest power-of-2
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Next we verify that the rounded-up size is valid (i.e. consistent
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * with the device limits and/or software-configured limits).
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor if (log_srq_size > state->ts_cfg_profile->cp_log_max_srq_sz) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Set "status" and "errormsg" and goto failure */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TAVOR_TNF_FAIL(IBT_HCA_WR_EXCEEDED, "max SRQ size");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Allocate the memory for newly resized Shared Receive Queue.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Note: If SRQ is not user-mappable, then it may come from either
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * kernel system memory or from HCA-attached local DDR memory.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Note2: We align this queue on a pagesize boundary. This is required
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * to make sure that all the resulting IB addresses will start at 0,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * for a zero-based queue. By making sure we are aligned on at least a
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * page, any offset we use into our queue will be the same as it was
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * when we allocated it at tavor_srq_alloc() time.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor new_srqinfo.qa_size = (1 << log_srq_size) * wqesz;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor new_srqinfo.qa_location = TAVOR_QUEUE_LOCATION_USERLAND;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor status = tavor_queue_alloc(state, &new_srqinfo, sleepflag);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Set "status" and "errormsg" and goto failure */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TAVOR_TNF_FAIL(IBT_INSUFF_RESOURCE, "failed srq");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Allocate the memory for the new WRE list. This will be used later
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * when we resize the wridlist based on the new SRQ size.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor wre_new = (tavor_wrid_entry_t *)kmem_zalloc((1 << log_srq_size) *
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Set "status" and "errormsg" and goto failure */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor "failed wre_new alloc");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Fill in the "bind" struct. This struct provides the majority
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * of the information that will be used to distinguish between an
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * "addr" binding (as is the case here) and a "buf" binding (see
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * below). The "bind" struct is later passed to tavor_mr_mem_bind()
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * which does most of the "heavy lifting" for the Tavor memory
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * registration routines.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor bind.bi_flags = sleepflag == TAVOR_SLEEP ? IBT_MR_SLEEP :
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor bind.bi_bypass = state->ts_cfg_profile->cp_iommu_bypass;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor if (wq_location == TAVOR_QUEUE_LOCATION_NORMAL) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor status = tavor_mr_mtt_bind(state, &bind, new_srqinfo.qa_dmahdl, &mtt,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Set "status" and "errormsg" and goto failure */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Calculate the offset between the kernel virtual address space
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * and the IB virtual address space. This will be used when
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * posting work requests to properly initialize each WQE.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Note: bind addr is zero-based (from alloc) so we calculate the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * correct new offset here.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor bind.bi_addr = bind.bi_addr & ((1 << mtt_pgsize_bits) - 1);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor srq_desc_off = (uint64_t)(uintptr_t)new_srqinfo.qa_buf_aligned -
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Get the base address for the MTT table. This will be necessary
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * below when we are modifying the MPT entry.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor mtt_ddrbaseaddr = (uint64_t)(uintptr_t)rsrc_pool->rsrc_ddr_offset;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Fill in the MPT entry. This is the final step before passing
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * ownership of the MPT entry to the Tavor hardware. We use all of
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the information collected/calculated above to fill in the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * requisite portions of the MPT.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor mtt_addr = mtt_ddrbaseaddr + (mtt->tr_indx << TAVOR_MTT_SIZE_SHIFT);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Now we grab the SRQ lock. Since we will be updating the actual
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * SRQ location and the producer/consumer indexes, we should hold
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * We do a TAVOR_NOSLEEP here (and below), though, because we are
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * holding the "srq_lock" and if we got raised to interrupt level
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * by priority inversion, we would not want to block in this routine
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * waiting for success.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Copy old entries to new buffer
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor bcopy(srq->srq_wq_buf, buf, srq_old_bufsz * wqesz);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Determine if later ddi_dma_sync will be necessary */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor srq_sync = TAVOR_SRQ_IS_SYNC_REQ(state, srq->srq_wqinfo);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Sync entire "new" SRQ for use by hardware (if necessary) */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Setup MPT information for use in the MODIFY_MPT command
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * MODIFY_MPT
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * If this fails for any reason, then it is an indication that
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * something (either in HW or SW) has gone seriously wrong. So we
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * print a warning message and return.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor status = tavor_modify_mpt_cmd_post(state, &mpt_entry, mpt->tr_indx,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor cmn_err(CE_CONT, "Tavor: MODIFY_MPT command failed: %08x\n",
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TNF_PROBE_1(tavor_mr_common_reg_sw2hw_mpt_cmd_fail,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TAVOR_TNF_FAIL(status, "MODIFY_MPT command failed");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor (void) tavor_mr_mtt_unbind(state, &srq->srq_mrhdl->mr_bindinfo,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Update the Tavor Shared Receive Queue handle with all the new
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * information. At the same time, save away all the necessary
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * information for freeing up the old resources
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Now set the new info */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor bcopy(&bind, &srq->srq_mrhdl->mr_bindinfo, sizeof (tavor_bind_info_t));
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor srq->srq_real_sizes.srq_wr_sz = (1 << log_srq_size);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Update MR mtt pagesize */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Initialize new wridlist, if needed.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * If a wridlist already is setup on an SRQ (the QP associated with an
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * SRQ has moved "from_reset") then we must update this wridlist based
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * on the new SRQ size. We allocate the new size of Work Request ID
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Entries, copy over the old entries to the new list, and
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * re-initialize the srq wridlist in non-umap case
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Setup new sizes in wre */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor tavor_wrid_list_srq_init(srq->srq_wridlist, srq,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * If "old" SRQ was a user-mappable SRQ that is currently mmap()'d out
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * to a user process, then we need to call devmap_devmem_remap() to
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * invalidate the mapping to the SRQ memory. We also need to
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * invalidate the SRQ tracking information for the user mapping.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Note: On failure, the remap really shouldn't ever happen. So, if it
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * does, it is an indication that something has gone seriously wrong.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * So we print a warning message and return error (knowing, of course,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * that the "old" SRQ memory will be leaked)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor if ((srq->srq_is_umap) && (srq->srq_umap_dhp != NULL)) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor state->ts_dip, 0, 0, srq->srq_wqinfo.qa_size, maxprot,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor "devmap_devmem_remap()");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* We can, however, free the memory for old wre */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Drop the SRQ lock now. The only thing left to do is to free up
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the old resources.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Unbind the MTT entries.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor status = tavor_mr_mtt_unbind(state, &old_bind, old_mtt);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TAVOR_WARNING(state, "failed to unbind old SRQ memory");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Set "status" and "errormsg" and goto failure */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor "failed to unbind (old)");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Free the memory for old wre */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Free the memory for the old SRQ */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Fill in the return arguments (if necessary). This includes the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * real new completion queue size.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TNF_PROBE_1(tavor_srq_modify_fail, TAVOR_TNF_ERROR, "",
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * tavor_srq_refcnt_inc()
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Context: Can be called from interrupt or base context.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TNF_PROBE_1_DEBUG(tavor_srq_refcnt_inc, TAVOR_TNF_TRACE, "",
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * tavor_srq_refcnt_dec()
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Context: Can be called from interrupt or base context.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TNF_PROBE_1_DEBUG(tavor_srq_refcnt_dec, TAVOR_TNF_TRACE, "",
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * tavor_srqhdl_from_srqnum()
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Context: Can be called from interrupt or base context.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * This routine is important because changing the unconstrained
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * portion of the SRQ number is critical to the detection of a
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * potential race condition in the SRQ handler code (i.e. the case
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * where a SRQ is freed and alloc'd again before an event for the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * "old" SRQ can be handled).
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * While this is not a perfect solution (not sure that one exists)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * it does help to mitigate the chance that this race condition will
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * cause us to deliver a "stale" event to the new SRQ owner. Note:
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * this solution does not scale well because the number of constrained
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * bits increases (and, hence, the number of unconstrained bits
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * decreases) as the number of supported SRQ grows. For small and
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * intermediate values, it should hopefully provide sufficient
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * protection.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylortavor_srqhdl_from_srqnum(tavor_state_t *state, uint_t srqnum)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Calculate the SRQ table index from the srqnum */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor srqmask = (1 << state->ts_cfg_profile->cp_log_num_srq) - 1;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * tavor_srq_sgl_to_logwqesz()
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Context: Can be called from interrupt or base context.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylortavor_srq_sgl_to_logwqesz(tavor_state_t *state, uint_t num_sgl,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor tavor_qp_wq_type_t wq_type, uint_t *logwqesz, uint_t *max_sgl)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Use requested maximum SGL to calculate max descriptor size
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * (while guaranteeing that the descriptor size is a
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * power-of-2 cachelines).
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor max_size = (TAVOR_QP_WQE_MLX_RCV_HDRS + (num_sgl << 4));
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Make sure descriptor is at least the minimum size */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Calculate actual number of SGL (given WQE size) */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor actual_sgl = ((1 << log2) - TAVOR_QP_WQE_MLX_RCV_HDRS) >> 4;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TAVOR_WARNING(state, "unexpected work queue type");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TNF_PROBE_0(tavor_srq_sgl_to_logwqesz_inv_wqtype_fail,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Fill in the return values */