9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * CDDL HEADER START
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * The contents of this file are subject to the terms of the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Common Development and Distribution License (the "License").
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * You may not use this file except in compliance with the License.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * See the License for the specific language governing permissions
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * and limitations under the License.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * When distributing Covered Code, include this CDDL HEADER in each
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * If applicable, add the following below this CDDL HEADER, with the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * fields enclosed by brackets "[]" replaced with your own identifying
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * information: Portions Copyright [yyyy] [name of copyright owner]
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * CDDL HEADER END
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Copyright 2009 Sun Microsystems, Inc. All rights reserved.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Use is subject to license terms.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Tavor Completion Queue Processing Routines
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Implements all the routines necessary for allocating, freeing, resizing,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * and handling the completion type events that the Tavor hardware can
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic void tavor_cq_doorbell(tavor_state_t *state, uint32_t cq_cmd,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic int tavor_cq_cqe_consume(tavor_state_t *state, tavor_cqhdl_t cq,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic int tavor_cq_errcqe_consume(tavor_state_t *state, tavor_cqhdl_t cq,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic void tavor_cqe_sync(tavor_cqhdl_t cq, tavor_hw_cqe_t *cqe,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic void tavor_cq_resize_helper(tavor_cqhdl_t cq, tavor_hw_cqe_t *new_cqbuf,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * tavor_cq_alloc()
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Context: Can be called only from user or kernel context.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylortavor_cq_alloc(tavor_state_t *state, ibt_cq_hdl_t ibt_cqhdl,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ibt_cq_attr_t *cq_attr, uint_t *actual_size, tavor_cqhdl_t *cqhdl,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Determine whether CQ is being allocated for userland access or
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * whether it is being allocated for kernel access. If the CQ is
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * being allocated for userland access, then lookup the UAR doorbell
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * page number for the current process. Note: If this is not found
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * (e.g. if the process has not previously open()'d the Tavor driver),
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * then an error is returned.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor cq_is_umap = (cq_attr->cq_flags & IBT_CQ_USER_MAP) ? 1 : 0;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor status = tavor_umap_db_find(state->ts_instance, ddi_get_pid(),
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Set "status" and "errormsg" and goto failure */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TAVOR_TNF_FAIL(IBT_INVALID_PARAM, "failed UAR page");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor uarpg = ((tavor_rsrc_t *)(uintptr_t)value)->tr_indx;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Use the internal protection domain (PD) for setting up CQs */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Increment the reference count on the protection domain (PD) */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Allocate an CQ context entry. This will be filled in with all
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the necessary parameters to define the Completion Queue. And then
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * ownership will be passed to the hardware in the final step
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * below. If we fail here, we must undo the protection domain
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * reference count.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor status = tavor_rsrc_alloc(state, TAVOR_CQC, 1, sleepflag, &cqc);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Set "status" and "errormsg" and goto failure */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TAVOR_TNF_FAIL(IBT_INSUFF_RESOURCE, "failed CQ context");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Allocate the software structure for tracking the completion queue
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * (i.e. the Tavor Completion Queue handle). If we fail here, we must
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * undo the protection domain reference count and the previous
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * resource allocation.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor status = tavor_rsrc_alloc(state, TAVOR_CQHDL, 1, sleepflag, &rsrc);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Set "status" and "errormsg" and goto failure */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TAVOR_TNF_FAIL(IBT_INSUFF_RESOURCE, "failed CQ handle");
36e3c6951df011a6cdb6b112be41a14ce9d1cf20Pramod Gunjikar /* Use the index as CQ number */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * If this will be a user-mappable CQ, then allocate an entry for
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the "userland resources database". This will later be added to
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the database (after all further CQ operations are successful).
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * If we fail here, we must undo the reference counts and the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * previous resource allocation.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor umapdb = tavor_umap_db_alloc(state->ts_instance, cq->cq_cqnum,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor MLNX_UMAP_CQMEM_RSRC, (uint64_t)(uintptr_t)rsrc);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Set "status" and "errormsg" and goto failure */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TAVOR_TNF_FAIL(IBT_INSUFF_RESOURCE, "failed umap add");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Calculate the appropriate size for the completion queue.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Note: All Tavor CQs must be a power-of-2 minus 1 in size. Also
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * they may not be any smaller than TAVOR_CQ_MIN_SIZE. This step is
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * to round the requested size up to the next highest power-of-2
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor cq_attr->cq_size = max(cq_attr->cq_size, TAVOR_CQ_MIN_SIZE);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Next we verify that the rounded-up size is valid (i.e. consistent
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * with the device limits and/or software-configured limits)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor if (log_cq_size > state->ts_cfg_profile->cp_log_max_cq_sz) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Set "status" and "errormsg" and goto failure */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TAVOR_TNF_FAIL(IBT_HCA_CQ_EXCEEDED, "max CQ size");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Allocate the memory for Completion Queue.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Note: Although we use the common queue allocation routine, we
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * always specify TAVOR_QUEUE_LOCATION_NORMAL (i.e. CQ located in
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * kernel system memory) for kernel CQs because it would be
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * inefficient to have CQs located in DDR memory. This is primarily
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * because CQs are read from (by software) more than they are written
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * to. (We always specify TAVOR_QUEUE_LOCATION_USERLAND for all
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * user-mappable CQs for a similar reason.)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * It is also worth noting that, unlike Tavor QP work queues,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * completion queues do not have the same strict alignment
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * requirements. It is sufficient for the CQ memory to be both
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * aligned to and bound to addresses which are a multiple of CQE size.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor cq->cq_cqinfo.qa_size = (1 << log_cq_size) * sizeof (tavor_hw_cqe_t);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor cq->cq_cqinfo.qa_alloc_align = sizeof (tavor_hw_cqe_t);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor cq->cq_cqinfo.qa_bind_align = sizeof (tavor_hw_cqe_t);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor cq->cq_cqinfo.qa_location = TAVOR_QUEUE_LOCATION_USERLAND;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor cq->cq_cqinfo.qa_location = TAVOR_QUEUE_LOCATION_NORMAL;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor status = tavor_queue_alloc(state, &cq->cq_cqinfo, sleepflag);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Set "status" and "errormsg" and goto failure */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TAVOR_TNF_FAIL(IBT_INSUFF_RESOURCE, "failed completion queue");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor buf = (tavor_hw_cqe_t *)cq->cq_cqinfo.qa_buf_aligned;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Initialize each of the Completion Queue Entries (CQE) by setting
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * their ownership to hardware ("owner" bit set to HW). This is in
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * preparation for the final transfer of ownership (below) of the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * CQ context itself.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Register the memory for the CQ. The memory for the CQ must
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * be registered in the Tavor TPT tables. This gives us the LKey
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * to specify in the CQ context below. Note: If this is a user-
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * mappable CQ, then we will force DDI_DMA_CONSISTENT mapping.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor flag = (sleepflag == TAVOR_SLEEP) ? IBT_MR_SLEEP : IBT_MR_NOSLEEP;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor mr_attr.mr_flags = flag | IBT_MR_ENABLE_LOCAL_WRITE;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor dma_xfer_mode = state->ts_cfg_profile->cp_streaming_consistent;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor op.mro_bind_type = state->ts_cfg_profile->cp_iommu_bypass;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor status = tavor_mr_register(state, pd, &mr_attr, &mr, &op);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Set "status" and "errormsg" and goto failure */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TAVOR_TNF_FAIL(IBT_INSUFF_RESOURCE, "failed register mr");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Determine if later ddi_dma_sync will be necessary */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor cq_sync = TAVOR_CQ_IS_SYNC_REQ(state, cq->cq_cqinfo);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Sync entire CQ for use by the hardware (if necessary). */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor (void) ddi_dma_sync(mr->mr_bindinfo.bi_dmahdl, 0,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Fill in the CQC entry. This is the final step before passing
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * ownership of the CQC entry to the Tavor hardware. We use all of
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the information collected/calculated above to fill in the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * requisite portions of the CQC. Note: If this CQ is going to be
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * used for userland access, then we need to set the UAR page number
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * appropriately (otherwise it's a "don't care")
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor cq->cq_eqnum = TAVOR_CQ_EQNUM_GET(cq->cq_cqnum);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor cq->cq_erreqnum = TAVOR_CQ_ERREQNUM_GET(cq->cq_cqnum);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Write the CQC entry to hardware. Lastly, we pass ownership of
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the entry to the hardware (using the Tavor SW2HW_CQ firmware
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * command). Note: In general, this operation shouldn't fail. But
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * if it does, we have to undo everything we've done above before
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * returning error.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor status = tavor_cmn_ownership_cmd_post(state, SW2HW_CQ, &cqc_entry,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor sizeof (tavor_hw_cqc_t), cq->cq_cqnum, sleepflag);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor cmn_err(CE_CONT, "Tavor: SW2HW_CQ command failed: %08x\n",
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Set "status" and "errormsg" and goto failure */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TAVOR_TNF_FAIL(ibc_get_ci_failure(0), "tavor SW2HW_CQ command");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Fill in the rest of the Tavor Completion Queue handle. Having
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * successfully transferred ownership of the CQC, we can update the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * following fields for use in further operations on the CQ.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor avl_create(&cq->cq_wrid_wqhdr_avl_tree, tavor_wrid_wqhdr_compare,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor sizeof (struct tavor_workq_hdr_s),
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor offsetof(struct tavor_workq_hdr_s, wq_avl_link));
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Put CQ handle in Tavor CQNum-to-CQHdl list. Then fill in the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * "actual_size" and "cqhdl" and return success
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * If this is a user-mappable CQ, then we need to insert the previously
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * allocated entry into the "userland resources database". This will
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * allow for later lookup during devmap() (i.e. mmap()) calls.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Fill in the return arguments (if necessary). This includes the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * real completion queue size.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * The following is cleanup for all possible failure cases in this routine
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor if (tavor_mr_deregister(state, &mr, TAVOR_MR_DEREG_ALL,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TAVOR_WARNING(state, "failed to deregister CQ memory");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TNF_PROBE_1(tavor_cq_alloc_fail, TAVOR_TNF_ERROR, "",
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * tavor_cq_free()
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Context: Can be called only from user or kernel context.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor/* ARGSUSED */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylortavor_cq_free(tavor_state_t *state, tavor_cqhdl_t *cqhdl, uint_t sleepflag)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Pull all the necessary information from the Tavor Completion Queue
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * handle. This is necessary here because the resource for the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * CQ handle is going to be freed up as part of this operation.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * If there are work queues still associated with the CQ, then return
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * an error. Otherwise, we will be holding the CQ lock.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TNF_PROBE_1(tavor_cq_free_refcnt_fail, TAVOR_TNF_ERROR, "",
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * If this was a user-mappable CQ, then we need to remove its entry
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * from the "userland resources database". If it is also currently
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * mmap()'d out to a user process, then we need to call
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * devmap_devmem_remap() to remap the CQ memory to an invalid mapping.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * We also need to invalidate the CQ tracking information for the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * user mapping.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor status = tavor_umap_db_find(state->ts_instance, cqnum,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor MLNX_UMAP_CQMEM_RSRC, &value, TAVOR_UMAP_DB_REMOVE,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TAVOR_WARNING(state, "failed to find in database");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor "devmap_devmem_remap()");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Put NULL into the Tavor CQNum-to-CQHdl list. This will allow any
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * in-progress events to detect that the CQ corresponding to this
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * number has been freed.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * While we hold the CQ lock, do a "forced reap" of the workQ WRID
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * list. This cleans up all the structures associated with the WRID
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * processing for this CQ. Once we complete, drop the lock and finish
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the deallocation of the CQ.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Reclaim CQC entry from hardware (using the Tavor HW2SW_CQ
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * firmware command). If the ownership transfer fails for any reason,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * then it is an indication that something (either in HW or SW) has
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * gone seriously wrong.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor status = tavor_cmn_ownership_cmd_post(state, HW2SW_CQ, &cqc_entry,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TAVOR_WARNING(state, "failed to reclaim CQC ownership");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor cmn_err(CE_CONT, "Tavor: HW2SW_CQ command failed: %08x\n",
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Deregister the memory for the Completion Queue. If this fails
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * for any reason, then it is an indication that something (either
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * in HW or SW) has gone seriously wrong. So we print a warning
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * message and return.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor status = tavor_mr_deregister(state, &mr, TAVOR_MR_DEREG_ALL,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TAVOR_WARNING(state, "failed to deregister CQ memory");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TNF_PROBE_0(tavor_cq_free_dereg_mr_fail, TAVOR_TNF_ERROR, "");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Free the memory for the CQ */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Free the Tavor Completion Queue handle */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Free up the CQC entry resource */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Decrement the reference count on the protection domain (PD) */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Set the cqhdl pointer to NULL and return success */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * tavor_cq_resize()
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Context: Can be called only from user or kernel context.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylortavor_cq_resize(tavor_state_t *state, tavor_cqhdl_t cq, uint_t req_size,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor uint_t dma_xfer_mode, cq_sync, log_cq_size, maxprot;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Use the internal protection domain (PD) for CQs */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Calculate the appropriate size for the new resized completion queue.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Note: All Tavor CQs must be a power-of-2 minus 1 in size. Also
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * they may not be any smaller than TAVOR_CQ_MIN_SIZE. This step is
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * to round the requested size up to the next highest power-of-2
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Next we verify that the rounded-up size is valid (i.e. consistent
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * with the device limits and/or software-configured limits)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor if (log_cq_size > state->ts_cfg_profile->cp_log_max_cq_sz) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Set "status" and "errormsg" and goto failure */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TAVOR_TNF_FAIL(IBT_HCA_CQ_EXCEEDED, "max CQ size");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Allocate the memory for newly resized Completion Queue.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Note: Although we use the common queue allocation routine, we
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * always specify TAVOR_QUEUE_LOCATION_NORMAL (i.e. CQ located in
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * kernel system memory) for kernel CQs because it would be
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * inefficient to have CQs located in DDR memory. This is the same
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * as we do when we first allocate completion queues primarily
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * because CQs are read from (by software) more than they are written
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * to. (We always specify TAVOR_QUEUE_LOCATION_USERLAND for all
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * user-mappable CQs for a similar reason.)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * It is also worth noting that, unlike Tavor QP work queues,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * completion queues do not have the same strict alignment
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * requirements. It is sufficient for the CQ memory to be both
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * aligned to and bound to addresses which are a multiple of CQE size.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor new_cqinfo.qa_size = (1 << log_cq_size) * sizeof (tavor_hw_cqe_t);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor new_cqinfo.qa_alloc_align = sizeof (tavor_hw_cqe_t);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor new_cqinfo.qa_bind_align = sizeof (tavor_hw_cqe_t);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor new_cqinfo.qa_location = TAVOR_QUEUE_LOCATION_USERLAND;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor new_cqinfo.qa_location = TAVOR_QUEUE_LOCATION_NORMAL;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor status = tavor_queue_alloc(state, &new_cqinfo, sleepflag);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Set "status" and "errormsg" and goto failure */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TAVOR_TNF_FAIL(IBT_INSUFF_RESOURCE, "failed completion queue");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor buf = (tavor_hw_cqe_t *)new_cqinfo.qa_buf_aligned;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Initialize each of the Completion Queue Entries (CQE) by setting
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * their ownership to hardware ("owner" bit set to HW). This is in
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * preparation for the final resize operation (below).
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Register the memory for the CQ. The memory for the CQ must
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * be registered in the Tavor TPT tables. This gives us the LKey
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * to specify in the CQ context below.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor flag = (sleepflag == TAVOR_SLEEP) ? IBT_MR_SLEEP : IBT_MR_NOSLEEP;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor mr_attr.mr_flags = flag | IBT_MR_ENABLE_LOCAL_WRITE;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor dma_xfer_mode = state->ts_cfg_profile->cp_streaming_consistent;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor op.mro_bind_type = state->ts_cfg_profile->cp_iommu_bypass;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor status = tavor_mr_register(state, pd, &mr_attr, &mr, &op);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Set "status" and "errormsg" and goto failure */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TAVOR_TNF_FAIL(IBT_INSUFF_RESOURCE, "failed register mr");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Determine if later ddi_dma_sync will be necessary */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor cq_sync = TAVOR_CQ_IS_SYNC_REQ(state, new_cqinfo);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Sync entire "new" CQ for use by hardware (if necessary) */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor (void) ddi_dma_sync(mr->mr_bindinfo.bi_dmahdl, 0,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Now we grab the CQ lock. Since we will be updating the actual
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * CQ location and the producer/consumer indexes, we should hold
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * We do a TAVOR_NOSLEEP here (and below), though, because we are
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * holding the "cq_lock" and if we got raised to interrupt level
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * by priority inversion, we would not want to block in this routine
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * waiting for success.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Determine the current CQ "consumer index".
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Note: This will depend on whether the CQ had previously been
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * mapped for user access or whether it is a kernel CQ. If this
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * is a kernel CQ, then all PollCQ() operations have come through
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the IBTF and, hence, the driver's CQ state structure will
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * contain the current consumer index. If, however, the user has
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * accessed this CQ by bypassing the driver (OS-bypass), then we
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * need to query the firmware to determine the current CQ consumer
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * index. This also assumes that the user process will not continue
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * to consume entries while at the same time doing the ResizeCQ()
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * operation. If the user process does not guarantee this, then it
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * may see duplicate or missed completions. But under no
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * circumstances should this panic the system.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor status = tavor_cmn_query_cmd_post(state, QUERY_CQ,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor cq->cq_cqnum, &cqc_entry, sizeof (tavor_hw_cqc_t),
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Query CQ has failed, drop CQ lock and cleanup */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor if (tavor_mr_deregister(state, &mr, TAVOR_MR_DEREG_ALL,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor "CQ memory");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TAVOR_WARNING(state, "failed to find in database");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Set "status" and "errormsg" and goto failure */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor "failed umap lookup");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Fill in the CQC entry. For the resize operation this is the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * final step before attempting the resize operation on the CQC entry.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * We use all of the information collected/calculated above to fill
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * in the requisite portions of the CQC.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor cqc_entry.start_addr_h = (mr->mr_bindinfo.bi_addr >> 32);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor cqc_entry.start_addr_l = (mr->mr_bindinfo.bi_addr & 0xFFFFFFFF);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Write the CQC entry to hardware. Lastly, we pass ownership of
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the entry to the hardware (using the Tavor RESIZE_CQ firmware
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * command). Note: In general, this operation shouldn't fail. But
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * if it does, we have to undo everything we've done above before
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * returning error. Also note that the status returned may indicate
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the code to return to the IBTF.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor status = tavor_resize_cq_cmd_post(state, &cqc_entry, cq->cq_cqnum,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Resize attempt has failed, drop CQ lock and cleanup */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor if (tavor_mr_deregister(state, &mr, TAVOR_MR_DEREG_ALL,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TAVOR_WARNING(state, "failed to deregister CQ memory");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor cmn_err(CE_CONT, "Tavor: RESIZE_CQ command failed: "
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * The CQ resize attempt was successful. Before dropping the CQ lock,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * copy all of the CQEs from the "old" CQ into the "new" CQ. Note:
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the Tavor firmware guarantees us that sufficient space is set aside
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * in the "new" CQ to handle any un-polled CQEs from the "old" CQ.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * The two parameters to this helper function ("old_cons_indx" and
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * "new_prod_indx") essentially indicate the starting index and number
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * of any CQEs that might remain in the "old" CQ memory.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor tavor_cq_resize_helper(cq, buf, old_cons_indx, new_prod_indx);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Sync entire "new" CQ for use by hardware (if necessary) */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor (void) ddi_dma_sync(mr->mr_bindinfo.bi_dmahdl, 0,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Update the Tavor Completion Queue handle with all the new
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * information. At the same time, save away all the necessary
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * information for freeing up the old resources
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * If "old" CQ was a user-mappable CQ that is currently mmap()'d out
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * to a user process, then we need to call devmap_devmem_remap() to
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * invalidate the mapping to the CQ memory. We also need to
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * invalidate the CQ tracking information for the user mapping.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor if ((cq->cq_is_umap) && (cq->cq_umap_dhp != NULL)) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor state->ts_dip, 0, 0, cq->cq_cqinfo.qa_size, maxprot,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor "devmap_devmem_remap()");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Drop the CQ lock now. The only thing left to do is to free up
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the old resources.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Deregister the memory for the old Completion Queue. Note: We
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * really can't return error here because we have no good way to
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * cleanup. Plus, the deregistration really shouldn't ever happen.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * So, if it does, it is an indication that something has gone
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * seriously wrong. So we print a warning message and return error
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * (knowing, of course, that the "old" CQ memory will be leaked)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor status = tavor_mr_deregister(state, &mr_old, TAVOR_MR_DEREG_ALL,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TAVOR_WARNING(state, "failed to deregister old CQ memory");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Set "status" and "errormsg" and goto failure */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor "failed deregister mr (old)");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Free the memory for the old CQ */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Fill in the return arguments (if necessary). This includes the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * real new completion queue size.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TNF_PROBE_1(tavor_cq_resize_fail, TAVOR_TNF_ERROR, "",
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * tavor_cq_notify()
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Context: Can be called from interrupt or base context.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylortavor_cq_notify(tavor_state_t *state, tavor_cqhdl_t cq,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Determine if we are trying to get the next completion or the next
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * "solicited" completion. Then hit the appropriate doorbell.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * NOTE: Please see the comment in tavor_event.c:tavor_eq_poll
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * regarding why we do not have to do an extra PIO read here, and we
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * will not lose an event after writing this doorbell.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor tavor_cq_doorbell(state, TAVOR_CQDB_NOTIFY_CQ, cqnum,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor tavor_cq_doorbell(state, TAVOR_CQDB_NOTIFY_CQ_SOLICIT,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TNF_PROBE_1(tavor_cq_notify_invflags_fail, TAVOR_TNF_ERROR, "",
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * tavor_cq_poll()
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Context: Can be called from interrupt or base context.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylortavor_cq_poll(tavor_state_t *state, tavor_cqhdl_t cq, ibt_wc_t *wc_p,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Check for user-mappable CQ memory. Note: We do not allow kernel
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * clients to poll CQ memory that is accessible directly by the user.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * If the CQ memory is user accessible, then return an error.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Get the consumer index */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Calculate the wrap around mask. Note: This operation only works
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * because all Tavor completion queues have power-of-2 sizes
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Calculate the pointer to the first CQ entry */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Sync the current CQE to read */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Keep pulling entries from the CQ until we find an entry owned by
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the hardware. As long as there the CQE's owned by SW, process
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * each entry by calling tavor_cq_cqe_consume() and updating the CQ
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * consumer index. Note: We only update the consumer index if
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * tavor_cq_cqe_consume() returns TAVOR_CQ_SYNC_AND_DB. Otherwise,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * it indicates that we are going to "recycle" the CQE (probably
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * because it is a error CQE and corresponds to more than one
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * completion).
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Reset entry to hardware ownership */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Sync the current CQE for device */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Increment the consumer index */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Update the pointer to the next CQ entry */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Sync the next CQE to read */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * If we have run out of space to store work completions,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * then stop and return the ones we have pulled of the CQ.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Now we only ring the doorbell (to update the consumer index) if
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * we've actually consumed a CQ entry. If we have, for example,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * pulled from a CQE that we are still in the process of "recycling"
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * for error purposes, then we would not update the consumer index.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor if ((polled_cnt != 0) && (cq->cq_consindx != cons_indx)) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Post doorbell to update the consumer index. Doorbell
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * value indicates number of entries consumed (minus 1)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor num_to_increment = (cons_indx - cq->cq_consindx) - 1;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor num_to_increment = ((cons_indx + cq->cq_bufsz) -
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor tavor_cq_doorbell(state, TAVOR_CQDB_INCR_CONSINDX,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor } else if (polled_cnt == 0) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * If the CQ is empty, we can try to free up some of the WRID
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * list containers. See tavor_wr.c for more details on this
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * operation.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Set "num_polled" (if necessary) */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Set CQ_EMPTY condition if needed, otherwise return success */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Check if the system is currently panicking. If it is, then call
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the Tavor interrupt service routine. This step is necessary here
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * because we might be in a polled I/O mode and without the call to
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * tavor_isr() - and its subsequent calls to poll and rearm each
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * event queue - we might overflow our EQs and render the system
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * unable to sync/dump.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor (void) tavor_isr((caddr_t)state, (caddr_t)NULL);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * tavor_cq_handler()
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Context: Only called from interrupt context
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylortavor_cq_handler(tavor_state_t *state, tavor_eqhdl_t eq,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TNF_PROBE_0(tavor_cq_handler_eq_overflow_condition,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Get the CQ handle from CQ number in event descriptor */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Post the EQ doorbell to move the CQ to the "disarmed" state.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * This operation is to enable subsequent CQ doorbells (e.g. those
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * that can be rung by tavor_cq_notify() above) to rearm the CQ.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor tavor_eq_doorbell(state, TAVOR_EQDB_DISARM_CQ, eq->eq_eqnum, cqnum);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * If the CQ handle is NULL, this is probably an indication
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * that the CQ has been freed already. In which case, we
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * should not deliver this event.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * We also check that the CQ number in the handle is the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * same as the CQ number in the event queue entry. This
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * extra check allows us to handle the case where a CQ was
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * freed and then allocated again in the time it took to
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * handle the event queue processing. By constantly incrementing
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the non-constrained portion of the CQ number every time
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * a new CQ is allocated, we mitigate (somewhat) the chance
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * that a stale event could be passed to the client's CQ
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Lastly, we check if "ts_ibtfpriv" is NULL. If it is then it
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * means that we've have either received this event before we
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * finished attaching to the IBTF or we've received it while we
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * are in the process of detaching.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * tavor_cq_err_handler()
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Context: Only called from interrupt context
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylortavor_cq_err_handler(tavor_state_t *state, tavor_eqhdl_t eq,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TNF_PROBE_0(tavor_cq_err_handler_eq_overflow_condition,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* cmn_err(CE_CONT, "CQ Error handler\n"); */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Get the CQ handle from CQ number in event descriptor */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * If the CQ handle is NULL, this is probably an indication
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * that the CQ has been freed already. In which case, we
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * should not deliver this event.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * We also check that the CQ number in the handle is the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * same as the CQ number in the event queue entry. This
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * extra check allows us to handle the case where a CQ was
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * freed and then allocated again in the time it took to
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * handle the event queue processing. By constantly incrementing
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the non-constrained portion of the CQ number every time
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * a new CQ is allocated, we mitigate (somewhat) the chance
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * that a stale event could be passed to the client's CQ
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * And then we check if "ts_ibtfpriv" is NULL. If it is then it
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * means that we've have either received this event before we
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * finished attaching to the IBTF or we've received it while we
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * are in the process of detaching.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * tavor_cq_refcnt_inc()
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Context: Can be called from interrupt or base context.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylortavor_cq_refcnt_inc(tavor_cqhdl_t cq, uint_t is_special)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Increment the completion queue's reference count. Note: In order
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * to ensure compliance with IBA C11-15, we must ensure that a given
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * CQ is not used for both special (SMI/GSI) QP and non-special QP.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * This is accomplished here by keeping track of how the referenced
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * CQ is being used.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TNF_PROBE_1_DEBUG(tavor_cq_refcnt_inc, TAVOR_TNF_TRACE, "",
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * tavor_cq_refcnt_dec()
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Context: Can be called from interrupt or base context.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Decrement the completion queue's reference count */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TNF_PROBE_1_DEBUG(tavor_cq_refcnt_dec, TAVOR_TNF_TRACE, "",
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * tavor_cq_doorbell()
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Context: Can be called from interrupt or base context.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylortavor_cq_doorbell(tavor_state_t *state, uint32_t cq_cmd, uint32_t cqn,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Build the doorbell from the parameters */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor doorbell = ((uint64_t)cq_cmd << TAVOR_CQDB_CMD_SHIFT) |
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ((uint64_t)cqn << TAVOR_CQDB_CQN_SHIFT) | cq_param;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TNF_PROBE_1_DEBUG(tavor_cq_doorbell, TAVOR_TNF_TRACE, "",
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Write the doorbell to UAR */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TAVOR_UAR_DOORBELL(state, (uint64_t *)&state->ts_uar->cq,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * tavor_cqhdl_from_cqnum()
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Context: Can be called from interrupt or base context.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * This routine is important because changing the unconstrained
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * portion of the CQ number is critical to the detection of a
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * potential race condition in the CQ handler code (i.e. the case
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * where a CQ is freed and alloc'd again before an event for the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * "old" CQ can be handled).
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * While this is not a perfect solution (not sure that one exists)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * it does help to mitigate the chance that this race condition will
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * cause us to deliver a "stale" event to the new CQ owner. Note:
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * this solution does not scale well because the number of constrained
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * bits increases (and, hence, the number of unconstrained bits
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * decreases) as the number of supported CQs grows. For small and
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * intermediate values, it should hopefully provide sufficient
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * protection.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylortavor_cqhdl_from_cqnum(tavor_state_t *state, uint_t cqnum)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Calculate the CQ table index from the cqnum */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor cqmask = (1 << state->ts_cfg_profile->cp_log_num_cq) - 1;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * tavor_cq_cqe_consume()
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Context: Can be called from interrupt or base context.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylortavor_cq_cqe_consume(tavor_state_t *state, tavor_cqhdl_t cq,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Determine if this is an "error" CQE by examining "opcode". If it
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * is an error CQE, then call tavor_cq_errcqe_consume() and return
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * whatever status it returns. Otherwise, this is a successful
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * completion.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor status = tavor_cq_errcqe_consume(state, cq, cqe, wc);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Fetch the Work Request ID using the information in the CQE.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * See tavor_wr.c for more details.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor wc->wc_id = tavor_wrid_get_entry(cq, cqe, NULL);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Parse the CQE opcode to determine completion type. This will set
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * not only the type of the completion, but also any flags that might
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * be associated with it (e.g. whether immediate data is present).
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor if (TAVOR_CQE_SENDRECV_GET(cq, cqe) != TAVOR_COMPLETION_RECV) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Send CQE */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* FALLTHROUGH */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* FALLTHROUGH */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TNF_PROBE_1(tavor_cq_cqe_consume_unknown_send_type,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Receive CQE */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* FALLTHROUGH */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Note: According to the Tavor PRM, all QP1 recv
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * completions look like the result of a Send with
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Immediate. They are not, however, (MADs are Send
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Only) so we need to check the QP number and set
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the flag only if it is non-QP1.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor if ((qpnum < qp1_indx) || (qpnum > qp1_indx + 1)) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* FALLTHROUGH */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* FALLTHROUGH */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* FALLTHROUGH */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TNF_PROBE_1(tavor_cq_cqe_consume_unknown_rcv_type,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Check for GRH, update the flags, then fill in "wc_flags" field
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * in the work completion
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* If we got here, completion status must be success */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Parse the remaining contents of the CQE into the work completion.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * This means filling in SL, QP number, SLID, immediate data, etc.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Note: Not all of these fields are valid in a given completion.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Many of them depend on the actual type of completion. So we fill
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * in all of the fields and leave it up to the IBTF and consumer to
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * sort out which are valid based on their context.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor wc->wc_immed_data = TAVOR_CQE_IMM_ETH_PKEY_CRED_GET(cq, cqe);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor wc->wc_ethertype = (wc->wc_immed_data & 0xFFFF);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Depending on whether the completion was a receive or a send
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * completion, fill in "bytes transferred" as appropriate. Also,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * if necessary, fill in the "path bits" field.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor if (TAVOR_CQE_SENDRECV_GET(cq, cqe) == TAVOR_COMPLETION_RECV) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor wc->wc_path_bits = TAVOR_CQE_PATHBITS_GET(cq, cqe);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor wc->wc_bytes_xfer = TAVOR_CQE_BYTECNT_GET(cq, cqe);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor (wc->wc_type == IBT_WRC_CSWAP) || (wc->wc_type == IBT_WRC_FADD)) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor wc->wc_bytes_xfer = TAVOR_CQE_BYTECNT_GET(cq, cqe);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * tavor_cq_errcqe_consume()
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Context: Can be called from interrupt or base context.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylortavor_cq_errcqe_consume(tavor_state_t *state, tavor_cqhdl_t cq,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Fetch the Work Request ID using the information in the CQE.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * See tavor_wr.c for more details.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor wc->wc_id = tavor_wrid_get_entry(cq, cqe, &wre);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Parse the CQE opcode to determine completion type. We know that
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the CQE is an error completion, so we extract only the completion
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * status here.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor imm_eth_pkey_cred = TAVOR_CQE_IMM_ETH_PKEY_CRED_GET(cq, cqe);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor status = imm_eth_pkey_cred >> TAVOR_CQE_ERR_STATUS_SHIFT;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * The following error codes are not supported in the Tavor driver
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * as they relate only to Reliable Datagram completion statuses:
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * case TAVOR_CQE_LOCAL_RDD_VIO_ERR:
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * case TAVOR_CQE_REM_INV_RD_REQ_ERR:
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * case TAVOR_CQE_EEC_REM_ABORTED_ERR:
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * case TAVOR_CQE_INV_EEC_NUM_ERR:
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * case TAVOR_CQE_INV_EEC_STATE_ERR:
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * case TAVOR_CQE_LOC_EEC_ERR:
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TAVOR_WARNING(state, "unknown error CQE status");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TNF_PROBE_1(tavor_cq_errcqe_consume_unknown_status,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Now we do all the checking that's necessary to handle completion
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * queue entry "recycling"
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * It is not necessary here to try to sync the WQE as we are only
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * attempting to read from the Work Queue (and hardware does not
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * write to it).
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * We can get doorbell info, WQE address, size for the next WQE
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * from the "wre" (which was filled in above in the call to the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * tavor_wrid_get_entry() routine)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor dbd = (wre.wr_signaled_dbd & TAVOR_WRID_ENTRY_DOORBELLED) ? 1 : 0;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor nextwqesize = wre.wr_wqeaddrsz & TAVOR_WQE_NDS_MASK;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Get the doorbell count from the CQE. This indicates how many
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * completions this one CQE represents.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor doorbell_cnt = imm_eth_pkey_cred & TAVOR_CQE_ERR_DBDCNT_MASK;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Determine if we're ready to consume this CQE yet or not. If the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * next WQE has size zero (i.e. no next WQE) or if the doorbell count
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * is down to zero, then this is the last/only completion represented
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * by the current CQE (return TAVOR_CQ_SYNC_AND_DB). Otherwise, the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * current CQE needs to be recycled (see below).
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor if ((nextwqesize == 0) || ((doorbell_cnt == 0) && (dbd == 1))) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Consume the CQE
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Return status to indicate that doorbell and sync may be
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * necessary.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Recycle the CQE for use in the next PollCQ() call
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Decrement the doorbell count, modify the error status,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * and update the WQE address and size (to point to the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * next WQE on the chain. Put these update entries back
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * into the CQE.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Despite the fact that we have updated the CQE, it is not
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * necessary for us to attempt to sync this entry just yet
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * as we have not changed the "hardware's view" of the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * entry (i.e. we have not modified the "owner" bit - which
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * is all that the Tavor hardware really cares about.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ((TAVOR_CQE_WR_FLUSHED_ERR << TAVOR_CQE_ERR_STATUS_SHIFT) |
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * tavor_cqe_sync()
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Context: Can be called from interrupt or base context.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylortavor_cqe_sync(tavor_cqhdl_t cq, tavor_hw_cqe_t *cqe, uint_t flag)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Determine if CQ needs to be synced or not */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Get the DMA handle from CQ context */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Calculate offset of next CQE */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor offset = (off_t)((uintptr_t)cqe - (uintptr_t)&cq->cq_buf[0]);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor status = ddi_dma_sync(dmahdl, offset, sizeof (tavor_hw_cqe_t), flag);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * tavor_cq_resize_helper()
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Context: Can be called only from user or kernel context.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylortavor_cq_resize_helper(tavor_cqhdl_t cq, tavor_hw_cqe_t *new_cqbuf,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Get the consumer index */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Calculate the wrap around mask. Note: This operation only works
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * because all Tavor completion queues have power-of-2 sizes
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Calculate the pointers to the first CQ entry (in the "old" CQ)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * and the first CQ entry in the "new" CQ
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Sync entire "old" CQ for use by software (if necessary). */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor (void) ddi_dma_sync(cq->cq_mrhdl->mr_bindinfo.bi_dmahdl,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Keep pulling entries from the "old" CQ until we find an entry owned
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * by the hardware. Process each entry by copying it into the "new"
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * CQ and updating respective indices and pointers in the "old" CQ.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor for (i = 0; i < num_newcqe; i++) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Copy this old CQE into the "new_cqe" pointer */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor bcopy(old_cqe, new_cqe, sizeof (tavor_hw_cqe_t));
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Increment the consumer index (for both CQs) */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor old_cons_indx = (old_cons_indx + 1) & wrap_around_mask;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Update the pointer to the next CQ entry */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * tavor_cq_srq_entries_flush()
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Context: Can be called from interrupt or base context.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylortavor_cq_srq_entries_flush(tavor_state_t *state, tavor_qphdl_t qp)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor uint32_t cons_indx, tail_cons_indx, wrap_around_mask;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Check for user-mapped CQ memory. Note: We do not allow kernel
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * clients to modify any userland mapping CQ. If the CQ is
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * user-mapped, then we simply return here, and this "flush" function
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * becomes a NO-OP in this case.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Get the consumer index */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Calculate the wrap around mask. Note: This operation only works
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * because all Tavor completion queues have power-of-2 sizes
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Calculate the pointer to the first CQ entry */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Sync the current CQE to read */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Loop through the CQ looking for entries owned by software. If an
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * entry is owned by software then we increment an 'outstanding_cqes'
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * count to know how many entries total we have on our CQ. We use this
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * value further down to know how many entries to loop through looking
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * for our same QP number.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* increment total cqes count */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* increment the consumer index */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor tail_cons_indx = (tail_cons_indx + 1) & wrap_around_mask;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* update the pointer to the next cq entry */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* sync the next cqe to read */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Using the 'tail_cons_indx' that was just set, we now know how many
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * total CQEs possible there are. Set the 'check_indx' and the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * 'new_indx' to the last entry identified by 'tail_cons_indx'
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor check_indx = new_indx = (tail_cons_indx - 1) & wrap_around_mask;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor for (i = 0; i < outstanding_cqes; i++) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Grab QP number from CQE */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * If the QP number is the same in the CQE as the QP that we
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * have on this SRQ, then we must free up the entry off the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * SRQ. We also make sure that the completion type is of the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * 'TAVOR_COMPLETION_RECV' type. So any send completions on
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * this CQ will be left as-is. The handling of returning
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * entries back to HW ownership happens further down.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Add back to SRQ free list */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor (void) tavor_wrid_find_match_srq(wqhdr->wq_wrid_post,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Do Copy */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Copy the CQE into the "next_cqe"
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Move index to next CQE to check */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor check_indx = (check_indx - 1) & wrap_around_mask;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Initialize removed cqes count */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* If an entry was removed */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Set current pointer back to the beginning consumer index.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * At this point, all unclaimed entries have been copied to the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * index specified by 'new_indx'. This 'new_indx' will be used
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * as the new consumer index after we mark all freed entries as
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * having HW ownership. We do that here.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Loop through all entries until we reach our new pointer */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Reset entry to hardware ownership */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Update consumer index to be the 'new_indx'. This moves it past all
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * removed entries. Because 'new_indx' is pointing to the last
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * previously valid SW owned entry, we add 1 to point the cons_indx to
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the first HW owned entry.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Now we only ring the doorbell (to update the consumer index) if
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * we've actually consumed a CQ entry. If we found no QP number
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * matches above, then we would not have removed anything. So only if
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * something was removed do we ring the doorbell.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor if ((removed_cqes != 0) && (cq->cq_consindx != cons_indx)) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Post doorbell to update the consumer index. Doorbell
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * value indicates number of entries consumed (minus 1)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor num_to_increment = (cons_indx - cq->cq_consindx) - 1;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor num_to_increment = ((cons_indx + cq->cq_bufsz) -