9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * CDDL HEADER START
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * The contents of this file are subject to the terms of the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Common Development and Distribution License (the "License").
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * You may not use this file except in compliance with the License.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * See the License for the specific language governing permissions
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * and limitations under the License.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * When distributing Covered Code, include this CDDL HEADER in each
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * If applicable, add the following below this CDDL HEADER, with the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * fields enclosed by brackets "[]" replaced with your own identifying
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * information: Portions Copyright [yyyy] [name of copyright owner]
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * CDDL HEADER END
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Hermon Completion Queue Processing Routines
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Implements all the routines necessary for allocating, freeing, resizing,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * and handling the completion type events that the Hermon hardware can
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Build the doorbell record data (low 24 bits only) */ \
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor HERMON_UAR_DB_RECORD_WRITE(cq->cq_arm_ci_vdbr, \
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic int hermon_cq_arm_doorbell(hermon_state_t *state, hermon_cqhdl_t cq,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic void hermon_arm_cq_dbr_init(hermon_dbr_t *cq_arm_dbr);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic void hermon_cq_cqe_consume(hermon_state_t *state, hermon_cqhdl_t cq,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic void hermon_cq_errcqe_consume(hermon_state_t *state, hermon_cqhdl_t cq,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * hermon_cq_alloc()
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Context: Can be called only from user or kernel context.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorhermon_cq_alloc(hermon_state_t *state, ibt_cq_hdl_t ibt_cqhdl,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ibt_cq_attr_t *cq_attr, uint_t *actual_size, hermon_cqhdl_t *cqhdl,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Determine whether CQ is being allocated for userland access or
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * whether it is being allocated for kernel access. If the CQ is
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * being allocated for userland access, then lookup the UAR
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * page number for the current process. Note: If this is not found
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * (e.g. if the process has not previously open()'d the Hermon driver),
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * then an error is returned.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor cq_is_umap = (cq_attr->cq_flags & IBT_CQ_USER_MAP) ? 1 : 0;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor status = hermon_umap_db_find(state->hs_instance, ddi_get_pid(),
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor uarpg = ((hermon_rsrc_t *)(uintptr_t)value)->hr_indx;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Use the internal protection domain (PD) for setting up CQs */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Increment the reference count on the protection domain (PD) */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Allocate an CQ context entry. This will be filled in with all
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the necessary parameters to define the Completion Queue. And then
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * ownership will be passed to the hardware in the final step
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * below. If we fail here, we must undo the protection domain
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * reference count.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor status = hermon_rsrc_alloc(state, HERMON_CQC, 1, sleepflag, &cqc);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Allocate the software structure for tracking the completion queue
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * (i.e. the Hermon Completion Queue handle). If we fail here, we must
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * undo the protection domain reference count and the previous
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * resource allocation.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor status = hermon_rsrc_alloc(state, HERMON_CQHDL, 1, sleepflag, &rsrc);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor cq->cq_cqnum = cqc->hr_indx; /* just use index, implicit in Hermon */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * If this will be a user-mappable CQ, then allocate an entry for
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the "userland resources database". This will later be added to
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the database (after all further CQ operations are successful).
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * If we fail here, we must undo the reference counts and the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * previous resource allocation.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor umapdb = hermon_umap_db_alloc(state->hs_instance, cq->cq_cqnum,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor MLNX_UMAP_CQMEM_RSRC, (uint64_t)(uintptr_t)rsrc);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Allocate the doorbell record. We'll need one for the CQ, handling
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * both consumer index (SET CI) and the CQ state (CQ ARM).
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor status = hermon_dbr_alloc(state, uarpg, &cq->cq_arm_ci_dbr_acchdl,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor &cq->cq_arm_ci_vdbr, &cq->cq_arm_ci_pdbr, &cq->cq_dbr_mapoffset);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Calculate the appropriate size for the completion queue.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Note: All Hermon CQs must be a power-of-2 minus 1 in size. Also
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * they may not be any smaller than HERMON_CQ_MIN_SIZE. This step is
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * to round the requested size up to the next highest power-of-2
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor cq_attr->cq_size = max(cq_attr->cq_size, HERMON_CQ_MIN_SIZE);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Next we verify that the rounded-up size is valid (i.e. consistent
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * with the device limits and/or software-configured limits)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor if (log_cq_size > state->hs_cfg_profile->cp_log_max_cq_sz) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Allocate the memory for Completion Queue.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Note: Although we use the common queue allocation routine, we
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * always specify HERMON_QUEUE_LOCATION_NORMAL (i.e. CQ located in
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * kernel system memory) for kernel CQs because it would be
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * inefficient to have CQs located in DDR memory. This is primarily
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * because CQs are read from (by software) more than they are written
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * to. (We always specify HERMON_QUEUE_LOCATION_USERLAND for all
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * user-mappable CQs for a similar reason.)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * It is also worth noting that, unlike Hermon QP work queues,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * completion queues do not have the same strict alignment
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * requirements. It is sufficient for the CQ memory to be both
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * aligned to and bound to addresses which are a multiple of CQE size.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor cq->cq_cqinfo.qa_size = (1 << log_cq_size) * sizeof (hermon_hw_cqe_t);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor cq->cq_cqinfo.qa_location = HERMON_QUEUE_LOCATION_USERLAND;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor cq->cq_cqinfo.qa_location = HERMON_QUEUE_LOCATION_NORMAL;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor status = hermon_queue_alloc(state, &cq->cq_cqinfo, sleepflag);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor buf = (hermon_hw_cqe_t *)cq->cq_cqinfo.qa_buf_aligned;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * The ownership bit of the CQE's is set by the HW during the process
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * of transferrring ownership of the CQ (PRM 09.35c, 14.2.1, note D1
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Register the memory for the CQ. The memory for the CQ must
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * be registered in the Hermon TPT tables. This gives us the LKey
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * to specify in the CQ context below. Note: If this is a user-
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * mappable CQ, then we will force DDI_DMA_CONSISTENT mapping.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor flag = (sleepflag == HERMON_SLEEP) ? IBT_MR_SLEEP : IBT_MR_NOSLEEP;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor mr_attr.mr_flags = flag | IBT_MR_ENABLE_LOCAL_WRITE;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor op.mro_bind_type = state->hs_cfg_profile->cp_iommu_bypass;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor status = hermon_mr_register(state, pd, &mr_attr, &mr, &op,
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor cq->cq_erreqnum = HERMON_CQ_ERREQNUM_GET(state);
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor if (!HERMON_HID_VALID(state, cq_attr->cq_hid)) {
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor IBTF_DPRINTF_L2("CQalloc", "bad handler id 0x%x",
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor cq->cq_eqnum = HERMON_HID_TO_EQNUM(state, cq_attr->cq_hid);
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor IBTF_DPRINTF_L2("cqalloc", "hid: eqn %d", cq->cq_eqnum);
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor cq_schedp = (hermon_cq_sched_t *)cq_attr->cq_sched;
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor } else if (cq_schedp != &state->hs_cq_sched_default) {
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor for (i = 0; i < state->hs_cq_sched_array_size; i++)
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor break; /* found it */
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor "ignored\n");
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor IBTF_DPRINTF_L2("cqalloc", "sched: first-1 %d, len %d, "
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Fill in the CQC entry. This is the final step before passing
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * ownership of the CQC entry to the Hermon hardware. We use all of
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the information collected/calculated above to fill in the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * requisite portions of the CQC. Note: If this CQ is going to be
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * used for userland access, then we need to set the UAR page number
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * appropriately (otherwise it's a "don't care")
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor cqc_entry.pg_offs = cq->cq_cqinfo.qa_pgoffs >> 5;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor cqc_entry.mtt_base_addh = (uint32_t)((mr->mr_mttaddr >> 32) & 0xFF);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor cqc_entry.dbr_addrh = (uint32_t)((uint64_t)cq->cq_arm_ci_pdbr >> 32);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor cqc_entry.dbr_addrl = (uint32_t)((uint64_t)cq->cq_arm_ci_pdbr >> 3);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Write the CQC entry to hardware - we pass ownership of
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the entry to the hardware (using the Hermon SW2HW_CQ firmware
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * command). Note: In general, this operation shouldn't fail. But
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * if it does, we have to undo everything we've done above before
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * returning error.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor status = hermon_cmn_ownership_cmd_post(state, SW2HW_CQ, &cqc_entry,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor sizeof (hermon_hw_cqc_t), cq->cq_cqnum, sleepflag);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor cmn_err(CE_CONT, "Hermon: SW2HW_CQ command failed: %08x\n",
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor hermon_fm_ereport(state, HCA_SYS_ERR, HCA_ERR_SRV_LOST);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Fill in the rest of the Hermon Completion Queue handle. Having
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * successfully transferred ownership of the CQC, we can update the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * following fields for use in further operations on the CQ.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* least restrictive */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor avl_create(&cq->cq_wrid_wqhdr_avl_tree, hermon_wrid_workq_compare,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor sizeof (struct hermon_workq_avl_s),
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Put CQ handle in Hermon CQNum-to-CQHdl list. Then fill in the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * "actual_size" and "cqhdl" and return success
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor hermon_icm_set_num_to_hdl(state, HERMON_CQC, cqc->hr_indx, cq);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * If this is a user-mappable CQ, then we need to insert the previously
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * allocated entry into the "userland resources database". This will
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * allow for later lookup during devmap() (i.e. mmap()) calls.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Fill in the return arguments (if necessary). This includes the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * real completion queue size.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * The following is cleanup for all possible failure cases in this routine
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor if (hermon_mr_deregister(state, &mr, HERMON_MR_DEREG_ALL,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor HERMON_WARNING(state, "failed to deregister CQ memory");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor hermon_dbr_free(state, uarpg, cq->cq_arm_ci_vdbr);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * hermon_cq_free()
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Context: Can be called only from user or kernel context.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor/* ARGSUSED */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorhermon_cq_free(hermon_state_t *state, hermon_cqhdl_t *cqhdl, uint_t sleepflag)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Pull all the necessary information from the Hermon Completion Queue
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * handle. This is necessary here because the resource for the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * CQ handle is going to be freed up as part of this operation.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor resize = cq->cq_resize_hdl; /* save the handle for later */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * If there are work queues still associated with the CQ, then return
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * an error. Otherwise, we will be holding the CQ lock.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * If this was a user-mappable CQ, then we need to remove its entry
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * from the "userland resources database". If it is also currently
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * mmap()'d out to a user process, then we need to call
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * devmap_devmem_remap() to remap the CQ memory to an invalid mapping.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * We also need to invalidate the CQ tracking information for the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * user mapping.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor status = hermon_umap_db_find(state->hs_instance, cqnum,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor MLNX_UMAP_CQMEM_RSRC, &value, HERMON_UMAP_DB_REMOVE,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor HERMON_WARNING(state, "failed to find in database");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor "devmap_devmem_remap()");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Put NULL into the Arbel CQNum-to-CQHdl list. This will allow any
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * in-progress events to detect that the CQ corresponding to this
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * number has been freed.
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor hermon_icm_set_num_to_hdl(state, HERMON_CQC, cqc->hr_indx, NULL);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Reclaim CQC entry from hardware (using the Hermon HW2SW_CQ
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * firmware command). If the ownership transfer fails for any reason,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * then it is an indication that something (either in HW or SW) has
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * gone seriously wrong.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor status = hermon_cmn_ownership_cmd_post(state, HW2SW_CQ, &cqc_entry,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor HERMON_WARNING(state, "failed to reclaim CQC ownership");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor cmn_err(CE_CONT, "Hermon: HW2SW_CQ command failed: %08x\n",
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor hermon_fm_ereport(state, HCA_SYS_ERR, HCA_ERR_SRV_LOST);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * From here on, we start reliquishing resources - but check to see
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * if a resize was in progress - if so, we need to relinquish those
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * resources as well
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Deregister the memory for the Completion Queue. If this fails
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * for any reason, then it is an indication that something (either
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * in HW or SW) has gone seriously wrong. So we print a warning
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * message and return.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor status = hermon_mr_deregister(state, &mr, HERMON_MR_DEREG_ALL,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor HERMON_WARNING(state, "failed to deregister CQ memory");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor if (resize) { /* there was a pointer to a handle */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor mr = resize->cq_mrhdl; /* reuse the pointer to the region */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor status = hermon_mr_deregister(state, &mr, HERMON_MR_DEREG_ALL,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor HERMON_WARNING(state, "failed to deregister resize CQ "
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Free the memory for the CQ */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* and the temporary handle */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor kmem_free(resize, sizeof (struct hermon_sw_cq_s));
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* everything else does not matter for the resize in progress */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Free the dbr */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor hermon_dbr_free(state, cq->cq_uarpg, cq->cq_arm_ci_vdbr);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Free the Hermon Completion Queue handle */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Free up the CQC entry resource */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Decrement the reference count on the protection domain (PD) */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Set the cqhdl pointer to NULL and return success */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * hermon_cq_resize()
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Context: Can be called only from user or kernel context.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorhermon_cq_resize(hermon_state_t *state, hermon_cqhdl_t cq, uint_t req_size,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor if (cq->cq_resize_hdl != 0) { /* already in process */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Use the internal protection domain (PD) for CQs */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Calculate the appropriate size for the new resized completion queue.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Note: All Hermon CQs must be a power-of-2 minus 1 in size. Also
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * they may not be any smaller than HERMON_CQ_MIN_SIZE. This step is
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * to round the requested size up to the next highest power-of-2
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Next we verify that the rounded-up size is valid (i.e. consistent
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * with the device limits and/or software-configured limits)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor if (log_cq_size > state->hs_cfg_profile->cp_log_max_cq_sz) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Allocate the memory for newly resized Completion Queue.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Note: Although we use the common queue allocation routine, we
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * always specify HERMON_QUEUE_LOCATION_NORMAL (i.e. CQ located in
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * kernel system memory) for kernel CQs because it would be
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * inefficient to have CQs located in DDR memory. This is the same
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * as we do when we first allocate completion queues primarily
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * because CQs are read from (by software) more than they are written
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * to. (We always specify HERMON_QUEUE_LOCATION_USERLAND for all
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * user-mappable CQs for a similar reason.)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * It is also worth noting that, unlike Hermon QP work queues,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * completion queues do not have the same strict alignment
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * requirements. It is sufficient for the CQ memory to be both
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * aligned to and bound to addresses which are a multiple of CQE size.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* first, alloc the resize_handle */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor resize_hdl = kmem_zalloc(sizeof (struct hermon_sw_cq_s), KM_SLEEP);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor new_cqinfo.qa_size = (1 << log_cq_size) * sizeof (hermon_hw_cqe_t);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor new_cqinfo.qa_location = HERMON_QUEUE_LOCATION_USERLAND;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor new_cqinfo.qa_location = HERMON_QUEUE_LOCATION_NORMAL;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor status = hermon_queue_alloc(state, &new_cqinfo, sleepflag);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* free the resize handle */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor kmem_free(resize_hdl, sizeof (struct hermon_sw_cq_s));
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor buf = (hermon_hw_cqe_t *)new_cqinfo.qa_buf_aligned;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * No initialization of the cq is needed - the command will do it
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Register the memory for the CQ. The memory for the CQ must
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * be registered in the Hermon TPT tables. This gives us the LKey
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * to specify in the CQ context below.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor flag = (sleepflag == HERMON_SLEEP) ? IBT_MR_SLEEP : IBT_MR_NOSLEEP;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor mr_attr.mr_flags = flag | IBT_MR_ENABLE_LOCAL_WRITE;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor op.mro_bind_type = state->hs_cfg_profile->cp_iommu_bypass;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor status = hermon_mr_register(state, pd, &mr_attr, &mr, &op,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* free the resize handle */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor kmem_free(resize_hdl, sizeof (struct hermon_sw_cq_s));
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Now we grab the CQ lock. Since we will be updating the actual
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * CQ location and the producer/consumer indexes, we should hold
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * We do a ARBEL_NOSLEEP here (and below), though, because we are
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * holding the "cq_lock" and if we got raised to interrupt level
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * by priority inversion, we would not want to block in this routine
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * waiting for success.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Fill in the CQC entry. For the resize operation this is the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * final step before attempting the resize operation on the CQC entry.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * We use all of the information collected/calculated above to fill
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * in the requisite portions of the CQC.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor cqc_entry.mtt_base_addh = (uint32_t)((mr->mr_mttaddr >> 32) & 0xFF);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Write the CQC entry to hardware. Lastly, we pass ownership of
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the entry to the hardware (using the Hermon RESIZE_CQ firmware
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * command). Note: In general, this operation shouldn't fail. But
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * if it does, we have to undo everything we've done above before
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * returning error. Also note that the status returned may indicate
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the code to return to the IBTF.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor status = hermon_resize_cq_cmd_post(state, &cqc_entry, cq->cq_cqnum,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Resize attempt has failed, drop CQ lock and cleanup */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor if (hermon_mr_deregister(state, &mr, HERMON_MR_DEREG_ALL,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor HERMON_WARNING(state, "failed to deregister CQ memory");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor kmem_free(resize_hdl, sizeof (struct hermon_sw_cq_s));
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor cmn_err(CE_CONT, "Hermon: RESIZE_CQ command failed: "
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * For Hermon, we've alloc'd another handle structure and save off the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * important things in it. Then, in polling we check to see if there's
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * a "resizing handle" and if so we look for the "special CQE", opcode
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * 0x16, that indicates the transition to the new buffer.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * At that point, we'll adjust everything - including dereg and
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * freeing of the original buffer, updating all the necessary fields
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * in the cq_hdl, and setting up for the next cqe polling
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor sizeof (struct hermon_qalloc_info_s));
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* now, save the address in the cq_handle */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Drop the CQ lock now.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Fill in the return arguments (if necessary). This includes the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * real new completion queue size.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * hermon_cq_modify()
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Context: Can be called base context.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor/* ARGSUSED */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorhermon_cq_modify(hermon_state_t *state, hermon_cqhdl_t cq,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor uint_t count, uint_t usec, ibt_cq_handler_id_t hid, uint_t sleepflag)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor status = hermon_modify_cq_cmd_post(state, &cqc_entry,
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor cmn_err(CE_CONT, "Hermon: MODIFY_MODERATION_CQ "
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor cqc_entry.c_eqn = HERMON_HID_TO_EQNUM(state, hid);
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor status = hermon_modify_cq_cmd_post(state, &cqc_entry,
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor cmn_err(CE_CONT, "Hermon: MODIFY_EQN command failed: "
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * hermon_cq_notify()
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Context: Can be called from interrupt or base context.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorhermon_cq_notify(hermon_state_t *state, hermon_cqhdl_t cq,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Validate IBT flags and call doorbell routine. */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor status = hermon_cq_arm_doorbell(state, cq, cmd);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * hermon_cq_poll()
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Context: Can be called from interrupt or base context.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorhermon_cq_poll(hermon_state_t *state, hermon_cqhdl_t cq, ibt_wc_t *wc_p,
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor uint32_t cons_indx, wrap_around_mask, shift, mask;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Check for user-mappable CQ memory. Note: We do not allow kernel
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * clients to poll CQ memory that is accessible directly by the user.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * If the CQ memory is user accessible, then return an error.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Get the consumer index */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Calculate the wrap around mask. Note: This operation only works
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * because all Hermon completion queues have power-of-2 sizes
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Calculate the pointer to the first CQ entry */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor cqe = &cq->cq_buf[cons_indx & wrap_around_mask];
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Keep pulling entries from the CQ until we find an entry owned by
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the hardware. As long as there the CQE's owned by SW, process
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * each entry by calling hermon_cq_cqe_consume() and updating the CQ
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * consumer index. Note: We only update the consumer index if
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * hermon_cq_cqe_consume() returns HERMON_CQ_SYNC_AND_DB. Otherwise,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * it indicates that we are going to "recycle" the CQE (probably
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * because it is a error CQE and corresponds to more than one
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * completion).
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor while (HERMON_CQE_OWNER_IS_SW(cq, cqe, cons_indx, shift, mask)) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor if (cq->cq_resize_hdl != 0) { /* in midst of resize */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* peek at the opcode */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Increment the consumer index */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor spec_op = 1; /* plus one for the limiting CQE */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Update the pointer to the next CQ entry */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor cqe = &cq->cq_buf[cons_indx & wrap_around_mask];
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor } /* in resizing CQ */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * either resizing and not the special opcode, or
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * not resizing at all
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor hermon_cq_cqe_consume(state, cq, cqe, &wc_p[polled_cnt++]);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Increment the consumer index */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Update the pointer to the next CQ entry */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor cqe = &cq->cq_buf[cons_indx & wrap_around_mask];
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * If we have run out of space to store work completions,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * then stop and return the ones we have pulled of the CQ.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Now we only ring the doorbell (to update the consumer index) if
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor * we've actually consumed a CQ entry.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor if ((polled_cnt != 0) && (cq->cq_consindx != cons_indx)) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Update the consumer index in both the CQ handle and the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * doorbell record.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor } else if (polled_cnt == 0) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* if we got the special opcode, update the consindx */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Set "num_polled" (if necessary) */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Set CQ_EMPTY condition if needed, otherwise return success */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Check if the system is currently panicking. If it is, then call
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the Hermon interrupt service routine. This step is necessary here
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * because we might be in a polled I/O mode and without the call to
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * hermon_isr() - and its subsequent calls to poll and rearm each
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * event queue - we might overflow our EQs and render the system
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * unable to sync/dump.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor (void) hermon_isr((caddr_t)state, (caddr_t)NULL);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * cmd_sn must be initialized to 1 to enable proper reenabling
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * by hermon_arm_cq_dbr_update().
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * User cmd_sn needs help from this kernel function to know
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * when it should be incremented (modulo 4). We do an atomic
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * update of the arm_cq dbr to communicate this fact. We retry
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * in the case that user library is racing with us. We zero
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * out the cmd field so that the user library can use the cmd
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * field to track the last command it issued (solicited verses any).
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorhermon_arm_cq_dbr_update(hermon_dbr_t *cq_arm_dbr)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor old_cmd_sn = tmp & (0x3 << HERMON_CQDB_CMDSN_SHIFT);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor new_cmd_sn = (old_cmd_sn + (0x1 << HERMON_CQDB_CMDSN_SHIFT)) &
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor new = htonl((tmp & ~(0x37 << HERMON_CQDB_CMD_SHIFT)) | new_cmd_sn);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor if (tmp != cmp) { /* cas failed, so need to retry */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * hermon_cq_handler()
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Context: Only called from interrupt context
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor/* ARGSUSED */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorhermon_cq_handler(hermon_state_t *state, hermon_eqhdl_t eq,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Get the CQ handle from CQ number in event descriptor */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * If the CQ handle is NULL, this is probably an indication
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * that the CQ has been freed already. In which case, we
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * should not deliver this event.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * We also check that the CQ number in the handle is the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * same as the CQ number in the event queue entry. This
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * extra check allows us to handle the case where a CQ was
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * freed and then allocated again in the time it took to
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * handle the event queue processing. By constantly incrementing
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the non-constrained portion of the CQ number every time
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * a new CQ is allocated, we mitigate (somewhat) the chance
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * that a stale event could be passed to the client's CQ
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Lastly, we check if "hs_ibtfpriv" is NULL. If it is then it
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * means that we've have either received this event before we
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * finished attaching to the IBTF or we've received it while we
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * are in the process of detaching.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * hermon_cq_err_handler()
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Context: Only called from interrupt context
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor/* ARGSUSED */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorhermon_cq_err_handler(hermon_state_t *state, hermon_eqhdl_t eq,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Get the CQ handle from CQ number in event descriptor */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * If the CQ handle is NULL, this is probably an indication
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * that the CQ has been freed already. In which case, we
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * should not deliver this event.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * We also check that the CQ number in the handle is the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * same as the CQ number in the event queue entry. This
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * extra check allows us to handle the case where a CQ was
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * freed and then allocated again in the time it took to
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * handle the event queue processing. By constantly incrementing
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the non-constrained portion of the CQ number every time
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * a new CQ is allocated, we mitigate (somewhat) the chance
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * that a stale event could be passed to the client's CQ
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * And then we check if "hs_ibtfpriv" is NULL. If it is then it
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * means that we've have either received this event before we
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * finished attaching to the IBTF or we've received it while we
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * are in the process of detaching.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor HERMON_DO_IBTF_ASYNC_CALLB(state, type, &event);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * hermon_cq_refcnt_inc()
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Context: Can be called from interrupt or base context.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorhermon_cq_refcnt_inc(hermon_cqhdl_t cq, uint_t is_special)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Increment the completion queue's reference count. Note: In order
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * to ensure compliance with IBA C11-15, we must ensure that a given
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * CQ is not used for both special (SMI/GSI) QP and non-special QP.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * This is accomplished here by keeping track of how the referenced
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * CQ is being used.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * hermon_cq_refcnt_dec()
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Context: Can be called from interrupt or base context.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Decrement the completion queue's reference count */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * hermon_cq_arm_doorbell()
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Context: Can be called from interrupt or base context.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorhermon_cq_arm_doorbell(hermon_state_t *state, hermon_cqhdl_t cq, uint_t cq_cmd)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ddi_acc_handle_t uarhdl = hermon_get_uarhdl(state);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* initialize the FMA retry loop */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor hermon_pio_init(fm_loop_cnt, fm_status, fm_test_num);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* the FMA retry loop starts for Hermon doorbell register. */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor hermon_pio_start(state, uarhdl, pio_error, fm_loop_cnt, fm_status,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor cmd_sn = tmp & (0x3 << HERMON_CQDB_CMDSN_SHIFT);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor new = htonl(cmd_sn | (cq->cq_consindx & 0xFFFFFF));
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor HERMON_UAR_DOORBELL(state, uarhdl, (uint64_t *)(void *)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor &state->hs_uar->cq, (((uint64_t)cmd_sn | cq_num) <<
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor } /* else it's already armed */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ASSERT(cq_cmd == HERMON_CQDB_NOTIFY_CQ_SOLICIT);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor new = htonl(cmd_sn | (cq->cq_consindx & 0xFFFFFF));
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor HERMON_UAR_DOORBELL(state, uarhdl, (uint64_t *)(void *)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor &state->hs_uar->cq, (((uint64_t)cmd_sn | cq_num) <<
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor } /* else it's already armed */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* the FMA retry loop ends. */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor hermon_pio_end(state, uarhdl, pio_error, fm_loop_cnt, fm_status,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor hermon_fm_ereport(state, HCA_SYS_ERR, HCA_ERR_SRV_LOST);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * hermon_cqhdl_from_cqnum()
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Context: Can be called from interrupt or base context.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * This routine is important because changing the unconstrained
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * portion of the CQ number is critical to the detection of a
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * potential race condition in the CQ handler code (i.e. the case
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * where a CQ is freed and alloc'd again before an event for the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * "old" CQ can be handled).
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * While this is not a perfect solution (not sure that one exists)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * it does help to mitigate the chance that this race condition will
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * cause us to deliver a "stale" event to the new CQ owner. Note:
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * this solution does not scale well because the number of constrained
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * bits increases (and, hence, the number of unconstrained bits
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * decreases) as the number of supported CQs grows. For small and
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * intermediate values, it should hopefully provide sufficient
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * protection.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorhermon_cqhdl_from_cqnum(hermon_state_t *state, uint_t cqnum)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Calculate the CQ table index from the cqnum */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor cqmask = (1 << state->hs_cfg_profile->cp_log_num_cq) - 1;
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor return (hermon_icm_num_to_hdl(state, HERMON_CQC, cqindx));
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * hermon_cq_cqe_consume()
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Context: Can be called from interrupt or base context.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorhermon_cq_cqe_consume(hermon_state_t *state, hermon_cqhdl_t cq,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Determine if this is an "error" CQE by examining "opcode". If it
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * is an error CQE, then call hermon_cq_errcqe_consume() and return
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * whatever status it returns. Otherwise, this is a successful
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * completion.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Fetch the Work Request ID using the information in the CQE.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * See hermon_wr.c for more details.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Parse the CQE opcode to determine completion type. This will set
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * not only the type of the completion, but also any flags that might
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * be associated with it (e.g. whether immediate data is present).
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(state->hs_fcoib_may_be_running))
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor if (HERMON_CQE_SENDRECV_GET(cq, cqe) != HERMON_COMPLETION_RECV) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Send CQE */
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor } else if ((state->hs_fcoib_may_be_running == B_TRUE) &&
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor hermon_fcoib_is_fexch_qpn(state, HERMON_CQE_QPNUM_GET(cq, cqe))) {
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor wc->wc_bytes_xfer = HERMON_CQE_BYTECNT_GET(cq, cqe);
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor wc->wc_fexch_seq_cnt = HERMON_CQE_FEXCH_SEQ_CNT(cq, cqe);
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor wc->wc_fexch_tx_bytes_xfer = HERMON_CQE_FEXCH_TX_BYTES(cq, cqe);
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor wc->wc_fexch_rx_bytes_xfer = HERMON_CQE_FEXCH_RX_BYTES(cq, cqe);
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor wc->wc_fexch_seq_id = HERMON_CQE_FEXCH_SEQ_ID(cq, cqe);
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor wc->wc_detail = HERMON_CQE_FEXCH_DETAIL(cq, cqe) &
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor wc->wc_rkey = HERMON_CQE_IMM_ETH_PKEY_CRED_GET(cq, cqe);
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor flags |= IBT_WC_FEXCH_FMT | IBT_WC_RKEY_INVALIDATED;
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor * Parse the remaining contents of the CQE into the work
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor * completion. This means filling in SL, QP number, SLID,
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor * immediate data, etc.
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor * Note: Not all of these fields are valid in a given
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor * completion. Many of them depend on the actual type of
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor * completion. So we fill in all of the fields and leave
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor * it up to the IBTF and consumer to sort out which are
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor * valid based on their context.
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor wc->wc_ethertype = (wc->wc_immed_data & 0xFFFF);
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor * Fill in "bytes transferred" as appropriate. Also,
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor * if necessary, fill in the "path bits" field.
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor wc->wc_path_bits = HERMON_CQE_PATHBITS_GET(cq, cqe);
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor wc->wc_bytes_xfer = HERMON_CQE_BYTECNT_GET(cq, cqe);
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor * Check for GRH, update the flags, then fill in "wc_flags"
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor * field in the work completion
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Receive CQE */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Note: According to the PRM, all QP1 recv
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * completions look like the result of a Send with
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Immediate. They are not, however, (MADs are Send
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Only) so we need to check the QP number and set
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the flag only if it is non-QP1.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor if ((qpnum < qp1_indx) || (qpnum > qp1_indx + 1)) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* FALLTHROUGH */
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor wc->wc_rkey = wc->wc_immed_data; /* same field in cqe */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * hermon_cq_errcqe_consume()
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Context: Can be called from interrupt or base context.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorhermon_cq_errcqe_consume(hermon_state_t *state, hermon_cqhdl_t cq,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Fetch the Work Request ID using the information in the CQE.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * See hermon_wr.c for more details.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Parse the CQE opcode to determine completion type. We know that
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the CQE is an error completion, so we extract only the completion
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor imm_eth_pkey_cred = HERMON_CQE_ERROR_SYNDROME_GET(cq, cqe);
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor IBTF_DPRINTF_L2("CQE ERR", "cqe %p QPN %x indx %x status 0x%x "
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor "vendor syndrome %x", cqe, HERMON_CQE_QPNUM_GET(cq, cqe),
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor "Local Protection Error\n");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * The following error codes are not supported in the Hermon driver
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * as they relate only to Reliable Datagram completion statuses:
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * case HERMON_CQE_LOCAL_RDD_VIO_ERR:
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * case HERMON_CQE_REM_INV_RD_REQ_ERR:
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * case HERMON_CQE_EEC_REM_ABORTED_ERR:
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * case HERMON_CQE_INV_EEC_NUM_ERR:
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * case HERMON_CQE_INV_EEC_STATE_ERR:
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * case HERMON_CQE_LOC_EEC_ERR:
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor HERMON_WARNING(state, "unknown error CQE status");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * hermon_cq_resize_helper()
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Context: Can be called only from user or kernel context.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorhermon_cq_resize_helper(hermon_state_t *state, hermon_cqhdl_t cq)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * we're here because we found the special cqe opcode, so we have
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * to update the cq_handle, release the old resources, clear the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * flag in the cq_hdl, and release the resize_hdl. When we return
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * above, it will take care of the rest
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Deregister the memory for the old Completion Queue. Note: We
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * really can't return error here because we have no good way to
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * cleanup. Plus, the deregistration really shouldn't ever happen.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * So, if it does, it is an indication that something has gone
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * seriously wrong. So we print a warning message and return error
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * (knowing, of course, that the "old" CQ memory will be leaked)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor status = hermon_mr_deregister(state, &cq->cq_mrhdl, HERMON_MR_DEREG_ALL,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor HERMON_WARNING(state, "failed to deregister old CQ memory");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Next, free the memory from the old CQ buffer */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* now we can update the cq_hdl with the new things saved */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor cq->cq_umap_dhp = cq->cq_resize_hdl->cq_umap_dhp;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor sizeof (struct hermon_qalloc_info_s));
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* finally, release the resizing handle */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor kmem_free(resize_hdl, sizeof (struct hermon_sw_cq_s));
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * hermon_cq_entries_flush()
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Context: Can be called from interrupt or base context.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor/* ARGSUSED */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorhermon_cq_entries_flush(hermon_state_t *state, hermon_qphdl_t qp)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor uint32_t cons_indx, tail_cons_indx, wrap_around_mask;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylordo_send_cq: /* loop back to here if send_cq is not the same as recv_cq */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Calculate the pointer to the first CQ entry */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor cqe = &cq->cq_buf[cons_indx & wrap_around_mask];
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Loop through the CQ looking for entries owned by software. If an
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * entry is owned by software then we increment an 'outstanding_cqes'
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * count to know how many entries total we have on our CQ. We use this
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * value further down to know how many entries to loop through looking
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * for our same QP number.
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor while (HERMON_CQE_OWNER_IS_SW(cq, cqe, tail_cons_indx, shift, mask)) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* increment total cqes count */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* increment the consumer index */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* update the pointer to the next cq entry */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor cqe = &cq->cq_buf[tail_cons_indx & wrap_around_mask];
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Using the 'tail_cons_indx' that was just set, we now know how many
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * total CQEs possible there are. Set the 'check_indx' and the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * 'new_indx' to the last entry identified by 'tail_cons_indx'
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor while (--outstanding_cqes >= 0) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor cqe = &cq->cq_buf[check_indx & wrap_around_mask];
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * If the QP number is the same in the CQE as the QP, then
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * we must "consume" it. If it is for an SRQ wqe, then we
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * also must free the wqe back onto the free list of the SRQ.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Add wqe back to SRQ free list */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor } else { /* CQEs for other QPNs need to remain */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Copy the CQE into the "next_cqe" pointer. */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor new_indx--; /* move index to next CQE to fill */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor check_indx--; /* move index to next CQE to check */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Update consumer index to be the 'new_indx'. This moves it past all
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * removed entries. Because 'new_indx' is pointing to the last
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * previously valid SW owned entry, we add 1 to point the cons_indx to
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the first HW owned entry.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Now we only ring the doorbell (to update the consumer index) if
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * we've actually consumed a CQ entry. If we found no QP number
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * matches above, then we would not have removed anything. So only if
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * something was removed do we ring the doorbell.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Update the consumer index in both the CQ handle and the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * doorbell record.
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor * hermon_get_cq_sched_list()
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor * Context: Only called from attach() path context
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor * Read properties, creating entries in hs_cq_sched_list with
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor * information about the requested "expected" and "minimum"
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor * number of MSI-X interrupt vectors per list entry.
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor if (ddi_prop_lookup_string_array(DDI_DEV_T_ANY, state->hs_dip,
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor DDI_PROP_DONTPASS, "cqh-group-list", &listp, &nlist) !=
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor state->hs_cq_sched_array = cq_schedp = kmem_zalloc(nlist *
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor for (i = 0; i < nlist; i++) {
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor if ((len = strlen(listp[i])) >= HERMON_CQH_MAX) {
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor cmn_err(CE_CONT, "'cqh' property name too long\n");
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor for (j = 0; j < i; j++) {
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor (void) strncpy(cq_schedp[i].cqs_name, listp[i], HERMON_CQH_MAX);
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor (void) strncpy(ulp_prop + 4, listp[i], len + 1);
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor if (ddi_prop_lookup_int_array(DDI_DEV_T_ANY, state->hs_dip,
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor cmn_err(CE_CONT, "property '%s' not found\n", ulp_prop);
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor if (ddi_prop_lookup_int_array(DDI_DEV_T_ANY, state->hs_dip,
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor DDI_PROP_DONTPASS, "cqh-default", &data, &ndata) !=
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor cmn_err(CE_CONT, "property 'cqh-default' not found\n");
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor cmn_err(CE_CONT, "property 'cqh-default' does not "
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor "have 2 integers\n");
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor cmn_err(CE_CONT, "Error in 'cqh' properties in hermon.conf\n");
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor cmn_err(CE_CONT, "completion handler groups not being used\n");
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor kmem_free(cq_schedp, nlist * sizeof (hermon_cq_sched_t));
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor * hermon_cq_sched_init()
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor * Context: Only called from attach() path context
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor * Read the hermon.conf properties looking for cq_sched info,
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor * creating reserved pools of MSI-X interrupt ranges for the
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor * specified ULPs.
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor mutex_init(&state->hs_cq_sched_lock, NULL, MUTEX_DRIVER,
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor /* initialize cq_sched_default */
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor /* Read properties to determine which ULPs use cq_sched */
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor /* Determine if we have enough vectors, or if we have to scale down */
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor desired = defp->cqs_desired; /* default desired (from hermon.conf) */
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor goto done; /* all interrupts in the default pool */
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor for (i = 0; i < array_size; i++)
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor cmn_err(CE_CONT, "#interrupts allocated (%d) is less than "
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor "the #interrupts desired (%d)\n",
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor cmn_err(CE_CONT, "completion handler groups not being used\n");
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor goto done; /* all interrupts in the default pool */
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor /* Game on. For each cq_sched group, reserve the MSI-X range */
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor for (i = 0; i < array_size; i++) {
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor cq_schedp[i].cqs_start_hid = defp->cqs_start_hid;
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor /* reset default's start allocation seed */
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor kmem_free(state->hs_cq_sched_array, sizeof (hermon_cq_sched_t) *
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylorhermon_cq_sched_alloc(hermon_state_t *state, ibt_cq_sched_attr_t *attr,
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor if ((flags & (IBT_CQS_SCHED_GROUP | IBT_CQS_EXACT_SCHED_GROUP)) == 0) {
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor for (i = 0; i < state->hs_cq_sched_array_size; i++, cq_schedp++) {
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor break; /* found it */
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor if ((i == state->hs_cq_sched_array_size) || /* not found, or */
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor (cq_schedp->cqs_len == 0)) /* defined, but no dedicated intr's */
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor *cq_sched_pp = cq_schedp; /* set to valid hdl, or to NULL */
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylorhermon_cq_sched_free(hermon_state_t *state, hermon_cq_sched_t *cq_schedp)
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor /* Just decrement refcnt */
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor HERMON_WARNING(state, "cq_sched free underflow\n");