9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * CDDL HEADER START
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * The contents of this file are subject to the terms of the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Common Development and Distribution License (the "License").
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * You may not use this file except in compliance with the License.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * See the License for the specific language governing permissions
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * and limitations under the License.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * When distributing Covered Code, include this CDDL HEADER in each
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * If applicable, add the following below this CDDL HEADER, with the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * fields enclosed by brackets "[]" replaced with your own identifying
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * information: Portions Copyright [yyyy] [name of copyright owner]
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * CDDL HEADER END
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Copyright 2008 Sun Microsystems, Inc. All rights reserved.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Use is subject to license terms.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * This file may contain confidential information of
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Mellanox Technologies, Ltd. and should not be distributed in source
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * form without approval from Sun Legal.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Function signatures
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorextern uint64_t dapls_tavor_wrid_get_entry(ib_cq_handle_t, tavor_hw_cqe_t *,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorextern void dapls_tavor_wrid_cq_reap(ib_cq_handle_t);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorextern void dapls_atomic_assign_64(uint64_t, uint64_t *);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic int dapli_tavor_wqe_send_build(ib_qp_handle_t, ibt_send_wr_t *,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic void dapli_tavor_wqe_send_linknext(ibt_send_wr_t *, uint64_t *,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor boolean_t, uint32_t, uint_t, uint64_t *, tavor_sw_wqe_dbinfo_t *);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic DAT_RETURN dapli_tavor_wqe_recv_build(ib_qp_handle_t, ibt_recv_wr_t *,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic void dapli_tavor_wqe_recv_linknext(uint64_t *, boolean_t, uint32_t,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic int dapli_tavor_cq_cqe_consume(ib_cq_handle_t, tavor_hw_cqe_t *,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic int dapli_tavor_cq_errcqe_consume(ib_cq_handle_t, tavor_hw_cqe_t *,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor/* exported to other HCAs */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorextern void dapli_tavor_wrid_add_entry(dapls_tavor_workq_hdr_t *, uint64_t,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorextern void dapli_tavor_wrid_add_entry_srq(ib_srq_handle_t, uint64_t, uint32_t);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Note: The 64 bit doorbells need to written atomically.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * In 32 bit libraries we need to use the special assembly rtn
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * because compiler generated code splits into 2 word writes
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor/* use a macro to ensure inlining on S10 amd64 compiler */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor#define dapli_tavor_cq_doorbell(ia_uar, cq_cmd, cqn, cq_param) \
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ((uint64_t)cqn << TAVOR_CQDB_CQN_SHIFT) | cq_param)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * dapli_tavor_cq_doorbell()
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Takes the specified cq cmd and cq number and rings the cq doorbell
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylordapli_tavor_cq_doorbell(dapls_hw_uar_t ia_uar, uint32_t cq_cmd, uint32_t cqn,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Build the doorbell from the parameters */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor doorbell = ((uint64_t)cq_cmd << TAVOR_CQDB_CMD_SHIFT) |
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ((uint64_t)cqn << TAVOR_CQDB_CQN_SHIFT) | cq_param;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Write the doorbell to UAR */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ((tavor_hw_uar_t *)ia_uar)->cq = HTOBE_64(doorbell);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* 32 bit version */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * For 32 bit intel we assign the doorbell in the order
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * prescribed by the Tavor PRM, lower to upper addresses
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor (uint32_t)HTOBE_32(doorbell & 0x00000000ffffffff);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor#endif /* _LP64 */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor#define dapli_tavor_qp_send_doorbell(ia_uar, nda, nds, qpn, fence, nopcode) \
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ((uint64_t)nopcode << TAVOR_QPSNDDB_NOPCODE_SHIFT) | \
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ((uint64_t)qpn << TAVOR_QPSNDDB_QPN_SHIFT) | nds)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * dapli_tavor_qp_send_doorbell()
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Takes the specified next descriptor information, qp number, opcode and
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * rings the send doorbell
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylordapli_tavor_qp_send_doorbell(dapls_hw_uar_t ia_uar, uint32_t nda,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor uint32_t nds, uint32_t qpn, uint32_t fence, uint32_t nopcode)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Build the doorbell from the parameters */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor doorbell = (((uint64_t)nda & TAVOR_QPSNDDB_NDA_MASK) <<
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ((uint64_t)nopcode << TAVOR_QPSNDDB_NOPCODE_SHIFT) |
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ((uint64_t)qpn << TAVOR_QPSNDDB_QPN_SHIFT) | nds;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Write the doorbell to UAR */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ((tavor_hw_uar_t *)ia_uar)->send = HTOBE_64(doorbell);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * For 32 bit intel we assign the doorbell in the order
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * prescribed by the Tavor PRM, lower to upper addresses
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor (uint32_t)HTOBE_32(doorbell & 0x00000000ffffffff);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor#endif /* _LP64 */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor#define dapli_tavor_qp_recv_doorbell(ia_uar, nda, nds, qpn, credits) \
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ((uint64_t)qpn << TAVOR_QPRCVDB_QPN_SHIFT) | credits)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * dapli_tavor_qp_recv_doorbell()
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Takes the specified next descriptor information, qp number and
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * rings the recv doorbell
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylordapli_tavor_qp_recv_doorbell(dapls_hw_uar_t ia_uar, uint32_t nda,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Build the doorbell from the parameters */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor doorbell = (((uint64_t)nda & TAVOR_QPRCVDB_NDA_MASK) <<
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ((uint64_t)qpn << TAVOR_QPRCVDB_QPN_SHIFT) | credits;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Write the doorbell to UAR */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ((tavor_hw_uar_t *)ia_uar)->recv = HTOBE_64(doorbell);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * For 32 bit intel we assign the doorbell in the order
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * prescribed by the Tavor PRM, lower to upper addresses
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor (uint32_t)HTOBE_32(doorbell & 0x00000000ffffffff);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor#endif /* _LP64 */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * dapls_tavor_max_inline()
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Return the max inline value that should be used.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Env variable DAPL_MAX_INLINE can override the default.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * If it's not set (or set to -1), default behavior is used.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * If it's zero or negative (except -1) inline is not done.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Check the env exactly once, otherwise return previous value. */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor max_inline_env = dapl_os_get_env_val("DAPL_MAX_INLINE", -1);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * dapls_ib_max_request_iov(), aka, max send sgl size.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * The send queue's scatter/gather list is used for "inline" data.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * By default, compute reasonable send queue size based on #iovs, #wqes,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * max_iovs, and max inline byte count. If the #wqes is large, then we
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * limit how much the SGL (space for inline data) can take. The heuristic
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * is to increase the memory for the send queue to a maximum of 32KB:
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * < 128 wqes increase to at most 256 minus header
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * < 256 wqes increase to at most 128 minus header
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * >= 256 wqes use SGL unaltered
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * If the env is supplied (max_inline >= 0), use it without checking.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylordapls_ib_max_request_iov(int iovs, int wqes, int max_iovs,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ret_iovs = max_inline_bytes / sizeof (tavor_hw_wqe_sgl_t);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor max_inline_bytes = 256 - TAVOR_INLINE_HEADER_SIZE_MAX;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ret_iovs = max_inline_bytes / sizeof (tavor_hw_wqe_sgl_t);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor max_inline_bytes = 128 - TAVOR_INLINE_HEADER_SIZE_MAX;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ret_iovs = max_inline_bytes / sizeof (tavor_hw_wqe_sgl_t);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor if (ret_iovs > max_iovs) /* do not exceed max */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * dapli_tavor_wqe_send_build()
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Constructs a WQE for a given ibt_send_wr_t
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylordapli_tavor_wqe_send_build(ib_qp_handle_t qp, ibt_send_wr_t *wr,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * RC is the only supported transport in UDAPL
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * For RC requests, we allow "Send", "RDMA Read", "RDMA Write"
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * If this is a Send request, then all we need is
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the Data Segment processing below.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Initialize the information for the Data Segments
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor qp->qp_sq_wqesz - TAVOR_INLINE_HEADER_SIZE_SEND;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor qp->qp_sq_wqesz - TAVOR_INLINE_HEADER_SIZE_RDMAW;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* FALLTHROUGH */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor if (qp->qp_sq_inline < 0 && wr->wr_opcode == IBT_WRC_RDMAR)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * If this is an RDMA Read or RDMA Write request, then fill
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * in the "Remote Address" header fields.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor rc = (tavor_hw_snd_wqe_remaddr_t *)((uintptr_t)addr +
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Build the Remote Address Segment for the WQE, using
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the information from the RC work request.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TAVOR_WQE_BUILD_REMADDR(rc, &wr->wr.rc.rcwr.rdma);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Update "ds" for filling in Data Segments (below) */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Generate a new R_key
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Increment the upper "unconstrained" bits and need to keep
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the lower "constrained" bits the same it represents
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the MPT index.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor tavor_num_mpt_mask = (uint32_t)(1 << qp->qp_num_mpt_shift) - 1;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor new_rkey = ((new_rkey << qp->qp_num_mpt_shift) |
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor bn = (tavor_hw_snd_wqe_bind_t *)((uintptr_t)addr +
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Build the Bind Memory Window Segments for the WQE,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * using the information from the RC Bind memory
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * window work request.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Update the "ds" pointer. Even though the "bind"
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * operation requires no SGLs, this is necessary to
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * facilitate the correct descriptor size calculations
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor "dapli_tavor_wqe_send_build: invalid wr_opcode=%d\n",
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Now fill in the Data Segments (SGL) for the Send WQE based on
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the values setup above (i.e. "sgl", "nds", and the "ds" pointer
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Start by checking for a valid number of SGL entries
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * For each SGL in the Send Work Request, fill in the Send WQE's data
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * segments. Note: We skip any SGL with zero size because Tavor
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * hardware cannot handle a zero for "byte_cnt" in the WQE. Actually
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the encoding for zero means a 2GB transfer. Because of this special
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * encoding in the hardware, we mask the requested length with
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * TAVOR_WQE_SGL_BYTE_CNT_MASK (so that 2GB will end up encoded as
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor if (max_inline_bytes != -1) { /* compute total_len */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor for (i = 0; i < nds; i++)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor max_inline_bytes = -1; /* too big, do not "inline" */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor HTOBE_32(total_len | TAVOR_WQE_SGL_INLINE_MASK);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor for (i = 0; i < nds; i++) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Return the size of descriptor (in 16-byte chunks) */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor *size = ((uintptr_t)dst - (uintptr_t)addr + 15) >> 4;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor for (i = 0; i < nds; i++) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Fill in the Data Segment(s) for the current WQE,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * using the information contained in the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * scatter-gather list of the work request.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Return the size of descriptor (in 16-byte chunks) */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor *size = ((uintptr_t)&ds[num_ds] - (uintptr_t)addr) >> 4;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * dapli_tavor_wqe_send_linknext()
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Takes a WQE and links it to the prev WQE chain
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylordapli_tavor_wqe_send_linknext(ibt_send_wr_t *curr_wr, uint64_t *curr_addr,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor boolean_t ns, uint32_t curr_desc, uint_t curr_descsz, uint64_t *prev_addr,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Set the "c" (i.e. "signaled") bit appropriately */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Set the "s" (i.e. "solicited") bit appropriately */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Set the "e" (i.e. "event") bit if notification is needed */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * The "i" bit is unused since uDAPL doesn't support
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the immediate data
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* initialize the ctrl and next fields of the current descriptor */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Calculate the "next" field of the prev descriptor. This amounts
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * to setting up the "next_wqe_addr", "nopcode", "fence", and "nds"
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * fields (see tavor_hw.h for more).
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Determine the value for the Tavor WQE "nopcode" field
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * by using the IBTF opcode from the work request
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Unsupported opcodes in UDAPL */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor "dapli_tavor_wqe_send_linknext: invalid nopcode=%d\n",
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor next = ((uint64_t)curr_desc & TAVOR_WQE_NDA_MASK) << 32;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor fence = (curr_wr->wr_flags & IBT_WR_SEND_FENCE) ? 1 : 0;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor next = next | (curr_descsz & TAVOR_WQE_NDS_MASK);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * A send queue doorbell will be rung for the next
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * WQE on the chain, set the current WQE's "dbd" bit.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Note: We also update the "dbinfo" structure here to pass
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * back information about what should (later) be included
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * in the send queue doorbell.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Send queue doorbell will be rung for the next WQE on
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the chain, update the prev WQE's "next" field and return.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * dapli_tavor_wqe_recv_build()
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Builds the recv WQE for a given ibt_recv_wr_t
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylordapli_tavor_wqe_recv_build(ib_qp_handle_t qp, ibt_recv_wr_t *wr,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Fill in the Data Segments (SGL) for the Recv WQE */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Check for valid number of SGL entries */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * For each SGL in the Recv Work Request, fill in the Recv WQE's data
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * segments. Note: We skip any SGL with zero size because Tavor
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * hardware cannot handle a zero for "byte_cnt" in the WQE. Actually
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the encoding for zero means a 2GB transfer. Because of this special
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * encoding in the hardware, we mask the requested length with
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * TAVOR_WQE_SGL_BYTE_CNT_MASK (so that 2GB will end up encoded as
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Fill in the Data Segment(s) for the receive WQE, using the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * information contained in the scatter-gather list of the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * work request.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TAVOR_WQE_BUILD_DATA_SEG(&ds[num_ds], &wr->wr_sgl[i]);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Return the size of descriptor (in 16-byte chunks) */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor *size = ((uintptr_t)&ds[num_ds] - (uintptr_t)addr) >> 0x4;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * dapli_tavor_wqe_recv_linknext()
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Links a recv WQE to the prev chain
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylordapli_tavor_wqe_recv_linknext(uint64_t *curr_addr, boolean_t ns,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor uint32_t curr_desc, uint_t curr_descsz, uint64_t *prev_addr)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Note: curr_addr is the last WQE (In uDAPL we manipulate 1 WQE
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * at a time. If there is no next descriptor (i.e. if the current
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * descriptor is the last WQE on the chain), then set "next" field
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * to TAVOR_WQE_DBD_MASK. This is because the Tavor hardware
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * requires the "dbd" bit to be set to one for all Recv WQEs.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * In either case, we must add a single bit in the "reserved" field
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * (TAVOR_RCV_WQE_NDA0_WA_MASK) following the NDA. This is the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * workaround for a known Tavor errata that can cause Recv WQEs with
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * zero in the NDA field to behave improperly.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * If notification suppression is not desired then we set
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the "E" bit in the ctrl field.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor next = TAVOR_WQE_DBD_MASK | TAVOR_RCV_WQE_NDA0_WA_MASK;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor if (!ns) { /* notification needed - so set the "E" bit */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* update the WQE */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Calculate the "next" field of the descriptor. This amounts
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * to setting up the "next_wqe_addr", "dbd", and "nds" fields
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * (see tavor_hw.h for more).
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor next = ((uint64_t)curr_desc & TAVOR_WQE_NDA_MASK) << 32;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor next = next | (curr_descsz & TAVOR_WQE_NDS_MASK) |
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TAVOR_WQE_DBD_MASK | TAVOR_RCV_WQE_NDA0_WA_MASK;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * If this WQE is supposed to be linked to the previous
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * descriptor, then we need to update not only the previous
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * WQE's "next" fields but we must not touch this WQE's
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * "ctrl" fields.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * dapli_tavor_wqe_srq_build()
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Builds the recv WQE for a given ibt_recv_wr_t
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylordapli_tavor_wqe_srq_build(ib_srq_handle_t srq, ibt_recv_wr_t *wr,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Fill in the Data Segments (SGL) for the Recv WQE */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Check for valid number of SGL entries */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * For each SGL in the Recv Work Request, fill in the Recv WQE's data
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * segments. Note: We skip any SGL with zero size because Tavor
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * hardware cannot handle a zero for "byte_cnt" in the WQE. Actually
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the encoding for zero means a 2GB transfer. Because of this special
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * encoding in the hardware, we mask the requested length with
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * TAVOR_WQE_SGL_BYTE_CNT_MASK (so that 2GB will end up encoded as
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Fill in the Data Segment(s) for the receive WQE, using the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * information contained in the scatter-gather list of the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * work request.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TAVOR_WQE_BUILD_DATA_SEG(&ds[num_ds], &wr->wr_sgl[i]);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * For SRQ, if the number of data segments is less than the maximum
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * specified at alloc, then we have to fill in a special "key" entry in
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the sgl entry after the last valid one in this post request. We do
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * that here.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TAVOR_WQE_BUILD_DATA_SEG(&ds[num_ds], &end_sgl);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * dapli_tavor_wqe_srq_linknext()
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Links a srq recv WQE to the prev chain
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylordapli_tavor_wqe_srq_linknext(uint64_t *curr_addr, boolean_t ns,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Note: curr_addr is the last WQE (In uDAPL we manipulate 1 WQE
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * at a time. If there is no next descriptor (i.e. if the current
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * descriptor is the last WQE on the chain), then set "next" field
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * to TAVOR_WQE_DBD_MASK. This is because the Tavor hardware
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * requires the "dbd" bit to be set to one for all Recv WQEs.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * In either case, we must add a single bit in the "reserved" field
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * (TAVOR_RCV_WQE_NDA0_WA_MASK) following the NDA. This is the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * workaround for a known Tavor errata that can cause Recv WQEs with
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * zero in the NDA field to behave improperly.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * If notification suppression is not desired then we set
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the "E" bit in the ctrl field.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor if (!ns) { /* notification needed - so set the "E" bit */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* update the WQE */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Calculate the "next" field of the descriptor. This amounts
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * to setting up the "next_wqe_addr", "dbd", and "nds" fields
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * (see tavor_hw.h for more).
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor next = ((uint64_t)curr_desc & TAVOR_WQE_NDA_MASK) << 32;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor next = next | TAVOR_WQE_DBD_MASK | TAVOR_RCV_WQE_NDA0_WA_MASK;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * If this WQE is supposed to be linked to the previous
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * descriptor, then we need to update not only the previous
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * WQE's "next" fields but we must not touch this WQE's
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * "ctrl" fields.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * dapli_tavor_cq_peek()
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Peeks into a given CQ to check if there are any events that can be
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * polled. It returns the number of CQEs that can be polled.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylordapli_tavor_cq_peek(ib_cq_handle_t cq, int *num_cqe)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Get the consumer index */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Calculate the wrap around mask. Note: This operation only works
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * because all Tavor completion queues have power-of-2 sizes
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Calculate the pointer to the first CQ entry */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Count entries in the CQ until we find an entry owned by
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the hardware.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Error CQE map to multiple work completions */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Increment the consumer index */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Update the pointer to the next CQ entry */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * dapli_tavor_cq_poll()
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * This routine polls CQEs out of a CQ and puts them into the ibt_wc_t
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * array that is passed in.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylordapli_tavor_cq_poll(ib_cq_handle_t cq, ibt_wc_t *wc_p, uint_t num_wc,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Get the consumer index */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Calculate the wrap around mask. Note: This operation only works
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * because all Tavor completion queues have power-of-2 sizes
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Calculate the pointer to the first CQ entry */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Keep pulling entries from the CQ until we find an entry owned by
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the hardware. As long as there the CQE's owned by SW, process
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * each entry by calling dapli_tavor_cq_cqe_consume() and updating the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * CQ consumer index. Note: We only update the consumer index if
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * dapli_tavor_cq_cqe_consume() returns TAVOR_CQ_SYNC_AND_DB.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Otherwise, it indicates that we are going to "recycle" the CQE
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * (probably because it is a error CQE and corresponds to more than one
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * completion).
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Reset entry to hardware ownership */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Increment the consumer index */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Update the pointer to the next CQ entry */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * If we have run out of space to store work completions,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * then stop and return the ones we have pulled of the CQ.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Now we only ring the doorbell (to update the consumer index) if
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * we've actually consumed a CQ entry. If we have, for example,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * pulled from a CQE that we are still in the process of "recycling"
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * for error purposes, then we would not update the consumer index.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor if ((polled_cnt != 0) && (cq->cq_consindx != cons_indx)) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Post doorbell to update the consumer index. Doorbell
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * value indicates number of entries consumed (minus 1)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor num_to_increment = (cons_indx - cq->cq_consindx) - 1;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor dapli_tavor_cq_doorbell(cq->cq_iauar, TAVOR_CQDB_INCR_CONSINDX,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor } else if (polled_cnt == 0) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * If the CQ is empty, we can try to free up some of the WRID
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * list containers.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor if (cq->cq_wrid_reap_head) /* look before leaping */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * dapli_tavor_cq_poll_one()
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * This routine polls one CQE out of a CQ and puts ot into the ibt_wc_t
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * that is passed in. See above for more comments/details.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylordapli_tavor_cq_poll_one(ib_cq_handle_t cq, ibt_wc_t *wc_p)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Get the consumer index */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Calculate the pointer to the first CQ entry */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Keep pulling entries from the CQ until we find an entry owned by
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the hardware. As long as there the CQE's owned by SW, process
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * each entry by calling dapli_tavor_cq_cqe_consume() and updating the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * CQ consumer index. Note: We only update the consumer index if
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * dapli_tavor_cq_cqe_consume() returns TAVOR_CQ_SYNC_AND_DB.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Otherwise, it indicates that we are going to "recycle" the CQE
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * (probably because it is a error CQE and corresponds to more than one
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * completion).
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor status = dapli_tavor_cq_cqe_consume(cq, cqe, wc_p);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Reset entry to hardware ownership */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Increment the consumer index */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor if (cq->cq_wrid_reap_head) /* look before leaping */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * dapli_tavor_cq_cqe_consume()
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Converts a given CQE into a ibt_wc_t object
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylordapli_tavor_cq_cqe_consume(ib_cq_handle_t cqhdl, tavor_hw_cqe_t *cqe,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Determine if this is an "error" CQE by examining "opcode". If it
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * is an error CQE, then call dapli_tavor_cq_errcqe_consume() and return
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * whatever status it returns. Otherwise, this is a successful
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * completion.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor status = dapli_tavor_cq_errcqe_consume(cqhdl, cqe, wc);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Fetch the Work Request ID using the information in the CQE.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * See tavor_wr.c for more details.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor wc->wc_id = dapls_tavor_wrid_get_entry(cqhdl, cqe,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Parse the CQE opcode to determine completion type. This will set
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * not only the type of the completion, but also any flags that might
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * be associated with it (e.g. whether immediate data is present).
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor if (TAVOR_CQE_SENDRECV_GET(cqe) != TAVOR_COMPLETION_RECV) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * The following opcodes will not be generated in uDAPL
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * case TAVOR_CQE_SND_RDMAWR_IMM:
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * case TAVOR_CQE_SND_SEND_IMM:
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * case TAVOR_CQE_SND_ATOMIC_CS:
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * case TAVOR_CQE_SND_ATOMIC_FA:
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Receive CQE
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * The following opcodes will not be generated in uDAPL
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * case TAVOR_CQE_RCV_RECV_IMM:
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * case TAVOR_CQE_RCV_RECV_IMM2:
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * case TAVOR_CQE_RCV_RDMAWR_IMM:
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * case TAVOR_CQE_RCV_RDMAWR_IMM2:
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* FALLTHROUGH */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* If we got here, completion status must be success */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * dapli_tavor_cq_errcqe_consume()
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylordapli_tavor_cq_errcqe_consume(ib_cq_handle_t cqhdl, tavor_hw_cqe_t *cqe,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor dapl_dbg_log(DAPL_DBG_TYPE_EVD, "errcqe_consume:cqe.eth=%x, wqe=%x\n",
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Fetch the Work Request ID using the information in the CQE.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * See tavor_wr.c for more details.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor wc->wc_id = dapls_tavor_wrid_get_entry(cqhdl, cqe,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor (opcode == TAVOR_CQE_SEND_ERR_OPCODE) ? TAVOR_COMPLETION_SEND :
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Parse the CQE opcode to determine completion type. We know that
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the CQE is an error completion, so we extract only the completion
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * status here.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor imm_eth_pkey_cred = TAVOR_CQE_IMM_ETH_PKEY_CRED_GET(cqe);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor status = imm_eth_pkey_cred >> TAVOR_CQE_ERR_STATUS_SHIFT;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * The following error codes are not supported in the Tavor driver
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * as they relate only to Reliable Datagram completion statuses:
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * case TAVOR_CQE_LOCAL_RDD_VIO_ERR:
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * case TAVOR_CQE_REM_INV_RD_REQ_ERR:
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * case TAVOR_CQE_EEC_REM_ABORTED_ERR:
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * case TAVOR_CQE_INV_EEC_NUM_ERR:
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * case TAVOR_CQE_INV_EEC_STATE_ERR:
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * case TAVOR_CQE_LOC_EEC_ERR:
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Now we do all the checking that's necessary to handle completion
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * queue entry "recycling"
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * It is not necessary here to try to sync the WQE as we are only
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * attempting to read from the Work Queue (and hardware does not
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * write to it).
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * We can get doorbell info, WQE address, size for the next WQE
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * from the "wre" (which was filled in above in the call to the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * tavor_wrid_get_entry() routine)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor dbd = (wre.wr_signaled_dbd & TAVOR_WRID_ENTRY_DOORBELLED) ? 1 : 0;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor nextwqesize = wre.wr_wqeaddrsz & TAVOR_WQE_NDS_MASK;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Get the doorbell count from the CQE. This indicates how many
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * completions this one CQE represents.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor doorbell_cnt = imm_eth_pkey_cred & TAVOR_CQE_ERR_DBDCNT_MASK;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Determine if we're ready to consume this CQE yet or not. If the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * next WQE has size zero (i.e. no next WQE) or if the doorbell count
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * is down to zero, then this is the last/only completion represented
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * by the current CQE (return TAVOR_CQ_SYNC_AND_DB). Otherwise, the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * current CQE needs to be recycled (see below).
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor if ((nextwqesize == 0) || ((doorbell_cnt == 0) && (dbd == 1))) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Consume the CQE
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Return status to indicate that doorbell and sync may be
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * necessary.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Recycle the CQE for use in the next PollCQ() call
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Decrement the doorbell count, modify the error status,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * and update the WQE address and size (to point to the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * next WQE on the chain. Put these update entries back
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * into the CQE.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Despite the fact that we have updated the CQE, it is not
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * necessary for us to attempt to sync this entry just yet
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * as we have not changed the "hardware's view" of the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * entry (i.e. we have not modified the "owner" bit - which
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * is all that the Tavor hardware really cares about.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ((TAVOR_CQE_WR_FLUSHED_ERR << TAVOR_CQE_ERR_STATUS_SHIFT) |
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor "errcqe_consume: recycling cqe.eth=%x, wqe=%x\n",
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * dapli_tavor_cq_notify()
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * This function is used for arming the CQ by ringing the CQ doorbell.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylordapli_tavor_cq_notify(ib_cq_handle_t cq, int flags, uint32_t param)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Determine if we are trying to get the next completion or the next
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * "solicited" completion. Then hit the appropriate doorbell.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor dapli_tavor_cq_doorbell(cq->cq_iauar, TAVOR_CQDB_NOTIFY_CQ,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor } else if (flags == IB_NOTIFY_ON_NEXT_SOLICITED) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor dapli_tavor_cq_doorbell(cq->cq_iauar, TAVOR_CQDB_NOTIFY_NCQ,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * dapli_tavor_post_send()
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylordapli_tavor_post_send(DAPL_EP *ep, ibt_send_wr_t *wr, boolean_t ns)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor "post_send: invalid qp_state %d\n", ep->qp_state);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Grab the lock for the WRID list */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor dapl_os_lock(&qp->qp_sq_wqhdr->wq_wrid_lock->wrl_lock);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Save away some initial QP state */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Check for "queue full" condition. If the queue is already full,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * then no more WQEs can be posted, return an error
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor dapl_os_unlock(&qp->qp_sq_wqhdr->wq_wrid_lock->wrl_lock);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Increment the "tail index" and check for "queue full" condition.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * If we detect that the current work request is going to fill the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * work queue, then we mark this condition and continue.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Get the user virtual address of the location where the next
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Send WQE should be built
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Call tavor_wqe_send_build() to build the WQE at the given address.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * This routine uses the information in the ibt_send_wr_t and
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * returns the size of the WQE when it returns.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor status = dapli_tavor_wqe_send_build(qp, wr, wqe_addr, &desc_sz);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor dapl_os_unlock(&qp->qp_sq_wqhdr->wq_wrid_lock->wrl_lock);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Get the descriptor (io address) corresponding to the location
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Send WQE was built.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Add a WRID entry to the WRID list. Need to calculate the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * "wqeaddrsz" and "signaled_dbd" values to pass to
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * dapli_tavor_wrid_add_entry()
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor dapli_tavor_wrid_add_entry(qp->qp_sq_wqhdr, wr->wr_id, wqeaddrsz,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Now link the wqe to the old chain (if there was one)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor dapli_tavor_wqe_send_linknext(wr, wqe_addr, ns, desc, desc_sz,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Now if the WRID tail entry is non-NULL, then this
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * represents the entry to which we are chaining the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * new entries. Since we are going to ring the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * doorbell for this WQE, we want set its "dbd" bit.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * On the other hand, if the tail is NULL, even though
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * we will have rung the doorbell for the previous WQE
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * (for the hardware's sake) it is irrelevant to our
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * purposes (for tracking WRIDs) because we know the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * request must have already completed.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor wre_last->wr_signaled_dbd |= TAVOR_WRID_ENTRY_DOORBELLED;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Update some of the state in the QP */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Ring the doorbell */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor dapli_tavor_qp_send_doorbell(qp->qp_iauar, desc, desc_sz,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor qp->qp_num, dbinfo.db_fence, dbinfo.db_nopcode);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor dapl_os_unlock(&qp->qp_sq_wqhdr->wq_wrid_lock->wrl_lock);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * dapli_tavor_post_recv()
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylordapli_tavor_post_recv(DAPL_EP *ep, ibt_recv_wr_t *wr, boolean_t ns)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor "post_recv: invalid qp_state %d\n", ep->qp_state);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Grab the lock for the WRID list */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor dapl_os_lock(&qp->qp_rq_wqhdr->wq_wrid_lock->wrl_lock);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Save away some initial QP state */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * For the ibt_recv_wr_t passed in, parse the request and build a
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Recv WQE. Link the WQE with the previous WQE and ring the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * door bell.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Check for "queue full" condition. If the queue is already full,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * then no more WQEs can be posted. So return an error.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor dapl_os_unlock(&qp->qp_rq_wqhdr->wq_wrid_lock->wrl_lock);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Increment the "tail index" and check for "queue
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * full" condition. If we detect that the current
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * work request is going to fill the work queue, then
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * we mark this condition and continue.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Get the descriptor (IO Address) of the WQE to be built */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* The user virtual address of the WQE to be built */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Call tavor_wqe_recv_build() to build the WQE at the given
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * address. This routine uses the information in the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * ibt_recv_wr_t and returns the size of the WQE.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor status = dapli_tavor_wqe_recv_build(qp, wr, wqe_addr, &desc_sz);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor dapl_os_unlock(&qp->qp_rq_wqhdr->wq_wrid_lock->wrl_lock);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Add a WRID entry to the WRID list. Need to calculate the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * "wqeaddrsz" and "signaled_dbd" values to pass to
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * dapli_tavor_wrid_add_entry().
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Note: all Recv WQEs are essentially "signaled"
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor dapli_tavor_wrid_add_entry(qp->qp_rq_wqhdr, wr->wr_id, wqeaddrsz,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Now link the chain to the old chain (if there was one)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * and ring the doorbel for the recv work queue.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor dapli_tavor_wqe_recv_linknext(wqe_addr, ns, desc, desc_sz,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Now if the WRID tail entry is non-NULL, then this
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * represents the entry to which we are chaining the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * new entries. Since we are going to ring the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * doorbell for this WQE, we want set its "dbd" bit.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * On the other hand, if the tail is NULL, even though
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * we will have rung the doorbell for the previous WQE
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * (for the hardware's sake) it is irrelevant to our
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * purposes (for tracking WRIDs) because we know the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * request must have already completed.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor wre_last->wr_signaled_dbd |= TAVOR_WRID_ENTRY_DOORBELLED;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Update some of the state in the QP */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Ring the doorbell */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor dapli_tavor_qp_recv_doorbell(qp->qp_iauar, desc, desc_sz,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor dapl_os_unlock(&qp->qp_rq_wqhdr->wq_wrid_lock->wrl_lock);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * dapli_tavor_post_srq()
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylordapli_tavor_post_srq(DAPL_SRQ *srqp, ibt_recv_wr_t *wr, boolean_t ns)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Grab the lock for the WRID list */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor dapl_os_lock(&srq->srq_wridlist->wl_lock->wrl_lock);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * For the ibt_recv_wr_t passed in, parse the request and build a
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Recv WQE. Link the WQE with the previous WQE and ring the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * door bell.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Check for "queue full" condition. If the queue is already full,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * ie. there are no free entries, then no more WQEs can be posted.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * So return an error.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor dapl_os_unlock(&srq->srq_wridlist->wl_lock->wrl_lock);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Save away some initial SRQ state */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Get the descriptor (IO Address) of the WQE to be built */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor wqe_index = TAVOR_SRQ_WQ_INDEX(srq->srq_wq_desc_addr, desc,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* The user virtual address of the WQE to be built */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Call dapli_tavor_wqe_srq_build() to build the WQE at the given
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * address. This routine uses the information in the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * ibt_recv_wr_t and returns the size of the WQE.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor status = dapli_tavor_wqe_srq_build(srq, wr, wqe_addr);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor dapl_os_unlock(&srq->srq_wridlist->wl_lock->wrl_lock);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Add a WRID entry to the WRID list.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor dapli_tavor_wrid_add_entry_srq(srq, wr->wr_id, wqe_index);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Now link the chain to the old chain (if there was one)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * and ring the doorbell for the SRQ.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor dapli_tavor_wqe_srq_linknext(wqe_addr, ns, desc, last_wqe_addr);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Update some of the state in the SRQ */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor dapl_os_assert(srq->srq_wridlist->wl_freel_entries <=
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Ring the doorbell - for SRQ nds = 0 */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor dapli_tavor_qp_recv_doorbell(srq->srq_iauar, desc, 0,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor dapl_os_unlock(&srq->srq_wridlist->wl_lock->wrl_lock);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * dapli_tavor_wrid_add_entry()
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylordapli_tavor_wrid_add_entry(dapls_tavor_workq_hdr_t *wq, uint64_t wrid,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Find the entry in the container pointed to by the "tail" index.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Add all of the relevant information to that entry, including WRID,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * "wqeaddrsz" parameter, and whether it was signaled/unsignaled
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * and/or doorbelled.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Update the "wrid_old_tail" pointer to point to the entry we just
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * inserted into the queue. By tracking this pointer (the pointer to
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the most recently inserted entry) it will possible later in the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * PostSend() and PostRecv() code paths to find the entry that needs
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * its "doorbelled" flag set (see comment in tavor_post_recv() and/or
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * tavor_post_send()).
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Update the tail index */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * If the "tail" index has just wrapped over into the "head" index,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * then we have filled the container. We use the "full" flag to
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * indicate this condition and to distinguish it from the "empty"
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * condition (where head and tail are also equal).
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * dapli_tavor_wrid_add_entry_srq()
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylordapli_tavor_wrid_add_entry_srq(ib_srq_handle_t srq, uint64_t wrid,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* ASSERT on impossible wqe_index values */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Setup the WRE.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Given the 'wqe_index' value, we store the WRID at this WRE offset.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * And we set the WRE to be signaled_dbd so that on poll CQ we can find
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * this information and associate the WRID to the WQE found on the CQE.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Note: all Recv WQEs are essentially "signaled"
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor wre->wr_signaled_dbd = (uint32_t)TAVOR_WRID_ENTRY_SIGNALED;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * dapli_tavor_cq_srq_entries_flush()
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylordapli_tavor_cq_srq_entries_flush(ib_qp_handle_t qp)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor uint32_t cons_indx, tail_cons_indx, wrap_around_mask;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* ASSERT(MUTEX_HELD(&qp->qp_rq_cqhdl->cq_lock)); */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor dapl_os_assert(wqhdr->wq_wrid_post->wl_srq_en != 0);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Get the consumer index */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Calculate the wrap around mask. Note: This operation only works
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * because all Tavor completion queues have power-of-2 sizes
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Calculate the pointer to the first CQ entry */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Loop through the CQ looking for entries owned by software. If an
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * entry is owned by software then we increment an 'outstanding_cqes'
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * count to know how many entries total we have on our CQ. We use this
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * value further down to know how many entries to loop through looking
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * for our same QP number.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* increment total cqes count */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* increment the consumer index */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor tail_cons_indx = (tail_cons_indx + 1) & wrap_around_mask;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* update the pointer to the next cq entry */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Using the 'tail_cons_indx' that was just set, we now know how many
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * total CQEs possible there are. Set the 'check_indx' and the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * 'new_indx' to the last entry identified by 'tail_cons_indx'
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor check_indx = new_indx = (tail_cons_indx - 1) & wrap_around_mask;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor for (i = 0; i < outstanding_cqes; i++) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Grab QP number from CQE */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * If the QP number is the same in the CQE as the QP that we
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * have on this SRQ, then we must free up the entry off the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * SRQ. We also make sure that the completion type is of the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * 'TAVOR_COMPLETION_RECV' type. So any send completions on
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * this CQ will be left as-is. The handling of returning
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * entries back to HW ownership happens further down.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Add back to SRQ free list */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Do Copy */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Copy the CQE into the "next_cqe"
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Move index to next CQE to check */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor check_indx = (check_indx - 1) & wrap_around_mask;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Initialize removed cqes count */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* If an entry was removed */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Set current pointer back to the beginning consumer index.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * At this point, all unclaimed entries have been copied to the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * index specified by 'new_indx'. This 'new_indx' will be used
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * as the new consumer index after we mark all freed entries as
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * having HW ownership. We do that here.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Loop through all entries until we reach our new pointer */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Reset entry to hardware ownership */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Update consumer index to be the 'new_indx'. This moves it past all
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * removed entries. Because 'new_indx' is pointing to the last
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * previously valid SW owned entry, we add 1 to point the cons_indx to
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the first HW owned entry.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Now we only ring the doorbell (to update the consumer index) if
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * we've actually consumed a CQ entry. If we found no QP number
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * matches above, then we would not have removed anything. So only if
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * something was removed do we ring the doorbell.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor if ((removed_cqes != 0) && (cq->cq_consindx != cons_indx)) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Post doorbell to update the consumer index. Doorbell
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * value indicates number of entries consumed (minus 1)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor num_to_increment = (cons_indx - cq->cq_consindx) - 1;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor dapli_tavor_cq_doorbell(cq->cq_iauar, TAVOR_CQDB_INCR_CONSINDX,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor/* ARGSUSED */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor/* ARGSUSED */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor/* ARGSUSED */