9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * CDDL HEADER START
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * The contents of this file are subject to the terms of the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Common Development and Distribution License (the "License").
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * You may not use this file except in compliance with the License.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * See the License for the specific language governing permissions
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * and limitations under the License.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * When distributing Covered Code, include this CDDL HEADER in each
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * If applicable, add the following below this CDDL HEADER, with the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * fields enclosed by brackets "[]" replaced with your own identifying
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * information: Portions Copyright [yyyy] [name of copyright owner]
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * CDDL HEADER END
86644ba28c50af32651bfc09efc3bd975a003ac1Bill Taylor * Copyright 2010 Sun Microsystems, Inc. All rights reserved.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Use is subject to license terms.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Tavor Work Request Processing Routines
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Implements all the routines necessary to provide the PostSend(),
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * PostRecv() and PostSRQ() verbs. Also contains all the code
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * necessary to implement the Tavor WRID tracking mechanism.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic void tavor_qp_send_doorbell(tavor_state_t *state, uint32_t nda,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor uint32_t nds, uint32_t qpn, uint32_t fence, uint32_t nopcode);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic void tavor_qp_recv_doorbell(tavor_state_t *state, uint32_t nda,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic uint32_t tavor_wr_get_immediate(ibt_send_wr_t *wr);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic int tavor_wr_bind_check(tavor_state_t *state, ibt_send_wr_t *wr);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic int tavor_wqe_send_build(tavor_state_t *state, tavor_qphdl_t qp,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ibt_send_wr_t *wr, uint64_t *desc, uint_t *size);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic void tavor_wqe_send_linknext(ibt_send_wr_t *curr_wr,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ibt_send_wr_t *prev_wr, uint64_t *curr_desc, uint_t curr_descsz,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor uint64_t *prev_desc, tavor_sw_wqe_dbinfo_t *dbinfo, tavor_qphdl_t qp);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic int tavor_wqe_mlx_build(tavor_state_t *state, tavor_qphdl_t qp,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ibt_send_wr_t *wr, uint64_t *desc, uint_t *size);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic void tavor_wqe_mlx_linknext(ibt_send_wr_t *prev_wr, uint64_t *curr_desc,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor uint_t curr_descsz, uint64_t *prev_desc, tavor_sw_wqe_dbinfo_t *dbinfo,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic int tavor_wqe_recv_build(tavor_state_t *state, tavor_qphdl_t qp,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ibt_recv_wr_t *wr, uint64_t *desc, uint_t *size);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic void tavor_wqe_recv_linknext(uint64_t *desc, uint_t desc_sz,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic int tavor_wqe_srq_build(tavor_state_t *state, tavor_srqhdl_t srq,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic void tavor_wqe_srq_linknext(uint64_t *desc, uint64_t *prev,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic void tavor_wqe_sync(void *hdl, uint_t sync_from,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic tavor_wrid_entry_t *tavor_wrid_find_match(tavor_workq_hdr_t *wq,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic void tavor_wrid_reaplist_add(tavor_cqhdl_t cq, tavor_workq_hdr_t *wq);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic tavor_workq_hdr_t *tavor_wrid_wqhdr_find(tavor_cqhdl_t cq, uint_t qpn,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic tavor_workq_hdr_t *tavor_wrid_wqhdr_create(tavor_state_t *state,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor tavor_cqhdl_t cq, uint_t qpn, uint_t wq_type, uint_t create_wql);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic uint32_t tavor_wrid_get_wqeaddrsz(tavor_workq_hdr_t *wq);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic void tavor_wrid_wqhdr_add(tavor_workq_hdr_t *wqhdr,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic void tavor_wrid_wqhdr_remove(tavor_workq_hdr_t *wqhdr,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic tavor_workq_hdr_t *tavor_wrid_list_reap(tavor_wrid_list_hdr_t *wq);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic void tavor_wrid_wqhdr_lock_both(tavor_qphdl_t qp);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic void tavor_wrid_wqhdr_unlock_both(tavor_qphdl_t qp);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic void tavor_cq_wqhdr_add(tavor_cqhdl_t cq, tavor_workq_hdr_t *wqhdr);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic void tavor_cq_wqhdr_remove(tavor_cqhdl_t cq, tavor_workq_hdr_t *wqhdr);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * tavor_post_send()
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Context: Can be called from interrupt or base context.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylortavor_post_send(tavor_state_t *state, tavor_qphdl_t qp,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ibt_send_wr_t *wr, uint_t num_wr, uint_t *num_posted)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Check for user-mappable QP memory. Note: We do not allow kernel
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * clients to post to QP memory that is accessible directly by the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * user. If the QP memory is user accessible, then return an error.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Initialize posted_cnt */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Check QP state. Can not post Send requests from the "Reset",
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * "Init", or "RTR" states
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Grab the lock for the WRID list */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor mutex_enter(&qp->qp_sq_wqhdr->wq_wrid_wql->wql_lock);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Save away some initial QP state */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * For each ibt_send_wr_t in the wr[] list passed in, parse the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * request and build a Send WQE. Note: Because we are potentially
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * building a chain of WQEs, we want to link them all together.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * However, we do not want to link the first one to the previous
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * WQE until the entire chain has been linked. Then in the last
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * step we ring the appropriate doorbell. Note: It is possible for
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * more Work Requests to be posted than the HW will support at one
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * shot. If this happens, we need to be able to post and ring
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * several chains here until the the entire request is complete.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor while ((wrindx < num_wr) && (status == DDI_SUCCESS)) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * For the first WQE on a new chain we need "prev" to point
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * to the current descriptor. As we begin to process
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * further, "prev" will be updated to point to the previous
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * WQE on the current chain (see below).
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Before we begin, save the current "tail index" for later
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Break the request up into chains that are less than or
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * equal to the maximum number of WQEs that can be posted
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * per doorbell ring
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor chainlen = (numremain > maxdb) ? maxdb : numremain;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor for (currindx = 0; currindx < chainlen; currindx++, wrindx++) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Check for "queue full" condition. If the queue
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * is already full, then no more WQEs can be posted.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * So break out, ring a doorbell (if necessary) and
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * return an error
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Increment the "tail index" and check for "queue
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * full" condition. If we detect that the current
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * work request is going to fill the work queue, then
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * we mark this condition and continue.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Get the address of the location where the next
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Send WQE should be built
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Call tavor_wqe_send_build() to build the WQE
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * at the given address. This routine uses the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * information in the ibt_send_wr_t list (wr[]) and
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * returns the size of the WQE when it returns.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Add a WRID entry to the WRID list. Need to
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * calculate the "wqeaddrsz" and "signaled_dbd"
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * values to pass to tavor_wrid_add_entry()
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor wqeaddrsz = TAVOR_QP_WQEADDRSZ((uint64_t *)(uintptr_t)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor if ((qp->qp_sq_sigtype == TAVOR_QP_SQ_ALL_SIGNALED) ||
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * If this is not the first descriptor on the current
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * chain, then link it to the previous WQE. Otherwise,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * save the address and size of this descriptor (in
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * "first" and "first_sz" respectively) and continue.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Note: Linking a WQE to the the previous one will
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * depend on whether the two WQEs are from "special
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * QPs" (i.e. MLX transport WQEs) or whether they are
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * normal Send WQEs.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Update the current "tail index" and increment
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * "posted_cnt"
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * If we reach here and there are one or more WQEs which have
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * been successfully chained together, then we need to link
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the current chain to the previously executing chain of
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * descriptor (if there is one) and ring the doorbell for the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * send work queue.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Before we link the chain, we need to ensure that the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * "next" field on the last WQE is set to NULL (to
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * indicate the end of the chain). Note: Just as it
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * did above, the format for the "next" fields in a
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * given WQE depend on whether the WQE is MLX
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * transport or not.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Save away updated "tail index" for the DMA sync */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Do a DMA sync for current send WQE(s) */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor tavor_wqe_sync(qp, sync_from, sync_to, TAVOR_WR_SEND,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Now link the chain to the old chain (if there was
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * one. Note: still need to pay attention to whether
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the QP used MLX transport WQEs or not.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * If there was a valid previous WQE (i.e. non-NULL),
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * then sync it too. This is because we have updated
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * its "next" fields and we want to ensure that the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * hardware can see the changes.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Now if the WRID tail entry is non-NULL, then this
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * represents the entry to which we are chaining the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * new entries. Since we are going to ring the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * doorbell for this WQE, we want set its "dbd" bit.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * On the other hand, if the tail is NULL, even though
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * we will have rung the doorbell for the previous WQE
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * (for the hardware's sake) it is irrelevant to our
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * purposes (for tracking WRIDs) because we know the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * request must have already completed.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Update some of the state in the QP */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Ring the doorbell */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Update the "num_posted" return value (if necessary). Then drop
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the locks and return success.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor mutex_exit(&qp->qp_sq_wqhdr->wq_wrid_wql->wql_lock);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * tavor_post_recv()
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Context: Can be called from interrupt or base context.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylortavor_post_recv(tavor_state_t *state, tavor_qphdl_t qp,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ibt_recv_wr_t *wr, uint_t num_wr, uint_t *num_posted)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Check for user-mappable QP memory. Note: We do not allow kernel
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * clients to post to QP memory that is accessible directly by the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * user. If the QP memory is user accessible, then return an error.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Initialize posted_cnt */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Check if QP is associated with an SRQ
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Check QP state. Can not post Recv requests from the "Reset" state
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Grab the lock for the WRID list */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor mutex_enter(&qp->qp_rq_wqhdr->wq_wrid_wql->wql_lock);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Save away some initial QP state */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * For each ibt_recv_wr_t in the wr[] list passed in, parse the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * request and build a Recv WQE. Note: Because we are potentially
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * building a chain of WQEs, we want to link them all together.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * However, we do not want to link the first one to the previous
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * WQE until the entire chain has been linked. Then in the last
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * step we ring the appropriate doorbell. Note: It is possible for
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * more Work Requests to be posted than the HW will support at one
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * shot. If this happens, we need to be able to post and ring
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * several chains here until the the entire request is complete.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor while ((wrindx < num_wr) && (status == DDI_SUCCESS)) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * For the first WQE on a new chain we need "prev" to point
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * to the current descriptor. As we begin to process
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * further, "prev" will be updated to point to the previous
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * WQE on the current chain (see below).
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Before we begin, save the current "tail index" for later
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Break the request up into chains that are less than or
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * equal to the maximum number of WQEs that can be posted
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * per doorbell ring
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor chainlen = (numremain > maxdb) ? maxdb : numremain;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor for (currindx = 0; currindx < chainlen; currindx++, wrindx++) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Check for "queue full" condition. If the queue
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * is already full, then no more WQEs can be posted.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * So break out, ring a doorbell (if necessary) and
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * return an error
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Increment the "tail index" and check for "queue
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * full" condition. If we detect that the current
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * work request is going to fill the work queue, then
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * we mark this condition and continue.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Get the address of the location where the next
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Recv WQE should be built
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Call tavor_wqe_recv_build() to build the WQE
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * at the given address. This routine uses the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * information in the ibt_recv_wr_t list (wr[]) and
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * returns the size of the WQE when it returns.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor status = tavor_wqe_recv_build(state, qp, &wr[wrindx],
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Add a WRID entry to the WRID list. Need to
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * calculate the "wqeaddrsz" and "signaled_dbd"
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * values to pass to tavor_wrid_add_entry(). Note:
86644ba28c50af32651bfc09efc3bd975a003ac1Bill Taylor * all Recv WQEs are essentially "signaled" and
86644ba28c50af32651bfc09efc3bd975a003ac1Bill Taylor * "doorbelled" (since Tavor HW requires all
86644ba28c50af32651bfc09efc3bd975a003ac1Bill Taylor * RecvWQE's to have their "DBD" bits set).
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor wqeaddrsz = TAVOR_QP_WQEADDRSZ((uint64_t *)(uintptr_t)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * If this is not the first descriptor on the current
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * chain, then link it to the previous WQE. Otherwise,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * save the address and size of this descriptor (in
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * "first" and "first_sz" respectively) and continue.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Update the current "tail index" and increment
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * "posted_cnt"
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * If we reach here and there are one or more WQEs which have
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * been successfully chained together, then we need to link
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the current chain to the previously executing chain of
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * descriptor (if there is one) and ring the doorbell for the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * recv work queue.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Before we link the chain, we need to ensure that the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * "next" field on the last WQE is set to NULL (to
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * indicate the end of the chain).
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Save away updated "tail index" for the DMA sync */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Do a DMA sync for current recv WQE(s) */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor tavor_wqe_sync(qp, sync_from, sync_to, TAVOR_WR_RECV,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Now link the chain to the old chain (if there was
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * If there was a valid previous WQE (i.e. non-NULL),
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * then sync it too. This is because we have updated
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * its "next" fields and we want to ensure that the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * hardware can see the changes.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Update some of the state in the QP */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Ring the doorbell */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Update the "num_posted" return value (if necessary). Then drop
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the locks and return success.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor mutex_exit(&qp->qp_rq_wqhdr->wq_wrid_wql->wql_lock);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * tavor_post_srq()
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Context: Can be called from interrupt or base context.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylortavor_post_srq(tavor_state_t *state, tavor_srqhdl_t srq,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ibt_recv_wr_t *wr, uint_t num_wr, uint_t *num_posted)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Check for user-mappable QP memory. Note: We do not allow kernel
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * clients to post to QP memory that is accessible directly by the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * user. If the QP memory is user accessible, then return an error.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Initialize posted_cnt */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Check SRQ state. Can not post Recv requests when SRQ is in error
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Grab the lock for the WRID list */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * For each ibt_recv_wr_t in the wr[] list passed in, parse the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * request and build a Recv WQE. Note: Because we are potentially
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * building a chain of WQEs, we want to link them all together.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * However, we do not want to link the first one to the previous
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * WQE until the entire chain has been linked. Then in the last
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * step we ring the appropriate doorbell. Note: It is possible for
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * more Work Requests to be posted than the HW will support at one
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * shot. If this happens, we need to be able to post and ring
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * several chains here until the the entire request is complete.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor while ((wrindx < num_wr) && (status == DDI_SUCCESS)) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * For the first WQE on a new chain we need "prev" to point
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * to the current descriptor. As we begin to process
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * further, "prev" will be updated to point to the previous
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * WQE on the current chain (see below).
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor prev = TAVOR_SRQ_WQE_ADDR(srq, srq->srq_wq_lastwqeindx);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Break the request up into chains that are less than or
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * equal to the maximum number of WQEs that can be posted
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * per doorbell ring
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor chainlen = (numremain > maxdb) ? maxdb : numremain;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor for (currindx = 0; currindx < chainlen; currindx++, wrindx++) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Check for "queue full" condition. If the queue
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * is already full, then no more WQEs can be posted.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * So break out, ring a doorbell (if necessary) and
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * return an error
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor if (srq->srq_wridlist->wl_free_list_indx == -1) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Get the address of the location where the next
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Recv WQE should be built
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Add a WRID entry to the WRID list. Need to
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * set the "signaled_dbd" values to pass to
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * tavor_wrid_add_entry(). Note: all Recv WQEs are
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * essentially "signaled"
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * The 'size' is stored at srq_alloc time, in the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * srq_wq_stride. This is a constant value required
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Call tavor_wqe_srq_build() to build the WQE
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * at the given address. This routine uses the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * information in the ibt_recv_wr_t list (wr[]) and
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * returns the size of the WQE when it returns.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor status = tavor_wqe_srq_build(state, srq, &wr[wrindx],
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * If this is not the first descriptor on the current
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * chain, then link it to the previous WQE. Otherwise,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * save the address of this descriptor (in "first") and
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Do a DMA sync for previous recv WQE */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * In this case, the last WQE on the chain is
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * also considered 'first'. So set prev to
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * first, here.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Increment "posted_cnt"
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * If we reach here and there are one or more WQEs which have
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * been successfully chained together, then we need to link
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the current chain to the previously executing chain of
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * descriptor (if there is one) and ring the doorbell for the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * recv work queue.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Before we link the chain, we need to ensure that the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * "next" field on the last WQE is set to NULL (to
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * indicate the end of the chain).
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor sync_indx = TAVOR_SRQ_WQE_INDEX(srq->srq_wq_buf, prev,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Do a DMA sync for current recv WQE */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Now link the chain to the old chain (if there was
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor tavor_wqe_srq_linknext(first, last_wqe_addr, srq);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * If there was a valid previous WQE (i.e. valid index),
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * then sync it too. This is because we have updated
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * its "next" fields and we want to ensure that the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * hardware can see the changes.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Update some of the state in the QP */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Ring the doorbell */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* SRQ needs NDS of 0 */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor (uint32_t)((uintptr_t)first - srq->srq_desc_off),
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Update the "num_posted" return value (if necessary). Then drop
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the locks and return success.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * tavor_qp_send_doorbell()
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Context: Can be called from interrupt or base context.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylortavor_qp_send_doorbell(tavor_state_t *state, uint32_t nda, uint32_t nds,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Build the doorbell from the parameters */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor doorbell = (((uint64_t)nda & TAVOR_QPSNDDB_NDA_MASK) <<
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ((uint64_t)nopcode << TAVOR_QPSNDDB_NOPCODE_SHIFT) |
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ((uint64_t)qpn << TAVOR_QPSNDDB_QPN_SHIFT) | nds;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TNF_PROBE_1_DEBUG(tavor_qp_send_doorbell, TAVOR_TNF_TRACE, "",
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Write the doorbell to UAR */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TAVOR_UAR_DOORBELL(state, (uint64_t *)&state->ts_uar->send,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * tavor_qp_recv_doorbell()
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Context: Can be called from interrupt or base context.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylortavor_qp_recv_doorbell(tavor_state_t *state, uint32_t nda, uint32_t nds,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Build the doorbell from the parameters */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor doorbell = (((uint64_t)nda & TAVOR_QPRCVDB_NDA_MASK) <<
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ((uint64_t)qpn << TAVOR_QPRCVDB_QPN_SHIFT) | credits;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TNF_PROBE_1_DEBUG(tavor_qp_recv_doorbell, TAVOR_TNF_TRACE, "",
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Write the doorbell to UAR */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TAVOR_UAR_DOORBELL(state, (uint64_t *)&state->ts_uar->recv,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * tavor_wqe_send_build()
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Context: Can be called from interrupt or base context.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylortavor_wqe_send_build(tavor_state_t *state, tavor_qphdl_t qp,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ibt_send_wr_t *wr, uint64_t *desc, uint_t *size)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Initialize the information for the Data Segments */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Build a Send WQE depends first and foremost on the transport
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * type of Work Request (i.e. UD, RC, or UC)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Ensure that work request transport type matches QP type */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TNF_PROBE_0(tavor_wqe_send_build_inv_servtype_fail,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Validate the operation type. For UD requests, only the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * "Send" operation is valid
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TNF_PROBE_0(tavor_wqe_send_build_inv_optype_fail,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * If this is a Special QP (QP0 or QP1), then we need to
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * build MLX WQEs instead. So jump to tavor_wqe_mlx_build()
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * and return whatever status it returns
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor status = tavor_wqe_mlx_build(state, qp, wr, desc, size);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Otherwise, if this is a normal UD Send request, then fill
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * all the fields in the Tavor UD header for the WQE. Note:
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * to do this we'll need to extract some information from the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Address Handle passed with the work request.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ud = (tavor_hw_snd_wqe_ud_t *)((uintptr_t)desc +
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Build the Unreliable Datagram Segment for the WQE, using
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the information from the address handle and the work
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Update "ds" for filling in Data Segments (below) */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Ensure that work request transport type matches QP type */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TNF_PROBE_0(tavor_wqe_send_build_inv_servtype_fail,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Validate the operation type. For RC requests, we allow
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * "Send", "RDMA Read", "RDMA Write", various "Atomic"
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * operations, and memory window "Bind"
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TNF_PROBE_0(tavor_wqe_send_build_inv_optype_fail,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * If this is a Send request, then all we need to do is break
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * out and here and begin the Data Segment processing below
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * If this is an RDMA Read or RDMA Write request, then fill
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * in the "Remote Address" header fields.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor rc = (tavor_hw_snd_wqe_remaddr_t *)((uintptr_t)desc +
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Build the Remote Address Segment for the WQE, using
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the information from the RC work request.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TAVOR_WQE_BUILD_REMADDR(qp, rc, &wr->wr.rc.rcwr.rdma);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Update "ds" for filling in Data Segments (below) */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * If this is one of the Atomic type operations (i.e
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Compare-Swap or Fetch-Add), then fill in both the "Remote
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Address" header fields and the "Atomic" header fields.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor rc = (tavor_hw_snd_wqe_remaddr_t *)((uintptr_t)desc +
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor at = (tavor_hw_snd_wqe_atomic_t *)((uintptr_t)rc +
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Build the Remote Address and Atomic Segments for
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the WQE, using the information from the RC Atomic
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * work request.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TAVOR_WQE_BUILD_ATOMIC(qp, at, wr->wr.rc.rcwr.atomic);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Update "ds" for filling in Data Segments (below) */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Update "nds" and "sgl" because Atomic requests have
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * only a single Data Segment (and they are encoded
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * somewhat differently in the work request.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * If this is memory window Bind operation, then we call the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * tavor_wr_bind_check() routine to validate the request and
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * to generate the updated RKey. If this is successful, then
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * we fill in the WQE's "Bind" header fields.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor bn = (tavor_hw_snd_wqe_bind_t *)((uintptr_t)desc +
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Build the Bind Memory Window Segments for the WQE,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * using the information from the RC Bind memory
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * window work request.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TAVOR_WQE_BUILD_BIND(qp, bn, wr->wr.rc.rcwr.bind);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Update the "ds" pointer. Even though the "bind"
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * operation requires no SGLs, this is necessary to
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * facilitate the correct descriptor size calculations
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Ensure that work request transport type matches QP type */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TNF_PROBE_0(tavor_wqe_send_build_inv_servtype_fail,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Validate the operation type. For UC requests, we only
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * allow "Send", "RDMA Write", and memory window "Bind".
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Note: Unlike RC, UC does not allow "RDMA Read" or "Atomic"
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * operations
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TNF_PROBE_0(tavor_wqe_send_build_inv_optype_fail,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * If this is a Send request, then all we need to do is break
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * out and here and begin the Data Segment processing below
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * If this is an RDMA Write request, then fill in the "Remote
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Address" header fields.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor uc = (tavor_hw_snd_wqe_remaddr_t *)((uintptr_t)desc +
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Build the Remote Address Segment for the WQE, using
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the information from the UC work request.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TAVOR_WQE_BUILD_REMADDR(qp, uc, &wr->wr.uc.ucwr.rdma);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Update "ds" for filling in Data Segments (below) */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * If this is memory window Bind operation, then we call the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * tavor_wr_bind_check() routine to validate the request and
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * to generate the updated RKey. If this is successful, then
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * we fill in the WQE's "Bind" header fields.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor bn = (tavor_hw_snd_wqe_bind_t *)((uintptr_t)desc +
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Build the Bind Memory Window Segments for the WQE,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * using the information from the UC Bind memory
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * window work request.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TAVOR_WQE_BUILD_BIND(qp, bn, wr->wr.uc.ucwr.bind);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Update the "ds" pointer. Even though the "bind"
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * operation requires no SGLs, this is necessary to
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * facilitate the correct descriptor size calculations
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TNF_PROBE_0(tavor_wqe_send_build_inv_tranport_fail,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Now fill in the Data Segments (SGL) for the Send WQE based on
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the values setup above (i.e. "sgl", "nds", and the "ds" pointer
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Start by checking for a valid number of SGL entries
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TNF_PROBE_0(tavor_wqe_send_build_toomanysgl_fail,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * For each SGL in the Send Work Request, fill in the Send WQE's data
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * segments. Note: We skip any SGL with zero size because Tavor
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * hardware cannot handle a zero for "byte_cnt" in the WQE. Actually
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the encoding for zero means a 2GB transfer. Because of this special
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * encoding in the hardware, we mask the requested length with
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * TAVOR_WQE_SGL_BYTE_CNT_MASK (so that 2GB will end up encoded as
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor for (i = 0; i < nds; i++) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Fill in the Data Segment(s) for the current WQE, using the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * information contained in the scatter-gather list of the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * work request.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TAVOR_WQE_BUILD_DATA_SEG(qp, &ds[num_ds], &sgl[i]);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Return the size of descriptor (in 16-byte chunks) */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor *size = ((uintptr_t)&ds[num_ds] - (uintptr_t)desc) >> 4;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * tavor_wqe_send_linknext()
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Context: Can be called from interrupt or base context.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylortavor_wqe_send_linknext(ibt_send_wr_t *curr_wr, ibt_send_wr_t *prev_wr,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor uint64_t *curr_desc, uint_t curr_descsz, uint64_t *prev_desc,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor tavor_sw_wqe_dbinfo_t *dbinfo, tavor_qphdl_t qp)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Calculate the "next" field of the descriptor. This amounts to
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * setting up the "next_wqe_addr", "nopcode", "fence", and "nds"
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * fields (see tavor_hw.h for more). Note: If there is no next
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * descriptor (i.e. if the current descriptor is the last WQE on
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the chain), then set "next" to zero.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Determine the value for the Tavor WQE "nopcode" field
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * by using the IBTF opcode from the work request
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor curr_desc = (uint64_t *)(uintptr_t)((uintptr_t)curr_desc
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor fence = (curr_wr->wr_flags & IBT_WR_SEND_FENCE) ? 1 : 0;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor next = next | (curr_descsz & TAVOR_WQE_NDS_MASK);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * If a send queue doorbell will be rung for the next
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * WQE on the chain, then set the current WQE's "dbd" bit.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Note: We also update the "dbinfo" structure here to pass
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * back information about what should (later) be included
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * in the send queue doorbell.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * If this WQE is supposed to be linked to the previous descriptor,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * then we need to update not only the previous WQE's "next" fields
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * but we must also update this WQE's "ctrl" fields (i.e. the "c", "e",
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * "s", "i" and "immediate" fields - see tavor_hw.h for more). Note:
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the "e" bit is always hardcoded to zero.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * If a send queue doorbell will be rung for the next WQE on
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the chain, then update the current WQE's "next" field and
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Note: We don't want to modify the "ctrl" field here because
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * that portion of the previous WQE has already been set
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * correctly at some previous point in time.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Set the "c" (i.e. "signaled") bit appropriately */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Set the "s" (i.e. "solicited") bit appropriately */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Set the "i" bit and the immediate data appropriately */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * tavor_wqe_mlx_build()
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Context: Can be called from interrupt or base context.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylortavor_wqe_mlx_build(tavor_state_t *state, tavor_qphdl_t qp,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ibt_send_wr_t *wr, uint64_t *desc, uint_t *size)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Initialize the information for the Data Segments */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Pull the address handle from the work request and read in
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the contents of the UDAV. This will be used to answer some
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * questions about the request.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor for (i = 0; i < udav_sz; i++) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * If the request is for QP1 and the destination LID is equal to
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the Permissive LID, then return an error. This combination is
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * not allowed
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TNF_PROBE_0(tavor_wqe_mlx_build_permissiveLIDonQP1_fail,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Calculate the size of the packet headers, including the GRH
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * (if necessary)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor desc_sz = sizeof (ib_lrh_hdr_t) + sizeof (ib_bth_hdr_t) +
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Begin to build the first "inline" data segment for the packet
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * headers. Note: By specifying "inline" we can build the contents
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * of the MAD packet headers directly into the work queue (as part
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * descriptor). This has the advantage of both speeding things up
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * and of not requiring the driver to allocate/register any additional
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * memory for the packet headers.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Build Local Route Header (LRH)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * We start here by building the LRH into a temporary location.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * When we have finished we copy the LRH data into the descriptor.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Notice that the VL values are hardcoded. This is not a problem
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * because VL15 is decided later based on the value in the MLX
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * transport "next/ctrl" header (see the "vl15" bit below), and it
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * is otherwise (meaning for QP1) chosen from the SL-to-VL table
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * values. This rule does not hold for loopback packets however
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * (all of which bypass the SL-to-VL tables) and it is the reason
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * that non-QP0 MADs are setup with VL hardcoded to zero below.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Notice also that Source LID is hardcoded to the Permissive LID
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * (0xFFFF). This is also not a problem because if the Destination
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * LID is not the Permissive LID, then the "slr" value in the MLX
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * transport "next/ctrl" header will be set to zero and the hardware
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * will pull the LID from value in the port.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Build Global Route Header (GRH)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * This is only built if necessary as defined by the "grh" bit in
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the address vector. Note: We also calculate the offset to the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * next header (BTH) based on whether or not the "grh" bit is set.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * If the request is for QP0, then return an error. The
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * combination of global routine (GRH) and QP0 is not allowed.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor grh = (ib_grh_t *)((uintptr_t)lrh + sizeof (ib_lrh_hdr_t));
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TAVOR_WQE_BUILD_MLX_GRH(state, grh, qp, udav, pktlen);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor bth = (ib_bth_hdr_t *)((uintptr_t)grh + sizeof (ib_grh_t));
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor bth = (ib_bth_hdr_t *)((uintptr_t)lrh + sizeof (ib_lrh_hdr_t));
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Build Base Transport Header (BTH)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Notice that the M, PadCnt, and TVer fields are all set
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * to zero implicitly. This is true for all Management Datagrams
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * MADs whether GSI are SMI.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Build Datagram Extended Transport Header (DETH)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor deth = (ib_deth_hdr_t *)((uintptr_t)bth + sizeof (ib_bth_hdr_t));
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Ensure that the Data Segment is aligned on a 16-byte boundary */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ds = (tavor_hw_wqe_sgl_t *)((uintptr_t)deth + sizeof (ib_deth_hdr_t));
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ds = (tavor_hw_wqe_sgl_t *)(((uintptr_t)ds + 0xF) & ~0xF);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Now fill in the Data Segments (SGL) for the MLX WQE based on the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * values set up above (i.e. "sgl", "nds", and the "ds" pointer
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Start by checking for a valid number of SGL entries
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TNF_PROBE_0(tavor_wqe_mlx_build_toomanysgl_fail,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * For each SGL in the Send Work Request, fill in the MLX WQE's data
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * segments. Note: We skip any SGL with zero size because Tavor
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * hardware cannot handle a zero for "byte_cnt" in the WQE. Actually
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the encoding for zero means a 2GB transfer. Because of this special
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * encoding in the hardware, we mask the requested length with
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * TAVOR_WQE_SGL_BYTE_CNT_MASK (so that 2GB will end up encoded as
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor for (i = 0; i < nds; i++) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Fill in the Data Segment(s) for the MLX send WQE, using
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the information contained in the scatter-gather list of
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the work request.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TAVOR_WQE_BUILD_DATA_SEG(qp, &ds[num_ds], &sgl[i]);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Search through the contents of all MADs posted to QP0 to
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * initialize pointers to the places where Directed Route "hop
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * pointer", "hop count", and "mgmtclass" would be. Tavor
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * needs these updated (i.e. incremented or decremented, as
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * necessary) by software.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Tavor's Directed Route MADs need to have the "hop pointer"
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * incremented/decremented (as necessary) depending on whether it is
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * currently less than or greater than the "hop count" (i.e. whether
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the MAD is a request or a response.)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TAVOR_SPECIAL_QP_DRMAD_DO_HOPPOINTER_MODIFY(*mgmtclass,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Now fill in the ICRC Data Segment. This data segment is inlined
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * just like the packets headers above, but it is only four bytes and
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * set to zero (to indicate that we wish the hardware to generate ICRC.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TAVOR_WQE_BUILD_INLINE_ICRC(qp, &ds[num_ds], 4, 0);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Return the size of descriptor (in 16-byte chunks) */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor *size = ((uintptr_t)&ds[num_ds] - (uintptr_t)desc) >> 0x4;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * tavor_wqe_mlx_linknext()
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Context: Can be called from interrupt or base context.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylortavor_wqe_mlx_linknext(ibt_send_wr_t *prev_wr, uint64_t *curr_desc,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor uint_t curr_descsz, uint64_t *prev_desc, tavor_sw_wqe_dbinfo_t *dbinfo,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Calculate the "next" field of the descriptor. This amounts to
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * setting up the "next_wqe_addr", "nopcode", and "nds" fields (see
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * tavor_hw.h for more). Note: If there is no next descriptor (i.e.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * if the current descriptor is the last WQE on the chain), then set
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * "next" to zero.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * The only valid Tavor WQE "nopcode" for MLX transport
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * requests is the "Send" code.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor next = next | (curr_descsz & TAVOR_WQE_NDS_MASK);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * If a send queue doorbell will be rung for the next
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * WQE on the chain, then set the current WQE's "dbd" bit.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Note: We also update the "dbinfo" structure here to pass
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * back information about what should (later) be included
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * in the send queue doorbell.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * If this WQE is supposed to be linked to the previous descriptor,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * then we need to update not only the previous WQE's "next" fields
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * but we must also update this WQE's "ctrl" fields (i.e. the "vl15",
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * "slr", "max_srate", "sl", "c", "e", "rlid", and "vcrc" fields -
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * see tavor_hw.h for more) Note: the "e" bit and "vcrc" fields are
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * always hardcoded to zero.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * If a send queue doorbell will be rung for the next WQE on
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the chain, then update the current WQE's "next" field and
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Note: We don't want to modify the "ctrl" field here because
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * that portion of the previous WQE has already been set
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * correctly at some previous point in time.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Pull the address handle from the work request and read in
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the contents of the UDAV. This will be used to answer some
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * questions about the request.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ah = (tavor_ahhdl_t)prev_wr->wr.ud.udwr_dest->ud_ah;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor for (i = 0; i < udav_sz; i++) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Only QP0 uses VL15, otherwise use VL in the packet */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * The SLR (Source LID Replace) bit determines whether the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * source LID for an outgoing MLX packet should come from the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * PortInfo (SLR = 0) or should be left as it is in the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * descriptor (SLR = 1). The latter is necessary for packets
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * to be sent with the Permissive LID.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Fill in the max static rate from the address handle */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* All VL15 (i.e. SMI) traffic is required to use SL 0 */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Set the "c" (i.e. "signaled") bit appropriately */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Fill in the destination LID from the address handle */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * tavor_wqe_recv_build()
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Context: Can be called from interrupt or base context.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor/* ARGSUSED */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylortavor_wqe_recv_build(tavor_state_t *state, tavor_qphdl_t qp,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ibt_recv_wr_t *wr, uint64_t *desc, uint_t *size)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Check that work request transport type is valid */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TNF_PROBE_0(tavor_build_recv_wqe_inv_servtype_fail,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Fill in the Data Segments (SGL) for the Recv WQE */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Check for valid number of SGL entries */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TNF_PROBE_0(tavor_wqe_recv_build_toomanysgl_fail,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * For each SGL in the Recv Work Request, fill in the Recv WQE's data
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * segments. Note: We skip any SGL with zero size because Tavor
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * hardware cannot handle a zero for "byte_cnt" in the WQE. Actually
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the encoding for zero means a 2GB transfer. Because of this special
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * encoding in the hardware, we mask the requested length with
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * TAVOR_WQE_SGL_BYTE_CNT_MASK (so that 2GB will end up encoded as
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Fill in the Data Segment(s) for the receive WQE, using the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * information contained in the scatter-gather list of the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * work request.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TAVOR_WQE_BUILD_DATA_SEG(qp, &ds[num_ds], &wr->wr_sgl[i]);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Return the size of descriptor (in 16-byte chunks) */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor *size = ((uintptr_t)&ds[num_ds] - (uintptr_t)desc) >> 0x4;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * tavor_wqe_recv_linknext()
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Context: Can be called from interrupt or base context.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylortavor_wqe_recv_linknext(uint64_t *curr_desc, uint_t curr_descsz,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Calculate the "next" field of the descriptor. This amounts to
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * setting up the "next_wqe_addr", "dbd", and "nds" fields (see
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * tavor_hw.h for more). Note: If there is no next descriptor (i.e.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * if the current descriptor is the last WQE on the chain), then set
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * "next" field to TAVOR_WQE_DBD_MASK. This is because the Tavor
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * hardware requires the "dbd" bit to be set to one for all Recv WQEs.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * In either case, we must add a single bit in the "reserved" field
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * (TAVOR_RCV_WQE_NDA0_WA_MASK) following the NDA. This is the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * workaround for a known Tavor errata that can cause Recv WQEs with
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * zero in the NDA field to behave improperly.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor curr_desc = (uint64_t *)(uintptr_t)((uintptr_t)curr_desc -
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor next = next | (curr_descsz & TAVOR_WQE_NDS_MASK) |
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TAVOR_WQE_DBD_MASK | TAVOR_RCV_WQE_NDA0_WA_MASK;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor next = TAVOR_WQE_DBD_MASK | TAVOR_RCV_WQE_NDA0_WA_MASK;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * If this WQE is supposed to be linked to the previous descriptor,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * then we need to update not only the previous WQE's "next" fields
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * but we must also update this WQE's "ctrl" fields (i.e. the "c" and
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * "e" bits - see tavor_hw.h for more). Note: both the "c" and "e"
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * bits are always hardcoded to zero.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * tavor_wqe_srq_build()
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Context: Can be called from interrupt or base context.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor/* ARGSUSED */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylortavor_wqe_srq_build(tavor_state_t *state, tavor_srqhdl_t srq,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Fill in the Data Segments (SGL) for the Recv WQE */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Check for valid number of SGL entries */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TNF_PROBE_0(tavor_wqe_srq_build_toomanysgl_fail,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * For each SGL in the Recv Work Request, fill in the Recv WQE's data
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * segments. Note: We skip any SGL with zero size because Tavor
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * hardware cannot handle a zero for "byte_cnt" in the WQE. Actually
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the encoding for zero means a 2GB transfer. Because of this special
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * encoding in the hardware, we mask the requested length with
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * TAVOR_WQE_SGL_BYTE_CNT_MASK (so that 2GB will end up encoded as
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Fill in the Data Segment(s) for the receive WQE, using the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * information contained in the scatter-gather list of the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * work request.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TAVOR_WQE_BUILD_DATA_SEG_SRQ(srq, &ds[num_ds], &wr->wr_sgl[i]);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * For SRQ, if the number of data segments is less than the maximum
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * specified at alloc, then we have to fill in a special "key" entry in
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the sgl entry after the last valid one in this post request. We do
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * that here.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TAVOR_WQE_BUILD_DATA_SEG_SRQ(srq, &ds[num_ds], &end_sgl);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * tavor_wqe_srq_linknext()
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Context: Can be called from interrupt or base context.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylortavor_wqe_srq_linknext(uint64_t *curr_desc, uint64_t *prev_desc,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Calculate the "next" field of the descriptor. This amounts to
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * setting up the "next_wqe_addr", "dbd", and "nds" fields (see
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * tavor_hw.h for more). Note: If there is no next descriptor (i.e.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * if the current descriptor is the last WQE on the chain), then set
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * "next" field to TAVOR_WQE_DBD_MASK. This is because the Tavor
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * hardware requires the "dbd" bit to be set to one for all Recv WQEs.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * In either case, we must add a single bit in the "reserved" field
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * (TAVOR_RCV_WQE_NDA0_WA_MASK) following the NDA. This is the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * workaround for a known Tavor errata that can cause Recv WQEs with
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * zero in the NDA field to behave improperly.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor curr_desc = (uint64_t *)(uintptr_t)((uintptr_t)curr_desc -
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor next = next | TAVOR_WQE_DBD_MASK | TAVOR_RCV_WQE_NDA0_WA_MASK;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * If this WQE is supposed to be linked to the previous descriptor,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * then we need to update not only the previous WQE's "next" fields
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * but we must also update this WQE's "ctrl" fields (i.e. the "c" and
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * "e" bits - see tavor_hw.h for more). Note: both the "c" and "e"
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * bits are always hardcoded to zero.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TAVOR_WQE_LINKNEXT_SRQ(srq, prev_desc, 0, next);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * tavor_wr_get_immediate()
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Context: Can be called from interrupt or base context.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * This routine extracts the "immediate data" from the appropriate
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * location in the IBTF work request. Because of the way the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * work request structure is defined, the location for this data
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * depends on the actual work request operation type.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* For RDMA Write, test if RC or UC */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor } else { /* IBT_UC_SRV */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* For Send, test if RC, UD, or UC */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor } else { /* IBT_UC_SRV */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * If any other type of request, then immediate is undefined
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * tavor_wqe_sync()
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Context: Can be called from interrupt or base context.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylortavor_wqe_sync(void *hdl, uint_t sync_from, uint_t sync_to,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor uint64_t *wqe_from, *wqe_to, *wqe_base, *wqe_top;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Get the DMA handle from SRQ context */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Get the DMA handle from QP context */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Determine if the work queues need to be synced or not */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Depending on the type of the work queue, we grab information
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * about the address ranges we need to DMA sync.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * There are two possible cases for the beginning and end of the WQE
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * chain we are trying to sync. Either this is the simple case, where
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the end of the chain is below the beginning of the chain, or it is
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the "wrap-around" case, where the end of the chain has wrapped over
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the end of the queue. In the former case, we simply need to
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * calculate the span from beginning to end and sync it. In the latter
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * case, however, we need to calculate the span from the top of the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * work queue to the end of the chain and sync that, and then we need
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * to find the other portion (from beginning of chain to end of queue)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * and sync that as well. Note: if the "top to end" span is actually
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * zero length, then we don't do a DMA sync because a zero length DMA
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * sync unnecessarily syncs the entire work queue.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* "From Beginning to End" */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor offset = (off_t)((uintptr_t)wqe_from - (uintptr_t)wqe_base);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor length = (size_t)((uintptr_t)wqe_to - (uintptr_t)wqe_from);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor status = ddi_dma_sync(dmahdl, offset, length, flag);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TNF_PROBE_0(tavor_wqe_sync_fail, TAVOR_TNF_ERROR, "");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* "From Top to End" */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor length = (size_t)((uintptr_t)wqe_to - (uintptr_t)wqe_base);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor status = ddi_dma_sync(dmahdl, offset, length, flag);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* "From Beginning to Bottom" */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor offset = (off_t)((uintptr_t)wqe_from - (uintptr_t)wqe_base);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor length = (size_t)((uintptr_t)wqe_top - (uintptr_t)wqe_from);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor status = ddi_dma_sync(dmahdl, offset, length, flag);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TNF_PROBE_0(tavor_wqe_sync_fail, TAVOR_TNF_ERROR, "");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * tavor_wr_bind_check()
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Context: Can be called from interrupt or base context.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylortavor_wr_bind_check(tavor_state_t *state, ibt_send_wr_t *wr)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Check for a valid Memory Window handle in the WR */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor mw = (tavor_mwhdl_t)wr->wr.rc.rcwr.bind->bind_ibt_mw_hdl;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Check for a valid Memory Region handle in the WR */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor mr = (tavor_mrhdl_t)wr->wr.rc.rcwr.bind->bind_ibt_mr_hdl;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Check here to see if the memory region has already been partially
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * deregistered as a result of a tavor_umap_umemlock_cb() callback.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * If so, this is an error, return failure.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor if ((mr->mr_is_umem) && (mr->mr_umemcookie == NULL)) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Check for a valid Memory Window RKey (i.e. a matching RKey) */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor if (mw->mr_rkey != wr->wr.rc.rcwr.bind->bind_rkey) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Check for a valid Memory Region LKey (i.e. a matching LKey) */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor if (mr->mr_lkey != wr->wr.rc.rcwr.bind->bind_lkey) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Now check for valid "vaddr" and "len". Note: We don't check the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * "vaddr" range when "len == 0" (i.e. on unbind operations)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor if ((vaddr < reg_start_addr) || (vaddr > reg_end_addr)) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Validate the bind access flags. Remote Write and Atomic access for
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the Memory Window require that Local Write access be set in the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * corresponding Memory Region.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Calculate the new RKey for the Memory Window */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor tavor_mr_keycalc(state, mpt->tr_indx, &new_rkey);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * tavor_wrid_from_reset_handling()
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Context: Can be called from interrupt or base context.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylortavor_wrid_from_reset_handling(tavor_state_t *state, tavor_qphdl_t qp)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TAVOR_TNF_ENTER(tavor_wrid_from_reset_handling);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * For each of this QP's Work Queues, make sure we have a (properly
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * initialized) Work Request ID list attached to the relevant
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * completion queue. Grab the CQ lock(s) before manipulating the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor swq = tavor_wrid_wqhdr_find(qp->qp_sq_cqhdl, qp->qp_qpnum,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Couldn't find matching work queue header, create it */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor swq = tavor_wrid_wqhdr_create(state, qp->qp_sq_cqhdl,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * If we couldn't find/allocate space for the workq
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * header, then drop the lock(s) and return failure.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TNF_PROBE_0(tavor_wrid_from_reset_handling_wqhdr_fail,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Allocate space for the tavor_wrid_entry_t container
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * If we couldn't allocate space for tracking the WRID
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * entries, then cleanup the workq header from above (if
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * necessary, i.e. if we created the workq header). Then
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * drop the lock(s) and return failure.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TNF_PROBE_0(tavor_wrid_from_reset_handling_wridlist_fail,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*s_wridlist))
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Chain the new WRID list container to the workq hdr list */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Now we repeat all the above operations for the receive work queue,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * or shared receive work queue.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Note: We still use the 'qp_rq_cqhdl' even in the SRQ case.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor rwq = tavor_wrid_wqhdr_find(qp->qp_rq_cqhdl, qp->qp_qpnum,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * If this QP is associated with an SRQ, and this isn't the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * first QP on the SRQ, then the 'srq_wrid_wql' will already be
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * created. Since the WQL is created at 'wqhdr_create' time we
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * pass in the flag 'create_wql' here to be 0 if we have
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * already created it. And later on below we then next setup
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the WQL and rwq information based off the existing SRQ info.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor rwq = tavor_wrid_wqhdr_create(state, qp->qp_rq_cqhdl,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * If we couldn't find/allocate space for the workq
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * header, then free all the send queue resources we
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * just allocated and setup (above), drop the lock(s)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * and return failure.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TNF_PROBE_0(tavor_wrid_from_reset_handling_wqhdr_fail,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Setup receive workq hdr
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * If the QP is on an SRQ, we setup the SRQ specific fields, setting
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * keeping a copy of the rwq pointer, setting the rwq bufsize
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * appropriately, and initializing our part of the WQLock.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * In the normal QP case, the QP recv queue bufsize is used.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor tavor_wql_refcnt_inc(qp->qp_srqhdl->srq_wrid_wql);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Allocate space for the tavor_wrid_entry_t container.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * If QP is on an SRQ, and the wrq_wridlist is NULL then we must
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * allocate the wridlist normally. However, if the srq_wridlist is !=
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * NULL, then we know this SRQ has already been initialized, thus the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * wridlist has already been initialized. So we re-use the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * srq_wridlist as the r_wridlist for this QP in this case.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Use existing srq_wridlist pointer */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Allocate memory for the r_wridlist */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * If the memory allocation failed for r_wridlist (or the SRQ pointer
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * is mistakenly NULL), we cleanup our previous swq allocation from
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * If we couldn't allocate space for tracking the WRID
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * entries, then cleanup all the stuff from above. Then
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * drop the lock(s) and return failure.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TNF_PROBE_0(tavor_wrid_from_reset_handling_wridlist_fail,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*r_wridlist))
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Initialize the wridlist
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * In the normal QP case, there is no special initialization needed.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * We simply setup the wridlist backpointer to be the receive wqhdr
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * But in the SRQ case, there is no backpointer to the wqhdr possible.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Instead we set 'wl_srq_en', specifying this wridlist is on an SRQ
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * and thus potentially shared across multiple QPs with the SRQ. We
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * also setup the srq_wridlist pointer to be the r_wridlist, and
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * intialize the freelist to an invalid index. This srq_wridlist
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * pointer is used above on future moves from_reset to let us know that
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the srq_wridlist has been initialized already.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * And finally, if we are in a non-UMAP case, we setup the srq wrid
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * free list.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Initialize srq wrid free list */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor tavor_wrid_list_srq_init(r_wridlist, qp->qp_srqhdl, 0);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Chain the WRID list "container" to the workq hdr list */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*r_wridlist))
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*s_wridlist))
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * tavor_wrid_to_reset_handling()
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Context: Can be called from interrupt or base context.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylortavor_wrid_to_reset_handling(tavor_state_t *state, tavor_qphdl_t qp)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * For each of this QP's Work Queues, move the WRID "container" to
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the "reapable" list. Although there may still be unpolled
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * entries in these containers, it is not a big deal. We will not
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * reap the list until either the Poll CQ command detects an empty
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * condition or the CQ itself is freed. Grab the CQ lock(s) before
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * manipulating the lists.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor tavor_wrid_reaplist_add(qp->qp_sq_cqhdl, qp->qp_sq_wqhdr);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Add the receive work queue header on to the reaplist. But if we are
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * on SRQ, then don't add anything to the reaplist. Instead we flush
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the SRQ entries on the CQ, remove wridlist from WQHDR, and free the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * WQHDR (if needed). We must hold the WQL for these operations, yet
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the call to tavor_cq_wqhdr_remove grabs the WQL internally. So we
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * drop WQL before that call. Then release the CQ WQHDR locks and the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * CQ lock and return.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Pull off all (if any) entries for this QP from CQ. This
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * only includes entries that have not yet been polled
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor mutex_enter(&qp->qp_rq_wqhdr->wq_wrid_wql->wql_lock);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Remove wridlist from WQHDR */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* If wridlist chain is now empty, remove the wqhdr as well */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor mutex_exit(&qp->qp_rq_wqhdr->wq_wrid_wql->wql_lock);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Free the WQHDR */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor tavor_cq_wqhdr_remove(qp->qp_rq_cqhdl, qp->qp_rq_wqhdr);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor tavor_wrid_reaplist_add(qp->qp_rq_cqhdl, qp->qp_rq_wqhdr);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * tavor_wrid_add_entry()
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Context: Can be called from interrupt or base context.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylortavor_wrid_add_entry(tavor_workq_hdr_t *wq, uint64_t wrid, uint32_t wqeaddrsz,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Find the entry in the container pointed to by the "tail" index.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Add all of the relevant information to that entry, including WRID,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * "wqeaddrsz" parameter, and whether it was signaled/unsignaled
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * and/or doorbelled.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Update the "wrid_old_tail" pointer to point to the entry we just
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * inserted into the queue. By tracking this pointer (the pointer to
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the most recently inserted entry) it will possible later in the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * PostSend() and PostRecv() code paths to find the entry that needs
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * its "doorbelled" flag set (see comment in tavor_post_recv() and/or
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * tavor_post_send()).
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Update the tail index */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * If the "tail" index has just wrapped over into the "head" index,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * then we have filled the container. We use the "full" flag to
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * indicate this condition and to distinguish it from the "empty"
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * condition (where head and tail are also equal).
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * tavor_wrid_add_entry_srq()
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Context: Can be called from interrupt or base context
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylortavor_wrid_add_entry_srq(tavor_srqhdl_t srq, uint64_t wrid, uint_t signaled_dbd)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Find the next available WQE from the SRQ free_list. Then update the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * free_list to point to the next entry
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor wl_wqe = TAVOR_SRQ_WQE_ADDR(srq, srq->srq_wridlist->wl_free_list_indx);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor wqe_index = srq->srq_wridlist->wl_free_list_indx;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* ASSERT on impossible wqe_index values */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Setup the WRE.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Given the 'wqe_index' value, we store the WRID at this WRE offset.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * And we set the WRE to be signaled_dbd so that on poll CQ we can find
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * this information and associate the WRID to the WQE found on the CQE.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Update the free list index */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor srq->srq_wridlist->wl_free_list_indx = ddi_get32(
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor srq->srq_wridlist->wl_acchdl, (uint32_t *)wl_wqe);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * tavor_wrid_get_entry()
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Context: Can be called from interrupt or base context.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylortavor_wrid_get_entry(tavor_cqhdl_t cq, tavor_hw_cqe_t *cqe,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Lock the list of work queues associated with this CQ */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Determine whether this CQE is a send or receive completion (and
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * whether it was a "successful" completion or not)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor send_or_recv = (opcode == TAVOR_CQE_SEND_ERR_OPCODE) ?
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Find the work queue for this QP number (send or receive side) */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor wq = tavor_wrid_wqhdr_find(cq, qpnum, send_or_recv);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Regardless of whether the completion is the result of a "success"
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * or a "failure", we lock the list of "containers" and attempt to
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * search for the the first matching completion (i.e. the first WR
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * with a matching WQE addr and size). Once we find it, we pull out
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the "wrid" field and return it (see below). Note: One possible
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * future enhancement would be to enable this routine to skip over
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * any "unsignaled" completions to go directly to the next "signaled"
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * entry on success. XXX
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * If this is a "successful" completion, then we assert that this
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * completion must be a "signaled" completion.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ASSERT(error || (wre_tmp->wr_signaled_dbd & TAVOR_WRID_ENTRY_SIGNALED));
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * If the completion is a "failed" completion, then we save away the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * contents of the entry (into the "wre" field passed in) for use
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * in later CQE processing. Note: We use the tavor_wrid_get_wqeaddrsz()
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * function to grab "wqeaddrsz" from the next entry in the container.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * This is required for error processing (where updating these fields
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * properly is necessary to correct handling of the "error" CQE)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor wre->wr_wqeaddrsz = tavor_wrid_get_wqeaddrsz(wq);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Pull out the WRID and return it */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * tavor_wrid_find_match()
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Context: Can be called from interrupt or base context.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylortavor_wrid_find_match(tavor_workq_hdr_t *wq, tavor_cqhdl_t cq,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Pull the "wqeaddrsz" information from the CQE */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor wqeaddr_size = TAVOR_CQE_WQEADDRSZ_GET(cq, cqe);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Walk the "containers" list(s), find first WR with a matching WQE
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * addr. If the current "container" is not the last one on the list,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * i.e. not the current one to which we are posting new WRID entries,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * then we do not attempt to update the "q_head", "q_tail", and
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * "q_full" indicators on the main work queue header. We do, however,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * update the "head" and "full" indicators on the individual containers
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * as we go. This is imperative because we need to be able to
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * determine when the current container has been emptied (so that we
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * can move on to the next container).
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Is this the last/only "container" on the list */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor last_container = (container != wq->wq_wrid_post) ? 0 : 1;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * First check if we are on an SRQ. If so, we grab the entry
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * and break out. Since SRQ wridlist's are never added to
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * reaplist, they can only be the last container.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor curr = tavor_wrid_find_match_srq(container, cq, cqe);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Grab the current "head", "tail" and "size" fields before
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * walking the list in the current container. Note: the "size"
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * field here must always be a power-of-2. The "full"
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * parameter is checked (and updated) here to distinguish the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * "queue full" condition from "queue empty".
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor while ((head != tail) || (container->wl_full)) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * If the current entry's "wqeaddrsz" matches the one
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * we're searching for, then this must correspond to
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the work request that caused the completion. Set
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the "found" flag and bail out.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * If the current container is empty (having reached here the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * "head == tail" condition can only mean that the container
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * is empty), then NULL out the "wrid_old_tail" field (see
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * tavor_post_send() and tavor_post_recv() for more details)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * and (potentially) remove the current container from future
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * If this wasn't the last "container" on the chain,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * i.e. the one to which new WRID entries will be
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * added, then remove it from the list.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Note: we don't "lose" the memory pointed to by this
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * because we should have already put this container
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * on the "reapable" list (from where it will later be
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Update the head index for the container */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * If the entry was found in this container, then continue to
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * bail out. Else reset the "curr" pointer and move on to the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * next container (if there is one). Note: the only real
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * reason for setting "curr = NULL" here is so that the ASSERT
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * below can catch the case where no matching entry was found
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * on any of the lists.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Update work queue header's "head" and "full" conditions to match
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the last entry on the container list. (Note: Only if we're pulling
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * entries from the last work queue portion of the list, i.e. not from
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the previous portions that may be the "reapable" list.)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Ensure that we've actually found what we were searching for */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * tavor_wrid_find_match_srq()
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Context: Can be called from interrupt or base context.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylortavor_wrid_find_match_srq(tavor_wrid_list_hdr_t *wl, tavor_cqhdl_t cq,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Grab the WQE addr out of the CQE */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor cqe_wqe_addr = TAVOR_CQE_WQEADDRSZ_GET(cq, cqe) & 0xFFFFFFC0;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Use the WQE addr as the lower 32-bit, we add back on the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * 'wl_srq_desc_off' because we have a zero-based queue. Then the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * upper 32-bit of the 'wl_srq_wq_buf' OR'd on gives us the WQE addr in
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the SRQ Work Queue itself. We use this address as the index to find
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * out which Work Queue Entry this CQE corresponds with.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * We also use this address below to add the WQE back on to the free
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor wqe_addr = ((uintptr_t)wl->wl_srq_wq_buf & 0xFFFFFFFF00000000ull) |
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Given the 'wqe_addr' just calculated and the srq buf address, we
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * find the 'wqe_index'. The 'wre' returned below contains the WRID
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * that we are looking for. This indexes into the wre_list for this
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * specific WQE.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor wqe_index = TAVOR_SRQ_WQE_INDEX(wl->wl_srq_wq_buf, wqe_addr,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* ASSERT on impossible wqe_index values */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Get the pointer to this WQE */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Put this WQE index back on the free list */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ddi_put32(wl->wl_acchdl, (uint32_t *)wl_wqe, wl->wl_free_list_indx);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Using the index, return the Work Request ID Entry (wre) */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * tavor_wrid_cq_reap()
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Context: Can be called from interrupt or base context.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Lock the list of work queues associated with this CQ */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Walk the "reapable" list and free up containers */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * If reaping the WRID list containers pulls the last
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * container from the given work queue header, then we free
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the work queue header as well.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Once finished reaping, we reset the CQ's reap list */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor cq->cq_wrid_reap_head = cq->cq_wrid_reap_tail = NULL;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * tavor_wrid_cq_force_reap()
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Context: Can be called from interrupt or base context.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * The first step is to walk the "reapable" list and free up those
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * containers. This is necessary because the containers on the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * reapable list are not otherwise connected to the work queue headers
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Now lock the list of work queues associated with this CQ */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Walk the list of work queue headers and free up all the WRID list
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * containers chained to it. Note: We don't need to grab the locks
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * for each of the individual WRID lists here because the only way
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * things can be added or removed from the list at this point would be
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * through post a work request to a QP. But if we've come this far,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * then we can be assured that there are no longer any QP associated
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * with the CQ that we are trying to free.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor while ((curr = avl_destroy_nodes(treep, &cookie)) != NULL) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * If reaping the WRID list containers pulls the last
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * container from the given work queue header, then
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * we free the work queue header as well. Note: we
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * ignore the return value because we know that the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * work queue header should always be freed once the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * list of containers has come to an end.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * tavor_wrid_get_list()
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Context: Can be called from interrupt or base context.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * The WRID list "container" consists of the tavor_wrid_list_hdr_t,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * which holds the pointers necessary for maintaining the "reapable"
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * list, chaining together multiple "containers" old and new, and
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * tracking the head, tail, size, etc. for each container.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * The "container" also holds all the tavor_wrid_entry_t's, which is
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * allocated separately, one for each entry on the corresponding work
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Note that this allocation has to be a NOSLEEP operation here
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * because we are holding the "wqhdr_list_lock" and, therefore,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * could get raised to the interrupt level.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor wridlist = (tavor_wrid_list_hdr_t *)kmem_zalloc(size, KM_NOSLEEP);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*wridlist))
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Complete the "container" initialization */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor wridlist->wl_wre = (tavor_wrid_entry_t *)kmem_zalloc(qsize *
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * tavor_wrid_list_srq_init()
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Context: Can be called from interrupt or base context
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylortavor_wrid_list_srq_init(tavor_wrid_list_hdr_t *wridlist, tavor_srqhdl_t srq,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ASSERT(MUTEX_HELD(&srq->srq_wrid_wql->wql_lock));
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Setup pointers for use later when we are polling the CQ */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor wridlist->wl_srq_log_wqesz = srq->srq_wq_log_wqesz;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor wridlist->wl_acchdl = srq->srq_wqinfo.qa_acchdl;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Given wq_start to start initializing buf at, verify sanity */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ASSERT(wq_start >= 0 && wq_start < srq->srq_wq_bufsz);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Initialize wridlist free list
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * For each WQ up to the size of our queue, we store an index in the WQ
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * memory itself, representing the next available free entry. The
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * 'wl_free_list_indx' always holds the index of the next available
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * free entry in the WQ. If 'wl_free_list_indx' is -1, then we are
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * completely full. This gives us the advantage of being able to have
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * entries complete or be polled off the WQ out-of-order.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * For now, we write the free_list entries inside the WQ itself. It
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * may be useful in the future to store this information in a separate
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * structure for debugging purposes.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor for (wqe_index = wq_start; wqe_index < srq->srq_wq_bufsz; wqe_index++) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ddi_put32(wridlist->wl_acchdl, (uint32_t *)wl_wqe,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * tavor_wrid_reaplist_add()
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Context: Can be called from interrupt or base context.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylortavor_wrid_reaplist_add(tavor_cqhdl_t cq, tavor_workq_hdr_t *wq)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Add the "post" container (the last one on the current chain) to
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the CQ's "reapable" list
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor cq->cq_wrid_reap_tail->wl_reap_next = wq->wq_wrid_post;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylortavor_wrid_wqhdr_compare(const void *p1, const void *p2)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor return (-1);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor return (+1);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor return (-1);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor return (+1);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * tavor_wrid_wqhdr_find()
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Context: Can be called from interrupt or base context.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylortavor_wrid_wqhdr_find(tavor_cqhdl_t cq, uint_t qpn, uint_t wq_type)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Walk the CQ's work queue list, trying to find a send or recv queue
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * with the same QP number. We do this even if we are going to later
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * create a new entry because it helps us easily find the end of the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor curr = avl_find(&cq->cq_wrid_wqhdr_avl_tree, &cmp, NULL);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * tavor_wrid_wqhdr_create()
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Context: Can be called from interrupt or base context.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylortavor_wrid_wqhdr_create(tavor_state_t *state, tavor_cqhdl_t cq, uint_t qpn,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Allocate space a work queue header structure and initialize it.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Each work queue header structure includes a "wq_wrid_wql"
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * which needs to be initialized. Note that this allocation has to be
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * a NOSLEEP operation because we are holding the "cq_wrid_wqhdr_lock"
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * and, therefore, could get raised to the interrupt level.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*wqhdr_tmp))
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor wqhdr_tmp->wq_wrid_wql = tavor_wrid_wql_create(state);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor kmem_free(wqhdr_tmp, sizeof (tavor_workq_hdr_t));
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Chain the newly allocated work queue header to the CQ's list */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * tavor_wrid_wql_create()
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Context: Can be called from interrupt or base context.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Allocate the WQL and initialize it.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor wql = kmem_zalloc(sizeof (tavor_wq_lock_t), KM_NOSLEEP);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Add refcount to WQL */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * tavor_wrid_get_wqeaddrsz()
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Context: Can be called from interrupt or base context.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * If the container is empty, then there is no next entry. So just
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * return zero. Note: the "head == tail" condition here can only
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * mean that the container is empty because we have previously pulled
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * something from the container.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * If the container is not empty, then find the next entry and return
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the contents of its "wqeaddrsz" field.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor if (wq->wq_wrid_poll->wl_head == wq->wq_wrid_poll->wl_tail) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * We don't need to calculate the "next" head pointer here
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * because "head" should already point to the next entry on
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the list (since we just pulled something off - in
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * tavor_wrid_find_match() - and moved the head index forward.)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * tavor_wrid_wqhdr_add()
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Context: Can be called from interrupt or base context.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ASSERT(MUTEX_HELD(&wqhdr->wq_wrid_wql->wql_lock));
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Chain the new WRID list "container" to the work queue list */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * tavor_wrid_wqhdr_remove()
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Context: Can be called from interrupt or base context.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Note: this is only called to remove the most recently added WRID list
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * container (i.e. in tavor_from_reset() above)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylortavor_wrid_wqhdr_remove(tavor_workq_hdr_t *wqhdr,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ASSERT(MUTEX_HELD(&wqhdr->wq_wrid_wql->wql_lock));
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Unlink the WRID list "container" from the work queue list */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Update any pointers in the work queue hdr that may point to this
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * WRID list container
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * tavor_wrid_list_reap()
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Context: Can be called from interrupt or base context.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Note: The "wqhdr_list_lock" must be held.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylortavor_wrid_list_reap(tavor_wrid_list_hdr_t *wridlist)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor tavor_workq_hdr_t *wqhdr, *consume_wqhdr = NULL;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Get the back pointer to the work queue header (see below) */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Unlink the WRID list "container" from the work queue list */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * If the back pointer to the work queue header shows that it
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * was pointing to the entry we are about to remove, then the work
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * queue header is reapable as well.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Be sure to update the "poll" and "post" container pointers */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Calculate the size and free the container */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor size = (wridlist->wl_size * sizeof (tavor_wrid_entry_t));
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor kmem_free(wridlist, sizeof (tavor_wrid_list_hdr_t));
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * tavor_wrid_wqhdr_lock_both()
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Context: Can be called from interrupt or base context.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor_NOTE(MUTEX_ACQUIRED_AS_SIDE_EFFECT(&sq_cq->cq_wrid_wqhdr_lock))
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor_NOTE(MUTEX_ACQUIRED_AS_SIDE_EFFECT(&rq_cq->cq_wrid_wqhdr_lock))
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * If both work queues (send and recv) share a completion queue, then
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * grab the common lock. If they use different CQs (hence different
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * "cq_wrid_wqhdr_list" locks), then grab the send one first, then the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * receive. We do this consistently and correctly in
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * tavor_wrid_wqhdr_unlock_both() below to avoid introducing any kind
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * of dead lock condition. Note: We add the "__lock_lint" code here
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * to fake out warlock into thinking we've grabbed both locks (when,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * in fact, we only needed the one).
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * tavor_wrid_wqhdr_unlock_both()
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Context: Can be called from interrupt or base context.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor_NOTE(LOCK_RELEASED_AS_SIDE_EFFECT(&rq_cq->cq_wrid_wqhdr_lock))
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor_NOTE(LOCK_RELEASED_AS_SIDE_EFFECT(&sq_cq->cq_wrid_wqhdr_lock))
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * See tavor_wrid_wqhdr_lock_both() above for more detail
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * tavor_cq_wqhdr_add()
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Context: Can be called from interrupt or base context.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylortavor_cq_wqhdr_add(tavor_cqhdl_t cq, tavor_workq_hdr_t *wqhdr)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor (void) avl_find(&cq->cq_wrid_wqhdr_avl_tree, &cmp, &where);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * If the CQ's work queue list is empty, then just add it.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Otherwise, chain it to the beginning of the list.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor avl_insert(&cq->cq_wrid_wqhdr_avl_tree, wqhdr, where);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * tavor_cq_wqhdr_remove()
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Context: Can be called from interrupt or base context.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylortavor_cq_wqhdr_remove(tavor_cqhdl_t cq, tavor_workq_hdr_t *wqhdr)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Remove "wqhdr" from the work queue header list on "cq" */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Release reference to WQL; If this is the last reference, this call
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * also has the side effect of freeing up the 'wq_wrid_wql' memory.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Free the memory associated with "wqhdr" */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * tavor_wql_refcnt_inc()
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Context: Can be called from interrupt or base context
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * tavor_wql_refcnt_dec()
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Context: Can be called from interrupt or base context
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Free up WQL memory if we're the last one associated with this
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * structure.