9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * CDDL HEADER START
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * The contents of this file are subject to the terms of the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Common Development and Distribution License (the "License").
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * You may not use this file except in compliance with the License.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * See the License for the specific language governing permissions
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * and limitations under the License.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * When distributing Covered Code, include this CDDL HEADER in each
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * If applicable, add the following below this CDDL HEADER, with the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * fields enclosed by brackets "[]" replaced with your own identifying
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * information: Portions Copyright [yyyy] [name of copyright owner]
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * CDDL HEADER END
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Hermon Work Request Processing Routines
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Implements all the routines necessary to provide the PostSend(),
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * PostRecv() and PostSRQ() verbs. Also contains all the code
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * necessary to implement the Hermon WRID tracking mechanism.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic uint32_t hermon_wr_get_immediate(ibt_send_wr_t *wr);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic int hermon_wr_bind_check(hermon_state_t *state, ibt_send_wr_t *wr);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic int hermon_wqe_send_build(hermon_state_t *state, hermon_qphdl_t qp,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ibt_send_wr_t *wr, uint64_t *desc, uint_t *size);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic int hermon_wqe_mlx_build(hermon_state_t *state, hermon_qphdl_t qp,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ibt_send_wr_t *wr, uint64_t *desc, uint_t *size);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic void hermon_wqe_headroom(uint_t from, hermon_qphdl_t qp);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic int hermon_wqe_recv_build(hermon_state_t *state, hermon_qphdl_t qp,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic int hermon_wqe_srq_build(hermon_state_t *state, hermon_srqhdl_t srq,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic hermon_workq_avl_t *hermon_wrid_wqavl_find(hermon_cqhdl_t cq, uint_t qpn,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic void hermon_cq_workq_add(hermon_cqhdl_t cq, hermon_workq_avl_t *wqavl);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic void hermon_cq_workq_remove(hermon_cqhdl_t cq,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic ibt_wr_ds_t null_sgl = { 0, 0x00000100, 0 };
c7facc54c4abed9e554ff80225311e6b7048d3c9Bill Taylor * Add ability to try to debug RDMA_READ/RDMA_WRITE failures.
c7facc54c4abed9e554ff80225311e6b7048d3c9Bill Taylor * 0x1 - print rkey used during post_send
c7facc54c4abed9e554ff80225311e6b7048d3c9Bill Taylor * 0x2 - print sgls used during post_send
c7facc54c4abed9e554ff80225311e6b7048d3c9Bill Taylor * 0x4 - print FMR comings and goings
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorhermon_post_send_ud(hermon_state_t *state, hermon_qphdl_t qp,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ibt_send_wr_t *wr, uint_t num_wr, uint_t *num_posted)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* initialize the FMA retry loop */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor hermon_pio_init(fm_loop_cnt, fm_status, fm_test_num);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor _NOTE(LOCK_RELEASED_AS_SIDE_EFFECT(&qp->qp_sq_lock))
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Grab the lock for the WRID list */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Save away some initial QP state */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Check for "queue full" condition. If the queue
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * is already full, then no more WQEs can be posted.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * So break out, ring a doorbell (if necessary) and
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * return an error
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Build a Send or Send_LSO WQE
9c865d645a5c60028aed172eea31ca36d81a2ff1Rajkumar Sivaprakasam nopcode |= (1 << 6); /* ReRead bit must be set */
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor ud = (hermon_hw_snd_wqe_ud_t *)((uintptr_t)desc +
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor total_len = (4 + 0xf + wr->wr.ud_lso.lso_hdr_sz) & ~0xf;
9c865d645a5c60028aed172eea31ca36d81a2ff1Rajkumar Sivaprakasam bcopy(wr->wr.ud_lso.lso_hdr, (uint32_t *)old_ds + 1,
9c865d645a5c60028aed172eea31ca36d81a2ff1Rajkumar Sivaprakasam ds = (hermon_hw_wqe_sgl_t *)((uintptr_t)ds + total_len);
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor } else if (qp->qp_serv_type == HERMON_QP_RFCI) {
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor /* "|=" changes 0xa to 0xb without touching FCEOF */
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor ud = (hermon_hw_snd_wqe_ud_t *)((uintptr_t)desc +
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor ud = (hermon_hw_snd_wqe_ud_t *)((uintptr_t)desc +
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor /* move ds beyond the FCP-3 Init Segment */
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor ds = (hermon_hw_wqe_sgl_t *)((uintptr_t)ds + 0x10);
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor frwr = (hermon_hw_snd_wqe_frwr_t *)((uintptr_t)desc +
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor HERMON_WQE_BUILD_FRWR(qp, frwr, wr->wr.fc.reg_pmr);
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor /* firmware does not support this */
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor li = (hermon_hw_snd_wqe_local_inv_t *)((uintptr_t)desc +
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor desc_sz = ((uintptr_t)&ds[last_ds] - (uintptr_t)desc) >> 0x4;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor for (j = nds; --j >= i; ) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Fill in the Data Segment(s) for the current WQE, using the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * information contained in the scatter-gather list of the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * work request.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor HERMON_WQE_BUILD_DATA_SEG_SEND(&ds[last_ds], &sgl[j]);
9c865d645a5c60028aed172eea31ca36d81a2ff1Rajkumar Sivaprakasam if (wr->wr_opcode == IBT_WRC_SEND_LSO) {
9c865d645a5c60028aed172eea31ca36d81a2ff1Rajkumar Sivaprakasam HERMON_WQE_BUILD_LSO(qp, old_ds, wr->wr.ud_lso.lso_mss,
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor } else if (wr->wr_opcode == IBT_WRC_INIT_SEND_FCMD) {
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor /* This sits in the STAMP, so must be set after setting SGL */
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor HERMON_WQE_BUILD_FCP3_INIT(old_ds, is->is_ctl.fc_frame_ctrl,
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor is->is_cs_priority, is->is_tx_seq_id, is->is_fc_mtu,
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor /* The following will be used in HERMON_WQE_SET_CTRL_SEGMENT */
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor /* SIT bit in FCP-3 ctrl segment */
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor desc_sz |= (is->is_ctl.fc_frame_ctrl & IBT_FCTL_SIT) ? 0x80 : 0;
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor /* LS bit in FCP-3 ctrl segment */
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor fc_bits |= (is->is_ctl.fc_frame_ctrl & IBT_FCTL_LAST_SEQ) ?
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor 0x10000 : 0;
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor fc_bits |= ((is->is_ctl.fc_routing_ctrl & 0xF) << 20) |
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor fence = (wr->wr_flags & IBT_WR_SEND_FENCE) ? 1 : 0;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor signaled_dbd = ((qp->qp_sq_sigtype == HERMON_QP_SQ_ALL_SIGNALED) ||
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor solicited = (wr->wr_flags & IBT_WR_SEND_SOLICIT) ? 0x2 : 0;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor HERMON_WQE_SET_CTRL_SEGMENT(desc, desc_sz, fence, immed_data,
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor solicited, signaled_dbd, cksum, qp, strong_order, fc_bits);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Update some of the state in the QP */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Now set the ownership bit and opcode (first dword). */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor HERMON_SET_SEND_WQE_OWNER(qp, (uint32_t *)desc, nopcode);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* do the invalidate of the headroom */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ddi_acc_handle_t uarhdl = hermon_get_uarhdl(state);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* the FMA retry loop starts for Hermon doorbell register. */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor hermon_pio_start(state, uarhdl, pio_error, fm_loop_cnt,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* the FMA retry loop ends. */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor hermon_pio_end(state, uarhdl, pio_error, fm_loop_cnt,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* do the invalidate of the headroom */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor hermon_fm_ereport(state, HCA_SYS_ERR, HCA_ERR_SRV_LOST);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorhermon_post_send_rc(hermon_state_t *state, hermon_qphdl_t qp,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ibt_send_wr_t *wr, uint_t num_wr, uint_t *num_posted)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* initialize the FMA retry loop */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor hermon_pio_init(fm_loop_cnt, fm_status, fm_test_num);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor _NOTE(LOCK_RELEASED_AS_SIDE_EFFECT(&qp->qp_sq_lock))
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Save away some initial QP state */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Check for "queue full" condition. If the queue
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * is already full, then no more WQEs can be posted.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * So break out, ring a doorbell (if necessary) and
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * return an error
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Validate the operation type. For RC requests, we allow
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * "Send", "RDMA Read", "RDMA Write", various "Atomic"
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * operations, and memory window "Bind"
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * If this is an RDMA Read or RDMA Write request, then fill
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * in the "Remote Address" header fields.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* FALLTHROUGH */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor rc = (hermon_hw_snd_wqe_remaddr_t *)((uintptr_t)desc +
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Build the Remote Address Segment for the WQE, using
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the information from the RC work request.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor HERMON_WQE_BUILD_REMADDR(qp, rc, &wr->wr.rc.rcwr.rdma);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Update "ds" for filling in Data Segments (below) */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * If this is one of the Atomic type operations (i.e
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Compare-Swap or Fetch-Add), then fill in both the "Remote
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Address" header fields and the "Atomic" header fields.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* FALLTHROUGH */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor rc = (hermon_hw_snd_wqe_remaddr_t *)((uintptr_t)desc +
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor at = (hermon_hw_snd_wqe_atomic_t *)((uintptr_t)rc +
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Build the Remote Address and Atomic Segments for
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the WQE, using the information from the RC Atomic
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * work request.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor HERMON_WQE_BUILD_ATOMIC(qp, at, wr->wr.rc.rcwr.atomic);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Update "ds" for filling in Data Segments (below) */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Update "nds" and "sgl" because Atomic requests have
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * only a single Data Segment.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * If this is memory window Bind operation, then we call the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * hermon_wr_bind_check() routine to validate the request and
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * to generate the updated RKey. If this is successful, then
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * we fill in the WQE's "Bind" header fields.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor bn = (hermon_hw_snd_wqe_bind_t *)((uintptr_t)desc +
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Build the Bind Memory Window Segments for the WQE,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * using the information from the RC Bind memory
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * window work request.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor HERMON_WQE_BUILD_BIND(qp, bn, wr->wr.rc.rcwr.bind);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Update the "ds" pointer. Even though the "bind"
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * operation requires no SGLs, this is necessary to
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * facilitate the correct descriptor size calculations
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor frwr = (hermon_hw_snd_wqe_frwr_t *)((uintptr_t)desc +
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor HERMON_WQE_BUILD_FRWR(qp, frwr, wr->wr.rc.rcwr.reg_pmr);
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor li = (hermon_hw_snd_wqe_local_inv_t *)((uintptr_t)desc +
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Now fill in the Data Segments (SGL) for the Send WQE based
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * on the values setup above (i.e. "sgl", "nds", and the "ds"
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * pointer. Start by checking for a valid number of SGL entries
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor desc_sz = ((uintptr_t)&ds[last_ds] - (uintptr_t)desc) >> 0x4;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor for (i = nds; --i >= 0; ) {
c7facc54c4abed9e554ff80225311e6b7048d3c9Bill Taylor IBTF_DPRINTF_L2("rdma", "post: [%d]: laddr %llx "
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Fill in the Data Segment(s) for the current WQE, using the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * information contained in the scatter-gather list of the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * work request.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor HERMON_WQE_BUILD_DATA_SEG_SEND(&ds[last_ds], &sgl[i]);
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor /* ensure RDMA READ does not exceed HCA limit */
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor if ((wr->wr_opcode == IBT_WRC_RDMAR) && (desc_sz >
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor state->hs_ibtfinfo.hca_attr->hca_conn_rdma_read_sgl_sz + 2)) {
c7facc54c4abed9e554ff80225311e6b7048d3c9Bill Taylor IBTF_DPRINTF_L2("rdma", "post: indx %x rkey %x raddr %llx "
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor fence = (wr->wr_flags & IBT_WR_SEND_FENCE) ? 1 : 0;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor signaled_dbd = ((qp->qp_sq_sigtype == HERMON_QP_SQ_ALL_SIGNALED) ||
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor solicited = (wr->wr_flags & IBT_WR_SEND_SOLICIT) ? 0x2 : 0;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor HERMON_WQE_SET_CTRL_SEGMENT(desc, desc_sz, fence, immed_data, solicited,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Update some of the state in the QP */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Now set the ownership bit of the first one in the chain. */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor HERMON_SET_SEND_WQE_OWNER(qp, (uint32_t *)desc, nopcode);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* do the invalidate of the headroom */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ddi_acc_handle_t uarhdl = hermon_get_uarhdl(state);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* the FMA retry loop starts for Hermon doorbell register. */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor hermon_pio_start(state, uarhdl, pio_error, fm_loop_cnt,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Ring the doorbell */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* the FMA retry loop ends. */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor hermon_pio_end(state, uarhdl, pio_error, fm_loop_cnt,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* do the invalidate of the headroom */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Update the "num_posted" return value (if necessary).
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Then drop the locks and return success.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor hermon_fm_ereport(state, HCA_SYS_ERR, HCA_ERR_SRV_LOST);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * hermon_post_send()
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Context: Can be called from interrupt or base context.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorhermon_post_send(hermon_state_t *state, hermon_qphdl_t qp,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ibt_send_wr_t *wr, uint_t num_wr, uint_t *num_posted)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* initialize the FMA retry loop */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor hermon_pio_init(fm_loop_cnt, fm_status, fm_test);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Check for user-mappable QP memory. Note: We do not allow kernel
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * clients to post to QP memory that is accessible directly by the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * user. If the QP memory is user accessible, then return an error.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Check QP state. Can not post Send requests from the "Reset",
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * "Init", or "RTR" states
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Use these optimized functions most of the time */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor return (hermon_post_send_ud(state, qp, wr, num_wr, num_posted));
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor return (hermon_post_send_rc(state, qp, wr, num_wr, num_posted));
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* general loop for non-optimized posting */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Save away some initial QP state */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Initialize posted_cnt */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * For each ibt_send_wr_t in the wr[] list passed in, parse the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * request and build a Send WQE. NOTE: Because we are potentially
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * building a chain of WQEs to post, we want to build them all first,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * and set the valid (HW Ownership) bit on all but the first.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * However, we do not want to validate the first one until the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * entire chain of WQEs has been built. Then in the final
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * we set the valid bit in the first, flush if needed, and as a last
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * step ring the appropriate doorbell. NOTE: the doorbell ring may
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * NOT be needed if the HCA is already processing, but the doorbell
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * ring will be done regardless. NOTE ALSO: It is possible for
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * more Work Requests to be posted than the HW will support at one
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * shot. If this happens, we need to be able to post and ring
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * several chains here until the the entire request is complete.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * NOTE ALSO: the term "chain" is used to differentiate it from
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Work Request List passed in; and because that's the terminology
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * from the previous generations of HCA - but the WQEs are not, in fact
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * chained together for Hermon
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor while ((wrindx < num_wr) && (status == DDI_SUCCESS)) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * For the first WQE on a new chain we need "prev" to point
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * to the current descriptor.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Break the request up into lists that are less than or
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * equal to the maximum number of WQEs that can be posted
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * per doorbell ring - 256 currently
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor chainlen = (numremain > HERMON_QP_MAXDESC_PER_DB) ?
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor for (currindx = 0; currindx < chainlen; currindx++, wrindx++) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Check for "queue full" condition. If the queue
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * is already full, then no more WQEs can be posted.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * So break out, ring a doorbell (if necessary) and
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * return an error
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Increment the "tail index". Check for "queue
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * full" condition incl. headroom. If we detect that
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the current work request is going to fill the work
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * queue, then we mark this condition and continue.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Don't need >=, because going one-by-one we have to
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * hit it exactly sooner or later
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Get the address of the location where the next
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Send WQE should be built
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Call hermon_wqe_send_build() to build the WQE
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * at the given address. This routine uses the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * information in the ibt_send_wr_t list (wr[]) and
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * returns the size of the WQE when it returns.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Now, build the Ctrl Segment based on
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * what was just done
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor fence = (curr_wr->wr_flags & IBT_WR_SEND_FENCE) ? 1 : 0;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * now, build up the control segment, leaving the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * owner bit as it is
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor if ((qp->qp_sq_sigtype == HERMON_QP_SQ_ALL_SIGNALED) ||
949b58c70cf907006b9f724dfad665d44eca5881Bill Taylor /* Ensure correctness, set the ReRead bit */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * If this is not the first descriptor on the current
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * chain, then set the ownership bit.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Update the current "tail index" and increment
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * "posted_cnt"
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * If we reach here and there are one or more WQEs which have
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * been successfully built as a chain, we have to finish up
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * and prepare them for writing to the HW
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * The steps are:
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * 1. do the headroom fixup
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * 2. add in the size of the headroom for the sync
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * 3. write the owner bit for the first WQE
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * 4. sync them
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * 5. fix up the structures
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * 6. hit the doorbell in UAR
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ddi_acc_handle_t uarhdl = hermon_get_uarhdl(state);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* do the invalidate of the headroom */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Update some of the state in the QP */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Now set the ownership bit of the first
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * one in the chain
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* the FMA retry loop starts for Hermon doorbell. */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor hermon_pio_start(state, uarhdl, pio_error, fm_loop_cnt,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* the FMA retry loop ends. */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor hermon_pio_end(state, uarhdl, pio_error, fm_loop_cnt,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Update the "num_posted" return value (if necessary).
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Then drop the locks and return success.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor hermon_fm_ereport(state, HCA_SYS_ERR, HCA_ERR_SRV_LOST);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * hermon_post_recv()
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Context: Can be called from interrupt or base context.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorhermon_post_recv(hermon_state_t *state, hermon_qphdl_t qp,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ibt_recv_wr_t *wr, uint_t num_wr, uint_t *num_posted)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Check for user-mappable QP memory. Note: We do not allow kernel
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * clients to post to QP memory that is accessible directly by the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * user. If the QP memory is user accessible, then return an error.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Initialize posted_cnt */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Check if QP is associated with an SRQ
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Check QP state. Can not post Recv requests from the "Reset" state
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Check that work request transport type is valid */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Grab the lock for the WRID list, i.e., membar_consumer().
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * This is not needed because the mutex_enter() above has
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the same effect.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Save away some initial QP state */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor status = hermon_wqe_recv_build(state, qp, &wr[wrindx], desc);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor membar_producer(); /* ensure wrids are visible */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Update the doorbell record w/ wqecntr */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * hermon_post_srq()
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Context: Can be called from interrupt or base context.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorhermon_post_srq(hermon_state_t *state, hermon_srqhdl_t srq,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ibt_recv_wr_t *wr, uint_t num_wr, uint_t *num_posted)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Check for user-mappable QP memory. Note: We do not allow kernel
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * clients to post to QP memory that is accessible directly by the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * user. If the QP memory is user accessible, then return an error.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Check SRQ state. Can not post Recv requests when SRQ is in error
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor status = hermon_wqe_srq_build(state, srq, &wr[wrindx], desc);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor membar_producer(); /* ensure wrids are visible */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Ring the doorbell w/ wqecntr */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * hermon_wqe_send_build()
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Context: Can be called from interrupt or base context.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorhermon_wqe_send_build(hermon_state_t *state, hermon_qphdl_t qp,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ibt_send_wr_t *wr, uint64_t *desc, uint_t *size)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Initialize the information for the Data Segments */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Build a Send WQE depends first and foremost on the transport
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * type of Work Request (i.e. UD, RC, or UC)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Ensure that work request transport type matches QP type */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Validate the operation type. For UD requests, only the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * "Send" and "Send LSO" operations are valid.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * If this is a Special QP (QP0 or QP1), then we need to
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * build MLX WQEs instead. So jump to hermon_wqe_mlx_build()
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * and return whatever status it returns
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Otherwise, if this is a normal UD Send request, then fill
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * all the fields in the Hermon UD header for the WQE. Note:
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * to do this we'll need to extract some information from the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Address Handle passed with the work request.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ud = (hermon_hw_snd_wqe_ud_t *)((uintptr_t)desc +
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Build the Unreliable Datagram Segment for the WQE, using
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the information from the address handle and the work
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* mutex_enter(&ah->ah_lock); */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor HERMON_WQE_BUILD_UD(qp, ud, ah, wr->wr.ud.udwr_dest);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor } else { /* IBT_WRC_SEND_LSO */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* mutex_exit(&ah->ah_lock); */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Update "ds" for filling in Data Segments (below) */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor total_len = (4 + 0xf + wr->wr.ud_lso.lso_hdr_sz) & ~0xf;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor bcopy(wr->wr.ud_lso.lso_hdr, (uint32_t *)ds + 1,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ds = (hermon_hw_wqe_sgl_t *)((uintptr_t)ds + total_len);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor for (; i < nds; i++) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor HERMON_WQE_BUILD_LSO(qp, old_ds, wr->wr.ud_lso.lso_mss,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Ensure that work request transport type matches QP type */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Validate the operation type. For RC requests, we allow
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * "Send", "RDMA Read", "RDMA Write", various "Atomic"
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * operations, and memory window "Bind"
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * If this is a Send request, then all we need to do is break
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * out and here and begin the Data Segment processing below
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * If this is an RDMA Read or RDMA Write request, then fill
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * in the "Remote Address" header fields.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor rc = (hermon_hw_snd_wqe_remaddr_t *)((uintptr_t)desc +
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Build the Remote Address Segment for the WQE, using
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the information from the RC work request.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor HERMON_WQE_BUILD_REMADDR(qp, rc, &wr->wr.rc.rcwr.rdma);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Update "ds" for filling in Data Segments (below) */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * If this is one of the Atomic type operations (i.e
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Compare-Swap or Fetch-Add), then fill in both the "Remote
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Address" header fields and the "Atomic" header fields.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor rc = (hermon_hw_snd_wqe_remaddr_t *)((uintptr_t)desc +
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor at = (hermon_hw_snd_wqe_atomic_t *)((uintptr_t)rc +
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Build the Remote Address and Atomic Segments for
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the WQE, using the information from the RC Atomic
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * work request.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor HERMON_WQE_BUILD_ATOMIC(qp, at, wr->wr.rc.rcwr.atomic);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Update "ds" for filling in Data Segments (below) */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Update "nds" and "sgl" because Atomic requests have
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * only a single Data Segment (and they are encoded
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * somewhat differently in the work request.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * If this is memory window Bind operation, then we call the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * hermon_wr_bind_check() routine to validate the request and
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * to generate the updated RKey. If this is successful, then
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * we fill in the WQE's "Bind" header fields.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor bn = (hermon_hw_snd_wqe_bind_t *)((uintptr_t)desc +
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Build the Bind Memory Window Segments for the WQE,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * using the information from the RC Bind memory
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * window work request.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor HERMON_WQE_BUILD_BIND(qp, bn, wr->wr.rc.rcwr.bind);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Update the "ds" pointer. Even though the "bind"
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * operation requires no SGLs, this is necessary to
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * facilitate the correct descriptor size calculations
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Ensure that work request transport type matches QP type */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Validate the operation type. For UC requests, we only
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * allow "Send", "RDMA Write", and memory window "Bind".
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Note: Unlike RC, UC does not allow "RDMA Read" or "Atomic"
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * operations
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * If this is a Send request, then all we need to do is break
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * out and here and begin the Data Segment processing below
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * If this is an RDMA Write request, then fill in the "Remote
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Address" header fields.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor uc = (hermon_hw_snd_wqe_remaddr_t *)((uintptr_t)desc +
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Build the Remote Address Segment for the WQE, using
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the information from the UC work request.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor HERMON_WQE_BUILD_REMADDR(qp, uc, &wr->wr.uc.ucwr.rdma);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Update "ds" for filling in Data Segments (below) */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * If this is memory window Bind operation, then we call the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * hermon_wr_bind_check() routine to validate the request and
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * to generate the updated RKey. If this is successful, then
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * we fill in the WQE's "Bind" header fields.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor bn = (hermon_hw_snd_wqe_bind_t *)((uintptr_t)desc +
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Build the Bind Memory Window Segments for the WQE,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * using the information from the UC Bind memory
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * window work request.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor HERMON_WQE_BUILD_BIND(qp, bn, wr->wr.uc.ucwr.bind);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Update the "ds" pointer. Even though the "bind"
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * operation requires no SGLs, this is necessary to
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * facilitate the correct descriptor size calculations
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Now fill in the Data Segments (SGL) for the Send WQE based on
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the values setup above (i.e. "sgl", "nds", and the "ds" pointer
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Start by checking for a valid number of SGL entries
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * For each SGL in the Send Work Request, fill in the Send WQE's data
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * segments. Note: We skip any SGL with zero size because Hermon
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * hardware cannot handle a zero for "byte_cnt" in the WQE. Actually
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the encoding for zero means a 2GB transfer.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Return the size of descriptor (in 16-byte chunks)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * For Hermon, we want them (for now) to be on stride size
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * boundaries, which was implicit in Tavor/Arbel
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor tmpsize = ((uintptr_t)&ds[last_ds] - (uintptr_t)desc);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor for (j = nds; --j >= i; ) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Fill in the Data Segment(s) for the current WQE, using the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * information contained in the scatter-gather list of the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * work request.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor HERMON_WQE_BUILD_DATA_SEG_SEND(&ds[last_ds], &sgl[j]);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * hermon_wqe_mlx_build()
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Context: Can be called from interrupt or base context.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorhermon_wqe_mlx_build(hermon_state_t *state, hermon_qphdl_t qp,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ibt_send_wr_t *wr, uint64_t *desc, uint_t *size)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Initialize the information for the Data Segments */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Pull the address handle from the work request. The UDAV will
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * be used to answer some questions about the request.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ah = (hermon_ahhdl_t)wr->wr.ud.udwr_dest->ud_ah;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * If the request is for QP1 and the destination LID is equal to
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the Permissive LID, then return an error. This combination is
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * not allowed
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Calculate the size of the packet headers, including the GRH
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * (if necessary)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor desc_sz = sizeof (ib_lrh_hdr_t) + sizeof (ib_bth_hdr_t) +
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Begin to build the first "inline" data segment for the packet
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * headers. Note: By specifying "inline" we can build the contents
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * of the MAD packet headers directly into the work queue (as part
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * descriptor). This has the advantage of both speeding things up
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * and of not requiring the driver to allocate/register any additional
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * memory for the packet headers.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Build Local Route Header (LRH)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * We start here by building the LRH into a temporary location.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * When we have finished we copy the LRH data into the descriptor.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Notice that the VL values are hardcoded. This is not a problem
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * because VL15 is decided later based on the value in the MLX
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * transport "next/ctrl" header (see the "vl15" bit below), and it
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * is otherwise (meaning for QP1) chosen from the SL-to-VL table
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * values. This rule does not hold for loopback packets however
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * (all of which bypass the SL-to-VL tables) and it is the reason
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * that non-QP0 MADs are setup with VL hardcoded to zero below.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Notice also that Source LID is hardcoded to the Permissive LID
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * (0xFFFF). This is also not a problem because if the Destination
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * LID is not the Permissive LID, then the "slr" value in the MLX
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * transport "next/ctrl" header will be set to zero and the hardware
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * will pull the LID from value in the port.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor HERMON_WQE_BUILD_MLX_LRH(lrh, qp, udav, pktlen);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Build Global Route Header (GRH)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * This is only built if necessary as defined by the "grh" bit in
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the address vector. Note: We also calculate the offset to the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * next header (BTH) based on whether or not the "grh" bit is set.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * If the request is for QP0, then return an error. The
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * combination of global routine (GRH) and QP0 is not allowed.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor grh = (ib_grh_t *)((uintptr_t)lrh + sizeof (ib_lrh_hdr_t));
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor HERMON_WQE_BUILD_MLX_GRH(state, grh, qp, udav, pktlen);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor bth = (ib_bth_hdr_t *)((uintptr_t)grh + sizeof (ib_grh_t));
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor bth = (ib_bth_hdr_t *)((uintptr_t)lrh + sizeof (ib_lrh_hdr_t));
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Build Base Transport Header (BTH)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Notice that the M, PadCnt, and TVer fields are all set
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * to zero implicitly. This is true for all Management Datagrams
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * MADs whether GSI are SMI.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Build Datagram Extended Transport Header (DETH)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor deth = (ib_deth_hdr_t *)((uintptr_t)bth + sizeof (ib_bth_hdr_t));
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Ensure that the Data Segment is aligned on a 16-byte boundary */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ds = (hermon_hw_wqe_sgl_t *)((uintptr_t)deth + sizeof (ib_deth_hdr_t));
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ds = (hermon_hw_wqe_sgl_t *)(((uintptr_t)ds + 0xF) & ~0xF);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Now fill in the Data Segments (SGL) for the MLX WQE based on the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * values set up above (i.e. "sgl", "nds", and the "ds" pointer
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Start by checking for a valid number of SGL entries
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * For each SGL in the Send Work Request, fill in the MLX WQE's data
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * segments. Note: We skip any SGL with zero size because Hermon
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * hardware cannot handle a zero for "byte_cnt" in the WQE. Actually
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the encoding for zero means a 2GB transfer. Because of this special
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * encoding in the hardware, we mask the requested length with
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * HERMON_WQE_SGL_BYTE_CNT_MASK (so that 2GB will end up encoded as
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor for (i = 0; i < nds; i++) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Fill in the Data Segment(s) for the MLX send WQE, using
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the information contained in the scatter-gather list of
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the work request.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor HERMON_WQE_BUILD_DATA_SEG_SEND(&ds[num_ds], &sgl[i]);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Search through the contents of all MADs posted to QP0 to
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * initialize pointers to the places where Directed Route "hop
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * pointer", "hop count", and "mgmtclass" would be. Hermon
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * needs these updated (i.e. incremented or decremented, as
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * necessary) by software.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor HERMON_SPECIAL_QP_DRMAD_GET_MGMTCLASS(mgmtclass,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Hermon's Directed Route MADs need to have the "hop pointer"
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * incremented/decremented (as necessary) depending on whether it is
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * currently less than or greater than the "hop count" (i.e. whether
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the MAD is a request or a response.)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor HERMON_SPECIAL_QP_DRMAD_DO_HOPPOINTER_MODIFY(*mgmtclass,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Now fill in the ICRC Data Segment. This data segment is inlined
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * just like the packets headers above, but it is only four bytes and
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * set to zero (to indicate that we wish the hardware to generate ICRC.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor HERMON_WQE_BUILD_INLINE_ICRC(qp, &ds[num_ds], 4, 0);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Return the size of descriptor (in 16-byte chunks)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * For Hermon, we want them (for now) to be on stride size
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * boundaries, which was implicit in Tavor/Arbel
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor tmpsize = ((uintptr_t)&ds[num_ds] - (uintptr_t)desc);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * hermon_wqe_recv_build()
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Context: Can be called from interrupt or base context.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor/* ARGSUSED */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorhermon_wqe_recv_build(hermon_state_t *state, hermon_qphdl_t qp,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Fill in the Data Segments (SGL) for the Recv WQE - don't
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * need to have a reserved for the ctrl, there is none on the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * recv queue for hermon, but will need to put an invalid
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * (null) scatter pointer per PRM
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Check for valid number of SGL entries */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * For each SGL in the Recv Work Request, fill in the Recv WQE's data
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * segments. Note: We skip any SGL with zero size because Hermon
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * hardware cannot handle a zero for "byte_cnt" in the WQE. Actually
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the encoding for zero means a 2GB transfer. Because of this special
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * encoding in the hardware, we mask the requested length with
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * HERMON_WQE_SGL_BYTE_CNT_MASK (so that 2GB will end up encoded as
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Fill in the Data Segment(s) for the receive WQE, using the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * information contained in the scatter-gather list of the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * work request.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor HERMON_WQE_BUILD_DATA_SEG_RECV(&ds[num_ds], &wr->wr_sgl[i]);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* put the null sgl pointer as well if needed */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor HERMON_WQE_BUILD_DATA_SEG_RECV(&ds[num_ds], &null_sgl);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * hermon_wqe_srq_build()
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Context: Can be called from interrupt or base context.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor/* ARGSUSED */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorhermon_wqe_srq_build(hermon_state_t *state, hermon_srqhdl_t srq,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Fill in the Data Segments (SGL) for the Recv WQE */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Check for valid number of SGL entries */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * For each SGL in the Recv Work Request, fill in the Recv WQE's data
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * segments. Note: We skip any SGL with zero size because Hermon
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * hardware cannot handle a zero for "byte_cnt" in the WQE. Actually
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the encoding for zero means a 2GB transfer. Because of this special
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * encoding in the hardware, we mask the requested length with
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * HERMON_WQE_SGL_BYTE_CNT_MASK (so that 2GB will end up encoded as
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Fill in the Data Segment(s) for the receive WQE, using the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * information contained in the scatter-gather list of the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * work request.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor HERMON_WQE_BUILD_DATA_SEG_RECV(&ds[num_ds], &wr->wr_sgl[i]);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * put in the null sgl pointer as well, if needed
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor HERMON_WQE_BUILD_DATA_SEG_RECV(&ds[num_ds], &null_sgl);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * hermon_wr_get_immediate()
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Context: Can be called from interrupt or base context.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * This routine extracts the "immediate data" from the appropriate
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * location in the IBTF work request. Because of the way the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * work request structure is defined, the location for this data
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * depends on the actual work request operation type.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* For RDMA Write, test if RC or UC */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor } else { /* IBT_UC_SRV */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* For Send, test if RC, UD, or UC */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor } else { /* IBT_UC_SRV */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * If any other type of request, then immediate is undefined
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * hermon_wqe_headroom()
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Context: can be called from interrupt or base, currently only from
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * base context.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Routine that fills in the headroom for the Send Queue
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorhermon_wqe_headroom(uint_t from, hermon_qphdl_t qp)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor uint32_t *wqe_start, *wqe_top, *wqe_base, qsize;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor sectperwqe = wqesizebytes >> 6; /* 64 bytes/section */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor wqe_base = (uint32_t *)HERMON_QP_SQ_ENTRY(qp, 0);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor wqe_top = (uint32_t *)HERMON_QP_SQ_ENTRY(qp, qsize);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor wqe_start = (uint32_t *)HERMON_QP_SQ_ENTRY(qp, from);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor for (i = 0; i < hdrmwqes; i++) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor for (j = 0; j < sectperwqe; j++) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor if (j == 0) { /* 1st section of wqe */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* perserve ownership bit */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* or just invalidate it */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ddi_put32(qp->qp_wqinfo.qa_acchdl, wqe_start, invalue);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor if (wqe_start == wqe_top) /* hit the end of the queue */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * hermon_wr_bind_check()
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Context: Can be called from interrupt or base context.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor/* ARGSUSED */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorhermon_wr_bind_check(hermon_state_t *state, ibt_send_wr_t *wr)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Check for a valid Memory Window handle in the WR */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor mw = (hermon_mwhdl_t)wr->wr.rc.rcwr.bind->bind_ibt_mw_hdl;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Check for a valid Memory Region handle in the WR */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor mr = (hermon_mrhdl_t)wr->wr.rc.rcwr.bind->bind_ibt_mr_hdl;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Check here to see if the memory region has already been partially
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * deregistered as a result of a hermon_umap_umemlock_cb() callback.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * If so, this is an error, return failure.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor if ((mr->mr_is_umem) && (mr->mr_umemcookie == NULL)) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Check for a valid Memory Window RKey (i.e. a matching RKey) */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor if (mw->mr_rkey != wr->wr.rc.rcwr.bind->bind_rkey) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Check for a valid Memory Region LKey (i.e. a matching LKey) */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor if (mr->mr_lkey != wr->wr.rc.rcwr.bind->bind_lkey) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Now check for valid "vaddr" and "len". Note: We don't check the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * "vaddr" range when "len == 0" (i.e. on unbind operations)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor if ((vaddr < reg_start_addr) || (vaddr > reg_end_addr)) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Validate the bind access flags. Remote Write and Atomic access for
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the Memory Window require that Local Write access be set in the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * corresponding Memory Region.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Calculate the new RKey for the Memory Window */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * hermon_wrid_from_reset_handling()
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Context: Can be called from interrupt or base context.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor/* ARGSUSED */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorhermon_wrid_from_reset_handling(hermon_state_t *state, hermon_qphdl_t qp)
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor /* grab the cq lock(s) to modify the wqavl tree */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Chain the newly allocated work queue header to the CQ's list */
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor hermon_cq_workq_add(qp->qp_sq_cqhdl, &qp->qp_sq_wqavl);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Now we repeat all the above operations for the receive work queue,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * or shared receive work queue.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Note: We still use the 'qp_rq_cqhdl' even in the SRQ case.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor hermon_cq_workq_add(qp->qp_rq_cqhdl, &qp->qp_rq_wqavl);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * hermon_wrid_to_reset_handling()
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Context: Can be called from interrupt or base context.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorhermon_wrid_to_reset_handling(hermon_state_t *state, hermon_qphdl_t qp)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * If there are unpolled entries in these CQs, they are
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Grab the CQ lock(s) before manipulating the lists.
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor /* grab the cq lock(s) to modify the wqavl tree */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Flush the entries on the CQ for this QP's QPN.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor hermon_cq_workq_remove(qp->qp_rq_cqhdl, &qp->qp_rq_wqavl);
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor hermon_cq_workq_remove(qp->qp_sq_cqhdl, &qp->qp_sq_wqavl);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * hermon_wrid_get_entry()
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Context: Can be called from interrupt or base context.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorhermon_wrid_get_entry(hermon_cqhdl_t cq, hermon_hw_cqe_t *cqe)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Determine whether this CQE is a send or receive completion.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor send_or_recv = HERMON_CQE_SENDRECV_GET(cq, cqe);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Find the work queue for this QP number (send or receive side) */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor wqa = hermon_wrid_wqavl_find(cq, qpnum, send_or_recv);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Regardless of whether the completion is the result of a "success"
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * or a "failure", we lock the list of "containers" and attempt to
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * search for the the first matching completion (i.e. the first WR
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * with a matching WQE addr and size). Once we find it, we pull out
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the "wrid" field and return it (see below). XXX Note: One possible
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * future enhancement would be to enable this routine to skip over
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * any "unsignaled" completions to go directly to the next "signaled"
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * entry on success.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor indx = HERMON_CQE_WQEADDRSZ_GET(cq, cqe) & wq->wq_mask;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* put wqe back on the srq free list */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorhermon_wrid_workq_compare(const void *p1, const void *p2)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor return (-1);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor return (+1);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor return (-1);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor return (+1);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * hermon_wrid_workq_find()
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Context: Can be called from interrupt or base context.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorhermon_wrid_wqavl_find(hermon_cqhdl_t cq, uint_t qpn, uint_t wq_type)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Walk the CQ's work queue list, trying to find a send or recv queue
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * with the same QP number. We do this even if we are going to later
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * create a new entry because it helps us easily find the end of the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor curr = avl_find(&cq->cq_wrid_wqhdr_avl_tree, &cmp, NULL);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * hermon_wrid_wqhdr_create()
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Context: Can be called from base context.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor/* ARGSUSED */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Allocate space for the wqhdr, and an array to record all the wrids.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor wqhdr = (hermon_workq_hdr_t *)kmem_zalloc(sizeof (*wqhdr), KM_NOSLEEP);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor wqhdr->wq_wrid = kmem_zalloc(bufsz * sizeof (uint64_t), KM_NOSLEEP);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorhermon_wrid_wqhdr_destroy(hermon_workq_hdr_t *wqhdr)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor kmem_free(wqhdr->wq_wrid, wqhdr->wq_size * sizeof (uint64_t));
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * hermon_cq_workq_add()
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Context: Can be called from interrupt or base context.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorhermon_cq_workq_add(hermon_cqhdl_t cq, hermon_workq_avl_t *wqavl)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor (void) avl_find(&cq->cq_wrid_wqhdr_avl_tree, &cmp, &where);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor avl_insert(&cq->cq_wrid_wqhdr_avl_tree, wqavl, where);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * hermon_cq_workq_remove()
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Context: Can be called from interrupt or base context.