hermon_wr.c revision 9c865d645a5c60028aed172eea31ca36d81a2ff1
/*
* CDDL HEADER START
*
* The contents of this file are subject to the terms of the
* Common Development and Distribution License (the "License").
* You may not use this file except in compliance with the License.
*
* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
* See the License for the specific language governing permissions
* and limitations under the License.
*
* When distributing Covered Code, include this CDDL HEADER in each
* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
* If applicable, add the following below this CDDL HEADER, with the
* fields enclosed by brackets "[]" replaced with your own identifying
* information: Portions Copyright [yyyy] [name of copyright owner]
*
* CDDL HEADER END
*/
/*
* Copyright 2009 Sun Microsystems, Inc. All rights reserved.
* Use is subject to license terms.
*/
/*
* Hermon Work Request Processing Routines
*
* Implements all the routines necessary to provide the PostSend(),
* PostRecv() and PostSRQ() verbs. Also contains all the code
* necessary to implement the Hermon WRID tracking mechanism.
*/
static int
{
int sectperwqe;
uint_t posted_cnt = 0;
/* initialize the FMA retry loop */
/* Grab the lock for the WRID list */
/* Save away some initial QP state */
/*
* Check for "queue full" condition. If the queue
* is already full, then no more WQEs can be posted.
* So break out, ring a doorbell (if necessary) and
* return an error
*/
goto done;
}
}
sizeof (hermon_hw_snd_wqe_ctrl_t));
sizeof (hermon_hw_snd_wqe_ud_t));
num_ds = 0;
/* need to know the count of destination nds for backward loop */
dnds++;
}
/*
* Build a Send or Send_LSO WQE
*/
int total_len;
}
goto done;
}
goto done;
}
i = 0;
} else {
}
goto done;
}
i = 0;
} else {
goto done;
}
goto done;
}
last_ds++; /* real last ds of wqe to fill */
}
for (j = nds; --j >= i; ) {
continue;
}
/*
* Fill in the Data Segment(s) for the current WQE, using the
* information contained in the scatter-gather list of the
* work request.
*/
last_ds--;
}
}
/* Update some of the state in the QP */
/* Now set the ownership bit and opcode (first dword). */
posted_cnt++;
if (--num_wr > 0) {
/* do the invalidate of the headroom */
wqe_start[i] = 0xFFFFFFFF;
}
wr++;
goto post_next;
}
done:
if (posted_cnt != 0) {
/* the FMA retry loop starts for Hermon doorbell register. */
/* the FMA retry loop ends. */
/* do the invalidate of the headroom */
wqe_start[i] = 0xFFFFFFFF;
}
}
if (num_posted != NULL)
*num_posted = posted_cnt;
return (status);
return (ibc_get_ci_failure(0));
}
static int
{
int status;
int sectperwqe;
uint_t posted_cnt = 0;
/* initialize the FMA retry loop */
/* make sure we see any update of wq_head */
/* Save away some initial QP state */
/*
* Check for "queue full" condition. If the queue
* is already full, then no more WQEs can be posted.
* So break out, ring a doorbell (if necessary) and
* return an error
*/
goto done;
}
}
sizeof (hermon_hw_snd_wqe_ctrl_t));
num_ds = 0;
/*
* Validate the operation type. For RC requests, we allow
* "Send", "RDMA Read", "RDMA Write", various "Atomic"
* operations, and memory window "Bind"
*/
default:
goto done;
case IBT_WRC_SEND:
} else {
}
break;
/*
* If this is an RDMA Read or RDMA Write request, then fill
* in the "Remote Address" header fields.
*/
case IBT_WRC_RDMAW:
} else {
}
/* FALLTHROUGH */
case IBT_WRC_RDMAR:
sizeof (hermon_hw_snd_wqe_ctrl_t));
/*
* Build the Remote Address Segment for the WQE, using
* the information from the RC work request.
*/
/* Update "ds" for filling in Data Segments (below) */
sizeof (hermon_hw_snd_wqe_remaddr_t));
break;
/*
* If this is one of the Atomic type operations (i.e
* Compare-Swap or Fetch-Add), then fill in both the "Remote
* Address" header fields and the "Atomic" header fields.
*/
case IBT_WRC_CSWAP:
/* FALLTHROUGH */
case IBT_WRC_FADD:
sizeof (hermon_hw_snd_wqe_ctrl_t));
sizeof (hermon_hw_snd_wqe_remaddr_t));
/*
* Build the Remote Address and Atomic Segments for
* the WQE, using the information from the RC Atomic
* work request.
*/
/* Update "ds" for filling in Data Segments (below) */
sizeof (hermon_hw_snd_wqe_atomic_t));
/*
* Update "nds" and "sgl" because Atomic requests have
* only a single Data Segment.
*/
nds = 1;
break;
/*
* If this is memory window Bind operation, then we call the
* hermon_wr_bind_check() routine to validate the request and
* to generate the updated RKey. If this is successful, then
* we fill in the WQE's "Bind" header fields.
*/
case IBT_WRC_BIND:
if (status != DDI_SUCCESS)
goto done;
sizeof (hermon_hw_snd_wqe_ctrl_t));
/*
* Build the Bind Memory Window Segments for the WQE,
* using the information from the RC Bind memory
* window work request.
*/
/*
* Update the "ds" pointer. Even though the "bind"
* operation requires no SGLs, this is necessary to
* facilitate the correct descriptor size calculations
* (below).
*/
sizeof (hermon_hw_snd_wqe_bind_t));
nds = 0;
}
/*
* Now fill in the Data Segments (SGL) for the Send WQE based
* on the values setup above (i.e. "sgl", "nds", and the "ds"
* pointer. Start by checking for a valid number of SGL entries
*/
goto done;
}
last_ds++; /* real last ds of wqe to fill */
}
for (i = nds; --i >= 0; ) {
continue;
}
/*
* Fill in the Data Segment(s) for the current WQE, using the
* information contained in the scatter-gather list of the
* work request.
*/
last_ds--;
}
/* Update some of the state in the QP */
/* Now set the ownership bit of the first one in the chain. */
posted_cnt++;
if (--num_wr > 0) {
/* do the invalidate of the headroom */
wqe_start[i] = 0xFFFFFFFF;
}
wr++;
goto post_next;
}
done:
if (posted_cnt != 0) {
/* the FMA retry loop starts for Hermon doorbell register. */
/* Ring the doorbell */
/* the FMA retry loop ends. */
/* do the invalidate of the headroom */
wqe_start[i] = 0xFFFFFFFF;
}
}
/*
* Update the "num_posted" return value (if necessary).
* Then drop the locks and return success.
*/
if (num_posted != NULL) {
*num_posted = posted_cnt;
}
return (status);
return (ibc_get_ci_failure(0));
}
/*
* hermon_post_send()
* Context: Can be called from interrupt or base context.
*/
int
{
int status;
/* initialize the FMA retry loop */
/*
* Check for user-mappable QP memory. Note: We do not allow kernel
* clients to post to QP memory that is accessible directly by the
* user. If the QP memory is user accessible, then return an error.
*/
if (qp->qp_is_umap) {
return (IBT_QP_HDL_INVALID);
}
/*
* Check QP state. Can not post Send requests from the "Reset",
* "Init", or "RTR" states
*/
return (IBT_QP_STATE_INVALID);
}
if (qp->qp_is_special)
goto post_many;
/* Use these optimized functions most of the time */
goto post_many;
return (IBT_QP_SRV_TYPE_INVALID);
/* general loop for non-optimized posting */
/* Grab the lock for the WRID list */
/* Save away some initial QP state */
/* Initialize posted_cnt */
posted_cnt = 0;
total_posted = 0;
/*
* For each ibt_send_wr_t in the wr[] list passed in, parse the
* request and build a Send WQE. NOTE: Because we are potentially
* building a chain of WQEs to post, we want to build them all first,
* and set the valid (HW Ownership) bit on all but the first.
* However, we do not want to validate the first one until the
* entire chain of WQEs has been built. Then in the final
* we set the valid bit in the first, flush if needed, and as a last
* step ring the appropriate doorbell. NOTE: the doorbell ring may
* NOT be needed if the HCA is already processing, but the doorbell
* ring will be done regardless. NOTE ALSO: It is possible for
* more Work Requests to be posted than the HW will support at one
* shot. If this happens, we need to be able to post and ring
* several chains here until the the entire request is complete.
* NOTE ALSO: the term "chain" is used to differentiate it from
* Work Request List passed in; and because that's the terminology
* from the previous generations of HCA - but the WQEs are not, in fact
* chained together for Hermon
*/
wrindx = 0;
/*
* For the first WQE on a new chain we need "prev" to point
* to the current descriptor.
*/
/*
* unlike Tavor & Arbel, tail will maintain the number of the
* next (this) WQE to be posted. Since there is no backward linking
* in Hermon, we can always just look ahead
*/
/*
* Before we begin, save the current "tail index" for later
* DMA sync
*/
/*
* Break the request up into lists that are less than or
* equal to the maximum number of WQEs that can be posted
* per doorbell ring - 256 currently
*/
/*
* Check for "queue full" condition. If the queue
* is already full, then no more WQEs can be posted.
* So break out, ring a doorbell (if necessary) and
* return an error
*/
break;
}
/*
* Increment the "tail index". Check for "queue
* full" condition incl. headroom. If we detect that
* the current work request is going to fill the work
* queue, then we mark this condition and continue.
* Don't need >=, because going one-by-one we have to
* hit it exactly sooner or later
*/
}
/*
* Get the address of the location where the next
* Send WQE should be built
*/
/*
* Call hermon_wqe_send_build() to build the WQE
* at the given address. This routine uses the
* information in the ibt_send_wr_t list (wr[]) and
* returns the size of the WQE when it returns.
*/
if (status != DDI_SUCCESS) {
break;
}
/*
* Now, build the Ctrl Segment based on
* what was just done
*/
case IBT_WRC_RDMAW:
nopcode =
} else {
}
break;
case IBT_WRC_SEND:
} else {
}
break;
case IBT_WRC_SEND_LSO:
break;
case IBT_WRC_RDMAR:
break;
case IBT_WRC_CSWAP:
break;
case IBT_WRC_FADD:
break;
case IBT_WRC_BIND:
break;
}
/*
* now, build up the control segment, leaving the
* owner bit as it is
*/
signaled_dbd = 1;
} else {
signaled_dbd = 0;
}
solicited = 1;
else
solicited = 0;
if (qp->qp_is_special) {
ah = (hermon_ahhdl_t)
} else {
}
/*
* If this is not the first descriptor on the current
* chain, then set the ownership bit.
*/
if (currindx != 0) { /* not the first */
} else
/*
* Update the current "tail index" and increment
* "posted_cnt"
*/
posted_cnt++;
}
/*
* If we reach here and there are one or more WQEs which have
* been successfully built as a chain, we have to finish up
* and prepare them for writing to the HW
* The steps are:
* 1. do the headroom fixup
* 2. add in the size of the headroom for the sync
* 3. write the owner bit for the first WQE
* 4. sync them
* 5. fix up the structures
* 6. hit the doorbell in UAR
*/
if (posted_cnt != 0) {
/*
* Save away updated "tail index" for the DMA sync
* including the headroom that will be needed
*/
/* do the invalidate of the headroom */
/* Do a DMA sync for current send WQE(s) */
/* Update some of the state in the QP */
posted_cnt = 0;
/*
* Now set the ownership bit of the first
* one in the chain
*/
/* the FMA retry loop starts for Hermon doorbell. */
/* the FMA retry loop ends. */
}
}
/*
* Update the "num_posted" return value (if necessary).
* Then drop the locks and return success.
*/
if (num_posted != NULL) {
}
return (status);
return (ibc_get_ci_failure(0));
}
/*
* hermon_post_recv()
* Context: Can be called from interrupt or base context.
*/
int
{
int status;
/*
* Check for user-mappable QP memory. Note: We do not allow kernel
* clients to post to QP memory that is accessible directly by the
* user. If the QP memory is user accessible, then return an error.
*/
if (qp->qp_is_umap) {
return (IBT_QP_HDL_INVALID);
}
/* Initialize posted_cnt */
posted_cnt = 0;
/*
* Check if QP is associated with an SRQ
*/
return (IBT_SRQ_IN_USE);
}
/*
* Check QP state. Can not post Recv requests from the "Reset" state
*/
return (IBT_QP_STATE_INVALID);
}
/* Check that work request transport type is valid */
return (IBT_QP_SRV_TYPE_INVALID);
}
/*
* Grab the lock for the WRID list, i.e., membar_consumer().
* This is not needed because the mutex_enter() above has
* the same effect.
*/
/* Save away some initial QP state */
wrindx = 0;
/*
* Before we begin, save the current "tail index" for later
* DMA sync
*/
break;
}
}
if (status != DDI_SUCCESS) {
break;
}
qp->qp_rq_wqecntr++;
posted_cnt++;
}
if (posted_cnt != 0) {
/* Save away updated "tail index" for the DMA sync */
membar_producer(); /* ensure wrids are visible */
/* Update the doorbell record w/ wqecntr */
}
if (num_posted != NULL) {
*num_posted = posted_cnt;
}
return (status);
}
/*
* hermon_post_srq()
* Context: Can be called from interrupt or base context.
*/
int
{
int status;
/*
* Check for user-mappable QP memory. Note: We do not allow kernel
* clients to post to QP memory that is accessible directly by the
* user. If the QP memory is user accessible, then return an error.
*/
if (srq->srq_is_umap) {
return (IBT_SRQ_HDL_INVALID);
}
/*
* Check SRQ state. Can not post Recv requests when SRQ is in error
*/
return (IBT_QP_STATE_INVALID);
}
posted_cnt = 0;
break;
}
if (status != DDI_SUCCESS) {
break;
}
posted_cnt++;
}
if (posted_cnt != 0) {
membar_producer(); /* ensure wrids are visible */
/* Ring the doorbell w/ wqecntr */
}
if (num_posted != NULL) {
*num_posted = posted_cnt;
}
return (status);
}
/*
* hermon_wqe_send_build()
* Context: Can be called from interrupt or base context.
*/
static int
{
int tmpsize;
/* Initialize the information for the Data Segments */
sizeof (hermon_hw_snd_wqe_ctrl_t));
num_ds = 0;
i = 0;
/*
* Build a Send WQE depends first and foremost on the transport
* type of Work Request (i.e. UD, RC, or UC)
*/
case IBT_UD_SRV:
/* Ensure that work request transport type matches QP type */
return (IBT_QP_SRV_TYPE_INVALID);
}
/*
* Validate the operation type. For UD requests, only the
* "Send" and "Send LSO" operations are valid.
*/
return (IBT_QP_OP_TYPE_INVALID);
}
/*
* If this is a Special QP (QP0 or QP1), then we need to
* build MLX WQEs instead. So jump to hermon_wqe_mlx_build()
* and return whatever status it returns
*/
if (qp->qp_is_special) {
return (IBT_QP_OP_TYPE_INVALID);
}
return (status);
}
/*
* Otherwise, if this is a normal UD Send request, then fill
* all the fields in the Hermon UD header for the WQE. Note:
* to do this we'll need to extract some information from the
* Address Handle passed with the work request.
*/
sizeof (hermon_hw_snd_wqe_ctrl_t));
} else {
}
return (IBT_AH_HDL_INVALID);
}
/*
* Build the Unreliable Datagram Segment for the WQE, using
* the information from the address handle and the work
* request.
*/
/* mutex_enter(&ah->ah_lock); */
} else { /* IBT_WRC_SEND_LSO */
}
/* mutex_exit(&ah->ah_lock); */
/* Update "ds" for filling in Data Segments (below) */
sizeof (hermon_hw_snd_wqe_ud_t));
int total_len;
return (IBT_QP_SGL_LEN_INVALID);
for (; i < nds; i++) {
continue;
&sgl[i]);
num_ds++;
i++;
break;
}
}
break;
case IBT_RC_SRV:
/* Ensure that work request transport type matches QP type */
return (IBT_QP_SRV_TYPE_INVALID);
}
/*
* Validate the operation type. For RC requests, we allow
* "Send", "RDMA Read", "RDMA Write", various "Atomic"
* operations, and memory window "Bind"
*/
return (IBT_QP_OP_TYPE_INVALID);
}
/*
* If this is a Send request, then all we need to do is break
* out and here and begin the Data Segment processing below
*/
break;
}
/*
* If this is an RDMA Read or RDMA Write request, then fill
* in the "Remote Address" header fields.
*/
sizeof (hermon_hw_snd_wqe_ctrl_t));
/*
* Build the Remote Address Segment for the WQE, using
* the information from the RC work request.
*/
/* Update "ds" for filling in Data Segments (below) */
sizeof (hermon_hw_snd_wqe_remaddr_t));
break;
}
/*
* If this is one of the Atomic type operations (i.e
* Compare-Swap or Fetch-Add), then fill in both the "Remote
* Address" header fields and the "Atomic" header fields.
*/
sizeof (hermon_hw_snd_wqe_ctrl_t));
sizeof (hermon_hw_snd_wqe_remaddr_t));
/*
* Build the Remote Address and Atomic Segments for
* the WQE, using the information from the RC Atomic
* work request.
*/
/* Update "ds" for filling in Data Segments (below) */
sizeof (hermon_hw_snd_wqe_atomic_t));
/*
* Update "nds" and "sgl" because Atomic requests have
* only a single Data Segment (and they are encoded
* somewhat differently in the work request.
*/
nds = 1;
break;
}
/*
* If this is memory window Bind operation, then we call the
* hermon_wr_bind_check() routine to validate the request and
* to generate the updated RKey. If this is successful, then
* we fill in the WQE's "Bind" header fields.
*/
if (status != DDI_SUCCESS) {
return (status);
}
sizeof (hermon_hw_snd_wqe_ctrl_t));
/*
* Build the Bind Memory Window Segments for the WQE,
* using the information from the RC Bind memory
* window work request.
*/
/*
* Update the "ds" pointer. Even though the "bind"
* operation requires no SGLs, this is necessary to
* facilitate the correct descriptor size calculations
* (below).
*/
sizeof (hermon_hw_snd_wqe_bind_t));
nds = 0;
}
break;
case IBT_UC_SRV:
/* Ensure that work request transport type matches QP type */
return (IBT_QP_SRV_TYPE_INVALID);
}
/*
* Validate the operation type. For UC requests, we only
* allow "Send", "RDMA Write", and memory window "Bind".
* Note: Unlike RC, UC does not allow "RDMA Read" or "Atomic"
* operations
*/
return (IBT_QP_OP_TYPE_INVALID);
}
/*
* If this is a Send request, then all we need to do is break
* out and here and begin the Data Segment processing below
*/
break;
}
/*
* If this is an RDMA Write request, then fill in the "Remote
* Address" header fields.
*/
sizeof (hermon_hw_snd_wqe_ctrl_t));
/*
* Build the Remote Address Segment for the WQE, using
* the information from the UC work request.
*/
/* Update "ds" for filling in Data Segments (below) */
sizeof (hermon_hw_snd_wqe_remaddr_t));
break;
}
/*
* If this is memory window Bind operation, then we call the
* hermon_wr_bind_check() routine to validate the request and
* to generate the updated RKey. If this is successful, then
* we fill in the WQE's "Bind" header fields.
*/
if (status != DDI_SUCCESS) {
return (status);
}
sizeof (hermon_hw_snd_wqe_ctrl_t));
/*
* Build the Bind Memory Window Segments for the WQE,
* using the information from the UC Bind memory
* window work request.
*/
/*
* Update the "ds" pointer. Even though the "bind"
* operation requires no SGLs, this is necessary to
* facilitate the correct descriptor size calculations
* (below).
*/
sizeof (hermon_hw_snd_wqe_bind_t));
nds = 0;
}
break;
default:
return (IBT_QP_SRV_TYPE_INVALID);
}
/*
* Now fill in the Data Segments (SGL) for the Send WQE based on
* the values setup above (i.e. "sgl", "nds", and the "ds" pointer
* Start by checking for a valid number of SGL entries
*/
return (IBT_QP_SGL_LEN_INVALID);
}
/*
* For each SGL in the Send Work Request, fill in the Send WQE's data
* segments. Note: We skip any SGL with zero size because Hermon
* hardware cannot handle a zero for "byte_cnt" in the WQE. Actually
* the encoding for zero means a 2GB transfer.
*/
last_ds++; /* real last ds of wqe to fill */
}
/*
* Return the size of descriptor (in 16-byte chunks)
* For Hermon, we want them (for now) to be on stride size
*
*/
for (j = nds; --j >= i; ) {
continue;
}
/*
* Fill in the Data Segment(s) for the current WQE, using the
* information contained in the scatter-gather list of the
* work request.
*/
last_ds--;
}
return (DDI_SUCCESS);
}
/*
* hermon_wqe_mlx_build()
* Context: Can be called from interrupt or base context.
*/
static int
{
int i, num_ds;
int tmpsize;
/* Initialize the information for the Data Segments */
sizeof (hermon_hw_mlx_wqe_nextctrl_t));
/*
* Pull the address handle from the work request. The UDAV will
* be used to answer some questions about the request.
*/
return (IBT_AH_HDL_INVALID);
}
/*
* If the request is for QP1 and the destination LID is equal to
* the Permissive LID, then return an error. This combination is
* not allowed
*/
return (IBT_AH_HDL_INVALID);
}
/*
* Calculate the size of the packet headers, including the GRH
* (if necessary)
*/
sizeof (ib_deth_hdr_t);
}
/*
* Begin to build the first "inline" data segment for the packet
* headers. Note: By specifying "inline" we can build the contents
* of the MAD packet headers directly into the work queue (as part
* descriptor). This has the advantage of both speeding things up
* memory for the packet headers.
*/
desc_sz += 4;
/*
* Build Local Route Header (LRH)
* We start here by building the LRH into a temporary location.
* When we have finished we copy the LRH data into the descriptor.
*
* Notice that the VL values are hardcoded. This is not a problem
* because VL15 is decided later based on the value in the MLX
* is otherwise (meaning for QP1) chosen from the SL-to-VL table
* values. This rule does not hold for loopback packets however
* (all of which bypass the SL-to-VL tables) and it is the reason
* that non-QP0 MADs are setup with VL hardcoded to zero below.
*
* Notice also that Source LID is hardcoded to the Permissive LID
* (0xFFFF). This is also not a problem because if the Destination
* LID is not the Permissive LID, then the "slr" value in the MLX
* will pull the LID from value in the port.
*/
/*
* Build Global Route Header (GRH)
* This is only built if necessary as defined by the "grh" bit in
* the address vector. Note: We also calculate the offset to the
* next header (BTH) based on whether or not the "grh" bit is set.
*/
/*
* If the request is for QP0, then return an error. The
* combination of global routine (GRH) and QP0 is not allowed.
*/
return (IBT_AH_HDL_INVALID);
}
} else {
}
/*
* Build Base Transport Header (BTH)
* Notice that the M, PadCnt, and TVer fields are all set
* to zero implicitly. This is true for all Management Datagrams
* MADs whether GSI are SMI.
*/
/*
* Build Datagram Extended Transport Header (DETH)
*/
/* Ensure that the Data Segment is aligned on a 16-byte boundary */
num_ds = 0;
/*
* Now fill in the Data Segments (SGL) for the MLX WQE based on the
* values set up above (i.e. "sgl", "nds", and the "ds" pointer
* Start by checking for a valid number of SGL entries
*/
return (IBT_QP_SGL_LEN_INVALID);
}
/*
* For each SGL in the Send Work Request, fill in the MLX WQE's data
* segments. Note: We skip any SGL with zero size because Hermon
* hardware cannot handle a zero for "byte_cnt" in the WQE. Actually
* the encoding for zero means a 2GB transfer. Because of this special
* encoding in the hardware, we mask the requested length with
* HERMON_WQE_SGL_BYTE_CNT_MASK (so that 2GB will end up encoded as
* zero.)
*/
offset = 0;
for (i = 0; i < nds; i++) {
continue;
}
/*
* Fill in the Data Segment(s) for the MLX send WQE, using
* the information contained in the scatter-gather list of
* the work request.
*/
/*
* Search through the contents of all MADs posted to QP0 to
* initialize pointers to the places where Directed Route "hop
* pointer", "hop count", and "mgmtclass" would be. Hermon
* needs these updated (i.e. incremented or decremented, as
* necessary) by software.
*/
}
num_ds++;
}
/*
* Hermon's Directed Route MADs need to have the "hop pointer"
* incremented/decremented (as necessary) depending on whether it is
* currently less than or greater than the "hop count" (i.e. whether
* the MAD is a request or a response.)
*/
}
/*
* Now fill in the ICRC Data Segment. This data segment is inlined
* just like the packets headers above, but it is only four bytes and
* set to zero (to indicate that we wish the hardware to generate ICRC.
*/
num_ds++;
/*
* Return the size of descriptor (in 16-byte chunks)
* For Hermon, we want them (for now) to be on stride size
*/
return (DDI_SUCCESS);
}
/*
* hermon_wqe_recv_build()
* Context: Can be called from interrupt or base context.
*/
/* ARGSUSED */
static int
{
int i, num_ds;
/*
* Fill in the Data Segments (SGL) for the Recv WQE - don't
* need to have a reserved for the ctrl, there is none on the
* recv queue for hermon, but will need to put an invalid
* (null) scatter pointer per PRM
*/
num_ds = 0;
/* Check for valid number of SGL entries */
return (IBT_QP_SGL_LEN_INVALID);
}
/*
* For each SGL in the Recv Work Request, fill in the Recv WQE's data
* segments. Note: We skip any SGL with zero size because Hermon
* hardware cannot handle a zero for "byte_cnt" in the WQE. Actually
* the encoding for zero means a 2GB transfer. Because of this special
* encoding in the hardware, we mask the requested length with
* HERMON_WQE_SGL_BYTE_CNT_MASK (so that 2GB will end up encoded as
* zero.)
*/
continue;
}
/*
* Fill in the Data Segment(s) for the receive WQE, using the
* information contained in the scatter-gather list of the
* work request.
*/
num_ds++;
}
/* put the null sgl pointer as well if needed */
}
return (DDI_SUCCESS);
}
/*
* hermon_wqe_srq_build()
* Context: Can be called from interrupt or base context.
*/
/* ARGSUSED */
static int
{
int i, num_ds;
/* Fill in the Data Segments (SGL) for the Recv WQE */
sizeof (hermon_hw_srq_wqe_next_t));
num_ds = 0;
/* Check for valid number of SGL entries */
return (IBT_QP_SGL_LEN_INVALID);
}
/*
* For each SGL in the Recv Work Request, fill in the Recv WQE's data
* segments. Note: We skip any SGL with zero size because Hermon
* hardware cannot handle a zero for "byte_cnt" in the WQE. Actually
* the encoding for zero means a 2GB transfer. Because of this special
* encoding in the hardware, we mask the requested length with
* HERMON_WQE_SGL_BYTE_CNT_MASK (so that 2GB will end up encoded as
* zero.)
*/
continue;
}
/*
* Fill in the Data Segment(s) for the receive WQE, using the
* information contained in the scatter-gather list of the
* work request.
*/
num_ds++;
}
/*
* put in the null sgl pointer as well, if needed
*/
}
return (DDI_SUCCESS);
}
/*
* hermon_wr_get_immediate()
* Context: Can be called from interrupt or base context.
*/
static uint32_t
{
/*
* This routine extracts the "immediate data" from the appropriate
* location in the IBTF work request. Because of the way the
* work request structure is defined, the location for this data
* depends on the actual work request operation type.
*/
/* For RDMA Write, test if RC or UC */
} else { /* IBT_UC_SRV */
}
}
/* For Send, test if RC, UD, or UC */
} else { /* IBT_UC_SRV */
}
}
/*
* If any other type of request, then immediate is undefined
*/
return (0);
}
/*
* hermon_wqe_headroom()
* Context: can be called from interrupt or base, currently only from
* base context.
* Routine that fills in the headroom for the Send Queue
*/
static void
{
int i, j;
for (i = 0; i < hdrmwqes; i++) {
for (j = 0; j < sectperwqe; j++) {
if (j == 0) { /* 1st section of wqe */
/* perserve ownership bit */
wqe_start) | 0x7FFFFFFF;
} else {
/* or just invalidate it */
invalue = 0xFFFFFFFF;
}
}
}
}
/*
* hermon_wqe_sync()
* Context: Can be called from interrupt or base context.
*/
static void
{
int status;
if (sync_type == HERMON_WR_SRQ) {
/* Get the DMA handle from SRQ context */
/* get base addr of the buffer */
} else {
/* Get the DMA handle from QP context */
/* Determine the base address of the QP buffer */
if (qp->qp_sq_baseaddr == 0) {
} else {
}
}
/*
* Depending on the type of the work queue, we grab information
* about the address ranges we need to DMA sync.
*/
if (sync_type == HERMON_WR_SEND) {
} else if (sync_type == HERMON_WR_RECV) {
} else {
}
/*
* There are two possible cases for the beginning and end of the WQE
* chain we are trying to sync. Either this is the simple case, where
* the end of the chain is below the beginning of the chain, or it is
* the "wrap-around" case, where the end of the chain has wrapped over
* the end of the queue. In the former case, we simply need to
* calculate the span from beginning to end and sync it. In the latter
* case, however, we need to calculate the span from the top of the
* work queue to the end of the chain and sync that, and then we need
* to find the other portion (from beginning of chain to end of queue)
* and sync that as well. Note: if the "top to end" span is actually
* zero length, then we don't do a DMA sync because a zero length DMA
* sync unnecessarily syncs the entire work queue.
*/
/* "From Beginning to End" */
if (status != DDI_SUCCESS) {
return;
}
} else {
/* "From Top to End" */
if (length) {
if (status != DDI_SUCCESS) {
return;
}
}
/* "From Beginning to Bottom" */
if (status != DDI_SUCCESS) {
return;
}
}
}
/*
* hermon_wr_bind_check()
* Context: Can be called from interrupt or base context.
*/
/* ARGSUSED */
static int
{
/* Check for a valid Memory Window handle in the WR */
return (IBT_MW_HDL_INVALID);
}
/* Check for a valid Memory Region handle in the WR */
return (IBT_MR_HDL_INVALID);
}
/*
* Check here to see if the memory region has already been partially
* deregistered as a result of a hermon_umap_umemlock_cb() callback.
* If so, this is an error, return failure.
*/
return (IBT_MR_HDL_INVALID);
}
/* Check for a valid Memory Window RKey (i.e. a matching RKey) */
return (IBT_MR_RKEY_INVALID);
}
/* Check for a valid Memory Region LKey (i.e. a matching LKey) */
return (IBT_MR_LKEY_INVALID);
}
/*
* Now check for valid "vaddr" and "len". Note: We don't check the
* "vaddr" range when "len == 0" (i.e. on unbind operations)
*/
if (len != 0) {
return (IBT_MR_VA_INVALID);
}
if (vaddr > reg_end_addr) {
return (IBT_MR_LEN_INVALID);
}
}
/*
* Validate the bind access flags. Remote Write and Atomic access for
* the Memory Window require that Local Write access be set in the
* corresponding Memory Region.
*/
if (((bind_flags & IBT_WR_BIND_WRITE) ||
(bind_flags & IBT_WR_BIND_ATOMIC)) &&
return (IBT_MR_ACCESS_REQ_INVALID);
}
/* Calculate the new RKey for the Memory Window */
return (DDI_SUCCESS);
}
/*
* hermon_wrid_from_reset_handling()
* Context: Can be called from interrupt or base context.
*/
/* ARGSUSED */
int
{
if (qp->qp_is_umap)
return (DDI_SUCCESS);
/* grab the cq lock(s) to modify the wqavl tree */
#ifdef __lock_lint
#else
#endif
/* Chain the newly allocated work queue header to the CQ's list */
/*
* Now we repeat all the above operations for the receive work queue,
* or shared receive work queue.
*
* Note: We still use the 'qp_rq_cqhdl' even in the SRQ case.
*/
#ifdef __lock_lint
#else
if (qp_srq_en == HERMON_QP_SRQ_ENABLED) {
} else {
qp->qp_rq_wqecntr = 0;
}
#endif
#ifdef __lock_lint
#else
if (qp_srq_en == HERMON_QP_SRQ_ENABLED) {
}
#endif
#ifdef __lock_lint
#else
#endif
return (DDI_SUCCESS);
}
/*
* hermon_wrid_to_reset_handling()
* Context: Can be called from interrupt or base context.
*/
int
{
if (qp->qp_is_umap)
return (DDI_SUCCESS);
/*
* If there are unpolled entries in these CQs, they are
* Grab the CQ lock(s) before manipulating the lists.
*/
#ifdef __lock_lint
#else
#endif
#ifdef __lock_lint
#else
if (qp_srq_en == HERMON_QP_SRQ_ENABLED) {
}
#endif
/*
* Flush the entries on the CQ for this QP's QPN.
*/
#ifdef __lock_lint
#else
if (qp_srq_en == HERMON_QP_SRQ_ENABLED) {
}
#endif
#ifdef __lock_lint
#else
#endif
return (IBT_SUCCESS);
}
/*
* hermon_wrid_get_entry()
* Context: Can be called from interrupt or base context.
*/
{
/*
* Determine whether this CQE is a send or receive completion.
*/
/* Find the work queue for this QP number (send or receive side) */
/*
* Regardless of whether the completion is the result of a "success"
* or a "failure", we lock the list of "containers" and attempt to
* search for the the first matching completion (i.e. the first WR
* with a matching WQE addr and size). Once we find it, we pull out
* the "wrid" field and return it (see below). XXX Note: One possible
* future enhancement would be to enable this routine to skip over
* any "unsignaled" completions to go directly to the next "signaled"
* entry on success.
*/
if (wqa->wqa_srq_en) {
struct hermon_sw_srq_s *srq;
/* put wqe back on the srq free list */
} else {
}
return (wrid);
}
int
{
return (-1);
return (+1);
return (-1);
return (+1);
else
return (0);
}
/*
* hermon_wrid_workq_find()
* Context: Can be called from interrupt or base context.
*/
static hermon_workq_avl_t *
{
/*
* Walk the CQ's work queue list, trying to find a send or recv queue
* with the same QP number. We do this even if we are going to later
* create a new entry because it helps us easily find the end of the
* list.
*/
#ifdef __lock_lint
#endif
return (curr);
}
/*
* hermon_wrid_wqhdr_create()
* Context: Can be called from base context.
*/
/* ARGSUSED */
{
/*
* Allocate space for the wqhdr, and an array to record all the wrids.
*/
return (NULL);
}
return (NULL);
}
return (wqhdr);
}
void
{
}
/*
* hermon_cq_workq_add()
* Context: Can be called from interrupt or base context.
*/
static void
{
#ifdef __lock_lint
#endif
}
/*
* hermon_cq_workq_remove()
* Context: Can be called from interrupt or base context.
*/
static void
{
#ifdef __lock_lint
#endif
}