tavor_wr.c revision 9e39c5ba00a55fa05777cc94b148296af305e135
/*
* CDDL HEADER START
*
* The contents of this file are subject to the terms of the
* Common Development and Distribution License (the "License").
* You may not use this file except in compliance with the License.
*
* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
* See the License for the specific language governing permissions
* and limitations under the License.
*
* When distributing Covered Code, include this CDDL HEADER in each
* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
* If applicable, add the following below this CDDL HEADER, with the
* fields enclosed by brackets "[]" replaced with your own identifying
* information: Portions Copyright [yyyy] [name of copyright owner]
*
* CDDL HEADER END
*/
/*
* Copyright 2006 Sun Microsystems, Inc. All rights reserved.
* Use is subject to license terms.
*/
/*
* Tavor Work Request Processing Routines
*
* Implements all the routines necessary to provide the PostSend(),
* PostRecv() and PostSRQ() verbs. Also contains all the code
* necessary to implement the Tavor WRID tracking mechanism.
*/
#pragma inline(tavor_qp_send_doorbell)
#pragma inline(tavor_qp_recv_doorbell)
/*
* tavor_post_send()
* Context: Can be called from interrupt or base context.
*/
int
{
int status;
/*
* Check for user-mappable QP memory. Note: We do not allow kernel
* clients to post to QP memory that is accessible directly by the
* user. If the QP memory is user accessible, then return an error.
*/
if (qp->qp_is_umap) {
TAVOR_TNF_ERROR, "");
return (IBT_QP_HDL_INVALID);
}
/* Initialize posted_cnt */
posted_cnt = 0;
/*
* Check QP state. Can not post Send requests from the "Reset",
* "Init", or "RTR" states
*/
TAVOR_TNF_ERROR, "");
return (IBT_QP_STATE_INVALID);
}
/* Grab the lock for the WRID list */
/* Save away some initial QP state */
/*
* For each ibt_send_wr_t in the wr[] list passed in, parse the
* request and build a Send WQE. Note: Because we are potentially
* building a chain of WQEs, we want to link them all together.
* However, we do not want to link the first one to the previous
* WQE until the entire chain has been linked. Then in the last
* step we ring the appropriate doorbell. Note: It is possible for
* more Work Requests to be posted than the HW will support at one
* shot. If this happens, we need to be able to post and ring
* several chains here until the the entire request is complete.
*/
wrindx = 0;
/*
* For the first WQE on a new chain we need "prev" to point
* to the current descriptor. As we begin to process
* further, "prev" will be updated to point to the previous
* WQE on the current chain (see below).
*/
/*
* Before we begin, save the current "tail index" for later
* DMA sync
*/
/*
* Break the request up into chains that are less than or
* equal to the maximum number of WQEs that can be posted
* per doorbell ring
*/
chainbegin = wrindx;
/*
* Check for "queue full" condition. If the queue
* is already full, then no more WQEs can be posted.
* So break out, ring a doorbell (if necessary) and
* return an error
*/
TAVOR_TNF_TRACE, "");
break;
}
/*
* Increment the "tail index" and check for "queue
* full" condition. If we detect that the current
* work request is going to fill the work queue, then
* we mark this condition and continue.
*/
}
/*
* Get the address of the location where the next
* Send WQE should be built
*/
/*
* Call tavor_wqe_send_build() to build the WQE
* at the given address. This routine uses the
* information in the ibt_send_wr_t list (wr[]) and
* returns the size of the WQE when it returns.
*/
if (status != DDI_SUCCESS) {
TAVOR_TNF_ERROR, "");
break;
}
/*
* Add a WRID entry to the WRID list. Need to
* calculate the "wqeaddrsz" and "signaled_dbd"
* values to pass to tavor_wrid_add_entry()
*/
desc_sz);
} else {
signaled_dbd = 0;
}
/*
* If this is not the first descriptor on the current
* chain, then link it to the previous WQE. Otherwise,
* save the address and size of this descriptor (in
* "first" and "first_sz" respectively) and continue.
* Note: Linking a WQE to the the previous one will
* depend on whether the two WQEs are from "special
* QPs" (i.e. MLX transport WQEs) or whether they are
* normal Send WQEs.
*/
if (currindx != 0) {
if (qp->qp_is_special) {
} else {
}
} else {
}
/*
* Update the current "tail index" and increment
* "posted_cnt"
*/
posted_cnt++;
}
/*
* If we reach here and there are one or more WQEs which have
* been successfully chained together, then we need to link
* the current chain to the previously executing chain of
* descriptor (if there is one) and ring the doorbell for the
* send work queue.
*/
if (currindx != 0) {
/*
* Before we link the chain, we need to ensure that the
* "next" field on the last WQE is set to NULL (to
* indicate the end of the chain). Note: Just as it
* did above, the format for the "next" fields in a
* given WQE depend on whether the WQE is MLX
* transport or not.
*/
if (qp->qp_is_special) {
} else {
}
/* Save away updated "tail index" for the DMA sync */
/* Do a DMA sync for current send WQE(s) */
/*
* Now link the chain to the old chain (if there was
* one. Note: still need to pay attention to whether
* the QP used MLX transport WQEs or not.
*/
if (qp->qp_is_special) {
} else {
}
/*
* If there was a valid previous WQE (i.e. non-NULL),
* then sync it too. This is because we have updated
* its "next" fields and we want to ensure that the
* hardware can see the changes.
*/
}
/*
* Now if the WRID tail entry is non-NULL, then this
* represents the entry to which we are chaining the
* new entries. Since we are going to ring the
* doorbell for this WQE, we want set its "dbd" bit.
*
* On the other hand, if the tail is NULL, even though
* we will have rung the doorbell for the previous WQE
* (for the hardware's sake) it is irrelevant to our
* purposes (for tracking WRIDs) because we know the
* request must have already completed.
*/
}
/* Update some of the state in the QP */
/* Ring the doorbell */
}
}
/*
* Update the "num_posted" return value (if necessary). Then drop
* the locks and return success.
*/
if (num_posted != NULL) {
*num_posted = posted_cnt;
}
return (status);
}
/*
* tavor_post_recv()
* Context: Can be called from interrupt or base context.
*/
int
{
int status;
/*
* Check for user-mappable QP memory. Note: We do not allow kernel
* clients to post to QP memory that is accessible directly by the
* user. If the QP memory is user accessible, then return an error.
*/
if (qp->qp_is_umap) {
TAVOR_TNF_ERROR, "");
return (IBT_QP_HDL_INVALID);
}
/* Initialize posted_cnt */
posted_cnt = 0;
/*
* Check if QP is associated with an SRQ
*/
TAVOR_TNF_ERROR, "");
return (IBT_SRQ_IN_USE);
}
/*
* Check QP state. Can not post Recv requests from the "Reset" state
*/
TAVOR_TNF_ERROR, "");
return (IBT_QP_STATE_INVALID);
}
/* Grab the lock for the WRID list */
/* Save away some initial QP state */
/*
* For each ibt_recv_wr_t in the wr[] list passed in, parse the
* request and build a Recv WQE. Note: Because we are potentially
* building a chain of WQEs, we want to link them all together.
* However, we do not want to link the first one to the previous
* WQE until the entire chain has been linked. Then in the last
* step we ring the appropriate doorbell. Note: It is possible for
* more Work Requests to be posted than the HW will support at one
* shot. If this happens, we need to be able to post and ring
* several chains here until the the entire request is complete.
*/
wrindx = 0;
/*
* For the first WQE on a new chain we need "prev" to point
* to the current descriptor. As we begin to process
* further, "prev" will be updated to point to the previous
* WQE on the current chain (see below).
*/
/*
* Before we begin, save the current "tail index" for later
* DMA sync
*/
/*
* Break the request up into chains that are less than or
* equal to the maximum number of WQEs that can be posted
* per doorbell ring
*/
/*
* Check for "queue full" condition. If the queue
* is already full, then no more WQEs can be posted.
* So break out, ring a doorbell (if necessary) and
* return an error
*/
TAVOR_TNF_TRACE, "");
break;
}
/*
* Increment the "tail index" and check for "queue
* full" condition. If we detect that the current
* work request is going to fill the work queue, then
* we mark this condition and continue.
*/
}
/*
* Get the address of the location where the next
* Recv WQE should be built
*/
/*
* Call tavor_wqe_recv_build() to build the WQE
* at the given address. This routine uses the
* information in the ibt_recv_wr_t list (wr[]) and
* returns the size of the WQE when it returns.
*/
if (status != DDI_SUCCESS) {
TAVOR_TNF_ERROR, "");
break;
}
/*
* Add a WRID entry to the WRID list. Need to
* calculate the "wqeaddrsz" and "signaled_dbd"
* values to pass to tavor_wrid_add_entry(). Note:
* all Recv WQEs are essentially "signaled"
*/
desc_sz);
/*
* If this is not the first descriptor on the current
* chain, then link it to the previous WQE. Otherwise,
* save the address and size of this descriptor (in
* "first" and "first_sz" respectively) and continue.
*/
if (currindx != 0) {
qp);
} else {
}
/*
* Update the current "tail index" and increment
* "posted_cnt"
*/
posted_cnt++;
}
/*
* If we reach here and there are one or more WQEs which have
* been successfully chained together, then we need to link
* the current chain to the previously executing chain of
* descriptor (if there is one) and ring the doorbell for the
* recv work queue.
*/
if (currindx != 0) {
/*
* Before we link the chain, we need to ensure that the
* "next" field on the last WQE is set to NULL (to
* indicate the end of the chain).
*/
/* Save away updated "tail index" for the DMA sync */
/* Do a DMA sync for current recv WQE(s) */
/*
* Now link the chain to the old chain (if there was
* one.
*/
/*
* If there was a valid previous WQE (i.e. non-NULL),
* then sync it too. This is because we have updated
* its "next" fields and we want to ensure that the
* hardware can see the changes.
*/
}
/*
* Now if the WRID tail entry is non-NULL, then this
* represents the entry to which we are chaining the
* new entries. Since we are going to ring the
* doorbell for this WQE, we want set its "dbd" bit.
*
* On the other hand, if the tail is NULL, even though
* we will have rung the doorbell for the previous WQE
* (for the hardware's sake) it is irrelevant to our
* purposes (for tracking WRIDs) because we know the
* request must have already completed.
*/
}
/* Update some of the state in the QP */
/* Ring the doorbell */
}
}
/*
* Update the "num_posted" return value (if necessary). Then drop
* the locks and return success.
*/
if (num_posted != NULL) {
*num_posted = posted_cnt;
}
return (status);
}
/*
* tavor_post_srq()
* Context: Can be called from interrupt or base context.
*/
int
{
int status;
/*
* Check for user-mappable QP memory. Note: We do not allow kernel
* clients to post to QP memory that is accessible directly by the
* user. If the QP memory is user accessible, then return an error.
*/
if (srq->srq_is_umap) {
TAVOR_TNF_ERROR, "");
return (IBT_SRQ_HDL_INVALID);
}
/* Initialize posted_cnt */
posted_cnt = 0;
/*
* Check SRQ state. Can not post Recv requests when SRQ is in error
*/
TAVOR_TNF_ERROR, "");
return (IBT_QP_STATE_INVALID);
}
/* Grab the lock for the WRID list */
/*
* For each ibt_recv_wr_t in the wr[] list passed in, parse the
* request and build a Recv WQE. Note: Because we are potentially
* building a chain of WQEs, we want to link them all together.
* However, we do not want to link the first one to the previous
* WQE until the entire chain has been linked. Then in the last
* step we ring the appropriate doorbell. Note: It is possible for
* more Work Requests to be posted than the HW will support at one
* shot. If this happens, we need to be able to post and ring
* several chains here until the the entire request is complete.
*/
wrindx = 0;
/*
* For the first WQE on a new chain we need "prev" to point
* to the current descriptor. As we begin to process
* further, "prev" will be updated to point to the previous
* WQE on the current chain (see below).
*/
} else {
}
/*
* Break the request up into chains that are less than or
* equal to the maximum number of WQEs that can be posted
* per doorbell ring
*/
/*
* Check for "queue full" condition. If the queue
* is already full, then no more WQEs can be posted.
* So break out, ring a doorbell (if necessary) and
* return an error
*/
TAVOR_TNF_TRACE, "");
break;
}
/*
* Get the address of the location where the next
* Recv WQE should be built
*/
/*
* Add a WRID entry to the WRID list. Need to
* set the "signaled_dbd" values to pass to
* tavor_wrid_add_entry(). Note: all Recv WQEs are
* essentially "signaled"
*
* The 'size' is stored at srq_alloc time, in the
* srq_wq_stride. This is a constant value required
* for SRQ.
*/
/*
* Call tavor_wqe_srq_build() to build the WQE
* at the given address. This routine uses the
* information in the ibt_recv_wr_t list (wr[]) and
* returns the size of the WQE when it returns.
*/
desc);
if (status != DDI_SUCCESS) {
TAVOR_TNF_ERROR, "");
break;
}
/*
* If this is not the first descriptor on the current
* chain, then link it to the previous WQE. Otherwise,
* save the address of this descriptor (in "first") and
* continue.
*/
if (currindx != 0) {
/* Do a DMA sync for previous recv WQE */
} else {
/*
* In this case, the last WQE on the chain is
* also considered 'first'. So set prev to
* first, here.
*/
}
/*
* Increment "posted_cnt"
*/
posted_cnt++;
}
/*
* If we reach here and there are one or more WQEs which have
* been successfully chained together, then we need to link
* the current chain to the previously executing chain of
* descriptor (if there is one) and ring the doorbell for the
* recv work queue.
*/
if (currindx != 0) {
/*
* Before we link the chain, we need to ensure that the
* "next" field on the last WQE is set to NULL (to
* indicate the end of the chain).
*/
/* Do a DMA sync for current recv WQE */
/*
* Now link the chain to the old chain (if there was
* one).
*/
} else {
}
/*
* If there was a valid previous WQE (i.e. valid index),
* then sync it too. This is because we have updated
* its "next" fields and we want to ensure that the
* hardware can see the changes.
*/
}
/* Update some of the state in the QP */
/* Ring the doorbell */
/* SRQ needs NDS of 0 */
}
}
/*
* Update the "num_posted" return value (if necessary). Then drop
* the locks and return success.
*/
if (num_posted != NULL) {
*num_posted = posted_cnt;
}
return (status);
}
/*
* tavor_qp_send_doorbell()
* Context: Can be called from interrupt or base context.
*/
static void
{
/* Build the doorbell from the parameters */
/* Write the doorbell to UAR */
doorbell);
}
/*
* tavor_qp_recv_doorbell()
* Context: Can be called from interrupt or base context.
*/
static void
{
/* Build the doorbell from the parameters */
/* Write the doorbell to UAR */
doorbell);
}
/*
* tavor_wqe_send_build()
* Context: Can be called from interrupt or base context.
*/
static int
{
/* Initialize the information for the Data Segments */
sizeof (tavor_hw_snd_wqe_nextctrl_t));
num_ds = 0;
/*
* Build a Send WQE depends first and foremost on the transport
* type of Work Request (i.e. UD, RC, or UC)
*/
case IBT_UD_SRV:
/* Ensure that work request transport type matches QP type */
TAVOR_TNF_ERROR, "");
return (IBT_QP_SRV_TYPE_INVALID);
}
/*
* Validate the operation type. For UD requests, only the
* "Send" operation is valid
*/
TAVOR_TNF_ERROR, "");
return (IBT_QP_OP_TYPE_INVALID);
}
/*
* If this is a Special QP (QP0 or QP1), then we need to
* build MLX WQEs instead. So jump to tavor_wqe_mlx_build()
* and return whatever status it returns
*/
if (qp->qp_is_special) {
return (status);
}
/*
* Otherwise, if this is a normal UD Send request, then fill
* all the fields in the Tavor UD header for the WQE. Note:
* to do this we'll need to extract some information from the
* Address Handle passed with the work request.
*/
sizeof (tavor_hw_snd_wqe_nextctrl_t));
TAVOR_TNF_ERROR, "");
return (IBT_AH_HDL_INVALID);
}
/*
* Build the Unreliable Datagram Segment for the WQE, using
* the information from the address handle and the work
* request.
*/
/* Update "ds" for filling in Data Segments (below) */
sizeof (tavor_hw_snd_wqe_ud_t));
break;
case IBT_RC_SRV:
/* Ensure that work request transport type matches QP type */
TAVOR_TNF_ERROR, "");
return (IBT_QP_SRV_TYPE_INVALID);
}
/*
* Validate the operation type. For RC requests, we allow
* "Send", "RDMA Read", "RDMA Write", various "Atomic"
* operations, and memory window "Bind"
*/
TAVOR_TNF_ERROR, "");
return (IBT_QP_OP_TYPE_INVALID);
}
/*
* If this is a Send request, then all we need to do is break
* out and here and begin the Data Segment processing below
*/
break;
}
/*
* If this is an RDMA Read or RDMA Write request, then fill
* in the "Remote Address" header fields.
*/
sizeof (tavor_hw_snd_wqe_nextctrl_t));
/*
* Build the Remote Address Segment for the WQE, using
* the information from the RC work request.
*/
/* Update "ds" for filling in Data Segments (below) */
sizeof (tavor_hw_snd_wqe_remaddr_t));
break;
}
/*
* If this is one of the Atomic type operations (i.e
* Compare-Swap or Fetch-Add), then fill in both the "Remote
* Address" header fields and the "Atomic" header fields.
*/
sizeof (tavor_hw_snd_wqe_nextctrl_t));
sizeof (tavor_hw_snd_wqe_remaddr_t));
/*
* Build the Remote Address and Atomic Segments for
* the WQE, using the information from the RC Atomic
* work request.
*/
/* Update "ds" for filling in Data Segments (below) */
sizeof (tavor_hw_snd_wqe_atomic_t));
/*
* Update "nds" and "sgl" because Atomic requests have
* only a single Data Segment (and they are encoded
* somewhat differently in the work request.
*/
nds = 1;
break;
}
/*
* If this is memory window Bind operation, then we call the
* tavor_wr_bind_check() routine to validate the request and
* to generate the updated RKey. If this is successful, then
* we fill in the WQE's "Bind" header fields.
*/
if (status != DDI_SUCCESS) {
TAVOR_TNF_ERROR, "");
return (status);
}
sizeof (tavor_hw_snd_wqe_nextctrl_t));
/*
* Build the Bind Memory Window Segments for the WQE,
* using the information from the RC Bind memory
* window work request.
*/
/*
* Update the "ds" pointer. Even though the "bind"
* operation requires no SGLs, this is necessary to
* facilitate the correct descriptor size calculations
* (below).
*/
sizeof (tavor_hw_snd_wqe_bind_t));
nds = 0;
}
break;
case IBT_UC_SRV:
/* Ensure that work request transport type matches QP type */
TAVOR_TNF_ERROR, "");
return (IBT_QP_SRV_TYPE_INVALID);
}
/*
* Validate the operation type. For UC requests, we only
* allow "Send", "RDMA Write", and memory window "Bind".
* Note: Unlike RC, UC does not allow "RDMA Read" or "Atomic"
* operations
*/
TAVOR_TNF_ERROR, "");
return (IBT_QP_OP_TYPE_INVALID);
}
/*
* If this is a Send request, then all we need to do is break
* out and here and begin the Data Segment processing below
*/
break;
}
/*
* If this is an RDMA Write request, then fill in the "Remote
* Address" header fields.
*/
sizeof (tavor_hw_snd_wqe_nextctrl_t));
/*
* Build the Remote Address Segment for the WQE, using
* the information from the UC work request.
*/
/* Update "ds" for filling in Data Segments (below) */
sizeof (tavor_hw_snd_wqe_remaddr_t));
break;
}
/*
* If this is memory window Bind operation, then we call the
* tavor_wr_bind_check() routine to validate the request and
* to generate the updated RKey. If this is successful, then
* we fill in the WQE's "Bind" header fields.
*/
if (status != DDI_SUCCESS) {
TAVOR_TNF_ERROR, "");
return (status);
}
sizeof (tavor_hw_snd_wqe_nextctrl_t));
/*
* Build the Bind Memory Window Segments for the WQE,
* using the information from the UC Bind memory
* window work request.
*/
/*
* Update the "ds" pointer. Even though the "bind"
* operation requires no SGLs, this is necessary to
* facilitate the correct descriptor size calculations
* (below).
*/
sizeof (tavor_hw_snd_wqe_bind_t));
nds = 0;
}
break;
default:
TAVOR_TNF_ERROR, "");
return (IBT_QP_SRV_TYPE_INVALID);
}
/*
* Now fill in the Data Segments (SGL) for the Send WQE based on
* the values setup above (i.e. "sgl", "nds", and the "ds" pointer
* Start by checking for a valid number of SGL entries
*/
TAVOR_TNF_ERROR, "");
return (IBT_QP_SGL_LEN_INVALID);
}
/*
* For each SGL in the Send Work Request, fill in the Send WQE's data
* segments. Note: We skip any SGL with zero size because Tavor
* hardware cannot handle a zero for "byte_cnt" in the WQE. Actually
* the encoding for zero means a 2GB transfer. Because of this special
* encoding in the hardware, we mask the requested length with
* TAVOR_WQE_SGL_BYTE_CNT_MASK (so that 2GB will end up encoded as
* zero.)
*/
for (i = 0; i < nds; i++) {
continue;
}
/*
* Fill in the Data Segment(s) for the current WQE, using the
* information contained in the scatter-gather list of the
* work request.
*/
num_ds++;
}
/* Return the size of descriptor (in 16-byte chunks) */
return (DDI_SUCCESS);
}
/*
* tavor_wqe_send_linknext()
* Context: Can be called from interrupt or base context.
*/
static void
{
/*
* Calculate the "next" field of the descriptor. This amounts to
* setting up the "next_wqe_addr", "nopcode", "fence", and "nds"
* fields (see tavor_hw.h for more). Note: If there is no next
* descriptor (i.e. if the current descriptor is the last WQE on
* the chain), then set "next" to zero.
*/
/*
* Determine the value for the Tavor WQE "nopcode" field
* by using the IBTF opcode from the work request
*/
case IBT_WRC_RDMAW:
} else {
}
break;
case IBT_WRC_SEND:
} else {
}
break;
case IBT_WRC_RDMAR:
break;
case IBT_WRC_CSWAP:
break;
case IBT_WRC_FADD:
break;
case IBT_WRC_BIND:
break;
}
- qp->qp_desc_off);
TAVOR_WQE_NDA_MASK) << 32;
if (fence) {
}
/*
* If a send queue doorbell will be rung for the next
* WQE on the chain, then set the current WQE's "dbd" bit.
* Note: We also update the "dbinfo" structure here to pass
* back information about what should (later) be included
* in the send queue doorbell.
*/
if (dbinfo) {
}
} else {
next = 0;
}
/*
* If this WQE is supposed to be linked to the previous descriptor,
* then we need to update not only the previous WQE's "next" fields
* but we must also update this WQE's "ctrl" fields (i.e. the "c", "e",
* "s", "i" and "immediate" fields - see tavor_hw.h for more). Note:
* the "e" bit is always hardcoded to zero.
*/
/*
* If a send queue doorbell will be rung for the next WQE on
* the chain, then update the current WQE's "next" field and
* return.
* Note: We don't want to modify the "ctrl" field here because
* that portion of the previous WQE has already been set
* correctly at some previous point in time.
*/
if (dbinfo) {
return;
}
ctrl = 0;
/* Set the "c" (i.e. "signaled") bit appropriately */
}
/* Set the "s" (i.e. "solicited") bit appropriately */
}
/* Set the "i" bit and the immediate data appropriately */
}
}
}
/*
* tavor_wqe_mlx_build()
* Context: Can be called from interrupt or base context.
*/
static int
{
int i, num_ds;
/* Initialize the information for the Data Segments */
sizeof (tavor_hw_mlx_wqe_nextctrl_t));
/*
* Pull the address handle from the work request and read in
* the contents of the UDAV. This will be used to answer some
* questions about the request.
*/
TAVOR_TNF_ERROR, "");
return (IBT_AH_HDL_INVALID);
}
for (i = 0; i < udav_sz; i++) {
}
/*
* If the request is for QP1 and the destination LID is equal to
* the Permissive LID, then return an error. This combination is
* not allowed
*/
TAVOR_TNF_ERROR, "");
return (IBT_AH_HDL_INVALID);
}
/*
* Calculate the size of the packet headers, including the GRH
* (if necessary)
*/
sizeof (ib_deth_hdr_t);
}
/*
* Begin to build the first "inline" data segment for the packet
* headers. Note: By specifying "inline" we can build the contents
* of the MAD packet headers directly into the work queue (as part
* descriptor). This has the advantage of both speeding things up
* memory for the packet headers.
*/
desc_sz += 4;
/*
* Build Local Route Header (LRH)
* We start here by building the LRH into a temporary location.
* When we have finished we copy the LRH data into the descriptor.
*
* Notice that the VL values are hardcoded. This is not a problem
* because VL15 is decided later based on the value in the MLX
* is otherwise (meaning for QP1) chosen from the SL-to-VL table
* values. This rule does not hold for loopback packets however
* (all of which bypass the SL-to-VL tables) and it is the reason
* that non-QP0 MADs are setup with VL hardcoded to zero below.
*
* Notice also that Source LID is hardcoded to the Permissive LID
* (0xFFFF). This is also not a problem because if the Destination
* LID is not the Permissive LID, then the "slr" value in the MLX
* will pull the LID from value in the port.
*/
/*
* Build Global Route Header (GRH)
* This is only built if necessary as defined by the "grh" bit in
* the address vector. Note: We also calculate the offset to the
* next header (BTH) based on whether or not the "grh" bit is set.
*/
/*
* If the request is for QP0, then return an error. The
* combination of global routine (GRH) and QP0 is not allowed.
*/
TAVOR_TNF_ERROR, "");
return (IBT_AH_HDL_INVALID);
}
} else {
}
/*
* Build Base Transport Header (BTH)
* Notice that the M, PadCnt, and TVer fields are all set
* to zero implicitly. This is true for all Management Datagrams
* MADs whether GSI are SMI.
*/
/*
* Build Datagram Extended Transport Header (DETH)
*/
/* Ensure that the Data Segment is aligned on a 16-byte boundary */
num_ds = 0;
/*
* Now fill in the Data Segments (SGL) for the MLX WQE based on the
* values set up above (i.e. "sgl", "nds", and the "ds" pointer
* Start by checking for a valid number of SGL entries
*/
TAVOR_TNF_ERROR, "");
return (IBT_QP_SGL_LEN_INVALID);
}
/*
* For each SGL in the Send Work Request, fill in the MLX WQE's data
* segments. Note: We skip any SGL with zero size because Tavor
* hardware cannot handle a zero for "byte_cnt" in the WQE. Actually
* the encoding for zero means a 2GB transfer. Because of this special
* encoding in the hardware, we mask the requested length with
* TAVOR_WQE_SGL_BYTE_CNT_MASK (so that 2GB will end up encoded as
* zero.)
*/
offset = 0;
for (i = 0; i < nds; i++) {
continue;
}
/*
* Fill in the Data Segment(s) for the MLX send WQE, using
* the information contained in the scatter-gather list of
* the work request.
*/
/*
* Search through the contents of all MADs posted to QP0 to
* initialize pointers to the places where Directed Route "hop
* pointer", "hop count", and "mgmtclass" would be. Tavor
* needs these updated (i.e. incremented or decremented, as
* necessary) by software.
*/
}
num_ds++;
}
/*
* Tavor's Directed Route MADs need to have the "hop pointer"
* incremented/decremented (as necessary) depending on whether it is
* currently less than or greater than the "hop count" (i.e. whether
* the MAD is a request or a response.)
*/
}
/*
* Now fill in the ICRC Data Segment. This data segment is inlined
* just like the packets headers above, but it is only four bytes and
* set to zero (to indicate that we wish the hardware to generate ICRC.
*/
num_ds++;
/* Return the size of descriptor (in 16-byte chunks) */
return (DDI_SUCCESS);
}
/*
* tavor_wqe_mlx_linknext()
* Context: Can be called from interrupt or base context.
*/
static void
{
int i;
/*
* Calculate the "next" field of the descriptor. This amounts to
* setting up the "next_wqe_addr", "nopcode", and "nds" fields (see
* tavor_hw.h for more). Note: If there is no next descriptor (i.e.
* if the current descriptor is the last WQE on the chain), then set
* "next" to zero.
*/
/*
* The only valid Tavor WQE "nopcode" for MLX transport
* requests is the "Send" code.
*/
TAVOR_WQE_NDA_MASK) << 32;
/*
* If a send queue doorbell will be rung for the next
* WQE on the chain, then set the current WQE's "dbd" bit.
* Note: We also update the "dbinfo" structure here to pass
* back information about what should (later) be included
* in the send queue doorbell.
*/
if (dbinfo) {
}
} else {
next = 0;
}
/*
* If this WQE is supposed to be linked to the previous descriptor,
* then we need to update not only the previous WQE's "next" fields
* but we must also update this WQE's "ctrl" fields (i.e. the "vl15",
* "slr", "max_srate", "sl", "c", "e", "rlid", and "vcrc" fields -
* see tavor_hw.h for more) Note: the "e" bit and "vcrc" fields are
* always hardcoded to zero.
*/
/*
* If a send queue doorbell will be rung for the next WQE on
* the chain, then update the current WQE's "next" field and
* return.
* Note: We don't want to modify the "ctrl" field here because
* that portion of the previous WQE has already been set
* correctly at some previous point in time.
*/
if (dbinfo) {
return;
}
/*
* Pull the address handle from the work request and read in
* the contents of the UDAV. This will be used to answer some
* questions about the request.
*/
for (i = 0; i < udav_sz; i++) {
}
ctrl = 0;
/* Only QP0 uses VL15, otherwise use VL in the packet */
}
/*
* The SLR (Source LID Replace) bit determines whether the
* source LID for an outgoing MLX packet should come from the
* PortInfo (SLR = 0) or should be left as it is in the
* descriptor (SLR = 1). The latter is necessary for packets
* to be sent with the Permissive LID.
*/
}
/* Fill in the max static rate from the address handle */
/* All VL15 (i.e. SMI) traffic is required to use SL 0 */
}
/* Set the "c" (i.e. "signaled") bit appropriately */
}
/* Fill in the destination LID from the address handle */
}
}
/*
* tavor_wqe_recv_build()
* Context: Can be called from interrupt or base context.
*/
/* ARGSUSED */
static int
{
int i, num_ds;
/* Check that work request transport type is valid */
TAVOR_TNF_ERROR, "");
return (IBT_QP_SRV_TYPE_INVALID);
}
/* Fill in the Data Segments (SGL) for the Recv WQE */
sizeof (tavor_hw_rcv_wqe_nextctrl_t));
num_ds = 0;
/* Check for valid number of SGL entries */
TAVOR_TNF_ERROR, "");
return (IBT_QP_SGL_LEN_INVALID);
}
/*
* For each SGL in the Recv Work Request, fill in the Recv WQE's data
* segments. Note: We skip any SGL with zero size because Tavor
* hardware cannot handle a zero for "byte_cnt" in the WQE. Actually
* the encoding for zero means a 2GB transfer. Because of this special
* encoding in the hardware, we mask the requested length with
* TAVOR_WQE_SGL_BYTE_CNT_MASK (so that 2GB will end up encoded as
* zero.)
*/
continue;
}
/*
* Fill in the Data Segment(s) for the receive WQE, using the
* information contained in the scatter-gather list of the
* work request.
*/
num_ds++;
}
/* Return the size of descriptor (in 16-byte chunks) */
return (DDI_SUCCESS);
}
/*
* tavor_wqe_recv_linknext()
* Context: Can be called from interrupt or base context.
*/
static void
{
/*
* Calculate the "next" field of the descriptor. This amounts to
* setting up the "next_wqe_addr", "dbd", and "nds" fields (see
* tavor_hw.h for more). Note: If there is no next descriptor (i.e.
* if the current descriptor is the last WQE on the chain), then set
* "next" field to TAVOR_WQE_DBD_MASK. This is because the Tavor
* hardware requires the "dbd" bit to be set to one for all Recv WQEs.
* In either case, we must add a single bit in the "reserved" field
* (TAVOR_RCV_WQE_NDA0_WA_MASK) following the NDA. This is the
* workaround for a known Tavor errata that can cause Recv WQEs with
* zero in the NDA field to behave improperly.
*/
qp->qp_desc_off);
TAVOR_WQE_NDA_MASK) << 32;
} else {
}
/*
* If this WQE is supposed to be linked to the previous descriptor,
* then we need to update not only the previous WQE's "next" fields
* but we must also update this WQE's "ctrl" fields (i.e. the "c" and
* "e" bits - see tavor_hw.h for more). Note: both the "c" and "e"
* bits are always hardcoded to zero.
*/
}
}
/*
* tavor_wqe_srq_build()
* Context: Can be called from interrupt or base context.
*/
/* ARGSUSED */
static int
{
int i, num_ds;
/* Fill in the Data Segments (SGL) for the Recv WQE */
sizeof (tavor_hw_rcv_wqe_nextctrl_t));
num_ds = 0;
/* Check for valid number of SGL entries */
TAVOR_TNF_ERROR, "");
return (IBT_QP_SGL_LEN_INVALID);
}
/*
* For each SGL in the Recv Work Request, fill in the Recv WQE's data
* segments. Note: We skip any SGL with zero size because Tavor
* hardware cannot handle a zero for "byte_cnt" in the WQE. Actually
* the encoding for zero means a 2GB transfer. Because of this special
* encoding in the hardware, we mask the requested length with
* TAVOR_WQE_SGL_BYTE_CNT_MASK (so that 2GB will end up encoded as
* zero.)
*/
continue;
}
/*
* Fill in the Data Segment(s) for the receive WQE, using the
* information contained in the scatter-gather list of the
* work request.
*/
num_ds++;
}
/*
* For SRQ, if the number of data segments is less than the maximum
* specified at alloc, then we have to fill in a special "key" entry in
* the sgl entry after the last valid one in this post request. We do
* that here.
*/
}
return (DDI_SUCCESS);
}
/*
* tavor_wqe_srq_linknext()
* Context: Can be called from interrupt or base context.
*/
static void
{
/*
* Calculate the "next" field of the descriptor. This amounts to
* setting up the "next_wqe_addr", "dbd", and "nds" fields (see
* tavor_hw.h for more). Note: If there is no next descriptor (i.e.
* if the current descriptor is the last WQE on the chain), then set
* "next" field to TAVOR_WQE_DBD_MASK. This is because the Tavor
* hardware requires the "dbd" bit to be set to one for all Recv WQEs.
* In either case, we must add a single bit in the "reserved" field
* (TAVOR_RCV_WQE_NDA0_WA_MASK) following the NDA. This is the
* workaround for a known Tavor errata that can cause Recv WQEs with
* zero in the NDA field to behave improperly.
*/
srq->srq_desc_off);
TAVOR_WQE_NDA_MASK) << 32;
} else {
}
/*
* If this WQE is supposed to be linked to the previous descriptor,
* then we need to update not only the previous WQE's "next" fields
* but we must also update this WQE's "ctrl" fields (i.e. the "c" and
* "e" bits - see tavor_hw.h for more). Note: both the "c" and "e"
* bits are always hardcoded to zero.
*/
}
}
/*
* tavor_wr_get_immediate()
* Context: Can be called from interrupt or base context.
*/
static uint32_t
{
/*
* This routine extracts the "immediate data" from the appropriate
* location in the IBTF work request. Because of the way the
* work request structure is defined, the location for this data
* depends on the actual work request operation type.
*/
/* For RDMA Write, test if RC or UC */
} else { /* IBT_UC_SRV */
}
}
/* For Send, test if RC, UD, or UC */
} else { /* IBT_UC_SRV */
}
}
/*
* If any other type of request, then immediate is undefined
*/
return (0);
}
/*
* tavor_wqe_sync()
* Context: Can be called from interrupt or base context.
*/
static void
{
int status;
if (sync_type == TAVOR_WR_SRQ) {
/* Get the DMA handle from SRQ context */
} else {
/* Get the DMA handle from QP context */
}
/* Determine if the work queues need to be synced or not */
if (is_sync_req == 0) {
return;
}
/*
* Depending on the type of the work queue, we grab information
* about the address ranges we need to DMA sync.
*/
if (sync_type == TAVOR_WR_SEND) {
} else if (sync_type == TAVOR_WR_RECV) {
} else {
}
/*
* There are two possible cases for the beginning and end of the WQE
* chain we are trying to sync. Either this is the simple case, where
* the end of the chain is below the beginning of the chain, or it is
* the "wrap-around" case, where the end of the chain has wrapped over
* the end of the queue. In the former case, we simply need to
* calculate the span from beginning to end and sync it. In the latter
* case, however, we need to calculate the span from the top of the
* work queue to the end of the chain and sync that, and then we need
* to find the other portion (from beginning of chain to end of queue)
* and sync that as well. Note: if the "top to end" span is actually
* zero length, then we don't do a DMA sync because a zero length DMA
* sync unnecessarily syncs the entire work queue.
*/
/* "From Beginning to End" */
if (status != DDI_SUCCESS) {
return;
}
} else {
/* "From Top to End" */
if (length) {
if (status != DDI_SUCCESS) {
TAVOR_TNF_ERROR, "");
return;
}
}
/* "From Beginning to Bottom" */
if (status != DDI_SUCCESS) {
return;
}
}
}
/*
* tavor_wr_bind_check()
* Context: Can be called from interrupt or base context.
*/
static int
{
/* Check for a valid Memory Window handle in the WR */
TAVOR_TNF_ERROR, "");
return (IBT_MW_HDL_INVALID);
}
/* Check for a valid Memory Region handle in the WR */
TAVOR_TNF_ERROR, "");
return (IBT_MR_HDL_INVALID);
}
/*
* Check here to see if the memory region has already been partially
* deregistered as a result of a tavor_umap_umemlock_cb() callback.
* If so, this is an error, return failure.
*/
TAVOR_TNF_ERROR, "");
return (IBT_MR_HDL_INVALID);
}
/* Check for a valid Memory Window RKey (i.e. a matching RKey) */
TAVOR_TNF_ERROR, "");
return (IBT_MR_RKEY_INVALID);
}
/* Check for a valid Memory Region LKey (i.e. a matching LKey) */
TAVOR_TNF_ERROR, "");
return (IBT_MR_LKEY_INVALID);
}
/*
* Now check for valid "vaddr" and "len". Note: We don't check the
* "vaddr" range when "len == 0" (i.e. on unbind operations)
*/
if (len != 0) {
TAVOR_TNF_ERROR, "");
return (IBT_MR_VA_INVALID);
}
if (vaddr > reg_end_addr) {
TAVOR_TNF_ERROR, "");
return (IBT_MR_LEN_INVALID);
}
}
/*
* Validate the bind access flags. Remote Write and Atomic access for
* the Memory Window require that Local Write access be set in the
* corresponding Memory Region.
*/
if (((bind_flags & IBT_WR_BIND_WRITE) ||
(bind_flags & IBT_WR_BIND_ATOMIC)) &&
TAVOR_TNF_ERROR, "");
return (IBT_MR_ACCESS_REQ_INVALID);
}
/* Calculate the new RKey for the Memory Window */
return (DDI_SUCCESS);
}
/*
* tavor_wrid_from_reset_handling()
* Context: Can be called from interrupt or base context.
*/
int
{
uint_t create_wql = 0;
/*
* For each of this QP's Work Queues, make sure we have a (properly
* initialized) Work Request ID list attached to the relevant
* completion queue. Grab the CQ lock(s) before manipulating the
* lists.
*/
/* Couldn't find matching work queue header, create it */
/*
* header, then drop the lock(s) and return failure.
*/
TAVOR_TNF_ERROR, "");
return (ibc_get_ci_failure(0));
}
}
/*
* Allocate space for the tavor_wrid_entry_t container
*/
if (s_wridlist == NULL) {
/*
* If we couldn't allocate space for tracking the WRID
* entries, then cleanup the workq header from above (if
* necessary, i.e. if we created the workq header). Then
* drop the lock(s) and return failure.
*/
if (create_new_swq) {
}
TAVOR_TNF_ERROR, "");
return (ibc_get_ci_failure(0));
}
/* Chain the new WRID list container to the workq hdr list */
#ifdef __lock_lint
#else
if (qp_srq_en == TAVOR_QP_SRQ_ENABLED) {
}
#endif
/*
* Now we repeat all the above operations for the receive work queue,
* or shared receive work queue.
*
* Note: We still use the 'qp_rq_cqhdl' even in the SRQ case.
*/
/*
* If this QP is associated with an SRQ, and this isn't the
* first QP on the SRQ, then the 'srq_wrid_wql' will already be
* created. Since the WQL is created at 'wqhdr_create' time we
* pass in the flag 'create_wql' here to be 0 if we have
* already created it. And later on below we then next setup
* the WQL and rwq information based off the existing SRQ info.
*/
if (qp_srq_en == TAVOR_QP_SRQ_ENABLED &&
create_wql = 0;
}
/*
* header, then free all the send queue resources we
* just allocated and setup (above), drop the lock(s)
* and return failure.
*/
if (create_new_swq) {
swq);
}
#ifdef __lock_lint
#else
if (qp_srq_en == TAVOR_QP_SRQ_ENABLED) {
}
#endif
TAVOR_TNF_ERROR, "");
return (ibc_get_ci_failure(0));
}
}
/*
* Setup receive workq hdr
*
* If the QP is on an SRQ, we setup the SRQ specific fields, setting
* keeping a copy of the rwq pointer, setting the rwq bufsize
* appropriately, and initializing our part of the WQLock.
*
* In the normal QP case, the QP recv queue bufsize is used.
*/
if (qp_srq_en == TAVOR_QP_SRQ_ENABLED) {
} else {
}
} else {
}
/*
* Allocate space for the tavor_wrid_entry_t container.
*
* If QP is on an SRQ, and the wrq_wridlist is NULL then we must
* allocate the wridlist normally. However, if the srq_wridlist is !=
* NULL, then we know this SRQ has already been initialized, thus the
* wridlist has already been initialized. So we re-use the
* srq_wridlist as the r_wridlist for this QP in this case.
*/
if (qp_srq_en == TAVOR_QP_SRQ_ENABLED &&
/* Use existing srq_wridlist pointer */
} else {
/* Allocate memory for the r_wridlist */
}
/*
* If the memory allocation failed for r_wridlist (or the SRQ pointer
* is mistakenly NULL), we cleanup our previous swq allocation from
* above
*/
if (r_wridlist == NULL) {
/*
* If we couldn't allocate space for tracking the WRID
* entries, then cleanup all the stuff from above. Then
* drop the lock(s) and return failure.
*/
if (create_new_swq) {
}
if (create_new_rwq) {
}
#ifdef __lock_lint
#else
if (qp_srq_en == TAVOR_QP_SRQ_ENABLED) {
}
#endif
TAVOR_TNF_ERROR, "");
return (ibc_get_ci_failure(0));
}
/*
* Initialize the wridlist
*
* In the normal QP case, there is no special initialization needed.
* We simply setup the wridlist backpointer to be the receive wqhdr
* (rwq).
*
* But in the SRQ case, there is no backpointer to the wqhdr possible.
* Instead we set 'wl_srq_en', specifying this wridlist is on an SRQ
* and thus potentially shared across multiple QPs with the SRQ. We
* also setup the srq_wridlist pointer to be the r_wridlist, and
* intialize the freelist to an invalid index. This srq_wridlist
* pointer is used above on future moves from_reset to let us know that
* the srq_wridlist has been initialized already.
*
* And finally, if we are in a non-UMAP case, we setup the srq wrid
* free list.
*/
if (qp_srq_en == TAVOR_QP_SRQ_ENABLED &&
/* Initialize srq wrid free list */
}
} else {
}
/* Chain the WRID list "container" to the workq hdr list */
#ifdef __lock_lint
#else
if (qp_srq_en == TAVOR_QP_SRQ_ENABLED) {
}
#endif
return (DDI_SUCCESS);
}
/*
* tavor_wrid_to_reset_handling()
* Context: Can be called from interrupt or base context.
*/
void
{
uint_t free_wqhdr = 0;
/*
* For each of this QP's Work Queues, move the WRID "container" to
* the "reapable" list. Although there may still be unpolled
* entries in these containers, it is not a big deal. We will not
* reap the list until either the Poll CQ command detects an empty
* condition or the CQ itself is freed. Grab the CQ lock(s) before
* manipulating the lists.
*/
/*
* Add the receive work queue header on to the reaplist. But if we are
* on SRQ, then don't add anything to the reaplist. Instead we flush
* the SRQ entries on the CQ, remove wridlist from WQHDR, and free the
* WQHDR (if needed). We must hold the WQL for these operations, yet
* the call to tavor_cq_wqhdr_remove grabs the WQL internally. So we
* drop WQL before that call. Then release the CQ WQHDR locks and the
* CQ lock and return.
*/
/*
* Pull off all (if any) entries for this QP from CQ. This
* only includes entries that have not yet been polled
*/
/* Remove wridlist from WQHDR */
/* If wridlist chain is now empty, remove the wqhdr as well */
free_wqhdr = 1;
} else {
free_wqhdr = 0;
}
/* Free the WQHDR */
if (free_wqhdr) {
}
} else {
}
}
/*
* tavor_wrid_add_entry()
* Context: Can be called from interrupt or base context.
*/
void
{
/*
* Find the entry in the container pointed to by the "tail" index.
* Add all of the relevant information to that entry, including WRID,
* "wqeaddrsz" parameter, and whether it was signaled/unsignaled
*/
/*
* Update the "wrid_old_tail" pointer to point to the entry we just
* inserted into the queue. By tracking this pointer (the pointer to
* the most recently inserted entry) it will possible later in the
* PostSend() and PostRecv() code paths to find the entry that needs
* tavor_post_send()).
*/
/* Update the tail index */
/*
* If the "tail" index has just wrapped over into the "head" index,
* then we have filled the container. We use the "full" flag to
* indicate this condition and to distinguish it from the "empty"
* condition (where head and tail are also equal).
*/
}
}
/*
* tavor_wrid_add_entry_srq()
* Context: Can be called from interrupt or base context
*/
void
{
/*
* Find the next available WQE from the SRQ free_list. Then update the
* free_list to point to the next entry
*/
/* ASSERT on impossible wqe_index values */
/*
* Setup the WRE.
*
* Given the 'wqe_index' value, we store the WRID at this WRE offset.
* And we set the WRE to be signaled_dbd so that on poll CQ we can find
* this information and associate the WRID to the WQE found on the CQE.
*/
/* Update the free list index */
}
/*
* tavor_wrid_get_entry()
* Context: Can be called from interrupt or base context.
*/
{
/* Lock the list of work queues associated with this CQ */
/*
* Determine whether this CQE is a send or receive completion (and
* whether it was a "successful" completion or not)
*/
if ((opcode == TAVOR_CQE_SEND_ERR_OPCODE) ||
(opcode == TAVOR_CQE_RECV_ERR_OPCODE)) {
error = 1;
} else {
error = 0;
}
/* Find the work queue for this QP number (send or receive side) */
/*
* Regardless of whether the completion is the result of a "success"
* or a "failure", we lock the list of "containers" and attempt to
* search for the the first matching completion (i.e. the first WR
* with a matching WQE addr and size). Once we find it, we pull out
* the "wrid" field and return it (see below). Note: One possible
* future enhancement would be to enable this routine to skip over
* any "unsignaled" completions to go directly to the next "signaled"
* entry on success. XXX
*/
/*
* If this is a "successful" completion, then we assert that this
* completion must be a "signaled" completion.
*/
/*
* If the completion is a "failed" completion, then we save away the
* contents of the entry (into the "wre" field passed in) for use
* in later CQE processing. Note: We use the tavor_wrid_get_wqeaddrsz()
* function to grab "wqeaddrsz" from the next entry in the container.
* This is required for error processing (where updating these fields
* properly is necessary to correct handling of the "error" CQE)
*/
}
/* Pull out the WRID and return it */
return (wrid);
}
/*
* tavor_wrid_find_match()
* Context: Can be called from interrupt or base context.
*/
static tavor_wrid_entry_t *
{
int found = 0, last_container;
/* Pull the "wqeaddrsz" information from the CQE */
/*
* Walk the "containers" list(s), find first WR with a matching WQE
* addr. If the current "container" is not the last one on the list,
* i.e. not the current one to which we are posting new WRID entries,
* then we do not attempt to update the "q_head", "q_tail", and
* "q_full" indicators on the main work queue header. We do, however,
* update the "head" and "full" indicators on the individual containers
* as we go. This is imperative because we need to be able to
* determine when the current container has been emptied (so that we
* can move on to the next container).
*/
/*
* First check if we are on an SRQ. If so, we grab the entry
* and break out. Since SRQ wridlist's are never added to
* reaplist, they can only be the last container.
*/
break;
}
/*
* Grab the current "head", "tail" and "size" fields before
* walking the list in the current container. Note: the "size"
* field here must always be a power-of-2. The "full"
* parameter is checked (and updated) here to distinguish the
* "queue full" condition from "queue empty".
*/
/*
* If the current entry's "wqeaddrsz" matches the one
* we're searching for, then this must correspond to
* the work request that caused the completion. Set
* the "found" flag and bail out.
*/
found = 1;
break;
}
}
/*
* If the current container is empty (having reached here the
* "head == tail" condition can only mean that the container
* is empty), then NULL out the "wrid_old_tail" field (see
* tavor_post_send() and tavor_post_recv() for more details)
* and (potentially) remove the current container from future
* searches.
*/
/*
* If this wasn't the last "container" on the chain,
* i.e. the one to which new WRID entries will be
* added, then remove it from the list.
* Note: we don't "lose" the memory pointed to by this
* because we should have already put this container
* on the "reapable" list (from where it will later be
* pulled).
*/
if (!last_container) {
}
}
/* Update the head index for the container */
/*
* If the entry was found in this container, then continue to
* bail out. Else reset the "curr" pointer and move on to the
* next container (if there is one). Note: the only real
* reason for setting "curr = NULL" here is so that the ASSERT
* below can catch the case where no matching entry was found
* on any of the lists.
*/
if (found) {
break;
} else {
}
}
/*
* Update work queue header's "head" and "full" conditions to match
* the last entry on the container list. (Note: Only if we're pulling
* entries from the last work queue portion of the list, i.e. not from
* the previous portions that may be the "reapable" list.)
*/
if (last_container) {
}
/* Ensure that we've actually found what we were searching for */
return (curr);
}
/*
* tavor_wrid_find_match_srq()
* Context: Can be called from interrupt or base context.
*/
{
/* Grab the WQE addr out of the CQE */
/*
* Use the WQE addr as the lower 32-bit, we add back on the
* 'wl_srq_desc_off' because we have a zero-based queue. Then the
* upper 32-bit of the 'wl_srq_wq_buf' OR'd on gives us the WQE addr in
* the SRQ Work Queue itself. We use this address as the index to find
* out which Work Queue Entry this CQE corresponds with.
*
* We also use this address below to add the WQE back on to the free
* list.
*/
/*
* Given the 'wqe_addr' just calculated and the srq buf address, we
* find the 'wqe_index'. The 'wre' returned below contains the WRID
* that we are looking for. This indexes into the wre_list for this
* specific WQE.
*/
/* ASSERT on impossible wqe_index values */
/* Get the pointer to this WQE */
/* Put this WQE index back on the free list */
/* Using the index, return the Work Request ID Entry (wre) */
return (wre);
}
/*
* tavor_wrid_cq_reap()
* Context: Can be called from interrupt or base context.
*/
void
{
/* Lock the list of work queues associated with this CQ */
/* Walk the "reapable" list and free up containers */
/*
* If reaping the WRID list containers pulls the last
* container from the given work queue header, then we free
* the work queue header as well.
*/
if (consume_wqhdr != NULL) {
}
}
/* Once finished reaping, we reset the CQ's reap list */
}
/*
* tavor_wrid_cq_force_reap()
* Context: Can be called from interrupt or base context.
*/
void
{
/*
* The first step is to walk the "reapable" list and free up those
* containers. This is necessary because the containers on the
* reapable list are not otherwise connected to the work queue headers
* anymore.
*/
/* Now lock the list of work queues associated with this CQ */
/*
* Walk the list of work queue headers and free up all the WRID list
* containers chained to it. Note: We don't need to grab the locks
* for each of the individual WRID lists here because the only way
* things can be added or removed from the list at this point would be
* through post a work request to a QP. But if we've come this far,
* then we can be assured that there are no longer any QP associated
* with the CQ that we are trying to free.
*/
#ifdef __lock_lint
#endif
/*
* If reaping the WRID list containers pulls the last
* container from the given work queue header, then
* we free the work queue header as well. Note: we
* ignore the return value because we know that the
* work queue header should always be freed once the
* list of containers has come to an end.
*/
(void) tavor_wrid_list_reap(to_free);
}
}
}
}
/*
* tavor_wrid_get_list()
* Context: Can be called from interrupt or base context.
*/
{
/*
* The WRID list "container" consists of the tavor_wrid_list_hdr_t,
* which holds the pointers necessary for maintaining the "reapable"
* list, chaining together multiple "containers" old and new, and
* tracking the head, tail, size, etc. for each container.
*
* The "container" also holds all the tavor_wrid_entry_t's, which is
* allocated separately, one for each entry on the corresponding work
* queue.
*/
size = sizeof (tavor_wrid_list_hdr_t);
/*
* Note that this allocation has to be a NOSLEEP operation here
* because we are holding the "wqhdr_list_lock" and, therefore,
* could get raised to the interrupt level.
*/
return (NULL);
}
/* Complete the "container" initialization */
sizeof (tavor_wrid_entry_t), KM_NOSLEEP);
return (NULL);
}
return (wridlist);
}
/*
* tavor_wrid_list_srq_init()
* Context: Can be called from interrupt or base context
*/
void
{
int wqe_index;
/* Setup pointers for use later when we are polling the CQ */
/* Given wq_start to start initializing buf at, verify sanity */
/*
* Initialize wridlist free list
*
* For each WQ up to the size of our queue, we store an index in the WQ
* memory itself, representing the next available free entry. The
* 'wl_free_list_indx' always holds the index of the next available
* free entry in the WQ. If 'wl_free_list_indx' is -1, then we are
* completely full. This gives us the advantage of being able to have
* entries complete or be polled off the WQ out-of-order.
*
* For now, we write the free_list entries inside the WQ itself. It
* may be useful in the future to store this information in a separate
* structure for debugging purposes.
*/
}
}
/*
* tavor_wrid_reaplist_add()
* Context: Can be called from interrupt or base context.
*/
static void
{
/*
* Add the "post" container (the last one on the current chain) to
* the CQ's "reapable" list
*/
} else {
}
}
int
{
return (-1);
return (+1);
return (-1);
return (+1);
else
return (0);
}
/*
* tavor_wrid_wqhdr_find()
* Context: Can be called from interrupt or base context.
*/
static tavor_workq_hdr_t *
{
/*
* Walk the CQ's work queue list, trying to find a send or recv queue
* with the same QP number. We do this even if we are going to later
* create a new entry because it helps us easily find the end of the
* list.
*/
#ifdef __lock_lint
#endif
return (curr);
}
/*
* tavor_wrid_wqhdr_create()
* Context: Can be called from interrupt or base context.
*/
static tavor_workq_hdr_t *
{
/*
* Allocate space a work queue header structure and initialize it.
* Each work queue header structure includes a "wq_wrid_wql"
* which needs to be initialized. Note that this allocation has to be
* a NOSLEEP operation because we are holding the "cq_wrid_wqhdr_lock"
* and, therefore, could get raised to the interrupt level.
*/
sizeof (tavor_workq_hdr_t), KM_NOSLEEP);
return (NULL);
}
if (create_wql) {
return (NULL);
}
}
/* Chain the newly allocated work queue header to the CQ's list */
return (wqhdr_tmp);
}
/*
* tavor_wrid_wql_create()
* Context: Can be called from interrupt or base context.
*/
{
/*
* Allocate the WQL and initialize it.
*/
return (NULL);
}
/* Add refcount to WQL */
return (wql);
}
/*
* tavor_wrid_get_wqeaddrsz()
* Context: Can be called from interrupt or base context.
*/
static uint32_t
{
/*
* If the container is empty, then there is no next entry. So just
* return zero. Note: the "head == tail" condition here can only
* mean that the container is empty because we have previously pulled
* something from the container.
*
* If the container is not empty, then find the next entry and return
* the contents of its "wqeaddrsz" field.
*/
wqeaddrsz = 0;
} else {
/*
* We don't need to calculate the "next" head pointer here
* because "head" should already point to the next entry on
* the list (since we just pulled something off - in
* tavor_wrid_find_match() - and moved the head index forward.)
*/
}
return (wqeaddrsz);
}
/*
* tavor_wrid_wqhdr_add()
* Context: Can be called from interrupt or base context.
*/
static void
{
/* Chain the new WRID list "container" to the work queue list */
} else {
}
}
/*
* tavor_wrid_wqhdr_remove()
* Context: Can be called from interrupt or base context.
*
* Note: this is only called to remove the most recently added WRID list
* container (i.e. in tavor_from_reset() above)
*/
static void
{
/* Unlink the WRID list "container" from the work queue list */
}
}
/*
* Update any pointers in the work queue hdr that may point to this
* WRID list container
*/
}
}
}
/*
* tavor_wrid_list_reap()
* Context: Can be called from interrupt or base context.
* Note: The "wqhdr_list_lock" must be held.
*/
static tavor_workq_hdr_t *
{
/* Get the back pointer to the work queue header (see below) */
/* Unlink the WRID list "container" from the work queue list */
}
}
/*
* If the back pointer to the work queue header shows that it
* was pointing to the entry we are about to remove, then the work
* queue header is reapable as well.
*/
}
/* Be sure to update the "poll" and "post" container pointers */
}
}
/* Calculate the size and free the container */
return (consume_wqhdr);
}
/*
* tavor_wrid_wqhdr_lock_both()
* Context: Can be called from interrupt or base context.
*/
static void
{
/*
* If both work queues (send and recv) share a completion queue, then
* grab the common lock. If they use different CQs (hence different
* "cq_wrid_wqhdr_list" locks), then grab the send one first, then the
* receive. We do this consistently and correctly in
* tavor_wrid_wqhdr_unlock_both() below to avoid introducing any kind
* of dead lock condition. Note: We add the "__lock_lint" code here
* to fake out warlock into thinking we've grabbed both locks (when,
* in fact, we only needed the one).
*/
#ifdef __lock_lint
#endif
} else {
}
}
/*
* tavor_wrid_wqhdr_unlock_both()
* Context: Can be called from interrupt or base context.
*/
static void
{
/*
* See tavor_wrid_wqhdr_lock_both() above for more detail
*/
#ifdef __lock_lint
#endif
} else {
}
}
/*
* tavor_cq_wqhdr_add()
* Context: Can be called from interrupt or base context.
*/
static void
{
#ifdef __lock_lint
#endif
/*
* If the CQ's work queue list is empty, then just add it.
* Otherwise, chain it to the beginning of the list.
*/
}
/*
* tavor_cq_wqhdr_remove()
* Context: Can be called from interrupt or base context.
*/
static void
{
#ifdef __lock_lint
#endif
/* Remove "wqhdr" from the work queue header list on "cq" */
/*
* Release reference to WQL; If this is the last reference, this call
* also has the side effect of freeing up the 'wq_wrid_wql' memory.
*/
/* Free the memory associated with "wqhdr" */
}
/*
* tavor_wql_refcnt_inc()
* Context: Can be called from interrupt or base context
*/
void
{
wql->wql_refcnt++;
}
/*
* tavor_wql_refcnt_dec()
* Context: Can be called from interrupt or base context
*/
void
{
int refcnt;
wql->wql_refcnt--;
/*
*
* Free up WQL memory if we're the last one associated with this
* structure.
*/
if (refcnt == 0) {
}
}