dapl_arbel_hw.c revision 9e39c5ba00a55fa05777cc94b148296af305e135
/*
* CDDL HEADER START
*
* The contents of this file are subject to the terms of the
* Common Development and Distribution License (the "License").
* You may not use this file except in compliance with the License.
*
* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
* See the License for the specific language governing permissions
* and limitations under the License.
*
* When distributing Covered Code, include this CDDL HEADER in each
* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
* If applicable, add the following below this CDDL HEADER, with the
* fields enclosed by brackets "[]" replaced with your own identifying
* information: Portions Copyright [yyyy] [name of copyright owner]
*
* CDDL HEADER END
*/
/*
* Copyright 2008 Sun Microsystems, Inc. All rights reserved.
* Use is subject to license terms.
*/
#include "dapl.h"
#include "dapl_tavor_hw.h"
#include "dapl_tavor_wr.h"
#include "dapl_tavor_ibtf_impl.h"
#define bt_debug 0
enum arbel_db_type_e {
};
#define ARBEL_WQE_SGL_INVALID_LKEY 0x00000100
#define ARBEL_WQE_SEND_SIGNALED_MASK 0x0000000800000000ull
#define ARBEL_WQE_SEND_SOLICIT_MASK 0x0000000200000000ull
#define ARBEL_WQE_CTRL_REQBIT_MASK 0x0000000100000000ull
#define ARBEL_WQE_NEXT_REQBIT_MASK 0x80
{ \
}
#define ARBEL_WQE_SEND_FENCE_MASK 0x40
#define ARBEL_WQE_SEND_NOPCODE_RDMAW 0x8
#define ARBEL_WQE_SEND_NOPCODE_SEND 0xA
#define ARBEL_WQE_SEND_NOPCODE_RDMAR 0x10
#define ARBEL_WQE_SEND_NOPCODE_BIND 0x18
#define ARBEL_WQE_NDA_MASK 0x00000000FFFFFFC0ull
#define ARBEL_WQE_NDS_MASK 0x3F
#define ARBEL_QPSNDDB_WQE_CNT_SHIFT 0x38
#define ARBEL_QPSNDDB_WQE_COUNTER_SHIFT 0x28
#define ARBEL_QPSNDDB_F_SHIFT 0x25
#define ARBEL_QPSNDDB_NOPCODE_SHIFT 0x20
#define ARBEL_QPSNDDB_QPN_SHIFT 0x8
#define ARBEL_DBR_QP_WQE_COUNTER_SHIFT 0x20
#define ARBEL_DBR_QN_SHIFT 0x8
#define ARBEL_CQDB_NOTIFY_CQ_SOLICIT 0x1
#define ARBEL_CQDB_NOTIFY_CQ 0x2
/*
* Function signatures
*/
extern void dapls_tavor_wrid_cq_reap(ib_cq_handle_t);
extern DAPL_OS_LOCK g_tavor_uar_lock;
#ifndef _LP64
#endif
ibt_wc_t *);
ibt_wc_t *);
/*
* Note: The 64 bit doorbells need to written atomically.
* In 32 bit libraries we need to use the special assembly rtn
* because compiler generated code splits into 2 word writes
*/
/*
* dapli_arbel_cq_doorbell()
* Takes the specified cq cmd and cq number and rings the cq doorbell
*/
static void
{
/* Build the doorbell from the parameters */
/* Write the doorbell to UAR */
#ifdef _LP64
/* 32 bit version */
/*
* For 32 bit intel we assign the doorbell in the order
* prescribed by the Tavor PRM, lower to upper addresses
*/
#else
#endif
}
/*
* dapli_arbel_qp_send_doorbell()
* Takes the specified next descriptor information, qp number, opcode and
* rings the send doorbell
*/
static void
{
}
static void
{
/* Write the doorbell to UAR */
#ifdef _LP64
#else
#if defined(i386)
/*
* For 32 bit intel we assign the doorbell in the order
* prescribed by the Tavor PRM, lower to upper addresses
*/
#else
#endif
#endif
}
/*
* dapli_arbel_wqe_send_build()
* Constructs a WQE for a given ibt_send_wr_t
*/
static int
{
int i, num_ds;
int max_inline_bytes = -1;
num_ds = 0;
ARBEL_WQE_SEND_SIGNALED_MASK : 0) |
ARBEL_WQE_SEND_SOLICIT_MASK : 0) |
/*
* RC is the only supported transport in UDAPL
* For RC requests, we allow "Send", "RDMA Read", "RDMA Write"
*/
case IBT_WRC_SEND:
/*
* If this is a Send request, then all we need is
* the Data Segment processing below.
* Initialize the information for the Data Segments
*/
sizeof (tavor_hw_snd_wqe_nextctrl_t));
if (qp->qp_sq_inline != 0)
break;
case IBT_WRC_RDMAW:
if (qp->qp_sq_inline != 0)
/* FALLTHROUGH */
case IBT_WRC_RDMAR:
qp->qp_sq_inline = 0;
/*
* If this is an RDMA Read or RDMA Write request, then fill
* in the "Remote Address" header fields.
*/
sizeof (tavor_hw_snd_wqe_nextctrl_t));
/*
* Build the Remote Address Segment for the WQE, using
* the information from the RC work request.
*/
/* Update "ds" for filling in Data Segments (below) */
sizeof (tavor_hw_snd_wqe_remaddr_t));
break;
case IBT_WRC_BIND:
/*
* Generate a new R_key
* Increment the upper "unconstrained" bits and need to keep
* the lower "constrained" bits the same it represents
* the MPT index.
*/
#if 0
/* XXX - need equiv of "arbel_wr_bind_check(state, wr);" */
/* XXX - uses arbel_mr_keycalc - what about Sinai vs. Arbel??? */
#endif
sizeof (tavor_hw_snd_wqe_nextctrl_t));
/*
* Build the Bind Memory Window Segments for the WQE,
* using the information from the RC Bind memory
* window work request.
*/
/*
* Update the "ds" pointer. Even though the "bind"
* operation requires no SGLs, this is necessary to
* facilitate the correct descriptor size calculations
* (below).
*/
sizeof (tavor_hw_snd_wqe_bind_t));
nds = 0;
break;
default:
"dapli_arbel_wqe_send_build: invalid wr_opcode=%d\n",
return (DAT_INTERNAL_ERROR);
}
/*
* Now fill in the Data Segments (SGL) for the Send WQE based on
* the values setup above (i.e. "sgl", "nds", and the "ds" pointer
* Start by checking for a valid number of SGL entries
*/
return (DAT_INVALID_PARAMETER);
}
/*
* For each SGL in the Send Work Request, fill in the Send WQE's data
* segments. Note: We skip any SGL with zero size because Tavor
* hardware cannot handle a zero for "byte_cnt" in the WQE. Actually
* the encoding for zero means a 2GB transfer. Because of this special
* encoding in the hardware, we mask the requested length with
* TAVOR_WQE_SGL_BYTE_CNT_MASK (so that 2GB will end up encoded as
* zero.)
*/
total_len = 0;
for (i = 0; i < nds; i++)
if (total_len > max_inline_bytes)
}
for (i = 0; i < nds; i++) {
continue;
}
(void) dapl_os_memcpy(dst,
}
/* Return the size of descriptor (in 16-byte chunks) */
} else {
for (i = 0; i < nds; i++) {
continue;
}
/*
* Fill in the Data Segment(s) for the current WQE,
* using the information contained in the
* scatter-gather list of the work request.
*/
num_ds++;
}
/* Return the size of descriptor (in 16-byte chunks) */
}
return (DAT_SUCCESS);
}
/*
* dapli_arbel_wqe_send_linknext()
* Takes a WQE and links it to the prev WQE chain
*/
static void
{
/*
* Calculate the "next" field of the prev descriptor. This amounts
* to setting up the "next_wqe_addr", "nopcode", "fence", and "nds"
* fields (see tavor_hw.h for more).
*/
/*
* Determine the value for the Tavor WQE "nopcode" field
* by using the IBTF opcode from the work request
*/
case IBT_WRC_RDMAW:
break;
case IBT_WRC_SEND:
break;
case IBT_WRC_RDMAR:
break;
case IBT_WRC_BIND:
break;
default:
/* Unsupported opcodes in UDAPL */
"dapli_arbel_wqe_send_linknext: invalid nopcode=%d\n",
nopcode);
return;
}
/*
* A send queue doorbell will be rung for the next
* WQE on the chain, set the current WQE's "dbd" bit.
* Note: We also update the "dbinfo" structure here to pass
* back information about what should (later) be included
* in the send queue doorbell.
*/
}
/*
* dapli_arbel_wqe_recv_build()
* Builds the recv WQE for a given ibt_recv_wr_t
*/
static DAT_RETURN
{
int i;
int num_ds;
/* Fill in the Data Segments (SGL) for the Recv WQE */
sizeof (tavor_hw_rcv_wqe_nextctrl_t));
num_ds = 0;
/* Check for valid number of SGL entries */
return (DAT_INVALID_PARAMETER);
}
/*
* For each SGL in the Recv Work Request, fill in the Recv WQE's data
* segments. Note: We skip any SGL with zero size because Tavor
* hardware cannot handle a zero for "byte_cnt" in the WQE. Actually
* the encoding for zero means a 2GB transfer. Because of this special
* encoding in the hardware, we mask the requested length with
* TAVOR_WQE_SGL_BYTE_CNT_MASK (so that 2GB will end up encoded as
* zero.)
*/
continue;
}
/*
* Fill in the Data Segment(s) for the receive WQE, using the
* information contained in the scatter-gather list of the
* work request.
*/
num_ds++;
}
}
/* Return the size of descriptor (in 16-byte chunks) */
return (DAT_SUCCESS);
}
/*
* dapli_arbel_wqe_srq_build()
* Builds the recv WQE for a given ibt_recv_wr_t
*/
static DAT_RETURN
{
int i;
int num_ds;
/* Fill in the Data Segments (SGL) for the Recv WQE */
sizeof (tavor_hw_rcv_wqe_nextctrl_t));
num_ds = 0;
/* Check for valid number of SGL entries */
return (DAT_INVALID_PARAMETER);
}
/*
* For each SGL in the Recv Work Request, fill in the Recv WQE's data
* segments. Note: We skip any SGL with zero size because Tavor
* hardware cannot handle a zero for "byte_cnt" in the WQE. Actually
* the encoding for zero means a 2GB transfer. Because of this special
* encoding in the hardware, we mask the requested length with
* TAVOR_WQE_SGL_BYTE_CNT_MASK (so that 2GB will end up encoded as
* zero.)
*/
continue;
}
/*
* Fill in the Data Segment(s) for the receive WQE, using the
* information contained in the scatter-gather list of the
* work request.
*/
num_ds++;
}
/*
* For SRQ, if the number of data segments is less than the maximum
* specified at alloc, then we have to fill in a special "key" entry in
* the sgl entry after the last valid one in this post request. We do
* that here.
*/
}
return (DAT_SUCCESS);
}
/*
* dapli_arbel_cq_peek()
* Peeks into a given CQ to check if there are any events that can be
* polled. It returns the number of CQEs that can be polled.
*/
static void
{
/* Get the consumer index */
/*
* Calculate the wrap around mask. Note: This operation only works
* because all Tavor completion queues have power-of-2 sizes
*/
/* Calculate the pointer to the first CQ entry */
/*
* Count entries in the CQ until we find an entry owned by
* the hardware.
*/
polled_cnt = 0;
while (TAVOR_CQE_OWNER_IS_SW(cqe)) {
/* Error CQE map to multiple work completions */
if ((opcode == TAVOR_CQE_SEND_ERR_OPCODE) ||
(opcode == TAVOR_CQE_RECV_ERR_OPCODE)) {
} else {
polled_cnt++;
}
/* Increment the consumer index */
/* Update the pointer to the next CQ entry */
}
*num_cqe = polled_cnt;
}
/*
* dapli_arbel_cq_poll()
* This routine polls CQEs out of a CQ and puts them into the ibt_wc_t
* array that is passed in.
*/
static DAT_RETURN
{
int status;
/* Get the consumer index */
/*
* Calculate the wrap around mask. Note: This operation only works
* because all Tavor completion queues have power-of-2 sizes
*/
/* Calculate the pointer to the first CQ entry */
/*
* Keep pulling entries from the CQ until we find an entry owned by
* the hardware. As long as there the CQE's owned by SW, process
* each entry by calling dapli_arbel_cq_cqe_consume() and updating the
* CQ consumer index. Note: We only update the consumer index if
* dapli_arbel_cq_cqe_consume() returns TAVOR_CQ_SYNC_AND_DB.
* Otherwise, it indicates that we are going to "recycle" the CQE
* (probably because it is a error CQE and corresponds to more than one
* completion).
*/
polled_cnt = 0;
while (TAVOR_CQE_OWNER_IS_SW(cqe)) {
&wc_p[polled_cnt++]);
if (status == TAVOR_CQ_SYNC_AND_DB) {
/* Reset entry to hardware ownership */
/* Increment the consumer index */
/* Update the pointer to the next CQ entry */
}
/*
* If we have run out of space to store work completions,
* then stop and return the ones we have pulled of the CQ.
*/
if (polled_cnt >= num_wc) {
break;
}
}
/*
* Now we only ring the doorbell (to update the consumer index) if
* we've actually consumed a CQ entry. If we have, for example,
* pulled from a CQE that we are still in the process of "recycling"
* for error purposes, then we would not update the consumer index.
*/
/*
* Update the consumer index in both the CQ handle and the
* doorbell record.
*/
} else if (polled_cnt == 0) {
/*
* If the CQ is empty, we can try to free up some of the WRID
* list containers.
*/
}
if (num_polled != NULL) {
*num_polled = polled_cnt;
}
return (dat_status);
}
/*
* dapli_arbel_cq_poll_one()
* This routine polls one CQE out of a CQ and puts ot into the ibt_wc_t
*/
static DAT_RETURN
{
int status;
/* Get the consumer index */
/* Calculate the pointer to the first CQ entry */
/*
* Keep pulling entries from the CQ until we find an entry owned by
* the hardware. As long as there the CQE's owned by SW, process
* each entry by calling dapli_arbel_cq_cqe_consume() and updating the
* CQ consumer index. Note: We only update the consumer index if
* dapli_arbel_cq_cqe_consume() returns TAVOR_CQ_SYNC_AND_DB.
* Otherwise, it indicates that we are going to "recycle" the CQE
* (probably because it is a error CQE and corresponds to more than one
* completion).
*/
if (TAVOR_CQE_OWNER_IS_SW(cqe)) {
if (status == TAVOR_CQ_SYNC_AND_DB) {
/* Reset entry to hardware ownership */
/* Increment the consumer index */
cq->cq_consindx =
}
} else {
}
return (dat_status);
}
/*
* dapli_arbel_cq_cqe_consume()
* Converts a given CQE into a ibt_wc_t object
*/
static int
{
int status;
/* strip off the size in wqeaddrsz */
/*
* Determine if this is an "error" CQE by examining "opcode". If it
* is an error CQE, then call dapli_arbel_cq_errcqe_consume() and return
* whatever status it returns. Otherwise, this is a successful
* completion.
*/
if ((opcode == TAVOR_CQE_SEND_ERR_OPCODE) ||
(opcode == TAVOR_CQE_RECV_ERR_OPCODE)) {
return (status);
}
/*
* Fetch the Work Request ID using the information in the CQE.
* See tavor_wr.c for more details.
*/
/*
* Parse the CQE opcode to determine completion type. This will set
* not only the type of the completion, but also any flags that might
* be associated with it (e.g. whether immediate data is present).
*/
/*
* Send CQE
*
* The following opcodes will not be generated in uDAPL
* case TAVOR_CQE_SND_RDMAWR_IMM:
* case TAVOR_CQE_SND_SEND_IMM:
* case TAVOR_CQE_SND_ATOMIC_CS:
* case TAVOR_CQE_SND_ATOMIC_FA:
*/
switch (opcode) {
case TAVOR_CQE_SND_RDMAWR:
break;
case TAVOR_CQE_SND_SEND:
type = IBT_WRC_SEND;
break;
case TAVOR_CQE_SND_RDMARD:
break;
case TAVOR_CQE_SND_BIND_MW:
type = IBT_WRC_BIND;
break;
default:
return (TAVOR_CQ_SYNC_AND_DB);
}
} else {
/*
* Receive CQE
*
* The following opcodes will not be generated in uDAPL
*
* case TAVOR_CQE_RCV_RECV_IMM:
* case TAVOR_CQE_RCV_RECV_IMM2:
* case TAVOR_CQE_RCV_RDMAWR_IMM:
* case TAVOR_CQE_RCV_RDMAWR_IMM2:
*/
switch (opcode & 0x1F) {
case TAVOR_CQE_RCV_RECV:
/* FALLTHROUGH */
case TAVOR_CQE_RCV_RECV2:
type = IBT_WRC_RECV;
break;
default:
return (TAVOR_CQ_SYNC_AND_DB);
}
}
/* If we got here, completion status must be success */
return (TAVOR_CQ_SYNC_AND_DB);
}
/*
* dapli_arbel_cq_errcqe_consume()
*/
static int
{
/*
* Fetch the Work Request ID using the information in the CQE.
* See tavor_wr.c for more details.
*/
/*
* Parse the CQE opcode to determine completion type. We know that
* the CQE is an error completion, so we extract only the completion
* status here.
*/
switch (status) {
case TAVOR_CQE_LOC_LEN_ERR:
break;
case TAVOR_CQE_LOC_OP_ERR:
break;
case TAVOR_CQE_LOC_PROT_ERR:
break;
case TAVOR_CQE_WR_FLUSHED_ERR:
break;
case TAVOR_CQE_MW_BIND_ERR:
break;
break;
break;
break;
case TAVOR_CQE_REM_ACC_ERR:
break;
case TAVOR_CQE_REM_OP_ERR:
break;
case TAVOR_CQE_TRANS_TO_ERR:
break;
case TAVOR_CQE_RNRNAK_TO_ERR:
break;
/*
* The following error codes are not supported in the Tavor driver
* as they relate only to Reliable Datagram completion statuses:
* case TAVOR_CQE_LOCAL_RDD_VIO_ERR:
* case TAVOR_CQE_REM_INV_RD_REQ_ERR:
* case TAVOR_CQE_EEC_REM_ABORTED_ERR:
* case TAVOR_CQE_INV_EEC_NUM_ERR:
* case TAVOR_CQE_INV_EEC_STATE_ERR:
* case TAVOR_CQE_LOC_EEC_ERR:
*/
default:
break;
}
/*
* Consume the CQE
* Return status to indicate that doorbell and sync may be
* necessary.
*/
return (TAVOR_CQ_SYNC_AND_DB);
}
/*
* dapli_arbel_cq_notify()
* This function is used for arming the CQ by ringing the CQ doorbell.
*
* Note: there is something very subtle here. This code assumes a very
* specific behavior of the kernel driver. The cmd_sn field of the
* arm_dbr is updated by the kernel driver whenever a notification
* event for the cq is received. This code extracts the cmd_sn field
* from the arm_dbr to know the right value to use. The arm_dbr is
* always updated atomically so that neither the kernel driver nor this
* will get confused about what the other is doing.
*
* Note: param is not used here. It is necessary for arming a CQ for
* N completions (param is N), but no uDAPL API supports this for now.
* Thus, we declare ARGSUSED to make lint happy.
*/
/*ARGSUSED*/
static DAT_RETURN
{
/*
* Determine if we are trying to get the next completion or the next
* "solicited" completion. Then hit the appropriate doorbell.
*/
if (flags == IB_NOTIFY_ON_NEXT_COMP) {
if (old_cmd != ARBEL_CQDB_NOTIFY_CQ) {
goto retry;
} /* else it's already armed */
} else if (flags == IB_NOTIFY_ON_NEXT_SOLICITED) {
if (old_cmd != ARBEL_CQDB_NOTIFY_CQ &&
goto retry;
} /* else it's already armed */
} else {
return (DAT_INVALID_PARAMETER);
}
return (DAT_SUCCESS);
}
/*
* dapli_arbel_post_send()
*/
/* ARGSUSED */
static DAT_RETURN
{
int status;
return (DAT_INVALID_STATE);
}
/* Grab the lock for the WRID list */
/* Save away some initial QP state */
/*
* Check for "queue full" condition. If the queue is already full,
* then no more WQEs can be posted, return an error
*/
return (DAT_INSUFFICIENT_RESOURCES);
}
/*
* Increment the "tail index" and check for "queue full" condition.
* If we detect that the current work request is going to fill the
* work queue, then we mark this condition and continue.
*/
}
/*
* Get the user virtual address of the location where the next
* Send WQE should be built
*/
/*
* Call tavor_wqe_send_build() to build the WQE at the given address.
* This routine uses the information in the ibt_send_wr_t and
* returns the size of the WQE when it returns.
*/
if (status != DAT_SUCCESS) {
return (status);
}
/*
* Get the descriptor (io address) corresponding to the location
* Send WQE was built.
*/
/*
* Add a WRID entry to the WRID list. Need to calculate the
* "wqeaddr" to pass to dapli_tavor_wrid_add_entry().
* signaled_dbd is still calculated, but ignored.
*/
}
/*
* Now link the wqe to the old chain (if there was one)
*/
/*
* Now if the WRID tail entry is non-NULL, then this
* represents the entry to which we are chaining the
* new entries. Since we are going to ring the
* doorbell for this WQE, we want set its "dbd" bit.
*
* On the other hand, if the tail is NULL, even though
* we will have rung the doorbell for the previous WQE
* (for the hardware's sake) it is irrelevant to our
* purposes (for tracking WRIDs) because we know the
* request must have already completed.
*/
}
/* Update some of the state in the QP */
/* Set the doorbell decord */
/* Ring the doorbell */
qp->qp_sq_counter++;
return (DAT_SUCCESS);
}
/*
* dapli_arbel_post_recv()
*/
/* ARGSUSED */
static DAT_RETURN
{
return (DAT_INVALID_STATE);
}
/* Grab the lock for the WRID list */
/* Save away some initial QP state */
/*
* For the ibt_recv_wr_t passed in, parse the request and build a
* Recv WQE. Link the WQE with the previous WQE and ring the
* door bell.
*/
/*
* Check for "queue full" condition. If the queue is already full,
* then no more WQEs can be posted. So return an error.
*/
return (DAT_INSUFFICIENT_RESOURCES);
}
/*
* Increment the "tail index" and check for "queue
* full" condition. If we detect that the current
* work request is going to fill the work queue, then
* we mark this condition and continue.
*/
}
/* Get the descriptor (IO Address) of the WQE to be built */
/* The user virtual address of the WQE to be built */
/*
* Call tavor_wqe_recv_build() to build the WQE at the given
* address. This routine uses the information in the
* ibt_recv_wr_t and returns the size of the WQE.
*/
if (status != DAT_SUCCESS) {
return (DAT_INTERNAL_ERROR);
}
/*
* Add a WRID entry to the WRID list. Need to calculate the
* "wqeaddr" and "signaled_dbd" values to pass to
* dapli_tavor_wrid_add_entry().
* Note: all Recv WQEs are essentially "signaled"
*/
/*
* Now if the WRID tail entry is non-NULL, then this
* represents the entry to which we are chaining the
* new entries. Since we are going to ring the
* doorbell for this WQE, we want set its "dbd" bit.
*
* On the other hand, if the tail is NULL, even though
* we will have rung the doorbell for the previous WQE
* (for the hardware's sake) it is irrelevant to our
* purposes (for tracking WRIDs) because we know the
* request must have already completed.
*/
}
/* Update some of the state in the QP */
/* Update the doorbell record */
qp->qp_rq_counter++;
return (DAT_SUCCESS);
}
/*
* dapli_arbel_post_srq()
*/
/* ARGSUSED */
static DAT_RETURN
{
/* Grab the lock for the WRID list */
/*
* For the ibt_recv_wr_t passed in, parse the request and build a
* Recv WQE. Link the WQE with the previous WQE and ring the
* door bell.
*/
/*
* Check for "queue full" condition. If the queue is already full,
* ie. there are no free entries, then no more WQEs can be posted.
* So return an error.
*/
return (DAT_INSUFFICIENT_RESOURCES);
}
/* Save away some initial SRQ state */
/* Get the descriptor (IO Address) of the WQE to be built */
srq->srq_wq_wqesz);
/* The user virtual address of the WQE to be built */
/*
* Call dapli_arbel_wqe_srq_build() to build the WQE at the given
* address. This routine uses the information in the
* ibt_recv_wr_t and returns the size of the WQE.
*/
if (status != DAT_SUCCESS) {
return (status);
}
/*
* Add a WRID entry to the WRID list.
*/
#if 0
} else {
}
/*
* Now link the chain to the old chain (if there was one)
* and update the wqe_counter in the doorbell record.
*/
#endif
/* Update some of the state in the SRQ */
/* Update the doorbell record */
srq->srq_counter++;
return (DAT_SUCCESS);
}
/*
* dapli_arbel_cq_srq_entries_flush()
*/
static void
{
int outstanding_cqes, removed_cqes;
int i;
/* ASSERT(MUTEX_HELD(&qp->qp_rq_cqhdl->cq_lock)); */
/* Get the consumer index */
/*
* Calculate the wrap around mask. Note: This operation only works
* because all Tavor completion queues have power-of-2 sizes
*/
/* Calculate the pointer to the first CQ entry */
/*
* Loop through the CQ looking for entries owned by software. If an
* entry is owned by software then we increment an 'outstanding_cqes'
* count to know how many entries total we have on our CQ. We use this
* value further down to know how many entries to loop through looking
* for our same QP number.
*/
outstanding_cqes = 0;
while (TAVOR_CQE_OWNER_IS_SW(cqe)) {
/* increment total cqes count */
/* increment the consumer index */
/* update the pointer to the next cq entry */
}
/*
* Using the 'tail_cons_indx' that was just set, we now know how many
* total CQEs possible there are. Set the 'check_indx' and the
* 'new_indx' to the last entry identified by 'tail_cons_indx'
*/
for (i = 0; i < outstanding_cqes; i++) {
/* Grab QP number from CQE */
/*
* If the QP number is the same in the CQE as the QP that we
* have on this SRQ, then we must free up the entry off the
* SRQ. We also make sure that the completion type is of the
* 'TAVOR_COMPLETION_RECV' type. So any send completions on
* this CQ will be left as-is. The handling of returning
* entries back to HW ownership happens further down.
*/
/* Add back to SRQ free list */
(void) dapli_tavor_wrid_find_match_srq(
} else {
/* Do Copy */
if (check_indx != new_indx) {
/*
* Copy the CQE into the "next_cqe"
* pointer.
*/
sizeof (tavor_hw_cqe_t));
}
}
/* Move index to next CQE to check */
}
/* Initialize removed cqes count */
removed_cqes = 0;
/* If an entry was removed */
if (check_indx != new_indx) {
/*
* Set current pointer back to the beginning consumer index.
* At this point, all unclaimed entries have been copied to the
* index specified by 'new_indx'. This 'new_indx' will be used
* as the new consumer index after we mark all freed entries as
* having HW ownership. We do that here.
*/
/* Loop through all entries until we reach our new pointer */
removed_cqes++;
/* Reset entry to hardware ownership */
}
}
/*
* Update consumer index to be the 'new_indx'. This moves it past all
* removed entries. Because 'new_indx' is pointing to the last
* previously valid SW owned entry, we add 1 to point the cons_indx to
* the first HW owned entry.
*/
/*
* Now we only ring the doorbell (to update the consumer index) if
* we've actually consumed a CQ entry. If we found no QP number
* matches above, then we would not have removed anything. So only if
* something was removed do we ring the doorbell.
*/
/*
* Update the consumer index in both the CQ handle and the
* doorbell record.
*/
}
}
static void
{
int i;
}
p[1] = nds;
}
static void
{
int i;
}
}
static void
{
if (qp->qp_srq_enabled == 0) {
/* pre-link the whole receive queue */
}
qp->qp_rq_counter = 0;
qp->qp_sq_counter = 0;
}
static void
{
/* cq_resize -- needs testing */
}
static void
{
/* pre-link the whole shared receive queue */
srq->srq_counter = 0;
/* needs testing */
}
void
{
hca_ptr->hermon_resize_cq = 0;
}