dapl_tavor_wr.c revision 9e39c5ba00a55fa05777cc94b148296af305e135
/*
* CDDL HEADER START
*
* The contents of this file are subject to the terms of the
* Common Development and Distribution License (the "License").
* You may not use this file except in compliance with the License.
*
* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
* See the License for the specific language governing permissions
* and limitations under the License.
*
* When distributing Covered Code, include this CDDL HEADER in each
* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
* If applicable, add the following below this CDDL HEADER, with the
* fields enclosed by brackets "[]" replaced with your own identifying
* information: Portions Copyright [yyyy] [name of copyright owner]
*
* CDDL HEADER END
*/
/*
* Copyright 2009 Sun Microsystems, Inc. All rights reserved.
* Use is subject to license terms.
*/
#include "dapl.h"
#include "dapl_tavor_wr.h"
#include "dapl_hash.h"
#include "dapl_tavor_ibtf_impl.h"
static void dapli_tavor_wrid_reaplist_add(ib_cq_handle_t,
static void dapli_tavor_wrid_wqhdr_add(dapls_tavor_workq_hdr_t *,
static void dapli_tavor_wrid_wqhdr_remove(dapls_tavor_workq_hdr_t *,
static void dapli_tavor_wrid_wqhdr_lock_both(ib_qp_handle_t);
static void dapli_tavor_wrid_wqhdr_unlock_both(ib_qp_handle_t);
static void dapli_tavor_cq_wqhdr_remove(ib_cq_handle_t,
/*
* dapls_tavor_wrid_get_entry()
*/
{
/* Lock the list of work queues associated with this CQ */
/* Find the work queue for this QP number (send or receive side) */
/*
* Regardless of whether the completion is the result of a "success"
* or a "failure", we lock the list of "containers" and attempt to
* search for the the first matching completion (i.e. the first WR
* with a matching WQE addr and size). Once we find it, we pull out
* the "wrid" field and return it (see below). Note: One possible
* future enhancement would be to enable this routine to skip over
* any "unsignaled" completions to go directly to the next "signaled"
* entry on success. XXX
*/
/*
* If this is a "successful" completion, then we assert that this
* completion must be a "signaled" completion.
*/
/*
* If the completion is a "failed" completion, then we save away the
* contents of the entry (into the "wre" field passed in) for use
* in later CQE processing. Note: We use the
* dapli_tavor_wrid_get_wqeaddrsz() function to grab "wqeaddrsz" from
* the next entry in the container.
* This is required for error processing (where updating these fields
* properly is necessary to correct handling of the "error" CQE)
*/
}
/* Pull out the WRID and return it */
return (wrid);
}
/*
* dapli_tavor_wrid_find_match()
*/
static dapls_tavor_wrid_entry_t *
{
int found = 0, last_container;
/* dapl_os_assert(MUTEX_HELD(&wq->wq_wrid_lock)); */
/* Pull the "wqeaddrsz" information from the CQE */
/*
* Walk the "containers" list(s), find first WR with a matching WQE
* addr. If the current "container" is not the last one on the list,
* i.e. not the current one to which we are posting new WRID entries,
* then we do not attempt to update the "q_head", "q_tail", and
* "q_full" indicators on the main work queue header. We do, however,
* update the "head" and "full" indicators on the individual containers
* as we go. This is imperative because we need to be able to
* determine when the current container has been emptied (so that we
* can move on to the next container).
*/
/*
* First check if we are on an SRQ. If so, we grab the entry
* and break out. Since SRQ wridlist's are never added to
* reaplist, they can only be the last container.
*/
break;
}
/*
* Grab the current "head", "tail" and "size" fields before
* walking the list in the current container. Note: the "size"
* field here must always be a power-of-2. The "full"
* parameter is checked (and updated) here to distinguish the
* "queue full" condition from "queue empty".
*/
/*
* If the current entry's "wqeaddrsz" matches the one
* we're searching for, then this must correspond to
* the work request that caused the completion. Set
* the "found" flag and bail out.
*/
found = 1;
break;
}
}
/*
* If the current container is empty (having reached here the
* "head == tail" condition can only mean that the container
* is empty), then NULL out the "wrid_old_tail" field (see
* tavor_post_send() and tavor_post_recv() for more details)
* and (potentially) remove the current container from future
* searches.
*/
/*
* If this wasn't the last "container" on the chain,
* i.e. the one to which new WRID entries will be
* added, then remove it from the list.
* Note: we don't "lose" the memory pointed to by this
* because we should have already put this container
* on the "reapable" list (from where it will later be
* pulled).
*/
if (!last_container) {
}
}
/* Update the head index for the container */
/*
* If the entry was found in this container, then continue to
* bail out. Else reset the "curr" pointer and move on to the
* next container (if there is one). Note: the only real
* reason for setting "curr = NULL" here is so that the ASSERT
* below can catch the case where no matching entry was found
* on any of the lists.
*/
if (found) {
break;
} else {
}
}
/*
* Update work queue header's "head" and "full" conditions to match
* the last entry on the container list. (Note: Only if we're pulling
* entries from the last work queue portion of the list, i.e. not from
* the previous portions that may be the "reapable" list.)
*/
if (last_container) {
}
/* Ensure that we've actually found what we were searching for */
return (curr);
}
/*
* tavor_wrid_find_match_srq()
* Context: Can be called from interrupt or base context.
*/
{
/* Grab the WQE addr out of the CQE */
/*
* Given the 'wqe_addr' just calculated and the srq buf address, we
* find the 'wqe_index'. The 'wre' returned below contains the WRID
* that we are looking for. This indexes into the wre_list for this
* specific WQE.
*/
wl->wl_srq_wqesz);
/* ASSERT on impossible wqe_index values */
/* Put this WQE back on the free list */
wl->wl_freel_entries++;
/* Get the descriptor (IO Address) of the WQE to be built */
/* Using the index, return the Work Request ID Entry (wre) */
return (wre);
}
/*
* dapls_tavor_wrid_cq_reap()
*/
void
{
/* dapl_os_assert(MUTEX_HELD(&cq->cq_lock)); */
/* Lock the list of work queues associated with this CQ */
/* Walk the "reapable" list and free up containers */
/*
* If reaping the WRID list containers pulls the last
* container from the given work queue header, then we free
* the work queue header as well.
*/
if (consume_wqhdr != NULL) {
}
}
/* Once finished reaping, we reset the CQ's reap list */
}
/*
* dapls_tavor_wrid_cq_force_reap()
*/
void
{
/* dapl_os_assert(MUTEX_HELD(&cq->cq_lock)); */
/*
* The first step is to walk the "reapable" list and free up those
* containers. This is necessary because the containers on the
* reapable list are not otherwise connected to the work queue headers
* anymore.
*/
/* Now lock the list of work queues associated with this CQ */
/*
* Walk the list of work queue headers and free up all the WRID list
* containers chained to it. Note: We don't need to grab the locks
* for each of the individual WRID lists here because the only way
* things can be added or removed from the list at this point would be
* through post a work request to a QP. But if we've come this far,
* then we can be assured that there are no longer any QP associated
* with the CQ that we are trying to free.
*/
/*
* If reaping the WRID list containers pulls the last
* container from the given work queue header, then
* we free the work queue header as well. Note: we
* ignore the return value because we know that the
* work queue header should always be freed once the
* list of containers has come to an end.
*/
(void) dapli_tavor_wrid_list_reap(to_free);
}
}
}
}
/*
* dapli_tavor_wrid_get_list()
*/
static dapls_tavor_wrid_list_hdr_t *
{
/*
* The WRID list "container" consists of the dapls_tavor_wrid_list_hdr_t
* which holds the pointers necessary for maintaining the "reapable"
* list, chaining together multiple "containers" old and new, and
* tracking the head, tail, size, etc. for each container. The
* "container" also holds all the tavor_wrid_entry_t's, one for
* each entry on the corresponding work queue.
*/
/*
* For wridlist associated with SRQs the wridlock needs to be
* allocated and initialized here.
*/
size = sizeof (dapls_tavor_wrid_list_hdr_t);
if (wrid_for_srq) {
}
goto bail;
}
if (wrid_for_srq) {
} else {
}
goto bail;
}
if (wrid_for_srq) { /* memory for the SRQ free list */
goto bail;
}
}
/* Complete the "container" initialization */
if (wrid_for_srq) {
wridlist->wl_freel_head = 0;
wridlist->wl_freel_tail = 0;
} else {
wridlist->wl_freel_head = 0;
wridlist->wl_freel_tail = 0;
wridlist->wl_freel_entries = 0;
wridlist->wl_srq_wqesz = 0;
wridlist->wl_srq_desc_addr = 0;
}
return (wridlist);
bail:
if (wridlist) {
if (wrid_for_srq) {
}
}
if (wl_wre) {
}
if (wl_freel) {
}
return (NULL);
}
/*
* dapli_tavor_wrid_reaplist_add()
*/
static void
{
/* dapl_os_assert(MUTEX_HELD(&cq->cq_wrid_wqhdr_lock)); */
/*
* Add the "post" container (the last one on the current chain) to
* the CQ's "reapable" list
*/
} else {
}
}
/*
* dapli_tavor_wrid_wqhdr_find()
*/
static dapls_tavor_workq_hdr_t *
{
/* dapl_os_assert(MUTEX_HELD(&cq->cq_wrid_wqhdr_lock)); */
/*
* Walk the CQ's work queue list, trying to find a send or recv queue
* with the same QP number. We do this even if we are going to later
* create a new entry because it helps us easily find the end of the
* list.
*/
if (status == DAT_SUCCESS) {
return ((dapls_tavor_workq_hdr_t *)curr);
} else {
return (NULL);
}
}
/*
* dapli_tavor_wrid_get_wqeaddrsz()
*/
static uint32_t
{
/*
* If the container is empty, then there is no next entry. So just
* return zero. Note: the "head == tail" condition here can only
* mean that the container is empty because we have previously pulled
* something from the container.
*
* If the container is not empty, then find the next entry and return
* the contents of its "wqeaddrsz" field.
*/
wqeaddrsz = 0;
} else {
/*
* We don't need to calculate the "next" head pointer here
* because "head" should already point to the next entry on
* the list (since we just pulled something off - in
* dapli_tavor_wrid_find_match() - and moved the head index
* forward.)
*/
}
return (wqeaddrsz);
}
/*
* dapli_tavor_wrid_list_reap()
* Note: The "wqhdr_list_lock" must be held.
*/
static dapls_tavor_workq_hdr_t *
{
/* Get the back pointer to the work queue header (see below) */
/* Unlink the WRID list "container" from the work queue list */
}
}
/*
* If the back pointer to the work queue header shows that it
* was pointing to the entry we are about to remove, then the work
* queue header is reapable as well.
*/
}
/* Be sure to update the "poll" and "post" container pointers */
}
}
/*
* Calculate the size and free the container, for SRQ wridlist is
* freed when srq gets freed
*/
sizeof (dapls_tavor_wrid_entry_t));
}
}
return (consume_wqhdr);
}
/*
* dapls_tavor_srq_wrid_init()
*/
{
int i;
return (DAT_INSUFFICIENT_RESOURCES | DAT_RESOURCE_MEMORY);
}
/* initialize the free list with the descriptor addresses */
}
return (DAT_SUCCESS);
}
void
{
if (wridlist) {
sizeof (dapls_tavor_wrid_entry_t));
}
if (wridlist->wl_free_list) {
sizeof (uint32_t));
}
size = sizeof (dapls_tavor_wrid_lock_t);
}
sizeof (dapls_tavor_wrid_list_hdr_t));
}
}
/*
* dapls_tavor_wrid_init()
*/
{
uint_t create_new_swq = 0;
uint_t create_new_rwq = 0;
/*
* For each of this QP's Work Queues, make sure we have a (properly
* initialized) Work Request ID list attached to the relevant
* completion queue. Grab the CQ lock(s) before manipulating the
* lists.
*/
/* Couldn't find matching work queue header, create it */
create_new_swq = 1;
/*
* header, then drop the lock(s) and return failure.
*/
return (DAT_INSUFFICIENT_RESOURCES);
}
}
/*
* Allocate space for the dapls_tavor_wrid_entry_t container
*/
if (s_wridlist == NULL) {
/*
* If we couldn't allocate space for tracking the WRID
* entries, then cleanup the workq header from above (if
* necessary, i.e. if we created the workq header). Then
* drop the lock(s) and return failure.
*/
if (create_new_swq) {
}
return (DAT_INSUFFICIENT_RESOURCES | DAT_RESOURCE_MEMORY);
}
/* Chain the new WRID list container to the workq hdr list */
/*
* Now we repeat all the above operations for the receive work queue
*/
create_new_rwq = 1;
/* if qp is attached to an SRQ don't need to alloc wrid_lock */
/*
* header, then free all the send queue resources we
* just allocated and setup (above), drop the lock(s)
* and return failure.
*/
if (create_new_swq) {
swq);
}
return (DAT_INSUFFICIENT_RESOURCES |
}
}
/*
* Allocate space for the dapls_tavor_wrid_entry_t container
* For qp associated with SRQs the SRQ wridlist is used
*/
if (qp->qp_srq_enabled) {
/* Use existing srq_wridlist pointer */
/* store the wl_lock in the wqhdr */
} else {
/* Allocate memory for the r_wridlist */
}
if (r_wridlist == NULL) {
/*
* If we couldn't allocate space for tracking the WRID
* entries, then cleanup all the stuff from above. Then
* drop the lock(s) and return failure.
*/
if (create_new_swq) {
}
if (create_new_rwq) {
}
return (DAT_INSUFFICIENT_RESOURCES | DAT_RESOURCE_MEMORY);
}
/* For SRQ based QPs r_wridlist does not point to recv wqhdr */
if (!qp->qp_srq_enabled) {
}
/* Chain the new WRID list "container" to the workq hdr list */
return (DAT_SUCCESS);
}
/*
* dapls_tavor_wrid_cleanup()
*/
void
{
/*
* For each of this QP's Work Queues, move the WRID "container" to
* the "reapable" list. Although there may still be unpolled
* entries in these containers, it is not a big deal. We will not
* reap the list until either the Poll CQ command detects an empty
* condition or the CQ itself is freed. Grab the CQ lock(s) before
* manipulating the lists.
*/
/*
* Repeat the above operation for the Recv work queue "container".
* However for qps with SRQ we flush the cq entries, remove the
* wridlist and wqhdr.
* Then drop the CQ lock(s) and return
*/
if (qp->qp_srq_enabled) {
/*
* Pull off all (if any) entries for this QP from CQ. This
* only includes entries that have not yet been polled
*/
/* Remove wridlist from WQHDR */
/* Free the WQHDR */
} else {
}
}
/*
* dapli_tavor_wrid_wqhdr_create()
*/
static dapls_tavor_workq_hdr_t *
{
/* dapl_os_assert(MUTEX_HELD(&cq->cq_wrid_wqhdr_lock)); */
/*
* Allocate space for a work queue header structure and initialize it.
* Each work queue header structure includes a "wq_wrid_lock"
* which needs to be initialized.
*
* Note: the address smashing is needed to ensure wq_wrid_lock is
* 8-byte aligned, which is not always the case on 32-bit sparc.
*/
aligned_size = size;
if (alloc_wrl) {
/* for non-srq wqhdr the lock is allocated with the wqhdr */
}
return (NULL);
}
if (alloc_wrl) {
/* wrl allocated with wqhdr don't have srq enabled */
}
/* Chain the newly allocated work queue header to the CQ's list */
if (alloc_wrl) {
wrl_lock);
}
}
return (wqhdr_tmp);
}
/*
* dapli_tavor_wrid_wqhdr_add()
*/
static void
{
/* dapl_os_assert(MUTEX_HELD(&wqhdr->wq_wrid_lock)); */
/* Chain the new WRID list "container" to the work queue list */
} else {
}
}
/*
* dapli_tavor_wrid_wqhdr_remove()
* Note: this is only called to remove the most recently added WRID list
* container.
*/
static void
{
/* dapl_os_assert(MUTEX_HELD(&wqhdr->wq_wrid_lock)); */
/* Unlink the WRID list "container" from the work queue list */
}
}
/*
* Update any pointers in the work queue hdr that may point to this
* WRID list container
*/
}
}
}
/*
* dapli_tavor_wrid_wqhdr_lock_both()
*/
static void
{
/*
* If both work queues (send and recv) share a completion queue, then
* grab the common lock. If they use different CQs (hence different
* "cq_wrid_wqhdr_list" locks), then grab the send one first, then the
* receive. We do this consistently and correctly in
* tavor_wrid_wqhdr_unlock_both() below to avoid introducing any kind
* of dead lock condition.
*/
} else {
}
}
/*
* dapli_tavor_wrid_wqhdr_unlock_both()
*/
static void
{
/*
* See tavor_wrid_wqhdr_lock_both() above for more detail
*/
} else {
}
}
/*
* dapli_tavor_cq_wqhdr_add()
*/
static DAT_RETURN
{
/* dapl_os_assert(MUTEX_HELD(&cq->cq_wrid_wqhdr_lock)); */
/*
* If the CQ's work queue list is empty, then just add it.
* Otherwise, chain it to the beginning of the list.
*/
}
/*
* dapli_tavor_cq_wqhdr_remove
*/
static void
{
/* dapl_os_assert(MUTEX_HELD(&cq->cq_wrid_wqhdr_lock)); */
/* Remove "wqhdr" from the work queue header list on "cq" */
size += sizeof (dapls_tavor_wrid_lock_t);
}
/* Free the memory associated with "wqhdr" */
}
/*
* dapls_tavor_srq_wrid_resize() is called to resize the wridlist
* associated with SRQS as a result of dat_srq_resize().
*
* Returns: DAT_TRUE if successful, otherwise DAT_FALSE
*/
{
uint32_t i;
return (DAT_FALSE);
}
sizeof (dapls_tavor_wrid_entry_t));
if (new_wl_wre == NULL) {
goto bail;
}
if (new_wl_freel == NULL) {
goto bail;
}
/*
* we just need to copy the old WREs to the new array. Since the
* descriptors are relatively addressed the descriptor to index
* mapping doesn't change.
*/
old_size * sizeof (dapls_tavor_wrid_entry_t));
/*
* Copy the old free list to the new one
*/
for (i = 0; i < wridlist->wl_freel_entries; i++) {
}
/*
* Add the new entries in wl_wre to the new free list
*/
}
wridlist->wl_freel_head = 0;
if (old_wl_wre) {
sizeof (dapls_tavor_wrid_entry_t));
}
if (old_wl_freel) {
}
return (DAT_TRUE);
bail:
if (new_wl_wre) {
sizeof (dapls_tavor_wrid_entry_t));
}
return (DAT_FALSE);
}