tavor_srq.c revision 9e39c5ba00a55fa05777cc94b148296af305e135
/*
* CDDL HEADER START
*
* The contents of this file are subject to the terms of the
* Common Development and Distribution License (the "License").
* You may not use this file except in compliance with the License.
*
* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
* See the License for the specific language governing permissions
* and limitations under the License.
*
* When distributing Covered Code, include this CDDL HEADER in each
* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
* If applicable, add the following below this CDDL HEADER, with the
* fields enclosed by brackets "[]" replaced with your own identifying
* information: Portions Copyright [yyyy] [name of copyright owner]
*
* CDDL HEADER END
*/
/*
* Copyright 2007 Sun Microsystems, Inc. All rights reserved.
* Use is subject to license terms.
*/
/*
* Tavor Shared Receive Queue Processing Routines
*
* Implements all the routines necessary for allocating, freeing, querying,
* modifying and posting shared receive queues.
*/
/*
* Used by tavor_srq_numcalc() below to fill in the "unconstrained" portion of
* Tavor shared receive queue number
*/
/*
* tavor_srq_alloc()
* Context: Can be called only from user or kernel context.
*/
int
{
char *errormsg;
/*
* Check the "options" flag. Currently this flag tells the driver
* whether or not the SRQ's work queues should be come from normal
* system memory or whether they should be allocated from DDR memory.
*/
} else {
}
/*
* Extract the necessary info from the tavor_srq_info_t structure
*/
/*
* Determine whether SRQ is being allocated for userland access or
* whether it is being allocated for kernel access. If the SRQ is
* being allocated for userland access, then lookup the UAR doorbell
* page number for the current process. Note: If this is not found
* (e.g. if the process has not previously open()'d the Tavor driver),
* then an error is returned.
*/
if (srq_is_umap) {
if (status != DDI_SUCCESS) {
/* Set "status" and "errormsg" and goto failure */
goto srqalloc_fail3;
}
}
/* Increase PD refcnt */
/* Allocate an SRQ context entry */
if (status != DDI_SUCCESS) {
/* Set "status" and "errormsg" and goto failure */
goto srqalloc_fail1;
}
/* Allocate the SRQ Handle entry */
if (status != DDI_SUCCESS) {
/* Set "status" and "errormsg" and goto failure */
goto srqalloc_fail2;
}
/* Calculate the SRQ number */
/*
* If this will be a user-mappable SRQ, then allocate an entry for
* the "userland resources database". This will later be added to
* the database (after all further SRQ operations are successful).
* If we fail here, we must undo the reference counts and the
* previous resource allocation.
*/
if (srq_is_umap) {
/* Set "status" and "errormsg" and goto failure */
goto srqalloc_fail3;
}
}
/*
* Calculate the appropriate size for the SRQ.
* Note: All Tavor SRQs must be a power-of-2 in size. Also
* they may not be any smaller than TAVOR_SRQ_MIN_SIZE. This step
* is to round the requested size up to the next highest power-of-2
*/
}
/*
* Next we verify that the rounded-up size is valid (i.e. consistent
* then obviously we have a lot of cleanup to do before returning.
*/
/* Set "status" and "errormsg" and goto failure */
goto srqalloc_fail4;
}
/*
* Next we verify that the requested number of SGL is valid (i.e.
* limits). If not, then obviously the same cleanup needs to be done.
*/
/* Set "status" and "errormsg" and goto failure */
goto srqalloc_fail4;
}
/*
* Determine the SRQ's WQE sizes. This depends on the requested
* number of SGLs. Note: This also has the side-effect of
* calculating the real number of SGLs (for the calculated WQE size)
*/
&srq->srq_wq_sgl);
/*
* Allocate the memory for SRQ work queues. Note: The location from
* which we will allocate these work queues has been passed in through
* the tavor_qp_options_t structure. Since Tavor work queues are not
* allowed to cross a 32-bit (4GB) boundary, the alignment of the work
* queue memory is very important. We used to allocate work queues
* (the combined receive and send queues) so that they would be aligned
* on their combined size. That alignment guaranteed that they would
* never cross the 4GB boundary (Tavor work queues are on the order of
* MBs at maximum). Now we are able to relax this alignment constraint
* by ensuring that the IB address assigned to the queue memory (as a
* result of the tavor_mr_register() call) is offset from zero.
* Previously, we had wanted to use the ddi_dma_mem_alloc() routine to
* guarantee the alignment, but when attempting to use IOMMU bypass
* mode we found that we were not allowed to specify any alignment that
* was more restrictive than the system page size. So we avoided this
* constraint by passing two alignment values, one for the memory
* allocation itself and the other for the DMA handle (for later bind).
* This used to cause more memory than necessary to be allocated (in
* order to guarantee the more restrictive alignment contraint). But
* be guaranteeing the zero-based IB virtual address for the queue, we
* are able to conserve this memory.
*
* Note: If SRQ is not user-mappable, then it may come from either
* kernel system memory or from HCA-attached local DDR memory.
*
* Note2: We align this queue on a pagesize boundary. This is required
* to make sure that all the resulting IB addresses will start at 0, for
* a zero-based queue. By making sure we are aligned on at least a
* page, any offset we use into our queue will be the same as when we
* perform tavor_srq_modify() operations later.
*/
if (srq_is_umap) {
} else {
}
if (status != DDI_SUCCESS) {
/* Set "status" and "errormsg" and goto failure */
goto srqalloc_fail4;
}
/*
* Register the memory for the SRQ work queues. The memory for the SRQ
* must be registered in the Tavor TPT tables. This gives us the LKey
* to specify in the SRQ context later. Note: If the work queue is to
* be allocated from DDR memory, then only a "bypass" mapping is
* appropriate. And if the SRQ memory is user-mappable, then we force
* DDI_DMA_CONSISTENT mapping. Also, in order to meet the alignment
* restriction, we pass the "mro_bind_override_addr" flag in the call
* to tavor_mr_register(). This guarantees that the resulting IB vaddr
* will be zero-based (modulo the offset into the first page). If we
* fail here, we still have the bunch of resource and reference count
* cleanup to do.
*/
if (srq_is_umap) {
} else {
if (wq_location == TAVOR_QUEUE_LOCATION_NORMAL) {
if (dma_xfer_mode == DDI_DMA_STREAMING) {
}
} else {
}
}
if (status != DDI_SUCCESS) {
/* Set "status" and "errormsg" and goto failure */
goto srqalloc_fail5;
}
/*
* Calculate the offset between the kernel virtual address space
* and the IB virtual address space. This will be used when
* posting work requests to properly initialize each WQE.
*/
/*
* Create WQL and Wridlist for use by this SRQ
*/
/* Set "status" and "errormsg" and goto failure */
goto srqalloc_fail6;
}
/* Set "status" and "errormsg" and goto failure */
goto srqalloc_fail7;
}
/*
* Fill in all the return arguments (if necessary). This includes
* real queue size and real SGLs.
*/
if (real_sizes != NULL) {
}
/*
* Fill in the SRQC entry. This is the final step before passing
* ownership of the SRQC entry to the Tavor hardware. We use all of
* the information collected/calculated above to fill in the
* requisite portions of the SRQC. Note: If this SRQ is going to be
* used for userland access, then we need to set the UAR page number
* appropriately (otherwise it's a "don't care")
*/
srqc_entry.wqe_cnt = 0;
if (srq_is_umap) {
} else {
srqc_entry.uar = 0;
}
/*
* Write the SRQC entry to hardware. Lastly, we pass ownership of
* the entry to the hardware (using the Tavor SW2HW_SRQ firmware
* command). Note: In general, this operation shouldn't fail. But
* if it does, we have to undo everything we've done above before
* returning error.
*/
if (status != TAVOR_CMD_SUCCESS) {
status);
/* Set "status" and "errormsg" and goto failure */
goto srqalloc_fail8;
}
/*
* Fill in the rest of the Tavor SRQ handle. We can update
* the following fields for use in further operations on the SRQ.
*/
srq->srq_refcnt = 0;
/* Determine if later ddi_dma_sync will be necessary */
/*
* Put SRQ handle in Tavor SRQNum-to-SRQhdl list. Then fill in the
* "srqhdl" and return success
*/
/*
* If this is a user-mappable SRQ, then we need to insert the
* previously allocated entry into the "userland resources database".
* This will allow for later lookup during devmap() (i.e. mmap())
* calls.
*/
if (srq->srq_is_umap) {
} else {
}
return (status);
/*
* The following is cleanup for all possible failure cases in this routine
*/
sizeof (tavor_wrid_entry_t));
TAVOR_SLEEPFLAG_FOR_CONTEXT()) != DDI_SUCCESS) {
}
if (srq_is_umap) {
}
return (status);
}
/*
* tavor_srq_free()
* Context: Can be called only from user or kernel context.
*/
/* ARGSUSED */
int
{
int status;
/*
* Pull all the necessary information from the Tavor Shared Receive
* Queue handle. This is necessary here because the resource for the
* SRQ handle is going to be freed up as part of this operation.
*/
/*
* If there are work queues still associated with the SRQ, then return
* an error. Otherwise, we will be holding the SRQ lock.
*/
if (srq->srq_refcnt != 0) {
return (IBT_SRQ_IN_USE);
}
/*
* If this was a user-mappable SRQ, then we need to remove its entry
* from the "userland resources database". If it is also currently
* mmap()'d out to a user process, then we need to call
* devmap_devmem_remap() to remap the SRQ memory to an invalid mapping.
* We also need to invalidate the SRQ tracking information for the
* user mapping.
*/
if (srq->srq_is_umap) {
&umapdb);
if (status != DDI_SUCCESS) {
return (ibc_get_ci_failure(0));
}
if (status != DDI_SUCCESS) {
"devmap_devmem_remap()");
return (ibc_get_ci_failure(0));
}
}
}
/*
* Put NULL into the Tavor SRQNum-to-SRQHdl list. This will allow any
* in-progress events to detect that the SRQ corresponding to this
* number has been freed.
*/
/*
* Reclaim SRQC entry from hardware (using the Tavor HW2SW_SRQ
* firmware command). If the ownership transfer fails for any reason,
* then it is an indication that something (either in HW or SW) has
* gone seriously wrong.
*/
if (status != TAVOR_CMD_SUCCESS) {
status);
return (IBT_FAILURE);
}
/*
* Deregister the memory for the Shared Receive Queue. If this fails
* for any reason, then it is an indication that something (either
* in HW or SW) has gone seriously wrong. So we print a warning
* message and return.
*/
if (status != DDI_SUCCESS) {
return (IBT_FAILURE);
}
/* Calculate the size and free the wridlist container */
sizeof (tavor_wrid_entry_t));
/*
* Release reference to WQL; If this is the last reference,
* this call also has the side effect of freeing up the
* 'srq_wrid_wql' memory.
*/
}
/* Free the memory for the SRQ */
/* Free the Tavor SRQ Handle */
/* Free the SRQC entry resource */
/* Decrement the reference count on the protection domain (PD) */
/* Set the srqhdl pointer to NULL and return success */
return (DDI_SUCCESS);
}
/*
* tavor_srq_modify()
* Context: Can be called only from user or kernel context.
*/
int
{
int status;
char *errormsg;
/*
* Check the "inddr" flag. This flag tells the driver whether or not
* the SRQ's work queues should be come from normal system memory or
* whether they should be allocated from DDR memory.
*/
/*
* If size requested is larger than device capability, return
* Insufficient Resources
*/
if (size > max_srq_size) {
TAVOR_TNF_ERROR, "");
return (IBT_HCA_WR_EXCEEDED);
}
/*
* Calculate the appropriate size for the SRQ.
* Note: All Tavor SRQs must be a power-of-2 in size. Also
* they may not be any smaller than TAVOR_SRQ_MIN_SIZE. This step
* is to round the requested size up to the next highest power-of-2
*/
}
/*
* Next we verify that the rounded-up size is valid (i.e. consistent
*/
/* Set "status" and "errormsg" and goto failure */
goto srqmodify_fail;
}
/*
* Allocate the memory for newly resized Shared Receive Queue.
*
* Note: If SRQ is not user-mappable, then it may come from either
* kernel system memory or from HCA-attached local DDR memory.
*
* Note2: We align this queue on a pagesize boundary. This is required
* to make sure that all the resulting IB addresses will start at 0,
* for a zero-based queue. By making sure we are aligned on at least a
* page, any offset we use into our queue will be the same as it was
* when we allocated it at tavor_srq_alloc() time.
*/
if (srq->srq_is_umap) {
} else {
}
if (status != DDI_SUCCESS) {
/* Set "status" and "errormsg" and goto failure */
goto srqmodify_fail;
}
/*
* Allocate the memory for the new WRE list. This will be used later
* when we resize the wridlist based on the new SRQ size.
*/
sizeof (tavor_wrid_entry_t), sleepflag);
/* Set "status" and "errormsg" and goto failure */
"failed wre_new alloc");
goto srqmodify_fail;
}
/*
* Fill in the "bind" struct. This struct provides the majority
* of the information that will be used to distinguish between an
* "addr" binding (as is the case here) and a "buf" binding (see
* below). The "bind" struct is later passed to tavor_mr_mem_bind()
* which does most of the "heavy lifting" for the Tavor memory
* registration routines.
*/
if (srq->srq_is_umap) {
} else {
if (wq_location == TAVOR_QUEUE_LOCATION_NORMAL) {
if (dma_xfer_mode == DDI_DMA_STREAMING) {
}
} else {
}
}
if (status != DDI_SUCCESS) {
/* Set "status" and "errormsg" and goto failure */
sizeof (tavor_wrid_entry_t));
goto srqmodify_fail;
}
/*
* Calculate the offset between the kernel virtual address space
* and the IB virtual address space. This will be used when
* posting work requests to properly initialize each WQE.
*
* Note: bind addr is zero-based (from alloc) so we calculate the
* correct new offset here.
*/
/*
* Get the base address for the MTT table. This will be necessary
* below when we are modifying the MPT entry.
*/
/*
* Fill in the MPT entry. This is the final step before passing
* ownership of the MPT entry to the Tavor hardware. We use all of
* the information collected/calculated above to fill in the
* requisite portions of the MPT.
*/
/*
* Now we grab the SRQ lock. Since we will be updating the actual
* the lock.
*
* We do a TAVOR_NOSLEEP here (and below), though, because we are
* holding the "srq_lock" and if we got raised to interrupt level
* by priority inversion, we would not want to block in this routine
* waiting for success.
*/
/*
* Copy old entries to new buffer
*/
/* Determine if later ddi_dma_sync will be necessary */
/* Sync entire "new" SRQ for use by hardware (if necessary) */
if (srq_sync) {
}
/*
* Setup MPT information for use in the MODIFY_MPT command
*/
/*
* MODIFY_MPT
*
* If this fails for any reason, then it is an indication that
* something (either in HW or SW) has gone seriously wrong. So we
* print a warning message and return.
*/
if (status != TAVOR_CMD_SUCCESS) {
status);
sizeof (tavor_wrid_entry_t));
return (ibc_get_ci_failure(0));
}
/*
* Update the Tavor Shared Receive Queue handle with all the new
* information. At the same time, save away all the necessary
* information for freeing up the old resources
*/
sizeof (tavor_bind_info_t));
/* Now set the new info */
/* Update MR mtt pagesize */
#ifdef __lock_lint
#else
}
#endif
/*
* Initialize new wridlist, if needed.
*
* If a wridlist already is setup on an SRQ (the QP associated with an
* SRQ has moved "from_reset") then we must update this wridlist based
* on the new SRQ size. We allocate the new size of Work Request ID
* Entries, copy over the old entries to the new list, and
* re-initialize the srq wridlist in non-umap case
*/
sizeof (tavor_wrid_entry_t));
/* Setup new sizes in wre */
if (!srq->srq_is_umap) {
}
}
#ifdef __lock_lint
#else
}
#endif
/*
* If "old" SRQ was a user-mappable SRQ that is currently mmap()'d out
* to a user process, then we need to call devmap_devmem_remap() to
* invalidate the mapping to the SRQ memory. We also need to
* invalidate the SRQ tracking information for the user mapping.
*
* Note: On failure, the remap really shouldn't ever happen. So, if it
* does, it is an indication that something has gone seriously wrong.
* So we print a warning message and return error (knowing, of course,
* that the "old" SRQ memory will be leaked)
*/
if (status != DDI_SUCCESS) {
"devmap_devmem_remap()");
/* We can, however, free the memory for old wre */
sizeof (tavor_wrid_entry_t));
}
return (ibc_get_ci_failure(0));
}
}
/*
* Drop the SRQ lock now. The only thing left to do is to free up
* the old resources.
*/
/*
* Unbind the MTT entries.
*/
if (status != DDI_SUCCESS) {
/* Set "status" and "errormsg" and goto failure */
"failed to unbind (old)");
goto srqmodify_fail;
}
/* Free the memory for old wre */
sizeof (tavor_wrid_entry_t));
}
/* Free the memory for the old SRQ */
/*
* Fill in the return arguments (if necessary). This includes the
* real new completion queue size.
*/
}
return (DDI_SUCCESS);
return (status);
}
/*
* tavor_srq_numcalc()
* Context: Can be called from interrupt or base context.
*/
static void
{
/*
* Generate a simple key from counter. Note: We increment this
* static variable _intentionally_ without any kind of mutex around
* it. First, single-threading all operations through a single lock
* would be a bad idea (from a performance point-of-view). Second,
* the upper "unconstrained" bits don't really have to be unique
* because the lower bits are guaranteed to be (although we do make a
* best effort to ensure that they are). Third, the window for the
* race (where both threads read and update the counter at the same
* time) is incredibly small.
*/
}
/*
* tavor_srq_refcnt_inc()
* Context: Can be called from interrupt or base context.
*/
void
{
srq->srq_refcnt++;
}
/*
* tavor_srq_refcnt_dec()
* Context: Can be called from interrupt or base context.
*/
void
{
srq->srq_refcnt--;
}
/*
* tavor_srqhdl_from_srqnum()
* Context: Can be called from interrupt or base context.
*
* This routine is important because changing the unconstrained
* portion of the SRQ number is critical to the detection of a
* potential race condition in the SRQ handler code (i.e. the case
* where a SRQ is freed and alloc'd again before an event for the
* "old" SRQ can be handled).
*
* While this is not a perfect solution (not sure that one exists)
* it does help to mitigate the chance that this race condition will
* cause us to deliver a "stale" event to the new SRQ owner. Note:
* this solution does not scale well because the number of constrained
* bits increases (and, hence, the number of unconstrained bits
* decreases) as the number of supported SRQ grows. For small and
* intermediate values, it should hopefully provide sufficient
* protection.
*/
{
/* Calculate the SRQ table index from the srqnum */
}
/*
* tavor_srq_sgl_to_logwqesz()
* Context: Can be called from interrupt or base context.
*/
static void
{
switch (wq_type) {
case TAVOR_QP_WQ_TYPE_RECVQ:
/*
* Use requested maximum SGL to calculate max descriptor size
* (while guaranteeing that the descriptor size is a
* power-of-2 cachelines).
*/
}
/* Make sure descriptor is at least the minimum size */
/* Calculate actual number of SGL (given WQE size) */
break;
default:
TAVOR_TNF_ERROR, "");
break;
}
/* Fill in the return values */
}