ib_send.c revision c0dd49bdd68c0d758a67d56f07826f3b45cfc664
/*
* CDDL HEADER START
*
* The contents of this file are subject to the terms of the
* Common Development and Distribution License (the "License").
* You may not use this file except in compliance with the License.
*
* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
* See the License for the specific language governing permissions
* and limitations under the License.
*
* When distributing Covered Code, include this CDDL HEADER in each
* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
* If applicable, add the following below this CDDL HEADER, with the
* fields enclosed by brackets "[]" replaced with your own identifying
* information: Portions Copyright [yyyy] [name of copyright owner]
*
* CDDL HEADER END
*/
/*
*/
/*
* Copyright (c) 2006 Oracle. All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
* General Public License (GPL) Version 2, available from the file
* COPYING in the main directory of this source tree, or the
* OpenIB.org BSD license below:
*
* Redistribution and use in source and binary forms, with or
* without modification, are permitted provided that the following
* conditions are met:
*
* - Redistributions of source code must retain the above
* copyright notice, this list of conditions and the following
* disclaimer.
*
* - Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following
* provided with the distribution.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*
*/
static void
int wc_status)
{
int notify_status;
switch (wc_status) {
case IBT_WC_WR_FLUSHED_ERR:
return;
case IBT_WC_SUCCESS:
break;
case IBT_WC_REMOTE_ACCESS_ERR:
break;
default:
break;
}
}
void
struct rdsv3_rdma_op *op)
{
} else {
}
}
}
static void
struct rdsv3_ib_send_work *send,
int wc_status)
{
/*
* If the user asked for a completion notification on this
* message, we can implement three different semantics:
* 1. Notify when we received the ACK on the RDS message
* that was queued with the RDMA. This provides reliable
* notification of RDMA status at the expense of a one-way
* packet delay.
* 2. Notify when the IB stack gives us the completion
* event for the RDMA operation.
* 3. Notify when the IB stack gives us the completion
* event for the accompanying RDS messages.
* Here, we implement approach #3. To implement approach #2,
* call rdsv3_rdma_send_complete from the cq_handler.
* To implement #1,
* don't call rdsv3_rdma_send_complete at all, and fall back to
* the notify
* handling in the ACK processing code.
*
* Note: There's no need to explicitly sync any RDMA buffers
* using
* ib_dma_sync_sg_for_cpu - the completion for the RDMA
* operation itself unmapped the RDMA buffers, which takes care
* of synching.
*/
else
}
/*
* If anyone waited for this message to get flushed out, wake
* them up now
*/
}
void
{
struct rdsv3_ib_send_work *send;
uint32_t i;
}
}
void
{
struct rdsv3_ib_send_work *send;
uint32_t i;
continue;
}
}
/*
* operations performed in the send path. As the sender allocs and potentially
* unallocs the next free entry in the ring it doesn't alter which is
* the next to be freed, which is what this is concerned with.
*/
void
{
struct rdsv3_ib_send_work *send;
uint32_t i = 0;
int ret;
if (ret)
RDSV3_DPRINTF2("rdsv3_ib_send_cq_comp_handler",
"ib_req_notify_cq send failed: %d", ret);
IBT_SUCCESS) {
RDSV3_DPRINTF5("rdsv3_ib_send_cq_comp_handler",
"swc wr_id 0x%llx status %u byte_len %u imm_data %u\n",
continue;
}
for (i = 0; i < completed; i++) {
/*
* In the error case, wc.opcode sometimes contains
* garbage
*/
case IBT_WRC_SEND:
break;
case IBT_WRC_RDMAW:
case IBT_WRC_RDMAR:
/*
* Nothing to be done - the SG list will
* be unmapped
* when the SEND completes.
*/
break;
default:
#ifndef __lock_lint
RDSV3_DPRINTF0("rdsv3_ib_send_cq_comp_handler",
"0x%x in WR!",
#endif
break;
}
/*
* If a RDMA operation produced an error, signal
* this right
* away. If we don't, the subsequent SEND that goes
* with this
* RDMA will be canceled with ERR_WFLUSH, and the
* application
* never learn that the RDMA failed.
*/
struct rdsv3_message *rm;
if (rm) {
}
}
}
/* We expect errors as the qp is drained during shutdown */
RDSV3_DPRINTF2("rdsv3_ib_send_cq_comp_handler",
"send completion on %u.%u.%u.%u "
"had status %u, disconnecting and reconnecting\n",
}
}
RDSV3_DPRINTF4("rdsv3_ib_send_cq_comp_handler",
}
/*
* This is the main function for allocating credits when sending
* messages.
*
* Conceptually, we have two counters:
* - send credits: this tells us how many WRs we're allowed
* to submit without overruning the reciever's queue. For
* each SEND WR we post, we decrement this by one.
*
* - posted credits: this tells us how many WRs we recently
* posted to the receive queue. This value is transferred
* to the peer as a "credit update" in a RDS header field.
* Every time we transmit credits to the peer, we subtract
* the amount of transferred credits from this counter.
*
* It is essential that we avoid situations where both sides have
* exhausted their send credits, and are unable to send new credits
* to the peer. We achieve this by requiring that we send at least
* one credit update to the peer before exhausting our credits.
* When new credits arrive, we subtract one credit that is withheld
* until we've posted new buffers and are ready to transmit these
* credits (see rdsv3_ib_send_add_credits below).
*
* The RDS send code is essentially single-threaded; rdsv3_send_xmit
* grabs c_send_lock to ensure exclusive access to the send ring.
* However, the ACK sending code is independent and can race with
* message SENDs.
*
* In the send path, we need to update the counters for send credits
* and the counter of posted buffers atomically - when we use the
* last available credit, we cannot allow another thread to race us
* and grab the posted credits counter. Hence, we have to use a
* spinlock to protect the credit counter, or use atomics.
*
* Spinlocks shared between the send and the receive path are bad,
* because they create unnecessary delays. An early implementation
* using a spinlock showed a 5% degradation in throughput at some
* loads.
*
* This implementation avoids spinlocks completely, putting both
* counters into a single atomic, and updating that atomic using
* atomic_add (in the receive path, when receiving fresh credits),
* and using atomic_cmpxchg when updating the two counters.
*/
int
{
*adv_credits = 0;
return (wanted);
advertise = 0;
RDSV3_DPRINTF5("rdsv3_ib_send_grab_credits",
/* The last credit must be used to send a credit update. */
avail--;
/* Oops, there aren't that many credits left! */
} else {
/* Sometimes you get what you want, lalala. */
}
/*
* If need_posted is non-zero, then the caller wants
* the posted regardless of whether any send credits are
* available.
*/
}
/* Finally bill everything */
goto try_again;
*adv_credits = advertise;
return (got);
}
void
{
if (credits == 0)
return;
RDSV3_DPRINTF5("rdsv3_ib_send_add_credits",
"credits (%u): current=%u%s\n",
", ll_send_full" : "");
RDSV3_DPRINTF4("rdsv3_ib_send_add_credits",
"Return: conn: %p, credits: %d",
}
void
{
if (posted == 0)
return;
/*
* Decide whether to send an update to the peer now.
* If we would send a credit update for every single buffer we
* post, we would end up with an ACK storm (ACK arrives,
* consumes buffer, we refill the ring, send ACK to remote
* advertising the newly posted buffer... ad inf)
*
* Performance pretty much depends on how often we send
* credit updates - too frequent updates mean lots of ACKs.
* Too infrequent updates, and the peer will run out of
* credits and has to throttle.
* For the time being, 16 seems to be a good compromise.
*/
}
static inline void
int send_flags)
{
RDSV3_DPRINTF4("rdsv3_ib_xmit_populate_wr",
"ic: %p, wr: %p scat: %p %d %d %d %d",
if (length != 0) {
if (off != 0) {
/* find the right sgl to begin with */
sgl++;
}
}
assigned = 0;
do {
if (len != 0) {
sgl++;
off = 0;
}
} while (len > 0);
} else {
/*
* We're sending a packet with no payload. There is only
* one SGE
*/
}
RDSV3_DPRINTF4("rdsv3_ib_xmit_populate_wr",
}
/*
* This can be called multiple times for a given message. The first time
* we see a message we map its scatterlist into the IB device so that
* we can provide that mapped address to the IB scatter gather entries
* in the IB work requests. We translate the scatterlist into a series
* of work requests that fragment the message. These work requests complete
* in order so we pass ownership of the message to the completion handler
* once we send the final fragment.
*
* The RDS core uses the c_send_lock to only enter this function once
* don't get out of sync and confuse the ring.
*/
int
{
struct rdsv3_ib_send_work *first;
struct rdsv3_ib_send_work *prev;
struct rdsv3_scatterlist *scat;
uint32_t i;
uint32_t adv_credits = 0;
int send_flags = 0;
int sent;
int ret;
int flow_controlled = 0;
/* Do not send cong updates to IB loopback */
if (conn->c_loopback &&
return (sizeof (struct rdsv3_header) + RDSV3_CONG_MAP_BYTES);
}
#ifndef __lock_lint
/* FIXME we may overallocate here */
i = 1;
else
#endif
if (work_alloc == 0) {
goto out;
}
&posted, 0, RDSV3_MAX_ADV_CREDIT);
adv_credits += posted;
if (credit_alloc < work_alloc) {
}
if (work_alloc == 0) {
goto out;
}
}
/* map the message the first time we see it */
/*
* printk(KERN_NOTICE
* "rdsv3_ib_xmit prep msg dport=%u flags=0x%x len=%d\n",
* be16_to_cpu(rm->m_inc.i_hdr.h_dport),
* rm->m_inc.i_hdr.h_flags,
* be32_to_cpu(rm->m_inc.i_hdr.h_len));
*/
RDSV3_DPRINTF5("rdsv3_ib_xmit",
RDSV3_DPRINTF2("rdsv3_ib_xmit",
"fail: ic %p mapping rm %p: %d\n",
goto out;
}
} else {
}
/* Finalize the header */
/*
* If it has a RDMA op, tell the peer we did it. This is
* used by the peer to release use-once RDMA MRs.
*/
struct rdsv3_ext_header_rdma ext_hdr;
sizeof (ext_hdr));
}
if (rm->m_rdma_cookie) {
}
/*
* Note - rdsv3_ib_piggyb_ack clears the ACK_REQUIRED bit, so
* we should not do this unless we have a chance of at least
* sticking the header into the send ring. Which is why we
* should call rdsv3_ib_ring_alloc first.
*/
/*
* Update adv_credits since we reset the ACK_REQUIRED bit.
*/
adv_credits += posted;
RDSV3_PANIC();
sent = 0;
i = 0;
/*
* Sometimes you want to put a fence between an RDMA
* READ and the following SEND.
* We could either do this all the time
* or when requested by the user. Right now, we let
* the application choose.
*/
/*
* We could be copying the header into the unused tail of the page.
* That would need to be changed in the future when those pages might
* be mapped userspace pages or page cache pages. So instead we always
* use a second sge and our long-lived ring of mapped headers. We send
* the header after the data so that the data payload can be aligned on
* the receiver.
*/
/* handle a 0-len message */
goto add_header;
}
/* if there's data reference it with a chain of work reqs */
unsigned int len;
/*
* We want to delay signaling completions just enough to get
* the batching benefits but not so much that we create dead
* time
* on the wire.
*/
if (ic->i_unsignaled_wrs-- == 0) {
}
if (ic->i_unsignaled_bytes <= 0) {
}
/*
* Always signal the last one if we're stopping due to flow
* control.
*/
}
scat++;
off = 0;
}
/*
* Tack on the header after the data. The header SGE
* should already
* have been set up to point to the right header buffer.
*/
sizeof (struct rdsv3_header));
if (0) {
RDSV3_DPRINTF0("rdsv3_ib_xmit",
"send WR dport=%u flags=0x%x len=%d",
}
if (adv_credits) {
/* add credit and redo the header checksum */
adv_credits = 0;
}
}
/*
* Account the RDS header in the number of bytes we sent, but just once.
* The caller has no concept of fragmentation.
*/
if (hdr_off == 0)
sent += sizeof (struct rdsv3_header);
/* if we finished the message then send completion owns it */
}
if (i < work_alloc) {
work_alloc = i;
}
/* XXX need to worry about failed_wr and partial sends. */
if (posted != i) {
RDSV3_DPRINTF1("rdsv3_ib_xmit",
"ic %p first %p nwr: %d ret %d:%d",
}
if (ret) {
RDSV3_DPRINTF0("rdsv3_ib_xmit",
}
#if 1
#else
/* Finesse this later */
RDSV3_PANIC();
#endif
goto out;
}
out:
return (ret);
}
static void
struct rdsv3_rdma_sg scat[])
{
int i;
int num_sgl;
if (dev) {
} else {
RDSV3_DPRINTF2("rdsv3_ib_dma_unmap_sg_rdma",
"NULL dev use cached hca_hdl %p", hca_hdl);
}
return;
for (i = 0; i < num; i++) {
(num_sgl * sizeof (ibt_wr_ds_t)));
} else
break;
}
}
/* ARGSUSED */
{
uint_t i, j, k;
struct rdsv3_scatterlist *sg;
int ret;
iov_attr.iov_lso_hdr_sz = 0;
/* transpose umem_cookie to buf structure */
/* free resources and return error */
goto out;
}
/* setup ibt_map_mem_iov() attributes */
KM_SLEEP);
if (ret != IBT_SUCCESS) {
RDSV3_DPRINTF2("rdsv3_ib_dma_map_sg_rdma",
"ibt_map_mem_iov returned: %d", ret);
/* free resources and return error */
goto out;
}
#ifdef DEBUG
RDSV3_DPRINTF5("rdsv3_ib_dma_map_sg_rdma",
"sgl[%d] va %llx len %x", j,
}
#endif
RDSV3_DPRINTF4("rdsv3_ib_dma_map_sg_rdma",
"iovec.bytes: 0x%x scat[%d]swr.wr_nds: %d",
}
return (count);
out:
return (0);
}
int
{
struct rdsv3_rdma_sg *scat;
int sent;
/* map the message the first time we see it */
RDSV3_DPRINTF2("rdsv3_ib_xmit_rdma",
"fail: ic %p mapping op %p: %d",
return (-ENOMEM); /* XXX ? */
}
}
/*
* we insist that there
* be enough work requests to send the entire message.
*/
return (-ENOMEM);
}
/*
* take the scatter list and transpose into a list of
* send wr's each with a scatter list of RDSV3_IB_MAX_SGE
*/
sent = 0;
if (left > RDSV3_IB_MAX_SGE) {
left -= RDSV3_IB_MAX_SGE;
} else {
left = 0;
}
for (j = 0; j < count; j++) {
idx++;
RDSV3_DPRINTF4("xmit_rdma",
"send_wrs[%d]sgl[%d] va %llx len %x",
}
RDSV3_DPRINTF4("rdsv3_ib_xmit_rdma",
"wr[%d] %p key: %x code: %d tlen: %d",
/*
* We want to delay signaling completions just enough
* to get the batching benefits but not so much that
* we create dead time on the wire.
*/
if (ic->i_unsignaled_wrs-- == 0) {
}
}
}
if (status != IBT_SUCCESS) {
RDSV3_DPRINTF0("rdsv3_ib_xmit_rdma",
}
return (status);
}
void
{
/*
* We may have a pending ACK or window update we were unable
* to send previously (due to flow control). Try again.
*/
}