/*
*/
/*
* This file contains code imported from the OFED rds source file send.c
* Oracle elects to have and use the contents of send.c under and governed
* by the OpenIB.org BSD license (see below for full license text). However,
* the following notice accompanied the original version of this file:
*/
/*
* Copyright (c) 2006 Oracle. All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
* General Public License (GPL) Version 2, available from the file
* COPYING in the main directory of this source tree, or the
* OpenIB.org BSD license below:
*
* Redistribution and use in source and binary forms, with or
* without modification, are permitted provided that the following
* conditions are met:
*
* - Redistributions of source code must retain the above
* copyright notice, this list of conditions and the following
* disclaimer.
*
* - Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following
* provided with the distribution.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*
*/
#include <sys/socketvar.h>
/*
* When transmitting messages in rdsv3_send_xmit, we need to emerge from
* time to time and briefly release the CPU. Otherwise the softlock watchdog
* will kick our shin.
* Also, it seems fairer to not let one busy connection stall all the
* others.
*
* send_batch_count is the number of times we'll loop in send_xmit. Setting
* it to 0 will restore the old behavior (where we looped until we had
* drained the queue).
*/
/*
* Reset the send state. Caller must hold c_send_lock when calling here.
*/
void
{
RDSV3_DPRINTF2("rdsv3_send_reset",
"rm %p mflg 0x%x map %d mihdl %p sgl %p",
}
/*
* Tell the user the RDMA op is no longer mapped by the
* transport. This isn't entirely true (it's flushed out
* independently) but as the connection is down, there's
*/
}
conn->c_xmit_hdr_off = 0;
conn->c_xmit_data_off = 0;
conn->c_xmit_rdma_sent = 0;
conn->c_map_queued = 0;
/* Mark messages as retransmissions, and move them to the send q */
RDSV3_DPRINTF4("_send_reset",
"RT rm %p mflg 0x%x sgl %p",
}
}
}
/*
* We're making the concious trade-off here to only send one message
* down the connection at a time.
* Pro:
* - tx queueing is a simple fifo list
* - reassembly is optional and easily done by transports per conn
* - no per flow rx lookup at all, straight to the socket
* - less per-frag memory and wire overhead
* Con:
* - queued acks can be delayed behind large messages
* Depends:
* - small message latency is higher behind queued large messages
* - large message latency isn't starved by intervening small sends
*/
int
{
unsigned int tmp;
int ret = 0;
int was_empty = 0;
if (!rdsv3_conn_up(conn))
goto out;
/*
* sendmsg calls here after having queued its message on the send
* queue. We only have one task feeding the connection at a time. If
* another thread is already feeding the queue then we back off. This
* avoids blocking the caller and trading per-connection data between
* caches per message.
*/
RDSV3_DPRINTF4("rdsv3_send_xmit",
"Another thread running(conn: %p)", conn);
goto out;
}
/*
* spin trying to push headers and data down the connection until
* the connection doesn't make forward progress.
*/
while (--send_quota) {
/*
* See if need to send a congestion map update if we're
* between sending messages. The send_sem protects our sole
* use of c_map_offset and _bytes.
* Note this is used only by transports that define a special
* xmit_cong_map function. For all others, we create allocate
* a cong_map message and treat it just like any other send.
*/
if (conn->c_map_bytes) {
conn->c_map_offset);
if (ret <= 0)
break;
if (conn->c_map_bytes)
continue;
}
/*
* If we're done sending the current message, clear the
* offset and S/G temporaries.
*/
conn->c_xmit_hdr_off = 0;
conn->c_xmit_data_off = 0;
conn->c_xmit_rdma_sent = 0;
/* Release the reference to the previous message. */
}
/* If we're asked to send a cong map update, do so. */
conn->c_map_offset = 0;
conn->c_map_bytes =
sizeof (struct rdsv3_header) +
continue;
}
break;
}
}
/*
* Grab the next message from the send queue, if there is one.
*
* c_xmit_rm holds a ref while we're sending this message down
* the connction. We can use this ref while holding the
* send_sem.. rdsv3_send_reset() is serialized with it.
*/
unsigned int len;
/*
* Move the message from the send queue to
* the retransmit
* list right away.
*/
}
was_empty = 1;
break;
}
/*
* Unfortunately, the way Infiniband deals with
* RDMA to a bad MR key is by moving the entire
* queue pair to error state. We cold possibly
* recover from that, but right now we drop the
* connection.
* Therefore, we never retransmit messages with
* RDMA ops.
*/
continue;
}
/* Require an ACK every once in a while */
if (conn->c_unacked_packets == 0 ||
} else {
}
}
/*
* Try and send an rdma message. Let's see if we can
* keep this simple and require that the transport either
* send the whole rdma or none of it.
*/
if (ret)
break;
/*
* The transport owns the mapped memory for now.
* You can't unmap it while it's on the send queue
*/
}
if (ret <= 0)
break;
if (conn->c_xmit_hdr_off <
sizeof (struct rdsv3_header)) {
sizeof (struct rdsv3_header) -
}
while (ret) {
conn->c_xmit_data_off = 0;
sg++;
}
}
}
}
/* Nuke any messages we decided not to retransmit. */
if (!list_is_empty(&to_be_dropped))
/*
* We might be racing with another sender who queued a message but
* backed off on noticing that we held the c_send_lock. If we check
* for queued messages after dropping the sem then either we'll
* see the queued message or the queuer will get the sem. If we
* notice the queued message then we trigger an immediate retry.
*
* We need to be careful only to do this when we stopped processing
* the send queue because it was empty. It's the only way we
* stop processing the loop when the transport hasn't taken
* responsibility for forward progress.
*/
/*
* We exhausted the send quota, but there's work left to
* do. Return and (re-)schedule the send worker.
*/
}
/*
* A simple bit test would be way faster than taking the
* spin lock
*/
}
}
out:
return (ret);
}
static void
{
if (rs->rs_snd_bytes == 0)
}
static inline int
{
if (is_acked)
}
/*
* Returns true if there are no messages on the send and retransmit queues
* which have a sequence number greater than or equal to the given sequence
* number.
*/
int
{
/* XXX - original code spits out warning */
ret = 0;
/* XXX - original code spits out warning */
ret = 0;
return (ret);
}
/*
* This is pretty similar to what happens below in the ACK
* handling code - except that we call here as soon as we get
* the IB send completion on the RDMA op and the accompanying
* message.
*/
void
{
}
if (rs) {
int error;
/* wake up anyone waiting in poll */
if (error != 0) {
RDSV3_DPRINTF2("rdsv3_recv_incoming",
"su_recv returned: %d", error);
}
}
}
/*
* This is the same as rdsv3_rdma_send_complete except we
* don't do any locking - we have all the ingredients (message,
* socket, socket lock) and can just move the notifier.
*/
static inline void
int status)
{
void *ic;
RDSV3_DPRINTF4("__rdsv3_rdma_send_complete",
}
/* No need to wake the app - caller does this */
}
/*
* This is called from the IB send completion when we detect
* a RDMA operation that failed with remote access error.
* So speed is not an issue here.
*/
struct rdsv3_message *
struct rdsv3_rdma_op *op)
{
goto out;
}
}
m_conn_item) {
break;
}
}
out:
return (found);
}
/*
* This removes messages from the socket's list if they're on it. The list
* argument must be private to the caller, we must be able to modify it
* without locks. The messages must have a reference held for their
* position on the list. This function will drop that reference after
* removing the messages from the 'messages' list regardless of if it found
* the messages on the socket list or not.
*/
void
{
while (!list_is_empty(messages)) {
int was_on_sock = 0;
/*
* If we see this flag cleared then we're *sure* that someone
* else beat us to removing it from the sock. If we race
* with their flag update we'll get the lock and then really
* see that the flag has been cleared.
*
* The message spinlock makes sure nobody clears rm->m_rs
* while we're messing with it. It does not prevent the
* message from being removed from the socket, though.
*/
goto unlock_and_drop;
if (rs) {
}
}
notifier);
}
was_on_sock = 1;
}
if (was_on_sock)
}
if (rs) {
}
}
/*
* Transports call here when they've determined that the receiver queued
* messages up to, and including, the given sequence number. Messages are
* moved to the retrans queue when rdsv3_send_xmit picks them off the send
* queue. This means that in the TCP case, the message may not have been
* assigned the m_ack_seq yet - but that's fine as long as tcp_is_acked
* checks the RDSV3_MSG_HAS_ACK_SEQ bit.
*
* XXX It's not clear to me how this is safely serialized with socket
* destruction. Maybe it should bail if it sees SOCK_DEAD.
*/
void
{
break;
}
#if 0
/* order flag updates with spin locks */
if (!list_is_empty(&list))
#endif
/* now remove the messages from the sock list as needed */
}
void
{
int wake = 0;
/* get all the messages we're dropping under the rs lock */
m_sock_item) {
continue;
wake = 1;
}
/* now remove the messages from the conn list as needed */
/*
* We do this here rather than in the loop above, so that
* we don't have to nest m_rs_lock under rs->rs_lock
*/
/* If this is a RDMA operation, notify the app. */
/*
* If we see this flag cleared then we're *sure* that someone
* else beat us to removing it from the conn. If we race
* with their flag update we'll get the lock and then really
* see that the flag has been cleared.
*/
continue;
if (conn)
}
}
}
if (conn)
if (wake)
while (!list_is_empty(&list)) {
}
}
/*
* we only want this to fire once so we use the callers 'queued'. It's
* possible that another thread can race with us and remove the
* message from the flow with RDSV3_CANCEL_SENT_TO.
*/
static int
{
if (*queued)
goto out;
/*
* this is the only place which holds both the socket's rs_lock
* and the connection's c_lock
*/
/*
* If there is a little space in sndbuf, we don't queue anything,
* and userspace gets -EAGAIN. But poll() indicates there's send
* room. This can lead to bad behavior (spinning) if snd_bytes isn't
* freed up by incoming acks. So we check the *old* value of
* rs_snd_bytes here to allow the last msg to exceed the buffer,
* and poll() now knows no more data can be sent.
*/
/*
* let recv side know we are close to send space exhaustion.
* This is probably not the optimal way to do it, as this
* means we set the flag on *all* messages as soon as our
* throughput hits a certain threshold.
*/
/*
* The code ordering is a little weird, but we're
* trying to minimize the time we hold c_lock
*/
dport, 0);
RDSV3_DPRINTF5("rdsv3_send_queue_rm",
"queued msg %p len %d, rs %p bytes %d seq %llu",
(unsigned long long)ntohll(
*queued = 1;
}
out:
return (*queued);
}
static int
{
int ret = 0;
continue;
/*
* As a side effect, RDMA_DEST and RDMA_MAP will set
* rm->m_rdma_cookie and rm->m_rdma_mr.
*/
case RDS_CMSG_RDMA_ARGS:
break;
case RDS_CMSG_RDMA_DEST:
break;
case RDS_CMSG_RDMA_MAP:
if (ret)
*allocated_mr = 1;
break;
default:
return (-EINVAL);
}
if (ret)
break;
}
return (ret);
}
extern unsigned long rdsv3_max_bcopy_size;
int
{
int ret = 0;
if (msg->msg_namelen) {
/* XXX fail non-unicast destination IPs? */
goto out;
}
} else {
/* We only care about consistency with ->connect() */
}
/* racing with another thread binding seems ok here */
goto out;
}
if (payload_len > rdsv3_max_bcopy_size) {
goto out;
}
RDSV3_DPRINTF2("rdsv3_sendmsg",
"rdsv3_message_copy_from_user failed %d", -ret);
goto out;
}
/* Parse any control messages the user may have included. */
if (ret) {
RDSV3_DPRINTF2("rdsv3_sendmsg",
"rdsv3_cmsg_send(rs: %p rm: %p msg: %p) returned: %d",
goto out;
}
/*
* rdsv3_conn_create has a spinlock that runs with IRQ off.
* Caching the conn in the socket helps a lot.
*/
} else {
RDSV3_DPRINTF2("rdsv3_sendmsg",
"rdsv3_conn_create_outgoing failed %d",
-ret);
goto out;
}
}
ret = -EOPNOTSUPP;
goto out;
}
/*
* If the connection is down, trigger a connect. We may
* have scheduled a delayed reconnect however - in this case
* we should not interfere.
*/
if (ret) {
RDSV3_DPRINTF2("rdsv3_sendmsg",
goto out;
}
&queued);
if (!queued) {
/* rdsv3_stats_inc(s_send_queue_full); */
/* XXX make sure this is reasonable */
RDSV3_DPRINTF2("rdsv3_sendmsg",
"msgsize(%d) too big, returning: %d",
payload_len, -ret);
goto out;
}
if (nonblock) {
RDSV3_DPRINTF3("rdsv3_sendmsg",
"send queue full (%d), returning: %d",
payload_len, -ret);
goto out;
}
#if 0
if (ret == 0) {
RDSV3_DPRINTF2("rdsv3_sendmsg",
"woke due to signal: %d", ret);
goto out;
}
#else
if (ret == 0) {
RDSV3_DPRINTF2("rdsv3_sendmsg",
"woke due to signal: %d", ret);
goto out;
}
}
#endif
queued);
ret = 0;
}
/*
* By now we've committed to the send. We reuse rdsv3_send_worker()
* to retry sends in the rds thread if the transport asks us to.
*/
rs, payload_len);
return (payload_len);
out:
/*
* If the user included a RDMA_MAP cmsg, we allocated a MR on the fly.
* If the sendmsg goes through, we keep the MR. If it fails with EAGAIN
* or in any other way, we need to destroy the MR again
*/
if (allocated_mr)
1);
if (rm)
return (ret);
}
/*
* Reply to a ping packet.
*/
int
{
int ret = 0;
if (!rm) {
goto out;
}
/*
* If the connection is down, trigger a connect. We may
* have scheduled a delayed reconnect however - in this case
* we should not interfere.
*/
if (ret)
goto out;
conn->c_next_tx_seq++;
(void) rdsv3_send_xmit(conn);
return (0);
out:
if (rm)
return (ret);
}