16e76cdd6e3cfaac7d91c3b0644ee1bc6cf52347agiri * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved.
16e76cdd6e3cfaac7d91c3b0644ee1bc6cf52347agiri * This file contains code imported from the OFED rds source file send.c
16e76cdd6e3cfaac7d91c3b0644ee1bc6cf52347agiri * Oracle elects to have and use the contents of send.c under and governed
16e76cdd6e3cfaac7d91c3b0644ee1bc6cf52347agiri * by the OpenIB.org BSD license (see below for full license text). However,
16e76cdd6e3cfaac7d91c3b0644ee1bc6cf52347agiri * the following notice accompanied the original version of this file:
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * Copyright (c) 2006 Oracle. All rights reserved.
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * This software is available to you under a choice of one of two
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * licenses. You may choose to be licensed under the terms of the GNU
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * General Public License (GPL) Version 2, available from the file
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * COPYING in the main directory of this source tree, or the
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * OpenIB.org BSD license below:
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * Redistribution and use in source and binary forms, with or
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * without modification, are permitted provided that the following
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * conditions are met:
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * - Redistributions of source code must retain the above
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * copyright notice, this list of conditions and the following
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * disclaimer.
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * - Redistributions in binary form must reproduce the above
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * copyright notice, this list of conditions and the following
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * disclaimer in the documentation and/or other materials
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * provided with the distribution.
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * SOFTWARE.
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * When transmitting messages in rdsv3_send_xmit, we need to emerge from
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * time to time and briefly release the CPU. Otherwise the softlock watchdog
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * will kick our shin.
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * Also, it seems fairer to not let one busy connection stall all the
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * send_batch_count is the number of times we'll loop in send_xmit. Setting
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * it to 0 will restore the old behavior (where we looped until we had
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * drained the queue).
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Otaextern void rdsv3_ib_send_unmap_rdma(void *ic, struct rdsv3_rdma_op *op);
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * Reset the send state. Caller must hold c_send_lock when calling here.
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota RDSV3_DPRINTF4("rdsv3_send_reset", "Enter(conn: %p)", conn);
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota "rm %p mflg 0x%x map %d mihdl %p sgl %p",
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota rdsv3_ib_send_unmap_rdma(conn->c_transport_data, ro);
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * Tell the user the RDMA op is no longer mapped by the
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * transport. This isn't entirely true (it's flushed out
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * independently) but as the connection is down, there's
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * no ongoing RDMA to/from that memory
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota conn->c_unacked_packets = rdsv3_sysctl_max_unacked_packets;
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota conn->c_unacked_bytes = rdsv3_sysctl_max_unacked_bytes;
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota /* Mark messages as retransmissions, and move them to the send q */
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota RDSV3_FOR_EACH_LIST_NODE_SAFE(rm, tmp, &conn->c_retrans, m_conn_item) {
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota "RT rm %p mflg 0x%x sgl %p",
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota list_move_tail(&conn->c_send_queue, &conn->c_retrans);
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota RDSV3_DPRINTF4("rdsv3_send_reset", "Return(conn: %p)", conn);
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * We're making the concious trade-off here to only send one message
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * down the connection at a time.
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * - tx queueing is a simple fifo list
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * - reassembly is optional and easily done by transports per conn
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * - no per flow rx lookup at all, straight to the socket
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * - less per-frag memory and wire overhead
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * - queued acks can be delayed behind large messages
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * - small message latency is higher behind queued large messages
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * - large message latency isn't starved by intervening small sends
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota RDSV3_DPRINTF4("rdsv3_send_xmit", "Enter(conn: %p)", conn);
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota list_create(&to_be_dropped, sizeof (struct rdsv3_message),
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * sendmsg calls here after having queued its message on the send
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * queue. We only have one task feeding the connection at a time. If
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * another thread is already feeding the queue then we back off. This
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * avoids blocking the caller and trading per-connection data between
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * caches per message.
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * spin trying to push headers and data down the connection until
5d5562f583b2b6affe19bdce0b3c8b1840d667a4Eiji Ota * the connection doesn't make forward progress.
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * See if need to send a congestion map update if we're
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * between sending messages. The send_sem protects our sole
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * use of c_map_offset and _bytes.
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * Note this is used only by transports that define a special
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * xmit_cong_map function. For all others, we create allocate
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * a cong_map message and treat it just like any other send.
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota ret = conn->c_trans->xmit_cong_map(conn, conn->c_lcong,
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * If we're done sending the current message, clear the
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * offset and S/G temporaries.
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota conn->c_xmit_hdr_off == sizeof (struct rdsv3_header) &&
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota /* Release the reference to the previous message. */
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota /* If we're asked to send a cong map update, do so. */
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota if (rm == NULL && test_and_clear_bit(0, &conn->c_map_queued)) {
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota sizeof (struct rdsv3_header) +
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * Grab the next message from the send queue, if there is one.
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * c_xmit_rm holds a ref while we're sending this message down
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * the connction. We can use this ref while holding the
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * send_sem.. rdsv3_send_reset() is serialized with it.
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * Move the message from the send queue to
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * the retransmit
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * list right away.
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * Unfortunately, the way Infiniband deals with
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * RDMA to a bad MR key is by moving the entire
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * queue pair to error state. We cold possibly
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * recover from that, but right now we drop the
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * connection.
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * Therefore, we never retransmit messages with
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * RDMA ops.
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota /* Require an ACK every once in a while */
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * Try and send an rdma message. Let's see if we can
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * keep this simple and require that the transport either
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * send the whole rdma or none of it.
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota ret = conn->c_trans->xmit_rdma(conn, rm->m_rdma_op);
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * The transport owns the mapped memory for now.
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * You can't unmap it while it's on the send queue
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota if (conn->c_xmit_hdr_off < sizeof (struct rdsv3_header) ||
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota sizeof (struct rdsv3_header)) {
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota sizeof (struct rdsv3_header) -
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota /* Nuke any messages we decided not to retransmit. */
fe817b6022080da0a98b5d2d8cd179f594d6ca5eEiji Ota rdsv3_send_remove_from_sock(&to_be_dropped, RDS_RDMA_DROPPED);
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * We might be racing with another sender who queued a message but
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * backed off on noticing that we held the c_send_lock. If we check
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * for queued messages after dropping the sem then either we'll
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * see the queued message or the queuer will get the sem. If we
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * notice the queued message then we trigger an immediate retry.
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * We need to be careful only to do this when we stopped processing
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * the send queue because it was empty. It's the only way we
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * stop processing the loop when the transport hasn't taken
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * responsibility for forward progress.
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota if (conn->c_map_bytes || (send_quota == 0 && !was_empty)) {
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * We exhausted the send quota, but there's work left to
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * do. Return and (re-)schedule the send worker.
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * A simple bit test would be way faster than taking the
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * spin lock
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota RDSV3_DPRINTF4("rdsv3_send_xmit", "Return(conn: %p, ret: %d)",
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Otardsv3_send_sndbuf_remove(struct rdsv3_sock *rs, struct rdsv3_message *rm)
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Otastatic inline int
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Otardsv3_send_is_acked(struct rdsv3_message *rm, uint64_t ack,
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota return (ntohll(rm->m_inc.i_hdr.h_sequence) <= ack);
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * Returns true if there are no messages on the send and retransmit queues
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * which have a sequence number greater than or equal to the given sequence
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Otardsv3_send_acked_before(struct rdsv3_connection *conn, uint64_t seq)
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota RDSV3_DPRINTF4("rdsv3_send_acked_before", "Enter(conn: %p)", conn);
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota /* XXX - original code spits out warning */
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota /* XXX - original code spits out warning */
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota RDSV3_DPRINTF4("rdsv3_send_acked_before", "Return(conn: %p)", conn);
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * This is pretty similar to what happens below in the ACK
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * handling code - except that we call here as soon as we get
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * the IB send completion on the RDMA op and the accompanying
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Otardsv3_rdma_send_complete(struct rdsv3_message *rm, int status)
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota RDSV3_DPRINTF4("rdsv3_rdma_send_complete", "Enter(rm: %p)", rm);
3f756f37bfdcab561a4d84fb80f41f1e1a15c135agiri /* wake up anyone waiting in poll */
3f756f37bfdcab561a4d84fb80f41f1e1a15c135agiri if (error != 0) {
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota RDSV3_DPRINTF4("rdsv3_rdma_send_complete", "Return(rm: %p)", rm);
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * This is the same as rdsv3_rdma_send_complete except we
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * don't do any locking - we have all the ingredients (message,
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * socket, socket lock) and can just move the notifier.
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Otastatic inline void
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota__rdsv3_rdma_send_complete(struct rdsv3_sock *rs, struct rdsv3_message *rm,
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota list_insert_tail(&rs->rs_notify_queue, ro->r_notifier);
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota /* No need to wake the app - caller does this */
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * This is called from the IB send completion when we detect
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * a RDMA operation that failed with remote access error.
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * So speed is not an issue here.
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Otardsv3_send_get_message(struct rdsv3_connection *conn,
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota RDSV3_DPRINTF4("rdsv3_send_get_message", "Enter(conn: %p)", conn);
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota RDSV3_FOR_EACH_LIST_NODE_SAFE(rm, tmp, &conn->c_retrans, m_conn_item) {
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota RDSV3_FOR_EACH_LIST_NODE_SAFE(rm, tmp, &conn->c_send_queue,
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * This removes messages from the socket's list if they're on it. The list
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * argument must be private to the caller, we must be able to modify it
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * without locks. The messages must have a reference held for their
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * position on the list. This function will drop that reference after
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * removing the messages from the 'messages' list regardless of if it found
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * the messages on the socket list or not.
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Otardsv3_send_remove_from_sock(struct list *messages, int status)
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota RDSV3_DPRINTF4("rdsv3_send_remove_from_sock", "Enter");
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * If we see this flag cleared then we're *sure* that someone
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * else beat us to removing it from the sock. If we race
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * with their flag update we'll get the lock and then really
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * see that the flag has been cleared.
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * The message spinlock makes sure nobody clears rm->m_rs
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * while we're messing with it. It does not prevent the
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * message from being removed from the socket, though.
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota if (test_and_clear_bit(RDSV3_MSG_ON_SOCK, &rm->m_flags)) {
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota RDSV3_DPRINTF4("rdsv3_send_remove_from_sock", "Return");
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * Transports call here when they've determined that the receiver queued
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * messages up to, and including, the given sequence number. Messages are
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * moved to the retrans queue when rdsv3_send_xmit picks them off the send
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * queue. This means that in the TCP case, the message may not have been
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * assigned the m_ack_seq yet - but that's fine as long as tcp_is_acked
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * checks the RDSV3_MSG_HAS_ACK_SEQ bit.
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * XXX It's not clear to me how this is safely serialized with socket
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * destruction. Maybe it should bail if it sees SOCK_DEAD.
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Otardsv3_send_drop_acked(struct rdsv3_connection *conn, uint64_t ack,
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota RDSV3_DPRINTF4("rdsv3_send_drop_acked", "Enter(conn: %p)", conn);
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota RDSV3_FOR_EACH_LIST_NODE_SAFE(rm, tmp, &conn->c_retrans, m_conn_item) {
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota /* order flag updates with spin locks */
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota /* now remove the messages from the sock list as needed */
fe817b6022080da0a98b5d2d8cd179f594d6ca5eEiji Ota rdsv3_send_remove_from_sock(&list, RDS_RDMA_SUCCESS);
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota RDSV3_DPRINTF4("rdsv3_send_drop_acked", "Return(conn: %p)", conn);
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Otardsv3_send_drop_to(struct rdsv3_sock *rs, struct sockaddr_in *dest)
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota RDSV3_DPRINTF4("rdsv3_send_drop_to", "Enter(rs: %p)", rs);
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota /* get all the messages we're dropping under the rs lock */
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota RDSV3_FOR_EACH_LIST_NODE_SAFE(rm, tmp, &rs->rs_send_queue,
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota if (dest && (dest->sin_addr.s_addr != rm->m_daddr ||
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota /* now remove the messages from the conn list as needed */
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * We do this here rather than in the loop above, so that
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * we don't have to nest m_rs_lock under rs->rs_lock
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota /* If this is a RDMA operation, notify the app. */
fe817b6022080da0a98b5d2d8cd179f594d6ca5eEiji Ota __rdsv3_rdma_send_complete(rs, rm, RDS_RDMA_CANCELED);
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * If we see this flag cleared then we're *sure* that someone
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * else beat us to removing it from the conn. If we race
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * with their flag update we'll get the lock and then really
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * see that the flag has been cleared.
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota if (test_and_clear_bit(RDSV3_MSG_ON_CONN, &rm->m_flags)) {
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota RDSV3_DPRINTF4("rdsv3_send_drop_to", "Return(rs: %p)", rs);
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * we only want this to fire once so we use the callers 'queued'. It's
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * possible that another thread can race with us and remove the
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * message from the flow with RDSV3_CANCEL_SENT_TO.
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Otardsv3_send_queue_rm(struct rdsv3_sock *rs, struct rdsv3_connection *conn,
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota RDSV3_DPRINTF4("rdsv3_send_queue_rm", "Enter(rs: %p, rm: %p)", rs, rm);
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * this is the only place which holds both the socket's rs_lock
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * and the connection's c_lock
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * If there is a little space in sndbuf, we don't queue anything,
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * and userspace gets -EAGAIN. But poll() indicates there's send
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * room. This can lead to bad behavior (spinning) if snd_bytes isn't
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * freed up by incoming acks. So we check the *old* value of
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * rs_snd_bytes here to allow the last msg to exceed the buffer,
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * and poll() now knows no more data can be sent.
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * let recv side know we are close to send space exhaustion.
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * This is probably not the optimal way to do it, as this
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * means we set the flag on *all* messages as soon as our
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * throughput hits a certain threshold.
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * The code ordering is a little weird, but we're
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * trying to minimize the time we hold c_lock
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota rdsv3_message_populate_header(&rm->m_inc.i_hdr, sport,
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota rm->m_inc.i_hdr.h_sequence = htonll(conn->c_next_tx_seq++);
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota "queued msg %p len %d, rs %p bytes %d seq %llu",
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota (unsigned long long)ntohll(
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota RDSV3_DPRINTF4("rdsv3_send_queue_rm", "Return(rs: %p)", rs);
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Otardsv3_cmsg_send(struct rdsv3_sock *rs, struct rdsv3_message *rm,
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota RDSV3_DPRINTF4("rdsv3_cmsg_send", "Enter(rs: %p)", rs);
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota for (cmsg = CMSG_FIRSTHDR(msg); cmsg; cmsg = CMSG_NXTHDR(msg, cmsg)) {
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota RDSV3_DPRINTF4("rdsv3_cmsg_send", "cmsg(%p, %p) type %d",
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * As a side effect, RDMA_DEST and RDMA_MAP will set
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * rm->m_rdma_cookie and rm->m_rdma_mr.
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota RDSV3_DPRINTF4("rdsv3_cmsg_send", "Return(rs: %p)", rs);
3f756f37bfdcab561a4d84fb80f41f1e1a15c135agiriextern unsigned long rdsv3_max_bcopy_size;
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Otardsv3_sendmsg(struct rdsv3_sock *rs, uio_t *uio, struct nmsghdr *msg,
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota struct sockaddr_in *usin = (struct sockaddr_in *)msg->msg_name;
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota RDSV3_DPRINTF4("rdsv3_sendmsg", "Enter(rs: %p)", rs);
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota /* XXX fail non-unicast destination IPs? */
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota RDSV3_DPRINTF2("rdsv3_sendmsg", "returning: %d", -ret);
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota /* We only care about consistency with ->connect() */
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota /* racing with another thread binding seems ok here */
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota RDSV3_DPRINTF2("rdsv3_sendmsg", "returning: %d", -ret);
3f756f37bfdcab561a4d84fb80f41f1e1a15c135agiri RDSV3_DPRINTF2("rdsv3_sendmsg", "Message too large: %d",
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota rm = rdsv3_message_copy_from_user(uio, payload_len);
cadbfdc3bdb156e92d7a88978bc98ea87f6e037fEiji Ota /* Parse any control messages the user may have included. */
cadbfdc3bdb156e92d7a88978bc98ea87f6e037fEiji Ota "rdsv3_cmsg_send(rs: %p rm: %p msg: %p) returned: %d",
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * rdsv3_conn_create has a spinlock that runs with IRQ off.
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * Caching the conn in the socket helps a lot.
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota if (rs->rs_conn && rs->rs_conn->c_faddr == daddr) {
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota conn = rdsv3_conn_create_outgoing(rs->rs_bound_addr,
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota "rdsv3_conn_create_outgoing failed %d",
6e18d381c642549b8bb1774a803d3510aec6baafagiri RDSV3_DPRINTF2("rdsv3_sendmsg", "rdma_op %p conn xmit_rdma %p",
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * If the connection is down, trigger a connect. We may
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * have scheduled a delayed reconnect however - in this case
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * we should not interfere.
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota !test_and_set_bit(RDSV3_RECONNECT_PENDING, &conn->c_flags))
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota rdsv3_queue_delayed_work(rdsv3_wq, &conn->c_conn_w, 0);
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota ret = rdsv3_cong_wait(conn->c_fcong, dport, nonblock, rs);
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota "rdsv3_cong_wait (dport: %d) returned: %d", dport, ret);
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota (void) rdsv3_send_queue_rm(rs, conn, rm, rs->rs_bound_port, dport,
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota /* rdsv3_stats_inc(s_send_queue_full); */
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota /* XXX make sure this is reasonable */
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota "msgsize(%d) too big, returning: %d",
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota "send queue full (%d), returning: %d",
6e18d381c642549b8bb1774a803d3510aec6baafagiri if (ret == 0) {
6e18d381c642549b8bb1774a803d3510aec6baafagiri /* signal/timeout pending */
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota while (!rdsv3_send_queue_rm(rs, conn, rm, rs->rs_bound_port,
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota if (ret == 0) {
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota /* signal/timeout pending */
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota RDSV3_DPRINTF5("rdsv3_sendmsg", "sendmsg woke queued %d",
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * By now we've committed to the send. We reuse rdsv3_send_worker()
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * to retry sends in the rds thread if the transport asks us to.
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota RDSV3_DPRINTF4("rdsv3_sendmsg", "Return(rs: %p, len: %d)",
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * If the user included a RDMA_MAP cmsg, we allocated a MR on the fly.
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * If the sendmsg goes through, we keep the MR. If it fails with EAGAIN
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * or in any other way, we need to destroy the MR again
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota rdsv3_rdma_unuse(rs, rdsv3_rdma_cookie_key(rm->m_rdma_cookie),
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * Reply to a ping packet.
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Otardsv3_send_pong(struct rdsv3_connection *conn, uint16_be_t dport)
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota RDSV3_DPRINTF4("rdsv3_send_pong", "Enter(conn: %p)", conn);
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * If the connection is down, trigger a connect. We may
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * have scheduled a delayed reconnect however - in this case
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * we should not interfere.
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota !test_and_set_bit(RDSV3_RECONNECT_PENDING, &conn->c_flags))
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota rdsv3_queue_delayed_work(rdsv3_wq, &conn->c_conn_w, 0);
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota ret = rdsv3_cong_wait(conn->c_fcong, dport, 1, NULL);
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota rdsv3_message_populate_header(&rm->m_inc.i_hdr, 0, dport,
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota RDSV3_DPRINTF4("rdsv3_send_pong", "Return(conn: %p)", conn);
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota return (0);