16e76cdd6e3cfaac7d91c3b0644ee1bc6cf52347agiri * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved.
16e76cdd6e3cfaac7d91c3b0644ee1bc6cf52347agiri * This file contains code imported from the OFED rds source file recv.c
16e76cdd6e3cfaac7d91c3b0644ee1bc6cf52347agiri * Oracle elects to have and use the contents of rds_recv.c under and governed
16e76cdd6e3cfaac7d91c3b0644ee1bc6cf52347agiri * by the OpenIB.org BSD license (see below for full license text). However,
16e76cdd6e3cfaac7d91c3b0644ee1bc6cf52347agiri * the following notice accompanied the original version of this file:
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * Copyright (c) 2006 Oracle. All rights reserved.
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * This software is available to you under a choice of one of two
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * licenses. You may choose to be licensed under the terms of the GNU
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * General Public License (GPL) Version 2, available from the file
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * COPYING in the main directory of this source tree, or the
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * OpenIB.org BSD license below:
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * Redistribution and use in source and binary forms, with or
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * without modification, are permitted provided that the following
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * conditions are met:
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * - Redistributions of source code must retain the above
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * copyright notice, this list of conditions and the following
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * disclaimer.
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * - Redistributions in binary form must reproduce the above
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * copyright notice, this list of conditions and the following
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * disclaimer in the documentation and/or other materials
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * provided with the distribution.
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * SOFTWARE.
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Otardsv3_inc_init(struct rdsv3_incoming *inc, struct rdsv3_connection *conn,
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota RDSV3_DPRINTF5("rdsv3_inc_init", "Enter(inc: %p, conn: %p)", inc, conn);
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota "addref inc %p ref %d", inc, atomic_get(&inc->i_refcount));
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota RDSV3_DPRINTF4("rdsv3_inc_put", "put inc %p ref %d",
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota/*ARGSUSED*/
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Otardsv3_recv_rcvbuf_delta(struct rdsv3_sock *rs, struct rsock *sk,
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota "Enter(rs: %p, map: %p, delta: %d, port: %d)",
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota now_congested = rs->rs_rcv_bytes > rdsv3_sk_rcvbuf(rs);
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota "rs %p (%u.%u.%u.%u:%u) recv bytes %d buf %d "
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota "now_cong %d delta %d",
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota /* wasn't -> am congested */
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota /* was -> aren't congested */
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * Require more free space before reporting uncongested to prevent
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * bouncing cong/uncong state too often
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota /* do nothing if no change in cong state */
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota RDSV3_DPRINTF4("rdsv3_recv_rcvbuf_delta", "Return(rs: %p)", rs);
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * Process all extension headers that come with this message.
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Otardsv3_recv_incoming_exthdrs(struct rdsv3_incoming *inc, struct rdsv3_sock *rs)
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota RDSV3_DPRINTF4("rdsv3_recv_incoming_exthdrs", "Enter");
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota while (1) {
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota type = rdsv3_message_next_extension(hdr, &pos, &buffer, &len);
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota RDSV3_DPRINTF4("recv_incoming_exthdrs", "type %d", type);
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota /* Process extension header here */
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota rdsv3_rdma_unuse(rs, ntohl(buffer.rdma.h_rdma_rkey),
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * We ignore the size for now. We could stash it
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * somewhere and use it for error checking.
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota RDSV3_DPRINTF4("rdsv3_recv_incoming_exthdrs", "Return");
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * The transport must make sure that this is serialized against other
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * rx and conn reset on this specific conn.
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * We currently assert that only one fragmented message will be sent
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * down a connection at a time. This lets us reassemble in the conn
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * instead of per-flow which means that we don't have to go digging through
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * flows to tear down partial reassembly progress on conn failure and
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * we save flow lookup and locking for each frag arrival. It does mean
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * that small messages will wait behind large ones. Fragmenting at all
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * is only to reduce the memory consumption of pre-posted buffers.
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * The caller passes in saddr and daddr instead of us getting it from the
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * conn. This lets loopback, who only has one conn for both directions,
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * tell us which roles the addrs in the conn are playing for this message.
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota/* ARGSUSED */
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Otardsv3_recv_incoming(struct rdsv3_connection *conn, uint32_be_t saddr,
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota uint32_be_t daddr, struct rdsv3_incoming *inc, int gfp)
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota "conn %p next %llu inc %p seq %llu len %u sport %u dport %u "
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * Sequence numbers should only increase. Messages get their
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * sequence number as they're queued in a sending conn. They
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * can be dropped, though, if the sending socket is closed before
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * they hit the wire. So sequence numbers can skip forward
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * under normal operation. They can also drop back in the conn
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * failover case as previously sent messages are resent down the
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * new instance of a conn. We drop those, otherwise we have
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * to assume that the next valid seq does not come after a
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * hole in the fragment stream.
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * The headers don't give us a way to realize if fragments of
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * a message have been dropped. We assume that frags that arrive
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * to a flow are part of the current message on the flow that is
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * being reassembled. This means that senders can't drop messages
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * from the sending conn until all their frags are sent.
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * XXX we could spend more on the wire to get more robust failure
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * detection, arguably worth it to avoid data corruption.
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota if (ntohll(inc->i_hdr.h_sequence) < conn->c_next_rx_seq &&
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota conn->c_next_rx_seq = ntohll(inc->i_hdr.h_sequence) + 1;
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota if (rdsv3_sysctl_ping_enable && inc->i_hdr.h_dport == 0) {
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota /* Process extension headers */
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota /* We can be racing with rdsv3_release() which marks the socket dead. */
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota /* serialize with rdsv3_release -> sock_orphan */
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota rdsv3_recv_rcvbuf_delta(rs, sk, inc->i_conn->c_lcong,
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota /* wake up anyone waiting in poll */
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * be very careful here. This is being called as the condition in
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * wait_event_*() needs to cope with being called many times.
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Otardsv3_next_incoming(struct rdsv3_sock *rs, struct rdsv3_incoming **inc)
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Otardsv3_still_queued(struct rdsv3_sock *rs, struct rdsv3_incoming *inc,
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota RDSV3_DPRINTF4("rdsv3_still_queued", "Enter rs: %p inc: %p drop: %d",
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota /* XXX make sure this i_conn is reliable */
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota rdsv3_recv_rcvbuf_delta(rs, sk, inc->i_conn->c_lcong,
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota "inc %p rs %p still %d dropped %d", inc, rs, ret, drop);
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * Pull errors off the error queue.
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * If msghdr is NULL, we will just purge the error queue.
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Otardsv3_notify_queue_get(struct rdsv3_sock *rs, struct msghdr *msghdr)
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota RDSV3_DPRINTF4("rdsv3_notify_queue_get", "Enter(rs: %p)", rs);
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * put_cmsg copies to user space and thus may sleep. We can't do this
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * with rs_lock held, so first grab as many notifications as we can
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * in the user provided cmsg buffer. We don't try to copy more, to avoid
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * losing notifications - except when the buffer is so small that
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * it wouldn't
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * even hold a single notification. Then we give him as much of this
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * msg as we can squeeze in, and set MSG_CTRUNC.
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota msghdr->msg_controllen / CMSG_SPACE(sizeof (cmsg));
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota while (!list_is_empty(&rs->rs_notify_queue) && count < max_messages) {
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota return (0);
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota kmem_free(notifier, sizeof (struct rdsv3_notifier));
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * If we bailed out because of an error in put_cmsg,
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * we may be left with one or more notifications that we
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * didn't process. Return them to the head of the list.
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota RDSV3_DPRINTF4("rdsv3_notify_queue_get", "Return(rs: %p)", rs);
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * Queue a congestion notification
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Otardsv3_notify_cong(struct rdsv3_sock *rs, struct msghdr *msghdr)
fe817b6022080da0a98b5d2d8cd179f594d6ca5eEiji Ota err = rdsv3_put_cmsg(msghdr, SOL_RDS, RDS_CMSG_CONG_UPDATE,
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota return (0);
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * Receive any control messages.
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Otardsv3_cmsg_recv(struct rdsv3_incoming *inc, struct msghdr *msg)
9b3d509c544ac523c29e6a3cfd08e87a552b622bEiji Ota ret = rdsv3_put_cmsg(msg, SOL_RDS, RDS_CMSG_RDMA_DEST,
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota "Enter(rs: %p size: %d msg_flags: 0x%x)", rs, size, msg_flags);
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota /* udp_recvmsg()->sock_recvtimeo() gets away without locking too.. */
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota /* mark the first cmsg position */
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota while (1) {
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * If there are pending notifications, do those -
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * and nothing else
6e18d381c642549b8bb1774a803d3510aec6baafagiri if (ret == 0) {
6e18d381c642549b8bb1774a803d3510aec6baafagiri /* signal/timeout pending */
6e18d381c642549b8bb1774a803d3510aec6baafagiri "woke due to signal");
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota if (ret == 0) {
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota /* signal/timeout pending */
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota "woke due to signal");
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota "recvmsg woke rs: %p inc %p ret %d",
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * if the wakeup was due to rs_notify_queue or
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * rs_cong_notify then we need to handle those first.
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota ret = inc->i_conn->c_trans->inc_copy_to_user(inc, uio, size);
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * if the message we just copied isn't at the head of the
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * recv queue then someone else raced us to return it, try
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * to get the next message.
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota if (!rdsv3_still_queued(rs, inc, !(msg_flags & MSG_PEEK))) {
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota sin = kmem_alloc(sizeof (struct sockaddr_in), KM_SLEEP);
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota RDSV3_DPRINTF4("rdsv3_recvmsg", "Return(rs: %p, ret: %d)", rs, ret);
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * The socket is being shut down and we're asked to drop messages that were
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * queued for recvmsg. The caller has unbound the socket so the receive path
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * won't queue any more incoming fragments or messages on the socket.
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota RDSV3_DPRINTF4("rdsv3_clear_recv_queue", "Enter(rs: %p)", rs);
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota RDSV3_FOR_EACH_LIST_NODE_SAFE(inc, tmp, &rs->rs_recv_queue, i_item) {
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota rdsv3_recv_rcvbuf_delta(rs, sk, inc->i_conn->c_lcong,
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota RDSV3_DPRINTF4("rdsv3_clear_recv_queue", "Return(rs: %p)", rs);
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * inc->i_saddr isn't used here because it is only set in the receive