tcp_fusion.c revision fc80c0dfb0c877aee828d778ea32b77fcf7b1ef4
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * CDDL HEADER START
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * The contents of this file are subject to the terms of the
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * Common Development and Distribution License (the "License").
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * You may not use this file except in compliance with the License.
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * See the License for the specific language governing permissions
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * and limitations under the License.
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * When distributing Covered Code, include this CDDL HEADER in each
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * If applicable, add the following below this CDDL HEADER, with the
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * fields enclosed by brackets "[]" replaced with your own identifying
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * information: Portions Copyright [yyyy] [name of copyright owner]
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * CDDL HEADER END
2a8164df8a5f42c8a00f10c67d7bc84f80ae9c41Zhong Wang * Copyright 2007 Sun Microsystems, Inc. All rights reserved.
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * Use is subject to license terms.
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte#pragma ident "%Z%%M% %I% %E% SMI"
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * This file implements TCP fusion - a protocol-less data path for TCP
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * loopback connections. The fusion of two local TCP endpoints occurs
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * at connection establishment time. Various conditions (see details
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * in tcp_fuse()) need to be met for fusion to be successful. If it
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * fails, we fall back to the regular TCP data path; if it succeeds,
2a8164df8a5f42c8a00f10c67d7bc84f80ae9c41Zhong Wang * both endpoints proceed to use tcp_fuse_output() as the transmit path.
2a8164df8a5f42c8a00f10c67d7bc84f80ae9c41Zhong Wang * tcp_fuse_output() enqueues application data directly onto the peer's
2a8164df8a5f42c8a00f10c67d7bc84f80ae9c41Zhong Wang * receive queue; no protocol processing is involved. After enqueueing
a79493184c9332129c9c91500069322f6f3fafddReed * the data, the sender can either push (putnext) data up the receiver's
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * read queue; or the sender can simply return and let the receiver
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * retrieve the enqueued data via the synchronous streams entry point
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * tcp_fuse_rrw(). The latter path is taken if synchronous streams is
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * enabled (the default). It is disabled if sockfs no longer resides
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * directly on top of tcp module due to a module insertion or removal.
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * It also needs to be temporarily disabled when sending urgent data
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * because the tcp_fuse_rrw() path bypasses the M_PROTO processing done
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * by strsock_proto() hook.
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * Sychronization is handled by squeue and the mutex tcp_non_sq_lock.
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * One of the requirements for fusion to succeed is that both endpoints
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * need to be using the same squeue. This ensures that neither side
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * can disappear while the other side is still sending data. By itself,
2a8164df8a5f42c8a00f10c67d7bc84f80ae9c41Zhong Wang * squeue is not sufficient for guaranteeing safety when synchronous
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * streams is enabled. The reason is that tcp_fuse_rrw() doesn't enter
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * the squeue and its access to tcp_rcv_list and other fusion-related
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * fields needs to be sychronized with the sender. tcp_non_sq_lock is
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * used for this purpose. When there is urgent data, the sender needs
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * to push the data up the receiver's streams read queue. In order to
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * avoid holding the tcp_non_sq_lock across putnext(), the sender sets
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * the peer tcp's tcp_fuse_syncstr_plugged bit and releases tcp_non_sq_lock
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * (see macro TCP_FUSE_SYNCSTR_PLUG_DRAIN()). If tcp_fuse_rrw() enters
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * after this point, it will see that synchronous streams is plugged and
2a8164df8a5f42c8a00f10c67d7bc84f80ae9c41Zhong Wang * will wait on tcp_fuse_plugcv. After the sender has finished pushing up
2a8164df8a5f42c8a00f10c67d7bc84f80ae9c41Zhong Wang * all urgent data, it will clear the tcp_fuse_syncstr_plugged bit using
7ff836697c120cb94bd30d5c2204eb9b74718e4cZhong Wang * TCP_FUSE_SYNCSTR_UNPLUG_DRAIN(). This will cause any threads waiting
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * on tcp_fuse_plugcv to return EBUSY, and in turn cause strget() to call
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * getq_noenab() to dequeue data from the stream head instead. Once the
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * data on the stream head has been consumed, tcp_fuse_rrw() may again
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * be used to process tcp_rcv_list. However, if TCP_FUSE_SYNCSTR_STOP()
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * has been called, all future calls to tcp_fuse_rrw() will return EBUSY,
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * effectively disabling synchronous streams.
2a8164df8a5f42c8a00f10c67d7bc84f80ae9c41Zhong Wang * The following note applies only to the synchronous streams mode.
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * Flow control is done by checking the size of receive buffer and
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * the number of data blocks, both set to different limits. This is
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * different than regular streams flow control where cumulative size
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * check dominates block count check -- streams queue high water mark
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * typically represents bytes. Each enqueue triggers notifications
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * to the receiving process; a build up of data blocks indicates a
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * slow receiver and the sender should be blocked or informed at the
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * earliest moment instead of further wasting system resources. In
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * effect, this is equivalent to limiting the number of outstanding
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * segments in flight.
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * Setting this to false means we disable fusion altogether and
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * loopback connections would go through the protocol paths.
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * Enabling this flag allows sockfs to retrieve data directly
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * from a fused tcp endpoint using synchronous streams interface.
2a8164df8a5f42c8a00f10c67d7bc84f80ae9c41Zhong Wang * This is the minimum amount of outstanding writes allowed on
7ff836697c120cb94bd30d5c2204eb9b74718e4cZhong Wang * a synchronous streams-enabled receiving endpoint before the
2a8164df8a5f42c8a00f10c67d7bc84f80ae9c41Zhong Wang * sender gets flow-controlled. Setting this value to 0 means
2a8164df8a5f42c8a00f10c67d7bc84f80ae9c41Zhong Wang * that the data block limit is equivalent to the byte count
2a8164df8a5f42c8a00f10c67d7bc84f80ae9c41Zhong Wang * limit, which essentially disables the check.
7ff836697c120cb94bd30d5c2204eb9b74718e4cZhong Wanguint_t tcp_fusion_rcv_unread_min = TCP_FUSION_RCV_UNREAD_MIN;
2a8164df8a5f42c8a00f10c67d7bc84f80ae9c41Zhong Wang * Return true if this connection needs some IP functionality
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * If ire is not cached, do not use fusion
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * There is no need to hold conn_lock here because when called
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * from tcp_fuse() there can be no window where conn_ire_cache
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * can change. This is not true whe called from
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * tcp_fuse_output(). conn_ire_cache can become null just
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * after the check, but it's ok if a few packets are delivered
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * in the fused state.
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte if (CONN_OUTBOUND_POLICY_PRESENT(tcp->tcp_connp, ipss))
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte if (CONN_INBOUND_POLICY_PRESENT(tcp->tcp_connp, ipss))
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte if (CONN_OUTBOUND_POLICY_PRESENT_V6(tcp->tcp_connp, ipss))
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte if (CONN_INBOUND_POLICY_PRESENT_V6(tcp->tcp_connp, ipss))
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * This routine gets called by the eager tcp upon changing state from
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * SYN_RCVD to ESTABLISHED. It fuses a direct path between itself
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * and the active connect tcp such that the regular tcp processings
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * may be bypassed under allowable circumstances. Because the fusion
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * requires both endpoints to be in the same squeue, it does not work
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * for simultaneous active connects because there is no easy way to
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * switch from one squeue to another once the connection is created.
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * This is different from the eager tcp case where we assign it the
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * same squeue as the one given to the active connect tcp during open.
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Fortetcp_fuse(tcp_t *tcp, uchar_t *iphdr, tcph_t *tcph)
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte ip_stack_t *ipst = tcps->tcps_netstack->netstack_ip;
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * We need to inherit q_hiwat of the listener tcp, but we can't
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * really use tcp_listener since we get here after sending up
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * T_CONN_IND and tcp_wput_accept() may be called independently,
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * at which point tcp_listener is cleared; this is why we use
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * tcp_saved_listener. The listener itself is guaranteed to be
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * around until tcp_accept_finish() is called on this eager --
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * this won't happen until we're done since we're inside the
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * eager's perimeter now.
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * Lookup peer endpoint; search for the remote endpoint having
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * the reversed address-port quadruplet in ESTABLISHED state,
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * which is guaranteed to be unique in the system. Zone check
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * is applied accordingly for loopback address, but not for
2a8164df8a5f42c8a00f10c67d7bc84f80ae9c41Zhong Wang * local address since we want fusion to happen across Zones.
2a8164df8a5f42c8a00f10c67d7bc84f80ae9c41Zhong Wang peer_connp = ipcl_conn_tcp_lookup_reversed_ipv4(connp,
2a8164df8a5f42c8a00f10c67d7bc84f80ae9c41Zhong Wang peer_connp = ipcl_conn_tcp_lookup_reversed_ipv6(connp,
2a8164df8a5f42c8a00f10c67d7bc84f80ae9c41Zhong Wang * We can only proceed if peer exists, resides in the same squeue
2a8164df8a5f42c8a00f10c67d7bc84f80ae9c41Zhong Wang * as our conn and is not raw-socket. The squeue assignment of
2a8164df8a5f42c8a00f10c67d7bc84f80ae9c41Zhong Wang * this eager tcp was done earlier at the time of SYN processing
2a8164df8a5f42c8a00f10c67d7bc84f80ae9c41Zhong Wang * in ip_fanout_tcp{_v6}. Note that similar squeues by itself
2a8164df8a5f42c8a00f10c67d7bc84f80ae9c41Zhong Wang * doesn't guarantee a safe condition to fuse, hence we perform
2a8164df8a5f42c8a00f10c67d7bc84f80ae9c41Zhong Wang * additional tests below.
2a8164df8a5f42c8a00f10c67d7bc84f80ae9c41Zhong Wang ASSERT(peer_connp == NULL || peer_connp != connp);
2a8164df8a5f42c8a00f10c67d7bc84f80ae9c41Zhong Wang if (peer_connp == NULL || peer_connp->conn_sqp != connp->conn_sqp ||
2a8164df8a5f42c8a00f10c67d7bc84f80ae9c41Zhong Wang peer_tcp = peer_connp->conn_tcp; /* active connect tcp */
2a8164df8a5f42c8a00f10c67d7bc84f80ae9c41Zhong Wang ASSERT(peer_tcp != NULL && peer_tcp != tcp && !peer_tcp->tcp_fused);
2a8164df8a5f42c8a00f10c67d7bc84f80ae9c41Zhong Wang ASSERT(peer_tcp->tcp_loopback && peer_tcp->tcp_loopback_peer == NULL);
a79493184c9332129c9c91500069322f6f3fafddReed * Fuse the endpoints; we perform further checks against both
a79493184c9332129c9c91500069322f6f3fafddReed * tcp endpoints to ensure that a fusion is allowed to happen.
a79493184c9332129c9c91500069322f6f3fafddReed * In particular we bail out for non-simple TCP/IP or if IPsec/
a79493184c9332129c9c91500069322f6f3fafddReed * IPQoS policy/kernel SSL exists.
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte !IPP_ENABLED(IPP_LOCAL_OUT|IPP_LOCAL_IN, ipst)) {
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte ASSERT(!TCP_IS_DETACHED(peer_tcp) && peer_rq != NULL);
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * We need to drain data on both endpoints during unfuse.
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * If we need to send up SIGURG at the time of draining,
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * we want to be sure that an mblk is readily available.
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * This is why we pre-allocate the M_PCSIG mblks for both
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * endpoints which will only be used during/after unfuse.
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte /* Allocate M_SETOPTS mblk */
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte if ((mp = allocb(sizeof (*stropt), BPRI_HI)) == NULL)
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte /* Fuse both endpoints */
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * We never use regular tcp paths in fusion and should
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * therefore clear tcp_unsent on both endpoints. Having
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * them set to non-zero values means asking for trouble
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * especially after unfuse, where we may end up sending
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * through regular tcp paths which expect xmit_list and
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * friends to be correctly setup.
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * At this point we are a detached eager tcp and therefore
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * don't have a queue assigned to us until accept happens.
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * In the mean time the peer endpoint may immediately send
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * us data as soon as fusion is finished, and we need to be
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * able to flow control it in case it sends down huge amount
2a8164df8a5f42c8a00f10c67d7bc84f80ae9c41Zhong Wang * of data while we're still detached. To prevent that we
2a8164df8a5f42c8a00f10c67d7bc84f80ae9c41Zhong Wang * inherit the listener's q_hiwat value; this is temporary
2a8164df8a5f42c8a00f10c67d7bc84f80ae9c41Zhong Wang * since we'll repeat the process in tcp_accept_finish().
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * Set the stream head's write offset value to zero since we
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * won't be needing any room for TCP/IP headers; tell it to
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * not break up the writes (this would reduce the amount of
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * work done by kmem); and configure our receive buffer.
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * Note that we can only do this for the active connect tcp
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * since our eager is still detached; it will be dealt with
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * later in tcp_accept_finish().
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte stropt->so_flags = SO_MAXBLK | SO_WROFF | SO_HIWAT;
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte stropt->so_maxblk = tcp_maxpsz_set(peer_tcp, B_FALSE);
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * Record the stream head's high water mark for
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * peer endpoint; this is used for flow-control
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte * purposes in tcp_fuse_output().
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte stropt->so_hiwat = tcp_fuse_set_rcv_hiwat(peer_tcp,
fcf3ce441efd61da9bb2884968af01cb7c1452ccJohn Forte /* Send the options up */
goto unfuse;
if (send_size == 0) {
return (B_TRUE);
if (urgent) {
goto unfuse;
goto unfuse;
goto unfuse;
goto unfuse;
NULL);
if (urgent)
return (B_TRUE);
return (B_FALSE);
#ifdef DEBUG
return (B_TRUE);
return (B_FALSE);
#ifdef DEBUG
return (B_TRUE);
return (EBUSY);
return (EBUSY);
goto plugged;
int res = 0;
int error = 0;
goto done;
goto done;
goto done;
done:
return (error);
if (on)
return (rwnd);
if (tcp_fusion_rcv_unread_min == 0) {
return (maxpsz);