tcp_fusion.c revision f4b3ec61df05330d25f55a36b975b4d7519fdeb1
/*
* CDDL HEADER START
*
* The contents of this file are subject to the terms of the
* Common Development and Distribution License (the "License").
* You may not use this file except in compliance with the License.
*
* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
* See the License for the specific language governing permissions
* and limitations under the License.
*
* When distributing Covered Code, include this CDDL HEADER in each
* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
* If applicable, add the following below this CDDL HEADER, with the
* fields enclosed by brackets "[]" replaced with your own identifying
* information: Portions Copyright [yyyy] [name of copyright owner]
*
* CDDL HEADER END
*/
/*
* Copyright 2007 Sun Microsystems, Inc. All rights reserved.
* Use is subject to license terms.
*/
#pragma ident "%Z%%M% %I% %E% SMI"
#include <inet/tcp_impl.h>
#include <inet/ipsec_impl.h>
#include <inet/ipclassifier.h>
#include <inet/ipp_common.h>
/*
* This file implements TCP fusion - a protocol-less data path for TCP
* loopback connections. The fusion of two local TCP endpoints occurs
* at connection establishment time. Various conditions (see details
* in tcp_fuse()) need to be met for fusion to be successful. If it
* fails, we fall back to the regular TCP data path; if it succeeds,
* both endpoints proceed to use tcp_fuse_output() as the transmit path.
* tcp_fuse_output() enqueues application data directly onto the peer's
* receive queue; no protocol processing is involved. After enqueueing
* the data, the sender can either push (putnext) data up the receiver's
* read queue; or the sender can simply return and let the receiver
* retrieve the enqueued data via the synchronous streams entry point
* tcp_fuse_rrw(). The latter path is taken if synchronous streams is
* enabled (the default). It is disabled if sockfs no longer resides
* directly on top of tcp module due to a module insertion or removal.
* It also needs to be temporarily disabled when sending urgent data
* because the tcp_fuse_rrw() path bypasses the M_PROTO processing done
* by strsock_proto() hook.
*
* Sychronization is handled by squeue and the mutex tcp_non_sq_lock.
* One of the requirements for fusion to succeed is that both endpoints
* need to be using the same squeue. This ensures that neither side
* can disappear while the other side is still sending data. By itself,
* squeue is not sufficient for guaranteeing safety when synchronous
* streams is enabled. The reason is that tcp_fuse_rrw() doesn't enter
* the squeue and its access to tcp_rcv_list and other fusion-related
* fields needs to be sychronized with the sender. tcp_non_sq_lock is
* used for this purpose. When there is urgent data, the sender needs
* to push the data up the receiver's streams read queue. In order to
* avoid holding the tcp_non_sq_lock across putnext(), the sender sets
* the peer tcp's tcp_fuse_syncstr_plugged bit and releases tcp_non_sq_lock
* (see macro TCP_FUSE_SYNCSTR_PLUG_DRAIN()). If tcp_fuse_rrw() enters
* after this point, it will see that synchronous streams is plugged and
* will wait on tcp_fuse_plugcv. After the sender has finished pushing up
* all urgent data, it will clear the tcp_fuse_syncstr_plugged bit using
* TCP_FUSE_SYNCSTR_UNPLUG_DRAIN(). This will cause any threads waiting
* on tcp_fuse_plugcv to return EBUSY, and in turn cause strget() to call
* getq_noenab() to dequeue data from the stream head instead. Once the
* data on the stream head has been consumed, tcp_fuse_rrw() may again
* be used to process tcp_rcv_list. However, if TCP_FUSE_SYNCSTR_STOP()
* has been called, all future calls to tcp_fuse_rrw() will return EBUSY,
* effectively disabling synchronous streams.
*
* The following note applies only to the synchronous streams mode.
*
* Flow control is done by checking the size of receive buffer and
* the number of data blocks, both set to different limits. This is
* different than regular streams flow control where cumulative size
* check dominates block count check -- streams queue high water mark
* typically represents bytes. Each enqueue triggers notifications
* to the receiving process; a build up of data blocks indicates a
* slow receiver and the sender should be blocked or informed at the
* earliest moment instead of further wasting system resources. In
* effect, this is equivalent to limiting the number of outstanding
* segments in flight.
*/
/*
* Setting this to false means we disable fusion altogether and
* loopback connections would go through the protocol paths.
*/
/*
* Enabling this flag allows sockfs to retrieve data directly
* from a fused tcp endpoint using synchronous streams interface.
*/
/*
* This is the minimum amount of outstanding writes allowed on
* a synchronous streams-enabled receiving endpoint before the
* sender gets flow-controlled. Setting this value to 0 means
* that the data block limit is equivalent to the byte count
* limit, which essentially disables the check.
*/
#define TCP_FUSION_RCV_UNREAD_MIN 8
static void tcp_fuse_syncstr_enable(tcp_t *);
static void tcp_fuse_syncstr_disable(tcp_t *);
/*
* Return true if this connection needs some IP functionality
*/
static boolean_t
{
return (B_TRUE);
return (B_TRUE);
return (B_TRUE);
} else {
return (B_TRUE);
return (B_TRUE);
return (B_TRUE);
}
return (B_TRUE);
return (B_FALSE);
}
/*
* This routine gets called by the eager tcp upon changing state from
* SYN_RCVD to ESTABLISHED. It fuses a direct path between itself
* and the active connect tcp such that the regular tcp processings
* may be bypassed under allowable circumstances. Because the fusion
* requires both endpoints to be in the same squeue, it does not work
* for simultaneous active connects because there is no easy way to
* switch from one squeue to another once the connection is created.
* This is different from the eager tcp case where we assign it the
* same squeue as the one given to the active connect tcp during open.
*/
void
{
netstack_t *ns;
/*
* We need to inherit q_hiwat of the listener tcp, but we can't
* really use tcp_listener since we get here after sending up
* T_CONN_IND and tcp_wput_accept() may be called independently,
* at which point tcp_listener is cleared; this is why we use
* tcp_saved_listener. The listener itself is guaranteed to be
* around until tcp_accept_finish() is called on this eager --
* this won't happen until we're done since we're inside the
* eager's perimeter now.
*/
/*
* Lookup peer endpoint; search for the remote endpoint having
* the reversed address-port quadruplet in ESTABLISHED state,
* which is guaranteed to be unique in the system. Zone check
* is applied accordingly for loopback address, but not for
* local address since we want fusion to happen across Zones.
*/
} else {
}
/*
* We can only proceed if peer exists, resides in the same squeue
* as our conn and is not raw-socket. The squeue assignment of
* this eager tcp was done earlier at the time of SYN processing
* in ip_fanout_tcp{_v6}. Note that similar squeues by itself
* doesn't guarantee a safe condition to fuse, hence we perform
* additional tests below.
*/
!IPCL_IS_TCP(peer_connp)) {
if (peer_connp != NULL) {
}
return;
}
/*
* Fuse the endpoints; we perform further checks against both
* tcp endpoints to ensure that a fusion is allowed to happen.
*/
struct stroptions *stropt;
/*
* We need to drain data on both endpoints during unfuse.
* If we need to send up SIGURG at the time of draining,
* we want to be sure that an mblk is readily available.
* This is why we pre-allocate the M_PCSIG mblks for both
*/
goto failed;
goto failed;
/* Allocate M_SETOPTS mblk */
goto failed;
/* Fuse both endpoints */
/*
* We never use regular tcp paths in fusion and should
* therefore clear tcp_unsent on both endpoints. Having
* them set to non-zero values means asking for trouble
* especially after unfuse, where we may end up sending
* through regular tcp paths which expect xmit_list and
* friends to be correctly setup.
*/
/*
* At this point we are a detached eager tcp and therefore
* don't have a queue assigned to us until accept happens.
* In the mean time the peer endpoint may immediately send
* us data as soon as fusion is finished, and we need to be
* able to flow control it in case it sends down huge amount
* of data while we're still detached. To prevent that we
* inherit the listener's q_hiwat value; this is temporary
* since we'll repeat the process in tcp_accept_finish().
*/
(void) tcp_fuse_set_rcv_hiwat(tcp,
/*
* Set the stream head's write offset value to zero since we
* not break up the writes (this would reduce the amount of
* work done by kmem); and configure our receive buffer.
* Note that we can only do this for the active connect tcp
* since our eager is still detached; it will be dealt with
* later in tcp_accept_finish().
*/
/*
* Record the stream head's high water mark for
* peer endpoint; this is used for flow-control
* purposes in tcp_fuse_output().
*/
/* Send the options up */
} else {
}
return;
}
}
}
/*
* Unfuse a previously-fused pair of tcp loopback endpoints.
*/
void
{
/*
* We disable synchronous streams, drain any queued data and
* clear tcp_direct_sockfs. The synchronous streams entry
* points will become no-ops after this point.
*/
/*
* Update th_seq and th_ack in the header template
*/
/* Unfuse the endpoints */
}
/*
* Fusion output routine for urgent data. This routine is called by
* tcp_fuse_output() for handling non-M_DATA mblks.
*/
void
{
struct T_exdata_ind *tei;
/*
* Urgent data arrives in the form of T_EXDATA_REQ from above.
* Each occurence denotes a new urgent pointer. For each new
* urgent pointer we signal (SIGURG) the receiving app to indicate
* that it needs to go into urgent mode. This is similar to the
* urgent data handling in the regular tcp. We don't need to keep
* track of where the urgent pointer is, because each T_EXDATA_REQ
* "advances" the urgent pointer for us.
*
* The actual urgent data carried by T_EXDATA_REQ is then prepended
* by a T_EXDATA_IND before being enqueued behind any existing data
* destined for the receiving app. There is only a single urgent
* pointer (out-of-band mark) for a given tcp. If the new urgent
* data arrives before the receiving app reads some existing urgent
* data, the previous marker is lost. This behavior is emulated
* accordingly below, by removing any existing T_EXDATA_IND messages
* and essentially converting old urgent data into non-urgent.
*/
/* Let sender get out of urgent mode */
/*
* This flag indicates that a signal needs to be sent up.
* This flag will only get cleared once SIGURG is delivered and
* is not affected by the tcp_fused flag -- delivery will still
* happen even after an endpoint is unfused, to handle the case
* sending urgent data and the accept is not yet finished.
*/
/* Reuse T_EXDATA_REQ mblk for T_EXDATA_IND */
/*
* Remove existing T_EXDATA_IND, keep the data which follows
* it and relink our list. Note that we don't modify the
* tcp_rcv_last_tail since it never points to T_EXDATA_IND.
*/
}
}
}
/*
* Fusion output routine, called by tcp_output() and tcp_wput_proto().
* If we are modifying any member that can be changed outside the squeue,
* like tcp_flow_stopped, we need to take tcp_non_sq_lock.
*/
{
/* If this connection requires IP, unfuse and use regular path */
goto unfuse;
}
if (send_size == 0) {
return (B_TRUE);
}
/*
* Handle urgent data; we either send up SIGURG to the peer now
* or do it later when we drain, in case the peer is detached
* or if we're short of memory for M_PCSIG mblk.
*/
if (urgent) {
/*
* We stop synchronous streams when we have urgent data
* queued to prevent tcp_fuse_rrw() from pulling it. If
* for some reasons the urgent data can't be delivered
* below, synchronous streams will remain stopped until
* someone drains the tcp_rcv_list.
*/
}
/*
* Build ip and tcp header to satisfy FW_HOOKS.
* We only build it when any hook is present.
*/
/* If tcp_xmit_mp fails, use regular path */
goto unfuse;
/* PFHooks: LOOPBACK_OUT */
} else {
}
goto unfuse;
/* PFHooks: LOOPBACK_IN */
goto unfuse;
} else {
goto unfuse;
}
/* Data length might be changed by FW_HOOKS */
/*
* The message duplicated by tcp_xmit_mp is freed.
* Note: the original message passed in remains unchanged.
*/
}
/*
* Wake up and signal the peer; it is okay to do this before
* enqueueing because we are holding the lock. One of the
* advantages of synchronous streams is the ability for us to
* find out when the application performs a read on the socket,
* by way of tcp_fuse_rrw() entry point being called. Every
* data that gets enqueued onto the receiver is treated as if
* it has arrived at the receiving endpoint, thus generating
* case. However, we only wake up the application when necessary,
* i.e. during the first enqueue. When tcp_fuse_rrw() is called
* it will send everything upstream.
*/
!TCP_IS_DETACHED(peer_tcp)) {
}
/*
* Enqueue data into the peer's receive list; we may or may not
* drain the contents depending on the conditions below.
*/
/* In case it wrapped around and also to keep it constant */
/*
* Exercise flow-control when needed; we will get back-enabled
* in either tcp_accept_finish(), tcp_unfuse(), or tcp_fuse_rrw().
* If tcp_direct_sockfs is on or if the peer endpoint is detached,
* we emulate streams flow control by checking the peer's queue
* size and high water mark; otherwise we simply use canputnext()
* to decide if we need to stop our flow.
*
* The outstanding unread data block check does not apply for a
* detached receiver; this is to avoid unnecessary blocking of the
* sender while the accept is currently in progress and is quite
* similar to the regular tcp.
*/
/*
* Since we are accessing our tcp_flow_stopped and might modify it,
* we need to take tcp->tcp_non_sq_lock. The lock for the highest
* address is held first. Dropping peer_tcp->tcp_non_sq_lock should
* not be an issue here since we are within the squeue and the peer
* won't disappear.
*/
} else {
}
if (!flow_stopped &&
(!peer_tcp->tcp_direct_sockfs &&
} else if (flow_stopped &&
}
/* Need to adjust the following SNMP MIB-related variables */
if (!TCP_IS_DETACHED(peer_tcp)) {
/*
* Drain the peer's receive queue it has urgent data or if
* we're not flow-controlled. There is no need for draining
* normal data when tcp_direct_sockfs is on because the peer
* will pull the data via tcp_fuse_rrw().
*/
/*
* For TLI-based streams, a thread in tcp_accept_swap()
* can race with us. That thread will ensure that the
* correct peer_tcp->tcp_rq is globally visible before
* peer_tcp->tcp_detached is visible as clear, but we
* must also ensure that the load of tcp_rq cannot be
* reordered to be before the tcp_detached check.
*/
NULL);
/*
* If synchronous streams was stopped above due
* to the presence of urgent data, re-enable it.
*/
if (urgent)
}
}
return (B_TRUE);
return (B_FALSE);
}
/*
* This routine gets called to deliver data upstream on a fused or
* previously fused tcp loopback endpoint; the latter happens only
* when there is a pending SIGURG signal plus urgent data that can't
* be sent upstream in the past.
*/
{
#ifdef DEBUG
#endif
/* No need for the push timer now, in case it was scheduled */
if (tcp->tcp_push_tid != 0) {
tcp->tcp_push_tid = 0;
}
/*
* If there's urgent data sitting in receive list and we didn't
* get a chance to send up a SIGURG signal, make sure we send
* it first before draining in order to ensure that SIOCATMARK
* works properly.
*/
if (tcp->tcp_fused_sigurg) {
/*
* sigurg_mpp is normally NULL, i.e. when we're still
* fused and didn't get here because of tcp_unfuse().
* In this case try hard to allocate the M_PCSIG mblk.
*/
if (sigurg_mpp == NULL &&
/* Alloc failed; try again next time */
return (B_TRUE);
} else if (sigurg_mpp != NULL) {
/*
* Use the supplied M_PCSIG mblk; it means we're
* either unfused or in the process of unfusing,
* and the drain must happen now.
*/
mp = *sigurg_mpp;
*sigurg_mpp = NULL;
}
/* Send up the signal */
/*
* Let the regular tcp_rcv_drain() path handle
* draining the data if we're no longer fused.
*/
return (B_FALSE);
}
/*
* each M_DATA that gets enqueued onto the receiver. At this point
* we are about to drain any queued data via putnext(). In order
* to avoid extraneous signal generation from strrput(), we set
* STRGETINPROG flag at the stream head prior to the draining and
* restore it afterwards. This masks out signal generation only
* for M_DATA messages and does not affect urgent data.
*/
if (tcp->tcp_direct_sockfs)
strrput_sig(q, B_FALSE);
/* Drain the data */
#ifdef DEBUG
#endif
}
if (tcp->tcp_direct_sockfs)
strrput_sig(q, B_TRUE);
tcp->tcp_rcv_cnt = 0;
tcp->tcp_fuse_rcv_unread_cnt = 0;
return (B_TRUE);
}
/*
* Synchronous stream entry point for sockfs to retrieve
* data directly from tcp_rcv_list.
* tcp_fuse_rrw() might end up modifying the peer's tcp_flow_stopped,
* for which it must take the tcp_non_sq_lock of the peer as well
* making any change. The order of taking the locks is based on
* the TCP pointer itself. Before we get the peer we need to take
* our tcp_non_sq_lock so that the peer doesn't disappear. However,
* we cannot drop the lock if we have to grab the peer's lock (because
* of ordering), since the peer might disappear in the interim. So,
* we take our tcp_non_sq_lock, get the peer, increment the ref on the
* peer's conn, drop all the locks and then take the tcp_non_sq_lock in the
* desired order. Incrementing the conn ref on the peer means that the
* peer won't disappear when we drop our tcp_non_sq_lock.
*/
int
{
/*
* If tcp_fuse_syncstr_plugged is set, then another thread is moving
* the underlying data to the stream head. We need to wait until it's
* done, then return EBUSY so that strget() will dequeue data from the
* stream head to ensure data is drained in-order.
*/
if (tcp->tcp_fuse_syncstr_plugged) {
do {
} while (tcp->tcp_fuse_syncstr_plugged);
return (EBUSY);
}
/*
* If someone had turned off tcp_direct_sockfs or if synchronous
* streams is stopped, we return EBUSY. This causes strget() to
* dequeue data from the stream head instead.
*/
return (EBUSY);
}
/*
* Grab lock in order. The highest addressed tcp is locked first.
* We don't do this within the tcp_rcv_list check since if we
* have to drop the lock, for ordering, then the tcp_rcv_list
* could change.
*/
/* This might have changed in the interim */
if (tcp->tcp_fuse_syncstr_plugged) {
goto plugged;
}
} else {
}
/*
* At this point nothing should be left in tcp_rcv_list.
* The only possible case where we would have a chain of
* b_next-linked messages is urgent data, but we wouldn't
* be here if that's true since urgent data is delivered
* via putnext() and synchronous streams is stopped until
* tcp_fuse_rcv_drain() is finished.
*/
tcp->tcp_rcv_cnt = 0;
tcp->tcp_fuse_rcv_unread_cnt = 0;
if (peer_tcp->tcp_flow_stopped) {
}
}
/*
* Either we just dequeued everything or we get here from sockfs
* and have nothing to return; in this case clear RSLEEP.
*/
STR_WAKEUP_CLEAR(STREAM(q));
return (0);
}
/*
* Synchronous stream entry point used by certain ioctls to retrieve
* information about or peek into the tcp_rcv_list.
*/
int
{
int res = 0;
int error = 0;
/* If shutdown on read has happened, return nothing */
goto done;
}
/*
* It is OK not to return an answer if tcp_rcv_list is
* currently not accessible.
*/
goto done;
if (cmd & INFOD_COUNT) {
/*
* We have at least one message and
* could return only one at a time.
*/
res |= INFOD_COUNT;
}
if (cmd & INFOD_BYTES) {
/*
* Return size of all data messages.
*/
res |= INFOD_BYTES;
}
if (cmd & INFOD_FIRSTBYTES) {
/*
* Return size of first data message.
*/
res |= INFOD_FIRSTBYTES;
}
if (cmd & INFOD_COPYOUT) {
int n;
} else {
}
/*
* Return data contents of first message.
*/
goto done;
}
}
res |= INFOD_COPYOUT;
}
done:
return (error);
}
/*
* Enable synchronous streams on a fused tcp loopback endpoint.
*/
static void
{
/* We can only enable synchronous streams for sockfs mode */
if (!tcp->tcp_direct_sockfs)
return;
/*
* We replace our q_qinfo with one that has the qi_rwp entry point.
* Clear SR_SIGALLDATA because we generate the equivalent signal(s)
* for every enqueued data in tcp_fuse_output().
*/
}
/*
* Disable synchronous streams on a fused tcp loopback endpoint.
*/
static void
{
if (!tcp->tcp_direct_sockfs)
return;
/*
* Reset q_qinfo to point to the default tcp entry points.
* Also restore SR_SIGALLDATA so that strrput() can generate
* the signals again for future M_DATA messages.
*/
}
/*
* Enable synchronous streams on a pair of fused tcp endpoints.
*/
void
{
}
/*
* Allow or disallow signals to be generated by strrput().
*/
static void
{
if (on)
else
}
/*
* Disable synchronous streams on a pair of fused tcp endpoints and drain
* any queued data; called either during unfuse or upon transitioning from
* a socket to a stream endpoint due to _SIOCSOCKFALLBACK.
*/
void
{
/*
* Force any tcp_fuse_rrw() calls to block until we've moved the data
* onto the stream head.
*/
/*
* Drain any pending data; the detached check is needed because
* we may be called as a result of a tcp_unfuse() triggered by
* tcp_fuse_output(). Note that in case of a detached tcp, the
* draining will happen later after the tcp is unfused. For non-
* urgent data, this can be handled by the regular tcp_rcv_drain().
* If we have urgent data sitting in the receive list, we will
* need to send up a SIGURG signal first before draining the data.
* All of these will be handled by the code in tcp_fuse_rcv_drain()
* when called from tcp_rcv_drain().
*/
if (!TCP_IS_DETACHED(tcp)) {
}
if (!TCP_IS_DETACHED(peer_tcp)) {
}
/*
* Make all current and future tcp_fuse_rrw() calls fail with EBUSY.
* To ensure threads don't sneak past the checks in tcp_fuse_rrw(),
* a given stream must be stopped prior to being unplugged (but the
* ordering of operations between the streams is unimportant).
*/
/* Lift up any flow-control conditions */
if (tcp->tcp_flow_stopped) {
}
if (peer_tcp->tcp_flow_stopped) {
}
/* Disable synchronous streams */
}
/*
* Calculate the size of receive buffer for a fused tcp endpoint.
*/
{
/* Ensure that value is within the maximum upper bound */
/* Obey the absolute minimum tcp receive high water mark */
/*
* Round up to system page size in case SO_RCVBUF is modified
* after SO_SNDBUF; the latter is also similarly rounded up.
*/
return (rwnd);
}
/*
* Calculate the maximum outstanding unread data block for a fused tcp endpoint.
*/
int
{
/*
* In the fused loopback case, we want the stream head to split
* up larger writes into smaller chunks for a more accurate flow-
* control accounting. Our maxpsz is half of the sender's send
* buffer or the receiver's receive buffer, whichever is smaller.
* We round up the buffer to system page size due to the lack of
* TCP MSS concept in Fusion.
*/
/*
* Calculate the peer's limit for the number of outstanding unread
* data block. This is the amount of data blocks that are allowed
* to reside in the receiver's queue before the sender gets flow
* controlled. It is used only in the synchronous streams mode as
* a way to throttle the sender when it performs consecutive writes
* faster than can be read. The value is derived from SO_SNDBUF in
* order to give the sender some control; we divide it with a large
* value (16KB) to produce a fairly low initial limit.
*/
if (tcp_fusion_rcv_unread_min == 0) {
/* A value of 0 means that we disable the check */
} else {
}
return (maxpsz);
}