tcp_fusion.c revision 0a0e9771ca0211c15f3ac4466b661c145feeb9e4
/*
* CDDL HEADER START
*
* The contents of this file are subject to the terms of the
* Common Development and Distribution License (the "License").
* You may not use this file except in compliance with the License.
*
* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
* See the License for the specific language governing permissions
* and limitations under the License.
*
* When distributing Covered Code, include this CDDL HEADER in each
* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
* If applicable, add the following below this CDDL HEADER, with the
* fields enclosed by brackets "[]" replaced with your own identifying
* information: Portions Copyright [yyyy] [name of copyright owner]
*
* CDDL HEADER END
*/
/*
* Copyright 2009 Sun Microsystems, Inc. All rights reserved.
* Use is subject to license terms.
*/
#include <inet/tcp_impl.h>
#include <inet/ipsec_impl.h>
#include <inet/ipclassifier.h>
#include <inet/ipp_common.h>
/*
* This file implements TCP fusion - a protocol-less data path for TCP
* loopback connections. The fusion of two local TCP endpoints occurs
* at connection establishment time. Various conditions (see details
* in tcp_fuse()) need to be met for fusion to be successful. If it
* fails, we fall back to the regular TCP data path; if it succeeds,
* both endpoints proceed to use tcp_fuse_output() as the transmit path.
* tcp_fuse_output() enqueues application data directly onto the peer's
* receive queue; no protocol processing is involved.
*
* Sychronization is handled by squeue and the mutex tcp_non_sq_lock.
* One of the requirements for fusion to succeed is that both endpoints
* need to be using the same squeue. This ensures that neither side
* can disappear while the other side is still sending data. Flow
* control information is manipulated outside the squeue, so the
* tcp_non_sq_lock must be held when touching tcp_flow_stopped.
*/
/*
* Setting this to false means we disable fusion altogether and
* loopback connections would go through the protocol paths.
*/
/*
* Return true if this connection needs some IP functionality
*/
static boolean_t
{
/*
* If ire is not cached, do not use fusion
*/
/*
* There is no need to hold conn_lock here because when called
* from tcp_fuse() there can be no window where conn_ire_cache
* can change. This is not true when called from
* tcp_fuse_output() as conn_ire_cache can become null just
* after the check. It will be necessary to recheck for a NULL
* conn_ire_cache in tcp_fuse_output() to avoid passing a
* stale ill pointer to FW_HOOKS.
*/
return (B_TRUE);
}
return (B_TRUE);
return (B_TRUE);
return (B_TRUE);
} else {
return (B_TRUE);
return (B_TRUE);
return (B_TRUE);
}
return (B_TRUE);
return (B_FALSE);
}
/*
* This routine gets called by the eager tcp upon changing state from
* SYN_RCVD to ESTABLISHED. It fuses a direct path between itself
* and the active connect tcp such that the regular tcp processings
* may be bypassed under allowable circumstances. Because the fusion
* requires both endpoints to be in the same squeue, it does not work
* for simultaneous active connects because there is no easy way to
* switch from one squeue to another once the connection is created.
* This is different from the eager tcp case where we assign it the
* same squeue as the one given to the active connect tcp during open.
*/
void
{
netstack_t *ns;
/*
* We need to inherit tcp_recv_hiwater of the listener tcp,
* but we can't really use tcp_listener since we get here after
* sending up T_CONN_IND and tcp_wput_accept() may be called
* independently, at which point tcp_listener is cleared;
* this is why we use tcp_saved_listener. The listener itself
* is guaranteed to be around until tcp_accept_finish() is called
* on this eager -- this won't happen until we're done since we're
* inside the eager's perimeter now.
*
* We can also get called in the case were a connection needs
* to be re-fused. In this case tcp_saved_listener will be
* NULL but tcp_refuse will be true.
*/
/*
* Lookup peer endpoint; search for the remote endpoint having
* the reversed address-port quadruplet in ESTABLISHED state,
* which is guaranteed to be unique in the system. Zone check
* is applied accordingly for loopback address, but not for
* local address since we want fusion to happen across Zones.
*/
} else {
}
/*
* We can only proceed if peer exists, resides in the same squeue
* as our conn and is not raw-socket. We also restrict fusion to
* endpoints of the same type (STREAMS or non-STREAMS). The squeue
* assignment of this eager tcp was done earlier at the time of SYN
* processing in ip_fanout_tcp{_v6}. Note that similar squeues by
* itself doesn't guarantee a safe condition to fuse, hence we perform
* additional tests below.
*/
!IPCL_IS_TCP(peer_connp) ||
if (peer_connp != NULL) {
}
return;
}
/*
* Due to IRE changes the peer and us might not agree on tcp_loopback.
* We bail in that case.
*/
if (!peer_tcp->tcp_loopback) {
return;
}
/*
* Fuse the endpoints; we perform further checks against both
* tcp endpoints to ensure that a fusion is allowed to happen.
* the connection is quiescent to cover the case when we are
* trying to re-enable fusion after IPobservability is turned off.
*/
/*
* We need to drain data on both endpoints during unfuse.
* If we need to send up SIGURG at the time of draining,
* we want to be sure that an mblk is readily available.
* This is why we pre-allocate the M_PCSIG mblks for both
* The mblk might already exist if we are doing a re-fuse.
*/
goto failed;
}
goto failed;
}
goto failed;
}
/* Fuse both endpoints */
/*
* We never use regular tcp paths in fusion and should
* therefore clear tcp_unsent on both endpoints. Having
* them set to non-zero values means asking for trouble
* especially after unfuse, where we may end up sending
* through regular tcp paths which expect xmit_list and
* friends to be correctly setup.
*/
if (!tcp->tcp_refuse) {
/*
* Set receive buffer and max packet size for the
* active open tcp.
* eager's values will be set in tcp_accept_finish.
*/
(void) tcp_rwnd_set(peer_tcp,
/*
* Set the write offset value to zero since we won't
*/
struct stroptions *stropt;
/* Send the options up */
} else {
struct sock_proto_props sopp;
/* The peer is a non-STREAMS end point */
sopp.sopp_wroff = 0;
}
} else {
/*
* Endpoints are being re-fused, so options will not
* be sent up. In case of STREAMS, free the stroptions
* mblk.
*/
if (!IPCL_IS_NONSTR(connp))
}
} else {
}
return;
}
}
}
/*
* Unfuse a previously-fused pair of tcp loopback endpoints.
*/
void
{
/*
* Cancel any pending push timers.
*/
if (tcp->tcp_push_tid != 0) {
tcp->tcp_push_tid = 0;
}
if (peer_tcp->tcp_push_tid != 0) {
peer_tcp->tcp_push_tid = 0;
}
/*
* Drain any pending data; Note that in case of a detached tcp, the
* draining will happen later after the tcp is unfused. For non-
* urgent data, this can be handled by the regular tcp_rcv_drain().
* If we have urgent data sitting in the receive list, we will
* need to send up a SIGURG signal first before draining the data.
* All of these will be handled by the code in tcp_fuse_rcv_drain()
* when called from tcp_rcv_drain().
*/
if (!TCP_IS_DETACHED(tcp)) {
}
if (!TCP_IS_DETACHED(peer_tcp)) {
}
/* Lift up any flow-control conditions */
if (tcp->tcp_flow_stopped) {
}
if (peer_tcp->tcp_flow_stopped) {
}
/*
* Update th_seq and th_ack in the header template
*/
/* Unfuse the endpoints */
}
/*
* Fusion output routine used to handle urgent data sent by STREAMS based
* endpoints. This routine is called by tcp_fuse_output() for handling
* non-M_DATA mblks.
*/
void
{
struct T_exdata_ind *tei;
/*
* Urgent data arrives in the form of T_EXDATA_REQ from above.
* Each occurence denotes a new urgent pointer. For each new
* urgent pointer we signal (SIGURG) the receiving app to indicate
* that it needs to go into urgent mode. This is similar to the
* urgent data handling in the regular tcp. We don't need to keep
* track of where the urgent pointer is, because each T_EXDATA_REQ
* "advances" the urgent pointer for us.
*
* The actual urgent data carried by T_EXDATA_REQ is then prepended
* by a T_EXDATA_IND before being enqueued behind any existing data
* destined for the receiving app. There is only a single urgent
* pointer (out-of-band mark) for a given tcp. If the new urgent
* data arrives before the receiving app reads some existing urgent
* data, the previous marker is lost. This behavior is emulated
* accordingly below, by removing any existing T_EXDATA_IND messages
* and essentially converting old urgent data into non-urgent.
*/
/* Let sender get out of urgent mode */
/*
* This flag indicates that a signal needs to be sent up.
* This flag will only get cleared once SIGURG is delivered and
* is not affected by the tcp_fused flag -- delivery will still
* happen even after an endpoint is unfused, to handle the case
* sending urgent data and the accept is not yet finished.
*/
/* Reuse T_EXDATA_REQ mblk for T_EXDATA_IND */
/*
* Remove existing T_EXDATA_IND, keep the data which follows
* it and relink our list. Note that we don't modify the
* tcp_rcv_last_tail since it never points to T_EXDATA_IND.
*/
}
}
}
/*
* Fusion output routine, called by tcp_output() and tcp_wput_proto().
* If we are modifying any member that can be changed outside the squeue,
* like tcp_flow_stopped, we need to take tcp_non_sq_lock.
*/
{
/* If this connection requires IP, unfuse and use regular path */
sizeof (tcph_t));
sizeof (tcph_t));
sizeof (ipha_t));
sizeof (ipha_t));
} else {
sizeof (ip6_t));
sizeof (ip6_t));
}
goto unfuse;
}
if (send_size == 0) {
return (B_TRUE);
}
/*
* Handle urgent data; we either send up SIGURG to the peer now
* or do it later when we drain, in case the peer is detached
* or if we're short of memory for M_PCSIG mblk.
*/
if (urgent) {
}
/*
* Build ip and tcp header to satisfy FW_HOOKS.
* We only build it when any hook is present.
*/
/* If tcp_xmit_mp fails, use regular path */
goto unfuse;
/*
* The ipif and ill can be safely referenced under the
* protection of conn_lock - see head of function comment for
* conn_get_held_ipif(). It is necessary to check that both
* the ipif and ill can be looked up (i.e. not condemned). If
* not, bail out and unfuse this connection.
*/
== NULL) ||
(!IPIF_CAN_LOOKUP(oifp)) ||
(ill_check_and_refhold(olp) != 0)) {
goto unfuse;
}
/* PFHooks: LOOPBACK_OUT */
} else {
}
goto unfuse;
/*
* The ipif and ill can be safely referenced under the
* protection of conn_lock - see head of function comment for
* conn_get_held_ipif(). It is necessary to check that both
* the ipif and ill can be looked up (i.e. not condemned). If
* not, bail out and unfuse this connection.
*/
== NULL) ||
(!IPIF_CAN_LOOKUP(iifp)) ||
(ill_check_and_refhold(ilp) != 0)) {
goto unfuse;
}
/* PFHooks: LOOPBACK_IN */
goto unfuse;
} else {
goto unfuse;
}
/* Data length might be changed by FW_HOOKS */
/*
* The message duplicated by tcp_xmit_mp is freed.
* Note: the original message passed in remains unchanged.
*/
}
/*
* Enqueue data into the peer's receive list; we may or may not
* drain the contents depending on the conditions below.
*
* For non-STREAMS sockets we normally queue data directly in the
* socket by calling the su_recv upcall. However, if the peer is
* detached we use tcp_rcv_enqueue() instead. Queued data will be
* drained when the accept completes (in tcp_accept_finish()).
*/
!TCP_IS_DETACHED(peer_tcp)) {
int error;
int flags = 0;
}
}
} else {
/*
* Can not deal with urgent pointers
* that arrive before the connection has been
* accept()ed.
*/
return (B_TRUE);
}
/* In case it wrapped around and also to keep it constant */
}
/*
* Exercise flow-control when needed; we will get back-enabled
* in either tcp_accept_finish(), tcp_unfuse(), or when data is
* consumed. If peer endpoint is detached, we emulate streams flow
* control by checking the peer's queue size and high water mark;
* otherwise we simply use canputnext() to decide if we need to stop
* our flow.
*
* Since we are accessing our tcp_flow_stopped and might modify it,
* we need to take tcp->tcp_non_sq_lock.
*/
if ((TCP_IS_DETACHED(peer_tcp) &&
(!TCP_IS_DETACHED(peer_tcp) &&
}
if (!flow_stopped && (peer_data_queued ||
} else if (flow_stopped && !peer_data_queued &&
}
/* Need to adjust the following SNMP MIB-related variables */
!TCP_IS_DETACHED(peer_tcp)) {
/*
* Drain the peer's receive queue it has urgent data or if
* we're not flow-controlled.
*/
if (urgent || !flow_stopped) {
/*
* For TLI-based streams, a thread in tcp_accept_swap()
* can race with us. That thread will ensure that the
* correct peer_tcp->tcp_rq is globally visible before
* peer_tcp->tcp_detached is visible as clear, but we
* must also ensure that the load of tcp_rq cannot be
* reordered to be before the tcp_detached check.
*/
NULL);
}
}
return (B_TRUE);
return (B_FALSE);
}
/*
* This routine gets called to deliver data upstream on a fused or
* previously fused tcp loopback endpoint; the latter happens only
* when there is a pending SIGURG signal plus urgent data that can't
* be sent upstream in the past.
*/
{
#ifdef DEBUG
#endif
/* No need for the push timer now, in case it was scheduled */
if (tcp->tcp_push_tid != 0) {
tcp->tcp_push_tid = 0;
}
/*
* If there's urgent data sitting in receive list and we didn't
* get a chance to send up a SIGURG signal, make sure we send
* it first before draining in order to ensure that SIOCATMARK
* works properly.
*/
if (tcp->tcp_fused_sigurg) {
/*
* sigurg_mpp is normally NULL, i.e. when we're still
* fused and didn't get here because of tcp_unfuse().
* In this case try hard to allocate the M_PCSIG mblk.
*/
if (sigurg_mpp == NULL &&
/* Alloc failed; try again next time */
return (B_TRUE);
} else if (sigurg_mpp != NULL) {
/*
* Use the supplied M_PCSIG mblk; it means we're
* either unfused or in the process of unfusing,
* and the drain must happen now.
*/
mp = *sigurg_mpp;
*sigurg_mpp = NULL;
}
/* Send up the signal */
/*
* Let the regular tcp_rcv_drain() path handle
* draining the data if we're no longer fused.
*/
return (B_FALSE);
}
/* Drain the data */
#ifdef DEBUG
#endif
}
#ifdef DEBUG
#endif
tcp->tcp_rcv_cnt = 0;
peer_tcp->tcp_xmit_lowater)) {
}
return (B_TRUE);
}
/*
* Calculate the size of receive buffer for a fused tcp endpoint.
*/
{
/* Ensure that value is within the maximum upper bound */
/*
* Round up to system page size in case SO_RCVBUF is modified
* after SO_SNDBUF; the latter is also similarly rounded up.
*/
/*
* Record high water mark, this is used for flow-control
* purposes in tcp_fuse_output().
*/
return (rwnd);
}
/*
* Calculate the maximum outstanding unread data block for a fused tcp endpoint.
*/
int
{
/*
* In the fused loopback case, we want the stream head to split
* up larger writes into smaller chunks for a more accurate flow-
* control accounting. Our maxpsz is half of the sender's send
* buffer or the receiver's receive buffer, whichever is smaller.
* We round up the buffer to system page size due to the lack of
* TCP MSS concept in Fusion.
*/
return (maxpsz);
}
/*
* Called to release flow control.
*/
void
{
if (peer_tcp->tcp_flow_stopped &&
(TCP_UNSENT_BYTES(peer_tcp) <=
peer_tcp->tcp_xmit_lowater)) {
}
}