/*
* CDDL HEADER START
*
* The contents of this file are subject to the terms of the
* Common Development and Distribution License (the "License").
* You may not use this file except in compliance with the License.
*
* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
* See the License for the specific language governing permissions
* and limitations under the License.
*
* When distributing Covered Code, include this CDDL HEADER in each
* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
* If applicable, add the following below this CDDL HEADER, with the
* fields enclosed by brackets "[]" replaced with your own identifying
* information: Portions Copyright [yyyy] [name of copyright owner]
*
* CDDL HEADER END
*/
/*
*/
/*
* Copyright 2012 Joyent, Inc. All rights reserved.
*/
/*
* Squeues: General purpose serialization mechanism
* ------------------------------------------------
*
* Background:
* -----------
*
* This is a general purpose high-performance serialization mechanism
* a worker thread and a polling thread with are bound to the CPU
* associated with the squeue. The squeue is strictly FIFO for both read
* and write side and only one thread can process it at any given time.
* The design goal of squeue was to offer a very high degree of
* parallelization (on a per H/W execution pipeline basis) with at
* most one queuing.
*
* The modules needing protection typically calls SQUEUE_ENTER_ONE() or
* SQUEUE_ENTER() macro as soon as a thread enter the module
* from either direction. For each packet, the processing function
* and argument is stored in the mblk itself. When the packet is ready
* to be processed, the squeue retrieves the stored function and calls
* it with the supplied argument and the pointer to the packet itself.
* The called function can assume that no other thread is processing
* the squeue when it is executing.
*
* Squeue/connection binding:
* --------------------------
*
* connections are assigned to specific squeue (based on various policies),
* at the connection creation time. Once assigned, the connection to
* squeue mapping is never changed and all future packets for that
* connection are processed on that squeue. The connection ("conn") to
* squeue mapping is stored in "conn_t" member "conn_sqp".
*
* Since the processing of the connection cuts across multiple layers
* but still allows packets for different connnection to be processed on
* "Per Connection Vertical Perimeter".
*
* Processing Model:
* -----------------
*
* Squeue doesn't necessary processes packets with its own worker thread.
* The callers can pick if they just want to queue the packet, process
* their packet if nothing is queued or drain and process. The first two
* modes are typically employed when the packet was generated while
* already doing the processing behind the squeue and last mode (drain
* and process) is typically employed when the thread is entering squeue
* for the first time. The squeue still imposes a finite time limit
* for which a external thread can do processing after which it switches
* processing to its own worker thread.
*
* Once created, squeues are never deleted. Hence squeue pointers are
* always valid. This means that functions outside the squeue can still
* refer safely to conn_sqp and their is no need for ref counts.
*
* Only a thread executing in the squeue can change the squeue of the
* connection. It does so by calling a squeue framework function to do this.
* After changing the squeue, the thread must leave the squeue. It must not
* continue to execute any code that needs squeue protection.
*
* The squeue framework, after entering the squeue, checks if the current
* squeue matches the conn_sqp. If the check fails, the packet is delivered
* to right squeue.
*
* Polling Model:
* --------------
*
* Squeues can control the rate of packet arrival into itself from the
* NIC or specific Rx ring within a NIC. As part of capability negotiation
* between IP and MAC layer, squeue are created for each TCP soft ring
* (or TCP Rx ring - to be implemented in future). As part of this
* negotiation, squeues get a cookie for underlying soft ring or Rx
* ring, a function to turn off incoming packets and a function to call
* to poll for packets. This helps schedule the receive side packet
* processing so that queue backlog doesn't build up and packet processing
* doesn't keep getting disturbed by high priority interrupts. As part
* of this mode, as soon as a backlog starts building, squeue turns off
* the interrupts and switches to poll mode. In poll mode, when poll
* thread goes down to retrieve packets, it retrieves them in the form of
* system gets more packets, it gets more efficient by switching to
* polling more often and dealing with larger packet chains.
*
*/
#include <sys/condvar_impl.h>
#include <sys/archsystm.h>
#include <inet/ipclassifier.h>
#include <inet/udp_impl.h>
#include <sys/squeue_impl.h>
static void squeue_fire(void *);
int squeue_workerwait_ms = 0;
/* The values above converted to ticks or nano seconds */
static int squeue_drain_ns = 0;
static int squeue_workerwait_tick = 0;
/* \
* Enqueue our mblk chain. \
*/ \
\
else \
\
}
/*
* Blank the receive ring (in this case it is the soft ring). When
* blanked, the soft ring will not send any more packets up.
* Blanking may not succeed when there is a CPU already in the soft
* ring sending packets up. In that case, SQS_POLLING will not be
* set.
*/
if (sq_poll_capable) { \
} \
} \
}
if (sq_poll_capable) { \
} \
} \
}
/* Wakeup poll thread only if SQS_POLLING is set */
} \
} \
}
#ifdef DEBUG
}
}
#else
#endif
void
squeue_init(void)
{
}
/* ARGSUSED */
squeue_t *
{
return (sqp);
}
/*
* Bind squeue worker thread to the specified CPU, given by CPU id.
* If the CPU id value is -1, bind the worker thread to the value
* specified in sq_bind field. If a thread is already bound to a
* different CPU, unbind it from the old CPU and bind to the new one.
*/
void
{
return;
}
} else {
}
if (bind != PBIND_NONE)
}
void
{
return;
}
}
void
{
return;
}
/*
* Queue isn't being processed, so take
* any post enqueue actions needed before leaving.
*/
if (tid != 0) {
/*
* Waiting for an enter() to process mblk(s).
*/
/*
* Times up and have a worker thread
* waiting for work, so schedule it.
*/
return;
}
return;
return;
} else {
/*
* Wait up to sqp->sq_wait ms for an
* enter() to process this queue. We
* don't want to contend on timeout locks
* with sq_lock held for performance reasons,
* so drop the sq_lock before calling timeout
* but we need to check if timeout is required
* after re acquiring the sq_lock. Once
* the sq_lock is dropped, someone else could
* have processed the packet or the timeout could
* have already fired.
*/
/* Check again if we still need the timeout */
return;
} else {
} else {
/*
* The timer fired before we could
* reacquire the sq_lock. squeue_fire
* removes the SQS_TMO_PROG flag
* and we don't need to do anything
* else.
*/
}
}
}
}
/*
* squeue_enter() - enter squeue sqp with mblk mp (which can be
* a chain), while tail points to the end and cnt in number of
* mblks in the chain.
*
* For a chain of single packet (i.e. mp == tail), go through the
* fast path if no one is processing the squeue and nothing is queued.
*
* The proc and arg for each mblk is already stored in the mblk in
* appropriate places.
*
* The process_flag specifies if we are allowed to process the mblk
* and drain in the entering thread context. If process_flag is
* SQ_FILL, then we just queue the mblk and return (after signaling
* the worker thread if no one else is processing the squeue).
*
* The ira argument can be used when the count is one.
* For a chain the caller needs to prepend any needed mblks from
* ip_recv_attr_to_mblk().
*/
/* ARGSUSED */
void
{
/*
* Try to process the packet if SQ_FILL flag is not set and
* we are allowed to process the squeue. The SQ_NODRAIN is
* ignored if the packet chain consists of more than 1 packet.
*/
/*
* See if anything is already queued. If we are the
* first packet, do inline processing else queue the
* packet and do the drain.
*/
/*
* Fast-path, ok to process and nothing queued.
*/
/*
* We are the chain of 1 packet so
* go through this fast path.
*/
/*
* Handle squeue switching. More details in the
* block comment at the top of the file
*/
tag);
} else {
}
process_flag == SQ_NODRAIN) {
return;
}
/*
* We processed inline our packet and nothing
* new has arrived. We are done. In case any
* control actions are pending, wake up the
* worker.
*/
return;
}
} else {
ip_drop_input("squeue: "
"ip_recv_attr_to_mblk",
return;
}
/* Move connp and func to new */
}
#ifdef DEBUG
#endif
}
/*
* We are here because either we couldn't do inline
* processing (because something was already queued),
* or we had a chain of more than one packet,
* or something else arrived after we were done with
* inline processing.
*/
/*
* If we didn't do a complete drain, the worker
* thread was already signalled by squeue_drain.
* In case any control actions are pending, wake
* up the worker.
*/
return;
} else {
/*
* We let a thread processing a squeue reenter only
* once. This helps the case of incoming connection
* where a SYN-ACK-ACK that triggers the conn_ind
* doesn't have to queue the packet if listener and
* eager are on the same squeue. Also helps the
* loopback connection where the two ends are bound
* to the same squeue (which is typical on single
* CPU machines).
*
* We let the thread reenter only once for the fear
* of stack getting blown with multiple traversal.
*/
/*
* Handle squeue switching. More details in the
* block comment at the top of the file
*/
} else {
}
return;
}
/*
* Queue is already being processed or there is already
* one or more paquets on the queue. Enqueue the
* packet and wakeup the squeue worker thread if the
* squeue is not being processed.
*/
#ifdef DEBUG
#endif
ip_drop_input("squeue: ip_recv_attr_to_mblk",
return;
}
/* Move connp and func to new */
}
return;
}
/*
* In case any control actions are pending, wake
* up the worker.
*/
return;
}
}
/*
* PRIVATE FUNCTIONS
*/
static void
{
return;
}
/*
* The timeout fired before we got a chance to set it.
* Process it anyway but remove the SQS_TMO_PROG so that
* the guy trying to set the timeout knows that it has
* already been processed.
*/
if (state & SQS_TMO_PROG)
}
}
static void
{
/*
* Before doing any work, check our stack depth; if we're not a
* worker thread for this squeue and we're beginning to get tight on
* on stack, kick the worker, bump a counter and return.
*/
return;
}
/*
* We have backlog built up. Switch to polling mode if the
* device underneath allows it. Need to do it so that
* more packets don't come in and disturb us (by contending
* for sq_lock or higher priority thread preempting us).
*
* The worker thread is allowed to do active polling while we
* just disable the interrupts for drain by non worker (kernel
* or userland) threads so they can peacefully process the
* packets during time allocated to them.
*/
if (tid != 0)
/* Is there an ip_recv_attr_t to handle? */
if (ip_recv_attr_is_mblk(mp)) {
/* The ill or ip_stack_t disappeared on us */
ip_drop_input("ip_recv_attr_from_mblk",
continue;
}
} else {
}
/*
* Handle squeue switching. More details in the
* block comment at the top of the file
*/
} else {
}
}
/*
* Check if there is still work to do (either more arrived or timer
* expired). If we are the worker thread and we are polling capable,
* continue doing the work since no one else is around to do the
* work anyway (but signal the poll thread to retrieve some packets
* in the meanwhile). If we are not the worker thread, just
* signal the worker thread to take up the work if processing time
* has expired.
*/
/*
* Still more to process. If time quanta not expired, we
* should let the drain go on. The worker thread is allowed
* to drain as long as there is anything left.
*/
/*
* If time not expired or we are worker thread and
* this squeue is polling capable, continue to do
* the drain.
*
* We turn off interrupts for all userland threads
* doing drain but we do active polling only for
* worker thread.
*
* Calling SQS_POLL_RING() even in the case of
* SQS_POLLING_ON() not succeeding is ok as
* SQS_POLL_RING() will not wake up poll thread
* if SQS_POLLING bit is not set.
*/
if (proc_type == SQS_WORKER)
goto again;
} else {
did_wakeup = B_TRUE;
}
}
/*
* If the poll thread is already running, just return. The
* poll thread continues to hold the proc and will finish
* processing.
*/
return;
}
/*
*
* If we are the worker thread and no work is left, send the poll
* thread down once more to see if something arrived. Otherwise,
* turn the interrupts back on and we are done.
*/
/*
* Do one last check to see if anything arrived
* in the NIC. We leave the SQS_PROC set to ensure
* that poll thread keeps the PROC and can decide
* if it needs to turn polling off or continue
* processing.
*
* If we drop the SQS_PROC here and poll thread comes
* up empty handed, it can not safely turn polling off
* since someone else could have acquired the PROC
* and started draining. The previously running poll
* thread and the current thread doing drain would end
* complex code would be required to deal with it.
*
* Its lot simpler for drain to hand the SQS_PROC to
* poll thread (if running) and let poll thread finish
* without worrying about racing with any other thread.
*/
} else {
/*
* The squeue is either not capable of polling or the
* attempt to blank (i.e., turn SQS_POLLING_ON()) was
* unsuccessful or poll thread already finished
* processing and didn't find anything. Since there
* is nothing queued and we already turn polling on
* (for all threads doing drain), we should turn
* polling off and relinquish the PROC.
*/
}
/*
* If we are not the worker and there is a pending quiesce
* event, wake up the worker
*/
if ((proc_type != SQS_WORKER) &&
}
}
/*
* Quiesce, Restart, or Cleanup of the squeue poll thread.
*
* Quiesce and Restart: After an squeue poll thread has been quiesced, it does
* not attempt to poll the underlying soft ring any more. The quiesce is
* triggered by the mac layer when it wants to quiesce a soft ring. Typically
* control operations such as changing the fanout of a NIC or VNIC (dladm
* setlinkprop) need to quiesce data flow before changing the wiring.
* The operation is done by the mac layer, but it calls back into IP to
* quiesce the soft ring. After completing the operation (say increase or
* decrease of the fanout) the mac layer then calls back into IP to restart
* the quiesced soft ring.
*
* Cleanup: This is triggered when the squeue binding to a soft ring is
* removed permanently. Typically interface plumb and unplumb would trigger
* this. It can also be triggered from the mac layer when a soft ring is
* being deleted say as the result of a fanout reduction. Since squeues are
* never deleted, the cleanup marks the squeue as fit for recycling and
* moves it to the zeroth squeue set.
*/
static void
{
/* Restart implies a previous quiesce */
return;
}
return;
}
}
/*
* POLLING Notes
*
* With polling mode, we want to do as much processing as we possibly can
* in worker thread context. The sweet spot is worker thread keeps doing
* work all the time in polling mode and writers etc. keep dumping packets
* to worker thread. Occassionally, we send the poll thread (running at
* lower priority to NIC to get the chain of packets to feed to worker).
* Sending the poll thread down to NIC is dependant on 3 criterions
*
* 1) Its always driven from squeue_drain and only if worker thread is
* doing the drain.
* 2) We clear the backlog once and more packets arrived in between.
* Before starting drain again, send the poll thread down if
* the drain is being done by worker thread.
* 3) Before exiting the squeue_drain, if the poll thread is not already
* working and we are the worker thread, try to poll one more time.
*
* For latency sake, we do allow any thread calling squeue_enter
* to process its packet provided:
*
* 1) Nothing is queued
* 2) If more packets arrived in between, the non worker thread are allowed
* to do the drain till their time quanta expired provided SQS_GET_PKTS
* wasn't set in between.
*
* Avoiding deadlocks with interrupts
* ==================================
*
* One of the big problem is that we can't send poll_thr down while holding
* the sq_lock since the thread can block. So we drop the sq_lock before
* calling sq_get_pkts(). We keep holding the SQS_PROC as long as the
* poll thread is running so that no other thread can acquire the
* perimeter in between. If the squeue_drain gets done (no more work
* left), it leaves the SQS_PROC set if poll thread is running.
*/
/*
* This is the squeue poll thread. In poll mode, it polls the underlying
* TCP softring and feeds packets into the squeue. The worker thread then
* drains the squeue. The poll thread also responds to control signals for
* quiesceing, restarting, or cleanup of an squeue. These are driven by
* related operations that are driven from the mac layer.
*/
static void
{
void *sq_mac_handle;
for (;;) {
if (ctl_state != 0) {
/*
* If the squeue is quiesced, then wait for a control
* request. A quiesced squeue must not poll the
* underlying soft ring.
*/
if (ctl_state == SQS_POLL_THR_QUIESCED)
continue;
/*
* Act on control requests to quiesce, cleanup or
* restart an squeue
*/
continue;
}
continue;
/*
* We got the packet chain from the mac layer. It
* would be nice to be able to process it inline
* for better performance but we need to give
* IP a chance to look at this chain to ensure
* that packets are really meant for this squeue
* and do the IP processing.
*/
}
/*
* The ip_accept function has already added an
* ip_recv_attr_t mblk if that is needed.
*/
}
/*
* We have packets to process and worker thread
* is not running. Check to see if poll thread is
* allowed to process. Let it do processing only if it
* picked up some packets from the NIC otherwise
* wakeup the worker thread.
*/
goto poll_again;
/*
* Couldn't do the entire drain because the
* time limit expired, let the
* worker thread take over.
*/
}
/*
* Put the SQS_PROC_HELD on so the worker
* thread can distinguish where its called from. We
* can remove the SQS_PROC flag here and turn off the
* polling so that it wouldn't matter who gets the
* processing but we get better performance this way
* and save the cost of turn polling off and possibly
* on again as soon as we start draining again.
*
* We can't remove the SQS_PROC flag without turning
* polling off until we can guarantee that control
* will return to squeue_drain immediately.
*/
/*
* Nothing queued and worker thread not running.
* Since we hold the proc, no other thread is
* processing the squeue. This means that there
* is no work to be done and nothing is queued
* in squeue or in NIC. Turn polling off and go
* back to interrupt mode.
*/
/* LINTED: constant in conditional context */
/*
* If there is a pending control operation
* wake up the worker, since it is currently
* not running.
*/
} else {
/*
* Worker thread is already running. We don't need
* to do anything. Indicate that poll thread is done.
*/
}
/*
* Act on control requests to quiesce, cleanup or
* restart an squeue
*/
}
}
}
/*
* The squeue worker thread acts on any control requests to quiesce, cleanup
* or restart an ill_rx_ring_t by calling this function. The worker thread
* synchronizes with the squeue poll thread to complete the request and finally
* wakes up the requestor when the request is completed.
*/
static void
{
/* Restart implies a previous quiesce. */
/*
* Request the squeue poll thread to restart and wait till
* it actually restarts.
*/
/*
* Signal any waiter that is waiting for the restart
* to complete
*/
return;
}
/* The squeue poll thread handed control to us */
}
/*
* Prevent any other thread from processing the squeue
* until we finish the control actions by setting SQS_PROC.
* But allow ourself to reenter by setting SQS_WORKER
*/
/* Signal the squeue poll thread and wait for it to quiesce itself */
}
/*
* The lock hierarchy is as follows.
* cpu_lock -> ill_lock -> sqset_lock -> sq_lock
*/
sqp->sq_rx_ring);
/*
* Disassociate this squeue from its ill_rx_ring_t.
* The rr_sqp, sq_rx_ring fields are protected by the
* corresponding squeue, ill_lock* and sq_lock. Holding any
* of them will ensure that the ring to squeue mapping does
* not change.
*/
} else {
}
/*
* Signal any waiter that is waiting for the quiesce or cleanup
* to complete and also wait for it to actually see and reset the
* SQS_POLL_CLEANUP_DONE.
*/
}
}
static void
{
for (;;) {
for (;;) {
/*
* If the poll thread has handed control to us
* we need to break out of the wait.
*/
break;
/*
* If the squeue is not being processed and we either
* have messages to drain or some thread has signaled
* some control activity we need to break
*/
break;
/*
* If we have started some control action, then check
* for the SQS_WORKER flag (since we don't
* release the squeue) to make sure we own the squeue
* and break out
*/
break;
}
continue;
}
}
}
{
ASSERT(p < SQPRIVATE_MAX);
return (&sqp->sq_private[p]);
}
/* ARGSUSED */
void
{
/*
* Mark the squeue as paused before waking up the thread stuck
* in squeue_synch_enter().
*/
/*
* Notify the thread that it's OK to proceed; that is done by
* clearing the MSGWAITSYNC flag. The synch thread will free the mblk.
*/
/*
* We are doing something on behalf of another thread, so we have to
* pause and wait until it finishes.
*/
}
}
int
{
/*
* We are OK to proceed if the squeue is empty, and
* no one owns the squeue.
*
* The caller won't own the squeue as this is called from the
* application.
*/
/*
* Handle squeue switching. The conn's squeue can only change
* while there is a thread in the squeue, which is why we do
* the check after entering the squeue. If it has changed, exit
* this squeue and redo everything with the new sqeueue.
*/
goto again;
}
#if SQUEUE_DEBUG
#endif
return (0);
} else {
return (ENOMEM);
}
/*
* We mark the mblk as awaiting synchronous squeue access
* by setting the MSGWAITSYNC flag. Once squeue_wakeup_conn
* fires, MSGWAITSYNC is cleared, at which point we know we
* have exclusive access.
*/
/* Wait until the enqueued mblk get processed. */
return (0);
}
}
void
{
} else {
/*
* If this was a normal thread, then it would
* (most likely) continue processing the pending
* requests. Since the just completed operation
* was executed synchronously, the thread should
* not be delayed. To compensate, wake up the
* worker thread right away when there are outstanding
* requests.
*/
}
} else {
/*
* The caller doesn't own the squeue, clear the SQS_PAUSE flag,
* and wake up the squeue owner, such that owner can continue
* processing.
*/
/* There should be only one thread blocking on sq_synch_cv. */
}
}