mac_sched.c revision d3d50737e566cade9a08d73d2af95105ac7cd960
/*
* CDDL HEADER START
*
* The contents of this file are subject to the terms of the
* Common Development and Distribution License (the "License").
* You may not use this file except in compliance with the License.
*
* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
* See the License for the specific language governing permissions
* and limitations under the License.
*
* When distributing Covered Code, include this CDDL HEADER in each
* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
* If applicable, add the following below this CDDL HEADER, with the
* fields enclosed by brackets "[]" replaced with your own identifying
* information: Portions Copyright [yyyy] [name of copyright owner]
*
* CDDL HEADER END
*/
/*
* Copyright 2009 Sun Microsystems, Inc. All rights reserved.
* Use is subject to license terms.
*/
#include <inet/ipsec_impl.h>
#include <inet/ipsecesp.h>
#include <sys/mac_impl.h>
#include <sys/mac_client_impl.h>
#include <sys/mac_client_priv.h>
#include <sys/mac_soft_ring.h>
#include <sys/mac_flow_impl.h>
typedef struct mac_tx_mode_s {
/*
* There are five modes of operation on the Tx side. These modes get set
* in mac_tx_srs_setup(). Except for the experimental TX_SERIALIZE mode,
* none of the other modes are user configurable. They get selected by
* the system depending upon whether the link (or flow) has multiple Tx
* rings or a bandwidth configured, etc.
*/
};
/*
* Soft Ring Set (SRS) - The Run time code that deals with
* dynamic polling from the hardware, bandwidth enforcement,
* fanout etc.
*
* We try to use H/W classification on NIC and assign traffic for
* a MAC address to a particular Rx ring or ring group. There is a
* 1-1 mapping between a SRS and a Rx ring. The SRS dynamically
* switches the underlying Rx ring between interrupt and
* polling mode and enforces any specified B/W control.
*
* There is always a SRS created and tied to each H/W and S/W rule.
* Whenever we create a H/W rule, we always add the the same rule to
* S/W classifier and tie a SRS to it.
*
* In case a B/W control is specified, it is broken into bytes
* per ticks and as soon as the quota for a tick is exhausted,
* the underlying Rx ring is forced into poll mode for remainder of
* the tick. The SRS poll thread only polls for bytes that are
* allowed to come in the SRS. We typically let 4x the configured
* B/W worth of packets to come in the SRS (to prevent unnecessary
* drops due to bursts) but only process the specified amount.
*
* A MAC client (e.g. a VNIC or aggr) can have 1 or more
* Rx rings (and corresponding SRSs) assigned to it. The SRS
* in turn can have softrings to do protocol level fanout or
* softrings to do S/W based fanout or both. In case the NIC
* has no Rx rings, we do S/W classification to respective SRS.
* The S/W classification rule is always setup and ready. This
* allows the MAC layer to reassign Rx rings whenever needed
* but packets still continue to flow via the default path and
* getting S/W classified to correct SRS.
*
* The SRS's are used on both Tx and Rx side. They use the same
* data structure but the processing routines have slightly different
* semantics due to the fact that Rx side needs to do dynamic
* polling etc.
*
* Dynamic Polling Notes
* =====================
*
* Each Soft ring set is capable of switching its Rx ring between
* interrupt and poll mode and actively 'polls' for packets in
* poll mode. If the SRS is implementing a B/W limit, it makes
* sure that only Max allowed packets are pulled in poll mode
* and goes to poll mode as soon as B/W limit is exceeded. As
* such, there are no overheads to implement B/W limits.
*
* In poll mode, its better to keep the pipeline going where the
* SRS worker thread keeps processing packets and poll thread
* keeps bringing more packets (specially if they get to run
* on different CPUs). This also prevents the overheads associated
* by excessive signalling (on NUMA machines, this can be
* pretty devastating). The exception is latency optimized case
* where worker thread does no work and interrupt and poll thread
* are allowed to do their own drain.
*
* We use the following policy to control Dynamic Polling:
* 1) We switch to poll mode anytime the processing
* thread causes a backlog to build up in SRS and
* its associated Soft Rings (sr_poll_pkt_cnt > 0).
* 2) As long as the backlog stays under the low water
* mark (sr_lowat), we poll the H/W for more packets.
* 3) If the backlog (sr_poll_pkt_cnt) exceeds low
* water mark, we stay in poll mode but don't poll
* the H/W for more packets.
* 4) Anytime in polling mode, if we poll the H/W for
* packets and find nothing plus we have an existing
* backlog (sr_poll_pkt_cnt > 0), we stay in polling
* mode but don't poll the H/W for packets anymore
* (let the polling thread go to sleep).
* 5) Once the backlog is relived (packets are processed)
* we reenable polling (by signalling the poll thread)
* only when the backlog dips below sr_poll_thres.
* 6) sr_hiwat is used exclusively when we are not
* polling capable and is used to decide when to
* drop packets so the SRS queue length doesn't grow
* infinitely.
*
* NOTE: Also see the block level comment on top of mac_soft_ring.c
*/
/*
* mac_latency_optimize
*
* Controls whether the poll thread can process the packets inline
* or let the SRS worker thread do the processing. This applies if
* the SRS was not being processed. For latency sensitive traffic,
* this needs to be true to allow inline processing. For throughput
* under load, this should be false.
*
* This (and other similar) tunable should be rolled into a link
* or flow specific workload hint that can be set using dladm
* linkprop (instead of multiple such tunables).
*/
/*
* MAC_RX_SRS_ENQUEUE_CHAIN and MAC_TX_SRS_ENQUEUE_CHAIN
*
* queue a mp or chain in soft ring set and increment the
* local count (srs_count) for the SRS and the shared counter
* (srs_poll_pkt_cnt - shared between SRS and its soft rings
* to track the total unprocessed packets for polling to work
* correctly).
*
* The size (total bytes queued) counters are incremented only
* if we are doing B/W control.
*/
else \
}
\
} \
}
} \
}
/*
* Turn polling on routines
*/
#define MAC_SRS_POLLING_ON(mac_srs) { \
(void) mac_hwring_disable_intr((mac_ring_handle_t) \
} \
}
#define MAC_SRS_WORKER_POLLING_ON(mac_srs) { \
(SRS_POLLING_CAPAB|SRS_WORKER)) { \
(void) mac_hwring_disable_intr((mac_ring_handle_t) \
} \
}
/*
* MAC_SRS_POLL_RING
*
* Signal the SRS poll thread to poll the underlying H/W ring
* provided it wasn't already polling (SRS_GET_PKTS was set).
*
* Poll thread gets to run only from mac_rx_srs_drain() and only
* if the drain was being done by the worker thread.
*/
#define MAC_SRS_POLL_RING(mac_srs) { \
\
srs_rx->sr_poll_thr_sig++; \
(SRS_WORKER|SRS_POLLING_CAPAB)) { \
} else { \
srs_rx->sr_poll_thr_busy++; \
} \
}
/*
* MAC_SRS_CHECK_BW_CONTROL
*
* Check to see if next tick has started so we can reset the
* SRS_BW_ENFORCED flag and allow more packets to come in the
* system.
*/
#define MAC_SRS_CHECK_BW_CONTROL(mac_srs) { \
} \
}
/*
* MAC_SRS_WORKER_WAKEUP
*
* Wake up the SRS worker thread to process the queue as long as
* no one else is processing the queue. If we are optimizing for
* latency, we wake up the worker thread immediately or else we
* wait mac_srs_worker_wakeup_ticks before worker thread gets
* woken up.
*/
int mac_srs_worker_wakeup_ticks = 0;
#define MAC_SRS_WORKER_WAKEUP(mac_srs) { \
(mac_srs_worker_wakeup_ticks == 0)) \
else \
} \
}
#define TX_SINGLE_RING_MODE(mac_srs) \
#define TX_BANDWIDTH_MODE(mac_srs) \
}
/*
* MAC_TX_SRS_BLOCK
*
* Always called from mac_tx_srs_drain() function. SRS_TX_BLOCKED
* will be set only if srs_tx_woken_up is FALSE. If
* srs_tx_woken_up is TRUE, it indicates that the wakeup arrived
* before we grabbed srs_lock to set SRS_TX_BLOCKED. We need to
* attempt to transmit again and not setting SRS_TX_BLOCKED does
* that.
*/
} else { \
} \
}
/*
* MAC_TX_SRS_TEST_HIWAT
*
* Called before queueing a packet onto Tx SRS to test and set
* SRS_TX_HIWAT if srs_count exceeds srs_tx_hiwat.
*/
\
/* \
* flow-controlled. Store srs in cookie so that it \
* can be returned as mac_tx_cookie_t to client \
*/ \
/* increment freed stats */ \
/* \
* b_prev may be set to the fanout hint \
* hence can't use freemsg directly \
*/ \
mac_soft_ring_set_t *, srs); \
enqueue = 0; \
} \
} \
if (enqueue) \
}
/* Some utility macros */
#define MAC_SRS_BW_LOCK(srs) \
#define MAC_SRS_BW_UNLOCK(srs) \
/* increment freed stats */ \
}
}
/*
* Drop the rx packet and advance to the next one in the chain.
*/
static void
{
srs_rx->sr_drop_count++;
}
/* DATAPATH RUNTIME ROUTINES */
/*
* mac_srs_fire
*
* Timer callback routine for waking up the SRS worker thread.
*/
static void
mac_srs_fire(void *arg)
{
return;
}
}
/*
* and it is used on the TX path.
*/
/*
* hash based on the src address and the port information.
*/
} else { \
} \
(cnt)++; \
if ((bw_ctl)) \
}
#define MAC_FANOUT_DEFAULT 0
#define MAC_FANOUT_RND_ROBIN 1
#define MAX_SR_TYPES 3
/* fanout types for port based hashing */
enum pkt_type {
V4_TCP = 0,
OTH,
};
/*
* In general we do port based hashing to spread traffic over different
* softrings. The below tunable allows to override that behavior. Setting it
* to B_TRUE allows to do a fanout based on src ipv6 address. This behavior
* is also the applicable to ipv6 packets carrying multiple optional headers
* and other uncommon packet types.
*/
/*
* Pair of local and remote ports in the transport header
*/
#define PORTS_SIZE 4
/*
* mac_rx_srs_proto_fanout
*
* This routine delivers packets destined to an SRS into one of the
* protocol soft rings.
*
* Given a chain of packets we need to split it up into multiple sub chains
* destined into TCP, UDP or OTH soft ring. Instead of entering
* the soft ring one packet at a time, we want to enter it in the form of a
* goes to sleep and then next packets comes in forcing it to wake up etc.
*/
static void
{
struct ether_header *ehp;
struct ether_vlan_header *evhp;
int cnt[MAX_SR_TYPES];
/*
* If we don't have a Rx ring, S/W classification would have done
* its job and its a packet meant for us. If we were polling on
* the default ring (i.e. there was a ring assigned to this SRS),
* then we need to make sure that the mac address really belongs
* to us.
*/
/*
* Special clients (eg. VLAN, non ether, etc) need DLS
* processing in the Rx path. SRST_DLS_BYPASS will be clear for
* such SRSs. Another way of disabling bypass is to set the
* MCIS_RX_BYPASS_DISABLE flag.
*/
/*
* We got a chain from SRS that we need to send to the soft rings.
* Since squeues for TCP & IPv4 sap poll their soft rings (for
* performance reasons), we need to separate out v4_tcp, v4_udp
* and the rest goes in other.
*/
if (is_ether) {
/*
* At this point we can be sure the packet at least
* has an ether header.
*/
if (sz1 < sizeof (struct ether_header)) {
continue;
}
/*
* Determine if this is a VLAN or non-VLAN packet.
*/
hdrsize = sizeof (struct ether_vlan_header);
/*
* Check if the VID of the packet, if any,
* belongs to this client.
*/
continue;
}
} else {
hdrsize = sizeof (struct ether_header);
}
} else {
continue;
}
}
if (!dls_bypass) {
continue;
}
if (sap == ETHERTYPE_IP) {
/*
* If we are H/W classified, but we have promisc
* on, then we need to check for the unicast address.
*/
} else if (is_unicast) {
}
}
/*
* This needs to become a contract with the driver for
* the fast path.
*
* In the normal case the packet will have at least the L2
* header and the IP + Transport header in the same mblk.
* This is usually the case when the NIC driver sends up
* the packet. This is also true when the stack generates
* a packet that is looped back and when the stack uses the
* fastpath mechanism. The normal case is optimized for
* performance and may bypass DLS. All other cases go through
* the 'OTH' type path without DLS bypass.
*/
continue;
}
/*
* We look for at least 4 bytes past the IP header to get
* the port information. If we get an IP fragment, we don't
* have the port information, and we use just the protocol
* information.
*/
switch (ipha->ipha_protocol) {
case IPPROTO_TCP:
break;
case IPPROTO_UDP:
break;
default:
break;
}
}
switch (type) {
case V4_TCP:
break;
case V4_UDP:
break;
case OTH:
}
}
}
}
int fanout_unalligned = 0;
/*
* mac_rx_srs_long_fanout
*
* The fanout routine for IPv6
*/
static int
{
if (sap == ETHERTYPE_IPV6) {
/*
* The first mblk_t only includes the mac header.
* Note that it is safe to change the mp pointer here,
* as the subsequent operation does not assume mp
* points to the start of the mac header.
*/
/*
* Make sure ip6h holds the full ip6_t structure.
*/
return (-1);
if (modifiable &&
return (-1);
}
}
}
/*
* If either ip6h is not alligned, or ip6h does not
* hold the complete ip6_t structure (a pullupmsg()
* is not an option since it would result in an
* unalligned ip6h), fanout to the default ring. Note
* that this may cause packets reordering.
*/
*indx = 0;
return (0);
}
if (remlen < MIN_EHDR_LEN)
return (-1);
/*
* Do src based fanout if below tunable is set to B_TRUE or
* when mac_ip_hdr_length_v6() fails because of malformed
* packets or because mblk's need to be concatenated using
* pullupmsg().
*/
goto src_based_fanout;
}
/* If the transport is one of below, we do port based fanout */
switch (nexthdr) {
case IPPROTO_TCP:
case IPPROTO_UDP:
case IPPROTO_SCTP:
case IPPROTO_ESP:
/*
* If the ports in the transport header is not part of
* the mblk, do src_based_fanout, instead of calling
* pullupmsg().
*/
goto src_based_fanout;
}
break;
default:
break;
}
switch (nexthdr) {
case IPPROTO_TCP:
break;
case IPPROTO_UDP:
case IPPROTO_SCTP:
case IPPROTO_ESP:
if (mac_fanout_type == MAC_FANOUT_DEFAULT) {
} else {
}
break;
/* For all other protocol, do source based fanout */
default:
goto src_based_fanout;
}
} else {
*indx = 0;
}
return (0);
return (0);
}
/*
* mac_rx_srs_fanout
*
* This routine delivers packets destined to an SRS into a soft ring member
* of the set.
*
* Given a chain of packets we need to split it up into multiple sub chains
* destined for one of the TCP, UDP or OTH soft rings. Instead of entering
* the soft ring one packet at a time, we want to enter it in the form of a
* goes to sleep and then next packets comes in forcing it to wake up etc.
*
* Note:
* Since we know what is the maximum fanout possible, we create a 2D array
* of 'softring types * MAX_SR_FANOUT' for the head, tail, cnt and sz
* variables so that we can enter the softrings with chain. We need the
* MAX_SR_FANOUT so we can allocate the arrays on the stack (a kmem_alloc
* for each packet would be expensive). If we ever want to have the
* ability to have unlimited fanout, we should probably declare a head,
* tail, cnt, sz with each soft ring (a data struct which contains a softring
* along with these members) and create an array of this uber struct so we
* don't have to do kmem_alloc.
*/
int fanout_oth1 = 0;
int fanout_oth2 = 0;
int fanout_oth3 = 0;
int fanout_oth4 = 0;
int fanout_oth5 = 0;
static void
{
struct ether_header *ehp;
struct ether_vlan_header *evhp;
int fanout_cnt;
/*
* If we don't have a Rx ring, S/W classification would have done
* its job and its a packet meant for us. If we were polling on
* the default ring (i.e. there was a ring assigned to this SRS),
* then we need to make sure that the mac address really belongs
* to us.
*/
/*
* Special clients (eg. VLAN, non ether, etc) need DLS
* processing in the Rx path. SRST_DLS_BYPASS will be clear for
* such SRSs. Another way of disabling bypass is to set the
* MCIS_RX_BYPASS_DISABLE flag.
*/
/*
* Since the softrings are never destroyed and we always
* create equal number of softrings for TCP, UDP and rest,
* its OK to check one of them for count and use it without
* any lock. In future, if soft rings get destroyed because
* of reduction in fanout, we will need to ensure that happens
* behind the SRS_PROC.
*/
/*
* We got a chain from SRS that we need to send to the soft rings.
* Since squeues for TCP & IPv4 sap poll their soft rings (for
* performance reasons), we need to separate out v4_tcp, v4_udp
* and the rest goes in other.
*/
if (is_ether) {
/*
* At this point we can be sure the packet at least
* has an ether header.
*/
if (sz1 < sizeof (struct ether_header)) {
continue;
}
/*
* Determine if this is a VLAN or non-VLAN packet.
*/
hdrsize = sizeof (struct ether_vlan_header);
/*
* Check if the VID of the packet, if any,
* belongs to this client.
*/
continue;
}
} else {
hdrsize = sizeof (struct ether_header);
}
} else {
continue;
}
}
if (!dls_bypass) {
continue;
}
continue;
}
/*
* If we are using the default Rx ring where H/W or S/W
* classification has not happened, we need to verify if
* this unicast packet really belongs to us.
*/
if (sap == ETHERTYPE_IP) {
/*
* If we are H/W classified, but we have promisc
* on, then we need to check for the unicast address.
*/
} else if (is_unicast) {
}
}
/*
* This needs to become a contract with the driver for
* the fast path.
*/
fanout_oth1++;
}
switch (ipha->ipha_protocol) {
case IPPROTO_TCP:
case IPPROTO_UDP:
case IPPROTO_SCTP:
case IPPROTO_ESP:
break;
}
if ((frag_offset_flags &
(IPH_MF | IPH_OFFSET)) != 0) {
fanout_oth3++;
break;
}
break;
default:
fanout_oth4++;
break;
}
}
continue;
}
continue;
}
/*
* XXX-Sunay: We should hold srs_lock since ring_count
* below can change. But if we are always called from
* mac_rx_srs_drain and SRS_PROC is set, then we can
* enforce that ring_count can't be changed i.e.
* to change fanout type or ring count, the calling
* thread needs to be behind SRS_PROC.
*/
switch (ipha->ipha_protocol) {
case IPPROTO_TCP:
/*
* Note that for ESP, we fanout on SPI and it is at the
* same offset as the 2x16-bit ports. So it is clumped
* along with TCP, UDP and SCTP.
*/
break;
case IPPROTO_UDP:
case IPPROTO_SCTP:
case IPPROTO_ESP:
if (mac_fanout_type == MAC_FANOUT_DEFAULT) {
} else {
}
break;
default:
indx = 0;
}
}
int i;
for (i = 0; i < fanout_cnt; i++) {
switch (type) {
case V4_TCP:
softring =
break;
case V4_UDP:
softring =
break;
case OTH:
softring =
break;
}
}
}
}
}
#define SRS_BYTES_TO_PICKUP 150000
/*
* mac_rx_srs_poll_ring
*
* This SRS Poll thread uses this routine to poll the underlying hardware
* Rx ring to get a chain of packets. It can inline process that chain
* if mac_latency_optimize is set (default) or signal the SRS worker thread
* to do the remaining processing.
*
* Since packets come in the system via interrupt or poll path, we also
* update the stats and deal with promiscous clients here.
*/
void
{
int count;
for (;;) {
goto done;
goto done;
/*
* We pick as many bytes as we are allowed to queue.
* Its possible that we will exceed the total
* packets queued in case this SRS is part of the
* Rx ring group since > 1 poll thread can be pulling
* upto the max allowed packets at the same time
* but that should be OK.
*/
/*
* We shouldn't have been signalled if we
* have 0 or less bytes to pick but since
* some of the bytes accounting is driver
* dependant, we do the safety check.
*/
if (bytes_to_pickup < 0)
bytes_to_pickup = 0;
} else {
/*
* ToDO: Need to change the polling API
* to add a packet count and a flag which
* tells the driver whether we want packets
* based on a count, or bytes, or all the
* way, we never have to check the limits
* on poll path. We truly let only as many
* packets enter the system as we are willing
* to process or queue.
*
* Something along the lines of
* pkts_to_pickup = mac_soft_ring_max_q_cnt -
* mac_srs->srs_poll_pkt_cnt
*/
/*
* Since we are not doing B/W control, pick
* as many packets as allowed.
*/
}
/* Poll the underlying Hardware */
count = 0;
sz = 0;
count++;
}
}
/*
* If there are any promiscuous mode callbacks
* defined for this MAC client, pass them a copy
* if appropriate and also update the counters.
*/
}
}
}
if (count <= 10)
else
}
/*
* We are guaranteed that SRS_PROC will be set if we
* are here. Also, poll thread gets to run only if
* the drain was being done by a worker thread although
* its possible that worker thread is still running
* and poll thread was sent down to keep the pipeline
* going instead of doing a complete drain and then
* trying to poll the NIC.
*
* So we need to check SRS_WORKER flag to make sure
* that the worker thread is not processing the queue
* in parallel to us. The flags and conditions are
* protected by the srs_lock to prevent any race. We
* ensure that we don't drop the srs_lock from now
* till the end and similarly we don't drop the srs_lock
* in mac_rx_srs_drain() till similar condition check
* are complete. The mac_rx_srs_drain() needs to ensure
* that SRS_WORKER flag remains set as long as its
* processing the queue.
*/
/*
* We have packets to process and worker thread
* is not running. Check to see if poll thread is
* allowed to process.
*/
srs_rx->sr_poll_again++;
goto check_again;
}
/*
* We are already above low water mark
* so stay in the polling mode but no
* need to poll. Once we dip below
* the polling threshold, the processing
* thread (soft ring) will signal us
* to poll again (MAC_UPDATE_SRS_COUNT)
*/
/*
* In B/W control case, its possible
* that the backlog built up due to
* B/W limit being reached and packets
* are queued only in SRS. In this case,
* we should schedule worker thread
* since no one else will wake us up.
*/
}
} else {
/*
* Wakeup the worker thread for more processing.
* We optimize for throughput in this case.
*/
}
/*
* There is nothing queued in SRS and
* no worker thread running. Plus we
* didn't get anything from the H/W
* as well (head == NULL);
*/
~(SRS_PROC|SRS_GET_PKTS);
/*
* If we have a packets in soft ring, don't allow
* more packets to come into this SRS by keeping the
* interrupts off but not polling the H/W. The
* poll thread will get signaled as soon as
* srs_poll_pkt_cnt dips below poll threshold.
*/
if (srs_rx->sr_poll_pkt_cnt == 0) {
} else {
/*
* We know nothing is queued in SRS
* since we are here after checking
* srs_first is NULL. The backlog
* is entirely due to packets queued
* in Soft ring which will wake us up
* and get the interface out of polling
* mode once the backlog dips below
* sr_poll_thres.
*/
}
} else {
/*
* Worker thread is already running.
* Nothing much to do. If the polling
* was enabled, worker thread will deal
* with that.
*/
}
}
done:
/*
* If this is a temporary quiesce then wait for the restart signal
* from the srs worker. Then clear the flags and signal the srs worker
* to ensure a positive handshake and go back to start.
*/
goto start;
} else {
thread_exit();
}
}
/*
* mac_srs_pick_chain
*
* In Bandwidth control case, checks how many packets can be processed
* and return them in a sub chain.
*/
static mblk_t *
{
int cnt = 0;
return (head);
}
/*
* Can't clear the entire backlog.
* Need to find how many packets to pick
*/
break;
}
/*
* The _size & cnt is decremented from the softrings
* when they send up the packet for polling to work
* properly.
*/
cnt++;
else
}
*chain_tail = tail;
return (head);
}
/*
* mac_rx_srs_drain
*
* The SRS drain routine. Gets to run to clear the queue. Any thread
* (worker, interrupt, poll) can call this based on processing model.
* The first thing we do is disable interrupts if possible and then
* drain the queue. we also try to poll the underlying hardware if
* there is a dedicated hardware Rx ring assigned to this SRS.
*
* There is a equivalent drain routine in bandwidth control mode
* mac_rx_srs_drain_bw. There is some code duplication between the two
* routines but they are highly performance sensitive and are easier
* also apply to mac_rx_srs_drain_bw as well.
*/
void
{
int cnt = 0;
/* If we are blanked i.e. can't do upcalls, then we are done */
goto out;
}
goto out;
/*
* In the normal case, the SRS worker thread does no
* work and we wait for a backlog to build up before
* we switch into polling mode. In case we are
* optimizing for throughput, we use the worker thread
* as well. The goal is to let worker thread process
* the queue and poll thread to feed packets into
* the queue. As such, we should signal the poll
* thread to try and get more packets.
*
* We could have pulled this check in the POLL_RING
* macro itself but keeping it explicit here makes
* the architecture more human understandable.
*/
}
/*
* mcip is NULL for broadcast and multicast flows. The promisc
* callbacks for broadcast and multicast packets are delivered from
* mac_rx() and we don't need to worry about that case in this path
*/
}
/*
* Check if SRS itself is doing the processing
* This direct path does not apply when subflows are present. In this
* case, packets need to be dispatched to a soft ring according to the
* flow's bandwidth and other resources contraints.
*/
void *arg1;
/*
* This is the case when a Rx is directly
* assigned and we have a fully classified
* protocol chain. We can deal with it in
* one shot.
*/
if (tid != 0) {
tid = 0;
}
/*
* Decrement the size and count here itelf
* since the packet has been processed.
*/
} else {
/* Some kind of softrings based fanout is required */
if (tid != 0) {
tid = 0;
}
/*
* Since the fanout routines can deal with chains,
* shoot the entire chain up.
*/
else
}
/*
* More packets arrived while we were clearing the
* SRS. This can be possible because of one of
* three conditions below:
* 1) The driver is using multiple worker threads
* to send the packets to us.
* 2) The driver has a race in switching
* between interrupt and polling mode or
* 3) Packets are arriving in this SRS via the
* S/W classification as well.
*
* We should switch to polling mode and see if we
* need to send the poll thread down. Also, signal
* the worker thread to process whats just arrived.
*/
}
/*
* If we didn't signal the poll thread, we need
* to deal with the pending packets ourselves.
*/
if (proc_type == SRS_WORKER) {
srs_rx->sr_drain_again++;
goto again;
} else {
}
}
out:
/*
* Poll thread is already running. Leave the
* SRS_RPOC set and hand over the control to
* poll thread.
*/
return;
}
/*
* Even if there are no packets queued in SRS, we
* need to make sure that the shared counter is
* clear and any associated softrings have cleared
* all the backlog. Otherwise, leave the interface
* in polling mode and the poll thread will get
* signalled once the count goes down to zero.
*
* If someone is already draining the queue (SRS_PROC is
* set) when the srs_poll_pkt_cnt goes down to zero,
* then it means that drain is already running and we
* will turn off polling at that time if there is
* no backlog.
*
* As long as there are packets queued either
* in soft ring set or its soft rings, we will leave
* the interface in polling mode (even if the drain
* was done being the interrupt thread). We signal
* the poll thread as well if we have dipped below
* low water mark.
*
* NOTE: We can't use the MAC_SRS_POLLING_ON macro
* since that turn polling on only for worker thread.
* Its not worth turning polling on for interrupt
* thread (since NIC will not issue another interrupt)
* unless a backlog builds up.
*/
if ((srs_rx->sr_poll_pkt_cnt > 0) &&
return;
}
/* Nothing else to do. Get out of poll mode */
}
/*
* mac_rx_srs_drain_bw
*
* The SRS BW drain routine. Gets to run to clear the queue. Any thread
* (worker, interrupt, poll) can call this based on processing model.
* The first thing we do is disable interrupts if possible and then
* drain the queue. we also try to poll the underlying hardware if
* there is a dedicated hardware Rx ring assigned to this SRS.
*
* There is a equivalent drain routine in non bandwidth control mode
* mac_rx_srs_drain. There is some code duplication between the two
* routines but they are highly performance sensitive and are easier
* also apply to mac_rx_srs_drain as well.
*/
void
{
int cnt = 0;
/* Check if we are doing B/W control */
now = ddi_get_lbolt();
goto done;
goto done;
}
/* If we are blanked i.e. can't do upcalls, then we are done */
goto done;
}
sz = 0;
cnt = 0;
/*
* We couldn't pick up a single packet.
*/
/*
* Seems like configured B/W doesn't
* even allow processing of 1 packet
* per tick.
*
* XXX: raise the limit to processing
* at least 1 packet per tick.
*/
"raised B/W limit to %d since not even a "
"single packet can be processed per "
"tick %d\n", (void *)mac_srs,
}
goto done;
}
/* zero bandwidth: drop all and return to interrupt mode */
goto leave_poll;
} else {
}
/*
* mcip is NULL for broadcast and multicast flows. The promisc
* callbacks for broadcast and multicast packets are delivered from
* mac_rx() and we don't need to worry about that case in this path
*/
}
/*
* Check if SRS itself is doing the processing
* This direct path does not apply when subflows are present. In this
* case, packets need to be dispatched to a soft ring according to the
* flow's bandwidth and other resources contraints.
*/
void *arg1;
/*
* This is the case when a Rx is directly
* assigned and we have a fully classified
* protocol chain. We can deal with it in
* one shot.
*/
if (tid != 0) {
tid = 0;
}
/*
* Decrement the size and count here itelf
* since the packet has been processed.
*/
} else {
/* Some kind of softrings based fanout is required */
if (tid != 0) {
tid = 0;
}
/*
* Since the fanout routines can deal with chains,
* shoot the entire chain up.
*/
else
}
/*
* Send the poll thread to pick up any packets arrived
* so far. This also serves as the last check in case
* nothing else is queued in the SRS. The poll thread
* is signalled only in the case the drain was done
* by the worker thread and SRS_WORKER is set. The
* worker thread can run in parallel as long as the
* SRS_WORKER flag is set. We we have nothing else to
* process, we can exit while leaving SRS_PROC set
* which gives the poll thread control to process and
* cleanup once it returns from the NIC.
*
* If we have nothing else to process, we need to
* ensure that we keep holding the srs_lock till
* all the checks below are done and control is
* handed to the poll thread if it was running.
*/
if (proc_type == SRS_WORKER) {
if (srs_rx->sr_poll_pkt_cnt <=
goto again;
} else {
}
}
}
done:
/*
* Poll thread is already running. Leave the
* SRS_RPOC set and hand over the control to
* poll thread.
*/
return;
}
/*
* If we can't process packets because we have exceeded
* B/W limit for this tick, just set the timeout
* and leave.
*
* Even if there are no packets queued in SRS, we
* need to make sure that the shared counter is
* clear and any associated softrings have cleared
* all the backlog. Otherwise, leave the interface
* in polling mode and the poll thread will get
* signalled once the count goes down to zero.
*
* If someone is already draining the queue (SRS_PROC is
* set) when the srs_poll_pkt_cnt goes down to zero,
* then it means that drain is already running and we
* will turn off polling at that time if there is
* no backlog. As long as there are packets queued either
* is soft ring set or its soft rings, we will leave
* the interface in polling mode.
*/
(srs_rx->sr_poll_pkt_cnt > 0))) {
mac_srs, 1);
return;
}
/* Nothing else to do. Get out of poll mode */
}
/*
* mac_srs_worker
*
* The SRS worker routine. Drains the queue when no one else is
* processing it.
*/
void
{
for (;;) {
}
/*
* The SRS_BW_ENFORCED flag may change since we have dropped
* the mac_bw_lock. However the drain function can handle both
* a drainable SRS or a bandwidth controlled SRS, and the
* effect of scheduling a timeout is to wakeup the worker
* thread which in turn will call the drain function. Since
* we release the srs_lock atomically only in the cv_wait there
* isn't a fear of waiting for ever.
*/
/*
* If we have packets queued and we are here
* because B/W control is in place, we better
* schedule the worker wakeup after 1 tick
* to see if bandwidth control can be relaxed.
*/
/*
* We need to ensure that a timer is already
* scheduled or we force schedule one for
* later so that we can continue processing
* after this quanta is over.
*/
mac_srs, 1);
}
wait:
goto done;
goto wait;
}
}
}
goto done;
}
done:
/*
* The Rx SRS quiesce logic first cuts off packet supply to the SRS
* from both hard and soft classifications and waits for such threads
* to finish before signaling the worker. So at this point the only
* thread left that could be competing with the worker is the poll
* thread. In the case of Tx, there shouldn't be any thread holding
* SRS_PROC at this point.
*/
} else {
/*
* Poll thread still owns the SRS and is still running
*/
}
/*
* Wait for the SRS_RESTART or SRS_CONDEMNED signal from the initiator
* of the quiesce operation
*/
goto start;
}
/* The macro drops the srs_lock */
thread_exit();
}
/*
* mac_rx_srs_subflow_process
*
* Receive side routine called from interrupt path when there are
* sub flows present on this SRS.
*/
/* ARGSUSED */
void
{
/*
* We need to determine the SRS for every packet
* by walking the flow table, if we don't get any,
* then we proceed using the SRS we came with.
*/
/*
* We will increment the stats for the mactching subflow.
* later in mac_rx_srs_process.
*/
FLOW_INBOUND, &flent);
if (prev_flent != NULL)
prev_flent = flent;
continue;
}
/*
* A null indicates, this is for the mac_srs itself.
* XXX-venu : probably assert for fe_rx_srs_cnt == 0.
*/
loopback);
} else {
}
prev_flent = flent;
}
/* Last chain */
} else {
}
}
/*
* mac_rx_srs_process
*
* Receive side routine called from the interrupt path.
*
* loopback is set to force a context switch on the loopback
* path between MAC clients.
*/
/* ARGSUSED */
void
{
int count = 0;
int count1;
/*
* Set the tail, count and sz. We set the sz irrespective
* of whether we are doing B/W control or not for the
* purpose of updating the stats.
*/
count++;
}
}
}
/*
* If the SRS in already being processed; has been blanked;
* can be processed by worker thread only; or the B/W limit
* has been reached, then queue the chain and check if
* worker thread needs to be awakend.
*/
/* Count the packets and bytes via interrupt */
if (mac_bw->mac_bw_limit == 0) {
/* zero bandwidth: drop all */
return;
} else {
} else {
chain_sz = 0;
count1 = 0;
break;
count1++;
}
} else {
/* Can't pick up any */
}
/* Drop any packet over the threshold */
}
}
return;
}
}
/*
* If the total number of packets queued in the SRS and
* its associated soft rings exceeds the max allowed,
* then drop the chain. If we are polling capable, this
* shouldn't be happening.
*/
return;
}
/* Count the packets entering via interrupt path */
/*
* If we are coming via loopback or if we are not
* optimizing for latency, we should signal the
* worker thread.
*/
/*
* For loopback, We need to let the worker take
* over as we don't want to continue in the same
* thread even if we can. This could lead to stack
* overflows and may also end up using
* resources (cpu) incorrectly.
*/
} else {
/*
* Seems like no one is processing the SRS and
* there is no backlog. We also inline process
* our packet if its a single packet in non
* latency optimized case (in latency optimized
* case, we inline process chains of any size).
*/
}
}
}
/* TX SIDE ROUTINES (RUNTIME) */
/*
* mac_tx_srs_no_desc
*
* This routine is called by Tx single ring default mode
* when Tx ring runs out of descs.
*/
{
if (flag & MAC_DROP_ON_NO_DESC) {
} else {
if (flag & MAC_TX_NO_ENQUEUE) {
/*
* If TX_QUEUED is not set, queue the
* packet and let mac_tx_srs_drain()
* set the TX_BLOCKED bit for the
* reasons explained above. Otherwise,
* return the mblks.
*/
if (wakeup_worker) {
} else {
}
} else {
}
if (wakeup_worker)
}
return (cookie);
}
/*
* mac_tx_srs_enqueue
*
* This routine is called when Tx SRS is operating in either serializer
* or bandwidth mode. In serializer mode, a packet will get enqueued
* when a thread cannot enter SRS exclusively. In bandwidth mode,
* packets gets queued if allowed byte-count limit for a tick is
* exceeded. The action that gets taken when MAC_DROP_ON_NO_DESC and
* MAC_TX_NO_ENQUEUE is set is different than when operaing in either
* the default mode or fanout mode. Here packets get dropped or
* returned back to the caller only after hi-watermark worth of data
* is queued.
*/
static mac_tx_cookie_t
{
/*
* Ignore fanout hint if we don't have multiple tx rings.
*/
if (!TX_MULTI_RING_MODE(mac_srs))
fanout_hint = 0;
if (flag & MAC_DROP_ON_NO_DESC) {
} else {
}
} else if (flag & MAC_TX_NO_ENQUEUE) {
} else {
}
} else {
/*
* If you are BW_ENFORCED, just enqueue the
* packet. srs_worker will drain it at the
* prescribed rate. Before enqueueing, save
* the fanout hint.
*/
}
if (wakeup_worker)
return (cookie);
}
/*
* There are five tx modes:
*
* 1) Default mode (SRS_TX_DEFAULT)
* 2) Serialization mode (SRS_TX_SERIALIZE)
* 3) Fanout mode (SRS_TX_FANOUT)
* 4) Bandwdith mode (SRS_TX_BW)
* 5) Fanout and Bandwidth mode (SRS_TX_BW_FANOUT)
*
* The tx mode in which an SRS operates is decided in mac_tx_srs_setup()
* based on the number of Tx rings requested for an SRS and whether
* bandwidth control is requested or not.
*
* pass-thru. Packets will go directly to mac_tx_send(). When the underlying
* Tx ring runs out of Tx descs, it starts queueing up packets in SRS.
* When flow-control is relieved, the srs_worker drains the queued
* packets and informs blocked clients to restart sending packets.
*
* In the SRS_TX_SERIALIZE mode, all calls to mac_tx() are serialized.
*
* In the SRS_TX_FANOUT mode, packets will be fanned out to multiple
* Tx rings. Each Tx ring will have a soft ring associated with it.
* These soft rings will be hung off the Tx SRS. Queueing if it happens
* due to lack of Tx desc will be in individual soft ring (and not srs)
* associated with Tx ring.
*
* In the TX_BW mode, tx srs will allow packets to go down to Tx ring
* only if bw is available. Otherwise the packets will be queued in
* SRS. If fanout to multiple Tx rings is configured, the packets will
* be fanned out among the soft rings associated with the Tx rings.
*
* Four flags are used in srs_state for indicating flow control
* conditions : SRS_TX_BLOCKED, SRS_TX_HIWAT, SRS_TX_WAKEUP_CLIENT.
* SRS_TX_BLOCKED indicates out of Tx descs. SRS expects a wakeup from the
* driver below.
* SRS_TX_HIWAT indicates packet count enqueued in Tx SRS exceeded Tx hiwat
* and flow-control pressure is applied back to clients. The clients expect
* wakeup when flow-control is relieved.
* SRS_TX_WAKEUP_CLIENT get set when (flag == MAC_TX_NO_ENQUEUE) and mblk
* got returned back to client either due to lack of Tx descs or due to bw
* control reasons. The clients expect a wakeup when condition is relieved.
*
* The fourth argument to mac_tx() is the flag. Normally it will be 0 but
* some clients set the following values too: MAC_DROP_ON_NO_DESC,
* MAC_TX_NO_ENQUEUE
* Mac clients that do not want packets to be enqueued in the mac layer set
* MAC_DROP_ON_NO_DESC value. The packets won't be queued in the Tx SRS or
* Tx soft rings but instead get dropped when the NIC runs out of desc. The
* behaviour of this flag is different when the Tx is running in serializer
* or bandwidth mode. Under these (Serializer, bandwidth) modes, the packet
* get dropped when Tx high watermark is reached.
* There are some mac clients like vsw, aggr that want the mblks to be
* returned back to clients instead of being queued in Tx SRS (or Tx soft
* rings) under flow-control (i.e., out of desc or exceeding bw limits)
* conditions. These clients call mac_tx() with MAC_TX_NO_ENQUEUE flag set.
* In the default and Tx fanout mode, the un-transmitted mblks will be
* returned back to the clients when the driver runs out of Tx descs.
* SRS_TX_WAKEUP_CLIENT (or S_RING_WAKEUP_CLIENT) will be set in SRS (or
* soft ring) so that the clients can be woken up when Tx desc become
* available. When running in serializer or bandwidth mode mode,
* SRS_TX_WAKEUP_CLIENT will be set when tx hi-watermark is reached.
*/
{
}
/* ARGSUSED */
static mac_tx_cookie_t
{
/* Regular case with a single Tx ring */
/*
* SRS_TX_BLOCKED is set when underlying NIC runs
* out of Tx descs and messages start getting
* queued. It won't get reset until
* tx_srs_drain() completely drains out the
* messages.
*/
return (cookie);
}
/*
* While we were computing mblk count, the
* flow control condition got relieved.
* Continue with the transmission.
*/
}
/*
* Multiple threads could be here sending packets.
* Under such conditions, it is not possible to
* automically set SRS_TX_BLOCKED bit to indicate
* out of tx desc condition. To atomically set
* this, we queue the returned packet and do
* the setting of SRS_TX_BLOCKED in
* mac_tx_srs_drain().
*/
return (cookie);
}
if (is_subflow)
return (NULL);
}
/*
* mac_tx_serialize_mode
*
* This is an experimental mode implemented as per the request of PAE.
* In this mode, all callers attempting to send a packet to the NIC
* will get serialized. Only one thread at any time will access the
* NIC to send the packet out.
*/
/* ARGSUSED */
static mac_tx_cookie_t
{
/* Single ring, serialize below */
/*
* In serialization mode, queue all packets until
* TX_HIWAT is set.
* If drop bit is set, drop if TX_HIWAT is set.
* If no_enqueue is set, still enqueue until hiwat
* is set and return mblks after TX_HIWAT is set.
*/
return (cookie);
}
/*
* No packets queued, nothing on proc and no flow
* control condition. Fast-path, ok. Do inline
* processing.
*/
}
/*
* We processed inline our packet and a new
* packet/s got queued while we were
* processing. Wakeup srs worker
*/
}
return (cookie);
}
/*
* mac_tx_fanout_mode
*
* In this mode, the SRS will have access to multiple Tx rings to send
* the packet out. The fanout hint that is passed as an argument is
* used to find an appropriate ring to fanout the traffic. Each Tx
* ring, in turn, will have a soft ring associated with it. If a Tx
* ring runs out of Tx desc's the returned packet will be queued in
* the soft ring associated with that Tx ring. The srs itself will not
* queue any packets.
*/
#define MAC_TX_SOFT_RING_PROCESS(chain) { \
}
static mac_tx_cookie_t
{
if (fanout_hint != 0) {
/*
* The hint is specified by the caller, simply pass the
* whole chain to the soft ring.
*/
} else {
/*
* Compute the hash from the contents (headers) of the
* packets of the mblk chain. Split the chains into
* subchains of the same conversation.
*
* Since there may be more than one ring used for
* sub-chains of the same call, and since the caller
* does not maintain per conversation state since it
* passed a zero hint, unsent subchains will be
* dropped.
*/
B_TRUE);
/*
* Starting a different subchain, send current
* chain out.
*/
}
/* add packet to subchain */
}
/* send last subchain */
}
}
return (cookie);
}
/*
* mac_tx_bw_mode
*
* In the bandwidth mode, Tx srs will allow packets to go down to Tx ring
* only if bw is available. Otherwise the packets will be queued in
* SRS. If the SRS has multiple Tx rings, then packets will get fanned
* out to a Tx rings.
*/
static mac_tx_cookie_t
{
/*
* zero bandwidth, no traffic is sent: drop the packets,
* or return the whole chain if the caller requests all
* unsent packets back.
*/
if (flag & MAC_TX_NO_ENQUEUE) {
} else {
}
return (cookie);
return (cookie);
}
now = ddi_get_lbolt();
/*
* Wakeup worker thread. Note that worker
* thread has to be woken up so that it
* can fire up the timer to be woken up
* on the next tick. Also once
* BW_ENFORCED is set, it can only be
* reset by srs_worker thread. Until then
* all packets will get queued up in SRS
* and hence this this code path won't be
* entered until BW_ENFORCED is reset.
*/
return (cookie);
}
ret_mp));
} else {
else
return (cookie);
}
if (is_subflow)
return (NULL);
}
}
/* ARGSUSED */
void
{
saved_pkt_count = 0;
/* Device out of tx desc, set block */
} else {
if (is_subflow) {
}
}
}
/*
* We are here because the timer fired and we have some data
* to tranmit. Also mac_tx_srs_worker should have reset
* SRS_BW_ENFORCED flag
*/
continue;
now = ddi_get_lbolt();
continue;
}
break;
}
/* Device out of tx desc, set block */
} else {
}
} else {
if (is_subflow) {
}
}
}
} else if (tx_mode == SRS_TX_BW_FANOUT) {
/*
* We are here because the timer fired and we
* have some quota to tranmit.
*/
}
continue;
now = ddi_get_lbolt();
continue;
}
break;
}
}
}
/*
* SRS_TX_FANOUT case not considered here because packets
* won't be queued in the SRS for this case. Packets will
* be sent directly to soft rings underneath and if there
* is any queueing at all, it would be in Tx side soft
* rings.
*/
/*
* When srs_count becomes 0, reset SRS_TX_HIWAT and
* SRS_TX_WAKEUP_CLIENT and wakeup registered clients.
*/
}
if (wakeup_required) {
/* Wakeup callback registered clients */
}
/*
* If the client is not the primary MAC client, then we
* need to send the notification to the clients upper
* MAC, i.e. mci_upper_mip.
*/
}
}
}
/*
* Given a packet, get the flow_entry that identifies the flow
* to which that packet belongs. The flow_entry will contain
* the transmit function to be used to send the packet. If the
* function returns NULL, the packet should be sent using the
* underlying NIC.
*/
static flow_entry_t *
{
int err;
/*
* Do classification on the packet.
*/
if (err != 0)
return (NULL);
/*
* This flent might just be an additional one on the MAC client,
* i.e. for classification purposes (different fdesc), however
* the resources, SRS et. al., are in the mci_flent, so if
* this isn't the mci_flent, we need to get it.
*/
if (err != 0)
return (NULL);
}
return (flent);
}
/*
* This macro is only meant to be used by mac_tx_send().
*/
#define CHECK_VID_AND_ADD_TAG(mp) { \
if (vid_check) { \
int err = 0; \
\
if (err != 0) { \
oerrors++; \
continue; \
} \
} \
if (add_tag) { \
oerrors++; \
continue; \
} \
} \
}
mblk_t *
{
if (add_tag)
} else {
}
/*
* Fastpath: if there's only one client, and there's no
* multicast listeners, we simply send the packet down to the
* underlying NIC.
*/
opackets++;
0));
/*
* If the driver is out of descriptors and does a
* partial send it will return a chain of unsent
* mblks. Adjust the accounting stats.
*/
opackets--;
break;
}
}
goto done;
}
/*
* No fastpath, we either have more than one MAC client
* defined on top of the same MAC, or one or more MAC
* client promiscuous callbacks.
*/
void *flow_cookie;
opackets++;
/*
* Check if there are promiscuous mode callbacks defined.
*/
/*
* Find the destination.
*/
if (dst_flow_ent != NULL) {
int err = 0;
struct ether_vlan_header *evhp =
else
hdrsize = sizeof (struct ether_header);
} else {
if (err == 0)
}
/*
* Got a matching flow. It's either another
* Make sure the packet size is within the
* allowed size. If not drop the packet and
* move to next packet.
*/
if (err != 0 ||
oerrors++;
continue;
}
if (flow_cookie != NULL) {
/*
* The vnic_bcast_send function expects
* to receive the sender MAC client
* as value for arg2.
*/
B_TRUE);
} else {
/*
* loopback the packet to a
* local MAC client. We force a context
* switch if both source and destination
* MAC clients are used by IP, i.e. bypass
* is set.
*/
MCIS_CLIENT_POLL_CAPABLE) != 0);
(dst_flow_ent->fe_cb_fn)(
}
}
} else {
/*
* Unknown destination, send via the underlying
* NIC.
*/
0));
/*
* Adjust for the last packet that
* could not be transmitted
*/
opackets--;
break;
}
}
}
done:
}
return (mp);
}
/*
* mac_tx_srs_ring_present
*
* Returns whether the specified ring is part of the specified SRS.
*/
{
int i;
return (B_TRUE);
for (i = 0; i < srs->srs_oth_ring_count; i++) {
return (B_TRUE);
}
return (B_FALSE);
}
/*
* mac_tx_srs_wakeup
*
* Called when Tx desc become available. Wakeup the appropriate worker
* thread after resetting the SRS_TX_BLOCKED/S_RING_BLOCK bit in the
* state field.
*/
void
{
int i;
if (TX_SINGLE_RING_MODE(mac_srs)) {
}
/*
* A wakeup can come before tx_srs_drain() could
* grab srs lock and set SRS_TX_BLOCKED. So
* always set woken_up flag when we come here.
*/
return;
}
/* If you are here, it is for FANOUT or BW_FANOUT case */
for (i = 0; i < mac_srs->srs_oth_ring_count; i++) {
}
}
}
}
/*
* Once the driver is done draining, send a MAC_NOTE_TX notification to unleash
* the blocked clients again.
*/
void
{
}
/*
* RX SOFTRING RELATED FUNCTIONS
*
* These functions really belong in mac_soft_ring.c and here for
* a short period.
*/
/* \
* Enqueue our mblk chain. \
*/ \
\
else \
} \
}
/*
* Default entry point to deliver a packet chain to a MAC client.
* If the MAC client has flows, do the classification with these
* flows as well.
*/
/* ARGSUSED */
void
{
/*
* If the client has exactly one VID associated with it
* and striping of VLAN header is not disabled,
* remove the VLAN tag from the packet before
* passing it on to the client's receive callback.
* Note that this needs to be done after we dispatch
* the packet to the promiscuous listeners of the
* client, since they expect to see the whole
* frame including the VLAN headers.
*/
}
}
/*
* mac_rx_soft_ring_process
*
* process a chain for a given soft ring. The number of packets queued
* in the SRS and its associated soft rings (including this one) is
* very small (tracked by srs_poll_pkt_cnt), then allow the entering
* thread (interrupt or poll thread) to do inline processing. This
* helps keep the latency down under low load.
*
* The proc and arg for each mblk is already stored in the mblk in
* appropriate places.
*/
/* ARGSUSED */
void
{
void *arg1;
/* If on processor or blanking on, then enqueue and return */
return;
}
/*
* See if anything is already queued. If we are the
* first packet, do inline processing else queue the
* packet and do the drain.
*/
/*
* Fast-path, ok to process and nothing queued.
*/
/*
* We are the chain of 1 packet so
* go through this fast path.
*/
/*
* If we have a soft ring set which is doing
* bandwidth control, we need to decrement
* srs_size and count so it the SRS can have a
* accurate idea of what is the real data
* queued between SRS and its soft rings. We
* decrement the counters only when the packet
* gets processed by both SRS and the soft ring.
*/
/*
* We processed inline our packet and
* nothing new has arrived or our
* receiver doesn't want to receive
* any packets. We are done.
*/
return;
}
} else {
}
/*
* We are here because either we couldn't do inline
* processing (because something was already
* queued), or we had a chain of more than one
* packet, or something else arrived after we were
* done with inline processing.
*/
return;
} else {
/* ST_RING_WORKER_ONLY case */
}
}
/*
* TX SOFTRING RELATED FUNCTIONS
*
* These functions really belong in mac_soft_ring.c and here for
* a short period.
*/
}
/*
* mac_tx_sring_queued
*
* When we are out of transmit descriptors and we already have a
* queue that exceeds hiwat (or the client called us with
* MAC_TX_NO_ENQUEUE or MAC_DROP_ON_NO_DESC flag), return the
* soft ring pointer as the opaque cookie for the client enable
* flow control.
*/
static mac_tx_cookie_t
{
int cnt;
if (flag & MAC_DROP_ON_NO_DESC) {
/* increment freed stats */
} else {
if (flag & MAC_TX_NO_ENQUEUE) {
/*
* If QUEUED is not set, queue the packet
* and let mac_tx_soft_ring_drain() set
* the TX_BLOCKED bit for the reasons
* explained above. Otherwise, return the
* mblks.
*/
if (wakeup_worker) {
} else {
}
} else {
/*
* flow-controlled. Store ringp in cookie
* so that it can be returned as
* mac_tx_cookie_t to client
*/
if (ringp->s_ring_count >
/* increment freed stats */
/*
* b_prev may be set to the fanout hint
* hence can't use freemsg directly
*/
mac_soft_ring_t *, ringp);
}
}
if (enqueue) {
}
}
if (wakeup_worker)
}
return (cookie);
}
/*
* mac_tx_soft_ring_process
*
* This routine is called when fanning out outgoing traffic among
* multipe Tx rings.
* Note that a soft ring is associated with a h/w Tx ring.
*/
{
int cnt;
/*
* Only two modes can come here; either it can be
* SRS_TX_BW_FANOUT or SRS_TX_FANOUT
*/
/* Serialization mode */
return (cookie);
}
/*
* If ring is blocked due to lack of Tx
* descs, just return. Worker thread
* will get scheduled when Tx desc's
* become available.
*/
return (cookie);
}
return (cookie);
} else {
/* Default fanout mode */
/*
* S_RING_BLOCKED is set when underlying NIC runs
* out of Tx descs and messages start getting
* queued. It won't get reset until
* tx_srs_drain() completely drains out the
* messages.
*/
return (cookie);
}
/*
* While we were computing mblk count, the
* flow control condition got relieved.
* Continue with the transmission.
*/
}
/*
* Multiple threads could be here sending packets.
* Under such conditions, it is not possible to
* automically set S_RING_BLOCKED bit to indicate
* out of tx desc condition. To atomically set
* this, we queue the returned packet and do
* the setting of S_RING_BLOCKED in
* mac_tx_soft_ring_drain().
*/
cookie =
return (cookie);
}
if (is_subflow) {
}
return (NULL);
}
}