/*
* CDDL HEADER START
*
* The contents of this file are subject to the terms of the
* Common Development and Distribution License (the "License").
* You may not use this file except in compliance with the License.
*
* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
* See the License for the specific language governing permissions
* and limitations under the License.
*
* When distributing Covered Code, include this CDDL HEADER in each
* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
* If applicable, add the following below this CDDL HEADER, with the
* fields enclosed by brackets "[]" replaced with your own identifying
* information: Portions Copyright [yyyy] [name of copyright owner]
*
* CDDL HEADER END
*/
/*
* Copyright 2005 Sun Microsystems, Inc. All rights reserved.
* Use is subject to license terms.
*/
/*
* This file is part of the Chelsio T1 Ethernet driver.
*
* Copyright (C) 2003-2005 Chelsio Communications. All rights reserved.
*/
#pragma ident "%Z%%M% %I% %E% SMI"
#include <sys/byteorder.h>
#include <sys/ethernet.h>
#include "ostypes.h"
#include "common.h"
#ifdef CONFIG_CHELSIO_T1_1G
#include "fpga_defs.h"
#endif
#include "regs.h"
#include "suni1x10gexp_regs.h"
#include "sge.h"
#include "espi.h"
#include "ch.h"
extern uint32_t buffers_in_use[];
#ifdef HOST_PAUSE
#endif
#ifdef HOST_PAUSE
#endif
#ifdef SUN_KSTATS
static int sge_kstat_setup(pesge *);
static void sge_kstat_remove(pesge *);
static int sge_kstat_update(p_kstat_t, int);
#endif
/*
* Local routines.
*/
static inline void
{
}
/*
* DESC:
*
* NOTES: Must have at least 1 command queue and 1 freelist queue.
*
*/
pesge *
{
goto error_no_mem;
/*
* PR2928 & PR3309
* set default timeout value - 20 msec
* we set the initial value to 2 which gurantees at least one tick.
*/
#ifdef SUN_KSTATS
if (sge_kstat_setup(sge) != 0)
goto t1_sge_create_fail1;
#endif
p->cmdQ_size[0] = sge_cmdq0_cnt;
/* note that jumbo frame index is inverted for T2 */
p->freelQ_size[0] = sge_flq1_cnt;
} else {
p->freelQ_size[0] = sge_flq0_cnt;
}
#if CH_DEBUG
/* DEBUG only */
#endif
#ifdef SUN_KSTATS
goto error_no_mem;
#endif
return (sge);
}
int
{
/* PR2928 & PR3309 */
#ifdef SUN_KSTATS
#endif
}
return (0);
}
/*
* PR2928 & PR3309
* call out event from timeout
*
* there is a potential race between the timeout and the close.
* unless we protect the timeout, the close could occur at the
* same time. Then if the timeout service routine was slow or
* interrupted, the sge_stop() could complete with a timeoutID
* that has expired, thus letting another timeout occur. If the
* service routine was delayed still further, a detach could occur.
* the second time could then end up accessing memory that has been
* released back to the system. Bad things could then occur. We
* set a flag in sge_stop() to tell the service routine not to
* issue further timeouts. sge_stop() will block until a timeout
* has occured. If the command Q is full then we shouldn't put out
* an arp.
*/
void
{
/* after first arp */
CH_ARP);
if (!rv)
}
}
#ifdef HOST_PAUSE
/*
* If we are already in sge_data_in, then we can skip calling
* t1_sge_check_pause() this clock cycle. lockstat showed that
* we were blocking on the mutex ~ 2% of the time.
*/
}
#endif
}
int
{
/* PR2928 & PR3309, also need to avoid Pause deadlock */
return (0);
}
/*
* Disables SGE queues.
*/
int
{
int loops;
/* PR2928 & PR3309, also need to avoid Pause deadlock */
/* wait until there's no more outstanding interrupts pending */
loops = 0;
do {
drv_usecwait(125);
loops++;
return (0);
}
int
{
cmdQ_e *e;
cmdQ_e *q = Q->cq_entries;
uint32_t j = 0;
#if defined(TX_CKSUM_FIX)
#endif
#ifdef TX_THREAD_RECLAIM
#endif
/*
* We must exit if we don't have enough free command queue entries
* available.
*/
#if defined(TX_CKSUM_FIX)
/*
* This checksum fix will address a fragmented datagram
* checksum error. Which will lead to the next packet after
* the last packet with the More fragment bit set having its
* checksum corrupted. When the packet reaches this point
* the 'flg' variable indicates whether a checksum is needed
* or not. The algorithm is as follows, if the current packet
* is a More fragment set the count of packets to be checksummed
* after it to 3. If it't not and the count of is more than 0
* then calculate the checksum in software, if a hardware checksum
* was requested. Then decrment the count. Same algorithm applies
* to TCP.
*/
if ((flg & CH_NO_HWCKSUM) == 0) {
/*
* Calc Checksum here.
*/
sizeof (struct ether_header) + CPL_FORMAT_0_SIZE);
sizeof (struct ether_header) + CPL_FORMAT_0_SIZE);
}
sge->do_udp_csum--;
} else if (sge->do_tcp_csum != 0) {
if ((flg & CH_NO_HWCKSUM) == 0) {
/*
* Calc Checksum here.
*/
}
sge->do_tcp_csum--;
}
#endif /* TX_CKSUM_FIX */
#ifdef TX_THREAD_RECLAIM
reclaim_cnt = Q->cq_complete;
if (reclaim_cnt > SGE_BATCH_THRESH) {
Q->cq_complete = 0;
}
#endif
credits = Q->cq_credits;
return (1);
}
Q->cq_genbit ^= 1;
}
#ifdef SUN_KSTATS
else
#endif
j++;
e = &q[pidx];
e->Sop = 1;
e->DataValid = 1;
--count;
if (count > 0) {
unsigned int i;
e->Eop = 0;
wmb();
for (i = 0; i < count; i++) {
ce++;
e++;
cmp++;
pidx = 0;
genbit ^= 1;
/* sync from offset to end of cmdQ */
j*sizeof (*e), DDI_DMA_SYNC_FORDEV);
offset = j = 0;
e = q;
}
j++;
e->Sop = 0;
e->DataValid = 1;
if (i < (count - 1)) {
e->Eop = 0;
wmb();
}
}
}
e->Eop = 1;
wmb();
/*
* We always ring the doorbell for cmdQ1. For cmdQ0, we only ring
* the doorbell if the Q is asleep. There is a natural race, where
* the hardware is going to sleep just after we checked, however,
* then the interrupt handler will detect the outstanding TX packet
* and ring the doorbell for us.
*/
if (qid) {
} else {
if (atomic_read(Q->cq_asleep)) {
atomic_set(&Q->cq_asleep, 0);
/* NOT YET doorbell_pio(sge, F_CMDQ0_ENABLE); */
}
}
return (0);
}
/*
* Disable SGE error interrupts.
*/
int
{
return (0);
}
/*
* Enable SGE error interrupts.
*/
int
{
en &= ~F_PACKET_TOO_BIG;
return (0);
}
/*
* Clear SGE error interrupts.
*/
int
{
return (0);
}
int
{
if (cause & F_RESPQ_EXHAUSTED)
if (cause & F_RESPQ_OVERFLOW) {
}
if (cause & F_FL_EXHAUSTED) {
}
if (cause & F_PACKET_TOO_BIG) {
}
if (cause & F_PACKET_MISMATCH) {
}
if (cause & SGE_INT_FATAL)
return (0);
}
/*
*
* PARAM: sge - SGE instance pointer.
*/
int
{
respQ_e *e; /* response queue entry */
#ifndef TX_THREAD_RECLAIM
#endif
/*
* Catch the case where an interrupt arrives
* early.
*/
goto check_slow_ints;
}
/* initial response queue entry */
e = &q[cidx];
/* pull physical memory of response queue entry into cache */
sizeof (*e), DDI_DMA_SYNC_FORKERNEL);
while (e->GenerationBit == genbit) {
if (--credits < credits_thresh) {
credits += n;
}
e->BufferLength, e->Offload);
}
}
if (flagt & F_CMDQ0_ENABLE)
if (flagt & F_CMDQ1_ENABLE)
if (flagt & F_FL0_ENABLE)
if (flagt & F_FL1_ENABLE)
#ifdef TX_THREAD_RECLAIM
}
#else
credits_pend[0] += e->Cmdq0CreditReturn;
#ifdef CONFIG_SMP
credits_pend[0] = 0;
}
credits_pend[1] = 0;
}
#endif
#endif
#ifdef HOST_PAUSE
#endif
e++;
cidx = 0;
genbit ^= 1;
e = q;
}
/* pull physical memory of response queue entry into cache */
sizeof (*e), DDI_DMA_SYNC_FORKERNEL);
ret = 1;
}
#ifndef TX_THREAD_RECLAIM
if (credits_pend[0])
if (credits_pend[1])
#endif
if (flags & F_CMDQ0_ENABLE) {
}
}
/* the SGE told us one of the free lists is empty */
#ifdef CONFIG_CHELSIO_T1_OFFLOAD
if (adapter->ch_tx_overflow_mutex)
if (adapter->ch_tx_overflow_cv)
}
if (adapter->ch_tx_overflow_mutex)
#else
#ifndef TX_THREAD_RECLAIM
}
#endif
#endif /* CONFIG_CHELSIO_T1_OFFLOAD */
Q->rq_credits = credits;
/* handle non-data interrupts */
return (ret);
}
/*
* allocate a mblk with DMA mapped mblk.
* When checksum offload is enabled, we start the DMA at a 2 byte offset so
* the IP header will be aligned. We do this for sparc only.
*/
static uint64_t
{
/* get pre-mapped buffer */
return ((uint64_t)0);
}
return ((uint64_t)0);
}
} else {
/* get pre-mapped buffer */
return ((uint64_t)0);
}
return ((uint64_t)0);
}
}
}
static inline unsigned int
{
#ifdef CONFIG_CHELSIO_T1_OFFLOAD
#endif
if (Q->fq_id)
else
/*
* If pkt size falls below threshold, then we'll copy data to
* an blk and reuse mblk.
*
* NOTE that rxoff is 2 for T1 adapters. We align the the start
* of the DMA buffer begin at rxoff offset for T1 cards instead of
* at the beginning of the buffer, thus the length of the received
* data does not include this offset. We therefore always add
* SGE_RX_OFFSET to the allocb size so we have space to provide the
* offset for the copied data.
*/
#ifdef HOST_PAUSE
/*
* If we have Host pause compiled in, then we look at the
* free list, if the pause is on and we're not in offload
* mode then we drop packets, this is designed to avoid
* overwhelming the machine. If the machine is powerfull enough
* this will not happen. The 'rx_pkt_drops' will show when
* packets are being dropped and how much.
*/
freelQ_e *e;
/* Ditch the packet and reuse original buffer */
e = &Q->fq_entries[cidx];
e->GenerationBit ^= 1;
e->GenerationBit2 ^= 1;
goto rx_entry_consumed;
(len <= SGE_RX_COPY_THRESHOLD)) &&
#else
if ((len <= SGE_RX_COPY_THRESHOLD) &&
#endif
{
freelQ_e *e;
/*
* pull physical memory of pkt data into cache
* Note that len does not include offset for T1.
*/
if (offload == 0) {
/*
* create 2 byte offset so IP header aligned on
* 4 byte boundry
*/
/*
* if hardware inserted 2 byte offset then need to
* start copying with extra offset
*/
}
useit = 0; /* mblk copy, don't inc esballoc in use cnt */
/* so we can reuse original buffer */
e = &Q->fq_entries[cidx];
e->GenerationBit ^= 1;
e->GenerationBit2 ^= 1;
} else {
/* consume buffer off the ring */
/*
* if not offload (tunneled pkt), & hardward padded, then
* adjust start of pkt to point to start of data i.e.
* skip pad (2 bytes).
*/
/*
* pull physical memory of pkt data into cache
* Note that len does not include offset for T1.
*/
}
/* set length of data in skb */
#ifdef CONFIG_CHELSIO_T1_OFFLOAD
/* sends pkt upstream to toe layer */
if (useit) {
atomic_add(1,
} else {
atomic_add(1,
}
}
else
} else {
"%s: unexpected offloaded packet, cmd %u\n",
/* discard packet */
}
}
#else
"%s: unexpected offloaded packet, cmd %u\n",
/* discard paket */
}
#endif
else {
int flg = 0;
/* adjust beginning of data to skip CPL header */
/* extract checksum from CPL header here */
/*
* bump count of mlbks in used by protocol stack(s)
*/
if (useit) {
atomic_add(1,
} else {
atomic_add(1,
}
}
#ifdef CONFIG_CHELSIO_T1_OFFLOAD
/*
* let the TOE layer have a crack at the packet first.
*/
if (adapter->toe_tunnel) {
/*
* The TOE may have consumed the packet.
*/
if (rv)
goto rx_entry_consumed;
}
#endif /* CONFIG_CHELSIO_T1_OFFLOAD */
/*
* NOTE: 14+9 = size of MAC + offset to IP protocol field
*/
ETHERTYPE_IP) &&
flg = 1;
}
}
cidx = 0;
/* allocate new buffers on the free list */
alloc_freelQ_buffers(sge, Q);
return (1);
}
#ifdef HOST_PAUSE
static void
{
/*
* If the number of available credits shrinks below
* the Pause on threshold then enable the pause and
* try and allocate more buffers.
* On the next pass, if there's more credits returned
* then check that you've went above the pause
* threshold and then disable the pause.
*/
if (Q->fq_credits < Q->fq_pause_on_thresh) {
if (do_host_pause) {
(void) t1_tpi_write(adapter,
}
alloc_freelQ_buffers(sge, Q);
(Q->fq_credits > Q->fq_pause_off_thresh)) {
(void) t1_tpi_write(adapter,
}
}
#endif /* HOST_PAUSE */
static void
{
if (e->GenerationBit != genbit) {
if (mapping == 0) {
break;
}
/*
* Note that for T1, we've started the beginning of
* of the buffer by an offset of 2 bytes. We thus
* decrement the length to account for this.
*/
wmb();
}
len += sizeof (*e);
ce++;
e++;
credits++;
/*
* sync freelist entries to physical memory up to
* end of the table.
*/
offset = 0;
len = 0;
pidx = 0;
genbit ^= 1;
ce = Q->fq_centries;
e = Q->fq_entries;
}
}
/* sync freelist entries that have been modified. */
if (len)
Q->fq_credits = credits;
}
static void
{
} else {
/* Clear the F_FL_EXHAUSTED interrupts for now */
irq_reg &= ~F_FL_EXHAUSTED;
}
/* We reenable the Qs to force an Freelist GTS interrupt later */
}
/*
* Frees 'credits_pend' TX buffers and returns the credits to Q->credits.
* Free xmit buffers
*/
static void
{
uint32_t i = credits_pend;
#ifdef CONFIG_CHELSIO_T1_OFFLOAD
#endif
while (i--) {
#ifdef CONFIG_CHELSIO_T1_OFFLOAD
/* if flag set, then toe buffer */
case DH_DMA:
}
}
break;
#if defined(__sparc)
case DH_DVMA:
}
}
break;
#endif /* __sparc */
case DH_TOE:
break;
}
#else /* CONFIG_CHELSIO_T1_OFFLOAD */
}
#if defined(__sparc)
else {
}
#endif /* __sparc */
}
}
#endif /* !CONFIG_CHELSIO_T1_OFFLOAD */
ce++;
cidx = 0;
}
}
}
struct sge_intr_counts *
{
}
/*
* Allocates both RX and TX resources and configures the SGE. However,
* the hardware is not enabled yet.
*
* rx_pkt_pad is set, if the hardware supports aligning non-offload traffic.
* jumbo_fl is set to the index of the freelist containing the jumbo buffers.
*/
int
{
/* if we're a T2 card, then we have hardware offset support */
if (alloc_rx_resources(sge, p))
return (-ENOMEM);
if (alloc_tx_resources(sge, p)) {
return (-ENOMEM);
}
configure_sge(sge, p);
/*
* Now that we have sized the free lists calculate the payload
* capacity of the large buffers. Other parts of the driver use
* this to set the max offload coalescing size so that RX packets
* do not overflow our large buffers.
*/
return (0);
}
/*
* Allocates basic RX resources, consisting of memory mapped freelist Qs and a
* response Q.
*/
static int
{
unsigned int size, i;
for (i = 0; i < SGE_FREELQ_N; i++) {
Q->fq_id = i;
Q->fq_genbit = 1;
Q->fq_entries_n = p->freelQ_size[i];
#ifdef HOST_PAUSE
#endif
if (!Q->fq_entries)
goto err_no_mem;
if (!Q->fq_centries)
goto err_no_mem;
}
/*
* Calculate the buffer sizes for the two free lists. FL0 accommodates
* regular sized Ethernet frames, FL1 is sized not to exceed 16K,
* including all the sk_buff overhead.
* For T1C FL0 and FL1 are reversed.
*/
#ifdef NOTYET
sizeof (struct cpl_rx_data) +
#else
else
#endif
#ifdef NOTYET
SKB_DATA_ALIGN(sizeof (struct skb_shared_info));
#else
else
#endif
goto err_no_mem;
return (0);
return (1);
}
/*
* Allocates basic TX resources, consisting of memory mapped command Qs.
*/
static int
{
unsigned int size, i;
for (i = 0; i < SGE_CMDQ_N; i++) {
Q->cq_genbit = 1;
Q->cq_entries_n = p->cmdQ_size[i];
if (!Q->cq_entries)
goto err_no_mem;
if (!Q->cq_centries)
goto err_no_mem;
/* allocate pre-mapped dma headers */
}
return (0);
return (1);
}
/*
* Sets the interrupt latency timer when the adaptive Rx coalescing
* is turned off. Do nothing when it is turned on again.
*
* This routine relies on the fact that the caller has already set
* the adaptive policy in adapter->sge_params before calling it.
*/
int
{
if (!p->coalesce_enable) {
}
return (0);
}
/*
* Programs the various SGE registers. However, the engine is not yet enabled,
* but sge->sge_control is setup and ready to go.
*/
static void
{
int i;
/* The threshold comparison uses <. */
#if 1
/*
* if the the following bit is not set, then we'll get an
* interrupt everytime command Q 0 goes empty. Since we're
* always ringing the doorbell, we can turn it on.
*/
#endif
#if BYTE_ORDER == BIG_ENDIAN
#endif
/*
* Initialize the SGE Interrupt Timer arrray:
* intrtimer[0] = (SGE_INTRTIMER0) usec
* intrtimer[0<i<10] = (SGE_INTRTIMER0 + 2*i) usec
* intrtimer[10] = (SGE_INTRTIMER1) usec
*
*/
}
/* Initialize resource timer */
/* Finally finish initialization of intrtimer[0] */
/* Initialize for a throughput oriented workload */
if (p->coalesce_enable)
else
(void) t1_sge_set_coalesce_params(sge, p);
}
static inline void
int base_reg_hi, int size_reg)
{
}
/*
* Frees RX resources.
*/
static void
{
unsigned int size, i;
}
for (i = 0; i < SGE_FREELQ_N; i++) {
if (Q->fq_centries) {
free_freelQ_buffers(sge, Q);
Q->fq_entries_n * sizeof (freelQ_ce_t));
}
if (Q->fq_entries) {
/* free the freelist queue */
}
}
}
/*
* Frees all RX buffers on the freelist Q. The caller must make sure that
* the SGE is turned off before calling this function.
*/
static void
{
while (credits--) {
/* bump in-use count of receive buffers */
atomic_add(1,
} else {
atomic_add(1,
}
/*
* note. freeb() callback of esb-alloced mblk will
* cause receive buffer to be put back on sa free list.
*/
}
ce++;
cidx = 0;
}
}
Q->fq_credits = credits;
}
/*
* Free TX resources.
*
* Assumes that SGE is stopped and all interrupts are disabled.
*/
static void
{
unsigned int size;
uint32_t i;
for (i = 0; i < SGE_CMDQ_N; i++) {
if (Q->cq_centries) {
atomic_read(Q->cq_credits);
mutex_destroy(&Q->cq_qlock);
if (pending)
}
if (Q->cq_entries) {
}
}
}
/*
* Return the payload capacity of the jumbo free-list buffers.
*/
{
}
/* PR2928 & PR3309 */
void
{
}
/* PR2928 & PR3309 */
{
}
void
{
}
#ifdef SUN_KSTATS
static int
{
int status;
int instance;
int i;
status = -1;
ch_kstat_sz = sizeof (ch_kstat_t);
goto sge_kstat_setup_exit;
for (i = 0; i < MBLK_MAX; i++) {
}
status = 0;
return (status);
}
static void
{
}
static int
{
int i;
if (rw == KSTAT_WRITE) {
for (i = 0; i < MBLK_MAX; i++) {
}
} else {
= statsp->rx_flbuf_fails;
for (i = 0; i < MBLK_MAX; i++) {
}
}
return (0);
}
#endif
static uint16_t
{
int len;
if (sum > 0xffff)
sum -= 0xffff;
do {
if (sum > 0xffff)
sum -= 0xffff;
if (mp)
} while (mp);
return (sum);
}