/*
* Copyright (c) 2008-2016 Solarflare Communications Inc.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
* THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
* PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
* EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
* PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
* WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
* OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
* EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
* The views and conclusions contained in the software and documentation are
* those of the authors and should not be interpreted as representing official
* policies, either expressed or implied, of the FreeBSD Project.
*/
#include <sys/types.h>
#include <sys/sysmacros.h>
#include <sys/ddi.h>
#include <sys/sunddi.h>
#include <sys/atomic.h>
#include <sys/stream.h>
#include <sys/strsun.h>
#include <sys/strsubr.h>
#include <sys/strft.h>
#include <sys/ksynch.h>
#include <sys/ethernet.h>
#include <sys/crc32.h>
#include <sys/pattr.h>
#include <sys/cpu.h>
#include <sys/ethernet.h>
#include <inet/ip.h>
#include <netinet/in.h>
#include <netinet/ip.h>
#include <netinet/tcp.h>
#include "sfxge.h"
#include "efx.h"
/* RXQ flush response timeout (in microseconds) */
#define SFXGE_RX_QFLUSH_USEC (2000000)
/* RXQ flush tries in the case of failure */
#define SFXGE_RX_QFLUSH_TRIES (5)
/* RXQ default packet buffer preallocation (number of packet buffers) */
#define SFXGE_RX_QPREALLOC (0)
/* Receive packet DMA attributes */
static ddi_device_acc_attr_t sfxge_rx_packet_devacc = {
DDI_DEVICE_ATTR_V0, /* devacc_attr_version */
DDI_NEVERSWAP_ACC, /* devacc_attr_endian_flags */
DDI_STRICTORDER_ACC /* devacc_attr_dataorder */
};
static ddi_dma_attr_t sfxge_rx_packet_dma_attr = {
DMA_ATTR_V0, /* dma_attr_version */
0, /* dma_attr_addr_lo */
0xffffffffffffffffull, /* dma_attr_addr_hi */
0xffffffffffffffffull, /* dma_attr_count_max */
SFXGE_CPU_CACHE_SIZE, /* dma_attr_align */
0xffffffff, /* dma_attr_burstsizes */
1, /* dma_attr_minxfer */
0xffffffffffffffffull, /* dma_attr_maxxfer */
0xffffffffffffffffull, /* dma_attr_seg */
1, /* dma_attr_sgllen */
1, /* dma_attr_granular */
0 /* dma_attr_flags */
};
/* Receive queue DMA attributes */
static ddi_device_acc_attr_t sfxge_rxq_devacc = {
DDI_DEVICE_ATTR_V0, /* devacc_attr_version */
DDI_NEVERSWAP_ACC, /* devacc_attr_endian_flags */
DDI_STRICTORDER_ACC /* devacc_attr_dataorder */
};
static ddi_dma_attr_t sfxge_rxq_dma_attr = {
DMA_ATTR_V0, /* dma_attr_version */
0, /* dma_attr_addr_lo */
0xffffffffffffffffull, /* dma_attr_addr_hi */
0xffffffffffffffffull, /* dma_attr_count_max */
EFX_BUF_SIZE, /* dma_attr_align */
0xffffffff, /* dma_attr_burstsizes */
1, /* dma_attr_minxfer */
0xffffffffffffffffull, /* dma_attr_maxxfer */
0xffffffffffffffffull, /* dma_attr_seg */
1, /* dma_attr_sgllen */
1, /* dma_attr_granular */
0 /* dma_attr_flags */
};
/* Forward declaration */
static void sfxge_rx_qpreallocate(sfxge_rxq_t *srp, int nprealloc);
static int
sfxge_rx_packet_ctor(void *buf, void *arg, int kmflags)
{
sfxge_rx_packet_t *srpp = buf;
sfxge_t *sp = arg;
dev_info_t *dip = sp->s_dip;
int err;
ASSERT3U(sizeof (srpp->__srp_u1.__srp_s1), <=,
sizeof (srpp->__srp_u1.__srp_pad));
ASSERT3U(sizeof (srpp->__srp_u2.__srp_s2), <=,
sizeof (srpp->__srp_u2.__srp_pad));
bzero(buf, sizeof (sfxge_rx_packet_t));
/* Allocate a DMA handle */
err = ddi_dma_alloc_handle(dip, &sfxge_rx_packet_dma_attr,
(kmflags == KM_SLEEP) ? DDI_DMA_SLEEP : DDI_DMA_DONTWAIT,
NULL, &(srpp->srp_dma_handle));
if (err != DDI_SUCCESS)
goto fail1;
return (0);
fail1:
DTRACE_PROBE1(fail1, int, err);
SFXGE_OBJ_CHECK(srpp, sfxge_rx_packet_t);
return (-1);
}
static void
sfxge_rx_packet_dtor(void *buf, void *arg)
{
sfxge_rx_packet_t *srpp = buf;
_NOTE(ARGUNUSED(arg))
/* Free the DMA handle */
ddi_dma_free_handle(&(srpp->srp_dma_handle));
srpp->srp_dma_handle = NULL;
SFXGE_OBJ_CHECK(srpp, sfxge_rx_packet_t);
}
static int
sfxge_rx_qctor(void *buf, void *arg, int kmflags)
{
sfxge_rxq_t *srp = buf;
efsys_mem_t *esmp = &(srp->sr_mem);
sfxge_t *sp = arg;
sfxge_dma_buffer_attr_t dma_attr;
sfxge_rx_fpp_t *srfppp;
int nprealloc;
unsigned int id;
int rc;
/* Compile-time structure layout checks */
EFX_STATIC_ASSERT(sizeof (srp->__sr_u1.__sr_s1) <=
sizeof (srp->__sr_u1.__sr_pad));
EFX_STATIC_ASSERT(sizeof (srp->__sr_u2.__sr_s2) <=
sizeof (srp->__sr_u2.__sr_pad));
EFX_STATIC_ASSERT(sizeof (srp->__sr_u3.__sr_s3) <=
sizeof (srp->__sr_u3.__sr_pad));
bzero(buf, sizeof (sfxge_rxq_t));
srp->sr_sp = sp;
dma_attr.sdba_dip = sp->s_dip;
dma_attr.sdba_dattrp = &sfxge_rxq_dma_attr;
dma_attr.sdba_callback = DDI_DMA_SLEEP;
dma_attr.sdba_length = EFX_RXQ_SIZE(sp->s_rxq_size);
dma_attr.sdba_memflags = DDI_DMA_CONSISTENT;
dma_attr.sdba_devaccp = &sfxge_rxq_devacc;
dma_attr.sdba_bindflags = DDI_DMA_READ | DDI_DMA_CONSISTENT;
dma_attr.sdba_maxcookies = 1;
dma_attr.sdba_zeroinit = B_FALSE;
if ((rc = sfxge_dma_buffer_create(esmp, &dma_attr)) != 0)
goto fail1;
/* Allocate some buffer table entries */
if ((rc = sfxge_sram_buf_tbl_alloc(sp, EFX_RXQ_NBUFS(sp->s_rxq_size),
&(srp->sr_id))) != 0)
goto fail2;
/* Allocate the context array */
if ((srp->sr_srpp = kmem_zalloc(sizeof (sfxge_rx_packet_t *) *
sp->s_rxq_size, kmflags)) == NULL) {
rc = ENOMEM;
goto fail3;
}
/* Allocate the flow table */
if ((srp->sr_flow = kmem_zalloc(sizeof (sfxge_rx_flow_t) *
SFXGE_MAX_FLOW, kmflags)) == NULL) {
rc = ENOMEM;
goto fail4;
}
srp->sr_srfpp = &(srp->sr_srfp);
srp->sr_rto = drv_usectohz(200000);
srp->sr_mpp = &(srp->sr_mp);
/* Initialize the free packet pool */
srfppp = &(srp->sr_fpp);
if ((srfppp->srfpp_putp = kmem_zalloc(SFXGE_CPU_CACHE_SIZE *
SFXGE_RX_FPP_NSLOTS, kmflags)) == NULL) {
rc = ENOMEM;
goto fail5;
}
for (id = 0; id < SFXGE_RX_FPP_NSLOTS; id++) {
sfxge_rx_fpp_putlist_t *putp;
size_t off;
off = id * SFXGE_CPU_CACHE_SIZE;
putp = (void *)(srfppp->srfpp_putp + off);
putp->srfpl_putp = NULL;
putp->srfpl_putpp = &(putp->srfpl_putp);
mutex_init(&(putp->srfpl_lock), NULL, MUTEX_DRIVER,
DDI_INTR_PRI(sp->s_intr.si_intr_pri));
}
cv_init(&(srp->sr_flush_kv), NULL, CV_DRIVER, NULL);
/* Preallocate some packets on the free packet pool */
nprealloc = ddi_prop_get_int(DDI_DEV_T_ANY, sp->s_dip,
DDI_PROP_DONTPASS, "rx_prealloc_pkt_buffers", SFXGE_RX_QPREALLOC);
sfxge_rx_qpreallocate(srp, nprealloc);
return (0);
fail5:
DTRACE_PROBE(fail5);
srp->sr_mpp = NULL;
srp->sr_rto = 0;
srp->sr_srfpp = NULL;
/* Free the flow table */
kmem_free(srp->sr_flow, sizeof (sfxge_rx_flow_t) *
SFXGE_MAX_FLOW);
srp->sr_flow = NULL;
fail4:
DTRACE_PROBE(fail4);
/* Free the context array */
kmem_free(srp->sr_srpp, sizeof (sfxge_rx_packet_t *) *
sp->s_rxq_size);
srp->sr_srpp = NULL;
fail3:
DTRACE_PROBE(fail3);
/* Free the buffer table entries */
sfxge_sram_buf_tbl_free(sp, srp->sr_id,
EFX_RXQ_NBUFS(sp->s_rxq_size));
srp->sr_id = 0;
fail2:
DTRACE_PROBE(fail2);
/* Remove dma setup */
sfxge_dma_buffer_destroy(esmp);
fail1:
DTRACE_PROBE1(fail1, int, rc);
srp->sr_sp = NULL;
SFXGE_OBJ_CHECK(srp, sfxge_rxq_t);
return (-1);
}
static void
sfxge_rx_qdtor(void *buf, void *arg)
{
sfxge_rxq_t *srp = buf;
efsys_mem_t *esmp = &(srp->sr_mem);
sfxge_t *sp = srp->sr_sp;
sfxge_rx_fpp_t *srfppp = &(srp->sr_fpp);
unsigned int id;
_NOTE(ARGUNUSED(arg))
cv_destroy(&(srp->sr_flush_kv));
/* Tear down the free packet pool */
for (id = 0; id < SFXGE_RX_FPP_NSLOTS; id++) {
sfxge_rx_fpp_putlist_t *putp;
size_t off;
off = id * SFXGE_CPU_CACHE_SIZE;
putp = (void *)(srfppp->srfpp_putp + off);
putp->srfpl_putpp = NULL;
mutex_destroy(&(putp->srfpl_lock));
SFXGE_OBJ_CHECK(putp, sfxge_rx_fpp_putlist_t);
}
kmem_free(srfppp->srfpp_putp, SFXGE_CPU_CACHE_SIZE *
SFXGE_RX_FPP_NSLOTS);
srfppp->srfpp_putp = NULL;
srp->sr_mpp = NULL;
srp->sr_rto = 0;
srp->sr_srfpp = NULL;
/* Free the flow table */
kmem_free(srp->sr_flow, sizeof (sfxge_rx_flow_t) *
SFXGE_MAX_FLOW);
srp->sr_flow = NULL;
/* Free the context array */
kmem_free(srp->sr_srpp, sizeof (sfxge_rx_packet_t *) *
sp->s_rxq_size);
srp->sr_srpp = NULL;
/* Free the buffer table entries */
sfxge_sram_buf_tbl_free(sp, srp->sr_id,
EFX_RXQ_NBUFS(sp->s_rxq_size));
srp->sr_id = 0;
/* Tear down dma setup */
sfxge_dma_buffer_destroy(esmp);
SFXGE_OBJ_CHECK(srp, sfxge_rxq_t);
}
/* Note: This function takes ownership of *srpp. */
static inline void
sfxge_rx_qfpp_put(sfxge_rxq_t *srp, sfxge_rx_packet_t *srpp)
{
sfxge_rx_fpp_t *srfppp = &(srp->sr_fpp);
mblk_t *mp = srpp->srp_mp;
unsigned int id;
size_t off;
sfxge_rx_fpp_putlist_t *putp;
ASSERT3P(mp->b_next, ==, NULL);
ASSERT3P(mp->b_prev, ==, NULL);
id = CPU->cpu_seqid & SFXGE_RX_FPP_MASK;
off = id * SFXGE_CPU_CACHE_SIZE;
ASSERT3P(srpp->srp_putp, ==, srfppp->srfpp_putp);
putp = (void *)(srpp->srp_putp + off);
mutex_enter(&(putp->srfpl_lock));
putp->srfpl_count++;
*putp->srfpl_putpp = mp;
putp->srfpl_putpp = &(mp->b_next);
mutex_exit(&(putp->srfpl_lock));
}
static unsigned int
sfxge_rx_qfpp_swizzle(sfxge_rxq_t *srp)
{
sfxge_t *sp = srp->sr_sp;
unsigned int index = srp->sr_index;
sfxge_evq_t *sep = sp->s_sep[index];
sfxge_rx_fpp_t *srfppp = &(srp->sr_fpp);
unsigned int start;
unsigned int id;
mblk_t *p;
mblk_t **pp;
unsigned int count;
unsigned int loaned;
ASSERT(mutex_owned(&(sep->se_lock)));
/* We want to access the put list for the current CPU last */
id = start = (CPU->cpu_seqid + 1) & SFXGE_RX_FPP_MASK;
do {
sfxge_rx_fpp_putlist_t *putp;
size_t off;
off = id * SFXGE_CPU_CACHE_SIZE;
id = (id + 1) & SFXGE_RX_FPP_MASK;
putp = (void *)(srfppp->srfpp_putp + off);
/* Acquire the put list */
mutex_enter(&(putp->srfpl_lock));
p = putp->srfpl_putp;
pp = putp->srfpl_putpp;
count = putp->srfpl_count;
putp->srfpl_putp = NULL;
putp->srfpl_putpp = &(putp->srfpl_putp);
putp->srfpl_count = 0;
mutex_exit(&(putp->srfpl_lock));
if (p == NULL)
continue;
/* Add the list to the head of the get list */
*pp = srfppp->srfpp_get;
srfppp->srfpp_get = p;
/* Adjust the counters */
ASSERT3U(srfppp->srfpp_loaned, >=, count);
srfppp->srfpp_loaned -= count;
srfppp->srfpp_count += count;
#if 0
/* NOTE: this probe is disabled because it is expensive!! */
DTRACE_PROBE2(count,
unsigned int, (id - 1) & SFXGE_RX_FPP_MASK,
unsigned int, count);
#endif
} while (id != start);
/* Return the number of packets yet to appear in the put list */
loaned = srfppp->srfpp_loaned;
return (loaned);
}
#define DB_FRTNP(mp) ((mp)->b_datap->db_frtnp)
static void
sfxge_rx_qfpp_empty(sfxge_rxq_t *srp)
{
sfxge_t *sp = srp->sr_sp;
unsigned int index = srp->sr_index;
sfxge_evq_t *sep = sp->s_sep[index];
sfxge_rx_fpp_t *srfppp;
mblk_t *mp;
mutex_enter(&(sep->se_lock));
srfppp = &(srp->sr_fpp);
/* Swizzle put list to get list */
(void) sfxge_rx_qfpp_swizzle(srp);
ASSERT3U(srfppp->srfpp_loaned, ==, 0);
mp = srfppp->srfpp_get;
srfppp->srfpp_get = NULL;
/* Free the remainder */
while (mp != NULL) {
mblk_t *next;
frtn_t *freep;
sfxge_rx_packet_t *srpp;
next = mp->b_next;
mp->b_next = NULL;
ASSERT3U(srfppp->srfpp_count, >, 0);
srfppp->srfpp_count--;
freep = DB_FRTNP(mp);
/*
* ASSERT3P(freep->free_func, ==, sfxge_rx_qpacket_free);
* is implied by srpp test below
*/
/*LINTED*/
srpp = (sfxge_rx_packet_t *)(freep->free_arg);
ASSERT3P(srpp->srp_mp, ==, mp);
ASSERT3P(mp->b_cont, ==, NULL);
srpp->srp_recycle = B_FALSE;
freeb(mp);
mp = next;
}
ASSERT3U(srfppp->srfpp_count, ==, 0);
srfppp->srfpp_min = 0;
mutex_exit(&(sep->se_lock));
}
/*
* This is an estimate of all memory consumed per RX packet
* it can be inaccurate but but sp->s_rx_pkt_mem_alloc mustn't drift
*/
static uint64_t
sfxge_rx_pkt_mem_approx(const sfxge_rx_packet_t *srpp)
{
return (srpp->srp_mblksize + sizeof (mblk_t) + sizeof (dblk_t) +
sizeof (sfxge_rx_packet_t));
}
static void
sfxge_rx_qpacket_destroy(sfxge_rxq_t *srp, sfxge_rx_packet_t *srpp)
{
sfxge_t *sp = srp->sr_sp;
int64_t delta = sfxge_rx_pkt_mem_approx(srpp);
ASSERT(!(srpp->srp_recycle));
ASSERT3P(srpp->srp_mp, ==, NULL);
srpp->srp_off = 0;
srpp->srp_thp = NULL;
srpp->srp_iphp = NULL;
srpp->srp_etherhp = NULL;
srpp->srp_size = 0;
srpp->srp_flags = 0;
bzero(&(srpp->srp_free), sizeof (frtn_t));
srpp->srp_mblksize = 0;
srpp->srp_base = NULL;
/* Unbind the DMA memory from the DMA handle */
srpp->srp_addr = 0;
(void) ddi_dma_unbind_handle(srpp->srp_dma_handle);
/* Free the DMA memory */
srpp->srp_base = NULL;
ddi_dma_mem_free(&(srpp->srp_acc_handle));
srpp->srp_acc_handle = NULL;
srpp->srp_putp = NULL;
srpp->srp_srp = NULL;
kmem_cache_free(sp->s_rpc, srpp);
if (sp->s_rx_pkt_mem_max)
atomic_add_64(&sp->s_rx_pkt_mem_alloc, -delta);
}
static void
sfxge_rx_qpacket_free(void *arg)
{
sfxge_rx_packet_t *srpp = arg;
sfxge_rxq_t *srp = srpp->srp_srp;
/*
* WARNING "man -s 9f esballoc" states:
* => runs sync from the thread calling freeb()
* => must not sleep, or access data structures that could be freed
*/
/* Check whether we want to recycle the receive packets */
if (srpp->srp_recycle) {
frtn_t *freep;
mblk_t *mp;
size_t size;
freep = &(srpp->srp_free);
ASSERT3P(freep->free_func, ==, sfxge_rx_qpacket_free);
ASSERT3P(freep->free_arg, ==, (caddr_t)srpp);
/*
* Allocate a matching mblk_t before the current one is
* freed.
*/
size = srpp->srp_mblksize;
if ((mp = desballoc(srpp->srp_base, size, BPRI_HI,
freep)) != NULL) {
srpp->srp_mp = mp;
/* NORMAL recycled case */
sfxge_rx_qfpp_put(srp, srpp);
return;
}
}
srpp->srp_mp = NULL;
sfxge_rx_qpacket_destroy(srp, srpp);
}
static sfxge_rx_packet_t *
sfxge_rx_qpacket_create(sfxge_rxq_t *srp)
{
sfxge_t *sp = srp->sr_sp;
sfxge_rx_fpp_t *srfppp = &(srp->sr_fpp);
sfxge_rx_packet_t *srpp;
size_t size;
caddr_t base;
size_t unit;
ddi_dma_cookie_t dmac;
unsigned int ncookies;
frtn_t *freep;
mblk_t *mp;
int err;
int rc;
size = sp->s_rx_buffer_size;
if (sp->s_rx_pkt_mem_max &&
(sp->s_rx_pkt_mem_alloc + size >= sp->s_rx_pkt_mem_max)) {
DTRACE_PROBE(rx_pkt_mem_max);
srp->sr_kstat.srk_rx_pkt_mem_limit++;
return (NULL);
}
/* Allocate a new packet */
if ((srpp = kmem_cache_alloc(sp->s_rpc, KM_NOSLEEP)) == NULL) {
srp->sr_kstat.srk_kcache_alloc_nomem++;
rc = ENOMEM;
goto fail1;
}
srpp->srp_srp = srp;
srpp->srp_putp = srfppp->srfpp_putp;
/* Allocate some DMA memory */
err = ddi_dma_mem_alloc(srpp->srp_dma_handle, size,
&sfxge_rx_packet_devacc, DDI_DMA_STREAMING, DDI_DMA_DONTWAIT,
NULL, &base, &unit, &(srpp->srp_acc_handle));
switch (err) {
case DDI_SUCCESS:
break;
case DDI_FAILURE:
srp->sr_kstat.srk_dma_alloc_nomem++;
rc = ENOMEM;
goto fail2;
default:
srp->sr_kstat.srk_dma_alloc_fail++;
rc = EFAULT;
goto fail2;
}
/* Adjust the buffer to align the start of the DMA area correctly */
base += sp->s_rx_buffer_align;
size -= sp->s_rx_buffer_align;
/* Bind the DMA memory to the DMA handle */
err = ddi_dma_addr_bind_handle(srpp->srp_dma_handle, NULL,
base, size, DDI_DMA_READ | DDI_DMA_STREAMING,
DDI_DMA_DONTWAIT, NULL, &dmac, &ncookies);
switch (err) {
case DDI_DMA_MAPPED:
break;
case DDI_DMA_INUSE:
srp->sr_kstat.srk_dma_bind_fail++;
rc = EEXIST;
goto fail3;
case DDI_DMA_NORESOURCES:
srp->sr_kstat.srk_dma_bind_nomem++;
rc = ENOMEM;
goto fail3;
case DDI_DMA_NOMAPPING:
srp->sr_kstat.srk_dma_bind_fail++;
rc = ENOTSUP;
goto fail3;
case DDI_DMA_TOOBIG:
srp->sr_kstat.srk_dma_bind_fail++;
rc = EFBIG;
goto fail3;
default:
srp->sr_kstat.srk_dma_bind_fail++;
rc = EFAULT;
goto fail3;
}
ASSERT3U(ncookies, ==, 1);
srpp->srp_addr = dmac.dmac_laddress;
srpp->srp_base = (unsigned char *)base;
srpp->srp_mblksize = size;
/*
* Allocate a STREAMS block: We use size 1 so that the allocator will
* use the first (and smallest) dblk cache.
*/
freep = &(srpp->srp_free);
freep->free_func = sfxge_rx_qpacket_free;
freep->free_arg = (caddr_t)srpp;
if ((mp = desballoc(srpp->srp_base, size, BPRI_HI, freep)) == NULL) {
srp->sr_kstat.srk_desballoc_fail++;
rc = ENOMEM;
goto fail4;
}
srpp->srp_mp = mp;
srpp->srp_recycle = B_TRUE;
if (sp->s_rx_pkt_mem_max) {
int64_t delta = sfxge_rx_pkt_mem_approx(srpp);
atomic_add_64(&sp->s_rx_pkt_mem_alloc, delta);
}
return (srpp);
fail4:
DTRACE_PROBE(fail4);
bzero(&(srpp->srp_free), sizeof (frtn_t));
srpp->srp_mblksize = 0;
srpp->srp_base = NULL;
/* Unbind the DMA memory from the DMA handle */
srpp->srp_addr = 0;
(void) ddi_dma_unbind_handle(srpp->srp_dma_handle);
fail3:
DTRACE_PROBE(fail3);
/* Free the DMA memory */
ddi_dma_mem_free(&(srpp->srp_acc_handle));
srpp->srp_acc_handle = NULL;
fail2:
DTRACE_PROBE(fail2);
srpp->srp_putp = NULL;
srpp->srp_srp = NULL;
kmem_cache_free(sp->s_rpc, srpp);
fail1:
DTRACE_PROBE1(fail1, int, rc);
return (NULL);
}
#define SFXGE_REFILL_BATCH 64
/* Try to refill the RX descriptor ring from the associated free pkt pool */
static void
sfxge_rx_qrefill(sfxge_rxq_t *srp, unsigned int target)
{
sfxge_t *sp = srp->sr_sp;
sfxge_rx_fpp_t *srfppp = &(srp->sr_fpp);
unsigned int index = srp->sr_index;
sfxge_evq_t *sep = sp->s_sep[index];
efsys_dma_addr_t addr[SFXGE_REFILL_BATCH];
mblk_t *mp;
int ntodo;
unsigned int count;
unsigned int batch;
unsigned int rxfill;
unsigned int mblksize;
prefetch_read_many(sp->s_enp);
prefetch_read_many(srp->sr_erp);
ASSERT(mutex_owned(&(sep->se_lock)));
if (srp->sr_state != SFXGE_RXQ_STARTED)
return;
rxfill = srp->sr_added - srp->sr_completed;
ASSERT3U(rxfill, <=, EFX_RXQ_LIMIT(sp->s_rxq_size));
ntodo = min(EFX_RXQ_LIMIT(sp->s_rxq_size) - rxfill, target);
ASSERT3U(ntodo, <=, EFX_RXQ_LIMIT(sp->s_rxq_size));
if (ntodo == 0)
goto out;
(void) sfxge_rx_qfpp_swizzle(srp);
mp = srfppp->srfpp_get;
count = srfppp->srfpp_count;
mblksize = sp->s_rx_buffer_size - sp->s_rx_buffer_align;
batch = 0;
while (ntodo-- > 0) {
mblk_t *next;
frtn_t *freep;
sfxge_rx_packet_t *srpp;
unsigned int id;
if (mp == NULL)
break;
next = mp->b_next;
mp->b_next = NULL;
if (next != NULL)
prefetch_read_many(next);
freep = DB_FRTNP(mp);
/*LINTED*/
srpp = (sfxge_rx_packet_t *)(freep->free_arg);
ASSERT3P(srpp->srp_mp, ==, mp);
/* The MTU may have changed since the packet was allocated */
if (MBLKSIZE(mp) != mblksize) {
srpp->srp_recycle = B_FALSE;
freeb(mp);
--count;
mp = next;
continue;
}
srpp->srp_off = 0;
srpp->srp_thp = NULL;
srpp->srp_iphp = NULL;
srpp->srp_etherhp = NULL;
srpp->srp_size = 0;
srpp->srp_flags = EFX_DISCARD;
id = (srp->sr_added + batch) & (sp->s_rxq_size - 1);
ASSERT(srp->sr_srpp[id] == NULL);
srp->sr_srpp[id] = srpp;
addr[batch++] = srpp->srp_addr;
if (batch == SFXGE_REFILL_BATCH) {
efx_rx_qpost(srp->sr_erp, addr, mblksize, batch,
srp->sr_completed, srp->sr_added);
srp->sr_added += batch;
batch = 0;
}
--count;
mp = next;
}
srfppp->srfpp_get = mp;
srfppp->srfpp_count = count;
if (batch != 0) {
efx_rx_qpost(srp->sr_erp, addr, mblksize, batch,
srp->sr_completed, srp->sr_added);
srp->sr_added += batch;
}
efx_rx_qpush(srp->sr_erp, srp->sr_added, &srp->sr_pushed);
out:
if (srfppp->srfpp_count < srfppp->srfpp_min)
srfppp->srfpp_min = srfppp->srfpp_count;
}
/* Preallocate packets and put them in the free packet pool */
static void
sfxge_rx_qpreallocate(sfxge_rxq_t *srp, int nprealloc)
{
sfxge_rx_fpp_t *srfppp = &((srp)->sr_fpp);
srfppp->srfpp_lowat = nprealloc;
while (nprealloc-- > 0) {
sfxge_rx_packet_t *srpp;
if ((srpp = sfxge_rx_qpacket_create(srp)) == NULL)
break;
sfxge_rx_qfpp_put(srp, srpp);
}
}
/* Try to refill the RX descriptor ring by allocating new packets */
static void
sfxge_rx_qfill(sfxge_rxq_t *srp, unsigned int target)
{
sfxge_t *sp = srp->sr_sp;
unsigned int index = srp->sr_index;
sfxge_evq_t *sep = sp->s_sep[index];
unsigned int batch;
unsigned int rxfill;
unsigned int mblksize;
int ntodo;
efsys_dma_addr_t addr[SFXGE_REFILL_BATCH];
mblk_t *mp = NULL;
prefetch_read_many(sp->s_enp);
prefetch_read_many(srp->sr_erp);
ASSERT(mutex_owned(&(sep->se_lock)));
if (srp->sr_state != SFXGE_RXQ_STARTED)
return;
rxfill = srp->sr_added - srp->sr_completed;
ASSERT3U(rxfill, <=, EFX_RXQ_LIMIT(sp->s_rxq_size));
ntodo = min(EFX_RXQ_LIMIT(sp->s_rxq_size) - rxfill, target);
ASSERT3U(ntodo, <=, EFX_RXQ_LIMIT(sp->s_rxq_size));
if (ntodo == 0)
return;
mblksize = sp->s_rx_buffer_size - sp->s_rx_buffer_align;
batch = 0;
while (ntodo-- > 0) {
sfxge_rx_packet_t *srpp;
unsigned int id;
if ((srpp = sfxge_rx_qpacket_create(srp)) == NULL)
break;
mp = srpp->srp_mp;
ASSERT3U(MBLKSIZE(mp), ==, mblksize);
ASSERT3U(srpp->srp_off, ==, 0);
ASSERT3P(srpp->srp_thp, ==, NULL);
ASSERT3P(srpp->srp_iphp, ==, NULL);
ASSERT3P(srpp->srp_etherhp, ==, NULL);
ASSERT3U(srpp->srp_size, ==, 0);
srpp->srp_flags = EFX_DISCARD;
id = (srp->sr_added + batch) & (sp->s_rxq_size - 1);
ASSERT(srp->sr_srpp[id] == NULL);
srp->sr_srpp[id] = srpp;
addr[batch++] = srpp->srp_addr;
if (batch == SFXGE_REFILL_BATCH) {
efx_rx_qpost(srp->sr_erp, addr, mblksize, batch,
srp->sr_completed, srp->sr_added);
srp->sr_added += batch;
batch = 0;
}
}
if (batch != 0) {
efx_rx_qpost(srp->sr_erp, addr, mblksize, batch,
srp->sr_completed, srp->sr_added);
srp->sr_added += batch;
}
efx_rx_qpush(srp->sr_erp, srp->sr_added, &srp->sr_pushed);
}
void
sfxge_rx_qfpp_trim(sfxge_rxq_t *srp)
{
sfxge_rx_fpp_t *srfppp = &(srp->sr_fpp);
sfxge_t *sp = srp->sr_sp;
unsigned int index = srp->sr_index;
sfxge_evq_t *sep = sp->s_sep[index];
mblk_t *p;
mblk_t **pp;
int count;
ASSERT(mutex_owned(&(sep->se_lock)));
if (srp->sr_state != SFXGE_RXQ_STARTED)
goto done;
/* Make sure the queue is full */
sfxge_rx_qrefill(srp, EFX_RXQ_LIMIT(sp->s_rxq_size));
/* The refill may have emptied the pool */
if (srfppp->srfpp_min == 0)
goto done;
/* Don't trim below the pool's low water mark */
if (srfppp->srfpp_count <= srfppp->srfpp_lowat)
goto done;
ASSERT(srfppp->srfpp_min <= srfppp->srfpp_count);
/* Trim to the largest of srfppp->srfpp_min and srfpp->srfpp_lowat */
if (srfppp->srfpp_lowat > srfppp->srfpp_min)
count = srfppp->srfpp_count - srfppp->srfpp_lowat;
else
count = srfppp->srfpp_count - srfppp->srfpp_min;
/* Walk the get list */
pp = &(srfppp->srfpp_get);
while (--count >= 0) {
ASSERT(pp);
p = *pp;
ASSERT(p != NULL);
pp = &(p->b_next);
}
ASSERT(pp);
p = *pp;
/* Truncate the get list */
*pp = NULL;
/* Free the remainder */
while (p != NULL) {
mblk_t *next;
frtn_t *freep;
sfxge_rx_packet_t *srpp;
next = p->b_next;
p->b_next = NULL;
ASSERT3U(srfppp->srfpp_min, >, 0);
srfppp->srfpp_min--;
srfppp->srfpp_count--;
freep = DB_FRTNP(p);
/*LINTED*/
srpp = (sfxge_rx_packet_t *)(freep->free_arg);
ASSERT3P(srpp->srp_mp, ==, p);
srpp->srp_recycle = B_FALSE;
freeb(p);
p = next;
}
done:
srfppp->srfpp_min = srfppp->srfpp_count;
}
static void
sfxge_rx_qpoll(void *arg)
{
sfxge_rxq_t *srp = arg;
sfxge_t *sp = srp->sr_sp;
unsigned int index = srp->sr_index;
sfxge_evq_t *sep = sp->s_sep[index];
uint16_t magic;
/*
* man timeout(9f) states that this code should adhere to the
* same requirements as a softirq handler - DO NOT BLOCK
*/
/*
* Post an event to the event queue to cause the free packet pool to be
* trimmed if it is oversize.
*/
magic = SFXGE_MAGIC_RX_QFPP_TRIM | index;
#if defined(DEBUG)
/* This is guaranteed due to the start/stop order of rx and ev */
ASSERT3U(sep->se_state, ==, SFXGE_EVQ_STARTED);
ASSERT3U(srp->sr_state, ==, SFXGE_RXQ_STARTED);
#else
/*
* Bug22691 WORKAROUND:
* This handler has been observed in the field to be invoked for a
* queue in the INITIALIZED state, which should never happen.
* Until the mechanism for this is properly understood, add defensive
* checks.
*/
if ((sep->se_state != SFXGE_EVQ_STARTED) ||
(srp->sr_state != SFXGE_RXQ_STARTED) ||
(!sep->se_eep)) {
dev_err(sp->s_dip, CE_WARN, SFXGE_CMN_ERR
"RXQ[%d] bad state in sfxge_rx_qpoll %d %d %p",
index, sep->se_state, srp->sr_state, sep->se_eep);
return;
}
#endif
efx_ev_qpost(sep->se_eep, magic);
srp->sr_tid = timeout(sfxge_rx_qpoll, srp,
drv_usectohz(sp->s_rxq_poll_usec));
}
static void
sfxge_rx_qpoll_start(sfxge_rxq_t *srp)
{
sfxge_t *sp = srp->sr_sp;
unsigned int index = srp->sr_index;
sfxge_evq_t *sep = sp->s_sep[index];
ASSERT(mutex_owned(&(sep->se_lock)));
ASSERT3U(srp->sr_state, ==, SFXGE_RXQ_STARTED);
/* Schedule a poll */
ASSERT3P(srp->sr_tid, ==, 0);
srp->sr_tid = timeout(sfxge_rx_qpoll, srp, 0);
}
static void
sfxge_rx_qpoll_stop(sfxge_rxq_t *srp)
{
sfxge_t *sp = srp->sr_sp;
unsigned int index = srp->sr_index;
sfxge_evq_t *sep = sp->s_sep[index];
timeout_id_t tid;
ASSERT(mutex_owned(&(sep->se_lock)));
ASSERT3U(srp->sr_state, ==, SFXGE_RXQ_STARTED);
/*
* Cancel the qpoll timer. Care is needed as this function
* can race with sfxge_rx_qpoll() for timeout id updates.
*
* Do not hold locks used by any timeout(9f) handlers across
* calls to untimeout(9f) as this will deadlock.
*/
tid = 0;
while ((srp->sr_tid != 0) && (srp->sr_tid != tid)) {
tid = srp->sr_tid;
(void) untimeout(tid);
}
srp->sr_tid = 0;
}
static int
sfxge_rx_kstat_update(kstat_t *ksp, int rw)
{
sfxge_rxq_t *srp = ksp->ks_private;
sfxge_t *sp = srp->sr_sp;
unsigned int index = srp->sr_index;
sfxge_evq_t *sep = sp->s_sep[index];
kstat_named_t *knp;
int rc;
if (rw != KSTAT_READ) {
rc = EACCES;
goto fail1;
}
ASSERT(mutex_owned(&(sep->se_lock)));
if (srp->sr_state != SFXGE_RXQ_STARTED)
goto done;
knp = ksp->ks_data;
/* NB pointer post-increment below */
knp++->value.ui32 = srp->sr_kstat.srk_rx_pkt_mem_limit;
knp++->value.ui32 = srp->sr_kstat.srk_kcache_alloc_nomem;
knp++->value.ui32 = srp->sr_kstat.srk_dma_alloc_nomem;
knp++->value.ui32 = srp->sr_kstat.srk_dma_alloc_fail;
knp++->value.ui32 = srp->sr_kstat.srk_dma_bind_nomem;
knp++->value.ui32 = srp->sr_kstat.srk_dma_bind_fail;
knp++->value.ui32 = srp->sr_kstat.srk_desballoc_fail;
knp++->value.ui32 = srp->sr_kstat.srk_rxq_empty_discard;
done:
return (0);
fail1:
DTRACE_PROBE1(fail1, int, rc);
return (rc);
}
static int
sfxge_rx_kstat_init(sfxge_rxq_t *srp)
{
sfxge_t *sp = srp->sr_sp;
unsigned int index = srp->sr_index;
sfxge_evq_t *sep = sp->s_sep[index];
dev_info_t *dip = sp->s_dip;
char name[MAXNAMELEN];
kstat_t *ksp;
kstat_named_t *knp;
int rc;
/* Create the set */
(void) snprintf(name, MAXNAMELEN - 1, "%s_rxq%04d",
ddi_driver_name(dip), index);
if ((ksp = kstat_create((char *)ddi_driver_name(dip),
ddi_get_instance(dip), name, "rxq", KSTAT_TYPE_NAMED,
SFXGE_RX_NSTATS, 0)) == NULL) {
rc = ENOMEM;
goto fail1;
}
srp->sr_ksp = ksp;
ksp->ks_update = sfxge_rx_kstat_update;
ksp->ks_private = srp;
ksp->ks_lock = &(sep->se_lock);
/* Initialise the named stats */
knp = ksp->ks_data;
kstat_named_init(knp, "rx_pkt_mem_limit", KSTAT_DATA_UINT32);
knp++;
kstat_named_init(knp, "kcache_alloc_nomem", KSTAT_DATA_UINT32);
knp++;
kstat_named_init(knp, "dma_alloc_nomem", KSTAT_DATA_UINT32);
knp++;
kstat_named_init(knp, "dma_alloc_fail", KSTAT_DATA_UINT32);
knp++;
kstat_named_init(knp, "dma_bind_nomem", KSTAT_DATA_UINT32);
knp++;
kstat_named_init(knp, "dma_bind_fail", KSTAT_DATA_UINT32);
knp++;
kstat_named_init(knp, "desballoc_fail", KSTAT_DATA_UINT32);
knp++;
kstat_named_init(knp, "rxq_empty_discard", KSTAT_DATA_UINT32);
kstat_install(ksp);
return (0);
fail1:
DTRACE_PROBE1(fail1, int, rc);
return (rc);
}
static int
sfxge_rx_qinit(sfxge_t *sp, unsigned int index)
{
sfxge_rxq_t *srp;
int rc;
ASSERT3U(index, <, SFXGE_RX_SCALE_MAX);
if ((srp = kmem_cache_alloc(sp->s_rqc, KM_SLEEP)) == NULL) {
rc = ENOMEM;
goto fail1;
}
ASSERT3U(srp->sr_state, ==, SFXGE_RXQ_UNINITIALIZED);
srp->sr_index = index;
sp->s_srp[index] = srp;
if ((rc = sfxge_rx_kstat_init(srp)) != 0)
goto fail2;
srp->sr_state = SFXGE_RXQ_INITIALIZED;
return (0);
fail2:
DTRACE_PROBE(fail2);
kmem_cache_free(sp->s_rqc, srp);
fail1:
DTRACE_PROBE1(fail1, int, rc);
return (rc);
}
static int
sfxge_rx_qstart(sfxge_t *sp, unsigned int index)
{
sfxge_evq_t *sep = sp->s_sep[index];
sfxge_rxq_t *srp;
efsys_mem_t *esmp;
efx_nic_t *enp;
unsigned int level;
int rc;
mutex_enter(&(sep->se_lock));
srp = sp->s_srp[index];
enp = sp->s_enp;
esmp = &(srp->sr_mem);
ASSERT3U(srp->sr_state, ==, SFXGE_RXQ_INITIALIZED);
ASSERT3U(sep->se_state, ==, SFXGE_EVQ_STARTED);
/* Zero the memory */
bzero(esmp->esm_base, EFX_RXQ_SIZE(sp->s_rxq_size));
/* Program the buffer table */
if ((rc = sfxge_sram_buf_tbl_set(sp, srp->sr_id, esmp,
EFX_RXQ_NBUFS(sp->s_rxq_size))) != 0)
goto fail1;
/* Create the receive queue */
if ((rc = efx_rx_qcreate(enp, index, index, EFX_RXQ_TYPE_DEFAULT,
esmp, sp->s_rxq_size, srp->sr_id, sep->se_eep, &(srp->sr_erp)))
!= 0)
goto fail2;
/* Enable the receive queue */
efx_rx_qenable(srp->sr_erp);
/* Set the water marks */
srp->sr_hiwat = EFX_RXQ_LIMIT(sp->s_rxq_size) * 9 / 10;
srp->sr_lowat = srp->sr_hiwat / 2;
srp->sr_state = SFXGE_RXQ_STARTED;
srp->sr_flush = SFXGE_FLUSH_INACTIVE;
sfxge_rx_qpoll_start(srp);
/* Try to fill the queue from the pool */
sfxge_rx_qrefill(srp, EFX_RXQ_LIMIT(sp->s_rxq_size));
/*
* If there were insufficient buffers in the pool to reach the at
* least a batch then allocate some.
*/
level = srp->sr_added - srp->sr_completed;
if (level < SFXGE_RX_BATCH)
sfxge_rx_qfill(srp, SFXGE_RX_BATCH);
mutex_exit(&(sep->se_lock));
return (0);
fail2:
DTRACE_PROBE(fail2);
/* Clear entries from the buffer table */
sfxge_sram_buf_tbl_clear(sp, srp->sr_id,
EFX_RXQ_NBUFS(sp->s_rxq_size));
fail1:
DTRACE_PROBE1(fail1, int, rc);
mutex_exit(&(sep->se_lock));
return (rc);
}
static void
sfxge_rx_qflow_complete(sfxge_rxq_t *srp, sfxge_rx_flow_t *srfp)
{
mblk_t *mp;
struct ether_header *etherhp;
struct ip *iphp;
struct tcphdr *thp;
if (srfp->srf_mp == NULL)
return;
mp = srfp->srf_mp;
etherhp = srfp->srf_etherhp;
iphp = srfp->srf_iphp;
thp = srfp->srf_last_thp;
ASSERT3U(((etherhp->ether_type == htons(ETHERTYPE_VLAN)) ?
sizeof (struct ether_vlan_header) :
sizeof (struct ether_header)) +
srfp->srf_len, ==, msgdsize(mp));
ASSERT3U(srfp->srf_len & 0xffff, ==, srfp->srf_len);
iphp->ip_len = htons(srfp->srf_len);
srfp->srf_first_thp->th_ack = thp->th_ack;
srfp->srf_first_thp->th_win = thp->th_win;
srfp->srf_first_thp->th_flags = thp->th_flags;
DTRACE_PROBE2(flow_complete, uint32_t, srfp->srf_tag,
size_t, srfp->srf_len);
srfp->srf_mp = NULL;
srfp->srf_len = 0;
ASSERT(mp->b_next == NULL);
*(srp->sr_mpp) = mp;
srp->sr_mpp = &(mp->b_next);
}
static boolean_t
sfxge_rx_qflow_add(sfxge_rxq_t *srp, sfxge_rx_flow_t *srfp,
sfxge_rx_packet_t *srpp, clock_t now)
{
sfxge_t *sp = srp->sr_sp;
struct ether_header *etherhp = srpp->srp_etherhp;
struct ip *iphp = srpp->srp_iphp;
struct tcphdr *thp = srpp->srp_thp;
size_t off = srpp->srp_off;
size_t size = (size_t)(srpp->srp_size);
mblk_t *mp = srpp->srp_mp;
uint32_t seq;
unsigned int shift;
ASSERT3U(MBLKL(mp), ==, off + size);
ASSERT3U(DB_CKSUMFLAGS(mp), ==,
HCK_FULLCKSUM | HCK_FULLCKSUM_OK | HCK_IPV4_HDRCKSUM);
seq = htonl(thp->th_seq);
/*
* If the time between this segment and the last is greater than RTO
* then consider this a new flow.
*/
if (now - srfp->srf_lbolt > srp->sr_rto) {
srfp->srf_count = 1;
srfp->srf_seq = seq + size;
goto fail1;
}
if (seq != srfp->srf_seq) {
if (srfp->srf_count > SFXGE_SLOW_START)
srfp->srf_count = SFXGE_SLOW_START;
srfp->srf_count >>= 1;
srfp->srf_count++;
srfp->srf_seq = seq + size;
goto fail2;
}
/* Update the in-order segment count and sequence number */
srfp->srf_count++;
srfp->srf_seq = seq + size;
/* Don't merge across pure ACK, URG, SYN or RST segments */
if (size == 0 || thp->th_flags & (TH_URG | TH_SYN | TH_RST) ||
thp->th_urp != 0)
goto fail3;
/*
* If the in-order segment count has not yet reached the slow-start
* threshold then we cannot coalesce.
*/
if (srfp->srf_count < SFXGE_SLOW_START)
goto fail4;
/* Scale up the packet size from 4k (the maximum being 64k) */
ASSERT3U(srfp->srf_count, >=, SFXGE_SLOW_START);
shift = MIN(srfp->srf_count - SFXGE_SLOW_START + 12, 16);
if (srfp->srf_len + size >= (1 << shift))
sfxge_rx_qflow_complete(srp, srfp);
ASSERT(mp->b_cont == NULL);
if (srfp->srf_mp == NULL) {
/* First packet in this flow */
srfp->srf_etherhp = etherhp;
srfp->srf_iphp = iphp;
srfp->srf_first_thp = srfp->srf_last_thp = thp;
ASSERT3P(mp->b_cont, ==, NULL);
srfp->srf_mp = mp;
srfp->srf_mpp = &(mp->b_cont);
srfp->srf_len = ntohs(iphp->ip_len);
/*
* If the flow is not already in the list of occupied flows then
* add it.
*/
if (srfp->srf_next == NULL &&
srp->sr_srfpp != &(srfp->srf_next)) {
*(srp->sr_srfpp) = srfp;
srp->sr_srfpp = &(srfp->srf_next);
}
} else {
/* Later packet in this flow - skip TCP header */
srfp->srf_last_thp = thp;
mp->b_rptr += off;
ASSERT3U(MBLKL(mp), ==, size);
ASSERT3P(mp->b_cont, ==, NULL);
*(srfp->srf_mpp) = mp;
srfp->srf_mpp = &(mp->b_cont);
srfp->srf_len += size;
ASSERT(srfp->srf_next != NULL ||
srp->sr_srfpp == &(srfp->srf_next));
}
DTRACE_PROBE2(flow_add, uint32_t, srfp->srf_tag, size_t, size);
/*
* Try to align coalesced segments on push boundaries, unless they
* are too frequent.
*/
if (sp->s_rx_coalesce_mode == SFXGE_RX_COALESCE_ALLOW_PUSH &&
thp->th_flags & TH_PUSH)
sfxge_rx_qflow_complete(srp, srfp);
srfp->srf_lbolt = now;
return (B_TRUE);
fail4:
fail3:
fail2:
fail1:
sfxge_rx_qflow_complete(srp, srfp);
srfp->srf_lbolt = now;
return (B_FALSE);
}
void
sfxge_rx_qpacket_coalesce(sfxge_rxq_t *srp)
{
sfxge_t *sp = srp->sr_sp;
clock_t now;
mblk_t *mp;
sfxge_rx_flow_t *srfp;
ASSERT(sp->s_rx_coalesce_mode != SFXGE_RX_COALESCE_OFF);
now = ddi_get_lbolt();
mp = srp->sr_mp;
srp->sr_mp = NULL;
srp->sr_mpp = &(srp->sr_mp);
/* Start with the last flow to be appended to */
srfp = *(srp->sr_srfpp);
while (mp != NULL) {
frtn_t *freep;
sfxge_rx_packet_t *srpp;
struct ether_header *etherhp;
struct ip *iphp;
struct tcphdr *thp;
size_t off;
size_t size;
uint16_t ether_tci;
uint32_t hash;
uint32_t tag;
mblk_t *next;
sfxge_packet_type_t pkt_type;
uint16_t sport, dport;
next = mp->b_next;
mp->b_next = NULL;
if (next != NULL)
prefetch_read_many(next);
freep = DB_FRTNP(mp);
/*LINTED*/
srpp = (sfxge_rx_packet_t *)(freep->free_arg);
ASSERT3P(srpp->srp_mp, ==, mp);
/* If the packet is not TCP then we cannot coalesce it */
if (~(srpp->srp_flags) & EFX_PKT_TCP)
goto reject;
/*
* If the packet is not fully checksummed then we cannot
* coalesce it.
*/
if (~(srpp->srp_flags) & (EFX_CKSUM_TCPUDP | EFX_CKSUM_IPV4))
goto reject;
/* Parse the TCP header */
pkt_type = sfxge_pkthdr_parse(mp, &etherhp, &iphp, &thp, &off,
&size, &sport, &dport);
ASSERT(pkt_type == SFXGE_PACKET_TYPE_IPV4_TCP);
ASSERT(etherhp != NULL);
ASSERT(iphp != NULL);
ASSERT(thp != NULL);
ASSERT(off != 0);
if ((iphp->ip_off & ~htons(IP_DF)) != 0)
goto reject;
if (etherhp->ether_type == htons(ETHERTYPE_VLAN)) {
struct ether_vlan_header *ethervhp;
ethervhp = (struct ether_vlan_header *)etherhp;
ether_tci = ethervhp->ether_tci;
} else {
ether_tci = 0;
}
/*
* Make sure any minimum length padding is stripped
* before we try to add the packet to a flow.
*/
ASSERT3U(sp->s_rx_prefix_size + MBLKL(mp), ==,
(size_t)(srpp->srp_size));
ASSERT3U(sp->s_rx_prefix_size + off + size, <=,
(size_t)(srpp->srp_size));
if (sp->s_rx_prefix_size + off + size <
(size_t)(srpp->srp_size))
mp->b_wptr = mp->b_rptr + off + size;
/*
* If there is no current flow, or the segment does not match
* the current flow then we must attempt to look up the
* correct flow in the table.
*/
if (srfp == NULL)
goto lookup;
if (srfp->srf_saddr != iphp->ip_src.s_addr ||
srfp->srf_daddr != iphp->ip_dst.s_addr)
goto lookup;
if (srfp->srf_sport != thp->th_sport ||
srfp->srf_dport != thp->th_dport)
goto lookup;
if (srfp->srf_tci != ether_tci)
goto lookup;
add:
ASSERT(srfp != NULL);
srpp->srp_etherhp = etherhp;
srpp->srp_iphp = iphp;
srpp->srp_thp = thp;
srpp->srp_off = off;
ASSERT3U(size, <, (1 << 16));
srpp->srp_size = (uint16_t)size;
/* Try to append the packet to the flow */
if (!sfxge_rx_qflow_add(srp, srfp, srpp, now))
goto reject;
mp = next;
continue;
lookup:
/*
* If there is a prefix area then read the hash from that,
* otherwise calculate it.
*/
if (sp->s_rx_prefix_size != 0) {
hash = efx_psuedo_hdr_hash_get(sp->s_enp,
EFX_RX_HASHALG_TOEPLITZ,
DB_BASE(mp));
} else {
SFXGE_TCP_HASH(sp,
&iphp->ip_src.s_addr,
thp->th_sport,
&iphp->ip_dst.s_addr,
thp->th_dport,
hash);
}
srfp = &(srp->sr_flow[(hash >> 6) % SFXGE_MAX_FLOW]);
tag = hash + 1; /* Make sure it's not zero */
/*
* If the flow we have found does not match the hash then
* it may be an unused flow, or it may be stale.
*/
if (tag != srfp->srf_tag) {
if (srfp->srf_count != 0) {
if (now - srfp->srf_lbolt <= srp->sr_rto)
goto reject;
}
if (srfp->srf_mp != NULL)
goto reject;
/* Start a new flow */
ASSERT(srfp->srf_next == NULL);
srfp->srf_tag = tag;
srfp->srf_saddr = iphp->ip_src.s_addr;
srfp->srf_daddr = iphp->ip_dst.s_addr;
srfp->srf_sport = thp->th_sport;
srfp->srf_dport = thp->th_dport;
srfp->srf_tci = ether_tci;
srfp->srf_count = 0;
srfp->srf_seq = ntohl(thp->th_seq);
srfp->srf_lbolt = now;
goto add;
}
/*
* If the flow we have found does match the hash then it could
* still be an alias.
*/
if (srfp->srf_saddr != iphp->ip_src.s_addr ||
srfp->srf_daddr != iphp->ip_dst.s_addr)
goto reject;
if (srfp->srf_sport != thp->th_sport ||
srfp->srf_dport != thp->th_dport)
goto reject;
if (srfp->srf_tci != ether_tci)
goto reject;
goto add;
reject:
*(srp->sr_mpp) = mp;
srp->sr_mpp = &(mp->b_next);
mp = next;
}
}
void
sfxge_rx_qcomplete(sfxge_rxq_t *srp, boolean_t eop)
{
sfxge_t *sp = srp->sr_sp;
unsigned int index = srp->sr_index;
sfxge_evq_t *sep = sp->s_sep[index];
unsigned int completed;
sfxge_rx_fpp_t *srfppp = &(srp->sr_fpp);
unsigned int level;
ASSERT(mutex_owned(&(sep->se_lock)));
ASSERT(srp->sr_mp == NULL);
ASSERT(srp->sr_mpp == &(srp->sr_mp));
completed = srp->sr_completed;
while (completed != srp->sr_pending) {
unsigned int id;
sfxge_rx_packet_t *srpp;
mblk_t *mp;
size_t size;
uint16_t flags;
int rc;
id = completed++ & (sp->s_rxq_size - 1);
if (srp->sr_pending - completed >= 4) {
unsigned int prefetch;
prefetch = (id + 4) & (sp->s_rxq_size - 1);
srpp = srp->sr_srpp[prefetch];
ASSERT(srpp != NULL);
mp = srpp->srp_mp;
prefetch_read_many(mp->b_datap);
} else if (completed == srp->sr_pending) {
prefetch_read_many(srp->sr_mp);
}
srpp = srp->sr_srpp[id];
ASSERT(srpp != NULL);
srp->sr_srpp[id] = NULL;
mp = srpp->srp_mp;
ASSERT(mp->b_cont == NULL);
/* when called from sfxge_rx_qstop() */
if (srp->sr_state != SFXGE_RXQ_STARTED)
goto discard;
if (srpp->srp_flags & (EFX_ADDR_MISMATCH | EFX_DISCARD))
goto discard;
/* Make the data visible to the kernel */
rc = ddi_dma_sync(srpp->srp_dma_handle, 0,
sp->s_rx_buffer_size, DDI_DMA_SYNC_FORKERNEL);
ASSERT3P(rc, ==, DDI_SUCCESS);
/* Read the length from the psuedo header if required */
if (srpp->srp_flags & EFX_PKT_PREFIX_LEN) {
rc = efx_psuedo_hdr_pkt_length_get(sp->s_enp,
mp->b_rptr,
&srpp->srp_size);
ASSERT3P(rc, ==, 0);
srpp->srp_size += sp->s_rx_prefix_size;
}
/* Set up the packet length */
ASSERT3P(mp->b_rptr, ==, DB_BASE(mp));
mp->b_rptr += sp->s_rx_prefix_size;
prefetch_read_many(mp->b_rptr);
ASSERT3P(mp->b_wptr, ==, DB_BASE(mp));
mp->b_wptr += (size_t)(srpp->srp_size);
ASSERT3P(mp->b_wptr, <=, DB_LIM(mp));
/* Calculate the maximum packet size */
size = sp->s_mtu;
size += (srpp->srp_flags & EFX_PKT_VLAN_TAGGED) ?
sizeof (struct ether_vlan_header) :
sizeof (struct ether_header);
if (MBLKL(mp) > size)
goto discard;
/* Check for loopback packets */
if (!(srpp->srp_flags & EFX_PKT_IPV4) &&
!(srpp->srp_flags & EFX_PKT_IPV6)) {
struct ether_header *etherhp;
/*LINTED*/
etherhp = (struct ether_header *)(mp->b_rptr);
if (etherhp->ether_type ==
htons(SFXGE_ETHERTYPE_LOOPBACK)) {
DTRACE_PROBE(loopback);
srp->sr_loopback++;
goto discard;
}
}
/* Set up the checksum information */
flags = 0;
if (srpp->srp_flags & EFX_CKSUM_IPV4) {
ASSERT(srpp->srp_flags & EFX_PKT_IPV4);
flags |= HCK_IPV4_HDRCKSUM;
}
if (srpp->srp_flags & EFX_CKSUM_TCPUDP) {
ASSERT(srpp->srp_flags & EFX_PKT_TCP ||
srpp->srp_flags & EFX_PKT_UDP);
flags |= HCK_FULLCKSUM | HCK_FULLCKSUM_OK;
}
DB_CKSUMSTART(mp) = 0;
DB_CKSUMSTUFF(mp) = 0;
DB_CKSUMEND(mp) = 0;
DB_CKSUMFLAGS(mp) = flags;
DB_CKSUM16(mp) = 0;
/* Add the packet to the tail of the chain */
srfppp->srfpp_loaned++;
ASSERT(mp->b_next == NULL);
*(srp->sr_mpp) = mp;
srp->sr_mpp = &(mp->b_next);
continue;
discard:
/* Return the packet to the pool */
srfppp->srfpp_loaned++;
freeb(mp); /* Equivalent to freemsg() as b_cont==0 */
}
srp->sr_completed = completed;
/* Attempt to coalesce any TCP packets */
if (sp->s_rx_coalesce_mode != SFXGE_RX_COALESCE_OFF)
sfxge_rx_qpacket_coalesce(srp);
/*
* If there are any pending flows and this is the end of the
* poll then they must be completed.
*/
if (srp->sr_srfp != NULL && eop) {
sfxge_rx_flow_t *srfp;
srfp = srp->sr_srfp;
srp->sr_srfp = NULL;
srp->sr_srfpp = &(srp->sr_srfp);
do {
sfxge_rx_flow_t *next;
next = srfp->srf_next;
srfp->srf_next = NULL;
sfxge_rx_qflow_complete(srp, srfp);
srfp = next;
} while (srfp != NULL);
}
level = srp->sr_pushed - srp->sr_completed;
/* If there are any packets then pass them up the stack */
if (srp->sr_mp != NULL) {
mblk_t *mp;
mp = srp->sr_mp;
srp->sr_mp = NULL;
srp->sr_mpp = &(srp->sr_mp);
if (level == 0) {
/* Try to refill ASAP */
sfxge_rx_qrefill(srp, EFX_RXQ_LIMIT(sp->s_rxq_size));
level = srp->sr_pushed - srp->sr_completed;
}
/*
* If the RXQ is still empty, discard and recycle the
* current entry to ensure that the ring always
* contains at least one descriptor. This ensures that
* the next hardware RX will trigger an event
* (possibly delayed by interrupt moderation) and
* trigger another refill/fill attempt.
*
* Note this drops a complete LRO fragment from the
* start of the batch.
*
* Note also that copymsgchain() does not help with
* resource starvation here, unless we are short of DMA
* mappings.
*/
if (level == 0) {
mblk_t *nmp;
srp->sr_kstat.srk_rxq_empty_discard++;
DTRACE_PROBE1(rxq_empty_discard, int, index);
nmp = mp->b_next;
if (nmp)
sfxge_gld_rx_post(sp, index, nmp);
/* as level==0 will swizzle,rxpost below */
freemsg(mp);
} else {
sfxge_gld_rx_post(sp, index, mp);
}
}
/* Top up the queue if necessary */
if (level < srp->sr_hiwat) {
sfxge_rx_qrefill(srp, EFX_RXQ_LIMIT(sp->s_rxq_size));
level = srp->sr_added - srp->sr_completed;
if (level < srp->sr_lowat)
sfxge_rx_qfill(srp, EFX_RXQ_LIMIT(sp->s_rxq_size));
}
}
void
sfxge_rx_qflush_done(sfxge_rxq_t *srp)
{
sfxge_t *sp = srp->sr_sp;
unsigned int index = srp->sr_index;
sfxge_evq_t *sep = sp->s_sep[index];
boolean_t flush_pending;
ASSERT(mutex_owned(&(sep->se_lock)));
/*
* Flush successful: wakeup sfxge_rx_qstop() if flush is pending.
*
* A delayed flush event received after RxQ stop has timed out
* will be ignored, as then the flush state will not be PENDING
* (see SFCbug22989).
*/
flush_pending = (srp->sr_flush == SFXGE_FLUSH_PENDING);
srp->sr_flush = SFXGE_FLUSH_DONE;
if (flush_pending)
cv_broadcast(&(srp->sr_flush_kv));
}
void
sfxge_rx_qflush_failed(sfxge_rxq_t *srp)
{
sfxge_t *sp = srp->sr_sp;
unsigned int index = srp->sr_index;
sfxge_evq_t *sep = sp->s_sep[index];
boolean_t flush_pending;
ASSERT(mutex_owned(&(sep->se_lock)));
/*
* Flush failed: wakeup sfxge_rx_qstop() if flush is pending.
*
* A delayed flush event received after RxQ stop has timed out
* will be ignored, as then the flush state will not be PENDING
* (see SFCbug22989).
*/
flush_pending = (srp->sr_flush == SFXGE_FLUSH_PENDING);
srp->sr_flush = SFXGE_FLUSH_FAILED;
if (flush_pending)
cv_broadcast(&(srp->sr_flush_kv));
}
static void
sfxge_rx_qstop(sfxge_t *sp, unsigned int index)
{
dev_info_t *dip = sp->s_dip;
sfxge_evq_t *sep = sp->s_sep[index];
sfxge_rxq_t *srp;
clock_t timeout;
unsigned int flush_tries = SFXGE_RX_QFLUSH_TRIES;
int rc;
ASSERT(mutex_owned(&(sp->s_state_lock)));
mutex_enter(&(sep->se_lock));
srp = sp->s_srp[index];
ASSERT3U(srp->sr_state, ==, SFXGE_RXQ_STARTED);
sfxge_rx_qpoll_stop(srp);
/* Further packets are discarded by sfxge_rx_qcomplete() */
srp->sr_state = SFXGE_RXQ_INITIALIZED;
if (sp->s_hw_err != SFXGE_HW_OK) {
/*
* Flag indicates possible hardware failure.
* Attempt flush but do not wait for it to complete.
*/
srp->sr_flush = SFXGE_FLUSH_DONE;
(void) efx_rx_qflush(srp->sr_erp);
}
/* Wait upto 2sec for queue flushing to complete */
timeout = ddi_get_lbolt() + drv_usectohz(SFXGE_RX_QFLUSH_USEC);
while (srp->sr_flush != SFXGE_FLUSH_DONE && flush_tries-- > 0) {
if ((rc = efx_rx_qflush(srp->sr_erp)) != 0) {
if (rc == EALREADY)
srp->sr_flush = SFXGE_FLUSH_DONE;
else
srp->sr_flush = SFXGE_FLUSH_FAILED;
break;
}
srp->sr_flush = SFXGE_FLUSH_PENDING;
if (cv_timedwait(&(srp->sr_flush_kv), &(sep->se_lock),
timeout) < 0) {
/* Timeout waiting for successful or failed flush */
dev_err(dip, CE_NOTE,
SFXGE_CMN_ERR "rxq[%d] flush timeout", index);
break;
}
}
if (srp->sr_flush == SFXGE_FLUSH_FAILED)
dev_err(dip, CE_NOTE,
SFXGE_CMN_ERR "rxq[%d] flush failed", index);
DTRACE_PROBE1(flush, sfxge_flush_state_t, srp->sr_flush);
srp->sr_flush = SFXGE_FLUSH_DONE;
/* Destroy the receive queue */
efx_rx_qdestroy(srp->sr_erp);
srp->sr_erp = NULL;
/* Clear entries from the buffer table */
sfxge_sram_buf_tbl_clear(sp, srp->sr_id,
EFX_RXQ_NBUFS(sp->s_rxq_size));
/*
* Free any unused RX packets which had descriptors on the RXQ
* Packets will be discard as state != STARTED
*/
srp->sr_pending = srp->sr_added;
sfxge_rx_qcomplete(srp, B_TRUE);
ASSERT3U(srp->sr_completed, ==, srp->sr_pending);
srp->sr_added = 0;
srp->sr_pushed = 0;
srp->sr_pending = 0;
srp->sr_completed = 0;
srp->sr_loopback = 0;
srp->sr_lowat = 0;
srp->sr_hiwat = 0;
mutex_exit(&(sep->se_lock));
}
static void
sfxge_rx_kstat_fini(sfxge_rxq_t *srp)
{
kstat_delete(srp->sr_ksp);
srp->sr_ksp = NULL;
}
static void
sfxge_rx_qfini(sfxge_t *sp, unsigned int index)
{
sfxge_rxq_t *srp = sp->s_srp[index];
ASSERT3U(srp->sr_state, ==, SFXGE_RXQ_INITIALIZED);
sp->s_srp[index] = NULL;
srp->sr_state = SFXGE_RXQ_UNINITIALIZED;
sfxge_rx_kstat_fini(srp);
/* Empty the pool */
sfxge_rx_qfpp_empty(srp);
srp->sr_index = 0;
kmem_cache_free(sp->s_rqc, srp);
}
static int
sfxge_rx_scale_kstat_update(kstat_t *ksp, int rw)
{
sfxge_t *sp = ksp->ks_private;
sfxge_rx_scale_t *srsp = &(sp->s_rx_scale);
sfxge_intr_t *sip = &(sp->s_intr);
kstat_named_t *knp;
unsigned int index;
unsigned int entry;
unsigned int *freq;
int rc;
ASSERT(mutex_owned(&(srsp->srs_lock)));
if (rw != KSTAT_READ) {
rc = EACCES;
goto fail1;
}
if ((freq = kmem_zalloc(sizeof (unsigned int) * sip->si_nalloc,
KM_NOSLEEP)) == NULL) {
rc = ENOMEM;
goto fail2;
}
for (entry = 0; entry < SFXGE_RX_SCALE_MAX; entry++) {
index = srsp->srs_tbl[entry];
freq[index]++;
}
knp = ksp->ks_data;
for (index = 0; index < sip->si_nalloc; index++) {
knp->value.ui64 = freq[index];
knp++;
}
knp->value.ui64 = srsp->srs_count;
kmem_free(freq, sizeof (unsigned int) * sip->si_nalloc);
return (0);
fail2:
DTRACE_PROBE(fail2);
fail1:
DTRACE_PROBE1(fail1, int, rc);
return (rc);
}
static int
sfxge_rx_scale_kstat_init(sfxge_t *sp)
{
dev_info_t *dip = sp->s_dip;
sfxge_rx_scale_t *srsp = &(sp->s_rx_scale);
sfxge_intr_t *sip = &(sp->s_intr);
char name[MAXNAMELEN];
kstat_t *ksp;
kstat_named_t *knp;
unsigned int index;
int rc;
/* Create the set */
(void) snprintf(name, MAXNAMELEN - 1, "%s_rss", ddi_driver_name(dip));
if ((ksp = kstat_create((char *)ddi_driver_name(dip),
ddi_get_instance(dip), name, "rss", KSTAT_TYPE_NAMED,
sip->si_nalloc + 1, 0)) == NULL) {
rc = ENOMEM;
goto fail1;
}
srsp->srs_ksp = ksp;
ksp->ks_update = sfxge_rx_scale_kstat_update;
ksp->ks_private = sp;
ksp->ks_lock = &(srsp->srs_lock);
/* Initialise the named stats */
knp = ksp->ks_data;
for (index = 0; index < sip->si_nalloc; index++) {
char name[MAXNAMELEN];
(void) snprintf(name, MAXNAMELEN - 1, "evq%04d_count", index);
kstat_named_init(knp, name, KSTAT_DATA_UINT64);
knp++;
}
kstat_named_init(knp, "scale", KSTAT_DATA_UINT64);
kstat_install(ksp);
return (0);
fail1:
DTRACE_PROBE1(fail1, int, rc);
return (rc);
}
static void
sfxge_rx_scale_kstat_fini(sfxge_t *sp)
{
sfxge_rx_scale_t *srsp = &(sp->s_rx_scale);
/* Destroy the set */
kstat_delete(srsp->srs_ksp);
srsp->srs_ksp = NULL;
}
unsigned int
sfxge_rx_scale_prop_get(sfxge_t *sp)
{
int rx_scale;
rx_scale = ddi_prop_get_int(DDI_DEV_T_ANY, sp->s_dip,
DDI_PROP_DONTPASS, "rx_scale_count", SFXGE_RX_SCALE_MAX);
/* 0 and all -ve numbers sets to number of logical CPUs */
if (rx_scale <= 0)
rx_scale = ncpus;
return (rx_scale);
}
static int
sfxge_rx_scale_init(sfxge_t *sp)
{
sfxge_rx_scale_t *srsp = &(sp->s_rx_scale);
sfxge_intr_t *sip = &(sp->s_intr);
int rc;
ASSERT3U(srsp->srs_state, ==, SFXGE_RX_SCALE_UNINITIALIZED);
/* Create tables for CPU, core, cache and chip counts */
srsp->srs_cpu = kmem_zalloc(sizeof (unsigned int) * NCPU, KM_SLEEP);
mutex_init(&(srsp->srs_lock), NULL, MUTEX_DRIVER, NULL);
/* We need at least one event queue */
srsp->srs_count = sfxge_rx_scale_prop_get(sp);
if (srsp->srs_count > sip->si_nalloc)
srsp->srs_count = sip->si_nalloc;
if (srsp->srs_count < 1)
srsp->srs_count = 1;
/* Set up the kstats */
if ((rc = sfxge_rx_scale_kstat_init(sp)) != 0)
goto fail1;
srsp->srs_state = SFXGE_RX_SCALE_INITIALIZED;
return (0);
fail1:
DTRACE_PROBE1(fail1, int, rc);
mutex_destroy(&(srsp->srs_lock));
return (rc);
}
void
sfxge_rx_scale_update(void *arg)
{
sfxge_t *sp = arg;
sfxge_rx_scale_t *srsp = &(sp->s_rx_scale);
sfxge_intr_t *sip;
processorid_t id;
unsigned int count;
unsigned int *tbl;
unsigned int *rating;
unsigned int entry;
int rc;
mutex_enter(&(srsp->srs_lock));
if (srsp->srs_state != SFXGE_RX_SCALE_STARTED) {
rc = EFAULT;
goto fail1;
}
if ((tbl = kmem_zalloc(sizeof (unsigned int) * SFXGE_RX_SCALE_MAX,
KM_NOSLEEP)) == NULL) {
rc = ENOMEM;
goto fail2;
}
sip = &(sp->s_intr);
if ((rating = kmem_zalloc(sizeof (unsigned int) * sip->si_nalloc,
KM_NOSLEEP)) == NULL) {
rc = ENOMEM;
goto fail3;
}
mutex_enter(&cpu_lock);
/*
* Substract any current CPU, core, cache and chip usage from the
* global contention tables.
*/
for (id = 0; id < NCPU; id++) {
ASSERT3U(sfxge_cpu[id], >=, srsp->srs_cpu[id]);
sfxge_cpu[id] -= srsp->srs_cpu[id];
srsp->srs_cpu[id] = 0;
}
ASSERT(srsp->srs_count != 0);
/* Choose as many event queues as we need */
for (count = 0; count < srsp->srs_count; count++) {
unsigned int index;
sfxge_evq_t *sep;
unsigned int choice;
unsigned int choice_rating;
bzero(rating, sizeof (unsigned int) * sip->si_nalloc);
/*
* Rate each event queue on its global level of CPU
* contention.
*/
for (index = 0; index < sip->si_nalloc; index++) {
sep = sp->s_sep[index];
id = sep->se_cpu_id;
rating[index] += sfxge_cpu[id];
}
/* Choose the queue with the lowest CPU contention */
choice = 0;
choice_rating = rating[0];
for (index = 1; index < sip->si_nalloc; index++) {
if (rating[index] < choice_rating) {
choice = index;
choice_rating = rating[index];
}
}
/* Add our choice to the condensed RSS table */
tbl[count] = choice;
/* Add information to the global contention tables */
sep = sp->s_sep[choice];
id = sep->se_cpu_id;
srsp->srs_cpu[id]++;
sfxge_cpu[id]++;
}
mutex_exit(&cpu_lock);
/* Build the expanded RSS table */
count = 0;
for (entry = 0; entry < SFXGE_RX_SCALE_MAX; entry++) {
unsigned int index;
index = tbl[count];
count = (count + 1) % srsp->srs_count;
srsp->srs_tbl[entry] = index;
}
/* Program the expanded RSS table into the hardware */
(void) efx_rx_scale_tbl_set(sp->s_enp, srsp->srs_tbl,
SFXGE_RX_SCALE_MAX);
mutex_exit(&(srsp->srs_lock));
kmem_free(rating, sizeof (unsigned int) * sip->si_nalloc);
kmem_free(tbl, sizeof (unsigned int) * SFXGE_RX_SCALE_MAX);
return;
fail3:
DTRACE_PROBE(fail3);
kmem_free(tbl, sizeof (unsigned int) * SFXGE_RX_SCALE_MAX);
fail2:
DTRACE_PROBE(fail2);
fail1:
DTRACE_PROBE1(fail1, int, rc);
mutex_exit(&(srsp->srs_lock));
}
static int
sfxge_rx_scale_start(sfxge_t *sp)
{
sfxge_rx_scale_t *srsp = &(sp->s_rx_scale);
int rc;
mutex_enter(&(srsp->srs_lock));
ASSERT3U(srsp->srs_state, ==, SFXGE_RX_SCALE_INITIALIZED);
/* Clear down the RSS table */
bzero(srsp->srs_tbl, sizeof (unsigned int) * SFXGE_RX_SCALE_MAX);
(void) efx_rx_scale_tbl_set(sp->s_enp, srsp->srs_tbl,
SFXGE_RX_SCALE_MAX);
if ((rc = sfxge_toeplitz_hash_init(sp)) != 0)
goto fail1;
srsp->srs_state = SFXGE_RX_SCALE_STARTED;
mutex_exit(&(srsp->srs_lock));
/* sfxge_t->s_state_lock held */
(void) ddi_taskq_dispatch(sp->s_tqp, sfxge_rx_scale_update, sp,
DDI_SLEEP);
return (0);
fail1:
DTRACE_PROBE1(fail1, int, rc);
mutex_exit(&(srsp->srs_lock));
return (rc);
}
int
sfxge_rx_scale_count_get(sfxge_t *sp, unsigned int *countp)
{
sfxge_rx_scale_t *srsp = &(sp->s_rx_scale);
int rc;
mutex_enter(&(srsp->srs_lock));
if (srsp->srs_state != SFXGE_RX_SCALE_INITIALIZED &&
srsp->srs_state != SFXGE_RX_SCALE_STARTED) {
rc = ENOTSUP;
goto fail1;
}
*countp = srsp->srs_count;
mutex_exit(&(srsp->srs_lock));
return (0);
fail1:
DTRACE_PROBE1(fail1, int, rc);
mutex_exit(&(srsp->srs_lock));
return (rc);
}
int
sfxge_rx_scale_count_set(sfxge_t *sp, unsigned int count)
{
sfxge_rx_scale_t *srsp = &(sp->s_rx_scale);
sfxge_intr_t *sip = &(sp->s_intr);
int dispatch = 1;
int rc;
if (count < 1 || count > sip->si_nalloc) {
rc = EINVAL;
goto fail1;
}
mutex_enter(&(srsp->srs_lock));
if (srsp->srs_state != SFXGE_RX_SCALE_INITIALIZED &&
srsp->srs_state != SFXGE_RX_SCALE_STARTED) {
rc = ENOTSUP;
goto fail2;
}
srsp->srs_count = count;
if (srsp->srs_state != SFXGE_RX_SCALE_STARTED)
dispatch = 0;
mutex_exit(&(srsp->srs_lock));
if (dispatch)
/* no locks held */
(void) ddi_taskq_dispatch(sp->s_tqp, sfxge_rx_scale_update, sp,
DDI_SLEEP);
return (0);
fail2:
DTRACE_PROBE(fail2);
mutex_exit(&(srsp->srs_lock));
fail1:
DTRACE_PROBE1(fail1, int, rc);
return (rc);
}
static void
sfxge_rx_scale_stop(sfxge_t *sp)
{
sfxge_rx_scale_t *srsp = &(sp->s_rx_scale);
processorid_t id;
mutex_enter(&(srsp->srs_lock));
ASSERT3U(srsp->srs_state, ==, SFXGE_RX_SCALE_STARTED);
srsp->srs_state = SFXGE_RX_SCALE_INITIALIZED;
mutex_enter(&cpu_lock);
/*
* Substract any current CPU, core, cache and chip usage from the
* global contention tables.
*/
for (id = 0; id < NCPU; id++) {
ASSERT3U(sfxge_cpu[id], >=, srsp->srs_cpu[id]);
sfxge_cpu[id] -= srsp->srs_cpu[id];
srsp->srs_cpu[id] = 0;
}
mutex_exit(&cpu_lock);
/* Clear down the RSS table */
bzero(srsp->srs_tbl, sizeof (unsigned int) * SFXGE_RX_SCALE_MAX);
(void) efx_rx_scale_tbl_set(sp->s_enp, srsp->srs_tbl,
SFXGE_RX_SCALE_MAX);
mutex_exit(&(srsp->srs_lock));
}
static void
sfxge_rx_scale_fini(sfxge_t *sp)
{
sfxge_rx_scale_t *srsp = &(sp->s_rx_scale);
ASSERT3U(srsp->srs_state, ==, SFXGE_RX_SCALE_INITIALIZED);
srsp->srs_state = SFXGE_RX_SCALE_UNINITIALIZED;
/* Tear down the kstats */
sfxge_rx_scale_kstat_fini(sp);
srsp->srs_count = 0;
mutex_destroy(&(srsp->srs_lock));
/* Destroy tables */
kmem_free(srsp->srs_cpu, sizeof (unsigned int) * NCPU);
srsp->srs_cpu = NULL;
sfxge_toeplitz_hash_fini(sp);
}
int
sfxge_rx_init(sfxge_t *sp)
{
sfxge_intr_t *sip = &(sp->s_intr);
char name[MAXNAMELEN];
int index;
int rc;
if (sip->si_state == SFXGE_INTR_UNINITIALIZED) {
rc = EINVAL;
goto fail1;
}
if ((rc = sfxge_rx_scale_init(sp)) != 0)
goto fail2;
(void) snprintf(name, MAXNAMELEN - 1, "%s%d_rx_packet_cache",
ddi_driver_name(sp->s_dip), ddi_get_instance(sp->s_dip));
sp->s_rpc = kmem_cache_create(name, sizeof (sfxge_rx_packet_t),
SFXGE_CPU_CACHE_SIZE, sfxge_rx_packet_ctor, sfxge_rx_packet_dtor,
NULL, sp, NULL, 0);
ASSERT(sp->s_rpc != NULL);
(void) snprintf(name, MAXNAMELEN - 1, "%s%d_rxq_cache",
ddi_driver_name(sp->s_dip), ddi_get_instance(sp->s_dip));
sp->s_rqc = kmem_cache_create(name, sizeof (sfxge_rxq_t),
SFXGE_CPU_CACHE_SIZE, sfxge_rx_qctor, sfxge_rx_qdtor, NULL, sp,
NULL, 0);
ASSERT(sp->s_rqc != NULL);
sp->s_rx_pkt_mem_max = ddi_prop_get_int64(DDI_DEV_T_ANY, sp->s_dip,
DDI_PROP_DONTPASS, "rx_pkt_mem_max", 0); /* disabled */
/* Initialize the receive queue(s) */
for (index = 0; index < sip->si_nalloc; index++) {
if ((rc = sfxge_rx_qinit(sp, index)) != 0)
goto fail3;
}
sp->s_rx_coalesce_mode = ddi_prop_get_int(DDI_DEV_T_ANY, sp->s_dip,
DDI_PROP_DONTPASS, "rx_coalesce_mode", SFXGE_RX_COALESCE_OFF);
return (0);
fail3:
DTRACE_PROBE(fail3);
/* Tear down the receive queue(s) */
while (--index >= 0)
sfxge_rx_qfini(sp, index);
kmem_cache_destroy(sp->s_rqc);
sp->s_rqc = NULL;
kmem_cache_destroy(sp->s_rpc);
sp->s_rpc = NULL;
sfxge_rx_scale_fini(sp);
fail2:
DTRACE_PROBE(fail2);
fail1:
DTRACE_PROBE1(fail1, int, rc);
return (rc);
}
int
sfxge_rx_start(sfxge_t *sp)
{
sfxge_mac_t *smp = &(sp->s_mac);
sfxge_intr_t *sip;
const efx_nic_cfg_t *encp;
size_t hdrlen, align;
int index;
int rc;
mutex_enter(&(smp->sm_lock));
/* Calculate the receive packet buffer size and alignment */
sp->s_rx_buffer_size = EFX_MAC_PDU(sp->s_mtu);
encp = efx_nic_cfg_get(sp->s_enp);
/* Packet buffer allocations are cache line aligned */
EFSYS_ASSERT3U(encp->enc_rx_buf_align_start, <=, SFXGE_CPU_CACHE_SIZE);
if (sp->s_family == EFX_FAMILY_HUNTINGTON) {
sp->s_rx_prefix_size = encp->enc_rx_prefix_size;
hdrlen = sp->s_rx_prefix_size + sizeof (struct ether_header);
/* Ensure IP headers are 32bit aligned */
sp->s_rx_buffer_align = P2ROUNDUP(hdrlen, 4) - hdrlen;
sp->s_rx_buffer_size += sp->s_rx_buffer_align;
} else if (encp->enc_features & EFX_FEATURE_LFSR_HASH_INSERT) {
sp->s_rx_prefix_size = encp->enc_rx_prefix_size;
/*
* Place the start of the buffer a prefix length minus 2
* before the start of a cache line. This ensures that the
* last two bytes of the prefix (which is where the LFSR hash
* is located) are in the same cache line as the headers, and
* the IP header is 32-bit aligned.
*/
sp->s_rx_buffer_align =
SFXGE_CPU_CACHE_SIZE - (encp->enc_rx_prefix_size - 2);
sp->s_rx_buffer_size += sp->s_rx_buffer_align;
} else {
sp->s_rx_prefix_size = 0;
/*
* Place the start of the buffer 2 bytes after a cache line
* boundary so that the headers fit into the cache line and
* the IP header is 32-bit aligned.
*/
hdrlen = sp->s_rx_prefix_size + sizeof (struct ether_header);
sp->s_rx_buffer_align = P2ROUNDUP(hdrlen, 4) - hdrlen;
sp->s_rx_buffer_size += sp->s_rx_buffer_align;
}
/* Align end of packet buffer for RX DMA end padding */
align = MAX(1, encp->enc_rx_buf_align_end);
EFSYS_ASSERT(ISP2(align));
sp->s_rx_buffer_size = P2ROUNDUP(sp->s_rx_buffer_size, align);
/* Initialize the receive module */
if ((rc = efx_rx_init(sp->s_enp)) != 0)
goto fail1;
mutex_exit(&(smp->sm_lock));
if ((rc = sfxge_rx_scale_start(sp)) != 0)
goto fail2;
/* Start the receive queue(s) */
sip = &(sp->s_intr);
for (index = 0; index < sip->si_nalloc; index++) {
if ((rc = sfxge_rx_qstart(sp, index)) != 0)
goto fail3;
}
ASSERT3U(sp->s_srp[0]->sr_state, ==, SFXGE_RXQ_STARTED);
/* It is sufficient to have Rx scale initialized */
ASSERT3U(sp->s_rx_scale.srs_state, ==, SFXGE_RX_SCALE_STARTED);
rc = efx_mac_filter_default_rxq_set(sp->s_enp, sp->s_srp[0]->sr_erp,
sp->s_rx_scale.srs_count > 1);
if (rc != 0)
goto fail4;
return (0);
fail4:
DTRACE_PROBE(fail4);
fail3:
DTRACE_PROBE(fail3);
/* Stop the receive queue(s) */
while (--index >= 0)
sfxge_rx_qstop(sp, index);
sfxge_rx_scale_stop(sp);
fail2:
DTRACE_PROBE(fail2);
mutex_enter(&(smp->sm_lock));
/* Tear down the receive module */
efx_rx_fini(sp->s_enp);
fail1:
DTRACE_PROBE1(fail1, int, rc);
mutex_exit(&(smp->sm_lock));
return (rc);
}
void
sfxge_rx_coalesce_mode_get(sfxge_t *sp, sfxge_rx_coalesce_mode_t *modep)
{
*modep = sp->s_rx_coalesce_mode;
}
int
sfxge_rx_coalesce_mode_set(sfxge_t *sp, sfxge_rx_coalesce_mode_t mode)
{
int rc;
switch (mode) {
case SFXGE_RX_COALESCE_OFF:
case SFXGE_RX_COALESCE_DISALLOW_PUSH:
case SFXGE_RX_COALESCE_ALLOW_PUSH:
break;
default:
rc = EINVAL;
goto fail1;
}
sp->s_rx_coalesce_mode = mode;
return (0);
fail1:
DTRACE_PROBE1(fail1, int, rc);
return (rc);
}
void
sfxge_rx_stop(sfxge_t *sp)
{
sfxge_mac_t *smp = &(sp->s_mac);
sfxge_intr_t *sip = &(sp->s_intr);
efx_nic_t *enp = sp->s_enp;
int index;
ASSERT(mutex_owned(&(sp->s_state_lock)));
efx_mac_filter_default_rxq_clear(enp);
/* Stop the receive queue(s) */
index = sip->si_nalloc;
while (--index >= 0) {
/* TBD: Flush RXQs in parallel; HW has limit + may need retry */
sfxge_rx_qstop(sp, index);
}
sfxge_rx_scale_stop(sp);
mutex_enter(&(smp->sm_lock));
/* Tear down the receive module */
efx_rx_fini(enp);
sp->s_rx_buffer_align = 0;
sp->s_rx_prefix_size = 0;
sp->s_rx_buffer_size = 0;
mutex_exit(&(smp->sm_lock));
}
unsigned int
sfxge_rx_loaned(sfxge_t *sp)
{
sfxge_intr_t *sip = &(sp->s_intr);
int index;
unsigned int loaned;
ASSERT3U(sip->si_state, ==, SFXGE_INTR_INITIALIZED);
loaned = 0;
for (index = 0; index < sip->si_nalloc; index++) {
sfxge_rxq_t *srp = sp->s_srp[index];
sfxge_evq_t *sep = sp->s_sep[srp->sr_index];
mutex_enter(&(sep->se_lock));
loaned += sfxge_rx_qfpp_swizzle(srp);
mutex_exit(&(sep->se_lock));
}
return (loaned);
}
void
sfxge_rx_fini(sfxge_t *sp)
{
sfxge_intr_t *sip = &(sp->s_intr);
int index;
ASSERT3U(sip->si_state, ==, SFXGE_INTR_INITIALIZED);
sp->s_rx_coalesce_mode = SFXGE_RX_COALESCE_OFF;
/* Tear down the receive queue(s) */
index = sip->si_nalloc;
while (--index >= 0)
sfxge_rx_qfini(sp, index);
ASSERT3U(sp->s_rx_pkt_mem_alloc, ==, 0);
kmem_cache_destroy(sp->s_rqc);
sp->s_rqc = NULL;
kmem_cache_destroy(sp->s_rpc);
sp->s_rpc = NULL;
sfxge_rx_scale_fini(sp);
}