49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore * Copyright (c) 2008-2016 Solarflare Communications Inc.
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore * All rights reserved.
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore * Redistribution and use in source and binary forms, with or without
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore * modification, are permitted provided that the following conditions are met:
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore * 1. Redistributions of source code must retain the above copyright notice,
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore * this list of conditions and the following disclaimer.
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore * 2. Redistributions in binary form must reproduce the above copyright notice,
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore * this list of conditions and the following disclaimer in the documentation
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore * and/or other materials provided with the distribution.
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore * The views and conclusions contained in the software and documentation are
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore * those of the authors and should not be interpreted as representing official
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore * policies, either expressed or implied, of the FreeBSD Project.
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore/* RXQ flush response timeout (in microseconds) */
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore/* RXQ flush tries in the case of failure */
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore/* RXQ default packet buffer preallocation (number of packet buffers) */
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore/* Receive packet DMA attributes */
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amorestatic ddi_device_acc_attr_t sfxge_rx_packet_devacc = {
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore DDI_DEVICE_ATTR_V0, /* devacc_attr_version */
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore DDI_NEVERSWAP_ACC, /* devacc_attr_endian_flags */
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore DDI_STRICTORDER_ACC /* devacc_attr_dataorder */
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amorestatic ddi_dma_attr_t sfxge_rx_packet_dma_attr = {
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore 0, /* dma_attr_addr_lo */
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore 0xffffffffffffffffull, /* dma_attr_addr_hi */
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore 0xffffffffffffffffull, /* dma_attr_count_max */
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore 0xffffffffffffffffull, /* dma_attr_maxxfer */
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore 0 /* dma_attr_flags */
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore/* Receive queue DMA attributes */
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amorestatic ddi_device_acc_attr_t sfxge_rxq_devacc = {
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore DDI_DEVICE_ATTR_V0, /* devacc_attr_version */
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore DDI_NEVERSWAP_ACC, /* devacc_attr_endian_flags */
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore DDI_STRICTORDER_ACC /* devacc_attr_dataorder */
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore 0, /* dma_attr_addr_lo */
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore 0xffffffffffffffffull, /* dma_attr_addr_hi */
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore 0xffffffffffffffffull, /* dma_attr_count_max */
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore 0xffffffffffffffffull, /* dma_attr_maxxfer */
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore 0 /* dma_attr_flags */
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore/* Forward declaration */
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amorestatic void sfxge_rx_qpreallocate(sfxge_rxq_t *srp, int nprealloc);
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amoresfxge_rx_packet_ctor(void *buf, void *arg, int kmflags)
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore ASSERT3U(sizeof (srpp->__srp_u1.__srp_s1), <=,
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore ASSERT3U(sizeof (srpp->__srp_u2.__srp_s2), <=,
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore /* Allocate a DMA handle */
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore err = ddi_dma_alloc_handle(dip, &sfxge_rx_packet_dma_attr,
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore (kmflags == KM_SLEEP) ? DDI_DMA_SLEEP : DDI_DMA_DONTWAIT,
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore /* Free the DMA handle */
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore ddi_dma_free_handle(&(srpp->srp_dma_handle));
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amoresfxge_rx_qctor(void *buf, void *arg, int kmflags)
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore /* Compile-time structure layout checks */
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore EFX_STATIC_ASSERT(sizeof (srp->__sr_u1.__sr_s1) <=
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore EFX_STATIC_ASSERT(sizeof (srp->__sr_u2.__sr_s2) <=
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore EFX_STATIC_ASSERT(sizeof (srp->__sr_u3.__sr_s3) <=
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore dma_attr.sdba_length = EFX_RXQ_SIZE(sp->s_rxq_size);
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore dma_attr.sdba_memflags = DDI_DMA_CONSISTENT;
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore dma_attr.sdba_bindflags = DDI_DMA_READ | DDI_DMA_CONSISTENT;
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore if ((rc = sfxge_dma_buffer_create(esmp, &dma_attr)) != 0)
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore /* Allocate some buffer table entries */
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore if ((rc = sfxge_sram_buf_tbl_alloc(sp, EFX_RXQ_NBUFS(sp->s_rxq_size),
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore /* Allocate the context array */
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore if ((srp->sr_srpp = kmem_zalloc(sizeof (sfxge_rx_packet_t *) *
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore /* Allocate the flow table */
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore if ((srp->sr_flow = kmem_zalloc(sizeof (sfxge_rx_flow_t) *
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore /* Initialize the free packet pool */
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore if ((srfppp->srfpp_putp = kmem_zalloc(SFXGE_CPU_CACHE_SIZE *
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore for (id = 0; id < SFXGE_RX_FPP_NSLOTS; id++) {
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore mutex_init(&(putp->srfpl_lock), NULL, MUTEX_DRIVER,
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore cv_init(&(srp->sr_flush_kv), NULL, CV_DRIVER, NULL);
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore /* Preallocate some packets on the free packet pool */
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore nprealloc = ddi_prop_get_int(DDI_DEV_T_ANY, sp->s_dip,
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore DDI_PROP_DONTPASS, "rx_prealloc_pkt_buffers", SFXGE_RX_QPREALLOC);
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore /* Free the flow table */
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore kmem_free(srp->sr_flow, sizeof (sfxge_rx_flow_t) *
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore /* Free the context array */
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore kmem_free(srp->sr_srpp, sizeof (sfxge_rx_packet_t *) *
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore /* Free the buffer table entries */
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore /* Remove dma setup */
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore /* Tear down the free packet pool */
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore for (id = 0; id < SFXGE_RX_FPP_NSLOTS; id++) {
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore SFXGE_OBJ_CHECK(putp, sfxge_rx_fpp_putlist_t);
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore kmem_free(srfppp->srfpp_putp, SFXGE_CPU_CACHE_SIZE *
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore /* Free the flow table */
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore kmem_free(srp->sr_flow, sizeof (sfxge_rx_flow_t) *
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore /* Free the context array */
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore kmem_free(srp->sr_srpp, sizeof (sfxge_rx_packet_t *) *
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore /* Free the buffer table entries */
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore /* Tear down dma setup */
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore/* Note: This function takes ownership of *srpp. */
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amorestatic inline void
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amoresfxge_rx_qfpp_put(sfxge_rxq_t *srp, sfxge_rx_packet_t *srpp)
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore ASSERT3P(srpp->srp_putp, ==, srfppp->srfpp_putp);
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amorestatic unsigned int
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore /* We want to access the put list for the current CPU last */
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore id = start = (CPU->cpu_seqid + 1) & SFXGE_RX_FPP_MASK;
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore /* Acquire the put list */
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore /* Add the list to the head of the get list */
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore /* Adjust the counters */
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore /* NOTE: this probe is disabled because it is expensive!! */
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore unsigned int, count);
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore /* Return the number of packets yet to appear in the put list */
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore#define DB_FRTNP(mp) ((mp)->b_datap->db_frtnp)
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore /* Swizzle put list to get list */
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore /* Free the remainder */
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore * ASSERT3P(freep->free_func, ==, sfxge_rx_qpacket_free);
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore * is implied by srpp test below
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore srpp = (sfxge_rx_packet_t *)(freep->free_arg);
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore * This is an estimate of all memory consumed per RX packet
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore * it can be inaccurate but but sp->s_rx_pkt_mem_alloc mustn't drift
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amoresfxge_rx_pkt_mem_approx(const sfxge_rx_packet_t *srpp)
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore return (srpp->srp_mblksize + sizeof (mblk_t) + sizeof (dblk_t) +
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amoresfxge_rx_qpacket_destroy(sfxge_rxq_t *srp, sfxge_rx_packet_t *srpp)
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore int64_t delta = sfxge_rx_pkt_mem_approx(srpp);
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore /* Unbind the DMA memory from the DMA handle */
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore (void) ddi_dma_unbind_handle(srpp->srp_dma_handle);
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore /* Free the DMA memory */
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore atomic_add_64(&sp->s_rx_pkt_mem_alloc, -delta);
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore * WARNING "man -s 9f esballoc" states:
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore * => runs sync from the thread calling freeb()
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore * => must not sleep, or access data structures that could be freed
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore /* Check whether we want to recycle the receive packets */
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore ASSERT3P(freep->free_func, ==, sfxge_rx_qpacket_free);
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore ASSERT3P(freep->free_arg, ==, (caddr_t)srpp);
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore * Allocate a matching mblk_t before the current one is
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore if ((mp = desballoc(srpp->srp_base, size, BPRI_HI,
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore /* NORMAL recycled case */
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore (sp->s_rx_pkt_mem_alloc + size >= sp->s_rx_pkt_mem_max)) {
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore /* Allocate a new packet */
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore if ((srpp = kmem_cache_alloc(sp->s_rpc, KM_NOSLEEP)) == NULL) {
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore /* Allocate some DMA memory */
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore err = ddi_dma_mem_alloc(srpp->srp_dma_handle, size,
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore &sfxge_rx_packet_devacc, DDI_DMA_STREAMING, DDI_DMA_DONTWAIT,
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore NULL, &base, &unit, &(srpp->srp_acc_handle));
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore /* Adjust the buffer to align the start of the DMA area correctly */
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore /* Bind the DMA memory to the DMA handle */
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore err = ddi_dma_addr_bind_handle(srpp->srp_dma_handle, NULL,
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore base, size, DDI_DMA_READ | DDI_DMA_STREAMING,
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore * Allocate a STREAMS block: We use size 1 so that the allocator will
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore * use the first (and smallest) dblk cache.
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore if ((mp = desballoc(srpp->srp_base, size, BPRI_HI, freep)) == NULL) {
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore int64_t delta = sfxge_rx_pkt_mem_approx(srpp);
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore atomic_add_64(&sp->s_rx_pkt_mem_alloc, delta);
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore /* Unbind the DMA memory from the DMA handle */
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore (void) ddi_dma_unbind_handle(srpp->srp_dma_handle);
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore /* Free the DMA memory */
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore/* Try to refill the RX descriptor ring from the associated free pkt pool */
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amoresfxge_rx_qrefill(sfxge_rxq_t *srp, unsigned int target)
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore ASSERT3U(rxfill, <=, EFX_RXQ_LIMIT(sp->s_rxq_size));
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore ntodo = min(EFX_RXQ_LIMIT(sp->s_rxq_size) - rxfill, target);
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore ASSERT3U(ntodo, <=, EFX_RXQ_LIMIT(sp->s_rxq_size));
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore mblksize = sp->s_rx_buffer_size - sp->s_rx_buffer_align;
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore while (ntodo-- > 0) {
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore srpp = (sfxge_rx_packet_t *)(freep->free_arg);
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore /* The MTU may have changed since the packet was allocated */
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore id = (srp->sr_added + batch) & (sp->s_rxq_size - 1);
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore efx_rx_qpost(srp->sr_erp, addr, mblksize, batch,
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore efx_rx_qpost(srp->sr_erp, addr, mblksize, batch,
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore efx_rx_qpush(srp->sr_erp, srp->sr_added, &srp->sr_pushed);
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore if (srfppp->srfpp_count < srfppp->srfpp_min)
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore/* Preallocate packets and put them in the free packet pool */
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amoresfxge_rx_qpreallocate(sfxge_rxq_t *srp, int nprealloc)
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore while (nprealloc-- > 0) {
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore if ((srpp = sfxge_rx_qpacket_create(srp)) == NULL)
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore/* Try to refill the RX descriptor ring by allocating new packets */
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amoresfxge_rx_qfill(sfxge_rxq_t *srp, unsigned int target)
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore ASSERT3U(rxfill, <=, EFX_RXQ_LIMIT(sp->s_rxq_size));
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore ntodo = min(EFX_RXQ_LIMIT(sp->s_rxq_size) - rxfill, target);
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore ASSERT3U(ntodo, <=, EFX_RXQ_LIMIT(sp->s_rxq_size));
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore mblksize = sp->s_rx_buffer_size - sp->s_rx_buffer_align;
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore while (ntodo-- > 0) {
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore if ((srpp = sfxge_rx_qpacket_create(srp)) == NULL)
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore id = (srp->sr_added + batch) & (sp->s_rxq_size - 1);
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore efx_rx_qpost(srp->sr_erp, addr, mblksize, batch,
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore efx_rx_qpost(srp->sr_erp, addr, mblksize, batch,
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore efx_rx_qpush(srp->sr_erp, srp->sr_added, &srp->sr_pushed);
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore /* Make sure the queue is full */
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore sfxge_rx_qrefill(srp, EFX_RXQ_LIMIT(sp->s_rxq_size));
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore /* The refill may have emptied the pool */
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore /* Don't trim below the pool's low water mark */
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore if (srfppp->srfpp_count <= srfppp->srfpp_lowat)
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore ASSERT(srfppp->srfpp_min <= srfppp->srfpp_count);
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore /* Trim to the largest of srfppp->srfpp_min and srfpp->srfpp_lowat */
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore if (srfppp->srfpp_lowat > srfppp->srfpp_min)
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore count = srfppp->srfpp_count - srfppp->srfpp_lowat;
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore count = srfppp->srfpp_count - srfppp->srfpp_min;
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore /* Walk the get list */
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore while (--count >= 0) {
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore /* Truncate the get list */
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore /* Free the remainder */
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore while (p != NULL) {
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore srpp = (sfxge_rx_packet_t *)(freep->free_arg);
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore * man timeout(9f) states that this code should adhere to the
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore * same requirements as a softirq handler - DO NOT BLOCK
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore * Post an event to the event queue to cause the free packet pool to be
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore * trimmed if it is oversize.
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore /* This is guaranteed due to the start/stop order of rx and ev */
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore ASSERT3U(sep->se_state, ==, SFXGE_EVQ_STARTED);
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore ASSERT3U(srp->sr_state, ==, SFXGE_RXQ_STARTED);
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore * Bug22691 WORKAROUND:
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore * This handler has been observed in the field to be invoked for a
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore * queue in the INITIALIZED state, which should never happen.
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore * Until the mechanism for this is properly understood, add defensive
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore "RXQ[%d] bad state in sfxge_rx_qpoll %d %d %p",
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore index, sep->se_state, srp->sr_state, sep->se_eep);
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore ASSERT3U(srp->sr_state, ==, SFXGE_RXQ_STARTED);
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore /* Schedule a poll */
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore srp->sr_tid = timeout(sfxge_rx_qpoll, srp, 0);
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore ASSERT3U(srp->sr_state, ==, SFXGE_RXQ_STARTED);
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore * Cancel the qpoll timer. Care is needed as this function
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore * can race with sfxge_rx_qpoll() for timeout id updates.
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore * Do not hold locks used by any timeout(9f) handlers across
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore * calls to untimeout(9f) as this will deadlock.
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore while ((srp->sr_tid != 0) && (srp->sr_tid != tid)) {
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore /* NB pointer post-increment below */
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore knp++->value.ui32 = srp->sr_kstat.srk_rx_pkt_mem_limit;
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore knp++->value.ui32 = srp->sr_kstat.srk_kcache_alloc_nomem;
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore knp++->value.ui32 = srp->sr_kstat.srk_dma_alloc_nomem;
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore knp++->value.ui32 = srp->sr_kstat.srk_dma_alloc_fail;
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore knp++->value.ui32 = srp->sr_kstat.srk_dma_bind_nomem;
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore knp++->value.ui32 = srp->sr_kstat.srk_dma_bind_fail;
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore knp++->value.ui32 = srp->sr_kstat.srk_desballoc_fail;
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore knp++->value.ui32 = srp->sr_kstat.srk_rxq_empty_discard;
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore /* Create the set */
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore (void) snprintf(name, MAXNAMELEN - 1, "%s_rxq%04d",
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore if ((ksp = kstat_create((char *)ddi_driver_name(dip),
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore ddi_get_instance(dip), name, "rxq", KSTAT_TYPE_NAMED,
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore /* Initialise the named stats */
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore kstat_named_init(knp, "rx_pkt_mem_limit", KSTAT_DATA_UINT32);
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore kstat_named_init(knp, "kcache_alloc_nomem", KSTAT_DATA_UINT32);
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore kstat_named_init(knp, "dma_alloc_nomem", KSTAT_DATA_UINT32);
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore kstat_named_init(knp, "dma_alloc_fail", KSTAT_DATA_UINT32);
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore kstat_named_init(knp, "dma_bind_nomem", KSTAT_DATA_UINT32);
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore kstat_named_init(knp, "dma_bind_fail", KSTAT_DATA_UINT32);
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore kstat_named_init(knp, "desballoc_fail", KSTAT_DATA_UINT32);
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore kstat_named_init(knp, "rxq_empty_discard", KSTAT_DATA_UINT32);
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amoresfxge_rx_qinit(sfxge_t *sp, unsigned int index)
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore if ((srp = kmem_cache_alloc(sp->s_rqc, KM_SLEEP)) == NULL) {
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore ASSERT3U(srp->sr_state, ==, SFXGE_RXQ_UNINITIALIZED);
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amoresfxge_rx_qstart(sfxge_t *sp, unsigned int index)
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore ASSERT3U(srp->sr_state, ==, SFXGE_RXQ_INITIALIZED);
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore ASSERT3U(sep->se_state, ==, SFXGE_EVQ_STARTED);
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore /* Zero the memory */
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore bzero(esmp->esm_base, EFX_RXQ_SIZE(sp->s_rxq_size));
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore /* Program the buffer table */
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore if ((rc = sfxge_sram_buf_tbl_set(sp, srp->sr_id, esmp,
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore /* Create the receive queue */
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore if ((rc = efx_rx_qcreate(enp, index, index, EFX_RXQ_TYPE_DEFAULT,
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore esmp, sp->s_rxq_size, srp->sr_id, sep->se_eep, &(srp->sr_erp)))
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore /* Enable the receive queue */
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore /* Set the water marks */
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore srp->sr_hiwat = EFX_RXQ_LIMIT(sp->s_rxq_size) * 9 / 10;
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore /* Try to fill the queue from the pool */
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore sfxge_rx_qrefill(srp, EFX_RXQ_LIMIT(sp->s_rxq_size));
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore * If there were insufficient buffers in the pool to reach the at
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore * least a batch then allocate some.
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore /* Clear entries from the buffer table */
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amoresfxge_rx_qflow_complete(sfxge_rxq_t *srp, sfxge_rx_flow_t *srfp)
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore ASSERT3U(((etherhp->ether_type == htons(ETHERTYPE_VLAN)) ?
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore sizeof (struct ether_vlan_header) :
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore sizeof (struct ether_header)) +
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore ASSERT3U(srfp->srf_len & 0xffff, ==, srfp->srf_len);
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore srfp->srf_first_thp->th_flags = thp->th_flags;
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore DTRACE_PROBE2(flow_complete, uint32_t, srfp->srf_tag,
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amoresfxge_rx_qflow_add(sfxge_rxq_t *srp, sfxge_rx_flow_t *srfp,
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore struct ether_header *etherhp = srpp->srp_etherhp;
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore HCK_FULLCKSUM | HCK_FULLCKSUM_OK | HCK_IPV4_HDRCKSUM);
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore * If the time between this segment and the last is greater than RTO
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore * then consider this a new flow.
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore /* Update the in-order segment count and sequence number */
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore /* Don't merge across pure ACK, URG, SYN or RST segments */
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore if (size == 0 || thp->th_flags & (TH_URG | TH_SYN | TH_RST) ||
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore * If the in-order segment count has not yet reached the slow-start
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore * threshold then we cannot coalesce.
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore /* Scale up the packet size from 4k (the maximum being 64k) */
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore ASSERT3U(srfp->srf_count, >=, SFXGE_SLOW_START);
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore shift = MIN(srfp->srf_count - SFXGE_SLOW_START + 12, 16);
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore /* First packet in this flow */
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore srfp->srf_first_thp = srfp->srf_last_thp = thp;
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore * If the flow is not already in the list of occupied flows then
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore /* Later packet in this flow - skip TCP header */
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore DTRACE_PROBE2(flow_add, uint32_t, srfp->srf_tag, size_t, size);
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore * Try to align coalesced segments on push boundaries, unless they
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore * are too frequent.
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore if (sp->s_rx_coalesce_mode == SFXGE_RX_COALESCE_ALLOW_PUSH &&
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore ASSERT(sp->s_rx_coalesce_mode != SFXGE_RX_COALESCE_OFF);
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore /* Start with the last flow to be appended to */
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore srpp = (sfxge_rx_packet_t *)(freep->free_arg);
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore /* If the packet is not TCP then we cannot coalesce it */
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore * If the packet is not fully checksummed then we cannot
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore * coalesce it.
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore if (~(srpp->srp_flags) & (EFX_CKSUM_TCPUDP | EFX_CKSUM_IPV4))
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore /* Parse the TCP header */
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore pkt_type = sfxge_pkthdr_parse(mp, ðerhp, &iphp, &thp, &off,
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore ASSERT(pkt_type == SFXGE_PACKET_TYPE_IPV4_TCP);
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore if (etherhp->ether_type == htons(ETHERTYPE_VLAN)) {
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore ethervhp = (struct ether_vlan_header *)etherhp;
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore * Make sure any minimum length padding is stripped
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore * before we try to add the packet to a flow.
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore ASSERT3U(sp->s_rx_prefix_size + MBLKL(mp), ==,
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore ASSERT3U(sp->s_rx_prefix_size + off + size, <=,
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore * If there is no current flow, or the segment does not match
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore * the current flow then we must attempt to look up the
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore * correct flow in the table.
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore if (srfp->srf_saddr != iphp->ip_src.s_addr ||
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore /* Try to append the packet to the flow */
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore if (!sfxge_rx_qflow_add(srp, srfp, srpp, now))
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore * If there is a prefix area then read the hash from that,
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore * otherwise calculate it.
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore srfp = &(srp->sr_flow[(hash >> 6) % SFXGE_MAX_FLOW]);
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore tag = hash + 1; /* Make sure it's not zero */
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore * If the flow we have found does not match the hash then
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore * it may be an unused flow, or it may be stale.
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore /* Start a new flow */
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore * If the flow we have found does match the hash then it could
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore * still be an alias.
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore if (srfp->srf_saddr != iphp->ip_src.s_addr ||
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amoresfxge_rx_qcomplete(sfxge_rxq_t *srp, boolean_t eop)
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore /* when called from sfxge_rx_qstop() */
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore if (srpp->srp_flags & (EFX_ADDR_MISMATCH | EFX_DISCARD))
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore /* Make the data visible to the kernel */
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore sp->s_rx_buffer_size, DDI_DMA_SYNC_FORKERNEL);
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore /* Read the length from the psuedo header if required */
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore rc = efx_psuedo_hdr_pkt_length_get(sp->s_enp,
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore /* Set up the packet length */
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore /* Calculate the maximum packet size */
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore size += (srpp->srp_flags & EFX_PKT_VLAN_TAGGED) ?
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore sizeof (struct ether_vlan_header) :
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore sizeof (struct ether_header);
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore /* Check for loopback packets */
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore etherhp = (struct ether_header *)(mp->b_rptr);
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore /* Set up the checksum information */
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore /* Add the packet to the tail of the chain */
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore /* Return the packet to the pool */
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore freeb(mp); /* Equivalent to freemsg() as b_cont==0 */
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore /* Attempt to coalesce any TCP packets */
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore if (sp->s_rx_coalesce_mode != SFXGE_RX_COALESCE_OFF)
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore * If there are any pending flows and this is the end of the
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore * poll then they must be completed.
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore /* If there are any packets then pass them up the stack */
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore /* Try to refill ASAP */
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore sfxge_rx_qrefill(srp, EFX_RXQ_LIMIT(sp->s_rxq_size));
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore * If the RXQ is still empty, discard and recycle the
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore * current entry to ensure that the ring always
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore * contains at least one descriptor. This ensures that
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore * the next hardware RX will trigger an event
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore * (possibly delayed by interrupt moderation) and
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore * trigger another refill/fill attempt.
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore * Note this drops a complete LRO fragment from the
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore * start of the batch.
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore * Note also that copymsgchain() does not help with
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore * resource starvation here, unless we are short of DMA
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore DTRACE_PROBE1(rxq_empty_discard, int, index);
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore /* as level==0 will swizzle,rxpost below */
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore /* Top up the queue if necessary */
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore sfxge_rx_qrefill(srp, EFX_RXQ_LIMIT(sp->s_rxq_size));
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore sfxge_rx_qfill(srp, EFX_RXQ_LIMIT(sp->s_rxq_size));
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore * Flush successful: wakeup sfxge_rx_qstop() if flush is pending.
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore * A delayed flush event received after RxQ stop has timed out
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore * will be ignored, as then the flush state will not be PENDING
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore * (see SFCbug22989).
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore flush_pending = (srp->sr_flush == SFXGE_FLUSH_PENDING);
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore * Flush failed: wakeup sfxge_rx_qstop() if flush is pending.
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore * A delayed flush event received after RxQ stop has timed out
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore * will be ignored, as then the flush state will not be PENDING
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore * (see SFCbug22989).
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore flush_pending = (srp->sr_flush == SFXGE_FLUSH_PENDING);
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amoresfxge_rx_qstop(sfxge_t *sp, unsigned int index)
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore unsigned int flush_tries = SFXGE_RX_QFLUSH_TRIES;
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore ASSERT3U(srp->sr_state, ==, SFXGE_RXQ_STARTED);
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore /* Further packets are discarded by sfxge_rx_qcomplete() */
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore * Flag indicates possible hardware failure.
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore * Attempt flush but do not wait for it to complete.
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore /* Wait upto 2sec for queue flushing to complete */
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore timeout = ddi_get_lbolt() + drv_usectohz(SFXGE_RX_QFLUSH_USEC);
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore while (srp->sr_flush != SFXGE_FLUSH_DONE && flush_tries-- > 0) {
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore if ((rc = efx_rx_qflush(srp->sr_erp)) != 0) {
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore if (cv_timedwait(&(srp->sr_flush_kv), &(sep->se_lock),
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore /* Timeout waiting for successful or failed flush */
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore SFXGE_CMN_ERR "rxq[%d] flush timeout", index);
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore SFXGE_CMN_ERR "rxq[%d] flush failed", index);
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore DTRACE_PROBE1(flush, sfxge_flush_state_t, srp->sr_flush);
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore /* Destroy the receive queue */
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore /* Clear entries from the buffer table */
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore * Free any unused RX packets which had descriptors on the RXQ
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore * Packets will be discard as state != STARTED
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore ASSERT3U(srp->sr_completed, ==, srp->sr_pending);
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amoresfxge_rx_qfini(sfxge_t *sp, unsigned int index)
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore ASSERT3U(srp->sr_state, ==, SFXGE_RXQ_INITIALIZED);
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore /* Empty the pool */
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amoresfxge_rx_scale_kstat_update(kstat_t *ksp, int rw)
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore if ((freq = kmem_zalloc(sizeof (unsigned int) * sip->si_nalloc,
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore for (entry = 0; entry < SFXGE_RX_SCALE_MAX; entry++) {
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore for (index = 0; index < sip->si_nalloc; index++) {
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore kmem_free(freq, sizeof (unsigned int) * sip->si_nalloc);
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore /* Create the set */
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore (void) snprintf(name, MAXNAMELEN - 1, "%s_rss", ddi_driver_name(dip));
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore if ((ksp = kstat_create((char *)ddi_driver_name(dip),
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore ddi_get_instance(dip), name, "rss", KSTAT_TYPE_NAMED,
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore ksp->ks_update = sfxge_rx_scale_kstat_update;
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore /* Initialise the named stats */
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore for (index = 0; index < sip->si_nalloc; index++) {
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore (void) snprintf(name, MAXNAMELEN - 1, "evq%04d_count", index);
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore kstat_named_init(knp, name, KSTAT_DATA_UINT64);
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore kstat_named_init(knp, "scale", KSTAT_DATA_UINT64);
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore /* Destroy the set */
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore rx_scale = ddi_prop_get_int(DDI_DEV_T_ANY, sp->s_dip,
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore DDI_PROP_DONTPASS, "rx_scale_count", SFXGE_RX_SCALE_MAX);
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore /* 0 and all -ve numbers sets to number of logical CPUs */
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore ASSERT3U(srsp->srs_state, ==, SFXGE_RX_SCALE_UNINITIALIZED);
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore /* Create tables for CPU, core, cache and chip counts */
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore srsp->srs_cpu = kmem_zalloc(sizeof (unsigned int) * NCPU, KM_SLEEP);
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore mutex_init(&(srsp->srs_lock), NULL, MUTEX_DRIVER, NULL);
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore /* We need at least one event queue */
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore srsp->srs_count = sfxge_rx_scale_prop_get(sp);
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore /* Set up the kstats */
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore if ((rc = sfxge_rx_scale_kstat_init(sp)) != 0)
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore srsp->srs_state = SFXGE_RX_SCALE_INITIALIZED;
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore if (srsp->srs_state != SFXGE_RX_SCALE_STARTED) {
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore if ((tbl = kmem_zalloc(sizeof (unsigned int) * SFXGE_RX_SCALE_MAX,
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore if ((rating = kmem_zalloc(sizeof (unsigned int) * sip->si_nalloc,
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore * Substract any current CPU, core, cache and chip usage from the
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore * global contention tables.
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore ASSERT3U(sfxge_cpu[id], >=, srsp->srs_cpu[id]);
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore /* Choose as many event queues as we need */
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore for (count = 0; count < srsp->srs_count; count++) {
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore bzero(rating, sizeof (unsigned int) * sip->si_nalloc);
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore * Rate each event queue on its global level of CPU
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore * contention.
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore for (index = 0; index < sip->si_nalloc; index++) {
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore /* Choose the queue with the lowest CPU contention */
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore for (index = 1; index < sip->si_nalloc; index++) {
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore /* Add our choice to the condensed RSS table */
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore /* Add information to the global contention tables */
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore /* Build the expanded RSS table */
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore for (entry = 0; entry < SFXGE_RX_SCALE_MAX; entry++) {
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore /* Program the expanded RSS table into the hardware */
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore (void) efx_rx_scale_tbl_set(sp->s_enp, srsp->srs_tbl,
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore kmem_free(rating, sizeof (unsigned int) * sip->si_nalloc);
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore kmem_free(tbl, sizeof (unsigned int) * SFXGE_RX_SCALE_MAX);
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore kmem_free(tbl, sizeof (unsigned int) * SFXGE_RX_SCALE_MAX);
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore ASSERT3U(srsp->srs_state, ==, SFXGE_RX_SCALE_INITIALIZED);
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore /* Clear down the RSS table */
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore bzero(srsp->srs_tbl, sizeof (unsigned int) * SFXGE_RX_SCALE_MAX);
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore (void) efx_rx_scale_tbl_set(sp->s_enp, srsp->srs_tbl,
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore if ((rc = sfxge_toeplitz_hash_init(sp)) != 0)
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore /* sfxge_t->s_state_lock held */
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore (void) ddi_taskq_dispatch(sp->s_tqp, sfxge_rx_scale_update, sp,
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amoresfxge_rx_scale_count_get(sfxge_t *sp, unsigned int *countp)
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore if (srsp->srs_state != SFXGE_RX_SCALE_INITIALIZED &&
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore srsp->srs_state != SFXGE_RX_SCALE_STARTED) {
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amoresfxge_rx_scale_count_set(sfxge_t *sp, unsigned int count)
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore if (srsp->srs_state != SFXGE_RX_SCALE_INITIALIZED &&
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore srsp->srs_state != SFXGE_RX_SCALE_STARTED) {
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore if (srsp->srs_state != SFXGE_RX_SCALE_STARTED)
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore /* no locks held */
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore (void) ddi_taskq_dispatch(sp->s_tqp, sfxge_rx_scale_update, sp,
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore ASSERT3U(srsp->srs_state, ==, SFXGE_RX_SCALE_STARTED);
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore srsp->srs_state = SFXGE_RX_SCALE_INITIALIZED;
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore * Substract any current CPU, core, cache and chip usage from the
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore * global contention tables.
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore ASSERT3U(sfxge_cpu[id], >=, srsp->srs_cpu[id]);
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore /* Clear down the RSS table */
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore bzero(srsp->srs_tbl, sizeof (unsigned int) * SFXGE_RX_SCALE_MAX);
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore (void) efx_rx_scale_tbl_set(sp->s_enp, srsp->srs_tbl,
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore ASSERT3U(srsp->srs_state, ==, SFXGE_RX_SCALE_INITIALIZED);
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore srsp->srs_state = SFXGE_RX_SCALE_UNINITIALIZED;
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore /* Tear down the kstats */
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore /* Destroy tables */
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore kmem_free(srsp->srs_cpu, sizeof (unsigned int) * NCPU);
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore if (sip->si_state == SFXGE_INTR_UNINITIALIZED) {
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore (void) snprintf(name, MAXNAMELEN - 1, "%s%d_rx_packet_cache",
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore ddi_driver_name(sp->s_dip), ddi_get_instance(sp->s_dip));
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore sp->s_rpc = kmem_cache_create(name, sizeof (sfxge_rx_packet_t),
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore SFXGE_CPU_CACHE_SIZE, sfxge_rx_packet_ctor, sfxge_rx_packet_dtor,
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore (void) snprintf(name, MAXNAMELEN - 1, "%s%d_rxq_cache",
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore ddi_driver_name(sp->s_dip), ddi_get_instance(sp->s_dip));
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore sp->s_rqc = kmem_cache_create(name, sizeof (sfxge_rxq_t),
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore SFXGE_CPU_CACHE_SIZE, sfxge_rx_qctor, sfxge_rx_qdtor, NULL, sp,
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore sp->s_rx_pkt_mem_max = ddi_prop_get_int64(DDI_DEV_T_ANY, sp->s_dip,
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore DDI_PROP_DONTPASS, "rx_pkt_mem_max", 0); /* disabled */
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore /* Initialize the receive queue(s) */
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore for (index = 0; index < sip->si_nalloc; index++) {
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore sp->s_rx_coalesce_mode = ddi_prop_get_int(DDI_DEV_T_ANY, sp->s_dip,
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore DDI_PROP_DONTPASS, "rx_coalesce_mode", SFXGE_RX_COALESCE_OFF);
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore /* Tear down the receive queue(s) */
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore while (--index >= 0)
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore /* Calculate the receive packet buffer size and alignment */
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore sp->s_rx_buffer_size = EFX_MAC_PDU(sp->s_mtu);
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore /* Packet buffer allocations are cache line aligned */
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore EFSYS_ASSERT3U(encp->enc_rx_buf_align_start, <=, SFXGE_CPU_CACHE_SIZE);
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore if (sp->s_family == EFX_FAMILY_HUNTINGTON) {
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore sp->s_rx_prefix_size = encp->enc_rx_prefix_size;
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore hdrlen = sp->s_rx_prefix_size + sizeof (struct ether_header);
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore /* Ensure IP headers are 32bit aligned */
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore sp->s_rx_buffer_align = P2ROUNDUP(hdrlen, 4) - hdrlen;
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore sp->s_rx_buffer_size += sp->s_rx_buffer_align;
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore } else if (encp->enc_features & EFX_FEATURE_LFSR_HASH_INSERT) {
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore sp->s_rx_prefix_size = encp->enc_rx_prefix_size;
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore * Place the start of the buffer a prefix length minus 2
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore * before the start of a cache line. This ensures that the
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore * last two bytes of the prefix (which is where the LFSR hash
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore * is located) are in the same cache line as the headers, and
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore * the IP header is 32-bit aligned.
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore SFXGE_CPU_CACHE_SIZE - (encp->enc_rx_prefix_size - 2);
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore sp->s_rx_buffer_size += sp->s_rx_buffer_align;
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore * Place the start of the buffer 2 bytes after a cache line
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore * boundary so that the headers fit into the cache line and
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore * the IP header is 32-bit aligned.
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore hdrlen = sp->s_rx_prefix_size + sizeof (struct ether_header);
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore sp->s_rx_buffer_align = P2ROUNDUP(hdrlen, 4) - hdrlen;
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore sp->s_rx_buffer_size += sp->s_rx_buffer_align;
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore /* Align end of packet buffer for RX DMA end padding */
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore sp->s_rx_buffer_size = P2ROUNDUP(sp->s_rx_buffer_size, align);
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore /* Initialize the receive module */
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore /* Start the receive queue(s) */
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore for (index = 0; index < sip->si_nalloc; index++) {
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore ASSERT3U(sp->s_srp[0]->sr_state, ==, SFXGE_RXQ_STARTED);
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore /* It is sufficient to have Rx scale initialized */
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore ASSERT3U(sp->s_rx_scale.srs_state, ==, SFXGE_RX_SCALE_STARTED);
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore rc = efx_mac_filter_default_rxq_set(sp->s_enp, sp->s_srp[0]->sr_erp,
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore /* Stop the receive queue(s) */
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore while (--index >= 0)
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore /* Tear down the receive module */
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amoresfxge_rx_coalesce_mode_get(sfxge_t *sp, sfxge_rx_coalesce_mode_t *modep)
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amoresfxge_rx_coalesce_mode_set(sfxge_t *sp, sfxge_rx_coalesce_mode_t mode)
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore /* Stop the receive queue(s) */
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore while (--index >= 0) {
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore /* TBD: Flush RXQs in parallel; HW has limit + may need retry */
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore /* Tear down the receive module */
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore ASSERT3U(sip->si_state, ==, SFXGE_INTR_INITIALIZED);
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore for (index = 0; index < sip->si_nalloc; index++) {
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore sfxge_evq_t *sep = sp->s_sep[srp->sr_index];
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore ASSERT3U(sip->si_state, ==, SFXGE_INTR_INITIALIZED);
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore sp->s_rx_coalesce_mode = SFXGE_RX_COALESCE_OFF;
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore /* Tear down the receive queue(s) */
49ef7e0638c8b771d8a136eae78b1c0f99acc8e0Garrett D'Amore while (--index >= 0)