d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw * CDDL HEADER START
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw * The contents of this file are subject to the terms of the
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw * Common Development and Distribution License (the "License").
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw * You may not use this file except in compliance with the License.
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw * See the License for the specific language governing permissions
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw * and limitations under the License.
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw * When distributing Covered Code, include this CDDL HEADER in each
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw * If applicable, add the following below this CDDL HEADER, with the
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw * fields enclosed by brackets "[]" replaced with your own identifying
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw * information: Portions Copyright [yyyy] [name of copyright owner]
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw * CDDL HEADER END
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw * Copyright 2005 Sun Microsystems, Inc. All rights reserved.
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw * Use is subject to license terms.
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw * This file is part of the Chelsio T1 Ethernet driver.
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw * Copyright (C) 2003-2005 Chelsio Communications. All rights reserved.
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw#pragma ident "%Z%%M% %I% %E% SMI"
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xwstatic uint64_t os_freelist_buffer_alloc(ch_t *sa, int sz, mblk_t **mb,
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xwvoid pe_os_free_contig(ch_t *, size_t, void *, uint64_t, ulong_t, ulong_t);
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xwstatic void t1_sge_check_pause(pesge *sge, struct freelQ *Q);
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xwstatic void alloc_freelQ_buffers(pesge *sge, struct freelQ *Q);
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xwstatic void free_cmdQ_buffers(pesge *sge, cmdQ_t *Q, uint32_t credits_pend);
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xwstatic int alloc_rx_resources(pesge *sge, struct sge_params *p);
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xwstatic int alloc_tx_resources(pesge *sge, struct sge_params *p);
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xwstatic inline void setup_ring_params(ch_t *adapter, u64 addr, u32 size,
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xwstatic void configure_sge(pesge *sge, struct sge_params *p);
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xwstatic void free_freelQ_buffers(pesge *sge, struct freelQ *Q);
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xwstatic inline unsigned int jumbo_payload_capacity(pesge *sge);
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw * Local routines.
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xwstatic inline void sge_ring_doorbell(pesge *sge, u32 control_reg);
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xwstatic inline void
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw * NOTES: Must have at least 1 command queue and 1 freelist queue.
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw * PR2928 & PR3309
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw * set default timeout value - 20 msec
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw * we set the initial value to 2 which gurantees at least one tick.
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw /* note that jumbo frame index is inverted for T2 */
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw /* DEBUG only */
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw cmn_err(CE_NOTE, "&sge->freelQ[0]: %p\n", &sge->freelQ[0]);
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw cmn_err(CE_NOTE, "&sge->freelQ[1]: %p\n", &sge->freelQ[1]);
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw cmn_err(CE_NOTE, "&sge->intr_cnt: %p\n", &sge->intr_cnt);
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw /* PR2928 & PR3309 */
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw return (0);
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw * PR2928 & PR3309
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw * call out event from timeout
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw * there is a potential race between the timeout and the close.
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw * unless we protect the timeout, the close could occur at the
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw * same time. Then if the timeout service routine was slow or
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw * interrupted, the sge_stop() could complete with a timeoutID
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw * that has expired, thus letting another timeout occur. If the
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw * service routine was delayed still further, a detach could occur.
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw * the second time could then end up accessing memory that has been
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw * released back to the system. Bad things could then occur. We
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw * set a flag in sge_stop() to tell the service routine not to
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw * issue further timeouts. sge_stop() will block until a timeout
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw * has occured. If the command Q is full then we shouldn't put out
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw /* after first arp */
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw * If we are already in sge_data_in, then we can skip calling
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw * t1_sge_check_pause() this clock cycle. lockstat showed that
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw * we were blocking on the mutex ~ 2% of the time.
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw t1_write_reg_4(sge->obj, A_SG_CONTROL, sge->sge_control);
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw /* PR2928 & PR3309, also need to avoid Pause deadlock */
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw return (0);
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw * Disables SGE queues.
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw /* PR2928 & PR3309, also need to avoid Pause deadlock */
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw /* wait until there's no more outstanding interrupts pending */
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw return (0);
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw ddi_dma_handle_t dh = (ddi_dma_handle_t)sge->cmdQ[qid].cq_dh;
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw * We must exit if we don't have enough free command queue entries
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw * available.
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw * This checksum fix will address a fragmented datagram
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw * checksum error. Which will lead to the next packet after
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw * the last packet with the More fragment bit set having its
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw * checksum corrupted. When the packet reaches this point
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw * the 'flg' variable indicates whether a checksum is needed
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw * or not. The algorithm is as follows, if the current packet
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw * is a More fragment set the count of packets to be checksummed
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw * after it to 3. If it't not and the count of is more than 0
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw * then calculate the checksum in software, if a hardware checksum
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw * was requested. Then decrment the count. Same algorithm applies
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw * Calc Checksum here.
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw * Calc Checksum here.
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw#endif /* TX_CKSUM_FIX */
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw return (1);
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw if (count > 0) {
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw unsigned int i;
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw for (i = 0; i < count; i++) {
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw /* sync from offset to end of cmdQ */
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw * We always ring the doorbell for cmdQ1. For cmdQ0, we only ring
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw * the doorbell if the Q is asleep. There is a natural race, where
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw * the hardware is going to sleep just after we checked, however,
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw * then the interrupt handler will detect the outstanding TX packet
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw * and ring the doorbell for us.
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw/* NOT YET doorbell_pio(sge, F_CMDQ0_ENABLE); */
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw return (0);
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw#define SGE_PL_INTR_MASK (F_PL_INTR_SGE_ERR | F_PL_INTR_SGE_DATA)
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw * Disable SGE error interrupts.
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw t1_write_reg_4(sge->obj, A_PL_ENABLE, val & ~SGE_PL_INTR_MASK);
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw return (0);
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw#define SGE_INT_ENABLE (F_RESPQ_EXHAUSTED | F_RESPQ_OVERFLOW | \
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw * Enable SGE error interrupts.
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw t1_write_reg_4(sge->obj, A_PL_ENABLE, val | SGE_PL_INTR_MASK);
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw return (0);
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw * Clear SGE error interrupts.
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw return (0);
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw#define SGE_INT_FATAL (F_RESPQ_OVERFLOW | F_PACKET_TOO_BIG | F_PACKET_MISMATCH)
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw return (0);
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw * PARAM: sge - SGE instance pointer.
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw * Catch the case where an interrupt arrives
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw /* initial response queue entry */
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw /* pull physical memory of response queue entry into cache */
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw (void) ddi_dma_sync(dh, (off_t)((caddr_t)e - (caddr_t)q),
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw /* pull physical memory of response queue entry into cache */
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw (void) ddi_dma_sync(dh, (off_t)((caddr_t)e - (caddr_t)q),
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw /* the SGE told us one of the free lists is empty */
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw (sge->cmdQ[0].cq_credits > (sge->cmdQ[0].cq_entries_n>>2)) &&
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw (sge->cmdQ[1].cq_credits > (sge->cmdQ[1].cq_entries_n>>2))) {
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw (sge->cmdQ[0].cq_credits > (sge->cmdQ[0].cq_entries_n>>1)) &&
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw (sge->cmdQ[1].cq_credits > (sge->cmdQ[1].cq_entries_n>>1))) {
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw#endif /* CONFIG_CHELSIO_T1_OFFLOAD */
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw /* handle non-data interrupts */
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw * allocate a mblk with DMA mapped mblk.
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw * When checksum offload is enabled, we start the DMA at a 2 byte offset so
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw * the IP header will be aligned. We do this for sparc only.
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xwos_freelist_buffer_alloc(ch_t *sa, int sz, mblk_t **mb, ulong_t *dh)
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw /* get pre-mapped buffer */
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw return ((uint64_t)0);
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw return ((uint64_t)0);
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw /* get pre-mapped buffer */
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw return ((uint64_t)0);
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw return ((uint64_t)0);
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xwstatic inline unsigned int
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xwt1_sge_rx(pesge *sge, struct freelQ *Q, unsigned int len, unsigned int offload)
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw * If pkt size falls below threshold, then we'll copy data to
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw * an blk and reuse mblk.
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw * NOTE that rxoff is 2 for T1 adapters. We align the the start
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw * of the DMA buffer begin at rxoff offset for T1 cards instead of
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw * at the beginning of the buffer, thus the length of the received
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw * data does not include this offset. We therefore always add
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw * SGE_RX_OFFSET to the allocb size so we have space to provide the
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw * offset for the copied data.
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw * If we have Host pause compiled in, then we look at the
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw * free list, if the pause is on and we're not in offload
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw * mode then we drop packets, this is designed to avoid
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw * overwhelming the machine. If the machine is powerfull enough
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw * this will not happen. The 'rx_pkt_drops' will show when
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw * packets are being dropped and how much.
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw /* Ditch the packet and reuse original buffer */
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw * pull physical memory of pkt data into cache
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw * Note that len does not include offset for T1.
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw if (offload == 0) {
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw * create 2 byte offset so IP header aligned on
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw * 4 byte boundry
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw * if hardware inserted 2 byte offset then need to
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw * start copying with extra offset
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw useit = 0; /* mblk copy, don't inc esballoc in use cnt */
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw /* so we can reuse original buffer */
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw /* consume buffer off the ring */
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw * if not offload (tunneled pkt), & hardward padded, then
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw * adjust start of pkt to point to start of data i.e.
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw * skip pad (2 bytes).
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw * pull physical memory of pkt data into cache
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw * Note that len does not include offset for T1.
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw /* set length of data in skb */
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw /* sends pkt upstream to toe layer */
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw "%s: unexpected offloaded packet, cmd %u\n",
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw /* discard packet */
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw "%s: unexpected offloaded packet, cmd %u\n",
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw /* discard paket */
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw /* adjust beginning of data to skip CPL header */
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw /* extract checksum from CPL header here */
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw * bump count of mlbks in used by protocol stack(s)
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw * let the TOE layer have a crack at the packet first.
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw * The TOE may have consumed the packet.
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw#endif /* CONFIG_CHELSIO_T1_OFFLOAD */
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw * NOTE: 14+9 = size of MAC + offset to IP protocol field
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw (ntohs(((struct ether_header *)skb->b_rptr)->ether_type) ==
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw /* allocate new buffers on the free list */
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw return (1);
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xwstatic void
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw * If the number of available credits shrinks below
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw * the Pause on threshold then enable the pause and
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw * try and allocate more buffers.
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw * On the next pass, if there's more credits returned
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw * then check that you've went above the pause
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw * threshold and then disable the pause.
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw adapter->txxg_cfg1 &= ~SUNI1x10GEXP_BITMSK_TXXG_HOSTPAUSE;
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw#endif /* HOST_PAUSE */
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xwstatic void
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw if (mapping == 0) {
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw * Note that for T1, we've started the beginning of
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw * of the buffer by an offset of 2 bytes. We thus
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw * decrement the length to account for this.
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw len += sizeof (*e);
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw * sync freelist entries to physical memory up to
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw * end of the table.
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw /* sync freelist entries that have been modified. */
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw (void) ddi_dma_sync(th, offset, len, DDI_DMA_SYNC_FORDEV);
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xwstatic void
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw if ((sge->freelQ[0].fq_credits > sge->freelQ[0].fq_entries_n >> 2) &&
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw (sge->freelQ[1].fq_credits > sge->freelQ[1].fq_entries_n >> 2)) {
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw /* Clear the F_FL_EXHAUSTED interrupts for now */
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw t1_write_reg_4(sge->obj, A_SG_INTRTIMER, irqholdoff_reg);
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw /* We reenable the Qs to force an Freelist GTS interrupt later */
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw * Frees 'credits_pend' TX buffers and returns the credits to Q->credits.
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw * Free xmit buffers
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xwstatic void
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xwfree_cmdQ_buffers(pesge *sge, struct cmdQ *Q, unsigned int credits_pend)
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw while (i--) {
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw /* if flag set, then toe buffer */
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw#if defined(__sparc)
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw#endif /* __sparc */
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw#else /* CONFIG_CHELSIO_T1_OFFLOAD */
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw#if defined(__sparc)
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw#endif /* __sparc */
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw#endif /* !CONFIG_CHELSIO_T1_OFFLOAD */
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw * Allocates both RX and TX resources and configures the SGE. However,
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw * the hardware is not enabled yet.
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw * rx_pkt_pad is set, if the hardware supports aligning non-offload traffic.
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw * jumbo_fl is set to the index of the freelist containing the jumbo buffers.
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw sge->rx_pkt_pad = t1_is_T1B(sge->obj) ? 0 : SGE_RX_OFFSET;
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw /* if we're a T2 card, then we have hardware offset support */
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw return (-ENOMEM);
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw return (-ENOMEM);
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw * Now that we have sized the free lists calculate the payload
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw * capacity of the large buffers. Other parts of the driver use
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw * this to set the max offload coalescing size so that RX packets
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw * do not overflow our large buffers.
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw return (0);
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw * Allocates basic RX resources, consisting of memory mapped freelist Qs and a
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw * response Q.
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw for (i = 0; i < SGE_FREELQ_N; i++) {
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw * Calculate the buffer sizes for the two free lists. FL0 accommodates
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw * regular sized Ethernet frames, FL1 is sized not to exceed 16K,
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw * including all the sk_buff overhead.
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw * For T1C FL0 and FL1 are reversed.
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw sge->freelQ[1 ^ sge->jumbo_fl].fq_rx_buffer_size = SGE_RX_SM_BUF_SIZE +
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw sizeof (struct cpl_rx_data) +
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw sge->freelQ[sge->jumbo_fl].fq_rx_buffer_size = (16 * 1024) -
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw sge->freelQ[sge->jumbo_fl].fq_rx_buffer_size = sge->obj->ch_bg_buf_sz;
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw sge->respQ.rq_credits_thresh = sge_respq_cnt - (sge_respq_cnt >> 2);
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw sge->respQ.rq_entries = pe_os_malloc_contig_wait_zero(sge->obj,
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw return (0);
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw return (1);
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw * Allocates basic TX resources, consisting of memory mapped command Qs.
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw for (i = 0; i < SGE_CMDQ_N; i++) {
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw /* allocate pre-mapped dma headers */
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw return (0);
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw return (1);
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw * Sets the interrupt latency timer when the adaptive Rx coalescing
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw * is turned off. Do nothing when it is turned on again.
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw * This routine relies on the fact that the caller has already set
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw * the adaptive policy in adapter->sge_params before calling it.
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xwt1_sge_set_coalesce_params(pesge *sge, struct sge_params *p)
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw return (0);
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw * Programs the various SGE registers. However, the engine is not yet enabled,
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw * but sge->sge_control is setup and ready to go.
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xwstatic void
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw setup_ring_params(ap, sge->cmdQ[0].cq_pa, sge->cmdQ[0].cq_entries_n,
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw setup_ring_params(ap, sge->cmdQ[1].cq_pa, sge->cmdQ[1].cq_entries_n,
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw /* The threshold comparison uses <. */
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw t1_write_reg_4(ap, A_SG_FLTHRESHOLD, SGE_RX_SM_BUF_SIZE(ap) -
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw setup_ring_params(ap, sge->respQ.rq_pa, sge->respQ.rq_entries_n,
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw t1_write_reg_4(ap, A_SG_RSPQUEUECREDIT, (u32)sge->respQ.rq_entries_n);
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw sge->sge_control = F_CMDQ0_ENABLE | F_CMDQ1_ENABLE | F_FL0_ENABLE |
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw V_CMDQ_PRIORITY(2) | F_DISABLE_CMDQ1_GTS | F_ISCSI_COALESCE |
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw * if the the following bit is not set, then we'll get an
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw * interrupt everytime command Q 0 goes empty. Since we're
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw * always ringing the doorbell, we can turn it on.
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw * Initialize the SGE Interrupt Timer arrray:
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw * intrtimer[0] = (SGE_INTRTIMER0) usec
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw * intrtimer[0<i<10] = (SGE_INTRTIMER0 + 2*i) usec
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw * intrtimer[10] = (SGE_INTRTIMER1) usec
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw sge->intrtimer[0] = board_info(sge->obj)->clock_core / 1000000;
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw /* Initialize resource timer */
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw /* Finally finish initialization of intrtimer[0] */
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw sge->intrtimer[0] = (uint32_t)(sge->intrtimer[0] * SGE_INTRTIMER0);
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw /* Initialize for a throughput oriented workload */
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xwstatic inline void
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xwsetup_ring_params(ch_t *adapter, u64 addr, u32 size, int base_reg_lo,
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw * Frees RX resources.
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xwstatic void
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw for (i = 0; i < SGE_FREELQ_N; i++) {
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw /* free the freelist queue */
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw * Frees all RX buffers on the freelist Q. The caller must make sure that
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw * the SGE is turned off before calling this function.
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xwstatic void
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw while (credits--) {
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw /* bump in-use count of receive buffers */
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw * note. freeb() callback of esb-alloced mblk will
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw * cause receive buffer to be put back on sa free list.
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw * Free TX resources.
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw * Assumes that SGE is stopped and all interrupts are disabled.
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xwstatic void
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw for (i = 0; i < SGE_CMDQ_N; i++) {
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw * Return the payload capacity of the jumbo free-list buffers.
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xwstatic inline unsigned int jumbo_payload_capacity(pesge *sge)
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw sizeof (struct cpl_rx_data) - sge->rx_pkt_pad - sge->rx_offset);
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw/* PR2928 & PR3309 */
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw/* PR2928 & PR3309 */
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw kstat_named_init(&chkp->respQ_overflow, "respQ_overflow",
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw kstat_named_init(&chkp->freelistQ_empty, "freelistQ_empty",
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw kstat_named_init(&chkp->tx_reclaims[0], "tx_reclaims[0]",
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw kstat_named_init(&chkp->tx_reclaims[1], "tx_reclaims[1]",
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw kstat_named_init(&chkp->tx_msg_pullups, "tx_msg_pullups",
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw kstat_named_init(&chkp->tx_hdr_pullups, "tx_hdr_pullups",
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw kstat_named_init(&chkp->tx_tcp_ip_frag, "tx_tcp_ip_frag",
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw kstat_named_init(&chkp->tx_udp_ip_frag, "tx_udp_ip_frag",
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw kstat_named_init(&chkp->tx_soft_cksums, "tx_soft_cksums",
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw kstat_named_init(&chkp->tx_need_cpl_space, "tx_need_cpl_space",
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw kstat_named_init(&chkp->tx_multi_mblks, "tx_multi_mblks",
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw kstat_named_init(&chkp->tx_no_dvma1, "tx_num_multi_dvma_fails",
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw kstat_named_init(&chkp->tx_no_dvma2, "tx_num_single_dvma_fails",
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw kstat_named_init(&chkp->tx_no_dma1, "tx_num_multi_dma_fails",
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw kstat_named_init(&chkp->tx_no_dma2, "tx_num_single_dma_fails",
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw kstat_named_init(&chkp->rx_pause_spike, "rx_pause_spike",
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw kstat_named_init(&chkp->rx_flbuf_fails, "rx_flbuf_fails",
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw kstat_named_init(&chkp->rx_flbuf_allocs, "rx_flbuf_allocs",
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw kstat_named_init(&chkp->intr_doorbells, "intr_doorbells",
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw kstat_named_init(&chkp->intr1_doorbells, "intr1_doorbells",
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw for (i = 0; i < MBLK_MAX; i++) {
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xwstatic void
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw statsp->respQ_overflow = chkp->respQ_overflow.value.ui32;
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw statsp->freelistQ_empty = chkp->freelistQ_empty.value.ui32;
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw statsp->tx_reclaims[0] = chkp->tx_reclaims[0].value.ui32;
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw statsp->tx_reclaims[1] = chkp->tx_reclaims[1].value.ui32;
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw statsp->tx_msg_pullups = chkp->tx_msg_pullups.value.ui32;
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw statsp->tx_hdr_pullups = chkp->tx_hdr_pullups.value.ui32;
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw statsp->tx_tcp_ip_frag = chkp->tx_tcp_ip_frag.value.ui32;
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw statsp->tx_udp_ip_frag = chkp->tx_udp_ip_frag.value.ui32;
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw statsp->tx_soft_cksums = chkp->tx_soft_cksums.value.ui32;
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw statsp->tx_multi_mblks = chkp->tx_multi_mblks.value.ui32;
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw statsp->rx_pause_spike = chkp->rx_pause_spike.value.ui32;
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw statsp->rx_flbuf_fails = chkp->rx_flbuf_fails.value.ui32;
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw statsp->rx_flbuf_allocs = chkp->rx_flbuf_allocs.value.ui32;
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw statsp->intr_doorbells = chkp->intr_doorbells.value.ui32;
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw statsp->intr1_doorbells = chkp->intr1_doorbells.value.ui32;
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw for (i = 0; i < MBLK_MAX; i++) {
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw chkp->respQ_overflow.value.ui32 = statsp->respQ_overflow;
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw chkp->tx_reclaims[0].value.ui32 = statsp->tx_reclaims[0];
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw chkp->tx_reclaims[1].value.ui32 = statsp->tx_reclaims[1];
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw chkp->tx_msg_pullups.value.ui32 = statsp->tx_msg_pullups;
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw chkp->tx_hdr_pullups.value.ui32 = statsp->tx_hdr_pullups;
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw chkp->tx_tcp_ip_frag.value.ui32 = statsp->tx_tcp_ip_frag;
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw chkp->tx_udp_ip_frag.value.ui32 = statsp->tx_udp_ip_frag;
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw chkp->tx_soft_cksums.value.ui32 = statsp->tx_soft_cksums;
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw chkp->tx_multi_mblks.value.ui32 = statsp->tx_multi_mblks;
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw chkp->rx_pause_spike.value.ui32 = statsp->rx_pause_spike;
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw chkp->intr_doorbells.value.ui32 = statsp->intr_doorbells;
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw for (i = 0; i < MBLK_MAX; i++) {
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw return (0);
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw struct udphdr *udpp = (struct udphdr *)(mp->b_rptr + offset + iplen);
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw src = ((uint32_t)(addrp[0]) << 24) | ((uint32_t)(addrp[1]) << 16) |
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw dst = ((uint32_t)(addrp[0]) << 24) | ((uint32_t)(addrp[1]) << 16) |
d39a76e7b087a3d0927cbe6898dc0a6770fa6c68xw } while (mp);