9da57d7b0ddd8d73b676ce12c040362132cdd538bt/*
9da57d7b0ddd8d73b676ce12c040362132cdd538bt * CDDL HEADER START
9da57d7b0ddd8d73b676ce12c040362132cdd538bt *
9da57d7b0ddd8d73b676ce12c040362132cdd538bt * The contents of this file are subject to the terms of the
9da57d7b0ddd8d73b676ce12c040362132cdd538bt * Common Development and Distribution License (the "License").
9da57d7b0ddd8d73b676ce12c040362132cdd538bt * You may not use this file except in compliance with the License.
9da57d7b0ddd8d73b676ce12c040362132cdd538bt *
da14cebe459d3275048785f25bd869cb09b5307fEric Cheng * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
da14cebe459d3275048785f25bd869cb09b5307fEric Cheng * or http://www.opensolaris.org/os/licensing.
9da57d7b0ddd8d73b676ce12c040362132cdd538bt * See the License for the specific language governing permissions
9da57d7b0ddd8d73b676ce12c040362132cdd538bt * and limitations under the License.
9da57d7b0ddd8d73b676ce12c040362132cdd538bt *
da14cebe459d3275048785f25bd869cb09b5307fEric Cheng * When distributing Covered Code, include this CDDL HEADER in each
da14cebe459d3275048785f25bd869cb09b5307fEric Cheng * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
9da57d7b0ddd8d73b676ce12c040362132cdd538bt * If applicable, add the following below this CDDL HEADER, with the
9da57d7b0ddd8d73b676ce12c040362132cdd538bt * fields enclosed by brackets "[]" replaced with your own identifying
9da57d7b0ddd8d73b676ce12c040362132cdd538bt * information: Portions Copyright [yyyy] [name of copyright owner]
9da57d7b0ddd8d73b676ce12c040362132cdd538bt *
9da57d7b0ddd8d73b676ce12c040362132cdd538bt * CDDL HEADER END
9da57d7b0ddd8d73b676ce12c040362132cdd538bt */
9da57d7b0ddd8d73b676ce12c040362132cdd538bt
da14cebe459d3275048785f25bd869cb09b5307fEric Cheng/*
5b6dd21f5401160f9a62ac2e76a858c2bc105370chenlu chen - Sun Microsystems - Beijing China * Copyright(c) 2007-2010 Intel Corporation. All rights reserved.
5b6dd21f5401160f9a62ac2e76a858c2bc105370chenlu chen - Sun Microsystems - Beijing China */
5b6dd21f5401160f9a62ac2e76a858c2bc105370chenlu chen - Sun Microsystems - Beijing China
5b6dd21f5401160f9a62ac2e76a858c2bc105370chenlu chen - Sun Microsystems - Beijing China/*
5b6dd21f5401160f9a62ac2e76a858c2bc105370chenlu chen - Sun Microsystems - Beijing China * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved.
da14cebe459d3275048785f25bd869cb09b5307fEric Cheng */
9da57d7b0ddd8d73b676ce12c040362132cdd538bt
9da57d7b0ddd8d73b676ce12c040362132cdd538bt#include "ixgbe_sw.h"
9da57d7b0ddd8d73b676ce12c040362132cdd538bt
9da57d7b0ddd8d73b676ce12c040362132cdd538bt/* function prototypes */
ea65739ebec0cbd625d8c89e19592be58fa186a6chenlu chen - Sun Microsystems - Beijing Chinastatic mblk_t *ixgbe_rx_bind(ixgbe_rx_data_t *, uint32_t, uint32_t);
ea65739ebec0cbd625d8c89e19592be58fa186a6chenlu chen - Sun Microsystems - Beijing Chinastatic mblk_t *ixgbe_rx_copy(ixgbe_rx_data_t *, uint32_t, uint32_t);
9da57d7b0ddd8d73b676ce12c040362132cdd538btstatic void ixgbe_rx_assoc_hcksum(mblk_t *, uint32_t);
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing Chinastatic mblk_t *ixgbe_lro_bind(ixgbe_rx_data_t *, uint32_t, uint32_t, uint32_t);
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing Chinastatic mblk_t *ixgbe_lro_copy(ixgbe_rx_data_t *, uint32_t, uint32_t, uint32_t);
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing Chinastatic int ixgbe_lro_get_start(ixgbe_rx_data_t *, uint32_t);
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing Chinastatic uint32_t ixgbe_lro_get_first(ixgbe_rx_data_t *, uint32_t);
9da57d7b0ddd8d73b676ce12c040362132cdd538bt
9da57d7b0ddd8d73b676ce12c040362132cdd538bt#ifndef IXGBE_DEBUG
9da57d7b0ddd8d73b676ce12c040362132cdd538bt#pragma inline(ixgbe_rx_assoc_hcksum)
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China#pragma inline(ixgbe_lro_get_start)
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China#pragma inline(ixgbe_lro_get_first)
9da57d7b0ddd8d73b676ce12c040362132cdd538bt#endif
9da57d7b0ddd8d73b676ce12c040362132cdd538bt
9da57d7b0ddd8d73b676ce12c040362132cdd538bt/*
9da57d7b0ddd8d73b676ce12c040362132cdd538bt * ixgbe_rx_recycle - The call-back function to reclaim rx buffer.
9da57d7b0ddd8d73b676ce12c040362132cdd538bt *
9da57d7b0ddd8d73b676ce12c040362132cdd538bt * This function is called when an mp is freed by the user thru
9da57d7b0ddd8d73b676ce12c040362132cdd538bt * freeb call (Only for mp constructed through desballoc call).
9da57d7b0ddd8d73b676ce12c040362132cdd538bt * It returns back the freed buffer to the free list.
9da57d7b0ddd8d73b676ce12c040362132cdd538bt */
9da57d7b0ddd8d73b676ce12c040362132cdd538btvoid
9da57d7b0ddd8d73b676ce12c040362132cdd538btixgbe_rx_recycle(caddr_t arg)
9da57d7b0ddd8d73b676ce12c040362132cdd538bt{
ea65739ebec0cbd625d8c89e19592be58fa186a6chenlu chen - Sun Microsystems - Beijing China ixgbe_t *ixgbe;
9da57d7b0ddd8d73b676ce12c040362132cdd538bt ixgbe_rx_ring_t *rx_ring;
ea65739ebec0cbd625d8c89e19592be58fa186a6chenlu chen - Sun Microsystems - Beijing China ixgbe_rx_data_t *rx_data;
9da57d7b0ddd8d73b676ce12c040362132cdd538bt rx_control_block_t *recycle_rcb;
9da57d7b0ddd8d73b676ce12c040362132cdd538bt uint32_t free_index;
ea65739ebec0cbd625d8c89e19592be58fa186a6chenlu chen - Sun Microsystems - Beijing China uint32_t ref_cnt;
9da57d7b0ddd8d73b676ce12c040362132cdd538bt
9da57d7b0ddd8d73b676ce12c040362132cdd538bt recycle_rcb = (rx_control_block_t *)(uintptr_t)arg;
ea65739ebec0cbd625d8c89e19592be58fa186a6chenlu chen - Sun Microsystems - Beijing China rx_data = recycle_rcb->rx_data;
ea65739ebec0cbd625d8c89e19592be58fa186a6chenlu chen - Sun Microsystems - Beijing China rx_ring = rx_data->rx_ring;
ea65739ebec0cbd625d8c89e19592be58fa186a6chenlu chen - Sun Microsystems - Beijing China ixgbe = rx_ring->ixgbe;
9da57d7b0ddd8d73b676ce12c040362132cdd538bt
ea65739ebec0cbd625d8c89e19592be58fa186a6chenlu chen - Sun Microsystems - Beijing China if (recycle_rcb->ref_cnt == 0) {
ea65739ebec0cbd625d8c89e19592be58fa186a6chenlu chen - Sun Microsystems - Beijing China /*
ea65739ebec0cbd625d8c89e19592be58fa186a6chenlu chen - Sun Microsystems - Beijing China * This case only happens when rx buffers are being freed
ea65739ebec0cbd625d8c89e19592be58fa186a6chenlu chen - Sun Microsystems - Beijing China * in ixgbe_stop() and freemsg() is called.
ea65739ebec0cbd625d8c89e19592be58fa186a6chenlu chen - Sun Microsystems - Beijing China */
9da57d7b0ddd8d73b676ce12c040362132cdd538bt return;
ea65739ebec0cbd625d8c89e19592be58fa186a6chenlu chen - Sun Microsystems - Beijing China }
9da57d7b0ddd8d73b676ce12c040362132cdd538bt
9da57d7b0ddd8d73b676ce12c040362132cdd538bt ASSERT(recycle_rcb->mp == NULL);
9da57d7b0ddd8d73b676ce12c040362132cdd538bt
9da57d7b0ddd8d73b676ce12c040362132cdd538bt /*
9da57d7b0ddd8d73b676ce12c040362132cdd538bt * Using the recycled data buffer to generate a new mblk
9da57d7b0ddd8d73b676ce12c040362132cdd538bt */
9da57d7b0ddd8d73b676ce12c040362132cdd538bt recycle_rcb->mp = desballoc((unsigned char *)
73cd555c10e70dac413ae4b40de8450a291750acBin Tu - Sun Microsystems - Beijing China recycle_rcb->rx_buf.address,
73cd555c10e70dac413ae4b40de8450a291750acBin Tu - Sun Microsystems - Beijing China recycle_rcb->rx_buf.size,
9da57d7b0ddd8d73b676ce12c040362132cdd538bt 0, &recycle_rcb->free_rtn);
9da57d7b0ddd8d73b676ce12c040362132cdd538bt
9da57d7b0ddd8d73b676ce12c040362132cdd538bt /*
9da57d7b0ddd8d73b676ce12c040362132cdd538bt * Put the recycled rx control block into free list
9da57d7b0ddd8d73b676ce12c040362132cdd538bt */
ea65739ebec0cbd625d8c89e19592be58fa186a6chenlu chen - Sun Microsystems - Beijing China mutex_enter(&rx_data->recycle_lock);
9da57d7b0ddd8d73b676ce12c040362132cdd538bt
ea65739ebec0cbd625d8c89e19592be58fa186a6chenlu chen - Sun Microsystems - Beijing China free_index = rx_data->rcb_tail;
ea65739ebec0cbd625d8c89e19592be58fa186a6chenlu chen - Sun Microsystems - Beijing China ASSERT(rx_data->free_list[free_index] == NULL);
9da57d7b0ddd8d73b676ce12c040362132cdd538bt
ea65739ebec0cbd625d8c89e19592be58fa186a6chenlu chen - Sun Microsystems - Beijing China rx_data->free_list[free_index] = recycle_rcb;
ea65739ebec0cbd625d8c89e19592be58fa186a6chenlu chen - Sun Microsystems - Beijing China rx_data->rcb_tail = NEXT_INDEX(free_index, 1, rx_data->free_list_size);
9da57d7b0ddd8d73b676ce12c040362132cdd538bt
ea65739ebec0cbd625d8c89e19592be58fa186a6chenlu chen - Sun Microsystems - Beijing China mutex_exit(&rx_data->recycle_lock);
9da57d7b0ddd8d73b676ce12c040362132cdd538bt
9da57d7b0ddd8d73b676ce12c040362132cdd538bt /*
9da57d7b0ddd8d73b676ce12c040362132cdd538bt * The atomic operation on the number of the available rx control
9da57d7b0ddd8d73b676ce12c040362132cdd538bt * blocks in the free list is used to make the recycling mutual
9da57d7b0ddd8d73b676ce12c040362132cdd538bt * exclusive with the receiving.
9da57d7b0ddd8d73b676ce12c040362132cdd538bt */
ea65739ebec0cbd625d8c89e19592be58fa186a6chenlu chen - Sun Microsystems - Beijing China atomic_inc_32(&rx_data->rcb_free);
ea65739ebec0cbd625d8c89e19592be58fa186a6chenlu chen - Sun Microsystems - Beijing China ASSERT(rx_data->rcb_free <= rx_data->free_list_size);
ea65739ebec0cbd625d8c89e19592be58fa186a6chenlu chen - Sun Microsystems - Beijing China
ea65739ebec0cbd625d8c89e19592be58fa186a6chenlu chen - Sun Microsystems - Beijing China /*
ea65739ebec0cbd625d8c89e19592be58fa186a6chenlu chen - Sun Microsystems - Beijing China * Considering the case that the interface is unplumbed
ea65739ebec0cbd625d8c89e19592be58fa186a6chenlu chen - Sun Microsystems - Beijing China * and there are still some buffers held by the upper layer.
ea65739ebec0cbd625d8c89e19592be58fa186a6chenlu chen - Sun Microsystems - Beijing China * When the buffer is returned back, we need to free it.
ea65739ebec0cbd625d8c89e19592be58fa186a6chenlu chen - Sun Microsystems - Beijing China */
ea65739ebec0cbd625d8c89e19592be58fa186a6chenlu chen - Sun Microsystems - Beijing China ref_cnt = atomic_dec_32_nv(&recycle_rcb->ref_cnt);
ea65739ebec0cbd625d8c89e19592be58fa186a6chenlu chen - Sun Microsystems - Beijing China if (ref_cnt == 0) {
ea65739ebec0cbd625d8c89e19592be58fa186a6chenlu chen - Sun Microsystems - Beijing China if (recycle_rcb->mp != NULL) {
ea65739ebec0cbd625d8c89e19592be58fa186a6chenlu chen - Sun Microsystems - Beijing China freemsg(recycle_rcb->mp);
ea65739ebec0cbd625d8c89e19592be58fa186a6chenlu chen - Sun Microsystems - Beijing China recycle_rcb->mp = NULL;
ea65739ebec0cbd625d8c89e19592be58fa186a6chenlu chen - Sun Microsystems - Beijing China }
ea65739ebec0cbd625d8c89e19592be58fa186a6chenlu chen - Sun Microsystems - Beijing China
ea65739ebec0cbd625d8c89e19592be58fa186a6chenlu chen - Sun Microsystems - Beijing China ixgbe_free_dma_buffer(&recycle_rcb->rx_buf);
ea65739ebec0cbd625d8c89e19592be58fa186a6chenlu chen - Sun Microsystems - Beijing China
ea65739ebec0cbd625d8c89e19592be58fa186a6chenlu chen - Sun Microsystems - Beijing China mutex_enter(&ixgbe->rx_pending_lock);
ea65739ebec0cbd625d8c89e19592be58fa186a6chenlu chen - Sun Microsystems - Beijing China atomic_dec_32(&rx_data->rcb_pending);
ea65739ebec0cbd625d8c89e19592be58fa186a6chenlu chen - Sun Microsystems - Beijing China atomic_dec_32(&ixgbe->rcb_pending);
ea65739ebec0cbd625d8c89e19592be58fa186a6chenlu chen - Sun Microsystems - Beijing China
ea65739ebec0cbd625d8c89e19592be58fa186a6chenlu chen - Sun Microsystems - Beijing China /*
ea65739ebec0cbd625d8c89e19592be58fa186a6chenlu chen - Sun Microsystems - Beijing China * When there is not any buffer belonging to this rx_data
ea65739ebec0cbd625d8c89e19592be58fa186a6chenlu chen - Sun Microsystems - Beijing China * held by the upper layer, the rx_data can be freed.
ea65739ebec0cbd625d8c89e19592be58fa186a6chenlu chen - Sun Microsystems - Beijing China */
ea65739ebec0cbd625d8c89e19592be58fa186a6chenlu chen - Sun Microsystems - Beijing China if ((rx_data->flag & IXGBE_RX_STOPPED) &&
ea65739ebec0cbd625d8c89e19592be58fa186a6chenlu chen - Sun Microsystems - Beijing China (rx_data->rcb_pending == 0))
ea65739ebec0cbd625d8c89e19592be58fa186a6chenlu chen - Sun Microsystems - Beijing China ixgbe_free_rx_ring_data(rx_data);
ea65739ebec0cbd625d8c89e19592be58fa186a6chenlu chen - Sun Microsystems - Beijing China
ea65739ebec0cbd625d8c89e19592be58fa186a6chenlu chen - Sun Microsystems - Beijing China mutex_exit(&ixgbe->rx_pending_lock);
ea65739ebec0cbd625d8c89e19592be58fa186a6chenlu chen - Sun Microsystems - Beijing China }
9da57d7b0ddd8d73b676ce12c040362132cdd538bt}
9da57d7b0ddd8d73b676ce12c040362132cdd538bt
9da57d7b0ddd8d73b676ce12c040362132cdd538bt/*
9da57d7b0ddd8d73b676ce12c040362132cdd538bt * ixgbe_rx_copy - Use copy to process the received packet.
9da57d7b0ddd8d73b676ce12c040362132cdd538bt *
9da57d7b0ddd8d73b676ce12c040362132cdd538bt * This function will use bcopy to process the packet
9da57d7b0ddd8d73b676ce12c040362132cdd538bt * and send the copied packet upstream.
9da57d7b0ddd8d73b676ce12c040362132cdd538bt */
9da57d7b0ddd8d73b676ce12c040362132cdd538btstatic mblk_t *
ea65739ebec0cbd625d8c89e19592be58fa186a6chenlu chen - Sun Microsystems - Beijing Chinaixgbe_rx_copy(ixgbe_rx_data_t *rx_data, uint32_t index, uint32_t pkt_len)
9da57d7b0ddd8d73b676ce12c040362132cdd538bt{
ea65739ebec0cbd625d8c89e19592be58fa186a6chenlu chen - Sun Microsystems - Beijing China ixgbe_t *ixgbe;
9da57d7b0ddd8d73b676ce12c040362132cdd538bt rx_control_block_t *current_rcb;
9da57d7b0ddd8d73b676ce12c040362132cdd538bt mblk_t *mp;
9da57d7b0ddd8d73b676ce12c040362132cdd538bt
ea65739ebec0cbd625d8c89e19592be58fa186a6chenlu chen - Sun Microsystems - Beijing China ixgbe = rx_data->rx_ring->ixgbe;
ea65739ebec0cbd625d8c89e19592be58fa186a6chenlu chen - Sun Microsystems - Beijing China current_rcb = rx_data->work_list[index];
9da57d7b0ddd8d73b676ce12c040362132cdd538bt
9da57d7b0ddd8d73b676ce12c040362132cdd538bt DMA_SYNC(&current_rcb->rx_buf, DDI_DMA_SYNC_FORKERNEL);
9da57d7b0ddd8d73b676ce12c040362132cdd538bt
9da57d7b0ddd8d73b676ce12c040362132cdd538bt if (ixgbe_check_dma_handle(current_rcb->rx_buf.dma_handle) !=
9da57d7b0ddd8d73b676ce12c040362132cdd538bt DDI_FM_OK) {
ea65739ebec0cbd625d8c89e19592be58fa186a6chenlu chen - Sun Microsystems - Beijing China ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_DEGRADED);
62e6e1adfc8d795f477410703cc7a88dc5b82622Paul Guo atomic_or_32(&ixgbe->ixgbe_state, IXGBE_ERROR);
62e6e1adfc8d795f477410703cc7a88dc5b82622Paul Guo return (NULL);
9da57d7b0ddd8d73b676ce12c040362132cdd538bt }
9da57d7b0ddd8d73b676ce12c040362132cdd538bt
9da57d7b0ddd8d73b676ce12c040362132cdd538bt /*
9da57d7b0ddd8d73b676ce12c040362132cdd538bt * Allocate buffer to receive this packet
9da57d7b0ddd8d73b676ce12c040362132cdd538bt */
9da57d7b0ddd8d73b676ce12c040362132cdd538bt mp = allocb(pkt_len + IPHDR_ALIGN_ROOM, 0);
9da57d7b0ddd8d73b676ce12c040362132cdd538bt if (mp == NULL) {
ea65739ebec0cbd625d8c89e19592be58fa186a6chenlu chen - Sun Microsystems - Beijing China ixgbe_log(ixgbe, "ixgbe_rx_copy: allocate buffer failed");
9da57d7b0ddd8d73b676ce12c040362132cdd538bt return (NULL);
9da57d7b0ddd8d73b676ce12c040362132cdd538bt }
9da57d7b0ddd8d73b676ce12c040362132cdd538bt
9da57d7b0ddd8d73b676ce12c040362132cdd538bt /*
9da57d7b0ddd8d73b676ce12c040362132cdd538bt * Copy the data received into the new cluster
9da57d7b0ddd8d73b676ce12c040362132cdd538bt */
9da57d7b0ddd8d73b676ce12c040362132cdd538bt mp->b_rptr += IPHDR_ALIGN_ROOM;
9da57d7b0ddd8d73b676ce12c040362132cdd538bt bcopy(current_rcb->rx_buf.address, mp->b_rptr, pkt_len);
9da57d7b0ddd8d73b676ce12c040362132cdd538bt mp->b_wptr = mp->b_rptr + pkt_len;
9da57d7b0ddd8d73b676ce12c040362132cdd538bt
9da57d7b0ddd8d73b676ce12c040362132cdd538bt return (mp);
9da57d7b0ddd8d73b676ce12c040362132cdd538bt}
9da57d7b0ddd8d73b676ce12c040362132cdd538bt
9da57d7b0ddd8d73b676ce12c040362132cdd538bt/*
9da57d7b0ddd8d73b676ce12c040362132cdd538bt * ixgbe_rx_bind - Use existing DMA buffer to build mblk for receiving.
9da57d7b0ddd8d73b676ce12c040362132cdd538bt *
9da57d7b0ddd8d73b676ce12c040362132cdd538bt * This function will use pre-bound DMA buffer to receive the packet
9da57d7b0ddd8d73b676ce12c040362132cdd538bt * and build mblk that will be sent upstream.
9da57d7b0ddd8d73b676ce12c040362132cdd538bt */
9da57d7b0ddd8d73b676ce12c040362132cdd538btstatic mblk_t *
ea65739ebec0cbd625d8c89e19592be58fa186a6chenlu chen - Sun Microsystems - Beijing Chinaixgbe_rx_bind(ixgbe_rx_data_t *rx_data, uint32_t index, uint32_t pkt_len)
9da57d7b0ddd8d73b676ce12c040362132cdd538bt{
9da57d7b0ddd8d73b676ce12c040362132cdd538bt rx_control_block_t *current_rcb;
9da57d7b0ddd8d73b676ce12c040362132cdd538bt rx_control_block_t *free_rcb;
9da57d7b0ddd8d73b676ce12c040362132cdd538bt uint32_t free_index;
9da57d7b0ddd8d73b676ce12c040362132cdd538bt mblk_t *mp;
ea65739ebec0cbd625d8c89e19592be58fa186a6chenlu chen - Sun Microsystems - Beijing China ixgbe_t *ixgbe = rx_data->rx_ring->ixgbe;
9da57d7b0ddd8d73b676ce12c040362132cdd538bt
9da57d7b0ddd8d73b676ce12c040362132cdd538bt /*
9da57d7b0ddd8d73b676ce12c040362132cdd538bt * If the free list is empty, we cannot proceed to send
9da57d7b0ddd8d73b676ce12c040362132cdd538bt * the current DMA buffer upstream. We'll have to return
9da57d7b0ddd8d73b676ce12c040362132cdd538bt * and use bcopy to process the packet.
9da57d7b0ddd8d73b676ce12c040362132cdd538bt */
ea65739ebec0cbd625d8c89e19592be58fa186a6chenlu chen - Sun Microsystems - Beijing China if (ixgbe_atomic_reserve(&rx_data->rcb_free, 1) < 0)
9da57d7b0ddd8d73b676ce12c040362132cdd538bt return (NULL);
9da57d7b0ddd8d73b676ce12c040362132cdd538bt
ea65739ebec0cbd625d8c89e19592be58fa186a6chenlu chen - Sun Microsystems - Beijing China current_rcb = rx_data->work_list[index];
9da57d7b0ddd8d73b676ce12c040362132cdd538bt /*
9da57d7b0ddd8d73b676ce12c040362132cdd538bt * If the mp of the rx control block is NULL, try to do
9da57d7b0ddd8d73b676ce12c040362132cdd538bt * desballoc again.
9da57d7b0ddd8d73b676ce12c040362132cdd538bt */
9da57d7b0ddd8d73b676ce12c040362132cdd538bt if (current_rcb->mp == NULL) {
9da57d7b0ddd8d73b676ce12c040362132cdd538bt current_rcb->mp = desballoc((unsigned char *)
73cd555c10e70dac413ae4b40de8450a291750acBin Tu - Sun Microsystems - Beijing China current_rcb->rx_buf.address,
73cd555c10e70dac413ae4b40de8450a291750acBin Tu - Sun Microsystems - Beijing China current_rcb->rx_buf.size,
9da57d7b0ddd8d73b676ce12c040362132cdd538bt 0, &current_rcb->free_rtn);
9da57d7b0ddd8d73b676ce12c040362132cdd538bt /*
9da57d7b0ddd8d73b676ce12c040362132cdd538bt * If it is failed to built a mblk using the current
9da57d7b0ddd8d73b676ce12c040362132cdd538bt * DMA buffer, we have to return and use bcopy to
9da57d7b0ddd8d73b676ce12c040362132cdd538bt * process the packet.
9da57d7b0ddd8d73b676ce12c040362132cdd538bt */
73cd555c10e70dac413ae4b40de8450a291750acBin Tu - Sun Microsystems - Beijing China if (current_rcb->mp == NULL) {
ea65739ebec0cbd625d8c89e19592be58fa186a6chenlu chen - Sun Microsystems - Beijing China atomic_inc_32(&rx_data->rcb_free);
9da57d7b0ddd8d73b676ce12c040362132cdd538bt return (NULL);
9da57d7b0ddd8d73b676ce12c040362132cdd538bt }
9da57d7b0ddd8d73b676ce12c040362132cdd538bt }
9da57d7b0ddd8d73b676ce12c040362132cdd538bt /*
9da57d7b0ddd8d73b676ce12c040362132cdd538bt * Sync up the data received
9da57d7b0ddd8d73b676ce12c040362132cdd538bt */
9da57d7b0ddd8d73b676ce12c040362132cdd538bt DMA_SYNC(&current_rcb->rx_buf, DDI_DMA_SYNC_FORKERNEL);
9da57d7b0ddd8d73b676ce12c040362132cdd538bt
9da57d7b0ddd8d73b676ce12c040362132cdd538bt if (ixgbe_check_dma_handle(current_rcb->rx_buf.dma_handle) !=
9da57d7b0ddd8d73b676ce12c040362132cdd538bt DDI_FM_OK) {
ea65739ebec0cbd625d8c89e19592be58fa186a6chenlu chen - Sun Microsystems - Beijing China ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_DEGRADED);
62e6e1adfc8d795f477410703cc7a88dc5b82622Paul Guo atomic_inc_32(&rx_data->rcb_free);
62e6e1adfc8d795f477410703cc7a88dc5b82622Paul Guo atomic_or_32(&ixgbe->ixgbe_state, IXGBE_ERROR);
62e6e1adfc8d795f477410703cc7a88dc5b82622Paul Guo return (NULL);
9da57d7b0ddd8d73b676ce12c040362132cdd538bt }
9da57d7b0ddd8d73b676ce12c040362132cdd538bt
9da57d7b0ddd8d73b676ce12c040362132cdd538bt mp = current_rcb->mp;
9da57d7b0ddd8d73b676ce12c040362132cdd538bt current_rcb->mp = NULL;
ea65739ebec0cbd625d8c89e19592be58fa186a6chenlu chen - Sun Microsystems - Beijing China atomic_inc_32(&current_rcb->ref_cnt);
9da57d7b0ddd8d73b676ce12c040362132cdd538bt
9da57d7b0ddd8d73b676ce12c040362132cdd538bt mp->b_wptr = mp->b_rptr + pkt_len;
9da57d7b0ddd8d73b676ce12c040362132cdd538bt mp->b_next = mp->b_cont = NULL;
9da57d7b0ddd8d73b676ce12c040362132cdd538bt
9da57d7b0ddd8d73b676ce12c040362132cdd538bt /*
9da57d7b0ddd8d73b676ce12c040362132cdd538bt * Strip off one free rx control block from the free list
9da57d7b0ddd8d73b676ce12c040362132cdd538bt */
ea65739ebec0cbd625d8c89e19592be58fa186a6chenlu chen - Sun Microsystems - Beijing China free_index = rx_data->rcb_head;
ea65739ebec0cbd625d8c89e19592be58fa186a6chenlu chen - Sun Microsystems - Beijing China free_rcb = rx_data->free_list[free_index];
9da57d7b0ddd8d73b676ce12c040362132cdd538bt ASSERT(free_rcb != NULL);
ea65739ebec0cbd625d8c89e19592be58fa186a6chenlu chen - Sun Microsystems - Beijing China rx_data->free_list[free_index] = NULL;
ea65739ebec0cbd625d8c89e19592be58fa186a6chenlu chen - Sun Microsystems - Beijing China rx_data->rcb_head = NEXT_INDEX(free_index, 1, rx_data->free_list_size);
9da57d7b0ddd8d73b676ce12c040362132cdd538bt
9da57d7b0ddd8d73b676ce12c040362132cdd538bt /*
9da57d7b0ddd8d73b676ce12c040362132cdd538bt * Put the rx control block to the work list
9da57d7b0ddd8d73b676ce12c040362132cdd538bt */
ea65739ebec0cbd625d8c89e19592be58fa186a6chenlu chen - Sun Microsystems - Beijing China rx_data->work_list[index] = free_rcb;
9da57d7b0ddd8d73b676ce12c040362132cdd538bt
9da57d7b0ddd8d73b676ce12c040362132cdd538bt return (mp);
9da57d7b0ddd8d73b676ce12c040362132cdd538bt}
9da57d7b0ddd8d73b676ce12c040362132cdd538bt
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China/*
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China * ixgbe_lro_bind - Use existing DMA buffer to build LRO mblk for receiving.
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China *
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China * This function will use pre-bound DMA buffers to receive the packet
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China * and build LRO mblk that will be sent upstream.
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China */
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing Chinastatic mblk_t *
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing Chinaixgbe_lro_bind(ixgbe_rx_data_t *rx_data, uint32_t lro_start,
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China uint32_t lro_num, uint32_t pkt_len)
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China{
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China rx_control_block_t *current_rcb;
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China union ixgbe_adv_rx_desc *current_rbd;
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China rx_control_block_t *free_rcb;
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China uint32_t free_index;
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China int lro_next;
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China uint32_t last_pkt_len;
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China uint32_t i;
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China mblk_t *mp;
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China mblk_t *mblk_head;
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China mblk_t **mblk_tail;
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China ixgbe_t *ixgbe = rx_data->rx_ring->ixgbe;
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China /*
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China * If the free list is empty, we cannot proceed to send
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China * the current DMA buffer upstream. We'll have to return
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China * and use bcopy to process the packet.
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China */
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China if (ixgbe_atomic_reserve(&rx_data->rcb_free, lro_num) < 0)
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China return (NULL);
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China current_rcb = rx_data->work_list[lro_start];
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China /*
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China * If any one of the rx data blocks can not support
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China * lro bind operation, We'll have to return and use
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China * bcopy to process the lro packet.
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China */
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China for (i = lro_num; i > 0; i--) {
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China /*
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China * Sync up the data received
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China */
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China DMA_SYNC(&current_rcb->rx_buf, DDI_DMA_SYNC_FORKERNEL);
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China if (ixgbe_check_dma_handle(current_rcb->rx_buf.dma_handle) !=
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China DDI_FM_OK) {
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_DEGRADED);
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China atomic_add_32(&rx_data->rcb_free, lro_num);
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China atomic_or_32(&ixgbe->ixgbe_state, IXGBE_ERROR);
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China return (NULL);
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China }
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China /*
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China * If the mp of the rx control block is NULL, try to do
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China * desballoc again.
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China */
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China if (current_rcb->mp == NULL) {
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China current_rcb->mp = desballoc((unsigned char *)
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China current_rcb->rx_buf.address,
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China current_rcb->rx_buf.size,
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China 0, &current_rcb->free_rtn);
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China /*
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China * If it is failed to built a mblk using the current
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China * DMA buffer, we have to return and use bcopy to
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China * process the packet.
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China */
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China if (current_rcb->mp == NULL) {
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China atomic_add_32(&rx_data->rcb_free, lro_num);
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China return (NULL);
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China }
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China }
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China if (current_rcb->lro_next != -1)
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China lro_next = current_rcb->lro_next;
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China current_rcb = rx_data->work_list[lro_next];
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China }
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China mblk_head = NULL;
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China mblk_tail = &mblk_head;
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China lro_next = lro_start;
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China last_pkt_len = pkt_len - ixgbe->rx_buf_size * (lro_num - 1);
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China current_rcb = rx_data->work_list[lro_next];
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China current_rbd = &rx_data->rbd_ring[lro_next];
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China while (lro_num --) {
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China mp = current_rcb->mp;
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China current_rcb->mp = NULL;
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China atomic_inc_32(&current_rcb->ref_cnt);
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China if (lro_num != 0)
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China mp->b_wptr = mp->b_rptr + ixgbe->rx_buf_size;
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China else
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China mp->b_wptr = mp->b_rptr + last_pkt_len;
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China mp->b_next = mp->b_cont = NULL;
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China *mblk_tail = mp;
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China mblk_tail = &mp->b_cont;
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China /*
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China * Strip off one free rx control block from the free list
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China */
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China free_index = rx_data->rcb_head;
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China free_rcb = rx_data->free_list[free_index];
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China ASSERT(free_rcb != NULL);
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China rx_data->free_list[free_index] = NULL;
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China rx_data->rcb_head = NEXT_INDEX(free_index, 1,
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China rx_data->free_list_size);
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China /*
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China * Put the rx control block to the work list
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China */
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China rx_data->work_list[lro_next] = free_rcb;
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China lro_next = current_rcb->lro_next;
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China current_rcb->lro_next = -1;
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China current_rcb->lro_prev = -1;
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China current_rcb->lro_pkt = B_FALSE;
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China current_rbd->read.pkt_addr = free_rcb->rx_buf.dma_address;
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China current_rbd->read.hdr_addr = 0;
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China if (lro_next == -1)
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China break;
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China current_rcb = rx_data->work_list[lro_next];
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China current_rbd = &rx_data->rbd_ring[lro_next];
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China }
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China return (mblk_head);
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China}
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China/*
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China * ixgbe_lro_copy - Use copy to process the received LRO packet.
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China *
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China * This function will use bcopy to process the LRO packet
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China * and send the copied packet upstream.
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China */
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing Chinastatic mblk_t *
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing Chinaixgbe_lro_copy(ixgbe_rx_data_t *rx_data, uint32_t lro_start,
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China uint32_t lro_num, uint32_t pkt_len)
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China{
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China ixgbe_t *ixgbe;
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China rx_control_block_t *current_rcb;
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China union ixgbe_adv_rx_desc *current_rbd;
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China mblk_t *mp;
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China uint32_t last_pkt_len;
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China int lro_next;
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China uint32_t i;
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China ixgbe = rx_data->rx_ring->ixgbe;
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China /*
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China * Allocate buffer to receive this LRO packet
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China */
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China mp = allocb(pkt_len + IPHDR_ALIGN_ROOM, 0);
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China if (mp == NULL) {
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China ixgbe_log(ixgbe, "LRO copy MP alloc failed");
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China return (NULL);
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China }
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China current_rcb = rx_data->work_list[lro_start];
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China /*
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China * Sync up the LRO packet data received
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China */
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China for (i = lro_num; i > 0; i--) {
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China DMA_SYNC(&current_rcb->rx_buf, DDI_DMA_SYNC_FORKERNEL);
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China if (ixgbe_check_dma_handle(current_rcb->rx_buf.dma_handle) !=
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China DDI_FM_OK) {
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_DEGRADED);
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China atomic_or_32(&ixgbe->ixgbe_state, IXGBE_ERROR);
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China return (NULL);
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China }
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China if (current_rcb->lro_next != -1)
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China lro_next = current_rcb->lro_next;
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China current_rcb = rx_data->work_list[lro_next];
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China }
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China lro_next = lro_start;
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China current_rcb = rx_data->work_list[lro_next];
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China current_rbd = &rx_data->rbd_ring[lro_next];
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China last_pkt_len = pkt_len - ixgbe->rx_buf_size * (lro_num - 1);
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China /*
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China * Copy the data received into the new cluster
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China */
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China mp->b_rptr += IPHDR_ALIGN_ROOM;
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China mp->b_wptr += IPHDR_ALIGN_ROOM;
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China while (lro_num --) {
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China if (lro_num != 0) {
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China bcopy(current_rcb->rx_buf.address, mp->b_wptr,
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China ixgbe->rx_buf_size);
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China mp->b_wptr += ixgbe->rx_buf_size;
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China } else {
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China bcopy(current_rcb->rx_buf.address, mp->b_wptr,
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China last_pkt_len);
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China mp->b_wptr += last_pkt_len;
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China }
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China lro_next = current_rcb->lro_next;
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China current_rcb->lro_next = -1;
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China current_rcb->lro_prev = -1;
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China current_rcb->lro_pkt = B_FALSE;
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China current_rbd->read.pkt_addr = current_rcb->rx_buf.dma_address;
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China current_rbd->read.hdr_addr = 0;
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China if (lro_next == -1)
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China break;
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China current_rcb = rx_data->work_list[lro_next];
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China current_rbd = &rx_data->rbd_ring[lro_next];
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China }
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China return (mp);
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China}
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China/*
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China * ixgbe_lro_get_start - get the start rcb index in one LRO packet
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China */
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing Chinastatic int
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing Chinaixgbe_lro_get_start(ixgbe_rx_data_t *rx_data, uint32_t rx_next)
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China{
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China int lro_prev;
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China int lro_start;
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China uint32_t lro_num = 1;
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China rx_control_block_t *prev_rcb;
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China rx_control_block_t *current_rcb = rx_data->work_list[rx_next];
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China lro_prev = current_rcb->lro_prev;
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China while (lro_prev != -1) {
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China lro_num ++;
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China prev_rcb = rx_data->work_list[lro_prev];
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China lro_start = lro_prev;
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China lro_prev = prev_rcb->lro_prev;
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China }
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China rx_data->lro_num = lro_num;
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China return (lro_start);
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China}
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China/*
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China * ixgbe_lro_get_first - get the first LRO rcb index
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China */
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing Chinastatic uint32_t
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing Chinaixgbe_lro_get_first(ixgbe_rx_data_t *rx_data, uint32_t rx_next)
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China{
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China rx_control_block_t *current_rcb;
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China uint32_t lro_first;
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China lro_first = rx_data->lro_first;
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China current_rcb = rx_data->work_list[lro_first];
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China while ((!current_rcb->lro_pkt) && (lro_first != rx_next)) {
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China lro_first = NEXT_INDEX(lro_first, 1, rx_data->ring_size);
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China current_rcb = rx_data->work_list[lro_first];
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China }
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China rx_data->lro_first = lro_first;
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China return (lro_first);
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China}
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China
9da57d7b0ddd8d73b676ce12c040362132cdd538bt/*
9da57d7b0ddd8d73b676ce12c040362132cdd538bt * ixgbe_rx_assoc_hcksum - Check the rx hardware checksum status and associate
9da57d7b0ddd8d73b676ce12c040362132cdd538bt * the hcksum flags.
9da57d7b0ddd8d73b676ce12c040362132cdd538bt */
9da57d7b0ddd8d73b676ce12c040362132cdd538btstatic void
9da57d7b0ddd8d73b676ce12c040362132cdd538btixgbe_rx_assoc_hcksum(mblk_t *mp, uint32_t status_error)
9da57d7b0ddd8d73b676ce12c040362132cdd538bt{
9da57d7b0ddd8d73b676ce12c040362132cdd538bt uint32_t hcksum_flags = 0;
9da57d7b0ddd8d73b676ce12c040362132cdd538bt
9da57d7b0ddd8d73b676ce12c040362132cdd538bt /*
9da57d7b0ddd8d73b676ce12c040362132cdd538bt * Check TCP/UDP checksum
9da57d7b0ddd8d73b676ce12c040362132cdd538bt */
9da57d7b0ddd8d73b676ce12c040362132cdd538bt if ((status_error & IXGBE_RXD_STAT_L4CS) &&
9da57d7b0ddd8d73b676ce12c040362132cdd538bt !(status_error & IXGBE_RXDADV_ERR_TCPE))
0dc2366f7b9f9f36e10909b1e95edbf2a261c2acVenugopal Iyer hcksum_flags |= HCK_FULLCKSUM_OK;
9da57d7b0ddd8d73b676ce12c040362132cdd538bt
9da57d7b0ddd8d73b676ce12c040362132cdd538bt /*
9da57d7b0ddd8d73b676ce12c040362132cdd538bt * Check IP Checksum
9da57d7b0ddd8d73b676ce12c040362132cdd538bt */
9da57d7b0ddd8d73b676ce12c040362132cdd538bt if ((status_error & IXGBE_RXD_STAT_IPCS) &&
9da57d7b0ddd8d73b676ce12c040362132cdd538bt !(status_error & IXGBE_RXDADV_ERR_IPE))
0dc2366f7b9f9f36e10909b1e95edbf2a261c2acVenugopal Iyer hcksum_flags |= HCK_IPV4_HDRCKSUM_OK;
9da57d7b0ddd8d73b676ce12c040362132cdd538bt
9da57d7b0ddd8d73b676ce12c040362132cdd538bt if (hcksum_flags != 0) {
0dc2366f7b9f9f36e10909b1e95edbf2a261c2acVenugopal Iyer mac_hcksum_set(mp, 0, 0, 0, 0, hcksum_flags);
9da57d7b0ddd8d73b676ce12c040362132cdd538bt }
9da57d7b0ddd8d73b676ce12c040362132cdd538bt}
9da57d7b0ddd8d73b676ce12c040362132cdd538bt
9da57d7b0ddd8d73b676ce12c040362132cdd538bt/*
da14cebe459d3275048785f25bd869cb09b5307fEric Cheng * ixgbe_ring_rx - Receive the data of one ring.
9da57d7b0ddd8d73b676ce12c040362132cdd538bt *
9da57d7b0ddd8d73b676ce12c040362132cdd538bt * This function goes throught h/w descriptor in one specified rx ring,
9da57d7b0ddd8d73b676ce12c040362132cdd538bt * receives the data if the descriptor status shows the data is ready.
9da57d7b0ddd8d73b676ce12c040362132cdd538bt * It returns a chain of mblks containing the received data, to be
9da57d7b0ddd8d73b676ce12c040362132cdd538bt * passed up to mac_rx().
9da57d7b0ddd8d73b676ce12c040362132cdd538bt */
9da57d7b0ddd8d73b676ce12c040362132cdd538btmblk_t *
da14cebe459d3275048785f25bd869cb09b5307fEric Chengixgbe_ring_rx(ixgbe_rx_ring_t *rx_ring, int poll_bytes)
9da57d7b0ddd8d73b676ce12c040362132cdd538bt{
9da57d7b0ddd8d73b676ce12c040362132cdd538bt union ixgbe_adv_rx_desc *current_rbd;
9da57d7b0ddd8d73b676ce12c040362132cdd538bt rx_control_block_t *current_rcb;
9da57d7b0ddd8d73b676ce12c040362132cdd538bt mblk_t *mp;
9da57d7b0ddd8d73b676ce12c040362132cdd538bt mblk_t *mblk_head;
9da57d7b0ddd8d73b676ce12c040362132cdd538bt mblk_t **mblk_tail;
9da57d7b0ddd8d73b676ce12c040362132cdd538bt uint32_t rx_next;
9da57d7b0ddd8d73b676ce12c040362132cdd538bt uint32_t rx_tail;
9da57d7b0ddd8d73b676ce12c040362132cdd538bt uint32_t pkt_len;
9da57d7b0ddd8d73b676ce12c040362132cdd538bt uint32_t status_error;
9da57d7b0ddd8d73b676ce12c040362132cdd538bt uint32_t pkt_num;
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China uint32_t rsc_cnt;
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China uint32_t lro_first;
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China uint32_t lro_start;
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China uint32_t lro_next;
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China boolean_t lro_eop;
da14cebe459d3275048785f25bd869cb09b5307fEric Cheng uint32_t received_bytes;
9da57d7b0ddd8d73b676ce12c040362132cdd538bt ixgbe_t *ixgbe = rx_ring->ixgbe;
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China ixgbe_rx_data_t *rx_data;
9da57d7b0ddd8d73b676ce12c040362132cdd538bt
62e6e1adfc8d795f477410703cc7a88dc5b82622Paul Guo if ((ixgbe->ixgbe_state & IXGBE_SUSPENDED) ||
62e6e1adfc8d795f477410703cc7a88dc5b82622Paul Guo (ixgbe->ixgbe_state & IXGBE_ERROR) ||
5b6dd21f5401160f9a62ac2e76a858c2bc105370chenlu chen - Sun Microsystems - Beijing China (ixgbe->ixgbe_state & IXGBE_OVERTEMP) ||
62e6e1adfc8d795f477410703cc7a88dc5b82622Paul Guo !(ixgbe->ixgbe_state & IXGBE_STARTED))
62e6e1adfc8d795f477410703cc7a88dc5b82622Paul Guo return (NULL);
62e6e1adfc8d795f477410703cc7a88dc5b82622Paul Guo
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China rx_data = rx_ring->rx_data;
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China lro_eop = B_FALSE;
9da57d7b0ddd8d73b676ce12c040362132cdd538bt mblk_head = NULL;
9da57d7b0ddd8d73b676ce12c040362132cdd538bt mblk_tail = &mblk_head;
9da57d7b0ddd8d73b676ce12c040362132cdd538bt
9da57d7b0ddd8d73b676ce12c040362132cdd538bt /*
9da57d7b0ddd8d73b676ce12c040362132cdd538bt * Sync the receive descriptors before accepting the packets
9da57d7b0ddd8d73b676ce12c040362132cdd538bt */
ea65739ebec0cbd625d8c89e19592be58fa186a6chenlu chen - Sun Microsystems - Beijing China DMA_SYNC(&rx_data->rbd_area, DDI_DMA_SYNC_FORKERNEL);
9da57d7b0ddd8d73b676ce12c040362132cdd538bt
ea65739ebec0cbd625d8c89e19592be58fa186a6chenlu chen - Sun Microsystems - Beijing China if (ixgbe_check_dma_handle(rx_data->rbd_area.dma_handle) != DDI_FM_OK) {
ea65739ebec0cbd625d8c89e19592be58fa186a6chenlu chen - Sun Microsystems - Beijing China ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_DEGRADED);
62e6e1adfc8d795f477410703cc7a88dc5b82622Paul Guo atomic_or_32(&ixgbe->ixgbe_state, IXGBE_ERROR);
62e6e1adfc8d795f477410703cc7a88dc5b82622Paul Guo return (NULL);
9da57d7b0ddd8d73b676ce12c040362132cdd538bt }
9da57d7b0ddd8d73b676ce12c040362132cdd538bt
9da57d7b0ddd8d73b676ce12c040362132cdd538bt /*
9da57d7b0ddd8d73b676ce12c040362132cdd538bt * Get the start point of rx bd ring which should be examined
9da57d7b0ddd8d73b676ce12c040362132cdd538bt * during this cycle.
9da57d7b0ddd8d73b676ce12c040362132cdd538bt */
ea65739ebec0cbd625d8c89e19592be58fa186a6chenlu chen - Sun Microsystems - Beijing China rx_next = rx_data->rbd_next;
ea65739ebec0cbd625d8c89e19592be58fa186a6chenlu chen - Sun Microsystems - Beijing China current_rbd = &rx_data->rbd_ring[rx_next];
da14cebe459d3275048785f25bd869cb09b5307fEric Cheng received_bytes = 0;
9da57d7b0ddd8d73b676ce12c040362132cdd538bt pkt_num = 0;
9da57d7b0ddd8d73b676ce12c040362132cdd538bt status_error = current_rbd->wb.upper.status_error;
9da57d7b0ddd8d73b676ce12c040362132cdd538bt while (status_error & IXGBE_RXD_STAT_DD) {
9da57d7b0ddd8d73b676ce12c040362132cdd538bt /*
9da57d7b0ddd8d73b676ce12c040362132cdd538bt * If adapter has found errors, but the error
9da57d7b0ddd8d73b676ce12c040362132cdd538bt * is hardware checksum error, this does not discard the
9da57d7b0ddd8d73b676ce12c040362132cdd538bt * packet: let upper layer compute the checksum;
9da57d7b0ddd8d73b676ce12c040362132cdd538bt * Otherwise discard the packet.
9da57d7b0ddd8d73b676ce12c040362132cdd538bt */
9da57d7b0ddd8d73b676ce12c040362132cdd538bt if ((status_error & IXGBE_RXDADV_ERR_FRAME_ERR_MASK) ||
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China ((!ixgbe->lro_enable) &&
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China (!(status_error & IXGBE_RXD_STAT_EOP)))) {
9da57d7b0ddd8d73b676ce12c040362132cdd538bt IXGBE_DEBUG_STAT(rx_ring->stat_frame_error);
9da57d7b0ddd8d73b676ce12c040362132cdd538bt goto rx_discard;
9da57d7b0ddd8d73b676ce12c040362132cdd538bt }
9da57d7b0ddd8d73b676ce12c040362132cdd538bt
9da57d7b0ddd8d73b676ce12c040362132cdd538bt IXGBE_DEBUG_STAT_COND(rx_ring->stat_cksum_error,
9da57d7b0ddd8d73b676ce12c040362132cdd538bt (status_error & IXGBE_RXDADV_ERR_TCPE) ||
9da57d7b0ddd8d73b676ce12c040362132cdd538bt (status_error & IXGBE_RXDADV_ERR_IPE));
9da57d7b0ddd8d73b676ce12c040362132cdd538bt
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China if (ixgbe->lro_enable) {
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China rsc_cnt = (current_rbd->wb.lower.lo_dword.data &
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China IXGBE_RXDADV_RSCCNT_MASK) >>
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China IXGBE_RXDADV_RSCCNT_SHIFT;
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China if (rsc_cnt != 0) {
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China if (status_error & IXGBE_RXD_STAT_EOP) {
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China pkt_len = current_rbd->wb.upper.length;
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China if (rx_data->work_list[rx_next]->
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China lro_prev != -1) {
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China lro_start =
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China ixgbe_lro_get_start(rx_data,
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China rx_next);
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China ixgbe->lro_pkt_count++;
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China pkt_len +=
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China (rx_data->lro_num - 1) *
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China ixgbe->rx_buf_size;
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China lro_eop = B_TRUE;
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China }
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China } else {
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China lro_next = (status_error &
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China IXGBE_RXDADV_NEXTP_MASK) >>
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China IXGBE_RXDADV_NEXTP_SHIFT;
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China rx_data->work_list[lro_next]->lro_prev
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China = rx_next;
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China rx_data->work_list[rx_next]->lro_next =
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China lro_next;
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China rx_data->work_list[rx_next]->lro_pkt =
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China B_TRUE;
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China goto rx_discard;
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China }
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China } else {
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China pkt_len = current_rbd->wb.upper.length;
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China }
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China } else {
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China pkt_len = current_rbd->wb.upper.length;
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China }
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China
da14cebe459d3275048785f25bd869cb09b5307fEric Cheng
da14cebe459d3275048785f25bd869cb09b5307fEric Cheng if ((poll_bytes != IXGBE_POLL_NULL) &&
da14cebe459d3275048785f25bd869cb09b5307fEric Cheng ((received_bytes + pkt_len) > poll_bytes))
da14cebe459d3275048785f25bd869cb09b5307fEric Cheng break;
da14cebe459d3275048785f25bd869cb09b5307fEric Cheng
da14cebe459d3275048785f25bd869cb09b5307fEric Cheng received_bytes += pkt_len;
9da57d7b0ddd8d73b676ce12c040362132cdd538bt mp = NULL;
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China
9da57d7b0ddd8d73b676ce12c040362132cdd538bt /*
9da57d7b0ddd8d73b676ce12c040362132cdd538bt * For packets with length more than the copy threshold,
9da57d7b0ddd8d73b676ce12c040362132cdd538bt * we'll first try to use the existing DMA buffer to build
9da57d7b0ddd8d73b676ce12c040362132cdd538bt * an mblk and send the mblk upstream.
9da57d7b0ddd8d73b676ce12c040362132cdd538bt *
9da57d7b0ddd8d73b676ce12c040362132cdd538bt * If the first method fails, or the packet length is less
9da57d7b0ddd8d73b676ce12c040362132cdd538bt * than the copy threshold, we'll allocate a new mblk and
9da57d7b0ddd8d73b676ce12c040362132cdd538bt * copy the packet data to the new mblk.
9da57d7b0ddd8d73b676ce12c040362132cdd538bt */
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China if (lro_eop) {
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China mp = ixgbe_lro_bind(rx_data, lro_start,
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China rx_data->lro_num, pkt_len);
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China if (mp == NULL)
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China mp = ixgbe_lro_copy(rx_data, lro_start,
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China rx_data->lro_num, pkt_len);
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China lro_eop = B_FALSE;
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China rx_data->lro_num = 0;
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China } else {
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China if (pkt_len > ixgbe->rx_copy_thresh)
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China mp = ixgbe_rx_bind(rx_data, rx_next, pkt_len);
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China if (mp == NULL)
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China mp = ixgbe_rx_copy(rx_data, rx_next, pkt_len);
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China }
9da57d7b0ddd8d73b676ce12c040362132cdd538bt if (mp != NULL) {
9da57d7b0ddd8d73b676ce12c040362132cdd538bt /*
9da57d7b0ddd8d73b676ce12c040362132cdd538bt * Check h/w checksum offload status
9da57d7b0ddd8d73b676ce12c040362132cdd538bt */
9da57d7b0ddd8d73b676ce12c040362132cdd538bt if (ixgbe->rx_hcksum_enable)
9da57d7b0ddd8d73b676ce12c040362132cdd538bt ixgbe_rx_assoc_hcksum(mp, status_error);
9da57d7b0ddd8d73b676ce12c040362132cdd538bt
9da57d7b0ddd8d73b676ce12c040362132cdd538bt *mblk_tail = mp;
9da57d7b0ddd8d73b676ce12c040362132cdd538bt mblk_tail = &mp->b_next;
9da57d7b0ddd8d73b676ce12c040362132cdd538bt }
9da57d7b0ddd8d73b676ce12c040362132cdd538bt
9da57d7b0ddd8d73b676ce12c040362132cdd538btrx_discard:
9da57d7b0ddd8d73b676ce12c040362132cdd538bt /*
9da57d7b0ddd8d73b676ce12c040362132cdd538bt * Reset rx descriptor read bits
9da57d7b0ddd8d73b676ce12c040362132cdd538bt */
ea65739ebec0cbd625d8c89e19592be58fa186a6chenlu chen - Sun Microsystems - Beijing China current_rcb = rx_data->work_list[rx_next];
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China if (ixgbe->lro_enable) {
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China if (!current_rcb->lro_pkt) {
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China current_rbd->read.pkt_addr =
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China current_rcb->rx_buf.dma_address;
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China current_rbd->read.hdr_addr = 0;
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China }
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China } else {
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China current_rbd->read.pkt_addr =
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China current_rcb->rx_buf.dma_address;
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China current_rbd->read.hdr_addr = 0;
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China }
9da57d7b0ddd8d73b676ce12c040362132cdd538bt
ea65739ebec0cbd625d8c89e19592be58fa186a6chenlu chen - Sun Microsystems - Beijing China rx_next = NEXT_INDEX(rx_next, 1, rx_data->ring_size);
9da57d7b0ddd8d73b676ce12c040362132cdd538bt
9da57d7b0ddd8d73b676ce12c040362132cdd538bt /*
9da57d7b0ddd8d73b676ce12c040362132cdd538bt * The receive function is in interrupt context, so here
ea65739ebec0cbd625d8c89e19592be58fa186a6chenlu chen - Sun Microsystems - Beijing China * rx_limit_per_intr is used to avoid doing receiving too long
9da57d7b0ddd8d73b676ce12c040362132cdd538bt * per interrupt.
9da57d7b0ddd8d73b676ce12c040362132cdd538bt */
ea65739ebec0cbd625d8c89e19592be58fa186a6chenlu chen - Sun Microsystems - Beijing China if (++pkt_num > ixgbe->rx_limit_per_intr) {
9da57d7b0ddd8d73b676ce12c040362132cdd538bt IXGBE_DEBUG_STAT(rx_ring->stat_exceed_pkt);
9da57d7b0ddd8d73b676ce12c040362132cdd538bt break;
9da57d7b0ddd8d73b676ce12c040362132cdd538bt }
9da57d7b0ddd8d73b676ce12c040362132cdd538bt
ea65739ebec0cbd625d8c89e19592be58fa186a6chenlu chen - Sun Microsystems - Beijing China current_rbd = &rx_data->rbd_ring[rx_next];
9da57d7b0ddd8d73b676ce12c040362132cdd538bt status_error = current_rbd->wb.upper.status_error;
9da57d7b0ddd8d73b676ce12c040362132cdd538bt }
9da57d7b0ddd8d73b676ce12c040362132cdd538bt
0dc2366f7b9f9f36e10909b1e95edbf2a261c2acVenugopal Iyer rx_ring->stat_rbytes += received_bytes;
0dc2366f7b9f9f36e10909b1e95edbf2a261c2acVenugopal Iyer rx_ring->stat_ipackets += pkt_num;
0dc2366f7b9f9f36e10909b1e95edbf2a261c2acVenugopal Iyer
ea65739ebec0cbd625d8c89e19592be58fa186a6chenlu chen - Sun Microsystems - Beijing China DMA_SYNC(&rx_data->rbd_area, DDI_DMA_SYNC_FORDEV);
9da57d7b0ddd8d73b676ce12c040362132cdd538bt
ea65739ebec0cbd625d8c89e19592be58fa186a6chenlu chen - Sun Microsystems - Beijing China rx_data->rbd_next = rx_next;
9da57d7b0ddd8d73b676ce12c040362132cdd538bt
9da57d7b0ddd8d73b676ce12c040362132cdd538bt /*
9da57d7b0ddd8d73b676ce12c040362132cdd538bt * Update the h/w tail accordingly
9da57d7b0ddd8d73b676ce12c040362132cdd538bt */
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China if (ixgbe->lro_enable) {
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China lro_first = ixgbe_lro_get_first(rx_data, rx_next);
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China rx_tail = PREV_INDEX(lro_first, 1, rx_data->ring_size);
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China } else
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China rx_tail = PREV_INDEX(rx_next, 1, rx_data->ring_size);
ffd8e8832e01f996f32459073f4dd308fe5265baWinson Wang - Sun Microsystems - Beijing China
0dc2366f7b9f9f36e10909b1e95edbf2a261c2acVenugopal Iyer IXGBE_WRITE_REG(&ixgbe->hw, IXGBE_RDT(rx_ring->hw_index), rx_tail);
9da57d7b0ddd8d73b676ce12c040362132cdd538bt
9da57d7b0ddd8d73b676ce12c040362132cdd538bt if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) {
ea65739ebec0cbd625d8c89e19592be58fa186a6chenlu chen - Sun Microsystems - Beijing China ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_DEGRADED);
62e6e1adfc8d795f477410703cc7a88dc5b82622Paul Guo atomic_or_32(&ixgbe->ixgbe_state, IXGBE_ERROR);
9da57d7b0ddd8d73b676ce12c040362132cdd538bt }
9da57d7b0ddd8d73b676ce12c040362132cdd538bt
9da57d7b0ddd8d73b676ce12c040362132cdd538bt return (mblk_head);
9da57d7b0ddd8d73b676ce12c040362132cdd538bt}
da14cebe459d3275048785f25bd869cb09b5307fEric Cheng
da14cebe459d3275048785f25bd869cb09b5307fEric Chengmblk_t *
da14cebe459d3275048785f25bd869cb09b5307fEric Chengixgbe_ring_rx_poll(void *arg, int n_bytes)
da14cebe459d3275048785f25bd869cb09b5307fEric Cheng{
da14cebe459d3275048785f25bd869cb09b5307fEric Cheng ixgbe_rx_ring_t *rx_ring = (ixgbe_rx_ring_t *)arg;
da14cebe459d3275048785f25bd869cb09b5307fEric Cheng mblk_t *mp = NULL;
da14cebe459d3275048785f25bd869cb09b5307fEric Cheng
da14cebe459d3275048785f25bd869cb09b5307fEric Cheng ASSERT(n_bytes >= 0);
da14cebe459d3275048785f25bd869cb09b5307fEric Cheng
da14cebe459d3275048785f25bd869cb09b5307fEric Cheng if (n_bytes == 0)
62e6e1adfc8d795f477410703cc7a88dc5b82622Paul Guo return (NULL);
da14cebe459d3275048785f25bd869cb09b5307fEric Cheng
da14cebe459d3275048785f25bd869cb09b5307fEric Cheng mutex_enter(&rx_ring->rx_lock);
da14cebe459d3275048785f25bd869cb09b5307fEric Cheng mp = ixgbe_ring_rx(rx_ring, n_bytes);
da14cebe459d3275048785f25bd869cb09b5307fEric Cheng mutex_exit(&rx_ring->rx_lock);
da14cebe459d3275048785f25bd869cb09b5307fEric Cheng
da14cebe459d3275048785f25bd869cb09b5307fEric Cheng return (mp);
da14cebe459d3275048785f25bd869cb09b5307fEric Cheng}