io/sfxge/sfxge.h

	sfxge.h revision 49ef7e0638c8b771d8a136eae78b1c0f99acc8e0
/*
 * Copyright (c) 2008-2016 Solarflare Communications Inc.
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *
 * 1. Redistributions of source code must retain the above copyright notice,
 *    this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright notice,
 *    this list of conditions and the following disclaimer in the documentation
 *    and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
 * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 *
 * The views and conclusions contained in the software and documentation are
 * those of the authors and should not be interpreted as representing official
 * policies, either expressed or implied, of the FreeBSD Project.
 */

#ifndef _SYS_SFXGE_H
#define _SYS_SFXGE_H

#ifdef  __cplusplus
extern "C" {
#endif

#include <sys/types.h>
#include <sys/ddi.h>
#include <sys/sunddi.h>
#include <sys/stream.h>
#include <sys/ethernet.h>
#include <sys/cpuvar.h>

#include <sys/mac.h>
#include <sys/mac_ether.h>
#include <sys/mac_provider.h>

#include "sfxge_ioc.h"
#include "sfxge_debug.h"

#include "efx.h"
#include "efx_regs.h"

#ifdef  _KERNEL

#define SFXGE_DRIVER_NAME "sfxge"

#define SFXGE_CPU_CACHE_SIZE    64

typedef struct sfxge_s  sfxge_t;

typedef enum sfxge_intr_state_e {
    SFXGE_INTR_UNINITIALIZED = 0,
    SFXGE_INTR_INITIALIZED,
    SFXGE_INTR_TESTING,
    SFXGE_INTR_STARTED
} sfxge_intr_state_t;

typedef struct sfxge_intr_s {
    ddi_intr_handle_t   *si_table;
    int         si_table_size;
    int         si_nalloc;
    int         si_type;
    int         si_cap;
    efsys_mem_t     si_mem;
    uint64_t        si_mask;
    sfxge_intr_state_t  si_state;
    uint32_t        si_zero_count;
    int         si_intr_pri;
} sfxge_intr_t;

typedef enum sfxge_promisc_type_e {
    SFXGE_PROMISC_OFF = 0,
    SFXGE_PROMISC_ALL_MULTI,
    SFXGE_PROMISC_ALL_PHYS
} sfxge_promisc_type_t;

typedef enum sfxge_link_duplex_e {
    SFXGE_LINK_DUPLEX_UNKNOWN = 0,
    SFXGE_LINK_DUPLEX_HALF,
    SFXGE_LINK_DUPLEX_FULL
} sfxge_link_duplex_t;

typedef enum sfxge_unicst_type_e {
    SFXGE_UNICST_BIA = 0,
    SFXGE_UNICST_LAA,
    SFXGE_UNICST_NTYPES
} sfxge_unicst_type_t;

typedef struct sfxge_phy_s {
        kstat_t         *sp_ksp;
        kstat_named_t       *sp_stat;
        uint32_t        *sp_statbuf;
        efsys_mem_t     sp_mem;
} sfxge_phy_t;

typedef enum sfxge_mac_state_e {
    SFXGE_MAC_UNINITIALIZED = 0,
    SFXGE_MAC_INITIALIZED,
    SFXGE_MAC_STARTED
} sfxge_mac_state_t;

typedef struct sfxge_mac_s {
    sfxge_t         *sm_sp;
    efsys_mem_t     sm_mem;
    kstat_t         *sm_ksp;
    kstat_named_t       *sm_stat;
    uint8_t         sm_bia[ETHERADDRL];
    uint8_t         sm_laa[ETHERADDRL];
    boolean_t       sm_laa_valid;
    unsigned int        sm_fcntl;
    sfxge_promisc_type_t    sm_promisc;
    uint8_t         sm_mcast_addr[EFX_MAC_MULTICAST_LIST_MAX *
        ETHERADDRL]; /* List of multicast addresses to filter on */
    int         sm_mcast_count;
    clock_t         sm_lbolt;
    kmutex_t        sm_lock;
    efx_link_mode_t     sm_link_mode;
    unsigned int        sm_link_speed;
    sfxge_link_duplex_t sm_link_duplex;
    boolean_t       sm_link_up;
    boolean_t       sm_link_poll_reqd;
    kcondvar_t      sm_link_poll_kv;
    boolean_t       sm_mac_stats_timer_reqd;
    boolean_t       sm_mac_stats_pend;
    ddi_taskq_t     *sm_tqp;
    sfxge_mac_state_t   sm_state;
    sfxge_phy_t     sm_phy;
    uint32_t        sm_phy_cap_to_set;
    uint32_t        sm_phy_cap_to_unset;
} sfxge_mac_t;

typedef enum sfxge_mon_state_e {
    SFXGE_MON_UNINITIALIZED = 0,
    SFXGE_MON_INITIALIZED,
    SFXGE_MON_STARTED
} sfxge_mon_state_t;

typedef struct sfxge_mon_s {
    sfxge_t         *sm_sp;
    efx_mon_type_t      sm_type;
    unsigned int        sm_devid;
    kstat_t         *sm_ksp;
    kstat_named_t       *sm_stat;
    efx_mon_stat_value_t    *sm_statbuf;
    kmutex_t        sm_lock;
    sfxge_mon_state_t   sm_state;
    efsys_mem_t     sm_mem;
    int         sm_polling;
} sfxge_mon_t;

typedef enum sfxge_sram_state_e {
    SFXGE_SRAM_UNINITIALIZED = 0,
    SFXGE_SRAM_INITIALIZED,
    SFXGE_SRAM_STARTED
} sfxge_sram_state_t;

typedef struct sfxge_sram_s {
    sfxge_t         *ss_sp;
    kmutex_t        ss_lock;
    struct map      *ss_buf_tbl_map;
    unsigned int        ss_count;
    sfxge_sram_state_t  ss_state;
} sfxge_sram_t;

typedef enum sfxge_mcdi_state_e {
    SFXGE_MCDI_UNINITIALIZED = 0,
    SFXGE_MCDI_INITIALIZED,
    SFXGE_MCDI_BUSY,
    SFXGE_MCDI_COMPLETED
} sfxge_mcdi_state_t;

typedef struct sfxge_mcdi_s {
    sfxge_t         *sm_sp;
    kmutex_t        sm_lock;
    sfxge_mcdi_state_t  sm_state;
    efx_mcdi_transport_t    sm_emt;
    efsys_mem_t     sm_mem;
    kcondvar_t      sm_kv;      /* MCDI poll complete */
} sfxge_mcdi_t;

#define SFXGE_NEVS          4096
#define SFXGE_RX_NDESCS         1024
#define SFXGE_TX_NDESCS         1024
#define SFXGE_TX_NLABELS        EFX_EV_TX_NLABELS

#define SFXGE_DEFAULT_RXQ_SIZE      1024
#define SFXGE_DEFAULT_MODERATION    30

typedef enum sfxge_evq_state_e {
    SFXGE_EVQ_UNINITIALIZED = 0,
    SFXGE_EVQ_INITIALIZED,
    SFXGE_EVQ_STARTING,
    SFXGE_EVQ_STARTED
} sfxge_evq_state_t;

#define SFXGE_EV_BATCH  (SFXGE_NEVS / 4)

typedef struct sfxge_txq_s  sfxge_txq_t;

typedef struct sfxge_evq_s {
    union {
        struct {
            sfxge_t         *__se_sp;
            unsigned int        __se_index;
            efsys_mem_t     __se_mem;
            unsigned int        __se_id;
            kstat_t         *__se_ksp;
            kstat_named_t       *__se_stat;
            efx_ev_callbacks_t  __se_eec;
            sfxge_evq_state_t   __se_state;
            boolean_t       __se_exception;
        } __se_s1;
        uint8_t __se_pad[SFXGE_CPU_CACHE_SIZE * 4];
    } __se_u1;
    union {
        struct {
            kmutex_t        __se_lock;
            kcondvar_t      __se_init_kv;
            efx_evq_t       *__se_eep;
            unsigned int        __se_count;
            unsigned int        __se_rx;
            unsigned int        __se_tx;
            sfxge_txq_t     *__se_stp;
            sfxge_txq_t     **__se_stpp;
            processorid_t       __se_cpu_id;
            uint16_t        __se_ev_batch;
        } __se_s2;
        uint8_t __se_pad[SFXGE_CPU_CACHE_SIZE];
    } __se_u2;
    union {
        struct {
            sfxge_txq_t *__se_label_stp[SFXGE_TX_NLABELS];
        } __se_s3;
        uint8_t __se_pad[SFXGE_CPU_CACHE_SIZE * 4];
    } __se_u3;
} sfxge_evq_t;

#define se_sp       __se_u1.__se_s1.__se_sp
#define se_index    __se_u1.__se_s1.__se_index
#define se_mem      __se_u1.__se_s1.__se_mem
#define se_id       __se_u1.__se_s1.__se_id
#define se_ksp      __se_u1.__se_s1.__se_ksp
#define se_stat     __se_u1.__se_s1.__se_stat
#define se_eec      __se_u1.__se_s1.__se_eec
#define se_state    __se_u1.__se_s1.__se_state
#define se_exception    __se_u1.__se_s1.__se_exception

#define se_lock     __se_u2.__se_s2.__se_lock
#define se_init_kv  __se_u2.__se_s2.__se_init_kv
#define se_eep      __se_u2.__se_s2.__se_eep
#define se_count    __se_u2.__se_s2.__se_count
#define se_rx       __se_u2.__se_s2.__se_rx
#define se_tx       __se_u2.__se_s2.__se_tx
#define se_stp      __se_u2.__se_s2.__se_stp
#define se_stpp     __se_u2.__se_s2.__se_stpp
#define se_cpu_id   __se_u2.__se_s2.__se_cpu_id
#define se_ev_batch __se_u2.__se_s2.__se_ev_batch

#define se_label_stp    __se_u3.__se_s3.__se_label_stp


#define SFXGE_MAGIC_RESERVED    0x8000

#define SFXGE_MAGIC_DMAQ_LABEL_WIDTH  5
#define SFXGE_MAGIC_DMAQ_LABEL_MASK   ((1 << SFXGE_MAGIC_DMAQ_LABEL_WIDTH) - 1)

#define SFXGE_MAGIC_RX_QFLUSH_DONE                  \
    (SFXGE_MAGIC_RESERVED | (1 << SFXGE_MAGIC_DMAQ_LABEL_WIDTH))

#define SFXGE_MAGIC_RX_QFLUSH_FAILED                    \
    (SFXGE_MAGIC_RESERVED | (2 << SFXGE_MAGIC_DMAQ_LABEL_WIDTH))

#define SFXGE_MAGIC_RX_QFPP_TRIM                    \
    (SFXGE_MAGIC_RESERVED | (3 << SFXGE_MAGIC_DMAQ_LABEL_WIDTH))

#define SFXGE_MAGIC_TX_QFLUSH_DONE                  \
    (SFXGE_MAGIC_RESERVED | (4 << SFXGE_MAGIC_DMAQ_LABEL_WIDTH))

typedef struct sfxge_rxq_s      sfxge_rxq_t;

#define SFXGE_ETHERTYPE_LOOPBACK    0x9000  /* Xerox loopback */

typedef struct sfxge_rx_packet_s    sfxge_rx_packet_t;

struct sfxge_rx_packet_s {
    union {
        struct {
            frtn_t          __srp_free;
            uint16_t        __srp_flags;
            uint16_t        __srp_size;
            mblk_t          *__srp_mp;
            struct ether_header *__srp_etherhp;
            struct ip       *__srp_iphp;
            struct tcphdr       *__srp_thp;
            size_t          __srp_off;
        } __srp_s1;
        uint8_t __srp_pad[SFXGE_CPU_CACHE_SIZE];
    } __srp_u1;
    union {
        struct {
            sfxge_rxq_t     *__srp_srp;
            ddi_dma_handle_t    __srp_dma_handle;
            ddi_acc_handle_t    __srp_acc_handle;
            unsigned char       *__srp_base;
            size_t          __srp_mblksize;
            uint64_t        __srp_addr;
            boolean_t       __srp_recycle;
            caddr_t         __srp_putp;
        } __srp_s2;
        uint8_t __srp_pad[SFXGE_CPU_CACHE_SIZE * 2];
    } __srp_u2;
};

#define srp_free    __srp_u1.__srp_s1.__srp_free
#define srp_flags   __srp_u1.__srp_s1.__srp_flags
#define srp_size    __srp_u1.__srp_s1.__srp_size
#define srp_mp      __srp_u1.__srp_s1.__srp_mp
#define srp_etherhp __srp_u1.__srp_s1.__srp_etherhp
#define srp_iphp    __srp_u1.__srp_s1.__srp_iphp
#define srp_thp     __srp_u1.__srp_s1.__srp_thp
#define srp_off     __srp_u1.__srp_s1.__srp_off

#define srp_srp     __srp_u2.__srp_s2.__srp_srp
#define srp_dma_handle  __srp_u2.__srp_s2.__srp_dma_handle
#define srp_acc_handle  __srp_u2.__srp_s2.__srp_acc_handle
#define srp_base    __srp_u2.__srp_s2.__srp_base
#define srp_mblksize    __srp_u2.__srp_s2.__srp_mblksize
#define srp_addr    __srp_u2.__srp_s2.__srp_addr
#define srp_recycle __srp_u2.__srp_s2.__srp_recycle
#define srp_putp    __srp_u2.__srp_s2.__srp_putp

#define SFXGE_RX_FPP_NSLOTS 8
#define SFXGE_RX_FPP_MASK   (SFXGE_RX_FPP_NSLOTS - 1)

/* Free packet pool putlist (dynamically allocated) */
typedef struct sfxge_rx_fpp_putlist_s {
    kmutex_t        srfpl_lock;
    unsigned int        srfpl_count;
    mblk_t          *srfpl_putp;
    mblk_t          **srfpl_putpp;
} sfxge_rx_fpp_putlist_t;

/* Free packet pool */
typedef struct sfxge_rx_fpp_s {
    caddr_t     srfpp_putp;
    unsigned int    srfpp_loaned;
    mblk_t      *srfpp_get;
    unsigned int    srfpp_count;
    unsigned int    srfpp_min;
    /* Low water mark: Don't trim to below this */
    unsigned int    srfpp_lowat;
} sfxge_rx_fpp_t;

typedef struct sfxge_rx_flow_s  sfxge_rx_flow_t;

struct sfxge_rx_flow_s {
    uint32_t        srf_tag;
    /* in-order segment count */
    unsigned int        srf_count;
    uint16_t        srf_tci;
    uint32_t        srf_saddr;
    uint32_t        srf_daddr;
    uint16_t        srf_sport;
    uint16_t        srf_dport;
    /* sequence number */
    uint32_t        srf_seq;
    clock_t         srf_lbolt;
    mblk_t          *srf_mp;
    mblk_t          **srf_mpp;
    struct ether_header *srf_etherhp;
    struct ip       *srf_iphp;
    struct tcphdr       *srf_first_thp;
    struct tcphdr       *srf_last_thp;
    size_t          srf_len;
    sfxge_rx_flow_t     *srf_next;
};

#define SFXGE_MAX_FLOW      1024
#define SFXGE_SLOW_START    20

typedef enum sfxge_flush_state_e {
    SFXGE_FLUSH_INACTIVE = 0,
    SFXGE_FLUSH_DONE,
    SFXGE_FLUSH_PENDING,
    SFXGE_FLUSH_FAILED
} sfxge_flush_state_t;

typedef enum sfxge_rxq_state_e {
    SFXGE_RXQ_UNINITIALIZED = 0,
    SFXGE_RXQ_INITIALIZED,
    SFXGE_RXQ_STARTED
} sfxge_rxq_state_t;


#define SFXGE_RX_BATCH  128
#define SFXGE_RX_NSTATS 8 /* note that *esballoc share one kstat */

struct sfxge_rxq_s {
    union {
        struct {
            sfxge_t             *__sr_sp;
            unsigned int            __sr_index;
            efsys_mem_t         __sr_mem;
            unsigned int            __sr_id;
            unsigned int            __sr_lowat;
            unsigned int            __sr_hiwat;
            volatile timeout_id_t       __sr_tid;
            sfxge_rxq_state_t       __sr_state;
        } __sr_s1;
        uint8_t __sr_pad[SFXGE_CPU_CACHE_SIZE * 2];
    } __sr_u1;
    union {
        struct {
            sfxge_rx_packet_t       **__sr_srpp;
            unsigned int            __sr_added;
            unsigned int            __sr_pushed;
            unsigned int            __sr_pending;
            unsigned int            __sr_completed;
            unsigned int            __sr_loopback;
            mblk_t              *__sr_mp;
            mblk_t              **__sr_mpp;
            sfxge_rx_flow_t         *__sr_flow;
            sfxge_rx_flow_t         *__sr_srfp;
            sfxge_rx_flow_t         **__sr_srfpp;
            clock_t             __sr_rto;
        } __sr_s2;
        uint8_t __sr_pad[SFXGE_CPU_CACHE_SIZE * 2];
    } __sr_u2;
    union {
        struct {
            sfxge_rx_fpp_t          __sr_fpp;
            efx_rxq_t           *__sr_erp;
            volatile sfxge_flush_state_t    __sr_flush;
            kcondvar_t          __sr_flush_kv;
            kstat_t             *__sr_ksp;
        } __sr_s3;
        uint8_t __sr_pad[SFXGE_CPU_CACHE_SIZE];
    } __sr_u3;
    struct {
        /* NB must match SFXGE_RX_NSTATS */
        uint32_t    srk_rx_pkt_mem_limit;
        uint32_t    srk_kcache_alloc_nomem;
        uint32_t    srk_dma_alloc_nomem;
        uint32_t    srk_dma_alloc_fail;
        uint32_t    srk_dma_bind_nomem;
        uint32_t    srk_dma_bind_fail;
        uint32_t    srk_desballoc_fail;
        uint32_t    srk_rxq_empty_discard;
    } sr_kstat;
};

#define sr_sp       __sr_u1.__sr_s1.__sr_sp
#define sr_index    __sr_u1.__sr_s1.__sr_index
#define sr_mem      __sr_u1.__sr_s1.__sr_mem
#define sr_id       __sr_u1.__sr_s1.__sr_id
#define sr_mrh      __sr_u1.__sr_s1.__sr_mrh
#define sr_lowat    __sr_u1.__sr_s1.__sr_lowat
#define sr_hiwat    __sr_u1.__sr_s1.__sr_hiwat
#define sr_tid      __sr_u1.__sr_s1.__sr_tid
#define sr_state    __sr_u1.__sr_s1.__sr_state

#define sr_srpp     __sr_u2.__sr_s2.__sr_srpp
#define sr_added    __sr_u2.__sr_s2.__sr_added
#define sr_pushed   __sr_u2.__sr_s2.__sr_pushed
#define sr_pending  __sr_u2.__sr_s2.__sr_pending
#define sr_completed    __sr_u2.__sr_s2.__sr_completed
#define sr_loopback __sr_u2.__sr_s2.__sr_loopback
#define sr_mp       __sr_u2.__sr_s2.__sr_mp
#define sr_mpp      __sr_u2.__sr_s2.__sr_mpp
#define sr_flow     __sr_u2.__sr_s2.__sr_flow
#define sr_srfp     __sr_u2.__sr_s2.__sr_srfp
#define sr_srfpp    __sr_u2.__sr_s2.__sr_srfpp
#define sr_rto      __sr_u2.__sr_s2.__sr_rto

#define sr_fpp      __sr_u3.__sr_s3.__sr_fpp
#define sr_erp      __sr_u3.__sr_s3.__sr_erp
#define sr_flush    __sr_u3.__sr_s3.__sr_flush
#define sr_flush_kv __sr_u3.__sr_s3.__sr_flush_kv
#define sr_ksp      __sr_u3.__sr_s3.__sr_ksp

typedef struct sfxge_tx_packet_s    sfxge_tx_packet_t;

/* Packet type from parsing transmit packet */
typedef enum sfxge_packet_type_e {
    SFXGE_PACKET_TYPE_UNKNOWN = 0,
    SFXGE_PACKET_TYPE_IPV4_TCP,
    SFXGE_PACKET_TYPE_IPV4_UDP,
    SFXGE_PACKET_TYPE_IPV4_SCTP,
    SFXGE_PACKET_TYPE_IPV4_OTHER,
    SFXGE_PACKET_NTYPES
} sfxge_packet_type_t;

struct sfxge_tx_packet_s {
    sfxge_tx_packet_t   *stp_next;
    mblk_t          *stp_mp;
    struct ether_header *stp_etherhp;
    struct ip       *stp_iphp;
    struct tcphdr       *stp_thp;
    size_t          stp_off;
    size_t          stp_size;
    size_t          stp_mss;
    uint32_t        stp_dpl_put_len;
};

#define SFXGE_TX_FPP_MAX    64

typedef struct sfxge_tx_fpp_s {
    sfxge_tx_packet_t   *stf_stpp;
    unsigned int        stf_count;
} sfxge_tx_fpp_t;

typedef struct sfxge_tx_mapping_s   sfxge_tx_mapping_t;

#define SFXGE_TX_MAPPING_NADDR  (((1 << 16) >> 12) + 2)

struct sfxge_tx_mapping_s {
    sfxge_tx_mapping_t  *stm_next;
    sfxge_t         *stm_sp;
    mblk_t          *stm_mp;
    ddi_dma_handle_t    stm_dma_handle;
    caddr_t         stm_base;
    size_t          stm_size;
    size_t          stm_off;
    uint64_t        stm_addr[SFXGE_TX_MAPPING_NADDR];
};

typedef struct sfxge_tx_fmp_s {
    sfxge_tx_mapping_t  *stf_stmp;
    unsigned int        stf_count;
} sfxge_tx_fmp_t;

typedef struct sfxge_tx_buffer_s    sfxge_tx_buffer_t;

struct sfxge_tx_buffer_s {
    sfxge_tx_buffer_t   *stb_next;
    size_t          stb_off;
    efsys_mem_t     stb_esm;
};

#define SFXGE_TX_BUFFER_SIZE    0x400
#define SFXGE_TX_HEADER_SIZE    0x100
#define SFXGE_TX_COPY_THRESHOLD 0x200

typedef struct sfxge_tx_fbp_s {
    sfxge_tx_buffer_t   *stf_stbp;
    unsigned int        stf_count;
} sfxge_tx_fbp_t;

typedef struct sfxge_tx_dpl_s {
    uintptr_t       std_put;
    sfxge_tx_packet_t   *std_get;
    sfxge_tx_packet_t   **std_getp;
    unsigned int        std_count; /* only get list count */
    unsigned int        get_pkt_limit;
    unsigned int        put_pkt_limit;
    unsigned int        get_full_count;
    unsigned int        put_full_count;
} sfxge_tx_dpl_t;

typedef enum sfxge_txq_state_e {
    SFXGE_TXQ_UNINITIALIZED = 0,
    SFXGE_TXQ_INITIALIZED,
    SFXGE_TXQ_STARTED,
    SFXGE_TXQ_FLUSH_PENDING,
    SFXGE_TXQ_FLUSH_DONE,
    SFXGE_TXQ_FLUSH_FAILED
} sfxge_txq_state_t;

typedef enum sfxge_txq_type_e {
    SFXGE_TXQ_NON_CKSUM = 0,
    SFXGE_TXQ_IP_CKSUM,
    SFXGE_TXQ_IP_TCP_UDP_CKSUM,
    SFXGE_TXQ_NTYPES
} sfxge_txq_type_t;

#define SFXGE_TXQ_UNBLOCK_LEVEL1    (EFX_TXQ_LIMIT(SFXGE_TX_NDESCS) / 4)
#define SFXGE_TXQ_UNBLOCK_LEVEL2    0
#define SFXGE_TXQ_NOT_BLOCKED       -1

#define SFXGE_TX_BATCH  64

struct sfxge_txq_s {
    union {
        struct {
            sfxge_t             *__st_sp;
            unsigned int            __st_index;
            unsigned int            __st_label;
            sfxge_txq_type_t        __st_type;
            unsigned int            __st_evq;
            efsys_mem_t         __st_mem;
            unsigned int            __st_id;
            kstat_t             *__st_ksp;
            kstat_named_t           *__st_stat;
            sfxge_txq_state_t       __st_state;
        } __st_s1;
        uint8_t __st_pad[SFXGE_CPU_CACHE_SIZE * 2];
    } __st_u1;
    union {
        struct {
            sfxge_tx_dpl_t          __st_dpl;
        } __st_s2;
        uint8_t __st_pad[SFXGE_CPU_CACHE_SIZE];
    } __st_u2;
    union {
        struct {
            kmutex_t            __st_lock;
            /* mapping pool - sfxge_tx_mapping_t */
            sfxge_tx_fmp_t          __st_fmp;
            /* buffer pool - sfxge_tx_buffer_t */
            sfxge_tx_fbp_t          __st_fbp;
            /* packet pool - sfxge_tx_packet_t */
            sfxge_tx_fpp_t          __st_fpp;
            efx_buffer_t            *__st_eb;
            unsigned int            __st_n;
            efx_txq_t           *__st_etp;
            sfxge_tx_mapping_t      **__st_stmp;
            sfxge_tx_buffer_t       **__st_stbp;
            mblk_t              **__st_mp;
            unsigned int            __st_added;
            unsigned int            __st_reaped;
            int             __st_unblock;
        } __st_s3;
        uint8_t __st_pad[SFXGE_CPU_CACHE_SIZE * 3];
    } __st_u3;
    union {
        struct {
            sfxge_txq_t         *__st_next;
            unsigned int            __st_pending;
            unsigned int            __st_completed;

        } __st_s4;
        uint8_t __st_pad[SFXGE_CPU_CACHE_SIZE];
    } __st_u4;
};

#define st_sp       __st_u1.__st_s1.__st_sp
#define st_index    __st_u1.__st_s1.__st_index
#define st_label    __st_u1.__st_s1.__st_label
#define st_type     __st_u1.__st_s1.__st_type
#define st_evq      __st_u1.__st_s1.__st_evq
#define st_mem      __st_u1.__st_s1.__st_mem
#define st_id       __st_u1.__st_s1.__st_id
#define st_ksp      __st_u1.__st_s1.__st_ksp
#define st_stat     __st_u1.__st_s1.__st_stat
#define st_state    __st_u1.__st_s1.__st_state

#define st_dpl      __st_u2.__st_s2.__st_dpl

#define st_lock     __st_u3.__st_s3.__st_lock
#define st_fmp      __st_u3.__st_s3.__st_fmp
#define st_fbp      __st_u3.__st_s3.__st_fbp
#define st_fpp      __st_u3.__st_s3.__st_fpp
#define st_eb       __st_u3.__st_s3.__st_eb
#define st_n        __st_u3.__st_s3.__st_n
#define st_etp      __st_u3.__st_s3.__st_etp
#define st_stmp     __st_u3.__st_s3.__st_stmp
#define st_stbp     __st_u3.__st_s3.__st_stbp
#define st_mp       __st_u3.__st_s3.__st_mp
#define st_added    __st_u3.__st_s3.__st_added
#define st_reaped   __st_u3.__st_s3.__st_reaped
#define st_unblock  __st_u3.__st_s3.__st_unblock

#define st_next     __st_u4.__st_s4.__st_next
#define st_pending  __st_u4.__st_s4.__st_pending
#define st_completed    __st_u4.__st_s4.__st_completed

typedef enum sfxge_rx_scale_state_e {
    SFXGE_RX_SCALE_UNINITIALIZED = 0,
    SFXGE_RX_SCALE_INITIALIZED,
    SFXGE_RX_SCALE_STARTED
} sfxge_rx_scale_state_t;

#define SFXGE_RX_SCALE_MAX  EFX_RSS_TBL_SIZE

typedef struct sfxge_rx_scale_s {
    kmutex_t        srs_lock;
    unsigned int        *srs_cpu;
    unsigned int        srs_tbl[SFXGE_RX_SCALE_MAX];
    unsigned int        srs_count;
    kstat_t         *srs_ksp;
    sfxge_rx_scale_state_t  srs_state;
} sfxge_rx_scale_t;


typedef enum sfxge_rx_coalesce_mode_e {
    SFXGE_RX_COALESCE_OFF = 0,
    SFXGE_RX_COALESCE_DISALLOW_PUSH = 1,
    SFXGE_RX_COALESCE_ALLOW_PUSH = 2
} sfxge_rx_coalesce_mode_t;

typedef enum sfxge_vpd_type_e {
    SFXGE_VPD_ID = 0,
    SFXGE_VPD_PN = 1,
    SFXGE_VPD_SN = 2,
    SFXGE_VPD_EC = 3,
    SFXGE_VPD_MN = 4,
    SFXGE_VPD_VD = 5,
    SFXGE_VPD_VE = 6,
    SFXGE_VPD_MAX = 7,
} sfxge_vpd_type_t;

typedef struct sfxge_vpd_kstat_s {
    kstat_t     *svk_ksp;
    kstat_named_t   svk_stat[SFXGE_VPD_MAX];
    efx_vpd_value_t *svk_vv;
} sfxge_vpd_kstat_t;

typedef struct sfxge_cfg_kstat_s {
    struct {
        kstat_named_t   sck_mac;
        kstat_named_t   sck_version;
    } kstat;
    struct {
        char        sck_mac[64 + 1];
    } buf;
} sfxge_cfg_kstat_t;

typedef enum sfxge_state_e {
    SFXGE_UNINITIALIZED = 0,
    SFXGE_INITIALIZED,
    SFXGE_REGISTERED,
    SFXGE_STARTING,
    SFXGE_STARTED,
    SFXGE_STOPPING
} sfxge_state_t;

typedef enum sfxge_hw_err_e {
    SFXGE_HW_OK = 0,
    SFXGE_HW_ERR,
} sfxge_hw_err_t;

typedef enum sfxge_action_on_hw_err_e {
    SFXGE_RECOVER = 0,
    SFXGE_INVISIBLE = 1,
    SFXGE_LEAVE_DEAD = 2,
} sfxge_action_on_hw_err_t;

typedef char *sfxge_mac_priv_prop_t;

#define SFXGE_TOEPLITZ_KEY_LEN 40

struct sfxge_s {
    kmutex_t            s_state_lock;
    sfxge_state_t           s_state;
    dev_info_t          *s_dip;
    ddi_taskq_t         *s_tqp;
    ddi_acc_handle_t        s_pci_handle;
    uint16_t            s_pci_venid;
    uint16_t            s_pci_devid;
#if EFSYS_OPT_MCDI_LOGGING
    unsigned int            s_bus_addr;
#endif
    efx_family_t            s_family;
    unsigned int            s_pcie_nlanes;
    unsigned int            s_pcie_linkspeed;
    kmutex_t            s_nic_lock;
    efsys_bar_t         s_bar;
    sfxge_intr_t            s_intr;
    sfxge_mac_t         s_mac;
    sfxge_mon_t         s_mon;
    sfxge_sram_t            s_sram;
    sfxge_mcdi_t            s_mcdi;
    kmem_cache_t            *s_eq0c; /* eventQ 0 */
    kmem_cache_t            *s_eqXc; /* all other eventQs */
    sfxge_evq_t         *s_sep[SFXGE_RX_SCALE_MAX];
    unsigned int            s_ev_moderation;
    kmem_cache_t            *s_rqc;
    sfxge_rxq_t         *s_srp[SFXGE_RX_SCALE_MAX];
    sfxge_rx_scale_t        s_rx_scale;
    size_t              s_rx_prefix_size;
    size_t              s_rx_buffer_size;
    size_t              s_rx_buffer_align;
    sfxge_rx_coalesce_mode_t    s_rx_coalesce_mode;
    int64_t             s_rx_pkt_mem_max;
    volatile uint64_t       s_rx_pkt_mem_alloc;
    kmem_cache_t            *s_rpc;
    kmem_cache_t            *s_tqc;
    unsigned int            s_tx_scale_base[SFXGE_TXQ_NTYPES];
    unsigned int            s_tx_scale_max[SFXGE_TXQ_NTYPES];
    int                 s_tx_qcount;
    sfxge_txq_t         *s_stp[SFXGE_RX_SCALE_MAX *
        SFXGE_TXQ_NTYPES]; /* Sparse array */
    kmem_cache_t            *s_tpc;
    int             s_tx_flush_pending;
    kmutex_t            s_tx_flush_lock;
    kcondvar_t          s_tx_flush_kv;
    kmem_cache_t            *s_tbc;
    kmem_cache_t            *s_tmc;
    efx_nic_t           *s_enp;
    sfxge_vpd_kstat_t       s_vpd_kstat;
    sfxge_cfg_kstat_t       s_cfg_kstat;
    kstat_t             *s_cfg_ksp;
    size_t              s_mtu;
    int             s_rxq_poll_usec;
    mac_callbacks_t         s_mc;
    mac_handle_t            s_mh;
    sfxge_mac_priv_prop_t       *s_mac_priv_props;
    int             s_mac_priv_props_alloc;
    volatile uint32_t       s_nested_restarts;
    uint32_t            s_num_restarts;
    uint32_t            s_num_restarts_hw_err;
    sfxge_hw_err_t          s_hw_err;
    sfxge_action_on_hw_err_t    s_action_on_hw_err;
    uint16_t            s_rxq_size;
    uint16_t            s_evq0_size;
    uint16_t            s_evqX_size;
#if EFSYS_OPT_MCDI_LOGGING
    int             s_mcdi_logging;
#endif
    const uint32_t          *s_toeplitz_cache;
};

typedef struct sfxge_dma_buffer_attr_s {
    dev_info_t      *sdba_dip;
    ddi_dma_attr_t      *sdba_dattrp;
    int         (*sdba_callback) (caddr_t);
    size_t          sdba_length;
    uint_t          sdba_memflags;
    ddi_device_acc_attr_t   *sdba_devaccp;
    uint_t          sdba_bindflags;
    int         sdba_maxcookies;
    boolean_t       sdba_zeroinit;
} sfxge_dma_buffer_attr_t;

extern const char       sfxge_ident[];
extern uint8_t          sfxge_brdcst[];

extern kmutex_t         sfxge_global_lock;

extern unsigned int     *sfxge_cpu;

extern int          sfxge_start(sfxge_t *, boolean_t);
extern void         sfxge_stop(sfxge_t *);
extern void         sfxge_ioctl(sfxge_t *, queue_t *, mblk_t *);
extern int          sfxge_restart_dispatch(sfxge_t *, uint_t,
    sfxge_hw_err_t, const char *, uint32_t);

extern void         sfxge_gld_link_update(sfxge_t *);
extern void         sfxge_gld_mtu_update(sfxge_t *);
extern void         sfxge_gld_rx_post(sfxge_t *, unsigned int,
    mblk_t *);
extern void         sfxge_gld_rx_push(sfxge_t *);
extern int          sfxge_gld_register(sfxge_t *);
extern int          sfxge_gld_unregister(sfxge_t *);

extern int          sfxge_dma_buffer_create(efsys_mem_t *,
    const sfxge_dma_buffer_attr_t *);
extern void         sfxge_dma_buffer_destroy(efsys_mem_t *);

extern int          sfxge_intr_init(sfxge_t *);
extern int          sfxge_intr_start(sfxge_t *);
extern void         sfxge_intr_stop(sfxge_t *);
extern void         sfxge_intr_fini(sfxge_t *);
extern void         sfxge_intr_fatal(sfxge_t *);

extern int          sfxge_ev_init(sfxge_t *);
extern int          sfxge_ev_start(sfxge_t *);
extern void         sfxge_ev_moderation_get(sfxge_t *,
    unsigned int *);
extern int          sfxge_ev_moderation_set(sfxge_t *,
    unsigned int);
extern int          sfxge_ev_qmoderate(sfxge_t *, unsigned int,
    unsigned int);
extern int          sfxge_ev_qpoll(sfxge_t *, unsigned int);
extern int          sfxge_ev_qprime(sfxge_t *, unsigned int);
extern void         sfxge_ev_stop(sfxge_t *);
extern void         sfxge_ev_fini(sfxge_t *);
extern int          sfxge_ev_txlabel_alloc(sfxge_t *sp,
    unsigned int evq, sfxge_txq_t *stp, unsigned int *labelp);
extern int          sfxge_ev_txlabel_free(sfxge_t *sp,
    unsigned int evq, sfxge_txq_t *stp, unsigned int label);

extern int          sfxge_mon_init(sfxge_t *);
extern int          sfxge_mon_start(sfxge_t *);
extern void         sfxge_mon_stop(sfxge_t *);
extern void         sfxge_mon_fini(sfxge_t *);

extern int          sfxge_mac_init(sfxge_t *);
extern int          sfxge_mac_start(sfxge_t *, boolean_t);
extern void         sfxge_mac_stat_get(sfxge_t *, unsigned int,
    uint64_t *);
extern void         sfxge_mac_link_check(sfxge_t *, boolean_t *);
extern void         sfxge_mac_link_speed_get(sfxge_t *,
    unsigned int *);
extern void         sfxge_mac_link_duplex_get(sfxge_t *,
    sfxge_link_duplex_t *);
extern void         sfxge_mac_fcntl_get(sfxge_t *, unsigned int *);
extern int          sfxge_mac_fcntl_set(sfxge_t *, unsigned int);
extern int          sfxge_mac_unicst_get(sfxge_t *,
    sfxge_unicst_type_t, uint8_t *);
extern int          sfxge_mac_unicst_set(sfxge_t *,
    uint8_t *);
extern int          sfxge_mac_promisc_set(sfxge_t *,
    sfxge_promisc_type_t);
extern int          sfxge_mac_multicst_add(sfxge_t *,
    uint8_t const *addr);
extern int          sfxge_mac_multicst_remove(sfxge_t *,
    uint8_t const *addr);
extern void         sfxge_mac_stop(sfxge_t *);
extern void         sfxge_mac_fini(sfxge_t *);
extern void         sfxge_mac_link_update(sfxge_t *sp,
    efx_link_mode_t mode);

extern int          sfxge_mcdi_init(sfxge_t *sp);
extern void         sfxge_mcdi_fini(sfxge_t *sp);
extern int          sfxge_mcdi_ioctl(sfxge_t *sp,
    sfxge_mcdi_ioc_t *smip);
extern int          sfxge_mcdi2_ioctl(sfxge_t *sp,
    sfxge_mcdi2_ioc_t *smip);

extern int          sfxge_phy_init(sfxge_t *);
extern void         sfxge_phy_link_mode_get(sfxge_t *,
    efx_link_mode_t *);
extern void         sfxge_phy_fini(sfxge_t *);
extern int          sfxge_phy_kstat_init(sfxge_t *sp);
extern void         sfxge_phy_kstat_fini(sfxge_t *sp);
extern uint8_t          sfxge_phy_lp_cap_test(sfxge_t *sp,
    uint32_t field);
extern int          sfxge_phy_cap_apply(sfxge_t *sp,
    boolean_t use_default);
extern uint8_t          sfxge_phy_cap_test(sfxge_t *sp, uint32_t flags,
    uint32_t field, boolean_t *mutablep);
extern int          sfxge_phy_cap_set(sfxge_t *sp, uint32_t field,
    int set);

extern int          sfxge_rx_init(sfxge_t *);
extern int          sfxge_rx_start(sfxge_t *);
extern void         sfxge_rx_coalesce_mode_get(sfxge_t *,
    sfxge_rx_coalesce_mode_t *);
extern int          sfxge_rx_coalesce_mode_set(sfxge_t *,
    sfxge_rx_coalesce_mode_t);
extern unsigned int     sfxge_rx_scale_prop_get(sfxge_t *);
extern void         sfxge_rx_scale_update(void *);
extern int          sfxge_rx_scale_count_get(sfxge_t *,
    unsigned int *);
extern int          sfxge_rx_scale_count_set(sfxge_t *,
    unsigned int);
extern void         sfxge_rx_qcomplete(sfxge_rxq_t *, boolean_t);
extern void         sfxge_rx_qflush_done(sfxge_rxq_t *);
extern void         sfxge_rx_qflush_failed(sfxge_rxq_t *);
extern void         sfxge_rx_qfpp_trim(sfxge_rxq_t *);
extern void         sfxge_rx_stop(sfxge_t *);
extern unsigned int         sfxge_rx_loaned(sfxge_t *);
extern void         sfxge_rx_fini(sfxge_t *);

extern int          sfxge_tx_init(sfxge_t *);
extern int          sfxge_tx_start(sfxge_t *);
extern int          sfxge_tx_packet_add(sfxge_t *, mblk_t *);
extern void         sfxge_tx_qcomplete(sfxge_txq_t *);
extern void         sfxge_tx_qflush_done(sfxge_txq_t *);
extern void         sfxge_tx_stop(sfxge_t *);
extern void         sfxge_tx_fini(sfxge_t *);
extern void         sfxge_tx_qdpl_flush(sfxge_txq_t *stp);

extern void         sfxge_sram_init(sfxge_t *);
extern int          sfxge_sram_buf_tbl_alloc(sfxge_t *, size_t,
    uint32_t *);
extern int          sfxge_sram_start(sfxge_t *);
extern int          sfxge_sram_buf_tbl_set(sfxge_t *, uint32_t,
    efsys_mem_t *, size_t);
extern void         sfxge_sram_buf_tbl_clear(sfxge_t *, uint32_t,
    size_t);
extern void         sfxge_sram_stop(sfxge_t *);
extern void         sfxge_sram_buf_tbl_free(sfxge_t *, uint32_t,
    size_t);
extern void         sfxge_sram_fini(sfxge_t *);

extern sfxge_packet_type_t  sfxge_pkthdr_parse(mblk_t *,
    struct ether_header **, struct ip **, struct tcphdr **, size_t *, size_t *,
    uint16_t *, uint16_t *);

extern int sfxge_toeplitz_hash_init(sfxge_t *);
extern void sfxge_toeplitz_hash_fini(sfxge_t *);
extern uint32_t sfxge_toeplitz_hash(sfxge_t *, unsigned int,
    uint8_t *, uint16_t, uint8_t *, uint16_t);

/*
 * 4-tuple hash for TCP/IPv4 used for LRO, TSO and TX queue selection.
 * To compute the same hash value as Siena/Huntington hardware, the inputs
 * must be in big endian (network) byte order.
 */
#define SFXGE_TCP_HASH(_sp, _raddr, _rport, _laddr, _lport, _hash)  \
    do { \
        (_hash) = sfxge_toeplitz_hash(_sp, \
                    sizeof (struct in_addr), \
                    (uint8_t *)(_raddr), \
                    (_rport), \
                    (uint8_t *)(_laddr), \
                    (_lport)); \
        _NOTE(CONSTANTCONDITION) \
    } while (B_FALSE)

/*
 * 4-tuple hash for non-TCP IPv4 packets, used for TX queue selection.
 * For UDP or SCTP packets, calculate a 4-tuple hash using port numbers.
 * For other IPv4 non-TCP packets, use zero for the port numbers.
 */
#define SFXGE_IP_HASH(_sp, _raddr, _rport, _laddr, _lport, _hash)   \
    SFXGE_TCP_HASH((_sp), (_raddr), (_rport), (_laddr), (_lport), (_hash))


extern int      sfxge_nvram_ioctl(sfxge_t *, sfxge_nvram_ioc_t *);

extern int      sfxge_pci_init(sfxge_t *);
extern void     sfxge_pcie_check_link(sfxge_t *, unsigned int,
    unsigned int);
extern void     sfxge_pci_fini(sfxge_t *);

extern int      sfxge_bar_init(sfxge_t *);
extern void     sfxge_bar_fini(sfxge_t *);

extern int      sfxge_vpd_ioctl(sfxge_t *, sfxge_vpd_ioc_t *);


#endif /* _KERNEL */

#ifdef  __cplusplus
}
#endif

#endif  /* _SYS_SFXGE_H */