sfxge/common/efx_tx.c

	efx_tx.c revision 49ef7e0638c8b771d8a136eae78b1c0f99acc8e0
/*
 * Copyright (c) 2007-2015 Solarflare Communications Inc.
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *
 * 1. Redistributions of source code must retain the above copyright notice,
 *    this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright notice,
 *    this list of conditions and the following disclaimer in the documentation
 *    and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
 * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 *
 * The views and conclusions contained in the software and documentation are
 * those of the authors and should not be interpreted as representing official
 * policies, either expressed or implied, of the FreeBSD Project.
 */

#include "efx.h"
#include "efx_impl.h"

#if EFSYS_OPT_QSTATS
#define EFX_TX_QSTAT_INCR(_etp, _stat)                  \
    do {                                \
        (_etp)->et_stat[_stat]++;               \
    _NOTE(CONSTANTCONDITION)                    \
    } while (B_FALSE)
#else
#define EFX_TX_QSTAT_INCR(_etp, _stat)
#endif

#if EFSYS_OPT_SIENA

static  __checkReturn   efx_rc_t
siena_tx_init(
    __in        efx_nic_t *enp);

static          void
siena_tx_fini(
    __in        efx_nic_t *enp);

static  __checkReturn   efx_rc_t
siena_tx_qcreate(
    __in        efx_nic_t *enp,
    __in        unsigned int index,
    __in        unsigned int label,
    __in        efsys_mem_t *esmp,
    __in        size_t n,
    __in        uint32_t id,
    __in        uint16_t flags,
    __in        efx_evq_t *eep,
    __in        efx_txq_t *etp,
    __out       unsigned int *addedp);

static      void
siena_tx_qdestroy(
    __in    efx_txq_t *etp);

static  __checkReturn   efx_rc_t
siena_tx_qpost(
    __in        efx_txq_t *etp,
    __in_ecount(n)  efx_buffer_t *eb,
    __in        unsigned int n,
    __in        unsigned int completed,
    __inout     unsigned int *addedp);

static          void
siena_tx_qpush(
    __in    efx_txq_t *etp,
    __in    unsigned int added,
    __in    unsigned int pushed);

static  __checkReturn   efx_rc_t
siena_tx_qpace(
    __in        efx_txq_t *etp,
    __in        unsigned int ns);

static  __checkReturn   efx_rc_t
siena_tx_qflush(
    __in        efx_txq_t *etp);

static          void
siena_tx_qenable(
    __in    efx_txq_t *etp);

    __checkReturn   efx_rc_t
siena_tx_qdesc_post(
    __in        efx_txq_t *etp,
    __in_ecount(n)  efx_desc_t *ed,
    __in        unsigned int n,
    __in        unsigned int completed,
    __inout     unsigned int *addedp);

    void
siena_tx_qdesc_dma_create(
    __in    efx_txq_t *etp,
    __in    efsys_dma_addr_t addr,
    __in    size_t size,
    __in    boolean_t eop,
    __out   efx_desc_t *edp);

#if EFSYS_OPT_QSTATS
static          void
siena_tx_qstats_update(
    __in                efx_txq_t *etp,
    __inout_ecount(TX_NQSTATS)  efsys_stat_t *stat);
#endif

#endif /* EFSYS_OPT_SIENA */


#if EFSYS_OPT_SIENA
static const efx_tx_ops_t   __efx_tx_siena_ops = {
    siena_tx_init,              /* etxo_init */
    siena_tx_fini,              /* etxo_fini */
    siena_tx_qcreate,           /* etxo_qcreate */
    siena_tx_qdestroy,          /* etxo_qdestroy */
    siena_tx_qpost,             /* etxo_qpost */
    siena_tx_qpush,             /* etxo_qpush */
    siena_tx_qpace,             /* etxo_qpace */
    siena_tx_qflush,            /* etxo_qflush */
    siena_tx_qenable,           /* etxo_qenable */
    NULL,                   /* etxo_qpio_enable */
    NULL,                   /* etxo_qpio_disable */
    NULL,                   /* etxo_qpio_write */
    NULL,                   /* etxo_qpio_post */
    siena_tx_qdesc_post,            /* etxo_qdesc_post */
    siena_tx_qdesc_dma_create,      /* etxo_qdesc_dma_create */
    NULL,                   /* etxo_qdesc_tso_create */
    NULL,                   /* etxo_qdesc_tso2_create */
    NULL,                   /* etxo_qdesc_vlantci_create */
#if EFSYS_OPT_QSTATS
    siena_tx_qstats_update,         /* etxo_qstats_update */
#endif
};
#endif /* EFSYS_OPT_SIENA */

#if EFSYS_OPT_HUNTINGTON
static const efx_tx_ops_t   __efx_tx_hunt_ops = {
    ef10_tx_init,               /* etxo_init */
    ef10_tx_fini,               /* etxo_fini */
    ef10_tx_qcreate,            /* etxo_qcreate */
    ef10_tx_qdestroy,           /* etxo_qdestroy */
    ef10_tx_qpost,              /* etxo_qpost */
    ef10_tx_qpush,              /* etxo_qpush */
    ef10_tx_qpace,              /* etxo_qpace */
    ef10_tx_qflush,             /* etxo_qflush */
    ef10_tx_qenable,            /* etxo_qenable */
    ef10_tx_qpio_enable,            /* etxo_qpio_enable */
    ef10_tx_qpio_disable,           /* etxo_qpio_disable */
    ef10_tx_qpio_write,         /* etxo_qpio_write */
    ef10_tx_qpio_post,          /* etxo_qpio_post */
    ef10_tx_qdesc_post,         /* etxo_qdesc_post */
    ef10_tx_qdesc_dma_create,       /* etxo_qdesc_dma_create */
    ef10_tx_qdesc_tso_create,       /* etxo_qdesc_tso_create */
    ef10_tx_qdesc_tso2_create,      /* etxo_qdesc_tso2_create */
    ef10_tx_qdesc_vlantci_create,       /* etxo_qdesc_vlantci_create */
#if EFSYS_OPT_QSTATS
    ef10_tx_qstats_update,          /* etxo_qstats_update */
#endif
};
#endif /* EFSYS_OPT_HUNTINGTON */

#if EFSYS_OPT_MEDFORD
static const efx_tx_ops_t   __efx_tx_medford_ops = {
    ef10_tx_init,               /* etxo_init */
    ef10_tx_fini,               /* etxo_fini */
    ef10_tx_qcreate,            /* etxo_qcreate */
    ef10_tx_qdestroy,           /* etxo_qdestroy */
    ef10_tx_qpost,              /* etxo_qpost */
    ef10_tx_qpush,              /* etxo_qpush */
    ef10_tx_qpace,              /* etxo_qpace */
    ef10_tx_qflush,             /* etxo_qflush */
    ef10_tx_qenable,            /* etxo_qenable */
    ef10_tx_qpio_enable,            /* etxo_qpio_enable */
    ef10_tx_qpio_disable,           /* etxo_qpio_disable */
    ef10_tx_qpio_write,         /* etxo_qpio_write */
    ef10_tx_qpio_post,          /* etxo_qpio_post */
    ef10_tx_qdesc_post,         /* etxo_qdesc_post */
    ef10_tx_qdesc_dma_create,       /* etxo_qdesc_dma_create */
    NULL,                   /* etxo_qdesc_tso_create */
    ef10_tx_qdesc_tso2_create,      /* etxo_qdesc_tso2_create */
    ef10_tx_qdesc_vlantci_create,       /* etxo_qdesc_vlantci_create */
#if EFSYS_OPT_QSTATS
    ef10_tx_qstats_update,          /* etxo_qstats_update */
#endif
};
#endif /* EFSYS_OPT_MEDFORD */

    __checkReturn   efx_rc_t
efx_tx_init(
    __in        efx_nic_t *enp)
{
    const efx_tx_ops_t *etxop;
    efx_rc_t rc;

    EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
    EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC);

    if (!(enp->en_mod_flags & EFX_MOD_EV)) {
        rc = EINVAL;
        goto fail1;
    }

    if (enp->en_mod_flags & EFX_MOD_TX) {
        rc = EINVAL;
        goto fail2;
    }

    switch (enp->en_family) {
#if EFSYS_OPT_SIENA
    case EFX_FAMILY_SIENA:
        etxop = &__efx_tx_siena_ops;
        break;
#endif /* EFSYS_OPT_SIENA */

#if EFSYS_OPT_HUNTINGTON
    case EFX_FAMILY_HUNTINGTON:
        etxop = &__efx_tx_hunt_ops;
        break;
#endif /* EFSYS_OPT_HUNTINGTON */

#if EFSYS_OPT_MEDFORD
    case EFX_FAMILY_MEDFORD:
        etxop = &__efx_tx_medford_ops;
        break;
#endif /* EFSYS_OPT_MEDFORD */

    default:
        EFSYS_ASSERT(0);
        rc = ENOTSUP;
        goto fail3;
    }

    EFSYS_ASSERT3U(enp->en_tx_qcount, ==, 0);

    if ((rc = etxop->etxo_init(enp)) != 0)
        goto fail4;

    enp->en_etxop = etxop;
    enp->en_mod_flags |= EFX_MOD_TX;
    return (0);

fail4:
    EFSYS_PROBE(fail4);
fail3:
    EFSYS_PROBE(fail3);
fail2:
    EFSYS_PROBE(fail2);
fail1:
    EFSYS_PROBE1(fail1, efx_rc_t, rc);

    enp->en_etxop = NULL;
    enp->en_mod_flags &= ~EFX_MOD_TX;
    return (rc);
}

            void
efx_tx_fini(
    __in    efx_nic_t *enp)
{
    const efx_tx_ops_t *etxop = enp->en_etxop;

    EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
    EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC);
    EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_TX);
    EFSYS_ASSERT3U(enp->en_tx_qcount, ==, 0);

    etxop->etxo_fini(enp);

    enp->en_etxop = NULL;
    enp->en_mod_flags &= ~EFX_MOD_TX;
}

    __checkReturn   efx_rc_t
efx_tx_qcreate(
    __in        efx_nic_t *enp,
    __in        unsigned int index,
    __in        unsigned int label,
    __in        efsys_mem_t *esmp,
    __in        size_t n,
    __in        uint32_t id,
    __in        uint16_t flags,
    __in        efx_evq_t *eep,
    __deref_out efx_txq_t **etpp,
    __out       unsigned int *addedp)
{
    const efx_tx_ops_t *etxop = enp->en_etxop;
    efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
    efx_txq_t *etp;
    efx_rc_t rc;

    EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
    EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_TX);

    EFSYS_ASSERT3U(enp->en_tx_qcount + 1, <, encp->enc_txq_limit);

    /* Allocate an TXQ object */
    EFSYS_KMEM_ALLOC(enp->en_esip, sizeof (efx_txq_t), etp);

    if (etp == NULL) {
        rc = ENOMEM;
        goto fail1;
    }

    etp->et_magic = EFX_TXQ_MAGIC;
    etp->et_enp = enp;
    etp->et_index = index;
    etp->et_mask = n - 1;
    etp->et_esmp = esmp;

    /* Initial descriptor index may be modified by etxo_qcreate */
    *addedp = 0;

    if ((rc = etxop->etxo_qcreate(enp, index, label, esmp,
        n, id, flags, eep, etp, addedp)) != 0)
            goto fail2;

    enp->en_tx_qcount++;
    *etpp = etp;

    return (0);

fail2:
    EFSYS_PROBE(fail2);
    EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_txq_t), etp);
fail1:
    EFSYS_PROBE1(fail1, efx_rc_t, rc);
    return (rc);
}

        void
efx_tx_qdestroy(
    __in    efx_txq_t *etp)
{
    efx_nic_t *enp = etp->et_enp;
    const efx_tx_ops_t *etxop = enp->en_etxop;

    EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);

    EFSYS_ASSERT(enp->en_tx_qcount != 0);
    --enp->en_tx_qcount;

    etxop->etxo_qdestroy(etp);

    /* Free the TXQ object */
    EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_txq_t), etp);
}

    __checkReturn   efx_rc_t
efx_tx_qpost(
    __in        efx_txq_t *etp,
    __in_ecount(n)  efx_buffer_t *eb,
    __in        unsigned int n,
    __in        unsigned int completed,
    __inout     unsigned int *addedp)
{
    efx_nic_t *enp = etp->et_enp;
    const efx_tx_ops_t *etxop = enp->en_etxop;
    efx_rc_t rc;

    EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);

    if ((rc = etxop->etxo_qpost(etp, eb,
        n, completed, addedp)) != 0)
        goto fail1;

    return (0);

fail1:
    EFSYS_PROBE1(fail1, efx_rc_t, rc);
    return (rc);
}

            void
efx_tx_qpush(
    __in    efx_txq_t *etp,
    __in    unsigned int added,
    __in    unsigned int pushed)
{
    efx_nic_t *enp = etp->et_enp;
    const efx_tx_ops_t *etxop = enp->en_etxop;

    EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);

    etxop->etxo_qpush(etp, added, pushed);
}

    __checkReturn   efx_rc_t
efx_tx_qpace(
    __in        efx_txq_t *etp,
    __in        unsigned int ns)
{
    efx_nic_t *enp = etp->et_enp;
    const efx_tx_ops_t *etxop = enp->en_etxop;
    efx_rc_t rc;

    EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);

    if ((rc = etxop->etxo_qpace(etp, ns)) != 0)
        goto fail1;

    return (0);

fail1:
    EFSYS_PROBE1(fail1, efx_rc_t, rc);
    return (rc);
}

    __checkReturn   efx_rc_t
efx_tx_qflush(
    __in    efx_txq_t *etp)
{
    efx_nic_t *enp = etp->et_enp;
    const efx_tx_ops_t *etxop = enp->en_etxop;
    efx_rc_t rc;

    EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);

    if ((rc = etxop->etxo_qflush(etp)) != 0)
        goto fail1;

    return (0);

fail1:
    EFSYS_PROBE1(fail1, efx_rc_t, rc);
    return (rc);
}

            void
efx_tx_qenable(
    __in    efx_txq_t *etp)
{
    efx_nic_t *enp = etp->et_enp;
    const efx_tx_ops_t *etxop = enp->en_etxop;

    EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);

    etxop->etxo_qenable(etp);
}

    __checkReturn   efx_rc_t
efx_tx_qpio_enable(
    __in    efx_txq_t *etp)
{
    efx_nic_t *enp = etp->et_enp;
    const efx_tx_ops_t *etxop = enp->en_etxop;
    efx_rc_t rc;

    EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);

    if (~enp->en_features & EFX_FEATURE_PIO_BUFFERS) {
        rc = ENOTSUP;
        goto fail1;
    }
    if (etxop->etxo_qpio_enable == NULL) {
        rc = ENOTSUP;
        goto fail2;
    }
    if ((rc = etxop->etxo_qpio_enable(etp)) != 0)
        goto fail3;

    return (0);

fail3:
    EFSYS_PROBE(fail3);
fail2:
    EFSYS_PROBE(fail2);
fail1:
    EFSYS_PROBE1(fail1, efx_rc_t, rc);
    return (rc);
}

        void
efx_tx_qpio_disable(
    __in    efx_txq_t *etp)
{
    efx_nic_t *enp = etp->et_enp;
    const efx_tx_ops_t *etxop = enp->en_etxop;

    EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);

    if (etxop->etxo_qpio_disable != NULL)
        etxop->etxo_qpio_disable(etp);
}

    __checkReturn   efx_rc_t
efx_tx_qpio_write(
    __in            efx_txq_t *etp,
    __in_ecount(buf_length) uint8_t *buffer,
    __in            size_t buf_length,
    __in            size_t pio_buf_offset)
{
    efx_nic_t *enp = etp->et_enp;
    const efx_tx_ops_t *etxop = enp->en_etxop;
    efx_rc_t rc;

    EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);

    if (etxop->etxo_qpio_write != NULL) {
        if ((rc = etxop->etxo_qpio_write(etp, buffer, buf_length,
                        pio_buf_offset)) != 0)
            goto fail1;
        return (0);
    }

    return (ENOTSUP);

fail1:
    EFSYS_PROBE1(fail1, efx_rc_t, rc);
    return (rc);
}

    __checkReturn   efx_rc_t
efx_tx_qpio_post(
    __in            efx_txq_t *etp,
    __in            size_t pkt_length,
    __in            unsigned int completed,
    __inout         unsigned int *addedp)
{
    efx_nic_t *enp = etp->et_enp;
    const efx_tx_ops_t *etxop = enp->en_etxop;
    efx_rc_t rc;

    EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);

    if (etxop->etxo_qpio_post != NULL) {
        if ((rc = etxop->etxo_qpio_post(etp, pkt_length, completed,
                        addedp)) != 0)
            goto fail1;
        return (0);
    }

    return (ENOTSUP);

fail1:
    EFSYS_PROBE1(fail1, efx_rc_t, rc);
    return (rc);
}

    __checkReturn   efx_rc_t
efx_tx_qdesc_post(
    __in        efx_txq_t *etp,
    __in_ecount(n)  efx_desc_t *ed,
    __in        unsigned int n,
    __in        unsigned int completed,
    __inout     unsigned int *addedp)
{
    efx_nic_t *enp = etp->et_enp;
    const efx_tx_ops_t *etxop = enp->en_etxop;
    efx_rc_t rc;

    EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);

    if ((rc = etxop->etxo_qdesc_post(etp, ed,
        n, completed, addedp)) != 0)
        goto fail1;

    return (0);

fail1:
    EFSYS_PROBE1(fail1, efx_rc_t, rc);
    return (rc);
}

    void
efx_tx_qdesc_dma_create(
    __in    efx_txq_t *etp,
    __in    efsys_dma_addr_t addr,
    __in    size_t size,
    __in    boolean_t eop,
    __out   efx_desc_t *edp)
{
    efx_nic_t *enp = etp->et_enp;
    const efx_tx_ops_t *etxop = enp->en_etxop;

    EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
    EFSYS_ASSERT(etxop->etxo_qdesc_dma_create != NULL);

    etxop->etxo_qdesc_dma_create(etp, addr, size, eop, edp);
}

    void
efx_tx_qdesc_tso_create(
    __in    efx_txq_t *etp,
    __in    uint16_t ipv4_id,
    __in    uint32_t tcp_seq,
    __in    uint8_t  tcp_flags,
    __out   efx_desc_t *edp)
{
    efx_nic_t *enp = etp->et_enp;
    const efx_tx_ops_t *etxop = enp->en_etxop;

    EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
    EFSYS_ASSERT(etxop->etxo_qdesc_tso_create != NULL);

    etxop->etxo_qdesc_tso_create(etp, ipv4_id, tcp_seq, tcp_flags, edp);
}

    void
efx_tx_qdesc_tso2_create(
    __in            efx_txq_t *etp,
    __in            uint16_t ipv4_id,
    __in            uint32_t tcp_seq,
    __in            uint16_t mss,
    __out_ecount(count) efx_desc_t *edp,
    __in            int count)
{
    efx_nic_t *enp = etp->et_enp;
    const efx_tx_ops_t *etxop = enp->en_etxop;

    EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
    EFSYS_ASSERT(etxop->etxo_qdesc_tso2_create != NULL);

    etxop->etxo_qdesc_tso2_create(etp, ipv4_id, tcp_seq, mss, edp, count);
}

    void
efx_tx_qdesc_vlantci_create(
    __in    efx_txq_t *etp,
    __in    uint16_t tci,
    __out   efx_desc_t *edp)
{
    efx_nic_t *enp = etp->et_enp;
    const efx_tx_ops_t *etxop = enp->en_etxop;

    EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
    EFSYS_ASSERT(etxop->etxo_qdesc_vlantci_create != NULL);

    etxop->etxo_qdesc_vlantci_create(etp, tci, edp);
}


#if EFSYS_OPT_QSTATS
            void
efx_tx_qstats_update(
    __in                efx_txq_t *etp,
    __inout_ecount(TX_NQSTATS)  efsys_stat_t *stat)
{
    efx_nic_t *enp = etp->et_enp;
    const efx_tx_ops_t *etxop = enp->en_etxop;

    EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);

    etxop->etxo_qstats_update(etp, stat);
}
#endif


#if EFSYS_OPT_SIENA

static  __checkReturn   efx_rc_t
siena_tx_init(
    __in        efx_nic_t *enp)
{
    efx_oword_t oword;

    /*
     * Disable the timer-based TX DMA backoff and allow TX DMA to be
     * controlled by the RX FIFO fill level (although always allow a
     * minimal trickle).
     */
    EFX_BAR_READO(enp, FR_AZ_TX_RESERVED_REG, &oword);
    EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_RX_SPACER, 0xfe);
    EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_RX_SPACER_EN, 1);
    EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_ONE_PKT_PER_Q, 1);
    EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_PUSH_EN, 0);
    EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DIS_NON_IP_EV, 1);
    EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_PREF_THRESHOLD, 2);
    EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_PREF_WD_TMR, 0x3fffff);

    /*
     * Filter all packets less than 14 bytes to avoid parsing
     * errors.
     */
    EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_FLUSH_MIN_LEN_EN, 1);
    EFX_BAR_WRITEO(enp, FR_AZ_TX_RESERVED_REG, &oword);

    /*
     * Do not set TX_NO_EOP_DISC_EN, since it limits packets to 16
     * descriptors (which is bad).
     */
    EFX_BAR_READO(enp, FR_AZ_TX_CFG_REG, &oword);
    EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_NO_EOP_DISC_EN, 0);
    EFX_BAR_WRITEO(enp, FR_AZ_TX_CFG_REG, &oword);

    return (0);
}

#define EFX_TX_DESC(_etp, _addr, _size, _eop, _added)           \
    do {                                \
        unsigned int id;                    \
        size_t offset;                      \
        efx_qword_t qword;                  \
                                    \
        id = (_added)++ & (_etp)->et_mask;          \
        offset = id * sizeof (efx_qword_t);         \
                                    \
        EFSYS_PROBE5(tx_post, unsigned int, (_etp)->et_index,   \
            unsigned int, id, efsys_dma_addr_t, (_addr),    \
            size_t, (_size), boolean_t, (_eop));        \
                                    \
        EFX_POPULATE_QWORD_4(qword,             \
            FSF_AZ_TX_KER_CONT, (_eop) ? 0 : 1,         \
            FSF_AZ_TX_KER_BYTE_COUNT, (uint32_t)(_size),    \
            FSF_AZ_TX_KER_BUF_ADDR_DW0,             \
            (uint32_t)((_addr) & 0xffffffff),           \
            FSF_AZ_TX_KER_BUF_ADDR_DW1,             \
            (uint32_t)((_addr) >> 32));             \
        EFSYS_MEM_WRITEQ((_etp)->et_esmp, offset, &qword);  \
                                    \
        _NOTE(CONSTANTCONDITION)                \
    } while (B_FALSE)

static  __checkReturn   efx_rc_t
siena_tx_qpost(
    __in        efx_txq_t *etp,
    __in_ecount(n)  efx_buffer_t *eb,
    __in        unsigned int n,
    __in        unsigned int completed,
    __inout     unsigned int *addedp)
{
    unsigned int added = *addedp;
    unsigned int i;
    int rc = ENOSPC;

    if (added - completed + n > EFX_TXQ_LIMIT(etp->et_mask + 1))
        goto fail1;

    for (i = 0; i < n; i++) {
        efx_buffer_t *ebp = &eb[i];
        efsys_dma_addr_t start = ebp->eb_addr;
        size_t size = ebp->eb_size;
        efsys_dma_addr_t end = start + size;

        /* Fragments must not span 4k boundaries. */
        EFSYS_ASSERT(P2ROUNDUP(start + 1, 4096) >= end);

        EFX_TX_DESC(etp, start, size, ebp->eb_eop, added);
    }

    EFX_TX_QSTAT_INCR(etp, TX_POST);

    *addedp = added;
    return (0);

fail1:
    EFSYS_PROBE1(fail1, efx_rc_t, rc);

    return (rc);
}

static      void
siena_tx_qpush(
    __in    efx_txq_t *etp,
    __in    unsigned int added,
    __in    unsigned int pushed)
{
    efx_nic_t *enp = etp->et_enp;
    uint32_t wptr;
    efx_dword_t dword;
    efx_oword_t oword;

    /* Push the populated descriptors out */
    wptr = added & etp->et_mask;

    EFX_POPULATE_OWORD_1(oword, FRF_AZ_TX_DESC_WPTR, wptr);

    /* Only write the third DWORD */
    EFX_POPULATE_DWORD_1(dword,
        EFX_DWORD_0, EFX_OWORD_FIELD(oword, EFX_DWORD_3));

    /* Guarantee ordering of memory (descriptors) and PIO (doorbell) */
    EFX_DMA_SYNC_QUEUE_FOR_DEVICE(etp->et_esmp, etp->et_mask + 1,
        wptr, pushed & etp->et_mask);
    EFSYS_PIO_WRITE_BARRIER();
    EFX_BAR_TBL_WRITED3(enp, FR_BZ_TX_DESC_UPD_REGP0,
                etp->et_index, &dword, B_FALSE);
}

#define EFX_MAX_PACE_VALUE 20

static  __checkReturn   efx_rc_t
siena_tx_qpace(
    __in        efx_txq_t *etp,
    __in        unsigned int ns)
{
    efx_nic_t *enp = etp->et_enp;
    efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
    efx_oword_t oword;
    unsigned int pace_val;
    unsigned int timer_period;
    efx_rc_t rc;

    if (ns == 0) {
        pace_val = 0;
    } else {
        /*
         * The pace_val to write into the table is s.t
         * ns <= timer_period * (2 ^ pace_val)
         */
        timer_period = 104 / encp->enc_clk_mult;
        for (pace_val = 1; pace_val <= EFX_MAX_PACE_VALUE; pace_val++) {
            if ((timer_period << pace_val) >= ns)
                break;
        }
    }
    if (pace_val > EFX_MAX_PACE_VALUE) {
        rc = EINVAL;
        goto fail1;
    }

    /* Update the pacing table */
    EFX_POPULATE_OWORD_1(oword, FRF_AZ_TX_PACE, pace_val);
    EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_PACE_TBL, etp->et_index,
        &oword, B_TRUE);

    return (0);

fail1:
    EFSYS_PROBE1(fail1, efx_rc_t, rc);

    return (rc);
}

static  __checkReturn   efx_rc_t
siena_tx_qflush(
    __in        efx_txq_t *etp)
{
    efx_nic_t *enp = etp->et_enp;
    efx_oword_t oword;
    uint32_t label;

    (void) efx_tx_qpace(etp, 0);

    label = etp->et_index;

    /* Flush the queue */
    EFX_POPULATE_OWORD_2(oword, FRF_AZ_TX_FLUSH_DESCQ_CMD, 1,
        FRF_AZ_TX_FLUSH_DESCQ, label);
    EFX_BAR_WRITEO(enp, FR_AZ_TX_FLUSH_DESCQ_REG, &oword);

    return (0);
}

static      void
siena_tx_qenable(
    __in    efx_txq_t *etp)
{
    efx_nic_t *enp = etp->et_enp;
    efx_oword_t oword;

    EFX_BAR_TBL_READO(enp, FR_AZ_TX_DESC_PTR_TBL,
                etp->et_index, &oword, B_TRUE);

    EFSYS_PROBE5(tx_descq_ptr, unsigned int, etp->et_index,
        uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_3),
        uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_2),
        uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_1),
        uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_0));

    EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DC_HW_RPTR, 0);
    EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DESCQ_HW_RPTR, 0);
    EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DESCQ_EN, 1);

    EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_DESC_PTR_TBL,
                etp->et_index, &oword, B_TRUE);
}

static  __checkReturn   efx_rc_t
siena_tx_qcreate(
    __in        efx_nic_t *enp,
    __in        unsigned int index,
    __in        unsigned int label,
    __in        efsys_mem_t *esmp,
    __in        size_t n,
    __in        uint32_t id,
    __in        uint16_t flags,
    __in        efx_evq_t *eep,
    __in        efx_txq_t *etp,
    __out       unsigned int *addedp)
{
    efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
    efx_oword_t oword;
    uint32_t size;
    efx_rc_t rc;

    _NOTE(ARGUNUSED(esmp));

    EFX_STATIC_ASSERT(EFX_EV_TX_NLABELS ==
        (1 << FRF_AZ_TX_DESCQ_LABEL_WIDTH));
    EFSYS_ASSERT3U(label, <, EFX_EV_TX_NLABELS);

    EFSYS_ASSERT(ISP2(EFX_TXQ_MAXNDESCS(encp)));
    EFX_STATIC_ASSERT(ISP2(EFX_TXQ_MINNDESCS));

    if (!ISP2(n) || (n < EFX_TXQ_MINNDESCS) || (n > EFX_EVQ_MAXNEVS)) {
        rc = EINVAL;
        goto fail1;
    }
    if (index >= encp->enc_txq_limit) {
        rc = EINVAL;
        goto fail2;
    }
    for (size = 0;
        (1 << size) <= (EFX_TXQ_MAXNDESCS(encp) / EFX_TXQ_MINNDESCS);
        size++)
        if ((1 << size) == (int)(n / EFX_TXQ_MINNDESCS))
            break;
    if (id + (1 << size) >= encp->enc_buftbl_limit) {
        rc = EINVAL;
        goto fail3;
    }

    /* Set up the new descriptor queue */
    *addedp = 0;

    EFX_POPULATE_OWORD_6(oword,
        FRF_AZ_TX_DESCQ_BUF_BASE_ID, id,
        FRF_AZ_TX_DESCQ_EVQ_ID, eep->ee_index,
        FRF_AZ_TX_DESCQ_OWNER_ID, 0,
        FRF_AZ_TX_DESCQ_LABEL, label,
        FRF_AZ_TX_DESCQ_SIZE, size,
        FRF_AZ_TX_DESCQ_TYPE, 0);

    EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_NON_IP_DROP_DIS, 1);
    EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_IP_CHKSM_DIS,
        (flags & EFX_TXQ_CKSUM_IPV4) ? 0 : 1);
    EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_TCP_CHKSM_DIS,
        (flags & EFX_TXQ_CKSUM_TCPUDP) ? 0 : 1);

    EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_DESC_PTR_TBL,
        etp->et_index, &oword, B_TRUE);

    return (0);

fail3:
    EFSYS_PROBE(fail3);
fail2:
    EFSYS_PROBE(fail2);
fail1:
    EFSYS_PROBE1(fail1, efx_rc_t, rc);

    return (rc);
}

    __checkReturn   efx_rc_t
siena_tx_qdesc_post(
    __in        efx_txq_t *etp,
    __in_ecount(n)  efx_desc_t *ed,
    __in        unsigned int n,
    __in        unsigned int completed,
    __inout     unsigned int *addedp)
{
    unsigned int added = *addedp;
    unsigned int i;
    efx_rc_t rc;

    if (added - completed + n > EFX_TXQ_LIMIT(etp->et_mask + 1)) {
        rc = ENOSPC;
        goto fail1;
    }

    for (i = 0; i < n; i++) {
        efx_desc_t *edp = &ed[i];
        unsigned int id;
        size_t offset;

        id = added++ & etp->et_mask;
        offset = id * sizeof (efx_desc_t);

        EFSYS_MEM_WRITEQ(etp->et_esmp, offset, &edp->ed_eq);
    }

    EFSYS_PROBE3(tx_desc_post, unsigned int, etp->et_index,
            unsigned int, added, unsigned int, n);

    EFX_TX_QSTAT_INCR(etp, TX_POST);

    *addedp = added;
    return (0);

fail1:
    EFSYS_PROBE1(fail1, efx_rc_t, rc);
    return (rc);
}

    void
siena_tx_qdesc_dma_create(
    __in    efx_txq_t *etp,
    __in    efsys_dma_addr_t addr,
    __in    size_t size,
    __in    boolean_t eop,
    __out   efx_desc_t *edp)
{
    /* Fragments must not span 4k boundaries. */
    EFSYS_ASSERT(P2ROUNDUP(addr + 1, 4096) >= addr + size);

    EFSYS_PROBE4(tx_desc_dma_create, unsigned int, etp->et_index,
            efsys_dma_addr_t, addr,
            size_t, size, boolean_t, eop);

    EFX_POPULATE_QWORD_4(edp->ed_eq,
                FSF_AZ_TX_KER_CONT, eop ? 0 : 1,
                FSF_AZ_TX_KER_BYTE_COUNT, (uint32_t)size,
                FSF_AZ_TX_KER_BUF_ADDR_DW0,
                (uint32_t)(addr & 0xffffffff),
                FSF_AZ_TX_KER_BUF_ADDR_DW1,
                (uint32_t)(addr >> 32));
}

#endif /* EFSYS_OPT_SIENA */

#if EFSYS_OPT_QSTATS
#if EFSYS_OPT_NAMES
/* START MKCONFIG GENERATED EfxTransmitQueueStatNamesBlock 9d8d26a0a5e2c453 */
static const char   *__efx_tx_qstat_name[] = {
    "post",
    "post_pio",
};
/* END MKCONFIG GENERATED EfxTransmitQueueStatNamesBlock */

        const char *
efx_tx_qstat_name(
    __in    efx_nic_t *enp,
    __in    unsigned int id)
{
    _NOTE(ARGUNUSED(enp))
    EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
    EFSYS_ASSERT3U(id, <, TX_NQSTATS);

    return (__efx_tx_qstat_name[id]);
}
#endif  /* EFSYS_OPT_NAMES */
#endif /* EFSYS_OPT_QSTATS */

#if EFSYS_OPT_SIENA

#if EFSYS_OPT_QSTATS
static                  void
siena_tx_qstats_update(
    __in                efx_txq_t *etp,
    __inout_ecount(TX_NQSTATS)  efsys_stat_t *stat)
{
    unsigned int id;

    for (id = 0; id < TX_NQSTATS; id++) {
        efsys_stat_t *essp = &stat[id];

        EFSYS_STAT_INCR(essp, etp->et_stat[id]);
        etp->et_stat[id] = 0;
    }
}
#endif  /* EFSYS_OPT_QSTATS */

static      void
siena_tx_qdestroy(
    __in    efx_txq_t *etp)
{
    efx_nic_t *enp = etp->et_enp;
    efx_oword_t oword;

    /* Purge descriptor queue */
    EFX_ZERO_OWORD(oword);

    EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_DESC_PTR_TBL,
                etp->et_index, &oword, B_TRUE);
}

static      void
siena_tx_fini(
    __in    efx_nic_t *enp)
{
    _NOTE(ARGUNUSED(enp))
}

#endif /* EFSYS_OPT_SIENA */