io/vmxnet3s/vmxnet3_tx.c

62dadd654b88164ac263978699c78aa01647a39bYuri Pankov/*
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov * Copyright (C) 2007 VMware, Inc. All rights reserved.
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov *
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov * The contents of this file are subject to the terms of the Common
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov * Development and Distribution License (the "License") version 1.0
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov * and no later version.  You may not use this file except in
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov * compliance with the License.
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov *
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov * You can obtain a copy of the License at
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov *         http://www.opensource.org/licenses/cddl1.php
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov *
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov * See the License for the specific language governing permissions
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov * and limitations under the License.
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov */
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov/*
6849994e8263545ed3c0f6f5676e47b38e14f63eSebastien Roy * Copyright (c) 2012, 2016 by Delphix. All rights reserved.
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov */
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov#include <vmxnet3.h>
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov
62dadd654b88164ac263978699c78aa01647a39bYuri Pankovtypedef enum vmxnet3_txstatus {
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov    VMXNET3_TX_OK,
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov    VMXNET3_TX_FAILURE,
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov    VMXNET3_TX_PULLUP,
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov    VMXNET3_TX_RINGFULL
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov} vmxnet3_txstatus;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov
62dadd654b88164ac263978699c78aa01647a39bYuri Pankovtypedef struct vmxnet3_offload_t {
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov    uint16_t om;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov    uint16_t hlen;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov    uint16_t msscof;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov} vmxnet3_offload_t;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov/*
ca5345b6a28e9e9bfd0c135121d62c6b35a5390dSebastien Roy * Initialize a TxQueue. Currently nothing needs to be done.
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov */
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov/* ARGSUSED */
62dadd654b88164ac263978699c78aa01647a39bYuri Pankovint
62dadd654b88164ac263978699c78aa01647a39bYuri Pankovvmxnet3_txqueue_init(vmxnet3_softc_t *dp, vmxnet3_txqueue_t *txq)
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov{
6849994e8263545ed3c0f6f5676e47b38e14f63eSebastien Roy    return (0);
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov}
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov/*
ca5345b6a28e9e9bfd0c135121d62c6b35a5390dSebastien Roy * Finish a TxQueue by freeing all pending Tx.
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov */
62dadd654b88164ac263978699c78aa01647a39bYuri Pankovvoid
62dadd654b88164ac263978699c78aa01647a39bYuri Pankovvmxnet3_txqueue_fini(vmxnet3_softc_t *dp, vmxnet3_txqueue_t *txq)
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov{
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov    unsigned int i;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov    ASSERT(!dp->devEnabled);
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov    for (i = 0; i < txq->cmdRing.size; i++) {
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov        mblk_t *mp = txq->metaRing[i].mp;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov        if (mp) {
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov            freemsg(mp);
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov        }
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov    }
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov}
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov/*
ca5345b6a28e9e9bfd0c135121d62c6b35a5390dSebastien Roy * Build the offload context of a msg.
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov *
ca5345b6a28e9e9bfd0c135121d62c6b35a5390dSebastien Roy * Returns:
ca5345b6a28e9e9bfd0c135121d62c6b35a5390dSebastien Roy *  0 if everything went well.
ca5345b6a28e9e9bfd0c135121d62c6b35a5390dSebastien Roy *  +n if n bytes need to be pulled up.
ca5345b6a28e9e9bfd0c135121d62c6b35a5390dSebastien Roy *  -1 in case of error (not used).
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov */
62dadd654b88164ac263978699c78aa01647a39bYuri Pankovstatic int
62dadd654b88164ac263978699c78aa01647a39bYuri Pankovvmxnet3_tx_prepare_offload(vmxnet3_softc_t *dp, vmxnet3_offload_t *ol,
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov    mblk_t *mp)
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov{
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov    int ret = 0;
ca5345b6a28e9e9bfd0c135121d62c6b35a5390dSebastien Roy    uint32_t start, stuff, value, flags, lso_flag, mss;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov    ol->om = VMXNET3_OM_NONE;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov    ol->hlen = 0;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov    ol->msscof = 0;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov    hcksum_retrieve(mp, NULL, NULL, &start, &stuff, NULL, &value, &flags);
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov
ca5345b6a28e9e9bfd0c135121d62c6b35a5390dSebastien Roy    mac_lso_get(mp, &mss, &lso_flag);
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov
ca5345b6a28e9e9bfd0c135121d62c6b35a5390dSebastien Roy    if (flags || lso_flag) {
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov        struct ether_vlan_header *eth = (void *) mp->b_rptr;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov        uint8_t ethLen;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov        if (eth->ether_tpid == htons(ETHERTYPE_VLAN)) {
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov            ethLen = sizeof (struct ether_vlan_header);
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov        } else {
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov            ethLen = sizeof (struct ether_header);
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov        }
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov        VMXNET3_DEBUG(dp, 4, "flags=0x%x, ethLen=%u, start=%u, "
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov            "stuff=%u, value=%u\n", flags, ethLen, start, stuff, value);
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov
ca5345b6a28e9e9bfd0c135121d62c6b35a5390dSebastien Roy        if (lso_flag & HW_LSO) {
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov            mblk_t *mblk = mp;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov            uint8_t *ip, *tcp;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov            uint8_t ipLen, tcpLen;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov            /*
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov             * Copy e1000g's behavior:
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov             * - Do not assume all the headers are in the same mblk.
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov             * - Assume each header is always within one mblk.
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov             * - Assume the ethernet header is in the first mblk.
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov             */
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov            ip = mblk->b_rptr + ethLen;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov            if (ip >= mblk->b_wptr) {
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov                mblk = mblk->b_cont;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov                ip = mblk->b_rptr;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov            }
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov            ipLen = IPH_HDR_LENGTH((ipha_t *)ip);
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov            tcp = ip + ipLen;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov            if (tcp >= mblk->b_wptr) {
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov                mblk = mblk->b_cont;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov                tcp = mblk->b_rptr;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov            }
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov            tcpLen = TCP_HDR_LENGTH((tcph_t *)tcp);
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov            /* Careful, '>' instead of '>=' here */
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov            if (tcp + tcpLen > mblk->b_wptr) {
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov                mblk = mblk->b_cont;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov            }
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov            ol->om = VMXNET3_OM_TSO;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov            ol->hlen = ethLen + ipLen + tcpLen;
ca5345b6a28e9e9bfd0c135121d62c6b35a5390dSebastien Roy            ol->msscof = mss;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov            if (mblk != mp) {
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov                ret = ol->hlen;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov            }
ca5345b6a28e9e9bfd0c135121d62c6b35a5390dSebastien Roy        } else if (flags & HCK_PARTIALCKSUM) {
ca5345b6a28e9e9bfd0c135121d62c6b35a5390dSebastien Roy            ol->om = VMXNET3_OM_CSUM;
ca5345b6a28e9e9bfd0c135121d62c6b35a5390dSebastien Roy            ol->hlen = start + ethLen;
ca5345b6a28e9e9bfd0c135121d62c6b35a5390dSebastien Roy            ol->msscof = stuff + ethLen;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov        }
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov    }
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov    return (ret);
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov}
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov/*
ca5345b6a28e9e9bfd0c135121d62c6b35a5390dSebastien Roy * Map a msg into the Tx command ring of a vmxnet3 device.
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov *
ca5345b6a28e9e9bfd0c135121d62c6b35a5390dSebastien Roy * Returns:
ca5345b6a28e9e9bfd0c135121d62c6b35a5390dSebastien Roy *  VMXNET3_TX_OK if everything went well.
ca5345b6a28e9e9bfd0c135121d62c6b35a5390dSebastien Roy *  VMXNET3_TX_RINGFULL if the ring is nearly full.
ca5345b6a28e9e9bfd0c135121d62c6b35a5390dSebastien Roy *  VMXNET3_TX_PULLUP if the msg is overfragmented.
ca5345b6a28e9e9bfd0c135121d62c6b35a5390dSebastien Roy *  VMXNET3_TX_FAILURE if there was a DMA or offload error.
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov *
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov * Side effects:
ca5345b6a28e9e9bfd0c135121d62c6b35a5390dSebastien Roy *  The ring is filled if VMXNET3_TX_OK is returned.
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov */
62dadd654b88164ac263978699c78aa01647a39bYuri Pankovstatic vmxnet3_txstatus
62dadd654b88164ac263978699c78aa01647a39bYuri Pankovvmxnet3_tx_one(vmxnet3_softc_t *dp, vmxnet3_txqueue_t *txq,
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov    vmxnet3_offload_t *ol, mblk_t *mp)
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov{
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov    int ret = VMXNET3_TX_OK;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov    unsigned int frags = 0, totLen = 0;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov    vmxnet3_cmdring_t *cmdRing = &txq->cmdRing;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov    Vmxnet3_TxQueueCtrl *txqCtrl = txq->sharedCtrl;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov    Vmxnet3_GenericDesc *txDesc;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov    uint16_t sopIdx, eopIdx;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov    uint8_t sopGen, curGen;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov    mblk_t *mblk;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov    mutex_enter(&dp->txLock);
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov    sopIdx = eopIdx = cmdRing->next2fill;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov    sopGen = cmdRing->gen;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov    curGen = !cmdRing->gen;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov    for (mblk = mp; mblk != NULL; mblk = mblk->b_cont) {
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov        unsigned int len = MBLKL(mblk);
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov        ddi_dma_cookie_t cookie;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov        uint_t cookieCount;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov        if (len) {
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov            totLen += len;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov        } else {
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov            continue;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov        }
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov        if (ddi_dma_addr_bind_handle(dp->txDmaHandle, NULL,
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov            (caddr_t)mblk->b_rptr, len,
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov            DDI_DMA_RDWR | DDI_DMA_STREAMING, DDI_DMA_DONTWAIT, NULL,
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov            &cookie, &cookieCount) != DDI_DMA_MAPPED) {
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov            VMXNET3_WARN(dp, "ddi_dma_addr_bind_handle() failed\n");
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov            ret = VMXNET3_TX_FAILURE;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov            goto error;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov        }
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov        ASSERT(cookieCount);
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov        do {
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov            uint64_t addr = cookie.dmac_laddress;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov            size_t len = cookie.dmac_size;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov            do {
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov                uint32_t dw2, dw3;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov                size_t chunkLen;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov                ASSERT(!txq->metaRing[eopIdx].mp);
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov                ASSERT(cmdRing->avail - frags);
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov                if (frags >= cmdRing->size - 1 ||
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov                    (ol->om != VMXNET3_OM_TSO &&
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov                    frags >= VMXNET3_MAX_TXD_PER_PKT)) {
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov                    VMXNET3_DEBUG(dp, 2,
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov                        "overfragmented mp (%u)\n", frags);
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov                    (void) ddi_dma_unbind_handle(
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov                        dp->txDmaHandle);
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov                    ret = VMXNET3_TX_PULLUP;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov                    goto error;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov                }
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov                if (cmdRing->avail - frags <= 1) {
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov                    dp->txMustResched = B_TRUE;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov                    (void) ddi_dma_unbind_handle(
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov                        dp->txDmaHandle);
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov                    ret = VMXNET3_TX_RINGFULL;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov                    goto error;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov                }
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov                if (len > VMXNET3_MAX_TX_BUF_SIZE) {
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov                    chunkLen = VMXNET3_MAX_TX_BUF_SIZE;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov                } else {
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov                    chunkLen = len;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov                }
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov                frags++;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov                eopIdx = cmdRing->next2fill;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov                txDesc = VMXNET3_GET_DESC(cmdRing, eopIdx);
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov                ASSERT(txDesc->txd.gen != cmdRing->gen);
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov                /* txd.addr */
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov                txDesc->txd.addr = addr;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov                /* txd.dw2 */
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov                dw2 = chunkLen == VMXNET3_MAX_TX_BUF_SIZE ?
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov                    0 : chunkLen;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov                dw2 |= curGen << VMXNET3_TXD_GEN_SHIFT;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov                txDesc->dword[2] = dw2;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov                ASSERT(txDesc->txd.len == len ||
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov                    txDesc->txd.len == 0);
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov                /* txd.dw3 */
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov                dw3 = 0;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov                txDesc->dword[3] = dw3;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov                VMXNET3_INC_RING_IDX(cmdRing,
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov                    cmdRing->next2fill);
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov                curGen = cmdRing->gen;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov                addr += chunkLen;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov                len -= chunkLen;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov            } while (len);
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov            if (--cookieCount) {
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov                ddi_dma_nextcookie(dp->txDmaHandle, &cookie);
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov            }
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov        } while (cookieCount);
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov        (void) ddi_dma_unbind_handle(dp->txDmaHandle);
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov    }
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov    /* Update the EOP descriptor */
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov    txDesc = VMXNET3_GET_DESC(cmdRing, eopIdx);
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov    txDesc->dword[3] |= VMXNET3_TXD_CQ | VMXNET3_TXD_EOP;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov    /* Update the SOP descriptor. Must be done last */
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov    txDesc = VMXNET3_GET_DESC(cmdRing, sopIdx);
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov    if (ol->om == VMXNET3_OM_TSO && txDesc->txd.len != 0 &&
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov        txDesc->txd.len < ol->hlen) {
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov        ret = VMXNET3_TX_FAILURE;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov        goto error;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov    }
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov    txDesc->txd.om = ol->om;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov    txDesc->txd.hlen = ol->hlen;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov    txDesc->txd.msscof = ol->msscof;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov    membar_producer();
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov    txDesc->txd.gen = sopGen;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov    /* Update the meta ring & metadata */
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov    txq->metaRing[sopIdx].mp = mp;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov    txq->metaRing[eopIdx].sopIdx = sopIdx;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov    txq->metaRing[eopIdx].frags = frags;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov    cmdRing->avail -= frags;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov    if (ol->om == VMXNET3_OM_TSO) {
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov        txqCtrl->txNumDeferred +=
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov            (totLen - ol->hlen + ol->msscof - 1) / ol->msscof;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov    } else {
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov        txqCtrl->txNumDeferred++;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov    }
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov    VMXNET3_DEBUG(dp, 3, "tx 0x%p on [%u;%u]\n", mp, sopIdx, eopIdx);
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov    goto done;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov
62dadd654b88164ac263978699c78aa01647a39bYuri Pankoverror:
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov    /* Reverse the generation bits */
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov    while (sopIdx != cmdRing->next2fill) {
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov        VMXNET3_DEC_RING_IDX(cmdRing, cmdRing->next2fill);
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov        txDesc = VMXNET3_GET_DESC(cmdRing, cmdRing->next2fill);
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov        txDesc->txd.gen = !cmdRing->gen;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov    }
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov
62dadd654b88164ac263978699c78aa01647a39bYuri Pankovdone:
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov    mutex_exit(&dp->txLock);
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov    return (ret);
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov}
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov/*
ca5345b6a28e9e9bfd0c135121d62c6b35a5390dSebastien Roy * Send packets on a vmxnet3 device.
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov *
ca5345b6a28e9e9bfd0c135121d62c6b35a5390dSebastien Roy * Returns:
ca5345b6a28e9e9bfd0c135121d62c6b35a5390dSebastien Roy *  NULL in case of success or failure.
ca5345b6a28e9e9bfd0c135121d62c6b35a5390dSebastien Roy *  The mps to be retransmitted later if the ring is full.
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov */
62dadd654b88164ac263978699c78aa01647a39bYuri Pankovmblk_t *
62dadd654b88164ac263978699c78aa01647a39bYuri Pankovvmxnet3_tx(void *data, mblk_t *mps)
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov{
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov    vmxnet3_softc_t *dp = data;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov    vmxnet3_txqueue_t *txq = &dp->txQueue;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov    vmxnet3_cmdring_t *cmdRing = &txq->cmdRing;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov    Vmxnet3_TxQueueCtrl *txqCtrl = txq->sharedCtrl;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov    vmxnet3_txstatus status = VMXNET3_TX_OK;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov    mblk_t *mp;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov    ASSERT(mps != NULL);
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov    do {
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov        vmxnet3_offload_t ol;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov        int pullup;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov        mp = mps;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov        mps = mp->b_next;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov        mp->b_next = NULL;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov        if (DB_TYPE(mp) != M_DATA) {
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov            /*
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov             * PR #315560: M_PROTO mblks could be passed for
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov             * some reason. Drop them because we don't understand
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov             * them and because their contents are not Ethernet
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov             * frames anyway.
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov             */
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov            ASSERT(B_FALSE);
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov            freemsg(mp);
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov            continue;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov        }
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov        /*
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov         * Prepare the offload while we're still handling the original
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov         * message -- msgpullup() discards the metadata afterwards.
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov         */
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov        pullup = vmxnet3_tx_prepare_offload(dp, &ol, mp);
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov        if (pullup) {
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov            mblk_t *new_mp = msgpullup(mp, pullup);
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov            atomic_inc_32(&dp->tx_pullup_needed);
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov            freemsg(mp);
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov            if (new_mp) {
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov                mp = new_mp;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov            } else {
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov                atomic_inc_32(&dp->tx_pullup_failed);
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov                continue;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov            }
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov        }
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov        /*
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov         * Try to map the message in the Tx ring.
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov         * This call might fail for non-fatal reasons.
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov         */
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov        status = vmxnet3_tx_one(dp, txq, &ol, mp);
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov        if (status == VMXNET3_TX_PULLUP) {
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov            /*
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov             * Try one more time after flattening
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov             * the message with msgpullup().
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov             */
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov            if (mp->b_cont != NULL) {
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov                mblk_t *new_mp = msgpullup(mp, -1);
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov                atomic_inc_32(&dp->tx_pullup_needed);
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov                freemsg(mp);
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov                if (new_mp) {
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov                    mp = new_mp;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov                    status = vmxnet3_tx_one(dp, txq, &ol,
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov                        mp);
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov                } else {
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov                    atomic_inc_32(&dp->tx_pullup_failed);
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov                    continue;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov                }
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov            }
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov        }
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov        if (status != VMXNET3_TX_OK && status != VMXNET3_TX_RINGFULL) {
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov            /* Fatal failure, drop it */
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov            atomic_inc_32(&dp->tx_error);
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov            freemsg(mp);
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov        }
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov    } while (mps && status != VMXNET3_TX_RINGFULL);
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov    if (status == VMXNET3_TX_RINGFULL) {
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov        atomic_inc_32(&dp->tx_ring_full);
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov        mp->b_next = mps;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov        mps = mp;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov    } else {
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov        ASSERT(!mps);
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov    }
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov    /* Notify the device */
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov    mutex_enter(&dp->txLock);
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov    if (txqCtrl->txNumDeferred >= txqCtrl->txThreshold) {
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov        txqCtrl->txNumDeferred = 0;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov        VMXNET3_BAR0_PUT32(dp, VMXNET3_REG_TXPROD, cmdRing->next2fill);
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov    }
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov    mutex_exit(&dp->txLock);
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov    return (mps);
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov}
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov/*
ca5345b6a28e9e9bfd0c135121d62c6b35a5390dSebastien Roy * Parse a transmit queue and complete packets.
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov *
ca5345b6a28e9e9bfd0c135121d62c6b35a5390dSebastien Roy * Returns:
ca5345b6a28e9e9bfd0c135121d62c6b35a5390dSebastien Roy *  B_TRUE if Tx must be updated or B_FALSE if no action is required.
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov */
62dadd654b88164ac263978699c78aa01647a39bYuri Pankovboolean_t
62dadd654b88164ac263978699c78aa01647a39bYuri Pankovvmxnet3_tx_complete(vmxnet3_softc_t *dp, vmxnet3_txqueue_t *txq)
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov{
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov    vmxnet3_cmdring_t *cmdRing = &txq->cmdRing;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov    vmxnet3_compring_t *compRing = &txq->compRing;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov    Vmxnet3_GenericDesc *compDesc;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov    boolean_t completedTx = B_FALSE;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov    boolean_t ret = B_FALSE;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov    mutex_enter(&dp->txLock);
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov    compDesc = VMXNET3_GET_DESC(compRing, compRing->next2comp);
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov    while (compDesc->tcd.gen == compRing->gen) {
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov        vmxnet3_metatx_t *sopMetaDesc, *eopMetaDesc;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov        uint16_t sopIdx, eopIdx;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov        mblk_t *mp;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov        eopIdx = compDesc->tcd.txdIdx;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov        eopMetaDesc = &txq->metaRing[eopIdx];
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov        sopIdx = eopMetaDesc->sopIdx;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov        sopMetaDesc = &txq->metaRing[sopIdx];
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov        ASSERT(eopMetaDesc->frags);
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov        cmdRing->avail += eopMetaDesc->frags;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov        ASSERT(sopMetaDesc->mp);
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov        mp = sopMetaDesc->mp;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov        freemsg(mp);
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov        eopMetaDesc->sopIdx = 0;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov        eopMetaDesc->frags = 0;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov        sopMetaDesc->mp = NULL;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov        completedTx = B_TRUE;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov        VMXNET3_DEBUG(dp, 3, "cp 0x%p on [%u;%u]\n", mp, sopIdx,
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov            eopIdx);
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov        VMXNET3_INC_RING_IDX(compRing, compRing->next2comp);
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov        compDesc = VMXNET3_GET_DESC(compRing, compRing->next2comp);
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov    }
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov    if (dp->txMustResched && completedTx) {
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov        dp->txMustResched = B_FALSE;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov        ret = B_TRUE;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov    }
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov    mutex_exit(&dp->txLock);
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov    return (ret);
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov}