io/vmxnet3s/vmxnet3_rx.c

62dadd654b88164ac263978699c78aa01647a39bYuri Pankov/*
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov * Copyright (C) 2007 VMware, Inc. All rights reserved.
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov *
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov * The contents of this file are subject to the terms of the Common
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov * Development and Distribution License (the "License") version 1.0
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov * and no later version.  You may not use this file except in
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov * compliance with the License.
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov *
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov * You can obtain a copy of the License at
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov *         http://www.opensource.org/licenses/cddl1.php
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov *
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov * See the License for the specific language governing permissions
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov * and limitations under the License.
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov */
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov/*
6849994e8263545ed3c0f6f5676e47b38e14f63eSebastien Roy * Copyright (c) 2013, 2016 by Delphix. All rights reserved.
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov */
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov#include <vmxnet3.h>
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov
ca5345b6a28e9e9bfd0c135121d62c6b35a5390dSebastien Roystatic void vmxnet3_put_rxbuf(vmxnet3_rxbuf_t *);
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov/*
ca5345b6a28e9e9bfd0c135121d62c6b35a5390dSebastien Roy * Allocate a new rxBuf from memory. All its fields are set except
ca5345b6a28e9e9bfd0c135121d62c6b35a5390dSebastien Roy * for its associated mblk which has to be allocated later.
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov *
ca5345b6a28e9e9bfd0c135121d62c6b35a5390dSebastien Roy * Returns:
ca5345b6a28e9e9bfd0c135121d62c6b35a5390dSebastien Roy *  A new rxBuf or NULL.
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov */
62dadd654b88164ac263978699c78aa01647a39bYuri Pankovstatic vmxnet3_rxbuf_t *
62dadd654b88164ac263978699c78aa01647a39bYuri Pankovvmxnet3_alloc_rxbuf(vmxnet3_softc_t *dp, boolean_t canSleep)
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov{
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov    vmxnet3_rxbuf_t *rxBuf;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov    int flag = canSleep ? KM_SLEEP : KM_NOSLEEP;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov    int err;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov    rxBuf = kmem_zalloc(sizeof (vmxnet3_rxbuf_t), flag);
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov    if (!rxBuf) {
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov        atomic_inc_32(&dp->rx_alloc_failed);
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov        return (NULL);
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov    }
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov    if ((err = vmxnet3_alloc_dma_mem_1(dp, &rxBuf->dma, (dp->cur_mtu + 18),
6849994e8263545ed3c0f6f5676e47b38e14f63eSebastien Roy        canSleep)) != 0) {
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov        VMXNET3_DEBUG(dp, 0, "Failed to allocate %d bytes for rx buf, "
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov            "err:%d\n", (dp->cur_mtu + 18), err);
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov        kmem_free(rxBuf, sizeof (vmxnet3_rxbuf_t));
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov        atomic_inc_32(&dp->rx_alloc_failed);
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov        return (NULL);
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov    }
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov    rxBuf->freeCB.free_func = vmxnet3_put_rxbuf;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov    rxBuf->freeCB.free_arg = (caddr_t)rxBuf;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov    rxBuf->dp = dp;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov
6849994e8263545ed3c0f6f5676e47b38e14f63eSebastien Roy    atomic_inc_32(&dp->rx_num_bufs);
6849994e8263545ed3c0f6f5676e47b38e14f63eSebastien Roy    atomic_inc_32(&dp->rx_alloc_buf);
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov    return (rxBuf);
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov}
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov
62dadd654b88164ac263978699c78aa01647a39bYuri Pankovstatic void
62dadd654b88164ac263978699c78aa01647a39bYuri Pankovvmxnet3_free_rxbuf(vmxnet3_softc_t *dp, vmxnet3_rxbuf_t *rxBuf)
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov{
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov    vmxnet3_free_dma_mem(&rxBuf->dma);
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov    kmem_free(rxBuf, sizeof (vmxnet3_rxbuf_t));
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov
6849994e8263545ed3c0f6f5676e47b38e14f63eSebastien Roy#ifndef DEBUG
6849994e8263545ed3c0f6f5676e47b38e14f63eSebastien Roy    atomic_dec_32(&dp->rx_num_bufs);
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov#else
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov    {
6849994e8263545ed3c0f6f5676e47b38e14f63eSebastien Roy        uint32_t nv = atomic_dec_32_nv(&dp->rx_num_bufs);
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov        ASSERT(nv != (uint32_t)-1);
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov    }
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov#endif
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov}
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov/*
6849994e8263545ed3c0f6f5676e47b38e14f63eSebastien Roy * Return a rxBuf to the pool. The init argument, when B_TRUE, indicates
6849994e8263545ed3c0f6f5676e47b38e14f63eSebastien Roy * that we're being called for the purpose of pool initialization, and
6849994e8263545ed3c0f6f5676e47b38e14f63eSebastien Roy * therefore, we should place the buffer in the pool even if the device
6849994e8263545ed3c0f6f5676e47b38e14f63eSebastien Roy * isn't enabled.
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov *
ca5345b6a28e9e9bfd0c135121d62c6b35a5390dSebastien Roy * Returns:
6849994e8263545ed3c0f6f5676e47b38e14f63eSebastien Roy *  B_TRUE if the buffer was returned to the pool, or B_FALSE if it
6849994e8263545ed3c0f6f5676e47b38e14f63eSebastien Roy *  wasn't (e.g. if the device is stopped).
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov */
62dadd654b88164ac263978699c78aa01647a39bYuri Pankovstatic boolean_t
6849994e8263545ed3c0f6f5676e47b38e14f63eSebastien Royvmxnet3_put_rxpool_buf(vmxnet3_softc_t *dp, vmxnet3_rxbuf_t *rxBuf,
6849994e8263545ed3c0f6f5676e47b38e14f63eSebastien Roy    boolean_t init)
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov{
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov    vmxnet3_rxpool_t *rxPool = &dp->rxPool;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov    boolean_t returned = B_FALSE;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov    mutex_enter(&dp->rxPoolLock);
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov    ASSERT(rxPool->nBufs <= rxPool->nBufsLimit);
6849994e8263545ed3c0f6f5676e47b38e14f63eSebastien Roy    if ((dp->devEnabled || init) && rxPool->nBufs < rxPool->nBufsLimit) {
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov        ASSERT((rxPool->listHead == NULL && rxPool->nBufs == 0) ||
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov            (rxPool->listHead != NULL && rxPool->nBufs != 0));
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov        rxBuf->next = rxPool->listHead;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov        rxPool->listHead = rxBuf;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov        rxPool->nBufs++;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov        returned = B_TRUE;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov    }
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov    mutex_exit(&dp->rxPoolLock);
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov    return (returned);
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov}
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov/*
ca5345b6a28e9e9bfd0c135121d62c6b35a5390dSebastien Roy * Return a rxBuf to the pool or free it.
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov */
62dadd654b88164ac263978699c78aa01647a39bYuri Pankovstatic void
62dadd654b88164ac263978699c78aa01647a39bYuri Pankovvmxnet3_put_rxbuf(vmxnet3_rxbuf_t *rxBuf)
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov{
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov    vmxnet3_softc_t *dp = rxBuf->dp;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov
6849994e8263545ed3c0f6f5676e47b38e14f63eSebastien Roy    if (!vmxnet3_put_rxpool_buf(dp, rxBuf, B_FALSE))
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov        vmxnet3_free_rxbuf(dp, rxBuf);
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov}
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov/*
ca5345b6a28e9e9bfd0c135121d62c6b35a5390dSebastien Roy * Get an unused rxBuf from the pool.
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov *
ca5345b6a28e9e9bfd0c135121d62c6b35a5390dSebastien Roy * Returns:
ca5345b6a28e9e9bfd0c135121d62c6b35a5390dSebastien Roy *  A rxBuf or NULL if there are no buffers in the pool.
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov */
62dadd654b88164ac263978699c78aa01647a39bYuri Pankovstatic vmxnet3_rxbuf_t *
62dadd654b88164ac263978699c78aa01647a39bYuri Pankovvmxnet3_get_rxpool_buf(vmxnet3_softc_t *dp)
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov{
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov    vmxnet3_rxpool_t *rxPool = &dp->rxPool;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov    vmxnet3_rxbuf_t *rxBuf = NULL;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov    mutex_enter(&dp->rxPoolLock);
6849994e8263545ed3c0f6f5676e47b38e14f63eSebastien Roy    if (rxPool->listHead != NULL) {
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov        rxBuf = rxPool->listHead;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov        rxPool->listHead = rxBuf->next;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov        rxPool->nBufs--;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov        ASSERT((rxPool->listHead == NULL && rxPool->nBufs == 0) ||
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov            (rxPool->listHead != NULL && rxPool->nBufs != 0));
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov    }
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov    mutex_exit(&dp->rxPoolLock);
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov    return (rxBuf);
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov}
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov/*
6849994e8263545ed3c0f6f5676e47b38e14f63eSebastien Roy * Fill a rxPool by allocating the maximum number of buffers.
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov *
ca5345b6a28e9e9bfd0c135121d62c6b35a5390dSebastien Roy * Returns:
6849994e8263545ed3c0f6f5676e47b38e14f63eSebastien Roy *  0 on success, non-zero on failure.
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov */
6849994e8263545ed3c0f6f5676e47b38e14f63eSebastien Roystatic int
6849994e8263545ed3c0f6f5676e47b38e14f63eSebastien Royvmxnet3_rxpool_init(vmxnet3_softc_t *dp)
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov{
6849994e8263545ed3c0f6f5676e47b38e14f63eSebastien Roy    int err = 0;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov    vmxnet3_rxbuf_t *rxBuf;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov
6849994e8263545ed3c0f6f5676e47b38e14f63eSebastien Roy    ASSERT(dp->rxPool.nBufsLimit > 0);
6849994e8263545ed3c0f6f5676e47b38e14f63eSebastien Roy    while (dp->rxPool.nBufs < dp->rxPool.nBufsLimit) {
6849994e8263545ed3c0f6f5676e47b38e14f63eSebastien Roy        if ((rxBuf = vmxnet3_alloc_rxbuf(dp, B_FALSE)) == NULL) {
6849994e8263545ed3c0f6f5676e47b38e14f63eSebastien Roy            err = ENOMEM;
6849994e8263545ed3c0f6f5676e47b38e14f63eSebastien Roy            break;
6849994e8263545ed3c0f6f5676e47b38e14f63eSebastien Roy        }
6849994e8263545ed3c0f6f5676e47b38e14f63eSebastien Roy        VERIFY(vmxnet3_put_rxpool_buf(dp, rxBuf, B_TRUE));
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov    }
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov
6849994e8263545ed3c0f6f5676e47b38e14f63eSebastien Roy    if (err != 0) {
6849994e8263545ed3c0f6f5676e47b38e14f63eSebastien Roy        while ((rxBuf = vmxnet3_get_rxpool_buf(dp)) != NULL) {
6849994e8263545ed3c0f6f5676e47b38e14f63eSebastien Roy            vmxnet3_free_rxbuf(dp, rxBuf);
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov        }
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov    }
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov
6849994e8263545ed3c0f6f5676e47b38e14f63eSebastien Roy    return (err);
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov}
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov/*
6849994e8263545ed3c0f6f5676e47b38e14f63eSebastien Roy * Populate a Rx descriptor with a new rxBuf. If the pool argument is B_TRUE,
6849994e8263545ed3c0f6f5676e47b38e14f63eSebastien Roy * then try to take a buffer from rxPool. If the pool is empty and the
6849994e8263545ed3c0f6f5676e47b38e14f63eSebastien Roy * dp->alloc_ok is true, then fall back to dynamic allocation. If pool is
6849994e8263545ed3c0f6f5676e47b38e14f63eSebastien Roy * B_FALSE, then always allocate a new buffer (this is only used when
6849994e8263545ed3c0f6f5676e47b38e14f63eSebastien Roy * populating the initial set of buffers in the receive queue during start).
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov *
ca5345b6a28e9e9bfd0c135121d62c6b35a5390dSebastien Roy * Returns:
6849994e8263545ed3c0f6f5676e47b38e14f63eSebastien Roy *  0 on success, non-zero on failure.
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov */
62dadd654b88164ac263978699c78aa01647a39bYuri Pankovstatic int
62dadd654b88164ac263978699c78aa01647a39bYuri Pankovvmxnet3_rx_populate(vmxnet3_softc_t *dp, vmxnet3_rxqueue_t *rxq, uint16_t idx,
6849994e8263545ed3c0f6f5676e47b38e14f63eSebastien Roy    boolean_t canSleep, boolean_t pool)
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov{
6849994e8263545ed3c0f6f5676e47b38e14f63eSebastien Roy    vmxnet3_rxbuf_t *rxBuf = NULL;
6849994e8263545ed3c0f6f5676e47b38e14f63eSebastien Roy
6849994e8263545ed3c0f6f5676e47b38e14f63eSebastien Roy    if (pool && (rxBuf = vmxnet3_get_rxpool_buf(dp)) == NULL) {
6849994e8263545ed3c0f6f5676e47b38e14f63eSebastien Roy        /* The maximum number of pool buffers have been allocated. */
6849994e8263545ed3c0f6f5676e47b38e14f63eSebastien Roy        atomic_inc_32(&dp->rx_pool_empty);
6849994e8263545ed3c0f6f5676e47b38e14f63eSebastien Roy        if (!dp->alloc_ok) {
6849994e8263545ed3c0f6f5676e47b38e14f63eSebastien Roy            atomic_inc_32(&dp->rx_alloc_failed);
6849994e8263545ed3c0f6f5676e47b38e14f63eSebastien Roy        }
6849994e8263545ed3c0f6f5676e47b38e14f63eSebastien Roy    }
6849994e8263545ed3c0f6f5676e47b38e14f63eSebastien Roy
6849994e8263545ed3c0f6f5676e47b38e14f63eSebastien Roy    if (rxBuf == NULL && (!pool || dp->alloc_ok)) {
6849994e8263545ed3c0f6f5676e47b38e14f63eSebastien Roy        rxBuf = vmxnet3_alloc_rxbuf(dp, canSleep);
6849994e8263545ed3c0f6f5676e47b38e14f63eSebastien Roy    }
6849994e8263545ed3c0f6f5676e47b38e14f63eSebastien Roy
6849994e8263545ed3c0f6f5676e47b38e14f63eSebastien Roy    if (rxBuf != NULL) {
6849994e8263545ed3c0f6f5676e47b38e14f63eSebastien Roy        rxBuf->mblk = desballoc((uchar_t *)rxBuf->dma.buf,
6849994e8263545ed3c0f6f5676e47b38e14f63eSebastien Roy            rxBuf->dma.bufLen, BPRI_MED, &rxBuf->freeCB);
6849994e8263545ed3c0f6f5676e47b38e14f63eSebastien Roy        if (rxBuf->mblk == NULL) {
6849994e8263545ed3c0f6f5676e47b38e14f63eSebastien Roy            if (pool) {
6849994e8263545ed3c0f6f5676e47b38e14f63eSebastien Roy                VERIFY(vmxnet3_put_rxpool_buf(dp, rxBuf,
6849994e8263545ed3c0f6f5676e47b38e14f63eSebastien Roy                    B_FALSE));
6849994e8263545ed3c0f6f5676e47b38e14f63eSebastien Roy            } else {
6849994e8263545ed3c0f6f5676e47b38e14f63eSebastien Roy                vmxnet3_free_rxbuf(dp, rxBuf);
6849994e8263545ed3c0f6f5676e47b38e14f63eSebastien Roy            }
6849994e8263545ed3c0f6f5676e47b38e14f63eSebastien Roy            atomic_inc_32(&dp->rx_alloc_failed);
6849994e8263545ed3c0f6f5676e47b38e14f63eSebastien Roy            return (ENOMEM);
6849994e8263545ed3c0f6f5676e47b38e14f63eSebastien Roy        }
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov        vmxnet3_cmdring_t *cmdRing = &rxq->cmdRing;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov        Vmxnet3_GenericDesc *rxDesc = VMXNET3_GET_DESC(cmdRing, idx);
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov        rxq->bufRing[idx].rxBuf = rxBuf;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov        rxDesc->rxd.addr = rxBuf->dma.bufPA;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov        rxDesc->rxd.len = rxBuf->dma.bufLen;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov        /* rxDesc->rxd.btype = 0; */
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov        membar_producer();
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov        rxDesc->rxd.gen = cmdRing->gen;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov    } else {
6849994e8263545ed3c0f6f5676e47b38e14f63eSebastien Roy        return (ENOMEM);
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov    }
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov
6849994e8263545ed3c0f6f5676e47b38e14f63eSebastien Roy    return (0);
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov}
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov/*
ca5345b6a28e9e9bfd0c135121d62c6b35a5390dSebastien Roy * Initialize a RxQueue by populating the whole Rx ring with rxBufs.
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov *
ca5345b6a28e9e9bfd0c135121d62c6b35a5390dSebastien Roy * Returns:
6849994e8263545ed3c0f6f5676e47b38e14f63eSebastien Roy *  0 on success, non-zero on failure.
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov */
62dadd654b88164ac263978699c78aa01647a39bYuri Pankovint
62dadd654b88164ac263978699c78aa01647a39bYuri Pankovvmxnet3_rxqueue_init(vmxnet3_softc_t *dp, vmxnet3_rxqueue_t *rxq)
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov{
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov    vmxnet3_cmdring_t *cmdRing = &rxq->cmdRing;
6849994e8263545ed3c0f6f5676e47b38e14f63eSebastien Roy    int err;
6849994e8263545ed3c0f6f5676e47b38e14f63eSebastien Roy
6849994e8263545ed3c0f6f5676e47b38e14f63eSebastien Roy    dp->rxPool.nBufsLimit = vmxnet3_getprop(dp, "RxBufPoolLimit", 0,
6849994e8263545ed3c0f6f5676e47b38e14f63eSebastien Roy        cmdRing->size * 10, cmdRing->size * 2);
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov    do {
6849994e8263545ed3c0f6f5676e47b38e14f63eSebastien Roy        if ((err = vmxnet3_rx_populate(dp, rxq, cmdRing->next2fill,
6849994e8263545ed3c0f6f5676e47b38e14f63eSebastien Roy            B_TRUE, B_FALSE)) != 0) {
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov            goto error;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov        }
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov        VMXNET3_INC_RING_IDX(cmdRing, cmdRing->next2fill);
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov    } while (cmdRing->next2fill);
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov
6849994e8263545ed3c0f6f5676e47b38e14f63eSebastien Roy    /*
6849994e8263545ed3c0f6f5676e47b38e14f63eSebastien Roy     * Pre-allocate rxPool buffers so that we never have to allocate
6849994e8263545ed3c0f6f5676e47b38e14f63eSebastien Roy     * new buffers from interrupt context when we need to replace a buffer
6849994e8263545ed3c0f6f5676e47b38e14f63eSebastien Roy     * in the rxqueue.
6849994e8263545ed3c0f6f5676e47b38e14f63eSebastien Roy     */
6849994e8263545ed3c0f6f5676e47b38e14f63eSebastien Roy    if ((err = vmxnet3_rxpool_init(dp)) != 0) {
6849994e8263545ed3c0f6f5676e47b38e14f63eSebastien Roy        goto error;
6849994e8263545ed3c0f6f5676e47b38e14f63eSebastien Roy    }
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov
6849994e8263545ed3c0f6f5676e47b38e14f63eSebastien Roy    return (0);
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov
62dadd654b88164ac263978699c78aa01647a39bYuri Pankoverror:
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov    while (cmdRing->next2fill) {
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov        VMXNET3_DEC_RING_IDX(cmdRing, cmdRing->next2fill);
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov        vmxnet3_free_rxbuf(dp, rxq->bufRing[cmdRing->next2fill].rxBuf);
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov    }
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov
6849994e8263545ed3c0f6f5676e47b38e14f63eSebastien Roy    return (err);
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov}
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov/*
ca5345b6a28e9e9bfd0c135121d62c6b35a5390dSebastien Roy * Finish a RxQueue by freeing all the related rxBufs.
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov */
62dadd654b88164ac263978699c78aa01647a39bYuri Pankovvoid
62dadd654b88164ac263978699c78aa01647a39bYuri Pankovvmxnet3_rxqueue_fini(vmxnet3_softc_t *dp, vmxnet3_rxqueue_t *rxq)
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov{
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov    vmxnet3_rxbuf_t *rxBuf;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov    unsigned int i;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov    ASSERT(!dp->devEnabled);
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov    /* First the rxPool */
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov    while ((rxBuf = vmxnet3_get_rxpool_buf(dp)))
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov        vmxnet3_free_rxbuf(dp, rxBuf);
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov    /* Then the ring */
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov    for (i = 0; i < rxq->cmdRing.size; i++) {
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov        rxBuf = rxq->bufRing[i].rxBuf;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov        ASSERT(rxBuf);
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov        ASSERT(rxBuf->mblk);
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov        /*
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov         * Here, freemsg() will trigger a call to vmxnet3_put_rxbuf()
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov         * which will then call vmxnet3_free_rxbuf() because the
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov         * underlying device is disabled.
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov         */
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov        freemsg(rxBuf->mblk);
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov    }
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov}
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov/*
ca5345b6a28e9e9bfd0c135121d62c6b35a5390dSebastien Roy * Determine if a received packet was checksummed by the Vmxnet3
ca5345b6a28e9e9bfd0c135121d62c6b35a5390dSebastien Roy * device and tag the mp appropriately.
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov */
62dadd654b88164ac263978699c78aa01647a39bYuri Pankovstatic void
62dadd654b88164ac263978699c78aa01647a39bYuri Pankovvmxnet3_rx_hwcksum(vmxnet3_softc_t *dp, mblk_t *mp,
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov    Vmxnet3_GenericDesc *compDesc)
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov{
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov    uint32_t flags = 0;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov    if (!compDesc->rcd.cnc) {
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov        if (compDesc->rcd.v4 && compDesc->rcd.ipc) {
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov            flags |= HCK_IPV4_HDRCKSUM;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov            if ((compDesc->rcd.tcp || compDesc->rcd.udp) &&
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov                compDesc->rcd.tuc) {
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov                flags |= HCK_FULLCKSUM | HCK_FULLCKSUM_OK;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov            }
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov        }
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov        VMXNET3_DEBUG(dp, 3, "rx cksum flags = 0x%x\n", flags);
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov        (void) hcksum_assoc(mp, NULL, NULL, 0, 0, 0, 0, flags, 0);
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov    }
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov}
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov/*
ca5345b6a28e9e9bfd0c135121d62c6b35a5390dSebastien Roy * Interrupt handler for Rx. Look if there are any pending Rx and
ca5345b6a28e9e9bfd0c135121d62c6b35a5390dSebastien Roy * put them in mplist.
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov *
ca5345b6a28e9e9bfd0c135121d62c6b35a5390dSebastien Roy * Returns:
ca5345b6a28e9e9bfd0c135121d62c6b35a5390dSebastien Roy *  A list of messages to pass to the MAC subystem.
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov */
62dadd654b88164ac263978699c78aa01647a39bYuri Pankovmblk_t *
62dadd654b88164ac263978699c78aa01647a39bYuri Pankovvmxnet3_rx_intr(vmxnet3_softc_t *dp, vmxnet3_rxqueue_t *rxq)
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov{
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov    vmxnet3_compring_t *compRing = &rxq->compRing;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov    vmxnet3_cmdring_t *cmdRing = &rxq->cmdRing;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov    Vmxnet3_RxQueueCtrl *rxqCtrl = rxq->sharedCtrl;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov    Vmxnet3_GenericDesc *compDesc;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov    mblk_t *mplist = NULL, **mplistTail = &mplist;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov    ASSERT(mutex_owned(&dp->intrLock));
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov    compDesc = VMXNET3_GET_DESC(compRing, compRing->next2comp);
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov    while (compDesc->rcd.gen == compRing->gen) {
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov        mblk_t *mp = NULL, **mpTail = &mp;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov        boolean_t mpValid = B_TRUE;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov        boolean_t eop;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov        ASSERT(compDesc->rcd.sop);
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov        do {
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov            uint16_t rxdIdx = compDesc->rcd.rxdIdx;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov            vmxnet3_rxbuf_t *rxBuf = rxq->bufRing[rxdIdx].rxBuf;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov            mblk_t *mblk = rxBuf->mblk;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov            Vmxnet3_GenericDesc *rxDesc;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov            while (compDesc->rcd.gen != compRing->gen) {
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov                /*
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov                 * H/W may be still be in the middle of
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov                 * generating this entry, so hold on until
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov                 * the gen bit is flipped.
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov                 */
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov                membar_consumer();
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov            }
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov            ASSERT(compDesc->rcd.gen == compRing->gen);
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov            ASSERT(rxBuf);
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov            ASSERT(mblk);
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov            /* Some Rx descriptors may have been skipped */
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov            while (cmdRing->next2fill != rxdIdx) {
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov                rxDesc = VMXNET3_GET_DESC(cmdRing,
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov                    cmdRing->next2fill);
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov                rxDesc->rxd.gen = cmdRing->gen;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov                VMXNET3_INC_RING_IDX(cmdRing,
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov                    cmdRing->next2fill);
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov            }
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov            eop = compDesc->rcd.eop;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov            /*
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov             * Now we have a piece of the packet in the rxdIdx
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov             * descriptor. Grab it only if we achieve to replace
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov             * it with a fresh buffer.
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov             */
6849994e8263545ed3c0f6f5676e47b38e14f63eSebastien Roy            if (vmxnet3_rx_populate(dp, rxq, rxdIdx, B_FALSE,
6849994e8263545ed3c0f6f5676e47b38e14f63eSebastien Roy                B_TRUE) == 0) {
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov                /* Success, we can chain the mblk with the mp */
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov                mblk->b_wptr = mblk->b_rptr + compDesc->rcd.len;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov                *mpTail = mblk;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov                mpTail = &mblk->b_cont;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov                ASSERT(*mpTail == NULL);
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov                VMXNET3_DEBUG(dp, 3, "rx 0x%p on [%u]\n", mblk,
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov                    rxdIdx);
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov                if (eop) {
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov                    if (!compDesc->rcd.err) {
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov                        /*
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov                         * Tag the mp if it was
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov                         * checksummed by the H/W
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov                         */
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov                        vmxnet3_rx_hwcksum(dp, mp,
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov                            compDesc);
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov                    } else {
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov                        mpValid = B_FALSE;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov                    }
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov                }
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov            } else {
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov                /*
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov                 * Keep the same buffer, we still need
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov                 * to flip the gen bit
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov                 */
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov                rxDesc = VMXNET3_GET_DESC(cmdRing, rxdIdx);
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov                rxDesc->rxd.gen = cmdRing->gen;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov                mpValid = B_FALSE;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov            }
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov            VMXNET3_INC_RING_IDX(compRing, compRing->next2comp);
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov            VMXNET3_INC_RING_IDX(cmdRing, cmdRing->next2fill);
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov            compDesc = VMXNET3_GET_DESC(compRing,
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov                compRing->next2comp);
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov        } while (!eop);
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov        if (mp) {
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov            if (mpValid) {
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov                *mplistTail = mp;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov                mplistTail = &mp->b_next;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov                ASSERT(*mplistTail == NULL);
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov            } else {
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov                /* This message got holes, drop it */
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov                freemsg(mp);
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov            }
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov        }
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov    }
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov    if (rxqCtrl->updateRxProd) {
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov        uint32_t rxprod;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov        /*
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov         * All buffers are actually available, but we can't tell that to
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov         * the device because it may interpret that as an empty ring.
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov         * So skip one buffer.
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov         */
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov        if (cmdRing->next2fill) {
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov            rxprod = cmdRing->next2fill - 1;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov        } else {
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov            rxprod = cmdRing->size - 1;
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov        }
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov        VMXNET3_BAR0_PUT32(dp, VMXNET3_REG_RXPROD, rxprod);
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov    }
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov    return (mplist);
62dadd654b88164ac263978699c78aa01647a39bYuri Pankov}