a23fd118e437af0a7877dd313db8fdaa3537c675yl/*
a23fd118e437af0a7877dd313db8fdaa3537c675yl * CDDL HEADER START
a23fd118e437af0a7877dd313db8fdaa3537c675yl *
a23fd118e437af0a7877dd313db8fdaa3537c675yl * The contents of this file are subject to the terms of the
a23fd118e437af0a7877dd313db8fdaa3537c675yl * Common Development and Distribution License (the "License").
a23fd118e437af0a7877dd313db8fdaa3537c675yl * You may not use this file except in compliance with the License.
a23fd118e437af0a7877dd313db8fdaa3537c675yl *
a23fd118e437af0a7877dd313db8fdaa3537c675yl * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
a23fd118e437af0a7877dd313db8fdaa3537c675yl * or http://www.opensolaris.org/os/licensing.
a23fd118e437af0a7877dd313db8fdaa3537c675yl * See the License for the specific language governing permissions
a23fd118e437af0a7877dd313db8fdaa3537c675yl * and limitations under the License.
a23fd118e437af0a7877dd313db8fdaa3537c675yl *
a23fd118e437af0a7877dd313db8fdaa3537c675yl * When distributing Covered Code, include this CDDL HEADER in each
a23fd118e437af0a7877dd313db8fdaa3537c675yl * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
a23fd118e437af0a7877dd313db8fdaa3537c675yl * If applicable, add the following below this CDDL HEADER, with the
a23fd118e437af0a7877dd313db8fdaa3537c675yl * fields enclosed by brackets "[]" replaced with your own identifying
a23fd118e437af0a7877dd313db8fdaa3537c675yl * information: Portions Copyright [yyyy] [name of copyright owner]
a23fd118e437af0a7877dd313db8fdaa3537c675yl *
a23fd118e437af0a7877dd313db8fdaa3537c675yl * CDDL HEADER END
a23fd118e437af0a7877dd313db8fdaa3537c675yl *
8347601bcb0a439f6e50fc36b4039a73d08700e1yl * Copyright (c) 2002-2006 Neterion, Inc.
a23fd118e437af0a7877dd313db8fdaa3537c675yl */
a23fd118e437af0a7877dd313db8fdaa3537c675yl
a23fd118e437af0a7877dd313db8fdaa3537c675yl#include "xge-os-pal.h"
a23fd118e437af0a7877dd313db8fdaa3537c675yl#include "xgehal-mm.h"
a23fd118e437af0a7877dd313db8fdaa3537c675yl#include "xge-debug.h"
a23fd118e437af0a7877dd313db8fdaa3537c675yl
a23fd118e437af0a7877dd313db8fdaa3537c675yl/*
a23fd118e437af0a7877dd313db8fdaa3537c675yl * __hal_mempool_grow
a23fd118e437af0a7877dd313db8fdaa3537c675yl *
a23fd118e437af0a7877dd313db8fdaa3537c675yl * Will resize mempool up to %num_allocate value.
a23fd118e437af0a7877dd313db8fdaa3537c675yl */
a23fd118e437af0a7877dd313db8fdaa3537c675ylxge_hal_status_e
a23fd118e437af0a7877dd313db8fdaa3537c675yl__hal_mempool_grow(xge_hal_mempool_t *mempool, int num_allocate,
a23fd118e437af0a7877dd313db8fdaa3537c675yl int *num_allocated)
a23fd118e437af0a7877dd313db8fdaa3537c675yl{
a23fd118e437af0a7877dd313db8fdaa3537c675yl int i, first_time = mempool->memblocks_allocated == 0 ? 1 : 0;
a23fd118e437af0a7877dd313db8fdaa3537c675yl int n_items = mempool->items_per_memblock;
a23fd118e437af0a7877dd313db8fdaa3537c675yl
a23fd118e437af0a7877dd313db8fdaa3537c675yl *num_allocated = 0;
a23fd118e437af0a7877dd313db8fdaa3537c675yl
a23fd118e437af0a7877dd313db8fdaa3537c675yl if ((mempool->memblocks_allocated + num_allocate) >
a23fd118e437af0a7877dd313db8fdaa3537c675yl mempool->memblocks_max) {
a23fd118e437af0a7877dd313db8fdaa3537c675yl xge_debug_mm(XGE_ERR, "%s",
a23fd118e437af0a7877dd313db8fdaa3537c675yl "__hal_mempool_grow: can grow anymore");
a23fd118e437af0a7877dd313db8fdaa3537c675yl return XGE_HAL_ERR_OUT_OF_MEMORY;
a23fd118e437af0a7877dd313db8fdaa3537c675yl }
a23fd118e437af0a7877dd313db8fdaa3537c675yl
a23fd118e437af0a7877dd313db8fdaa3537c675yl for (i = mempool->memblocks_allocated;
a23fd118e437af0a7877dd313db8fdaa3537c675yl i < mempool->memblocks_allocated + num_allocate; i++) {
a23fd118e437af0a7877dd313db8fdaa3537c675yl int j;
a23fd118e437af0a7877dd313db8fdaa3537c675yl int is_last =
a23fd118e437af0a7877dd313db8fdaa3537c675yl ((mempool->memblocks_allocated+num_allocate-1) == i);
a23fd118e437af0a7877dd313db8fdaa3537c675yl xge_hal_mempool_dma_t *dma_object =
a23fd118e437af0a7877dd313db8fdaa3537c675yl mempool->memblocks_dma_arr + i;
a23fd118e437af0a7877dd313db8fdaa3537c675yl void *the_memblock;
a23fd118e437af0a7877dd313db8fdaa3537c675yl int dma_flags;
a23fd118e437af0a7877dd313db8fdaa3537c675yl
a23fd118e437af0a7877dd313db8fdaa3537c675yl dma_flags = XGE_OS_DMA_CACHELINE_ALIGNED;
a23fd118e437af0a7877dd313db8fdaa3537c675yl#ifdef XGE_HAL_DMA_DTR_CONSISTENT
a23fd118e437af0a7877dd313db8fdaa3537c675yl dma_flags |= XGE_OS_DMA_CONSISTENT;
a23fd118e437af0a7877dd313db8fdaa3537c675yl#else
a23fd118e437af0a7877dd313db8fdaa3537c675yl dma_flags |= XGE_OS_DMA_STREAMING;
a23fd118e437af0a7877dd313db8fdaa3537c675yl#endif
a23fd118e437af0a7877dd313db8fdaa3537c675yl
a23fd118e437af0a7877dd313db8fdaa3537c675yl /* allocate DMA-capable memblock */
a23fd118e437af0a7877dd313db8fdaa3537c675yl mempool->memblocks_arr[i] = xge_os_dma_malloc(mempool->pdev,
a23fd118e437af0a7877dd313db8fdaa3537c675yl mempool->memblock_size,
a23fd118e437af0a7877dd313db8fdaa3537c675yl dma_flags,
a23fd118e437af0a7877dd313db8fdaa3537c675yl &dma_object->handle,
a23fd118e437af0a7877dd313db8fdaa3537c675yl &dma_object->acc_handle);
a23fd118e437af0a7877dd313db8fdaa3537c675yl if (mempool->memblocks_arr[i] == NULL) {
a23fd118e437af0a7877dd313db8fdaa3537c675yl xge_debug_mm(XGE_ERR,
a23fd118e437af0a7877dd313db8fdaa3537c675yl "memblock[%d]: out of DMA memory", i);
a23fd118e437af0a7877dd313db8fdaa3537c675yl return XGE_HAL_ERR_OUT_OF_MEMORY;
a23fd118e437af0a7877dd313db8fdaa3537c675yl }
a23fd118e437af0a7877dd313db8fdaa3537c675yl xge_os_memzero(mempool->memblocks_arr[i],
a23fd118e437af0a7877dd313db8fdaa3537c675yl mempool->memblock_size);
a23fd118e437af0a7877dd313db8fdaa3537c675yl the_memblock = mempool->memblocks_arr[i];
a23fd118e437af0a7877dd313db8fdaa3537c675yl
a23fd118e437af0a7877dd313db8fdaa3537c675yl /* allocate memblock's private part. Each DMA memblock
a23fd118e437af0a7877dd313db8fdaa3537c675yl * has a space allocated for item's private usage upon
a23fd118e437af0a7877dd313db8fdaa3537c675yl * mempool's user request. Each time mempool grows, it will
a23fd118e437af0a7877dd313db8fdaa3537c675yl * allocate new memblock and its private part at once.
a23fd118e437af0a7877dd313db8fdaa3537c675yl * This helps to minimize memory usage a lot. */
a23fd118e437af0a7877dd313db8fdaa3537c675yl mempool->memblocks_priv_arr[i] = xge_os_malloc(mempool->pdev,
a23fd118e437af0a7877dd313db8fdaa3537c675yl mempool->items_priv_size * n_items);
a23fd118e437af0a7877dd313db8fdaa3537c675yl if (mempool->memblocks_priv_arr[i] == NULL) {
a23fd118e437af0a7877dd313db8fdaa3537c675yl xge_os_dma_free(mempool->pdev,
a23fd118e437af0a7877dd313db8fdaa3537c675yl the_memblock,
a23fd118e437af0a7877dd313db8fdaa3537c675yl mempool->memblock_size,
a23fd118e437af0a7877dd313db8fdaa3537c675yl &dma_object->acc_handle,
a23fd118e437af0a7877dd313db8fdaa3537c675yl &dma_object->handle);
a23fd118e437af0a7877dd313db8fdaa3537c675yl xge_debug_mm(XGE_ERR,
a23fd118e437af0a7877dd313db8fdaa3537c675yl "memblock_priv[%d]: out of virtual memory, "
a23fd118e437af0a7877dd313db8fdaa3537c675yl "requested %d(%d:%d) bytes", i,
a23fd118e437af0a7877dd313db8fdaa3537c675yl mempool->items_priv_size * n_items,
a23fd118e437af0a7877dd313db8fdaa3537c675yl mempool->items_priv_size, n_items);
a23fd118e437af0a7877dd313db8fdaa3537c675yl return XGE_HAL_ERR_OUT_OF_MEMORY;
a23fd118e437af0a7877dd313db8fdaa3537c675yl }
a23fd118e437af0a7877dd313db8fdaa3537c675yl xge_os_memzero(mempool->memblocks_priv_arr[i],
a23fd118e437af0a7877dd313db8fdaa3537c675yl mempool->items_priv_size * n_items);
a23fd118e437af0a7877dd313db8fdaa3537c675yl
a23fd118e437af0a7877dd313db8fdaa3537c675yl /* map memblock to physical memory */
a23fd118e437af0a7877dd313db8fdaa3537c675yl dma_object->addr = xge_os_dma_map(mempool->pdev,
a23fd118e437af0a7877dd313db8fdaa3537c675yl dma_object->handle,
a23fd118e437af0a7877dd313db8fdaa3537c675yl the_memblock,
a23fd118e437af0a7877dd313db8fdaa3537c675yl mempool->memblock_size,
a23fd118e437af0a7877dd313db8fdaa3537c675yl XGE_OS_DMA_DIR_BIDIRECTIONAL,
a23fd118e437af0a7877dd313db8fdaa3537c675yl#ifdef XGE_HAL_DMA_DTR_CONSISTENT
a23fd118e437af0a7877dd313db8fdaa3537c675yl XGE_OS_DMA_CONSISTENT
a23fd118e437af0a7877dd313db8fdaa3537c675yl#else
a23fd118e437af0a7877dd313db8fdaa3537c675yl XGE_OS_DMA_STREAMING
a23fd118e437af0a7877dd313db8fdaa3537c675yl#endif
a23fd118e437af0a7877dd313db8fdaa3537c675yl );
a23fd118e437af0a7877dd313db8fdaa3537c675yl if (dma_object->addr == XGE_OS_INVALID_DMA_ADDR) {
a23fd118e437af0a7877dd313db8fdaa3537c675yl xge_os_free(mempool->pdev, mempool->memblocks_priv_arr[i],
a23fd118e437af0a7877dd313db8fdaa3537c675yl mempool->items_priv_size *
a23fd118e437af0a7877dd313db8fdaa3537c675yl n_items);
a23fd118e437af0a7877dd313db8fdaa3537c675yl xge_os_dma_free(mempool->pdev,
a23fd118e437af0a7877dd313db8fdaa3537c675yl the_memblock,
a23fd118e437af0a7877dd313db8fdaa3537c675yl mempool->memblock_size,
a23fd118e437af0a7877dd313db8fdaa3537c675yl &dma_object->acc_handle,
a23fd118e437af0a7877dd313db8fdaa3537c675yl &dma_object->handle);
a23fd118e437af0a7877dd313db8fdaa3537c675yl return XGE_HAL_ERR_OUT_OF_MAPPING;
a23fd118e437af0a7877dd313db8fdaa3537c675yl }
a23fd118e437af0a7877dd313db8fdaa3537c675yl
a23fd118e437af0a7877dd313db8fdaa3537c675yl /* fill the items hash array */
a23fd118e437af0a7877dd313db8fdaa3537c675yl for (j=0; j<n_items; j++) {
a23fd118e437af0a7877dd313db8fdaa3537c675yl int index = i*n_items + j;
a23fd118e437af0a7877dd313db8fdaa3537c675yl
a23fd118e437af0a7877dd313db8fdaa3537c675yl if (first_time && index >= mempool->items_initial) {
a23fd118e437af0a7877dd313db8fdaa3537c675yl break;
a23fd118e437af0a7877dd313db8fdaa3537c675yl }
a23fd118e437af0a7877dd313db8fdaa3537c675yl
a23fd118e437af0a7877dd313db8fdaa3537c675yl mempool->items_arr[index] =
a23fd118e437af0a7877dd313db8fdaa3537c675yl ((char *)the_memblock + j*mempool->item_size);
a23fd118e437af0a7877dd313db8fdaa3537c675yl
a23fd118e437af0a7877dd313db8fdaa3537c675yl /* let caller to do more job on each item */
a23fd118e437af0a7877dd313db8fdaa3537c675yl if (mempool->item_func_alloc != NULL) {
a23fd118e437af0a7877dd313db8fdaa3537c675yl xge_hal_status_e status;
a23fd118e437af0a7877dd313db8fdaa3537c675yl
a23fd118e437af0a7877dd313db8fdaa3537c675yl if ((status = mempool->item_func_alloc(
a23fd118e437af0a7877dd313db8fdaa3537c675yl mempool,
a23fd118e437af0a7877dd313db8fdaa3537c675yl the_memblock,
a23fd118e437af0a7877dd313db8fdaa3537c675yl i,
a23fd118e437af0a7877dd313db8fdaa3537c675yl dma_object,
a23fd118e437af0a7877dd313db8fdaa3537c675yl mempool->items_arr[index],
a23fd118e437af0a7877dd313db8fdaa3537c675yl index,
a23fd118e437af0a7877dd313db8fdaa3537c675yl is_last,
a23fd118e437af0a7877dd313db8fdaa3537c675yl mempool->userdata)) != XGE_HAL_OK) {
a23fd118e437af0a7877dd313db8fdaa3537c675yl
a23fd118e437af0a7877dd313db8fdaa3537c675yl if (mempool->item_func_free != NULL) {
a23fd118e437af0a7877dd313db8fdaa3537c675yl int k;
a23fd118e437af0a7877dd313db8fdaa3537c675yl
a23fd118e437af0a7877dd313db8fdaa3537c675yl for (k=0; k<j; k++) {
a23fd118e437af0a7877dd313db8fdaa3537c675yl
a23fd118e437af0a7877dd313db8fdaa3537c675yl index =i*n_items + k;
a23fd118e437af0a7877dd313db8fdaa3537c675yl
a23fd118e437af0a7877dd313db8fdaa3537c675yl (void)mempool->item_func_free(
a23fd118e437af0a7877dd313db8fdaa3537c675yl mempool, the_memblock,
a23fd118e437af0a7877dd313db8fdaa3537c675yl i, dma_object,
a23fd118e437af0a7877dd313db8fdaa3537c675yl mempool->items_arr[index],
a23fd118e437af0a7877dd313db8fdaa3537c675yl index, is_last,
a23fd118e437af0a7877dd313db8fdaa3537c675yl mempool->userdata);
a23fd118e437af0a7877dd313db8fdaa3537c675yl }
a23fd118e437af0a7877dd313db8fdaa3537c675yl }
a23fd118e437af0a7877dd313db8fdaa3537c675yl
a23fd118e437af0a7877dd313db8fdaa3537c675yl xge_os_free(mempool->pdev,
a23fd118e437af0a7877dd313db8fdaa3537c675yl mempool->memblocks_priv_arr[i],
a23fd118e437af0a7877dd313db8fdaa3537c675yl mempool->items_priv_size *
a23fd118e437af0a7877dd313db8fdaa3537c675yl n_items);
a23fd118e437af0a7877dd313db8fdaa3537c675yl xge_os_dma_unmap(mempool->pdev,
a23fd118e437af0a7877dd313db8fdaa3537c675yl dma_object->handle,
a23fd118e437af0a7877dd313db8fdaa3537c675yl dma_object->addr,
a23fd118e437af0a7877dd313db8fdaa3537c675yl mempool->memblock_size,
a23fd118e437af0a7877dd313db8fdaa3537c675yl XGE_OS_DMA_DIR_BIDIRECTIONAL);
a23fd118e437af0a7877dd313db8fdaa3537c675yl xge_os_dma_free(mempool->pdev,
a23fd118e437af0a7877dd313db8fdaa3537c675yl the_memblock,
a23fd118e437af0a7877dd313db8fdaa3537c675yl mempool->memblock_size,
a23fd118e437af0a7877dd313db8fdaa3537c675yl &dma_object->acc_handle,
a23fd118e437af0a7877dd313db8fdaa3537c675yl &dma_object->handle);
a23fd118e437af0a7877dd313db8fdaa3537c675yl return status;
a23fd118e437af0a7877dd313db8fdaa3537c675yl }
a23fd118e437af0a7877dd313db8fdaa3537c675yl }
a23fd118e437af0a7877dd313db8fdaa3537c675yl
a23fd118e437af0a7877dd313db8fdaa3537c675yl mempool->items_current = index + 1;
a23fd118e437af0a7877dd313db8fdaa3537c675yl }
a23fd118e437af0a7877dd313db8fdaa3537c675yl
a23fd118e437af0a7877dd313db8fdaa3537c675yl xge_debug_mm(XGE_TRACE,
8347601bcb0a439f6e50fc36b4039a73d08700e1yl "memblock%d: allocated %dk, vaddr 0x"XGE_OS_LLXFMT", "
8347601bcb0a439f6e50fc36b4039a73d08700e1yl "dma_addr 0x"XGE_OS_LLXFMT, i, mempool->memblock_size / 1024,
a23fd118e437af0a7877dd313db8fdaa3537c675yl (unsigned long long)(ulong_t)mempool->memblocks_arr[i],
a23fd118e437af0a7877dd313db8fdaa3537c675yl (unsigned long long)dma_object->addr);
a23fd118e437af0a7877dd313db8fdaa3537c675yl
a23fd118e437af0a7877dd313db8fdaa3537c675yl (*num_allocated)++;
a23fd118e437af0a7877dd313db8fdaa3537c675yl
a23fd118e437af0a7877dd313db8fdaa3537c675yl if (first_time && mempool->items_current ==
a23fd118e437af0a7877dd313db8fdaa3537c675yl mempool->items_initial) {
a23fd118e437af0a7877dd313db8fdaa3537c675yl break;
a23fd118e437af0a7877dd313db8fdaa3537c675yl }
a23fd118e437af0a7877dd313db8fdaa3537c675yl }
a23fd118e437af0a7877dd313db8fdaa3537c675yl
a23fd118e437af0a7877dd313db8fdaa3537c675yl /* increment actual number of allocated memblocks */
a23fd118e437af0a7877dd313db8fdaa3537c675yl mempool->memblocks_allocated += *num_allocated;
a23fd118e437af0a7877dd313db8fdaa3537c675yl
a23fd118e437af0a7877dd313db8fdaa3537c675yl return XGE_HAL_OK;
a23fd118e437af0a7877dd313db8fdaa3537c675yl}
a23fd118e437af0a7877dd313db8fdaa3537c675yl
a23fd118e437af0a7877dd313db8fdaa3537c675yl/*
a23fd118e437af0a7877dd313db8fdaa3537c675yl * xge_hal_mempool_create
a23fd118e437af0a7877dd313db8fdaa3537c675yl * @memblock_size:
a23fd118e437af0a7877dd313db8fdaa3537c675yl * @items_initial:
a23fd118e437af0a7877dd313db8fdaa3537c675yl * @items_max:
a23fd118e437af0a7877dd313db8fdaa3537c675yl * @item_size:
a23fd118e437af0a7877dd313db8fdaa3537c675yl * @item_func:
a23fd118e437af0a7877dd313db8fdaa3537c675yl *
a23fd118e437af0a7877dd313db8fdaa3537c675yl * This function will create memory pool object. Pool may grow but will
a23fd118e437af0a7877dd313db8fdaa3537c675yl * never shrink. Pool consists of number of dynamically allocated blocks
a23fd118e437af0a7877dd313db8fdaa3537c675yl * with size enough to hold %items_initial number of items. Memory is
a23fd118e437af0a7877dd313db8fdaa3537c675yl * DMA-able but client must map/unmap before interoperating with the device.
a23fd118e437af0a7877dd313db8fdaa3537c675yl * See also: xge_os_dma_map(), xge_hal_dma_unmap(), xge_hal_status_e{}.
a23fd118e437af0a7877dd313db8fdaa3537c675yl */
a23fd118e437af0a7877dd313db8fdaa3537c675ylxge_hal_mempool_t*
a23fd118e437af0a7877dd313db8fdaa3537c675yl__hal_mempool_create(pci_dev_h pdev, int memblock_size, int item_size,
a23fd118e437af0a7877dd313db8fdaa3537c675yl int items_priv_size, int items_initial, int items_max,
a23fd118e437af0a7877dd313db8fdaa3537c675yl xge_hal_mempool_item_f item_func_alloc,
a23fd118e437af0a7877dd313db8fdaa3537c675yl xge_hal_mempool_item_f item_func_free, void *userdata)
a23fd118e437af0a7877dd313db8fdaa3537c675yl{
a23fd118e437af0a7877dd313db8fdaa3537c675yl xge_hal_status_e status;
a23fd118e437af0a7877dd313db8fdaa3537c675yl int memblocks_to_allocate;
a23fd118e437af0a7877dd313db8fdaa3537c675yl xge_hal_mempool_t *mempool;
a23fd118e437af0a7877dd313db8fdaa3537c675yl int allocated;
a23fd118e437af0a7877dd313db8fdaa3537c675yl
a23fd118e437af0a7877dd313db8fdaa3537c675yl if (memblock_size < item_size) {
a23fd118e437af0a7877dd313db8fdaa3537c675yl xge_debug_mm(XGE_ERR,
a23fd118e437af0a7877dd313db8fdaa3537c675yl "memblock_size %d < item_size %d: misconfiguration",
a23fd118e437af0a7877dd313db8fdaa3537c675yl memblock_size, item_size);
a23fd118e437af0a7877dd313db8fdaa3537c675yl return NULL;
a23fd118e437af0a7877dd313db8fdaa3537c675yl }
a23fd118e437af0a7877dd313db8fdaa3537c675yl
8347601bcb0a439f6e50fc36b4039a73d08700e1yl mempool = (xge_hal_mempool_t *) \
8347601bcb0a439f6e50fc36b4039a73d08700e1yl xge_os_malloc(pdev, sizeof(xge_hal_mempool_t));
a23fd118e437af0a7877dd313db8fdaa3537c675yl if (mempool == NULL) {
a23fd118e437af0a7877dd313db8fdaa3537c675yl xge_debug_mm(XGE_ERR, "mempool allocation failure");
a23fd118e437af0a7877dd313db8fdaa3537c675yl return NULL;
a23fd118e437af0a7877dd313db8fdaa3537c675yl }
a23fd118e437af0a7877dd313db8fdaa3537c675yl xge_os_memzero(mempool, sizeof(xge_hal_mempool_t));
a23fd118e437af0a7877dd313db8fdaa3537c675yl
a23fd118e437af0a7877dd313db8fdaa3537c675yl mempool->pdev = pdev;
a23fd118e437af0a7877dd313db8fdaa3537c675yl mempool->memblock_size = memblock_size;
a23fd118e437af0a7877dd313db8fdaa3537c675yl mempool->items_max = items_max;
a23fd118e437af0a7877dd313db8fdaa3537c675yl mempool->items_initial = items_initial;
a23fd118e437af0a7877dd313db8fdaa3537c675yl mempool->item_size = item_size;
a23fd118e437af0a7877dd313db8fdaa3537c675yl mempool->items_priv_size = items_priv_size;
a23fd118e437af0a7877dd313db8fdaa3537c675yl mempool->item_func_alloc = item_func_alloc;
a23fd118e437af0a7877dd313db8fdaa3537c675yl mempool->item_func_free = item_func_free;
a23fd118e437af0a7877dd313db8fdaa3537c675yl mempool->userdata = userdata;
a23fd118e437af0a7877dd313db8fdaa3537c675yl
a23fd118e437af0a7877dd313db8fdaa3537c675yl mempool->memblocks_allocated = 0;
a23fd118e437af0a7877dd313db8fdaa3537c675yl
a23fd118e437af0a7877dd313db8fdaa3537c675yl mempool->items_per_memblock = memblock_size / item_size;
a23fd118e437af0a7877dd313db8fdaa3537c675yl
a23fd118e437af0a7877dd313db8fdaa3537c675yl mempool->memblocks_max = (items_max + mempool->items_per_memblock - 1) /
a23fd118e437af0a7877dd313db8fdaa3537c675yl mempool->items_per_memblock;
a23fd118e437af0a7877dd313db8fdaa3537c675yl
a23fd118e437af0a7877dd313db8fdaa3537c675yl /* allocate array of memblocks */
8347601bcb0a439f6e50fc36b4039a73d08700e1yl mempool->memblocks_arr = (void ** ) xge_os_malloc(mempool->pdev,
a23fd118e437af0a7877dd313db8fdaa3537c675yl sizeof(void*) * mempool->memblocks_max);
a23fd118e437af0a7877dd313db8fdaa3537c675yl if (mempool->memblocks_arr == NULL) {
a23fd118e437af0a7877dd313db8fdaa3537c675yl xge_debug_mm(XGE_ERR, "memblocks_arr allocation failure");
a23fd118e437af0a7877dd313db8fdaa3537c675yl __hal_mempool_destroy(mempool);
a23fd118e437af0a7877dd313db8fdaa3537c675yl return NULL;
a23fd118e437af0a7877dd313db8fdaa3537c675yl }
a23fd118e437af0a7877dd313db8fdaa3537c675yl xge_os_memzero(mempool->memblocks_arr,
a23fd118e437af0a7877dd313db8fdaa3537c675yl sizeof(void*) * mempool->memblocks_max);
a23fd118e437af0a7877dd313db8fdaa3537c675yl
a23fd118e437af0a7877dd313db8fdaa3537c675yl /* allocate array of private parts of items per memblocks */
8347601bcb0a439f6e50fc36b4039a73d08700e1yl mempool->memblocks_priv_arr = (void **) xge_os_malloc(mempool->pdev,
a23fd118e437af0a7877dd313db8fdaa3537c675yl sizeof(void*) * mempool->memblocks_max);
a23fd118e437af0a7877dd313db8fdaa3537c675yl if (mempool->memblocks_priv_arr == NULL) {
a23fd118e437af0a7877dd313db8fdaa3537c675yl xge_debug_mm(XGE_ERR, "memblocks_priv_arr allocation failure");
a23fd118e437af0a7877dd313db8fdaa3537c675yl __hal_mempool_destroy(mempool);
a23fd118e437af0a7877dd313db8fdaa3537c675yl return NULL;
a23fd118e437af0a7877dd313db8fdaa3537c675yl }
a23fd118e437af0a7877dd313db8fdaa3537c675yl xge_os_memzero(mempool->memblocks_priv_arr,
a23fd118e437af0a7877dd313db8fdaa3537c675yl sizeof(void*) * mempool->memblocks_max);
a23fd118e437af0a7877dd313db8fdaa3537c675yl
a23fd118e437af0a7877dd313db8fdaa3537c675yl /* allocate array of memblocks DMA objects */
8347601bcb0a439f6e50fc36b4039a73d08700e1yl mempool->memblocks_dma_arr =
8347601bcb0a439f6e50fc36b4039a73d08700e1yl (xge_hal_mempool_dma_t *) xge_os_malloc(mempool->pdev,
8347601bcb0a439f6e50fc36b4039a73d08700e1yl sizeof(xge_hal_mempool_dma_t) * mempool->memblocks_max);
8347601bcb0a439f6e50fc36b4039a73d08700e1yl
a23fd118e437af0a7877dd313db8fdaa3537c675yl if (mempool->memblocks_dma_arr == NULL) {
a23fd118e437af0a7877dd313db8fdaa3537c675yl xge_debug_mm(XGE_ERR, "memblocks_dma_arr allocation failure");
a23fd118e437af0a7877dd313db8fdaa3537c675yl __hal_mempool_destroy(mempool);
a23fd118e437af0a7877dd313db8fdaa3537c675yl return NULL;
a23fd118e437af0a7877dd313db8fdaa3537c675yl }
a23fd118e437af0a7877dd313db8fdaa3537c675yl xge_os_memzero(mempool->memblocks_dma_arr,
a23fd118e437af0a7877dd313db8fdaa3537c675yl sizeof(xge_hal_mempool_dma_t) * mempool->memblocks_max);
a23fd118e437af0a7877dd313db8fdaa3537c675yl
a23fd118e437af0a7877dd313db8fdaa3537c675yl /* allocate hash array of items */
8347601bcb0a439f6e50fc36b4039a73d08700e1yl mempool->items_arr = (void **) xge_os_malloc(mempool->pdev,
a23fd118e437af0a7877dd313db8fdaa3537c675yl sizeof(void*) * mempool->items_max);
a23fd118e437af0a7877dd313db8fdaa3537c675yl if (mempool->items_arr == NULL) {
a23fd118e437af0a7877dd313db8fdaa3537c675yl xge_debug_mm(XGE_ERR, "items_arr allocation failure");
a23fd118e437af0a7877dd313db8fdaa3537c675yl __hal_mempool_destroy(mempool);
a23fd118e437af0a7877dd313db8fdaa3537c675yl return NULL;
a23fd118e437af0a7877dd313db8fdaa3537c675yl }
a23fd118e437af0a7877dd313db8fdaa3537c675yl xge_os_memzero(mempool->items_arr, sizeof(void *) * mempool->items_max);
a23fd118e437af0a7877dd313db8fdaa3537c675yl
8347601bcb0a439f6e50fc36b4039a73d08700e1yl mempool->shadow_items_arr = (void **) xge_os_malloc(mempool->pdev,
8347601bcb0a439f6e50fc36b4039a73d08700e1yl sizeof(void*) * mempool->items_max);
a23fd118e437af0a7877dd313db8fdaa3537c675yl if (mempool->shadow_items_arr == NULL) {
a23fd118e437af0a7877dd313db8fdaa3537c675yl xge_debug_mm(XGE_ERR, "shadow_items_arr allocation failure");
a23fd118e437af0a7877dd313db8fdaa3537c675yl __hal_mempool_destroy(mempool);
a23fd118e437af0a7877dd313db8fdaa3537c675yl return NULL;
a23fd118e437af0a7877dd313db8fdaa3537c675yl }
a23fd118e437af0a7877dd313db8fdaa3537c675yl xge_os_memzero(mempool->shadow_items_arr,
a23fd118e437af0a7877dd313db8fdaa3537c675yl sizeof(void *) * mempool->items_max);
a23fd118e437af0a7877dd313db8fdaa3537c675yl
a23fd118e437af0a7877dd313db8fdaa3537c675yl /* calculate initial number of memblocks */
a23fd118e437af0a7877dd313db8fdaa3537c675yl memblocks_to_allocate = (mempool->items_initial +
a23fd118e437af0a7877dd313db8fdaa3537c675yl mempool->items_per_memblock - 1) /
a23fd118e437af0a7877dd313db8fdaa3537c675yl mempool->items_per_memblock;
a23fd118e437af0a7877dd313db8fdaa3537c675yl
a23fd118e437af0a7877dd313db8fdaa3537c675yl xge_debug_mm(XGE_TRACE, "allocating %d memblocks, "
a23fd118e437af0a7877dd313db8fdaa3537c675yl "%d items per memblock", memblocks_to_allocate,
a23fd118e437af0a7877dd313db8fdaa3537c675yl mempool->items_per_memblock);
a23fd118e437af0a7877dd313db8fdaa3537c675yl
a23fd118e437af0a7877dd313db8fdaa3537c675yl /* pre-allocate the mempool */
a23fd118e437af0a7877dd313db8fdaa3537c675yl status = __hal_mempool_grow(mempool, memblocks_to_allocate, &allocated);
a23fd118e437af0a7877dd313db8fdaa3537c675yl xge_os_memcpy(mempool->shadow_items_arr, mempool->items_arr,
a23fd118e437af0a7877dd313db8fdaa3537c675yl sizeof(void*) * mempool->items_max);
a23fd118e437af0a7877dd313db8fdaa3537c675yl if (status != XGE_HAL_OK) {
a23fd118e437af0a7877dd313db8fdaa3537c675yl xge_debug_mm(XGE_ERR, "mempool_grow failure");
a23fd118e437af0a7877dd313db8fdaa3537c675yl __hal_mempool_destroy(mempool);
a23fd118e437af0a7877dd313db8fdaa3537c675yl return NULL;
a23fd118e437af0a7877dd313db8fdaa3537c675yl }
a23fd118e437af0a7877dd313db8fdaa3537c675yl
a23fd118e437af0a7877dd313db8fdaa3537c675yl xge_debug_mm(XGE_TRACE,
a23fd118e437af0a7877dd313db8fdaa3537c675yl "total: allocated %dk of DMA-capable memory",
a23fd118e437af0a7877dd313db8fdaa3537c675yl mempool->memblock_size * allocated / 1024);
a23fd118e437af0a7877dd313db8fdaa3537c675yl
a23fd118e437af0a7877dd313db8fdaa3537c675yl return mempool;
a23fd118e437af0a7877dd313db8fdaa3537c675yl}
a23fd118e437af0a7877dd313db8fdaa3537c675yl
a23fd118e437af0a7877dd313db8fdaa3537c675yl/*
a23fd118e437af0a7877dd313db8fdaa3537c675yl * xge_hal_mempool_destroy
a23fd118e437af0a7877dd313db8fdaa3537c675yl */
a23fd118e437af0a7877dd313db8fdaa3537c675ylvoid
a23fd118e437af0a7877dd313db8fdaa3537c675yl__hal_mempool_destroy(xge_hal_mempool_t *mempool)
a23fd118e437af0a7877dd313db8fdaa3537c675yl{
a23fd118e437af0a7877dd313db8fdaa3537c675yl int i, j;
a23fd118e437af0a7877dd313db8fdaa3537c675yl
a23fd118e437af0a7877dd313db8fdaa3537c675yl for (i=0; i<mempool->memblocks_allocated; i++) {
a23fd118e437af0a7877dd313db8fdaa3537c675yl xge_hal_mempool_dma_t *dma_object;
a23fd118e437af0a7877dd313db8fdaa3537c675yl
a23fd118e437af0a7877dd313db8fdaa3537c675yl xge_assert(mempool->memblocks_arr[i]);
a23fd118e437af0a7877dd313db8fdaa3537c675yl xge_assert(mempool->memblocks_dma_arr + i);
a23fd118e437af0a7877dd313db8fdaa3537c675yl
a23fd118e437af0a7877dd313db8fdaa3537c675yl dma_object = mempool->memblocks_dma_arr + i;
a23fd118e437af0a7877dd313db8fdaa3537c675yl
a23fd118e437af0a7877dd313db8fdaa3537c675yl for (j=0; j<mempool->items_per_memblock; j++) {
a23fd118e437af0a7877dd313db8fdaa3537c675yl int index = i*mempool->items_per_memblock + j;
a23fd118e437af0a7877dd313db8fdaa3537c675yl
a23fd118e437af0a7877dd313db8fdaa3537c675yl /* to skip last partially filled(if any) memblock */
a23fd118e437af0a7877dd313db8fdaa3537c675yl if (index >= mempool->items_current) {
a23fd118e437af0a7877dd313db8fdaa3537c675yl break;
a23fd118e437af0a7877dd313db8fdaa3537c675yl }
a23fd118e437af0a7877dd313db8fdaa3537c675yl
a23fd118e437af0a7877dd313db8fdaa3537c675yl /* let caller to do more job on each item */
a23fd118e437af0a7877dd313db8fdaa3537c675yl if (mempool->item_func_free != NULL) {
a23fd118e437af0a7877dd313db8fdaa3537c675yl
a23fd118e437af0a7877dd313db8fdaa3537c675yl mempool->item_func_free(mempool,
a23fd118e437af0a7877dd313db8fdaa3537c675yl mempool->memblocks_arr[i],
a23fd118e437af0a7877dd313db8fdaa3537c675yl i, dma_object,
a23fd118e437af0a7877dd313db8fdaa3537c675yl mempool->shadow_items_arr[index],
a23fd118e437af0a7877dd313db8fdaa3537c675yl index, /* unused */ -1,
a23fd118e437af0a7877dd313db8fdaa3537c675yl mempool->userdata);
a23fd118e437af0a7877dd313db8fdaa3537c675yl }
a23fd118e437af0a7877dd313db8fdaa3537c675yl }
a23fd118e437af0a7877dd313db8fdaa3537c675yl
a23fd118e437af0a7877dd313db8fdaa3537c675yl xge_os_dma_unmap(mempool->pdev,
a23fd118e437af0a7877dd313db8fdaa3537c675yl dma_object->handle, dma_object->addr,
a23fd118e437af0a7877dd313db8fdaa3537c675yl mempool->memblock_size, XGE_OS_DMA_DIR_BIDIRECTIONAL);
a23fd118e437af0a7877dd313db8fdaa3537c675yl
a23fd118e437af0a7877dd313db8fdaa3537c675yl xge_os_free(mempool->pdev, mempool->memblocks_priv_arr[i],
a23fd118e437af0a7877dd313db8fdaa3537c675yl mempool->items_priv_size * mempool->items_per_memblock);
a23fd118e437af0a7877dd313db8fdaa3537c675yl
a23fd118e437af0a7877dd313db8fdaa3537c675yl xge_os_dma_free(mempool->pdev, mempool->memblocks_arr[i],
a23fd118e437af0a7877dd313db8fdaa3537c675yl mempool->memblock_size, &dma_object->acc_handle,
a23fd118e437af0a7877dd313db8fdaa3537c675yl &dma_object->handle);
a23fd118e437af0a7877dd313db8fdaa3537c675yl }
a23fd118e437af0a7877dd313db8fdaa3537c675yl
a23fd118e437af0a7877dd313db8fdaa3537c675yl if (mempool->items_arr) {
a23fd118e437af0a7877dd313db8fdaa3537c675yl xge_os_free(mempool->pdev, mempool->items_arr, sizeof(void*) *
a23fd118e437af0a7877dd313db8fdaa3537c675yl mempool->items_max);
a23fd118e437af0a7877dd313db8fdaa3537c675yl }
a23fd118e437af0a7877dd313db8fdaa3537c675yl
a23fd118e437af0a7877dd313db8fdaa3537c675yl if (mempool->shadow_items_arr) {
a23fd118e437af0a7877dd313db8fdaa3537c675yl xge_os_free(mempool->pdev, mempool->shadow_items_arr,
a23fd118e437af0a7877dd313db8fdaa3537c675yl sizeof(void*) * mempool->items_max);
a23fd118e437af0a7877dd313db8fdaa3537c675yl }
a23fd118e437af0a7877dd313db8fdaa3537c675yl
a23fd118e437af0a7877dd313db8fdaa3537c675yl if (mempool->memblocks_dma_arr) {
a23fd118e437af0a7877dd313db8fdaa3537c675yl xge_os_free(mempool->pdev, mempool->memblocks_dma_arr,
a23fd118e437af0a7877dd313db8fdaa3537c675yl sizeof(xge_hal_mempool_dma_t) *
a23fd118e437af0a7877dd313db8fdaa3537c675yl mempool->memblocks_max);
a23fd118e437af0a7877dd313db8fdaa3537c675yl }
a23fd118e437af0a7877dd313db8fdaa3537c675yl
a23fd118e437af0a7877dd313db8fdaa3537c675yl if (mempool->memblocks_priv_arr) {
a23fd118e437af0a7877dd313db8fdaa3537c675yl xge_os_free(mempool->pdev, mempool->memblocks_priv_arr,
a23fd118e437af0a7877dd313db8fdaa3537c675yl sizeof(void*) * mempool->memblocks_max);
a23fd118e437af0a7877dd313db8fdaa3537c675yl }
a23fd118e437af0a7877dd313db8fdaa3537c675yl
a23fd118e437af0a7877dd313db8fdaa3537c675yl if (mempool->memblocks_arr) {
a23fd118e437af0a7877dd313db8fdaa3537c675yl xge_os_free(mempool->pdev, mempool->memblocks_arr,
a23fd118e437af0a7877dd313db8fdaa3537c675yl sizeof(void*) * mempool->memblocks_max);
a23fd118e437af0a7877dd313db8fdaa3537c675yl }
a23fd118e437af0a7877dd313db8fdaa3537c675yl
a23fd118e437af0a7877dd313db8fdaa3537c675yl xge_os_free(mempool->pdev, mempool, sizeof(xge_hal_mempool_t));
a23fd118e437af0a7877dd313db8fdaa3537c675yl}