30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap * CDDL HEADER START
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap * The contents of this file are subject to the terms of the
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap * Common Development and Distribution License (the "License").
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap * You may not use this file except in compliance with the License.
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap * See the License for the specific language governing permissions
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap * and limitations under the License.
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap * When distributing Covered Code, include this CDDL HEADER in each
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap * If applicable, add the following below this CDDL HEADER, with the
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap * fields enclosed by brackets "[]" replaced with your own identifying
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap * information: Portions Copyright [yyyy] [name of copyright owner]
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap * CDDL HEADER END
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap * Copyright 2009 Sun Microsystems, Inc. All rights reserved.
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap * Use is subject to license terms.
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap * Routines for allocating resources for iSER
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlapstatic iser_mr_t *iser_vmem_chunk_alloc(iser_hca_t *hca, ib_memlen_t chunksize,
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlapstatic void iser_vmem_chunk_free(iser_hca_t *hca, iser_mr_t *iser_mr);
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlapstatic iser_mr_t *iser_reg_mem(iser_hca_t *hca, ib_vaddr_t vaddr,
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlapstatic void iser_dereg_mem(iser_hca_t *hca, iser_mr_t *mr);
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlapstatic int iser_vmem_mr_compare(const void *void_mr1, const void *void_mr2);
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap * iser_init_hca_caches()
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap * Invoked per HCA instance initialization, to establish HCA-wide
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap * message and buffer kmem caches. Note we'll uniquify cache names
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap * with the lower 32-bits of the HCA GUID.
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap (void) snprintf(name, ISER_CACHE_NAMELEN, "iser_msg_pool_%08x",
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap hca->hca_msg_pool = iser_vmem_create(name, hca, ISER_MSG_MR_CHUNKSIZE,
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap (void) snprintf(name, ISER_CACHE_NAMELEN, "iser_msg_cache_%08x",
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap hca->iser_msg_cache = kmem_cache_create(name, sizeof (iser_msg_t),
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap 0, &iser_msg_cache_constructor, &iser_msg_cache_destructor,
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap (void) snprintf(name, ISER_CACHE_NAMELEN, "iser_buf_pool_%08x",
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap hca->hca_buf_pool = iser_vmem_create(name, hca, ISER_BUF_MR_CHUNKSIZE,
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap (void) snprintf(name, ISER_CACHE_NAMELEN, "iser_buf_cache_%08x",
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap hca->iser_buf_cache = kmem_cache_create(name, sizeof (iser_buf_t),
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap 0, &iser_buf_cache_constructor, &iser_buf_cache_destructor,
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap * iser_fini_hca_caches()
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap * Invoked per HCA instance teardown, this routine cleans up the
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap * message and buffer handle caches.
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap * Allocate and initialize an iSER WR handle
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap iser_wr = kmem_cache_alloc(iser_state->iser_wr_cache, KM_NOSLEEP);
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap * Free an iSER WR handle back to the global cache
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap kmem_cache_free(iser_state->iser_wr_cache, iser_wr);
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap * iser_msg_cache_constructor()
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap * Allocate and register memory for an iSER Control-type PDU message.
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap * The cached objects will retain this memory registration in the HCA,
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap * and thus provide a cache of pre-allocated and registered messages
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap * for use in iSER.
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap/* ARGSUSED */
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlapiser_msg_cache_constructor(void *msg_void, void *arg, int flags)
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap memp = iser_vmem_alloc(hca->hca_msg_pool, ISER_MAX_CTRLPDU_LEN);
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap ISER_LOG(CE_NOTE, "iser_msg_cache_constructor: "
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap "failed to allocate backing memory");
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap /* Fill in iser_mr for the memory we just allocated */
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap ISER_LOG(CE_NOTE, "iser_msg_cache_constructor: "
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap iser_vmem_free(hca->hca_msg_pool, memp, ISER_MAX_CTRLPDU_LEN);
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap msg->msg_ds.ds_va = (ib_vaddr_t)(uintptr_t)memp;
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap /* Set a backpointer to this cache to save a lookup on free */
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap * Deregister and free registered memory from an iser_msg_t handle.
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap memp = (uint8_t *)(uintptr_t)(ib_vaddr_t)msg->msg_ds.ds_va;
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap iser_vmem_free(hca->hca_msg_pool, memp, ISER_MAX_CTRLPDU_LEN);
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap * Pull a msg handle off of hca's msg cache. If no object is available
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap * on the cache, a new message buffer will be allocated and registered
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap * with the HCA. Once freed, this message will not be unregistered, thus
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap * building up a cache of pre-allocated and registered message buffers
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap * over time.
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlapiser_msg_get(iser_hca_t *hca, int num, int *ret)
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap * Pull num number of message handles off the cache, linking
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap * them if more than one have been requested.
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap for (i = 0; i < num; i++) {
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap tmp = kmem_cache_alloc(hca->iser_msg_cache, KM_NOSLEEP);
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap ISER_LOG(CE_NOTE, "iser_msg_get: alloc failed, "
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap * Free this msg back to its cache, leaving the memory contained by
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap * it registered for later re-use.
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap * iser_buf_cache_constructor()
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap * Allocate and register memory for an iSER RDMA operation. The cached
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap * objects will retain this memory registration in the HCA, and thus
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap * provide a cache of pre-allocated and registered messages for use in
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap/* ARGSUSED */
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlapiser_buf_cache_constructor(void *mr, void *arg, int flags)
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap /* Allocate an iser_mr handle for this buffer */
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap iser_buf->iser_mr = kmem_zalloc(sizeof (iser_mr_t), KM_NOSLEEP);
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap ISER_LOG(CE_NOTE, "iser_buf_cache_constructor: "
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap "failed to allocate memory for iser_mr handle");
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap memp = iser_vmem_alloc(hca->hca_buf_pool, ISER_DEFAULT_BUFLEN);
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap kmem_free(iser_buf->iser_mr, sizeof (iser_mr_t));
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap /* Fill in iser_mr for the memory we just allocated */
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap status = iser_vmem_mr(hca->hca_buf_pool, memp, ISER_DEFAULT_BUFLEN,
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap /* Set buf pointer and len for later manipulation (if necessary) */
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap /* Populate the SGE Vaddr and L_key for the xfer operation later */
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap iser_buf->buf_ds.ds_va = iser_buf->iser_mr->is_mrva;
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap iser_buf->buf_ds.ds_key = iser_buf->iser_mr->is_mrlkey;
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap /* Set a backpointer to this cache to save a lookup on free */
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap * Deregister and free registered memory from an iser_buf_t handle.
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap iser_vmem_free(hca->hca_buf_pool, iser_buf->buf, iser_buf->buflen);
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap kmem_free(iser_buf->iser_mr, sizeof (iser_mr_t));
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap * Registration for initiator buffers
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlapiser_reg_rdma_mem(iser_hca_t *hca, idm_buf_t *idb)
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap iser_mr = iser_reg_mem(hca, (ib_vaddr_t)(uintptr_t)idb->idb_buf,
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap idb->idb_buflen, ISER_BUF_MR_FLAGS | IBT_MR_NOSLEEP);
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap ISER_LOG(CE_NOTE, "iser_reg_rdma_mem: failed to register "
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap "memory for idm_buf_t");
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlapiser_dereg_rdma_mem(iser_hca_t *hca, idm_buf_t *idb)
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlapiser_vmem_create(const char *name, iser_hca_t *hca, ib_memlen_t chunksize,
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap uint64_t max_total_size, ibt_mr_flags_t arena_mr_flags)
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap result = kmem_zalloc(sizeof (*result), KM_SLEEP);
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap mutex_init(&result->ivmp_mutex, NULL, MUTEX_DRIVER, NULL);
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap avl_create(&result->ivmp_mr_list, iser_vmem_mr_compare,
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap sizeof (iser_mr_t), offsetof(iser_mr_t, is_avl_ln));
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap first_chunk = iser_vmem_chunk_alloc(hca, chunksize,
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap (size_t)first_chunk->is_mrlen, ISER_MR_QUANTSIZE,
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlapiser_vmem_destroy(iser_vmem_mr_pool_t *vmr_pool)
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap for (chunk = avl_first(&vmr_pool->ivmp_mr_list); chunk != NULL;
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap next_chunk = AVL_NEXT(&vmr_pool->ivmp_mr_list, chunk);
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap iser_vmem_chunk_free(vmr_pool->ivmp_hca, chunk);
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlapiser_vmem_alloc(iser_vmem_mr_pool_t *vmr_pool, size_t size)
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap * Don't go over the pool size limit. We can allocate
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap * partial chunks so it's not always the case that
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap * current_size + chunk_size == max_total_size
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap next_chunk = iser_vmem_chunk_alloc(vmr_pool->ivmp_hca,
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap chunk_len, vmr_pool->ivmp_mr_flags | IBT_MR_NOSLEEP);
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap /* Free the chunk we just allocated */
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlapiser_vmem_free(iser_vmem_mr_pool_t *vmr_pool, void *vaddr, size_t size)
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlapiser_vmem_mr(iser_vmem_mr_pool_t *vmr_pool, void *vaddr, size_t size,
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap ib_vaddr_t mrva = (ib_vaddr_t)(uintptr_t)vaddr;
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap nearest_chunk = avl_find(&vmr_pool->ivmp_mr_list, &search_chunk,
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap nearest_chunk = avl_nearest(&vmr_pool->ivmp_mr_list, where,
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap /* See if this chunk contains the specified address range */
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap chunk_end = nearest_chunk->is_mrva + nearest_chunk->is_mrlen;
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap /* Yes, this chunk contains the address range */
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlapiser_vmem_chunk_alloc(iser_hca_t *hca, ib_memlen_t chunksize,
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap while ((chunk == NULL) && (chunksize >= ISER_MIN_CHUNKSIZE)) {
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap "chunk alloc of %d failed, trying %d",
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap "New chunk %p size %d", chunk, (int)chunksize);
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap result = iser_reg_mem(hca, (ib_vaddr_t)(uintptr_t)chunk,
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap "Chunk registration failed");
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlapiser_vmem_chunk_free(iser_hca_t *hca, iser_mr_t *iser_mr)
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap void *chunk = (void *)(uintptr_t)iser_mr->is_mrva;
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlapiser_reg_mem(iser_hca_t *hca, ib_vaddr_t vaddr, ib_memlen_t len,
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap result = (iser_mr_t *)kmem_zalloc(sizeof (iser_mr_t), km_flags);
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap ISER_LOG(CE_NOTE, "iser_reg_mem: failed to allocate "
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap "memory for iser_mr handle");
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap status = ibt_register_mr(hca->hca_hdl, hca->hca_pdhdl, &mr_attr,
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap ISER_LOG(CE_NOTE, "iser_reg_mem: ibt_register_mr "
aedf2b3bb56b025fcaf87b49ec6c8aeea07f16d7srivijitha dugganapalli (void) ibt_deregister_mr(hca->hca_hdl, mr->is_mrhdl);
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlapiser_vmem_mr_compare(const void *void_mr1, const void *void_mr2)
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap /* Sort memory chunks by their virtual address */
30e7468f8f41aa30ada067b2c1d5d284046514daPeter Dunlap return (-1);