9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * CDDL HEADER START
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * The contents of this file are subject to the terms of the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Common Development and Distribution License (the "License").
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * You may not use this file except in compliance with the License.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * See the License for the specific language governing permissions
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * and limitations under the License.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * When distributing Covered Code, include this CDDL HEADER in each
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * If applicable, add the following below this CDDL HEADER, with the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * fields enclosed by brackets "[]" replaced with your own identifying
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * information: Portions Copyright [yyyy] [name of copyright owner]
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * CDDL HEADER END
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Tavor Memory Region/Window Routines
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Implements all the routines necessary to provide the requisite memory
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * registration verbs. These include operations like RegisterMemRegion(),
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * DeregisterMemRegion(), ReregisterMemRegion, RegisterSharedMemRegion,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * etc., that affect Memory Regions. It also includes the verbs that
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * affect Memory Windows, including AllocMemWindow(), FreeMemWindow(),
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * and QueryMemWindow().
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Used by tavor_mr_keycalc() below to fill in the "unconstrained" portion
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * of Tavor memory keys (LKeys and RKeys)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic uint_t tavor_debug_memkey_cnt = 0x00000000;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic int tavor_mr_common_reg(tavor_state_t *state, tavor_pdhdl_t pd,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor tavor_bind_info_t *bind, tavor_mrhdl_t *mrhdl, tavor_mr_options_t *op);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic int tavor_mr_common_rereg(tavor_state_t *state, tavor_mrhdl_t mr,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor tavor_pdhdl_t pd, tavor_bind_info_t *bind, tavor_mrhdl_t *mrhdl_new,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic int tavor_mr_rereg_xlat_helper(tavor_state_t *state, tavor_mrhdl_t mr,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor tavor_bind_info_t *bind, tavor_mr_options_t *op, uint64_t *mtt_addr,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic uint64_t tavor_mr_nummtt_needed(tavor_state_t *state,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic int tavor_mr_mem_bind(tavor_state_t *state, tavor_bind_info_t *bind,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic void tavor_mr_mem_unbind(tavor_state_t *state,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic int tavor_mr_fast_mtt_write(tavor_rsrc_t *mtt, tavor_bind_info_t *bind,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic int tavor_mtt_refcnt_inc(tavor_rsrc_t *rsrc);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic int tavor_mtt_refcnt_dec(tavor_rsrc_t *rsrc);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * The Tavor umem_lockmemory() callback ops. When userland memory is
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * registered, these callback ops are specified. The tavor_umap_umemlock_cb()
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * callback will be called whenever the memory for the corresponding
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * ddi_umem_cookie_t is being freed.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic struct umem_callback_ops tavor_umem_cbops = {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * tavor_mr_register()
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Context: Can be called from interrupt or base context.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylortavor_mr_register(tavor_state_t *state, tavor_pdhdl_t pd,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ibt_mr_attr_t *mr_attr, tavor_mrhdl_t *mrhdl, tavor_mr_options_t *op)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Fill in the "bind" struct. This struct provides the majority
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * of the information that will be used to distinguish between an
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * "addr" binding (as is the case here) and a "buf" binding (see
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * below). The "bind" struct is later passed to tavor_mr_mem_bind()
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * which does most of the "heavy lifting" for the Tavor memory
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * registration routines.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor status = tavor_mr_common_reg(state, pd, &bind, mrhdl, op);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * tavor_mr_register_buf()
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Context: Can be called from interrupt or base context.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylortavor_mr_register_buf(tavor_state_t *state, tavor_pdhdl_t pd,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ibt_smr_attr_t *mr_attr, struct buf *buf, tavor_mrhdl_t *mrhdl,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Fill in the "bind" struct. This struct provides the majority
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * of the information that will be used to distinguish between an
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * "addr" binding (see above) and a "buf" binding (as is the case
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * here). The "bind" struct is later passed to tavor_mr_mem_bind()
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * which does most of the "heavy lifting" for the Tavor memory
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * registration routines. Note: We have chosen to provide
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * "b_un.b_addr" as the IB address (when the IBT_MR_PHYS_IOVA flag is
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * not set). It is not critical what value we choose here as it need
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * only be unique for the given RKey (which will happen by default),
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * so the choice here is somewhat arbitrary.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor bind.bi_addr = (uint64_t)(uintptr_t)buf->b_un.b_addr;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor status = tavor_mr_common_reg(state, pd, &bind, mrhdl, op);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * tavor_mr_register_shared()
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Context: Can be called from interrupt or base context.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylortavor_mr_register_shared(tavor_state_t *state, tavor_mrhdl_t mrhdl,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor tavor_pdhdl_t pd, ibt_smr_attr_t *mr_attr, tavor_mrhdl_t *mrhdl_new)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Check the sleep flag. Ensure that it is consistent with the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * current thread context (i.e. if we are currently in the interrupt
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * context, then we shouldn't be attempting to sleep).
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor sleep = (mr_attr->mr_flags & IBT_MR_NOSLEEP) ? TAVOR_NOSLEEP :
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Set "status" and "errormsg" and goto failure */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TAVOR_TNF_FAIL(IBT_INVALID_PARAM, "invalid flags");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Increment the reference count on the protection domain (PD) */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Allocate an MPT entry. This will be filled in with all the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * necessary parameters to define the shared memory region.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Specifically, it will be made to reference the currently existing
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * MTT entries and ownership of the MPT will be passed to the hardware
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * in the last step below. If we fail here, we must undo the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * protection domain reference count.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor status = tavor_rsrc_alloc(state, TAVOR_MPT, 1, sleep, &mpt);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Set "status" and "errormsg" and goto failure */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TAVOR_TNF_FAIL(IBT_INSUFF_RESOURCE, "failed MPT");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Allocate the software structure for tracking the shared memory
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * region (i.e. the Tavor Memory Region handle). If we fail here, we
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * must undo the protection domain reference count and the previous
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * resource allocation.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor status = tavor_rsrc_alloc(state, TAVOR_MRHDL, 1, sleep, &rsrc);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Set "status" and "errormsg" and goto failure */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TAVOR_TNF_FAIL(IBT_INSUFF_RESOURCE, "failed MR handle");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Setup and validate the memory region access flags. This means
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * translating the IBTF's enable flags into the access flags that
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * will be used in later operations.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor if (mr_attr->mr_flags & IBT_MR_ENABLE_WINDOW_BIND)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor if (mr_attr->mr_flags & IBT_MR_ENABLE_LOCAL_WRITE)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor if (mr_attr->mr_flags & IBT_MR_ENABLE_REMOTE_READ)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor if (mr_attr->mr_flags & IBT_MR_ENABLE_REMOTE_WRITE)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor if (mr_attr->mr_flags & IBT_MR_ENABLE_REMOTE_ATOMIC)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Calculate keys (Lkey, Rkey) from MPT index. Each key is formed
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * from a certain number of "constrained" bits (the least significant
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * bits) and some number of "unconstrained" bits. The constrained
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * bits must be set to the index of the entry in the MPT table, but
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the unconstrained bits can be set to any value we wish. Note:
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * if no remote access is required, then the RKey value is not filled
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * in. Otherwise both Rkey and LKey are given the same value.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor tavor_mr_keycalc(state, mpt->tr_indx, &mr->mr_lkey);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Grab the MR lock for the current memory region */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Check here to see if the memory region has already been partially
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * deregistered as a result of a tavor_umap_umemlock_cb() callback.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * If so, this is an error, return failure.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor if ((mrhdl->mr_is_umem) && (mrhdl->mr_umemcookie == NULL)) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Set "status" and "errormsg" and goto failure */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TAVOR_TNF_FAIL(IBT_MR_HDL_INVALID, "invalid mrhdl");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Determine if the original memory was from userland and, if so, pin
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the pages (again) with umem_lockmemory(). This will guarantee a
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * separate callback for each of this shared region's MR handles.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * If this is userland memory, then allocate an entry in the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * "userland resources database". This will later be added to
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the database (after all further memory registration operations are
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * successful). If we fail here, we must undo all the above setup.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor umem_len = ptob(btopr(mrhdl->mr_bindinfo.bi_len +
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ((uintptr_t)mrhdl->mr_bindinfo.bi_addr & PAGEOFFSET)));
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor umem_addr = (caddr_t)((uintptr_t)mrhdl->mr_bindinfo.bi_addr &
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor umem_flags = (DDI_UMEMLOCK_WRITE | DDI_UMEMLOCK_READ |
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor status = umem_lockmemory(umem_addr, umem_len, umem_flags,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Set "status" and "errormsg" and goto failure */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TAVOR_TNF_FAIL(IBT_INSUFF_RESOURCE, "failed umem pin");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor umapdb = tavor_umap_db_alloc(state->ts_instance,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor (uint64_t)(uintptr_t)umem_cookie, MLNX_UMAP_MRMEM_RSRC,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Set "status" and "errormsg" and goto failure */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TAVOR_TNF_FAIL(IBT_INSUFF_RESOURCE, "failed umap add");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Copy the MTT resource pointer (and additional parameters) from
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the original Tavor Memory Region handle. Note: this is normally
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * where the tavor_mr_mem_bind() routine would be called, but because
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * we already have bound and filled-in MTT entries it is simply a
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * matter here of managing the MTT reference count and grabbing the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * address of the MTT table entries (for filling in the shared region's
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * MPT entry).
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Increment the MTT reference count (to reflect the fact that
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the MTT is now shared)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Update the new "bind" virtual address. Do some extra work here
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * to ensure proper alignment. That is, make sure that the page
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * offset for the beginning of the old range is the same as the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * offset for this new mapping
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor pgsize_msk = (((uint64_t)1 << mr->mr_logmttpgsz) - 1);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor bind->bi_addr = ((mr_attr->mr_vaddr & ~pgsize_msk) |
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Get the base address for the MTT table. This will be necessary
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * in the next step when we are setting up the MPT entry.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor mtt_ddrbaseaddr = (uint64_t)(uintptr_t)rsrc_pool->rsrc_ddr_offset;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Fill in the MPT entry. This is the final step before passing
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * ownership of the MPT entry to the Tavor hardware. We use all of
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the information collected/calculated above to fill in the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * requisite portions of the MPT.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor mpt_entry.en_bind = (mr->mr_accflag & IBT_MR_WINDOW_BIND) ? 1 : 0;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor mpt_entry.atomic = (mr->mr_accflag & IBT_MR_REMOTE_ATOMIC) ? 1 : 0;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor mpt_entry.rw = (mr->mr_accflag & IBT_MR_REMOTE_WRITE) ? 1 : 0;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor mpt_entry.rr = (mr->mr_accflag & IBT_MR_REMOTE_READ) ? 1 : 0;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor mpt_entry.lw = (mr->mr_accflag & IBT_MR_LOCAL_WRITE) ? 1 : 0;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor mpt_entry.win_cnt_limit = TAVOR_UNLIMITED_WIN_BIND;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor mtt_addr = mtt_ddrbaseaddr + (mtt->tr_indx << TAVOR_MTT_SIZE_SHIFT);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Write the MPT entry to hardware. Lastly, we pass ownership of
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the entry to the hardware. Note: in general, this operation
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * shouldn't fail. But if it does, we have to undo everything we've
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * done above before returning error.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor status = tavor_cmn_ownership_cmd_post(state, SW2HW_MPT, &mpt_entry,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor cmn_err(CE_CONT, "Tavor: SW2HW_MPT command failed: %08x\n",
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TNF_PROBE_1(tavor_mr_register_shared_sw2hw_mpt_cmd_fail,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Set "status" and "errormsg" and goto failure */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor "tavor SW2HW_MPT command");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Fill in the rest of the Tavor Memory Region handle. Having
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * successfully transferred ownership of the MPT, we can update the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * following fields for use in further operations on the MR.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor mr->mr_umemcookie = (mr_is_umem != 0) ? umem_cookie : NULL;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * If this is userland memory, then we need to insert the previously
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * allocated entry into the "userland resources database". This will
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * allow for later coordination between the tavor_umap_umemlock_cb()
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * callback and tavor_mr_deregister().
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * The following is cleanup for all possible failure cases in this routine
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TNF_PROBE_1(tavor_mr_register_shared_fail, TAVOR_TNF_ERROR, "",
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * tavor_mr_deregister()
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Context: Can be called from interrupt or base context.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor/* ARGSUSED */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylortavor_mr_deregister(tavor_state_t *state, tavor_mrhdl_t *mrhdl, uint_t level,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Check the sleep flag. Ensure that it is consistent with the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * current thread context (i.e. if we are currently in the interrupt
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * context, then we shouldn't be attempting to sleep).
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Set "status" and "errormsg" and goto failure */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TAVOR_TNF_FAIL(IBT_INVALID_PARAM, "invalid sleep flags");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TNF_PROBE_1(tavor_mr_deregister_fail, TAVOR_TNF_ERROR, "",
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Pull all the necessary information from the Tavor Memory Region
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * handle. This is necessary here because the resource for the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * MR handle is going to be freed up as part of the this
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * deregistration
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Check here to see if the memory region has already been partially
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * deregistered as a result of the tavor_umap_umemlock_cb() callback.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * If so, then jump to the end and free the remaining resources.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor if ((mr->mr_is_umem) && (mr->mr_umemcookie == NULL)) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * We must drop the "mr_lock" here to ensure that both SLEEP and
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * NOSLEEP calls into the firmware work as expected. Also, if two
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * threads are attemping to access this MR (via de-register,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * re-register, or otherwise), then we allow the firmware to enforce
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the checking, that only one deregister is valid.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Reclaim MPT entry from hardware (if necessary). Since the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * tavor_mr_deregister() routine is used in the memory region
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * reregistration process as well, it is possible that we will
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * not always wish to reclaim ownership of the MPT. Check the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * "level" arg and, if necessary, attempt to reclaim it. If
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the ownership transfer fails for any reason, we check to see
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * what command status was returned from the hardware. The only
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * "expected" error status is the one that indicates an attempt to
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * deregister a memory region that has memory windows bound to it
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor status = tavor_cmn_ownership_cmd_post(state, HW2SW_MPT,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Re-grab the mr_lock here. Since further access to the protected
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * 'mr' structure is needed, and we would have returned previously for
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the multiple deregistration case, we can safely grab the lock here.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * If the memory had come from userland, then we do a lookup in the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * "userland resources database". On success, we free the entry, call
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * ddi_umem_unlock(), and continue the cleanup. On failure (which is
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * an indication that the umem_lockmemory() callback has called
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * tavor_mr_deregister()), we call ddi_umem_unlock() and invalidate
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the "mr_umemcookie" field in the MR handle (this will be used
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * later to detect that only partial cleaup still remains to be done
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * on the MR handle).
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor MLNX_UMAP_MRMEM_RSRC, &value, TAVOR_UMAP_DB_REMOVE,
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor /* mtt_refcnt is NULL in the case of tavor_dma_mr_register() */
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor * Decrement the MTT reference count. Since the MTT resource
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor * may be shared between multiple memory regions (as a result
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor * of a "RegisterSharedMR" verb) it is important that we not
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor * free up or unbind resources prematurely. If it's not shared
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor * (as indicated by the return status), then free the resource.
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor * Free up the MTT entries and unbind the memory. Here,
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor * as above, we attempt to free these resources only if
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor * it is appropriate to do so.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * If the MR handle has been invalidated, then drop the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * lock and return success. Note: This only happens because
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the umem_lockmemory() callback has been triggered. The
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * cleanup here is partial, and further cleanup (in a
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * subsequent tavor_mr_deregister() call) will be necessary.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor if ((mr->mr_is_umem) && (mr->mr_umemcookie == NULL)) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Free the Tavor Memory Region handle */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Free up the MPT entry resource */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Decrement the reference count on the protection domain (PD) */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Set the mrhdl pointer to NULL and return success */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * tavor_mr_query()
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Context: Can be called from interrupt or base context.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor/* ARGSUSED */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylortavor_mr_query(tavor_state_t *state, tavor_mrhdl_t mr,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Check here to see if the memory region has already been partially
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * deregistered as a result of a tavor_umap_umemlock_cb() callback.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * If so, this is an error, return failure.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor if ((mr->mr_is_umem) && (mr->mr_umemcookie == NULL)) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TNF_PROBE_0(tavor_mr_query_inv_mrhdl_fail, TAVOR_TNF_ERROR, "");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Fill in the queried attributes */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Fill in the "local" attributes */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor attr->mr_lbounds.pb_addr = (ib_vaddr_t)mr->mr_bindinfo.bi_addr;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor attr->mr_lbounds.pb_len = (size_t)mr->mr_bindinfo.bi_len;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Fill in the "remote" attributes (if necessary). Note: the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * remote attributes are only valid if the memory region has one
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * or more of the remote access flags set.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor attr->mr_rbounds.pb_addr = (ib_vaddr_t)mr->mr_bindinfo.bi_addr;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor attr->mr_rbounds.pb_len = (size_t)mr->mr_bindinfo.bi_len;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * If region is mapped for streaming (i.e. noncoherent), then set sync
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * is required
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor attr->mr_sync_required = (mr->mr_bindinfo.bi_flags &
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * tavor_mr_reregister()
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Context: Can be called from interrupt or base context.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylortavor_mr_reregister(tavor_state_t *state, tavor_mrhdl_t mr,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor tavor_pdhdl_t pd, ibt_mr_attr_t *mr_attr, tavor_mrhdl_t *mrhdl_new,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Fill in the "bind" struct. This struct provides the majority
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * of the information that will be used to distinguish between an
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * "addr" binding (as is the case here) and a "buf" binding (see
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * below). The "bind" struct is later passed to tavor_mr_mem_bind()
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * which does most of the "heavy lifting" for the Tavor memory
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * registration (and reregistration) routines.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor status = tavor_mr_common_rereg(state, mr, pd, &bind, mrhdl_new, op);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * tavor_mr_reregister_buf()
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Context: Can be called from interrupt or base context.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylortavor_mr_reregister_buf(tavor_state_t *state, tavor_mrhdl_t mr,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor tavor_pdhdl_t pd, ibt_smr_attr_t *mr_attr, struct buf *buf,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor tavor_mrhdl_t *mrhdl_new, tavor_mr_options_t *op)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Fill in the "bind" struct. This struct provides the majority
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * of the information that will be used to distinguish between an
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * "addr" binding (see above) and a "buf" binding (as is the case
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * here). The "bind" struct is later passed to tavor_mr_mem_bind()
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * which does most of the "heavy lifting" for the Tavor memory
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * registration routines. Note: We have chosen to provide
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * "b_un.b_addr" as the IB address (when the IBT_MR_PHYS_IOVA flag is
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * not set). It is not critical what value we choose here as it need
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * only be unique for the given RKey (which will happen by default),
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * so the choice here is somewhat arbitrary.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor bind.bi_addr = (uint64_t)(uintptr_t)buf->b_un.b_addr;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor status = tavor_mr_common_rereg(state, mr, pd, &bind, mrhdl_new, op);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TNF_PROBE_0(tavor_mr_reregister_buf_cmnreg_fail,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * tavor_mr_sync()
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Context: Can be called from interrupt or base context.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor/* ARGSUSED */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylortavor_mr_sync(tavor_state_t *state, ibt_mr_sync_t *mr_segs, size_t num_segs)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Process each of the ibt_mr_sync_t's */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor for (i = 0; i < num_segs; i++) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Check for valid memory region handle */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Set "status" and "errormsg" and goto failure */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TAVOR_TNF_FAIL(IBT_MR_HDL_INVALID, "invalid mrhdl");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Check here to see if the memory region has already been
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * partially deregistered as a result of a
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * tavor_umap_umemlock_cb() callback. If so, this is an
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * error, return failure.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor if ((mrhdl->mr_is_umem) && (mrhdl->mr_umemcookie == NULL)) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Set "status" and "errormsg" and goto failure */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TAVOR_TNF_FAIL(IBT_MR_HDL_INVALID, "invalid mrhdl2");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Check for valid bounds on sync request */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor mr_end = mr_start + mrhdl->mr_bindinfo.bi_len - 1;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor if ((seg_vaddr < mr_start) || (seg_vaddr > mr_end)) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Set "status" and "errormsg" and goto failure */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TAVOR_TNF_FAIL(IBT_MR_VA_INVALID, "invalid vaddr");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor if ((seg_end < mr_start) || (seg_end > mr_end)) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Set "status" and "errormsg" and goto failure */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TAVOR_TNF_FAIL(IBT_MR_LEN_INVALID, "invalid length");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Determine what type (i.e. direction) for sync */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor } else if (mr_segs[i].ms_flags & IBT_SYNC_WRITE) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Set "status" and "errormsg" and goto failure */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TAVOR_TNF_FAIL(IBT_INVALID_PARAM, "invalid sync type");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor (void) ddi_dma_sync(mrhdl->mr_bindinfo.bi_dmahdl,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor (off_t)(seg_vaddr - mr_start), (size_t)seg_len, type);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TNF_PROBE_1(tavor_mr_sync_fail, TAVOR_TNF_ERROR, "", tnf_string, msg,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * tavor_mw_alloc()
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Context: Can be called from interrupt or base context.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylortavor_mw_alloc(tavor_state_t *state, tavor_pdhdl_t pd, ibt_mw_flags_t flags,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Check the sleep flag. Ensure that it is consistent with the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * current thread context (i.e. if we are currently in the interrupt
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * context, then we shouldn't be attempting to sleep).
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor sleep = (flags & IBT_MW_NOSLEEP) ? TAVOR_NOSLEEP : TAVOR_SLEEP;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Set "status" and "errormsg" and goto failure */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TAVOR_TNF_FAIL(IBT_INVALID_PARAM, "invalid flags");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Increment the reference count on the protection domain (PD) */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Allocate an MPT entry (for use as a memory window). Since the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Tavor hardware uses the MPT entry for memory regions and for
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * memory windows, we will fill in this MPT with all the necessary
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * parameters for the memory window. And then (just as we do for
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * memory regions) ownership will be passed to the hardware in the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * final step below. If we fail here, we must undo the protection
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * domain reference count.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor status = tavor_rsrc_alloc(state, TAVOR_MPT, 1, sleep, &mpt);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Set "status" and "errormsg" and goto failure */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TAVOR_TNF_FAIL(IBT_INSUFF_RESOURCE, "failed MPT");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Allocate the software structure for tracking the memory window (i.e.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the Tavor Memory Window handle). Note: This is actually the same
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * software structure used for tracking memory regions, but since many
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * of the same properties are needed, only a single structure is
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * necessary. If we fail here, we must undo the protection domain
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * reference count and the previous resource allocation.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor status = tavor_rsrc_alloc(state, TAVOR_MRHDL, 1, sleep, &rsrc);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Set "status" and "errormsg" and goto failure */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TAVOR_TNF_FAIL(IBT_INSUFF_RESOURCE, "failed MR handle");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Calculate an "unbound" RKey from MPT index. In much the same way
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * as we do for memory regions (above), this key is constructed from
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * a "constrained" (which depends on the MPT index) and an
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * "unconstrained" portion (which may be arbitrarily chosen).
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor tavor_mr_keycalc(state, mpt->tr_indx, &mw->mr_rkey);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Fill in the MPT entry. This is the final step before passing
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * ownership of the MPT entry to the Tavor hardware. We use all of
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the information collected/calculated above to fill in the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * requisite portions of the MPT. Note: fewer entries in the MPT
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * entry are necessary to allocate a memory window.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Write the MPT entry to hardware. Lastly, we pass ownership of
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the entry to the hardware. Note: in general, this operation
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * shouldn't fail. But if it does, we have to undo everything we've
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * done above before returning error.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor status = tavor_cmn_ownership_cmd_post(state, SW2HW_MPT, &mpt_entry,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor cmn_err(CE_CONT, "Tavor: SW2HW_MPT command failed: %08x\n",
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Set "status" and "errormsg" and goto failure */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor "tavor SW2HW_MPT command");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Fill in the rest of the Tavor Memory Window handle. Having
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * successfully transferred ownership of the MPT, we can update the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * following fields for use in further operations on the MW.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TNF_PROBE_1(tavor_mw_alloc_fail, TAVOR_TNF_ERROR, "",
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * tavor_mw_free()
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Context: Can be called from interrupt or base context.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylortavor_mw_free(tavor_state_t *state, tavor_mwhdl_t *mwhdl, uint_t sleep)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Check the sleep flag. Ensure that it is consistent with the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * current thread context (i.e. if we are currently in the interrupt
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * context, then we shouldn't be attempting to sleep).
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Set "status" and "errormsg" and goto failure */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TAVOR_TNF_FAIL(IBT_INVALID_PARAM, "invalid sleep flags");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TNF_PROBE_1(tavor_mw_free_fail, TAVOR_TNF_ERROR, "",
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Pull all the necessary information from the Tavor Memory Window
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * handle. This is necessary here because the resource for the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * MW handle is going to be freed up as part of the this operation.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Reclaim the MPT entry from hardware. Note: in general, it is
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * unexpected for this operation to return an error.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor status = tavor_cmn_ownership_cmd_post(state, HW2SW_MPT, NULL,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor cmn_err(CE_CONT, "Tavor: HW2SW_MPT command failed: %08x\n",
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TNF_PROBE_1(tavor_hw2sw_mpt_cmd_fail, TAVOR_TNF_ERROR, "",
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Free the Tavor Memory Window handle */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Free up the MPT entry resource */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Decrement the reference count on the protection domain (PD) */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Set the mwhdl pointer to NULL and return success */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * tavor_mr_keycalc()
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Context: Can be called from interrupt or base context.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylortavor_mr_keycalc(tavor_state_t *state, uint32_t indx, uint32_t *key)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Generate a simple key from counter. Note: We increment this
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * static variable _intentionally_ without any kind of mutex around
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * it. First, single-threading all operations through a single lock
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * would be a bad idea (from a performance point-of-view). Second,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the upper "unconstrained" bits don't really have to be unique
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * because the lower bits are guaranteed to be (although we do make a
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * best effort to ensure that they are). Third, the window for the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * race (where both threads read and update the counter at the same
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * time) is incredibly small.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * And, lastly, we'd like to make this into a "random" key XXX
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(tavor_debug_memkey_cnt))
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor log_num_mpt = state->ts_cfg_profile->cp_log_num_mpt;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor tmp = (tavor_debug_memkey_cnt++) << log_num_mpt;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * tavor_mr_common_reg()
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Context: Can be called from interrupt or base context.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylortavor_mr_common_reg(tavor_state_t *state, tavor_pdhdl_t pd,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor tavor_bind_info_t *bind, tavor_mrhdl_t *mrhdl, tavor_mr_options_t *op)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor uint_t sleep, mtt_pgsize_bits, bind_type, mr_is_umem;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Check the "options" flag. Currently this flag tells the driver
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * whether or not the region should be bound normally (i.e. with
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * entries written into the PCI IOMMU), whether it should be
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * registered to bypass the IOMMU, and whether or not the resulting
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * address should be "zero-based" (to aid the alignment restrictions
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor bind_override_addr = op->mro_bind_override_addr;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Extract the flags field from the tavor_bind_info_t */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Check for invalid length. Check is the length is zero or if the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * length is larger than the maximum configured value. Return error
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor max_sz = ((uint64_t)1 << state->ts_cfg_profile->cp_log_max_mrw_sz);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor if ((bind->bi_len == 0) || (bind->bi_len > max_sz)) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Set "status" and "errormsg" and goto failure */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TAVOR_TNF_FAIL(IBT_MR_LEN_INVALID, "invalid length");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Check the sleep flag. Ensure that it is consistent with the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * current thread context (i.e. if we are currently in the interrupt
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * context, then we shouldn't be attempting to sleep).
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor sleep = (flags & IBT_MR_NOSLEEP) ? TAVOR_NOSLEEP: TAVOR_SLEEP;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Set "status" and "errormsg" and goto failure */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TAVOR_TNF_FAIL(IBT_INVALID_PARAM, "invalid flags");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Get the base address for the MTT table. This will be necessary
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * below when we are setting up the MPT entry.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor mtt_ddrbaseaddr = (uint64_t)(uintptr_t)rsrc_pool->rsrc_ddr_offset;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Increment the reference count on the protection domain (PD) */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Allocate an MPT entry. This will be filled in with all the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * necessary parameters to define the memory region. And then
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * ownership will be passed to the hardware in the final step
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * below. If we fail here, we must undo the protection domain
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * reference count.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor status = tavor_rsrc_alloc(state, TAVOR_MPT, 1, sleep, &mpt);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Set "status" and "errormsg" and goto failure */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TAVOR_TNF_FAIL(IBT_INSUFF_RESOURCE, "failed MPT");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Allocate the software structure for tracking the memory region (i.e.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the Tavor Memory Region handle). If we fail here, we must undo
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the protection domain reference count and the previous resource
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * allocation.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor status = tavor_rsrc_alloc(state, TAVOR_MRHDL, 1, sleep, &rsrc);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Set "status" and "errormsg" and goto failure */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TAVOR_TNF_FAIL(IBT_INSUFF_RESOURCE, "failed MR handle");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Setup and validate the memory region access flags. This means
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * translating the IBTF's enable flags into the access flags that
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * will be used in later operations.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Calculate keys (Lkey, Rkey) from MPT index. Each key is formed
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * from a certain number of "constrained" bits (the least significant
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * bits) and some number of "unconstrained" bits. The constrained
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * bits must be set to the index of the entry in the MPT table, but
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the unconstrained bits can be set to any value we wish. Note:
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * if no remote access is required, then the RKey value is not filled
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * in. Otherwise both Rkey and LKey are given the same value.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor tavor_mr_keycalc(state, mpt->tr_indx, &mr->mr_lkey);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Determine if the memory is from userland and pin the pages
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * with umem_lockmemory() if necessary.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Then, if this is userland memory, allocate an entry in the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * "userland resources database". This will later be added to
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the database (after all further memory registration operations are
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * successful). If we fail here, we must undo the reference counts
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * and the previous resource allocations.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor mr_is_umem = (((bind->bi_as != NULL) && (bind->bi_as != &kas)) ? 1 : 0);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor umem_addr = (caddr_t)((uintptr_t)bind->bi_addr & ~PAGEOFFSET);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor umem_flags = (DDI_UMEMLOCK_WRITE | DDI_UMEMLOCK_READ |
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor status = umem_lockmemory(umem_addr, umem_len, umem_flags,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Set "status" and "errormsg" and goto failure */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TAVOR_TNF_FAIL(IBT_INSUFF_RESOURCE, "failed umem pin");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*bind->bi_buf))
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor bind->bi_buf = ddi_umem_iosetup(umem_cookie, 0, umem_len,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Set "status" and "errormsg" and goto failure */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TAVOR_TNF_FAIL(IBT_INSUFF_RESOURCE, "failed iosetup");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*bind->bi_buf))
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor umapdb = tavor_umap_db_alloc(state->ts_instance,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor (uint64_t)(uintptr_t)umem_cookie, MLNX_UMAP_MRMEM_RSRC,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Set "status" and "errormsg" and goto failure */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TAVOR_TNF_FAIL(IBT_INSUFF_RESOURCE, "failed umap add");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Setup the bindinfo for the mtt bind call
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor status = tavor_mr_mtt_bind(state, bh, bind_dmahdl, &mtt,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Set "status" and "errormsg" and goto failure */
949b58c70cf907006b9f724dfad665d44eca5881Bill Taylor * When mtt_bind fails, freerbuf has already been done,
949b58c70cf907006b9f724dfad665d44eca5881Bill Taylor * so make sure not to call it again.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Allocate MTT reference count (to track shared memory regions).
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * This reference count resource may never be used on the given
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * memory region, but if it is ever later registered as "shared"
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * memory region then this resource will be necessary. If we fail
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * here, we do pretty much the same as above to clean up.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor status = tavor_rsrc_alloc(state, TAVOR_REFCNT, 1, sleep,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Set "status" and "errormsg" and goto failure */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TAVOR_TNF_FAIL(IBT_INSUFF_RESOURCE, "failed refence count");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor swrc_tmp = (tavor_sw_refcnt_t *)mtt_refcnt->tr_addr;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*swrc_tmp))
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Fill in the MPT entry. This is the final step before passing
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * ownership of the MPT entry to the Tavor hardware. We use all of
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the information collected/calculated above to fill in the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * requisite portions of the MPT.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor mpt_entry.en_bind = (mr->mr_accflag & IBT_MR_WINDOW_BIND) ? 1 : 0;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor mpt_entry.atomic = (mr->mr_accflag & IBT_MR_REMOTE_ATOMIC) ? 1 : 0;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor mpt_entry.rw = (mr->mr_accflag & IBT_MR_REMOTE_WRITE) ? 1 : 0;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor mpt_entry.rr = (mr->mr_accflag & IBT_MR_REMOTE_READ) ? 1 : 0;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor mpt_entry.lw = (mr->mr_accflag & IBT_MR_LOCAL_WRITE) ? 1 : 0;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor bh->bi_addr = bh->bi_addr & ((1 << mr->mr_logmttpgsz) - 1);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor mpt_entry.win_cnt_limit = TAVOR_UNLIMITED_WIN_BIND;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor mtt_addr = mtt_ddrbaseaddr + (mtt->tr_indx << TAVOR_MTT_SIZE_SHIFT);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Write the MPT entry to hardware. Lastly, we pass ownership of
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the entry to the hardware. Note: in general, this operation
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * shouldn't fail. But if it does, we have to undo everything we've
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * done above before returning error.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor status = tavor_cmn_ownership_cmd_post(state, SW2HW_MPT, &mpt_entry,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor cmn_err(CE_CONT, "Tavor: SW2HW_MPT command failed: %08x\n",
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TNF_PROBE_1(tavor_mr_common_reg_sw2hw_mpt_cmd_fail,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Set "status" and "errormsg" and goto failure */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor "tavor SW2HW_MPT command");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Fill in the rest of the Tavor Memory Region handle. Having
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * successfully transferred ownership of the MPT, we can update the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * following fields for use in further operations on the MR.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor mr->mr_umemcookie = (mr_is_umem != 0) ? umem_cookie : NULL;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * If this is userland memory, then we need to insert the previously
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * allocated entry into the "userland resources database". This will
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * allow for later coordination between the tavor_umap_umemlock_cb()
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * callback and tavor_mr_deregister().
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * The following is cleanup for all possible failure cases in this routine
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Free up the memory ddi_umem_iosetup() allocates
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * internally.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TNF_PROBE_1(tavor_mr_common_reg_fail, TAVOR_TNF_ERROR, "",
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylortavor_dma_mr_register(tavor_state_t *state, tavor_pdhdl_t pd,
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor /* Extract the flags field */
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor * Check the sleep flag. Ensure that it is consistent with the
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor * current thread context (i.e. if we are currently in the interrupt
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor * context, then we shouldn't be attempting to sleep).
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor sleep = (flags & IBT_MR_NOSLEEP) ? TAVOR_NOSLEEP: TAVOR_SLEEP;
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor /* Increment the reference count on the protection domain (PD) */
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor * Allocate an MPT entry. This will be filled in with all the
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor * necessary parameters to define the memory region. And then
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor * ownership will be passed to the hardware in the final step
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor * below. If we fail here, we must undo the protection domain
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor * reference count.
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor status = tavor_rsrc_alloc(state, TAVOR_MPT, 1, sleep, &mpt);
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor * Allocate the software structure for tracking the memory region (i.e.
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor * the Tavor Memory Region handle). If we fail here, we must undo
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor * the protection domain reference count and the previous resource
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor * allocation.
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor status = tavor_rsrc_alloc(state, TAVOR_MRHDL, 1, sleep, &rsrc);
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor * Setup and validate the memory region access flags. This means
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor * translating the IBTF's enable flags into the access flags that
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor * will be used in later operations.
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor * Calculate keys (Lkey, Rkey) from MPT index. Each key is formed
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor * from a certain number of "constrained" bits (the least significant
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor * bits) and some number of "unconstrained" bits. The constrained
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor * bits must be set to the index of the entry in the MPT table, but
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor * the unconstrained bits can be set to any value we wish. Note:
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor * if no remote access is required, then the RKey value is not filled
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor * in. Otherwise both Rkey and LKey are given the same value.
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor tavor_mr_keycalc(state, mpt->tr_indx, &mr->mr_lkey);
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor * Fill in the MPT entry. This is the final step before passing
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor * ownership of the MPT entry to the Tavor hardware. We use all of
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor * the information collected/calculated above to fill in the
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor * requisite portions of the MPT.
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor mpt_entry.en_bind = (mr->mr_accflag & IBT_MR_WINDOW_BIND) ? 1 : 0;
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor mpt_entry.atomic = (mr->mr_accflag & IBT_MR_REMOTE_ATOMIC) ? 1 : 0;
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor mpt_entry.rw = (mr->mr_accflag & IBT_MR_REMOTE_WRITE) ? 1 : 0;
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor mpt_entry.rr = (mr->mr_accflag & IBT_MR_REMOTE_READ) ? 1 : 0;
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor mpt_entry.lw = (mr->mr_accflag & IBT_MR_LOCAL_WRITE) ? 1 : 0;
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor mpt_entry.phys_addr = 1; /* critical bit for this */
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor mpt_entry.win_cnt_limit = TAVOR_UNLIMITED_WIN_BIND;
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor * Write the MPT entry to hardware. Lastly, we pass ownership of
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor * the entry to the hardware if needed. Note: in general, this
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor * operation shouldn't fail. But if it does, we have to undo
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor * everything we've done above before returning error.
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor * For Tavor, this routine (which is common to the contexts) will only
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor * set the ownership if needed - the process of passing the context
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor * itself to HW will take care of setting up the MPT (based on type
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor * and index).
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor status = tavor_cmn_ownership_cmd_post(state, SW2HW_MPT, &mpt_entry,
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor cmn_err(CE_CONT, "Tavor: SW2HW_MPT command failed: %08x\n",
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor * Fill in the rest of the Tavor Memory Region handle. Having
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor * successfully transferred ownership of the MPT, we can update the
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor * following fields for use in further operations on the MR.
17a2b317610f531d565bf4e940433aab2d9e6985Bill Taylor * The following is cleanup for all possible failure cases in this routine
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * tavor_mr_mtt_bind()
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Context: Can be called from interrupt or base context.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylortavor_mr_mtt_bind(tavor_state_t *state, tavor_bind_info_t *bind,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ddi_dma_handle_t bind_dmahdl, tavor_rsrc_t **mtt, uint_t *mtt_pgsize_bits)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Check the sleep flag. Ensure that it is consistent with the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * current thread context (i.e. if we are currently in the interrupt
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * context, then we shouldn't be attempting to sleep).
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor sleep = (bind->bi_flags & IBT_MR_NOSLEEP) ? TAVOR_NOSLEEP: TAVOR_SLEEP;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Set "status" and "errormsg" and goto failure */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TAVOR_TNF_FAIL(IBT_INVALID_PARAM, "invalid flags");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Bind the memory and determine the mapped addresses. This is
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the first of two routines that do all the "heavy lifting" for
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the Tavor memory registration routines. The tavor_mr_mem_bind()
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * routine takes the "bind" struct with all its fields filled
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * in and returns a list of DMA cookies (for the PCI mapped addresses
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * corresponding to the specified address region) which are used by
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the tavor_mr_fast_mtt_write() routine below. If we fail here, we
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * must undo all the previous resource allocation (and PD reference
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor status = tavor_mr_mem_bind(state, bind, bind_dmahdl, sleep);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Set "status" and "errormsg" and goto failure */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TAVOR_TNF_FAIL(IBT_INSUFF_RESOURCE, "failed mem bind");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Determine number of pages spanned. This routine uses the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * information in the "bind" struct to determine the required
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * number of MTT entries needed (and returns the suggested page size -
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * as a "power-of-2" - for each MTT entry).
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor nummtt = tavor_mr_nummtt_needed(state, bind, mtt_pgsize_bits);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Allocate the MTT entries. Use the calculations performed above to
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * allocate the required number of MTT entries. Note: MTT entries are
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * allocated in "MTT segments" which consist of complete cachelines
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * (i.e. 8 entries, 16 entries, etc.) So the TAVOR_NUMMTT_TO_MTTSEG()
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * macro is used to do the proper conversion. If we fail here, we
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * must not only undo all the previous resource allocation (and PD
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * reference count), but we must also unbind the memory.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Set "status" and "errormsg" and goto failure */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TAVOR_TNF_FAIL(IBT_INSUFF_RESOURCE, "failed MTT");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Write the mapped addresses into the MTT entries. This is part two
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * of the "heavy lifting" routines that we talked about above. Note:
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * we pass the suggested page size from the earlier operation here.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * And if we fail here, we again do pretty much the same huge clean up.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor status = tavor_mr_fast_mtt_write(*mtt, bind, *mtt_pgsize_bits);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Set "status" and "errormsg" and goto failure */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TAVOR_TNF_FAIL(ibc_get_ci_failure(0), "failed write mtt");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * The following is cleanup for all possible failure cases in this routine
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TNF_PROBE_1(tavor_mr_mtt_bind_fail, TAVOR_TNF_ERROR, "",
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * tavor_mr_mtt_unbind()
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Context: Can be called from interrupt or base context.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylortavor_mr_mtt_unbind(tavor_state_t *state, tavor_bind_info_t *bind,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Free up the MTT entries and unbind the memory. Here, as above, we
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * attempt to free these resources only if it is appropriate to do so.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * tavor_mr_common_rereg()
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Context: Can be called from interrupt or base context.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylortavor_mr_common_rereg(tavor_state_t *state, tavor_mrhdl_t mr,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor tavor_pdhdl_t pd, tavor_bind_info_t *bind, tavor_mrhdl_t *mrhdl_new,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor uint64_t mtt_addr_to_use, vaddr_to_use, len_to_use;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Check here to see if the memory region corresponds to a userland
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * mapping. Reregistration of userland memory regions is not
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * currently supported. Return failure. XXX
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Set "status" and "errormsg" and goto failure */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TAVOR_TNF_FAIL(IBT_MR_HDL_INVALID, "invalid mrhdl");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Pull MPT resource pointer from the Tavor Memory Region handle */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Extract the flags field from the tavor_bind_info_t */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Check the sleep flag. Ensure that it is consistent with the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * current thread context (i.e. if we are currently in the interrupt
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * context, then we shouldn't be attempting to sleep).
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor sleep = (flags & IBT_MR_NOSLEEP) ? TAVOR_NOSLEEP: TAVOR_SLEEP;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Set "status" and "errormsg" and goto failure */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TAVOR_TNF_FAIL(IBT_INVALID_PARAM, "invalid flags");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * First step is to temporarily invalidate the MPT entry. This
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * regains ownership from the hardware, and gives us the opportunity
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * to modify the entry. Note: The HW2SW_MPT command returns the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * current MPT entry contents. These are saved away here because
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * they will be reused in a later step below. If the region has
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * bound memory windows that we fail returning an "in use" error code.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Otherwise, this is an unexpected error and we deregister the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * memory region and return error.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * We use TAVOR_CMD_NOSLEEP_SPIN here always because we must protect
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * against holding the lock around this rereg call in all contexts.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor status = tavor_cmn_ownership_cmd_post(state, HW2SW_MPT, &mpt_entry,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor sizeof (tavor_hw_mpt_t), mpt->tr_indx, TAVOR_CMD_NOSLEEP_SPIN);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor cmn_err(CE_CONT, "Tavor: HW2SW_MPT command failed: "
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Call deregister and ensure that all current
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * resources get freed up
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor "memory region");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TNF_PROBE_1(tavor_mr_common_rereg_hw2sw_mpt_cmd_fail,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * If we're changing the protection domain, then validate the new one
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Check for valid PD handle pointer */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Call deregister and ensure that all current
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * resources get properly freed up. Unnecessary
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * here to attempt to regain software ownership
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * of the MPT entry as that has already been
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * done above.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor "memory region");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Set "status" and "errormsg" and goto failure */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TAVOR_TNF_FAIL(IBT_PD_HDL_INVALID, "invalid PD handle");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Use the new PD handle in all operations below */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Use the current PD handle in all operations below */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * If we're changing access permissions, then validate the new ones
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Validate the access flags. Both remote write and remote
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * atomic require the local write flag to be set
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Call deregister and ensure that all current
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * resources get properly freed up. Unnecessary
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * here to attempt to regain software ownership
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * of the MPT entry as that has already been
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * done above.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor "memory region");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Set "status" and "errormsg" and goto failure */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor "invalid access flags");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Setup and validate the memory region access flags. This
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * means translating the IBTF's enable flags into the access
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * flags that will be used in later operations.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * If we're modifying the translation, then figure out whether
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * we can reuse the current MTT resources. This means calling
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * tavor_mr_rereg_xlat_helper() which does most of the heavy lifting
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * for the reregistration. If the current memory region contains
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * sufficient MTT entries for the new regions, then it will be
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * reused and filled in. Otherwise, new entries will be allocated,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the old ones will be freed, and the new entries will be filled
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * in. Note: If we're not modifying the translation, then we
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * should already have all the information we need to update the MPT.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Also note: If tavor_mr_rereg_xlat_helper() fails, it will return
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * a "dereg_level" which is the level of cleanup that needs to be
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * passed to tavor_mr_deregister() to finish the cleanup.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor status = tavor_mr_rereg_xlat_helper(state, mr, bind, op,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Call deregister and ensure that all resources get
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * properly freed up.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor if (tavor_mr_deregister(state, &mr, dereg_level,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor "memory region");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Set "status" and "errormsg" and goto failure */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor mtt_addr_to_use = (((uint64_t)mpt_entry.mttseg_addr_h << 32) |
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Calculate new keys (Lkey, Rkey) from MPT index. Just like they were
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * when the region was first registered, each key is formed from
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * "constrained" bits and "unconstrained" bits. Note: If no remote
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * access is required, then the RKey value is not filled in. Otherwise
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * both Rkey and LKey are given the same value.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor tavor_mr_keycalc(state, mpt->tr_indx, &mr->mr_lkey);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Update the MPT entry with the new information. Some of this
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * information is retained from the previous operation, some of
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * it is new based on request.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor mpt_entry.en_bind = (acc_flags_to_use & IBT_MR_WINDOW_BIND) ? 1 : 0;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor mpt_entry.atomic = (acc_flags_to_use & IBT_MR_REMOTE_ATOMIC) ? 1 : 0;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor mpt_entry.rw = (acc_flags_to_use & IBT_MR_REMOTE_WRITE) ? 1 : 0;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor mpt_entry.rr = (acc_flags_to_use & IBT_MR_REMOTE_READ) ? 1 : 0;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor mpt_entry.lw = (acc_flags_to_use & IBT_MR_LOCAL_WRITE) ? 1 : 0;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor mpt_entry.mttseg_addr_h = mtt_addr_to_use >> 32;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Write the updated MPT entry to hardware
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * We use TAVOR_CMD_NOSLEEP_SPIN here always because we must protect
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * against holding the lock around this rereg call in all contexts.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor status = tavor_cmn_ownership_cmd_post(state, SW2HW_MPT, &mpt_entry,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor sizeof (tavor_hw_mpt_t), mpt->tr_indx, TAVOR_CMD_NOSLEEP_SPIN);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor cmn_err(CE_CONT, "Tavor: SW2HW_MPT command failed: %08x\n",
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Call deregister and ensure that all current resources get
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * properly freed up. Unnecessary here to attempt to regain
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * software ownership of the MPT entry as that has already
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * been done above.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TAVOR_MR_DEREG_NO_HW2SW_MPT, sleep) != DDI_SUCCESS) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TAVOR_WARNING(state, "failed to deregister memory "
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TNF_PROBE_1(tavor_mr_common_rereg_sw2hw_mpt_cmd_fail,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * If we're changing PD, then update their reference counts now.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * This means decrementing the reference count on the old PD and
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * incrementing the reference count on the new PD.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Update the contents of the Tavor Memory Region handle to reflect
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * what has been changed.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* New MR handle is same as the old */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TNF_PROBE_1(tavor_mr_common_rereg_fail, TAVOR_TNF_ERROR, "",
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * tavor_mr_rereg_xlat_helper
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Context: Can be called from interrupt or base context.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Note: This routine expects the "mr_lock" to be held when it
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * is called. Upon returning failure, this routine passes information
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * about what "dereg_level" should be passed to tavor_mr_deregister().
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylortavor_mr_rereg_xlat_helper(tavor_state_t *state, tavor_mrhdl_t mr,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor tavor_bind_info_t *bind, tavor_mr_options_t *op, uint64_t *mtt_addr,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor uint64_t nummtt_needed, nummtt_in_currrsrc, max_sz;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor uint_t mtt_pgsize_bits, bind_type, reuse_dmahdl;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Check the "options" flag. Currently this flag tells the driver
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * whether or not the region should be bound normally (i.e. with
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * entries written into the PCI IOMMU) or whether it should be
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * registered to bypass the IOMMU.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Check for invalid length. Check is the length is zero or if the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * length is larger than the maximum configured value. Return error
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor max_sz = ((uint64_t)1 << state->ts_cfg_profile->cp_log_max_mrw_sz);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor if ((bind->bi_len == 0) || (bind->bi_len > max_sz)) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Deregister will be called upon returning failure from this
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * routine. This will ensure that all current resources get
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * properly freed up. Unnecessary to attempt to regain
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * software ownership of the MPT entry as that has already
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * been done above (in tavor_mr_reregister())
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Set "status" and "errormsg" and goto failure */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TAVOR_TNF_FAIL(IBT_MR_LEN_INVALID, "invalid length");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Determine the number of pages necessary for new region and the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * number of pages supported by the current MTT resources
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor nummtt_needed = tavor_mr_nummtt_needed(state, bind, &mtt_pgsize_bits);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor nummtt_in_currrsrc = mr->mr_mttrsrcp->tr_len >> TAVOR_MTT_SIZE_SHIFT;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Depending on whether we have enough pages or not, the next step is
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * to fill in a set of MTT entries that reflect the new mapping. In
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the first case below, we already have enough entries. This means
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * we need to unbind the memory from the previous mapping, bind the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * memory for the new mapping, write the new MTT entries, and update
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the mr to reflect the changes.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * In the second case below, we do not have enough entries in the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * current mapping. So, in this case, we need not only to unbind the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * current mapping, but we need to free up the MTT resources associated
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * with that mapping. After we've successfully done that, we continue
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * by binding the new memory, allocating new MTT entries, writing the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * new MTT entries, and updating the mr to reflect the changes.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * If this region is being shared (i.e. MTT refcount != 1), then we
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * can't reuse the current MTT resources regardless of their size.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Instead we'll need to alloc new ones (below) just as if there
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * hadn't been enough room in the current entries.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor swrc_old = (tavor_sw_refcnt_t *)mr->mr_mttrefcntp->tr_addr;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Unbind the old mapping for this memory region, but retain
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the ddi_dma_handle_t (if possible) for reuse in the bind
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * operation below. Note: If original memory region was
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * bound for IOMMU bypass and the new region can not use
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * bypass, then a new DMA handle will be necessary.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor if (TAVOR_MR_REUSE_DMAHDL(mr, bind->bi_flags)) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Bind the new memory and determine the mapped addresses.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * As described, this routine and tavor_mr_fast_mtt_write()
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * do the majority of the work for the memory registration
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * operations. Note: When we successfully finish the binding,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * we will set the "bi_free_dmahdl" flag to indicate that
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * even though we may have reused the ddi_dma_handle_t we do
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * wish it to be freed up at some later time. Note also that
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * if we fail, we may need to cleanup the ddi_dma_handle_t.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor status = tavor_mr_mem_bind(state, bind, dmahdl, sleep);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Deregister will be called upon returning failure
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * from this routine. This will ensure that all
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * current resources get properly freed up.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Unnecessary to attempt to regain software ownership
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * of the MPT entry as that has already been done
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * above (in tavor_mr_reregister()). Also unnecessary
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * to attempt to unbind the memory.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor *dereg_level = TAVOR_MR_DEREG_NO_HW2SW_MPT_OR_UNBIND;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Set "status" and "errormsg" and goto failure */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TAVOR_TNF_FAIL(IBT_INSUFF_RESOURCE, "failed mem bind");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Using the new mapping, but reusing the current MTT
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * resources, write the updated entries to MTT
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor status = tavor_mr_fast_mtt_write(mtt, bind, mtt_pgsize_bits);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Deregister will be called upon returning failure
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * from this routine. This will ensure that all
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * current resources get properly freed up.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Unnecessary to attempt to regain software ownership
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * of the MPT entry as that has already been done
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * above (in tavor_mr_reregister()). Also unnecessary
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * to attempt to unbind the memory.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * But we do need to unbind the newly bound memory
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * before returning.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor *dereg_level = TAVOR_MR_DEREG_NO_HW2SW_MPT_OR_UNBIND;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Set "status" and "errormsg" and goto failure */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor "failed write mtt");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Put the updated information into the Mem Region handle */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Check if the memory region MTT is shared by any other MRs.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Since the resource may be shared between multiple memory
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * regions (as a result of a "RegisterSharedMR()" verb) it is
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * important that we not unbind any resources prematurely.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Unbind the old mapping for this memory region, but
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * retain the ddi_dma_handle_t for reuse in the bind
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * operation below. Note: This can only be done here
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * because the region being reregistered is not
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * currently shared. Also if original memory region
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * was bound for IOMMU bypass and the new region can
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * not use bypass, then a new DMA handle will be
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * necessary.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor if (TAVOR_MR_REUSE_DMAHDL(mr, bind->bi_flags)) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Bind the new memory and determine the mapped addresses.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * As described, this routine and tavor_mr_fast_mtt_write()
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * do the majority of the work for the memory registration
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * operations. Note: When we successfully finish the binding,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * we will set the "bi_free_dmahdl" flag to indicate that
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * even though we may have reused the ddi_dma_handle_t we do
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * wish it to be freed up at some later time. Note also that
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * if we fail, we may need to cleanup the ddi_dma_handle_t.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor status = tavor_mr_mem_bind(state, bind, dmahdl, sleep);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Deregister will be called upon returning failure
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * from this routine. This will ensure that all
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * current resources get properly freed up.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Unnecessary to attempt to regain software ownership
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * of the MPT entry as that has already been done
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * above (in tavor_mr_reregister()). Also unnecessary
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * to attempt to unbind the memory.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor *dereg_level = TAVOR_MR_DEREG_NO_HW2SW_MPT_OR_UNBIND;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Set "status" and "errormsg" and goto failure */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TAVOR_TNF_FAIL(IBT_INSUFF_RESOURCE, "failed mem bind");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Allocate the new MTT entries resource
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TAVOR_NUMMTT_TO_MTTSEG(nummtt_needed), sleep, &mtt);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Deregister will be called upon returning failure
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * from this routine. This will ensure that all
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * current resources get properly freed up.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Unnecessary to attempt to regain software ownership
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * of the MPT entry as that has already been done
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * above (in tavor_mr_reregister()). Also unnecessary
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * to attempt to unbind the memory.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * But we do need to unbind the newly bound memory
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * before returning.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor *dereg_level = TAVOR_MR_DEREG_NO_HW2SW_MPT_OR_UNBIND;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Set "status" and "errormsg" and goto failure */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TAVOR_TNF_FAIL(IBT_INSUFF_RESOURCE, "failed MTT");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Allocate MTT reference count (to track shared memory
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * regions). As mentioned elsewhere above, this reference
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * count resource may never be used on the given memory region,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * but if it is ever later registered as a "shared" memory
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * region then this resource will be necessary. Note: This
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * is only necessary here if the existing memory region is
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * already being shared (because otherwise we already have
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * a useable reference count resource).
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor status = tavor_rsrc_alloc(state, TAVOR_REFCNT, 1,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Deregister will be called upon returning
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * failure from this routine. This will ensure
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * that all current resources get properly
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * freed up. Unnecessary to attempt to regain
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * software ownership of the MPT entry as that
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * has already been done above (in
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * tavor_mr_reregister()). Also unnecessary
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * to attempt to unbind the memory.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * But we need to unbind the newly bound
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * memory and free up the newly allocated MTT
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * entries before returning.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Set "status"/"errormsg", goto failure */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor "failed reference count");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor swrc_new = (tavor_sw_refcnt_t *)mtt_refcnt->tr_addr;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*swrc_new))
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Using the new mapping and the new MTT resources, write the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * updated entries to MTT
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor status = tavor_mr_fast_mtt_write(mtt, bind, mtt_pgsize_bits);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Deregister will be called upon returning failure
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * from this routine. This will ensure that all
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * current resources get properly freed up.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Unnecessary to attempt to regain software ownership
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * of the MPT entry as that has already been done
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * above (in tavor_mr_reregister()). Also unnecessary
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * to attempt to unbind the memory.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * But we need to unbind the newly bound memory,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * free up the newly allocated MTT entries, and
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * (possibly) free the new MTT reference count
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * resource before returning.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor *dereg_level = TAVOR_MR_DEREG_NO_HW2SW_MPT_OR_UNBIND;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Set "status" and "errormsg" and goto failure */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TAVOR_TNF_FAIL(IBT_INSUFF_RESOURCE, "failed write mtt");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Check if the memory region MTT is shared by any other MRs.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Since the resource may be shared between multiple memory
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * regions (as a result of a "RegisterSharedMR()" verb) it is
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * important that we not free up any resources prematurely.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Decrement MTT reference count for "old" region */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Free up the old MTT entries resource */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Put the updated information into the mrhdl */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Calculate and return the updated MTT address (in the DDR address
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * space). This will be used by the caller (tavor_mr_reregister) in
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the updated MPT entry
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor mtt_ddrbaseaddr = (uint64_t)(uintptr_t)rsrc_pool->rsrc_ddr_offset;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TNF_PROBE_1(tavor_mr_rereg_xlat_helper_fail, TAVOR_TNF_ERROR, "",
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * tavor_mr_nummtt_needed()
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Context: Can be called from interrupt or base context.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor/* ARGSUSED */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylortavor_mr_nummtt_needed(tavor_state_t *state, tavor_bind_info_t *bind,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * For now we specify the page size as 8Kb (the default page size for
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the sun4u architecture), or 4Kb for x86. Figure out optimal page
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * size by examining the dmacookies XXX
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor pg_offset_mask = ((uint64_t)1 << *mtt_pgsize_bits) - 1;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * tavor_mr_mem_bind()
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Context: Can be called from interrupt or base context.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylortavor_mr_mem_bind(tavor_state_t *state, tavor_bind_info_t *bind,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* bi_type must be set to a meaningful value to get a bind handle */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Set the callback flag appropriately */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor callback = (sleep == TAVOR_SLEEP) ? DDI_DMA_SLEEP : DDI_DMA_DONTWAIT;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Determine whether to map STREAMING or CONSISTENT */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor dma_xfer_mode = (bind->bi_flags & IBT_MR_NONCOHERENT) ?
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Initialize many of the default DMA attributes. Then, if we're
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * bypassing the IOMMU, set the DDI_DMA_FORCE_PHYSICAL flag.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * First, disable streaming and switch to consistent if
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * configured to do so and IOMMU BYPASS is enabled.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor if (state->ts_cfg_profile->cp_disable_streaming_on_bypass &&
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Then, if streaming is still specified, then "bypass" is not
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor dma_attr.dma_attr_flags = DDI_DMA_FORCE_PHYSICAL;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Allocate a DMA handle for the binding */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor status = ddi_dma_alloc_handle(state->ts_dip, &dma_attr,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Bind the memory to get the PCI mapped addresses. The decision
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * to call ddi_dma_addr_bind_handle() or ddi_dma_buf_bind_handle()
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * is determined by the "bi_type" flag. Note: if the bind operation
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * fails then we have to free up the DMA handle and return error.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor status = ddi_dma_addr_bind_handle(bind->bi_dmahdl, NULL,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor (caddr_t)(uintptr_t)bind->bi_addr, bind->bi_len,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor } else { /* TAVOR_BINDHDL_BUF || TAVOR_BINDHDL_UBUF */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor status = ddi_dma_buf_bind_handle(bind->bi_dmahdl,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor bind->bi_buf, (DDI_DMA_RDWR | dma_xfer_mode), callback,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor NULL, &bind->bi_dmacookie, &bind->bi_cookiecnt);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TNF_PROBE_0(tavor_mr_mem_bind_dmabind_fail, TAVOR_TNF_ERROR,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * tavor_mr_mem_unbind()
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Context: Can be called from interrupt or base context.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylortavor_mr_mem_unbind(tavor_state_t *state, tavor_bind_info_t *bind)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * In case of TAVOR_BINDHDL_UBUF, the memory bi_buf points to
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * is actually allocated by ddi_umem_iosetup() internally, then
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * it's required to free it here. Reset bi_type to TAVOR_BINDHDL_NONE
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * not to free it again later.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Unbind the DMA memory for the region
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Note: The only way ddi_dma_unbind_handle() currently
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * can return an error is if the handle passed in is invalid.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Since this should never happen, we choose to return void
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * from this function! If this does return an error, however,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * then we print a warning message to the console.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor status = ddi_dma_unbind_handle(bind->bi_dmahdl);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TAVOR_WARNING(state, "failed to unbind DMA mapping");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Free up the DMA handle */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * tavor_mr_fast_mtt_write()
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Context: Can be called from interrupt or base context.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylortavor_mr_fast_mtt_write(tavor_rsrc_t *mtt, tavor_bind_info_t *bind,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Calculate page size from the suggested value passed in */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Walk the "cookie list" and fill in the MTT table entries
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor while (cookie_cnt-- > 0) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Fill in the mapped addresses (calculated above) and
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * set TAVOR_MTT_ENTRY_PRESET flag for each MTT entry.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ddi_put64(mtt->tr_acchdl, &mtt_table[i], mtt_entry);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor "dma_cookie address from caller\n");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * When we've reached the end of the current DMA cookie,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * jump to the next cookie (if there are more)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ddi_dma_nextcookie(bind->bi_dmahdl, &dmacookie);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * tavor_mtt_refcnt_inc()
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Context: Can be called from interrupt or base context.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Increment the MTT's reference count */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor TNF_PROBE_1_DEBUG(tavor_mtt_refcnt_inc, TAVOR_TNF_TRACE, "",
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * tavor_mtt_refcnt_dec()
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Context: Can be called from interrupt or base context.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Decrement the MTT's reference count */