/*
* CDDL HEADER START
*
* The contents of this file are subject to the terms of the
* Common Development and Distribution License (the "License").
* You may not use this file except in compliance with the License.
*
* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
* or http://www.opensolaris.org/os/licensing.
* See the License for the specific language governing permissions
* and limitations under the License.
*
* When distributing Covered Code, include this CDDL HEADER in each
* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
* If applicable, add the following below this CDDL HEADER, with the
* fields enclosed by brackets "[]" replaced with your own identifying
* information: Portions Copyright [yyyy] [name of copyright owner]
*
* CDDL HEADER END
*/
/*
* Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved.
*/
/*
* hermon_misc.c
* Hermon Miscellaneous routines - Address Handle, Multicast, Protection
* Domain, and port-related operations
*
* Implements all the routines necessary for allocating, freeing, querying
* and modifying Address Handles and Protection Domains. Also implements
* all the routines necessary for adding and removing Queue Pairs to/from
* Multicast Groups. Lastly, it implements the routines necessary for
* port-related query and modify operations.
*/
#include <sys/types.h>
#include <sys/conf.h>
#include <sys/ddi.h>
#include <sys/sunddi.h>
#include <sys/modctl.h>
#include <sys/bitmap.h>
#include <sys/sysmacros.h>
#include <sys/ib/adapters/hermon/hermon.h>
extern int hermon_rdma_debug;
int hermon_fmr_verbose = 0;
static int hermon_mcg_qplist_add(hermon_state_t *state, hermon_mcghdl_t mcg,
hermon_hw_mcg_qp_list_t *mcg_qplist, hermon_qphdl_t qp, uint_t *qp_found);
static int hermon_mcg_qplist_remove(hermon_mcghdl_t mcg,
hermon_hw_mcg_qp_list_t *mcg_qplist, hermon_qphdl_t qp);
static void hermon_qp_mcg_refcnt_inc(hermon_qphdl_t qp);
static void hermon_qp_mcg_refcnt_dec(hermon_qphdl_t qp);
static uint_t hermon_mcg_walk_mgid_hash(hermon_state_t *state,
uint64_t start_indx, ib_gid_t mgid, uint_t *prev_indx);
static void hermon_mcg_setup_new_hdr(hermon_mcghdl_t mcg,
hermon_hw_mcg_t *mcg_hdr, ib_gid_t mgid, hermon_rsrc_t *mcg_rsrc);
static int hermon_mcg_hash_list_remove(hermon_state_t *state, uint_t curr_indx,
uint_t prev_indx, hermon_hw_mcg_t *mcg_entry);
static int hermon_mcg_entry_invalidate(hermon_state_t *state,
hermon_hw_mcg_t *mcg_entry, uint_t indx);
static int hermon_mgid_is_valid(ib_gid_t gid);
static int hermon_mlid_is_valid(ib_lid_t lid);
static void hermon_fmr_cleanup(hermon_fmrhdl_t pool);
#define HERMON_MAX_DBR_PAGES_PER_USER 64
#define HERMON_DBR_KEY(index, page) \
(((uint64_t)index) * HERMON_MAX_DBR_PAGES_PER_USER + (page))
static hermon_udbr_page_t *
hermon_dbr_new_user_page(hermon_state_t *state, uint_t index,
uint_t page)
{
hermon_udbr_page_t *pagep;
ddi_dma_attr_t dma_attr;
uint_t cookiecnt;
int status;
hermon_umap_db_entry_t *umapdb;
ulong_t pagesize = PAGESIZE;
pagep = kmem_alloc(sizeof (*pagep), KM_SLEEP);
pagep->upg_index = page;
pagep->upg_nfree = pagesize / sizeof (hermon_dbr_t);
/* Allocate 1 bit per dbr for free/alloc management (0 => "free") */
pagep->upg_free = kmem_zalloc(pagesize / sizeof (hermon_dbr_t) / 8,
KM_SLEEP);
pagep->upg_kvaddr = ddi_umem_alloc(pagesize, DDI_UMEM_SLEEP,
&pagep->upg_umemcookie); /* not HERMON_PAGESIZE here */
pagep->upg_buf = ddi_umem_iosetup(pagep->upg_umemcookie, 0,
pagesize, B_WRITE, 0, 0, NULL, DDI_UMEM_SLEEP);
hermon_dma_attr_init(state, &dma_attr);
#ifdef __sparc
if (state->hs_cfg_profile->cp_iommu_bypass == HERMON_BINDMEM_BYPASS)
dma_attr.dma_attr_flags = DDI_DMA_FORCE_PHYSICAL;
#endif
status = ddi_dma_alloc_handle(state->hs_dip, &dma_attr,
DDI_DMA_SLEEP, NULL, &pagep->upg_dmahdl);
if (status != DDI_SUCCESS) {
IBTF_DPRINTF_L2("hermon", "hermon_new_user_page: "
"ddi_dma_buf_bind_handle failed: %d", status);
return (NULL);
}
status = ddi_dma_buf_bind_handle(pagep->upg_dmahdl,
pagep->upg_buf, DDI_DMA_RDWR | DDI_DMA_CONSISTENT,
DDI_DMA_SLEEP, NULL, &pagep->upg_dmacookie, &cookiecnt);
if (status != DDI_SUCCESS) {
IBTF_DPRINTF_L2("hermon", "hermon_dbr_new_user_page: "
"ddi_dma_buf_bind_handle failed: %d", status);
ddi_dma_free_handle(&pagep->upg_dmahdl);
return (NULL);
}
ASSERT(cookiecnt == 1);
/* create db entry for mmap */
umapdb = hermon_umap_db_alloc(state->hs_instance,
HERMON_DBR_KEY(index, page), MLNX_UMAP_DBRMEM_RSRC,
(uint64_t)(uintptr_t)pagep);
hermon_umap_db_add(umapdb);
return (pagep);
}
/*ARGSUSED*/
static int
hermon_user_dbr_alloc(hermon_state_t *state, uint_t index,
ddi_acc_handle_t *acchdl, hermon_dbr_t **vdbr, uint64_t *pdbr,
uint64_t *mapoffset)
{
hermon_user_dbr_t *udbr;
hermon_udbr_page_t *pagep;
uint_t next_page;
int dbr_index;
int i1, i2, i3, last;
uint64_t u64, mask;
mutex_enter(&state->hs_dbr_lock);
for (udbr = state->hs_user_dbr; udbr != NULL; udbr = udbr->udbr_link)
if (udbr->udbr_index == index)
break;
if (udbr == NULL) {
udbr = kmem_alloc(sizeof (*udbr), KM_SLEEP);
udbr->udbr_link = state->hs_user_dbr;
state->hs_user_dbr = udbr;
udbr->udbr_index = index;
udbr->udbr_pagep = NULL;
}
pagep = udbr->udbr_pagep;
next_page = (pagep == NULL) ? 0 : (pagep->upg_index + 1);
while (pagep != NULL)
if (pagep->upg_nfree > 0)
break;
else
pagep = pagep->upg_link;
if (pagep == NULL) {
pagep = hermon_dbr_new_user_page(state, index, next_page);
if (pagep == NULL) {
mutex_exit(&state->hs_dbr_lock);
return (DDI_FAILURE);
}
pagep->upg_link = udbr->udbr_pagep;
udbr->udbr_pagep = pagep;
}
/* Since nfree > 0, we're assured the loops below will succeed */
/* First, find a 64-bit (not ~0) that has a free dbr */
last = PAGESIZE / sizeof (uint64_t) / 64;
mask = ~0ull;
for (i1 = 0; i1 < last; i1++)
if ((pagep->upg_free[i1] & mask) != mask)
break;
u64 = pagep->upg_free[i1];
/* Second, find a byte (not 0xff) that has a free dbr */
last = sizeof (uint64_t) / sizeof (uint8_t);
for (i2 = 0, mask = 0xff; i2 < last; i2++, mask <<= 8)
if ((u64 & mask) != mask)
break;
/* Third, find a bit that is free (0) */
for (i3 = 0; i3 < sizeof (uint64_t) / sizeof (uint8_t); i3++)
if ((u64 & (1ul << (i3 + 8 * i2))) == 0)
break;
/* Mark it as allocated */
pagep->upg_free[i1] |= (1ul << (i3 + 8 * i2));
dbr_index = ((i1 * sizeof (uint64_t)) + i2) * sizeof (uint64_t) + i3;
pagep->upg_nfree--;
((uint64_t *)(void *)pagep->upg_kvaddr)[dbr_index] = 0; /* clear dbr */
*mapoffset = ((HERMON_DBR_KEY(index, pagep->upg_index) <<
MLNX_UMAP_RSRC_TYPE_SHIFT) | MLNX_UMAP_DBRMEM_RSRC) << PAGESHIFT;
*vdbr = (hermon_dbr_t *)((uint64_t *)(void *)pagep->upg_kvaddr +
dbr_index);
*pdbr = pagep->upg_dmacookie.dmac_laddress + dbr_index *
sizeof (uint64_t);
mutex_exit(&state->hs_dbr_lock);
return (DDI_SUCCESS);
}
static void
hermon_user_dbr_free(hermon_state_t *state, uint_t index, hermon_dbr_t *record)
{
hermon_user_dbr_t *udbr;
hermon_udbr_page_t *pagep;
caddr_t kvaddr;
uint_t dbr_index;
uint_t max_free = PAGESIZE / sizeof (hermon_dbr_t);
int i1, i2;
dbr_index = (uintptr_t)record & PAGEOFFSET; /* offset (not yet index) */
kvaddr = (caddr_t)record - dbr_index;
dbr_index /= sizeof (hermon_dbr_t); /* now it's the index */
mutex_enter(&state->hs_dbr_lock);
for (udbr = state->hs_user_dbr; udbr != NULL; udbr = udbr->udbr_link)
if (udbr->udbr_index == index)
break;
if (udbr == NULL) {
IBTF_DPRINTF_L2("hermon", "free user dbr: udbr struct not "
"found for index %x", index);
mutex_exit(&state->hs_dbr_lock);
return;
}
for (pagep = udbr->udbr_pagep; pagep != NULL; pagep = pagep->upg_link)
if (pagep->upg_kvaddr == kvaddr)
break;
if (pagep == NULL) {
IBTF_DPRINTF_L2("hermon", "free user dbr: pagep struct not"
" found for index %x, kvaddr %p, DBR index %x",
index, kvaddr, dbr_index);
mutex_exit(&state->hs_dbr_lock);
return;
}
if (pagep->upg_nfree >= max_free) {
IBTF_DPRINTF_L2("hermon", "free user dbr: overflow: "
"UCE index %x, DBR index %x", index, dbr_index);
mutex_exit(&state->hs_dbr_lock);
return;
}
ASSERT(dbr_index < max_free);
i1 = dbr_index / 64;
i2 = dbr_index % 64;
ASSERT((pagep->upg_free[i1] & (1ul << i2)) == (1ul << i2));
pagep->upg_free[i1] &= ~(1ul << i2);
pagep->upg_nfree++;
mutex_exit(&state->hs_dbr_lock);
}
/*
* hermon_dbr_page_alloc()
* first page allocation - called from attach or open
* in this case, we want exactly one page per call, and aligned on a
* page - and may need to be mapped to the user for access
*/
int
hermon_dbr_page_alloc(hermon_state_t *state, hermon_dbr_info_t **dinfo)
{
int status;
ddi_dma_handle_t dma_hdl;
ddi_acc_handle_t acc_hdl;
ddi_dma_attr_t dma_attr;
ddi_dma_cookie_t cookie;
uint_t cookie_cnt;
int i;
hermon_dbr_info_t *info;
caddr_t dmaaddr;
uint64_t dmalen;
ulong_t pagesize = PAGESIZE;
info = kmem_zalloc(sizeof (hermon_dbr_info_t), KM_SLEEP);
/*
* Initialize many of the default DMA attributes. Then set additional
* alignment restrictions if necessary for the dbr memory, meaning
* page aligned. Also use the configured value for IOMMU bypass
*/
hermon_dma_attr_init(state, &dma_attr);
dma_attr.dma_attr_align = pagesize;
dma_attr.dma_attr_sgllen = 1; /* make sure only one cookie */
#ifdef __sparc
if (state->hs_cfg_profile->cp_iommu_bypass == HERMON_BINDMEM_BYPASS)
dma_attr.dma_attr_flags = DDI_DMA_FORCE_PHYSICAL;
#endif
status = ddi_dma_alloc_handle(state->hs_dip, &dma_attr,
DDI_DMA_SLEEP, NULL, &dma_hdl);
if (status != DDI_SUCCESS) {
kmem_free((void *)info, sizeof (hermon_dbr_info_t));
cmn_err(CE_NOTE, "dbr DMA handle alloc failed\n");
return (DDI_FAILURE);
}
status = ddi_dma_mem_alloc(dma_hdl, pagesize,
&state->hs_reg_accattr, DDI_DMA_CONSISTENT, DDI_DMA_SLEEP,
NULL, &dmaaddr, (size_t *)&dmalen, &acc_hdl);
if (status != DDI_SUCCESS) {
ddi_dma_free_handle(&dma_hdl);
cmn_err(CE_CONT, "dbr DMA mem alloc failed(status %d)", status);
kmem_free((void *)info, sizeof (hermon_dbr_info_t));
return (DDI_FAILURE);
}
/* this memory won't be IB registered, so do the bind here */
status = ddi_dma_addr_bind_handle(dma_hdl, NULL,
dmaaddr, (size_t)dmalen, DDI_DMA_RDWR |
DDI_DMA_CONSISTENT, DDI_DMA_SLEEP, NULL, &cookie, &cookie_cnt);
if (status != DDI_SUCCESS) {
ddi_dma_mem_free(&acc_hdl);
ddi_dma_free_handle(&dma_hdl);
kmem_free((void *)info, sizeof (hermon_dbr_info_t));
cmn_err(CE_CONT, "dbr DMA bind handle failed (status %d)",
status);
return (DDI_FAILURE);
}
*dinfo = info; /* Pass back the pointer */
/* init the info structure with returned info */
info->dbr_dmahdl = dma_hdl;
info->dbr_acchdl = acc_hdl;
info->dbr_page = (hermon_dbr_t *)(void *)dmaaddr;
info->dbr_link = NULL;
/* extract the phys addr from the cookie */
info->dbr_paddr = cookie.dmac_laddress;
info->dbr_firstfree = 0;
info->dbr_nfree = HERMON_NUM_DBR_PER_PAGE;
/* link all DBrs onto the free list */
for (i = 0; i < HERMON_NUM_DBR_PER_PAGE; i++) {
info->dbr_page[i] = i + 1;
}
return (DDI_SUCCESS);
}
/*
* hermon_dbr_alloc()
* DBr record allocation - called from alloc cq/qp/srq
* will check for available dbrs in current
* page - if needed it will allocate another and link them
*/
int
hermon_dbr_alloc(hermon_state_t *state, uint_t index, ddi_acc_handle_t *acchdl,
hermon_dbr_t **vdbr, uint64_t *pdbr, uint64_t *mapoffset)
{
hermon_dbr_t *record = NULL;
hermon_dbr_info_t *info = NULL;
uint32_t idx;
int status;
if (index != state->hs_kernel_uar_index)
return (hermon_user_dbr_alloc(state, index, acchdl, vdbr, pdbr,
mapoffset));
mutex_enter(&state->hs_dbr_lock);
for (info = state->hs_kern_dbr; info != NULL; info = info->dbr_link)
if (info->dbr_nfree != 0)
break; /* found a page w/ one available */
if (info == NULL) { /* did NOT find a page with one available */
status = hermon_dbr_page_alloc(state, &info);
if (status != DDI_SUCCESS) {
/* do error handling */
mutex_exit(&state->hs_dbr_lock);
return (DDI_FAILURE);
}
/* got a new page, so link it in. */
info->dbr_link = state->hs_kern_dbr;
state->hs_kern_dbr = info;
}
idx = info->dbr_firstfree;
record = info->dbr_page + idx;
info->dbr_firstfree = *record;
info->dbr_nfree--;
*record = 0;
*acchdl = info->dbr_acchdl;
*vdbr = record;
*pdbr = info->dbr_paddr + idx * sizeof (hermon_dbr_t);
mutex_exit(&state->hs_dbr_lock);
return (DDI_SUCCESS);
}
/*
* hermon_dbr_free()
* DBr record deallocation - called from free cq/qp
* will update the counter in the header, and invalidate
* the dbr, but will NEVER free pages of dbrs - small
* price to pay, but userland access never will anyway
*/
void
hermon_dbr_free(hermon_state_t *state, uint_t indx, hermon_dbr_t *record)
{
hermon_dbr_t *page;
hermon_dbr_info_t *info;
if (indx != state->hs_kernel_uar_index) {
hermon_user_dbr_free(state, indx, record);
return;
}
page = (hermon_dbr_t *)(uintptr_t)((uintptr_t)record & PAGEMASK);
mutex_enter(&state->hs_dbr_lock);
for (info = state->hs_kern_dbr; info != NULL; info = info->dbr_link)
if (info->dbr_page == page)
break;
ASSERT(info != NULL);
*record = info->dbr_firstfree;
info->dbr_firstfree = record - info->dbr_page;
info->dbr_nfree++;
mutex_exit(&state->hs_dbr_lock);
}
/*
* hermon_dbr_kern_free()
* Context: Can be called only from detach context.
*
* Free all kernel dbr pages. This includes the freeing of all the dma
* resources acquired during the allocation of the pages.
*
* Also, free all the user dbr pages.
*/
void
hermon_dbr_kern_free(hermon_state_t *state)
{
hermon_dbr_info_t *info, *link;
hermon_user_dbr_t *udbr, *next;
hermon_udbr_page_t *pagep, *nextp;
hermon_umap_db_entry_t *umapdb;
int instance, status;
uint64_t value;
extern hermon_umap_db_t hermon_userland_rsrc_db;
mutex_enter(&state->hs_dbr_lock);
for (info = state->hs_kern_dbr; info != NULL; info = link) {
(void) ddi_dma_unbind_handle(info->dbr_dmahdl);
ddi_dma_mem_free(&info->dbr_acchdl); /* free page */
ddi_dma_free_handle(&info->dbr_dmahdl);
link = info->dbr_link;
kmem_free(info, sizeof (hermon_dbr_info_t));
}
udbr = state->hs_user_dbr;
instance = state->hs_instance;
mutex_enter(&hermon_userland_rsrc_db.hdl_umapdb_lock);
while (udbr != NULL) {
pagep = udbr->udbr_pagep;
while (pagep != NULL) {
/* probably need to remove "db" */
(void) ddi_dma_unbind_handle(pagep->upg_dmahdl);
ddi_dma_free_handle(&pagep->upg_dmahdl);
freerbuf(pagep->upg_buf);
ddi_umem_free(pagep->upg_umemcookie);
status = hermon_umap_db_find_nolock(instance,
HERMON_DBR_KEY(udbr->udbr_index,
pagep->upg_index), MLNX_UMAP_DBRMEM_RSRC,
&value, HERMON_UMAP_DB_REMOVE, &umapdb);
if (status == DDI_SUCCESS)
hermon_umap_db_free(umapdb);
kmem_free(pagep->upg_free,
PAGESIZE / sizeof (hermon_dbr_t) / 8);
nextp = pagep->upg_link;
kmem_free(pagep, sizeof (*pagep));
pagep = nextp;
}
next = udbr->udbr_link;
kmem_free(udbr, sizeof (*udbr));
udbr = next;
}
mutex_exit(&hermon_userland_rsrc_db.hdl_umapdb_lock);
mutex_exit(&state->hs_dbr_lock);
}
/*
* hermon_ah_alloc()
* Context: Can be called only from user or kernel context.
*/
int
hermon_ah_alloc(hermon_state_t *state, hermon_pdhdl_t pd,
ibt_adds_vect_t *attr_p, hermon_ahhdl_t *ahhdl, uint_t sleepflag)
{
hermon_rsrc_t *rsrc;
hermon_hw_udav_t *udav;
hermon_ahhdl_t ah;
int status;
/*
* Someday maybe the "ibt_adds_vect_t *attr_p" will be NULL to
* indicate that we wish to allocate an "invalid" (i.e. empty)
* address handle XXX
*/
/* Validate that specified port number is legal */
if (!hermon_portnum_is_valid(state, attr_p->av_port_num)) {
return (IBT_HCA_PORT_INVALID);
}
/*
* Allocate the software structure for tracking the address handle
* (i.e. the Hermon Address Handle struct).
*/
status = hermon_rsrc_alloc(state, HERMON_AHHDL, 1, sleepflag, &rsrc);
if (status != DDI_SUCCESS) {
return (IBT_INSUFF_RESOURCE);
}
ah = (hermon_ahhdl_t)rsrc->hr_addr;
_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*ah))
/* Increment the reference count on the protection domain (PD) */
hermon_pd_refcnt_inc(pd);
udav = (hermon_hw_udav_t *)kmem_zalloc(sizeof (hermon_hw_udav_t),
KM_SLEEP);
_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*udav))
/*
* Fill in the UDAV data. We first zero out the UDAV, then populate
* it by then calling hermon_set_addr_path() to fill in the common
* portions that can be pulled from the "ibt_adds_vect_t" passed in
*/
status = hermon_set_addr_path(state, attr_p,
(hermon_hw_addr_path_t *)udav, HERMON_ADDRPATH_UDAV);
if (status != DDI_SUCCESS) {
hermon_pd_refcnt_dec(pd);
hermon_rsrc_free(state, &rsrc);
return (status);
}
udav->pd = pd->pd_pdnum;
udav->sl = attr_p->av_srvl;
/*
* Fill in the rest of the Hermon Address Handle struct.
*
* NOTE: We are saving away a copy of the "av_dgid.gid_guid" field
* here because we may need to return it later to the IBTF (as a
* result of a subsequent query operation). Unlike the other UDAV
* parameters, the value of "av_dgid.gid_guid" is not always preserved.
* The reason for this is described in hermon_set_addr_path().
*/
ah->ah_rsrcp = rsrc;
ah->ah_pdhdl = pd;
ah->ah_udav = udav;
ah->ah_save_guid = attr_p->av_dgid.gid_guid;
*ahhdl = ah;
return (DDI_SUCCESS);
}
/*
* hermon_ah_free()
* Context: Can be called only from user or kernel context.
*/
/* ARGSUSED */
int
hermon_ah_free(hermon_state_t *state, hermon_ahhdl_t *ahhdl, uint_t sleepflag)
{
hermon_rsrc_t *rsrc;
hermon_pdhdl_t pd;
hermon_ahhdl_t ah;
/*
* Pull all the necessary information from the Hermon Address Handle
* struct. This is necessary here because the resource for the
* AH is going to be freed up as part of this operation.
*/
ah = *ahhdl;
mutex_enter(&ah->ah_lock);
rsrc = ah->ah_rsrcp;
pd = ah->ah_pdhdl;
mutex_exit(&ah->ah_lock);
_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*ah))
/* Free the UDAV memory */
kmem_free(ah->ah_udav, sizeof (hermon_hw_udav_t));
/* Decrement the reference count on the protection domain (PD) */
hermon_pd_refcnt_dec(pd);
/* Free the Hermon Address Handle structure */
hermon_rsrc_free(state, &rsrc);
/* Set the ahhdl pointer to NULL and return success */
*ahhdl = NULL;
return (DDI_SUCCESS);
}
/*
* hermon_ah_query()
* Context: Can be called from interrupt or base context.
*/
/* ARGSUSED */
int
hermon_ah_query(hermon_state_t *state, hermon_ahhdl_t ah, hermon_pdhdl_t *pd,
ibt_adds_vect_t *attr_p)
{
mutex_enter(&ah->ah_lock);
_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*attr_p))
/*
* Pull the PD and UDAV from the Hermon Address Handle structure
*/
*pd = ah->ah_pdhdl;
/*
* Fill in "ibt_adds_vect_t". We call hermon_get_addr_path() to fill
* the common portions that can be pulled from the UDAV we pass in.
*
* NOTE: We will also fill the "av_dgid.gid_guid" field from the
* "ah_save_guid" field we have previously saved away. The reason
* for this is described in hermon_ah_alloc() and hermon_ah_modify().
*/
hermon_get_addr_path(state, (hermon_hw_addr_path_t *)ah->ah_udav,
attr_p, HERMON_ADDRPATH_UDAV);
attr_p->av_dgid.gid_guid = ah->ah_save_guid;
mutex_exit(&ah->ah_lock);
return (DDI_SUCCESS);
}
/*
* hermon_ah_modify()
* Context: Can be called from interrupt or base context.
*/
/* ARGSUSED */
int
hermon_ah_modify(hermon_state_t *state, hermon_ahhdl_t ah,
ibt_adds_vect_t *attr_p)
{
hermon_hw_udav_t old_udav;
uint64_t data_old;
int status, size, i;
/* Validate that specified port number is legal */
if (!hermon_portnum_is_valid(state, attr_p->av_port_num)) {
return (IBT_HCA_PORT_INVALID);
}
mutex_enter(&ah->ah_lock);
/* Save a copy of the current UDAV data in old_udav. */
bcopy(ah->ah_udav, &old_udav, sizeof (hermon_hw_udav_t));
/*
* Fill in the new UDAV with the caller's data, passed in via the
* "ibt_adds_vect_t" structure.
*
* NOTE: We also need to save away a copy of the "av_dgid.gid_guid"
* field here (just as we did during hermon_ah_alloc()) because we
* may need to return it later to the IBTF (as a result of a
* subsequent query operation). As explained in hermon_ah_alloc(),
* unlike the other UDAV parameters, the value of "av_dgid.gid_guid"
* is not always preserved. The reason for this is described in
* hermon_set_addr_path().
*/
status = hermon_set_addr_path(state, attr_p,
(hermon_hw_addr_path_t *)ah->ah_udav, HERMON_ADDRPATH_UDAV);
if (status != DDI_SUCCESS) {
mutex_exit(&ah->ah_lock);
return (status);
}
ah->ah_save_guid = attr_p->av_dgid.gid_guid;
_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*(ah->ah_udav)))
ah->ah_udav->sl = attr_p->av_srvl;
/*
* Copy changes into the new UDAV.
* Note: We copy in 64-bit chunks. For the first two of these
* chunks it is necessary to read the current contents of the
* UDAV, mask off the modifiable portions (maintaining any
* of the "reserved" portions), and then mask on the new data.
*/
size = sizeof (hermon_hw_udav_t) >> 3;
for (i = 0; i < size; i++) {
data_old = ((uint64_t *)&old_udav)[i];
/*
* Apply mask to change only the relevant values.
*/
if (i == 0) {
data_old = data_old & HERMON_UDAV_MODIFY_MASK0;
} else if (i == 1) {
data_old = data_old & HERMON_UDAV_MODIFY_MASK1;
} else {
data_old = 0;
}
/* Store the updated values to the UDAV */
((uint64_t *)ah->ah_udav)[i] |= data_old;
}
/*
* Put the valid PD number back into the UDAV entry, as it
* might have been clobbered above.
*/
ah->ah_udav->pd = old_udav.pd;
mutex_exit(&ah->ah_lock);
return (DDI_SUCCESS);
}
/*
* hermon_mcg_attach()
* Context: Can be called only from user or kernel context.
*/
int
hermon_mcg_attach(hermon_state_t *state, hermon_qphdl_t qp, ib_gid_t gid,
ib_lid_t lid)
{
hermon_rsrc_t *rsrc;
hermon_hw_mcg_t *mcg_entry;
hermon_hw_mcg_qp_list_t *mcg_entry_qplist;
hermon_mcghdl_t mcg, newmcg;
uint64_t mgid_hash;
uint32_t end_indx;
int status;
uint_t qp_found;
/*
* It is only allowed to attach MCG to UD queue pairs. Verify
* that the intended QP is of the appropriate transport type
*/
if (qp->qp_serv_type != HERMON_QP_UD) {
return (IBT_QP_SRV_TYPE_INVALID);
}
/*
* Check for invalid Multicast DLID. Specifically, all Multicast
* LIDs should be within a well defined range. If the specified LID
* is outside of that range, then return an error.
*/
if (hermon_mlid_is_valid(lid) == 0) {
return (IBT_MC_MLID_INVALID);
}
/*
* Check for invalid Multicast GID. All Multicast GIDs should have
* a well-defined pattern of bits and flags that are allowable. If
* the specified GID does not meet the criteria, then return an error.
*/
if (hermon_mgid_is_valid(gid) == 0) {
return (IBT_MC_MGID_INVALID);
}
/*
* Compute the MGID hash value. Since the MCG table is arranged as
* a number of separate hash chains, this operation converts the
* specified MGID into the starting index of an entry in the hash
* table (i.e. the index for the start of the appropriate hash chain).
* Subsequent operations below will walk the chain searching for the
* right place to add this new QP.
*/
status = hermon_mgid_hash_cmd_post(state, gid.gid_prefix, gid.gid_guid,
&mgid_hash, HERMON_SLEEPFLAG_FOR_CONTEXT());
if (status != HERMON_CMD_SUCCESS) {
cmn_err(CE_CONT, "Hermon: MGID_HASH command failed: %08x\n",
status);
if (status == HERMON_CMD_INVALID_STATUS) {
hermon_fm_ereport(state, HCA_SYS_ERR, HCA_ERR_SRV_LOST);
}
return (ibc_get_ci_failure(0));
}
/*
* Grab the multicast group mutex. Then grab the pre-allocated
* temporary buffer used for holding and/or modifying MCG entries.
* Zero out the temporary MCG entry before we begin.
*/
mutex_enter(&state->hs_mcglock);
mcg_entry = state->hs_mcgtmp;
mcg_entry_qplist = HERMON_MCG_GET_QPLIST_PTR(mcg_entry);
bzero(mcg_entry, HERMON_MCGMEM_SZ(state));
/*
* Walk through the array of MCG entries starting at "mgid_hash".
* Try to find the appropriate place for this new QP to be added.
* This could happen when the first entry of the chain has MGID == 0
* (which means that the hash chain is empty), or because we find
* an entry with the same MGID (in which case we'll add the QP to
* that MCG), or because we come to the end of the chain (in which
* case this is the first QP being added to the multicast group that
* corresponds to the MGID. The hermon_mcg_walk_mgid_hash() routine
* walks the list and returns an index into the MCG table. The entry
* at this index is then checked to determine which case we have
* fallen into (see below). Note: We are using the "shadow" MCG
* list (of hermon_mcg_t structs) for this lookup because the real
* MCG entries are in hardware (and the lookup process would be much
* more time consuming).
*/
end_indx = hermon_mcg_walk_mgid_hash(state, mgid_hash, gid, NULL);
mcg = &state->hs_mcghdl[end_indx];
/*
* If MGID == 0, then the hash chain is empty. Just fill in the
* current entry. Note: No need to allocate an MCG table entry
* as all the hash chain "heads" are already preallocated.
*/
if ((mcg->mcg_mgid_h == 0) && (mcg->mcg_mgid_l == 0)) {
/* Fill in the current entry in the "shadow" MCG list */
hermon_mcg_setup_new_hdr(mcg, mcg_entry, gid, NULL);
/*
* Try to add the new QP number to the list. This (and the
* above) routine fills in a temporary MCG. The "mcg_entry"
* and "mcg_entry_qplist" pointers simply point to different
* offsets within the same temporary copy of the MCG (for
* convenience). Note: If this fails, we need to invalidate
* the entries we've already put into the "shadow" list entry
* above.
*/
status = hermon_mcg_qplist_add(state, mcg, mcg_entry_qplist, qp,
&qp_found);
if (status != DDI_SUCCESS) {
bzero(mcg, sizeof (struct hermon_sw_mcg_list_s));
mutex_exit(&state->hs_mcglock);
return (status);
}
if (!qp_found)
mcg_entry->member_cnt = (mcg->mcg_num_qps + 1);
/* set the member count */
/*
* Once the temporary MCG has been filled in, write the entry
* into the appropriate location in the Hermon MCG entry table.
* If it's successful, then drop the lock and return success.
* Note: In general, this operation shouldn't fail. If it
* does, then it is an indication that something (probably in
* HW, but maybe in SW) has gone seriously wrong. We still
* want to zero out the entries that we've filled in above
* (in the hermon_mcg_setup_new_hdr() routine).
*/
status = hermon_write_mgm_cmd_post(state, mcg_entry, end_indx,
HERMON_CMD_NOSLEEP_SPIN);
if (status != HERMON_CMD_SUCCESS) {
bzero(mcg, sizeof (struct hermon_sw_mcg_list_s));
mutex_exit(&state->hs_mcglock);
HERMON_WARNING(state, "failed to write MCG entry");
cmn_err(CE_CONT, "Hermon: WRITE_MGM command failed: "
"%08x\n", status);
if (status == HERMON_CMD_INVALID_STATUS) {
hermon_fm_ereport(state, HCA_SYS_ERR,
HCA_ERR_SRV_LOST);
}
return (ibc_get_ci_failure(0));
}
/*
* Now that we know all the Hermon firmware accesses have been
* successful, we update the "shadow" MCG entry by incrementing
* the "number of attached QPs" count.
*
* We increment only if the QP is not already part of the
* MCG by checking the 'qp_found' flag returned from the
* qplist_add above.
*/
if (!qp_found) {
mcg->mcg_num_qps++;
/*
* Increment the refcnt for this QP. Because the QP
* was added to this MCG, the refcnt must be
* incremented.
*/
hermon_qp_mcg_refcnt_inc(qp);
}
/*
* We drop the lock and return success.
*/
mutex_exit(&state->hs_mcglock);
return (DDI_SUCCESS);
}
/*
* If the specified MGID matches the MGID in the current entry, then
* we need to try to add the QP to the current MCG entry. In this
* case, it means that we need to read the existing MCG entry (into
* the temporary MCG), add the new QP number to the temporary entry
* (using the same method we used above), and write the entry back
* to the hardware (same as above).
*/
if ((mcg->mcg_mgid_h == gid.gid_prefix) &&
(mcg->mcg_mgid_l == gid.gid_guid)) {
/*
* Read the current MCG entry into the temporary MCG. Note:
* In general, this operation shouldn't fail. If it does,
* then it is an indication that something (probably in HW,
* but maybe in SW) has gone seriously wrong.
*/
status = hermon_read_mgm_cmd_post(state, mcg_entry, end_indx,
HERMON_CMD_NOSLEEP_SPIN);
if (status != HERMON_CMD_SUCCESS) {
mutex_exit(&state->hs_mcglock);
HERMON_WARNING(state, "failed to read MCG entry");
cmn_err(CE_CONT, "Hermon: READ_MGM command failed: "
"%08x\n", status);
if (status == HERMON_CMD_INVALID_STATUS) {
hermon_fm_ereport(state, HCA_SYS_ERR,
HCA_ERR_SRV_LOST);
}
return (ibc_get_ci_failure(0));
}
/*
* Try to add the new QP number to the list. This routine
* fills in the necessary pieces of the temporary MCG. The
* "mcg_entry_qplist" pointer is used to point to the portion
* of the temporary MCG that holds the QP numbers.
*
* Note: hermon_mcg_qplist_add() returns SUCCESS if it
* already found the QP in the list. In this case, the QP is
* not added on to the list again. Check the flag 'qp_found'
* if this value is needed to be known.
*
*/
status = hermon_mcg_qplist_add(state, mcg, mcg_entry_qplist, qp,
&qp_found);
if (status != DDI_SUCCESS) {
mutex_exit(&state->hs_mcglock);
return (status);
}
if (!qp_found)
mcg_entry->member_cnt = (mcg->mcg_num_qps + 1);
/* set the member count */
/*
* Once the temporary MCG has been updated, write the entry
* into the appropriate location in the Hermon MCG entry table.
* If it's successful, then drop the lock and return success.
* Note: In general, this operation shouldn't fail. If it
* does, then it is an indication that something (probably in
* HW, but maybe in SW) has gone seriously wrong.
*/
status = hermon_write_mgm_cmd_post(state, mcg_entry, end_indx,
HERMON_CMD_NOSLEEP_SPIN);
if (status != HERMON_CMD_SUCCESS) {
mutex_exit(&state->hs_mcglock);
HERMON_WARNING(state, "failed to write MCG entry");
cmn_err(CE_CONT, "Hermon: WRITE_MGM command failed: "
"%08x\n", status);
if (status == HERMON_CMD_INVALID_STATUS) {
hermon_fm_ereport(state, HCA_SYS_ERR,
HCA_ERR_SRV_LOST);
}
return (ibc_get_ci_failure(0));
}
/*
* Now that we know all the Hermon firmware accesses have been
* successful, we update the current "shadow" MCG entry by
* incrementing the "number of attached QPs" count.
*
* We increment only if the QP is not already part of the
* MCG by checking the 'qp_found' flag returned
* hermon_mcg_walk_mgid_hashfrom the qplist_add above.
*/
if (!qp_found) {
mcg->mcg_num_qps++;
/*
* Increment the refcnt for this QP. Because the QP
* was added to this MCG, the refcnt must be
* incremented.
*/
hermon_qp_mcg_refcnt_inc(qp);
}
/*
* We drop the lock and return success.
*/
mutex_exit(&state->hs_mcglock);
return (DDI_SUCCESS);
}
/*
* If we've reached here, then we're at the end of the hash chain.
* We need to allocate a new MCG entry, fill it in, write it to Hermon,
* and update the previous entry to link the new one to the end of the
* chain.
*/
/*
* Allocate an MCG table entry. This will be filled in with all
* the necessary parameters to define the multicast group. Then it
* will be written to the hardware in the next-to-last step below.
*/
status = hermon_rsrc_alloc(state, HERMON_MCG, 1, HERMON_NOSLEEP, &rsrc);
if (status != DDI_SUCCESS) {
mutex_exit(&state->hs_mcglock);
return (IBT_INSUFF_RESOURCE);
}
/*
* Fill in the new entry in the "shadow" MCG list. Note: Just as
* it does above, hermon_mcg_setup_new_hdr() also fills in a portion
* of the temporary MCG entry (the rest of which will be filled in by
* hermon_mcg_qplist_add() below)
*/
newmcg = &state->hs_mcghdl[rsrc->hr_indx];
hermon_mcg_setup_new_hdr(newmcg, mcg_entry, gid, rsrc);
/*
* Try to add the new QP number to the list. This routine fills in
* the final necessary pieces of the temporary MCG. The
* "mcg_entry_qplist" pointer is used to point to the portion of the
* temporary MCG that holds the QP numbers. If we fail here, we
* must undo the previous resource allocation.
*
* Note: hermon_mcg_qplist_add() can we return SUCCESS if it already
* found the QP in the list. In this case, the QP is not added on to
* the list again. Check the flag 'qp_found' if this value is needed
* to be known.
*/
status = hermon_mcg_qplist_add(state, newmcg, mcg_entry_qplist, qp,
&qp_found);
if (status != DDI_SUCCESS) {
bzero(newmcg, sizeof (struct hermon_sw_mcg_list_s));
hermon_rsrc_free(state, &rsrc);
mutex_exit(&state->hs_mcglock);
return (status);
}
mcg_entry->member_cnt = (newmcg->mcg_num_qps + 1);
/* set the member count */
/*
* Once the temporary MCG has been updated, write the entry into the
* appropriate location in the Hermon MCG entry table. If this is
* successful, then we need to chain the previous entry to this one.
* Note: In general, this operation shouldn't fail. If it does, then
* it is an indication that something (probably in HW, but maybe in
* SW) has gone seriously wrong.
*/
status = hermon_write_mgm_cmd_post(state, mcg_entry, rsrc->hr_indx,
HERMON_CMD_NOSLEEP_SPIN);
if (status != HERMON_CMD_SUCCESS) {
bzero(newmcg, sizeof (struct hermon_sw_mcg_list_s));
hermon_rsrc_free(state, &rsrc);
mutex_exit(&state->hs_mcglock);
HERMON_WARNING(state, "failed to write MCG entry");
cmn_err(CE_CONT, "Hermon: WRITE_MGM command failed: %08x\n",
status);
if (status == HERMON_CMD_INVALID_STATUS) {
hermon_fm_ereport(state, HCA_SYS_ERR, HCA_ERR_SRV_LOST);
}
return (ibc_get_ci_failure(0));
}
/*
* Now read the current MCG entry (the one previously at the end of
* hash chain) into the temporary MCG. We are going to update its
* "next_gid_indx" now and write the entry back to the MCG table.
* Note: In general, this operation shouldn't fail. If it does, then
* it is an indication that something (probably in HW, but maybe in SW)
* has gone seriously wrong. We will free up the MCG entry resource,
* but we will not undo the previously written MCG entry in the HW.
* This is OK, though, because the MCG entry is not currently attached
* to any hash chain.
*/
status = hermon_read_mgm_cmd_post(state, mcg_entry, end_indx,
HERMON_CMD_NOSLEEP_SPIN);
if (status != HERMON_CMD_SUCCESS) {
bzero(newmcg, sizeof (struct hermon_sw_mcg_list_s));
hermon_rsrc_free(state, &rsrc);
mutex_exit(&state->hs_mcglock);
HERMON_WARNING(state, "failed to read MCG entry");
cmn_err(CE_CONT, "Hermon: READ_MGM command failed: %08x\n",
status);
if (status == HERMON_CMD_INVALID_STATUS) {
hermon_fm_ereport(state, HCA_SYS_ERR, HCA_ERR_SRV_LOST);
}
return (ibc_get_ci_failure(0));
}
/*
* Finally, we update the "next_gid_indx" field in the temporary MCG
* and attempt to write the entry back into the Hermon MCG table. If
* this succeeds, then we update the "shadow" list to reflect the
* change, drop the lock, and return success. Note: In general, this
* operation shouldn't fail. If it does, then it is an indication
* that something (probably in HW, but maybe in SW) has gone seriously
* wrong. Just as we do above, we will free up the MCG entry resource,
* but we will not try to undo the previously written MCG entry. This
* is OK, though, because (since we failed here to update the end of
* the chain) that other entry is not currently attached to any chain.
*/
mcg_entry->next_gid_indx = rsrc->hr_indx;
status = hermon_write_mgm_cmd_post(state, mcg_entry, end_indx,
HERMON_CMD_NOSLEEP_SPIN);
if (status != HERMON_CMD_SUCCESS) {
bzero(newmcg, sizeof (struct hermon_sw_mcg_list_s));
hermon_rsrc_free(state, &rsrc);
mutex_exit(&state->hs_mcglock);
HERMON_WARNING(state, "failed to write MCG entry");
cmn_err(CE_CONT, "Hermon: WRITE_MGM command failed: %08x\n",
status);
if (status == HERMON_CMD_INVALID_STATUS) {
hermon_fm_ereport(state, HCA_SYS_ERR, HCA_ERR_SRV_LOST);
}
return (ibc_get_ci_failure(0));
}
mcg = &state->hs_mcghdl[end_indx];
mcg->mcg_next_indx = rsrc->hr_indx;
/*
* Now that we know all the Hermon firmware accesses have been
* successful, we update the new "shadow" MCG entry by incrementing
* the "number of attached QPs" count. Then we drop the lock and
* return success.
*/
newmcg->mcg_num_qps++;
/*
* Increment the refcnt for this QP. Because the QP
* was added to this MCG, the refcnt must be
* incremented.
*/
hermon_qp_mcg_refcnt_inc(qp);
mutex_exit(&state->hs_mcglock);
return (DDI_SUCCESS);
}
/*
* hermon_mcg_detach()
* Context: Can be called only from user or kernel context.
*/
int
hermon_mcg_detach(hermon_state_t *state, hermon_qphdl_t qp, ib_gid_t gid,
ib_lid_t lid)
{
hermon_hw_mcg_t *mcg_entry;
hermon_hw_mcg_qp_list_t *mcg_entry_qplist;
hermon_mcghdl_t mcg;
uint64_t mgid_hash;
uint32_t end_indx, prev_indx;
int status;
/*
* Check for invalid Multicast DLID. Specifically, all Multicast
* LIDs should be within a well defined range. If the specified LID
* is outside of that range, then return an error.
*/
if (hermon_mlid_is_valid(lid) == 0) {
return (IBT_MC_MLID_INVALID);
}
/*
* Compute the MGID hash value. As described above, the MCG table is
* arranged as a number of separate hash chains. This operation
* converts the specified MGID into the starting index of an entry in
* the hash table (i.e. the index for the start of the appropriate
* hash chain). Subsequent operations below will walk the chain
* searching for a matching entry from which to attempt to remove
* the specified QP.
*/
status = hermon_mgid_hash_cmd_post(state, gid.gid_prefix, gid.gid_guid,
&mgid_hash, HERMON_SLEEPFLAG_FOR_CONTEXT());
if (status != HERMON_CMD_SUCCESS) {
cmn_err(CE_CONT, "Hermon: MGID_HASH command failed: %08x\n",
status);
if (status == HERMON_CMD_INVALID_STATUS) {
hermon_fm_ereport(state, HCA_SYS_ERR, HCA_ERR_SRV_LOST);
}
return (ibc_get_ci_failure(0));
}
/*
* Grab the multicast group mutex. Then grab the pre-allocated
* temporary buffer used for holding and/or modifying MCG entries.
*/
mutex_enter(&state->hs_mcglock);
mcg_entry = state->hs_mcgtmp;
mcg_entry_qplist = HERMON_MCG_GET_QPLIST_PTR(mcg_entry);
/*
* Walk through the array of MCG entries starting at "mgid_hash".
* Try to find an MCG entry with a matching MGID. The
* hermon_mcg_walk_mgid_hash() routine walks the list and returns an
* index into the MCG table. The entry at this index is checked to
* determine whether it is a match or not. If it is a match, then
* we continue on to attempt to remove the QP from the MCG. If it
* is not a match (or not a valid MCG entry), then we return an error.
*/
end_indx = hermon_mcg_walk_mgid_hash(state, mgid_hash, gid, &prev_indx);
mcg = &state->hs_mcghdl[end_indx];
/*
* If MGID == 0 (the hash chain is empty) or if the specified MGID
* does not match the MGID in the current entry, then return
* IBT_MC_MGID_INVALID (to indicate that the specified MGID is not
* valid).
*/
if (((mcg->mcg_mgid_h == 0) && (mcg->mcg_mgid_l == 0)) ||
((mcg->mcg_mgid_h != gid.gid_prefix) ||
(mcg->mcg_mgid_l != gid.gid_guid))) {
mutex_exit(&state->hs_mcglock);
return (IBT_MC_MGID_INVALID);
}
/*
* Read the current MCG entry into the temporary MCG. Note: In
* general, this operation shouldn't fail. If it does, then it is
* an indication that something (probably in HW, but maybe in SW)
* has gone seriously wrong.
*/
status = hermon_read_mgm_cmd_post(state, mcg_entry, end_indx,
HERMON_CMD_NOSLEEP_SPIN);
if (status != HERMON_CMD_SUCCESS) {
mutex_exit(&state->hs_mcglock);
HERMON_WARNING(state, "failed to read MCG entry");
cmn_err(CE_CONT, "Hermon: READ_MGM command failed: %08x\n",
status);
if (status == HERMON_CMD_INVALID_STATUS) {
hermon_fm_ereport(state, HCA_SYS_ERR, HCA_ERR_SRV_LOST);
}
return (ibc_get_ci_failure(0));
}
/*
* Search the QP number list for a match. If a match is found, then
* remove the entry from the QP list. Otherwise, if no match is found,
* return an error.
*/
status = hermon_mcg_qplist_remove(mcg, mcg_entry_qplist, qp);
if (status != DDI_SUCCESS) {
mutex_exit(&state->hs_mcglock);
return (status);
}
/*
* Decrement the MCG count for this QP. When the 'qp_mcg'
* field becomes 0, then this QP is no longer a member of any
* MCG.
*/
hermon_qp_mcg_refcnt_dec(qp);
/*
* If the current MCG's QP number list is about to be made empty
* ("mcg_num_qps" == 1), then remove the entry itself from the hash
* chain. Otherwise, just write the updated MCG entry back to the
* hardware. In either case, once we successfully update the hardware
* chain, then we decrement the "shadow" list entry's "mcg_num_qps"
* count (or zero out the entire "shadow" list entry) before returning
* success. Note: Zeroing out the "shadow" list entry is done
* inside of hermon_mcg_hash_list_remove().
*/
if (mcg->mcg_num_qps == 1) {
/* Remove an MCG entry from the hash chain */
status = hermon_mcg_hash_list_remove(state, end_indx, prev_indx,
mcg_entry);
if (status != DDI_SUCCESS) {
mutex_exit(&state->hs_mcglock);
return (status);
}
} else {
/*
* Write the updated MCG entry back to the Hermon MCG table.
* If this succeeds, then we update the "shadow" list to
* reflect the change (i.e. decrement the "mcg_num_qps"),
* drop the lock, and return success. Note: In general,
* this operation shouldn't fail. If it does, then it is an
* indication that something (probably in HW, but maybe in SW)
* has gone seriously wrong.
*/
mcg_entry->member_cnt = (mcg->mcg_num_qps - 1);
status = hermon_write_mgm_cmd_post(state, mcg_entry, end_indx,
HERMON_CMD_NOSLEEP_SPIN);
if (status != HERMON_CMD_SUCCESS) {
mutex_exit(&state->hs_mcglock);
HERMON_WARNING(state, "failed to write MCG entry");
cmn_err(CE_CONT, "Hermon: WRITE_MGM command failed: "
"%08x\n", status);
if (status == HERMON_CMD_INVALID_STATUS) {
hermon_fm_ereport(state, HCA_SYS_ERR,
HCA_ERR_SRV_LOST);
}
return (ibc_get_ci_failure(0));
}
mcg->mcg_num_qps--;
}
mutex_exit(&state->hs_mcglock);
return (DDI_SUCCESS);
}
/*
* hermon_qp_mcg_refcnt_inc()
* Context: Can be called from interrupt or base context.
*/
static void
hermon_qp_mcg_refcnt_inc(hermon_qphdl_t qp)
{
/* Increment the QP's MCG reference count */
mutex_enter(&qp->qp_lock);
qp->qp_mcg_refcnt++;
mutex_exit(&qp->qp_lock);
}
/*
* hermon_qp_mcg_refcnt_dec()
* Context: Can be called from interrupt or base context.
*/
static void
hermon_qp_mcg_refcnt_dec(hermon_qphdl_t qp)
{
/* Decrement the QP's MCG reference count */
mutex_enter(&qp->qp_lock);
qp->qp_mcg_refcnt--;
mutex_exit(&qp->qp_lock);
}
/*
* hermon_mcg_qplist_add()
* Context: Can be called from interrupt or base context.
*/
static int
hermon_mcg_qplist_add(hermon_state_t *state, hermon_mcghdl_t mcg,
hermon_hw_mcg_qp_list_t *mcg_qplist, hermon_qphdl_t qp,
uint_t *qp_found)
{
uint_t qplist_indx;
ASSERT(MUTEX_HELD(&state->hs_mcglock));
qplist_indx = mcg->mcg_num_qps;
/*
* Determine if we have exceeded the maximum number of QP per
* multicast group. If we have, then return an error
*/
if (qplist_indx >= state->hs_cfg_profile->cp_num_qp_per_mcg) {
return (IBT_HCA_MCG_QP_EXCEEDED);
}
/*
* Determine if the QP is already attached to this MCG table. If it
* is, then we break out and treat this operation as a NO-OP
*/
for (qplist_indx = 0; qplist_indx < mcg->mcg_num_qps;
qplist_indx++) {
if (mcg_qplist[qplist_indx].qpn == qp->qp_qpnum) {
break;
}
}
/*
* If the QP was already on the list, set 'qp_found' to TRUE. We still
* return SUCCESS in this case, but the qplist will not have been
* updated because the QP was already on the list.
*/
if (qplist_indx < mcg->mcg_num_qps) {
*qp_found = 1;
} else {
/*
* Otherwise, append the new QP number to the end of the
* current QP list. Note: We will increment the "mcg_num_qps"
* field on the "shadow" MCG list entry later (after we know
* that all necessary Hermon firmware accesses have been
* successful).
*
* Set 'qp_found' to 0 so we know the QP was added on to the
* list for sure.
*/
mcg_qplist[qplist_indx].qpn =
(qp->qp_qpnum | HERMON_MCG_QPN_BLOCK_LB);
*qp_found = 0;
}
return (DDI_SUCCESS);
}
/*
* hermon_mcg_qplist_remove()
* Context: Can be called from interrupt or base context.
*/
static int
hermon_mcg_qplist_remove(hermon_mcghdl_t mcg,
hermon_hw_mcg_qp_list_t *mcg_qplist, hermon_qphdl_t qp)
{
uint_t i, qplist_indx;
/*
* Search the MCG QP list for a matching QPN. When
* it's found, we swap the last entry with the current
* one, set the last entry to zero, decrement the last
* entry, and return. If it's not found, then it's
* and error.
*/
qplist_indx = mcg->mcg_num_qps;
for (i = 0; i < qplist_indx; i++) {
if (mcg_qplist[i].qpn == qp->qp_qpnum) {
mcg_qplist[i] = mcg_qplist[qplist_indx - 1];
mcg_qplist[qplist_indx - 1].qpn = 0;
return (DDI_SUCCESS);
}
}
return (IBT_QP_HDL_INVALID);
}
/*
* hermon_mcg_walk_mgid_hash()
* Context: Can be called from interrupt or base context.
*/
static uint_t
hermon_mcg_walk_mgid_hash(hermon_state_t *state, uint64_t start_indx,
ib_gid_t mgid, uint_t *p_indx)
{
hermon_mcghdl_t curr_mcghdl;
uint_t curr_indx, prev_indx;
ASSERT(MUTEX_HELD(&state->hs_mcglock));
/* Start at the head of the hash chain */
curr_indx = (uint_t)start_indx;
prev_indx = curr_indx;
curr_mcghdl = &state->hs_mcghdl[curr_indx];
/* If the first entry in the chain has MGID == 0, then stop */
if ((curr_mcghdl->mcg_mgid_h == 0) &&
(curr_mcghdl->mcg_mgid_l == 0)) {
goto end_mgid_hash_walk;
}
/* If the first entry in the chain matches the MGID, then stop */
if ((curr_mcghdl->mcg_mgid_h == mgid.gid_prefix) &&
(curr_mcghdl->mcg_mgid_l == mgid.gid_guid)) {
goto end_mgid_hash_walk;
}
/* Otherwise, walk the hash chain looking for a match */
while (curr_mcghdl->mcg_next_indx != 0) {
prev_indx = curr_indx;
curr_indx = curr_mcghdl->mcg_next_indx;
curr_mcghdl = &state->hs_mcghdl[curr_indx];
if ((curr_mcghdl->mcg_mgid_h == mgid.gid_prefix) &&
(curr_mcghdl->mcg_mgid_l == mgid.gid_guid)) {
break;
}
}
end_mgid_hash_walk:
/*
* If necessary, return the index of the previous entry too. This
* is primarily used for detaching a QP from a multicast group. It
* may be necessary, in that case, to delete an MCG entry from the
* hash chain and having the index of the previous entry is helpful.
*/
if (p_indx != NULL) {
*p_indx = prev_indx;
}
return (curr_indx);
}
/*
* hermon_mcg_setup_new_hdr()
* Context: Can be called from interrupt or base context.
*/
static void
hermon_mcg_setup_new_hdr(hermon_mcghdl_t mcg, hermon_hw_mcg_t *mcg_hdr,
ib_gid_t mgid, hermon_rsrc_t *mcg_rsrc)
{
/*
* Fill in the fields of the "shadow" entry used by software
* to track MCG hardware entry
*/
mcg->mcg_mgid_h = mgid.gid_prefix;
mcg->mcg_mgid_l = mgid.gid_guid;
mcg->mcg_rsrcp = mcg_rsrc;
mcg->mcg_next_indx = 0;
mcg->mcg_num_qps = 0;
/*
* Fill the header fields of the MCG entry (in the temporary copy)
*/
mcg_hdr->mgid_h = mgid.gid_prefix;
mcg_hdr->mgid_l = mgid.gid_guid;
mcg_hdr->next_gid_indx = 0;
}
/*
* hermon_mcg_hash_list_remove()
* Context: Can be called only from user or kernel context.
*/
static int
hermon_mcg_hash_list_remove(hermon_state_t *state, uint_t curr_indx,
uint_t prev_indx, hermon_hw_mcg_t *mcg_entry)
{
hermon_mcghdl_t curr_mcg, prev_mcg, next_mcg;
uint_t next_indx;
int status;
/* Get the pointer to "shadow" list for current entry */
curr_mcg = &state->hs_mcghdl[curr_indx];
/*
* If this is the first entry on a hash chain, then attempt to replace
* the entry with the next entry on the chain. If there are no
* subsequent entries on the chain, then this is the only entry and
* should be invalidated.
*/
if (curr_indx == prev_indx) {
/*
* If this is the only entry on the chain, then invalidate it.
* Note: Invalidating an MCG entry means writing all zeros
* to the entry. This is only necessary for those MCG
* entries that are the "head" entries of the individual hash
* chains. Regardless of whether this operation returns
* success or failure, return that result to the caller.
*/
next_indx = curr_mcg->mcg_next_indx;
if (next_indx == 0) {
status = hermon_mcg_entry_invalidate(state, mcg_entry,
curr_indx);
bzero(curr_mcg, sizeof (struct hermon_sw_mcg_list_s));
return (status);
}
/*
* Otherwise, this is just the first entry on the chain, so
* grab the next one
*/
next_mcg = &state->hs_mcghdl[next_indx];
/*
* Read the next MCG entry into the temporary MCG. Note:
* In general, this operation shouldn't fail. If it does,
* then it is an indication that something (probably in HW,
* but maybe in SW) has gone seriously wrong.
*/
status = hermon_read_mgm_cmd_post(state, mcg_entry, next_indx,
HERMON_CMD_NOSLEEP_SPIN);
if (status != HERMON_CMD_SUCCESS) {
HERMON_WARNING(state, "failed to read MCG entry");
cmn_err(CE_CONT, "Hermon: READ_MGM command failed: "
"%08x\n", status);
if (status == HERMON_CMD_INVALID_STATUS) {
hermon_fm_ereport(state, HCA_SYS_ERR,
HCA_ERR_SRV_LOST);
}
return (ibc_get_ci_failure(0));
}
/*
* Copy/Write the temporary MCG back to the hardware MCG list
* using the current index. This essentially removes the
* current MCG entry from the list by writing over it with
* the next one. If this is successful, then we can do the
* same operation for the "shadow" list. And we can also
* free up the Hermon MCG entry resource that was associated
* with the (old) next entry. Note: In general, this
* operation shouldn't fail. If it does, then it is an
* indication that something (probably in HW, but maybe in SW)
* has gone seriously wrong.
*/
status = hermon_write_mgm_cmd_post(state, mcg_entry, curr_indx,
HERMON_CMD_NOSLEEP_SPIN);
if (status != HERMON_CMD_SUCCESS) {
HERMON_WARNING(state, "failed to write MCG entry");
cmn_err(CE_CONT, "Hermon: WRITE_MGM command failed: "
"%08x\n", status);
if (status == HERMON_CMD_INVALID_STATUS) {
hermon_fm_ereport(state, HCA_SYS_ERR,
HCA_ERR_SRV_LOST);
}
return (ibc_get_ci_failure(0));
}
/*
* Copy all the software tracking information from the next
* entry on the "shadow" MCG list into the current entry on
* the list. Then invalidate (zero out) the other "shadow"
* list entry.
*/
bcopy(next_mcg, curr_mcg, sizeof (struct hermon_sw_mcg_list_s));
bzero(next_mcg, sizeof (struct hermon_sw_mcg_list_s));
/*
* Free up the Hermon MCG entry resource used by the "next"
* MCG entry. That resource is no longer needed by any
* MCG entry which is first on a hash chain (like the "next"
* entry has just become).
*/
hermon_rsrc_free(state, &curr_mcg->mcg_rsrcp);
return (DDI_SUCCESS);
}
/*
* Else if this is the last entry on the hash chain (or a middle
* entry, then we update the previous entry's "next_gid_index" field
* to make it point instead to the next entry on the chain. By
* skipping over the removed entry in this way, we can then free up
* any resources associated with the current entry. Note: We don't
* need to invalidate the "skipped over" hardware entry because it
* will no be longer connected to any hash chains, and if/when it is
* finally re-used, it will be written with entirely new values.
*/
/*
* Read the next MCG entry into the temporary MCG. Note: In general,
* this operation shouldn't fail. If it does, then it is an
* indication that something (probably in HW, but maybe in SW) has
* gone seriously wrong.
*/
status = hermon_read_mgm_cmd_post(state, mcg_entry, prev_indx,
HERMON_CMD_NOSLEEP_SPIN);
if (status != HERMON_CMD_SUCCESS) {
HERMON_WARNING(state, "failed to read MCG entry");
cmn_err(CE_CONT, "Hermon: READ_MGM command failed: %08x\n",
status);
if (status == HERMON_CMD_INVALID_STATUS) {
hermon_fm_ereport(state, HCA_SYS_ERR, HCA_ERR_SRV_LOST);
}
return (ibc_get_ci_failure(0));
}
/*
* Finally, we update the "next_gid_indx" field in the temporary MCG
* and attempt to write the entry back into the Hermon MCG table. If
* this succeeds, then we update the "shadow" list to reflect the
* change, free up the Hermon MCG entry resource that was associated
* with the current entry, and return success. Note: In general,
* this operation shouldn't fail. If it does, then it is an indication
* that something (probably in HW, but maybe in SW) has gone seriously
* wrong.
*/
mcg_entry->next_gid_indx = curr_mcg->mcg_next_indx;
status = hermon_write_mgm_cmd_post(state, mcg_entry, prev_indx,
HERMON_CMD_NOSLEEP_SPIN);
if (status != HERMON_CMD_SUCCESS) {
HERMON_WARNING(state, "failed to write MCG entry");
cmn_err(CE_CONT, "Hermon: WRITE_MGM command failed: %08x\n",
status);
if (status == HERMON_CMD_INVALID_STATUS) {
hermon_fm_ereport(state, HCA_SYS_ERR,
HCA_ERR_SRV_LOST);
}
return (ibc_get_ci_failure(0));
}
/*
* Get the pointer to the "shadow" MCG list entry for the previous
* MCG. Update its "mcg_next_indx" to point to the next entry
* the one after the current entry. Note: This next index may be
* zero, indicating the end of the list.
*/
prev_mcg = &state->hs_mcghdl[prev_indx];
prev_mcg->mcg_next_indx = curr_mcg->mcg_next_indx;
/*
* Free up the Hermon MCG entry resource used by the current entry.
* This resource is no longer needed because the chain now skips over
* the current entry. Then invalidate (zero out) the current "shadow"
* list entry.
*/
hermon_rsrc_free(state, &curr_mcg->mcg_rsrcp);
bzero(curr_mcg, sizeof (struct hermon_sw_mcg_list_s));
return (DDI_SUCCESS);
}
/*
* hermon_mcg_entry_invalidate()
* Context: Can be called only from user or kernel context.
*/
static int
hermon_mcg_entry_invalidate(hermon_state_t *state, hermon_hw_mcg_t *mcg_entry,
uint_t indx)
{
int status;
/*
* Invalidate the hardware MCG entry by zeroing out this temporary
* MCG and writing it the the hardware. Note: In general, this
* operation shouldn't fail. If it does, then it is an indication
* that something (probably in HW, but maybe in SW) has gone seriously
* wrong.
*/
bzero(mcg_entry, HERMON_MCGMEM_SZ(state));
status = hermon_write_mgm_cmd_post(state, mcg_entry, indx,
HERMON_CMD_NOSLEEP_SPIN);
if (status != HERMON_CMD_SUCCESS) {
HERMON_WARNING(state, "failed to write MCG entry");
cmn_err(CE_CONT, "Hermon: WRITE_MGM command failed: %08x\n",
status);
if (status == HERMON_CMD_INVALID_STATUS) {
hermon_fm_ereport(state, HCA_SYS_ERR, HCA_ERR_SRV_LOST);
}
return (ibc_get_ci_failure(0));
}
return (DDI_SUCCESS);
}
/*
* hermon_mgid_is_valid()
* Context: Can be called from interrupt or base context.
*/
static int
hermon_mgid_is_valid(ib_gid_t gid)
{
uint_t topbits, flags, scope;
/*
* According to IBA 1.1 specification (section 4.1.1) a valid
* "multicast GID" must have its top eight bits set to all ones
*/
topbits = (gid.gid_prefix >> HERMON_MCG_TOPBITS_SHIFT) &
HERMON_MCG_TOPBITS_MASK;
if (topbits != HERMON_MCG_TOPBITS) {
return (0);
}
/*
* The next 4 bits are the "flag" bits. These are valid only
* if they are "0" (which correspond to permanently assigned/
* "well-known" multicast GIDs) or "1" (for so-called "transient"
* multicast GIDs). All other values are reserved.
*/
flags = (gid.gid_prefix >> HERMON_MCG_FLAGS_SHIFT) &
HERMON_MCG_FLAGS_MASK;
if (!((flags == HERMON_MCG_FLAGS_PERM) ||
(flags == HERMON_MCG_FLAGS_NONPERM))) {
return (0);
}
/*
* The next 4 bits are the "scope" bits. These are valid only
* if they are "2" (Link-local), "5" (Site-local), "8"
* (Organization-local) or "E" (Global). All other values
* are reserved (or currently unassigned).
*/
scope = (gid.gid_prefix >> HERMON_MCG_SCOPE_SHIFT) &
HERMON_MCG_SCOPE_MASK;
if (!((scope == HERMON_MCG_SCOPE_LINKLOC) ||
(scope == HERMON_MCG_SCOPE_SITELOC) ||
(scope == HERMON_MCG_SCOPE_ORGLOC) ||
(scope == HERMON_MCG_SCOPE_GLOBAL))) {
return (0);
}
/*
* If it passes all of the above checks, then we will consider it
* a valid multicast GID.
*/
return (1);
}
/*
* hermon_mlid_is_valid()
* Context: Can be called from interrupt or base context.
*/
static int
hermon_mlid_is_valid(ib_lid_t lid)
{
/*
* According to IBA 1.1 specification (section 4.1.1) a valid
* "multicast DLID" must be between 0xC000 and 0xFFFE.
*/
if ((lid < IB_LID_MC_FIRST) || (lid > IB_LID_MC_LAST)) {
return (0);
}
return (1);
}
/*
* hermon_pd_alloc()
* Context: Can be called only from user or kernel context.
*/
int
hermon_pd_alloc(hermon_state_t *state, hermon_pdhdl_t *pdhdl, uint_t sleepflag)
{
hermon_rsrc_t *rsrc;
hermon_pdhdl_t pd;
int status;
/*
* Allocate the software structure for tracking the protection domain
* (i.e. the Hermon Protection Domain handle). By default each PD
* structure will have a unique PD number assigned to it. All that
* is necessary is for software to initialize the PD reference count
* (to zero) and return success.
*/
status = hermon_rsrc_alloc(state, HERMON_PDHDL, 1, sleepflag, &rsrc);
if (status != DDI_SUCCESS) {
return (IBT_INSUFF_RESOURCE);
}
pd = (hermon_pdhdl_t)rsrc->hr_addr;
_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*pd))
pd->pd_refcnt = 0;
*pdhdl = pd;
return (DDI_SUCCESS);
}
/*
* hermon_pd_free()
* Context: Can be called only from user or kernel context.
*/
int
hermon_pd_free(hermon_state_t *state, hermon_pdhdl_t *pdhdl)
{
hermon_rsrc_t *rsrc;
hermon_pdhdl_t pd;
/*
* Pull all the necessary information from the Hermon Protection Domain
* handle. This is necessary here because the resource for the
* PD is going to be freed up as part of this operation.
*/
pd = *pdhdl;
_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*pd))
rsrc = pd->pd_rsrcp;
/*
* Check the PD reference count. If the reference count is non-zero,
* then it means that this protection domain is still referenced by
* some memory region, queue pair, address handle, or other IB object
* If it is non-zero, then return an error. Otherwise, free the
* Hermon resource and return success.
*/
if (pd->pd_refcnt != 0) {
return (IBT_PD_IN_USE);
}
/* Free the Hermon Protection Domain handle */
hermon_rsrc_free(state, &rsrc);
/* Set the pdhdl pointer to NULL and return success */
*pdhdl = (hermon_pdhdl_t)NULL;
return (DDI_SUCCESS);
}
/*
* hermon_pd_refcnt_inc()
* Context: Can be called from interrupt or base context.
*/
void
hermon_pd_refcnt_inc(hermon_pdhdl_t pd)
{
/* Increment the protection domain's reference count */
atomic_inc_32(&pd->pd_refcnt);
}
/*
* hermon_pd_refcnt_dec()
* Context: Can be called from interrupt or base context.
*/
void
hermon_pd_refcnt_dec(hermon_pdhdl_t pd)
{
/* Decrement the protection domain's reference count */
atomic_dec_32(&pd->pd_refcnt);
}
/*
* hermon_port_query()
* Context: Can be called only from user or kernel context.
*/
int
hermon_port_query(hermon_state_t *state, uint_t port, ibt_hca_portinfo_t *pi)
{
sm_portinfo_t portinfo;
sm_guidinfo_t guidinfo;
sm_pkey_table_t pkeytable;
ib_gid_t *sgid;
uint_t sgid_max, pkey_max, tbl_size;
int i, j, indx, status;
ib_pkey_t *pkeyp;
ib_guid_t *guidp;
_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*pi))
_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*state))
/* Validate that specified port number is legal */
if (!hermon_portnum_is_valid(state, port)) {
return (IBT_HCA_PORT_INVALID);
}
pkeyp = state->hs_pkey[port - 1];
guidp = state->hs_guid[port - 1];
/*
* We use the Hermon MAD_IFC command to post a GetPortInfo MAD
* to the firmware (for the specified port number). This returns
* a full PortInfo MAD (in "portinfo") which we subsequently
* parse to fill in the "ibt_hca_portinfo_t" structure returned
* to the IBTF.
*/
status = hermon_getportinfo_cmd_post(state, port,
HERMON_SLEEPFLAG_FOR_CONTEXT(), &portinfo);
if (status != HERMON_CMD_SUCCESS) {
cmn_err(CE_CONT, "Hermon: GetPortInfo (port %02d) command "
"failed: %08x\n", port, status);
if (status == HERMON_CMD_INVALID_STATUS) {
hermon_fm_ereport(state, HCA_SYS_ERR, HCA_ERR_SRV_LOST);
}
return (ibc_get_ci_failure(0));
}
/*
* Parse the PortInfo MAD and fill in the IBTF structure
*/
pi->p_base_lid = portinfo.LID;
pi->p_qkey_violations = portinfo.Q_KeyViolations;
pi->p_pkey_violations = portinfo.P_KeyViolations;
pi->p_sm_sl = portinfo.MasterSMSL;
pi->p_sm_lid = portinfo.MasterSMLID;
pi->p_linkstate = portinfo.PortState;
pi->p_port_num = portinfo.LocalPortNum;
pi->p_phys_state = portinfo.PortPhysicalState;
pi->p_width_supported = portinfo.LinkWidthSupported;
pi->p_width_enabled = portinfo.LinkWidthEnabled;
pi->p_width_active = portinfo.LinkWidthActive;
pi->p_speed_supported = portinfo.LinkSpeedSupported;
pi->p_speed_enabled = portinfo.LinkSpeedEnabled;
pi->p_speed_active = portinfo.LinkSpeedActive;
pi->p_mtu = portinfo.MTUCap;
pi->p_lmc = portinfo.LMC;
pi->p_max_vl = portinfo.VLCap;
pi->p_subnet_timeout = portinfo.SubnetTimeOut;
pi->p_msg_sz = ((uint32_t)1 << HERMON_QP_LOG_MAX_MSGSZ);
tbl_size = state->hs_cfg_profile->cp_log_max_gidtbl;
pi->p_sgid_tbl_sz = (1 << tbl_size);
tbl_size = state->hs_cfg_profile->cp_log_max_pkeytbl;
pi->p_pkey_tbl_sz = (1 << tbl_size);
state->hs_sn_prefix[port - 1] = portinfo.GidPrefix;
/*
* Convert InfiniBand-defined port capability flags to the format
* specified by the IBTF
*/
if (portinfo.CapabilityMask & SM_CAP_MASK_IS_SM)
pi->p_capabilities |= IBT_PORT_CAP_SM;
if (portinfo.CapabilityMask & SM_CAP_MASK_IS_SM_DISABLED)
pi->p_capabilities |= IBT_PORT_CAP_SM_DISABLED;
if (portinfo.CapabilityMask & SM_CAP_MASK_IS_SNMP_SUPPD)
pi->p_capabilities |= IBT_PORT_CAP_SNMP_TUNNEL;
if (portinfo.CapabilityMask & SM_CAP_MASK_IS_DM_SUPPD)
pi->p_capabilities |= IBT_PORT_CAP_DM;
if (portinfo.CapabilityMask & SM_CAP_MASK_IS_VM_SUPPD)
pi->p_capabilities |= IBT_PORT_CAP_VENDOR;
if (portinfo.CapabilityMask & SM_CAP_MASK_IS_CLNT_REREG_SUPPD)
pi->p_capabilities |= IBT_PORT_CAP_CLNT_REREG;
/*
* Fill in the SGID table. Since the only access to the Hermon
* GID tables is through the firmware's MAD_IFC interface, we
* post as many GetGUIDInfo MADs as necessary to read in the entire
* contents of the SGID table (for the specified port). Note: The
* GetGUIDInfo command only gets eight GUIDs per operation. These
* GUIDs are then appended to the GID prefix for the port (from the
* GetPortInfo above) to form the entire SGID table.
*/
for (i = 0; i < pi->p_sgid_tbl_sz; i += 8) {
status = hermon_getguidinfo_cmd_post(state, port, i >> 3,
HERMON_SLEEPFLAG_FOR_CONTEXT(), &guidinfo);
if (status != HERMON_CMD_SUCCESS) {
cmn_err(CE_CONT, "Hermon: GetGUIDInfo (port %02d) "
"command failed: %08x\n", port, status);
if (status == HERMON_CMD_INVALID_STATUS) {
hermon_fm_ereport(state, HCA_SYS_ERR,
HCA_ERR_SRV_LOST);
}
return (ibc_get_ci_failure(0));
}
/* Figure out how many of the entries are valid */
sgid_max = min((pi->p_sgid_tbl_sz - i), 8);
for (j = 0; j < sgid_max; j++) {
indx = (i + j);
sgid = &pi->p_sgid_tbl[indx];
_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*sgid))
sgid->gid_prefix = portinfo.GidPrefix;
guidp[indx] = sgid->gid_guid =
guidinfo.GUIDBlocks[j];
}
}
/*
* Fill in the PKey table. Just as for the GID tables above, the
* only access to the Hermon PKey tables is through the firmware's
* MAD_IFC interface. We post as many GetPKeyTable MADs as necessary
* to read in the entire contents of the PKey table (for the specified
* port). Note: The GetPKeyTable command only gets 32 PKeys per
* operation.
*/
for (i = 0; i < pi->p_pkey_tbl_sz; i += 32) {
status = hermon_getpkeytable_cmd_post(state, port, i,
HERMON_SLEEPFLAG_FOR_CONTEXT(), &pkeytable);
if (status != HERMON_CMD_SUCCESS) {
cmn_err(CE_CONT, "Hermon: GetPKeyTable (port %02d) "
"command failed: %08x\n", port, status);
if (status == HERMON_CMD_INVALID_STATUS) {
hermon_fm_ereport(state, HCA_SYS_ERR,
HCA_ERR_SRV_LOST);
}
return (ibc_get_ci_failure(0));
}
/* Figure out how many of the entries are valid */
pkey_max = min((pi->p_pkey_tbl_sz - i), 32);
for (j = 0; j < pkey_max; j++) {
indx = (i + j);
pkeyp[indx] = pi->p_pkey_tbl[indx] =
pkeytable.P_KeyTableBlocks[j];
}
}
return (DDI_SUCCESS);
}
/*
* hermon_port_modify()
* Context: Can be called only from user or kernel context.
*/
/* ARGSUSED */
int
hermon_port_modify(hermon_state_t *state, uint8_t port,
ibt_port_modify_flags_t flags, uint8_t init_type)
{
sm_portinfo_t portinfo;
uint32_t capmask;
int status;
hermon_hw_set_port_t set_port;
/*
* Return an error if either of the unsupported flags are set
*/
if ((flags & IBT_PORT_SHUTDOWN) ||
(flags & IBT_PORT_SET_INIT_TYPE)) {
return (IBT_NOT_SUPPORTED);
}
bzero(&set_port, sizeof (set_port));
/*
* Determine whether we are trying to reset the QKey counter
*/
if (flags & IBT_PORT_RESET_QKEY)
set_port.rqk = 1;
/* Validate that specified port number is legal */
if (!hermon_portnum_is_valid(state, port)) {
return (IBT_HCA_PORT_INVALID);
}
/*
* Use the Hermon MAD_IFC command to post a GetPortInfo MAD to the
* firmware (for the specified port number). This returns a full
* PortInfo MAD (in "portinfo") from which we pull the current
* capability mask. We then modify the capability mask as directed
* by the "pmod_flags" field, and write the updated capability mask
* using the Hermon SET_IB command (below).
*/
status = hermon_getportinfo_cmd_post(state, port,
HERMON_SLEEPFLAG_FOR_CONTEXT(), &portinfo);
if (status != HERMON_CMD_SUCCESS) {
if (status == HERMON_CMD_INVALID_STATUS) {
hermon_fm_ereport(state, HCA_SYS_ERR, HCA_ERR_SRV_LOST);
}
return (ibc_get_ci_failure(0));
}
/*
* Convert InfiniBand-defined port capability flags to the format
* specified by the IBTF. Specifically, we modify the capability
* mask based on the specified values.
*/
capmask = portinfo.CapabilityMask;
if (flags & IBT_PORT_RESET_SM)
capmask &= ~SM_CAP_MASK_IS_SM;
else if (flags & IBT_PORT_SET_SM)
capmask |= SM_CAP_MASK_IS_SM;
if (flags & IBT_PORT_RESET_SNMP)
capmask &= ~SM_CAP_MASK_IS_SNMP_SUPPD;
else if (flags & IBT_PORT_SET_SNMP)
capmask |= SM_CAP_MASK_IS_SNMP_SUPPD;
if (flags & IBT_PORT_RESET_DEVMGT)
capmask &= ~SM_CAP_MASK_IS_DM_SUPPD;
else if (flags & IBT_PORT_SET_DEVMGT)
capmask |= SM_CAP_MASK_IS_DM_SUPPD;
if (flags & IBT_PORT_RESET_VENDOR)
capmask &= ~SM_CAP_MASK_IS_VM_SUPPD;
else if (flags & IBT_PORT_SET_VENDOR)
capmask |= SM_CAP_MASK_IS_VM_SUPPD;
set_port.cap_mask = capmask;
/*
* Use the Hermon SET_PORT command to update the capability mask and
* (possibly) reset the QKey violation counter for the specified port.
* Note: In general, this operation shouldn't fail. If it does, then
* it is an indication that something (probably in HW, but maybe in
* SW) has gone seriously wrong.
*/
status = hermon_set_port_cmd_post(state, &set_port, port,
HERMON_SLEEPFLAG_FOR_CONTEXT());
if (status != HERMON_CMD_SUCCESS) {
HERMON_WARNING(state, "failed to modify port capabilities");
cmn_err(CE_CONT, "Hermon: SET_IB (port %02d) command failed: "
"%08x\n", port, status);
if (status == HERMON_CMD_INVALID_STATUS) {
hermon_fm_ereport(state, HCA_SYS_ERR, HCA_ERR_SRV_LOST);
}
return (ibc_get_ci_failure(0));
}
return (DDI_SUCCESS);
}
/*
* hermon_set_addr_path()
* Context: Can be called from interrupt or base context.
*
* Note: This routine is used for two purposes. It is used to fill in the
* Hermon UDAV fields, and it is used to fill in the address path information
* for QPs. Because the two Hermon structures are similar, common fields can
* be filled in here. Because they are different, however, we pass
* an additional flag to indicate which type is being filled and do each one
* uniquely
*/
int hermon_srate_override = -1; /* allows ease of testing */
int
hermon_set_addr_path(hermon_state_t *state, ibt_adds_vect_t *av,
hermon_hw_addr_path_t *path, uint_t type)
{
uint_t gidtbl_sz;
hermon_hw_udav_t *udav;
_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*av))
_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*path))
udav = (hermon_hw_udav_t *)(void *)path;
_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*udav))
path->mlid = av->av_src_path;
path->rlid = av->av_dlid;
switch (av->av_srate) {
case IBT_SRATE_2: /* 1xSDR-2.5Gb/s injection rate */
path->max_stat_rate = 7; break;
case IBT_SRATE_10: /* 4xSDR-10.0Gb/s injection rate */
path->max_stat_rate = 8; break;
case IBT_SRATE_30: /* 12xSDR-30Gb/s injection rate */
path->max_stat_rate = 9; break;
case IBT_SRATE_5: /* 1xDDR-5Gb/s injection rate */
path->max_stat_rate = 10; break;
case IBT_SRATE_20: /* 4xDDR-20Gb/s injection rate */
path->max_stat_rate = 11; break;
case IBT_SRATE_40: /* 4xQDR-40Gb/s injection rate */
path->max_stat_rate = 12; break;
case IBT_SRATE_60: /* 12xDDR-60Gb/s injection rate */
path->max_stat_rate = 13; break;
case IBT_SRATE_80: /* 8xQDR-80Gb/s injection rate */
path->max_stat_rate = 14; break;
case IBT_SRATE_120: /* 12xQDR-120Gb/s injection rate */
path->max_stat_rate = 15; break;
case IBT_SRATE_NOT_SPECIFIED: /* Max */
path->max_stat_rate = 0; break;
default:
return (IBT_STATIC_RATE_INVALID);
}
if (hermon_srate_override != -1) /* for evaluating HCA firmware */
path->max_stat_rate = hermon_srate_override;
/* If "grh" flag is set, then check for valid SGID index too */
gidtbl_sz = (1 << state->hs_queryport.log_max_gid);
if ((av->av_send_grh) && (av->av_sgid_ix > gidtbl_sz)) {
return (IBT_SGID_INVALID);
}
/*
* Fill in all "global" values regardless of the value in the GRH
* flag. Because "grh" is not set unless "av_send_grh" is set, the
* hardware will ignore the other "global" values as necessary. Note:
* SW does this here to enable later query operations to return
* exactly the same params that were passed when the addr path was
* last written.
*/
path->grh = av->av_send_grh;
if (type == HERMON_ADDRPATH_QP) {
path->mgid_index = av->av_sgid_ix;
} else {
/*
* For Hermon UDAV, the "mgid_index" field is the index into
* a combined table (not a per-port table), but having sections
* for each port. So some extra calculations are necessary.
*/
path->mgid_index = ((av->av_port_num - 1) * gidtbl_sz) +
av->av_sgid_ix;
udav->portnum = av->av_port_num;
}
/*
* According to Hermon PRM, the (31:0) part of rgid_l must be set to
* "0x2" if the 'grh' or 'g' bit is cleared. It also says that we
* only need to do it for UDAV's. So we enforce that here.
*
* NOTE: The entire 64 bits worth of GUID info is actually being
* preserved (for UDAVs) by the callers of this function
* (hermon_ah_alloc() and hermon_ah_modify()) and as long as the
* 'grh' bit is not set, the upper 32 bits (63:32) of rgid_l are
* "don't care".
*/
if ((path->grh) || (type == HERMON_ADDRPATH_QP)) {
path->flow_label = av->av_flow;
path->tclass = av->av_tclass;
path->hop_limit = av->av_hop;
bcopy(&(av->av_dgid.gid_prefix), &(path->rgid_h),
sizeof (uint64_t));
bcopy(&(av->av_dgid.gid_guid), &(path->rgid_l),
sizeof (uint64_t));
} else {
path->rgid_l = 0x2;
path->flow_label = 0;
path->tclass = 0;
path->hop_limit = 0;
path->rgid_h = 0;
}
/* extract the default service level */
udav->sl = (HERMON_DEF_SCHED_SELECTION & 0x3C) >> 2;
return (DDI_SUCCESS);
}
/*
* hermon_get_addr_path()
* Context: Can be called from interrupt or base context.
*
* Note: Just like hermon_set_addr_path() above, this routine is used for two
* purposes. It is used to read in the Hermon UDAV fields, and it is used to
* read in the address path information for QPs. Because the two Hermon
* structures are similar, common fields can be read in here. But because
* they are slightly different, we pass an additional flag to indicate which
* type is being read.
*/
void
hermon_get_addr_path(hermon_state_t *state, hermon_hw_addr_path_t *path,
ibt_adds_vect_t *av, uint_t type)
{
uint_t gidtbl_sz;
_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*path))
_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*av))
av->av_src_path = path->mlid;
av->av_dlid = path->rlid;
/* Set "av_ipd" value from max_stat_rate */
switch (path->max_stat_rate) {
case 7: /* 1xSDR-2.5Gb/s injection rate */
av->av_srate = IBT_SRATE_2; break;
case 8: /* 4xSDR-10.0Gb/s injection rate */
av->av_srate = IBT_SRATE_10; break;
case 9: /* 12xSDR-30Gb/s injection rate */
av->av_srate = IBT_SRATE_30; break;
case 10: /* 1xDDR-5Gb/s injection rate */
av->av_srate = IBT_SRATE_5; break;
case 11: /* 4xDDR-20Gb/s injection rate */
av->av_srate = IBT_SRATE_20; break;
case 12: /* xQDR-40Gb/s injection rate */
av->av_srate = IBT_SRATE_40; break;
case 13: /* 12xDDR-60Gb/s injection rate */
av->av_srate = IBT_SRATE_60; break;
case 14: /* 8xQDR-80Gb/s injection rate */
av->av_srate = IBT_SRATE_80; break;
case 15: /* 12xQDR-120Gb/s injection rate */
av->av_srate = IBT_SRATE_120; break;
case 0: /* max */
av->av_srate = IBT_SRATE_NOT_SPECIFIED; break;
default: /* 1x injection rate */
av->av_srate = IBT_SRATE_1X;
}
/*
* Extract all "global" values regardless of the value in the GRH
* flag. Because "av_send_grh" is set only if "grh" is set, software
* knows to ignore the other "global" values as necessary. Note: SW
* does it this way to enable these query operations to return exactly
* the same params that were passed when the addr path was last written.
*/
av->av_send_grh = path->grh;
if (type == HERMON_ADDRPATH_QP) {
av->av_sgid_ix = path->mgid_index;
} else {
/*
* For Hermon UDAV, the "mgid_index" field is the index into
* a combined table (not a per-port table).
*/
gidtbl_sz = (1 << state->hs_queryport.log_max_gid);
av->av_sgid_ix = path->mgid_index - ((av->av_port_num - 1) *
gidtbl_sz);
av->av_port_num = ((hermon_hw_udav_t *)(void *)path)->portnum;
}
av->av_flow = path->flow_label;
av->av_tclass = path->tclass;
av->av_hop = path->hop_limit;
/* this is for alignment issue w/ the addr path struct in Hermon */
bcopy(&(path->rgid_h), &(av->av_dgid.gid_prefix), sizeof (uint64_t));
bcopy(&(path->rgid_l), &(av->av_dgid.gid_guid), sizeof (uint64_t));
}
/*
* hermon_portnum_is_valid()
* Context: Can be called from interrupt or base context.
*/
int
hermon_portnum_is_valid(hermon_state_t *state, uint_t portnum)
{
uint_t max_port;
max_port = state->hs_cfg_profile->cp_num_ports;
if ((portnum <= max_port) && (portnum != 0)) {
return (1);
} else {
return (0);
}
}
/*
* hermon_pkeyindex_is_valid()
* Context: Can be called from interrupt or base context.
*/
int
hermon_pkeyindex_is_valid(hermon_state_t *state, uint_t pkeyindx)
{
uint_t max_pkeyindx;
max_pkeyindx = 1 << state->hs_cfg_profile->cp_log_max_pkeytbl;
if (pkeyindx < max_pkeyindx) {
return (1);
} else {
return (0);
}
}
/*
* hermon_queue_alloc()
* Context: Can be called from interrupt or base context.
*/
int
hermon_queue_alloc(hermon_state_t *state, hermon_qalloc_info_t *qa_info,
uint_t sleepflag)
{
ddi_dma_attr_t dma_attr;
int (*callback)(caddr_t);
uint64_t realsize, alloc_mask;
int flag, status;
_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*qa_info))
/* Set the callback flag appropriately */
callback = (sleepflag == HERMON_SLEEP) ? DDI_DMA_SLEEP :
DDI_DMA_DONTWAIT;
/*
* Initialize many of the default DMA attributes. Then set additional
* alignment restrictions as necessary for the queue memory. Also
* respect the configured value for IOMMU bypass
*/
hermon_dma_attr_init(state, &dma_attr);
dma_attr.dma_attr_align = qa_info->qa_bind_align;
#ifdef __sparc
if (state->hs_cfg_profile->cp_iommu_bypass == HERMON_BINDMEM_BYPASS) {
dma_attr.dma_attr_flags = DDI_DMA_FORCE_PHYSICAL;
}
#endif
/* Allocate a DMA handle */
status = ddi_dma_alloc_handle(state->hs_dip, &dma_attr, callback, NULL,
&qa_info->qa_dmahdl);
if (status != DDI_SUCCESS) {
return (DDI_FAILURE);
}
/*
* Determine the amount of memory to allocate, depending on the values
* in "qa_bind_align" and "qa_alloc_align". The problem we are trying
* to solve here is that allocating a DMA handle with IOMMU bypass
* (DDI_DMA_FORCE_PHYSICAL) constrains us to only requesting alignments
* that are less restrictive than the page size. Since we may need
* stricter alignments on the memory allocated by ddi_dma_mem_alloc()
* (e.g. in Hermon QP work queue memory allocation), we use the
* following method to calculate how much additional memory to request,
* and we enforce our own alignment on the allocated result.
*/
alloc_mask = qa_info->qa_alloc_align - 1;
if (qa_info->qa_bind_align == qa_info->qa_alloc_align) {
realsize = qa_info->qa_size;
} else {
realsize = qa_info->qa_size + alloc_mask;
}
/*
* If we are to allocate the queue from system memory, then use
* ddi_dma_mem_alloc() to find the space. Otherwise, this is a
* host memory allocation, use ddi_umem_alloc(). In either case,
* return a pointer to the memory range allocated (including any
* necessary alignment adjustments), the "real" memory pointer,
* the "real" size, and a ddi_acc_handle_t to use when reading
* from/writing to the memory.
*/
if (qa_info->qa_location == HERMON_QUEUE_LOCATION_NORMAL) {
/* Allocate system memory for the queue */
status = ddi_dma_mem_alloc(qa_info->qa_dmahdl, realsize,
&state->hs_reg_accattr, DDI_DMA_CONSISTENT, callback, NULL,
(caddr_t *)&qa_info->qa_buf_real,
(size_t *)&qa_info->qa_buf_realsz, &qa_info->qa_acchdl);
if (status != DDI_SUCCESS) {
ddi_dma_free_handle(&qa_info->qa_dmahdl);
return (DDI_FAILURE);
}
/*
* Save temporary copy of the real pointer. (This may be
* modified in the last step below).
*/
qa_info->qa_buf_aligned = qa_info->qa_buf_real;
bzero(qa_info->qa_buf_real, qa_info->qa_buf_realsz);
} else { /* HERMON_QUEUE_LOCATION_USERLAND */
/* Allocate userland mappable memory for the queue */
flag = (sleepflag == HERMON_SLEEP) ? DDI_UMEM_SLEEP :
DDI_UMEM_NOSLEEP;
qa_info->qa_buf_real = ddi_umem_alloc(realsize, flag,
&qa_info->qa_umemcookie);
if (qa_info->qa_buf_real == NULL) {
ddi_dma_free_handle(&qa_info->qa_dmahdl);
return (DDI_FAILURE);
}
/*
* Save temporary copy of the real pointer. (This may be
* modified in the last step below).
*/
qa_info->qa_buf_aligned = qa_info->qa_buf_real;
}
/*
* The next to last step is to ensure that the final address
* ("qa_buf_aligned") has the appropriate "alloc" alignment
* restriction applied to it (if necessary).
*/
if (qa_info->qa_bind_align != qa_info->qa_alloc_align) {
qa_info->qa_buf_aligned = (uint32_t *)(uintptr_t)(((uintptr_t)
qa_info->qa_buf_aligned + alloc_mask) & ~alloc_mask);
}
/*
* The last step is to figure out the offset of the start relative
* to the first page of the region - will be used in the eqc/cqc
* passed to the HW
*/
qa_info->qa_pgoffs = (uint_t)((uintptr_t)
qa_info->qa_buf_aligned & HERMON_PAGEOFFSET);
return (DDI_SUCCESS);
}
/*
* hermon_queue_free()
* Context: Can be called from interrupt or base context.
*/
void
hermon_queue_free(hermon_qalloc_info_t *qa_info)
{
_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*qa_info))
/*
* Depending on how (i.e. from where) we allocated the memory for
* this queue, we choose the appropriate method for releasing the
* resources.
*/
if (qa_info->qa_location == HERMON_QUEUE_LOCATION_NORMAL) {
ddi_dma_mem_free(&qa_info->qa_acchdl);
} else if (qa_info->qa_location == HERMON_QUEUE_LOCATION_USERLAND) {
ddi_umem_free(qa_info->qa_umemcookie);
}
/* Always free the dma handle */
ddi_dma_free_handle(&qa_info->qa_dmahdl);
}
/*
* hermon_create_fmr_pool()
* Create a pool of FMRs.
* Context: Can be called from kernel context only.
*/
int
hermon_create_fmr_pool(hermon_state_t *state, hermon_pdhdl_t pd,
ibt_fmr_pool_attr_t *fmr_attr, hermon_fmrhdl_t *fmrpoolp)
{
hermon_fmrhdl_t fmrpool;
hermon_fmr_list_t *fmr, *fmr_next;
hermon_mrhdl_t mr;
int status;
int sleep;
int i;
sleep = (fmr_attr->fmr_flags & IBT_MR_SLEEP) ? HERMON_SLEEP :
HERMON_NOSLEEP;
if ((sleep == HERMON_SLEEP) &&
(sleep != HERMON_SLEEPFLAG_FOR_CONTEXT())) {
return (IBT_INVALID_PARAM);
}
fmrpool = (hermon_fmrhdl_t)kmem_zalloc(sizeof (*fmrpool), sleep);
if (fmrpool == NULL) {
status = IBT_INSUFF_RESOURCE;
goto fail;
}
_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*fmrpool))
mutex_init(&fmrpool->fmr_lock, NULL, MUTEX_DRIVER,
DDI_INTR_PRI(state->hs_intrmsi_pri));
mutex_init(&fmrpool->remap_lock, NULL, MUTEX_DRIVER,
DDI_INTR_PRI(state->hs_intrmsi_pri));
mutex_init(&fmrpool->dirty_lock, NULL, MUTEX_DRIVER,
DDI_INTR_PRI(state->hs_intrmsi_pri));
fmrpool->fmr_state = state;
fmrpool->fmr_flush_function = fmr_attr->fmr_func_hdlr;
fmrpool->fmr_flush_arg = fmr_attr->fmr_func_arg;
fmrpool->fmr_pool_size = 0;
fmrpool->fmr_max_pages = fmr_attr->fmr_max_pages_per_fmr;
fmrpool->fmr_page_sz = fmr_attr->fmr_page_sz;
fmrpool->fmr_dirty_watermark = fmr_attr->fmr_pool_size / 4;
fmrpool->fmr_dirty_len = 0;
fmrpool->fmr_remap_watermark = fmr_attr->fmr_pool_size / 32;
fmrpool->fmr_remap_len = 0;
fmrpool->fmr_flags = fmr_attr->fmr_flags;
fmrpool->fmr_stat_register = 0;
fmrpool->fmr_max_remaps = state->hs_cfg_profile->cp_fmr_max_remaps;
fmrpool->fmr_remap_gen = 1;
fmrpool->fmr_free_list_tail = &fmrpool->fmr_free_list;
fmrpool->fmr_dirty_list = NULL;
fmrpool->fmr_dirty_list_tail = &fmrpool->fmr_dirty_list;
fmrpool->fmr_remap_list = NULL;
fmrpool->fmr_remap_list_tail = &fmrpool->fmr_remap_list;
fmrpool->fmr_pool_size = fmrpool->fmr_free_len =
fmr_attr->fmr_pool_size;
for (i = 0; i < fmr_attr->fmr_pool_size; i++) {
status = hermon_mr_alloc_fmr(state, pd, fmrpool, &mr);
if (status != DDI_SUCCESS) {
goto fail2;
}
fmr = (hermon_fmr_list_t *)kmem_zalloc(
sizeof (hermon_fmr_list_t), sleep);
_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*fmr))
fmr->fmr = mr;
fmr->fmr_remaps = 0;
fmr->fmr_remap_gen = fmrpool->fmr_remap_gen;
fmr->fmr_pool = fmrpool;
_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*mr))
mr->mr_fmr = fmr;
if (!i) /* address of last entry's link */
fmrpool->fmr_free_list_tail = &fmr->fmr_next;
fmr->fmr_next = fmrpool->fmr_free_list;
fmrpool->fmr_free_list = fmr;
}
/* Set to return pool */
*fmrpoolp = fmrpool;
IBTF_DPRINTF_L2("fmr", "create_fmr_pool SUCCESS");
return (IBT_SUCCESS);
fail2:
for (fmr = fmrpool->fmr_free_list; fmr != NULL; fmr = fmr_next) {
_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*fmr))
fmr_next = fmr->fmr_next;
(void) hermon_mr_dealloc_fmr(state, &fmr->fmr);
kmem_free(fmr, sizeof (hermon_fmr_list_t));
}
kmem_free(fmrpool, sizeof (*fmrpool));
fail:
*fmrpoolp = NULL;
IBTF_DPRINTF_L2("fmr", "create_fmr_pool FAILED");
if (status == DDI_FAILURE) {
return (ibc_get_ci_failure(0));
} else {
return (status);
}
}
/*
* hermon_destroy_fmr_pool()
* Destroy an FMR pool and free all associated resources.
* Context: Can be called from kernel context only.
*/
int
hermon_destroy_fmr_pool(hermon_state_t *state, hermon_fmrhdl_t fmrpool)
{
hermon_fmr_list_t *fmr, *fmr_next;
mutex_enter(&fmrpool->fmr_lock);
hermon_fmr_cleanup(fmrpool);
for (fmr = fmrpool->fmr_free_list; fmr != NULL; fmr = fmr_next) {
fmr_next = fmr->fmr_next;
(void) hermon_mr_dealloc_fmr(state, &fmr->fmr);
kmem_free(fmr, sizeof (hermon_fmr_list_t));
--fmrpool->fmr_pool_size;
}
ASSERT(fmrpool->fmr_pool_size == 0);
mutex_exit(&fmrpool->fmr_lock);
mutex_destroy(&fmrpool->fmr_lock);
mutex_destroy(&fmrpool->dirty_lock);
mutex_destroy(&fmrpool->remap_lock);
kmem_free(fmrpool, sizeof (*fmrpool));
IBTF_DPRINTF_L2("fmr", "destroy_fmr_pool SUCCESS");
return (DDI_SUCCESS);
}
/*
* hermon_flush_fmr_pool()
* Ensure that all unmapped FMRs are fully invalidated.
* Context: Can be called from kernel context only.
*/
/* ARGSUSED */
int
hermon_flush_fmr_pool(hermon_state_t *state, hermon_fmrhdl_t fmrpool)
{
/*
* Force the unmapping of all entries on the dirty list, regardless of
* whether the watermark has been hit yet.
*/
/* grab the pool lock */
mutex_enter(&fmrpool->fmr_lock);
hermon_fmr_cleanup(fmrpool);
mutex_exit(&fmrpool->fmr_lock);
return (DDI_SUCCESS);
}
/*
* hermon_register_physical_fmr()
* Map memory into FMR
* Context: Can be called from interrupt or base context.
*/
int
hermon_register_physical_fmr(hermon_state_t *state, hermon_fmrhdl_t fmrpool,
ibt_pmr_attr_t *mem_pattr, hermon_mrhdl_t *mr,
ibt_pmr_desc_t *mem_desc_p)
{
hermon_fmr_list_t *fmr;
int status;
/* Check length */
if (mem_pattr->pmr_len < 1 || (mem_pattr->pmr_num_buf >
fmrpool->fmr_max_pages)) {
return (IBT_MR_LEN_INVALID);
}
mutex_enter(&fmrpool->fmr_lock);
if (fmrpool->fmr_free_list == NULL) {
if (hermon_fmr_verbose & 2)
IBTF_DPRINTF_L2("fmr", "register needs remap");
mutex_enter(&fmrpool->remap_lock);
if (fmrpool->fmr_remap_list) {
/* add to free list */
*(fmrpool->fmr_free_list_tail) =
fmrpool->fmr_remap_list;
fmrpool->fmr_remap_list = NULL;
fmrpool->fmr_free_list_tail =
fmrpool->fmr_remap_list_tail;
/* reset list */
fmrpool->fmr_remap_list_tail = &fmrpool->fmr_remap_list;
fmrpool->fmr_free_len += fmrpool->fmr_remap_len;
fmrpool->fmr_remap_len = 0;
}
mutex_exit(&fmrpool->remap_lock);
}
if (fmrpool->fmr_free_list == NULL) {
if (hermon_fmr_verbose & 2)
IBTF_DPRINTF_L2("fmr", "register needs cleanup");
hermon_fmr_cleanup(fmrpool);
}
/* grab next free entry */
fmr = fmrpool->fmr_free_list;
if (fmr == NULL) {
IBTF_DPRINTF_L2("fmr", "WARNING: no free fmr resource");
cmn_err(CE_CONT, "no free fmr resource\n");
mutex_exit(&fmrpool->fmr_lock);
return (IBT_INSUFF_RESOURCE);
}
if ((fmrpool->fmr_free_list = fmr->fmr_next) == NULL)
fmrpool->fmr_free_list_tail = &fmrpool->fmr_free_list;
fmr->fmr_next = NULL;
fmrpool->fmr_stat_register++;
mutex_exit(&fmrpool->fmr_lock);
_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*fmr))
status = hermon_mr_register_physical_fmr(state, mem_pattr, fmr->fmr,
mem_desc_p);
if (status != DDI_SUCCESS) {
return (status);
}
_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*fmr->fmr))
if (hermon_rdma_debug & 0x4)
IBTF_DPRINTF_L2("fmr", " reg: mr %p key %x",
fmr->fmr, fmr->fmr->mr_rkey);
_NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*fmr->fmr))
if (fmr->fmr_remap_gen != fmrpool->fmr_remap_gen) {
fmr->fmr_remap_gen = fmrpool->fmr_remap_gen;
fmr->fmr_remaps = 0;
}
fmr->fmr_remaps++;
*mr = (hermon_mrhdl_t)fmr->fmr;
return (DDI_SUCCESS);
}
/*
* hermon_deregister_fmr()
* Unmap FMR
* Context: Can be called from kernel context only.
*/
int
hermon_deregister_fmr(hermon_state_t *state, hermon_mrhdl_t mr)
{
hermon_fmrhdl_t fmrpool;
hermon_fmr_list_t *fmr, **fmrlast;
int len;
fmr = mr->mr_fmr;
_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*fmr))
fmrpool = fmr->fmr_pool;
/* mark as owned by software */
_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*(fmr->fmr)))
*(uint8_t *)(fmr->fmr->mr_mptrsrcp->hr_addr) = 0xF0;
if (fmr->fmr_remaps <
state->hs_cfg_profile->cp_fmr_max_remaps) {
/* add to remap list */
_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*(fmr->fmr)))
if (hermon_rdma_debug & 0x4)
IBTF_DPRINTF_L2("fmr", "dereg: mr %p key %x",
fmr->fmr, fmr->fmr->mr_rkey);
_NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*(fmr->fmr)))
mutex_enter(&fmrpool->remap_lock);
fmr->fmr_next = NULL;
*(fmrpool->fmr_remap_list_tail) = fmr;
fmrpool->fmr_remap_list_tail = &fmr->fmr_next;
fmrpool->fmr_remap_len++;
/* conditionally add remap list back to free list */
fmrlast = NULL;
if (fmrpool->fmr_remap_len >=
fmrpool->fmr_remap_watermark) {
fmr = fmrpool->fmr_remap_list;
fmrlast = fmrpool->fmr_remap_list_tail;
len = fmrpool->fmr_remap_len;
fmrpool->fmr_remap_len = 0;
fmrpool->fmr_remap_list = NULL;
fmrpool->fmr_remap_list_tail =
&fmrpool->fmr_remap_list;
}
mutex_exit(&fmrpool->remap_lock);
if (fmrlast) {
mutex_enter(&fmrpool->fmr_lock);
*(fmrpool->fmr_free_list_tail) = fmr;
fmrpool->fmr_free_list_tail = fmrlast;
fmrpool->fmr_free_len += len;
mutex_exit(&fmrpool->fmr_lock);
}
} else {
/* add to dirty list */
_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*(fmr->fmr)))
if (hermon_rdma_debug & 0x4)
IBTF_DPRINTF_L2("fmr", "dirty: mr %p key %x",
fmr->fmr, fmr->fmr->mr_rkey);
_NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*(fmr->fmr)))
mutex_enter(&fmrpool->dirty_lock);
fmr->fmr_next = NULL;
*(fmrpool->fmr_dirty_list_tail) = fmr;
fmrpool->fmr_dirty_list_tail = &fmr->fmr_next;
fmrpool->fmr_dirty_len++;
if (fmrpool->fmr_dirty_len >=
fmrpool->fmr_dirty_watermark) {
mutex_exit(&fmrpool->dirty_lock);
mutex_enter(&fmrpool->fmr_lock);
hermon_fmr_cleanup(fmrpool);
mutex_exit(&fmrpool->fmr_lock);
} else
mutex_exit(&fmrpool->dirty_lock);
}
return (DDI_SUCCESS);
}
/*
* hermon_fmr_cleanup()
* Context: Called from any context.
*/
static void
hermon_fmr_cleanup(hermon_fmrhdl_t fmrpool)
{
int status;
ASSERT(MUTEX_HELD(&fmrpool->fmr_lock));
if (fmrpool->fmr_stat_register == 0)
return;
fmrpool->fmr_stat_register = 0;
membar_producer();
if (hermon_fmr_verbose)
IBTF_DPRINTF_L2("fmr", "TPT_SYNC");
status = hermon_sync_tpt_cmd_post(fmrpool->fmr_state,
HERMON_CMD_NOSLEEP_SPIN);
if (status != HERMON_CMD_SUCCESS) {
cmn_err(CE_WARN, "fmr SYNC_TPT failed(%x)\n", status);
}
fmrpool->fmr_remap_gen++;
/* add everything back to the free list */
mutex_enter(&fmrpool->dirty_lock);
if (fmrpool->fmr_dirty_list) {
/* add to free list */
*(fmrpool->fmr_free_list_tail) = fmrpool->fmr_dirty_list;
fmrpool->fmr_dirty_list = NULL;
fmrpool->fmr_free_list_tail = fmrpool->fmr_dirty_list_tail;
/* reset list */
fmrpool->fmr_dirty_list_tail = &fmrpool->fmr_dirty_list;
fmrpool->fmr_free_len += fmrpool->fmr_dirty_len;
fmrpool->fmr_dirty_len = 0;
}
mutex_exit(&fmrpool->dirty_lock);
mutex_enter(&fmrpool->remap_lock);
if (fmrpool->fmr_remap_list) {
/* add to free list */
*(fmrpool->fmr_free_list_tail) = fmrpool->fmr_remap_list;
fmrpool->fmr_remap_list = NULL;
fmrpool->fmr_free_list_tail = fmrpool->fmr_remap_list_tail;
/* reset list */
fmrpool->fmr_remap_list_tail = &fmrpool->fmr_remap_list;
fmrpool->fmr_free_len += fmrpool->fmr_remap_len;
fmrpool->fmr_remap_len = 0;
}
mutex_exit(&fmrpool->remap_lock);
if (fmrpool->fmr_flush_function != NULL) {
(void) fmrpool->fmr_flush_function(
(ibc_fmr_pool_hdl_t)fmrpool,
fmrpool->fmr_flush_arg);
}
}