b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * CDDL HEADER START
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * The contents of this file are subject to the terms of the
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * Common Development and Distribution License (the "License").
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * You may not use this file except in compliance with the License.
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * or http://www.opensolaris.org/os/licensing.
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * See the License for the specific language governing permissions
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * and limitations under the License.
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * When distributing Covered Code, include this CDDL HEADER in each
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * If applicable, add the following below this CDDL HEADER, with the
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * fields enclosed by brackets "[]" replaced with your own identifying
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * information: Portions Copyright [yyyy] [name of copyright owner]
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * CDDL HEADER END
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved.
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * Module (static) info passed to IBTL during ibt_attach
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalanstatic ibt_clnt_modinfo_t eibnx_clnt_modinfo = {
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * Static function declarations
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalanstatic int eibnx_setup_txbufs(eibnx_thr_info_t *);
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalanstatic int eibnx_setup_rxbufs(eibnx_thr_info_t *);
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalanstatic int eibnx_join_solicit_mcg(eibnx_thr_info_t *);
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalanstatic int eibnx_join_advertise_mcg(eibnx_thr_info_t *);
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalanstatic void eibnx_rb_state_init(void);
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalanstatic void eibnx_rb_setup_txbufs(eibnx_thr_info_t *);
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalanstatic void eibnx_rb_setup_rxbufs(eibnx_thr_info_t *);
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalanstatic void eibnx_rb_join_solicit_mcg(eibnx_thr_info_t *);
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalanstatic void eibnx_rb_join_advertise_mcg(eibnx_thr_info_t *);
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * eibnx_ibt_init() is expected to be called during the nexus driver's
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * attach time; given that there is only one instance of the nexus
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * driver allowed, and no threads are active before the initialization
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * is complete, we don't really have to acquire any driver specific mutex
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * within this routine.
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * Do per-state initialization
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * Attach to IBTL
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan if ((ret = ibt_attach(&eibnx_clnt_modinfo, ss->nx_dip, ss,
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan ENX_DPRINTF_ERR("ibt_attach() failed, ret=%d", ret);
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * Get the list of HCA guids on the system
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan if ((num_hcas = ibt_get_hca_list(&hca_guids)) == 0) {
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan ENX_DPRINTF_VERBOSE("no HCAs found on the system");
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan if ((ret = ibt_detach(ss->nx_ibt_hdl)) != IBT_SUCCESS) {
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan ENX_DPRINTF_ERR("ibt_detach() failed, ret=%d", ret);
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * Open the HCAs and store the handles
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan for (i = 0; i < num_hcas; i++) {
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * If we cannot open a HCA, allocate a protection domain
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * on it or get portinfo on it, print an error and move on
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * to the next HCA. Otherwise, queue it up in our hca list
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan if ((hca = eibnx_prepare_hca(hca_guids[i])) == NULL)
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * Free the HCA guid list we've allocated via ibt_get_hca_list()
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * Put the hca list in the state structure
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * Register for subnet notices
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan ibt_register_subnet_notices(ss->nx_ibt_hdl,
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * Initialize synchronization primitives
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan mutex_init(&ss->nx_lock, NULL, MUTEX_DRIVER, NULL);
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan mutex_init(&ss->nx_nodeq_lock, NULL, MUTEX_DRIVER, NULL);
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan cv_init(&ss->nx_nodeq_cv, NULL, CV_DEFAULT, NULL);
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan mutex_init(&ss->nx_busop_lock, NULL, MUTEX_DRIVER, NULL);
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan cv_init(&ss->nx_busop_cv, NULL, CV_DEFAULT, NULL);
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * Initialize well-known mgids: there must be a better way to
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * do this instead of having to express every single gid as a
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * tuple of two 8-byte integer quantities.
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan enx_solicit_mgid.gid_prefix = EIB_GUID_SOLICIT_PREFIX;
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan enx_advertise_mgid.gid_prefix = EIB_GUID_ADVERTISE_PREFIX;
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * Start up the eoib node creation thread
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan kt = thread_create(NULL, 0, eibnx_create_eoib_node, NULL, 0,
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * Locate the two multicast groups: the All-EoIB-GWs-GID and
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * All-EoIB-ENodes-GID. Make sure the MTU is something that
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * we can work with and Qkey is as expected.
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan if ((info->ti_mcg_status & ENX_MCGS_FOUND) == ENX_MCGS_FOUND) {
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * Request GID defining this port
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * First, locate the multicast group to use for sending solicit
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * requests to the GW
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan bzero(&mcg_attr, sizeof (ibt_mcg_attr_t));
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan mcg_attr.mc_pkey = (ib_pkey_t)EIB_ADMIN_PKEY;
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan mcg_attr.mc_qkey = (ib_qkey_t)EIB_FIP_QKEY;
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan if ((ret = ibt_query_mcg(rgid, &mcg_attr, 1, &info->ti_solicit_mcg,
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan ENX_DPRINTF_WARN("solicit mcg (gid=%llx.%llx) not found, "
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan "ibt_query_mcg() returned %d", enx_solicit_mgid.gid_prefix,
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * Make sure the multicast mtu isn't bigger than the port mtu
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * and the multicast group's qkey is the same as EIB_FIP_QKEY.
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan if (info->ti_solicit_mcg->mc_mtu > pi->p_mtu) {
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan ENX_DPRINTF_WARN("solicit mcg (gid=%llx.%llx) mtu too big, "
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan "0x%x > 0x%x", enx_solicit_mgid.gid_prefix,
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan enx_solicit_mgid.gid_guid, info->ti_solicit_mcg->mc_mtu,
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan if (info->ti_solicit_mcg->mc_qkey != EIB_FIP_QKEY) {
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan ENX_DPRINTF_WARN("solicit mcg (gid=%llx.%llx) qkey bad, "
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan "actual=0x%x, expected=0x%x", enx_solicit_mgid.gid_prefix,
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan enx_solicit_mgid.gid_guid, info->ti_solicit_mcg->mc_qkey,
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * Now, locate the multicast group for receiving discover
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * advertisements from the GW
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan bzero(&mcg_attr, sizeof (ibt_mcg_attr_t));
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan mcg_attr.mc_pkey = (ib_pkey_t)EIB_ADMIN_PKEY;
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan mcg_attr.mc_qkey = (ib_qkey_t)EIB_FIP_QKEY;
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan if ((ret = ibt_query_mcg(rgid, &mcg_attr, 1, &info->ti_advertise_mcg,
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan ENX_DPRINTF_WARN("advertise mcg (gid=%llx.%llx) not found, "
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan "ibt_query_mcg() returned %d",
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * Verify the multicast group's mtu and qkey as before
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan if (info->ti_advertise_mcg->mc_mtu > pi->p_mtu) {
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan ENX_DPRINTF_WARN("advertise mcg (gid=%llx.%llx) mtu too big, "
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan "0x%x > 0x%x", enx_advertise_mgid.gid_prefix,
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan info->ti_advertise_mcg->mc_mtu, pi->p_mtu);
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan if (info->ti_advertise_mcg->mc_qkey != EIB_FIP_QKEY) {
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan ENX_DPRINTF_WARN("advertise mcg (gid=%llx.%llx) qkey bad, "
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan "actual=0x%x, expected=0x%x",
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan enx_advertise_mgid.gid_prefix, enx_advertise_mgid.gid_guid,
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan info->ti_advertise_mcg->mc_qkey, EIB_FIP_QKEY);
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan ibt_free_mcg_info(info->ti_advertise_mcg, 1);
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan ibt_free_mcg_info(info->ti_solicit_mcg, 1);
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * Allocate and setup a single completion queue for tx and rx
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * Get this HCA's attributes
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan ret = ibt_query_hca(info->ti_hca, &hca_attr);
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan ENX_DPRINTF_ERR("ibt_query_hca(hca_hdl=0x%llx) failed, ret=%d",
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * Allocate a completion queue for our sends and receives
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan cq_attr.cq_size = (hca_attr.hca_max_cq_sz < ENX_CQ_SIZE) ?
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan ret = ibt_alloc_cq(info->ti_hca, &cq_attr, &info->ti_cq_hdl, &sz);
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan ENX_DPRINTF_ERR("ibt_alloc_cq(hca_hdl=0x%llx, cq_sz=0x%lx) "
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan "failed, ret=%d", info->ti_hca, cq_attr.cq_size, ret);
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * Set up other parameters for collecting completion information
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan info->ti_wc = kmem_zalloc(sizeof (ibt_wc_t) * sz, KM_SLEEP);
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * Allocate and setup the UD channel parameters
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalaneibnx_setup_ud_channel(eibnx_thr_info_t *info)
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * Protect against arbitrary additions to the chan_alloc_args
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * and chan_query_attr structures (make sure the ones we don't
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * use are zero'd).
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan bzero(&alloc_attr, sizeof (ibt_ud_chan_alloc_args_t));
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan bzero(&query_attr, sizeof (ibt_ud_chan_query_attr_t));
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * This ud channel is not going to be used by the nexus driver
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * to send any LSO packets, so we won't need the IBT_USES_LSO flag.
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan alloc_attr.ud_hca_port_num = info->ti_pi->p_port_num;
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan ret = ibt_pkey2index(info->ti_hca, info->ti_pi->p_port_num,
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan (ib_pkey_t)EIB_ADMIN_PKEY, &(alloc_attr.ud_pkey_ix));
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan ENX_DPRINTF_ERR("ibt_pkey2index(hca_hdl=0x%llx, "
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan "port_num=0x%x, pkey=0x%x) failed, ret=%d",
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan ret = ibt_alloc_ud_channel(info->ti_hca, IBT_ACHAN_NO_FLAGS,
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan ENX_DPRINTF_ERR("ibt_alloc_ud_channel(hca_hdl=0x%llx, "
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan "cs_sq=0x%lx, cs_rq=0x%lx) failed, ret=%d",
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan ret = ibt_query_ud_channel(info->ti_chan, &query_attr);
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan ENX_DPRINTF_ERR("ibt_query_ud_channel(chan_hdl=0x%llx) "
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan if ((ret = ibt_free_channel(info->ti_chan)) != IBT_SUCCESS) {
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan ENX_DPRINTF_WARN("ibt_free_channel(chan_hdl=0x%llx) "
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * Set up the transmit buffers for communicating with the gateway. Since
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * the EoIB Nexus driver only exchanges control messages with the
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * gateway, we don't really need too much space.
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * Allocate for the tx buf
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan snd_p->tx_vaddr = (ib_vaddr_t)(uintptr_t)kmem_zalloc(tx_bufsz,
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * Register the memory region with IBTF for use
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan if ((ret = ibt_register_mr(info->ti_hca, info->ti_pd, &attr,
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan ENX_DPRINTF_ERR("ibt_register_mr() failed for tx "
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan "region (0x%llx, 0x%llx) with ret=%d",
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan kmem_free((void *)(uintptr_t)(snd_p->tx_vaddr), tx_bufsz);
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * Now setup the send wqes
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan buf = (uint8_t *)(uintptr_t)(snd_p->tx_vaddr);
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan for (i = 0; i < ENX_NUM_SWQE; i++) {
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * Allocate a UD destination handle
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan ret = ibt_alloc_ud_dest(info->ti_hca, IBT_UD_DEST_NO_FLAGS,
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan ENX_DPRINTF_ERR("ibt_alloc_ud_dest(hca_hdl=0x%llx) "
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * We set up everything in the send wqes except initialize
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * the UD destination and the state of the entry. The ds_len
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * should also be adjusted correctly. All this should be
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * done later in the appropriate routines, before posting.
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan swqe->qe_sgl.ds_va = (ib_vaddr_t)(uintptr_t)buf;
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan swqe->qe_wr.send.wr_id = (ibt_wrid_t)(uintptr_t)swqe;
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan swqe->qe_wr.send.wr_flags = IBT_WR_NO_FLAGS;
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan swqe->qe_wr.send.wr_opcode = IBT_WRC_SEND;
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan mutex_init(&swqe->qe_lock, NULL, MUTEX_DRIVER, NULL);
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * Set up bufs for receiving gateway advertisements
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * Allocate for the rx buf
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan rx_bufsz = ENX_NUM_RWQE * (mtu + ENX_GRH_SZ);
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan rcv_p->rx_vaddr = (ib_vaddr_t)(uintptr_t)kmem_zalloc(rx_bufsz,
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan attr.mr_flags = IBT_MR_SLEEP | IBT_MR_ENABLE_LOCAL_WRITE;
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan if ((ret = ibt_register_mr(info->ti_hca, info->ti_pd, &attr,
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan ENX_DPRINTF_ERR("ibt_register_mr() failed for rx "
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan "region (0x%llx, 0x%llx) with ret=%d",
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan kmem_free((void *)(uintptr_t)(rcv_p->rx_vaddr), rx_bufsz);
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan buf = (uint8_t *)(uintptr_t)(rcv_p->rx_vaddr);
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan for (i = 0; i < ENX_NUM_RWQE; i++) {
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan rwqe->qe_sgl.ds_va = (ib_vaddr_t)(uintptr_t)buf;
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan rwqe->qe_wr.recv.wr_id = (ibt_wrid_t)(uintptr_t)rwqe;
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan mutex_init(&rwqe->qe_lock, NULL, MUTEX_DRIVER, NULL);
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * Set up transmit and receive buffers and post the receive buffers
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan if (eibnx_setup_txbufs(info) != ENX_E_SUCCESS)
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan if (eibnx_setup_rxbufs(info) != ENX_E_SUCCESS) {
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan for (i = 0; i < ENX_NUM_RWQE; i++) {
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan rwqe->qe_flags |= (ENX_QEFL_INUSE | ENX_QEFL_POSTED);
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan ret = ibt_post_recv(info->ti_chan, &(rwqe->qe_wr.recv), 1,
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan ENX_DPRINTF_ERR("ibt_post_recv(chan_hdl=0x%llx) "
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan "(chan_hdl=0x%llx) failed, ret=%d",
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * Set up the completion queue handler. While we don't quit if we cannot
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * use soft interrupts, that path is really unreliable and untested.
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalaneibnx_setup_cq_handler(eibnx_thr_info_t *info)
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * We'll try to use a softintr if possible. If not, it's not
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * fatal, we'll try and use the completion handler directly from
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * the interrupt handler.
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan rv = ddi_intr_add_softint(ss->nx_dip, &info->ti_softint_hdl,
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan EIB_SOFTPRI_ADM, eibnx_comp_handler, info);
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan ENX_DPRINTF_WARN("ddi_intr_add_softint(dip=0x%llx) "
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan ibt_set_cq_handler(info->ti_cq_hdl, eibnx_comp_intr, info);
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan ret = ibt_enable_cq_notify(info->ti_cq_hdl, IBT_NEXT_COMPLETION);
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan ENX_DPRINTF_WARN("ibt_enable_cq_notify(cq_hdl=0x%llx) "
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan (void) ddi_intr_remove_softint(info->ti_softint_hdl);
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * Join the solicit multicast group (All-EoIB-GWs-GID) as a full member
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalaneibnx_join_solicit_mcg(eibnx_thr_info_t *info)
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan ib_gid_t rgid = info->ti_pi->p_sgid_tbl[0];
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan bzero(&mcg_attr, sizeof (ibt_mcg_attr_t));
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan mcg_attr.mc_qkey = (ib_qkey_t)EIB_FIP_QKEY;
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan mcg_attr.mc_pkey = (ib_pkey_t)EIB_ADMIN_PKEY;
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan mcg_attr.mc_join_state = IB_MC_JSTATE_FULL;
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan mcg_attr.mc_flow = info->ti_solicit_mcg->mc_adds_vect.av_flow;
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan mcg_attr.mc_tclass = info->ti_solicit_mcg->mc_adds_vect.av_tclass;
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan mcg_attr.mc_sl = info->ti_solicit_mcg->mc_adds_vect.av_srvl;
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan mcg_attr.mc_scope = IB_MC_SCOPE_SUBNET_LOCAL;
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * We only need to send to solicit mcg, so we only need to join
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * the multicast group, no need to attach our qp to it
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan ret = ibt_join_mcg(rgid, &mcg_attr, &mcg_info, NULL, NULL);
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan ENX_DPRINTF_ERR("ibt_join_mcg() failed for solicit "
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan "mgid=%llx.%llx, ret=%x", enx_solicit_mgid.gid_prefix,
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * We can throw away the old mcg info we got when we queried
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * for the mcg and use the new one. They both should be the
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * same, really.
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * Join and attach to the advertise multicast group (All-EoIB-ENodes-GID)
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * to receive unsolicitied advertisements from the gateways.
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalaneibnx_join_advertise_mcg(eibnx_thr_info_t *info)
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan ib_gid_t rgid = info->ti_pi->p_sgid_tbl[0];
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan bzero(&mcg_attr, sizeof (ibt_mcg_attr_t));
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan mcg_attr.mc_qkey = (ib_qkey_t)EIB_FIP_QKEY;
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan mcg_attr.mc_pkey = (ib_pkey_t)EIB_ADMIN_PKEY;
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan mcg_attr.mc_join_state = IB_MC_JSTATE_FULL;
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan mcg_attr.mc_flow = info->ti_advertise_mcg->mc_adds_vect.av_flow;
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan mcg_attr.mc_tclass = info->ti_advertise_mcg->mc_adds_vect.av_tclass;
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan mcg_attr.mc_sl = info->ti_advertise_mcg->mc_adds_vect.av_srvl;
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan mcg_attr.mc_scope = IB_MC_SCOPE_SUBNET_LOCAL;
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan ret = ibt_join_mcg(rgid, &mcg_attr, &mcg_info, NULL, NULL);
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan ENX_DPRINTF_ERR("ibt_join_mcg() failed for advertise "
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan "mgid=%llx.%llx, ret=%x", enx_advertise_mgid.gid_prefix,
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * We can throw away the old mcg info we got when we queried
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * for the mcg and use the new one. They both should be the
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * same, really.
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * Since we need to receive advertisements, we'll attach our qp
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * to the advertise mcg
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan ret = ibt_attach_mcg(info->ti_chan, info->ti_advertise_mcg);
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan ENX_DPRINTF_ERR("ibt_attach_mcg(chan_hdl=0x%llx, "
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan "advt_mcg=0x%llx) failed, ret=%d", info->ti_chan,
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * Join the multicast groups we're interested in
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * We should've located the mcg first
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan if ((info->ti_mcg_status & ENX_MCGS_FOUND) == 0) {
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * If we're already joined to the mcgs, we must leave first
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan if ((info->ti_mcg_status & ENX_MCGS_JOINED) == ENX_MCGS_JOINED) {
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * Join the two mcgs
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan if (eibnx_join_advertise_mcg(info) != ENX_E_SUCCESS) {
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan if (eibnx_join_solicit_mcg(info) != ENX_E_SUCCESS) {
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * Lookup the MCGs again and join them
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan if (eibnx_find_mgroups(info) != ENX_E_SUCCESS)
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan if (eibnx_join_mcgs(info) != ENX_E_SUCCESS)
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * Disable subnet notices callbacks
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan ibt_register_subnet_notices(ss->nx_ibt_hdl, NULL, NULL);
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * Remove the hca list from the state structure
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * For each HCA in the list, free up the portinfo/port structs,
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * free the pd, close the hca handle and release the hca struct.
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * If something goes wrong, try to put back whatever good remains
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * back on the hca list and return failure.
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan for (hca = hca_list; hca; hca = hca_next) {
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan if (eibnx_cleanup_hca(hca) != ENX_E_SUCCESS) {
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan if ((ret = ibt_detach(ss->nx_ibt_hdl)) != IBT_SUCCESS) {
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan ENX_DPRINTF_WARN("ibt_detach(ibt_hdl=0x%llx) "
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * Ask the eoib node creation thread to die and wait for
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * it to happen
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalaneibnx_rb_find_mgroups(eibnx_thr_info_t *info)
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan if ((info->ti_mcg_status & ENX_MCGS_FOUND) == ENX_MCGS_FOUND) {
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan ibt_free_mcg_info(info->ti_advertise_mcg, 1);
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan ibt_free_mcg_info(info->ti_solicit_mcg, 1);
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan kmem_free(info->ti_wc, sizeof (ibt_wc_t) * info->ti_cq_sz);
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan ENX_DPRINTF_WARN("ibt_free_cq(cq_hdl=0x%llx) "
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalaneibnx_rb_setup_ud_channel(eibnx_thr_info_t *info)
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan if ((ret = ibt_free_channel(info->ti_chan)) != IBT_SUCCESS) {
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan ENX_DPRINTF_WARN("ibt_free_channel(chan=0x%llx) "
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalaneibnx_rb_setup_txbufs(eibnx_thr_info_t *info)
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * Release any UD destination handle we may have allocated. Note that
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * the per swqe lock would've been initialized only if we were able to
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * allocate the UD dest handle.
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan for (i = 0; i < ENX_NUM_SWQE; i++) {
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan ibt_free_ud_dest(swqe->qe_wr.send.wr.ud.udwr_dest);
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan ENX_DPRINTF_WARN("ibt_free_ud_dest(dest=0x%llx)"
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan " failed, ret=%d",
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * Clear all the workq entries
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan bzero(snd_p->tx_wqe, sizeof (eibnx_wqe_t) * ENX_NUM_SWQE);
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * Clear Lkey and deregister any memory region we may have
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * registered earlier
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan if ((ret = ibt_deregister_mr(info->ti_hca,
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan ENX_DPRINTF_WARN("ibt_deregister_TXmr(hca_hdl=0x%llx,"
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan "mr=0x%llx) failed, ret=%d", info->ti_hca,
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * Release any memory allocated for the tx bufs
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan kmem_free((void *)(uintptr_t)(snd_p->tx_vaddr),
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalaneibnx_rb_setup_rxbufs(eibnx_thr_info_t *info)
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan for (i = 0; i < ENX_NUM_RWQE; i++) {
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan bzero(rcv_p->rx_wqe, sizeof (eibnx_wqe_t) * ENX_NUM_RWQE);
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan if ((ret = ibt_deregister_mr(info->ti_hca,
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan ENX_DPRINTF_WARN("ibt_deregister_RXmr(hca_hdl=0x%llx,"
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan "mr=0x%llx) failed, ret=%d", info->ti_hca,
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan kmem_free((void *)(uintptr_t)(rcv_p->rx_vaddr),
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalaneibnx_rb_setup_bufs(eibnx_thr_info_t *info)
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan if ((ret = ibt_flush_channel(info->ti_chan)) != IBT_SUCCESS) {
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan ENX_DPRINTF_WARN("ibt_flush_channel(chan_hdl=0x%llx) "
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalaneibnx_rb_setup_cq_handler(eibnx_thr_info_t *info)
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan ibt_set_cq_handler(info->ti_cq_hdl, NULL, NULL);
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan (void) ddi_intr_remove_softint(info->ti_softint_hdl);
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalaneibnx_rb_join_solicit_mcg(eibnx_thr_info_t *info)
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan ib_gid_t rgid = info->ti_pi->p_sgid_tbl[0];
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan ret = ibt_leave_mcg(rgid, enx_solicit_mgid,
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan ENX_DPRINTF_WARN("ibt_leave_mcg(slct_mgid=%llx.%llx) "
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan "failed, ret=%d", enx_solicit_mgid.gid_prefix,
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalaneibnx_rb_join_advertise_mcg(eibnx_thr_info_t *info)
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan ib_gid_t rgid = info->ti_pi->p_sgid_tbl[0];
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan ret = ibt_detach_mcg(info->ti_chan, info->ti_advertise_mcg);
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan ENX_DPRINTF_WARN("ibt_detach_mcg(chan_hdl=0x%llx, "
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan "advt_mcg=0x%llx) failed, ret=%d",
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan info->ti_chan, info->ti_advertise_mcg, ret);
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan ret = ibt_leave_mcg(rgid, enx_advertise_mgid,
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan ENX_DPRINTF_WARN("ibt_leave_mcg(advt_mgid=%llx.%llx) "
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan "failed, ret=%d", enx_advertise_mgid.gid_prefix,
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan if ((info->ti_mcg_status & ENX_MCGS_JOINED) == ENX_MCGS_JOINED) {
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan info->ti_mcg_status &= (~ENX_MCGS_JOINED);
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan ret = ibt_open_hca(ss->nx_ibt_hdl, hca_guid, &hca_hdl);
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan ENX_DPRINTF_ERR("ibt_open_hca(hca_guid=0x%llx) "
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan bzero(&hca_attr, sizeof (ibt_hca_attr_t));
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan if ((ret = ibt_query_hca(hca_hdl, &hca_attr)) != IBT_SUCCESS) {
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan ENX_DPRINTF_ERR("ibt_query_hca(hca_hdl=0x%llx, "
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan "hca_guid=0x%llx) failed, ret=%d",
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan if ((ret = ibt_close_hca(hca_hdl)) != IBT_SUCCESS) {
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan ENX_DPRINTF_WARN("ibt_close_hca(hca_hdl=0x%llx) "
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan ret = ibt_alloc_pd(hca_hdl, IBT_PD_NO_FLAGS, &pd_hdl);
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan ENX_DPRINTF_ERR("ibt_alloc_pd(hca_hdl=0x%llx, "
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan "hca_guid=0x%llx) failed, ret=%d",
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan if ((ret = ibt_close_hca(hca_hdl)) != IBT_SUCCESS) {
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan ENX_DPRINTF_WARN("ibt_close_hca(hca_hdl=0x%llx) "
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * We have all the information we want about this hca, create
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * a new struct and return it.
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan hca = kmem_zalloc(sizeof (eibnx_hca_t), KM_SLEEP);
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan for (i = 0; i < hca_attr.hca_nports; i++) {
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan ret = ibt_query_hca_ports(hca_hdl, i + 1, &pi,
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan ENX_DPRINTF_WARN("ibt_query_hca_ports(hca_hdl=0x%llx, "
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan "port=0x%x) failed, ret=%d", hca_hdl, i + 1, ret);
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan port = kmem_zalloc(sizeof (eibnx_port_t), KM_SLEEP);
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * If we couldn't query about any ports on the HCA, return failure
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan ENX_DPRINTF_ERR("all hca port queries failed for "
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan for (port = hca->hc_port; port; port = port_next) {
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan ibt_free_portinfo(port->po_pi, port->po_pi_size);
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan if ((ret = ibt_free_pd(hca->hc_hdl, hca->hc_pd)) != IBT_SUCCESS) {
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan ENX_DPRINTF_WARN("ibt_free_pd(hca_hdl=0x%lx, pd_hd=0x%lx) "
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan "failed, ret=%d", hca->hc_hdl, hca->hc_pd, ret);
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan if ((ret = ibt_close_hca(hca->hc_hdl)) != IBT_SUCCESS) {
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan ENX_DPRINTF_WARN("ibt_close_hca(hca_hdl=0x%lx) failed, "