b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri * CDDL HEADER START
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri * The contents of this file are subject to the terms of the
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri * Common Development and Distribution License (the "License").
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri * You may not use this file except in compliance with the License.
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri * See the License for the specific language governing permissions
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri * and limitations under the License.
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri * When distributing Covered Code, include this CDDL HEADER in each
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri * If applicable, add the following below this CDDL HEADER, with the
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri * fields enclosed by brackets "[]" replaced with your own identifying
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri * information: Portions Copyright [yyyy] [name of copyright owner]
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri * CDDL HEADER END
00d1d19828f3122eb24ae7a68175ba64744f8366Suhasini Peddada * Copyright (c) 2006, 2010, Oracle and/or its affiliates. All rights reserved.
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri * Copyright (c) 2005 SilverStorm Technologies, Inc. All rights reserved.
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri * This software is available to you under a choice of one of two
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri * licenses. You may choose to be licensed under the terms of the GNU
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri * General Public License (GPL) Version 2, available from the file
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri * COPYING in the main directory of this source tree, or the
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri * OpenIB.org BSD license below:
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri * Redistribution and use in source and binary forms, with or
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri * without modification, are permitted provided that the following
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri * conditions are met:
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri * - Redistributions of source code must retain the above
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri * copyright notice, this list of conditions and the following
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri * disclaimer.
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri * - Redistributions in binary form must reproduce the above
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri * copyright notice, this list of conditions and the following
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri * disclaimer in the documentation and/or other materials
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri * provided with the distribution.
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri * SOFTWARE.
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri * Sun elects to include this software in Sun product
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri * under the OpenIB BSD license.
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri * POSSIBILITY OF SUCH DAMAGE.
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiristatic void rds_async_handler(void *clntp, ibt_hca_hdl_t hdl,
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri/* performance tunables */
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri /* The SQ size should not be more than that supported by the HCA */
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri if (((MaxDataSendBuffers + RDS_NUM_ACKS) > hattrp->hca_max_chan_sz) ||
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri ((MaxDataSendBuffers + RDS_NUM_ACKS) > hattrp->hca_max_cq_sz)) {
7424242225ff3dee57fcc8963bf073b4ba0297fdagiri RDS_DPRINTF2("RDSIB", "MaxDataSendBuffers + %d is greater "
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri "than that supported by the HCA driver "
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri "(%d + %d > %d or %d), lowering it to a supported value.",
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri /* The RQ size should not be more than that supported by the HCA */
7424242225ff3dee57fcc8963bf073b4ba0297fdagiri RDS_DPRINTF2("RDSIB", "MaxDataRecvBuffers is greater than that "
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri "supported by the HCA driver (%d > %d or %d), lowering it "
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri /* The SQ size should not be more than that supported by the HCA */
7424242225ff3dee57fcc8963bf073b4ba0297fdagiri RDS_DPRINTF2("RDSIB", "MaxCtrlSendBuffers is greater than that "
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri "supported by the HCA driver (%d > %d or %d), lowering it "
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri /* The RQ size should not be more than that supported by the HCA */
7424242225ff3dee57fcc8963bf073b4ba0297fdagiri RDS_DPRINTF2("RDSIB", "MaxCtrlRecvBuffers is greater than that "
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri "supported by the HCA driver (%d > %d or %d), lowering it "
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri /* The MaxRecvMemory should be less than that supported by the HCA */
5763ba1e357fad1d57b5875c499307b7ea6e2cd4agiri if ((NDataRX * RdsPktSize) > hattrp->hca_max_memr_len) {
7424242225ff3dee57fcc8963bf073b4ba0297fdagiri RDS_DPRINTF2("RDSIB", "MaxRecvMemory is greater than that "
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri "supported by the HCA driver (%d > %d), lowering it to %d",
00a3eaf3896a33935e11fd5c5fb5c1714225c067Ramaswamy Tummala/* Return hcap, given the hca guid */
00a3eaf3896a33935e11fd5c5fb5c1714225c067Ramaswamy Tummala RDS_DPRINTF4("rds_lkup_hca", "Enter: statep: 0x%p "
00a3eaf3896a33935e11fd5c5fb5c1714225c067Ramaswamy Tummala rw_enter(&rdsib_statep->rds_hca_lock, RW_READER);
00a3eaf3896a33935e11fd5c5fb5c1714225c067Ramaswamy Tummala while ((hcap != NULL) && (hcap->hca_guid != hca_guid)) {
00a3eaf3896a33935e11fd5c5fb5c1714225c067Ramaswamy Tummala RDS_DPRINTF2("rdsib_init_hca", "enter: HCA 0x%llx", hca_guid);
00a3eaf3896a33935e11fd5c5fb5c1714225c067Ramaswamy Tummala /* Do a HCA lookup */
00a3eaf3896a33935e11fd5c5fb5c1714225c067Ramaswamy Tummala if (hcap != NULL && hcap->hca_hdl != NULL) {
00a3eaf3896a33935e11fd5c5fb5c1714225c067Ramaswamy Tummala * This can happen if we get IBT_HCA_ATTACH_EVENT on an HCA
00a3eaf3896a33935e11fd5c5fb5c1714225c067Ramaswamy Tummala * that we have already opened. Just return NULL so that
00a3eaf3896a33935e11fd5c5fb5c1714225c067Ramaswamy Tummala * we'll not end up reinitializing the HCA again.
00a3eaf3896a33935e11fd5c5fb5c1714225c067Ramaswamy Tummala RDS_DPRINTF2("rdsib_init_hca", "HCA already initialized");
00a3eaf3896a33935e11fd5c5fb5c1714225c067Ramaswamy Tummala RDS_DPRINTF2("rdsib_init_hca", "New HCA is added");
00a3eaf3896a33935e11fd5c5fb5c1714225c067Ramaswamy Tummala hcap = (rds_hca_t *)kmem_zalloc(sizeof (rds_hca_t), KM_SLEEP);
00a3eaf3896a33935e11fd5c5fb5c1714225c067Ramaswamy Tummala ret = ibt_open_hca(rdsib_statep->rds_ibhdl, hca_guid,
00a3eaf3896a33935e11fd5c5fb5c1714225c067Ramaswamy Tummala "ibt_open_hca: 0x%llx returned IBT_HCA_IN_USE",
00a3eaf3896a33935e11fd5c5fb5c1714225c067Ramaswamy Tummala "ibt_open_hca: 0x%llx failed: %d", hca_guid, ret);
00a3eaf3896a33935e11fd5c5fb5c1714225c067Ramaswamy Tummala ret = ibt_query_hca(hcap->hca_hdl, &hcap->hca_attr);
00a3eaf3896a33935e11fd5c5fb5c1714225c067Ramaswamy Tummala "Query HCA: 0x%llx failed: %d", hca_guid, ret);
00a3eaf3896a33935e11fd5c5fb5c1714225c067Ramaswamy Tummala ret = ibt_query_hca_ports(hcap->hca_hdl, 0,
00a3eaf3896a33935e11fd5c5fb5c1714225c067Ramaswamy Tummala &hcap->hca_pinfop, &hcap->hca_nports, &hcap->hca_pinfo_sz);
00a3eaf3896a33935e11fd5c5fb5c1714225c067Ramaswamy Tummala "Query HCA 0x%llx ports failed: %d", hca_guid,
00a3eaf3896a33935e11fd5c5fb5c1714225c067Ramaswamy Tummala /* Only one PD per HCA is allocated, so do it here */
00a3eaf3896a33935e11fd5c5fb5c1714225c067Ramaswamy Tummala ret = ibt_alloc_pd(hcap->hca_hdl, IBT_PD_NO_FLAGS,
00a3eaf3896a33935e11fd5c5fb5c1714225c067Ramaswamy Tummala "ibt_alloc_pd 0x%llx failed: %d", hca_guid, ret);
00a3eaf3896a33935e11fd5c5fb5c1714225c067Ramaswamy Tummala (void) ibt_free_portinfo(hcap->hca_pinfop,
00a3eaf3896a33935e11fd5c5fb5c1714225c067Ramaswamy Tummala rdsib_validate_chan_sizes(&hcap->hca_attr);
0c19630b1592aa30d3e4d9db1a2a8cf9a91c0e72agiri /* To minimize stale connections after ungraceful reboots */
00a3eaf3896a33935e11fd5c5fb5c1714225c067Ramaswamy Tummala rw_enter(&rdsib_statep->rds_hca_lock, RW_WRITER);
00a3eaf3896a33935e11fd5c5fb5c1714225c067Ramaswamy Tummala /* this is a new HCA, add it to the list */
00a3eaf3896a33935e11fd5c5fb5c1714225c067Ramaswamy Tummala hcap->hca_nextp = rdsib_statep->rds_hcalistp;
00a3eaf3896a33935e11fd5c5fb5c1714225c067Ramaswamy Tummala RDS_DPRINTF2("rdsib_init_hca", "return: HCA 0x%llx", hca_guid);
d99cb22f7f0de8584336bda08cb86c562ffbab55agiri * Called from attach
d99cb22f7f0de8584336bda08cb86c562ffbab55agiri RDS_DPRINTF2("rdsib_initialize_ib", "enter: statep %p", rdsib_statep);
d99cb22f7f0de8584336bda08cb86c562ffbab55agiri "RDS Statep not initialized");
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri return (-1);
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri /* How many hcas are there? */
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri if (nhcas == 0) {
d99cb22f7f0de8584336bda08cb86c562ffbab55agiri RDS_DPRINTF2("rdsib_initialize_ib", "No IB HCAs Available");
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri return (-1);
d99cb22f7f0de8584336bda08cb86c562ffbab55agiri RDS_DPRINTF3("rdsib_initialize_ib", "Number of HCAs: %d", nhcas);
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri /* Register with IBTF */
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri ret = ibt_attach(&rds_ib_modinfo, rdsib_dev_info, rdsib_statep,
d99cb22f7f0de8584336bda08cb86c562ffbab55agiri RDS_DPRINTF2("rdsib_initialize_ib", "ibt_attach failed: %d",
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri return (-1);
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri * Open each HCA and gather its information. Don't care about HCAs
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri * that cannot be opened. It is OK as long as atleast one HCA can be
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri * Initialize a HCA only if all the information is available.
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri /* free the HCA list, we are done with it */
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri if (hcaix == 0) {
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri /* Failed to Initialize even one HCA */
d99cb22f7f0de8584336bda08cb86c562ffbab55agiri RDS_DPRINTF2("rdsib_initialize_ib", "No HCAs are initialized");
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri return (-1);
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri RDS_DPRINTF2("rdsib_open_ib", "HCAs %d/%d failed to initialize",
d99cb22f7f0de8584336bda08cb86c562ffbab55agiri RDS_DPRINTF2("rdsib_initialize_ib", "return: statep %p", rdsib_statep);
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri return (0);
d99cb22f7f0de8584336bda08cb86c562ffbab55agiri * Called from detach
d99cb22f7f0de8584336bda08cb86c562ffbab55agiri RDS_DPRINTF2("rdsib_deinitialize_ib", "enter: statep %p", rdsib_statep);
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri /* close and destroy all the sessions */
c1f8b08e52d9b30bd55daeac694e3a7f50d3cd21agiri /* Release all HCA resources */
00a3eaf3896a33935e11fd5c5fb5c1714225c067Ramaswamy Tummala RDS_DPRINTF2("rdsib_deinitialize_ib", "HCA List: %p, NHCA: %d",
00a3eaf3896a33935e11fd5c5fb5c1714225c067Ramaswamy Tummala rdsib_statep->rds_hcalistp, rdsib_statep->rds_nhcas);
00a3eaf3896a33935e11fd5c5fb5c1714225c067Ramaswamy Tummala ret = ibt_free_pd(hcap->hca_hdl, hcap->hca_pdhdl);
00a3eaf3896a33935e11fd5c5fb5c1714225c067Ramaswamy Tummala (void) ibt_free_portinfo(hcap->hca_pinfop,
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri /* Deregister with IBTF */
d99cb22f7f0de8584336bda08cb86c562ffbab55agiri RDS_DPRINTF2("rdsib_deinitialize_ib", "return: statep %p",
d99cb22f7f0de8584336bda08cb86c562ffbab55agiri * Called on open of first RDS socket
d99cb22f7f0de8584336bda08cb86c562ffbab55agiri RDS_DPRINTF2("rdsib_open_ib", "enter: statep %p", rdsib_statep);
d99cb22f7f0de8584336bda08cb86c562ffbab55agiri /* Enable incoming connection requests */
d99cb22f7f0de8584336bda08cb86c562ffbab55agiri "Service registration failed");
d99cb22f7f0de8584336bda08cb86c562ffbab55agiri return (-1);
d99cb22f7f0de8584336bda08cb86c562ffbab55agiri /* bind the service on all available ports */
d99cb22f7f0de8584336bda08cb86c562ffbab55agiri if (ret != 0) {
d99cb22f7f0de8584336bda08cb86c562ffbab55agiri RDS_DPRINTF2("rdsib_open_ib", "return: statep %p", rdsib_statep);
d99cb22f7f0de8584336bda08cb86c562ffbab55agiri return (0);
d99cb22f7f0de8584336bda08cb86c562ffbab55agiri * Called when all ports are closed.
d99cb22f7f0de8584336bda08cb86c562ffbab55agiri RDS_DPRINTF2("rdsib_close_ib", "enter: statep %p", rdsib_statep);
d99cb22f7f0de8584336bda08cb86c562ffbab55agiri /* Disable incoming connection requests */
d99cb22f7f0de8584336bda08cb86c562ffbab55agiri ret = ibt_unbind_all_services(rdsib_statep->rds_srvhdl);
d99cb22f7f0de8584336bda08cb86c562ffbab55agiri if (ret != 0) {
d99cb22f7f0de8584336bda08cb86c562ffbab55agiri if (ret != 0) {
d99cb22f7f0de8584336bda08cb86c562ffbab55agiri RDS_DPRINTF2("rdsib_close_ib", "return: statep %p", rdsib_statep);
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri/* Return hcap, given the hca guid */
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri RDS_DPRINTF4("rds_get_hcap", "rds_get_hcap: Enter: statep: 0x%p "
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri while ((hcap != NULL) && (hcap->hca_guid != hca_guid)) {
00a3eaf3896a33935e11fd5c5fb5c1714225c067Ramaswamy Tummala * don't let anyone use this HCA until the RECV memory
00a3eaf3896a33935e11fd5c5fb5c1714225c067Ramaswamy Tummala * is registered with this HCA
00a3eaf3896a33935e11fd5c5fb5c1714225c067Ramaswamy Tummala (hcap->hca_state == RDS_HCA_STATE_MEM_REGISTERED)) {
00a3eaf3896a33935e11fd5c5fb5c1714225c067Ramaswamy Tummala "HCA (0x%p, 0x%llx) is not initialized", hcap, hca_guid);
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri/* Return hcap, given a gid */
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri RDS_DPRINTF4("rds_gid_to_hcap", "Enter: statep: 0x%p gid: %llx:%llx",
00a3eaf3896a33935e11fd5c5fb5c1714225c067Ramaswamy Tummala * don't let anyone use this HCA until the RECV memory
00a3eaf3896a33935e11fd5c5fb5c1714225c067Ramaswamy Tummala * is registered with this HCA
00a3eaf3896a33935e11fd5c5fb5c1714225c067Ramaswamy Tummala if (hcap->hca_state != RDS_HCA_STATE_MEM_REGISTERED) {
00a3eaf3896a33935e11fd5c5fb5c1714225c067Ramaswamy Tummala "HCA (0x%p, 0x%llx) is not initialized",
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri/* This is called from the send CQ handler */
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri RDS_DPRINTF4("rds_send_acknowledgement", "Enter EP(%p)", ep);
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri * The previous ACK completed successfully, send the next one
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri * if more messages were received after sending the last ACK
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri if (ep->ep_rbufid != *(uintptr_t *)(uintptr_t)ep->ep_ackds.ds_va) {
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri *(uintptr_t *)(uintptr_t)ep->ep_ackds.ds_va = ep->ep_rbufid;
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri /* send acknowledgement */
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri ret = ibt_post_send(ep->ep_chanhdl, &ep->ep_ackwr, 1, &ix);
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri "EP(%p): ibt_post_send for acknowledgement "
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri "failed: %d, SQ depth: %d",
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri /* ACKed all messages, no more to ACK */
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri RDS_DPRINTF4("rds_send_acknowledgement", "Return EP(%p)", ep);
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagirirds_poll_ctrl_completions(ibt_cq_hdl_t cq, rds_ep_t *ep)
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri RDS_DPRINTF4("rds_poll_ctrl_completions", "Enter: EP(%p)", ep);
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri /* Free the buffer */
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri /* Receive completion failure */
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri "EP(%p) CQ(%p) BP(%p): WC Error Status: %d",
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri /* there is one less in the RQ */
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri /* Time to post more buffers into the RQ */
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri rds_post_recv_buf, (void *)ep->ep_chanhdl, DDI_NOSLEEP);
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri RDS_DPRINTF4("rds_poll_ctrl_completions", "Return: EP(%p)", ep);
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri/* Post recv WRs into the RQ. Assumes the ep->refcnt is already incremented */
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri RDS_DPRINTF4("rds_post_recv_buf", "Enter: CHAN(%p)", chanhdl);
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri /* get the hcap for the HCA hosting this channel */
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri RDS_DPRINTF2("rds_post_recv_buf", "HCA (0x%llx) not found",
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri /* Make sure the session is still connected */
00a3eaf3896a33935e11fd5c5fb5c1714225c067Ramaswamy Tummala (sp->session_state != RDS_SESSION_STATE_CONNECTED) &&
00a3eaf3896a33935e11fd5c5fb5c1714225c067Ramaswamy Tummala (sp->session_state != RDS_SESSION_STATE_HCA_CLOSING)) {
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri RDS_DPRINTF2("rds_post_recv_buf", "EP(%p): Session is not "
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri /* how many can be posted */
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri if (nspace == 0) {
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri RDS_DPRINTF2(LABEL, "EP(%p): No Recv buffers available", ep);
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri /* try again later */
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri ret = ddi_taskq_dispatch(rds_taskq, rds_post_recv_buf,
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri wrp = kmem_zalloc(RDS_POST_FEW_ATATIME * sizeof (ibt_recv_wr_t),
7424242225ff3dee57fcc8963bf073b4ba0297fdagiri RDS_DPRINTF2(LABEL, "ibt_post_recv for %d WRs failed: "
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri if (npost != 0) {
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri * sometimes, the recv WRs can get consumed as soon as they are
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri * posted. In that case, taskq thread to post more WRs to the RQ will
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri * not be scheduled as the taskqpending flag is still set.
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri kmem_free(wrp, RDS_POST_FEW_ATATIME * sizeof (ibt_recv_wr_t));
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri RDS_DPRINTF4("rds_post_recv_buf", "Return: EP(%p)", ep);
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagirirds_poll_data_completions(ibt_cq_hdl_t cq, rds_ep_t *ep)
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri RDS_DPRINTF4("rds_poll_data_completions", "Enter: EP(%p)", ep);
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri /* free the buffer */
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri /* Receive completion failure */
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri "EP(%p) CQ(%p) BP(%p): WC Error Status: %d",
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri /* there is one less in the RQ */
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri /* Time to post more buffers into the RQ */
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri rds_post_recv_buf, (void *)ep->ep_chanhdl, DDI_NOSLEEP);
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri RDS_DPRINTF5(LABEL, "Message Received: sendIP: 0x%x recvIP: 0x%x "
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri "sendport: %d recvport: %d npkts: %d pktno: %d", ep->ep_remip,
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri /* single pkt or last packet */
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri /* last packet of a segmented message */
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri /* single packet */
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri /* multi-pkt msg */
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri /* first packet */
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri /* intermediate packet */
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri RDS_DPRINTF4("rds_poll_data_completions", "Return: EP(%p)", ep);
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri RDS_DPRINTF4("rds_recvcq_handler", "enter: EP(%p)", ep);
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri /* enable the CQ */
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri RDS_DPRINTF2(LABEL, "EP(%p) CQ(%p): ibt_enable_cq_notify "
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri RDS_DPRINTF4("rds_recvcq_handler", "Return: EP(%p)", ep);
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagirirds_poll_send_completions(ibt_cq_hdl_t cq, rds_ep_t *ep, boolean_t lock)
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri RDS_DPRINTF4("rds_poll_send_completions", "Enter EP(%p)", ep);
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri ret = ibt_poll_cq(cq, wc, RDS_NUM_DATA_SEND_WCS, &nret);
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri "ibt_poll_cq returned: IBT_CQ_EMPTY",
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri } else if (wc[ix].wc_status == IBT_WC_WR_FLUSHED_ERR) {
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri /* don't let anyone send anymore */
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri /* Make this the active end */
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri RDS_DPRINTF5("rds_poll_send_completions", "Npolled: %d send_error: %d",
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri /* put the buffers to the pool */
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri if (npolled != 0) {
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri RDS_DPRINTF4("rds_poll_send_completions", "Return EP(%p)", ep);
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri RDS_DPRINTF4("rds_sendcq_handler", "Enter: EP(%p)", ep);
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri /* enable the CQ */
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri RDS_DPRINTF2(LABEL, "EP(%p) CQ(%p): ibt_enable_cq_notify "
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri RDS_DPRINTF4("rds_sendcq_handler", "Return: EP(%p)", ep);
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri RDS_DPRINTF2("rds_ep_free_rc_channel", "EP(%p) - Enter", ep);
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri /* free the QP */
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri /* wait until the RQ is empty */
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri /* free the Send CQ */
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri "EP(%p) - for sendcq, ibt_free_cq returned %d",
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri /* free the Recv CQ */
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri "EP(%p) - for recvcq, ibt_free_cq returned %d",
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri RDS_DPRINTF2("rds_ep_free_rc_channel", "EP(%p) - Return", ep);
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri/* Allocate resources for RC channel */
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagirirds_ep_alloc_rc_channel(rds_ep_t *ep, uint8_t hca_port)
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri RDS_DPRINTF4("rds_ep_alloc_rc_channel", "Enter: 0x%p port: %d",
c1f8b08e52d9b30bd55daeac694e3a7f50d3cd21agiri /* Update the EP with the right IP address and HCA guid */
c1f8b08e52d9b30bd55daeac694e3a7f50d3cd21agiri hcap = rds_gid_to_hcap(rdsib_statep, sp->session_lgid);
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri /* reset taskqpending flag here */
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri chanargs.rc_sizes.cs_sq = MaxDataSendBuffers + RDS_NUM_ACKS;
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri /* returned size is always greater than the requested size */
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri (void) ibt_set_cq_handler(ep->ep_sendcq, rds_sendcq_handler,
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri /* returned size is always greater than the requested size */
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri (void) ibt_set_cq_handler(ep->ep_recvcq, rds_recvcq_handler,
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri ret = ibt_enable_cq_notify(ep->ep_recvcq, rds_wc_signal);
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri chanargs.rc_control = IBT_CEP_RDMA_RD | IBT_CEP_RDMA_WR |
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri /* Chan private should contain the ep */
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri RDS_DPRINTF4("rds_ep_alloc_rc_channel", "Return: 0x%p", chanhdl);
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri/* Return node guid given a port gid */
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri RDS_DPRINTF4("rds_gid_to_node_guid", "Enter: gid: %llx:%llx",
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri RDS_DPRINTF2(LABEL, "ibt_gid_node_info for gid: %llx:%llx "
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri return (0LL);
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri RDS_DPRINTF4("rds_gid_to_node_guid", "Return: Node guid: %llx",
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagirirds_handle_portup_event(rds_state_t *statep, ibt_hca_hdl_t hdl,
c1f8b08e52d9b30bd55daeac694e3a7f50d3cd21agiri "Enter: GUID: 0x%llx Statep: %p", event->ev_hca_guid, statep);
00a3eaf3896a33935e11fd5c5fb5c1714225c067Ramaswamy Tummala rw_enter(&statep->rds_hca_lock, RW_WRITER);
00a3eaf3896a33935e11fd5c5fb5c1714225c067Ramaswamy Tummala while ((hcap != NULL) && (hcap->hca_guid != event->ev_hca_guid)) {
c1f8b08e52d9b30bd55daeac694e3a7f50d3cd21agiri RDS_DPRINTF2("rds_handle_portup_event", "HCA: 0x%llx is "
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri ret = ibt_query_hca_ports(hdl, 0, &newpinfop, &nport, &newsize);
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri RDS_DPRINTF2(LABEL, "ibt_query_hca_ports failed: %d", ret);
00a3eaf3896a33935e11fd5c5fb5c1714225c067Ramaswamy Tummala (void) ibt_free_portinfo(oldpinfop, oldsize);
00a3eaf3896a33935e11fd5c5fb5c1714225c067Ramaswamy Tummala /* If RDS service is not registered then no bind is needed */
00a3eaf3896a33935e11fd5c5fb5c1714225c067Ramaswamy Tummala "RDS Service is not registered, so no action needed");
00a3eaf3896a33935e11fd5c5fb5c1714225c067Ramaswamy Tummala * If the service was previously bound on this port and
00a3eaf3896a33935e11fd5c5fb5c1714225c067Ramaswamy Tummala * if this port has changed state down and now up, we do not
00a3eaf3896a33935e11fd5c5fb5c1714225c067Ramaswamy Tummala * need to bind the service again. The bind is expected to
00a3eaf3896a33935e11fd5c5fb5c1714225c067Ramaswamy Tummala * persist across state changes. If the service was never bound
00a3eaf3896a33935e11fd5c5fb5c1714225c067Ramaswamy Tummala * before then we bind it this time.
00a3eaf3896a33935e11fd5c5fb5c1714225c067Ramaswamy Tummala if (hcap->hca_bindhdl[event->ev_port - 1] == NULL) {
00a3eaf3896a33935e11fd5c5fb5c1714225c067Ramaswamy Tummala /* structure copy */
00a3eaf3896a33935e11fd5c5fb5c1714225c067Ramaswamy Tummala gid = newpinfop[event->ev_port - 1].p_sgid_tbl[0];
00a3eaf3896a33935e11fd5c5fb5c1714225c067Ramaswamy Tummala /* bind RDS service on the port, pass statep as cm_private */
00a3eaf3896a33935e11fd5c5fb5c1714225c067Ramaswamy Tummala ret = ibt_bind_service(statep->rds_srvhdl, gid, NULL, statep,
00a3eaf3896a33935e11fd5c5fb5c1714225c067Ramaswamy Tummala "Bind service for HCA: 0x%llx Port: %d "
00a3eaf3896a33935e11fd5c5fb5c1714225c067Ramaswamy Tummala "gid %llx:%llx returned: %d", event->ev_hca_guid,
00a3eaf3896a33935e11fd5c5fb5c1714225c067Ramaswamy Tummala event->ev_port, gid.gid_prefix, gid.gid_guid, ret);
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri RDS_DPRINTF2("rds_handle_portup_event", "Return: GUID: 0x%llx",
00a3eaf3896a33935e11fd5c5fb5c1714225c067Ramaswamy Tummala RDS_DPRINTF2("rdsib_add_hca", "Enter: GUID: 0x%llx", hca_guid);
00a3eaf3896a33935e11fd5c5fb5c1714225c067Ramaswamy Tummala /* register the recv memory with this hca */
00a3eaf3896a33935e11fd5c5fb5c1714225c067Ramaswamy Tummala /* no memory to register */
00a3eaf3896a33935e11fd5c5fb5c1714225c067Ramaswamy Tummala RDS_DPRINTF2("rdsib_add_hca", "No memory to register");
00a3eaf3896a33935e11fd5c5fb5c1714225c067Ramaswamy Tummala mem_attr.mr_vaddr = (ib_vaddr_t)(uintptr_t)rds_dpool.pool_memp;
00a3eaf3896a33935e11fd5c5fb5c1714225c067Ramaswamy Tummala mem_attr.mr_flags = IBT_MR_ENABLE_LOCAL_WRITE;
00a3eaf3896a33935e11fd5c5fb5c1714225c067Ramaswamy Tummala ret = ibt_register_mr(hcap->hca_hdl, hcap->hca_pdhdl, &mem_attr,
00a3eaf3896a33935e11fd5c5fb5c1714225c067Ramaswamy Tummala RDS_DPRINTF2("rdsib_add_hca", "ibt_register_mr failed: %d",
00a3eaf3896a33935e11fd5c5fb5c1714225c067Ramaswamy Tummala rw_enter(&rdsib_statep->rds_hca_lock, RW_WRITER);
00a3eaf3896a33935e11fd5c5fb5c1714225c067Ramaswamy Tummala hcap->hca_state = RDS_HCA_STATE_MEM_REGISTERED;
00a3eaf3896a33935e11fd5c5fb5c1714225c067Ramaswamy Tummala RDS_DPRINTF2("rdsib_add_hca", "Retrun: GUID: 0x%llx", hca_guid);
00a3eaf3896a33935e11fd5c5fb5c1714225c067Ramaswamy Tummalavoid rds_close_this_session(rds_session_t *sp, uint8_t wait);
00a3eaf3896a33935e11fd5c5fb5c1714225c067Ramaswamy Tummalaint rds_post_control_message(rds_session_t *sp, uint8_t code, in_port_t port);
00a3eaf3896a33935e11fd5c5fb5c1714225c067Ramaswamy Tummalardsib_del_hca(rds_state_t *statep, ib_guid_t hca_guid)
00a3eaf3896a33935e11fd5c5fb5c1714225c067Ramaswamy Tummala RDS_DPRINTF2("rdsib_del_hca", "Enter: GUID: 0x%llx", hca_guid);
00a3eaf3896a33935e11fd5c5fb5c1714225c067Ramaswamy Tummala * This should be a write lock as we don't want anyone to get access
00a3eaf3896a33935e11fd5c5fb5c1714225c067Ramaswamy Tummala * to the hcap while we are modifing its contents
00a3eaf3896a33935e11fd5c5fb5c1714225c067Ramaswamy Tummala rw_enter(&statep->rds_hca_lock, RW_WRITER);
00a3eaf3896a33935e11fd5c5fb5c1714225c067Ramaswamy Tummala while ((hcap != NULL) && (hcap->hca_guid != hca_guid)) {
00a3eaf3896a33935e11fd5c5fb5c1714225c067Ramaswamy Tummala /* Prevent initiating any new activity on this HCA */
00a3eaf3896a33935e11fd5c5fb5c1714225c067Ramaswamy Tummala * stop the outgoing traffic and close any active sessions on this hca.
00a3eaf3896a33935e11fd5c5fb5c1714225c067Ramaswamy Tummala * Any pending messages in the SQ will be allowed to complete.
00a3eaf3896a33935e11fd5c5fb5c1714225c067Ramaswamy Tummala rw_enter(&statep->rds_sessionlock, RW_READER);
00a3eaf3896a33935e11fd5c5fb5c1714225c067Ramaswamy Tummala RDS_DPRINTF2("rdsib_del_hca", "SP(%p) State: %d", sp,
00a3eaf3896a33935e11fd5c5fb5c1714225c067Ramaswamy Tummala * We are changing the session state in advance. This prevents
00a3eaf3896a33935e11fd5c5fb5c1714225c067Ramaswamy Tummala * further messages to be posted to the SQ. We then
00a3eaf3896a33935e11fd5c5fb5c1714225c067Ramaswamy Tummala * send a control message to the remote and tell it close
00a3eaf3896a33935e11fd5c5fb5c1714225c067Ramaswamy Tummala * the session.
00a3eaf3896a33935e11fd5c5fb5c1714225c067Ramaswamy Tummala sp->session_state = RDS_SESSION_STATE_HCA_CLOSING;
00a3eaf3896a33935e11fd5c5fb5c1714225c067Ramaswamy Tummala RDS_DPRINTF3("rds_handle_cm_conn_closed", "SP(%p) State "
00a3eaf3896a33935e11fd5c5fb5c1714225c067Ramaswamy Tummala * wait until the sendq is empty then tell the remote to
00a3eaf3896a33935e11fd5c5fb5c1714225c067Ramaswamy Tummala * close this session. This enables for graceful shutdown of
00a3eaf3896a33935e11fd5c5fb5c1714225c067Ramaswamy Tummala /* wait until all the sessions are off this HCA */
00a3eaf3896a33935e11fd5c5fb5c1714225c067Ramaswamy Tummala RDS_DPRINTF2("rdsib_del_hca", "SP(%p) State: %d", sp,
00a3eaf3896a33935e11fd5c5fb5c1714225c067Ramaswamy Tummala while ((sp->session_state == RDS_SESSION_STATE_HCA_CLOSING) ||
00a3eaf3896a33935e11fd5c5fb5c1714225c067Ramaswamy Tummala (sp->session_state == RDS_SESSION_STATE_ERROR) ||
00a3eaf3896a33935e11fd5c5fb5c1714225c067Ramaswamy Tummala (sp->session_state == RDS_SESSION_STATE_PASSIVE_CLOSING) ||
00a3eaf3896a33935e11fd5c5fb5c1714225c067Ramaswamy Tummala (sp->session_state == RDS_SESSION_STATE_CLOSED)) {
00a3eaf3896a33935e11fd5c5fb5c1714225c067Ramaswamy Tummala RDS_DPRINTF2("rdsib_del_hca", "SP(%p) State: %d", sp,
00a3eaf3896a33935e11fd5c5fb5c1714225c067Ramaswamy Tummala * if rdsib_close_ib was called before this, then that would have
00a3eaf3896a33935e11fd5c5fb5c1714225c067Ramaswamy Tummala * unbound the service on all ports. In that case, the HCA structs
00a3eaf3896a33935e11fd5c5fb5c1714225c067Ramaswamy Tummala * will contain stale bindhdls. Hence, we do not call unbind unless
00a3eaf3896a33935e11fd5c5fb5c1714225c067Ramaswamy Tummala * the service is still registered.
00a3eaf3896a33935e11fd5c5fb5c1714225c067Ramaswamy Tummala /* unbind RDS service on all ports on this HCA */
00a3eaf3896a33935e11fd5c5fb5c1714225c067Ramaswamy Tummala for (ix = 0; ix < hcap->hca_nports; ix++) {
00a3eaf3896a33935e11fd5c5fb5c1714225c067Ramaswamy Tummala "Unbinding Service: port: %d, bindhdl: %p",
00a3eaf3896a33935e11fd5c5fb5c1714225c067Ramaswamy Tummala (void) ibt_unbind_service(rdsib_statep->rds_srvhdl,
00a3eaf3896a33935e11fd5c5fb5c1714225c067Ramaswamy Tummala RDS_DPRINTF2("rdsib_del_hca", "HCA(%p) State: %d", hcap,
00a3eaf3896a33935e11fd5c5fb5c1714225c067Ramaswamy Tummala ret = ibt_deregister_mr(hcap->hca_hdl, hcap->hca_mrhdl);
00a3eaf3896a33935e11fd5c5fb5c1714225c067Ramaswamy Tummala /* FALLTHRU */
00a3eaf3896a33935e11fd5c5fb5c1714225c067Ramaswamy Tummala ret = ibt_free_pd(hcap->hca_hdl, hcap->hca_pdhdl);
00a3eaf3896a33935e11fd5c5fb5c1714225c067Ramaswamy Tummala (void) ibt_free_portinfo(hcap->hca_pinfop, hcap->hca_pinfo_sz);
00a3eaf3896a33935e11fd5c5fb5c1714225c067Ramaswamy Tummala * This should be a write lock as we don't want anyone to get access
00a3eaf3896a33935e11fd5c5fb5c1714225c067Ramaswamy Tummala * to the hcap while we are modifing its contents
00a3eaf3896a33935e11fd5c5fb5c1714225c067Ramaswamy Tummala rw_enter(&statep->rds_hca_lock, RW_WRITER);
00a3eaf3896a33935e11fd5c5fb5c1714225c067Ramaswamy Tummala RDS_DPRINTF2("rdsib_del_hca", "Return: GUID: 0x%llx", hca_guid);
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagirirds_async_handler(void *clntp, ibt_hca_hdl_t hdl, ibt_async_code_t code,
00a3eaf3896a33935e11fd5c5fb5c1714225c067Ramaswamy Tummala rds_state_t *statep = (rds_state_t *)clntp;
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri RDS_DPRINTF2("rds_async_handler", "Async code: %d", code);
00a3eaf3896a33935e11fd5c5fb5c1714225c067Ramaswamy Tummala * NOTE: In some error recovery paths, it is possible to
00a3eaf3896a33935e11fd5c5fb5c1714225c067Ramaswamy Tummala * receive IBT_HCA_ATTACH_EVENTs on already known HCAs.
00a3eaf3896a33935e11fd5c5fb5c1714225c067Ramaswamy Tummala (void) rdsib_del_hca(statep, event->ev_hca_guid);
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri RDS_DPRINTF2(LABEL, "Async event: %d not handled", code);
b86efd96f8acd85ddaa930a2f0c1d664237e4aafagiri RDS_DPRINTF2("rds_async_handler", "Return: code: %d", code);
0c19630b1592aa30d3e4d9db1a2a8cf9a91c0e72agiri * This routine exists to minimize stale connections across ungraceful
0c19630b1592aa30d3e4d9db1a2a8cf9a91c0e72agiri * reboots of nodes in a cluster.
0c19630b1592aa30d3e4d9db1a2a8cf9a91c0e72agiri ret = ibt_alloc_cq(hcap->hca_hdl, &cqattr, &cq_hdl, NULL);
0c19630b1592aa30d3e4d9db1a2a8cf9a91c0e72agiri chanargs.rc_control = IBT_CEP_RDMA_RD | IBT_CEP_RDMA_WR |
0c19630b1592aa30d3e4d9db1a2a8cf9a91c0e72agiri RDS_DPRINTF2("rds_randomize_qps", "rand1: %d rand2: %d",
0c19630b1592aa30d3e4d9db1a2a8cf9a91c0e72agiri "Bailing at i: %d", i);
0c19630b1592aa30d3e4d9db1a2a8cf9a91c0e72agiri "Bailing at i: %d j: %d", i, j);