b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * CDDL HEADER START
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * The contents of this file are subject to the terms of the
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * Common Development and Distribution License (the "License").
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * You may not use this file except in compliance with the License.
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * or http://www.opensolaris.org/os/licensing.
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * See the License for the specific language governing permissions
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * and limitations under the License.
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * When distributing Covered Code, include this CDDL HEADER in each
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * If applicable, add the following below this CDDL HEADER, with the
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * fields enclosed by brackets "[]" replaced with your own identifying
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * information: Portions Copyright [yyyy] [name of copyright owner]
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * CDDL HEADER END
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved.
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * Declarations private to this file
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalanstatic int eib_rsrc_setup_txbufs(eib_t *, int *);
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalanstatic int eib_rsrc_setup_rxbufs(eib_t *, int *);
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalanstatic int eib_rsrc_setup_lsobufs(eib_t *, int *);
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalanstatic void eib_rsrc_init_wqe_pool(eib_t *, eib_wqe_pool_t **,
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalanstatic void eib_rsrc_fini_wqe_pool(eib_t *, eib_wqe_pool_t **);
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalanstatic boolean_t eib_rsrc_ok_to_free_pool(eib_t *, eib_wqe_pool_t *, boolean_t);
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalanstatic int eib_rsrc_grab_wqes(eib_t *, eib_wqe_pool_t *, eib_wqe_t **, uint_t,
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalanstatic void eib_rsrc_return_wqes(eib_t *, eib_wqe_pool_t *, eib_wqe_t **,
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalanstatic void eib_rb_rsrc_setup_txbufs(eib_t *, boolean_t);
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalanstatic void eib_rb_rsrc_setup_rxbufs(eib_t *, boolean_t);
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalanstatic void eib_rb_rsrc_setup_lsobufs(eib_t *, boolean_t);
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * Definitions private to this file
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalanstatic uint_t eib_lso_num_bufs = EIB_LSO_NUM_BUFS; /* tunable? */
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan if (eib_rsrc_setup_txbufs(ss, err) != EIB_E_SUCCESS)
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan if (ss->ei_caps->cp_lso_maxlen && ss->ei_caps->cp_cksum_flags &&
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan if (eib_rsrc_setup_lsobufs(ss, err) != EIB_E_SUCCESS) {
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan if (eib_rsrc_setup_rxbufs(ss, err) != EIB_E_SUCCESS) {
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalaneib_rsrc_grab_swqes(eib_t *ss, eib_wqe_t **wqes, uint_t n_req, uint_t *actual,
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan ret = eib_rsrc_grab_wqes(ss, ss->ei_tx, wqes, n_req, actual, pri);
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * See note for eib_rsrc_grab_swqe()
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan for (i = 0; i < (*actual); i++) {
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan wqe->qe_wr.send.wr_flags = IBT_WR_NO_FLAGS;
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan wqe->qe_wr.send.wr.ud.udwr_dest = wqe->qe_dest;
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan encap_hdr = (uint32_t *)(void *)wqe->qe_payload_hdr;
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalaneib_rsrc_grab_rwqes(eib_t *ss, eib_wqe_t **wqes, uint_t n_req, uint_t *actual,
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan return (eib_rsrc_grab_wqes(ss, ss->ei_rx, wqes, n_req, actual, pri));
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalaneib_rsrc_grab_lsobufs(eib_t *ss, uint_t req_sz, ibt_wr_ds_t *sgl, uint32_t *nds)
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * Determine how many bufs we'd need for the size requested
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan if ((frag_sz = req_sz % EIB_LSO_BUFSZ) != 0)
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * If we don't have enough lso bufs, return failure
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * Pick the first "num_needed" bufs from the free list
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan for (i = 0; i < num_needed; i++) {
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan sgl[i].ds_va = (ib_vaddr_t)(uintptr_t)elem->lb_buf;
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * If the requested size is not a multiple of EIB_LSO_BUFSZ, we need
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * to adjust the last sgl entry's length. Since we know we need atleast
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * one, the i-1 use below is ok.
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * Update nfree count and return
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan (void) eib_rsrc_grab_wqes(ss, ss->ei_tx, &wqe, 1, NULL, pri);
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * Let's reset the swqe basic wr parameters to default. We need
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * to do this because this swqe could've previously been used
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * for a checksum offload (when the flags would've been set)
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * or for an LSO send (in which case the opcode would've been set
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * to a different value), or been iov mapped (in which case the
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * sgl/nds could've been set to different values). We'll make
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * it easy and initialize it here, so simple transactions can
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * go through without any special effort by the caller.
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * Note that even though the wqe structure is common for both
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * send and recv, they're in two independent pools and the wqe
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * type remains the same throughout its lifetime. So we don't
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * have to worry about resetting any other field.
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan wqe->qe_wr.send.wr_flags = IBT_WR_NO_FLAGS;
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan wqe->qe_wr.send.wr.ud.udwr_dest = wqe->qe_dest;
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan encap_hdr = (uint32_t *)(void *)wqe->qe_payload_hdr;
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan (void) eib_rsrc_grab_wqes(ss, ss->ei_rx, &wqe, 1, NULL, pri);
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalaneib_rsrc_return_swqe(eib_t *ss, eib_wqe_t *wqe, eib_chan_t *chan)
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan eib_rsrc_return_wqes(ss, ss->ei_tx, &wqe, 1);
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalaneib_rsrc_return_rwqe(eib_t *ss, eib_wqe_t *wqe, eib_chan_t *chan)
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan eib_rsrc_return_wqes(ss, ss->ei_rx, &wqe, 1);
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalaneib_rsrc_return_lsobufs(eib_t *ss, ibt_wr_ds_t *sgl_p, uint32_t nds)
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * Nowhere to return the buffers to ??
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan for (i = 0; i < nds; i++) {
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan va = (uint8_t *)(uintptr_t)sgl_p[i].ds_va;
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan ASSERT(va < (bkt->bk_mem + bkt->bk_nelem * EIB_LSO_BUFSZ));
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * Figure out the buflist element this sgl buffer corresponds
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * to and put it back at the head
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan ndx = ((uintptr_t)va - (uintptr_t)bkt->bk_mem) / EIB_LSO_BUFSZ;
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * If the number of available lso buffers just crossed the
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * threshold, wakeup anyone who may be sleeping on the event.
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan if (((bkt->bk_nfree - nds) < EIB_LSO_FREE_BUFS_THRESH) &&
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan (bkt->bk_nfree >= EIB_LSO_FREE_BUFS_THRESH)) {
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalaneib_rsrc_decr_posted_swqe(eib_t *ss, eib_chan_t *chan)
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan if ((chan->ch_tear_down) && (chan->ch_tx_posted == 0)) {
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalaneib_rsrc_decr_posted_rwqe(eib_t *ss, eib_chan_t *chan)
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * Decrement the ch_rx_posted count. If we are tearing this channel
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * down, signal the waiter when the count reaches 0. If we aren't
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * tearing the channel down, see if the count has gone below the low
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * water mark. If it has, and if this channel isn't already being
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * refilled, queue the channel up with the service thread for a
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * rwqe refill.
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan } else if (chan->ch_rx_posted < chan->ch_lwm_rwqes) {
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan for (tail = ss->ei_rxpost; tail; tail = tail->ch_rxpost_next) {
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan EIB_INCR_COUNTER(&ss->ei_stats->st_noxmitbuf);
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan if ((wp->wp_status & EIB_TXWQE_SHORT) == 0) {
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan EIB_INCR_COUNTER(&ss->ei_stats->st_noxmitbuf);
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan "eib_rsrc_lsobufs_needed: "
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan "lso bufs seem to be needed even though "
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan "LSO support was not advertised");
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan if ((bkt->bk_status & EIB_LBUF_SHORT) == 0) {
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * Set the EIB_RXWQE_SHORT flag when the number of free wqes
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * in the rx pool falls below the low threshold for rwqes and
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * clear it only when the number of free wqes gets back above
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * the high water mark.
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan if (wp->wp_nfree <= EIB_NFREE_RWQES_LOW) {
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan } else if (wp->wp_nfree >= EIB_NFREE_RWQES_HWM) {
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan if ((wp->wp_status & EIB_RXWQE_SHORT) == EIB_RXWQE_SHORT)
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalaneib_rb_rsrc_setup_bufs(eib_t *ss, boolean_t force)
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * Try to allocate and initialize the tx wqe pool
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * If we keep the tx buffers as mtu-sized, then potentially every
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * LSO request that cannot be satisfactorily mapped, will use up
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * the 8K large (default size) lso buffers. This may be inadvisable
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * given that lso buffers are a scarce resource. Instead, we'll
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * slightly raise the size of the copy buffers in the send wqes
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * (say to EIB_TX_COPY_THRESH) so that requests that cannot be
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * mapped could still avoid using the 8K LSO buffers if they're
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * less than the copy threshold size.
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan lso_enabled = ss->ei_caps->cp_lso_maxlen &&
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan ss->ei_caps->cp_cksum_flags && ss->ei_caps->cp_resv_lkey_capab;
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan tx_bufsz = ((lso_enabled) && (EIB_TX_COPY_THRESH > mtu)) ?
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan eib_rsrc_init_wqe_pool(ss, &ss->ei_tx, tx_bufsz, EIB_WP_TYPE_TX);
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * Register the TX memory region with IBTF for use
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan ret = ibt_register_mr(ss->ei_hca_hdl, ss->ei_pd_hdl, &attr,
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan EIB_DPRINTF_ERR(ss->ei_instance, "eib_rsrc_setup_txbufs: "
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan "ibt_register_mr() failed for tx "
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan "region (0x%llx, 0x%llx) with ret=%d",
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * Now setup the tx wqes
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan buf = (uint8_t *)(uintptr_t)(tx->wp_vaddr);
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan for (i = 0, blk = 0; blk < EIB_BLKS_PER_POOL; blk++) {
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan for (ndx = 0; ndx < EIB_WQES_PER_BLK; ndx++, i++) {
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * Allocate a UD destination handle
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan IBT_UD_DEST_NO_FLAGS, ss->ei_pd_hdl, &dest);
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan "eib_rsrc_setup_txbufs: "
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan "ibt_alloc_ud_dest(hca_hdl=0x%llx) "
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * These parameters should remain fixed throughout the
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * lifetime of this wqe.
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * The qe_dest and qe_payload_hdr are specific to tx
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * only, but remain unchanged throughout the lifetime
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * The payload header is normally used when we have an
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * LSO packet to send. Since the EoIB encapsulation
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * header won't be part of the message we get from the
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * network layer, we'll need to copy the lso header into
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * a new buffer every time before we hand over the LSO
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * send request to the hca driver.
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan kmem_zalloc(EIB_MAX_PAYLOAD_HDR_SZ, KM_SLEEP);
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * The encapsulation header is at the start of the
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * payload header and is initialized to the default
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * encapsulation header we use (no multiple segments,
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * no FCS). This part of the header is not expected
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan encap_hdr = (uint32_t *)(void *)wqe->qe_payload_hdr;
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * The parameter set below are used in tx and rx paths.
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * These parameters (except ds_key) are reset to these
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * default values in eib_rsrc_return_wqes().
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan wqe->qe_sgl.ds_va = (ib_vaddr_t)(uintptr_t)buf;
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan ((blk & EIB_WQEBLK_MASK) << EIB_WQEBLK_SHIFT) |
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan ((ndx & EIB_WQENDX_MASK) << EIB_WQENDX_SHIFT) |
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * These tx-specific parameters (except wr_id and
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * wr_trans) are reset in eib_rsrc_grab_swqes() to make
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * sure any freshly acquired swqe from the pool has
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * these default settings for the caller.
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan wqe->qe_wr.send.wr_id = (ibt_wrid_t)(uintptr_t)wqe;
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan wqe->qe_wr.send.wr_flags = IBT_WR_NO_FLAGS;
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan wqe->qe_wr.send.wr.ud.udwr_dest = wqe->qe_dest;
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * Before returning, create a kernel thread to monitor the status
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * of wqes in the tx wqe pool. Note that this thread cannot be
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * created from eib_state_init() during attach(), since the thread
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * expects the wqe pool to be allocated and ready when it starts,
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * and the tx bufs initialization only happens during eib_m_start().
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan kt = thread_create(NULL, 0, eib_monitor_tx_wqes, ss, 0,
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * Try to allocate and initialize the wqe pool. When this is called
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * during a plumb via the mac m_start callback, we need to make
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * sure there is a need to allocate a wqe pool afresh. If during a
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * previous unplumb we didn't free the wqe pool because the nw layer
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * was holding on to some rx buffers, we don't need to allocate new
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * pool and set up the buffers again; we'll just start re-using the
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * previous one.
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * The receive buffer has to work for all channels, specifically the
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * data qp of the vnics. This means that the buffer must be large
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * enough to hold MTU sized IB payload (including the EoIB and ethernet
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * headers) plus the GRH. In addition, because the ethernet header is
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * either 14 or 18 bytes (tagless or vlan tagged), we should have the
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * buffer filled in such a way that the IP header starts at atleast a
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * 4-byte aligned address. In order to do this, we need to have some
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * additional room.
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan mtu + EIB_GRH_SZ + EIB_IPHDR_ALIGN_ROOM, EIB_WP_TYPE_RX);
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * Register the RX memory region with IBTF for use
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan attr.mr_flags = IBT_MR_SLEEP | IBT_MR_ENABLE_LOCAL_WRITE;
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan ret = ibt_register_mr(ss->ei_hca_hdl, ss->ei_pd_hdl, &attr,
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan EIB_DPRINTF_ERR(ss->ei_instance, "eib_rsrc_setup_rxbufs: "
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan "ibt_register_mr() failed for rx "
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan "region (0x%llx, 0x%llx) with ret=%d",
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * Now setup the rx wqes
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan buf = (uint8_t *)(uintptr_t)(rx->wp_vaddr);
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan for (i = 0, blk = 0; blk < EIB_BLKS_PER_POOL; blk++) {
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan for (ndx = 0; ndx < EIB_WQES_PER_BLK; ndx++, i++) {
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * These parameters should remain fixed throughout the
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * lifetime of this recv wqe. The qe_frp will only be
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * used by the data channel of vnics and will remain
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * unused by other channels.
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan wqe->qe_bufsz = mtu + EIB_GRH_SZ + EIB_IPHDR_ALIGN_ROOM;
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan wqe->qe_wr.recv.wr_id = (ibt_wrid_t)(uintptr_t)wqe;
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan wqe->qe_frp.free_func = eib_data_rx_recycle;
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * The parameter set below are used in tx and rx paths.
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * These parameters (except ds_key) are reset to these
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * default values in eib_rsrc_return_wqes().
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan wqe->qe_sgl.ds_va = (ib_vaddr_t)(uintptr_t)buf;
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan ((blk & EIB_WQEBLK_MASK) << EIB_WQEBLK_SHIFT) |
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan ((ndx & EIB_WQENDX_MASK) << EIB_WQENDX_SHIFT) |
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * These rx-specific parameters are also reset to
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * these default values in eib_rsrc_return_wqes().
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan buf += (mtu + EIB_GRH_SZ + EIB_IPHDR_ALIGN_ROOM);
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalaneib_rsrc_setup_lsobufs(eib_t *ss, int *err)
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * Allocate the lso bucket and space for buffers
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan bkt = kmem_zalloc(sizeof (eib_lsobkt_t), KM_SLEEP);
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan lsomem = kmem_zalloc(eib_lso_num_bufs * EIB_LSO_BUFSZ, KM_SLEEP);
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * Register lso memory and save the lkey
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan attr.mr_vaddr = (uint64_t)(uintptr_t)lsomem;
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan attr.mr_len = eib_lso_num_bufs * EIB_LSO_BUFSZ;
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan ret = ibt_register_mr(ss->ei_hca_hdl, ss->ei_pd_hdl, &attr,
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan EIB_DPRINTF_ERR(ss->ei_instance, "eib_rsrc_setup_lsobufs: "
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan "ibt_register_mr() failed for LSO "
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan "region (0x%llx, 0x%llx) with ret=%d",
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan kmem_free(lsomem, eib_lso_num_bufs * EIB_LSO_BUFSZ);
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * Now allocate the buflist. Note that the elements in the buflist and
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * the buffers in the lso memory have a permanent 1-1 relation, so we
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * can always derive the address of a buflist entry from the address of
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * an lso buffer.
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan bkt->bk_bufl = kmem_zalloc(eib_lso_num_bufs * sizeof (eib_lsobuf_t),
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * Set up the lso buf chain
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan for (i = 0; i < eib_lso_num_bufs; i++) {
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * Set up the LSO buffer information in eib state
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan mutex_init(&bkt->bk_lock, NULL, MUTEX_DRIVER, NULL);
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan cv_init(&bkt->bk_cv, NULL, CV_DEFAULT, NULL);
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * Before returning, create a kernel thread to monitor the status
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan kt = thread_create(NULL, 0, eib_monitor_lso_bufs, ss, 0,
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalaneib_rsrc_init_wqe_pool(eib_t *ss, eib_wqe_pool_t **wpp, ib_memlen_t bufsz,
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * Allocate the wqe pool, wqes and bufs
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan wp = kmem_zalloc(sizeof (eib_wqe_pool_t), KM_SLEEP);
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan wp_wqesz = EIB_WQES_PER_POOL * sizeof (eib_wqe_t);
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan wp->wp_wqe = (eib_wqe_t *)kmem_zalloc(wp_wqesz, KM_SLEEP);
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan wp->wp_vaddr = (ib_vaddr_t)(uintptr_t)kmem_zalloc(wp->wp_memsz,
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan wp->wp_nfree_lwm = (wp_type == EIB_WP_TYPE_TX) ?
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan EIB_NFREE_SWQES_LWM : EIB_NFREE_RWQES_LWM;
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * Initialize the lock and bitmaps: everything is available at first,
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * but note that if the number of blocks per pool is less than 64, we
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * need to initialize those extra bits as "unavailable" - these will
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * remain unavailable throughout.
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan mutex_init(&wp->wp_lock, NULL, MUTEX_DRIVER, NULL);
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan cv_init(&wp->wp_cv, NULL, CV_DEFAULT, NULL);
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan wp->wp_free_blks = (EIB_BLKS_PER_POOL >= 64) ? (~0) :
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan for (i = 0; i < EIB_BLKS_PER_POOL; i++)
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalaneib_rsrc_fini_wqe_pool(eib_t *ss, eib_wqe_pool_t **wpp)
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan kmem_free((void *)(uintptr_t)(wp->wp_vaddr), wp->wp_memsz);
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan kmem_free(wp->wp_wqe, EIB_WQES_PER_POOL * sizeof (eib_wqe_t));
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalaneib_rsrc_ok_to_free_pool(eib_t *ss, eib_wqe_pool_t *wp, boolean_t force)
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * See if we can release all memory allocated for buffers, wqes and
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * the pool. Note that in the case of data channel rx buffers, some
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * of the buffers may not be free if the nw layer is holding on to
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * them still. If this is the case, we cannot free the wqe pool now
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * or a subsequent access by the nw layer to the buffers will cause
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * If force-free flag is set, we can always release the memory.
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * Note that this flag is unused currently, and should be removed.
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * If a whole block remains allocated, obviously we cannot free
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan free_blks = (EIB_BLKS_PER_POOL >= 64) ? (~0) :
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * If even a single wqe within any one block remains in-use, we
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * cannot free the pool
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan for (i = 0; i < EIB_BLKS_PER_POOL; i++) {
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalaneib_rsrc_grab_wqes(eib_t *ss, eib_wqe_pool_t *wp, eib_wqe_t **wqes,
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * If this is a low priority request, adjust the number requested
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * so we don't allocate beyond the low-water-mark
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan else if ((wp->wp_nfree - n_req) < wp->wp_nfree_lwm)
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan for (n_allocd = 0; n_allocd < n_req; n_allocd++) {
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * If the entire pool is unavailable, quit
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * Find the first wqe that's available
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan ndx = EIB_FIND_LSB_SET(wp->wp_free_wqes[blk]);
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * Mark the wqe as allocated
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan wp->wp_free_wqes[blk] &= (~((uint64_t)1 << ndx));
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * If this was the last free wqe in this block, mark
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * the block itself as unavailable
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan wp->wp_free_blks &= (~((uint64_t)1 << blk));
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * Return this wqe to the caller
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalaneib_rsrc_return_wqes(eib_t *ss, eib_wqe_pool_t *wp, eib_wqe_t **wqes,
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan for (n_freed = 0; n_freed < n_wqes; n_freed++) {
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * This wqe is being returned back to the pool, so clear
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * any wqe flags and reset buffer address and size in the
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * single segment sgl back to what they were initially.
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * Also erase any mblk pointer and callback function ptrs.
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan wqe->qe_sgl.ds_va = (ib_vaddr_t)(uintptr_t)wqe->qe_cpbuf;
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * Mark the wqe free in its block
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan wp->wp_free_wqes[blk] |= ((uint64_t)1 << ndx);
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * This block now has atleast one wqe free, so mark
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * the block itself as available and move on to the
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * next wqe to free
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * If the number of available wqes in the pool has just crossed
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * the high-water-mark, wakeup anyone who may be sleeping on it.
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan ((wp->wp_nfree - n_freed) < EIB_NFREE_SWQES_HWM) &&
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalaneib_rb_rsrc_setup_txbufs(eib_t *ss, boolean_t force)
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * Check if it's ok to free the tx wqe pool (i.e. all buffers have
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * been reclaimed) and if so, stop the txwqe monitor thread (and wait
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * for it to die), release the UD destination handles, deregister
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * memory and fini the wqe pool.
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan if (eib_rsrc_ok_to_free_pool(ss, wp, force)) {
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan for (i = 0; i < EIB_WQES_PER_POOL; i++) {
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan if ((plhdr = wqe->qe_payload_hdr) != NULL) {
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan "eib_rb_rsrc_setup_txbufs: "
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan "ibt_free_ud_dest() failed, ret=%d",
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan if ((ret = ibt_deregister_mr(ss->ei_hca_hdl,
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan "eib_rb_rsrc_setup_txbufs: "
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan "ibt_deregister_mr() failed, ret=%d", ret);
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalaneib_rb_rsrc_setup_rxbufs(eib_t *ss, boolean_t force)
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * Check if it's ok to free the rx wqe pool (i.e. all buffers have
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * been reclaimed) and if so, deregister memory and fini the wqe pool.
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan if (eib_rsrc_ok_to_free_pool(ss, rx, force)) {
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan if ((ret = ibt_deregister_mr(ss->ei_hca_hdl,
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan "eib_rb_rsrc_setup_rxbufs: "
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan "ibt_deregister_mr() failed, ret=%d", ret);
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalaneib_rb_rsrc_setup_lsobufs(eib_t *ss, boolean_t force)
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * Remove the lso bucket from the state
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * Try to stop the lso bufs monitor thread. If we fail, we simply
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * return. We'll have another shot at it later from detach() with
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * the force flag set.
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan if (eib_stop_monitor_lso_bufs(ss, force) != EIB_E_SUCCESS)
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * Free the buflist
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan kmem_free(bkt->bk_bufl, bkt->bk_nelem * sizeof (eib_lsobuf_t));
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * Deregister LSO memory and free it
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan if ((ret = ibt_deregister_mr(ss->ei_hca_hdl,
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan "eib_rb_rsrc_setup_lsobufs: "
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan "ibt_deregister_mr() failed, ret=%d", ret);
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan kmem_free(bkt->bk_mem, bkt->bk_nelem * EIB_LSO_BUFSZ);
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * Destroy the mutex and condvar
b494511a9cf72b1fc4eb13a0e593f55c624ab829Venki Rajagopalan * Finally, free the lso bucket itself