Lines Matching refs:state
81 * instances will be created independent of the port state.
93 * managed by the DDI framework for the partition objects. However, the state is
127 * When such a partition object is plumbed, the link state will be displayed as
138 * initialization of the partition object and transitions it to a usable state.
429 static int ibd_part_attach(ibd_state_t *state, dev_info_t *dip);
430 static void ibd_part_unattach(ibd_state_t *state);
432 static int ibd_port_unattach(ibd_state_t *state, dev_info_t *dip);
598 ibd_print_warn(ibd_state_t *state, char *fmt, ...)
605 datalink_id_t linkid = state->id_plinkid;
607 hca_guid = ddi_prop_get_int64(DDI_DEV_T_ANY, state->id_dip,
612 ddi_driver_name(state->id_dip), ddi_get_instance(state->id_dip),
613 (u_longlong_t)hca_guid, state->id_port, state->id_pkey,
629 * state->id_lso->bkt_nfree may be accessed without a lock to
843 ibd_rc_chan_s::state
1089 ibd_get_allroutergroup(ibd_state_t *state, ipoib_mac_t *mcmac,
1093 uint32_t adjscope = state->id_scope << 16;
1110 ((uint32_t)(state->id_pkey << 16))),
1148 #define IBD_MCACHE_INSERT_FULL(state, mce) \
1149 list_insert_head(&state->id_mc_full, mce)
1150 #define IBD_MCACHE_INSERT_NON(state, mce) \
1151 list_insert_head(&state->id_mc_non, mce)
1152 #define IBD_MCACHE_FIND_FULL(state, mgid) \
1153 ibd_mcache_find(mgid, &state->id_mc_full)
1154 #define IBD_MCACHE_FIND_NON(state, mgid) \
1155 ibd_mcache_find(mgid, &state->id_mc_non)
1156 #define IBD_MCACHE_PULLOUT_FULL(state, mce) \
1157 list_remove(&state->id_mc_full, mce)
1158 #define IBD_MCACHE_PULLOUT_NON(state, mce) \
1159 list_remove(&state->id_mc_non, mce)
1175 ibd_queue_work_slot(ibd_state_t *state, ibd_req_t *ptr, int op)
1184 mutex_enter(&state->id_acache_req_lock);
1185 list_insert_tail(&state->id_req_list, ptr);
1188 cv_signal(&state->id_acache_req_cv);
1189 mutex_exit(&state->id_acache_req_lock);
1196 ibd_async_work(ibd_state_t *state)
1201 mutex_enter(&state->id_acache_req_lock);
1202 CALLB_CPR_INIT(&cprinfo, &state->id_acache_req_lock,
1206 ptr = list_get_head(&state->id_req_list);
1208 mutex_exit(&state->id_acache_req_lock);
1216 if ((state->id_mac_state & IBD_DRV_IN_LATE_HCA_INIT) &&
1231 ibd_async_acache(state, &ptr->rq_mac);
1235 ibd_async_multicast(state,
1239 ibd_async_setprom(state);
1242 ibd_async_unsetprom(state);
1245 ibd_async_reap_group(state,
1256 ibd_async_trap(state, ptr);
1259 ibd_async_txsched(state);
1262 ibd_async_link(state, ptr);
1265 mutex_enter(&state->id_acache_req_lock);
1269 mutex_exit(&state->id_acache_req_lock);
1273 ibd_async_rc_process_too_big(state,
1277 ibd_async_rc_close_act_chan(state, ptr);
1280 ibd_async_rc_recycle_ace(state, ptr);
1289 kmem_cache_free(state->id_req_kmc, ptr);
1291 mutex_enter(&state->id_acache_req_lock);
1298 cv_wait(&state->id_acache_req_cv,
1299 &state->id_acache_req_lock);
1301 &state->id_acache_req_lock);
1316 ibd_async_safe(ibd_state_t *state)
1318 mutex_enter(&state->id_trap_lock);
1319 if (state->id_trap_stop) {
1320 mutex_exit(&state->id_trap_lock);
1323 state->id_trap_inprog++;
1324 mutex_exit(&state->id_trap_lock);
1334 ibd_async_done(ibd_state_t *state)
1336 mutex_enter(&state->id_trap_lock);
1337 if (--state->id_trap_inprog == 0)
1338 cv_signal(&state->id_trap_cv);
1339 mutex_exit(&state->id_trap_lock);
1383 ibd_acache_init(ibd_state_t *state)
1388 mutex_init(&state->id_ac_mutex, NULL, MUTEX_DRIVER, NULL);
1389 mutex_init(&state->id_mc_mutex, NULL, MUTEX_DRIVER, NULL);
1390 mutex_enter(&state->id_ac_mutex);
1391 list_create(&state->id_ah_free, sizeof (ibd_ace_t),
1393 list_create(&state->id_ah_active, sizeof (ibd_ace_t),
1395 state->id_ah_active_hash = mod_hash_create_extended("IBD AH hash",
1396 state->id_hash_size, mod_hash_null_keydtor, mod_hash_null_valdtor,
1398 list_create(&state->id_mc_full, sizeof (ibd_mce_t),
1400 list_create(&state->id_mc_non, sizeof (ibd_mce_t),
1402 state->id_ac_hot_ace = NULL;
1404 state->id_ac_list = ce = (ibd_ace_t *)kmem_zalloc(sizeof (ibd_ace_t) *
1405 state->id_num_ah, KM_SLEEP);
1406 for (i = 0; i < state->id_num_ah; i++, ce++) {
1407 if (ibt_alloc_ud_dest(state->id_hca_hdl, IBT_UD_DEST_NO_FLAGS,
1408 state->id_pd_hdl, &ce->ac_dest) != IBT_SUCCESS) {
1409 mutex_exit(&state->id_ac_mutex);
1410 ibd_acache_fini(state);
1417 IBD_ACACHE_INSERT_FREE(state, ce);
1420 mutex_exit(&state->id_ac_mutex);
1425 ibd_acache_fini(ibd_state_t *state)
1429 mutex_enter(&state->id_ac_mutex);
1431 while ((ptr = IBD_ACACHE_GET_ACTIVE(state)) != NULL) {
1437 while ((ptr = IBD_ACACHE_GET_FREE(state)) != NULL) {
1443 list_destroy(&state->id_ah_free);
1444 list_destroy(&state->id_ah_active);
1445 list_destroy(&state->id_mc_full);
1446 list_destroy(&state->id_mc_non);
1447 kmem_free(state->id_ac_list, sizeof (ibd_ace_t) * state->id_num_ah);
1448 mutex_exit(&state->id_ac_mutex);
1449 mutex_destroy(&state->id_ac_mutex);
1450 mutex_destroy(&state->id_mc_mutex);
1460 ibd_acache_find(ibd_state_t *state, ipoib_mac_t *mac, boolean_t hold, int num)
1464 ASSERT(mutex_owned(&state->id_ac_mutex));
1469 if (mod_hash_find(state->id_ah_active_hash,
1484 ibd_acache_lookup(ibd_state_t *state, ipoib_mac_t *mac, int *err, int numwqe)
1501 mutex_enter(&state->id_ac_mutex);
1503 if (((ptr = state->id_ac_hot_ace) != NULL) &&
1506 mutex_exit(&state->id_ac_mutex);
1509 if (((ptr = ibd_acache_find(state, mac, B_TRUE, numwqe)) != NULL)) {
1510 state->id_ac_hot_ace = ptr;
1511 mutex_exit(&state->id_ac_mutex);
1518 * to ongoing state. Remember in id_ah_addr for which address
1528 if (state->id_ah_op == IBD_OP_NOTSTARTED) {
1529 req = kmem_cache_alloc(state->id_req_kmc, KM_NOSLEEP);
1536 state->id_ah_op = IBD_OP_ONGOING;
1537 ibd_queue_work_slot(state, req, IBD_ASYNC_GETAH);
1538 bcopy(mac, &state->id_ah_addr, IPOIB_ADDRL);
1540 } else if ((state->id_ah_op != IBD_OP_ONGOING) &&
1541 (bcmp(&state->id_ah_addr, mac, IPOIB_ADDRL) == 0)) {
1546 if (state->id_ah_op == IBD_OP_ERRORED) {
1548 state->id_ah_error++;
1559 (void) ibd_get_allroutergroup(state, mac, &routermac);
1560 ptr = ibd_acache_find(state, &routermac, B_TRUE,
1563 state->id_ah_op = IBD_OP_NOTSTARTED;
1564 } else if ((state->id_ah_op != IBD_OP_ONGOING) &&
1565 (bcmp(&state->id_ah_addr, mac, IPOIB_ADDRL) != 0)) {
1568 * packet. The easiest way is to reset the state machine
1571 state->id_ah_op = IBD_OP_NOTSTARTED;
1573 mutex_exit(&state->id_ac_mutex);
1584 ibd_acache_get_unref(ibd_state_t *state)
1586 ibd_ace_t *ptr = list_tail(&state->id_ah_active);
1589 ASSERT(mutex_owned(&state->id_ac_mutex));
1605 ASSERT(state->id_enable_rc == B_TRUE);
1608 ibd_rc_signal_ace_recycle(state, ptr);
1611 IBD_ACACHE_PULLOUT_ACTIVE(state, ptr);
1615 ptr = list_prev(&state->id_ah_active, ptr);
1635 ibd_acache_recycle(ibd_state_t *state, ipoib_mac_t *acmac, boolean_t force)
1640 ASSERT(mutex_owned(&state->id_ac_mutex));
1642 if ((acactive = ibd_acache_find(state, acmac, B_FALSE, 0)) != NULL) {
1667 IBD_ACACHE_PULLOUT_ACTIVE(state, acactive);
1677 IBD_ACACHE_PULLOUT_ACTIVE(state, acactive);
1679 IBD_ACACHE_INSERT_FREE(state, acactive);
1694 ibd_async_mcache(ibd_state_t *state, ipoib_mac_t *mac, boolean_t *redirect)
1708 mce = IBD_MCACHE_FIND_FULL(state, mgid);
1718 if ((mce = ibd_join_group(state, mgid, IB_MC_JSTATE_SEND_ONLY_NON)) !=
1730 if (!ibd_get_allroutergroup(state, mac, &routermac))
1741 if ((mce = IBD_MCACHE_FIND_FULL(state, mgid)) != NULL) {
1751 if ((mce = ibd_join_group(state, mgid, IB_MC_JSTATE_SEND_ONLY_NON)) !=
1764 ibd_async_acache(ibd_state_t *state, ipoib_mac_t *mac)
1790 if ((mce = ibd_async_mcache(state, mac, &redirected)) ==
1792 state->id_ah_op = IBD_OP_ERRORED;
1814 mutex_enter(&state->id_ac_mutex);
1815 if (ibd_acache_find(state, mac, B_FALSE, 0) != NULL) {
1816 state->id_ah_op = IBD_OP_ROUTERED;
1817 mutex_exit(&state->id_ac_mutex);
1821 mutex_exit(&state->id_ac_mutex);
1828 mutex_enter(&state->id_ac_mutex);
1829 if ((ce = IBD_ACACHE_GET_FREE(state)) == NULL) {
1839 if ((ce = ibd_acache_get_unref(state)) == NULL) {
1843 state->id_ah_op = IBD_OP_NOTSTARTED;
1844 mutex_exit(&state->id_ac_mutex);
1856 mutex_exit(&state->id_ac_mutex);
1866 path_attr.pa_sgid = state->id_sgid;
1870 path_attr.pa_sl = state->id_mcinfo->mc_adds_vect.av_srvl;
1871 path_attr.pa_pkey = state->id_pkey;
1872 if (ibt_get_paths(state->id_ibt_hdl, IBT_PATH_PKEY, &path_attr, 1,
1877 if (ibt_modify_ud_dest(ce->ac_dest, state->id_mcinfo->mc_qkey,
1895 if (state->id_enable_rc && (mac->ipoib_qpn != htonl(IB_MC_QPN)) &&
1900 ibd_rc_try_connect(state, ce, &path_info);
1904 state->rc_conn_fail++;
1909 mutex_enter(&state->id_ac_mutex);
1910 IBD_ACACHE_INSERT_ACTIVE(state, ce);
1911 state->id_ah_op = ret;
1912 mutex_exit(&state->id_ac_mutex);
1920 mutex_enter(&state->id_ac_mutex);
1921 state->id_ah_op = IBD_OP_ERRORED;
1922 IBD_ACACHE_INSERT_FREE(state, ce);
1923 mutex_exit(&state->id_ac_mutex);
1931 ibd_async_link(ibd_state_t *state, ibd_req_t *req)
1949 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*state))
1950 state->id_link_speed = ibd_get_portspeed(state);
1951 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*state))
1962 if (state->id_prom_op == IBD_OP_COMPLETED) {
1966 ibd_async_unsetprom(state);
1971 ibd_async_setprom(state);
1979 mce = list_head(&state->id_mc_full);
1981 mce = list_next(&state->id_mc_full, mce);
1983 ibd_leave_group(state,
1987 ibd_reacquire_group(state, pmce);
1996 mutex_enter(&state->id_ac_mutex);
1997 ace = list_head(&state->id_ah_active);
2001 ace = list_next(&state->id_ah_active, ace);
2005 ASSERT(state->id_enable_rc == B_TRUE);
2009 IBD_ACACHE_PULLOUT_ACTIVE(state, pace);
2012 ibd_rc_signal_act_close(state, pace);
2014 state->rc_act_close_simultaneous++;
2022 cycled = ibd_acache_recycle(state,
2045 ibd_async_reap_group(state, mce,
2049 mutex_exit(&state->id_ac_mutex);
2058 mutex_enter(&state->id_link_mutex);
2059 state->id_link_state = lstate;
2060 mac_link_update(state->id_mh, lstate);
2061 mutex_exit(&state->id_link_mutex);
2063 ibd_async_done(state);
2109 ibd_link_mod(ibd_state_t *state, ibt_async_code_t code)
2125 ibd_set_mac_progress(state, IBD_DRV_RESTART_IN_PROGRESS);
2127 mutex_enter(&state->id_link_mutex);
2130 * If the link state is unknown, a plumb has not yet been attempted
2133 if (state->id_link_state == LINK_STATE_UNKNOWN) {
2134 mutex_exit(&state->id_link_mutex);
2139 * If link state is down because of plumb failure, and we are not in
2142 if ((state->id_link_state == LINK_STATE_DOWN) &&
2143 ((state->id_mac_state & IBD_DRV_IN_LATE_HCA_INIT) == 0) &&
2144 ((state->id_mac_state & IBD_DRV_STARTED) == 0)) {
2145 mutex_exit(&state->id_link_mutex);
2165 ibt_status = ibt_query_hca_ports(state->id_hca_hdl, state->id_port,
2175 * port state being down, or due to it's absence in the pkey table,
2178 if (state->id_mac_state & IBD_DRV_IN_LATE_HCA_INIT) {
2179 mutex_exit(&state->id_link_mutex);
2180 if ((ret = ibd_start(state)) != 0) {
2203 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(state->id_sgid))
2205 &state->id_sgid, sizeof (ib_gid_t)) != 0) {
2209 } else if (port_infop->p_pkey_tbl[state->id_pkix] ==
2210 state->id_pkey) {
2215 port_infop->p_pkey_tbl_sz, state->id_pkey, &pkix) == 0) {
2218 mutex_exit(&state->id_link_mutex);
2228 (void) ibd_undo_start(state, LINK_STATE_DOWN);
2229 if ((ret = ibd_start(state)) != 0) {
2238 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(state->id_sgid))
2263 * If the old state is the same as the new state, and the SM indicated
2266 if ((state->id_link_state == new_link_state) && (opcode !=
2268 mutex_exit(&state->id_link_mutex);
2273 * Ok, so there was a link state change; see if it's safe to ask
2276 if (!ibd_async_safe(state)) {
2277 state->id_link_state = new_link_state;
2278 mutex_exit(&state->id_link_mutex);
2282 mutex_exit(&state->id_link_mutex);
2286 * state change event
2288 req = kmem_cache_alloc(state->id_req_kmc, KM_SLEEP);
2290 ibd_queue_work_slot(state, req, IBD_ASYNC_LINK);
2293 ibd_clr_mac_progress(state, IBD_DRV_RESTART_IN_PROGRESS);
2300 * invoke the handler with last known state
2306 ibd_state_t *state = (ibd_state_t *)clnt_private;
2310 ibd_print_warn(state, "catastrophic channel error");
2313 ibd_print_warn(state, "completion queue error");
2324 ASSERT(state->id_hca_hdl == hca_hdl);
2325 if (state->id_port != event->ev_port)
2330 ibd_link_mod(state, code);
2344 ASSERT(state->id_hca_hdl == hca_hdl);
2345 if (state->id_port != event->ev_port)
2348 ibd_link_mod(state, code);
2372 ibd_register_mac(ibd_state_t *state, dev_info_t *dip)
2390 macp->m_driver = state;
2392 macp->m_src_addr = (uint8_t *)&state->id_macaddr;
2396 if (state->id_type == IBD_PORT_DRIVER) {
2398 } else if (state->id_enable_rc) {
2399 macp->m_max_sdu = state->rc_mtu - IPOIB_HDRSIZE;
2408 if ((ret = mac_register(macp, &state->id_mh)) != 0) {
2420 ibd_record_capab(ibd_state_t *state)
2425 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*state))
2430 ibt_status = ibt_query_hca(state->id_hca_hdl, &hca_attrs);
2437 if (state->id_enable_rc) {
2438 state->id_hwcksum_capab = 0;
2442 state->id_hwcksum_capab = IBT_HCA_CKSUM_FULL;
2449 if (state->id_enable_rc) {
2450 state->id_lso_capable = B_FALSE;
2451 state->id_lso_maxlen = 0;
2454 state->id_lso_capable = B_TRUE;
2456 state->id_lso_maxlen = IBD_LSO_MAXLEN;
2458 state->id_lso_maxlen =
2461 state->id_lso_capable = B_FALSE;
2462 state->id_lso_maxlen = 0;
2470 state->id_hca_res_lkey_capab = 1;
2471 state->id_res_lkey = hca_attrs.hca_reserved_lkey;
2472 state->rc_enable_iov_map = B_TRUE;
2475 state->rc_enable_iov_map = B_FALSE;
2483 state->id_max_sqseg = hca_attrs.hca_ud_send_sgl_sz;
2484 state->rc_tx_max_sqseg = hca_attrs.hca_conn_send_sgl_sz;
2486 state->id_max_sqseg = hca_attrs.hca_max_sgl;
2487 state->rc_tx_max_sqseg = hca_attrs.hca_max_sgl;
2489 if (state->id_max_sqseg > IBD_MAX_SQSEG) {
2490 state->id_max_sqseg = IBD_MAX_SQSEG;
2491 } else if (state->id_max_sqseg < IBD_MAX_SQSEG) {
2492 ibd_print_warn(state, "Set #sgl = %d instead of default %d",
2493 state->id_max_sqseg, IBD_MAX_SQSEG);
2495 if (state->rc_tx_max_sqseg > IBD_MAX_SQSEG) {
2496 state->rc_tx_max_sqseg = IBD_MAX_SQSEG;
2497 } else if (state->rc_tx_max_sqseg < IBD_MAX_SQSEG) {
2498 ibd_print_warn(state, "RC mode: Set #sgl = %d instead of "
2499 "default %d", state->rc_tx_max_sqseg, IBD_MAX_SQSEG);
2508 state->id_max_sqseg_hiwm = (state->id_max_sqseg * 65) / 100;
2509 state->rc_max_sqseg_hiwm = (state->rc_tx_max_sqseg * 65) / 100;
2513 * channel size. Store the max channel size in the state so that it
2518 state->id_hca_max_chan_sz = hca_attrs.hca_max_chan_sz;
2520 if (hca_attrs.hca_max_chan_sz < state->id_ud_num_rwqe)
2521 state->id_ud_num_rwqe = hca_attrs.hca_max_chan_sz;
2523 state->id_rx_bufs_outstanding_limit = state->id_ud_num_rwqe -
2526 if (hca_attrs.hca_max_chan_sz < state->id_ud_num_swqe)
2527 state->id_ud_num_swqe = hca_attrs.hca_max_chan_sz;
2529 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*state))
2535 ibd_part_busy(ibd_state_t *state)
2537 if (atomic_add_32_nv(&state->id_rx_list.dl_bufs_outstanding, 0) != 0) {
2542 if (state->rc_srq_rwqe_list.dl_bufs_outstanding != 0) {
2548 * "state->id_ah_op == IBD_OP_ONGOING" means this IPoIB port is
2551 if (state->id_ah_op == IBD_OP_ONGOING) {
2561 ibd_part_unattach(ibd_state_t *state)
2563 uint32_t progress = state->id_mac_state;
2567 ibd_free_rx_rsrcs(state);
2570 ASSERT(state->id_enable_rc);
2571 ibd_rc_fini_srq_list(state);
2572 state->id_mac_state &= (~IBD_DRV_RC_SRQ_ALLOCD);
2576 (void) mac_unregister(state->id_mh);
2577 state->id_mac_state &= (~IBD_DRV_MAC_REGISTERED);
2583 * link state has been marked as unknown; completion handlers
2594 ibd_queue_work_slot(state, &state->id_ah_req, IBD_ASYNC_EXIT);
2595 thread_join(state->id_async_thrid);
2597 state->id_mac_state &= (~IBD_DRV_ASYNC_THR_CREATED);
2601 list_destroy(&state->id_req_list);
2602 mutex_destroy(&state->id_acache_req_lock);
2603 cv_destroy(&state->id_acache_req_cv);
2604 state->id_mac_state &= ~IBD_DRV_REQ_LIST_INITED;
2608 if ((ret = ibt_free_pd(state->id_hca_hdl,
2609 state->id_pd_hdl)) != IBT_SUCCESS) {
2610 ibd_print_warn(state, "failed to free "
2613 state->id_pd_hdl = NULL;
2614 state->id_mac_state &= (~IBD_DRV_PD_ALLOCD);
2618 if ((ret = ibt_close_hca(state->id_hca_hdl)) !=
2620 ibd_print_warn(state, "failed to close "
2623 state->id_hca_hdl = NULL;
2624 state->id_mac_state &= (~IBD_DRV_HCA_OPENED);
2629 if ((ret = ibt_detach(state->id_ibt_hdl)) !=
2631 ibd_print_warn(state,
2634 state->id_ibt_hdl = NULL;
2635 state->id_mac_state &= (~IBD_DRV_IBTL_ATTACH_DONE);
2642 ibd_print_warn(state, "ibt_detach(): global "
2650 ddi_remove_softintr(state->id_tx);
2651 state->id_tx = NULL;
2652 state->id_mac_state &= (~IBD_DRV_TXINTR_ADDED);
2656 ddi_remove_softintr(state->id_rx);
2657 state->id_rx = NULL;
2658 state->id_mac_state &= (~IBD_DRV_RXINTR_ADDED);
2663 kstat_delete(state->rc_ksp);
2664 state->id_mac_state &= (~IBD_DRV_RC_PRIVATE_STATE);
2669 ibd_state_fini(state);
2670 state->id_mac_state &= (~IBD_DRV_STATE_INITIALIZED);
2675 ibd_part_attach(ibd_state_t *state, dev_info_t *dip)
2684 if (ibd_state_init(state, dip) != DDI_SUCCESS) {
2688 state->id_mac_state |= IBD_DRV_STATE_INITIALIZED;
2694 if ((rv = ddi_add_softintr(dip, DDI_SOFTINT_LOW, &state->id_rx,
2695 NULL, NULL, ibd_intr, (caddr_t)state)) != DDI_SUCCESS) {
2700 state->id_mac_state |= IBD_DRV_RXINTR_ADDED;
2703 if ((rv = ddi_add_softintr(dip, DDI_SOFTINT_LOW, &state->id_tx,
2705 (caddr_t)state)) != DDI_SUCCESS) {
2710 state->id_mac_state |= IBD_DRV_TXINTR_ADDED;
2718 if ((ret = ibt_attach(&ibd_clnt_modinfo, dip, state,
2726 if ((ret = ibt_attach(&ibd_clnt_modinfo, dip, state,
2727 &state->id_ibt_hdl)) != IBT_SUCCESS) {
2735 state->id_mac_state |= IBD_DRV_IBTL_ATTACH_DONE;
2740 if ((ret = ibt_open_hca(state->id_ibt_hdl, state->id_hca_guid,
2741 &state->id_hca_hdl)) != IBT_SUCCESS) {
2746 state->id_mac_state |= IBD_DRV_HCA_OPENED;
2750 if (state->id_enable_rc) {
2751 if (ibd_rc_init_stats(state) != DDI_SUCCESS) {
2756 state->id_mac_state |= IBD_DRV_RC_PRIVATE_STATE;
2763 (void) ibd_record_capab(state);
2768 if ((ret = ibt_alloc_pd(state->id_hca_hdl, IBT_PD_NO_FLAGS,
2769 &state->id_pd_hdl)) != IBT_SUCCESS) {
2774 state->id_mac_state |= IBD_DRV_PD_ALLOCD;
2781 mutex_init(&state->id_acache_req_lock, NULL, MUTEX_DRIVER, NULL);
2782 cv_init(&state->id_acache_req_cv, NULL, CV_DEFAULT, NULL);
2783 list_create(&state->id_req_list, sizeof (ibd_req_t),
2785 state->id_mac_state |= IBD_DRV_REQ_LIST_INITED;
2790 kht = thread_create(NULL, 0, ibd_async_work, state, 0, &p0,
2792 state->id_async_thrid = kht->t_did;
2793 state->id_mac_state |= IBD_DRV_ASYNC_THR_CREATED;
2823 ibd_state_t *state;
2836 state = ddi_get_soft_state(ibd_list, instance);
2843 return (ibd_port_unattach(state, dip));
2850 ibd_state_init(ibd_state_t *state, dev_info_t *dip)
2854 mutex_init(&state->id_link_mutex, NULL, MUTEX_DRIVER, NULL);
2855 state->id_link_state = LINK_STATE_UNKNOWN;
2857 mutex_init(&state->id_trap_lock, NULL, MUTEX_DRIVER, NULL);
2858 cv_init(&state->id_trap_cv, NULL, CV_DEFAULT, NULL);
2859 state->id_trap_stop = B_TRUE;
2860 state->id_trap_inprog = 0;
2862 mutex_init(&state->id_scq_poll_lock, NULL, MUTEX_DRIVER, NULL);
2863 mutex_init(&state->id_rcq_poll_lock, NULL, MUTEX_DRIVER, NULL);
2864 state->id_dip = dip;
2866 mutex_init(&state->id_sched_lock, NULL, MUTEX_DRIVER, NULL);
2868 mutex_init(&state->id_tx_list.dl_mutex, NULL, MUTEX_DRIVER, NULL);
2869 mutex_init(&state->id_tx_rel_list.dl_mutex, NULL, MUTEX_DRIVER, NULL);
2870 mutex_init(&state->id_txpost_lock, NULL, MUTEX_DRIVER, NULL);
2871 state->id_tx_busy = 0;
2872 mutex_init(&state->id_lso_lock, NULL, MUTEX_DRIVER, NULL);
2874 state->id_rx_list.dl_bufs_outstanding = 0;
2875 state->id_rx_list.dl_cnt = 0;
2876 mutex_init(&state->id_rx_list.dl_mutex, NULL, MUTEX_DRIVER, NULL);
2877 mutex_init(&state->id_rx_free_list.dl_mutex, NULL, MUTEX_DRIVER, NULL);
2879 state->id_pkey, state->id_plinkid);
2880 state->id_req_kmc = kmem_cache_create(buf, sizeof (ibd_req_t),
2884 mutex_init(&state->rc_rx_lock, NULL, MUTEX_DRIVER, NULL);
2885 mutex_init(&state->rc_tx_large_bufs_lock, NULL, MUTEX_DRIVER, NULL);
2886 mutex_init(&state->rc_srq_rwqe_list.dl_mutex, NULL, MUTEX_DRIVER, NULL);
2887 mutex_init(&state->rc_srq_free_list.dl_mutex, NULL, MUTEX_DRIVER, NULL);
2888 mutex_init(&state->rc_pass_chan_list.chan_list_mutex, NULL,
2890 mutex_init(&state->rc_timeout_lock, NULL, MUTEX_DRIVER, NULL);
2897 state->id_enable_rc = IBD_DEF_LINK_MODE;
2898 state->rc_mtu = IBD_DEF_RC_MAX_MTU;
2899 state->id_mtu = IBD_DEF_MAX_MTU;
2902 state->id_lso_policy = IBD_DEF_LSO_POLICY;
2903 state->id_num_lso_bufs = IBD_DEF_NUM_LSO_BUFS;
2904 state->id_num_ah = IBD_DEF_NUM_AH;
2905 state->id_hash_size = IBD_DEF_HASH_SIZE;
2906 state->id_create_broadcast_group = IBD_DEF_CREATE_BCAST_GROUP;
2907 state->id_allow_coalesce_comp_tuning = IBD_DEF_COALESCE_COMPLETIONS;
2908 state->id_ud_rx_comp_count = IBD_DEF_UD_RX_COMP_COUNT;
2909 state->id_ud_rx_comp_usec = IBD_DEF_UD_RX_COMP_USEC;
2910 state->id_ud_tx_comp_count = IBD_DEF_UD_TX_COMP_COUNT;
2911 state->id_ud_tx_comp_usec = IBD_DEF_UD_TX_COMP_USEC;
2912 state->id_rc_rx_comp_count = IBD_DEF_RC_RX_COMP_COUNT;
2913 state->id_rc_rx_comp_usec = IBD_DEF_RC_RX_COMP_USEC;
2914 state->id_rc_tx_comp_count = IBD_DEF_RC_TX_COMP_COUNT;
2915 state->id_rc_tx_comp_usec = IBD_DEF_RC_TX_COMP_USEC;
2916 state->id_ud_tx_copy_thresh = IBD_DEF_UD_TX_COPY_THRESH;
2917 state->id_rc_rx_copy_thresh = IBD_DEF_RC_RX_COPY_THRESH;
2918 state->id_rc_tx_copy_thresh = IBD_DEF_RC_TX_COPY_THRESH;
2919 state->id_ud_num_rwqe = IBD_DEF_UD_NUM_RWQE;
2920 state->id_ud_num_swqe = IBD_DEF_UD_NUM_SWQE;
2921 state->id_rc_num_rwqe = IBD_DEF_RC_NUM_RWQE;
2922 state->id_rc_num_swqe = IBD_DEF_RC_NUM_SWQE;
2923 state->rc_enable_srq = IBD_DEF_RC_ENABLE_SRQ;
2924 state->id_rc_num_srq = IBD_DEF_RC_NUM_SRQ;
2925 state->id_rc_rx_rwqe_thresh = IBD_DEF_RC_RX_RWQE_THRESH;
2934 ibd_state_fini(ibd_state_t *state)
2936 kmem_cache_destroy(state->id_req_kmc);
2938 mutex_destroy(&state->id_rx_list.dl_mutex);
2939 mutex_destroy(&state->id_rx_free_list.dl_mutex);
2941 mutex_destroy(&state->id_txpost_lock);
2942 mutex_destroy(&state->id_tx_list.dl_mutex);
2943 mutex_destroy(&state->id_tx_rel_list.dl_mutex);
2944 mutex_destroy(&state->id_lso_lock);
2946 mutex_destroy(&state->id_sched_lock);
2947 mutex_destroy(&state->id_scq_poll_lock);
2948 mutex_destroy(&state->id_rcq_poll_lock);
2950 cv_destroy(&state->id_trap_cv);
2951 mutex_destroy(&state->id_trap_lock);
2952 mutex_destroy(&state->id_link_mutex);
2955 mutex_destroy(&state->rc_timeout_lock);
2956 mutex_destroy(&state->rc_srq_free_list.dl_mutex);
2957 mutex_destroy(&state->rc_srq_rwqe_list.dl_mutex);
2958 mutex_destroy(&state->rc_pass_chan_list.chan_list_mutex);
2959 mutex_destroy(&state->rc_tx_large_bufs_lock);
2960 mutex_destroy(&state->rc_rx_lock);
2967 ibd_get_portspeed(ibd_state_t *state)
2987 path_attr.pa_dgids = &state->id_sgid;
2989 path_attr.pa_sgid = state->id_sgid;
2991 if (ibt_get_paths(state->id_ibt_hdl, IBT_PATH_NO_FLAGS,
3064 ibd_iba_join(ibd_state_t *state, ib_gid_t mgid, ibd_mce_t *mce)
3069 mcg_attr.mc_qkey = state->id_mcinfo->mc_qkey;
3072 mcg_attr.mc_scope = state->id_scope;
3073 mcg_attr.mc_pkey = state->id_pkey;
3074 mcg_attr.mc_flow = state->id_mcinfo->mc_adds_vect.av_flow;
3075 mcg_attr.mc_sl = state->id_mcinfo->mc_adds_vect.av_srvl;
3076 mcg_attr.mc_tclass = state->id_mcinfo->mc_adds_vect.av_tclass;
3077 return (ibt_join_mcg(state->id_sgid, &mcg_attr, &mce->mc_info,
3083 * state) so that IBA fabric will forward mcg packets to/from the port.
3087 * different join state, even though this is not required by SWG_0216,
3091 ibd_join_group(ibd_state_t *state, ib_gid_t mgid, uint8_t jstate)
3097 DPRINT(2, "ibd_join_group : join_group state %d : %016llx:%016llx\n",
3116 IBD_MCACHE_FIND_FULL(state, mgid)) != NULL)) {
3133 if ((ibt_status = ibd_iba_join(state, mgid, mce)) != IBT_SUCCESS) {
3142 * to the mcg in a different appropriate join state.
3145 tmce = IBD_MCACHE_FIND_FULL(state, mgid);
3149 if (IBD_MCACHE_FIND_NON(state, mgid) != NULL)
3160 if ((ibt_status = ibt_attach_mcg(state->id_chnl_hdl,
3169 (void) ibt_leave_mcg(state->id_sgid, mgid,
3170 state->id_sgid, jstate);
3180 IBD_MCACHE_INSERT_NON(state, mce);
3200 (void) ibt_leave_mcg(state->id_sgid, mgid,
3201 state->id_sgid, IB_MC_JSTATE_SEND_ONLY_NON);
3208 mutex_enter(&state->id_mc_mutex);
3209 IBD_MCACHE_INSERT_FULL(state, mce);
3210 mutex_exit(&state->id_mc_mutex);
3223 ibd_reacquire_group(ibd_state_t *state, ibd_mce_t *mce)
3241 (void) ibt_leave_mcg(state->id_sgid, mgid, state->id_sgid,
3243 if (ibd_iba_join(state, mgid, mce) != IBT_SUCCESS)
3244 ibd_print_warn(state, "Failure on port up to rejoin "
3254 * cleanups during driver detach time. Depending on the join state,
3260 ibd_async_reap_group(ibd_state_t *state, ibd_mce_t *mce, ib_gid_t mgid,
3279 mutex_enter(&state->id_mc_mutex);
3280 IBD_MCACHE_PULLOUT_FULL(state, mce);
3281 mutex_exit(&state->id_mc_mutex);
3282 if (IBD_MCACHE_FIND_NON(state, mgid) != NULL)
3285 IBD_MCACHE_PULLOUT_NON(state, mce);
3286 tmce = IBD_MCACHE_FIND_FULL(state, mgid);
3290 mutex_enter(&state->id_mc_mutex);
3291 IBD_MCACHE_PULLOUT_FULL(state, mce);
3292 mutex_exit(&state->id_mc_mutex);
3307 (void) ibt_detach_mcg(state->id_chnl_hdl, &mce->mc_info);
3310 (void) ibt_leave_mcg(state->id_sgid, mgid, state->id_sgid, jstate);
3322 ibd_leave_group(ibd_state_t *state, ib_gid_t mgid, uint8_t jstate)
3328 DPRINT(2, "ibd_leave_group : leave_group state %d : %016llx:%016llx\n",
3333 mce = IBD_MCACHE_FIND_NON(state, mgid);
3342 mce = IBD_MCACHE_FIND_FULL(state, mgid);
3382 mutex_enter(&state->id_ac_mutex);
3384 recycled = ibd_acache_recycle(state, &mcmac, (jstate ==
3386 mutex_exit(&state->id_ac_mutex);
3392 ibd_async_reap_group(state, mce, mgid, jstate);
3402 ibd_find_bgroup(ibd_state_t *state)
3414 state->id_bgroup_created = B_FALSE;
3415 state->id_bgroup_present = B_FALSE;
3419 mcg_attr.mc_pkey = state->id_pkey;
3420 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(state->id_mgid))
3421 state->id_mgid.gid_guid = IB_MGID_IPV4_LOWGRP_MASK;
3422 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(state->id_mgid))
3425 state->id_scope = mcg_attr.mc_scope = scopes[i];
3430 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(state->id_mgid))
3431 state->id_mgid.gid_prefix =
3433 ((uint64_t)state->id_scope << 48) |
3434 ((uint32_t)(state->id_pkey << 16)));
3435 mcg_attr.mc_mgid = state->id_mgid;
3436 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(state->id_mgid))
3437 if (ibt_query_mcg(state->id_sgid, &mcg_attr, 1,
3438 &state->id_mcinfo, &numg) == IBT_SUCCESS) {
3445 if (state->id_create_broadcast_group) {
3451 if (state->id_bgroup_created) {
3452 ibd_print_warn(state, "IPoIB broadcast group "
3464 mcg_attr.mc_pkey = state->id_pkey;
3468 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(state->id_mgid))
3469 state->id_mgid.gid_prefix =
3472 ((uint32_t)(state->id_pkey << 16)));
3473 mcg_attr.mc_mgid = state->id_mgid;
3474 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(state->id_mgid))
3476 if ((ret = ibt_join_mcg(state->id_sgid, &mcg_attr,
3478 ibd_print_warn(state, "IPoIB broadcast group "
3480 state->id_bgroup_created = B_FALSE;
3483 state->id_bgroup_created = B_TRUE;
3486 ibd_print_warn(state, "IPoIB broadcast group absent");
3494 mcgmtu = (128 << state->id_mcinfo->mc_mtu);
3495 if (state->id_mtu < mcgmtu) {
3496 ibd_print_warn(state, "IPoIB broadcast group MTU %d "
3498 state->id_mtu);
3499 ibt_free_mcg_info(state->id_mcinfo, 1);
3502 state->id_mtu = mcgmtu;
3503 state->id_bgroup_present = B_TRUE;
3508 if (state->id_bgroup_created) {
3509 (void) ibt_leave_mcg(state->id_sgid,
3510 mcg_info.mc_adds_vect.av_dgid, state->id_sgid,
3518 ibd_alloc_tx_copybufs(ibd_state_t *state)
3525 state->id_tx_buf_sz = state->id_mtu;
3526 if (state->id_lso_policy && state->id_lso_capable &&
3527 (state->id_ud_tx_copy_thresh > state->id_mtu)) {
3528 state->id_tx_buf_sz = state->id_ud_tx_copy_thresh;
3531 state->id_tx_bufs = kmem_zalloc(state->id_ud_num_swqe *
3532 state->id_tx_buf_sz, KM_SLEEP);
3534 state->id_tx_wqes = kmem_zalloc(state->id_ud_num_swqe *
3540 mem_attr.mr_vaddr = (uint64_t)(uintptr_t)state->id_tx_bufs;
3541 mem_attr.mr_len = state->id_ud_num_swqe * state->id_tx_buf_sz;
3544 if (ibt_register_mr(state->id_hca_hdl, state->id_pd_hdl, &mem_attr,
3545 &state->id_tx_mr_hdl, &state->id_tx_mr_desc) != IBT_SUCCESS) {
3547 kmem_free(state->id_tx_wqes,
3548 state->id_ud_num_swqe * sizeof (ibd_swqe_t));
3549 kmem_free(state->id_tx_bufs,
3550 state->id_ud_num_swqe * state->id_tx_buf_sz);
3551 state->id_tx_bufs = NULL;
3559 ibd_alloc_tx_lsobufs(ibd_state_t *state)
3579 memsz = state->id_num_lso_bufs * IBD_LSO_BUFSZ;
3586 if (ibt_register_mr(state->id_hca_hdl, state->id_pd_hdl,
3594 mutex_enter(&state->id_lso_lock);
3602 buflist = kmem_zalloc(state->id_num_lso_bufs * sizeof (ibd_lsobuf_t),
3610 for (i = 0; i < state->id_num_lso_bufs; i++) {
3623 * Set up the LSO buffer information in ibd state
3628 bktp->bkt_nelem = state->id_num_lso_bufs;
3631 state->id_lso = bktp;
3632 mutex_exit(&state->id_lso_lock);
3641 ibd_init_txlist(ibd_state_t *state)
3649 if (ibd_alloc_tx_copybufs(state) != DDI_SUCCESS)
3652 if (state->id_lso_policy && state->id_lso_capable) {
3653 if (ibd_alloc_tx_lsobufs(state) != DDI_SUCCESS)
3654 state->id_lso_capable = B_FALSE;
3657 mutex_enter(&state->id_tx_list.dl_mutex);
3658 state->id_tx_list.dl_head = NULL;
3659 state->id_tx_list.dl_pending_sends = B_FALSE;
3660 state->id_tx_list.dl_cnt = 0;
3661 mutex_exit(&state->id_tx_list.dl_mutex);
3662 mutex_enter(&state->id_tx_rel_list.dl_mutex);
3663 state->id_tx_rel_list.dl_head = NULL;
3664 state->id_tx_rel_list.dl_pending_sends = B_FALSE;
3665 state->id_tx_rel_list.dl_cnt = 0;
3666 mutex_exit(&state->id_tx_rel_list.dl_mutex);
3671 lkey = state->id_tx_mr_desc.md_lkey;
3672 bufaddr = state->id_tx_bufs;
3673 len = state->id_tx_buf_sz;
3674 swqe = state->id_tx_wqes;
3675 mutex_enter(&state->id_tx_list.dl_mutex);
3676 for (i = 0; i < state->id_ud_num_swqe; i++, swqe++, bufaddr += len) {
3695 state->id_tx_list.dl_cnt++;
3696 swqe->swqe_next = state->id_tx_list.dl_head;
3697 state->id_tx_list.dl_head = SWQE_TO_WQE(swqe);
3699 mutex_exit(&state->id_tx_list.dl_mutex);
3705 ibd_acquire_lsobufs(ibd_state_t *state, uint_t req_sz, ibt_wr_ds_t *sgl_p,
3727 mutex_enter(&state->id_lso_lock);
3732 ASSERT(state->id_lso != NULL);
3733 bktp = state->id_lso;
3735 mutex_exit(&state->id_lso_lock);
3775 mutex_exit(&state->id_lso_lock);
3783 ibd_release_lsobufs(ibd_state_t *state, ibt_wr_ds_t *sgl_p, uint32_t nds)
3791 mutex_enter(&state->id_lso_lock);
3793 bktp = state->id_lso;
3819 mutex_exit(&state->id_lso_lock);
3823 ibd_free_tx_copybufs(ibd_state_t *state)
3828 if (ibt_deregister_mr(state->id_hca_hdl,
3829 state->id_tx_mr_hdl) != IBT_SUCCESS) {
3832 state->id_tx_mr_hdl = NULL;
3837 kmem_free(state->id_tx_wqes, state->id_ud_num_swqe *
3839 kmem_free(state->id_tx_bufs, state->id_ud_num_swqe *
3840 state->id_tx_buf_sz);
3841 state->id_tx_wqes = NULL;
3842 state->id_tx_bufs = NULL;
3846 ibd_free_tx_lsobufs(ibd_state_t *state)
3850 mutex_enter(&state->id_lso_lock);
3852 if ((bktp = state->id_lso) == NULL) {
3853 mutex_exit(&state->id_lso_lock);
3867 if (ibt_deregister_mr(state->id_hca_hdl,
3879 state->id_lso = NULL;
3881 mutex_exit(&state->id_lso_lock);
3888 ibd_fini_txlist(ibd_state_t *state)
3893 mutex_enter(&state->id_tx_list.dl_mutex);
3894 mutex_enter(&state->id_tx_rel_list.dl_mutex);
3895 state->id_tx_list.dl_head = NULL;
3896 state->id_tx_list.dl_pending_sends = B_FALSE;
3897 state->id_tx_list.dl_cnt = 0;
3898 state->id_tx_rel_list.dl_head = NULL;
3899 state->id_tx_rel_list.dl_pending_sends = B_FALSE;
3900 state->id_tx_rel_list.dl_cnt = 0;
3901 mutex_exit(&state->id_tx_rel_list.dl_mutex);
3902 mutex_exit(&state->id_tx_list.dl_mutex);
3904 ibd_free_tx_lsobufs(state);
3905 ibd_free_tx_copybufs(state);
3912 ibd_post_recv_list(ibd_state_t *state, ibd_rwqe_t *rwqe)
3939 atomic_add_32(&state->id_rx_list.dl_cnt, i);
3942 ibt_status = ibt_post_recv(state->id_chnl_hdl, wrs, i,
3946 ibd_print_warn(state, "ibd_post_recv: FATAL: "
3950 atomic_add_32(&state->id_rx_list.dl_cnt,
3960 ibd_post_recv_intr(ibd_state_t *state)
3966 state->id_rx_post_queue_index =
3967 (state->id_rx_post_queue_index + 1) &
3968 (state->id_rx_nqueues - 1);
3970 rxp = state->id_rx_queues + state->id_rx_post_queue_index;
3976 ibd_post_recv_list(state, list);
3981 (((uintptr_t)(rwqe) >> 8) & (state->id_rx_nqueues - 1))
3995 ibd_post_recv(ibd_state_t *state, ibd_rwqe_t *rwqe)
3999 rxp = state->id_rx_queues + RX_QUEUE_HASH(rwqe);
4003 rxp = state->id_rx_queues + RX_QUEUE_HASH(rwqe + 16);
4008 uint_t active = atomic_inc_32_nv(&state->id_rx_post_active);
4011 if ((active & (state->id_rx_nqueues - 1)) == 0) {
4015 ibd_post_recv_list(state, rwqe);
4024 ibd_alloc_rx_copybufs(ibd_state_t *state)
4032 state->id_rx_buf_sz = state->id_mtu + IPOIB_GRH_SIZE;
4034 state->id_rx_bufs = kmem_zalloc(state->id_ud_num_rwqe *
4035 state->id_rx_buf_sz, KM_SLEEP);
4037 state->id_rx_wqes = kmem_zalloc(state->id_ud_num_rwqe *
4040 state->id_rx_nqueues = 1 << IBD_LOG_RX_POST;
4041 state->id_rx_queues = kmem_zalloc(state->id_rx_nqueues *
4043 for (i = 0; i < state->id_rx_nqueues; i++) {
4044 ibd_rx_queue_t *rxp = state->id_rx_queues + i;
4051 mem_attr.mr_vaddr = (uint64_t)(uintptr_t)state->id_rx_bufs;
4052 mem_attr.mr_len = state->id_ud_num_rwqe * state->id_rx_buf_sz;
4055 if (ibt_register_mr(state->id_hca_hdl, state->id_pd_hdl, &mem_attr,
4056 &state->id_rx_mr_hdl, &state->id_rx_mr_desc) != IBT_SUCCESS) {
4058 kmem_free(state->id_rx_wqes,
4059 state->id_ud_num_rwqe * sizeof (ibd_rwqe_t));
4060 kmem_free(state->id_rx_bufs,
4061 state->id_ud_num_rwqe * state->id_rx_buf_sz);
4062 state->id_rx_bufs = NULL;
4063 state->id_rx_wqes = NULL;
4074 ibd_init_rxlist(ibd_state_t *state)
4083 mutex_enter(&state->id_rx_free_list.dl_mutex);
4084 if (state->id_rx_free_list.dl_head != NULL) {
4086 len = state->id_rx_buf_sz;
4087 list = state->id_rx_free_list.dl_head;
4088 state->id_rx_free_list.dl_head = NULL;
4089 state->id_rx_free_list.dl_cnt = 0;
4090 mutex_exit(&state->id_rx_free_list.dl_mutex);
4097 if (atomic_dec_32_nv(&state->id_running) != 0) {
4107 atomic_inc_32(&state->
4112 ibd_free_rwqe(state, rwqe);
4114 atomic_inc_32(&state->id_running);
4118 ibd_post_recv_list(state, WQE_TO_RWQE(list));
4121 mutex_exit(&state->id_rx_free_list.dl_mutex);
4123 if (ibd_alloc_rx_copybufs(state) != DDI_SUCCESS)
4129 len = state->id_rx_buf_sz;
4130 lkey = state->id_rx_mr_desc.md_lkey;
4131 rwqe = state->id_rx_wqes;
4132 bufaddr = state->id_rx_bufs;
4134 for (i = 0; i < state->id_ud_num_rwqe; i++, rwqe++, bufaddr += len) {
4135 rwqe->w_state = state;
4145 if (atomic_dec_32_nv(&state->id_running) != 0) {
4156 atomic_inc_32(&state->id_running);
4159 mutex_enter(&state->id_rx_free_list.dl_mutex);
4160 state->id_rx_free_list.dl_head = NULL;
4161 state->id_rx_free_list.dl_cnt = 0;
4162 mutex_exit(&state->id_rx_free_list.dl_mutex);
4164 ibd_fini_rxlist(state);
4179 ibd_post_recv_list(state, WQE_TO_RWQE(list));
4185 ibd_free_rx_copybufs(ibd_state_t *state)
4192 if (ibt_deregister_mr(state->id_hca_hdl,
4193 state->id_rx_mr_hdl) != IBT_SUCCESS) {
4196 state->id_rx_mr_hdl = NULL;
4201 for (i = 0; i < state->id_rx_nqueues; i++) {
4202 ibd_rx_queue_t *rxp = state->id_rx_queues + i;
4205 kmem_free(state->id_rx_queues, state->id_rx_nqueues *
4207 kmem_free(state->id_rx_wqes, state->id_ud_num_rwqe *
4209 kmem_free(state->id_rx_bufs, state->id_ud_num_rwqe *
4210 state->id_rx_buf_sz);
4211 state->id_rx_queues = NULL;
4212 state->id_rx_wqes = NULL;
4213 state->id_rx_bufs = NULL;
4217 ibd_free_rx_rsrcs(ibd_state_t *state)
4219 mutex_enter(&state->id_rx_free_list.dl_mutex);
4220 if (state->id_rx_free_list.dl_head == NULL) {
4222 mutex_exit(&state->id_rx_free_list.dl_mutex);
4225 ASSERT(state->id_rx_free_list.dl_cnt == state->id_ud_num_rwqe);
4226 ibd_free_rx_copybufs(state);
4227 state->id_rx_free_list.dl_cnt = 0;
4228 state->id_rx_free_list.dl_head = NULL;
4229 mutex_exit(&state->id_rx_free_list.dl_mutex);
4236 ibd_fini_rxlist(ibd_state_t *state)
4242 for (i = 0; i < state->id_rx_nqueues; i++) {
4243 ibd_rx_queue_t *rxp = state->id_rx_queues + i;
4255 if (atomic_add_32_nv(&state->id_rx_list.dl_bufs_outstanding, 0) == 0)
4256 ibd_free_rx_rsrcs(state);
4264 ibd_free_rwqe(ibd_state_t *state, ibd_rwqe_t *rwqe)
4275 mutex_enter(&state->id_rx_free_list.dl_mutex);
4276 state->id_rx_free_list.dl_cnt++;
4277 rwqe->rwqe_next = state->id_rx_free_list.dl_head;
4278 state->id_rx_free_list.dl_head = RWQE_TO_WQE(rwqe);
4279 mutex_exit(&state->id_rx_free_list.dl_mutex);
4290 ibd_state_t *state = (ibd_state_t *)arg;
4292 atomic_inc_64(&state->id_num_intrs);
4295 mutex_enter(&state->id_rcq_poll_lock);
4296 if (state->id_rcq_poll_busy & IBD_CQ_POLLING) {
4297 state->id_rcq_poll_busy |= IBD_REDO_CQ_POLLING;
4298 mutex_exit(&state->id_rcq_poll_lock);
4301 mutex_exit(&state->id_rcq_poll_lock);
4302 ddi_trigger_softintr(state->id_rx);
4305 (void) ibd_intr((caddr_t)state);
4316 ibd_state_t *state = (ibd_state_t *)arg;
4318 atomic_inc_64(&state->id_num_intrs);
4321 mutex_enter(&state->id_scq_poll_lock);
4322 if (state->id_scq_poll_busy & IBD_CQ_POLLING) {
4323 state->id_scq_poll_busy |= IBD_REDO_CQ_POLLING;
4324 mutex_exit(&state->id_scq_poll_lock);
4327 mutex_exit(&state->id_scq_poll_lock);
4328 ddi_trigger_softintr(state->id_tx);
4331 (void) ibd_tx_recycle((caddr_t)state);
4345 ibd_state_t *state = (ibd_state_t *)arg;
4356 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(state->id_sgid))
4357 if (bcmp(&gid, &state->id_sgid, sizeof (ib_gid_t)) != 0)
4359 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(state->id_sgid))
4373 ibd_print_warn(state, "IBA multicast support "
4383 ibd_print_warn(state, "IBA multicast support "
4393 if (((state->id_mac_state & IBD_DRV_IN_LATE_HCA_INIT) ==
4407 if (!ibd_async_safe(state))
4410 req = kmem_cache_alloc(state->id_req_kmc, KM_SLEEP);
4413 ibd_queue_work_slot(state, req, IBD_ASYNC_TRAP);
4419 ibd_async_trap(ibd_state_t *state, ibd_req_t *req)
4435 if (((state->id_mac_state & IBD_DRV_IN_LATE_HCA_INIT) ==
4436 IBD_DRV_IN_LATE_HCA_INIT) && (state->id_bgroup_present == 0) &&
4444 state->id_pkey)) {
4445 ibd_async_done(state);
4448 ibd_set_mac_progress(state, IBD_DRV_RESTART_IN_PROGRESS);
4455 if (((state->id_mac_state & IBD_DRV_IN_LATE_HCA_INIT) ==
4457 ((ret = ibd_start(state)) != 0)) {
4461 ibd_clr_mac_progress(state, IBD_DRV_RESTART_IN_PROGRESS);
4462 ibd_async_done(state);
4470 ibd_leave_group(state, mgid, IB_MC_JSTATE_SEND_ONLY_NON);
4472 if (state->id_prom_op == IBD_OP_COMPLETED) {
4473 ibd_leave_group(state, mgid, IB_MC_JSTATE_NON);
4485 if ((ibd_join_group(state, mgid, IB_MC_JSTATE_NON) ==
4487 ibd_print_warn(state, "IBA promiscuous mode missed "
4496 ibd_async_done(state);
4505 ibd_state_t *state = arg;
4507 if (state->id_type == IBD_PORT_DRIVER)
4517 if (state->id_hwcksum_capab & IBT_HCA_CKSUM_FULL)
4535 if (!state->id_lso_policy || !state->id_lso_capable)
4538 if ((state->id_hwcksum_capab & IBT_HCA_CKSUM_FULL) == 0)
4541 if (state->id_hca_res_lkey_capab == 0) {
4542 ibd_print_warn(state, "no reserved-lkey capability, "
4548 cap_lso->lso_basic_tcp_ipv4.lso_max = state->id_lso_maxlen - 1;
4566 ibd_state_t *state = arg;
4571 if (state->id_type == IBD_PORT_DRIVER) {
4577 if (state->id_mac_state & IBD_DRV_STARTED) {
4591 if (state->id_enable_rc) {
4594 state->id_enable_rc = 1;
4596 err = mac_maxsdu_update2(state->id_mh,
4597 state->rc_mtu - IPOIB_HDRSIZE,
4598 state->id_mtu - IPOIB_HDRSIZE);
4600 if (!state->id_enable_rc) {
4603 state->id_enable_rc = 0;
4604 err = mac_maxsdu_update2(state->id_mh,
4605 state->id_mtu - IPOIB_HDRSIZE,
4606 state->id_mtu - IPOIB_HDRSIZE);
4608 (void) ibd_record_capab(state);
4609 mac_capab_update(state->id_mh);
4613 err = ibd_set_priv_prop(state, pr_name,
4627 ibd_state_t *state = arg;
4634 if (state->id_type == IBD_PORT_DRIVER) {
4642 *(uint_t *)pr_val = state->id_enable_rc;
4645 err = ibd_get_priv_prop(state, pr_name, pr_valsize,
4659 ibd_state_t *state = arg;
4668 if (state->id_type == IBD_PORT_DRIVER) {
4671 } else if (state->id_enable_rc) {
4674 min = max = state->id_mtu - IPOIB_HDRSIZE;
4749 ibd_set_priv_prop(ibd_state_t *state, const char *pr_name,
4763 state->id_allow_coalesce_comp_tuning = (result == 1) ?
4769 if (state->id_mac_state & IBD_DRV_STARTED) {
4779 state->id_create_broadcast_group = (result == 1) ?
4785 if (state->id_mac_state & IBD_DRV_STARTED) {
4795 state->id_hash_size = (uint32_t)result;
4800 if (state->id_mac_state & IBD_DRV_STARTED) {
4810 state->id_lso_policy = (result == 1) ?
4813 mac_capab_update(state->id_mh);
4817 if (state->id_mac_state & IBD_DRV_STARTED) {
4827 state->id_num_ah = (uint32_t)result;
4832 if (state->id_mac_state & IBD_DRV_STARTED) {
4835 if (!state->id_lso_policy || !state->id_lso_capable) {
4846 state->id_num_lso_bufs = (uint32_t)result;
4851 if (state->id_mac_state & IBD_DRV_STARTED) {
4861 state->rc_enable_srq = (result == 1) ?
4864 if (!state->rc_enable_srq) {
4865 state->id_rc_num_srq = 0;
4870 if (state->id_mac_state & IBD_DRV_STARTED) {
4881 state->id_rc_num_rwqe = (uint32_t)result;
4882 if (state->id_allow_coalesce_comp_tuning &&
4883 state->id_rc_rx_comp_count > state->id_rc_num_rwqe)
4884 state->id_rc_rx_comp_count =
4885 state->id_rc_num_rwqe;
4886 if (state->id_rc_num_srq > state->id_rc_num_rwqe)
4887 state->id_rc_num_srq =
4888 state->id_rc_num_rwqe - 1;
4893 if (state->id_rc_rx_rwqe_thresh > state->id_rc_num_rwqe)
4894 state->id_rc_rx_rwqe_thresh =
4895 (state->id_rc_num_rwqe >> 2);
4901 if (state->id_mac_state & IBD_DRV_STARTED) {
4907 if (!state->rc_enable_srq)
4912 result >= state->id_rc_num_rwqe) {
4915 state->id_rc_num_srq = (uint32_t)result;
4919 if (state->id_mac_state & IBD_DRV_STARTED) {
4930 state->id_rc_num_swqe = (uint32_t)result;
4931 if (state->id_allow_coalesce_comp_tuning &&
4932 state->id_rc_tx_comp_count > state->id_rc_num_swqe)
4933 state->id_rc_tx_comp_count =
4934 state->id_rc_num_swqe;
4939 if (!state->id_allow_coalesce_comp_tuning) {
4946 if (result < 1 || result > state->id_rc_num_rwqe) {
4949 state->id_rc_rx_comp_count = (uint32_t)result;
4954 if (!state->id_allow_coalesce_comp_tuning) {
4964 state->id_rc_rx_comp_usec = (uint32_t)result;
4969 if (state->id_mac_state & IBD_DRV_STARTED) {
4977 result > state->rc_mtu) {
4980 state->id_rc_rx_copy_thresh = (uint32_t)result;
4985 if (state->id_mac_state & IBD_DRV_STARTED) {
4993 result >= state->id_rc_num_rwqe) {
4996 state->id_rc_rx_rwqe_thresh = (uint32_t)result;
5001 if (!state->id_allow_coalesce_comp_tuning) {
5008 if (result < 1 || result > state->id_rc_num_swqe) {
5011 state->id_rc_tx_comp_count = (uint32_t)result;
5016 if (!state->id_allow_coalesce_comp_tuning) {
5026 state->id_rc_tx_comp_usec = (uint32_t)result;
5031 if (state->id_mac_state & IBD_DRV_STARTED) {
5039 result > state->rc_mtu) {
5042 state->id_rc_tx_copy_thresh = (uint32_t)result;
5047 if (state->id_mac_state & IBD_DRV_STARTED) {
5058 if (result > state->id_hca_max_chan_sz) {
5059 state->id_ud_num_rwqe =
5060 state->id_hca_max_chan_sz;
5062 state->id_ud_num_rwqe = (uint32_t)result;
5064 if (state->id_allow_coalesce_comp_tuning &&
5065 state->id_ud_rx_comp_count > state->id_ud_num_rwqe)
5066 state->id_ud_rx_comp_count =
5067 state->id_ud_num_rwqe;
5072 if (state->id_mac_state & IBD_DRV_STARTED) {
5083 if (result > state->id_hca_max_chan_sz) {
5084 state->id_ud_num_swqe =
5085 state->id_hca_max_chan_sz;
5087 state->id_ud_num_swqe = (uint32_t)result;
5089 if (state->id_allow_coalesce_comp_tuning &&
5090 state->id_ud_tx_comp_count > state->id_ud_num_swqe)
5091 state->id_ud_tx_comp_count =
5092 state->id_ud_num_swqe;
5097 if (!state->id_allow_coalesce_comp_tuning) {
5104 if (result < 1 || result > state->id_ud_num_rwqe) {
5107 state->id_ud_rx_comp_count = (uint32_t)result;
5112 if (!state->id_allow_coalesce_comp_tuning) {
5122 state->id_ud_rx_comp_usec = (uint32_t)result;
5127 if (!state->id_allow_coalesce_comp_tuning) {
5134 if (result < 1 || result > state->id_ud_num_swqe) {
5137 state->id_ud_tx_comp_count = (uint32_t)result;
5142 if (!state->id_allow_coalesce_comp_tuning) {
5152 state->id_ud_tx_comp_usec = (uint32_t)result;
5157 if (state->id_mac_state & IBD_DRV_STARTED) {
5168 state->id_ud_tx_copy_thresh = (uint32_t)result;
5176 ibd_get_priv_prop(ibd_state_t *state, const char *pr_name, uint_t pr_valsize,
5183 value = state->id_bgroup_present;
5188 value = state->id_allow_coalesce_comp_tuning;
5193 value = state->id_create_broadcast_group;
5198 value = state->id_hash_size;
5203 value = state->id_lso_policy;
5208 value = state->id_num_ah;
5213 value = state->id_num_lso_bufs;
5218 value = state->rc_enable_srq;
5223 value = state->id_rc_num_rwqe;
5228 value = state->id_rc_num_srq;
5233 value = state->id_rc_num_swqe;
5238 value = state->id_rc_rx_comp_count;
5243 value = state->id_rc_rx_comp_usec;
5248 value = state->id_rc_rx_copy_thresh;
5253 value = state->id_rc_rx_rwqe_thresh;
5258 value = state->id_rc_tx_comp_count;
5263 value = state->id_rc_tx_comp_usec;
5268 value = state->id_rc_tx_copy_thresh;
5273 value = state->id_ud_num_rwqe;
5278 value = state->id_ud_num_swqe;
5283 value = state->id_ud_rx_comp_count;
5288 value = state->id_ud_rx_comp_usec;
5293 value = state->id_ud_tx_comp_count;
5298 value = state->id_ud_tx_comp_usec;
5303 value = state->id_ud_tx_copy_thresh;
5315 ibd_get_port_details(ibd_state_t *state)
5321 mutex_enter(&state->id_link_mutex);
5326 ret = ibt_query_hca_ports(state->id_hca_hdl, state->id_port,
5329 mutex_exit(&state->id_link_mutex);
5339 if ((ret = ibt_pkey2index(state->id_hca_hdl, state->id_port,
5340 state->id_pkey, &state->id_pkix)) != IBT_SUCCESS) {
5341 state->id_link_state = LINK_STATE_DOWN;
5343 state->id_link_state = LINK_STATE_UP;
5345 state->id_mtu = (128 << port_infop->p_mtu);
5346 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(state->id_sgid))
5347 state->id_sgid = *port_infop->p_sgid_tbl;
5348 _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(state->id_sgid))
5352 state->id_link_speed = ibd_get_portspeed(state);
5355 state->id_mtu = 0;
5356 state->id_link_state = LINK_STATE_DOWN;
5357 state->id_link_speed = 0;
5359 mutex_exit(&state->id_link_mutex);
5366 ibd_alloc_cqs(ibd_state_t *state)
5375 ret = ibt_query_hca(state->id_hca_hdl, &hca_attrs);
5392 if (hca_attrs.hca_max_cq_sz >= (state->id_ud_num_rwqe + 1)) {
5393 cq_attr.cq_size = state->id_ud_num_rwqe + 1;
5396 num_rwqe_change = state->id_ud_num_rwqe;
5397 state->id_ud_num_rwqe = cq_attr.cq_size - 1;
5400 if ((ret = ibt_alloc_cq(state->id_hca_hdl, &cq_attr,
5401 &state->id_rcq_hdl, &real_size)) != IBT_SUCCESS) {
5407 if ((ret = ibt_modify_cq(state->id_rcq_hdl, state->id_ud_rx_comp_count,
5408 state->id_ud_rx_comp_usec, 0)) != IBT_SUCCESS) {
5414 state->id_rxwcs_size = IBD_MAX_RX_MP_LEN;
5415 state->id_rxwcs = kmem_alloc(sizeof (ibt_wc_t) *
5416 state->id_rxwcs_size, KM_SLEEP);
5421 if (hca_attrs.hca_max_cq_sz >= (state->id_ud_num_swqe + 1)) {
5422 cq_attr.cq_size = state->id_ud_num_swqe + 1;
5425 num_swqe_change = state->id_ud_num_swqe;
5426 state->id_ud_num_swqe = cq_attr.cq_size - 1;
5429 if ((ret = ibt_alloc_cq(state->id_hca_hdl, &cq_attr,
5430 &state->id_scq_hdl, &real_size)) != IBT_SUCCESS) {
5433 kmem_free(state->id_rxwcs, sizeof (ibt_wc_t) *
5434 state->id_rxwcs_size);
5435 (void) ibt_free_cq(state->id_rcq_hdl);
5438 if ((ret = ibt_modify_cq(state->id_scq_hdl, state->id_ud_tx_comp_count,
5439 state->id_ud_tx_comp_usec, 0)) != IBT_SUCCESS) {
5444 state->id_txwcs_size = IBD_TX_POLL_THRESH;
5445 state->id_txwcs = kmem_alloc(sizeof (ibt_wc_t) *
5446 state->id_txwcs_size, KM_SLEEP);
5453 ibd_print_warn(state, "Setting #rwqe = %d instead of default "
5454 "%d", state->id_ud_num_rwqe, num_rwqe_change);
5457 ibd_print_warn(state, "Setting #swqe = %d instead of default "
5458 "%d", state->id_ud_num_swqe, num_swqe_change);
5465 ibd_setup_ud_channel(ibd_state_t *state)
5472 if (state->id_hca_res_lkey_capab)
5474 if (state->id_lso_policy && state->id_lso_capable)
5477 ud_alloc_attr.ud_hca_port_num = state->id_port;
5478 ud_alloc_attr.ud_sizes.cs_sq_sgl = state->id_max_sqseg;
5480 ud_alloc_attr.ud_sizes.cs_sq = state->id_ud_num_swqe;
5481 ud_alloc_attr.ud_sizes.cs_rq = state->id_ud_num_rwqe;
5482 ud_alloc_attr.ud_qkey = state->id_mcinfo->mc_qkey;
5483 ud_alloc_attr.ud_scq = state->id_scq_hdl;
5484 ud_alloc_attr.ud_rcq = state->id_rcq_hdl;
5485 ud_alloc_attr.ud_pd = state->id_pd_hdl;
5486 ud_alloc_attr.ud_pkey_ix = state->id_pkix;
5489 if ((ret = ibt_alloc_ud_channel(state->id_hca_hdl, IBT_ACHAN_NO_FLAGS,
5490 &ud_alloc_attr, &state->id_chnl_hdl, NULL)) != IBT_SUCCESS) {
5496 if ((ret = ibt_query_ud_channel(state->id_chnl_hdl,
5500 (void) ibt_free_channel(state->id_chnl_hdl);
5504 state->id_qpnum = ud_chan_attr.ud_qpn;
5510 ibd_undo_start(ibd_state_t *state, link_state_t cur_link_state)
5512 uint32_t progress = state->id_mac_state;
5520 if (atomic_dec_32_nv(&state->id_running) != 0)
5525 * we need to mark the link state appropriately to prevent the
5527 * that if the original state of the link was "up" when we're
5528 * here, we'll set the final link state to "unknown", to behave
5531 mutex_enter(&state->id_link_mutex);
5533 state->id_link_state = cur_link_state;
5535 state->id_link_state = LINK_STATE_UNKNOWN;
5537 mutex_exit(&state->id_link_mutex);
5538 bzero(&state->id_macaddr, sizeof (ipoib_mac_t));
5539 mac_link_update(state->id_mh, state->id_link_state);
5541 state->id_mac_state &= (~IBD_DRV_PORT_DETAILS_OBTAINED);
5543 state->id_mac_state &= (~IBD_DRV_STARTED);
5547 state->id_mac_state &= (~IBD_DRV_IN_LATE_HCA_INIT);
5552 ASSERT(state->id_enable_rc);
5553 if (state->rc_listen_hdl != NULL) {
5554 ibd_rc_stop_listen(state);
5556 state->id_mac_state &= (~IBD_DRV_RC_LISTEN);
5561 ASSERT(state->id_enable_rc);
5562 mutex_enter(&state->rc_timeout_lock);
5563 state->rc_timeout_start = B_FALSE;
5564 tid = state->rc_timeout;
5565 state->rc_timeout = 0;
5566 mutex_exit(&state->rc_timeout_lock);
5569 state->id_mac_state &= (~IBD_DRV_RC_TIMEOUT);
5572 if ((state->id_enable_rc) && (progress & IBD_DRV_ACACHE_INITIALIZED)) {
5574 while (state->id_ah_op == IBD_OP_ONGOING) {
5576 * "state->id_ah_op == IBD_OP_ONGOING" means this IPoIB
5582 state->rc_stop_connect++;
5587 mutex_enter(&state->id_sched_lock);
5588 state->id_sched_needed = 0;
5589 mutex_exit(&state->id_sched_lock);
5590 (void) ibd_rc_close_all_chan(state);
5600 while (atomic_add_32_nv(&state->id_rx_list.dl_bufs_outstanding,
5618 state->id_mac_state &= (~IBD_DRV_RCQ_NOTIFY_ENABLED);
5622 ibd_rc_fini_tx_largebuf_list(state);
5623 state->id_mac_state &= (~IBD_DRV_RC_LARGEBUF_ALLOCD);
5627 ASSERT(state->id_enable_rc);
5628 if (state->rc_srq_rwqe_list.dl_bufs_outstanding == 0) {
5629 if (state->id_ah_op == IBD_OP_ONGOING) {
5631 if (state->id_ah_op == IBD_OP_ONGOING) {
5633 * "state->id_ah_op == IBD_OP_ONGOING"
5638 state->rc_stop_connect++;
5642 ibd_rc_fini_srq_list(state);
5643 state->id_mac_state &=
5647 ibd_rc_fini_srq_list(state);
5648 state->id_mac_state &= (~IBD_DRV_RC_SRQ_ALLOCD);
5656 ibt_register_subnet_notices(state->id_ibt_hdl, NULL, NULL);
5658 mutex_enter(&state->id_trap_lock);
5659 state->id_trap_stop = B_TRUE;
5660 while (state->id_trap_inprog > 0)
5661 cv_wait(&state->id_trap_cv, &state->id_trap_lock);
5662 mutex_exit(&state->id_trap_lock);
5664 state->id_mac_state &= (~IBD_DRV_SM_NOTICES_REGISTERED);
5675 if ((ret = ibt_flush_channel(state->id_chnl_hdl)) !=
5686 mutex_enter(&state->id_tx_list.dl_mutex);
5687 mutex_enter(&state->id_tx_rel_list.dl_mutex);
5688 while (state->id_tx_list.dl_cnt + state->id_tx_rel_list.dl_cnt
5689 != state->id_ud_num_swqe) {
5692 mutex_exit(&state->id_tx_rel_list.dl_mutex);
5693 mutex_exit(&state->id_tx_list.dl_mutex);
5695 mutex_enter(&state->id_tx_list.dl_mutex);
5696 mutex_enter(&state->id_tx_rel_list.dl_mutex);
5698 ibt_set_cq_handler(state->id_scq_hdl, 0, 0);
5699 if (state->id_tx_list.dl_cnt + state->id_tx_rel_list.dl_cnt !=
5700 state->id_ud_num_swqe) {
5703 mutex_exit(&state->id_tx_rel_list.dl_mutex);
5704 mutex_exit(&state->id_tx_list.dl_mutex);
5707 while (atomic_add_32_nv(&state->id_rx_list.dl_cnt, 0) != 0) {
5712 ibt_set_cq_handler(state->id_rcq_hdl, 0, 0);
5713 if (atomic_add_32_nv(&state->id_rx_list.dl_cnt, 0) != 0) {
5717 state->id_mac_state &= (~IBD_DRV_SCQ_NOTIFY_ENABLED);
5732 mce = list_head(&state->id_mc_full);
5736 mce = list_next(&state->id_mc_full, mce);
5737 ibd_leave_group(state, mgid, jstate);
5739 state->id_mac_state &= (~IBD_DRV_BCAST_GROUP_JOINED);
5743 ibd_fini_rxlist(state);
5744 state->id_mac_state &= (~IBD_DRV_RXLIST_ALLOCD);
5748 ibd_fini_txlist(state);
5749 state->id_mac_state &= (~IBD_DRV_TXLIST_ALLOCD);
5753 if ((ret = ibt_free_channel(state->id_chnl_hdl)) !=
5759 state->id_mac_state &= (~IBD_DRV_UD_CHANNEL_SETUP);
5763 kmem_free(state->id_txwcs,
5764 sizeof (ibt_wc_t) * state->id_txwcs_size);
5765 if ((ret = ibt_free_cq(state->id_scq_hdl)) !=
5771 kmem_free(state->id_rxwcs,
5772 sizeof (ibt_wc_t) * state->id_rxwcs_size);
5773 if ((ret = ibt_free_cq(state->id_rcq_hdl)) != IBT_SUCCESS) {
5778 state->id_txwcs = NULL;
5779 state->id_rxwcs = NULL;
5780 state->id_scq_hdl = NULL;
5781 state->id_rcq_hdl = NULL;
5783 state->id_mac_state &= (~IBD_DRV_CQS_ALLOCD);
5787 mutex_enter(&state->id_ac_mutex);
5788 mod_hash_destroy_hash(state->id_ah_active_hash);
5789 mutex_exit(&state->id_ac_mutex);
5790 ibd_acache_fini(state);
5792 state->id_mac_state &= (~IBD_DRV_ACACHE_INITIALIZED);
5800 if (state->id_bgroup_created) {
5801 mgid = state->id_mcinfo->mc_adds_vect.av_dgid;
5803 (void) ibt_leave_mcg(state->id_sgid, mgid,
5804 state->id_sgid, jstate);
5806 ibt_free_mcg_info(state->id_mcinfo, 1);
5808 state->id_mac_state &= (~IBD_DRV_BCAST_GROUP_FOUND);
5823 ibd_set_mac_progress(ibd_state_t *state, uint_t flag)
5825 mutex_enter(&state->id_macst_lock);
5826 while (state->id_mac_state & IBD_DRV_RESTART_IN_PROGRESS)
5827 cv_wait(&state->id_macst_cv, &state->id_macst_lock);
5829 state->id_mac_state |= flag;
5830 mutex_exit(&state->id_macst_lock);
5834 ibd_clr_mac_progress(ibd_state_t *state, uint_t flag)
5836 mutex_enter(&state->id_macst_lock);
5837 state->id_mac_state &= (~flag);
5838 cv_signal(&state->id_macst_cv);
5839 mutex_exit(&state->id_macst_lock);
5849 ibd_state_t *state = arg;
5852 if (state->id_type == IBD_PORT_DRIVER)
5855 ibd_set_mac_progress(state, IBD_DRV_START_IN_PROGRESS);
5856 if (state->id_mac_state & IBD_DRV_IN_DELETION) {
5857 ibd_clr_mac_progress(state, IBD_DRV_START_IN_PROGRESS);
5861 ret = ibd_start(state);
5862 ibd_clr_mac_progress(state, IBD_DRV_START_IN_PROGRESS);
5867 ibd_start(ibd_state_t *state)
5873 if (state->id_mac_state & IBD_DRV_STARTED)
5878 * a result of some event which moves the state away from late HCA
5881 if (!(state->id_mac_state & IBD_DRV_IN_LATE_HCA_INIT) &&
5882 (atomic_inc_32_nv(&state->id_running) != 1)) {
5885 atomic_dec_32(&state->id_running);
5893 if ((err = ibd_get_port_details(state)) != 0) {
5898 * If state->id_link_state is DOWN, it indicates that either the port
5902 state->id_mac_state |= IBD_DRV_PORT_DETAILS_OBTAINED;
5903 if (state->id_link_state == LINK_STATE_DOWN) {
5911 if (ibd_find_bgroup(state) != IBT_SUCCESS) {
5916 state->id_mac_state |= IBD_DRV_BCAST_GROUP_FOUND;
5922 if (ibd_acache_init(state) != DDI_SUCCESS) {
5927 state->id_mac_state |= IBD_DRV_ACACHE_INITIALIZED;
5932 if (ibd_alloc_cqs(state) != DDI_SUCCESS) {
5937 state->id_mac_state |= IBD_DRV_CQS_ALLOCD;
5942 if (ibd_setup_ud_channel(state) != DDI_SUCCESS) {
5947 state->id_mac_state |= IBD_DRV_UD_CHANNEL_SETUP;
5952 if (ibd_init_txlist(state) != DDI_SUCCESS) {
5957 state->id_mac_state |= IBD_DRV_TXLIST_ALLOCD;
5962 ibt_set_cq_handler(state->id_scq_hdl, ibd_scq_handler, state);
5963 if ((ret = ibt_enable_cq_notify(state->id_scq_hdl,
5970 state->id_mac_state |= IBD_DRV_SCQ_NOTIFY_ENABLED;
5975 if (ibd_init_rxlist(state) != DDI_SUCCESS) {
5980 state->id_mac_state |= IBD_DRV_RXLIST_ALLOCD;
5985 if (ibd_join_group(state, state->id_mgid, IB_MC_JSTATE_FULL) == NULL) {
5990 state->id_mac_state |= IBD_DRV_BCAST_GROUP_JOINED;
5999 if (state->id_enable_rc) {
6000 ibd_h2n_mac(&state->id_macaddr,
6001 IBD_MAC_ADDR_RC + state->id_qpnum,
6002 state->id_sgid.gid_prefix, state->id_sgid.gid_guid);
6003 ibd_h2n_mac(&state->rc_macaddr_loopback, state->id_qpnum,
6004 state->id_sgid.gid_prefix, state->id_sgid.gid_guid);
6006 ibd_h2n_mac(&state->id_macaddr, state->id_qpnum,
6007 state->id_sgid.gid_prefix, state->id_sgid.gid_guid);
6009 ibd_h2n_mac(&state->id_bcaddr, IB_QPN_MASK,
6010 state->id_mgid.gid_prefix, state->id_mgid.gid_guid);
6012 if (!state->id_enable_rc) {
6013 (void) mac_maxsdu_update2(state->id_mh,
6014 state->id_mtu - IPOIB_HDRSIZE,
6015 state->id_mtu - IPOIB_HDRSIZE);
6017 mac_unicst_update(state->id_mh, (uint8_t *)&state->id_macaddr);
6022 ibt_set_cq_handler(state->id_rcq_hdl, ibd_rcq_handler, state);
6023 if ((ret = ibt_enable_cq_notify(state->id_rcq_hdl,
6030 state->id_mac_state |= IBD_DRV_RCQ_NOTIFY_ENABLED;
6052 if ((state->id_mac_state & IBD_DRV_SM_NOTICES_REGISTERED) == 0) {
6053 ibt_register_subnet_notices(state->id_ibt_hdl,
6054 ibd_snet_notices_handler, state);
6055 mutex_enter(&state->id_trap_lock);
6056 state->id_trap_stop = B_FALSE;
6057 mutex_exit(&state->id_trap_lock);
6058 state->id_mac_state |= IBD_DRV_SM_NOTICES_REGISTERED;
6063 state->id_mac_state |= IBD_DRV_IN_LATE_HCA_INIT;
6065 * In case of late initialization, mark the link state as down,
6066 * immaterial of the actual link state as reported in the
6069 state->id_link_state = LINK_STATE_DOWN;
6070 mac_unicst_update(state->id_mh, (uint8_t *)&state->id_macaddr);
6071 mac_link_update(state->id_mh, state->id_link_state);
6075 if (state->id_enable_rc) {
6076 if (state->rc_enable_srq) {
6077 if (state->id_mac_state & IBD_DRV_RC_SRQ_ALLOCD) {
6078 if (ibd_rc_repost_srq_free_list(state) !=
6085 if (ibd_rc_init_srq_list(state) !=
6090 state->id_mac_state |= IBD_DRV_RC_SRQ_ALLOCD;
6094 if (ibd_rc_init_tx_largebuf_list(state) != IBT_SUCCESS) {
6100 state->id_mac_state |= IBD_DRV_RC_LARGEBUF_ALLOCD;
6103 if (ibd_rc_listen(state) != IBT_SUCCESS) {
6108 state->id_mac_state |= IBD_DRV_RC_LISTEN;
6113 * we assume we are in up state (which must have been true at
6116 * async handler will have updated last known state, which we
6121 mac_link_update(state->id_mh, state->id_link_state);
6122 state->id_mac_state &= ~IBD_DRV_IN_LATE_HCA_INIT;
6123 state->id_mac_state |= IBD_DRV_STARTED;
6126 if (state->id_enable_rc) {
6127 mutex_enter(&state->rc_timeout_lock);
6128 state->rc_timeout_start = B_TRUE;
6129 state->rc_timeout = timeout(ibd_rc_conn_timeout_call, state,
6131 mutex_exit(&state->rc_timeout_lock);
6132 state->id_mac_state |= IBD_DRV_RC_TIMEOUT;
6144 (void) ibd_undo_start(state, LINK_STATE_DOWN);
6155 ibd_state_t *state = (ibd_state_t *)arg;
6157 if (state->id_type == IBD_PORT_DRIVER)
6160 ibd_set_mac_progress(state, IBD_DRV_STOP_IN_PROGRESS);
6162 (void) ibd_undo_start(state, state->id_link_state);
6164 ibd_clr_mac_progress(state, IBD_DRV_STOP_IN_PROGRESS);
6174 ibd_state_t *state = arg;
6176 if (state->id_type == IBD_PORT_DRIVER)
6183 if ((state->id_mac_state & IBD_DRV_STARTED) == 0)
6186 if (bcmp(macaddr, &state->id_macaddr, IPOIB_ADDRL) == 0)
6197 ibd_async_multicast(ibd_state_t *state, ib_gid_t mgid, int op)
6203 if (ibd_join_group(state, mgid, IB_MC_JSTATE_FULL) == NULL) {
6204 ibd_print_warn(state, "Join multicast group failed :"
6212 ibd_leave_group(state, mgid, IB_MC_JSTATE_FULL);
6224 ibd_state_t *state = (ibd_state_t *)arg;
6229 if (state->id_type == IBD_PORT_DRIVER)
6237 if ((state->id_mac_state & IBD_DRV_STARTED) == 0)
6263 IBD_FILL_SCOPE_PKEY(mcast, state->id_scope, state->id_pkey);
6273 if (bcmp(mcast, &state->id_bcaddr, IPOIB_ADDRL) == 0)
6277 req = kmem_cache_alloc(state->id_req_kmc, KM_NOSLEEP);
6286 ibd_queue_work_slot(state, req, IBD_ASYNC_JOIN);
6290 ibd_queue_work_slot(state, req, IBD_ASYNC_LEAVE);
6302 ibd_async_unsetprom(ibd_state_t *state)
6304 ibd_mce_t *mce = list_head(&state->id_mc_non);
6311 mce = list_next(&state->id_mc_non, mce);
6312 ibd_leave_group(state, mgid, IB_MC_JSTATE_NON);
6314 state->id_prom_op = IBD_OP_NOTSTARTED;
6324 ibd_async_setprom(ibd_state_t *state)
6340 mcg_attr.mc_pkey = state->id_pkey;
6341 mcg_attr.mc_scope = state->id_scope;
6342 mcg_attr.mc_qkey = state->id_mcinfo->mc_qkey;
6343 mcg_attr.mc_mtu_req.r_mtu = state->id_mcinfo->mc_mtu;
6345 if (ibt_query_mcg(state->id_sgid, &mcg_attr, 0, &mcg_info, &numg) !=
6347 ibd_print_warn(state, "Could not get list of IBA multicast "
6362 if (ibd_join_group(state, mgid, IB_MC_JSTATE_NON) == NULL)
6363 ibd_print_warn(state, "IBA promiscuous mode missed "
6372 state->id_prom_op = ret;
6377 * GLDv3 assumes phys state receives more packets than multi state,
6384 ibd_state_t *state = (ibd_state_t *)arg;
6387 if (state->id_type == IBD_PORT_DRIVER)
6394 if ((state->id_mac_state & IBD_DRV_STARTED) == 0)
6397 req = kmem_cache_alloc(state->id_req_kmc, KM_NOSLEEP);
6402 ibd_queue_work_slot(state, req, IBD_ASYNC_PROMON);
6405 ibd_queue_work_slot(state, req, IBD_ASYNC_PROMOFF);
6417 ibd_state_t *state = (ibd_state_t *)arg;
6421 *val = state->id_link_speed;
6424 *val = state->id_multi_rcv;
6427 *val = state->id_brd_rcv;
6430 *val = state->id_multi_xmt;
6433 *val = state->id_brd_xmt;
6436 *val = state->id_rcv_bytes + state->rc_rcv_trans_byte
6437 + state->rc_rcv_copy_byte;
6440 *val = state->id_rcv_pkt + state->rc_rcv_trans_pkt
6441 + state->rc_rcv_copy_pkt;
6444 *val = state->id_xmt_bytes + state->rc_xmt_bytes;
6447 *val = state->id_xmt_pkt + state->rc_xmt_small_pkt +
6448 state->rc_xmt_fragmented_pkt +
6449 state->rc_xmt_map_fail_pkt + state->rc_xmt_map_succ_pkt;
6452 *val = state->id_ah_error; /* failed AH translation */
6458 *val = state->id_tx_short + state->rc_swqe_short +
6459 state->rc_xmt_buf_short;
6470 ibd_async_txsched(ibd_state_t *state)
6472 ibd_resume_transmission(state);
6476 ibd_resume_transmission(ibd_state_t *state)
6483 mutex_enter(&state->id_sched_lock);
6484 if (state->id_sched_needed & IBD_RSRC_SWQE) {
6485 mutex_enter(&state->id_tx_list.dl_mutex);
6486 mutex_enter(&state->id_tx_rel_list.dl_mutex);
6487 met_thresh = state->id_tx_list.dl_cnt +
6488 state->id_tx_rel_list.dl_cnt;
6489 mutex_exit(&state->id_tx_rel_list.dl_mutex);
6490 mutex_exit(&state->id_tx_list.dl_mutex);
6493 } else if (state->id_sched_needed & IBD_RSRC_LSOBUF) {
6494 ASSERT(state->id_lso != NULL);
6495 mutex_enter(&state->id_lso_lock);
6496 met_thresh = state->id_lso->bkt_nfree;
6498 mutex_exit(&state->id_lso_lock);
6501 state->id_sched_lso_cnt++;
6504 state->id_sched_needed &= ~flag;
6505 state->id_sched_cnt++;
6508 mutex_exit(&state->id_sched_lock);
6511 mac_tx_update(state->id_mh);
6518 ibd_release_swqe(ibd_state_t *state, ibd_swqe_t *head, ibd_swqe_t *tail, int n)
6524 mutex_enter(&state->id_tx_rel_list.dl_mutex);
6525 state->id_tx_rel_list.dl_pending_sends = B_FALSE;
6526 tail->swqe_next = state->id_tx_rel_list.dl_head;
6527 state->id_tx_rel_list.dl_head = SWQE_TO_WQE(head);
6528 state->id_tx_rel_list.dl_cnt += n;
6529 mutex_exit(&state->id_tx_rel_list.dl_mutex);
6537 ibd_acquire_swqe(ibd_state_t *state)
6541 mutex_enter(&state->id_tx_rel_list.dl_mutex);
6542 if (state->id_tx_rel_list.dl_head != NULL) {
6544 state->id_tx_list.dl_head =
6545 state->id_tx_rel_list.dl_head;
6546 state->id_tx_list.dl_cnt =
6547 state->id_tx_rel_list.dl_cnt;
6548 state->id_tx_list.dl_pending_sends = B_FALSE;
6551 state->id_tx_rel_list.dl_head = NULL;
6552 state->id_tx_rel_list.dl_cnt = 0;
6553 mutex_exit(&state->id_tx_rel_list.dl_mutex);
6555 wqe = WQE_TO_SWQE(state->id_tx_list.dl_head);
6556 state->id_tx_list.dl_cnt -= 1;
6557 state->id_tx_list.dl_head = wqe->swqe_next;
6559 mutex_exit(&state->id_tx_rel_list.dl_mutex);
6560 state->id_tx_list.dl_pending_sends = B_TRUE;
6562 state->id_tx_short++;
6679 ibd_post_send(ibd_state_t *state, ibd_swqe_t *node)
6690 ibt_status = ibt_post_send(state->id_chnl_hdl,
6693 ibd_print_warn(state, "ibd_post_send: "
6695 ibd_tx_cleanup(state, node);
6701 mutex_enter(&state->id_txpost_lock);
6702 tx_head = state->id_tx_head;
6704 state->id_tx_busy = 0;
6705 mutex_exit(&state->id_txpost_lock);
6708 state->id_tx_head = NULL;
6709 mutex_exit(&state->id_txpost_lock);
6736 ibt_status = ibt_post_send(state->id_chnl_hdl,
6739 ibd_print_warn(state, "ibd_post_send: "
6745 ibd_tx_cleanup(state, nodes[i]);
6751 ibd_prepare_sgl(ibd_state_t *state, mblk_t *mp, ibd_swqe_t *node,
6800 if ((state->id_hca_res_lkey_capab) &&
6801 (pktsize > state->id_ud_tx_copy_thresh) &&
6802 (nmblks < state->id_max_sqseg_hiwm)) {
6810 iov_attr.iov_wr_nds = state->id_max_sqseg;
6826 ibt_status = ibt_map_mem_iov(state->id_hca_hdl, &iov_attr,
6829 ibd_print_warn(state, "ibd_send: ibt_map_mem_iov "
6838 if (pktsize <= state->id_tx_buf_sz) {
6865 if (ibd_acquire_lsobufs(state, pktsize,
6909 ibd_sched_poll(ibd_state_t *state, int resource_type, int q_flag)
6913 mutex_enter(&state->id_sched_lock);
6914 state->id_sched_needed |= resource_type;
6915 mutex_exit(&state->id_sched_lock);
6921 req = kmem_cache_alloc(state->id_req_kmc, KM_NOSLEEP);
6925 ibd_queue_work_slot(state, req, IBD_ASYNC_SCHED);
6936 ibd_send(ibd_state_t *state, mblk_t *mp)
6960 if ((state->id_mac_state & IBD_DRV_STARTED) == 0)
6969 IBD_FILL_SCOPE_PKEY(dest, state->id_scope, state->id_pkey);
6972 ace = ibd_acache_lookup(state, dest, &ret, 1);
6973 if (state->id_enable_rc && (ace != NULL) &&
6976 state->rc_null_conn++;
6995 state->rc_swqe_short++;
6996 mutex_enter(&state->id_sched_lock);
6997 state->id_sched_needed |=
6999 mutex_exit(&state->id_sched_lock);
7000 ibd_dec_ref_ace(state, ace);
7004 state->rc_no_estab_conn++;
7010 mutex_enter(&state->id_tx_list.dl_mutex);
7011 node = WQE_TO_SWQE(state->id_tx_list.dl_head);
7013 state->id_tx_list.dl_cnt -= 1;
7014 state->id_tx_list.dl_head = node->swqe_next;
7016 node = ibd_acquire_swqe(state);
7018 mutex_exit(&state->id_tx_list.dl_mutex);
7025 if (ibd_sched_poll(state, IBD_RSRC_SWQE, 0) == 0) {
7027 ibd_dec_ref_ace(state, ace);
7036 ibd_print_warn(state, "ibd_send: no swqe, pkt drop");
7038 ibd_dec_ref_ace(state, ace);
7063 if (bcmp(&ipibp->ib_dst, &state->id_bcaddr, IPOIB_ADDRL) == 0)
7064 atomic_inc_64(&state->id_brd_xmt);
7066 atomic_inc_64(&state->id_multi_xmt);
7079 state->rc_ace_not_found++;
7097 } else if (ibd_sched_poll(state, IBD_RSRC_SWQE, 1) != 0) {
7162 atomic_add_64(&state->rc_xmt_bytes, pktsize);
7175 if (pktsize <= state->id_rc_tx_copy_thresh) {
7176 atomic_inc_64(&state->rc_xmt_small_pkt);
7195 if ((state->rc_enable_iov_map) &&
7196 (nmblks < state->rc_max_sqseg_hiwm)) {
7202 iov_attr.iov_wr_nds = state->rc_tx_max_sqseg;
7218 ret = ibt_map_mem_iov(state->id_hca_hdl,
7223 &state->rc_xmt_map_fail_pkt);
7230 atomic_inc_64(&state->rc_xmt_map_succ_pkt);
7234 atomic_inc_64(&state->rc_xmt_fragmented_pkt);
7236 mutex_enter(&state->rc_tx_large_bufs_lock);
7237 if (state->rc_tx_largebuf_nfree == 0) {
7238 state->rc_xmt_buf_short++;
7240 (&state->rc_tx_large_bufs_lock);
7241 mutex_enter(&state->id_sched_lock);
7242 state->id_sched_needed |=
7244 mutex_exit(&state->id_sched_lock);
7258 lbufp = state->rc_tx_largebuf_free_head;
7260 state->rc_tx_largebuf_free_head =
7264 state->rc_tx_largebuf_nfree --;
7265 mutex_exit(&state->rc_tx_large_bufs_lock);
7270 state->rc_tx_mr_desc.md_lkey;
7310 if ((state->id_enable_rc) && (pktsize > state->id_mtu)) {
7313 * state->id_mtu + sizeof (ib_addrs_t)
7321 state->rc_xmt_reenter_too_long_pkt++;
7326 state->rc_xmt_icmp_too_long_pkt++;
7328 req = kmem_cache_alloc(state->id_req_kmc,
7331 ibd_print_warn(state, "ibd_send: alloc "
7338 ibd_queue_work_slot(state, req,
7344 ibd_print_warn(state, "Reliable Connected mode is on. "
7347 pktsize, state->id_mtu);
7348 state->rc_xmt_drop_too_long_pkt++;
7356 atomic_add_64(&state->id_xmt_bytes, pktsize);
7357 atomic_inc_64(&state->id_xmt_pkt);
7374 ibd_print_warn(state,
7396 if (ibd_prepare_sgl(state, mp, node, lsohdr_sz) != 0) {
7397 if (ibd_sched_poll(state, IBD_RSRC_LSOBUF, 1) != 0) {
7415 mutex_enter(&state->id_txpost_lock);
7416 if (state->id_tx_busy) {
7417 if (state->id_tx_head) {
7418 state->id_tx_tail->swqe_next =
7421 state->id_tx_head = node;
7423 state->id_tx_tail = node;
7424 mutex_exit(&state->id_txpost_lock);
7426 state->id_tx_busy = 1;
7427 mutex_exit(&state->id_txpost_lock);
7428 ibd_post_send(state, node);
7444 ibd_tx_cleanup(state, node);
7457 ibd_state_t *state = (ibd_state_t *)arg;
7460 if (state->id_type == IBD_PORT_DRIVER) {
7465 if ((state->id_link_state != LINK_STATE_UP) ||
7466 !(state->id_mac_state & IBD_DRV_STARTED)) {
7474 if (ibd_send(state, mp) == B_FALSE) {
7492 ibd_state_t *state = (ibd_state_t *)arg;
7494 ibd_poll_rcq(state, state->id_rcq_hdl);
7503 ibd_drain_scq(ibd_state_t *state, ibt_cq_hdl_t cq_hdl)
7505 ibt_wc_t *wcs = state->id_txwcs;
7506 uint_t numwcs = state->id_txwcs_size;
7545 ibd_tx_cleanup_list(state, head, tail);
7550 ibd_resume_transmission(state);
7558 ibd_drain_rcq(ibd_state_t *state, ibt_cq_hdl_t cq_hdl)
7560 ibt_wc_t *wcs = state->id_rxwcs;
7561 uint_t numwcs = state->id_rxwcs_size;
7585 &state->id_rx_list.dl_bufs_outstanding);
7589 mp = ibd_process_rx(state, rwqe, wc);
7603 mac_rx(state->id_mh, state->id_rh, head);
7609 if (atomic_add_32_nv(&state->id_rx_list.dl_cnt, -num_polled) <
7610 (state->id_ud_num_rwqe / 4))
7611 ibd_post_recv_intr(state);
7620 ibd_poll_scq(ibd_state_t *state, ibt_cq_hdl_t cq_hdl)
7628 mutex_enter(&state->id_scq_poll_lock);
7629 if (state->id_scq_poll_busy & flag) {
7630 ibd_print_warn(state, "ibd_poll_scq: multiple polling threads");
7631 state->id_scq_poll_busy |= redo_flag;
7632 mutex_exit(&state->id_scq_poll_lock);
7635 state->id_scq_poll_busy |= flag;
7636 mutex_exit(&state->id_scq_poll_lock);
7650 ibd_drain_scq(state, cq_hdl);
7665 ibd_drain_scq(state, cq_hdl);
7667 mutex_enter(&state->id_scq_poll_lock);
7668 if (state->id_scq_poll_busy & redo_flag)
7669 state->id_scq_poll_busy &= ~redo_flag;
7671 state->id_scq_poll_busy &= ~flag;
7674 mutex_exit(&state->id_scq_poll_lock);
7684 ibd_poll_rcq(ibd_state_t *state, ibt_cq_hdl_t rcq)
7692 mutex_enter(&state->id_rcq_poll_lock);
7693 if (state->id_rcq_poll_busy & flag) {
7694 ibd_print_warn(state, "ibd_poll_rcq: multiple polling threads");
7695 state->id_rcq_poll_busy |= redo_flag;
7696 mutex_exit(&state->id_rcq_poll_lock);
7699 state->id_rcq_poll_busy |= flag;
7700 mutex_exit(&state->id_rcq_poll_lock);
7705 ibd_drain_rcq(state, rcq);
7720 ibd_drain_rcq(state, rcq);
7722 mutex_enter(&state->id_rcq_poll_lock);
7723 if (state->id_rcq_poll_busy & redo_flag)
7724 state->id_rcq_poll_busy &= ~redo_flag;
7726 state->id_rcq_poll_busy &= ~flag;
7729 mutex_exit(&state->id_rcq_poll_lock);
7738 ibd_unmap_mem(ibd_state_t *state, ibd_swqe_t *swqe)
7745 if ((stat = ibt_unmap_mem_iov(state->id_hca_hdl,
7756 ibd_dec_ref_ace(ibd_state_t *state, ibd_ace_t *ace)
7787 mutex_enter(&state->id_ac_mutex);
7800 IBD_ACACHE_PULLOUT_ACTIVE(state, ace);
7805 ibd_queue_work_slot(state,
7808 IBD_ACACHE_INSERT_FREE(state, ace);
7810 mutex_exit(&state->id_ac_mutex);
7819 ibd_tx_cleanup(ibd_state_t *state, ibd_swqe_t *swqe)
7834 ibd_unmap_mem(state, swqe);
7836 ibd_release_lsobufs(state,
7853 ibd_dec_ref_ace(state, ace);
7860 ibd_release_swqe(state, swqe, swqe, 1);
7864 ibd_tx_cleanup_list(ibd_state_t *state, ibd_swqe_t *head, ibd_swqe_t *tail)
7884 ibd_unmap_mem(state, swqe);
7886 ibd_release_lsobufs(state,
7904 ibd_dec_ref_ace(state, ace);
7912 ibd_release_swqe(state, head, tail, n);
7921 ibd_process_rx(ibd_state_t *state, ibd_rwqe_t *rwqe, ibt_wc_t *wc)
7935 bufs = atomic_inc_32_nv(&state->id_rx_list.dl_bufs_outstanding);
7938 if (bufs >= state->id_rx_bufs_outstanding_limit) {
7939 atomic_dec_32(&state->id_rx_list.dl_bufs_outstanding);
7940 atomic_inc_32(&state->id_rx_allocb);
7944 ibd_post_recv(state, rwqe);
7946 atomic_inc_32(&state->id_rx_allocb_failed);
7947 ibd_post_recv(state, rwqe);
7964 ibd_print_warn(state,
7983 if (state->id_enable_rc) {
7985 &state->rc_macaddr_loopback,
7991 if (bcmp(&phdr->ib_grh.ipoib_sqpn, &state->id_macaddr,
8004 phdr->ib_dst.ipoib_qpn = state->id_macaddr.ipoib_qpn;
8012 ovbcopy(&state->id_macaddr, &phdr->ib_dst,
8034 atomic_add_64(&state->id_rcv_bytes, pkt_len);
8035 atomic_inc_64(&state->id_rcv_pkt);
8036 if (bcmp(&phdr->ib_dst, &state->id_bcaddr, IPOIB_ADDRL) == 0)
8037 atomic_inc_64(&state->id_brd_rcv);
8039 atomic_inc_64(&state->id_multi_rcv);
8068 ibd_state_t *state = rwqe->w_state;
8070 atomic_dec_32(&state->id_rx_list.dl_bufs_outstanding);
8075 if (atomic_add_32_nv(&state->id_running, 0) == 0) {
8078 ibd_free_rwqe(state, rwqe);
8083 state->id_mtu + IPOIB_GRH_SIZE, 0, &rwqe->w_freemsg_cb);
8085 ibd_free_rwqe(state, rwqe);
8090 ibd_post_recv(state, rwqe);
8096 ibd_state_t *state = (ibd_state_t *)arg;
8101 ibd_poll_scq(state, state->id_scq_hdl);
8169 ibd_state_t *state, *port_state, *p;
8185 DPRINT(10, "ibd_create_partition: failed to get state %d",
8243 state = kmem_zalloc(sizeof (ibd_state_t), KM_SLEEP);
8245 state->id_type = IBD_PARTITION_OBJ;
8247 state->id_plinkid = cmd->ioc_partid;
8248 state->id_dlinkid = cmd->ibdioc.ioc_linkid;
8249 state->id_port_inst = cmd->ibdioc.ioc_port_inst;
8251 state->id_dip = port_state->id_dip;
8252 state->id_port = port_state->id_port;
8253 state->id_pkey = cmd->ioc_pkey;
8254 state->id_hca_guid = port_state->id_hca_guid;
8255 state->id_port_guid = port_state->id_port_guid;
8256 state->id_force_create = force_create;
8258 mutex_init(&state->id_macst_lock, NULL, MUTEX_DRIVER, NULL);
8259 cv_init(&state->id_macst_cv, NULL, CV_DEFAULT, NULL);
8261 if (ibd_part_attach(state, state->id_dip) != DDI_SUCCESS) {
8275 macp->m_driver = state;
8276 macp->m_src_addr = (uint8_t *)&state->id_macaddr;
8280 if (state->id_enable_rc) {
8287 err = mac_register(macp, &state->id_mh);
8297 err = dls_devnet_create(state->id_mh,
8303 (void) mac_unregister(state->id_mh);
8308 * Add the new partition state structure to the list
8312 state->id_next = ibd_objlist_head;
8314 ibd_objlist_head = state;
8327 ibd_part_unattach(state);
8328 kmem_free(state, sizeof (ibd_state_t));
8347 /* Find the ibd state structure corresponding to the partition */
8391 /* Remove the partition state structure from the linked list */
8422 ibd_state_t *state, *port_state;
8442 /* Find the ibd state structure corresponding the partition */
8443 for (state = ibd_objlist_head; state; state = state->id_next) {
8444 if (state->id_plinkid == cmd.ioc_linkid) {
8449 if (state == NULL) {
8454 partioc.ibdioc.ioc_linkid = state->id_dlinkid;
8455 partioc.ibdioc.ioc_port_inst = state->id_port_inst;
8456 partioc.ibdioc.ioc_portnum = state->id_port;
8457 partioc.ibdioc.ioc_hcaguid = state->id_hca_guid;
8458 partioc.ibdioc.ioc_portguid = state->id_port_guid;
8460 partioc.ioc_partid = state->id_plinkid;
8461 partioc.ioc_pkey = state->id_pkey;
8462 partioc.ioc_force_create = state->id_force_create;
8475 " state %d", cmd.ioc_port_inst);
8608 " state %d", cmd.ioc_port_inst);
8697 ibd_state_t *state = (ibd_state_t *)arg;
8703 if (ibd_get_port_state(state, &lstate) != 0)
8706 if (state->id_link_state != lstate) {
8707 state->id_link_state = lstate;
8708 mac_link_update(state->id_mh, lstate);
8717 ibd_get_port_state(ibd_state_t *state, link_state_t *lstate)
8723 ret = ibt_query_hca_ports(state->id_hca_hdl, state->id_port,
8728 state->id_sgid = *port_infop->p_sgid_tbl;
8729 state->id_link_speed = ibd_get_portspeed(state);
8743 ibd_state_t *state;
8757 state = ddi_get_soft_state(ibd_list, instance);
8759 state->id_dip = dip;
8760 state->id_type = IBD_PORT_DRIVER;
8762 if ((state->id_port = ddi_prop_get_int(DDI_DEV_T_ANY, dip, 0,
8765 state->id_port);
8768 if ((state->id_hca_guid = ddi_prop_get_int64(DDI_DEV_T_ANY, dip, 0,
8771 state->id_hca_guid);
8774 if ((state->id_port_guid = ddi_prop_get_int64(DDI_DEV_T_ANY, dip, 0,
8777 state->id_port_guid);
8784 if ((ret = ibt_attach(&ibdpd_clnt_modinfo, dip, state,
8785 &state->id_ibt_hdl)) != IBT_SUCCESS) {
8791 state->id_mac_state |= IBD_DRV_IBTL_ATTACH_DONE;
8793 if ((ret = ibt_open_hca(state->id_ibt_hdl, state->id_hca_guid,
8794 &state->id_hca_hdl)) != IBT_SUCCESS) {
8799 state->id_mac_state |= IBD_DRV_HCA_OPENED;
8803 if (ibd_get_port_state(state, &lstate) != 0) {
8808 state->id_link_state = lstate;
8812 if (ibd_register_mac(state, dip) != IBT_SUCCESS) {
8816 state->id_mac_state |= IBD_DRV_MAC_REGISTERED;
8818 mac_link_update(state->id_mh, lstate);
8822 (void) ibd_port_unattach(state, dip);
8827 ibd_port_unattach(ibd_state_t *state, dev_info_t *dip)
8830 uint32_t progress = state->id_mac_state;
8834 (void) mac_unregister(state->id_mh);
8835 state->id_mac_state &= (~IBD_DRV_MAC_REGISTERED);
8839 if ((ret = ibt_close_hca(state->id_hca_hdl)) !=
8841 ibd_print_warn(state, "failed to close "
8844 state->id_hca_hdl = NULL;
8845 state->id_mac_state &= (~IBD_DRV_HCA_OPENED);
8849 if ((ret = ibt_detach(state->id_ibt_hdl)) != IBT_SUCCESS) {
8850 ibd_print_warn(state,
8853 state->id_ibt_hdl = NULL;
8854 state->id_mac_state &= (~IBD_DRV_IBTL_ATTACH_DONE);
8865 ibd_state_t *state;
8869 /* Find the ibd state structure corresponding the partition */
8870 for (state = ibd_objlist_head; state; state = state->id_next) {
8871 if (state->id_plinkid == linkid) {
8876 if (state == NULL) {
8881 attr->pa_dlinkid = state->id_dlinkid;
8882 attr->pa_plinkid = state->id_plinkid;
8883 attr->pa_port = state->id_port;
8884 attr->pa_hca_guid = state->id_hca_guid;
8885 attr->pa_port_guid = state->id_port_guid;
8886 attr->pa_pkey = state->id_pkey;
8896 ibd_state_t *state;
8902 for (state = ibd_objlist_head; state; state = state->id_next)
8914 for (state = ibd_objlist_head; state; state = state->id_next) {
8919 attr->pa_dlinkid = state->id_dlinkid;
8920 attr->pa_plinkid = state->id_plinkid;
8921 attr->pa_port = state->id_port;
8922 attr->pa_hca_guid = state->id_hca_guid;
8923 attr->pa_port_guid = state->id_port_guid;
8924 attr->pa_pkey = state->id_pkey;