ibcm_impl.c revision 43ed929a6988c3778f00123f4a4a8541e515ec16
fb3fb4f3d76d55b64440afd0af72775dfad3bd1dtomee * CDDL HEADER START
fb3fb4f3d76d55b64440afd0af72775dfad3bd1dtomee * The contents of this file are subject to the terms of the
fb3fb4f3d76d55b64440afd0af72775dfad3bd1dtomee * Common Development and Distribution License, Version 1.0 only
fb3fb4f3d76d55b64440afd0af72775dfad3bd1dtomee * (the "License"). You may not use this file except in compliance
fb3fb4f3d76d55b64440afd0af72775dfad3bd1dtomee * with the License.
fb3fb4f3d76d55b64440afd0af72775dfad3bd1dtomee * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
fb3fb4f3d76d55b64440afd0af72775dfad3bd1dtomee * See the License for the specific language governing permissions
fb3fb4f3d76d55b64440afd0af72775dfad3bd1dtomee * and limitations under the License.
fb3fb4f3d76d55b64440afd0af72775dfad3bd1dtomee * When distributing Covered Code, include this CDDL HEADER in each
fb3fb4f3d76d55b64440afd0af72775dfad3bd1dtomee * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
fb3fb4f3d76d55b64440afd0af72775dfad3bd1dtomee * If applicable, add the following below this CDDL HEADER, with the
fb3fb4f3d76d55b64440afd0af72775dfad3bd1dtomee * fields enclosed by brackets "[]" replaced with your own identifying
fb3fb4f3d76d55b64440afd0af72775dfad3bd1dtomee * information: Portions Copyright [yyyy] [name of copyright owner]
fb3fb4f3d76d55b64440afd0af72775dfad3bd1dtomee * CDDL HEADER END
91cfa10a8e55050a5103c4b2e83b0bf8d783a7cbtomee * Copyright 2005 Sun Microsystems, Inc. All rights reserved.
fb3fb4f3d76d55b64440afd0af72775dfad3bd1dtomee * Use is subject to license terms.
fb3fb4f3d76d55b64440afd0af72775dfad3bd1dtomee#pragma ident "%Z%%M% %I% %E% SMI"
fb3fb4f3d76d55b64440afd0af72775dfad3bd1dtomee * contains internal functions of IB CM module.
fb3fb4f3d76d55b64440afd0af72775dfad3bd1dtomee * 1. HCA CATASTROPHIC/RECOVERED not handled yet
fb3fb4f3d76d55b64440afd0af72775dfad3bd1dtomee/* function prototypes */
fb3fb4f3d76d55b64440afd0af72775dfad3bd1dtomee/* Routines to initialize and destory CM global locks and CVs */
fb3fb4f3d76d55b64440afd0af72775dfad3bd1dtomeestatic void ibcm_init_locks(void);
fb3fb4f3d76d55b64440afd0af72775dfad3bd1dtomeestatic void ibcm_fini_locks(void);
fb3fb4f3d76d55b64440afd0af72775dfad3bd1dtomee/* Routines that initialize/teardown CM's global hca structures */
fb3fb4f3d76d55b64440afd0af72775dfad3bd1dtomeestatic void ibcm_init_hcas();
fb3fb4f3d76d55b64440afd0af72775dfad3bd1dtomee/* Routines that handle HCA attach/detach asyncs */
fb3fb4f3d76d55b64440afd0af72775dfad3bd1dtomeestatic ibcm_status_t ibcm_hca_detach(ibcm_hca_info_t *);
fb3fb4f3d76d55b64440afd0af72775dfad3bd1dtomee/* Routines that initialize the HCA's port related fields */
fb3fb4f3d76d55b64440afd0af72775dfad3bd1dtomeestatic ibt_status_t ibcm_hca_init_port(ibcm_hca_info_t *hcap,
fb3fb4f3d76d55b64440afd0af72775dfad3bd1dtomeestatic ibcm_status_t ibcm_hca_fini_port(ibcm_hca_info_t *hcap,
fb3fb4f3d76d55b64440afd0af72775dfad3bd1dtomeestatic void ibcm_rc_flow_control_init(void);
fb3fb4f3d76d55b64440afd0af72775dfad3bd1dtomeestatic void ibcm_rc_flow_control_fini(void);
fb3fb4f3d76d55b64440afd0af72775dfad3bd1dtomee * Routines that check if hca's avl trees and sidr lists are free of any
fb3fb4f3d76d55b64440afd0af72775dfad3bd1dtomee * active client resources ie., RC or UD state structures in certain states
fb3fb4f3d76d55b64440afd0af72775dfad3bd1dtomeestatic ibcm_status_t ibcm_check_avl_clean(ibcm_hca_info_t *hcap);
fb3fb4f3d76d55b64440afd0af72775dfad3bd1dtomeestatic ibcm_status_t ibcm_check_sidr_clean(ibcm_hca_info_t *hcap);
fb3fb4f3d76d55b64440afd0af72775dfad3bd1dtomee/* Add a new hca structure to CM's global hca list */
fb3fb4f3d76d55b64440afd0af72775dfad3bd1dtomeestatic ibcm_hca_info_t *ibcm_add_hca_entry(ib_guid_t hcaguid, uint_t nports);
fb3fb4f3d76d55b64440afd0af72775dfad3bd1dtomeestatic void ibcm_comm_est_handler(ibt_async_event_t *);
fb3fb4f3d76d55b64440afd0af72775dfad3bd1dtomee/* Global variables */
fb3fb4f3d76d55b64440afd0af72775dfad3bd1dtomeekmutex_t ibcm_trace_print_mutex; /* Trace print mutex */
fb3fb4f3d76d55b64440afd0af72775dfad3bd1dtomeeint ibcm_enable_trace = 4; /* Trace level 4 by default */
fb3fb4f3d76d55b64440afd0af72775dfad3bd1dtomee_NOTE(MUTEX_PROTECTS_DATA(ibcm_svc_info_lock, ibcm_svc_info_s::{svc_bind_list
fb3fb4f3d76d55b64440afd0af72775dfad3bd1dtomee_NOTE(MUTEX_PROTECTS_DATA(ibcm_svc_info_lock, ibcm_svc_bind_s::{sbind_link}))
fb3fb4f3d76d55b64440afd0af72775dfad3bd1dtomee_NOTE(MUTEX_PROTECTS_DATA(ibcm_trace_mutex, ibcm_conn_trace_s))
fb3fb4f3d76d55b64440afd0af72775dfad3bd1dtomee_NOTE(MUTEX_PROTECTS_DATA(ibcm_trace_print_mutex, ibcm_debug_buf))
fb3fb4f3d76d55b64440afd0af72775dfad3bd1dtomee * Initial state is INIT. All hca dr's return success immediately in this
fb3fb4f3d76d55b64440afd0af72775dfad3bd1dtomee * state, without adding or deleting any hca's to CM.
fb3fb4f3d76d55b64440afd0af72775dfad3bd1dtomee/* mutex and cv to manage hca's reference and resource count(s) */
fb3fb4f3d76d55b64440afd0af72775dfad3bd1dtomee/* mutex and cv to sa session open */
fb3fb4f3d76d55b64440afd0af72775dfad3bd1dtomee ibcm_port_info_s::{port_ibmf_saa_hdl port_saa_open_in_progress}))
fb3fb4f3d76d55b64440afd0af72775dfad3bd1dtomee_NOTE(DATA_READABLE_WITHOUT_LOCK(ibcm_port_info_s::{port_ibmf_saa_hdl}))
fb3fb4f3d76d55b64440afd0af72775dfad3bd1dtomee/* serialize sm notice callbacks */
fb3fb4f3d76d55b64440afd0af72775dfad3bd1dtomee_NOTE(LOCK_ORDER(ibcm_sm_notice_serialize_lock ibcm_global_hca_lock))
fb3fb4f3d76d55b64440afd0af72775dfad3bd1dtomee_NOTE(MUTEX_PROTECTS_DATA(ibcm_global_hca_lock, ibcm_hca_info_s::{hca_state
fb3fb4f3d76d55b64440afd0af72775dfad3bd1dtomee_NOTE(MUTEX_PROTECTS_DATA(ibcm_sm_notice_serialize_lock,
fb3fb4f3d76d55b64440afd0af72775dfad3bd1dtomee_NOTE(DATA_READABLE_WITHOUT_LOCK(ibcm_hca_info_s::{hca_state}))
fb3fb4f3d76d55b64440afd0af72775dfad3bd1dtomee/* mutex for CM's qp list management */
fb3fb4f3d76d55b64440afd0af72775dfad3bd1dtomee_NOTE(MUTEX_PROTECTS_DATA(ibcm_qp_list_lock, ibcm_port_info_s::{port_qplist}))
fb3fb4f3d76d55b64440afd0af72775dfad3bd1dtomee_NOTE(MUTEX_PROTECTS_DATA(ibcm_qp_list_lock, ibcm_qp_list_s))
fb3fb4f3d76d55b64440afd0af72775dfad3bd1dtomee_NOTE(MUTEX_PROTECTS_DATA(ibcm_qp_list_lock, ibcm_qp_list_s))
fb3fb4f3d76d55b64440afd0af72775dfad3bd1dtomeeibcm_state_data_t *ibcm_timeout_list_hdr, *ibcm_timeout_list_tail;
fb3fb4f3d76d55b64440afd0af72775dfad3bd1dtomeeibcm_ud_state_data_t *ibcm_ud_timeout_list_hdr, *ibcm_ud_timeout_list_tail;
fb3fb4f3d76d55b64440afd0af72775dfad3bd1dtomeestatic ibt_clnt_modinfo_t ibcm_ibt_modinfo = { /* Client's modinfop */
fb3fb4f3d76d55b64440afd0af72775dfad3bd1dtomee/* IBCM's list of HCAs registered with it */
fb3fb4f3d76d55b64440afd0af72775dfad3bd1dtomeestatic ibcm_hca_info_t *ibcm_hca_listp = NULL; /* CM's HCA list */
fb3fb4f3d76d55b64440afd0af72775dfad3bd1dtomee/* Array of CM state call table functions */
fb3fb4f3d76d55b64440afd0af72775dfad3bd1dtomee/* the following globals are CM tunables */
fb3fb4f3d76d55b64440afd0af72775dfad3bd1dtomeeibt_rnr_nak_time_t ibcm_default_rnr_nak_time = IBT_RNR_NAK_655ms;
fb3fb4f3d76d55b64440afd0af72775dfad3bd1dtomeeclock_t ibcm_local_processing_time = IBCM_LOCAL_RESPONSE_TIME;
fb3fb4f3d76d55b64440afd0af72775dfad3bd1dtomeeclock_t ibcm_remote_response_time = IBCM_REMOTE_RESPONSE_TIME;
fb3fb4f3d76d55b64440afd0af72775dfad3bd1dtomeeib_time_t ibcm_max_sidr_rep_proctime = IBCM_MAX_SIDR_PROCESS_TIME;
fb3fb4f3d76d55b64440afd0af72775dfad3bd1dtomeeib_time_t ibcm_max_sidr_pktlife_time = IBCM_MAX_SIDR_PKT_LIFE_TIME;
fb3fb4f3d76d55b64440afd0af72775dfad3bd1dtomeeuint32_t ibcm_wait_for_acc_cnt_timeout = 500000; /* 500 ms */
fb3fb4f3d76d55b64440afd0af72775dfad3bd1dtomeeuint32_t ibcm_wait_for_res_cnt_timeout = 500000; /* 500 ms */
fb3fb4f3d76d55b64440afd0af72775dfad3bd1dtomeeib_time_t ibcm_max_ib_mad_pkt_lt = IBCM_MAX_IB_MAD_PKT_LT;
fb3fb4f3d76d55b64440afd0af72775dfad3bd1dtomee * This delay accounts for time involved in various activities as follows :
fb3fb4f3d76d55b64440afd0af72775dfad3bd1dtomee * IBMF delays for posting the MADs in non-blocking mode
fb3fb4f3d76d55b64440afd0af72775dfad3bd1dtomee * IBMF delays for receiving the MADs and delivering to CM
fb3fb4f3d76d55b64440afd0af72775dfad3bd1dtomee * CM delays in processing the MADs before invoking client handlers,
fb3fb4f3d76d55b64440afd0af72775dfad3bd1dtomee * Any other delays associated with HCA driver in processing the MADs and
fb3fb4f3d76d55b64440afd0af72775dfad3bd1dtomee * other subsystems that CM may invoke (ex : SA, HCA driver)
char *event_str[] = {
"LOCAL_COMID/HCA/PORT ",
"REMOTE_COMID/HCA ",
#ifdef DEBUG
_init(void)
int rval;
return (EINVAL);
if (rval != 0) {
rval);
(void) ibcm_fini();
return (rval);
_fini(void)
int status;
return (EBUSY);
status);
return (status);
return (status);
#ifdef DEBUG
sizeof (ibcm_req_msg_t));
sizeof (ibcm_rep_msg_t));
sizeof (ibcm_rtu_msg_t));
sizeof (ibcm_mra_msg_t));
sizeof (ibcm_rej_msg_t));
sizeof (ibcm_lap_msg_t));
sizeof (ibcm_apr_msg_t));
sizeof (ibcm_dreq_msg_t));
sizeof (ibcm_drep_msg_t));
sizeof (ibcm_sidr_req_msg_t));
sizeof (ibcm_sidr_rep_msg_t));
sizeof (ibcm_svc_info_t),
static ibcm_status_t
ibcm_init(void)
kthread_t *t;
return (IBCM_FAILURE);
return (IBCM_FAILURE);
status);
(void) ibcm_ar_fini();
return (IBCM_FAILURE);
return (IBCM_SUCCESS);
for (i = 0; i < num_hcas; i++)
if (num_hcas)
static ibcm_status_t
ibcm_fini(void)
return (IBCM_FAILURE);
return (IBCM_FAILURE);
return (IBCM_FAILURE);
return (IBCM_SUCCESS);
static ibcm_status_t
return (IBCM_FAILURE);
return (IBCM_SUCCESS);
sizeof (ibcm_state_data_t),
sizeof (ibcm_state_data_t),
sizeof (ibcm_state_data_t),
static ibcm_status_t
int port_index, i;
#ifdef DEBUG
if (ibcm_test_mode > 0)
return (IBCM_FAILURE);
return (IBCM_FAILURE);
return (IBCM_FAILURE);
return (IBCM_FAILURE);
#ifdef DEBUG
if (ibcm_test_mode > 0)
return (IBCM_FAILURE);
IBCM_SUCCESS) {
for (i = 0; i < port_index; i++) {
return (IBCM_FAILURE);
return (IBCM_SUCCESS);
static ibcm_status_t
return (IBCM_FAILURE);
return (IBCM_SUCCESS);
static ibcm_status_t
return (IBCM_FAILURE);
return (IBCM_SUCCESS);
static ibcm_hca_info_t *
hcaguid);
return (NULL);
return (hcap);
if (prevp) {
sizeof (ibcm_port_info_t));
return (NULL);
return (hcap);
return (NULL);
return (hcap);
return (IBCM_SUCCESS);
return (IBCM_FAILURE);
typedef struct ibcm_flow1_s {
} ibcm_flow1_t;
typedef struct ibcm_flow_s {
} ibcm_flow_t;
static ibcm_flow1_t *
ibcm_flow1_t *f;
f = f->link;
f = f->link;
return (flow1);
void *callback_arg)
switch (saa_event_code) {
int num_failed_sgids;
ibt_hdl);
num_failed_sgids = 0;
portp++;
if (num_failed_sgids != 0) {
portp++;
if (num_failed_sgids != 0) {
int status;
return (NULL);
return (saa_handle);
return (NULL);
return (saa_handle);
int status;
return (IBT_SUCCESS);
return (status);
static ibcm_status_t
int ibmf_status;
return (IBCM_FAILURE);
return (IBCM_FAILURE);
return (IBCM_FAILURE);
NULL;
return (IBCM_FAILURE);
return (IBCM_SUCCESS);
if (timer_val) {
code);
switch (code) {
case IBT_EVENT_PORT_UP:
case IBT_HCA_ATTACH_EVENT:
case IBT_HCA_DETACH_EVENT:
NULL) {
case IBT_EVENT_COM_EST_QP:
case IBT_EVENT_COM_EST_EEC: