ibtl.h revision 9c468ea9d266203f8dac0165f60fc9b92d8aead3
/*
* CDDL HEADER START
*
* The contents of this file are subject to the terms of the
* Common Development and Distribution License (the "License").
* You may not use this file except in compliance with the License.
*
* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
* See the License for the specific language governing permissions
* and limitations under the License.
*
* When distributing Covered Code, include this CDDL HEADER in each
* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
* If applicable, add the following below this CDDL HEADER, with the
* fields enclosed by brackets "[]" replaced with your own identifying
* information: Portions Copyright [yyyy] [name of copyright owner]
*
* CDDL HEADER END
*/
/*
* Copyright 2009 Sun Microsystems, Inc. All rights reserved.
* Use is subject to license terms.
*/
#ifndef _SYS_IB_IBTL_IMPL_IBTL_H
#define _SYS_IB_IBTL_IMPL_IBTL_H
/*
* ibtl.h
*
* All data structures and function prototypes that are specific to the
* IBTL implementation.
*/
#ifdef __cplusplus
extern "C" {
#endif
/*
* Define a per IBT Client state structure. Its address is returned
* to the IBT client as an opaque IBT Client Handle - ibt_clnt_hdl_t.
*
* ibt_attach() allocates one of these structures.
*
* For each IBT Client registered with the IBTL, we maintain a list
* of HCAs, clnt_hca_list, that this IBT Client is using.
*
* This list is updated by ibt_open_hca().
*/
typedef struct ibtl_clnt_s {
/* module information */
void *clnt_private; /* IBT Client's private ptr */
struct ibtl_clnt_s *clnt_list_link;
/* link is ha_hca_link */
void *clnt_sm_trap_handler_arg;
} ibtl_clnt_t;
/* HCA Device State. */
typedef enum ibtl_hca_state_e {
/*
* Define a type to record hca async PORT_UP and PORT_DOWN events for
* processing by async thread(s). At the time an async is made by an
* HCA driver (presumably at interrupt level), a call is made to IBTL.
* IBTL marks this field, and wakes up an async thread for delivery
* to IBT clients as appropriate.
*/
typedef enum ibtl_async_port_status_e {
IBTL_HCA_PORT_UP = 0x001,
IBTL_HCA_PORT_DOWN = 0x002,
IBTL_HCA_PORT_CHG = 0x004,
IBTL_HCA_PORT_ASYNC_CLNT_REREG = 0x008,
/*
* Define a type to record the PORT async events and port change flags.
*/
typedef struct ibtl_async_port_event_s {
/*
* Bit definition(s) for {qp,cq,eec,hd,ha,srq}_async_flags.
*
* IBTL_ASYNC_PENDING This structure is known by the async_threads.
* It will be checked for additional async work
* before this bit is cleared, so new async
* to be linked onto its async list.
*
* IBTL_ASYNC_FREE_OBJECT Client has called ibt_free_*, and the
* the structure should be kmem_freed when
* the outstanding asyncs complete.
*/
typedef enum ibtl_async_flags_e {
IBTL_ASYNC_PENDING = 0x1,
IBTL_ASYNC_FREE_OBJECT = 0x2
/*
* Keeps track of all data associated with HCA port kstats.
*/
typedef struct ibtl_hca_port_kstat_s {
struct ibtl_hca_devinfo_s *pks_hca_devp;
struct kstat *pks_stats_ksp;
struct kstat *pks_pkeys_ksp;
/*
* Define a per CI HCA Device structure. Its address is returned
* to the CI as an opaque IBTL HCA Handle - ibc_hdl_t.
*
* ibc_ci_attach() allocates one of these and adds it to ibtl_hca_list.
*
* The hd_hca_dev_link is the link for the ibtl_hca_list. It is the
* list of HCA devices registered with the IBTL.
*
* The hd_clnt_list is a list of IBT Clients using this HCA.
* The hd_clnt_list->l_head points to the ha_clnt_link field of a client's
* ibtl_hca_s structure.
*
* This list is updated by ibt_open_hca().
*/
typedef struct ibtl_hca_devinfo_s {
int hd_portinfo_waiters; /* any waiters */
/* port whose info is queried */
int hd_async_busy; /* only 1 async at a time */
/* The following must be at the end of this struct */
/*
* Define a HCA info structure.
*
* The IBTL function ibt_open_hca() allocates one of these.
*
* For each client instance registered with the IBTL, we maintain a list
* of HCAs that it is using. The elements of that list include the
* address of the CI HCA device structure, a pointer to the client
* structure, and reference counts of HCA resources that this client
* device is using.
*
* Note: ha_qpn_cnt is protected by a global mutex to deal with a client
* trying to open the HCA while it is actively being closed.
*
* ha_hca_link is the link to the next HCA info struct that this client is
* using.
*
* ha_clnt_link is the link to the next IBT client (ibtl_clnt_t) that is using
* the same CI HCA (ibtl_hca_devinfo_t). The link points to that client's
* ibtl_hca_t because an IBT client can use more than one CI HCA.
*/
typedef struct ibtl_hca_s {
void *ha_clnt_private;
/* counters. */
int ha_flags; /* misc. flags */
} ibtl_hca_t;
/* ha_flags values */
/*
* Bit definition(s) for cq_impl_flags.
*
* IBTL_CQ_PENDING This CQ is known by the ibtl_cq_threads,
* and it will be checked for additional work
* before this bit is cleared, so new work
* will be seen without this cq being added
* to the cq list.
*
* IBTL_CQ_CALL_CLIENT Mark that the HCA driver has called
* ibc_cq_handler with new work on this CQ,
* so IBTL should call the client handler
* again before it is considered done.
*
* IBTL_CQ_FREE Mark that ibt_free_cq is sleeping until
* ibtl_cq_threads is done with this CQ.
*/
typedef enum ibtl_cq_impl_flags_e {
IBTL_CQ_PENDING = 0x1,
IBTL_CQ_CALL_CLIENT = 0x2,
IBTL_CQ_FREE = 0x4
/*
* Define a per CQ state structure.
*
* The ibt_alloc_cq() allocates one of these. A CQ is associated with a
* particular HCA, whose handle is recorded in the cq_hca field.
* The cq_ibc_cq_hdl field is initialized with the CI CQ handle returned
* from the ibc_alloc_cq() call to the HCA driver.
*
* need to use ibt_set_cq_private() and ibt_get_cq_private() calls.
*
* An IBT client registers a CQ completion handler callback and private
* callback argument (probably the client instance soft state structure) using
* the ibt_set_cq_handler() IBT routine. The comp_handler, arg fields of the
* structure are initialized with the values passed in by the IBTL client.
* These two fields are the only fields protected by the cq_mutex.
*
* When a completion event is posted to an IBT client, the
* client completion handler is called with the following arguments:
*
* - The Client Handle, that is passed into the IBTL on ibt_attach call.
* - The CQ Handle upon which the completion occurred.
* - The private client argument, set during handler registration via
* ibt_set_cq_handler() call.
*
* The address of the ibtl_cq_s structure is passed in as the ibt_cq_hdl_t
* (callback arg) in the CI ibc_alloc_cq() function. Thus when a CI calls
* the IBTL completion handler (ibc_ci_cq_handler()) we can de-mux
* directly to the targeted IBT client.
*
*/
typedef struct ibtl_cq_s {
void *cq_arg; /* CQ handler's argument */
void *cq_clnt_private; /* Client's Private. */
/* to be handled in a thread */
/* handler runs in a thread */
int cq_in_thread; /* mark if cq handler is to */
/* be called in a thread */
} ibtl_cq_t;
/*
* Define a per SRQ state structure.
*
* ibt_alloc_srq() allocates one of these. A SRQ is associated with a
* particular HCA, whose handle is recorded in the srq_hca field.
* The srq_ibc_srq_hdl field is initialized with the CI SRQ handle returned
* from the ibc_alloc_srq() call to the HCA driver.
*
* need to use ibt_set_srq_private() and ibt_get_srq_private() calls.
*
* The address of the ibtl_srq_s structure is passed in as the ibt_srq_hdl_t
* (callback arg) in the CI ibc_alloc_srq() function.
*/
typedef struct ibtl_srq_s {
void *srq_clnt_private; /* Client's Private. */
} ibtl_srq_t;
/*
* Define a per QP state structure.
*
* The qp_hca field is initialized with the ibtl_hca_hdl_t of the HCA in
* which the QP was allocated. The qp_ibc_qp_hdl field is initialized with
* the CI QP handle.
*
* The ibtl_qp_t structure also maintains a channel connection state
* structure that is only valid for RC and RD QP's. The information about
* the respective Send and Receive CQ, the RDD and PD Handles are also stored.
*
* The IBTA spec does not include the signal type or PD on a QP query
* operation. In order to implement the "CLONE" feature of the alloc rc|ud
* channel functions we need to cache these values.
*/
typedef struct ibtl_qp_s {
} ibtl_qp_t;
/*
* Define a per EEC state structure.
*
* The ibt_alloc_eec() allocates an ibt_eec_s structure and initializes
* the eec_hca field with the ibtl_hca_hdl_t of the HCA in which the EEC
* was allocated. The eec_ibc_eec_hdl field is initialized with the
* CI EEC handle.
*
* The information about CI's RDD Handle and channel connection state structure
* is also maintained.
*/
typedef struct ibtl_eec_s {
struct ibtl_channel_s *eec_channel;
} ibtl_eec_t;
/*
* Define an ibt RD communication channel struct. This holds information
* specific to an RD QP.
*/
typedef struct ibtl_rd_chan_s {
/*
* Define an ibt UD communication channel struct. This holds information
* specific to a UD QP.
*/
typedef struct ibtl_ud_chan_s {
/* ibt_modify_reply_ud_dest() */
/*
* Define an ibt RC communication channel struct. This holds information
* specific to an RC QP.
*/
typedef struct ibtl_rc_chan_s {
int rc_free_flags; /* Track connection state as */
/* we will need to delay for */
/* TIMEWAIT before freeing. */
/* TIMEWAIT delay. */
/* bit definitions for rc_free_flags */
#define IBTL_RC_QP_CONNECTED 0x1
#define IBTL_RC_QP_CLOSING 0x2
#define IBTL_RC_QP_CLOSED 0x4
#define IBTL_RC_QP_FREED 0x8
#define IBTL_RC_QP_CONNECTING 0x10
/*
* Define a per Channel state structure.
*
* A ibtl_channel_s is allocated each time a TI client calls a
* channel allocation routine ibt_alloc_rc_channel() or ibt_alloc_ud_channel()
* or VTI client calls ibt_alloc_qp() or ibt_alloc_special_qp().
*
* TI client's need to use ibt_set_chan_private() and ibt_get_chan_private()
* or VTI clients need to use ibt_set_qp_private() and ibt_get_qp_private().
*/
typedef struct ibtl_channel_s {
/* The ibtl_qp_t must be at the first of this struct */
union { /* transport specific */
} ch_transport;
void *ch_clnt_private; /* Client's Private data */
void *ch_cm_private; /* Ptr to CM state */
/*
* MACROS
*/
/*
* Get IBC HCA Handle from IBT Handles.
*/
/*
* Get a pointer to the HCA ops structure from IBT handles.
*/
#define IBTL_HCA2CIHCAOPS_P(ibtl_hca) \
#define IBTL_CHAN2CIHCAOPS_P(ibt_chan) \
#define IBTL_SRQ2CIHCAOPS_P(ibtl_srq) \
/*
* Get Client Handle from IBT Handles.
*/
/*
* Get a Pointer to the client modinfo from IBT Handles.
*/
#define IBTL_HCA2MODI_P(ibtl_hca) \
#define IBTL_EEC2MODI_P(ibtl_eec) \
/*
* Using HCA Device Info Pointer, access HCA Attributes values for
* Max SGID Table Size, Max PKEY Table Size.
*/
#define IBTL_HDIP2SGIDTBLSZ(hca) \
#define IBTL_HDIP2PKEYTBLSZ(hca) \
/*
* Using IBTL HCA Handle, access HCA Attributes values.
* viz. HCA Node GUID,
* Number of Ports on this HCA Device,
* Max SGID Table Size
* Max PKEY Table Size
*/
#define IBTL_HCA2HCAGUID(hca_hdl) \
#define IBTL_HCA2NPORTS(hca_hdl) \
#define IBTL_HCA2SGIDTBLSZ(hca_hdl) \
#define IBTL_HCA2PKEYTBLSZ(hca_hdl) \
/* possible strlen of a IB driver's name */
#define IBTL_DRVNAME_LEN 40
/* strings passed to ib_dprintfN() are this long */
#define IBTL_PRINT_BUF_LEN 4096
#define IBTL_GENERIC_CLIENT(clntp) \
/*
* Function Prototypes that are specific to the IBTL implementation.
*/
void ibtl_init_cep_states(void);
void ibtl_ib2usec_init(void);
void ibtl_logging_initialization(void);
void ibtl_logging_destroy(void);
void ibtl_thread_init(void);
void ibtl_thread_init2(void);
void ibtl_thread_fini(void);
void ibtl_another_cq_handler_in_thread(void);
void ibtl_qp_flow_control_enter(void);
void ibtl_qp_flow_control_exit(void);
/* synchronization of asyncs when freeing an object */
/* synchronization of cq_handler callbacks and free_cq */
/* release_qpn and close_hca synchronization */
/* Global List of HCA devices, and associated lock. */
/* Global List of IBT Client Instances, and associated lock. */
extern kmutex_t ibtl_clnt_list_mutex;
/* Lock for the race between the client and CM to free QPs. */
extern kmutex_t ibtl_free_qp_mutex;
/* Lock for the race between the client closing the HCA and QPN being freed. */
extern kcondvar_t ibtl_close_hca_cv;
/* Limit the flow of QP verb calls */
extern kmutex_t ibtl_qp_mutex;
extern kcondvar_t ibtl_qp_cv;
/* Async handlers and client private for well known clients of IBTL */
extern void *ibtl_cm_clnt_private;
extern void *ibtl_dm_clnt_private;
extern void *ibtl_ibma_clnt_private;
/* cache for fast GID => portinfo lookup */
extern boolean_t ibtl_fast_gid_cache_valid;
/* The following structs are used to pass info in and out of the APIs */
#ifdef __cplusplus
}
#endif
#endif /* _SYS_IB_IBTL_IMPL_IBTL_H */