ibti_cm.h revision 7c478bd95313f5f23a4c958a745db2134aa03244
/*
* CDDL HEADER START
*
* The contents of this file are subject to the terms of the
* Common Development and Distribution License, Version 1.0 only
* (the "License"). You may not use this file except in compliance
* with the License.
*
* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
* See the License for the specific language governing permissions
* and limitations under the License.
*
* When distributing Covered Code, include this CDDL HEADER in each
* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
* If applicable, add the following below this CDDL HEADER, with the
* fields enclosed by brackets "[]" replaced with your own identifying
* information: Portions Copyright [yyyy] [name of copyright owner]
*
* CDDL HEADER END
*/
/*
* Copyright 2004 Sun Microsystems, Inc. All rights reserved.
* Use is subject to license terms.
*/
#ifndef _SYS_IB_IBTL_IBTI_CM_H
#define _SYS_IB_IBTL_IBTI_CM_H
#pragma ident "%Z%%M% %I% %E% SMI"
/*
*
* This file contains the data structure definitions for the IBTI
* communication manager (CM). It is only included in ibti.h
*/
#ifdef __cplusplus
extern "C" {
#endif
/*
* Defines.
*/
#define IBT_MAX_PRIV_DATA_SZ 224
#define IBT_DREP_PRIV_DATA_SZ 224
#define IBT_RTU_PRIV_DATA_SZ 224
#define IBT_MRA_PRIV_DATA_SZ 222
#define IBT_DREQ_PRIV_DATA_SZ 220
#define IBT_REP_PRIV_DATA_SZ 196
#define IBT_LAP_PRIV_DATA_SZ 168
#define IBT_APR_PRIV_DATA_SZ 148
#define IBT_REJ_PRIV_DATA_SZ 148
#define IBT_REQ_PRIV_DATA_SZ 92
#define IBT_SIDR_REQ_PRIV_DATA_SZ 216
#define IBT_SIDR_REP_PRIV_DATA_SZ 136
/* This is the max consumer addl */
/* reject info len */
typedef uint8_t ibt_priv_data_len_t;
/*
* CM channel handler reject reasons.
*
* Refer to InfiniBand Architecture Release Volume 1 Rev 1.0a:
* Section 12.6.7.2 Rejection Reason
*/
typedef enum ibt_cm_reason_e {
IBT_CM_SUCCESS = 0, /* Success */
/* a msg */
/* does not refer to a valid */
/* communication Instance. */
/* recognized */
/* not supported */
/* accepted */
/* supported */
/* not supported */
/* accepted */
/* state */
/*
* CM flags.
*/
typedef uint8_t ibt_cm_flags_t;
#define IBT_CM_NO_FLAGS 0x0
#define IBT_CM_FLOW_CONTROL 0x1
#define IBT_CM_SRQ_EXISTS 0x2
/*
* The CM Handler function return values.
*
* Applicable as follows :
* Note : default return value is used by the CM, if the cm handler return
* value is IBT_CM_DEFAULT
*
*
* RC handler return:
* ==================
*
* Event type Allowed return values default return value
* ---------- --------------------- --------------------
* IBT_CM_EVENT_REQ_RCV IBT_CM_ACCEPT IBT_CM_REJECT
* IBT_CM_REJECT
* IBT_CM_REDIRECT_PORT
* IBT_CM_REDIRECT
* IBT_CM_NO_CHANNEL
* IBT_CM_NO_RESOURCE
* IBT_CM_DEFER
*
* IBT_CM_EVENT_REP_RCV IBT_CM_ACCEPT IBT_CM_REJECT
* IBT_CM_REJECT
* IBT_CM_NO_RESOURCE
* IBT_CM_DEFER
*
* IBT_CM_EVENT_MRA_RCV IBT_CM_ACCEPT IBT_CM_ACCEPT
*
* IBT_CM_EVENT_CONN_CLOSED IDT_CM_ACCEPT IBT_CM_ACCEPT
* IBT_CM_DEFER
*
* IBT_CM_EVENT_LAP_RCV IBT_CM_ACCEPT IBT_CM_REJECT
* IBT_CM_REJECT
* IBT_CM_REDIRECT_PORT
* IBT_CM_NO_RESOURCE
* IBT_CM_DEFER
*
* IBT_CM_EVENT_APR_RCV IBT_CM_ACCEPT IBT_CM_ACCEPT
*
* IBT_CM_EVENT_CONN_EST IBT_CM_ACCEPT IBT_CM_ACCEPT
*
* IBT_CM_EVENT_FAILURE IBT_CM_ACCEPT IBT_CM_ACCEPT
*
*
* UD handler return
* =================
*
* Event type Allowed return values default return value
* ---------- --------------------- --------------------
* IBT_CM_UD_EVENT_SIDR_REQ IBT_CM_ACCEPT IBT_CM_REJECT
* IBT_CM_REJECT
* IBT_CM_REDIRECT
* IBT_CM_NO_CHANNEL
* IBT_CM_NO_RESOURCE
* IBT_CM_DEFER
*
* IBT_CM_UD_EVENT_SIDR_REP IBT_CM_ACCEPT IBT_CM_ACCEPT
*
* The following are the meanings of cm handler return values:
*
* IBT_CM_ACCEPT The CM handler function accepts the specified
* event.
* IBT_CM_REJECT The CM handler rejects the specified event.
* IBT_CM_REDIRECT The CM handler is redirecting a REQ indicating
* that the service is provided at another port.
* IBT_CM_NO_RESOURCE Unable to allocate resources.
* IBT_CM_DEFER Client would callback CM via ibt_cm_proceed or
* ibt_cm_ud_proceed for return args and outgoing
* private data. Once cm handler is returned to CM,
* client cannot access any of the handler args.
* IBT_CM_DEFAULT The default return value.
*
*
*/
typedef enum ibt_cm_status_e {
IBT_CM_ACCEPT = 0,
IBT_CM_REJECT = 1,
} ibt_cm_status_t; /* will call ibt_cm_proceed() */
/* later */
/*
* SIDR_REP status type
*/
typedef enum ibt_sidr_status_e {
/*
* Alternate path status type
* The implementation defined status codes begin from 20. The status codes
* below 20 are based on apr_status in the APR mad.
*/
typedef enum ibt_ap_status_e {
IBT_CM_AP_LOADED = 0, /* AP loaded successfully */
/* earlier because of connection */
/* getting closed */
/*
* Communication event types.
*/
typedef enum ibt_cm_event_type_e {
IBT_CM_EVENT_REQ_RCV = 0x1,
IBT_CM_EVENT_CONN_EST, /* RTU has been sent/recvd and it is */
/* OK to use the connection */
IBT_CM_EVENT_CONN_CLOSED, /* Connection has been closed and it */
/* is OK to free resources associated */
/* with the connection. */
IBT_CM_EVENT_FAILURE /* The CM Failure see cf_code in the */
/* ibt_cm_conn_failed_t struct for */
/* details of the failure */
/*
* CM and Port redirect information.
*/
typedef struct ibt_redirect_info_s {
/*
* Values for rep_failover_status.
*/
/*
* CM REP_RCV event structure.
*/
typedef struct ibt_cm_rep_rcv_s {
/* resources (rdma_ra_in) */
/* depth (rdma_ra_out) */
/* Time to complete */
/* processing of REP event */
/*
* Values for mra_msg_type.
*/
#define IBT_CM_MRA_TYPE_REP 0x01
#define IBT_CM_MRA_TYPE_LAP 0x02
/*
* CM MRA_RCV event structure.
*/
typedef struct ibt_cm_mra_rcv_s {
/*
* CM LAP_RCV event structure.
*/
typedef struct ibt_cm_lap_rcv_s {
/* This is the time that the */
/* Service handler has to */
/* return to the CM */
/*
* Consumer defined Additional reject information.
*/
typedef struct ibt_ari_con_s {
/*
* Additional reject information.
*/
typedef union ibt_arej_info_u {
/* IBT_CM_ALT_GID, */
/* IBT_CM_PORT_REDIRECT */
/* IBT_CM_ALT_LID */
/* IBT_CM_INVALID_ALT_SL */
/* IBT_CM_INVALID_ALT_TC */
/* IBT_CM_INVALID_ALT_HOP */
/* IBT_CM_INVALID_ALT_FLOW */
/* IBT_CM_INVALID_ALT_RATE */
/*
* CM APR_RCV event structure.
*/
typedef struct ibt_cm_apr_rcv_s {
/*
* Connection Failed Message type (values of cf_msg).
* Identifies the CM message that either timed out or is being rejected.
*/
#define IBT_CM_FAILURE_REQ 0x00
#define IBT_CM_FAILURE_REP 0x01
#define IBT_CM_FAILURE_UNKNOWN 0x02
#define IBT_CM_FAILURE_LAP 0x03
/*
* CM Failure code (values of cf_code).
* Identifies the reason for failure.
*/
#define IBT_CM_FAILURE_REJ_SENT 0x00
#define IBT_CM_FAILURE_REJ_RCV 0x01
#define IBT_CM_FAILURE_TIMEOUT 0x02
#define IBT_CM_FAILURE_DUP 0x03
#define IBT_CM_FAILURE_STALE 0x04
/*
* IBT_CM_EVENT_FAILURE event struct.
*
* This event is generated by the CM when it decides to fail a connection
* establishment or the CM operation times out. The cf_code identifies the
* reason for failure as:
*
* IBT_CM_FAILURE_REJ_SENT - The CM decided to fail the connection and
* is going to send a REJ. cf_reason and
* cf_additional are the rej_reason code and
* additional reject information that the CM
* will be placing in the REJ message.
* Returning this event gives the client an
* opportunity to specify some private data
* to be placed in the REJ message.
* IBT_CM_FAILURE_REJ_RCV - The CM received a reject message.
* IBT_CM_FAILURE_TIMEOUT - A CM message timed out.
* IBT_CM_FAILURE_DUP - The CM detected a duplicate connection
* request and deferred to the remote CM.
* IBT_CM_FAILURE_STALE - Stale Connection.
*
*
* The table below shows additional details about which ibt_cm_conn_failed
* struct members are defined.
*
* code msg reason additional
* Defined? Defined? Defined?
* ---- --- ------ ----------
* IBT_CM_FAILURE_REJ_SENT Yes Yes Yes
* IBT_CM_FAILURE_REJ_RCV Yes Yes Yes
* IBT_CM_FAILURE_TIMEOUT Yes No No
* IBT_CM_FAILURE_DUP No No No
* IBT_CM_FAILURE_STALE No No No
*
*/
typedef struct ibt_cm_conn_failed_s {
/* being rejected */
/*
* CM REQ_RCV event structure.
*
* The req_cm_opaque is an IBTF CM opaque (to the client) value, that should
* ibt_cm_delay().
*
* prim_addr->av_dgid Is the GID of the requester.
* prim_addr->av_sgid Is the local GID to which the
* requester is attempting to establish
* a connection to.
* hca_guid Is the HCA GUID that contains the
* prim_addr->av_sgid
* prim_hca_port Is the port on the hca_guid that
* prim_addr->av_sgid is on.
*/
typedef struct ibt_cm_req_rcv_s {
/* resources. */
/* This is the time that the */
/* Service handler has to */
/* return to the CM */
/*
* The IBT_CM_EVENT_CONN_CLOSED event is generated by the CM when a connection
* has been closed. The reason the connection was closed is given in the
* "closed" member of the cm_event as one of:
*
* IBT_CM_CLOSED_DREP_RCVD Closed DREP received. This event is posted to
* the client that initiated the close, and
* indicates that the DREP was received from
* the passive side.
* IBT_CM_CLOSED_DREQ_RCVD Closed DREQ received. This event is posted to
* the passive side and indicates that a DREQ has
* been received. Clients should free the channel
* on receipt of this event.
* IBT_CM_CLOSED_REJ_RCVD Closed REJECT received. This is posted to the
* client that opened the channel in question
* (active side) if a reject is received at any
* time after a connection is established.
* IBT_CM_CLOSED_DREQ_TIMEOUT
* Closed DREQ timeout. This event is posted to
* the client that initiated the close (active
* side) if the close request has timed out.
* IBT_CM_CLOSED_DUP Closed crossing DREQ, deferred to remote. This
* event is posted to a client that initiated a
* close but also received a close request when
* the CM chooses to adopt a passive side roll
* and defer to the remote client.
* IBT_CM_CLOSED_ABORT A connection that was in progress was aborted.
* IBT_CM_CLOSED_STALE Closed as CM detected that connection has became
* stale. The remote node probably crashed or DREQ
* from remote node got lost in the fabric.
*
* A client can only call ibt_free_channel() to free channel resources on
* receipt of the IBT_CM_EVENT_CONN_CLOSED event.
*/
#define IBT_CM_CLOSED_DREP_RCVD 0x01
#define IBT_CM_CLOSED_DREQ_RCVD 0x02
#define IBT_CM_CLOSED_REJ_RCVD 0x03
#define IBT_CM_CLOSED_DREQ_TIMEOUT 0x04
#define IBT_CM_CLOSED_DUP 0x05
#define IBT_CM_CLOSED_ABORT 0x06
#define IBT_CM_CLOSED_STALE 0x07
#define IBT_CM_CLOSED_ALREADY 0x08
/*
* Operation type in ibt_cm_event_type_t.
*
* Note:
* The IBT_CM_EVENT_CONN_EST event has no associated "cm_event"
* field in the ibt_cm_event_t structure.
*
* The cm_session_id is a CM opaque (to the client) value, that
* to call ibt_cm_delay() or ibt_cm_proceed().
*
*/
typedef struct ibt_cm_event_s {
void *cm_session_id; /* Used by the CM */
/* for ibt_cm_req_rcv_t */
void *cm_priv_data; /* data returned */
union {
} cm_event; /* operation specific */
/*
* CM Return structure for an incoming REQ
* Server handler that wishes to accept the connection, fills all the
* values before returning to CM
*/
typedef struct ibt_cm_ret_rep_s {
/* connection will be */
/* established */
/* can be returned NULL */
/* if no resources available */
/* Number of RDMA RD's & */
/* Atomics outstanding */
/* Responder resources for */
/* handling incoming */
/* RDMA RD's & Atomics */
/*
* Define an ibt_cm_proceed() argument union.
*
* rep is valid when an IBT client cm handler has decided to continue a
* REQ_RCV, accepting the connection.
*
* rej is valid when an IBT client cm handler has decided to continue a
* REQ_RCV or REP_RCV, rejecting the message.
*
* apr is valid when an IBT client cm handler has decided to continue a
* LAP_RCV, redirecting the lap request.
*/
typedef union ibt_cm_proceed_reply_s {
/*
* Define a CM handler return arguments structure.
*
* cm_ret.rep is returned to the CM when an IBT client cm handler has
* decided to accept a connection in response to a CM REQ_RCV event.
*
* cm_ret.rej is returned to the CM when an IBT client cm handler
* has decided to reject the connection in response to a CM REQ_RCV event.
*
* cm_ret.apr is returned to the CM when an IBT client cm handler
* has decided to redirect the lap request in response to a CM LAP RCV event.
*
* bytes filled in priv_data arg of the cm handler.
*/
typedef struct ibt_cm_return_args_s {
union {
} cm_ret; /* reply specific */
/*
* Communication Manager UD event types.
*/
typedef enum ibt_cm_ud_event_type_e {
/*
* CM SIDR_REQ event structure.
*/
typedef struct ibt_cm_sidr_req_s {
/*
* CM SIDR_REP event structure.
*/
typedef struct ibt_cm_sidr_rep_s {
/*
* Operation type in ibt_cm_ud_event_type_t.
*
* The cm_session_id is a CM opaque (to the client) value, that
* wishes to call ibt_cm_ud_proceed().
*/
typedef struct ibt_cm_ud_event_s {
void *cm_session_id; /* Used by the CM */
void *cm_priv_data;
union {
} cm_event; /* operation specific */
/*
* Define a CM UD handler return arguments structure.
*
* The information here is returned to the CM when an IBT client CM UD
* handler has decided to communicate (via UD messages) with the requester
* of the SIDR_REQ_RCV event.
*
* The ud_channel encodes the QPN and Q_Key to be placed in the SIDR_REP.
*
* CM sets ud_ret_len to 0 before calling the UD CM handler. If the CM UD
* handler wishes to send private data back, it needs to update ud_ret_len
* with the actual number of bytes to be sent back in the SIDR_REP MAD. It
* copies said data to the buffer pointed to by the ret_priv_data argument.
*
* The ud_redirect structure should only be updated if the UD CM handler is
* Redirecting a CM request.
*/
typedef struct ibt_cm_ud_return_args_s {
/*
* IBT Client CM Callback function typedefs.
*
* ibt_cm_handler_t:
*
* The CM event handler function. An IBT client callback that handles
* CM events. If the handler will exceed the event service timeout
* then it should call ibt_cm_delay(), specifying the maximum time it
* will take to complete processing the CM event.
*
* Clients are advised not to issue blocking calls from a cm handler, as this
* would block the CM threads, and could delay or block other client
* connections.
*
* such as ibt_free_cq, ibt_free_rc/ud_channel, etc. on connection failure
* or tear down.
*
* Clients should not call ibt_close_rc_channel in the cm handler for connection
* failure or tear down events, as these events already perform the processing
* necessary to close the channel.
*
* Clients can call ibt_close_rc_channel only in the non-blocking mode from
* the cm handler for connection abort.
*
* Arguments:
*
* cm_private A pointer to the client private data, set by the
* client on a call to ibt_open_rc_channel (client side),
* or from a call to ibt_bind_service (server side).
*
* event Pointer to an CM event structure, contains the
* following:
*
* cm_type Describes the event
* cm_channel Event channel, not valid for ibt_cm_req_rcv_t
* cm_priv_data Pointer to the private data associated with
* the event. This field may point to an
* unaligned address and can be NULL, if there is
* no valid private data for the cm event. The
* following events can contain private data:
* IBT_CM_EVENT_REQ_RCV
* IBT_CM_EVENT_REP_RCV
* IBT_CM_EVENT_CONN_EST (passive)
* IBT_CM_EVENT_LAP_RCV
* IBT_CM_EVENT_APR_RCV
* IBT_CM_EVENT_MRA_RCV
* IBT_CM_EVENT_FAILURE
* (IBT_CM_FAILURE_REJ_RCVD)
* IBT_CM_EVENT_CONN_CLOSED
* (IBT_CM_CLOSED_DREQ_RCV,
* IBT_CM_CLOSED_DREP_RCVD,
* IBT_CM_CLOSED_REJ_RCVD,
* IBT_CM_CLOSED_DUP)
*
* cm_priv_data_len Number of valid bytes pointed by
* cm_priv_data. It shall be set to 0 by CM, if
* cm_priv_data is NULL
*
* union {
* ibt_cm_rep_rcv_t rep;
* ibt_cm_req_rcv_t req;
* ibt_cm_lap_rcv_t lap;
* ibt_cm_apr_rcv_t apr;
* ibt_cm_mra_rcv_t mra;
* ibt_cm_conn_failed_t conn_failed;
* } cm_event;
*
* ret_args A pointer to a ibt_cm_return_args_t structure.
*
* cm_ret_len Actual number of private_data bytes
* to copy into the outgoing MAD.
*
* The structure members of cm_ret.rep should only
* be updated on the return from a REQ_RCV event if
*
* channel An ibt_channel_hdl_t. The service
* handler function, if it decides to
* accept the connection request, should
* specify a ibt_channel_hdl_t over which
* the connection will be established and
* return the handle to the CM via this
* parameter.
* sent Number of RDMA RD's & Atomics
* outstanding.
* rdma_ra_in Responder resources - Incoming RDMA-R/
* Atomics
* If IBT client does not specify values for rdma_ra_out
* and rdma_ra_in and accepts the connection, CM uses the
* respectives values from the REQ message.
*
* The below structure member should only be updated on
* the return from a REQ_RCV or REP_RCV event if
*
* cm_ret.rej Additional reject information.
*
* ret_priv_data Points to a location where an IBT client can copy
* client private data to. This data will be placed in
* the next protocol message generated by the CM.
* ret_priv_data may point to a unaligned address.
* If ret_priv_data is NULL, the client cannot specify any
* private data for that event type. ret_priv_data is valid
* for the following events:
* IBT_CM_EVENT_REQ_RCV
* IBT_CM_EVENT_REP_RCV
* IBT_CM_EVENT_LAP_RCV
* IBT_CM_EVENT_CONN_CLOSED
* (IBT_CM_CLOSED_DREQ_RCV,
* IBT_CM_CLOSED_DUP)
*
* ret_len_max The maximum number of bytes that the client may
* copy into ret_priv_data. The actual number of
* bytes that client provides is set in the cm_ret_len
* field in ret_args above. ret_len_max will be set
* to the maximum size for the possible response MADs.
*
* Returns:
* IBT_CM_ACCEPT
* IBT_CM_REJECT
* IBT_CM_REDIRECT
* IBT_CM_REDIRECT_PORT
* IBT_CM_NO_CHANNEL
* IBT_CM_NO_RESOURCE
* IBT_CM_DEFER
* IBT_CM_DEFAULT
*
* See ibt_cm_status_t at the start of the file for details.
*
*/
/*
* ibt_cm_ud_handler_t - Pointer to the CM UD event handler function.
* This function should handle the following CM
* events:
* IBT_CM_UD_EVENT_SIDR_REQ
* IBT_CM_UD_EVENT_SIDR_REP
*
* Arguments:
*
* ud_cm_private A pointer to the client private data, (probably the
* client device soft state struct), set by the client
* during resource allocations - ibt_alloc_ud_dest().
* Set by the server with ibt_bind_service().
*
* event Pointer to an CM UD event structure.
*
* ret_args A pointer to a ibt_cm_ud_return_args_t structure. This
* should only be updated on the return from a SIDR_REQ
* event.
*
* ret_priv_data Points to a location where an IBT client can copy
* client private data to. This data will be placed in
* the next protocol message generated by the CM.
* This field may point to a unaligned address
*
* ret_len_max The maximum number of bytes that the client may
* copy into ret_priv_data. The actual number of
* bytes that client provides is set in the ud_ret_len
* field in ret_args above. ret_len_max will be set
* to the maximum size for the possible response MADs.
*
* Returns:
* IBT_CM_ACCEPT
* IBT_CM_REJECT
* IBT_CM_REDIRECT
* IBT_CM_NO_CHANNEL
* IBT_CM_NO_RESOURCE
* IBT_CM_DEFER
* IBT_CM_DEFAULT
*
* See ibt_cm_status_t at the start of the file for details.
*/
#ifdef __cplusplus
}
#endif
#endif /* _SYS_IB_IBTL_IBTI_CM_H */