ibmf_impl.h revision 7c478bd95313f5f23a4c958a745db2134aa03244
/*
* CDDL HEADER START
*
* The contents of this file are subject to the terms of the
* Common Development and Distribution License, Version 1.0 only
* (the "License"). You may not use this file except in compliance
* with the License.
*
* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
* See the License for the specific language governing permissions
* and limitations under the License.
*
* When distributing Covered Code, include this CDDL HEADER in each
* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
* If applicable, add the following below this CDDL HEADER, with the
* fields enclosed by brackets "[]" replaced with your own identifying
* information: Portions Copyright [yyyy] [name of copyright owner]
*
* CDDL HEADER END
*/
/*
* Copyright 2005 Sun Microsystems, Inc. All rights reserved.
* Use is subject to license terms.
*/
#ifndef _SYS_IB_MGT_IBMF_IBMF_IMPL_H
#define _SYS_IB_MGT_IBMF_IBMF_IMPL_H
#pragma ident "%Z%%M% %I% %E% SMI"
/*
* This file contains the IBMF implementation dependent structures and defines.
*/
#ifdef __cplusplus
extern "C" {
#endif
#define IBMF_MAX_SQ_WRE 64
#define IBMF_MAX_RQ_WRE 64
#define IBMF_MAX_POSTED_RQ_PER_QP 512
#define IBMF_MAX_POSTED_SQ_PER_QP 512
#define IBMF_MAX_SQ_WR_SGL_ELEMENTS 1
#define IBMF_MAX_RQ_WR_SGL_ELEMENTS 1
#define IBMF_MGMT_Q_KEY 0x80010000
#define IBMF_P_KEY_DEF_FULL 0xFFFF
#define IBMF_P_KEY_DEF_LIMITED 0x7FFF
#define IBMF_P_KEY_BASE_MASK 0x7FFF
#define IBMF_PKEY_MEMBERSHIP_MASK 0x8000
#define IBMF_TASKQ_1THREAD 1
#define IBMF_TASKQ_NTHREADS 128
/*
* Work request ID format used for receive requests.
*
* bit 0 set to 1
*/
#define IBMF_RCV_CQE 0x1
/*
* Convenience macro used in the RMPP protocol to obtain R_Method field
* of MAD header with Response bit flipped.
*/
#define IBMF_FLIP_RESP_BIT(r_method) \
/* Work Request ID macros */
#define IBMF_IS_RECV_WR_ID(id) \
#define IBMF_IS_SEND_WR_ID(id) \
(!(IBMF_IS_RECV_WR_ID((id))))
/* Decrement IBMF message reference count */
#define IBMF_MSG_DECR_REFCNT(msg) { \
(msg)->im_ref_count--; \
}
/* Increment IBMF message reference count */
#define IBMF_MSG_INCR_REFCNT(msg) \
(msg)->im_ref_count++;
#define IBMF_RECV_CB_SETUP(clp) { \
(clp)->ic_recvs_active++; \
}
#define IBMF_RECV_CB_CLEANUP(clp) { \
(clp)->ic_recvs_active--; \
if ((clp)->ic_recvs_active == 0) \
}
#define IBMF_ALT_RECV_CB_SETUP(altqp) { \
(altqp)->isq_recvs_active++; \
}
#define IBMF_ALT_RECV_CB_CLEANUP(altqp) { \
(altqp)->isq_recvs_active--; \
if ((altqp)->isq_recvs_active == 0) \
}
/* warlock annotations for ibmf.h and ibmf_msg.h structures */
/*
* WQE pool management contexts
*/
typedef struct _ibmf_wqe_mgt {
void *wqes_kmem; /* kmem allocated for WQEs */
/*
* structure used to keep track of qp handles
*/
typedef struct _ibmf_qp_t {
int iq_port_num; /* port num for this qp */
int iq_qp_num; /* qp num */
int iq_qp_ref; /* no. of clients using this */
} ibmf_qp_t;
/* defines for iq_flags */
#define IBMF_QP_FLAGS_INVALID 0x0001
#define IBMF_QP_FLAGS_INITING 0x0002
#define IBMF_QP_FLAGS_INITED 0x0004
#define IBMF_QP_FLAGS_UNINITING 0x0008
/*
* structure used to keep track of qp handles for qps other than
* the special qps
*/
typedef struct _ibmf_alt_qp_t {
void *isq_recv_cb_arg; /* arg for recv cb */
int isq_flags; /* to keep track of state */
int isq_sends_active; /* outstanding sends */
int isq_recvs_active; /* outstanding recvs */
int isq_port_num; /* port num for this qp */
int isq_wqes_alloced; /* wqes allocated for QP */
/*
* This structure is used to keep track of IBT returned ibt_ud_dest_t
* structures.
*/
typedef struct ibmf_ud_dest_s {
struct ibmf_ud_dest_s *ud_next;
/*
* ibmf_msg_impl definition
* The IBMF client initializes various members of the msg while sending
* the message. IBMF fills in the various members of the msg when a message
* is received.
*/
typedef struct _ibmf_msg_impl {
void *im_client; /* client that allocd the pkt */
void *im_trans_cb_arg; /* arg for completion cb */
int im_ref_count; /* reference count */
int im_pending_send_compls; /* send completions */
/* im_flags */
#define IBMF_MSG_FLAGS_SEQUENCED 0x1
#define IBMF_MSG_FLAGS_SEND_RMPP 0x2
#define IBMF_MSG_FLAGS_RECV_RMPP 0x4
#define IBMF_MSG_FLAGS_NOT_RMPP 0x8
#define IBMF_MSG_FLAGS_BUSY 0x10
#define IBMF_MSG_FLAGS_FREE 0x20
#define IBMF_MSG_FLAGS_ON_LIST 0x40
#define IBMF_MSG_FLAGS_SET_TERMINATION 0x80
#define IBMF_MSG_FLAGS_TERMINATION 0x100
/* retransmission parameter defaults for im_retrans field */
#define IBMF_RETRANS_DEF_RETRIES 0
/*
* Transaction state flags (im_trans_state_flags) definitions
* Don't use 0x0 as a flag value since clients OR and AND the flags
*/
#define IBMF_TRANS_STATE_FLAG_UNINIT 0x1
#define IBMF_TRANS_STATE_FLAG_INIT 0x2
#define IBMF_TRANS_STATE_FLAG_WAIT 0x4
#define IBMF_TRANS_STATE_FLAG_DONE 0x8
#define IBMF_TRANS_STATE_FLAG_SIGNALED 0x10
#define IBMF_TRANS_STATE_FLAG_TIMEOUT 0x20
#define IBMF_TRANS_STATE_FLAG_RECV_ACTIVE 0x40
#define IBMF_TRANS_STATE_FLAG_RECV_DONE 0x80
#define IBMF_TRANS_STATE_FLAG_SEND_DONE 0x100
/* Timer types */
typedef enum _ibmf_timer_t {
IBMF_RESP_TIMER = 1,
IBMF_TRANS_TIMER = 2
} ibmf_timer_t;
/*
* structure to hold specific client info taken from ibmf_register_info_t
* since we can register for more than one client at a time, but each specific
* ibmf_client_t only holds one client itself.
*/
typedef struct _ibmf_client_info {
/*
* Bits 16-19 of the client_class specify the client type.
*/
#define IBMF_AGENT_ID 0x00010000
#define IBMF_MANAGER_ID 0x00020000
#define IBMF_AGENT_MANAGER_ID 0x00030000
/*
* structure used to keep track of clients
*/
typedef struct _ibmf_client {
void *ic_client_sig; /* set for valid handles */
/* IBTL asynchronous event callback (eg. HCA offline) */
void *ic_async_cb_arg; /* args for async cb */
/* Asynchronous/Unsolicited message handler */
void *ic_recv_cb_arg;
int ic_flags; /* to keep track of state */
int ic_reg_flags; /* flags specified during */
/* registration */
/* Statistics */
int ic_msgs_alloced; /* no. msgs alloced by/for */
int ic_msgs_active; /* no. msgs active */
int ic_trans_active; /* outstanding transacts */
int ic_sends_active; /* outstanding sends */
int ic_recvs_active; /* outstanding recvs */
/* remove recv_cb */
/* IBMF_MAD_ONLY is used by the alternate QP context only (isq_flags) */
#define IBMF_MAD_ONLY 0x00002000
#define IBMF_RAW_ONLY 0x00004000
#define IBMF_REG_MSG_LIST 0
#define IBMF_TERM_MSG_LIST 1
/*
* Send WQE context
*/
typedef struct _ibmf_send_wqe {
struct _ibmf_send_wqe *send_wqe_next;
void *send_mem; /* memory used in send */
/*
* Receive WQE context
*/
typedef struct _ibmf_recv_wqe {
struct _ibmf_recv_wqe *recv_wqe_next;
void *recv_mem; /* memory used in WQEs */
/*
* Struct that keeps track of the underlying IB channel interface. There
* is one per CI. Each clients on a given ci gets a reference to the CI.
* References are tracked used ci_ref field; when ci_ref drops to 0, the
* structure can be freed.
*/
typedef struct _ibmf_ci {
/* UD destination resources */
/* free QP synchronization with WQE completion processing */
int ci_wqes_alloced; /* wqes alloced for sp QPs */
/* port kstats */
} ibmf_ci_t;
/* defines for ci_init_state */
#define IBMF_CI_INIT_HCA_INITED 0x0001
#define IBMF_CI_INIT_MUTEX_CV_INITED 0x0002
#define IBMF_CI_INIT_SEND_TASKQ_DONE 0x0004
#define IBMF_CI_INIT_RECV_TASKQ_DONE 0x0008
#define IBMF_CI_INIT_CQ_INITED 0x0010
#define IBMF_CI_INIT_WQES_ALLOCED 0x0020
#define IBMF_CI_INIT_HCA_LINKED 0x0040
#define IBMF_CI_INIT_QP_LIST_INITED 0x0080
/* defines for ci_state */
#define IBMF_CI_STATE_PRESENT 0x0001
#define IBMF_CI_STATE_INITED 0x0002
#define IBMF_CI_STATE_GONE 0x0003
/* defines for ci_state_flags */
#define IBMF_CI_STATE_INIT_WAIT 0x0001
#define IBMF_CI_STATE_UNINIT_WAIT 0x0002
#define IBMF_CI_STATE_VALIDATE_WAIT 0x0004
#define IBMF_CI_STATE_INVALIDATING 0x0100
#define IBMF_CI_STATE_VALIDATING 0x0200
#define IBMF_CI_STATE_UNINITING 0x0400
#define IBMF_CI_STATE_INITING 0x0800
/*
* for keeping track of ibmf state
*/
typedef struct _ibmf_state {
struct _ibmf_ci *ibmf_ci_list;
struct _ibmf_ci *ibmf_ci_list_tail;
/* for classes not registered */
} ibmf_state_t;
/*
* Endian macros for swapping ib_mad_hdr components. These are used by both
* ibmf and ibmf_saa. MAD header fields only need to be swapped, so in-place
* swapping using these #defines is better. However, the SA header and data
* portions need to be unpacked as well (they can have unaligned 64-bit fields).
* So the ibmf_utils pack, unpack functions are used for swapping and unpacking
* the data portions of ibmf_saa messages.
* The macros below are copied from the ibtl macros.
*/
/*
* Endian Macros
* h2b - host endian to big endian protocol
* b2h - big endian protocol to host endian
* h2l - host endian to little endian protocol
* l2h - little endian protocol to host endian
*/
#if defined(_LITTLE_ENDIAN)
#define h2b64(x) (ddi_swap64(x))
#define b2h64(x) (ddi_swap64(x))
#elif defined(_BIG_ENDIAN)
#define h2b16(x) (x)
#define h2b32(x) (x)
#define h2b64(x) (x)
#define b2h16(x) (x)
#define b2h32(x) (x)
#define b2h64(x) (x)
#else
#endif
/* UD Destination resource cache definitions */
/*
* It is preferred that the difference between the hi and lo water
* marks be only a few ud_dest resources. The intent is that a
* thread that needs to run ibmf_i_populate_ud_dest_list() does not
* spend too much time in this ud_dest resource population process
* before it returns to its caller. A benefit of a higher lo water
* mark is that the larger available pool of resources supports high
* stress scenarios better.
*/
#define IBMF_UD_DEST_HI_WATER_MARK 512
#define IBMF_UD_DEST_LO_WATER_MARK 500
/*
* Prototypes
*/
/* ci related functions */
/* client related functions */
/* qp related functions */
/* pkt related functions */
void ibmf_i_pop_ud_dest_thread(void *argp);
/* WQE related functions */
#ifdef DEBUG
#endif
/* msg related functions */
int km_flags);
/* transaction related functions */
int msg_flags);
/* timer related functions */
void ibmf_i_recv_timeout(void *argp);
void ibmf_i_send_timeout(void *argp);
void ibmf_i_err_terminate_timeout(void *msgp);
/* rmpp related functions */
#ifdef NOTDEF
#endif
int block);
/* Alternate QP WQE cache functions */
int kmflags);
int kmflags);
/* Receive callback functions */
/* UD Dest population thread */
#ifdef __cplusplus
}
#endif
#endif /* _SYS_IB_MGT_IBMF_IBMF_IMPL_H */