ibtl_chan.c revision 17a2b317610f531d565bf4e940433aab2d9e6985
/*
* CDDL HEADER START
*
* The contents of this file are subject to the terms of the
* Common Development and Distribution License (the "License").
* You may not use this file except in compliance with the License.
*
* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
* See the License for the specific language governing permissions
* and limitations under the License.
*
* When distributing Covered Code, include this CDDL HEADER in each
* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
* If applicable, add the following below this CDDL HEADER, with the
* fields enclosed by brackets "[]" replaced with your own identifying
* information: Portions Copyright [yyyy] [name of copyright owner]
*
* CDDL HEADER END
*/
/*
*/
/*
*
* This file contains Transport API functions related to Channel Functions
* and internal Protection Domain and Address Handle Verbs functions.
*/
static char ibtl_chan[] = "ibtl_chan";
/*
* RC Channel.
*/
/*
* Function:
* ibt_alloc_rc_channel
* Input:
* hca_hdl HCA Handle.
* flags Channel allocate flags.
* args A pointer to an ibt_rc_chan_alloc_args_t struct that
* specifies required channel attributes.
* Output:
* rc_chan_p The returned RC Channel handle.
* sizes NULL or a pointer to ibt_chan_sizes_s struct where
* Returns:
* IBT_SUCCESS
* IBT_INVALID_PARAM
* Description:
* Allocates a RC communication channels that satisfy the specified
* channel attributes.
*/
{
if (flags & IBT_ACHAN_USER_MAP)
if (flags & IBT_ACHAN_DEFER_ALLOC)
if (flags & IBT_ACHAN_USES_SRQ) {
"NULL SRQ Handle specified.");
return (IBT_INVALID_PARAM);
}
}
/*
* Check if this request is to clone the channel, or to allocate a
* fresh one.
*/
if (flags & IBT_ACHAN_CLONE) {
"Clone Channel info not available.");
return (IBT_INVALID_PARAM);
"Clone Channel's & requested HCA Handle mismatch");
return (IBT_INVALID_PARAM);
}
/*
* Query the source channel, to obtained the attributes
* so that the new channel share the same attributes.
*/
if (retval != IBT_SUCCESS) {
"Failed to query the source channel: %d", retval);
return (retval);
}
/* Setup QP alloc attributes. */
sizeof (ibt_chan_sizes_t));
} else {
/* Setup QP alloc attributes. */
sizeof (ibt_chan_sizes_t));
if ((args->rc_hca_port_num == 0) ||
"Invalid port_num %d, range is (1 to %d)",
return (IBT_HCA_PORT_INVALID);
}
/*
* We allocate the Channel initially with the default PKey,
* and later client can update this when the channel is opened
* with the pkey returned from a path record lookup.
*/
}
/* Allocate Channel and Initialize the channel. */
&chanp);
if (retval != IBT_SUCCESS) {
"Failed to allocate QP: %d", retval);
return (retval);
}
/* Initialize RC Channel by transitioning it to INIT State. */
if (retval != IBT_SUCCESS) {
"Failed to Initialize QP: %d", retval);
/* Free the QP as we failed to initialize it. */
(void) ibt_free_qp(chanp);
return (retval);
}
return (IBT_SUCCESS);
}
/*
* Function:
* ibt_query_rc_channel
* Input:
* rc_chan A previously allocated channel handle.
* chan_attrs A pointer to an ibt_rc_chan_query_args_t struct where
* Channel's current attributes are returned.
* Output:
* chan_attrs A pointer to an ibt_rc_chan_query_args_t struct where
* Channel's current attributes are returned.
* Returns:
* IBT_SUCCESS
* Description:
* Query an RC channel's attributes.
*/
{
return (IBT_CHAN_SRV_TYPE_INVALID);
}
/* Query the channel (QP) */
if (retval != IBT_SUCCESS) {
return (retval);
}
return (retval);
}
/*
* Function:
* ibt_modify_rc_channel
* Input:
* rc_chan A previously allocated channel handle.
* flags Specifies which attributes in ibt_rc_chan_modify_attr_t
* are to be modified.
* attrs Attributes to be modified.
* Output:
* actual_sz On return contains the new send and receive queue sizes.
* Returns:
* IBT_SUCCESS
* Description:
* Modifies an RC channel's attributes, as specified by a
* ibt_cep_modify_flags_t parameter to those specified in the
* ibt_rc_chan_modify_attr_t structure.
*/
{
int retries = 1;
return (IBT_CHAN_SRV_TYPE_INVALID);
}
if (flags & IBT_CEP_SET_ADDS_VECT) {
sizeof (ibt_adds_vect_t));
}
/* Current channel state must be either SQD or RTS. */
if (flags & IBT_CEP_SET_ALT_PATH) {
sizeof (ibt_adds_vect_t));
}
if (retval != IBT_SUCCESS) {
/* give it one more shot if the old current state was stale */
--retries >= 0 &&
goto retry;
}
return (retval);
}
/*
* UD Channel.
*/
/*
* Function:
* ibt_alloc_ud_channel
* Input:
* hca_hdl HCA Handle.
* flags Channel allocate flags.
* args A pointer to an ibt_ud_chan_alloc_args_t struct that
* specifies required channel attributes.
* Output:
* ud_chan_p The returned UD Channel handle.
* sizes NULL or a pointer to ibt_chan_sizes_s struct where
* Returns:
* IBT_SUCCESS
* IBT_INVALID_PARAM
* Description:
* Allocate UD channels that satisfy the specified channel attributes.
*/
{
if (flags & IBT_ACHAN_USES_FEXCH) {
"FEXCH QPs are allocated by ibt_alloc_ud_channel_range()");
return (IBT_CHAN_SRV_TYPE_INVALID);
}
/* allow at most one of these flags */
switch (variant_flags) {
case IBT_ACHAN_USER_MAP:
break;
case IBT_ACHAN_USES_RSS:
break;
case IBT_ACHAN_USES_RFCI:
break;
case IBT_ACHAN_USES_FCMD:
break;
case IBT_ACHAN_CLONE:
case 0:
break;
default:
return (IBT_INVALID_PARAM);
}
if (flags & IBT_ACHAN_DEFER_ALLOC)
if (flags & IBT_ACHAN_USES_SRQ) {
"NULL SRQ Handle specified.");
return (IBT_INVALID_PARAM);
}
if (flags & IBT_ACHAN_USES_RSS) {
"SRQ not allowed with RSS.");
return (IBT_INVALID_PARAM);
}
}
/*
* Check if this request is to clone the channel, or to allocate a
* fresh one.
*/
if (flags & IBT_ACHAN_CLONE) {
"Clone Channel info not available.");
return (IBT_INVALID_PARAM);
"Clone Channel and HCA Handle mismatch");
return (IBT_INVALID_PARAM);
}
if (retval != IBT_SUCCESS) {
"Failed to Query the source channel: %d", retval);
return (retval);
}
/* Setup QP alloc attributes. */
sizeof (ibt_chan_sizes_t));
} else {
/* Setup QP alloc attributes. */
sizeof (ibt_chan_sizes_t));
/* Validate input hca_port_num and pkey_ix values. */
"ibt_index2pkey failed, status: %d", retval);
return (retval);
}
}
/* Allocate Channel and Initialize the channel. */
&chanp);
if (retval != IBT_SUCCESS) {
"Failed to allocate QP: %d", retval);
return (retval);
}
/* Initialize UD Channel by transitioning it to RTS State. */
if (retval != IBT_SUCCESS) {
"Failed to Initialize QP: %d", retval);
/* Free the QP as we failed to initialize it. */
(void) ibt_free_qp(chanp);
return (retval);
}
return (IBT_SUCCESS);
}
/*
* Function:
* ibt_alloc_ud_channel_range
* Input:
* hca_hdl HCA Handle.
* log2 Log (base 2) of the number of QPs to allocate.
* flags Channel allocate flags.
* args A pointer to an ibt_ud_chan_alloc_args_t struct that
* specifies required channel attributes.
* send_cq A pointer to an array of CQ handles.
* recv_cq A pointer to an array of CQ handles.
* Output:
* base_qpn_p The returned QP number of the base QP.
* ud_chan_p The returned UD Channel handle.
* sizes NULL or a pointer to ibt_chan_sizes_s struct where
* Returns:
* IBT_SUCCESS
* IBT_INVALID_PARAM
* Description:
* Allocate UD channels that satisfy the specified channel attributes.
*/
{
int i, n = 1 << log2;
if (flags & IBT_ACHAN_CLONE)
return (IBT_INVALID_PARAM);
if (flags & IBT_ACHAN_USER_MAP)
if (flags & IBT_ACHAN_DEFER_ALLOC)
if (flags & IBT_ACHAN_USES_SRQ) {
"NULL SRQ Handle specified.");
return (IBT_INVALID_PARAM);
}
}
if (flags & IBT_ACHAN_USES_FEXCH) {
}
if (flags & IBT_ACHAN_USES_RSS) {
if (log2 >
return (IBT_INSUFF_RESOURCE);
}
for (i = 0; i < 1 << log2; i++) {
ibt_cq_hdl = send_cq[i];
ibt_cq_hdl = recv_cq[i];
}
/* Setup QP alloc attributes. */
sizeof (ibt_chan_sizes_t));
/* Validate input hca_port_num and pkey_ix values. */
" ibt_index2pkey failed, status: %d", retval);
goto fail;
}
/* Allocate Channel and Initialize the channel. */
if (retval != IBT_SUCCESS) {
"Failed to allocate QPs: %d", retval);
goto fail;
}
/* Initialize UD Channel by transitioning it to RTS State. */
for (i = 0; i < n; i++) {
/* Initialize the internal QP struct. */
if (retval != IBT_SUCCESS) {
int j;
" Failed to Initialize QP: %d", retval);
/* Free the QP as we failed to initialize it. */
(void) ibt_free_qp(chanp);
for (j = 0; j < i; j++) {
(void) ibt_free_qp(chanp);
}
goto fail;
}
/*
* The IBTA spec does not include the signal type or PD on a QP
* query operation. In order to implement the "CLONE" feature
* we need to cache these values.
*/
}
fail:
if (retval != IBT_SUCCESS) {
for (i = 0; i < 1 << log2; i++) {
}
"failed: %d", retval);
}
return (retval);
}
/*
* Function:
* ibt_query_ud_channel
* Input:
* ud_chan A previously allocated UD channel handle.
* Output:
* chan_attrs Channel's current attributes.
* Returns:
* IBT_SUCCESS
* Description:
* Query a UD channel's attributes.
*/
{
return (IBT_CHAN_SRV_TYPE_INVALID);
}
/* Query the channel (QP) */
if (retval != IBT_SUCCESS) {
return (retval);
}
return (retval);
}
/*
* Function:
* ibt_modify_ud_channel
* Input:
* ud_chan A previously allocated UD channel handle.
* flags Specifies which attributes in ibt_ud_chan_modify_attr_t
* are to be modified.
* attrs Attributes to be modified.
* Output:
* actual_sz On return contains the new send and receive queue sizes.
* Returns:
* IBT_SUCCESS
* Description:
* Modifies an UD channel's attributes, as specified by a
* ibt_cep_modify_flags_t parameter to those specified in the
* ibt_ud_chan_modify_attr_t structure.
*/
{
int retries = 1;
return (IBT_CHAN_SRV_TYPE_INVALID);
}
if (flags & ~good_flags) {
"Invalid Modify Flags: %x", flags);
return (IBT_INVALID_PARAM);
}
if (retval != IBT_SUCCESS) {
/* give it one more shot if the old current state was stale */
--retries >= 0 &&
goto retry;
}
return (retval);
}
/*
* Function:
* ibt_recover_ud_channel
* Input:
* ud_chan An UD channel handle which is in SQError state.
* Output:
* none.
* Returns:
* IBT_SUCCESS
* IBT_CHAN_HDL_INVALID
* IBT_CHAN_SRV_TYPE_INVALID
* IBT_CHAN_STATE_INVALID
* Description:
* Recover an UD Channel which has transitioned to SQ Error state. The
* ibt_recover_ud_channel() transitions the channel from SQ Error state
* to Ready-To-Send channel state.
*
* If a work request posted to a UD channel's send queue completes with
* an error (see ibt_wc_status_t), the channel gets transitioned to SQ
* Error state. In order to reuse this channel, ibt_recover_ud_channel()
* can be used to recover the channel to a usable (Ready-to-Send) state.
*/
{
return (IBT_CHAN_SRV_TYPE_INVALID);
}
/* Set the channel state to RTS, to activate the send processing. */
if (retval != IBT_SUCCESS)
"ibt_modify_qp failed on qp %p: status = %d",
return (retval);
}
/*
* Function:
* ibt_flush_channel
* Input:
* chan The opaque channel handle returned in a previous call
* to ibt_alloc_ud_channel() or ibt_alloc_rc_channel().
* Output:
* none.
* Returns:
* IBT_SUCCESS
* Description:
* Flush the specified channel. Outstanding work requests are flushed
* so that the client can do the associated clean up. After that, the
* client will usually deregister the previously registered memory,
* then free the channel by calling ibt_free_channel(). This function
* applies to UD channels, or to RC channels that have not successfully
* been opened.
*/
{
if (retval != IBT_SUCCESS) {
}
return (retval);
}
/*
* Function:
* ibt_free_channel
* Input:
* chan The opaque channel handle returned in a previous
* call to ibt_alloc_{ud,rc}_channel().
* Output:
* none.
* Returns:
* IBT_SUCCESS
* Description:
* Releases the resources associated with the specified channel.
* It is well assumed that channel has been closed before this.
*/
{
return (ibt_free_qp(chan));
}
/*
* UD Destination.
*/
/*
* Function:
* ibt_alloc_ud_dest
* Input:
* hca_hdl HCA Handle.
* pd Protection Domain
* Output:
* ud_dest_p Address to store the returned UD destination handle.
* Returns:
* IBT_SUCCESS
* Description:
* Allocate a UD destination handle. The returned UD destination handle
* has no useful contents, but is usable after calling ibt_modify_ud_dest,
* ibt_modify_reply_ud_dest, or ibt_open_ud_dest.
*/
{
if (retval != IBT_SUCCESS) {
"Address Handle Allocation failed: %d", retval);
return (retval);
}
ud_destp->ud_dst_qpn = 0;
return (IBT_SUCCESS);
}
/*
* Function:
* ibt_query_ud_dest
* Input:
* ud_dest A previously allocated UD destination handle.
* Output:
* dest_attrs UD destination's current attributes.
* Returns:
* IBT_SUCCESS
* Description:
* Query a UD destination's attributes.
*/
{
/* Query Address Handle */
if (retval != IBT_SUCCESS) {
"Failed to Query Address Handle: %d", retval);
return (retval);
}
/* Update the return struct. */
return (retval);
}
/*
* Function:
* ibt_modify_ud_dest
* Input:
* ud_dest A previously allocated UD destination handle
* as returned by ibt_alloc_ud_dest().
* qkey QKey of the destination.
* dest_qpn QPN of the destination.
* adds_vect NULL or Address Vector for the destination.
*
* Output:
* none.
* Returns:
* IBT_SUCCESS
* Description:
* Modify a previously allocated UD destination handle from the
* arguments supplied by the caller.
*/
{
adds_vect)) != IBT_SUCCESS) {
"ibt_modify_ah() failed: status = %d", retval);
return (retval);
}
return (IBT_SUCCESS);
}
/*
* Function:
* ibt_free_ud_dest
* Input:
* ud_dest The opaque destination handle returned in a previous
* call to ibt_alloc_ud_dest() or ibt_alloc_mcg_dest().
* Output:
* none.
* Returns:
* IBT_SUCCESS
* Description:
* Releases the resources associated with the specified destination
* handle.
*/
{
if (retval != IBT_SUCCESS) {
"Address Handle free failed: %d", retval);
return (retval);
}
return (IBT_SUCCESS);
}
static ibt_status_t
{
uint_t i;
*sgid_ix_p = 0;
return (IBT_INVALID_PARAM);
}
for (i = 0; i < sgid_tbl_sz; i++, sgidp++) {
continue;
*sgid_ix_p = i;
return (IBT_SUCCESS);
}
*sgid_ix_p = 0;
return (IBT_INVALID_PARAM);
}
/*
* Function:
* ibt_modify_reply_ud_dest
* Input:
* ud_dest A previously allocated UD reply destination handle
* as returned by ibt_alloc_ud_dest().
* qkey Qkey. 0 means "not specified", so use the Q_Key
* in the QP context.
* recv_buf Pointer to the first data buffer associated with the
* receive work request.
* Output:
* Returns:
* IBT_SUCCESS
* Description:
* Modify a previously allocated UD destination handle, so that it
* can be used to reply to the sender of the datagram contained in the
* specified work request completion. If the qkey is not supplied (0),
* then use the qkey in the QP (we just set qkey to a privileged QKEY).
*/
{
"type of channel (%d) is not UD",
return (IBT_CHAN_SRV_TYPE_INVALID);
}
if (qkey == 0)
} else {
adds_vect.av_sgid_ix = 0;
}
&adds_vect)) != IBT_SUCCESS) {
"ibt_alloc_ah() failed: status = %d", retval);
return (retval);
}
return (IBT_SUCCESS);
}
/*
* Function:
* ibt_is_privileged_ud_dest
* Input:
* ud_dest A previously allocated destination handle.
* Output:
* none
* Returns:
* Description:
* Determine if a UD destination Handle is a privileged handle.
*/
{
}
/*
* Function:
* ibt_update_channel_qkey
* Input:
* ud_chan The UD channel handle, that is to be used to
* communicate with the specified destination.
*
* ud_dest A UD destination handle returned from
* ibt_alloc_ud_dest(9F).
* Output:
* none
* Returns:
* IBT_SUCCESS
* Description:
* ibt_update_channel_qkey() sets the Q_Key in the specified channel context
* to the Q_Key in the specified destination handle. This function can be used
* to enable sends to a privileged destination. All posted send work requests
* that contain a privileged destination handle now use the Q_Key in the
* channel context.
*
* ibt_update_channel_qkey() can also be used to enable the caller to receive
* from the specified remote destination on the specified channel.
*/
{
"type of channel (%d) is not UD",
return (IBT_CHAN_SRV_TYPE_INVALID);
}
if (retval != IBT_SUCCESS) {
} else {
}
return (retval);
}
/*
* Function:
* ibt_set_chan_private
* Input:
* chan A previously allocated channel handle.
* clnt_private The client private data.
* Output:
* none.
* Returns:
* none.
* Description:
* Set the client private data.
*/
void
{
}
/*
* Function:
* ibt_get_chan_private
* Input:
* chan A previously allocated channel handle.
* Output:
* A pointer to the client private data.
* Returns:
* none.
* Description:
* Get a pointer to client private data.
*/
void *
{
return (chan->ch_clnt_private);
}
/*
* Function:
* ibt_channel_to_hca_guid
* Input:
* chan Channel Handle.
* Output:
* none.
* Returns:
* hca_guid Returned HCA GUID on which the specified Channel is
* allocated. Valid if it is non-NULL on return.
* Description:
* A helper function to retrieve HCA GUID for the specified Channel.
*/
{
}
/*
* Protection Domain Verbs Functions.
*/
/*
* Function:
* ibt_alloc_pd
* Input:
* hca_hdl The IBT HCA handle, the device on which we need
* to create the requested Protection Domain.
* flags IBT_PD_NO_FLAGS, IBT_PD_USER_MAP or IBT_PD_DEFER_ALLOC
* Output:
* pd IBT Protection Domain Handle.
* Returns:
* IBT_SUCCESS
* IBT_HCA_HDL_INVALID
* Description:
* Allocate a Protection Domain.
*/
{
/* re-direct the call to CI's call */
if (retval != IBT_SUCCESS) {
return (retval);
}
/* Update the PDs Resource Count per HCA Device. */
return (retval);
}
/*
* Function:
* ibt_free_pd
* Input:
* hca_hdl The IBT HCA handle, the device on which we need
* to free the requested Protection Domain.
* pd IBT Protection Domain Handle.
* Output:
* none.
* Returns:
* IBT_SUCCESS
* IBT_HCA_HDL_INVALID
* IBT_MEM_PD_HDL_INVALID
* IBT_MEM_PD_IN_USE
* Description:
* Release/de-allocate a Protection Domain.
*/
{
/* re-direct the call to CI's call */
if (retval != IBT_SUCCESS) {
return (retval);
}
/* Update the PDs Resource Count per HCA Device. */
return (retval);
}
/*
* Address Handle Verbs Functions.
*/
/*
* Function:
* ibt_alloc_ah
* Input:
* hca_hdl The IBT HCA Handle.
* pd The IBT Protection Domain to associate with this handle.
* adds_vectp Points to an ibt_adds_vect_t struct.
* Output:
* ah IBT Address Handle.
* Returns:
* IBT_SUCCESS
* IBT_HCA_HDL_INVALID
* IBT_INSUFF_RESOURCE
* IBT_MEM_PD_HDL_INVALID
* Description:
* Allocate and returns an Address Handle.
*/
{
/* XXX - if av_send_grh, need to compute av_sgid_ix from av_sgid */
/* re-direct the call to CI's call */
if (retval != IBT_SUCCESS) {
"ibc_alloc_ah failed: status = %d", retval);
return (retval);
}
/* Update the AHs Resource Count per HCA Device. */
return (retval);
}
/*
* Function:
* ibt_free_ah
* Input:
* hca_hdl The IBT HCA Handle.
* ah IBT Address Handle.
* Output:
* none.
* Returns:
* IBT_SUCCESS
* IBT_HCA_HDL_INVALID
* IBT_AH_HDL_INVALID
* Description:
* Release/de-allocate the specified Address Handle.
*/
{
/* re-direct the call to CI's call */
if (retval != IBT_SUCCESS) {
return (retval);
}
/* Update the AHs Resource Count per HCA Device. */
return (retval);
}
/*
* Function:
* ibt_query_ah
* Input:
* hca_hdl The IBT HCA Handle.
* ah IBT Address Handle.
* Output:
* pd The Protection Domain Handle with which this
* Address Handle is associated.
* adds_vectp Points to an ibt_adds_vect_t struct.
* Returns:
* Description:
* Obtain the address vector information for the specified address handle.
*/
{
/* re-direct the call to CI's call */
/*
* av_sgid_ix.
*/
if (retval == IBT_SUCCESS) {
} else {
}
}
return (retval);
}
/*
* Function:
* ibt_modify_ah
* Input:
* hca_hdl The IBT HCA Handle.
* ah IBT Address Handle.
* Output:
* adds_vectp Points to an ibt_adds_vect_t struct. The new address
* vector information is specified is returned in this
* structure.
* Returns:
* Description:
* Modify the address vector information for the specified Address Handle.
*/
{
/* XXX - if av_send_grh, need to compute av_sgid_ix from av_sgid */
/* re-direct the call to CI's call */
}