/*
* CDDL HEADER START
*
* The contents of this file are subject to the terms of the
* Common Development and Distribution License (the "License").
* You may not use this file except in compliance with the License.
*
* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
* or http://www.opensolaris.org/os/licensing.
* See the License for the specific language governing permissions
* and limitations under the License.
*
* When distributing Covered Code, include this CDDL HEADER in each
* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
* If applicable, add the following below this CDDL HEADER, with the
* fields enclosed by brackets "[]" replaced with your own identifying
* information: Portions Copyright [yyyy] [name of copyright owner]
*
* CDDL HEADER END
*/
/*
* Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved.
*/
/*
* hermon_ci.c
* Hermon Channel Interface (CI) Routines
*
* Implements all the routines necessary to interface with the IBTF.
* Pointers to all of these functions are passed to the IBTF at attach()
* time in the ibc_operations_t structure. These functions include all
* of the necessary routines to implement the required InfiniBand "verbs"
* and additional IBTF-specific interfaces.
*/
#include <sys/types.h>
#include <sys/conf.h>
#include <sys/ddi.h>
#include <sys/sunddi.h>
#include <sys/ib/adapters/hermon/hermon.h>
extern uint32_t hermon_kernel_data_ro;
extern uint32_t hermon_user_data_ro;
/* HCA and port related operations */
static ibt_status_t hermon_ci_query_hca_ports(ibc_hca_hdl_t, uint8_t,
ibt_hca_portinfo_t *);
static ibt_status_t hermon_ci_modify_ports(ibc_hca_hdl_t, uint8_t,
ibt_port_modify_flags_t, uint8_t);
static ibt_status_t hermon_ci_modify_system_image(ibc_hca_hdl_t, ib_guid_t);
/* Protection Domains */
static ibt_status_t hermon_ci_alloc_pd(ibc_hca_hdl_t, ibt_pd_flags_t,
ibc_pd_hdl_t *);
static ibt_status_t hermon_ci_free_pd(ibc_hca_hdl_t, ibc_pd_hdl_t);
/* Reliable Datagram Domains */
static ibt_status_t hermon_ci_alloc_rdd(ibc_hca_hdl_t, ibc_rdd_flags_t,
ibc_rdd_hdl_t *);
static ibt_status_t hermon_ci_free_rdd(ibc_hca_hdl_t, ibc_rdd_hdl_t);
/* Address Handles */
static ibt_status_t hermon_ci_alloc_ah(ibc_hca_hdl_t, ibt_ah_flags_t,
ibc_pd_hdl_t, ibt_adds_vect_t *, ibc_ah_hdl_t *);
static ibt_status_t hermon_ci_free_ah(ibc_hca_hdl_t, ibc_ah_hdl_t);
static ibt_status_t hermon_ci_query_ah(ibc_hca_hdl_t, ibc_ah_hdl_t,
ibc_pd_hdl_t *, ibt_adds_vect_t *);
static ibt_status_t hermon_ci_modify_ah(ibc_hca_hdl_t, ibc_ah_hdl_t,
ibt_adds_vect_t *);
/* Queue Pairs */
static ibt_status_t hermon_ci_alloc_qp(ibc_hca_hdl_t, ibtl_qp_hdl_t,
ibt_qp_type_t, ibt_qp_alloc_attr_t *, ibt_chan_sizes_t *, ib_qpn_t *,
ibc_qp_hdl_t *);
static ibt_status_t hermon_ci_alloc_special_qp(ibc_hca_hdl_t, uint8_t,
ibtl_qp_hdl_t, ibt_sqp_type_t, ibt_qp_alloc_attr_t *,
ibt_chan_sizes_t *, ibc_qp_hdl_t *);
static ibt_status_t hermon_ci_alloc_qp_range(ibc_hca_hdl_t, uint_t,
ibtl_qp_hdl_t *, ibt_qp_type_t, ibt_qp_alloc_attr_t *, ibt_chan_sizes_t *,
ibc_cq_hdl_t *, ibc_cq_hdl_t *, ib_qpn_t *, ibc_qp_hdl_t *);
static ibt_status_t hermon_ci_free_qp(ibc_hca_hdl_t, ibc_qp_hdl_t,
ibc_free_qp_flags_t, ibc_qpn_hdl_t *);
static ibt_status_t hermon_ci_release_qpn(ibc_hca_hdl_t, ibc_qpn_hdl_t);
static ibt_status_t hermon_ci_query_qp(ibc_hca_hdl_t, ibc_qp_hdl_t,
ibt_qp_query_attr_t *);
static ibt_status_t hermon_ci_modify_qp(ibc_hca_hdl_t, ibc_qp_hdl_t,
ibt_cep_modify_flags_t, ibt_qp_info_t *, ibt_queue_sizes_t *);
/* Completion Queues */
static ibt_status_t hermon_ci_alloc_cq(ibc_hca_hdl_t, ibt_cq_hdl_t,
ibt_cq_attr_t *, ibc_cq_hdl_t *, uint_t *);
static ibt_status_t hermon_ci_free_cq(ibc_hca_hdl_t, ibc_cq_hdl_t);
static ibt_status_t hermon_ci_query_cq(ibc_hca_hdl_t, ibc_cq_hdl_t,
uint_t *, uint_t *, uint_t *, ibt_cq_handler_id_t *);
static ibt_status_t hermon_ci_resize_cq(ibc_hca_hdl_t, ibc_cq_hdl_t,
uint_t, uint_t *);
static ibt_status_t hermon_ci_modify_cq(ibc_hca_hdl_t, ibc_cq_hdl_t,
uint_t, uint_t, ibt_cq_handler_id_t);
static ibt_status_t hermon_ci_alloc_cq_sched(ibc_hca_hdl_t,
ibt_cq_sched_attr_t *, ibc_sched_hdl_t *);
static ibt_status_t hermon_ci_free_cq_sched(ibc_hca_hdl_t, ibc_sched_hdl_t);
static ibt_status_t hermon_ci_query_cq_handler_id(ibc_hca_hdl_t,
ibt_cq_handler_id_t, ibt_cq_handler_attr_t *);
/* EE Contexts */
static ibt_status_t hermon_ci_alloc_eec(ibc_hca_hdl_t, ibc_eec_flags_t,
ibt_eec_hdl_t, ibc_rdd_hdl_t, ibc_eec_hdl_t *);
static ibt_status_t hermon_ci_free_eec(ibc_hca_hdl_t, ibc_eec_hdl_t);
static ibt_status_t hermon_ci_query_eec(ibc_hca_hdl_t, ibc_eec_hdl_t,
ibt_eec_query_attr_t *);
static ibt_status_t hermon_ci_modify_eec(ibc_hca_hdl_t, ibc_eec_hdl_t,
ibt_cep_modify_flags_t, ibt_eec_info_t *);
/* Memory Registration */
static ibt_status_t hermon_ci_register_mr(ibc_hca_hdl_t, ibc_pd_hdl_t,
ibt_mr_attr_t *, void *, ibc_mr_hdl_t *, ibt_mr_desc_t *);
static ibt_status_t hermon_ci_register_buf(ibc_hca_hdl_t, ibc_pd_hdl_t,
ibt_smr_attr_t *, struct buf *, void *, ibt_mr_hdl_t *, ibt_mr_desc_t *);
static ibt_status_t hermon_ci_register_shared_mr(ibc_hca_hdl_t,
ibc_mr_hdl_t, ibc_pd_hdl_t, ibt_smr_attr_t *, void *,
ibc_mr_hdl_t *, ibt_mr_desc_t *);
static ibt_status_t hermon_ci_deregister_mr(ibc_hca_hdl_t, ibc_mr_hdl_t);
static ibt_status_t hermon_ci_query_mr(ibc_hca_hdl_t, ibc_mr_hdl_t,
ibt_mr_query_attr_t *);
static ibt_status_t hermon_ci_reregister_mr(ibc_hca_hdl_t, ibc_mr_hdl_t,
ibc_pd_hdl_t, ibt_mr_attr_t *, void *, ibc_mr_hdl_t *,
ibt_mr_desc_t *);
static ibt_status_t hermon_ci_reregister_buf(ibc_hca_hdl_t, ibc_mr_hdl_t,
ibc_pd_hdl_t, ibt_smr_attr_t *, struct buf *, void *, ibc_mr_hdl_t *,
ibt_mr_desc_t *);
static ibt_status_t hermon_ci_sync_mr(ibc_hca_hdl_t, ibt_mr_sync_t *, size_t);
static ibt_status_t hermon_ci_register_dma_mr(ibc_hca_hdl_t, ibc_pd_hdl_t,
ibt_dmr_attr_t *, void *, ibc_mr_hdl_t *, ibt_mr_desc_t *);
/* Memory Windows */
static ibt_status_t hermon_ci_alloc_mw(ibc_hca_hdl_t, ibc_pd_hdl_t,
ibt_mw_flags_t, ibc_mw_hdl_t *, ibt_rkey_t *);
static ibt_status_t hermon_ci_free_mw(ibc_hca_hdl_t, ibc_mw_hdl_t);
static ibt_status_t hermon_ci_query_mw(ibc_hca_hdl_t, ibc_mw_hdl_t,
ibt_mw_query_attr_t *);
/* Multicast Groups */
static ibt_status_t hermon_ci_attach_mcg(ibc_hca_hdl_t, ibc_qp_hdl_t,
ib_gid_t, ib_lid_t);
static ibt_status_t hermon_ci_detach_mcg(ibc_hca_hdl_t, ibc_qp_hdl_t,
ib_gid_t, ib_lid_t);
/* Work Request and Completion Processing */
static ibt_status_t hermon_ci_post_send(ibc_hca_hdl_t, ibc_qp_hdl_t,
ibt_send_wr_t *, uint_t, uint_t *);
static ibt_status_t hermon_ci_post_recv(ibc_hca_hdl_t, ibc_qp_hdl_t,
ibt_recv_wr_t *, uint_t, uint_t *);
static ibt_status_t hermon_ci_poll_cq(ibc_hca_hdl_t, ibc_cq_hdl_t,
ibt_wc_t *, uint_t, uint_t *);
static ibt_status_t hermon_ci_notify_cq(ibc_hca_hdl_t, ibc_cq_hdl_t,
ibt_cq_notify_flags_t);
/* CI Object Private Data */
static ibt_status_t hermon_ci_ci_data_in(ibc_hca_hdl_t, ibt_ci_data_flags_t,
ibt_object_type_t, void *, void *, size_t);
/* CI Object Private Data */
static ibt_status_t hermon_ci_ci_data_out(ibc_hca_hdl_t, ibt_ci_data_flags_t,
ibt_object_type_t, void *, void *, size_t);
/* Shared Receive Queues */
static ibt_status_t hermon_ci_alloc_srq(ibc_hca_hdl_t, ibt_srq_flags_t,
ibt_srq_hdl_t, ibc_pd_hdl_t, ibt_srq_sizes_t *, ibc_srq_hdl_t *,
ibt_srq_sizes_t *);
static ibt_status_t hermon_ci_free_srq(ibc_hca_hdl_t, ibc_srq_hdl_t);
static ibt_status_t hermon_ci_query_srq(ibc_hca_hdl_t, ibc_srq_hdl_t,
ibc_pd_hdl_t *, ibt_srq_sizes_t *, uint_t *);
static ibt_status_t hermon_ci_modify_srq(ibc_hca_hdl_t, ibc_srq_hdl_t,
ibt_srq_modify_flags_t, uint_t, uint_t, uint_t *);
static ibt_status_t hermon_ci_post_srq(ibc_hca_hdl_t, ibc_srq_hdl_t,
ibt_recv_wr_t *, uint_t, uint_t *);
/* Address translation */
static ibt_status_t hermon_ci_map_mem_area(ibc_hca_hdl_t, ibt_va_attr_t *,
void *, uint_t, ibt_reg_req_t *, ibc_ma_hdl_t *);
static ibt_status_t hermon_ci_unmap_mem_area(ibc_hca_hdl_t, ibc_ma_hdl_t);
static ibt_status_t hermon_ci_map_mem_iov(ibc_hca_hdl_t, ibt_iov_attr_t *,
ibt_all_wr_t *, ibc_mi_hdl_t *);
static ibt_status_t hermon_ci_unmap_mem_iov(ibc_hca_hdl_t, ibc_mi_hdl_t);
/* Allocate L_Key */
static ibt_status_t hermon_ci_alloc_lkey(ibc_hca_hdl_t, ibc_pd_hdl_t,
ibt_lkey_flags_t, uint_t, ibc_mr_hdl_t *, ibt_pmr_desc_t *);
/* Physical Register Memory Region */
static ibt_status_t hermon_ci_register_physical_mr(ibc_hca_hdl_t, ibc_pd_hdl_t,
ibt_pmr_attr_t *, void *, ibc_mr_hdl_t *, ibt_pmr_desc_t *);
static ibt_status_t hermon_ci_reregister_physical_mr(ibc_hca_hdl_t,
ibc_mr_hdl_t, ibc_pd_hdl_t, ibt_pmr_attr_t *, void *, ibc_mr_hdl_t *,
ibt_pmr_desc_t *);
/* Mellanox FMR */
static ibt_status_t hermon_ci_create_fmr_pool(ibc_hca_hdl_t hca,
ibc_pd_hdl_t pd, ibt_fmr_pool_attr_t *fmr_params,
ibc_fmr_pool_hdl_t *fmr_pool);
static ibt_status_t hermon_ci_destroy_fmr_pool(ibc_hca_hdl_t hca,
ibc_fmr_pool_hdl_t fmr_pool);
static ibt_status_t hermon_ci_flush_fmr_pool(ibc_hca_hdl_t hca,
ibc_fmr_pool_hdl_t fmr_pool);
static ibt_status_t hermon_ci_register_physical_fmr(ibc_hca_hdl_t hca,
ibc_fmr_pool_hdl_t fmr_pool, ibt_pmr_attr_t *mem_pattr,
void *ibtl_reserved, ibc_mr_hdl_t *mr_hdl_p, ibt_pmr_desc_t *mem_desc_p);
static ibt_status_t hermon_ci_deregister_fmr(ibc_hca_hdl_t hca,
ibc_mr_hdl_t mr);
/* Memory Allocation/Deallocation */
static ibt_status_t hermon_ci_alloc_io_mem(ibc_hca_hdl_t hca, size_t size,
ibt_mr_flags_t mr_flag, caddr_t *kaddrp,
ibc_mem_alloc_hdl_t *mem_alloc_hdl_p);
static ibt_status_t hermon_ci_free_io_mem(ibc_hca_hdl_t hca,
ibc_mem_alloc_hdl_t mem_alloc_hdl);
static ibt_status_t hermon_ci_not_supported();
/*
* This ibc_operations_t structure includes pointers to all the entry points
* provided by the Hermon driver. This structure is passed to the IBTF at
* driver attach time, using the ibc_attach() call.
*/
ibc_operations_t hermon_ibc_ops = {
/* HCA and port related operations */
hermon_ci_query_hca_ports,
hermon_ci_modify_ports,
hermon_ci_modify_system_image,
/* Protection Domains */
hermon_ci_alloc_pd,
hermon_ci_free_pd,
/* Reliable Datagram Domains */
hermon_ci_alloc_rdd,
hermon_ci_free_rdd,
/* Address Handles */
hermon_ci_alloc_ah,
hermon_ci_free_ah,
hermon_ci_query_ah,
hermon_ci_modify_ah,
/* Queue Pairs */
hermon_ci_alloc_qp,
hermon_ci_alloc_special_qp,
hermon_ci_alloc_qp_range,
hermon_ci_free_qp,
hermon_ci_release_qpn,
hermon_ci_query_qp,
hermon_ci_modify_qp,
/* Completion Queues */
hermon_ci_alloc_cq,
hermon_ci_free_cq,
hermon_ci_query_cq,
hermon_ci_resize_cq,
hermon_ci_modify_cq,
hermon_ci_alloc_cq_sched,
hermon_ci_free_cq_sched,
hermon_ci_query_cq_handler_id,
/* EE Contexts */
hermon_ci_alloc_eec,
hermon_ci_free_eec,
hermon_ci_query_eec,
hermon_ci_modify_eec,
/* Memory Registration */
hermon_ci_register_mr,
hermon_ci_register_buf,
hermon_ci_register_shared_mr,
hermon_ci_deregister_mr,
hermon_ci_query_mr,
hermon_ci_reregister_mr,
hermon_ci_reregister_buf,
hermon_ci_sync_mr,
/* Memory Windows */
hermon_ci_alloc_mw,
hermon_ci_free_mw,
hermon_ci_query_mw,
/* Multicast Groups */
hermon_ci_attach_mcg,
hermon_ci_detach_mcg,
/* Work Request and Completion Processing */
hermon_ci_post_send,
hermon_ci_post_recv,
hermon_ci_poll_cq,
hermon_ci_notify_cq,
/* CI Object Mapping Data */
hermon_ci_ci_data_in,
hermon_ci_ci_data_out,
/* Shared Receive Queue */
hermon_ci_alloc_srq,
hermon_ci_free_srq,
hermon_ci_query_srq,
hermon_ci_modify_srq,
hermon_ci_post_srq,
/* Address translation */
hermon_ci_map_mem_area,
hermon_ci_unmap_mem_area,
hermon_ci_map_mem_iov,
hermon_ci_unmap_mem_iov,
/* Allocate L_key */
hermon_ci_alloc_lkey,
/* Physical Register Memory Region */
hermon_ci_register_physical_mr,
hermon_ci_reregister_physical_mr,
/* Mellanox FMR */
hermon_ci_create_fmr_pool,
hermon_ci_destroy_fmr_pool,
hermon_ci_flush_fmr_pool,
hermon_ci_register_physical_fmr,
hermon_ci_deregister_fmr,
/* Memory allocation */
hermon_ci_alloc_io_mem,
hermon_ci_free_io_mem,
/* XRC not yet supported */
hermon_ci_not_supported, /* ibc_alloc_xrc_domain */
hermon_ci_not_supported, /* ibc_free_xrc_domain */
hermon_ci_not_supported, /* ibc_alloc_xrc_srq */
hermon_ci_not_supported, /* ibc_free_xrc_srq */
hermon_ci_not_supported, /* ibc_query_xrc_srq */
hermon_ci_not_supported, /* ibc_modify_xrc_srq */
hermon_ci_not_supported, /* ibc_alloc_xrc_tgt_qp */
hermon_ci_not_supported, /* ibc_free_xrc_tgt_qp */
hermon_ci_not_supported, /* ibc_query_xrc_tgt_qp */
hermon_ci_not_supported, /* ibc_modify_xrc_tgt_qp */
/* Memory Region (physical) */
hermon_ci_register_dma_mr,
/* Next enhancements */
hermon_ci_not_supported, /* ibc_enhancement1 */
hermon_ci_not_supported, /* ibc_enhancement2 */
hermon_ci_not_supported, /* ibc_enhancement3 */
hermon_ci_not_supported, /* ibc_enhancement4 */
};
/*
* Not yet implemented OPS
*/
/* ARGSUSED */
static ibt_status_t
hermon_ci_not_supported()
{
return (IBT_NOT_SUPPORTED);
}
/*
* hermon_ci_query_hca_ports()
* Returns HCA port attributes for either one or all of the HCA's ports.
* Context: Can be called only from user or kernel context.
*/
static ibt_status_t
hermon_ci_query_hca_ports(ibc_hca_hdl_t hca, uint8_t query_port,
ibt_hca_portinfo_t *info_p)
{
hermon_state_t *state;
uint_t start, end, port;
int status, indx;
/* Grab the Hermon softstate pointer */
state = (hermon_state_t *)hca;
/*
* If the specified port is zero, then we are supposed to query all
* ports. Otherwise, we query only the port number specified.
* Setup the start and end port numbers as appropriate for the loop
* below. Note: The first Hermon port is port number one (1).
*/
if (query_port == 0) {
start = 1;
end = start + (state->hs_cfg_profile->cp_num_ports - 1);
} else {
end = start = query_port;
}
/* Query the port(s) */
for (port = start, indx = 0; port <= end; port++, indx++) {
status = hermon_port_query(state, port, &info_p[indx]);
if (status != DDI_SUCCESS) {
return (status);
}
}
return (IBT_SUCCESS);
}
/*
* hermon_ci_modify_ports()
* Modify HCA port attributes
* Context: Can be called only from user or kernel context.
*/
static ibt_status_t
hermon_ci_modify_ports(ibc_hca_hdl_t hca, uint8_t port,
ibt_port_modify_flags_t flags, uint8_t init_type)
{
hermon_state_t *state;
int status;
/* Grab the Hermon softstate pointer */
state = (hermon_state_t *)hca;
/* Modify the port(s) */
status = hermon_port_modify(state, port, flags, init_type);
return (status);
}
/*
* hermon_ci_modify_system_image()
* Modify the System Image GUID
* Context: Can be called only from user or kernel context.
*/
/* ARGSUSED */
static ibt_status_t
hermon_ci_modify_system_image(ibc_hca_hdl_t hca, ib_guid_t sys_guid)
{
/*
* This is an unsupported interface for the Hermon driver. This
* interface is necessary to support modification of the System
* Image GUID. Hermon is only capable of modifying this parameter
* once (during driver initialization).
*/
return (IBT_NOT_SUPPORTED);
}
/*
* hermon_ci_alloc_pd()
* Allocate a Protection Domain
* Context: Can be called only from user or kernel context.
*/
/* ARGSUSED */
static ibt_status_t
hermon_ci_alloc_pd(ibc_hca_hdl_t hca, ibt_pd_flags_t flags, ibc_pd_hdl_t *pd_p)
{
hermon_state_t *state;
hermon_pdhdl_t pdhdl;
int status;
ASSERT(pd_p != NULL);
/* Grab the Hermon softstate pointer */
state = (hermon_state_t *)hca;
/* Allocate the PD */
status = hermon_pd_alloc(state, &pdhdl, HERMON_NOSLEEP);
if (status != DDI_SUCCESS) {
return (status);
}
/* Return the Hermon PD handle */
*pd_p = (ibc_pd_hdl_t)pdhdl;
return (IBT_SUCCESS);
}
/*
* hermon_ci_free_pd()
* Free a Protection Domain
* Context: Can be called only from user or kernel context
*/
static ibt_status_t
hermon_ci_free_pd(ibc_hca_hdl_t hca, ibc_pd_hdl_t pd)
{
hermon_state_t *state;
hermon_pdhdl_t pdhdl;
int status;
/* Grab the Hermon softstate pointer and PD handle */
state = (hermon_state_t *)hca;
pdhdl = (hermon_pdhdl_t)pd;
/* Free the PD */
status = hermon_pd_free(state, &pdhdl);
return (status);
}
/*
* hermon_ci_alloc_rdd()
* Allocate a Reliable Datagram Domain
* Context: Can be called only from user or kernel context.
*/
/* ARGSUSED */
static ibt_status_t
hermon_ci_alloc_rdd(ibc_hca_hdl_t hca, ibc_rdd_flags_t flags,
ibc_rdd_hdl_t *rdd_p)
{
/*
* This is an unsupported interface for the Hermon driver. This
* interface is necessary to support Reliable Datagram (RD)
* operations. Hermon does not support RD.
*/
return (IBT_NOT_SUPPORTED);
}
/*
* hermon_free_rdd()
* Free a Reliable Datagram Domain
* Context: Can be called only from user or kernel context.
*/
/* ARGSUSED */
static ibt_status_t
hermon_ci_free_rdd(ibc_hca_hdl_t hca, ibc_rdd_hdl_t rdd)
{
/*
* This is an unsupported interface for the Hermon driver. This
* interface is necessary to support Reliable Datagram (RD)
* operations. Hermon does not support RD.
*/
return (IBT_NOT_SUPPORTED);
}
/*
* hermon_ci_alloc_ah()
* Allocate an Address Handle
* Context: Can be called only from user or kernel context.
*/
/* ARGSUSED */
static ibt_status_t
hermon_ci_alloc_ah(ibc_hca_hdl_t hca, ibt_ah_flags_t flags, ibc_pd_hdl_t pd,
ibt_adds_vect_t *attr_p, ibc_ah_hdl_t *ah_p)
{
hermon_state_t *state;
hermon_ahhdl_t ahhdl;
hermon_pdhdl_t pdhdl;
int status;
/* Grab the Hermon softstate pointer and PD handle */
state = (hermon_state_t *)hca;
pdhdl = (hermon_pdhdl_t)pd;
/* Allocate the AH */
status = hermon_ah_alloc(state, pdhdl, attr_p, &ahhdl, HERMON_NOSLEEP);
if (status != DDI_SUCCESS) {
return (status);
}
/* Return the Hermon AH handle */
*ah_p = (ibc_ah_hdl_t)ahhdl;
return (IBT_SUCCESS);
}
/*
* hermon_ci_free_ah()
* Free an Address Handle
* Context: Can be called only from user or kernel context.
*/
static ibt_status_t
hermon_ci_free_ah(ibc_hca_hdl_t hca, ibc_ah_hdl_t ah)
{
hermon_state_t *state;
hermon_ahhdl_t ahhdl;
int status;
/* Grab the Hermon softstate pointer and AH handle */
state = (hermon_state_t *)hca;
ahhdl = (hermon_ahhdl_t)ah;
/* Free the AH */
status = hermon_ah_free(state, &ahhdl, HERMON_NOSLEEP);
return (status);
}
/*
* hermon_ci_query_ah()
* Return the Address Vector information for a specified Address Handle
* Context: Can be called from interrupt or base context.
*/
static ibt_status_t
hermon_ci_query_ah(ibc_hca_hdl_t hca, ibc_ah_hdl_t ah, ibc_pd_hdl_t *pd_p,
ibt_adds_vect_t *attr_p)
{
hermon_state_t *state;
hermon_ahhdl_t ahhdl;
hermon_pdhdl_t pdhdl;
int status;
/* Grab the Hermon softstate pointer and AH handle */
state = (hermon_state_t *)hca;
ahhdl = (hermon_ahhdl_t)ah;
/* Query the AH */
status = hermon_ah_query(state, ahhdl, &pdhdl, attr_p);
if (status != DDI_SUCCESS) {
return (status);
}
/* Return the Hermon PD handle */
*pd_p = (ibc_pd_hdl_t)pdhdl;
return (IBT_SUCCESS);
}
/*
* hermon_ci_modify_ah()
* Modify the Address Vector information of a specified Address Handle
* Context: Can be called from interrupt or base context.
*/
static ibt_status_t
hermon_ci_modify_ah(ibc_hca_hdl_t hca, ibc_ah_hdl_t ah, ibt_adds_vect_t *attr_p)
{
hermon_state_t *state;
hermon_ahhdl_t ahhdl;
int status;
/* Grab the Hermon softstate pointer and AH handle */
state = (hermon_state_t *)hca;
ahhdl = (hermon_ahhdl_t)ah;
/* Modify the AH */
status = hermon_ah_modify(state, ahhdl, attr_p);
return (status);
}
/*
* hermon_ci_alloc_qp()
* Allocate a Queue Pair
* Context: Can be called only from user or kernel context.
*/
static ibt_status_t
hermon_ci_alloc_qp(ibc_hca_hdl_t hca, ibtl_qp_hdl_t ibt_qphdl,
ibt_qp_type_t type, ibt_qp_alloc_attr_t *attr_p,
ibt_chan_sizes_t *queue_sizes_p, ib_qpn_t *qpn, ibc_qp_hdl_t *qp_p)
{
hermon_state_t *state;
hermon_qp_info_t qpinfo;
int status;
_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*attr_p))
_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*queue_sizes_p))
/* Grab the Hermon softstate pointer */
state = (hermon_state_t *)hca;
/* Allocate the QP */
qpinfo.qpi_attrp = attr_p;
qpinfo.qpi_type = type;
qpinfo.qpi_ibt_qphdl = ibt_qphdl;
qpinfo.qpi_queueszp = queue_sizes_p;
qpinfo.qpi_qpn = qpn;
status = hermon_qp_alloc(state, &qpinfo, HERMON_NOSLEEP);
if (status != DDI_SUCCESS) {
return (status);
}
/* Return the Hermon QP handle */
*qp_p = (ibc_qp_hdl_t)qpinfo.qpi_qphdl;
return (IBT_SUCCESS);
}
/*
* hermon_ci_alloc_special_qp()
* Allocate a Special Queue Pair
* Context: Can be called only from user or kernel context.
*/
static ibt_status_t
hermon_ci_alloc_special_qp(ibc_hca_hdl_t hca, uint8_t port,
ibtl_qp_hdl_t ibt_qphdl, ibt_sqp_type_t type,
ibt_qp_alloc_attr_t *attr_p, ibt_chan_sizes_t *queue_sizes_p,
ibc_qp_hdl_t *qp_p)
{
hermon_state_t *state;
hermon_qp_info_t qpinfo;
int status;
_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*attr_p))
_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*queue_sizes_p))
/* Grab the Hermon softstate pointer */
state = (hermon_state_t *)hca;
/* Allocate the Special QP */
qpinfo.qpi_attrp = attr_p;
qpinfo.qpi_type = type;
qpinfo.qpi_port = port;
qpinfo.qpi_ibt_qphdl = ibt_qphdl;
qpinfo.qpi_queueszp = queue_sizes_p;
status = hermon_special_qp_alloc(state, &qpinfo, HERMON_NOSLEEP);
if (status != DDI_SUCCESS) {
return (status);
}
/* Return the Hermon QP handle */
*qp_p = (ibc_qp_hdl_t)qpinfo.qpi_qphdl;
return (IBT_SUCCESS);
}
/*
* hermon_ci_alloc_qp_range()
* Free a Queue Pair
* Context: Can be called only from user or kernel context.
*/
/* ARGSUSED */
static ibt_status_t
hermon_ci_alloc_qp_range(ibc_hca_hdl_t hca, uint_t log2,
ibtl_qp_hdl_t *ibtl_qp, ibt_qp_type_t type,
ibt_qp_alloc_attr_t *attr_p, ibt_chan_sizes_t *queue_sizes_p,
ibc_cq_hdl_t *send_cq, ibc_cq_hdl_t *recv_cq,
ib_qpn_t *qpn, ibc_qp_hdl_t *qp_p)
{
hermon_state_t *state;
hermon_qp_info_t qpinfo;
int status;
_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*attr_p))
_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*queue_sizes_p))
/* Grab the Hermon softstate pointer */
state = (hermon_state_t *)hca;
/* Allocate the QP */
qpinfo.qpi_attrp = attr_p;
qpinfo.qpi_type = type;
qpinfo.qpi_queueszp = queue_sizes_p;
qpinfo.qpi_qpn = qpn;
status = hermon_qp_alloc_range(state, log2, &qpinfo, ibtl_qp,
send_cq, recv_cq, (hermon_qphdl_t *)qp_p, HERMON_NOSLEEP);
return (status);
}
/*
* hermon_ci_free_qp()
* Free a Queue Pair
* Context: Can be called only from user or kernel context.
*/
static ibt_status_t
hermon_ci_free_qp(ibc_hca_hdl_t hca, ibc_qp_hdl_t qp,
ibc_free_qp_flags_t free_qp_flags, ibc_qpn_hdl_t *qpnh_p)
{
hermon_state_t *state;
hermon_qphdl_t qphdl;
int status;
/* Grab the Hermon softstate pointer and QP handle */
state = (hermon_state_t *)hca;
qphdl = (hermon_qphdl_t)qp;
/* Free the QP */
status = hermon_qp_free(state, &qphdl, free_qp_flags, qpnh_p,
HERMON_NOSLEEP);
return (status);
}
/*
* hermon_ci_release_qpn()
* Release a Queue Pair Number (QPN)
* Context: Can be called only from user or kernel context.
*/
static ibt_status_t
hermon_ci_release_qpn(ibc_hca_hdl_t hca, ibc_qpn_hdl_t qpnh)
{
hermon_state_t *state;
hermon_qpn_entry_t *entry;
/* Grab the Hermon softstate pointer and QP handle */
state = (hermon_state_t *)hca;
entry = (hermon_qpn_entry_t *)qpnh;
/* Release the QP number */
hermon_qp_release_qpn(state, entry, HERMON_QPN_RELEASE);
return (IBT_SUCCESS);
}
/*
* hermon_ci_query_qp()
* Query a Queue Pair
* Context: Can be called from interrupt or base context.
*/
static ibt_status_t
hermon_ci_query_qp(ibc_hca_hdl_t hca, ibc_qp_hdl_t qp,
ibt_qp_query_attr_t *attr_p)
{
hermon_state_t *state;
hermon_qphdl_t qphdl;
int status;
/* Grab the Hermon softstate pointer and QP handle */
state = (hermon_state_t *)hca;
qphdl = (hermon_qphdl_t)qp;
/* Query the QP */
status = hermon_qp_query(state, qphdl, attr_p);
return (status);
}
/*
* hermon_ci_modify_qp()
* Modify a Queue Pair
* Context: Can be called from interrupt or base context.
*/
static ibt_status_t
hermon_ci_modify_qp(ibc_hca_hdl_t hca, ibc_qp_hdl_t qp,
ibt_cep_modify_flags_t flags, ibt_qp_info_t *info_p,
ibt_queue_sizes_t *actual_sz)
{
hermon_state_t *state;
hermon_qphdl_t qphdl;
int status;
/* Grab the Hermon softstate pointer and QP handle */
state = (hermon_state_t *)hca;
qphdl = (hermon_qphdl_t)qp;
/* Modify the QP */
status = hermon_qp_modify(state, qphdl, flags, info_p, actual_sz);
return (status);
}
/*
* hermon_ci_alloc_cq()
* Allocate a Completion Queue
* Context: Can be called only from user or kernel context.
*/
/* ARGSUSED */
static ibt_status_t
hermon_ci_alloc_cq(ibc_hca_hdl_t hca, ibt_cq_hdl_t ibt_cqhdl,
ibt_cq_attr_t *attr_p, ibc_cq_hdl_t *cq_p, uint_t *actual_size)
{
hermon_state_t *state;
hermon_cqhdl_t cqhdl;
int status;
state = (hermon_state_t *)hca;
/* Allocate the CQ */
status = hermon_cq_alloc(state, ibt_cqhdl, attr_p, actual_size,
&cqhdl, HERMON_NOSLEEP);
if (status != DDI_SUCCESS) {
return (status);
}
/* Return the Hermon CQ handle */
*cq_p = (ibc_cq_hdl_t)cqhdl;
return (IBT_SUCCESS);
}
/*
* hermon_ci_free_cq()
* Free a Completion Queue
* Context: Can be called only from user or kernel context.
*/
static ibt_status_t
hermon_ci_free_cq(ibc_hca_hdl_t hca, ibc_cq_hdl_t cq)
{
hermon_state_t *state;
hermon_cqhdl_t cqhdl;
int status;
/* Grab the Hermon softstate pointer and CQ handle */
state = (hermon_state_t *)hca;
cqhdl = (hermon_cqhdl_t)cq;
/* Free the CQ */
status = hermon_cq_free(state, &cqhdl, HERMON_NOSLEEP);
return (status);
}
/*
* hermon_ci_query_cq()
* Return the size of a Completion Queue
* Context: Can be called only from user or kernel context.
*/
static ibt_status_t
hermon_ci_query_cq(ibc_hca_hdl_t hca, ibc_cq_hdl_t cq, uint_t *entries_p,
uint_t *count_p, uint_t *usec_p, ibt_cq_handler_id_t *hid_p)
{
hermon_state_t *state;
hermon_cqhdl_t cqhdl;
/* Grab the CQ handle */
state = (hermon_state_t *)hca;
cqhdl = (hermon_cqhdl_t)cq;
/* Query the current CQ size */
*entries_p = cqhdl->cq_bufsz;
*count_p = cqhdl->cq_intmod_count;
*usec_p = cqhdl->cq_intmod_usec;
_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*cqhdl))
*hid_p = HERMON_EQNUM_TO_HID(state, cqhdl->cq_eqnum);
return (IBT_SUCCESS);
}
/*
* hermon_ci_resize_cq()
* Change the size of a Completion Queue
* Context: Can be called only from user or kernel context.
*/
static ibt_status_t
hermon_ci_resize_cq(ibc_hca_hdl_t hca, ibc_cq_hdl_t cq, uint_t size,
uint_t *actual_size)
{
hermon_state_t *state;
hermon_cqhdl_t cqhdl;
int status;
/* Grab the Hermon softstate pointer and CQ handle */
state = (hermon_state_t *)hca;
cqhdl = (hermon_cqhdl_t)cq;
/* Resize the CQ */
status = hermon_cq_resize(state, cqhdl, size, actual_size,
HERMON_NOSLEEP);
if (status != DDI_SUCCESS) {
return (status);
}
return (IBT_SUCCESS);
}
/*
* hermon_ci_modify_cq()
* Change the interrupt moderation values of a Completion Queue
* Context: Can be called only from user or kernel context.
*/
static ibt_status_t
hermon_ci_modify_cq(ibc_hca_hdl_t hca, ibc_cq_hdl_t cq, uint_t count,
uint_t usec, ibt_cq_handler_id_t hid)
{
hermon_state_t *state;
hermon_cqhdl_t cqhdl;
int status;
/* Grab the Hermon softstate pointer and CQ handle */
state = (hermon_state_t *)hca;
cqhdl = (hermon_cqhdl_t)cq;
/* Resize the CQ */
status = hermon_cq_modify(state, cqhdl, count, usec, hid,
HERMON_NOSLEEP);
return (status);
}
/*
* hermon_ci_alloc_cq_sched()
* Reserve a CQ scheduling class resource
* Context: Can be called only from user or kernel context.
*/
/* ARGSUSED */
static ibt_status_t
hermon_ci_alloc_cq_sched(ibc_hca_hdl_t hca, ibt_cq_sched_attr_t *attr,
ibc_sched_hdl_t *sched_hdl_p)
{
int status;
status = hermon_cq_sched_alloc((hermon_state_t *)hca, attr,
(hermon_cq_sched_t **)sched_hdl_p);
return (status);
}
/*
* hermon_ci_free_cq_sched()
* Free a CQ scheduling class resource
* Context: Can be called only from user or kernel context.
*/
/* ARGSUSED */
static ibt_status_t
hermon_ci_free_cq_sched(ibc_hca_hdl_t hca, ibc_sched_hdl_t sched_hdl)
{
int status;
status = hermon_cq_sched_free((hermon_state_t *)hca,
(hermon_cq_sched_t *)sched_hdl);
return (status);
}
static ibt_status_t
hermon_ci_query_cq_handler_id(ibc_hca_hdl_t hca,
ibt_cq_handler_id_t hid, ibt_cq_handler_attr_t *attrs)
{
hermon_state_t *state;
state = (hermon_state_t *)hca;
if (!HERMON_HID_VALID(state, hid))
return (IBT_CQ_HID_INVALID);
if (attrs == NULL)
return (IBT_INVALID_PARAM);
_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*attrs))
attrs->cha_ih = state->hs_intrmsi_hdl[hid - 1];
attrs->cha_dip = state->hs_dip;
return (IBT_SUCCESS);
}
/*
* hermon_ci_alloc_eec()
* Allocate an End-to-End context
* Context: Can be called only from user or kernel context.
*/
/* ARGSUSED */
static ibt_status_t
hermon_ci_alloc_eec(ibc_hca_hdl_t hca, ibc_eec_flags_t flags,
ibt_eec_hdl_t ibt_eec, ibc_rdd_hdl_t rdd, ibc_eec_hdl_t *eec_p)
{
/*
* This is an unsupported interface for the Hermon driver. This
* interface is necessary to support Reliable Datagram (RD)
* operations. Hermon does not support RD.
*/
return (IBT_NOT_SUPPORTED);
}
/*
* hermon_ci_free_eec()
* Free an End-to-End context
* Context: Can be called only from user or kernel context.
*/
/* ARGSUSED */
static ibt_status_t
hermon_ci_free_eec(ibc_hca_hdl_t hca, ibc_eec_hdl_t eec)
{
/*
* This is an unsupported interface for the Hermon driver. This
* interface is necessary to support Reliable Datagram (RD)
* operations. Hermon does not support RD.
*/
return (IBT_NOT_SUPPORTED);
}
/*
* hermon_ci_query_eec()
* Query an End-to-End context
* Context: Can be called from interrupt or base context.
*/
/* ARGSUSED */
static ibt_status_t
hermon_ci_query_eec(ibc_hca_hdl_t hca, ibc_eec_hdl_t eec,
ibt_eec_query_attr_t *attr_p)
{
/*
* This is an unsupported interface for the Hermon driver. This
* interface is necessary to support Reliable Datagram (RD)
* operations. Hermon does not support RD.
*/
return (IBT_NOT_SUPPORTED);
}
/*
* hermon_ci_modify_eec()
* Modify an End-to-End context
* Context: Can be called from interrupt or base context.
*/
/* ARGSUSED */
static ibt_status_t
hermon_ci_modify_eec(ibc_hca_hdl_t hca, ibc_eec_hdl_t eec,
ibt_cep_modify_flags_t flags, ibt_eec_info_t *info_p)
{
/*
* This is an unsupported interface for the Hermon driver. This
* interface is necessary to support Reliable Datagram (RD)
* operations. Hermon does not support RD.
*/
return (IBT_NOT_SUPPORTED);
}
/*
* hermon_ci_register_mr()
* Prepare a virtually addressed Memory Region for use by an HCA
* Context: Can be called from interrupt or base context.
*/
/* ARGSUSED */
static ibt_status_t
hermon_ci_register_mr(ibc_hca_hdl_t hca, ibc_pd_hdl_t pd,
ibt_mr_attr_t *mr_attr, void *ibtl_reserved, ibc_mr_hdl_t *mr_p,
ibt_mr_desc_t *mr_desc)
{
hermon_mr_options_t op;
hermon_state_t *state;
hermon_pdhdl_t pdhdl;
hermon_mrhdl_t mrhdl;
int status;
_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*mr_desc))
ASSERT(mr_attr != NULL);
ASSERT(mr_p != NULL);
ASSERT(mr_desc != NULL);
/*
* Validate the access flags. Both Remote Write and Remote Atomic
* require the Local Write flag to be set
*/
if (((mr_attr->mr_flags & IBT_MR_ENABLE_REMOTE_WRITE) ||
(mr_attr->mr_flags & IBT_MR_ENABLE_REMOTE_ATOMIC)) &&
!(mr_attr->mr_flags & IBT_MR_ENABLE_LOCAL_WRITE)) {
return (IBT_MR_ACCESS_REQ_INVALID);
}
/* Grab the Hermon softstate pointer and PD handle */
state = (hermon_state_t *)hca;
pdhdl = (hermon_pdhdl_t)pd;
/* Register the memory region */
op.mro_bind_type = state->hs_cfg_profile->cp_iommu_bypass;
op.mro_bind_dmahdl = NULL;
op.mro_bind_override_addr = 0;
status = hermon_mr_register(state, pdhdl, mr_attr, &mrhdl,
&op, HERMON_MPT_DMPT);
if (status != DDI_SUCCESS) {
return (status);
}
_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*mrhdl))
/* Fill in the mr_desc structure */
mr_desc->md_vaddr = mrhdl->mr_bindinfo.bi_addr;
mr_desc->md_lkey = mrhdl->mr_lkey;
/* Only set RKey if remote access was requested */
if ((mr_attr->mr_flags & IBT_MR_ENABLE_REMOTE_ATOMIC) ||
(mr_attr->mr_flags & IBT_MR_ENABLE_REMOTE_WRITE) ||
(mr_attr->mr_flags & IBT_MR_ENABLE_REMOTE_READ)) {
mr_desc->md_rkey = mrhdl->mr_rkey;
}
/*
* If region is mapped for streaming (i.e. noncoherent), then set
* sync is required
*/
mr_desc->md_sync_required = (mrhdl->mr_bindinfo.bi_flags &
IBT_MR_NONCOHERENT) ? B_TRUE : B_FALSE;
/* Return the Hermon MR handle */
*mr_p = (ibc_mr_hdl_t)mrhdl;
return (IBT_SUCCESS);
}
/*
* hermon_ci_register_buf()
* Prepare a Memory Region specified by buf structure for use by an HCA
* Context: Can be called from interrupt or base context.
*/
/* ARGSUSED */
static ibt_status_t
hermon_ci_register_buf(ibc_hca_hdl_t hca, ibc_pd_hdl_t pd,
ibt_smr_attr_t *attrp, struct buf *buf, void *ibtl_reserved,
ibt_mr_hdl_t *mr_p, ibt_mr_desc_t *mr_desc)
{
hermon_mr_options_t op;
hermon_state_t *state;
hermon_pdhdl_t pdhdl;
hermon_mrhdl_t mrhdl;
int status;
ibt_mr_flags_t flags = attrp->mr_flags;
_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*mr_desc))
ASSERT(mr_p != NULL);
ASSERT(mr_desc != NULL);
/*
* Validate the access flags. Both Remote Write and Remote Atomic
* require the Local Write flag to be set
*/
if (((flags & IBT_MR_ENABLE_REMOTE_WRITE) ||
(flags & IBT_MR_ENABLE_REMOTE_ATOMIC)) &&
!(flags & IBT_MR_ENABLE_LOCAL_WRITE)) {
return (IBT_MR_ACCESS_REQ_INVALID);
}
/* Grab the Hermon softstate pointer and PD handle */
state = (hermon_state_t *)hca;
pdhdl = (hermon_pdhdl_t)pd;
/* Register the memory region */
op.mro_bind_type = state->hs_cfg_profile->cp_iommu_bypass;
op.mro_bind_dmahdl = NULL;
op.mro_bind_override_addr = 0;
status = hermon_mr_register_buf(state, pdhdl, attrp, buf,
&mrhdl, &op, HERMON_MPT_DMPT);
if (status != DDI_SUCCESS) {
return (status);
}
_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*mrhdl))
/* Fill in the mr_desc structure */
mr_desc->md_vaddr = mrhdl->mr_bindinfo.bi_addr;
mr_desc->md_lkey = mrhdl->mr_lkey;
/* Only set RKey if remote access was requested */
if ((flags & IBT_MR_ENABLE_REMOTE_ATOMIC) ||
(flags & IBT_MR_ENABLE_REMOTE_WRITE) ||
(flags & IBT_MR_ENABLE_REMOTE_READ)) {
mr_desc->md_rkey = mrhdl->mr_rkey;
}
/*
* If region is mapped for streaming (i.e. noncoherent), then set
* sync is required
*/
mr_desc->md_sync_required = (mrhdl->mr_bindinfo.bi_flags &
IBT_MR_NONCOHERENT) ? B_TRUE : B_FALSE;
/* Return the Hermon MR handle */
*mr_p = (ibc_mr_hdl_t)mrhdl;
return (IBT_SUCCESS);
}
/*
* hermon_ci_deregister_mr()
* Deregister a Memory Region from an HCA translation table
* Context: Can be called only from user or kernel context.
*/
static ibt_status_t
hermon_ci_deregister_mr(ibc_hca_hdl_t hca, ibc_mr_hdl_t mr)
{
hermon_state_t *state;
hermon_mrhdl_t mrhdl;
int status;
/* Grab the Hermon softstate pointer */
state = (hermon_state_t *)hca;
mrhdl = (hermon_mrhdl_t)mr;
/*
* Deregister the memory region.
*/
status = hermon_mr_deregister(state, &mrhdl, HERMON_MR_DEREG_ALL,
HERMON_NOSLEEP);
return (status);
}
/*
* hermon_ci_query_mr()
* Retrieve information about a specified Memory Region
* Context: Can be called from interrupt or base context.
*/
static ibt_status_t
hermon_ci_query_mr(ibc_hca_hdl_t hca, ibc_mr_hdl_t mr,
ibt_mr_query_attr_t *mr_attr)
{
hermon_state_t *state;
hermon_mrhdl_t mrhdl;
int status;
ASSERT(mr_attr != NULL);
/* Grab the Hermon softstate pointer and MR handle */
state = (hermon_state_t *)hca;
mrhdl = (hermon_mrhdl_t)mr;
/* Query the memory region */
status = hermon_mr_query(state, mrhdl, mr_attr);
return (status);
}
/*
* hermon_ci_register_shared_mr()
* Create a shared memory region matching an existing Memory Region
* Context: Can be called from interrupt or base context.
*/
/* ARGSUSED */
static ibt_status_t
hermon_ci_register_shared_mr(ibc_hca_hdl_t hca, ibc_mr_hdl_t mr,
ibc_pd_hdl_t pd, ibt_smr_attr_t *mr_attr, void *ibtl_reserved,
ibc_mr_hdl_t *mr_p, ibt_mr_desc_t *mr_desc)
{
hermon_state_t *state;
hermon_pdhdl_t pdhdl;
hermon_mrhdl_t mrhdl, mrhdl_new;
int status;
_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*mr_desc))
ASSERT(mr_attr != NULL);
ASSERT(mr_p != NULL);
ASSERT(mr_desc != NULL);
/*
* Validate the access flags. Both Remote Write and Remote Atomic
* require the Local Write flag to be set
*/
if (((mr_attr->mr_flags & IBT_MR_ENABLE_REMOTE_WRITE) ||
(mr_attr->mr_flags & IBT_MR_ENABLE_REMOTE_ATOMIC)) &&
!(mr_attr->mr_flags & IBT_MR_ENABLE_LOCAL_WRITE)) {
return (IBT_MR_ACCESS_REQ_INVALID);
}
/* Grab the Hermon softstate pointer and handles */
state = (hermon_state_t *)hca;
pdhdl = (hermon_pdhdl_t)pd;
mrhdl = (hermon_mrhdl_t)mr;
/* Register the shared memory region */
status = hermon_mr_register_shared(state, mrhdl, pdhdl, mr_attr,
&mrhdl_new);
if (status != DDI_SUCCESS) {
return (status);
}
_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*mrhdl_new))
/* Fill in the mr_desc structure */
mr_desc->md_vaddr = mrhdl_new->mr_bindinfo.bi_addr;
mr_desc->md_lkey = mrhdl_new->mr_lkey;
/* Only set RKey if remote access was requested */
if ((mr_attr->mr_flags & IBT_MR_ENABLE_REMOTE_ATOMIC) ||
(mr_attr->mr_flags & IBT_MR_ENABLE_REMOTE_WRITE) ||
(mr_attr->mr_flags & IBT_MR_ENABLE_REMOTE_READ)) {
mr_desc->md_rkey = mrhdl_new->mr_rkey;
}
/*
* If shared region is mapped for streaming (i.e. noncoherent), then
* set sync is required
*/
mr_desc->md_sync_required = (mrhdl_new->mr_bindinfo.bi_flags &
IBT_MR_NONCOHERENT) ? B_TRUE : B_FALSE;
/* Return the Hermon MR handle */
*mr_p = (ibc_mr_hdl_t)mrhdl_new;
return (IBT_SUCCESS);
}
/*
* hermon_ci_reregister_mr()
* Modify the attributes of an existing Memory Region
* Context: Can be called from interrupt or base context.
*/
/* ARGSUSED */
static ibt_status_t
hermon_ci_reregister_mr(ibc_hca_hdl_t hca, ibc_mr_hdl_t mr, ibc_pd_hdl_t pd,
ibt_mr_attr_t *mr_attr, void *ibtl_reserved, ibc_mr_hdl_t *mr_new,
ibt_mr_desc_t *mr_desc)
{
hermon_mr_options_t op;
hermon_state_t *state;
hermon_pdhdl_t pdhdl;
hermon_mrhdl_t mrhdl, mrhdl_new;
int status;
_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*mr_desc))
ASSERT(mr_attr != NULL);
ASSERT(mr_new != NULL);
ASSERT(mr_desc != NULL);
/* Grab the Hermon softstate pointer, mrhdl, and pdhdl */
state = (hermon_state_t *)hca;
mrhdl = (hermon_mrhdl_t)mr;
pdhdl = (hermon_pdhdl_t)pd;
/* Reregister the memory region */
op.mro_bind_type = state->hs_cfg_profile->cp_iommu_bypass;
status = hermon_mr_reregister(state, mrhdl, pdhdl, mr_attr,
&mrhdl_new, &op);
if (status != DDI_SUCCESS) {
return (status);
}
_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*mrhdl_new))
/* Fill in the mr_desc structure */
mr_desc->md_vaddr = mrhdl_new->mr_bindinfo.bi_addr;
mr_desc->md_lkey = mrhdl_new->mr_lkey;
/* Only set RKey if remote access was requested */
if ((mr_attr->mr_flags & IBT_MR_ENABLE_REMOTE_ATOMIC) ||
(mr_attr->mr_flags & IBT_MR_ENABLE_REMOTE_WRITE) ||
(mr_attr->mr_flags & IBT_MR_ENABLE_REMOTE_READ)) {
mr_desc->md_rkey = mrhdl_new->mr_rkey;
}
/*
* If region is mapped for streaming (i.e. noncoherent), then set
* sync is required
*/
mr_desc->md_sync_required = (mrhdl_new->mr_bindinfo.bi_flags &
IBT_MR_NONCOHERENT) ? B_TRUE : B_FALSE;
/* Return the Hermon MR handle */
*mr_new = (ibc_mr_hdl_t)mrhdl_new;
return (IBT_SUCCESS);
}
/*
* hermon_ci_reregister_buf()
* Modify the attributes of an existing Memory Region
* Context: Can be called from interrupt or base context.
*/
/* ARGSUSED */
static ibt_status_t
hermon_ci_reregister_buf(ibc_hca_hdl_t hca, ibc_mr_hdl_t mr, ibc_pd_hdl_t pd,
ibt_smr_attr_t *attrp, struct buf *buf, void *ibtl_reserved,
ibc_mr_hdl_t *mr_new, ibt_mr_desc_t *mr_desc)
{
hermon_mr_options_t op;
hermon_state_t *state;
hermon_pdhdl_t pdhdl;
hermon_mrhdl_t mrhdl, mrhdl_new;
int status;
ibt_mr_flags_t flags = attrp->mr_flags;
_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*mr_desc))
ASSERT(mr_new != NULL);
ASSERT(mr_desc != NULL);
/* Grab the Hermon softstate pointer, mrhdl, and pdhdl */
state = (hermon_state_t *)hca;
mrhdl = (hermon_mrhdl_t)mr;
pdhdl = (hermon_pdhdl_t)pd;
/* Reregister the memory region */
op.mro_bind_type = state->hs_cfg_profile->cp_iommu_bypass;
status = hermon_mr_reregister_buf(state, mrhdl, pdhdl, attrp, buf,
&mrhdl_new, &op);
if (status != DDI_SUCCESS) {
return (status);
}
_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*mrhdl_new))
/* Fill in the mr_desc structure */
mr_desc->md_vaddr = mrhdl_new->mr_bindinfo.bi_addr;
mr_desc->md_lkey = mrhdl_new->mr_lkey;
/* Only set RKey if remote access was requested */
if ((flags & IBT_MR_ENABLE_REMOTE_ATOMIC) ||
(flags & IBT_MR_ENABLE_REMOTE_WRITE) ||
(flags & IBT_MR_ENABLE_REMOTE_READ)) {
mr_desc->md_rkey = mrhdl_new->mr_rkey;
}
/*
* If region is mapped for streaming (i.e. noncoherent), then set
* sync is required
*/
mr_desc->md_sync_required = (mrhdl_new->mr_bindinfo.bi_flags &
IBT_MR_NONCOHERENT) ? B_TRUE : B_FALSE;
/* Return the Hermon MR handle */
*mr_new = (ibc_mr_hdl_t)mrhdl_new;
return (IBT_SUCCESS);
}
/*
* hermon_ci_sync_mr()
* Synchronize access to a Memory Region
* Context: Can be called from interrupt or base context.
*/
static ibt_status_t
hermon_ci_sync_mr(ibc_hca_hdl_t hca, ibt_mr_sync_t *mr_segs, size_t num_segs)
{
hermon_state_t *state;
int status;
ASSERT(mr_segs != NULL);
/* Grab the Hermon softstate pointer */
state = (hermon_state_t *)hca;
/* Sync the memory region */
status = hermon_mr_sync(state, mr_segs, num_segs);
return (status);
}
/*
* hermon_ci_alloc_mw()
* Allocate a Memory Window
* Context: Can be called from interrupt or base context.
*/
static ibt_status_t
hermon_ci_alloc_mw(ibc_hca_hdl_t hca, ibc_pd_hdl_t pd, ibt_mw_flags_t flags,
ibc_mw_hdl_t *mw_p, ibt_rkey_t *rkey_p)
{
hermon_state_t *state;
hermon_pdhdl_t pdhdl;
hermon_mwhdl_t mwhdl;
int status;
ASSERT(mw_p != NULL);
ASSERT(rkey_p != NULL);
/* Grab the Hermon softstate pointer and PD handle */
state = (hermon_state_t *)hca;
pdhdl = (hermon_pdhdl_t)pd;
/* Allocate the memory window */
status = hermon_mw_alloc(state, pdhdl, flags, &mwhdl);
if (status != DDI_SUCCESS) {
return (status);
}
_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*mwhdl))
/* Return the MW handle and RKey */
*mw_p = (ibc_mw_hdl_t)mwhdl;
*rkey_p = mwhdl->mr_rkey;
return (IBT_SUCCESS);
}
/*
* hermon_ci_free_mw()
* Free a Memory Window
* Context: Can be called from interrupt or base context.
*/
static ibt_status_t
hermon_ci_free_mw(ibc_hca_hdl_t hca, ibc_mw_hdl_t mw)
{
hermon_state_t *state;
hermon_mwhdl_t mwhdl;
int status;
/* Grab the Hermon softstate pointer and MW handle */
state = (hermon_state_t *)hca;
mwhdl = (hermon_mwhdl_t)mw;
/* Free the memory window */
status = hermon_mw_free(state, &mwhdl, HERMON_NOSLEEP);
return (status);
}
/*
* hermon_ci_query_mw()
* Return the attributes of the specified Memory Window
* Context: Can be called from interrupt or base context.
*/
/* ARGSUSED */
static ibt_status_t
hermon_ci_query_mw(ibc_hca_hdl_t hca, ibc_mw_hdl_t mw,
ibt_mw_query_attr_t *mw_attr_p)
{
hermon_mwhdl_t mwhdl;
ASSERT(mw_attr_p != NULL);
/* Query the memory window pointer and fill in the return values */
mwhdl = (hermon_mwhdl_t)mw;
mutex_enter(&mwhdl->mr_lock);
mw_attr_p->mw_pd = (ibc_pd_hdl_t)mwhdl->mr_pdhdl;
mw_attr_p->mw_rkey = mwhdl->mr_rkey;
mutex_exit(&mwhdl->mr_lock);
return (IBT_SUCCESS);
}
/*
* hermon_ci_register_dma_mr()
* Allocate a memory region that maps physical addresses.
* Context: Can be called only from user or kernel context.
*/
/* ARGSUSED */
static ibt_status_t
hermon_ci_register_dma_mr(ibc_hca_hdl_t hca, ibc_pd_hdl_t pd,
ibt_dmr_attr_t *mr_attr, void *ibtl_reserved, ibc_mr_hdl_t *mr_p,
ibt_mr_desc_t *mr_desc)
{
hermon_state_t *state;
hermon_pdhdl_t pdhdl;
hermon_mrhdl_t mrhdl;
int status;
_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*mr_desc))
ASSERT(mr_attr != NULL);
ASSERT(mr_p != NULL);
ASSERT(mr_desc != NULL);
/*
* Validate the access flags. Both Remote Write and Remote Atomic
* require the Local Write flag to be set
*/
if (((mr_attr->dmr_flags & IBT_MR_ENABLE_REMOTE_WRITE) ||
(mr_attr->dmr_flags & IBT_MR_ENABLE_REMOTE_ATOMIC)) &&
!(mr_attr->dmr_flags & IBT_MR_ENABLE_LOCAL_WRITE)) {
return (IBT_MR_ACCESS_REQ_INVALID);
}
/* Grab the Hermon softstate pointer and PD handle */
state = (hermon_state_t *)hca;
pdhdl = (hermon_pdhdl_t)pd;
status = hermon_dma_mr_register(state, pdhdl, mr_attr, &mrhdl);
if (status != DDI_SUCCESS) {
return (status);
}
_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*mrhdl))
/* Fill in the mr_desc structure */
mr_desc->md_vaddr = mr_attr->dmr_paddr;
mr_desc->md_lkey = mrhdl->mr_lkey;
/* Only set RKey if remote access was requested */
if ((mr_attr->dmr_flags & IBT_MR_ENABLE_REMOTE_ATOMIC) ||
(mr_attr->dmr_flags & IBT_MR_ENABLE_REMOTE_WRITE) ||
(mr_attr->dmr_flags & IBT_MR_ENABLE_REMOTE_READ)) {
mr_desc->md_rkey = mrhdl->mr_rkey;
}
/*
* If region is mapped for streaming (i.e. noncoherent), then set
* sync is required
*/
mr_desc->md_sync_required = B_FALSE;
/* Return the Hermon MR handle */
*mr_p = (ibc_mr_hdl_t)mrhdl;
return (IBT_SUCCESS);
}
/*
* hermon_ci_attach_mcg()
* Attach a Queue Pair to a Multicast Group
* Context: Can be called only from user or kernel context.
*/
static ibt_status_t
hermon_ci_attach_mcg(ibc_hca_hdl_t hca, ibc_qp_hdl_t qp, ib_gid_t gid,
ib_lid_t lid)
{
hermon_state_t *state;
hermon_qphdl_t qphdl;
int status;
/* Grab the Hermon softstate pointer and QP handles */
state = (hermon_state_t *)hca;
qphdl = (hermon_qphdl_t)qp;
/* Attach the QP to the multicast group */
status = hermon_mcg_attach(state, qphdl, gid, lid);
return (status);
}
/*
* hermon_ci_detach_mcg()
* Detach a Queue Pair to a Multicast Group
* Context: Can be called only from user or kernel context.
*/
static ibt_status_t
hermon_ci_detach_mcg(ibc_hca_hdl_t hca, ibc_qp_hdl_t qp, ib_gid_t gid,
ib_lid_t lid)
{
hermon_state_t *state;
hermon_qphdl_t qphdl;
int status;
/* Grab the Hermon softstate pointer and QP handle */
state = (hermon_state_t *)hca;
qphdl = (hermon_qphdl_t)qp;
/* Detach the QP from the multicast group */
status = hermon_mcg_detach(state, qphdl, gid, lid);
return (status);
}
/*
* hermon_ci_post_send()
* Post send work requests to the send queue on the specified QP
* Context: Can be called from interrupt or base context.
*/
static ibt_status_t
hermon_ci_post_send(ibc_hca_hdl_t hca, ibc_qp_hdl_t qp, ibt_send_wr_t *wr_p,
uint_t num_wr, uint_t *num_posted_p)
{
hermon_state_t *state;
hermon_qphdl_t qphdl;
int status;
ASSERT(wr_p != NULL);
ASSERT(num_wr != 0);
/* Grab the Hermon softstate pointer and QP handle */
state = (hermon_state_t *)hca;
qphdl = (hermon_qphdl_t)qp;
/* Post the send WQEs */
status = hermon_post_send(state, qphdl, wr_p, num_wr, num_posted_p);
return (status);
}
/*
* hermon_ci_post_recv()
* Post receive work requests to the receive queue on the specified QP
* Context: Can be called from interrupt or base context.
*/
static ibt_status_t
hermon_ci_post_recv(ibc_hca_hdl_t hca, ibc_qp_hdl_t qp, ibt_recv_wr_t *wr_p,
uint_t num_wr, uint_t *num_posted_p)
{
hermon_state_t *state;
hermon_qphdl_t qphdl;
int status;
ASSERT(wr_p != NULL);
ASSERT(num_wr != 0);
state = (hermon_state_t *)hca;
qphdl = (hermon_qphdl_t)qp;
/* Post the receive WQEs */
status = hermon_post_recv(state, qphdl, wr_p, num_wr, num_posted_p);
return (status);
}
/*
* hermon_ci_poll_cq()
* Poll for a work request completion
* Context: Can be called from interrupt or base context.
*/
static ibt_status_t
hermon_ci_poll_cq(ibc_hca_hdl_t hca, ibc_cq_hdl_t cq, ibt_wc_t *wc_p,
uint_t num_wc, uint_t *num_polled)
{
hermon_state_t *state;
hermon_cqhdl_t cqhdl;
int status;
ASSERT(wc_p != NULL);
/* Check for valid num_wc field */
if (num_wc == 0) {
return (IBT_INVALID_PARAM);
}
/* Grab the Hermon softstate pointer and CQ handle */
state = (hermon_state_t *)hca;
cqhdl = (hermon_cqhdl_t)cq;
/* Poll for work request completions */
status = hermon_cq_poll(state, cqhdl, wc_p, num_wc, num_polled);
return (status);
}
/*
* hermon_ci_notify_cq()
* Enable notification events on the specified CQ
* Context: Can be called from interrupt or base context.
*/
static ibt_status_t
hermon_ci_notify_cq(ibc_hca_hdl_t hca, ibc_cq_hdl_t cq_hdl,
ibt_cq_notify_flags_t flags)
{
hermon_state_t *state;
hermon_cqhdl_t cqhdl;
int status;
/* Grab the Hermon softstate pointer and CQ handle */
state = (hermon_state_t *)hca;
cqhdl = (hermon_cqhdl_t)cq_hdl;
/* Enable the CQ notification */
status = hermon_cq_notify(state, cqhdl, flags);
return (status);
}
/*
* hermon_ci_ci_data_in()
* Exchange CI-specific data.
* Context: Can be called only from user or kernel context.
*/
static ibt_status_t
hermon_ci_ci_data_in(ibc_hca_hdl_t hca, ibt_ci_data_flags_t flags,
ibt_object_type_t object, void *ibc_object_handle, void *data_p,
size_t data_sz)
{
hermon_state_t *state;
int status;
/* Grab the Hermon softstate pointer */
state = (hermon_state_t *)hca;
/* Get the Hermon userland mapping information */
status = hermon_umap_ci_data_in(state, flags, object,
ibc_object_handle, data_p, data_sz);
return (status);
}
/*
* hermon_ci_ci_data_out()
* Exchange CI-specific data.
* Context: Can be called only from user or kernel context.
*/
static ibt_status_t
hermon_ci_ci_data_out(ibc_hca_hdl_t hca, ibt_ci_data_flags_t flags,
ibt_object_type_t object, void *ibc_object_handle, void *data_p,
size_t data_sz)
{
hermon_state_t *state;
int status;
/* Grab the Hermon softstate pointer */
state = (hermon_state_t *)hca;
/* Get the Hermon userland mapping information */
status = hermon_umap_ci_data_out(state, flags, object,
ibc_object_handle, data_p, data_sz);
return (status);
}
/*
* hermon_ci_alloc_srq()
* Allocate a Shared Receive Queue (SRQ)
* Context: Can be called only from user or kernel context
*/
static ibt_status_t
hermon_ci_alloc_srq(ibc_hca_hdl_t hca, ibt_srq_flags_t flags,
ibt_srq_hdl_t ibt_srq, ibc_pd_hdl_t pd, ibt_srq_sizes_t *sizes,
ibc_srq_hdl_t *ibc_srq_p, ibt_srq_sizes_t *ret_sizes_p)
{
hermon_state_t *state;
hermon_pdhdl_t pdhdl;
hermon_srqhdl_t srqhdl;
hermon_srq_info_t srqinfo;
int status;
state = (hermon_state_t *)hca;
pdhdl = (hermon_pdhdl_t)pd;
srqinfo.srqi_ibt_srqhdl = ibt_srq;
srqinfo.srqi_pd = pdhdl;
srqinfo.srqi_sizes = sizes;
srqinfo.srqi_real_sizes = ret_sizes_p;
srqinfo.srqi_srqhdl = &srqhdl;
srqinfo.srqi_flags = flags;
status = hermon_srq_alloc(state, &srqinfo, HERMON_NOSLEEP);
if (status != DDI_SUCCESS) {
return (status);
}
*ibc_srq_p = (ibc_srq_hdl_t)srqhdl;
return (IBT_SUCCESS);
}
/*
* hermon_ci_free_srq()
* Free a Shared Receive Queue (SRQ)
* Context: Can be called only from user or kernel context
*/
static ibt_status_t
hermon_ci_free_srq(ibc_hca_hdl_t hca, ibc_srq_hdl_t srq)
{
hermon_state_t *state;
hermon_srqhdl_t srqhdl;
int status;
state = (hermon_state_t *)hca;
/* Check for valid SRQ handle pointer */
if (srq == NULL) {
return (IBT_SRQ_HDL_INVALID);
}
srqhdl = (hermon_srqhdl_t)srq;
/* Free the SRQ */
status = hermon_srq_free(state, &srqhdl, HERMON_NOSLEEP);
return (status);
}
/*
* hermon_ci_query_srq()
* Query properties of a Shared Receive Queue (SRQ)
* Context: Can be called from interrupt or base context.
*/
/* ARGSUSED */
static ibt_status_t
hermon_ci_query_srq(ibc_hca_hdl_t hca, ibc_srq_hdl_t srq, ibc_pd_hdl_t *pd_p,
ibt_srq_sizes_t *sizes_p, uint_t *limit_p)
{
hermon_srqhdl_t srqhdl;
srqhdl = (hermon_srqhdl_t)srq;
mutex_enter(&srqhdl->srq_lock);
if (srqhdl->srq_state == HERMON_SRQ_STATE_ERROR) {
mutex_exit(&srqhdl->srq_lock);
return (IBT_SRQ_ERROR_STATE);
}
*pd_p = (ibc_pd_hdl_t)srqhdl->srq_pdhdl;
sizes_p->srq_wr_sz = srqhdl->srq_real_sizes.srq_wr_sz - 1;
sizes_p->srq_sgl_sz = srqhdl->srq_real_sizes.srq_sgl_sz;
mutex_exit(&srqhdl->srq_lock);
*limit_p = 0;
return (IBT_SUCCESS);
}
/*
* hermon_ci_modify_srq()
* Modify properties of a Shared Receive Queue (SRQ)
* Context: Can be called from interrupt or base context.
*/
/* ARGSUSED */
static ibt_status_t
hermon_ci_modify_srq(ibc_hca_hdl_t hca, ibc_srq_hdl_t srq,
ibt_srq_modify_flags_t flags, uint_t size, uint_t limit, uint_t *ret_size_p)
{
hermon_state_t *state;
hermon_srqhdl_t srqhdl;
uint_t resize_supported, cur_srq_size;
int status;
state = (hermon_state_t *)hca;
srqhdl = (hermon_srqhdl_t)srq;
/*
* Check Error State of SRQ.
* Also, while we are holding the lock we save away the current SRQ
* size for later use.
*/
mutex_enter(&srqhdl->srq_lock);
cur_srq_size = srqhdl->srq_wq_bufsz;
if (srqhdl->srq_state == HERMON_SRQ_STATE_ERROR) {
mutex_exit(&srqhdl->srq_lock);
return (IBT_SRQ_ERROR_STATE);
}
mutex_exit(&srqhdl->srq_lock);
/*
* Setting the limit watermark is not currently supported. This is a
* hermon hardware (firmware) limitation. We return NOT_SUPPORTED here,
* and have the limit code commented out for now.
*
* XXX If we enable the limit watermark support, we need to do checks
* and set the 'srq->srq_wr_limit' here, instead of returning not
* supported. The 'hermon_srq_modify' operation below is for resizing
* the SRQ only, the limit work should be done here. If this is
* changed to use the 'limit' field, the 'ARGSUSED' comment for this
* function should also be removed at that time.
*/
if (flags & IBT_SRQ_SET_LIMIT) {
return (IBT_NOT_SUPPORTED);
}
/*
* Check the SET_SIZE flag. If not set, we simply return success here.
* However if it is set, we check if resize is supported and only then
* do we continue on with our resize processing.
*/
if (!(flags & IBT_SRQ_SET_SIZE)) {
return (IBT_SUCCESS);
}
resize_supported = state->hs_ibtfinfo.hca_attr->hca_flags &
IBT_HCA_RESIZE_SRQ;
if ((flags & IBT_SRQ_SET_SIZE) && !resize_supported) {
return (IBT_NOT_SUPPORTED);
}
/*
* We do not support resizing an SRQ to be smaller than it's current
* size. If a smaller (or equal) size is requested, then we simply
* return success, and do nothing.
*/
if (size <= cur_srq_size) {
*ret_size_p = cur_srq_size;
return (IBT_SUCCESS);
}
status = hermon_srq_modify(state, srqhdl, size, ret_size_p,
HERMON_NOSLEEP);
if (status != DDI_SUCCESS) {
/* Set return value to current SRQ size */
*ret_size_p = cur_srq_size;
return (status);
}
return (IBT_SUCCESS);
}
/*
* hermon_ci_post_srq()
* Post a Work Request to the specified Shared Receive Queue (SRQ)
* Context: Can be called from interrupt or base context.
*/
static ibt_status_t
hermon_ci_post_srq(ibc_hca_hdl_t hca, ibc_srq_hdl_t srq,
ibt_recv_wr_t *wr, uint_t num_wr, uint_t *num_posted_p)
{
hermon_state_t *state;
hermon_srqhdl_t srqhdl;
int status;
state = (hermon_state_t *)hca;
srqhdl = (hermon_srqhdl_t)srq;
status = hermon_post_srq(state, srqhdl, wr, num_wr, num_posted_p);
return (status);
}
/* Address translation */
struct ibc_ma_s {
int h_ma_addr_list_len;
void *h_ma_addr_list;
ddi_dma_handle_t h_ma_dmahdl;
ddi_dma_handle_t h_ma_list_hdl;
ddi_acc_handle_t h_ma_list_acc_hdl;
size_t h_ma_real_len;
caddr_t h_ma_kaddr;
ibt_phys_addr_t h_ma_list_cookie;
};
static ibt_status_t
hermon_map_mem_area_fmr(ibc_hca_hdl_t hca, ibt_va_attr_t *va_attrs,
uint_t list_len, ibt_pmr_attr_t *pmr, ibc_ma_hdl_t *ma_hdl_p)
{
int status;
ibt_status_t ibt_status;
ibc_ma_hdl_t ma_hdl;
ib_memlen_t len;
ddi_dma_attr_t dma_attr;
uint_t cookie_cnt;
ddi_dma_cookie_t dmacookie;
hermon_state_t *state;
uint64_t *kaddr;
uint64_t addr, endaddr, pagesize;
int i, kmflag;
int (*callback)(caddr_t);
if ((va_attrs->va_flags & IBT_VA_BUF) == 0) {
return (IBT_NOT_SUPPORTED); /* XXX - not yet implemented */
}
state = (hermon_state_t *)hca;
hermon_dma_attr_init(state, &dma_attr);
if (va_attrs->va_flags & IBT_VA_NOSLEEP) {
kmflag = KM_NOSLEEP;
callback = DDI_DMA_DONTWAIT;
} else {
kmflag = KM_SLEEP;
callback = DDI_DMA_SLEEP;
}
ma_hdl = kmem_zalloc(sizeof (*ma_hdl), kmflag);
if (ma_hdl == NULL) {
return (IBT_INSUFF_RESOURCE);
}
#ifdef __sparc
if (state->hs_cfg_profile->cp_iommu_bypass == HERMON_BINDMEM_BYPASS)
dma_attr.dma_attr_flags = DDI_DMA_FORCE_PHYSICAL;
if (hermon_kernel_data_ro == HERMON_RO_ENABLED)
dma_attr.dma_attr_flags |= DDI_DMA_RELAXED_ORDERING;
#endif
_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*ma_hdl))
status = ddi_dma_alloc_handle(state->hs_dip, &dma_attr,
callback, NULL, &ma_hdl->h_ma_dmahdl);
if (status != DDI_SUCCESS) {
kmem_free(ma_hdl, sizeof (*ma_hdl));
return (IBT_INSUFF_RESOURCE);
}
status = ddi_dma_buf_bind_handle(ma_hdl->h_ma_dmahdl,
va_attrs->va_buf, DDI_DMA_RDWR | DDI_DMA_CONSISTENT,
callback, NULL, &dmacookie, &cookie_cnt);
if (status != DDI_DMA_MAPPED) {
status = ibc_get_ci_failure(0);
goto marea_fail3;
}
ma_hdl->h_ma_real_len = list_len * sizeof (ibt_phys_addr_t);
ma_hdl->h_ma_kaddr = kmem_zalloc(ma_hdl->h_ma_real_len, kmflag);
if (ma_hdl->h_ma_kaddr == NULL) {
ibt_status = IBT_INSUFF_RESOURCE;
goto marea_fail4;
}
i = 0;
len = 0;
pagesize = PAGESIZE;
kaddr = (uint64_t *)(void *)ma_hdl->h_ma_kaddr;
while (cookie_cnt-- > 0) {
addr = dmacookie.dmac_laddress;
len += dmacookie.dmac_size;
endaddr = addr + (dmacookie.dmac_size - 1);
addr = addr & ~(pagesize - 1);
while (addr <= endaddr) {
if (i >= list_len) {
status = IBT_PBL_TOO_SMALL;
goto marea_fail5;
}
kaddr[i] = htonll(addr | HERMON_MTT_ENTRY_PRESENT);
i++;
addr += pagesize;
if (addr == 0) {
static int do_once = 1;
_NOTE(SCHEME_PROTECTS_DATA("safe sharing",
do_once))
if (do_once) {
do_once = 0;
cmn_err(CE_NOTE, "probable error in "
"dma_cookie address: map_mem_area");
}
break;
}
}
if (cookie_cnt != 0)
ddi_dma_nextcookie(ma_hdl->h_ma_dmahdl, &dmacookie);
}
_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*pmr))
pmr->pmr_addr_list = (ibt_phys_addr_t *)(void *)ma_hdl->h_ma_kaddr;
pmr->pmr_iova = va_attrs->va_vaddr;
pmr->pmr_len = len;
pmr->pmr_offset = va_attrs->va_vaddr & PAGEOFFSET;
pmr->pmr_buf_sz = PAGESHIFT; /* PRM says "Page Sice", but... */
pmr->pmr_num_buf = i;
pmr->pmr_ma = ma_hdl;
*ma_hdl_p = ma_hdl;
return (IBT_SUCCESS);
marea_fail5:
kmem_free(ma_hdl->h_ma_kaddr, ma_hdl->h_ma_real_len);
marea_fail4:
status = ddi_dma_unbind_handle(ma_hdl->h_ma_dmahdl);
marea_fail3:
ddi_dma_free_handle(&ma_hdl->h_ma_dmahdl);
kmem_free(ma_hdl, sizeof (*ma_hdl));
*ma_hdl_p = NULL;
return (ibt_status);
}
/*
* hermon_ci_map_mem_area()
* Context: Can be called from user or base context.
*
* Creates the memory mapping suitable for a subsequent posting of an
* FRWR work request. All the info about the memory area for the
* FRWR work request (wr member of "union ibt_reg_req_u") is filled
* such that the client only needs to point wr.rc.rcwr.reg_pmr to it,
* and then fill in the additional information only it knows.
*
* Alternatively, creates the memory mapping for FMR.
*/
/* ARGSUSED */
static ibt_status_t
hermon_ci_map_mem_area(ibc_hca_hdl_t hca, ibt_va_attr_t *va_attrs,
void *ibtl_reserved, uint_t list_len, ibt_reg_req_t *reg_req,
ibc_ma_hdl_t *ma_hdl_p)
{
ibt_status_t ibt_status;
int status;
ibc_ma_hdl_t ma_hdl;
ibt_wr_reg_pmr_t *pmr;
ib_memlen_t len;
ddi_dma_attr_t dma_attr;
ddi_dma_handle_t khdl;
uint_t cookie_cnt;
ddi_dma_cookie_t dmacookie, kcookie;
hermon_state_t *state;
uint64_t *kaddr;
uint64_t addr, endaddr, pagesize, kcookie_paddr;
int i, j, kmflag;
int (*callback)(caddr_t);
if (va_attrs->va_flags & (IBT_VA_FMR | IBT_VA_REG_FN)) {
/* delegate FMR and Physical Register to other function */
return (hermon_map_mem_area_fmr(hca, va_attrs, list_len,
&reg_req->fn_arg, ma_hdl_p));
}
/* FRWR */
state = (hermon_state_t *)hca;
if (!(state->hs_ibtfinfo.hca_attr->hca_flags2 & IBT_HCA2_MEM_MGT_EXT))
return (IBT_NOT_SUPPORTED);
hermon_dma_attr_init(state, &dma_attr);
#ifdef __sparc
if (state->hs_cfg_profile->cp_iommu_bypass == HERMON_BINDMEM_BYPASS)
dma_attr.dma_attr_flags = DDI_DMA_FORCE_PHYSICAL;
if (hermon_kernel_data_ro == HERMON_RO_ENABLED)
dma_attr.dma_attr_flags |= DDI_DMA_RELAXED_ORDERING;
#endif
if (va_attrs->va_flags & IBT_VA_NOSLEEP) {
kmflag = KM_NOSLEEP;
callback = DDI_DMA_DONTWAIT;
} else {
kmflag = KM_SLEEP;
callback = DDI_DMA_SLEEP;
}
ma_hdl = kmem_zalloc(sizeof (*ma_hdl), kmflag);
if (ma_hdl == NULL) {
return (IBT_INSUFF_RESOURCE);
}
_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*ma_hdl))
status = ddi_dma_alloc_handle(state->hs_dip, &dma_attr,
callback, NULL, &ma_hdl->h_ma_dmahdl);
if (status != DDI_SUCCESS) {
ibt_status = IBT_INSUFF_RESOURCE;
goto marea_fail0;
}
dma_attr.dma_attr_align = 64; /* as per PRM */
status = ddi_dma_alloc_handle(state->hs_dip, &dma_attr,
callback, NULL, &ma_hdl->h_ma_list_hdl);
if (status != DDI_SUCCESS) {
ibt_status = IBT_INSUFF_RESOURCE;
goto marea_fail1;
}
/*
* Entries in the list in the last slot on each page cannot be used,
* so 1 extra ibt_phys_addr_t is allocated per page. We add 1 more
* to deal with the possibility of a less than 1 page allocation
* across a page boundary.
*/
status = ddi_dma_mem_alloc(ma_hdl->h_ma_list_hdl, (list_len + 1 +
list_len / (HERMON_PAGESIZE / sizeof (ibt_phys_addr_t))) *
sizeof (ibt_phys_addr_t),
&state->hs_reg_accattr, DDI_DMA_CONSISTENT, callback, NULL,
&ma_hdl->h_ma_kaddr, &ma_hdl->h_ma_real_len,
&ma_hdl->h_ma_list_acc_hdl);
if (status != DDI_SUCCESS) {
ibt_status = IBT_INSUFF_RESOURCE;
goto marea_fail2;
}
status = ddi_dma_addr_bind_handle(ma_hdl->h_ma_list_hdl, NULL,
ma_hdl->h_ma_kaddr, ma_hdl->h_ma_real_len, DDI_DMA_RDWR |
DDI_DMA_CONSISTENT, callback, NULL,
&kcookie, &cookie_cnt);
if (status != DDI_SUCCESS) {
ibt_status = IBT_INSUFF_RESOURCE;
goto marea_fail3;
}
if ((kcookie.dmac_laddress & 0x3f) != 0) {
cmn_err(CE_NOTE, "64-byte alignment assumption wrong");
ibt_status = ibc_get_ci_failure(0);
goto marea_fail4;
}
ma_hdl->h_ma_list_cookie.p_laddr = kcookie.dmac_laddress;
if (va_attrs->va_flags & IBT_VA_BUF) {
status = ddi_dma_buf_bind_handle(ma_hdl->h_ma_dmahdl,
va_attrs->va_buf, DDI_DMA_RDWR | DDI_DMA_CONSISTENT,
callback, NULL, &dmacookie, &cookie_cnt);
} else {
status = ddi_dma_addr_bind_handle(ma_hdl->h_ma_dmahdl,
va_attrs->va_as, (caddr_t)(uintptr_t)va_attrs->va_vaddr,
va_attrs->va_len, DDI_DMA_RDWR | DDI_DMA_CONSISTENT,
callback, NULL, &dmacookie, &cookie_cnt);
}
if (status != DDI_DMA_MAPPED) {
ibt_status = ibc_get_ci_failure(0);
goto marea_fail4;
}
i = 0; /* count the number of pbl entries */
j = 0; /* count the number of links to next HERMON_PAGE */
len = 0;
pagesize = PAGESIZE;
kaddr = (uint64_t *)(void *)ma_hdl->h_ma_kaddr;
kcookie.dmac_size += kcookie.dmac_laddress & HERMON_PAGEOFFSET;
kcookie_paddr = kcookie.dmac_laddress & HERMON_PAGEMASK;
khdl = ma_hdl->h_ma_list_hdl;
while (cookie_cnt-- > 0) {
addr = dmacookie.dmac_laddress;
len += dmacookie.dmac_size;
endaddr = addr + (dmacookie.dmac_size - 1);
addr = addr & ~(pagesize - 1);
while (addr <= endaddr) {
if (i >= list_len) {
ibt_status = IBT_PBL_TOO_SMALL;
goto marea_fail5;
}
/* Deal with last entry on page. */
if (!((uintptr_t)&kaddr[i+j+1] & HERMON_PAGEOFFSET)) {
if (kcookie.dmac_size > HERMON_PAGESIZE) {
kcookie_paddr += HERMON_PAGESIZE;
kcookie.dmac_size -= HERMON_PAGESIZE;
} else {
ddi_dma_nextcookie(khdl, &kcookie);
kcookie_paddr = kcookie.dmac_laddress;
}
kaddr[i+j] = htonll(kcookie_paddr);
j++;
}
kaddr[i+j] = htonll(addr | HERMON_MTT_ENTRY_PRESENT);
i++;
addr += pagesize;
if (addr == 0) {
static int do_once = 1;
_NOTE(SCHEME_PROTECTS_DATA("safe sharing",
do_once))
if (do_once) {
do_once = 0;
cmn_err(CE_NOTE, "probable error in "
"dma_cookie address: map_mem_area");
}
break;
}
}
if (cookie_cnt != 0)
ddi_dma_nextcookie(ma_hdl->h_ma_dmahdl, &dmacookie);
}
pmr = &reg_req->wr;
_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*pmr))
pmr->pmr_len = len;
pmr->pmr_offset = va_attrs->va_vaddr & PAGEOFFSET;
pmr->pmr_buf_sz = PAGESHIFT; /* PRM says "Page Size", but... */
pmr->pmr_num_buf = i;
pmr->pmr_addr_list = &ma_hdl->h_ma_list_cookie;
*ma_hdl_p = ma_hdl;
return (IBT_SUCCESS);
marea_fail5:
status = ddi_dma_unbind_handle(ma_hdl->h_ma_dmahdl);
if (status != DDI_SUCCESS)
HERMON_WARNING(state, "failed to unbind DMA mapping");
marea_fail4:
status = ddi_dma_unbind_handle(ma_hdl->h_ma_list_hdl);
if (status != DDI_SUCCESS)
HERMON_WARNING(state, "failed to unbind DMA mapping");
marea_fail3:
ddi_dma_mem_free(&ma_hdl->h_ma_list_acc_hdl);
marea_fail2:
ddi_dma_free_handle(&ma_hdl->h_ma_list_hdl);
marea_fail1:
ddi_dma_free_handle(&ma_hdl->h_ma_dmahdl);
marea_fail0:
kmem_free(ma_hdl, sizeof (*ma_hdl));
*ma_hdl_p = NULL;
return (ibt_status);
}
/*
* hermon_ci_unmap_mem_area()
* Unmap the memory area
* Context: Can be called from interrupt or base context.
*/
/* ARGSUSED */
static ibt_status_t
hermon_ci_unmap_mem_area(ibc_hca_hdl_t hca, ibc_ma_hdl_t ma_hdl)
{
int status;
hermon_state_t *state;
if (ma_hdl == NULL) {
return (IBT_MA_HDL_INVALID);
}
state = (hermon_state_t *)hca;
if (ma_hdl->h_ma_list_hdl != NULL) {
status = ddi_dma_unbind_handle(ma_hdl->h_ma_list_hdl);
if (status != DDI_SUCCESS)
HERMON_WARNING(state, "failed to unbind DMA mapping");
ddi_dma_mem_free(&ma_hdl->h_ma_list_acc_hdl);
ddi_dma_free_handle(&ma_hdl->h_ma_list_hdl);
} else {
kmem_free(ma_hdl->h_ma_kaddr, ma_hdl->h_ma_real_len);
}
status = ddi_dma_unbind_handle(ma_hdl->h_ma_dmahdl);
if (status != DDI_SUCCESS)
HERMON_WARNING(state, "failed to unbind DMA mapping");
ddi_dma_free_handle(&ma_hdl->h_ma_dmahdl);
kmem_free(ma_hdl, sizeof (*ma_hdl));
return (IBT_SUCCESS);
}
struct ibc_mi_s {
int imh_len;
ddi_dma_handle_t imh_dmahandle[1];
};
_NOTE(SCHEME_PROTECTS_DATA("safe sharing",
ibc_mi_s::imh_len
ibc_mi_s::imh_dmahandle))
/*
* hermon_ci_map_mem_iov()
* Map the memory
* Context: Can be called from interrupt or base context.
*/
/* ARGSUSED */
static ibt_status_t
hermon_ci_map_mem_iov(ibc_hca_hdl_t hca, ibt_iov_attr_t *iov_attr,
ibt_all_wr_t *wr, ibc_mi_hdl_t *mi_hdl_p)
{
int status;
int i, j, nds, max_nds;
uint_t len;
ibt_status_t ibt_status;
ddi_dma_handle_t dmahdl;
ddi_dma_cookie_t dmacookie;
ddi_dma_attr_t dma_attr;
uint_t cookie_cnt;
ibc_mi_hdl_t mi_hdl;
ibt_lkey_t rsvd_lkey;
ibt_wr_ds_t *sgl;
hermon_state_t *state;
int kmflag;
int (*callback)(caddr_t);
_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*wr))
state = (hermon_state_t *)hca;
hermon_dma_attr_init(state, &dma_attr);
#ifdef __sparc
if (state->hs_cfg_profile->cp_iommu_bypass == HERMON_BINDMEM_BYPASS)
dma_attr.dma_attr_flags = DDI_DMA_FORCE_PHYSICAL;
if (hermon_kernel_data_ro == HERMON_RO_ENABLED)
dma_attr.dma_attr_flags |= DDI_DMA_RELAXED_ORDERING;
#endif
nds = 0;
max_nds = iov_attr->iov_wr_nds;
if (iov_attr->iov_lso_hdr_sz)
max_nds -= (iov_attr->iov_lso_hdr_sz + sizeof (uint32_t) +
0xf) >> 4; /* 0xf is for rounding up to a multiple of 16 */
rsvd_lkey = (iov_attr->iov_flags & IBT_IOV_ALT_LKEY) ?
iov_attr->iov_alt_lkey : state->hs_devlim.rsv_lkey;
if ((iov_attr->iov_flags & IBT_IOV_NOSLEEP) == 0) {
kmflag = KM_SLEEP;
callback = DDI_DMA_SLEEP;
} else {
kmflag = KM_NOSLEEP;
callback = DDI_DMA_DONTWAIT;
}
if (iov_attr->iov_flags & IBT_IOV_BUF) {
mi_hdl = kmem_alloc(sizeof (*mi_hdl), kmflag);
if (mi_hdl == NULL)
return (IBT_INSUFF_RESOURCE);
sgl = wr->send.wr_sgl;
_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*sgl))
status = ddi_dma_alloc_handle(state->hs_dip, &dma_attr,
callback, NULL, &dmahdl);
if (status != DDI_SUCCESS) {
kmem_free(mi_hdl, sizeof (*mi_hdl));
return (IBT_INSUFF_RESOURCE);
}
status = ddi_dma_buf_bind_handle(dmahdl, iov_attr->iov_buf,
DDI_DMA_RDWR | DDI_DMA_CONSISTENT, callback, NULL,
&dmacookie, &cookie_cnt);
if (status != DDI_DMA_MAPPED) {
ddi_dma_free_handle(&dmahdl);
kmem_free(mi_hdl, sizeof (*mi_hdl));
return (ibc_get_ci_failure(0));
}
while (cookie_cnt-- > 0) {
if (nds > max_nds) {
status = ddi_dma_unbind_handle(dmahdl);
if (status != DDI_SUCCESS)
HERMON_WARNING(state, "failed to "
"unbind DMA mapping");
ddi_dma_free_handle(&dmahdl);
return (IBT_SGL_TOO_SMALL);
}
sgl[nds].ds_va = dmacookie.dmac_laddress;
sgl[nds].ds_key = rsvd_lkey;
sgl[nds].ds_len = (ib_msglen_t)dmacookie.dmac_size;
nds++;
if (cookie_cnt != 0)
ddi_dma_nextcookie(dmahdl, &dmacookie);
}
wr->send.wr_nds = nds;
mi_hdl->imh_len = 1;
mi_hdl->imh_dmahandle[0] = dmahdl;
*mi_hdl_p = mi_hdl;
return (IBT_SUCCESS);
}
if (iov_attr->iov_flags & IBT_IOV_RECV)
sgl = wr->recv.wr_sgl;
else
sgl = wr->send.wr_sgl;
_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*sgl))
len = iov_attr->iov_list_len;
for (i = 0, j = 0; j < len; j++) {
if (iov_attr->iov[j].iov_len == 0)
continue;
i++;
}
mi_hdl = kmem_alloc(sizeof (*mi_hdl) +
(i - 1) * sizeof (ddi_dma_handle_t), kmflag);
if (mi_hdl == NULL)
return (IBT_INSUFF_RESOURCE);
mi_hdl->imh_len = i;
for (i = 0, j = 0; j < len; j++) {
if (iov_attr->iov[j].iov_len == 0)
continue;
status = ddi_dma_alloc_handle(state->hs_dip, &dma_attr,
callback, NULL, &dmahdl);
if (status != DDI_SUCCESS) {
ibt_status = IBT_INSUFF_RESOURCE;
goto fail2;
}
status = ddi_dma_addr_bind_handle(dmahdl, iov_attr->iov_as,
iov_attr->iov[j].iov_addr, iov_attr->iov[j].iov_len,
DDI_DMA_RDWR | DDI_DMA_CONSISTENT, callback, NULL,
&dmacookie, &cookie_cnt);
if (status != DDI_DMA_MAPPED) {
ibt_status = ibc_get_ci_failure(0);
goto fail1;
}
if (nds + cookie_cnt > max_nds) {
ibt_status = IBT_SGL_TOO_SMALL;
goto fail2;
}
while (cookie_cnt-- > 0) {
sgl[nds].ds_va = dmacookie.dmac_laddress;
sgl[nds].ds_key = rsvd_lkey;
sgl[nds].ds_len = (ib_msglen_t)dmacookie.dmac_size;
nds++;
if (cookie_cnt != 0)
ddi_dma_nextcookie(dmahdl, &dmacookie);
}
mi_hdl->imh_dmahandle[i] = dmahdl;
i++;
}
if (iov_attr->iov_flags & IBT_IOV_RECV)
wr->recv.wr_nds = nds;
else
wr->send.wr_nds = nds;
*mi_hdl_p = mi_hdl;
return (IBT_SUCCESS);
fail1:
ddi_dma_free_handle(&dmahdl);
fail2:
while (--i >= 0) {
status = ddi_dma_unbind_handle(mi_hdl->imh_dmahandle[i]);
if (status != DDI_SUCCESS)
HERMON_WARNING(state, "failed to unbind DMA mapping");
ddi_dma_free_handle(&mi_hdl->imh_dmahandle[i]);
}
kmem_free(mi_hdl, sizeof (*mi_hdl) +
(len - 1) * sizeof (ddi_dma_handle_t));
*mi_hdl_p = NULL;
return (ibt_status);
}
/*
* hermon_ci_unmap_mem_iov()
* Unmap the memory
* Context: Can be called from interrupt or base context.
*/
static ibt_status_t
hermon_ci_unmap_mem_iov(ibc_hca_hdl_t hca, ibc_mi_hdl_t mi_hdl)
{
int status, i;
hermon_state_t *state;
state = (hermon_state_t *)hca;
for (i = mi_hdl->imh_len; --i >= 0; ) {
status = ddi_dma_unbind_handle(mi_hdl->imh_dmahandle[i]);
if (status != DDI_SUCCESS)
HERMON_WARNING(state, "failed to unbind DMA mapping");
ddi_dma_free_handle(&mi_hdl->imh_dmahandle[i]);
}
kmem_free(mi_hdl, sizeof (*mi_hdl) +
(mi_hdl->imh_len - 1) * sizeof (ddi_dma_handle_t));
return (IBT_SUCCESS);
}
/*
* hermon_ci_alloc_lkey()
* Allocate an empty memory region for use with FRWR.
* Context: Can be called from user or base context.
*/
/* ARGSUSED */
static ibt_status_t
hermon_ci_alloc_lkey(ibc_hca_hdl_t hca, ibc_pd_hdl_t pd,
ibt_lkey_flags_t flags, uint_t list_sz, ibc_mr_hdl_t *mr_p,
ibt_pmr_desc_t *mem_desc_p)
{
hermon_state_t *state;
hermon_pdhdl_t pdhdl;
hermon_mrhdl_t mrhdl;
int status;
_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*mem_desc_p))
ASSERT(mr_p != NULL);
ASSERT(mem_desc_p != NULL);
state = (hermon_state_t *)hca;
pdhdl = (hermon_pdhdl_t)pd;
if (!(state->hs_ibtfinfo.hca_attr->hca_flags2 & IBT_HCA2_MEM_MGT_EXT))
return (IBT_NOT_SUPPORTED);
status = hermon_mr_alloc_lkey(state, pdhdl, flags, list_sz, &mrhdl);
if (status != DDI_SUCCESS) {
return (status);
}
_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*mrhdl))
/* Fill in the mem_desc_p structure */
mem_desc_p->pmd_iova = 0;
mem_desc_p->pmd_phys_buf_list_sz = list_sz;
mem_desc_p->pmd_lkey = mrhdl->mr_lkey;
/* Only set RKey if remote access was requested */
if (flags & IBT_KEY_REMOTE) {
mem_desc_p->pmd_rkey = mrhdl->mr_rkey;
}
mem_desc_p->pmd_sync_required = B_FALSE;
/* Return the Hermon MR handle */
*mr_p = (ibc_mr_hdl_t)mrhdl;
return (IBT_SUCCESS);
}
/* Physical Register Memory Region */
/*
* hermon_ci_register_physical_mr()
*/
/* ARGSUSED */
static ibt_status_t
hermon_ci_register_physical_mr(ibc_hca_hdl_t hca, ibc_pd_hdl_t pd,
ibt_pmr_attr_t *mem_pattrs, void *ibtl_reserved, ibc_mr_hdl_t *mr_p,
ibt_pmr_desc_t *mem_desc_p)
{
return (IBT_NOT_SUPPORTED);
}
/*
* hermon_ci_reregister_physical_mr()
*/
/* ARGSUSED */
static ibt_status_t
hermon_ci_reregister_physical_mr(ibc_hca_hdl_t hca, ibc_mr_hdl_t mr,
ibc_pd_hdl_t pd, ibt_pmr_attr_t *mem_pattrs, void *ibtl_reserved,
ibc_mr_hdl_t *mr_p, ibt_pmr_desc_t *mr_desc_p)
{
return (IBT_NOT_SUPPORTED);
}
/* Mellanox FMR Support */
/*
* hermon_ci_create_fmr_pool()
* Creates a pool of memory regions suitable for FMR registration
* Context: Can be called from base context only
*/
static ibt_status_t
hermon_ci_create_fmr_pool(ibc_hca_hdl_t hca, ibc_pd_hdl_t pd,
ibt_fmr_pool_attr_t *params, ibc_fmr_pool_hdl_t *fmr_pool_p)
{
hermon_state_t *state;
hermon_pdhdl_t pdhdl;
hermon_fmrhdl_t fmrpoolhdl;
int status;
state = (hermon_state_t *)hca;
/* Check for valid PD handle pointer */
if (pd == NULL) {
return (IBT_PD_HDL_INVALID);
}
pdhdl = (hermon_pdhdl_t)pd;
/*
* Validate the access flags. Both Remote Write and Remote Atomic
* require the Local Write flag to be set
*/
if (((params->fmr_flags & IBT_MR_ENABLE_REMOTE_WRITE) ||
(params->fmr_flags & IBT_MR_ENABLE_REMOTE_ATOMIC)) &&
!(params->fmr_flags & IBT_MR_ENABLE_LOCAL_WRITE)) {
return (IBT_MR_ACCESS_REQ_INVALID);
}
status = hermon_create_fmr_pool(state, pdhdl, params, &fmrpoolhdl);
if (status != DDI_SUCCESS) {
return (status);
}
/* Set fmr_pool from hermon handle */
*fmr_pool_p = (ibc_fmr_pool_hdl_t)fmrpoolhdl;
return (IBT_SUCCESS);
}
/*
* hermon_ci_destroy_fmr_pool()
* Free all resources associated with an FMR pool.
* Context: Can be called from base context only.
*/
static ibt_status_t
hermon_ci_destroy_fmr_pool(ibc_hca_hdl_t hca, ibc_fmr_pool_hdl_t fmr_pool)
{
hermon_state_t *state;
hermon_fmrhdl_t fmrpoolhdl;
int status;
state = (hermon_state_t *)hca;
fmrpoolhdl = (hermon_fmrhdl_t)fmr_pool;
status = hermon_destroy_fmr_pool(state, fmrpoolhdl);
return (status);
}
/*
* hermon_ci_flush_fmr_pool()
* Force a flush of the memory tables, cleaning up used FMR resources.
* Context: Can be called from interrupt or base context.
*/
static ibt_status_t
hermon_ci_flush_fmr_pool(ibc_hca_hdl_t hca, ibc_fmr_pool_hdl_t fmr_pool)
{
hermon_state_t *state;
hermon_fmrhdl_t fmrpoolhdl;
int status;
state = (hermon_state_t *)hca;
fmrpoolhdl = (hermon_fmrhdl_t)fmr_pool;
status = hermon_flush_fmr_pool(state, fmrpoolhdl);
return (status);
}
/*
* hermon_ci_register_physical_fmr()
* From the 'pool' of FMR regions passed in, performs register physical
* operation.
* Context: Can be called from interrupt or base context.
*/
/* ARGSUSED */
static ibt_status_t
hermon_ci_register_physical_fmr(ibc_hca_hdl_t hca,
ibc_fmr_pool_hdl_t fmr_pool, ibt_pmr_attr_t *mem_pattr,
void *ibtl_reserved, ibc_mr_hdl_t *mr_p, ibt_pmr_desc_t *mem_desc_p)
{
hermon_state_t *state;
hermon_mrhdl_t mrhdl;
hermon_fmrhdl_t fmrpoolhdl;
int status;
ASSERT(mem_pattr != NULL);
ASSERT(mr_p != NULL);
ASSERT(mem_desc_p != NULL);
/* Grab the Hermon softstate pointer */
state = (hermon_state_t *)hca;
fmrpoolhdl = (hermon_fmrhdl_t)fmr_pool;
status = hermon_register_physical_fmr(state, fmrpoolhdl, mem_pattr,
&mrhdl, mem_desc_p);
if (status != DDI_SUCCESS) {
return (status);
}
/*
* If region is mapped for streaming (i.e. noncoherent), then set
* sync is required
*/
_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*mem_desc_p))
mem_desc_p->pmd_sync_required = (mrhdl->mr_bindinfo.bi_flags &
IBT_MR_NONCOHERENT) ? B_TRUE : B_FALSE;
if (mem_desc_p->pmd_sync_required == B_TRUE) {
/* Fill in DMA handle for future sync operations */
_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(mrhdl->mr_bindinfo))
mrhdl->mr_bindinfo.bi_dmahdl =
(ddi_dma_handle_t)mem_pattr->pmr_ma;
}
/* Return the Hermon MR handle */
*mr_p = (ibc_mr_hdl_t)mrhdl;
return (IBT_SUCCESS);
}
/*
* hermon_ci_deregister_fmr()
* Moves an FMR (specified by 'mr') to the deregistered state.
* Context: Can be called from base context only.
*/
static ibt_status_t
hermon_ci_deregister_fmr(ibc_hca_hdl_t hca, ibc_mr_hdl_t mr)
{
hermon_state_t *state;
hermon_mrhdl_t mrhdl;
int status;
/* Grab the Hermon softstate pointer */
state = (hermon_state_t *)hca;
mrhdl = (hermon_mrhdl_t)mr;
/*
* Deregister the memory region, either "unmap" the FMR or deregister
* the normal memory region.
*/
status = hermon_deregister_fmr(state, mrhdl);
return (status);
}
static int
hermon_mem_alloc(hermon_state_t *state, size_t size, ibt_mr_flags_t flags,
caddr_t *kaddrp, ibc_mem_alloc_hdl_t *mem_hdl)
{
ddi_dma_handle_t dma_hdl;
ddi_dma_attr_t dma_attr;
ddi_acc_handle_t acc_hdl;
size_t real_len;
int status;
int (*ddi_cb)(caddr_t);
ibc_mem_alloc_hdl_t mem_alloc_hdl;
hermon_dma_attr_init(state, &dma_attr);
ddi_cb = (flags & IBT_MR_NOSLEEP) ? DDI_DMA_DONTWAIT : DDI_DMA_SLEEP;
/* Allocate a DMA handle */
status = ddi_dma_alloc_handle(state->hs_dip, &dma_attr, ddi_cb,
NULL, &dma_hdl);
if (status != DDI_SUCCESS) {
return (DDI_FAILURE);
}
/* Allocate DMA memory */
status = ddi_dma_mem_alloc(dma_hdl, size,
&state->hs_reg_accattr, DDI_DMA_CONSISTENT, ddi_cb,
NULL, kaddrp, &real_len, &acc_hdl);
if (status != DDI_SUCCESS) {
ddi_dma_free_handle(&dma_hdl);
return (DDI_FAILURE);
}
/* Package the hermon_dma_info contents and return */
mem_alloc_hdl = kmem_alloc(sizeof (**mem_hdl),
(flags & IBT_MR_NOSLEEP) ? KM_NOSLEEP : KM_SLEEP);
if (mem_alloc_hdl == NULL) {
ddi_dma_mem_free(&acc_hdl);
ddi_dma_free_handle(&dma_hdl);
return (DDI_FAILURE);
}
mem_alloc_hdl->ibc_dma_hdl = dma_hdl;
mem_alloc_hdl->ibc_acc_hdl = acc_hdl;
*mem_hdl = mem_alloc_hdl;
return (DDI_SUCCESS);
}
/*
* hermon_ci_alloc_io_mem()
* Allocate dma-able memory
*
*/
static ibt_status_t
hermon_ci_alloc_io_mem(ibc_hca_hdl_t hca, size_t size, ibt_mr_flags_t mr_flag,
caddr_t *kaddrp, ibc_mem_alloc_hdl_t *mem_alloc_hdl_p)
{
hermon_state_t *state;
int status;
/* Grab the Hermon softstate pointer and mem handle */
state = (hermon_state_t *)hca;
/* Allocate the memory and handles */
status = hermon_mem_alloc(state, size, mr_flag, kaddrp,
mem_alloc_hdl_p);
if (status != DDI_SUCCESS) {
*mem_alloc_hdl_p = NULL;
*kaddrp = NULL;
return (status);
}
return (IBT_SUCCESS);
}
/*
* hermon_ci_free_io_mem()
* Unbind handl and free the memory
*/
/* ARGSUSED */
static ibt_status_t
hermon_ci_free_io_mem(ibc_hca_hdl_t hca, ibc_mem_alloc_hdl_t mem_alloc_hdl)
{
/* Unbind the handles and free the memory */
(void) ddi_dma_unbind_handle(mem_alloc_hdl->ibc_dma_hdl);
ddi_dma_mem_free(&mem_alloc_hdl->ibc_acc_hdl);
ddi_dma_free_handle(&mem_alloc_hdl->ibc_dma_hdl);
kmem_free(mem_alloc_hdl, sizeof (*mem_alloc_hdl));
return (IBT_SUCCESS);
}