9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * CDDL HEADER START
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * The contents of this file are subject to the terms of the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Common Development and Distribution License (the "License").
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * You may not use this file except in compliance with the License.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * See the License for the specific language governing permissions
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * and limitations under the License.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * When distributing Covered Code, include this CDDL HEADER in each
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * If applicable, add the following below this CDDL HEADER, with the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * fields enclosed by brackets "[]" replaced with your own identifying
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * information: Portions Copyright [yyyy] [name of copyright owner]
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * CDDL HEADER END
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Copyright 2009 Sun Microsystems, Inc. All rights reserved.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Use is subject to license terms.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * UDAPL kernel agent
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * The following variables support the debug log buffer scheme.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor#else /* DEBUG */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor#endif /* DEBUG */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic int daplka_dbgsize = sizeof (daplka_dbgbuf);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic void daplka_console(const char *, ...);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic void daplka_debug(const char *, ...);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic int daplka_apm = 0x1; /* default enable */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic int daplka_failback = 0x1; /* default enable */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor#else /* DEBUG */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor#endif /* DEBUG */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * driver entry points
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic int daplka_open(dev_t *, int, int, struct cred *);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic int daplka_close(dev_t, int, int, struct cred *);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic int daplka_attach(dev_info_t *, ddi_attach_cmd_t);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic int daplka_detach(dev_info_t *, ddi_detach_cmd_t);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic int daplka_info(dev_info_t *, ddi_info_cmd_t, void *, void **);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic int daplka_ioctl(dev_t, int, intptr_t, int, cred_t *, int *);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * types of ioctls
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic int daplka_common_ioctl(int, minor_t, intptr_t, int, cred_t *, int *);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic int daplka_misc_ioctl(int, daplka_ia_resource_t *, intptr_t, int,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic int daplka_ep_ioctl(int, daplka_ia_resource_t *, intptr_t, int,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic int daplka_evd_ioctl(int, daplka_ia_resource_t *, intptr_t, int,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic int daplka_mr_ioctl(int, daplka_ia_resource_t *, intptr_t, int,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic int daplka_cno_ioctl(int, daplka_ia_resource_t *, intptr_t, int,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic int daplka_pd_ioctl(int, daplka_ia_resource_t *, intptr_t, int,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic int daplka_sp_ioctl(int, daplka_ia_resource_t *, intptr_t, int,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic int daplka_srq_ioctl(int, daplka_ia_resource_t *, intptr_t, int,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * common ioctls and supporting functions
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic int daplka_ia_create(minor_t, intptr_t, int, cred_t *, int *);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic int daplka_ia_destroy(daplka_resource_t *);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * EP ioctls and supporting functions
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic int daplka_ep_create(daplka_ia_resource_t *, intptr_t, int,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic int daplka_ep_modify(daplka_ia_resource_t *, intptr_t, int,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic int daplka_ep_free(daplka_ia_resource_t *, intptr_t, int,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic int daplka_ep_connect(daplka_ia_resource_t *, intptr_t, int,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic int daplka_ep_disconnect(daplka_ia_resource_t *, intptr_t, int,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic int daplka_ep_reinit(daplka_ia_resource_t *, intptr_t, int,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic int daplka_ep_destroy(daplka_resource_t *);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic void daplka_hash_ep_free(void *);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic int daplka_ep_failback(void *objp, void *arg);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic int daplka_ep_altpath(daplka_ep_resource_t *, ib_gid_t *);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic uint32_t daplka_ep_get_state(daplka_ep_resource_t *);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic void daplka_ep_set_state(daplka_ep_resource_t *, uint32_t, uint32_t);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic boolean_t daplka_ep_transition_is_valid(uint32_t, uint32_t);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic daplka_timer_info_t *daplka_timer_info_alloc(daplka_ep_resource_t *);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic void daplka_timer_info_free(daplka_timer_info_t *);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic void daplka_timer_handler(void *);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic void daplka_timer_dispatch(void *);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic void daplka_timer_thread(void *);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic int daplka_cancel_timer(daplka_ep_resource_t *);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic void daplka_hash_timer_free(void *);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * EVD ioctls and supporting functions
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic int daplka_evd_create(daplka_ia_resource_t *, intptr_t, int,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic int daplka_cq_resize(daplka_ia_resource_t *, intptr_t, int,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic int daplka_evd_free(daplka_ia_resource_t *, intptr_t, int,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic int daplka_event_poll(daplka_ia_resource_t *, intptr_t, int,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic int daplka_evd_destroy(daplka_resource_t *);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic void daplka_cq_handler(ibt_cq_hdl_t, void *);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic void daplka_evd_wakeup(daplka_evd_resource_t *,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor daplka_evd_event_list_t *, daplka_evd_event_t *);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic void daplka_evd_event_enqueue(daplka_evd_event_list_t *,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic daplka_evd_event_t *daplka_evd_event_dequeue(daplka_evd_event_list_t *);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic void daplka_hash_evd_free(void *);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * SRQ ioctls and supporting functions
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic int daplka_srq_create(daplka_ia_resource_t *, intptr_t, int,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic int daplka_srq_resize(daplka_ia_resource_t *, intptr_t, int,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic int daplka_srq_free(daplka_ia_resource_t *, intptr_t, int,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic int daplka_srq_destroy(daplka_resource_t *);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic void daplka_hash_srq_free(void *);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Miscellaneous ioctls
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic int daplka_cr_accept(daplka_ia_resource_t *, intptr_t, int,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic int daplka_cr_reject(daplka_ia_resource_t *, intptr_t, int,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic int daplka_cr_handoff(daplka_ia_resource_t *, intptr_t, int,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic int daplka_ia_query(daplka_ia_resource_t *, intptr_t, int,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * PD ioctls and supporting functions
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic int daplka_pd_alloc(daplka_ia_resource_t *, intptr_t, int,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic int daplka_pd_free(daplka_ia_resource_t *, intptr_t, int,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic int daplka_pd_destroy(daplka_resource_t *);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic void daplka_hash_pd_free(void *);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * SP ioctls and supporting functions
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic int daplka_service_register(daplka_ia_resource_t *, intptr_t, int,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic int daplka_service_deregister(daplka_ia_resource_t *, intptr_t, int,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic int daplka_sp_destroy(daplka_resource_t *);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic void daplka_hash_sp_free(void *);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic void daplka_hash_sp_unref(void *);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * MR ioctls and supporting functions
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic int daplka_mr_register(daplka_ia_resource_t *, intptr_t, int,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic int daplka_mr_register_lmr(daplka_ia_resource_t *, intptr_t, int,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic int daplka_mr_register_shared(daplka_ia_resource_t *, intptr_t, int,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic int daplka_mr_deregister(daplka_ia_resource_t *, intptr_t, int,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic int daplka_mr_sync(daplka_ia_resource_t *, intptr_t, int,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic int daplka_mr_destroy(daplka_resource_t *);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic void daplka_hash_mr_free(void *);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic void daplka_shared_mr_free(daplka_mr_resource_t *);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * MW ioctls and supporting functions
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic int daplka_mw_alloc(daplka_ia_resource_t *, intptr_t, int,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic int daplka_mw_free(daplka_ia_resource_t *, intptr_t, int,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic int daplka_mw_destroy(daplka_resource_t *);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic void daplka_hash_mw_free(void *);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * CNO ioctls and supporting functions
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic int daplka_cno_alloc(daplka_ia_resource_t *, intptr_t, int,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic int daplka_cno_free(daplka_ia_resource_t *, intptr_t, int,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic int daplka_cno_wait(daplka_ia_resource_t *, intptr_t, int,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic int daplka_cno_destroy(daplka_resource_t *);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic void daplka_hash_cno_free(void *);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * CM handlers
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic ibt_cm_status_t daplka_cm_rc_handler(void *, ibt_cm_event_t *,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ibt_cm_return_args_t *, void *, ibt_priv_data_len_t);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic ibt_cm_status_t daplka_cm_service_handler(void *, ibt_cm_event_t *,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ibt_cm_return_args_t *, void *, ibt_priv_data_len_t);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic ibt_cm_status_t daplka_cm_service_req(daplka_sp_resource_t *,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ibt_cm_event_t *, ibt_cm_return_args_t *, void *, ibt_priv_data_len_t);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * resource management routines
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic int daplka_resource_insert(minor_t, daplka_resource_t *);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic daplka_resource_t *daplka_resource_remove(minor_t rnum);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic daplka_resource_t *daplka_resource_lookup(minor_t);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic void daplka_resource_init(void);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic void daplka_resource_fini(void);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic struct daplka_resource_table daplka_resource;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * hash table routines
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic int daplka_hash_insert(daplka_hash_table_t *, uint64_t *, void *);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic int daplka_hash_remove(daplka_hash_table_t *, uint64_t, void **);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic void daplka_hash_walk(daplka_hash_table_t *, int (*)(void *, void *),
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic void *daplka_hash_lookup(daplka_hash_table_t *, uint64_t);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic int daplka_hash_create(daplka_hash_table_t *, uint_t,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor void (*)(void *), void (*)(void *));
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic void daplka_hash_destroy(daplka_hash_table_t *);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic uint32_t daplka_hash_getsize(daplka_hash_table_t *);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic void daplka_hash_generic_lookup(void *);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * async event handlers
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic void daplka_async_event_create(ibt_async_code_t, ibt_async_event_t *,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic void daplka_rc_async_handler(void *, ibt_hca_hdl_t, ibt_async_code_t,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic void daplka_cq_async_handler(void *, ibt_hca_hdl_t, ibt_async_code_t,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic void daplka_un_async_handler(void *, ibt_hca_hdl_t, ibt_async_code_t,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic void daplka_async_handler(void *, ibt_hca_hdl_t, ibt_async_code_t,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic void daplka_sm_notice_handler(void *, ib_gid_t, ibt_subnet_event_code_t,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic void daplka_sm_gid_avail(ib_gid_t *, ib_gid_t *);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * IBTF wrappers and default limits used for resource accounting
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic boolean_t daplka_accounting_enabled = B_TRUE;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylordaplka_ibt_alloc_rc_channel(daplka_ep_resource_t *, ibt_hca_hdl_t,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ibt_chan_alloc_flags_t, ibt_rc_chan_alloc_args_t *,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylordaplka_ibt_free_channel(daplka_ep_resource_t *, ibt_channel_hdl_t);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylordaplka_ibt_alloc_cq(daplka_evd_resource_t *, ibt_hca_hdl_t,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylordaplka_ibt_free_cq(daplka_evd_resource_t *, ibt_cq_hdl_t);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylordaplka_ibt_alloc_pd(daplka_pd_resource_t *, ibt_hca_hdl_t,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylordaplka_ibt_free_pd(daplka_pd_resource_t *, ibt_hca_hdl_t, ibt_pd_hdl_t);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylordaplka_ibt_alloc_mw(daplka_mw_resource_t *, ibt_hca_hdl_t, ibt_pd_hdl_t,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylordaplka_ibt_free_mw(daplka_mw_resource_t *, ibt_hca_hdl_t, ibt_mw_hdl_t);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylordaplka_ibt_register_mr(daplka_mr_resource_t *, ibt_hca_hdl_t, ibt_pd_hdl_t,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ibt_mr_attr_t *, ibt_mr_hdl_t *, ibt_mr_desc_t *);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylordaplka_ibt_register_shared_mr(daplka_mr_resource_t *, ibt_hca_hdl_t,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ibt_mr_hdl_t, ibt_pd_hdl_t, ibt_smr_attr_t *, ibt_mr_hdl_t *,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylordaplka_ibt_deregister_mr(daplka_mr_resource_t *, ibt_hca_hdl_t, ibt_mr_hdl_t);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylordaplka_ibt_alloc_srq(daplka_srq_resource_t *, ibt_hca_hdl_t, ibt_srq_flags_t,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ibt_pd_hdl_t, ibt_srq_sizes_t *, ibt_srq_hdl_t *, ibt_srq_sizes_t *);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylordaplka_ibt_free_srq(daplka_srq_resource_t *, ibt_srq_hdl_t);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * macros for manipulating resource objects.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * these macros can be used on objects that begin with a
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * daplka_resource_t header.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor#define DAPLKA_RS_REFCNT(rp) ((rp)->header.rs_refcnt)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor (rp)->header.rs_free((daplka_resource_t *)rp); \
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor#define DAPLKA_RS_INIT(rp, type, rnum, free_func) { \
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor atomic_add_32(&(rp)->header.rs_charged, (cnt)); \
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor atomic_add_32(&(rp)->header.rs_charged, -(cnt)); \
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor#define DAPLKA_RS_ACCT_CHARGED(rp) ((rp)->header.rs_charged)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor#define DAPLKA_RS_RNUM(rp) ((rp)->header.rs_rnum)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor#define DAPLKA_RS_TYPE(rp) ((rp)->header.rs_type)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor#define DAPLKA_RS_RESERVED(rp) ((intptr_t)(rp) == DAPLKA_RC_RESERVED)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * depending on the timeout value does a cv_wait_sig or cv_timedwait_sig
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ((timeout) == LONG_MAX) ? cv_wait_sig((cvp), (mp)) : \
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor#define DAPLKA_HOLD_HCA_WITHOUT_LOCK(hca) ((hca)->hca_ref_cnt++)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor#define DAPLKA_RELE_HCA_WITHOUT_LOCK(hca) ((hca)->hca_ref_cnt--)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor 0, /* devo_refcnt */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Module linkage information for the kernel.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor "uDAPL Service Driver",
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor MODREV_1, { (void *) &modldrv, NULL, NULL, NULL, NULL, NULL, NULL }
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor MODREV_1, { (void *) &modldrv, NULL, NULL, NULL }
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * daplka_dev holds global driver state and a list of HCAs
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * global SP hash table
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic daplka_hash_table_t daplka_global_sp_htbl;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * timer_info hash table
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic daplka_hash_table_t daplka_timer_info_htbl;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * shared MR avl tree
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic int daplka_shared_mr_cmp(const void *, const void *);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * default kmem flags used by this driver
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * taskq used for handling background tasks
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * daplka_cm_delay is the length of time the active
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * side needs to wait before timing out on the REP message.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * modunload will fail if pending_close is non-zero
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic struct ibt_clnt_modinfo_s daplka_clnt_modinfo = {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Module Installation
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor status = ddi_soft_state_init(&daplka_state, sizeof (daplka_t), 1);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor mutex_init(&daplka_dbglock, NULL, MUTEX_DRIVER, NULL);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* undo inits done before mod_install */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Module Removal
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * mod_remove causes detach to be called
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("fini: mod_remove failed: 0x%x\n", status);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Return Module Info.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylordaplka_enqueue_hca(daplka_t *dp, daplka_hca_t *hca)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylordaplka_dequeue_hca(daplka_t *dp, daplka_hca_t *hca)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylordaplka_init_hca(daplka_t *dp, ib_guid_t hca_guid)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor hca = kmem_zalloc(sizeof (daplka_hca_t), KM_SLEEP);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * open the HCA for use
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor status = ibt_open_hca(dp->daplka_clnt_hdl, hca_guid, &hca->hca_hdl);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("ibt_open_hca() returned IBT_HCA_IN_USE\n");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * query HCA to get its info
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor status = ibt_query_hca(hca->hca_hdl, &hca->hca_attr);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("ibt_query_hca returned %d (hca_guid 0x%llx)\n",
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * query HCA to get info of all ports
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor "guid %016llx\n",
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * this function obtains the list of HCAs from IBTF.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the HCAs are then opened and the returned handles
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * and attributes are stored into the global daplka_dev
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * structure.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * get the num & list of HCAs present
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * get the info for each available HCA
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor for (i = 0; i < hca_count; i++)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ibt_free_portinfo(hca->hca_ports, hca->hca_pinfosz);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * closes all HCAs and frees up the HCA list
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor while ((hca = dp->daplka_hca_list_head) != NULL) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor if ((status = daplka_fini_hca(dp, hca)) != IBT_SUCCESS)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Attach the device, create and fill in daplka_dev
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylordaplka_attach(dev_info_t *dip, ddi_attach_cmd_t cmd)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Allocate soft data structure
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor if (ddi_soft_state_zalloc(daplka_state, instance) != DDI_SUCCESS) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor dp = ddi_get_soft_state(daplka_state, instance);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Stuff private info into dip.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor mutex_init(&dp->daplka_mutex, NULL, MUTEX_DRIVER, NULL);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Register driver with IBTF
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor retval = ibt_attach(&daplka_clnt_modinfo, dip, dp,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("attach: ibt_attach failed: error = %d\n", retval);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Register to receive SM events */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ibt_register_subnet_notices(dp->daplka_clnt_hdl,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("attach: hca_init failed: error = %d\n", retval);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * this table is used by cr_handoff
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor retval = daplka_hash_create(&daplka_global_sp_htbl,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * this table stores per EP timer information.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * timer_info_t objects are inserted into this table whenever
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * a EP timer is set. timers get removed when they expire
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * or when they get cancelled.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor retval = daplka_hash_create(&daplka_timer_info_htbl,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DAPLKA_TIMER_HTBL_SZ, daplka_hash_timer_free, NULL);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("attach: cannot create timer hash table\n");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * this taskq is currently only used for processing timers.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * other processing may also use this taskq in the future.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor daplka_taskq = taskq_create(DAPLKA_DRV_NAME, DAPLKA_TQ_NTHREADS,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor maxclsyspri, 1, DAPLKA_TQ_NTHREADS, TASKQ_DYNAMIC);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * daplka_shared_mr_tree holds daplka_shared_mr_t objects that
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * gets retrieved or created when daplka_mr_register_shared is
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor mutex_init(&daplka_shared_mr_lock, NULL, MUTEX_DRIVER, NULL);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor avl_create(&daplka_shared_mr_tree, daplka_shared_mr_cmp,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Create the filesystem device node.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor if (ddi_create_minor_node(dip, DAPLKA_MINOR_NAME, S_IFCHR,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* unregister SM event notification */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ibt_register_subnet_notices(dp->daplka_clnt_hdl,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor if (dp->daplka_status == DAPLKA_STATE_ATTACHED) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Detach - Free resources allocated in attach
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor/* ARGSUSED */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylordaplka_detach(dev_info_t *dip, ddi_detach_cmd_t cmd)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor dp = ddi_get_soft_state(daplka_state, instance);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* unregister SM event notification */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ibt_register_subnet_notices(dp->daplka_clnt_hdl,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor if (dp->daplka_status == DAPLKA_STATE_ATTACHED) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * by the time we get here, all clients of dapl should
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * have exited and completed their cleanup properly.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * we can assert that all global data structures are now
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ASSERT(avl_destroy_nodes(&daplka_shared_mr_tree, &cookie) == NULL);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ASSERT(daplka_hash_getsize(&daplka_timer_info_htbl) == 0);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ASSERT(daplka_hash_getsize(&daplka_global_sp_htbl) == 0);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor/* ARGSUSED */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylordaplka_info(dev_info_t *dip, ddi_info_cmd_t infocmd, void *arg, void **result)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * creates a EP resource.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * A EP resource contains a RC channel. A EP resource holds a
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * reference to a send_evd (for the send CQ), recv_evd (for the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * recv CQ), a connection evd and a PD. These references ensure
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * that the referenced resources are not freed until the EP itself
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * gets freed.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor/* ARGSUSED */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylordaplka_ep_create(daplka_ia_resource_t *ia_rp, intptr_t arg, int mode,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor retval = ddi_copyin((void *)arg, &args, sizeof (dapl_ep_create_t),
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ep_rp = kmem_zalloc(sizeof (daplka_ep_resource_t), daplka_km_flags);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor mutex_init(&ep_rp->ep_lock, NULL, MUTEX_DRIVER, NULL);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * we don't have to use ep_get_state here because ep_rp is not in
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * ep_htbl yet. refer to the description of daplka_ep_set_state
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * for details about the EP state machine.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ep_rp->ep_state = DAPLKA_EP_STATE_TRANSITIONING;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* get reference to send evd and get cq handle */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor daplka_hash_lookup(&ia_rp->ia_evd_htbl, args.ep_snd_evd_hkey);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor chan_args.rc_scq = ep_rp->ep_snd_evd->evd_cq_hdl;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* get reference to recv evd and get cq handle */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor daplka_hash_lookup(&ia_rp->ia_evd_htbl, args.ep_rcv_evd_hkey);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor chan_args.rc_rcq = ep_rp->ep_rcv_evd->evd_cq_hdl;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* get reference to conn evd */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor daplka_hash_lookup(&ia_rp->ia_evd_htbl, args.ep_conn_evd_hkey);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* get reference to SRQ if needed */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ep_rp->ep_srq_res = (daplka_srq_resource_t *)daplka_hash_lookup(
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ASSERT(DAPLKA_RS_TYPE(ep_rp->ep_srq_res) == DAPL_TYPE_SRQ);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor D3("ep_create: ep_srq %p %llx\n", ep_rp->ep_srq_res,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* get pd handle */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor daplka_hash_lookup(&ia_rp->ia_pd_htbl, args.ep_pd_hkey);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * these checks ensure that the requested channel sizes
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * are within the limits supported by the chosen HCA.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor if (args.ep_ch_sizes.dcs_sq_sgl > hca_attrp->hca_max_sgl) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor if (args.ep_ch_sizes.dcs_rq_sgl > hca_attrp->hca_max_sgl) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor if (args.ep_ch_sizes.dcs_sq > hca_attrp->hca_max_chan_sz) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor if (args.ep_ch_sizes.dcs_rq > hca_attrp->hca_max_chan_sz) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor chan_args.rc_sizes.cs_sq_sgl = args.ep_ch_sizes.dcs_sq_sgl;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor chan_args.rc_sizes.cs_rq_sgl = args.ep_ch_sizes.dcs_rq_sgl;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor chan_args.rc_sizes.cs_sq = args.ep_ch_sizes.dcs_sq;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor chan_args.rc_sizes.cs_rq = args.ep_ch_sizes.dcs_rq;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor chan_args.rc_control = IBT_CEP_RDMA_RD | IBT_CEP_RDMA_WR;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor D3("ep_create: sq_sgl %d, rq_sgl %d, sq %d, rq %d, "
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor "sig_type 0x%x, control 0x%x, portnum %d, clone_chan 0x%p\n",
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor args.ep_ch_sizes.dcs_sq_sgl, args.ep_ch_sizes.dcs_rq_sgl,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor args.ep_ch_sizes.dcs_sq, args.ep_ch_sizes.dcs_rq,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor chan_args.rc_hca_port_num, chan_args.rc_clone_chan);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor achan_flags = IBT_ACHAN_USER_MAP | IBT_ACHAN_USES_SRQ;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* create rc channel */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor status = daplka_ibt_alloc_rc_channel(ep_rp, ia_rp->ia_hca_hdl,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("ep_create: alloc_rc_channel returned %d\n", status);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor args.ep_ch_real_sizes.dcs_sq = chan_real_sizes.cs_sq;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor args.ep_ch_real_sizes.dcs_rq = chan_real_sizes.cs_rq;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor args.ep_ch_real_sizes.dcs_sq_sgl = chan_real_sizes.cs_sq_sgl;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor args.ep_ch_real_sizes.dcs_rq_sgl = chan_real_sizes.cs_rq_sgl;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * store ep ptr with chan_hdl.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * this ep_ptr is used by the CM handlers (both active and
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * mutex is only needed for race of "destroy" and "async"
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ibt_set_chan_private(ep_rp->ep_chan_hdl, (void *)ep_rp);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Get HCA-specific data_out info */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor IBT_CI_NO_FLAGS, IBT_HDL_CHANNEL, (void *)ep_rp->ep_chan_hdl,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor &args.ep_qp_data_out, sizeof (args.ep_qp_data_out));
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* insert into ep hash table */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("ep_create: cannot insert ep resource into ep_htbl\n");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * at this point, the ep_rp can be looked up by other threads
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * if they manage to guess the correct hkey. but they are not
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * permitted to operate on ep_rp until we transition to the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * CLOSED state.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* return hkey to library */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor retval = ddi_copyout(&args, (void *)arg, sizeof (dapl_ep_create_t),
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor daplka_ep_set_state(ep_rp, old_state, new_state);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor (void) daplka_hash_remove(&ia_rp->ia_ep_htbl, ep_hkey,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * this case is impossible because ep_free will
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * wait until our state transition is complete.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("ep_create: cannot remove ep from hash table\n");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor daplka_ep_set_state(ep_rp, old_state, new_state);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * daplka_ep_get_state retrieves the current state of the EP and
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * sets the state to TRANSITIONING. if the current state is already
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * TRANSITIONING, this function will wait until the state becomes one
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * of the other EP states. Most of the EP related ioctls follow the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * call sequence:
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * new_state = old_state = daplka_ep_get_state(ep_rp);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * ...some code that affects the EP
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * new_state = <NEW_STATE>;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * daplka_ep_set_state(ep_rp, old_state, new_state);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * this call sequence ensures that only one thread may access the EP
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * during the time ep_state is in TRANSITIONING. daplka_ep_set_state
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * transitions ep_state to new_state and wakes up any waiters blocking
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor while (ep_rp->ep_state == DAPLKA_EP_STATE_TRANSITIONING) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor D2("get_state: wait for state transition to complete\n");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor D2("get_state: done, curr state = %d\n", ep_rp->ep_state);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ASSERT(ep_rp->ep_state != DAPLKA_EP_STATE_TRANSITIONING);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * an ep that is in the FREED state cannot transition
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * back to any of the regular states
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ep_rp->ep_state = DAPLKA_EP_STATE_TRANSITIONING;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * EP state transition diagram
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * CLOSED<-------------------
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * ------------------------ |
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * CONNECTING ACCEPTING |
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * | | | | | |
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * | | | | | |
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * | | | | | |
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * | | |_______|_______| |
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * | | | | | |
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * | |___________| | | |
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * | v | |---->DISCONNECTED
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * | CONNECTED | ^
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * ABORTING |---------|--------------|
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * | |-------->DISCONNECTING--|
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * |---------------------------------|
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * *not shown in this diagram:
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * -loopback transitions
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * -transitions to the FREED state
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylordaplka_ep_transition_is_valid(uint32_t old_state, uint32_t new_state)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * reseting to the same state is a no-op and is always
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * permitted. transitioning to the FREED state indicates
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * that the ep is about to be freed and no further operation
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * is allowed on it. to support abrupt close, the ep is
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * permitted to transition to the FREED state from any state.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * this is the initial ep_state.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * a transition to CONNECTING or ACCEPTING may occur
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * upon calling daplka_ep_connect or daplka_cr_accept,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * respectively.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * we transition to this state if daplka_ep_connect
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * is successful. from this state, we can transition
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * to CONNECTED if daplka_cm_rc_conn_est gets called;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * or to DISCONNECTED if daplka_cm_rc_conn_closed or
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * daplka_cm_rc_event_failure gets called. If the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * client calls daplka_ep_disconnect, we transition
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * to DISCONNECTING. If a timer was set at ep_connect
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * time and if the timer expires prior to any of the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * CM callbacks, we transition to ABORTING and then
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * to DISCONNECTED.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * we transition to this state if daplka_cr_accept
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * is successful. from this state, we can transition
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * to CONNECTED if daplka_cm_service_conn_est gets called;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * or to DISCONNECTED if daplka_cm_service_conn_closed or
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * daplka_cm_service_event_failure gets called. If the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * client calls daplka_ep_disconnect, we transition to
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * DISCONNECTING.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * we transition to this state if a active or passive
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * connection gets established. if the client calls
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * daplka_ep_disconnect, we transition to the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * DISCONNECTING state. subsequent CM callbacks will
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * cause ep_state to be set to DISCONNECTED. If the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * remote peer terminates the connection before we do,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * it is possible for us to transition directly from
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * CONNECTED to DISCONNECTED.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor if (new_state == DAPLKA_EP_STATE_DISCONNECTING ||
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * we transition to this state if the client calls
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * daplka_ep_disconnect.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor if (new_state == DAPLKA_EP_STATE_DISCONNECTED) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * we transition to this state if the active side
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * EP timer has expired. this is only a transient
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * state that is set during timer processing. when
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * timer processing completes, ep_state will become
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * DISCONNECTED.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor if (new_state == DAPLKA_EP_STATE_DISCONNECTED) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * we transition to this state if we get a closed
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * or event_failure CM callback. an expired timer
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * can also cause us to be in this state. this
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * is the only state in which we permit the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * ep_reinit operation.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("ep_transition: invalid state change %d -> %d\n",
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * first check if the transition is valid. then set ep_state
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * to new_state and wake up all waiters.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylordaplka_ep_set_state(daplka_ep_resource_t *ep_rp, uint32_t old_state,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ASSERT(new_state != DAPLKA_EP_STATE_TRANSITIONING);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor valid = daplka_ep_transition_is_valid(old_state, new_state);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * this case is impossible.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * we have a serious problem if we get here.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * instead of panicing, we reset the state to
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * old_state. doing this would at least prevent
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * threads from hanging due to ep_state being
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * stuck in TRANSITIONING.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * modifies RC channel attributes.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * currently, only the rdma_in and rdma_out attributes may
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * be modified. the channel must be in quiescent state when
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * this function is called.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor/* ARGSUSED */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylordaplka_ep_modify(daplka_ia_resource_t *ia_rp, intptr_t arg, int mode,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor retval = ddi_copyin((void *)arg, &args, sizeof (dapl_ep_modify_t),
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor daplka_hash_lookup(&ia_rp->ia_ep_htbl, args.epm_hkey);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor new_state = old_state = daplka_ep_get_state(ep_rp);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("ep_modify: invalid state %d\n", old_state);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor good_flags = IBT_CEP_SET_RDMARA_OUT | IBT_CEP_SET_RDMARA_IN;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("ep_modify: invalid flags 0x%x\n", args.epm_flags);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor bzero(&rcm_attr, sizeof (ibt_rc_chan_modify_attr_t));
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor if ((args.epm_flags & IBT_CEP_SET_RDMARA_OUT) != 0) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor if (args.epm_rdma_ra_out > hca_attrp->hca_max_rdma_out_chan) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor if ((args.epm_flags & IBT_CEP_SET_RDMARA_IN) != 0) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor if (args.epm_rdma_ra_in > hca_attrp->hca_max_rdma_in_chan) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor status = ibt_modify_rc_channel(ep_rp->ep_chan_hdl, args.epm_flags,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("ep_modify: modify_rc_channel returned %d\n", status);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * ep_modify does not change ep_state
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor daplka_ep_set_state(ep_rp, old_state, new_state);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Frees a EP resource.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * a EP may only be freed when it is in the CLOSED or
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * DISCONNECTED state.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor/* ARGSUSED */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylordaplka_ep_free(daplka_ia_resource_t *ia_rp, intptr_t arg, int mode,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor retval = ddi_copyin((void *)arg, &args, sizeof (dapl_ep_free_t), mode);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor daplka_hash_lookup(&ia_rp->ia_ep_htbl, args.epf_hkey);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor new_state = old_state = daplka_ep_get_state(ep_rp);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * ep cannot be freed if it is in an invalid state.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * this is only possible if we have two threads
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * calling ep_free in parallel.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* there should not be any outstanding timers */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor daplka_ep_set_state(ep_rp, old_state, new_state);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* remove reference obtained by lookup */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* UNREF calls the actual free function when refcnt is zero */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor daplka_ep_set_state(ep_rp, old_state, new_state);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* remove reference obtained by lookup */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * The following routines supports the timeout feature of ep_connect.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Refer to the description of ep_connect for details.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * this is the timer processing thread.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor daplka_timer_info_t *timerp = (daplka_timer_info_t *)arg;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor new_state = old_state = daplka_ep_get_state(ep_rp);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* unblock hash_ep_free */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* reset state to original state */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor daplka_ep_set_state(ep_rp, old_state, new_state);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* this function will also unref ep_rp */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * we cannot keep ep_state in TRANSITIONING if we call
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * ibt_close_rc_channel in blocking mode. this would cause
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * a deadlock because the cm callbacks will be blocked and
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * will not be able to wake us up.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor daplka_ep_set_state(ep_rp, old_state, new_state);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * when we return from close_rc_channel, all callbacks should have
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * completed. we can also be certain that these callbacks did not
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * enqueue any events to conn_evd.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor status = ibt_close_rc_channel(ep_rp->ep_chan_hdl, IBT_BLOCKING,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("timer_thread: ibt_close_rc_channel returned %d\n",
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * this is the only thread that can transition ep_state out
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * of ABORTING. all other ep operations would fail when
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * ep_state is in ABORTING.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor disc_ev = kmem_zalloc(sizeof (daplka_evd_event_t), KM_SLEEP);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor disc_ev->ee_cmev.ec_cm_ev_type = DAPL_IB_CME_TIMED_OUT;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor disc_ev->ee_cmev.ec_cm_cookie = ep_rp->ep_cookie;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor D2("timer_thread: enqueue event(%p) evdp(%p)\n",
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor daplka_ep_set_state(ep_rp, old_state, new_state);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* this function will also unref ep_rp */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * dispatches a thread to continue with timer processing.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * keep rescheduling this function until
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * taskq_dispatch succeeds.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("timer_dispatch: taskq_dispatch failed, retrying...\n");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * this function is called by the kernel's callout thread.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * we first attempt to remove the timer object from the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * global timer table. if it is found, we dispatch a thread
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * to continue processing the timer object. if it is not
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * found, that means the timer has been cancelled by someone
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor D2("timer_handler: timer_hkey 0x%llx\n", (longlong_t)timer_hkey);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor (void) daplka_hash_remove(&daplka_timer_info_htbl,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * allocates a timer_info object.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * a reference to a EP is held by this object. this ensures
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * that the EP stays valid when a timer is outstanding.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylordaplka_timer_info_alloc(daplka_ep_resource_t *ep_rp)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor timerp = kmem_zalloc(sizeof (*timerp), daplka_km_flags);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("timer_info_alloc: cannot allocate timer info\n");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Frees the timer_info object.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * we release the EP reference before freeing the object.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylordaplka_timer_info_free(daplka_timer_info_t *timerp)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * cancels the timer set by ep_connect.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * returns -1 if timer handling is in progress
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * and 0 otherwise.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * this function can only be called when ep_state
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * is frozen.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ASSERT(ep_rp->ep_state == DAPLKA_EP_STATE_TRANSITIONING);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor (void) daplka_hash_remove(&daplka_timer_info_htbl,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * this is possible if the timer_handler has
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * removed the timerp but the taskq thread has
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * not transitioned the ep_state to DISCONNECTED.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * we need to reset the ep_state to allow the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * taskq thread to continue with its work. the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * taskq thread will set the ep_timer_hkey to 0
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * so we don't have to do it here.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("cancel_timer: timer is being processed\n");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor return (-1);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * we got the timer object. if the handler fires at
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * this point, it will not be able to find the object
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * and will return immediately. normally, ti_tmo_id gets
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * cleared when the handler fires.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * note that untimeout can possibly call the handler.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * we are safe because the handler will be a no-op.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * this function is called by daplka_hash_destroy for
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * freeing timer_info objects
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor daplka_timer_info_free((daplka_timer_info_t *)obj);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor/* ARGSUSED */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor for (i = 0; i < sizeof (DAPL_PRIVATE); i++) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * ep_connect is called by the client to initiate a connection to a
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * remote service point. It is a non-blocking call. If a non-zero
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * timeout is specified by the client, a timer will be set just before
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * returning from ep_connect. Upon a successful return from ep_connect,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the client will call evd_wait to wait for the connection to complete.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * If the connection is rejected or has failed due to an error, the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * client will be notified with an event containing the appropriate error
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * code. If the connection is accepted, the client will be notified with
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the CONN_ESTABLISHED event. If the timer expires before either of the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * above events (error or established), a TIMED_OUT event will be delivered
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * to the client.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the complicated part of the timer logic is the handling of race
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * conditions with CM callbacks. we need to ensure that either the CM or
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the timer thread gets to deliver an event, but not both. when the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * CM callback is about to deliver an event, it always tries to cancel
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the outstanding timer. if cancel_timer indicates a that the timer is
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * already being processed, the CM callback will simply return without
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * delivering an event. when the timer thread executes, it tries to check
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * if the EP is still in CONNECTING state (timers only work on the active
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * side). if the EP is not in this state, the timer thread will return
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * without delivering an event.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor/* ARGSUSED */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylordaplka_ep_connect(daplka_ia_resource_t *ia_rp, intptr_t arg, int mode,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor retval = ddi_copyin((void *)arg, &args, sizeof (dapl_ep_connect_t),
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor daplka_hash_lookup(&ia_rp->ia_ep_htbl, args.epc_hkey);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor new_state = old_state = daplka_ep_get_state(ep_rp);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("ep_connect: invalid state %d\n", old_state);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor if (args.epc_priv_sz > DAPL_MAX_PRIVATE_DATA_SIZE) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("ep_connect: private data len (%d) exceeded "
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * check for remote ipaddress to dgid resolution needs ATS
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor#endif /* DAPLKA_DEBUG_FORCE_ATS */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* check for unidentified dgid */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * setup for ibt_query_ar()
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor status = ibt_query_ar(sgid, &ar_query_s, &ar_result_s);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("ep_connect: ibt_query_ar returned %d\n", status);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * dgid identified from SA record
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor (longlong_t)dgid->gid_prefix, (longlong_t)dgid->gid_guid);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor bzero(&chan_args, sizeof (ibt_chan_open_args_t));
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * don't set sid in path_attr saves 1 SA query
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Also makes server side not to write the service record
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* save the connection ep - struct copy */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* enable APM on remote port but not on loopback case */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor if (daplka_apm && ((dgid->gid_prefix != path_attr.pa_sgid.gid_prefix) ||
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor (dgid->gid_guid != path_attr.pa_sgid.gid_guid))) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor status = ibt_get_paths(daplka_dev->daplka_clnt_hdl,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor pathflags, &path_attr, 1, &path_info, &num_paths);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor if (status != IBT_SUCCESS && status != IBT_INSUFF_DATA) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("ep_connect: ibt_get_paths returned %d paths %d\n",
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* fill in the sid directly to path_info */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* fill in open channel args */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor chan_args.oc_rdma_ra_out = hca_attrp->hca_max_rdma_out_chan;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor chan_args.oc_rdma_ra_in = hca_attrp->hca_max_rdma_in_chan;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor chan_args.oc_path_retry_cnt = 7; /* 3-bit field */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor chan_args.oc_path_rnr_retry_cnt = IBT_RNR_INFINITE_RETRY;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * calculate checksum value of hello message and
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * put hello message in networking byte order
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor dp->hello_msg.hi_port = htons(dp->hello_msg.hi_port);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor dp->hello_msg.hi_checksum = htons(daplka_hellomsg_cksum(dp));
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * increment refcnt before passing reference to
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * timer_info_alloc.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * we need to remove the reference if
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * allocation failed.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * We generate our own hkeys so that timer_hkey can fit
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * into a pointer and passed as an arg to timeout()
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor retval = daplka_hash_insert(&daplka_timer_info_htbl,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor status = ibt_open_rc_channel(ep_rp->ep_chan_hdl, IBT_OCHAN_NO_FLAGS,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("ep_connect: ibt_open_rc_channel returned %d\n", status);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * if a cm callback gets called at this point, it'll have to wait until
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * ep_state becomes connecting (or some other state if another thread
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * manages to get ahead of the callback). this guarantees that the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * callback will not touch the timer until it gets set.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * We generate our own 32 bit timer_hkey so that it can fit
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * into a pointer
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor timerp->ti_tmo_id = timeout(daplka_timer_handler,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor if (timerp != NULL && (retval != 0 || status != IBT_SUCCESS)) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * if ibt_open_rc_channel failed, the timerp must still
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * be in daplka_timer_info_htbl because neither the cm
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * callback nor the timer_handler will be called.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor (void) daplka_hash_remove(&daplka_timer_info_htbl,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor daplka_ep_set_state(ep_rp, old_state, new_state);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * ep_disconnect closes a connection with a remote peer.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * if a connection has not been established, ep_disconnect
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * will instead flush all recv bufs posted to this channel.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * if the EP state is CONNECTED, CONNECTING or ACCEPTING upon
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * entry to ep_disconnect, the EP state will transition to
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * DISCONNECTING upon exit. the CM callbacks triggered by
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * ibt_close_rc_channel will cause EP state to become
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * DISCONNECTED. This function is a no-op if EP state is
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * DISCONNECTED.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor/* ARGSUSED */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylordaplka_ep_disconnect(daplka_ia_resource_t *ia_rp, intptr_t arg, int mode,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor retval = ddi_copyin((void *)arg, &args, sizeof (dapl_ep_disconnect_t),
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("ep_disconnect: copyin error %d\n", retval);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor daplka_hash_lookup(&ia_rp->ia_ep_htbl, args.epd_hkey);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("ep_disconnect: cannot find ep resource\n");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor new_state = old_state = daplka_ep_get_state(ep_rp);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("ep_disconnect: invalid state %d\n", old_state);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor if ((old_state == DAPLKA_EP_STATE_DISCONNECTED) ||
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* we leave the state as DISCONNECTED */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor D2("ep_disconnect: aborting, old_state = %d\n", old_state);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * according to the udapl spec, ep_disconnect should
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * flush the channel if the channel is not CONNECTED.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("ep_disconnect: ibt_flush_channel failed %d\n",
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* we leave the state as CLOSED */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor daplka_ep_set_state(ep_rp, old_state, new_state);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor status = ibt_close_rc_channel(ep_rp->ep_chan_hdl, IBT_NONBLOCKING,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("ep_disconnect: ibt_close_rc_channel returned %d\n",
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor daplka_ep_set_state(ep_rp, old_state, new_state);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * this function resets the EP to a usable state (ie. from
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * DISCONNECTED to CLOSED). this function is best implemented using
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the ibt_recycle_channel interface. until that is available, we will
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * instead clone and tear down the existing channel and replace the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * existing channel with the cloned one.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor/* ARGSUSED */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylordaplka_ep_reinit(daplka_ia_resource_t *ia_rp, intptr_t arg, int mode,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor retval = ddi_copyin((void *)arg, &args, sizeof (dapl_ep_reinit_t),
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor daplka_hash_lookup(&ia_rp->ia_ep_htbl, args.epri_hkey);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor new_state = old_state = daplka_ep_get_state(ep_rp);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor daplka_ep_set_state(ep_rp, old_state, new_state);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * destroys a EP resource.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * called when refcnt drops to zero.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor daplka_ep_resource_t *ep_rp = (daplka_ep_resource_t *)gen_rp;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ASSERT(ep_rp->ep_state == DAPLKA_EP_STATE_FREED);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * by the time we get here, we can be sure that
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * there is no outstanding timer.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor D3("ep_destroy: entering, ep_rp 0x%p, rnum %d\n",
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * free rc channel
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor status = daplka_ibt_free_channel(ep_rp, ep_rp->ep_chan_hdl);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor D3("ep_destroy: qp freed, rnum %d\n", DAPLKA_RS_RNUM(ep_rp));
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * release all references
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor kmem_free(ep_rp, sizeof (daplka_ep_resource_t));
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * this function is called by daplka_hash_destroy for
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * freeing EP resource objects
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor daplka_ep_resource_t *ep_rp = (daplka_ep_resource_t *)obj;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor daplka_ep_set_state(ep_rp, old_state, new_state);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* call ibt_close_rc_channel regardless of what state we are in */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor status = ibt_close_rc_channel(ep_rp->ep_chan_hdl, IBT_BLOCKING,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor "unexpected error %d from close_rc_channel\n",
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor D2("hash_ep_free: close_rc_channel, status %d\n", status);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * creates a EVD resource.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * a EVD is used by the client to wait for events from one
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * or more sources.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor/* ARGSUSED */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylordaplka_evd_create(daplka_ia_resource_t *ia_rp, intptr_t arg, int mode,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor retval = ddi_copyin((void *)arg, &args, sizeof (dapl_evd_create_t),
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ~(DAT_EVD_DEFAULT_FLAG | DAT_EVD_SOFTWARE_FLAG)) != 0) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("evd_create: invalid flags 0x%x\n", args.evd_flags);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor evd_rp = kmem_zalloc(sizeof (daplka_evd_resource_t), daplka_km_flags);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor mutex_init(&evd_rp->evd_lock, NULL, MUTEX_DRIVER, NULL);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor cv_init(&evd_rp->evd_cv, NULL, CV_DRIVER, NULL);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor evd_rp->evd_cr_events.eel_event_type = DAPLKA_EVD_CM_EVENTS;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor evd_rp->evd_conn_events.eel_event_type = DAPLKA_EVD_CM_EVENTS;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor evd_rp->evd_async_events.eel_event_type = DAPLKA_EVD_ASYNC_EVENTS;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * if the client specified a non-zero cno_hkey, we
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * lookup the cno and save the reference for later use.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ASSERT(DAPLKA_RS_TYPE(cno_rp) == DAPL_TYPE_CNO);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor (DAT_EVD_DTO_FLAG | DAT_EVD_RMR_BIND_FLAG)) != 0) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor if (args.evd_cq_size > hca_attrp->hca_max_cq_sz) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor status = daplka_ibt_alloc_cq(evd_rp, evd_rp->evd_hca_hdl,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor &cq_attr, &evd_rp->evd_cq_hdl, &evd_rp->evd_cq_real_size);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("evd_create: ibt_alloc_cq returned %d", status);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * store evd ptr with cq_hdl
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * mutex is only needed for race of "destroy" and "async"
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ibt_set_cq_private(evd_rp->evd_cq_hdl, (void *)evd_rp);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Get HCA-specific data_out info */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor IBT_CI_NO_FLAGS, IBT_HDL_CQ, (void *)evd_rp->evd_cq_hdl,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor &args.evd_cq_data_out, sizeof (args.evd_cq_data_out));
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("evd_create: ibt_ci_data_out error(%d)", status);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor args.evd_cq_real_size = evd_rp->evd_cq_real_size;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ibt_set_cq_handler(evd_rp->evd_cq_hdl, daplka_cq_handler,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor retval = daplka_hash_insert(&ia_rp->ia_evd_htbl,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("evd_ceate: cannot insert evd %d\n", retval);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * If this evd handles async events need to add to the IA resource
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * async evd list
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor async_evd = kmem_zalloc(sizeof (daplka_async_evd_hkey_t),
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* add the evd to the head of the list */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor async_evd->aeh_next = ia_rp->ia_async_evd_hkeys;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor retval = copyout(&args, (void *)arg, sizeof (dapl_evd_create_t));
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor (void) daplka_hash_remove(&ia_rp->ia_evd_htbl, evd_hkey,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * we can only get here if another thread
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * has completed the cleanup in evd_free
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * resizes CQ and returns new mapping info to library.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor/* ARGSUSED */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylordaplka_cq_resize(daplka_ia_resource_t *ia_rp, intptr_t arg, int mode,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor retval = ddi_copyin((void *)arg, &args, sizeof (dapl_cq_resize_t),
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* get evd resource */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor daplka_hash_lookup(&ia_rp->ia_evd_htbl, args.cqr_evd_hkey);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ASSERT(DAPLKA_RS_TYPE(evd_rp) == DAPL_TYPE_EVD);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor if (args.cqr_cq_new_size > hca_attrp->hca_max_cq_sz) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("cq_resize: invalid cq size %d", args.cqr_cq_new_size);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * If ibt_resize_cq fails that it is primarily due to resource
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * shortage. Per IB spec resize will never loose events and
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * a resize error leaves the CQ intact. Therefore even if the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * resize request fails we proceed and get the mapping data
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * from the CQ so that the library can mmap it.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor status = ibt_resize_cq(evd_rp->evd_cq_hdl, args.cqr_cq_new_size,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* we return the size of the old CQ if resize fails */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor args.cqr_cq_real_size = evd_rp->evd_cq_real_size;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("cq_resize: ibt_resize_cq failed:%d\n", status);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor evd_rp->evd_cq_real_size = args.cqr_cq_real_size;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor D2("cq_resize(%d): done new_sz(%u) real_sz(%u)\n",
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Get HCA-specific data_out info */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor IBT_CI_NO_FLAGS, IBT_HDL_CQ, (void *)evd_rp->evd_cq_hdl,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor &args.cqr_cq_data_out, sizeof (args.cqr_cq_data_out));
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("cq_resize: ibt_ci_data_out error(%d)\n", status);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* return ibt_ci_data_out status */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor retval = ddi_copyout(&args, (void *)arg, sizeof (dapl_cq_resize_t),
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Routine to copyin the event poll message so that 32 bit libraries
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * can be safely supported
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylordaplka_event_poll_copyin(intptr_t inarg, dapl_event_poll_t *outarg, int mode)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor if ((mode & DATAMODEL_MASK) == DATAMODEL_ILP32) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("event_poll_copyin: 32bit error %d\n", retval);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor outarg->evp_ep = (dapl_ib_event_t *)(uintptr_t)args32.evp_ep;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor retval = ddi_copyin((void *)inarg, outarg, sizeof (dapl_event_poll_t),
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Routine to copyout the event poll message so that 32 bit libraries
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * can be safely supported
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylordaplka_event_poll_copyout(dapl_event_poll_t *inarg, intptr_t outarg, int mode)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor if ((mode & DATAMODEL_MASK) == DATAMODEL_ILP32) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor args32.evp_ep = (caddr32_t)(uintptr_t)inarg->evp_ep;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor retval = ddi_copyout((void *)&args32, (void *)outarg,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("event_poll_copyout: 32bit error %d\n", retval);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor retval = ddi_copyout((void *)inarg, (void *)outarg,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * fucntion to handle CM REQ RCV private data from Solaris or third parties
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor/* ARGSUSED */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylordaplka_crevent_privdata_post(daplka_ia_resource_t *ia_rp,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor dapl_ib_event_t *evd_rp, daplka_evd_event_t *cr_ev)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * get private data and len
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor dp = (DAPL_PRIVATE *)cr_ev->ee_cmev.ec_cm_ev_priv_data;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* skip the DAPL_PRIVATE chekcsum check */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* for remote connects */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* look up hello message in the CM private data area */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor (dp->hello_msg.hi_vers == DAPL_HELLO_MSG_VERS)) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor D2("daplka_crevent_privdata_post: Solaris msg\n");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor dp->hello_msg.hi_port = ntohs(dp->hello_msg.hi_port);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor bcopy(dp, evd_rp->ibe_ce.ibce_priv_data_ptr, clen);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor#endif /* DAPLKA_DEBUG_FORCE_ATS */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor D2("daplka_crevent_privdata_post: 3rd party msg\n");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* transpose CM private data into hello message */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor if (clen > DAPL_CONSUMER_MAX_PRIVATE_DATA_SIZE) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor bcopy(dp, evd_rp->ibe_ce.ibce_priv_data_ptr, clen);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor evd_rp->ibe_ce.ibce_priv_data_size = sizeof (DAPL_PRIVATE);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor dp = (DAPL_PRIVATE *)evd_rp->ibe_ce.ibce_priv_data_ptr;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * fill in hello message
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* assign sgid and dgid */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* reverse ip address lookup through ATS */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor status = ibt_query_ar(lgid, &ar_query_s, &ar_result_s);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor bcopy(ar_result_s.ar_data, hip->hi_saaddr, DAPL_ATS_NBYTES);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* determine the address families */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ipaddr_ord = hip->hi_v4pad[0] | hip->hi_v4pad[1] |
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor D3("daplka_privdata_post: family=%d :SA[8] %d.%d.%d.%d\n",
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor D3("daplka_privdata_post: SA[12] %d.%d.%d.%d\n",
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* non-conformed third parties */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * this function is called by evd_wait and evd_dequeue to wait for
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * connection events and CQ notifications. typically this function
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * is called when the userland CQ is empty and the client has
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * specified a non-zero timeout to evd_wait. if the client is
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * interested in CQ events, the CQ must be armed in userland prior
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * to calling this function.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor/* ARGSUSED */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylordaplka_event_poll(daplka_ia_resource_t *ia_rp, intptr_t arg, int mode,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor retval = daplka_event_poll_copyin(arg, &args, mode);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor if ((args.evp_num_ev > 0) && (args.evp_ep == NULL)) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("event_poll: evp_ep cannot be NULL if num_wc=%d",
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Note: dequeue requests have a threshold = 0, timeout = 0
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* ensure library is passing sensible values */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("event_poll: max_events(%d) < threshold(%d)\n",
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Do a sanity check to avoid excessive memory allocation */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor D4("event_poll: threshold(%d) timeout(0x%llx) max_events(%d)\n",
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor threshold, (longlong_t)args.evp_timeout, max_events);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* get evd resource */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor daplka_hash_lookup(&ia_rp->ia_evd_htbl, args.evp_evd_hkey);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ASSERT(DAPLKA_RS_TYPE(evd_rp) == DAPL_TYPE_EVD);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Use event array on the stack if possible
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor evp_size = max_events * sizeof (dapl_ib_event_t);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor evp_start = evp = kmem_zalloc(evp_size, daplka_km_flags);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("event_poll: kmem_zalloc failed, evp_size %d",
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * The Event poll algorithm is as follows -
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * The library passes a buffer big enough to hold "max_events"
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * events. max_events is >= threshold. If at any stage we get
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * max_events no. of events we bail. The events are polled in
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the following order -
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * 1) Check for CR events in the evd_cr_events list
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * 2) Check for Connection events in the evd_connection_events list
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * If after the above 2 steps we don't have enough(>= threshold) events
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * we block for CQ notification and sleep. Upon being woken up we start
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * at step 1 again.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Note: this could be 0 or INFINITE or anyother value in microsec
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * use the max value if we wrapped around
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * If this evd is waiting for CM events check that now.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* dequeue events from evd_cr_events list */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * populate the evp array
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor if ((evd_rp->evd_flags & DAT_EVD_CONNECTION_FLAG) &&
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor (evd_rp->evd_conn_events.eel_num_elements > 0)) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* dequeue events from evd_connection_events list */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * populate the evp array -
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor (evd_rp->evd_async_events.eel_num_elements > 0)) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* dequeue events from evd_async_events list */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * populate the evp array
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * We have sufficient events for this call so no need to wait
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor if ((threshold > 0) && (num_events >= threshold)) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * There are no new events and a timeout was specified.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Note: for CQ events threshold is 0 but timeout is
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * not necessarily 0.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor while ((evd_rp->evd_newevents == DAPLKA_EVD_NO_EVENTS) &&
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor if (evd_rp->evd_newevents != DAPLKA_EVD_NO_EVENTS) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * If we got woken up by the CQ handler due to events
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * in the CQ. Need to go to userland to check for
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * CQ events. Or if we were woken up due to S/W events
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* check for userland events only */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Clear newevents since we are going to loopback
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * back and check for both CM and CQ events
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor } else { /* error */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * At this point retval might have a value that we want to return
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * back to the user. So the copyouts shouldn't tamper retval.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor if (args.evp_num_polled > 0) { /* copyout the events */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor rc = ddi_copyout(evp, args.evp_ep, args.evp_num_polled *
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor if (rc != 0) { /* XXX: we are losing events here */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("event_poll: event array copyout error %d", rc);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor rc = daplka_event_poll_copyout(&args, arg, mode);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor if (rc != 0) { /* XXX: we are losing events here */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor if ((max_events > NUM_EVENTS_PER_POLL) && (evp_start != NULL)) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor/* ARGSUSED */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylordaplka_event_wakeup(daplka_ia_resource_t *ia_rp, intptr_t arg, int mode,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor retval = ddi_copyin((void *)arg, &args, sizeof (dapl_event_wakeup_t),
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("event_wakeup: copyin error %d\n", retval);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* get evd resource */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor daplka_hash_lookup(&ia_rp->ia_evd_htbl, args.evw_hkey);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("event_wakeup: cannot find evd resource\n");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ASSERT(DAPLKA_RS_TYPE(evd_rp) == DAPL_TYPE_EVD);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor/* ARGSUSED */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylordaplka_evd_modify_cno(daplka_ia_resource_t *ia_rp, intptr_t arg, int mode,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor retval = ddi_copyin((void *)arg, &args, sizeof (dapl_evd_modify_cno_t),
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("evd_modify_cno: copyin error %d\n", retval);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* get evd resource */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor daplka_hash_lookup(&ia_rp->ia_evd_htbl, args.evmc_hkey);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("evd_modify_cno: cannot find evd resource\n");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ASSERT(DAPLKA_RS_TYPE(evd_rp) == DAPL_TYPE_EVD);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* get cno resource corresponding to the new CNO */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("evd_modify_cno: cannot find CNO resource\n");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ASSERT(DAPLKA_RS_TYPE(cno_rp) == DAPL_TYPE_CNO);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * drop the refcnt on the old CNO, the refcnt on the new CNO is
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * retained since the evd holds a reference to it.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Frees the EVD and associated resources.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * If there are other threads still using this EVD, the destruction
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * will defer until the EVD's refcnt drops to zero.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor/* ARGSUSED */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylordaplka_evd_free(daplka_ia_resource_t *ia_rp, intptr_t arg, int mode,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor retval = ddi_copyin((void *)arg, &args, sizeof (dapl_evd_free_t), mode);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor retval = daplka_hash_remove(&ia_rp->ia_evd_htbl, args.evf_hkey,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ASSERT(DAPLKA_RS_TYPE(evd_rp) == DAPL_TYPE_EVD);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* If this is an async evd remove it from the IA's async evd list */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* unlink curr from the list */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * if first element in the list update
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the list head
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* free the curr entry */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor kmem_free(curr, sizeof (daplka_async_evd_hkey_t));
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* UNREF calls the actual free function when refcnt is zero */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * destroys EVD resource.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * called when refcnt drops to zero.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor daplka_evd_resource_t *evd_rp = (daplka_evd_resource_t *)gen_rp;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor D3("evd_destroy: entering, evd_rp 0x%p, rnum %d\n",
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ibt_set_cq_handler(evd_rp->evd_cq_hdl, NULL, NULL);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor status = daplka_ibt_free_cq(evd_rp, evd_rp->evd_cq_hdl);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("evd_destroy: ibt_free_cq returned %d\n", status);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor D2("evd_destroy: cq freed, rnum %d\n", DAPLKA_RS_RNUM(evd_rp));
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * release reference on CNO
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * discard all remaining events
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor while ((evt = daplka_evd_event_dequeue(&evd_rp->evd_cr_events))) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor kmem_free(evt->ee_cmev.ec_cm_ev_priv_data, len);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ASSERT(evd_rp->evd_cr_events.eel_num_elements == 0);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor while ((evt = daplka_evd_event_dequeue(&evd_rp->evd_conn_events))) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor kmem_free(evt->ee_cmev.ec_cm_ev_priv_data, len);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ASSERT(evd_rp->evd_conn_events.eel_num_elements == 0);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor while ((evt = daplka_evd_event_dequeue(&evd_rp->evd_async_events))) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("evd_destroy: discarding ASYNC event: %d\n",
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ASSERT(evd_rp->evd_async_events.eel_num_elements == 0);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor kmem_free(evd_rp, sizeof (daplka_evd_resource_t));
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor D3("evd_destroy: exiting, evd_rp 0x%p\n", evd_rp);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor daplka_evd_resource_t *evd_rp = (daplka_evd_resource_t *)obj;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ASSERT(DAPLKA_RS_TYPE(evd_rp) == DAPL_TYPE_EVD);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * this handler fires when new completions arrive.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor/* ARGSUSED */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylordaplka_cq_handler(ibt_cq_hdl_t ibt_cq, void *arg)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor D3("cq_handler: fired setting evd_newevents\n");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor daplka_evd_wakeup((daplka_evd_resource_t *)arg, NULL, NULL);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * this routine wakes up a client from evd_wait. if evtq and evt
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * are non-null, the event evt will be enqueued prior to waking
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * up the client. if the evd is associated with a CNO and if there
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * are no waiters on the evd, the CNO will be notified.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylordaplka_evd_wakeup(daplka_evd_resource_t *evd_rp, daplka_evd_event_list_t *evtq,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ASSERT((evtq->eel_event_type == DAPLKA_EVD_CM_EVENTS) ||
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor (evtq->eel_event_type == DAPLKA_EVD_ASYNC_EVENTS));
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor evd_rp->evd_newevents |= DAPLKA_EVD_ULAND_EVENTS;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * only wakeup the CNO if there are no waiters on this evd.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor if (evd_rp->evd_cno_res != NULL && waiters == 0) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor evd_rp->evd_cno_res->cno_evd_cookie = evd_rp->evd_cookie;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * daplka_evd_event_enqueue adds elem to the end of the event list
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * The caller is expected to acquire appropriate locks before
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * calling enqueue
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylordaplka_evd_event_enqueue(daplka_evd_event_list_t *evlist,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* list is empty */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * daplka_evd_event_dequeue removes and returns the first element of event
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * list. NULL is returned if the list is empty. The caller is expected to
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * acquire appropriate locks before calling enqueue.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylordaplka_evd_event_dequeue(daplka_evd_event_list_t *evlist)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* if it was the last element update the tail pointer too */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * A CNO allows the client to wait for notifications from multiple EVDs.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * To use a CNO, the client needs to follow the procedure below:
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * 1. allocate a CNO. this returns a cno_hkey that identifies the CNO.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * 2. create one or more EVDs using the returned cno_hkey.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * 3. call cno_wait. when one of the associated EVDs get notified, the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * CNO will also get notified. cno_wait will then return with a
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * evd_cookie identifying the EVD that triggered the event.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * A note about cno_wait:
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * -unlike a EVD, a CNO does not maintain a queue of notifications. For
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * example, suppose multiple EVDs triggered a CNO before the client calls
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * cno_wait; when the client calls cno_wait, it will return with the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * evd_cookie that identifies the *last* EVD that triggered the CNO. It
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * is the responsibility of the client, upon returning from cno_wait, to
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * check on all EVDs that can potentially trigger the CNO. the returned
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * evd_cookie is only meant to be a hint. there is no guarantee that the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * EVD identified by the evd_cookie still contains an event or still
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * exists by the time cno_wait returns.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * allocates a CNO.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the returned cno_hkey may subsequently be used in evd_create.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor/* ARGSUSED */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylordaplka_cno_alloc(daplka_ia_resource_t *ia_rp, intptr_t arg, int mode,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor cno_rp = kmem_zalloc(sizeof (*cno_rp), daplka_km_flags);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("cno_alloc: cannot allocate cno resource\n");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor mutex_init(&cno_rp->cno_lock, NULL, MUTEX_DRIVER, NULL);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor cv_init(&cno_rp->cno_cv, NULL, CV_DRIVER, NULL);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* insert into cno hash table */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor retval = daplka_hash_insert(&ia_rp->ia_cno_htbl,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("cno_alloc: cannot insert cno resource\n");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* return hkey to library */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor retval = ddi_copyout(&args, (void *)arg, sizeof (dapl_cno_alloc_t),
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor (void) daplka_hash_remove(&ia_rp->ia_cno_htbl, cno_hkey,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * we can only get here if another thread
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * has completed the cleanup in cno_free
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * destroys a CNO.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * this gets called when a CNO resource's refcnt drops to zero.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor daplka_cno_resource_t *cno_rp = (daplka_cno_resource_t *)gen_rp;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor D2("cno_destroy: entering, cno_rp %p, rnum %d\n",
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ASSERT(DAPLKA_RS_TYPE(cno_rp) == DAPL_TYPE_CNO);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor kmem_free(cno_rp, sizeof (daplka_cno_resource_t));
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor D2("cno_destroy: exiting, cno_rp %p\n", cno_rp);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor daplka_cno_resource_t *cno_rp = (daplka_cno_resource_t *)obj;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ASSERT(DAPLKA_RS_TYPE(cno_rp) == DAPL_TYPE_CNO);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * removes the CNO from the cno hash table and frees the CNO
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * if there are no references to it. if there are references to
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * it, the CNO will be destroyed when the last of the references
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * is released. once the CNO is removed from the cno hash table,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the client will no longer be able to call cno_wait on the CNO.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor/* ARGSUSED */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylordaplka_cno_free(daplka_ia_resource_t *ia_rp, intptr_t arg, int mode,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor retval = ddi_copyin((void *)arg, &args, sizeof (dapl_cno_free_t), mode);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor retval = daplka_hash_remove(&ia_rp->ia_cno_htbl,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ASSERT(DAPLKA_RS_TYPE(cno_rp) == DAPL_TYPE_CNO);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* UNREF calls the actual free function when refcnt is zero */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * wait for a notification from one of the associated EVDs.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor/* ARGSUSED */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylordaplka_cno_wait(daplka_ia_resource_t *ia_rp, intptr_t arg, int mode,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor retval = ddi_copyin((void *)arg, &args, sizeof (dapl_cno_wait_t), mode);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* get cno resource */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor daplka_hash_lookup(&ia_rp->ia_cno_htbl, args.cnw_hkey);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ASSERT(DAPLKA_RS_TYPE(cno_rp) == DAPL_TYPE_CNO);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor timeout = curr_time + drv_usectohz(args.cnw_timeout);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * use the max value if we wrapped around
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor if (args.cnw_timeout > 0 && timeout <= curr_time) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * clock_t (size long) changes between 32 and 64-bit kernels
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor retval = ddi_copyout((void *)&args, (void *)arg,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * this function is called by the client when it decides to
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * accept a connection request. a connection request is generated
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * when the active side generates REQ MAD to a service point on
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the destination node. this causes the CM service handler
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * (daplka_cm_service_req) on the passive side to be callee. This
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * handler will then enqueue this connection request to the backlog
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * array of the service point. A connection event containing the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * backlog array index and connection request private data is passed
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * to the client's service point EVD (sp_evd_res). once the event
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * is passed up to the userland, the client may examine the request
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * to decide whether to call daplka_cr_accept or dapka_cr_reject.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor/* ARGSUSED */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylordaplka_cr_accept(daplka_ia_resource_t *ia_rp, intptr_t arg, int mode,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor retval = ddi_copyin((void *)arg, &args, sizeof (dapl_cr_accept_t),
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor if (args.cra_priv_sz > DAPL_MAX_PRIVATE_DATA_SIZE) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("cr_accept: private data len (%d) exceeded "
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor priv_data = (args.cra_priv_sz > 0) ? (void *)args.cra_priv : NULL;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor D2("cr_accept: priv(0x%p) priv_len(%u) psep(0x%llx)\n", priv_data,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor args.cra_priv_sz, (longlong_t)args.cra_bkl_cookie);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* get sp resource */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor sp_rp = (daplka_sp_resource_t *)daplka_hash_lookup(&ia_rp->ia_sp_htbl,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* get ep resource */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ep_rp = (daplka_ep_resource_t *)daplka_hash_lookup(&ia_rp->ia_ep_htbl,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * accept is only allowed if ep_state is CLOSED.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * note that after this point, the ep_state is frozen
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * (i.e. TRANSITIONING) until we transition ep_state
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * to ACCEPTING or back to CLOSED if we get an error.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor new_state = old_state = daplka_ep_get_state(ep_rp);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("cr_accept: invalid ep state %d\n", old_state);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor bkl_index = DAPLKA_GET_PSEP_INDEX(args.cra_bkl_cookie);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * make sure the backlog index is not bogus.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("cr_accept: invalid backlog index 0x%llx %d\n",
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * make sure the backlog index indeed refers
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * to a pending connection.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * a ep_rp with a NULL chan_hdl is impossible.
05c0d196312ed468262dc0f5435a8310aa133484Bill Taylor proc_reply.rep.cm_rdma_ra_out = conn->spcp_rdma_ra_out;
05c0d196312ed468262dc0f5435a8310aa133484Bill Taylor proc_reply.rep.cm_rdma_ra_in = conn->spcp_rdma_ra_in;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor proc_reply.rep.cm_rnr_retry_cnt = IBT_RNR_INFINITE_RETRY;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * this clears our slot in the backlog array.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * this slot may now be used by other pending connections.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Set the unique cookie corresponding to the CR to this EP
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * so that is can be used in passive side CM callbacks
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor status = ibt_cm_proceed(IBT_CM_EVENT_REQ_RCV, sid, IBT_CM_ACCEPT,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor &proc_reply, priv_data, (ibt_priv_data_len_t)args.cra_priv_sz);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("cr_accept: ibt_cm_proceed returned %d\n", status);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * note that the CM handler may actually be called at this
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * point. but since ep_state is still in TRANSITIONING, the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * handler will wait until we transition to ACCEPTING. this
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * prevents the case where we set ep_state to ACCEPTING after
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * daplka_service_conn_est sets ep_state to CONNECTED.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor daplka_ep_set_state(ep_rp, old_state, new_state);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * this function is called by the client to reject a
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * connection request.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor/* ARGSUSED */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylordaplka_cr_reject(daplka_ia_resource_t *ia_rp, intptr_t arg, int mode,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor retval = ddi_copyin((void *)arg, &args, sizeof (dapl_cr_reject_t),
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* get sp resource */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor sp_rp = (daplka_sp_resource_t *)daplka_hash_lookup(&ia_rp->ia_sp_htbl,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor D2("cr_reject: psep(0x%llx)\n", (longlong_t)args.crr_bkl_cookie);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor bkl_index = DAPLKA_GET_PSEP_INDEX(args.crr_bkl_cookie);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * make sure the backlog index is not bogus.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("cr_reject: invalid backlog index 0x%llx %d\n",
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * make sure the backlog index indeed refers
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * to a pending connection.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * this clears our slot in the backlog array.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * this slot may now be used by other pending connections.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* results in IBT_CM_CONSUMER as the reason for reject */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /*FALLTHRU*/
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* results in IBT_CM_NO_RESC as the reason for reject */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* unexpect reason code */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor status = ibt_cm_proceed(IBT_CM_EVENT_REQ_RCV, sid, proc_status,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("cr_reject: ibt_cm_proceed returned %d\n", status);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * daplka_sp_match is used by daplka_hash_walk for finding SPs
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor_NOTE(SCHEME_PROTECTS_DATA("daplka", daplka_sp_match_s::spm_sp_rp))
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor daplka_sp_resource_t *sp_rp = (daplka_sp_resource_t *)objp;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor D2("daplka_sp_match: found sp, conn_qual %016llu\n",
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor (longlong_t)((daplka_sp_match_t *)arg)->spm_conn_qual);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * cr_handoff allows the client to handoff a connection request from
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * one service point to another.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor/* ARGSUSED */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylordaplka_cr_handoff(daplka_ia_resource_t *ia_rp, intptr_t arg, int mode,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor daplka_sp_resource_t *sp_rp = NULL, *new_sp_rp = NULL;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor retval = ddi_copyin((void *)arg, &args, sizeof (dapl_cr_handoff_t),
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* get sp resource */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor sp_rp = (daplka_sp_resource_t *)daplka_hash_lookup(&ia_rp->ia_sp_htbl,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * find the destination service point.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor daplka_hash_walk(&daplka_global_sp_htbl, daplka_sp_match,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * return if we cannot find the service point
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("cr_handoff: new sp not found, conn qual = %llu\n",
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the spec does not discuss the security implications of this
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * function. to be safe, we currently only allow processes
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * owned by the same user to handoff connection requests
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * to each other.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor D2("cr_handoff: psep(0x%llx)\n", (longlong_t)args.crh_bkl_cookie);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor bkl_index = DAPLKA_GET_PSEP_INDEX(args.crh_bkl_cookie);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * make sure the backlog index is not bogus.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("cr_handoff: invalid backlog index 0x%llx %d\n",
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * make sure the backlog index indeed refers
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * to a pending connection.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * this clears our slot in the backlog array.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * this slot may now be used by other pending connections.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* fill fake_event and call service_req handler */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor &fake_event, NULL, priv, (ibt_priv_data_len_t)priv_len);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("cr_handoff: service_req returned %d\n", cm_status);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * if for some reason cm_service_req failed, we
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * reject the connection.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor status = ibt_cm_proceed(IBT_CM_EVENT_REQ_RCV, sid,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("cr_handoff: ibt_cm_proceed returned %d\n",
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * returns a list of hca attributes
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor/* ARGSUSED */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylordaplka_ia_query(daplka_ia_resource_t *ia_rp, intptr_t arg, int mode,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Take the ibt_hca_attr_t and stuff them into dapl_hca_attr_t
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor args.hca_attr.dhca_vendor_id = hcap->hca_vendor_id;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor args.hca_attr.dhca_device_id = hcap->hca_device_id;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor args.hca_attr.dhca_version_id = hcap->hca_version_id;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor args.hca_attr.dhca_max_chans = hcap->hca_max_chans;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor args.hca_attr.dhca_max_chan_sz = hcap->hca_max_chan_sz;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor args.hca_attr.dhca_max_cq_sz = hcap->hca_max_cq_sz;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor args.hca_attr.dhca_max_memr = hcap->hca_max_memr;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor args.hca_attr.dhca_max_memr_len = hcap->hca_max_memr_len;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor args.hca_attr.dhca_max_mem_win = hcap->hca_max_mem_win;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor args.hca_attr.dhca_max_rdma_in_chan = hcap->hca_max_rdma_in_chan;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor args.hca_attr.dhca_max_rdma_out_chan = hcap->hca_max_rdma_out_chan;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor args.hca_attr.dhca_max_partitions = hcap->hca_max_partitions;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor args.hca_attr.dhca_node_guid = hcap->hca_node_guid;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor args.hca_attr.dhca_max_srqs = hcap->hca_max_srqs;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor args.hca_attr.dhca_max_srqs_sz = hcap->hca_max_srqs_sz;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor args.hca_attr.dhca_max_srq_sgl = hcap->hca_max_srq_sgl;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor retval = ddi_copyout(&args, (void *)arg, sizeof (dapl_ia_query_t),
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * This routine is passed to hash walk in the daplka_pre_mr_cleanup_callback,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * it frees the mw embedded in the mw resource object.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor/* ARGSUSED */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor daplka_mw_resource_t *mw_rp = (daplka_mw_resource_t *)objp;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor D3("mr_cb_freemw: entering, mw_rp 0x%p\n", mw_rp);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * we set mw_hdl to NULL so it won't get freed again
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor status = daplka_ibt_free_mw(mw_rp, mw_rp->mw_hca_hdl, mw_hdl);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("mr_cb_freemw: ibt_free_mw returned %d\n", status);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * This routine is called from HCA driver's umem lock undo callback
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * when the memory associated with an MR is being unmapped. In this callback
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * we free all the MW associated with the IA and post an unaffiliated
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * async event to tell the app that there was a catastrophic event.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * This allows the HCA to deregister the MR in its callback processing.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylordaplka_pre_mr_cleanup_callback(void *arg1, void *arg2 /*ARGSUSED*/)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ia_rp = (daplka_ia_resource_t *)daplka_resource_lookup(rnum);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("daplka_mr_unlock_callback: resource not found, rnum %d\n",
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("daplka_mr_unlock_callback: resource(%p) rnum(%d)\n", ia_rp, rnum);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * MW is being alloced OR MW freeze has already begun. In
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * both these cases we wait for that to complete before
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * continuing.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor while ((ia_rp->ia_state == DAPLKA_IA_MW_ALLOC_IN_PROGRESS) ||
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor (ia_rp->ia_state == DAPLKA_IA_MW_FREEZE_IN_PROGRESS)) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ia_rp->ia_state = DAPLKA_IA_MW_FREEZE_IN_PROGRESS;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* the mw on this ia have been freed */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor D2("daplka_mr_unlock_callback: ia_state %d nothing to do\n",
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ASSERT(!"daplka_mr_unlock_callback: IA state invalid");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("daplka_mr_unlock_callback: invalid ia_state %d\n",
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Walk the mw hash table and free the mws. Acquire a writer
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * lock since we don't want anyone else traversing this tree
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * while we are freeing the MW.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor daplka_hash_walk(&ia_rp->ia_mw_htbl, daplka_mr_cb_freemw, NULL,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ASSERT(ia_rp->ia_state == DAPLKA_IA_MW_FREEZE_IN_PROGRESS);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Currently commented out because Oracle skgxp is incapable
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * of handling async events correctly.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Enqueue an unaffiliated async error event to indicate this
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * IA has encountered a problem that caused the MW to freed up
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Create a fake event, only relevant field is the hca_guid */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor daplka_async_event_create(IBT_ERROR_LOCAL_CATASTROPHIC, &event, 0,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor#endif /* _THROW_ASYNC_EVENT_FROM_MRUNLOCKCB */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor D2("daplka_mr_unlock_callback: resource(%p) done\n", ia_rp);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * registers a memory region.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * memory locking will be done by the HCA driver.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor/* ARGSUSED */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylordaplka_mr_register(daplka_ia_resource_t *ia_rp, intptr_t arg, int mode,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor retval = ddi_copyin((void *)arg, &args, sizeof (dapl_mr_register_t),
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor mr_rp = kmem_zalloc(sizeof (daplka_mr_resource_t), daplka_km_flags);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("mr_register: cannot allocate mr resource\n");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor mutex_init(&mr_rp->mr_lock, NULL, MUTEX_DRIVER, NULL);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* get pd handle */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor daplka_hash_lookup(&ia_rp->ia_pd_htbl, args.mr_pd_hkey);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor mr_rp->mr_attr.mr_flags = args.mr_flags | IBT_MR_NOSLEEP;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor D3("mr_register: mr_vaddr %p, mr_len %llu, mr_flags 0x%x\n",
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor status = daplka_ibt_register_mr(mr_rp, ia_rp->ia_hca_hdl,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor mr_rp->mr_pd_res->pd_hdl, &mr_rp->mr_attr, &mr_rp->mr_hdl,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("mr_register: ibt_register_mr error %d\n", status);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor mr_cb_data_in.mr_rev = IBT_MR_DATA_IN_IF_VERSION;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor mr_cb_data_in.mr_func = daplka_pre_mr_cleanup_callback;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Pass the service driver mr cleanup handler to the hca driver */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor IBT_CI_NO_FLAGS, IBT_HDL_MR, (void *)mr_rp->mr_hdl,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("mr_register: ibt_ci_data_in error(%d) ver(%d)",
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* insert into mr hash table */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("mr_register: cannot insert mr resource into mr_htbl\n");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor retval = ddi_copyout((void *)&args, (void *)arg,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("mr_register: copyout error %d\n", retval);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor (void) daplka_hash_remove(&ia_rp->ia_mr_htbl, mr_hkey,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("mr_register: cannot remove mr from hash table\n");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * we can only get here if another thread
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * has completed the cleanup in mr_deregister
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * registers a shared memory region.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the client calls this function with the intention to share the memory
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * region with other clients. it is assumed that, prior to calling this
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * function, the client(s) are already sharing parts of their address
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * space using a mechanism such as SYSV shared memory. the first client
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * that calls this function will create and insert a daplka_shared_mr_t
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * object into the global daplka_shared_mr_tree. this shared mr object
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * will be identified by a unique 40-byte key and will maintain a list
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * of mr resources. every time this function gets called with the same
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * 40-byte key, a new mr resource (containing a new mr handle generated
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * by ibt_register_mr or ibt_register_shared_mr) is created and inserted
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * into this list. similarly, every time a shared mr gets deregistered
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * or invalidated by a callback, the mr resource gets removed from this
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * list. the shared mr object has a reference count. when it drops to
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * zero, the shared mr object will be removed from the global avl tree
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * and be freed.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor/* ARGSUSED */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylordaplka_mr_register_shared(daplka_ia_resource_t *ia_rp, intptr_t arg, int mode,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("mr_register_shared: copyin error %d\n", retval);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * find smrp from the global avl tree.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the 40-byte key is used as the lookup key.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor avl_find(&daplka_shared_mr_tree, &tmp_smr, &where);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor D2("mr_register_shared: smrp 0x%p, found cookie:\n"
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor "0x%016llx%016llx%016llx%016llx%016llx\n", smrp,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * if the smrp exists, other threads could still be
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * accessing it. we wait until they are done before
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * we continue.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor while (smrp->smr_state == DAPLKA_SMR_TRANSITIONING) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor "waiting in transitioning state, refcnt %d\n",
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor D2("mr_register_shared: smrp 0x%p, refcnt %d, ready\n",
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * we set smr_state to TRANSITIONING to temporarily
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * prevent other threads from trying to access smrp.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor "0x%016llx%016llx%016llx%016llx%016llx\n",
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * if we cannot find smrp, we need to create and
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * insert one into daplka_shared_mr_tree
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor avl_insert(&daplka_shared_mr_tree, smrp, where);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor mr_rp = kmem_zalloc(sizeof (daplka_mr_resource_t), daplka_km_flags);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("mr_register_shared: cannot allocate mr resource\n");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor mutex_init(&mr_rp->mr_lock, NULL, MUTEX_DRIVER, NULL);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* get pd handle */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor daplka_hash_lookup(&ia_rp->ia_pd_htbl, args.mrs_pd_hkey);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("mr_register_shared: cannot find pd resource\n");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor mr_rp->mr_attr.mr_flags = args.mrs_flags | IBT_MR_NOSLEEP;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor D2("mr_register_shared: mr_vaddr 0x%p, mr_len %llu, "
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor "mr_flags 0x%x, mr_as 0x%p, mr_exists %d, smrp 0x%p\n",
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * since we are in TRANSITIONING state, we are guaranteed
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * that we have exclusive access to smr_mr_list.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * a non-null smr_mr_list indicates that someone
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * else has already inserted an mr_resource into
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * smr_mr_list. we use the mr_handle from the first
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * element as an arg to ibt_register_shared_mr.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor mem_sattr.mr_vaddr = smrp->smr_mr_list->mr_desc.md_vaddr;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor D2("mr_register_shared: mem_sattr vaddr 0x%p flags 0x%x\n",
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor (void *)(uintptr_t)mem_sattr.mr_vaddr, mem_sattr.mr_flags);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor status = daplka_ibt_register_shared_mr(mr_rp, ia_rp->ia_hca_hdl,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor smrp->smr_mr_list->mr_hdl, mr_rp->mr_pd_res->pd_hdl,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * an mr does not exist yet. we need to create one
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * using ibt_register_mr.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor status = daplka_ibt_register_mr(mr_rp, ia_rp->ia_hca_hdl,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor mr_cb_data_in.mr_rev = IBT_MR_DATA_IN_IF_VERSION;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor mr_cb_data_in.mr_func = daplka_pre_mr_cleanup_callback;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Pass the service driver mr cleanup handler to the hca driver */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor IBT_CI_NO_FLAGS, IBT_HDL_MR, (void *)mr_rp->mr_hdl,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("mr_register_shared: ibt_ci_data_in error(%d) ver(%d)",
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * we bump reference of mr_rp and enqueue it onto smrp.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* insert into mr hash table */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("mr_register_shared: cannot insert mr resource\n");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * at this point, there are two references to our mr resource.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * one is kept in ia_mr_htbl. the other is kept in the list
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * within this shared mr object (smrp). when we deregister this
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * mr or when a callback invalidates this mr, the reference kept
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * by this shared mr object will be removed.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor retval = ddi_copyout((void *)&args, (void *)arg,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("mr_register_shared: copyout error %d\n", retval);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * set the state to READY to allow others to continue
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor (void) daplka_hash_remove(&ia_rp->ia_mr_htbl, mr_hkey,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor "cannot remove mr from hash table\n");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * we can only get here if another thread
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * has completed the cleanup in mr_deregister
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("mr_register_shared: freeing smrp 0x%p\n", smrp);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the refcnt is 0. if there is anything
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * left on the list, it must be ours.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ASSERT(smrp->smr_state == DAPLKA_SMR_TRANSITIONING);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor if (smrp->smr_mr_list != NULL && mr_rp != NULL) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * search and remove mr_rp from smr_mr_list
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * note that smr_state == READY does not necessarily
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * mean that smr_mr_list is non empty. for this case,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * we are doing cleanup because of a failure. we set
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the state to READY to allow other threads to
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * registers a memory region using the attributes of an
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * existing region.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor/* ARGSUSED */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylordaplka_mr_register_lmr(daplka_ia_resource_t *ia_rp, intptr_t arg, int mode,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("mr_register_lmr: copyin error %d\n", retval);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor daplka_hash_lookup(&ia_rp->ia_mr_htbl, args.mrl_orig_hkey);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("mr_register_lmr: cannot find mr resource\n");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ASSERT(DAPLKA_RS_TYPE(orig_mr_rp) == DAPL_TYPE_MR);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor mr_rp = kmem_zalloc(sizeof (daplka_mr_resource_t), daplka_km_flags);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("mr_register_lmr: cannot allocate mr resource\n");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor mutex_init(&mr_rp->mr_lock, NULL, MUTEX_DRIVER, NULL);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Pass the IO addr that was returned while allocating the orig MR */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor mem_sattr.mr_vaddr = orig_mr_rp->mr_desc.md_vaddr;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor mem_sattr.mr_flags = args.mrl_flags | IBT_MR_NOSLEEP;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor status = daplka_ibt_register_shared_mr(mr_rp, ia_rp->ia_hca_hdl,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor orig_mr_rp->mr_hdl, mr_rp->mr_pd_res->pd_hdl, &mem_sattr,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("mr_register_lmr: ibt_register_shared_mr error %d\n",
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor mr_cb_data_in.mr_rev = IBT_MR_DATA_IN_IF_VERSION;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor mr_cb_data_in.mr_func = daplka_pre_mr_cleanup_callback;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Pass the service driver mr cleanup handler to the hca driver */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor IBT_CI_NO_FLAGS, IBT_HDL_MR, (void *)mr_rp->mr_hdl,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("mr_register_lmr: ibt_ci_data_in error(%d) ver(%d)",
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor mr_rp->mr_attr.mr_len = orig_mr_rp->mr_attr.mr_len;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* insert into mr hash table */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor retval = daplka_hash_insert(&ia_rp->ia_mr_htbl, &mr_hkey,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("mr_register: cannot insert mr resource into mr_htbl\n");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor retval = ddi_copyout((void *)&args, (void *)arg,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("mr_register_lmr: copyout error %d\n", retval);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor (void) daplka_hash_remove(&ia_rp->ia_mr_htbl, mr_hkey,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("mr_register: cannot remove mr from hash table\n");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * we can only get here if another thread
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * has completed the cleanup in mr_deregister
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * this function is called by mr_deregister and mr_cleanup_callback to
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * remove a mr resource from the shared mr object mr_rp->mr_shared_mr.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * if mr_shared_mr is already NULL, that means the region being
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * deregistered or invalidated is not a shared mr region and we can
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * return immediately.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylordaplka_shared_mr_free(daplka_mr_resource_t *mr_rp)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * we need a lock because mr_callback also checks this field.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * for the rare case that mr_deregister and mr_cleanup_callback
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * gets called simultaneously, we are guaranteed that smrp won't
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * be dereferenced twice because either function will find
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * mr_shared_mr to be NULL.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor while (smrp->smr_state == DAPLKA_SMR_TRANSITIONING) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * search and remove mr_rp from smr_mr_list.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * also UNREF mr_rp because it is no longer
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * on the list.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * since mr_clean_callback may not touch smr_mr_list
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * at this time (due to smr_state), we can be sure
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * that we can find and remove mr_rp from smr_mr_list
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor D3("shared_mr_free: freeing smrp 0x%p\n", smrp);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * deregisters a memory region.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * if mr is shared, remove reference from global shared mr object.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * release the initial reference to the mr. if the mr's refcnt is
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * zero, call mr_destroy to free mr.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor/* ARGSUSED */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylordaplka_mr_deregister(daplka_ia_resource_t *ia_rp, intptr_t arg, int mode,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor retval = ddi_copyin((void *)arg, &args, sizeof (dapl_mr_deregister_t),
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("mr_deregister: copyin error %d\n", retval);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("mr_deregister: cannot find mr resource\n");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * sync local memory regions on RDMA read or write.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor/* ARGSUSED */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylordaplka_mr_sync(daplka_ia_resource_t *ia_rp, intptr_t arg, int mode,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor retval = ddi_copyin((void *)arg, &args, sizeof (dapl_mr_sync_t), mode);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* number of segments bound check */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("mr_sync: number of segments too large\n");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* translate MR sync direction flag */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor } else if (args.mrs_flags == DAPL_MR_SYNC_RDMA_WR) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * all the segments are going to be sync'd by ibtl together
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor mr_rp[i] = (daplka_mr_resource_t *)daplka_hash_lookup(
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor for (j = 0; j < i; j++) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ASSERT(DAPLKA_RS_TYPE(mr_rp[i]) == DAPL_TYPE_MR);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor status = ibt_sync_mr(ia_rp->ia_hca_hdl, mrs, args.mrs_numseg);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("mr_sync: ibt_sync_mr error %d\n", status);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * destroys a memory region.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * called when refcnt drops to zero.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor daplka_mr_resource_t *mr_rp = (daplka_mr_resource_t *)gen_rp;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor D3("mr_destroy: entering, mr_rp 0x%p, rnum %d\n",
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * deregister mr
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor status = daplka_ibt_deregister_mr(mr_rp, mr_rp->mr_hca_hdl,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("mr_destroy: ibt_deregister_mr returned %d\n",
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * release reference on PD
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor kmem_free(mr_rp, sizeof (daplka_mr_resource_t));
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * this function is called by daplka_hash_destroy for
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * freeing MR resource objects
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor daplka_mr_resource_t *mr_rp = (daplka_mr_resource_t *)obj;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * comparison function used for finding a shared mr object
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * from the global shared mr avl tree.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylordaplka_shared_mr_cmp(const void *smr1, const void *smr2)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor daplka_shared_mr_t *s1 = (daplka_shared_mr_t *)smr1;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor daplka_shared_mr_t *s2 = (daplka_shared_mr_t *)smr2;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor for (i = 4; i >= 0; i--) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor return (-1);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * allocates a protection domain.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor/* ARGSUSED */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylordaplka_pd_alloc(daplka_ia_resource_t *ia_rp, intptr_t arg, int mode,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor pd_rp = kmem_zalloc(sizeof (*pd_rp), daplka_km_flags);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("pd_alloc: cannot allocate pd resource\n");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor status = daplka_ibt_alloc_pd(pd_rp, pd_rp->pd_hca_hdl,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("pd_alloc: ibt_alloc_pd returned %d\n", status);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* insert into pd hash table */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("pd_alloc: cannot insert pd resource into pd_htbl\n");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* return hkey to library */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor retval = ddi_copyout(&args, (void *)arg, sizeof (dapl_pd_alloc_t),
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor (void) daplka_hash_remove(&ia_rp->ia_pd_htbl, pd_hkey,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("pd_alloc: cannot remove pd from hash table\n");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * we can only get here if another thread
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * has completed the cleanup in pd_free
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * destroys a protection domain.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * called when refcnt drops to zero.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor daplka_pd_resource_t *pd_rp = (daplka_pd_resource_t *)gen_rp;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor status = daplka_ibt_free_pd(pd_rp, pd_rp->pd_hca_hdl,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("pd_destroy: ibt_free_pd returned %d\n", status);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor kmem_free(pd_rp, sizeof (daplka_pd_resource_t));
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor daplka_pd_resource_t *pd_rp = (daplka_pd_resource_t *)obj;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * removes the pd reference from ia_pd_htbl and releases the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * initial reference to the pd. also destroys the pd if the refcnt
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor/* ARGSUSED */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylordaplka_pd_free(daplka_ia_resource_t *ia_rp, intptr_t arg, int mode,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor retval = ddi_copyin((void *)arg, &args, sizeof (dapl_pd_free_t), mode);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* UNREF calls the actual free function when refcnt is zero */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * allocates a memory window
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor/* ARGSUSED */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylordaplka_mw_alloc(daplka_ia_resource_t *ia_rp, intptr_t arg, int mode,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor retval = ddi_copyin((void *)arg, &args, sizeof (dapl_mw_alloc_t), mode);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Allocate and initialize a MW resource
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor mw_rp = kmem_zalloc(sizeof (daplka_mw_resource_t), daplka_km_flags);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("mw_alloc: cannot allocate mw resource\n");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor mutex_init(&mw_rp->mw_lock, NULL, MUTEX_DRIVER, NULL);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* get pd handle */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor daplka_hash_lookup(&ia_rp->ia_pd_htbl, args.mw_pd_hkey);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor status = daplka_ibt_alloc_mw(mw_rp, mw_rp->mw_hca_hdl,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor pd_rp->pd_hdl, IBT_MW_NOSLEEP, &mw_rp->mw_hdl, &mw_rkey);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("mw_alloc: ibt_alloc_mw returned %d\n", status);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ia_rp->ia_state = DAPLKA_IA_MW_ALLOC_IN_PROGRESS;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* another mw_alloc is already in progress increase cnt */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* FALLTHRU */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * IA is being or already frozen don't allow more MWs to be
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * allocated.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("mw_alloc: IA is freezing MWs (state=%d)\n",
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("mw_alloc: IA state=%d invalid\n", ia_rp->ia_state);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* retval is 0 when ia_mw_alloccnt is incremented */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* insert into mw hash table */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor retval = daplka_hash_insert(&ia_rp->ia_mw_htbl, &mw_hkey,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("mw_alloc: cannot insert mw resource into mw_htbl\n");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ASSERT(ia_rp->ia_state == DAPLKA_IA_MW_ALLOC_IN_PROGRESS);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor D3("mw_alloc: ibt_alloc_mw mw_hdl(%p) mw_rkey(0x%llx)\n",
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * We are done with mw_alloc if this was the last mw_alloc
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * change state back to DAPLKA_IA_INIT and wake up waiters
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * specifically the unlock callback.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ASSERT(ia_rp->ia_state == DAPLKA_IA_MW_ALLOC_IN_PROGRESS);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor retval = ddi_copyout(&args, (void *)arg, sizeof (dapl_mw_alloc_t),
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor (void) daplka_hash_remove(&ia_rp->ia_mw_htbl, mw_hkey,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("mw_alloc: cannot remove mw from hash table\n");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * we can only get here if another thread
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * has completed the cleanup in mw_free
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * removes the mw reference from ia_mw_htbl and releases the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * initial reference to the mw. also destroys the mw if the refcnt
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor/* ARGSUSED */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylordaplka_mw_free(daplka_ia_resource_t *ia_rp, intptr_t arg, int mode,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor retval = ddi_copyin((void *)arg, &args, sizeof (dapl_mw_free_t), mode);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor retval = daplka_hash_remove(&ia_rp->ia_mw_htbl, args.mw_hkey,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("mw_free: cannot find mw resrc (0x%llx)\n",
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* UNREF calls the actual free function when refcnt is zero */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * destroys the memory window.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * called when refcnt drops to zero.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor daplka_mw_resource_t *mw_rp = (daplka_mw_resource_t *)gen_rp;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor D3("mw_destroy: entering, mw_rp 0x%p, rnum %d\n",
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * free memory window
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor status = daplka_ibt_free_mw(mw_rp, mw_rp->mw_hca_hdl,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("mw_destroy: ibt_free_mw returned %d\n", status);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * release reference on PD
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor kmem_free(mw_rp, sizeof (daplka_mw_resource_t));
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor daplka_mw_resource_t *mw_rp = (daplka_mw_resource_t *)obj;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * SRQ ioctls and supporting functions
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor/* ARGSUSED */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylordaplka_srq_create(daplka_ia_resource_t *ia_rp, intptr_t arg, int mode,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor retval = ddi_copyin((void *)arg, &args, sizeof (dapl_srq_create_t),
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor srq_rp = kmem_zalloc(sizeof (daplka_srq_resource_t), daplka_km_flags);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor mutex_init(&srq_rp->srq_lock, NULL, MUTEX_DRIVER, NULL);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* get pd handle */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor daplka_hash_lookup(&ia_rp->ia_pd_htbl, args.srqc_pd_hkey);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * these checks ensure that the requested SRQ sizes
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * are within the limits supported by the chosen HCA.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor if (args.srqc_sizes.srqs_sz > hca_attrp->hca_max_srqs_sz) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor if (args.srqc_sizes.srqs_sgl > hca_attrp->hca_max_srq_sgl) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor args.srqc_sizes.srqs_sgl, args.srqc_sizes.srqs_sz);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor srq_sizes.srq_sgl_sz = args.srqc_sizes.srqs_sgl;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* create srq */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor status = daplka_ibt_alloc_srq(srq_rp, ia_rp->ia_hca_hdl,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor IBT_SRQ_USER_MAP, pd_rp->pd_hdl, &srq_sizes, &srq_rp->srq_hdl,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("srq_create: alloc_srq returned %d\n", status);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor args.srqc_real_sizes.srqs_sz = srq_real_sizes.srq_wr_sz;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor args.srqc_real_sizes.srqs_sgl = srq_real_sizes.srq_sgl_sz;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Get HCA-specific data_out info */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor IBT_CI_NO_FLAGS, IBT_HDL_SRQ, (void *)srq_rp->srq_hdl,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor &args.srqc_data_out, sizeof (args.srqc_data_out));
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("srq_create: ibt_ci_data_out error(%d)\n", status);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor srq_rp->srq_real_size = srq_real_sizes.srq_wr_sz;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* preparing to copyout map_data back to the library */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor args.srqc_real_sizes.srqs_sz = srq_real_sizes.srq_wr_sz;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor args.srqc_real_sizes.srqs_sgl = srq_real_sizes.srq_sgl_sz;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* insert into srq hash table */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor retval = daplka_hash_insert(&ia_rp->ia_srq_htbl,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("srq_create: cannot insert srq resource into srq_htbl\n");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* return hkey to library */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor retval = ddi_copyout(&args, (void *)arg, sizeof (dapl_srq_create_t),
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor D3("srq_create: %p, 0x%llx\n", srq_rp->srq_hdl, (longlong_t)srq_hkey);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor args.srqc_real_sizes.srqs_sz, args.srqc_real_sizes.srqs_sgl);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor (void) daplka_hash_remove(&ia_rp->ia_srq_htbl, srq_hkey,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * this case is impossible because ep_free will
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * wait until our state transition is complete.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("srq_create: cannot remove srq from hash table\n");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Resize an existing SRQ
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor/* ARGSUSED */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylordaplka_srq_resize(daplka_ia_resource_t *ia_rp, intptr_t arg, int mode,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor retval = ddi_copyin((void *)arg, &args, sizeof (dapl_srq_resize_t),
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* get srq resource */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor daplka_hash_lookup(&ia_rp->ia_srq_htbl, args.srqr_hkey);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ASSERT(DAPLKA_RS_TYPE(srq_rp) == DAPL_TYPE_SRQ);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor if (args.srqr_new_size > hca_attrp->hca_max_srqs_sz) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("srq_resize: invalid srq size %d", args.srqr_new_size);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * If ibt_resize_srq fails that it is primarily due to resource
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * shortage. Per IB spec resize will never loose events and
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * a resize error leaves the SRQ intact. Therefore even if the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * resize request fails we proceed and get the mapping data
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * from the SRQ so that the library can mmap it.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor status = ibt_modify_srq(srq_rp->srq_hdl, IBT_SRQ_SET_SIZE,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* we return the size of the old CQ if resize fails */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("srq_resize: ibt_modify_srq failed:%d\n", status);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor D2("srq_resize(%d): done new_sz(%u) real_sz(%u)\n",
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DAPLKA_RS_RNUM(srq_rp), args.srqr_new_size, args.srqr_real_size);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Get HCA-specific data_out info */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor IBT_CI_NO_FLAGS, IBT_HDL_SRQ, (void *)srq_rp->srq_hdl,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor &args.srqr_data_out, sizeof (args.srqr_data_out));
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("srq_resize: ibt_ci_data_out error(%d)\n", status);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* return ibt_ci_data_out status */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor retval = ddi_copyout(&args, (void *)arg, sizeof (dapl_srq_resize_t),
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Frees an SRQ resource.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor/* ARGSUSED */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylordaplka_srq_free(daplka_ia_resource_t *ia_rp, intptr_t arg, int mode,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor retval = ddi_copyin((void *)arg, &args, sizeof (dapl_srq_free_t), mode);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor retval = daplka_hash_remove(&ia_rp->ia_srq_htbl,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * this is only possible if we have two threads
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * calling ep_free in parallel.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("srq_free: cannot find resource retval(%d) 0x%llx\n",
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* UNREF calls the actual free function when refcnt is zero */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * destroys a SRQ resource.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * called when refcnt drops to zero.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor daplka_srq_resource_t *srq_rp = (daplka_srq_resource_t *)gen_rp;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor D3("srq_destroy: entering, srq_rp 0x%p, rnum %d\n",
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * destroy the srq
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor status = daplka_ibt_free_srq(srq_rp, srq_rp->srq_hdl);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor D3("srq_destroy: srq freed, rnum %d\n", DAPLKA_RS_RNUM(srq_rp));
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * release all references
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor kmem_free(srq_rp, sizeof (daplka_srq_resource_t));
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor D3("srq_destroy: exiting, srq_rp 0x%p\n", srq_rp);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor daplka_srq_resource_t *srq_rp = (daplka_srq_resource_t *)obj;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ASSERT(DAPLKA_RS_TYPE(srq_rp) == DAPL_TYPE_SRQ);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * This function tells the CM to start listening on a service id.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * It must be called by the passive side client before the client
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * can receive connection requests from remote endpoints. If the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * client specifies a non-zero service id (connection qualifier in
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * dapl terms), this function will attempt to bind to this service
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * id and return an error if the id is already in use. If the client
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * specifies zero as the service id, this function will try to find
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the next available service id and return it back to the client.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * To support the cr_handoff function, this function will, in addition
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * to creating and inserting an SP resource into the per-IA SP hash
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * table, insert the SP resource into a global SP table. This table
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * maintains all active service points created by all dapl clients.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * CR handoff locates the target SP by iterating through this global
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor/* ARGSUSED */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylordaplka_service_register(daplka_ia_resource_t *ia_rp, intptr_t arg, int mode,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("service_register: copyin error %d\n", retval);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor sp_rp = kmem_zalloc(sizeof (*sp_rp), daplka_km_flags);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("service_register: cannot allocate sp resource\n");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* check if evd exists */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor daplka_hash_lookup(&ia_rp->ia_evd_htbl, args.sr_evd_hkey);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("service_register: evd resource not found\n");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * initialize backlog size
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor D2("service_register: args.sr_sid = %llu\n", (longlong_t)args.sr_sid);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* save the userland sp ptr */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor D3("service_register: backlog set to %d\n", sp_rp->sp_backlog_size);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor sp_rp->sp_backlog = kmem_zalloc(sp_rp->sp_backlog_size *
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor sizeof (daplka_sp_conn_pend_t), daplka_km_flags);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* save evd resource pointer */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * save ruid here so that we can do a comparison later
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * when someone does cr_handoff. the check will prevent
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * a malicious app from passing a CR to us.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* fill in args for register_service */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor status = ibt_register_service(daplka_dev->daplka_clnt_hdl,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor &sd_args, args.sr_sid, 1, &sp_rp->sp_srv_hdl, &retsid);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("service_register: ibt_register_service returned %d\n",
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* save returned sid */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* fill in args for bind_service */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor status = ibt_bind_service(sp_rp->sp_srv_hdl, ia_rp->ia_hca_sgid,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("service_register: ibt_bind_service returned %d\n",
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * need to bump refcnt because the global hash table will
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * have a reference to sp_rp
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* insert into global sp hash table */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor retval = daplka_hash_insert(&daplka_global_sp_htbl,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("service_register: cannot insert sp resource\n");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* insert into per-IA sp hash table */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("service_register: cannot insert sp resource\n");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* pass index to application */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("service_register: copyout error %d\n", retval);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* remove from ia table */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * we can only get here if another thread
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * has completed the cleanup in svc_deregister
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* remove from global table */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * we get here if either the hash_insert into
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * ia_sp_htbl failed or the ddi_copyout failed.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * hash_insert failure implies that we are the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * only thread with a reference to sp. ddi_copyout
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * failure implies that svc_deregister could have
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * picked up the sp and destroyed it. but since
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * we got to this point, we must have removed
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the sp ourselves in hash_remove above and
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * that the sp can be destroyed by us.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor (void) daplka_hash_remove(&daplka_global_sp_htbl,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * this case is impossible. see explanation above.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* unreference sp */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* destroy sp resource */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * deregisters the service and removes SP from the global table.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor/* ARGSUSED */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylordaplka_service_deregister(daplka_ia_resource_t *ia_rp, intptr_t arg, int mode,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor daplka_sp_resource_t *sp_rp = NULL, *g_sp_rp = NULL;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("service_deregister: copyin error %d\n", retval);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("service_deregister: cannot find sp resource\n");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor retval = daplka_hash_remove(&daplka_global_sp_htbl,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("service_deregister: cannot find sp resource\n");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* remove the global reference */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * destroys a service point.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * called when the refcnt drops to zero.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor daplka_sp_resource_t *sp_rp = (daplka_sp_resource_t *)gen_rp;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * it is possible for pending connections to remain
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * on an SP. We need to clean them up here.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor "spcp_sid == NULL!\n");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor if (sp_rp->sp_srv_hdl != NULL && sp_rp->sp_bind_hdl != NULL) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor status = ibt_deregister_service(daplka_dev->daplka_clnt_hdl,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor sp_rp->sp_backlog_size * sizeof (daplka_sp_conn_pend_t));
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * release reference to evd
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * this function is called by daplka_hash_destroy for
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * freeing SP resource objects
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor daplka_sp_resource_t *sp_rp = (daplka_sp_resource_t *)obj;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor retval = daplka_hash_remove(&daplka_global_sp_htbl,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor daplka_sp_resource_t *sp_rp = (daplka_sp_resource_t *)obj;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Passive side CM handlers
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * processes the REQ_RCV event
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor/* ARGSUSED */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylordaplka_cm_service_req(daplka_sp_resource_t *spp, ibt_cm_event_t *event,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ibt_cm_return_args_t *ret_args, void *pr_data, ibt_priv_data_len_t pr_len)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * acquire a slot in the connection backlog of this service point
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor for (bkl_index = 0; bkl_index < spp->sp_backlog_size; bkl_index++) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor if (spp->sp_backlog[bkl_index].spcp_state == DAPLKA_SPCP_INIT) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * too many pending connections
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("service_req: connection pending exceeded %d limit\n",
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * save data for cr_handoff
05c0d196312ed468262dc0f5435a8310aa133484Bill Taylor conn->spcp_rdma_ra_in = event->cm_event.req.req_rdma_ra_in;
05c0d196312ed468262dc0f5435a8310aa133484Bill Taylor conn->spcp_rdma_ra_out = event->cm_event.req.req_rdma_ra_out;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * create a CR event
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor cr_ev = kmem_zalloc(sizeof (daplka_evd_event_t), KM_NOSLEEP);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor cr_ev->ee_cmev.ec_cm_psep_cookie = DAPLKA_CREATE_PSEP_COOKIE(bkl_index);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * save the requestor gid
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * daplka_event_poll needs this if this is a third party REQ_RCV
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor event->cm_event.req.req_prim_addr.av_dgid.gid_prefix;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor event->cm_event.req.req_prim_addr.av_dgid.gid_guid;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * set event type
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor if (cr_ev->ee_cmev.ec_cm_ev_priv_data == NULL) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor bcopy(pr_data, cr_ev->ee_cmev.ec_cm_ev_priv_data, pr_len);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DAPL_IB_CME_CONNECTION_REQUEST_PENDING_PRIVATE_DATA;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * tell the active side to expect the processing time to be
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * at most equal to daplka_cm_delay
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor status = ibt_cm_delay(IBT_CM_DELAY_REQ, event->cm_session_id,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("service_req: ibt_cm_delay failed %d\n", status);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * enqueue cr_ev onto the cr_events list of the EVD
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * corresponding to the SP
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor D2("service_req: enqueue event(%p) evdp(%p) priv_data(%p) "
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor "priv_len(%d) psep(0x%llx)\n", cr_ev, spp->sp_evd_res,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * free the cr event
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor if (cr_ev->ee_cmev.ec_cm_ev_priv_data != NULL) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor kmem_free(cr_ev->ee_cmev.ec_cm_ev_priv_data, pr_len);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * release our slot in the backlog array
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ASSERT(conn->spcp_state == DAPLKA_SPCP_PENDING);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * processes the CONN_CLOSED event
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor/* ARGSUSED */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylordaplka_cm_service_conn_closed(daplka_sp_resource_t *sp_rp,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ibt_cm_event_t *event, ibt_cm_return_args_t *ret_args,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * verify that the ep_state is either CONNECTED or
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * DISCONNECTING. if it is not in either states return
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * without generating an event.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor new_state = old_state = daplka_ep_get_state(ep_rp);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * we can get here if the connection is being aborted
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor D2("service_conn_closed: conn aborted, state = %d, "
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor "closed = %d\n", old_state, (int)event->cm_event.closed);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor daplka_ep_set_state(ep_rp, old_state, new_state);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * create a DAPL_IB_CME_DISCONNECTED event
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor disc_ev = kmem_zalloc(sizeof (daplka_evd_event_t), KM_NOSLEEP);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("service_conn_closed: cannot alloc disc_ev\n");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor daplka_ep_set_state(ep_rp, old_state, new_state);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor disc_ev->ee_cmev.ec_cm_ev_type = DAPL_IB_CME_DISCONNECTED;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor disc_ev->ee_cmev.ec_cm_cookie = sp_rp->sp_cookie;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor disc_ev->ee_cmev.ec_cm_psep_cookie = ep_rp->ep_psep_cookie;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor D2("service_conn_closed: enqueue event(%p) evdp(%p) psep(0x%llx)\n",
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor disc_ev, sp_rp->sp_evd_res, (longlong_t)ep_rp->ep_psep_cookie);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * transition ep_state to DISCONNECTED
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor daplka_ep_set_state(ep_rp, old_state, new_state);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * enqueue event onto the conn_evd owned by ep_rp
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * processes the CONN_EST event
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor/* ARGSUSED */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylordaplka_cm_service_conn_est(daplka_sp_resource_t *sp_rp, ibt_cm_event_t *event,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ibt_cm_return_args_t *ret_args, void *priv_data, ibt_priv_data_len_t len)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ibt_priv_data_len_t pr_len = event->cm_priv_data_len;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * verify that ep_state is ACCEPTING. if it is not in this
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * state, return without generating an event.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor new_state = old_state = daplka_ep_get_state(ep_rp);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * we can get here if the connection is being aborted
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("service_conn_est: conn aborted, state = %d\n",
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor daplka_ep_set_state(ep_rp, old_state, new_state);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * create a DAPL_IB_CME_CONNECTED event
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor conn_ev = kmem_zalloc(sizeof (daplka_evd_event_t), KM_NOSLEEP);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("service_conn_est: conn_ev alloc failed\n");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor daplka_ep_set_state(ep_rp, old_state, new_state);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor conn_ev->ee_cmev.ec_cm_ev_type = DAPL_IB_CME_CONNECTED;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor conn_ev->ee_cmev.ec_cm_cookie = sp_rp->sp_cookie;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor conn_ev->ee_cmev.ec_cm_psep_cookie = ep_rp->ep_psep_cookie;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * copy private data into event
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor if (conn_ev->ee_cmev.ec_cm_ev_priv_data == NULL) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("service_conn_est: pr_data alloc failed\n");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor daplka_ep_set_state(ep_rp, old_state, new_state);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor kmem_free(conn_ev, sizeof (daplka_evd_event_t));
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor bcopy(pr_data, conn_ev->ee_cmev.ec_cm_ev_priv_data, pr_len);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor conn_ev->ee_cmev.ec_cm_ev_priv_data_len = pr_len;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor D2("service_conn_est: enqueue event(%p) evdp(%p)\n",
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * transition ep_state to CONNECTED
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor daplka_ep_set_state(ep_rp, old_state, new_state);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * enqueue event onto the conn_evd owned by ep_rp
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * processes the FAILURE event
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor/* ARGSUSED */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylordaplka_cm_service_event_failure(daplka_sp_resource_t *sp_rp,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ibt_cm_event_t *event, ibt_cm_return_args_t *ret_args, void *priv_data,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * check that we still have a valid cm_channel before continuing
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("serice_event_failure: event->cm_channel == NULL\n");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * verify that ep_state is ACCEPTING or DISCONNECTING. if it
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * is not in either state, return without generating an event.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor new_state = old_state = daplka_ep_get_state(ep_rp);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * we can get here if the connection is being aborted
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("service_event_failure: conn aborted, state = %d, "
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor "cf_code = %d, cf_msg = %d, cf_reason = %d\n", old_state,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor daplka_ep_set_state(ep_rp, old_state, new_state);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor bzero(&chan_attrs, sizeof (ibt_rc_chan_query_attr_t));
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor status = ibt_query_rc_channel(ep_rp->ep_chan_hdl, &chan_attrs);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("service_event_failure: conn abort qpn %d state %d\n",
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* explicit transition the QP to ERROR state */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * create an event
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor disc_ev = kmem_zalloc(sizeof (daplka_evd_event_t), KM_NOSLEEP);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("service_event_failure: cannot alloc disc_ev\n");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor daplka_ep_set_state(ep_rp, old_state, new_state);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * fill in the appropriate event type
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor if (event->cm_event.failed.cf_code == IBT_CM_FAILURE_TIMEOUT) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor disc_ev->ee_cmev.ec_cm_ev_type = DAPL_IB_CME_TIMED_OUT;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor } else if (event->cm_event.failed.cf_code == IBT_CM_FAILURE_REJ_RCV) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor disc_ev->ee_cmev.ec_cm_ev_type = DAPL_IB_CME_LOCAL_FAILURE;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor disc_ev->ee_cmev.ec_cm_cookie = sp_rp->sp_cookie;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor disc_ev->ee_cmev.ec_cm_psep_cookie = ep_rp->ep_psep_cookie;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor D2("service_event_failure: enqueue event(%p) evdp(%p) cf_code(%d) "
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor "cf_msg(%d) cf_reason(%d) psep(0x%llx)\n", disc_ev,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ep_rp->ep_conn_evd, (int)event->cm_event.failed.cf_code,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * transition ep_state to DISCONNECTED
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor daplka_ep_set_state(ep_rp, old_state, new_state);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * enqueue event onto the conn_evd owned by ep_rp
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * this is the passive side CM handler. it gets registered
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * when an SP resource is created in daplka_service_register.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylordaplka_cm_service_handler(void *cm_private, ibt_cm_event_t *event,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Tayloribt_cm_return_args_t *ret_args, void *priv_data, ibt_priv_data_len_t len)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor daplka_sp_resource_t *sp_rp = (daplka_sp_resource_t *)cm_private;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * default is not to return priv data
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*ret_args))
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor return (daplka_cm_service_req(sp_rp, event, ret_args,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* passive side should not receive this event */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor D2("service_handler: IBT_CM_EVENT_CONN_CLOSED %d\n",
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor return (daplka_cm_service_conn_closed(sp_rp, event, ret_args,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* passive side does default processing MRA event */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor return (daplka_cm_service_conn_est(sp_rp, event, ret_args,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor return (daplka_cm_service_event_failure(sp_rp, event, ret_args,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* active side had initiated a path migration operation */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("service_handler: invalid event %d\n", event->cm_type);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Active side CM handlers
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Processes the REP_RCV event. When the passive side accepts the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * connection, this handler is called. We make a copy of the private
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * data into the ep so that it can be passed back to userland in when
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the CONN_EST event occurs.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor/* ARGSUSED */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylordaplka_cm_rc_rep_rcv(daplka_ep_resource_t *ep_rp, ibt_cm_event_t *event,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ibt_cm_return_args_t *ret_args, void *priv_data, ibt_priv_data_len_t len)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ibt_priv_data_len_t pr_len = event->cm_priv_data_len;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor D2("rc_rep_rcv: pr_data(0x%p), pr_len(%d)\n", pr_data,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor new_state = old_state = daplka_ep_get_state(ep_rp);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * we can get here if the connection is being aborted
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("rc_rep_rcv: conn aborted, state = %d\n", old_state);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor daplka_ep_set_state(ep_rp, old_state, new_state);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * we do not cancel the timer here because the connection
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * handshake is still in progress.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * save the private data. it will be passed up when
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the connection is established.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor bcopy(pr_data, ep_rp->ep_priv_data, (size_t)pr_len);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * we do not actually transition to a different state.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the state will change when we get a conn_est, failure,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * closed, or timeout event.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor daplka_ep_set_state(ep_rp, old_state, new_state);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Processes the CONN_CLOSED event. This gets called when either
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the active or passive side closes the rc channel.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor/* ARGSUSED */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylordaplka_cm_rc_conn_closed(daplka_ep_resource_t *ep_rp, ibt_cm_event_t *event,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ibt_cm_return_args_t *ret_args, void *priv_data, ibt_priv_data_len_t len)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor old_state = new_state = daplka_ep_get_state(ep_rp);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * we can get here if the connection is being aborted
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor "closed = %d\n", old_state, (int)event->cm_event.closed);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor daplka_ep_set_state(ep_rp, old_state, new_state);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * it's ok for the timer to fire at this point. the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * taskq thread that processes the timer will just wait
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * until we are done with our state transition.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * daplka_cancel_timer returns -1 if the timer is
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * being processed and 0 for all other cases.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * we need to reset ep_state to allow timer processing
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * to continue.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("rc_conn_closed: timer is being processed\n");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor daplka_ep_set_state(ep_rp, old_state, new_state);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * create a DAPL_IB_CME_DISCONNECTED event
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor disc_ev = kmem_zalloc(sizeof (daplka_evd_event_t), KM_NOSLEEP);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor daplka_ep_set_state(ep_rp, old_state, new_state);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor disc_ev->ee_cmev.ec_cm_ev_type = DAPL_IB_CME_DISCONNECTED;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor disc_ev->ee_cmev.ec_cm_cookie = ep_rp->ep_cookie;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor D2("rc_conn_closed: enqueue event(%p) evdp(%p) closed(%d)\n",
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor disc_ev, ep_rp->ep_conn_evd, (int)event->cm_event.closed);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * transition ep_state to DISCONNECTED
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor daplka_ep_set_state(ep_rp, old_state, new_state);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * enqueue event onto the conn_evd owned by ep_rp
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * processes the CONN_EST event
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor/* ARGSUSED */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylordaplka_cm_rc_conn_est(daplka_ep_resource_t *ep_rp, ibt_cm_event_t *event,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ibt_cm_return_args_t *ret_args, void *priv_data, ibt_priv_data_len_t len)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor old_state = new_state = daplka_ep_get_state(ep_rp);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * we can get here if the connection is being aborted
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("rc_conn_est: conn aborted, state = %d\n", old_state);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor daplka_ep_set_state(ep_rp, old_state, new_state);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * it's ok for the timer to fire at this point. the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * taskq thread that processes the timer will just wait
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * until we are done with our state transition.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * daplka_cancel_timer returns -1 if the timer is
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * being processed and 0 for all other cases.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * we need to reset ep_state to allow timer processing
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * to continue.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("rc_conn_est: timer is being processed\n");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor daplka_ep_set_state(ep_rp, old_state, new_state);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * create a DAPL_IB_CME_CONNECTED event
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor conn_ev = kmem_zalloc(sizeof (daplka_evd_event_t), KM_NOSLEEP);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor daplka_ep_set_state(ep_rp, old_state, new_state);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor conn_ev->ee_cmev.ec_cm_ev_type = DAPL_IB_CME_CONNECTED;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor conn_ev->ee_cmev.ec_cm_cookie = ep_rp->ep_cookie;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * The private data passed back in the connection established
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * event is what was recvd in the daplka_cm_rc_rep_rcv handler and
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * saved in ep resource structure.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor if (conn_ev->ee_cmev.ec_cm_ev_priv_data == NULL) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor kmem_free(conn_ev, sizeof (daplka_evd_event_t));
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor daplka_ep_set_state(ep_rp, old_state, new_state);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor bcopy(ep_rp->ep_priv_data, conn_ev->ee_cmev.ec_cm_ev_priv_data,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor conn_ev->ee_cmev.ec_cm_ev_priv_data_len = ep_rp->ep_priv_len;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor D2("rc_conn_est: enqueue event(%p) evdp(%p) pr_data(0x%p), "
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * transition ep_state to CONNECTED
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor daplka_ep_set_state(ep_rp, old_state, new_state);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * enqueue event onto the conn_evd owned by ep_rp
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * processes the FAILURE event
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor/* ARGSUSED */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylordaplka_cm_rc_event_failure(daplka_ep_resource_t *ep_rp, ibt_cm_event_t *event,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ibt_cm_return_args_t *ret_args, void *priv_data, ibt_priv_data_len_t len)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ibt_priv_data_len_t pr_len = event->cm_priv_data_len;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor old_state = new_state = daplka_ep_get_state(ep_rp);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * we can get here if the connection is being aborted
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("rc_event_failure: conn aborted, state = %d, "
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor "cf_code = %d, cf_msg = %d, cf_reason = %d\n", old_state,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor daplka_ep_set_state(ep_rp, old_state, new_state);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * it's ok for the timer to fire at this point. the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * taskq thread that processes the timer will just wait
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * until we are done with our state transition.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * daplka_cancel_timer returns -1 if the timer is
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * being processed and 0 for all other cases.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * we need to reset ep_state to allow timer processing
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * to continue.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("rc_event_failure: timer is being processed\n");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor daplka_ep_set_state(ep_rp, old_state, new_state);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor bzero(&chan_attrs, sizeof (ibt_rc_chan_query_attr_t));
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor status = ibt_query_rc_channel(ep_rp->ep_chan_hdl, &chan_attrs);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("rc_event_failure: conn abort qpn %d state %d\n",
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* explicit transition the QP to ERROR state */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * create an event
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor disc_ev = kmem_zalloc(sizeof (daplka_evd_event_t), KM_NOSLEEP);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("rc_event_failure: cannot alloc disc_ev\n");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor daplka_ep_set_state(ep_rp, old_state, new_state);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * copy private data into event
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor if (disc_ev->ee_cmev.ec_cm_ev_priv_data == NULL) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("rc_event_failure: cannot alloc pr data\n");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor kmem_free(disc_ev, sizeof (daplka_evd_event_t));
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor daplka_ep_set_state(ep_rp, old_state, new_state);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor bcopy(pr_data, disc_ev->ee_cmev.ec_cm_ev_priv_data, pr_len);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor disc_ev->ee_cmev.ec_cm_ev_priv_data_len = pr_len;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * fill in the appropriate event type
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor if (event->cm_event.failed.cf_code == IBT_CM_FAILURE_REJ_RCV) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor } else if (event->cm_event.failed.cf_code == IBT_CM_FAILURE_TIMEOUT) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor disc_ev->ee_cmev.ec_cm_ev_type = DAPL_IB_CME_TIMED_OUT;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* others we'll mark as local failure */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor disc_ev->ee_cmev.ec_cm_ev_type = DAPL_IB_CME_LOCAL_FAILURE;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor disc_ev->ee_cmev.ec_cm_cookie = ep_rp->ep_cookie;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor D2("rc_event_failure: enqueue event(%p) evdp(%p) cf_code(%d) "
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor "cf_msg(%d) cf_reason(%d)\n", disc_ev, ep_rp->ep_conn_evd,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * transition ep_state to DISCONNECTED
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor daplka_ep_set_state(ep_rp, old_state, new_state);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * enqueue event onto the conn_evd owned by ep_rp
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * This is the active side CM handler. It gets registered when
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * ibt_open_rc_channel is called.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylordaplka_cm_rc_handler(void *cm_private, ibt_cm_event_t *event,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ibt_cm_return_args_t *ret_args, void *priv_data, ibt_priv_data_len_t len)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor daplka_ep_resource_t *ep_rp = (daplka_ep_resource_t *)cm_private;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * default is not to return priv data
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*ret_args))
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* active side should not receive this event */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* connection accepted by passive side */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor return (daplka_cm_rc_rep_rcv(ep_rp, event, ret_args,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor return (daplka_cm_rc_conn_closed(ep_rp, event, ret_args,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* passive side does default processing MRA event */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor return (daplka_cm_rc_conn_est(ep_rp, event, ret_args,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor return (daplka_cm_rc_event_failure(ep_rp, event, ret_args,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor D2("rc_handler: invalid event %d\n", event->cm_type);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * creates an IA resource and inserts it into the global resource table.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor/* ARGSUSED */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylordaplka_ia_create(minor_t rnum, intptr_t arg, int mode,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor retval = ddi_copyin((void *)arg, &args, sizeof (dapl_ia_create_t),
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("ia_create: invalid version %d, expected version %d\n",
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * find the hca with the matching guid
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor for (hca = daplka_dev->daplka_hca_list_head; hca != NULL;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * check whether port number is valid and whether it is up
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("ia_create: invalid hca_port %d\n", args.ia_port);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor status = ibt_query_hca_ports(hca_hdl, (uint8_t)args.ia_port,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("ia_create: ibt_query_hca_ports returned %d\n", status);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ia_rp = kmem_zalloc(sizeof (daplka_ia_resource_t), daplka_km_flags);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DAPLKA_RS_INIT(ia_rp, DAPL_TYPE_IA, rnum, daplka_ia_destroy);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor mutex_init(&ia_rp->ia_lock, NULL, MUTEX_DRIVER, NULL);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor bcopy(args.ia_sadata, ia_rp->ia_sadata, DAPL_ATS_NBYTES);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* register Address Record */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor bcopy(ia_rp->ia_sadata, ar_s.ar_data, DAPL_ATS_NBYTES);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor retval = ibt_register_ar(daplka_dev->daplka_clnt_hdl, &ar_s);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("ia_create: failed to register Address Record.\n");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * create hash tables for all object types
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor retval = daplka_hash_create(&ia_rp->ia_ep_htbl, DAPLKA_EP_HTBL_SZ,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor daplka_hash_ep_free, daplka_hash_generic_lookup);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("ia_create: cannot create ep hash table\n");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor retval = daplka_hash_create(&ia_rp->ia_mr_htbl, DAPLKA_MR_HTBL_SZ,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor daplka_hash_mr_free, daplka_hash_generic_lookup);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("ia_create: cannot create mr hash table\n");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor retval = daplka_hash_create(&ia_rp->ia_mw_htbl, DAPLKA_MW_HTBL_SZ,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor daplka_hash_mw_free, daplka_hash_generic_lookup);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("ia_create: cannot create mw hash table\n");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor retval = daplka_hash_create(&ia_rp->ia_pd_htbl, DAPLKA_PD_HTBL_SZ,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor daplka_hash_pd_free, daplka_hash_generic_lookup);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("ia_create: cannot create pd hash table\n");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor retval = daplka_hash_create(&ia_rp->ia_evd_htbl, DAPLKA_EVD_HTBL_SZ,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor daplka_hash_evd_free, daplka_hash_generic_lookup);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("ia_create: cannot create evd hash table\n");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor retval = daplka_hash_create(&ia_rp->ia_cno_htbl, DAPLKA_CNO_HTBL_SZ,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor daplka_hash_cno_free, daplka_hash_generic_lookup);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("ia_create: cannot create cno hash table\n");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor retval = daplka_hash_create(&ia_rp->ia_sp_htbl, DAPLKA_SP_HTBL_SZ,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor daplka_hash_sp_free, daplka_hash_generic_lookup);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("ia_create: cannot create sp hash table\n");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor retval = daplka_hash_create(&ia_rp->ia_srq_htbl, DAPLKA_SRQ_HTBL_SZ,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor daplka_hash_srq_free, daplka_hash_generic_lookup);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("ia_create: cannot create srq hash table\n");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * insert ia_rp into the global resource table
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor retval = daplka_resource_insert(rnum, (daplka_resource_t *)ia_rp);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor retval = copyout(&args, (void *)arg, sizeof (dapl_ia_create_t));
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor tmp_rp = (daplka_ia_resource_t *)daplka_resource_remove(rnum);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * we can return here because another thread must
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * have freed up the resource
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * destroys an IA resource
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor daplka_ia_resource_t *ia_rp = (daplka_ia_resource_t *)gen_rp;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor D3("ia_destroy: entering, ia_rp 0x%p\n", ia_rp);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* deregister Address Record */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor bcopy(ia_rp->ia_sadata, ar_s.ar_data, DAPL_ATS_NBYTES);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor (void) ibt_deregister_ar(daplka_dev->daplka_clnt_hdl, &ar_s);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * destroy hash tables. make sure resources are
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * destroyed in the correct order.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * free the async evd list
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor kmem_free(ia_rp, sizeof (daplka_ia_resource_t));
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylordaplka_async_event_create(ibt_async_code_t code, ibt_async_event_t *event,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Note: this allocation does not zero out the buffer
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * since we init all the fields.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor evp = kmem_alloc(sizeof (daplka_evd_event_t), KM_NOSLEEP);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Lookup the async evd corresponding to this ia and enqueue
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * evp and wakeup any waiter.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor daplka_hash_lookup(&ia_rp->ia_evd_htbl, curr->aeh_evd_hkey);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor if (async_evd == NULL) { /* async evd is being freed */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("async_event_enqueue: ia_rp(%p) asycn_evd %llx "
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor "!found\n", ia_rp, (longlong_t)curr->aeh_evd_hkey);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor daplka_evd_wakeup(async_evd, &async_evd->evd_async_events, evp);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* decrement refcnt on async_evd */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * This routine is called in kernel context
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor/* ARGSUSED */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylordaplka_rc_async_handler(void *clnt_private, ibt_hca_hdl_t hca_hdl,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ibt_async_code_t code, ibt_async_event_t *event)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("daplka_rc_async_handler: ev_chan_hdl is NULL\n");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("daplka_rc_async_handler: chan_private is NULL\n");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* grab a reference to this ep */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * The endpoint resource has the resource number corresponding to
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the IA resource. Use that to lookup the ia resource entry
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ia_rp = (daplka_ia_resource_t *)daplka_resource_lookup(ia_rnum);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor if ((ia_rp == NULL) || DAPLKA_RS_RESERVED(ia_rp)) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor D2("daplka_rc_async_handler: resource (%d) not found\n",
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Create an async event and chain it to the async evd
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor daplka_async_event_create(code, event, epp->ep_cookie, ia_rp);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * This routine is called in kernel context
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor/* ARGSUSED */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylordaplka_cq_async_handler(void *clnt_private, ibt_hca_hdl_t hca_hdl,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ibt_async_code_t code, ibt_async_event_t *event)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("daplka_cq_async_handler: get cq private(%p) failed\n",
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* grab a reference to this evd resource */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * The endpoint resource has the resource number corresponding to
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the IA resource. Use that to lookup the ia resource entry
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ia_rp = (daplka_ia_resource_t *)daplka_resource_lookup(ia_rnum);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor if ((ia_rp == NULL) || DAPLKA_RS_RESERVED(ia_rp)) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("daplka_cq_async_handler: resource (%d) not found\n",
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Create an async event and chain it to the async evd
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor daplka_async_event_create(code, event, evdp->evd_cookie, ia_rp);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* release all the refcount that were acquired */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * This routine is called in kernel context, handles unaffiliated async errors
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor/* ARGSUSED */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylordaplka_un_async_handler(void *clnt_private, ibt_hca_hdl_t hca_hdl,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ibt_async_code_t code, ibt_async_event_t *event)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Walk the resource table looking for an ia that matches the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor rw_enter(&daplka_resource.daplka_rct_lock, RW_READER);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor for (i = 0; i < daplka_resource.daplka_rc_len; i++) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor for (j = 0; j < DAPLKA_RC_BLKSZ; j++) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * rp is an IA resource check if it belongs
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * to the hca/port for which we got the event
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * walk the ep hash table. Acquire a
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * reader lock. NULL dgid indicates
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * local port up event.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylordaplka_handle_hca_detach_event(ibt_async_event_t *event)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * find the hca with the matching guid
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor for (hca = daplka_dev->daplka_hca_list_head; hca != NULL;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * This routine is called in kernel context
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylordaplka_async_handler(void *clnt_private, ibt_hca_hdl_t hca_hdl,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ibt_async_code_t code, ibt_async_event_t *event)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor D2("daplka_async_handler(): Channel affiliated=0x%x\n", code);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* These events are affiliated with a the RC channel */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor daplka_rc_async_handler(clnt_private, hca_hdl, code, event);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* This event is affiliated with a the CQ */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor daplka_cq_async_handler(clnt_private, hca_hdl, code, event);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor daplka_un_async_handler(clnt_private, hca_hdl, code,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * NOTE: In some error recovery paths, it is possible to
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * receive IBT_HCA_ATTACH_EVENTs on already known HCAs.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor (void) daplka_init_hca(daplka_dev, event->ev_hca_guid);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Free all hca resources and close the HCA. */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* This event is affiliated with APM */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor D2("daplka_async_handler(): IBT_PATH_MIGRATED.\n");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor D2("daplka_async_handler(): unhandled code = 0x%x\n", code);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * This routine is called in kernel context related to Subnet events
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylordaplka_sm_notice_handler(void *arg, ib_gid_t gid, ibt_subnet_event_code_t code,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* This event is affiliated with remote port up */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor D2("daplka_sm_notice_handler(): IBT_SM_EVENT_GID_AVAIL\n");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* This event is affiliated with remote port down */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor D2("daplka_sm_notice_handler(): IBT_SM_EVENT_GID_UNAVAIL\n");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor D2("daplka_sm_notice_handler(): unhandled IBT_SM_EVENT_[%d]\n",
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * This routine is called in kernel context, handles Subnet GID avail events
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * which correspond to remote port up. Setting up alternate path or path
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * migration (failback) has to be initiated from the active side of the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * original connect.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylordaplka_sm_gid_avail(ib_gid_t *sgid, ib_gid_t *dgid)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor D2("daplka_sm_gid_avail: sgid=%llx:%llx dgid=%llx:%llx\n",
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor (longlong_t)sgid->gid_prefix, (longlong_t)sgid->gid_guid,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor (longlong_t)dgid->gid_prefix, (longlong_t)dgid->gid_guid);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Walk the resource table looking for an ia that matches the sgid
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor rw_enter(&daplka_resource.daplka_rct_lock, RW_READER);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor for (i = 0; i < daplka_resource.daplka_rc_len; i++) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor for (j = 0; j < DAPLKA_RC_BLKSZ; j++) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * rp is an IA resource check if its gid
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * matches with the calling sgid
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor (sgid->gid_guid == ia_rp->ia_hca_sgid.gid_guid)) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * walk the ep hash table. Acquire a
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * reader lock.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * This routine is called in kernel context to get and set an alternate path
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylordaplka_ep_altpath(daplka_ep_resource_t *ep_rp, ib_gid_t *dgid)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor bzero(&path_info, sizeof (ibt_alt_path_info_t));
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor bzero(&path_attr, sizeof (ibt_alt_path_attr_t));
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor status = ibt_get_alt_path(ep_rp->ep_chan_hdl, IBT_PATH_AVAIL,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("daplka_ep_altpath : ibt_get_alt_path failed %d\n",
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor status = ibt_set_alt_path(ep_rp->ep_chan_hdl, IBT_BLOCKING,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("daplka_ep_altpath : ibt_set_alt_path failed "
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor "status %d ap_status %d\n", status, ap_rets.ap_status);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * This routine is called in kernel context to failback to the original path
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor daplka_ep_resource_t *ep_rp = (daplka_ep_resource_t *)objp;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor D2("daplka_ep_failback ep : sgid=%llx:%llx dgid=%llx:%llx\n",
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * daplka_ep_failback is called from daplka_hash_walk
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * which holds the read lock on hash table to protect
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the endpoint resource from removal
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* check for unconnected endpoints */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* first check for ep state */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor if (ep_rp->ep_state != DAPLKA_EP_STATE_CONNECTED) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor D2("daplka_ep_failback : endpoints not connected\n");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* second check for gids */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor D2("daplka_ep_failback : skip unconnected endpoints\n");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * matching destination ep
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * when dgid is NULL, the async event is a local port up.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * dgid becomes wild card, i.e. all endpoints match
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* ignore loopback ep */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor if ((ep_rp->ep_sgid.gid_prefix == ep_rp->ep_dgid.gid_prefix) &&
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor (ep_rp->ep_sgid.gid_guid == ep_rp->ep_dgid.gid_guid)) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor D2("daplka_ep_failback : skip loopback endpoints\n");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* matching remote ep */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor if ((ep_rp->ep_dgid.gid_prefix != dgid->gid_prefix) ||
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor D2("daplka_ep_failback : unrelated endpoints\n");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* call get and set altpath with original dgid used in ep_connect */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor if (daplka_ep_altpath(ep_rp, &ep_rp->ep_dgid)) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * wait for migration state to be ARMed
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * e.g. a post_send msg will transit mig_state from REARM to ARM
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor for (i = 0; i < daplka_query_aft_setaltpath; i++) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor bzero(&chan_attrs, sizeof (ibt_rc_chan_query_attr_t));
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor status = ibt_query_rc_channel(ep_rp->ep_chan_hdl, &chan_attrs);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("daplka_ep_altpath : ibt_query_rc_channel err\n");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor D2("daplka_ep_altpath : P sgid=%llx:%llx dgid=%llx:%llx\n",
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor chan_attrs.rc_prim_path.cep_adds_vect.av_sgid.gid_prefix,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor (longlong_t)chan_attrs.rc_prim_path.cep_adds_vect.av_sgid.gid_guid,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor chan_attrs.rc_prim_path.cep_adds_vect.av_dgid.gid_prefix,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor (longlong_t)chan_attrs.rc_prim_path.cep_adds_vect.av_dgid.gid_guid);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor D2("daplka_ep_altpath : A sgid=%llx:%llx dgid=%llx:%llx\n",
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor (longlong_t)chan_attrs.rc_alt_path.cep_adds_vect.av_sgid.gid_prefix,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor (longlong_t)chan_attrs.rc_alt_path.cep_adds_vect.av_sgid.gid_guid,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor (longlong_t)chan_attrs.rc_alt_path.cep_adds_vect.av_dgid.gid_prefix,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor (longlong_t)chan_attrs.rc_alt_path.cep_adds_vect.av_dgid.gid_guid);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* skip failback on ARMed state not reached or env override */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor if ((i >= daplka_query_aft_setaltpath) || (daplka_failback == 0)) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("daplka_ep_altpath : ARMed state not reached\n");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor D2("daplka_ep_failback : ibt_migrate_path() to original ep\n");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* call get and altpath with NULL dgid to indicate unspecified dgid */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * IBTF wrappers used for resource accounting
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylordaplka_ibt_alloc_rc_channel(daplka_ep_resource_t *ep_rp, ibt_hca_hdl_t hca_hdl,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ibt_chan_alloc_flags_t flags, ibt_rc_chan_alloc_args_t *args,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ibt_channel_hdl_t *chan_hdl_p, ibt_chan_sizes_t *sizes)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor max_qps = daplka_max_qp_percent * hca_p->hca_attr.hca_max_chans / 100;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("ibt_alloc_rc_channel: resource limit exceeded "
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor status = ibt_alloc_rc_channel(hca_hdl, flags, args, chan_hdl_p, sizes);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylordaplka_ibt_free_channel(daplka_ep_resource_t *ep_rp, ibt_channel_hdl_t chan_hdl)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylordaplka_ibt_alloc_cq(daplka_evd_resource_t *evd_rp, ibt_hca_hdl_t hca_hdl,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ibt_cq_attr_t *cq_attr, ibt_cq_hdl_t *ibt_cq_p, uint32_t *real_size)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor max_cqs = daplka_max_cq_percent * hca_p->hca_attr.hca_max_cq / 100;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor status = ibt_alloc_cq(hca_hdl, cq_attr, ibt_cq_p, real_size);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylordaplka_ibt_free_cq(daplka_evd_resource_t *evd_rp, ibt_cq_hdl_t cq_hdl)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylordaplka_ibt_alloc_pd(daplka_pd_resource_t *pd_rp, ibt_hca_hdl_t hca_hdl,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor max_pds = daplka_max_pd_percent * hca_p->hca_attr.hca_max_pd / 100;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor status = ibt_alloc_pd(hca_hdl, flags, pd_hdl_p);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylordaplka_ibt_free_pd(daplka_pd_resource_t *pd_rp, ibt_hca_hdl_t hca_hdl,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylordaplka_ibt_alloc_mw(daplka_mw_resource_t *mw_rp, ibt_hca_hdl_t hca_hdl,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ibt_pd_hdl_t pd_hdl, ibt_mw_flags_t flags, ibt_mw_hdl_t *mw_hdl_p,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor max_mws = daplka_max_mw_percent * hca_p->hca_attr.hca_max_mem_win / 100;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor status = ibt_alloc_mw(hca_hdl, pd_hdl, flags, mw_hdl_p, rkey_p);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylordaplka_ibt_free_mw(daplka_mw_resource_t *mw_rp, ibt_hca_hdl_t hca_hdl,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylordaplka_ibt_register_mr(daplka_mr_resource_t *mr_rp, ibt_hca_hdl_t hca_hdl,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ibt_pd_hdl_t pd_hdl, ibt_mr_attr_t *mr_attr, ibt_mr_hdl_t *mr_hdl_p,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor max_mrs = daplka_max_mr_percent * hca_p->hca_attr.hca_max_memr / 100;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("ibt_register_mr: resource limit exceeded "
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor status = ibt_register_mr(hca_hdl, pd_hdl, mr_attr, mr_hdl_p, mr_desc_p);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylordaplka_ibt_register_shared_mr(daplka_mr_resource_t *mr_rp,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ibt_hca_hdl_t hca_hdl, ibt_mr_hdl_t mr_hdl, ibt_pd_hdl_t pd_hdl,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ibt_smr_attr_t *smr_attr_p, ibt_mr_hdl_t *mr_hdl_p,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor max_mrs = daplka_max_mr_percent * hca_p->hca_attr.hca_max_memr / 100;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("ibt_register_shared_mr: resource limit exceeded "
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor status = ibt_register_shared_mr(hca_hdl, mr_hdl, pd_hdl,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylordaplka_ibt_deregister_mr(daplka_mr_resource_t *mr_rp, ibt_hca_hdl_t hca_hdl,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylordaplka_ibt_alloc_srq(daplka_srq_resource_t *srq_rp, ibt_hca_hdl_t hca_hdl,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ibt_srq_flags_t flags, ibt_pd_hdl_t pd, ibt_srq_sizes_t *reqsz,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ibt_srq_hdl_t *srq_hdl_p, ibt_srq_sizes_t *realsz)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor max_srqs = daplka_max_srq_percent * hca_p->hca_attr.hca_max_srqs / 100;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor status = ibt_alloc_srq(hca_hdl, flags, pd, reqsz, srq_hdl_p, realsz);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylordaplka_ibt_free_srq(daplka_srq_resource_t *srq_rp, ibt_srq_hdl_t srq_hdl)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylordaplka_common_ioctl(int cmd, minor_t rnum, intptr_t arg, int mode,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor error = daplka_ia_create(rnum, arg, mode, cred, rvalp);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* can potentially add other commands here */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("daplka_common_ioctl: cmd not supported\n");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylordaplka_evd_ioctl(int cmd, daplka_ia_resource_t *rp, intptr_t arg, int mode,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor error = daplka_evd_create(rp, arg, mode, cred, rvalp);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor error = daplka_cq_resize(rp, arg, mode, cred, rvalp);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor error = daplka_event_poll(rp, arg, mode, cred, rvalp);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor error = daplka_event_wakeup(rp, arg, mode, cred, rvalp);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor error = daplka_evd_modify_cno(rp, arg, mode, cred, rvalp);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor error = daplka_evd_free(rp, arg, mode, cred, rvalp);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylordaplka_ep_ioctl(int cmd, daplka_ia_resource_t *rp, intptr_t arg, int mode,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor error = daplka_ep_modify(rp, arg, mode, cred, rvalp);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor error = daplka_ep_free(rp, arg, mode, cred, rvalp);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor error = daplka_ep_connect(rp, arg, mode, cred, rvalp);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor error = daplka_ep_disconnect(rp, arg, mode, cred, rvalp);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor error = daplka_ep_reinit(rp, arg, mode, cred, rvalp);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor error = daplka_ep_create(rp, arg, mode, cred, rvalp);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylordaplka_mr_ioctl(int cmd, daplka_ia_resource_t *rp, intptr_t arg, int mode,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor error = daplka_mr_register(rp, arg, mode, cred, rvalp);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor error = daplka_mr_register_lmr(rp, arg, mode, cred, rvalp);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor error = daplka_mr_register_shared(rp, arg, mode, cred, rvalp);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor error = daplka_mr_deregister(rp, arg, mode, cred, rvalp);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor error = daplka_mr_sync(rp, arg, mode, cred, rvalp);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylordaplka_mw_ioctl(int cmd, daplka_ia_resource_t *rp, intptr_t arg, int mode,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor error = daplka_mw_alloc(rp, arg, mode, cred, rvalp);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor error = daplka_mw_free(rp, arg, mode, cred, rvalp);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylordaplka_cno_ioctl(int cmd, daplka_ia_resource_t *rp, intptr_t arg, int mode,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor error = daplka_cno_alloc(rp, arg, mode, cred, rvalp);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor error = daplka_cno_free(rp, arg, mode, cred, rvalp);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor error = daplka_cno_wait(rp, arg, mode, cred, rvalp);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylordaplka_pd_ioctl(int cmd, daplka_ia_resource_t *rp, intptr_t arg, int mode,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor error = daplka_pd_alloc(rp, arg, mode, cred, rvalp);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor error = daplka_pd_free(rp, arg, mode, cred, rvalp);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylordaplka_sp_ioctl(int cmd, daplka_ia_resource_t *rp, intptr_t arg, int mode,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor error = daplka_service_register(rp, arg, mode, cred, rvalp);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor error = daplka_service_deregister(rp, arg, mode, cred, rvalp);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylordaplka_srq_ioctl(int cmd, daplka_ia_resource_t *rp, intptr_t arg, int mode,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor error = daplka_srq_create(rp, arg, mode, cred, rvalp);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor error = daplka_srq_resize(rp, arg, mode, cred, rvalp);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor error = daplka_srq_free(rp, arg, mode, cred, rvalp);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("daplka_srq_ioctl: cmd(%d) not supported\n", cmd);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylordaplka_misc_ioctl(int cmd, daplka_ia_resource_t *rp, intptr_t arg, int mode,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor error = daplka_cr_accept(rp, arg, mode, cred, rvalp);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor error = daplka_cr_reject(rp, arg, mode, cred, rvalp);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor error = daplka_ia_query(rp, arg, mode, cred, rvalp);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor error = daplka_cr_handoff(rp, arg, mode, cred, rvalp);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylordaplka_ioctl(dev_t dev, int cmd, intptr_t arg, int mode, cred_t *cred,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ia_rp = (daplka_ia_resource_t *)daplka_resource_lookup(rnum);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("ioctl: resource not found, rnum %d\n", rnum);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor D4("ioctl: rnum = %d, cmd = 0x%x\n", rnum, cmd);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor error = daplka_common_ioctl(cmd, rnum, arg, mode, cred, rvalp);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("ioctl: invalid type %d\n", DAPLKA_RS_TYPE(ia_rp));
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor error = daplka_evd_ioctl(cmd, ia_rp, arg, mode, cred, rvalp);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor error = daplka_ep_ioctl(cmd, ia_rp, arg, mode, cred, rvalp);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor error = daplka_mr_ioctl(cmd, ia_rp, arg, mode, cred, rvalp);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor error = daplka_mw_ioctl(cmd, ia_rp, arg, mode, cred, rvalp);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor error = daplka_pd_ioctl(cmd, ia_rp, arg, mode, cred, rvalp);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor error = daplka_sp_ioctl(cmd, ia_rp, arg, mode, cred, rvalp);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor error = daplka_cno_ioctl(cmd, ia_rp, arg, mode, cred, rvalp);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor error = daplka_misc_ioctl(cmd, ia_rp, arg, mode, cred, rvalp);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor error = daplka_srq_ioctl(cmd, ia_rp, arg, mode, cred, rvalp);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("ioctl: invalid dapl type = %d\n", DAPLKA_RS_TYPE(ia_rp));
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor/* ARGSUSED */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylordaplka_open(dev_t *devp, int flag, int otyp, struct cred *cred)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Only zero can be opened, clones are used for resources.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("daplka_open: bad minor %d\n", getminor(*devp));
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * - allocate new minor number
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * - update devp argument to new device
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor/* ARGSUSED */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylordaplka_close(dev_t dev, int flag, int otyp, struct cred *cred)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * remove from resource table.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ia_rp = (daplka_ia_resource_t *)daplka_resource_remove(rnum);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * remove the initial reference
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Resource management routines
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * We start with no resource array. Each time we run out of slots, we
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * reallocate a new larger array and copy the pointer to the new array and
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * a new resource blk is allocated and added to the hash table.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * The resource control block contains:
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * root - array of pointer of resource blks
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * sz - current size of array.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * len - last valid entry in array.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * A search operation based on a resource number is as follows:
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * index = rnum / RESOURCE_BLKSZ;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * ASSERT(index < resource_block.len);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * ASSERT(index < resource_block.sz);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * offset = rnum % RESOURCE_BLKSZ;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * ASSERT(offset >= resource_block.root[index]->base);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * ASSERT(offset < resource_block.root[index]->base + RESOURCE_BLKSZ);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * return resource_block.root[index]->blks[offset];
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * A resource blk is freed when its used count reaches zero.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * initializes the global resource table
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(daplka_resource))
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor rw_init(&daplka_resource.daplka_rct_lock, NULL, RW_DRIVER, NULL);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(daplka_resource))
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * destroys the global resource table
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor rw_enter(&daplka_resource.daplka_rct_lock, RW_WRITER);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor for (i = 0; i < daplka_resource.daplka_rc_len; i++) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor for (j = 0; j < DAPLKA_RC_BLKSZ; j++) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor kmem_free(daplka_resource.daplka_rc_root, (uint_t)sz);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * reserves a slot in the global resource table.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * this is called by the open() syscall. it is needed because
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * at open() time, we do not have sufficient information to
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * create an IA resource. the library needs to subsequently
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * call daplka_ia_create to insert an IA resource into this
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * reserved slot.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor rw_enter(&daplka_resource.daplka_rct_lock, RW_WRITER);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Try to find an empty slot
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor for (i = 0; i < daplka_resource.daplka_rc_len; i++) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor if (blk != NULL && blk->daplka_rcblk_avail > 0) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * found an empty slot in this blk
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor for (j = 0; j < DAPLKA_RC_BLKSZ; j++) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * remember first empty slot
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Couldn't find anything, allocate a new blk
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Do we need to reallocate the root array
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Allocate new array and copy current stuff into it
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor uint_t newsz = (uint_t)daplka_resource.daplka_rc_sz +
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor D3("resource_alloc: increasing no. of buckets to %d\n",
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor p = kmem_zalloc(newsz * sizeof (*p), daplka_km_flags);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor (int)sizeof (*p));
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Copy old data into new space and
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * free old stuff
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor bcopy(daplka_resource.daplka_rc_root, p, oldsz);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Allocate a new blk
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor blk = kmem_zalloc(sizeof (*blk), daplka_km_flags);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ASSERT(daplka_resource.daplka_rc_root[empty] == NULL);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Allocate slot
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor blk->daplka_rcblk_blks[0] = (daplka_resource_t *)DAPLKA_RC_RESERVED;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * removes resource from global resource table
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor rw_enter(&daplka_resource.daplka_rct_lock, RW_WRITER);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("resource_remove: invalid rnum %d\n", rnum);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("resource_remove: invalid rnum %d\n", rnum);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("resource_remove: blk->daplka_rcblk_blks[j] == NULL\n");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor if (blk->daplka_rcblk_avail == DAPLKA_RC_BLKSZ) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * free this blk
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * inserts resource into the slot designated by rnum
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylordaplka_resource_insert(minor_t rnum, daplka_resource_t *rp)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Find resource and lock it in WRITER mode
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * search for available resource slot
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor rw_enter(&daplka_resource.daplka_rct_lock, RW_WRITER);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("resource_insert: resource %d not found\n", rnum);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor return (-1);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor if ((intptr_t)blk->daplka_rcblk_blks[j] == DAPLKA_RC_RESERVED) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("resource_insert: %d not reserved, blk = %p\n",
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("resource_insert: resource %d not found\n", rnum);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * finds resource using minor device number
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Find resource and lock it in READER mode
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * search for available resource slot
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor rw_enter(&daplka_resource.daplka_rct_lock, RW_READER);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("resource_lookup: resource %d not found\n", rnum);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor if (rp == NULL || (intptr_t)rp == DAPLKA_RC_RESERVED) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("resource_lookup: resource %d not found\n", rnum);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * generic hash table implementation
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * daplka_hash_create:
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * initializes a hash table with the specified parameters
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * htblp pointer to hash table
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * nbuckets number of buckets (must be power of 2)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * free_func this function is called on each hash
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * table element when daplka_hash_destroy
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * lookup_func if daplka_hash_lookup is able to find
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the desired object, this function is
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * applied on the object before
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * daplka_hash_lookup returns
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * return value(s):
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * EINVAL nbuckets is not a power of 2
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * ENOMEM cannot allocate buckets
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylordaplka_hash_create(daplka_hash_table_t *htblp, uint_t nbuckets,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor void (*free_func)(void *), void (*lookup_func)(void *))
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor kmem_zalloc(sizeof (daplka_hash_bucket_t) * nbuckets,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor for (i = 0; i < nbuckets; i++) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(htblp->ht_buckets[i]))
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(htblp->ht_buckets[i]))
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor rw_init(&htblp->ht_table_lock, NULL, RW_DRIVER, NULL);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor mutex_init(&htblp->ht_key_lock, NULL, MUTEX_DRIVER, NULL);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor D3("hash_create: done, buckets = %d\n", nbuckets);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * daplka_hash_insert:
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * inserts an object into a hash table
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * htblp pointer to hash table
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * hkeyp pointer to hash key.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * *hkeyp being non-zero means that the caller
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * has generated its own hkey. if *hkeyp is zero,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * this function will generate an hkey for the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * caller. it is recommended that the caller
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * leave the hkey generation to this function
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * because the hkey is more likely to be evenly
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * distributed.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * objp pointer to object to be inserted into
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * hash table
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * hkeyp the generated hkey is returned via this pointer
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * return value(s):
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * EINVAL invalid parameter
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * ENOMEM cannot allocate hash entry
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * 0 successful
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylordaplka_hash_insert(daplka_hash_table_t *htblp, uint64_t *hkeyp, void *objp)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor hep = kmem_zalloc(sizeof (*hep), daplka_km_flags);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor if (*hkeyp == 0) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* generate a new key */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor hkey = htblp->ht_next_hkey = (uint64_t)gethrtime();
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* use user generated key */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* only works if ht_nbuckets is a power of 2 */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor bucket = (uint32_t)(hkey & (htblp->ht_nbuckets - 1));
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* look for duplicate entries */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor DERR("hash_insert: found duplicate hash entry: "
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor "bucket %d, hkey 0x%016llx\n",
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor if (*hkeyp == 0) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor D3("hash_insert: htblp 0x%p, hkey = 0x%016llx, bucket = %d\n",
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * daplka_hash_remove:
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * removes object identified by hkey from hash table
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * htblp pointer to hash table
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * hkey hkey that identifies the object to be removed
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * objpp pointer to pointer to object.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * if remove is successful, the removed object
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * will be returned via *objpp.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * return value(s):
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * EINVAL cannot find hash entry
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * 0 successful
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylordaplka_hash_remove(daplka_hash_table_t *htblp, uint64_t hkey, void **objpp)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor bucket = (uint32_t)(hkey & (htblp->ht_nbuckets - 1));
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor "bucket %d, hkey 0x%016llx\n", bucket, (longlong_t)hkey);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor D3("hash_remove: removed entry, hkey 0x%016llx, bucket %d, "
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor "hb_count %d, hb_count %d\n",
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor (longlong_t)hkey, bucket, hbp->hb_count, htblp->ht_count);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * daplka_hash_walk:
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * walks through the entire hash table. applying func on each of
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the inserted objects. stops walking if func returns non-zero.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * htblp pointer to hash table
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * func function to be applied on each object
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * farg second argument to func
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * lockmode can be RW_WRITER or RW_READER. this
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * allows the caller to choose what type
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * of lock to acquire before walking the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * return value(s):
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylordaplka_hash_walk(daplka_hash_table_t *htblp, int (*func)(void *, void *),
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ASSERT(lockmode == RW_WRITER || lockmode == RW_READER);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* needed for warlock */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor for (bucket = 0; bucket < htblp->ht_nbuckets && retval == 0; bucket++) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * daplka_hash_lookup:
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * finds object from hkey
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * htblp pointer to hash table
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * hkey hkey that identifies the object to be looked up
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * return value(s):
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * NULL if not found
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * object pointer if found
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic void *
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylordaplka_hash_lookup(daplka_hash_table_t *htblp, uint64_t hkey)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor bucket = (uint32_t)(hkey & (htblp->ht_nbuckets - 1));
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor curr_hep = htblp->ht_buckets[bucket].hb_entries;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor "bucket %d, hkey 0x%016llx\n", bucket, (longlong_t)hkey);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * daplka_hash_destroy:
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * destroys hash table. applies free_func on all inserted objects.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * htblp pointer to hash table
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * return value(s):
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* free all elements from hash table */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor for (bucket = 0; bucket < htblp->ht_nbuckets; bucket++) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* build list of elements to be freed */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor D3("hash_destroy: htblp 0x%p, nbuckets %d, freed %d hash entries\n",
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* free all objects, now without holding the hash table lock */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* free hash buckets and destroy locks */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor sizeof (daplka_hash_bucket_t) * htblp->ht_nbuckets);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * daplka_hash_getsize:
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * return the number of objects in hash table
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * htblp pointer to hash table
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * return value(s):
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * number of objects in hash table
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * this function is used as ht_lookup_func above when lookup is called.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * other types of objs may use a more elaborate lookup_func.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor daplka_resource_t *rp = (daplka_resource_t *)obj;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Generates a non-zero 32 bit hash key used for the timer hash table.
1a5e258f5471356ca102c7176637cdce45bac147Josef 'Jeff' Sipek new_hkey = atomic_inc_32_nv(&daplka_timer_hkey);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor } while (new_hkey == 0);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * The DAPL KA debug logging routines
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Add the string str to the end of the debug log, followed by a newline.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * If this is the first time we've written to the log, initialize it.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Note the log is circular; if this string would run over the end,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * we copy the first piece to the end and then the last piece to
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the beginning of the log.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor remlen = (size_t)sizeof (daplka_dbgbuf) - daplka_dbgnext - 1;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor bcopy(str, daplka_dbgbuf + daplka_dbgnext, remlen);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor daplka_dbgbuf[sizeof (daplka_dbgbuf) - 1] = (char)NULL;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor bcopy(str, daplka_dbgbuf + daplka_dbgnext, length);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Add a printf-style message to whichever debug logs we're currently using.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * The system prepends the thread id and high resolution time
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * (nanoseconds are dropped and so are the upper digits)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * to the specified string.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * The unit for timestamp is 10 microseconds.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * It wraps around every 10000 seconds.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Ex: gethrtime() = X ns = X/1000 us = X/10000 10 micro sec.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor int micro_time = (int)((gethrtime() / 10000) % 1000000000);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor (void) sprintf(buff, "th %p tm %9d: ", (void *)curthread, micro_time);