9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * CDDL HEADER START
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * The contents of this file are subject to the terms of the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Common Development and Distribution License (the "License").
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * You may not use this file except in compliance with the License.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * See the License for the specific language governing permissions
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * and limitations under the License.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * When distributing Covered Code, include this CDDL HEADER in each
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * If applicable, add the following below this CDDL HEADER, with the
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * fields enclosed by brackets "[]" replaced with your own identifying
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * information: Portions Copyright [yyyy] [name of copyright owner]
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * CDDL HEADER END
1cfa752f4e24c34133009b0f6c139127a5c461deRamaswamy Tummala * Copyright (c) 2009, 2010, Oracle and/or its affiliates. All rights reserved.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor/* function prototypes */
1cfa752f4e24c34133009b0f6c139127a5c461deRamaswamy Tummalastatic DAT_RETURN dapli_process_tavor_node(char *dev_path, int *hca_idx,
1cfa752f4e24c34133009b0f6c139127a5c461deRamaswamy Tummalastatic DAT_RETURN dapli_process_ia(dladm_ib_attr_t *ib_attr, DAPL_HCA *hca_ptr,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic int dapli_process_fake_ibds(DAPL_HCA **hca_list, int hca_idx);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor#endif /* IBHOSTS_NAMING */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylorstatic struct dapls_ib_hca_state g_tavor_state[MAX_HCAS];
1cfa752f4e24c34133009b0f6c139127a5c461deRamaswamy Tummala if (dladm_open(&dlh) != DLADM_STATUS_OK) {
1cfa752f4e24c34133009b0f6c139127a5c461deRamaswamy Tummala "init_hca: dladm_open failed\n");
1cfa752f4e24c34133009b0f6c139127a5c461deRamaswamy Tummala if ((ibnex_fd = open(IBNEX_DEVCTL_DEV, O_RDONLY)) < 0) {
1cfa752f4e24c34133009b0f6c139127a5c461deRamaswamy Tummala dat_status = DAT_ERROR(DAT_NAME_NOT_FOUND, 0);
1cfa752f4e24c34133009b0f6c139127a5c461deRamaswamy Tummala "init_hca: could not open ib nexus (%s)\n",
1cfa752f4e24c34133009b0f6c139127a5c461deRamaswamy Tummala if ((dladm_name2info(dlh, hca_ptr->name, &linkid, NULL, &class,
1cfa752f4e24c34133009b0f6c139127a5c461deRamaswamy Tummala "init_hca: %s not found - couldn't get partition info\n",
1cfa752f4e24c34133009b0f6c139127a5c461deRamaswamy Tummala query_hca.hca_guid = ib_attr.dia_hca_guid;
1cfa752f4e24c34133009b0f6c139127a5c461deRamaswamy Tummala query_hca.hca_device_path = hca_device_path;
1cfa752f4e24c34133009b0f6c139127a5c461deRamaswamy Tummala query_hca.hca_device_path_alloc_sz = sizeof (hca_device_path);
1cfa752f4e24c34133009b0f6c139127a5c461deRamaswamy Tummala if (ioctl(ibnex_fd, IBNEX_CTL_QUERY_HCA, &query_hca) == -1) {
1cfa752f4e24c34133009b0f6c139127a5c461deRamaswamy Tummala dat_status = DAT_ERROR(DAT_NAME_NOT_FOUND, 0);
1cfa752f4e24c34133009b0f6c139127a5c461deRamaswamy Tummala "init_hca: %s not found; query_hca failed\n",
1cfa752f4e24c34133009b0f6c139127a5c461deRamaswamy Tummala if (strcmp(query_hca.hca_info.hca_driver_name, "tavor") == 0)
1cfa752f4e24c34133009b0f6c139127a5c461deRamaswamy Tummala else if (strcmp(query_hca.hca_info.hca_driver_name, "arbel") == 0)
1cfa752f4e24c34133009b0f6c139127a5c461deRamaswamy Tummala else if (strcmp(query_hca.hca_info.hca_driver_name, "hermon") == 0) {
1cfa752f4e24c34133009b0f6c139127a5c461deRamaswamy Tummala "init_hca: %s not found\n", hca_ptr->name);
1cfa752f4e24c34133009b0f6c139127a5c461deRamaswamy Tummala dat_status = dapli_process_tavor_node(hca_device_path, &hca_idx,
1cfa752f4e24c34133009b0f6c139127a5c461deRamaswamy Tummala "init_hcas: %s process_tavor_node failed(0x%x)\n",
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor if (dapli_process_fake_ibds(hca_ptr, hca_idx) == 0) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* no entries were found */
1cfa752f4e24c34133009b0f6c139127a5c461deRamaswamy Tummala dat_status = dapli_process_ia(&ib_attr, hca_ptr, hca_idx);
1cfa752f4e24c34133009b0f6c139127a5c461deRamaswamy Tummala "init_hcas: %s process_ia failed(0x%x)\n",
1cfa752f4e24c34133009b0f6c139127a5c461deRamaswamy Tummaladapli_process_tavor_node(char *dev_path, int *hca_idx, int try_blueflame)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * page size == 0 means this entry is not occupied
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor "process_tavor: all hcas are being used!\n");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor return (DAT_ERROR(DAT_INSUFFICIENT_RESOURCES, 0));
1ed53a3f65abecaadc1b967e341970ad0f6b2aebBill Taylor for (i = 0; i < idx; i++) {
1ed53a3f65abecaadc1b967e341970ad0f6b2aebBill Taylor if (strcmp(dev_path, g_tavor_state[i].hca_path) == 0) {
1ed53a3f65abecaadc1b967e341970ad0f6b2aebBill Taylor /* no need for a refcnt */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Add 16 to accomodate the prefix "/devices" and suffix ":devctl" */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor if (strlen("/devices") + strlen(dev_path) + strlen(":devctl") + 1 >
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor "process_tavor: devfs path %s is too long\n",
1ed53a3f65abecaadc1b967e341970ad0f6b2aebBill Taylor (void) dapl_os_strcpy(g_tavor_state[idx].hca_path, dev_path);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor "process_tavor: page_size == 0\n");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor "process_tavor: cannot open %s: %s\n",
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor return (DAT_ERROR(DAT_INSUFFICIENT_RESOURCES, 0));
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * libc can't handle fd's greater than 255, in order to
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * ensure that these values remain available make fd > 255.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Note: not needed for LP64
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor "process_tavor: cannot F_DUPFD: %s\n", strerror(errno));
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor#endif /* _LP64 */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor "process_tavor: cannot F_SETFD: %s\n", strerror(errno));
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor uarpg_offset = (((off64_t)cur_pid << MLNX_UMAP_RSRC_TYPE_SHIFT) |
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor mapaddr = mmap64((void *)0, pagesize, PROT_READ | PROT_WRITE,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor "process_tavor: mmap failed %s\n", strerror(errno));
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor return (DAT_ERROR(DAT_INSUFFICIENT_RESOURCES, 0));
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* Try to do the Hermon Blueflame page mapping */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor uarpg_offset = (((off64_t)cur_pid << MLNX_UMAP_RSRC_TYPE_SHIFT) |
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor mapaddr = mmap64((void *)0, pagesize, PROT_READ | PROT_WRITE,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* This is not considered to be fatal. Charge on! */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor "process_tavor: mmap of blueflame page failed %s\n",
1cfa752f4e24c34133009b0f6c139127a5c461deRamaswamy Tummaladapli_process_ia(dladm_ib_attr_t *ib_attr, DAPL_HCA *hca_ptr, int hca_idx)
1cfa752f4e24c34133009b0f6c139127a5c461deRamaswamy Tummala if (ib_attr->dia_hca_guid == 0 || ib_attr->dia_portnum == 0 ||
1cfa752f4e24c34133009b0f6c139127a5c461deRamaswamy Tummala "process_ia: invalid properties: guid 0x%016llx, "
1cfa752f4e24c34133009b0f6c139127a5c461deRamaswamy Tummala "port %d, pkey 0x%08x\n", ib_attr->dia_hca_guid,
1cfa752f4e24c34133009b0f6c139127a5c461deRamaswamy Tummala ib_attr->dia_portnum, (uint_t)ib_attr->dia_pkey);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * if an interface has both v4 and v6 addresses plumbed,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * we'll take the v4 address.
1cfa752f4e24c34133009b0f6c139127a5c461deRamaswamy Tummala "process_ia: socket failed: %s\n", strerror(errno));
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor return (DAT_ERROR(DAT_INSUFFICIENT_RESOURCES, 0));
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* check if name will fit in lifr_name */
1cfa752f4e24c34133009b0f6c139127a5c461deRamaswamy Tummala if (dapl_os_strlen(hca_ptr->name) >= LIFNAMSIZ) {
1cfa752f4e24c34133009b0f6c139127a5c461deRamaswamy Tummala "process_ia: if name overflow %s\n",
1cfa752f4e24c34133009b0f6c139127a5c461deRamaswamy Tummala (void) dapl_os_strcpy(lifreq.lifr_name, hca_ptr->name);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor retval = ioctl(sfd, SIOCGLIFADDR, (caddr_t)&lifreq);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * the interface is not plumbed.
1cfa752f4e24c34133009b0f6c139127a5c461deRamaswamy Tummala "process_ia: %s: ip address not found\n",
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor return (DAT_ERROR(DAT_INSUFFICIENT_RESOURCES, 0));
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * we've failed to find a v4 address. now
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * let's try v6.
1cfa752f4e24c34133009b0f6c139127a5c461deRamaswamy Tummala hca_ptr->node_GUID = ib_attr->dia_hca_guid;
1cfa752f4e24c34133009b0f6c139127a5c461deRamaswamy Tummala hca_ptr->partition_key = ib_attr->dia_pkey;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor (void) dapl_os_memcpy((void *)&hca_ptr->hca_address,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor (void *)&lifreq.lifr_addr, sizeof (hca_ptr->hca_address));
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor hca_ptr->max_inline_send = dapls_tavor_max_inline();
1cfa752f4e24c34133009b0f6c139127a5c461deRamaswamy Tummala "process_ia: interface %s, hca guid 0x%016llx, port %d, "
1cfa752f4e24c34133009b0f6c139127a5c461deRamaswamy Tummala "pkey 0x%08x, ip addr %s\n", lifreq.lifr_name, hca_ptr->node_GUID,
1cfa752f4e24c34133009b0f6c139127a5c461deRamaswamy Tummala hca_ptr->port_num, hca_ptr->partition_key, dapls_inet_ntop(
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor (struct sockaddr *)&hca_ptr->hca_address, addr_buf, 64));
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor for (i = 0; i < MAX_HCAS; i++) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Uinitialize the per hca instance state
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor for (i = 0; i < MAX_HCAS; i++) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor dapl_os_assert(g_tavor_state[i].uarpg_baseaddr ==
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor "ib_state_fini: "
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor "munmap(0x%p, 0x%llx) failed(%d)\n",
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor if ((g_tavor_state[i].bf_pg_baseaddr != NULL) &&
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor "ib_state_fini: "
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor "munmap(0x%p, 0x%llx) of blueflame failed(%d)\n",
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * dapls_ib_open_hca
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * *hca_ptr pointer to hca device
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * *ib_hca_handle_p pointer to provide HCA handle
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * DAT_SUCCESS
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * DAT_INSUFFICIENT_RESOURCES
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor "dapls_ib_open_hca: init_hca failed %d\n", dat_status);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * libc can't handle fd's greater than 255, in order to
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * ensure that these values remain available make fd > 255.
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * Note: not needed for LP64
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor "dapls_ib_open_hca: cannot F_DUPFD: %s\n",
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor#endif /* _LP64 */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor "dapls_ib_open_hca: cannot F_SETFD: %s\n", strerror(errno));
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor hca_p = (struct dapls_ib_hca_handle *)dapl_os_alloc(
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor sizeof (struct dapls_ib_hca_handle));
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor (void) dapl_os_memzero((void *)args.ia_sadata, DAPL_ATS_NBYTES);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* pass down local ip address to be stored in SA */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* LINTED: E_BAD_PTR_CAST_ALIGN */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor switch (s->sa_family) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* LINTED: E_BAD_PTR_CAST_ALIGN */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor /* LINTED: E_BAD_PTR_CAST_ALIGN */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor break; /* fall through */
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor "open_hca: ia_create failed, fd %d, "
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor "guid 0x%016llx, port %d, pkey 0x%x, version %d\n",
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor hca_p->hca_fd = g_tavor_state[hca_ptr->tavor_idx].hca_fd;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor hca_p->ia_uar = g_tavor_state[hca_ptr->tavor_idx].uarpg_baseaddr;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor hca_p->ia_bf = g_tavor_state[hca_ptr->tavor_idx].bf_pg_baseaddr;
1ed53a3f65abecaadc1b967e341970ad0f6b2aebBill Taylor hca_p->ia_bf_toggle = &g_tavor_state[hca_ptr->tavor_idx].bf_toggle;
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor "open_hca: ia_created, hca_p 0x%p, fd %d, "
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor "rnum %d, guid 0x%016llx, port %d, pkey 0x%x\n",
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor hca_p, hca_p->ia_fd, hca_p->ia_rnum, hca_ptr->node_GUID,
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * dapls_ib_close_hca
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * ib_hca_handle provide HCA handle
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * DAT_SUCCESS
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor * DAT_INSUFFICIENT_RESOURCES
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor "close_hca: ib_hca_handle == NULL\n");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor "close_hca: closing hca 0x%p, fd %d, rnum %d\n",
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor ib_hca_handle, ib_hca_handle->ia_fd, ib_hca_handle->ia_rnum);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor sizeof (struct dapls_ib_hca_handle));
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylordapli_process_fake_ibds(DAPL_HCA *hca_ptr, int hca_idx)
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor "fake_ibds: ibhosts not found!\n");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor "fake_ibds: hostname not found!\n");
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor sscanf(line_buf, "%s %llx %llx", host_buf, &prefix, &guid);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor (void) sprintf(line_buf, "%s-ib%d", localhost, count + 1);
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor if (strncmp(line_buf, host_buf, strlen(line_buf)) == 0) {
9e39c5ba00a55fa05777cc94b148296af305e135Bill Taylor#endif /* IBHOSTS_NAMING */