c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota/*
16e76cdd6e3cfaac7d91c3b0644ee1bc6cf52347agiri * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved.
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota */
16e76cdd6e3cfaac7d91c3b0644ee1bc6cf52347agiri
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota/*
16e76cdd6e3cfaac7d91c3b0644ee1bc6cf52347agiri * This file contains code imported from the OFED rds source file ib_rdma.c
16e76cdd6e3cfaac7d91c3b0644ee1bc6cf52347agiri * Oracle elects to have and use the contents of ib_rdma.c under and governed
16e76cdd6e3cfaac7d91c3b0644ee1bc6cf52347agiri * by the OpenIB.org BSD license (see below for full license text). However,
16e76cdd6e3cfaac7d91c3b0644ee1bc6cf52347agiri * the following notice accompanied the original version of this file:
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota */
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota/*
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * Copyright (c) 2006 Oracle. All rights reserved.
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota *
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * This software is available to you under a choice of one of two
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * licenses. You may choose to be licensed under the terms of the GNU
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * General Public License (GPL) Version 2, available from the file
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * COPYING in the main directory of this source tree, or the
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * OpenIB.org BSD license below:
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota *
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * Redistribution and use in source and binary forms, with or
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * without modification, are permitted provided that the following
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * conditions are met:
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota *
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * - Redistributions of source code must retain the above
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * copyright notice, this list of conditions and the following
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * disclaimer.
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota *
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * - Redistributions in binary form must reproduce the above
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * copyright notice, this list of conditions and the following
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * disclaimer in the documentation and/or other materials
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * provided with the distribution.
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota *
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * SOFTWARE.
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota *
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota */
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota#include <sys/rds.h>
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota#include <netinet/in.h>
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota#include <sys/ib/clients/rdsv3/rdsv3.h>
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota#include <sys/ib/clients/rdsv3/rdma.h>
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota#include <sys/ib/clients/rdsv3/ib.h>
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota#include <sys/ib/clients/rdsv3/rdsv3_debug.h>
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota/*
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * This is stored as mr->r_trans_private.
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota */
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Otastruct rdsv3_ib_mr {
5d5562f583b2b6affe19bdce0b3c8b1840d667a4Eiji Ota list_node_t m_obj; /* list obj of rdsv3_fmr_pool list */
5d5562f583b2b6affe19bdce0b3c8b1840d667a4Eiji Ota struct rdsv3_ib_device *m_device;
5d5562f583b2b6affe19bdce0b3c8b1840d667a4Eiji Ota struct rdsv3_fmr_pool *m_pool; /* hca fmr pool */
5d5562f583b2b6affe19bdce0b3c8b1840d667a4Eiji Ota unsigned int m_inval:1;
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota struct rdsv3_scatterlist *sg;
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota unsigned int sg_len;
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota uint64_t *dma;
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota int sg_dma_len;
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota /* DDI pinned memory */
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota ddi_umem_cookie_t umem_cookie;
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota /* IBTF type definitions */
5d5562f583b2b6affe19bdce0b3c8b1840d667a4Eiji Ota ibt_hca_hdl_t rc_hca_hdl;
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota ibt_fmr_pool_hdl_t fmr_pool_hdl;
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota ibt_ma_hdl_t rc_ma_hdl;
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota ibt_mr_hdl_t rc_fmr_hdl;
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota ibt_pmr_desc_t rc_mem_desc;
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota};
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota/*
5d5562f583b2b6affe19bdce0b3c8b1840d667a4Eiji Ota * delayed freed fmr's
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota */
5d5562f583b2b6affe19bdce0b3c8b1840d667a4Eiji Otastruct rdsv3_fmr_pool {
5d5562f583b2b6affe19bdce0b3c8b1840d667a4Eiji Ota list_t f_list; /* list of freed mr */
5d5562f583b2b6affe19bdce0b3c8b1840d667a4Eiji Ota kmutex_t f_lock; /* lock of fmr pool */
5d5562f583b2b6affe19bdce0b3c8b1840d667a4Eiji Ota int32_t f_listcnt;
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota};
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Otastatic int rdsv3_ib_flush_mr_pool(struct rdsv3_ib_device *rds_ibdev,
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota ibt_fmr_pool_hdl_t pool_hdl, int free_all);
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Otastatic void rdsv3_ib_teardown_mr(struct rdsv3_ib_mr *ibmr);
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Otastatic void rdsv3_ib_mr_pool_flush_worker(struct rdsv3_work_s *work);
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Otastatic struct rdsv3_ib_mr *rdsv3_ib_alloc_fmr(struct rdsv3_ib_device
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota *rds_ibdev);
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Otastatic int rdsv3_ib_map_fmr(struct rdsv3_ib_device *rds_ibdev,
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota struct rdsv3_ib_mr *ibmr, struct buf *bp, unsigned int nents);
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Otastatic struct rdsv3_ib_device *
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Otardsv3_ib_get_device(uint32_be_t ipaddr)
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota{
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota struct rdsv3_ib_device *rds_ibdev;
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota struct rdsv3_ib_ipaddr *i_ipaddr;
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota RDSV3_DPRINTF4("rdsv3_ib_get_device", "Enter: ipaddr: 0x%x", ipaddr);
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota RDSV3_FOR_EACH_LIST_NODE(rds_ibdev, &rdsv3_ib_devices, list) {
5d5562f583b2b6affe19bdce0b3c8b1840d667a4Eiji Ota rw_enter(&rds_ibdev->rwlock, RW_READER);
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota RDSV3_FOR_EACH_LIST_NODE(i_ipaddr, &rds_ibdev->ipaddr_list,
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota list) {
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota if (i_ipaddr->ipaddr == ipaddr) {
5d5562f583b2b6affe19bdce0b3c8b1840d667a4Eiji Ota rw_exit(&rds_ibdev->rwlock);
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota return (rds_ibdev);
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota }
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota }
5d5562f583b2b6affe19bdce0b3c8b1840d667a4Eiji Ota rw_exit(&rds_ibdev->rwlock);
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota }
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota RDSV3_DPRINTF4("rdsv3_ib_get_device", "Return: ipaddr: 0x%x", ipaddr);
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota return (NULL);
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota}
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Otastatic int
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Otardsv3_ib_add_ipaddr(struct rdsv3_ib_device *rds_ibdev, uint32_be_t ipaddr)
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota{
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota struct rdsv3_ib_ipaddr *i_ipaddr;
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota RDSV3_DPRINTF4("rdsv3_ib_add_ipaddr", "rds_ibdev: %p ipaddr: %x",
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota rds_ibdev, ipaddr);
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota i_ipaddr = kmem_alloc(sizeof (*i_ipaddr), KM_NOSLEEP);
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota if (!i_ipaddr)
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota return (-ENOMEM);
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota i_ipaddr->ipaddr = ipaddr;
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota
5d5562f583b2b6affe19bdce0b3c8b1840d667a4Eiji Ota rw_enter(&rds_ibdev->rwlock, RW_WRITER);
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota list_insert_tail(&rds_ibdev->ipaddr_list, i_ipaddr);
5d5562f583b2b6affe19bdce0b3c8b1840d667a4Eiji Ota rw_exit(&rds_ibdev->rwlock);
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota return (0);
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota}
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Otastatic void
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Otardsv3_ib_remove_ipaddr(struct rdsv3_ib_device *rds_ibdev, uint32_be_t ipaddr)
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota{
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota struct rdsv3_ib_ipaddr *i_ipaddr, *next;
5d5562f583b2b6affe19bdce0b3c8b1840d667a4Eiji Ota struct rdsv3_ib_ipaddr *to_free = NULL;
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota RDSV3_DPRINTF4("rdsv3_ib_remove_ipaddr", "rds_ibdev: %p, ipaddr: %x",
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota rds_ibdev, ipaddr);
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota
5d5562f583b2b6affe19bdce0b3c8b1840d667a4Eiji Ota rw_enter(&rds_ibdev->rwlock, RW_WRITER);
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota RDSV3_FOR_EACH_LIST_NODE_SAFE(i_ipaddr, next, &rds_ibdev->ipaddr_list,
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota list) {
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota if (i_ipaddr->ipaddr == ipaddr) {
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota list_remove_node(&i_ipaddr->list);
5d5562f583b2b6affe19bdce0b3c8b1840d667a4Eiji Ota to_free = i_ipaddr;
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota break;
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota }
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota }
5d5562f583b2b6affe19bdce0b3c8b1840d667a4Eiji Ota rw_exit(&rds_ibdev->rwlock);
5d5562f583b2b6affe19bdce0b3c8b1840d667a4Eiji Ota
5d5562f583b2b6affe19bdce0b3c8b1840d667a4Eiji Ota if (to_free) {
5d5562f583b2b6affe19bdce0b3c8b1840d667a4Eiji Ota kmem_free(i_ipaddr, sizeof (*i_ipaddr));
5d5562f583b2b6affe19bdce0b3c8b1840d667a4Eiji Ota }
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota RDSV3_DPRINTF4("rdsv3_ib_remove_ipaddr",
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota "Return: rds_ibdev: %p, ipaddr: %x", rds_ibdev, ipaddr);
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota}
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Otaint
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Otardsv3_ib_update_ipaddr(struct rdsv3_ib_device *rds_ibdev, uint32_be_t ipaddr)
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota{
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota struct rdsv3_ib_device *rds_ibdev_old;
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota RDSV3_DPRINTF4("rdsv3_ib_update_ipaddr", "rds_ibdev: %p, ipaddr: %x",
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota rds_ibdev, ipaddr);
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota rds_ibdev_old = rdsv3_ib_get_device(ipaddr);
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota if (rds_ibdev_old)
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota rdsv3_ib_remove_ipaddr(rds_ibdev_old, ipaddr);
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota return (rdsv3_ib_add_ipaddr(rds_ibdev, ipaddr));
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota}
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Otavoid
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Otardsv3_ib_add_conn(struct rdsv3_ib_device *rds_ibdev,
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota struct rdsv3_connection *conn)
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota{
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota struct rdsv3_ib_connection *ic = conn->c_transport_data;
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota RDSV3_DPRINTF4("rdsv3_ib_add_conn", "rds_ibdev: %p, conn: %p",
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota rds_ibdev, conn);
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota /* conn was previously on the nodev_conns_list */
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota mutex_enter(&ib_nodev_conns_lock);
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota ASSERT(!list_is_empty(&ib_nodev_conns));
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota ASSERT(list_link_active(&ic->ib_node));
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota list_remove_node(&ic->ib_node);
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota mutex_enter(&rds_ibdev->spinlock);
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota list_insert_tail(&rds_ibdev->conn_list, ic);
d2b539e744e90927cf7a57df3475145c279d68f9agiri ic->i_on_dev_list = B_TRUE;
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota mutex_exit(&rds_ibdev->spinlock);
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota mutex_exit(&ib_nodev_conns_lock);
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota}
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Otavoid
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Otardsv3_ib_remove_conn(struct rdsv3_ib_device *rds_ibdev,
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota struct rdsv3_connection *conn)
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota{
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota struct rdsv3_ib_connection *ic = conn->c_transport_data;
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota RDSV3_DPRINTF4("rdsv3_ib_remove_conn", "rds_ibdev: %p, conn: %p",
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota rds_ibdev, conn);
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota /* place conn on nodev_conns_list */
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota mutex_enter(&ib_nodev_conns_lock);
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota mutex_enter(&rds_ibdev->spinlock);
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota ASSERT(list_link_active(&ic->ib_node));
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota list_remove_node(&ic->ib_node);
d2b539e744e90927cf7a57df3475145c279d68f9agiri ic->i_on_dev_list = B_FALSE;
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota mutex_exit(&rds_ibdev->spinlock);
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota list_insert_tail(&ib_nodev_conns, ic);
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota mutex_exit(&ib_nodev_conns_lock);
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota RDSV3_DPRINTF4("rdsv3_ib_remove_conn",
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota "Return: rds_ibdev: %p, conn: %p", rds_ibdev, conn);
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota}
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Otavoid
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota__rdsv3_ib_destroy_conns(struct list *list, kmutex_t *list_lock)
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota{
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota struct rdsv3_ib_connection *ic, *_ic;
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota list_t tmp_list;
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota RDSV3_DPRINTF4("__rdsv3_ib_destroy_conns", "Enter: list: %p", list);
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota /* avoid calling conn_destroy with irqs off */
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota mutex_enter(list_lock);
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota list_splice(list, &tmp_list);
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota mutex_exit(list_lock);
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota RDSV3_FOR_EACH_LIST_NODE_SAFE(ic, _ic, &tmp_list, ib_node) {
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota rdsv3_conn_destroy(ic->conn);
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota }
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota RDSV3_DPRINTF4("__rdsv3_ib_destroy_conns", "Return: list: %p", list);
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota}
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Otavoid
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Otardsv3_ib_destroy_mr_pool(struct rdsv3_ib_device *rds_ibdev)
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota{
5d5562f583b2b6affe19bdce0b3c8b1840d667a4Eiji Ota struct rdsv3_fmr_pool *pool = rds_ibdev->fmr_pool;
5d5562f583b2b6affe19bdce0b3c8b1840d667a4Eiji Ota
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota RDSV3_DPRINTF4("rdsv3_ib_destroy_mr_pool", "Enter: ibdev: %p",
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota rds_ibdev);
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota if (rds_ibdev->fmr_pool_hdl == NULL)
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota return;
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota
5d5562f583b2b6affe19bdce0b3c8b1840d667a4Eiji Ota if (pool) {
5d5562f583b2b6affe19bdce0b3c8b1840d667a4Eiji Ota list_destroy(&pool->f_list);
5d5562f583b2b6affe19bdce0b3c8b1840d667a4Eiji Ota kmem_free((void *) pool, sizeof (*pool));
5d5562f583b2b6affe19bdce0b3c8b1840d667a4Eiji Ota }
5d5562f583b2b6affe19bdce0b3c8b1840d667a4Eiji Ota
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota (void) rdsv3_ib_flush_mr_pool(rds_ibdev, rds_ibdev->fmr_pool_hdl, 1);
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota (void) ibt_destroy_fmr_pool(ib_get_ibt_hca_hdl(rds_ibdev->dev),
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota rds_ibdev->fmr_pool_hdl);
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota}
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota#define IB_FMR_MAX_BUF_SIZE 0x1000000 /* 16MB max buf */
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Otaint
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Otardsv3_ib_create_mr_pool(struct rdsv3_ib_device *rds_ibdev)
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota{
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota uint_t h_page_sz;
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota ibt_fmr_pool_attr_t fmr_attr;
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota ibt_status_t ibt_status;
5d5562f583b2b6affe19bdce0b3c8b1840d667a4Eiji Ota struct rdsv3_fmr_pool *pool;
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota RDSV3_DPRINTF4("rdsv3_ib_create_mr_pool",
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota "Enter: ibdev: %p", rds_ibdev);
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota
5d5562f583b2b6affe19bdce0b3c8b1840d667a4Eiji Ota pool = (struct rdsv3_fmr_pool *)kmem_zalloc(sizeof (*pool), KM_NOSLEEP);
5d5562f583b2b6affe19bdce0b3c8b1840d667a4Eiji Ota if (pool == NULL) {
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota return (-ENOMEM);
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota }
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota /* setup FMR pool attributes */
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota h_page_sz = rds_ibdev->hca_attr.hca_page_sz * 1024;
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota fmr_attr.fmr_max_pages_per_fmr = (IB_FMR_MAX_BUF_SIZE / h_page_sz) + 2;
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota fmr_attr.fmr_pool_size = RDSV3_FMR_POOL_SIZE;
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota fmr_attr.fmr_dirty_watermark = 128;
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota fmr_attr.fmr_cache = B_FALSE;
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota fmr_attr.fmr_flags = IBT_MR_NOSLEEP | IBT_MR_ENABLE_LOCAL_WRITE |
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota IBT_MR_ENABLE_REMOTE_WRITE | IBT_MR_ENABLE_REMOTE_READ;
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota fmr_attr.fmr_page_sz = h_page_sz;
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota fmr_attr.fmr_func_hdlr = NULL;
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota fmr_attr.fmr_func_arg = (void *) NULL;
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota /* create the FMR pool */
5d5562f583b2b6affe19bdce0b3c8b1840d667a4Eiji Ota ibt_status = ibt_create_fmr_pool(rds_ibdev->ibt_hca_hdl,
5d5562f583b2b6affe19bdce0b3c8b1840d667a4Eiji Ota rds_ibdev->pd->ibt_pd, &fmr_attr, &rds_ibdev->fmr_pool_hdl);
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota if (ibt_status != IBT_SUCCESS) {
5d5562f583b2b6affe19bdce0b3c8b1840d667a4Eiji Ota kmem_free((void *) pool, sizeof (*pool));
5d5562f583b2b6affe19bdce0b3c8b1840d667a4Eiji Ota rds_ibdev->fmr_pool = NULL;
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota return (-ENOMEM);
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota }
5d5562f583b2b6affe19bdce0b3c8b1840d667a4Eiji Ota
5d5562f583b2b6affe19bdce0b3c8b1840d667a4Eiji Ota list_create(&pool->f_list, sizeof (struct rdsv3_ib_mr),
5d5562f583b2b6affe19bdce0b3c8b1840d667a4Eiji Ota offsetof(struct rdsv3_ib_mr, m_obj));
5d5562f583b2b6affe19bdce0b3c8b1840d667a4Eiji Ota mutex_init(&pool->f_lock, NULL, MUTEX_DRIVER, NULL);
5d5562f583b2b6affe19bdce0b3c8b1840d667a4Eiji Ota rds_ibdev->fmr_pool = pool;
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota rds_ibdev->max_fmrs = fmr_attr.fmr_pool_size;
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota rds_ibdev->fmr_message_size = fmr_attr.fmr_max_pages_per_fmr;
5d5562f583b2b6affe19bdce0b3c8b1840d667a4Eiji Ota
5d5562f583b2b6affe19bdce0b3c8b1840d667a4Eiji Ota RDSV3_DPRINTF2("rdsv3_ib_create_mr_pool",
5d5562f583b2b6affe19bdce0b3c8b1840d667a4Eiji Ota "Exit: ibdev: %p fmr_pool: %p", rds_ibdev, pool);
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota return (0);
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota}
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Otavoid
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Otardsv3_ib_get_mr_info(struct rdsv3_ib_device *rds_ibdev,
fe817b6022080da0a98b5d2d8cd179f594d6ca5eEiji Ota struct rds_info_rdma_connection *iinfo)
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota{
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota iinfo->rdma_mr_max = rds_ibdev->max_fmrs;
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota iinfo->rdma_mr_size = rds_ibdev->fmr_message_size;
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota}
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Otavoid *
fe817b6022080da0a98b5d2d8cd179f594d6ca5eEiji Otardsv3_ib_get_mr(struct rds_iovec *args, unsigned long nents,
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota struct rdsv3_sock *rs, uint32_t *key_ret)
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota{
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota struct rdsv3_ib_device *rds_ibdev;
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota struct rdsv3_ib_mr *ibmr = NULL;
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota ddi_umem_cookie_t umem_cookie;
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota size_t umem_len;
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota caddr_t umem_addr;
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota int ret;
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota struct buf *bp;
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota RDSV3_DPRINTF4("rdsv3_ib_get_mr", "Enter: args.addr: %p", args->addr);
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota rds_ibdev = rdsv3_ib_get_device(rs->rs_bound_addr);
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota if (rds_ibdev == NULL)
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota return (void *)(PTR_ERR(-EFAULT));
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota ibmr = rdsv3_ib_alloc_fmr(rds_ibdev);
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota if (IS_ERR(ibmr))
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota return (ibmr);
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota /* pin user memory pages */
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota umem_len = ptob(btopr(args->bytes +
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota ((uintptr_t)args->addr & PAGEOFFSET)));
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota umem_addr = (caddr_t)((uintptr_t)args->addr & ~PAGEOFFSET);
d2b539e744e90927cf7a57df3475145c279d68f9agiri ret = umem_lockmemory(umem_addr, umem_len,
d2b539e744e90927cf7a57df3475145c279d68f9agiri DDI_UMEMLOCK_WRITE | DDI_UMEMLOCK_READ,
d2b539e744e90927cf7a57df3475145c279d68f9agiri &umem_cookie, NULL, NULL);
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota if (ret != 0) {
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota kmem_free((void *) ibmr, sizeof (*ibmr));
b27516f55237249607f754e6e42e865f12456675agiri ibmr = ERR_PTR(-ret);
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota return (ibmr);
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota }
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota /* transpose umem_cookie to buf structure for rdsv3_ib_map_fmr() */
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota bp = ddi_umem_iosetup(umem_cookie, 0, umem_len,
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota B_WRITE, 0, 0, NULL, DDI_UMEM_SLEEP);
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota ret = rdsv3_ib_map_fmr(rds_ibdev, ibmr, bp, nents);
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota freerbuf(bp); /* free bp */
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota if (ret == 0) {
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota ibmr->umem_cookie = umem_cookie;
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota *key_ret = (uint32_t)ibmr->rc_mem_desc.pmd_rkey;
5d5562f583b2b6affe19bdce0b3c8b1840d667a4Eiji Ota ibmr->m_device = rds_ibdev;
5d5562f583b2b6affe19bdce0b3c8b1840d667a4Eiji Ota ibmr->m_pool = rds_ibdev->fmr_pool;
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota RDSV3_DPRINTF4("rdsv3_ib_get_mr",
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota "Return: ibmr: %p umem_cookie %p", ibmr, ibmr->umem_cookie);
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota return (ibmr);
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota } else { /* error return */
6e18d381c642549b8bb1774a803d3510aec6baafagiri RDSV3_DPRINTF2("rdsv3_ib_get_mr", "map_fmr failed (errno=%d)\n",
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota ret);
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota ddi_umem_unlock(umem_cookie);
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota kmem_free((void *)ibmr, sizeof (*ibmr));
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota return (ERR_PTR(ret));
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota }
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota}
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Otastatic struct rdsv3_ib_mr *
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Otardsv3_ib_alloc_fmr(struct rdsv3_ib_device *rds_ibdev)
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota{
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota struct rdsv3_ib_mr *ibmr;
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota RDSV3_DPRINTF4("rdsv3_ib_alloc_fmr", "Enter: ibdev: %p", rds_ibdev);
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota if (rds_ibdev->fmr_pool_hdl) {
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota ibmr = (struct rdsv3_ib_mr *)kmem_zalloc(sizeof (*ibmr),
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota KM_SLEEP);
5d5562f583b2b6affe19bdce0b3c8b1840d667a4Eiji Ota ibmr->rc_hca_hdl = ib_get_ibt_hca_hdl(rds_ibdev->dev);
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota ibmr->fmr_pool_hdl = rds_ibdev->fmr_pool_hdl;
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota return (ibmr);
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota }
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota return (struct rdsv3_ib_mr *)(PTR_ERR(-ENOMEM));
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota}
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Otastatic int
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Otardsv3_ib_map_fmr(struct rdsv3_ib_device *rds_ibdev, struct rdsv3_ib_mr *ibmr,
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota struct buf *bp, unsigned int nents)
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota{
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota ibt_va_attr_t va_attr;
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota ibt_reg_req_t reg_req;
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota uint_t paddr_list_len;
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota uint_t page_sz;
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota ibt_status_t ibt_status;
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota /* LINTED E_FUNC_SET_NOT_USED */
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota unsigned int l_nents = nents;
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota RDSV3_DPRINTF4("rdsv3_ib_map_fmr", "Enter: ibmr: %p", ibmr);
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota RDSV3_DPRINTF4("rdsv3_ib_map_fmr", "buf addr: %p", bp->b_un.b_addr);
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota /* setup ibt_map_mem_area attributes */
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota bzero(&va_attr, sizeof (ibt_va_attr_t));
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota va_attr.va_buf = bp;
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota va_attr.va_flags = IBT_VA_FMR | IBT_VA_BUF;
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota page_sz = rds_ibdev->hca_attr.hca_page_sz * 1024; /* in kbytes */
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota paddr_list_len = (bp->b_bcount / page_sz) + 2; /* start + end pg */
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota /* map user buffer to HCA address */
5d5562f583b2b6affe19bdce0b3c8b1840d667a4Eiji Ota ibt_status = ibt_map_mem_area(ibmr->rc_hca_hdl,
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota &va_attr, paddr_list_len, &reg_req, &ibmr->rc_ma_hdl);
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota if (ibt_status != IBT_SUCCESS) {
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota return (-ENOMEM);
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota }
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota /* use a free entry from FMR pool to register the specified memory */
5d5562f583b2b6affe19bdce0b3c8b1840d667a4Eiji Ota ibt_status = ibt_register_physical_fmr(ibmr->rc_hca_hdl,
5d5562f583b2b6affe19bdce0b3c8b1840d667a4Eiji Ota ibmr->fmr_pool_hdl,
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota &reg_req.fn_arg, &ibmr->rc_fmr_hdl, &ibmr->rc_mem_desc);
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota if (ibt_status != IBT_SUCCESS) {
5d5562f583b2b6affe19bdce0b3c8b1840d667a4Eiji Ota RDSV3_DPRINTF2("rdsv3_ib_map_fmr", "reg_phy_fmr failed %d",
5d5562f583b2b6affe19bdce0b3c8b1840d667a4Eiji Ota ibt_status);
5d5562f583b2b6affe19bdce0b3c8b1840d667a4Eiji Ota (void) ibt_unmap_mem_area(ibmr->rc_hca_hdl,
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota ibmr->rc_ma_hdl);
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota if (ibt_status == IBT_INSUFF_RESOURCE) {
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota return (-ENOBUFS);
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota }
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota return (-EINVAL);
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota }
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota RDSV3_DPRINTF4("rdsv3_ib_map_fmr", "Return: ibmr: %p rkey: 0x%x",
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota ibmr, (uint32_t)ibmr->rc_mem_desc.pmd_rkey);
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota return (0);
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota}
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Otavoid
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Otardsv3_ib_sync_mr(void *trans_private, int direction)
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota{
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota /* LINTED E_FUNC_SET_NOT_USED */
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota void *l_trans_private = trans_private;
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota /* LINTED E_FUNC_SET_NOT_USED */
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota int l_direction = direction;
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota /* FMR Sync not needed in Solaris on PCI-ex systems */
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota RDSV3_DPRINTF4("rdsv3_ib_sync_mr", "Enter:");
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota}
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Otavoid
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Otardsv3_ib_flush_mrs(void)
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota{
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota struct rdsv3_ib_device *rds_ibdev;
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota RDSV3_DPRINTF4("rdsv3_ib_flush_mrs", "Enter:");
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota RDSV3_FOR_EACH_LIST_NODE(rds_ibdev, &rdsv3_ib_devices, list) {
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota if (rds_ibdev->fmr_pool_hdl) {
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota (void) rdsv3_ib_flush_mr_pool(rds_ibdev,
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota rds_ibdev->fmr_pool_hdl, 0);
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota }
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota }
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota}
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Otastatic void
5d5562f583b2b6affe19bdce0b3c8b1840d667a4Eiji Otardsv3_ib_drop_mr(struct rdsv3_ib_mr *ibmr)
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota{
5d5562f583b2b6affe19bdce0b3c8b1840d667a4Eiji Ota /* return the fmr to the IBTF pool */
5d5562f583b2b6affe19bdce0b3c8b1840d667a4Eiji Ota (void) ibt_deregister_fmr(ibmr->rc_hca_hdl, ibmr->rc_fmr_hdl);
5d5562f583b2b6affe19bdce0b3c8b1840d667a4Eiji Ota (void) ibt_unmap_mem_area(ibmr->rc_hca_hdl, ibmr->rc_ma_hdl);
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota (void) ddi_umem_unlock(ibmr->umem_cookie);
5d5562f583b2b6affe19bdce0b3c8b1840d667a4Eiji Ota kmem_free((void *) ibmr, sizeof (*ibmr));
5d5562f583b2b6affe19bdce0b3c8b1840d667a4Eiji Ota}
5d5562f583b2b6affe19bdce0b3c8b1840d667a4Eiji Ota
5d5562f583b2b6affe19bdce0b3c8b1840d667a4Eiji Otavoid
5d5562f583b2b6affe19bdce0b3c8b1840d667a4Eiji Otardsv3_ib_drain_mrlist_fn(void *data)
5d5562f583b2b6affe19bdce0b3c8b1840d667a4Eiji Ota{
5d5562f583b2b6affe19bdce0b3c8b1840d667a4Eiji Ota struct rdsv3_fmr_pool *pool = (struct rdsv3_fmr_pool *)data;
5d5562f583b2b6affe19bdce0b3c8b1840d667a4Eiji Ota ibt_hca_hdl_t hca_hdl;
5d5562f583b2b6affe19bdce0b3c8b1840d667a4Eiji Ota ibt_fmr_pool_hdl_t fmr_pool_hdl;
5d5562f583b2b6affe19bdce0b3c8b1840d667a4Eiji Ota unsigned int inval;
5d5562f583b2b6affe19bdce0b3c8b1840d667a4Eiji Ota struct rdsv3_ib_mr *ibmr;
5d5562f583b2b6affe19bdce0b3c8b1840d667a4Eiji Ota list_t *listp = &pool->f_list;
5d5562f583b2b6affe19bdce0b3c8b1840d667a4Eiji Ota kmutex_t *lockp = &pool->f_lock;
5d5562f583b2b6affe19bdce0b3c8b1840d667a4Eiji Ota int i;
5d5562f583b2b6affe19bdce0b3c8b1840d667a4Eiji Ota
5d5562f583b2b6affe19bdce0b3c8b1840d667a4Eiji Ota inval = 0;
5d5562f583b2b6affe19bdce0b3c8b1840d667a4Eiji Ota i = 0;
5d5562f583b2b6affe19bdce0b3c8b1840d667a4Eiji Ota for (;;) {
5d5562f583b2b6affe19bdce0b3c8b1840d667a4Eiji Ota mutex_enter(lockp);
5d5562f583b2b6affe19bdce0b3c8b1840d667a4Eiji Ota ibmr = (struct rdsv3_ib_mr *)list_remove_head(listp);
5d5562f583b2b6affe19bdce0b3c8b1840d667a4Eiji Ota if (ibmr)
5d5562f583b2b6affe19bdce0b3c8b1840d667a4Eiji Ota pool->f_listcnt--;
5d5562f583b2b6affe19bdce0b3c8b1840d667a4Eiji Ota mutex_exit(lockp);
5d5562f583b2b6affe19bdce0b3c8b1840d667a4Eiji Ota if (!ibmr)
5d5562f583b2b6affe19bdce0b3c8b1840d667a4Eiji Ota break;
5d5562f583b2b6affe19bdce0b3c8b1840d667a4Eiji Ota if ((inval == 0) && ibmr->m_inval) {
5d5562f583b2b6affe19bdce0b3c8b1840d667a4Eiji Ota inval = 1;
5d5562f583b2b6affe19bdce0b3c8b1840d667a4Eiji Ota hca_hdl = ibmr->rc_hca_hdl;
5d5562f583b2b6affe19bdce0b3c8b1840d667a4Eiji Ota fmr_pool_hdl = ibmr->fmr_pool_hdl;
5d5562f583b2b6affe19bdce0b3c8b1840d667a4Eiji Ota }
5d5562f583b2b6affe19bdce0b3c8b1840d667a4Eiji Ota i++;
5d5562f583b2b6affe19bdce0b3c8b1840d667a4Eiji Ota rdsv3_ib_drop_mr(ibmr);
5d5562f583b2b6affe19bdce0b3c8b1840d667a4Eiji Ota }
5d5562f583b2b6affe19bdce0b3c8b1840d667a4Eiji Ota if (inval)
5d5562f583b2b6affe19bdce0b3c8b1840d667a4Eiji Ota (void) ibt_flush_fmr_pool(hca_hdl, fmr_pool_hdl);
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota}
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Otavoid
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Otardsv3_ib_free_mr(void *trans_private, int invalidate)
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota{
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota struct rdsv3_ib_mr *ibmr = trans_private;
5d5562f583b2b6affe19bdce0b3c8b1840d667a4Eiji Ota rdsv3_af_thr_t *af_thr;
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota RDSV3_DPRINTF4("rdsv3_ib_free_mr", "Enter: ibmr: %p inv: %d",
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota ibmr, invalidate);
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota
5d5562f583b2b6affe19bdce0b3c8b1840d667a4Eiji Ota /* save af_thr at local as ibmr might be freed at mutex_exit */
5d5562f583b2b6affe19bdce0b3c8b1840d667a4Eiji Ota af_thr = ibmr->m_device->fmr_soft_cq;
5d5562f583b2b6affe19bdce0b3c8b1840d667a4Eiji Ota ibmr->m_inval = (unsigned int) invalidate;
5d5562f583b2b6affe19bdce0b3c8b1840d667a4Eiji Ota mutex_enter(&ibmr->m_pool->f_lock);
5d5562f583b2b6affe19bdce0b3c8b1840d667a4Eiji Ota list_insert_tail(&ibmr->m_pool->f_list, ibmr);
5d5562f583b2b6affe19bdce0b3c8b1840d667a4Eiji Ota ibmr->m_pool->f_listcnt++;
5d5562f583b2b6affe19bdce0b3c8b1840d667a4Eiji Ota mutex_exit(&ibmr->m_pool->f_lock);
5d5562f583b2b6affe19bdce0b3c8b1840d667a4Eiji Ota
5d5562f583b2b6affe19bdce0b3c8b1840d667a4Eiji Ota rdsv3_af_thr_fire(af_thr);
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota}
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Otastatic int
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Otardsv3_ib_flush_mr_pool(struct rdsv3_ib_device *rds_ibdev,
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota ibt_fmr_pool_hdl_t pool_hdl, int free_all)
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota{
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota /* LINTED E_FUNC_SET_NOT_USED */
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota int l_free_all = free_all;
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota RDSV3_DPRINTF4("rdsv3_ib_flush_mr_pool", "Enter: pool: %p", pool_hdl);
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota rdsv3_ib_stats_inc(s_ib_rdma_mr_pool_flush);
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota (void) ibt_flush_fmr_pool(ib_get_ibt_hca_hdl(rds_ibdev->dev),
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota pool_hdl);
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota return (0);
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota}