16e76cdd6e3cfaac7d91c3b0644ee1bc6cf52347agiri * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved.
16e76cdd6e3cfaac7d91c3b0644ee1bc6cf52347agiri * This file contains code imported from the OFED rds source file rdma.c
16e76cdd6e3cfaac7d91c3b0644ee1bc6cf52347agiri * Oracle elects to have and use the contents of rdma.c under and governed
16e76cdd6e3cfaac7d91c3b0644ee1bc6cf52347agiri * by the OpenIB.org BSD license (see below for full license text). However,
16e76cdd6e3cfaac7d91c3b0644ee1bc6cf52347agiri * the following notice accompanied the original version of this file:
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * Copyright (c) 2007 Oracle. All rights reserved.
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * This software is available to you under a choice of one of two
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * licenses. You may choose to be licensed under the terms of the GNU
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * General Public License (GPL) Version 2, available from the file
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * COPYING in the main directory of this source tree, or the
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * OpenIB.org BSD license below:
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * Redistribution and use in source and binary forms, with or
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * without modification, are permitted provided that the following
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * conditions are met:
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * - Redistributions of source code must retain the above
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * copyright notice, this list of conditions and the following
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * disclaimer.
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * - Redistributions in binary form must reproduce the above
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * copyright notice, this list of conditions and the following
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * disclaimer in the documentation and/or other materials
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * provided with the distribution.
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * SOFTWARE.
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota#define RB_CLEAR_NODE(nodep) AVL_SETPARENT(nodep, nodep);
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * - build with sparse
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * - should we limit the size of a mr region? let transport return failure?
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * - should we detect duplicate keys on a socket? hmm.
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * - an rdma is an mlock, apply rlimit?
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * get the number of pages by looking at the page indices that the start and
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * end addresses fall in.
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * Returns 0 if the vec is invalid. It is invalid if the number of bytes
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * causes the address to wrap or overflows an unsigned int. This comes
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * from being stored in the 'length' member of 'struct rdsv3_scatterlist'.
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Otastatic unsigned int
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota return (0);
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota return (((vec->addr + vec->bytes + PAGESIZE - 1) >>
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Otastatic struct rdsv3_mr *
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Otardsv3_mr_tree_walk(struct avl_tree *root, uint32_t key,
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * Destroy the transport-specific part of a MR.
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota "RDS: destroy mr key is %x refcnt %u",
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota mr->r_trans->free_mr(trans_private, mr->r_invalidate);
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * By the time this is called we can't have any more ioctls called on
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * the socket so we don't need to worry about racing with others.
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota /* Release any MRs associated with this socket */
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota mr = container_of(node, struct rdsv3_mr, r_rb_node);
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota if (rs->rs_transport && rs->rs_transport->flush_mrs)
fe817b6022080da0a98b5d2d8cd179f594d6ca5eEiji Ota__rdsv3_rdma_map(struct rdsv3_sock *rs, struct rds_get_mr_args *args,
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota mr = kmem_zalloc(sizeof (struct rdsv3_mr), KM_NOSLEEP);
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * Obtain a transport specific MR. If this succeeds, the
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * s/g list is now owned by the MR.
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * Note that dma_map() implies that pending writes are
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * flushed to RAM, so no dma_sync is needed here.
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota trans_private = rs->rs_transport->get_mr(&args->vec, nents, rs,
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * The user may pass us an unaligned address, but we can only
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * map page aligned regions. So we keep the offset, and build
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * a 64bit cookie containing <R_Key, offset> and pass that
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota cookie = rdsv3_rdma_make_cookie(mr->r_key, args->vec.addr & ~PAGEMASK);
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * copy value of cookie to user address at args->cookie_addr
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota if (ret != 0) {
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota "RDS: get_mr mr 0x%p addr 0x%llx key 0x%x",
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * Inserting the new MR into the rbtree bumps its
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * reference count.
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota found = rdsv3_mr_tree_walk(&rs->rs_rdma_keys, mr->r_key, mr);
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Otardsv3_get_mr(struct rdsv3_sock *rs, const void *optval, int optlen)
fe817b6022080da0a98b5d2d8cd179f594d6ca5eEiji Ota sizeof (struct rds_get_mr_args));
cadbfdc3bdb156e92d7a88978bc98ea87f6e037fEiji Otardsv3_get_mr_for_dest(struct rdsv3_sock *rs, const void *optval,
fe817b6022080da0a98b5d2d8cd179f594d6ca5eEiji Ota if (optlen != sizeof (struct rds_get_mr_for_dest_args))
fe817b6022080da0a98b5d2d8cd179f594d6ca5eEiji Ota bcopy((struct rds_get_mr_for_dest_args *)optval, &args,
fe817b6022080da0a98b5d2d8cd179f594d6ca5eEiji Ota sizeof (struct rds_get_mr_for_dest_args));
cadbfdc3bdb156e92d7a88978bc98ea87f6e037fEiji Ota * Initially, just behave like get_mr().
cadbfdc3bdb156e92d7a88978bc98ea87f6e037fEiji Ota * TODO: Implement get_mr as wrapper around this
cadbfdc3bdb156e92d7a88978bc98ea87f6e037fEiji Ota * and deprecate it.
cadbfdc3bdb156e92d7a88978bc98ea87f6e037fEiji Ota return (__rdsv3_rdma_map(rs, &new_args, NULL, NULL));
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * Free the MR indicated by the given R_Key
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Otardsv3_free_mr(struct rdsv3_sock *rs, const void *optval, int optlen)
fe817b6022080da0a98b5d2d8cd179f594d6ca5eEiji Ota sizeof (struct rds_free_mr_args));
fe817b6022080da0a98b5d2d8cd179f594d6ca5eEiji Ota if (ddi_copyin((struct rds_free_mr_args *)optval, &args,
fe817b6022080da0a98b5d2d8cd179f594d6ca5eEiji Ota sizeof (struct rds_free_mr_args), 0))
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota /* Special case - a null cookie means flush all unused MRs */
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota if (!rs->rs_transport || !rs->rs_transport->flush_mrs)
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota return (0);
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * Look up the MR given its R_key and remove it from the rbtree
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * so nobody else finds it.
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * This should also prevent races with rdsv3_rdma_unuse.
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * call rdsv3_destroy_mr() ourselves so that we're sure it's done
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * by time we return. If we let rdsv3_mr_put() do it it might not
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * happen until someone else drops their ref.
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota return (0);
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * This is called when we receive an extension header that
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * tells us this MR was used. It allows us to implement
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * use_once semantics
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Otardsv3_rdma_unuse(struct rdsv3_sock *rs, uint32_t r_key, int force)
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota RDSV3_DPRINTF4("rdsv3_rdma_unuse", "Enter rkey: 0x%x", r_key);
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota mr = rdsv3_mr_tree_walk(&rs->rs_rdma_keys, r_key, NULL);
5d5562f583b2b6affe19bdce0b3c8b1840d667a4Eiji Ota "rdsv3: trying to unuse MR with unknown r_key %u!", r_key);
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * May have to issue a dma_sync on this memory region.
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * Note we could avoid this if the operation was a RDMA READ,
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * but at this point we can't tell.
5d5562f583b2b6affe19bdce0b3c8b1840d667a4Eiji Ota mr->r_trans->sync_mr(mr->r_trans_private, DMA_FROM_DEVICE);
5d5562f583b2b6affe19bdce0b3c8b1840d667a4Eiji Ota * If the MR was marked as invalidate, this will
5d5562f583b2b6affe19bdce0b3c8b1840d667a4Eiji Ota * trigger an async flush.
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota unsigned int i;
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota /* deallocate RDMA resources on rdsv3_message */
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota kmem_free(ro->r_notifier, sizeof (*ro->r_notifier));
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * args is a pointer to an in-kernel copy in the sendmsg cmsg.
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Otastatic struct rdsv3_rdma_op *
fe817b6022080da0a98b5d2d8cd179f594d6ca5eEiji Otardsv3_rdma_prepare(struct rdsv3_sock *rs, struct rds_rdma_args *args)
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota unsigned int i;
fe817b6022080da0a98b5d2d8cd179f594d6ca5eEiji Ota op->r_write = !!(args->flags & RDS_RDMA_READWRITE);
fe817b6022080da0a98b5d2d8cd179f594d6ca5eEiji Ota op->r_notify = !!(args->flags & RDS_RDMA_NOTIFY_ME);
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * We allocate an uninitialized notifier here, because
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * we don't want to do that in the completion handler. We
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * would have to use GFP_ATOMIC there, and don't want to deal
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * with failed allocations.
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota op->r_notifier = kmem_alloc(sizeof (struct rdsv3_notifier),
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * The cookie contains the R_Key of the remote memory region, and
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * optionally an offset into it. This is how we implement RDMA into
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * unaligned memory.
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * When setting up the RDMA, we need to add that offset to the
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * destination address (which is really an offset into the MR)
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * FIXME: We may want to move this into ib_rdma.c
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota "RDS: rdma prepare nr_local %llu rva %llx rkey %x",
fe817b6022080da0a98b5d2d8cd179f594d6ca5eEiji Ota local_vec = (struct rds_iovec *)(unsigned long) args->local_vec_addr;
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota /* pin the scatter list of user buffers */
fe817b6022080da0a98b5d2d8cd179f594d6ca5eEiji Ota sizeof (struct rds_iovec), 0)) {
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota if (nr == 0) {
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota "rdsv3_pages_in_vec returned 0");
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota /* pin user memory pages */
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota umem_addr = (caddr_t)((uintptr_t)vec.addr & ~PAGEOFFSET);
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota if (ret != 0) {
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota "RDS: nr_bytes %u nr %u vec.bytes %llu vec.addr %llx",
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota "RDS nr_bytes %u remote_bytes %u do not match",
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * The application asks for a RDMA transfer.
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * Extract all arguments and set up the rdma_op
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Otardsv3_cmsg_rdma_args(struct rdsv3_sock *rs, struct rdsv3_message *rm,
265a964d7aa43c47170d21d2f01bcf873d7fd79dEiji Ota /* uint64_t alignment on the buffer */
8ddb3a46e1a09773609249d578c6decb530951c2Eiji Ota uint64_t buf[CEIL(CMSG_LEN(sizeof (struct rds_rdma_args)),
265a964d7aa43c47170d21d2f01bcf873d7fd79dEiji Ota if (cmsg->cmsg_len != CMSG_LEN(sizeof (struct rds_rdma_args)) ||
265a964d7aa43c47170d21d2f01bcf873d7fd79dEiji Ota ASSERT(sizeof (buf) >= cmsg->cmsg_len && ((uintptr_t)buf & 0x7) == 0);
265a964d7aa43c47170d21d2f01bcf873d7fd79dEiji Ota bcopy(CMSG_DATA(cmsg), (char *)buf, cmsg->cmsg_len);
265a964d7aa43c47170d21d2f01bcf873d7fd79dEiji Ota op = rdsv3_rdma_prepare(rs, (struct rds_rdma_args *)buf);
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota return (0);
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * The application wants us to pass an RDMA destination (aka MR)
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * to the remote
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Otardsv3_cmsg_rdma_dest(struct rdsv3_sock *rs, struct rdsv3_message *rm,
265a964d7aa43c47170d21d2f01bcf873d7fd79dEiji Ota if (cmsg->cmsg_len != CMSG_LEN(sizeof (rds_rdma_cookie_t)) ||
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * We are reusing a previously mapped MR here. Most likely, the
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * application has written to the buffer, so we need to explicitly
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * flush those writes to RAM. Otherwise the HCA may not see them
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * when doing a DMA from that buffer.
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota mr = rdsv3_mr_tree_walk(&rs->rs_rdma_keys, r_key, NULL);
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota mr->r_trans->sync_mr(mr->r_trans_private, DMA_TO_DEVICE);
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * The application passes us an address range it wants to enable RDMA
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * to/from. We map the area, and save the <R_Key,offset> pair
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * in rm->m_rdma_cookie. This causes it to be sent along to the peer
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Ota * in an extension header.
c0dd49bdd68c0d758a67d56f07826f3b45cfc664Eiji Otardsv3_cmsg_rdma_map(struct rdsv3_sock *rs, struct rdsv3_message *rm,
265a964d7aa43c47170d21d2f01bcf873d7fd79dEiji Ota /* uint64_t alignment on the buffer */
8ddb3a46e1a09773609249d578c6decb530951c2Eiji Ota uint64_t buf[CEIL(CMSG_LEN(sizeof (struct rds_get_mr_args)),
265a964d7aa43c47170d21d2f01bcf873d7fd79dEiji Ota if (cmsg->cmsg_len != CMSG_LEN(sizeof (struct rds_get_mr_args)) ||
265a964d7aa43c47170d21d2f01bcf873d7fd79dEiji Ota ASSERT(sizeof (buf) >= cmsg->cmsg_len && ((uintptr_t)buf & 0x7) == 0);
265a964d7aa43c47170d21d2f01bcf873d7fd79dEiji Ota bcopy(CMSG_DATA(cmsg), (char *)buf, cmsg->cmsg_len);