clnt_rdma.c revision 1160694128cd3980cc06abe31af529a887efd310
4fd606d1f5abe38e1f42c38de1d2e895166bd0f4vboxsync * CDDL HEADER START
4fd606d1f5abe38e1f42c38de1d2e895166bd0f4vboxsync * The contents of this file are subject to the terms of the
4fd606d1f5abe38e1f42c38de1d2e895166bd0f4vboxsync * Common Development and Distribution License, Version 1.0 only
4fd606d1f5abe38e1f42c38de1d2e895166bd0f4vboxsync * (the "License"). You may not use this file except in compliance
4fd606d1f5abe38e1f42c38de1d2e895166bd0f4vboxsync * with the License.
4fd606d1f5abe38e1f42c38de1d2e895166bd0f4vboxsync * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
4fd606d1f5abe38e1f42c38de1d2e895166bd0f4vboxsync * See the License for the specific language governing permissions
4fd606d1f5abe38e1f42c38de1d2e895166bd0f4vboxsync * and limitations under the License.
4fd606d1f5abe38e1f42c38de1d2e895166bd0f4vboxsync * When distributing Covered Code, include this CDDL HEADER in each
4fd606d1f5abe38e1f42c38de1d2e895166bd0f4vboxsync * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
4fd606d1f5abe38e1f42c38de1d2e895166bd0f4vboxsync * If applicable, add the following below this CDDL HEADER, with the
4fd606d1f5abe38e1f42c38de1d2e895166bd0f4vboxsync * fields enclosed by brackets "[]" replaced with your own identifying
4fd606d1f5abe38e1f42c38de1d2e895166bd0f4vboxsync * information: Portions Copyright [yyyy] [name of copyright owner]
4fd606d1f5abe38e1f42c38de1d2e895166bd0f4vboxsync * CDDL HEADER END
4fd606d1f5abe38e1f42c38de1d2e895166bd0f4vboxsync * Copyright 2005 Sun Microsystems, Inc. All rights reserved.
4fd606d1f5abe38e1f42c38de1d2e895166bd0f4vboxsync * Use is subject to license terms.
4fd606d1f5abe38e1f42c38de1d2e895166bd0f4vboxsync/* Copyright (c) 1983, 1984, 1985, 1986, 1987, 1988, 1989 AT&T */
4fd606d1f5abe38e1f42c38de1d2e895166bd0f4vboxsync/* All Rights Reserved */
4fd606d1f5abe38e1f42c38de1d2e895166bd0f4vboxsync * Portions of this source code were derived from Berkeley
4fd606d1f5abe38e1f42c38de1d2e895166bd0f4vboxsync * 4.3 BSD under license from the Regents of the University of
4fd606d1f5abe38e1f42c38de1d2e895166bd0f4vboxsync * California.
4fd606d1f5abe38e1f42c38de1d2e895166bd0f4vboxsync#pragma ident "%Z%%M% %I% %E% SMI"
4fd606d1f5abe38e1f42c38de1d2e895166bd0f4vboxsyncstatic enum clnt_stat clnt_rdma_kcallit(CLIENT *, rpcproc_t, xdrproc_t,
4fd606d1f5abe38e1f42c38de1d2e895166bd0f4vboxsyncstatic void clnt_rdma_kerror(CLIENT *, struct rpc_err *);
4fd606d1f5abe38e1f42c38de1d2e895166bd0f4vboxsyncstatic bool_t clnt_rdma_kfreeres(CLIENT *, xdrproc_t, caddr_t);
4fd606d1f5abe38e1f42c38de1d2e895166bd0f4vboxsyncstatic bool_t clnt_rdma_kcontrol(CLIENT *, int, char *);
4fd606d1f5abe38e1f42c38de1d2e895166bd0f4vboxsyncstatic int clnt_rdma_ksettimers(CLIENT *, struct rpc_timers *,
4fd606d1f5abe38e1f42c38de1d2e895166bd0f4vboxsync struct rpc_timers *, int, void(*)(int, int, caddr_t), caddr_t, uint32_t);
4fd606d1f5abe38e1f42c38de1d2e895166bd0f4vboxsync * Operations vector for RDMA based RPC
4fd606d1f5abe38e1f42c38de1d2e895166bd0f4vboxsync * The size of the preserialized RPC header information.
4fd606d1f5abe38e1f42c38de1d2e895166bd0f4vboxsync * Per RPC RDMA endpoint details
4fd606d1f5abe38e1f42c38de1d2e895166bd0f4vboxsynctypedef struct cku_private {
4fd606d1f5abe38e1f42c38de1d2e895166bd0f4vboxsync struct netbuf cku_addr; /* remote netbuf address */
4fd606d1f5abe38e1f42c38de1d2e895166bd0f4vboxsync#define CLNT_RDMA_DELAY 10 /* secs to delay after a connection failure */
4fd606d1f5abe38e1f42c38de1d2e895166bd0f4vboxsynckstat_named_t *rdmarcstat_ptr = (kstat_named_t *)&rdmarcstat;
4fd606d1f5abe38e1f42c38de1d2e895166bd0f4vboxsyncuint_t rdmarcstat_ndata = sizeof (rdmarcstat) / sizeof (kstat_named_t);
4fd606d1f5abe38e1f42c38de1d2e895166bd0f4vboxsyncextern kmutex_t rdmarcstat_lock; /* mutex for rcstat updates */
4fd606d1f5abe38e1f42c38de1d2e895166bd0f4vboxsync#define htop(h) ((cku_private_t *)((h)->cl_private))
4fd606d1f5abe38e1f42c38de1d2e895166bd0f4vboxsyncclnt_rdma_kcreate(char *proto, void *handle, struct netbuf *raddr, int family,
4fd606d1f5abe38e1f42c38de1d2e895166bd0f4vboxsync rpcprog_t pgm, rpcvers_t vers, struct cred *cred, CLIENT **cl)
4fd606d1f5abe38e1f42c38de1d2e895166bd0f4vboxsync * Find underlying RDMATF plugin
4fd606d1f5abe38e1f42c38de1d2e895166bd0f4vboxsync * Should not happen.
4fd606d1f5abe38e1f42c38de1d2e895166bd0f4vboxsync * No matching RDMATF plugin.
4fd606d1f5abe38e1f42c38de1d2e895166bd0f4vboxsync /* call message, just used to pre-serialize below */
4fd606d1f5abe38e1f42c38de1d2e895166bd0f4vboxsync xdrmem_create(&p->cku_outxdr, p->cku_rpchdr, CKU_HDRSIZE, XDR_ENCODE);
4fd606d1f5abe38e1f42c38de1d2e895166bd0f4vboxsync /* pre-serialize call message header */
4fd606d1f5abe38e1f42c38de1d2e895166bd0f4vboxsync * Set up the rpc information
4fd606d1f5abe38e1f42c38de1d2e895166bd0f4vboxsync p->cku_addr.buf = kmem_zalloc(raddr->maxlen, KM_SLEEP);
4fd606d1f5abe38e1f42c38de1d2e895166bd0f4vboxsync return (0);
4fd606d1f5abe38e1f42c38de1d2e895166bd0f4vboxsync kmem_free(p, sizeof (*p));
4fd606d1f5abe38e1f42c38de1d2e895166bd0f4vboxsyncclnt_rdma_kinit(CLIENT *h, char *proto, void *handle, struct netbuf *raddr,
4fd606d1f5abe38e1f42c38de1d2e895166bd0f4vboxsync * Find underlying RDMATF plugin
4fd606d1f5abe38e1f42c38de1d2e895166bd0f4vboxsync * Set up the rpc information
4fd606d1f5abe38e1f42c38de1d2e895166bd0f4vboxsync if (p->cku_addr.maxlen != 0 && p->cku_addr.buf != NULL)
4fd606d1f5abe38e1f42c38de1d2e895166bd0f4vboxsync p->cku_addr.buf = kmem_zalloc(raddr->maxlen, KM_SLEEP);
4fd606d1f5abe38e1f42c38de1d2e895166bd0f4vboxsync/* ARGSUSED */
4fd606d1f5abe38e1f42c38de1d2e895166bd0f4vboxsyncclnt_rdma_kcallit(CLIENT *h, rpcproc_t procnum, xdrproc_t xdr_args,
4fd606d1f5abe38e1f42c38de1d2e895166bd0f4vboxsync caddr_t argsp, xdrproc_t xdr_results, caddr_t resultsp, struct timeval wait)
4fd606d1f5abe38e1f42c38de1d2e895166bd0f4vboxsync XDR *cxdrp = NULL, callxdr; /* for xdrrdma encoding the RPC call */
4fd606d1f5abe38e1f42c38de1d2e895166bd0f4vboxsync XDR *rxdrp = NULL, replxdr; /* for xdrrdma decoding the RPC reply */
4fd606d1f5abe38e1f42c38de1d2e895166bd0f4vboxsync * Get unique xid
4fd606d1f5abe38e1f42c38de1d2e895166bd0f4vboxsync if (p->cku_xid == 0)
4fd606d1f5abe38e1f42c38de1d2e895166bd0f4vboxsync status = RDMA_GET_CONN(p->cku_rd_mod->rdma_ops, &p->cku_addr,
4fd606d1f5abe38e1f42c38de1d2e895166bd0f4vboxsync * Connect failed to server. Could be because of one
4fd606d1f5abe38e1f42c38de1d2e895166bd0f4vboxsync * of several things. In some cases we don't want
4fd606d1f5abe38e1f42c38de1d2e895166bd0f4vboxsync * the caller to retry immediately - delay before
4fd606d1f5abe38e1f42c38de1d2e895166bd0f4vboxsync * returning to caller.
4fd606d1f5abe38e1f42c38de1d2e895166bd0f4vboxsync * Already timed out. No need to delay
4fd606d1f5abe38e1f42c38de1d2e895166bd0f4vboxsync * some more.
4fd606d1f5abe38e1f42c38de1d2e895166bd0f4vboxsync * Failed because of an signal. Very likely
4fd606d1f5abe38e1f42c38de1d2e895166bd0f4vboxsync * the caller will not retry.
4fd606d1f5abe38e1f42c38de1d2e895166bd0f4vboxsync * All other failures - server down or service
4fd606d1f5abe38e1f42c38de1d2e895166bd0f4vboxsync * down or temporary resource failure. Delay before
4fd606d1f5abe38e1f42c38de1d2e895166bd0f4vboxsync * returning to caller.
4fd606d1f5abe38e1f42c38de1d2e895166bd0f4vboxsync ticks = clnt_rdma_min_delay * drv_usectohz(1000000);
4fd606d1f5abe38e1f42c38de1d2e895166bd0f4vboxsync * Get the size of the rpc call message. Need this
4fd606d1f5abe38e1f42c38de1d2e895166bd0f4vboxsync * to determine if the rpc call message will fit in
4fd606d1f5abe38e1f42c38de1d2e895166bd0f4vboxsync * the pre-allocated RDMA buffers. If the rpc call
4fd606d1f5abe38e1f42c38de1d2e895166bd0f4vboxsync * message length is greater that the pre-allocated
4fd606d1f5abe38e1f42c38de1d2e895166bd0f4vboxsync * buffers then, it is a Long RPC. A one time use
4fd606d1f5abe38e1f42c38de1d2e895166bd0f4vboxsync * buffer is allocated and registered for the Long
4fd606d1f5abe38e1f42c38de1d2e895166bd0f4vboxsync * RPC call.
4fd606d1f5abe38e1f42c38de1d2e895166bd0f4vboxsync msglen += xdrrdma_authsize(h->cl_auth, p->cku_cred,
4fd606d1f5abe38e1f42c38de1d2e895166bd0f4vboxsync msglen += xdrrdma_sizeof(xdr_args, argsp, rdma_minchunk);
4fd606d1f5abe38e1f42c38de1d2e895166bd0f4vboxsync * Long RPC. Allocate one time use custom buffer.
goto done;
if (cle)
"clnt_rdma_kcallit: XDR_PUTINT32/AUTH_MARSHAL/xdr_args failed");
goto done;
goto done;
goto done;
goto done;
goto done;
if (cl) {
goto done;
goto done;
if (cl) {
goto done;
#ifdef DEBUG
if (rdma_clnt_debug) {
if (status != 0) {
#ifdef DEBUG
if (rdma_clnt_debug)
goto done;
#ifdef DEBUG
if (rdma_clnt_debug)
if (status) {
goto rdma_done;
#ifdef DEBUG
if (rdma_clnt_debug)
if (status) {
goto rdma_done;
goto rdma_done;
case AUTH_BADCRED:
case AUTH_BADVERF:
case AUTH_INVALIDRESP:
case AUTH_TOOWEAK:
case AUTH_FAILED:
case RPCSEC_GSS_NOCRED:
case RPCSEC_GSS_FAILED:
case AUTH_REJECTEDCRED:
case AUTH_REJECTEDVERF:
if (cl) {
goto done;
goto done;
p->cku_xid);
#ifdef DEBUG
if (rdma_clnt_debug)
p->cku_xid);
done:
if (cxdrp)
if (rxdrp) {
if (recvlist) {
static bool_t
static bool_t
return (TRUE);
char *pf, *p;
int error = 0;
if (!rdma_modloaded) {
if (!rdma_modloaded) {
if (error)
if (!rdma_dev_available)
&handle);
KM_SLEEP);