rpc_rdma.h revision 7523bef8c782ba29fe98b26ccdad47b1cc70f7d7
/*
* CDDL HEADER START
*
* The contents of this file are subject to the terms of the
* Common Development and Distribution License (the "License").
* You may not use this file except in compliance with the License.
*
* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
* See the License for the specific language governing permissions
* and limitations under the License.
*
* When distributing Covered Code, include this CDDL HEADER in each
* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
* If applicable, add the following below this CDDL HEADER, with the
* fields enclosed by brackets "[]" replaced with your own identifying
* information: Portions Copyright [yyyy] [name of copyright owner]
*
* CDDL HEADER END
*/
/*
* Copyright 2009 Sun Microsystems, Inc. All rights reserved.
* Use is subject to license terms.
*/
/*
* Copyright (c) 2007, The Ohio State University. All rights reserved.
*
* Portions of this source code is developed by the team members of
* The Ohio State University's Network-Based Computing Laboratory (NBCL),
* headed by Professor Dhabaleswar K. (DK) Panda.
*
* Acknowledgements to contributions from developors:
* Ranjit Noronha: noronha@cse.ohio-state.edu
* Lei Chai : chail@cse.ohio-state.edu
* Weikuan Yu : yuw@cse.ohio-state.edu
*
*/
#ifndef _RPC_RPC_RDMA_H
#define _RPC_RPC_RDMA_H
#include <rpc/rpc_sztypes.h>
#ifdef __cplusplus
extern "C" {
#endif
/*
* The size of an RPC call or reply message
*/
#define RPC_MSG_SZ 1024
/*
* RDMA chunk size
*/
#define RDMA_MINCHUNK 1024
/*
* Storage for a chunk list
*/
#define RPC_CL_SZ 1024
/*
* Chunk size
*/
#define MINCHUNK 1024
/*
* Size of receive buffer
*/
#define RPC_BUF_SIZE 2048
#define NOWAIT 0 /* don't wait for operation of complete */
/*
* RDMA xdr buffer control and other control flags. Add new flags here,
* set them in private structure for xdr over RDMA in xdr_rdma.c
*/
#define XDR_RDMA_CHUNK 0x1
#define XDR_RDMA_WLIST_REG 0x2
#define XDR_RDMA_RLIST_REG 0x4
#define LONG_REPLY_LEN 65536
#define WCL_BUF_LEN 32768
#define RCL_BUF_LEN 32768
struct xdr_ops *xdrrdma_xops(void);
/*
* Credit Control Structures.
*/
typedef enum rdma_cc_type {
RDMA_CC_CLNT, /* CONN is for a client */
RDMA_CC_SRV /* CONN is for a server */
/*
* Client side credit control data structure.
*/
typedef struct rdma_clnt_cred_ctrl {
/*
* Server side credit control data structure.
*/
typedef struct rdma_srv_cred_ctrl {
typedef enum {
typedef enum {
CLIST_REG_SOURCE = 1,
} clist_dstsrc;
/*
* Return codes from RDMA operations
*/
typedef enum {
RDMA_SUCCESS = 0, /* successful operation */
/*
* connection errors
*/
/*
* RDMATF errors
*/
RDMA_HCA_ATTACH = 18,
RDMA_HCA_DETACH = 19,
/*
* fallback error
*/
} rdma_stat;
/*
* Memory region context. This is an RDMA provider generated
* handle for a registered arbitrary size contiguous virtual
* memory. The RDMA Interface Adapter needs this for local or
* remote memory access.
*
* The mrc_rmr field holds the remote memory region context
* which is sent over-the-wire to provide the remote host
* with RDMA access to the memory region.
*/
struct mrc {
union {
struct mr {
} mr;
} lhdl;
};
/*
* Memory management for the RDMA buffers
*/
/*
* RDMA buffer types
*/
typedef enum {
SEND_BUFFER, /* buf for send msg */
SEND_DESCRIPTOR, /* buf used for send msg descriptor in plugins only */
RECV_BUFFER, /* buf for recv msg */
RECV_DESCRIPTOR, /* buf used for recv msg descriptor in plugins only */
RDMA_LONG_BUFFER /* chunk buf used in RDMATF only and not in plugins */
} rdma_btype;
/*
* RDMA buffer information
*/
typedef struct rdma_buf {
} rdma_buf_t;
/*
* The XDR offset value is used by the XDR
* routine to identify the position in the
* RPC message where the opaque object would
* normally occur. Neither the data content
* of the chunk, nor its size field are included
* in the RPC message. The XDR offset is calculated
* as if the chunks were present.
*
* The remaining fields identify the chunk of data
* on the sender. The c_memhandle identifies a
* registered RDMA memory region and the c_addr
* and c_len fields identify the chunk within it.
*/
struct clist {
union {
} w;
union {
} u;
};
/*
* max 4M wlist xfer size
* This is defined because the rfs3_tsize service requires
* svc_req struct (which we don't have that in krecv).
*/
enum rdma_proc {
RDMA_MSG = 0, /* chunk list and RPC msg follow */
};
/*
* Listener information for a service
*/
struct rdma_svc_data {
queue_t q; /* queue_t to place incoming pkts */
int active; /* If active, after registeration startup */
};
/*
* Per RDMA plugin module information.
* Will be populated by each plugin
* module during its initialization.
*/
typedef struct rdma_mod {
char *rdma_api; /* "kvipl", "ibtf", etc */
int rdma_count; /* # of devices */
} rdma_mod_t;
/*
* Registry of RDMA plugins
*/
typedef struct rdma_registry {
/*
* RDMA MODULE state flags (r_mod_state).
*/
#define RDMA_MOD_ACTIVE 1
#define RDMA_MOD_INACTIVE 0
/*
* RDMA transport information
*/
typedef struct rdma_info {
} rdma_info_t;
typedef enum {
C_IDLE = 0x00000001,
C_CONN_PEND = 0x00000002,
C_CONNECTED = 0x00000004,
C_ERROR_CONN = 0x00000008,
C_DISCONN_PEND = 0x00000010,
C_REMOTE_DOWN = 0x00000020
} conn_c_state;
/* c_flags */
/*
* RDMA Connection information
*/
typedef struct conn {
char *c_netid; /* tcp or tcp6 token */
int c_ref; /* no. of clients of connection */
int c_flags; /* flags for connection management */
union {
} CONN;
/*
* Data transferred from plugin interrupt to svc_queuereq()
*/
typedef struct rdma_recv_data {
int status;
/* structure used to pass information for READ over rdma write */
typedef enum {
RCI_WRITE_UIO_CHUNK = 1,
RCI_WRITE_ADDR_CHUNK = 2,
RCI_REPLY_CHUNK = 3
} rci_type_t;
typedef struct {
union {
} rci_a;
typedef struct {
typedef struct {
struct clist *rwci_wlist;
/*
* Operations vector for RDMA transports.
*/
typedef struct rdmaops {
/* Network */
void **handle);
/* Connection */
/* Server side listner start and stop routines */
void (*rdma_svc_listen)(struct rdma_svc_data *);
void (*rdma_svc_stop)(struct rdma_svc_data *);
/* Memory */
struct mrc *, void **, void *);
void *, void *);
/* Buffer */
/* Transfer */
/* RDMA */
/* INFO */
} rdmaops_t;
typedef struct rdma_svc_wait {
extern rdma_svc_wait_t rdma_wait;
/*
* RDMA operations.
*/
#define RDMA_REL_CONN(conn) \
#ifdef _KERNEL
extern rdma_registry_t *rdma_mod_head;
extern int rdma_modloaded; /* flag for loading RDMA plugins */
extern int rdma_dev_available; /* rdma device is loaded or not */
extern uint_t rdma_minchunk;
/*
* General RDMA routines
*/
extern struct clist *clist_alloc(void);
extern void clist_free(struct clist *);
extern void clist_zero_len(struct clist *);
extern int rdma_modload();
extern rdma_stat rdma_kwait(void);
/*
* RDMA XDR
*/
extern void xdrrdma_destroy(XDR *);
#endif /* _KERNEL */
#ifdef __cplusplus
}
#endif
#endif /* _RPC_RPC_RDMA_H */