/*
* CDDL HEADER START
*
* The contents of this file are subject to the terms of the
* Common Development and Distribution License (the "License").
* You may not use this file except in compliance with the License.
*
* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
* or http://www.opensolaris.org/os/licensing.
* See the License for the specific language governing permissions
* and limitations under the License.
*
* When distributing Covered Code, include this CDDL HEADER in each
* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
* If applicable, add the following below this CDDL HEADER, with the
* fields enclosed by brackets "[]" replaced with your own identifying
* information: Portions Copyright [yyyy] [name of copyright owner]
*
* CDDL HEADER END
*/
/*
* Copyright 2009 Sun Microsystems, Inc. All rights reserved.
* Use is subject to license terms.
*/
#ifndef _RDC_IO_H
#define _RDC_IO_H
#ifdef __cplusplus
extern "C" {
#endif
#include <sys/unistat/spcs_s.h>
#ifdef DS_DDICT
#define bool_t int
#endif
#include <sys/nsctl/rdc_prot.h>
#include <sys/nsctl/nsctl.h>
#include <sys/nsctl/rdc_ioctl.h>
/*
* Definitions for kstats
*/
#define RDC_MKSTAT_MAXSETS "maxsets"
#define RDC_MKSTAT_MAXFBAS "maxfbas"
#define RDC_MKSTAT_RPC_TIMEOUT "rpc_timeout"
#define RDC_MKSTAT_HEALTH_THRES "health_thres"
#define RDC_MKSTAT_BITMAP_WRITES "bitmap_writes"
#define RDC_MKSTAT_CLNT_COTS_CALLS "clnt_cots_calls"
#define RDC_MKSTAT_CLNT_CLTS_CALLS "clnt_clts_calls"
#define RDC_MKSTAT_SVC_COTS_CALLS "svc_cots_calls"
#define RDC_MKSTAT_SVC_CLTS_CALLS "svc_clts_calls"
#define RDC_MKSTAT_BITMAP_REF_DELAY "bitmap_ref_delay"
#define RDC_IKSTAT_FLAGS "flags"
#define RDC_IKSTAT_SYNCFLAGS "syncflags"
#define RDC_IKSTAT_BMPFLAGS "bmpflags"
#define RDC_IKSTAT_SYNCPOS "syncpos"
#define RDC_IKSTAT_VOLSIZE "volsize"
#define RDC_IKSTAT_BITSSET "bitsset"
#define RDC_IKSTAT_AUTOSYNC "autosync"
#define RDC_IKSTAT_MAXQFBAS "maxqfbas"
#define RDC_IKSTAT_MAXQITEMS "maxqitems"
#define RDC_IKSTAT_FILE "primary_vol"
#define RDC_IKSTAT_SECFILE "secondary_vol"
#define RDC_IKSTAT_BITMAP "bitmap"
#define RDC_IKSTAT_PRIMARY_HOST "primary_host"
#define RDC_IKSTAT_SECONDARY_HOST "secondary_host"
#define RDC_IKSTAT_TYPE_FLAG "type_flag"
#define RDC_IKSTAT_BMP_SIZE "bmp_size"
#define RDC_IKSTAT_DISK_STATUS "disk_status"
#define RDC_IKSTAT_IF_DOWN "if_down"
#define RDC_IKSTAT_IF_RPC_VERSION "if_rpc_version"
#define RDC_IKSTAT_ASYNC_THROTTLE_DELAY "async_throttle_delay"
#define RDC_IKSTAT_ASYNC_BLOCK_HWM "async_block_hwm"
#define RDC_IKSTAT_ASYNC_ITEM_HWM "async_item_hwm"
#define RDC_IKSTAT_QUEUE_TYPE "async_queue_type"
#define RDC_IKSTAT_ASYNC_ITEMS "async_queue_items"
#define RDC_IKSTAT_ASYNC_BLOCKS "async_queue_blocks"
/*
* Queue types
*/
#define RDC_DISKQUE 0X01
#define RDC_MEMQUE 0x02
#define RDC_NOQUE -1
#define RDC_ACTIVE 0x1
#define RDC_INACTIVE 0x2
#ifdef _KERNEL
extern nstset_t *_rdc_ioset;
extern nstset_t *_rdc_flset;
#ifdef DEBUG
extern int RDC_MAX_SYNC_THREADS;
extern int rdc_maxthreads_last;
int num_sync_threads;
#else
#define RDC_MAX_SYNC_THREADS 8
#endif
#ifdef DEBUG
#define RDC_AVAIL_THR_TUNE(n) \
do { \
if (rdc_maxthreads_last < RDC_MAX_SYNC_THREADS) { \
(void) nst_add_thread(n.rdc_syncset, \
RDC_MAX_SYNC_THREADS - rdc_maxthreads_last);\
} \
if (rdc_maxthreads_last > RDC_MAX_SYNC_THREADS) { \
(void) nst_del_thread(n.rdc_syncset, \
rdc_maxthreads_last - RDC_MAX_SYNC_THREADS); \
} \
n.avail_thr = RDC_MAX_SYNC_THREADS - n.active_thr; \
if (n.avail_thr < 0) { \
n.avail_thr = 0; \
} \
rdc_maxthreads_last = RDC_MAX_SYNC_THREADS; \
num_sync_threads = nst_nthread(n.rdc_syncset); \
} while (0);
#else
#define RDC_AVAIL_THR_TUNE(n) \
do { \
n.avail_thr = RDC_MAX_SYNC_THREADS - n.active_thr; \
if (n.avail_thr < 0) \
n.avail_thr = 0; \
} while (0);
#endif
typedef struct syncloop_info {
int active_thr;
int avail_thr; /* should be MAX_RDC_SYNC_THREADS - active */
kmutex_t lock;
nstset_t *rdc_syncset;
} sync_info_t;
sync_info_t sync_info;
/*
* Static server information
*/
typedef struct servinfo {
struct knetconfig *ri_knconf; /* bound TLI fd */
struct netbuf ri_addr; /* server's address */
struct sec_data *ri_secdata; /* sec data for rpcsec module */
char *ri_hostname; /* server's hostname */
int ri_hostnamelen; /* server's hostname length */
} rdc_srv_t;
/*
* Interface structure, including health monitoring.
*/
typedef struct rdc_if_s {
struct rdc_if_s *next; /* chain pointer */
struct netbuf ifaddr;
struct netbuf r_ifaddr;
rdc_srv_t *srv; /* servinfo of server end */
int if_down; /* i/f is down (set on primary) */
int isprimary; /* this end is a primary */
int issecondary; /* this end is a secondary */
rpcvers_t rpc_version; /* RPC protocol version in use */
int no_ping; /* set on secondary to hold off RPCs */
int old_pulse; /* previous (current) pulse value */
int new_pulse; /* new (incoming) pulse value */
int deadness; /* how close to death are we? */
volatile int exiting; /* daemon exit flag */
time_t last; /* time of last ping */
} rdc_if_t;
typedef struct rdc_aio_s {
struct rdc_aio_s *next;
nsc_buf_t *handle;
nsc_buf_t *qhandle;
nsc_off_t pos;
nsc_off_t qpos;
nsc_size_t len;
nsc_size_t orig_len;
int flag;
int iostatus;
int index;
uint_t seq; /* sequence on async Q */
} rdc_aio_t;
/* values for (rdc_aio_t *)->iostatus */
enum {
RDC_IO_NONE = 0, /* not used */
RDC_IO_INIT, /* io started */
RDC_IO_DONE, /* io done successfully */
RDC_IO_FAILED, /* io failed */
RDC_IO_DISCARDED, /* io discarded */
RDC_IO_CANCELLED /* group_log in progress */
};
#define RDC_MAX_QBLOCKS 16384 /* 8MB temporary q for diskq to flush to */
#define RDC_LOW_QBLOCKS 13927 /* roughly 85% of queue full */
#define RDC_HALF_MQUEUE 8192 /* half of the memory queue */
typedef struct netqueue {
rdc_aio_t *net_qhead;
rdc_aio_t *net_qtail;
kmutex_t net_qlock;
int hwmhit; /* queue full hit? reset after hwm */
int qfill_sleeping; /* waiting for work? */
int qfflags; /* diskq/memq flusher flags */
kcondvar_t qfcv; /* for timed waits */
volatile nsc_size_t blocks; /* number of FBAs in q */
volatile uint64_t nitems; /* number of items in q */
volatile int inflbls; /* number of inflight blocks */
volatile int inflitems; /* number of inflight items */
uint64_t nitems_hwm; /* highest items on queue */
nsc_size_t blocks_hwm; /* highest blocks on queue */
long throttle_delay; /* Number of times we delayed x 2 */
} net_queue;
/*
* Bitmap header structures.
* These must be fixed size in all data models.
* If we ever support little-endian machines (eg. Intel) we will need
* to add byte-swapping logic.
*/
typedef struct {
int32_t magic;
int32_t serial_mode;
int32_t use_mirror;
int32_t mirror_down;
int32_t sync_needed;
char bitmapname[NSC_MAXPATH];
char filename[NSC_MAXPATH];
int32_t volume_failed;
} rdc_headerv2_t;
#define RDC_HDR_V2 0x52444302 /* RDC2 */
#define RDC_SYNC 0x1
#define RDC_REV_SYNC 0x2
#define RDC_FULL_SYNC 0x3
#define RDC_FAILED 0x1
#define RDC_COMPLETED 0x2
typedef struct {
char file[NSC_MAXPATH];
char bitmap[NSC_MAXPATH];
} rdc_hdr_addr_t;
typedef struct {
int32_t magic;
rdc_hdr_addr_t primary;
rdc_hdr_addr_t secondary;
int32_t flags;
int32_t autosync;
int32_t maxqfbas;
int32_t maxqitems;
int32_t syshostid; /* for cluster bitmaps */
} rdc_headerv3_t;
#define RDC_HDR_V3 0x52444303 /* RDC3 */
typedef struct {
int32_t magic;
rdc_hdr_addr_t primary;
rdc_hdr_addr_t secondary;
int32_t flags;
int32_t autosync;
int32_t maxqfbas;
int32_t maxqitems;
int32_t syshostid; /* for cluster bitmaps */
int32_t asyncthr;
} rdc_headerv4_t;
#define RDC_HDR_V4 0x52444304 /* RDC4 */
typedef struct {
int32_t magic;
rdc_hdr_addr_t primary;
rdc_hdr_addr_t secondary;
int32_t flags;
int32_t autosync;
int64_t maxqfbas;
int64_t maxqitems;
int32_t syshostid; /* for cluster bitmaps */
int32_t asyncthr;
int32_t refcntsize; /* size in bytes of each refcount */
} rdc_headerv5_t;
#define RDC_HDR_V5 0x52444305 /* RDC5 */
typedef rdc_headerv5_t rdc_header_t; /* Current header type */
#define RDC_HDR_MAGIC RDC_HDR_V5 /* Current header magic number */
#endif /* _KERNEL */
#define RDC_BITMAP_FBA 1 /* Offset at which the bitmap starts */
#define RDC_BITREF_FBA(krdc) (RDC_BITMAP_FBA + FBA_LEN(krdc->bitmap_size))
#ifdef _KERNEL
#define RDC_FUTILE_ATTEMPTS 50
typedef struct aio_buf_s {
struct aio_buf_s *next; /* next aio_buf */
nsc_buf_t *rdc_abufp; /* actual anon buf */
int kindex; /* index we are attached to */
} aio_buf_t;
typedef struct rdc_thrsync {
kmutex_t lock;
int threads;
int complete;
kcondvar_t cv;
} rdc_thrsync_t;
typedef struct sync_status_s {
int offset;
struct sync_status_s *next;
} sync_status_t;
typedef struct rdc_syncthr {
nsc_off_t offset;
nsc_size_t len;
struct rdc_k_info *krdc;
sync_status_t *status;
} rdc_syncthr_t;
/*
* RDC buffer header
*/
typedef struct rdc_buf_s {
nsc_buf_t rdc_bufh; /* exported buffer header */
nsc_buf_t *rdc_bufp; /* underlying buffer */
aio_buf_t *rdc_anon; /* ANON async buffer */
struct rdc_fd_s *rdc_fd; /* back link */
size_t rdc_vsize; /* size of allocated nsc_vec_t */
int rdc_flags; /* flags */
kmutex_t aio_lock; /* lock for rdc_anon */
rdc_thrsync_t rdc_sync; /* for thread syncronization */
} rdc_buf_t;
#define RDC_VEC_ALLOC 0x1 /* local kmem vector for remote io */
#define RDC_ALLOC 0x2 /* rdc_bufp is nsc_buf_alloc'd */
#define RDC_ASYNC_VEC 0x4 /* Keep tmp handle for async flusher */
#define RDC_REMOTE_BUF 0x8 /* buffer alloc'd for remote io only */
#define RDC_NULL_BUF 0x10 /* tell diskq to only store io_hdr */
#define RDC_ASYNC_BUF 0x20 /* this buf is to an async vol */
#define RDC_NULLBUFREAD 0x0f000000 /* read because RDC_NULL_BUF detected */
#define BUF_IS_ASYNC(h) (((h) != NULL) && (h)->rdc_flags & RDC_ASYNC_BUF)
#define RDC_REMOTE(h) (((h) != NULL) && ((h)->rdc_flags & RDC_REMOTE_BUF) && \
(((h)->rdc_flags & RDC_ASYNC_VEC) == 0))
/* check a handle against a supplied pos/len pair */
#define RDC_HANDLE_LIMITS(h, p, l) \
(((h)->sb_user & RDC_DISKQUE) || \
((p) >= (h)->sb_pos) && \
(((p) + (l)) <= ((h)->sb_pos + (h)->sb_len)))
/* check a dset against a supplied pos/len pair */
#define RDC_DSET_LIMITS(d, p, l) \
(((p) >= (d)->pos) && \
(((p) + (l)) <= ((d)->pos + (d)->fbalen)))
/*
* RDC device info structures
*/
typedef struct _rdc_info_dev_s {
nsc_fd_t *bi_fd; /* file descriptor */
nsc_iodev_t *bi_iodev; /* I/O device structure */
struct rdc_k_info *bi_krdc; /* back link */
int bi_rsrv; /* Count of reserves held */
int bi_orsrv; /* Reserves for other io provider */
int bi_failed; /* Count of failed (faked) reserves */
int bi_ofailed; /* Other io provider failed reserves */
int bi_flag; /* Reserve flags */
} _rdc_info_dev_t;
typedef struct rdc_info_dev_s {
struct rdc_info_dev_s *id_next; /* forward link */
_rdc_info_dev_t id_cache_dev; /* cached device info */
_rdc_info_dev_t id_raw_dev; /* raw device info */
kmutex_t id_rlock; /* reserve/release lock */
kcondvar_t id_rcv; /* nsc_release pending cv */
int id_sets; /* # of sets referencing */
int id_release; /* # of pending nsc_releases */
int id_flag; /* flags */
} rdc_info_dev_t;
typedef struct rdc_path_s {
nsc_path_t *rp_tok; /* nsc_register_path token */
int rp_ref; /* # of rdc_fd_t's */
} rdc_path_t;
/*
* Values for id_flag
*/
#define RDC_ID_CLOSING 0x1 /* device is closing */
#include <sys/nsctl/rdc_diskq.h>
/*
* value for diskio.seq.
*/
#define RDC_NOSEQ (0) /* ignore sequence */
#define RDC_NEWSEQ (1) /* start of sequence */
typedef struct rdc_sleepq {
struct rdc_sleepq *next;
uint_t seq; /* sequence in queue */
int idx; /* idx number of request */
int pindex; /* primary host set index */
int sindex; /* secondary host set index */
uint64_t qpos; /* offset on primary's queue */
int nocache; /* cache flag to alloc_buf */
} rdc_sleepq_t;
/*
* RDC group structure
*/
typedef struct rdc_group {
int count;
int rdc_writer;
int unregistering;
kmutex_t lock;
net_queue ra_queue; /* io todo async queues */
kcondvar_t iowaitcv; /* wait for flusher */
kcondvar_t unregistercv; /* wait for unregister */
int rdc_thrnum; /* number of threads */
int rdc_addthrnum; /* number threads added to thr set */
kmutex_t addthrnumlk; /* lock for above */
rdc_sleepq_t *sleepq; /* head of waiting tasks */
/*
* Dual use, the outgoing sequence number on the client.
* The next expected sequence number on the server.
* Protected by the ra_queue lock.
*/
uint_t seq;
/*
* Dual use, the last acknowledged sequence number.
* Used to ensure that the queue doesn't overflow on server
* and to stall transmissions on the client.
* Protected by the ra_queue lock.
*/
uint_t seqack;
int asyncstall; /* count of asleep threads */
int asyncdis; /* discard stalled output */
kcondvar_t asyncqcv; /* output stall here */
int flags; /* memory or disk. status etc */
disk_queue diskq; /* disk queue */
nsc_fd_t *diskqfd; /* diskq handle */
nsc_path_t *q_tok; /* q registration */
int diskqrsrv; /* reserve count */
kmutex_t diskqmutex; /* enables/disables/reserves */
uint_t synccount; /* number of group members syncing */
} rdc_group_t;
/* group state */
#define RDC_DISKQ_KILL 0x01 /* a force kill of diskq pending */
#define RDC_IS_DISKQ(grp) (grp->flags & RDC_DISKQUE)
#define RDC_IS_MEMQ(grp) (grp->flags & RDC_MEMQUE)
/*
* These flags are used in the
* aux_state field, and are used to track:
* AUXSYNCIP: When the code has a sync thread running, used instead
* of the RC_SYNCING flag which gets cleared before the sync thread
* terminates.
* AUXWRITE: Set when rdc_sync_write_thr is running, so the rdc_unintercept
* code can wait until a one-to-many write has actually terminated.
*/
#define RDC_AUXSYNCIP 0x01 /* a sync is in progress */
#define RDC_AUXWRITE 0x02 /* I've got a write in progress */
/*
* RDC kernel-private information
*/
typedef struct rdc_k_info {
int index; /* Index into array */
int remote_index; /* -1 means unknown */
int type_flag;
int rpc_version; /* RPC version this set supps */
int spare1;
nsc_off_t syncbitpos;
kmutex_t syncbitmutex; /* lock for syncbitpos */
volatile int busy_count; /* ioctls in progress */
volatile int sync_done;
int aux_state; /* syncing ,don't disable */
rdc_thrsync_t syncs; /* _rdc_sync thread tracking */
rdc_info_dev_t *devices;
nsc_iodev_t *iodev; /* I/O device structure */
rdc_path_t cache_path;
rdc_path_t raw_path;
rdc_if_t *intf;
rdc_srv_t *lsrv; /* list of servinfo */
nsc_size_t maxfbas; /* returned from nsc_maxfbas */
unsigned char *dcio_bitmap;
void *bitmap_ref; /* Incore bitmap bit ref */
struct rdc_group *group;
nsc_size_t bitmap_size;
int bmaprsrv; /* bitmap reserve count */
int bitmap_write;
nsc_fd_t *bitmapfd;
nsc_fd_t *remote_fd; /* FCAL direct io */
volatile int disk_status; /* set to halt sync */
int closing;
nsc_path_t *b_tok; /* Bitmap registration */
int b_ref;
kmutex_t dc_sleep;
kmutex_t bmapmutex; /* mutex for bitmap ops */
kcondvar_t busycv; /* wait for ioctl to complete */
kcondvar_t closingcv; /* unregister_path/close */
kcondvar_t haltcv; /* wait for sync to halt */
kcondvar_t synccv; /* wait for sync to halt */
struct rdc_net_dataset *net_dataset; /* replaces hnds */
int64_t io_time; /* moved from cd_info */
struct rdc_k_info *many_next; /* 1-to-many circular list */
struct rdc_k_info *multi_next; /* to multihop krdc */
struct rdc_k_info *group_next; /* group circular list */
kstat_t *io_kstats; /* io kstat */
kstat_t *bmp_kstats; /* bitmap io kstat */
kstat_t *set_kstats; /* set kstat */
kmutex_t kstat_mutex; /* mutex for kstats */
kmutex_t bmp_kstat_mutex; /* mutex for kstats */
struct bm_ref_ops *bm_refs;
} rdc_k_info_t;
#define c_fd devices->id_cache_dev.bi_fd
#define c_rsrv devices->id_cache_dev.bi_rsrv
#define c_failed devices->id_cache_dev.bi_failed
#define c_flag devices->id_cache_dev.bi_flag
#define c_tok cache_path.rp_tok
#define c_ref cache_path.rp_ref
#define r_fd devices->id_raw_dev.bi_fd
#define r_rsrv devices->id_raw_dev.bi_rsrv
#define r_failed devices->id_raw_dev.bi_failed
#define r_flag devices->id_raw_dev.bi_flag
#define r_tok raw_path.rp_tok
#define r_ref raw_path.rp_ref
/*
* flags for _rdc_rsrv_devs()
*/
/*
* which device(s) to reserve - integer bitmap.
*/
#define RDC_CACHE 0x1 /* data device in cache mode */
#define RDC_RAW 0x2 /* data device in raw mode */
#define RDC_BMP 0x4 /* bitmap device */
#define RDC_QUE 0x8 /* diskq device */
/*
* device usage after reserve - integer flag.
*/
#define RDC_INTERNAL 0x1 /* reserve for rdc internal purposes */
#define RDC_EXTERNAL 0x2 /* reserve in response to io provider Attach */
/*
* Utility macro for nsc_*() io function returns.
*/
#define RDC_SUCCESS(rc) (((rc) == NSC_DONE) || ((rc) == NSC_HIT))
/*
* RDC file descriptor structure
*/
typedef struct rdc_fd_s {
rdc_k_info_t *rdc_info; /* devices info structure */
int rdc_type; /* open type, diskq or bitmap */
int rdc_oflags; /* raw or cached open type */
} rdc_fd_t;
/*
* fd and rsrv macros
*/
#define RSRV(bi) (((bi)->bi_rsrv > 0) || ((bi)->bi_failed > 0))
#define ORSRV(bi) (((bi)->bi_orsrv > 0) || ((bi)->bi_ofailed > 0))
#define RFAILED(bi) (((bi)->bi_failed > 0) || ((bi)->bi_ofailed > 0))
#define IS_RSRV(bi) (RSRV(bi) || ORSRV(bi))
#define IS_CRSRV(gcd) (IS_RSRV(&(gcd)->devices->id_cache_dev))
#define IS_RRSRV(gcd) (IS_RSRV(&(gcd)->devices->id_raw_dev))
#define IS_RFAILED(gcd) \
(RFAILED(&(gcd)->devices->id_cache_dev) || \
RFAILED(&(gcd)->devices->id_raw_dev))
#define RDC_IS_BMP(rdc) ((rdc)->rdc_type == RDC_BMP)
#define RDC_IS_QUE(rdc) ((rdc)->rdc_type == RDC_QUE)
#define RDC_IS_RAW(rdc) (((rdc)->rdc_oflags & NSC_CACHE) == 0)
#define RDC_U_FD(gcd) (IS_CRSRV(gcd) ? (gcd)->c_fd : (gcd)->r_fd)
#define RDC_FD(rdc) (RDC_U_FD(rdc->rdc_info))
typedef struct rdc_host_u {
char *nodename;
int netaddr;
struct netbuf *naddr;
} rdc_host_t;
/*
* Reply from remote read
* - convenience defines for the client side code.
* - keep this in sync with the readres structure in rdc_prot.h/.x
*/
#define rdcrdresult readres
#define rr_status status
#define rr_ok readres_u.reply
#define rr_bufsize rr_ok.data.data_len
#define rr_data rr_ok.data.data_val
/*
* Flags for remote read rpc
*
* _START must be a unique rpc, _DATA and _END may be OR-d together.
*/
#define RDC_RREAD_DATA 0x1 /* Intermediate rpc with data payload */
#define RDC_RREAD_START 0x2 /* Setup rpc */
#define RDC_RREAD_END 0x4 /* End rpc */
#define RDC_RREAD_FAIL 0x8 /* Primary is failed */
/*
* Flags for remote write rpc
*/
#define RDC_RWRITE_FAIL 0x8 /* Primary is failed */
/*
* macro used to determine if the incomming sq, with sequence
* value x, should be placed before the sq with sequence value y.
* This has to account for integer wrap. We account for integer
* wrap by checking if the difference between x and y is within
* half of the maximum integer value (RDC_MAXINT) or not.
*/
#define RDC_BITSPERBYTE 8
#define RDC_BITS(type) (RDC_BITSPERBYTE * (long)sizeof (type))
#define RDC_HIBITI ((unsigned)1 << (RDC_BITS(int) - 1))
#define RDC_MAXINT ((int)(~RDC_HIBITI))
#define RDC_RANGE ((RDC_MAXINT / 2) -1)
#define RDC_INFRONT(x, y) (((x < y) && ((y - x) < RDC_RANGE)) ? 1 : \
((x > y) && ((x - y) > RDC_RANGE)) ? 1 : 0)
#endif /* _KERNEL */
/*
* RDC user-visible information
*/
typedef rdc_set_t rdc_u_info_t;
/*
* RDC flags for set state / set cd RPC.
* Must remain compatible with rdc RPC protocol version v3.
*/
#define CCIO_NONE 0x0000
#define CCIO_ENABLE 0x0008
#define CCIO_SLAVE 0x0010
#define CCIO_DONE 0x0020
#define CCIO_ENABLELOG 0x0100
#define CCIO_RSYNC 0x0400
#define CCIO_REMOTE 0x2000
/*
* In kernel type flags (krdc->type_flag).
*/
#define RDC_CONFIGURED 0x1
#define RDC_DISABLEPEND 0x2 /* Suspend/Disable is in progress */
#define RDC_ASYNCMODE 0x4
#define RDC_RESUMEPEND 0x8
#define RDC_RESPONSIBLE 0x10
#define RDC_BUSYWAIT 0x20
#define RDC_UNREGISTER 0x40 /* Unregister is in progress */
#define RDC_QDISABLEPEND 0x100 /* Q Suspend/Disable is in progress */
#define IS_ENABLED(urdc) ((IS_CONFIGURED(&rdc_k_info[(urdc)->index]) && \
(rdc_get_vflags(urdc) & RDC_ENABLED)))
#define IS_CONFIGURED(krdc) ((krdc)->type_flag & RDC_CONFIGURED)
#define IS_MANY(krdc) ((krdc)->many_next != (krdc))
#define IS_MULTI(krdc) ((krdc)->multi_next != NULL)
#define IS_VALID_INDEX(index) ((index) >= 0 && (index) < rdc_max_sets && \
IS_CONFIGURED(&rdc_k_info[(index)]))
#define RDC_NOFLUSH 0 /* Do not do a flush when starting logging */
#define RDC_NOREMOTE 0 /* Do no remote logging notifications */
#define RDC_FLUSH 1 /* Do a flush when starting logging */
#define RDC_ALLREMOTE 2 /* Notify all remote group members */
#define RDC_OTHERREMOTE 4 /* Notify all remote group members except */
/* the one corresponding to the current set, */
/* to prevent recursion in the case where */
/* the request was initiated from the remote */
/* node. */
#define RDC_FORCE_GROUP 8 /* set all group memebers logging regardless */
#ifdef _KERNEL
/*
* Functions, vars
*/
#define RDC_SYNC_EVENT_TIMEOUT (60 * HZ)
typedef struct {
clock_t lbolt;
int event;
int ack;
int daemon_waiting; /* Daemon waiting in ioctl */
int kernel_waiting; /* Kernel waiting for daemon to reply */
char master[NSC_MAXPATH];
char group[NSC_MAXPATH];
kmutex_t mutex;
kcondvar_t cv;
kcondvar_t done_cv;
} rdc_sync_event_t;
extern rdc_sync_event_t rdc_sync_event;
extern clock_t rdc_sync_event_timeout;
extern kmutex_t rdc_sync_mutex;
extern rdc_u_info_t *rdc_u_info;
extern rdc_k_info_t *rdc_k_info;
extern int rdc_max_sets;
extern unsigned long rdc_async_timeout;
extern int rdc_self_host();
extern uint64_t mirror_getsize(int index);
extern void rdc_sleepqdiscard(rdc_group_t *);
#ifdef DEBUG
extern void rdc_stallzero(int);
#endif
struct rdc_net_dataitem {
void *dptr;
int len; /* byte count */
int mlen; /* actual malloced size */
struct rdc_net_dataitem *next;
};
typedef struct rdc_net_dataitem rdc_net_dataitem_t;
struct rdc_net_dataset {
int id;
int inuse;
int delpend;
int nitems;
nsc_off_t pos;
nsc_size_t fbalen;
rdc_net_dataitem_t *head;
rdc_net_dataitem_t *tail;
struct rdc_net_dataset *next;
};
typedef struct rdc_net_dataset rdc_net_dataset_t;
#endif /* _KERNEL */
#define RDC_TCP_DEV "/dev/tcp"
#define RDC_VERS_MIN RDC_VERSION5
#define RDC_VERS_MAX RDC_VERSION7
#define RDC_HEALTH_THRESHOLD 20
#define RDC_MIN_HEALTH_THRES 5
#define SNDR_MAXTHREADS 16
/*
* These next two defines are the default value of the async queue size
* They have been calculated to be 8MB of data with an average of
* 2K IO size
*/
#define RDC_MAXTHRES_QUEUE 16384 /* max # of fbas on async q */
#define RDC_MAX_QITEMS 4096 /* max # of items on async q */
#define RDC_ASYNCTHR 2 /* number of async threads */
#define RDC_RPC_MAX (RDC_MAXDATA + sizeof (net_data5) +\
(RPC_MAXDATASIZE - 8192))
#define ATM_NONE 0
#define ATM_INIT 1
#define ATM_EXIT 2
#define RDC_CLNT_TMOUT 16
#define BMAP_BLKSIZE 1024
#define BMAP_BLKSIZEV7 RDC_MAXDATA
/* right now we can only trace 1m or less writes to the bitmap (32 bits wide) */
#define RDC_MAX_MAXFBAS 2048
#if defined(_KERNEL)
/* kstat interface */
/*
* Per module kstats
* only one instance
*/
typedef struct {
kstat_named_t m_maxsets; /* Max # of sndr sets */
kstat_named_t m_maxfbas; /* Max # of FBAS from nsctl */
kstat_named_t m_rpc_timeout; /* global RPC timeout */
kstat_named_t m_health_thres; /* Health thread timeout */
kstat_named_t m_bitmap_writes; /* True for bitmap writes */
kstat_named_t m_clnt_cots_calls; /* # of clnt COTS calls */
kstat_named_t m_clnt_clts_calls; /* # of clnt CLTS calls */
kstat_named_t m_svc_cots_calls; /* # of server COTS calls */
kstat_named_t m_svc_clts_calls; /* # of server CLTS calls */
kstat_named_t m_bitmap_ref_delay; /* # of bitmap ref overflows */
} sndr_m_stats_t;
/*
* Per set kstats
* one instance per configured set
*/
typedef struct {
kstat_named_t s_flags; /* from rdc_set_t */
kstat_named_t s_syncflags; /* from rdc_set_t */
kstat_named_t s_bmpflags; /* from rdc_set_t */
kstat_named_t s_syncpos; /* from rdc_set_t */
kstat_named_t s_volsize; /* from rdc_set_t */
kstat_named_t s_bits_set; /* from rdc_set_t */
kstat_named_t s_autosync; /* from rdc_set_t */
kstat_named_t s_maxqfbas; /* from rdc_set_t */
kstat_named_t s_maxqitems; /* from rdc_set_t */
kstat_named_t s_primary_vol; /* from rdc_set_t */
kstat_named_t s_secondary_vol; /* from rdc_set_t */
kstat_named_t s_bitmap; /* from rdc_set_t */
kstat_named_t s_primary_intf; /* from rdc_set_t */
kstat_named_t s_secondary_intf; /* from rdc_set_t */
kstat_named_t s_type_flag; /* from rdc_k_info_t */
kstat_named_t s_bitmap_size; /* from rdc_k_info_t */
kstat_named_t s_disk_status; /* from rdc_k_info_t */
kstat_named_t s_if_if_down; /* from rdc_if_t */
kstat_named_t s_if_rpc_version; /* from rdc_if_t */
kstat_named_t s_aqueue_blk_hwm; /* from rdc_k_info_t */
kstat_named_t s_aqueue_itm_hwm; /* from rdc_k_info_t */
kstat_named_t s_aqueue_throttle; /* from rdc_k_info_t */
kstat_named_t s_aqueue_items;
kstat_named_t s_aqueue_blocks;
kstat_named_t s_aqueue_type;
} rdc_info_stats_t;
#endif /* _KERNEL */
#ifndef _SunOS_5_6 /* i.e. 2.7+ */
typedef int xdr_t;
#else /* i.e. 2.6- */
typedef unsigned long rpcprog_t;
typedef unsigned long rpcvers_t;
typedef unsigned long rpcproc_t;
typedef unsigned long rpcprot_t;
typedef unsigned long rpcport_t;
#endif /* _SunOS_5_6 */
#ifdef _KERNEL
extern nsc_size_t MAX_RDC_FBAS;
extern volatile int net_exit;
extern nsc_size_t rdc_maxthres_queue; /* max # of fbas on async q */
extern int rdc_max_qitems; /* max # of items on async q */
extern int rdc_asyncthr; /* # of async threads */
#ifdef DEBUG
extern kmutex_t rdc_cntlock;
extern int rdc_datasetcnt;
#endif
/*
* Macro to keep tabs on dataset memory usage.
*/
#ifdef DEBUG
#define RDC_DSMEMUSE(x) \
mutex_enter(&rdc_cntlock);\
rdc_datasetcnt += (x);\
mutex_exit(&rdc_cntlock);
#else
#define RDC_DSMEMUSE(x)
#endif
extern kmutex_t rdc_ping_lock;
extern rdc_if_t *rdc_if_top;
extern int _rdc_enqueue_write(rdc_k_info_t *, nsc_off_t, nsc_size_t, int,
nsc_buf_t *);
extern int rdc_net_state(int, int);
extern int rdc_net_getbmap(int, int);
extern int rdc_net_getsize(int, uint64_t *);
extern int rdc_net_write(int, int, nsc_buf_t *, nsc_off_t, nsc_size_t, uint_t,
int, netwriteres *);
extern int rdc_net_read(int, int, nsc_buf_t *, nsc_off_t, nsc_size_t);
extern int _rdc_remote_read(rdc_k_info_t *, nsc_buf_t *, nsc_off_t, nsc_size_t,
int);
extern int _rdc_multi_write(nsc_buf_t *, nsc_off_t, nsc_size_t, int,
rdc_k_info_t *);
extern int rdc_start_server(struct rdc_svc_args *, int);
extern aio_buf_t *rdc_aio_buf_get(rdc_buf_t *, int);
extern void rdc_aio_buf_del(rdc_buf_t *, rdc_k_info_t *);
extern aio_buf_t *rdc_aio_buf_add(int, rdc_buf_t *);
extern int rdc_net_getstate(rdc_k_info_t *, int *, int *, int *, int);
extern kmutex_t rdc_conf_lock;
extern kmutex_t rdc_many_lock;
extern int rdc_drain_queue(int);
extern int flush_group_queue(int);
extern void rdc_dev_close(rdc_k_info_t *);
extern int rdc_dev_open(rdc_set_t *, int);
extern void rdc_get_details(rdc_k_info_t *);
extern int rdc_lookup_bitmap(char *);
extern int rdc_lookup_enabled(char *, int);
extern int rdc_lookup_byaddr(rdc_set_t *);
extern int rdc_lookup_byname(rdc_set_t *);
extern int rdc_intercept(rdc_k_info_t *);
extern int rdc_unintercept(rdc_k_info_t *);
extern int _rdc_rsrv_devs(rdc_k_info_t *, int, int);
extern void _rdc_rlse_devs(rdc_k_info_t *, int);
extern void _rdc_unload(void);
extern int _rdc_load(void);
extern int _rdc_configure(void);
extern void _rdc_deconfigure(void);
extern void _rdc_async_throttle(rdc_k_info_t *, long);
extern int rdc_writer(int);
extern int rdc_dump_alloc_bufs_cd(int);
extern void rdc_dump_alloc_bufs(rdc_if_t *);
extern int rdc_check_secondary(rdc_if_t *, int);
extern void rdc_dump_queue(int);
extern int rdc_isactive_if(struct netbuf *, struct netbuf *);
extern rdc_if_t *rdc_add_to_if(rdc_srv_t *, struct netbuf *, struct netbuf *,
int);
extern void rdc_remove_from_if(rdc_if_t *);
extern void rdc_set_if_vers(rdc_u_info_t *, rpcvers_t);
extern void rdc_print_svinfo(rdc_srv_t *, char *);
extern rdc_srv_t *rdc_create_svinfo(char *, struct netbuf *,
struct knetconfig *);
extern void rdc_destroy_svinfo(rdc_srv_t *);
extern void init_rdc_netbuf(struct netbuf *);
extern void free_rdc_netbuf(struct netbuf *);
extern void dup_rdc_netbuf(const struct netbuf *, struct netbuf *);
extern int rdc_netbuf_toint(struct netbuf *);
extern struct netbuf *rdc_int_tonetbuf(int);
extern void rdc_lor(const uchar_t *, uchar_t *, int);
extern int rdc_resume2(rdc_k_info_t *);
extern void rdc_set_flags(rdc_u_info_t *, int);
extern void rdc_clr_flags(rdc_u_info_t *, int);
extern int rdc_get_vflags(rdc_u_info_t *);
extern void rdc_set_mflags(rdc_u_info_t *, int);
extern void rdc_clr_mflags(rdc_u_info_t *, int);
extern int rdc_get_mflags(rdc_u_info_t *);
extern void rdc_set_flags_log(rdc_u_info_t *, int, char *);
extern void rdc_group_log(rdc_k_info_t *krdc, int flush, char *why);
extern int _rdc_config(void *, int, spcs_s_info_t, int *);
extern void rdc_many_enter(rdc_k_info_t *);
extern void rdc_many_exit(rdc_k_info_t *);
extern void rdc_group_enter(rdc_k_info_t *);
extern void rdc_group_exit(rdc_k_info_t *);
extern int _rdc_sync_event_wait(void *, void *, int, spcs_s_info_t, int *);
extern int _rdc_sync_event_notify(int, char *, char *);
extern int _rdc_link_down(void *, int, spcs_s_info_t, int *);
extern void rdc_delgroup(rdc_group_t *);
extern int rdc_write_bitmap_fba(rdc_k_info_t *, nsc_off_t);
extern int rdc_bitmapset(int, char *, char *, void *, int, nsc_off_t, int);
extern rdc_net_dataset_t *rdc_net_add_set(int);
extern rdc_net_dataset_t *rdc_net_get_set(int, int);
extern void rdc_net_put_set(int, rdc_net_dataset_t *);
extern void rdc_net_del_set(int, rdc_net_dataset_t *);
extern void rdc_net_free_set(rdc_k_info_t *, rdc_net_dataset_t *);
extern int rdc_lookup_byhostdev(char *intf, char *file);
extern int rdc_lookup_configured(char *path);
extern void rdc_dump_dsets(int);
extern void set_busy(rdc_k_info_t *);
extern void wakeup_busy(rdc_k_info_t *);
#ifdef DEBUG
extern int rdc_async6(void *, int mode, int *);
extern int rdc_readgen(void *, int, int *);
#endif
#endif /* _KERNEL */
#ifdef __cplusplus
}
#endif
#endif /* _RDC_IO_H */