/*
* CDDL HEADER START
*
* The contents of this file are subject to the terms of the
* Common Development and Distribution License (the "License").
* You may not use this file except in compliance with the License.
*
* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
* See the License for the specific language governing permissions
* and limitations under the License.
*
* When distributing Covered Code, include this CDDL HEADER in each
* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
* If applicable, add the following below this CDDL HEADER, with the
* fields enclosed by brackets "[]" replaced with your own identifying
* information: Portions Copyright [yyyy] [name of copyright owner]
*
* CDDL HEADER END
*/
/*
* Copyright 2009 Sun Microsystems, Inc. All rights reserved.
* Use is subject to license terms.
*/
#ifndef _SYS_MDI_IMPLDEFS_H
#define _SYS_MDI_IMPLDEFS_H
#ifdef __cplusplus
extern "C" {
#endif
#ifdef _KERNEL
/*
* Multipath Driver Interfaces
*
* The multipathing framework is provided in two modules. The 'mpxio' misc.
* module provides the core multipath framework and the 'scsi_vhci' nexus
* driver provides the SCSI-III command set driver functionality for
* managing Fibre-Channel storage devices.
*
* As in any multipathing solution there are three major problems to solve:
*
* 1) Identification and enumeration of multipath client devices.
* 2) Optimal path selection when routing I/O requests.
* 3) Observability interfaces to snapshot the multipath configuration,
* and infrastructure to provide performance and error statistics.
*
* The mpxio framework consists of several major components:
*
* 1) The MDI is the Multiplexed Device Interface; this is the core glue which
* holds the following components together.
* 2) vHCI (Virtual Host Controller Interconnect) drivers provide multipathing
* services for a given bus technology (example: 'scsi_vhci' provides
* multipathing support for SCSI-III fibre-channel devices).
* 3) pHCI (Physical Host Controller Interconnect) drivers provide transport
* services for a given host controller (example: 'fcp' provides transport
* for fibre-channel devices).
* 4) Client Devices are standard Solaris target (or leaf) drivers
* (example: 'ssd' is the standard disk driver for fibre-channel arrays).
* 5) Multipath information nodes ('pathinfo' nodes) connect client device
* nodes and pHCI device nodes in the device tree.
*
* With the scsi_vhci, a QLC card, and mpxio enabled, the device tree might
* look like this:
*
* /\
* / ............
* <vHCI>:/ \
* +-----------+ +-----------+
* | scsi_vhci | | pci@1f,0 |
* +-----------+ +-----------+
* / \ \
* <Client>: / \ :<Client> \ :parent(pHCI)
* +----------+ +-----------+ +-------------+
* | ssd 1 | | ssd 2 | | qlc@0,0 |
* +----------+ +-----------+ +-------------+
* | | / \
* | | <pHCI>: / \ :<pHCI>
* | | +-------------+ +-------------+
* | | | pHCI 1 (fp) | | pHCI 2 (fp) |
* | | +-------------+ +-------------+
* | | / | / |
* | | +------+ | +------+ |
* | | | ssd 3| | | ssd | |
* | | |!mpxio| | | (OBP)| |
* | | +------+ | +------+ |
* | | | |
* | | <pathinfo>: | |
* | | +-------+ +--------+
* | +-------------->| path |-------->| path |
* | | info | | info |
* | | node 1| | node 3 |
* | +-------+ +--------+
* | | |
* | | +~~~~~~~~+
* | +-------+ :+--------+
* +--------------------------->| path |-------->| path |
* | info | :| info |
* | node 2| +| node 4 |
* +-------+ +--------+
*
* The multipath information nodes (mdi_pathinfo nodes) establish the
* relationship between the pseudo client driver instance nodes (children
* of the vHCI) and the physical host controller interconnect (pHCI
* drivers) forming a matrix structure.
*
* The mpxio module implements locking at multiple granularity levels to
* support the needs of various consumers. The multipath matrix can be
* column locked, or row locked depending on the consumer. The intention
* is to balance simplicity and performance.
*
* Locking:
*
* The devinfo locking still applies:
*
* 2) state >= DS_INITIALIZED adds devi_ref of parent
* 3) devi_ref at state >= DS_ATTACHED prevents detach(9E).
*
* The ordering of 1) is (vHCI, pHCI). For a DEBUG kernel this ordering
* is asserted by the ndi_devi_enter() implementation. There is also an
* ndi_devi_enter(Client), which is atypical since the client is a leaf.
* This is done to synchronize pathinfo nodes during devinfo snapshot (see
* di_register_pip) by pretending that the pathinfo nodes are children
* of the client.
*
* In addition to devinfo locking the current implementation utilizes
* the following locks:
*
* mdi_mutex: protects the global list of vHCIs.
*
* vh_phci_mutex: per-vHCI (mutex) lock: protects list of pHCIs registered
* with vHCI.
*
* associated with vHCI.
*
* ph_mutex: per-pHCI (mutex) lock: protects the column (pHCI-mdi_pathinfo
* node list) and per-pHCI structure fields. mdi_pathinfo node creation,
* deletion and child mdi_pathinfo node state changes are serialized on per
* pHCI basis (Protection against DR).
*
* ct_mutex: per-client (mutex) lock: protects the row (client-mdi_pathinfo
* node list) and per-client structure fields. The client-mdi_pathinfo node
* list is typically walked to select an optimal path when routing I/O
* requests.
*
* pi_mutex: per-mdi_pathinfo (mutex) lock: protects the mdi_pathinfo node
* structure fields.
*
* Note that per-Client structure and per-pHCI fields are freely readable when
* corresponding mdi_pathinfo locks are held, since holding an mdi_pathinfo
* node guarantees that its corresponding client and pHCI devices will not be
* freed.
*/
/*
* MDI Client global unique identifier property name string definition
*/
extern const char *mdi_client_guid_prop;
/*
* MDI Client load balancing policy definitions
*
* Load balancing policies are determined on a per-vHCI basis and are
* configurable via the vHCI's driver.conf file.
*/
typedef enum {
} client_lb_t;
typedef struct {
int region_size;
/*
*/
extern const char *mdi_load_balance;
extern const char *mdi_load_balance_none;
extern const char *mdi_load_balance_ap;
extern const char *mdi_load_balance_rr;
extern const char *mdi_load_balance_lba;
/* default for region size */
/*
* vHCI drivers:
*
* vHCI drivers are pseudo nexus drivers which implement multipath services
* for a specific command set or bus architecture ('class'). There is a
* single instance of the vHCI driver for each command set which supports
* multipath devices.
*
* Each vHCI driver registers the following callbacks from attach(9e).
*/
typedef struct mdi_vhci_ops {
/* revision management */
int vo_revision;
/* mdi_pathinfo node init callback */
/* mdi_pathinfo node uninit callback */
int flags);
/* mdi_pathinfo node state change callback */
/* Client path failover callback */
/* Client attached callback */
/* Ask vHCI if 'cinfo' device is support as a client */
void *cinfo);
/*
* An mdi_vhci structure is created and bound to the devinfo node of every
* registered vHCI class driver; this happens when a vHCI registers itself from
* attach(9e). This structure is unbound and freed when the vHCI unregisters
* at detach(9e) time;
*
* Each vHCI driver is associated with a vHCI class name; this is the handle
* used to register and unregister pHCI drivers for a given transport.
*
* Locking: Different parts of this structure are guarded by different
* locks: global threading of multiple vHCIs and initialization is protected
* by mdi_mutex, the list of pHCIs associated with a vHCI is protected by
* vh_phci_mutex, and Clients are protected by vh_client_mutex.
*
* XXX Depending on the context, some of the fields can be freely read without
* holding any locks (ex. holding vh_client_mutex lock also guarantees that
* the vHCI (parent) cannot be unexpectedly freed).
*/
typedef struct mdi_vhci {
/* protected by mdi_mutex... */
/* protected by MDI_VHCI_PHCI_LOCK vh_phci_mutex... */
/* protected by MDI_VHCI_CLIENT_LOCK vh_client_mutex... */
} mdi_vhci_t;
/*
* per-vHCI lock macros
*/
#ifdef DEBUG
#endif /* DEBUG */
#ifdef DEBUG
#endif /* DEBUG */
/*
* GUID Hash definitions
*
* Since all the mpxio managed devices for a given class are enumerated under
* the single vHCI instance for that class, sequentially walking through the
* client device link to find a client would be prohibitively slow.
*/
/*
* Client hash table structure
*/
struct client_hash {
};
/*
* pHCI Drivers:
*
* Physical HBA drivers provide transport services for mpxio-managed devices.
* As each pHCI instance is attached, it must register itself with the mpxio
* framework using mdi_phci_register(). When the pHCI is detached it must
* similarly call mdi_phci_unregister().
*
* The framework maintains a list of registered pHCI device instances for each
* vHCI. This list involves (vh_phci_count, vh_phci_head, vh_phci_tail) and
* (ph_next, ph_prev, ph_vhci) and is protected by vh_phci_mutex.
*
* Locking order:
*
* _NOTE(LOCK_ORDER(mdi_mutex, mdi_phci::ph_mutex)) XXX
* _NOTE(LOCK_ORDER(mdi_phci::ph_mutex devinfo_tree_lock)) XXX
*/
typedef struct mdi_phci {
/* protected by MDI_VHCI_PHCI_LOCK vh_phci_mutex... */
/* protected by MDI_PHCI_LOCK ph_mutex... */
/* protected by mdi_phci_[gs]et_vhci_private caller... */
} mdi_phci_t;
/*
* A pHCI device is 'unstable' while one or more paths are in a transitional
* state. Hotplugging is prevented during this state.
*/
(ph)->ph_unstable--; \
if ((ph)->ph_unstable == 0) { \
} \
}
/*
* per-pHCI lock macros
*/
#ifdef DEBUG
#endif /* DEBUG */
/*
* pHCI state definitions and macros to track the pHCI driver instance state
*/
#define MDI_PHCI_DISABLE_MASK \
/*
* mpxio Managed Clients:
*
* This framework creates a struct mdi_client for every client device created
* by the framework as a result of self-enumeration of target devices by the
* registered pHCI devices. This structure is bound to client device dev_info
* node at the time of client device allocation (ndi_devi_alloc(9e)). This
* structure is unbound from the dev_info node when mpxio framework removes a
* client device node from the system.
*
* This structure is created when a first path is enumerated and removed when
* last path is de-enumerated from the system.
*
* Multipath client devices are instantiated as children of corresponding vHCI
* driver instance. Each client device is uniquely identified by a GUID
* provided by target device itself. The parent vHCI device also maintains a
* hashed list of client devices, protected by vh_client_mutex.
*
* Typically pHCI devices self-enumerate their child devices using taskq,
* resulting in multiple paths to the same client device to be enumerated by
* competing threads.
*
* Currently this framework supports two kinds of load-balancing policy
* configurable through the vHCI driver configuration files.
*
* NONE - Legacy AP mode
* Round Robin - Balance the pHCI load in a Round Robin fashion.
*
* This framework identifies the client device in three distinct states:
*
* OPTIMAL - Client device has at least one redundant path.
* DEGRADED - No redundant paths (critical). Failure in the current active
* path would result in data access failures.
* FAILED - No paths are available to access this device.
*
* Locking order:
*
* _NOTE(LOCK_ORDER(mdi_mutex, mdi_client::ct_mutex)) XXX
* _NOTE(LOCK_ORDER(mdi_client::ct_mutex devinfo_tree_lock)) XXX
*/
typedef struct mdi_client {
/* protected by MDI_VHCI_CLIENT_LOCK vh_client_mutex... */
/* protected by MDI_CLIENT_LOCK ct_mutex... */
/* Paths in power transient state */
short ct_powercnt_config;
short ct_powercnt_unconfig;
int ct_powercnt_reset;
/* ct_power_cnt was reset */
} mdi_client_t;
/*
* per-Client device locking definitions
*/
#ifdef DEBUG
#endif /* DEBUG */
/*
* A Client device is in unstable while one or more paths are in transitional
* state. We do not allow failover to take place while paths are in transient
* state. Similarly we do not allow state transition while client device
* failover is in progress.
*/
(ct)->ct_unstable--; \
if ((ct)->ct_unstable == 0) { \
} \
}
/*
* Client driver instance state definitions:
*/
/*
* Client operating states.
*/
/*
* mdi_pathinfo nodes:
*
* From this framework's perspective, a 'path' is a tuple consisting of a
* client or end device, a host controller which provides device
* identification and transport services (pHCI), and bus specific unit
* addressing information. A path may be decorated with properties which
* describe the capabilities of the path; such properties are analogous to
* device node and minor node properties.
*
* The framework maintains link list of mdi_pathinfo nodes created by every
* pHCI driver instance via the pi_phci_link linkage; this is used (for example)
* to make sure that all relevant pathinfo nodes are freed before the pHCI
* is unregistered.
*
* Locking order:
*
* _NOTE(LOCK_ORDER(mdi_phci::ph_mutex mdi_pathinfo::pi_mutex)) XXX
* _NOTE(LOCK_ORDER(mdi_client::ct_mutex mdi_pathinfo::pi_mutex)) XXX
* _NOTE(LOCK_ORDER(mdi_phci::ph_mutex mdi_client::ct_mutex)) XXX
* _NOTE(LOCK_ORDER(devinfo_tree_lock mdi_pathinfo::pi_mutex)) XXX
*
* mdi_pathinfo node structure definition
*/
struct mdi_pathinfo {
/* protected by MDI_PHCI_LOCK ph_mutex... */
/* protected by MDI_CLIENT_LOCK ct_mutex... */
/* protected by MDI_VHCI_CLIENT_LOCK vh_client_mutex... */
/* protected by MDI_PI_LOCK pi_mutex... */
};
/*
* pathinfo statistics:
*
* The mpxio architecture allows for multiple pathinfo nodes for each
* client-pHCI combination. For statistics purposes, these statistics are
* aggregated into a single client-pHCI set of kstats.
*/
struct mdi_pi_kstats {
};
/*
* pathinfo error kstat
*/
struct pi_errs {
};
/*
* increment an error counter
*/
} \
}
/*
* error codes which can be passed to MDI_PI_ERRSTAT
*/
#ifdef DEBUG
#endif /* DEBUG */
(MDI_PI_IS_USER_DISABLE(pip) || \
MDI_PI_IS_DRV_DISABLE(pip) || \
/*
* mdi_vhcache_client, mdi_vhcache_pathinfo, and mdi_vhcache_phci structures
* hold the vhci to phci client mappings of the on-disk vhci busconfig cache.
*/
/* phci structure of vhci cache */
typedef struct mdi_vhcache_phci {
/* pathinfo structure of vhci cache */
typedef struct mdi_vhcache_pathinfo {
/*
* cpi_flags
*
* MDI_CPI_HINT_PATH_DOES_NOT_EXIST - set when configuration of the path has
* failed.
*/
/* client structure of vhci cache */
typedef struct mdi_vhcache_client {
/* vhci cache structure - one for vhci instance */
typedef struct mdi_vhci_cache {
/* vhcache_flags */
/* vhci bus config structure - one for vhci instance */
typedef struct mdi_vhci_config {
/* flush vhci cache when lbolt reaches vhc_flush_at_ticks */
/*
* Head and tail of the client list whose paths are being configured
* asynchronously. vhc_acc_count is the number of clients on this list.
* vhc_acc_thrcount is the number threads running to configure
* the paths for these clients.
*/
int vhc_acc_count;
int vhc_acc_thrcount;
/* callback id - for flushing the cache during system shutdown */
/*
* vhc_path_discovery_boot - number of times path discovery will be
* attempted during early boot.
* vhc_path_discovery_postboot number of times path discovery will be
* attempted during late boot.
* vhc_path_discovery_cutoff_time - time at which paths were last
* discovered + some timeout
*/
/* vhc_flags */
typedef struct mdi_phys_path {
char *phys_path;
/*
* Lookup tokens are used to cache the result of the vhci cache client lookup
* operations (to reduce the number of real lookup operations).
*/
typedef struct mdi_vhcache_lookup_token {
/* asynchronous configuration of client paths */
typedef struct mdi_async_client_config {
/*
* vHCI driver instance registration/unregistration
*
* mdi_vhci_register() is called by a vHCI driver to register itself as the
* manager of devices from a particular 'class'. This should be called from
* attach(9e).
*
* mdi_vhci_unregister() is called from detach(9E) to unregister a vHCI
* instance from the framework.
*/
int mdi_vhci_unregister(dev_info_t *, int);
/*
* Utility functions
*/
int mdi_phci_get_path_count(dev_info_t *);
/*
* Path Selection Functions:
*
* mdi_select_path() is called by a vHCI driver to select to which path an
* I/O request should be routed. The caller passes the 'buf' structure as
* one of the parameters. The mpxio framework uses the buf's contents to
* maintain per path statistics (total I/O size / count pending). If more
* than one online path is available, the framework automatically selects
* a suitable one. If a failover operation is active for this client device
* the call fails, returning MDI_BUSY.
*
* By default this function returns a suitable path in the 'online' state,
* based on the current load balancing policy. Currently we support
* LOAD_BALANCE_NONE (Previously selected online path will continue to be
* used as long as the path is usable) and LOAD_BALANCE_RR (Online paths
* will be selected in a round robin fashion). The load balancing scheme
* can be configured in the vHCI driver's configuration file (driver.conf).
*
* vHCI drivers may override this default behavior by specifying appropriate
* flags. If start_pip is specified (non NULL), it is used as the routine's
* starting point; it starts walking from there to find the next appropriate
* path.
*
* The following values for 'flags' are currently defined, the third argument
* to mdi_select_path depends on the flags used.
*
* <none>: default, arg is pip
* MDI_SELECT_ONLINE_PATH: select an ONLINE path preferred-first,
* arg is pip
* MDI_SELECT_STANDBY_PATH: select a STANDBY path, arg is pip
* MDI_SELECT_USER_DISABLE_PATH: select user disable for failover and
* auto_failback
* MDI_SELECT_PATH_INSTANCE: select a specific path, arg is
* path instance
* MDI_SELECT_NO_PREFERRED: select path without preferred-first
*
* The selected paths are returned in an mdi_hold_path() state (pi_ref_cnt),
* caller should release the hold by calling mdi_rele_path() at the end of
* operation.
*/
void *, mdi_pathinfo_t **);
int mdi_set_lb_region_size(dev_info_t *, int);
/*
* flags for mdi_select_path() routine
*/
/*
* MDI client device utility functions
*/
int mdi_client_get_path_count(dev_info_t *);
/*
* Failover:
*
* The vHCI driver calls mdi_failover() to initiate a failover operation.
* mdi_failover() calls back into the vHCI driver's vo_failover()
* entry point to perform the actual failover operation. The reason
* for requiring the vHCI driver to initiate failover by calling
* mdi_failover(), instead of directly executing vo_failover() itself,
* is to ensure that the mdi framework can keep track of the client
* state properly. Additionally, mdi_failover() provides as a
* convenience the option of performing the failover operation
* synchronously or asynchronously
*
* Upon successful completion of the failover operation, the paths that were
* previously ONLINE will be in the STANDBY state, and the newly activated
* paths will be in the ONLINE state.
*
* The flags modifier determines whether the activation is done synchronously
*/
/*
* Client device failover mode of operation
*/
/*
* mdi_is_dev_supported: The pHCI driver bus_config implementation calls
* mdi_is_dev_supported to determine if a child device should is supported as
* a vHCI child (i.e. as a client). The method used to specify the child
* device, via the cinfo argument, is by agreement between the pHCI and the
* vHCI. In the case of SCSA and scsi_vhci cinfo is a pointer to the pHCI
* probe dev_info node, which is decorated with the device idenity information
* necessary to determine scsi_vhci support.
*/
/*
* mdi_pathinfo node kstat functions.
*/
int mdi_pi_kstat_exists(mdi_pathinfo_t *);
/*
* mdi_pathinfo node extended state change functions.
*/
int mdi_pi_get_preferred(mdi_pathinfo_t *);
/*
* mdi_pathinfo node member functions
*/
void *mdi_pi_get_client_private(mdi_pathinfo_t *);
void mdi_pi_set_client_private(mdi_pathinfo_t *, void *);
void mdi_pi_set_preferred(mdi_pathinfo_t *, int);
void *mdi_client_get_vhci_private(dev_info_t *);
void mdi_client_set_vhci_private(dev_info_t *, void *);
void *mdi_phci_get_vhci_private(dev_info_t *);
void mdi_phci_set_vhci_private(dev_info_t *, void *);
void *mdi_pi_get_vhci_private(mdi_pathinfo_t *);
void mdi_pi_set_vhci_private(mdi_pathinfo_t *, void *);
/*
* mdi_pathinfo Property utilities
*/
/* obsolete interface, to be removed */
int mdi_get_component_type(dev_info_t *);
#endif /* _KERNEL */
#ifdef __cplusplus
}
#endif
#endif /* _SYS_MDI_IMPLDEFS_H */