/*
* CDDL HEADER START
*
* The contents of this file are subject to the terms of the
* Common Development and Distribution License (the "License").
* You may not use this file except in compliance with the License.
*
* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
* See the License for the specific language governing permissions
* and limitations under the License.
*
* When distributing Covered Code, include this CDDL HEADER in each
* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
* If applicable, add the following below this CDDL HEADER, with the
* fields enclosed by brackets "[]" replaced with your own identifying
* information: Portions Copyright [yyyy] [name of copyright owner]
*
* CDDL HEADER END
*/
/*
* Copyright (c) 2012, 2015 by Delphix. All rights reserved.
*/
/*
* ZFS fault injection
*
* To handle fault injection, we keep track of a series of zinject_record_t
* structures which describe which logical block(s) should be injected with a
* fault. These are kept in a global list. Each record corresponds to a given
* spa_t and maintains a special hold on the spa_t so that it cannot be deleted
* or exported while the injection record exists.
*
* Device level injection is done using the 'zi_guid' field. If this is set, it
* means that the error is destined for a particular device, not a piece of
* data.
*
* This is a rather poor data structure and algorithm, but we don't expect more
* than a few faults at any one time, so it should be sufficient for our needs.
*/
#include <sys/zio_impl.h>
#include <sys/zfs_ioctl.h>
#include <sys/vdev_impl.h>
#include <sys/dmu_objset.h>
/*
* Data describing each zinject handler registered on the system, and
* contains the list node linking the handler in the global zinject
* handler list.
*/
typedef struct inject_handler {
int zi_id;
int zi_next_lane;
/*
* List of all zinject handlers registered on the system, protected by
* the inject_lock defined below.
*/
/*
* This protects insertion into, and traversal of, the inject handler
* list defined above; as well as the inject_delay_count. Any time a
* handler is inserted or removed from the list, this lock should be
* taken as a RW_WRITER; and any time traversal is done over the list
* (without modification to it) this lock should be taken as a RW_READER.
*/
/*
* This holds the number of zinject delay handlers that have been
* registered on the system. It is protected by the inject_lock defined
* above. Thus modifications to this count must be a RW_WRITER of the
* inject_lock, and reads of this count must be (at least) a RW_READER
* of the lock.
*/
static int inject_delay_count = 0;
/*
* This lock is used only in zio_handle_io_delay(), refer to the comment
* in that function for more details.
*/
/*
* Used to assign unique identifying numbers to each new zinject handler.
*/
/*
* Returns true if the given record matches the I/O in progress.
*/
static boolean_t
{
/*
* Check for a match against the MOS, which is based on type
*/
else
return (B_FALSE);
}
/*
* Check for an exact match.
*/
return (B_FALSE);
}
/*
* Panic the system when a config change happens in the function
* specified by tag.
*/
void
{
continue;
}
}
/*
* Determine if the I/O in question should return failure. Returns the errno
* to be returned to the caller.
*/
int
{
int ret = 0;
/*
* Ignore I/O not associated with any logical data.
*/
return (0);
/*
* Currently, we only support fault injection on reads.
*/
return (0);
continue;
/* If this handler matches, return EIO */
break;
}
}
return (ret);
}
/*
* Determine if the zio is part of a label update and has an injection
* handler associated with that portion of the label. Currently, we
* allow error injection in either the nvlist or the uberblock region of
* of the vdev label.
*/
int
{
int label;
int ret = 0;
if (offset >= VDEV_LABEL_START_SIZE &&
return (0);
continue;
/*
* The injection region is the relative offsets within a
* vdev label. We must determine the label which is being
* updated and adjust our region accordingly.
*/
break;
}
}
return (ret);
}
int
{
int ret = 0;
/*
* We skip over faults in the labels unless it's during
* device open (i.e. zio == NULL).
*/
if (offset < VDEV_LABEL_START_SIZE ||
return (0);
}
continue;
(ZIO_FLAG_IO_RETRY | ZIO_FLAG_TRYHARD)))) {
continue;
}
/* Handle type specific I/O failures */
continue;
/*
* For a failed open, pretend like the device
* has gone away.
*/
/*
* Treat these errors as if they had been
* retried so that all the appropriate stats
* and FMA events are generated.
*/
break;
}
break;
}
}
}
return (ret);
}
/*
* Simulate hardware that ignores cache flushes. For requested number
* of seconds nix the actual writing to disk.
*/
void
{
/* Ignore errors not destined for this pool */
continue;
/*
* Positive duration implies # of seconds, negative
* a number of txgs
*/
else
}
/* Have a "problem" writing 60% of the time */
break;
}
}
void
{
if (zio_injection_enabled == 0)
return;
continue;
ddi_get_lbolt64());
} else {
/* duration is negative so the subtraction here adds */
}
}
}
{
/*
* inject_delay_count is a subset of zio_injection_enabled that
* is only incremented for delay handlers. These checks are
* mainly added to remind the reader why we're not explicitly
* checking zio_injection_enabled like the other functions.
*/
/*
* If there aren't any inject delay handlers registered, then we
* can short circuit and simply return 0 here. A value of zero
* informs zio_delay_interrupt() that this request should not be
* delayed. This short circuit keeps us from acquiring the
* inject_delay_mutex unnecessarily.
*/
if (inject_delay_count == 0) {
return (0);
}
/*
* Each inject handler has a number of "lanes" associated with
* it. Each lane is able to handle requests independently of one
* another, and at a latency defined by the inject handler
* record's zi_timer field. Thus if a handler in configured with
* a single lane with a 10ms latency, it will delay requests
* such that only a single request is completed every 10ms. So,
* if more than one request is attempted per each 10ms interval,
* the average latency of the requests will be greater than
* 10ms; but if only a single request is submitted each 10ms
* interval the average latency will be 10ms.
*
* We need to acquire this mutex to prevent multiple concurrent
* threads being assigned to the same lane of a given inject
* handler. The mutex allows us to perform the following two
* operations atomically:
*
* 1. determine the minimum handler and minimum target
* value of all the possible handlers
* 2. update that minimum handler's lane array
*
* Without atomicity, two (or more) threads could pick the same
* lane in step (1), and then conflict with each other in step
* (2). This could allow a single lane handler to process
* multiple requests simultaneously, which shouldn't be possible.
*/
continue;
continue;
/*
* Defensive; should never happen as the array allocation
* occurs prior to inserting this handler on the list.
*/
/*
* This should never happen, the zinject command should
* prevent a user from setting an IO delay with zero lanes.
*/
/*
* We want to issue this IO to the lane that will become
* idle the soonest, so we compare the soonest this
* specific handler can complete the IO with all other
* handlers, to find the lowest value of all possible
* lanes. We then use this lane to submit the request.
*
* Since each handler has a constant value for its
* delay, we can just use the "next" lane for that
* handler; as it will always be the lane with the
* lowest value for that particular handler (i.e. the
* lane that will become idle the soonest). This saves a
* scan of each handler's lanes array.
*
* There's two cases to consider when determining when
* this specific IO request should complete. If this
* lane is idle, we want to "submit" the request now so
* it will complete after zi_timer milliseconds. Thus,
* we set the target to now + zi_timer.
*
* If the lane is busy, we want this request to complete
* zi_timer milliseconds after the lane becomes idle.
* Since the 'zi_lanes' array holds the time at which
* each lane will become idle, we use that value to
* determine when this request should complete.
*/
if (min_handler == NULL) {
min_target = target;
continue;
}
ASSERT3U(min_target, !=, 0);
/*
* We don't yet increment the "next lane" variable since
* we still might find a lower value lane in another
* handler during any remaining iterations. Once we're
* sure we've selected the absolute minimum, we'll claim
* the lane and increment the handler's "next lane"
* field below.
*/
if (target < min_target) {
min_target = target;
}
}
/*
* 'min_handler' will be NULL if no IO delays are registered for
* this vdev, otherwise it will point to the handler containing
* the lane that will become idle the soonest.
*/
if (min_handler != NULL) {
ASSERT3U(min_target, !=, 0);
/*
* If we've used all possible lanes for this handler,
* loop back and start using the first lane again;
* otherwise, just increment the lane index.
*/
}
return (min_target);
}
/*
* Create a new handler for the given record. We add it to the list, adding
* a reference to the spa_t in the process. We increment zio_injection_enabled,
* which is the switch to trigger all fault injection.
*/
int
{
int error;
/*
* If this is pool-wide metadata, make sure we unload the corresponding
* spa_t, so that the next attempt to load it will trigger the fault.
* We call spa_reset() to unload the pool appropriately.
*/
if (flags & ZINJECT_UNLOAD_SPA)
return (error);
/*
* A value of zero for the number of lanes or for the
* delay time doesn't make sense.
*/
/*
* The number of lanes is directly mapped to the size of
* an array used by the handler. Thus, to ensure the
* user doesn't trigger an allocation that's "too large"
* we cap the number of lanes here.
*/
}
if (!(flags & ZINJECT_NULL)) {
/*
* spa_inject_ref() will add an injection reference, which will
* prevent the pool from being removed from the namespace while
* still allowing it to be unloaded.
*/
handler->zi_next_lane = 0;
} else {
handler->zi_next_lane = 0;
}
/*
* We can't move this increment into the conditional
* above because we need to hold the RW_WRITER lock of
* inject_lock, and we don't want to hold that while
* allocating the handler's zi_lanes array.
*/
ASSERT3S(inject_delay_count, >=, 0);
ASSERT3S(inject_delay_count, >, 0);
}
}
/*
* Flush the ARC, so that any attempts to read this data will end up
* going to the ZIO layer. Note that this is a little overkill, but
* we don't have the necessary ARC interfaces to do anything else, and
* fault injection isn't a performance critical path.
*/
if (flags & ZINJECT_FLUSH_ARC)
/*
* We must use FALSE to ensure arc_flush returns, since
* we're not preventing concurrent ARC insertions.
*/
return (0);
}
/*
* Returns the next record with an ID greater than that supplied to the
* function. Used to iterate over all handlers in the system.
*/
int
{
int ret;
break;
if (handler) {
ret = 0;
} else {
}
return (ret);
}
/*
* Clear the fault handler with the given identifier, or return ENOENT if none
* exists.
*/
int
{
break;
}
ASSERT3S(inject_delay_count, >, 0);
ASSERT3S(inject_delay_count, >=, 0);
}
} else {
}
return (0);
}
void
zio_inject_init(void)
{
}
void
zio_inject_fini(void)
{
}