/*
* CDDL HEADER START
*
* The contents of this file are subject to the terms of the
* Common Development and Distribution License (the "License").
* You may not use this file except in compliance with the License.
*
* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
* See the License for the specific language governing permissions
* and limitations under the License.
*
* When distributing Covered Code, include this CDDL HEADER in each
* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
* If applicable, add the following below this CDDL HEADER, with the
* fields enclosed by brackets "[]" replaced with your own identifying
* information: Portions Copyright [yyyy] [name of copyright owner]
*
* CDDL HEADER END
*/
/*
* Copyright 2009 Sun Microsystems, Inc. All rights reserved.
* Use is subject to license terms.
*/
/*
* Copyright (c) 2012, 2014 by Delphix. All rights reserved.
* Copyright (c) 2014 Integros [integros.com]
*/
#include <sys/zfs_context.h>
#include <sys/vdev_impl.h>
#include <sys/spa_impl.h>
#include <sys/dsl_pool.h>
#include <sys/metaslab_impl.h>
/*
* ZFS I/O Scheduler
* ---------------
*
* ZFS issues I/O operations to leaf vdevs to satisfy and complete zios. The
* I/O scheduler determines when and in what order those operations are
* issued. The I/O scheduler divides operations into five I/O classes
* prioritized in the following order: sync read, sync write, async read,
* maximum number of concurrent operations that may be issued to the device.
* In addition, the device has an aggregate maximum. Note that the sum of the
* per-queue minimums must not exceed the aggregate maximum, and if the
* aggregate maximum is equal to or greater than the sum of the per-queue
* maximums, the per-queue minimum has no effect.
*
* For many physical devices, throughput increases with the number of
* concurrent operations, but latency typically suffers. Further, physical
* devices typically have a limit at which more concurrent operations have no
* effect on throughput or can actually cause it to decrease.
*
* The scheduler selects the next operation to issue by first looking for an
* I/O class whose minimum has not been satisfied. Once all are satisfied and
* the aggregate maximum has not been hit, the scheduler looks for classes
* whose maximum has not been satisfied. Iteration through the I/O classes is
* done in the order specified above. No further operations are issued if the
* aggregate maximum number of concurrent operations has been hit or if there
* are no operations queued for an I/O class that has not hit its maximum.
* Every time an i/o is queued or an operation completes, the I/O scheduler
* looks for new operations to issue.
*
* All I/O classes have a fixed maximum number of outstanding operations
* except for the async write class. Asynchronous writes represent the data
* that is committed to stable storage during the syncing stage for
* transaction groups (see txg.c). Transaction groups enter the syncing state
* periodically so the number of queued async writes will quickly burst up and
* then bleed down to zero. Rather than servicing them as quickly as possible,
* according to the amount of dirty data in the pool (see dsl_pool.c). Since
* both throughput and latency typically increase with the number of
* concurrent operations issued to physical devices, reducing the burstiness
* in the number of concurrent operations also stabilizes the response time of
* operations from other -- and in particular synchronous -- queues. In broad
* strokes, the I/O scheduler will issue more concurrent operations from the
* async write queue as there's more dirty data in the pool.
*
* Async Writes
*
* The number of concurrent operations issued for the async write I/O class
* follows a piece-wise linear function defined by a few adjustable points.
*
* | o---------| <-- zfs_vdev_async_write_max_active
* ^ | /^ |
* | | / | |
* active | / | |
* I/O | / | |
* count | / | |
* | / | |
* |------------o | | <-- zfs_vdev_async_write_min_active
* 0|____________^______|_________|
* 0% | | 100% of zfs_dirty_data_max
* | |
* | `-- zfs_vdev_async_write_active_max_dirty_percent
* `--------- zfs_vdev_async_write_active_min_dirty_percent
*
* Until the amount of dirty data exceeds a minimum percentage of the dirty
* data allowed in the pool, the I/O scheduler will limit the number of
* concurrent operations to the minimum. As that threshold is crossed, the
* number of concurrent operations issued increases linearly to the maximum at
* the specified maximum percentage of the dirty data allowed in the pool.
*
* Ideally, the amount of dirty data on a busy pool will stay in the sloped
* part of the function between zfs_vdev_async_write_active_min_dirty_percent
* and zfs_vdev_async_write_active_max_dirty_percent. If it exceeds the
* maximum percentage, this indicates that the rate of incoming data is
* greater than the rate that the backend storage can handle. In this case, we
* must further throttle incoming writes (see dmu_tx_delay() for details).
*/
/*
* the sum of each queue's max_active. It must be at least the sum of each
* queue's min_active.
*/
/*
* sum of the queue's max_active is < zfs_vdev_max_active, then the
* min_active comes into play. We will send min_active from each queue,
* and then select from queues in the order defined by zio_priority_t.
*
* In general, smaller max_active's will lead to lower latency of synchronous
* operations. Larger max_active's may lead to higher overall throughput,
* depending on underlying storage.
*
* The ratio of the queues' max_actives determines the balance of performance
* between reads, writes, and scrubs. E.g., increasing
* zfs_vdev_scrub_max_active will cause the scrub or resilver to complete
* more quickly, but reads and writes to have higher latency and lower
* throughput.
*/
/*
* When the pool has less than zfs_vdev_async_write_active_min_dirty_percent
* dirty data, use zfs_vdev_async_write_min_active. When it has more than
* zfs_vdev_async_write_active_max_dirty_percent, use
* zfs_vdev_async_write_max_active. The value is linearly interpolated
* between min and max.
*/
/*
* they aren't able to help us aggregate at this level.
*/
/*
* Define the queue depth percentage for each top-level. This percentage is
* used in conjunction with zfs_vdev_async_max_active to determine how many
* allocations a specific top-level vdev should handle. Once the queue depth
* reaches zfs_vdev_queue_depth_pct * zfs_vdev_async_write_max_active / 100
* then allocator will stop allocating blocks on that top-level device.
* The default kernel setting is 1000% which will yield 100 allocations per
* device. For userland testing, the default setting is 300% which equates
* to 30 allocations per device.
*/
#ifdef _KERNEL
#else
#endif
int
{
return (-1);
return (1);
return (-1);
return (1);
return (0);
}
static inline avl_tree_t *
{
}
static inline avl_tree_t *
{
if (t == ZIO_TYPE_READ)
return (&vq->vq_read_offset_tree);
else
return (&vq->vq_write_offset_tree);
}
int
{
return (-1);
return (1);
return (-1);
return (1);
return (0);
}
void
{
for (zio_priority_t p = 0; p < ZIO_PRIORITY_NUM_QUEUEABLE; p++) {
int (*compfn) (const void *, const void *);
/*
* The synchronous i/o queues are dispatched in FIFO rather
* than LBA order. This provides more consistent latency for
*/
if (p == ZIO_PRIORITY_SYNC_READ || p == ZIO_PRIORITY_SYNC_WRITE)
else
}
}
void
{
for (zio_priority_t p = 0; p < ZIO_PRIORITY_NUM_QUEUEABLE; p++)
}
static void
{
}
static void
{
}
static void
{
}
static void
{
}
}
}
static void
{
}
}
}
static int
{
switch (p) {
case ZIO_PRIORITY_SYNC_READ:
return (zfs_vdev_sync_read_min_active);
case ZIO_PRIORITY_SYNC_WRITE:
return (zfs_vdev_sync_write_min_active);
case ZIO_PRIORITY_ASYNC_READ:
return (zfs_vdev_async_read_min_active);
case ZIO_PRIORITY_ASYNC_WRITE:
return (zfs_vdev_async_write_min_active);
case ZIO_PRIORITY_SCRUB:
return (zfs_vdev_scrub_min_active);
default:
panic("invalid priority %u", p);
return (0);
}
}
static int
{
int writes;
/*
* Sync tasks correspond to interactive user actions. To reduce the
* execution time of those actions we push data out as fast as possible.
*/
if (spa_has_pending_synctask(spa)) {
return (zfs_vdev_async_write_max_active);
}
return (zfs_vdev_async_write_min_active);
return (zfs_vdev_async_write_max_active);
/*
* linear interpolation:
* slope = (max_writes - min_writes) / (max_bytes - min_bytes)
* move right by min_bytes
* move up by min_writes
*/
return (writes);
}
static int
{
switch (p) {
case ZIO_PRIORITY_SYNC_READ:
return (zfs_vdev_sync_read_max_active);
case ZIO_PRIORITY_SYNC_WRITE:
return (zfs_vdev_sync_write_max_active);
case ZIO_PRIORITY_ASYNC_READ:
return (zfs_vdev_async_read_max_active);
case ZIO_PRIORITY_ASYNC_WRITE:
return (vdev_queue_max_async_writes(spa));
case ZIO_PRIORITY_SCRUB:
return (zfs_vdev_scrub_max_active);
default:
panic("invalid priority %u", p);
return (0);
}
}
/*
* Return the i/o class to issue from, or ZIO_PRIORITY_MAX_QUEUEABLE if
* there is no eligible class.
*/
static zio_priority_t
{
return (ZIO_PRIORITY_NUM_QUEUEABLE);
for (p = 0; p < ZIO_PRIORITY_NUM_QUEUEABLE; p++) {
return (p);
}
/*
* If we haven't found a queue, look for one that hasn't reached its
*/
for (p = 0; p < ZIO_PRIORITY_NUM_QUEUEABLE; p++) {
return (p);
}
return (ZIO_PRIORITY_NUM_QUEUEABLE);
}
/*
* (lio->io_offset + lio->io_size) minus start of the first (fio->io_offset).
* Conveniently, the gap between fio and lio is given by -IO_SPAN(lio, fio);
* thus fio and lio are adjacent if and only if IO_SPAN(lio, fio) == 0.
*/
static zio_t *
{
return (NULL);
/*
* the same flavor, as expressed by the AGG_INHERIT flags.
* The latter requirement is necessary so that certain
* attributes of the I/O, such as whether it's a normal I/O
* to begin a range as they add no benefit in that situation.
*/
/*
* We keep track of the last non-optional I/O.
*/
/*
* recording the last non-option I/O.
*/
}
/*
*/
}
/*
*/
}
/*
* Now that we've established the range of the I/O aggregation
* For reads, there's nothing to do. While we are unable to
* aggregate further, it's possible that a trailing optional
* I/O would allow the underlying device to aggregate with
* non-optional I/O is close enough to make aggregation
* worthwhile.
*/
break;
}
}
}
if (stretch) {
/* This may be a no-op. */
} else {
}
}
return (NULL);
do {
}
return (aio);
}
static zio_t *
{
p = vdev_queue_class_to_issue(vq);
if (p == ZIO_PRIORITY_NUM_QUEUEABLE) {
return (NULL);
}
/*
* For LBA-ordered queues (async / scrub), issue the i/o which follows
* the most recently issued i/o in LBA (offset) order.
*
* For FIFO queues (sync), issue the i/o with the lowest timestamp.
*/
search.io_timestamp = 0;
else
/*
* If the I/O is or was optional and therefore has no data, we need to
* simply discard it. We need to drop the vdev queue's lock to avoid a
* deadlock that we could encounter since this I/O will complete
* immediately.
*/
goto again;
}
return (zio);
}
zio_t *
{
return (zio);
/*
* not match the child's i/o type. Fix it up here.
*/
} else {
}
return (NULL);
return (NULL);
}
return (nio);
}
void
{
} else {
}
}
}