vdev_queue.c revision 97e81309571898df9fdd94aab1216dfcf23e060b
4496171313bed39e96f21bc2f9faf2868e267ae3girish * CDDL HEADER START
4496171313bed39e96f21bc2f9faf2868e267ae3girish * The contents of this file are subject to the terms of the
4496171313bed39e96f21bc2f9faf2868e267ae3girish * Common Development and Distribution License (the "License").
4496171313bed39e96f21bc2f9faf2868e267ae3girish * You may not use this file except in compliance with the License.
4496171313bed39e96f21bc2f9faf2868e267ae3girish * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
4496171313bed39e96f21bc2f9faf2868e267ae3girish * See the License for the specific language governing permissions
4496171313bed39e96f21bc2f9faf2868e267ae3girish * and limitations under the License.
4496171313bed39e96f21bc2f9faf2868e267ae3girish * When distributing Covered Code, include this CDDL HEADER in each
4496171313bed39e96f21bc2f9faf2868e267ae3girish * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
4496171313bed39e96f21bc2f9faf2868e267ae3girish * If applicable, add the following below this CDDL HEADER, with the
4496171313bed39e96f21bc2f9faf2868e267ae3girish * fields enclosed by brackets "[]" replaced with your own identifying
4496171313bed39e96f21bc2f9faf2868e267ae3girish * information: Portions Copyright [yyyy] [name of copyright owner]
4496171313bed39e96f21bc2f9faf2868e267ae3girish * CDDL HEADER END
4496171313bed39e96f21bc2f9faf2868e267ae3girish * Copyright 2009 Sun Microsystems, Inc. All rights reserved.
fb2f18f820d90b001aea4fb27dd654bc1263c440esaxe * Use is subject to license terms.
4496171313bed39e96f21bc2f9faf2868e267ae3girish * Copyright (c) 2012, 2014 by Delphix. All rights reserved.
102033aa92edf302ad31b3bdd7c6fcd2d6910903dp * ZFS I/O Scheduler
4496171313bed39e96f21bc2f9faf2868e267ae3girish * ---------------
4496171313bed39e96f21bc2f9faf2868e267ae3girish * ZFS issues I/O operations to leaf vdevs to satisfy and complete zios. The
4496171313bed39e96f21bc2f9faf2868e267ae3girish * I/O scheduler determines when and in what order those operations are
4496171313bed39e96f21bc2f9faf2868e267ae3girish * issued. The I/O scheduler divides operations into five I/O classes
4496171313bed39e96f21bc2f9faf2868e267ae3girish * prioritized in the following order: sync read, sync write, async read,
4496171313bed39e96f21bc2f9faf2868e267ae3girish * async write, and scrub/resilver. Each queue defines the minimum and
4496171313bed39e96f21bc2f9faf2868e267ae3girish * maximum number of concurrent operations that may be issued to the device.
4496171313bed39e96f21bc2f9faf2868e267ae3girish * In addition, the device has an aggregate maximum. Note that the sum of the
4496171313bed39e96f21bc2f9faf2868e267ae3girish * per-queue minimums must not exceed the aggregate maximum, and if the
4496171313bed39e96f21bc2f9faf2868e267ae3girish * aggregate maximum is equal to or greater than the sum of the per-queue
4496171313bed39e96f21bc2f9faf2868e267ae3girish * maximums, the per-queue minimum has no effect.
4496171313bed39e96f21bc2f9faf2868e267ae3girish * For many physical devices, throughput increases with the number of
4496171313bed39e96f21bc2f9faf2868e267ae3girish * concurrent operations, but latency typically suffers. Further, physical
4496171313bed39e96f21bc2f9faf2868e267ae3girish * devices typically have a limit at which more concurrent operations have no
4496171313bed39e96f21bc2f9faf2868e267ae3girish * effect on throughput or can actually cause it to decrease.
4496171313bed39e96f21bc2f9faf2868e267ae3girish * The scheduler selects the next operation to issue by first looking for an
4496171313bed39e96f21bc2f9faf2868e267ae3girish * I/O class whose minimum has not been satisfied. Once all are satisfied and
4496171313bed39e96f21bc2f9faf2868e267ae3girish * the aggregate maximum has not been hit, the scheduler looks for classes
4496171313bed39e96f21bc2f9faf2868e267ae3girish * whose maximum has not been satisfied. Iteration through the I/O classes is
4496171313bed39e96f21bc2f9faf2868e267ae3girish * done in the order specified above. No further operations are issued if the
4496171313bed39e96f21bc2f9faf2868e267ae3girish * aggregate maximum number of concurrent operations has been hit or if there
4496171313bed39e96f21bc2f9faf2868e267ae3girish * are no operations queued for an I/O class that has not hit its maximum.
4496171313bed39e96f21bc2f9faf2868e267ae3girish * Every time an i/o is queued or an operation completes, the I/O scheduler
4496171313bed39e96f21bc2f9faf2868e267ae3girish * looks for new operations to issue.
4496171313bed39e96f21bc2f9faf2868e267ae3girish * All I/O classes have a fixed maximum number of outstanding operations
4496171313bed39e96f21bc2f9faf2868e267ae3girish * except for the async write class. Asynchronous writes represent the data
4496171313bed39e96f21bc2f9faf2868e267ae3girish * that is committed to stable storage during the syncing stage for
4496171313bed39e96f21bc2f9faf2868e267ae3girish * transaction groups (see txg.c). Transaction groups enter the syncing state
4496171313bed39e96f21bc2f9faf2868e267ae3girish * periodically so the number of queued async writes will quickly burst up and
4496171313bed39e96f21bc2f9faf2868e267ae3girish * then bleed down to zero. Rather than servicing them as quickly as possible,
4496171313bed39e96f21bc2f9faf2868e267ae3girish * the I/O scheduler changes the maximum number of active async write i/os
4496171313bed39e96f21bc2f9faf2868e267ae3girish * according to the amount of dirty data in the pool (see dsl_pool.c). Since
4496171313bed39e96f21bc2f9faf2868e267ae3girish * both throughput and latency typically increase with the number of
4496171313bed39e96f21bc2f9faf2868e267ae3girish * concurrent operations issued to physical devices, reducing the burstiness
4496171313bed39e96f21bc2f9faf2868e267ae3girish * in the number of concurrent operations also stabilizes the response time of
4496171313bed39e96f21bc2f9faf2868e267ae3girish * operations from other -- and in particular synchronous -- queues. In broad
4496171313bed39e96f21bc2f9faf2868e267ae3girish * strokes, the I/O scheduler will issue more concurrent operations from the
4496171313bed39e96f21bc2f9faf2868e267ae3girish * async write queue as there's more dirty data in the pool.
4496171313bed39e96f21bc2f9faf2868e267ae3girish * Async Writes
4496171313bed39e96f21bc2f9faf2868e267ae3girish * The number of concurrent operations issued for the async write I/O class
4496171313bed39e96f21bc2f9faf2868e267ae3girish * follows a piece-wise linear function defined by a few adjustable points.
4496171313bed39e96f21bc2f9faf2868e267ae3girish * | o---------| <-- zfs_vdev_async_write_max_active
4496171313bed39e96f21bc2f9faf2868e267ae3girish * | | / | |
4496171313bed39e96f21bc2f9faf2868e267ae3girish * active | / | |
4496171313bed39e96f21bc2f9faf2868e267ae3girish * I/O | / | |
4496171313bed39e96f21bc2f9faf2868e267ae3girish * count | / | |
4496171313bed39e96f21bc2f9faf2868e267ae3girish * |------------o | | <-- zfs_vdev_async_write_min_active
4496171313bed39e96f21bc2f9faf2868e267ae3girish * 0|____________^______|_________|
4496171313bed39e96f21bc2f9faf2868e267ae3girish * 0% | | 100% of zfs_dirty_data_max
4496171313bed39e96f21bc2f9faf2868e267ae3girish * | `-- zfs_vdev_async_write_active_max_dirty_percent
4496171313bed39e96f21bc2f9faf2868e267ae3girish * `--------- zfs_vdev_async_write_active_min_dirty_percent
4496171313bed39e96f21bc2f9faf2868e267ae3girish * Until the amount of dirty data exceeds a minimum percentage of the dirty
4496171313bed39e96f21bc2f9faf2868e267ae3girish * data allowed in the pool, the I/O scheduler will limit the number of
4496171313bed39e96f21bc2f9faf2868e267ae3girish * concurrent operations to the minimum. As that threshold is crossed, the
4496171313bed39e96f21bc2f9faf2868e267ae3girish * number of concurrent operations issued increases linearly to the maximum at
4496171313bed39e96f21bc2f9faf2868e267ae3girish * the specified maximum percentage of the dirty data allowed in the pool.
4496171313bed39e96f21bc2f9faf2868e267ae3girish * Ideally, the amount of dirty data on a busy pool will stay in the sloped
4496171313bed39e96f21bc2f9faf2868e267ae3girish * part of the function between zfs_vdev_async_write_active_min_dirty_percent
4496171313bed39e96f21bc2f9faf2868e267ae3girish * and zfs_vdev_async_write_active_max_dirty_percent. If it exceeds the
4496171313bed39e96f21bc2f9faf2868e267ae3girish * maximum percentage, this indicates that the rate of incoming data is
4496171313bed39e96f21bc2f9faf2868e267ae3girish * greater than the rate that the backend storage can handle. In this case, we
4496171313bed39e96f21bc2f9faf2868e267ae3girish * must further throttle incoming writes (see dmu_tx_delay() for details).
4496171313bed39e96f21bc2f9faf2868e267ae3girish * The maximum number of i/os active to each device. Ideally, this will be >=
4496171313bed39e96f21bc2f9faf2868e267ae3girish * the sum of each queue's max_active. It must be at least the sum of each
4496171313bed39e96f21bc2f9faf2868e267ae3girish * queue's min_active.
4496171313bed39e96f21bc2f9faf2868e267ae3girish * Per-queue limits on the number of i/os active to each device. If the
4496171313bed39e96f21bc2f9faf2868e267ae3girish * sum of the queue's max_active is < zfs_vdev_max_active, then the
4496171313bed39e96f21bc2f9faf2868e267ae3girish * min_active comes into play. We will send min_active from each queue,
4496171313bed39e96f21bc2f9faf2868e267ae3girish * and then select from queues in the order defined by zio_priority_t.
4496171313bed39e96f21bc2f9faf2868e267ae3girish * In general, smaller max_active's will lead to lower latency of synchronous
102033aa92edf302ad31b3bdd7c6fcd2d6910903dp * operations. Larger max_active's may lead to higher overall throughput,
102033aa92edf302ad31b3bdd7c6fcd2d6910903dp * depending on underlying storage.
102033aa92edf302ad31b3bdd7c6fcd2d6910903dp * The ratio of the queues' max_actives determines the balance of performance
102033aa92edf302ad31b3bdd7c6fcd2d6910903dp * between reads, writes, and scrubs. E.g., increasing
102033aa92edf302ad31b3bdd7c6fcd2d6910903dp * zfs_vdev_scrub_max_active will cause the scrub or resilver to complete
102033aa92edf302ad31b3bdd7c6fcd2d6910903dp * more quickly, but reads and writes to have higher latency and lower
4496171313bed39e96f21bc2f9faf2868e267ae3girish * throughput.
4496171313bed39e96f21bc2f9faf2868e267ae3girish * When the pool has less than zfs_vdev_async_write_active_min_dirty_percent
4496171313bed39e96f21bc2f9faf2868e267ae3girish * dirty data, use zfs_vdev_async_write_min_active. When it has more than
4496171313bed39e96f21bc2f9faf2868e267ae3girish * zfs_vdev_async_write_active_max_dirty_percent, use
4496171313bed39e96f21bc2f9faf2868e267ae3girish * zfs_vdev_async_write_max_active. The value is linearly interpolated
4496171313bed39e96f21bc2f9faf2868e267ae3girish * between min and max.
4496171313bed39e96f21bc2f9faf2868e267ae3girishint zfs_vdev_async_write_active_min_dirty_percent = 30;
4496171313bed39e96f21bc2f9faf2868e267ae3girishint zfs_vdev_async_write_active_max_dirty_percent = 60;
459190a5c46206e7885f6a649a055ceb46be49a7rsmaeda * To reduce IOPs, we aggregate small adjacent I/Os into one large I/O.
4496171313bed39e96f21bc2f9faf2868e267ae3girish * For read I/Os, we also aggregate across small adjacency gaps; for writes
4496171313bed39e96f21bc2f9faf2868e267ae3girish * we include spans of optional I/Os to aid aggregation at the disk even when
fb2f18f820d90b001aea4fb27dd654bc1263c440esaxe * they aren't able to help us aggregate at this level.
459190a5c46206e7885f6a649a055ceb46be49a7rsmaedaint zfs_vdev_aggregation_limit = SPA_OLD_MAXBLOCKSIZE;
4496171313bed39e96f21bc2f9faf2868e267ae3girishvdev_queue_offset_compare(const void *x1, const void *x2)
fb2f18f820d90b001aea4fb27dd654bc1263c440esaxe return (-1);
fb2f18f820d90b001aea4fb27dd654bc1263c440esaxe return (1);
459190a5c46206e7885f6a649a055ceb46be49a7rsmaeda return (-1);
459190a5c46206e7885f6a649a055ceb46be49a7rsmaeda return (1);
459190a5c46206e7885f6a649a055ceb46be49a7rsmaeda return (0);
4496171313bed39e96f21bc2f9faf2868e267ae3girishstatic inline avl_tree_t *
459190a5c46206e7885f6a649a055ceb46be49a7rsmaedavdev_queue_class_tree(vdev_queue_t *vq, zio_priority_t p)
4496171313bed39e96f21bc2f9faf2868e267ae3girishstatic inline avl_tree_t *
459190a5c46206e7885f6a649a055ceb46be49a7rsmaedavdev_queue_timestamp_compare(const void *x1, const void *x2)
4496171313bed39e96f21bc2f9faf2868e267ae3girish return (-1);
4496171313bed39e96f21bc2f9faf2868e267ae3girish return (1);
4496171313bed39e96f21bc2f9faf2868e267ae3girish return (-1);
4496171313bed39e96f21bc2f9faf2868e267ae3girish return (1);
4496171313bed39e96f21bc2f9faf2868e267ae3girish return (0);
4496171313bed39e96f21bc2f9faf2868e267ae3girish avl_create(&vq->vq_active_tree, vdev_queue_offset_compare,
4496171313bed39e96f21bc2f9faf2868e267ae3girish sizeof (zio_t), offsetof(struct zio, io_queue_node));
4496171313bed39e96f21bc2f9faf2868e267ae3girish for (zio_priority_t p = 0; p < ZIO_PRIORITY_NUM_QUEUEABLE; p++) {
4496171313bed39e96f21bc2f9faf2868e267ae3girish int (*compfn) (const void *, const void *);
4496171313bed39e96f21bc2f9faf2868e267ae3girish * The synchronous i/o queues are dispatched in FIFO rather
4496171313bed39e96f21bc2f9faf2868e267ae3girish * than LBA order. This provides more consistent latency for
4496171313bed39e96f21bc2f9faf2868e267ae3girish if (p == ZIO_PRIORITY_SYNC_READ || p == ZIO_PRIORITY_SYNC_WRITE)
4496171313bed39e96f21bc2f9faf2868e267ae3girish sizeof (zio_t), offsetof(struct zio, io_queue_node));
102033aa92edf302ad31b3bdd7c6fcd2d6910903dp for (zio_priority_t p = 0; p < ZIO_PRIORITY_NUM_QUEUEABLE; p++)
102033aa92edf302ad31b3bdd7c6fcd2d6910903dpstatic void
102033aa92edf302ad31b3bdd7c6fcd2d6910903dp ASSERT3U(zio->io_priority, <, ZIO_PRIORITY_NUM_QUEUEABLE);
102033aa92edf302ad31b3bdd7c6fcd2d6910903dp avl_add(vdev_queue_class_tree(vq, zio->io_priority), zio);
102033aa92edf302ad31b3bdd7c6fcd2d6910903dpstatic void
102033aa92edf302ad31b3bdd7c6fcd2d6910903dp ASSERT3U(zio->io_priority, <, ZIO_PRIORITY_NUM_QUEUEABLE);
102033aa92edf302ad31b3bdd7c6fcd2d6910903dp avl_remove(vdev_queue_class_tree(vq, zio->io_priority), zio);
102033aa92edf302ad31b3bdd7c6fcd2d6910903dp ASSERT3U(spa->spa_queue_stats[zio->io_priority].spa_queued, >, 0);
102033aa92edf302ad31b3bdd7c6fcd2d6910903dpstatic void
102033aa92edf302ad31b3bdd7c6fcd2d6910903dp ASSERT3U(zio->io_priority, <, ZIO_PRIORITY_NUM_QUEUEABLE);
102033aa92edf302ad31b3bdd7c6fcd2d6910903dpstatic void
102033aa92edf302ad31b3bdd7c6fcd2d6910903dp ASSERT3U(zio->io_priority, <, ZIO_PRIORITY_NUM_QUEUEABLE);
102033aa92edf302ad31b3bdd7c6fcd2d6910903dp ASSERT3U(spa->spa_queue_stats[zio->io_priority].spa_active, >, 0);
102033aa92edf302ad31b3bdd7c6fcd2d6910903dpstatic void
102033aa92edf302ad31b3bdd7c6fcd2d6910903dp switch (p) {
102033aa92edf302ad31b3bdd7c6fcd2d6910903dp return (0);
102033aa92edf302ad31b3bdd7c6fcd2d6910903dp * Sync tasks correspond to interactive user actions. To reduce the
102033aa92edf302ad31b3bdd7c6fcd2d6910903dp * execution time of those actions we push data out as fast as possible.
102033aa92edf302ad31b3bdd7c6fcd2d6910903dp * linear interpolation:
102033aa92edf302ad31b3bdd7c6fcd2d6910903dp * slope = (max_writes - min_writes) / (max_bytes - min_bytes)
102033aa92edf302ad31b3bdd7c6fcd2d6910903dp * move right by min_bytes
102033aa92edf302ad31b3bdd7c6fcd2d6910903dp * move up by min_writes
102033aa92edf302ad31b3bdd7c6fcd2d6910903dp switch (p) {
102033aa92edf302ad31b3bdd7c6fcd2d6910903dp return (0);
102033aa92edf302ad31b3bdd7c6fcd2d6910903dp * Return the i/o class to issue from, or ZIO_PRIORITY_MAX_QUEUEABLE if
102033aa92edf302ad31b3bdd7c6fcd2d6910903dp * there is no eligible class.
102033aa92edf302ad31b3bdd7c6fcd2d6910903dp if (avl_numnodes(&vq->vq_active_tree) >= zfs_vdev_max_active)
102033aa92edf302ad31b3bdd7c6fcd2d6910903dp /* find a queue that has not reached its minimum # outstanding i/os */
102033aa92edf302ad31b3bdd7c6fcd2d6910903dp for (p = 0; p < ZIO_PRIORITY_NUM_QUEUEABLE; p++) {
fe70c9cf90dfc23d18485fb7b4b20a1175d53a8bdp return (p);
fe70c9cf90dfc23d18485fb7b4b20a1175d53a8bdp * If we haven't found a queue, look for one that hasn't reached its
fe70c9cf90dfc23d18485fb7b4b20a1175d53a8bdp * maximum # outstanding i/os.
fe70c9cf90dfc23d18485fb7b4b20a1175d53a8bdp for (p = 0; p < ZIO_PRIORITY_NUM_QUEUEABLE; p++) {
fe70c9cf90dfc23d18485fb7b4b20a1175d53a8bdp return (p);
fe70c9cf90dfc23d18485fb7b4b20a1175d53a8bdp /* No eligible queued i/os */
fe70c9cf90dfc23d18485fb7b4b20a1175d53a8bdp * Compute the range spanned by two i/os, which is the endpoint of the last
fe70c9cf90dfc23d18485fb7b4b20a1175d53a8bdp * (lio->io_offset + lio->io_size) minus start of the first (fio->io_offset).
fe70c9cf90dfc23d18485fb7b4b20a1175d53a8bdp * Conveniently, the gap between fio and lio is given by -IO_SPAN(lio, fio);
fe70c9cf90dfc23d18485fb7b4b20a1175d53a8bdp * thus fio and lio are adjacent if and only if IO_SPAN(lio, fio) == 0.
fe70c9cf90dfc23d18485fb7b4b20a1175d53a8bdp#define IO_SPAN(fio, lio) ((lio)->io_offset + (lio)->io_size - (fio)->io_offset)
return (NULL);
if (stretch) {
return (NULL);
return (aio);
static zio_t *
if (p == ZIO_PRIORITY_NUM_QUEUEABLE) {
return (NULL);
goto again;
return (zio);
zio_t *
return (zio);
return (NULL);
return (NULL);
return (nio);