fa9e4066f08beec538e775443c5be79dd423fcabahrens * CDDL HEADER START
fa9e4066f08beec538e775443c5be79dd423fcabahrens * The contents of this file are subject to the terms of the
ea8dc4b6d2251b437950c0056bc626b311c73c27eschrock * Common Development and Distribution License (the "License").
ea8dc4b6d2251b437950c0056bc626b311c73c27eschrock * You may not use this file except in compliance with the License.
fa9e4066f08beec538e775443c5be79dd423fcabahrens * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
fa9e4066f08beec538e775443c5be79dd423fcabahrens * See the License for the specific language governing permissions
fa9e4066f08beec538e775443c5be79dd423fcabahrens * and limitations under the License.
fa9e4066f08beec538e775443c5be79dd423fcabahrens * When distributing Covered Code, include this CDDL HEADER in each
fa9e4066f08beec538e775443c5be79dd423fcabahrens * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
fa9e4066f08beec538e775443c5be79dd423fcabahrens * If applicable, add the following below this CDDL HEADER, with the
fa9e4066f08beec538e775443c5be79dd423fcabahrens * fields enclosed by brackets "[]" replaced with your own identifying
fa9e4066f08beec538e775443c5be79dd423fcabahrens * information: Portions Copyright [yyyy] [name of copyright owner]
fa9e4066f08beec538e775443c5be79dd423fcabahrens * CDDL HEADER END
3f9d6ad73e45c6823b409f93b0c8d4f62861d2d5Lin Ling * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
8df0bcf0df7622a075cc6e52f659d2fcfdd08cdcPaul Dagnelie * Copyright (c) 2011, 2016 by Delphix. All rights reserved.
5aeb94743e3be0c51e86f73096334611ae3a058eGarrett D'Amore * Copyright (c) 2011 Nexenta Systems, Inc. All rights reserved.
c3d26abc9ee97b4f60233556aadeb57e0bd30bb9Matthew Ahrens * Copyright (c) 2014 Integros [integros.com]
fa9e4066f08beec538e775443c5be79dd423fcabahrens * ==========================================================================
fa9e4066f08beec538e775443c5be79dd423fcabahrens * I/O type descriptions
fa9e4066f08beec538e775443c5be79dd423fcabahrens * ==========================================================================
80eb36f241abf8c076119fb4c49a55fd61ebc710George Wilson "zio_null", "zio_read", "zio_write", "zio_free", "zio_claim",
fa9e4066f08beec538e775443c5be79dd423fcabahrens * ==========================================================================
fa9e4066f08beec538e775443c5be79dd423fcabahrens * I/O kmem caches
fa9e4066f08beec538e775443c5be79dd423fcabahrens * ==========================================================================
fa9e4066f08beec538e775443c5be79dd423fcabahrenskmem_cache_t *zio_buf_cache[SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT];
ad23a2db4cfc94c0ed1d58554479ce8d2e7e5768johansenkmem_cache_t *zio_data_buf_cache[SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT];
a2cdcdd260232b58202b11a9bfc0103c9449ed52Paul Dagnelie (((uint64_t)1) << ((level) * ((indblkshift) - SPA_BLKPTRSHIFT)))
01f55e48fb4d524eaf70687728aa51b7762e2e97George Wilson * The following actions directly effect the spa's sync-to-convergence logic.
01f55e48fb4d524eaf70687728aa51b7762e2e97George Wilson * The values below define the sync pass when we start performing the action.
01f55e48fb4d524eaf70687728aa51b7762e2e97George Wilson * Care should be taken when changing these values as they directly impact
01f55e48fb4d524eaf70687728aa51b7762e2e97George Wilson * spa_sync() performance. Tuning these values may introduce subtle performance
01f55e48fb4d524eaf70687728aa51b7762e2e97George Wilson * pathologies and should only be done in the context of performance analysis.
01f55e48fb4d524eaf70687728aa51b7762e2e97George Wilson * These tunables will eventually be removed and replaced with #defines once
01f55e48fb4d524eaf70687728aa51b7762e2e97George Wilson * enough analysis has been done to determine optimal values.
01f55e48fb4d524eaf70687728aa51b7762e2e97George Wilson * The 'zfs_sync_pass_deferred_free' pass must be greater than 1 to ensure that
01f55e48fb4d524eaf70687728aa51b7762e2e97George Wilson * regular blocks are not deferred.
01f55e48fb4d524eaf70687728aa51b7762e2e97George Wilsonint zfs_sync_pass_deferred_free = 2; /* defer frees starting in this pass */
01f55e48fb4d524eaf70687728aa51b7762e2e97George Wilsonint zfs_sync_pass_dont_compress = 5; /* don't compress starting in this pass */
01f55e48fb4d524eaf70687728aa51b7762e2e97George Wilsonint zfs_sync_pass_rewrite = 2; /* rewrite new bps starting in this pass */
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick * An allocating zio is one that either currently has the DVA allocate
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick * stage set or will have it later in its lifetime.
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick#define IO_IS_ALLOCATING(zio) ((zio)->io_orig_pipeline & ZIO_STAGE_DVA_ALLOCATE)
35a5a3587fd94b666239c157d3722745250ccbd7Jonathan Adamsboolean_t zio_requeue_io_start_cut_in_line = B_TRUE;
0f7643c7376dd69a08acbfc9d1d7d548b10c846aGeorge Wilsonstatic void zio_taskq_dispatch(zio_t *, zio_taskq_type_t, boolean_t);
a3f829ae41ece20e7f5f63604e177aeeb8b24628Bill Moore sizeof (zio_t), 0, NULL, NULL, NULL, NULL, NULL, 0);
a3f829ae41ece20e7f5f63604e177aeeb8b24628Bill Moore zio_link_cache = kmem_cache_create("zio_link_cache",
a3f829ae41ece20e7f5f63604e177aeeb8b24628Bill Moore sizeof (zio_link_t), 0, NULL, NULL, NULL, NULL, NULL, 0);
fa9e4066f08beec538e775443c5be79dd423fcabahrens * For small buffers, we want a cache for each multiple of
b515258426fed6c7311fd3f1dea697cfbd4085c6Matthew Ahrens * SPA_MINBLOCKSIZE. For larger buffers, we want a cache
b515258426fed6c7311fd3f1dea697cfbd4085c6Matthew Ahrens * for each quarter-power of 2.
fa9e4066f08beec538e775443c5be79dd423fcabahrens for (c = 0; c < SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT; c++) {
e291592ab12a560fc73b0610963bb3fe66aab341Jonathan Adams size_t cflags = (size > zio_buf_debug_limit) ? KMC_NODEBUG : 0;
cd1c8b85eb30b568e9816221430c479ace7a559dMatthew Ahrens * If we are using watchpoints, put each buffer on its own page,
cd1c8b85eb30b568e9816221430c479ace7a559dMatthew Ahrens * to eliminate the performance overhead of trapping to the
cd1c8b85eb30b568e9816221430c479ace7a559dMatthew Ahrens * kernel when modifying a non-watched buffer that shares the
cd1c8b85eb30b568e9816221430c479ace7a559dMatthew Ahrens * page with a watched buffer.
cd1c8b85eb30b568e9816221430c479ace7a559dMatthew Ahrens if (arc_watch && !IS_P2ALIGNED(size, PAGESIZE))
fa9e4066f08beec538e775443c5be79dd423fcabahrens if (align != 0) {
e291592ab12a560fc73b0610963bb3fe66aab341Jonathan Adams align, NULL, NULL, NULL, NULL, NULL, cflags);
e291592ab12a560fc73b0610963bb3fe66aab341Jonathan Adams * Since zio_data bufs do not appear in crash dumps, we
e291592ab12a560fc73b0610963bb3fe66aab341Jonathan Adams * pass KMC_NOTOUCH so that no allocator metadata is
e291592ab12a560fc73b0610963bb3fe66aab341Jonathan Adams * stored with the buffers.
ad23a2db4cfc94c0ed1d58554479ce8d2e7e5768johansen (void) sprintf(name, "zio_data_buf_%lu", (ulong_t)size);
ad23a2db4cfc94c0ed1d58554479ce8d2e7e5768johansen zio_data_buf_cache[c] = kmem_cache_create(name, size,
fa9e4066f08beec538e775443c5be79dd423fcabahrens while (--c != 0) {
fa9e4066f08beec538e775443c5be79dd423fcabahrens for (c = 0; c < SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT; c++) {
fa9e4066f08beec538e775443c5be79dd423fcabahrens * ==========================================================================
fa9e4066f08beec538e775443c5be79dd423fcabahrens * Allocate and free I/O buffers
fa9e4066f08beec538e775443c5be79dd423fcabahrens * ==========================================================================
ad23a2db4cfc94c0ed1d58554479ce8d2e7e5768johansen * Use zio_buf_alloc to allocate ZFS metadata. This data will appear in a
ad23a2db4cfc94c0ed1d58554479ce8d2e7e5768johansen * crashdump if the kernel panics, so use it judiciously. Obviously, it's
ad23a2db4cfc94c0ed1d58554479ce8d2e7e5768johansen * useful to inspect ZFS metadata, but if possible, we should avoid keeping
ad23a2db4cfc94c0ed1d58554479ce8d2e7e5768johansen * excess / transient data in-core during a crashdump.
f63ab3d5a84a12b474655fc7e700db3efba6c4c9Matthew Ahrens VERIFY3U(c, <, SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT);
1ab7f2ded02e7a1bc3c73516eb27efa79bf2a2ffmaybee return (kmem_cache_alloc(zio_buf_cache[c], KM_PUSHPAGE));
ad23a2db4cfc94c0ed1d58554479ce8d2e7e5768johansen * Use zio_data_buf_alloc to allocate data. The data will not appear in a
ad23a2db4cfc94c0ed1d58554479ce8d2e7e5768johansen * crashdump if the kernel panics. This exists so that we will limit the amount
ad23a2db4cfc94c0ed1d58554479ce8d2e7e5768johansen * of ZFS data that shows up in a kernel crashdump. (Thus reducing the amount
ad23a2db4cfc94c0ed1d58554479ce8d2e7e5768johansen * of kernel heap dumped to disk when the kernel panics)
f63ab3d5a84a12b474655fc7e700db3efba6c4c9Matthew Ahrens VERIFY3U(c, <, SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT);
1ab7f2ded02e7a1bc3c73516eb27efa79bf2a2ffmaybee return (kmem_cache_alloc(zio_data_buf_cache[c], KM_PUSHPAGE));
f63ab3d5a84a12b474655fc7e700db3efba6c4c9Matthew Ahrens VERIFY3U(c, <, SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT);
f63ab3d5a84a12b474655fc7e700db3efba6c4c9Matthew Ahrens VERIFY3U(c, <, SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT);
fa9e4066f08beec538e775443c5be79dd423fcabahrens * ==========================================================================
fa9e4066f08beec538e775443c5be79dd423fcabahrens * Push and pop I/O transform buffers
fa9e4066f08beec538e775443c5be79dd423fcabahrens * ==========================================================================
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwickzio_push_transform(zio_t *zio, void *data, uint64_t size, uint64_t bufsize,
fa9e4066f08beec538e775443c5be79dd423fcabahrens zio_transform_t *zt = kmem_alloc(sizeof (zio_transform_t), KM_SLEEP);
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick while ((zt = zio->io_transform_stack) != NULL) {
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick * ==========================================================================
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick * I/O transform callbacks for subblocks and decompression
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick * ==========================================================================
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwickzio_subblock(zio_t *zio, void *data, uint64_t size)
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwickzio_decompress(zio_t *zio, void *data, uint64_t size)
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick zio_decompress_data(BP_GET_COMPRESS(zio->io_bp),
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick * ==========================================================================
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick * I/O parent/child relationships and pipeline interlocks
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick * ==========================================================================
0f7643c7376dd69a08acbfc9d1d7d548b10c846aGeorge Wilson *zl = (*zl == NULL) ? list_head(pl) : list_next(pl, *zl);
0f7643c7376dd69a08acbfc9d1d7d548b10c846aGeorge Wilson *zl = (*zl == NULL) ? list_head(cl) : list_next(cl, *zl);
0f7643c7376dd69a08acbfc9d1d7d548b10c846aGeorge Wilson VERIFY3P(zio_walk_parents(cio, &zl), ==, NULL);
a3f829ae41ece20e7f5f63604e177aeeb8b24628Bill Moore zio_link_t *zl = kmem_cache_alloc(zio_link_cache, KM_SLEEP);
a3f829ae41ece20e7f5f63604e177aeeb8b24628Bill Moore * Logical I/Os can have logical, gang, or vdev children.
a3f829ae41ece20e7f5f63604e177aeeb8b24628Bill Moore * Gang I/Os can have gang or vdev children.
a3f829ae41ece20e7f5f63604e177aeeb8b24628Bill Moore * Vdev I/Os can only have vdev children.
a3f829ae41ece20e7f5f63604e177aeeb8b24628Bill Moore * The following ASSERT captures all of these constraints.
a3f829ae41ece20e7f5f63604e177aeeb8b24628Bill Moore ASSERT(cio->io_child_type <= pio->io_child_type);
a3f829ae41ece20e7f5f63604e177aeeb8b24628Bill Moore for (int w = 0; w < ZIO_WAIT_TYPES; w++)
a3f829ae41ece20e7f5f63604e177aeeb8b24628Bill Moore pio->io_children[cio->io_child_type][w] += !cio->io_state[w];
a3f829ae41ece20e7f5f63604e177aeeb8b24628Bill Moorezio_remove_child(zio_t *pio, zio_t *cio, zio_link_t *zl)
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwickzio_wait_for_children(zio_t *zio, enum zio_child child, enum zio_wait_type wait)
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick uint64_t *countp = &zio->io_children[child][wait];
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwickzio_notify_parent(zio_t *pio, zio_t *zio, enum zio_wait_type wait)
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick uint64_t *countp = &pio->io_children[zio->io_child_type][wait];
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick int *errorp = &pio->io_child_error[zio->io_child_type];
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick if (zio->io_error && !(zio->io_flags & ZIO_FLAG_DONT_PROPAGATE))
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick *errorp = zio_worst_error(*errorp, zio->io_error);
69962b5647e4a8b9b14998733b765925381b727eMatthew Ahrens if (*countp == 0 && pio->io_stall == countp) {
0f7643c7376dd69a08acbfc9d1d7d548b10c846aGeorge Wilson pio->io_stage < ZIO_STAGE_VDEV_IO_START ? ZIO_TASKQ_ISSUE :
0f7643c7376dd69a08acbfc9d1d7d548b10c846aGeorge Wilson * Dispatch the parent zio in its own taskq so that
0f7643c7376dd69a08acbfc9d1d7d548b10c846aGeorge Wilson * the child can continue to make progress. This also
0f7643c7376dd69a08acbfc9d1d7d548b10c846aGeorge Wilson * prevents overflowing the stack when we have deeply nested
0f7643c7376dd69a08acbfc9d1d7d548b10c846aGeorge Wilson * parent-child relationships.
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwickzio_inherit_child_errors(zio_t *zio, enum zio_child c)
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick if (zio->io_child_error[c] != 0 && zio->io_error == 0)
0f7643c7376dd69a08acbfc9d1d7d548b10c846aGeorge Wilsonzio_timestamp_compare(const void *x1, const void *x2)
0f7643c7376dd69a08acbfc9d1d7d548b10c846aGeorge Wilson if (z1->io_queued_timestamp < z2->io_queued_timestamp)
0f7643c7376dd69a08acbfc9d1d7d548b10c846aGeorge Wilson if (z1->io_queued_timestamp > z2->io_queued_timestamp)
fa9e4066f08beec538e775443c5be79dd423fcabahrens * ==========================================================================
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick * Create the various types of I/O (read, write, free, etc)
fa9e4066f08beec538e775443c5be79dd423fcabahrens * ==========================================================================
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwickzio_create(zio_t *pio, spa_t *spa, uint64_t txg, const blkptr_t *bp,
5602294fda888d923d57a78bafdaf48ae6223deaDan Kimmel void *data, uint64_t lsize, uint64_t psize, zio_done_func_t *done,
5602294fda888d923d57a78bafdaf48ae6223deaDan Kimmel void *private, zio_type_t type, zio_priority_t priority,
5602294fda888d923d57a78bafdaf48ae6223deaDan Kimmel enum zio_flag flags, vdev_t *vd, uint64_t offset,
5602294fda888d923d57a78bafdaf48ae6223deaDan Kimmel const zbookmark_phys_t *zb, enum zio_stage stage, enum zio_stage pipeline)
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick ASSERT(P2PHASE(offset, SPA_MINBLOCKSIZE) == 0);
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick ASSERT(!vd || spa_config_held(spa, SCL_STATE_ALL, RW_READER));
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick ASSERT(!bp || !(flags & ZIO_FLAG_CONFIG_WRITER));
5602294fda888d923d57a78bafdaf48ae6223deaDan Kimmel IMPLY(lsize != psize, (flags & ZIO_FLAG_RAW) != 0);
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick mutex_init(&zio->io_lock, NULL, MUTEX_DEFAULT, NULL);
a3f829ae41ece20e7f5f63604e177aeeb8b24628Bill Moore list_create(&zio->io_parent_list, sizeof (zio_link_t),
a3f829ae41ece20e7f5f63604e177aeeb8b24628Bill Moore list_create(&zio->io_child_list, sizeof (zio_link_t),
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick zio->io_bp = &zio->io_bp_copy; /* so caller can free */
f5383399322244b3008a624bcd7e93d67037d177Bill Moore if (zio->io_child_type > ZIO_CHILD_GANG && BP_IS_GANG(bp))
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick zio->io_orig_pipeline = zio->io_pipeline = pipeline;
a3f829ae41ece20e7f5f63604e177aeeb8b24628Bill Moore zio->io_state[ZIO_WAIT_READY] = (stage >= ZIO_STAGE_READY);
a3f829ae41ece20e7f5f63604e177aeeb8b24628Bill Moore zio->io_state[ZIO_WAIT_DONE] = (stage >= ZIO_STAGE_DONE);
0a4e9518a44f226be6d39383330b5b1792d2f184gwstatic void
a3f829ae41ece20e7f5f63604e177aeeb8b24628Bill Moorezio_null(zio_t *pio, spa_t *spa, vdev_t *vd, zio_done_func_t *done,
5602294fda888d923d57a78bafdaf48ae6223deaDan Kimmel zio = zio_create(pio, spa, 0, NULL, NULL, 0, 0, done, private,
a3f829ae41ece20e7f5f63604e177aeeb8b24628Bill Moore ZIO_TYPE_NULL, ZIO_PRIORITY_NOW, flags, vd, 0, NULL,
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwickzio_root(spa_t *spa, zio_done_func_t *done, void *private, enum zio_flag flags)
a3f829ae41ece20e7f5f63604e177aeeb8b24628Bill Moore return (zio_null(NULL, spa, NULL, done, private, flags));
f63ab3d5a84a12b474655fc7e700db3efba6c4c9Matthew Ahrenszfs_blkptr_verify(spa_t *spa, const blkptr_t *bp)
f63ab3d5a84a12b474655fc7e700db3efba6c4c9Matthew Ahrens zfs_panic_recover("blkptr at %p has invalid TYPE %llu",
f63ab3d5a84a12b474655fc7e700db3efba6c4c9Matthew Ahrens if (BP_GET_CHECKSUM(bp) >= ZIO_CHECKSUM_FUNCTIONS ||
f63ab3d5a84a12b474655fc7e700db3efba6c4c9Matthew Ahrens zfs_panic_recover("blkptr at %p has invalid CHECKSUM %llu",
f63ab3d5a84a12b474655fc7e700db3efba6c4c9Matthew Ahrens if (BP_GET_COMPRESS(bp) >= ZIO_COMPRESS_FUNCTIONS ||
f63ab3d5a84a12b474655fc7e700db3efba6c4c9Matthew Ahrens zfs_panic_recover("blkptr at %p has invalid COMPRESS %llu",
f63ab3d5a84a12b474655fc7e700db3efba6c4c9Matthew Ahrens zfs_panic_recover("blkptr at %p has invalid LSIZE %llu",
f63ab3d5a84a12b474655fc7e700db3efba6c4c9Matthew Ahrens zfs_panic_recover("blkptr at %p has invalid PSIZE %llu",
f63ab3d5a84a12b474655fc7e700db3efba6c4c9Matthew Ahrens if (BPE_GET_ETYPE(bp) > NUM_BP_EMBEDDED_TYPES) {
f63ab3d5a84a12b474655fc7e700db3efba6c4c9Matthew Ahrens zfs_panic_recover("blkptr at %p has invalid ETYPE %llu",
f63ab3d5a84a12b474655fc7e700db3efba6c4c9Matthew Ahrens * Pool-specific checks.
f63ab3d5a84a12b474655fc7e700db3efba6c4c9Matthew Ahrens * Note: it would be nice to verify that the blk_birth and
f63ab3d5a84a12b474655fc7e700db3efba6c4c9Matthew Ahrens * BP_PHYSICAL_BIRTH() are not too large. However, spa_freeze()
f63ab3d5a84a12b474655fc7e700db3efba6c4c9Matthew Ahrens * allows the birth time of log blocks (and dmu_sync()-ed blocks
f63ab3d5a84a12b474655fc7e700db3efba6c4c9Matthew Ahrens * that are in the log) to be arbitrarily large.
f63ab3d5a84a12b474655fc7e700db3efba6c4c9Matthew Ahrens uint64_t vdevid = DVA_GET_VDEV(&bp->blk_dva[i]);
f63ab3d5a84a12b474655fc7e700db3efba6c4c9Matthew Ahrens if (vdevid >= spa->spa_root_vdev->vdev_children) {
f63ab3d5a84a12b474655fc7e700db3efba6c4c9Matthew Ahrens zfs_panic_recover("blkptr at %p DVA %u has invalid "
f63ab3d5a84a12b474655fc7e700db3efba6c4c9Matthew Ahrens vdev_t *vd = spa->spa_root_vdev->vdev_child[vdevid];
f63ab3d5a84a12b474655fc7e700db3efba6c4c9Matthew Ahrens zfs_panic_recover("blkptr at %p DVA %u has invalid "
f63ab3d5a84a12b474655fc7e700db3efba6c4c9Matthew Ahrens zfs_panic_recover("blkptr at %p DVA %u has hole "
f63ab3d5a84a12b474655fc7e700db3efba6c4c9Matthew Ahrens * "missing" vdevs are valid during import, but we
f63ab3d5a84a12b474655fc7e700db3efba6c4c9Matthew Ahrens * don't have their detailed info (e.g. asize), so
f63ab3d5a84a12b474655fc7e700db3efba6c4c9Matthew Ahrens * we can't perform any more checks on them.
f63ab3d5a84a12b474655fc7e700db3efba6c4c9Matthew Ahrens uint64_t offset = DVA_GET_OFFSET(&bp->blk_dva[i]);
f63ab3d5a84a12b474655fc7e700db3efba6c4c9Matthew Ahrens uint64_t asize = DVA_GET_ASIZE(&bp->blk_dva[i]);
f63ab3d5a84a12b474655fc7e700db3efba6c4c9Matthew Ahrens asize = vdev_psize_to_asize(vd, SPA_GANGBLOCKSIZE);
f63ab3d5a84a12b474655fc7e700db3efba6c4c9Matthew Ahrens zfs_panic_recover("blkptr at %p DVA %u has invalid "
f63ab3d5a84a12b474655fc7e700db3efba6c4c9Matthew Ahrens "OFFSET %llu",
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwickzio_read(zio_t *pio, spa_t *spa, const blkptr_t *bp,
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick void *data, uint64_t size, zio_done_func_t *done, void *private,
7802d7bf98dec568dadf72286893b1fe5abd8602Matthew Ahrens zio_priority_t priority, enum zio_flag flags, const zbookmark_phys_t *zb)
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick zio = zio_create(pio, spa, BP_PHYSICAL_BIRTH(bp), bp,
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick ZIO_DDT_CHILD_READ_PIPELINE : ZIO_READ_PIPELINE);
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwickzio_write(zio_t *pio, spa_t *spa, uint64_t txg, blkptr_t *bp,
5602294fda888d923d57a78bafdaf48ae6223deaDan Kimmel void *data, uint64_t lsize, uint64_t psize, const zio_prop_t *zp,
8df0bcf0df7622a075cc6e52f659d2fcfdd08cdcPaul Dagnelie zio_done_func_t *ready, zio_done_func_t *children_ready,
8df0bcf0df7622a075cc6e52f659d2fcfdd08cdcPaul Dagnelie zio_done_func_t *physdone, zio_done_func_t *done,
8df0bcf0df7622a075cc6e52f659d2fcfdd08cdcPaul Dagnelie void *private, zio_priority_t priority, enum zio_flag flags,
5602294fda888d923d57a78bafdaf48ae6223deaDan Kimmel zio = zio_create(pio, spa, txg, bp, data, lsize, psize, done, private,
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick ZIO_DDT_CHILD_WRITE_PIPELINE : ZIO_WRITE_PIPELINE);
5d7b4d438c4a51eccc95e77a83a437b4d48380ebMatthew Ahrens * Data can be NULL if we are going to call zio_write_override() to
5d7b4d438c4a51eccc95e77a83a437b4d48380ebMatthew Ahrens * provide the already-allocated BP. But we may need the data to
5d7b4d438c4a51eccc95e77a83a437b4d48380ebMatthew Ahrens * verify a dedup hit (if requested). In this case, don't try to
5d7b4d438c4a51eccc95e77a83a437b4d48380ebMatthew Ahrens * dedup (just take the already-allocated BP verbatim).
5d7b4d438c4a51eccc95e77a83a437b4d48380ebMatthew Ahrens if (data == NULL && zio->io_prop.zp_dedup_verify) {
5d7b4d438c4a51eccc95e77a83a437b4d48380ebMatthew Ahrens zio->io_prop.zp_dedup = zio->io_prop.zp_dedup_verify = B_FALSE;
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwickzio_rewrite(zio_t *pio, spa_t *spa, uint64_t txg, blkptr_t *bp, void *data,
69962b5647e4a8b9b14998733b765925381b727eMatthew Ahrens uint64_t size, zio_done_func_t *done, void *private,
7802d7bf98dec568dadf72286893b1fe5abd8602Matthew Ahrens zio_priority_t priority, enum zio_flag flags, zbookmark_phys_t *zb)
5602294fda888d923d57a78bafdaf48ae6223deaDan Kimmel zio = zio_create(pio, spa, txg, bp, data, size, size, done, private,
0f7643c7376dd69a08acbfc9d1d7d548b10c846aGeorge Wilson ZIO_TYPE_WRITE, priority, flags | ZIO_FLAG_IO_REWRITE, NULL, 0, zb,
80901aea8e78a2c20751f61f01bebd1d5b5c2ba5George Wilsonzio_write_override(zio_t *zio, blkptr_t *bp, int copies, boolean_t nopwrite)
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick ASSERT(zio->io_child_type == ZIO_CHILD_LOGICAL);
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick ASSERT(zio->io_txg == spa_syncing_txg(zio->io_spa));
80901aea8e78a2c20751f61f01bebd1d5b5c2ba5George Wilson * We must reset the io_prop to match the values that existed
80901aea8e78a2c20751f61f01bebd1d5b5c2ba5George Wilson * when the bp was first written by dmu_sync() keeping in mind
80901aea8e78a2c20751f61f01bebd1d5b5c2ba5George Wilson * that nopwrite and dedup are mutually exclusive.
80901aea8e78a2c20751f61f01bebd1d5b5c2ba5George Wilson zio->io_prop.zp_dedup = nopwrite ? B_FALSE : zio->io_prop.zp_dedup;
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwickzio_free(spa_t *spa, uint64_t txg, const blkptr_t *bp)
5d7b4d438c4a51eccc95e77a83a437b4d48380ebMatthew Ahrens * The check for EMBEDDED is a performance optimization. We
5d7b4d438c4a51eccc95e77a83a437b4d48380ebMatthew Ahrens * process the free here (by ignoring it) rather than
5d7b4d438c4a51eccc95e77a83a437b4d48380ebMatthew Ahrens * putting it on the list and then processing it in zio_free_sync().
9cb154a3c9f170904dce9bad5bd5a7d256b922a4Matthew Ahrens * Frees that are for the currently-syncing txg, are not going to be
9cb154a3c9f170904dce9bad5bd5a7d256b922a4Matthew Ahrens * deferred, and which will not need to do a read (i.e. not GANG or
9cb154a3c9f170904dce9bad5bd5a7d256b922a4Matthew Ahrens * DEDUP), can be processed immediately. Otherwise, put them on the
9cb154a3c9f170904dce9bad5bd5a7d256b922a4Matthew Ahrens * in-memory list for later processing.
9cb154a3c9f170904dce9bad5bd5a7d256b922a4Matthew Ahrens spa_sync_pass(spa) >= zfs_sync_pass_deferred_free) {
9cb154a3c9f170904dce9bad5bd5a7d256b922a4Matthew Ahrens bplist_append(&spa->spa_free_bplist[txg & TXG_MASK], bp);
9cb154a3c9f170904dce9bad5bd5a7d256b922a4Matthew Ahrens VERIFY0(zio_wait(zio_free_sync(NULL, spa, txg, bp, 0)));
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwickzio_free_sync(zio_t *pio, spa_t *spa, uint64_t txg, const blkptr_t *bp,
01f55e48fb4d524eaf70687728aa51b7762e2e97George Wilson ASSERT(spa_sync_pass(spa) < zfs_sync_pass_deferred_free);
5d7b4d438c4a51eccc95e77a83a437b4d48380ebMatthew Ahrens return (zio_null(pio, spa, NULL, NULL, NULL, 0));
9cb154a3c9f170904dce9bad5bd5a7d256b922a4Matthew Ahrens * GANG and DEDUP blocks can induce a read (for the gang block header,
9cb154a3c9f170904dce9bad5bd5a7d256b922a4Matthew Ahrens * or the DDT), so issue them asynchronously so that this thread is
9cb154a3c9f170904dce9bad5bd5a7d256b922a4Matthew Ahrens * not tied up.
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick zio = zio_create(pio, spa, txg, bp, NULL, BP_GET_PSIZE(bp),
5602294fda888d923d57a78bafdaf48ae6223deaDan Kimmel BP_GET_PSIZE(bp), NULL, NULL, ZIO_TYPE_FREE, ZIO_PRIORITY_NOW,
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwickzio_claim(zio_t *pio, spa_t *spa, uint64_t txg, const blkptr_t *bp,
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick zio_done_func_t *done, void *private, enum zio_flag flags)
5d7b4d438c4a51eccc95e77a83a437b4d48380ebMatthew Ahrens return (zio_null(pio, spa, NULL, NULL, NULL, 0));
fa9e4066f08beec538e775443c5be79dd423fcabahrens * A claim is an allocation of a specific block. Claims are needed
fa9e4066f08beec538e775443c5be79dd423fcabahrens * to support immediate writes in the intent log. The issue is that
fa9e4066f08beec538e775443c5be79dd423fcabahrens * immediate writes contain committed data, but in a txg that was
fa9e4066f08beec538e775443c5be79dd423fcabahrens * *not* committed. Upon opening the pool after an unclean shutdown,
fa9e4066f08beec538e775443c5be79dd423fcabahrens * the intent log claims all blocks that contain immediate write data
fa9e4066f08beec538e775443c5be79dd423fcabahrens * so that the SPA knows they're in use.
fa9e4066f08beec538e775443c5be79dd423fcabahrens * All claims *must* be resolved in the first txg -- before the SPA
fa9e4066f08beec538e775443c5be79dd423fcabahrens * starts allocating blocks -- so that nothing is allocated twice.
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick * If txg == 0 we just verify that the block is claimable.
fa9e4066f08beec538e775443c5be79dd423fcabahrens ASSERT3U(spa->spa_uberblock.ub_rootbp.blk_birth, <, spa_first_txg(spa));
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick ASSERT(!BP_GET_DEDUP(bp) || !spa_writeable(spa)); /* zdb(1M) */
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick zio = zio_create(pio, spa, txg, bp, NULL, BP_GET_PSIZE(bp),
5602294fda888d923d57a78bafdaf48ae6223deaDan Kimmel BP_GET_PSIZE(bp), done, private, ZIO_TYPE_CLAIM, ZIO_PRIORITY_NOW,
5602294fda888d923d57a78bafdaf48ae6223deaDan Kimmel flags, NULL, 0, NULL, ZIO_STAGE_OPEN, ZIO_CLAIM_PIPELINE);
fa9e4066f08beec538e775443c5be79dd423fcabahrenszio_ioctl(zio_t *pio, spa_t *spa, vdev_t *vd, int cmd,
69962b5647e4a8b9b14998733b765925381b727eMatthew Ahrens zio_done_func_t *done, void *private, enum zio_flag flags)
5602294fda888d923d57a78bafdaf48ae6223deaDan Kimmel zio = zio_create(pio, spa, 0, NULL, NULL, 0, 0, done, private,
69962b5647e4a8b9b14998733b765925381b727eMatthew Ahrens ZIO_TYPE_IOCTL, ZIO_PRIORITY_NOW, flags, vd, 0, NULL,
a3f829ae41ece20e7f5f63604e177aeeb8b24628Bill Moore zio = zio_null(pio, spa, NULL, NULL, NULL, flags);
fa9e4066f08beec538e775443c5be79dd423fcabahrens zio_nowait(zio_ioctl(zio, spa, vd->vdev_child[c], cmd,
fa9e4066f08beec538e775443c5be79dd423fcabahrenszio_read_phys(zio_t *pio, vdev_t *vd, uint64_t offset, uint64_t size,
fa9e4066f08beec538e775443c5be79dd423fcabahrens void *data, int checksum, zio_done_func_t *done, void *private,
69962b5647e4a8b9b14998733b765925381b727eMatthew Ahrens zio_priority_t priority, enum zio_flag flags, boolean_t labels)
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick ASSERT(!labels || offset + size <= VDEV_LABEL_START_SIZE ||
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick offset >= vd->vdev_psize - VDEV_LABEL_END_SIZE);
5602294fda888d923d57a78bafdaf48ae6223deaDan Kimmel zio = zio_create(pio, vd->vdev_spa, 0, NULL, data, size, size, done,
5602294fda888d923d57a78bafdaf48ae6223deaDan Kimmel private, ZIO_TYPE_READ, priority, flags | ZIO_FLAG_PHYSICAL, vd,
5602294fda888d923d57a78bafdaf48ae6223deaDan Kimmel offset, NULL, ZIO_STAGE_OPEN, ZIO_READ_PHYS_PIPELINE);
fa9e4066f08beec538e775443c5be79dd423fcabahrenszio_write_phys(zio_t *pio, vdev_t *vd, uint64_t offset, uint64_t size,
fa9e4066f08beec538e775443c5be79dd423fcabahrens void *data, int checksum, zio_done_func_t *done, void *private,
69962b5647e4a8b9b14998733b765925381b727eMatthew Ahrens zio_priority_t priority, enum zio_flag flags, boolean_t labels)
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick ASSERT(!labels || offset + size <= VDEV_LABEL_START_SIZE ||
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick offset >= vd->vdev_psize - VDEV_LABEL_END_SIZE);
5602294fda888d923d57a78bafdaf48ae6223deaDan Kimmel zio = zio_create(pio, vd->vdev_spa, 0, NULL, data, size, size, done,
5602294fda888d923d57a78bafdaf48ae6223deaDan Kimmel private, ZIO_TYPE_WRITE, priority, flags | ZIO_FLAG_PHYSICAL, vd,
5602294fda888d923d57a78bafdaf48ae6223deaDan Kimmel offset, NULL, ZIO_STAGE_OPEN, ZIO_WRITE_PHYS_PIPELINE);
45818ee124adeaaf947698996b4f4c722afc6d1fMatthew Ahrens if (zio_checksum_table[checksum].ci_flags & ZCHECKSUM_FLAG_EMBEDDED) {
6e1f5caa9321646aa4212d48e32a0d241866d85dNeil Perrin * zec checksums are necessarily destructive -- they modify
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick * the end of the write buffer to hold the verifier/checksum.
fa9e4066f08beec538e775443c5be79dd423fcabahrens * Therefore, we must make a local copy in case the data is
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick * being written to multiple places in parallel.
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick zio_push_transform(zio, wbuf, size, size, NULL);
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick * Create a child I/O to do some work for us.
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwickzio_vdev_child_io(zio_t *pio, blkptr_t *bp, vdev_t *vd, uint64_t offset,
dcbf3bd6a1f1360fc1afcee9e22c6dcff7844bf2George Wilson void *data, uint64_t size, int type, zio_priority_t priority,
dcbf3bd6a1f1360fc1afcee9e22c6dcff7844bf2George Wilson enum zio_flag flags, zio_done_func_t *done, void *private)
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick enum zio_stage pipeline = ZIO_VDEV_CHILD_PIPELINE;
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick (pio->io_vd ? pio->io_vd : pio->io_spa->spa_root_vdev));
fa9e4066f08beec538e775443c5be79dd423fcabahrens * If we have the bp, then the child should perform the
fa9e4066f08beec538e775443c5be79dd423fcabahrens * checksum and the parent need not. This pushes error
fa9e4066f08beec538e775443c5be79dd423fcabahrens * detection as close to the leaves as possible and
fa9e4066f08beec538e775443c5be79dd423fcabahrens * eliminates redundant checksums in the interior nodes.
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick pio->io_pipeline &= ~ZIO_STAGE_CHECKSUM_VERIFY;
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick flags |= ZIO_VDEV_CHILD_FLAGS(pio) | ZIO_FLAG_DONT_PROPAGATE;
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick * If we've decided to do a repair, the write is not speculative --
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick * even if the original read was.
0f7643c7376dd69a08acbfc9d1d7d548b10c846aGeorge Wilson * If we're creating a child I/O that is not associated with a
0f7643c7376dd69a08acbfc9d1d7d548b10c846aGeorge Wilson * top-level vdev, then the child zio is not an allocating I/O.
0f7643c7376dd69a08acbfc9d1d7d548b10c846aGeorge Wilson * If this is a retried I/O then we ignore it since we will
0f7643c7376dd69a08acbfc9d1d7d548b10c846aGeorge Wilson * have already processed the original allocating I/O.
0f7643c7376dd69a08acbfc9d1d7d548b10c846aGeorge Wilson (vd != vd->vdev_top || (flags & ZIO_FLAG_IO_RETRY))) {
0f7643c7376dd69a08acbfc9d1d7d548b10c846aGeorge Wilson metaslab_class_t *mc = spa_normal_class(pio->io_spa);
0f7643c7376dd69a08acbfc9d1d7d548b10c846aGeorge Wilson ASSERT(!(pio->io_flags & ZIO_FLAG_IO_REWRITE) ||
5602294fda888d923d57a78bafdaf48ae6223deaDan Kimmel zio = zio_create(pio, pio->io_spa, pio->io_txg, bp, data, size, size,
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick done, private, type, priority, flags, vd, offset, &pio->io_bookmark,
0f7643c7376dd69a08acbfc9d1d7d548b10c846aGeorge Wilson ASSERT3U(zio->io_child_type, ==, ZIO_CHILD_VDEV);
69962b5647e4a8b9b14998733b765925381b727eMatthew Ahrens if (vd->vdev_ops->vdev_op_leaf && zio->io_logical != NULL)
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwickzio_vdev_delegated_io(vdev_t *vd, uint64_t offset, void *data, uint64_t size,
9a686fbc186e8e2a64e9a5094d44c7d6fa0ea167Paul Dagnelie int type, zio_priority_t priority, enum zio_flag flags,
69962b5647e4a8b9b14998733b765925381b727eMatthew Ahrens flags | ZIO_FLAG_CANFAIL | ZIO_FLAG_DONT_RETRY | ZIO_FLAG_DELEGATED,
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick ZIO_STAGE_VDEV_IO_START >> 1, ZIO_VDEV_CHILD_PIPELINE);
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick zio_nowait(zio_ioctl(zio, zio->io_spa, vd, DKIOCFLUSHWRITECACHE,
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick ZIO_FLAG_CANFAIL | ZIO_FLAG_DONT_PROPAGATE | ZIO_FLAG_DONT_RETRY));
6e1f5caa9321646aa4212d48e32a0d241866d85dNeil Perrin * We don't shrink for raidz because of problems with the
6e1f5caa9321646aa4212d48e32a0d241866d85dNeil Perrin * reconstruction when reading back less than the block size.
6e1f5caa9321646aa4212d48e32a0d241866d85dNeil Perrin * Note, BP_IS_RAIDZ() assumes no compression.
6e1f5caa9321646aa4212d48e32a0d241866d85dNeil Perrin ASSERT(BP_GET_COMPRESS(zio->io_bp) == ZIO_COMPRESS_OFF);
5602294fda888d923d57a78bafdaf48ae6223deaDan Kimmel /* we are not doing a raw write */
5602294fda888d923d57a78bafdaf48ae6223deaDan Kimmel zio->io_orig_size = zio->io_size = zio->io_lsize = size;
fa9e4066f08beec538e775443c5be79dd423fcabahrens * ==========================================================================
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick * Prepare to read and write logical blocks
fa9e4066f08beec538e775443c5be79dd423fcabahrens * ==========================================================================
5d7b4d438c4a51eccc95e77a83a437b4d48380ebMatthew Ahrens BP_IS_EMBEDDED(bp) ? BPE_GET_PSIZE(bp) : BP_GET_PSIZE(bp);
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick zio_push_transform(zio, cbuf, psize, psize, zio_decompress);
5d7b4d438c4a51eccc95e77a83a437b4d48380ebMatthew Ahrens if (BP_IS_EMBEDDED(bp) && BPE_GET_ETYPE(bp) == BP_EMBEDDED_TYPE_DATA) {
5d7b4d438c4a51eccc95e77a83a437b4d48380ebMatthew Ahrens decode_embedded_bp_compressed(bp, zio->io_data);
ad135b5d644628e791c3188a6ecbd9c257961ef8Christopher Siden if (!DMU_OT_IS_METADATA(BP_GET_TYPE(bp)) && BP_GET_LEVEL(bp) == 0)
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick if (BP_GET_DEDUP(bp) && zio->io_child_type == ZIO_CHILD_LOGICAL)
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick ASSERT(BP_GET_DEDUP(zio->io_bp_override) == 0);
80901aea8e78a2c20751f61f01bebd1d5b5c2ba5George Wilson * If we've been overridden and nopwrite is set then
80901aea8e78a2c20751f61f01bebd1d5b5c2ba5George Wilson * set the flag accordingly to indicate that a nopwrite
80901aea8e78a2c20751f61f01bebd1d5b5c2ba5George Wilson * has already occurred.
0f7643c7376dd69a08acbfc9d1d7d548b10c846aGeorge Wilson ASSERT3U(BP_GET_CHECKSUM(bp), ==, zp->zp_checksum);
45818ee124adeaaf947698996b4f4c722afc6d1fMatthew Ahrens ASSERT((zio_checksum_table[zp->zp_checksum].ci_flags &
45818ee124adeaaf947698996b4f4c722afc6d1fMatthew Ahrens ZCHECKSUM_FLAG_DEDUP) || zp->zp_dedup_verify);
0f7643c7376dd69a08acbfc9d1d7d548b10c846aGeorge Wilson * We were unable to handle this as an override bp, treat
0f7643c7376dd69a08acbfc9d1d7d548b10c846aGeorge Wilson * it as a regular write I/O.
5602294fda888d923d57a78bafdaf48ae6223deaDan Kimmel EQUIV(lsize != psize, (zio->io_flags & ZIO_FLAG_RAW) != 0);
0f7643c7376dd69a08acbfc9d1d7d548b10c846aGeorge Wilson * If our children haven't all reached the ready stage,
0f7643c7376dd69a08acbfc9d1d7d548b10c846aGeorge Wilson * wait for them and then repeat this pipeline stage.
0f7643c7376dd69a08acbfc9d1d7d548b10c846aGeorge Wilson if (zio_wait_for_children(zio, ZIO_CHILD_GANG, ZIO_WAIT_READY) ||
0f7643c7376dd69a08acbfc9d1d7d548b10c846aGeorge Wilson zio_wait_for_children(zio, ZIO_CHILD_LOGICAL, ZIO_WAIT_READY))
0f7643c7376dd69a08acbfc9d1d7d548b10c846aGeorge Wilson * Now that all our children are ready, run the callback
0f7643c7376dd69a08acbfc9d1d7d548b10c846aGeorge Wilson * associated with this zio in case it wants to modify the
0f7643c7376dd69a08acbfc9d1d7d548b10c846aGeorge Wilson * data to be written.
43466aae47bfcd2ad9bf501faec8e75c08095e4fMax Grossman if (!BP_IS_HOLE(bp) && bp->blk_birth == zio->io_txg) {
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick * We're rewriting an existing block, which means we're
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick * working on behalf of spa_sync(). For spa_sync() to
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick * converge, it must eventually be the case that we don't
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick * have to allocate new blocks. But compression changes
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick * the blocksize, which forces a reallocate, and makes
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick * convergence take longer. Therefore, after the first
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick * few passes, stop compressing to ensure convergence.
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick ASSERT(zio->io_child_type == ZIO_CHILD_LOGICAL);
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick /* Make sure someone doesn't change their mind on overwrites */
5d7b4d438c4a51eccc95e77a83a437b4d48380ebMatthew Ahrens ASSERT(BP_IS_EMBEDDED(bp) || MIN(zp->zp_copies + BP_IS_GANG(bp),
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick spa_max_replication(spa)) == BP_GET_NDVAS(bp));
5602294fda888d923d57a78bafdaf48ae6223deaDan Kimmel /* If it's a compressed write that is not raw, compress the buffer. */
5602294fda888d923d57a78bafdaf48ae6223deaDan Kimmel if (compress != ZIO_COMPRESS_OFF && psize == lsize) {
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick psize = zio_compress_data(compress, zio->io_data, cbuf, lsize);
5d7b4d438c4a51eccc95e77a83a437b4d48380ebMatthew Ahrens } else if (!zp->zp_dedup && psize <= BPE_PAYLOAD_SIZE &&
5d7b4d438c4a51eccc95e77a83a437b4d48380ebMatthew Ahrens zp->zp_level == 0 && !DMU_OT_HAS_FILL(zp->zp_type) &&
5d7b4d438c4a51eccc95e77a83a437b4d48380ebMatthew Ahrens spa_feature_is_enabled(spa, SPA_FEATURE_EMBEDDED_DATA)) {
81cd5c555f505484180a62ca5a2fbb00d70c57d6Matthew Ahrens * Round up compressed size up to the ashift
81cd5c555f505484180a62ca5a2fbb00d70c57d6Matthew Ahrens * of the smallest-ashift device, and zero the tail.
81cd5c555f505484180a62ca5a2fbb00d70c57d6Matthew Ahrens * This ensures that the compressed size of the BP
81cd5c555f505484180a62ca5a2fbb00d70c57d6Matthew Ahrens * (and thus compressratio property) are correct,
81cd5c555f505484180a62ca5a2fbb00d70c57d6Matthew Ahrens * in that we charge for the padding used to fill out
81cd5c555f505484180a62ca5a2fbb00d70c57d6Matthew Ahrens * the last sector.
81cd5c555f505484180a62ca5a2fbb00d70c57d6Matthew Ahrens ASSERT3U(spa->spa_min_ashift, >=, SPA_MINBLOCKSHIFT);
81cd5c555f505484180a62ca5a2fbb00d70c57d6Matthew Ahrens bzero((char *)cbuf + psize, rounded - psize);
0f7643c7376dd69a08acbfc9d1d7d548b10c846aGeorge Wilson * We were unable to handle this as an override bp, treat
0f7643c7376dd69a08acbfc9d1d7d548b10c846aGeorge Wilson * it as a regular write I/O.
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick * The final pass of spa_sync() must be all rewrites, but the first
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick * few passes offer a trade-off: allocating blocks defers convergence,
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick * but newly allocated blocks are sequential, so they can be written
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick * to disk faster. Therefore, we allow the first few passes of
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick * spa_sync() to allocate new blocks, but force rewrites after that.
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick * There should only be a handful of blocks after pass 1 in any case.
43466aae47bfcd2ad9bf501faec8e75c08095e4fMax Grossman if (!BP_IS_HOLE(bp) && bp->blk_birth == zio->io_txg &&
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick enum zio_stage gang_stages = zio->io_pipeline & ZIO_GANG_STAGES;
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick zio->io_pipeline = ZIO_REWRITE_PIPELINE | gang_stages;
43466aae47bfcd2ad9bf501faec8e75c08095e4fMax Grossman spa_feature_is_active(spa, SPA_FEATURE_HOLE_BIRTH)) {
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick ASSERT(zp->zp_checksum != ZIO_CHECKSUM_GANG_HEADER);
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick ASSERT(zio->io_child_type == ZIO_CHILD_LOGICAL);
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick ASSERT(!(zio->io_flags & ZIO_FLAG_IO_REWRITE));
80901aea8e78a2c20751f61f01bebd1d5b5c2ba5George Wilson ASSERT(zio->io_child_type == ZIO_CHILD_LOGICAL);
80901aea8e78a2c20751f61f01bebd1d5b5c2ba5George Wilson ASSERT(!(zio->io_flags & ZIO_FLAG_IO_REWRITE));
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick * ==========================================================================
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick * Execute the I/O pipeline
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick * ==========================================================================
ec94d32216ed5705f5176582355cc311cf848e73Adam Leventhalzio_taskq_dispatch(zio_t *zio, zio_taskq_type_t q, boolean_t cutinline)
bbe36defdfa03da1119804d4ca2d48c1fc6c4ce1George Wilson * If we're a config writer or a probe, the normal issue and
bbe36defdfa03da1119804d4ca2d48c1fc6c4ce1George Wilson * interrupt threads may all be blocked waiting for the config lock.
bbe36defdfa03da1119804d4ca2d48c1fc6c4ce1George Wilson * In this case, select the otherwise-unused taskq for ZIO_TYPE_NULL.
bbe36defdfa03da1119804d4ca2d48c1fc6c4ce1George Wilson if (zio->io_flags & (ZIO_FLAG_CONFIG_WRITER | ZIO_FLAG_PROBE))
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick * A similar issue exists for the L2ARC write thread until L2ARC 2.0.
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick if (t == ZIO_TYPE_WRITE && zio->io_vd && zio->io_vd->vdev_aux)
ec94d32216ed5705f5176582355cc311cf848e73Adam Leventhal * If this is a high priority I/O, then use the high priority taskq if
ec94d32216ed5705f5176582355cc311cf848e73Adam Leventhal spa->spa_zio_taskq[t][q + 1].stqs_count != 0)
5aeb94743e3be0c51e86f73096334611ae3a058eGarrett D'Amore * NB: We are assuming that the zio can only be dispatched
5aeb94743e3be0c51e86f73096334611ae3a058eGarrett D'Amore * to a single taskq at a time. It would be a grievous error
5aeb94743e3be0c51e86f73096334611ae3a058eGarrett D'Amore * to dispatch the zio to another taskq at the same time.
ec94d32216ed5705f5176582355cc311cf848e73Adam Leventhal spa_taskq_dispatch_ent(spa, t, q, (task_func_t *)zio_execute, zio,
ec94d32216ed5705f5176582355cc311cf848e73Adam Leventhalzio_taskq_member(zio_t *zio, zio_taskq_type_t q)
ec94d32216ed5705f5176582355cc311cf848e73Adam Leventhal spa_taskqs_t *tqs = &spa->spa_zio_taskq[t][q];
ec94d32216ed5705f5176582355cc311cf848e73Adam Leventhal if (taskq_member(tqs->stqs_taskq[i], executor))
35a5a3587fd94b666239c157d3722745250ccbd7Jonathan Adams zio_taskq_dispatch(zio, ZIO_TASKQ_ISSUE, B_FALSE);
35a5a3587fd94b666239c157d3722745250ccbd7Jonathan Adams zio_taskq_dispatch(zio, ZIO_TASKQ_INTERRUPT, B_FALSE);
97e81309571898df9fdd94aab1216dfcf23e060bPrakash Surya * The timeout_generic() function isn't defined in userspace, so
97e81309571898df9fdd94aab1216dfcf23e060bPrakash Surya * rather than trying to implement the function, the zio delay
97e81309571898df9fdd94aab1216dfcf23e060bPrakash Surya * functionality has been disabled for userspace builds.
97e81309571898df9fdd94aab1216dfcf23e060bPrakash Surya * If io_target_timestamp is zero, then no delay has been registered
97e81309571898df9fdd94aab1216dfcf23e060bPrakash Surya * for this IO, thus jump to the end of this function and "skip" the
97e81309571898df9fdd94aab1216dfcf23e060bPrakash Surya * delay; issuing it directly to the zio layer.
97e81309571898df9fdd94aab1216dfcf23e060bPrakash Surya * This IO has already taken longer than the target
97e81309571898df9fdd94aab1216dfcf23e060bPrakash Surya * delay to complete, so we don't want to delay it
97e81309571898df9fdd94aab1216dfcf23e060bPrakash Surya * any longer; we "miss" the delay and issue it
97e81309571898df9fdd94aab1216dfcf23e060bPrakash Surya * directly to the zio layer. This is likely due to
97e81309571898df9fdd94aab1216dfcf23e060bPrakash Surya * the target latency being set to a value less than
97e81309571898df9fdd94aab1216dfcf23e060bPrakash Surya * the underlying hardware can satisfy (e.g. delay
97e81309571898df9fdd94aab1216dfcf23e060bPrakash Surya * set to 1ms, but the disks take 10ms to complete an
97e81309571898df9fdd94aab1216dfcf23e060bPrakash Surya * IO request).
97e81309571898df9fdd94aab1216dfcf23e060bPrakash Surya hrtime_t diff = zio->io_target_timestamp - now;
97e81309571898df9fdd94aab1216dfcf23e060bPrakash Surya (void (*)(void *))zio_interrupt, zio, diff, 1, 0);
97e81309571898df9fdd94aab1216dfcf23e060bPrakash Surya DTRACE_PROBE1(zio__delay__skip, zio_t *, zio);
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick * Execute the I/O pipeline until one of the following occurs:
f7170741490edba9d1d9c697c177c887172bc741Will Andrews * (1) the I/O completes
f7170741490edba9d1d9c697c177c887172bc741Will Andrews * (2) the pipeline stalls waiting for dependent child I/Os
f7170741490edba9d1d9c697c177c887172bc741Will Andrews * (3) the I/O issues, so we're waiting for an I/O completion interrupt
f7170741490edba9d1d9c697c177c887172bc741Will Andrews * (4) the I/O is delegated by vdev-level caching or aggregation
f7170741490edba9d1d9c697c177c887172bc741Will Andrews * (5) the I/O is deferred due to vdev-level queueing
f7170741490edba9d1d9c697c177c887172bc741Will Andrews * (6) the I/O is handed off to another thread.
f7170741490edba9d1d9c697c177c887172bc741Will Andrews * In all cases, the pipeline stops whenever there's no CPU work; it never
f7170741490edba9d1d9c697c177c887172bc741Will Andrews * burns a thread in cv_wait().
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick * There's no locking on io_stage because there's no legitimate way
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick * for multiple threads to be attempting to process the same I/O.
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick * If we are in interrupt context and this pipeline stage
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick * will grab a config lock that is held across I/O,
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick * or may wait for an I/O that needs an interrupt thread
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick * to complete, issue async to avoid deadlock.
35a5a3587fd94b666239c157d3722745250ccbd7Jonathan Adams * For VDEV_IO_START, we cut in line so that the io will
35a5a3587fd94b666239c157d3722745250ccbd7Jonathan Adams * be sent to disk promptly.
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick if ((stage & ZIO_BLOCKING_STAGES) && zio->io_vd == NULL &&
35a5a3587fd94b666239c157d3722745250ccbd7Jonathan Adams boolean_t cut = (stage == ZIO_STAGE_VDEV_IO_START) ?
35a5a3587fd94b666239c157d3722745250ccbd7Jonathan Adams zio_taskq_dispatch(zio, ZIO_TASKQ_ISSUE, cut);
bf16b11e8deb633dd6c4296d46e92399d1582df4Matthew Ahrens rv = zio_pipeline[highbit64(stage) - 1](zio);
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick * ==========================================================================
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick * Initiate I/O, either sync or async
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick * ==========================================================================
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick * This is a logical async I/O with no parent to wait for it.
54d692b75b7a6f90ce7787309da5451f7458e66aGeorge Wilson * We add it to the spa_async_root_zio "Godfather" I/O which
54d692b75b7a6f90ce7787309da5451f7458e66aGeorge Wilson * will ensure they complete prior to unloading the pool.
6f834bc197c703a6568554c889157fb345bac079Matthew Ahrens zio_add_child(spa->spa_async_zio_root[CPU_SEQID], zio);
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick * ==========================================================================
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick * Reexecute or suspend/resume failed I/O
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick * ==========================================================================
a3f829ae41ece20e7f5f63604e177aeeb8b24628Bill Moore for (int w = 0; w < ZIO_WAIT_TYPES; w++)
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick for (int c = 0; c < ZIO_CHILD_TYPES; c++)
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick * As we reexecute pio's children, new children could be created.
a3f829ae41ece20e7f5f63604e177aeeb8b24628Bill Moore * New children go to the head of pio's io_child_list, however,
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick * so we will (correctly) not reexecute them. The key is that
a3f829ae41ece20e7f5f63604e177aeeb8b24628Bill Moore * the remainder of pio's io_child_list, from 'cio_next' onward,
a3f829ae41ece20e7f5f63604e177aeeb8b24628Bill Moore * cannot be affected by any side effects of reexecuting 'cio'.
0f7643c7376dd69a08acbfc9d1d7d548b10c846aGeorge Wilson for (cio = zio_walk_children(pio, &zl); cio != NULL; cio = cio_next) {
a3f829ae41ece20e7f5f63604e177aeeb8b24628Bill Moore for (int w = 0; w < ZIO_WAIT_TYPES; w++)
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick * Now that all children have been reexecuted, execute the parent.
54d692b75b7a6f90ce7787309da5451f7458e66aGeorge Wilson * We don't reexecute "The Godfather" I/O here as it's the
54d692b75b7a6f90ce7787309da5451f7458e66aGeorge Wilson * responsibility of the caller to wait on him.
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick if (spa_get_failmode(spa) == ZIO_FAILURE_MODE_PANIC)
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick fm_panic("Pool '%s' has encountered an uncorrectable I/O "
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick "failure and the failure mode property for this pool "
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick zfs_ereport_post(FM_EREPORT_ZFS_IO_FAILURE, spa, NULL, NULL, 0, 0);
54d692b75b7a6f90ce7787309da5451f7458e66aGeorge Wilson spa->spa_suspend_zio_root = zio_root(spa, NULL, NULL,
54d692b75b7a6f90ce7787309da5451f7458e66aGeorge Wilson ASSERT(!(zio->io_flags & ZIO_FLAG_GODFATHER));
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick ASSERT(zio->io_child_type == ZIO_CHILD_LOGICAL);
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick * Reexecute all previously suspended i/o.
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick cv_wait(&spa->spa_suspend_cv, &spa->spa_suspend_lock);
fa9e4066f08beec538e775443c5be79dd423fcabahrens * ==========================================================================
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick * Gang blocks.
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick * A gang block is a collection of small blocks that looks to the DMU
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick * like one large block. When zio_dva_allocate() cannot find a block
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick * of the requested size, due to either severe fragmentation or the pool
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick * being nearly full, it calls zio_write_gang_block() to construct the
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick * block from smaller fragments.
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick * A gang block consists of a gang header (zio_gbh_phys_t) and up to
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick * three (SPA_GBH_NBLKPTRS) gang members. The gang header is just like
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick * an indirect block: it's an array of block pointers. It consumes
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick * only one sector and hence is allocatable regardless of fragmentation.
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick * The gang header's bps point to its gang members, which hold the data.
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick * Gang blocks are self-checksumming, using the bp's <vdev, offset, txg>
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick * as the verifier to ensure uniqueness of the SHA256 checksum.
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick * Critically, the gang block bp's blk_cksum is the checksum of the data,
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick * not the gang header. This ensures that data block signatures (needed for
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick * deduplication) are independent of how the block is physically stored.
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick * Gang blocks can be nested: a gang member may itself be a gang block.
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick * Thus every gang block is a tree in which root and all interior nodes are
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick * gang headers, and the leaves are normal blocks that contain user data.
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick * The root of the gang tree is called the gang leader.
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick * To perform any operation (read, rewrite, free, claim) on a gang block,
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick * zio_gang_assemble() first assembles the gang tree (minus data leaves)
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick * in the io_gang_tree field of the original logical i/o by recursively
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick * reading the gang leader and all gang headers below it. This yields
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick * an in-core tree containing the contents of every gang header and the
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick * bps for every constituent of the gang block.
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick * With the gang tree now assembled, zio_gang_issue() just walks the gang tree
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick * and invokes a callback on each bp. To free a gang block, zio_gang_issue()
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick * calls zio_free_gang() -- a trivial wrapper around zio_free() -- for each bp.
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick * zio_claim_gang() provides a similarly trivial wrapper for zio_claim().
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick * zio_read_gang() is a wrapper around zio_read() that omits reading gang
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick * headers, since we already have those in io_gang_tree. zio_rewrite_gang()
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick * performs a zio_rewrite() of the data or, for gang headers, a zio_rewrite()
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick * of the gang header plus zio_checksum_compute() of the data to update the
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick * gang header's blk_cksum as described above.
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick * The two-phase assemble/issue model solves the problem of partial failure --
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick * what if you'd freed part of a gang block but then couldn't read the
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick * gang header for another part? Assembling the entire gang tree first
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick * ensures that all the necessary gang header I/O has succeeded before
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick * starting the actual work of free, claim, or write. Once the gang tree
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick * is assembled, free and claim are in-memory operations that cannot fail.
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick * In the event that a gang write fails, zio_dva_unallocate() walks the
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick * gang tree to immediately free (i.e. insert back into the space map)
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick * everything we've allocated. This ensures that we don't get ENOSPC
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick * errors during repeated suspend/resume cycles due to a flaky device.
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick * Gang rewrites only happen during sync-to-convergence. If we can't assemble
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick * the gang tree, we won't modify the block, so we can safely defer the free
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick * (knowing that the block is still intact). If we *can* assemble the gang
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick * tree, then even if some of the rewrites fail, zio_dva_unallocate() will free
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick * each constituent bp and we can allocate a new block on the next sync pass.
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick * In all cases, the gang tree allows complete recovery from partial failure.
fa9e4066f08beec538e775443c5be79dd423fcabahrens * ==========================================================================
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwickzio_read_gang(zio_t *pio, blkptr_t *bp, zio_gang_node_t *gn, void *data)
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick return (zio_read(pio, pio->io_spa, bp, data, BP_GET_PSIZE(bp),
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick NULL, NULL, pio->io_priority, ZIO_GANG_CHILD_FLAGS(pio),
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwickzio_rewrite_gang(zio_t *pio, blkptr_t *bp, zio_gang_node_t *gn, void *data)
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick zio = zio_rewrite(pio, pio->io_spa, pio->io_txg, bp,
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick gn->gn_gbh, SPA_GANGBLOCKSIZE, NULL, NULL, pio->io_priority,
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick * As we rewrite each gang header, the pipeline will compute
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick * a new gang block header checksum for it; but no one will
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick * compute a new data checksum, so we do that here. The one
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick * exception is the gang leader: the pipeline already computed
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick * its data checksum because that stage precedes gang assembly.
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick * (Presently, nothing actually uses interior data checksums;
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick * this is just good hygiene.)
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick * If we are here to damage data for testing purposes,
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick * leave the GBH alone so that we can detect the damage.
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick if (pio->io_gang_leader->io_flags & ZIO_FLAG_INDUCE_DAMAGE)
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick zio = zio_rewrite(pio, pio->io_spa, pio->io_txg, bp,
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick data, BP_GET_PSIZE(bp), NULL, NULL, pio->io_priority,
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick/* ARGSUSED */
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwickzio_free_gang(zio_t *pio, blkptr_t *bp, zio_gang_node_t *gn, void *data)
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick return (zio_free_sync(pio, pio->io_spa, pio->io_txg, bp,
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick/* ARGSUSED */
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwickzio_claim_gang(zio_t *pio, blkptr_t *bp, zio_gang_node_t *gn, void *data)
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick return (zio_claim(pio, pio->io_spa, pio->io_txg, bp,
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwickstatic zio_gang_issue_func_t *zio_gang_issue_func[ZIO_TYPES] = {
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwickstatic void zio_gang_tree_assemble_done(zio_t *zio);
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick for (int g = 0; g < SPA_GBH_NBLKPTRS; g++)
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick for (int g = 0; g < SPA_GBH_NBLKPTRS; g++)
f5383399322244b3008a624bcd7e93d67037d177Bill Moorezio_gang_tree_assemble(zio_t *gio, blkptr_t *bp, zio_gang_node_t **gnpp)
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick zio_gang_node_t *gn = zio_gang_node_alloc(gnpp);
f5383399322244b3008a624bcd7e93d67037d177Bill Moore zio_nowait(zio_read(gio, gio->io_spa, bp, gn->gn_gbh,
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick SPA_GANGBLOCKSIZE, zio_gang_tree_assemble_done, gn,
f5383399322244b3008a624bcd7e93d67037d177Bill Moore gio->io_priority, ZIO_GANG_CHILD_FLAGS(gio), &gio->io_bookmark));
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick byteswap_uint64_array(zio->io_data, zio->io_size);
6e1f5caa9321646aa4212d48e32a0d241866d85dNeil Perrin ASSERT(gn->gn_gbh->zg_tail.zec_magic == ZEC_MAGIC);
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick for (int g = 0; g < SPA_GBH_NBLKPTRS; g++) {
f5383399322244b3008a624bcd7e93d67037d177Bill Moore zio_gang_tree_assemble(gio, gbp, &gn->gn_child[g]);
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwickzio_gang_tree_issue(zio_t *pio, zio_gang_node_t *gn, blkptr_t *bp, void *data)
f5383399322244b3008a624bcd7e93d67037d177Bill Moore ASSERT(BP_GET_CHECKSUM(bp) == BP_GET_CHECKSUM(gio->io_bp));
f5383399322244b3008a624bcd7e93d67037d177Bill Moore ASSERT(BP_GET_LSIZE(bp) == BP_GET_PSIZE(bp) || gn == gio->io_gang_tree);
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick * If you're a gang header, your data is in gn->gn_gbh.
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick * If you're a gang member, your data is in 'data' and gn == NULL.
f5383399322244b3008a624bcd7e93d67037d177Bill Moore zio = zio_gang_issue_func[gio->io_type](pio, bp, gn, data);
6e1f5caa9321646aa4212d48e32a0d241866d85dNeil Perrin ASSERT(gn->gn_gbh->zg_tail.zec_magic == ZEC_MAGIC);
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick for (int g = 0; g < SPA_GBH_NBLKPTRS; g++) {
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick zio_gang_tree_issue(zio, gn->gn_child[g], gbp, data);
f5383399322244b3008a624bcd7e93d67037d177Bill Moore ASSERT3P((char *)gio->io_data + gio->io_size, ==, data);
f5383399322244b3008a624bcd7e93d67037d177Bill Moore ASSERT(BP_IS_GANG(bp) && zio->io_gang_leader == NULL);
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick zio_gang_tree_assemble(zio, bp, &zio->io_gang_tree);
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick if (zio_wait_for_children(zio, ZIO_CHILD_GANG, ZIO_WAIT_DONE))
f5383399322244b3008a624bcd7e93d67037d177Bill Moore ASSERT(BP_IS_GANG(bp) && zio->io_gang_leader == zio);
f5383399322244b3008a624bcd7e93d67037d177Bill Moore zio_gang_tree_issue(zio, zio->io_gang_tree, bp, zio->io_data);
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick ASSERT3U(zio->io_prop.zp_copies, ==, gio->io_prop.zp_copies);
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick ASSERT3U(zio->io_prop.zp_copies, <=, BP_GET_NDVAS(zio->io_bp));
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick ASSERT3U(pio->io_prop.zp_copies, <=, BP_GET_NDVAS(pio->io_bp));
44cd46cadd9aab751dae6a4023c1cb5bf316d274billm ASSERT3U(BP_GET_NDVAS(zio->io_bp), <=, BP_GET_NDVAS(pio->io_bp));
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick for (int d = 0; d < BP_GET_NDVAS(zio->io_bp); d++) {
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick int gbh_copies = MIN(copies + 1, spa_max_replication(spa));
0f7643c7376dd69a08acbfc9d1d7d548b10c846aGeorge Wilson int flags = METASLAB_HINTBP_FAVOR | METASLAB_GANG_HEADER;
0f7643c7376dd69a08acbfc9d1d7d548b10c846aGeorge Wilson ASSERT(pio->io_priority == ZIO_PRIORITY_ASYNC_WRITE);
0f7643c7376dd69a08acbfc9d1d7d548b10c846aGeorge Wilson VERIFY(refcount_held(&mc->mc_alloc_slots, pio));
0f7643c7376dd69a08acbfc9d1d7d548b10c846aGeorge Wilson * The logical zio has already placed a reservation for
0f7643c7376dd69a08acbfc9d1d7d548b10c846aGeorge Wilson * 'copies' allocation slots but gang blocks may require
0f7643c7376dd69a08acbfc9d1d7d548b10c846aGeorge Wilson * additional copies. These additional copies
0f7643c7376dd69a08acbfc9d1d7d548b10c846aGeorge Wilson * (i.e. gbh_copies - copies) are guaranteed to succeed
0f7643c7376dd69a08acbfc9d1d7d548b10c846aGeorge Wilson * since metaslab_class_throttle_reserve() always allows
0f7643c7376dd69a08acbfc9d1d7d548b10c846aGeorge Wilson * additional reservations for gang blocks.
0f7643c7376dd69a08acbfc9d1d7d548b10c846aGeorge Wilson VERIFY(metaslab_class_throttle_reserve(mc, gbh_copies - copies,
0f7643c7376dd69a08acbfc9d1d7d548b10c846aGeorge Wilson error = metaslab_alloc(spa, mc, SPA_GANGBLOCKSIZE,
8363e80ae72609660f6090766ca8c2c18aa53f0cGeorge Wilson bp, gbh_copies, txg, pio == gio ? NULL : gio->io_bp, flags,
0f7643c7376dd69a08acbfc9d1d7d548b10c846aGeorge Wilson ASSERT(pio->io_priority == ZIO_PRIORITY_ASYNC_WRITE);
0f7643c7376dd69a08acbfc9d1d7d548b10c846aGeorge Wilson * If we failed to allocate the gang block header then
0f7643c7376dd69a08acbfc9d1d7d548b10c846aGeorge Wilson * we remove any additional allocation reservations that
0f7643c7376dd69a08acbfc9d1d7d548b10c846aGeorge Wilson * we placed here. The original reservation will
0f7643c7376dd69a08acbfc9d1d7d548b10c846aGeorge Wilson * be removed when the logical I/O goes to the ready
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick ASSERT(pio->io_ready == zio_write_gang_member_ready);
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick * Create the gang header.
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick zio = zio_rewrite(pio, spa, txg, bp, gbh, SPA_GANGBLOCKSIZE, NULL, NULL,
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick pio->io_priority, ZIO_GANG_CHILD_FLAGS(pio), &pio->io_bookmark);
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick * Create and nowait the gang children.
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick for (int g = 0; resid != 0; resid -= lsize, g++) {
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick lsize = P2ROUNDUP(resid / (SPA_GBH_NBLKPTRS - g),
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick ASSERT(lsize >= SPA_MINBLOCKSIZE && lsize <= resid);
0f7643c7376dd69a08acbfc9d1d7d548b10c846aGeorge Wilson zio_t *cio = zio_write(zio, spa, txg, &gbh->zg_blkptr[g],
5602294fda888d923d57a78bafdaf48ae6223deaDan Kimmel (char *)pio->io_data + (pio->io_size - resid), lsize, lsize,
5602294fda888d923d57a78bafdaf48ae6223deaDan Kimmel &zp, zio_write_gang_member_ready, NULL, NULL, NULL,
0f7643c7376dd69a08acbfc9d1d7d548b10c846aGeorge Wilson ZIO_GANG_CHILD_FLAGS(pio), &pio->io_bookmark);
0f7643c7376dd69a08acbfc9d1d7d548b10c846aGeorge Wilson ASSERT(pio->io_priority == ZIO_PRIORITY_ASYNC_WRITE);
0f7643c7376dd69a08acbfc9d1d7d548b10c846aGeorge Wilson * Gang children won't throttle but we should
0f7643c7376dd69a08acbfc9d1d7d548b10c846aGeorge Wilson * account for their work, so reserve an allocation
0f7643c7376dd69a08acbfc9d1d7d548b10c846aGeorge Wilson * slot for them here.
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick * Set pio's pipeline to just wait for zio to finish.
45818ee124adeaaf947698996b4f4c722afc6d1fMatthew Ahrens * The zio_nop_write stage in the pipeline determines if allocating a
45818ee124adeaaf947698996b4f4c722afc6d1fMatthew Ahrens * new bp is necessary. The nopwrite feature can handle writes in
45818ee124adeaaf947698996b4f4c722afc6d1fMatthew Ahrens * either syncing or open context (i.e. zil writes) and as a result is
45818ee124adeaaf947698996b4f4c722afc6d1fMatthew Ahrens * mutually exclusive with dedup.
45818ee124adeaaf947698996b4f4c722afc6d1fMatthew Ahrens * By leveraging a cryptographically secure checksum, such as SHA256, we
45818ee124adeaaf947698996b4f4c722afc6d1fMatthew Ahrens * can compare the checksums of the new data and the old to determine if
45818ee124adeaaf947698996b4f4c722afc6d1fMatthew Ahrens * allocating a new block is required. Note that our requirements for
45818ee124adeaaf947698996b4f4c722afc6d1fMatthew Ahrens * cryptographic strength are fairly weak: there can't be any accidental
45818ee124adeaaf947698996b4f4c722afc6d1fMatthew Ahrens * hash collisions, but we don't need to be secure against intentional
45818ee124adeaaf947698996b4f4c722afc6d1fMatthew Ahrens * (malicious) collisions. To trigger a nopwrite, you have to be able
45818ee124adeaaf947698996b4f4c722afc6d1fMatthew Ahrens * to write the file to begin with, and triggering an incorrect (hash
45818ee124adeaaf947698996b4f4c722afc6d1fMatthew Ahrens * collision) nopwrite is no worse than simply writing to the file.
45818ee124adeaaf947698996b4f4c722afc6d1fMatthew Ahrens * That said, there are no known attacks against the checksum algorithms
45818ee124adeaaf947698996b4f4c722afc6d1fMatthew Ahrens * used for nopwrite, assuming that the salt and the checksums
45818ee124adeaaf947698996b4f4c722afc6d1fMatthew Ahrens * themselves remain secret.
80901aea8e78a2c20751f61f01bebd1d5b5c2ba5George Wilson ASSERT(!(zio->io_flags & ZIO_FLAG_IO_REWRITE));
80901aea8e78a2c20751f61f01bebd1d5b5c2ba5George Wilson * Check to see if the original bp and the new bp have matching
80901aea8e78a2c20751f61f01bebd1d5b5c2ba5George Wilson * characteristics (i.e. same checksum, compression algorithms, etc).
80901aea8e78a2c20751f61f01bebd1d5b5c2ba5George Wilson * If they don't then just continue with the pipeline which will
80901aea8e78a2c20751f61f01bebd1d5b5c2ba5George Wilson * allocate a new bp.
45818ee124adeaaf947698996b4f4c722afc6d1fMatthew Ahrens !(zio_checksum_table[BP_GET_CHECKSUM(bp)].ci_flags &
80901aea8e78a2c20751f61f01bebd1d5b5c2ba5George Wilson BP_GET_CHECKSUM(bp) != BP_GET_CHECKSUM(bp_orig) ||
80901aea8e78a2c20751f61f01bebd1d5b5c2ba5George Wilson BP_GET_COMPRESS(bp) != BP_GET_COMPRESS(bp_orig) ||
80901aea8e78a2c20751f61f01bebd1d5b5c2ba5George Wilson * If the checksums match then reset the pipeline so that we
80901aea8e78a2c20751f61f01bebd1d5b5c2ba5George Wilson * avoid allocating a new bp and issuing any I/O.
80901aea8e78a2c20751f61f01bebd1d5b5c2ba5George Wilson if (ZIO_CHECKSUM_EQUAL(bp->blk_cksum, bp_orig->blk_cksum)) {
45818ee124adeaaf947698996b4f4c722afc6d1fMatthew Ahrens ASSERT(zio_checksum_table[zp->zp_checksum].ci_flags &
80901aea8e78a2c20751f61f01bebd1d5b5c2ba5George Wilson ASSERT3U(BP_GET_PSIZE(bp), ==, BP_GET_PSIZE(bp_orig));
80901aea8e78a2c20751f61f01bebd1d5b5c2ba5George Wilson ASSERT3U(BP_GET_LSIZE(bp), ==, BP_GET_LSIZE(bp_orig));
80901aea8e78a2c20751f61f01bebd1d5b5c2ba5George Wilson ASSERT(bcmp(&bp->blk_prop, &bp_orig->blk_prop,
80901aea8e78a2c20751f61f01bebd1d5b5c2ba5George Wilson sizeof (uint64_t)) == 0);
fa9e4066f08beec538e775443c5be79dd423fcabahrens * ==========================================================================
fa9e4066f08beec538e775443c5be79dd423fcabahrens * ==========================================================================
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick ddt_phys_clear(ddp); /* this ddp doesn't need repair */
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick if (zio->io_error == 0 && dde->dde_repair_data == NULL)
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick ASSERT(zio->io_child_type == ZIO_CHILD_LOGICAL);
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick ddt_phys_t *ddp_self = ddt_phys_select(dde, bp);
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick for (int p = 0; p < DDT_PHYS_TYPES; p++, ddp++) {
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick if (ddp->ddp_phys_birth == 0 || ddp == ddp_self)
bbfd46c40e81c7d954cec28db66453ec5ab44613Jeff Bonwick ddt_bp_create(ddt->ddt_checksum, &dde->dde_key, ddp,
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick zio_ddt_child_read_done, dde, zio->io_priority,
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick ZIO_DDT_CHILD_FLAGS(zio) | ZIO_FLAG_DONT_PROPAGATE,
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick zio->io_data, zio->io_size, NULL, NULL, zio->io_priority,
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick if (zio_wait_for_children(zio, ZIO_CHILD_DDT, ZIO_WAIT_DONE))
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick ASSERT(zio->io_child_type == ZIO_CHILD_LOGICAL);
b16da2e29e074fb6eaeadc4fd7d17ae7340ba240George Wilson ASSERT(spa_load_state(zio->io_spa) != SPA_LOAD_NONE);
35a5a3587fd94b666239c157d3722745250ccbd7Jonathan Adams zio_taskq_dispatch(zio, ZIO_TASKQ_ISSUE, B_FALSE);
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick bcopy(dde->dde_repair_data, zio->io_data, zio->io_size);
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwickzio_ddt_collision(zio_t *zio, ddt_t *ddt, ddt_entry_t *dde)
5602294fda888d923d57a78bafdaf48ae6223deaDan Kimmel boolean_t do_raw = (zio->io_flags & ZIO_FLAG_RAW);
5602294fda888d923d57a78bafdaf48ae6223deaDan Kimmel /* We should never get a raw, override zio */
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick * Note: we compare the original data, not the transformed data,
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick * because when zio->io_bp is an override bp, we will not have
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick * pushed the I/O transforms. That's an important optimization
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick * because otherwise we'd compress/encrypt all dmu_sync() data twice.
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick for (int p = DDT_PHYS_SINGLE; p <= DDT_PHYS_TRIPLE; p++) {
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick return (lio->io_orig_size != zio->io_orig_size ||
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick for (int p = DDT_PHYS_SINGLE; p <= DDT_PHYS_TRIPLE; p++) {
5602294fda888d923d57a78bafdaf48ae6223deaDan Kimmel int zio_flags = ZIO_FLAG_CANFAIL | ZIO_FLAG_SPECULATIVE;
5602294fda888d923d57a78bafdaf48ae6223deaDan Kimmel * Intuitively, it would make more sense to compare
5602294fda888d923d57a78bafdaf48ae6223deaDan Kimmel * io_data than io_orig_data in the raw case since you
5602294fda888d923d57a78bafdaf48ae6223deaDan Kimmel * don't want to look at any transformations that have
5602294fda888d923d57a78bafdaf48ae6223deaDan Kimmel * happened to the data. However, for raw I/Os the
5602294fda888d923d57a78bafdaf48ae6223deaDan Kimmel * data will actually be the same in io_data and
5602294fda888d923d57a78bafdaf48ae6223deaDan Kimmel * io_orig_data, so all we have to do is issue this as
5602294fda888d923d57a78bafdaf48ae6223deaDan Kimmel * a raw ARC read.
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick arc_getbuf_func, &abuf, ZIO_PRIORITY_SYNC_READ,
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick return (error != 0);
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick ddt_t *ddt = ddt_select(zio->io_spa, zio->io_bp);
0f7643c7376dd69a08acbfc9d1d7d548b10c846aGeorge Wilson while ((pio = zio_walk_parents(zio, &zl)) != NULL)
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick ddt_t *ddt = ddt_select(zio->io_spa, zio->io_bp);
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick ASSERT(ZIO_CHECKSUM_EQUAL(bp->blk_cksum, ddk->ddk_cksum));
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick ASSERT(zp->zp_copies == BP_GET_NDVAS(bp) - BP_IS_GANG(bp));
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick ASSERT(BP_GET_CHECKSUM(bp) == zp->zp_checksum);
5602294fda888d923d57a78bafdaf48ae6223deaDan Kimmel ASSERT(!(zio->io_bp_override && (zio->io_flags & ZIO_FLAG_RAW)));
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick if (zp->zp_dedup_verify && zio_ddt_collision(zio, ddt, dde)) {
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick * If we're using a weak checksum, upgrade to a strong checksum
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick * and try again. If we're already using a strong checksum,
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick * we can't resolve it, so just convert to an ordinary write.
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick * (And automatically e-mail a paper to Nature?)
45818ee124adeaaf947698996b4f4c722afc6d1fMatthew Ahrens if (!(zio_checksum_table[zp->zp_checksum].ci_flags &
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick ditto_copies = ddt_ditto_copies_needed(ddt, dde, ddp);
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick if (ditto_copies > ddt_ditto_copies_present(dde) &&
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick * If we arrived here with an override bp, we won't have run
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick * the transform stack, so we won't have the data we need to
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick * generate a child i/o. So, toss the override bp and restart.
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick * This is safe, because using the override bp is just an
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick * optimization; and it's rare, so the cost doesn't matter.
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick dio = zio_write(zio, spa, txg, bp, zio->io_orig_data,
5602294fda888d923d57a78bafdaf48ae6223deaDan Kimmel zio->io_orig_size, zio->io_orig_size, &czp, NULL, NULL,
8df0bcf0df7622a075cc6e52f659d2fcfdd08cdcPaul Dagnelie NULL, zio_ddt_ditto_write_done, dde, zio->io_priority,
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick zio_push_transform(dio, zio->io_data, zio->io_size, 0, NULL);
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick if (ddp->ddp_phys_birth != 0 || dde->dde_lead_zio[p] != NULL) {
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick cio = zio_write(zio, spa, txg, bp, zio->io_orig_data,
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick zio_ddt_child_write_done, dde, zio->io_priority,
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick zio_push_transform(cio, zio->io_data, zio->io_size, 0, NULL);
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick ASSERT(zio->io_child_type == ZIO_CHILD_LOGICAL);
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick * ==========================================================================
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick * Allocate and free blocks
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick * ==========================================================================
0f7643c7376dd69a08acbfc9d1d7d548b10c846aGeorge Wilson * Try to place a reservation for this zio. If we're unable to
0f7643c7376dd69a08acbfc9d1d7d548b10c846aGeorge Wilson * reserve then we throttle.
0f7643c7376dd69a08acbfc9d1d7d548b10c846aGeorge Wilson if (!metaslab_class_throttle_reserve(spa_normal_class(spa),
0f7643c7376dd69a08acbfc9d1d7d548b10c846aGeorge Wilson ASSERT3U(zio->io_stage, <, ZIO_STAGE_DVA_ALLOCATE);
0f7643c7376dd69a08acbfc9d1d7d548b10c846aGeorge Wilson if (zio->io_priority == ZIO_PRIORITY_SYNC_WRITE ||
0f7643c7376dd69a08acbfc9d1d7d548b10c846aGeorge Wilson !spa_normal_class(zio->io_spa)->mc_alloc_throttle_enabled ||
0f7643c7376dd69a08acbfc9d1d7d548b10c846aGeorge Wilson ASSERT(zio->io_stage == ZIO_STAGE_DVA_THROTTLE);
0f7643c7376dd69a08acbfc9d1d7d548b10c846aGeorge Wilson ASSERT(nio->io_stage == ZIO_STAGE_DVA_THROTTLE);
0f7643c7376dd69a08acbfc9d1d7d548b10c846aGeorge Wilson * We are passing control to a new zio so make sure that
0f7643c7376dd69a08acbfc9d1d7d548b10c846aGeorge Wilson * it is processed by a different thread. We do this to
0f7643c7376dd69a08acbfc9d1d7d548b10c846aGeorge Wilson * avoid stack overflows that can occur when parents are
0f7643c7376dd69a08acbfc9d1d7d548b10c846aGeorge Wilson * throttled and children are making progress. We allow
0f7643c7376dd69a08acbfc9d1d7d548b10c846aGeorge Wilson * it to go to the head of the taskq since it's already
0f7643c7376dd69a08acbfc9d1d7d548b10c846aGeorge Wilson * been waiting.
0f7643c7376dd69a08acbfc9d1d7d548b10c846aGeorge Wilson zio_taskq_dispatch(nio, ZIO_TASKQ_ISSUE, B_TRUE);
0f7643c7376dd69a08acbfc9d1d7d548b10c846aGeorge Wilson ASSERT3U(zio->io_stage, ==, ZIO_STAGE_DVA_THROTTLE);
0f7643c7376dd69a08acbfc9d1d7d548b10c846aGeorge Wilson zio_taskq_dispatch(zio, ZIO_TASKQ_ISSUE, B_TRUE);
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick ASSERT3U(zio->io_prop.zp_copies, <=, spa_max_replication(spa));
0f7643c7376dd69a08acbfc9d1d7d548b10c846aGeorge Wilson if (zio->io_priority == ZIO_PRIORITY_ASYNC_WRITE) {
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick error = metaslab_alloc(spa, mc, zio->io_size, bp,
8363e80ae72609660f6090766ca8c2c18aa53f0cGeorge Wilson zio->io_prop.zp_copies, zio->io_txg, NULL, flags,
09c9d376e8ccb8fbba74f33cc268964464092b62George Wilson spa_dbgmsg(spa, "%s: metaslab allocation failure: zio %p, "
09c9d376e8ccb8fbba74f33cc268964464092b62George Wilson "size %llu, error %d", spa_name(spa), zio, zio->io_size,
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick if (error == ENOSPC && zio->io_size > SPA_MINBLOCKSIZE)
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick metaslab_free(zio->io_spa, zio->io_bp, zio->io_txg, B_FALSE);
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick error = metaslab_claim(zio->io_spa, zio->io_bp, zio->io_txg);
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick * Undo an allocation. This is used by zio_done() when an I/O fails
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick * and we want to give back the block we just allocated.
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick * This handles both normal blocks and gang blocks.
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwickzio_dva_unallocate(zio_t *zio, zio_gang_node_t *gn, blkptr_t *bp)
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick ASSERT(bp->blk_birth == zio->io_txg || BP_IS_HOLE(bp));
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick metaslab_free(zio->io_spa, bp, bp->blk_birth, B_TRUE);
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick for (int g = 0; g < SPA_GBH_NBLKPTRS; g++) {
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick * Try to allocate an intent log block. Return 0 on success, errno on failure.
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwickzio_alloc_zil(spa_t *spa, uint64_t txg, blkptr_t *new_bp, blkptr_t *old_bp,
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick error = metaslab_alloc(spa, spa_log_class(spa), size,
8363e80ae72609660f6090766ca8c2c18aa53f0cGeorge Wilson new_bp, 1, txg, old_bp, METASLAB_HINTBP_AVOID,
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick error = metaslab_alloc(spa, spa_normal_class(spa), size,
8363e80ae72609660f6090766ca8c2c18aa53f0cGeorge Wilson new_bp, 1, txg, old_bp, METASLAB_HINTBP_AVOID,
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick * Free an intent log block.
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwickzio_free_zil(spa_t *spa, uint64_t txg, blkptr_t *bp)
fa9e4066f08beec538e775443c5be79dd423fcabahrens * ==========================================================================
fa9e4066f08beec538e775443c5be79dd423fcabahrens * Read and write to physical devices
fa9e4066f08beec538e775443c5be79dd423fcabahrens * ==========================================================================
738f37bc3dcd61e8a893af0f2d466d76690b70ecGeorge Wilson * Issue an I/O to the underlying vdev. Typically the issue pipeline
738f37bc3dcd61e8a893af0f2d466d76690b70ecGeorge Wilson * stops after this stage and will resume upon I/O completion.
738f37bc3dcd61e8a893af0f2d466d76690b70ecGeorge Wilson * However, there are instances where the vdev layer may need to
738f37bc3dcd61e8a893af0f2d466d76690b70ecGeorge Wilson * continue the pipeline when an I/O was not issued. Since the I/O
738f37bc3dcd61e8a893af0f2d466d76690b70ecGeorge Wilson * that was sent to the vdev layer might be different than the one
738f37bc3dcd61e8a893af0f2d466d76690b70ecGeorge Wilson * currently active in the pipeline (see vdev_queue_io()), we explicitly
738f37bc3dcd61e8a893af0f2d466d76690b70ecGeorge Wilson * force the underlying vdev layers to call either zio_execute() or
738f37bc3dcd61e8a893af0f2d466d76690b70ecGeorge Wilson * zio_interrupt() to ensure that the pipeline continues with the correct I/O.
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick ASSERT(zio->io_child_error[ZIO_CHILD_VDEV] == 0);
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick spa_config_enter(spa, SCL_ZIO, zio, RW_READER);
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick * The mirror_ops handle multiple DVAs in a single BP.
44ecc5327ab4ce0750dcca2a17e05566bf2812e2George Wilson * We keep track of time-sensitive I/Os so that the scan thread
44ecc5327ab4ce0750dcca2a17e05566bf2812e2George Wilson * can quickly react to certain workloads. In particular, we care
44ecc5327ab4ce0750dcca2a17e05566bf2812e2George Wilson * about non-scrubbing, top-level reads and writes with the following
44ecc5327ab4ce0750dcca2a17e05566bf2812e2George Wilson * characteristics:
738f37bc3dcd61e8a893af0f2d466d76690b70ecGeorge Wilson * - synchronous writes of user data to non-slog devices
44ecc5327ab4ce0750dcca2a17e05566bf2812e2George Wilson * - any reads of user data
44ecc5327ab4ce0750dcca2a17e05566bf2812e2George Wilson * When these conditions are met, adjust the timestamp of spa_last_io
44ecc5327ab4ce0750dcca2a17e05566bf2812e2George Wilson * which allows the scan thread to adjust its workload accordingly.
44ecc5327ab4ce0750dcca2a17e05566bf2812e2George Wilson if (!(zio->io_flags & ZIO_FLAG_SCAN_THREAD) && zio->io_bp != NULL &&
44ecc5327ab4ce0750dcca2a17e05566bf2812e2George Wilson zio->io_bookmark.zb_objset != DMU_META_OBJSET &&
44ecc5327ab4ce0750dcca2a17e05566bf2812e2George Wilson (void) atomic_cas_64(&spa->spa_last_io, old, new);
2a104a5236475eb73aa41eaaf3ed9f3ccbe0ca55Alex Reece /* Transform logical writes to be a full physical block size. */
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick zio_push_transform(zio, abuf, asize, asize, zio_subblock);
2a104a5236475eb73aa41eaaf3ed9f3ccbe0ca55Alex Reece * If this is not a physical io, make sure that it is properly aligned
2a104a5236475eb73aa41eaaf3ed9f3ccbe0ca55Alex Reece * before proceeding.
2a104a5236475eb73aa41eaaf3ed9f3ccbe0ca55Alex Reece * For physical writes, we allow 512b aligned writes and assume
2a104a5236475eb73aa41eaaf3ed9f3ccbe0ca55Alex Reece * the device will perform a read-modify-write as necessary.
2a104a5236475eb73aa41eaaf3ed9f3ccbe0ca55Alex Reece ASSERT0(P2PHASE(zio->io_offset, SPA_MINBLOCKSIZE));
2a104a5236475eb73aa41eaaf3ed9f3ccbe0ca55Alex Reece ASSERT0(P2PHASE(zio->io_size, SPA_MINBLOCKSIZE));
f9af39bacaaa0f9dda3b75ff6858b9f3988a39afGeorge Wilson VERIFY(zio->io_type != ZIO_TYPE_WRITE || spa_writeable(spa));
8ad4d6dd86f5bc65fb3afa566c8133f3bac21648Jeff Bonwick * If this is a repair I/O, and there's no self-healing involved --
8ad4d6dd86f5bc65fb3afa566c8133f3bac21648Jeff Bonwick * that is, we're just resilvering what we expect to resilver --
8ad4d6dd86f5bc65fb3afa566c8133f3bac21648Jeff Bonwick * then don't do the I/O unless zio's txg is actually in vd's DTL.
8ad4d6dd86f5bc65fb3afa566c8133f3bac21648Jeff Bonwick * This prevents spurious resilvering with nested replication.
8ad4d6dd86f5bc65fb3afa566c8133f3bac21648Jeff Bonwick * For example, given a mirror of mirrors, (A+B)+(C+D), if only
8ad4d6dd86f5bc65fb3afa566c8133f3bac21648Jeff Bonwick * A is out of date, we'll read from C+D, then use the data to
8ad4d6dd86f5bc65fb3afa566c8133f3bac21648Jeff Bonwick * resilver A+B -- but we don't actually want to resilver B, just A.
8ad4d6dd86f5bc65fb3afa566c8133f3bac21648Jeff Bonwick * The top-level mirror has no way to know this, so instead we just
8ad4d6dd86f5bc65fb3afa566c8133f3bac21648Jeff Bonwick * discard unnecessary repairs as we work our way down the vdev tree.
8ad4d6dd86f5bc65fb3afa566c8133f3bac21648Jeff Bonwick * The same logic applies to any form of nested replication:
8ad4d6dd86f5bc65fb3afa566c8133f3bac21648Jeff Bonwick * ditto + mirror, RAID-Z + replacing, etc. This covers them all.
8ad4d6dd86f5bc65fb3afa566c8133f3bac21648Jeff Bonwick !vdev_dtl_contains(vd, DTL_PARTIAL, zio->io_txg, 1)) {
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick (zio->io_type == ZIO_TYPE_READ || zio->io_type == ZIO_TYPE_WRITE)) {
43466aae47bfcd2ad9bf501faec8e75c08095e4fMax Grossman if (zio->io_type == ZIO_TYPE_READ && vdev_cache_read(zio))
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick vdev_ops_t *ops = vd ? vd->vdev_ops : &vdev_mirror_ops;
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick if (zio_wait_for_children(zio, ZIO_CHILD_VDEV, ZIO_WAIT_DONE))
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick ASSERT(zio->io_type == ZIO_TYPE_READ || zio->io_type == ZIO_TYPE_WRITE);
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick if (vd != NULL && vd->vdev_ops->vdev_op_leaf) {
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick if (zio_injection_enabled && zio->io_error == 0)
8956713aded83a741173fcd4f9ef1c83521fbea9Eric Schrock zio->io_error = zio_handle_device_injection(vd,
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick if (zio_injection_enabled && zio->io_error == 0)
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick zio->io_error = zio_handle_label_injection(zio, EIO);
22fe2c8844be88ebae6478ca1b0b92c8ec2aef54Jonathan Adams * For non-raidz ZIOs, we can just copy aside the bad data read from the
22fe2c8844be88ebae6478ca1b0b92c8ec2aef54Jonathan Adams * disk, and use that to finish the checksum ereport later.
22fe2c8844be88ebae6478ca1b0b92c8ec2aef54Jonathan Adamszio_vsd_default_cksum_finish(zio_cksum_report_t *zcr,
22fe2c8844be88ebae6478ca1b0b92c8ec2aef54Jonathan Adams /* no processing needed */
22fe2c8844be88ebae6478ca1b0b92c8ec2aef54Jonathan Adams zfs_ereport_finish_checksum(zcr, good_buf, zcr->zcr_cbdata, B_FALSE);
22fe2c8844be88ebae6478ca1b0b92c8ec2aef54Jonathan Adamszio_vsd_default_cksum_report(zio_t *zio, zio_cksum_report_t *zcr, void *ignored)
22fe2c8844be88ebae6478ca1b0b92c8ec2aef54Jonathan Adams zcr->zcr_finish = zio_vsd_default_cksum_finish;
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick if (zio_wait_for_children(zio, ZIO_CHILD_VDEV, ZIO_WAIT_DONE))
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick if (vd == NULL && !(zio->io_flags & ZIO_FLAG_CONFIG_WRITER))
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick if (zio_injection_enabled && zio->io_error == 0)
ea8dc4b6d2251b437950c0056bc626b311c73c27eschrock zio->io_error = zio_handle_fault_injection(zio, EIO);
fa9e4066f08beec538e775443c5be79dd423fcabahrens * If the I/O failed, determine whether we should attempt to retry it.
35a5a3587fd94b666239c157d3722745250ccbd7Jonathan Adams * On retry, we cut in line in the issue queue, since we don't want
35a5a3587fd94b666239c157d3722745250ccbd7Jonathan Adams * compression/checksumming/etc. work to prevent our (cheap) IO reissue.
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick !(zio->io_flags & (ZIO_FLAG_DONT_RETRY | ZIO_FLAG_IO_RETRY))) {
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick ASSERT(!(zio->io_flags & ZIO_FLAG_DONT_QUEUE)); /* not a leaf */
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick ASSERT(!(zio->io_flags & ZIO_FLAG_IO_BYPASS)); /* not a leaf */
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick * If we got an error on a leaf device, convert it to ENXIO
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick * if the device is not accessible at all.
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick if (zio->io_error && vd != NULL && vd->vdev_ops->vdev_op_leaf &&
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick * If we can't write to an interior vdev (mirror or RAID-Z),
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick * set vdev_cant_write so that we stop trying to allocate from it.
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick if (zio->io_error == ENXIO && zio->io_type == ZIO_TYPE_WRITE &&
295438ba3230419314faaa889a2616f561658bd5Hans Rosenfeld * If a cache flush returns ENOTSUP or ENOTTY, we know that no future
295438ba3230419314faaa889a2616f561658bd5Hans Rosenfeld * attempts will ever succeed. In this case we set a persistent bit so
295438ba3230419314faaa889a2616f561658bd5Hans Rosenfeld * that we don't bother with it in the future.
295438ba3230419314faaa889a2616f561658bd5Hans Rosenfeld if ((zio->io_error == ENOTSUP || zio->io_error == ENOTTY) &&
295438ba3230419314faaa889a2616f561658bd5Hans Rosenfeld zio->io_cmd == DKIOCFLUSHWRITECACHE && vd != NULL)
69962b5647e4a8b9b14998733b765925381b727eMatthew Ahrens if (vd != NULL && vd->vdev_ops->vdev_op_leaf &&
69962b5647e4a8b9b14998733b765925381b727eMatthew Ahrens ASSERT(!(zio->io_flags & ZIO_FLAG_DELEGATED));
69962b5647e4a8b9b14998733b765925381b727eMatthew Ahrens ASSERT(zio->io_child_type == ZIO_CHILD_VDEV);
fa9e4066f08beec538e775443c5be79dd423fcabahrens * ==========================================================================
fa9e4066f08beec538e775443c5be79dd423fcabahrens * Generate and verify checksums
fa9e4066f08beec538e775443c5be79dd423fcabahrens * ==========================================================================
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick * This is zio_write_phys().
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick * We're either generating a label checksum, or none at all.
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick if (BP_IS_GANG(bp) && zio->io_child_type == ZIO_CHILD_GANG) {
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick zio_checksum_compute(zio, checksum, zio->io_data, zio->io_size);
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick * This is zio_read_phys().
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick * We're either verifying a label checksum, or nothing at all.
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick if (zio->io_prop.zp_checksum == ZIO_CHECKSUM_OFF)
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick ASSERT(zio->io_prop.zp_checksum == ZIO_CHECKSUM_LABEL);
22fe2c8844be88ebae6478ca1b0b92c8ec2aef54Jonathan Adams if ((error = zio_checksum_error(zio, &info)) != 0) {
fa9e4066f08beec538e775443c5be79dd423fcabahrens * Called by RAID-Z to ensure we don't compute the checksum twice.
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick zio->io_pipeline &= ~ZIO_STAGE_CHECKSUM_VERIFY;
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick * ==========================================================================
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick * Error rank. Error are ranked in the order 0, ENXIO, ECKSUM, EIO, other.
5d7b4d438c4a51eccc95e77a83a437b4d48380ebMatthew Ahrens * An error of 0 indicates success. ENXIO indicates whole-device failure,
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick * which may be transient (e.g. unplugged) or permament. ECKSUM and EIO
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick * indicate errors that are specific to one I/O, and most likely permanent.
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick * Any other error is presumed to be worse because we weren't expecting it.
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick * ==========================================================================
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick static int zio_error_rank[] = { 0, ENXIO, ECKSUM, EIO };
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick for (r1 = 0; r1 < sizeof (zio_error_rank) / sizeof (int); r1++)
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick for (r2 = 0; r2 < sizeof (zio_error_rank) / sizeof (int); r2++)
fa9e4066f08beec538e775443c5be79dd423fcabahrens * ==========================================================================
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick * I/O completion
fa9e4066f08beec538e775443c5be79dd423fcabahrens * ==========================================================================
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick if (zio_wait_for_children(zio, ZIO_CHILD_GANG, ZIO_WAIT_READY) ||
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick zio_wait_for_children(zio, ZIO_CHILD_DDT, ZIO_WAIT_READY))
80901aea8e78a2c20751f61f01bebd1d5b5c2ba5George Wilson ASSERT(bp->blk_birth == zio->io_txg || BP_IS_HOLE(bp) ||
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick ASSERT(zio->io_children[ZIO_CHILD_GANG][ZIO_WAIT_READY] == 0);
0f7643c7376dd69a08acbfc9d1d7d548b10c846aGeorge Wilson ASSERT(zio->io_priority == ZIO_PRIORITY_ASYNC_WRITE);
0f7643c7376dd69a08acbfc9d1d7d548b10c846aGeorge Wilson * We were unable to allocate anything, unreserve and
0f7643c7376dd69a08acbfc9d1d7d548b10c846aGeorge Wilson * issue the next I/O to allocate.
a3f829ae41ece20e7f5f63604e177aeeb8b24628Bill Moore * As we notify zio's parents, new parents could be added.
a3f829ae41ece20e7f5f63604e177aeeb8b24628Bill Moore * New parents go to the head of zio's io_parent_list, however,
a3f829ae41ece20e7f5f63604e177aeeb8b24628Bill Moore * so we will (correctly) not notify them. The remainder of zio's
a3f829ae41ece20e7f5f63604e177aeeb8b24628Bill Moore * io_parent_list, from 'pio_next' onward, cannot change because
a3f829ae41ece20e7f5f63604e177aeeb8b24628Bill Moore * all parents must wait for us to be done before they can be done.
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick ASSERT((uintptr_t)zio->io_data < SPA_MAXBLOCKSIZE);
0f7643c7376dd69a08acbfc9d1d7d548b10c846aGeorge Wilson * Update the allocation throttle accounting.
0f7643c7376dd69a08acbfc9d1d7d548b10c846aGeorge Wilson ASSERT3U(zio->io_priority, ==, ZIO_PRIORITY_ASYNC_WRITE);
0f7643c7376dd69a08acbfc9d1d7d548b10c846aGeorge Wilson ASSERT3U(zio->io_child_type, ==, ZIO_CHILD_VDEV);
0f7643c7376dd69a08acbfc9d1d7d548b10c846aGeorge Wilson ASSERT(!(zio->io_flags & (ZIO_FLAG_IO_REPAIR | ZIO_FLAG_IO_RETRY)));
0f7643c7376dd69a08acbfc9d1d7d548b10c846aGeorge Wilson ASSERT(zio->io_flags & ZIO_FLAG_IO_ALLOCATING);
0f7643c7376dd69a08acbfc9d1d7d548b10c846aGeorge Wilson ASSERT(!(lio->io_flags & ZIO_FLAG_IO_REWRITE));
0f7643c7376dd69a08acbfc9d1d7d548b10c846aGeorge Wilson ASSERT(!(lio->io_orig_flags & ZIO_FLAG_NODATA));
0f7643c7376dd69a08acbfc9d1d7d548b10c846aGeorge Wilson * Parents of gang children can have two flavors -- ones that
0f7643c7376dd69a08acbfc9d1d7d548b10c846aGeorge Wilson * allocated the gang header (will have ZIO_FLAG_IO_REWRITE set)
0f7643c7376dd69a08acbfc9d1d7d548b10c846aGeorge Wilson * and ones that allocated the constituent blocks. The allocation
0f7643c7376dd69a08acbfc9d1d7d548b10c846aGeorge Wilson * throttle needs to know the allocating parent zio so we must find
0f7643c7376dd69a08acbfc9d1d7d548b10c846aGeorge Wilson * If our parent is a rewrite gang child then our grandparent
0f7643c7376dd69a08acbfc9d1d7d548b10c846aGeorge Wilson * would have been the one that performed the allocation.
0f7643c7376dd69a08acbfc9d1d7d548b10c846aGeorge Wilson ASSERT(!(zio->io_flags & ZIO_FLAG_IO_REPAIR));
0f7643c7376dd69a08acbfc9d1d7d548b10c846aGeorge Wilson metaslab_group_alloc_decrement(zio->io_spa, vd->vdev_id, pio, flags);
0f7643c7376dd69a08acbfc9d1d7d548b10c846aGeorge Wilson metaslab_class_throttle_unreserve(spa_normal_class(zio->io_spa),
0f7643c7376dd69a08acbfc9d1d7d548b10c846aGeorge Wilson * Call into the pipeline to see if there is more work that
0f7643c7376dd69a08acbfc9d1d7d548b10c846aGeorge Wilson * needs to be done. If there is work to be done it will be
0f7643c7376dd69a08acbfc9d1d7d548b10c846aGeorge Wilson * dispatched to another taskq thread.
f5383399322244b3008a624bcd7e93d67037d177Bill Moore * If our children haven't all completed,
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick * wait for them and then repeat this pipeline stage.
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick if (zio_wait_for_children(zio, ZIO_CHILD_VDEV, ZIO_WAIT_DONE) ||
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick zio_wait_for_children(zio, ZIO_CHILD_GANG, ZIO_WAIT_DONE) ||
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick zio_wait_for_children(zio, ZIO_CHILD_DDT, ZIO_WAIT_DONE) ||
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick zio_wait_for_children(zio, ZIO_CHILD_LOGICAL, ZIO_WAIT_DONE))
0f7643c7376dd69a08acbfc9d1d7d548b10c846aGeorge Wilson * If the allocation throttle is enabled, then update the accounting.
0f7643c7376dd69a08acbfc9d1d7d548b10c846aGeorge Wilson * We only track child I/Os that are part of an allocating async
0f7643c7376dd69a08acbfc9d1d7d548b10c846aGeorge Wilson * write. We must do this since the allocation is performed
0f7643c7376dd69a08acbfc9d1d7d548b10c846aGeorge Wilson * by the logical I/O but the actual write is done by child I/Os.
0f7643c7376dd69a08acbfc9d1d7d548b10c846aGeorge Wilson * If the allocation throttle is enabled, verify that
0f7643c7376dd69a08acbfc9d1d7d548b10c846aGeorge Wilson * we have decremented the refcounts for every I/O that was throttled.
0f7643c7376dd69a08acbfc9d1d7d548b10c846aGeorge Wilson ASSERT(zio->io_priority == ZIO_PRIORITY_ASYNC_WRITE);
0f7643c7376dd69a08acbfc9d1d7d548b10c846aGeorge Wilson metaslab_group_alloc_verify(spa, zio->io_bp, zio);
0f7643c7376dd69a08acbfc9d1d7d548b10c846aGeorge Wilson VERIFY(refcount_not_held(&mc->mc_alloc_slots, zio));
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick for (int c = 0; c < ZIO_CHILD_TYPES; c++)
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick for (int w = 0; w < ZIO_WAIT_TYPES; w++)
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick ASSERT(bcmp(bp, &zio->io_bp_copy, sizeof (blkptr_t)) == 0 ||
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick if (zio->io_type == ZIO_TYPE_WRITE && !BP_IS_HOLE(bp) &&
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick ASSERT3U(zio->io_prop.zp_copies, <=, BP_GET_NDVAS(bp));
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick * If there were child vdev/gang/ddt errors, they apply to us now.
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick * If the I/O on the transformed data was successful, generate any
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick * checksum reports now while we still have the transformed data.
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick zio_cksum_report_t *zcr = zio->io_cksum_report;
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick zio_pop_transforms(zio); /* note: may set zio->io_error */
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick * If this I/O is attached to a particular vdev,
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick * generate an error message describing the I/O failure
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick * at the block level. We ignore these errors if the
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick * device is currently unavailable.
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick if (zio->io_error != ECKSUM && vd != NULL && !vdev_is_dead(vd))
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick zfs_ereport_post(FM_EREPORT_ZFS_IO, spa, vd, zio, 0, 0);
8f18d1fadf6a0c20fac9ff7259a5368faa3c3bfbGeorge Wilson if ((zio->io_error == EIO || !(zio->io_flags &
8f18d1fadf6a0c20fac9ff7259a5368faa3c3bfbGeorge Wilson (ZIO_FLAG_SPECULATIVE | ZIO_FLAG_DONT_PROPAGATE))) &&
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick * For logical I/O requests, tell the SPA to log the
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick * error and generate a logical data ereport.
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick zfs_ereport_post(FM_EREPORT_ZFS_DATA, spa, NULL, zio,
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick * Determine whether zio should be reexecuted. This will
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick * propagate all the way to the root via zio_notify_parent().
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick ASSERT(zio->io_child_type == ZIO_CHILD_LOGICAL);
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick spa_get_failmode(spa) != ZIO_FAILURE_MODE_CONTINUE)
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick if (!(zio->io_flags & ZIO_FLAG_CANFAIL) && !zio->io_reexecute)
22fe2c8844be88ebae6478ca1b0b92c8ec2aef54Jonathan Adams * Here is a possibly good place to attempt to do
22fe2c8844be88ebae6478ca1b0b92c8ec2aef54Jonathan Adams * either combinatorial reconstruction or error correction
22fe2c8844be88ebae6478ca1b0b92c8ec2aef54Jonathan Adams * based on checksums. It also might be a good place
22fe2c8844be88ebae6478ca1b0b92c8ec2aef54Jonathan Adams * to send out preliminary ereports before we suspend
22fe2c8844be88ebae6478ca1b0b92c8ec2aef54Jonathan Adams * processing.
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick * If there were logical child errors, they apply to us now.
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick * We defer this until now to avoid conflating logical child
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick * errors with errors that happened to the zio itself when
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick * updating vdev stats and reporting FMA events above.
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick zio_inherit_child_errors(zio, ZIO_CHILD_LOGICAL);
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick IO_IS_ALLOCATING(zio) && zio->io_gang_leader == zio &&
80901aea8e78a2c20751f61f01bebd1d5b5c2ba5George Wilson !(zio->io_flags & (ZIO_FLAG_IO_REWRITE | ZIO_FLAG_NOPWRITE)))
33a372ed2393167a0e329c831ace971f147dddb6George Wilson * Godfather I/Os should never suspend.
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick * This is a logical I/O that wants to reexecute.
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick * Reexecute is top-down. When an i/o fails, if it's not
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick * the root, it simply notifies its parent and sticks around.
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick * The parent, seeing that it still has children in zio_done(),
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick * does the same. This percolates all the way up to the root.
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick * The root i/o will reexecute or suspend the entire tree.
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick * This approach ensures that zio_reexecute() honors
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick * all the original i/o dependency relationships, e.g.
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick * parents not executing until children are ready.
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick ASSERT(zio->io_child_type == ZIO_CHILD_LOGICAL);
54d692b75b7a6f90ce7787309da5451f7458e66aGeorge Wilson * "The Godfather" I/O monitors its children but is
54d692b75b7a6f90ce7787309da5451f7458e66aGeorge Wilson * not a true parent to them. It will track them through
54d692b75b7a6f90ce7787309da5451f7458e66aGeorge Wilson * the pipeline but severs its ties whenever they get into
54d692b75b7a6f90ce7787309da5451f7458e66aGeorge Wilson * trouble (e.g. suspended). This allows "The Godfather"
54d692b75b7a6f90ce7787309da5451f7458e66aGeorge Wilson * I/O to return status without blocking.
0f7643c7376dd69a08acbfc9d1d7d548b10c846aGeorge Wilson for (pio = zio_walk_parents(zio, &zl); pio != NULL;
54d692b75b7a6f90ce7787309da5451f7458e66aGeorge Wilson (zio->io_reexecute & ZIO_REEXECUTE_SUSPEND)) {
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick * We're not a root i/o, so there's nothing to do
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick * but notify our parent. Don't propagate errors
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick * upward since we haven't permanently failed yet.
33a372ed2393167a0e329c831ace971f147dddb6George Wilson ASSERT(!(zio->io_flags & ZIO_FLAG_GODFATHER));
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick } else if (zio->io_reexecute & ZIO_REEXECUTE_SUSPEND) {
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick * We'd fail again if we reexecuted now, so suspend
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick * until conditions improve (e.g. device comes online).
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick * Reexecution is potentially a huge amount of work.
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick * Hand it off to the otherwise-unused claim taskq.
ec94d32216ed5705f5176582355cc311cf848e73Adam Leventhal ZIO_TASKQ_ISSUE, (task_func_t *)zio_reexecute, zio,
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick ASSERT(zio->io_error == 0 || (zio->io_flags & ZIO_FLAG_CANFAIL));
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick * Report any checksum errors, since the I/O is complete.
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick zio_cksum_report_t *zcr = zio->io_cksum_report;
a3f829ae41ece20e7f5f63604e177aeeb8b24628Bill Moore * It is the responsibility of the done callback to ensure that this
a3f829ae41ece20e7f5f63604e177aeeb8b24628Bill Moore * particular zio is no longer discoverable for adoption, and as
a3f829ae41ece20e7f5f63604e177aeeb8b24628Bill Moore * such, cannot acquire any new parents.
0f7643c7376dd69a08acbfc9d1d7d548b10c846aGeorge Wilson for (pio = zio_walk_parents(zio, &zl); pio != NULL; pio = pio_next) {
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick * ==========================================================================
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick * I/O pipeline definition
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick * ==========================================================================
a2cdcdd260232b58202b11a9bfc0103c9449ed52Paul Dagnelie * Compare two zbookmark_phys_t's to see which we would reach first in a
a2cdcdd260232b58202b11a9bfc0103c9449ed52Paul Dagnelie * pre-order traversal of the object tree.
a2cdcdd260232b58202b11a9bfc0103c9449ed52Paul Dagnelie * This is simple in every case aside from the meta-dnode object. For all other
a2cdcdd260232b58202b11a9bfc0103c9449ed52Paul Dagnelie * objects, we traverse them in order (object 1 before object 2, and so on).
a2cdcdd260232b58202b11a9bfc0103c9449ed52Paul Dagnelie * However, all of these objects are traversed while traversing object 0, since
a2cdcdd260232b58202b11a9bfc0103c9449ed52Paul Dagnelie * the data it points to is the list of objects. Thus, we need to convert to a
a2cdcdd260232b58202b11a9bfc0103c9449ed52Paul Dagnelie * canonical representation so we can compare meta-dnode bookmarks to
a2cdcdd260232b58202b11a9bfc0103c9449ed52Paul Dagnelie * non-meta-dnode bookmarks.
a2cdcdd260232b58202b11a9bfc0103c9449ed52Paul Dagnelie * We do this by calculating "equivalents" for each field of the zbookmark.
a2cdcdd260232b58202b11a9bfc0103c9449ed52Paul Dagnelie * zbookmarks outside of the meta-dnode use their own object and level, and
a2cdcdd260232b58202b11a9bfc0103c9449ed52Paul Dagnelie * calculate the level 0 equivalent (the first L0 blkid that is contained in the
a2cdcdd260232b58202b11a9bfc0103c9449ed52Paul Dagnelie * blocks this bookmark refers to) by multiplying their blkid by their span
a2cdcdd260232b58202b11a9bfc0103c9449ed52Paul Dagnelie * (the number of L0 blocks contained within one block at their level).
a2cdcdd260232b58202b11a9bfc0103c9449ed52Paul Dagnelie * zbookmarks inside the meta-dnode calculate their object equivalent
a2cdcdd260232b58202b11a9bfc0103c9449ed52Paul Dagnelie * (which is L0equiv * dnodes per data block), use 0 for their L0equiv, and use
a2cdcdd260232b58202b11a9bfc0103c9449ed52Paul Dagnelie * level + 1<<31 (any value larger than a level could ever be) for their level.
a2cdcdd260232b58202b11a9bfc0103c9449ed52Paul Dagnelie * This causes them to always compare before a bookmark in their object
a2cdcdd260232b58202b11a9bfc0103c9449ed52Paul Dagnelie * equivalent, compare appropriately to bookmarks in other objects, and to
a2cdcdd260232b58202b11a9bfc0103c9449ed52Paul Dagnelie * compare appropriately to other bookmarks in the meta-dnode.
a2cdcdd260232b58202b11a9bfc0103c9449ed52Paul Dagneliezbookmark_compare(uint16_t dbss1, uint8_t ibs1, uint16_t dbss2, uint8_t ibs2,
a2cdcdd260232b58202b11a9bfc0103c9449ed52Paul Dagnelie const zbookmark_phys_t *zb1, const zbookmark_phys_t *zb2)
a2cdcdd260232b58202b11a9bfc0103c9449ed52Paul Dagnelie * These variables represent the "equivalent" values for the zbookmark,
a2cdcdd260232b58202b11a9bfc0103c9449ed52Paul Dagnelie * after converting zbookmarks inside the meta dnode to their
a2cdcdd260232b58202b11a9bfc0103c9449ed52Paul Dagnelie * normal-object equivalents.
a2cdcdd260232b58202b11a9bfc0103c9449ed52Paul Dagnelie * BP_SPANB calculates the span in blocks.
a2cdcdd260232b58202b11a9bfc0103c9449ed52Paul Dagnelie zb1L0 = (zb1->zb_blkid) * BP_SPANB(ibs1, zb1->zb_level);
a2cdcdd260232b58202b11a9bfc0103c9449ed52Paul Dagnelie zb2L0 = (zb2->zb_blkid) * BP_SPANB(ibs2, zb2->zb_level);
ad135b5d644628e791c3188a6ecbd9c257961ef8Christopher Siden if (zb1->zb_object == DMU_META_DNODE_OBJECT) {
a2cdcdd260232b58202b11a9bfc0103c9449ed52Paul Dagnelie zb1obj = zb1L0 * (dbss1 << (SPA_MINBLOCKSHIFT - DNODE_SHIFT));
a2cdcdd260232b58202b11a9bfc0103c9449ed52Paul Dagnelie zb1level = zb1->zb_level + COMPARE_META_LEVEL;
a2cdcdd260232b58202b11a9bfc0103c9449ed52Paul Dagnelie if (zb2->zb_object == DMU_META_DNODE_OBJECT) {
a2cdcdd260232b58202b11a9bfc0103c9449ed52Paul Dagnelie zb2obj = zb2L0 * (dbss2 << (SPA_MINBLOCKSHIFT - DNODE_SHIFT));
a2cdcdd260232b58202b11a9bfc0103c9449ed52Paul Dagnelie zb2level = zb2->zb_level + COMPARE_META_LEVEL;
a2cdcdd260232b58202b11a9bfc0103c9449ed52Paul Dagnelie /* Now that we have a canonical representation, do the comparison. */
a2cdcdd260232b58202b11a9bfc0103c9449ed52Paul Dagnelie * This can (theoretically) happen if the bookmarks have the same object
a2cdcdd260232b58202b11a9bfc0103c9449ed52Paul Dagnelie * and level, but different blkids, if the block sizes are not the same.
a2cdcdd260232b58202b11a9bfc0103c9449ed52Paul Dagnelie * There is presently no way to change the indirect block sizes
a2cdcdd260232b58202b11a9bfc0103c9449ed52Paul Dagnelie * This function checks the following: given that last_block is the place that
a2cdcdd260232b58202b11a9bfc0103c9449ed52Paul Dagnelie * our traversal stopped last time, does that guarantee that we've visited
a2cdcdd260232b58202b11a9bfc0103c9449ed52Paul Dagnelie * every node under subtree_root? Therefore, we can't just use the raw output
a2cdcdd260232b58202b11a9bfc0103c9449ed52Paul Dagnelie * of zbookmark_compare. We have to pass in a modified version of
a2cdcdd260232b58202b11a9bfc0103c9449ed52Paul Dagnelie * subtree_root; by incrementing the block id, and then checking whether
a2cdcdd260232b58202b11a9bfc0103c9449ed52Paul Dagnelie * last_block is before or equal to that, we can tell whether or not having
a2cdcdd260232b58202b11a9bfc0103c9449ed52Paul Dagnelie * visited last_block implies that all of subtree_root's children have been
a2cdcdd260232b58202b11a9bfc0103c9449ed52Paul Dagneliezbookmark_subtree_completed(const dnode_phys_t *dnp,
a2cdcdd260232b58202b11a9bfc0103c9449ed52Paul Dagnelie const zbookmark_phys_t *subtree_root, const zbookmark_phys_t *last_block)
a2cdcdd260232b58202b11a9bfc0103c9449ed52Paul Dagnelie /* The objset_phys_t isn't before anything. */
a2cdcdd260232b58202b11a9bfc0103c9449ed52Paul Dagnelie * We pass in 1ULL << (DNODE_BLOCK_SHIFT - SPA_MINBLOCKSHIFT) for the
a2cdcdd260232b58202b11a9bfc0103c9449ed52Paul Dagnelie * data block size in sectors, because that variable is only used if
a2cdcdd260232b58202b11a9bfc0103c9449ed52Paul Dagnelie * the bookmark refers to a block in the meta-dnode. Since we don't
a2cdcdd260232b58202b11a9bfc0103c9449ed52Paul Dagnelie * know without examining it what object it refers to, and there's no
a2cdcdd260232b58202b11a9bfc0103c9449ed52Paul Dagnelie * harm in passing in this value in other cases, we always pass it in.
a2cdcdd260232b58202b11a9bfc0103c9449ed52Paul Dagnelie * We pass in 0 for the indirect block size shift because zb2 must be
a2cdcdd260232b58202b11a9bfc0103c9449ed52Paul Dagnelie * level 0. The indirect block size is only used to calculate the span
a2cdcdd260232b58202b11a9bfc0103c9449ed52Paul Dagnelie * of the bookmark, but since the bookmark must be level 0, the span is
a2cdcdd260232b58202b11a9bfc0103c9449ed52Paul Dagnelie * always 1, so the math works out.
a2cdcdd260232b58202b11a9bfc0103c9449ed52Paul Dagnelie * If you make changes to how the zbookmark_compare code works, be sure
a2cdcdd260232b58202b11a9bfc0103c9449ed52Paul Dagnelie * to make sure that this code still works afterwards.
a2cdcdd260232b58202b11a9bfc0103c9449ed52Paul Dagnelie return (zbookmark_compare(dnp->dn_datablkszsec, dnp->dn_indblkshift,