fa9e4066f08beec538e775443c5be79dd423fcabahrens/*
fa9e4066f08beec538e775443c5be79dd423fcabahrens * CDDL HEADER START
fa9e4066f08beec538e775443c5be79dd423fcabahrens *
fa9e4066f08beec538e775443c5be79dd423fcabahrens * The contents of this file are subject to the terms of the
ea8dc4b6d2251b437950c0056bc626b311c73c27eschrock * Common Development and Distribution License (the "License").
ea8dc4b6d2251b437950c0056bc626b311c73c27eschrock * You may not use this file except in compliance with the License.
fa9e4066f08beec538e775443c5be79dd423fcabahrens *
fa9e4066f08beec538e775443c5be79dd423fcabahrens * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
fa9e4066f08beec538e775443c5be79dd423fcabahrens * or http://www.opensolaris.org/os/licensing.
fa9e4066f08beec538e775443c5be79dd423fcabahrens * See the License for the specific language governing permissions
fa9e4066f08beec538e775443c5be79dd423fcabahrens * and limitations under the License.
fa9e4066f08beec538e775443c5be79dd423fcabahrens *
fa9e4066f08beec538e775443c5be79dd423fcabahrens * When distributing Covered Code, include this CDDL HEADER in each
fa9e4066f08beec538e775443c5be79dd423fcabahrens * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
fa9e4066f08beec538e775443c5be79dd423fcabahrens * If applicable, add the following below this CDDL HEADER, with the
fa9e4066f08beec538e775443c5be79dd423fcabahrens * fields enclosed by brackets "[]" replaced with your own identifying
fa9e4066f08beec538e775443c5be79dd423fcabahrens * information: Portions Copyright [yyyy] [name of copyright owner]
fa9e4066f08beec538e775443c5be79dd423fcabahrens *
fa9e4066f08beec538e775443c5be79dd423fcabahrens * CDDL HEADER END
fa9e4066f08beec538e775443c5be79dd423fcabahrens */
c9431fa1e59a88c2f0abf611f25b97af964449e5ahl
fa9e4066f08beec538e775443c5be79dd423fcabahrens/*
c7cd242109c82107ec2e50013369e92be9d77702George Wilson * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
5aeb94743e3be0c51e86f73096334611ae3a058eGarrett D'Amore * Copyright 2011 Nexenta Systems, Inc. All rights reserved.
8df0bcf0df7622a075cc6e52f659d2fcfdd08cdcPaul Dagnelie * Copyright (c) 2012, 2016 by Delphix. All rights reserved.
a6f561b4aee75d0d028e7b36b151c8ed8a86bc76Sašo Kiselkov * Copyright (c) 2013 by Saso Kiselkov. All rights reserved.
810e43b2eb0e320833671a403fdda51917e8b036Bill Pijewski * Copyright (c) 2013, Joyent, Inc. All rights reserved.
c8811bd3e2427dddbac6c05a59cfe117d8fea370Toomas Soome * Copyright 2016 Toomas Soome <tsoome@me.com>
5aeb94743e3be0c51e86f73096334611ae3a058eGarrett D'Amore */
fa9e4066f08beec538e775443c5be79dd423fcabahrens
fa9e4066f08beec538e775443c5be79dd423fcabahrens#ifndef _ZIO_H
fa9e4066f08beec538e775443c5be79dd423fcabahrens#define _ZIO_H
fa9e4066f08beec538e775443c5be79dd423fcabahrens
a2cdcdd260232b58202b11a9bfc0103c9449ed52Paul Dagnelie#include <sys/zio_priority.h>
fa9e4066f08beec538e775443c5be79dd423fcabahrens#include <sys/zfs_context.h>
fa9e4066f08beec538e775443c5be79dd423fcabahrens#include <sys/spa.h>
fa9e4066f08beec538e775443c5be79dd423fcabahrens#include <sys/txg.h>
fa9e4066f08beec538e775443c5be79dd423fcabahrens#include <sys/avl.h>
fa9e4066f08beec538e775443c5be79dd423fcabahrens#include <sys/fs/zfs.h>
44cd46cadd9aab751dae6a4023c1cb5bf316d274billm#include <sys/zio_impl.h>
fa9e4066f08beec538e775443c5be79dd423fcabahrens
fa9e4066f08beec538e775443c5be79dd423fcabahrens#ifdef __cplusplus
fa9e4066f08beec538e775443c5be79dd423fcabahrensextern "C" {
fa9e4066f08beec538e775443c5be79dd423fcabahrens#endif
fa9e4066f08beec538e775443c5be79dd423fcabahrens
6e1f5caa9321646aa4212d48e32a0d241866d85dNeil Perrin/*
6e1f5caa9321646aa4212d48e32a0d241866d85dNeil Perrin * Embedded checksum
6e1f5caa9321646aa4212d48e32a0d241866d85dNeil Perrin */
6e1f5caa9321646aa4212d48e32a0d241866d85dNeil Perrin#define ZEC_MAGIC 0x210da7ab10c7a11ULL
fa9e4066f08beec538e775443c5be79dd423fcabahrens
6e1f5caa9321646aa4212d48e32a0d241866d85dNeil Perrintypedef struct zio_eck {
6e1f5caa9321646aa4212d48e32a0d241866d85dNeil Perrin uint64_t zec_magic; /* for validation, endianness */
6e1f5caa9321646aa4212d48e32a0d241866d85dNeil Perrin zio_cksum_t zec_cksum; /* 256-bit checksum */
6e1f5caa9321646aa4212d48e32a0d241866d85dNeil Perrin} zio_eck_t;
fa9e4066f08beec538e775443c5be79dd423fcabahrens
fa9e4066f08beec538e775443c5be79dd423fcabahrens/*
fa9e4066f08beec538e775443c5be79dd423fcabahrens * Gang block headers are self-checksumming and contain an array
fa9e4066f08beec538e775443c5be79dd423fcabahrens * of block pointers.
fa9e4066f08beec538e775443c5be79dd423fcabahrens */
fa9e4066f08beec538e775443c5be79dd423fcabahrens#define SPA_GANGBLOCKSIZE SPA_MINBLOCKSIZE
fa9e4066f08beec538e775443c5be79dd423fcabahrens#define SPA_GBH_NBLKPTRS ((SPA_GANGBLOCKSIZE - \
6e1f5caa9321646aa4212d48e32a0d241866d85dNeil Perrin sizeof (zio_eck_t)) / sizeof (blkptr_t))
fa9e4066f08beec538e775443c5be79dd423fcabahrens#define SPA_GBH_FILLER ((SPA_GANGBLOCKSIZE - \
6e1f5caa9321646aa4212d48e32a0d241866d85dNeil Perrin sizeof (zio_eck_t) - \
fa9e4066f08beec538e775443c5be79dd423fcabahrens (SPA_GBH_NBLKPTRS * sizeof (blkptr_t))) /\
fa9e4066f08beec538e775443c5be79dd423fcabahrens sizeof (uint64_t))
fa9e4066f08beec538e775443c5be79dd423fcabahrens
fa9e4066f08beec538e775443c5be79dd423fcabahrenstypedef struct zio_gbh {
fa9e4066f08beec538e775443c5be79dd423fcabahrens blkptr_t zg_blkptr[SPA_GBH_NBLKPTRS];
fa9e4066f08beec538e775443c5be79dd423fcabahrens uint64_t zg_filler[SPA_GBH_FILLER];
6e1f5caa9321646aa4212d48e32a0d241866d85dNeil Perrin zio_eck_t zg_tail;
fa9e4066f08beec538e775443c5be79dd423fcabahrens} zio_gbh_phys_t;
fa9e4066f08beec538e775443c5be79dd423fcabahrens
fa9e4066f08beec538e775443c5be79dd423fcabahrensenum zio_checksum {
fa9e4066f08beec538e775443c5be79dd423fcabahrens ZIO_CHECKSUM_INHERIT = 0,
fa9e4066f08beec538e775443c5be79dd423fcabahrens ZIO_CHECKSUM_ON,
fa9e4066f08beec538e775443c5be79dd423fcabahrens ZIO_CHECKSUM_OFF,
fa9e4066f08beec538e775443c5be79dd423fcabahrens ZIO_CHECKSUM_LABEL,
fa9e4066f08beec538e775443c5be79dd423fcabahrens ZIO_CHECKSUM_GANG_HEADER,
fa9e4066f08beec538e775443c5be79dd423fcabahrens ZIO_CHECKSUM_ZILOG,
fa9e4066f08beec538e775443c5be79dd423fcabahrens ZIO_CHECKSUM_FLETCHER_2,
fa9e4066f08beec538e775443c5be79dd423fcabahrens ZIO_CHECKSUM_FLETCHER_4,
fa9e4066f08beec538e775443c5be79dd423fcabahrens ZIO_CHECKSUM_SHA256,
6e1f5caa9321646aa4212d48e32a0d241866d85dNeil Perrin ZIO_CHECKSUM_ZILOG2,
810e43b2eb0e320833671a403fdda51917e8b036Bill Pijewski ZIO_CHECKSUM_NOPARITY,
45818ee124adeaaf947698996b4f4c722afc6d1fMatthew Ahrens ZIO_CHECKSUM_SHA512,
45818ee124adeaaf947698996b4f4c722afc6d1fMatthew Ahrens ZIO_CHECKSUM_SKEIN,
45818ee124adeaaf947698996b4f4c722afc6d1fMatthew Ahrens ZIO_CHECKSUM_EDONR,
fa9e4066f08beec538e775443c5be79dd423fcabahrens ZIO_CHECKSUM_FUNCTIONS
fa9e4066f08beec538e775443c5be79dd423fcabahrens};
fa9e4066f08beec538e775443c5be79dd423fcabahrens
5d7b4d438c4a51eccc95e77a83a437b4d48380ebMatthew Ahrens/*
5d7b4d438c4a51eccc95e77a83a437b4d48380ebMatthew Ahrens * The number of "legacy" compression functions which can be set on individual
5d7b4d438c4a51eccc95e77a83a437b4d48380ebMatthew Ahrens * objects.
5d7b4d438c4a51eccc95e77a83a437b4d48380ebMatthew Ahrens */
5d7b4d438c4a51eccc95e77a83a437b4d48380ebMatthew Ahrens#define ZIO_CHECKSUM_LEGACY_FUNCTIONS ZIO_CHECKSUM_ZILOG2
5d7b4d438c4a51eccc95e77a83a437b4d48380ebMatthew Ahrens
7e322df5ee63a00c1c57398abec50fc1dc54b67aJonathan Adams#define ZIO_CHECKSUM_ON_VALUE ZIO_CHECKSUM_FLETCHER_4
fa9e4066f08beec538e775443c5be79dd423fcabahrens#define ZIO_CHECKSUM_DEFAULT ZIO_CHECKSUM_ON
fa9e4066f08beec538e775443c5be79dd423fcabahrens
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick#define ZIO_CHECKSUM_MASK 0xffULL
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick#define ZIO_CHECKSUM_VERIFY (1 << 8)
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick#define ZIO_DEDUPCHECKSUM ZIO_CHECKSUM_SHA256
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick#define ZIO_DEDUPDITTO_MIN 100
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick
5d7b4d438c4a51eccc95e77a83a437b4d48380ebMatthew Ahrens/*
5d7b4d438c4a51eccc95e77a83a437b4d48380ebMatthew Ahrens * The number of "legacy" compression functions which can be set on individual
5d7b4d438c4a51eccc95e77a83a437b4d48380ebMatthew Ahrens * objects.
5d7b4d438c4a51eccc95e77a83a437b4d48380ebMatthew Ahrens */
5d7b4d438c4a51eccc95e77a83a437b4d48380ebMatthew Ahrens#define ZIO_COMPRESS_LEGACY_FUNCTIONS ZIO_COMPRESS_LZ4
5d7b4d438c4a51eccc95e77a83a437b4d48380ebMatthew Ahrens
db1741f555ec79def5e9846e6bfd132248514ffeJustin T. Gibbs/*
db1741f555ec79def5e9846e6bfd132248514ffeJustin T. Gibbs * The meaning of "compress = on" selected by the compression features enabled
db1741f555ec79def5e9846e6bfd132248514ffeJustin T. Gibbs * on a given pool.
db1741f555ec79def5e9846e6bfd132248514ffeJustin T. Gibbs */
db1741f555ec79def5e9846e6bfd132248514ffeJustin T. Gibbs#define ZIO_COMPRESS_LEGACY_ON_VALUE ZIO_COMPRESS_LZJB
db1741f555ec79def5e9846e6bfd132248514ffeJustin T. Gibbs#define ZIO_COMPRESS_LZ4_ON_VALUE ZIO_COMPRESS_LZ4
db1741f555ec79def5e9846e6bfd132248514ffeJustin T. Gibbs
db1741f555ec79def5e9846e6bfd132248514ffeJustin T. Gibbs#define ZIO_COMPRESS_DEFAULT ZIO_COMPRESS_OFF
fa9e4066f08beec538e775443c5be79dd423fcabahrens
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick#define BOOTFS_COMPRESS_VALID(compress) \
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick ((compress) == ZIO_COMPRESS_LZJB || \
a6f561b4aee75d0d028e7b36b151c8ed8a86bc76Sašo Kiselkov (compress) == ZIO_COMPRESS_LZ4 || \
c8811bd3e2427dddbac6c05a59cfe117d8fea370Toomas Soome (compress) == ZIO_COMPRESS_GZIP_1 || \
c8811bd3e2427dddbac6c05a59cfe117d8fea370Toomas Soome (compress) == ZIO_COMPRESS_GZIP_2 || \
c8811bd3e2427dddbac6c05a59cfe117d8fea370Toomas Soome (compress) == ZIO_COMPRESS_GZIP_3 || \
c8811bd3e2427dddbac6c05a59cfe117d8fea370Toomas Soome (compress) == ZIO_COMPRESS_GZIP_4 || \
c8811bd3e2427dddbac6c05a59cfe117d8fea370Toomas Soome (compress) == ZIO_COMPRESS_GZIP_5 || \
c8811bd3e2427dddbac6c05a59cfe117d8fea370Toomas Soome (compress) == ZIO_COMPRESS_GZIP_6 || \
c8811bd3e2427dddbac6c05a59cfe117d8fea370Toomas Soome (compress) == ZIO_COMPRESS_GZIP_7 || \
c8811bd3e2427dddbac6c05a59cfe117d8fea370Toomas Soome (compress) == ZIO_COMPRESS_GZIP_8 || \
c8811bd3e2427dddbac6c05a59cfe117d8fea370Toomas Soome (compress) == ZIO_COMPRESS_GZIP_9 || \
db1741f555ec79def5e9846e6bfd132248514ffeJustin T. Gibbs (compress) == ZIO_COMPRESS_ON || \
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick (compress) == ZIO_COMPRESS_OFF)
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick
0a4e9518a44f226be6d39383330b5b1792d2f184gw#define ZIO_FAILURE_MODE_WAIT 0
0a4e9518a44f226be6d39383330b5b1792d2f184gw#define ZIO_FAILURE_MODE_CONTINUE 1
0a4e9518a44f226be6d39383330b5b1792d2f184gw#define ZIO_FAILURE_MODE_PANIC 2
0a4e9518a44f226be6d39383330b5b1792d2f184gw
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwickenum zio_flag {
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick /*
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick * Flags inherited by gang, ddt, and vdev children,
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick * and that must be equal for two zios to aggregate
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick */
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick ZIO_FLAG_DONT_AGGREGATE = 1 << 0,
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick ZIO_FLAG_IO_REPAIR = 1 << 1,
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick ZIO_FLAG_SELF_HEAL = 1 << 2,
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick ZIO_FLAG_RESILVER = 1 << 3,
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick ZIO_FLAG_SCRUB = 1 << 4,
44ecc5327ab4ce0750dcca2a17e05566bf2812e2George Wilson ZIO_FLAG_SCAN_THREAD = 1 << 5,
2a104a5236475eb73aa41eaaf3ed9f3ccbe0ca55Alex Reece ZIO_FLAG_PHYSICAL = 1 << 6,
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick#define ZIO_FLAG_AGG_INHERIT (ZIO_FLAG_CANFAIL - 1)
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick /*
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick * Flags inherited by ddt, gang, and vdev children.
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick */
2a104a5236475eb73aa41eaaf3ed9f3ccbe0ca55Alex Reece ZIO_FLAG_CANFAIL = 1 << 7, /* must be first for INHERIT */
2a104a5236475eb73aa41eaaf3ed9f3ccbe0ca55Alex Reece ZIO_FLAG_SPECULATIVE = 1 << 8,
2a104a5236475eb73aa41eaaf3ed9f3ccbe0ca55Alex Reece ZIO_FLAG_CONFIG_WRITER = 1 << 9,
2a104a5236475eb73aa41eaaf3ed9f3ccbe0ca55Alex Reece ZIO_FLAG_DONT_RETRY = 1 << 10,
2a104a5236475eb73aa41eaaf3ed9f3ccbe0ca55Alex Reece ZIO_FLAG_DONT_CACHE = 1 << 11,
2a104a5236475eb73aa41eaaf3ed9f3ccbe0ca55Alex Reece ZIO_FLAG_NODATA = 1 << 12,
2a104a5236475eb73aa41eaaf3ed9f3ccbe0ca55Alex Reece ZIO_FLAG_INDUCE_DAMAGE = 1 << 13,
0f7643c7376dd69a08acbfc9d1d7d548b10c846aGeorge Wilson ZIO_FLAG_IO_ALLOCATING = 1 << 14,
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick#define ZIO_FLAG_DDT_INHERIT (ZIO_FLAG_IO_RETRY - 1)
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick#define ZIO_FLAG_GANG_INHERIT (ZIO_FLAG_IO_RETRY - 1)
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick /*
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick * Flags inherited by vdev children.
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick */
0f7643c7376dd69a08acbfc9d1d7d548b10c846aGeorge Wilson ZIO_FLAG_IO_RETRY = 1 << 15, /* must be first for INHERIT */
0f7643c7376dd69a08acbfc9d1d7d548b10c846aGeorge Wilson ZIO_FLAG_PROBE = 1 << 16,
0f7643c7376dd69a08acbfc9d1d7d548b10c846aGeorge Wilson ZIO_FLAG_TRYHARD = 1 << 17,
0f7643c7376dd69a08acbfc9d1d7d548b10c846aGeorge Wilson ZIO_FLAG_OPTIONAL = 1 << 18,
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick#define ZIO_FLAG_VDEV_INHERIT (ZIO_FLAG_DONT_QUEUE - 1)
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick /*
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick * Flags not inherited by any children.
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick */
0f7643c7376dd69a08acbfc9d1d7d548b10c846aGeorge Wilson ZIO_FLAG_DONT_QUEUE = 1 << 19, /* must be first for INHERIT */
0f7643c7376dd69a08acbfc9d1d7d548b10c846aGeorge Wilson ZIO_FLAG_DONT_PROPAGATE = 1 << 20,
0f7643c7376dd69a08acbfc9d1d7d548b10c846aGeorge Wilson ZIO_FLAG_IO_BYPASS = 1 << 21,
0f7643c7376dd69a08acbfc9d1d7d548b10c846aGeorge Wilson ZIO_FLAG_IO_REWRITE = 1 << 22,
0f7643c7376dd69a08acbfc9d1d7d548b10c846aGeorge Wilson ZIO_FLAG_RAW = 1 << 23,
0f7643c7376dd69a08acbfc9d1d7d548b10c846aGeorge Wilson ZIO_FLAG_GANG_CHILD = 1 << 24,
0f7643c7376dd69a08acbfc9d1d7d548b10c846aGeorge Wilson ZIO_FLAG_DDT_CHILD = 1 << 25,
0f7643c7376dd69a08acbfc9d1d7d548b10c846aGeorge Wilson ZIO_FLAG_GODFATHER = 1 << 26,
0f7643c7376dd69a08acbfc9d1d7d548b10c846aGeorge Wilson ZIO_FLAG_NOPWRITE = 1 << 27,
0f7643c7376dd69a08acbfc9d1d7d548b10c846aGeorge Wilson ZIO_FLAG_REEXECUTED = 1 << 28,
0f7643c7376dd69a08acbfc9d1d7d548b10c846aGeorge Wilson ZIO_FLAG_DELEGATED = 1 << 29,
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick};
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick#define ZIO_FLAG_MUSTSUCCEED 0
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick#define ZIO_DDT_CHILD_FLAGS(zio) \
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick (((zio)->io_flags & ZIO_FLAG_DDT_INHERIT) | \
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick ZIO_FLAG_DDT_CHILD | ZIO_FLAG_CANFAIL)
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick#define ZIO_GANG_CHILD_FLAGS(zio) \
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick (((zio)->io_flags & ZIO_FLAG_GANG_INHERIT) | \
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick ZIO_FLAG_GANG_CHILD | ZIO_FLAG_CANFAIL)
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick#define ZIO_VDEV_CHILD_FLAGS(zio) \
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick (((zio)->io_flags & ZIO_FLAG_VDEV_INHERIT) | \
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick ZIO_FLAG_CANFAIL)
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwickenum zio_child {
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick ZIO_CHILD_VDEV = 0,
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick ZIO_CHILD_GANG,
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick ZIO_CHILD_DDT,
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick ZIO_CHILD_LOGICAL,
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick ZIO_CHILD_TYPES
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick};
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwickenum zio_wait_type {
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick ZIO_WAIT_READY = 0,
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick ZIO_WAIT_DONE,
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick ZIO_WAIT_TYPES
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick};
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick
fa9e4066f08beec538e775443c5be79dd423fcabahrens/*
e7cbe64f7a72dae5cb44f100db60ca88f3313c65gw * We'll take the unused errnos, 'EBADE' and 'EBADR' (from the Convergent
e7cbe64f7a72dae5cb44f100db60ca88f3313c65gw * graveyard) to indicate checksum errors and fragmentation.
fa9e4066f08beec538e775443c5be79dd423fcabahrens */
fa9e4066f08beec538e775443c5be79dd423fcabahrens#define ECKSUM EBADE
e7cbe64f7a72dae5cb44f100db60ca88f3313c65gw#define EFRAGS EBADR
fa9e4066f08beec538e775443c5be79dd423fcabahrens
fa9e4066f08beec538e775443c5be79dd423fcabahrenstypedef void zio_done_func_t(zio_t *zio);
fa9e4066f08beec538e775443c5be79dd423fcabahrens
0f7643c7376dd69a08acbfc9d1d7d548b10c846aGeorge Wilsonextern boolean_t zio_dva_throttle_enabled;
69962b5647e4a8b9b14998733b765925381b727eMatthew Ahrensextern const char *zio_type_name[ZIO_TYPES];
fa9e4066f08beec538e775443c5be79dd423fcabahrens
ea8dc4b6d2251b437950c0056bc626b311c73c27eschrock/*
ea8dc4b6d2251b437950c0056bc626b311c73c27eschrock * A bookmark is a four-tuple <objset, object, level, blkid> that uniquely
ea8dc4b6d2251b437950c0056bc626b311c73c27eschrock * identifies any block in the pool. By convention, the meta-objset (MOS)
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick * is objset 0, and the meta-dnode is object 0. This covers all blocks
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick * except root blocks and ZIL blocks, which are defined as follows:
ea8dc4b6d2251b437950c0056bc626b311c73c27eschrock *
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick * Root blocks (objset_phys_t) are object 0, level -1: <objset, 0, -1, 0>.
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick * ZIL blocks are bookmarked <objset, 0, -2, blkid == ZIL sequence number>.
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick * dmu_sync()ed ZIL data blocks are bookmarked <objset, object, -2, blkid>.
a2cdcdd260232b58202b11a9bfc0103c9449ed52Paul Dagnelie * dnode visit bookmarks are <objset, object id of dnode, -3, 0>.
ea8dc4b6d2251b437950c0056bc626b311c73c27eschrock *
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick * Note: this structure is called a bookmark because its original purpose
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick * was to remember where to resume a pool-wide traverse.
ea8dc4b6d2251b437950c0056bc626b311c73c27eschrock *
7802d7bf98dec568dadf72286893b1fe5abd8602Matthew Ahrens * Note: this structure is passed between userland and the kernel, and is
7802d7bf98dec568dadf72286893b1fe5abd8602Matthew Ahrens * stored on disk (by virtue of being incorporated into other on-disk
7802d7bf98dec568dadf72286893b1fe5abd8602Matthew Ahrens * structures, e.g. dsl_scan_phys_t).
ea8dc4b6d2251b437950c0056bc626b311c73c27eschrock */
7802d7bf98dec568dadf72286893b1fe5abd8602Matthew Ahrenstypedef struct zbookmark_phys {
ea8dc4b6d2251b437950c0056bc626b311c73c27eschrock uint64_t zb_objset;
ea8dc4b6d2251b437950c0056bc626b311c73c27eschrock uint64_t zb_object;
ea8dc4b6d2251b437950c0056bc626b311c73c27eschrock int64_t zb_level;
ea8dc4b6d2251b437950c0056bc626b311c73c27eschrock uint64_t zb_blkid;
7802d7bf98dec568dadf72286893b1fe5abd8602Matthew Ahrens} zbookmark_phys_t;
ea8dc4b6d2251b437950c0056bc626b311c73c27eschrock
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick#define SET_BOOKMARK(zb, objset, object, level, blkid) \
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick{ \
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick (zb)->zb_objset = objset; \
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick (zb)->zb_object = object; \
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick (zb)->zb_level = level; \
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick (zb)->zb_blkid = blkid; \
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick}
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick#define ZB_DESTROYED_OBJSET (-1ULL)
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick#define ZB_ROOT_OBJECT (0ULL)
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick#define ZB_ROOT_LEVEL (-1LL)
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick#define ZB_ROOT_BLKID (0ULL)
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick#define ZB_ZIL_OBJECT (0ULL)
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick#define ZB_ZIL_LEVEL (-2LL)
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick
a2cdcdd260232b58202b11a9bfc0103c9449ed52Paul Dagnelie#define ZB_DNODE_LEVEL (-3LL)
a2cdcdd260232b58202b11a9bfc0103c9449ed52Paul Dagnelie#define ZB_DNODE_BLKID (0ULL)
a2cdcdd260232b58202b11a9bfc0103c9449ed52Paul Dagnelie
ad135b5d644628e791c3188a6ecbd9c257961ef8Christopher Siden#define ZB_IS_ZERO(zb) \
ad135b5d644628e791c3188a6ecbd9c257961ef8Christopher Siden ((zb)->zb_objset == 0 && (zb)->zb_object == 0 && \
ad135b5d644628e791c3188a6ecbd9c257961ef8Christopher Siden (zb)->zb_level == 0 && (zb)->zb_blkid == 0)
ad135b5d644628e791c3188a6ecbd9c257961ef8Christopher Siden#define ZB_IS_ROOT(zb) \
ad135b5d644628e791c3188a6ecbd9c257961ef8Christopher Siden ((zb)->zb_object == ZB_ROOT_OBJECT && \
ad135b5d644628e791c3188a6ecbd9c257961ef8Christopher Siden (zb)->zb_level == ZB_ROOT_LEVEL && \
ad135b5d644628e791c3188a6ecbd9c257961ef8Christopher Siden (zb)->zb_blkid == ZB_ROOT_BLKID)
ad135b5d644628e791c3188a6ecbd9c257961ef8Christopher Siden
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwicktypedef struct zio_prop {
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick enum zio_checksum zp_checksum;
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick enum zio_compress zp_compress;
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick dmu_object_type_t zp_type;
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick uint8_t zp_level;
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick uint8_t zp_copies;
80901aea8e78a2c20751f61f01bebd1d5b5c2ba5George Wilson boolean_t zp_dedup;
80901aea8e78a2c20751f61f01bebd1d5b5c2ba5George Wilson boolean_t zp_dedup_verify;
80901aea8e78a2c20751f61f01bebd1d5b5c2ba5George Wilson boolean_t zp_nopwrite;
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick} zio_prop_t;
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick
22fe2c8844be88ebae6478ca1b0b92c8ec2aef54Jonathan Adamstypedef struct zio_cksum_report zio_cksum_report_t;
22fe2c8844be88ebae6478ca1b0b92c8ec2aef54Jonathan Adams
22fe2c8844be88ebae6478ca1b0b92c8ec2aef54Jonathan Adamstypedef void zio_cksum_finish_f(zio_cksum_report_t *rep,
22fe2c8844be88ebae6478ca1b0b92c8ec2aef54Jonathan Adams const void *good_data);
22fe2c8844be88ebae6478ca1b0b92c8ec2aef54Jonathan Adamstypedef void zio_cksum_free_f(void *cbdata, size_t size);
22fe2c8844be88ebae6478ca1b0b92c8ec2aef54Jonathan Adams
22fe2c8844be88ebae6478ca1b0b92c8ec2aef54Jonathan Adamsstruct zio_bad_cksum; /* defined in zio_checksum.h */
ad135b5d644628e791c3188a6ecbd9c257961ef8Christopher Sidenstruct dnode_phys;
22fe2c8844be88ebae6478ca1b0b92c8ec2aef54Jonathan Adams
22fe2c8844be88ebae6478ca1b0b92c8ec2aef54Jonathan Adamsstruct zio_cksum_report {
22fe2c8844be88ebae6478ca1b0b92c8ec2aef54Jonathan Adams struct zio_cksum_report *zcr_next;
22fe2c8844be88ebae6478ca1b0b92c8ec2aef54Jonathan Adams nvlist_t *zcr_ereport;
22fe2c8844be88ebae6478ca1b0b92c8ec2aef54Jonathan Adams nvlist_t *zcr_detector;
22fe2c8844be88ebae6478ca1b0b92c8ec2aef54Jonathan Adams void *zcr_cbdata;
22fe2c8844be88ebae6478ca1b0b92c8ec2aef54Jonathan Adams size_t zcr_cbinfo; /* passed to zcr_free() */
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick uint64_t zcr_align;
22fe2c8844be88ebae6478ca1b0b92c8ec2aef54Jonathan Adams uint64_t zcr_length;
22fe2c8844be88ebae6478ca1b0b92c8ec2aef54Jonathan Adams zio_cksum_finish_f *zcr_finish;
22fe2c8844be88ebae6478ca1b0b92c8ec2aef54Jonathan Adams zio_cksum_free_f *zcr_free;
22fe2c8844be88ebae6478ca1b0b92c8ec2aef54Jonathan Adams
22fe2c8844be88ebae6478ca1b0b92c8ec2aef54Jonathan Adams /* internal use only */
22fe2c8844be88ebae6478ca1b0b92c8ec2aef54Jonathan Adams struct zio_bad_cksum *zcr_ckinfo; /* information from failure */
22fe2c8844be88ebae6478ca1b0b92c8ec2aef54Jonathan Adams};
22fe2c8844be88ebae6478ca1b0b92c8ec2aef54Jonathan Adams
22fe2c8844be88ebae6478ca1b0b92c8ec2aef54Jonathan Adamstypedef void zio_vsd_cksum_report_f(zio_t *zio, zio_cksum_report_t *zcr,
22fe2c8844be88ebae6478ca1b0b92c8ec2aef54Jonathan Adams void *arg);
22fe2c8844be88ebae6478ca1b0b92c8ec2aef54Jonathan Adams
22fe2c8844be88ebae6478ca1b0b92c8ec2aef54Jonathan Adamszio_vsd_cksum_report_f zio_vsd_default_cksum_report;
22fe2c8844be88ebae6478ca1b0b92c8ec2aef54Jonathan Adams
22fe2c8844be88ebae6478ca1b0b92c8ec2aef54Jonathan Adamstypedef struct zio_vsd_ops {
22fe2c8844be88ebae6478ca1b0b92c8ec2aef54Jonathan Adams zio_done_func_t *vsd_free;
22fe2c8844be88ebae6478ca1b0b92c8ec2aef54Jonathan Adams zio_vsd_cksum_report_f *vsd_cksum_report;
22fe2c8844be88ebae6478ca1b0b92c8ec2aef54Jonathan Adams} zio_vsd_ops_t;
22fe2c8844be88ebae6478ca1b0b92c8ec2aef54Jonathan Adams
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwicktypedef struct zio_gang_node {
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick zio_gbh_phys_t *gn_gbh;
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick struct zio_gang_node *gn_child[SPA_GBH_NBLKPTRS];
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick} zio_gang_node_t;
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwicktypedef zio_t *zio_gang_issue_func_t(zio_t *zio, blkptr_t *bp,
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick zio_gang_node_t *gn, void *data);
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwicktypedef void zio_transform_func_t(zio_t *zio, void *data, uint64_t size);
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwicktypedef struct zio_transform {
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick void *zt_orig_data;
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick uint64_t zt_orig_size;
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick uint64_t zt_bufsize;
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick zio_transform_func_t *zt_transform;
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick struct zio_transform *zt_next;
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick} zio_transform_t;
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwicktypedef int zio_pipe_stage_t(zio_t *zio);
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick/*
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick * The io_reexecute flags are distinct from io_flags because the child must
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick * be able to propagate them to the parent. The normal io_flags are local
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick * to the zio, not protected by any lock, and not modifiable by children;
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick * the reexecute flags are protected by io_lock, modifiable by children,
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick * and always propagated -- even when ZIO_FLAG_DONT_PROPAGATE is set.
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick */
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick#define ZIO_REEXECUTE_NOW 0x01
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick#define ZIO_REEXECUTE_SUSPEND 0x02
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick
8363e80ae72609660f6090766ca8c2c18aa53f0cGeorge Wilsontypedef struct zio_alloc_list {
8363e80ae72609660f6090766ca8c2c18aa53f0cGeorge Wilson list_t zal_list;
8363e80ae72609660f6090766ca8c2c18aa53f0cGeorge Wilson uint64_t zal_size;
8363e80ae72609660f6090766ca8c2c18aa53f0cGeorge Wilson} zio_alloc_list_t;
8363e80ae72609660f6090766ca8c2c18aa53f0cGeorge Wilson
a3f829ae41ece20e7f5f63604e177aeeb8b24628Bill Mooretypedef struct zio_link {
a3f829ae41ece20e7f5f63604e177aeeb8b24628Bill Moore zio_t *zl_parent;
a3f829ae41ece20e7f5f63604e177aeeb8b24628Bill Moore zio_t *zl_child;
a3f829ae41ece20e7f5f63604e177aeeb8b24628Bill Moore list_node_t zl_parent_node;
a3f829ae41ece20e7f5f63604e177aeeb8b24628Bill Moore list_node_t zl_child_node;
a3f829ae41ece20e7f5f63604e177aeeb8b24628Bill Moore} zio_link_t;
a3f829ae41ece20e7f5f63604e177aeeb8b24628Bill Moore
fa9e4066f08beec538e775443c5be79dd423fcabahrensstruct zio {
fa9e4066f08beec538e775443c5be79dd423fcabahrens /* Core information about this I/O */
7802d7bf98dec568dadf72286893b1fe5abd8602Matthew Ahrens zbookmark_phys_t io_bookmark;
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick zio_prop_t io_prop;
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick zio_type_t io_type;
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick enum zio_child io_child_type;
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick int io_cmd;
69962b5647e4a8b9b14998733b765925381b727eMatthew Ahrens zio_priority_t io_priority;
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick uint8_t io_reexecute;
a3f829ae41ece20e7f5f63604e177aeeb8b24628Bill Moore uint8_t io_state[ZIO_WAIT_TYPES];
fa9e4066f08beec538e775443c5be79dd423fcabahrens uint64_t io_txg;
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick spa_t *io_spa;
fa9e4066f08beec538e775443c5be79dd423fcabahrens blkptr_t *io_bp;
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick blkptr_t *io_bp_override;
fa9e4066f08beec538e775443c5be79dd423fcabahrens blkptr_t io_bp_copy;
a3f829ae41ece20e7f5f63604e177aeeb8b24628Bill Moore list_t io_parent_list;
a3f829ae41ece20e7f5f63604e177aeeb8b24628Bill Moore list_t io_child_list;
ea8dc4b6d2251b437950c0056bc626b311c73c27eschrock zio_t *io_logical;
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick zio_transform_t *io_transform_stack;
fa9e4066f08beec538e775443c5be79dd423fcabahrens
fa9e4066f08beec538e775443c5be79dd423fcabahrens /* Callback info */
c717a56157ae0e6fca6a1e3689ae1edc385716a3maybee zio_done_func_t *io_ready;
8df0bcf0df7622a075cc6e52f659d2fcfdd08cdcPaul Dagnelie zio_done_func_t *io_children_ready;
69962b5647e4a8b9b14998733b765925381b727eMatthew Ahrens zio_done_func_t *io_physdone;
fa9e4066f08beec538e775443c5be79dd423fcabahrens zio_done_func_t *io_done;
fa9e4066f08beec538e775443c5be79dd423fcabahrens void *io_private;
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick int64_t io_prev_space_delta; /* DMU private */
fa9e4066f08beec538e775443c5be79dd423fcabahrens blkptr_t io_bp_orig;
fa9e4066f08beec538e775443c5be79dd423fcabahrens
fa9e4066f08beec538e775443c5be79dd423fcabahrens /* Data represented by this I/O */
fa9e4066f08beec538e775443c5be79dd423fcabahrens void *io_data;
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick void *io_orig_data;
fa9e4066f08beec538e775443c5be79dd423fcabahrens uint64_t io_size;
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick uint64_t io_orig_size;
5602294fda888d923d57a78bafdaf48ae6223deaDan Kimmel /* io_lsize != io_orig_size iff this is a raw write */
5602294fda888d923d57a78bafdaf48ae6223deaDan Kimmel uint64_t io_lsize;
fa9e4066f08beec538e775443c5be79dd423fcabahrens
fa9e4066f08beec538e775443c5be79dd423fcabahrens /* Stuff for the vdev stack */
fa9e4066f08beec538e775443c5be79dd423fcabahrens vdev_t *io_vd;
fa9e4066f08beec538e775443c5be79dd423fcabahrens void *io_vsd;
22fe2c8844be88ebae6478ca1b0b92c8ec2aef54Jonathan Adams const zio_vsd_ops_t *io_vsd_ops;
22fe2c8844be88ebae6478ca1b0b92c8ec2aef54Jonathan Adams
fa9e4066f08beec538e775443c5be79dd423fcabahrens uint64_t io_offset;
c55e05cb35da47582b7afd38734d2f0d9c6deb40Matthew Ahrens hrtime_t io_timestamp;
0f7643c7376dd69a08acbfc9d1d7d548b10c846aGeorge Wilson hrtime_t io_queued_timestamp;
97e81309571898df9fdd94aab1216dfcf23e060bPrakash Surya hrtime_t io_target_timestamp;
69962b5647e4a8b9b14998733b765925381b727eMatthew Ahrens avl_node_t io_queue_node;
fe319232d24f4ae183730a5a24a09423d8ab4429Justin T. Gibbs avl_node_t io_offset_node;
0f7643c7376dd69a08acbfc9d1d7d548b10c846aGeorge Wilson avl_node_t io_alloc_node;
8363e80ae72609660f6090766ca8c2c18aa53f0cGeorge Wilson zio_alloc_list_t io_alloc_list;
fa9e4066f08beec538e775443c5be79dd423fcabahrens
fa9e4066f08beec538e775443c5be79dd423fcabahrens /* Internal pipeline state */
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick enum zio_flag io_flags;
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick enum zio_stage io_stage;
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick enum zio_stage io_pipeline;
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick enum zio_flag io_orig_flags;
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick enum zio_stage io_orig_stage;
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick enum zio_stage io_orig_pipeline;
0f7643c7376dd69a08acbfc9d1d7d548b10c846aGeorge Wilson enum zio_stage io_pipeline_trace;
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick int io_error;
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick int io_child_error[ZIO_CHILD_TYPES];
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick uint64_t io_children[ZIO_CHILD_TYPES][ZIO_WAIT_TYPES];
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick uint64_t io_child_count;
69962b5647e4a8b9b14998733b765925381b727eMatthew Ahrens uint64_t io_phys_children;
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick uint64_t io_parent_count;
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick uint64_t *io_stall;
f5383399322244b3008a624bcd7e93d67037d177Bill Moore zio_t *io_gang_leader;
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick zio_gang_node_t *io_gang_tree;
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick void *io_executor;
fa9e4066f08beec538e775443c5be79dd423fcabahrens void *io_waiter;
fa9e4066f08beec538e775443c5be79dd423fcabahrens kmutex_t io_lock;
fa9e4066f08beec538e775443c5be79dd423fcabahrens kcondvar_t io_cv;
ea8dc4b6d2251b437950c0056bc626b311c73c27eschrock
ea8dc4b6d2251b437950c0056bc626b311c73c27eschrock /* FMA state */
22fe2c8844be88ebae6478ca1b0b92c8ec2aef54Jonathan Adams zio_cksum_report_t *io_cksum_report;
ea8dc4b6d2251b437950c0056bc626b311c73c27eschrock uint64_t io_ena;
5aeb94743e3be0c51e86f73096334611ae3a058eGarrett D'Amore
5aeb94743e3be0c51e86f73096334611ae3a058eGarrett D'Amore /* Taskq dispatching state */
5aeb94743e3be0c51e86f73096334611ae3a058eGarrett D'Amore taskq_ent_t io_tqent;
fa9e4066f08beec538e775443c5be79dd423fcabahrens};
fa9e4066f08beec538e775443c5be79dd423fcabahrens
0f7643c7376dd69a08acbfc9d1d7d548b10c846aGeorge Wilsonextern int zio_timestamp_compare(const void *, const void *);
0f7643c7376dd69a08acbfc9d1d7d548b10c846aGeorge Wilson
a3f829ae41ece20e7f5f63604e177aeeb8b24628Bill Mooreextern zio_t *zio_null(zio_t *pio, spa_t *spa, vdev_t *vd,
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick zio_done_func_t *done, void *private, enum zio_flag flags);
fa9e4066f08beec538e775443c5be79dd423fcabahrens
fa9e4066f08beec538e775443c5be79dd423fcabahrensextern zio_t *zio_root(spa_t *spa,
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick zio_done_func_t *done, void *private, enum zio_flag flags);
fa9e4066f08beec538e775443c5be79dd423fcabahrens
088f389458728c464569a5506b58070254fa4f7dahrensextern zio_t *zio_read(zio_t *pio, spa_t *spa, const blkptr_t *bp, void *data,
5602294fda888d923d57a78bafdaf48ae6223deaDan Kimmel uint64_t lsize, zio_done_func_t *done, void *private,
7802d7bf98dec568dadf72286893b1fe5abd8602Matthew Ahrens zio_priority_t priority, enum zio_flag flags, const zbookmark_phys_t *zb);
fa9e4066f08beec538e775443c5be79dd423fcabahrens
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwickextern zio_t *zio_write(zio_t *pio, spa_t *spa, uint64_t txg, blkptr_t *bp,
5602294fda888d923d57a78bafdaf48ae6223deaDan Kimmel void *data, uint64_t size, uint64_t psize, const zio_prop_t *zp,
8df0bcf0df7622a075cc6e52f659d2fcfdd08cdcPaul Dagnelie zio_done_func_t *ready, zio_done_func_t *children_ready,
8df0bcf0df7622a075cc6e52f659d2fcfdd08cdcPaul Dagnelie zio_done_func_t *physdone, zio_done_func_t *done,
8df0bcf0df7622a075cc6e52f659d2fcfdd08cdcPaul Dagnelie void *private, zio_priority_t priority, enum zio_flag flags,
8df0bcf0df7622a075cc6e52f659d2fcfdd08cdcPaul Dagnelie const zbookmark_phys_t *zb);
fa9e4066f08beec538e775443c5be79dd423fcabahrens
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwickextern zio_t *zio_rewrite(zio_t *pio, spa_t *spa, uint64_t txg, blkptr_t *bp,
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick void *data, uint64_t size, zio_done_func_t *done, void *private,
7802d7bf98dec568dadf72286893b1fe5abd8602Matthew Ahrens zio_priority_t priority, enum zio_flag flags, zbookmark_phys_t *zb);
fa9e4066f08beec538e775443c5be79dd423fcabahrens
80901aea8e78a2c20751f61f01bebd1d5b5c2ba5George Wilsonextern void zio_write_override(zio_t *zio, blkptr_t *bp, int copies,
80901aea8e78a2c20751f61f01bebd1d5b5c2ba5George Wilson boolean_t nopwrite);
82c9918f28a2d8d9e93769e16f7fa7bee347e80fTim Haley
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwickextern void zio_free(spa_t *spa, uint64_t txg, const blkptr_t *bp);
fa9e4066f08beec538e775443c5be79dd423fcabahrens
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwickextern zio_t *zio_claim(zio_t *pio, spa_t *spa, uint64_t txg,
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick const blkptr_t *bp,
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick zio_done_func_t *done, void *private, enum zio_flag flags);
fa9e4066f08beec538e775443c5be79dd423fcabahrens
fa9e4066f08beec538e775443c5be79dd423fcabahrensextern zio_t *zio_ioctl(zio_t *pio, spa_t *spa, vdev_t *vd, int cmd,
69962b5647e4a8b9b14998733b765925381b727eMatthew Ahrens zio_done_func_t *done, void *private, enum zio_flag flags);
fa9e4066f08beec538e775443c5be79dd423fcabahrens
fa9e4066f08beec538e775443c5be79dd423fcabahrensextern zio_t *zio_read_phys(zio_t *pio, vdev_t *vd, uint64_t offset,
fa9e4066f08beec538e775443c5be79dd423fcabahrens uint64_t size, void *data, int checksum,
69962b5647e4a8b9b14998733b765925381b727eMatthew Ahrens zio_done_func_t *done, void *private, zio_priority_t priority,
69962b5647e4a8b9b14998733b765925381b727eMatthew Ahrens enum zio_flag flags, boolean_t labels);
fa9e4066f08beec538e775443c5be79dd423fcabahrens
fa9e4066f08beec538e775443c5be79dd423fcabahrensextern zio_t *zio_write_phys(zio_t *pio, vdev_t *vd, uint64_t offset,
fa9e4066f08beec538e775443c5be79dd423fcabahrens uint64_t size, void *data, int checksum,
69962b5647e4a8b9b14998733b765925381b727eMatthew Ahrens zio_done_func_t *done, void *private, zio_priority_t priority,
69962b5647e4a8b9b14998733b765925381b727eMatthew Ahrens enum zio_flag flags, boolean_t labels);
fa9e4066f08beec538e775443c5be79dd423fcabahrens
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwickextern zio_t *zio_free_sync(zio_t *pio, spa_t *spa, uint64_t txg,
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick const blkptr_t *bp, enum zio_flag flags);
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwickextern int zio_alloc_zil(spa_t *spa, uint64_t txg, blkptr_t *new_bp,
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick blkptr_t *old_bp, uint64_t size, boolean_t use_slog);
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwickextern void zio_free_zil(spa_t *spa, uint64_t txg, blkptr_t *bp);
17f17c2df4ca603e787c5fc03b7af83519edfd2cbonwickextern void zio_flush(zio_t *zio, vdev_t *vd);
6e1f5caa9321646aa4212d48e32a0d241866d85dNeil Perrinextern void zio_shrink(zio_t *zio, uint64_t size);
fa9e4066f08beec538e775443c5be79dd423fcabahrens
fa9e4066f08beec538e775443c5be79dd423fcabahrensextern int zio_wait(zio_t *zio);
fa9e4066f08beec538e775443c5be79dd423fcabahrensextern void zio_nowait(zio_t *zio);
e05725b117836db173257fae43fb0746eb857fb5bonwickextern void zio_execute(zio_t *zio);
e05725b117836db173257fae43fb0746eb857fb5bonwickextern void zio_interrupt(zio_t *zio);
97e81309571898df9fdd94aab1216dfcf23e060bPrakash Suryaextern void zio_delay_init(zio_t *zio);
97e81309571898df9fdd94aab1216dfcf23e060bPrakash Suryaextern void zio_delay_interrupt(zio_t *zio);
e05725b117836db173257fae43fb0746eb857fb5bonwick
0f7643c7376dd69a08acbfc9d1d7d548b10c846aGeorge Wilsonextern zio_t *zio_walk_parents(zio_t *cio, zio_link_t **);
0f7643c7376dd69a08acbfc9d1d7d548b10c846aGeorge Wilsonextern zio_t *zio_walk_children(zio_t *pio, zio_link_t **);
a3f829ae41ece20e7f5f63604e177aeeb8b24628Bill Mooreextern zio_t *zio_unique_parent(zio_t *cio);
a3f829ae41ece20e7f5f63604e177aeeb8b24628Bill Mooreextern void zio_add_child(zio_t *pio, zio_t *cio);
a3f829ae41ece20e7f5f63604e177aeeb8b24628Bill Moore
fa9e4066f08beec538e775443c5be79dd423fcabahrensextern void *zio_buf_alloc(size_t size);
fa9e4066f08beec538e775443c5be79dd423fcabahrensextern void zio_buf_free(void *buf, size_t size);
ad23a2db4cfc94c0ed1d58554479ce8d2e7e5768johansenextern void *zio_data_buf_alloc(size_t size);
ad23a2db4cfc94c0ed1d58554479ce8d2e7e5768johansenextern void zio_data_buf_free(void *buf, size_t size);
fa9e4066f08beec538e775443c5be79dd423fcabahrens
dcbf3bd6a1f1360fc1afcee9e22c6dcff7844bf2George Wilsonextern void zio_push_transform(zio_t *zio, void *data, uint64_t size,
dcbf3bd6a1f1360fc1afcee9e22c6dcff7844bf2George Wilson uint64_t bufsize, zio_transform_func_t *transform);
dcbf3bd6a1f1360fc1afcee9e22c6dcff7844bf2George Wilsonextern void zio_pop_transforms(zio_t *zio);
dcbf3bd6a1f1360fc1afcee9e22c6dcff7844bf2George Wilson
0a4e9518a44f226be6d39383330b5b1792d2f184gwextern void zio_resubmit_stage_async(void *);
fa9e4066f08beec538e775443c5be79dd423fcabahrens
fa9e4066f08beec538e775443c5be79dd423fcabahrensextern zio_t *zio_vdev_child_io(zio_t *zio, blkptr_t *bp, vdev_t *vd,
69962b5647e4a8b9b14998733b765925381b727eMatthew Ahrens uint64_t offset, void *data, uint64_t size, int type,
69962b5647e4a8b9b14998733b765925381b727eMatthew Ahrens zio_priority_t priority, enum zio_flag flags,
69962b5647e4a8b9b14998733b765925381b727eMatthew Ahrens zio_done_func_t *done, void *private);
fa9e4066f08beec538e775443c5be79dd423fcabahrens
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwickextern zio_t *zio_vdev_delegated_io(vdev_t *vd, uint64_t offset,
69962b5647e4a8b9b14998733b765925381b727eMatthew Ahrens void *data, uint64_t size, int type, zio_priority_t priority,
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick enum zio_flag flags, zio_done_func_t *done, void *private);
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwick
fa9e4066f08beec538e775443c5be79dd423fcabahrensextern void zio_vdev_io_bypass(zio_t *zio);
fa9e4066f08beec538e775443c5be79dd423fcabahrensextern void zio_vdev_io_reissue(zio_t *zio);
fa9e4066f08beec538e775443c5be79dd423fcabahrensextern void zio_vdev_io_redone(zio_t *zio);
fa9e4066f08beec538e775443c5be79dd423fcabahrens
fa9e4066f08beec538e775443c5be79dd423fcabahrensextern void zio_checksum_verified(zio_t *zio);
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwickextern int zio_worst_error(int e1, int e2);
fa9e4066f08beec538e775443c5be79dd423fcabahrens
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwickextern enum zio_checksum zio_checksum_select(enum zio_checksum child,
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick enum zio_checksum parent);
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwickextern enum zio_checksum zio_checksum_dedup_select(spa_t *spa,
b24ab6762772a3f6a89393947930c7fa61306783Jeff Bonwick enum zio_checksum child, enum zio_checksum parent);
db1741f555ec79def5e9846e6bfd132248514ffeJustin T. Gibbsextern enum zio_compress zio_compress_select(spa_t *spa,
db1741f555ec79def5e9846e6bfd132248514ffeJustin T. Gibbs enum zio_compress child, enum zio_compress parent);
fa9e4066f08beec538e775443c5be79dd423fcabahrens
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwickextern void zio_suspend(spa_t *spa, zio_t *zio);
54d692b75b7a6f90ce7787309da5451f7458e66aGeorge Wilsonextern int zio_resume(spa_t *spa);
e14bb3258d05c1b1077e2db7cf77088924e56919Jeff Bonwickextern void zio_resume_wait(spa_t *spa);
ea8dc4b6d2251b437950c0056bc626b311c73c27eschrock
fa9e4066f08beec538e775443c5be79dd423fcabahrens/*
fa9e4066f08beec538e775443c5be79dd423fcabahrens * Initial setup and teardown.
fa9e4066f08beec538e775443c5be79dd423fcabahrens */
fa9e4066f08beec538e775443c5be79dd423fcabahrensextern void zio_init(void);
fa9e4066f08beec538e775443c5be79dd423fcabahrensextern void zio_fini(void);
fa9e4066f08beec538e775443c5be79dd423fcabahrens
ea8dc4b6d2251b437950c0056bc626b311c73c27eschrock/*
ea8dc4b6d2251b437950c0056bc626b311c73c27eschrock * Fault injection
ea8dc4b6d2251b437950c0056bc626b311c73c27eschrock */
ea8dc4b6d2251b437950c0056bc626b311c73c27eschrockstruct zinject_record;
ea8dc4b6d2251b437950c0056bc626b311c73c27eschrockextern uint32_t zio_injection_enabled;
ea8dc4b6d2251b437950c0056bc626b311c73c27eschrockextern int zio_inject_fault(char *name, int flags, int *id,
ea8dc4b6d2251b437950c0056bc626b311c73c27eschrock struct zinject_record *record);
ea8dc4b6d2251b437950c0056bc626b311c73c27eschrockextern int zio_inject_list_next(int *id, char *name, size_t buflen,
ea8dc4b6d2251b437950c0056bc626b311c73c27eschrock struct zinject_record *record);
ea8dc4b6d2251b437950c0056bc626b311c73c27eschrockextern int zio_clear_fault(int id);
1195e687f1c03c8d57417b5999578922e20a3554Mark J Musanteextern void zio_handle_panic_injection(spa_t *spa, char *tag, uint64_t type);
ea8dc4b6d2251b437950c0056bc626b311c73c27eschrockextern int zio_handle_fault_injection(zio_t *zio, int error);
8956713aded83a741173fcd4f9ef1c83521fbea9Eric Schrockextern int zio_handle_device_injection(vdev_t *vd, zio_t *zio, int error);
21bf64a78855d076f09716ea1c06175d954e934cgwextern int zio_handle_label_injection(zio_t *zio, int error);
468c413a79615e77179e8d98f22a7e513a8135bdTim Haleyextern void zio_handle_ignored_writes(zio_t *zio);
97e81309571898df9fdd94aab1216dfcf23e060bPrakash Suryaextern hrtime_t zio_handle_io_delay(zio_t *zio);
ea8dc4b6d2251b437950c0056bc626b311c73c27eschrock
22fe2c8844be88ebae6478ca1b0b92c8ec2aef54Jonathan Adams/*
22fe2c8844be88ebae6478ca1b0b92c8ec2aef54Jonathan Adams * Checksum ereport functions
22fe2c8844be88ebae6478ca1b0b92c8ec2aef54Jonathan Adams */
22fe2c8844be88ebae6478ca1b0b92c8ec2aef54Jonathan Adamsextern void zfs_ereport_start_checksum(spa_t *spa, vdev_t *vd, struct zio *zio,
22fe2c8844be88ebae6478ca1b0b92c8ec2aef54Jonathan Adams uint64_t offset, uint64_t length, void *arg, struct zio_bad_cksum *info);
22fe2c8844be88ebae6478ca1b0b92c8ec2aef54Jonathan Adamsextern void zfs_ereport_finish_checksum(zio_cksum_report_t *report,
22fe2c8844be88ebae6478ca1b0b92c8ec2aef54Jonathan Adams const void *good_data, const void *bad_data, boolean_t drop_if_identical);
22fe2c8844be88ebae6478ca1b0b92c8ec2aef54Jonathan Adams
22fe2c8844be88ebae6478ca1b0b92c8ec2aef54Jonathan Adamsextern void zfs_ereport_send_interim_checksum(zio_cksum_report_t *report);
22fe2c8844be88ebae6478ca1b0b92c8ec2aef54Jonathan Adamsextern void zfs_ereport_free_checksum(zio_cksum_report_t *report);
22fe2c8844be88ebae6478ca1b0b92c8ec2aef54Jonathan Adams
22fe2c8844be88ebae6478ca1b0b92c8ec2aef54Jonathan Adams/* If we have the good data in hand, this function can be used */
22fe2c8844be88ebae6478ca1b0b92c8ec2aef54Jonathan Adamsextern void zfs_ereport_post_checksum(spa_t *spa, vdev_t *vd,
22fe2c8844be88ebae6478ca1b0b92c8ec2aef54Jonathan Adams struct zio *zio, uint64_t offset, uint64_t length,
22fe2c8844be88ebae6478ca1b0b92c8ec2aef54Jonathan Adams const void *good_data, const void *bad_data, struct zio_bad_cksum *info);
22fe2c8844be88ebae6478ca1b0b92c8ec2aef54Jonathan Adams
468c413a79615e77179e8d98f22a7e513a8135bdTim Haley/* Called from spa_sync(), but primarily an injection handler */
468c413a79615e77179e8d98f22a7e513a8135bdTim Haleyextern void spa_handle_ignored_writes(spa_t *spa);
468c413a79615e77179e8d98f22a7e513a8135bdTim Haley
7802d7bf98dec568dadf72286893b1fe5abd8602Matthew Ahrens/* zbookmark_phys functions */
a2cdcdd260232b58202b11a9bfc0103c9449ed52Paul Dagnelieboolean_t zbookmark_subtree_completed(const struct dnode_phys *dnp,
a2cdcdd260232b58202b11a9bfc0103c9449ed52Paul Dagnelie const zbookmark_phys_t *subtree_root, const zbookmark_phys_t *last_block);
a2cdcdd260232b58202b11a9bfc0103c9449ed52Paul Dagnelieint zbookmark_compare(uint16_t dbss1, uint8_t ibs1, uint16_t dbss2,
a2cdcdd260232b58202b11a9bfc0103c9449ed52Paul Dagnelie uint8_t ibs2, const zbookmark_phys_t *zb1, const zbookmark_phys_t *zb2);
ad135b5d644628e791c3188a6ecbd9c257961ef8Christopher Siden
fa9e4066f08beec538e775443c5be79dd423fcabahrens#ifdef __cplusplus
fa9e4066f08beec538e775443c5be79dd423fcabahrens}
fa9e4066f08beec538e775443c5be79dd423fcabahrens#endif
fa9e4066f08beec538e775443c5be79dd423fcabahrens
fa9e4066f08beec538e775443c5be79dd423fcabahrens#endif /* _ZIO_H */