Lines Matching refs:hdr
192 * arc_buf_hdr_t is cached. If the hdr is cached then the ARC allocates a new
194 * existing uncompressed arc_buf_t, decompresses the hdr's b_pdata buffer into a
195 * new data buffer, or shares the hdr's b_pdata buffer, depending on whether the
196 * hdr is compressed and the desired compression characteristics of the
199 * the last buffer in the hdr's b_buf list, however a shared compressed buf can
200 * be anywhere in the hdr's list.
202 * The diagram below shows an example of an uncompressed ARC hdr that is
233 * Writing to the ARC requires that the ARC first discard the hdr's b_pdata
240 * buffers which have been released (and hence have their own hdr, if there
241 * were originally other readers of the buf's original hdr). This ensures that
242 * the ARC only needs to update a single buf and its hdr after a write occurs.
951 #define HDR_IN_HASH_TABLE(hdr) ((hdr)->b_flags & ARC_FLAG_IN_HASH_TABLE)
952 #define HDR_IO_IN_PROGRESS(hdr) ((hdr)->b_flags & ARC_FLAG_IO_IN_PROGRESS)
953 #define HDR_IO_ERROR(hdr) ((hdr)->b_flags & ARC_FLAG_IO_ERROR)
954 #define HDR_PREFETCH(hdr) ((hdr)->b_flags & ARC_FLAG_PREFETCH)
955 #define HDR_COMPRESSION_ENABLED(hdr) \
956 ((hdr)->b_flags & ARC_FLAG_COMPRESSED_ARC)
958 #define HDR_L2CACHE(hdr) ((hdr)->b_flags & ARC_FLAG_L2CACHE)
959 #define HDR_L2_READING(hdr) \
960 (((hdr)->b_flags & ARC_FLAG_IO_IN_PROGRESS) && \
961 ((hdr)->b_flags & ARC_FLAG_HAS_L2HDR))
962 #define HDR_L2_WRITING(hdr) ((hdr)->b_flags & ARC_FLAG_L2_WRITING)
963 #define HDR_L2_EVICTED(hdr) ((hdr)->b_flags & ARC_FLAG_L2_EVICTED)
964 #define HDR_L2_WRITE_HEAD(hdr) ((hdr)->b_flags & ARC_FLAG_L2_WRITE_HEAD)
965 #define HDR_SHARED_DATA(hdr) ((hdr)->b_flags & ARC_FLAG_SHARED_DATA)
967 #define HDR_ISTYPE_METADATA(hdr) \
968 ((hdr)->b_flags & ARC_FLAG_BUFC_METADATA)
969 #define HDR_ISTYPE_DATA(hdr) (!HDR_ISTYPE_METADATA(hdr))
971 #define HDR_HAS_L1HDR(hdr) ((hdr)->b_flags & ARC_FLAG_HAS_L1HDR)
972 #define HDR_HAS_L2HDR(hdr) ((hdr)->b_flags & ARC_FLAG_HAS_L2HDR)
977 #define HDR_GET_COMPRESS(hdr) ((enum zio_compress)BF32_GET((hdr)->b_flags, \
979 #define HDR_SET_COMPRESS(hdr, cmp) BF32_SET((hdr)->b_flags, \
1019 #define HDR_LOCK(hdr) \
1020 (BUF_HASH_LOCK(BUF_HASH_INDEX(hdr->b_spa, &hdr->b_dva, hdr->b_birth)))
1104 static void arc_hdr_free_pdata(arc_buf_hdr_t *hdr);
1112 static inline void arc_hdr_set_flags(arc_buf_hdr_t *hdr, arc_flags_t flags);
1113 static inline void arc_hdr_clear_flags(arc_buf_hdr_t *hdr, arc_flags_t flags);
1135 #define HDR_EMPTY(hdr) \
1136 ((hdr)->b_dva.dva_word[0] == 0 && \
1137 (hdr)->b_dva.dva_word[1] == 0)
1139 #define HDR_EQUAL(spa, dva, birth, hdr) \
1140 ((hdr)->b_dva.dva_word[0] == (dva)->dva_word[0]) && \
1141 ((hdr)->b_dva.dva_word[1] == (dva)->dva_word[1]) && \
1142 ((hdr)->b_birth == birth) && ((hdr)->b_spa == spa)
1145 buf_discard_identity(arc_buf_hdr_t *hdr)
1147 hdr->b_dva.dva_word[0] = 0;
1148 hdr->b_dva.dva_word[1] = 0;
1149 hdr->b_birth = 0;
1159 arc_buf_hdr_t *hdr;
1162 for (hdr = buf_hash_table.ht_table[idx]; hdr != NULL;
1163 hdr = hdr->b_hash_next) {
1164 if (HDR_EQUAL(spa, dva, birth, hdr)) {
1166 return (hdr);
1182 buf_hash_insert(arc_buf_hdr_t *hdr, kmutex_t **lockp)
1184 uint64_t idx = BUF_HASH_INDEX(hdr->b_spa, &hdr->b_dva, hdr->b_birth);
1189 ASSERT(!DVA_IS_EMPTY(&hdr->b_dva));
1190 ASSERT(hdr->b_birth != 0);
1191 ASSERT(!HDR_IN_HASH_TABLE(hdr));
1202 if (HDR_EQUAL(hdr->b_spa, &hdr->b_dva, hdr->b_birth, fhdr))
1206 hdr->b_hash_next = buf_hash_table.ht_table[idx];
1207 buf_hash_table.ht_table[idx] = hdr;
1208 arc_hdr_set_flags(hdr, ARC_FLAG_IN_HASH_TABLE);
1226 buf_hash_remove(arc_buf_hdr_t *hdr)
1229 uint64_t idx = BUF_HASH_INDEX(hdr->b_spa, &hdr->b_dva, hdr->b_birth);
1232 ASSERT(HDR_IN_HASH_TABLE(hdr));
1235 while ((fhdr = *hdrp) != hdr) {
1239 *hdrp = hdr->b_hash_next;
1240 hdr->b_hash_next = NULL;
1241 arc_hdr_clear_flags(hdr, ARC_FLAG_IN_HASH_TABLE);
1280 arc_buf_hdr_t *hdr = vbuf;
1282 bzero(hdr, HDR_FULL_SIZE);
1283 cv_init(&hdr->b_l1hdr.b_cv, NULL, CV_DEFAULT, NULL);
1284 refcount_create(&hdr->b_l1hdr.b_refcnt);
1285 mutex_init(&hdr->b_l1hdr.b_freeze_lock, NULL, MUTEX_DEFAULT, NULL);
1286 multilist_link_init(&hdr->b_l1hdr.b_arc_node);
1296 arc_buf_hdr_t *hdr = vbuf;
1298 bzero(hdr, HDR_L2ONLY_SIZE);
1325 arc_buf_hdr_t *hdr = vbuf;
1327 ASSERT(HDR_EMPTY(hdr));
1328 cv_destroy(&hdr->b_l1hdr.b_cv);
1329 refcount_destroy(&hdr->b_l1hdr.b_refcnt);
1330 mutex_destroy(&hdr->b_l1hdr.b_freeze_lock);
1331 ASSERT(!multilist_link_active(&hdr->b_l1hdr.b_arc_node));
1339 arc_buf_hdr_t *hdr = vbuf;
1341 ASSERT(HDR_EMPTY(hdr));
1451 * It would be nice to assert arc_can_share() too, but the "hdr isn't
1463 arc_cksum_free(arc_buf_hdr_t *hdr)
1465 ASSERT(HDR_HAS_L1HDR(hdr));
1466 mutex_enter(&hdr->b_l1hdr.b_freeze_lock);
1467 if (hdr->b_l1hdr.b_freeze_cksum != NULL) {
1468 kmem_free(hdr->b_l1hdr.b_freeze_cksum, sizeof (zio_cksum_t));
1469 hdr->b_l1hdr.b_freeze_cksum = NULL;
1471 mutex_exit(&hdr->b_l1hdr.b_freeze_lock);
1475 * Return true iff at least one of the bufs on hdr is not compressed.
1478 arc_hdr_has_uncompressed_buf(arc_buf_hdr_t *hdr)
1480 for (arc_buf_t *b = hdr->b_l1hdr.b_buf; b != NULL; b = b->b_next) {
1490 * matches the checksum that is stored in the hdr. If there is no checksum,
1496 arc_buf_hdr_t *hdr = buf->b_hdr;
1503 ASSERT(hdr->b_l1hdr.b_freeze_cksum == NULL ||
1504 arc_hdr_has_uncompressed_buf(hdr));
1508 ASSERT(HDR_HAS_L1HDR(hdr));
1510 mutex_enter(&hdr->b_l1hdr.b_freeze_lock);
1511 if (hdr->b_l1hdr.b_freeze_cksum == NULL || HDR_IO_ERROR(hdr)) {
1512 mutex_exit(&hdr->b_l1hdr.b_freeze_lock);
1517 if (!ZIO_CHECKSUM_EQUAL(*hdr->b_l1hdr.b_freeze_cksum, zc))
1519 mutex_exit(&hdr->b_l1hdr.b_freeze_lock);
1523 arc_cksum_is_equal(arc_buf_hdr_t *hdr, zio_t *zio)
1529 VERIFY3U(BP_GET_PSIZE(zio->io_bp), ==, HDR_GET_PSIZE(hdr));
1543 if (!HDR_COMPRESSION_ENABLED(hdr) && compress != ZIO_COMPRESS_OFF) {
1544 ASSERT3U(HDR_GET_COMPRESS(hdr), ==, ZIO_COMPRESS_OFF);
1545 uint64_t lsize = HDR_GET_LSIZE(hdr);
1548 void *cbuf = zio_buf_alloc(HDR_GET_PSIZE(hdr));
1550 ASSERT3U(csize, <=, HDR_GET_PSIZE(hdr));
1551 if (csize < HDR_GET_PSIZE(hdr)) {
1558 * we rely on the fact that the hdr's psize
1564 bzero((char *)cbuf + csize, HDR_GET_PSIZE(hdr) - csize);
1565 csize = HDR_GET_PSIZE(hdr);
1567 zio_push_transform(zio, cbuf, csize, HDR_GET_PSIZE(hdr), NULL);
1592 * checksum and attaches it to the buf's hdr so that we can ensure that the buf
1594 * on the hdr, this is a no-op (we only checksum uncompressed bufs).
1599 arc_buf_hdr_t *hdr = buf->b_hdr;
1604 ASSERT(HDR_HAS_L1HDR(hdr));
1607 if (hdr->b_l1hdr.b_freeze_cksum != NULL) {
1608 ASSERT(arc_hdr_has_uncompressed_buf(hdr));
1609 mutex_exit(&hdr->b_l1hdr.b_freeze_lock);
1612 mutex_exit(&hdr->b_l1hdr.b_freeze_lock);
1617 hdr->b_l1hdr.b_freeze_cksum = kmem_alloc(sizeof (zio_cksum_t),
1620 hdr->b_l1hdr.b_freeze_cksum);
1621 mutex_exit(&hdr->b_l1hdr.b_freeze_lock);
1669 arc_buf_type(arc_buf_hdr_t *hdr)
1672 if (HDR_ISTYPE_METADATA(hdr)) {
1677 VERIFY3U(hdr->b_type, ==, type);
1706 arc_buf_hdr_t *hdr = buf->b_hdr;
1708 ASSERT3P(hdr->b_l1hdr.b_state, ==, arc_anon);
1709 ASSERT(!HDR_IO_IN_PROGRESS(hdr));
1718 ASSERT(hdr->b_l1hdr.b_freeze_cksum == NULL ||
1719 arc_hdr_has_uncompressed_buf(hdr));
1723 ASSERT(HDR_HAS_L1HDR(hdr));
1724 arc_cksum_free(hdr);
1726 mutex_enter(&hdr->b_l1hdr.b_freeze_lock);
1729 if (hdr->b_l1hdr.b_thawed != NULL)
1730 kmem_free(hdr->b_l1hdr.b_thawed, 1);
1731 hdr->b_l1hdr.b_thawed = kmem_alloc(1, KM_SLEEP);
1735 mutex_exit(&hdr->b_l1hdr.b_freeze_lock);
1743 arc_buf_hdr_t *hdr = buf->b_hdr;
1750 ASSERT(hdr->b_l1hdr.b_freeze_cksum == NULL ||
1751 arc_hdr_has_uncompressed_buf(hdr));
1755 hash_lock = HDR_LOCK(hdr);
1758 ASSERT(HDR_HAS_L1HDR(hdr));
1759 ASSERT(hdr->b_l1hdr.b_freeze_cksum != NULL ||
1760 hdr->b_l1hdr.b_state == arc_anon);
1769 * the hash_lock must be held or the hdr must be undiscoverable. This
1774 arc_hdr_set_flags(arc_buf_hdr_t *hdr, arc_flags_t flags)
1776 ASSERT(MUTEX_HELD(HDR_LOCK(hdr)) || HDR_EMPTY(hdr));
1777 hdr->b_flags |= flags;
1781 arc_hdr_clear_flags(arc_buf_hdr_t *hdr, arc_flags_t flags)
1783 ASSERT(MUTEX_HELD(HDR_LOCK(hdr)) || HDR_EMPTY(hdr));
1784 hdr->b_flags &= ~flags;
1795 arc_hdr_set_compress(arc_buf_hdr_t *hdr, enum zio_compress cmp)
1797 ASSERT(MUTEX_HELD(HDR_LOCK(hdr)) || HDR_EMPTY(hdr));
1806 if (!zfs_compressed_arc_enabled || HDR_GET_PSIZE(hdr) == 0) {
1807 arc_hdr_clear_flags(hdr, ARC_FLAG_COMPRESSED_ARC);
1808 HDR_SET_COMPRESS(hdr, ZIO_COMPRESS_OFF);
1809 ASSERT(!HDR_COMPRESSION_ENABLED(hdr));
1810 ASSERT3U(HDR_GET_COMPRESS(hdr), ==, ZIO_COMPRESS_OFF);
1812 arc_hdr_set_flags(hdr, ARC_FLAG_COMPRESSED_ARC);
1813 HDR_SET_COMPRESS(hdr, cmp);
1814 ASSERT3U(HDR_GET_COMPRESS(hdr), ==, cmp);
1815 ASSERT(HDR_COMPRESSION_ENABLED(hdr));
1820 * Looks for another buf on the same hdr which has the data decompressed, copies
1826 arc_buf_hdr_t *hdr = buf->b_hdr;
1829 ASSERT(HDR_HAS_L1HDR(hdr));
1833 for (arc_buf_t *from = hdr->b_l1hdr.b_buf; from != NULL;
1849 * checksum on the hdr either.
1851 EQUIV(!copied, hdr->b_l1hdr.b_freeze_cksum == NULL);
1859 * the hdr and update the hdr's b_freeze_cksum if necessary. If the buf and hdr
1864 * buf with uncompressed data. You can't request a compressed buf on a hdr with
1872 arc_buf_hdr_t *hdr = buf->b_hdr;
1873 boolean_t hdr_compressed = (HDR_GET_COMPRESS(hdr) != ZIO_COMPRESS_OFF);
1874 dmu_object_byteswap_t bswap = hdr->b_l1hdr.b_byteswap;
1882 bcopy(hdr->b_l1hdr.b_pdata, buf->b_data,
1888 ASSERT3U(HDR_GET_LSIZE(hdr), !=, HDR_GET_PSIZE(hdr));
1891 * If the buf is sharing its data with the hdr, unlink it and
1900 arc_get_data_buf(hdr, HDR_GET_LSIZE(hdr), buf);
1901 arc_hdr_clear_flags(hdr, ARC_FLAG_SHARED_DATA);
1904 ARCSTAT_INCR(arcstat_overhead_size, HDR_GET_LSIZE(hdr));
1907 arc_free_data_buf(hdr, buf->b_data, HDR_GET_PSIZE(hdr),
1910 arc_get_data_buf(hdr, HDR_GET_LSIZE(hdr), buf);
1914 HDR_GET_LSIZE(hdr) - HDR_GET_PSIZE(hdr));
1926 * bite the bullet and decompress the data from the hdr.
1930 ASSERT3P(hdr->b_l1hdr.b_freeze_cksum, !=, NULL);
1933 int error = zio_decompress_data(HDR_GET_COMPRESS(hdr),
1934 hdr->b_l1hdr.b_pdata, buf->b_data,
1935 HDR_GET_PSIZE(hdr), HDR_GET_LSIZE(hdr));
1943 "hdr %p, compress %d, psize %d, lsize %d",
1944 hdr, HDR_GET_COMPRESS(hdr),
1945 HDR_GET_PSIZE(hdr), HDR_GET_LSIZE(hdr));
1953 ASSERT(!HDR_SHARED_DATA(hdr));
1955 dmu_ot_byteswap[bswap].ob_func(buf->b_data, HDR_GET_LSIZE(hdr));
1958 /* Compute the hdr's checksum if necessary */
1974 arc_hdr_size(arc_buf_hdr_t *hdr)
1978 if (HDR_GET_COMPRESS(hdr) != ZIO_COMPRESS_OFF &&
1979 HDR_GET_PSIZE(hdr) > 0) {
1980 size = HDR_GET_PSIZE(hdr);
1982 ASSERT3U(HDR_GET_LSIZE(hdr), !=, 0);
1983 size = HDR_GET_LSIZE(hdr);
1990 * We account for the space used by the hdr and the arc buf individually
1994 arc_evictable_space_increment(arc_buf_hdr_t *hdr, arc_state_t *state)
1996 arc_buf_contents_t type = arc_buf_type(hdr);
1998 ASSERT(HDR_HAS_L1HDR(hdr));
2001 ASSERT0(hdr->b_l1hdr.b_bufcnt);
2002 ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL);
2003 ASSERT3P(hdr->b_l1hdr.b_pdata, ==, NULL);
2005 HDR_GET_LSIZE(hdr), hdr);
2010 if (hdr->b_l1hdr.b_pdata != NULL) {
2012 arc_hdr_size(hdr), hdr);
2014 for (arc_buf_t *buf = hdr->b_l1hdr.b_buf; buf != NULL;
2025 * We account for the space used by the hdr and the arc buf individually
2029 arc_evictable_space_decrement(arc_buf_hdr_t *hdr, arc_state_t *state)
2031 arc_buf_contents_t type = arc_buf_type(hdr);
2033 ASSERT(HDR_HAS_L1HDR(hdr));
2036 ASSERT0(hdr->b_l1hdr.b_bufcnt);
2037 ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL);
2038 ASSERT3P(hdr->b_l1hdr.b_pdata, ==, NULL);
2040 HDR_GET_LSIZE(hdr), hdr);
2045 if (hdr->b_l1hdr.b_pdata != NULL) {
2047 arc_hdr_size(hdr), hdr);
2049 for (arc_buf_t *buf = hdr->b_l1hdr.b_buf; buf != NULL;
2059 * Add a reference to this hdr indicating that someone is actively
2065 add_reference(arc_buf_hdr_t *hdr, void *tag)
2067 ASSERT(HDR_HAS_L1HDR(hdr));
2068 if (!MUTEX_HELD(HDR_LOCK(hdr))) {
2069 ASSERT(hdr->b_l1hdr.b_state == arc_anon);
2070 ASSERT(refcount_is_zero(&hdr->b_l1hdr.b_refcnt));
2071 ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL);
2074 arc_state_t *state = hdr->b_l1hdr.b_state;
2076 if ((refcount_add(&hdr->b_l1hdr.b_refcnt, tag) == 1) &&
2080 multilist_remove(&state->arcs_list[arc_buf_type(hdr)],
2081 hdr);
2082 arc_evictable_space_decrement(hdr, state);
2085 arc_hdr_clear_flags(hdr, ARC_FLAG_PREFETCH);
2090 * Remove a reference from this hdr. When the reference transitions from
2091 * 1 to 0 and we're not anonymous, then we add this hdr to the arc_state_t's
2095 remove_reference(arc_buf_hdr_t *hdr, kmutex_t *hash_lock, void *tag)
2098 arc_state_t *state = hdr->b_l1hdr.b_state;
2100 ASSERT(HDR_HAS_L1HDR(hdr));
2108 if (((cnt = refcount_remove(&hdr->b_l1hdr.b_refcnt, tag)) == 0) &&
2110 multilist_insert(&state->arcs_list[arc_buf_type(hdr)], hdr);
2111 ASSERT3U(hdr->b_l1hdr.b_bufcnt, >, 0);
2112 arc_evictable_space_increment(hdr, state);
2122 arc_change_state(arc_state_t *new_state, arc_buf_hdr_t *hdr,
2129 arc_buf_contents_t buftype = arc_buf_type(hdr);
2132 * We almost always have an L1 hdr here, since we call arc_hdr_realloc()
2134 * L1 hdr doesn't always exist when we change state to arc_anon before
2135 * destroying a header, in which case reallocating to add the L1 hdr is
2138 if (HDR_HAS_L1HDR(hdr)) {
2139 old_state = hdr->b_l1hdr.b_state;
2140 refcnt = refcount_count(&hdr->b_l1hdr.b_refcnt);
2141 bufcnt = hdr->b_l1hdr.b_bufcnt;
2142 update_old = (bufcnt > 0 || hdr->b_l1hdr.b_pdata != NULL);
2162 ASSERT(HDR_HAS_L1HDR(hdr));
2163 multilist_remove(&old_state->arcs_list[buftype], hdr);
2167 ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL);
2170 arc_evictable_space_decrement(hdr, old_state);
2180 ASSERT(HDR_HAS_L1HDR(hdr));
2181 multilist_insert(&new_state->arcs_list[buftype], hdr);
2185 ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL);
2188 arc_evictable_space_increment(hdr, new_state);
2192 ASSERT(!HDR_EMPTY(hdr));
2193 if (new_state == arc_anon && HDR_IN_HASH_TABLE(hdr))
2194 buf_hash_remove(hdr);
2199 ASSERT(HDR_HAS_L1HDR(hdr));
2211 HDR_GET_LSIZE(hdr), hdr);
2212 ASSERT3P(hdr->b_l1hdr.b_pdata, ==, NULL);
2221 for (arc_buf_t *buf = hdr->b_l1hdr.b_buf; buf != NULL;
2228 * block with the hdr, the owner of the
2229 * reference belongs to the hdr. Only
2241 if (hdr->b_l1hdr.b_pdata != NULL) {
2243 arc_hdr_size(hdr), hdr);
2251 ASSERT(HDR_HAS_L1HDR(hdr));
2254 ASSERT3P(hdr->b_l1hdr.b_pdata, ==, NULL);
2265 HDR_GET_LSIZE(hdr), hdr);
2274 for (arc_buf_t *buf = hdr->b_l1hdr.b_buf; buf != NULL;
2281 * block with the hdr, the owner of the
2282 * reference belongs to the hdr. Only
2294 ASSERT3P(hdr->b_l1hdr.b_pdata, !=, NULL);
2296 &old_state->arcs_size, arc_hdr_size(hdr), hdr);
2300 if (HDR_HAS_L1HDR(hdr))
2301 hdr->b_l1hdr.b_state = new_state;
2375 * Given a hdr and a buf, returns whether that buf can share its b_data buffer
2376 * with the hdr's b_pdata.
2379 arc_can_share(arc_buf_hdr_t *hdr, arc_buf_t *buf)
2382 * The criteria for sharing a hdr's data are:
2383 * 1. the hdr's compression matches the buf's compression
2384 * 2. the hdr doesn't need to be byteswapped
2385 * 3. the hdr isn't already being shared
2386 * 4. the buf is either compressed or it is the last buf in the hdr list
2389 * bufs must be the final buf in the hdr's b_buf list. Reading this, you
2392 * a shared uncompressed buf anyway (because the hdr must be compressed
2402 ASSERT3P(buf->b_hdr, ==, hdr);
2403 boolean_t hdr_compressed = HDR_GET_COMPRESS(hdr) != ZIO_COMPRESS_OFF;
2406 hdr->b_l1hdr.b_byteswap == DMU_BSWAP_NUMFUNCS &&
2407 !HDR_SHARED_DATA(hdr) &&
2412 * Allocate a buf for this hdr. If you care about the data that's in the hdr,
2417 arc_buf_alloc_impl(arc_buf_hdr_t *hdr, void *tag, boolean_t compressed,
2422 ASSERT(HDR_HAS_L1HDR(hdr));
2423 ASSERT3U(HDR_GET_LSIZE(hdr), >, 0);
2424 VERIFY(hdr->b_type == ARC_BUFC_DATA ||
2425 hdr->b_type == ARC_BUFC_METADATA);
2430 buf->b_hdr = hdr;
2432 buf->b_next = hdr->b_l1hdr.b_buf;
2435 add_reference(hdr, tag);
2438 * We're about to change the hdr's b_flags. We must either
2441 ASSERT(MUTEX_HELD(HDR_LOCK(hdr)) || HDR_EMPTY(hdr));
2444 * Only honor requests for compressed bufs if the hdr is actually
2447 if (compressed && HDR_GET_COMPRESS(hdr) != ZIO_COMPRESS_OFF)
2451 * If the hdr's data can be shared then we share the data buffer and
2452 * set the appropriate bit in the hdr's b_flags to indicate the hdr is
2457 * hdr -> buf instead of the usual buf -> hdr: the hdr can't be actively
2459 * arc_write() then the hdr's data buffer will be released when the
2462 boolean_t can_share = arc_can_share(hdr, buf) && !HDR_L2_WRITING(hdr);
2466 buf->b_data = hdr->b_l1hdr.b_pdata;
2468 arc_hdr_set_flags(hdr, ARC_FLAG_SHARED_DATA);
2471 arc_get_data_buf(hdr, arc_buf_size(buf), buf);
2476 hdr->b_l1hdr.b_buf = buf;
2477 hdr->b_l1hdr.b_bufcnt += 1;
2480 * If the user wants the data from the hdr, we need to either copy or
2537 arc_buf_hdr_t *hdr = buf->b_hdr;
2540 ASSERT(HDR_HAS_L1HDR(hdr));
2541 (void) refcount_add(&hdr->b_l1hdr.b_refcnt, tag);
2542 (void) refcount_remove(&hdr->b_l1hdr.b_refcnt, arc_onloan_tag);
2551 arc_buf_hdr_t *hdr = buf->b_hdr;
2554 ASSERT(HDR_HAS_L1HDR(hdr));
2555 (void) refcount_add(&hdr->b_l1hdr.b_refcnt, arc_onloan_tag);
2556 (void) refcount_remove(&hdr->b_l1hdr.b_refcnt, tag);
2575 arc_hdr_free_on_write(arc_buf_hdr_t *hdr)
2577 arc_state_t *state = hdr->b_l1hdr.b_state;
2578 arc_buf_contents_t type = arc_buf_type(hdr);
2579 uint64_t size = arc_hdr_size(hdr);
2582 if (multilist_link_active(&hdr->b_l1hdr.b_arc_node)) {
2583 ASSERT(refcount_is_zero(&hdr->b_l1hdr.b_refcnt));
2587 size, hdr);
2589 (void) refcount_remove_many(&state->arcs_size, size, hdr);
2591 l2arc_free_data_on_write(hdr->b_l1hdr.b_pdata, size, type);
2595 * Share the arc_buf_t's data with the hdr. Whenever we are sharing the
2596 * data buffer, we transfer the refcount ownership to the hdr and update
2600 arc_share_buf(arc_buf_hdr_t *hdr, arc_buf_t *buf)
2602 arc_state_t *state = hdr->b_l1hdr.b_state;
2604 ASSERT(arc_can_share(hdr, buf));
2605 ASSERT3P(hdr->b_l1hdr.b_pdata, ==, NULL);
2606 ASSERT(MUTEX_HELD(HDR_LOCK(hdr)) || HDR_EMPTY(hdr));
2610 * refcount ownership to the hdr since it always owns
2613 refcount_transfer_ownership(&state->arcs_size, buf, hdr);
2614 hdr->b_l1hdr.b_pdata = buf->b_data;
2615 arc_hdr_set_flags(hdr, ARC_FLAG_SHARED_DATA);
2619 * Since we've transferred ownership to the hdr we need
2623 ARCSTAT_INCR(arcstat_compressed_size, arc_hdr_size(hdr));
2624 ARCSTAT_INCR(arcstat_uncompressed_size, HDR_GET_LSIZE(hdr));
2629 arc_unshare_buf(arc_buf_hdr_t *hdr, arc_buf_t *buf)
2631 arc_state_t *state = hdr->b_l1hdr.b_state;
2634 ASSERT3P(hdr->b_l1hdr.b_pdata, !=, NULL);
2635 ASSERT(MUTEX_HELD(HDR_LOCK(hdr)) || HDR_EMPTY(hdr));
2641 refcount_transfer_ownership(&state->arcs_size, hdr, buf);
2642 arc_hdr_clear_flags(hdr, ARC_FLAG_SHARED_DATA);
2643 hdr->b_l1hdr.b_pdata = NULL;
2648 * the arc buf and the hdr, count it as overhead.
2650 ARCSTAT_INCR(arcstat_compressed_size, -arc_hdr_size(hdr));
2651 ARCSTAT_INCR(arcstat_uncompressed_size, -HDR_GET_LSIZE(hdr));
2656 * Remove an arc_buf_t from the hdr's buf list and return the last
2661 arc_buf_remove(arc_buf_hdr_t *hdr, arc_buf_t *buf)
2663 ASSERT(HDR_HAS_L1HDR(hdr));
2664 ASSERT(MUTEX_HELD(HDR_LOCK(hdr)) || HDR_EMPTY(hdr));
2666 arc_buf_t **bufp = &hdr->b_l1hdr.b_buf;
2670 * Remove the buf from the hdr list and locate the last
2689 IMPLY(hdr->b_l1hdr.b_bufcnt > 0, lastbuf != NULL);
2690 IMPLY(hdr->b_l1hdr.b_bufcnt > 0, hdr->b_l1hdr.b_buf != NULL);
2703 arc_buf_hdr_t *hdr = buf->b_hdr;
2707 * sharing this with the hdr. If we are sharing it with the hdr, the
2708 * hdr is responsible for doing the free.
2712 * We're about to change the hdr's b_flags. We must either
2715 ASSERT(MUTEX_HELD(HDR_LOCK(hdr)) || HDR_EMPTY(hdr));
2721 arc_hdr_clear_flags(hdr, ARC_FLAG_SHARED_DATA);
2724 arc_free_data_buf(hdr, buf->b_data, size, buf);
2729 ASSERT(hdr->b_l1hdr.b_bufcnt > 0);
2730 hdr->b_l1hdr.b_bufcnt -= 1;
2733 arc_buf_t *lastbuf = arc_buf_remove(hdr, buf);
2738 * hdr, then reassign the hdr's b_pdata to share it with the new
2740 * the last one on the hdr's buffer list.
2750 /* hdr is uncompressed so can't have compressed buf */
2753 ASSERT3P(hdr->b_l1hdr.b_pdata, !=, NULL);
2754 arc_hdr_free_pdata(hdr);
2758 * last buffer and the hdr. The data would have
2760 * ownership to the hdr since it's now being shared.
2762 arc_share_buf(hdr, lastbuf);
2764 } else if (HDR_SHARED_DATA(hdr)) {
2770 * we rely on the hdr's compression flags to determine
2775 HDR_GET_COMPRESS(hdr) != ZIO_COMPRESS_OFF);
2780 * this hdr.
2782 if (!arc_hdr_has_uncompressed_buf(hdr)) {
2783 arc_cksum_free(hdr);
2792 arc_hdr_alloc_pdata(arc_buf_hdr_t *hdr)
2794 ASSERT3U(HDR_GET_LSIZE(hdr), >, 0);
2795 ASSERT(HDR_HAS_L1HDR(hdr));
2796 ASSERT(!HDR_SHARED_DATA(hdr));
2798 ASSERT3P(hdr->b_l1hdr.b_pdata, ==, NULL);
2799 hdr->b_l1hdr.b_pdata = arc_get_data_buf(hdr, arc_hdr_size(hdr), hdr);
2800 hdr->b_l1hdr.b_byteswap = DMU_BSWAP_NUMFUNCS;
2801 ASSERT3P(hdr->b_l1hdr.b_pdata, !=, NULL);
2803 ARCSTAT_INCR(arcstat_compressed_size, arc_hdr_size(hdr));
2804 ARCSTAT_INCR(arcstat_uncompressed_size, HDR_GET_LSIZE(hdr));
2808 arc_hdr_free_pdata(arc_buf_hdr_t *hdr)
2810 ASSERT(HDR_HAS_L1HDR(hdr));
2811 ASSERT3P(hdr->b_l1hdr.b_pdata, !=, NULL);
2814 * If the hdr is currently being written to the l2arc then
2819 if (HDR_L2_WRITING(hdr)) {
2820 arc_hdr_free_on_write(hdr);
2823 arc_free_data_buf(hdr, hdr->b_l1hdr.b_pdata,
2824 arc_hdr_size(hdr), hdr);
2826 hdr->b_l1hdr.b_pdata = NULL;
2827 hdr->b_l1hdr.b_byteswap = DMU_BSWAP_NUMFUNCS;
2829 ARCSTAT_INCR(arcstat_compressed_size, -arc_hdr_size(hdr));
2830 ARCSTAT_INCR(arcstat_uncompressed_size, -HDR_GET_LSIZE(hdr));
2837 arc_buf_hdr_t *hdr;
2841 hdr = kmem_cache_alloc(hdr_full_cache, KM_PUSHPAGE);
2842 ASSERT(HDR_EMPTY(hdr));
2843 ASSERT3P(hdr->b_l1hdr.b_freeze_cksum, ==, NULL);
2844 ASSERT3P(hdr->b_l1hdr.b_thawed, ==, NULL);
2845 HDR_SET_PSIZE(hdr, psize);
2846 HDR_SET_LSIZE(hdr, lsize);
2847 hdr->b_spa = spa;
2848 hdr->b_type = type;
2849 hdr->b_flags = 0;
2850 arc_hdr_set_flags(hdr, arc_bufc_to_flags(type) | ARC_FLAG_HAS_L1HDR);
2851 arc_hdr_set_compress(hdr, compression_type);
2853 hdr->b_l1hdr.b_state = arc_anon;
2854 hdr->b_l1hdr.b_arc_access = 0;
2855 hdr->b_l1hdr.b_bufcnt = 0;
2856 hdr->b_l1hdr.b_buf = NULL;
2859 * Allocate the hdr's buffer. This will contain either
2863 arc_hdr_alloc_pdata(hdr);
2864 ASSERT(refcount_is_zero(&hdr->b_l1hdr.b_refcnt));
2866 return (hdr);
2877 arc_hdr_realloc(arc_buf_hdr_t *hdr, kmem_cache_t *old, kmem_cache_t *new)
2879 ASSERT(HDR_HAS_L2HDR(hdr));
2882 l2arc_dev_t *dev = hdr->b_l2hdr.b_dev;
2889 ASSERT(MUTEX_HELD(HDR_LOCK(hdr)));
2890 buf_hash_remove(hdr);
2892 bcopy(hdr, nhdr, HDR_L2ONLY_SIZE);
2906 ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL);
2907 ASSERT0(hdr->b_l1hdr.b_bufcnt);
2908 ASSERT3P(hdr->b_l1hdr.b_freeze_cksum, ==, NULL);
2917 ASSERT(!multilist_link_active(&hdr->b_l1hdr.b_arc_node));
2925 VERIFY(!HDR_L2_WRITING(hdr));
2926 VERIFY3P(hdr->b_l1hdr.b_pdata, ==, NULL);
2929 if (hdr->b_l1hdr.b_thawed != NULL) {
2930 kmem_free(hdr->b_l1hdr.b_thawed, 1);
2931 hdr->b_l1hdr.b_thawed = NULL;
2943 ASSERT(list_link_active(&hdr->b_l2hdr.b_l2node));
2953 list_insert_after(&dev->l2ad_buflist, hdr, nhdr);
2954 list_remove(&dev->l2ad_buflist, hdr);
2966 (void) refcount_remove_many(&dev->l2ad_alloc, arc_hdr_size(hdr), hdr);
2969 buf_discard_identity(hdr);
2970 kmem_cache_free(old, hdr);
2982 arc_buf_hdr_t *hdr = arc_hdr_alloc(spa_load_guid(spa), size, size,
2984 ASSERT(!MUTEX_HELD(HDR_LOCK(hdr)));
2987 VERIFY0(arc_buf_alloc_impl(hdr, tag, B_FALSE, B_FALSE, &buf));
3006 arc_buf_hdr_t *hdr = arc_hdr_alloc(spa_load_guid(spa), psize, lsize,
3008 ASSERT(!MUTEX_HELD(HDR_LOCK(hdr)));
3011 VERIFY0(arc_buf_alloc_impl(hdr, tag, B_TRUE, B_FALSE, &buf));
3013 ASSERT3P(hdr->b_l1hdr.b_freeze_cksum, ==, NULL);
3019 arc_hdr_l2hdr_destroy(arc_buf_hdr_t *hdr)
3021 l2arc_buf_hdr_t *l2hdr = &hdr->b_l2hdr;
3023 uint64_t asize = arc_hdr_size(hdr);
3026 ASSERT(HDR_HAS_L2HDR(hdr));
3028 list_remove(&dev->l2ad_buflist, hdr);
3031 ARCSTAT_INCR(arcstat_l2_size, -HDR_GET_LSIZE(hdr));
3035 (void) refcount_remove_many(&dev->l2ad_alloc, asize, hdr);
3036 arc_hdr_clear_flags(hdr, ARC_FLAG_HAS_L2HDR);
3040 arc_hdr_destroy(arc_buf_hdr_t *hdr)
3042 if (HDR_HAS_L1HDR(hdr)) {
3043 ASSERT(hdr->b_l1hdr.b_buf == NULL ||
3044 hdr->b_l1hdr.b_bufcnt > 0);
3045 ASSERT(refcount_is_zero(&hdr->b_l1hdr.b_refcnt));
3046 ASSERT3P(hdr->b_l1hdr.b_state, ==, arc_anon);
3048 ASSERT(!HDR_IO_IN_PROGRESS(hdr));
3049 ASSERT(!HDR_IN_HASH_TABLE(hdr));
3051 if (!HDR_EMPTY(hdr))
3052 buf_discard_identity(hdr);
3054 if (HDR_HAS_L2HDR(hdr)) {
3055 l2arc_dev_t *dev = hdr->b_l2hdr.b_dev;
3070 if (HDR_HAS_L2HDR(hdr))
3071 arc_hdr_l2hdr_destroy(hdr);
3077 if (HDR_HAS_L1HDR(hdr)) {
3078 arc_cksum_free(hdr);
3080 while (hdr->b_l1hdr.b_buf != NULL)
3081 arc_buf_destroy_impl(hdr->b_l1hdr.b_buf);
3084 if (hdr->b_l1hdr.b_thawed != NULL) {
3085 kmem_free(hdr->b_l1hdr.b_thawed, 1);
3086 hdr->b_l1hdr.b_thawed = NULL;
3090 if (hdr->b_l1hdr.b_pdata != NULL) {
3091 arc_hdr_free_pdata(hdr);
3095 ASSERT3P(hdr->b_hash_next, ==, NULL);
3096 if (HDR_HAS_L1HDR(hdr)) {
3097 ASSERT(!multilist_link_active(&hdr->b_l1hdr.b_arc_node));
3098 ASSERT3P(hdr->b_l1hdr.b_acb, ==, NULL);
3099 kmem_cache_free(hdr_full_cache, hdr);
3101 kmem_cache_free(hdr_l2only_cache, hdr);
3108 arc_buf_hdr_t *hdr = buf->b_hdr;
3109 kmutex_t *hash_lock = HDR_LOCK(hdr);
3111 if (hdr->b_l1hdr.b_state == arc_anon) {
3112 ASSERT3U(hdr->b_l1hdr.b_bufcnt, ==, 1);
3113 ASSERT(!HDR_IO_IN_PROGRESS(hdr));
3114 VERIFY0(remove_reference(hdr, NULL, tag));
3115 arc_hdr_destroy(hdr);
3120 ASSERT3P(hdr, ==, buf->b_hdr);
3121 ASSERT(hdr->b_l1hdr.b_bufcnt > 0);
3122 ASSERT3P(hash_lock, ==, HDR_LOCK(hdr));
3123 ASSERT3P(hdr->b_l1hdr.b_state, !=, arc_anon);
3126 (void) remove_reference(hdr, hash_lock, tag);
3144 arc_evict_hdr(arc_buf_hdr_t *hdr, kmutex_t *hash_lock)
3150 ASSERT(HDR_HAS_L1HDR(hdr));
3152 state = hdr->b_l1hdr.b_state;
3154 ASSERT(!HDR_IO_IN_PROGRESS(hdr));
3155 ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL);
3164 if (HDR_HAS_L2HDR(hdr) && HDR_L2_WRITING(hdr)) {
3170 bytes_evicted += HDR_GET_LSIZE(hdr);
3172 DTRACE_PROBE1(arc__delete, arc_buf_hdr_t *, hdr);
3174 ASSERT3P(hdr->b_l1hdr.b_pdata, ==, NULL);
3175 if (HDR_HAS_L2HDR(hdr)) {
3180 arc_change_state(arc_l2c_only, hdr, hash_lock);
3185 hdr = arc_hdr_realloc(hdr, hdr_full_cache,
3188 arc_change_state(arc_anon, hdr, hash_lock);
3189 arc_hdr_destroy(hdr);
3198 if (HDR_IO_IN_PROGRESS(hdr) ||
3199 ((hdr->b_flags & (ARC_FLAG_PREFETCH | ARC_FLAG_INDIRECT)) &&
3200 ddi_get_lbolt() - hdr->b_l1hdr.b_arc_access <
3206 ASSERT0(refcount_count(&hdr->b_l1hdr.b_refcnt));
3207 while (hdr->b_l1hdr.b_buf) {
3208 arc_buf_t *buf = hdr->b_l1hdr.b_buf;
3214 bytes_evicted += HDR_GET_LSIZE(hdr);
3219 if (HDR_HAS_L2HDR(hdr)) {
3220 ARCSTAT_INCR(arcstat_evict_l2_cached, HDR_GET_LSIZE(hdr));
3222 if (l2arc_write_eligible(hdr->b_spa, hdr)) {
3224 HDR_GET_LSIZE(hdr));
3227 HDR_GET_LSIZE(hdr));
3231 if (hdr->b_l1hdr.b_bufcnt == 0) {
3232 arc_cksum_free(hdr);
3234 bytes_evicted += arc_hdr_size(hdr);
3237 * If this hdr is being evicted and has a compressed
3242 arc_hdr_free_pdata(hdr);
3244 arc_change_state(evicted_state, hdr, hash_lock);
3245 ASSERT(HDR_IN_HASH_TABLE(hdr));
3246 arc_hdr_set_flags(hdr, ARC_FLAG_IN_HASH_TABLE);
3247 DTRACE_PROBE1(arc__evict, arc_buf_hdr_t *, hdr);
3259 arc_buf_hdr_t *hdr;
3268 for (hdr = multilist_sublist_prev(mls, marker); hdr != NULL;
3269 hdr = multilist_sublist_prev(mls, marker)) {
3276 * forward. Since we're not holding hdr's hash lock, we
3277 * must be very careful and not remove 'hdr' from the
3279 * 'hdr' as not being on a sublist when they call the
3296 if (hdr->b_spa == 0)
3300 if (spa != 0 && hdr->b_spa != spa) {
3305 hash_lock = HDR_LOCK(hdr);
3319 uint64_t evicted = arc_evict_hdr(hdr, hash_lock);
4255 arc_get_data_buf(arc_buf_hdr_t *hdr, uint64_t size, void *tag)
4258 arc_state_t *state = hdr->b_l1hdr.b_state;
4259 arc_buf_contents_t type = arc_buf_type(hdr);
4299 VERIFY3U(hdr->b_type, ==, type);
4326 if (multilist_link_active(&hdr->b_l1hdr.b_arc_node)) {
4327 ASSERT(refcount_is_zero(&hdr->b_l1hdr.b_refcnt));
4336 if (arc_size < arc_c && hdr->b_l1hdr.b_state == arc_anon &&
4348 arc_free_data_buf(arc_buf_hdr_t *hdr, void *data, uint64_t size, void *tag)
4350 arc_state_t *state = hdr->b_l1hdr.b_state;
4351 arc_buf_contents_t type = arc_buf_type(hdr);
4354 if (multilist_link_active(&hdr->b_l1hdr.b_arc_node)) {
4355 ASSERT(refcount_is_zero(&hdr->b_l1hdr.b_refcnt));
4363 VERIFY3U(hdr->b_type, ==, type);
4379 arc_access(arc_buf_hdr_t *hdr, kmutex_t *hash_lock)
4384 ASSERT(HDR_HAS_L1HDR(hdr));
4386 if (hdr->b_l1hdr.b_state == arc_anon) {
4393 ASSERT0(hdr->b_l1hdr.b_arc_access);
4394 hdr->b_l1hdr.b_arc_access = ddi_get_lbolt();
4395 DTRACE_PROBE1(new_state__mru, arc_buf_hdr_t *, hdr);
4396 arc_change_state(arc_mru, hdr, hash_lock);
4398 } else if (hdr->b_l1hdr.b_state == arc_mru) {
4409 if (HDR_PREFETCH(hdr)) {
4410 if (refcount_count(&hdr->b_l1hdr.b_refcnt) == 0) {
4413 &hdr->b_l1hdr.b_arc_node));
4415 arc_hdr_clear_flags(hdr, ARC_FLAG_PREFETCH);
4418 hdr->b_l1hdr.b_arc_access = now;
4427 if (now > hdr->b_l1hdr.b_arc_access + ARC_MINTIME) {
4433 hdr->b_l1hdr.b_arc_access = now;
4434 DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, hdr);
4435 arc_change_state(arc_mfu, hdr, hash_lock);
4438 } else if (hdr->b_l1hdr.b_state == arc_mru_ghost) {
4446 if (HDR_PREFETCH(hdr)) {
4448 if (refcount_count(&hdr->b_l1hdr.b_refcnt) > 0)
4449 arc_hdr_clear_flags(hdr, ARC_FLAG_PREFETCH);
4450 DTRACE_PROBE1(new_state__mru, arc_buf_hdr_t *, hdr);
4453 DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, hdr);
4456 hdr->b_l1hdr.b_arc_access = ddi_get_lbolt();
4457 arc_change_state(new_state, hdr, hash_lock);
4460 } else if (hdr->b_l1hdr.b_state == arc_mfu) {
4470 if ((HDR_PREFETCH(hdr)) != 0) {
4471 ASSERT(refcount_is_zero(&hdr->b_l1hdr.b_refcnt));
4473 ASSERT(multilist_link_active(&hdr->b_l1hdr.b_arc_node));
4476 hdr->b_l1hdr.b_arc_access = ddi_get_lbolt();
4477 } else if (hdr->b_l1hdr.b_state == arc_mfu_ghost) {
4485 if (HDR_PREFETCH(hdr)) {
4490 ASSERT0(refcount_count(&hdr->b_l1hdr.b_refcnt));
4494 hdr->b_l1hdr.b_arc_access = ddi_get_lbolt();
4495 DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, hdr);
4496 arc_change_state(new_state, hdr, hash_lock);
4499 } else if (hdr->b_l1hdr.b_state == arc_l2c_only) {
4504 hdr->b_l1hdr.b_arc_access = ddi_get_lbolt();
4505 DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, hdr);
4506 arc_change_state(arc_mfu, hdr, hash_lock);
4537 arc_hdr_verify(arc_buf_hdr_t *hdr, blkptr_t *bp)
4540 ASSERT3U(HDR_GET_PSIZE(hdr), ==, 0);
4541 ASSERT3U(HDR_GET_COMPRESS(hdr), ==, ZIO_COMPRESS_OFF);
4543 if (HDR_COMPRESSION_ENABLED(hdr)) {
4544 ASSERT3U(HDR_GET_COMPRESS(hdr), ==,
4547 ASSERT3U(HDR_GET_LSIZE(hdr), ==, BP_GET_LSIZE(bp));
4548 ASSERT3U(HDR_GET_PSIZE(hdr), ==, BP_GET_PSIZE(bp));
4555 arc_buf_hdr_t *hdr = zio->io_private;
4563 * The hdr was inserted into hash-table and removed from lists
4570 if (HDR_IN_HASH_TABLE(hdr)) {
4571 ASSERT3U(hdr->b_birth, ==, BP_PHYSICAL_BIRTH(zio->io_bp));
4572 ASSERT3U(hdr->b_dva.dva_word[0], ==,
4574 ASSERT3U(hdr->b_dva.dva_word[1], ==,
4577 arc_buf_hdr_t *found = buf_hash_find(hdr->b_spa, zio->io_bp,
4580 ASSERT((found == hdr &&
4581 DVA_EQUAL(&hdr->b_dva, BP_IDENTITY(zio->io_bp))) ||
4582 (found == hdr && HDR_L2_READING(hdr)));
4590 hdr->b_l1hdr.b_byteswap = DMU_BSWAP_UINT64;
4592 hdr->b_l1hdr.b_byteswap =
4596 hdr->b_l1hdr.b_byteswap = DMU_BSWAP_NUMFUNCS;
4600 arc_hdr_clear_flags(hdr, ARC_FLAG_L2_EVICTED);
4601 if (l2arc_noprefetch && HDR_PREFETCH(hdr))
4602 arc_hdr_clear_flags(hdr, ARC_FLAG_L2CACHE);
4604 callback_list = hdr->b_l1hdr.b_acb;
4607 if (hash_lock && no_zio_error && hdr->b_l1hdr.b_state == arc_anon) {
4614 arc_access(hdr, hash_lock);
4631 int error = arc_buf_alloc_impl(hdr, acb->acb_private,
4637 hdr->b_l1hdr.b_acb = NULL;
4638 arc_hdr_clear_flags(hdr, ARC_FLAG_IO_IN_PROGRESS);
4640 ASSERT(HDR_PREFETCH(hdr));
4641 ASSERT0(hdr->b_l1hdr.b_bufcnt);
4642 ASSERT3P(hdr->b_l1hdr.b_pdata, !=, NULL);
4645 ASSERT(refcount_is_zero(&hdr->b_l1hdr.b_refcnt) ||
4649 arc_hdr_verify(hdr, zio->io_bp);
4651 arc_hdr_set_flags(hdr, ARC_FLAG_IO_ERROR);
4652 if (hdr->b_l1hdr.b_state != arc_anon)
4653 arc_change_state(arc_anon, hdr, hash_lock);
4654 if (HDR_IN_HASH_TABLE(hdr))
4655 buf_hash_remove(hdr);
4656 freeable = refcount_is_zero(&hdr->b_l1hdr.b_refcnt);
4661 * that the hdr (and hence the cv) might be freed before we get to
4664 cv_broadcast(&hdr->b_l1hdr.b_cv);
4675 ASSERT3P(hdr->b_l1hdr.b_state, ==, arc_anon);
4676 freeable = refcount_is_zero(&hdr->b_l1hdr.b_refcnt);
4694 arc_hdr_destroy(hdr);
4720 arc_buf_hdr_t *hdr = NULL;
4735 hdr = buf_hash_find(guid, bp, &hash_lock);
4738 if (hdr != NULL && HDR_HAS_L1HDR(hdr) && hdr->b_l1hdr.b_pdata != NULL) {
4742 if (HDR_IO_IN_PROGRESS(hdr)) {
4744 if ((hdr->b_flags & ARC_FLAG_PRIO_ASYNC_READ) &&
4767 arc_buf_hdr_t *, hdr);
4770 if (hdr->b_flags & ARC_FLAG_PREDICTIVE_PREFETCH) {
4771 arc_hdr_clear_flags(hdr,
4776 cv_wait(&hdr->b_l1hdr.b_cv, hash_lock);
4795 acb->acb_next = hdr->b_l1hdr.b_acb;
4796 hdr->b_l1hdr.b_acb = acb;
4804 ASSERT(hdr->b_l1hdr.b_state == arc_mru ||
4805 hdr->b_l1hdr.b_state == arc_mfu);
4808 if (hdr->b_flags & ARC_FLAG_PREDICTIVE_PREFETCH) {
4816 arc_buf_hdr_t *, hdr);
4819 arc_hdr_clear_flags(hdr,
4825 VERIFY0(arc_buf_alloc_impl(hdr, private,
4828 refcount_count(&hdr->b_l1hdr.b_refcnt) == 0) {
4829 arc_hdr_set_flags(hdr, ARC_FLAG_PREFETCH);
4831 DTRACE_PROBE1(arc__hit, arc_buf_hdr_t *, hdr);
4832 arc_access(hdr, hash_lock);
4834 arc_hdr_set_flags(hdr, ARC_FLAG_L2CACHE);
4837 ARCSTAT_CONDSTAT(!HDR_PREFETCH(hdr),
4838 demand, prefetch, !HDR_ISTYPE_METADATA(hdr),
4852 if (hdr == NULL) {
4856 hdr = arc_hdr_alloc(spa_load_guid(spa), psize, lsize,
4860 hdr->b_dva = *BP_IDENTITY(bp);
4861 hdr->b_birth = BP_PHYSICAL_BIRTH(bp);
4862 exists = buf_hash_insert(hdr, &hash_lock);
4867 buf_discard_identity(hdr);
4868 arc_hdr_destroy(hdr);
4874 * (and thus didn't have an L1 hdr), we realloc the
4875 * header to add an L1 hdr.
4877 if (!HDR_HAS_L1HDR(hdr)) {
4878 hdr = arc_hdr_realloc(hdr, hdr_l2only_cache,
4881 ASSERT3P(hdr->b_l1hdr.b_pdata, ==, NULL);
4882 ASSERT(GHOST_STATE(hdr->b_l1hdr.b_state));
4883 ASSERT(!HDR_IO_IN_PROGRESS(hdr));
4884 ASSERT(refcount_is_zero(&hdr->b_l1hdr.b_refcnt));
4885 ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL);
4886 ASSERT3P(hdr->b_l1hdr.b_freeze_cksum, ==, NULL);
4890 * This hdr is in the ghost list so we access it
4898 arc_access(hdr, hash_lock);
4899 arc_hdr_alloc_pdata(hdr);
4901 ASSERT3P(hdr->b_l1hdr.b_pdata, !=, NULL);
4902 size = arc_hdr_size(hdr);
4905 * If compression is enabled on the hdr, then will do
4906 * RAW I/O and will store the compressed data in the hdr's
4907 * data block. Otherwise, the hdr's data block will contain
4910 if (HDR_GET_COMPRESS(hdr) != ZIO_COMPRESS_OFF) {
4915 arc_hdr_set_flags(hdr, ARC_FLAG_PREFETCH);
4917 arc_hdr_set_flags(hdr, ARC_FLAG_L2CACHE);
4919 arc_hdr_set_flags(hdr, ARC_FLAG_INDIRECT);
4921 arc_hdr_set_flags(hdr, ARC_FLAG_PREDICTIVE_PREFETCH);
4922 ASSERT(!GHOST_STATE(hdr->b_l1hdr.b_state));
4929 ASSERT3P(hdr->b_l1hdr.b_acb, ==, NULL);
4930 hdr->b_l1hdr.b_acb = acb;
4931 arc_hdr_set_flags(hdr, ARC_FLAG_IO_IN_PROGRESS);
4933 if (HDR_HAS_L2HDR(hdr) &&
4934 (vd = hdr->b_l2hdr.b_dev->l2ad_vdev) != NULL) {
4935 devw = hdr->b_l2hdr.b_dev->l2ad_writing;
4936 addr = hdr->b_l2hdr.b_daddr;
4946 arc_hdr_set_flags(hdr, ARC_FLAG_PRIO_ASYNC_READ);
4948 arc_hdr_clear_flags(hdr, ARC_FLAG_PRIO_ASYNC_READ);
4957 ASSERT3U(HDR_GET_LSIZE(hdr), ==, lsize);
4959 DTRACE_PROBE4(arc__miss, arc_buf_hdr_t *, hdr, blkptr_t *, bp,
4962 ARCSTAT_CONDSTAT(!HDR_PREFETCH(hdr),
4963 demand, prefetch, !HDR_ISTYPE_METADATA(hdr),
4976 if (HDR_HAS_L2HDR(hdr) &&
4977 !HDR_L2_WRITING(hdr) && !HDR_L2_EVICTED(hdr) &&
4978 !(l2arc_noprefetch && HDR_PREFETCH(hdr))) {
4981 DTRACE_PROBE1(l2arc__hit, arc_buf_hdr_t *, hdr);
4986 cb->l2rcb_hdr = hdr;
5001 ASSERT3U(HDR_GET_COMPRESS(hdr), !=,
5004 size, hdr->b_l1hdr.b_pdata,
5027 arc_buf_hdr_t *, hdr);
5029 if (HDR_L2_WRITING(hdr))
5038 arc_buf_hdr_t *, hdr);
5043 rzio = zio_read(pio, spa, bp, hdr->b_l1hdr.b_pdata, size,
5044 arc_read_done, hdr, priority, zio_flags, zb);
5061 arc_buf_hdr_t *hdr;
5067 hdr = buf_hash_find(guid, bp, &hash_lock);
5068 if (hdr == NULL)
5075 * would still have the ARC_FLAG_IO_IN_PROGRESS flag set on the hdr
5079 * without the dedup flag set. This would have left the hdr in the MRU
5091 * this hdr, then we don't destroy the hdr.
5093 if (!HDR_HAS_L1HDR(hdr) || (!HDR_IO_IN_PROGRESS(hdr) &&
5094 refcount_is_zero(&hdr->b_l1hdr.b_refcnt))) {
5095 arc_change_state(arc_anon, hdr, hash_lock);
5096 arc_hdr_destroy(hdr);
5108 * a new hdr for the buffer.
5113 arc_buf_hdr_t *hdr = buf->b_hdr;
5123 ASSERT(HDR_HAS_L1HDR(hdr));
5130 if (hdr->b_l1hdr.b_state == arc_anon) {
5132 ASSERT(!HDR_IO_IN_PROGRESS(hdr));
5133 ASSERT(!HDR_IN_HASH_TABLE(hdr));
5134 ASSERT(!HDR_HAS_L2HDR(hdr));
5135 ASSERT(HDR_EMPTY(hdr));
5137 ASSERT3U(hdr->b_l1hdr.b_bufcnt, ==, 1);
5138 ASSERT3S(refcount_count(&hdr->b_l1hdr.b_refcnt), ==, 1);
5139 ASSERT(!list_link_active(&hdr->b_l1hdr.b_arc_node));
5141 hdr->b_l1hdr.b_arc_access = 0;
5145 * have a hdr that is not empty.
5147 buf_discard_identity(hdr);
5153 kmutex_t *hash_lock = HDR_LOCK(hdr);
5161 arc_state_t *state = hdr->b_l1hdr.b_state;
5162 ASSERT3P(hash_lock, ==, HDR_LOCK(hdr));
5166 ASSERT3S(refcount_count(&hdr->b_l1hdr.b_refcnt), >, 0);
5168 if (HDR_HAS_L2HDR(hdr)) {
5169 mutex_enter(&hdr->b_l2hdr.b_dev->l2ad_mtx);
5179 if (HDR_HAS_L2HDR(hdr))
5180 arc_hdr_l2hdr_destroy(hdr);
5182 mutex_exit(&hdr->b_l2hdr.b_dev->l2ad_mtx);
5188 if (hdr->b_l1hdr.b_bufcnt > 1) {
5190 uint64_t spa = hdr->b_spa;
5191 uint64_t psize = HDR_GET_PSIZE(hdr);
5192 uint64_t lsize = HDR_GET_LSIZE(hdr);
5193 enum zio_compress compress = HDR_GET_COMPRESS(hdr);
5194 arc_buf_contents_t type = arc_buf_type(hdr);
5195 VERIFY3U(hdr->b_type, ==, type);
5197 ASSERT(hdr->b_l1hdr.b_buf != buf || buf->b_next != NULL);
5198 (void) remove_reference(hdr, hash_lock, tag);
5201 ASSERT3P(hdr->b_l1hdr.b_buf, !=, buf);
5206 * Pull the data off of this hdr and attach it to
5207 * a new anonymous hdr. Also find the last buffer
5208 * in the hdr's buffer list.
5210 arc_buf_t *lastbuf = arc_buf_remove(hdr, buf);
5214 * If the current arc_buf_t and the hdr are sharing their data
5224 arc_unshare_buf(hdr, buf);
5227 * Now we need to recreate the hdr's b_pdata. Since we
5232 if (arc_can_share(hdr, lastbuf)) {
5233 arc_share_buf(hdr, lastbuf);
5235 arc_hdr_alloc_pdata(hdr);
5236 bcopy(buf->b_data, hdr->b_l1hdr.b_pdata, psize);
5239 } else if (HDR_SHARED_DATA(hdr)) {
5245 * we rely on the hdr's compression flags to determine
5249 HDR_GET_COMPRESS(hdr) != ZIO_COMPRESS_OFF);
5252 ASSERT3P(hdr->b_l1hdr.b_pdata, !=, NULL);
5258 if (refcount_is_zero(&hdr->b_l1hdr.b_refcnt)) {
5264 hdr->b_l1hdr.b_bufcnt -= 1;
5271 * Allocate a new hdr. The new hdr will contain a b_pdata
5291 ASSERT(refcount_count(&hdr->b_l1hdr.b_refcnt) == 1);
5292 /* protected by hash lock, or hdr is on arc_anon */
5293 ASSERT(!multilist_link_active(&hdr->b_l1hdr.b_arc_node));
5294 ASSERT(!HDR_IO_IN_PROGRESS(hdr));
5295 arc_change_state(arc_anon, hdr, hash_lock);
5296 hdr->b_l1hdr.b_arc_access = 0;
5299 buf_discard_identity(hdr);
5334 arc_buf_hdr_t *hdr = buf->b_hdr;
5337 ASSERT(HDR_HAS_L1HDR(hdr));
5339 ASSERT(hdr->b_l1hdr.b_bufcnt > 0);
5347 arc_cksum_free(hdr);
5349 if (hdr->b_l1hdr.b_pdata != NULL) {
5351 arc_unshare_buf(hdr, buf);
5353 arc_hdr_free_pdata(hdr);
5357 ASSERT3P(hdr->b_l1hdr.b_pdata, ==, NULL);
5358 ASSERT(!HDR_SHARED_DATA(hdr));
5363 if (HDR_IO_IN_PROGRESS(hdr))
5367 arc_hdr_set_flags(hdr, ARC_FLAG_IO_IN_PROGRESS);
5373 ASSERT3U(HDR_GET_LSIZE(hdr), ==, BP_GET_LSIZE(zio->io_bp));
5376 HDR_SET_PSIZE(hdr, psize);
5377 arc_hdr_set_compress(hdr, compress);
5380 * If the hdr is compressed, then copy the compressed
5382 * data buf into the hdr. Ideally, we would like to always copy the
5385 * in the hdr's b_pdata field.
5387 if (HDR_GET_COMPRESS(hdr) != ZIO_COMPRESS_OFF &&
5391 arc_hdr_alloc_pdata(hdr);
5392 bcopy(zio->io_data, hdr->b_l1hdr.b_pdata, psize);
5396 ASSERT3U(hdr->b_l1hdr.b_bufcnt, ==, 1);
5399 * This hdr is not compressed so we're able to share
5400 * the arc_buf_t data buffer with the hdr.
5402 arc_share_buf(hdr, buf);
5403 ASSERT0(bcmp(zio->io_orig_data, hdr->b_l1hdr.b_pdata,
5404 HDR_GET_LSIZE(hdr)));
5406 arc_hdr_verify(hdr, zio->io_bp);
5435 arc_buf_hdr_t *hdr = buf->b_hdr;
5437 ASSERT3P(hdr->b_l1hdr.b_acb, ==, NULL);
5440 arc_hdr_verify(hdr, zio->io_bp);
5443 buf_discard_identity(hdr);
5445 hdr->b_dva = *BP_IDENTITY(zio->io_bp);
5446 hdr->b_birth = BP_PHYSICAL_BIRTH(zio->io_bp);
5449 ASSERT(HDR_EMPTY(hdr));
5458 if (!HDR_EMPTY(hdr)) {
5466 exists = buf_hash_insert(hdr, &hash_lock);
5475 panic("bad overwrite, hdr=%p exists=%p",
5476 (void *)hdr, (void *)exists);
5482 exists = buf_hash_insert(hdr, &hash_lock);
5488 panic("bad nopwrite, hdr=%p exists=%p",
5489 (void *)hdr, (void *)exists);
5492 ASSERT(hdr->b_l1hdr.b_bufcnt == 1);
5493 ASSERT(hdr->b_l1hdr.b_state == arc_anon);
5498 arc_hdr_clear_flags(hdr, ARC_FLAG_IO_IN_PROGRESS);
5500 if (exists == NULL && hdr->b_l1hdr.b_state == arc_anon)
5501 arc_access(hdr, hash_lock);
5504 arc_hdr_clear_flags(hdr, ARC_FLAG_IO_IN_PROGRESS);
5507 ASSERT(!refcount_is_zero(&hdr->b_l1hdr.b_refcnt));
5520 arc_buf_hdr_t *hdr = buf->b_hdr;
5526 ASSERT(!HDR_IO_ERROR(hdr));
5527 ASSERT(!HDR_IO_IN_PROGRESS(hdr));
5528 ASSERT3P(hdr->b_l1hdr.b_acb, ==, NULL);
5529 ASSERT3U(hdr->b_l1hdr.b_bufcnt, >, 0);
5531 arc_hdr_set_flags(hdr, ARC_FLAG_L2CACHE);
5534 ASSERT3U(HDR_GET_LSIZE(hdr), !=, arc_buf_size(buf));
5546 * The hdr's b_pdata is now stale, free it now. A new data block
5549 if (hdr->b_l1hdr.b_pdata != NULL) {
5552 * the hdr then we need to break that relationship here.
5553 * The hdr will remain with a NULL data pointer and the
5557 arc_unshare_buf(hdr, buf);
5559 arc_hdr_free_pdata(hdr);
5562 arc_hdr_set_compress(hdr, ZIO_COMPRESS_OFF);
5565 ASSERT3P(hdr->b_l1hdr.b_pdata, ==, NULL);
5568 HDR_GET_LSIZE(hdr), arc_buf_size(buf), zp, arc_write_ready,
5734 arc_buf_hdr_t *hdr = obj;
5741 ASSERT(!HDR_EMPTY(hdr));
5755 return (buf_hash(hdr->b_spa, &hdr->b_dva, hdr->b_birth) %
6189 l2arc_write_eligible(uint64_t spa_guid, arc_buf_hdr_t *hdr)
6198 if (hdr->b_spa != spa_guid || HDR_HAS_L2HDR(hdr) ||
6199 HDR_IO_IN_PROGRESS(hdr) || !HDR_L2CACHE(hdr))
6350 arc_buf_hdr_t *head, *hdr, *hdr_prev;
6373 for (hdr = list_prev(buflist, head); hdr; hdr = hdr_prev) {
6374 hdr_prev = list_prev(buflist, hdr);
6376 hash_lock = HDR_LOCK(hdr);
6397 list_insert_after(buflist, hdr, head);
6417 ASSERT(HDR_HAS_L1HDR(hdr));
6423 list_remove(buflist, hdr);
6424 arc_hdr_clear_flags(hdr, ARC_FLAG_HAS_L2HDR);
6426 ARCSTAT_INCR(arcstat_l2_asize, -arc_hdr_size(hdr));
6427 ARCSTAT_INCR(arcstat_l2_size, -HDR_GET_LSIZE(hdr));
6429 bytes_dropped += arc_hdr_size(hdr);
6431 arc_hdr_size(hdr), hdr);
6438 arc_hdr_clear_flags(hdr, ARC_FLAG_L2_WRITING);
6464 arc_buf_hdr_t *hdr;
6475 hdr = cb->l2rcb_hdr;
6476 ASSERT3P(hdr, !=, NULL);
6478 hash_lock = HDR_LOCK(hdr);
6480 ASSERT3P(hash_lock, ==, HDR_LOCK(hdr));
6487 ASSERT3P(zio->io_data, ==, hdr->b_l1hdr.b_pdata);
6491 valid_cksum = arc_cksum_is_equal(hdr, zio);
6492 if (valid_cksum && zio->io_error == 0 && !HDR_L2_EVICTED(hdr)) {
6494 zio->io_private = hdr;
6521 hdr->b_l1hdr.b_pdata, zio->io_size, arc_read_done,
6522 hdr, zio->io_priority, cb->l2rcb_flags,
6583 arc_buf_hdr_t *hdr, *hdr_prev;
6611 for (hdr = list_tail(buflist); hdr; hdr = hdr_prev) {
6612 hdr_prev = list_prev(buflist, hdr);
6614 hash_lock = HDR_LOCK(hdr);
6632 if (HDR_L2_WRITE_HEAD(hdr)) {
6637 list_remove(buflist, hdr);
6642 if (!all && HDR_HAS_L2HDR(hdr) &&
6643 (hdr->b_l2hdr.b_daddr > taddr ||
6644 hdr->b_l2hdr.b_daddr < dev->l2ad_hand)) {
6653 ASSERT(HDR_HAS_L2HDR(hdr));
6654 if (!HDR_HAS_L1HDR(hdr)) {
6655 ASSERT(!HDR_L2_READING(hdr));
6661 arc_change_state(arc_anon, hdr, hash_lock);
6662 arc_hdr_destroy(hdr);
6664 ASSERT(hdr->b_l1hdr.b_state != arc_l2c_only);
6671 if (HDR_L2_READING(hdr)) {
6673 arc_hdr_set_flags(hdr, ARC_FLAG_L2_EVICTED);
6677 ASSERT(!HDR_L2_WRITING(hdr));
6679 arc_hdr_l2hdr_destroy(hdr);
6700 arc_buf_hdr_t *hdr, *hdr_prev, *head;
6729 hdr = multilist_sublist_head(mls);
6731 hdr = multilist_sublist_tail(mls);
6737 for (; hdr; hdr = hdr_prev) {
6741 hdr_prev = multilist_sublist_next(mls, hdr);
6743 hdr_prev = multilist_sublist_prev(mls, hdr);
6745 hash_lock = HDR_LOCK(hdr);
6753 passed_sz += HDR_GET_LSIZE(hdr);
6762 if (!l2arc_write_eligible(guid, hdr)) {
6767 if ((write_asize + HDR_GET_LSIZE(hdr)) > target_sz) {
6791 hdr->b_l2hdr.b_dev = dev;
6792 hdr->b_l2hdr.b_daddr = dev->l2ad_hand;
6793 arc_hdr_set_flags(hdr,
6797 list_insert_head(&dev->l2ad_buflist, hdr);
6806 ASSERT(HDR_HAS_L1HDR(hdr));
6808 ASSERT3U(HDR_GET_PSIZE(hdr), >, 0);
6809 ASSERT3P(hdr->b_l1hdr.b_pdata, !=, NULL);
6810 ASSERT3U(arc_hdr_size(hdr), >, 0);
6811 uint64_t size = arc_hdr_size(hdr);
6813 (void) refcount_add_many(&dev->l2ad_alloc, size, hdr);
6816 * Normally the L2ARC can use the hdr's data, but if
6817 * we're sharing data between the hdr and one of its
6825 if (!HDR_SHARED_DATA(hdr)) {
6826 to_write = hdr->b_l1hdr.b_pdata;
6828 arc_buf_contents_t type = arc_buf_type(hdr);
6836 bcopy(hdr->b_l1hdr.b_pdata, to_write, size);
6840 hdr->b_l2hdr.b_daddr, size, to_write,
6841 ZIO_CHECKSUM_OFF, NULL, hdr,
6845 write_sz += HDR_GET_LSIZE(hdr);