f38cb554a534c6df738be3f4d23327e69888e634John Wren Kennedy# CDDL HEADER START
f38cb554a534c6df738be3f4d23327e69888e634John Wren Kennedy# The contents of this file are subject to the terms of the
f38cb554a534c6df738be3f4d23327e69888e634John Wren Kennedy# Common Development and Distribution License (the "License").
f38cb554a534c6df738be3f4d23327e69888e634John Wren Kennedy# You may not use this file except in compliance with the License.
f38cb554a534c6df738be3f4d23327e69888e634John Wren Kennedy# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
f38cb554a534c6df738be3f4d23327e69888e634John Wren Kennedy# or http://www.opensolaris.org/os/licensing.
f38cb554a534c6df738be3f4d23327e69888e634John Wren Kennedy# See the License for the specific language governing permissions
f38cb554a534c6df738be3f4d23327e69888e634John Wren Kennedy# and limitations under the License.
f38cb554a534c6df738be3f4d23327e69888e634John Wren Kennedy# When distributing Covered Code, include this CDDL HEADER in each
f38cb554a534c6df738be3f4d23327e69888e634John Wren Kennedy# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
f38cb554a534c6df738be3f4d23327e69888e634John Wren Kennedy# If applicable, add the following below this CDDL HEADER, with the
f38cb554a534c6df738be3f4d23327e69888e634John Wren Kennedy# fields enclosed by brackets "[]" replaced with your own identifying
f38cb554a534c6df738be3f4d23327e69888e634John Wren Kennedy# information: Portions Copyright [yyyy] [name of copyright owner]
f38cb554a534c6df738be3f4d23327e69888e634John Wren Kennedy# CDDL HEADER END
f38cb554a534c6df738be3f4d23327e69888e634John Wren Kennedy# Copyright 2009 Sun Microsystems, Inc. All rights reserved.
f38cb554a534c6df738be3f4d23327e69888e634John Wren Kennedy# Use is subject to license terms.
1d32ba663e202c24a5a1f2e5aef83fffb447cb7fJohn Wren Kennedy# Copyright (c) 2013, 2016 by Delphix. All rights reserved.
f38cb554a534c6df738be3f4d23327e69888e634John Wren Kennedy. $STF_SUITE/tests/functional/rsend/rsend.cfg
f38cb554a534c6df738be3f4d23327e69888e634John Wren Kennedy# Set up test model which includes various datasets
f38cb554a534c6df738be3f4d23327e69888e634John Wren Kennedy# ______ pclone
f38cb554a534c6df738be3f4d23327e69888e634John Wren Kennedy# ||@final @final @snapC
f38cb554a534c6df738be3f4d23327e69888e634John Wren Kennedy# ||@snapC @snapC @snapB
f38cb554a534c6df738be3f4d23327e69888e634John Wren Kennedy# ||@snapA @snapB @snapA
f38cb554a534c6df738be3f4d23327e69888e634John Wren Kennedy# ||@init @init @init
f38cb554a534c6df738be3f4d23327e69888e634John Wren Kennedy# $pool -------- $FS ------- fs1 ------- fs2
f38cb554a534c6df738be3f4d23327e69888e634John Wren Kennedy# \ \\_____ \ |
f38cb554a534c6df738be3f4d23327e69888e634John Wren Kennedy# vol vol \____ \ @fsnap
f38cb554a534c6df738be3f4d23327e69888e634John Wren Kennedy# @init @vsnap | ------------ fclone
f38cb554a534c6df738be3f4d23327e69888e634John Wren Kennedy# @snapA @init \ | |
f38cb554a534c6df738be3f4d23327e69888e634John Wren Kennedy# @final @snapB \ | @init
f38cb554a534c6df738be3f4d23327e69888e634John Wren Kennedy# @snapC vclone @snapA
f38cb554a534c6df738be3f4d23327e69888e634John Wren Kennedy# @final | @final
1d32ba663e202c24a5a1f2e5aef83fffb447cb7fJohn Wren Kennedy log_must zfs clone $pool@psnap $pool/pclone
1d32ba663e202c24a5a1f2e5aef83fffb447cb7fJohn Wren Kennedy log_must zfs clone $pool/$FS/vol@vsnap $pool/$FS/vclone
f38cb554a534c6df738be3f4d23327e69888e634John Wren Kennedy log_must snapshot_tree $pool/$FS/fs1/fs2@fsnap
1d32ba663e202c24a5a1f2e5aef83fffb447cb7fJohn Wren Kennedy log_must zfs clone $pool/$FS/fs1/fs2@fsnap $pool/$FS/fs1/fclone
f38cb554a534c6df738be3f4d23327e69888e634John Wren Kennedy log_must snapshot_tree $pool/$FS/fs1@snapA
f38cb554a534c6df738be3f4d23327e69888e634John Wren Kennedy log_must snapshot_tree $pool/$FS/fs1@snapB
f38cb554a534c6df738be3f4d23327e69888e634John Wren Kennedy log_must snapshot_tree $pool/$FS/fs1@snapC
f38cb554a534c6df738be3f4d23327e69888e634John Wren Kennedy log_must snapshot_tree $pool/$FS/fs1/fclone@snapA
1d32ba663e202c24a5a1f2e5aef83fffb447cb7fJohn Wren Kennedy log_must zfs snapshot $pool/$FS/vclone@snapC
f38cb554a534c6df738be3f4d23327e69888e634John Wren Kennedy# Cleanup the BACKDIR and given pool content and all the sub datasets
ac89d1e83d4735655102d06f099fb2d21bf120f6John Kennedy typeset list=$(zfs list -H -r -t all -o name $pool)
f38cb554a534c6df738be3f4d23327e69888e634John Wren Kennedy if [[ $ds != $pool ]] ; then
f38cb554a534c6df738be3f4d23327e69888e634John Wren Kennedy typeset mntpnt=$(get_prop mountpoint $pool)
f38cb554a534c6df738be3f4d23327e69888e634John Wren Kennedy if [[ -d $mntpnt ]]; then
f38cb554a534c6df738be3f4d23327e69888e634John Wren Kennedy if [[ -d $mntpnt ]]; then
f38cb554a534c6df738be3f4d23327e69888e634John Wren Kennedy# Detect if the given two filesystems have same sub-datasets
f38cb554a534c6df738be3f4d23327e69888e634John Wren Kennedy# $1 source filesystem
f38cb554a534c6df738be3f4d23327e69888e634John Wren Kennedy# $2 destination filesystem
ac89d1e83d4735655102d06f099fb2d21bf120f6John Kennedy zfs list -r -H -t all -o name $src_fs > $BACKDIR/src1
ac89d1e83d4735655102d06f099fb2d21bf120f6John Kennedy zfs list -r -H -t all -o name $dst_fs > $BACKDIR/dst1
1d32ba663e202c24a5a1f2e5aef83fffb447cb7fJohn Wren Kennedy eval sed -e 's:^$src_fs:PREFIX:g' < $BACKDIR/src1 > $BACKDIR/src
1d32ba663e202c24a5a1f2e5aef83fffb447cb7fJohn Wren Kennedy eval sed -e 's:^$dst_fs:PREFIX:g' < $BACKDIR/dst1 > $BACKDIR/dst
f38cb554a534c6df738be3f4d23327e69888e634John Wren Kennedy typeset -i ret=$?
1d32ba663e202c24a5a1f2e5aef83fffb447cb7fJohn Wren Kennedy rm -f $BACKDIR/src $BACKDIR/dst $BACKDIR/src1 $BACKDIR/dst1
f38cb554a534c6df738be3f4d23327e69888e634John Wren Kennedy# Compare all the directores and files in two filesystems
f38cb554a534c6df738be3f4d23327e69888e634John Wren Kennedy# $1 source filesystem
f38cb554a534c6df738be3f4d23327e69888e634John Wren Kennedy# $2 destination filesystem
f38cb554a534c6df738be3f4d23327e69888e634John Wren Kennedy# Compare the given two dataset properties
f38cb554a534c6df738be3f4d23327e69888e634John Wren Kennedy for item in "type" "origin" "volblocksize" "aclinherit" "aclmode" \
9c3fd1216fa7fb02cfbc78a2518a686d54b48ab8Matthew Ahrens "atime" "canmount" "checksum" "compression" "copies" "devices" \
9c3fd1216fa7fb02cfbc78a2518a686d54b48ab8Matthew Ahrens "exec" "quota" "readonly" "recordsize" "reservation" "setuid" \
9c3fd1216fa7fb02cfbc78a2518a686d54b48ab8Matthew Ahrens "sharenfs" "snapdir" "version" "volsize" "xattr" "zoned" \
1d32ba663e202c24a5a1f2e5aef83fffb447cb7fJohn Wren Kennedy zfs get -H -o property,value,source $item $dtst1 >> \
1d32ba663e202c24a5a1f2e5aef83fffb447cb7fJohn Wren Kennedy zfs get -H -o property,value,source $item $dtst2 >> \
1d32ba663e202c24a5a1f2e5aef83fffb447cb7fJohn Wren Kennedy eval sed -e 's:$dtst1:PREFIX:g' < $BACKDIR/dtst1 > $BACKDIR/dtst1
1d32ba663e202c24a5a1f2e5aef83fffb447cb7fJohn Wren Kennedy eval sed -e 's:$dtst2:PREFIX:g' < $BACKDIR/dtst2 > $BACKDIR/dtst2
f38cb554a534c6df738be3f4d23327e69888e634John Wren Kennedy typeset -i ret=$?
f38cb554a534c6df738be3f4d23327e69888e634John Wren Kennedy# Random create directories and files
f38cb554a534c6df738be3f4d23327e69888e634John Wren Kennedy if [[ -d $dir ]]; then
f38cb554a534c6df738be3f4d23327e69888e634John Wren Kennedy typeset -i ret=$?
f38cb554a534c6df738be3f4d23327e69888e634John Wren Kennedy# Put data in filesystem and take snapshot
f38cb554a534c6df738be3f4d23327e69888e634John Wren Kennedy# $1 snapshot name
f38cb554a534c6df738be3f4d23327e69888e634John Wren Kennedy typeset ds=${snap%%@*}
f38cb554a534c6df738be3f4d23327e69888e634John Wren Kennedy# Destroy the given snapshot and stuff
f38cb554a534c6df738be3f4d23327e69888e634John Wren Kennedy typeset ds=${snap%%@*}
f38cb554a534c6df738be3f4d23327e69888e634John Wren Kennedy# Get all the sub-datasets of give dataset with specific suffix
f38cb554a534c6df738be3f4d23327e69888e634John Wren Kennedy# $1 Given dataset
ac89d1e83d4735655102d06f099fb2d21bf120f6John Kennedy typeset list=$(zfs list -r -H -t all -o name $ds | grep "$suffix$")
f38cb554a534c6df738be3f4d23327e69888e634John Wren Kennedy# Output inherited properties whitch is edited for file system
1d32ba663e202c24a5a1f2e5aef83fffb447cb7fJohn Wren Kennedy fs_prop=$(echo $fs_prop | grep -v "mlslabel")
1d32ba663e202c24a5a1f2e5aef83fffb447cb7fJohn Wren Kennedy egrep -v "devices|mlslabel|sharenfs|sharesmb|zoned")
f38cb554a534c6df738be3f4d23327e69888e634John Wren Kennedy# Output inherited properties for volume
f38cb554a534c6df738be3f4d23327e69888e634John Wren Kennedy# Get the destination dataset to compare
1d32ba663e202c24a5a1f2e5aef83fffb447cb7fJohn Wren Kennedy if ! zpool list $srcfs > /dev/null 2>&1 ; then
9c3fd1216fa7fb02cfbc78a2518a686d54b48ab8Matthew Ahrens# Make test files
9c3fd1216fa7fb02cfbc78a2518a686d54b48ab8Matthew Ahrens# $1 Number of files to create
9c3fd1216fa7fb02cfbc78a2518a686d54b48ab8Matthew Ahrens# $2 Maximum file size
9c3fd1216fa7fb02cfbc78a2518a686d54b48ab8Matthew Ahrens# $3 File ID offset
9c3fd1216fa7fb02cfbc78a2518a686d54b48ab8Matthew Ahrens# $4 File system to create the files on
9c3fd1216fa7fb02cfbc78a2518a686d54b48ab8Matthew Ahrens of=/$fs/file-$maxsize-$((i+$file_id_offset)) \
9c3fd1216fa7fb02cfbc78a2518a686d54b48ab8Matthew Ahrens "Failed to create /$fs/file-$maxsize-$((i+$file_id_offset))"
1d32ba663e202c24a5a1f2e5aef83fffb447cb7fJohn Wren Kennedy echo Created $nfiles files of random sizes up to $maxsize bytes
9c3fd1216fa7fb02cfbc78a2518a686d54b48ab8Matthew Ahrens# Remove test files
9c3fd1216fa7fb02cfbc78a2518a686d54b48ab8Matthew Ahrens# $1 Number of files to remove
9c3fd1216fa7fb02cfbc78a2518a686d54b48ab8Matthew Ahrens# $2 Maximum file size
9c3fd1216fa7fb02cfbc78a2518a686d54b48ab8Matthew Ahrens# $3 File ID offset
9c3fd1216fa7fb02cfbc78a2518a686d54b48ab8Matthew Ahrens# $4 File system to remove the files from
1d32ba663e202c24a5a1f2e5aef83fffb447cb7fJohn Wren Kennedy rm -f /$fs/file-$maxsize-$((i+$file_id_offset))
1d32ba663e202c24a5a1f2e5aef83fffb447cb7fJohn Wren Kennedy echo Removed $nfiles files of random sizes up to $maxsize bytes
9c3fd1216fa7fb02cfbc78a2518a686d54b48ab8Matthew Ahrens# Mess up file contents
9c3fd1216fa7fb02cfbc78a2518a686d54b48ab8Matthew Ahrens# $1 The file path
9c3fd1216fa7fb02cfbc78a2518a686d54b48ab8Matthew Ahrens # We corrupt 2 bytes to minimize the chance that we
1d32ba663e202c24a5a1f2e5aef83fffb447cb7fJohn Wren Kennedy log_must eval "dd if=/dev/random of=$file conv=notrunc " \
9c3fd1216fa7fb02cfbc78a2518a686d54b48ab8Matthew Ahrens# Diff the send/receive filesystems
9c3fd1216fa7fb02cfbc78a2518a686d54b48ab8Matthew Ahrens# $1 The sent filesystem
9c3fd1216fa7fb02cfbc78a2518a686d54b48ab8Matthew Ahrens# $2 The received filesystem
9c3fd1216fa7fb02cfbc78a2518a686d54b48ab8Matthew Ahrens if [[ -d /$recvfs/.zfs/snapshot/a && -d \
1d32ba663e202c24a5a1f2e5aef83fffb447cb7fJohn Wren Kennedy diff -r /$recvfs/.zfs/snapshot/a /$sendfs/.zfs/snapshot/a
9c3fd1216fa7fb02cfbc78a2518a686d54b48ab8Matthew Ahrens [[ $? -eq 0 ]] || log_fail "Differences found in snap a"
9c3fd1216fa7fb02cfbc78a2518a686d54b48ab8Matthew Ahrens if [[ -d /$recvfs/.zfs/snapshot/b && -d \
1d32ba663e202c24a5a1f2e5aef83fffb447cb7fJohn Wren Kennedy diff -r /$recvfs/.zfs/snapshot/b /$sendfs/.zfs/snapshot/b
9c3fd1216fa7fb02cfbc78a2518a686d54b48ab8Matthew Ahrens [[ $? -eq 0 ]] || log_fail "Differences found in snap b"
9c3fd1216fa7fb02cfbc78a2518a686d54b48ab8Matthew Ahrens# Resume test helper
9c3fd1216fa7fb02cfbc78a2518a686d54b48ab8Matthew Ahrens# $1 The ZFS send command
9c3fd1216fa7fb02cfbc78a2518a686d54b48ab8Matthew Ahrens# $2 The filesystem where the streams are sent
9c3fd1216fa7fb02cfbc78a2518a686d54b48ab8Matthew Ahrens# $3 The receive filesystem
9c3fd1216fa7fb02cfbc78a2518a686d54b48ab8Matthew Ahrens log_must eval "$sendcmd >/$streamfs/$stream_num"
ac89d1e83d4735655102d06f099fb2d21bf120f6John Kennedy log_mustnot zfs recv -suv $recvfs </$streamfs/$stream_num
1d32ba663e202c24a5a1f2e5aef83fffb447cb7fJohn Wren Kennedy token=$(zfs get -Hp -o value receive_resume_token $recvfs)
1d32ba663e202c24a5a1f2e5aef83fffb447cb7fJohn Wren Kennedy log_must eval "zfs send -v -t $token >/$streamfs/$stream_num"
ac89d1e83d4735655102d06f099fb2d21bf120f6John Kennedy log_must zfs recv -suv $recvfs </$streamfs/$stream_num
9c3fd1216fa7fb02cfbc78a2518a686d54b48ab8Matthew Ahrens# Setup filesystems for the resumable send/receive tests
ac89d1e83d4735655102d06f099fb2d21bf120f6John Kennedy# $1 The "send" filesystem
ac89d1e83d4735655102d06f099fb2d21bf120f6John Kennedy# $2 The "recv" filesystem
ac89d1e83d4735655102d06f099fb2d21bf120f6John Kennedy typeset sendpool=${sendfs%%/*}
ac89d1e83d4735655102d06f099fb2d21bf120f6John Kennedy typeset recvpool=${recvfs%%/*}
5602294fda888d923d57a78bafdaf48ae6223deaDan Kimmel datasetexists $sendfs && log_must zfs destroy -r $sendpool
5602294fda888d923d57a78bafdaf48ae6223deaDan Kimmel datasetexists $recvfs && log_must zfs destroy -r $recvpool
ac89d1e83d4735655102d06f099fb2d21bf120f6John Kennedy if $(datasetexists $sendfs || zfs create -o compress=lz4 $sendfs); then
9c3fd1216fa7fb02cfbc78a2518a686d54b48ab8Matthew Ahrens mk_files 1000 131072 0 $sendfs &
9c3fd1216fa7fb02cfbc78a2518a686d54b48ab8Matthew Ahrens mk_files 100 1048576 0 $sendfs &
9c3fd1216fa7fb02cfbc78a2518a686d54b48ab8Matthew Ahrens mk_files 10 10485760 0 $sendfs &
9c3fd1216fa7fb02cfbc78a2518a686d54b48ab8Matthew Ahrens mk_files 1 104857600 0 $sendfs &
1d32ba663e202c24a5a1f2e5aef83fffb447cb7fJohn Wren Kennedy log_must eval "zfs send -v $sendfs@a >/$sendpool/initial.zsend"
1d32ba663e202c24a5a1f2e5aef83fffb447cb7fJohn Wren Kennedy log_must eval "zfs send -v -i @a $sendfs@b " \
1d32ba663e202c24a5a1f2e5aef83fffb447cb7fJohn Wren Kennedy log_must zfs create -o compress=lz4 $sendpool/stream
5602294fda888d923d57a78bafdaf48ae6223deaDan Kimmel# Check to see if the specified features are set in a send stream.
5602294fda888d923d57a78bafdaf48ae6223deaDan Kimmel# The values for these features are found in uts/common/fs/zfs/sys/zfs_ioctl.h
5602294fda888d923d57a78bafdaf48ae6223deaDan Kimmel# $1 The stream file
5602294fda888d923d57a78bafdaf48ae6223deaDan Kimmel# $2-$n The flags expected in the stream
5602294fda888d923d57a78bafdaf48ae6223deaDan Kimmel [[ -f $file ]] || log_fail "Couldn't find file: $file"
5602294fda888d923d57a78bafdaf48ae6223deaDan Kimmel typeset flags=$(cat $file | zstreamdump | awk '/features =/ {print $3}')
5602294fda888d923d57a78bafdaf48ae6223deaDan Kimmel [[ -z $known ]] && log_fail "Unknown feature: $flag"
5602294fda888d923d57a78bafdaf48ae6223deaDan Kimmel derived=$(echo "$flags & ${feature[$flag]} = X" | mdb | sed 's/ //g')
5602294fda888d923d57a78bafdaf48ae6223deaDan Kimmel [[ $derived = $known ]] || return 1
5602294fda888d923d57a78bafdaf48ae6223deaDan Kimmel# Parse zstreamdump -v output. The output varies for each kind of record:
5602294fda888d923d57a78bafdaf48ae6223deaDan Kimmel# BEGIN records are simply output as "BEGIN"
5602294fda888d923d57a78bafdaf48ae6223deaDan Kimmel# END records are output as "END"
5602294fda888d923d57a78bafdaf48ae6223deaDan Kimmel# OBJECT records become "OBJECT <object num>"
5602294fda888d923d57a78bafdaf48ae6223deaDan Kimmel# FREEOBJECTS records become "FREEOBJECTS <startobj> <numobjs>"
5602294fda888d923d57a78bafdaf48ae6223deaDan Kimmel# FREE records become "<record type> <start> <length>"
5602294fda888d923d57a78bafdaf48ae6223deaDan Kimmel# WRITE records become:
5602294fda888d923d57a78bafdaf48ae6223deaDan Kimmel# "<record type> <compression type> <start> <logical size> <compressed size>
5602294fda888d923d57a78bafdaf48ae6223deaDan Kimmel# <data size>"
5602294fda888d923d57a78bafdaf48ae6223deaDan Kimmel sed '/^WRITE/{N;s/\n/ /;}' | grep "^[A-Z]" | awk '{
5602294fda888d923d57a78bafdaf48ae6223deaDan Kimmel if ($1 == "BEGIN" || $1 == "END") print $1
5602294fda888d923d57a78bafdaf48ae6223deaDan Kimmel if ($1 == "OBJECT") print $1" "$4
5602294fda888d923d57a78bafdaf48ae6223deaDan Kimmel if ($1 == "FREEOBJECTS") print $1" "$4" "$7
5602294fda888d923d57a78bafdaf48ae6223deaDan Kimmel if ($1 == "FREE") print $1" "$7" "$10
5602294fda888d923d57a78bafdaf48ae6223deaDan Kimmel if ($1 == "WRITE") print $1" "$15" "$18" "$21" "$24" "$27}'
5602294fda888d923d57a78bafdaf48ae6223deaDan Kimmel# Given a send stream, verify that the size of the stream matches what's
5602294fda888d923d57a78bafdaf48ae6223deaDan Kimmel# expected based on the source or target dataset. If the stream is an
5602294fda888d923d57a78bafdaf48ae6223deaDan Kimmel# incremental stream, subtract the size of the source snapshot before
5602294fda888d923d57a78bafdaf48ae6223deaDan Kimmel# comparing. This function does not currently handle incremental streams
5602294fda888d923d57a78bafdaf48ae6223deaDan Kimmel# that remove data.
5602294fda888d923d57a78bafdaf48ae6223deaDan Kimmel# $1 The zstreamdump output file
5602294fda888d923d57a78bafdaf48ae6223deaDan Kimmel# $2 The dataset to compare against
5602294fda888d923d57a78bafdaf48ae6223deaDan Kimmel# This can be a source of a send or recv target (fs, not snapshot)
5602294fda888d923d57a78bafdaf48ae6223deaDan Kimmel# $3 The percentage below which verification is deemed a failure
5602294fda888d923d57a78bafdaf48ae6223deaDan Kimmel# $4 The source snapshot of an incremental send
5602294fda888d923d57a78bafdaf48ae6223deaDan Kimmel typeset percent=${3:-90}
5602294fda888d923d57a78bafdaf48ae6223deaDan Kimmel [[ -f $stream ]] || log_fail "No such file: $stream"
5602294fda888d923d57a78bafdaf48ae6223deaDan Kimmel datasetexists $ds || log_fail "No such dataset: $ds"
5602294fda888d923d57a78bafdaf48ae6223deaDan Kimmel typeset stream_size=$(cat $stream | zstreamdump | sed -n \
5602294fda888d923d57a78bafdaf48ae6223deaDan Kimmel if [[ -n $inc_src ]]; then
5602294fda888d923d57a78bafdaf48ae6223deaDan Kimmel within_percent $stream_size $ds_size $percent || log_fail \
5602294fda888d923d57a78bafdaf48ae6223deaDan Kimmel# Cleanup function for tests involving resumable send
5602294fda888d923d57a78bafdaf48ae6223deaDan Kimmel datasetexists $sendfs && log_must zfs destroy -r $sendfs
5602294fda888d923d57a78bafdaf48ae6223deaDan Kimmel datasetexists $streamfs && log_must zfs destroy -r $streamfs