perf.shlib revision dcbf3bd6a1f1360fc1afcee9e22c6dcff7844bf2
dcbf3bd6a1f1360fc1afcee9e22c6dcff7844bf2George Wilson# This file and its contents are supplied under the terms of the
dcbf3bd6a1f1360fc1afcee9e22c6dcff7844bf2George Wilson# Common Development and Distribution License ("CDDL"), version 1.0.
dcbf3bd6a1f1360fc1afcee9e22c6dcff7844bf2George Wilson# You may only use this file in accordance with the terms of version
dcbf3bd6a1f1360fc1afcee9e22c6dcff7844bf2George Wilson# 1.0 of the CDDL.
dcbf3bd6a1f1360fc1afcee9e22c6dcff7844bf2George Wilson# A full copy of the text of the CDDL should have accompanied this
dcbf3bd6a1f1360fc1afcee9e22c6dcff7844bf2George Wilson# source. A copy of the CDDL is also available via the Internet at
dcbf3bd6a1f1360fc1afcee9e22c6dcff7844bf2George Wilson# Copyright (c) 2015 by Delphix. All rights reserved.
dcbf3bd6a1f1360fc1afcee9e22c6dcff7844bf2George Wilson# If neither is specified, do a nightly run.
dcbf3bd6a1f1360fc1afcee9e22c6dcff7844bf2George Wilson[[ -z $PERF_REGRESSION_WEEKLY ]] && export PERF_REGRESSION_NIGHTLY=1
dcbf3bd6a1f1360fc1afcee9e22c6dcff7844bf2George Wilson# Default runtime for each type of test run.
dcbf3bd6a1f1360fc1afcee9e22c6dcff7844bf2George Wilsonexport PERF_RUNTIME_WEEKLY=$((30 * 60))
dcbf3bd6a1f1360fc1afcee9e22c6dcff7844bf2George Wilsonexport PERF_RUNTIME_NIGHTLY=$((10 * 60))
dcbf3bd6a1f1360fc1afcee9e22c6dcff7844bf2George Wilson# Default fs creation options
dcbf3bd6a1f1360fc1afcee9e22c6dcff7844bf2George Wilsonexport PERF_FS_OPTS=${PERF_FS_OPTS:-'-o recsize=8k -o compress=lz4' \
dcbf3bd6a1f1360fc1afcee9e22c6dcff7844bf2George Wilson ' -o checksum=sha256 -o redundant_metadata=most'}
dcbf3bd6a1f1360fc1afcee9e22c6dcff7844bf2George Wilsonfunction get_sync_str
dcbf3bd6a1f1360fc1afcee9e22c6dcff7844bf2George Wilson typeset sync=$1
dcbf3bd6a1f1360fc1afcee9e22c6dcff7844bf2George Wilson typeset sync_str=''
dcbf3bd6a1f1360fc1afcee9e22c6dcff7844bf2George Wilson [[ $sync -eq 0 ]] && sync_str='async'
dcbf3bd6a1f1360fc1afcee9e22c6dcff7844bf2George Wilson [[ $sync -eq 1 ]] && sync_str='sync'
dcbf3bd6a1f1360fc1afcee9e22c6dcff7844bf2George Wilson echo $sync_str
dcbf3bd6a1f1360fc1afcee9e22c6dcff7844bf2George Wilson# This function will run fio in a loop, according to the .fio file passed
dcbf3bd6a1f1360fc1afcee9e22c6dcff7844bf2George Wilson# in and a number of environment variables. The following variables can be
dcbf3bd6a1f1360fc1afcee9e22c6dcff7844bf2George Wilson# set before launching zfstest to override the defaults.
dcbf3bd6a1f1360fc1afcee9e22c6dcff7844bf2George Wilson# PERF_RUNTIME: The time in seconds each fio invocation should run.
dcbf3bd6a1f1360fc1afcee9e22c6dcff7844bf2George Wilson# PERF_RUNTYPE: A human readable tag that appears in logs. The defaults are
dcbf3bd6a1f1360fc1afcee9e22c6dcff7844bf2George Wilson# nightly and weekly.
dcbf3bd6a1f1360fc1afcee9e22c6dcff7844bf2George Wilson# PERF_NTHREADS: A list of how many threads each fio invocation will use.
dcbf3bd6a1f1360fc1afcee9e22c6dcff7844bf2George Wilson# PERF_SYNC_TYPES: Whether to use (O_SYNC) or not. 1 is sync IO, 0 is async IO.
dcbf3bd6a1f1360fc1afcee9e22c6dcff7844bf2George Wilson# PERF_IOSIZES: A list of blocksizes in which each fio invocation will do IO.
dcbf3bd6a1f1360fc1afcee9e22c6dcff7844bf2George Wilson# PERF_COLLECT_SCRIPTS: A comma delimited list of 'command args, logfile_tag'
dcbf3bd6a1f1360fc1afcee9e22c6dcff7844bf2George Wilson# pairs that will be added to the scripts specified in each test.
dcbf3bd6a1f1360fc1afcee9e22c6dcff7844bf2George Wilsonfunction do_fio_run
dcbf3bd6a1f1360fc1afcee9e22c6dcff7844bf2George Wilson typeset script=$1
dcbf3bd6a1f1360fc1afcee9e22c6dcff7844bf2George Wilson typeset do_recreate=$2
dcbf3bd6a1f1360fc1afcee9e22c6dcff7844bf2George Wilson typeset clear_cache=$3
dcbf3bd6a1f1360fc1afcee9e22c6dcff7844bf2George Wilson typeset threads sync iosize
dcbf3bd6a1f1360fc1afcee9e22c6dcff7844bf2George Wilson for threads in $PERF_NTHREADS; do
dcbf3bd6a1f1360fc1afcee9e22c6dcff7844bf2George Wilson for sync in $PERF_SYNC_TYPES; do
dcbf3bd6a1f1360fc1afcee9e22c6dcff7844bf2George Wilson for iosize in $PERF_IOSIZES; do
dcbf3bd6a1f1360fc1afcee9e22c6dcff7844bf2George Wilson log_note "Running with $threads" \
dcbf3bd6a1f1360fc1afcee9e22c6dcff7844bf2George Wilson "$(get_sync_str $sync) threads, $iosize ios"
dcbf3bd6a1f1360fc1afcee9e22c6dcff7844bf2George Wilson if $do_recreate; then
dcbf3bd6a1f1360fc1afcee9e22c6dcff7844bf2George Wilson recreate_perfpool
dcbf3bd6a1f1360fc1afcee9e22c6dcff7844bf2George Wilson log_must $ZFS create $PERF_FS_OPTS \
dcbf3bd6a1f1360fc1afcee9e22c6dcff7844bf2George Wilson if $clear_cache; then
dcbf3bd6a1f1360fc1afcee9e22c6dcff7844bf2George Wilson # Clear the ARC
dcbf3bd6a1f1360fc1afcee9e22c6dcff7844bf2George Wilson $ZPOOL export $PERFPOOL
dcbf3bd6a1f1360fc1afcee9e22c6dcff7844bf2George Wilson $ZPOOL import $PERFPOOL
dcbf3bd6a1f1360fc1afcee9e22c6dcff7844bf2George Wilson export RUNTIME=$PERF_RUNTIME
dcbf3bd6a1f1360fc1afcee9e22c6dcff7844bf2George Wilson export FILESIZE=$((TOTAL_SIZE / threads))
dcbf3bd6a1f1360fc1afcee9e22c6dcff7844bf2George Wilson export NUMJOBS=$threads
dcbf3bd6a1f1360fc1afcee9e22c6dcff7844bf2George Wilson export SYNC_TYPE=$sync
dcbf3bd6a1f1360fc1afcee9e22c6dcff7844bf2George Wilson export BLOCKSIZE=$iosize
dcbf3bd6a1f1360fc1afcee9e22c6dcff7844bf2George Wilson # Start the data collection
dcbf3bd6a1f1360fc1afcee9e22c6dcff7844bf2George Wilson do_collect_scripts $threads $sync $iosize
dcbf3bd6a1f1360fc1afcee9e22c6dcff7844bf2George Wilson # Start the load
dcbf3bd6a1f1360fc1afcee9e22c6dcff7844bf2George Wilson log_must $FIO $FIO_SCRIPTS/$script
dcbf3bd6a1f1360fc1afcee9e22c6dcff7844bf2George Wilson# This function iterates through the value pairs in $PERF_COLLECT_SCRIPTS.
dcbf3bd6a1f1360fc1afcee9e22c6dcff7844bf2George Wilson# The script at index N is launched in the background, with its output
dcbf3bd6a1f1360fc1afcee9e22c6dcff7844bf2George Wilson# redirected to a logfile containing the tag specified at index N + 1.
dcbf3bd6a1f1360fc1afcee9e22c6dcff7844bf2George Wilsonfunction do_collect_scripts
dcbf3bd6a1f1360fc1afcee9e22c6dcff7844bf2George Wilson typeset threads=$1
dcbf3bd6a1f1360fc1afcee9e22c6dcff7844bf2George Wilson typeset sync=$2
dcbf3bd6a1f1360fc1afcee9e22c6dcff7844bf2George Wilson typeset iosize=$3
dcbf3bd6a1f1360fc1afcee9e22c6dcff7844bf2George Wilson [[ -n $collect_scripts ]] || log_fail "No data collection scripts."
dcbf3bd6a1f1360fc1afcee9e22c6dcff7844bf2George Wilson [[ -n $PERF_RUNTIME ]] || log_fail "No runtime specified."
dcbf3bd6a1f1360fc1afcee9e22c6dcff7844bf2George Wilson # This will be part of the output filename.
dcbf3bd6a1f1360fc1afcee9e22c6dcff7844bf2George Wilson typeset sync_str=$(get_sync_str $sync)
dcbf3bd6a1f1360fc1afcee9e22c6dcff7844bf2George Wilson typeset suffix="$sync_str.$iosize-ios.$threads-threads"
dcbf3bd6a1f1360fc1afcee9e22c6dcff7844bf2George Wilson # Add in user supplied scripts and logfiles, if any.
dcbf3bd6a1f1360fc1afcee9e22c6dcff7844bf2George Wilson typeset oIFS=$IFS
dcbf3bd6a1f1360fc1afcee9e22c6dcff7844bf2George Wilson for item in $PERF_COLLECT_SCRIPTS; do
dcbf3bd6a1f1360fc1afcee9e22c6dcff7844bf2George Wilson collect_scripts+=($($ECHO $item | $SED 's/^ *//g'))
dcbf3bd6a1f1360fc1afcee9e22c6dcff7844bf2George Wilson typeset idx=0
dcbf3bd6a1f1360fc1afcee9e22c6dcff7844bf2George Wilson while [[ $idx -lt "${#collect_scripts[@]}" ]]; do
dcbf3bd6a1f1360fc1afcee9e22c6dcff7844bf2George Wilson typeset logbase="$(get_perf_output_dir)/$($BASENAME \
dcbf3bd6a1f1360fc1afcee9e22c6dcff7844bf2George Wilson $SUDO_COMMAND)"
dcbf3bd6a1f1360fc1afcee9e22c6dcff7844bf2George Wilson typeset outfile="$logbase.${collect_scripts[$idx + 1]}.$suffix"
dcbf3bd6a1f1360fc1afcee9e22c6dcff7844bf2George Wilson $TIMEOUT $PERF_RUNTIME ${collect_scripts[$idx]} >$outfile 2>&1 &
dcbf3bd6a1f1360fc1afcee9e22c6dcff7844bf2George Wilson # Need to explicitly return 0 because timeout(1) will kill
dcbf3bd6a1f1360fc1afcee9e22c6dcff7844bf2George Wilson # a child process and cause us to return non-zero.
dcbf3bd6a1f1360fc1afcee9e22c6dcff7844bf2George Wilson# Find a place to deposit performance data collected while under load.
dcbf3bd6a1f1360fc1afcee9e22c6dcff7844bf2George Wilsonfunction get_perf_output_dir
dcbf3bd6a1f1360fc1afcee9e22c6dcff7844bf2George Wilson typeset dir="$(pwd)/perf_data"
dcbf3bd6a1f1360fc1afcee9e22c6dcff7844bf2George Wilson [[ -d $dir ]] || $MKDIR -p $dir
dcbf3bd6a1f1360fc1afcee9e22c6dcff7844bf2George Wilson# Destroy and create the pool used for performance tests. The
dcbf3bd6a1f1360fc1afcee9e22c6dcff7844bf2George Wilson# PERFPOOL_CREATE_CMD variable allows users to test with a custom pool
dcbf3bd6a1f1360fc1afcee9e22c6dcff7844bf2George Wilson# configuration by specifying the pool creation command in their environment.
dcbf3bd6a1f1360fc1afcee9e22c6dcff7844bf2George Wilson# If PERFPOOL_CREATE_CMD is empty, a pool using all available disks is created.
dcbf3bd6a1f1360fc1afcee9e22c6dcff7844bf2George Wilsonfunction recreate_perfpool
dcbf3bd6a1f1360fc1afcee9e22c6dcff7844bf2George Wilson [[ -n $PERFPOOL ]] || log_fail "The \$PERFPOOL variable isn't set."
dcbf3bd6a1f1360fc1afcee9e22c6dcff7844bf2George Wilson poolexists $PERFPOOL && destroy_pool $PERFPOOL
dcbf3bd6a1f1360fc1afcee9e22c6dcff7844bf2George Wilson if [[ -n $PERFPOOL_CREATE_CMD ]]; then
dcbf3bd6a1f1360fc1afcee9e22c6dcff7844bf2George Wilson log_must $PERFPOOL_CREATE_CMD
dcbf3bd6a1f1360fc1afcee9e22c6dcff7844bf2George Wilson log_must eval "$ZPOOL create -f $PERFPOOL $DISKS"
dcbf3bd6a1f1360fc1afcee9e22c6dcff7844bf2George Wilsonfunction get_max_arc_size
dcbf3bd6a1f1360fc1afcee9e22c6dcff7844bf2George Wilson typeset -l max_arc_size=$(dtrace -qn 'BEGIN {
dcbf3bd6a1f1360fc1afcee9e22c6dcff7844bf2George Wilson printf("%u\n", `arc_stats.arcstat_c_max.value.ui64);
dcbf3bd6a1f1360fc1afcee9e22c6dcff7844bf2George Wilson [[ $? -eq 0 ]] || log_fail "get_max_arc_size failed"
dcbf3bd6a1f1360fc1afcee9e22c6dcff7844bf2George Wilson echo $max_arc_size
dcbf3bd6a1f1360fc1afcee9e22c6dcff7844bf2George Wilson# Create a file with some information about how this system is configured.
dcbf3bd6a1f1360fc1afcee9e22c6dcff7844bf2George Wilsonfunction get_system_config
dcbf3bd6a1f1360fc1afcee9e22c6dcff7844bf2George Wilson typeset config=$PERF_DATA_DIR/$1
dcbf3bd6a1f1360fc1afcee9e22c6dcff7844bf2George Wilson echo "{" >>$config
dcbf3bd6a1f1360fc1afcee9e22c6dcff7844bf2George Wilson $DTRACE -qn 'BEGIN{
dcbf3bd6a1f1360fc1afcee9e22c6dcff7844bf2George Wilson printf(" \"ncpus\": %d,\n", `ncpus);
dcbf3bd6a1f1360fc1afcee9e22c6dcff7844bf2George Wilson printf(" \"physmem\": %u,\n", `physmem * `_pagesize);
dcbf3bd6a1f1360fc1afcee9e22c6dcff7844bf2George Wilson printf(" \"c_max\": %u,\n", `arc_stats.arcstat_c_max.value.ui64);
dcbf3bd6a1f1360fc1afcee9e22c6dcff7844bf2George Wilson printf(" \"kmem_flags\": \"0x%x\",", `kmem_flags);
dcbf3bd6a1f1360fc1afcee9e22c6dcff7844bf2George Wilson exit(0)}' >>$config
dcbf3bd6a1f1360fc1afcee9e22c6dcff7844bf2George Wilson $ECHO " \"hostname\": \"$($UNAME -n)\"," >>$config
dcbf3bd6a1f1360fc1afcee9e22c6dcff7844bf2George Wilson $ECHO " \"kernel version\": \"$($UNAME -v)\"," >>$config
dcbf3bd6a1f1360fc1afcee9e22c6dcff7844bf2George Wilson $IOSTAT -En | $AWK 'BEGIN {
dcbf3bd6a1f1360fc1afcee9e22c6dcff7844bf2George Wilson printf(" \"disks\": {\n"); first = 1}
dcbf3bd6a1f1360fc1afcee9e22c6dcff7844bf2George Wilson /^c/ {disk = $1}
dcbf3bd6a1f1360fc1afcee9e22c6dcff7844bf2George Wilson /^Size: [^0]/ {size = $2;
dcbf3bd6a1f1360fc1afcee9e22c6dcff7844bf2George Wilson if (first != 1) {printf(",\n")} else {first = 0}
dcbf3bd6a1f1360fc1afcee9e22c6dcff7844bf2George Wilson printf(" \"%s\": \"%s\"", disk, size)}
dcbf3bd6a1f1360fc1afcee9e22c6dcff7844bf2George Wilson END {printf("\n },\n")}' >>$config
dcbf3bd6a1f1360fc1afcee9e22c6dcff7844bf2George Wilson $SED -n 's/^set \(.*\)[ ]=[ ]\(.*\)/\1=\2/p' /etc/system | \
dcbf3bd6a1f1360fc1afcee9e22c6dcff7844bf2George Wilson $AWK -F= 'BEGIN {printf(" \"system\": {\n"); first = 1}
dcbf3bd6a1f1360fc1afcee9e22c6dcff7844bf2George Wilson {if (first != 1) {printf(",\n")} else {first = 0};
dcbf3bd6a1f1360fc1afcee9e22c6dcff7844bf2George Wilson printf(" \"%s\": %s", $1, $2)}
dcbf3bd6a1f1360fc1afcee9e22c6dcff7844bf2George Wilson END {printf("\n }\n")}' >>$config
dcbf3bd6a1f1360fc1afcee9e22c6dcff7844bf2George Wilson echo "}" >>$config
dcbf3bd6a1f1360fc1afcee9e22c6dcff7844bf2George Wilsonfunction num_jobs_by_cpu
dcbf3bd6a1f1360fc1afcee9e22c6dcff7844bf2George Wilson typeset ncpu=$($PSRINFO | $WC -l)
dcbf3bd6a1f1360fc1afcee9e22c6dcff7844bf2George Wilson typeset num_jobs=$ncpu
dcbf3bd6a1f1360fc1afcee9e22c6dcff7844bf2George Wilson [[ $ncpu -gt 8 ]] && num_jobs=$($ECHO "$ncpu * 3 / 4" | $BC)
dcbf3bd6a1f1360fc1afcee9e22c6dcff7844bf2George Wilson $ECHO $num_jobs
dcbf3bd6a1f1360fc1afcee9e22c6dcff7844bf2George Wilsonfunction pool_to_lun_list
dcbf3bd6a1f1360fc1afcee9e22c6dcff7844bf2George Wilson typeset pool=$1
dcbf3bd6a1f1360fc1afcee9e22c6dcff7844bf2George Wilson typeset ctd ctds devname lun
dcbf3bd6a1f1360fc1afcee9e22c6dcff7844bf2George Wilson typeset lun_list=':'
dcbf3bd6a1f1360fc1afcee9e22c6dcff7844bf2George Wilson ctds=$($ZPOOL list -v $pool | $AWK '/c[0-9]*t[0-9a-fA-F]*d[0-9]*/ \
dcbf3bd6a1f1360fc1afcee9e22c6dcff7844bf2George Wilson for ctd in $ctds; do
dcbf3bd6a1f1360fc1afcee9e22c6dcff7844bf2George Wilson # Get the device name as it appears in /etc/path_to_inst
dcbf3bd6a1f1360fc1afcee9e22c6dcff7844bf2George Wilson devname=$($READLINK -f /dev/dsk/${ctd}s0 | $SED -n \
dcbf3bd6a1f1360fc1afcee9e22c6dcff7844bf2George Wilson 's/\/devices\([^:]*\):.*/\1/p')
dcbf3bd6a1f1360fc1afcee9e22c6dcff7844bf2George Wilson # Add a string composed of the driver name and instance
dcbf3bd6a1f1360fc1afcee9e22c6dcff7844bf2George Wilson # number to the list for comparison with dev_statname.
dcbf3bd6a1f1360fc1afcee9e22c6dcff7844bf2George Wilson lun=$($SED 's/"//g' /etc/path_to_inst | $GREP $devname | $AWK \
dcbf3bd6a1f1360fc1afcee9e22c6dcff7844bf2George Wilson '{print $3$2}')
dcbf3bd6a1f1360fc1afcee9e22c6dcff7844bf2George Wilson lun_list="$lun_list$lun:"
dcbf3bd6a1f1360fc1afcee9e22c6dcff7844bf2George Wilson echo $lun_list
dcbf3bd6a1f1360fc1afcee9e22c6dcff7844bf2George Wilson# Create a perf_data directory to hold performance statistics and
dcbf3bd6a1f1360fc1afcee9e22c6dcff7844bf2George Wilson# configuration information.
dcbf3bd6a1f1360fc1afcee9e22c6dcff7844bf2George Wilsonexport PERF_DATA_DIR=$(get_perf_output_dir)
dcbf3bd6a1f1360fc1afcee9e22c6dcff7844bf2George Wilson[[ -f $PERF_DATA_DIR/config.json ]] || get_system_config config.json