diff options
Diffstat (limited to 'src/spdk/test/common/autotest_common.sh')
| -rwxr-xr-x | src/spdk/test/common/autotest_common.sh | 1350 |
1 files changed, 1350 insertions, 0 deletions
diff --git a/src/spdk/test/common/autotest_common.sh b/src/spdk/test/common/autotest_common.sh new file mode 100755 index 000000000..32cd4e32a --- /dev/null +++ b/src/spdk/test/common/autotest_common.sh @@ -0,0 +1,1350 @@ +#!/usr/bin/env bash + +function xtrace_disable() { + if [ "$XTRACE_DISABLED" != "yes" ]; then + PREV_BASH_OPTS="$-" + if [[ "$PREV_BASH_OPTS" == *"x"* ]]; then + XTRACE_DISABLED="yes" + fi + set +x + elif [ -z $XTRACE_NESTING_LEVEL ]; then + XTRACE_NESTING_LEVEL=1 + else + XTRACE_NESTING_LEVEL=$((++XTRACE_NESTING_LEVEL)) + fi +} + +xtrace_disable +set -e +shopt -s expand_aliases + +source "$rootdir/test/common/applications.sh" +if [[ -e $rootdir/test/common/build_config.sh ]]; then + source "$rootdir/test/common/build_config.sh" +elif [[ -e $rootdir/mk/config.mk ]]; then + build_config=$(< "$rootdir/mk/config.mk") + source <(echo "${build_config//\?=/=}") +else + source "$rootdir/CONFIG" +fi + +# Dummy function to be called after restoring xtrace just so that it appears in the +# xtrace log. This way we can consistently track when xtrace is enabled/disabled. +function xtrace_enable() { + # We have to do something inside a function in bash, and calling any command + # (even `:`) will produce an xtrace entry, so we just define another function. + function xtrace_dummy() { :; } +} + +# Keep it as alias to avoid xtrace_enable backtrace always pointing to xtrace_restore. +# xtrace_enable will appear as called directly from the user script, from the same line +# that "called" xtrace_restore. +alias xtrace_restore='if [ -z $XTRACE_NESTING_LEVEL ]; then + if [[ "$PREV_BASH_OPTS" == *"x"* ]]; then + XTRACE_DISABLED="no"; PREV_BASH_OPTS=""; set -x; xtrace_enable; + fi +else + XTRACE_NESTING_LEVEL=$((--XTRACE_NESTING_LEVEL)); + if [ $XTRACE_NESTING_LEVEL -eq "0" ]; then + unset XTRACE_NESTING_LEVEL + fi +fi' + +: ${RUN_NIGHTLY:=0} +export RUN_NIGHTLY + +# Set defaults for missing test config options +: ${SPDK_AUTOTEST_DEBUG_APPS:=0} +export SPDK_AUTOTEST_DEBUG_APPS +: ${SPDK_RUN_VALGRIND=0} +export SPDK_RUN_VALGRIND +: ${SPDK_RUN_FUNCTIONAL_TEST=0} +export SPDK_RUN_FUNCTIONAL_TEST +: ${SPDK_TEST_UNITTEST=0} +export SPDK_TEST_UNITTEST +: ${SPDK_TEST_AUTOBUILD=0} +export SPDK_TEST_AUTOBUILD +: ${SPDK_TEST_ISAL=0} +export SPDK_TEST_ISAL +: ${SPDK_TEST_ISCSI=0} +export SPDK_TEST_ISCSI +: ${SPDK_TEST_ISCSI_INITIATOR=0} +export SPDK_TEST_ISCSI_INITIATOR +: ${SPDK_TEST_NVME=0} +export SPDK_TEST_NVME +: ${SPDK_TEST_NVME_CLI=0} +export SPDK_TEST_NVME_CLI +: ${SPDK_TEST_NVME_CUSE=0} +export SPDK_TEST_NVME_CUSE +: ${SPDK_TEST_NVMF=0} +export SPDK_TEST_NVMF +: ${SPDK_TEST_NVMF_TRANSPORT="rdma"} +export SPDK_TEST_NVMF_TRANSPORT +: ${SPDK_TEST_RBD=0} +export SPDK_TEST_RBD +: ${SPDK_TEST_VHOST=0} +export SPDK_TEST_VHOST +: ${SPDK_TEST_BLOCKDEV=0} +export SPDK_TEST_BLOCKDEV +: ${SPDK_TEST_IOAT=0} +export SPDK_TEST_IOAT +: ${SPDK_TEST_BLOBFS=0} +export SPDK_TEST_BLOBFS +: ${SPDK_TEST_VHOST_INIT=0} +export SPDK_TEST_VHOST_INIT +: ${SPDK_TEST_PMDK=0} +export SPDK_TEST_PMDK +: ${SPDK_TEST_LVOL=0} +export SPDK_TEST_LVOL +: ${SPDK_TEST_JSON=0} +export SPDK_TEST_JSON +: ${SPDK_TEST_REDUCE=0} +export SPDK_TEST_REDUCE +: ${SPDK_TEST_VPP=0} +export SPDK_TEST_VPP +: ${SPDK_RUN_ASAN=0} +export SPDK_RUN_ASAN +: ${SPDK_RUN_UBSAN=0} +export SPDK_RUN_UBSAN +: ${SPDK_RUN_INSTALLED_DPDK=0} +export SPDK_RUN_INSTALLED_DPDK +: ${SPDK_RUN_NON_ROOT=0} +export SPDK_RUN_NON_ROOT +: ${SPDK_TEST_CRYPTO=0} +export SPDK_TEST_CRYPTO +: ${SPDK_TEST_FTL=0} +export SPDK_TEST_FTL +: ${SPDK_TEST_OCF=0} +export SPDK_TEST_OCF +: ${SPDK_TEST_FTL_EXTENDED=0} +export SPDK_TEST_FTL_EXTENDED +: ${SPDK_TEST_VMD=0} +export SPDK_TEST_VMD +: ${SPDK_TEST_OPAL=0} +export SPDK_TEST_OPAL +: ${SPDK_AUTOTEST_X=true} +export SPDK_AUTOTEST_X +: ${SPDK_TEST_RAID5=0} +export SPDK_TEST_RAID5 +: ${SPDK_TEST_URING=0} +export SPDK_TEST_URING + +# Export PYTHONPATH with addition of RPC framework. New scripts can be created +# specific use cases for tests. +export PYTHONPATH=$PYTHONPATH:$rootdir/scripts + +# Don't create Python .pyc files. When running with sudo these will be +# created with root ownership and can cause problems when cleaning the repository. +export PYTHONDONTWRITEBYTECODE=1 + +# Export flag to skip the known bug that exists in librados +# Bug is reported on ceph bug tracker with number 24078 +export ASAN_OPTIONS=new_delete_type_mismatch=0 +export UBSAN_OPTIONS='halt_on_error=1:print_stacktrace=1:abort_on_error=1' + +# Export LeakSanitizer option to use suppression file in order to prevent false positives +# and known leaks in external executables or libraries from showing up. +asan_suppression_file="/var/tmp/asan_suppression_file" +sudo rm -rf "$asan_suppression_file" +cat << EOL >> "$asan_suppression_file" +# ASAN has some bugs around thread_local variables. We have a destructor in place +# to free the thread contexts, but ASAN complains about the leak before those +# destructors have a chance to run. So suppress this one specific leak using +# LSAN_OPTIONS. +leak:spdk_fs_alloc_thread_ctx + +# Suppress known leaks in fio project +leak:$CONFIG_FIO_SOURCE_DIR/parse.c +leak:$CONFIG_FIO_SOURCE_DIR/iolog.c +leak:$CONFIG_FIO_SOURCE_DIR/init.c +leak:$CONFIG_FIO_SOURCE_DIR/filesetup.c +leak:fio_memalign +leak:spdk_fio_io_u_init + +# Suppress leaks in libiscsi +leak:libiscsi.so +EOL + +# Suppress leaks in libfuse3 +echo "leak:libfuse3.so" >> "$asan_suppression_file" + +export LSAN_OPTIONS=suppressions="$asan_suppression_file" + +export DEFAULT_RPC_ADDR="/var/tmp/spdk.sock" + +if [ -z "$DEPENDENCY_DIR" ]; then + export DEPENDENCY_DIR=/home/sys_sgsw +else + export DEPENDENCY_DIR +fi + +# Export location of where all the SPDK binaries are +export SPDK_BIN_DIR="$rootdir/build/bin" +export SPDK_EXAMPLE_DIR="$rootdir/build/examples" + +# pass our valgrind desire on to unittest.sh +if [ $SPDK_RUN_VALGRIND -eq 0 ]; then + export valgrind='' +fi + +if [ "$(uname -s)" = "Linux" ]; then + MAKE="make" + MAKEFLAGS=${MAKEFLAGS:--j$(nproc)} + DPDK_LINUX_DIR=/usr/share/dpdk/x86_64-default-linuxapp-gcc + if [ -d $DPDK_LINUX_DIR ] && [ $SPDK_RUN_INSTALLED_DPDK -eq 1 ]; then + WITH_DPDK_DIR=$DPDK_LINUX_DIR + fi + # Override the default HUGEMEM in scripts/setup.sh to allocate 8GB in hugepages. + export HUGEMEM=8192 +elif [ "$(uname -s)" = "FreeBSD" ]; then + MAKE="gmake" + MAKEFLAGS=${MAKEFLAGS:--j$(sysctl -a | grep -E -i 'hw.ncpu' | awk '{print $2}')} + DPDK_FREEBSD_DIR=/usr/local/share/dpdk/x86_64-native-bsdapp-clang + if [ -d $DPDK_FREEBSD_DIR ] && [ $SPDK_RUN_INSTALLED_DPDK -eq 1 ]; then + WITH_DPDK_DIR=$DPDK_FREEBSD_DIR + fi + # FreeBSD runs a much more limited set of tests, so keep the default 2GB. + export HUGEMEM=2048 +else + echo "Unknown OS \"$(uname -s)\"" + exit 1 +fi + +if [ -z "$output_dir" ]; then + if [ -z "$rootdir" ] || [ ! -d "$rootdir/../output" ]; then + output_dir=. + else + output_dir=$rootdir/../output + fi + export output_dir +fi + +TEST_MODE= +for i in "$@"; do + case "$i" in + --iso) + TEST_MODE=iso + ;; + --transport=*) + TEST_TRANSPORT="${i#*=}" + ;; + --sock=*) + TEST_SOCK="${i#*=}" + ;; + esac +done + +# start rpc.py coprocess if it's not started yet +if [[ -z $RPC_PIPE_PID ]] || ! kill -0 "$RPC_PIPE_PID" &> /dev/null; then + coproc RPC_PIPE { "$rootdir/scripts/rpc.py" --server; } + exec {RPC_PIPE_OUTPUT}<&${RPC_PIPE[0]} {RPC_PIPE_INPUT}>&${RPC_PIPE[1]} + # all descriptors will automatically close together with this bash + # process, this will make rpc.py stop reading and exit gracefully +fi + +if [ $SPDK_TEST_VPP -eq 1 ]; then + VPP_PATH="/usr/local/src/vpp-19.04/build-root/install-vpp_debug-native/vpp/" + export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:${VPP_PATH}/lib/ + export PATH=${PATH}:${VPP_PATH}/bin/ +fi + +function set_test_storage() { + [[ -v testdir ]] || return 0 + + local requested_size=$1 # bytes + local mount target_dir + + local -A mounts fss sizes avails uses + local source fs size avail mount use + + local storage_fallback storage_candidates + local storage_fallback_purge + + storage_fallback_purge=("${TMPDIR:-/tmp}/spdk."??????) + + if ((${#storage_fallback_purge[@]} > 0)); then + printf '* Purging old temporary test storage (%s)\n' \ + "${storage_fallback_purge[*]}" >&2 + rm -rf "${storage_fallback_purge[@]}" + fi + + storage_fallback=$(mktemp -udt spdk.XXXXXX) + storage_candidates=( + "$testdir" + "$storage_fallback/tests/${testdir##*/}" + "$storage_fallback" + ) + + if [[ -n $ADD_TEST_STORAGE ]]; then + # List of dirs|mounts separated by whitespaces + storage_candidates+=($ADD_TEST_STORAGE) + fi + + if [[ -n $DEDICATED_TEST_STORAGE ]]; then + # Single, dedicated dir|mount + storage_candidates=("$DEDICATED_TEST_STORAGE") + fi + + mkdir -p "${storage_candidates[@]}" + + # add some headroom - 64M + requested_size=$((requested_size + (64 << 20))) + + while read -r source fs size use avail _ mount; do + mounts["$mount"]=$source fss["$mount"]=$fs + avails["$mount"]=$((avail * 1024)) sizes["$mount"]=$((size * 1024)) + uses["$mount"]=$((use * 1024)) + done < <(df -T | grep -v Filesystem) + + printf '* Looking for test storage...\n' >&2 + + local target_space new_size + for target_dir in "${storage_candidates[@]}"; do + # FreeBSD's df is lacking the --output arg + # mount=$(df --output=target "$target_dir" | grep -v "Mounted on") + mount=$(df "$target_dir" | awk '$1 !~ /Filesystem/{print $6}') + + target_space=${avails["$mount"]} + if ((target_space == 0 || target_space < requested_size)); then + continue + fi + if ((target_space >= requested_size)); then + # For in-memory fs, and / make sure our requested size won't fill most of the space. + if [[ ${fss["$mount"]} == tmpfs ]] || [[ ${fss["$mount"]} == ramfs ]] || [[ $mount == / ]]; then + new_size=$((uses["$mount"] + requested_size)) + if ((new_size * 100 / sizes["$mount"] > 95)); then + continue + fi + fi + fi + export SPDK_TEST_STORAGE=$target_dir + printf '* Found test storage at %s\n' "$SPDK_TEST_STORAGE" >&2 + return 0 + done + printf '* Test storage is not available\n' + return 1 +} + +function get_config_params() { + xtrace_disable + config_params='--enable-debug --enable-werror' + + # for options with dependencies but no test flag, set them here + if [ -f /usr/include/infiniband/verbs.h ]; then + config_params+=' --with-rdma' + fi + + if [ $(uname -s) == "FreeBSD" ]; then + intel="hw.model: Intel" + cpu_vendor=$(sysctl -a | grep hw.model | cut -c 1-15) + else + intel="GenuineIntel" + cpu_vendor=$(grep -i 'vendor' /proc/cpuinfo --max-count=1) + fi + if [[ "$cpu_vendor" != *"$intel"* ]]; then + config_params+=" --without-idxd" + else + config_params+=" --with-idxd" + fi + + if [[ -d $CONFIG_FIO_SOURCE_DIR ]]; then + config_params+=" --with-fio=$CONFIG_FIO_SOURCE_DIR" + fi + + if [ -d ${DEPENDENCY_DIR}/vtune_codes ]; then + config_params+=' --with-vtune='${DEPENDENCY_DIR}'/vtune_codes' + fi + + if [ -d /usr/include/iscsi ]; then + libiscsi_version=$(grep LIBISCSI_API_VERSION /usr/include/iscsi/iscsi.h | head -1 | awk '{print $3}' | awk -F '(' '{print $2}' | awk -F ')' '{print $1}') + if [ $libiscsi_version -ge 20150621 ]; then + config_params+=' --with-iscsi-initiator' + fi + fi + + if [ $SPDK_TEST_UNITTEST -eq 0 ]; then + config_params+=' --disable-unit-tests' + fi + + if [ $SPDK_TEST_NVME_CUSE -eq 1 ]; then + config_params+=' --with-nvme-cuse' + fi + + # for options with both dependencies and a test flag, set them here + if [ -f /usr/include/libpmemblk.h ] && [ $SPDK_TEST_PMDK -eq 1 ]; then + config_params+=' --with-pmdk' + fi + + if [ -f /usr/include/libpmem.h ] && [ $SPDK_TEST_REDUCE -eq 1 ]; then + if [ $SPDK_TEST_ISAL -eq 1 ]; then + config_params+=' --with-reduce' + fi + fi + + if [ -d /usr/include/rbd ] && [ -d /usr/include/rados ] && [ $SPDK_TEST_RBD -eq 1 ]; then + config_params+=' --with-rbd' + fi + + if [ $SPDK_TEST_VPP -eq 1 ]; then + config_params+=" --with-vpp=${VPP_PATH}" + fi + + # for options with no required dependencies, just test flags, set them here + if [ $SPDK_TEST_CRYPTO -eq 1 ]; then + config_params+=' --with-crypto' + fi + + if [ $SPDK_TEST_OCF -eq 1 ]; then + config_params+=" --with-ocf" + fi + + if [ $SPDK_RUN_UBSAN -eq 1 ]; then + config_params+=' --enable-ubsan' + fi + + if [ $SPDK_RUN_ASAN -eq 1 ]; then + config_params+=' --enable-asan' + fi + + if [ "$(uname -s)" = "Linux" ]; then + config_params+=' --enable-coverage' + fi + + if [ $SPDK_TEST_ISAL -eq 0 ]; then + config_params+=' --without-isal' + fi + + if [ $SPDK_TEST_BLOBFS -eq 1 ]; then + if [[ -d /usr/include/fuse3 ]] || [[ -d /usr/local/include/fuse3 ]]; then + config_params+=' --with-fuse' + fi + fi + + if [ $SPDK_TEST_RAID5 -eq 1 ]; then + config_params+=' --with-raid5' + fi + + # Check whether liburing library header exists + if [ -f /usr/include/liburing/io_uring.h ] && [ $SPDK_TEST_URING -eq 1 ]; then + config_params+=' --with-uring' + fi + + # By default, --with-dpdk is not set meaning the SPDK build will use the DPDK submodule. + # If a DPDK installation is found in a well-known location though, WITH_DPDK_DIR will be + # set which will override the default and use that DPDK installation instead. + if [ -n "$WITH_DPDK_DIR" ]; then + config_params+=" --with-dpdk=$WITH_DPDK_DIR" + fi + + echo "$config_params" + xtrace_restore +} + +function rpc_cmd() { + xtrace_disable + local rsp rc + + echo "$@" >&$RPC_PIPE_INPUT + while read -t 5 -ru $RPC_PIPE_OUTPUT rsp; do + if [[ $rsp == "**STATUS="* ]]; then + break + fi + echo "$rsp" + done + + rc=${rsp#*=} + xtrace_restore + [[ $rc == 0 ]] +} + +function rpc_cmd_simple_data_json() { + + local elems="$1[@]" elem + local -gA jq_out=() + local jq val + + local lvs=( + "uuid" + "name" + "base_bdev" + "total_data_clusters" + "free_clusters" + "block_size" + "cluster_size" + ) + + local bdev=( + "name" + "aliases[0]" + "block_size" + "num_blocks" + "uuid" + "product_name" + ) + + [[ -v $elems ]] || return 1 + + for elem in "${!elems}"; do + jq="${jq:+$jq,\"\\n\",}\"$elem\",\" \",.[0].$elem" + done + jq+=',"\n"' + + shift + while read -r elem val; do + jq_out["$elem"]=$val + done < <(rpc_cmd "$@" | jq -jr "$jq") + ((${#jq_out[@]} > 0)) || return 1 +} + +# invert error code of any command and also trigger ERR on 0 (unlike bash ! prefix) +function NOT() { + if "$@"; then + return 1 + fi +} + +function timing() { + direction="$1" + testname="$2" + + now=$(date +%s) + + if [ "$direction" = "enter" ]; then + export timing_stack="${timing_stack};${now}" + export test_stack="${test_stack};${testname}" + else + touch "$output_dir/timing.txt" + child_time=$(grep "^${test_stack:1};" $output_dir/timing.txt | awk '{s+=$2} END {print s}') + + start_time=$(echo "$timing_stack" | sed -e 's@^.*;@@') + timing_stack=$(echo "$timing_stack" | sed -e 's@;[^;]*$@@') + + elapsed=$((now - start_time - child_time)) + echo "${test_stack:1} $elapsed" >> $output_dir/timing.txt + + test_stack=$(echo "$test_stack" | sed -e 's@;[^;]*$@@') + fi +} + +function timing_enter() { + xtrace_disable + timing "enter" "$1" + xtrace_restore +} + +function timing_exit() { + xtrace_disable + timing "exit" "$1" + xtrace_restore +} + +function timing_finish() { + flamegraph='/usr/local/FlameGraph/flamegraph.pl' + if [ -x "$flamegraph" ]; then + "$flamegraph" \ + --title 'Build Timing' \ + --nametype 'Step:' \ + --countname seconds \ + $output_dir/timing.txt \ + > $output_dir/timing.svg + fi +} + +function create_test_list() { + xtrace_disable + # First search all scripts in main SPDK directory. + completion=$(grep -shI -d skip --include="*.sh" -e "run_test " $rootdir/*) + # Follow up with search in test directory recursively. + completion+=$(grep -rshI --include="*.sh" --exclude="autotest_common.sh" -e "run_test " $rootdir/test) + printf "%s" "$completion" | grep -v "#" \ + | sed 's/^.*run_test/run_test/' | awk '{print $2}' \ + | sed 's/\"//g' | sort > $output_dir/all_tests.txt || true + xtrace_restore +} + +function gdb_attach() { + gdb -q --batch \ + -ex 'handle SIGHUP nostop pass' \ + -ex 'handle SIGQUIT nostop pass' \ + -ex 'handle SIGPIPE nostop pass' \ + -ex 'handle SIGALRM nostop pass' \ + -ex 'handle SIGTERM nostop pass' \ + -ex 'handle SIGUSR1 nostop pass' \ + -ex 'handle SIGUSR2 nostop pass' \ + -ex 'handle SIGCHLD nostop pass' \ + -ex 'set print thread-events off' \ + -ex 'cont' \ + -ex 'thread apply all bt' \ + -ex 'quit' \ + --tty=/dev/stdout \ + -p $1 +} + +function process_core() { + ret=0 + while IFS= read -r -d '' core; do + exe=$(eu-readelf -n "$core" | grep psargs | sed "s/.*psargs: \([^ \'\" ]*\).*/\1/") + if [[ ! -f "$exe" ]]; then + exe=$(eu-readelf -n "$core" | grep -oP -m1 "$exe.+") + fi + echo "exe for $core is $exe" + if [[ -n "$exe" ]]; then + if hash gdb &> /dev/null; then + gdb -batch -ex "thread apply all bt full" $exe $core + fi + cp $exe $output_dir + fi + mv $core $output_dir + chmod a+r $output_dir/$core + ret=1 + done < <(find . -type f \( -name 'core\.?[0-9]*' -o -name '*.core' \) -print0) + return $ret +} + +function process_shm() { + type=$1 + id=$2 + if [ "$type" = "--pid" ]; then + id="pid${id}" + elif [ "$type" = "--id" ]; then + id="${id}" + else + echo "Please specify to search for pid or shared memory id." + return 1 + fi + + shm_files=$(find /dev/shm -name "*.${id}" -printf "%f\n") + + if [[ -z $shm_files ]]; then + echo "SHM File for specified PID or shared memory id: ${id} not found!" + return 1 + fi + for n in $shm_files; do + tar -C /dev/shm/ -cvzf $output_dir/${n}_shm.tar.gz ${n} + done + return 0 +} + +function waitforlisten() { + # $1 = process pid + if [ -z "$1" ]; then + exit 1 + fi + + local rpc_addr="${2:-$DEFAULT_RPC_ADDR}" + + echo "Waiting for process to start up and listen on UNIX domain socket $rpc_addr..." + # turn off trace for this loop + xtrace_disable + local ret=0 + local i + for ((i = 40; i != 0; i--)); do + # if the process is no longer running, then exit the script + # since it means the application crashed + if ! kill -s 0 $1; then + echo "ERROR: process (pid: $1) is no longer running" + ret=1 + break + fi + + if $rootdir/scripts/rpc.py -t 1 -s "$rpc_addr" rpc_get_methods &> /dev/null; then + break + fi + + sleep 0.5 + done + + xtrace_restore + if ((i == 0)); then + echo "ERROR: timeout while waiting for process (pid: $1) to start listening on '$rpc_addr'" + ret=1 + fi + return $ret +} + +function waitfornbd() { + local nbd_name=$1 + local i + + for ((i = 1; i <= 20; i++)); do + if grep -q -w $nbd_name /proc/partitions; then + break + else + sleep 0.1 + fi + done + + # The nbd device is now recognized as a block device, but there can be + # a small delay before we can start I/O to that block device. So loop + # here trying to read the first block of the nbd block device to a temp + # file. Note that dd returns success when reading an empty file, so we + # need to check the size of the output file instead. + for ((i = 1; i <= 20; i++)); do + dd if=/dev/$nbd_name of="$SPDK_TEST_STORAGE/nbdtest" bs=4096 count=1 iflag=direct + size=$(stat -c %s "$SPDK_TEST_STORAGE/nbdtest") + rm -f "$SPDK_TEST_STORAGE/nbdtest" + if [ "$size" != "0" ]; then + return 0 + else + sleep 0.1 + fi + done + + return 1 +} + +function waitforbdev() { + local bdev_name=$1 + local i + + for ((i = 1; i <= 20; i++)); do + if $rpc_py bdev_get_bdevs | jq -r '.[] .name' | grep -qw $bdev_name; then + return 0 + fi + + if $rpc_py bdev_get_bdevs | jq -r '.[] .aliases' | grep -qw $bdev_name; then + return 0 + fi + + sleep 0.1 + done + + return 1 +} + +function make_filesystem() { + local fstype=$1 + local dev_name=$2 + local i=0 + local force + + if [ $fstype = ext4 ]; then + force=-F + else + force=-f + fi + + while ! mkfs.${fstype} $force ${dev_name}; do + if [ $i -ge 15 ]; then + return 1 + fi + i=$((i + 1)) + sleep 1 + done + + return 0 +} + +function killprocess() { + # $1 = process pid + if [ -z "$1" ]; then + exit 1 + fi + + if kill -0 $1; then + if [ $(uname) = Linux ]; then + process_name=$(ps --no-headers -o comm= $1) + else + process_name=$(ps -c -o command $1 | tail -1) + fi + if [ "$process_name" = "sudo" ]; then + # kill the child process, which is the actual app + # (assume $1 has just one child) + local child + child="$(pgrep -P $1)" + echo "killing process with pid $child" + kill $child + else + echo "killing process with pid $1" + kill $1 + fi + + # wait for the process regardless if its the dummy sudo one + # or the actual app - it should terminate anyway + wait $1 + else + # the process is not there anymore + echo "Process with pid $1 is not found" + exit 1 + fi +} + +function iscsicleanup() { + echo "Cleaning up iSCSI connection" + iscsiadm -m node --logout || true + iscsiadm -m node -o delete || true + rm -rf /var/lib/iscsi/nodes/* +} + +function stop_iscsi_service() { + if cat /etc/*-release | grep Ubuntu; then + service open-iscsi stop + else + service iscsid stop + fi +} + +function start_iscsi_service() { + if cat /etc/*-release | grep Ubuntu; then + service open-iscsi start + else + service iscsid start + fi +} + +function rbd_setup() { + # $1 = monitor ip address + # $2 = name of the namespace + if [ -z "$1" ]; then + echo "No monitor IP address provided for ceph" + exit 1 + fi + if [ -n "$2" ]; then + if ip netns list | grep "$2"; then + NS_CMD="ip netns exec $2" + else + echo "No namespace $2 exists" + exit 1 + fi + fi + + if hash ceph; then + export PG_NUM=128 + export RBD_POOL=rbd + export RBD_NAME=foo + $NS_CMD $rootdir/scripts/ceph/stop.sh || true + $NS_CMD $rootdir/scripts/ceph/start.sh $1 + + $NS_CMD ceph osd pool create $RBD_POOL $PG_NUM || true + $NS_CMD rbd create $RBD_NAME --size 1000 + fi +} + +function rbd_cleanup() { + if hash ceph; then + $rootdir/scripts/ceph/stop.sh || true + rm -f /var/tmp/ceph_raw.img + fi +} + +function nvme_cli_build() { + if [[ -z "${DEPENDENCY_DIR}" ]]; then + echo DEPENDENCY_DIR not defined! + exit 1 + fi + + spdk_nvme_cli="${DEPENDENCY_DIR}/nvme-cli" + + if [[ ! -d $spdk_nvme_cli ]]; then + echo "nvme-cli repository not found at $spdk_nvme_cli; skipping tests." + exit 1 + fi + + if ! grep -q "DEF_VER=v1.6" $spdk_nvme_cli/NVME-VERSION-GEN; then + echo "SPDK supports only \"spdk/nvme-cli\" project on \"spdk-1.6\" branch." + exit 1 + fi + + # Build against the version of SPDK under test + pushd $spdk_nvme_cli + + # Remove and recreate git index in case it became corrupted + if ! git clean -dfx; then + rm -f .git/index + git clean -dfx + git reset --hard + fi + + rm -f "$spdk_nvme_cli/spdk" + ln -sf "$rootdir" "$spdk_nvme_cli/spdk" + + make -j$(nproc) LDFLAGS="$(make -s -C $spdk_nvme_cli/spdk ldflags)" + popd +} + +function _start_stub() { + # Disable ASLR for multi-process testing. SPDK does support using DPDK multi-process, + # but ASLR can still be unreliable in some cases. + # We will reenable it again after multi-process testing is complete in kill_stub(). + # Save current setting so it can be restored upon calling kill_stub(). + _randomize_va_space=$(< /proc/sys/kernel/randomize_va_space) + echo 0 > /proc/sys/kernel/randomize_va_space + $rootdir/test/app/stub/stub $1 & + stubpid=$! + echo Waiting for stub to ready for secondary processes... + while ! [ -e /var/run/spdk_stub0 ]; do + # If stub dies while we wait, bail + [[ -e /proc/$stubpid ]] || return 1 + sleep 1s + done + echo done. +} + +function start_stub() { + if ! _start_stub "$@"; then + echo "stub failed" >&2 + return 1 + fi +} + +function kill_stub() { + if [[ -e /proc/$stubpid ]]; then + kill $1 $stubpid + wait $stubpid + fi 2> /dev/null || : + rm -f /var/run/spdk_stub0 + # Re-enable ASLR now that we are done with multi-process testing + # Note: "1" enables ASLR w/o randomizing data segments, "2" adds data segment + # randomizing and is the default on all recent Linux kernels + echo "${_randomize_va_space:-2}" > /proc/sys/kernel/randomize_va_space +} + +function run_test() { + if [ $# -le 1 ]; then + echo "Not enough parameters" + echo "usage: run_test test_name test_script [script_params]" + exit 1 + fi + + xtrace_disable + local test_name="$1" + shift + + if [ -n "$test_domain" ]; then + export test_domain="${test_domain}.${test_name}" + else + export test_domain="$test_name" + fi + + timing_enter $test_name + echo "************************************" + echo "START TEST $test_name" + echo "************************************" + xtrace_restore + time "$@" + xtrace_disable + echo "************************************" + echo "END TEST $test_name" + echo "************************************" + timing_exit $test_name + + export test_domain=${test_domain%"$test_name"} + if [ -n "$test_domain" ]; then + export test_domain=${test_domain%?} + fi + + if [ -z "$test_domain" ]; then + echo "top_level $test_name" >> $output_dir/test_completions.txt + else + echo "$test_domain $test_name" >> $output_dir/test_completions.txt + fi + xtrace_restore +} + +function skip_run_test_with_warning() { + echo "WARNING: $1" + echo "Test run may fail if run with autorun.sh" + echo "Please check your $rootdir/test/common/skipped_tests.txt" +} + +function print_backtrace() { + # if errexit is not enabled, don't print a backtrace + [[ "$-" =~ e ]] || return 0 + + local args=("${BASH_ARGV[@]}") + + xtrace_disable + echo "========== Backtrace start: ==========" + echo "" + for i in $(seq 1 $((${#FUNCNAME[@]} - 1))); do + local func="${FUNCNAME[$i]}" + local line_nr="${BASH_LINENO[$((i - 1))]}" + local src="${BASH_SOURCE[$i]}" + local bt="" cmdline=() + + if [[ -f $src ]]; then + bt=$(nl -w 4 -ba -nln $src | grep -B 5 -A 5 "^${line_nr}[^0-9]" \ + | sed "s/^/ /g" | sed "s/^ $line_nr /=> $line_nr /g") + fi + + # If extdebug set the BASH_ARGC[i], try to fetch all the args + if ((BASH_ARGC[i] > 0)); then + # Use argc as index to reverse the stack + local argc=${BASH_ARGC[i]} arg + for arg in "${args[@]::BASH_ARGC[i]}"; do + cmdline[argc--]="[\"$arg\"]" + done + args=("${args[@]:BASH_ARGC[i]}") + fi + + echo "in $src:$line_nr -> $func($( + IFS="," + printf '%s\n' "${cmdline[*]:-[]}" + ))" + echo " ..." + echo "${bt:-backtrace unavailable}" + echo " ..." + done + echo "" + echo "========== Backtrace end ==========" + xtrace_restore + return 0 +} + +function waitforserial() { + local i=0 + local nvme_device_counter=1 + if [[ -n "$2" ]]; then + nvme_device_counter=$2 + fi + + while [ $(lsblk -l -o NAME,SERIAL | grep -c $1) -lt $nvme_device_counter ]; do + [ $i -lt 15 ] || break + i=$((i + 1)) + echo "Waiting for devices" + sleep 1 + done + + if [[ $(lsblk -l -o NAME,SERIAL | grep -c $1) -lt $nvme_device_counter ]]; then + return 1 + fi + + return 0 +} + +function waitforserial_disconnect() { + local i=0 + while lsblk -o NAME,SERIAL | grep -q -w $1; do + [ $i -lt 15 ] || break + i=$((i + 1)) + echo "Waiting for disconnect devices" + sleep 1 + done + + if lsblk -l -o NAME | grep -q -w $1; then + return 1 + fi + + return 0 +} + +function waitforblk() { + local i=0 + while ! lsblk -l -o NAME | grep -q -w $1; do + [ $i -lt 15 ] || break + i=$((i + 1)) + sleep 1 + done + + if ! lsblk -l -o NAME | grep -q -w $1; then + return 1 + fi + + return 0 +} + +function waitforblk_disconnect() { + local i=0 + while lsblk -l -o NAME | grep -q -w $1; do + [ $i -lt 15 ] || break + i=$((i + 1)) + sleep 1 + done + + if lsblk -l -o NAME | grep -q -w $1; then + return 1 + fi + + return 0 +} + +function waitforfile() { + local i=0 + while [ ! -e $1 ]; do + [ $i -lt 200 ] || break + i=$((i + 1)) + sleep 0.1 + done + + if [ ! -e $1 ]; then + return 1 + fi + + return 0 +} + +function fio_config_gen() { + local config_file=$1 + local workload=$2 + local bdev_type=$3 + local fio_dir=$CONFIG_FIO_SOURCE_DIR + + if [ -e "$config_file" ]; then + echo "Configuration File Already Exists!: $config_file" + return 1 + fi + + if [ -z "$workload" ]; then + workload=randrw + fi + + touch $1 + + cat > $1 << EOL +[global] +thread=1 +group_reporting=1 +direct=1 +norandommap=1 +percentile_list=50:99:99.9:99.99:99.999 +time_based=1 +ramp_time=0 +EOL + + if [ "$workload" == "verify" ]; then + cat <<- EOL >> $config_file + verify=sha1 + verify_backlog=1024 + rw=randwrite + EOL + + # To avoid potential data race issue due to the AIO device + # flush mechanism, add the flag to serialize the writes. + # This is to fix the intermittent IO failure issue of #935 + if [ "$bdev_type" == "AIO" ]; then + if [[ $($fio_dir/fio --version) == *"fio-3"* ]]; then + echo "serialize_overlap=1" >> $config_file + fi + fi + elif [ "$workload" == "trim" ]; then + echo "rw=trimwrite" >> $config_file + else + echo "rw=$workload" >> $config_file + fi +} + +function fio_bdev() { + # Setup fio binary cmd line + local fio_dir=$CONFIG_FIO_SOURCE_DIR + local bdev_plugin="$rootdir/build/fio/spdk_bdev" + + # Preload AddressSanitizer library to fio if fio_plugin was compiled with it + local asan_lib + asan_lib=$(ldd $bdev_plugin | grep libasan | awk '{print $3}') + + LD_PRELOAD="$asan_lib $bdev_plugin" "$fio_dir"/fio "$@" +} + +function fio_nvme() { + # Setup fio binary cmd line + local fio_dir=$CONFIG_FIO_SOURCE_DIR + local nvme_plugin="$rootdir/build/fio/spdk_nvme" + + # Preload AddressSanitizer library to fio if fio_plugin was compiled with it + asan_lib=$(ldd $nvme_plugin | grep libasan | awk '{print $3}') + + LD_PRELOAD="$asan_lib $nvme_plugin" "$fio_dir"/fio "$@" +} + +function get_lvs_free_mb() { + local lvs_uuid=$1 + local lvs_info + local fc + local cs + lvs_info=$($rpc_py bdev_lvol_get_lvstores) + fc=$(jq ".[] | select(.uuid==\"$lvs_uuid\") .free_clusters" <<< "$lvs_info") + cs=$(jq ".[] | select(.uuid==\"$lvs_uuid\") .cluster_size" <<< "$lvs_info") + + # Change to MB's + free_mb=$((fc * cs / 1024 / 1024)) + echo "$free_mb" +} + +function get_bdev_size() { + local bdev_name=$1 + local bdev_info + local bs + local nb + bdev_info=$($rpc_py bdev_get_bdevs -b $bdev_name) + bs=$(jq ".[] .block_size" <<< "$bdev_info") + nb=$(jq ".[] .num_blocks" <<< "$bdev_info") + + # Change to MB's + bdev_size=$((bs * nb / 1024 / 1024)) + echo "$bdev_size" +} + +function autotest_cleanup() { + $rootdir/scripts/setup.sh reset + $rootdir/scripts/setup.sh cleanup + if [ $(uname -s) = "Linux" ]; then + if grep -q '#define SPDK_CONFIG_IGB_UIO_DRIVER 1' $rootdir/include/spdk/config.h; then + [[ -e /sys/module/igb_uio ]] && rmmod igb_uio + else + modprobe -r uio_pci_generic + fi + fi + rm -rf "$asan_suppression_file" +} + +function freebsd_update_contigmem_mod() { + if [ $(uname) = FreeBSD ]; then + kldunload contigmem.ko || true + if [ -n "$WITH_DPDK_DIR" ]; then + echo "Warning: SPDK only works on FreeBSD with patches that only exist in SPDK's dpdk submodule" + cp -f "$WITH_DPDK_DIR/kmod/contigmem.ko" /boot/modules/ + cp -f "$WITH_DPDK_DIR/kmod/contigmem.ko" /boot/kernel/ + cp -f "$WITH_DPDK_DIR/kmod/nic_uio.ko" /boot/modules/ + cp -f "$WITH_DPDK_DIR/kmod/nic_uio.ko" /boot/kernel/ + else + cp -f "$rootdir/dpdk/build/kmod/contigmem.ko" /boot/modules/ + cp -f "$rootdir/dpdk/build/kmod/contigmem.ko" /boot/kernel/ + cp -f "$rootdir/dpdk/build/kmod/nic_uio.ko" /boot/modules/ + cp -f "$rootdir/dpdk/build/kmod/nic_uio.ko" /boot/kernel/ + fi + fi +} + +function get_nvme_name_from_bdf() { + blkname=() + + nvme_devs=$(lsblk -d --output NAME | grep "^nvme") || true + if [ -z "$nvme_devs" ]; then + return + fi + for dev in $nvme_devs; do + link_name=$(readlink /sys/block/$dev/device/device) || true + if [ -z "$link_name" ]; then + link_name=$(readlink /sys/block/$dev/device) + fi + bdf=$(basename "$link_name") + if [ "$bdf" = "$1" ]; then + blkname+=($dev) + fi + done + + printf '%s\n' "${blkname[@]}" +} + +function get_nvme_ctrlr_from_bdf() { + bdf_sysfs_path=$(readlink -f /sys/class/nvme/nvme* | grep "$1/nvme/nvme") + if [[ -z "$bdf_sysfs_path" ]]; then + return + fi + + printf '%s\n' "$(basename $bdf_sysfs_path)" +} + +# Get BDF addresses of all NVMe drives currently attached to +# uio-pci-generic or vfio-pci +function get_nvme_bdfs() { + xtrace_disable + bdfs=$(jq -r .config[].params.traddr <<< $($rootdir/scripts/gen_nvme.sh --json)) + if [[ -z $bdfs ]]; then + echo "No devices to test on!" + exit 1 + fi + echo "$bdfs" + xtrace_restore +} + +# Same as function above, but just get the first disks BDF address +function get_first_nvme_bdf() { + head -1 <<< "$(get_nvme_bdfs)" +} + +function nvme_namespace_revert() { + $rootdir/scripts/setup.sh + sleep 1 + bdfs=$(get_nvme_bdfs) + + $rootdir/scripts/setup.sh reset + sleep 1 + + for bdf in $bdfs; do + nvme_ctrlr=/dev/$(get_nvme_ctrlr_from_bdf ${bdf}) + if [[ -z "$nvme_ctrlr" ]]; then + continue + fi + + # Check Optional Admin Command Support for Namespace Management + oacs=$(nvme id-ctrl ${nvme_ctrlr} | grep oacs | cut -d: -f2) + oacs_ns_manage=$((oacs & 0x8)) + + if [[ "$oacs_ns_manage" -ne 0 ]]; then + # This assumes every NVMe controller contains single namespace, + # encompassing Total NVM Capacity and formatted as 512 block size. + # 512 block size is needed for test/vhost/vhost_boot.sh to + # succesfully run. + + unvmcap=$(nvme id-ctrl ${nvme_ctrlr} | grep unvmcap | cut -d: -f2) + if [[ "$unvmcap" -eq 0 ]]; then + # All available space already used + continue + fi + tnvmcap=$(nvme id-ctrl ${nvme_ctrlr} | grep tnvmcap | cut -d: -f2) + blksize=512 + + size=$((tnvmcap / blksize)) + + nvme detach-ns ${nvme_ctrlr} -n 0xffffffff -c 0 || true + nvme delete-ns ${nvme_ctrlr} -n 0xffffffff || true + nvme create-ns ${nvme_ctrlr} -s ${size} -c ${size} -b ${blksize} + nvme attach-ns ${nvme_ctrlr} -n 1 -c 0 + nvme reset ${nvme_ctrlr} + waitforblk "${nvme_ctrlr}n1" + fi + done +} + +# Get BDFs based on device ID, such as 0x0a54 +function get_nvme_bdfs_by_id() { + local bdfs=() + + for bdf in $(get_nvme_bdfs); do + device=$(cat /sys/bus/pci/devices/$bdf/device) || true + if [[ "$device" == "$1" ]]; then + bdfs+=($bdf) + fi + done + + printf '%s\n' "${bdfs[@]}" +} + +function opal_revert_cleanup() { + # The OPAL CI tests is only used for P4510 devices. + mapfile -t bdfs < <(get_nvme_bdfs_by_id 0x0a54) + if [[ -z ${bdfs[0]} ]]; then + return 0 + fi + + $SPDK_BIN_DIR/spdk_tgt & + spdk_tgt_pid=$! + waitforlisten $spdk_tgt_pid + + for bdf in "${bdfs[@]}"; do + $rootdir/scripts/rpc.py bdev_nvme_attach_controller -b "nvme0" -t "pcie" -a ${bdf} + # Ignore if this fails. + $rootdir/scripts/rpc.py bdev_nvme_opal_revert -b nvme0 -p test || true + done + + killprocess $spdk_tgt_pid +} + +# Define temp storage for all the tests. Look for 2GB at minimum +set_test_storage "${TEST_MIN_STORAGE_SIZE:-$((1 << 31))}" + +set -o errtrace +shopt -s extdebug +trap "trap - ERR; print_backtrace >&2" ERR + +PS4=' \t \$ ' +if $SPDK_AUTOTEST_X; then + # explicitly enable xtraces, overriding any tracking information. + unset XTRACE_DISABLED + unset XTRACE_NESTING_LEVEL + set -x + xtrace_enable +else + xtrace_restore +fi |