Adding upstream version 18.2.2.upstream/18.2.2

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

5 files changed, 507 insertions, 0 deletions

diff --git a/src/spdk/test/dd/basic_rw.sh b/src/spdk/test/dd/basic_rw.sh
new file mode 100755
index 000000000..5e9be5363
--- /dev/null
+++ b/src/spdk/test/dd/basic_rw.sh
@@ -0,0 +1,107 @@
+#!/usr/bin/env bash
+testdir=$(readlink -f "$(dirname "$0")")
+rootdir=$(readlink -f "$testdir/../../")
+source "$testdir/common.sh"
+
+basic_rw() {
+	local native_bs=$1
+	local count size
+	local qds bss
+
+	qds=(1 64)
+	# Generate some bs for tests based on the native_bs
+	for bs in {0..4}; do
+		bss+=($((native_bs << bs)))
+	done
+
+	for bs in "${bss[@]}"; do
+		for qd in "${qds[@]}"; do
+			count=$((0xffff / bs))
+			count=$((count == 0 ? 1 : count))
+			size=$((count * bs))
+
+			gen_bytes "$size" > "$test_file0"
+
+			"${DD_APP[@]}" \
+				--if="$test_file0" \
+				--ob="$bdev0" \
+				--bs="$bs" \
+				--qd="$qd" \
+				--json <(gen_conf)
+
+			"${DD_APP[@]}" \
+				--ib="$bdev0" \
+				--of="$test_file1" \
+				--bs="$bs" \
+				--qd="$qd" \
+				--count="$count" \
+				--json <(gen_conf)
+
+			diff -q "$test_file0" "$test_file1"
+			clear_nvme "$bdev0" "" "$size"
+		done
+	done
+}
+
+basic_offset() {
+	# Check if offseting works - using default io size of 4k
+	local count seek skip data data_check
+
+	gen_bytes 4096 > "$test_file0"
+	((count = seek = skip = 1))
+	data=$(< "$test_file0")
+
+	"${DD_APP[@]}" \
+		--if="$test_file0" \
+		--ob="$bdev0" \
+		--seek="$seek" \
+		--json <(gen_conf)
+
+	"${DD_APP[@]}" \
+		--ib="$bdev0" \
+		--of="$test_file1" \
+		--skip="$skip" \
+		--count="$count" \
+		--json <(gen_conf)
+
+	read -rn${#data} data_check < "$test_file1"
+	[[ $data == "$data_check" ]]
+}
+
+plain_copy() {
+	# Test if copy between plain files works as well
+	"${DD_APP[@]}" --if="$test_file0" --of="$test_file1"
+	diff -q "$test_file0" "$test_file1"
+}
+
+cleanup() {
+	clear_nvme "$bdev0"
+	rm -f "$test_file0" "$test_file1"
+}
+
+trap "cleanup" EXIT
+
+nvmes=("$@")
+nvme0=Nvme0 nvme0_pci=${nvmes[0]} bdev0=Nvme0n1
+
+declare -A method_bdev_nvme_attach_controller_0=(
+	["name"]=$nvme0
+	["traddr"]=$nvme0_pci
+	["trtype"]=pcie
+)
+
+test_file0=$SPDK_TEST_STORAGE/dd.dump0
+test_file1=$SPDK_TEST_STORAGE/dd.dump1
+native_bs=$(get_native_nvme_bs "$nvme0_pci")
+
+# Test if running with bs < native_bs successfully fails
+run_test "dd_bs_lt_native_bs" \
+	NOT "${DD_APP[@]}" \
+	--if=<(:) \
+	--ob="$bdev0" \
+	--bs=$((native_bs >> 1)) \
+	--json <(gen_conf)
+
+run_test "dd_rw" basic_rw "$native_bs"
+run_test "dd_rw_offset" basic_offset
+run_test "dd_rw_file_copy" plain_copy
diff --git a/src/spdk/test/dd/bdev_to_bdev.sh b/src/spdk/test/dd/bdev_to_bdev.sh
new file mode 100755
index 000000000..f18705ef7
--- /dev/null
+++ b/src/spdk/test/dd/bdev_to_bdev.sh
@@ -0,0 +1,111 @@
+#!/usr/bin/env bash
+testdir=$(readlink -f "$(dirname "$0")")
+rootdir=$(readlink -f "$testdir/../../")
+source "$testdir/common.sh"
+
+nvmes=("$@")
+
+offset_magic() {
+	local magic_check
+	local offsets offset
+
+	offsets=(16 256 4096) # * bs
+
+	for offset in "${offsets[@]}"; do
+		"${DD_APP[@]}" \
+			--ib="$bdev0" \
+			--ob="$bdev1" \
+			--count="$count" \
+			--seek="$offset" \
+			--bs="$bs" \
+			--json <(gen_conf)
+
+		"${DD_APP[@]}" \
+			--ib="$bdev1" \
+			--of="$test_file1" \
+			--count=1 \
+			--skip="$offset" \
+			--bs="$bs" \
+			--json <(gen_conf)
+
+		read -rn${#magic} magic_check < "$test_file1"
+		[[ $magic_check == "$magic" ]]
+	done
+}
+
+cleanup() {
+	# Zero up to 1G on input bdev, 4G on out bdev to consider offsetting
+	clear_nvme "$bdev0" "" $((0x40000000 + ${#magic}))
+	clear_nvme "$bdev1" "" $((0x100000000 + ${#magic}))
+	rm -f "$test_file0" "$test_file1" "$aio1"
+}
+
+trap "cleanup" EXIT
+
+bs=$((1024 << 10))
+
+if ((${#nvmes[@]} > 1)); then
+	nvme0=Nvme0 bdev0=Nvme0n1 nvme0_pci=${nvmes[0]} # input bdev
+	nvme1=Nvme1 bdev1=Nvme1n1 nvme1_pci=${nvmes[1]} # output bdev
+
+	declare -A method_bdev_nvme_attach_controller_0=(
+		["name"]=$nvme0
+		["traddr"]=$nvme0_pci
+		["trtype"]=pcie
+	)
+	declare -A method_bdev_nvme_attach_controller_1=(
+		["name"]=$nvme1
+		["traddr"]=$nvme1_pci
+		["trtype"]=pcie
+	)
+else
+	# Use AIO to compensate lack of actual hardware
+	nvme0=Nvme0 bdev0=Nvme0n1 nvme0_pci=${nvmes[0]} # input bdev
+	aio1=$SPDK_TEST_STORAGE/aio1 bdev1=aio1         # output bdev
+
+	declare -A method_bdev_nvme_attach_controller_1=(
+		["name"]=$nvme0
+		["traddr"]=$nvme0_pci
+		["trtype"]=pcie
+	)
+	declare -A method_bdev_aio_create_0=(
+		["name"]=$bdev1
+		["filename"]=$aio1
+		["block_size"]=4096
+	)
+
+	# 8G AIO file
+	"${DD_APP[@]}" \
+		--if=/dev/zero \
+		--of="$aio1" \
+		--bs="$bs" \
+		--count=8192
+fi
+
+test_file0=$SPDK_TEST_STORAGE/dd.dump0
+test_file1=$SPDK_TEST_STORAGE/dd.dump1
+
+magic="This Is Our Magic, find it"
+echo "$magic" > "$test_file0"
+
+# Make the file a bit bigger (~1GB)
+run_test "dd_inflate_file" \
+	"${DD_APP[@]}" \
+	--if=/dev/zero \
+	--of="$test_file0" \
+	--oflag=append \
+	--bs="$bs" \
+	--count=1024
+
+test_file0_size=$(wc -c < "$test_file0")
+
+# Now, copy it over to first nvme with default bs (4k)
+run_test "dd_copy_to_out_bdev" \
+	"${DD_APP[@]}" \
+	--if="$test_file0" \
+	--ob="$bdev0" \
+	--json <(gen_conf)
+
+count=$(((test_file0_size / bs) + 1))
+
+run_test "dd_offset_magic" offset_magic
diff --git a/src/spdk/test/dd/common.sh b/src/spdk/test/dd/common.sh
new file mode 100644
index 000000000..d2f7defa3
--- /dev/null
+++ b/src/spdk/test/dd/common.sh
@@ -0,0 +1,154 @@
+source "$rootdir/test/common/autotest_common.sh"
+source "$rootdir/scripts/common.sh"
+
+clear_nvme() {
+	local bdev=$1
+	local nvme_ref=$2
+	local size=${3:-0xffff}
+
+	local bs=$((1024 << 10)) # 1M
+	local count=$(((size / bs) + (size % bs ? 1 : 0)))
+
+	"${DD_APP[@]}" \
+		--if="/dev/zero" \
+		--bs="$bs" \
+		--ob="$bdev" \
+		--count="$count" \
+		--json <(gen_conf $nvme_ref)
+}
+
+trunc_files() {
+	local f
+	for f; do : > "$f"; done
+}
+
+gen_conf() {
+	xtrace_disable
+
+	local ref_name
+	local method methods
+	local param params
+	local config
+
+	# Pick references to all assoc arrays and build subsystem's config
+	# around them. The assoc array should be the name of the rpc method
+	# suffixed with unique _ID (ID may be any string). Default arrays
+	# should be prefixed with _method string. The keys of the array
+	# should store names of the method's parameters - proper quoting
+	# of the values is done here. extra_subsystems[] can store extra
+	# json configuration for different subsystems, other than bdev.
+
+	methods=("${@:-${!method_@}}")
+	local IFS=","
+
+	for ref_name in "${methods[@]}"; do
+		method=${ref_name#*method_} method=${method%_*} params=()
+
+		# FIXME: centos7's Bash got trapped in 2011:
+		# local -n ref=$ref_name -> local: -n: invalid option
+		# HACK: it with eval and partial refs instead.
+		eval "local refs=(\${!${ref_name}[@]})"
+		local param_ref
+
+		for param in "${refs[@]}"; do
+			param_ref="${ref_name}[$param]"
+			if [[ ${!param_ref} =~ ^([0-9]+|true|false|\{.*\})$ ]]; then
+				params+=("\"$param\": ${!param_ref}")
+			else
+				params+=("\"$param\": \"${!param_ref}\"")
+			fi
+		done
+
+		config+=("$(
+			cat <<- JSON
+				{
+				  "params": {
+				    ${params[*]}
+				  },
+				  "method": "$method"
+				}
+			JSON
+		)")
+	done
+
+	jq . <<- JSON | tee /dev/stderr
+		{
+		  "subsystems": [
+		    {
+		      "subsystem": "bdev",
+		      "config": [
+		        ${config[*]}
+		      ]
+		    }
+		    ${extra_subsystems[*]:+,${extra_subsystems[*]}}
+		  ]
+		}
+	JSON
+
+	xtrace_restore
+}
+
+gen_bytes() {
+	xtrace_disable
+
+	local max=$1
+	local bytes
+	local byte
+	local string
+	shift
+
+	bytes=({a..z} {0..9})
+	if (($#)); then
+		bytes=("$@")
+	fi
+
+	for ((byte = 0; byte < max; byte++)); do
+		string+=${bytes[RANDOM % ${#bytes[@]}]}
+	done
+	printf '%b' "$string"
+
+	xtrace_restore
+}
+
+get_native_nvme_bs() {
+	# This is now needed since spdk_dd will reject all bs smaller than the
+	# native bs of given nvme. We need to make sure all tests are using
+	# bs >= native_bs. Use identify here so we don't have to switch nvmes
+	# between user space and the kernel back and forth.
+	local pci=$1 lbaf id
+
+	mapfile -t id < <("$rootdir/build/examples/identify" -r trtype:pcie "traddr:$pci")
+
+	# Get size of the current LBAF
+	[[ ${id[*]} =~ "Current LBA Format:"\ *"LBA Format #"([0-9]+) ]]
+	lbaf=${BASH_REMATCH[1]}
+	[[ ${id[*]} =~ "LBA Format #$lbaf: Data Size:"\ *([0-9]+) ]]
+	lbaf=${BASH_REMATCH[1]}
+
+	echo "$lbaf"
+}
+
+check_liburing() {
+	# Simply check if spdk_dd links to liburing. If yes, log that information.
+	local lib so
+	local -g liburing_in_use=0
+
+	while read -r lib _ so _; do
+		if [[ $lib == liburing.so.* ]]; then
+			printf '* spdk_dd linked to liburing\n'
+			# For sanity, check build config to see if liburing was requested.
+			if [[ -e $rootdir/test/common/build_config.sh ]]; then
+				source "$rootdir/test/common/build_config.sh"
+			fi
+			if [[ $CONFIG_URING != y ]]; then
+				printf '* spdk_dd built with liburing, but no liburing support requested?\n'
+			fi
+			if [[ ! -e $so ]]; then
+				printf '* %s is missing, aborting\n' "$lib"
+				return 1
+			fi
+			export liburing_in_use=1
+			return 0
+		fi
+	done < <(LD_TRACE_LOADED_OBJECTS=1 "${DD_APP[@]}") >&2
+}
diff --git a/src/spdk/test/dd/dd.sh b/src/spdk/test/dd/dd.sh
new file mode 100755
index 000000000..e2b8bb86a
--- /dev/null
+++ b/src/spdk/test/dd/dd.sh
@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+testdir=$(readlink -f "$(dirname "$0")")
+rootdir=$(readlink -f "$testdir/../../")
+source "$testdir/common.sh"
+
+"$rootdir/scripts/setup.sh"
+nvmes=($(nvme_in_userspace))
+
+check_liburing
+
+run_test "spdk_dd_basic_rw" "$testdir/basic_rw.sh" "${nvmes[@]}"
+run_test "spdk_dd_posix" "$testdir/posix.sh"
+run_test "spdk_dd_bdev_to_bdev" "$testdir/bdev_to_bdev.sh" "${nvmes[@]}"
diff --git a/src/spdk/test/dd/posix.sh b/src/spdk/test/dd/posix.sh
new file mode 100755
index 000000000..15346d8d3
--- /dev/null
+++ b/src/spdk/test/dd/posix.sh
@@ -0,0 +1,122 @@
+#!/usr/bin/env bash
+testdir=$(readlink -f "$(dirname "$0")")
+rootdir=$(readlink -f "$testdir/../../")
+source "$testdir/common.sh"
+
+cleanup() {
+	rm -f "$test_file0"{,.link}
+	rm -f "$test_file1"{,.link}
+}
+
+append() {
+	local dump0
+	local dump1
+
+	dump0=$(gen_bytes 32)
+	dump1=$(gen_bytes 32)
+
+	printf '%s' "$dump0" > "$test_file0"
+	printf '%s' "$dump1" > "$test_file1"
+
+	"${DD_APP[@]}" --if="$test_file0" --of="$test_file1" --oflag=append
+
+	[[ $(< "$test_file1") == "${dump1}${dump0}" ]]
+}
+
+directory() {
+	NOT "${DD_APP[@]}" --if="$test_file0" --iflag=directory --of="$test_file0"
+	NOT "${DD_APP[@]}" --if="$test_file0" --of="$test_file0" --oflag=directory
+}
+
+nofollow() {
+	local test_file0_link=$test_file0.link
+	local test_file1_link=$test_file1.link
+
+	ln -fs "$test_file0" "$test_file0_link"
+	ln -fs "$test_file1" "$test_file1_link"
+
+	NOT "${DD_APP[@]}" --if="$test_file0_link" --iflag=nofollow --of="$test_file1"
+	NOT "${DD_APP[@]}" --if="$test_file0" --of="$test_file1_link" --oflag=nofollow
+
+	# Do an extra step of checking if we actually can follow symlinks
+	gen_bytes 512 > "$test_file0"
+
+	"${DD_APP[@]}" --if="$test_file0_link" --of="$test_file1"
+	[[ $(< "$test_file0") == "$(< "$test_file1")" ]]
+}
+
+noatime() {
+	local atime_if
+	local atime_of
+
+	# It seems like spdk_dd doesn't update the atime in case 0 bytes are copied.
+	# This differs from how standard dd works for instance
+	gen_bytes 512 > "$test_file0"
+
+	atime_if=$(stat --printf="%X" "$test_file0")
+	atime_of=$(stat --printf="%X" "$test_file1")
+
+	"${DD_APP[@]}" --if="$test_file0" --iflag=noatime --of="$test_file1"
+	((atime_if == $(stat --printf="%X" "$test_file0")))
+	((atime_of == $(stat --printf="%X" "$test_file1")))
+
+	"${DD_APP[@]}" --if="$test_file0" --of="$test_file1"
+	((atime_if < $(stat --printf="%X" "$test_file0")))
+}
+
+io() {
+	local flags_ro flags_rw flag_ro flag_rw
+
+	# O_NONBLOCK is actually a no-op, from a functional perspective, while
+	# open()ing a regular file, but let's keep it just to test its usage.
+	flags_ro=(direct nonblock)
+	flags_rw=("${flags_ro[@]}" sync dsync)
+
+	# simply check if data was correctly copied between files
+	for flag_ro in "${flags_ro[@]}"; do
+		gen_bytes 512 > "$test_file0"
+		for flag_rw in "${flags_rw[@]}"; do
+			"${DD_APP[@]}" \
+				--if="$test_file0" \
+				--iflag="$flag_ro" \
+				--of="$test_file1" \
+				--oflag="$flag_rw"
+			[[ $(< "$test_file0") == "$(< "$test_file1")" ]]
+		done
+	done
+}
+
+tests() {
+	printf '* First test run%s\n' \
+		"${msg[liburing_in_use]}" >&2
+
+	run_test "dd_flag_append" append
+	run_test "dd_flag_directory" directory
+	run_test "dd_flag_nofollow" nofollow
+	run_test "dd_flag_noatime" noatime
+	run_test "dd_flags_misc" io
+}
+
+tests_forced_aio() {
+	printf '* Second test run%s\n' \
+		"${msg[liburing_in_use ? 2 : 0]}" >&2
+
+	DD_APP+=("--aio")
+	run_test "dd_flag_append_forced_aio" append
+	run_test "dd_flag_directory_forced_aio" directory
+	run_test "dd_flag_nofollow_forced_aio" nofollow
+	run_test "dd_flag_noatime_forced_aio" noatime
+	run_test "dd_flags_misc_forced_aio" io
+}
+
+msg[0]=", using AIO"
+msg[1]=", liburing in use"
+msg[2]=", disabling liburing, forcing AIO"
+
+trap "cleanup" EXIT
+
+test_file0=$SPDK_TEST_STORAGE/dd.dump0
+test_file1=$SPDK_TEST_STORAGE/dd.dump1
+
+tests
+tests_forced_aio