Adding upstream version 16.2.11+ds.upstream/16.2.11+dsupstream

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

11 files changed, 598 insertions, 0 deletions

diff --git a/src/spdk/test/ocf/common.sh b/src/spdk/test/ocf/common.sh
new file mode 100644
index 000000000..6c196ab97
--- /dev/null
+++ b/src/spdk/test/ocf/common.sh
@@ -0,0 +1,27 @@
+source $rootdir/scripts/common.sh
+source $rootdir/test/common/autotest_common.sh
+
+rpc_py=$rootdir/scripts/rpc.py
+
+function nvme_cfg() {
+	if [ -z "$ocf_nvme_cfg" ]; then
+		ocf_nvme_cfg=$($rootdir/scripts/gen_nvme.sh)
+	fi
+	echo "$ocf_nvme_cfg"
+}
+
+function clear_nvme() {
+	mapfile -t bdf < <(get_first_nvme_bdf)
+
+	# Clear metadata on NVMe device
+	$rootdir/scripts/setup.sh reset
+	sleep 5
+
+	name=$(get_nvme_name_from_bdf "${bdf[0]}")
+	mountpoints=$(lsblk /dev/$name --output MOUNTPOINT -n | wc -w)
+	if [ "$mountpoints" != "0" ]; then
+		exit 1
+	fi
+	dd if=/dev/zero of=/dev/$name bs=1M count=1000 oflag=direct
+	$rootdir/scripts/setup.sh
+}
diff --git a/src/spdk/test/ocf/integrity/bdevperf-iotypes.sh b/src/spdk/test/ocf/integrity/bdevperf-iotypes.sh
new file mode 100755
index 000000000..2effa21bd
--- /dev/null
+++ b/src/spdk/test/ocf/integrity/bdevperf-iotypes.sh
@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+
+curdir=$(dirname $(readlink -f "${BASH_SOURCE[0]}"))
+rootdir=$(readlink -f $curdir/../../..)
+source $rootdir/test/common/autotest_common.sh
+
+bdevperf=$rootdir/test/bdev/bdevperf/bdevperf
+rpc_py="$rootdir/scripts/rpc.py"
+
+source "$curdir/mallocs.conf"
+$bdevperf --json <(gen_malloc_ocf_json) -q 128 -o 4096 -t 4 -w flush
+$bdevperf --json <(gen_malloc_ocf_json) -q 128 -o 4096 -t 4 -w unmap
+$bdevperf --json <(gen_malloc_ocf_json) -q 128 -o 4096 -t 4 -w write
diff --git a/src/spdk/test/ocf/integrity/fio-modes.sh b/src/spdk/test/ocf/integrity/fio-modes.sh
new file mode 100755
index 000000000..0c90f999f
--- /dev/null
+++ b/src/spdk/test/ocf/integrity/fio-modes.sh
@@ -0,0 +1,90 @@
+#!/usr/bin/env bash
+
+curdir=$(dirname $(readlink -f "${BASH_SOURCE[0]}"))
+rootdir=$(readlink -f $curdir/../../..)
+
+source $rootdir/test/ocf/common.sh
+
+function fio_verify() {
+	fio_bdev $curdir/test.fio --aux-path=/tmp/ --ioengine=spdk_bdev "$@"
+}
+
+function cleanup() {
+	rm -f $curdir/modes.conf
+}
+
+# Clear nvme device which we will use in test
+clear_nvme
+
+trap "cleanup; exit 1" SIGINT SIGTERM EXIT
+
+# Building config is not backtrace worthy ...
+xtrace_disable
+
+config=() ocf_names=() ocf_modes=()
+
+ocf_names[1]=PT_Nvme ocf_modes[1]=pt
+ocf_names[2]=WT_Nvme ocf_modes[2]=wt
+ocf_names[3]=WB_Nvme0 ocf_modes[3]=wb
+ocf_names[4]=WB_Nvme1 ocf_modes[4]=wb
+
+mapfile -t config < <("$rootdir/scripts/gen_nvme.sh" --json)
+
+# Drop anything from last closing ] so we can inject our own config pieces ...
+config=("${config[@]::${#config[@]}-2}")
+# ... and now convert entire array to a single string item
+config=("${config[*]}")
+
+config+=(
+	"$(
+		cat <<- JSON
+			{
+			  "method": "bdev_split_create",
+			  "params": {
+			    "base_bdev": "Nvme0n1",
+			     "split_count": 8,
+			     "split_size_mb": 101
+			    }
+			}
+		JSON
+	)"
+)
+
+for ((d = 0, c = 1; d <= ${#ocf_names[@]} + 2; d += 2, c++)); do
+	config+=(
+		"$(
+			cat <<- JSON
+				{
+				  "method": "bdev_ocf_create",
+				    "params": {
+				      "name": "${ocf_names[c]}",
+				      "mode": "${ocf_modes[c]}",
+				      "cache_bdev_name": "Nvme0n1p$d",
+				      "core_bdev_name": "Nvme0n1p$((d + 1))"
+				    }
+				}
+			JSON
+		)"
+	)
+done
+
+# First ']}' closes our config and bdev subsystem blocks
+cat <<- CONFIG > "$curdir/modes.conf"
+	{"subsystems":[
+	$(
+	IFS=","
+	printf '%s\n' "${config[*]}"
+	)
+	]}]}
+CONFIG
+
+# Format the config nicely and dump it to stdout for everyone to marvel at it ...
+jq . "$curdir/modes.conf"
+
+# ... and now back to our regularly scheduled program
+xtrace_restore
+
+fio_verify --filename=PT_Nvme:WT_Nvme:WB_Nvme0:WB_Nvme1 --spdk_json_conf="$curdir/modes.conf" --thread=1
+
+trap - SIGINT SIGTERM EXIT
+cleanup
diff --git a/src/spdk/test/ocf/integrity/mallocs.conf b/src/spdk/test/ocf/integrity/mallocs.conf
new file mode 100644
index 000000000..245dd23cf
--- /dev/null
+++ b/src/spdk/test/ocf/integrity/mallocs.conf
@@ -0,0 +1,59 @@
+gen_malloc_ocf_json () {
+	local size=300 # MB
+	local block_size=512
+	local config
+
+	local malloc malloc_devs=3
+	for (( malloc = 0; malloc < malloc_devs; malloc++ )); do
+		config+=(
+			"$(
+				cat <<-JSON
+					{
+					  "method": "bdev_malloc_create",
+					  "params": {
+					    "name": "Malloc$malloc",
+					    "num_blocks": $(( (size << 20) / block_size )),
+					    "block_size": 512
+					  }
+					}
+				JSON
+			)"
+		)
+	done
+
+	local ocfs ocf ocf_mode ocf_cache ocf_core
+	ocfs=(1 2)
+	ocf_mode[1]=wt ocf_cache[1]=Malloc0 ocf_core[1]=Malloc1
+	ocf_mode[2]=pt ocf_cache[2]=Malloc0 ocf_core[2]=Malloc2
+
+	for ocf in "${ocfs[@]}"; do
+		config+=(
+			"$(
+				cat <<-JSON
+					{
+					  "method": "bdev_ocf_create",
+					  "params": {
+					    "name": "MalCache$ocf",
+					    "mode": "${ocf_mode[ocf]}",
+					    "cache_bdev_name": "${ocf_cache[ocf]}",
+					    "core_bdev_name": "${ocf_core[ocf]}"
+					  }
+					}
+				JSON
+			)"
+		)
+	done
+
+	jq . <<-JSON
+		{
+		  "subsystems": [
+		    {
+		      "subsystem": "bdev",
+		      "config": [
+		        $(IFS=","; printf '%s\n' "${config[*]}")
+		      ]
+		    }
+		  ]
+		}
+	JSON
+}
diff --git a/src/spdk/test/ocf/integrity/stats.sh b/src/spdk/test/ocf/integrity/stats.sh
new file mode 100755
index 000000000..c82f89d89
--- /dev/null
+++ b/src/spdk/test/ocf/integrity/stats.sh
@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+
+curdir=$(dirname $(readlink -f "${BASH_SOURCE[0]}"))
+rootdir=$(readlink -f $curdir/../../..)
+source $rootdir/test/common/autotest_common.sh
+
+bdevperf=$rootdir/test/bdev/bdevperf/bdevperf
+rpc_py="$rootdir/scripts/rpc.py"
+
+source "$curdir/mallocs.conf"
+$bdevperf --json <(gen_malloc_ocf_json) -q 128 -o 4096 -w write -t 120 -r /var/tmp/spdk.sock &
+bdev_perf_pid=$!
+waitforlisten $bdev_perf_pid
+sleep 1
+$rpc_py bdev_ocf_get_stats MalCache1
+kill -9 $bdev_perf_pid
+wait $bdev_perf_pid || true
diff --git a/src/spdk/test/ocf/integrity/test.fio b/src/spdk/test/ocf/integrity/test.fio
new file mode 100644
index 000000000..e56895c28
--- /dev/null
+++ b/src/spdk/test/ocf/integrity/test.fio
@@ -0,0 +1,39 @@
+[global]
+thread=1
+group_reporting=1
+direct=1
+serialize_overlap=1
+time_based=1
+do_verify=1
+verify=md5
+verify_backlog=1024
+iodepth=128
+bs=4K
+runtime=10
+
+size=20%
+
+[job_1]
+offset=0
+rw=randwrite
+name=randwrite
+
+[job_2]
+offset=20%
+rw=randrw
+name=randrw
+
+[job_3]
+offset=40%
+rw=write
+name=write
+
+[job_4]
+offset=60%
+rw=rw
+name=rw
+
+[job_5]
+offset=80%
+rw=randwrite
+name=randwrite
diff --git a/src/spdk/test/ocf/management/create-destruct.sh b/src/spdk/test/ocf/management/create-destruct.sh
new file mode 100755
index 000000000..162f7a679
--- /dev/null
+++ b/src/spdk/test/ocf/management/create-destruct.sh
@@ -0,0 +1,88 @@
+#!/usr/bin/env bash
+
+curdir=$(dirname $(readlink -f "${BASH_SOURCE[0]}"))
+rootdir=$(readlink -f $curdir/../../..)
+source $rootdir/test/common/autotest_common.sh
+
+rpc_py=$rootdir/scripts/rpc.py
+
+function bdev_check_claimed() {
+	if [ "$($rpc_py get_bdevs -b "$@" | jq '.[0].claimed')" = "true" ]; then
+		return 0
+	else
+		return 1
+	fi
+}
+
+$SPDK_BIN_DIR/iscsi_tgt &
+spdk_pid=$!
+
+trap 'killprocess $spdk_pid; exit 1' SIGINT SIGTERM EXIT
+
+waitforlisten $spdk_pid
+
+$rpc_py bdev_malloc_create 101 512 -b Malloc0
+$rpc_py bdev_malloc_create 101 512 -b Malloc1
+
+$rpc_py bdev_ocf_create PartCache wt Malloc0 NonExisting
+
+$rpc_py bdev_ocf_get_bdevs PartCache | jq -e \
+	'.[0] | .started == false and .cache.attached and .core.attached == false'
+
+$rpc_py bdev_ocf_get_bdevs NonExisting | jq -e \
+	'.[0] | .name == "PartCache"'
+
+if ! bdev_check_claimed Malloc0; then
+	echo >&2 "Base device expected to be claimed now"
+	exit 1
+fi
+
+$rpc_py bdev_ocf_delete PartCache
+if bdev_check_claimed Malloc0; then
+	echo >&2 "Base device is not expected to be claimed now"
+	exit 1
+fi
+
+$rpc_py bdev_ocf_create FullCache wt Malloc0 Malloc1
+
+$rpc_py bdev_ocf_get_bdevs FullCache | jq -e \
+	'.[0] | .started and .cache.attached and .core.attached'
+
+if ! (bdev_check_claimed Malloc0 && bdev_check_claimed Malloc1); then
+	echo >&2 "Base devices expected to be claimed now"
+	exit 1
+fi
+
+$rpc_py bdev_ocf_delete FullCache
+if bdev_check_claimed Malloc0 && bdev_check_claimed Malloc1; then
+	echo >&2 "Base devices are not expected to be claimed now"
+	exit 1
+fi
+
+$rpc_py bdev_ocf_create HotCache wt Malloc0 Malloc1
+
+if ! (bdev_check_claimed Malloc0 && bdev_check_claimed Malloc1); then
+	echo >&2 "Base devices expected to be claimed now"
+	exit 1
+fi
+
+$rpc_py bdev_malloc_delete Malloc0
+
+if bdev_check_claimed Malloc1; then
+	echo >&2 "Base device is not expected to be claimed now"
+	exit 1
+fi
+
+status=$($rpc_py get_bdevs)
+gone=$(echo $status | jq 'map(select(.name == "HotCache")) == []')
+if [[ $gone == false ]]; then
+	echo >&2 "OCF bdev is expected to unregister"
+	exit 1
+fi
+
+# check if shutdown of running CAS bdev is ok
+$rpc_py bdev_ocf_create PartCache wt NonExisting Malloc1
+
+trap - SIGINT SIGTERM EXIT
+
+killprocess $spdk_pid
diff --git a/src/spdk/test/ocf/management/multicore.sh b/src/spdk/test/ocf/management/multicore.sh
new file mode 100755
index 000000000..8c4f89e1b
--- /dev/null
+++ b/src/spdk/test/ocf/management/multicore.sh
@@ -0,0 +1,82 @@
+#!/usr/bin/env bash
+
+curdir=$(dirname $(readlink -f "${BASH_SOURCE[0]}"))
+rootdir=$(readlink -f $curdir/../../..)
+source $rootdir/test/common/autotest_common.sh
+
+rpc_py=$rootdir/scripts/rpc.py
+
+spdk_pid='?'
+function start_spdk() {
+	$SPDK_BIN_DIR/iscsi_tgt &
+	spdk_pid=$!
+	trap 'killprocess $spdk_pid; exit 1' SIGINT SIGTERM EXIT
+	waitforlisten $spdk_pid
+}
+function stop_spdk() {
+	killprocess $spdk_pid
+	trap - SIGINT SIGTERM EXIT
+}
+
+start_spdk
+
+# Hotplug case
+
+$rpc_py bdev_malloc_create 1 512 -b Core0
+$rpc_py bdev_malloc_create 1 512 -b Core1
+
+$rpc_py bdev_ocf_create C1 wt Cache Core0
+$rpc_py bdev_ocf_create C2 wt Cache Core1
+
+$rpc_py bdev_ocf_get_bdevs | jq -e \
+	'any(select(.started)) == false'
+
+$rpc_py bdev_malloc_create 101 512 -b Cache
+
+$rpc_py bdev_ocf_get_bdevs | jq -e \
+	'all(select(.started)) == true'
+
+#Be sure that we will not fail delete because examine is still in progress
+waitforbdev C2
+
+# Detaching cores
+
+$rpc_py bdev_ocf_delete C2
+
+$rpc_py bdev_ocf_get_bdevs C1 | jq -e \
+	'.[0] | .started'
+
+$rpc_py bdev_ocf_create C2 wt Cache Core1
+
+$rpc_py bdev_ocf_get_bdevs C2 | jq -e \
+	'.[0] | .started'
+
+# Normal shutdown
+
+stop_spdk
+
+# Hotremove case
+start_spdk
+
+$rpc_py bdev_malloc_create 101 512 -b Cache
+$rpc_py bdev_malloc_create 101 512 -b Malloc
+$rpc_py bdev_malloc_create 1 512 -b Core
+
+$rpc_py bdev_ocf_create C1 wt Cache Malloc
+$rpc_py bdev_ocf_create C2 wt Cache Core
+
+$rpc_py bdev_ocf_get_bdevs Cache | jq \
+	'length == 2'
+
+$rpc_py bdev_malloc_delete Cache
+
+$rpc_py bdev_ocf_get_bdevs | jq -e \
+	'. == []'
+
+# Not fully initialized shutdown
+
+$rpc_py bdev_ocf_create C1 wt Malloc NonExisting
+$rpc_py bdev_ocf_create C2 wt Malloc NonExisting
+$rpc_py bdev_ocf_create C3 wt Malloc Core
+
+stop_spdk
diff --git a/src/spdk/test/ocf/management/persistent-metadata.sh b/src/spdk/test/ocf/management/persistent-metadata.sh
new file mode 100755
index 000000000..cbfcab341
--- /dev/null
+++ b/src/spdk/test/ocf/management/persistent-metadata.sh
@@ -0,0 +1,88 @@
+#!/usr/bin/env bash
+
+curdir=$(dirname $(readlink -f "${BASH_SOURCE[0]}"))
+rootdir=$(readlink -f $curdir/../../..)
+source $rootdir/test/ocf/common.sh
+
+source $rootdir/scripts/common.sh
+source $rootdir/test/common/autotest_common.sh
+
+rpc_py=$rootdir/scripts/rpc.py
+
+$rootdir/scripts/setup.sh
+
+mapfile -t config < <("$rootdir/scripts/gen_nvme.sh" --json)
+# Drop anything from last closing ] so we can inject our own config pieces ...
+config=("${config[@]::${#config[@]}-2}")
+# ... and now convert entire array to a single string item
+config=("${config[*]}")
+
+config+=(
+	"$(
+		cat <<- JSON
+			{
+			  "method": "bdev_split_create",
+			  "params": {
+			    "base_bdev": "Nvme0n1",
+			    "split_count": 7,
+			    "split_size_mb": 128
+			  }
+			}
+		JSON
+	)"
+)
+
+# First ']}' closes our config and bdev subsystem blocks
+jq . <<- CONFIG > "$curdir/config"
+	{"subsystems":[
+	$(
+	IFS=","
+	printf '%s\n' "${config[*]}"
+	)
+	]}]}
+CONFIG
+
+# Clear nvme device which we will use in test
+clear_nvme
+
+"$SPDK_BIN_DIR/iscsi_tgt" --json "$curdir/config" &
+spdk_pid=$!
+
+waitforlisten $spdk_pid
+
+# Create ocf on persistent storage
+
+$rpc_py bdev_ocf_create ocfWT wt Nvme0n1p0 Nvme0n1p1
+$rpc_py bdev_ocf_create ocfPT pt Nvme0n1p2 Nvme0n1p3
+$rpc_py bdev_ocf_create ocfWB0 wb Nvme0n1p4 Nvme0n1p5
+$rpc_py bdev_ocf_create ocfWB1 wb Nvme0n1p4 Nvme0n1p6
+
+# Sorting bdevs because we dont guarantee that they are going to be
+# in the same order after shutdown
+($rpc_py bdev_ocf_get_bdevs | jq '(.. | arrays) |= sort') > ./ocf_bdevs
+
+trap - SIGINT SIGTERM EXIT
+
+killprocess $spdk_pid
+
+# Check for ocf persistency after restart
+"$SPDK_BIN_DIR/iscsi_tgt" --json "$curdir/config" &
+spdk_pid=$!
+
+trap 'killprocess $spdk_pid; rm -f $curdir/config ocf_bdevs ocf_bdevs_verify; exit 1' SIGINT SIGTERM EXIT
+
+waitforlisten $spdk_pid
+sleep 5
+
+# OCF should be loaded now as well
+
+($rpc_py bdev_ocf_get_bdevs | jq '(.. | arrays) |= sort') > ./ocf_bdevs_verify
+
+diff ocf_bdevs ocf_bdevs_verify
+
+trap - SIGINT SIGTERM EXIT
+
+killprocess $spdk_pid
+rm -f $curdir/config ocf_bdevs ocf_bdevs_verify
+
+clear_nvme $bdf
diff --git a/src/spdk/test/ocf/management/remove.sh b/src/spdk/test/ocf/management/remove.sh
new file mode 100755
index 000000000..1302f16cd
--- /dev/null
+++ b/src/spdk/test/ocf/management/remove.sh
@@ -0,0 +1,81 @@
+#!/usr/bin/env bash
+
+curdir=$(dirname $(readlink -f "${BASH_SOURCE[0]}"))
+rootdir=$(readlink -f $curdir/../../..)
+source $rootdir/test/common/autotest_common.sh
+
+rpc_py=$rootdir/scripts/rpc.py
+
+rm -f aio*
+truncate -s 128M aio0
+truncate -s 128M aio1
+
+jq . <<- JSON > "$curdir/config"
+	{
+	  "subsystems": [
+	    {
+	      "subsystem": "bdev",
+	      "config": [
+	        {
+	          "method": "bdev_aio_create",
+	          "params": {
+	            "name": "ai0",
+	            "block_size": 512,
+	            "filename": "./aio0"
+	          }
+	        },
+	        {
+	          "method": "bdev_aio_create",
+	          "params": {
+	            "name": "aio1",
+	            "block_size": 512,
+	            "filename": "./aio1"
+	          }
+	        }
+	      ]
+	    }
+	  ]
+	}
+JSON
+
+"$SPDK_BIN_DIR/iscsi_tgt" --json "$curdir/config" &
+spdk_pid=$!
+
+waitforlisten $spdk_pid
+
+# Create ocf on persistent storage
+
+$rpc_py bdev_ocf_create ocfWT wt aio0 aio1
+
+# Check that ocfWT was created properly
+
+$rpc_py bdev_ocf_get_bdevs | jq -r '.[] .name' | grep -qw ocfWT
+
+# Remove ocfWT, after delete via rpc ocf bdev should not load on next app start
+
+$rpc_py bdev_ocf_delete ocfWT
+
+# Check that ocfWT was deleted properly
+
+! $rpc_py bdev_ocf_get_bdevs | jq -r '.[] .name' | grep -qw ocfWT
+
+trap - SIGINT SIGTERM EXIT
+
+killprocess $spdk_pid
+
+# Check for ocfWT was deleted permanently
+"$SPDK_BIN_DIR/iscsi_tgt" --json "$curdir/config" &
+spdk_pid=$!
+
+trap 'killprocess $spdk_pid; rm -f aio* $curdir/config ocf_bdevs ocf_bdevs_verify; exit 1' SIGINT SIGTERM EXIT
+
+waitforlisten $spdk_pid
+
+# Check that ocfWT was not loaded on app start
+
+! $rpc_py bdev_ocf_get_bdevs | jq -r '.[] .name' | grep -qw ocfWT
+
+trap - SIGINT SIGTERM EXIT
+
+killprocess $spdk_pid
+rm -f aio* $curdir/config ocf_bdevs ocf_bdevs_verify
diff --git a/src/spdk/test/ocf/ocf.sh b/src/spdk/test/ocf/ocf.sh
new file mode 100755
index 000000000..415befc67
--- /dev/null
+++ b/src/spdk/test/ocf/ocf.sh
@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+
+testdir=$(readlink -f $(dirname $0))
+rootdir=$(readlink -f $testdir/../..)
+
+source $rootdir/test/common/autotest_common.sh
+
+run_test "ocf_fio_modes" "$testdir/integrity/fio-modes.sh"
+run_test "ocf_bdevperf_iotypes" "$testdir/integrity/bdevperf-iotypes.sh"
+run_test "ocf_stats" "$testdir/integrity/stats.sh"
+run_test "ocf_create_destruct" "$testdir/management/create-destruct.sh"
+run_test "ocf_multicore" "$testdir/management/multicore.sh"
+run_test "ocf_persistent_metadata" "$testdir/management/persistent-metadata.sh"
+run_test "ocf_remove" "$testdir/management/remove.sh"