Adding upstream version 14.2.21.upstream/14.2.21upstream

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

3 files changed, 531 insertions, 0 deletions

diff --git a/src/spdk/test/json_config/clear_config.py b/src/spdk/test/json_config/clear_config.py
new file mode 100755
index 00000000..e6d8dd71
--- /dev/null
+++ b/src/spdk/test/json_config/clear_config.py
@@ -0,0 +1,212 @@
+#!/usr/bin/env python3
+
+import os
+import sys
+import argparse
+sys.path.append(os.path.join(os.path.dirname(__file__), "../../scripts"))
+import rpc   # noqa
+from rpc.client import print_dict, JSONRPCException  # noqa
+
+
+def get_bdev_name_key(bdev):
+    bdev_name_key = 'name'
+    if 'method' in bdev and bdev['method'] == 'construct_split_vbdev':
+        bdev_name_key = "base_bdev"
+    return bdev_name_key
+
+
+def get_bdev_name(bdev):
+    bdev_name = None
+    if 'params' in bdev:
+        if 'name' in bdev['params']:
+            bdev_name = bdev['params']['name']
+        elif 'base_name' in bdev['params']:
+            bdev_name = bdev['params']['base_name']
+        elif 'base_bdev' in bdev['params']:
+            bdev_name = bdev['params']['base_bdev']
+    if 'method' in bdev and bdev['method'] == 'construct_error_bdev':
+        bdev_name = "EE_%s" % bdev_name
+    return bdev_name
+
+
+def delete_subbdevs(args, bdev, rpc_bdevs):
+    ret_value = False
+    bdev_name = get_bdev_name(bdev)
+    if bdev_name and 'method' in bdev:
+        construct_method = bdev['method']
+        if construct_method == 'construct_nvme_bdev':
+            for rpc_bdev in rpc_bdevs:
+                if bdev_name in rpc_bdev['name'] and rpc_bdev['product_name'] == "NVMe disk":
+                    args.client.call('delete_nvme_controller', {'name': "%s" % rpc_bdev['name'].split('n')[0]})
+                    ret_value = True
+
+    return ret_value
+
+
+def get_bdev_destroy_method(bdev):
+    destroy_method_map = {'construct_nvme_bdev': "delete_nvme_controller",
+                          'construct_malloc_bdev': "delete_malloc_bdev",
+                          'construct_null_bdev': "delete_null_bdev",
+                          'construct_rbd_bdev': "delete_rbd_bdev",
+                          'construct_pmem_bdev': "delete_pmem_bdev",
+                          'construct_aio_bdev': "delete_aio_bdev",
+                          'construct_error_bdev': "delete_error_bdev",
+                          'construct_split_vbdev': "destruct_split_vbdev",
+                          'construct_virtio_dev': "remove_virtio_bdev",
+                          'construct_crypto_bdev': "delete_crypto_bdev"
+                          }
+    destroy_method = None
+    if 'method' in bdev:
+        construct_method = bdev['method']
+        if construct_method in list(destroy_method_map.keys()):
+            destroy_method = destroy_method_map[construct_method]
+
+    return destroy_method
+
+
+def clear_bdev_subsystem(args, bdev_config):
+    rpc_bdevs = args.client.call("get_bdevs")
+    for bdev in bdev_config:
+        if delete_subbdevs(args, bdev, rpc_bdevs):
+            continue
+        bdev_name_key = get_bdev_name_key(bdev)
+        bdev_name = get_bdev_name(bdev)
+        destroy_method = get_bdev_destroy_method(bdev)
+        if destroy_method:
+            args.client.call(destroy_method, {bdev_name_key: bdev_name})
+
+    ''' Disable and reset hotplug '''
+    rpc.bdev.set_bdev_nvme_hotplug(args.client, False)
+
+
+def get_nvmf_destroy_method(nvmf):
+    destroy_method_map = {'nvmf_subsystem_create': "delete_nvmf_subsystem"}
+    try:
+        return destroy_method_map[nvmf['method']]
+    except KeyError:
+        return None
+
+
+def clear_nvmf_subsystem(args, nvmf_config):
+    for nvmf in nvmf_config:
+        destroy_method = get_nvmf_destroy_method(nvmf)
+        if destroy_method:
+            args.client.call(destroy_method, {'nqn': nvmf['params']['nqn']})
+
+
+def get_iscsi_destroy_method(iscsi):
+    destroy_method_map = {'add_portal_group': "delete_portal_group",
+                          'add_initiator_group': "delete_initiator_group",
+                          'construct_target_node': "delete_target_node",
+                          'set_iscsi_options': None
+                          }
+    return destroy_method_map[iscsi['method']]
+
+
+def get_iscsi_name(iscsi):
+    if 'name' in iscsi['params']:
+        return iscsi['params']['name']
+    else:
+        return iscsi['params']['tag']
+
+
+def get_iscsi_name_key(iscsi):
+    if iscsi['method'] == 'construct_target_node':
+        return "name"
+    else:
+        return 'tag'
+
+
+def clear_iscsi_subsystem(args, iscsi_config):
+    for iscsi in iscsi_config:
+        destroy_method = get_iscsi_destroy_method(iscsi)
+        if destroy_method:
+            args.client.call(destroy_method, {get_iscsi_name_key(iscsi): get_iscsi_name(iscsi)})
+
+
+def get_nbd_destroy_method(nbd):
+    destroy_method_map = {'start_nbd_disk': "stop_nbd_disk"
+                          }
+    return destroy_method_map[nbd['method']]
+
+
+def clear_nbd_subsystem(args, nbd_config):
+    for nbd in nbd_config:
+        destroy_method = get_nbd_destroy_method(nbd)
+        if destroy_method:
+            args.client.call(destroy_method, {'nbd_device': nbd['params']['nbd_device']})
+
+
+def clear_net_framework_subsystem(args, net_framework_config):
+    pass
+
+
+def clear_copy_subsystem(args, copy_config):
+    pass
+
+
+def clear_interface_subsystem(args, interface_config):
+    pass
+
+
+def clear_vhost_subsystem(args, vhost_config):
+    for vhost in reversed(vhost_config):
+        if 'method' in vhost:
+            method = vhost['method']
+            if method in ['add_vhost_scsi_lun']:
+                args.client.call("remove_vhost_scsi_target",
+                                 {"ctrlr": vhost['params']['ctrlr'],
+                                  "scsi_target_num": vhost['params']['scsi_target_num']})
+            elif method in ['construct_vhost_scsi_controller', 'construct_vhost_blk_controller',
+                            'construct_vhost_nvme_controller']:
+                args.client.call("remove_vhost_controller", {'ctrlr': vhost['params']['ctrlr']})
+
+
+def call_test_cmd(func):
+    def rpc_test_cmd(*args, **kwargs):
+        try:
+            func(*args, **kwargs)
+        except JSONRPCException as ex:
+            print((ex.message))
+            exit(1)
+    return rpc_test_cmd
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description='Clear config command')
+    parser.add_argument('-s', dest='server_addr', default='/var/tmp/spdk.sock')
+    parser.add_argument('-p', dest='port', default=5260, type=int)
+    parser.add_argument('-t', dest='timeout', default=60.0, type=float)
+    parser.add_argument('-v', dest='verbose', action='store_true')
+    subparsers = parser.add_subparsers(help='RPC methods')
+
+    @call_test_cmd
+    def clear_config(args):
+        for subsystem_item in reversed(args.client.call('get_subsystems')):
+            args.subsystem = subsystem_item['subsystem']
+            clear_subsystem(args)
+
+    p = subparsers.add_parser('clear_config', help="""Clear configuration of all SPDK subsystems and targets using JSON RPC""")
+    p.set_defaults(func=clear_config)
+
+    @call_test_cmd
+    def clear_subsystem(args):
+        config = args.client.call('get_subsystem_config', {"name": args.subsystem})
+        if config is None:
+            return
+        if args.verbose:
+            print("Calling clear_%s_subsystem" % args.subsystem)
+        globals()["clear_%s_subsystem" % args.subsystem](args, config)
+
+    p = subparsers.add_parser('clear_subsystem', help="""Clear configuration of SPDK subsystem using JSON RPC""")
+    p.add_argument('--subsystem', help="""Subsystem name""")
+    p.set_defaults(func=clear_subsystem)
+
+    args = parser.parse_args()
+
+    try:
+        args.client = rpc.client.JSONRPCClient(args.server_addr, args.port, args.verbose, args.timeout)
+    except JSONRPCException as ex:
+        print((ex.message))
+        exit(1)
+    args.func(args)
diff --git a/src/spdk/test/json_config/common.sh b/src/spdk/test/json_config/common.sh
new file mode 100644
index 00000000..a0a56951
--- /dev/null
+++ b/src/spdk/test/json_config/common.sh
@@ -0,0 +1,249 @@
+JSON_DIR=$(readlink -f $(dirname ${BASH_SOURCE[0]}))
+SPDK_BUILD_DIR=$JSON_DIR/../../
+source $JSON_DIR/../common/autotest_common.sh
+source $JSON_DIR/../nvmf/common.sh
+
+spdk_rpc_py="$SPDK_BUILD_DIR/scripts/rpc.py -s /var/tmp/spdk.sock"
+spdk_clear_config_py="$JSON_DIR/clear_config.py -s /var/tmp/spdk.sock"
+initiator_rpc_py="$SPDK_BUILD_DIR/scripts/rpc.py -s /var/tmp/virtio.sock"
+initiator_clear_config_py="$JSON_DIR/clear_config.py -s /var/tmp/virtio.sock"
+base_json_config=$JSON_DIR/base_config.json
+last_json_config=$JSON_DIR/last_config.json
+full_config=$JSON_DIR/full_config.json
+base_bdevs=$JSON_DIR/bdevs_base.txt
+last_bdevs=$JSON_DIR/bdevs_last.txt
+null_json_config=$JSON_DIR/null_json_config.json
+
+function run_spdk_tgt() {
+	echo "Running spdk target"
+	$SPDK_BUILD_DIR/app/spdk_tgt/spdk_tgt -m 0x1 -p 0 -s 4096 --wait-for-rpc &
+	spdk_tgt_pid=$!
+
+	echo "Waiting for app to run..."
+	waitforlisten $spdk_tgt_pid
+	echo "spdk_tgt started - pid=$spdk_tgt_pid but waits for subsystem initialization"
+
+	echo ""
+}
+
+function load_nvme() {
+	echo '{"subsystems": [' > nvme_config.json
+	$SPDK_BUILD_DIR/scripts/gen_nvme.sh --json >> nvme_config.json
+	echo ']}' >> nvme_config.json
+	$rpc_py load_config < nvme_config.json
+	rm nvme_config.json
+}
+
+function run_initiator() {
+	$SPDK_BUILD_DIR/app/spdk_tgt/spdk_tgt -m 0x2 -p 0 -g -u -s 1024 -r /var/tmp/virtio.sock --wait-for-rpc &
+	virtio_pid=$!
+	waitforlisten $virtio_pid /var/tmp/virtio.sock
+}
+
+function upload_vhost() {
+	$rpc_py construct_split_vbdev Nvme0n1 8
+	$rpc_py construct_vhost_scsi_controller sample1
+	$rpc_py add_vhost_scsi_lun sample1 0 Nvme0n1p3
+	$rpc_py add_vhost_scsi_lun sample1 1 Nvme0n1p4
+	$rpc_py set_vhost_controller_coalescing sample1 1 100
+	$rpc_py construct_vhost_blk_controller sample2 Nvme0n1p5
+	$rpc_py construct_vhost_nvme_controller sample3 16
+	$rpc_py add_vhost_nvme_ns sample3 Nvme0n1p6
+}
+
+function kill_targets() {
+	if [ ! -z $virtio_pid ]; then
+		killprocess $virtio_pid
+	fi
+	if [ ! -z $spdk_tgt_pid ]; then
+		killprocess $spdk_tgt_pid
+	fi
+}
+
+# Compare two JSON files.
+#
+# NOTE: Order of objects in JSON can change by just doing loads -> dumps so all JSON objects (not arrays) are sorted by
+# config_filter.py script. Sorted output is used to compare JSON output.
+#
+function json_diff()
+{
+	local tmp_file_1=$(mktemp ${1}.XXX)
+	local tmp_file_2=$(mktemp ${2}.XXX)
+	local ret=0
+
+	cat $1 | $JSON_DIR/config_filter.py -method "sort" > $tmp_file_1
+	cat $2 | $JSON_DIR/config_filter.py -method "sort" > $tmp_file_2
+
+	if ! diff -u $tmp_file_1 $tmp_file_2; then
+		ret=1
+	fi
+
+	rm $tmp_file_1 $tmp_file_2
+	return $ret
+}
+
+# This function test if json config was properly saved and loaded.
+# 1. Get a list of bdevs and save it to the file "base_bdevs".
+# 2. Save only configuration of the running spdk_tgt to the file "base_json_config"
+#    (global parameters are not saved).
+# 3. Clear configuration of the running spdk_tgt.
+# 4. Save only configuration of the running spdk_tgt to the file "null_json_config"
+#    (global parameters are not saved).
+# 5. Check if configuration of the running spdk_tgt is cleared by checking
+#    if the file "null_json_config" doesn't have any configuration.
+# 6. Load the file "base_json_config" to the running spdk_tgt.
+# 7. Get a list of bdevs and save it to the file "last_bdevs".
+# 8. Save only configuration of the running spdk_tgt to the file "last_json_config".
+# 9. Check if the file "base_json_config" matches the file "last_json_config".
+# 10. Check if the file "base_bdevs" matches the file "last_bdevs".
+# 11. Remove all files.
+function test_json_config() {
+	$rpc_py get_bdevs | jq '.|sort_by(.name)' > $base_bdevs
+	$rpc_py save_config > $full_config
+	$JSON_DIR/config_filter.py -method "delete_global_parameters" < $full_config > $base_json_config
+	$clear_config_py clear_config
+	$rpc_py save_config | $JSON_DIR/config_filter.py -method "delete_global_parameters" > $null_json_config
+	if [ "[]" != "$(jq '.subsystems | map(select(.config != null)) | map(select(.config != []))' $null_json_config)" ]; then
+		echo "Config has not been cleared"
+		return 1
+	fi
+	$rpc_py load_config < $base_json_config
+	$rpc_py get_bdevs | jq '.|sort_by(.name)' > $last_bdevs
+	$rpc_py save_config | $JSON_DIR/config_filter.py -method "delete_global_parameters" > $last_json_config
+
+	json_diff $base_json_config $last_json_config
+	json_diff $base_bdevs $last_bdevs
+	remove_config_files_after_test_json_config
+}
+
+function remove_config_files_after_test_json_config() {
+	rm -f $last_bdevs $base_bdevs
+	rm -f $last_json_config $base_json_config
+	rm -f $full_config $null_json_config
+}
+
+function create_pmem_bdev_subsytem_config() {
+        $rpc_py create_pmem_pool /tmp/pool_file1 128 512
+        $rpc_py construct_pmem_bdev -n pmem1 /tmp/pool_file1
+}
+
+function clear_pmem_bdev_subsystem_config() {
+	$clear_config_py clear_config
+	$rpc_py  delete_pmem_pool /tmp/pool_file1
+}
+
+function create_rbd_bdev_subsystem_config() {
+	rbd_setup 127.0.0.1
+	$rpc_py construct_rbd_bdev $RBD_POOL $RBD_NAME 4096
+}
+
+function clear_rbd_bdev_subsystem_config() {
+	$clear_config_py clear_config
+	rbd_cleanup
+}
+
+function create_bdev_subsystem_config() {
+	$rpc_py construct_split_vbdev Nvme0n1 2
+	$rpc_py construct_null_bdev Null0 32 512
+	$rpc_py construct_malloc_bdev 128 512 --name Malloc0
+	$rpc_py construct_malloc_bdev 64 4096 --name Malloc1
+	$rpc_py construct_malloc_bdev 8 1024 --name Malloc2
+	if [ $SPDK_TEST_CRYPTO -eq 1 ]; then
+		$rpc_py construct_malloc_bdev 8 1024 --name Malloc3
+		if [ $(lspci -d:37c8 | wc -l) -eq 0 ]; then
+			$rpc_py construct_crypto_bdev -b Malloc3 -c CryMalloc3 -d crypto_aesni_mb -k 0123456789123456
+		else
+			$rpc_py construct_crypto_bdev -b Malloc3 -c CryMalloc3 -d crypto_qat -k 0123456789123456
+		fi
+	fi
+	$rpc_py construct_error_bdev Malloc2
+	if [ $(uname -s) = Linux ]; then
+		dd if=/dev/zero of=/tmp/sample_aio bs=2048 count=5000
+		$rpc_py construct_aio_bdev /tmp/sample_aio aio_disk 1024
+	fi
+	$rpc_py construct_lvol_store -c 1048576 Nvme0n1p0 lvs_test
+	$rpc_py construct_lvol_bdev -l lvs_test lvol0 32
+	$rpc_py construct_lvol_bdev -l lvs_test -t lvol1 32
+	$rpc_py snapshot_lvol_bdev lvs_test/lvol0 snapshot0
+	$rpc_py clone_lvol_bdev lvs_test/snapshot0 clone0
+}
+
+function create_nvmf_subsystem_config() {
+	rdma_device_init
+	RDMA_IP_LIST=$(get_available_rdma_ips)
+	NVMF_FIRST_TARGET_IP=$(echo "$RDMA_IP_LIST" | head -n 1)
+	if [ -z $NVMF_FIRST_TARGET_IP ]; then
+		echo "Error: no NIC for nvmf test"
+		return 1
+	fi
+
+	bdevs="$($rpc_py construct_malloc_bdev 64 512) "
+	bdevs+="$($rpc_py construct_malloc_bdev 64 512)"
+	$rpc_py nvmf_subsystem_create nqn.2016-06.io.spdk:cnode1 -a -s SPDK00000000000001
+	for bdev in $bdevs; do
+		$rpc_py nvmf_subsystem_add_ns nqn.2016-06.io.spdk:cnode1 $bdev
+	done
+	$rpc_py nvmf_subsystem_add_listener nqn.2016-06.io.spdk:cnode1 -t RDMA -a $NVMF_FIRST_TARGET_IP -s "$NVMF_PORT"
+}
+
+function clear_nvmf_subsystem_config() {
+	$clear_config_py clear_config
+}
+
+function clear_bdev_subsystem_config() {
+	$rpc_py destroy_lvol_bdev lvs_test/clone0
+	$rpc_py destroy_lvol_bdev lvs_test/lvol0
+	$rpc_py destroy_lvol_bdev lvs_test/snapshot0
+	$rpc_py destroy_lvol_store -l lvs_test
+	$clear_config_py clear_config
+	if [ $(uname -s) = Linux ]; then
+		rm -f /tmp/sample_aio
+	fi
+}
+
+# In this test, target is spdk_tgt or virtio_initiator.
+# 1. Save current spdk config to full_config
+#    and save only global parameters to the file "base_json_config".
+# 2. Exit the running spdk target.
+# 3. Start the spdk target and wait for loading config.
+# 4. Load global parameters and configuration to the spdk target from the file full_config.
+# 5. Save json config to the file "full_config".
+# 6. Save only global parameters to the file "last_json_config".
+# 7. Check if the file "base_json_config" matches the file "last_json_config".
+# 8. Delete all files.
+function test_global_params() {
+	target=$1
+	$rpc_py save_config > $full_config
+	$JSON_DIR/config_filter.py -method "delete_configs" < $full_config > $base_json_config
+	if [ $target == "spdk_tgt" ]; then
+		killprocess $spdk_tgt_pid
+		run_spdk_tgt
+	elif [ $target == "virtio_initiator" ]; then
+		killprocess $virtio_pid
+                run_initiator
+	else
+		echo "Target is not specified for test_global_params"
+		return 1
+	fi
+	$rpc_py load_config < $full_config
+	$rpc_py save_config > $full_config
+	$JSON_DIR/config_filter.py -method "delete_configs" < $full_config > $last_json_config
+
+	json_diff $base_json_config $last_json_config
+	rm $base_json_config $last_json_config
+	rm $full_config
+}
+
+function on_error_exit() {
+	set +e
+	echo "Error on $1 - $2"
+	remove_config_files_after_test_json_config
+	rpc_py="$spdk_rpc_py"
+	clear_config_py="$spdk_clear_config_py"
+	clear_bdev_subsystem_config
+
+	kill_targets
+
+	print_backtrace
+	exit 1
+}
diff --git a/src/spdk/test/json_config/config_filter.py b/src/spdk/test/json_config/config_filter.py
new file mode 100755
index 00000000..59e96f94
--- /dev/null
+++ b/src/spdk/test/json_config/config_filter.py
@@ -0,0 +1,70 @@
+#!/usr/bin/env python3
+
+import sys
+import json
+import argparse
+from collections import OrderedDict
+
+
+def sort_json_object(o):
+    if isinstance(o, dict):
+        sorted_o = OrderedDict()
+        """ Order of keys in JSON object is irrelevant but we need to pick one
+        to be able to compare JSONS. """
+        for key in sorted(o.keys()):
+            sorted_o[key] = sort_json_object(o[key])
+        return sorted_o
+    if isinstance(o, list):
+        """ Keep list in the same orded but sort each item """
+        return [sort_json_object(item) for item in o]
+    else:
+        return o
+
+
+def filter_methods(do_remove_global_rpcs):
+    global_rpcs = [
+        'set_iscsi_options',
+        'set_nvmf_target_config',
+        'set_nvmf_target_options',
+        'nvmf_create_transport',
+        'set_bdev_options',
+        'set_bdev_nvme_options',
+        'set_bdev_nvme_hotplug',
+    ]
+
+    data = json.loads(sys.stdin.read())
+    out = {'subsystems': []}
+    for s in data['subsystems']:
+        if s['config']:
+            s_config = []
+            for config in s['config']:
+                m_name = config['method']
+                is_global_rpc = m_name in global_rpcs
+                if do_remove_global_rpcs != is_global_rpc:
+                    s_config.append(config)
+        else:
+            s_config = None
+        out['subsystems'].append({
+            'subsystem': s['subsystem'],
+            'config': s_config,
+        })
+
+    print(json.dumps(out, indent=2))
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument('-method', dest='method')
+
+    args = parser.parse_args()
+    if args.method == "delete_global_parameters":
+        filter_methods(True)
+    elif args.method == "delete_configs":
+        filter_methods(False)
+    elif args.method == "sort":
+        """ Wrap input into JSON object so any input is possible here
+        like output from get_bdevs RPC method"""
+        o = json.loads('{ "the_object": ' + sys.stdin.read() + ' }')
+        print(json.dumps(sort_json_object(o)['the_object'], indent=2))
+    else:
+        raise ValueError("Invalid method '{}'".format(args.method))