diff options
Diffstat (limited to '')
34 files changed, 2362 insertions, 0 deletions
diff --git a/src/spdk/test/nvmf/README.md b/src/spdk/test/nvmf/README.md new file mode 100644 index 000000000..19d6954c2 --- /dev/null +++ b/src/spdk/test/nvmf/README.md @@ -0,0 +1,5 @@ +# NVMe-oF test scripts + +The test scripts in this directory hierarchy can be run in isolation by passing +the --iso flag when running the test script. This will set up the RDMA NIC for +testing and then tear it back down again when the test is completed. diff --git a/src/spdk/test/nvmf/common.sh b/src/spdk/test/nvmf/common.sh new file mode 100644 index 000000000..5f52ef127 --- /dev/null +++ b/src/spdk/test/nvmf/common.sh @@ -0,0 +1,292 @@ +NVMF_PORT=4420 +NVMF_IP_PREFIX="192.168.100" +NVMF_IP_LEAST_ADDR=8 +NVMF_TCP_IP_ADDRESS="127.0.0.1" +NVMF_TRANSPORT_OPTS="" +NVMF_SERIAL=SPDK00000000000001 + +function build_nvmf_app_args() { + if [ $SPDK_RUN_NON_ROOT -eq 1 ]; then + NVMF_APP=(sudo -u "$USER" "${NVMF_APP[@]}") + NVMF_APP+=(-i "$NVMF_APP_SHM_ID" -e 0xFFFF) + else + NVMF_APP+=(-i "$NVMF_APP_SHM_ID" -e 0xFFFF) + fi +} + +: ${NVMF_APP_SHM_ID="0"} +export NVMF_APP_SHM_ID +build_nvmf_app_args + +have_pci_nics=0 + +function rxe_cfg() { + "$rootdir/scripts/rxe_cfg_small.sh" "$@" +} + +function load_ib_rdma_modules() { + if [ $(uname) != Linux ]; then + return 0 + fi + + modprobe ib_cm + modprobe ib_core + # Newer kernels do not have the ib_ucm module + modprobe ib_ucm || true + modprobe ib_umad + modprobe ib_uverbs + modprobe iw_cm + modprobe rdma_cm + modprobe rdma_ucm +} + +function detect_soft_roce_nics() { + rxe_cfg start +} + +# args 1 and 2 represent the grep filters for finding our NICS. +# subsequent args are all drivers that should be loaded if we find these NICs. +# Those drivers should be supplied in the correct order. +function detect_nics_and_probe_drivers() { + NIC_VENDOR="$1" + NIC_CLASS="$2" + + nvmf_nic_bdfs=$(lspci | grep Ethernet | grep "$NIC_VENDOR" | grep "$NIC_CLASS" | awk -F ' ' '{print "0000:"$1}') + + if [ -z "$nvmf_nic_bdfs" ]; then + return 0 + fi + + have_pci_nics=1 + if [ $# -ge 2 ]; then + # shift out the first two positional arguments. + shift 2 + # Iterate through the remaining arguments. + for i; do + modprobe "$i" + done + fi +} + +function detect_pci_nics() { + + if ! hash lspci; then + return 0 + fi + + detect_nics_and_probe_drivers "Mellanox" "ConnectX-4" "mlx4_core" "mlx4_ib" "mlx4_en" + detect_nics_and_probe_drivers "Mellanox" "ConnectX-5" "mlx5_core" "mlx5_ib" + detect_nics_and_probe_drivers "Intel" "X722" "i40e" "i40iw" + detect_nics_and_probe_drivers "Chelsio" "Unified Wire" "cxgb4" "iw_cxgb4" + + if [ "$have_pci_nics" -eq "0" ]; then + return 0 + fi + + # Provide time for drivers to properly load. + sleep 5 +} + +function detect_rdma_nics() { + detect_pci_nics + if [ "$have_pci_nics" -eq "0" ]; then + detect_soft_roce_nics + fi +} + +function allocate_nic_ips() { + ((count = NVMF_IP_LEAST_ADDR)) + for nic_name in $(get_rdma_if_list); do + ip="$(get_ip_address $nic_name)" + if [ -z $ip ]; then + ip addr add $NVMF_IP_PREFIX.$count/24 dev $nic_name + ip link set $nic_name up + ((count = count + 1)) + fi + # dump configuration for debug log + ip addr show $nic_name + done +} + +function get_available_rdma_ips() { + for nic_name in $(get_rdma_if_list); do + get_ip_address $nic_name + done +} + +function get_rdma_if_list() { + for nic_type in /sys/class/infiniband/*; do + [[ -e "$nic_type" ]] || break + for nic_name in /sys/class/infiniband/"$(basename ${nic_type})"/device/net/*; do + [[ -e "$nic_name" ]] || break + basename "$nic_name" + done + done +} + +function get_ip_address() { + interface=$1 + ip -o -4 addr show $interface | awk '{print $4}' | cut -d"/" -f1 +} + +function nvmfcleanup() { + sync + set +e + for i in {1..20}; do + modprobe -v -r nvme-$TEST_TRANSPORT + if modprobe -v -r nvme-fabrics; then + set -e + return 0 + fi + sleep 1 + done + set -e + + # So far unable to remove the kernel modules. Try + # one more time and let it fail. + # Allow the transport module to fail for now. See Jim's comment + # about the nvme-tcp module below. + modprobe -v -r nvme-$TEST_TRANSPORT || true + modprobe -v -r nvme-fabrics +} + +function nvmftestinit() { + if [ -z $TEST_TRANSPORT ]; then + echo "transport not specified - use --transport= to specify" + return 1 + fi + if [ "$TEST_MODE" == "iso" ]; then + $rootdir/scripts/setup.sh + if [ "$TEST_TRANSPORT" == "rdma" ]; then + rdma_device_init + fi + fi + + NVMF_TRANSPORT_OPTS="-t $TEST_TRANSPORT" + if [ "$TEST_TRANSPORT" == "rdma" ]; then + RDMA_IP_LIST=$(get_available_rdma_ips) + NVMF_FIRST_TARGET_IP=$(echo "$RDMA_IP_LIST" | head -n 1) + NVMF_SECOND_TARGET_IP=$(echo "$RDMA_IP_LIST" | tail -n +2 | head -n 1) + if [ -z $NVMF_FIRST_TARGET_IP ]; then + echo "no NIC for nvmf test" + exit 0 + fi + elif [ "$TEST_TRANSPORT" == "tcp" ]; then + NVMF_FIRST_TARGET_IP=127.0.0.1 + NVMF_TRANSPORT_OPTS="$NVMF_TRANSPORT_OPTS -o" + fi + + # currently we run the host/perf test for TCP even on systems without kernel nvme-tcp + # support; that's fine since the host/perf test uses the SPDK initiator + # maybe later we will enforce modprobe to succeed once we have systems in the test pool + # with nvme-tcp kernel support - but until then let this pass so we can still run the + # host/perf test with the tcp transport + modprobe nvme-$TEST_TRANSPORT || true +} + +function nvmfappstart() { + timing_enter start_nvmf_tgt + "${NVMF_APP[@]}" "$@" & + nvmfpid=$! + trap 'process_shm --id $NVMF_APP_SHM_ID; nvmftestfini; exit 1' SIGINT SIGTERM EXIT + waitforlisten $nvmfpid + timing_exit start_nvmf_tgt +} + +function nvmftestfini() { + nvmfcleanup || : + if [ -n "$nvmfpid" ]; then + killprocess $nvmfpid + fi + if [ "$TEST_MODE" == "iso" ]; then + $rootdir/scripts/setup.sh reset + if [ "$TEST_TRANSPORT" == "rdma" ]; then + rdma_device_init + fi + fi +} + +function rdma_device_init() { + load_ib_rdma_modules + detect_rdma_nics + allocate_nic_ips +} + +function revert_soft_roce() { + rxe_cfg stop +} + +function check_ip_is_soft_roce() { + rxe_cfg status rxe | grep -wq "$1" +} + +function nvme_connect() { + local init_count + init_count=$(nvme list | wc -l) + + if ! nvme connect "$@"; then return $?; fi + + for i in $(seq 1 10); do + if [ $(nvme list | wc -l) -gt $init_count ]; then + return 0 + else + sleep 1s + fi + done + return 1 +} + +function get_nvme_devs() { + local dev rest + + nvmes=() + while read -r dev rest; do + if [[ $dev == /dev/nvme* ]]; then + nvmes+=("$dev") + fi + if [[ $1 == print ]]; then + echo "$dev $rest" + fi + done < <(nvme list) + ((${#nvmes[@]})) || return 1 + echo "${#nvmes[@]}" >&2 +} + +function gen_nvmf_target_json() { + local subsystem config=() + + for subsystem in "${@:-1}"; do + config+=( + "$( + cat <<- EOF + { + "params": { + "name": "Nvme$subsystem", + "trtype": "$TEST_TRANSPORT", + "traddr": "$NVMF_FIRST_TARGET_IP", + "adrfam": "ipv4", + "trsvcid": "$NVMF_PORT", + "subnqn": "nqn.2016-06.io.spdk:cnode$subsystem" + }, + "method": "bdev_nvme_attach_controller" + } + EOF + )" + ) + done + jq . <<- JSON + { + "subsystems": [ + { + "subsystem": "bdev", + "config": [ + $( + IFS="," + printf '%s\n' "${config[*]}" + ) + ] + } + ] + } + JSON +} diff --git a/src/spdk/test/nvmf/host/aer.sh b/src/spdk/test/nvmf/host/aer.sh new file mode 100755 index 000000000..1c438c686 --- /dev/null +++ b/src/spdk/test/nvmf/host/aer.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash + +testdir=$(readlink -f $(dirname $0)) +rootdir=$(readlink -f $testdir/../../..) +source $rootdir/test/common/autotest_common.sh +source $rootdir/test/nvmf/common.sh + +rpc_py="$rootdir/scripts/rpc.py" + +nvmftestinit +nvmfappstart -m 0xF + +$rpc_py nvmf_create_transport $NVMF_TRANSPORT_OPTS -u 8192 + +$rpc_py bdev_malloc_create 64 512 --name Malloc0 +$rpc_py nvmf_create_subsystem nqn.2016-06.io.spdk:cnode1 -a -s SPDK00000000000001 -m 2 +$rpc_py nvmf_subsystem_add_ns nqn.2016-06.io.spdk:cnode1 Malloc0 +$rpc_py nvmf_subsystem_add_listener nqn.2016-06.io.spdk:cnode1 -t $TEST_TRANSPORT -a $NVMF_FIRST_TARGET_IP -s $NVMF_PORT + +$rpc_py nvmf_get_subsystems + +AER_TOUCH_FILE=/tmp/aer_touch_file +rm -f $AER_TOUCH_FILE + +# Namespace Attribute Notice Tests +$rootdir/test/nvme/aer/aer -r "\ + trtype:$TEST_TRANSPORT \ + adrfam:IPv4 \ + traddr:$NVMF_FIRST_TARGET_IP \ + trsvcid:$NVMF_PORT \ + subnqn:nqn.2016-06.io.spdk:cnode1" -n 2 -t $AER_TOUCH_FILE & +aerpid=$! + +# Waiting for aer start to work +waitforfile $AER_TOUCH_FILE + +# Add a new namespace +$rpc_py bdev_malloc_create 64 4096 --name Malloc1 +$rpc_py nvmf_subsystem_add_ns nqn.2016-06.io.spdk:cnode1 Malloc1 -n 2 +$rpc_py nvmf_get_subsystems + +wait $aerpid + +$rpc_py bdev_malloc_delete Malloc0 +$rpc_py bdev_malloc_delete Malloc1 +$rpc_py nvmf_delete_subsystem nqn.2016-06.io.spdk:cnode1 + +trap - SIGINT SIGTERM EXIT + +nvmftestfini diff --git a/src/spdk/test/nvmf/host/bdevperf.sh b/src/spdk/test/nvmf/host/bdevperf.sh new file mode 100755 index 000000000..776550c4d --- /dev/null +++ b/src/spdk/test/nvmf/host/bdevperf.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash + +testdir=$(readlink -f $(dirname $0)) +rootdir=$(readlink -f $testdir/../../..) +source $rootdir/test/common/autotest_common.sh +source $rootdir/test/nvmf/common.sh + +MALLOC_BDEV_SIZE=64 +MALLOC_BLOCK_SIZE=512 + +rpc_py="$rootdir/scripts/rpc.py" + +function tgt_init() { + nvmfappstart -m 0xF + + $rpc_py nvmf_create_transport $NVMF_TRANSPORT_OPTS -u 8192 + $rpc_py bdev_malloc_create $MALLOC_BDEV_SIZE $MALLOC_BLOCK_SIZE -b Malloc0 + $rpc_py nvmf_create_subsystem nqn.2016-06.io.spdk:cnode1 -a -s SPDK00000000000001 + $rpc_py nvmf_subsystem_add_ns nqn.2016-06.io.spdk:cnode1 Malloc0 + $rpc_py nvmf_subsystem_add_listener nqn.2016-06.io.spdk:cnode1 -t $TEST_TRANSPORT -a $NVMF_FIRST_TARGET_IP -s $NVMF_PORT +} + +nvmftestinit +# There is an intermittent error relating to this test and Soft-RoCE. for now, just +# skip this test if we are using rxe. TODO: get to the bottom of GitHub issue #1165 +if [ $TEST_TRANSPORT == "rdma" ] && check_ip_is_soft_roce $NVMF_FIRST_TARGET_IP; then + echo "Using software RDMA, skipping the host bdevperf tests." + exit 0 +fi + +tgt_init + +"$rootdir/test/bdev/bdevperf/bdevperf" --json <(gen_nvmf_target_json) -q 128 -o 4096 -w verify -t 1 + +"$rootdir/test/bdev/bdevperf/bdevperf" --json <(gen_nvmf_target_json) -q 128 -o 4096 -w verify -t 15 -f & +bdevperfpid=$! + +sleep 3 +kill -9 $nvmfpid + +sleep 3 +tgt_init + +wait $bdevperfpid +sync +$rpc_py nvmf_delete_subsystem nqn.2016-06.io.spdk:cnode1 + +trap - SIGINT SIGTERM EXIT + +nvmftestfini diff --git a/src/spdk/test/nvmf/host/fio.sh b/src/spdk/test/nvmf/host/fio.sh new file mode 100755 index 000000000..85f9a00f1 --- /dev/null +++ b/src/spdk/test/nvmf/host/fio.sh @@ -0,0 +1,82 @@ +#!/usr/bin/env bash + +testdir=$(readlink -f $(dirname $0)) +rootdir=$(readlink -f $testdir/../../..) +source $rootdir/test/common/autotest_common.sh +source $rootdir/scripts/common.sh +source $rootdir/test/nvmf/common.sh + +rpc_py="$rootdir/scripts/rpc.py" + +nvmftestinit + +if [[ $CONFIG_FIO_PLUGIN != y ]]; then + echo "FIO not available" + exit 1 +fi + +timing_enter start_nvmf_tgt + +"${NVMF_APP[@]}" -m 0xF & +nvmfpid=$! + +trap 'process_shm --id $NVMF_APP_SHM_ID; nvmftestfini; exit 1' SIGINT SIGTERM EXIT + +waitforlisten $nvmfpid +$rpc_py nvmf_create_transport $NVMF_TRANSPORT_OPTS -u 8192 +timing_exit start_nvmf_tgt + +$rpc_py bdev_malloc_create 64 512 -b Malloc1 +$rpc_py nvmf_create_subsystem nqn.2016-06.io.spdk:cnode1 -a -s SPDK00000000000001 +$rpc_py nvmf_subsystem_add_ns nqn.2016-06.io.spdk:cnode1 Malloc1 +$rpc_py nvmf_subsystem_add_listener nqn.2016-06.io.spdk:cnode1 -t $TEST_TRANSPORT -a $NVMF_FIRST_TARGET_IP -s $NVMF_PORT + +PLUGIN_DIR=$rootdir/examples/nvme/fio_plugin + +# Test fio_plugin as host with malloc backend +fio_nvme $PLUGIN_DIR/example_config.fio --filename="trtype=$TEST_TRANSPORT adrfam=IPv4 \ +traddr=$NVMF_FIRST_TARGET_IP trsvcid=$NVMF_PORT ns=1" + +# second test mocking multiple SGL elements +fio_nvme $PLUGIN_DIR/mock_sgl_config.fio --filename="trtype=$TEST_TRANSPORT adrfam=IPv4 \ +traddr=$NVMF_FIRST_TARGET_IP trsvcid=$NVMF_PORT ns=1" +$rpc_py nvmf_delete_subsystem nqn.2016-06.io.spdk:cnode1 + +if [ $RUN_NIGHTLY -eq 1 ]; then + # Test fio_plugin as host with nvme lvol backend + bdfs=$(get_nvme_bdfs) + $rpc_py bdev_nvme_attach_controller -b Nvme0 -t PCIe -a $(echo $bdfs | awk '{ print $1 }') -i $NVMF_FIRST_TARGET_IP + ls_guid=$($rpc_py bdev_lvol_create_lvstore -c 1073741824 Nvme0n1 lvs_0) + get_lvs_free_mb $ls_guid + $rpc_py bdev_lvol_create -l lvs_0 lbd_0 $free_mb + $rpc_py nvmf_create_subsystem nqn.2016-06.io.spdk:cnode2 -a -s SPDK00000000000001 + $rpc_py nvmf_subsystem_add_ns nqn.2016-06.io.spdk:cnode2 lvs_0/lbd_0 + $rpc_py nvmf_subsystem_add_listener nqn.2016-06.io.spdk:cnode2 -t $TEST_TRANSPORT -a $NVMF_FIRST_TARGET_IP -s $NVMF_PORT + fio_nvme $PLUGIN_DIR/example_config.fio --filename="trtype=$TEST_TRANSPORT adrfam=IPv4 \ + traddr=$NVMF_FIRST_TARGET_IP trsvcid=$NVMF_PORT ns=1" + $rpc_py nvmf_delete_subsystem nqn.2016-06.io.spdk:cnode2 + + # Test fio_plugin as host with nvme lvol nested backend + ls_nested_guid=$($rpc_py bdev_lvol_create_lvstore --clear-method none lvs_0/lbd_0 lvs_n_0) + get_lvs_free_mb $ls_nested_guid + $rpc_py bdev_lvol_create -l lvs_n_0 lbd_nest_0 $free_mb + $rpc_py nvmf_create_subsystem nqn.2016-06.io.spdk:cnode3 -a -s SPDK00000000000001 + $rpc_py nvmf_subsystem_add_ns nqn.2016-06.io.spdk:cnode3 lvs_n_0/lbd_nest_0 + $rpc_py nvmf_subsystem_add_listener nqn.2016-06.io.spdk:cnode3 -t $TEST_TRANSPORT -a $NVMF_FIRST_TARGET_IP -s $NVMF_PORT + fio_nvme $PLUGIN_DIR/example_config.fio --filename="trtype=$TEST_TRANSPORT adrfam=IPv4 \ + traddr=$NVMF_FIRST_TARGET_IP trsvcid=$NVMF_PORT ns=1" + $rpc_py nvmf_delete_subsystem nqn.2016-06.io.spdk:cnode3 + + sync + # Delete lvol_bdev and destroy lvol_store. + $rpc_py bdev_lvol_delete lvs_n_0/lbd_nest_0 + $rpc_py bdev_lvol_delete_lvstore -l lvs_n_0 + $rpc_py bdev_lvol_delete lvs_0/lbd_0 + $rpc_py bdev_lvol_delete_lvstore -l lvs_0 + $rpc_py bdev_nvme_detach_controller Nvme0 +fi + +trap - SIGINT SIGTERM EXIT + +rm -f ./local-test-0-verify.state +nvmftestfini diff --git a/src/spdk/test/nvmf/host/identify.sh b/src/spdk/test/nvmf/host/identify.sh new file mode 100755 index 000000000..412626388 --- /dev/null +++ b/src/spdk/test/nvmf/host/identify.sh @@ -0,0 +1,54 @@ +#!/usr/bin/env bash + +testdir=$(readlink -f $(dirname $0)) +rootdir=$(readlink -f $testdir/../../..) +source $rootdir/test/common/autotest_common.sh +source $rootdir/test/nvmf/common.sh + +MALLOC_BDEV_SIZE=64 +MALLOC_BLOCK_SIZE=512 + +rpc_py="$rootdir/scripts/rpc.py" + +nvmftestinit + +timing_enter start_nvmf_tgt + +"${NVMF_APP[@]}" -m 0xF & +nvmfpid=$! + +trap 'process_shm --id $NVMF_APP_SHM_ID; nvmftestfini; exit 1' SIGINT SIGTERM EXIT + +waitforlisten $nvmfpid +$rpc_py nvmf_create_transport $NVMF_TRANSPORT_OPTS -u 8192 +timing_exit start_nvmf_tgt + +$rpc_py bdev_malloc_create $MALLOC_BDEV_SIZE $MALLOC_BLOCK_SIZE -b Malloc0 +$rpc_py nvmf_create_subsystem nqn.2016-06.io.spdk:cnode1 -a -s SPDK00000000000001 +# NOTE: This will assign the same NGUID and EUI64 to all bdevs, +# but currently we only have one (see above), so this is OK. +$rpc_py nvmf_subsystem_add_ns nqn.2016-06.io.spdk:cnode1 Malloc0 \ + --nguid "ABCDEF0123456789ABCDEF0123456789" \ + --eui64 "ABCDEF0123456789" +$rpc_py nvmf_subsystem_add_listener nqn.2016-06.io.spdk:cnode1 -t $TEST_TRANSPORT -a $NVMF_FIRST_TARGET_IP -s $NVMF_PORT + +$rpc_py nvmf_get_subsystems + +$SPDK_EXAMPLE_DIR/identify -r "\ + trtype:$TEST_TRANSPORT \ + adrfam:IPv4 \ + traddr:$NVMF_FIRST_TARGET_IP \ + trsvcid:$NVMF_PORT \ + subnqn:nqn.2014-08.org.nvmexpress.discovery" -L all +$SPDK_EXAMPLE_DIR/identify -r "\ + trtype:$TEST_TRANSPORT \ + adrfam:IPv4 \ + traddr:$NVMF_FIRST_TARGET_IP \ + trsvcid:$NVMF_PORT \ + subnqn:nqn.2016-06.io.spdk:cnode1" -L all +sync +$rpc_py nvmf_delete_subsystem nqn.2016-06.io.spdk:cnode1 + +trap - SIGINT SIGTERM EXIT + +nvmftestfini diff --git a/src/spdk/test/nvmf/host/identify_kernel_nvmf.sh b/src/spdk/test/nvmf/host/identify_kernel_nvmf.sh new file mode 100755 index 000000000..d6dd2916b --- /dev/null +++ b/src/spdk/test/nvmf/host/identify_kernel_nvmf.sh @@ -0,0 +1,71 @@ +#!/usr/bin/env bash + +testdir=$(readlink -f $(dirname $0)) +rootdir=$(readlink -f $testdir/../../..) +source $rootdir/test/common/autotest_common.sh +source $rootdir/test/nvmf/common.sh + +nvmftestinit + +subsystemname=nqn.2016-06.io.spdk:testnqn + +modprobe null_blk nr_devices=1 +modprobe nvmet +modprobe nvmet-rdma +modprobe nvmet-fc +modprobe lpfc + +if [ ! -d /sys/kernel/config/nvmet/subsystems/$subsystemname ]; then + mkdir /sys/kernel/config/nvmet/subsystems/$subsystemname +fi +echo 1 > /sys/kernel/config/nvmet/subsystems/$subsystemname/attr_allow_any_host + +if [ ! -d /sys/kernel/config/nvmet/subsystems/$subsystemname/namespaces/1 ]; then + mkdir /sys/kernel/config/nvmet/subsystems/$subsystemname/namespaces/1 +fi + +echo -n /dev/nullb0 > /sys/kernel/config/nvmet/subsystems/$subsystemname/namespaces/1/device_path +echo 1 > /sys/kernel/config/nvmet/subsystems/$subsystemname/namespaces/1/enable + +if [ ! -d /sys/kernel/config/nvmet/ports/1 ]; then + mkdir /sys/kernel/config/nvmet/ports/1 +fi + +echo -n rdma > /sys/kernel/config/nvmet/ports/1/addr_trtype +echo -n ipv4 > /sys/kernel/config/nvmet/ports/1/addr_adrfam +echo -n $NVMF_FIRST_TARGET_IP > /sys/kernel/config/nvmet/ports/1/addr_traddr +echo -n $NVMF_PORT > /sys/kernel/config/nvmet/ports/1/addr_trsvcid + +ln -s /sys/kernel/config/nvmet/subsystems/$subsystemname /sys/kernel/config/nvmet/ports/1/subsystems/$subsystemname + +sleep 4 + +$SPDK_EXAMPLE_DIR/identify -r "\ + trtype:$TEST_TRANSPORT \ + adrfam:IPv4 \ + traddr:$NVMF_FIRST_TARGET_IP \ + trsvcid:$NVMF_PORT \ + subnqn:nqn.2014-08.org.nvmexpress.discovery" -t all +$SPDK_EXAMPLE_DIR/identify -r "\ + trtype:$TEST_TRANSPORT \ + adrfam:IPv4 \ + traddr:$NVMF_FIRST_TARGET_IP \ + trsvcid:$NVMF_PORT \ + subnqn:$subsystemname" + +rm -rf /sys/kernel/config/nvmet/ports/1/subsystems/$subsystemname + +echo 0 > /sys/kernel/config/nvmet/subsystems/$subsystemname/namespaces/1/enable +echo -n 0 > /sys/kernel/config/nvmet/subsystems/$subsystemname/namespaces/1/device_path + +rmdir --ignore-fail-on-non-empty /sys/kernel/config/nvmet/subsystems/$subsystemname/namespaces/1 +rmdir --ignore-fail-on-non-empty /sys/kernel/config/nvmet/subsystems/$subsystemname +rmdir --ignore-fail-on-non-empty /sys/kernel/config/nvmet/ports/1 + +rmmod lpfc +rmmod nvmet_fc +rmmod nvmet-rdma +rmmod null_blk +rmmod nvmet + +nvmftestfini diff --git a/src/spdk/test/nvmf/host/perf.sh b/src/spdk/test/nvmf/host/perf.sh new file mode 100755 index 000000000..69fa28f0b --- /dev/null +++ b/src/spdk/test/nvmf/host/perf.sh @@ -0,0 +1,93 @@ +#!/usr/bin/env bash + +testdir=$(readlink -f $(dirname $0)) +rootdir=$(readlink -f $testdir/../../..) +source $rootdir/test/common/autotest_common.sh +source $rootdir/test/nvmf/common.sh + +MALLOC_BDEV_SIZE=64 +MALLOC_BLOCK_SIZE=512 + +rpc_py="$rootdir/scripts/rpc.py" + +nvmftestinit +nvmfappstart -m 0xF + +$rootdir/scripts/gen_nvme.sh --json | $rpc_py load_subsystem_config + +local_nvme_trid="trtype:PCIe traddr:"$($rpc_py framework_get_config bdev | jq -r '.[].params | select(.name=="Nvme0").traddr') +bdevs="$bdevs $($rpc_py bdev_malloc_create $MALLOC_BDEV_SIZE $MALLOC_BLOCK_SIZE)" + +if [ -n "$local_nvme_trid" ]; then + bdevs="$bdevs Nvme0n1" +fi + +$rpc_py nvmf_create_transport $NVMF_TRANSPORT_OPTS +$rpc_py nvmf_create_subsystem nqn.2016-06.io.spdk:cnode1 -a -s SPDK00000000000001 +for bdev in $bdevs; do + $rpc_py nvmf_subsystem_add_ns nqn.2016-06.io.spdk:cnode1 $bdev +done +$rpc_py nvmf_subsystem_add_listener nqn.2016-06.io.spdk:cnode1 -t $TEST_TRANSPORT -a $NVMF_FIRST_TARGET_IP -s $NVMF_PORT + +# Test multi-process access to local NVMe device +if [ -n "$local_nvme_trid" ]; then + if [ $SPDK_RUN_NON_ROOT -eq 1 ]; then + perf_app="sudo -u $(logname) $SPDK_EXAMPLE_DIR/perf" + else + perf_app="$SPDK_EXAMPLE_DIR/perf" + fi + $perf_app -i $NVMF_APP_SHM_ID -q 32 -o 4096 -w randrw -M 50 -t 1 -r "$local_nvme_trid" +fi + +$SPDK_EXAMPLE_DIR/perf -q 1 -o 4096 -w randrw -M 50 -t 1 -r "trtype:$TEST_TRANSPORT adrfam:IPv4 traddr:$NVMF_FIRST_TARGET_IP trsvcid:$NVMF_PORT" +$SPDK_EXAMPLE_DIR/perf -q 32 -o 4096 -w randrw -M 50 -t 1 -r "trtype:$TEST_TRANSPORT adrfam:IPv4 traddr:$NVMF_FIRST_TARGET_IP trsvcid:$NVMF_PORT" +$SPDK_EXAMPLE_DIR/perf -q 128 -o 262144 -w randrw -M 50 -t 2 -r "trtype:$TEST_TRANSPORT adrfam:IPv4 traddr:$NVMF_FIRST_TARGET_IP trsvcid:$NVMF_PORT" +sync +$rpc_py nvmf_delete_subsystem nqn.2016-06.io.spdk:cnode1 + +if [ $RUN_NIGHTLY -eq 1 ]; then + # Configure nvme devices with nvmf lvol_bdev backend + if [ -n "$local_nvme_trid" ]; then + ls_guid=$($rpc_py bdev_lvol_create_lvstore Nvme0n1 lvs_0) + get_lvs_free_mb $ls_guid + # We don't need to create an lvol larger than 20G for this test. + # decreasing the size of the nested lvol allows us to take less time setting up + #before running I/O. + if [ $free_mb -gt 20480 ]; then + free_mb=20480 + fi + lb_guid=$($rpc_py bdev_lvol_create -u $ls_guid lbd_0 $free_mb) + + # Create lvol bdev for nested lvol stores + ls_nested_guid=$($rpc_py bdev_lvol_create_lvstore $lb_guid lvs_n_0) + get_lvs_free_mb $ls_nested_guid + if [ $free_mb -gt 20480 ]; then + free_mb=20480 + fi + lb_nested_guid=$($rpc_py bdev_lvol_create -u $ls_nested_guid lbd_nest_0 $free_mb) + $rpc_py nvmf_create_subsystem nqn.2016-06.io.spdk:cnode1 -a -s SPDK00000000000001 + for bdev in $lb_nested_guid; do + $rpc_py nvmf_subsystem_add_ns nqn.2016-06.io.spdk:cnode1 $bdev + done + $rpc_py nvmf_subsystem_add_listener nqn.2016-06.io.spdk:cnode1 -t $TEST_TRANSPORT -a $NVMF_FIRST_TARGET_IP -s $NVMF_PORT + # Test perf as host with different io_size and qd_depth in nightly + qd_depth=("1" "32" "128") + io_size=("512" "131072") + for qd in "${qd_depth[@]}"; do + for o in "${io_size[@]}"; do + $SPDK_EXAMPLE_DIR/perf -q $qd -o $o -w randrw -M 50 -t 10 -r "trtype:$TEST_TRANSPORT adrfam:IPv4 traddr:$NVMF_FIRST_TARGET_IP trsvcid:$NVMF_PORT" + done + done + + # Delete subsystems, lvol_bdev and destroy lvol_store. + $rpc_py nvmf_delete_subsystem nqn.2016-06.io.spdk:cnode1 + $rpc_py bdev_lvol_delete "$lb_nested_guid" + $rpc_py bdev_lvol_delete_lvstore -l lvs_n_0 + $rpc_py bdev_lvol_delete "$lb_guid" + $rpc_py bdev_lvol_delete_lvstore -l lvs_0 + fi +fi + +trap - SIGINT SIGTERM EXIT + +nvmftestfini diff --git a/src/spdk/test/nvmf/host/target_disconnect.sh b/src/spdk/test/nvmf/host/target_disconnect.sh new file mode 100755 index 000000000..82521196b --- /dev/null +++ b/src/spdk/test/nvmf/host/target_disconnect.sh @@ -0,0 +1,89 @@ +#!/usr/bin/env bash + +testdir=$(readlink -f $(dirname $0)) +rootdir=$(readlink -f $testdir/../../..) +source $rootdir/test/common/autotest_common.sh +source $rootdir/test/nvmf/common.sh + +PLUGIN_DIR=$rootdir/examples/nvme/fio_plugin + +MALLOC_BDEV_SIZE=64 +MALLOC_BLOCK_SIZE=512 + +rpc_py="$rootdir/scripts/rpc.py" + +function disconnect_init() { + nvmfappstart -m 0xF0 + + $rpc_py bdev_malloc_create $MALLOC_BDEV_SIZE $MALLOC_BLOCK_SIZE -b Malloc0 + + $rpc_py nvmf_create_transport $NVMF_TRANSPORT_OPTS + $rpc_py nvmf_subsystem_create nqn.2016-06.io.spdk:cnode1 -a -s SPDK00000000000001 + + $rpc_py nvmf_subsystem_add_ns nqn.2016-06.io.spdk:cnode1 Malloc0 + $rpc_py nvmf_subsystem_add_listener nqn.2016-06.io.spdk:cnode1 -t $TEST_TRANSPORT -a $1 -s $NVMF_PORT +} + +# Test to make sure we don't segfault or access null pointers when we try to connect to +# a discovery controller that doesn't exist yet. +function nvmf_target_disconnect_tc1() { + set +e + $SPDK_EXAMPLE_DIR/reconnect -q 32 -o 4096 -w randrw -M 50 -t 10 -c 0xF \ + -r "trtype:$TEST_TRANSPORT adrfam:IPv4 traddr:$NVMF_FIRST_TARGET_IP trsvcid:$NVMF_PORT" + # If the program crashes, the high bit of $? will be set so we will get a value in the hundreds. + # But if the reconnect code detects errors and exits normally it will return 1. + if [ $? != 1 ]; then + set -e + exit 1 + fi + set -e +} + +function nvmf_target_disconnect_tc2() { + disconnect_init $NVMF_FIRST_TARGET_IP + + # If perf doesn't shut down, this test will time out. + $SPDK_EXAMPLE_DIR/reconnect -q 32 -o 4096 -w randrw -M 50 -t 10 -c 0xF \ + -r "trtype:$TEST_TRANSPORT adrfam:IPv4 traddr:$NVMF_FIRST_TARGET_IP trsvcid:$NVMF_PORT" & + reconnectpid=$! + + sleep 2 + kill -9 $nvmfpid + + sleep 2 + disconnect_init $NVMF_FIRST_TARGET_IP + + wait $reconnectpid + sync +} + +function nvmf_target_disconnect_tc3() { + $SPDK_EXAMPLE_DIR/reconnect -q 32 -o 4096 -w randrw -M 50 -t 10 -c 0xF \ + -r "trtype:$TEST_TRANSPORT adrfam:IPv4 traddr:$NVMF_FIRST_TARGET_IP trsvcid:$NVMF_PORT alt_traddr:$NVMF_SECOND_TARGET_IP" & + reconnectpid=$! + + sleep 2 + kill -9 $nvmfpid + + sleep 2 + disconnect_init $NVMF_SECOND_TARGET_IP + + wait $reconnectpid + sync +} + +nvmftestinit +# There is an intermittent error relating to this test and Soft-RoCE. for now, just +# skip this test if we are using rxe. TODO: get to the bottom of GitHub issue #1043 +if [ $TEST_TRANSPORT == "rdma" ] && check_ip_is_soft_roce $NVMF_FIRST_TARGET_IP; then + echo "Using software RDMA, skipping the target disconnect tests." +else + run_test "nvmf_target_disconnect_tc1" nvmf_target_disconnect_tc1 + run_test "nvmf_target_disconnect_tc2" nvmf_target_disconnect_tc2 + if [ -n "$NVMF_SECOND_TARGET_IP" ]; then + run_test "nvmf_target_disconnect_tc3" nvmf_target_disconnect_tc3 + fi +fi + +trap - SIGINT SIGTERM EXIT +nvmftestfini diff --git a/src/spdk/test/nvmf/nvmf.sh b/src/spdk/test/nvmf/nvmf.sh new file mode 100755 index 000000000..b33d55cff --- /dev/null +++ b/src/spdk/test/nvmf/nvmf.sh @@ -0,0 +1,60 @@ +#!/usr/bin/env bash +testdir=$(readlink -f $(dirname $0)) +rootdir=$(readlink -f $testdir/../..) +source $rootdir/test/common/autotest_common.sh + +if [ ! $(uname -s) = Linux ]; then + exit 0 +fi + +source $rootdir/test/nvmf/common.sh + +trap "exit 1" SIGINT SIGTERM EXIT + +TEST_ARGS=("$@") + +run_test "nvmf_example" test/nvmf/target/nvmf_example.sh "${TEST_ARGS[@]}" +run_test "nvmf_filesystem" test/nvmf/target/filesystem.sh "${TEST_ARGS[@]}" +run_test "nvmf_discovery" test/nvmf/target/discovery.sh "${TEST_ARGS[@]}" +run_test "nvmf_connect_disconnect" test/nvmf/target/connect_disconnect.sh "${TEST_ARGS[@]}" +if [ $SPDK_TEST_NVME_CLI -eq 1 ]; then + run_test "nvmf_nvme_cli" test/nvmf/target/nvme_cli.sh "${TEST_ARGS[@]}" +fi +run_test "nvmf_lvol" test/nvmf/target/nvmf_lvol.sh "${TEST_ARGS[@]}" +#TODO: disabled due to intermittent failures. Need to triage. +# run_test "nvmf_srq_overwhelm" test/nvmf/target/srq_overwhelm.sh $TEST_ARGS +run_test "nvmf_vhost" test/nvmf/target/nvmf_vhost.sh "${TEST_ARGS[@]}" +run_test "nvmf_bdev_io_wait" test/nvmf/target/bdev_io_wait.sh "${TEST_ARGS[@]}" +run_test "nvmf_create_transport." test/nvmf/target/create_transport.sh "${TEST_ARGS[@]}" +run_test "nvmf_multitarget" test/nvmf/target/multitarget.sh "${TEST_ARGS[@]}" + +if [ $RUN_NIGHTLY -eq 1 ]; then + run_test "nvmf_fuzz" test/nvmf/target/fuzz.sh "${TEST_ARGS[@]}" + run_test "nvmf_multiconnection" test/nvmf/target/multiconnection.sh "${TEST_ARGS[@]}" + run_test "nvmf_initiator_timeout" test/nvmf/target/initiator_timeout.sh "${TEST_ARGS[@]}" +fi + +run_test "nvmf_nmic" test/nvmf/target/nmic.sh "${TEST_ARGS[@]}" +run_test "nvmf_rpc" test/nvmf/target/rpc.sh "${TEST_ARGS[@]}" +run_test "nvmf_fio" test/nvmf/target/fio.sh "${TEST_ARGS[@]}" +run_test "nvmf_shutdown" test/nvmf/target/shutdown.sh "${TEST_ARGS[@]}" +run_test "nvmf_bdevio" test/nvmf/target/bdevio.sh "${TEST_ARGS[@]}" +run_test "nvmf_invalid" test/nvmf/target/invalid.sh "${TEST_ARGS[@]}" +run_test "nvmf_abort" test/nvmf/target/abort.sh "${TEST_ARGS[@]}" + +timing_enter host + +run_test "nvmf_bdevperf" test/nvmf/host/bdevperf.sh "${TEST_ARGS[@]}" +run_test "nvmf_identify" test/nvmf/host/identify.sh "${TEST_ARGS[@]}" +run_test "nvmf_perf" test/nvmf/host/perf.sh "${TEST_ARGS[@]}" + +# TODO: disabled due to intermittent failures (RDMA_CM_EVENT_UNREACHABLE/ETIMEDOUT) +#run_test test/nvmf/host/identify_kernel_nvmf.sh $TEST_ARGS +run_test "nvmf_aer" test/nvmf/host/aer.sh "${TEST_ARGS[@]}" +run_test "nvmf_fio" test/nvmf/host/fio.sh "${TEST_ARGS[@]}" +run_test "nvmf_target_disconnect" test/nvmf/host/target_disconnect.sh "${TEST_ARGS[@]}" + +timing_exit host + +trap - SIGINT SIGTERM EXIT +revert_soft_roce diff --git a/src/spdk/test/nvmf/target/abort.sh b/src/spdk/test/nvmf/target/abort.sh new file mode 100755 index 000000000..913c17e19 --- /dev/null +++ b/src/spdk/test/nvmf/target/abort.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash + +testdir=$(readlink -f $(dirname $0)) +rootdir=$(readlink -f $testdir/../../..) +source $rootdir/test/common/autotest_common.sh +source $rootdir/test/nvmf/common.sh + +MALLOC_BDEV_SIZE=64 +MALLOC_BLOCK_SIZE=4096 + +rpc_py="$rootdir/scripts/rpc.py" + +nvmftestinit +nvmfappstart -m 0xE + +$rpc_py nvmf_create_transport $NVMF_TRANSPORT_OPTS -u 8192 + +# Construct a delay bdev on a malloc bdev which has constant 10ms delay for all read or write I/Os +$rpc_py bdev_malloc_create $MALLOC_BDEV_SIZE $MALLOC_BLOCK_SIZE -b Malloc0 +$rpc_py bdev_delay_create -b Malloc0 -d Delay0 -r 1000000 -t 1000000 -w 1000000 -n 1000000 + +# Create an NVMe-oF subsystem and add the delay bdev as a namespace +$rpc_py nvmf_create_subsystem nqn.2016-06.io.spdk:cnode0 -a -s SPDK0 +$rpc_py nvmf_subsystem_add_ns nqn.2016-06.io.spdk:cnode0 Delay0 +$rpc_py nvmf_subsystem_add_listener nqn.2016-06.io.spdk:cnode0 -t $TEST_TRANSPORT -a $NVMF_FIRST_TARGET_IP -s $NVMF_PORT + +# Run abort application +$SPDK_EXAMPLE_DIR/abort -r "trtype:$TEST_TRANSPORT adrfam:IPv4 traddr:$NVMF_FIRST_TARGET_IP trsvcid:$NVMF_PORT" -c 0x1 + +# Clean up +$rpc_py nvmf_delete_subsystem nqn.2016-06.io.spdk:cnode0 + +trap - SIGINT SIGTERM EXIT + +nvmftestfini diff --git a/src/spdk/test/nvmf/target/bdev_io_wait.sh b/src/spdk/test/nvmf/target/bdev_io_wait.sh new file mode 100755 index 000000000..e57ffc36d --- /dev/null +++ b/src/spdk/test/nvmf/target/bdev_io_wait.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash + +testdir=$(readlink -f $(dirname $0)) +rootdir=$(readlink -f $testdir/../../..) +source $rootdir/test/common/autotest_common.sh +source $rootdir/test/nvmf/common.sh + +MALLOC_BDEV_SIZE=64 +MALLOC_BLOCK_SIZE=512 + +rpc_py="$rootdir/scripts/rpc.py" + +nvmftestinit +nvmfappstart -m 0xF --wait-for-rpc + +# Minimal number of bdev io pool (5) and cache (1) +$rpc_py bdev_set_options -p 5 -c 1 +$rpc_py framework_start_init +$rpc_py nvmf_create_transport $NVMF_TRANSPORT_OPTS -u 8192 + +$rpc_py bdev_malloc_create $MALLOC_BDEV_SIZE $MALLOC_BLOCK_SIZE -b Malloc0 +$rpc_py nvmf_create_subsystem nqn.2016-06.io.spdk:cnode1 -a -s SPDK00000000000001 +$rpc_py nvmf_subsystem_add_ns nqn.2016-06.io.spdk:cnode1 Malloc0 +$rpc_py nvmf_subsystem_add_listener nqn.2016-06.io.spdk:cnode1 -t $TEST_TRANSPORT -a $NVMF_FIRST_TARGET_IP -s $NVMF_PORT + +"$rootdir/test/bdev/bdevperf/bdevperf" -m 0x10 -i 1 --json <(gen_nvmf_target_json) -q 128 -o 4096 -w write -t 1 & +WRITE_PID=$! +"$rootdir/test/bdev/bdevperf/bdevperf" -m 0x20 -i 2 --json <(gen_nvmf_target_json) -q 128 -o 4096 -w read -t 1 & +READ_PID=$! +"$rootdir/test/bdev/bdevperf/bdevperf" -m 0x40 -i 3 --json <(gen_nvmf_target_json) -q 128 -o 4096 -w flush -t 1 & +FLUSH_PID=$! +"$rootdir/test/bdev/bdevperf/bdevperf" -m 0x80 -i 4 --json <(gen_nvmf_target_json) -q 128 -o 4096 -w unmap -t 1 & +UNMAP_PID=$! +sync + +wait $WRITE_PID +wait $READ_PID +wait $FLUSH_PID +wait $UNMAP_PID + +$rpc_py nvmf_delete_subsystem nqn.2016-06.io.spdk:cnode1 + +trap - SIGINT SIGTERM EXIT + +nvmftestfini diff --git a/src/spdk/test/nvmf/target/bdevio.sh b/src/spdk/test/nvmf/target/bdevio.sh new file mode 100755 index 000000000..f4d7eb1b5 --- /dev/null +++ b/src/spdk/test/nvmf/target/bdevio.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash + +testdir=$(readlink -f $(dirname $0)) +rootdir=$(readlink -f $testdir/../../..) +source $rootdir/test/common/autotest_common.sh +source $rootdir/test/nvmf/common.sh + +MALLOC_BDEV_SIZE=64 +MALLOC_BLOCK_SIZE=512 + +rpc_py="$rootdir/scripts/rpc.py" + +nvmftestinit +# Don't use cores 0 - 2 to avoid overlap with bdevio. +nvmfappstart -m 0x78 + +$rpc_py nvmf_create_transport $NVMF_TRANSPORT_OPTS -u 8192 +$rpc_py bdev_malloc_create $MALLOC_BDEV_SIZE $MALLOC_BLOCK_SIZE -b Malloc0 +$rpc_py nvmf_create_subsystem nqn.2016-06.io.spdk:cnode1 -a -s SPDK00000000000001 +$rpc_py nvmf_subsystem_add_ns nqn.2016-06.io.spdk:cnode1 Malloc0 +$rpc_py nvmf_subsystem_add_listener nqn.2016-06.io.spdk:cnode1 -t $TEST_TRANSPORT -a $NVMF_FIRST_TARGET_IP -s $NVMF_PORT + +$rootdir/test/bdev/bdevio/bdevio --json <(gen_nvmf_target_json) + +$rpc_py nvmf_delete_subsystem nqn.2016-06.io.spdk:cnode1 + +trap - SIGINT SIGTERM EXIT + +nvmftestfini diff --git a/src/spdk/test/nvmf/target/connect_disconnect.sh b/src/spdk/test/nvmf/target/connect_disconnect.sh new file mode 100755 index 000000000..b74394123 --- /dev/null +++ b/src/spdk/test/nvmf/target/connect_disconnect.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash + +testdir=$(readlink -f $(dirname $0)) +rootdir=$(readlink -f $testdir/../../..) +source $rootdir/test/common/autotest_common.sh +source $rootdir/test/nvmf/common.sh + +MALLOC_BDEV_SIZE=64 +MALLOC_BLOCK_SIZE=512 + +rpc_py="$rootdir/scripts/rpc.py" + +# connect disconnect is geared towards ensuring that we are properly freeing resources after disconnecting qpairs. +nvmftestinit +nvmfappstart -m 0xF + +$rpc_py nvmf_create_transport $NVMF_TRANSPORT_OPTS -u 8192 -c 0 + +bdev="$($rpc_py bdev_malloc_create $MALLOC_BDEV_SIZE $MALLOC_BLOCK_SIZE)" + +$rpc_py nvmf_create_subsystem nqn.2016-06.io.spdk:cnode1 -a -s $NVMF_SERIAL +$rpc_py nvmf_subsystem_add_ns nqn.2016-06.io.spdk:cnode1 $bdev +$rpc_py nvmf_subsystem_add_listener nqn.2016-06.io.spdk:cnode1 -t $TEST_TRANSPORT -a $NVMF_FIRST_TARGET_IP -s $NVMF_PORT + +if [ $RUN_NIGHTLY -eq 1 ]; then + num_iterations=200 + IO_QUEUES="-i 8" +else + num_iterations=10 +fi + +set +x +for i in $(seq 1 $num_iterations); do + nvme connect -t $TEST_TRANSPORT -n "nqn.2016-06.io.spdk:cnode1" -a "$NVMF_FIRST_TARGET_IP" -s "$NVMF_PORT" $IO_QUEUES + waitforserial "$NVMF_SERIAL" + nvme disconnect -n "nqn.2016-06.io.spdk:cnode1" + waitforserial_disconnect "$NVMF_SERIAL" +done +set -x + +trap - SIGINT SIGTERM EXIT + +nvmftestfini diff --git a/src/spdk/test/nvmf/target/create_transport.sh b/src/spdk/test/nvmf/target/create_transport.sh new file mode 100755 index 000000000..e2766467b --- /dev/null +++ b/src/spdk/test/nvmf/target/create_transport.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash + +testdir=$(readlink -f $(dirname $0)) +rootdir=$(readlink -f $testdir/../../..) +source $rootdir/test/common/autotest_common.sh +source $rootdir/test/nvmf/common.sh + +NULL_BDEV_SIZE=102400 +NULL_BLOCK_SIZE=512 + +rpc_py="$rootdir/scripts/rpc.py" + +if ! hash nvme; then + echo "nvme command not found; skipping create transport test" + exit 0 +fi + +nvmftestinit +nvmfappstart -m 0xF + +# Use nvmf_create_transport call to create transport +$rpc_py nvmf_create_transport $NVMF_TRANSPORT_OPTS -u 8192 + +null_bdevs="$($rpc_py bdev_null_create Null0 $NULL_BDEV_SIZE $NULL_BLOCK_SIZE) " +null_bdevs+="$($rpc_py bdev_null_create Null1 $NULL_BDEV_SIZE $NULL_BLOCK_SIZE)" + +$rpc_py nvmf_create_subsystem nqn.2016-06.io.spdk:cnode1 -a -s SPDK00000000000001 +for null_bdev in $null_bdevs; do + $rpc_py nvmf_subsystem_add_ns nqn.2016-06.io.spdk:cnode1 $null_bdev +done +$rpc_py nvmf_subsystem_add_listener nqn.2016-06.io.spdk:cnode1 -t $TEST_TRANSPORT -a $NVMF_FIRST_TARGET_IP -s $NVMF_PORT + +nvme discover -t $TEST_TRANSPORT -a $NVMF_FIRST_TARGET_IP -s $NVMF_PORT + +echo "Perform nvmf subsystem discovery via RPC" +$rpc_py nvmf_get_subsystems + +$rpc_py nvmf_delete_subsystem nqn.2016-06.io.spdk:cnode1 + +for null_bdev in $null_bdevs; do + $rpc_py bdev_null_delete $null_bdev +done + +check_bdevs=$($rpc_py bdev_get_bdevs | jq -r '.[].name') +if [ -n "$check_bdevs" ]; then + echo $check_bdevs + exit 1 +fi + +trap - SIGINT SIGTERM EXIT + +nvmftestfini diff --git a/src/spdk/test/nvmf/target/discovery.sh b/src/spdk/test/nvmf/target/discovery.sh new file mode 100755 index 000000000..ad5a6ce96 --- /dev/null +++ b/src/spdk/test/nvmf/target/discovery.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash + +testdir=$(readlink -f $(dirname $0)) +rootdir=$(readlink -f $testdir/../../..) +source $rootdir/test/common/autotest_common.sh +source $rootdir/test/nvmf/common.sh + +NULL_BDEV_SIZE=102400 +NULL_BLOCK_SIZE=512 + +rpc_py="$rootdir/scripts/rpc.py" + +if ! hash nvme; then + echo "nvme command not found; skipping discovery test" + exit 0 +fi + +nvmftestinit +nvmfappstart -m 0xF + +$rpc_py nvmf_create_transport $NVMF_TRANSPORT_OPTS -u 8192 + +# Use at least 4 subsystems so they spill over to a second discovery log page +for i in $(seq 1 4); do + $rpc_py bdev_null_create Null$i $NULL_BDEV_SIZE $NULL_BLOCK_SIZE + $rpc_py nvmf_create_subsystem nqn.2016-06.io.spdk:cnode$i -a -s SPDK0000000000000$i + $rpc_py nvmf_subsystem_add_ns nqn.2016-06.io.spdk:cnode$i Null$i + $rpc_py nvmf_subsystem_add_listener nqn.2016-06.io.spdk:cnode$i -t $TEST_TRANSPORT -a $NVMF_FIRST_TARGET_IP -s $NVMF_PORT +done + +nvme discover -t $TEST_TRANSPORT -a $NVMF_FIRST_TARGET_IP -s $NVMF_PORT + +echo "Perform nvmf subsystem discovery via RPC" +$rpc_py nvmf_get_subsystems + +for i in $(seq 1 4); do + $rpc_py nvmf_delete_subsystem nqn.2016-06.io.spdk:cnode$i + $rpc_py bdev_null_delete Null$i +done + +check_bdevs=$($rpc_py bdev_get_bdevs | jq -r '.[].name') +if [ -n "$check_bdevs" ]; then + echo $check_bdevs + exit 1 +fi + +trap - SIGINT SIGTERM EXIT + +nvmftestfini diff --git a/src/spdk/test/nvmf/target/filesystem.sh b/src/spdk/test/nvmf/target/filesystem.sh new file mode 100755 index 000000000..ff819fdb6 --- /dev/null +++ b/src/spdk/test/nvmf/target/filesystem.sh @@ -0,0 +1,92 @@ +#!/usr/bin/env bash + +testdir=$(readlink -f $(dirname $0)) +rootdir=$(readlink -f $testdir/../../..) +source $rootdir/test/common/autotest_common.sh +source $rootdir/test/nvmf/common.sh + +MALLOC_BDEV_SIZE=64 +MALLOC_BLOCK_SIZE=512 + +rpc_py="$rootdir/scripts/rpc.py" + +nvmftestinit + +function nvmf_filesystem_create() { + fstype=$1 + nvme_name=$2 + + make_filesystem ${fstype} /dev/${nvme_name}p1 + + mount /dev/${nvme_name}p1 /mnt/device + touch /mnt/device/aaa + sync + rm /mnt/device/aaa + sync + + i=0 + while ! umount /mnt/device; do + [ $i -lt 15 ] || break + i=$((i + 1)) + sleep 1 + done + + # Make sure the target did not crash + kill -0 $nvmfpid + + # Make sure the device is still present + lsblk -l -o NAME | grep -q -w "${nvme_name}" + + # Make sure the partition is still present + lsblk -l -o NAME | grep -q -w "${nvme_name}p1" +} + +function nvmf_filesystem_part() { + incapsule=$1 + + nvmfappstart -m 0xF + + $rpc_py nvmf_create_transport $NVMF_TRANSPORT_OPTS -u 8192 -c $incapsule + $rpc_py bdev_malloc_create $MALLOC_BDEV_SIZE $MALLOC_BLOCK_SIZE -b Malloc1 + $rpc_py nvmf_create_subsystem nqn.2016-06.io.spdk:cnode1 -a -s $NVMF_SERIAL + $rpc_py nvmf_subsystem_add_ns nqn.2016-06.io.spdk:cnode1 Malloc1 + $rpc_py nvmf_subsystem_add_listener nqn.2016-06.io.spdk:cnode1 -t $TEST_TRANSPORT -a $NVMF_FIRST_TARGET_IP -s $NVMF_PORT + + nvme connect -t $TEST_TRANSPORT -n "nqn.2016-06.io.spdk:cnode1" -a "$NVMF_FIRST_TARGET_IP" -s "$NVMF_PORT" + + waitforserial "$NVMF_SERIAL" + nvme_name=$(lsblk -l -o NAME,SERIAL | grep -oP "([\w]*)(?=\s+${NVMF_SERIAL})") + + mkdir -p /mnt/device + + parted -s /dev/${nvme_name} mklabel msdos mkpart primary '0%' '100%' + partprobe + sleep 1 + + if [ $incapsule -eq 0 ]; then + run_test "filesystem_ext4" nvmf_filesystem_create "ext4" ${nvme_name} + run_test "filesystem_btrfs" nvmf_filesystem_create "btrfs" ${nvme_name} + run_test "filesystem_xfs" nvmf_filesystem_create "xfs" ${nvme_name} + else + run_test "filesystem_incapsule_ext4" nvmf_filesystem_create "ext4" ${nvme_name} + run_test "filesystem_incapsule_btrfs" nvmf_filesystem_create "btrfs" ${nvme_name} + run_test "filesystem_incapsule_xfs" nvmf_filesystem_create "xfs" ${nvme_name} + fi + + parted -s /dev/${nvme_name} rm 1 + + sync + nvme disconnect -n "nqn.2016-06.io.spdk:cnode1" || true + + $rpc_py nvmf_delete_subsystem nqn.2016-06.io.spdk:cnode1 + + trap - SIGINT SIGTERM EXIT + + killprocess $nvmfpid + nvmfpid= +} + +run_test "nvmf_filesystem_no_incapsule" nvmf_filesystem_part 0 +run_test "nvmf_filesystem_incapsule" nvmf_filesystem_part 4096 + +nvmftestfini diff --git a/src/spdk/test/nvmf/target/fio.sh b/src/spdk/test/nvmf/target/fio.sh new file mode 100755 index 000000000..4e98d7083 --- /dev/null +++ b/src/spdk/test/nvmf/target/fio.sh @@ -0,0 +1,77 @@ +#!/usr/bin/env bash + +testdir=$(readlink -f $(dirname $0)) +rootdir=$(readlink -f $testdir/../../..) +source $rootdir/test/common/autotest_common.sh +source $rootdir/test/nvmf/common.sh + +MALLOC_BDEV_SIZE=64 +MALLOC_BLOCK_SIZE=512 + +rpc_py="$rootdir/scripts/rpc.py" + +nvmftestinit +nvmfappstart -m 0xF + +$rpc_py nvmf_create_transport $NVMF_TRANSPORT_OPTS -u 8192 + +malloc_bdevs="$($rpc_py bdev_malloc_create $MALLOC_BDEV_SIZE $MALLOC_BLOCK_SIZE) " +malloc_bdevs+="$($rpc_py bdev_malloc_create $MALLOC_BDEV_SIZE $MALLOC_BLOCK_SIZE)" +# Create a RAID-0 bdev from two malloc bdevs +raid_malloc_bdevs="$($rpc_py bdev_malloc_create $MALLOC_BDEV_SIZE $MALLOC_BLOCK_SIZE) " +raid_malloc_bdevs+="$($rpc_py bdev_malloc_create $MALLOC_BDEV_SIZE $MALLOC_BLOCK_SIZE)" +$rpc_py bdev_raid_create -n raid0 -z 64 -r 0 -b "$raid_malloc_bdevs" + +$rpc_py nvmf_create_subsystem nqn.2016-06.io.spdk:cnode1 -a -s $NVMF_SERIAL +for malloc_bdev in $malloc_bdevs; do + $rpc_py nvmf_subsystem_add_ns nqn.2016-06.io.spdk:cnode1 "$malloc_bdev" +done +$rpc_py nvmf_subsystem_add_listener nqn.2016-06.io.spdk:cnode1 -t $TEST_TRANSPORT -a $NVMF_FIRST_TARGET_IP -s $NVMF_PORT + +# Append the raid0 bdev into subsystem +$rpc_py nvmf_subsystem_add_ns nqn.2016-06.io.spdk:cnode1 raid0 + +nvme connect -t $TEST_TRANSPORT -n "nqn.2016-06.io.spdk:cnode1" -a "$NVMF_FIRST_TARGET_IP" -s "$NVMF_PORT" + +waitforserial $NVMF_SERIAL 3 + +$rootdir/scripts/fio.py -p nvmf -i 4096 -d 1 -t write -r 1 -v +$rootdir/scripts/fio.py -p nvmf -i 4096 -d 1 -t randwrite -r 1 -v +$rootdir/scripts/fio.py -p nvmf -i 4096 -d 128 -t write -r 1 -v +$rootdir/scripts/fio.py -p nvmf -i 4096 -d 128 -t randwrite -r 1 -v + +sync + +#start hotplug test case +$rootdir/scripts/fio.py -p nvmf -i 4096 -d 1 -t read -r 10 & +fio_pid=$! + +sleep 3 + +$rpc_py bdev_raid_delete "raid0" +for malloc_bdev in $malloc_bdevs; do + $rpc_py bdev_malloc_delete "$malloc_bdev" +done + +fio_status=0 +wait $fio_pid || fio_status=$? + +nvme disconnect -n "nqn.2016-06.io.spdk:cnode1" || true + +if [ $fio_status -eq 0 ]; then + echo "nvmf hotplug test: fio successful - expected failure" + nvmftestfini + exit 1 +else + echo "nvmf hotplug test: fio failed as expected" +fi + +$rpc_py nvmf_delete_subsystem nqn.2016-06.io.spdk:cnode1 + +rm -f ./local-job0-0-verify.state +rm -f ./local-job1-1-verify.state +rm -f ./local-job2-2-verify.state + +trap - SIGINT SIGTERM EXIT + +nvmftestfini diff --git a/src/spdk/test/nvmf/target/fuzz.sh b/src/spdk/test/nvmf/target/fuzz.sh new file mode 100755 index 000000000..5a18be856 --- /dev/null +++ b/src/spdk/test/nvmf/target/fuzz.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash + +testdir=$(readlink -f $(dirname $0)) +rootdir=$(readlink -f $testdir/../../..) +source $rootdir/test/common/autotest_common.sh +source $rootdir/test/nvmf/common.sh + +rpc_py="$rootdir/scripts/rpc.py" + +nvmftestinit + +"${NVMF_APP[@]}" -m 0xF > $output_dir/nvmf_fuzz_tgt_output.txt 2>&1 & +nvmfpid=$! + +trap 'process_shm --id $NVMF_APP_SHM_ID; rm -f $testdir/nvmf_fuzz.conf; killprocess $nvmfpid; nvmftestfini $1; exit 1' SIGINT SIGTERM EXIT + +waitforlisten $nvmfpid +$rpc_py nvmf_create_transport $NVMF_TRANSPORT_OPTS -u 8192 + +$rpc_py bdev_malloc_create -b Malloc0 64 512 + +$rpc_py nvmf_create_subsystem nqn.2016-06.io.spdk:cnode1 -a -s SPDK00000000000001 +$rpc_py nvmf_subsystem_add_ns nqn.2016-06.io.spdk:cnode1 Malloc0 +$rpc_py nvmf_subsystem_add_listener nqn.2016-06.io.spdk:cnode1 -t $TEST_TRANSPORT -a $NVMF_FIRST_TARGET_IP -s $NVMF_PORT + +echo "[Nvme]" > $testdir/nvmf_fuzz.conf +echo " TransportID \"trtype:$TEST_TRANSPORT adrfam:IPv4 subnqn:nqn.2016-06.io.spdk:cnode1 traddr:$NVMF_FIRST_TARGET_IP trsvcid:$NVMF_PORT\" Nvme0" >> $testdir/nvmf_fuzz.conf + +# Note that we chose a consistent seed to ensure that this test is consistent in nightly builds. +$rootdir/test/app/fuzz/nvme_fuzz/nvme_fuzz -m 0xF0 -r "/var/tmp/nvme_fuzz" -t 30 -S 123456 -C $testdir/nvmf_fuzz.conf -N -a 2> $output_dir/nvmf_fuzz_logs1.txt +# We don't specify a seed for this test. Instead we run a static list of commands from example.json. +$rootdir/test/app/fuzz/nvme_fuzz/nvme_fuzz -m 0xF0 -r "/var/tmp/nvme_fuzz" -C $testdir/nvmf_fuzz.conf -j $rootdir/test/app/fuzz/nvme_fuzz/example.json -a 2> $output_dir/nvmf_fuzz_logs2.txt + +rm -f $testdir/nvmf_fuzz.conf +$rpc_py nvmf_delete_subsystem nqn.2016-06.io.spdk:cnode1 + +trap - SIGINT SIGTERM EXIT + +nvmfcleanup +killprocess $nvmfpid +nvmfpid= + +nvmftestfini diff --git a/src/spdk/test/nvmf/target/identify_passthru.sh b/src/spdk/test/nvmf/target/identify_passthru.sh new file mode 100755 index 000000000..2ce52fe38 --- /dev/null +++ b/src/spdk/test/nvmf/target/identify_passthru.sh @@ -0,0 +1,76 @@ +#!/usr/bin/env bash + +testdir=$(readlink -f $(dirname $0)) +rootdir=$(readlink -f $testdir/../../..) +source $rootdir/test/common/autotest_common.sh +source $rootdir/test/nvmf/common.sh +source $rootdir/scripts/common.sh + +rpc_py="$rootdir/scripts/rpc.py" + +nvmftestinit + +timing_enter nvme_identify + +bdf=$(get_first_nvme_bdf) +if [ -z "${bdf}" ]; then + echo "No NVMe drive found but test requires it. Failing the test." + exit 1 +fi + +# Expected values +nvme_serial_number=$($SPDK_EXAMPLE_DIR/identify -r "trtype:PCIe traddr:${bdf}" -i 0 | grep "Serial Number:" | awk '{print $3}') +nvme_model_number=$($SPDK_EXAMPLE_DIR/identify -r "trtype:PCIe traddr:${bdf}" -i 0 | grep "Model Number:" | awk '{print $3}') + +timing_exit nvme_identify + +timing_enter start_nvmf_tgt + +"${NVMF_APP[@]}" -m 0xF --wait-for-rpc & +nvmfpid=$! + +trap 'process_shm --id $NVMF_APP_SHM_ID; nvmftestfini; exit 1' SIGINT SIGTERM EXIT + +waitforlisten $nvmfpid +$rpc_py -v nvmf_set_config --passthru-identify-ctrlr +$rpc_py -v framework_start_init +$rpc_py nvmf_create_transport $NVMF_TRANSPORT_OPTS -u 8192 +timing_exit start_nvmf_tgt + +$rpc_py bdev_nvme_attach_controller -b Nvme0 -t PCIe -a ${bdf} +$rpc_py nvmf_create_subsystem nqn.2016-06.io.spdk:cnode1 -a -s SPDK00000000000001 +$rpc_py nvmf_subsystem_add_ns nqn.2016-06.io.spdk:cnode1 Nvme0n1 +$rpc_py nvmf_subsystem_add_listener nqn.2016-06.io.spdk:cnode1 -t $TEST_TRANSPORT -a $NVMF_FIRST_TARGET_IP -s $NVMF_PORT + +$rpc_py nvmf_get_subsystems + +# Discovered values +nvmf_serial_number=$($SPDK_EXAMPLE_DIR/identify -r "\ + trtype:$TEST_TRANSPORT \ + adrfam:IPv4 \ + traddr:$NVMF_FIRST_TARGET_IP \ + trsvcid:$NVMF_PORT \ + subnqn:nqn.2016-06.io.spdk:cnode1" | grep "Serial Number:" | awk '{print $3}') + +nvmf_model_number=$($SPDK_EXAMPLE_DIR/identify -r "\ + trtype:$TEST_TRANSPORT \ + adrfam:IPv4 \ + traddr:$NVMF_FIRST_TARGET_IP \ + trsvcid:$NVMF_PORT \ + subnqn:nqn.2016-06.io.spdk:cnode1" | grep "Model Number:" | awk '{print $3}') + +if [ ${nvme_serial_number} != ${nvmf_serial_number} ]; then + echo "Serial number doesn't match" + exit 1 +fi + +if [ ${nvme_model_number} != ${nvmf_model_number} ]; then + echo "Model number doesn't match" + exit 1 +fi + +$rpc_py nvmf_delete_subsystem nqn.2016-06.io.spdk:cnode1 + +trap - SIGINT SIGTERM EXIT + +nvmftestfini diff --git a/src/spdk/test/nvmf/target/initiator_timeout.sh b/src/spdk/test/nvmf/target/initiator_timeout.sh new file mode 100755 index 000000000..199983be5 --- /dev/null +++ b/src/spdk/test/nvmf/target/initiator_timeout.sh @@ -0,0 +1,71 @@ +#!/usr/bin/env bash + +testdir=$(readlink -f $(dirname $0)) +rootdir=$(readlink -f $testdir/../../..) +source $rootdir/test/common/autotest_common.sh +source $rootdir/test/nvmf/common.sh + +MALLOC_BDEV_SIZE=64 +MALLOC_BLOCK_SIZE=512 + +rpc_py="$rootdir/scripts/rpc.py" + +nvmftestinit +nvmfappstart -m 0xF + +trap 'process_shm --id $NVMF_APP_SHM_ID; killprocess $nvmfpid; nvmftestfini $1; exit 1' SIGINT SIGTERM EXIT + +$rpc_py bdev_malloc_create $MALLOC_BDEV_SIZE $MALLOC_BLOCK_SIZE -b Malloc0 + +# We cannot configure the bdev with an incredibly high latency up front because connect will not work properly. +$rpc_py bdev_delay_create -b Malloc0 -d Delay0 -r 30 -t 30 -w 30 -n 30 + +$rpc_py nvmf_create_transport $NVMF_TRANSPORT_OPTS -u 8192 +$rpc_py nvmf_create_subsystem nqn.2016-06.io.spdk:cnode1 -a -s $NVMF_SERIAL +$rpc_py nvmf_subsystem_add_ns nqn.2016-06.io.spdk:cnode1 Delay0 +$rpc_py nvmf_subsystem_add_listener nqn.2016-06.io.spdk:cnode1 -t $TEST_TRANSPORT -a $NVMF_FIRST_TARGET_IP -s $NVMF_PORT + +nvme connect -t $TEST_TRANSPORT -n "nqn.2016-06.io.spdk:cnode1" -a "$NVMF_FIRST_TARGET_IP" -s "$NVMF_PORT" + +waitforserial "$NVMF_SERIAL" + +# Once our timed out I/O complete, we will still have 10 sec of I/O. +$rootdir/scripts/fio.py -p nvmf -i 4096 -d 1 -t write -r 60 -v & +fio_pid=$! + +sleep 3 + +# The kernel initiator has a default timeout of 30 seconds. delay for 31 to trigger initiator reconnect. +$rpc_py bdev_delay_update_latency Delay0 avg_read 31000000 +$rpc_py bdev_delay_update_latency Delay0 avg_write 31000000 +$rpc_py bdev_delay_update_latency Delay0 p99_read 31000000 +$rpc_py bdev_delay_update_latency Delay0 p99_write 310000000 + +sleep 3 + +# Reset these values so that subsequent I/O will complete in a timely manner. +$rpc_py bdev_delay_update_latency Delay0 avg_read 30 +$rpc_py bdev_delay_update_latency Delay0 avg_write 30 +$rpc_py bdev_delay_update_latency Delay0 p99_read 30 +$rpc_py bdev_delay_update_latency Delay0 p99_write 30 + +fio_status=0 +wait $fio_pid || fio_status=$? + +nvme disconnect -n "nqn.2016-06.io.spdk:cnode1" || true + +if [ $fio_status -eq 0 ]; then + echo "nvmf hotplug test: fio successful as expected" +else + echo "nvmf hotplug test: fio failed, expected success" + nvmftestfini + exit 1 +fi + +$rpc_py nvmf_delete_subsystem nqn.2016-06.io.spdk:cnode1 + +rm -f ./local-job0-0-verify.state + +trap - SIGINT SIGTERM EXIT + +nvmftestfini diff --git a/src/spdk/test/nvmf/target/invalid.sh b/src/spdk/test/nvmf/target/invalid.sh new file mode 100755 index 000000000..98246efeb --- /dev/null +++ b/src/spdk/test/nvmf/target/invalid.sh @@ -0,0 +1,63 @@ +#!/usr/bin/env bash + +testdir=$(readlink -f "$(dirname "$0")") +rootdir=$(readlink -f "$testdir/../../..") +source "$rootdir/test/common/autotest_common.sh" +source "$rootdir/test/nvmf/common.sh" + +multi_target_rpc=$rootdir/test/nvmf/target/multitarget_rpc.py +rpc=$rootdir/scripts/rpc.py +nqn=nqn.2016-06.io.spdk:cnode +target=foobar +# pre-seed the rng to generate predictive values across different test runs +RANDOM=0 + +gen_random_s() { + local length=$1 ll + # generate ascii table which nvme supports + local chars=({32..127}) + local string + + for ((ll = 0; ll < length; ll++)); do + string+="$(echo -e "\x$(printf '%x' "${chars[RANDOM % ${#chars[@]}]}")")" + done + # Be nice to rpc.py's arg parser and escape `-` in case it's a first character + if [[ ${string::1} == "-" ]]; then + string=${string/-/\\-} + fi + echo "$string" +} + +nvmftestinit +nvmfappstart -m 0xF + +trap 'process_shm --id $NVMF_APP_SHM_ID; nvmftestfini $1; exit 1' SIGINT SIGTERM EXIT + +# Attempt to create subsystem with non-existing target +out=$("$rpc" nvmf_create_subsystem -t "$target" "$nqn$RANDOM" 2>&1) && false +[[ $out == *"Unable to find target"* ]] + +# Attempt to create subsystem with invalid serial number - inject ASCII char that's +# not in the range (0x20-0x7e) of these supported by the nvme spec. +out=$("$rpc" nvmf_create_subsystem -s "$NVMF_SERIAL$(echo -e "\x1f")" "$nqn$RANDOM" 2>&1) && false +[[ $out == *"Invalid SN"* ]] + +# Attempt to create subsystem with invalid model - inject ASCII char that's not in the +# range (0x20-0x7e) of these supported by the nvme spec. +out=$("$rpc" nvmf_create_subsystem -d "SPDK_Controller$(echo -e "\x1f")" "$nqn$RANDOM" 2>&1) && false +[[ $out == *"Invalid MN"* ]] + +# Attempt to create subsystem with invalid serial number - exceed SPDK_NVME_CTRLR_SN_LEN (20) +out=$("$rpc" nvmf_create_subsystem -s "$(gen_random_s 21)" "$nqn$RANDOM" 2>&1) && false +[[ $out == *"Invalid SN"* ]] + +# Attempt to create subsystem with invalid model - exceed SPDK_NVME_CTRLR_MN_LEN (40) +out=$("$rpc" nvmf_create_subsystem -d "$(gen_random_s 41)" "$nqn$RANDOM" 2>&1) && false +[[ $out == *"Invalid MN"* ]] + +# Attempt to delete non-existing target +out=$("$multi_target_rpc" nvmf_delete_target --name "$target" 2>&1) && false +[[ $out == *"The specified target doesn't exist, cannot delete it."* ]] + +trap - SIGINT SIGTERM EXIT +nvmftestfini diff --git a/src/spdk/test/nvmf/target/multiconnection.sh b/src/spdk/test/nvmf/target/multiconnection.sh new file mode 100755 index 000000000..d7e490861 --- /dev/null +++ b/src/spdk/test/nvmf/target/multiconnection.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash + +testdir=$(readlink -f $(dirname $0)) +rootdir=$(readlink -f $testdir/../../..) +source $rootdir/test/common/autotest_common.sh +source $rootdir/test/nvmf/common.sh + +MALLOC_BDEV_SIZE=64 +MALLOC_BLOCK_SIZE=512 + +NVMF_SUBSYS=11 + +rpc_py="$rootdir/scripts/rpc.py" + +nvmftestinit +nvmfappstart -m 0xF + +# SoftRoce does not have enough queues available for +# multiconnection tests. Detect if we're using software RDMA. +# If so - lower the number of subsystems for test. +if check_ip_is_soft_roce $NVMF_FIRST_TARGET_IP; then + echo "Using software RDMA, lowering number of NVMeOF subsystems." + NVMF_SUBSYS=1 +fi + +$rpc_py nvmf_create_transport $NVMF_TRANSPORT_OPTS -u 8192 + +for i in $(seq 1 $NVMF_SUBSYS); do + $rpc_py bdev_malloc_create $MALLOC_BDEV_SIZE $MALLOC_BLOCK_SIZE -b Malloc$i + $rpc_py nvmf_create_subsystem nqn.2016-06.io.spdk:cnode$i -a -s SPDK$i + $rpc_py nvmf_subsystem_add_ns nqn.2016-06.io.spdk:cnode$i Malloc$i + $rpc_py nvmf_subsystem_add_listener nqn.2016-06.io.spdk:cnode$i -t $TEST_TRANSPORT -a $NVMF_FIRST_TARGET_IP -s $NVMF_PORT +done + +for i in $(seq 1 $NVMF_SUBSYS); do + nvme connect -t $TEST_TRANSPORT -n "nqn.2016-06.io.spdk:cnode${i}" -a "$NVMF_FIRST_TARGET_IP" -s "$NVMF_PORT" + waitforserial SPDK$i +done + +$rootdir/scripts/fio.py -p nvmf -i 262144 -d 64 -t read -r 10 +$rootdir/scripts/fio.py -p nvmf -i 262144 -d 64 -t randwrite -r 10 + +sync +for i in $(seq 1 $NVMF_SUBSYS); do + nvme disconnect -n "nqn.2016-06.io.spdk:cnode${i}" || true + $rpc_py nvmf_delete_subsystem nqn.2016-06.io.spdk:cnode${i} +done + +rm -f ./local-job0-0-verify.state + +trap - SIGINT SIGTERM EXIT + +nvmftestfini diff --git a/src/spdk/test/nvmf/target/multitarget.sh b/src/spdk/test/nvmf/target/multitarget.sh new file mode 100755 index 000000000..4c3ece7c0 --- /dev/null +++ b/src/spdk/test/nvmf/target/multitarget.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash +testdir=$(readlink -f $(dirname $0)) +rootdir=$(readlink -f $testdir/../../..) +source $rootdir/test/common/autotest_common.sh +source $rootdir/test/nvmf/common.sh + +# For the time being this script is just menat to confirm the basic functionality of the +# multitarget RPCs as the in-tree applications don't support multi-target functionality. +rpc_py="$rootdir/test/nvmf/target/multitarget_rpc.py" + +nvmftestinit +nvmfappstart -m 0xF + +trap 'process_shm --id $NVMF_APP_SHM_ID; nvmftestfini $1; exit 1' SIGINT SIGTERM EXIT + +# Target application should start with a single target. +if [ "$($rpc_py nvmf_get_targets | jq 'length')" != "1" ]; then + echo "SPDK application did not start with the proper number of targets." && false +fi + +$rpc_py nvmf_create_target -n nvmf_tgt_1 -s 32 +$rpc_py nvmf_create_target -n nvmf_tgt_2 -s 32 + +if [ "$($rpc_py nvmf_get_targets | jq 'length')" != "3" ]; then + echo "nvmf_create_target RPC didn't properly create targets." && false +fi + +$rpc_py nvmf_delete_target -n nvmf_tgt_1 +$rpc_py nvmf_delete_target -n nvmf_tgt_2 + +if [ "$($rpc_py nvmf_get_targets | jq 'length')" != "1" ]; then + echo "nvmf_delete_target RPC didn't properly destroy targets." && false +fi + +trap - SIGINT SIGTERM EXIT + +nvmftestfini diff --git a/src/spdk/test/nvmf/target/multitarget_rpc.py b/src/spdk/test/nvmf/target/multitarget_rpc.py new file mode 100755 index 000000000..c5ccbcece --- /dev/null +++ b/src/spdk/test/nvmf/target/multitarget_rpc.py @@ -0,0 +1,84 @@ +#!/usr/bin/env python3 + +# Not for use in production. Please see the changelog for v19.10. + +from rpc.client import print_dict, JSONRPCException + +import logging +import argparse +import rpc +import sys +import shlex + +try: + from shlex import quote +except ImportError: + from pipes import quote + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description='NVMe-oF RPC command line interface. NOTE: spdk/scripts/ is expected in PYTHONPATH') + parser.add_argument('-s', dest='server_addr', + help='RPC domain socket path or IP address', default='/var/tmp/spdk.sock') + parser.add_argument('-p', dest='port', + help='RPC port number (if server_addr is IP address)', + default=5260, type=int) + parser.add_argument('-t', dest='timeout', + help='Timeout as a floating point number expressed in seconds waiting for response. Default: 60.0', + default=60.0, type=float) + parser.add_argument('-v', dest='verbose', action='store_const', const="INFO", + help='Set verbose mode to INFO', default="ERROR") + parser.add_argument('--verbose', dest='verbose', choices=['DEBUG', 'INFO', 'ERROR'], + help="""Set verbose level. """) + subparsers = parser.add_subparsers(help='RPC methods') + + def nvmf_create_target(args): + print_dict(rpc.nvmf.nvmf_create_target(args.client, + name=args.name, + max_subsystems=args.max_subsystems)) + + p = subparsers.add_parser('nvmf_create_target', help='Create a new NVMe-oF target') + p.add_argument('-n', '--name', help='Target name (unique to application)', type=str, required=True) + p.add_argument('-s', '--max-subsystems', help='Max number of NVMf subsystems defaults to SPDK_NVMF_DEFAULT_MAX_SUBSYSTEMS', + type=int, required=False) + p.set_defaults(func=nvmf_create_target) + + def nvmf_delete_target(args): + print_dict(rpc.nvmf.nvmf_delete_target(args.client, + name=args.name)) + + p = subparsers.add_parser('nvmf_delete_target', help='Destroy the given NVMe-oF Target') + p.add_argument('-n', '--name', help='Target name (unique to application)', type=str, required=True) + p.set_defaults(func=nvmf_delete_target) + + def nvmf_get_targets(args): + print_dict(rpc.nvmf.nvmf_get_targets(args.client)) + + p = subparsers.add_parser('nvmf_get_targets', help='Get the list of NVMe-oF Targets') + p.set_defaults(func=nvmf_get_targets) + + def call_rpc_func(args): + try: + args.func(args) + except JSONRPCException as ex: + print(ex.message) + exit(1) + + def execute_script(parser, client, fd): + for rpc_call in map(str.rstrip, fd): + if not rpc_call.strip(): + continue + args = parser.parse_args(shlex.split(rpc_call)) + args.client = client + call_rpc_func(args) + + args = parser.parse_args() + args.client = rpc.client.JSONRPCClient(args.server_addr, args.port, args.timeout, log_level=getattr(logging, args.verbose.upper())) + if hasattr(args, 'func'): + call_rpc_func(args) + elif sys.stdin.isatty(): + # No arguments and no data piped through stdin + parser.print_help() + exit(1) + else: + execute_script(parser, args.client, sys.stdin) diff --git a/src/spdk/test/nvmf/target/nmic.sh b/src/spdk/test/nvmf/target/nmic.sh new file mode 100755 index 000000000..6a967dc08 --- /dev/null +++ b/src/spdk/test/nvmf/target/nmic.sh @@ -0,0 +1,56 @@ +#!/usr/bin/env bash + +testdir=$(readlink -f $(dirname $0)) +rootdir=$(readlink -f $testdir/../../..) +source $rootdir/test/common/autotest_common.sh +source $rootdir/test/nvmf/common.sh + +MALLOC_BDEV_SIZE=64 +MALLOC_BLOCK_SIZE=512 + +rpc_py="$rootdir/scripts/rpc.py" + +nvmftestinit +nvmfappstart -m 0xF + +NVMF_SECOND_TARGET_IP=$(echo "$RDMA_IP_LIST" | sed -n 2p) + +$rpc_py nvmf_create_transport $NVMF_TRANSPORT_OPTS -u 8192 + +# Create subsystems +$rpc_py bdev_malloc_create $MALLOC_BDEV_SIZE $MALLOC_BLOCK_SIZE -b Malloc0 +$rpc_py nvmf_create_subsystem nqn.2016-06.io.spdk:cnode1 -a -s $NVMF_SERIAL +$rpc_py nvmf_subsystem_add_ns nqn.2016-06.io.spdk:cnode1 Malloc0 +$rpc_py nvmf_subsystem_add_listener nqn.2016-06.io.spdk:cnode1 -t $TEST_TRANSPORT -a $NVMF_FIRST_TARGET_IP -s "$NVMF_PORT" + +echo "test case1: single bdev can't be used in multiple subsystems" +$rpc_py nvmf_create_subsystem nqn.2016-06.io.spdk:cnode2 -a -s SPDK2 +$rpc_py nvmf_subsystem_add_listener nqn.2016-06.io.spdk:cnode2 -t $TEST_TRANSPORT -a $NVMF_FIRST_TARGET_IP -s "$NVMF_PORT" +nmic_status=0 +$rpc_py nvmf_subsystem_add_ns nqn.2016-06.io.spdk:cnode2 Malloc0 || nmic_status=$? + +if [ $nmic_status -eq 0 ]; then + echo " Adding namespace passed - failure expected." + nvmftestfini + exit 1 +else + echo " Adding namespace failed - expected result." +fi + +echo "test case2: host connect to nvmf target in multiple paths" +if [ -n "$NVMF_SECOND_TARGET_IP" ]; then + $rpc_py nvmf_subsystem_add_listener nqn.2016-06.io.spdk:cnode1 -t $TEST_TRANSPORT -a $NVMF_SECOND_TARGET_IP -s $NVMF_PORT + + nvme connect -t $TEST_TRANSPORT -n "nqn.2016-06.io.spdk:cnode1" -a "$NVMF_FIRST_TARGET_IP" -s "$NVMF_PORT" + nvme connect -t $TEST_TRANSPORT -n "nqn.2016-06.io.spdk:cnode1" -a "$NVMF_SECOND_TARGET_IP" -s "$NVMF_PORT" + + waitforserial "$NVMF_SERIAL" + + $rootdir/scripts/fio.py -p nvmf -i 4096 -d 1 -t write -r 1 -v +fi + +nvme disconnect -n "nqn.2016-06.io.spdk:cnode1" || true + +trap - SIGINT SIGTERM EXIT + +nvmftestfini diff --git a/src/spdk/test/nvmf/target/nvme_cli.sh b/src/spdk/test/nvmf/target/nvme_cli.sh new file mode 100755 index 000000000..29359689b --- /dev/null +++ b/src/spdk/test/nvmf/target/nvme_cli.sh @@ -0,0 +1,75 @@ +#!/usr/bin/env bash + +testdir=$(readlink -f $(dirname $0)) +rootdir=$(readlink -f $testdir/../../..) +source $rootdir/test/common/autotest_common.sh +source $rootdir/test/nvmf/common.sh + +if [ -z "${DEPENDENCY_DIR}" ]; then + echo DEPENDENCY_DIR not defined! + exit 1 +fi + +MALLOC_BDEV_SIZE=64 +MALLOC_BLOCK_SIZE=512 + +rpc_py="$rootdir/scripts/rpc.py" + +nvmftestinit +nvmfappstart -m 0xF + +$rpc_py nvmf_create_transport $NVMF_TRANSPORT_OPTS -u 8192 + +$rpc_py bdev_malloc_create $MALLOC_BDEV_SIZE $MALLOC_BLOCK_SIZE -b Malloc0 +$rpc_py bdev_malloc_create $MALLOC_BDEV_SIZE $MALLOC_BLOCK_SIZE -b Malloc1 + +$rpc_py nvmf_create_subsystem nqn.2016-06.io.spdk:cnode1 -a -s $NVMF_SERIAL -d SPDK_Controller1 +$rpc_py nvmf_subsystem_add_ns nqn.2016-06.io.spdk:cnode1 Malloc0 +$rpc_py nvmf_subsystem_add_ns nqn.2016-06.io.spdk:cnode1 Malloc1 +$rpc_py nvmf_subsystem_add_listener nqn.2016-06.io.spdk:cnode1 -t $TEST_TRANSPORT -a $NVMF_FIRST_TARGET_IP -s $NVMF_PORT + +nvme connect -t $TEST_TRANSPORT -n "nqn.2016-06.io.spdk:cnode1" -a "$NVMF_FIRST_TARGET_IP" -s "$NVMF_PORT" + +waitforserial $NVMF_SERIAL 2 +if ! get_nvme_devs print 2> /dev/null; then + echo "Could not find any nvme devices to work with, aborting the test" >&2 + exit 1 +fi + +for ctrl in "${nvmes[@]}"; do + nvme id-ctrl $ctrl + nvme smart-log $ctrl + nvme_model=$(nvme id-ctrl $ctrl | grep -w mn | sed 's/^.*: //' | sed 's/ *$//') + if [ "$nvme_model" != "SPDK_Controller1" ]; then + echo "Wrong model number for controller" $nvme_model + exit 1 + fi +done + +for ns in "${nvmes[@]}"; do + nvme id-ns $ns +done + +nvme disconnect -n "nqn.2016-06.io.spdk:cnode1" + +# Test spdk/nvme-cli NVMe-oF commands: discover, connect and disconnect +nvme_cli_build +pushd "${DEPENDENCY_DIR}/nvme-cli" + +sed -i 's/shm_id=.*/shm_id=-1/g' spdk.conf +./nvme discover -t $TEST_TRANSPORT -a $NVMF_FIRST_TARGET_IP -s "$NVMF_PORT" +nvme_num_before_connection=$(get_nvme_devs 2>&1 || echo 0) +./nvme connect -t $TEST_TRANSPORT -n "nqn.2016-06.io.spdk:cnode1" -a "$NVMF_FIRST_TARGET_IP" -s "$NVMF_PORT" +sleep 1 +nvme_num=$(get_nvme_devs 2>&1) +./nvme disconnect -n "nqn.2016-06.io.spdk:cnode1" +if [ $nvme_num -le $nvme_num_before_connection ]; then + echo "spdk/nvme-cli connect target devices failed" + exit 1 +fi +popd + +$rpc_py nvmf_delete_subsystem nqn.2016-06.io.spdk:cnode1 +trap - SIGINT SIGTERM EXIT + +nvmftestfini diff --git a/src/spdk/test/nvmf/target/nvmf_example.sh b/src/spdk/test/nvmf/target/nvmf_example.sh new file mode 100755 index 000000000..28045bc49 --- /dev/null +++ b/src/spdk/test/nvmf/target/nvmf_example.sh @@ -0,0 +1,59 @@ +#!/usr/bin/env bash + +testdir=$(readlink -f $(dirname $0)) +rootdir=$(readlink -f $testdir/../../..) +source $rootdir/test/common/autotest_common.sh +source $rootdir/test/nvmf/common.sh + +rpc_py="$rootdir/scripts/rpc.py" + +MALLOC_BDEV_SIZE=64 +MALLOC_BLOCK_SIZE=512 + +function build_nvmf_example_args() { + if [ $SPDK_RUN_NON_ROOT -eq 1 ]; then + echo "sudo -u $(logname) $SPDK_EXAMPLE_DIR/nvmf -i $NVMF_APP_SHM_ID" -g 10000 + else + echo "$SPDK_EXAMPLE_DIR/nvmf -i $NVMF_APP_SHM_ID" -g 10000 + fi +} + +NVMF_EXAMPLE="$(build_nvmf_example_args)" + +function nvmfexamplestart() { + timing_enter start_nvmf_example + $NVMF_EXAMPLE $1 & + nvmfpid=$! + trap 'process_shm --id $NVMF_APP_SHM_ID; nvmftestfini; exit 1' SIGINT SIGTERM EXIT + waitforlisten $nvmfpid + timing_exit start_nvmf_example +} + +timing_enter nvmf_example_test +nvmftestinit +nvmfexamplestart "-m 0xF" + +#create transport +$rpc_py nvmf_create_transport $NVMF_TRANSPORT_OPTS -u 8192 +#create malloc bdev +malloc_bdevs="$($rpc_py bdev_malloc_create $MALLOC_BDEV_SIZE $MALLOC_BLOCK_SIZE) " +#create subsystem +$rpc_py nvmf_create_subsystem nqn.2016-06.io.spdk:cnode1 -a -s SPDK00000000000001 + +#add ns to subsystem +for malloc_bdev in $malloc_bdevs; do + $rpc_py nvmf_subsystem_add_ns nqn.2016-06.io.spdk:cnode1 "$malloc_bdev" +done + +#add listener to subsystem +$rpc_py nvmf_subsystem_add_listener nqn.2016-06.io.spdk:cnode1 -t $TEST_TRANSPORT -a $NVMF_FIRST_TARGET_IP -s $NVMF_PORT + +perf="$SPDK_EXAMPLE_DIR/perf" + +$perf -q 64 -o 4096 -w randrw -M 30 -t 10 \ + -r "trtype:${TEST_TRANSPORT} adrfam:IPv4 traddr:${NVMF_FIRST_TARGET_IP} trsvcid:${NVMF_PORT} \ +subnqn:nqn.2016-06.io.spdk:cnode1" + +trap - SIGINT SIGTERM EXIT +nvmftestfini +timing_exit nvmf_example_test diff --git a/src/spdk/test/nvmf/target/nvmf_lvol.sh b/src/spdk/test/nvmf/target/nvmf_lvol.sh new file mode 100755 index 000000000..d44bc9332 --- /dev/null +++ b/src/spdk/test/nvmf/target/nvmf_lvol.sh @@ -0,0 +1,60 @@ +#!/usr/bin/env bash + +testdir=$(readlink -f $(dirname $0)) +rootdir=$(readlink -f $testdir/../../..) +source $rootdir/test/common/autotest_common.sh +source $rootdir/test/nvmf/common.sh + +MALLOC_BDEV_SIZE=64 +MALLOC_BLOCK_SIZE=512 +LVOL_BDEV_INIT_SIZE=20 +LVOL_BDEV_FINAL_SIZE=30 + +rpc_py="$rootdir/scripts/rpc.py" + +nvmftestinit +nvmfappstart -m 0x7 + +$rpc_py nvmf_create_transport $NVMF_TRANSPORT_OPTS -u 8192 + +# Construct a RAID volume for the logical volume store +base_bdevs="$($rpc_py bdev_malloc_create $MALLOC_BDEV_SIZE $MALLOC_BLOCK_SIZE) " +base_bdevs+=$($rpc_py bdev_malloc_create $MALLOC_BDEV_SIZE $MALLOC_BLOCK_SIZE) +$rpc_py bdev_raid_create -n raid0 -z 64 -r 0 -b "$base_bdevs" + +# Create the logical volume store on the RAID volume +lvs=$($rpc_py bdev_lvol_create_lvstore raid0 lvs) + +# Create a logical volume on the logical volume store +lvol=$($rpc_py bdev_lvol_create -u $lvs lvol $LVOL_BDEV_INIT_SIZE) + +# Create an NVMe-oF subsystem and add the logical volume as a namespace +$rpc_py nvmf_create_subsystem nqn.2016-06.io.spdk:cnode0 -a -s SPDK0 +$rpc_py nvmf_subsystem_add_ns nqn.2016-06.io.spdk:cnode0 $lvol +$rpc_py nvmf_subsystem_add_listener nqn.2016-06.io.spdk:cnode0 -t $TEST_TRANSPORT -a $NVMF_FIRST_TARGET_IP -s $NVMF_PORT + +# Start random writes in the background +$SPDK_EXAMPLE_DIR/perf -r "trtype:$TEST_TRANSPORT adrfam:IPv4 traddr:$NVMF_FIRST_TARGET_IP trsvcid:$NVMF_PORT" -o 4096 -q 128 -s 512 -w randwrite -t 10 -c 0x18 & +perf_pid=$! + +sleep 1 + +# Perform some operations on the logical volume +snapshot=$($rpc_py bdev_lvol_snapshot $lvol "MY_SNAPSHOT") +$rpc_py bdev_lvol_resize $lvol $LVOL_BDEV_FINAL_SIZE +clone=$($rpc_py bdev_lvol_clone $snapshot "MY_CLONE") +$rpc_py bdev_lvol_inflate $clone + +# Wait for I/O to complete +wait $perf_pid + +# Clean up +$rpc_py nvmf_delete_subsystem nqn.2016-06.io.spdk:cnode0 +$rpc_py bdev_lvol_delete $lvol +$rpc_py bdev_lvol_delete_lvstore -u $lvs + +rm -f ./local-job* + +trap - SIGINT SIGTERM EXIT + +nvmftestfini diff --git a/src/spdk/test/nvmf/target/nvmf_vhost.sh b/src/spdk/test/nvmf/target/nvmf_vhost.sh new file mode 100755 index 000000000..48e78d6d2 --- /dev/null +++ b/src/spdk/test/nvmf/target/nvmf_vhost.sh @@ -0,0 +1,69 @@ +#!/usr/bin/env bash + +testdir=$(readlink -f $(dirname $0)) +rootdir=$testdir/../../.. +source $rootdir/test/common/autotest_common.sh +source $rootdir/test/nvmf/common.sh + +if [ ! -d "/usr/local/qemu/spdk-3.0.0" ]; then + echo "Qemu not installed on this machine. It may be a VM. Skipping nvmf_vhost test." + exit 0 +fi + +source $rootdir/test/vhost/common.sh + +MALLOC_BDEV_SIZE=128 +MALLOC_BLOCK_SIZE=512 +NVMF_SOCK="/tmp/nvmf_rpc.sock" +NVMF_RPC="$rootdir/scripts/rpc.py -s $NVMF_SOCK" + +VHOST_SOCK="/tmp/vhost_rpc.sock" +VHOST_APP+=(-p 0 -r "$VHOST_SOCK" -u) +VHOST_RPC="$rootdir/scripts/rpc.py -s $VHOST_SOCK" + +nvmftestinit + +# Start Apps +"${NVMF_APP[@]}" -r $NVMF_SOCK & +nvmfpid=$! +waitforlisten $nvmfpid $NVMF_SOCK + +trap 'process_shm --id $NVMF_APP_SHM_ID; nvmftestfini; exit 1' SIGINT SIGTERM EXIT + +mkdir -p "$(get_vhost_dir 3)" + +"${VHOST_APP[@]}" -S "$(get_vhost_dir 3)" & +vhostpid=$! +waitforlisten $vhostpid $NVMF_SOCK + +trap 'process_shm --id $NVMF_APP_SHM_ID; killprocess $vhostpid nvmftestfini; exit 1' SIGINT SIGTERM EXIT + +# Configure NVMF tgt on host machine +malloc_bdev="$($NVMF_RPC bdev_malloc_create $MALLOC_BDEV_SIZE $MALLOC_BLOCK_SIZE)" + +$NVMF_RPC nvmf_create_transport $NVMF_TRANSPORT_OPTS -u 8192 -p 4 +$NVMF_RPC nvmf_create_subsystem nqn.2016-06.io.spdk:cnode1 -a -s SPDK00000000000001 +$NVMF_RPC nvmf_subsystem_add_ns nqn.2016-06.io.spdk:cnode1 "$malloc_bdev" +$NVMF_RPC nvmf_subsystem_add_listener nqn.2016-06.io.spdk:cnode1 -t $TEST_TRANSPORT -a $NVMF_FIRST_TARGET_IP -s $NVMF_PORT + +# Configure VHost on host machine +$VHOST_RPC bdev_nvme_attach_controller -b Nvme0 -t $TEST_TRANSPORT -f ipv4 -a $NVMF_FIRST_TARGET_IP -s $NVMF_PORT -n nqn.2016-06.io.spdk:cnode1 +$VHOST_RPC vhost_create_scsi_controller naa.VhostScsi0.3 +$VHOST_RPC vhost_scsi_controller_add_target naa.VhostScsi0.3 0 "Nvme0n1" + +# start qemu based VM. +vm_setup --os="$VM_IMAGE" --disk-type=spdk_vhost_scsi --disks="VhostScsi0" --force=3 --vhost-name=3 + +vm_run 3 + +vm_wait_for_boot 300 3 + +# Run the fio workload remotely +vm_scp 3 $testdir/nvmf_vhost_fio.job 127.0.0.1:/root/nvmf_vhost_fio.job +vm_exec 3 "fio /root/nvmf_vhost_fio.job" +vm_shutdown_all + +trap - SIGINT SIGTERM EXIT + +killprocess $vhostpid +nvmftestfini diff --git a/src/spdk/test/nvmf/target/nvmf_vhost_fio.job b/src/spdk/test/nvmf/target/nvmf_vhost_fio.job new file mode 100644 index 000000000..350aa895e --- /dev/null +++ b/src/spdk/test/nvmf/target/nvmf_vhost_fio.job @@ -0,0 +1,19 @@ +[global] +blocksize=4k-512k +iodepth=128 +ioengine=libaio +filename=/dev/sdb +group_reporting +thread +numjobs=1 +direct=1 +do_verify=1 +verify=md5 +verify_fatal=1 +verify_dump=1 +verify_backlog=8 + +[randwrite] +rw=randwrite +runtime=15 +time_based diff --git a/src/spdk/test/nvmf/target/rpc.sh b/src/spdk/test/nvmf/target/rpc.sh new file mode 100755 index 000000000..d715e4b4f --- /dev/null +++ b/src/spdk/test/nvmf/target/rpc.sh @@ -0,0 +1,124 @@ +#!/usr/bin/env bash + +testdir=$(readlink -f $(dirname $0)) +rootdir=$(readlink -f $testdir/../../..) +source $rootdir/test/common/autotest_common.sh +source $rootdir/test/nvmf/common.sh + +rpc_py="$rootdir/scripts/rpc.py" + +function jcount() { + local filter=$1 + jq "$filter" | wc -l +} + +function jsum() { + local filter=$1 + jq "$filter" | awk '{s+=$1}END{print s}' +} + +nvmftestinit +nvmfappstart -m 0xF + +stats=$($rpc_py nvmf_get_stats) +# Expect 4 poll groups (from CPU mask) and no transports yet +[ "4" -eq $(jcount .poll_groups[].name <<< "$stats") ] +[ "null" == $(jq .poll_groups[0].transports[0] <<< "$stats") ] + +$rpc_py nvmf_create_transport $NVMF_TRANSPORT_OPTS -u 8192 + +stats=$($rpc_py nvmf_get_stats) +# Expect no QPs +[ "0" -eq $(jsum .poll_groups[].admin_qpairs <<< "$stats") ] +[ "0" -eq $(jsum .poll_groups[].io_qpairs <<< "$stats") ] +# Transport statistics is currently implemented for RDMA only +if [ 'rdma' == $TEST_TRANSPORT ]; then + # Expect RDMA transport and some devices + [ "1" -eq $(jcount .poll_groups[0].transports[].trtype <<< "$stats") ] + transport_type=$(jq -r .poll_groups[0].transports[0].trtype <<< "$stats") + [ "${transport_type,,}" == "${TEST_TRANSPORT,,}" ] + [ "0" -lt $(jcount .poll_groups[0].transports[0].devices[].name <<< "$stats") ] +fi + +# set times for subsystem construct/delete +if [ $RUN_NIGHTLY -eq 1 ]; then + times=50 +else + times=3 +fi + +MALLOC_BDEV_SIZE=64 +MALLOC_BLOCK_SIZE=512 + +$rpc_py bdev_malloc_create $MALLOC_BDEV_SIZE $MALLOC_BLOCK_SIZE -b Malloc1 + +# Disallow host NQN and make sure connect fails +$rpc_py nvmf_create_subsystem nqn.2016-06.io.spdk:cnode1 -a -s $NVMF_SERIAL +$rpc_py nvmf_subsystem_add_ns nqn.2016-06.io.spdk:cnode1 Malloc1 +$rpc_py nvmf_subsystem_allow_any_host -d nqn.2016-06.io.spdk:cnode1 +$rpc_py nvmf_subsystem_add_listener nqn.2016-06.io.spdk:cnode1 -t $TEST_TRANSPORT -a $NVMF_FIRST_TARGET_IP -s $NVMF_PORT + +# This connect should fail - the host NQN is not allowed +! nvme connect -t $TEST_TRANSPORT -n nqn.2016-06.io.spdk:cnode1 -q nqn.2016-06.io.spdk:host1 -a "$NVMF_FIRST_TARGET_IP" -s "$NVMF_PORT" + +# Add the host NQN and verify that the connect succeeds +$rpc_py nvmf_subsystem_add_host nqn.2016-06.io.spdk:cnode1 nqn.2016-06.io.spdk:host1 +nvme connect -t $TEST_TRANSPORT -n nqn.2016-06.io.spdk:cnode1 -q nqn.2016-06.io.spdk:host1 -a "$NVMF_FIRST_TARGET_IP" -s "$NVMF_PORT" +waitforserial "$NVMF_SERIAL" +nvme disconnect -n nqn.2016-06.io.spdk:cnode1 + +# Remove the host and verify that the connect fails +$rpc_py nvmf_subsystem_remove_host nqn.2016-06.io.spdk:cnode1 nqn.2016-06.io.spdk:host1 +! nvme connect -t $TEST_TRANSPORT -n nqn.2016-06.io.spdk:cnode1 -q nqn.2016-06.io.spdk:host1 -a "$NVMF_FIRST_TARGET_IP" -s "$NVMF_PORT" + +# Allow any host and verify that the connect succeeds +$rpc_py nvmf_subsystem_allow_any_host -e nqn.2016-06.io.spdk:cnode1 +nvme connect -t $TEST_TRANSPORT -n nqn.2016-06.io.spdk:cnode1 -q nqn.2016-06.io.spdk:host1 -a "$NVMF_FIRST_TARGET_IP" -s "$NVMF_PORT" +waitforserial "$NVMF_SERIAL" +nvme disconnect -n nqn.2016-06.io.spdk:cnode1 + +$rpc_py nvmf_delete_subsystem nqn.2016-06.io.spdk:cnode1 + +# do frequent add delete of namespaces with different nsid. +for i in $(seq 1 $times); do + $rpc_py nvmf_create_subsystem nqn.2016-06.io.spdk:cnode1 -s $NVMF_SERIAL + $rpc_py nvmf_subsystem_add_listener nqn.2016-06.io.spdk:cnode1 -t $TEST_TRANSPORT -a $NVMF_FIRST_TARGET_IP -s $NVMF_PORT + $rpc_py nvmf_subsystem_add_ns nqn.2016-06.io.spdk:cnode1 Malloc1 -n 5 + $rpc_py nvmf_subsystem_allow_any_host nqn.2016-06.io.spdk:cnode1 + nvme connect -t $TEST_TRANSPORT -n nqn.2016-06.io.spdk:cnode1 -a "$NVMF_FIRST_TARGET_IP" -s "$NVMF_PORT" + + waitforserial "$NVMF_SERIAL" + + nvme disconnect -n nqn.2016-06.io.spdk:cnode1 + + $rpc_py nvmf_subsystem_remove_ns nqn.2016-06.io.spdk:cnode1 5 + $rpc_py nvmf_delete_subsystem nqn.2016-06.io.spdk:cnode1 + +done + +# do frequent add delete. +for i in $(seq 1 $times); do + $rpc_py nvmf_create_subsystem nqn.2016-06.io.spdk:cnode1 -s $NVMF_SERIAL + $rpc_py nvmf_subsystem_add_listener nqn.2016-06.io.spdk:cnode1 -t $TEST_TRANSPORT -a $NVMF_FIRST_TARGET_IP -s $NVMF_PORT + $rpc_py nvmf_subsystem_add_ns nqn.2016-06.io.spdk:cnode1 Malloc1 + $rpc_py nvmf_subsystem_allow_any_host nqn.2016-06.io.spdk:cnode1 + + $rpc_py nvmf_subsystem_remove_ns nqn.2016-06.io.spdk:cnode1 1 + + $rpc_py nvmf_delete_subsystem nqn.2016-06.io.spdk:cnode1 +done + +stats=$($rpc_py nvmf_get_stats) +# Expect some admin and IO qpairs +[ "0" -lt $(jsum .poll_groups[].admin_qpairs <<< "$stats") ] +[ "0" -lt $(jsum .poll_groups[].io_qpairs <<< "$stats") ] +# Transport statistics is currently implemented for RDMA only +if [ 'rdma' == $TEST_TRANSPORT ]; then + # Expect non-zero completions and request latencies accumulated + [ "0" -lt $(jsum .poll_groups[].transports[].devices[].completions <<< "$stats") ] + [ "0" -lt $(jsum .poll_groups[].transports[].devices[].request_latency <<< "$stats") ] +fi + +trap - SIGINT SIGTERM EXIT + +nvmftestfini diff --git a/src/spdk/test/nvmf/target/shutdown.sh b/src/spdk/test/nvmf/target/shutdown.sh new file mode 100755 index 000000000..8ad73bd6f --- /dev/null +++ b/src/spdk/test/nvmf/target/shutdown.sh @@ -0,0 +1,155 @@ +#!/usr/bin/env bash + +testdir=$(readlink -f $(dirname $0)) +rootdir=$(readlink -f $testdir/../../..) +source $rootdir/test/common/autotest_common.sh +source $rootdir/test/nvmf/common.sh + +MALLOC_BDEV_SIZE=64 +MALLOC_BLOCK_SIZE=512 + +rpc_py="$rootdir/scripts/rpc.py" + +function starttarget() { + # Start the target + nvmfappstart -m 0x1E + + $rpc_py nvmf_create_transport $NVMF_TRANSPORT_OPTS -u 8192 + + num_subsystems=({1..10}) + # SoftRoce does not have enough queues available for + # this test. Detect if we're using software RDMA. + # If so, only use two subsystem. + if check_ip_is_soft_roce "$NVMF_FIRST_TARGET_IP"; then + num_subsystems=({1..2}) + fi + + timing_enter create_subsystems + # Create subsystems + rm -rf $testdir/rpcs.txt + for i in "${num_subsystems[@]}"; do + cat <<- EOL >> $testdir/rpcs.txt + bdev_malloc_create $MALLOC_BDEV_SIZE $MALLOC_BLOCK_SIZE -b Malloc$i + nvmf_create_subsystem nqn.2016-06.io.spdk:cnode$i -a -s SPDK$i + nvmf_subsystem_add_ns nqn.2016-06.io.spdk:cnode$i Malloc$i + nvmf_subsystem_add_listener nqn.2016-06.io.spdk:cnode$i -t $TEST_TRANSPORT -a $NVMF_FIRST_TARGET_IP -s $NVMF_PORT + EOL + done + $rpc_py < $testdir/rpcs.txt + timing_exit create_subsystems + +} + +function stoptarget() { + rm -f ./local-job0-0-verify.state + rm -rf $testdir/bdevperf.conf + rm -rf $testdir/rpcs.txt + + nvmftestfini +} + +function waitforio() { + # $1 = RPC socket + if [ -z "$1" ]; then + exit 1 + fi + # $2 = bdev name + if [ -z "$2" ]; then + exit 1 + fi + local ret=1 + local i + for ((i = 10; i != 0; i--)); do + read_io_count=$($rpc_py -s $1 bdev_get_iostat -b $2 | jq -r '.bdevs[0].num_read_ops') + # A few I/O will happen during initial examine. So wait until at least 100 I/O + # have completed to know that bdevperf is really generating the I/O. + if [ $read_io_count -ge 100 ]; then + ret=0 + break + fi + sleep 0.25 + done + return $ret +} + +# Test 1: Kill the initiator unexpectedly with no I/O outstanding +function nvmf_shutdown_tc1() { + starttarget + + # Run bdev_svc, which connects but does not issue I/O + $rootdir/test/app/bdev_svc/bdev_svc -m 0x1 -i 1 -r /var/tmp/bdevperf.sock --json <(gen_nvmf_target_json "${num_subsystems[@]}") & + perfpid=$! + waitforlisten $perfpid /var/tmp/bdevperf.sock + $rpc_py -s /var/tmp/bdevperf.sock framework_wait_init + + # Kill bdev_svc + kill -9 $perfpid || true + rm -f /var/run/spdk_bdev1 + + # Verify the target stays up + sleep 1 + kill -0 $nvmfpid + + # Connect with bdevperf and confirm it works + $rootdir/test/bdev/bdevperf/bdevperf -r /var/tmp/bdevperf.sock --json <(gen_nvmf_target_json "${num_subsystems[@]}") -q 64 -o 65536 -w verify -t 1 + + stoptarget +} + +# Test 2: Kill initiator unexpectedly with I/O outstanding +function nvmf_shutdown_tc2() { + starttarget + + # Run bdevperf + $rootdir/test/bdev/bdevperf/bdevperf -r /var/tmp/bdevperf.sock --json <(gen_nvmf_target_json "${num_subsystems[@]}") -q 64 -o 65536 -w verify -t 10 & + perfpid=$! + waitforlisten $perfpid /var/tmp/bdevperf.sock + $rpc_py -s /var/tmp/bdevperf.sock framework_wait_init + + waitforio /var/tmp/bdevperf.sock Nvme1n1 + + # Kill bdevperf half way through + killprocess $perfpid + + # Verify the target stays up + sleep 1 + kill -0 $nvmfpid + + stoptarget +} + +# Test 3: Kill the target unexpectedly with I/O outstanding +function nvmf_shutdown_tc3() { + starttarget + + # Run bdevperf + $rootdir/test/bdev/bdevperf/bdevperf -r /var/tmp/bdevperf.sock --json <(gen_nvmf_target_json "${num_subsystems[@]}") -q 64 -o 65536 -w verify -t 10 & + perfpid=$! + waitforlisten $perfpid /var/tmp/bdevperf.sock + $rpc_py -s /var/tmp/bdevperf.sock framework_wait_init + + # Expand the trap to clean up bdevperf if something goes wrong + trap 'process_shm --id $NVMF_APP_SHM_ID; kill -9 $perfpid || true; nvmftestfini; exit 1' SIGINT SIGTERM EXIT + + waitforio /var/tmp/bdevperf.sock Nvme1n1 + + # Kill the target half way through + killprocess $nvmfpid + nvmfpid= + + # Verify bdevperf exits successfully + sleep 1 + # TODO: Right now the NVMe-oF initiator will not correctly detect broken connections + # and so it will never shut down. Just kill it. + kill -9 $perfpid || true + + stoptarget +} + +nvmftestinit + +run_test "nvmf_shutdown_tc1" nvmf_shutdown_tc1 +run_test "nvmf_shutdown_tc2" nvmf_shutdown_tc2 +run_test "nvmf_shutdown_tc3" nvmf_shutdown_tc3 + +trap - SIGINT SIGTERM EXIT diff --git a/src/spdk/test/nvmf/target/srq_overwhelm.sh b/src/spdk/test/nvmf/target/srq_overwhelm.sh new file mode 100755 index 000000000..fe4dd7d29 --- /dev/null +++ b/src/spdk/test/nvmf/target/srq_overwhelm.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash + +testdir=$(readlink -f $(dirname $0)) +rootdir=$(readlink -f $testdir/../../..) +source $rootdir/test/common/autotest_common.sh +source $rootdir/test/nvmf/common.sh + +MALLOC_BDEV_SIZE=64 +MALLOC_BLOCK_SIZE=512 + +rpc_py="$rootdir/scripts/rpc.py" + +nvmftestinit + +if check_ip_is_soft_roce $NVMF_FIRST_TARGET_IP; then + echo "Using software RDMA, Likely not enough memory to run this test. aborting." + exit 0 +fi + +nvmfappstart -m 0xF + +# create the rdma transport with an intentionally small SRQ depth +$rpc_py nvmf_create_transport $NVMF_TRANSPORT_OPTS -u 8192 -s 1024 + +for i in $(seq 0 5); do + $rpc_py nvmf_create_subsystem nqn.2016-06.io.spdk:cnode$i -a -s SPDK00000000000001 + $rpc_py bdev_malloc_create $MALLOC_BDEV_SIZE $MALLOC_BLOCK_SIZE -b Malloc$i + $rpc_py nvmf_subsystem_add_ns nqn.2016-06.io.spdk:cnode$i Malloc$i + $rpc_py nvmf_subsystem_add_listener nqn.2016-06.io.spdk:cnode$i -t $TEST_TRANSPORT -a $NVMF_FIRST_TARGET_IP -s $NVMF_PORT + nvme connect -t $TEST_TRANSPORT -n "nqn.2016-06.io.spdk:cnode${i}" -a "$NVMF_FIRST_TARGET_IP" -s "$NVMF_PORT" -i 16 + waitforblk "nvme${i}n1" +done + +# by running 6 different FIO jobs, each with 13 subjobs, we end up with 78 fio threads trying to write to +# our target at once. This completely overwhelms the target SRQ, but allows us to verify that rnr_retry is +# working even at very high queue depths because the rdma qpair doesn't fail. +# It is normal to see the initiator timeout and reconnect waiting for completions from an overwhelmmed target, +# but the connection should come up and FIO should complete without errors. +$rootdir/scripts/fio.py -p nvmf -i 1048576 -d 128 -t read -r 10 -n 13 + +sync + +for i in $(seq 0 5); do + nvme disconnect -n "nqn.2016-06.io.spdk:cnode${i}" + $rpc_py nvmf_delete_subsystem nqn.2016-06.io.spdk:cnode$i +done + +trap - SIGINT SIGTERM EXIT + +nvmftestfini |