diff options
Diffstat (limited to 'src/spdk/test/vhost/lvol/lvol_test.sh')
| -rwxr-xr-x | src/spdk/test/vhost/lvol/lvol_test.sh | 286 |
1 files changed, 286 insertions, 0 deletions
diff --git a/src/spdk/test/vhost/lvol/lvol_test.sh b/src/spdk/test/vhost/lvol/lvol_test.sh new file mode 100755 index 00000000..5190b5f2 --- /dev/null +++ b/src/spdk/test/vhost/lvol/lvol_test.sh @@ -0,0 +1,286 @@ +#!/usr/bin/env bash +set -e + +rootdir=$(readlink -f $(dirname $0))/../../.. +source "$rootdir/scripts/common.sh" + +LVOL_TEST_DIR=$(readlink -f $(dirname $0)) +[[ -z "$TEST_DIR" ]] && TEST_DIR="$(cd $LVOL_TEST_DIR/../../../../ && pwd)" +[[ -z "$COMMON_DIR" ]] && COMMON_DIR="$(cd $LVOL_TEST_DIR/../common && pwd)" + +. $COMMON_DIR/common.sh +rpc_py="$SPDK_BUILD_DIR/scripts/rpc.py -s $(get_vhost_dir)/rpc.sock" + +vm_count=1 +max_disks="" +ctrl_type="spdk_vhost_scsi" +use_fs=false +nested_lvol=false +distribute_cores=false + +function usage() +{ + [[ ! -z $2 ]] && ( echo "$2"; echo ""; ) + echo "Shortcut script for doing automated test" + echo "Usage: $(basename $1) [OPTIONS]" + echo + echo "-h, --help Print help and exit" + echo " --fio-bin=PATH Path to FIO binary.;" + echo " --vm-count=INT Virtual machines to use in test;" + echo " Each VM will get one lvol bdev on each NVMe." + echo " Default: 1" + echo " --max-disks=INT Maximum number of NVMe drives to use in test." + echo " Default: will use all available NVMes." + echo " --ctrl-type=TYPE Controller type to use for test:" + echo " spdk_vhost_scsi - use spdk vhost scsi" + echo " spdk_vhost_blk - use spdk vhost block" + echo " --nested-lvol If enabled will create additional lvol bdev" + echo " on each NVMe for use as base device for next" + echo " lvol store and lvol bdevs." + echo " (NVMe->lvol_store->lvol_bdev->lvol_store->lvol_bdev)" + echo " Default: False" + echo " --thin-provisioning Create lvol bdevs thin provisioned instead of" + echo " allocating space up front" + echo " --distribute-cores Use custom config file and run vhost controllers" + echo " on different CPU cores instead of single core." + echo " Default: False" + echo "-x set -x for script debug" + echo " --multi-os Run tests on different os types in VMs" + echo " Default: False" + exit 0 +} + +function clean_lvol_cfg() +{ + notice "Removing nested lvol bdevs" + for lvol_bdev in "${nest_lvol_bdevs[@]}"; do + $rpc_py destroy_lvol_bdev $lvol_bdev + notice "nested lvol bdev $lvol_bdev removed" + done + + notice "Removing nested lvol stores" + for lvol_store in "${nest_lvol_stores[@]}"; do + $rpc_py destroy_lvol_store -u $lvol_store + notice "nested lvol store $lvol_store removed" + done + + notice "Removing lvol bdevs" + for lvol_bdev in "${lvol_bdevs[@]}"; do + $rpc_py destroy_lvol_bdev $lvol_bdev + notice "lvol bdev $lvol_bdev removed" + done + + notice "Removing lvol stores" + for lvol_store in "${lvol_stores[@]}"; do + $rpc_py destroy_lvol_store -u $lvol_store + notice "lvol store $lvol_store removed" + done +} + +while getopts 'xh-:' optchar; do + case "$optchar" in + -) + case "$OPTARG" in + help) usage $0 ;; + fio-bin=*) fio_bin="--fio-bin=${OPTARG#*=}" ;; + vm-count=*) vm_count="${OPTARG#*=}" ;; + max-disks=*) max_disks="${OPTARG#*=}" ;; + ctrl-type=*) ctrl_type="${OPTARG#*=}" ;; + nested-lvol) nested_lvol=true ;; + distribute-cores) distribute_cores=true ;; + thin-provisioning) thin=" -t " ;; + multi-os) multi_os=true ;; + *) usage $0 "Invalid argument '$OPTARG'" ;; + esac + ;; + h) usage $0 ;; + x) set -x + x="-x" ;; + *) usage $0 "Invalid argument '$OPTARG'" + esac +done + +notice "Get NVMe disks:" +nvmes=($(iter_pci_class_code 01 08 02)) + +if [[ -z $max_disks ]]; then + max_disks=${#nvmes[@]} +fi + +if [[ ${#nvmes[@]} -lt max_disks ]]; then + fail "Number of NVMe drives (${#nvmes[@]}) is lower than number of requested disks for test ($max_disks)" +fi + +if $distribute_cores; then + # FIXME: this need to be handled entirely in common.sh + source $LVOL_TEST_DIR/autotest.config +fi + +trap 'error_exit "${FUNCNAME}" "${LINENO}"' SIGTERM SIGABRT ERR + +vm_kill_all + +notice "running SPDK vhost" +spdk_vhost_run +notice "..." + +trap 'clean_lvol_cfg; error_exit "${FUNCNAME}" "${LINENO}"' SIGTERM SIGABRT ERR + +lvol_stores=() +lvol_bdevs=() +nest_lvol_stores=() +nest_lvol_bdevs=() +used_vms="" + +# On each NVMe create one lvol store +for (( i=0; i<$max_disks; i++ ));do + + # Create base lvol store on NVMe + notice "Creating lvol store on device Nvme${i}n1" + ls_guid=$($rpc_py construct_lvol_store Nvme${i}n1 lvs_$i -c 4194304) + lvol_stores+=("$ls_guid") + + if $nested_lvol; then + free_mb=$(get_lvs_free_mb "$ls_guid") + size=$((free_mb / (vm_count+1) )) + + notice "Creating lvol bdev on lvol store: $ls_guid" + lb_name=$($rpc_py construct_lvol_bdev -u $ls_guid lbd_nest $size $thin) + + notice "Creating nested lvol store on lvol bdev: $lb_name" + nest_ls_guid=$($rpc_py construct_lvol_store $lb_name lvs_n_$i -c 4194304) + nest_lvol_stores+=("$nest_ls_guid") + + for (( j=0; j<$vm_count; j++)); do + notice "Creating nested lvol bdev for VM $i on lvol store $nest_ls_guid" + free_mb=$(get_lvs_free_mb "$nest_ls_guid") + nest_size=$((free_mb / (vm_count-j) )) + lb_name=$($rpc_py construct_lvol_bdev -u $nest_ls_guid lbd_vm_$j $nest_size $thin) + nest_lvol_bdevs+=("$lb_name") + done + fi + + # Create base lvol bdevs + for (( j=0; j<$vm_count; j++)); do + notice "Creating lvol bdev for VM $i on lvol store $ls_guid" + free_mb=$(get_lvs_free_mb "$ls_guid") + size=$((free_mb / (vm_count-j) )) + lb_name=$($rpc_py construct_lvol_bdev -u $ls_guid lbd_vm_$j $size $thin) + lvol_bdevs+=("$lb_name") + done +done + +bdev_info=$($rpc_py get_bdevs) +notice "Configuration after initial set-up:" +$rpc_py get_lvol_stores +echo "$bdev_info" + +# Set up VMs +for (( i=0; i<$vm_count; i++)); do + vm="vm_$i" + + # Get all lvol bdevs associated with this VM number + bdevs=$(jq -r "map(select(.aliases[] | contains(\"$vm\")) | \ + .aliases[]) | join(\" \")" <<< "$bdev_info") + bdevs=($bdevs) + + setup_cmd="vm_setup --disk-type=$ctrl_type --force=$i" + if [[ $i%2 -ne 0 ]] && [[ $multi_os ]]; then + setup_cmd+=" --os=/home/sys_sgsw/spdk_vhost_CentOS_vm_image.qcow2" + else + setup_cmd+=" --os=/home/sys_sgsw/vhost_vm_image.qcow2" + fi + + # Create single SCSI controller or multiple BLK controllers for this VM + if $distribute_cores; then + mask="VM_${i}_qemu_mask" + mask_arg="--cpumask ${!mask}" + fi + + if [[ "$ctrl_type" == "spdk_vhost_scsi" ]]; then + $rpc_py construct_vhost_scsi_controller naa.0.$i $mask_arg + for (( j=0; j<${#bdevs[@]}; j++)); do + $rpc_py add_vhost_scsi_lun naa.0.$i $j ${bdevs[$j]} + done + setup_cmd+=" --disks=0" + elif [[ "$ctrl_type" == "spdk_vhost_blk" ]]; then + disk="" + for (( j=0; j<${#bdevs[@]}; j++)); do + $rpc_py construct_vhost_blk_controller naa.$j.$i ${bdevs[$j]} $mask_arg + disk+="${j}:" + done + disk="${disk::-1}" + setup_cmd+=" --disks=$disk" + fi + + $setup_cmd + used_vms+=" $i" +done + +$rpc_py get_vhost_controllers + +# Run VMs +vm_run $used_vms +vm_wait_for_boot 600 $used_vms + +# Get disk names from VMs and run FIO traffic + +fio_disks="" +for vm_num in $used_vms; do + vm_dir=$VM_BASE_DIR/$vm_num + qemu_mask_param="VM_${vm_num}_qemu_mask" + + host_name="VM-$vm_num-${!qemu_mask_param}" + vm_ssh $vm_num "hostname $host_name" + vm_start_fio_server $fio_bin $vm_num + + if [[ "$ctrl_type" == "spdk_vhost_scsi" ]]; then + vm_check_scsi_location $vm_num + elif [[ "$ctrl_type" == "spdk_vhost_blk" ]]; then + vm_check_blk_location $vm_num + fi + + fio_disks+=" --vm=${vm_num}$(printf ':/dev/%s' $SCSI_DISK)" +done + +if [[ $RUN_NIGHTLY -eq 1 ]]; then + job_file="default_integrity_nightly.job" +else + job_file="default_integrity.job" +fi +# Run FIO traffic +run_fio $fio_bin --job-file=$COMMON_DIR/fio_jobs/$job_file --out="$TEST_DIR/fio_results" $fio_disks + +notice "Shutting down virtual machines..." +vm_shutdown_all +sleep 2 + +notice "Cleaning up vhost - remove LUNs, controllers, lvol bdevs and lvol stores" +if [[ "$ctrl_type" == "spdk_vhost_scsi" ]]; then + for (( i=0; i<$vm_count; i++)); do + notice "Removing devices from vhost SCSI controller naa.0.$i" + for (( j=0; j<${#bdevs[@]}; j++)); do + $rpc_py remove_vhost_scsi_target naa.0.$i $j + notice "Removed device $j" + done + notice "Removing vhost SCSI controller naa.0.$i" + $rpc_py remove_vhost_controller naa.0.$i + done +elif [[ "$ctrl_type" == "spdk_vhost_blk" ]]; then + for (( i=0; i<$vm_count; i++)); do + for (( j=0; j<${#bdevs[@]}; j++)); do + notice "Removing vhost BLK controller naa.$j.$i" + $rpc_py remove_vhost_controller naa.$j.$i + notice "Removed naa.$j.$i" + done + done +fi + +clean_lvol_cfg + +$rpc_py get_lvol_stores +$rpc_py get_bdevs +$rpc_py get_vhost_controllers + +notice "Shutting down SPDK vhost app..." +spdk_vhost_kill |