Adding upstream version 14.2.21.upstream/14.2.21upstream

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

9 files changed, 870 insertions, 0 deletions

diff --git a/src/spdk/test/vhost/migration/autotest.config b/src/spdk/test/vhost/migration/autotest.config
new file mode 100644
index 00000000..ccda306e
--- /dev/null
+++ b/src/spdk/test/vhost/migration/autotest.config
@@ -0,0 +1,14 @@
+vhost_0_reactor_mask=["0"]
+vhost_0_master_core=0
+
+vhost_1_reactor_mask=["0"]
+vhost_1_master_core=0
+
+VM_0_qemu_mask=1
+VM_0_qemu_numa_node=0
+
+VM_1_qemu_mask=1
+VM_1_qemu_numa_node=0
+
+VM_2_qemu_mask=1
+VM_2_qemu_numa_node=0
diff --git a/src/spdk/test/vhost/migration/migration-tc1.job b/src/spdk/test/vhost/migration/migration-tc1.job
new file mode 100644
index 00000000..5383b243
--- /dev/null
+++ b/src/spdk/test/vhost/migration/migration-tc1.job
@@ -0,0 +1,25 @@
+[global]
+blocksize_range=4k-512k
+#bs=512k
+iodepth=128
+ioengine=libaio
+filename=
+group_reporting
+thread
+numjobs=1
+direct=1
+do_verify=1
+verify=md5
+verify_fatal=1
+verify_dump=1
+size=100%
+
+[write]
+rw=write
+stonewall
+
+[randread]
+rw=randread
+runtime=10
+time_based
+stonewall
diff --git a/src/spdk/test/vhost/migration/migration-tc1.sh b/src/spdk/test/vhost/migration/migration-tc1.sh
new file mode 100644
index 00000000..ec89545d
--- /dev/null
+++ b/src/spdk/test/vhost/migration/migration-tc1.sh
@@ -0,0 +1,123 @@
+function migration_tc1_clean_vhost_config()
+{
+	# Restore trap
+	trap 'error_exit "${FUNCNAME}" "${LINENO}"' INT ERR EXIT
+
+	notice "Removing vhost devices & controllers via RPC ..."
+	# Delete bdev first to remove all LUNs and SCSI targets
+	$rpc delete_malloc_bdev Malloc0
+
+	# Delete controllers
+	$rpc remove_vhost_controller $incoming_vm_ctrlr
+	$rpc remove_vhost_controller $target_vm_ctrlr
+
+	unset -v incoming_vm target_vm incoming_vm_ctrlr target_vm_ctrlr rpc
+}
+
+function migration_tc1_configure_vhost()
+{
+	# Those are global intentionally - they will be unset in cleanup handler
+	incoming_vm=0
+	target_vm=1
+	incoming_vm_ctrlr=naa.Malloc0.$incoming_vm
+	target_vm_ctrlr=naa.Malloc0.$target_vm
+	rpc="$SPDK_BUILD_DIR/scripts/rpc.py -s $(get_vhost_dir)/rpc.sock"
+
+	trap 'migration_tc1_error_handler; error_exit "${FUNCNAME}" "${LINENO}"' INT ERR EXIT
+
+	# Construct shared Malloc Bdev
+	$rpc construct_malloc_bdev -b Malloc0 128 4096
+
+	# And two controllers - one for each VM. Both are using the same Malloc Bdev as LUN 0
+	$rpc construct_vhost_scsi_controller $incoming_vm_ctrlr
+	$rpc add_vhost_scsi_lun $incoming_vm_ctrlr 0 Malloc0
+
+	$rpc construct_vhost_scsi_controller $target_vm_ctrlr
+	$rpc add_vhost_scsi_lun $target_vm_ctrlr 0 Malloc0
+}
+
+function migration_tc1_error_handler()
+{
+	trap - SIGINT ERR EXIT
+	warning "Migration TC1 ERROR HANDLER"
+	print_backtrace
+	set -x
+
+	vm_kill_all
+	migration_tc1_clean_vhost_config
+
+	warning "Migration TC1 FAILED"
+}
+
+function migration_tc1()
+{
+	# Use 2 VMs:
+	# incoming VM - the one we want to migrate
+	# targe VM - the one which will accept migration
+	local job_file="$MIGRATION_DIR/migration-tc1.job"
+
+	# Run vhost
+	spdk_vhost_run
+	migration_tc1_configure_vhost
+
+	notice "Setting up VMs"
+	vm_setup --os="$os_image" --force=$incoming_vm --disk-type=spdk_vhost_scsi --disks=Malloc0 --migrate-to=$target_vm
+	vm_setup --force=$target_vm --disk-type=spdk_vhost_scsi --disks=Malloc0 --incoming=$incoming_vm
+
+	# Run everything
+	vm_run $incoming_vm $target_vm
+
+	# Wait only for incoming VM, as target is waiting for migration
+	vm_wait_for_boot 600 $incoming_vm
+
+	# Run fio before migration
+	notice "Starting FIO"
+
+	vm_check_scsi_location $incoming_vm
+	run_fio $fio_bin --job-file="$job_file" --local --vm="${incoming_vm}$(printf ':/dev/%s' $SCSI_DISK)"
+
+	# Wait a while to let the FIO time to issue some IO
+	sleep 5
+
+	# Check if fio is still running before migration
+	if ! is_fio_running $incoming_vm; then
+		vm_ssh $incoming_vm "cat /root/$(basename ${job_file}).out"
+		error "FIO is not running before migration: process crashed or finished too early"
+	fi
+
+	vm_migrate $incoming_vm
+	sleep 3
+
+	# Check if fio is still running after migration
+	if ! is_fio_running $target_vm; then
+		vm_ssh $target_vm "cat /root/$(basename ${job_file}).out"
+		error "FIO is not running after migration: process crashed or finished too early"
+	fi
+
+	notice "Waiting for fio to finish"
+	local timeout=40
+	while is_fio_running $target_vm; do
+		sleep 1
+		echo -n "."
+		if (( timeout-- == 0 )); then
+			error "timeout while waiting for FIO!"
+		fi
+	done
+
+	notice "Fio result is:"
+	vm_ssh $target_vm "cat /root/$(basename ${job_file}).out"
+
+	notice "Migration DONE"
+
+	notice "Shutting down all VMs"
+	vm_shutdown_all
+
+	migration_tc1_clean_vhost_config
+
+	notice "killing vhost app"
+	spdk_vhost_kill
+
+	notice "Migration TC1 SUCCESS"
+}
+
+migration_tc1
diff --git a/src/spdk/test/vhost/migration/migration-tc2.job b/src/spdk/test/vhost/migration/migration-tc2.job
new file mode 100644
index 00000000..df78a3cd
--- /dev/null
+++ b/src/spdk/test/vhost/migration/migration-tc2.job
@@ -0,0 +1,20 @@
+[global]
+blocksize_range=4k-512k
+iodepth=128
+ioengine=libaio
+filename=
+group_reporting
+thread
+numjobs=1
+direct=1
+do_verify=1
+verify=md5
+verify_fatal=1
+verify_dump=1
+verify_backlog=8
+
+[randwrite]
+rw=randwrite
+runtime=15
+time_based
+stonewall
diff --git a/src/spdk/test/vhost/migration/migration-tc2.sh b/src/spdk/test/vhost/migration/migration-tc2.sh
new file mode 100644
index 00000000..bc4a0f53
--- /dev/null
+++ b/src/spdk/test/vhost/migration/migration-tc2.sh
@@ -0,0 +1,209 @@
+source $SPDK_BUILD_DIR/test/nvmf/common.sh
+
+function migration_tc2_cleanup_nvmf_tgt()
+{
+	local i
+
+	if [[ ! -r "$nvmf_dir/nvmf_tgt.pid" ]]; then
+		warning "Pid file '$nvmf_dir/nvmf_tgt.pid' does not exist. "
+		return
+	fi
+
+	if [[ ! -z "$1" ]]; then
+		trap 'error_exit "${FUNCNAME}" "${LINENO}"' INT ERR EXIT
+		pkill --signal $1 -F $nvmf_dir/nvmf_tgt.pid || true
+		sleep 5
+		if ! pkill -F $nvmf_dir/nvmf_tgt.pid; then
+			fail "failed to kill nvmf_tgt app"
+		fi
+	else
+		pkill --signal SIGTERM -F $nvmf_dir/nvmf_tgt.pid || true
+		for (( i=0; i<20; i++ )); do
+			if ! pkill --signal 0 -F $nvmf_dir/nvmf_tgt.pid; then
+				break
+			fi
+			sleep 0.5
+		done
+
+		if pkill --signal 0 -F $nvmf_dir/nvmf_tgt.pid; then
+			error "nvmf_tgt failed to shutdown"
+		fi
+	fi
+
+	rm $nvmf_dir/nvmf_tgt.pid
+	unset -v nvmf_dir rpc_nvmf
+}
+
+function migration_tc2_cleanup_vhost_config()
+{
+	timing_enter migration_tc2_cleanup_vhost_config
+
+	trap 'migration_tc2_cleanup_nvmf_tgt SIGKILL; error_exit "${FUNCNAME}" "${LINENO}"' INT ERR EXIT
+
+	notice "Shutting down all VMs"
+	vm_shutdown_all
+
+	notice "Removing vhost devices & controllers via RPC ..."
+	# Delete bdev first to remove all LUNs and SCSI targets
+	$rpc_0 delete_nvme_controller Nvme0
+	$rpc_0 remove_vhost_controller $incoming_vm_ctrlr
+
+	$rpc_1 delete_nvme_controller Nvme0
+	$rpc_1 remove_vhost_controller $target_vm_ctrlr
+
+	notice "killing vhost app"
+	spdk_vhost_kill 0
+	spdk_vhost_kill 1
+
+	unset -v incoming_vm target_vm incoming_vm_ctrlr target_vm_ctrlr
+	unset -v rpc_0 rpc_1
+
+	trap 'error_exit "${FUNCNAME}" "${LINENO}"' INT ERR EXIT
+	migration_tc2_cleanup_nvmf_tgt
+
+	timing_exit migration_tc2_cleanup_vhost_config
+}
+
+function migration_tc2_configure_vhost()
+{
+	timing_enter migration_tc2_configure_vhost
+
+	# Those are global intentionally - they will be unset in cleanup handler
+	nvmf_dir="$TEST_DIR/nvmf_tgt"
+
+	incoming_vm=1
+	target_vm=2
+	incoming_vm_ctrlr=naa.VhostScsi0.$incoming_vm
+	target_vm_ctrlr=naa.VhostScsi0.$target_vm
+
+	rpc_nvmf="$SPDK_BUILD_DIR/scripts/rpc.py -s $nvmf_dir/rpc.sock"
+	rpc_0="$SPDK_BUILD_DIR/scripts/rpc.py -s $(get_vhost_dir 0)/rpc.sock"
+	rpc_1="$SPDK_BUILD_DIR/scripts/rpc.py -s $(get_vhost_dir 1)/rpc.sock"
+
+	# Default cleanup/error handlers will not shutdown nvmf_tgt app so setup it
+	# here to teardown in cleanup function
+	trap 'migration_tc2_error_cleanup; error_exit "${FUNCNAME}" "${LINENO}"' INT ERR EXIT
+
+	# Run nvmf_tgt and two vhost instances:
+	# nvmf_tgt uses core id 2 (-m 0x4)
+	# First uses core id 0 (vhost_0_reactor_mask=0x1)
+	# Second uses core id 1 (vhost_1_reactor_mask=0x2)
+	# This force to use VM 1 and 2.
+	timing_enter start_nvmf_tgt
+	notice "Running nvmf_tgt..."
+	mkdir -p $nvmf_dir
+	rm -f $nvmf_dir/*
+	$SPDK_BUILD_DIR/app/nvmf_tgt/nvmf_tgt -s 512 -m 0x4 -r $nvmf_dir/rpc.sock --wait-for-rpc &
+	local nvmf_tgt_pid=$!
+	echo $nvmf_tgt_pid > $nvmf_dir/nvmf_tgt.pid
+	waitforlisten "$nvmf_tgt_pid" "$nvmf_dir/rpc.sock"
+	$rpc_nvmf start_subsystem_init
+	$rpc_nvmf nvmf_create_transport -t RDMA -u 8192 -p 4
+	$SPDK_BUILD_DIR/scripts/gen_nvme.sh --json | $rpc_nvmf load_subsystem_config
+	timing_exit start_nvmf_tgt
+
+	spdk_vhost_run --memory=512 --vhost-num=0 --no-pci
+	# Those are global intentionally
+	vhost_1_reactor_mask=0x2
+	vhost_1_master_core=1
+	spdk_vhost_run --memory=512 --vhost-num=1 --no-pci
+
+	local rdma_ip_list=$(get_available_rdma_ips)
+	local nvmf_target_ip=$(echo "$rdma_ip_list" | head -n 1)
+
+	if [[ -z "$nvmf_target_ip" ]]; then
+		fail "no NIC for nvmf target"
+	fi
+
+	notice "Configuring nvmf_tgt, vhost devices & controllers via RPC ..."
+
+	# Construct shared bdevs and controllers
+	$rpc_nvmf nvmf_subsystem_create nqn.2016-06.io.spdk:cnode1 -a -s SPDK00000000000001
+	$rpc_nvmf nvmf_subsystem_add_ns nqn.2016-06.io.spdk:cnode1 Nvme0n1
+	$rpc_nvmf nvmf_subsystem_add_listener nqn.2016-06.io.spdk:cnode1 -t rdma -a $nvmf_target_ip -s 4420
+
+	$rpc_0 construct_nvme_bdev -b Nvme0 -t rdma -f ipv4 -a $nvmf_target_ip -s 4420 -n "nqn.2016-06.io.spdk:cnode1"
+	$rpc_0 construct_vhost_scsi_controller $incoming_vm_ctrlr
+	$rpc_0 add_vhost_scsi_lun $incoming_vm_ctrlr 0 Nvme0n1
+
+	$rpc_1 construct_nvme_bdev -b Nvme0 -t rdma -f ipv4 -a $nvmf_target_ip -s 4420 -n "nqn.2016-06.io.spdk:cnode1"
+	$rpc_1 construct_vhost_scsi_controller $target_vm_ctrlr
+	$rpc_1 add_vhost_scsi_lun $target_vm_ctrlr 0 Nvme0n1
+
+	notice "Setting up VMs"
+	vm_setup --os="$os_image" --force=$incoming_vm --disk-type=spdk_vhost_scsi --disks=VhostScsi0 \
+		--migrate-to=$target_vm  --memory=1024 --vhost-num=0
+	vm_setup --force=$target_vm --disk-type=spdk_vhost_scsi --disks=VhostScsi0 --incoming=$incoming_vm --memory=1024 \
+		--vhost-num=1
+
+	# Run everything
+	vm_run $incoming_vm $target_vm
+
+	# Wait only for incoming VM, as target is waiting for migration
+	vm_wait_for_boot 600 $incoming_vm
+
+	notice "Configuration done"
+
+	timing_exit migration_tc2_configure_vhost
+}
+
+function migration_tc2_error_cleanup()
+{
+	trap - SIGINT ERR EXIT
+	set -x
+
+	vm_kill_all
+	migration_tc2_cleanup_vhost_config
+	notice "Migration TC2 FAILED"
+}
+
+function migration_tc2()
+{
+	# Use 2 VMs:
+	# incoming VM - the one we want to migrate
+	# targe VM - the one which will accept migration
+	local job_file="$MIGRATION_DIR/migration-tc2.job"
+
+	migration_tc2_configure_vhost
+
+	# Run fio before migration
+	notice "Starting FIO"
+	vm_check_scsi_location $incoming_vm
+	run_fio $fio_bin --job-file="$job_file" --local --vm="${incoming_vm}$(printf ':/dev/%s' $SCSI_DISK)"
+
+	# Wait a while to let the FIO time to issue some IO
+	sleep 5
+
+	# Check if fio is still running before migration
+	if ! is_fio_running $incoming_vm; then
+		vm_ssh $incoming_vm "cat /root/$(basename ${job_file}).out"
+		error "FIO is not running before migration: process crashed or finished too early"
+	fi
+
+	vm_migrate $incoming_vm
+	sleep 3
+
+	# Check if fio is still running after migration
+	if ! is_fio_running $target_vm; then
+		vm_ssh $target_vm "cat /root/$(basename ${job_file}).out"
+		error "FIO is not running after migration: process crashed or finished too early"
+	fi
+
+	notice "Waiting for fio to finish"
+	local timeout=40
+	while is_fio_running $target_vm; do
+		sleep 1
+		echo -n "."
+		if (( timeout-- == 0 )); then
+			error "timeout while waiting for FIO!"
+		fi
+	done
+
+	notice "Fio result is:"
+	vm_ssh $target_vm "cat /root/$(basename ${job_file}).out"
+
+	migration_tc2_cleanup_vhost_config
+	notice "Migration TC2 SUCCESS"
+}
+
+migration_tc2
diff --git a/src/spdk/test/vhost/migration/migration-tc3.job b/src/spdk/test/vhost/migration/migration-tc3.job
new file mode 100644
index 00000000..fe192966
--- /dev/null
+++ b/src/spdk/test/vhost/migration/migration-tc3.job
@@ -0,0 +1,20 @@
+[global]
+blocksize=4k-512k
+iodepth=128
+ioengine=libaio
+filename=
+group_reporting
+thread
+numjobs=1
+direct=1
+do_verify=1
+verify=md5
+verify_fatal=1
+verify_dump=1
+verify_backlog=8
+
+[randwrite]
+rw=randwrite
+runtime=15
+time_based
+stonewall
diff --git a/src/spdk/test/vhost/migration/migration-tc3a.sh b/src/spdk/test/vhost/migration/migration-tc3a.sh
new file mode 100644
index 00000000..0f20b994
--- /dev/null
+++ b/src/spdk/test/vhost/migration/migration-tc3a.sh
@@ -0,0 +1,227 @@
+source $SPDK_BUILD_DIR/test/nvmf/common.sh
+source $MIGRATION_DIR/autotest.config
+
+incoming_vm=1
+target_vm=2
+incoming_vm_ctrlr=naa.VhostScsi0.$incoming_vm
+target_vm_ctrlr=naa.VhostScsi0.$target_vm
+share_dir=$TEST_DIR/share
+spdk_repo_share_dir=$TEST_DIR/share_spdk
+job_file=$MIGRATION_DIR/migration-tc3.job
+
+if [ -z "$MGMT_TARGET_IP" ]; then
+	error "No IP address of target is given"
+fi
+
+if [ -z "$MGMT_INITIATOR_IP" ]; then
+	error "No IP address of initiator  is given"
+fi
+
+if [ -z "$RDMA_TARGET_IP" ]; then
+	error "No IP address of targets RDMA capable NIC is given"
+fi
+
+if [ -z "$RDMA_INITIATOR_IP" ]; then
+	error "No IP address of initiators RDMA capable NIC is given"
+fi
+
+function ssh_remote()
+{
+	local ssh_cmd="ssh -i $SPDK_VHOST_SSH_KEY_FILE \
+		-o UserKnownHostsFile=/dev/null \
+		-o StrictHostKeyChecking=no -o ControlMaster=auto \
+		root@$1"
+
+	shift
+	$ssh_cmd "$@"
+}
+
+function wait_for_remote()
+{
+	local timeout=40
+	set +x
+	while [[ ! -f $share_dir/DONE ]]; do
+		echo -n "."
+		if (( timeout-- == 0 )); then
+			error "timeout while waiting for FIO!"
+		fi
+		sleep 1
+	done
+	set -x
+	rm -f $share_dir/DONE
+}
+
+function check_rdma_connection()
+{
+	local nic_name=$(ip -4 -o addr show to $RDMA_TARGET_IP up | cut -d' ' -f2)
+	if [[ -z $nic_name ]]; then
+		error "There is no NIC with IP address $RDMA_TARGET_IP configured"
+	fi
+
+	if ! ls /sys/class/infiniband/*/device/net/$nic_name &> /dev/null; then
+		error "$nic_name with IP $RDMA_TARGET_IP is not a RDMA capable NIC"
+	fi
+
+}
+
+function host1_cleanup_nvmf()
+{
+	notice "Shutting down nvmf_tgt on local server"
+	if [[ ! -z "$1" ]]; then
+		pkill --signal $1 -F $nvmf_dir/nvmf_tgt.pid
+	else
+		pkill -F $nvmf_dir/nvmf_tgt.pid
+	fi
+	rm -f $nvmf_dir/nvmf_tgt.pid
+}
+
+function host1_cleanup_vhost()
+{
+	trap 'host1_cleanup_nvmf SIGKILL; error_exit "${FUNCNAME}" "${LINENO}"' INT ERR EXIT
+	notice "Shutting down VM $incoming_vm"
+	vm_kill $incoming_vm
+
+	notice "Removing bdev & controller from vhost on local server"
+	$rpc_0 delete_nvme_controller Nvme0
+	$rpc_0 remove_vhost_controller $incoming_vm_ctrlr
+
+	notice "Shutting down vhost app"
+	spdk_vhost_kill 0
+
+	host1_cleanup_nvmf
+}
+
+function host1_start_nvmf()
+{
+	nvmf_dir="$TEST_DIR/nvmf_tgt"
+	rpc_nvmf="$SPDK_BUILD_DIR/scripts/rpc.py -s $nvmf_dir/nvmf_rpc.sock"
+
+	notice "Starting nvmf_tgt instance on local server"
+	mkdir -p $nvmf_dir
+	rm -rf $nvmf_dir/*
+
+	trap 'host1_cleanup_nvmf SIGKILL; error_exit "${FUNCNAME}" "${LINENO}"' INT ERR EXIT
+	$SPDK_BUILD_DIR/app/nvmf_tgt/nvmf_tgt -s 512 -m 0xF -r $nvmf_dir/nvmf_rpc.sock --wait-for-rpc &
+	nvmf_tgt_pid=$!
+	echo $nvmf_tgt_pid > $nvmf_dir/nvmf_tgt.pid
+	waitforlisten "$nvmf_tgt_pid" "$nvmf_dir/nvmf_rpc.sock"
+	$rpc_nvmf start_subsystem_init
+	$rpc_nvmf nvmf_create_transport -t RDMA -u 8192 -p 4
+	$SPDK_BUILD_DIR/scripts/gen_nvme.sh --json | $rpc_nvmf load_subsystem_config
+
+	$rpc_nvmf nvmf_subsystem_create nqn.2018-02.io.spdk:cnode1 -a -s SPDK01
+	$rpc_nvmf nvmf_subsystem_add_ns nqn.2018-02.io.spdk:cnode1 Nvme0n1
+	$rpc_nvmf nvmf_subsystem_add_listener nqn.2018-02.io.spdk:cnode1 -t rdma -a $RDMA_TARGET_IP -s 4420
+}
+
+function host1_start_vhost()
+{
+	rpc_0="$SPDK_BUILD_DIR/scripts/rpc.py -s $(get_vhost_dir 0)/rpc.sock"
+
+	notice "Starting vhost0 instance on local server"
+	trap 'host1_cleanup_vhost; error_exit "${FUNCNAME}" "${LINENO}"' INT ERR EXIT
+	spdk_vhost_run --vhost-num=0 --no-pci
+	$rpc_0 construct_nvme_bdev -b Nvme0 -t rdma -f ipv4 -a $RDMA_TARGET_IP -s 4420 -n "nqn.2018-02.io.spdk:cnode1"
+	$rpc_0 construct_vhost_scsi_controller $incoming_vm_ctrlr
+	$rpc_0 add_vhost_scsi_lun $incoming_vm_ctrlr 0 Nvme0n1
+
+	vm_setup --os="$share_dir/migration.qcow2" --force=$incoming_vm --disk-type=spdk_vhost_scsi --disks=VhostScsi0 \
+		--migrate-to=$target_vm --memory=512 --queue_num=1
+
+	# TODO: Fix loop calculating cpu_num in common.sh
+	# We need -smp 1 and -queue_num 1 for this test to work, and this loop
+	# in some cases calculates wrong cpu_num.
+	sed -i "s#smp 2#smp 1#g" $VM_BASE_DIR/$incoming_vm/run.sh
+	vm_run $incoming_vm
+	vm_wait_for_boot 300 $incoming_vm
+}
+
+function cleanup_share()
+{
+	set +e
+	notice "Cleaning up share directory on remote and local server"
+	ssh_remote $MGMT_INITIATOR_IP "umount $VM_BASE_DIR"
+	ssh_remote $MGMT_INITIATOR_IP "umount $share_dir; rm -f $share_dir/* rm -rf $spdk_repo_share_dir"
+	rm -f $share_dir/migration.qcow2
+	rm -f $share_dir/spdk.tar.gz
+	set -e
+}
+
+function host_1_create_share()
+{
+	notice "Creating share directory on local server to re-use on remote"
+	mkdir -p $share_dir
+	mkdir -p $VM_BASE_DIR # This dir would've been created later but we need it now
+	rm -rf $share_dir/spdk.tar.gz $share_dir/spdk || true
+	cp $os_image $share_dir/migration.qcow2
+	tar --exclude="*.o"--exclude="*.d" --exclude="*.git" -C $SPDK_BUILD_DIR -zcf $share_dir/spdk.tar.gz .
+}
+
+function host_2_create_share()
+{
+	# Copy & compile the sources for later use on remote server.
+	ssh_remote $MGMT_INITIATOR_IP "uname -a"
+	ssh_remote $MGMT_INITIATOR_IP "mkdir -p $share_dir"
+	ssh_remote $MGMT_INITIATOR_IP "mkdir -p $spdk_repo_share_dir"
+	ssh_remote $MGMT_INITIATOR_IP "mkdir -p $VM_BASE_DIR"
+	ssh_remote $MGMT_INITIATOR_IP "sshfs -o\
+	 ssh_command=\"ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o ControlMaster=auto\
+	 -i $SPDK_VHOST_SSH_KEY_FILE\" root@$MGMT_TARGET_IP:$VM_BASE_DIR $VM_BASE_DIR"
+	ssh_remote $MGMT_INITIATOR_IP "sshfs -o\
+	 ssh_command=\"ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o ControlMaster=auto\
+	 -i $SPDK_VHOST_SSH_KEY_FILE\" root@$MGMT_TARGET_IP:$share_dir $share_dir"
+	ssh_remote $MGMT_INITIATOR_IP "mkdir -p $spdk_repo_share_dir/spdk"
+	ssh_remote $MGMT_INITIATOR_IP "tar -zxf $share_dir/spdk.tar.gz -C $spdk_repo_share_dir/spdk --strip-components=1"
+	ssh_remote $MGMT_INITIATOR_IP "cd $spdk_repo_share_dir/spdk; make clean; ./configure --with-rdma --enable-debug; make -j40"
+}
+
+function host_2_start_vhost()
+{
+	ssh_remote $MGMT_INITIATOR_IP "nohup $spdk_repo_share_dir/spdk/test/vhost/migration/migration.sh\
+	 --test-cases=3b --work-dir=$TEST_DIR --os=$share_dir/migration.qcow2\
+	 --rdma-tgt-ip=$RDMA_TARGET_IP &>$share_dir/output.log &"
+	notice "Waiting for remote to be done with vhost & VM setup..."
+	wait_for_remote
+}
+
+function setup_share()
+{
+	trap 'cleanup_share; error_exit "${FUNCNAME}" "${LINENO}"' INT ERR EXIT
+	host_1_create_share
+	host_2_create_share
+}
+
+function migration_tc3()
+{
+	check_rdma_connection
+	setup_share
+	host1_start_nvmf
+	host1_start_vhost
+	host_2_start_vhost
+
+	# Do migration
+	notice "Starting fio on local VM"
+	vm_check_scsi_location $incoming_vm
+
+	run_fio $fio_bin --job-file="$job_file" --local --vm="${incoming_vm}$(printf ':/dev/%s' $SCSI_DISK)"
+	sleep 5
+
+	if ! is_fio_running $incoming_vm; then
+		vh_ssh $incoming_vm "cat /root/$(basename ${job_file}).out"
+		error "Fio not running on local VM before starting migration!"
+	fi
+
+	vm_migrate $incoming_vm $RDMA_INITIATOR_IP
+	sleep 1
+
+	# Verify migration on remote host and clean up vhost
+	ssh_remote $MGMT_INITIATOR_IP "pkill -CONT -F $TEST_DIR/tc3b.pid"
+	notice "Waiting for remote to finish FIO on VM and clean up..."
+	wait_for_remote
+
+	# Clean up local stuff
+	host1_cleanup_vhost
+	cleanup_share
+}
+
+migration_tc3
diff --git a/src/spdk/test/vhost/migration/migration-tc3b.sh b/src/spdk/test/vhost/migration/migration-tc3b.sh
new file mode 100644
index 00000000..babba0dc
--- /dev/null
+++ b/src/spdk/test/vhost/migration/migration-tc3b.sh
@@ -0,0 +1,79 @@
+# Set -m option is needed to be able to use "suspend" command
+# as we are usin non-interactive session to connect to remote.
+# Without -m it would be not possible to suspend the process.
+set -m
+source $MIGRATION_DIR/autotest.config
+
+incoming_vm=1
+target_vm=2
+target_vm_ctrl=naa.VhostScsi0.$target_vm
+rpc="$SPDK_BUILD_DIR/scripts/rpc.py -s $(get_vhost_dir 1)/rpc.sock"
+share_dir=$TEST_DIR/share
+
+function host_2_cleanup_vhost()
+{
+	notice "Shutting down VM $target_vm"
+	vm_kill $target_vm
+
+	notice "Removing bdev & controller from vhost 1 on remote server"
+	$rpc delete_nvme_controller Nvme0
+	$rpc remove_vhost_controller $target_vm_ctrl
+
+	notice "Shutting down vhost app"
+	spdk_vhost_kill 1
+	sleep 1
+}
+
+function host_2_start_vhost()
+{
+	echo "BASE DIR $TEST_DIR"
+	vhost_work_dir=$TEST_DIR/vhost1
+	mkdir -p $vhost_work_dir
+	rm -f $vhost_work_dir/*
+
+	notice "Starting vhost 1 instance on remote server"
+	trap 'host_2_cleanup_vhost; error_exit "${FUNCNAME}" "${LINENO}"' INT ERR EXIT
+	spdk_vhost_run --vhost-num=1 --no-pci
+
+	$rpc construct_nvme_bdev -b Nvme0 -t rdma -f ipv4 -a $RDMA_TARGET_IP -s 4420 -n "nqn.2018-02.io.spdk:cnode1"
+	$rpc construct_vhost_scsi_controller $target_vm_ctrl
+	$rpc add_vhost_scsi_lun $target_vm_ctrl 0 Nvme0n1
+
+	vm_setup --os="$os_image" --force=$target_vm --disk-type=spdk_vhost_scsi --disks=VhostScsi0 \
+		--memory=512 --vhost-num=1 --incoming=$incoming_vm
+	vm_run $target_vm
+	sleep 1
+
+	# Use this file as a flag to notify main script
+	# that setup on remote server is done
+	echo "DONE" > $share_dir/DONE
+}
+
+echo $$ > $TEST_DIR/tc3b.pid
+host_2_start_vhost
+suspend -f
+
+if ! vm_os_booted $target_vm; then
+	fail "VM$target_vm is not running!"
+fi
+
+if ! is_fio_running $target_vm; then
+	vm_ssh $target_vm "cat /root/migration-tc3.job.out"
+	error "FIO is not running on remote server after migration!"
+fi
+
+notice "Waiting for FIO to finish on remote server VM"
+timeout=40
+while is_fio_running $target_vm; do
+	sleep 1
+	echo -n "."
+	if (( timeout-- == 0 )); then
+		error "timeout while waiting for FIO!"
+	fi
+done
+
+notice "FIO result after migration:"
+vm_ssh $target_vm "cat /root/migration-tc3.job.out"
+
+host_2_cleanup_vhost
+echo "DONE" > $share_dir/DONE
diff --git a/src/spdk/test/vhost/migration/migration.sh b/src/spdk/test/vhost/migration/migration.sh
new file mode 100755
index 00000000..bdfcd845
--- /dev/null
+++ b/src/spdk/test/vhost/migration/migration.sh
@@ -0,0 +1,153 @@
+#!/usr/bin/env bash
+
+set -e
+
+vms=()
+declare -A vms_os
+declare -A vms_raw_disks
+declare -A vms_ctrlrs
+declare -A vms_ctrlrs_disks
+
+# By default use Guest fio
+fio_bin=""
+test_cases=""
+MGMT_TARGET_IP=""
+MGMT_INITIATOR_IP=""
+RDMA_TARGET_IP=""
+RDMA_INITIATOR_IP=""
+function usage()
+{
+	[[ ! -z $2 ]] && ( echo "$2"; echo ""; )
+	echo "Shortcut script for doing automated test of live migration."
+	echo "Usage: $(basename $1) [OPTIONS]"
+	echo
+	echo "    --work-dir=WORK_DIR   Where to find build file. Must exist. [default: $TEST_DIR]"
+	echo "    --os ARGS             VM configuration. This parameter might be used more than once:"
+	echo "    --fio-bin=FIO         Use specific fio binary (will be uploaded to VM)"
+	echo "    --test-cases=TESTS    Coma-separated list of tests to run. Implemented test cases are: 1"
+	echo "                          See test/vhost/test_plan.md for more info."
+	echo "    --mgmt-tgt-ip=IP      IP address of target."
+	echo "    --mgmt-init-ip=IP     IP address of initiator."
+	echo "    --rdma-tgt-ip=IP      IP address of targets rdma capable NIC."
+	echo "    --rdma-init-ip=IP     IP address of initiators rdma capable NIC."
+	echo "-x                        set -x for script debug"
+}
+
+for param in "$@"; do
+	case "$param" in
+		--help|-h)
+			usage $0
+			exit 0
+			;;
+		--work-dir=*) TEST_DIR="${param#*=}" ;;
+		--os=*) os_image="${param#*=}" ;;
+		--fio-bin=*) fio_bin="${param}" ;;
+		--test-cases=*) test_cases="${param#*=}" ;;
+		--mgmt-tgt-ip=*) MGMT_TARGET_IP="${param#*=}" ;;
+		--mgmt-init-ip=*) MGMT_INITIATOR_IP="${param#*=}" ;;
+		--rdma-tgt-ip=*) RDMA_TARGET_IP="${param#*=}" ;;
+		--rdma-init-ip=*) RDMA_INITIATOR_IP="${param#*=}" ;;
+		-x) set -x ;;
+		-v) SPDK_VHOST_VERBOSE=true	;;
+		*)
+			usage $0 "Invalid argument '$param'"
+			exit 1;;
+	esac
+done
+
+. $(readlink -e "$(dirname $0)/../common/common.sh") || exit 1
+MIGRATION_DIR=$(readlink -f $(dirname $0))
+
+[[ ! -z "$test_cases" ]] || fail "Need '--test-cases=' parameter"
+
+trap 'error_exit "${FUNCNAME}" "${LINENO}"' INT ERR EXIT
+
+function vm_monitor_send()
+{
+	local vm_num=$1
+	local cmd_result_file="$2"
+	local vm_dir="$VM_BASE_DIR/$1"
+	local vm_monitor_port=$(cat $vm_dir/monitor_port)
+
+	[[ ! -z "$vm_monitor_port" ]] || fail "No monitor port!"
+
+	shift 2
+	nc 127.0.0.1 $vm_monitor_port "$@" > $cmd_result_file
+}
+
+# Migrate VM $1
+function vm_migrate()
+{
+	local from_vm_dir="$VM_BASE_DIR/$1"
+	local target_vm_dir="$(readlink -e $from_vm_dir/vm_migrate_to)"
+	local target_vm="$(basename $target_vm_dir)"
+	local target_vm_migration_port="$(cat $target_vm_dir/migration_port)"
+	if [[ -n "$2" ]]; then
+		local target_ip=$2
+	else
+		local target_ip="127.0.0.1"
+	fi
+
+	# Sanity check if target VM (QEMU) is configured to accept source VM (QEMU) migration
+	if [[ "$(readlink -e ${target_vm_dir}/vm_incoming)" != "$(readlink -e ${from_vm_dir})" ]]; then
+		fail "source VM $1 or destination VM is not properly configured for live migration"
+	fi
+
+	timing_enter vm_migrate
+	notice "Migrating VM $1 to VM "$(basename $target_vm_dir)
+	echo -e \
+		"migrate_set_speed 1g\n" \
+		"migrate tcp:$target_ip:$target_vm_migration_port\n" \
+		"info migrate\n" \
+		"quit" | vm_monitor_send $1 "$from_vm_dir/migration_result"
+
+	# Post migration checks:
+	if ! grep "Migration status: completed"  $from_vm_dir/migration_result -q; then
+		cat $from_vm_dir/migration_result
+		fail "Migration failed:\n"
+	fi
+
+	# Don't perform the following check if target VM is on remote server
+	# as we won't have access to it.
+	# If you need this check then perform it on your own.
+	if [[ "$target_ip" == "127.0.0.1" ]]; then
+		if ! vm_os_booted $target_vm; then
+			fail "VM$target_vm is not running"
+			cat $target_vm $target_vm_dir/cont_result
+		fi
+	fi
+
+	notice "Migration complete"
+	timing_exit vm_migrate
+}
+
+function is_fio_running()
+{
+	local shell_restore_x="$( [[ "$-" =~ x ]] && echo 'set -x' )"
+	set +x
+
+	if vm_ssh $1 'kill -0 $(cat /root/fio.pid)'; then
+		local ret=0
+	else
+		local ret=1
+	fi
+
+	$shell_restore_x
+	return $ret
+}
+
+for test_case in ${test_cases//,/ }; do
+	assert_number "$test_case"
+	notice "==============================="
+	notice "Running Migration test case ${test_case}"
+	notice "==============================="
+
+	timing_enter migration-tc${test_case}
+	source $MIGRATION_DIR/migration-tc${test_case}.sh
+	timing_exit migration-tc${test_case}
+done
+
+notice "Migration Test SUCCESS"
+notice "==============="
+
+trap - SIGINT ERR EXIT