Adding upstream version 14.2.21.upstream/14.2.21upstream

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

6 files changed, 1988 insertions, 0 deletions

diff --git a/src/spdk/test/common/autotest_common.sh b/src/spdk/test/common/autotest_common.sh
new file mode 100644
index 00000000..86511ccf
--- /dev/null
+++ b/src/spdk/test/common/autotest_common.sh
@@ -0,0 +1,706 @@
+: ${SPDK_AUTOTEST_X=true}; export SPDK_AUTOTEST_X
+
+if $SPDK_AUTOTEST_X; then
+	set -x
+fi
+
+set -e
+
+# Export flag to skip the known bug that exists in librados
+# Bug is reported on ceph bug tracker with number 24078
+export ASAN_OPTIONS=new_delete_type_mismatch=0
+
+PS4=' \t	\$ '
+ulimit -c unlimited
+
+: ${RUN_NIGHTLY:=0}
+export RUN_NIGHTLY
+
+: ${RUN_NIGHTLY_FAILING:=0}
+export RUN_NIGHTLY_FAILING
+
+if [[ ! -z $1 ]]; then
+	if [ -f $1 ]; then
+		source $1
+	fi
+fi
+
+# If certain utilities are not installed, preemptively disable the tests
+if ! hash ceph; then
+	SPDK_TEST_RBD=0
+fi
+
+if ! hash pmempool; then
+	SPDK_TEST_PMDK=0
+fi
+
+# Set defaults for missing test config options
+: ${SPDK_BUILD_DOC=1}; export SPDK_BUILD_DOC
+: ${SPDK_BUILD_SHARED_OBJECT=1}; export SPDK_BUILD_SHARED_OBJECT
+: ${SPDK_RUN_CHECK_FORMAT=1}; export SPDK_RUN_CHECK_FORMAT
+: ${SPDK_RUN_SCANBUILD=1}; export SPDK_RUN_SCANBUILD
+: ${SPDK_RUN_VALGRIND=1}; export SPDK_RUN_VALGRIND
+: ${SPDK_TEST_UNITTEST=1}; export SPDK_TEST_UNITTEST
+: ${SPDK_TEST_ISCSI=1}; export SPDK_TEST_ISCSI
+: ${SPDK_TEST_ISCSI_INITIATOR=1}; export SPDK_TEST_ISCSI_INITIATOR
+: ${SPDK_TEST_NVME=1}; export SPDK_TEST_NVME
+: ${SPDK_TEST_NVME_CLI=1}; export SPDK_TEST_NVME_CLI
+: ${SPDK_TEST_NVMF=1}; export SPDK_TEST_NVMF
+: ${SPDK_TEST_RBD=1}; export SPDK_TEST_RBD
+: ${SPDK_TEST_VHOST=1}; export SPDK_TEST_VHOST
+: ${SPDK_TEST_BLOCKDEV=1}; export SPDK_TEST_BLOCKDEV
+: ${SPDK_TEST_IOAT=1}; export SPDK_TEST_IOAT
+: ${SPDK_TEST_EVENT=1}; export SPDK_TEST_EVENT
+: ${SPDK_TEST_BLOBFS=1}; export SPDK_TEST_BLOBFS
+: ${SPDK_TEST_VHOST_INIT=1}; export SPDK_TEST_VHOST_INIT
+: ${SPDK_TEST_PMDK=1}; export SPDK_TEST_PMDK
+: ${SPDK_TEST_LVOL=1}; export SPDK_TEST_LVOL
+: ${SPDK_TEST_JSON=1}; export SPDK_TEST_JSON
+: ${SPDK_RUN_ASAN=1}; export SPDK_RUN_ASAN
+: ${SPDK_RUN_UBSAN=1}; export SPDK_RUN_UBSAN
+: ${SPDK_RUN_INSTALLED_DPDK=1}; export SPDK_RUN_INSTALLED_DPDK
+: ${SPDK_TEST_CRYPTO=1}; export SPDK_TEST_CRYPTO
+
+if [ -z "$DEPENDENCY_DIR" ]; then
+	export DEPENDENCY_DIR=/home/sys_sgsw
+else
+	export DEPENDENCY_DIR
+fi
+
+if [ ! -z "$HUGEMEM" ]; then
+	export HUGEMEM
+fi
+
+# pass our valgrind desire on to unittest.sh
+if [ $SPDK_RUN_VALGRIND -eq 0 ]; then
+	export valgrind=''
+fi
+
+config_params='--enable-debug --enable-werror'
+
+if echo -e "#include <libunwind.h>\nint main(int argc, char *argv[]) {return 0;}\n" | \
+	gcc -o /dev/null -lunwind -x c - 2>/dev/null; then
+	config_params+=' --enable-log-bt'
+fi
+
+if [ $SPDK_TEST_CRYPTO -eq 1 ]; then
+	config_params+=' --with-crypto'
+fi
+
+export UBSAN_OPTIONS='halt_on_error=1:print_stacktrace=1:abort_on_error=1'
+
+# On Linux systems, override the default HUGEMEM in scripts/setup.sh to
+#  allocate 8GB in hugepages.
+# FreeBSD runs a much more limited set of tests, so keep the default 2GB.
+if [ `uname -s` = "Linux" ]; then
+	export HUGEMEM=8192
+fi
+
+DEFAULT_RPC_ADDR=/var/tmp/spdk.sock
+
+case `uname` in
+	FreeBSD)
+		DPDK_FREEBSD_DIR=/usr/local/share/dpdk/x86_64-native-bsdapp-clang
+		if [ -d $DPDK_FREEBSD_DIR ] && [ $SPDK_RUN_INSTALLED_DPDK -eq 1 ]; then
+			WITH_DPDK_DIR=$DPDK_FREEBSD_DIR
+		fi
+		MAKE=gmake
+		MAKEFLAGS=${MAKEFLAGS:--j$(sysctl -a | egrep -i 'hw.ncpu' | awk '{print $2}')}
+		SPDK_RUN_ASAN=0
+		SPDK_RUN_UBSAN=0
+		;;
+	Linux)
+		DPDK_LINUX_DIR=/usr/share/dpdk/x86_64-default-linuxapp-gcc
+		if [ -d $DPDK_LINUX_DIR ] && [ $SPDK_RUN_INSTALLED_DPDK -eq 1 ]; then
+			WITH_DPDK_DIR=$DPDK_LINUX_DIR
+		fi
+		MAKE=make
+		MAKEFLAGS=${MAKEFLAGS:--j$(nproc)}
+		config_params+=' --enable-coverage'
+		if [ $SPDK_RUN_UBSAN -eq 1 ]; then
+			config_params+=' --enable-ubsan'
+		fi
+		if [ $SPDK_RUN_ASAN -eq 1 ]; then
+			if ldconfig -p | grep -q asan; then
+				config_params+=' --enable-asan'
+			else
+				SPDK_RUN_ASAN=0
+			fi
+		fi
+		;;
+	*)
+		echo "Unknown OS in $0"
+		exit 1
+		;;
+esac
+
+# By default, --with-dpdk is not set meaning the SPDK build will use the DPDK submodule.
+# If a DPDK installation is found in a well-known location though, WITH_DPDK_DIR will be
+# set which will override the default and use that DPDK installation instead.
+if [ ! -z "$WITH_DPDK_DIR" ]; then
+	config_params+=" --with-dpdk=$WITH_DPDK_DIR"
+fi
+
+if [ -f /usr/include/infiniband/verbs.h ]; then
+	config_params+=' --with-rdma'
+fi
+
+if [ -f /usr/include/libpmemblk.h ]; then
+	config_params+=' --with-pmdk'
+else
+	# PMDK not installed so disable PMDK tests explicitly here
+	SPDK_TEST_PMDK=0; export SPDK_TEST_PMDK
+fi
+
+if [ -d /usr/src/fio ]; then
+	config_params+=' --with-fio=/usr/src/fio'
+fi
+
+if [ -d ${DEPENDENCY_DIR}/vtune_codes ]; then
+	config_params+=' --with-vtune='${DEPENDENCY_DIR}'/vtune_codes'
+fi
+
+if [ -d /usr/include/rbd ] &&  [ -d /usr/include/rados ]; then
+	config_params+=' --with-rbd'
+fi
+
+if [ -d /usr/include/iscsi ]; then
+	libiscsi_version=`grep LIBISCSI_API_VERSION /usr/include/iscsi/iscsi.h | head -1 | awk '{print $3}' | awk -F '(' '{print $2}' | awk -F ')' '{print $1}'`
+	if [ $libiscsi_version -ge 20150621 ]; then
+		config_params+=' --with-iscsi-initiator'
+	else
+		export SPDK_TEST_ISCSI_INITIATOR=0
+	fi
+else
+	export SPDK_TEST_ISCSI_INITIATOR=0
+fi
+
+if [ ! -d "${DEPENDENCY_DIR}/nvme-cli" ]; then
+	export SPDK_TEST_NVME_CLI=0
+fi
+
+export config_params
+
+if [ -z "$output_dir" ]; then
+	if [ -z "$rootdir" ] || [ ! -d "$rootdir/../output" ]; then
+		output_dir=.
+	else
+		output_dir=$rootdir/../output
+	fi
+	export output_dir
+fi
+
+function timing() {
+	direction="$1"
+	testname="$2"
+
+	now=$(date +%s)
+
+	if [ "$direction" = "enter" ]; then
+		export timing_stack="${timing_stack};${now}"
+		export test_stack="${test_stack};${testname}"
+	else
+		child_time=$(grep "^${test_stack:1};" $output_dir/timing.txt | awk '{s+=$2} END {print s}')
+
+		start_time=$(echo "$timing_stack" | sed -e 's@^.*;@@')
+		timing_stack=$(echo "$timing_stack" | sed -e 's@;[^;]*$@@')
+
+		elapsed=$((now - start_time - child_time))
+		echo "${test_stack:1} $elapsed" >> $output_dir/timing.txt
+
+		test_stack=$(echo "$test_stack" | sed -e 's@;[^;]*$@@')
+	fi
+}
+
+function timing_enter() {
+	set +x
+	timing "enter" "$1"
+	set -x
+}
+
+function timing_exit() {
+	set +x
+	timing "exit" "$1"
+	set -x
+}
+
+function timing_finish() {
+	flamegraph='/usr/local/FlameGraph/flamegraph.pl'
+	if [ -x "$flamegraph" ]; then
+		"$flamegraph" \
+			--title 'Build Timing' \
+			--nametype 'Step:' \
+			--countname seconds \
+			$output_dir/timing.txt \
+			>$output_dir/timing.svg
+	fi
+}
+
+function create_test_list() {
+	grep -rsh --exclude="autotest_common.sh" --exclude="$rootdir/test/common/autotest_common.sh" -e "report_test_completion" $rootdir | sed 's/report_test_completion//g; s/[[:blank:]]//g; s/"//g;' > $output_dir/all_tests.txt || true
+}
+
+function report_test_completion() {
+	echo "$1" >> $output_dir/test_completions.txt
+}
+
+function process_core() {
+	ret=0
+	for core in $(find . -type f \( -name 'core*' -o -name '*.core' \)); do
+		exe=$(eu-readelf -n "$core" | grep psargs | sed "s/.*psargs: \([^ \'\" ]*\).*/\1/")
+		if [[ ! -f "$exe" ]]; then
+			exe=$(eu-readelf -n "$core" | grep -oP -m1 "$exe.+")
+		fi
+		echo "exe for $core is $exe"
+		if [[ ! -z "$exe" ]]; then
+			if hash gdb; then
+				gdb -batch -ex "thread apply all bt full" $exe $core
+			fi
+			cp $exe $output_dir
+		fi
+		mv $core $output_dir
+		chmod a+r $output_dir/$core
+		ret=1
+	done
+	return $ret
+}
+
+function process_shm() {
+	type=$1
+	id=$2
+	if [ "$type" = "--pid" ]; then
+		id="pid${id}"
+	elif [ "$type" = "--id" ]; then
+		id="${id}"
+	else
+		echo "Please specify to search for pid or shared memory id."
+		return 1
+	fi
+
+	shm_files=$(find /dev/shm -name "*.${id}" -printf "%f\n")
+
+	if [[ -z $shm_files ]]; then
+		echo "SHM File for specified PID or shared memory id: ${id} not found!"
+		return 1
+	fi
+	for n in $shm_files; do
+		tar -C /dev/shm/ -cvzf $output_dir/${n}_shm.tar.gz ${n}
+	done
+	return 0
+}
+
+function waitforlisten() {
+	# $1 = process pid
+	if [ -z "$1" ]; then
+		exit 1
+	fi
+
+	rpc_addr="${2:-$DEFAULT_RPC_ADDR}"
+
+	echo "Waiting for process to start up and listen on UNIX domain socket $rpc_addr..."
+	# turn off trace for this loop
+	set +x
+	ret=1
+	while [ $ret -ne 0 ]; do
+		# if the process is no longer running, then exit the script
+		#  since it means the application crashed
+		if ! kill -s 0 $1; then
+			exit 1
+		fi
+
+		namespace=$(ip netns identify $1)
+		if [ -n "$namespace" ]; then
+			ns_cmd="ip netns exec $namespace"
+		fi
+
+		if hash ss; then
+			if $ns_cmd ss -lx | grep -q $rpc_addr; then
+				ret=0
+			fi
+		else
+			# if system doesn't have ss, just assume it has netstat
+			if $ns_cmd netstat -an -x | grep -iw LISTENING | grep -q $rpc_addr; then
+				ret=0
+			fi
+		fi
+	done
+	set -x
+}
+
+function waitfornbd() {
+	nbd_name=$1
+
+	for ((i=1; i<=20; i++)); do
+		if grep -q -w $nbd_name /proc/partitions; then
+			break
+		else
+			sleep 0.1
+		fi
+	done
+
+	# The nbd device is now recognized as a block device, but there can be
+	#  a small delay before we can start I/O to that block device.  So loop
+	#  here trying to read the first block of the nbd block device to a temp
+	#  file.  Note that dd returns success when reading an empty file, so we
+	#  need to check the size of the output file instead.
+	for ((i=1; i<=20; i++)); do
+		dd if=/dev/$nbd_name of=/tmp/nbdtest bs=4096 count=1 iflag=direct
+		size=`stat -c %s /tmp/nbdtest`
+		rm -f /tmp/nbdtest
+		if [ "$size" != "0" ]; then
+			return 0
+		else
+			sleep 0.1
+		fi
+	done
+
+	return 1
+}
+
+function killprocess() {
+	# $1 = process pid
+	if [ -z "$1" ]; then
+		exit 1
+	fi
+
+	if kill -0 $1; then
+		echo "killing process with pid $1"
+		kill $1
+		wait $1
+	fi
+}
+
+function iscsicleanup() {
+	echo "Cleaning up iSCSI connection"
+	iscsiadm -m node --logout || true
+	iscsiadm -m node -o delete || true
+}
+
+function stop_iscsi_service() {
+	if cat /etc/*-release | grep Ubuntu; then
+		service open-iscsi stop
+	else
+		service iscsid stop
+	fi
+}
+
+function start_iscsi_service() {
+	if cat /etc/*-release | grep Ubuntu; then
+		service open-iscsi start
+	else
+		service iscsid start
+	fi
+}
+
+function rbd_setup() {
+	# $1 = monitor ip address
+	# $2 = name of the namespace
+	if [ -z "$1" ]; then
+		echo "No monitor IP address provided for ceph"
+		exit 1
+	fi
+	if [ -n "$2" ]; then
+		if ip netns list | grep "$2"; then
+			NS_CMD="ip netns exec $2"
+		else
+			echo "No namespace $2 exists"
+			exit 1
+		fi
+	fi
+
+	if hash ceph; then
+		export PG_NUM=128
+		export RBD_POOL=rbd
+		export RBD_NAME=foo
+		$NS_CMD $rootdir/scripts/ceph/start.sh $1
+
+		$NS_CMD ceph osd pool create $RBD_POOL $PG_NUM || true
+		$NS_CMD rbd create $RBD_NAME --size 1000
+	fi
+}
+
+function rbd_cleanup() {
+	if hash ceph; then
+		$rootdir/scripts/ceph/stop.sh || true
+	fi
+}
+
+function start_stub() {
+	# Disable ASLR for multi-process testing.  SPDK does support using DPDK multi-process,
+	# but ASLR can still be unreliable in some cases.
+	# We will reenable it again after multi-process testing is complete in kill_stub()
+	echo 0 > /proc/sys/kernel/randomize_va_space
+	$rootdir/test/app/stub/stub $1 &
+	stubpid=$!
+	echo Waiting for stub to ready for secondary processes...
+	while ! [ -e /var/run/spdk_stub0 ]; do
+		sleep 1s
+	done
+	# dump process memory map contents to help debug random ASLR failures
+	pmap -pX $stubpid || pmap -x $stubpid || true
+	echo done.
+}
+
+function kill_stub() {
+	kill $stubpid
+	wait $stubpid
+	rm -f /var/run/spdk_stub0
+	# Re-enable ASLR now that we are done with multi-process testing
+	# Note: "1" enables ASLR w/o randomizing data segments, "2" adds data segment
+	#  randomizing and is the default on all recent Linux kernels
+	echo 2 > /proc/sys/kernel/randomize_va_space
+}
+
+function run_test() {
+	set +x
+	local test_type="$(echo $1 | tr 'a-z' 'A-Z')"
+	shift
+	echo "************************************"
+	echo "START TEST $test_type $@"
+	echo "************************************"
+	set -x
+	time "$@"
+	set +x
+	echo "************************************"
+	echo "END TEST $test_type $@"
+	echo "************************************"
+	set -x
+}
+
+function print_backtrace() {
+	# if errexit is not enabled, don't print a backtrace
+	[[ "$-" =~ e ]] || return 0
+
+	local shell_options="$-"
+	set +x
+	echo "========== Backtrace start: =========="
+	echo ""
+	for i in $(seq 1 $((${#FUNCNAME[@]} - 1))); do
+		local func="${FUNCNAME[$i]}"
+		local line_nr="${BASH_LINENO[$((i - 1))]}"
+		local src="${BASH_SOURCE[$i]}"
+		echo "in $src:$line_nr -> $func()"
+		echo "     ..."
+		nl -w 4 -ba -nln $src | grep -B 5 -A 5 "^$line_nr[^0-9]" | \
+			sed "s/^/   /g" | sed "s/^   $line_nr /=> $line_nr /g"
+		echo "     ..."
+	done
+	echo ""
+	echo "========== Backtrace end =========="
+	[[ "$shell_options" =~ x ]] && set -x
+	return 0
+}
+
+function part_dev_by_gpt () {
+	if [ $(uname -s) = Linux ] && hash sgdisk && modprobe nbd; then
+		conf=$1
+		devname=$2
+		rootdir=$3
+		operation=$4
+		local nbd_path=/dev/nbd0
+		local rpc_server=/var/tmp/spdk-gpt-bdevs.sock
+
+		if [ ! -e $conf ]; then
+			return 1
+		fi
+
+		if [ -z "$operation" ]; then
+			operation="create"
+		fi
+
+		cp $conf ${conf}.gpt
+		echo "[Gpt]" >> ${conf}.gpt
+		echo "  Disable Yes" >> ${conf}.gpt
+
+		$rootdir/test/app/bdev_svc/bdev_svc -r $rpc_server -i 0 -c ${conf}.gpt &
+		nbd_pid=$!
+		echo "Process nbd pid: $nbd_pid"
+		waitforlisten $nbd_pid $rpc_server
+
+		# Start bdev as a nbd device
+		$rootdir/scripts/rpc.py -s "$rpc_server" start_nbd_disk $devname $nbd_path
+
+		waitfornbd ${nbd_path:5}
+
+		if [ "$operation" = create ]; then
+			parted -s $nbd_path mklabel gpt mkpart first '0%' '50%' mkpart second '50%' '100%'
+
+			# change the GUID to SPDK GUID value
+			SPDK_GPT_GUID=`grep SPDK_GPT_PART_TYPE_GUID $rootdir/lib/bdev/gpt/gpt.h \
+				| awk -F "(" '{ print $2}' | sed 's/)//g' \
+				| awk -F ", " '{ print $1 "-" $2 "-" $3 "-" $4 "-" $5}' | sed 's/0x//g'`
+			sgdisk -t 1:$SPDK_GPT_GUID $nbd_path
+			sgdisk -t 2:$SPDK_GPT_GUID $nbd_path
+		elif [ "$operation" = reset ]; then
+			# clear the partition table
+			dd if=/dev/zero of=$nbd_path bs=4096 count=8 oflag=direct
+		fi
+
+		$rootdir/scripts/rpc.py -s "$rpc_server" stop_nbd_disk $nbd_path
+
+		killprocess $nbd_pid
+		rm -f ${conf}.gpt
+	fi
+
+	return 0
+}
+
+function discover_bdevs()
+{
+	local rootdir=$1
+	local config_file=$2
+	local rpc_server=/var/tmp/spdk-discover-bdevs.sock
+
+	if [ ! -e $config_file ]; then
+		echo "Invalid Configuration File: $config_file"
+		return -1
+	fi
+
+	# Start the bdev service to query for the list of available
+	# bdevs.
+	$rootdir/test/app/bdev_svc/bdev_svc -r $rpc_server -i 0 \
+		-c $config_file &>/dev/null &
+	stubpid=$!
+	while ! [ -e /var/run/spdk_bdev0 ]; do
+		sleep 1
+	done
+
+	# Get all of the bdevs
+	if [ -z "$rpc_server" ]; then
+		$rootdir/scripts/rpc.py get_bdevs
+	else
+		$rootdir/scripts/rpc.py -s "$rpc_server" get_bdevs
+	fi
+
+	# Shut down the bdev service
+	kill $stubpid
+	wait $stubpid
+	rm -f /var/run/spdk_bdev0
+}
+
+function waitforblk()
+{
+	local i=0
+	while ! lsblk -l -o NAME | grep -q -w $1; do
+		[ $i -lt 15 ] || break
+		i=$[$i+1]
+		sleep 1
+	done
+
+	if ! lsblk -l -o NAME | grep -q -w $1; then
+		return 1
+	fi
+
+	return 0
+}
+
+function fio_config_gen()
+{
+	local config_file=$1
+	local workload=$2
+
+	if [ -e "$config_file" ]; then
+		echo "Configuration File Already Exists!: $config_file"
+		return -1
+	fi
+
+	if [ -z "$workload" ]; then
+		workload=randrw
+	fi
+
+	touch $1
+
+	cat > $1 << EOL
+[global]
+thread=1
+group_reporting=1
+direct=1
+norandommap=1
+percentile_list=50:99:99.9:99.99:99.999
+time_based=1
+ramp_time=0
+EOL
+
+	if [ "$workload" == "verify" ]; then
+		echo "verify=sha1" >> $config_file
+		echo "rw=randwrite" >> $config_file
+	elif [ "$workload" == "trim" ]; then
+		echo "rw=trimwrite" >> $config_file
+	else
+		echo "rw=$workload" >> $config_file
+	fi
+}
+
+function fio_config_add_job()
+{
+	config_file=$1
+	filename=$2
+
+	if [ ! -e "$config_file" ]; then
+		echo "Configuration File Doesn't Exist: $config_file"
+		return -1
+	fi
+
+	if [ -z "$filename" ]; then
+		echo "No filename provided"
+		return -1
+	fi
+
+	echo "[job_$filename]" >> $config_file
+	echo "filename=$filename" >> $config_file
+}
+
+function get_lvs_free_mb()
+{
+	local lvs_uuid=$1
+	local lvs_info=$($rpc_py get_lvol_stores)
+	local fc=$(jq ".[] | select(.uuid==\"$lvs_uuid\") .free_clusters" <<< "$lvs_info")
+	local cs=$(jq ".[] | select(.uuid==\"$lvs_uuid\") .cluster_size" <<< "$lvs_info")
+
+	# Change to MB's
+	free_mb=$((fc*cs/1024/1024))
+	echo "$free_mb"
+}
+
+function get_bdev_size()
+{
+	local bdev_name=$1
+	local bdev_info=$($rpc_py get_bdevs -b $bdev_name)
+	local bs=$(jq ".[] .block_size" <<< "$bdev_info")
+	local nb=$(jq ".[] .num_blocks" <<< "$bdev_info")
+
+	# Change to MB's
+	bdev_size=$((bs*nb/1024/1024))
+	echo "$bdev_size"
+}
+
+function autotest_cleanup()
+{
+	$rootdir/scripts/setup.sh reset
+	$rootdir/scripts/setup.sh cleanup
+	if [ $(uname -s) = "Linux" ]; then
+		if grep -q '#define SPDK_CONFIG_IGB_UIO_DRIVER 1' $rootdir/include/spdk/config.h; then
+			rmmod igb_uio
+		else
+			modprobe -r uio_pci_generic
+		fi
+	fi
+}
+
+function freebsd_update_contigmem_mod()
+{
+	if [ `uname` = FreeBSD ]; then
+		kldunload contigmem.ko || true
+		if [ ! -z "$WITH_DPDK_DIR" ]; then
+			echo "Warning: SPDK only works on FreeBSD with patches that only exist in SPDK's dpdk submodule"
+			cp -f "$WITH_DPDK_DIR/kmod/contigmem.ko" /boot/modules/
+			cp -f "$WITH_DPDK_DIR/kmod/contigmem.ko" /boot/kernel/
+		else
+			cp -f "$rootdir/dpdk/build/kmod/contigmem.ko" /boot/modules/
+			cp -f "$rootdir/dpdk/build/kmod/contigmem.ko" /boot/kernel/
+		fi
+	fi
+}
+
+set -o errtrace
+trap "trap - ERR; print_backtrace >&2" ERR
diff --git a/src/spdk/test/common/config/README.md b/src/spdk/test/common/config/README.md
new file mode 100644
index 00000000..609f9de8
--- /dev/null
+++ b/src/spdk/test/common/config/README.md
@@ -0,0 +1,99 @@
+# Virtual Test Configuration
+
+This readme and the associated bash script, vm_setup.sh, are intended to assist developers in quickly
+preparing a virtual test environment on which to run the SPDK validation tests rooted at autorun.sh.
+This file contains basic information about SPDK environment requirements, an introduction to the
+autorun-spdk.conf files used to moderate which tests are run by autorun.sh, and step-by-step instructions
+for spinning up a VM capable of running the SPDK test suite.
+There is no need for external hardware to run these tests. The linux kernel comes with the drivers necessary
+to emulate an RDMA enabled NIC. NVMe controllers can also be virtualized in emulators such as QEMU.
+
+
+## VM Envronment Requirements (Host):
+- 8 GiB of RAM (for DPDK)
+- Enable intel_kvm on the host machine from the bios.
+- Enable nesting for VMs in kernel command line (for vhost tests).
+  - In `/etc/default/grub` append the following to the GRUB_CMDLINE_LINUX line: intel_iommu=on kvm-intel.nested=1.
+
+## VM Specs
+When creating the user during the fedora installation, it is best to use the name sys_sgsw. Efforts are being made
+to remove all references to this user, or files specific to this user from the codebase, but there are still some
+trailing references to it.
+
+## Autorun-spdk.conf
+Every machine that runs the autotest scripts should include a file titled autorun-spdk.conf in the home directory
+of the user that will run them. This file consists of several lines of the form 'variable_name=0/1'. autorun.sh sources
+this file each time it is run, and determines which tests to attempt based on which variables are defined in the
+configuration file. For a full list of the variable declarations available for autorun-spdk.conf, please see
+`test/common/autotest_common.sh` starting at line 13.
+
+## Steps for Configuring the VM
+1. Download a fresh Fedora 26 image.
+2. Perform the installation of Fedora 26 server.
+3. Create an admin user sys_sgsw (enabling passwordless sudo for this account will make life easier during the tests).
+4. Run the vm_setup.sh script which will install all proper dependencies.
+5. Modify the autorun-spdk.conf file in the home directory.
+6. Reboot the VM.
+7. Run autorun.sh for SPDK. Any output files will be placed in `~/spdk_repo/output/`.
+
+## Additional Steps for Preparing the Vhost Tests
+The Vhost tests also require the creation of a second virtual machine nested inside of the test VM.
+Please follow the directions below to complete that installation. Note that host refers to the Fedora VM
+created above and guest or VM refer to the Ubuntu VM created in this section.
+
+1. Follow instructions from spdk/scripts/vagrant/README.md
+	- install all needed packages mentioned in "Mac OSX Setup" or "Windows 10 Setup" sections
+	- follow steps from "Configure Vagrant" section
+
+2. Use Vagrant scripts located in spdk/scripts/vagrant to automatically generate
+	VM image to use in SPDK vhost tests.
+	Example command:
+	~~~{.sh}
+	spdk/scripts/vagrant/create_vhost_vm.sh --move-to-def-dirs ubuntu16
+	~~~
+	This command will:
+		- Download a Ubuntu 16.04 image file
+		- upgrade the system and install needed dependencies (fio, sg3-utils, bc)
+		- add entry to VM's ~/.ssh/autorized_keys
+		- add appropriate options to GRUB command line and update grub
+		- convert the image to .qcow2 format
+		- move .qcow2 file and ssh keys to default locations used by vhost test scripts
+
+Alternatively it is possible to create the VM image manually using following steps:
+1. Create an image file for the VM. It does not have to be large, about 3.5G should suffice.
+2. Create an ssh keypair for host-guest communications (performed on the host):
+    - Generate an ssh keypair with the name spdk_vhost_id_rsa and save it in `/root/.ssh`.
+    - Make sure that only root has read access to the private key.
+3. Install the OS in the VM image (performed on guest):
+    - Use the latest Ubuntu server (Currently 16.04 LTS).
+    - When partitioning the disk, make one partion that consumes the whole disk mounted at /. Do not encrypt the disk or enable LVM.
+    - Choose the OpenSSH server packages during install.
+4. Post installation configuration (performed on guest):
+    - Run the following commands to enable all necessary dependencies:
+      ~~~{.sh}
+      sudo apt update
+      sudo apt upgrade
+      sudo apt install fio sg3-utils bc
+      ~~~
+    - Enable the root user: "sudo passwd root -> root".
+    - Enable root login over ssh: vim `/etc/ssh/sshd_config` -> PermitRootLogin=yes.
+    - Disable DNS for ssh: `/etc/ssh/sshd_config` -> UseDNS=no.
+    - Add the spdk_vhost key to root's known hosts: `/root/.ssh/authorized_keys` -> add spdk_vhost_id_rsa.pub key to authorized keys.
+    Remember to save the private key in `~/.ssh/spdk_vhost_id_rsa` on the host.
+    - Change the grub boot options for the guest as follows:
+      - Add "console=ttyS0 earlyprintk=ttyS0" to the boot options in `/etc/default/grub` (for serial output redirect).
+      - Add "scsi_mod.use_blk_mq=1" to boot options in `/etc/default/grub`.
+      ~~~{.sh}
+      sudo update-grub
+      ~~~
+    - Reboot the VM.
+    - Remove any unnecessary packages (this is to make booting the VM faster):
+      ~~~{.sh}
+      apt purge snapd
+      apt purge Ubuntu-core-launcher
+      apt purge squashfs-tools
+      apt purge unattended-upgrades
+      ~~~
+5. Copy the fio binary from the guest location `/usr/bin/fio` to the host location `/home/sys_sgsw/fio_ubuntu`.
+6. Place the guest VM in the host at the following location: `/home/sys_sgsw/vhost_vm_image.qcow2`.
+7. On the host, edit the `~/autorun-spdk.conf` file to include the following line: SPDK_TEST_VHOST=1.
diff --git a/src/spdk/test/common/config/vm_setup.conf b/src/spdk/test/common/config/vm_setup.conf
new file mode 100644
index 00000000..40acea66
--- /dev/null
+++ b/src/spdk/test/common/config/vm_setup.conf
@@ -0,0 +1,12 @@
+# This configuration file is provided for reference purposes.
+GIT_REPO_SPDK=https://review.gerrithub.io/spdk/spdk
+GIT_REPO_DPDK=https://github.com/spdk/dpdk.git
+GIT_REPO_LIBRXE=https://github.com/SoftRoCE/librxe-dev.git
+GIT_REPO_OPEN_ISCSI=https://github.com/open-iscsi/open-iscsi
+GIT_REPO_ROCKSDB=https://review.gerrithub.io/spdk/rocksdb
+GIT_REPO_FIO=http://git.kernel.dk/fio.git
+GIT_REPO_FLAMEGRAPH=https://github.com/brendangregg/FlameGraph.git
+GIT_REPO_QEMU=https://github.com/spdk/qemu
+GIT_REPO_VPP=https://gerrit.fd.io/r/vpp
+GIT_REPO_LIBISCSI=https://github.com/sahlberg/libiscsi
+GIT_REPO_SPDK_NVME_CLI=https://github.com/spdk/nvme-cli
diff --git a/src/spdk/test/common/config/vm_setup.sh b/src/spdk/test/common/config/vm_setup.sh
new file mode 100755
index 00000000..e01b8879
--- /dev/null
+++ b/src/spdk/test/common/config/vm_setup.sh
@@ -0,0 +1,424 @@
+#!/usr/bin/env bash
+
+# Virtual Machine environment requirements:
+# 8 GiB of RAM (for DPDK)
+# enable intel_kvm on your host machine
+
+# The purpose of this script is to provide a simple procedure for spinning up a new
+# virtual test environment capable of running our whole test suite. This script, when
+# applied to a fresh install of fedora 26 server will install all of the necessary dependencies
+# to run almost the complete test suite. The main exception being VHost. Vhost requires the
+# configuration of a second virtual machine. instructions for how to configure
+# that vm are included in the file TEST_ENV_SETUP_README inside this repository
+
+# it is important to enable nesting for vms in kernel command line of your machine for the vhost tests.
+#     in /etc/default/grub
+#     append the following to the GRUB_CMDLINE_LINUX line
+#     intel_iommu=on kvm-intel.nested=1
+
+# We have made a lot of progress with removing hardcoded paths from the tests,
+
+set -e
+
+VM_SETUP_PATH=$(readlink -f ${BASH_SOURCE%/*})
+
+UPGRADE=false
+INSTALL=false
+CONF="librxe,iscsi,rocksdb,fio,flamegraph,tsocks,qemu,vpp,libiscsi,nvmecli"
+
+function install_rxe_cfg()
+{
+    if echo $CONF | grep -q librxe; then
+        # rxe_cfg is used in the NVMe-oF tests
+        # The librxe-dev repository provides a command line tool called rxe_cfg which makes it
+        # very easy to use Soft-RoCE. The build pool utilizes this command line tool in the absence
+        # of any real RDMA NICs to simulate one for the NVMe-oF tests.
+        if hash rxe_cfg 2> /dev/null; then
+            echo "rxe_cfg is already installed. skipping"
+        else
+            if [ -d librxe-dev ]; then
+                echo "librxe-dev source already present, not cloning"
+            else
+                git clone "${GIT_REPO_LIBRXE}"
+            fi
+
+            ./librxe-dev/configure --libdir=/usr/lib64/ --prefix=
+            make -C librxe-dev -j${jobs}
+            sudo make -C librxe-dev install
+        fi
+    fi
+}
+
+function install_iscsi_adm()
+{
+    if echo $CONF | grep -q iscsi; then
+        # iscsiadm is used in the iscsi_tgt tests
+        # The version of iscsiadm that ships with fedora 26 was broken as of November 3 2017.
+        # There is already a bug report out about it, and hopefully it is fixed soon, but in the event that
+        # that version is still broken when you do your setup, the below steps will fix the issue.
+        CURRENT_VERSION=$(iscsiadm --version)
+        OPEN_ISCSI_VER='iscsiadm version 6.2.0.874'
+        if [ "$CURRENT_VERSION" == "$OPEN_ISCSI_VER" ]; then
+            if [ ! -d open-iscsi-install ]; then
+                mkdir -p open-iscsi-install/patches
+                sudo dnf download --downloaddir=./open-iscsi-install --source iscsi-initiator-utils
+                rpm2cpio open-iscsi-install/$(ls ~/open-iscsi-install) | cpio -D open-iscsi-install -idmv
+                mv open-iscsi-install/00* open-iscsi-install/patches/
+                git clone "${GIT_REPO_OPEN_ISCSI}" open-iscsi-install/open-iscsi
+
+                # the configurations of username and email are needed for applying patches to iscsiadm.
+                git -C open-iscsi-install/open-iscsi config user.name none
+                git -C open-iscsi-install/open-iscsi config user.email none
+
+                git -C open-iscsi-install/open-iscsi checkout 86e8892
+                for patch in `ls open-iscsi-install/patches`; do
+                    git -C open-iscsi-install/open-iscsi am ../patches/$patch
+                done
+                sed -i '427s/.*/-1);/' open-iscsi-install/open-iscsi/usr/session_info.c
+                make -C open-iscsi-install/open-iscsi -j${jobs}
+                sudo make -C open-iscsi-install/open-iscsi install
+            else
+                echo "custom open-iscsi install located, not reinstalling"
+            fi
+        fi
+    fi
+}
+
+function install_rocksdb()
+{
+    if echo $CONF | grep -q rocksdb; then
+        # Rocksdb is installed for use with the blobfs tests.
+        if [ ! -d /usr/src/rocksdb ]; then
+            git clone "${GIT_REPO_ROCKSDB}"
+            git -C ./rocksdb checkout spdk-v5.6.1
+            sudo mv rocksdb /usr/src/
+        else
+            sudo git -C /usr/src/rocksdb checkout spdk-v5.6.1
+            echo "rocksdb already in /usr/src. Not checking out again"
+        fi
+    fi
+}
+
+function install_fio()
+{
+    if echo $CONF | grep -q fio; then
+        # This version of fio is installed in /usr/src/fio to enable
+        # building the spdk fio plugin.
+        if [ ! -d /usr/src/fio ]; then
+            if [ ! -d fio ]; then
+                git clone "${GIT_REPO_FIO}"
+                sudo mv fio /usr/src/
+            else
+                sudo mv fio /usr/src/
+            fi
+            (
+                git -C /usr/src/fio checkout master &&
+                git -C /usr/src/fio pull &&
+                git -C /usr/src/fio checkout fio-3.3 &&
+                make -C /usr/src/fio -j${jobs} &&
+                sudo make -C /usr/src/fio install
+            )
+        else
+            echo "fio already in /usr/src/fio. Not installing"
+        fi
+    fi
+}
+
+function install_flamegraph()
+{
+    if echo $CONF | grep -q flamegraph; then
+        # Flamegraph is used when printing out timing graphs for the tests.
+        if [ ! -d /usr/local/FlameGraph ]; then
+            git clone "${GIT_REPO_FLAMEGRAPH}"
+            mkdir -p /usr/local
+            sudo mv FlameGraph /usr/local/FlameGraph
+        else
+            echo "flamegraph already installed. Skipping"
+        fi
+    fi
+}
+
+function install_qemu()
+{
+    if echo $CONF | grep -q qemu; then
+        # Qemu is used in the vhost tests.
+        SPDK_QEMU_BRANCH=spdk-2.12
+        mkdir -p qemu
+        if [ ! -d "qemu/$SPDK_QEMU_BRANCH" ]; then
+            git -C ./qemu clone "${GIT_REPO_QEMU}" -b "$SPDK_QEMU_BRANCH" "$SPDK_QEMU_BRANCH"
+        else
+            echo "qemu already checked out. Skipping"
+        fi
+
+        declare -a opt_params=("--prefix=/usr/local/qemu/$SPDK_QEMU_BRANCH")
+
+        # Most tsocks proxies rely on a configuration file in /etc/tsocks.conf.
+        # If using tsocks, please make sure to complete this config before trying to build qemu.
+        if echo $CONF | grep -q tsocks; then
+            if hash tsocks 2> /dev/null; then
+                opt_params+=(--with-git='tsocks git')
+            fi
+        fi
+
+        # The qemu configure script places several output files in the CWD.
+        (cd qemu/$SPDK_QEMU_BRANCH && ./configure "${opt_params[@]}" --target-list="x86_64-softmmu" --enable-kvm --enable-linux-aio --enable-numa)
+
+        make -C ./qemu/$SPDK_QEMU_BRANCH -j${jobs}
+        sudo make -C ./qemu/$SPDK_QEMU_BRANCH install
+    fi
+}
+
+function install_vpp()
+{
+    if echo $CONF | grep -q vpp; then
+        # Vector packet processing (VPP) is installed for use with iSCSI tests.
+        # At least on fedora 28, the yum setup that vpp uses is deprecated and fails.
+        # The actions taken under the vpp_setup script are necessary to fix this issue.
+        if [ -d vpp_setup ]; then
+            echo "vpp setup already done."
+        else
+            echo "%_topdir  $HOME/vpp_setup/src/rpm" >> ~/.rpmmacros
+            sudo dnf install -y perl-generators
+            mkdir -p ~/vpp_setup/src/rpm
+            mkdir -p vpp_setup/src/rpm/BUILD vpp_setup/src/rpm/RPMS vpp_setup/src/rpm/SOURCES \
+            vpp_setup/src/rpm/SPECS vpp_setup/src/rpm/SRPMS
+            dnf download --downloaddir=./vpp_setup/src/rpm --source redhat-rpm-config
+            rpm -ivh ~/vpp_setup/src/rpm/redhat-rpm-config*
+            sed -i s/"Requires: (annobin if gcc)"//g ~/vpp_setup/src/rpm/SPECS/redhat-rpm-config.spec
+            rpmbuild -ba ~/vpp_setup/src/rpm/SPECS/*.spec
+            sudo dnf remove -y --noautoremove redhat-rpm-config
+            sudo rpm -Uvh ~/vpp_setup/src/rpm/RPMS/noarch/*
+        fi
+
+        if [ -d vpp ]; then
+            echo "vpp already cloned."
+            if [ ! -d vpp/build-root ]; then
+                echo "build-root has not been done"
+                echo "remove the `pwd` and start again"
+                exit 1
+            fi
+        else
+            git clone "${GIT_REPO_VPP}"
+            git -C ./vpp checkout v18.01.1
+            # VPP 18.01.1 does not support OpenSSL 1.1.
+            # For compilation, a compatibility package is used temporarily.
+            sudo dnf install -y --allowerasing compat-openssl10-devel
+            # Installing required dependencies for building VPP
+            yes | make -C ./vpp install-dep
+
+            make -C ./vpp pkg-rpm -j${jobs}
+            # Reinstall latest OpenSSL devel package.
+            sudo dnf install -y --allowerasing openssl-devel
+            sudo dnf install -y \
+                ./vpp/build_root/vpp-lib-18.01.1-release.x86_64.rpm \
+                ./vpp/build_root/vpp-devel-18.01.1-release.x86_64.rpm \
+                ./vpp/build_root/vpp-18.01.1-release.x86_64.rpm
+            # Since hugepage configuration is done via spdk/scripts/setup.sh,
+            # this default config is not needed.
+            #
+            # NOTE: Parameters kernel.shmmax and vm.max_map_count are set to
+            # very low count and cause issues with hugepage total sizes above 1GB.
+            sudo rm -f /etc/sysctl.d/80-vpp.conf
+        fi
+    fi
+}
+
+function install_nvmecli()
+{
+    if echo $CONF | grep -q nvmecli; then
+        SPDK_NVME_CLI_BRANCH=spdk-1.6
+        if [ ! -d nvme-cli ]; then
+            git clone "${GIT_REPO_SPDK_NVME_CLI}" -b "$SPDK_NVME_CLI_BRANCH"
+        else
+            echo "nvme-cli already checked out. Skipping"
+        fi
+    fi
+}
+
+function install_libiscsi()
+{
+    if echo $CONF | grep -q libiscsi; then
+        # We currently don't make any changes to the libiscsi repository for our tests, but it is possible that we will need
+        # to later. Cloning from git is just future proofing the machines.
+        if [ ! -d libiscsi ]; then
+            git clone "${GIT_REPO_LIBISCSI}"
+        else
+            echo "libiscsi already checked out. Skipping"
+        fi
+        ( cd libiscsi && ./autogen.sh &&  ./configure --prefix=/usr/local/libiscsi)
+        make -C ./libiscsi -j${jobs}
+        sudo make -C ./libiscsi install
+    fi
+}
+
+function usage()
+{
+    echo "This script is intended to automate the environment setup for a fedora linux virtual machine."
+    echo "Please run this script as your regular user. The script will make calls to sudo as needed."
+    echo ""
+    echo "./vm_setup.sh"
+    echo "  -h --help"
+    echo "  -u --upgrade Run dnf upgrade"
+    echo "  -i --install-deps Install dnf based dependencies"
+    echo "  -t --test-conf List of test configurations to enable (${CONF})"
+    echo "  -c --conf-path Path to configuration file"
+    exit 0
+}
+
+while getopts 'iuht:c:-:' optchar; do
+    case "$optchar" in
+        -)
+        case "$OPTARG" in
+            help) usage;;
+            upgrade) UPGRADE=true;;
+            install-deps) INSTALL=true;;
+            test-conf=*) CONF="${OPTARG#*=}";;
+            conf-path=*) CONF_PATH="${OPTARG#*=}";;
+            *) echo "Invalid argument '$OPTARG'"
+            usage;;
+        esac
+        ;;
+    h) usage;;
+    u) UPGRADE=true;;
+    i) INSTALL=true;;
+    t) CONF="$OPTARG";;
+    c) CONF_PATH="$OPTARG";;
+    *) echo "Invalid argument '$OPTARG'"
+    usage;;
+    esac
+done
+
+if [ ! -z "$CONF_PATH" ]; then
+    if [ ! -f "$CONF_PATH" ]; then
+        echo Configuration file does not exist: "$CONF_PATH"
+        exit 1
+    else
+        source "$CONF_PATH"
+    fi
+fi
+
+cd ~
+
+: ${GIT_REPO_SPDK=https://review.gerrithub.io/spdk/spdk}; export GIT_REPO_SPDK
+: ${GIT_REPO_DPDK=https://github.com/spdk/dpdk.git}; export GIT_REPO_DPDK
+: ${GIT_REPO_LIBRXE=https://github.com/SoftRoCE/librxe-dev.git}; export GIT_REPO_LIBRXE
+: ${GIT_REPO_OPEN_ISCSI=https://github.com/open-iscsi/open-iscsi}; export GIT_REPO_OPEN_ISCSI
+: ${GIT_REPO_ROCKSDB=https://review.gerrithub.io/spdk/rocksdb}; export GIT_REPO_ROCKSDB
+: ${GIT_REPO_FIO=http://git.kernel.dk/fio.git}; export GIT_REPO_FIO
+: ${GIT_REPO_FLAMEGRAPH=https://github.com/brendangregg/FlameGraph.git}; export GIT_REPO_FLAMEGRAPH
+: ${GIT_REPO_QEMU=https://github.com/spdk/qemu}; export GIT_REPO_QEMU
+: ${GIT_REPO_VPP=https://gerrit.fd.io/r/vpp}; export GIT_REPO_VPP
+: ${GIT_REPO_LIBISCSI=https://github.com/sahlberg/libiscsi}; export GIT_REPO_LIBISCSI
+: ${GIT_REPO_SPDK_NVME_CLI=https://github.com/spdk/nvme-cli}; export GIT_REPO_SPDK_NVME_CLI
+
+jobs=$(($(nproc)*2))
+
+if $UPGRADE; then
+    sudo dnf upgrade -y
+fi
+
+if $INSTALL; then
+    sudo dnf install -y git
+fi
+
+mkdir -p spdk_repo/output
+
+if [ -d spdk_repo/spdk ]; then
+    echo "spdk source already present, not cloning"
+else
+    git -C spdk_repo clone "${GIT_REPO_SPDK}"
+fi
+git -C spdk_repo/spdk config submodule.dpdk.url "${GIT_REPO_DPDK}"
+git -C spdk_repo/spdk submodule update --init --recursive
+
+if $INSTALL; then
+    sudo ./scripts/pkgdep.sh
+
+    if echo $CONF | grep -q tsocks; then
+        sudo dnf install -y tsocks
+    fi
+
+    sudo dnf install -y \
+    valgrind \
+    jq \
+    nvme-cli \
+    ceph \
+    gdb \
+    fio \
+    librbd-devel \
+    kernel-devel \
+    gflags-devel \
+    libasan \
+    libubsan \
+    autoconf \
+    automake \
+    libtool \
+    libmount-devel \
+    iscsi-initiator-utils \
+    isns-utils-devel \
+    pmempool \
+    perl-open \
+    glib2-devel \
+    pixman-devel \
+    astyle-devel \
+    elfutils \
+    elfutils-libelf-devel \
+    flex \
+    bison \
+    targetcli \
+    perl-Switch \
+    librdmacm-utils \
+    libibverbs-utils \
+    gdisk \
+    socat \
+    sshfs
+fi
+
+sudo mkdir -p /usr/src
+
+install_rxe_cfg&
+install_iscsi_adm&
+install_rocksdb&
+install_fio&
+install_flamegraph&
+install_qemu&
+install_vpp&
+install_nvmecli&
+install_libiscsi&
+
+wait
+# create autorun-spdk.conf in home folder. This is sourced by the autotest_common.sh file.
+# By setting any one of the values below to 0, you can skip that specific test. If you are
+# using your autotest platform to do sanity checks before uploading to the build pool, it is
+# probably best to only run the tests that you believe your changes have modified along with
+# Scanbuild and check format. This is because running the whole suite of tests in series can
+# take ~40 minutes to complete.
+if [ ! -e ~/autorun-spdk.conf ]; then
+	cat > ~/autorun-spdk.conf << EOF
+# assign a value of 1 to all of the pertinent tests
+SPDK_BUILD_DOC=1
+SPDK_RUN_CHECK_FORMAT=1
+SPDK_RUN_SCANBUILD=1
+SPDK_RUN_VALGRIND=1
+SPDK_TEST_UNITTEST=1
+SPDK_TEST_ISCSI=1
+SPDK_TEST_ISCSI_INITIATOR=1
+# nvme and nvme-cli cannot be run at the same time on a VM.
+SPDK_TEST_NVME=1
+SPDK_TEST_NVME_CLI=0
+SPDK_TEST_NVMF=1
+SPDK_TEST_RBD=1
+# requires some extra configuration. see TEST_ENV_SETUP_README
+SPDK_TEST_VHOST=0
+SPDK_TEST_VHOST_INIT=0
+SPDK_TEST_BLOCKDEV=1
+# doesn't work on vm
+SPDK_TEST_IOAT=0
+SPDK_TEST_EVENT=1
+SPDK_TEST_BLOBFS=1
+SPDK_TEST_PMDK=1
+SPDK_TEST_LVOL=1
+SPDK_RUN_ASAN=1
+SPDK_RUN_UBSAN=1
+EOF
+fi
diff --git a/src/spdk/test/common/lib/test_env.c b/src/spdk/test/common/lib/test_env.c
new file mode 100644
index 00000000..4c600516
--- /dev/null
+++ b/src/spdk/test/common/lib/test_env.c
@@ -0,0 +1,469 @@
+/*-
+ *   BSD LICENSE
+ *
+ *   Copyright (c) Intel Corporation.
+ *   All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Intel Corporation nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "spdk/stdinc.h"
+
+#include "spdk_internal/mock.h"
+
+#include "spdk/env.h"
+#include "spdk/queue.h"
+
+DEFINE_STUB(spdk_process_is_primary, bool, (void), true)
+DEFINE_STUB(spdk_memzone_lookup, void *, (const char *name), NULL)
+
+/*
+ * These mocks don't use the DEFINE_STUB macros because
+ * their default implementation is more complex.
+ */
+
+DEFINE_RETURN_MOCK(spdk_memzone_reserve, void *);
+void *
+spdk_memzone_reserve(const char *name, size_t len, int socket_id, unsigned flags)
+{
+	HANDLE_RETURN_MOCK(spdk_memzone_reserve);
+
+	return malloc(len);
+}
+
+DEFINE_RETURN_MOCK(spdk_memzone_reserve_aligned, void *);
+void *
+spdk_memzone_reserve_aligned(const char *name, size_t len, int socket_id,
+			     unsigned flags, unsigned align)
+{
+	HANDLE_RETURN_MOCK(spdk_memzone_reserve_aligned);
+
+	return malloc(len);
+}
+
+DEFINE_RETURN_MOCK(spdk_malloc, void *);
+void *
+spdk_malloc(size_t size, size_t align, uint64_t *phys_addr, int socket_id, uint32_t flags)
+{
+	HANDLE_RETURN_MOCK(spdk_malloc);
+
+	void *buf = NULL;
+	if (posix_memalign(&buf, align, size)) {
+		return NULL;
+	}
+	if (phys_addr) {
+		*phys_addr = (uint64_t)buf;
+	}
+
+	return buf;
+}
+
+DEFINE_RETURN_MOCK(spdk_zmalloc, void *);
+void *
+spdk_zmalloc(size_t size, size_t align, uint64_t *phys_addr, int socket_id, uint32_t flags)
+{
+	HANDLE_RETURN_MOCK(spdk_zmalloc);
+
+	void *buf = spdk_malloc(size, align, phys_addr, -1, 1);
+
+	if (buf != NULL) {
+		memset(buf, 0, size);
+	}
+	return buf;
+}
+
+DEFINE_RETURN_MOCK(spdk_dma_malloc, void *);
+void *
+spdk_dma_malloc(size_t size, size_t align, uint64_t *phys_addr)
+{
+	HANDLE_RETURN_MOCK(spdk_dma_malloc);
+
+	return spdk_malloc(size, align, phys_addr, -1, 1);
+}
+
+DEFINE_RETURN_MOCK(spdk_dma_zmalloc, void *);
+void *
+spdk_dma_zmalloc(size_t size, size_t align, uint64_t *phys_addr)
+{
+	HANDLE_RETURN_MOCK(spdk_dma_zmalloc);
+
+	return spdk_zmalloc(size, align, phys_addr, -1, 1);
+}
+
+DEFINE_RETURN_MOCK(spdk_dma_malloc_socket, void *);
+void *
+spdk_dma_malloc_socket(size_t size, size_t align, uint64_t *phys_addr, int socket_id)
+{
+	HANDLE_RETURN_MOCK(spdk_dma_malloc_socket);
+
+	return spdk_dma_malloc(size, align, phys_addr);
+}
+
+DEFINE_RETURN_MOCK(spdk_dma_zmalloc_socket, void *);
+void *
+spdk_dma_zmalloc_socket(size_t size, size_t align, uint64_t *phys_addr, int socket_id)
+{
+	HANDLE_RETURN_MOCK(spdk_dma_zmalloc_socket);
+
+	return spdk_dma_zmalloc(size, align, phys_addr);
+}
+
+DEFINE_RETURN_MOCK(spdk_dma_realloc, void *);
+void *
+spdk_dma_realloc(void *buf, size_t size, size_t align, uint64_t *phys_addr)
+{
+	HANDLE_RETURN_MOCK(spdk_dma_realloc);
+
+	return realloc(buf, size);
+}
+
+void
+spdk_free(void *buf)
+{
+	free(buf);
+}
+
+void
+spdk_dma_free(void *buf)
+{
+	return spdk_free(buf);
+}
+
+DEFINE_RETURN_MOCK(spdk_vtophys, uint64_t);
+uint64_t
+spdk_vtophys(void *buf)
+{
+	HANDLE_RETURN_MOCK(spdk_vtophys);
+
+	return (uintptr_t)buf;
+}
+
+void
+spdk_memzone_dump(FILE *f)
+{
+	return;
+}
+
+DEFINE_RETURN_MOCK(spdk_memzone_free, int);
+int
+spdk_memzone_free(const char *name)
+{
+	HANDLE_RETURN_MOCK(spdk_memzone_free);
+
+	return 0;
+}
+
+struct test_mempool {
+	size_t	count;
+};
+
+DEFINE_RETURN_MOCK(spdk_mempool_create, struct spdk_mempool *);
+struct spdk_mempool *
+spdk_mempool_create(const char *name, size_t count,
+		    size_t ele_size, size_t cache_size, int socket_id)
+{
+	struct test_mempool *mp;
+
+	HANDLE_RETURN_MOCK(spdk_mempool_create);
+
+	mp = calloc(1, sizeof(*mp));
+	if (mp == NULL) {
+		return NULL;
+	}
+
+	mp->count = count;
+
+	return (struct spdk_mempool *)mp;
+}
+
+void
+spdk_mempool_free(struct spdk_mempool *_mp)
+{
+	struct test_mempool *mp = (struct test_mempool *)_mp;
+
+	free(mp);
+}
+
+DEFINE_RETURN_MOCK(spdk_mempool_get, void *);
+void *
+spdk_mempool_get(struct spdk_mempool *_mp)
+{
+	struct test_mempool *mp = (struct test_mempool *)_mp;
+	void *buf;
+
+	HANDLE_RETURN_MOCK(spdk_mempool_get);
+
+	if (mp && mp->count == 0) {
+		return NULL;
+	}
+
+	if (posix_memalign(&buf, 64, 0x1000)) {
+		return NULL;
+	} else {
+		if (mp) {
+			mp->count--;
+		}
+		return buf;
+	}
+}
+
+int
+spdk_mempool_get_bulk(struct spdk_mempool *mp, void **ele_arr, size_t count)
+{
+	for (size_t i = 0; i < count; i++) {
+		ele_arr[i] = spdk_mempool_get(mp);
+		if (ele_arr[i] == NULL) {
+			return -1;
+		}
+	}
+	return 0;
+}
+
+void
+spdk_mempool_put(struct spdk_mempool *_mp, void *ele)
+{
+	struct test_mempool *mp = (struct test_mempool *)_mp;
+
+	if (mp) {
+		mp->count++;
+	}
+	free(ele);
+}
+
+void
+spdk_mempool_put_bulk(struct spdk_mempool *mp, void **ele_arr, size_t count)
+{
+	for (size_t i = 0; i < count; i++) {
+		spdk_mempool_put(mp, ele_arr[i]);
+	}
+}
+
+DEFINE_RETURN_MOCK(spdk_mempool_count, size_t);
+size_t
+spdk_mempool_count(const struct spdk_mempool *_mp)
+{
+	struct test_mempool *mp = (struct test_mempool *)_mp;
+
+	HANDLE_RETURN_MOCK(spdk_mempool_count);
+
+	if (mp) {
+		return mp->count;
+	} else {
+		return 1024;
+	}
+}
+
+struct spdk_ring_ele {
+	void *ele;
+	TAILQ_ENTRY(spdk_ring_ele) link;
+};
+
+struct spdk_ring {
+	TAILQ_HEAD(, spdk_ring_ele) elements;
+};
+
+DEFINE_RETURN_MOCK(spdk_ring_create, struct spdk_ring *);
+struct spdk_ring *
+spdk_ring_create(enum spdk_ring_type type, size_t count, int socket_id)
+{
+	struct spdk_ring *ring;
+
+	HANDLE_RETURN_MOCK(spdk_ring_create);
+
+	ring = calloc(1, sizeof(*ring));
+	if (ring) {
+		TAILQ_INIT(&ring->elements);
+	}
+
+	return ring;
+}
+
+void
+spdk_ring_free(struct spdk_ring *ring)
+{
+	free(ring);
+}
+
+DEFINE_RETURN_MOCK(spdk_ring_enqueue, size_t);
+size_t
+spdk_ring_enqueue(struct spdk_ring *ring, void **objs, size_t count)
+{
+	struct spdk_ring_ele *ele;
+	size_t i;
+
+	HANDLE_RETURN_MOCK(spdk_ring_enqueue);
+
+	for (i = 0; i < count; i++) {
+		ele = calloc(1, sizeof(*ele));
+		if (!ele) {
+			break;
+		}
+
+		ele->ele = objs[i];
+		TAILQ_INSERT_TAIL(&ring->elements, ele, link);
+	}
+
+	return i;
+}
+
+DEFINE_RETURN_MOCK(spdk_ring_dequeue, size_t);
+size_t
+spdk_ring_dequeue(struct spdk_ring *ring, void **objs, size_t count)
+{
+	struct spdk_ring_ele *ele, *tmp;
+	size_t i = 0;
+
+	HANDLE_RETURN_MOCK(spdk_ring_dequeue);
+
+	if (count == 0) {
+		return 0;
+	}
+
+	TAILQ_FOREACH_SAFE(ele, &ring->elements, link, tmp) {
+		TAILQ_REMOVE(&ring->elements, ele, link);
+		objs[i] = ele->ele;
+		free(ele);
+		i++;
+		if (i >= count) {
+			break;
+		}
+	}
+
+	return i;
+
+}
+
+DEFINE_RETURN_MOCK(spdk_get_ticks, uint64_t);
+uint64_t
+spdk_get_ticks(void)
+{
+	HANDLE_RETURN_MOCK(spdk_get_ticks);
+
+	return ut_spdk_get_ticks;
+}
+
+DEFINE_RETURN_MOCK(spdk_get_ticks_hz, uint64_t);
+uint64_t
+spdk_get_ticks_hz(void)
+{
+	HANDLE_RETURN_MOCK(spdk_get_ticks_hz);
+
+	return 1000000;
+}
+
+void
+spdk_delay_us(unsigned int us)
+{
+	/* spdk_get_ticks_hz is 1000000, meaning 1 tick per us. */
+	ut_spdk_get_ticks += us;
+}
+
+DEFINE_RETURN_MOCK(spdk_pci_addr_parse, int);
+int
+spdk_pci_addr_parse(struct spdk_pci_addr *addr, const char *bdf)
+{
+	unsigned domain, bus, dev, func;
+
+	HANDLE_RETURN_MOCK(spdk_pci_addr_parse);
+
+	if (addr == NULL || bdf == NULL) {
+		return -EINVAL;
+	}
+
+	if ((sscanf(bdf, "%x:%x:%x.%x", &domain, &bus, &dev, &func) == 4) ||
+	    (sscanf(bdf, "%x.%x.%x.%x", &domain, &bus, &dev, &func) == 4)) {
+		/* Matched a full address - all variables are initialized */
+	} else if (sscanf(bdf, "%x:%x:%x", &domain, &bus, &dev) == 3) {
+		func = 0;
+	} else if ((sscanf(bdf, "%x:%x.%x", &bus, &dev, &func) == 3) ||
+		   (sscanf(bdf, "%x.%x.%x", &bus, &dev, &func) == 3)) {
+		domain = 0;
+	} else if ((sscanf(bdf, "%x:%x", &bus, &dev) == 2) ||
+		   (sscanf(bdf, "%x.%x", &bus, &dev) == 2)) {
+		domain = 0;
+		func = 0;
+	} else {
+		return -EINVAL;
+	}
+
+	if (bus > 0xFF || dev > 0x1F || func > 7) {
+		return -EINVAL;
+	}
+
+	addr->domain = domain;
+	addr->bus = bus;
+	addr->dev = dev;
+	addr->func = func;
+
+	return 0;
+}
+
+DEFINE_RETURN_MOCK(spdk_pci_addr_fmt, int);
+int
+spdk_pci_addr_fmt(char *bdf, size_t sz, const struct spdk_pci_addr *addr)
+{
+	int rc;
+
+	HANDLE_RETURN_MOCK(spdk_pci_addr_fmt);
+
+	rc = snprintf(bdf, sz, "%04x:%02x:%02x.%x",
+		      addr->domain, addr->bus,
+		      addr->dev, addr->func);
+
+	if (rc > 0 && (size_t)rc < sz) {
+		return 0;
+	}
+
+	return -1;
+}
+
+DEFINE_RETURN_MOCK(spdk_pci_addr_compare, int);
+int
+spdk_pci_addr_compare(const struct spdk_pci_addr *a1, const struct spdk_pci_addr *a2)
+{
+	HANDLE_RETURN_MOCK(spdk_pci_addr_compare);
+
+	if (a1->domain > a2->domain) {
+		return 1;
+	} else if (a1->domain < a2->domain) {
+		return -1;
+	} else if (a1->bus > a2->bus) {
+		return 1;
+	} else if (a1->bus < a2->bus) {
+		return -1;
+	} else if (a1->dev > a2->dev) {
+		return 1;
+	} else if (a1->dev < a2->dev) {
+		return -1;
+	} else if (a1->func > a2->func) {
+		return 1;
+	} else if (a1->func < a2->func) {
+		return -1;
+	}
+
+	return 0;
+}
diff --git a/src/spdk/test/common/lib/ut_multithread.c b/src/spdk/test/common/lib/ut_multithread.c
new file mode 100644
index 00000000..85fcee2a
--- /dev/null
+++ b/src/spdk/test/common/lib/ut_multithread.c
@@ -0,0 +1,278 @@
+/*-
+ *   BSD LICENSE
+ *
+ *   Copyright (c) Intel Corporation.
+ *   All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Intel Corporation nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "spdk_cunit.h"
+#include "spdk/thread.h"
+#include "spdk_internal/mock.h"
+
+static uint32_t g_ut_num_threads;
+static uint64_t g_current_time_in_us = 0;
+
+int allocate_threads(int num_threads);
+void free_threads(void);
+void poll_threads(void);
+int poll_thread(uintptr_t thread_id);
+void increment_time(uint64_t time_in_us);
+void reset_time(void);
+
+struct ut_msg {
+	spdk_thread_fn		fn;
+	void			*ctx;
+	TAILQ_ENTRY(ut_msg)	link;
+};
+
+struct ut_thread {
+	struct spdk_thread	*thread;
+	struct spdk_io_channel	*ch;
+	TAILQ_HEAD(, ut_msg)	msgs;
+	TAILQ_HEAD(, ut_poller)	pollers;
+};
+
+struct ut_thread *g_ut_threads;
+
+struct ut_poller {
+	spdk_poller_fn		fn;
+	void			*arg;
+	TAILQ_ENTRY(ut_poller)	tailq;
+	uint64_t		period_us;
+	uint64_t		next_expiration_in_us;
+};
+
+static void
+__send_msg(spdk_thread_fn fn, void *ctx, void *thread_ctx)
+{
+	struct ut_thread *thread = thread_ctx;
+	struct ut_msg *msg;
+
+	msg = calloc(1, sizeof(*msg));
+	SPDK_CU_ASSERT_FATAL(msg != NULL);
+
+	msg->fn = fn;
+	msg->ctx = ctx;
+	TAILQ_INSERT_TAIL(&thread->msgs, msg, link);
+}
+
+static struct spdk_poller *
+__start_poller(void *thread_ctx, spdk_poller_fn fn, void *arg, uint64_t period_microseconds)
+{
+	struct ut_thread *thread = thread_ctx;
+	struct ut_poller *poller = calloc(1, sizeof(struct ut_poller));
+
+	SPDK_CU_ASSERT_FATAL(poller != NULL);
+
+	poller->fn = fn;
+	poller->arg = arg;
+	poller->period_us = period_microseconds;
+	poller->next_expiration_in_us = g_current_time_in_us + poller->period_us;
+
+	TAILQ_INSERT_TAIL(&thread->pollers, poller, tailq);
+
+	return (struct spdk_poller *)poller;
+}
+
+static void
+__stop_poller(struct spdk_poller *poller, void *thread_ctx)
+{
+	struct ut_thread *thread = thread_ctx;
+
+	TAILQ_REMOVE(&thread->pollers, (struct ut_poller *)poller, tailq);
+
+	free(poller);
+}
+
+#define INVALID_THREAD 0x1000
+
+static uintptr_t g_thread_id = INVALID_THREAD;
+
+static void
+set_thread(uintptr_t thread_id)
+{
+	g_thread_id = thread_id;
+	if (thread_id == INVALID_THREAD) {
+		MOCK_CLEAR(pthread_self);
+	} else {
+		MOCK_SET(pthread_self, (pthread_t)thread_id);
+	}
+}
+
+int
+allocate_threads(int num_threads)
+{
+	struct spdk_thread *thread;
+	uint32_t i;
+
+	g_ut_num_threads = num_threads;
+
+	g_ut_threads = calloc(num_threads, sizeof(*g_ut_threads));
+	SPDK_CU_ASSERT_FATAL(g_ut_threads != NULL);
+
+	for (i = 0; i < g_ut_num_threads; i++) {
+		set_thread(i);
+		spdk_allocate_thread(__send_msg, __start_poller, __stop_poller,
+				     &g_ut_threads[i], NULL);
+		thread = spdk_get_thread();
+		SPDK_CU_ASSERT_FATAL(thread != NULL);
+		g_ut_threads[i].thread = thread;
+		TAILQ_INIT(&g_ut_threads[i].msgs);
+		TAILQ_INIT(&g_ut_threads[i].pollers);
+	}
+
+	set_thread(INVALID_THREAD);
+	return 0;
+}
+
+void
+free_threads(void)
+{
+	uint32_t i;
+
+	for (i = 0; i < g_ut_num_threads; i++) {
+		set_thread(i);
+		spdk_free_thread();
+	}
+
+	g_ut_num_threads = 0;
+	free(g_ut_threads);
+	g_ut_threads = NULL;
+}
+
+void
+increment_time(uint64_t time_in_us)
+{
+	g_current_time_in_us += time_in_us;
+	spdk_delay_us(time_in_us);
+}
+
+static void
+reset_pollers(void)
+{
+	uint32_t		i = 0;
+	struct ut_thread	*thread = NULL;
+	struct ut_poller	*poller = NULL;
+	uintptr_t		original_thread_id = g_thread_id;
+
+	CU_ASSERT(g_current_time_in_us == 0);
+
+	for (i = 0; i < g_ut_num_threads; i++) {
+		set_thread(i);
+		thread = &g_ut_threads[i];
+
+		TAILQ_FOREACH(poller, &thread->pollers, tailq) {
+			poller->next_expiration_in_us = g_current_time_in_us + poller->period_us;
+		}
+	}
+
+	set_thread(original_thread_id);
+}
+
+void
+reset_time(void)
+{
+	g_current_time_in_us = 0;
+	reset_pollers();
+}
+
+int
+poll_thread(uintptr_t thread_id)
+{
+	int count = 0;
+	struct ut_thread *thread = &g_ut_threads[thread_id];
+	struct ut_msg *msg;
+	struct ut_poller *poller;
+	uintptr_t original_thread_id;
+	TAILQ_HEAD(, ut_poller)	tmp_pollers;
+
+	CU_ASSERT(thread_id != (uintptr_t)INVALID_THREAD);
+	CU_ASSERT(thread_id < g_ut_num_threads);
+
+	original_thread_id = g_thread_id;
+	set_thread(thread_id);
+
+	while (!TAILQ_EMPTY(&thread->msgs)) {
+		msg = TAILQ_FIRST(&thread->msgs);
+		TAILQ_REMOVE(&thread->msgs, msg, link);
+
+		msg->fn(msg->ctx);
+		count++;
+		free(msg);
+	}
+
+	TAILQ_INIT(&tmp_pollers);
+
+	while (!TAILQ_EMPTY(&thread->pollers)) {
+		poller = TAILQ_FIRST(&thread->pollers);
+		TAILQ_REMOVE(&thread->pollers, poller, tailq);
+
+		if (g_current_time_in_us >= poller->next_expiration_in_us) {
+			if (poller->fn) {
+				poller->fn(poller->arg);
+			}
+
+			if (poller->period_us == 0) {
+				break;
+			} else {
+				poller->next_expiration_in_us += poller->period_us;
+			}
+		}
+
+		TAILQ_INSERT_TAIL(&tmp_pollers, poller, tailq);
+	}
+
+	TAILQ_SWAP(&tmp_pollers, &thread->pollers, ut_poller, tailq);
+
+	set_thread(original_thread_id);
+
+	return count;
+}
+
+void
+poll_threads(void)
+{
+	bool msg_processed;
+	uint32_t i, count;
+
+	while (true) {
+		msg_processed = false;
+
+		for (i = 0; i < g_ut_num_threads; i++) {
+			count = poll_thread(i);
+			if (count > 0) {
+				msg_processed = true;
+			}
+		}
+
+		if (!msg_processed) {
+			break;
+		}
+	}
+}