Adding upstream version 2.5.5.upstream/2.5.5upstream

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

1 files changed, 1127 insertions, 0 deletions

diff --git a/tests/regression b/tests/regression
new file mode 100755
index 0000000..f938b1b
--- /dev/null
+++ b/tests/regression
@@ -0,0 +1,1127 @@
+#!/bin/bash
+
+#
+# libseccomp regression test automation script
+#
+# Copyright IBM Corp. 2012
+# Author: Corey Bryant <coreyb@linux.vnet.ibm.com>
+#
+
+#
+# This library is free software; you can redistribute it and/or modify it
+# under the terms of version 2.1 of the GNU Lesser General Public License as
+# published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License
+# for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with this library; if not, see <http://www.gnu.org/licenses>.
+#
+
+GLBL_ARCH_LE_SUPPORT=" \
+	x86 x86_64 x32 \
+	arm aarch64 \
+	mipsel mipsel64 mipsel64n32 \
+	ppc64le \
+	riscv64"
+GLBL_ARCH_BE_SUPPORT=" \
+	mips mips64 mips64n32 \
+	parisc parisc64 \
+	ppc ppc64 \
+	s390 s390x"
+
+GLBL_ARCH_32B_SUPPORT=" \
+	x86 x32 \
+	arm \
+	mips mipsel mips64n32 mipsel64n32 \
+	parisc \
+	ppc \
+	s390"
+
+GLBL_ARCH_64B_SUPPORT=" \
+	x86_64 \
+	aarch64 \
+	mips64 \
+	parisc64 \
+	ppc64 \
+	riscv64 \
+	s390x"
+
+GLBL_SYS_ARCH="../tools/scmp_arch_detect"
+GLBL_SYS_RESOLVER="../tools/scmp_sys_resolver"
+GLBL_SYS_SIM="../tools/scmp_bpf_sim"
+GLBL_SYS_API="../tools/scmp_api_level"
+
+####
+# functions
+
+#
+# Dependency check
+#
+# Arguments:
+#     1    Dependency to check for
+#
+function check_deps() {
+	[[ -z "$1" ]] && return
+	which "$1" >& /dev/null
+	return $?
+}
+
+#
+# Dependency verification
+#
+# Arguments:
+#     1    Dependency to check for
+#
+function verify_deps() {
+	[[ -z "$1" ]] && return
+	if ! check_deps "$1"; then
+		echo "error: install \"$1\" and include it in your \$PATH"
+		exit 1
+	fi
+}
+
+#
+# Print out script usage details
+#
+function usage() {
+cat << EOF
+usage: regression [-h] [-v] [-m MODE] [-a] [-b BATCH_NAME] [-l <LOG>]
+                  [-s SINGLE_TEST] [-t <TEMP_DIR>] [-T <TEST_TYPE>]
+
+libseccomp regression test automation script
+optional arguments:
+  -h             show this help message and exit
+  -m MODE        specified the test mode [c (default), python]
+                  can also be set via LIBSECCOMP_TSTCFG_MODE_LIST env variable
+  -a             specifies all tests are to be run
+  -b BATCH_NAME  specifies batch of tests to be run
+                  can also be set via LIBSECCOMP_TSTCFG_BATCHES env variable
+  -l [LOG]       specifies log file to write test results to
+  -s SINGLE_TEST specifies individual test number to be run
+  -t [TEMP_DIR]  specifies directory to create temporary files in
+  -T [TEST_TYPE] only run tests matching the specified type
+                  can also be set via LIBSECCOMP_TSTCFG_TYPE env variable
+  -v             specifies that verbose output be provided
+EOF
+}
+
+#
+# Match on a single word/column in a CSV string
+#
+# Arguments:
+#     1    string containing the CSV
+#     2    string containing the word to match
+#
+# Returns true/0 if a match is found false/1 otherwise.
+#
+function match_csv_word() {
+	[[ -z $1 || -z $2 ]] && return 1
+
+	echo "$1" | sed 's/,/ /g' | grep -w "$2"
+}
+
+#
+# Generate a string representing the test number
+#
+# Arguments:
+#     1    string containing the batch name
+#     2    value of the test number from the input test data file
+#     3    value of the subtest number that corresponds to argument 1
+#
+#  The actual test number from the input test data file is 1 for the first
+#  test found in the file, 2 for the second, etc.
+#
+#  The subtest number is useful for batches that generate multiple tests based
+#  on a single line of input from the test data file.  The subtest number
+#  should be set to zero if the  corresponding test data is actual test data
+#  that was read from the input file, and should be set to a value greater than
+#  zero if the corresponding test data is generated.
+#
+function generate_test_num() {
+	local testnumstr=$(printf '%s%%%%%03d-%05d' "$1" $2 $3)
+	echo "$testnumstr"
+}
+
+#
+# Print the test data to the log file
+#
+# Arguments:
+#     1    string containing generated test number
+#     2    string containing line of test data
+#
+function print_data() {
+	if [[ -n $verbose ]]; then
+		printf "Test %s data:     %s\n" "$1" "$2" >&$logfd
+	fi
+}
+
+#
+# Print the test result to the log file
+#
+# Arguments:
+#     1    string containing generated test number
+#     2    string containing the test result (INFO, SUCCESS, ERROR, or FAILURE)
+#     3    string containing addition details
+#
+function print_result() {
+	if [[ $2 == "INFO" && -z $verbose ]]; then
+		return
+	fi
+	if [[ $3 == "" ]]; then
+		printf "Test %s result:   %s\n" "$1" "$2" >&$logfd
+	else
+		printf "Test %s result:   %s %s\n" "$1" "$2" "$3" >&$logfd
+	fi
+}
+
+#
+# Print the valgrind header to the log file
+#
+# Arguments:
+#     1    string containing generated test number
+#
+function print_valgrind() {
+	if [[ -n $verbose ]]; then
+		printf "Test %s valgrind output\n" "$1" >&$logfd
+	fi
+}
+
+#
+# Get the low or high range value from a range specification
+#
+# Arguments:
+#     1    value specifying range value to retrieve: low (1) or high (2)
+#     2    string containing dash-separated range or a single value
+#
+function get_range() {
+	if [[ $2 =~ ^[0-9a-fA-Fx]+-[0-9a-fA-Fx]+$ ]]; then
+		# if there's a dash, get the low or high range value
+		range_val=$(echo "$2" | cut -d'-' -f "$1")
+	else
+		# otherwise there should just be a single value
+		range_val="$2"
+	fi
+	echo "$range_val"
+}
+
+#
+# Get the number sequence for a given range with increments of 1, i.e.
+# implement a specialized seq(1).
+#
+# We use our own implementation based on miniseq in favour to the standard seq
+# tool as, at least, seq of coreutils v8.23 and v8.24 has problems on 32 bit
+# ARM for large numbers (see the mailing thread at
+# https://groups.google.com/forum/#!topic/libseccomp/VtrClkXxLGA).
+#
+# Arguments:
+#     1    starting value
+#     2    last value
+#
+function get_seq() {
+	# NOTE: this whole thing is a bit hacky, but we need to search around
+	#       for miniseq to fix 'make distcheck', someday we should fix this
+	if [[ -x ./miniseq ]]; then
+		./miniseq "$1" "$2"
+	elif [[ -x $basedir/miniseq ]]; then
+		$basedir/miniseq "$1" "$2"
+	else
+		# we're often run from a subshell, so we can't simply exit
+		echo "error: unable to find miniseq" >&2
+		kill $pid
+	fi
+}
+
+#
+# Run the specified test command (with valgrind if requested)
+#
+# Arguments:
+#     1    string containing generated test number
+#     2    string containing command name
+#     3    string containing command options
+#     4    number for the stdout fd
+#     5    number for the stderr fd
+#
+function run_test_command() {
+	local cmd
+
+	if [[ $mode == "python" ]]; then
+		cmd="PYTHONPATH=$PYTHONPATH"
+		cmd="$cmd:$(cd $(pwd)/../src/python/build/lib.*; pwd)"
+		# check and adjust if we are doing a VPATH build
+		if [[ -e "./$2.py" ]]; then
+			cmd="$cmd /usr/bin/env python $2.py $3"
+		else
+			cmd="$cmd /usr/bin/env python ${srcdir}/$2.py $3"
+		fi
+	else
+		cmd="$2 $3"
+	fi
+
+	# setup the stdout/stderr redirects
+	local stdout=$4
+	local stderr=$5
+	[[ -z $stdout ]] && stdout=$logfd
+	[[ -z $stderr ]] && stderr=$logfd
+
+	# run the command
+	eval "$cmd" 1>&$stdout 2>&$stderr
+
+	# return the command's return code
+	return $?
+}
+
+#
+# Generate pseudo-random string of alphanumeric characters
+#
+# The generated string will be no larger than the corresponding
+# architecture's register size.
+#
+function generate_random_data() {
+	local rcount
+	local rdata
+	if [[ $arch == "x86_64" ]]; then
+		rcount=$[ ($RANDOM % 16) + 1 ]
+	else
+		rcount=$[ ($RANDOM % 8) + 1 ]
+	fi
+	rdata=$(dd if=/dev/urandom bs=64 count=1 status=none | \
+		md5sum | awk '{ print $1 }' | head -c"$rcount")
+	echo "$rdata"
+}
+
+#
+# Run the specified "bpf-sim-fuzz" test
+#
+# Tests that belong to the "bpf-sim-fuzz" test type generate a BPF filter and
+# then run a simulated system call test with pseudo-random fuzz data for the
+# syscall and argument values.  Tests that belong to this test type provide the
+# following data on a single line in the input batch file:
+#
+#     Testname - The executable test name (e.g. 01-allow, 02-basic, etc.)
+#     StressCount - The number of fuzz tests to run against the filter
+#
+# The following test data is output to the logfile for each generated test:
+#
+#     Testname - The executable test name (e.g. 01-allow, 02-basic, etc.)
+#     Syscall - The fuzzed syscall value to be simulated against the filter
+#     Arg0-5 - The fuzzed syscall arg values to be simulated against the filter
+#
+# Arguments:
+#     1    string containing the batch name
+#     2    value of test number from batch file
+#     3    string containing line of test data from batch file
+#
+function run_test_bpf_sim_fuzz() {
+	local rc
+
+	# begin splitting the test data from the line into individual variables
+	local line=($3)
+	local testname=${line[0]}
+	local stress_count=${line[1]}
+
+	# check for stress count configuration via environment variables
+	[[ -n $LIBSECCOMP_TSTCFG_STRESSCNT ]] && \
+		stress_count=$LIBSECCOMP_TSTCFG_STRESSCNT
+
+	for i in $(get_seq 1 $stress_count); do
+		local sys=$(generate_random_data)
+		local -a arg=($(generate_random_data) $(generate_random_data) \
+			      $(generate_random_data) $(generate_random_data) \
+			      $(generate_random_data) $(generate_random_data))
+
+		# get the generated sub-test num string
+		local testnumstr=$(generate_test_num "$1" $2 $i)
+
+		# set up log file test data line for this individual test,
+		# spacing is added to align the output in the correct columns
+		local -a COL_WIDTH=(26 17 17 17 17 17 17)
+		local testdata=$(printf "%-${COL_WIDTH[0]}s" $testname)
+		testdata+=$(printf "%-${COL_WIDTH[1]}s" $sys)
+		testdata+=$(printf "%-${COL_WIDTH[2]}s" ${arg[0]})
+		testdata+=$(printf "%-${COL_WIDTH[3]}s" ${arg[1]})
+		testdata+=$(printf "%-${COL_WIDTH[4]}s" ${arg[2]})
+		testdata+=$(printf "%-${COL_WIDTH[5]}s" ${arg[3]})
+		testdata+=$(printf "%-${COL_WIDTH[6]}s" ${arg[4]})
+		testdata+=$(printf "%s" ${arg[5]})
+
+		# print out the generated test data to the log file
+		print_data "$testnumstr" "$testdata"
+
+		# set up the syscall argument values to be passed to bpf_sim
+		for i in {0..5}; do
+			arg[$i]=" -$i ${arg[$i]} "
+		done
+
+		# run the test command and put the BPF filter in a temp file
+		exec 4>$tmpfile
+		run_test_command "$testnumstr" "./$testname" "-b" 4 ""
+		rc=$?
+		exec 4>&-
+		if [[ $rc -ne 0 ]]; then
+			print_result $testnumstr "ERROR" "$testname rc=$rc"
+			stats_error=$(($stats_error+1))
+			return
+		fi
+
+		# simulate the fuzzed syscall data against the BPF filter, we
+		# don't verify the resulting action since we're just testing for
+		# stability
+		allow=$($GLBL_SYS_SIM -f $tmpfile -s $sys \
+			${arg[0]} ${arg[1]} ${arg[2]} ${arg[3]} ${arg[4]} \
+			${arg[5]})
+		rc=$?
+		if [[ $rc -ne 0 ]]; then
+			print_result $testnumstr "ERROR" "bpf_sim rc=$rc"
+			stats_error=$(($stats_error+1))
+		else
+			print_result $testnumstr "SUCCESS" ""
+			stats_success=$(($stats_success+1))
+		fi
+		stats_all=$(($stats_all+1))
+	done
+}
+
+#
+# Run the specified "bpf-sim" test
+#
+# Tests that belong to the "bpf-sim" test type generate a BPF filter and then
+# run a simulated system call test to validate the filter.  Tests that belong to
+# this test type provide the following data on a single line in the input batch
+# file:
+#
+#     Testname - The executable test name (e.g. 01-allow, 02-basic, etc.)
+#     Arch - The architecture that the test should be run on (all, x86, x86_64)
+#     Syscall - The syscall to simulate against the generated filter
+#     Arg0-5 - The syscall arguments to simulate against the generated filter
+#     Result - The expected simulation result (ALLOW, KILL, etc.)
+#
+# If a range of syscall or argument values are specified (e.g. 1-9), a test is
+# generated for every combination of range values.  Otherwise, the individual
+# test is run.
+#
+# Arguments:
+#     1    string containing the batch name
+#     2    value of test number from batch file
+#     3    string containing line of test data from batch file
+#
+function run_test_bpf_sim() {
+	local rc
+	local LOW=1
+	local HIGH=2
+	local -a arg_empty=(false false false false false false)
+
+	# begin splitting the test data from the line into individual variables
+	local line=($3)
+	local testname=${line[0]}
+	local testarch=${line[1]}
+	local low_syscall  #line[2]
+	local high_syscall #line[2]
+	local -a low_arg   #line[3-8]
+	local -a high_arg  #line[3-8]
+	local result=${line[9]}
+
+	# expand the architecture list
+	local simarch_tmp
+	local simarch_avoid
+	simarch_tmp=""
+	simarch_avoid=""
+	for arch_i in $(echo $testarch | sed -e 's/,/ /g'); do
+		case $arch_i in
+		all)
+			# add the native arch
+			simarch_tmp+=" $arch"
+			;;
+		all_le)
+			# add the native arch only if it is little endian
+			if echo "$GLBL_ARCH_LE_SUPPORT" | grep -qw "$arch"; then
+				simarch_tmp+=" $arch"
+			fi
+			;;
+		+all_le)
+			# add all of the little endian architectures
+			simarch_tmp+=" $GLBL_ARCH_LE_SUPPORT"
+			;;
+		all_be)
+			# add the native arch only if it is big endian
+			if echo "$GLBL_ARCH_BE_SUPPORT" | grep -qw "$arch"; then
+				simarch_tmp+=" $arch"
+			fi
+			;;
+		+all_be)
+			# add all of the big endian architectures
+			simarch_tmp+=" $GLBL_ARCH_BE_SUPPORT"
+			;;
+		all_32)
+			# add the native arch only if it is 32-bit
+			if echo "$GLBL_ARCH_32B_SUPPORT" | grep -qw "$arch"; then
+				simarch_tmp+=" $arch"
+			fi
+			;;
+		+all_32)
+			# add all of the 32-bit architectures
+			simarch_tmp+=" $GLBL_ARCH_32B_SUPPORT"
+			;;
+		all_64)
+			# add the native arch only if it is 64-bit
+			if echo "$GLBL_ARCH_64B_SUPPORT" | grep -qw "$arch"; then
+				simarch_tmp+=" $arch"
+			fi
+			;;
+		+all_64)
+			# add all of the 64-bit architectures
+			simarch_tmp+=" $GLBL_ARCH_64B_SUPPORT"
+			;;
+		+*)
+			# add the architecture specified
+			simarch_tmp+=" ${arch_i:1}"
+			;;
+		-*)
+			# remove the architecture specified
+			simarch_avoid+=" ${arch_i:1}"
+			;;
+		*)
+			# add the architecture specified if it is native
+			if [[ "$arch_i" == "$arch" ]]; then
+				simarch_tmp+=" $arch_i"
+			fi
+			;;
+		esac
+	done
+
+	# make sure we remove any undesired architectures
+	local simarch_list
+	simarch_list=""
+	for arch_i in $simarch_tmp; do
+		if echo "$simarch_avoid" | grep -q -v -w "$arch_i"; then
+			simarch_list+=" $arch_i"
+		fi
+	done
+	simarch_list=$(echo $simarch_list | sed -e 's/  / /g;s/^ //;')
+
+	# do we have any architectures remaining in the list?
+	if [[ $simarch_list == "" ]]; then
+		print_result $(generate_test_num "$1" $2 1) "SKIPPED" \
+			"(architecture difference)"
+		stats_skipped=$(($stats_skipped+1))
+		return
+	fi
+
+	# get low and high range arg values
+	line_i=3
+	for arg_i in {0..5}; do
+		low_arg[$arg_i]=$(get_range $LOW "${line[$line_i]}")
+		high_arg[$arg_i]=$(get_range $HIGH "${line[$line_i]}")
+
+		# fix up empty arg values so the nested loops work
+		if [[ ${low_arg[$arg_i]} == "N" ]]; then
+			arg_empty[$arg_i]=true
+			low_arg[$arg_i]=0
+			high_arg[$arg_i]=0
+		fi
+
+		line_i=$(($line_i+1))
+	done
+
+	# loop through the selected architectures
+	for simarch in $simarch_list; do
+		# print architecture header if necessary
+		if [[ $simarch != $simarch_list ]]; then
+			echo " test arch:  $simarch" >&$logfd
+		fi
+
+		# reset the subtest number
+		local subtestnum=1
+
+		# get low and high syscall values and convert them to numbers
+		low_syscall=$(get_range $LOW "${line[2]}")
+		if [[ ! $low_syscall =~ ^\-?[0-9]+$ ]]; then
+			low_syscall=$($GLBL_SYS_RESOLVER -a $simarch -t \
+				      $low_syscall)
+			if [[ $? -ne 0 ]]; then
+				print_result $(generate_test_num "$1" $2 1) \
+					     "ERROR" "sys_resolver rc=$?"
+				stats_error=$(($stats_error+1))
+				return
+			fi
+		fi
+		high_syscall=$(get_range $HIGH "${line[2]}")
+		if [[ ! $high_syscall =~ ^\-?[0-9]+$ ]]; then
+			high_syscall=$($GLBL_SYS_RESOLVER -a $simarch -t \
+				       $high_syscall)
+			if [[ $? -ne 0 ]]; then
+				print_result $(generate_test_num "$1" $2 1) \
+					     "ERROR" "sys_resolver rc=$?"
+				stats_error=$(($stats_error+1))
+				return
+			fi
+		fi
+
+		# if ranges exist, the following will loop through all syscall
+		# and arg ranges and generate/run every combination of requested
+		# tests; if no ranges were specifed, then the single test is
+		# run
+		for sys in $(get_seq $low_syscall $high_syscall); do
+		for arg0 in $(get_seq ${low_arg[0]} ${high_arg[0]}); do
+		for arg1 in $(get_seq ${low_arg[1]} ${high_arg[1]}); do
+		for arg2 in $(get_seq ${low_arg[2]} ${high_arg[2]}); do
+		for arg3 in $(get_seq ${low_arg[3]} ${high_arg[3]}); do
+		for arg4 in $(get_seq ${low_arg[4]} ${high_arg[4]}); do
+		for arg5 in $(get_seq ${low_arg[5]} ${high_arg[5]}); do
+			local -a arg=($arg0 $arg1 $arg2 $arg3 $arg4 $arg5)
+
+			# Get the generated sub-test num string
+			local testnumstr=$(generate_test_num "$1" $2 \
+					   $subtestnum)
+
+			# format any empty args to print to log file
+			for i in {0..5}; do
+				if ${arg_empty[$i]}; then
+					arg[$i]="N"
+				fi
+			done
+
+			# set up log file test data line for this
+			# individual test, spacing is added to align
+			# the output in the correct columns
+			local -a COL_WIDTH=(26 08 14 11 17 21 09 06 06)
+			local testdata=$(printf "%-${COL_WIDTH[0]}s" $testname)
+			testdata+=$(printf "%-${COL_WIDTH[1]}s" $simarch)
+			testdata+=$(printf "%-${COL_WIDTH[2]}s" $sys)
+			testdata+=$(printf "%-${COL_WIDTH[3]}s" ${arg[0]})
+			testdata+=$(printf "%-${COL_WIDTH[4]}s" ${arg[1]})
+			testdata+=$(printf "%-${COL_WIDTH[5]}s" ${arg[2]})
+			testdata+=$(printf "%-${COL_WIDTH[6]}s" ${arg[3]})
+			testdata+=$(printf "%-${COL_WIDTH[7]}s" ${arg[4]})
+			testdata+=$(printf "%-${COL_WIDTH[8]}s" ${arg[5]})
+			testdata+=$(printf "%-${COL_WIDTH[9]}s" $result)
+
+			# print out the test data to the log file
+			print_data "$testnumstr" "$testdata"
+
+			# set up the syscall arguments to be passed to bpf_sim
+			for i in {0..5}; do
+				if ${arg_empty[$i]}; then
+					arg[$i]=""
+				else
+					arg[$i]=" -$i ${arg[$i]} "
+				fi
+			done
+
+			# run the test command and put the BPF in a temp file
+			exec 4>$tmpfile
+			run_test_command "$testnumstr" "./$testname" "-b" 4 ""
+			rc=$?
+			exec 4>&-
+			if [[ $rc -ne 0 ]]; then
+				print_result $testnumstr \
+					     "ERROR" "$testname rc=$rc"
+				stats_error=$(($stats_error+1))
+				return
+			fi
+
+			# simulate the specifed syscall against the BPF filter
+			# and verify the results
+			action=$($GLBL_SYS_SIM -a $simarch -f $tmpfile \
+				 -s $sys ${arg[0]} ${arg[1]} ${arg[2]} \
+				 ${arg[3]} ${arg[4]} ${arg[5]})
+			rc=$?
+			if [[ $rc -ne 0 ]]; then
+				print_result $testnumstr \
+					     "ERROR" "bpf_sim rc=$rc"
+				stats_error=$(($stats_error+1))
+			elif [[ "$action" != "$result" ]]; then
+				print_result $testnumstr "FAILURE" \
+					     "bpf_sim resulted in $action"
+				stats_failure=$(($stats_failure+1))
+			else
+				print_result $testnumstr "SUCCESS" ""
+				stats_success=$(($stats_success+1))
+			fi
+			stats_all=$(($stats_all+1))
+
+			subtestnum=$(($subtestnum+1))
+		done # syscall
+		done # arg0
+		done # arg1
+		done # arg2
+		done # arg3
+		done # arg4
+		done # arg5
+	done # architecture
+}
+
+#
+# Run the specified "basic" test
+#
+# Tests that belong to the "basic" test type will simply have the command
+# specified in the input batch file.  The command must return zero for success
+# and non-zero for failure.
+#
+# Arguments:
+#     1    value of test number from batch file
+#     2    string containing line of test data from batch file
+#
+function run_test_basic() {
+	local rc
+	local cmd
+
+	# if the test is a script, only run it in native/c mode
+	if [[ $mode != "c" && "$2" == *.sh ]]; then
+		print_result "$1" "SKIPPED" "(only valid in native/c mode)"
+		stats_skipped=$(($stats_skipped+1))
+		return
+	fi
+
+	# print out the input test data to the log file
+	print_data "$1" "$2"
+
+	# check and adjust if we are doing a VPATH build
+	if [[ -x "./$2" ]]; then
+		cmd="./$2"
+	else
+		cmd="${srcdir}/$2"
+	fi
+
+	# run the command
+	run_test_command "$1" "$cmd" "" "" ""
+	rc=$?
+	if [[ $rc -ne 0 ]]; then
+		print_result $1 "FAILURE" "$2 rc=$rc"
+		stats_failure=$(($stats_failure+1))
+	else
+		print_result $1 "SUCCESS" ""
+		stats_success=$(($stats_success+1))
+	fi
+	stats_all=$(($stats_all+1))
+}
+
+#
+# Run the specified "bpf-valgrind" test
+#
+# Tests that belong to the "bpf-valgrind" test type generate a BPF filter
+# while running under valgrind to detect any memory errors.
+#
+# Arguments:
+#     1    value of test number from batch file
+#     2    string containing line of test data from batch file
+#
+function run_test_bpf_valgrind() {
+	local rc
+
+	# we only support the native/c test mode here
+	if [[ $mode != "c" ]]; then
+		print_result "$1" "SKIPPED" "(only valid in native/c mode)"
+		stats_skipped=$(($stats_skipped+1))
+		return
+	fi
+
+	# print out the input test data to the log file
+	print_data "$1" "$2"
+
+	# build the command
+	testvalgrind="valgrind \
+		       --tool=memcheck \
+		       --error-exitcode=1 \
+		       --leak-check=full \
+		       --read-var-info=yes \
+		       --track-origins=yes \
+		       --suppressions=$basedir/valgrind_test.supp"
+	if [[ -n $logfile ]]; then
+		testvalgrind+=" --log-fd=$logfd"
+	fi
+	if [[ -z $verbose ]]; then
+		testvalgrind+=" --quiet --log-fd=4"
+	fi
+
+	# run the command
+	exec 4>/dev/null
+	print_valgrind "$1"
+	run_test_command "$1" "$testvalgrind --" "./$2 -b" 4 2
+	rc=$?
+	exec 4>&-
+	if [[ $rc -ne 0 ]]; then
+		print_result $1 "FAILURE" "$2 rc=$rc"
+		stats_failure=$(($stats_failure+1))
+	else
+		print_result $1 "SUCCESS" ""
+		stats_success=$(($stats_success+1))
+	fi
+	stats_all=$(($stats_all+1))
+}
+
+#
+# Run the specified "live" test
+#
+# Tests that belong to the "live" test type will attempt to run a live test
+# of the libseccomp library on the host system; for obvious reasons the host
+# system must support seccomp mode 2 for this to work correctly.
+#
+# Arguments:
+#     1    value of test number from batch file
+#     2    string containing line of test data from batch file
+#
+function run_test_live() {
+	local rc
+	local api
+	local line=($2)
+
+	# parse the test line
+	line_cmd=${line[0]}
+	line_api=${line[1]}
+	line_act=${line[2]}
+	line_test="$line_cmd $line_api $line_act"
+
+	# check the api level
+	api=$($GLBL_SYS_API)
+	if [[ $api -lt $line_api ]]; then
+		# runtime api level is too low
+		print_result "$1" "SKIPPED" "(api level)"
+		stats_skipped=$(($stats_skipped+1))
+		return
+	fi
+
+	# print out the input test data to the log file
+	print_data "$1" "$2"
+
+	# run the command
+	exec 4>/dev/null
+	run_test_command "$1" "./$line_cmd" "$line_act" "" 4
+	rc=$?
+	exec 4>&-
+	stats_all=$(($stats_all+1))
+
+	# setup the arch specific return values
+	case "$arch" in
+	x86|x86_64|x32|arm|aarch64|parisc|parisc64|ppc|ppc64|ppc64le|ppc|s390|s390x|riscv64)
+		rc_kill_process=159
+		rc_kill=159
+		rc_allow=160
+		rc_trap=161
+		rc_trace=162
+		rc_errno=163
+		rc_log=164
+		;;
+	mips|mipsel|mips64|mips64n32|mipsel64|mipsel64n32)
+		rc_kill_process=140
+		rc_kill=140
+		rc_allow=160
+		rc_trap=161
+		rc_trace=162
+		rc_errno=163
+		rc_log=164
+		;;
+	*)
+		print_result $testnumstr "ERROR" "arch $arch not supported"
+		stats_error=$(($stats_error+1))
+		return
+		;;
+	esac
+
+	# verify the results
+	if [[ $line_act == "KILL_PROCESS" && $rc -eq $rc_kill_process ]]; then
+		print_result $1 "SUCCESS" ""
+		stats_success=$(($stats_success+1))
+	elif [[ $line_act == "KILL" && $rc -eq $rc_kill ]]; then
+		print_result $1 "SUCCESS" ""
+		stats_success=$(($stats_success+1))
+	elif [[ $line_act == "ALLOW" && $rc -eq $rc_allow ]]; then
+		print_result $1 "SUCCESS" ""
+		stats_success=$(($stats_success+1))
+	elif [[ $line_act == "TRAP" && $rc -eq $rc_trap ]]; then
+		print_result $1 "SUCCESS" ""
+		stats_success=$(($stats_success+1))
+	elif [[ $line_act == "TRACE" ]]; then
+		print_result $1 "ERROR" "unsupported action \"$line_act\""
+		stats_error=$(($stats_error+1))
+	elif [[ $line_act == "ERRNO" && $rc -eq $rc_errno ]]; then
+		print_result $1 "SUCCESS" ""
+		stats_success=$(($stats_success+1))
+	elif [[ $line_act == "LOG" && $rc -eq $rc_log ]]; then
+		print_result $1 "SUCCESS" ""
+		stats_success=$(($stats_success+1))
+	else
+		print_result $1 "FAILURE" "$line_test rc=$rc"
+		stats_failure=$(($stats_failure+1))
+	fi
+}
+
+#
+# Run a single test from the specified batch
+#
+# Arguments:
+#     1    string containing the batch name
+#     2    value of test number from batch file
+#     3    string containing line of test data from batch file
+#     4    string containing test type that this test belongs to
+#
+function run_test() {
+	# generate the test number string for the line of batch test data
+	local testnumstr=$(generate_test_num "$1" $2 1)
+
+	# ensure we only run tests which match the specified type
+	match_csv_word "$type" "$4"
+	local type_match=$?
+	[[ -n $type && $type_match -eq 1 ]] && return
+
+	# execute the function corresponding to the test type
+	if [[ "$4" == "basic" ]]; then
+		run_test_basic "$testnumstr" "$3"
+	elif [[ "$4" == "bpf-sim" ]]; then
+		run_test_bpf_sim "$1" $2 "$3"
+	elif [[ "$4" == "bpf-sim-fuzz" ]]; then
+		run_test_bpf_sim_fuzz "$1" $2 "$3"
+	elif [[ "$4" == "bpf-valgrind" ]]; then
+		# only run this test if valgrind is installed
+		if check_deps valgrind; then
+			run_test_bpf_valgrind "$testnumstr" "$3"
+		else
+			print_result $testnumstr "SKIPPED" \
+				"(valgrind not installed)"
+			stats_skipped=$(($stats_skipped+1))
+		fi
+	elif [[ "$4" == "live" ]]; then
+		# only run this test if explicitly requested
+		if [[ -n $type ]]; then
+			run_test_live "$testnumstr" "$3"
+		else
+			print_result $testnumstr "SKIPPED" \
+				"(must specify live tests)"
+			stats_skipped=$(($stats_skipped+1))
+		fi
+	else
+		print_result $testnumstr "ERROR" "test type $4 not supported"
+		stats_error=$(($stats_error+1))
+	fi
+}
+
+#
+# Run the requested tests
+#
+function run_tests() {
+	# loop through all test files
+	for file in $basedir/*.tests; do
+		local testnum=1
+		local batch_requested=false
+		local batch_name=""
+
+		# extract the batch name from the file name
+		batch_name=$(basename $file .tests)
+
+		# check if this batch was requested
+		if [[ ${batch_list[@]} ]]; then
+			for b in ${batch_list[@]}; do
+				if [[ $b == $batch_name ]]; then
+					batch_requested=true
+					break
+				fi
+			done
+			if ! $batch_requested; then
+				continue
+			fi
+		fi
+
+		# print a test batch header
+		echo " batch name: $batch_name" >&$logfd
+
+		# loop through each line and run the requested tests
+		while read line; do
+			# strip whitespace, comments, and blank lines
+			line=$(echo "$line" | \
+			       sed -e 's/^[\t ]*//;s/[\t ]*$//;' | \
+			       sed -e '/^[#].*$/d;/^$/d')
+			if [[ -z $line ]]; then
+				continue
+			fi
+
+			if [[ $line =~ ^"test type": ]]; then
+				test_type=$(echo "$line" | \
+					    sed -e 's/^test type: //;')
+				# print a test mode and type header
+				echo " test mode:  $mode" >&$logfd
+				echo " test type:  $test_type" >&$logfd
+				continue
+			fi
+
+			if [[ ${single_list[@]} ]]; then
+				for i in ${single_list[@]}; do
+					if [ $i -eq $testnum ]; then
+						# we're running a single test
+						run_test "$batch_name" \
+							 $testnum "$line" \
+							 "$test_type"
+					fi
+				done
+			else
+				# we're running a test from a batch
+				run_test "$batch_name" \
+					 $testnum "$line" "$test_type"
+			fi
+			testnum=$(($testnum+1))
+		done < "$file"
+	done
+}
+
+####
+# main
+
+# verify general script dependencies
+verify_deps head
+verify_deps sed
+verify_deps awk
+verify_deps tr
+
+# global variables
+declare -a batch_list
+declare -a single_list
+arch=
+batch_count=0
+logfile=
+logfd=
+mode_list=""
+runall=
+singlecount=0
+tmpfile=""
+tmpdir=""
+type=
+verbose=
+stats_all=0
+stats_skipped=0
+stats_success=0
+stats_failure=0
+stats_error=0
+
+# set the test root directory
+basedir=$(dirname $0)
+
+# set the test harness pid
+pid=$$
+
+# parse the command line
+while getopts "ab:gl:m:s:t:T:vh" opt; do
+	case $opt in
+	a)
+		runall=1
+		;;
+	b)
+		batch_list[batch_count]="$OPTARG"
+		batch_count=$(($batch_count+1))
+		;;
+	l)
+		logfile="$OPTARG"
+		;;
+	m)
+		case $OPTARG in
+		c)
+			mode_list="$mode_list c"
+			;;
+		python)
+			verify_deps python
+			mode_list="$mode_list python"
+			;;
+		*)
+			usage
+			exit 1
+		esac
+		;;
+	s)
+		single_list[single_count]=$OPTARG
+		single_count=$(($single_count+1))
+		;;
+	t)
+		tmpdir="$OPTARG"
+		;;
+	T)
+		type="$OPTARG"
+		;;
+	v)
+		verbose=1
+		;;
+	h|*)
+		usage
+		exit 1
+		;;
+	esac
+done
+
+# use mode list from environment if provided
+[[ -z $mode_list && -n $LIBSECCOMP_TSTCFG_MODE_LIST ]] && mode_list=$LIBSECCOMP_TSTCFG_MODE_LIST
+
+# determine the mode test automatically
+if [[ -z $mode_list ]]; then
+	# always perform the native c tests
+	mode_list="c"
+
+	# query the build configuration
+	if [[ -r "../configure.h" ]]; then
+		# python tests
+		[[ "$(grep "ENABLE_PYTHON" ../configure.h | \
+		   awk '{ print $3 }')" = "1" ]] && \
+			mode_list="$mode_list python"
+	fi
+fi
+
+# check if we specified a list of tests via the environment variable
+if [[ -n $LIBSECCOMP_TSTCFG_BATCHES ]]; then
+	for i in $(echo "$LIBSECCOMP_TSTCFG_BATCHES" | sed 's/,/ /g'); do
+		batch_list[batch_count]="$i"
+		batch_count=$(($batch_count+1))
+	done
+fi
+
+# default to all tests if batch or single tests not requested
+if [[ -z $batch_list ]] && [[ -z $single_list ]]; then
+	runall=1
+fi
+
+# drop any requested batch and single tests if all tests were requested
+if [[ -n $runall ]]; then
+	batch_list=()
+	single_list=()
+fi
+
+# check for configuration via environment variables
+[[ -z $type && -n $LIBSECCOMP_TSTCFG_TYPE ]] && type=$LIBSECCOMP_TSTCFG_TYPE
+
+# open log file for append (default to stdout)
+if [[ -n $logfile ]]; then
+	logfd=3
+	exec 3>>"$logfile"
+else
+	logfd=1
+fi
+
+# open temporary file
+if [[ -n $tmpdir ]]; then
+	tmpfile=$(mktemp -t regression_XXXXXX --tmpdir=$tmpdir)
+else
+	tmpfile=$(mktemp -t regression_XXXXXX)
+fi
+
+# determine the current system's architecture
+arch=$($GLBL_SYS_ARCH)
+
+# display the test output and run the requested tests
+echo "=============== $(date) ===============" >&$logfd
+echo "Regression Test Report (\"regression $*\")" >&$logfd
+for mode in $mode_list; do
+	run_tests
+done
+echo "Regression Test Summary" >&$logfd
+echo " tests run: $stats_all" >&$logfd
+echo " tests skipped: $stats_skipped" >&$logfd
+echo " tests passed: $stats_success" >&$logfd
+echo " tests failed: $stats_failure" >&$logfd
+echo " tests errored: $stats_error" >&$logfd
+echo "============================================================" >&$logfd
+
+# cleanup and exit
+rm -f $tmpfile
+rc=0
+[[ $stats_failure -gt 0 ]] && rc=$(($rc + 2))
+[[ $stats_error -gt 0 ]] && rc=$(($rc + 4))
+
+exit $rc