summaryrefslogtreecommitdiffstats
path: root/scripts/run-regtests.sh
diff options
context:
space:
mode:
Diffstat (limited to 'scripts/run-regtests.sh')
-rwxr-xr-xscripts/run-regtests.sh416
1 files changed, 416 insertions, 0 deletions
diff --git a/scripts/run-regtests.sh b/scripts/run-regtests.sh
new file mode 100755
index 0000000..5126efb
--- /dev/null
+++ b/scripts/run-regtests.sh
@@ -0,0 +1,416 @@
+#!/bin/sh
+
+_help()
+{
+ cat << EOF
+### run-regtests.sh ###
+ Running run-regtests.sh --help shows this information about how to use it
+
+ Run without parameters to run all tests in the current folder (including subfolders)
+ run-regtests.sh
+
+ Provide paths to run tests from (including subfolders):
+ run-regtests.sh ./tests1 ./tests2
+
+ Parameters:
+ --j <NUM>, To run vtest with multiple jobs / threads for a faster overall result
+ run-regtests.sh ./fasttest --j 16
+
+ --v, to run verbose
+ run-regtests.sh --v, disables the default vtest 'quiet' parameter
+
+ --debug to show test logs on standard output (implies --v)
+ run-regtests.sh --debug
+
+ --keep-logs to keep all log directories (by default kept if test fails)
+ run-regtests.sh --keep-logs
+
+ --vtestparams <ARGS>, passes custom ARGS to vtest
+ run-regtests.sh --vtestparams "-n 10"
+
+ --type <reg tests types> filter the types of the tests to be run, depending on
+ the commented REGTESTS_TYPE variable value in each VTC file.
+ The value of REGTESTS_TYPE supported are: default, slow, bug, broken, devel
+ and experimental. When not specified, it is set to 'default' as default value.
+
+ run-regtest.sh --type slow,default
+
+ --clean to cleanup previous reg-tests log directories and exit
+ run-regtests.sh --clean
+
+ Including text below into a .vtc file will check for its requirements
+ related to haproxy's target and compilation options
+ # Below targets are not capable of completing this test successfully
+ #EXCLUDE_TARGET=freebsd, abns sockets are not available on freebsd
+
+ #EXCLUDE_TARGETS=dos,freebsd,windows
+
+ # Below option is required to complete this test successfully
+ #REQUIRE_OPTION=OPENSSL, this test needs OPENSSL compiled in.
+ #REQUIRE_OPTIONS=ZLIB|SLZ,OPENSSL,LUA
+
+ #REQUIRE_SERVICE=prometheus-exporter
+ #REQUIRE_SERVICES=prometheus-exporter,foo
+
+ # To define a range of versions that a test can run with:
+ #REQUIRE_VERSION=0.0
+ #REQUIRE_VERSION_BELOW=99.9
+
+ Configure environment variables to set the haproxy and vtest binaries to use
+ setenv HAPROXY_PROGRAM /usr/local/sbin/haproxy
+ setenv VTEST_PROGRAM /usr/local/bin/vtest
+ setenv HAPROXY_ARGS "-dM -de -m 50"
+ or
+ export HAPROXY_PROGRAM=/usr/local/sbin/haproxy
+ export VTEST_PROGRAM=/usr/local/bin/vtest
+ export HAPROXY_ARGS="-dM -de -m 50"
+EOF
+ exit 0
+}
+
+add_range_to_test_list()
+{
+ level0="*.vtc"
+ level1="h*.vtc"
+ level2="s*.vtc"
+ level3="l*.vtc"
+ level4="b*.vtc"
+ level5="k*.vtc"
+ level6="e*.vtc"
+
+ new_range=$(echo $1 | tr '-' ' ')
+ non_digit=$(echo $new_range | grep '[^0-9 ]')
+ if [ -n "$non_digit" ] ; then
+ return
+ fi
+ if [ "$new_range" = "$1" ] ; then
+ if [ $1 -gt 6 ] ; then
+ return
+ fi
+ eval echo '$'level$1
+ return
+ fi
+ if [ -z "$new_range" ] ; then
+ return
+ fi
+ list=
+ for l in $(seq $new_range) ; do
+ if [ -n "l" ] ; then
+ if [ -z "$list" ] ; then
+ list="$(eval echo '$'level${l})"
+ else
+ list="$list $(eval echo '$'level${l})"
+ fi
+ fi
+ done
+
+ echo $list
+}
+
+_startswith() {
+ _str="$1"
+ _sub="$2"
+ echo "$_str" | grep "^$_sub" >/dev/null 2>&1
+}
+
+_findtests() {
+ set -f
+
+ REGTESTS_TYPES="${REGTESTS_TYPES:-default,bug,devel,slow}"
+ any_test=$(echo $REGTESTS_TYPES | grep -cw "any")
+ for i in $( find "$1" -name *.vtc ); do
+ skiptest=
+ OLDIFS="$IFS"; IFS="$LINEFEED"
+ set -- $(grep '^#[0-9A-Z_]*=' "$i")
+ IFS="$OLDIFS"
+
+ require_version=""; require_version_below=""; require_options="";
+ require_services=""; exclude_targets=""; regtest_type=""
+ requiredoption=""; requiredservice=""; excludedtarget="";
+
+ while [ $# -gt 0 ]; do
+ v="$1"; v="${v#*=}"
+ case "$1" in
+ "#REQUIRE_VERSION="*) require_version="$v" ;;
+ "#REQUIRE_VERSION_BELOW="*) require_version_below="$v" ;;
+ "#REQUIRE_OPTIONS="*) require_options="$v" ;;
+ "#REQUIRE_SERVICES="*) require_services="$v" ;;
+ "#EXCLUDE_TARGETS="*) exclude_targets="$v" ;;
+ "#REGTEST_TYPE="*) regtest_type="$v" ;;
+ "#REQUIRE_OPTION="*) requiredoption="${v%,*}" ;;
+ "#REQUIRE_SERVICE="*) required_service="${v%,*}" ;;
+ "#EXCLUDE_TARGET="*) excludedtarget="${v%,*}" ;;
+ # Note: any new variable declared here must be initialized above.
+ esac
+ shift
+ done
+
+ if [ $any_test -ne 1 ] ; then
+ if [ -z $regtest_type ] ; then
+ regtest_type=default
+ fi
+ if ! $(echo $REGTESTS_TYPES | grep -wq $regtest_type) ; then
+ echo " Skip $i because its type '"$regtest_type"' is excluded"
+ skiptest=1
+ fi
+ fi
+
+ if [ -n "$requiredoption" ]; then
+ require_options="$require_options,$requiredoption"
+ fi
+
+ if [ -n "$requiredservice" ]; then
+ require_services="$require_services,$requiredservice"
+ fi
+
+ if [ -n "$excludedtarget" ]; then
+ exclude_targets="$exclude_targets,$excludedtarget"
+ fi
+
+ IFS=","; set -- $require_options; IFS=$OLDIFS; require_options="$*"
+ IFS=","; set -- $require_services; IFS=$OLDIFS; require_services="$*"
+ IFS=","; set -- $exclude_targets; IFS=$OLDIFS; exclude_targets="$*"
+
+ if [ -n "$require_version" ]; then
+ if [ $(_version "$HAPROXY_VERSION") -lt $(_version "$require_version") ]; then
+ echo " Skip $i because option haproxy is version: $HAPROXY_VERSION"
+ echo " REASON: this test requires at least version: $require_version"
+ skiptest=1
+ fi
+ fi
+ if [ -n "$require_version_below" ]; then
+ if [ $(_version "$HAPROXY_VERSION") -ge $(_version "$require_version_below") ]; then
+ echo " Skip $i because option haproxy is version: $HAPROXY_VERSION"
+ echo " REASON: this test requires a version below: $require_version_below"
+ skiptest=1
+ fi
+ fi
+
+ for excludedtarget in $exclude_targets; do
+ if [ "$excludedtarget" = "$TARGET" ]; then
+ echo " Skip $i because haproxy is compiled for the excluded target $TARGET"
+ skiptest=1
+ fi
+ done
+
+ for requiredoption in $require_options; do
+ IFS="|"; set -- $requiredoption; IFS=$OLDIFS; alternatives="$*"
+ found=
+ for alt in $alternatives; do
+ if [ -z "${FEATURES_PATTERN##* +$alt *}" ]; then
+ found=1;
+ fi
+ done
+ if [ -z $found ]; then
+ echo " Skip $i because haproxy is not compiled with the required option $requiredoption"
+ skiptest=1
+ fi
+ done
+
+ for requiredservice in $require_services; do
+ IFS="|"; set -- $requiredservice; IFS=$OLDIFS; alternatives="$*"
+ found=
+ for alt in $alternatives; do
+ if [ -z "${SERVICES_PATTERN##* $alt *}" ]; then
+ found=1;
+ fi
+ done
+ if [ -z $found ]; then
+ echo " Skip $i because haproxy is not compiled with the required service $requiredservice"
+ skiptest=1
+ fi
+ done
+
+ if [ -z $skiptest ]; then
+ echo " Add test: $i"
+ testlist="$testlist $i"
+ fi
+ done
+}
+
+_cleanup()
+{
+ DIRS=$(find "${TESTDIR}" -maxdepth 1 -type d -name "haregtests-*" -exec basename {} \; 2>/dev/null)
+ if [ -z "${DIRS}" ]; then
+ echo "No reg-tests log directory found"
+ else
+ echo "Cleanup following reg-tests log directories:"
+ for d in ${DIRS}; do
+ echo " o ${TESTDIR}/$d"
+ done
+ read -p "Continue (y/n)?" reply
+ case "$reply" in
+ y|Y)
+ for d in ${DIRS}; do
+ rm -r "${TESTDIR}/$d"
+ done
+ echo "done"
+ exit 0
+ ;;
+ *)
+ echo "aborted"
+ exit 1
+ ;;
+ esac
+ fi
+}
+
+
+_process() {
+ while [ ${#} -gt 0 ]; do
+ if _startswith "$1" "-"; then
+ case "${1}" in
+ --j)
+ jobcount="$2"
+ shift
+ ;;
+ --vtestparams)
+ vtestparams="$2"
+ shift
+ ;;
+ --v)
+ verbose=""
+ ;;
+ --debug)
+ verbose=""
+ debug="-v"
+ ;;
+ --keep-logs)
+ keep_logs="-L"
+ ;;
+ --type)
+ REGTESTS_TYPES="$2"
+ shift
+ ;;
+ --clean)
+ _cleanup
+ exit 0
+ ;;
+ --help)
+ _help
+ ;;
+ *)
+ echo "Unknown parameter : $1"
+ exit 1
+ ;;
+ esac
+ else
+ REGTESTS="${REGTESTS} $1"
+ fi
+ shift 1
+ done
+}
+
+# compute a version from up to 4 sub-version components, each multiplied
+# by a power of 1000, and padded left with 0, 1 or 2 zeroes.
+_version() {
+ OLDIFS="$IFS"; IFS="."; set -- $*; IFS="$OLDIFS"
+ set -- ${1%%[!0-9]*} 000${2%%[!0-9]*} 000${3%%[!0-9]*} 000${4%%[!0-9]*}
+ prf2=${2%???}; prf3=${3%???}; prf4=${4%???}
+ echo ${1}${2#$prf2}${3#$prf3}${4#$prf4}
+}
+
+
+HAPROXY_PROGRAM="${HAPROXY_PROGRAM:-${PWD}/haproxy}"
+HAPROXY_ARGS="${HAPROXY_ARGS--dM}"
+VTEST_PROGRAM="${VTEST_PROGRAM:-vtest}"
+TESTDIR="${TMPDIR:-/tmp}"
+REGTESTS=""
+LINEFEED="
+"
+
+jobcount=""
+verbose="-q"
+debug=""
+keep_logs="-l"
+testlist=""
+
+_process "$@";
+
+echo ""
+echo "########################## Preparing to run tests ##########################"
+
+preparefailed=
+if ! [ -x "$(command -v $HAPROXY_PROGRAM)" ]; then
+ echo "haproxy not found in path, please specify HAPROXY_PROGRAM environment variable"
+ preparefailed=1
+fi
+if ! [ -x "$(command -v $VTEST_PROGRAM)" ]; then
+ echo "vtest not found in path, please specify VTEST_PROGRAM environment variable"
+ preparefailed=1
+fi
+if [ $preparefailed ]; then
+ exit 1
+fi
+
+{ read HAPROXY_VERSION; read TARGET; read FEATURES; read SERVICES; } << EOF
+$($HAPROXY_PROGRAM $HAPROXY_ARGS -vv | grep 'HA-\?Proxy version\|TARGET.*=\|^Feature\|^Available services' | sed 's/.* [:=] //')
+EOF
+
+HAPROXY_VERSION=$(echo $HAPROXY_VERSION | cut -d " " -f 3)
+echo "Testing with haproxy version: $HAPROXY_VERSION"
+
+FEATURES_PATTERN=" $FEATURES "
+SERVICES_PATTERN=" $SERVICES "
+
+TESTRUNDATETIME="$(date '+%Y-%m-%d_%H-%M-%S')"
+
+mkdir -p "$TESTDIR" || exit 1
+TESTDIR=$(mktemp -d "$TESTDIR/haregtests-$TESTRUNDATETIME.XXXXXX") || exit 1
+
+export TMPDIR="$TESTDIR"
+export HAPROXY_PROGRAM="$HAPROXY_PROGRAM"
+if [ -n "$HAPROXY_ARGS" ]; then
+ export HAPROXY_ARGS
+fi
+
+echo "Target : $TARGET"
+echo "Options : $FEATURES"
+echo "Services : $SERVICES"
+
+echo "########################## Gathering tests to run ##########################"
+
+if [ -z "$REGTESTS" ]; then
+ _findtests reg-tests/
+else
+ for t in $REGTESTS; do
+ _findtests $t
+ done
+fi
+
+echo "########################## Starting vtest ##########################"
+echo "Testing with haproxy version: $HAPROXY_VERSION"
+_vtresult=0
+if [ -n "$testlist" ]; then
+ if [ -n "$jobcount" ]; then
+ jobcount="-j $jobcount"
+ fi
+ cmd="$VTEST_PROGRAM -b $((2<<20)) -k -t 10 $keep_logs $verbose $debug $jobcount $vtestparams $testlist"
+ eval $cmd
+ _vtresult=$?
+else
+ echo "No tests found that meet the required criteria"
+fi
+
+
+if [ $_vtresult -eq 0 ]; then
+ # all tests were successful, removing tempdir (the last part.)
+ # ignore errors is the directory is not empty or if it does not exist
+ rmdir "$TESTDIR" 2>/dev/null
+fi
+
+if [ -d "${TESTDIR}" ]; then
+ echo "########################## Gathering results ##########################"
+ export TESTDIR
+ find "$TESTDIR" -type d -name "vtc.*" -exec sh -c 'for i; do
+ if [ ! -e "$i/LOG" ] ; then continue; fi
+
+ cat <<- EOF | tee -a "$TESTDIR/failedtests.log"
+$(echo "###### $(cat "$i/INFO") ######")
+$(echo "## test results in: \"$i\"")
+$(grep -E -- "^(----|\* diag)" "$i/LOG")
+EOF
+ done' sh {} +
+fi
+
+exit $_vtresult