diff options
Diffstat (limited to 'bin/tests/system/run.sh')
| -rwxr-xr-x | bin/tests/system/run.sh | 346 |
1 files changed, 346 insertions, 0 deletions
diff --git a/bin/tests/system/run.sh b/bin/tests/system/run.sh new file mode 100755 index 0000000..2330d7c --- /dev/null +++ b/bin/tests/system/run.sh @@ -0,0 +1,346 @@ +#!/bin/sh + +# Copyright (C) Internet Systems Consortium, Inc. ("ISC") +# +# SPDX-License-Identifier: MPL-2.0 +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, you can obtain one at https://mozilla.org/MPL/2.0/. +# +# See the COPYRIGHT file distributed with this work for additional +# information regarding copyright ownership. + +# +# Run a system test. +# + +SYSTEMTESTTOP="$(cd -P -- "$(dirname -- "$0")" && pwd -P)" +. $SYSTEMTESTTOP/conf.sh + +if [ "$CI_SERVER" != "yes" ] && [ "$(id -u)" -eq "0" ] && ! ${NAMED} -V | grep -q -F -- "enable-developer"; then + echofail "Refusing to run test as root. Build with --enable-developer to override." >&2 + exit 1 +fi + +export SYSTEMTESTTOP + +date_with_args() ( + date "+%Y-%m-%dT%T%z" +) + +stopservers=true +baseport=5300 + +if [ ${SYSTEMTEST_NO_CLEAN:-0} -eq 1 ]; then + clean=false +else + clean=true +fi + +restart=false +while getopts "knp:r-:t" flag; do + case "$flag" in + -) case "${OPTARG}" in + keep) stopservers=false ;; + noclean) clean=false ;; + esac + ;; + k) stopservers=false ;; + n) clean=false ;; + p) baseport=$OPTARG ;; + t) restart=true ;; + esac +done +shift `expr $OPTIND - 1` + +if [ $# -eq 0 ]; then + echofail "Usage: $0 [-k] [-n] [-p <PORT>] test-directory [test-options]" >&2; + exit 1 +fi + +systest=${1%%/} +shift + +if [ ! -d $systest ]; then + echofail "$0: $systest: no such test" >&2 + exit 1 +fi + +# Define the number of ports allocated for each test, and the lowest and +# highest valid values for the "-p" option. +# +# The lowest valid value is one more than the highest privileged port number +# (1024). +# +# The highest valid value is calculated by noting that the value passed on the +# command line is the lowest port number in a block of "numports" consecutive +# ports and that the highest valid port number is 65,535. +numport=100 +minvalid=`expr 1024 + 1` +maxvalid=`expr 65535 - $numport + 1` + +test "$baseport" -eq "$baseport" > /dev/null 2>&1 +if [ $? -ne 0 ]; then + echofail "$0: $systest: must specify a numeric value for the port" >&2 + exit 1 +elif [ $baseport -lt $minvalid -o $baseport -gt $maxvalid ]; then + echofail "$0: $systest: the specified port must be in the range $minvalid to $maxvalid" >&2 + exit 1 +fi + +# Name the first 10 ports in the set (it is assumed that each test has access +# to ten or more ports): the query port, the control port and eight extra +# ports. Since the lowest numbered port (specified in the command line) +# will usually be a multiple of 10, the names are chosen so that if this is +# true, the last digit of EXTRAPORTn is "n". +PORT=$baseport +EXTRAPORT1=`expr $baseport + 1` +EXTRAPORT2=`expr $baseport + 2` +EXTRAPORT3=`expr $baseport + 3` +EXTRAPORT4=`expr $baseport + 4` +EXTRAPORT5=`expr $baseport + 5` +EXTRAPORT6=`expr $baseport + 6` +EXTRAPORT7=`expr $baseport + 7` +EXTRAPORT8=`expr $baseport + 8` +CONTROLPORT=`expr $baseport + 9` + +LOWPORT=$baseport +HIGHPORT=`expr $baseport + $numport - 1` + +export PORT +export EXTRAPORT1 +export EXTRAPORT2 +export EXTRAPORT3 +export EXTRAPORT4 +export EXTRAPORT5 +export EXTRAPORT6 +export EXTRAPORT7 +export EXTRAPORT8 +export CONTROLPORT + +export LOWPORT +export HIGHPORT + +# Start all servers used by the system test. Ensure all log files written +# during a system test (tests.sh + potentially multiple *.py scripts) are +# retained for each run by calling start.pl with the --restart command-line +# option for all invocations except the first one. +start_servers() { + echoinfo "I:$systest:starting servers" + if $restart || [ "$run" -gt 0 ]; then + restart_opt="--restart" + fi + if ! $PERL start.pl ${restart_opt} --port "$PORT" "$systest"; then + echoinfo "I:$systest:starting servers failed" + return 1 + fi +} + +stop_servers() { + if $stopservers; then + echoinfo "I:$systest:stopping servers" + if ! $PERL stop.pl "$systest"; then + echoinfo "I:$systest:stopping servers failed" + return 1 + fi + fi +} + +echostart "S:$systest:$(date_with_args)" +echoinfo "T:$systest:1:A" +echoinfo "A:$systest:System test $systest" +echoinfo "I:$systest:PORTRANGE:${LOWPORT} - ${HIGHPORT}" + +if [ x${PERL:+set} = x ] +then + echowarn "I:$systest:Perl not available. Skipping test." + echowarn "R:$systest:SKIPPED" + echoend "E:$systest:$(date_with_args)" + exit 0; +fi + +$PERL testsock.pl -p $PORT || { + echowarn "I:$systest:Network interface aliases not set up. Skipping test." + echowarn "R:$systest:SKIPPED" + echoend "E:$systest:$(date_with_args)" + exit 0; +} + +# Check for test-specific prerequisites. +test ! -f $systest/prereq.sh || ( cd $systest && $SHELL prereq.sh "$@" ) +result=$? + +if [ $result -eq 0 ]; then + : prereqs ok +else + echowarn "I:$systest:Prerequisites missing, skipping test." + echowarn "R:$systest:SKIPPED"; + echoend "E:$systest:$(date_with_args)" + exit 0 +fi + +# Check for PKCS#11 support +if + test ! -f $systest/usepkcs11 || $SHELL cleanpkcs11.sh +then + : pkcs11 ok +else + echowarn "I:$systest:Need PKCS#11, skipping test." + echowarn "R:$systest:PKCS11ONLY" + echoend "E:$systest:$(date_with_args)" + exit 0 +fi + +# Clean up files left from any potential previous runs except when +# started with the --restart option. +if ! $restart; then + if test -f "$systest/clean.sh"; then + if ! ( cd "${systest}" && $SHELL clean.sh "$@" ); then + echowarn "I:$systest:clean.sh script failed" + echofail "R:$systest:FAIL" + echoend "E:$systest:$(date_with_args)" + exit 1 + fi + fi +fi + +# Set up any dynamically generated test data +if test -f $systest/setup.sh +then + if ! ( cd "${systest}" && $SHELL setup.sh "$@" ); then + echowarn "I:$systest:setup.sh script failed" + echofail "R:$systest:FAIL" + echoend "E:$systest:$(date_with_args)" + exit 1 + fi +fi + +status=0 +run=0 +# Run the tests +if [ -r "$systest/tests.sh" ]; then + if start_servers; then + ( cd "$systest" && $SHELL tests.sh "$@" ) + status=$? + run=$((run+1)) + stop_servers || status=1 + else + status=1 + fi +fi + +if [ $status -eq 0 ]; then + if [ -n "$PYTEST" ]; then + for test in $(cd "${systest}" && find . -name "tests*.py"); do + rm -f "$systest/$test.status" + if start_servers; then + run=$((run+1)) + test_status=0 + (cd "$systest" && "$PYTEST" --confcutdir ../ -rsxX -v "$test" "$@" || echo "$?" > "$test.status") | SYSTESTDIR="$systest" cat_d + if [ -f "$systest/$test.status" ]; then + if [ "$(cat "$systest/$test.status")" != "5" ]; then + test_status=$(cat "$systest/$test.status") + fi + fi + status=$((status+test_status)) + stop_servers || status=1 + else + status=1 + fi + if [ $status -ne 0 ]; then + break + fi + done + rm -f "$systest/$test.status" + else + echoinfo "I:$systest:pytest not installed, skipping python tests" + fi +fi + +if [ "$run" -eq "0" ]; then + echoinfo "I:$systest:No tests were found and run" + status=255 +fi + + +if $stopservers +then + : +else + exit $status +fi + +get_core_dumps() { + find "$systest/" \( -name 'core' -or -name 'core.*' -or -name '*.core' \) ! -name '*.gz' ! -name '*.txt' | sort +} + +core_dumps=$(get_core_dumps | tr '\n' ' ') +if [ -n "$core_dumps" ]; then + echoinfo "I:$systest:Core dump(s) found: $core_dumps" + get_core_dumps | while read -r coredump; do + SYSTESTDIR="$systest" + echoinfo "D:$systest:backtrace from $coredump:" + echoinfo "D:$systest:--------------------------------------------------------------------------------" + binary=$(gdb --batch --core="$coredump" 2>/dev/null | sed -ne "s|Core was generated by \`\([^' ]*\)[' ].*|\1|p") + if [ ! -f "${binary}" ]; then + binary=$(find "${TOP}" -path "*/.libs/${binary}" -type f) + fi + "${TOP}/libtool" --mode=execute gdb \ + -batch \ + -ex bt \ + -core="$coredump" \ + -- \ + "$binary" 2>/dev/null | sed -n '/^Core was generated by/,$p' | cat_d + echoinfo "D:$systest:--------------------------------------------------------------------------------" + coredump_backtrace="${coredump}-backtrace.txt" + echoinfo "D:$systest:full backtrace from $coredump saved in $coredump_backtrace" + "${TOP}/libtool" --mode=execute gdb \ + -batch \ + -command="${TOP_SRCDIR}/bin/tests/system/run.gdb" \ + -core="$coredump" \ + -- \ + "$binary" > "$coredump_backtrace" 2>&1 + echoinfo "D:$systest:core dump $coredump archived as $coredump.gz" + gzip -1 "${coredump}" + done + status=$((status+1)) +fi + +assertion_failures=$(find "$systest/" -name named.run -exec grep "assertion failure" {} + | wc -l) +if [ "$assertion_failures" -ne 0 ]; then + SYSTESTDIR="$systest" + echoinfo "I:$systest:$assertion_failures assertion failure(s) found" + status=$((status+1)) +fi + +tsan_failures=$(find "$systest/" -name 'tsan.*' | wc -l) +if [ "$tsan_failures" -ne 0 ]; then + echoinfo "I:$systest:$tsan_failures sanitizer report(s) found" + find "$systest/" -name 'tsan.*' -exec grep "SUMMARY: " {} + | sort -u | cat_d + status=$((status+1)) +fi + +if [ "$status" -ne 0 ]; then + echofail "R:$systest:FAIL" +else + echopass "R:$systest:PASS" + if $clean && ! $restart; then + ( cd $systest && $SHELL clean.sh "$@" ) + if test -d ../../../.git; then + git status -su --ignored "${systest}/" 2>/dev/null | \ + sed -n -e 's|^?? \(.*\)|I:'${systest}':file \1 not removed|p' \ + -e 's|^!! \(.*/named.run\)$|I:'${systest}':file \1 not removed|p' \ + -e 's|^!! \(.*/named.memstats\)$|I:'${systest}':file \1 not removed|p' + fi + fi +fi + +NAMED_RUN_LINES_THRESHOLD=200000 +find "${systest}" -type f -name "named.run" -exec wc -l {} \; | awk "\$1 > ${NAMED_RUN_LINES_THRESHOLD} { print \$2 }" | sort | while read -r LOG_FILE; do + echowarn "I:${systest}:${LOG_FILE} contains more than ${NAMED_RUN_LINES_THRESHOLD} lines, consider tweaking the test to limit disk I/O" +done + +echoend "E:$systest:$(date_with_args)" + +exit $status |