diff options
author | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-04-07 15:26:00 +0000 |
---|---|---|
committer | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-04-07 15:26:00 +0000 |
commit | 830407e88f9d40d954356c3754f2647f91d5c06a (patch) | |
tree | d6a0ece6feea91f3c656166dbaa884ef8a29740e /tests/integration/deckard/ci | |
parent | Initial commit. (diff) | |
download | knot-resolver-830407e88f9d40d954356c3754f2647f91d5c06a.tar.xz knot-resolver-830407e88f9d40d954356c3754f2647f91d5c06a.zip |
Adding upstream version 5.6.0.upstream/5.6.0upstream
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to '')
-rw-r--r-- | tests/integration/deckard/ci/README.rst | 4 | ||||
-rw-r--r-- | tests/integration/deckard/ci/__init__.py | 0 | ||||
-rw-r--r-- | tests/integration/deckard/ci/common.sh | 17 | ||||
-rwxr-xr-x | tests/integration/deckard/ci/compare-rplint.sh | 31 | ||||
-rwxr-xr-x | tests/integration/deckard/ci/compare-tests.sh | 30 | ||||
-rwxr-xr-x | tests/integration/deckard/ci/junit-compare.py | 38 | ||||
-rwxr-xr-x | tests/integration/deckard/ci/mypy-run.sh | 17 | ||||
-rwxr-xr-x | tests/integration/deckard/ci/pylint-run.sh | 12 | ||||
-rwxr-xr-x | tests/integration/deckard/ci/raw_id.py | 20 | ||||
-rwxr-xr-x | tests/integration/deckard/ci/raw_id_check.sh | 7 | ||||
-rwxr-xr-x | tests/integration/deckard/ci/runlocally.sh | 41 |
11 files changed, 217 insertions, 0 deletions
diff --git a/tests/integration/deckard/ci/README.rst b/tests/integration/deckard/ci/README.rst new file mode 100644 index 0000000..39955f9 --- /dev/null +++ b/tests/integration/deckard/ci/README.rst @@ -0,0 +1,4 @@ +Deckard CI +========== + +For testing with Deckard in Docker use kresd ci image: registry.labs.nic.cz/knot/knot-resolver/ci:debian-stable diff --git a/tests/integration/deckard/ci/__init__.py b/tests/integration/deckard/ci/__init__.py new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/tests/integration/deckard/ci/__init__.py diff --git a/tests/integration/deckard/ci/common.sh b/tests/integration/deckard/ci/common.sh new file mode 100644 index 0000000..5430e64 --- /dev/null +++ b/tests/integration/deckard/ci/common.sh @@ -0,0 +1,17 @@ +set -o errexit -o nounset + +HEAD="$(git log -1 --format="%H" HEAD)" +MERGEBASE="$(git merge-base origin/master "${HEAD}")" +LOGDIR="$(pwd)" +PYTHON=${PYTHON:-"python3"} +CIDIR="$(dirname "${0}")" + +# workaround for Gitlab's missing support for absolute paths in artifacts: +# https://gitlab.com/gitlab-org/gitlab-ci-multi-runner/issues/1011 +declare -a LOGS +LOGS[0]="" # avoid unbound variable error if user does not specify own logs +function collect_logs { + set +o errexit + test -n "${LOGS[*]}" && cp "--target-directory=${LOGDIR}" ${LOGS[*]} +} +trap collect_logs EXIT diff --git a/tests/integration/deckard/ci/compare-rplint.sh b/tests/integration/deckard/ci/compare-rplint.sh new file mode 100755 index 0000000..0146dc9 --- /dev/null +++ b/tests/integration/deckard/ci/compare-rplint.sh @@ -0,0 +1,31 @@ +#!/bin/bash +set -o nounset -o xtrace -o errexit +source "$(dirname "$(readlink -f "$0")")/common.sh" + + +function find_new_tests { + : detect tests affected by current merge request + : store list of modified tests in ${NEW_TESTS_FILE} + git diff --name-only --diff-filter=AM ${MERGEBASE} ${HEAD} | fgrep .rpl > "${NEW_TESTS_FILE}" || : no new tests detected +} + +NEW_TESTS_FILE="/tmp/new_tests" +find_new_tests + +truncate -s0 /tmp/rplint_fails + +: run rplint of all new tests +FAIL=0 +cat /tmp/new_tests +for test in $(cat ${NEW_TESTS_FILE}) +do + ${PYTHON} -m rplint $test >> /tmp/rplint_fails || FAIL=1 +done + +: if even one of the test does not pass rplint, fail +if [ "$FAIL" -eq 1 ] +then + cat /tmp/rplint_fails + exit 1 +fi +exit 0 diff --git a/tests/integration/deckard/ci/compare-tests.sh b/tests/integration/deckard/ci/compare-tests.sh new file mode 100755 index 0000000..63d8c67 --- /dev/null +++ b/tests/integration/deckard/ci/compare-tests.sh @@ -0,0 +1,30 @@ +#!/bin/bash +set -o nounset -o xtrace -o errexit +source "$(dirname "$0")/common.sh" +TESTRUNNER="$1" +: comparing results from test script "${TESTRUNNER}" + +# Run specified test runner on HEAD and again on merge base for master..HEAD +# Fail if result of any test not modified between master..HEAD changed +# (i.e. any change in Deckard should not change results of non-modified tests) + +function find_modified_tests { + : detect tests affected by current merge request + : store list of modified tests in ${MODIFIED_TESTS_FILE} + git diff --numstat "${MERGEBASE}..${HEAD}" | cut -f 3 | fgrep .rpl > "${MODIFIED_TESTS_FILE}" || : no modified tests detected +} + +MODIFIED_TESTS_FILE="/tmp/modified_tests" +find_modified_tests +LOGS[0]="${MODIFIED_TESTS_FILE}" +LOGS[1]="/tmp/base.xml" +LOGS[2]="/tmp/head.xml" + +: get test results from version under test +"${TESTRUNNER}" -n $(nproc) --junit-xml=/tmp/head.xml || : some tests on HEAD ${HEAD} failed + +: get test results from common ancestor with master branch +git checkout --force --detach "${MERGEBASE}" +git clean -xdf +"${TESTRUNNER}" -n $(nproc) --junit-xml=/tmp/base.xml || : some tests on merge base ${MERGEBASE} failed +"${CIDIR}/junit-compare.py" /tmp/head.xml /tmp/base.xml /tmp/modified_tests && echo "OK, no differences found" diff --git a/tests/integration/deckard/ci/junit-compare.py b/tests/integration/deckard/ci/junit-compare.py new file mode 100755 index 0000000..23409bb --- /dev/null +++ b/tests/integration/deckard/ci/junit-compare.py @@ -0,0 +1,38 @@ +#!/usr/bin/python3 + +import sys + +import xml.etree.ElementTree as xml + + +def parse_junit_xml(filename): + """ + Transform junit XML file into set of tuples: + (test description, file name, test result) + """ + results = set() + suite = xml.parse(filename).getroot().find("testsuite") + for case in suite: + if case.find("failure") is not None: # Because empty XML elements are falsey + results.add((case.get("name"), case.get("name").split("'")[1], "FAILED")) + elif case.find("skipped") is not None: + results.add((case.get("name"), case.get("name").split("'")[1], "SKIPPED")) + else: + results.add((case.get("name"), case.get("name").split("'")[1], "PASSED")) + + return results + + +new = sys.argv[1] +old = sys.argv[2] +with open(sys.argv[3]) as f: + modified_tests = [line.strip() for line in f.readlines()] + +test_diffs = parse_junit_xml(old) ^ parse_junit_xml(new) +errorneous_rpls = [diff[1] for diff in test_diffs + if diff[1] not in modified_tests] +if errorneous_rpls: + print('FAIL! Following tests changed their result without test modification:') + for rpl in sorted(set(errorneous_rpls)): + print(rpl) + sys.exit(1) diff --git a/tests/integration/deckard/ci/mypy-run.sh b/tests/integration/deckard/ci/mypy-run.sh new file mode 100755 index 0000000..1abfef1 --- /dev/null +++ b/tests/integration/deckard/ci/mypy-run.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +set -o nounset -o errexit +source "$(dirname "$0")/common.sh" + +PYFILES=$(find . \ + -path ./.git -prune -o \ + -path ./contrib -o \ + -path ./tools -prune -o \ + -type d -exec test -e '{}/__init__.py' \; -print -prune -o \ + -name '*.py' -print -o \ + -type f -exec grep -qsm1 '^#!.*\bpython' '{}' \; -print) +set -e + +${PYTHON} -m mypy --ignore-missing-imports ${PYFILES} + +# tools +${PYTHON} -m mypy --ignore-missing-imports tools diff --git a/tests/integration/deckard/ci/pylint-run.sh b/tests/integration/deckard/ci/pylint-run.sh new file mode 100755 index 0000000..107c674 --- /dev/null +++ b/tests/integration/deckard/ci/pylint-run.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -o nounset -o errexit +source "$(dirname "$0")/common.sh" + +PYFILES=$(find . \ + -path ./.git -prune -o \ + -path ./contrib -o \ + -type d -exec test -e '{}/__init__.py' \; -print -prune -o \ + -name '*.py' -print -o \ + -type f -exec grep -qsm1 '^#!.*\bpython' '{}' \; -print) + +PYTHONPATH=.${PYTHONPATH:+":${PYTHONPATH}"} ${PYTHON} -m pylint -j 0 --rcfile pylintrc ${PYFILES} diff --git a/tests/integration/deckard/ci/raw_id.py b/tests/integration/deckard/ci/raw_id.py new file mode 100755 index 0000000..cc4fe8e --- /dev/null +++ b/tests/integration/deckard/ci/raw_id.py @@ -0,0 +1,20 @@ +#!/usr/bin/python3 +import sys + +import dns.message + +from pydnstest import mock_client + +q = dns.message.make_query("anything", "A") +qid = q.id.to_bytes(2, "big", signed=False) +sock = mock_client.setup_socket("1.2.3.4", 53) +mock_client.send_query(sock, q) +a = mock_client.get_answer(sock) +print(qid) +print(a) + + +if a == qid: + sys.exit(0) +else: + sys.exit(1) diff --git a/tests/integration/deckard/ci/raw_id_check.sh b/tests/integration/deckard/ci/raw_id_check.sh new file mode 100755 index 0000000..336b37b --- /dev/null +++ b/tests/integration/deckard/ci/raw_id_check.sh @@ -0,0 +1,7 @@ +#!/bin/bash +make depend +cat env.sh +source env.sh +python3 -m pydnstest.testserver --scenario $(pwd)/tests/deckard_raw_id.rpl & +sleep 1 +python3 -m ci.raw_id
\ No newline at end of file diff --git a/tests/integration/deckard/ci/runlocally.sh b/tests/integration/deckard/ci/runlocally.sh new file mode 100755 index 0000000..25bf0c9 --- /dev/null +++ b/tests/integration/deckard/ci/runlocally.sh @@ -0,0 +1,41 @@ +#!/bin/bash +set -o nounset -o errexit +CIDIR="$(dirname "$0")" +ORIGNAME="$(git symbolic-ref -q --short HEAD || git describe --all --always HEAD)" +FAILURE_DETECTED="?" + +function checkout_back { + git checkout --force "${ORIGNAME}" || { echo "Warning: unable to checkout back!"; exit 5; } + + test "${FAILURE_DETECTED}" "==" "0" && echo "All tests passed, good work!" + test "${FAILURE_DETECTED}" "!=" "0" && echo "Problem found, go fix it!" +} + +STATUS=$(git status --untracked-files=normal --porcelain) +test -n "${STATUS}" && echo "Working tree is dirty, commit your changes now." && exit 2 + +# return back to whatever branch we were on the beginning +# to avoid need for git checkout before fixing reported bugs +trap checkout_back EXIT +trap "{ FAILURE_DETECTED=1; }" ERR + +"${CIDIR}"/compare-rplint.sh +checkout_back +git clean -xdf + +"${CIDIR}"/compare-pylint.sh +checkout_back +git clean -xdf + +"${CIDIR}"/compare-pep8.sh +checkout_back +git clean -xdf + +"${CIDIR}"/compare-tests.sh "${CIDIR}/../kresd_run.sh" +checkout_back +git clean -xdf + +# at this point all the tests passed so we can clean up +git clean -xdf +FAILURE_DETECTED=0 +trap - ERR |