8 files changed, 185 insertions, 0 deletions
diff --git a/tests/deckard/ci/README.rst b/tests/deckard/ci/README.rst
new file mode 100644
index 0000000..39955f9
--- /dev/null
+++ b/tests/deckard/ci/README.rst
@@ -0,0 +1,4 @@
+Deckard CI
+==========
+
+For testing with Deckard in Docker use kresd ci image: registry.labs.nic.cz/knot/knot-resolver/ci:debian-stable
diff --git a/tests/deckard/ci/common.sh b/tests/deckard/ci/common.sh
new file mode 100644
index 0000000..5430e64
--- /dev/null
+++ b/tests/deckard/ci/common.sh
@@ -0,0 +1,17 @@
+set -o errexit -o nounset
+
+HEAD="$(git log -1 --format="%H" HEAD)"
+MERGEBASE="$(git merge-base origin/master "${HEAD}")"
+LOGDIR="$(pwd)"
+PYTHON=${PYTHON:-"python3"}
+CIDIR="$(dirname "${0}")"
+
+# workaround for Gitlab's missing support for absolute paths in artifacts:
+# https://gitlab.com/gitlab-org/gitlab-ci-multi-runner/issues/1011
+declare -a LOGS
+LOGS[0]=""  # avoid unbound variable error if user does not specify own logs
+function collect_logs {
+	set +o errexit
+	test -n "${LOGS[*]}" && cp "--target-directory=${LOGDIR}" ${LOGS[*]}
+}
+trap collect_logs EXIT
diff --git a/tests/deckard/ci/compare-rplint.sh b/tests/deckard/ci/compare-rplint.sh
new file mode 100755
index 0000000..0146dc9
--- /dev/null
+++ b/tests/deckard/ci/compare-rplint.sh
@@ -0,0 +1,31 @@
+#!/bin/bash
+set -o nounset -o xtrace -o errexit
+source "$(dirname "$(readlink -f "$0")")/common.sh"
+
+
+function find_new_tests {
+	: detect tests affected by current merge request
+	: store list of modified tests in ${NEW_TESTS_FILE}
+  git diff --name-only --diff-filter=AM ${MERGEBASE} ${HEAD} | fgrep .rpl > "${NEW_TESTS_FILE}" || : no new tests detected
+}
+
+NEW_TESTS_FILE="/tmp/new_tests"
+find_new_tests
+
+truncate -s0 /tmp/rplint_fails
+
+: run rplint of all new tests
+FAIL=0
+cat /tmp/new_tests
+for test in $(cat ${NEW_TESTS_FILE})
+do
+  ${PYTHON} -m rplint $test >> /tmp/rplint_fails || FAIL=1
+done
+
+: if even one of the test does not pass rplint, fail
+if [ "$FAIL" -eq 1 ]
+then
+  cat /tmp/rplint_fails
+  exit 1
+fi
+exit 0
diff --git a/tests/deckard/ci/compare-tests.sh b/tests/deckard/ci/compare-tests.sh
new file mode 100755
index 0000000..63d8c67
--- /dev/null
+++ b/tests/deckard/ci/compare-tests.sh
@@ -0,0 +1,30 @@
+#!/bin/bash
+set -o nounset -o xtrace -o errexit
+source "$(dirname "$0")/common.sh"
+TESTRUNNER="$1"
+: comparing results from test script "${TESTRUNNER}"
+
+# Run specified test runner on HEAD and again on merge base for master..HEAD
+# Fail if result of any test not modified between master..HEAD changed
+# (i.e. any change in Deckard should not change results of non-modified tests)
+
+function find_modified_tests {
+	: detect tests affected by current merge request
+	: store list of modified tests in ${MODIFIED_TESTS_FILE}
+	git diff --numstat "${MERGEBASE}..${HEAD}" | cut -f 3 | fgrep .rpl > "${MODIFIED_TESTS_FILE}" || : no modified tests detected
+}
+
+MODIFIED_TESTS_FILE="/tmp/modified_tests"
+find_modified_tests
+LOGS[0]="${MODIFIED_TESTS_FILE}"
+LOGS[1]="/tmp/base.xml"
+LOGS[2]="/tmp/head.xml"
+
+: get test results from version under test
+"${TESTRUNNER}" -n $(nproc) --junit-xml=/tmp/head.xml || : some tests on HEAD ${HEAD} failed
+
+: get test results from common ancestor with master branch
+git checkout --force --detach "${MERGEBASE}"
+git clean -xdf
+"${TESTRUNNER}" -n $(nproc) --junit-xml=/tmp/base.xml || : some tests on merge base ${MERGEBASE} failed
+"${CIDIR}/junit-compare.py" /tmp/head.xml /tmp/base.xml /tmp/modified_tests && echo "OK, no differences found"
diff --git a/tests/deckard/ci/junit-compare.py b/tests/deckard/ci/junit-compare.py
new file mode 100755
index 0000000..54ca71a
--- /dev/null
+++ b/tests/deckard/ci/junit-compare.py
@@ -0,0 +1,37 @@
+#!/usr/bin/python3
+
+import sys
+
+import xml.etree.ElementTree as xml
+
+
+def parse_junit_xml(filename):
+    """
+    Transform junit XML file into set of tuples:
+    (test description, file name, test result)
+    """
+    results = set()
+    root = xml.parse(filename).getroot()
+    for case in root:
+        if case.find("failure") is not None:  # Because empty XML elements are falsey
+            results.add((case.get("name"), case.get("name").split("'")[1], "FAILED"))
+        elif case.find("skipped") is not None:
+            results.add((case.get("name"), case.get("name").split("'")[1], "SKIPPED"))
+        else:
+            results.add((case.get("name"), case.get("name").split("'")[1], "PASSED"))
+
+    return results
+
+
+new = sys.argv[1]
+old = sys.argv[2]
+modified_tests = [line.strip() for line in open(sys.argv[3]).readlines()]
+
+test_diffs = parse_junit_xml(old) ^ parse_junit_xml(new)
+errorneous_rpls = [diff[1] for diff in test_diffs
+                   if diff[1] not in modified_tests]
+if errorneous_rpls:
+    print('FAIL! Following tests changed their result without test modification:')
+    for rpl in sorted(set(errorneous_rpls)):
+        print(rpl)
+    sys.exit(1)
diff --git a/tests/deckard/ci/mypy-run.sh b/tests/deckard/ci/mypy-run.sh
new file mode 100755
index 0000000..d99e256
--- /dev/null
+++ b/tests/deckard/ci/mypy-run.sh
@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+set -o nounset -o errexit
+source "$(dirname "$0")/common.sh"
+
+PYFILES=$(find . \
+	-path ./.git -prune -o \
+	-path ./contrib -o \
+	-type d -exec test -e '{}/__init__.py' \; -print -prune -o \
+	-name '*.py' -print -o \
+	-type f -exec grep -qsm1 '^#!.*\bpython' '{}' \; -print)
+set -e
+
+${PYTHON} -m mypy --ignore-missing-imports ${PYFILES}
diff --git a/tests/deckard/ci/pylint-run.sh b/tests/deckard/ci/pylint-run.sh
new file mode 100755
index 0000000..2df40ef
--- /dev/null
+++ b/tests/deckard/ci/pylint-run.sh
@@ -0,0 +1,12 @@
+#!/bin/bash
+set -o nounset -o errexit
+source "$(dirname "$0")/common.sh"
+
+PYFILES=$(find . \
+	-path ./.git -prune -o \
+	-path ./contrib -o \
+	-type d -exec test -e '{}/__init__.py' \; -print -prune -o \
+	-name '*.py' -print -o \
+	-type f -exec grep -qsm1 '^#!.*\bpython' '{}' \; -print)
+
+${PYTHON} -m pylint -j 0 --rcfile pylintrc ${PYFILES}
diff --git a/tests/deckard/ci/runlocally.sh b/tests/deckard/ci/runlocally.sh
new file mode 100755
index 0000000..25bf0c9
--- /dev/null
+++ b/tests/deckard/ci/runlocally.sh
@@ -0,0 +1,41 @@
+#!/bin/bash
+set -o nounset -o errexit
+CIDIR="$(dirname "$0")"
+ORIGNAME="$(git symbolic-ref -q --short HEAD || git describe --all --always HEAD)"
+FAILURE_DETECTED="?"
+
+function checkout_back {
+	git checkout --force "${ORIGNAME}" || { echo "Warning: unable to checkout back!"; exit 5; }
+
+	test "${FAILURE_DETECTED}" "==" "0" && echo "All tests passed, good work!"
+	test "${FAILURE_DETECTED}" "!=" "0" && echo "Problem found, go fix it!"
+}
+
+STATUS=$(git status --untracked-files=normal --porcelain)
+test -n "${STATUS}" && echo "Working tree is dirty, commit your changes now." && exit 2
+
+# return back to whatever branch we were on the beginning
+# to avoid need for git checkout before fixing reported bugs
+trap checkout_back EXIT
+trap "{ FAILURE_DETECTED=1; }" ERR
+
+"${CIDIR}"/compare-rplint.sh
+checkout_back
+git clean -xdf
+
+"${CIDIR}"/compare-pylint.sh
+checkout_back
+git clean -xdf
+
+"${CIDIR}"/compare-pep8.sh
+checkout_back
+git clean -xdf
+
+"${CIDIR}"/compare-tests.sh "${CIDIR}/../kresd_run.sh"
+checkout_back
+git clean -xdf
+
+# at this point all the tests passed so we can clean up
+git clean -xdf
+FAILURE_DETECTED=0
+trap - ERR