Adding upstream version 5.3.1.upstream/5.3.1upstream

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

13 files changed, 462 insertions, 0 deletions

diff --git a/ci/README.md b/ci/README.md
new file mode 100644
index 0000000..fdc7b9f
--- /dev/null
+++ b/ci/README.md
@@ -0,0 +1,12 @@
+Docker Build
+------------
+
+* debian-buster
+
+```
+$ export KNOT_BRANCH=3.0
+$ docker build --no-cache -t registry.nic.cz/knot/knot-resolver/ci/debian-buster:knot-$KNOT_BRANCH --build-arg KNOT_BRANCH=$KNOT_BRANCH debian-buster
+
+$ docker login registry.nic.cz
+$ docker push registry.nic.cz/knot/knot-resolver/ci/debian-buster:knot-$KNOT_BRANCH
+```
diff --git a/ci/debian-buster/Dockerfile b/ci/debian-buster/Dockerfile
new file mode 100644
index 0000000..bc215c2
--- /dev/null
+++ b/ci/debian-buster/Dockerfile
@@ -0,0 +1,143 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+FROM debian:buster
+MAINTAINER Knot Resolver <knot-resolver@labs.nic.cz>
+# >= 3.0 needed because of --enable-xdp=yes
+ARG KNOT_BRANCH=3.0
+ENV DEBIAN_FRONTEND=noninteractive
+
+WORKDIR /root
+CMD ["/bin/bash"]
+
+# generic cleanup
+RUN apt-get update -qq
+# TODO: run upgrade once buster reaches a stable release
+# RUN apt-get upgrade -y -qqq
+
+# Knot and Knot Resolver dependecies
+RUN apt-get install -y -qqq git make cmake pkg-config meson \
+	build-essential bsdmainutils libtool autoconf libcmocka-dev \
+	liburcu-dev libgnutls28-dev libedit-dev liblmdb-dev libcap-ng-dev libsystemd-dev \
+	libelf-dev libidn11-dev libuv1-dev \
+	libluajit-5.1-dev lua-http libssl-dev libnghttp2-dev
+
+# Build and testing deps for Resolver's dnstap module (go stuff is just for testing)
+RUN apt-get install -y -qqq \
+	protobuf-c-compiler libprotobuf-c-dev libfstrm-dev \
+	golang-any
+RUN bash -c "go get github.com/{FiloSottile/gvt,cloudflare/dns,dnstap/golang-dnstap}"
+
+# documentation dependecies
+RUN apt-get install -y -qqq doxygen python3-sphinx python3-breathe python3-sphinx-rtd-theme
+
+# Python packags required for Deckard CI
+# Python: grab latest versions from PyPi
+# (Augeas binding in Debian packages are slow and buggy)
+RUN apt-get install -y -qqq python3-pip wget augeas-tools
+RUN pip3 install --upgrade pip
+RUN pip3 install pylint
+RUN pip3 install pep8
+RUN pip3 install pytest-xdist
+# tests/pytest dependencies: skip over broken versions
+RUN pip3 install 'dnspython != 2.0.0' jinja2 'pytest != 6.0.0' pytest-html pytest-xdist
+
+# packet capture tools for Deckard
+RUN apt-get install --no-install-suggests --no-install-recommends -y -qqq tcpdump wireshark-common
+
+# Faketime for Deckard
+RUN apt-get install -y -qqq faketime
+
+# C dependencies for python-augeas
+RUN apt-get install -y -qqq libaugeas-dev libffi-dev
+# Python dependencies for Deckard
+RUN wget https://gitlab.nic.cz/knot/deckard/raw/master/requirements.txt -O /tmp/deckard-req.txt
+RUN pip3 install -r /tmp/deckard-req.txt
+
+# build and install latest version of Knot DNS
+RUN git clone --depth=1 --branch=$KNOT_BRANCH https://gitlab.nic.cz/knot/knot-dns.git /tmp/knot
+WORKDIR /tmp/knot
+RUN pwd
+RUN autoreconf -if
+RUN ./configure --prefix=/usr --enable-xdp=yes
+RUN CFLAGS="-g" make
+RUN make install
+RUN ldconfig
+
+# Valgrind for kresd CI
+RUN apt-get install valgrind -y -qqq
+RUN wget https://github.com/LuaJIT/LuaJIT/raw/v2.1.0-beta3/src/lj.supp -O /lj.supp
+# TODO: rebuild LuaJIT with Valgrind support
+
+# Lua lint for kresd CI
+RUN apt-get install luarocks -y -qqq
+RUN luarocks --lua-version 5.1 install luacheck
+
+# respdiff for kresd CI
+RUN apt-get install lmdb-utils -y -qqq
+RUN git clone --depth=1 https://gitlab.nic.cz/knot/respdiff /var/opt/respdiff
+RUN pip3 install -r /var/opt/respdiff/requirements.txt
+
+# Python static analysis for respdiff
+RUN pip3 install mypy
+RUN pip3 install flake8
+
+# Python requests for CI scripts
+RUN pip3 install requests
+
+# docker-py for packaging tests
+RUN pip3 install docker
+
+# Unbound for respdiff
+RUN apt-get install unbound unbound-anchor -y -qqq
+RUN printf "server:\n interface: 127.0.0.1@53535\n use-syslog: yes\n do-ip6: no\nremote-control:\n control-enable: no\n" >> /etc/unbound/unbound.conf
+
+# BIND for respdiff
+RUN apt-get install bind9 -y -qqq
+RUN printf '\nOPTIONS="-4 $OPTIONS"' >> /etc/default/bind9
+RUN printf 'options {\n directory "/var/cache/bind";\n listen-on port 53533 { 127.0.0.1; };\n listen-on-v6 port 53533 { ::1; };\n};\n' > /etc/bind/named.conf.options
+
+# PowerDNS Recursor for Deckard CI
+RUN apt-get install pdns-recursor -y -qqq
+
+# code coverage
+RUN apt-get install -y -qqq lcov
+RUN luarocks --lua-version 5.1 install luacov
+
+# LuaJIT binary for stand-alone scripting
+RUN apt-get install -y -qqq luajit
+
+# clang for kresd CI, version updated as debian updates it
+RUN apt-get install -y -qqq clang clang-tools clang-tidy
+
+# OpenBuildService CLI tool
+RUN apt-get install -y osc
+
+# curl (API)
+RUN apt-get install -y curl
+
+# configure knot-resolver-testing OBS repo for dependencies missing in Debian
+RUN echo 'deb http://download.opensuse.org/repositories/home:/CZ-NIC:/knot-resolver-testing/Debian_10/ /' > /etc/apt/sources.list.d/knot-resolver-testing.list
+RUN wget -nv https://download.opensuse.org/repositories/home:CZ-NIC:knot-resolver-testing/Debian_10/Release.key -O Release.key
+RUN APT_KEY_DONT_WARN_ON_DANGEROUS_USAGE=1 apt-key add Release.key
+RUN rm Release.key
+RUN apt-get update -qq
+
+# packages from our knot-resolver-testing repo
+RUN apt-get install -y -qqq lua-http lua-psl
+
+# en_US.UTF-8 locale for scripts.update-authors.sh
+RUN apt-get install -y -qqq locales
+RUN sed -i "/en_US.UTF-8/ s/^#\(.*\)/\1/" /etc/locale.gen
+RUN locale-gen
+
+# SonarCloud scanner
+RUN wget -O /var/opt/wrapper.zip https://sonarcloud.io/static/cpp/build-wrapper-linux-x86.zip
+RUN wget -O /var/opt/scanner.zip https://binaries.sonarsource.com/Distribution/sonar-scanner-cli/sonar-scanner-cli-4.4.0.2170-linux.zip
+RUN unzip -d /var/opt /var/opt/wrapper.zip
+RUN unzip -d /var/opt /var/opt/scanner.zip
+ENV PATH "$PATH:/var/opt/build-wrapper-linux-x86:/var/opt/sonar-scanner-4.4.0.2170-linux/bin"
+
+# let's get newer meson from backports
+RUN echo 'deb http://deb.debian.org/debian buster-backports main' > /etc/apt/sources.list.d/backports.list
+RUN apt-get update -qq
+RUN apt-get -t buster-backports install -y -qqq meson
diff --git a/ci/deckard_commit_check.sh b/ci/deckard_commit_check.sh
new file mode 100755
index 0000000..5b4016d
--- /dev/null
+++ b/ci/deckard_commit_check.sh
@@ -0,0 +1,13 @@
+DECKARD_COMMIT=$(git ls-tree HEAD:tests/integration/ | grep commit | grep deckard | cut -f1 | cut -f3 '-d ')
+DECKARD_PATH="tests/integration/deckard"
+pushd $DECKARD_PATH > /dev/null
+if git merge-base --is-ancestor $DECKARD_COMMIT origin/master; then
+	echo "Deckard submodule commit is on in its master branch. All good in the hood."
+	exit 0
+else
+	echo "Deckard submodule commit $DECKARD_COMMIT is not in Deckard's master branch."
+	echo "This WILL cause CI breakages so make sure your changes in Deckard are merged"
+	echo "or point the submodule to another commit."
+	exit 1
+fi
+
diff --git a/ci/gh_actions.py b/ci/gh_actions.py
new file mode 100755
index 0000000..c452c69
--- /dev/null
+++ b/ci/gh_actions.py
@@ -0,0 +1,53 @@
+#!/usr/bin/python3
+# SPDX-License-Identifier: GPL-3.0-or-later
+import json
+import time
+import sys
+
+import requests
+
+
+BRANCH_API_ENDPOINT = "https://api.github.com/repos/CZ-NIC/knot-resolver/actions/runs?branch={branch}"  # noqa
+TIMEOUT = 15*60  # 15 mins max
+POLL_DELAY = 60
+
+
+def exit(msg='', html_url='', code=1):
+    print(msg, file=sys.stderr)
+    print(html_url)
+    sys.exit(code)
+
+
+end_time = time.time() + TIMEOUT
+while time.time() < end_time:
+    response = requests.get(
+        BRANCH_API_ENDPOINT.format(branch=sys.argv[1]),
+        headers={"Accept": "application/vnd.github.v3+json"})
+    if response.status_code == 404:
+        pass  # not created yet?
+    elif response.status_code == 200:
+        data = json.loads(response.content.decode('utf-8'))
+        try:
+            run = data['workflow_runs'][0]
+            conclusion = run['conclusion']
+            html_url = run['html_url']
+            commit_sha = run['head_sha']
+        except (KeyError, IndexError):
+            time.sleep(POLL_DELAY)
+            continue
+
+        if commit_sha != sys.argv[2]:
+            exit("Fetched invalid GH Action: commit mismatch. Re-run or push again?")
+
+        if conclusion is None:
+            pass
+        if conclusion == "success":
+            exit("SUCCESS!", html_url, code=0)
+        elif isinstance(conclusion, str):
+            # failure, neutral, cancelled, skipped, timed_out, or action_required
+            exit("GitHub Actions Conclusion: {}!".format(conclusion.upper()), html_url)
+    else:
+        exit("API Response Code: {}".format(response.status_code), code=2)
+    time.sleep(POLL_DELAY)
+
+exit("Timed out!")
diff --git a/ci/respdiff/kresd.config b/ci/respdiff/kresd.config
new file mode 100644
index 0000000..be5c458
--- /dev/null
+++ b/ci/respdiff/kresd.config
@@ -0,0 +1,26 @@
+-- SPDX-License-Identifier: GPL-3.0-or-later
+-- Refer to manual: https://knot-resolver.readthedocs.io/en/stable/
+-- Listen on localhost and external interface
+net.listen('127.0.0.1', 5353)
+net.listen('127.0.0.1', 8853, { tls = true })
+net.ipv6=false
+
+-- Auto-maintain root TA
+trust_anchors.add_file('.local/etc/knot-resolver/root.keys')
+
+cache.size = 1024 * MB
+
+-- Load Useful modules
+modules = {
+	'workarounds < iterate',
+	'policy',   -- Block queries to local zones/bad sites
+	'view',     -- Views for certain clients
+	'hints',    -- Load /etc/hosts and allow custom root hints
+	'stats',    -- Track internal statistics
+}
+
+-- avoid TC flags returned to respdiff
+local _, up_bs = net.bufsize()
+net.bufsize(4096, up_bs)
+
+verbose(true)
diff --git a/ci/respdiff/respdiff-tcp.conf b/ci/respdiff/respdiff-tcp.conf
new file mode 100644
index 0000000..b2d40ff
--- /dev/null
+++ b/ci/respdiff/respdiff-tcp.conf
@@ -0,0 +1,52 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+[sendrecv]
+# in seconds
+timeout = 11
+# number of queries to run simultaneously
+jobs = 64
+# in seconds (float); delay each query by a random time (uniformly distributed) between min and max; set max to 0 to disable
+time_delay_min = 0
+time_delay_max = 0
+
+[servers]
+names = kresd, bind, unbound
+# symbolic names of DNS servers under test
+# separate multiple values by ,
+
+# each symbolic name in [servers] section refers to config section
+# containing IP address and port of particular server
+[kresd]
+ip = 127.0.0.1
+port = 5353
+transport = tcp
+graph_color = #00a2e2
+restart_script = ./ci/respdiff/restart-kresd.sh
+
+[bind]
+ip = 127.0.0.1
+port = 53533
+transport = udp
+graph_color = #e2a000
+restart_script = ./ci/respdiff/restart-bind.sh
+
+[unbound]
+ip = 127.0.0.1
+port = 53535
+transport = udp
+graph_color = #218669
+restart_script = ./ci/respdiff/restart-unbound.sh
+
+[diff]
+# symbolic name of server under test
+# other servers are used as reference when comparing answers from the target
+target = kresd
+
+# fields and comparison methods used when comparing two DNS messages
+criteria = opcode, rcode, flags, question, answertypes, answerrrsigs
+# other supported criteria values: authority, additional, edns, nsid
+
+[report]
+# diffsum reports mismatches in field values in this order
+# if particular message has multiple mismatches, it is counted only once into category with highest weight
+field_weights = timeout, malformed, opcode, question, rcode, flags, answertypes, answerrrsigs, answer, authority, additional, edns, nsid
diff --git a/ci/respdiff/respdiff-tls.conf b/ci/respdiff/respdiff-tls.conf
new file mode 100644
index 0000000..1a50eab
--- /dev/null
+++ b/ci/respdiff/respdiff-tls.conf
@@ -0,0 +1,52 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+[sendrecv]
+# in seconds
+timeout = 11
+# number of queries to run simultaneously
+jobs = 64
+# in seconds (float); delay each query by a random time (uniformly distributed) between min and max; set max to 0 to disable
+time_delay_min = 0
+time_delay_max = 0
+
+[servers]
+names = kresd, bind, unbound
+# symbolic names of DNS servers under test
+# separate multiple values by ,
+
+# each symbolic name in [servers] section refers to config section
+# containing IP address and port of particular server
+[kresd]
+ip = 127.0.0.1
+port = 8853
+transport = tls
+graph_color = #00a2e2
+restart_script = ./ci/respdiff/restart-kresd.sh
+
+[bind]
+ip = 127.0.0.1
+port = 53533
+transport = udp
+graph_color = #e2a000
+restart_script = ./ci/respdiff/restart-bind.sh
+
+[unbound]
+ip = 127.0.0.1
+port = 53535
+transport = udp
+graph_color = #218669
+restart_script = ./ci/respdiff/restart-unbound.sh
+
+[diff]
+# symbolic name of server under test
+# other servers are used as reference when comparing answers from the target
+target = kresd
+
+# fields and comparison methods used when comparing two DNS messages
+criteria = opcode, rcode, flags, question, answertypes, answerrrsigs
+# other supported criteria values: authority, additional, edns, nsid
+
+[report]
+# diffsum reports mismatches in field values in this order
+# if particular message has multiple mismatches, it is counted only once into category with highest weight
+field_weights = timeout, malformed, opcode, question, rcode, flags, answertypes, answerrrsigs, answer, authority, additional, edns, nsid
diff --git a/ci/respdiff/respdiff-udp.conf b/ci/respdiff/respdiff-udp.conf
new file mode 100644
index 0000000..35a69a9
--- /dev/null
+++ b/ci/respdiff/respdiff-udp.conf
@@ -0,0 +1,52 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+[sendrecv]
+# in seconds
+timeout = 11
+# number of queries to run simultaneously
+jobs = 64
+# in seconds (float); delay each query by a random time (uniformly distributed) between min and max; set max to 0 to disable
+time_delay_min = 0
+time_delay_max = 0
+
+[servers]
+names = kresd, bind, unbound
+# symbolic names of DNS servers under test
+# separate multiple values by ,
+
+# each symbolic name in [servers] section refers to config section
+# containing IP address and port of particular server
+[kresd]
+ip = 127.0.0.1
+port = 5353
+transport = udp
+graph_color = #00a2e2
+restart_script = ./ci/respdiff/restart-kresd.sh
+
+[bind]
+ip = 127.0.0.1
+port = 53533
+transport = udp
+graph_color = #e2a000
+restart_script = ./ci/respdiff/restart-bind.sh
+
+[unbound]
+ip = 127.0.0.1
+port = 53535
+transport = udp
+graph_color = #218669
+restart_script = ./ci/respdiff/restart-unbound.sh
+
+[diff]
+# symbolic name of server under test
+# other servers are used as reference when comparing answers from the target
+target = kresd
+
+# fields and comparison methods used when comparing two DNS messages
+criteria = opcode, rcode, flags, question, answertypes, answerrrsigs
+# other supported criteria values: authority, additional, edns, nsid
+
+[report]
+# diffsum reports mismatches in field values in this order
+# if particular message has multiple mismatches, it is counted only once into category with highest weight
+field_weights = timeout, malformed, opcode, question, rcode, flags, answertypes, answerrrsigs, answer, authority, additional, edns, nsid
diff --git a/ci/respdiff/restart-bind.sh b/ci/respdiff/restart-bind.sh
new file mode 100755
index 0000000..d7b5157
--- /dev/null
+++ b/ci/respdiff/restart-bind.sh
@@ -0,0 +1,3 @@
+#!/bin/sh
+# SPDX-License-Identifier: GPL-3.0-or-later
+service bind9 restart
diff --git a/ci/respdiff/restart-kresd.sh b/ci/respdiff/restart-kresd.sh
new file mode 100755
index 0000000..4e9387c
--- /dev/null
+++ b/ci/respdiff/restart-kresd.sh
@@ -0,0 +1,12 @@
+#!/bin/sh
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+exec > /dev/null
+exec 2>&1
+
+killall -w kresd
+rm -f '*.mdb'
+$PREFIX/sbin/kresd -n -q -c $(pwd)/ci/respdiff/kresd.config &>>kresd.log &
+
+# wait until socket is receiving connections
+sleep 1
diff --git a/ci/respdiff/restart-unbound.sh b/ci/respdiff/restart-unbound.sh
new file mode 100755
index 0000000..add24c9
--- /dev/null
+++ b/ci/respdiff/restart-unbound.sh
@@ -0,0 +1,4 @@
+#!/bin/sh
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+service unbound restart
diff --git a/ci/respdiff/run-respdiff-tests.sh b/ci/respdiff/run-respdiff-tests.sh
new file mode 100755
index 0000000..2bfc44d
--- /dev/null
+++ b/ci/respdiff/run-respdiff-tests.sh
@@ -0,0 +1,27 @@
+#!/bin/bash
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+# $1 == udp/tcp/tls, it selects configuration file to use
+# respdiff scripts must be present in /var/opt/respdiff
+set -o errexit -o nounset -o xtrace
+
+NDIFFREPRO=3
+
+wget -qO- https://gitlab.nic.cz/knot/respdiff/snippets/238/raw?inline=false | head -n 5000 > /tmp/queries.txt
+mkdir results
+rm -rf respdiff.db
+
+CONFIG="$(pwd)/ci/respdiff/respdiff-${1}.conf"
+/var/opt/respdiff/qprep.py respdiff.db < /tmp/queries.txt
+time /var/opt/respdiff/orchestrator.py respdiff.db -c "${CONFIG}"
+time /var/opt/respdiff/msgdiff.py respdiff.db -c "${CONFIG}"
+for i in $(seq $NDIFFREPRO); do
+	time /var/opt/respdiff/diffrepro.py -c "${CONFIG}" respdiff.db
+done
+/var/opt/respdiff/diffsum.py respdiff.db -c "${CONFIG}" > results/respdiff.txt
+/var/opt/respdiff/histogram.py respdiff.db -c "${CONFIG}" -o results/histogram.svg
+: minimize LMDB and log size so they can be effectively archived
+mkdir results/respdiff.db
+mdb_copy -c respdiff.db results/respdiff.db
+xz -9 results/respdiff.db/data.mdb
+xz kresd.log
diff --git a/ci/respdiff/start-resolvers.sh b/ci/respdiff/start-resolvers.sh
new file mode 100755
index 0000000..f9aa695
--- /dev/null
+++ b/ci/respdiff/start-resolvers.sh
@@ -0,0 +1,13 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+#run unbound
+service unbound start && service unbound status;
+# dig @localhost -p 53535
+
+#run bind
+service bind9 start && service bind9 status;
+# dig @localhost -p 53533
+
+#run kresd
+$PREFIX/sbin/kresd -n -q -c $(pwd)/ci/respdiff/kresd.config &>kresd.log &
+# dig @localhost -p 5353