From 1852910ef0fd7393da62b88aee66ee092208748e Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Sat, 27 Apr 2024 12:41:58 +0200 Subject: Adding upstream version 5.3.1. Signed-off-by: Daniel Baumann --- ci/README.md | 12 ++++ ci/debian-buster/Dockerfile | 143 ++++++++++++++++++++++++++++++++++++++ ci/deckard_commit_check.sh | 13 ++++ ci/gh_actions.py | 53 ++++++++++++++ ci/respdiff/kresd.config | 26 +++++++ ci/respdiff/respdiff-tcp.conf | 52 ++++++++++++++ ci/respdiff/respdiff-tls.conf | 52 ++++++++++++++ ci/respdiff/respdiff-udp.conf | 52 ++++++++++++++ ci/respdiff/restart-bind.sh | 3 + ci/respdiff/restart-kresd.sh | 12 ++++ ci/respdiff/restart-unbound.sh | 4 ++ ci/respdiff/run-respdiff-tests.sh | 27 +++++++ ci/respdiff/start-resolvers.sh | 13 ++++ 13 files changed, 462 insertions(+) create mode 100644 ci/README.md create mode 100644 ci/debian-buster/Dockerfile create mode 100755 ci/deckard_commit_check.sh create mode 100755 ci/gh_actions.py create mode 100644 ci/respdiff/kresd.config create mode 100644 ci/respdiff/respdiff-tcp.conf create mode 100644 ci/respdiff/respdiff-tls.conf create mode 100644 ci/respdiff/respdiff-udp.conf create mode 100755 ci/respdiff/restart-bind.sh create mode 100755 ci/respdiff/restart-kresd.sh create mode 100755 ci/respdiff/restart-unbound.sh create mode 100755 ci/respdiff/run-respdiff-tests.sh create mode 100755 ci/respdiff/start-resolvers.sh (limited to 'ci') diff --git a/ci/README.md b/ci/README.md new file mode 100644 index 0000000..fdc7b9f --- /dev/null +++ b/ci/README.md @@ -0,0 +1,12 @@ +Docker Build +------------ + +* debian-buster + +``` +$ export KNOT_BRANCH=3.0 +$ docker build --no-cache -t registry.nic.cz/knot/knot-resolver/ci/debian-buster:knot-$KNOT_BRANCH --build-arg KNOT_BRANCH=$KNOT_BRANCH debian-buster + +$ docker login registry.nic.cz +$ docker push registry.nic.cz/knot/knot-resolver/ci/debian-buster:knot-$KNOT_BRANCH +``` diff --git a/ci/debian-buster/Dockerfile b/ci/debian-buster/Dockerfile new file mode 100644 index 0000000..bc215c2 --- /dev/null +++ b/ci/debian-buster/Dockerfile @@ -0,0 +1,143 @@ +# SPDX-License-Identifier: GPL-3.0-or-later + +FROM debian:buster +MAINTAINER Knot Resolver +# >= 3.0 needed because of --enable-xdp=yes +ARG KNOT_BRANCH=3.0 +ENV DEBIAN_FRONTEND=noninteractive + +WORKDIR /root +CMD ["/bin/bash"] + +# generic cleanup +RUN apt-get update -qq +# TODO: run upgrade once buster reaches a stable release +# RUN apt-get upgrade -y -qqq + +# Knot and Knot Resolver dependecies +RUN apt-get install -y -qqq git make cmake pkg-config meson \ + build-essential bsdmainutils libtool autoconf libcmocka-dev \ + liburcu-dev libgnutls28-dev libedit-dev liblmdb-dev libcap-ng-dev libsystemd-dev \ + libelf-dev libidn11-dev libuv1-dev \ + libluajit-5.1-dev lua-http libssl-dev libnghttp2-dev + +# Build and testing deps for Resolver's dnstap module (go stuff is just for testing) +RUN apt-get install -y -qqq \ + protobuf-c-compiler libprotobuf-c-dev libfstrm-dev \ + golang-any +RUN bash -c "go get github.com/{FiloSottile/gvt,cloudflare/dns,dnstap/golang-dnstap}" + +# documentation dependecies +RUN apt-get install -y -qqq doxygen python3-sphinx python3-breathe python3-sphinx-rtd-theme + +# Python packags required for Deckard CI +# Python: grab latest versions from PyPi +# (Augeas binding in Debian packages are slow and buggy) +RUN apt-get install -y -qqq python3-pip wget augeas-tools +RUN pip3 install --upgrade pip +RUN pip3 install pylint +RUN pip3 install pep8 +RUN pip3 install pytest-xdist +# tests/pytest dependencies: skip over broken versions +RUN pip3 install 'dnspython != 2.0.0' jinja2 'pytest != 6.0.0' pytest-html pytest-xdist + +# packet capture tools for Deckard +RUN apt-get install --no-install-suggests --no-install-recommends -y -qqq tcpdump wireshark-common + +# Faketime for Deckard +RUN apt-get install -y -qqq faketime + +# C dependencies for python-augeas +RUN apt-get install -y -qqq libaugeas-dev libffi-dev +# Python dependencies for Deckard +RUN wget https://gitlab.nic.cz/knot/deckard/raw/master/requirements.txt -O /tmp/deckard-req.txt +RUN pip3 install -r /tmp/deckard-req.txt + +# build and install latest version of Knot DNS +RUN git clone --depth=1 --branch=$KNOT_BRANCH https://gitlab.nic.cz/knot/knot-dns.git /tmp/knot +WORKDIR /tmp/knot +RUN pwd +RUN autoreconf -if +RUN ./configure --prefix=/usr --enable-xdp=yes +RUN CFLAGS="-g" make +RUN make install +RUN ldconfig + +# Valgrind for kresd CI +RUN apt-get install valgrind -y -qqq +RUN wget https://github.com/LuaJIT/LuaJIT/raw/v2.1.0-beta3/src/lj.supp -O /lj.supp +# TODO: rebuild LuaJIT with Valgrind support + +# Lua lint for kresd CI +RUN apt-get install luarocks -y -qqq +RUN luarocks --lua-version 5.1 install luacheck + +# respdiff for kresd CI +RUN apt-get install lmdb-utils -y -qqq +RUN git clone --depth=1 https://gitlab.nic.cz/knot/respdiff /var/opt/respdiff +RUN pip3 install -r /var/opt/respdiff/requirements.txt + +# Python static analysis for respdiff +RUN pip3 install mypy +RUN pip3 install flake8 + +# Python requests for CI scripts +RUN pip3 install requests + +# docker-py for packaging tests +RUN pip3 install docker + +# Unbound for respdiff +RUN apt-get install unbound unbound-anchor -y -qqq +RUN printf "server:\n interface: 127.0.0.1@53535\n use-syslog: yes\n do-ip6: no\nremote-control:\n control-enable: no\n" >> /etc/unbound/unbound.conf + +# BIND for respdiff +RUN apt-get install bind9 -y -qqq +RUN printf '\nOPTIONS="-4 $OPTIONS"' >> /etc/default/bind9 +RUN printf 'options {\n directory "/var/cache/bind";\n listen-on port 53533 { 127.0.0.1; };\n listen-on-v6 port 53533 { ::1; };\n};\n' > /etc/bind/named.conf.options + +# PowerDNS Recursor for Deckard CI +RUN apt-get install pdns-recursor -y -qqq + +# code coverage +RUN apt-get install -y -qqq lcov +RUN luarocks --lua-version 5.1 install luacov + +# LuaJIT binary for stand-alone scripting +RUN apt-get install -y -qqq luajit + +# clang for kresd CI, version updated as debian updates it +RUN apt-get install -y -qqq clang clang-tools clang-tidy + +# OpenBuildService CLI tool +RUN apt-get install -y osc + +# curl (API) +RUN apt-get install -y curl + +# configure knot-resolver-testing OBS repo for dependencies missing in Debian +RUN echo 'deb http://download.opensuse.org/repositories/home:/CZ-NIC:/knot-resolver-testing/Debian_10/ /' > /etc/apt/sources.list.d/knot-resolver-testing.list +RUN wget -nv https://download.opensuse.org/repositories/home:CZ-NIC:knot-resolver-testing/Debian_10/Release.key -O Release.key +RUN APT_KEY_DONT_WARN_ON_DANGEROUS_USAGE=1 apt-key add Release.key +RUN rm Release.key +RUN apt-get update -qq + +# packages from our knot-resolver-testing repo +RUN apt-get install -y -qqq lua-http lua-psl + +# en_US.UTF-8 locale for scripts.update-authors.sh +RUN apt-get install -y -qqq locales +RUN sed -i "/en_US.UTF-8/ s/^#\(.*\)/\1/" /etc/locale.gen +RUN locale-gen + +# SonarCloud scanner +RUN wget -O /var/opt/wrapper.zip https://sonarcloud.io/static/cpp/build-wrapper-linux-x86.zip +RUN wget -O /var/opt/scanner.zip https://binaries.sonarsource.com/Distribution/sonar-scanner-cli/sonar-scanner-cli-4.4.0.2170-linux.zip +RUN unzip -d /var/opt /var/opt/wrapper.zip +RUN unzip -d /var/opt /var/opt/scanner.zip +ENV PATH "$PATH:/var/opt/build-wrapper-linux-x86:/var/opt/sonar-scanner-4.4.0.2170-linux/bin" + +# let's get newer meson from backports +RUN echo 'deb http://deb.debian.org/debian buster-backports main' > /etc/apt/sources.list.d/backports.list +RUN apt-get update -qq +RUN apt-get -t buster-backports install -y -qqq meson diff --git a/ci/deckard_commit_check.sh b/ci/deckard_commit_check.sh new file mode 100755 index 0000000..5b4016d --- /dev/null +++ b/ci/deckard_commit_check.sh @@ -0,0 +1,13 @@ +DECKARD_COMMIT=$(git ls-tree HEAD:tests/integration/ | grep commit | grep deckard | cut -f1 | cut -f3 '-d ') +DECKARD_PATH="tests/integration/deckard" +pushd $DECKARD_PATH > /dev/null +if git merge-base --is-ancestor $DECKARD_COMMIT origin/master; then + echo "Deckard submodule commit is on in its master branch. All good in the hood." + exit 0 +else + echo "Deckard submodule commit $DECKARD_COMMIT is not in Deckard's master branch." + echo "This WILL cause CI breakages so make sure your changes in Deckard are merged" + echo "or point the submodule to another commit." + exit 1 +fi + diff --git a/ci/gh_actions.py b/ci/gh_actions.py new file mode 100755 index 0000000..c452c69 --- /dev/null +++ b/ci/gh_actions.py @@ -0,0 +1,53 @@ +#!/usr/bin/python3 +# SPDX-License-Identifier: GPL-3.0-or-later +import json +import time +import sys + +import requests + + +BRANCH_API_ENDPOINT = "https://api.github.com/repos/CZ-NIC/knot-resolver/actions/runs?branch={branch}" # noqa +TIMEOUT = 15*60 # 15 mins max +POLL_DELAY = 60 + + +def exit(msg='', html_url='', code=1): + print(msg, file=sys.stderr) + print(html_url) + sys.exit(code) + + +end_time = time.time() + TIMEOUT +while time.time() < end_time: + response = requests.get( + BRANCH_API_ENDPOINT.format(branch=sys.argv[1]), + headers={"Accept": "application/vnd.github.v3+json"}) + if response.status_code == 404: + pass # not created yet? + elif response.status_code == 200: + data = json.loads(response.content.decode('utf-8')) + try: + run = data['workflow_runs'][0] + conclusion = run['conclusion'] + html_url = run['html_url'] + commit_sha = run['head_sha'] + except (KeyError, IndexError): + time.sleep(POLL_DELAY) + continue + + if commit_sha != sys.argv[2]: + exit("Fetched invalid GH Action: commit mismatch. Re-run or push again?") + + if conclusion is None: + pass + if conclusion == "success": + exit("SUCCESS!", html_url, code=0) + elif isinstance(conclusion, str): + # failure, neutral, cancelled, skipped, timed_out, or action_required + exit("GitHub Actions Conclusion: {}!".format(conclusion.upper()), html_url) + else: + exit("API Response Code: {}".format(response.status_code), code=2) + time.sleep(POLL_DELAY) + +exit("Timed out!") diff --git a/ci/respdiff/kresd.config b/ci/respdiff/kresd.config new file mode 100644 index 0000000..be5c458 --- /dev/null +++ b/ci/respdiff/kresd.config @@ -0,0 +1,26 @@ +-- SPDX-License-Identifier: GPL-3.0-or-later +-- Refer to manual: https://knot-resolver.readthedocs.io/en/stable/ +-- Listen on localhost and external interface +net.listen('127.0.0.1', 5353) +net.listen('127.0.0.1', 8853, { tls = true }) +net.ipv6=false + +-- Auto-maintain root TA +trust_anchors.add_file('.local/etc/knot-resolver/root.keys') + +cache.size = 1024 * MB + +-- Load Useful modules +modules = { + 'workarounds < iterate', + 'policy', -- Block queries to local zones/bad sites + 'view', -- Views for certain clients + 'hints', -- Load /etc/hosts and allow custom root hints + 'stats', -- Track internal statistics +} + +-- avoid TC flags returned to respdiff +local _, up_bs = net.bufsize() +net.bufsize(4096, up_bs) + +verbose(true) diff --git a/ci/respdiff/respdiff-tcp.conf b/ci/respdiff/respdiff-tcp.conf new file mode 100644 index 0000000..b2d40ff --- /dev/null +++ b/ci/respdiff/respdiff-tcp.conf @@ -0,0 +1,52 @@ +# SPDX-License-Identifier: GPL-3.0-or-later + +[sendrecv] +# in seconds +timeout = 11 +# number of queries to run simultaneously +jobs = 64 +# in seconds (float); delay each query by a random time (uniformly distributed) between min and max; set max to 0 to disable +time_delay_min = 0 +time_delay_max = 0 + +[servers] +names = kresd, bind, unbound +# symbolic names of DNS servers under test +# separate multiple values by , + +# each symbolic name in [servers] section refers to config section +# containing IP address and port of particular server +[kresd] +ip = 127.0.0.1 +port = 5353 +transport = tcp +graph_color = #00a2e2 +restart_script = ./ci/respdiff/restart-kresd.sh + +[bind] +ip = 127.0.0.1 +port = 53533 +transport = udp +graph_color = #e2a000 +restart_script = ./ci/respdiff/restart-bind.sh + +[unbound] +ip = 127.0.0.1 +port = 53535 +transport = udp +graph_color = #218669 +restart_script = ./ci/respdiff/restart-unbound.sh + +[diff] +# symbolic name of server under test +# other servers are used as reference when comparing answers from the target +target = kresd + +# fields and comparison methods used when comparing two DNS messages +criteria = opcode, rcode, flags, question, answertypes, answerrrsigs +# other supported criteria values: authority, additional, edns, nsid + +[report] +# diffsum reports mismatches in field values in this order +# if particular message has multiple mismatches, it is counted only once into category with highest weight +field_weights = timeout, malformed, opcode, question, rcode, flags, answertypes, answerrrsigs, answer, authority, additional, edns, nsid diff --git a/ci/respdiff/respdiff-tls.conf b/ci/respdiff/respdiff-tls.conf new file mode 100644 index 0000000..1a50eab --- /dev/null +++ b/ci/respdiff/respdiff-tls.conf @@ -0,0 +1,52 @@ +# SPDX-License-Identifier: GPL-3.0-or-later + +[sendrecv] +# in seconds +timeout = 11 +# number of queries to run simultaneously +jobs = 64 +# in seconds (float); delay each query by a random time (uniformly distributed) between min and max; set max to 0 to disable +time_delay_min = 0 +time_delay_max = 0 + +[servers] +names = kresd, bind, unbound +# symbolic names of DNS servers under test +# separate multiple values by , + +# each symbolic name in [servers] section refers to config section +# containing IP address and port of particular server +[kresd] +ip = 127.0.0.1 +port = 8853 +transport = tls +graph_color = #00a2e2 +restart_script = ./ci/respdiff/restart-kresd.sh + +[bind] +ip = 127.0.0.1 +port = 53533 +transport = udp +graph_color = #e2a000 +restart_script = ./ci/respdiff/restart-bind.sh + +[unbound] +ip = 127.0.0.1 +port = 53535 +transport = udp +graph_color = #218669 +restart_script = ./ci/respdiff/restart-unbound.sh + +[diff] +# symbolic name of server under test +# other servers are used as reference when comparing answers from the target +target = kresd + +# fields and comparison methods used when comparing two DNS messages +criteria = opcode, rcode, flags, question, answertypes, answerrrsigs +# other supported criteria values: authority, additional, edns, nsid + +[report] +# diffsum reports mismatches in field values in this order +# if particular message has multiple mismatches, it is counted only once into category with highest weight +field_weights = timeout, malformed, opcode, question, rcode, flags, answertypes, answerrrsigs, answer, authority, additional, edns, nsid diff --git a/ci/respdiff/respdiff-udp.conf b/ci/respdiff/respdiff-udp.conf new file mode 100644 index 0000000..35a69a9 --- /dev/null +++ b/ci/respdiff/respdiff-udp.conf @@ -0,0 +1,52 @@ +# SPDX-License-Identifier: GPL-3.0-or-later + +[sendrecv] +# in seconds +timeout = 11 +# number of queries to run simultaneously +jobs = 64 +# in seconds (float); delay each query by a random time (uniformly distributed) between min and max; set max to 0 to disable +time_delay_min = 0 +time_delay_max = 0 + +[servers] +names = kresd, bind, unbound +# symbolic names of DNS servers under test +# separate multiple values by , + +# each symbolic name in [servers] section refers to config section +# containing IP address and port of particular server +[kresd] +ip = 127.0.0.1 +port = 5353 +transport = udp +graph_color = #00a2e2 +restart_script = ./ci/respdiff/restart-kresd.sh + +[bind] +ip = 127.0.0.1 +port = 53533 +transport = udp +graph_color = #e2a000 +restart_script = ./ci/respdiff/restart-bind.sh + +[unbound] +ip = 127.0.0.1 +port = 53535 +transport = udp +graph_color = #218669 +restart_script = ./ci/respdiff/restart-unbound.sh + +[diff] +# symbolic name of server under test +# other servers are used as reference when comparing answers from the target +target = kresd + +# fields and comparison methods used when comparing two DNS messages +criteria = opcode, rcode, flags, question, answertypes, answerrrsigs +# other supported criteria values: authority, additional, edns, nsid + +[report] +# diffsum reports mismatches in field values in this order +# if particular message has multiple mismatches, it is counted only once into category with highest weight +field_weights = timeout, malformed, opcode, question, rcode, flags, answertypes, answerrrsigs, answer, authority, additional, edns, nsid diff --git a/ci/respdiff/restart-bind.sh b/ci/respdiff/restart-bind.sh new file mode 100755 index 0000000..d7b5157 --- /dev/null +++ b/ci/respdiff/restart-bind.sh @@ -0,0 +1,3 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-3.0-or-later +service bind9 restart diff --git a/ci/respdiff/restart-kresd.sh b/ci/respdiff/restart-kresd.sh new file mode 100755 index 0000000..4e9387c --- /dev/null +++ b/ci/respdiff/restart-kresd.sh @@ -0,0 +1,12 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-3.0-or-later + +exec > /dev/null +exec 2>&1 + +killall -w kresd +rm -f '*.mdb' +$PREFIX/sbin/kresd -n -q -c $(pwd)/ci/respdiff/kresd.config &>>kresd.log & + +# wait until socket is receiving connections +sleep 1 diff --git a/ci/respdiff/restart-unbound.sh b/ci/respdiff/restart-unbound.sh new file mode 100755 index 0000000..add24c9 --- /dev/null +++ b/ci/respdiff/restart-unbound.sh @@ -0,0 +1,4 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-3.0-or-later + +service unbound restart diff --git a/ci/respdiff/run-respdiff-tests.sh b/ci/respdiff/run-respdiff-tests.sh new file mode 100755 index 0000000..2bfc44d --- /dev/null +++ b/ci/respdiff/run-respdiff-tests.sh @@ -0,0 +1,27 @@ +#!/bin/bash +# SPDX-License-Identifier: GPL-3.0-or-later + +# $1 == udp/tcp/tls, it selects configuration file to use +# respdiff scripts must be present in /var/opt/respdiff +set -o errexit -o nounset -o xtrace + +NDIFFREPRO=3 + +wget -qO- https://gitlab.nic.cz/knot/respdiff/snippets/238/raw?inline=false | head -n 5000 > /tmp/queries.txt +mkdir results +rm -rf respdiff.db + +CONFIG="$(pwd)/ci/respdiff/respdiff-${1}.conf" +/var/opt/respdiff/qprep.py respdiff.db < /tmp/queries.txt +time /var/opt/respdiff/orchestrator.py respdiff.db -c "${CONFIG}" +time /var/opt/respdiff/msgdiff.py respdiff.db -c "${CONFIG}" +for i in $(seq $NDIFFREPRO); do + time /var/opt/respdiff/diffrepro.py -c "${CONFIG}" respdiff.db +done +/var/opt/respdiff/diffsum.py respdiff.db -c "${CONFIG}" > results/respdiff.txt +/var/opt/respdiff/histogram.py respdiff.db -c "${CONFIG}" -o results/histogram.svg +: minimize LMDB and log size so they can be effectively archived +mkdir results/respdiff.db +mdb_copy -c respdiff.db results/respdiff.db +xz -9 results/respdiff.db/data.mdb +xz kresd.log diff --git a/ci/respdiff/start-resolvers.sh b/ci/respdiff/start-resolvers.sh new file mode 100755 index 0000000..f9aa695 --- /dev/null +++ b/ci/respdiff/start-resolvers.sh @@ -0,0 +1,13 @@ +# SPDX-License-Identifier: GPL-3.0-or-later + +#run unbound +service unbound start && service unbound status; +# dig @localhost -p 53535 + +#run bind +service bind9 start && service bind9 status; +# dig @localhost -p 53533 + +#run kresd +$PREFIX/sbin/kresd -n -q -c $(pwd)/ci/respdiff/kresd.config &>kresd.log & +# dig @localhost -p 5353 -- cgit v1.2.3