From 830407e88f9d40d954356c3754f2647f91d5c06a Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Sun, 7 Apr 2024 17:26:00 +0200 Subject: Adding upstream version 5.6.0. Signed-off-by: Daniel Baumann --- ci/deckard_commit_check.sh | 13 ++ ci/fix-meson-junit.sh | 5 + ci/gh_actions.py | 58 ++++++ ci/images/README.md | 41 +++++ ci/images/build.sh | 13 ++ ci/images/debian-11-coverity/Dockerfile | 43 +++++ ci/images/debian-11/Dockerfile | 144 +++++++++++++++ ci/images/debian-buster/Dockerfile | 145 +++++++++++++++ ci/images/push.sh | 8 + ci/images/update.sh | 22 +++ ci/images/vars.sh | 13 ++ ci/no_assert_check.sh | 3 + ci/pkgtest.yaml | 300 ++++++++++++++++++++++++++++++++ ci/respdiff/kresd.config | 26 +++ ci/respdiff/respdiff-tcp.conf | 52 ++++++ ci/respdiff/respdiff-tls.conf | 52 ++++++ ci/respdiff/respdiff-udp.conf | 52 ++++++ ci/respdiff/restart-bind.sh | 3 + ci/respdiff/restart-kresd.sh | 12 ++ ci/respdiff/restart-unbound.sh | 4 + ci/respdiff/run-respdiff-tests.sh | 27 +++ ci/respdiff/start-resolvers.sh | 13 ++ 22 files changed, 1049 insertions(+) create mode 100755 ci/deckard_commit_check.sh create mode 100755 ci/fix-meson-junit.sh create mode 100755 ci/gh_actions.py create mode 100644 ci/images/README.md create mode 100755 ci/images/build.sh create mode 100644 ci/images/debian-11-coverity/Dockerfile create mode 100644 ci/images/debian-11/Dockerfile create mode 100644 ci/images/debian-buster/Dockerfile create mode 100755 ci/images/push.sh create mode 100755 ci/images/update.sh create mode 100755 ci/images/vars.sh create mode 100755 ci/no_assert_check.sh create mode 100644 ci/pkgtest.yaml create mode 100644 ci/respdiff/kresd.config create mode 100644 ci/respdiff/respdiff-tcp.conf create mode 100644 ci/respdiff/respdiff-tls.conf create mode 100644 ci/respdiff/respdiff-udp.conf create mode 100755 ci/respdiff/restart-bind.sh create mode 100755 ci/respdiff/restart-kresd.sh create mode 100755 ci/respdiff/restart-unbound.sh create mode 100755 ci/respdiff/run-respdiff-tests.sh create mode 100755 ci/respdiff/start-resolvers.sh (limited to 'ci') diff --git a/ci/deckard_commit_check.sh b/ci/deckard_commit_check.sh new file mode 100755 index 0000000..5b4016d --- /dev/null +++ b/ci/deckard_commit_check.sh @@ -0,0 +1,13 @@ +DECKARD_COMMIT=$(git ls-tree HEAD:tests/integration/ | grep commit | grep deckard | cut -f1 | cut -f3 '-d ') +DECKARD_PATH="tests/integration/deckard" +pushd $DECKARD_PATH > /dev/null +if git merge-base --is-ancestor $DECKARD_COMMIT origin/master; then + echo "Deckard submodule commit is on in its master branch. All good in the hood." + exit 0 +else + echo "Deckard submodule commit $DECKARD_COMMIT is not in Deckard's master branch." + echo "This WILL cause CI breakages so make sure your changes in Deckard are merged" + echo "or point the submodule to another commit." + exit 1 +fi + diff --git a/ci/fix-meson-junit.sh b/ci/fix-meson-junit.sh new file mode 100755 index 0000000..02cf488 --- /dev/null +++ b/ci/fix-meson-junit.sh @@ -0,0 +1,5 @@ +#!/bin/sh +sed 's||\n|g' -i "$@" +sed -e '//,/<\/testcase>/s/<\(\/\?\)system-\(out\|err\)>/<\1failure>/g' \ + -e 's///g' \ + -i "$@" diff --git a/ci/gh_actions.py b/ci/gh_actions.py new file mode 100755 index 0000000..8fe05b6 --- /dev/null +++ b/ci/gh_actions.py @@ -0,0 +1,58 @@ +#!/usr/bin/python3 +# SPDX-License-Identifier: GPL-3.0-or-later +import json +import time +import sys + +import requests + + +BRANCH_API_ENDPOINT = "https://api.github.com/repos/CZ-NIC/knot-resolver/actions/runs?branch={branch}" # noqa +TIMEOUT = 20*60 # 20 mins max +POLL_DELAY = 60 +SYNC_TIMEOUT = 10*60 + + +def exit(msg='', html_url='', code=1): + print(msg, file=sys.stderr) + print(html_url) + sys.exit(code) + + +end_time = time.time() + TIMEOUT +sync_timeout = time.time() + SYNC_TIMEOUT +while time.time() < end_time: + response = requests.get( + BRANCH_API_ENDPOINT.format(branch=sys.argv[1]), + headers={"Accept": "application/vnd.github.v3+json"}) + if response.status_code == 404: + pass # not created yet? + elif response.status_code == 200: + data = json.loads(response.content.decode('utf-8')) + try: + run = data['workflow_runs'][0] + conclusion = run['conclusion'] + html_url = run['html_url'] + commit_sha = run['head_sha'] + except (KeyError, IndexError): + time.sleep(POLL_DELAY) + continue + + if commit_sha != sys.argv[2]: + if time.time() < sync_timeout: + time.sleep(POLL_DELAY) + continue + exit("Fetched invalid GH Action: commit mismatch. Re-run or push again?") + + if conclusion is None: + pass + if conclusion == "success": + exit("SUCCESS!", html_url, code=0) + elif isinstance(conclusion, str): + # failure, neutral, cancelled, skipped, timed_out, or action_required + exit("GitHub Actions Conclusion: {}!".format(conclusion.upper()), html_url) + else: + exit("API Response Code: {}".format(response.status_code), code=2) + time.sleep(POLL_DELAY) + +exit("Timed out!") diff --git a/ci/images/README.md b/ci/images/README.md new file mode 100644 index 0000000..d9efe0e --- /dev/null +++ b/ci/images/README.md @@ -0,0 +1,41 @@ +# Container images for CI + +## Image purpose + +### debian-11 + +The main image used by shared runners to execute most CI builds and tests. + +### debian-11-coverity + +A stripped down version of `debian-11`. It only contains build (not test) +dependencies of `kresd`. It also contains the `cov-build` tool for generating +inputs for [Coverity Scan](https://scan.coverity.com/). + +It is used by the `coverity` CI job to generate and send data to Coverity Scan +for analysis. + +To build this image, you need to retrieve the Coverity Scan token from the +dashboard and pass it to the `build.sh` script using the `COVERITY_SCAN_TOKEN` +environment variable, e.g.: + +``` +$ COVERITY_SCAN_TOKEN=the_secret_token ./build.sh debian-11-coverity +``` + +### debian-buster (10) + +Used to serve the same purpose as `debian-11`. As of 2022-03-09, it is still +used by some jobs (linters). + +## Maintenance + +The `ci/images/` directory contains utility scripts to build, push or update +the container images. + +``` +$ ./build.sh debian-11 # builds a debian-11 image locally +$ ./push.sh debian-11 # pushes the local image into target registry +$ ./update.sh debian-11 # utility wrapper that both builds and pushes the image +$ ./update.sh */ # use shell expansion of dirnames to update all images +``` diff --git a/ci/images/build.sh b/ci/images/build.sh new file mode 100755 index 0000000..39ee617 --- /dev/null +++ b/ci/images/build.sh @@ -0,0 +1,13 @@ +#!/bin/bash +# build specified docker image + +CURRENT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" +source "${CURRENT_DIR}"/vars.sh "$@" +set -ex + +if [ -n "$COVERITY_SCAN_TOKEN" ]; then + SECRETS="$SECRETS --secret id=coverity-token,env=COVERITY_SCAN_TOKEN" +fi + +export DOCKER_BUILDKIT=1 # Enables using secrets in docker-build +docker build --pull --no-cache -t "${FULL_NAME}" "${IMAGE}" --build-arg KNOT_BRANCH=${KNOT_BRANCH} $SECRETS diff --git a/ci/images/debian-11-coverity/Dockerfile b/ci/images/debian-11-coverity/Dockerfile new file mode 100644 index 0000000..1915614 --- /dev/null +++ b/ci/images/debian-11-coverity/Dockerfile @@ -0,0 +1,43 @@ +# SPDX-License-Identifier: GPL-3.0-or-later + +FROM debian:bullseye +MAINTAINER Knot Resolver +# >= 3.0 needed because of --enable-xdp=yes +ARG KNOT_BRANCH=3.1 +ARG COVERITY_SCAN_PROJECT_NAME=CZ-NIC/knot-resolver +ENV DEBIAN_FRONTEND=noninteractive + +WORKDIR /root +CMD ["/bin/bash"] + +# generic cleanup +RUN apt-get update -qq + +# Knot and Knot Resolver dependencies +RUN apt-get install -y -qqq git make cmake pkg-config meson \ + build-essential bsdmainutils libtool autoconf libcmocka-dev \ + liburcu-dev libgnutls28-dev libedit-dev liblmdb-dev libcap-ng-dev libsystemd-dev \ + libelf-dev libmnl-dev libidn11-dev libuv1-dev \ + libluajit-5.1-dev lua-http libssl-dev libnghttp2-dev + +# LuaJIT binary for stand-alone scripting +RUN apt-get install -y -qqq luajit + +# build and install latest version of Knot DNS +RUN git clone --depth=1 --branch=$KNOT_BRANCH https://gitlab.nic.cz/knot/knot-dns.git /tmp/knot +WORKDIR /tmp/knot +RUN pwd +RUN autoreconf -if +RUN ./configure --prefix=/usr --enable-xdp=yes +RUN CFLAGS="-g" make +RUN make install +RUN ldconfig + +# curl and tar (for downloading Coverity tools and uploading logs) +RUN apt-get install -y curl tar + +RUN --mount=type=secret,id=coverity-token \ + curl -o /tmp/cov-analysis-linux64.tar.gz https://scan.coverity.com/download/cxx/linux64 \ + --form project=$COVERITY_SCAN_PROJECT_NAME --form token=$(cat /run/secrets/coverity-token) +RUN tar xfz /tmp/cov-analysis-linux64.tar.gz +RUN mv cov-analysis-linux64-* /opt/cov-analysis diff --git a/ci/images/debian-11/Dockerfile b/ci/images/debian-11/Dockerfile new file mode 100644 index 0000000..59f170b --- /dev/null +++ b/ci/images/debian-11/Dockerfile @@ -0,0 +1,144 @@ +# SPDX-License-Identifier: GPL-3.0-or-later + +FROM debian:bullseye +MAINTAINER Knot Resolver +# >= 3.0 needed because of --enable-xdp=yes +ARG KNOT_BRANCH=3.1 +ENV DEBIAN_FRONTEND=noninteractive + +WORKDIR /root +CMD ["/bin/bash"] + +# generic cleanup +RUN apt-get update -qq + +# Knot and Knot Resolver dependencies +RUN apt-get install -y -qqq git make cmake pkg-config meson \ + build-essential bsdmainutils libtool autoconf libcmocka-dev \ + liburcu-dev libgnutls28-dev libedit-dev liblmdb-dev libcap-ng-dev libsystemd-dev \ + libelf-dev libmnl-dev libidn11-dev libuv1-dev libjemalloc-dev \ + libluajit-5.1-dev lua-http libssl-dev libnghttp2-dev + +# Build and testing deps for Resolver's dnstap module (go stuff is just for testing) +RUN apt-get install -y -qqq \ + protobuf-c-compiler libprotobuf-c-dev libfstrm-dev +# Maintaining the go stuff in CI really seems more trouble than worth. +# golang-any +#RUN bash -c "go get github.com/{FiloSottile/gvt,cloudflare/dns,dnstap/golang-dnstap,golang/protobuf/proto}" + +# documentation dependencies +RUN apt-get install -y -qqq doxygen python3-sphinx python3-breathe python3-sphinx-rtd-theme + +# Python packages required for Deckard CI +# Python: grab latest versions from PyPi +# (Augeas binding in Debian packages are slow and buggy) +RUN apt-get install -y -qqq python3-pip wget augeas-tools +RUN pip3 install --upgrade pip +RUN pip3 install pylint +RUN pip3 install pep8 +# FIXME replace with dnspython >= 2.2.0 once released +RUN pip3 install git+https://github.com/bwelling/dnspython.git@72348d4698a8f8b209fbdf9e72738904ad31b930 +# tests/pytest dependencies: skip over broken versions +RUN pip3 install jinja2 'pytest != 6.0.0' pytest-html pytest-xdist pytest-forked +# apkg for packaging +RUN pip3 install apkg + +# packet capture tools for Deckard +RUN apt-get install --no-install-suggests --no-install-recommends -y -qqq tcpdump wireshark-common + +# Faketime for Deckard +RUN apt-get install -y -qqq faketime + +# C dependencies for python-augeas +RUN apt-get install -y -qqq libaugeas-dev libffi-dev +# Python dependencies for Deckard +RUN wget https://gitlab.nic.cz/knot/deckard/raw/master/requirements.txt -O /tmp/deckard-req.txt +RUN pip3 install -r /tmp/deckard-req.txt + +# build and install latest version of Knot DNS +RUN git clone --depth=1 --branch=$KNOT_BRANCH https://gitlab.nic.cz/knot/knot-dns.git /tmp/knot +WORKDIR /tmp/knot +RUN pwd +RUN autoreconf -if +RUN ./configure --prefix=/usr --enable-xdp=yes +RUN CFLAGS="-g" make +RUN make install +RUN ldconfig + +# Valgrind for kresd CI +RUN apt-get install valgrind -y -qqq +RUN wget https://github.com/LuaJIT/LuaJIT/raw/v2.1.0-beta3/src/lj.supp -O /lj.supp +# TODO: rebuild LuaJIT with Valgrind support + +# Lua lint for kresd CI +RUN apt-get install luarocks -y -qqq +RUN luarocks --lua-version 5.1 install luacheck + +# respdiff for kresd CI +RUN apt-get install lmdb-utils -y -qqq +RUN git clone --depth=1 https://gitlab.nic.cz/knot/respdiff /var/opt/respdiff +RUN pip3 install -r /var/opt/respdiff/requirements.txt + +# Python static analysis for respdiff +RUN pip3 install mypy +RUN pip3 install flake8 + +# Python requests for CI scripts +RUN pip3 install requests + +# docker-py for packaging tests +RUN pip3 install docker + +# Unbound for respdiff +RUN apt-get install unbound unbound-anchor -y -qqq +RUN printf "server:\n interface: 127.0.0.1@53535\n use-syslog: yes\n do-ip6: no\nremote-control:\n control-enable: no\n" >> /etc/unbound/unbound.conf + +# BIND for respdiff +RUN apt-get install bind9 -y -qqq +RUN printf '\nOPTIONS="-4 $OPTIONS"' >> /etc/default/bind9 +RUN printf 'options {\n directory "/var/cache/bind";\n listen-on port 53533 { 127.0.0.1; };\n listen-on-v6 port 53533 { ::1; };\n};\n' > /etc/bind/named.conf.options + +# PowerDNS Recursor for Deckard CI +RUN apt-get install pdns-recursor -y -qqq + +# dnsdist for Deckard CI +RUN apt-get install dnsdist -y -qqq + +# code coverage +RUN apt-get install -y -qqq lcov +RUN luarocks --lua-version 5.1 install luacov + +# LuaJIT binary for stand-alone scripting +RUN apt-get install -y -qqq luajit + +# clang for kresd CI, version updated as debian updates it +RUN apt-get install -y -qqq clang clang-tools clang-tidy + +# OpenBuildService CLI tool +RUN apt-get install -y osc + +# curl (API) +RUN apt-get install -y curl + +# configure knot-resolver-testing OBS repo for dependencies missing in Debian +RUN echo 'deb http://download.opensuse.org/repositories/home:/CZ-NIC:/knot-resolver-testing/Debian_11/ /' > /etc/apt/sources.list.d/knot-resolver-testing.list +RUN wget -nv https://download.opensuse.org/repositories/home:CZ-NIC:knot-resolver-testing/Debian_11/Release.key -O Release.key +RUN APT_KEY_DONT_WARN_ON_DANGEROUS_USAGE=1 apt-key add Release.key +RUN rm Release.key +RUN apt-get update -qq + +# packages from our knot-resolver-testing repo +RUN apt-get update +RUN apt-get install -y -qqq lua-psl + +# en_US.UTF-8 locale for scripts.update-authors.sh +RUN apt-get install -y -qqq locales +RUN sed -i "/en_US.UTF-8/ s/^#\(.*\)/\1/" /etc/locale.gen +RUN locale-gen + +# SonarCloud scanner +RUN wget -O /var/opt/wrapper.zip https://sonarcloud.io/static/cpp/build-wrapper-linux-x86.zip +RUN wget -O /var/opt/scanner.zip https://binaries.sonarsource.com/Distribution/sonar-scanner-cli/sonar-scanner-cli-4.4.0.2170-linux.zip +RUN unzip -d /var/opt /var/opt/wrapper.zip +RUN unzip -d /var/opt /var/opt/scanner.zip +ENV PATH "$PATH:/var/opt/build-wrapper-linux-x86:/var/opt/sonar-scanner-4.4.0.2170-linux/bin" diff --git a/ci/images/debian-buster/Dockerfile b/ci/images/debian-buster/Dockerfile new file mode 100644 index 0000000..4b47dda --- /dev/null +++ b/ci/images/debian-buster/Dockerfile @@ -0,0 +1,145 @@ +# SPDX-License-Identifier: GPL-3.0-or-later + +FROM debian:buster +MAINTAINER Knot Resolver +# >= 3.0 needed because of --enable-xdp=yes +ARG KNOT_BRANCH=3.0 +ENV DEBIAN_FRONTEND=noninteractive + +WORKDIR /root +CMD ["/bin/bash"] + +# generic cleanup +RUN apt-get update -qq +# TODO: run upgrade once buster reaches a stable release +# RUN apt-get upgrade -y -qqq + +# Knot and Knot Resolver dependencies +RUN apt-get install -y -qqq git make cmake pkg-config meson \ + build-essential bsdmainutils libtool autoconf libcmocka-dev \ + liburcu-dev libgnutls28-dev libedit-dev liblmdb-dev libcap-ng-dev libsystemd-dev \ + libelf-dev libmnl-dev libidn11-dev libuv1-dev \ + libluajit-5.1-dev lua-http libssl-dev libnghttp2-dev + +# Build and testing deps for Resolver's dnstap module (go stuff is just for testing) +RUN apt-get install -y -qqq \ + protobuf-c-compiler libprotobuf-c-dev libfstrm-dev \ + golang-any +# Some stuff won't work on buster: +# package crypto/ed25519: unrecognized import path "crypto/ed25519" +#RUN bash -c "go get github.com/{FiloSottile/gvt,cloudflare/dns,dnstap/golang-dnstap}" + +# documentation dependencies +RUN apt-get install -y -qqq doxygen python3-sphinx python3-breathe python3-sphinx-rtd-theme + +# Python packages required for Deckard CI +# Python: grab latest versions from PyPi +# (Augeas binding in Debian packages are slow and buggy) +RUN apt-get install -y -qqq python3-pip wget augeas-tools +RUN pip3 install --upgrade pip +RUN pip3 install pylint +RUN pip3 install pep8 +RUN pip3 install pytest-xdist +# tests/pytest dependencies: skip over broken versions +RUN pip3 install 'dnspython != 2.0.0' jinja2 'pytest != 6.0.0' pytest-html pytest-xdist + +# packet capture tools for Deckard +RUN apt-get install --no-install-suggests --no-install-recommends -y -qqq tcpdump wireshark-common + +# Faketime for Deckard +RUN apt-get install -y -qqq faketime + +# C dependencies for python-augeas +RUN apt-get install -y -qqq libaugeas-dev libffi-dev +# Python dependencies for Deckard +RUN wget https://gitlab.nic.cz/knot/deckard/raw/master/requirements.txt -O /tmp/deckard-req.txt +RUN pip3 install -r /tmp/deckard-req.txt + +# build and install latest version of Knot DNS +RUN git clone --depth=1 --branch=$KNOT_BRANCH https://gitlab.nic.cz/knot/knot-dns.git /tmp/knot +WORKDIR /tmp/knot +RUN pwd +RUN autoreconf -if +RUN ./configure --prefix=/usr --enable-xdp=yes +RUN CFLAGS="-g" make +RUN make install +RUN ldconfig + +# Valgrind for kresd CI +RUN apt-get install valgrind -y -qqq +RUN wget https://github.com/LuaJIT/LuaJIT/raw/v2.1.0-beta3/src/lj.supp -O /lj.supp +# TODO: rebuild LuaJIT with Valgrind support + +# Lua lint for kresd CI +RUN apt-get install luarocks -y -qqq +RUN luarocks --lua-version 5.1 install luacheck + +# respdiff for kresd CI +RUN apt-get install lmdb-utils -y -qqq +RUN git clone --depth=1 https://gitlab.nic.cz/knot/respdiff /var/opt/respdiff +RUN pip3 install -r /var/opt/respdiff/requirements.txt + +# Python static analysis for respdiff +RUN pip3 install mypy +RUN pip3 install flake8 + +# Python requests for CI scripts +RUN pip3 install requests + +# docker-py for packaging tests +RUN pip3 install docker + +# Unbound for respdiff +RUN apt-get install unbound unbound-anchor -y -qqq +RUN printf "server:\n interface: 127.0.0.1@53535\n use-syslog: yes\n do-ip6: no\nremote-control:\n control-enable: no\n" >> /etc/unbound/unbound.conf + +# BIND for respdiff +RUN apt-get install bind9 -y -qqq +RUN printf '\nOPTIONS="-4 $OPTIONS"' >> /etc/default/bind9 +RUN printf 'options {\n directory "/var/cache/bind";\n listen-on port 53533 { 127.0.0.1; };\n listen-on-v6 port 53533 { ::1; };\n};\n' > /etc/bind/named.conf.options + +# PowerDNS Recursor for Deckard CI +RUN apt-get install pdns-recursor -y -qqq + +# code coverage +RUN apt-get install -y -qqq lcov +RUN luarocks --lua-version 5.1 install luacov + +# LuaJIT binary for stand-alone scripting +RUN apt-get install -y -qqq luajit + +# clang for kresd CI, version updated as debian updates it +RUN apt-get install -y -qqq clang clang-tools clang-tidy + +# OpenBuildService CLI tool +RUN apt-get install -y osc + +# curl (API) +RUN apt-get install -y curl + +# configure knot-resolver-testing OBS repo for dependencies missing in Debian +RUN echo 'deb http://download.opensuse.org/repositories/home:/CZ-NIC:/knot-resolver-testing/Debian_10/ /' > /etc/apt/sources.list.d/knot-resolver-testing.list +RUN wget -nv https://download.opensuse.org/repositories/home:CZ-NIC:knot-resolver-testing/Debian_10/Release.key -O Release.key +RUN APT_KEY_DONT_WARN_ON_DANGEROUS_USAGE=1 apt-key add Release.key +RUN rm Release.key +RUN apt-get update -qq + +# packages from our knot-resolver-testing repo +RUN apt-get install -y -qqq lua-http lua-psl + +# en_US.UTF-8 locale for scripts.update-authors.sh +RUN apt-get install -y -qqq locales +RUN sed -i "/en_US.UTF-8/ s/^#\(.*\)/\1/" /etc/locale.gen +RUN locale-gen + +# SonarCloud scanner +RUN wget -O /var/opt/wrapper.zip https://sonarcloud.io/static/cpp/build-wrapper-linux-x86.zip +RUN wget -O /var/opt/scanner.zip https://binaries.sonarsource.com/Distribution/sonar-scanner-cli/sonar-scanner-cli-4.4.0.2170-linux.zip +RUN unzip -d /var/opt /var/opt/wrapper.zip +RUN unzip -d /var/opt /var/opt/scanner.zip +ENV PATH "$PATH:/var/opt/build-wrapper-linux-x86:/var/opt/sonar-scanner-4.4.0.2170-linux/bin" + +# let's get newer meson from backports +RUN echo 'deb http://deb.debian.org/debian buster-backports main' > /etc/apt/sources.list.d/backports.list +RUN apt-get update -qq +RUN apt-get -t buster-backports install -y -qqq meson diff --git a/ci/images/push.sh b/ci/images/push.sh new file mode 100755 index 0000000..75f5f87 --- /dev/null +++ b/ci/images/push.sh @@ -0,0 +1,8 @@ +#!/bin/bash +# upload docker image into registry + +CURRENT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" +source "${CURRENT_DIR}"/vars.sh "$@" +set -ex + +docker push "${FULL_NAME}" diff --git a/ci/images/update.sh b/ci/images/update.sh new file mode 100755 index 0000000..7be5172 --- /dev/null +++ b/ci/images/update.sh @@ -0,0 +1,22 @@ +#!/bin/bash +# build and upload docker image(s) into registry +# +# this is a simple wrapper around build.sh and update.sh +# +# to build & upload all images: ./update.sh */ + +if [[ $# -le 0 ]]; then + echo "usage: $0 IMAGE..." + exit 1 +fi +set -e + +for ARG in "$@" +do + IMAGE=${ARG%/} + echo "Building $IMAGE..." + ./build.sh $IMAGE + echo "Pushing $IMAGE..." + ./push.sh $IMAGE +done + diff --git a/ci/images/vars.sh b/ci/images/vars.sh new file mode 100755 index 0000000..f2ea465 --- /dev/null +++ b/ci/images/vars.sh @@ -0,0 +1,13 @@ +#!/bin/bash +# define common variables for image build scripts + +KNOT_BRANCH="${KNOT_BRANCH:-3.1}" + +REGISTRY="registry.nic.cz/knot/knot-resolver/ci" +IMAGE=$1 +if [ -z "${IMAGE}" ]; then + echo "image name not provided" + exit 1 +fi +TAG="knot-${KNOT_BRANCH}" +FULL_NAME="${REGISTRY}/${IMAGE}:${TAG}" diff --git a/ci/no_assert_check.sh b/ci/no_assert_check.sh new file mode 100755 index 0000000..a3f3563 --- /dev/null +++ b/ci/no_assert_check.sh @@ -0,0 +1,3 @@ +#!/bin/sh +grep '\' -- $(git ls-files | grep '\.[hc]$' | grep -vE '^(contrib|bench|tests)/') +test $? -eq 1 diff --git a/ci/pkgtest.yaml b/ci/pkgtest.yaml new file mode 100644 index 0000000..74f6ec8 --- /dev/null +++ b/ci/pkgtest.yaml @@ -0,0 +1,300 @@ +default: + interruptible: true + +stages: + - pkgbuild + - pkgtest + +# pkgbuild {{{ +.pkgbuild: &pkgbuild + stage: pkgbuild + tags: + - lxc + - amd64 + before_script: + - git config --global user.name CI + - git config --global user.email ci@nic + needs: # https://gitlab.nic.cz/help/ci/yaml/README.md#artifact-downloads-to-child-pipelines + - pipeline: $PARENT_PIPELINE_ID + job: archive + artifacts: + when: always + expire_in: '1 day' + paths: + - pkg/ + +.apkgbuild: &apkgbuild # new jinja2 breaks docs (sphinx/breathe) + - pip3 install -U apkg 'jinja2<3.1' + - apkg build-dep -y + - apkg build + +.pkgdebrepo: &pkgdebrepo + - apt-get update + - apt-get install -y curl gnupg2 + - echo "deb http://download.opensuse.org/repositories/home:/CZ-NIC:/$OBS_REPO/$DISTROTEST_REPO/ /" > /etc/apt/sources.list.d/obs.list + - curl -fsSL "https://download.opensuse.org/repositories/home:CZ-NIC:$OBS_REPO/$DISTROTEST_REPO/Release.key" | gpg --dearmor > /etc/apt/trusted.gpg.d/obs.gpg + - apt-get update + +.debpkgbuild: &debpkgbuild + - *pkgdebrepo + - apt-get install -y python3-pip devscripts + - *apkgbuild + +centos-7:pkgbuild: + <<: *pkgbuild + image: $CI_REGISTRY/labs/lxc-gitlab-runner/centos-7 + before_script: + - export LC_ALL=en_US.UTF-8 + - git config --global user.name CI + - git config --global user.email ci@nic + script: + - yum install -y rpm-build python3-pip epel-release + - *apkgbuild + +debian-9:pkgbuild: + <<: *pkgbuild + image: $CI_REGISTRY/labs/lxc-gitlab-runner/debian-9 + variables: + OBS_REPO: knot-resolver-build + DISTROTEST_REPO: Debian_9.0 + script: + - *debpkgbuild + +debian-10:pkgbuild: + <<: *pkgbuild + image: $CI_REGISTRY/labs/lxc-gitlab-runner/debian-10 + variables: + OBS_REPO: knot-resolver-build + DISTROTEST_REPO: Debian_10 + script: + - *debpkgbuild + +debian-11:pkgbuild: + <<: *pkgbuild + image: $CI_REGISTRY/labs/lxc-gitlab-runner/debian-11 + variables: + OBS_REPO: knot-resolver-build + DISTROTEST_REPO: Debian_11 + script: + - *debpkgbuild + +fedora-34:pkgbuild: + <<: *pkgbuild + image: $CI_REGISTRY/labs/lxc-gitlab-runner/fedora-34 + script: + - dnf install -y rpm-build python3-pip + - *apkgbuild + +fedora-35:pkgbuild: + <<: *pkgbuild + image: $CI_REGISTRY/labs/lxc-gitlab-runner/fedora-35 + script: + - dnf install -y rpm-build python3-pip + - *apkgbuild + +opensuse-15.2:pkgbuild: + <<: *pkgbuild + image: $CI_REGISTRY/labs/lxc-gitlab-runner/opensuse-15.2 + script: + - zypper addrepo -G -f https://download.opensuse.org/repositories/home:CZ-NIC:knot-resolver-build/openSUSE_Leap_15.2/home:CZ-NIC:knot-resolver-build.repo + - zypper install -y rpm-build python3-pip + - *apkgbuild + +opensuse-15.3:pkgbuild: + <<: *pkgbuild + image: $CI_REGISTRY/labs/lxc-gitlab-runner/opensuse-15.3 + script: + - zypper addrepo -G -f https://download.opensuse.org/repositories/home:CZ-NIC:knot-resolver-build/openSUSE_Leap_15.3/home:CZ-NIC:knot-resolver-build.repo + - zypper install -y rpm-build python3-pip + - *apkgbuild + +rocky-8:pkgbuild: + <<: *pkgbuild + image: $CI_REGISTRY/labs/lxc-gitlab-runner/rocky-8 + script: + - dnf install -y rpm-build python3-pip epel-release dnf-plugins-core + - dnf config-manager --set-enabled powertools + - *apkgbuild + +ubuntu-18.04:pkgbuild: + <<: *pkgbuild + image: $CI_REGISTRY/labs/lxc-gitlab-runner/ubuntu-18.04 + variables: + OBS_REPO: knot-resolver-build + DISTROTEST_REPO: xUbuntu_18.04 + script: + - *debpkgbuild + +ubuntu-20.04:pkgbuild: + <<: *pkgbuild + image: $CI_REGISTRY/labs/lxc-gitlab-runner/ubuntu-20.04 + variables: + OBS_REPO: knot-resolver-build + DISTROTEST_REPO: xUbuntu_20.04 + script: + - *debpkgbuild + +nixos-unstable:pkgbuild: + <<: *pkgbuild + # We do NOT use LXC, for now at least. + parallel: + matrix: + - PLATFORM: [ amd64, arm64 ] + tags: + - docker + - linux + - ${PLATFORM} + image: nixos/nix + + variables: + NIX_PATH: nixpkgs=https://github.com/nixos/nixpkgs/archive/nixos-unstable.tar.gz + before_script: + script: + - nix-build '' -QA apkg + # the image auto-detects as alpine distro + # If apkg version differs (too much), it will fail to reuse archive and fail. + - ./result/bin/apkg install -d nix + - kresd --version +# }}} + +# pkgtest {{{ +.pkgtest: &pkgtest + stage: pkgtest + tags: + - lxc + - amd64 + +.debpkgtest: &debpkgtest + - *pkgdebrepo + - apt-get install -y knot-dnsutils + - apt-get install -y $(find ./pkg/pkgs -name '*.deb' | grep -v module | grep -v debug | grep -v devel) + - systemctl start kresd@1 + - kdig @127.0.0.1 nic.cz | grep -qi NOERROR + +centos-7:pkgtest: + <<: *pkgtest + needs: + - centos-7:pkgbuild + image: $CI_REGISTRY/labs/lxc-gitlab-runner/centos-7 + before_script: + - export LC_ALL=en_US.UTF-8 + script: + - yum install -y epel-release + - yum install -y knot-utils findutils + - yum install -y $(find ./pkg/pkgs -name '*.rpm' | grep -v module | grep -v debug | grep -v devel) + - systemctl start kresd@1 + - kdig @127.0.0.1 nic.cz | grep -qi NOERROR + +debian-9:pkgtest: + <<: *pkgtest + needs: + - debian-9:pkgbuild + image: $CI_REGISTRY/labs/lxc-gitlab-runner/debian-9 + variables: + OBS_REPO: knot-resolver-build + DISTROTEST_REPO: Debian_9.0 + script: + - *debpkgtest + +debian-10:pkgtest: + <<: *pkgtest + needs: + - debian-10:pkgbuild + image: $CI_REGISTRY/labs/lxc-gitlab-runner/debian-10 + variables: + OBS_REPO: knot-resolver-build + DISTROTEST_REPO: Debian_10 + script: + - *debpkgtest + +debian-11:pkgtest: + <<: *pkgtest + needs: + - debian-11:pkgbuild + image: $CI_REGISTRY/labs/lxc-gitlab-runner/debian-11 + variables: + OBS_REPO: knot-resolver-build + DISTROTEST_REPO: Debian_11 + script: + - *debpkgtest + +fedora-34:pkgtest: + <<: *pkgtest + needs: + - fedora-34:pkgbuild + image: $CI_REGISTRY/labs/lxc-gitlab-runner/fedora-34 + script: + - dnf install -y knot-utils findutils + - dnf install -y $(find ./pkg/pkgs -name '*.rpm' | grep -v module | grep -v debug | grep -v devel) + - systemctl start kresd@1 + - kdig @127.0.0.1 nic.cz | grep -qi NOERROR + +fedora-35:pkgtest: + <<: *pkgtest + needs: + - fedora-35:pkgbuild + image: $CI_REGISTRY/labs/lxc-gitlab-runner/fedora-35 + script: + - dnf install -y knot-utils findutils + - dnf install -y $(find ./pkg/pkgs -name '*.rpm' | grep -v module | grep -v debug | grep -v devel) + - systemctl start kresd@1 + - kdig @127.0.0.1 nic.cz | grep -qi NOERROR + +opensuse-15.2:pkgtest: + <<: *pkgtest + needs: + - opensuse-15.2:pkgbuild + image: $CI_REGISTRY/labs/lxc-gitlab-runner/opensuse-15.2 + script: + - zypper addrepo -G -f https://download.opensuse.org/repositories/home:CZ-NIC:knot-resolver-build/openSUSE_Leap_15.2/home:CZ-NIC:knot-resolver-build.repo + - zypper install -y knot-utils + - zypper install --allow-unsigned-rpm -y $(find ./pkg/pkgs -name '*.rpm' | grep -v module | grep -v debug | grep -v devel) + - systemctl start kresd@1 + - kdig @127.0.0.1 nic.cz | grep -qi NOERROR + +opensuse-15.3:pkgtest: + <<: *pkgtest + needs: + - opensuse-15.3:pkgbuild + image: $CI_REGISTRY/labs/lxc-gitlab-runner/opensuse-15.3 + script: + - zypper addrepo -G -f https://download.opensuse.org/repositories/home:CZ-NIC:knot-resolver-build/openSUSE_Leap_15.3/home:CZ-NIC:knot-resolver-build.repo + - zypper install -y knot-utils + - zypper install --allow-unsigned-rpm -y $(find ./pkg/pkgs -name '*.rpm' | grep -v module | grep -v debug | grep -v devel) + - systemctl start kresd@1 + - kdig @127.0.0.1 nic.cz | grep -qi NOERROR + +rocky-8:pkgtest: + <<: *pkgtest + needs: + - rocky-8:pkgbuild + image: $CI_REGISTRY/labs/lxc-gitlab-runner/rocky-8 + script: + - dnf install -y epel-release + - dnf install -y knot-utils findutils + - dnf install -y $(find ./pkg/pkgs -name '*.rpm' | grep -v module | grep -v debug | grep -v devel) + - systemctl start kresd@1 + - kdig @127.0.0.1 nic.cz | grep -qi NOERROR + +ubuntu-18.04:pkgtest: + <<: *pkgtest + needs: + - ubuntu-18.04:pkgbuild + image: $CI_REGISTRY/labs/lxc-gitlab-runner/ubuntu-18.04 + variables: + OBS_REPO: knot-resolver-build + DISTROTEST_REPO: xUbuntu_18.04 + script: + - *debpkgtest + +ubuntu-20.04:pkgtest: + <<: *pkgtest + needs: + - ubuntu-20.04:pkgbuild + image: $CI_REGISTRY/labs/lxc-gitlab-runner/ubuntu-20.04 + variables: + OBS_REPO: knot-resolver-build + DISTROTEST_REPO: xUbuntu_20.04 + script: + - *debpkgtest +# }}} diff --git a/ci/respdiff/kresd.config b/ci/respdiff/kresd.config new file mode 100644 index 0000000..2b7b218 --- /dev/null +++ b/ci/respdiff/kresd.config @@ -0,0 +1,26 @@ +-- SPDX-License-Identifier: GPL-3.0-or-later +-- Refer to manual: https://knot-resolver.readthedocs.io/en/stable/ +-- Listen on localhost and external interface +net.listen('127.0.0.1', 5353) +net.listen('127.0.0.1', 8853, { tls = true }) +net.ipv6=false + +-- Auto-maintain root TA +trust_anchors.add_file('.local/etc/knot-resolver/root.keys') + +cache.size = 1024 * MB + +-- Load Useful modules +modules = { + 'workarounds < iterate', + 'policy', -- Block queries to local zones/bad sites + 'view', -- Views for certain clients + 'hints > iterate', -- Allow loading /etc/hosts or custom root hints + 'stats', -- Track internal statistics +} + +-- avoid TC flags returned to respdiff +local _, up_bs = net.bufsize() +net.bufsize(4096, up_bs) + +log_level('debug') diff --git a/ci/respdiff/respdiff-tcp.conf b/ci/respdiff/respdiff-tcp.conf new file mode 100644 index 0000000..b2d40ff --- /dev/null +++ b/ci/respdiff/respdiff-tcp.conf @@ -0,0 +1,52 @@ +# SPDX-License-Identifier: GPL-3.0-or-later + +[sendrecv] +# in seconds +timeout = 11 +# number of queries to run simultaneously +jobs = 64 +# in seconds (float); delay each query by a random time (uniformly distributed) between min and max; set max to 0 to disable +time_delay_min = 0 +time_delay_max = 0 + +[servers] +names = kresd, bind, unbound +# symbolic names of DNS servers under test +# separate multiple values by , + +# each symbolic name in [servers] section refers to config section +# containing IP address and port of particular server +[kresd] +ip = 127.0.0.1 +port = 5353 +transport = tcp +graph_color = #00a2e2 +restart_script = ./ci/respdiff/restart-kresd.sh + +[bind] +ip = 127.0.0.1 +port = 53533 +transport = udp +graph_color = #e2a000 +restart_script = ./ci/respdiff/restart-bind.sh + +[unbound] +ip = 127.0.0.1 +port = 53535 +transport = udp +graph_color = #218669 +restart_script = ./ci/respdiff/restart-unbound.sh + +[diff] +# symbolic name of server under test +# other servers are used as reference when comparing answers from the target +target = kresd + +# fields and comparison methods used when comparing two DNS messages +criteria = opcode, rcode, flags, question, answertypes, answerrrsigs +# other supported criteria values: authority, additional, edns, nsid + +[report] +# diffsum reports mismatches in field values in this order +# if particular message has multiple mismatches, it is counted only once into category with highest weight +field_weights = timeout, malformed, opcode, question, rcode, flags, answertypes, answerrrsigs, answer, authority, additional, edns, nsid diff --git a/ci/respdiff/respdiff-tls.conf b/ci/respdiff/respdiff-tls.conf new file mode 100644 index 0000000..1a50eab --- /dev/null +++ b/ci/respdiff/respdiff-tls.conf @@ -0,0 +1,52 @@ +# SPDX-License-Identifier: GPL-3.0-or-later + +[sendrecv] +# in seconds +timeout = 11 +# number of queries to run simultaneously +jobs = 64 +# in seconds (float); delay each query by a random time (uniformly distributed) between min and max; set max to 0 to disable +time_delay_min = 0 +time_delay_max = 0 + +[servers] +names = kresd, bind, unbound +# symbolic names of DNS servers under test +# separate multiple values by , + +# each symbolic name in [servers] section refers to config section +# containing IP address and port of particular server +[kresd] +ip = 127.0.0.1 +port = 8853 +transport = tls +graph_color = #00a2e2 +restart_script = ./ci/respdiff/restart-kresd.sh + +[bind] +ip = 127.0.0.1 +port = 53533 +transport = udp +graph_color = #e2a000 +restart_script = ./ci/respdiff/restart-bind.sh + +[unbound] +ip = 127.0.0.1 +port = 53535 +transport = udp +graph_color = #218669 +restart_script = ./ci/respdiff/restart-unbound.sh + +[diff] +# symbolic name of server under test +# other servers are used as reference when comparing answers from the target +target = kresd + +# fields and comparison methods used when comparing two DNS messages +criteria = opcode, rcode, flags, question, answertypes, answerrrsigs +# other supported criteria values: authority, additional, edns, nsid + +[report] +# diffsum reports mismatches in field values in this order +# if particular message has multiple mismatches, it is counted only once into category with highest weight +field_weights = timeout, malformed, opcode, question, rcode, flags, answertypes, answerrrsigs, answer, authority, additional, edns, nsid diff --git a/ci/respdiff/respdiff-udp.conf b/ci/respdiff/respdiff-udp.conf new file mode 100644 index 0000000..35a69a9 --- /dev/null +++ b/ci/respdiff/respdiff-udp.conf @@ -0,0 +1,52 @@ +# SPDX-License-Identifier: GPL-3.0-or-later + +[sendrecv] +# in seconds +timeout = 11 +# number of queries to run simultaneously +jobs = 64 +# in seconds (float); delay each query by a random time (uniformly distributed) between min and max; set max to 0 to disable +time_delay_min = 0 +time_delay_max = 0 + +[servers] +names = kresd, bind, unbound +# symbolic names of DNS servers under test +# separate multiple values by , + +# each symbolic name in [servers] section refers to config section +# containing IP address and port of particular server +[kresd] +ip = 127.0.0.1 +port = 5353 +transport = udp +graph_color = #00a2e2 +restart_script = ./ci/respdiff/restart-kresd.sh + +[bind] +ip = 127.0.0.1 +port = 53533 +transport = udp +graph_color = #e2a000 +restart_script = ./ci/respdiff/restart-bind.sh + +[unbound] +ip = 127.0.0.1 +port = 53535 +transport = udp +graph_color = #218669 +restart_script = ./ci/respdiff/restart-unbound.sh + +[diff] +# symbolic name of server under test +# other servers are used as reference when comparing answers from the target +target = kresd + +# fields and comparison methods used when comparing two DNS messages +criteria = opcode, rcode, flags, question, answertypes, answerrrsigs +# other supported criteria values: authority, additional, edns, nsid + +[report] +# diffsum reports mismatches in field values in this order +# if particular message has multiple mismatches, it is counted only once into category with highest weight +field_weights = timeout, malformed, opcode, question, rcode, flags, answertypes, answerrrsigs, answer, authority, additional, edns, nsid diff --git a/ci/respdiff/restart-bind.sh b/ci/respdiff/restart-bind.sh new file mode 100755 index 0000000..35838c7 --- /dev/null +++ b/ci/respdiff/restart-bind.sh @@ -0,0 +1,3 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-3.0-or-later +service named restart diff --git a/ci/respdiff/restart-kresd.sh b/ci/respdiff/restart-kresd.sh new file mode 100755 index 0000000..4e9387c --- /dev/null +++ b/ci/respdiff/restart-kresd.sh @@ -0,0 +1,12 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-3.0-or-later + +exec > /dev/null +exec 2>&1 + +killall -w kresd +rm -f '*.mdb' +$PREFIX/sbin/kresd -n -q -c $(pwd)/ci/respdiff/kresd.config &>>kresd.log & + +# wait until socket is receiving connections +sleep 1 diff --git a/ci/respdiff/restart-unbound.sh b/ci/respdiff/restart-unbound.sh new file mode 100755 index 0000000..add24c9 --- /dev/null +++ b/ci/respdiff/restart-unbound.sh @@ -0,0 +1,4 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-3.0-or-later + +service unbound restart diff --git a/ci/respdiff/run-respdiff-tests.sh b/ci/respdiff/run-respdiff-tests.sh new file mode 100755 index 0000000..2bfc44d --- /dev/null +++ b/ci/respdiff/run-respdiff-tests.sh @@ -0,0 +1,27 @@ +#!/bin/bash +# SPDX-License-Identifier: GPL-3.0-or-later + +# $1 == udp/tcp/tls, it selects configuration file to use +# respdiff scripts must be present in /var/opt/respdiff +set -o errexit -o nounset -o xtrace + +NDIFFREPRO=3 + +wget -qO- https://gitlab.nic.cz/knot/respdiff/snippets/238/raw?inline=false | head -n 5000 > /tmp/queries.txt +mkdir results +rm -rf respdiff.db + +CONFIG="$(pwd)/ci/respdiff/respdiff-${1}.conf" +/var/opt/respdiff/qprep.py respdiff.db < /tmp/queries.txt +time /var/opt/respdiff/orchestrator.py respdiff.db -c "${CONFIG}" +time /var/opt/respdiff/msgdiff.py respdiff.db -c "${CONFIG}" +for i in $(seq $NDIFFREPRO); do + time /var/opt/respdiff/diffrepro.py -c "${CONFIG}" respdiff.db +done +/var/opt/respdiff/diffsum.py respdiff.db -c "${CONFIG}" > results/respdiff.txt +/var/opt/respdiff/histogram.py respdiff.db -c "${CONFIG}" -o results/histogram.svg +: minimize LMDB and log size so they can be effectively archived +mkdir results/respdiff.db +mdb_copy -c respdiff.db results/respdiff.db +xz -9 results/respdiff.db/data.mdb +xz kresd.log diff --git a/ci/respdiff/start-resolvers.sh b/ci/respdiff/start-resolvers.sh new file mode 100755 index 0000000..87e98f3 --- /dev/null +++ b/ci/respdiff/start-resolvers.sh @@ -0,0 +1,13 @@ +# SPDX-License-Identifier: GPL-3.0-or-later + +#run unbound +service unbound start && service unbound status; +# dig @localhost -p 53535 + +#run bind +service named start && service named status; +# dig @localhost -p 53533 + +#run kresd +$PREFIX/sbin/kresd -n -q -c $(pwd)/ci/respdiff/kresd.config &>kresd.log & +# dig @localhost -p 5353 -- cgit v1.2.3