From 483926a283e118590da3f9ecfa75a8a4d62143ce Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Wed, 1 Dec 2021 07:15:11 +0100 Subject: Merging upstream version 1.32.0. Signed-off-by: Daniel Baumann --- .github/CODEOWNERS | 67 ++-- .github/data/build-matrix.json | 114 ++++++ .github/dockerfiles/Dockerfile.build_test | 9 +- .github/labeler.yml | 6 + .github/scripts/build-static-x86_64.sh | 58 --- .github/scripts/build-static.sh | 61 +++ .github/scripts/check-updater.sh | 6 +- .github/scripts/docker-test.sh | 58 +++ .github/scripts/pkg-test.sh | 128 ++++++ .github/scripts/run-updater-check.sh | 9 + .github/scripts/run_install_with_dist_file.sh | 3 +- .github/workflows/build-and-install.yml | 215 ---------- .github/workflows/build.yml | 538 ++++++++++++++++++++++++++ .github/workflows/checks.yml | 41 +- .github/workflows/coverity.yml | 7 +- .github/workflows/dashboard-pr.yml | 35 +- .github/workflows/docker.yml | 88 ++++- .github/workflows/docs.yml | 2 + .github/workflows/labeler.yml | 4 +- .github/workflows/packaging.yml | 95 +++-- .github/workflows/repoconfig-packages.yml | 80 ++++ .github/workflows/review.yml | 120 ++++-- .github/workflows/tests.yml | 9 +- .github/workflows/updater.yml | 85 ---- 24 files changed, 1307 insertions(+), 531 deletions(-) create mode 100644 .github/data/build-matrix.json delete mode 100755 .github/scripts/build-static-x86_64.sh create mode 100755 .github/scripts/build-static.sh create mode 100755 .github/scripts/docker-test.sh create mode 100755 .github/scripts/pkg-test.sh create mode 100755 .github/scripts/run-updater-check.sh delete mode 100644 .github/workflows/build-and-install.yml create mode 100644 .github/workflows/build.yml create mode 100644 .github/workflows/repoconfig-packages.yml delete mode 100644 .github/workflows/updater.yml (limited to '.github') diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 185ececd2..50aa6797f 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -2,20 +2,20 @@ # This way we prevent modifications which will be overwriten by automation. # Global (default) code owner -* @ktsaou @Ferroin +* @Ferroin # Ownership by directory structure -.travis/ @Ferroin @kaskavel @vkalintiris -.github/ @Ferroin @kaskavel @vkalintiris +.travis/ @Ferroin @iigorkarpov @maneamarius @kaskavel @vkalintiris +.github/ @Ferroin @iigorkarpov @maneamarius @kaskavel @vkalintiris aclk/ @stelfrag @underhood backends/ @thiagoftsm @vlvkobal backends/graphite/ @thiagoftsm @vlvkobal backends/json/ @thiagoftsm @vlvkobal backends/opentsdb/ @thiagoftsm @vlvkobal backends/prometheus/ @vlvkobal @thiagoftsm -build/ @Ferroin -contrib/debian @Ferroin @vkalintiris -collectors/ @vlvkobal @mfundul +build/ @Ferroin @iigorkarpov @maneamarius +contrib/debian @Ferroin @iigorkarpov @maneamarius @vkalintiris +collectors/ @vlvkobal collectors/charts.d.plugin/ @ilyam8 @Ferroin collectors/freebsd.plugin/ @vlvkobal @thiagoftsm collectors/ebpf.plugin/ @thiagoftsm @vlvkobal @@ -27,40 +27,41 @@ collectors/node.d.plugin/stiebeleltron/ @ccremer collectors/python.d.plugin/ @ilyam8 collectors/cups.plugin/ @simonnagl @vlvkobal @thiagoftsm exporting/ @vlvkobal @thiagoftsm -daemon/ @thiagoftsm @mfundul -database/ @mfundul @thiagoftsm -docs/ @joelhans -health/ @thiagoftsm @vlvkobal +daemon/ @thiagoftsm +database/ @thiagoftsm @vkalintiris +docs/ @DShreve2 @kickoke +health/ @thiagoftsm @vlvkobal @vkalintiris health/health.d/ @thiagoftsm @vlvkobal health/notifications/ @Ferroin @thiagoftsm -libnetdata/ @thiagofsm @mfundul -packaging/ @Ferroin @vkalintiris +ml/ @siamaktavakoli @vkalintiris +libnetdata/ @thiagofsm +packaging/ @Ferroin @iigorkarpov @maneamarius @vkalintiris registry/ @jacekkolasa streaming/ @thiagoftsm @vlvkobal -system/ @Ferroin @vkalintiris -tests/ @Ferroin @kaskavel -web/ @thiagoftsm @mfundul @vlvkobal +system/ @Ferroin @iigorkarpov @maneamarius @vkalintiris +tests/ @Ferroin @iigorkarpov @maneamarius @kaskavel @vkalintiris +web/ @thiagoftsm @vlvkobal @vkalintiris web/gui/ @jacekkolasa # Ownership by filetype (overwrites ownership by directory) -*.am @Ferroin -*.md @joelhans -Dockerfile* @Ferroin @knatsakis +*.am @Ferroin @iigorkarpov @maneamarius @vkalintiris +*.md @DShreve2 @kickoke +Dockerfile* @Ferroin @iigorkarpov @maneamarius @vkalintiris # Ownership of specific files -.gitignore @Ferroin @knatsakis -.travis.yml @Ferroin @knatsakis @kaskavel @vkalintiris -.lgtm.yml @Ferroin @knatsakis -.eslintrc @Ferroin @knatsakis -.eslintignore @Ferroin @knatsakis -.csslintrc @Ferroin @knatsakis -.codeclimate.yml @Ferroin @knatsakis -.codacy.yml @Ferroin @knatsakis -.yamllint.yml @Ferroin @knatsakis -netdata.spec.in @Ferroin @knatsakis @vkalintiris -netdata-installer.sh @Ferroin @knatsakis @vkalintiris -package.json @jacekkolasa @Ferroin @knatsakis -packaging/version @netdatabot @Ferroin @knatsakis +.gitignore @Ferroin @iigorkarpov @maneamarius @vkalintiris +.travis.yml @Ferroin @iigorkarpov @maneamarius @kaskavel @vkalintiris +.lgtm.yml @Ferroin @iigorkarpov @maneamarius @vkalintiris +.eslintrc @Ferroin @iigorkarpov @maneamarius @vkalintiris +.eslintignore @Ferroin @iigorkarpov @maneamarius @vkalintiris +.csslintrc @Ferroin @iigorkarpov @maneamarius @vkalintiris +.codeclimate.yml @Ferroin @iigorkarpov @maneamarius @vkalintiris +.codacy.yml @Ferroin @iigorkarpov @maneamarius @vkalintiris +.yamllint.yml @Ferroin @iigorkarpov @maneamarius @vkalintiris +netdata.spec.in @Ferroin @iigorkarpov @maneamarius @vkalintiris +netdata-installer.sh @Ferroin @iigorkarpov @maneamarius @vkalintiris +package.json @jacekkolasa @Ferroin @iigorkarpov @maneamarius @vkalintiris +packaging/version @netdatabot @Ferroin @iigorkarpov @maneamarius @vkalintiris -LICENSE.md @joelhans @Ferroin -CHANGELOG.md @netdatabot @Ferroin +LICENSE.md @DShreve2 @Ferroin @vkalintiris +CHANGELOG.md @netdatabot @Ferroin @iigorkarpov @maneamarius @vkalintiris diff --git a/.github/data/build-matrix.json b/.github/data/build-matrix.json new file mode 100644 index 000000000..b066311f3 --- /dev/null +++ b/.github/data/build-matrix.json @@ -0,0 +1,114 @@ +{ + "include": [ + { + "distro": "alpine:edge", + "artifact_key": "alpine-edge", + "pre": "apk add -U bash", + "rmjsonc": "apk del json-c-dev" + }, + { + "distro": "alpine:3.14", + "artifact_key": "alpine-3.14", + "pre": "apk add -U bash", + "rmjsonc": "apk del json-c-dev" + }, + { + "distro": "alpine:3.13", + "artifact_key": "alpine-3.13", + "pre": "apk add -U bash", + "rmjsonc": "apk del json-c-dev" + }, + { + "distro": "alpine:3.12", + "artifact_key": "alpine-3.12", + "pre": "apk add -U bash", + "rmjsonc": "apk del json-c-dev" + }, + { + "distro": "archlinux:latest", + "artifact_key": "archlinux", + "pre": "pacman --noconfirm -Syu && pacman --noconfirm -Sy grep libffi" + }, + { + "distro": "centos:7", + "artifact_key": "centos7" + }, + { + "distro": "centos:8", + "artifact_key": "centos8", + "rmjsonc": "dnf remove -y json-c-devel" + }, + { + "distro": "debian:11", + "artifact_key": "debian11", + "pre": "apt-get update", + "rmjsonc": "apt-get remove -y libjson-c-dev" + }, + { + "distro": "debian:10", + "artifact_key": "debian10", + "pre": "apt-get update", + "rmjsonc": "apt-get remove -y libjson-c-dev" + }, + { + "distro": "debian:9", + "artifact_key": "debian9", + "pre": "apt-get update", + "rmjsonc": "apt-get remove -y libjson-c-dev" + }, + { + "distro": "fedora:35", + "artifact_key": "fedora35", + "rmjsonc": "dnf remove -y json-c-devel" + }, + { + "distro": "fedora:34", + "artifact_key": "fedora34", + "rmjsonc": "dnf remove -y json-c-devel" + }, + { + "distro": "fedora:33", + "artifact_key": "fedora33", + "rmjsonc": "dnf remove -y json-c-devel" + }, + { + "distro": "opensuse/leap:15.3", + "artifact_key": "opensuse15.3", + "rmjsonc": "zypper rm -y libjson-c-devel" + }, + { + "distro": "opensuse/leap:15.2", + "artifact_key": "opensuse15.2", + "rmjsonc": "zypper rm -y libjson-c-devel" + }, + { + "distro": "oraclelinux:8", + "artifact_key": "oraclelinux8", + "rmjsonc": "dnf remove -y json-c-devel" + }, + { + "distro": "ubuntu:21.10", + "artifact_key": "ubuntu21.10", + "pre": "rm -f /etc/apt/apt.conf.d/docker && apt-get update", + "rmjsonc": "apt-get remove -y libjson-c-dev" + }, + { + "distro": "ubuntu:21.04", + "artifact_key": "ubuntu21.04", + "pre": "apt-get update", + "rmjsonc": "apt-get remove -y libjson-c-dev" + }, + { + "distro": "ubuntu:20.04", + "artifact_key": "ubuntu20.04", + "pre": "apt-get update", + "rmjsonc": "apt-get remove -y libjson-c-dev" + }, + { + "distro": "ubuntu:18.04", + "artifact_key": "ubuntu18.04", + "pre": "apt-get update", + "rmjsonc": "apt-get remove -y libjson-c-dev" + } + ] +} diff --git a/.github/dockerfiles/Dockerfile.build_test b/.github/dockerfiles/Dockerfile.build_test index 1dc3e303d..aca89c09b 100644 --- a/.github/dockerfiles/Dockerfile.build_test +++ b/.github/dockerfiles/Dockerfile.build_test @@ -4,8 +4,15 @@ FROM ${BASE} ARG PRE ENV PRE=${PRE} +ARG RMJSONC +ENV RMJSONC=${RMJSONC} +ENV DO_NOT_TRACK=1 +ENV GITHUB_ACTIONS=true + +RUN echo "${PRE}" > /prep-cmd.sh && \ + echo "${RMJSONC}" > /rmjsonc.sh && chmod +x /rmjsonc.sh && \ + /bin/sh /prep-cmd.sh COPY . /netdata -RUN /bin/sh /netdata/prep-cmd.sh RUN /netdata/packaging/installer/install-required-packages.sh --dont-wait --non-interactive netdata-all diff --git a/.github/labeler.yml b/.github/labeler.yml index 544b420c6..a1082a2a8 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -13,6 +13,7 @@ ACLK: - aclk/* - aclk/**/* + - mqtt_websockets area/backends: - backends/* @@ -52,6 +53,7 @@ area/database: area/docs: - "**/*.md" + - "**/*.mdx" - diagrams/* - diagrams/**/* @@ -69,6 +71,10 @@ area/health: - health/* - health/**/* +area/ml: + - ml/* + - ml/**/* + area/packaging: - contrib/* - contrib/**/* diff --git a/.github/scripts/build-static-x86_64.sh b/.github/scripts/build-static-x86_64.sh deleted file mode 100755 index 2676b6321..000000000 --- a/.github/scripts/build-static-x86_64.sh +++ /dev/null @@ -1,58 +0,0 @@ -#!/bin/sh -# -# Builds the netdata-vX.Y.Z-xxxx.gz.run (static x86_64) artifact. - -set -e - -# shellcheck source=.github/scripts/functions.sh -. "$(dirname "$0")/functions.sh" - -NAME="${NAME:-netdata}" -VERSION="${VERSION:-"$(git describe)"}" -BASENAME="$NAME-$VERSION" - -prepare_build() { - progress "Preparing build" - ( - test -d artifacts || mkdir -p artifacts - ) >&2 -} - -build_static_x86_64() { - progress "Building static x86_64" - ( - USER="" ./packaging/makeself/build-x86_64-static.sh - ) >&2 -} - -prepare_assets() { - progress "Preparing assets" - ( - cp packaging/version artifacts/latest-version.txt - - cd artifacts || exit 1 - ln -s "${BASENAME}.gz.run" netdata-latest.gz.run - sha256sum -b ./* > "sha256sums.txt" - ) >&2 -} - -steps="prepare_build build_static_x86_64" -steps="$steps prepare_assets" - -_main() { - for step in $steps; do - if ! run "$step"; then - if [ -t 1 ]; then - debug - else - fail "Build failed" - fi - fi - done - - echo "🎉 All Done!" -} - -if [ -n "$0" ] && [ x"$0" != x"-bash" ]; then - _main "$@" -fi diff --git a/.github/scripts/build-static.sh b/.github/scripts/build-static.sh new file mode 100755 index 000000000..e81051438 --- /dev/null +++ b/.github/scripts/build-static.sh @@ -0,0 +1,61 @@ +#!/bin/sh +# +# Builds the netdata-vX.Y.Z-xxxx.gz.run (static x86_64) artifact. + +set -e + +# shellcheck source=.github/scripts/functions.sh +. "$(dirname "$0")/functions.sh" + +BUILDARCH="${1}" +NAME="${NAME:-netdata}" +VERSION="${VERSION:-"$(git describe)"}" +BASENAME="$NAME-$BUILDARCH-$VERSION" + +prepare_build() { + progress "Preparing build" + ( + test -d artifacts || mkdir -p artifacts + ) >&2 +} + +build_static() { + progress "Building static ${BUILDARCH}" + ( + USER="" ./packaging/makeself/build-static.sh "${BUILDARCH}" + ) >&2 +} + +prepare_assets() { + progress "Preparing assets" + ( + cp packaging/version artifacts/latest-version.txt + + cd artifacts || exit 1 + ln -s "${BASENAME}.gz.run" "netdata-${BUILDARCH}-latest.gz.run" + if [ "${BUILDARCH}" = "x86_64" ]; then + ln -s "${BASENAME}.gz.run" netdata-latest.gz.run + fi + ) >&2 +} + +steps="prepare_build build_static" +steps="$steps prepare_assets" + +_main() { + for step in $steps; do + if ! run "$step"; then + if [ -t 1 ]; then + debug + else + fail "Build failed" + fi + fi + done + + echo "🎉 All Done!" +} + +if [ -n "$0" ] && [ x"$0" != x"-bash" ]; then + _main "$@" +fi diff --git a/.github/scripts/check-updater.sh b/.github/scripts/check-updater.sh index 3ef4857f9..1051f1eee 100755 --- a/.github/scripts/check-updater.sh +++ b/.github/scripts/check-updater.sh @@ -4,20 +4,20 @@ set -e # shellcheck source=.github/scripts/functions.sh . "$(dirname "$0")/functions.sh" -check_successfull_update() { +check_successful_update() { progress "Check netdata version after update" ( netdata_version=$(netdata -v | awk '{print $2}') updater_version=$(cat packaging/version) if [ "$netdata_version" = "$updater_version" ]; then - echo "Update successfull!" + echo "Update successful!" else exit 1 fi ) >&2 } -steps="check_successfull_update" +steps="check_successful_update" _main() { for step in $steps; do diff --git a/.github/scripts/docker-test.sh b/.github/scripts/docker-test.sh new file mode 100755 index 000000000..795711b1a --- /dev/null +++ b/.github/scripts/docker-test.sh @@ -0,0 +1,58 @@ +#!/bin/sh + +export DEBIAN_FRONTEND=noninteractive + +wait_for() { + host="${1}" + port="${2}" + name="${3}" + timeout="30" + + if command -v nc > /dev/null ; then + netcat="nc" + elif command -v netcat > /dev/null ; then + netcat="netcat" + else + printf "Unable to find a usable netcat command.\n" + return 1 + fi + + printf "Waiting for %s on %s:%s ... " "${name}" "${host}" "${port}" + + sleep 30 + + i=0 + while ! ${netcat} -z "${host}" "${port}"; do + sleep 1 + if [ "$i" -gt "$timeout" ]; then + printf "Timed out!\n" + return 1 + fi + i="$((i + 1))" + done + printf "OK\n" +} + +apt-get update && apt-get upgrade -y && apt get install -y netcat + +docker run -d --name=netdata \ + -p 19999:19999 \ + -v netdataconfig:/etc/netdata \ + -v netdatalib:/var/lib/netdata \ + -v netdatacache:/var/cache/netdata \ + -v /etc/passwd:/host/etc/passwd:ro \ + -v /etc/group:/host/etc/group:ro \ + -v /proc:/host/proc:ro \ + -v /sys:/host/sys:ro \ + -v /etc/os-release:/host/etc/os-release:ro \ + --cap-add SYS_PTRACE \ + --security-opt apparmor=unconfined \ + netdata/netdata:test + +wait_for localhost 19999 netdata || exit 1 + +curl -sS http://127.0.0.1:19999/api/v1/info > ./response || exit 1 + +cat ./response + +jq '.version' ./response || exit 1 diff --git a/.github/scripts/pkg-test.sh b/.github/scripts/pkg-test.sh new file mode 100755 index 000000000..196fefa99 --- /dev/null +++ b/.github/scripts/pkg-test.sh @@ -0,0 +1,128 @@ +#!/bin/sh + +install_debian_like() { + # This is needed to ensure package installs don't prompt for any user input. + export DEBIAN_FRONTEND=noninteractive + + apt-get update + + # Install Netdata + apt-get install -y /netdata/artifacts/netdata_"${VERSION}"_*.deb || exit 1 + + # Install testing tools + apt-get install -y --no-install-recommends curl netcat jq || exit 1 +} + +install_fedora_like() { + # Using a glob pattern here because I can't reliably determine what the + # resulting package name will be (TODO: There must be a better way!) + + PKGMGR="$( (command -v dnf > /dev/null && echo "dnf") || echo "yum")" + + pkg_version="$(echo "${VERSION}" | tr - .)" + + # Install Netdata + "$PKGMGR" install -y /netdata/artifacts/netdata-"${pkg_version}"-*.rpm + + # Install testing tools + "$PKGMGR" install -y curl nc jq || exit 1 +} + +install_centos() { + # Using a glob pattern here because I can't reliably determine what the + # resulting package name will be (TODO: There must be a better way!) + + PKGMGR="$( (command -v dnf > /dev/null && echo "dnf") || echo "yum")" + + pkg_version="$(echo "${VERSION}" | tr - .)" + + # Install EPEL (needed for `jq` + "$PKGMGR" install -y epel-release || exit 1 + + # Install Netdata + "$PKGMGR" install -y /netdata/artifacts/netdata-"${pkg_version}"-*.rpm + + # Install testing tools + "$PKGMGR" install -y curl nc jq || exit 1 +} + +install_suse_like() { + # Using a glob pattern here because I can't reliably determine what the + # resulting package name will be (TODO: There must be a better way!) + + pkg_version="$(echo "${VERSION}" | tr - .)" + + # Install Netdata + zypper install -y --allow-unsigned-rpm /netdata/artifacts/netdata-"${pkg_version}"-*.rpm + + # Install testing tools + zypper install -y --no-recommends curl netcat-openbsd jq || exit 1 +} + +dump_log() { + cat ./netdata.log +} + +wait_for() { + host="${1}" + port="${2}" + name="${3}" + timeout="30" + + if command -v nc > /dev/null ; then + netcat="nc" + elif command -v netcat > /dev/null ; then + netcat="netcat" + else + printf "Unable to find a usable netcat command.\n" + return 1 + fi + + printf "Waiting for %s on %s:%s ... " "${name}" "${host}" "${port}" + + sleep 30 + + i=0 + while ! ${netcat} -z "${host}" "${port}"; do + sleep 1 + if [ "$i" -gt "$timeout" ]; then + printf "Timed out!\n" + return 1 + fi + i="$((i + 1))" + done + printf "OK\n" +} + +case "${DISTRO}" in + debian | ubuntu) + install_debian_like + ;; + fedora | oraclelinux) + install_fedora_like + ;; + centos) + install_centos + ;; + opensuse) + install_suse_like + ;; + *) + printf "ERROR: unsupported distro: %s_%s\n" "${DISTRO}" "${DISTRO_VERSION}" + exit 1 + ;; +esac + +trap dump_log EXIT + +/usr/sbin/netdata -D > ./netdata.log 2>&1 & + +wait_for localhost 19999 netdata || exit 1 + +curl -sS http://127.0.0.1:19999/api/v1/info > ./response || exit 1 + +cat ./response + +jq '.version' ./response || exit 1 + +trap - EXIT diff --git a/.github/scripts/run-updater-check.sh b/.github/scripts/run-updater-check.sh new file mode 100755 index 000000000..e6969a2d1 --- /dev/null +++ b/.github/scripts/run-updater-check.sh @@ -0,0 +1,9 @@ +#!/bin/sh + +echo ">>> Installing Netdata..." +/netdata/packaging/installer/kickstart.sh --dont-wait --disable-telemetry || exit 1 +echo ">>> Updating Netdata..." +export NETDATA_NIGHTLIES_BASEURL="http://localhost:8080/artifacts/" # Pull the tarball from the local web server. +/netdata/packaging/installer/netdata-updater.sh --not-running-from-cron --no-updater-self-update || exit 1 +echo ">>> Checking if update was successful..." +/netdata/.github/scripts/check-updater.sh || exit 1 diff --git a/.github/scripts/run_install_with_dist_file.sh b/.github/scripts/run_install_with_dist_file.sh index 9453dff35..18cf80927 100755 --- a/.github/scripts/run_install_with_dist_file.sh +++ b/.github/scripts/run_install_with_dist_file.sh @@ -18,7 +18,7 @@ shift printf >&2 "Opening dist archive %s ... " "${distfile}" tar -xovf "${distfile}" -distdir="$(echo "${distfile}" | cut -d. -f1,2,3)" +distdir="$(echo "${distfile}" | rev | cut -d. -f3- | rev)" cp -a packaging/installer/install-required-packages.sh "${distdir}/install-required-packages.sh" if [ ! -d "${distdir}" ]; then printf >&2 "ERROR: %s is not a directory" "${distdir}" @@ -29,6 +29,7 @@ printf >&2 "Entering %s and starting docker run ..." "${distdir}" pushd "${distdir}" || exit 1 docker run \ + -e DO_NOT_TRACK=1 \ -v "${PWD}:/netdata" \ -w /netdata \ "ubuntu:latest" \ diff --git a/.github/workflows/build-and-install.yml b/.github/workflows/build-and-install.yml deleted file mode 100644 index 251a65162..000000000 --- a/.github/workflows/build-and-install.yml +++ /dev/null @@ -1,215 +0,0 @@ ---- -name: Builder -on: - push: - branches: - - master - pull_request: -jobs: - static-build: - name: Build (x86_64) - runs-on: ubuntu-latest - steps: - - name: Git clone repository - uses: actions/checkout@v2 - with: - submodules: recursive - - run: | - git fetch --prune --unshallow --tags - - name: Build - run: | - .github/scripts/build-static-x86_64.sh - source-build: - name: Build & Install - strategy: - fail-fast: false - max-parallel: 8 - matrix: - distro: - - 'alpine:edge' - - 'alpine:3.13' - - 'alpine:3.12' - - 'alpine:3.11' - - 'alpine:3.10' - - 'archlinux:latest' - - 'centos:8' - - 'centos:7' - - 'clearlinux:latest' - - 'debian:10' - - 'debian:9' - - 'fedora:34' - - 'fedora:33' - - 'fedora:32' - - 'opensuse/leap:15.2' - - 'opensuse/tumbleweed:latest' - - 'ubuntu:21.04' - - 'ubuntu:20.10' - - 'ubuntu:20.04' - - 'ubuntu:18.04' - - 'ubuntu:16.04' - include: - - distro: 'alpine:edge' - pre: 'apk add -U bash' - rmjsonc: 'apk del json-c-dev' - - distro: 'alpine:3.13' - pre: 'apk add -U bash' - rmjsonc: 'apk del json-c-dev' - - distro: 'alpine:3.12' - pre: 'apk add -U bash' - rmjsonc: 'apk del json-c-dev' - - distro: 'alpine:3.11' - pre: 'apk add -U bash' - rmjsonc: 'apk del json-c-dev' - - distro: 'alpine:3.10' - pre: 'apk add -U bash' - rmjsonc: 'apk del json-c-dev' - - - distro: 'archlinux:latest' - pre: 'pacman --noconfirm -Syu && pacman --noconfirm -Sy grep libffi' - - - distro: 'centos:8' - rmjsonc: 'dnf remove -y json-c-devel' - - - distro: 'debian:10' - pre: 'apt-get update' - rmjsonc: 'apt-get remove -y libjson-c-dev' - - distro: 'debian:9' - pre: 'apt-get update' - rmjsonc: 'apt-get remove -y libjson-c-dev' - - - distro: 'fedora:34' - rmjsonc: 'dnf remove -y json-c-devel' - - distro: 'fedora:33' - rmjsonc: 'dnf remove -y json-c-devel' - - distro: 'fedora:32' - rmjsonc: 'dnf remove -y json-c-devel' - - - distro: 'opensuse/leap:15.2' - rmjsonc: 'zypper rm -y libjson-c-devel' - - distro: 'opensuse/tumbleweed:latest' - rmjsonc: 'zypper rm -y libjson-c-devel' - - - distro: 'ubuntu:21.04' - pre: 'apt-get update' - rmjsonc: 'apt-get remove -y libjson-c-dev' - - distro: 'ubuntu:20.10' - pre: 'apt-get update' - rmjsonc: 'apt-get remove -y libjson-c-dev' - - distro: 'ubuntu:20.04' - pre: 'apt-get update' - rmjsonc: 'apt-get remove -y libjson-c-dev' - - distro: 'ubuntu:18.04' - pre: 'apt-get update' - rmjsonc: 'apt-get remove -y libjson-c-dev' - - distro: 'ubuntu:16.04' - pre: 'apt-get update' - rmjsonc: 'apt-get remove -y libjson-c-dev' - runs-on: ubuntu-latest - steps: - - name: Git clone repository - uses: actions/checkout@v2 - with: - submodules: recursive - - name: install-required-packages.sh on ${{ matrix.distro }} - env: - PRE: ${{ matrix.pre }} - RMJSONC: ${{ matrix.rmjsonc }} - run: | - echo $PRE > ./prep-cmd.sh - echo $RMJSONC > ./rmjsonc.sh && chmod +x ./rmjsonc.sh - docker build . -f .github/dockerfiles/Dockerfile.build_test -t test --build-arg BASE=${{ matrix.distro }} - - name: Regular build on ${{ matrix.distro }} - run: | - docker run -w /netdata test /bin/sh -c 'autoreconf -ivf && ./configure && make -j2' - - name: netdata-installer on ${{ matrix.distro }}, disable cloud - run: | - docker run -w /netdata test /bin/sh -c './netdata-installer.sh --dont-wait --dont-start-it --disable-cloud' - - name: netdata-installer on ${{ matrix.distro }}, require cloud - run: | - docker run -w /netdata test /bin/sh -c './netdata-installer.sh --dont-wait --dont-start-it --require-cloud' - - name: netdata-installer on ${{ matrix.distro }}, require cloud, no JSON-C - if: matrix.rmjsonc != '' - run: | - docker run -w /netdata test \ - /bin/sh -c '/netdata/rmjsonc.sh && ./netdata-installer.sh --dont-wait --dont-start-it --require-cloud' - aws-kinesis-build: - name: With AWS Kinesis SDK - strategy: - fail-fast: false - matrix: - distro: - - 'centos:8' - - 'debian:buster' - - 'fedora:32' - - 'ubuntu:20.04' - include: - - distro: 'centos:8' - pre: >- - yum -y update && - yum -y groupinstall 'Development Tools' && - yum -y install libcurl-devel openssl-devel libuuid-devel - build_kinesis: >- - git clone --branch 1.8.186 --depth 1 https://github.com/aws/aws-sdk-cpp.git && - cmake -DCMAKE_INSTALL_PREFIX=/usr - -DBUILD_ONLY=kinesis - ./aws-sdk-cpp && - make && - make install - - distro: 'debian:buster' - pre: >- - apt-get update && - DEBIAN_FRONTEND=noninteractive apt-get install -y build-essential && - DEBIAN_FRONTEND=noninteractive apt-get install -y libcurl4-openssl-dev libssl-dev uuid-dev zlib1g-dev libpulse-dev - build_kinesis: >- - git clone --branch 1.8.186 --depth 1 https://github.com/aws/aws-sdk-cpp.git && - cmake -DCMAKE_INSTALL_PREFIX=/usr - -DBUILD_ONLY=kinesis - ./aws-sdk-cpp && - make && - make install - - distro: 'fedora:32' - pre: >- - dnf -y update && - dnf -y groupinstall 'Development Tools' && - dnf -y install libcurl-devel openssl-devel libuuid-devel - build_kinesis: >- - git clone --branch 1.8.186 --depth 1 https://github.com/aws/aws-sdk-cpp.git && - cmake -DCMAKE_INSTALL_PREFIX=/usr - -DBUILD_ONLY=kinesis - ./aws-sdk-cpp && - make && - make install - - distro: 'ubuntu:20.04' - pre: >- - apt-get update && - DEBIAN_FRONTEND=noninteractive apt-get install -y build-essential && - DEBIAN_FRONTEND=noninteractive apt-get install -y libcurl4-openssl-dev libssl-dev uuid-dev zlib1g-dev libpulse-dev - build_kinesis: >- - git clone --branch 1.8.186 --depth 1 https://github.com/aws/aws-sdk-cpp.git && - cmake -DCMAKE_INSTALL_PREFIX=/usr - -DBUILD_ONLY=kinesis - ./aws-sdk-cpp && - make && - make install - runs-on: ubuntu-latest - steps: - - name: Git clone repository - uses: actions/checkout@v2 - with: - submodules: recursive - - name: install-required-packages.sh on ${{ matrix.distro }} - env: - PRE: ${{ matrix.pre }} - BUILD_KINESIS: ${{ matrix.build_kinesis }} - run: | - echo $PRE > ./prep-cmd.sh - echo $BUILD_KINESIS > ./build-kinesis.sh && chmod +x ./build-kinesis.sh - docker build . -f .github/dockerfiles/Dockerfile.build_test -t test --build-arg BASE=${{ matrix.distro }} - - name: Build on ${{ matrix.distro }} - env: - RUNCMD: >- - ./build-kinesis.sh && - ./netdata-installer.sh --dont-wait --dont-start-it --enable-backend-kinesis - run: | - docker run -w /netdata test \ - /bin/sh -c "$RUNCMD" diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 000000000..2732a1851 --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,538 @@ +--- +# Ci code for building release artifacts. +name: Build +on: + push: # Master branch checks only validate the build and generate artifacts for testing. + branches: + - master + pull_request: null # PR checks only validate the build and generate artifacts for testing. + workflow_dispatch: # Dispatch runs build and validate, then push to the appropriate storage location. + inputs: + type: + description: Build Type + default: nightly + required: true + version: + description: Version Tag + default: nightly + required: true +concurrency: # This keeps multiple instances of the job from running concurrently for the same ref and event type. + group: release-${{ github.ref }}-${{ github.event_name }} + cancel-in-progress: true +jobs: + build-dist: # Build the distribution tarball and store it as an artifact. + name: Build Distribution Tarball + runs-on: ubuntu-latest + outputs: + distfile: ${{ steps.build.outputs.distfile }} + steps: + - name: Checkout + uses: actions/checkout@v2 + with: + fetch-depth: 0 + submodules: recursive + - name: Mark Stable + if: github.event_name == 'workflow_dispatch' && github.event.inputs.type != 'nightly' + run: | + sed -i 's/^RELEASE_CHANNEL="nightly" *#/RELEASE_CHANNEL="stable" #/' netdata-installer.sh + - name: Build + id: build + run: | + mkdir -p artifacts + ./packaging/installer/install-required-packages.sh --dont-wait --non-interactive netdata + autoreconf -ivf + ./configure --prefix=/usr \ + --sysconfdir=/etc \ + --localstatedir=/var \ + --libexecdir=/usr/libexec \ + --with-zlib \ + --with-math \ + --with-user=netdata + make dist + echo "::set-output name=distfile::$(find . -name 'netdata-*.tar.gz')" + cp netdata-*.tar.gz artifacts/ + - name: Store + uses: actions/upload-artifact@v2 + with: + name: dist-tarball + path: artifacts/*.tar.gz + retention-days: 30 + - name: Failure Notification + uses: rtCamp/action-slack-notify@v2 + env: + SLACK_COLOR: 'danger' + SLACK_FOOTER: '' + SLACK_ICON_EMOJI: ':github-actions:' + SLACK_TITLE: 'Distribution tarball build failed:' + SLACK_USERNAME: 'GitHub Actions' + SLACK_MESSAGE: "Distribution tarball build failed." + SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }} + if: >- + ${{ + failure() + && startsWith(github.ref, 'refs/heads/master') + && github.event_name != 'pull_request' + }} + + build-static: # Build the static binary archives, and store them as artifacts. + name: Build Static + runs-on: ubuntu-latest + strategy: + matrix: + arch: + - 'x86_64' + - 'armv7l' + - 'aarch64' + - 'ppc64le' + steps: + - name: Checkout + uses: actions/checkout@v2 + with: + fetch-depth: 0 + submodules: recursive + - name: Mark Stable + if: github.event_name == 'workflow_dispatch' && github.event.inputs.type != 'nightly' + run: | + sed -i 's/^RELEASE_CHANNEL="nightly" *#/RELEASE_CHANNEL="stable" #/' netdata-installer.sh packaging/makeself/install-or-update.sh + - name: Build + run: .github/scripts/build-static.sh ${{ matrix.arch }} + - name: Store + uses: actions/upload-artifact@v2 + with: + name: static-archive + path: artifacts/*.gz.run + retention-days: 30 + - name: Failure Notification + uses: rtCamp/action-slack-notify@v2 + env: + SLACK_COLOR: 'danger' + SLACK_FOOTER: '' + SLACK_ICON_EMOJI: ':github-actions:' + SLACK_TITLE: 'Static build for ${{ matrix.arch }} failed:' + SLACK_USERNAME: 'GitHub Actions' + SLACK_MESSAGE: "Static build for ${{ matrix.arch }} failed." + SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }} + if: >- + ${{ + failure() + && startsWith(github.ref, 'refs/heads/master') + && github.event_name != 'pull_request' + }} + + matrix: # Generate the shared build matrix for our build tests. + name: Prepare Build Matrix + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.set-matrix.outputs.matrix }} + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Prepare tools + run: | + sudo apt-get update && sudo apt-get install -y jq + - name: Read build matrix + id: set-matrix + run: | + TASKS="$(jq -c . .github/data/build-matrix.json)" + echo "Generated Matrix: $TASKS" + echo "::set-output name=matrix::$TASKS" + + prepare-test-images: # Prepare the test environments for our build checks. This also checks dependency handling code for each tested environment. + name: Prepare Test Environments + runs-on: ubuntu-latest + needs: + - matrix + strategy: + # Unlike the actal build tests, this completes _very_ fast (average of about 3 minutes for each job), so we + # just run everything in parallel instead lof limiting job concurrency. + fail-fast: false + matrix: ${{ fromJson(needs.matrix.outputs.matrix) }} + steps: + - name: Git clone repository + uses: actions/checkout@v2 + - name: Setup Buildx + uses: docker/setup-buildx-action@v1 + - name: Build test environment + uses: docker/build-push-action@v2 + with: + push: false + load: false + file: .github/dockerfiles/Dockerfile.build_test + build-args: | + BASE=${{ matrix.distro }} + PRE=${{ matrix.pre }} + RMJSONC=${{ matrix.rmjsonc }} + outputs: type=oci,dest=/tmp/image.tar + tags: test:${{ matrix.artifact_key }} + - name: Upload image artifact + uses: actions/upload-artifact@v2 + with: + name: ${{ matrix.artifact_key }}-test-env + path: /tmp/image.tar + retention-days: 30 + - name: Failure Notification + uses: rtCamp/action-slack-notify@v2 + env: + SLACK_COLOR: 'danger' + SLACK_FOOTER: '' + SLACK_ICON_EMOJI: ':github-actions:' + SLACK_TITLE: 'Test environment preparation for ${{ matrix.distro }} failed:' + SLACK_USERNAME: 'GitHub Actions' + SLACK_MESSAGE: "Test environment preparation for ${{ matrix.distro }} failed." + SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }} + if: >- + ${{ + failure() + && startsWith(github.ref, 'refs/heads/master') + && github.event_name != 'pull_request' + }} + + source-build: # Test various source build arrangements. + name: Test Source Build + runs-on: ubuntu-latest + needs: + - matrix + - prepare-test-images + strategy: + fail-fast: false + max-parallel: 8 + matrix: ${{ fromJson(needs.matrix.outputs.matrix) }} + steps: + - name: Git clone repository + uses: actions/checkout@v2 + with: + submodules: recursive + - name: Fetch test environment + uses: actions/download-artifact@v2 + with: + name: ${{ matrix.artifact_key }}-test-env + - name: Load test environment + id: load + run: | + docker load --input image.tar | tee image-info.txt + echo "::set-output name=image::$(cut -d ':' -f 3 image-info.txt)" + - name: Regular build on ${{ matrix.distro }} + run: | + docker run --security-opt seccomp=unconfined -w /netdata sha256:${{ steps.load.outputs.image }} \ + /bin/sh -c 'autoreconf -ivf && ./configure && make -j2' + - name: netdata-installer on ${{ matrix.distro }}, disable cloud + run: | + docker run --security-opt seccomp=unconfined -w /netdata sha256:${{ steps.load.outputs.image }} \ + /bin/sh -c './netdata-installer.sh --dont-wait --dont-start-it --disable-cloud' + - name: netdata-installer on ${{ matrix.distro }}, require cloud + run: | + docker run --security-opt seccomp=unconfined -w /netdata sha256:${{ steps.load.outputs.image }} \ + /bin/sh -c './netdata-installer.sh --dont-wait --dont-start-it --require-cloud' + - name: netdata-installer on ${{ matrix.distro }}, require cloud, require ACLK-NG + run: | + docker run --security-opt seccomp=unconfined -w /netdata -e NETDATA_CONFIGURE_OPTIONS='--with-aclk-ng' \ + sha256:${{ steps.load.outputs.image }} /bin/sh -c './netdata-installer.sh --dont-wait --dont-start-it --require-cloud' + - name: netdata-installer on ${{ matrix.distro }}, require cloud, no JSON-C + if: matrix.rmjsonc != '' + run: | + docker run --security-opt seccomp=unconfined -w /netdata sha256:${{ steps.load.outputs.image }} \ + /bin/sh -c '/rmjsonc.sh && ./netdata-installer.sh --dont-wait --dont-start-it --require-cloud' + - name: Failure Notification + uses: rtCamp/action-slack-notify@v2 + env: + SLACK_COLOR: 'danger' + SLACK_FOOTER: '' + SLACK_ICON_EMOJI: ':github-actions:' + SLACK_TITLE: 'Build tests for ${{ matrix.distro }} failed:' + SLACK_USERNAME: 'GitHub Actions' + SLACK_MESSAGE: "Build tests for ${{ matrix.distro }} failed." + SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }} + if: >- + ${{ + failure() + && startsWith(github.ref, 'refs/heads/master') + && github.event_name != 'pull_request' + }} + + updater-check: # Test the generated dist archive using the updater code. + name: Test Generated Distfile and Updater Code + runs-on: ubuntu-latest + needs: + - build-dist + - matrix + - prepare-test-images + strategy: + fail-fast: false + max-parallel: 8 + matrix: ${{ fromJson(needs.matrix.outputs.matrix) }} + services: + apache: # This gets used to serve the dist tarball for the updater script. + image: httpd:2.4 + ports: + - 8080:80 + volumes: + - ${{ github.workspace }}:/usr/local/apache2/htdocs/ + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Fetch dist tarball artifacts + uses: actions/download-artifact@v2 + with: + name: dist-tarball + path: dist-tarball + - name: Prepare artifact directory + run: | + mkdir -p artifacts || exit 1 + echo "9999.0.0-0" > artifacts/latest-version.txt || exit 1 + cp dist-tarball/* artifacts || exit 1 + cd artifacts || exit 1 + ln -s ${{ needs.build-dist.outputs.distfile }} netdata-latest.tar.gz || exit 1 + sha256sum -b ./* > "sha256sums.txt" || exit 1 + cat sha256sums.txt + - name: Fetch test environment + uses: actions/download-artifact@v2 + with: + name: ${{ matrix.artifact_key }}-test-env + - name: Load test environment + id: load + run: | + docker load --input image.tar | tee image-info.txt + echo "::set-output name=image::$(cut -d ':' -f 3 image-info.txt)" + - name: Install netdata and run the updater on ${{ matrix.distro }} + run: | + docker run --security-opt seccomp=unconfined -e DO_NOT_TRACK=1 --network host -w /netdata sha256:${{ steps.load.outputs.image }} \ + /netdata/.github/scripts/run-updater-check.sh + - name: Failure Notification + uses: rtCamp/action-slack-notify@v2 + env: + SLACK_COLOR: 'danger' + SLACK_FOOTER: '' + SLACK_ICON_EMOJI: ':github-actions:' + SLACK_TITLE: 'Updater checks for ${{ matrix.distro }} failed:' + SLACK_USERNAME: 'GitHub Actions' + SLACK_MESSAGE: "Updater checks for ${{ matrix.distro }} failed." + SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }} + if: >- + ${{ + failure() + && startsWith(github.ref, 'refs/heads/master') + && github.event_name != 'pull_request' + }} + + prepare-upload: # Consolidate the artifacts for uploading or releasing. + name: Prepare Artifacts + runs-on: ubuntu-latest + needs: + - build-dist + - build-static + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Prepare Environment + run: mkdir -p artifacts + - name: Retrieve Dist Tarball + uses: actions/download-artifact@v2 + with: + name: dist-tarball + path: dist-tarball + - name: Retrieve Static Build Artifacts + uses: actions/download-artifact@v2 + with: + name: static-archive + path: static-archive + - name: Prepare Artifacts + working-directory: ./artifacts/ + run: | + mv ../dist-tarball/* . || exit 1 + mv ../static-archive/* . || exit 1 + ln -s ${{ needs.build-dist.outputs.distfile }} netdata-latest.tar.gz || exit 1 + cp ../packaging/version ./latest-version.txt || exit 1 + sha256sum -b ./* > sha256sums.txt || exit 1 + cat sha256sums.txt + - name: Store Artifacts + uses: actions/upload-artifact@v2 + with: + name: final-artifacts + path: artifacts/* + retention-days: 30 + - name: Failure Notification + uses: rtCamp/action-slack-notify@v2 + env: + SLACK_COLOR: 'danger' + SLACK_FOOTER: '' + SLACK_ICON_EMOJI: ':github-actions:' + SLACK_TITLE: 'Failed to prepare release artifacts for upload:' + SLACK_USERNAME: 'GitHub Actions' + SLACK_MESSAGE: "Failed to prepare release artifacts for upload." + SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }} + if: >- + ${{ + failure() + && startsWith(github.ref, 'refs/heads/master') + && github.event_name != 'pull_request' + }} + + artifact-verification-dist: # Verify the regular installer works with the consolidated artifacts. + name: Test Consolidated Artifacts (Source) + runs-on: ubuntu-latest + needs: + - prepare-upload + services: + apache: # This gets used to serve the dist tarball for the updater script. + image: httpd:2.4 + ports: + - 8080:80 + volumes: + - ${{ github.workspace }}:/usr/local/apache2/htdocs/ + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Fetch artifacts + uses: actions/download-artifact@v2 + with: + name: final-artifacts + path: artifacts + - name: Verify that artifacts work with installer + env: + NETDATA_TARBALL_BASEURL: http://localhost:8080/artifacts + run: packaging/installer/kickstart.sh --dont-start-it --disable-telemetry --dont-wait + - name: Failure Notification + uses: rtCamp/action-slack-notify@v2 + env: + SLACK_COLOR: 'danger' + SLACK_FOOTER: '' + SLACK_ICON_EMOJI: ':github-actions:' + SLACK_TITLE: 'Artifact verification for source tarball failed.' + SLACK_USERNAME: 'GitHub Actions' + SLACK_MESSAGE: "Artifact verification for source tarball failed." + SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }} + if: >- + ${{ + failure() + && startsWith(github.ref, 'refs/heads/master') + && github.event_name != 'pull_request' + }} + + artifact-verification-static: # Verify the static installer works with the consolidated artifacts. + name: Test Consolidated Artifacts (Static) + runs-on: ubuntu-latest + needs: + - prepare-upload + services: + apache: # This gets used to serve the static archives. + image: httpd:2.4 + ports: + - 8080:80 + volumes: + - ${{ github.workspace }}:/usr/local/apache2/htdocs/ + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Fetch artifacts + uses: actions/download-artifact@v2 + with: + name: final-artifacts + path: artifacts + - name: Verify that artifacts work with installer + env: + NETDATA_TARBALL_BASEURL: http://localhost:8080/artifacts + run: packaging/installer/kickstart-static64.sh --dont-start-it --disable-telemetry --dont-wait + - name: Failure Notification + uses: rtCamp/action-slack-notify@v2 + env: + SLACK_COLOR: 'danger' + SLACK_FOOTER: '' + SLACK_ICON_EMOJI: ':github-actions:' + SLACK_TITLE: 'Artifact verification for static build failed.' + SLACK_USERNAME: 'GitHub Actions' + SLACK_MESSAGE: "Artifact verification for static build failed." + SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }} + if: >- + ${{ + failure() + && startsWith(github.ref, 'refs/heads/master') + && github.event_name != 'pull_request' + }} + + upload-nightly: # Upload the nightly build artifacts to GCS. + name: Upload Nightly Artifacts + runs-on: ubuntu-latest + if: github.event_name == 'workflow_dispatch' && github.event.inputs.type == 'nightly' + needs: + - updater-check + - source-build + - artifact-verification-dist + - artifact-verification-static + steps: + - name: Retrieve Artifacts + uses: actions/download-artifact@v2 + with: + name: final-artifacts + path: final-artifacts + - name: Setup Gcloud + uses: google-github-actions/setup-gcloud@v0.2.1 + with: + project_id: ${{ secrets.GCP_NIGHTLY_STORAGE_PROJECT }} + service_account_key: ${{ secrets.GCP_STORAGE_SERVICE_ACCOUNT_KEY }} + export_default_credentials: true + - name: Upload Artifacts + uses: google-github-actions/upload-cloud-storage@v0.4.0 + with: + destination: ${{ secrets.GCP_NIGHTLY_STORAGE_BUCKET }} + gzip: false + path: ./final-artifacts + parent: false + - name: Failure Notification + uses: rtCamp/action-slack-notify@v2 + env: + SLACK_COLOR: 'danger' + SLACK_FOOTER: '' + SLACK_ICON_EMOJI: ':github-actions:' + SLACK_TITLE: 'Failed to upload nightly release artifacts:' + SLACK_USERNAME: 'GitHub Actions' + SLACK_MESSAGE: "Failed to upload nightly release artifacts." + SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }} + if: >- + ${{ + failure() + && startsWith(github.ref, 'refs/heads/master') + && github.event_name != 'pull_request' + }} + + upload-release: # Create the draft release and upload the build artifacts. + name: Create Release Draft + runs-on: ubuntu-latest + if: github.event_name == 'workflow_dispatch' && github.event.inputs.type == 'release' + needs: + - updater-check + - source-build + - artifact-verification-dist + - artifact-verification-static + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Retrieve Artifacts + uses: actions/download-artifact@v2 + with: + name: final-artifacts + path: final-artifacts + - name: Create Release + uses: ncipollo/release-action@v1 + with: + allowUpdates: false + artifactErrorsFailBuild: true + artifacts: 'final-artifacts/sha256sums.txt,final-artifacts/netdata-*.tar.gz,final-artifacts/netdata-*.gz.run' + draft: true + tag: ${{ github.event.inputs.version }} + token: ${{ secrets.NETDATABOT_TOKEN }} + - name: Failure Notification + uses: rtCamp/action-slack-notify@v2 + env: + SLACK_COLOR: 'danger' + SLACK_FOOTER: '' + SLACK_ICON_EMOJI: ':github-actions:' + SLACK_TITLE: 'Failed to draft release:' + SLACK_USERNAME: 'GitHub Actions' + SLACK_MESSAGE: "Failed to draft release." + SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }} + if: >- + ${{ + failure() + && startsWith(github.ref, 'refs/heads/master') + && github.event_name == 'workflow_dispatch' + }} diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index e4025dd20..bc746f9ac 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -4,7 +4,12 @@ on: push: branches: - master - pull_request: + pull_request: null +env: + DO_NOT_TRACK: 1 +concurrency: + group: checks-${{ github.ref }} + cancel-in-progress: true jobs: checksum-checks: name: Checksums @@ -40,44 +45,14 @@ jobs: clang-checks: name: Clang runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v2 - - name: Build - run: | - docker build -f .github/dockerfiles/Dockerfile.clang . - dist-checks: - name: Dist - runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v2 with: submodules: recursive - - name: Prepare environment - run: | - ./packaging/installer/install-required-packages.sh --dont-wait --non-interactive netdata - sudo apt-get install -y libjson-c-dev libipmimonitoring-dev libcups2-dev libsnappy-dev \ - libprotobuf-dev libprotoc-dev libssl-dev protobuf-compiler \ - libnetfilter-acct-dev - - name: Configure - run: | - autoreconf -ivf - ./configure \ - --with-zlib \ - --with-math \ - --with-user=netdata \ - CFLAGS=-O2 - - name: Make dist - run: | - make dist - - name: Verify & Set distfile - run: | - ls -lah netdata-*.tar.gz - echo "DISTFILE=$(ls netdata-*.tar.gz)" >> $GITHUB_ENV - - name: Run run_install_with_dist_file.sh + - name: Build run: | - ./.github/scripts/run_install_with_dist_file.sh "${DISTFILE}" + docker build -f .github/dockerfiles/Dockerfile.clang . gitignore-check: name: .gitignore runs-on: ubuntu-latest diff --git a/.github/workflows/coverity.yml b/.github/workflows/coverity.yml index 766275ed9..c9e25ebf9 100644 --- a/.github/workflows/coverity.yml +++ b/.github/workflows/coverity.yml @@ -8,6 +8,11 @@ on: paths: - .github/workflows/coverity.yml - coverity-scan.sh +env: + DO_NOT_TRACK: 1 +concurrency: + group: coverity-${{ github.ref }} + cancel-in-progress: true jobs: coverity: if: github.repository == 'netdata/netdata' @@ -38,7 +43,7 @@ jobs: uses: rtCamp/action-slack-notify@v2 env: SLACK_COLOR: 'danger' - SLACK_FOOTER: + SLACK_FOOTER: '' SLACK_ICON_EMOJI: ':github-actions:' SLACK_TITLE: 'Coverity run failed:' SLACK_USERNAME: 'GitHub Actions' diff --git a/.github/workflows/dashboard-pr.yml b/.github/workflows/dashboard-pr.yml index 8f5a40f2b..711cf1e1f 100644 --- a/.github/workflows/dashboard-pr.yml +++ b/.github/workflows/dashboard-pr.yml @@ -8,9 +8,12 @@ on: dashboard_version: # This must be specified, and must _exactly_ match the version # tag for the release to be used for the update. - name: Dashboard Version + description: Dashboard Version required: true +env: + DO_NOT_TRACK: 1 + jobs: dashboard-pr: name: Generate Dashboard Version Bump PR @@ -18,30 +21,16 @@ jobs: steps: - name: Checkout uses: actions/checkout@v2 - - name: Create Branch - # This is needed because we want to do a PR, and the commit - # action used below requires the branch it is commiting to to - # already exist. - run: | - git checkout -b dashboard-${{ github.event.inputs.dashboard_version }} - git push -u origin dashboard-${{ github.event.inputs.dashboard_version }} - name: Update Files run: | web/gui/bundle_dashboard.py ${{ github.event.inputs.dashboard_version }} - - name: Commit Changes - uses: swinton/commit@v2.x - env: - GH_TOKEN: ${{ secrets.NETDATABOT_GITHUB_TOKEN }} + - name: Create Pull Request + uses: peter-evans/create-pull-request@v3 with: - files: | - packaging/dashboard.version - packaging/dashboard.checksums + title: 'Update dashboard to version ${{ github.event.inputs.dashboard_version }}.' + body: 'See https://github.com/netdata/dashboard/releases/tag/${{ github.event.inputs.dashboard_version }} for changes.' + branch: dashboard-${{ github.event.inputs.dashboard_version }} + branch-suffix: timestamp + delete-branch: true commit-message: 'Update dashboard to version ${{ github.event.inputs.dashboard_version }}.' - ref: refs/heads/dashboard-${{ github.event.inputs.dashboard_version }} - - name: Create PR - uses: repo-sync/pull-request@v2 - with: - source_branch: dashboard-${{ github.event.inputs.dashboard_version }} - pr_title: 'Update dashboard to version ${{ github.event.inputs.dashboard_version }}.' - pr_body: 'See https://github.com/netdata/dashboard/releases/tag/${{ github.event.inputs.dashboard_version }} for changes.' - github_token: ${{ secrets.NETDATABOT_GITHUB_TOKEN }} + token: ${{ secrets.NETDATABOT_GITHUB_TOKEN }} diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 4f18ce5b6..f8a1d6855 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -4,43 +4,94 @@ on: push: branches: - master - pull_request: + pull_request: null workflow_dispatch: inputs: version: name: Version Tag default: nightly required: true +env: + DO_NOT_TRACK: 1 +concurrency: + group: docker-${{ github.ref }}-${{ github.event_name }} + cancel-in-progress: true jobs: - docker-build: - name: Docker Build + docker-test: + name: Docker Runtime Test runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v2 with: submodules: recursive - - name: Determine if we should push changes and which tags to use - if: github.event_name == 'workflow_dispatch' && github.event.inputs.version != 'nightly' + - name: Setup Buildx + uses: docker/setup-buildx-action@v1 + - name: Test Build + uses: docker/build-push-action@v2 + with: + load: true + push: false + tags: netdata/netdata:test + - name: Test Image + run: .github/scripts/docker-test.sh + + docker-ci: + if: github.event_name != 'workflow_dispatch' + name: Docker Alt Arch Builds + needs: docker-test + runs-on: ubuntu-latest + strategy: + matrix: + platforms: + - linux/i386 + - linux/arm/v7 + - linux/arm64 + - linux/ppc64le + steps: + - name: Checkout + uses: actions/checkout@v2 + with: + submodules: recursive + - name: Setup QEMU + if: matrix.platforms != 'linux/i386' + uses: docker/setup-qemu-action@v1 + - name: Setup Buildx + uses: docker/setup-buildx-action@v1 + - name: Build + uses: docker/build-push-action@v2 + with: + platforms: ${{ matrix.platforms }} + load: false + push: false + tags: netdata/netdata:test + + docker-publish: + if: github.event_name == 'workflow_dispatch' + name: Docker Build and Publish + needs: docker-test + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + with: + submodules: recursive + - name: Determine which tags to use + if: github.event.inputs.version != 'nightly' run: | - echo "publish=true" >> $GITHUB_ENV echo "tags=netdata/netdata:latest,netdata/netdata:stable,$(.github/scripts/gen-docker-tags.py ${{ github.event.inputs.version }})" >> $GITHUB_ENV - - name: Determine if we should push changes and which tags to use - if: github.event_name == 'workflow_dispatch' && github.event.inputs.version == 'nightly' + - name: Determine which tags to use + if: github.event.inputs.version == 'nightly' run: | - echo "publish=true" >> $GITHUB_ENV echo "tags=netdata/netdata:latest,netdata/netdata:edge" >> $GITHUB_ENV - - name: Determine if we should push changes and which tags to use - if: github.event_name != 'workflow_dispatch' - run: | - echo "publish=false" >> $GITHUB_ENV - echo "tags=netdata/netdata:test" >> $GITHUB_ENV + - name: Mark image as official + if: github.repository == 'netdata/netdata' + run: echo "OFFICIAL_IMAGE=true" >> $GITHUB_ENV - name: Setup QEMU uses: docker/setup-qemu-action@v1 - name: Setup Buildx uses: docker/setup-buildx-action@v1 - name: Docker Hub Login - if: github.event_name == 'workflow_dispatch' uses: docker/login-action@v1 with: username: ${{ secrets.DOCKER_HUB_USERNAME }} @@ -48,14 +99,15 @@ jobs: - name: Docker Build uses: docker/build-push-action@v2 with: - platforms: linux/amd64,linux/i386,linux/arm/v7,linux/arm64 - push: ${{ env.publish }} + platforms: linux/amd64,linux/i386,linux/arm/v7,linux/arm64,linux/ppc64le + push: true tags: ${{ env.tags }} + build-args: OFFICIAL_IMAGE=${{ env.OFFICIAL_IMAGE }} - name: Failure Notification uses: rtCamp/action-slack-notify@v2 env: SLACK_COLOR: 'danger' - SLACK_FOOTER: + SLACK_FOOTER: '' SLACK_ICON_EMOJI: ':github-actions:' SLACK_TITLE: 'Docker Build failed:' SLACK_USERNAME: 'GitHub Actions' diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 9f7234f92..73bcc8773 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -9,6 +9,8 @@ on: pull_request: paths: - '**.md' +env: + DO_NOT_TRACK: 1 jobs: markdown-link-check: name: Broken Links diff --git a/.github/workflows/labeler.yml b/.github/workflows/labeler.yml index 24842e73d..a4937a9db 100644 --- a/.github/workflows/labeler.yml +++ b/.github/workflows/labeler.yml @@ -3,7 +3,9 @@ name: Pull Request Labeler on: schedule: - - cron: '*/5 * * * *' + - cron: '*/10 * * * *' +env: + DO_NOT_TRACK: 1 jobs: labeler: runs-on: ubuntu-latest diff --git a/.github/workflows/packaging.yml b/.github/workflows/packaging.yml index 934aa264b..80c14e24b 100644 --- a/.github/workflows/packaging.yml +++ b/.github/workflows/packaging.yml @@ -15,6 +15,11 @@ on: version: name: Package version required: false +env: + DO_NOT_TRACK: 1 +concurrency: + group: packages-${{ github.ref }}-${{ github.event_name }} + cancel-in-progress: true jobs: build: name: Build @@ -26,21 +31,47 @@ jobs: include: - {distro: debian, version: "9", pkgclouddistro: debian/stretch, format: deb, base_image: debian, platform: linux/amd64, arch: amd64} - {distro: debian, version: "9", pkgclouddistro: debian/stretch, format: deb, base_image: debian, platform: linux/i386, arch: i386} + - {distro: debian, version: "9", pkgclouddistro: debian/stretch, format: deb, base_image: debian, platform: linux/arm/v7, arch: armhf} + - {distro: debian, version: "9", pkgclouddistro: debian/stretch, format: deb, base_image: debian, platform: linux/arm64/v8, arch: arm64} - {distro: debian, version: "10", pkgclouddistro: debian/buster, format: deb, base_image: debian, platform: linux/amd64, arch: amd64} - {distro: debian, version: "10", pkgclouddistro: debian/buster, format: deb, base_image: debian, platform: linux/i386, arch: i386} - - {distro: ubuntu, version: "16.04", pkgclouddistro: ubuntu/xenial, format: deb, base_image: ubuntu, platform: linux/amd64, arch: amd64} - - {distro: ubuntu, version: "16.04", pkgclouddistro: ubuntu/xenial, format: deb, base_image: ubuntu, platform: linux/i386, arch: i386} + - {distro: debian, version: "10", pkgclouddistro: debian/buster, format: deb, base_image: debian, platform: linux/arm/v7, arch: armhf} + - {distro: debian, version: "10", pkgclouddistro: debian/buster, format: deb, base_image: debian, platform: linux/arm64/v8, arch: arm64} + - {distro: debian, version: "11", pkgclouddistro: debian/bullseye, format: deb, base_image: debian, platform: linux/amd64, arch: amd64, alias: bullseye} + - {distro: debian, version: "11", pkgclouddistro: debian/bullseye, format: deb, base_image: debian, platform: linux/i386, arch: i386, alias: bullseye} + - {distro: debian, version: "11", pkgclouddistro: debian/bullseye, format: deb, base_image: debian, platform: linux/arm/v7, arch: armhf, alias: bullseye} + - {distro: debian, version: "11", pkgclouddistro: debian/bullseye, format: deb, base_image: debian, platform: linux/arm64/v8, arch: arm64, alias: bullseye} - {distro: ubuntu, version: "18.04", pkgclouddistro: ubuntu/bionic, format: deb, base_image: ubuntu, platform: linux/amd64, arch: amd64} - {distro: ubuntu, version: "18.04", pkgclouddistro: ubuntu/bionic, format: deb, base_image: ubuntu, platform: linux/i386, arch: i386} + - {distro: ubuntu, version: "18.04", pkgclouddistro: ubuntu/bionic, format: deb, base_image: ubuntu, platform: linux/arm/v7, arch: armhf} + - {distro: ubuntu, version: "18.04", pkgclouddistro: ubuntu/bionic, format: deb, base_image: ubuntu, platform: linux/arm64/v8, arch: arm64} - {distro: ubuntu, version: "20.04", pkgclouddistro: ubuntu/focal, format: deb, base_image: ubuntu, platform: linux/amd64, arch: amd64} - - {distro: ubuntu, version: "20.10", pkgclouddistro: ubuntu/groovy, format: deb, base_image: ubuntu, platform: linux/amd64, arch: amd64} + - {distro: ubuntu, version: "20.04", pkgclouddistro: ubuntu/focal, format: deb, base_image: ubuntu, platform: linux/arm/v7, arch: armhf} + - {distro: ubuntu, version: "20.04", pkgclouddistro: ubuntu/focal, format: deb, base_image: ubuntu, platform: linux/arm64/v8, arch: arm64} - {distro: ubuntu, version: "21.04", pkgclouddistro: ubuntu/hirsute, format: deb, base_image: ubuntu, platform: linux/amd64, arch: amd64} + - {distro: ubuntu, version: "21.04", pkgclouddistro: ubuntu/hirsute, format: deb, base_image: ubuntu, platform: linux/arm/v7, arch: armhf} + - {distro: ubuntu, version: "21.04", pkgclouddistro: ubuntu/hirsute, format: deb, base_image: ubuntu, platform: linux/arm64/v8, arch: arm64} + - {distro: ubuntu, version: "21.10", pkgclouddistro: ubuntu/impish, format: deb, base_image: ubuntu, platform: linux/amd64, arch: amd64} + - {distro: ubuntu, version: "21.10", pkgclouddistro: ubuntu/impish, format: deb, base_image: ubuntu, platform: linux/arm/v7, arch: armhf} + - {distro: ubuntu, version: "21.10", pkgclouddistro: ubuntu/impish, format: deb, base_image: ubuntu, platform: linux/arm64/v8, arch: arm64} - {distro: centos, version: "7", pkgclouddistro: el/7, format: rpm, base_image: centos, platform: linux/amd64, arch: amd64} - {distro: centos, version: "8", pkgclouddistro: el/8, format: rpm, base_image: centos, platform: linux/amd64, arch: amd64} - - {distro: fedora, version: "32", pkgclouddistro: fedora/32, format: rpm, base_image: fedora, platform: linux/amd64, arch: amd64} + - {distro: centos, version: "8", pkgclouddistro: el/8, format: rpm, base_image: centos, platform: linux/arm64/v8, arch: arm64} - {distro: fedora, version: "33", pkgclouddistro: fedora/33, format: rpm, base_image: fedora, platform: linux/amd64, arch: amd64} + - {distro: fedora, version: "33", pkgclouddistro: fedora/33, format: rpm, base_image: fedora, platform: linux/arm/v7, arch: armhf} + - {distro: fedora, version: "33", pkgclouddistro: fedora/33, format: rpm, base_image: fedora, platform: linux/arm64/v8, arch: arm64} - {distro: fedora, version: "34", pkgclouddistro: fedora/34, format: rpm, base_image: fedora, platform: linux/amd64, arch: amd64} + - {distro: fedora, version: "34", pkgclouddistro: fedora/34, format: rpm, base_image: fedora, platform: linux/arm/v7, arch: armhf} + - {distro: fedora, version: "34", pkgclouddistro: fedora/34, format: rpm, base_image: fedora, platform: linux/arm64/v8, arch: arm64} + - {distro: fedora, version: "35", pkgclouddistro: fedora/35, format: rpm, base_image: fedora, platform: linux/amd64, arch: amd64} + - {distro: fedora, version: "35", pkgclouddistro: fedora/35, format: rpm, base_image: fedora, platform: linux/arm/v7, arch: armhf} + - {distro: fedora, version: "35", pkgclouddistro: fedora/35, format: rpm, base_image: fedora, platform: linux/arm64/v8, arch: arm64} - {distro: opensuse, version: "15.2", pkgclouddistro: opensuse/15.2, format: rpm, base_image: opensuse/leap, platform: linux/amd64, arch: amd64} + - {distro: opensuse, version: "15.2", pkgclouddistro: opensuse/15.2, format: rpm, base_image: opensuse/leap, platform: linux/arm64/v8, arch: arm64} + - {distro: opensuse, version: "15.3", pkgclouddistro: opensuse/15.3, format: rpm, base_image: opensuse/leap, platform: linux/amd64, arch: amd64} + - {distro: opensuse, version: "15.3", pkgclouddistro: opensuse/15.3, format: rpm, base_image: opensuse/leap, platform: linux/arm64/v8, arch: arm64} + - {distro: oraclelinux, version: "8", pkgclouddistro: ol/8, format: rpm, base_image: oraclelinux, platform: linux/amd64, arch: amd64} + - {distro: oraclelinux, version: "8", pkgclouddistro: ol/8, format: rpm, base_image: oraclelinux, platform: linux/arm64/v8, arch: arm64} # We intentiaonally disable the fail-fast behavior so that a # build failure for one version doesn't prevent us from publishing # successfully built and tested packages for another version. @@ -52,14 +83,14 @@ jobs: uses: actions/checkout@v2 with: fetch-depth: 0 # We need full history for versioning - submodules: true + submodules: recursive - name: Checkout Tag # Otherwise check out the tag that triggered this. if: github.event_name == 'workflow_dispatch' uses: actions/checkout@v2 with: ref: ${{ github.event.ref }} fetch-depth: 0 # We need full history for versioning - submodules: true + submodules: recursive - name: Check Base Branch run: | if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then @@ -86,35 +117,44 @@ jobs: echo "pkg_version=$(cut -d'-' -f 1 packaging/version | sed -e 's/^v//')" >> $GITHUB_ENV fi - name: Setup QEMU - if: matrix.platform != 'linux/amd64' + if: matrix.platform != 'linux/amd64' && matrix.platform != 'linux/i386' uses: docker/setup-qemu-action@v1 - - name: Setup Buildx - uses: docker/setup-buildx-action@v1 - name: Prepare Docker Environment shell: bash run: | echo '{"cgroup-parent": "/actions_job", "experimental": true}' | sudo tee /etc/docker/daemon.json 2>/dev/null sudo service docker restart - - name: Build Packages - uses: docker/build-push-action@v2 + - name: Set Base Image Version + shell: bash + run: | + if [ -z "${{ matrix.alias }}" ] ; then + echo "version=${{ matrix.version }}" >> $GITHUB_ENV + else + echo "version=${{ matrix.alias }}" >> $GITHUB_ENV + fi + - name: Fetch base image + uses: nick-invision/retry@v2 with: - platforms: ${{ matrix.platform }} - file: packaging/Dockerfile.packager - tags: local/package-builder:${{ matrix.distro}}${{ matrix.version }} - push: false - load: true - build-args: | - ARCH=${{ matrix.arch }} - DISTRO=${{ matrix.distro }} - TEST_BASE=${{ matrix.base_image }} - DISTRO_VERSION=${{ matrix.version }} - PKG_VERSION=${{ env.pkg_version }} - - name: Extract Packages + max_attempts: 3 + retry_wait_seconds: 30 + timeout_seconds: 900 + command: | + docker pull --platform ${{ matrix.platform }} ${{ matrix.base_image }}:${{ env.version }} + docker pull --platform ${{ matrix.platform }} netdata/package-builders:${{ matrix.distro }}${{ matrix.version }} + - name: Build Packages + shell: bash + run: | + docker run --security-opt seccomp=unconfined -e DO_NOT_TRACK=1 -e VERSION=${{ env.pkg_version }} --platform=${{ matrix.platform }} -v $PWD:/netdata netdata/package-builders:${{ matrix.distro }}${{ matrix.version }} + - name: Test Packages shell: bash run: | - mkdir -p artifacts - docker run --platform ${{ matrix.platform }} -v $PWD/artifacts:/artifacts local/package-builder:${{ matrix.distro }}${{ matrix.version }} - - name: Upload + docker run --security-opt seccomp=unconfined -e DO_NOT_TRACK=1 -e DISTRO=${{ matrix.distro }} -e VERSION=${{ env.pkg_version }} -e DISTRO_VERSION=${{ env.version }} --platform=${{ matrix.platform }} -v $PWD:/netdata ${{ matrix.base_image }}:${{ env.version }} /netdata/.github/scripts/pkg-test.sh + - name: Save Packages + uses: actions/upload-artifact@v2 + with: + name: ${{ matrix.distro }}-${{ matrix.version }}-${{ matrix.arch }}-packages + path: ${{ github.workspace }}/artifacts/* + - name: Upload to PackageCloud if: github.event_name == 'workflow_dispatch' shell: bash env: @@ -122,7 +162,7 @@ jobs: run: | echo "Packages to upload:\n$(ls artifacts/*.${{ matrix.format }})" for pkgfile in artifacts/*.${{ matrix.format }} ; do - .github/scripts/package_cloud_wrapper.sh yank ${{ env.repo }}/${{ matrix.pkgclouddistro }} ${pkgfile} || true + .github/scripts/package_cloud_wrapper.sh yank ${{ env.repo }}/${{ matrix.pkgclouddistro }} $(basename ${pkgfile}) || true .github/scripts/package_cloud_wrapper.sh push ${{ env.repo }}/${{ matrix.pkgclouddistro }} ${pkgfile} done - name: Clean @@ -137,7 +177,6 @@ jobs: uses: rtCamp/action-slack-notify@v2 env: SLACK_COLOR: 'danger' - SLACK_FOOTER: SLACK_ICON_EMOJI: ':github-actions:' SLACK_TITLE: 'Package Build failed:' SLACK_USERNAME: 'GitHub Actions' diff --git a/.github/workflows/repoconfig-packages.yml b/.github/workflows/repoconfig-packages.yml new file mode 100644 index 000000000..c132ec8b5 --- /dev/null +++ b/.github/workflows/repoconfig-packages.yml @@ -0,0 +1,80 @@ +--- +# Handles building of binary packages for the agent. +name: Repository Packages +on: + workflow_dispatch: null +env: + DO_NOT_TRACK: 1 +jobs: + build: + name: Build + runs-on: ubuntu-latest + env: + DO_NOT_TRACK: 1 + DOCKER_CLI_EXPERIMENTAL: enabled + strategy: + # This needs to be kept in sync with the matrix in packaging.yml, but should only include the AMD64 lines. + matrix: + include: + - {distro: debian, version: "9", pkgclouddistro: debian/stretch, format: deb, base_image: debian, platform: linux/amd64, arch: amd64} + - {distro: debian, version: "10", pkgclouddistro: debian/buster, format: deb, base_image: debian, platform: linux/amd64, arch: amd64} + - {distro: debian, version: "11", pkgclouddistro: debian/bullseye, format: deb, base_image: debian, platform: linux/amd64, arch: amd64} + - {distro: ubuntu, version: "18.04", pkgclouddistro: ubuntu/bionic, format: deb, base_image: ubuntu, platform: linux/amd64, arch: amd64} + - {distro: ubuntu, version: "20.04", pkgclouddistro: ubuntu/focal, format: deb, base_image: ubuntu, platform: linux/amd64, arch: amd64} + - {distro: ubuntu, version: "21.04", pkgclouddistro: ubuntu/hirsute, format: deb, base_image: ubuntu, platform: linux/amd64, arch: amd64} + - {distro: ubuntu, version: "21.10", pkgclouddistro: ubuntu/impish, format: deb, base_image: ubuntu, platform: linux/amd64, arch: amd64} + - {distro: centos, version: "7", pkgclouddistro: el/7, format: rpm, base_image: centos, platform: linux/amd64, arch: amd64} + - {distro: centos, version: "8", pkgclouddistro: el/8, format: rpm, base_image: centos, platform: linux/amd64, arch: amd64} + - {distro: fedora, version: "33", pkgclouddistro: fedora/33, format: rpm, base_image: fedora, platform: linux/amd64, arch: amd64} + - {distro: fedora, version: "34", pkgclouddistro: fedora/34, format: rpm, base_image: fedora, platform: linux/amd64, arch: amd64} + - {distro: fedora, version: "35", pkgclouddistro: fedora/35, format: rpm, base_image: fedora, platform: linux/amd64, arch: amd64} + - {distro: opensuse, version: "15.2", pkgclouddistro: opensuse/15.2, format: rpm, base_image: opensuse/leap, platform: linux/amd64, arch: amd64} + - {distro: opensuse, version: "15.3", pkgclouddistro: opensuse/15.3, format: rpm, base_image: opensuse/leap, platform: linux/amd64, arch: amd64} + - {distro: oraclelinux, version: "8", pkgclouddistro: ol/8, format: rpm, base_image: oraclelinux, platform: linux/amd64, arch: amd64} + # We intentiaonally disable the fail-fast behavior so that a + # build failure for one version doesn't prevent us from publishing + # successfully built and tested packages for another version. + fail-fast: false + max-parallel: 8 + steps: + - name: Checkout + uses: actions/checkout@v2 + # Unlike normally, we do not need a deep clone or submodules for this. + - name: Fetch base image + uses: nick-invision/retry@v2 + with: + max_attempts: 3 + retry_wait_seconds: 30 + timeout_seconds: 900 + command: docker pull --platform ${{ matrix.platform }} ${{ matrix.base_image }}:${{ matrix.version }} + - name: Build Packages + shell: bash + run: | + docker run --security-opt seccomp=unconfined -e DO_NOT_TRACK=1 --platform ${{ matrix.platform }} \ + -v $PWD:/netdata ${{ matrix.base_image }}:${{ matrix.version }} \ + /netdata/packaging/repoconfig/build-${{ matrix.format }}.sh + - name: Upload Packages + shell: bash + env: + PKG_CLOUD_TOKEN: ${{ secrets.PACKAGE_CLOUD_API_KEY }} + run: | + echo "Packages to upload:\n$(ls artifacts/*.${{ matrix.format }})" + for pkgfile in artifacts/*.${{ matrix.format }} ; do + .github/scripts/package_cloud_wrapper.sh yank ${{ secrets.PACKAGE_CLOUD_REPO }}/${{ matrix.pkgclouddistro }} $(basename ${pkgfile}) || true + .github/scripts/package_cloud_wrapper.sh push ${{ secrets.PACKAGE_CLOUD_REPO }}/${{ matrix.pkgclouddistro }} ${pkgfile} + .github/scripts/package_cloud_wrapper.sh yank ${{ secrets.PACKAGE_CLOUD_REPO }}-edge/${{ matrix.pkgclouddistro }} $(basename ${pkgfile}) || true + .github/scripts/package_cloud_wrapper.sh push ${{ secrets.PACKAGE_CLOUD_REPO }}-edge/${{ matrix.pkgclouddistro }} ${pkgfile} + .github/scripts/package_cloud_wrapper.sh yank ${{ secrets.PACKAGE_CLOUD_REPO }}-repoconfig/${{ matrix.pkgclouddistro }} $(basename ${pkgfile}) || true + .github/scripts/package_cloud_wrapper.sh push ${{ secrets.PACKAGE_CLOUD_REPO }}-repoconfig/${{ matrix.pkgclouddistro }} ${pkgfile} + done + - name: Failure Notification + if: ${{ failure() }} + uses: rtCamp/action-slack-notify@v2 + env: + SLACK_COLOR: 'danger' + SLACK_FOOTER: '' + SLACK_ICON_EMOJI: ':github-actions:' + SLACK_TITLE: 'Repository Package Build failed:' + SLACK_USERNAME: 'GitHub Actions' + SLACK_MESSAGE: "${{ matrix.pkgclouddistro }} ${{ matrix.version }} repository package build failed." + SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }} diff --git a/.github/workflows/review.yml b/.github/workflows/review.yml index a267fea3f..e9972303b 100644 --- a/.github/workflows/review.yml +++ b/.github/workflows/review.yml @@ -2,15 +2,95 @@ # Runs various ReviewDog based checks against PR with suggested changes to improve quality name: Review on: - pull_request: + pull_request: null env: - run_eslint: 0 - run_hadolint: 0 - run_shellcheck: 0 - run_yamllint: 0 + DO_NOT_TRACK: 1 +concurrency: + group: review-${{ github.ref }} + cancel-in-progress: true jobs: + prep-review: + name: Prepare Review Jobs + runs-on: ubuntu-latest + outputs: + actionlint: ${{ steps.actionlint.outputs.run }} + eslint: ${{ steps.eslint.outputs.run }} + hadolint: ${{ steps.hadolint.outputs.run }} + shellcheck: ${{ steps.shellcheck.outputs.run }} + yamllint: ${{ steps.yamllint.outputs.run }} + steps: + - name: Clone repository + uses: actions/checkout@v2 + with: + submodules: recursive + fetch-depth: 0 + - name: Check files for actionlint + id: actionlint + run: | + if git diff --name-only origin/${{ github.base_ref }} HEAD | grep -Eq '\.github/workflows/.*' ; then + echo '::set-output name=run::true' + echo 'GitHub Actions workflows have changed, need to run actionlint.' + else + echo '::set-output name=run::false' + fi + - name: Check files for eslint + id: eslint + run: | + if git diff --name-only origin/${{ github.base_ref }} HEAD | grep -v "web/gui/dashboard" | grep -Eq '.*\.js|node\.d\.plugin\.in' ; then + echo '::set-output name=run::true' + echo 'JS files have changed, need to run ESLint.' + else + echo '::set-output name=run::false' + fi + - name: Check files for hadolint + id: hadolint + run: | + if git diff --name-only origin/${{ github.base_ref }} HEAD | grep -Eq '.*Dockerfile.*' ; then + echo '::set-output name=run::true' + echo 'Dockerfiles have changed, need to run Hadolint.' + else + echo '::set-output name=run::false' + fi + - name: Check files for shellcheck + id: shellcheck + run: | + if git diff --name-only origin/${{ github.base_ref }} HEAD | grep -Eq '.*\.sh.*' ; then + echo '::set-output name=run::true' + echo 'Shell scripts have changed, need to run shellcheck.' + else + echo '::set-output name=run::false' + fi + - name: Check files for yamllint + id: yamllint + run: | + if git diff --name-only origin/${{ github.base_ref }} HEAD | grep -Eq '.*\.ya?ml|python\.d/.*\.conf' ; then + echo '::set-output name=run::true' + echo 'YAML files have changed, need to run yamllint.' + else + echo '::set-output name=run::false' + fi + + actionlint: + name: actionlint + needs: prep-review + if: needs.prep-review.outputs.actionlint == 'true' + runs-on: ubuntu-latest + steps: + - name: Git clone repository + uses: actions/checkout@v2 + with: + submodules: recursive + fetch-depth: 0 + - name: Run actionlint + uses: reviewdog/action-actionlint@v1 + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + reporter: github-pr-check + eslint: name: eslint + needs: prep-review + if: needs.prep-review.outputs.eslint == 'true' runs-on: ubuntu-latest steps: - name: Git clone repository @@ -18,13 +98,7 @@ jobs: with: submodules: recursive fetch-depth: 0 - - name: Check files - run: | - if git diff --name-only origin/${{ github.base_ref }} HEAD | grep -Eq '*\.js|node\.d\.plugin\.in' ; then - echo 'run_eslint=1' >> $GITHUB_ENV - fi - name: Run eslint - if: env.run_eslint == 1 uses: reviewdog/action-eslint@v1 with: github_token: ${{ secrets.GITHUB_TOKEN }} @@ -33,19 +107,15 @@ jobs: hadolint: name: hadolint + needs: prep-review + if: needs.prep-review.outputs.hadolint == 'true' runs-on: ubuntu-latest steps: - name: Git clone repository uses: actions/checkout@v2 with: fetch-depth: 0 - - name: Check files - run: | - if git diff --name-only origin/${{ github.base_ref }} HEAD | grep -Eq '*Dockerfile*' ; then - echo 'run_hadolint=1' >> $GITHUB_ENV - fi - name: Run hadolint - if: env.run_hadolint == 1 uses: reviewdog/action-hadolint@v1 with: github_token: ${{ secrets.GITHUB_TOKEN }} @@ -53,6 +123,8 @@ jobs: shellcheck: name: shellcheck + needs: prep-review + if: needs.prep-review.outputs.shellcheck == 'true' runs-on: ubuntu-latest steps: - name: Git clone repository @@ -60,13 +132,7 @@ jobs: with: submodules: recursive fetch-depth: 0 - - name: Check files - run: | - if git diff --name-only origin/${{ github.base_ref }} HEAD | grep -Eq '*\.sh.*' ; then - echo 'run_shellcheck=1' >> $GITHUB_ENV - fi - name: Run shellcheck - if: env.run_shellcheck == 1 uses: reviewdog/action-shellcheck@v1 with: github_token: ${{ secrets.GITHUB_TOKEN }} @@ -77,6 +143,8 @@ jobs: yamllint: name: yamllint + needs: prep-review + if: needs.prep-review.outputs.yamllint == 'true' runs-on: ubuntu-latest steps: - name: Git clone repository @@ -84,13 +152,7 @@ jobs: with: submodules: recursive fetch-depth: 0 - - name: Check files - run: | - if git diff --name-only origin/${{ github.base_ref }} HEAD | grep -Eq '*\.ya?ml|python\.d/.*\.conf' ; then - echo 'run_yamllint=1' >> $GITHUB_ENV - fi - name: Run yamllint - if: env.run_yamllint == 1 uses: reviewdog/action-yamllint@v1 with: github_token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 7e47f12da..20c6f7b8c 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -14,6 +14,11 @@ on: - 'CMakeLists.txt' - '**.c' - '**.h' +env: + DO_NOT_TRACK: 1 +concurrency: + group: tests-${{ github.ref }} + cancel-in-progress: true jobs: unit-tests-legacy: name: Unit Tests (legacy) @@ -52,7 +57,7 @@ jobs: - name: Configure run: | autoreconf -ivf - ./configure --without-aclk-ng + ./configure --disable-ml # XXX: Work-around for bug with libbson-1.0 in Ubuntu 18.04 # See: https://bugs.launchpad.net/ubuntu/+source/libmongoc/+bug/1790771 # https://jira.mongodb.org/browse/CDRIVER-2818 @@ -85,7 +90,7 @@ jobs: find . -type f -name '*.log' -exec cp {} ../logs/ \; popd || exit 1 - name: Upload Artifacts - uses: actions/upload-artifact@v2.2.3 + uses: actions/upload-artifact@v2.2.4 if: always() with: name: logs diff --git a/.github/workflows/updater.yml b/.github/workflows/updater.yml deleted file mode 100644 index 5f8d77c4a..000000000 --- a/.github/workflows/updater.yml +++ /dev/null @@ -1,85 +0,0 @@ ---- -name: Updater -on: - push: - branches: - - master - pull_request: - branches: - - master - -jobs: - source-build: - name: Install, Build & Update - strategy: - fail-fast: false - max-parallel: 8 - matrix: - distro: - - 'alpine:3.12' - - 'alpine:3.13' - - 'archlinux:latest' - - 'centos:7' - - 'centos:8' - - 'clearlinux:latest' - - 'debian:9' - - 'debian:10' - - 'fedora:33' - - 'fedora:34' - - 'ubuntu:16.04' - - 'ubuntu:18.04' - - 'ubuntu:20.04' - - 'ubuntu:20.10' - - 'ubuntu:21.04' - include: - - distro: 'alpine:3.12' - pre: 'apk add -U bash' - - distro: 'alpine:3.13' - pre: 'apk add -U bash' - - distro: 'debian:9' - pre: 'apt-get update' - - distro: 'debian:10' - pre: 'apt-get update' - - distro: 'ubuntu:16.04' - pre: 'apt-get update' - - distro: 'ubuntu:18.04' - pre: 'apt-get update' - - distro: 'ubuntu:20.04' - pre: 'apt-get update' - - distro: 'ubuntu:20.10' - pre: 'apt-get update' - - distro: 'ubuntu:21.04' - pre: 'apt-get update' - runs-on: ubuntu-latest - steps: - - name: Git clone repository - uses: actions/checkout@v2 - with: - submodules: recursive - - name: Install required packages & build tarball - run: | - ./packaging/installer/install-required-packages.sh --dont-wait --non-interactive netdata-all - .github/scripts/build-dist.sh - - name: Run a dockerised web server to serve files used by the custom update script - run: | - docker run -dit --name my-apache-app -p 8080:80 -v "$PWD":/usr/local/apache2/htdocs/ httpd:2.4 - - name: Replace URLs in updater script to point at the local web server - run: | - ORIG_TARBALL="export NETDATA_TARBALL_URL=.*" - ORIG_CHECKSUM="export NETDATA_TARBALL_CHECKSUM_URL=.*" - CURRENT_VERSION="current_version=.*" - NEW_TARBALL="export NETDATA_TARBALL_URL=http://localhost:8080/artifacts/netdata-latest.tar.gz" - NEW_CHECKSUM="export NETDATA_TARBALL_CHECKSUM_URL=http://localhost:8080/artifacts/sha256sums.txt" - sed -i "s|${ORIG_TARBALL}|${NEW_TARBALL}|g" packaging/installer/netdata-updater.sh - sed -i "s|${ORIG_CHECKSUM}|${NEW_CHECKSUM}|g" packaging/installer/netdata-updater.sh - sed -i "s|"current_version=.*"|"current_version=1"|g" packaging/installer/netdata-updater.sh - - name: Install netdata and run the updater on ${{ matrix.distro }} - env: - PRE: ${{ matrix.pre }} - run: | - echo $PRE > ./prep-cmd.sh - docker build . -f .github/dockerfiles/Dockerfile.build_test -t test --build-arg BASE=${{ matrix.distro }} - docker run --network host -w /netdata test \ - /bin/sh -c '/netdata/packaging/installer/kickstart.sh --dont-wait \ - && /netdata/packaging/installer/netdata-updater.sh --not-running-from-cron --no-updater-self-update \ - && bash /netdata/.github/scripts/check-updater.sh' -- cgit v1.2.3