From bb8713bbc1c4594366fc735c04910edbf4c61aab Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Wed, 31 Mar 2021 14:59:21 +0200 Subject: Merging upstream version 1.30.0. Signed-off-by: Daniel Baumann --- .github/workflows/build-and-install.yml | 13 +++ .github/workflows/checks.yml | 10 +++ .github/workflows/coverity.yml | 16 ++++ .github/workflows/docker.yml | 28 ++++-- .github/workflows/docs.yml | 2 + .github/workflows/packaging.yml | 149 ++++++++++++++++++++++++++++++++ .github/workflows/review.yml | 3 + .github/workflows/tests.yml | 6 +- .github/workflows/updater.yml | 84 ++++++++++++++++++ 9 files changed, 301 insertions(+), 10 deletions(-) create mode 100644 .github/workflows/packaging.yml create mode 100644 .github/workflows/updater.yml (limited to '.github/workflows') diff --git a/.github/workflows/build-and-install.yml b/.github/workflows/build-and-install.yml index 26a144acd..4c4693601 100644 --- a/.github/workflows/build-and-install.yml +++ b/.github/workflows/build-and-install.yml @@ -12,6 +12,8 @@ jobs: steps: - name: Git clone repository uses: actions/checkout@v2 + with: + submodules: recursive - run: | git fetch --prune --unshallow --tags - name: Build @@ -34,10 +36,12 @@ jobs: - 'clearlinux:latest' - 'debian:10' - 'debian:9' + - 'fedora:34' - 'fedora:33' - 'fedora:32' - 'opensuse/leap:15.2' - 'opensuse/tumbleweed:latest' + - 'ubuntu:21.04' - 'ubuntu:20.10' - 'ubuntu:20.04' - 'ubuntu:18.04' @@ -72,6 +76,8 @@ jobs: pre: 'apt-get update' rmjsonc: 'apt-get remove -y libjson-c-dev' + - distro: 'fedora:34' + rmjsonc: 'dnf remove -y json-c-devel' - distro: 'fedora:33' rmjsonc: 'dnf remove -y json-c-devel' - distro: 'fedora:32' @@ -82,6 +88,9 @@ jobs: - distro: 'opensuse/tumbleweed:latest' rmjsonc: 'zypper rm -y libjson-c-devel' + - distro: 'ubuntu:21.04' + pre: 'apt-get update' + rmjsonc: 'apt-get remove -y libjson-c-dev' - distro: 'ubuntu:20.10' pre: 'apt-get update' rmjsonc: 'apt-get remove -y libjson-c-dev' @@ -98,6 +107,8 @@ jobs: steps: - name: Git clone repository uses: actions/checkout@v2 + with: + submodules: recursive - name: install-required-packages.sh on ${{ matrix.distro }} env: PRE: ${{ matrix.pre }} @@ -183,6 +194,8 @@ jobs: steps: - name: Git clone repository uses: actions/checkout@v2 + with: + submodules: recursive - name: install-required-packages.sh on ${{ matrix.distro }} env: PRE: ${{ matrix.pre }} diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index cf494e95c..7225d3dbe 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -12,6 +12,8 @@ jobs: steps: - name: Git clone repository uses: actions/checkout@v2 + with: + submodules: recursive - name: Run checksum checks on kickstart files env: LOCAL_ONLY: "true" @@ -23,6 +25,8 @@ jobs: steps: - name: Git clone repository uses: actions/checkout@v2 + with: + submodules: recursive - name: Install required packages run: | ./packaging/installer/install-required-packages.sh --dont-wait --non-interactive netdata @@ -43,6 +47,8 @@ jobs: steps: - name: Checkout uses: actions/checkout@v2 + with: + submodules: recursive - name: Build run: > docker run -v "$PWD":/netdata -w /netdata alpine:latest /bin/sh -c @@ -68,6 +74,8 @@ jobs: steps: - name: Checkout uses: actions/checkout@v2 + with: + submodules: recursive - name: Prepare environment run: | ./packaging/installer/install-required-packages.sh --dont-wait --non-interactive netdata @@ -98,6 +106,8 @@ jobs: steps: - name: Checkout uses: actions/checkout@v2 + with: + submodules: recursive - name: Prepare environment run: ./packaging/installer/install-required-packages.sh --dont-wait --non-interactive netdata - name: Build netdata diff --git a/.github/workflows/coverity.yml b/.github/workflows/coverity.yml index 926257dc0..766275ed9 100644 --- a/.github/workflows/coverity.yml +++ b/.github/workflows/coverity.yml @@ -15,6 +15,8 @@ jobs: steps: - name: Checkout uses: actions/checkout@v2 + with: + submodules: recursive - name: Prepare environment env: DEBIAN_FRONTEND: 'noninteractive' @@ -32,3 +34,17 @@ jobs: COVERITY_SCAN_SUBMIT_MAIL: ${{ secrets.COVERITY_SCAN_SUBMIT_MAIL }} run: | ./coverity-scan.sh --with-install + - name: Failure Notification + uses: rtCamp/action-slack-notify@v2 + env: + SLACK_COLOR: 'danger' + SLACK_FOOTER: + SLACK_ICON_EMOJI: ':github-actions:' + SLACK_TITLE: 'Coverity run failed:' + SLACK_USERNAME: 'GitHub Actions' + SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }} + if: ${{ + failure() + && github.event_name != 'pull_request' + && startsWith(github.ref, 'refs/heads/master') + }} diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 04f91bff5..a5648d2a9 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -4,15 +4,7 @@ on: push: branches: - master - paths: - - '.github/workflows/docker.yml' - - 'netdata-installer.sh' - - 'packaging/**' pull_request: - paths: - - '.github/workflows/docker.yml' - - 'netdata-installer.sh' - - 'packaging/**' workflow_dispatch: inputs: version: @@ -26,11 +18,13 @@ jobs: steps: - name: Checkout uses: actions/checkout@v2 + with: + submodules: recursive - name: Determine if we should push changes and which tags to use if: github.event_name == 'workflow_dispatch' && github.event.inputs.version != 'nightly' run: | echo "publish=true" >> $GITHUB_ENV - echo "tags=netdata/netdata:latest,netdata/netdata:stable,netdata/netdata:${{ github.event.inputs.version }}" >> $GITHUB_ENV + echo "tags=netdata/netdata:latest,netdata/netdata:stable,$(.github/scripts/gen-docker-tags.py ${{ github.event.inputs.version }})" >> $GITHUB_ENV - name: Determine if we should push changes and which tags to use if: github.event_name == 'workflow_dispatch' && github.event.inputs.version == 'nightly' run: | @@ -57,3 +51,19 @@ jobs: platforms: linux/amd64,linux/i386,linux/arm/v7,linux/arm64 push: ${{ env.publish }} tags: ${{ env.tags }} + - name: Failure Notification + uses: rtCamp/action-slack-notify@v2 + env: + SLACK_COLOR: 'danger' + SLACK_FOOTER: + SLACK_ICON_EMOJI: ':github-actions:' + SLACK_TITLE: 'Docker Build failed:' + SLACK_USERNAME: 'GitHub Actions' + SLACK_MESSAGE: "Docker image build failed." + SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }} + if: >- + ${{ + failure() + && github.event_name != 'pull_request' + && startsWith(github.ref, 'refs/heads/master') + }} diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 2a4fe87e4..9f7234f92 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -16,6 +16,8 @@ jobs: steps: - name: Checkout uses: actions/checkout@v2 + with: + submodules: recursive - name: Run link check uses: gaurav-nelson/github-action-markdown-link-check@v1 with: diff --git a/.github/workflows/packaging.yml b/.github/workflows/packaging.yml new file mode 100644 index 000000000..7340c27bb --- /dev/null +++ b/.github/workflows/packaging.yml @@ -0,0 +1,149 @@ +--- +# Handles building of binary packages for the agent. +name: Packages +on: + pull_request: + branches: + - master + - develop + workflow_dispatch: + inputs: + type: + name: Package build type + default: devel + required: true + version: + name: Package version + required: false +jobs: + build: + name: Build + runs-on: ubuntu-latest + env: + DOCKER_CLI_EXPERIMENTAL: enabled + strategy: + matrix: + include: + - {distro: debian, version: "9", pkgclouddistro: debian/stretch, format: deb, base_image: debian, platform: linux/amd64, arch: amd64} + - {distro: debian, version: "9", pkgclouddistro: debian/stretch, format: deb, base_image: debian, platform: linux/i386, arch: i386} + - {distro: debian, version: "10", pkgclouddistro: debian/buster, format: deb, base_image: debian, platform: linux/amd64, arch: amd64} + - {distro: debian, version: "10", pkgclouddistro: debian/buster, format: deb, base_image: debian, platform: linux/i386, arch: i386} + - {distro: ubuntu, version: "16.04", pkgclouddistro: ubuntu/xenial, format: deb, base_image: ubuntu, platform: linux/amd64, arch: amd64} + - {distro: ubuntu, version: "16.04", pkgclouddistro: ubuntu/xenial, format: deb, base_image: ubuntu, platform: linux/i386, arch: i386} + - {distro: ubuntu, version: "18.04", pkgclouddistro: ubuntu/bionic, format: deb, base_image: ubuntu, platform: linux/amd64, arch: amd64} + - {distro: ubuntu, version: "18.04", pkgclouddistro: ubuntu/bionic, format: deb, base_image: ubuntu, platform: linux/i386, arch: i386} + - {distro: ubuntu, version: "20.04", pkgclouddistro: ubuntu/focal, format: deb, base_image: ubuntu, platform: linux/amd64, arch: amd64} + - {distro: ubuntu, version: "20.10", pkgclouddistro: ubuntu/groovy, format: deb, base_image: ubuntu, platform: linux/amd64, arch: amd64} + - {distro: ubuntu, version: "21.04", pkgclouddistro: ubuntu/hirsute, format: deb, base_image: ubuntu, platform: linux/amd64, arch: amd64} + - {distro: centos, version: "7", pkgclouddistro: el/7, format: rpm, base_image: centos, platform: linux/amd64, arch: amd64} + - {distro: centos, version: "8", pkgclouddistro: el/8, format: rpm, base_image: centos, platform: linux/amd64, arch: amd64} + - {distro: fedora, version: "32", pkgclouddistro: fedora/32, format: rpm, base_image: fedora, platform: linux/amd64, arch: amd64} + - {distro: fedora, version: "33", pkgclouddistro: fedora/33, format: rpm, base_image: fedora, platform: linux/amd64, arch: amd64} + - {distro: fedora, version: "34", pkgclouddistro: fedora/34, format: rpm, base_image: fedora, platform: linux/amd64, arch: amd64} + - {distro: opensuse, version: "15.2", pkgclouddistro: opensuse/15.2, format: rpm, base_image: opensuse/leap, platform: linux/amd64, arch: amd64} + # We intentiaonally disable the fail-fast behavior so that a + # build failure for one version doesn't prevent us from publishing + # successfully built and tested packages for another version. + fail-fast: false + steps: + - name: Checkout PR # Checkout the PR if it's a PR. + if: github.event_name == 'pull_request' + uses: actions/checkout@v2 + with: + fetch-depth: 0 # We need full history for versioning + submodules: true + - name: Checkout Tag # Otherwise check out the tag that triggered this. + if: github.event_name == 'workflow_dispatch' + uses: actions/checkout@v2 + with: + ref: ${{ github.event.ref }} + fetch-depth: 0 # We need full history for versioning + submodules: true + - name: Check Base Branch + run: | + if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then + echo "runtype=${{ github.event.inputs.type }}" >> $GITHUB_ENV + case "${{ github.event.inputs.type }}" in + "release") + echo "repo=${{ secrets.PACKAGE_CLOUD_REPO }}" >> $GITHUB_ENV + echo "pkg_version=${{ github.event.inputs.version }}" >> $GITHUB_ENV + echo "pkg_retention_days=365" >> $GITHUB_ENV + ;; + "nightly") + echo "repo=${{ secrets.PACKAGE_CLOUD_REPO }}-edge" >> $GITHUB_ENV + echo "pkg_version=${{ github.event.inputs.version }}" >> $GITHUB_ENV + echo "pkg_retention_days=30" >> $GITHUB_ENV + ;; + *) + echo "repo=${{ secrets.PACKAGE_CLOUD_REPO }}-devel" >> $GITHUB_ENV + echo "pkg_version=0.${GITHUB_SHA}" >> $GITHUB_ENV + echo "pkg_retention_days=30" >> $GITHUB_ENV + ;; + esac + else + echo "runtype=test" >> $GITHUB_ENV + echo "pkg_version=$(cut -d'-' -f 1 packaging/version | sed -e 's/^v//')" >> $GITHUB_ENV + fi + - name: Setup QEMU + if: matrix.platform != 'linux/amd64' + uses: docker/setup-qemu-action@v1 + - name: Setup Buildx + uses: docker/setup-buildx-action@v1 + - name: Prepare Docker Environment + shell: bash + run: | + echo '{"cgroup-parent": "/actions_job", "experimental": true}' | sudo tee /etc/docker/daemon.json 2>/dev/null + sudo service docker restart + - name: Build Packages + uses: docker/build-push-action@v2 + with: + platforms: ${{ matrix.platform }} + file: packaging/Dockerfile.packager + tags: local/package-builder:${{ matrix.distro}}${{ matrix.version }} + push: false + load: true + build-args: | + ARCH=${{ matrix.arch }} + DISTRO=${{ matrix.distro }} + TEST_BASE=${{ matrix.base_image }} + DISTRO_VERSION=${{ matrix.version }} + PKG_VERSION=${{ env.pkg_version }} + - name: Extract Packages + shell: bash + run: | + mkdir -p artifacts + docker run --platform ${{ matrix.platform }} -v $PWD/artifacts:/artifacts local/package-builder:${{ matrix.distro }}${{ matrix.version }} + - name: Upload + if: github.event_name == 'workflow_dispatch' + shell: bash + env: + PKG_CLOUD_TOKEN: ${{ secrets.PACKAGE_CLOUD_API_KEY }} + run: | + echo "Packages to upload:\n$(ls artifacts/*.${{ matrix.format }})" + for pkgfile in artifacts/*.${{ matrix.format }} ; do + .github/scripts/package_cloud_wrapper.sh push ${{ env.repo }}/${{ matrix.pkgclouddistro }} ${pkgfile} + done + - name: Clean + if: github.event_name == 'workflow_dispatch' + shell: bash + env: + REPO: ${{ env.repo }} + PKG_CLOUD_TOKEN: ${{ secrets.PACKAGE_CLOUD_API_KEY }} + PACKAGE_CLOUD_RETENTION_DAYS: ${{ env.pkg_retention_days }} + run: .github/scripts/old_package_purging.sh + - name: Failure Notification + uses: rtCamp/action-slack-notify@v2 + env: + SLACK_COLOR: 'danger' + SLACK_FOOTER: + SLACK_ICON_EMOJI: ':github-actions:' + SLACK_TITLE: 'Package Build failed:' + SLACK_USERNAME: 'GitHub Actions' + SLACK_MESSAGE: "${{ matrix.pkgclouddistro }} ${{ matrix.version }} package build for ${{ matrix.arch }} failed." + SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }} + if: >- + ${{ + failure() + && github.event_name != 'pull_request' + && startsWith(github.ref, 'refs/heads/master') + }} diff --git a/.github/workflows/review.yml b/.github/workflows/review.yml index ca8f6de13..a267fea3f 100644 --- a/.github/workflows/review.yml +++ b/.github/workflows/review.yml @@ -16,6 +16,7 @@ jobs: - name: Git clone repository uses: actions/checkout@v2 with: + submodules: recursive fetch-depth: 0 - name: Check files run: | @@ -57,6 +58,7 @@ jobs: - name: Git clone repository uses: actions/checkout@v2 with: + submodules: recursive fetch-depth: 0 - name: Check files run: | @@ -80,6 +82,7 @@ jobs: - name: Git clone repository uses: actions/checkout@v2 with: + submodules: recursive fetch-depth: 0 - name: Check files run: | diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index ef6bfbc2a..c166c7442 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -21,6 +21,8 @@ jobs: steps: - name: Checkout uses: actions/checkout@v2 + with: + submodules: recursive - name: Prepare environment run: | ./packaging/installer/install-required-packages.sh --dont-wait --non-interactive netdata-all @@ -39,6 +41,8 @@ jobs: steps: - name: Checkout uses: actions/checkout@v2 + with: + submodules: recursive - name: Prepare environment run: | ./packaging/installer/install-required-packages.sh --dont-wait --non-interactive netdata-all @@ -48,7 +52,7 @@ jobs: - name: Configure run: | autoreconf -ivf - ./configure + ./configure --without-aclk-ng # XXX: Work-around for bug with libbson-1.0 in Ubuntu 18.04 # See: https://bugs.launchpad.net/ubuntu/+source/libmongoc/+bug/1790771 # https://jira.mongodb.org/browse/CDRIVER-2818 diff --git a/.github/workflows/updater.yml b/.github/workflows/updater.yml new file mode 100644 index 000000000..76cb2fdbe --- /dev/null +++ b/.github/workflows/updater.yml @@ -0,0 +1,84 @@ +--- +name: Updater +on: + push: + branches: + - master + pull_request: + branches: + - master + +jobs: + source-build: + name: Install, Build & Update + strategy: + fail-fast: false + matrix: + distro: + - 'alpine:3.12' + - 'alpine:3.13' + - 'archlinux:latest' + - 'centos:7' + - 'centos:8' + - 'clearlinux:latest' + - 'debian:9' + - 'debian:10' + - 'fedora:33' + - 'fedora:34' + - 'ubuntu:16.04' + - 'ubuntu:18.04' + - 'ubuntu:20.04' + - 'ubuntu:20.10' + - 'ubuntu:21.04' + include: + - distro: 'alpine:3.12' + pre: 'apk add -U bash' + - distro: 'alpine:3.13' + pre: 'apk add -U bash' + - distro: 'debian:9' + pre: 'apt-get update' + - distro: 'debian:10' + pre: 'apt-get update' + - distro: 'ubuntu:16.04' + pre: 'apt-get update' + - distro: 'ubuntu:18.04' + pre: 'apt-get update' + - distro: 'ubuntu:20.04' + pre: 'apt-get update' + - distro: 'ubuntu:20.10' + pre: 'apt-get update' + - distro: 'ubuntu:21.04' + pre: 'apt-get update' + runs-on: ubuntu-latest + steps: + - name: Git clone repository + uses: actions/checkout@v2 + with: + submodules: recursive + - name: Install required packages & build tarball + run: | + ./packaging/installer/install-required-packages.sh --dont-wait --non-interactive netdata-all + .github/scripts/build-dist.sh + - name: Run a dockerised web server to serve files used by the custom update script + run: | + docker run -dit --name my-apache-app -p 8080:80 -v "$PWD":/usr/local/apache2/htdocs/ httpd:2.4 + - name: Replace URLs in updater script to point at the local web server + run: | + ORIG_TARBALL="export NETDATA_TARBALL_URL=.*" + ORIG_CHECKSUM="export NETDATA_TARBALL_CHECKSUM_URL=.*" + CURRENT_VERSION="current_version=.*" + NEW_TARBALL="export NETDATA_TARBALL_URL=http://localhost:8080/artifacts/netdata-latest.tar.gz" + NEW_CHECKSUM="export NETDATA_TARBALL_CHECKSUM_URL=http://localhost:8080/artifacts/sha256sums.txt" + sed -i "s|${ORIG_TARBALL}|${NEW_TARBALL}|g" packaging/installer/netdata-updater.sh + sed -i "s|${ORIG_CHECKSUM}|${NEW_CHECKSUM}|g" packaging/installer/netdata-updater.sh + sed -i "s|"current_version=.*"|"current_version=1"|g" packaging/installer/netdata-updater.sh + - name: Install netdata and run the updater on ${{ matrix.distro }} + env: + PRE: ${{ matrix.pre }} + run: | + echo $PRE > ./prep-cmd.sh + docker build . -f .github/dockerfiles/Dockerfile.build_test -t test --build-arg BASE=${{ matrix.distro }} + docker run --network host -w /netdata test \ + /bin/sh -c '/netdata/packaging/installer/kickstart.sh --dont-wait \ + && /netdata/packaging/installer/netdata-updater.sh --not-running-from-cron --no-updater-self-update \ + && bash /netdata/.github/scripts/check-updater.sh' -- cgit v1.2.3