Merging upstream version 1.30.0.

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

9 files changed, 301 insertions, 10 deletions

diff --git a/.github/workflows/build-and-install.yml b/.github/workflows/build-and-install.yml
index 26a144acd..4c4693601 100644
--- a/.github/workflows/build-and-install.yml
+++ b/.github/workflows/build-and-install.yml
@@ -12,6 +12,8 @@ jobs:
     steps:
       - name: Git clone repository
         uses: actions/checkout@v2
+        with:
+          submodules: recursive
       - run: |
           git fetch --prune --unshallow --tags
       - name: Build
@@ -34,10 +36,12 @@ jobs:
           - 'clearlinux:latest'
           - 'debian:10'
           - 'debian:9'
+          - 'fedora:34'
           - 'fedora:33'
           - 'fedora:32'
           - 'opensuse/leap:15.2'
           - 'opensuse/tumbleweed:latest'
+          - 'ubuntu:21.04'
           - 'ubuntu:20.10'
           - 'ubuntu:20.04'
           - 'ubuntu:18.04'
@@ -72,6 +76,8 @@ jobs:
             pre: 'apt-get update'
             rmjsonc: 'apt-get remove -y libjson-c-dev'
 
+          - distro: 'fedora:34'
+            rmjsonc: 'dnf remove -y json-c-devel'
           - distro: 'fedora:33'
             rmjsonc: 'dnf remove -y json-c-devel'
           - distro: 'fedora:32'
@@ -82,6 +88,9 @@ jobs:
           - distro: 'opensuse/tumbleweed:latest'
             rmjsonc: 'zypper rm -y libjson-c-devel'
 
+          - distro: 'ubuntu:21.04'
+            pre: 'apt-get update'
+            rmjsonc: 'apt-get remove -y libjson-c-dev'
           - distro: 'ubuntu:20.10'
             pre: 'apt-get update'
             rmjsonc: 'apt-get remove -y libjson-c-dev'
@@ -98,6 +107,8 @@ jobs:
     steps:
       - name: Git clone repository
         uses: actions/checkout@v2
+        with:
+          submodules: recursive
       - name: install-required-packages.sh on ${{ matrix.distro }}
         env:
           PRE: ${{ matrix.pre }}
@@ -183,6 +194,8 @@ jobs:
     steps:
       - name: Git clone repository
         uses: actions/checkout@v2
+        with:
+          submodules: recursive
       - name: install-required-packages.sh on ${{ matrix.distro }}
         env:
           PRE: ${{ matrix.pre }}
diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml
index cf494e95c..7225d3dbe 100644
--- a/.github/workflows/checks.yml
+++ b/.github/workflows/checks.yml
@@ -12,6 +12,8 @@ jobs:
     steps:
       - name: Git clone repository
         uses: actions/checkout@v2
+        with:
+          submodules: recursive
       - name: Run checksum checks on kickstart files
         env:
           LOCAL_ONLY: "true"
@@ -23,6 +25,8 @@ jobs:
     steps:
       - name: Git clone repository
         uses: actions/checkout@v2
+        with:
+          submodules: recursive
       - name: Install required packages
         run: |
           ./packaging/installer/install-required-packages.sh --dont-wait --non-interactive netdata
@@ -43,6 +47,8 @@ jobs:
     steps:
       - name: Checkout
         uses: actions/checkout@v2
+        with:
+          submodules: recursive
       - name: Build
         run: >
           docker run -v "$PWD":/netdata -w /netdata alpine:latest /bin/sh -c
@@ -68,6 +74,8 @@ jobs:
     steps:
       - name: Checkout
         uses: actions/checkout@v2
+        with:
+          submodules: recursive
       - name: Prepare environment
         run: |
           ./packaging/installer/install-required-packages.sh --dont-wait --non-interactive netdata
@@ -98,6 +106,8 @@ jobs:
     steps:
       - name: Checkout
         uses: actions/checkout@v2
+        with:
+          submodules: recursive
       - name: Prepare environment
         run: ./packaging/installer/install-required-packages.sh --dont-wait --non-interactive netdata
       - name: Build netdata
diff --git a/.github/workflows/coverity.yml b/.github/workflows/coverity.yml
index 926257dc0..766275ed9 100644
--- a/.github/workflows/coverity.yml
+++ b/.github/workflows/coverity.yml
@@ -15,6 +15,8 @@ jobs:
     steps:
       - name: Checkout
         uses: actions/checkout@v2
+        with:
+          submodules: recursive
       - name: Prepare environment
         env:
           DEBIAN_FRONTEND: 'noninteractive'
@@ -32,3 +34,17 @@ jobs:
           COVERITY_SCAN_SUBMIT_MAIL: ${{ secrets.COVERITY_SCAN_SUBMIT_MAIL }}
         run: |
           ./coverity-scan.sh --with-install
+      - name: Failure Notification
+        uses: rtCamp/action-slack-notify@v2
+        env:
+          SLACK_COLOR: 'danger'
+          SLACK_FOOTER:
+          SLACK_ICON_EMOJI: ':github-actions:'
+          SLACK_TITLE: 'Coverity run failed:'
+          SLACK_USERNAME: 'GitHub Actions'
+          SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }}
+        if: ${{
+            failure()
+            && github.event_name != 'pull_request'
+            && startsWith(github.ref, 'refs/heads/master')
+          }}
diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml
index 04f91bff5..a5648d2a9 100644
--- a/.github/workflows/docker.yml
+++ b/.github/workflows/docker.yml
@@ -4,15 +4,7 @@ on:
   push:
     branches:
       - master
-    paths:
-      - '.github/workflows/docker.yml'
-      - 'netdata-installer.sh'
-      - 'packaging/**'
   pull_request:
-    paths:
-      - '.github/workflows/docker.yml'
-      - 'netdata-installer.sh'
-      - 'packaging/**'
   workflow_dispatch:
     inputs:
       version:
@@ -26,11 +18,13 @@ jobs:
     steps:
       - name: Checkout
         uses: actions/checkout@v2
+        with:
+          submodules: recursive
       - name: Determine if we should push changes and which tags to use
         if: github.event_name == 'workflow_dispatch' && github.event.inputs.version != 'nightly'
         run: |
           echo "publish=true" >> $GITHUB_ENV
-          echo "tags=netdata/netdata:latest,netdata/netdata:stable,netdata/netdata:${{ github.event.inputs.version }}" >> $GITHUB_ENV
+          echo "tags=netdata/netdata:latest,netdata/netdata:stable,$(.github/scripts/gen-docker-tags.py ${{ github.event.inputs.version }})" >> $GITHUB_ENV
       - name: Determine if we should push changes and which tags to use
         if: github.event_name == 'workflow_dispatch' && github.event.inputs.version == 'nightly'
         run: |
@@ -57,3 +51,19 @@ jobs:
           platforms: linux/amd64,linux/i386,linux/arm/v7,linux/arm64
           push: ${{ env.publish }}
           tags: ${{ env.tags }}
+      - name: Failure Notification
+        uses: rtCamp/action-slack-notify@v2
+        env:
+          SLACK_COLOR: 'danger'
+          SLACK_FOOTER:
+          SLACK_ICON_EMOJI: ':github-actions:'
+          SLACK_TITLE: 'Docker Build failed:'
+          SLACK_USERNAME: 'GitHub Actions'
+          SLACK_MESSAGE: "Docker image build failed."
+          SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }}
+        if: >-
+          ${{
+            failure()
+            && github.event_name != 'pull_request'
+            && startsWith(github.ref, 'refs/heads/master')
+          }}
diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml
index 2a4fe87e4..9f7234f92 100644
--- a/.github/workflows/docs.yml
+++ b/.github/workflows/docs.yml
@@ -16,6 +16,8 @@ jobs:
     steps:
       - name: Checkout
         uses: actions/checkout@v2
+        with:
+          submodules: recursive
       - name: Run link check
         uses: gaurav-nelson/github-action-markdown-link-check@v1
         with:
diff --git a/.github/workflows/packaging.yml b/.github/workflows/packaging.yml
new file mode 100644
index 000000000..7340c27bb
--- /dev/null
+++ b/.github/workflows/packaging.yml
@@ -0,0 +1,149 @@
+---
+# Handles building of binary packages for the agent.
+name: Packages
+on:
+  pull_request:
+    branches:
+      - master
+      - develop
+  workflow_dispatch:
+    inputs:
+      type:
+        name: Package build type
+        default: devel
+        required: true
+      version:
+        name: Package version
+        required: false
+jobs:
+  build:
+    name: Build
+    runs-on: ubuntu-latest
+    env:
+      DOCKER_CLI_EXPERIMENTAL: enabled
+    strategy:
+      matrix:
+        include:
+          - {distro: debian, version: "9", pkgclouddistro: debian/stretch, format: deb, base_image: debian, platform: linux/amd64, arch: amd64}
+          - {distro: debian, version: "9", pkgclouddistro: debian/stretch, format: deb, base_image: debian, platform: linux/i386, arch: i386}
+          - {distro: debian, version: "10", pkgclouddistro: debian/buster, format: deb, base_image: debian, platform: linux/amd64, arch: amd64}
+          - {distro: debian, version: "10", pkgclouddistro: debian/buster, format: deb, base_image: debian, platform: linux/i386, arch: i386}
+          - {distro: ubuntu, version: "16.04", pkgclouddistro: ubuntu/xenial, format: deb, base_image: ubuntu, platform: linux/amd64, arch: amd64}
+          - {distro: ubuntu, version: "16.04", pkgclouddistro: ubuntu/xenial, format: deb, base_image: ubuntu, platform: linux/i386, arch: i386}
+          - {distro: ubuntu, version: "18.04", pkgclouddistro: ubuntu/bionic, format: deb, base_image: ubuntu, platform: linux/amd64, arch: amd64}
+          - {distro: ubuntu, version: "18.04", pkgclouddistro: ubuntu/bionic, format: deb, base_image: ubuntu, platform: linux/i386, arch: i386}
+          - {distro: ubuntu, version: "20.04", pkgclouddistro: ubuntu/focal, format: deb, base_image: ubuntu, platform: linux/amd64, arch: amd64}
+          - {distro: ubuntu, version: "20.10", pkgclouddistro: ubuntu/groovy, format: deb, base_image: ubuntu, platform: linux/amd64, arch: amd64}
+          - {distro: ubuntu, version: "21.04", pkgclouddistro: ubuntu/hirsute, format: deb, base_image: ubuntu, platform: linux/amd64, arch: amd64}
+          - {distro: centos, version: "7", pkgclouddistro: el/7, format: rpm, base_image: centos, platform: linux/amd64, arch: amd64}
+          - {distro: centos, version: "8", pkgclouddistro: el/8, format: rpm, base_image: centos, platform: linux/amd64, arch: amd64}
+          - {distro: fedora, version: "32", pkgclouddistro: fedora/32, format: rpm, base_image: fedora, platform: linux/amd64, arch: amd64}
+          - {distro: fedora, version: "33", pkgclouddistro: fedora/33, format: rpm, base_image: fedora, platform: linux/amd64, arch: amd64}
+          - {distro: fedora, version: "34", pkgclouddistro: fedora/34, format: rpm, base_image: fedora, platform: linux/amd64, arch: amd64}
+          - {distro: opensuse, version: "15.2", pkgclouddistro: opensuse/15.2, format: rpm, base_image: opensuse/leap, platform: linux/amd64, arch: amd64}
+      # We intentiaonally disable the fail-fast behavior so that a
+      # build failure for one version doesn't prevent us from publishing
+      # successfully built and tested packages for another version.
+      fail-fast: false
+    steps:
+      - name: Checkout PR # Checkout the PR if it's a PR.
+        if: github.event_name == 'pull_request'
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0 # We need full history for versioning
+          submodules: true
+      - name: Checkout Tag # Otherwise check out the tag that triggered this.
+        if: github.event_name == 'workflow_dispatch'
+        uses: actions/checkout@v2
+        with:
+          ref: ${{ github.event.ref }}
+          fetch-depth: 0 # We need full history for versioning
+          submodules: true
+      - name: Check Base Branch
+        run: |
+          if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then
+            echo "runtype=${{ github.event.inputs.type }}" >> $GITHUB_ENV
+            case "${{ github.event.inputs.type }}" in
+              "release")
+                echo "repo=${{ secrets.PACKAGE_CLOUD_REPO }}" >> $GITHUB_ENV
+                echo "pkg_version=${{ github.event.inputs.version }}" >> $GITHUB_ENV
+                echo "pkg_retention_days=365" >> $GITHUB_ENV
+                ;;
+              "nightly")
+                echo "repo=${{ secrets.PACKAGE_CLOUD_REPO }}-edge" >> $GITHUB_ENV
+                echo "pkg_version=${{ github.event.inputs.version }}" >> $GITHUB_ENV
+                echo "pkg_retention_days=30" >> $GITHUB_ENV
+                ;;
+              *)
+                echo "repo=${{ secrets.PACKAGE_CLOUD_REPO }}-devel" >> $GITHUB_ENV
+                echo "pkg_version=0.${GITHUB_SHA}" >> $GITHUB_ENV
+                echo "pkg_retention_days=30" >> $GITHUB_ENV
+                ;;
+            esac
+          else
+            echo "runtype=test" >> $GITHUB_ENV
+            echo "pkg_version=$(cut -d'-' -f 1 packaging/version | sed -e 's/^v//')" >> $GITHUB_ENV
+          fi
+      - name: Setup QEMU
+        if: matrix.platform != 'linux/amd64'
+        uses: docker/setup-qemu-action@v1
+      - name: Setup Buildx
+        uses: docker/setup-buildx-action@v1
+      - name: Prepare Docker Environment
+        shell: bash
+        run: |
+          echo '{"cgroup-parent": "/actions_job", "experimental": true}' | sudo tee /etc/docker/daemon.json 2>/dev/null
+          sudo service docker restart
+      - name: Build Packages
+        uses: docker/build-push-action@v2
+        with:
+          platforms: ${{ matrix.platform }}
+          file: packaging/Dockerfile.packager
+          tags: local/package-builder:${{ matrix.distro}}${{ matrix.version }}
+          push: false
+          load: true
+          build-args: |
+            ARCH=${{ matrix.arch }}
+            DISTRO=${{ matrix.distro }}
+            TEST_BASE=${{ matrix.base_image }}
+            DISTRO_VERSION=${{ matrix.version }}
+            PKG_VERSION=${{ env.pkg_version }}
+      - name: Extract Packages
+        shell: bash
+        run: |
+          mkdir -p artifacts
+          docker run --platform ${{ matrix.platform }} -v $PWD/artifacts:/artifacts local/package-builder:${{ matrix.distro }}${{ matrix.version }}
+      - name: Upload
+        if: github.event_name == 'workflow_dispatch'
+        shell: bash
+        env:
+          PKG_CLOUD_TOKEN: ${{ secrets.PACKAGE_CLOUD_API_KEY }}
+        run: |
+          echo "Packages to upload:\n$(ls artifacts/*.${{ matrix.format }})"
+          for pkgfile in artifacts/*.${{ matrix.format }} ; do
+            .github/scripts/package_cloud_wrapper.sh push ${{ env.repo }}/${{ matrix.pkgclouddistro }} ${pkgfile}
+          done
+      - name: Clean
+        if: github.event_name == 'workflow_dispatch'
+        shell: bash
+        env:
+          REPO: ${{ env.repo }}
+          PKG_CLOUD_TOKEN: ${{ secrets.PACKAGE_CLOUD_API_KEY }}
+          PACKAGE_CLOUD_RETENTION_DAYS: ${{ env.pkg_retention_days }}
+        run: .github/scripts/old_package_purging.sh
+      - name: Failure Notification
+        uses: rtCamp/action-slack-notify@v2
+        env:
+          SLACK_COLOR: 'danger'
+          SLACK_FOOTER:
+          SLACK_ICON_EMOJI: ':github-actions:'
+          SLACK_TITLE: 'Package Build failed:'
+          SLACK_USERNAME: 'GitHub Actions'
+          SLACK_MESSAGE: "${{ matrix.pkgclouddistro }} ${{ matrix.version }} package build for ${{ matrix.arch }} failed."
+          SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }}
+        if: >-
+          ${{
+            failure()
+            && github.event_name != 'pull_request'
+            && startsWith(github.ref, 'refs/heads/master')
+          }}
diff --git a/.github/workflows/review.yml b/.github/workflows/review.yml
index ca8f6de13..a267fea3f 100644
--- a/.github/workflows/review.yml
+++ b/.github/workflows/review.yml
@@ -16,6 +16,7 @@ jobs:
       - name: Git clone repository
         uses: actions/checkout@v2
         with:
+          submodules: recursive
           fetch-depth: 0
       - name: Check files
         run: |
@@ -57,6 +58,7 @@ jobs:
       - name: Git clone repository
         uses: actions/checkout@v2
         with:
+          submodules: recursive
           fetch-depth: 0
       - name: Check files
         run: |
@@ -80,6 +82,7 @@ jobs:
       - name: Git clone repository
         uses: actions/checkout@v2
         with:
+          submodules: recursive
           fetch-depth: 0
       - name: Check files
         run: |
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index ef6bfbc2a..c166c7442 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -21,6 +21,8 @@ jobs:
     steps:
       - name: Checkout
         uses: actions/checkout@v2
+        with:
+          submodules: recursive
       - name: Prepare environment
         run: |
           ./packaging/installer/install-required-packages.sh --dont-wait --non-interactive netdata-all
@@ -39,6 +41,8 @@ jobs:
     steps:
       - name: Checkout
         uses: actions/checkout@v2
+        with:
+          submodules: recursive
       - name: Prepare environment
         run: |
           ./packaging/installer/install-required-packages.sh --dont-wait --non-interactive netdata-all
@@ -48,7 +52,7 @@ jobs:
       - name: Configure
         run: |
           autoreconf -ivf
-          ./configure
+          ./configure --without-aclk-ng
       # XXX: Work-around for bug with libbson-1.0 in Ubuntu 18.04
       # See: https://bugs.launchpad.net/ubuntu/+source/libmongoc/+bug/1790771
       #      https://jira.mongodb.org/browse/CDRIVER-2818
diff --git a/.github/workflows/updater.yml b/.github/workflows/updater.yml
new file mode 100644
index 000000000..76cb2fdbe
--- /dev/null
+++ b/.github/workflows/updater.yml
@@ -0,0 +1,84 @@
+---
+name: Updater
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+    branches:
+      - master
+
+jobs:
+  source-build:
+    name: Install, Build & Update
+    strategy:
+      fail-fast: false
+      matrix:
+        distro:
+          - 'alpine:3.12'
+          - 'alpine:3.13'
+          - 'archlinux:latest'
+          - 'centos:7'
+          - 'centos:8'
+          - 'clearlinux:latest'
+          - 'debian:9'
+          - 'debian:10'
+          - 'fedora:33'
+          - 'fedora:34'
+          - 'ubuntu:16.04'
+          - 'ubuntu:18.04'
+          - 'ubuntu:20.04'
+          - 'ubuntu:20.10'
+          - 'ubuntu:21.04'
+        include:
+          - distro: 'alpine:3.12'
+            pre: 'apk add -U bash'
+          - distro: 'alpine:3.13'
+            pre: 'apk add -U bash'
+          - distro: 'debian:9'
+            pre: 'apt-get update'
+          - distro: 'debian:10'
+            pre: 'apt-get update'
+          - distro: 'ubuntu:16.04'
+            pre: 'apt-get update'
+          - distro: 'ubuntu:18.04'
+            pre: 'apt-get update'
+          - distro: 'ubuntu:20.04'
+            pre: 'apt-get update'
+          - distro: 'ubuntu:20.10'
+            pre: 'apt-get update'
+          - distro: 'ubuntu:21.04'
+            pre: 'apt-get update'
+    runs-on: ubuntu-latest
+    steps:
+      - name: Git clone repository
+        uses: actions/checkout@v2
+        with:
+          submodules: recursive
+      - name: Install required packages & build tarball
+        run: |
+          ./packaging/installer/install-required-packages.sh --dont-wait --non-interactive netdata-all
+          .github/scripts/build-dist.sh
+      - name: Run a dockerised web server to serve files used by the custom update script
+        run: |
+          docker run -dit --name my-apache-app -p 8080:80 -v "$PWD":/usr/local/apache2/htdocs/ httpd:2.4
+      - name: Replace URLs in updater script to point at the local web server
+        run: |
+          ORIG_TARBALL="export NETDATA_TARBALL_URL=.*"
+          ORIG_CHECKSUM="export NETDATA_TARBALL_CHECKSUM_URL=.*"
+          CURRENT_VERSION="current_version=.*"
+          NEW_TARBALL="export NETDATA_TARBALL_URL=http://localhost:8080/artifacts/netdata-latest.tar.gz"
+          NEW_CHECKSUM="export NETDATA_TARBALL_CHECKSUM_URL=http://localhost:8080/artifacts/sha256sums.txt"
+          sed -i "s|${ORIG_TARBALL}|${NEW_TARBALL}|g" packaging/installer/netdata-updater.sh
+          sed -i "s|${ORIG_CHECKSUM}|${NEW_CHECKSUM}|g" packaging/installer/netdata-updater.sh
+          sed -i "s|"current_version=.*"|"current_version=1"|g" packaging/installer/netdata-updater.sh
+      - name: Install netdata and run the updater on ${{ matrix.distro }}
+        env:
+          PRE: ${{ matrix.pre }}
+        run: |
+          echo $PRE > ./prep-cmd.sh
+          docker build . -f .github/dockerfiles/Dockerfile.build_test -t test --build-arg BASE=${{ matrix.distro }}
+          docker run --network host -w /netdata test \
+              /bin/sh -c '/netdata/packaging/installer/kickstart.sh --dont-wait \
+              && /netdata/packaging/installer/netdata-updater.sh --not-running-from-cron --no-updater-self-update \
+              && bash /netdata/.github/scripts/check-updater.sh'