diff options
Diffstat (limited to '.github/workflows')
| -rw-r--r-- | .github/workflows/build-and-install.yml | 215 | ||||
| -rw-r--r-- | .github/workflows/build.yml | 538 | ||||
| -rw-r--r-- | .github/workflows/checks.yml | 41 | ||||
| -rw-r--r-- | .github/workflows/coverity.yml | 7 | ||||
| -rw-r--r-- | .github/workflows/dashboard-pr.yml | 35 | ||||
| -rw-r--r-- | .github/workflows/docker.yml | 88 | ||||
| -rw-r--r-- | .github/workflows/docs.yml | 2 | ||||
| -rw-r--r-- | .github/workflows/labeler.yml | 4 | ||||
| -rw-r--r-- | .github/workflows/packaging.yml | 95 | ||||
| -rw-r--r-- | .github/workflows/repoconfig-packages.yml | 80 | ||||
| -rw-r--r-- | .github/workflows/review.yml | 120 | ||||
| -rw-r--r-- | .github/workflows/tests.yml | 9 | ||||
| -rw-r--r-- | .github/workflows/updater.yml | 85 |
13 files changed, 884 insertions, 435 deletions
diff --git a/.github/workflows/build-and-install.yml b/.github/workflows/build-and-install.yml deleted file mode 100644 index 251a65162..000000000 --- a/.github/workflows/build-and-install.yml +++ /dev/null @@ -1,215 +0,0 @@ ---- -name: Builder -on: - push: - branches: - - master - pull_request: -jobs: - static-build: - name: Build (x86_64) - runs-on: ubuntu-latest - steps: - - name: Git clone repository - uses: actions/checkout@v2 - with: - submodules: recursive - - run: | - git fetch --prune --unshallow --tags - - name: Build - run: | - .github/scripts/build-static-x86_64.sh - source-build: - name: Build & Install - strategy: - fail-fast: false - max-parallel: 8 - matrix: - distro: - - 'alpine:edge' - - 'alpine:3.13' - - 'alpine:3.12' - - 'alpine:3.11' - - 'alpine:3.10' - - 'archlinux:latest' - - 'centos:8' - - 'centos:7' - - 'clearlinux:latest' - - 'debian:10' - - 'debian:9' - - 'fedora:34' - - 'fedora:33' - - 'fedora:32' - - 'opensuse/leap:15.2' - - 'opensuse/tumbleweed:latest' - - 'ubuntu:21.04' - - 'ubuntu:20.10' - - 'ubuntu:20.04' - - 'ubuntu:18.04' - - 'ubuntu:16.04' - include: - - distro: 'alpine:edge' - pre: 'apk add -U bash' - rmjsonc: 'apk del json-c-dev' - - distro: 'alpine:3.13' - pre: 'apk add -U bash' - rmjsonc: 'apk del json-c-dev' - - distro: 'alpine:3.12' - pre: 'apk add -U bash' - rmjsonc: 'apk del json-c-dev' - - distro: 'alpine:3.11' - pre: 'apk add -U bash' - rmjsonc: 'apk del json-c-dev' - - distro: 'alpine:3.10' - pre: 'apk add -U bash' - rmjsonc: 'apk del json-c-dev' - - - distro: 'archlinux:latest' - pre: 'pacman --noconfirm -Syu && pacman --noconfirm -Sy grep libffi' - - - distro: 'centos:8' - rmjsonc: 'dnf remove -y json-c-devel' - - - distro: 'debian:10' - pre: 'apt-get update' - rmjsonc: 'apt-get remove -y libjson-c-dev' - - distro: 'debian:9' - pre: 'apt-get update' - rmjsonc: 'apt-get remove -y libjson-c-dev' - - - distro: 'fedora:34' - rmjsonc: 'dnf remove -y json-c-devel' - - distro: 'fedora:33' - rmjsonc: 'dnf remove -y json-c-devel' - - distro: 'fedora:32' - rmjsonc: 'dnf remove -y json-c-devel' - - - distro: 'opensuse/leap:15.2' - rmjsonc: 'zypper rm -y libjson-c-devel' - - distro: 'opensuse/tumbleweed:latest' - rmjsonc: 'zypper rm -y libjson-c-devel' - - - distro: 'ubuntu:21.04' - pre: 'apt-get update' - rmjsonc: 'apt-get remove -y libjson-c-dev' - - distro: 'ubuntu:20.10' - pre: 'apt-get update' - rmjsonc: 'apt-get remove -y libjson-c-dev' - - distro: 'ubuntu:20.04' - pre: 'apt-get update' - rmjsonc: 'apt-get remove -y libjson-c-dev' - - distro: 'ubuntu:18.04' - pre: 'apt-get update' - rmjsonc: 'apt-get remove -y libjson-c-dev' - - distro: 'ubuntu:16.04' - pre: 'apt-get update' - rmjsonc: 'apt-get remove -y libjson-c-dev' - runs-on: ubuntu-latest - steps: - - name: Git clone repository - uses: actions/checkout@v2 - with: - submodules: recursive - - name: install-required-packages.sh on ${{ matrix.distro }} - env: - PRE: ${{ matrix.pre }} - RMJSONC: ${{ matrix.rmjsonc }} - run: | - echo $PRE > ./prep-cmd.sh - echo $RMJSONC > ./rmjsonc.sh && chmod +x ./rmjsonc.sh - docker build . -f .github/dockerfiles/Dockerfile.build_test -t test --build-arg BASE=${{ matrix.distro }} - - name: Regular build on ${{ matrix.distro }} - run: | - docker run -w /netdata test /bin/sh -c 'autoreconf -ivf && ./configure && make -j2' - - name: netdata-installer on ${{ matrix.distro }}, disable cloud - run: | - docker run -w /netdata test /bin/sh -c './netdata-installer.sh --dont-wait --dont-start-it --disable-cloud' - - name: netdata-installer on ${{ matrix.distro }}, require cloud - run: | - docker run -w /netdata test /bin/sh -c './netdata-installer.sh --dont-wait --dont-start-it --require-cloud' - - name: netdata-installer on ${{ matrix.distro }}, require cloud, no JSON-C - if: matrix.rmjsonc != '' - run: | - docker run -w /netdata test \ - /bin/sh -c '/netdata/rmjsonc.sh && ./netdata-installer.sh --dont-wait --dont-start-it --require-cloud' - aws-kinesis-build: - name: With AWS Kinesis SDK - strategy: - fail-fast: false - matrix: - distro: - - 'centos:8' - - 'debian:buster' - - 'fedora:32' - - 'ubuntu:20.04' - include: - - distro: 'centos:8' - pre: >- - yum -y update && - yum -y groupinstall 'Development Tools' && - yum -y install libcurl-devel openssl-devel libuuid-devel - build_kinesis: >- - git clone --branch 1.8.186 --depth 1 https://github.com/aws/aws-sdk-cpp.git && - cmake -DCMAKE_INSTALL_PREFIX=/usr - -DBUILD_ONLY=kinesis - ./aws-sdk-cpp && - make && - make install - - distro: 'debian:buster' - pre: >- - apt-get update && - DEBIAN_FRONTEND=noninteractive apt-get install -y build-essential && - DEBIAN_FRONTEND=noninteractive apt-get install -y libcurl4-openssl-dev libssl-dev uuid-dev zlib1g-dev libpulse-dev - build_kinesis: >- - git clone --branch 1.8.186 --depth 1 https://github.com/aws/aws-sdk-cpp.git && - cmake -DCMAKE_INSTALL_PREFIX=/usr - -DBUILD_ONLY=kinesis - ./aws-sdk-cpp && - make && - make install - - distro: 'fedora:32' - pre: >- - dnf -y update && - dnf -y groupinstall 'Development Tools' && - dnf -y install libcurl-devel openssl-devel libuuid-devel - build_kinesis: >- - git clone --branch 1.8.186 --depth 1 https://github.com/aws/aws-sdk-cpp.git && - cmake -DCMAKE_INSTALL_PREFIX=/usr - -DBUILD_ONLY=kinesis - ./aws-sdk-cpp && - make && - make install - - distro: 'ubuntu:20.04' - pre: >- - apt-get update && - DEBIAN_FRONTEND=noninteractive apt-get install -y build-essential && - DEBIAN_FRONTEND=noninteractive apt-get install -y libcurl4-openssl-dev libssl-dev uuid-dev zlib1g-dev libpulse-dev - build_kinesis: >- - git clone --branch 1.8.186 --depth 1 https://github.com/aws/aws-sdk-cpp.git && - cmake -DCMAKE_INSTALL_PREFIX=/usr - -DBUILD_ONLY=kinesis - ./aws-sdk-cpp && - make && - make install - runs-on: ubuntu-latest - steps: - - name: Git clone repository - uses: actions/checkout@v2 - with: - submodules: recursive - - name: install-required-packages.sh on ${{ matrix.distro }} - env: - PRE: ${{ matrix.pre }} - BUILD_KINESIS: ${{ matrix.build_kinesis }} - run: | - echo $PRE > ./prep-cmd.sh - echo $BUILD_KINESIS > ./build-kinesis.sh && chmod +x ./build-kinesis.sh - docker build . -f .github/dockerfiles/Dockerfile.build_test -t test --build-arg BASE=${{ matrix.distro }} - - name: Build on ${{ matrix.distro }} - env: - RUNCMD: >- - ./build-kinesis.sh && - ./netdata-installer.sh --dont-wait --dont-start-it --enable-backend-kinesis - run: | - docker run -w /netdata test \ - /bin/sh -c "$RUNCMD" diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 000000000..2732a1851 --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,538 @@ +--- +# Ci code for building release artifacts. +name: Build +on: + push: # Master branch checks only validate the build and generate artifacts for testing. + branches: + - master + pull_request: null # PR checks only validate the build and generate artifacts for testing. + workflow_dispatch: # Dispatch runs build and validate, then push to the appropriate storage location. + inputs: + type: + description: Build Type + default: nightly + required: true + version: + description: Version Tag + default: nightly + required: true +concurrency: # This keeps multiple instances of the job from running concurrently for the same ref and event type. + group: release-${{ github.ref }}-${{ github.event_name }} + cancel-in-progress: true +jobs: + build-dist: # Build the distribution tarball and store it as an artifact. + name: Build Distribution Tarball + runs-on: ubuntu-latest + outputs: + distfile: ${{ steps.build.outputs.distfile }} + steps: + - name: Checkout + uses: actions/checkout@v2 + with: + fetch-depth: 0 + submodules: recursive + - name: Mark Stable + if: github.event_name == 'workflow_dispatch' && github.event.inputs.type != 'nightly' + run: | + sed -i 's/^RELEASE_CHANNEL="nightly" *#/RELEASE_CHANNEL="stable" #/' netdata-installer.sh + - name: Build + id: build + run: | + mkdir -p artifacts + ./packaging/installer/install-required-packages.sh --dont-wait --non-interactive netdata + autoreconf -ivf + ./configure --prefix=/usr \ + --sysconfdir=/etc \ + --localstatedir=/var \ + --libexecdir=/usr/libexec \ + --with-zlib \ + --with-math \ + --with-user=netdata + make dist + echo "::set-output name=distfile::$(find . -name 'netdata-*.tar.gz')" + cp netdata-*.tar.gz artifacts/ + - name: Store + uses: actions/upload-artifact@v2 + with: + name: dist-tarball + path: artifacts/*.tar.gz + retention-days: 30 + - name: Failure Notification + uses: rtCamp/action-slack-notify@v2 + env: + SLACK_COLOR: 'danger' + SLACK_FOOTER: '' + SLACK_ICON_EMOJI: ':github-actions:' + SLACK_TITLE: 'Distribution tarball build failed:' + SLACK_USERNAME: 'GitHub Actions' + SLACK_MESSAGE: "Distribution tarball build failed." + SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }} + if: >- + ${{ + failure() + && startsWith(github.ref, 'refs/heads/master') + && github.event_name != 'pull_request' + }} + + build-static: # Build the static binary archives, and store them as artifacts. + name: Build Static + runs-on: ubuntu-latest + strategy: + matrix: + arch: + - 'x86_64' + - 'armv7l' + - 'aarch64' + - 'ppc64le' + steps: + - name: Checkout + uses: actions/checkout@v2 + with: + fetch-depth: 0 + submodules: recursive + - name: Mark Stable + if: github.event_name == 'workflow_dispatch' && github.event.inputs.type != 'nightly' + run: | + sed -i 's/^RELEASE_CHANNEL="nightly" *#/RELEASE_CHANNEL="stable" #/' netdata-installer.sh packaging/makeself/install-or-update.sh + - name: Build + run: .github/scripts/build-static.sh ${{ matrix.arch }} + - name: Store + uses: actions/upload-artifact@v2 + with: + name: static-archive + path: artifacts/*.gz.run + retention-days: 30 + - name: Failure Notification + uses: rtCamp/action-slack-notify@v2 + env: + SLACK_COLOR: 'danger' + SLACK_FOOTER: '' + SLACK_ICON_EMOJI: ':github-actions:' + SLACK_TITLE: 'Static build for ${{ matrix.arch }} failed:' + SLACK_USERNAME: 'GitHub Actions' + SLACK_MESSAGE: "Static build for ${{ matrix.arch }} failed." + SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }} + if: >- + ${{ + failure() + && startsWith(github.ref, 'refs/heads/master') + && github.event_name != 'pull_request' + }} + + matrix: # Generate the shared build matrix for our build tests. + name: Prepare Build Matrix + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.set-matrix.outputs.matrix }} + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Prepare tools + run: | + sudo apt-get update && sudo apt-get install -y jq + - name: Read build matrix + id: set-matrix + run: | + TASKS="$(jq -c . .github/data/build-matrix.json)" + echo "Generated Matrix: $TASKS" + echo "::set-output name=matrix::$TASKS" + + prepare-test-images: # Prepare the test environments for our build checks. This also checks dependency handling code for each tested environment. + name: Prepare Test Environments + runs-on: ubuntu-latest + needs: + - matrix + strategy: + # Unlike the actal build tests, this completes _very_ fast (average of about 3 minutes for each job), so we + # just run everything in parallel instead lof limiting job concurrency. + fail-fast: false + matrix: ${{ fromJson(needs.matrix.outputs.matrix) }} + steps: + - name: Git clone repository + uses: actions/checkout@v2 + - name: Setup Buildx + uses: docker/setup-buildx-action@v1 + - name: Build test environment + uses: docker/build-push-action@v2 + with: + push: false + load: false + file: .github/dockerfiles/Dockerfile.build_test + build-args: | + BASE=${{ matrix.distro }} + PRE=${{ matrix.pre }} + RMJSONC=${{ matrix.rmjsonc }} + outputs: type=oci,dest=/tmp/image.tar + tags: test:${{ matrix.artifact_key }} + - name: Upload image artifact + uses: actions/upload-artifact@v2 + with: + name: ${{ matrix.artifact_key }}-test-env + path: /tmp/image.tar + retention-days: 30 + - name: Failure Notification + uses: rtCamp/action-slack-notify@v2 + env: + SLACK_COLOR: 'danger' + SLACK_FOOTER: '' + SLACK_ICON_EMOJI: ':github-actions:' + SLACK_TITLE: 'Test environment preparation for ${{ matrix.distro }} failed:' + SLACK_USERNAME: 'GitHub Actions' + SLACK_MESSAGE: "Test environment preparation for ${{ matrix.distro }} failed." + SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }} + if: >- + ${{ + failure() + && startsWith(github.ref, 'refs/heads/master') + && github.event_name != 'pull_request' + }} + + source-build: # Test various source build arrangements. + name: Test Source Build + runs-on: ubuntu-latest + needs: + - matrix + - prepare-test-images + strategy: + fail-fast: false + max-parallel: 8 + matrix: ${{ fromJson(needs.matrix.outputs.matrix) }} + steps: + - name: Git clone repository + uses: actions/checkout@v2 + with: + submodules: recursive + - name: Fetch test environment + uses: actions/download-artifact@v2 + with: + name: ${{ matrix.artifact_key }}-test-env + - name: Load test environment + id: load + run: | + docker load --input image.tar | tee image-info.txt + echo "::set-output name=image::$(cut -d ':' -f 3 image-info.txt)" + - name: Regular build on ${{ matrix.distro }} + run: | + docker run --security-opt seccomp=unconfined -w /netdata sha256:${{ steps.load.outputs.image }} \ + /bin/sh -c 'autoreconf -ivf && ./configure && make -j2' + - name: netdata-installer on ${{ matrix.distro }}, disable cloud + run: | + docker run --security-opt seccomp=unconfined -w /netdata sha256:${{ steps.load.outputs.image }} \ + /bin/sh -c './netdata-installer.sh --dont-wait --dont-start-it --disable-cloud' + - name: netdata-installer on ${{ matrix.distro }}, require cloud + run: | + docker run --security-opt seccomp=unconfined -w /netdata sha256:${{ steps.load.outputs.image }} \ + /bin/sh -c './netdata-installer.sh --dont-wait --dont-start-it --require-cloud' + - name: netdata-installer on ${{ matrix.distro }}, require cloud, require ACLK-NG + run: | + docker run --security-opt seccomp=unconfined -w /netdata -e NETDATA_CONFIGURE_OPTIONS='--with-aclk-ng' \ + sha256:${{ steps.load.outputs.image }} /bin/sh -c './netdata-installer.sh --dont-wait --dont-start-it --require-cloud' + - name: netdata-installer on ${{ matrix.distro }}, require cloud, no JSON-C + if: matrix.rmjsonc != '' + run: | + docker run --security-opt seccomp=unconfined -w /netdata sha256:${{ steps.load.outputs.image }} \ + /bin/sh -c '/rmjsonc.sh && ./netdata-installer.sh --dont-wait --dont-start-it --require-cloud' + - name: Failure Notification + uses: rtCamp/action-slack-notify@v2 + env: + SLACK_COLOR: 'danger' + SLACK_FOOTER: '' + SLACK_ICON_EMOJI: ':github-actions:' + SLACK_TITLE: 'Build tests for ${{ matrix.distro }} failed:' + SLACK_USERNAME: 'GitHub Actions' + SLACK_MESSAGE: "Build tests for ${{ matrix.distro }} failed." + SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }} + if: >- + ${{ + failure() + && startsWith(github.ref, 'refs/heads/master') + && github.event_name != 'pull_request' + }} + + updater-check: # Test the generated dist archive using the updater code. + name: Test Generated Distfile and Updater Code + runs-on: ubuntu-latest + needs: + - build-dist + - matrix + - prepare-test-images + strategy: + fail-fast: false + max-parallel: 8 + matrix: ${{ fromJson(needs.matrix.outputs.matrix) }} + services: + apache: # This gets used to serve the dist tarball for the updater script. + image: httpd:2.4 + ports: + - 8080:80 + volumes: + - ${{ github.workspace }}:/usr/local/apache2/htdocs/ + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Fetch dist tarball artifacts + uses: actions/download-artifact@v2 + with: + name: dist-tarball + path: dist-tarball + - name: Prepare artifact directory + run: | + mkdir -p artifacts || exit 1 + echo "9999.0.0-0" > artifacts/latest-version.txt || exit 1 + cp dist-tarball/* artifacts || exit 1 + cd artifacts || exit 1 + ln -s ${{ needs.build-dist.outputs.distfile }} netdata-latest.tar.gz || exit 1 + sha256sum -b ./* > "sha256sums.txt" || exit 1 + cat sha256sums.txt + - name: Fetch test environment + uses: actions/download-artifact@v2 + with: + name: ${{ matrix.artifact_key }}-test-env + - name: Load test environment + id: load + run: | + docker load --input image.tar | tee image-info.txt + echo "::set-output name=image::$(cut -d ':' -f 3 image-info.txt)" + - name: Install netdata and run the updater on ${{ matrix.distro }} + run: | + docker run --security-opt seccomp=unconfined -e DO_NOT_TRACK=1 --network host -w /netdata sha256:${{ steps.load.outputs.image }} \ + /netdata/.github/scripts/run-updater-check.sh + - name: Failure Notification + uses: rtCamp/action-slack-notify@v2 + env: + SLACK_COLOR: 'danger' + SLACK_FOOTER: '' + SLACK_ICON_EMOJI: ':github-actions:' + SLACK_TITLE: 'Updater checks for ${{ matrix.distro }} failed:' + SLACK_USERNAME: 'GitHub Actions' + SLACK_MESSAGE: "Updater checks for ${{ matrix.distro }} failed." + SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }} + if: >- + ${{ + failure() + && startsWith(github.ref, 'refs/heads/master') + && github.event_name != 'pull_request' + }} + + prepare-upload: # Consolidate the artifacts for uploading or releasing. + name: Prepare Artifacts + runs-on: ubuntu-latest + needs: + - build-dist + - build-static + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Prepare Environment + run: mkdir -p artifacts + - name: Retrieve Dist Tarball + uses: actions/download-artifact@v2 + with: + name: dist-tarball + path: dist-tarball + - name: Retrieve Static Build Artifacts + uses: actions/download-artifact@v2 + with: + name: static-archive + path: static-archive + - name: Prepare Artifacts + working-directory: ./artifacts/ + run: | + mv ../dist-tarball/* . || exit 1 + mv ../static-archive/* . || exit 1 + ln -s ${{ needs.build-dist.outputs.distfile }} netdata-latest.tar.gz || exit 1 + cp ../packaging/version ./latest-version.txt || exit 1 + sha256sum -b ./* > sha256sums.txt || exit 1 + cat sha256sums.txt + - name: Store Artifacts + uses: actions/upload-artifact@v2 + with: + name: final-artifacts + path: artifacts/* + retention-days: 30 + - name: Failure Notification + uses: rtCamp/action-slack-notify@v2 + env: + SLACK_COLOR: 'danger' + SLACK_FOOTER: '' + SLACK_ICON_EMOJI: ':github-actions:' + SLACK_TITLE: 'Failed to prepare release artifacts for upload:' + SLACK_USERNAME: 'GitHub Actions' + SLACK_MESSAGE: "Failed to prepare release artifacts for upload." + SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }} + if: >- + ${{ + failure() + && startsWith(github.ref, 'refs/heads/master') + && github.event_name != 'pull_request' + }} + + artifact-verification-dist: # Verify the regular installer works with the consolidated artifacts. + name: Test Consolidated Artifacts (Source) + runs-on: ubuntu-latest + needs: + - prepare-upload + services: + apache: # This gets used to serve the dist tarball for the updater script. + image: httpd:2.4 + ports: + - 8080:80 + volumes: + - ${{ github.workspace }}:/usr/local/apache2/htdocs/ + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Fetch artifacts + uses: actions/download-artifact@v2 + with: + name: final-artifacts + path: artifacts + - name: Verify that artifacts work with installer + env: + NETDATA_TARBALL_BASEURL: http://localhost:8080/artifacts + run: packaging/installer/kickstart.sh --dont-start-it --disable-telemetry --dont-wait + - name: Failure Notification + uses: rtCamp/action-slack-notify@v2 + env: + SLACK_COLOR: 'danger' + SLACK_FOOTER: '' + SLACK_ICON_EMOJI: ':github-actions:' + SLACK_TITLE: 'Artifact verification for source tarball failed.' + SLACK_USERNAME: 'GitHub Actions' + SLACK_MESSAGE: "Artifact verification for source tarball failed." + SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }} + if: >- + ${{ + failure() + && startsWith(github.ref, 'refs/heads/master') + && github.event_name != 'pull_request' + }} + + artifact-verification-static: # Verify the static installer works with the consolidated artifacts. + name: Test Consolidated Artifacts (Static) + runs-on: ubuntu-latest + needs: + - prepare-upload + services: + apache: # This gets used to serve the static archives. + image: httpd:2.4 + ports: + - 8080:80 + volumes: + - ${{ github.workspace }}:/usr/local/apache2/htdocs/ + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Fetch artifacts + uses: actions/download-artifact@v2 + with: + name: final-artifacts + path: artifacts + - name: Verify that artifacts work with installer + env: + NETDATA_TARBALL_BASEURL: http://localhost:8080/artifacts + run: packaging/installer/kickstart-static64.sh --dont-start-it --disable-telemetry --dont-wait + - name: Failure Notification + uses: rtCamp/action-slack-notify@v2 + env: + SLACK_COLOR: 'danger' + SLACK_FOOTER: '' + SLACK_ICON_EMOJI: ':github-actions:' + SLACK_TITLE: 'Artifact verification for static build failed.' + SLACK_USERNAME: 'GitHub Actions' + SLACK_MESSAGE: "Artifact verification for static build failed." + SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }} + if: >- + ${{ + failure() + && startsWith(github.ref, 'refs/heads/master') + && github.event_name != 'pull_request' + }} + + upload-nightly: # Upload the nightly build artifacts to GCS. + name: Upload Nightly Artifacts + runs-on: ubuntu-latest + if: github.event_name == 'workflow_dispatch' && github.event.inputs.type == 'nightly' + needs: + - updater-check + - source-build + - artifact-verification-dist + - artifact-verification-static + steps: + - name: Retrieve Artifacts + uses: actions/download-artifact@v2 + with: + name: final-artifacts + path: final-artifacts + - name: Setup Gcloud + uses: google-github-actions/setup-gcloud@v0.2.1 + with: + project_id: ${{ secrets.GCP_NIGHTLY_STORAGE_PROJECT }} + service_account_key: ${{ secrets.GCP_STORAGE_SERVICE_ACCOUNT_KEY }} + export_default_credentials: true + - name: Upload Artifacts + uses: google-github-actions/upload-cloud-storage@v0.4.0 + with: + destination: ${{ secrets.GCP_NIGHTLY_STORAGE_BUCKET }} + gzip: false + path: ./final-artifacts + parent: false + - name: Failure Notification + uses: rtCamp/action-slack-notify@v2 + env: + SLACK_COLOR: 'danger' + SLACK_FOOTER: '' + SLACK_ICON_EMOJI: ':github-actions:' + SLACK_TITLE: 'Failed to upload nightly release artifacts:' + SLACK_USERNAME: 'GitHub Actions' + SLACK_MESSAGE: "Failed to upload nightly release artifacts." + SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }} + if: >- + ${{ + failure() + && startsWith(github.ref, 'refs/heads/master') + && github.event_name != 'pull_request' + }} + + upload-release: # Create the draft release and upload the build artifacts. + name: Create Release Draft + runs-on: ubuntu-latest + if: github.event_name == 'workflow_dispatch' && github.event.inputs.type == 'release' + needs: + - updater-check + - source-build + - artifact-verification-dist + - artifact-verification-static + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Retrieve Artifacts + uses: actions/download-artifact@v2 + with: + name: final-artifacts + path: final-artifacts + - name: Create Release + uses: ncipollo/release-action@v1 + with: + allowUpdates: false + artifactErrorsFailBuild: true + artifacts: 'final-artifacts/sha256sums.txt,final-artifacts/netdata-*.tar.gz,final-artifacts/netdata-*.gz.run' + draft: true + tag: ${{ github.event.inputs.version }} + token: ${{ secrets.NETDATABOT_TOKEN }} + - name: Failure Notification + uses: rtCamp/action-slack-notify@v2 + env: + SLACK_COLOR: 'danger' + SLACK_FOOTER: '' + SLACK_ICON_EMOJI: ':github-actions:' + SLACK_TITLE: 'Failed to draft release:' + SLACK_USERNAME: 'GitHub Actions' + SLACK_MESSAGE: "Failed to draft release." + SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }} + if: >- + ${{ + failure() + && startsWith(github.ref, 'refs/heads/master') + && github.event_name == 'workflow_dispatch' + }} diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index e4025dd20..bc746f9ac 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -4,7 +4,12 @@ on: push: branches: - master - pull_request: + pull_request: null +env: + DO_NOT_TRACK: 1 +concurrency: + group: checks-${{ github.ref }} + cancel-in-progress: true jobs: checksum-checks: name: Checksums @@ -43,41 +48,11 @@ jobs: steps: - name: Checkout uses: actions/checkout@v2 - - name: Build - run: | - docker build -f .github/dockerfiles/Dockerfile.clang . - dist-checks: - name: Dist - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v2 with: submodules: recursive - - name: Prepare environment - run: | - ./packaging/installer/install-required-packages.sh --dont-wait --non-interactive netdata - sudo apt-get install -y libjson-c-dev libipmimonitoring-dev libcups2-dev libsnappy-dev \ - libprotobuf-dev libprotoc-dev libssl-dev protobuf-compiler \ - libnetfilter-acct-dev - - name: Configure - run: | - autoreconf -ivf - ./configure \ - --with-zlib \ - --with-math \ - --with-user=netdata \ - CFLAGS=-O2 - - name: Make dist - run: | - make dist - - name: Verify & Set distfile - run: | - ls -lah netdata-*.tar.gz - echo "DISTFILE=$(ls netdata-*.tar.gz)" >> $GITHUB_ENV - - name: Run run_install_with_dist_file.sh + - name: Build run: | - ./.github/scripts/run_install_with_dist_file.sh "${DISTFILE}" + docker build -f .github/dockerfiles/Dockerfile.clang . gitignore-check: name: .gitignore runs-on: ubuntu-latest diff --git a/.github/workflows/coverity.yml b/.github/workflows/coverity.yml index 766275ed9..c9e25ebf9 100644 --- a/.github/workflows/coverity.yml +++ b/.github/workflows/coverity.yml @@ -8,6 +8,11 @@ on: paths: - .github/workflows/coverity.yml - coverity-scan.sh +env: + DO_NOT_TRACK: 1 +concurrency: + group: coverity-${{ github.ref }} + cancel-in-progress: true jobs: coverity: if: github.repository == 'netdata/netdata' @@ -38,7 +43,7 @@ jobs: uses: rtCamp/action-slack-notify@v2 env: SLACK_COLOR: 'danger' - SLACK_FOOTER: + SLACK_FOOTER: '' SLACK_ICON_EMOJI: ':github-actions:' SLACK_TITLE: 'Coverity run failed:' SLACK_USERNAME: 'GitHub Actions' diff --git a/.github/workflows/dashboard-pr.yml b/.github/workflows/dashboard-pr.yml index 8f5a40f2b..711cf1e1f 100644 --- a/.github/workflows/dashboard-pr.yml +++ b/.github/workflows/dashboard-pr.yml @@ -8,9 +8,12 @@ on: dashboard_version: # This must be specified, and must _exactly_ match the version # tag for the release to be used for the update. - name: Dashboard Version + description: Dashboard Version required: true +env: + DO_NOT_TRACK: 1 + jobs: dashboard-pr: name: Generate Dashboard Version Bump PR @@ -18,30 +21,16 @@ jobs: steps: - name: Checkout uses: actions/checkout@v2 - - name: Create Branch - # This is needed because we want to do a PR, and the commit - # action used below requires the branch it is commiting to to - # already exist. - run: | - git checkout -b dashboard-${{ github.event.inputs.dashboard_version }} - git push -u origin dashboard-${{ github.event.inputs.dashboard_version }} - name: Update Files run: | web/gui/bundle_dashboard.py ${{ github.event.inputs.dashboard_version }} - - name: Commit Changes - uses: swinton/commit@v2.x - env: - GH_TOKEN: ${{ secrets.NETDATABOT_GITHUB_TOKEN }} + - name: Create Pull Request + uses: peter-evans/create-pull-request@v3 with: - files: | - packaging/dashboard.version - packaging/dashboard.checksums + title: 'Update dashboard to version ${{ github.event.inputs.dashboard_version }}.' + body: 'See https://github.com/netdata/dashboard/releases/tag/${{ github.event.inputs.dashboard_version }} for changes.' + branch: dashboard-${{ github.event.inputs.dashboard_version }} + branch-suffix: timestamp + delete-branch: true commit-message: 'Update dashboard to version ${{ github.event.inputs.dashboard_version }}.' - ref: refs/heads/dashboard-${{ github.event.inputs.dashboard_version }} - - name: Create PR - uses: repo-sync/pull-request@v2 - with: - source_branch: dashboard-${{ github.event.inputs.dashboard_version }} - pr_title: 'Update dashboard to version ${{ github.event.inputs.dashboard_version }}.' - pr_body: 'See https://github.com/netdata/dashboard/releases/tag/${{ github.event.inputs.dashboard_version }} for changes.' - github_token: ${{ secrets.NETDATABOT_GITHUB_TOKEN }} + token: ${{ secrets.NETDATABOT_GITHUB_TOKEN }} diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 4f18ce5b6..f8a1d6855 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -4,43 +4,94 @@ on: push: branches: - master - pull_request: + pull_request: null workflow_dispatch: inputs: version: name: Version Tag default: nightly required: true +env: + DO_NOT_TRACK: 1 +concurrency: + group: docker-${{ github.ref }}-${{ github.event_name }} + cancel-in-progress: true jobs: - docker-build: - name: Docker Build + docker-test: + name: Docker Runtime Test runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v2 with: submodules: recursive - - name: Determine if we should push changes and which tags to use - if: github.event_name == 'workflow_dispatch' && github.event.inputs.version != 'nightly' + - name: Setup Buildx + uses: docker/setup-buildx-action@v1 + - name: Test Build + uses: docker/build-push-action@v2 + with: + load: true + push: false + tags: netdata/netdata:test + - name: Test Image + run: .github/scripts/docker-test.sh + + docker-ci: + if: github.event_name != 'workflow_dispatch' + name: Docker Alt Arch Builds + needs: docker-test + runs-on: ubuntu-latest + strategy: + matrix: + platforms: + - linux/i386 + - linux/arm/v7 + - linux/arm64 + - linux/ppc64le + steps: + - name: Checkout + uses: actions/checkout@v2 + with: + submodules: recursive + - name: Setup QEMU + if: matrix.platforms != 'linux/i386' + uses: docker/setup-qemu-action@v1 + - name: Setup Buildx + uses: docker/setup-buildx-action@v1 + - name: Build + uses: docker/build-push-action@v2 + with: + platforms: ${{ matrix.platforms }} + load: false + push: false + tags: netdata/netdata:test + + docker-publish: + if: github.event_name == 'workflow_dispatch' + name: Docker Build and Publish + needs: docker-test + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + with: + submodules: recursive + - name: Determine which tags to use + if: github.event.inputs.version != 'nightly' run: | - echo "publish=true" >> $GITHUB_ENV echo "tags=netdata/netdata:latest,netdata/netdata:stable,$(.github/scripts/gen-docker-tags.py ${{ github.event.inputs.version }})" >> $GITHUB_ENV - - name: Determine if we should push changes and which tags to use - if: github.event_name == 'workflow_dispatch' && github.event.inputs.version == 'nightly' + - name: Determine which tags to use + if: github.event.inputs.version == 'nightly' run: | - echo "publish=true" >> $GITHUB_ENV echo "tags=netdata/netdata:latest,netdata/netdata:edge" >> $GITHUB_ENV - - name: Determine if we should push changes and which tags to use - if: github.event_name != 'workflow_dispatch' - run: | - echo "publish=false" >> $GITHUB_ENV - echo "tags=netdata/netdata:test" >> $GITHUB_ENV + - name: Mark image as official + if: github.repository == 'netdata/netdata' + run: echo "OFFICIAL_IMAGE=true" >> $GITHUB_ENV - name: Setup QEMU uses: docker/setup-qemu-action@v1 - name: Setup Buildx uses: docker/setup-buildx-action@v1 - name: Docker Hub Login - if: github.event_name == 'workflow_dispatch' uses: docker/login-action@v1 with: username: ${{ secrets.DOCKER_HUB_USERNAME }} @@ -48,14 +99,15 @@ jobs: - name: Docker Build uses: docker/build-push-action@v2 with: - platforms: linux/amd64,linux/i386,linux/arm/v7,linux/arm64 - push: ${{ env.publish }} + platforms: linux/amd64,linux/i386,linux/arm/v7,linux/arm64,linux/ppc64le + push: true tags: ${{ env.tags }} + build-args: OFFICIAL_IMAGE=${{ env.OFFICIAL_IMAGE }} - name: Failure Notification uses: rtCamp/action-slack-notify@v2 env: SLACK_COLOR: 'danger' - SLACK_FOOTER: + SLACK_FOOTER: '' SLACK_ICON_EMOJI: ':github-actions:' SLACK_TITLE: 'Docker Build failed:' SLACK_USERNAME: 'GitHub Actions' diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 9f7234f92..73bcc8773 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -9,6 +9,8 @@ on: pull_request: paths: - '**.md' +env: + DO_NOT_TRACK: 1 jobs: markdown-link-check: name: Broken Links diff --git a/.github/workflows/labeler.yml b/.github/workflows/labeler.yml index 24842e73d..a4937a9db 100644 --- a/.github/workflows/labeler.yml +++ b/.github/workflows/labeler.yml @@ -3,7 +3,9 @@ name: Pull Request Labeler on: schedule: - - cron: '*/5 * * * *' + - cron: '*/10 * * * *' +env: + DO_NOT_TRACK: 1 jobs: labeler: runs-on: ubuntu-latest diff --git a/.github/workflows/packaging.yml b/.github/workflows/packaging.yml index 934aa264b..80c14e24b 100644 --- a/.github/workflows/packaging.yml +++ b/.github/workflows/packaging.yml @@ -15,6 +15,11 @@ on: version: name: Package version required: false +env: + DO_NOT_TRACK: 1 +concurrency: + group: packages-${{ github.ref }}-${{ github.event_name }} + cancel-in-progress: true jobs: build: name: Build @@ -26,21 +31,47 @@ jobs: include: - {distro: debian, version: "9", pkgclouddistro: debian/stretch, format: deb, base_image: debian, platform: linux/amd64, arch: amd64} - {distro: debian, version: "9", pkgclouddistro: debian/stretch, format: deb, base_image: debian, platform: linux/i386, arch: i386} + - {distro: debian, version: "9", pkgclouddistro: debian/stretch, format: deb, base_image: debian, platform: linux/arm/v7, arch: armhf} + - {distro: debian, version: "9", pkgclouddistro: debian/stretch, format: deb, base_image: debian, platform: linux/arm64/v8, arch: arm64} - {distro: debian, version: "10", pkgclouddistro: debian/buster, format: deb, base_image: debian, platform: linux/amd64, arch: amd64} - {distro: debian, version: "10", pkgclouddistro: debian/buster, format: deb, base_image: debian, platform: linux/i386, arch: i386} - - {distro: ubuntu, version: "16.04", pkgclouddistro: ubuntu/xenial, format: deb, base_image: ubuntu, platform: linux/amd64, arch: amd64} - - {distro: ubuntu, version: "16.04", pkgclouddistro: ubuntu/xenial, format: deb, base_image: ubuntu, platform: linux/i386, arch: i386} + - {distro: debian, version: "10", pkgclouddistro: debian/buster, format: deb, base_image: debian, platform: linux/arm/v7, arch: armhf} + - {distro: debian, version: "10", pkgclouddistro: debian/buster, format: deb, base_image: debian, platform: linux/arm64/v8, arch: arm64} + - {distro: debian, version: "11", pkgclouddistro: debian/bullseye, format: deb, base_image: debian, platform: linux/amd64, arch: amd64, alias: bullseye} + - {distro: debian, version: "11", pkgclouddistro: debian/bullseye, format: deb, base_image: debian, platform: linux/i386, arch: i386, alias: bullseye} + - {distro: debian, version: "11", pkgclouddistro: debian/bullseye, format: deb, base_image: debian, platform: linux/arm/v7, arch: armhf, alias: bullseye} + - {distro: debian, version: "11", pkgclouddistro: debian/bullseye, format: deb, base_image: debian, platform: linux/arm64/v8, arch: arm64, alias: bullseye} - {distro: ubuntu, version: "18.04", pkgclouddistro: ubuntu/bionic, format: deb, base_image: ubuntu, platform: linux/amd64, arch: amd64} - {distro: ubuntu, version: "18.04", pkgclouddistro: ubuntu/bionic, format: deb, base_image: ubuntu, platform: linux/i386, arch: i386} + - {distro: ubuntu, version: "18.04", pkgclouddistro: ubuntu/bionic, format: deb, base_image: ubuntu, platform: linux/arm/v7, arch: armhf} + - {distro: ubuntu, version: "18.04", pkgclouddistro: ubuntu/bionic, format: deb, base_image: ubuntu, platform: linux/arm64/v8, arch: arm64} - {distro: ubuntu, version: "20.04", pkgclouddistro: ubuntu/focal, format: deb, base_image: ubuntu, platform: linux/amd64, arch: amd64} - - {distro: ubuntu, version: "20.10", pkgclouddistro: ubuntu/groovy, format: deb, base_image: ubuntu, platform: linux/amd64, arch: amd64} + - {distro: ubuntu, version: "20.04", pkgclouddistro: ubuntu/focal, format: deb, base_image: ubuntu, platform: linux/arm/v7, arch: armhf} + - {distro: ubuntu, version: "20.04", pkgclouddistro: ubuntu/focal, format: deb, base_image: ubuntu, platform: linux/arm64/v8, arch: arm64} - {distro: ubuntu, version: "21.04", pkgclouddistro: ubuntu/hirsute, format: deb, base_image: ubuntu, platform: linux/amd64, arch: amd64} + - {distro: ubuntu, version: "21.04", pkgclouddistro: ubuntu/hirsute, format: deb, base_image: ubuntu, platform: linux/arm/v7, arch: armhf} + - {distro: ubuntu, version: "21.04", pkgclouddistro: ubuntu/hirsute, format: deb, base_image: ubuntu, platform: linux/arm64/v8, arch: arm64} + - {distro: ubuntu, version: "21.10", pkgclouddistro: ubuntu/impish, format: deb, base_image: ubuntu, platform: linux/amd64, arch: amd64} + - {distro: ubuntu, version: "21.10", pkgclouddistro: ubuntu/impish, format: deb, base_image: ubuntu, platform: linux/arm/v7, arch: armhf} + - {distro: ubuntu, version: "21.10", pkgclouddistro: ubuntu/impish, format: deb, base_image: ubuntu, platform: linux/arm64/v8, arch: arm64} - {distro: centos, version: "7", pkgclouddistro: el/7, format: rpm, base_image: centos, platform: linux/amd64, arch: amd64} - {distro: centos, version: "8", pkgclouddistro: el/8, format: rpm, base_image: centos, platform: linux/amd64, arch: amd64} - - {distro: fedora, version: "32", pkgclouddistro: fedora/32, format: rpm, base_image: fedora, platform: linux/amd64, arch: amd64} + - {distro: centos, version: "8", pkgclouddistro: el/8, format: rpm, base_image: centos, platform: linux/arm64/v8, arch: arm64} - {distro: fedora, version: "33", pkgclouddistro: fedora/33, format: rpm, base_image: fedora, platform: linux/amd64, arch: amd64} + - {distro: fedora, version: "33", pkgclouddistro: fedora/33, format: rpm, base_image: fedora, platform: linux/arm/v7, arch: armhf} + - {distro: fedora, version: "33", pkgclouddistro: fedora/33, format: rpm, base_image: fedora, platform: linux/arm64/v8, arch: arm64} - {distro: fedora, version: "34", pkgclouddistro: fedora/34, format: rpm, base_image: fedora, platform: linux/amd64, arch: amd64} + - {distro: fedora, version: "34", pkgclouddistro: fedora/34, format: rpm, base_image: fedora, platform: linux/arm/v7, arch: armhf} + - {distro: fedora, version: "34", pkgclouddistro: fedora/34, format: rpm, base_image: fedora, platform: linux/arm64/v8, arch: arm64} + - {distro: fedora, version: "35", pkgclouddistro: fedora/35, format: rpm, base_image: fedora, platform: linux/amd64, arch: amd64} + - {distro: fedora, version: "35", pkgclouddistro: fedora/35, format: rpm, base_image: fedora, platform: linux/arm/v7, arch: armhf} + - {distro: fedora, version: "35", pkgclouddistro: fedora/35, format: rpm, base_image: fedora, platform: linux/arm64/v8, arch: arm64} - {distro: opensuse, version: "15.2", pkgclouddistro: opensuse/15.2, format: rpm, base_image: opensuse/leap, platform: linux/amd64, arch: amd64} + - {distro: opensuse, version: "15.2", pkgclouddistro: opensuse/15.2, format: rpm, base_image: opensuse/leap, platform: linux/arm64/v8, arch: arm64} + - {distro: opensuse, version: "15.3", pkgclouddistro: opensuse/15.3, format: rpm, base_image: opensuse/leap, platform: linux/amd64, arch: amd64} + - {distro: opensuse, version: "15.3", pkgclouddistro: opensuse/15.3, format: rpm, base_image: opensuse/leap, platform: linux/arm64/v8, arch: arm64} + - {distro: oraclelinux, version: "8", pkgclouddistro: ol/8, format: rpm, base_image: oraclelinux, platform: linux/amd64, arch: amd64} + - {distro: oraclelinux, version: "8", pkgclouddistro: ol/8, format: rpm, base_image: oraclelinux, platform: linux/arm64/v8, arch: arm64} # We intentiaonally disable the fail-fast behavior so that a # build failure for one version doesn't prevent us from publishing # successfully built and tested packages for another version. @@ -52,14 +83,14 @@ jobs: uses: actions/checkout@v2 with: fetch-depth: 0 # We need full history for versioning - submodules: true + submodules: recursive - name: Checkout Tag # Otherwise check out the tag that triggered this. if: github.event_name == 'workflow_dispatch' uses: actions/checkout@v2 with: ref: ${{ github.event.ref }} fetch-depth: 0 # We need full history for versioning - submodules: true + submodules: recursive - name: Check Base Branch run: | if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then @@ -86,35 +117,44 @@ jobs: echo "pkg_version=$(cut -d'-' -f 1 packaging/version | sed -e 's/^v//')" >> $GITHUB_ENV fi - name: Setup QEMU - if: matrix.platform != 'linux/amd64' + if: matrix.platform != 'linux/amd64' && matrix.platform != 'linux/i386' uses: docker/setup-qemu-action@v1 - - name: Setup Buildx - uses: docker/setup-buildx-action@v1 - name: Prepare Docker Environment shell: bash run: | echo '{"cgroup-parent": "/actions_job", "experimental": true}' | sudo tee /etc/docker/daemon.json 2>/dev/null sudo service docker restart - - name: Build Packages - uses: docker/build-push-action@v2 + - name: Set Base Image Version + shell: bash + run: | + if [ -z "${{ matrix.alias }}" ] ; then + echo "version=${{ matrix.version }}" >> $GITHUB_ENV + else + echo "version=${{ matrix.alias }}" >> $GITHUB_ENV + fi + - name: Fetch base image + uses: nick-invision/retry@v2 with: - platforms: ${{ matrix.platform }} - file: packaging/Dockerfile.packager - tags: local/package-builder:${{ matrix.distro}}${{ matrix.version }} - push: false - load: true - build-args: | - ARCH=${{ matrix.arch }} - DISTRO=${{ matrix.distro }} - TEST_BASE=${{ matrix.base_image }} - DISTRO_VERSION=${{ matrix.version }} - PKG_VERSION=${{ env.pkg_version }} - - name: Extract Packages + max_attempts: 3 + retry_wait_seconds: 30 + timeout_seconds: 900 + command: | + docker pull --platform ${{ matrix.platform }} ${{ matrix.base_image }}:${{ env.version }} + docker pull --platform ${{ matrix.platform }} netdata/package-builders:${{ matrix.distro }}${{ matrix.version }} + - name: Build Packages + shell: bash + run: | + docker run --security-opt seccomp=unconfined -e DO_NOT_TRACK=1 -e VERSION=${{ env.pkg_version }} --platform=${{ matrix.platform }} -v $PWD:/netdata netdata/package-builders:${{ matrix.distro }}${{ matrix.version }} + - name: Test Packages shell: bash run: | - mkdir -p artifacts - docker run --platform ${{ matrix.platform }} -v $PWD/artifacts:/artifacts local/package-builder:${{ matrix.distro }}${{ matrix.version }} - - name: Upload + docker run --security-opt seccomp=unconfined -e DO_NOT_TRACK=1 -e DISTRO=${{ matrix.distro }} -e VERSION=${{ env.pkg_version }} -e DISTRO_VERSION=${{ env.version }} --platform=${{ matrix.platform }} -v $PWD:/netdata ${{ matrix.base_image }}:${{ env.version }} /netdata/.github/scripts/pkg-test.sh + - name: Save Packages + uses: actions/upload-artifact@v2 + with: + name: ${{ matrix.distro }}-${{ matrix.version }}-${{ matrix.arch }}-packages + path: ${{ github.workspace }}/artifacts/* + - name: Upload to PackageCloud if: github.event_name == 'workflow_dispatch' shell: bash env: @@ -122,7 +162,7 @@ jobs: run: | echo "Packages to upload:\n$(ls artifacts/*.${{ matrix.format }})" for pkgfile in artifacts/*.${{ matrix.format }} ; do - .github/scripts/package_cloud_wrapper.sh yank ${{ env.repo }}/${{ matrix.pkgclouddistro }} ${pkgfile} || true + .github/scripts/package_cloud_wrapper.sh yank ${{ env.repo }}/${{ matrix.pkgclouddistro }} $(basename ${pkgfile}) || true .github/scripts/package_cloud_wrapper.sh push ${{ env.repo }}/${{ matrix.pkgclouddistro }} ${pkgfile} done - name: Clean @@ -137,7 +177,6 @@ jobs: uses: rtCamp/action-slack-notify@v2 env: SLACK_COLOR: 'danger' - SLACK_FOOTER: SLACK_ICON_EMOJI: ':github-actions:' SLACK_TITLE: 'Package Build failed:' SLACK_USERNAME: 'GitHub Actions' diff --git a/.github/workflows/repoconfig-packages.yml b/.github/workflows/repoconfig-packages.yml new file mode 100644 index 000000000..c132ec8b5 --- /dev/null +++ b/.github/workflows/repoconfig-packages.yml @@ -0,0 +1,80 @@ +--- +# Handles building of binary packages for the agent. +name: Repository Packages +on: + workflow_dispatch: null +env: + DO_NOT_TRACK: 1 +jobs: + build: + name: Build + runs-on: ubuntu-latest + env: + DO_NOT_TRACK: 1 + DOCKER_CLI_EXPERIMENTAL: enabled + strategy: + # This needs to be kept in sync with the matrix in packaging.yml, but should only include the AMD64 lines. + matrix: + include: + - {distro: debian, version: "9", pkgclouddistro: debian/stretch, format: deb, base_image: debian, platform: linux/amd64, arch: amd64} + - {distro: debian, version: "10", pkgclouddistro: debian/buster, format: deb, base_image: debian, platform: linux/amd64, arch: amd64} + - {distro: debian, version: "11", pkgclouddistro: debian/bullseye, format: deb, base_image: debian, platform: linux/amd64, arch: amd64} + - {distro: ubuntu, version: "18.04", pkgclouddistro: ubuntu/bionic, format: deb, base_image: ubuntu, platform: linux/amd64, arch: amd64} + - {distro: ubuntu, version: "20.04", pkgclouddistro: ubuntu/focal, format: deb, base_image: ubuntu, platform: linux/amd64, arch: amd64} + - {distro: ubuntu, version: "21.04", pkgclouddistro: ubuntu/hirsute, format: deb, base_image: ubuntu, platform: linux/amd64, arch: amd64} + - {distro: ubuntu, version: "21.10", pkgclouddistro: ubuntu/impish, format: deb, base_image: ubuntu, platform: linux/amd64, arch: amd64} + - {distro: centos, version: "7", pkgclouddistro: el/7, format: rpm, base_image: centos, platform: linux/amd64, arch: amd64} + - {distro: centos, version: "8", pkgclouddistro: el/8, format: rpm, base_image: centos, platform: linux/amd64, arch: amd64} + - {distro: fedora, version: "33", pkgclouddistro: fedora/33, format: rpm, base_image: fedora, platform: linux/amd64, arch: amd64} + - {distro: fedora, version: "34", pkgclouddistro: fedora/34, format: rpm, base_image: fedora, platform: linux/amd64, arch: amd64} + - {distro: fedora, version: "35", pkgclouddistro: fedora/35, format: rpm, base_image: fedora, platform: linux/amd64, arch: amd64} + - {distro: opensuse, version: "15.2", pkgclouddistro: opensuse/15.2, format: rpm, base_image: opensuse/leap, platform: linux/amd64, arch: amd64} + - {distro: opensuse, version: "15.3", pkgclouddistro: opensuse/15.3, format: rpm, base_image: opensuse/leap, platform: linux/amd64, arch: amd64} + - {distro: oraclelinux, version: "8", pkgclouddistro: ol/8, format: rpm, base_image: oraclelinux, platform: linux/amd64, arch: amd64} + # We intentiaonally disable the fail-fast behavior so that a + # build failure for one version doesn't prevent us from publishing + # successfully built and tested packages for another version. + fail-fast: false + max-parallel: 8 + steps: + - name: Checkout + uses: actions/checkout@v2 + # Unlike normally, we do not need a deep clone or submodules for this. + - name: Fetch base image + uses: nick-invision/retry@v2 + with: + max_attempts: 3 + retry_wait_seconds: 30 + timeout_seconds: 900 + command: docker pull --platform ${{ matrix.platform }} ${{ matrix.base_image }}:${{ matrix.version }} + - name: Build Packages + shell: bash + run: | + docker run --security-opt seccomp=unconfined -e DO_NOT_TRACK=1 --platform ${{ matrix.platform }} \ + -v $PWD:/netdata ${{ matrix.base_image }}:${{ matrix.version }} \ + /netdata/packaging/repoconfig/build-${{ matrix.format }}.sh + - name: Upload Packages + shell: bash + env: + PKG_CLOUD_TOKEN: ${{ secrets.PACKAGE_CLOUD_API_KEY }} + run: | + echo "Packages to upload:\n$(ls artifacts/*.${{ matrix.format }})" + for pkgfile in artifacts/*.${{ matrix.format }} ; do + .github/scripts/package_cloud_wrapper.sh yank ${{ secrets.PACKAGE_CLOUD_REPO }}/${{ matrix.pkgclouddistro }} $(basename ${pkgfile}) || true + .github/scripts/package_cloud_wrapper.sh push ${{ secrets.PACKAGE_CLOUD_REPO }}/${{ matrix.pkgclouddistro }} ${pkgfile} + .github/scripts/package_cloud_wrapper.sh yank ${{ secrets.PACKAGE_CLOUD_REPO }}-edge/${{ matrix.pkgclouddistro }} $(basename ${pkgfile}) || true + .github/scripts/package_cloud_wrapper.sh push ${{ secrets.PACKAGE_CLOUD_REPO }}-edge/${{ matrix.pkgclouddistro }} ${pkgfile} + .github/scripts/package_cloud_wrapper.sh yank ${{ secrets.PACKAGE_CLOUD_REPO }}-repoconfig/${{ matrix.pkgclouddistro }} $(basename ${pkgfile}) || true + .github/scripts/package_cloud_wrapper.sh push ${{ secrets.PACKAGE_CLOUD_REPO }}-repoconfig/${{ matrix.pkgclouddistro }} ${pkgfile} + done + - name: Failure Notification + if: ${{ failure() }} + uses: rtCamp/action-slack-notify@v2 + env: + SLACK_COLOR: 'danger' + SLACK_FOOTER: '' + SLACK_ICON_EMOJI: ':github-actions:' + SLACK_TITLE: 'Repository Package Build failed:' + SLACK_USERNAME: 'GitHub Actions' + SLACK_MESSAGE: "${{ matrix.pkgclouddistro }} ${{ matrix.version }} repository package build failed." + SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }} diff --git a/.github/workflows/review.yml b/.github/workflows/review.yml index a267fea3f..e9972303b 100644 --- a/.github/workflows/review.yml +++ b/.github/workflows/review.yml @@ -2,15 +2,95 @@ # Runs various ReviewDog based checks against PR with suggested changes to improve quality name: Review on: - pull_request: + pull_request: null env: - run_eslint: 0 - run_hadolint: 0 - run_shellcheck: 0 - run_yamllint: 0 + DO_NOT_TRACK: 1 +concurrency: + group: review-${{ github.ref }} + cancel-in-progress: true jobs: + prep-review: + name: Prepare Review Jobs + runs-on: ubuntu-latest + outputs: + actionlint: ${{ steps.actionlint.outputs.run }} + eslint: ${{ steps.eslint.outputs.run }} + hadolint: ${{ steps.hadolint.outputs.run }} + shellcheck: ${{ steps.shellcheck.outputs.run }} + yamllint: ${{ steps.yamllint.outputs.run }} + steps: + - name: Clone repository + uses: actions/checkout@v2 + with: + submodules: recursive + fetch-depth: 0 + - name: Check files for actionlint + id: actionlint + run: | + if git diff --name-only origin/${{ github.base_ref }} HEAD | grep -Eq '\.github/workflows/.*' ; then + echo '::set-output name=run::true' + echo 'GitHub Actions workflows have changed, need to run actionlint.' + else + echo '::set-output name=run::false' + fi + - name: Check files for eslint + id: eslint + run: | + if git diff --name-only origin/${{ github.base_ref }} HEAD | grep -v "web/gui/dashboard" | grep -Eq '.*\.js|node\.d\.plugin\.in' ; then + echo '::set-output name=run::true' + echo 'JS files have changed, need to run ESLint.' + else + echo '::set-output name=run::false' + fi + - name: Check files for hadolint + id: hadolint + run: | + if git diff --name-only origin/${{ github.base_ref }} HEAD | grep -Eq '.*Dockerfile.*' ; then + echo '::set-output name=run::true' + echo 'Dockerfiles have changed, need to run Hadolint.' + else + echo '::set-output name=run::false' + fi + - name: Check files for shellcheck + id: shellcheck + run: | + if git diff --name-only origin/${{ github.base_ref }} HEAD | grep -Eq '.*\.sh.*' ; then + echo '::set-output name=run::true' + echo 'Shell scripts have changed, need to run shellcheck.' + else + echo '::set-output name=run::false' + fi + - name: Check files for yamllint + id: yamllint + run: | + if git diff --name-only origin/${{ github.base_ref }} HEAD | grep -Eq '.*\.ya?ml|python\.d/.*\.conf' ; then + echo '::set-output name=run::true' + echo 'YAML files have changed, need to run yamllint.' + else + echo '::set-output name=run::false' + fi + + actionlint: + name: actionlint + needs: prep-review + if: needs.prep-review.outputs.actionlint == 'true' + runs-on: ubuntu-latest + steps: + - name: Git clone repository + uses: actions/checkout@v2 + with: + submodules: recursive + fetch-depth: 0 + - name: Run actionlint + uses: reviewdog/action-actionlint@v1 + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + reporter: github-pr-check + eslint: name: eslint + needs: prep-review + if: needs.prep-review.outputs.eslint == 'true' runs-on: ubuntu-latest steps: - name: Git clone repository @@ -18,13 +98,7 @@ jobs: with: submodules: recursive fetch-depth: 0 - - name: Check files - run: | - if git diff --name-only origin/${{ github.base_ref }} HEAD | grep -Eq '*\.js|node\.d\.plugin\.in' ; then - echo 'run_eslint=1' >> $GITHUB_ENV - fi - name: Run eslint - if: env.run_eslint == 1 uses: reviewdog/action-eslint@v1 with: github_token: ${{ secrets.GITHUB_TOKEN }} @@ -33,19 +107,15 @@ jobs: hadolint: name: hadolint + needs: prep-review + if: needs.prep-review.outputs.hadolint == 'true' runs-on: ubuntu-latest steps: - name: Git clone repository uses: actions/checkout@v2 with: fetch-depth: 0 - - name: Check files - run: | - if git diff --name-only origin/${{ github.base_ref }} HEAD | grep -Eq '*Dockerfile*' ; then - echo 'run_hadolint=1' >> $GITHUB_ENV - fi - name: Run hadolint - if: env.run_hadolint == 1 uses: reviewdog/action-hadolint@v1 with: github_token: ${{ secrets.GITHUB_TOKEN }} @@ -53,6 +123,8 @@ jobs: shellcheck: name: shellcheck + needs: prep-review + if: needs.prep-review.outputs.shellcheck == 'true' runs-on: ubuntu-latest steps: - name: Git clone repository @@ -60,13 +132,7 @@ jobs: with: submodules: recursive fetch-depth: 0 - - name: Check files - run: | - if git diff --name-only origin/${{ github.base_ref }} HEAD | grep -Eq '*\.sh.*' ; then - echo 'run_shellcheck=1' >> $GITHUB_ENV - fi - name: Run shellcheck - if: env.run_shellcheck == 1 uses: reviewdog/action-shellcheck@v1 with: github_token: ${{ secrets.GITHUB_TOKEN }} @@ -77,6 +143,8 @@ jobs: yamllint: name: yamllint + needs: prep-review + if: needs.prep-review.outputs.yamllint == 'true' runs-on: ubuntu-latest steps: - name: Git clone repository @@ -84,13 +152,7 @@ jobs: with: submodules: recursive fetch-depth: 0 - - name: Check files - run: | - if git diff --name-only origin/${{ github.base_ref }} HEAD | grep -Eq '*\.ya?ml|python\.d/.*\.conf' ; then - echo 'run_yamllint=1' >> $GITHUB_ENV - fi - name: Run yamllint - if: env.run_yamllint == 1 uses: reviewdog/action-yamllint@v1 with: github_token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 7e47f12da..20c6f7b8c 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -14,6 +14,11 @@ on: - 'CMakeLists.txt' - '**.c' - '**.h' +env: + DO_NOT_TRACK: 1 +concurrency: + group: tests-${{ github.ref }} + cancel-in-progress: true jobs: unit-tests-legacy: name: Unit Tests (legacy) @@ -52,7 +57,7 @@ jobs: - name: Configure run: | autoreconf -ivf - ./configure --without-aclk-ng + ./configure --disable-ml # XXX: Work-around for bug with libbson-1.0 in Ubuntu 18.04 # See: https://bugs.launchpad.net/ubuntu/+source/libmongoc/+bug/1790771 # https://jira.mongodb.org/browse/CDRIVER-2818 @@ -85,7 +90,7 @@ jobs: find . -type f -name '*.log' -exec cp {} ../logs/ \; popd || exit 1 - name: Upload Artifacts - uses: actions/upload-artifact@v2.2.3 + uses: actions/upload-artifact@v2.2.4 if: always() with: name: logs diff --git a/.github/workflows/updater.yml b/.github/workflows/updater.yml deleted file mode 100644 index 5f8d77c4a..000000000 --- a/.github/workflows/updater.yml +++ /dev/null @@ -1,85 +0,0 @@ ---- -name: Updater -on: - push: - branches: - - master - pull_request: - branches: - - master - -jobs: - source-build: - name: Install, Build & Update - strategy: - fail-fast: false - max-parallel: 8 - matrix: - distro: - - 'alpine:3.12' - - 'alpine:3.13' - - 'archlinux:latest' - - 'centos:7' - - 'centos:8' - - 'clearlinux:latest' - - 'debian:9' - - 'debian:10' - - 'fedora:33' - - 'fedora:34' - - 'ubuntu:16.04' - - 'ubuntu:18.04' - - 'ubuntu:20.04' - - 'ubuntu:20.10' - - 'ubuntu:21.04' - include: - - distro: 'alpine:3.12' - pre: 'apk add -U bash' - - distro: 'alpine:3.13' - pre: 'apk add -U bash' - - distro: 'debian:9' - pre: 'apt-get update' - - distro: 'debian:10' - pre: 'apt-get update' - - distro: 'ubuntu:16.04' - pre: 'apt-get update' - - distro: 'ubuntu:18.04' - pre: 'apt-get update' - - distro: 'ubuntu:20.04' - pre: 'apt-get update' - - distro: 'ubuntu:20.10' - pre: 'apt-get update' - - distro: 'ubuntu:21.04' - pre: 'apt-get update' - runs-on: ubuntu-latest - steps: - - name: Git clone repository - uses: actions/checkout@v2 - with: - submodules: recursive - - name: Install required packages & build tarball - run: | - ./packaging/installer/install-required-packages.sh --dont-wait --non-interactive netdata-all - .github/scripts/build-dist.sh - - name: Run a dockerised web server to serve files used by the custom update script - run: | - docker run -dit --name my-apache-app -p 8080:80 -v "$PWD":/usr/local/apache2/htdocs/ httpd:2.4 - - name: Replace URLs in updater script to point at the local web server - run: | - ORIG_TARBALL="export NETDATA_TARBALL_URL=.*" - ORIG_CHECKSUM="export NETDATA_TARBALL_CHECKSUM_URL=.*" - CURRENT_VERSION="current_version=.*" - NEW_TARBALL="export NETDATA_TARBALL_URL=http://localhost:8080/artifacts/netdata-latest.tar.gz" - NEW_CHECKSUM="export NETDATA_TARBALL_CHECKSUM_URL=http://localhost:8080/artifacts/sha256sums.txt" - sed -i "s|${ORIG_TARBALL}|${NEW_TARBALL}|g" packaging/installer/netdata-updater.sh - sed -i "s|${ORIG_CHECKSUM}|${NEW_CHECKSUM}|g" packaging/installer/netdata-updater.sh - sed -i "s|"current_version=.*"|"current_version=1"|g" packaging/installer/netdata-updater.sh - - name: Install netdata and run the updater on ${{ matrix.distro }} - env: - PRE: ${{ matrix.pre }} - run: | - echo $PRE > ./prep-cmd.sh - docker build . -f .github/dockerfiles/Dockerfile.build_test -t test --build-arg BASE=${{ matrix.distro }} - docker run --network host -w /netdata test \ - /bin/sh -c '/netdata/packaging/installer/kickstart.sh --dont-wait \ - && /netdata/packaging/installer/netdata-updater.sh --not-running-from-cron --no-updater-self-update \ - && bash /netdata/.github/scripts/check-updater.sh' |