diff options
Diffstat (limited to '.github/workflows/build.yml')
| -rw-r--r-- | .github/workflows/build.yml | 283 |
1 files changed, 234 insertions, 49 deletions
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index abf08c2ea..caa49e290 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -17,7 +17,7 @@ on: default: nightly required: true concurrency: # This keeps multiple instances of the job from running concurrently for the same ref and event type. - group: release-${{ github.ref }}-${{ github.event_name }} + group: build-${{ github.ref }}-${{ github.event_name }} cancel-in-progress: true jobs: build-dist: # Build the distribution tarball and store it as an artifact. @@ -27,11 +27,13 @@ jobs: distfile: ${{ steps.build.outputs.distfile }} steps: - name: Checkout - uses: actions/checkout@v2 + id: checkout + uses: actions/checkout@v3 with: fetch-depth: 0 submodules: recursive - name: Mark Stable + id: channel if: github.event_name == 'workflow_dispatch' && github.event.inputs.type != 'nightly' run: | sed -i 's/^RELEASE_CHANNEL="nightly" *#/RELEASE_CHANNEL="stable" #/' netdata-installer.sh @@ -52,7 +54,8 @@ jobs: echo "::set-output name=distfile::$(find . -name 'netdata-*.tar.gz')" cp netdata-*.tar.gz artifacts/ - name: Store - uses: actions/upload-artifact@v2 + id: store + uses: actions/upload-artifact@v3 with: name: dist-tarball path: artifacts/*.tar.gz @@ -63,9 +66,14 @@ jobs: SLACK_COLOR: 'danger' SLACK_FOOTER: '' SLACK_ICON_EMOJI: ':github-actions:' - SLACK_TITLE: 'Distribution tarball build failed:' + SLACK_TITLE: 'Distribution tarball creation failed:' SLACK_USERNAME: 'GitHub Actions' - SLACK_MESSAGE: "Distribution tarball build failed." + SLACK_MESSAGE: |- + ${{ github.repository }}: Failed to create source tarball for distribution. + Checkout: ${{ steps.checkout.outcome }} + Mark stable: ${{ steps.channel.outcome }} + Build: ${{ steps.build.outcome }} + Store: ${{ steps.store.outcome }} SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }} if: >- ${{ @@ -86,18 +94,22 @@ jobs: - 'ppc64le' steps: - name: Checkout - uses: actions/checkout@v2 + id: checkout + uses: actions/checkout@v3 with: fetch-depth: 0 submodules: recursive - name: Mark Stable + id: channel if: github.event_name == 'workflow_dispatch' && github.event.inputs.type != 'nightly' run: | sed -i 's/^RELEASE_CHANNEL="nightly" *#/RELEASE_CHANNEL="stable" #/' netdata-installer.sh packaging/makeself/install-or-update.sh - name: Build + id: build run: .github/scripts/build-static.sh ${{ matrix.arch }} - name: Store - uses: actions/upload-artifact@v2 + id: store + uses: actions/upload-artifact@v3 with: name: static-archive path: artifacts/*.gz.run @@ -108,9 +120,14 @@ jobs: SLACK_COLOR: 'danger' SLACK_FOOTER: '' SLACK_ICON_EMOJI: ':github-actions:' - SLACK_TITLE: 'Static build for ${{ matrix.arch }} failed:' + SLACK_TITLE: 'Static build failed:' SLACK_USERNAME: 'GitHub Actions' - SLACK_MESSAGE: "Static build for ${{ matrix.arch }} failed." + SLACK_MESSAGE: |- + ${{ github.repository }}: Failed to create static installer archive for ${{ matrix.arch }}. + Checkout: ${{ steps.checkout.outcome }} + Mark stable: ${{ steps.channel.outcome }} + Build: ${{ steps.build.outcome }} + Store: ${{ steps.store.outcome }} SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }} if: >- ${{ @@ -126,33 +143,112 @@ jobs: matrix: ${{ steps.set-matrix.outputs.matrix }} steps: - name: Checkout - uses: actions/checkout@v2 + id: checkout + uses: actions/checkout@v3 - name: Prepare tools + id: prepare run: | - sudo apt-get update && sudo apt-get install -y jq + sudo apt-get update && sudo apt-get install -y python3-ruamel.yaml - name: Read build matrix id: set-matrix + shell: python3 {0} run: | - TASKS="$(jq -c . .github/data/build-matrix.json)" - echo "Generated Matrix: $TASKS" - echo "::set-output name=matrix::$TASKS" + from ruamel.yaml import YAML + import json + yaml = YAML(typ='safe') + with open('.github/data/distros.yml') as f: + data = yaml.load(f) + del data['platform_map'] + for i, v in enumerate(data['include']): + data['include'][i]['artifact_key'] = data['include'][i]['distro'] + str(data['include'][i]['version']).replace('.', '') + if 'packages' in data['include'][i]: + del data['include'][i]['packages'] + if 'base_image' in data['include'][i]: + data['include'][i]['distro'] = data['include'][i]['base_image'] + del data['include'][i]['base_image'] + data['include'][i]['distro'] = ':'.join([data['include'][i]['distro'], str(data['include'][i]['version'])]) + del data['include'][i]['version'] + matrix = json.dumps(data, sort_keys=True) + print('Generated Matrix: ' + matrix) + print('::set-output name=matrix::' + matrix) + - name: Failure Notification + uses: rtCamp/action-slack-notify@v2 + env: + SLACK_COLOR: 'danger' + SLACK_FOOTER: '' + SLACK_ICON_EMOJI: ':github-actions:' + SLACK_TITLE: 'Build matrix preparation failed:' + SLACK_USERNAME: 'GitHub Actions' + SLACK_MESSAGE: |- + ${{ github.repository }}: Failed to prepare build matrix for build checks. + Checkout: ${{ steps.checkout.outcome }} + Prepare tools: ${{ steps.prepare.outcome }} + Read build matrix: ${{ steps.set-matrix.outcome }} + SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }} + if: >- + ${{ + failure() + && startsWith(github.ref, 'refs/heads/master') + && github.event_name != 'pull_request' + }} prepare-test-images: # Prepare the test environments for our build checks. This also checks dependency handling code for each tested environment. name: Prepare Test Environments runs-on: ubuntu-latest needs: - matrix + env: + RETRY_DELAY: 300 strategy: # Unlike the actal build tests, this completes _very_ fast (average of about 3 minutes for each job), so we # just run everything in parallel instead lof limiting job concurrency. fail-fast: false matrix: ${{ fromJson(needs.matrix.outputs.matrix) }} steps: - - name: Git clone repository - uses: actions/checkout@v2 + - name: Checkout + id: checkout + uses: actions/checkout@v3 - name: Setup Buildx + id: buildx uses: docker/setup-buildx-action@v1 - name: Build test environment + id: build1 + uses: docker/build-push-action@v2 + continue-on-error: true # We retry 3 times at 5 minute intervals if there is a failure here. + with: + push: false + load: false + file: .github/dockerfiles/Dockerfile.build_test + build-args: | + BASE=${{ matrix.distro }} + PRE=${{ matrix.env_prep }} + RMJSONC=${{ matrix.jsonc_removal }} + outputs: type=oci,dest=/tmp/image.tar + tags: test:${{ matrix.artifact_key }} + - name: Retry delay + if: ${{ steps.build1.outcome }} == 'failure' + run: sleep "${RETRY_DELAY}" + - name: Build test environment (attempt 2) + if: ${{ steps.build1.outcome }} == 'failure' + id: build2 + uses: docker/build-push-action@v2 + continue-on-error: true # We retry 3 times at 5 minute intervals if there is a failure here. + with: + push: false + load: false + file: .github/dockerfiles/Dockerfile.build_test + build-args: | + BASE=${{ matrix.distro }} + PRE=${{ matrix.env_prep }} + RMJSONC=${{ matrix.jsonc_removal }} + outputs: type=oci,dest=/tmp/image.tar + tags: test:${{ matrix.artifact_key }} + - name: Retry delay + if: ${{ steps.build1.outcome }} == 'failure' && ${{ steps.build2.outcome }} == 'failure' + run: sleep "${RETRY_DELAY}" + - name: Build test environment (attempt 3) + if: ${{ steps.build1.outcome }} == 'failure' && ${{ steps.build2.outcome }} == 'failure' + id: build3 uses: docker/build-push-action@v2 with: push: false @@ -160,12 +256,13 @@ jobs: file: .github/dockerfiles/Dockerfile.build_test build-args: | BASE=${{ matrix.distro }} - PRE=${{ matrix.pre }} - RMJSONC=${{ matrix.rmjsonc }} + PRE=${{ matrix.env_prep }} + RMJSONC=${{ matrix.jsonc_removal }} outputs: type=oci,dest=/tmp/image.tar tags: test:${{ matrix.artifact_key }} - name: Upload image artifact - uses: actions/upload-artifact@v2 + id: upload + uses: actions/upload-artifact@v3 with: name: ${{ matrix.artifact_key }}-test-env path: /tmp/image.tar @@ -178,7 +275,14 @@ jobs: SLACK_ICON_EMOJI: ':github-actions:' SLACK_TITLE: 'Test environment preparation for ${{ matrix.distro }} failed:' SLACK_USERNAME: 'GitHub Actions' - SLACK_MESSAGE: "Test environment preparation for ${{ matrix.distro }} failed." + SLACK_MESSAGE: |- + ${{ github.repository }}: Test environment preparation for ${{ matrix.distro }} failed. + Checkout: ${{ steps.checkout.outcome }} + Set up Buildx: ${{ steps.buildx.outcome }} + Build test environment: ${{ steps.build1.outcome }} + Build test environment (attempt 2): ${{ steps.build2.outcome }} + Build test environment (attempt 3): ${{ steps.build3.outcome }} + Upload: ${{ steps.upload.outcome }} SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }} if: >- ${{ @@ -198,12 +302,14 @@ jobs: max-parallel: 8 matrix: ${{ fromJson(needs.matrix.outputs.matrix) }} steps: - - name: Git clone repository - uses: actions/checkout@v2 + - name: Checkout + id: checkout + uses: actions/checkout@v3 with: submodules: recursive - name: Fetch test environment - uses: actions/download-artifact@v2 + id: fetch + uses: actions/download-artifact@v3 with: name: ${{ matrix.artifact_key }}-test-env - name: Load test environment @@ -212,19 +318,23 @@ jobs: docker load --input image.tar | tee image-info.txt echo "::set-output name=image::$(cut -d ':' -f 3 image-info.txt)" - name: Regular build on ${{ matrix.distro }} + id: build-basic run: | docker run --security-opt seccomp=unconfined -w /netdata sha256:${{ steps.load.outputs.image }} \ /bin/sh -c 'autoreconf -ivf && ./configure && make -j2' - name: netdata-installer on ${{ matrix.distro }}, disable cloud + id: build-no-cloud run: | docker run --security-opt seccomp=unconfined -w /netdata sha256:${{ steps.load.outputs.image }} \ /bin/sh -c './netdata-installer.sh --dont-wait --dont-start-it --disable-cloud' - name: netdata-installer on ${{ matrix.distro }}, require cloud + id: build-cloud run: | docker run --security-opt seccomp=unconfined -w /netdata sha256:${{ steps.load.outputs.image }} \ /bin/sh -c './netdata-installer.sh --dont-wait --dont-start-it --require-cloud' - name: netdata-installer on ${{ matrix.distro }}, require cloud, no JSON-C - if: matrix.rmjsonc != '' + id: build-no-jsonc + if: matrix.jsonc_removal != '' run: | docker run --security-opt seccomp=unconfined -w /netdata sha256:${{ steps.load.outputs.image }} \ /bin/sh -c '/rmjsonc.sh && ./netdata-installer.sh --dont-wait --dont-start-it --require-cloud' @@ -236,7 +346,15 @@ jobs: SLACK_ICON_EMOJI: ':github-actions:' SLACK_TITLE: 'Build tests for ${{ matrix.distro }} failed:' SLACK_USERNAME: 'GitHub Actions' - SLACK_MESSAGE: "Build tests for ${{ matrix.distro }} failed." + SLACK_MESSAGE: |- + ${{ github.repository }}: Build tests for ${{ matrix.distro }} failed. + Checkout: ${{ steps.checkout.outcome }} + Fetch test environment: ${{ steps.fetch.outcome }} + Load test environment: ${{ steps.load.outcome }} + Regular build: ${{ steps.build-basic.outcome }} + netdata-installer, disable cloud: ${{ steps.build-no-cloud.outcome }} + netdata-installer, require cloud: ${{ steps.build-cloud.outcome }} + netdata-installer, no JSON-C: ${{ steps.build-no-jsonc.outcome }} SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }} if: >- ${{ @@ -265,13 +383,16 @@ jobs: - ${{ github.workspace }}:/usr/local/apache2/htdocs/ steps: - name: Checkout - uses: actions/checkout@v2 + id: checkout + uses: actions/checkout@v3 - name: Fetch dist tarball artifacts - uses: actions/download-artifact@v2 + id: fetch-tarball + uses: actions/download-artifact@v3 with: name: dist-tarball path: dist-tarball - name: Prepare artifact directory + id: prepare run: | mkdir -p artifacts || exit 1 echo "9999.0.0-0" > artifacts/latest-version.txt || exit 1 @@ -281,7 +402,8 @@ jobs: sha256sum -b ./* > "sha256sums.txt" || exit 1 cat sha256sums.txt - name: Fetch test environment - uses: actions/download-artifact@v2 + id: fetch-test-environment + uses: actions/download-artifact@v3 with: name: ${{ matrix.artifact_key }}-test-env - name: Load test environment @@ -290,8 +412,9 @@ jobs: docker load --input image.tar | tee image-info.txt echo "::set-output name=image::$(cut -d ':' -f 3 image-info.txt)" - name: Install netdata and run the updater on ${{ matrix.distro }} + id: updater-check run: | - docker run --security-opt seccomp=unconfined -e DO_NOT_TRACK=1 --network host -w /netdata sha256:${{ steps.load.outputs.image }} \ + docker run --security-opt seccomp=unconfined -e DISABLE_TELEMETRY=1 --network host -w /netdata sha256:${{ steps.load.outputs.image }} \ /netdata/.github/scripts/run-updater-check.sh - name: Failure Notification uses: rtCamp/action-slack-notify@v2 @@ -301,7 +424,14 @@ jobs: SLACK_ICON_EMOJI: ':github-actions:' SLACK_TITLE: 'Updater checks for ${{ matrix.distro }} failed:' SLACK_USERNAME: 'GitHub Actions' - SLACK_MESSAGE: "Updater checks for ${{ matrix.distro }} failed." + SLACK_MESSAGE: |- + ${{ github.repository }}: Updater checks for ${{ matrix.distro }} failed. + Checkout: ${{ steps.checkout.outcome }} + Fetch dist tarball: ${{ steps.fetch-tarball.outcome }} + Prepare artifact directory: ${{ steps.prepare.outcome }} + Fetch test environment: ${{ steps.fetch-test-environment.outcome }} + Load test environment: ${{ steps.load.outcome }} + Updater check: ${{ steps.updater-check.outcome }} SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }} if: >- ${{ @@ -318,20 +448,25 @@ jobs: - build-static steps: - name: Checkout - uses: actions/checkout@v2 + id: checkout + uses: actions/checkout@v3 - name: Prepare Environment + id: prepare run: mkdir -p artifacts - name: Retrieve Dist Tarball - uses: actions/download-artifact@v2 + id: fetch-dist + uses: actions/download-artifact@v3 with: name: dist-tarball path: dist-tarball - name: Retrieve Static Build Artifacts - uses: actions/download-artifact@v2 + id: fetch-static + uses: actions/download-artifact@v3 with: name: static-archive path: static-archive - name: Prepare Artifacts + id: consolidate working-directory: ./artifacts/ run: | mv ../dist-tarball/* . || exit 1 @@ -341,7 +476,8 @@ jobs: sha256sum -b ./* > sha256sums.txt || exit 1 cat sha256sums.txt - name: Store Artifacts - uses: actions/upload-artifact@v2 + id: store + uses: actions/upload-artifact@v3 with: name: final-artifacts path: artifacts/* @@ -354,7 +490,14 @@ jobs: SLACK_ICON_EMOJI: ':github-actions:' SLACK_TITLE: 'Failed to prepare release artifacts for upload:' SLACK_USERNAME: 'GitHub Actions' - SLACK_MESSAGE: "Failed to prepare release artifacts for upload." + SLACK_MESSAGE: |- + ${{ github.repository }}: Failed to prepare release artifacts for upload. + CHeckout: ${{ steps.checkout.outcome }} + Prepare environment: ${{ steps.prepare.outcome }} + Fetch dist tarball: ${{ steps.fetch-dist.outcome }} + Fetch static builds: ${{ steps.fetch-static.outcome }} + Consolidate artifacts: ${{ steps.consolidate.outcome }} + Store: ${{ steps.store.outcome }} SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }} if: >- ${{ @@ -377,13 +520,16 @@ jobs: - ${{ github.workspace }}:/usr/local/apache2/htdocs/ steps: - name: Checkout - uses: actions/checkout@v2 + id: checkout + uses: actions/checkout@v3 - name: Fetch artifacts - uses: actions/download-artifact@v2 + id: fetch + uses: actions/download-artifact@v3 with: name: final-artifacts path: artifacts - name: Verify that artifacts work with installer + id: verify env: NETDATA_TARBALL_BASEURL: http://localhost:8080/artifacts run: packaging/installer/kickstart.sh --build-only --dont-start-it --disable-telemetry --dont-wait @@ -395,7 +541,11 @@ jobs: SLACK_ICON_EMOJI: ':github-actions:' SLACK_TITLE: 'Artifact verification for source tarball failed.' SLACK_USERNAME: 'GitHub Actions' - SLACK_MESSAGE: "Artifact verification for source tarball failed." + SLACK_MESSAGE: |- + ${{ github.repository }}: Artifact verification for source tarball failed. + Checkout: ${{ steps.checkout.outcome }} + Fetch artifacts: ${{ steps.fetch.outcome }} + Verify artifacts: ${{ steps.verify.outcome }} SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }} if: >- ${{ @@ -418,13 +568,16 @@ jobs: - ${{ github.workspace }}:/usr/local/apache2/htdocs/ steps: - name: Checkout - uses: actions/checkout@v2 + id: checkout + uses: actions/checkout@v3 - name: Fetch artifacts - uses: actions/download-artifact@v2 + id: fetch-artifacts + uses: actions/download-artifact@v3 with: name: final-artifacts path: artifacts - name: Verify that artifacts work with installer + id: verify env: NETDATA_TARBALL_BASEURL: http://localhost:8080/artifacts run: packaging/installer/kickstart.sh --static-only --dont-start-it --disable-telemetry @@ -436,7 +589,11 @@ jobs: SLACK_ICON_EMOJI: ':github-actions:' SLACK_TITLE: 'Artifact verification for static build failed.' SLACK_USERNAME: 'GitHub Actions' - SLACK_MESSAGE: "Artifact verification for static build failed." + SLACK_MESSAGE: |- + ${{ github.repository }}: Artifact verification for static build failed. + Checkout: ${{ steps.checkout.outcome }} + Fetch artifacts: ${{ steps.fetch-artifacts.outcome }} + Verify artifacts: ${{ steps.verify.outcome }} SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }} if: >- ${{ @@ -456,18 +613,21 @@ jobs: - artifact-verification-static steps: - name: Retrieve Artifacts - uses: actions/download-artifact@v2 + id: fetch + uses: actions/download-artifact@v3 with: name: final-artifacts path: final-artifacts - name: Setup Gcloud - uses: google-github-actions/setup-gcloud@v0.5.0 + id: gcloud + uses: google-github-actions/setup-gcloud@v0.6.0 with: project_id: ${{ secrets.GCP_NIGHTLY_STORAGE_PROJECT }} service_account_key: ${{ secrets.GCP_STORAGE_SERVICE_ACCOUNT_KEY }} export_default_credentials: true - name: Upload Artifacts - uses: google-github-actions/upload-cloud-storage@v0.5.0 + id: upload + uses: google-github-actions/upload-cloud-storage@v0.9.0 with: destination: ${{ secrets.GCP_NIGHTLY_STORAGE_BUCKET }} gzip: false @@ -481,7 +641,11 @@ jobs: SLACK_ICON_EMOJI: ':github-actions:' SLACK_TITLE: 'Failed to upload nightly release artifacts:' SLACK_USERNAME: 'GitHub Actions' - SLACK_MESSAGE: "Failed to upload nightly release artifacts." + SLACK_MESSAGE: |- + ${{ github.repository }}: Failed to upload nightly release artifacts. + Fetch artifacts: ${{ steps.fetch.outcome }} + Setup GCloud: ${{ steps.gcloud.outcome }} + Upload artifacts: ${{ steps.upload.outcome }} SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }} if: >- ${{ @@ -501,13 +665,16 @@ jobs: - artifact-verification-static steps: - name: Checkout - uses: actions/checkout@v2 + id: checkout + uses: actions/checkout@v3 - name: Retrieve Artifacts - uses: actions/download-artifact@v2 + id: fetch + uses: actions/download-artifact@v3 with: name: final-artifacts path: final-artifacts - name: Create Release + id: create-release uses: ncipollo/release-action@v1 with: allowUpdates: false @@ -524,11 +691,29 @@ jobs: SLACK_ICON_EMOJI: ':github-actions:' SLACK_TITLE: 'Failed to draft release:' SLACK_USERNAME: 'GitHub Actions' - SLACK_MESSAGE: "Failed to draft release." + SLACK_MESSAGE: |- + ${{ github.repository }}: Failed to create draft release or attach artifacts. + Checkout: ${{ steps.checkout.outcome }} + Fetch artifacts: ${{ steps.fetch.outcome }} + Create draft release: ${{ steps.create-release.outcome }} SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }} if: >- ${{ failure() - && startsWith(github.ref, 'refs/heads/master') + && github.event_name == 'workflow_dispatch' + }} + - name: Success Notification + uses: rtCamp/action-slack-notify@v2 + env: + SLACK_COLOR: 'good' + SLACK_FOOTER: '' + SLACK_ICON_EMOJI: ':github-actions:' + SLACK_TITLE: 'Created agent draft release:' + SLACK_USERNAME: 'GitHub Actions' + SLACK_MESSAGE: "${{ github.repository }}: ${{ steps.create-release.outputs.html_url }}" + SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }} + if: >- + ${{ + success() && github.event_name == 'workflow_dispatch' }} |