diff options
Diffstat (limited to '.github/workflows/packaging.yml')
| -rw-r--r-- | .github/workflows/packaging.yml | 268 |
1 files changed, 174 insertions, 94 deletions
diff --git a/.github/workflows/packaging.yml b/.github/workflows/packaging.yml index 38a78731b..dbf334d59 100644 --- a/.github/workflows/packaging.yml +++ b/.github/workflows/packaging.yml @@ -6,164 +6,235 @@ on: branches: - master - develop + push: + branches: + - master workflow_dispatch: inputs: type: - name: Package build type + description: Package build type default: devel required: true version: - name: Package version + description: Package version required: false env: - DO_NOT_TRACK: 1 + DISABLE_TELEMETRY: 1 + REPO_PREFIX: netdata/netdata concurrency: group: packages-${{ github.ref }}-${{ github.event_name }} cancel-in-progress: true jobs: + matrix: + name: Prepare Build Matrix + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.set-matrix.outputs.matrix }} + steps: + - name: Checkout + id: checkout + uses: actions/checkout@v3 + - name: Prepare tools + id: prepare + run: | + sudo apt-get update && sudo apt-get install -y python3-ruamel.yaml + - name: Read build matrix + id: set-matrix + shell: python3 {0} + run: | + from ruamel.yaml import YAML + import json + import re + FULL_CI_REGEX = '/actions run full ci' + ALWAYS_RUN_ARCHES = ["amd64"] + PR_BODY = """${{ github.event.pull_request.body }}""" + yaml = YAML(typ='safe') + entries = list() + run_limited = False + + with open('.github/data/distros.yml') as f: + data = yaml.load(f) + + if "${{ github.event_name }}" == "pull_request" and re.search(FULL_CI_REGEX, PR_BODY, re.I) is None: + run_limited = True + + for i, v in enumerate(data['include']): + if 'packages' in data['include'][i]: + for arch in data['include'][i]['packages']['arches']: + if arch in ALWAYS_RUN_ARCHES or not run_limited: + entries.append({ + 'distro': data['include'][i]['distro'], + 'version': data['include'][i]['version'], + 'pkgclouddistro': data['include'][i]['packages']['repo_distro'], + 'format': data['include'][i]['packages']['type'], + 'base_image': data['include'][i]['base_image'] if 'base_image' in data['include'][i] else data['include'][i]['distro'], + 'platform': data['platform_map'][arch], + 'arch': arch + }) + + entries.sort(key=lambda k: (k['arch'], k['distro'], k['version'])) + matrix = json.dumps({'include': entries}, sort_keys=True) + print('Generated Matrix: ' + matrix) + print('::set-output name=matrix::' + matrix) + - name: Failure Notification + uses: rtCamp/action-slack-notify@v2 + env: + SLACK_COLOR: 'danger' + SLACK_ICON_EMOJI: ':github-actions:' + SLACK_TITLE: 'Package Build matrix generation failed:' + SLACK_USERNAME: 'GitHub Actions' + SLACK_MESSAGE: |- + ${{ github.repository }}: Failed to generate build matrix for package build. + Checkout: ${{ steps.checkout.outcome }} + Prepare Tools: ${{ steps.prepare.outcome }} + Read Build Matrix: ${{ steps.set-matrix.outcome }} + SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }} + if: >- + ${{ + failure() + && github.event_name != 'pull_request' + && startsWith(github.ref, 'refs/heads/master') + }} + + version-check: + name: Version check + runs-on: ubuntu-latest + outputs: + repo: ${{ steps.check-version.outputs.repo }} + version: ${{ steps.check-version.outputs.version }} + retention: ${{ steps.check-version.outputs.retention }} + steps: + - name: Checkout + id: checkout + uses: actions/checkout@v3 + - name: Check Version + id: check-version + run: | + if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then + case "${{ github.event.inputs.type }}" in + "release") + echo "::set-output name=repo::${REPO_PREFIX}" + echo "::set-output name=version::${{ github.event.inputs.version }}" + echo "::set-output name=retention::365" + ;; + "nightly") + echo "::set-output name=repo::${REPO_PREFIX}-edge" + echo "::set-output name=version::$(tr -d 'v' < packaging/version)" + echo "::set-output name=retention::30" + ;; + *) + echo "::set-output name=repo::${REPO_PREFIX}-devel" + echo "::set-output name=version::0.${GITHUB_SHA}" + echo "::set-output name=retention::30" + ;; + esac + else + echo "::set-output name=version::$(cut -d'-' -f 1 packaging/version | tr -d 'v')" + echo "::set-output name=retention::0" + fi + - name: Failure Notification + uses: rtCamp/action-slack-notify@v2 + env: + SLACK_COLOR: 'danger' + SLACK_ICON_EMOJI: ':github-actions:' + SLACK_TITLE: 'Package Build version check failed:' + SLACK_USERNAME: 'GitHub Actions' + SLACK_MESSAGE: |- + ${{ github.repository }}: Failed to generate version information for package build. + Checkout: ${{ steps.checkout.outcome }} + Check Version: ${{ steps.check-version.outcome }} + SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }} + if: >- + ${{ + failure() + && github.event_name != 'pull_request' + && startsWith(github.ref, 'refs/heads/master') + }} + build: name: Build runs-on: ubuntu-latest env: DOCKER_CLI_EXPERIMENTAL: enabled + needs: + - matrix + - version-check strategy: - matrix: - include: - - {distro: debian, version: "9", pkgclouddistro: debian/stretch, format: deb, base_image: debian, platform: linux/amd64, arch: amd64} - - {distro: debian, version: "9", pkgclouddistro: debian/stretch, format: deb, base_image: debian, platform: linux/i386, arch: i386} - - {distro: debian, version: "9", pkgclouddistro: debian/stretch, format: deb, base_image: debian, platform: linux/arm/v7, arch: armhf} - - {distro: debian, version: "9", pkgclouddistro: debian/stretch, format: deb, base_image: debian, platform: linux/arm64/v8, arch: arm64} - - {distro: debian, version: "10", pkgclouddistro: debian/buster, format: deb, base_image: debian, platform: linux/amd64, arch: amd64} - - {distro: debian, version: "10", pkgclouddistro: debian/buster, format: deb, base_image: debian, platform: linux/i386, arch: i386} - - {distro: debian, version: "10", pkgclouddistro: debian/buster, format: deb, base_image: debian, platform: linux/arm/v7, arch: armhf} - - {distro: debian, version: "10", pkgclouddistro: debian/buster, format: deb, base_image: debian, platform: linux/arm64/v8, arch: arm64} - - {distro: debian, version: "11", pkgclouddistro: debian/bullseye, format: deb, base_image: debian, platform: linux/amd64, arch: amd64, alias: bullseye} - - {distro: debian, version: "11", pkgclouddistro: debian/bullseye, format: deb, base_image: debian, platform: linux/i386, arch: i386, alias: bullseye} - - {distro: debian, version: "11", pkgclouddistro: debian/bullseye, format: deb, base_image: debian, platform: linux/arm/v7, arch: armhf, alias: bullseye} - - {distro: debian, version: "11", pkgclouddistro: debian/bullseye, format: deb, base_image: debian, platform: linux/arm64/v8, arch: arm64, alias: bullseye} - - {distro: ubuntu, version: "18.04", pkgclouddistro: ubuntu/bionic, format: deb, base_image: ubuntu, platform: linux/amd64, arch: amd64} - - {distro: ubuntu, version: "18.04", pkgclouddistro: ubuntu/bionic, format: deb, base_image: ubuntu, platform: linux/i386, arch: i386} - - {distro: ubuntu, version: "18.04", pkgclouddistro: ubuntu/bionic, format: deb, base_image: ubuntu, platform: linux/arm/v7, arch: armhf} - - {distro: ubuntu, version: "18.04", pkgclouddistro: ubuntu/bionic, format: deb, base_image: ubuntu, platform: linux/arm64/v8, arch: arm64} - - {distro: ubuntu, version: "20.04", pkgclouddistro: ubuntu/focal, format: deb, base_image: ubuntu, platform: linux/amd64, arch: amd64} - - {distro: ubuntu, version: "20.04", pkgclouddistro: ubuntu/focal, format: deb, base_image: ubuntu, platform: linux/arm/v7, arch: armhf} - - {distro: ubuntu, version: "20.04", pkgclouddistro: ubuntu/focal, format: deb, base_image: ubuntu, platform: linux/arm64/v8, arch: arm64} - - {distro: ubuntu, version: "21.10", pkgclouddistro: ubuntu/impish, format: deb, base_image: ubuntu, platform: linux/amd64, arch: amd64} - - {distro: ubuntu, version: "21.10", pkgclouddistro: ubuntu/impish, format: deb, base_image: ubuntu, platform: linux/arm/v7, arch: armhf} - - {distro: ubuntu, version: "21.10", pkgclouddistro: ubuntu/impish, format: deb, base_image: ubuntu, platform: linux/arm64/v8, arch: arm64} - - {distro: centos, version: "7", pkgclouddistro: el/7, format: rpm, base_image: centos, platform: linux/amd64, arch: amd64} - - {distro: rockylinux, version: "8", pkgclouddistro: el/8, format: rpm, base_image: rockylinux/rockylinux, platform: linux/amd64, arch: amd64} - - {distro: rockylinux, version: "8", pkgclouddistro: el/8, format: rpm, base_image: rockylinux/rockylinux, platform: linux/arm64/v8, arch: arm64} - - {distro: fedora, version: "34", pkgclouddistro: fedora/34, format: rpm, base_image: fedora, platform: linux/amd64, arch: amd64} - - {distro: fedora, version: "34", pkgclouddistro: fedora/34, format: rpm, base_image: fedora, platform: linux/arm/v7, arch: armhf} - - {distro: fedora, version: "34", pkgclouddistro: fedora/34, format: rpm, base_image: fedora, platform: linux/arm64/v8, arch: arm64} - - {distro: fedora, version: "35", pkgclouddistro: fedora/35, format: rpm, base_image: fedora, platform: linux/amd64, arch: amd64} - - {distro: fedora, version: "35", pkgclouddistro: fedora/35, format: rpm, base_image: fedora, platform: linux/arm/v7, arch: armhf} - - {distro: fedora, version: "35", pkgclouddistro: fedora/35, format: rpm, base_image: fedora, platform: linux/arm64/v8, arch: arm64} - - {distro: opensuse, version: "15.3", pkgclouddistro: opensuse/15.3, format: rpm, base_image: opensuse/leap, platform: linux/amd64, arch: amd64} - - {distro: opensuse, version: "15.3", pkgclouddistro: opensuse/15.3, format: rpm, base_image: opensuse/leap, platform: linux/arm64/v8, arch: arm64} - - {distro: oraclelinux, version: "8", pkgclouddistro: ol/8, format: rpm, base_image: oraclelinux, platform: linux/amd64, arch: amd64} - - {distro: oraclelinux, version: "8", pkgclouddistro: ol/8, format: rpm, base_image: oraclelinux, platform: linux/arm64/v8, arch: arm64} + matrix: ${{ fromJson(needs.matrix.outputs.matrix) }} # We intentiaonally disable the fail-fast behavior so that a # build failure for one version doesn't prevent us from publishing # successfully built and tested packages for another version. fail-fast: false max-parallel: 8 steps: - - name: Checkout PR # Checkout the PR if it's a PR. - if: github.event_name == 'pull_request' - uses: actions/checkout@v2 - with: - fetch-depth: 0 # We need full history for versioning - submodules: recursive - - name: Checkout Tag # Otherwise check out the tag that triggered this. - if: github.event_name == 'workflow_dispatch' - uses: actions/checkout@v2 + - name: Checkout + id: checkout + uses: actions/checkout@v3 with: - ref: ${{ github.event.ref }} fetch-depth: 0 # We need full history for versioning submodules: recursive - - name: Check Base Branch - run: | - if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then - echo "runtype=${{ github.event.inputs.type }}" >> $GITHUB_ENV - case "${{ github.event.inputs.type }}" in - "release") - echo "repo=${{ secrets.PACKAGE_CLOUD_REPO }}" >> $GITHUB_ENV - echo "pkg_version=${{ github.event.inputs.version }}" >> $GITHUB_ENV - echo "pkg_retention_days=365" >> $GITHUB_ENV - ;; - "nightly") - echo "repo=${{ secrets.PACKAGE_CLOUD_REPO }}-edge" >> $GITHUB_ENV - echo "pkg_version=${{ github.event.inputs.version }}" >> $GITHUB_ENV - echo "pkg_retention_days=30" >> $GITHUB_ENV - ;; - *) - echo "repo=${{ secrets.PACKAGE_CLOUD_REPO }}-devel" >> $GITHUB_ENV - echo "pkg_version=0.${GITHUB_SHA}" >> $GITHUB_ENV - echo "pkg_retention_days=30" >> $GITHUB_ENV - ;; - esac - else - echo "runtype=test" >> $GITHUB_ENV - echo "pkg_version=$(cut -d'-' -f 1 packaging/version | sed -e 's/^v//')" >> $GITHUB_ENV - fi - name: Setup QEMU + id: qemu if: matrix.platform != 'linux/amd64' && matrix.platform != 'linux/i386' uses: docker/setup-qemu-action@v1 - name: Prepare Docker Environment + id: docker-config shell: bash run: | echo '{"cgroup-parent": "/actions_job", "experimental": true}' | sudo tee /etc/docker/daemon.json 2>/dev/null sudo service docker restart - - name: Set Base Image Version - shell: bash - run: | - if [ -z "${{ matrix.alias }}" ] ; then - echo "version=${{ matrix.version }}" >> $GITHUB_ENV - else - echo "version=${{ matrix.alias }}" >> $GITHUB_ENV - fi - - name: Fetch base image + - name: Fetch images + id: fetch-images uses: nick-invision/retry@v2 with: max_attempts: 3 retry_wait_seconds: 30 timeout_seconds: 900 command: | - docker pull --platform ${{ matrix.platform }} ${{ matrix.base_image }}:${{ env.version }} + docker pull --platform ${{ matrix.platform }} ${{ matrix.base_image }}:${{ matrix.version }} docker pull --platform ${{ matrix.platform }} netdata/package-builders:${{ matrix.distro }}${{ matrix.version }} - name: Build Packages + id: build shell: bash run: | - docker run --security-opt seccomp=unconfined -e DO_NOT_TRACK=1 -e VERSION=${{ env.pkg_version }} --platform=${{ matrix.platform }} -v $PWD:/netdata netdata/package-builders:${{ matrix.distro }}${{ matrix.version }} + docker run --security-opt seccomp=unconfined -e DISABLE_TELEMETRY=1 -e VERSION=${{ needs.version-check.outputs.version }} \ + --platform=${{ matrix.platform }} -v "$PWD":/netdata netdata/package-builders:${{ matrix.distro }}${{ matrix.version }} - name: Test Packages + id: test shell: bash run: | - docker run --security-opt seccomp=unconfined -e DO_NOT_TRACK=1 -e DISTRO=${{ matrix.distro }} -e VERSION=${{ env.pkg_version }} -e DISTRO_VERSION=${{ env.version }} --platform=${{ matrix.platform }} -v $PWD:/netdata ${{ matrix.base_image }}:${{ env.version }} /netdata/.github/scripts/pkg-test.sh + docker run --security-opt seccomp=unconfined -e DISABLE_TELEMETRY=1 -e DISTRO=${{ matrix.distro }} \ + -e VERSION=${{ needs.version-check.outputs.version }} -e DISTRO_VERSION=${{ matrix.version }} \ + --platform=${{ matrix.platform }} -v "$PWD":/netdata ${{ matrix.base_image }}:${{ matrix.version }} \ + /netdata/.github/scripts/pkg-test.sh - name: Save Packages - uses: actions/upload-artifact@v2 + id: artifacts + continue-on-error: true + uses: actions/upload-artifact@v3 with: name: ${{ matrix.distro }}-${{ matrix.version }}-${{ matrix.arch }}-packages path: ${{ github.workspace }}/artifacts/* - name: Upload to PackageCloud + id: upload if: github.event_name == 'workflow_dispatch' shell: bash env: PKG_CLOUD_TOKEN: ${{ secrets.PACKAGE_CLOUD_API_KEY }} run: | - echo "Packages to upload:\n$(ls artifacts/*.${{ matrix.format }})" + printf "Packages to upload:\n%s" "$(ls artifacts/*.${{ matrix.format }})" for pkgfile in artifacts/*.${{ matrix.format }} ; do - .github/scripts/package_cloud_wrapper.sh yank ${{ env.repo }}/${{ matrix.pkgclouddistro }} $(basename ${pkgfile}) || true - .github/scripts/package_cloud_wrapper.sh push ${{ env.repo }}/${{ matrix.pkgclouddistro }} ${pkgfile} + .github/scripts/package_cloud_wrapper.sh yank ${{ needs.version-check.outputs.repo }}/${{ matrix.pkgclouddistro }} \ + "$(basename "${pkgfile}")" || true + .github/scripts/package_cloud_wrapper.sh push ${{ needs.version-check.outputs.repo }}/${{ matrix.pkgclouddistro }} "${pkgfile}" done - name: Clean + id: cleanup if: github.event_name == 'workflow_dispatch' shell: bash env: - REPO: ${{ env.repo }} + REPO: ${{ needs.version-check.outputs.repo }} PKG_CLOUD_TOKEN: ${{ secrets.PACKAGE_CLOUD_API_KEY }} - PACKAGE_CLOUD_RETENTION_DAYS: ${{ env.pkg_retention_days }} + PACKAGE_CLOUD_RETENTION_DAYS: ${{ needs.version-check.outputs.retention }} run: .github/scripts/old_package_purging.sh - name: Failure Notification uses: rtCamp/action-slack-notify@v2 @@ -172,7 +243,16 @@ jobs: SLACK_ICON_EMOJI: ':github-actions:' SLACK_TITLE: 'Package Build failed:' SLACK_USERNAME: 'GitHub Actions' - SLACK_MESSAGE: "${{ matrix.pkgclouddistro }} ${{ matrix.version }} package build for ${{ matrix.arch }} failed." + SLACK_MESSAGE: |- + ${{ github.repository }}: ${{ matrix.pkgclouddistro }} ${{ matrix.version }} package build for ${{ matrix.arch }} failed. + Checkout: ${{ steps.checkout.outcome }} + Setup QEMU: ${{ steps.qemu.outcome }} + Setup Docker: ${{ steps.docker-config.outcome }} + Fetch images: ${{ steps.fetch-images.outcome }} + Build: ${{ steps.build.outcome }} + Test: ${{ steps.test.outcome }} + Publish: ${{ steps.upload.outcome }} + Cleanup: ${{ steps.cleanup.outcome }} SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }} if: >- ${{ |