diff options
Diffstat (limited to '.github/workflows/packaging.yml')
| -rw-r--r-- | .github/workflows/packaging.yml | 313 |
1 files changed, 313 insertions, 0 deletions
diff --git a/.github/workflows/packaging.yml b/.github/workflows/packaging.yml new file mode 100644 index 00000000..eb936c4d --- /dev/null +++ b/.github/workflows/packaging.yml @@ -0,0 +1,313 @@ +--- +# Handles building of binary packages for the agent. +name: Packages +on: + pull_request: + types: + - opened + - reopened + - labeled + - synchronize + push: + branches: + - master + workflow_dispatch: + inputs: + type: + description: Package build type + default: devel + required: true + version: + description: Package version + required: false +env: + DISABLE_TELEMETRY: 1 + REPO_PREFIX: netdata/netdata +concurrency: + group: packages-${{ github.ref }}-${{ github.event_name }} + cancel-in-progress: true +jobs: + file-check: # Check what files changed if we’re being run in a PR or on a push. + name: Check Modified Files + runs-on: ubuntu-latest + outputs: + run: ${{ steps.check-run.outputs.run }} + steps: + - name: Checkout + id: checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + submodules: recursive + - name: Check files + id: file-check + uses: tj-actions/changed-files@v40 + with: + since_last_remote_commit: ${{ github.event_name != 'pull_request' }} + files: | + **.c + **.cc + **.h + **.hh + **.in + netdata.spec.in + configure.ac + **/Makefile* + Makefile* + .github/data/distros.yml + .github/workflows/packaging.yml + .github/scripts/gen-matrix-packaging.py + .github/scripts/pkg-test.sh + build/** + packaging/*.sh + packaging/*.checksums + packaging/*.version + contrib/debian/** + aclk/aclk-schemas/ + ml/dlib/ + mqtt_websockets + web/server/h2o/libh2o + files_ignore: | + **.md + - name: Check Run + id: check-run + run: | + if [ "${{ steps.file-check.outputs.any_modified }}" == "true" ] || [ "${{ github.event_name }}" == "workflow_dispatch" ]; then + echo 'run=true' >> "${GITHUB_OUTPUT}" + else + echo 'run=false' >> "${GITHUB_OUTPUT}" + fi + + matrix: + name: Prepare Build Matrix + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.set-matrix.outputs.matrix }} + steps: + - name: Checkout + id: checkout + uses: actions/checkout@v4 + - name: Prepare tools + id: prepare + run: | + sudo apt-get update && sudo apt-get install -y python3-ruamel.yaml + - name: Read build matrix + id: set-matrix + run: | + if [ "${{ github.event_name }}" = "pull_request" ] && \ + [ "${{ !contains(github.event.pull_request.labels.*.name, 'run-ci/packaging') }}" = "true" ]; then + matrix="$(.github/scripts/gen-matrix-packaging.py 1)" + else + matrix="$(.github/scripts/gen-matrix-packaging.py 0)" + fi + echo "Generated matrix: ${matrix}" + echo "matrix=${matrix}" >> "${GITHUB_OUTPUT}" + - name: Failure Notification + uses: rtCamp/action-slack-notify@v2 + env: + SLACK_COLOR: 'danger' + SLACK_ICON_EMOJI: ':github-actions:' + SLACK_TITLE: 'Package Build matrix generation failed:' + SLACK_USERNAME: 'GitHub Actions' + SLACK_MESSAGE: |- + ${{ github.repository }}: Failed to generate build matrix for package build. + Checkout: ${{ steps.checkout.outcome }} + Prepare Tools: ${{ steps.prepare.outcome }} + Read Build Matrix: ${{ steps.set-matrix.outcome }} + SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }} + if: >- + ${{ + failure() + && github.event_name != 'pull_request' + && startsWith(github.ref, 'refs/heads/master') + && github.repository == 'netdata/netdata' + }} + + version-check: + name: Version check + runs-on: ubuntu-latest + outputs: + repo: ${{ steps.check-version.outputs.repo }} + version: ${{ steps.check-version.outputs.version }} + retention: ${{ steps.check-version.outputs.retention }} + steps: + - name: Checkout + id: checkout + uses: actions/checkout@v4 + - name: Check Version + id: check-version + run: | + if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then + case "${{ github.event.inputs.type }}" in + "release") + echo "repo=${REPO_PREFIX}" >> "${GITHUB_OUTPUT}" + echo "version=${{ github.event.inputs.version }}" >> "${GITHUB_OUTPUT}" + echo "retention=365" >> "${GITHUB_OUTPUT}" + ;; + "nightly") + echo "repo=${REPO_PREFIX}-edge" >> "${GITHUB_OUTPUT}" + echo "version=$(tr -d 'v' < packaging/version)" >> "${GITHUB_OUTPUT}" + echo "retention=30" >> "${GITHUB_OUTPUT}" + ;; + *) + echo "repo=${REPO_PREFIX}-devel" >> "${GITHUB_OUTPUT}" + echo "version=0.${GITHUB_SHA}" >> "${GITHUB_OUTPUT}" + echo "retention=30" >> "${GITHUB_OUTPUT}" + ;; + esac + else + echo "version=$(cut -d'-' -f 1 packaging/version | tr -d 'v')" >> "${GITHUB_OUTPUT}" + echo "retention=0" >> "${GITHUB_OUTPUT}" + fi + - name: Failure Notification + uses: rtCamp/action-slack-notify@v2 + env: + SLACK_COLOR: 'danger' + SLACK_ICON_EMOJI: ':github-actions:' + SLACK_TITLE: 'Package Build version check failed:' + SLACK_USERNAME: 'GitHub Actions' + SLACK_MESSAGE: |- + ${{ github.repository }}: Failed to generate version information for package build. + Checkout: ${{ steps.checkout.outcome }} + Check Version: ${{ steps.check-version.outcome }} + SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }} + if: >- + ${{ + failure() + && github.event_name != 'pull_request' + && startsWith(github.ref, 'refs/heads/master') + && github.repository == 'netdata/netdata' + }} + + build: + name: Build + runs-on: ubuntu-latest + env: + DOCKER_CLI_EXPERIMENTAL: enabled + needs: + - matrix + - version-check + - file-check + strategy: + matrix: ${{ fromJson(needs.matrix.outputs.matrix) }} + # We intentiaonally disable the fail-fast behavior so that a + # build failure for one version doesn't prevent us from publishing + # successfully built and tested packages for another version. + fail-fast: false + max-parallel: 8 + steps: + - name: Skip Check + id: skip + if: needs.file-check.outputs.run != 'true' + run: echo "SKIPPED" + - name: Checkout + id: checkout + if: needs.file-check.outputs.run == 'true' + uses: actions/checkout@v4 + with: + fetch-depth: 0 # We need full history for versioning + submodules: recursive + - name: Setup QEMU + id: qemu + if: matrix.platform != 'linux/amd64' && matrix.platform != 'linux/i386' && needs.file-check.outputs.run == 'true' + uses: docker/setup-qemu-action@v3 + - name: Prepare Docker Environment + id: docker-config + if: needs.file-check.outputs.run == 'true' + shell: bash + run: | + echo '{"cgroup-parent": "actions-job.slice", "experimental": true}' | sudo tee /etc/docker/daemon.json 2>/dev/null + sudo service docker restart + - name: Fetch images + id: fetch-images + if: needs.file-check.outputs.run == 'true' + uses: nick-invision/retry@v2 + with: + max_attempts: 3 + retry_wait_seconds: 30 + timeout_seconds: 900 + command: | + docker pull --platform ${{ matrix.platform }} ${{ matrix.base_image }} + docker pull --platform ${{ matrix.platform }} netdata/package-builders:${{ matrix.distro }}${{ matrix.version }}-v1 + - name: Build Packages + id: build + if: needs.file-check.outputs.run == 'true' + shell: bash + run: | + docker run --security-opt seccomp=unconfined -e DISABLE_TELEMETRY=1 -e VERSION=${{ needs.version-check.outputs.version }} \ + --platform=${{ matrix.platform }} -v "$PWD":/netdata netdata/package-builders:${{ matrix.distro }}${{ matrix.version }}-v1 + - name: Save Packages + id: artifacts + if: needs.file-check.outputs.run == 'true' + continue-on-error: true + uses: actions/upload-artifact@v3 + with: + name: ${{ matrix.distro }}-${{ matrix.version }}-${{ matrix.arch }}-packages + path: ${{ github.workspace }}/artifacts/* + - name: Test Packages + id: test + if: needs.file-check.outputs.run == 'true' + shell: bash + run: | + docker run --security-opt seccomp=unconfined -e DISABLE_TELEMETRY=1 -e DISTRO=${{ matrix.distro }} \ + -e VERSION=${{ needs.version-check.outputs.version }} -e DISTRO_VERSION=${{ matrix.version }} \ + --platform=${{ matrix.platform }} -v "$PWD":/netdata ${{ matrix.base_image }} \ + /netdata/.github/scripts/pkg-test.sh + - name: Upload to PackageCloud + id: upload + if: github.event_name == 'workflow_dispatch' && github.repository == 'netdata/netdata' && needs.file-check.outputs.run == 'true' + continue-on-error: true + shell: bash + env: + PKG_CLOUD_TOKEN: ${{ secrets.PACKAGE_CLOUD_API_KEY }} + run: | + printf "Packages to upload:\n%s" "$(ls artifacts/*.${{ matrix.format }})" + for pkgfile in artifacts/*.${{ matrix.format }} ; do + .github/scripts/package_cloud_wrapper.sh yank ${{ needs.version-check.outputs.repo }}/${{ matrix.repo_distro }} \ + "$(basename "${pkgfile}")" || true + .github/scripts/package_cloud_wrapper.sh push ${{ needs.version-check.outputs.repo }}/${{ matrix.repo_distro }} "${pkgfile}" + done + - name: SSH setup + id: ssh-setup + if: github.event_name == 'workflow_dispatch' && github.repository == 'netdata/netdata' && needs.file-check.outputs.run == 'true' + uses: shimataro/ssh-key-action@v2 + with: + key: ${{ secrets.NETDATABOT_PACKAGES_SSH_KEY }} + name: id_ecdsa + known_hosts: ${{ secrets.PACKAGES_KNOWN_HOSTS }} + - name: Upload to packages.netdata.cloud + id: package-upload + if: github.event_name == 'workflow_dispatch' && github.repository == 'netdata/netdata' && needs.file-check.outputs.run == 'true' + run: | + .github/scripts/package-upload.sh \ + ${{ matrix.repo_distro }} \ + ${{ matrix.arch }} \ + ${{ matrix.format }} \ + ${{ needs.version-check.outputs.repo }} + - name: Failure Notification + uses: rtCamp/action-slack-notify@v2 + env: + SLACK_COLOR: 'danger' + SLACK_ICON_EMOJI: ':github-actions:' + SLACK_TITLE: 'Package Build failed:' + SLACK_USERNAME: 'GitHub Actions' + SLACK_MESSAGE: |- + ${{ github.repository }}: ${{ matrix.repo_distro }} ${{ matrix.version }} package build for ${{ matrix.arch }} failed. + Checkout: ${{ steps.checkout.outcome }} + Setup QEMU: ${{ steps.qemu.outcome }} + Setup Docker: ${{ steps.docker-config.outcome }} + Fetch images: ${{ steps.fetch-images.outcome }} + Build: ${{ steps.build.outcome }} + Test: ${{ steps.test.outcome }} + Publish to PackageCloud: ${{ steps.upload.outcome }} + Import SSH Key: ${{ steps.ssh-setup.outcome }} + Publish to packages.netdata.cloud: ${{ steps.package-upload.outcome }} + SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }} + if: >- + ${{ + failure() + && github.event_name != 'pull_request' + && startsWith(github.ref, 'refs/heads/master') + && github.repository == 'netdata/netdata' + && needs.file-check.outputs.run == 'true' + }} |