diff options
author | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-05-05 11:19:16 +0000 |
---|---|---|
committer | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-05-05 12:07:37 +0000 |
commit | b485aab7e71c1625cfc27e0f92c9509f42378458 (patch) | |
tree | ae9abe108601079d1679194de237c9a435ae5b55 /.github/workflows | |
parent | Adding upstream version 1.44.3. (diff) | |
download | netdata-b485aab7e71c1625cfc27e0f92c9509f42378458.tar.xz netdata-b485aab7e71c1625cfc27e0f92c9509f42378458.zip |
Adding upstream version 1.45.3+dfsg.
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to '.github/workflows')
-rw-r--r-- | .github/workflows/add-to-project.yml | 4 | ||||
-rw-r--r-- | .github/workflows/build-macos.yml | 142 | ||||
-rw-r--r-- | .github/workflows/build.yml | 211 | ||||
-rw-r--r-- | .github/workflows/checks.yml | 54 | ||||
-rw-r--r-- | .github/workflows/codeql.yml | 54 | ||||
-rw-r--r-- | .github/workflows/coverity.yml | 7 | ||||
-rw-r--r-- | .github/workflows/dashboard-pr.yml | 4 | ||||
-rw-r--r-- | .github/workflows/docker.yml | 677 | ||||
-rw-r--r-- | .github/workflows/docs.yml | 29 | ||||
-rw-r--r-- | .github/workflows/generate-integrations.yml | 25 | ||||
-rw-r--r-- | .github/workflows/go-tests.yml | 124 | ||||
-rw-r--r-- | .github/workflows/kickstart-upload.yml | 54 | ||||
-rw-r--r-- | .github/workflows/labeler.yml | 2 | ||||
-rw-r--r-- | .github/workflows/monitor-releases.yml | 2 | ||||
-rw-r--r-- | .github/workflows/packaging.yml | 65 | ||||
-rw-r--r-- | .github/workflows/release.yml | 4 | ||||
-rw-r--r-- | .github/workflows/repoconfig-packages.yml | 2 | ||||
-rw-r--r-- | .github/workflows/review.yml | 66 |
18 files changed, 1116 insertions, 410 deletions
diff --git a/.github/workflows/add-to-project.yml b/.github/workflows/add-to-project.yml index 986d836a1..838917146 100644 --- a/.github/workflows/add-to-project.yml +++ b/.github/workflows/add-to-project.yml @@ -13,13 +13,13 @@ jobs: runs-on: ubuntu-latest steps: - name: Add issues to Agent project board - uses: actions/add-to-project@v0.5.0 + uses: actions/add-to-project@v0.6.0 with: project-url: https://github.com/orgs/netdata/projects/32 github-token: ${{ secrets.NETDATABOT_ORG_GITHUB_TOKEN }} - name: Add issues to Product Bug project board - uses: actions/add-to-project@v0.5.0 + uses: actions/add-to-project@v0.6.0 with: project-url: https://github.com/orgs/netdata/projects/45 github-token: ${{ secrets.NETDATABOT_ORG_GITHUB_TOKEN }} diff --git a/.github/workflows/build-macos.yml b/.github/workflows/build-macos.yml new file mode 100644 index 000000000..dd6e41540 --- /dev/null +++ b/.github/workflows/build-macos.yml @@ -0,0 +1,142 @@ +--- +# CI code for build and test on macOS +name: macOS Build and test +on: + push: # Master branch checks only validate the build and generate artifacts for testing. + branches: + - master + pull_request: null # PR checks only validate the build and generate artifacts for testing. + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + +jobs: + file-check: # Check what files changed if we’re being run in a PR or on a push. + name: Check Modified Files + runs-on: ubuntu-latest + outputs: + run: ${{ steps.check-run.outputs.run }} + steps: + - name: Checkout + id: checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + submodules: recursive + - name: Check files + id: check-files + uses: tj-actions/changed-files@v43 + with: + since_last_remote_commit: ${{ github.event_name != 'pull_request' }} + files: | + **/*.c + **/*.cc + **/*.h + **/*.hh + **/*.in + **/*.patch + **/*.cmake + CMakeLists.txt + netdata-installer.sh + .github/workflows/build-macos.yml + .github/scripts/run-updater-check.sh + packaging/cmake/ + packaging/installer/ + packaging/*.sh + packaging/*.version + packaging/*.checksums + src/aclk/aclk-schemas/ + src/ml/dlib/ + src/fluent-bit/ + src/web/server/h2o/libh2o/ + files_ignore: | + netdata.spec.in + **/*.md + - name: List all changed files in pattern + continue-on-error: true + env: + ALL_CHANGED_FILES: ${{ steps.check-files.outputs.all_changed_files }} + run: | + for file in ${ALL_CHANGED_FILES}; do + echo "$file was changed" + done + - name: Check Run + id: check-run + run: | + if [ "${{ steps.check-files.outputs.any_modified }}" == "true" ] || [ "${{ github.event_name }}" == "workflow_dispatch" ]; then + echo 'run=true' >> "${GITHUB_OUTPUT}" + else + echo 'run=false' >> "${GITHUB_OUTPUT}" + fi + + build-test: + env: + DISABLE_TELEMETRY: 1 + runs-on: ${{ matrix.runner }} + needs: + - file-check + strategy: + fail-fast: false + max-parallel: 3 + matrix: + include: + - name: macos-12 + runner: macos-12 + - name: macos-13 + runner: macos-13 + - name: macos-14-M1 + runner: macos-14 + steps: + - name: Skip Check + id: skip + if: needs.file-check.outputs.run != 'true' + run: echo "SKIPPED" + - uses: actions/checkout@v4 + id: checkout + if: needs.file-check.outputs.run == 'true' + with: + submodules: recursive + - name: Install latest bash + id: install-bash + if: needs.file-check.outputs.run == 'true' + run: | + brew install bash + - name: Install netdata dependencies + id: install-nd-dep + if: needs.file-check.outputs.run == 'true' + run: | + bash ./packaging/installer/install-required-packages.sh --dont-wait --non-interactive netdata + - name: Build from source + id: build-source + if: needs.file-check.outputs.run == 'true' + run: | + sudo bash ./netdata-installer.sh --install-no-prefix /usr/local/netdata --dont-wait --dont-start-it --require-cloud --one-time-build + - name: Test Agent start up + id: test-agent + if: needs.file-check.outputs.run == 'true' + run: | + /usr/local/netdata/usr/sbin/netdata -D > ./netdata.log 2>&1 & + ./packaging/runtime-check.sh + - name: Failure Notification + uses: rtCamp/action-slack-notify@v2 + env: + SLACK_COLOR: 'danger' + SLACK_FOOTER: '' + SLACK_ICON_EMOJI: ':github-actions:' + SLACK_TITLE: 'Build & test from source macOS failed:' + SLACK_USERNAME: 'GitHub Actions' + SLACK_MESSAGE: |- + ${{ github.repository }}: macOS Build and test. + Checkout: ${{ steps.checkout.outcome }} + Setup runner: ${{ steps.install-bash.outcome }} + Install netdata required packages: ${{ steps.install-nd-dep.outcome }} + Build from source: ${{ steps.build-source.outcome }} + Test Agent runtime: ${{ steps.test-agent.outcome }} + SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }} + if: >- + ${{ + failure() + && startsWith(github.ref, 'refs/heads/master') + && github.event_name != 'pull_request' + && github.repository == 'netdata/netdata' + }} diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 4a6debc46..7ca83d867 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -34,35 +34,46 @@ jobs: submodules: recursive - name: Check files id: check-files - uses: tj-actions/changed-files@v40 + uses: tj-actions/changed-files@v43 with: since_last_remote_commit: ${{ github.event_name != 'pull_request' }} files: | - **.c - **.cc - **.h - **.hh - **.in - configure.ac + **/*.c + **/*.cc + **/*.h + **/*.hh + **/*.in + **/*.patch + **/*.cmake + CMakeLists.txt netdata-installer.sh - **/Makefile* - Makefile* .github/data/distros.yml .github/workflows/build.yml .github/scripts/build-static.sh .github/scripts/get-static-cache-key.sh .github/scripts/gen-matrix-build.py .github/scripts/run-updater-check.sh - build/** - packaging/makeself/** - packaging/installer/** - aclk/aclk-schemas/ - ml/dlib/ - mqtt_websockets - web/server/h2o/libh2o + packaging/cmake/ + packaging/makeself/ + packaging/installer/ + packaging/*.sh + packaging/*.version + packaging/*.checksums + src/aclk/aclk-schemas/ + src/ml/dlib/ + src/fluent-bit/ + src/web/server/h2o/libh2o/ files_ignore: | netdata.spec.in - **.md + **/*.md + - name: List all changed files in pattern + continue-on-error: true + env: + ALL_CHANGED_FILES: ${{ steps.check-files.outputs.all_changed_files }} + run: | + for file in ${ALL_CHANGED_FILES}; do + echo "$file was changed" + done - name: Check Run id: check-run run: | @@ -105,24 +116,17 @@ jobs: id: build if: needs.file-check.outputs.run == 'true' run: | - git describe - mkdir -p artifacts - ./packaging/installer/install-required-packages.sh --dont-wait --non-interactive netdata - autoreconf -ivf - ./configure --prefix=/usr \ - --sysconfdir=/etc \ - --localstatedir=/var \ - --libexecdir=/usr/libexec \ - --with-zlib \ - --with-math \ - --with-user=netdata - make dist + mkdir -p artifacts/ + tar --create --file "artifacts/netdata-$(git describe).tar.gz" \ + --sort=name --posix --auto-compress --exclude=artifacts/ --exclude=.git \ + --exclude=.gitignore --exclude=.gitattributes --exclude=.gitmodules \ + --transform "s/^\\.\\//netdata-$(git describe)\\//" --verbose . + cd artifacts/ echo "distfile=$(find . -name 'netdata-*.tar.gz')" >> "${GITHUB_OUTPUT}" - cp netdata-*.tar.gz artifacts/ - name: Store id: store if: needs.file-check.outputs.run == 'true' - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: dist-tarball path: artifacts/*.tar.gz @@ -161,6 +165,7 @@ jobs: matrix: arch: - x86_64 + - armv6l - armv7l - aarch64 - ppc64le @@ -193,7 +198,7 @@ jobs: - name: Cache if: (github.event_name != 'pull_request' || ! contains(github.event.pull_request.labels.*.name, 'run-ci/no-cache')) && needs.file-check.outputs.run == 'true' id: cache - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: artifacts/cache key: ${{ steps.cache-key.outputs.key }} @@ -203,7 +208,7 @@ jobs: - name: Build if: github.event_name == 'workflow_dispatch' && needs.file-check.outputs.run == 'true' id: build - uses: nick-fields/retry@v2 + uses: nick-fields/retry@v3 with: timeout_minutes: 180 max_attempts: 3 @@ -211,9 +216,9 @@ jobs: - name: Store id: store if: needs.file-check.outputs.run == 'true' - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: - name: static-archive + name: dist-static-${{ matrix.arch }} path: artifacts/*.gz.run retention-days: 30 - name: Failure Notification @@ -354,7 +359,7 @@ jobs: tags: test:${{ matrix.artifact_key }} - name: Upload image artifact id: upload - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: ${{ matrix.artifact_key }}-test-env path: /tmp/image.tar @@ -410,19 +415,18 @@ jobs: - name: Fetch test environment id: fetch if: needs.file-check.outputs.run == 'true' - uses: actions/download-artifact@v3 + uses: Wandalen/wretry.action@v1 with: - name: ${{ matrix.artifact_key }}-test-env + action: actions/download-artifact@v4 + with: | + name: ${{ matrix.artifact_key }}-test-env + path: . + attempt_limit: 3 + attempt_delay: 2000 - name: Load test environment id: load if: needs.file-check.outputs.run == 'true' run: docker load --input image.tar - - name: Regular build on ${{ matrix.distro }} - id: build-basic - if: needs.file-check.outputs.run == 'true' - run: | - docker run --security-opt seccomp=unconfined -w /netdata test:${{ matrix.artifact_key }} \ - /bin/sh -c 'autoreconf -ivf && ./configure --disable-dependency-tracking && make -j2' - name: netdata-installer on ${{ matrix.distro }}, disable cloud id: build-no-cloud if: needs.file-check.outputs.run == 'true' @@ -454,7 +458,6 @@ jobs: Checkout: ${{ steps.checkout.outcome }} Fetch test environment: ${{ steps.fetch.outcome }} Load test environment: ${{ steps.load.outcome }} - Regular build: ${{ steps.build-basic.outcome }} netdata-installer, disable cloud: ${{ steps.build-no-cloud.outcome }} netdata-installer, require cloud: ${{ steps.build-cloud.outcome }} netdata-installer, no JSON-C: ${{ steps.build-no-jsonc.outcome }} @@ -500,27 +503,40 @@ jobs: - name: Fetch dist tarball artifacts id: fetch-tarball if: needs.file-check.outputs.run == 'true' - uses: actions/download-artifact@v3 + uses: Wandalen/wretry.action@v1 with: - name: dist-tarball - path: dist-tarball + action: actions/download-artifact@v4 + with: | + name: dist-tarball + path: dist-tarball + attempt_limit: 3 + attempt_delay: 2000 - name: Prepare artifact directory id: prepare if: needs.file-check.outputs.run == 'true' run: | - mkdir -p artifacts/download/latest || exit 1 - echo "9999.0.0-0" > artifacts/download/latest/latest-version.txt || exit 1 - cp dist-tarball/* artifacts/download/latest || exit 1 - cd artifacts/download/latest || exit 1 + mkdir -p artifacts/download/v9999.0.0 || exit 1 + mkdir -p artifacts/latest || exit 1 + echo "v9999.0.0" > artifacts/latest/latest-version.txt || exit 1 + cp dist-tarball/* artifacts/download/v9999.0.0 || exit 1 + cd artifacts/download/v9999.0.0 || exit 1 ln -s ${{ needs.build-dist.outputs.distfile }} netdata-latest.tar.gz || exit 1 + ls -lFh sha256sum -b ./* > "sha256sums.txt" || exit 1 cat sha256sums.txt + cd ../.. || exit 1 + ls -lR - name: Fetch test environment id: fetch-test-environment if: needs.file-check.outputs.run == 'true' - uses: actions/download-artifact@v3 + uses: Wandalen/wretry.action@v1 with: - name: ${{ matrix.artifact_key }}-test-env + action: actions/download-artifact@v4 + with: | + name: ${{ matrix.artifact_key }}-test-env + path: . + attempt_limit: 3 + attempt_delay: 2000 - name: Load test environment id: load if: needs.file-check.outputs.run == 'true' @@ -577,27 +593,24 @@ jobs: id: prepare if: needs.file-check.outputs.run == 'true' run: mkdir -p artifacts - - name: Retrieve Dist Tarball + - name: Retrieve Build Artifacts id: fetch-dist if: needs.file-check.outputs.run == 'true' - uses: actions/download-artifact@v3 + uses: Wandalen/wretry.action@v1 with: - name: dist-tarball - path: dist-tarball - - name: Retrieve Static Build Artifacts - id: fetch-static - if: needs.file-check.outputs.run == 'true' - uses: actions/download-artifact@v3 - with: - name: static-archive - path: static-archive + action: actions/download-artifact@v4 + with: | + pattern: dist-* + path: dist-artifacts + merge-multiple: true + attempt_limit: 3 + attempt_delay: 2000 - name: Prepare Artifacts id: consolidate if: needs.file-check.outputs.run == 'true' working-directory: ./artifacts/ run: | - mv ../dist-tarball/* . || exit 1 - mv ../static-archive/* . || exit 1 + mv ../dist-artifacts/* . || exit 1 ln -s ${{ needs.build-dist.outputs.distfile }} netdata-latest.tar.gz || exit 1 cp ../packaging/version ./latest-version.txt || exit 1 cp ../integrations/integrations.js ./integrations.js || exit 1 @@ -606,7 +619,7 @@ jobs: - name: Store Artifacts id: store if: needs.file-check.outputs.run == 'true' - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: final-artifacts path: artifacts/* @@ -662,16 +675,21 @@ jobs: - name: Fetch artifacts id: fetch if: needs.file-check.outputs.run == 'true' - uses: actions/download-artifact@v3 + uses: Wandalen/wretry.action@v1 with: - name: final-artifacts - path: artifacts + action: actions/download-artifact@v4 + with: | + name: final-artifacts + path: artifacts + attempt_limit: 3 + attempt_delay: 2000 - name: Prepare artifacts directory id: prepare if: needs.file-check.outputs.run == 'true' run: | mkdir -p download/latest mv artifacts/* download/latest + ls -al download/latest - name: Verify that artifacts work with installer id: verify if: needs.file-check.outputs.run == 'true' @@ -726,16 +744,21 @@ jobs: - name: Fetch artifacts id: fetch-artifacts if: needs.file-check.outputs.run == 'true' - uses: actions/download-artifact@v3 + uses: Wandalen/wretry.action@v1 with: - name: final-artifacts - path: artifacts + action: actions/download-artifact@v4 + with: | + name: final-artifacts + path: artifacts + attempt_limit: 3 + attempt_delay: 2000 - name: Prepare artifacts directory id: prepare if: needs.file-check.outputs.run == 'true' run: | mkdir -p download/latest mv artifacts/* download/latest + ls -al download/latest - name: Verify that artifacts work with installer id: verify if: needs.file-check.outputs.run == 'true' @@ -775,26 +798,30 @@ jobs: steps: - name: Retrieve Artifacts id: fetch - uses: actions/download-artifact@v3 + uses: Wandalen/wretry.action@v1 with: - name: final-artifacts - path: final-artifacts + action: actions/download-artifact@v4 + with: | + name: final-artifacts + path: final-artifacts + attempt_limit: 3 + attempt_delay: 2000 - name: Authenticate to GCS id: gcs-auth - uses: google-github-actions/auth@v1 + uses: google-github-actions/auth@v2 with: project_id: ${{ secrets.GCP_NIGHTLY_STORAGE_PROJECT }} credentials_json: ${{ secrets.GCS_STORAGE_SERVICE_KEY_JSON }} - name: Setup GCS id: gcs-setup - uses: google-github-actions/setup-gcloud@v1.1.1 + uses: google-github-actions/setup-gcloud@v2.0.1 - name: Upload Artifacts id: upload - uses: google-github-actions/upload-cloud-storage@v1.0.3 + uses: google-github-actions/upload-cloud-storage@v2.0.0 with: destination: ${{ secrets.GCP_NIGHTLY_STORAGE_BUCKET }} gzip: false - path: ./final-artifacts + path: ./final-artifacts/latest-version.txt parent: false - name: Failure Notification uses: rtCamp/action-slack-notify@v2 @@ -840,10 +867,14 @@ jobs: token: ${{ secrets.NETDATABOT_GITHUB_TOKEN }} - name: Retrieve Artifacts id: fetch - uses: actions/download-artifact@v3 + uses: Wandalen/wretry.action@v1 with: - name: final-artifacts - path: final-artifacts + action: actions/download-artifact@v4 + with: | + name: final-artifacts + path: final-artifacts + attempt_limit: 3 + attempt_delay: 2000 - name: Prepare version info id: version run: | @@ -871,7 +902,7 @@ jobs: with: token: ${{ secrets.NETDATABOT_GITHUB_TOKEN }} - name: Init python environment for publish release metadata - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 id: init-python with: python-version: "3.12" @@ -954,10 +985,14 @@ jobs: uses: actions/checkout@v4 - name: Retrieve Artifacts id: fetch - uses: actions/download-artifact@v3 + uses: Wandalen/wretry.action@v1 with: - name: final-artifacts - path: final-artifacts + action: actions/download-artifact@v4 + with: | + name: final-artifacts + path: final-artifacts + attempt_limit: 3 + attempt_delay: 2000 - name: Create Release id: create-release uses: ncipollo/release-action@v1 diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 1308f45fa..01ada7422 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -25,28 +25,43 @@ jobs: submodules: recursive - name: Check files id: check-files - uses: tj-actions/changed-files@v40 + uses: tj-actions/changed-files@v43 with: since_last_remote_commit: ${{ github.event_name != 'pull_request' }} files: | - **.c - **.cc - **.h - **.hh - **.in - configure.ac - **/Makefile* - Makefile* + **/*.c + **/*.cc + **/*.h + **/*.hh + **/*.in + **/*.patch + **/*.cmake + CMakeLists.txt .gitignore - .github/workflows/checks.yml - build/** - aclk/aclk-schemas/ - ml/dlib/ - mqtt_websockets - web/server/h2o/libh2o + .github/data/distros.yml + .github/workflows/build.yml + .github/scripts/build-static.sh + .github/scripts/get-static-cache-key.sh + .github/scripts/gen-matrix-build.py + .github/scripts/run-updater-check.sh + packaging/cmake/ + packaging/*.version + packaging/*.checksums + src/aclk/aclk-schemas/ + src/ml/dlib/ + src/fluent-bit/ + src/web/server/h2o/libh2o/ files_ignore: | netdata.spec.in - **.md + **/*.md + - name: List all changed files in pattern + continue-on-error: true + env: + ALL_CHANGED_FILES: ${{ steps.check-files.outputs.all_changed_files }} + run: | + for file in ${ALL_CHANGED_FILES}; do + echo "$file was changed" + done - name: Check Run id: check-run run: | @@ -78,11 +93,8 @@ jobs: 'apk add bash; ./packaging/installer/install-required-packages.sh --dont-wait --non-interactive netdata; apk del openssl openssl-dev; - apk add libressl libressl-dev; - autoreconf -ivf; - ./configure --disable-dependency-tracking; - make;' - + apk add libressl libressl-dev protobuf-dev; + ./netdata-installer.sh --disable-telemetry --dont-start-it --dont-wait --one-time-build;' clang-checks: name: Clang needs: diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index ae5818afc..6c2c36365 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -21,6 +21,7 @@ jobs: outputs: cpp: ${{ steps.cpp.outputs.run }} python: ${{ steps.python.outputs.run }} + go: ${{ steps.go.outputs.run }} steps: - name: Clone repository uses: actions/checkout@v4 @@ -57,7 +58,7 @@ jobs: id: python run: | if [ "${{ steps.always.outputs.run }}" = "false" ]; then - if git diff --name-only origin/${{ github.base_ref }} HEAD | grep -Eq 'collectors/python.d.plugin/.*\.py' ; then + if git diff --name-only origin/${{ github.base_ref }} HEAD | grep -Eq 'src/collectors/python.d.plugin/.*\.py' ; then echo "run=true" >> "${GITHUB_OUTPUT}" echo '::notice::Python code has changed, need to run CodeQL.' else @@ -66,6 +67,19 @@ jobs: else echo "run=true" >> "${GITHUB_OUTPUT}" fi + - name: Check for Go changes + id: go + run: | + if [ "${{ steps.always.outputs.run }}" = "false" ]; then + if git diff --name-only origin/${{ github.base_ref }} HEAD | grep -Eq 'src/go/*\.go' ; then + echo "run=true" >> "${GITHUB_OUTPUT}" + echo '::notice::Go code has changed, need to run CodeQL.' + else + echo "run=false" >> "${GITHUB_OUTPUT}" + fi + else + echo "run=true" >> "${GITHUB_OUTPUT}" + fi analyze-cpp: name: Analyze C/C++ @@ -81,7 +95,7 @@ jobs: submodules: recursive fetch-depth: 0 - name: Initialize CodeQL - uses: github/codeql-action/init@v2 + uses: github/codeql-action/init@v3 with: languages: cpp config-file: ./.github/codeql/c-cpp-config.yml @@ -90,7 +104,7 @@ jobs: - name: Build netdata run: ./netdata-installer.sh --dont-start-it --disable-telemetry --dont-wait --install-prefix /tmp/install --one-time-build - name: Run CodeQL - uses: github/codeql-action/analyze@v2 + uses: github/codeql-action/analyze@v3 with: category: "/language:cpp" @@ -108,11 +122,41 @@ jobs: submodules: recursive fetch-depth: 0 - name: Initialize CodeQL - uses: github/codeql-action/init@v2 + uses: github/codeql-action/init@v3 with: config-file: ./.github/codeql/python-config.yml languages: python - name: Run CodeQL - uses: github/codeql-action/analyze@v2 + uses: github/codeql-action/analyze@v3 with: category: "/language:python" + + analyze-go: + name: Analyze Go + runs-on: ubuntu-latest + needs: prepare + if: needs.prepare.outputs.go == 'true' + strategy: + matrix: + tree: + - src/go/collectors/go.d.plugin + permissions: + security-events: write + steps: + - name: Git clone repository + uses: actions/checkout@v4 + with: + submodules: recursive + fetch-depth: 0 + - name: Initialize CodeQL + uses: github/codeql-action/init@v3 + with: + languages: go + - name: Autobuild + uses: github/codeql-action/autobuild@v3 + with: + working-directory: ${{ matrix.tree }} + - name: Run CodeQL + uses: github/codeql-action/analyze@v3 + with: + category: "/language:go" diff --git a/.github/workflows/coverity.yml b/.github/workflows/coverity.yml index eb68c302b..9113e179e 100644 --- a/.github/workflows/coverity.yml +++ b/.github/workflows/coverity.yml @@ -7,7 +7,7 @@ on: pull_request: paths: - .github/workflows/coverity.yml - - coverity-scan.sh + - packaging/utils/coverity-scan.sh env: DISABLE_TELEMETRY: 1 concurrency: @@ -33,7 +33,8 @@ jobs: sudo apt-get install -y libjson-c-dev libyaml-dev libipmimonitoring-dev \ libcups2-dev libsnappy-dev libprotobuf-dev \ libprotoc-dev libssl-dev protobuf-compiler \ - libnetfilter-acct-dev + libnetfilter-acct-dev libmongoc-dev libxen-dev \ + libsystemd-dev ninja-build - name: Run coverity-scan id: run env: @@ -41,7 +42,7 @@ jobs: COVERITY_SCAN_TOKEN: ${{ secrets.COVERITY_SCAN_TOKEN }} COVERITY_SCAN_SUBMIT_MAIL: ${{ secrets.COVERITY_SCAN_SUBMIT_MAIL }} run: | - bash -x ./coverity-scan.sh --with-install + bash -x ./packaging/utils/coverity-scan.sh --with-install - name: Failure Notification uses: rtCamp/action-slack-notify@v2 env: diff --git a/.github/workflows/dashboard-pr.yml b/.github/workflows/dashboard-pr.yml index f02cfb69d..418a8b8e6 100644 --- a/.github/workflows/dashboard-pr.yml +++ b/.github/workflows/dashboard-pr.yml @@ -25,10 +25,10 @@ jobs: - name: Update Files id: update run: | - web/gui/bundle_dashboard_v1.py ${{ github.event.inputs.dashboard_version }} + src/web/gui/bundle_dashboard_v1.py ${{ github.event.inputs.dashboard_version }} - name: Create Pull Request id: pr - uses: peter-evans/create-pull-request@v5 + uses: peter-evans/create-pull-request@v6 with: title: 'Update dashboard to version ${{ github.event.inputs.dashboard_version }}.' body: 'See https://github.com/netdata/dashboard/releases/tag/${{ github.event.inputs.dashboard_version }} for changes.' diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index b7fe0a866..aa1528a17 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -1,4 +1,13 @@ --- +# Handle building docker images both for CI checks and for eleases. +# +# The case of releaases is unfortunately rather complicated, as Docker +# tooling does not have great support for handling of multiarch images +# published to multiple registries. As a result, we have to build the +# images, export the cache, and then _rebuild_ the images using the exported +# cache but with different output parameters for buildx. We also need to +# do the second build step as a separate job for each registry so that a +# failure to publish one place won’t break publishing elsewhere. name: Docker on: push: @@ -25,52 +34,78 @@ jobs: steps: - name: Checkout id: checkout + if: github.event_name != 'workflow_dispatch' uses: actions/checkout@v4 with: fetch-depth: 0 submodules: recursive - name: Check files - id: file-check - uses: tj-actions/changed-files@v40 + id: check-files + if: github.event_name != 'workflow_dispatch' + uses: tj-actions/changed-files@v43 with: since_last_remote_commit: ${{ github.event_name != 'pull_request' }} files: | - **.c - **.cc - **.h - **.hh - **.in + **/*.c + **/*.cc + **/*.h + **/*.hh + **/*.in + **/*.patch + **/*.cmake .dockerignore - configure.ac + CMakeLists.txt netdata-installer.sh - **/Makefile* - Makefile* .github/workflows/docker.yml .github/scripts/docker-test.sh - build/** - packaging/docker/** - packaging/installer/** - aclk/aclk-schemas/ - ml/dlib/ - mqtt_websockets - web/server/h2o/libh2o + .github/scripts/gen-docker-tags.py + .github/scripts/gen-docker-imagetool-args.py + packaging/cmake/ + packaging/docker/ + packaging/installer/ + packaging/runtime-check.sh + packaging/*.version + packaging/*.checksums + src/aclk/aclk-schemas/ + src/ml/dlib/ + src/fluent-bit/ + src/web/server/h2o/libh2o/ files_ignore: | netdata.spec.in - **.md + **/*.md + - name: List all changed files in pattern + continue-on-error: true + if: github.event_name != 'workflow_dispatch' + env: + ALL_CHANGED_FILES: ${{ steps.check-files.outputs.all_changed_files }} + run: | + for file in ${ALL_CHANGED_FILES}; do + echo "$file was changed" + done - name: Check Run id: check-run run: | - if [ "${{ steps.file-check.outputs.any_modified }}" == "true" ] || [ "${{ github.event_name }}" == "workflow_dispatch" ]; then + if [ "${{ steps.check-files.outputs.any_modified }}" == "true" ] || [ "${{ github.event_name }}" == "workflow_dispatch" ]; then echo 'run=true' >> "${GITHUB_OUTPUT}" else echo 'run=false' >> "${GITHUB_OUTPUT}" fi - docker-test: - name: Docker Runtime Test + build-images: + name: Build Docker Images needs: - file-check runs-on: ubuntu-latest + strategy: + matrix: + platform: + - linux/amd64 + - linux/i386 + - linux/arm/v7 + - linux/arm64 + - linux/ppc64le + # Fail fast on releases, but run everything to completion on other triggers. + fail-fast: ${{ github.event_name == 'workflow_dispatch' }} steps: - name: Skip Check id: skip @@ -81,328 +116,604 @@ jobs: if: needs.file-check.outputs.run == 'true' uses: actions/checkout@v4 with: + fetch-depth: 0 submodules: recursive + - name: Generate Artifact Name + id: artifact-name + if: github.repository == 'netdata/netdata' && needs.file-check.outputs.run == 'true' && github.event_name == 'workflow_dispatch' + run: echo "platform=$(echo ${{ matrix.platform }} | tr '/' '-' | cut -f 2- -d '-')" >> "${GITHUB_OUTPUT}" + - name: Mark image as official + id: env + if: github.repository == 'netdata/netdata' && needs.file-check.outputs.run == 'true' && github.event_name == 'workflow_dispatch' + run: echo "OFFICIAL_IMAGE=true" >> "${GITHUB_ENV}" + - name: Setup QEMU + id: qemu + if: matrix.platform != 'linux/i386' && matrix.platform != 'linux/amd64' && needs.file-check.outputs.run == 'true' + uses: docker/setup-qemu-action@v3 - name: Setup Buildx id: prepare if: needs.file-check.outputs.run == 'true' uses: docker/setup-buildx-action@v3 - - name: Test Build + - name: Build Image id: build if: needs.file-check.outputs.run == 'true' uses: docker/build-push-action@v5 with: - load: true - push: false + platforms: ${{ matrix.platform }} tags: netdata/netdata:test + load: true + cache-to: type=local,dest=/tmp/build-cache,mode=max + build-args: OFFICIAL_IMAGE=${{ env.OFFICIAL_IMAGE }} - name: Test Image id: test - if: needs.file-check.outputs.run == 'true' + if: needs.file-check.outputs.run == 'true' && matrix.platform == 'linux/amd64' run: .github/scripts/docker-test.sh + - name: Upload Cache + id: upload-cache + if: github.repository == 'netdata/netdata' && needs.file-check.outputs.run == 'true' && github.event_name == 'workflow_dispatch' + uses: actions/upload-artifact@v4 + with: + name: cache-${{ steps.artifact-name.outputs.platform }} + path: /tmp/build-cache/* + retention-days: 1 - name: Failure Notification uses: rtCamp/action-slack-notify@v2 env: SLACK_COLOR: 'danger' SLACK_FOOTER: '' SLACK_ICON_EMOJI: ':github-actions:' - SLACK_TITLE: 'Docker runtime testing failed:' + SLACK_TITLE: 'Docker build failed:' SLACK_USERNAME: 'GitHub Actions' SLACK_MESSAGE: |- - ${{ github.repository }}: Building or testing Docker image for linux/amd64 failed. - CHeckout: ${{ steps.checkout.outcome }} + ${{ github.repository }}: Building or testing Docker image for ${{ matrix.platform }} failed. + Checkout: ${{ steps.checkout.outcome }} + Determine artifact name: ${{ steps.artifact-name.outcome }} + Setup environment: ${{ steps.env.outcome }} + Setup QEMU: ${{ steps.qemu.outcome }} Setup buildx: ${{ steps.prepare.outcome }} Build image: ${{ steps.build.outcome }} Test image: ${{ steps.test.outcome }} + Upload build cache: ${{ steps.upload-cache.outcome }} SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }} if: >- ${{ failure() && github.event_name != 'pull_request' - && startsWith(github.ref, 'refs/heads/master') && github.repository == 'netdata/netdata' && needs.file-check.outputs.run == 'true' }} - docker-ci: - if: github.event_name != 'workflow_dispatch' - name: Docker Alt Arch Builds - needs: - - docker-test - - file-check + gen-tags: + name: Generate Docker Tags runs-on: ubuntu-latest + if: github.event_name == 'workflow_dispatch' + outputs: + tags: ${{ steps.tag.outputs.tags }} + steps: + - name: Checkout + id: checkout + uses: actions/checkout@v4 + - name: Generate Tags + id: tag + run: | + if [ ${{ github.event_name }} = 'workflow_dispatch' ]; then + echo "tags=$(.github/scripts/gen-docker-tags.py ${{ github.event_name }} ${{ github.event.inputs.version }})" >> "${GITHUB_OUTPUT}" + else + echo "tags=$(.github/scripts/gen-docker-tags.py ${{ github.event_name }} '')" >> "${GITHUB_OUTPUT}" + fi + + build-images-docker-hub: + name: Push Images to Docker Hub + if: github.event_name == 'workflow_dispatch' + needs: + - build-images + - gen-tags strategy: matrix: - platforms: + platform: + - linux/amd64 - linux/i386 - linux/arm/v7 - linux/arm64 - linux/ppc64le + runs-on: ubuntu-latest steps: - - name: Skip Check - id: skip - if: needs.file-check.outputs.run != 'true' - run: echo "SKIPPED" - name: Checkout id: checkout - if: needs.file-check.outputs.run == 'true' uses: actions/checkout@v4 with: + fetch-depth: 0 submodules: recursive + - name: Generate Artifact Name + id: artifact-name + run: echo "platform=$(echo ${{ matrix.platform }} | tr '/' '-' | cut -f 2- -d '-')" >> "${GITHUB_OUTPUT}" + - name: Download Cache + id: fetch-cache + uses: actions/download-artifact@v4 + with: + name: cache-${{ steps.artifact-name.outputs.platform }} + path: /tmp/build-cache + - name: Mark image as official + id: env + if: github.repository == 'netdata/netdata' + run: echo "OFFICIAL_IMAGE=true" >> "${GITHUB_ENV}" - name: Setup QEMU id: qemu - if: matrix.platforms != 'linux/i386' && needs.file-check.outputs.run == 'true' + if: matrix.platform != 'linux/i386' && matrix.platform != 'linux/amd64' uses: docker/setup-qemu-action@v3 - name: Setup Buildx - id: buildx - if: needs.file-check.outputs.run == 'true' + id: prepare uses: docker/setup-buildx-action@v3 - - name: Build + - name: Registry Login + id: login + if: github.repository == 'netdata/netdata' + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKER_HUB_USERNAME }} + password: ${{ secrets.DOCKER_HUB_PASSWORD }} + - name: Build Image id: build - if: needs.file-check.outputs.run == 'true' uses: docker/build-push-action@v5 with: - platforms: ${{ matrix.platforms }} - load: false - push: false - tags: netdata/netdata:test + platforms: ${{ matrix.platform }} + cache-from: type=local,src=/tmp/build-cache + build-args: OFFICIAL_IMAGE=${{ env.OFFICIAL_IMAGE }} + outputs: type=image,name=netdata/netdata,push-by-digest=true,name-canonical=true,push=true + - name: Export Digest + id: export-digest + if: github.repository == 'netdata/netdata' + run: | + mkdir -p /tmp/digests + digest="${{ steps.build.outputs.digest }}" + touch "/tmp/digests/${digest#sha256:}" + - name: Upload digest + id: upload-digest + if: github.repository == 'netdata/netdata' + uses: actions/upload-artifact@v4 + with: + name: docker-digests-${{ steps.artifact-name.outputs.platform }} + path: /tmp/digests/* + if-no-files-found: error + retention-days: 1 - name: Failure Notification uses: rtCamp/action-slack-notify@v2 env: SLACK_COLOR: 'danger' SLACK_FOOTER: '' SLACK_ICON_EMOJI: ':github-actions:' - SLACK_TITLE: 'Docker build testing failed:' + SLACK_TITLE: 'Docker Hub upload failed:' SLACK_USERNAME: 'GitHub Actions' SLACK_MESSAGE: |- - ${{ github.repository }}: Building Docker image for ${{ matrix.platforms }} failed. - CHeckout: ${{ steps.checkout.outcome }} + ${{ github.repository }}: Creating or uploading Docker image for ${{ matrix.platform }} on Docker Hub failed. + Checkout: ${{ steps.checkout.outcome }} + Determine artifact name: ${{ steps.artifact-name.outcome }} + Fetch build cache: ${{ steps.fetch-cache.outcome }} + Setup environment: ${{ steps.env.outcome }} Setup QEMU: ${{ steps.qemu.outcome }} - Setup buildx: ${{ steps.buildx.outcome }} + Setup buildx: ${{ steps.prepare.outcome }} + Login to registry: ${{ steps.login.outcome }} Build image: ${{ steps.build.outcome }} + Export digest: ${{ steps.export-digest.outcome }} + Upload digest: ${{ steps.upload-digest.outcome }} SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }} if: >- ${{ failure() - && github.event_name != 'pull_request' - && startsWith(github.ref, 'refs/heads/master') && github.repository == 'netdata/netdata' - && needs.file-check.outputs.run == 'true' }} - normalize-tag: # Fix the release tag if needed - name: Normalize Release Tag - runs-on: ubuntu-latest + publish-docker-hub: + name: Consolidate and tag images for DockerHub if: github.event_name == 'workflow_dispatch' - outputs: - tag: ${{ steps.tag.outputs.tag }} + needs: + - build-images-docker-hub + - gen-tags + runs-on: ubuntu-latest steps: - - name: Normalize Tag - id: tag - run: | - if echo ${{ github.event.inputs.version }} | grep -qE '^[[:digit:]]+\.[[:digit:]]+\.[[:digit:]]+$'; then - echo "tag=v${{ github.event.inputs.version }}" >> "${GITHUB_OUTPUT}" - else - echo "tag=${{ github.event.inputs.version }}" >> "${GITHUB_OUTPUT}" - fi + - name: Checkout + id: checkout + uses: actions/checkout@v4 + - name: Download digests + id: fetch-digests + uses: actions/download-artifact@v4 + with: + path: /tmp/digests + pattern: docker-digests-* + merge-multiple: true + - name: Setup Buildx + id: prepare + uses: docker/setup-buildx-action@v3 + - name: Registry Login + id: login + if: github.repository == 'netdata/netdata' + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKER_HUB_USERNAME }} + password: ${{ secrets.DOCKER_HUB_PASSWORD }} + - name: Create and Push Manifest + id: manifest + if: github.repository == 'netdata/netdata' + run: docker buildx imagetools create $(.github/scripts/gen-docker-imagetool-args.py /tmp/digests '' ${{ needs.gen-tags.outputs.tags }}) + - name: Failure Notification + uses: rtCamp/action-slack-notify@v2 + env: + SLACK_COLOR: 'danger' + SLACK_FOOTER: '' + SLACK_ICON_EMOJI: ':github-actions:' + SLACK_TITLE: 'Publishing Docker images to Docker Hub failed:' + SLACK_USERNAME: 'GitHub Actions' + SLACK_MESSAGE: |- + ${{ github.repository }}: Publishing Docker images to Docker Hub failed. + Checkout: ${{ steps.checkout.outcome }} + Download digests: ${{ steps.fetch-digests.outcome }} + Setup buildx: ${{ steps.prepare.outcome }} + Login to registry: ${{ steps.login.outcome }} + Create and push manifest: ${{ steps.manifest.outcome }} + SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }} + if: >- + ${{ + failure() + && github.repository == 'netdata/netdata' + }} - docker-publish: + build-images-quay: + name: Push Images to Quay.io if: github.event_name == 'workflow_dispatch' - name: Docker Build and Publish needs: - - docker-test - - normalize-tag + - build-images + - gen-tags + strategy: + matrix: + platform: + - linux/amd64 + - linux/i386 + - linux/arm/v7 + - linux/arm64 + - linux/ppc64le runs-on: ubuntu-latest steps: - name: Checkout id: checkout uses: actions/checkout@v4 with: + fetch-depth: 0 submodules: recursive - - name: Determine which tags to use - id: release-tags - if: github.event.inputs.version != 'nightly' - run: | - echo "tags=netdata/netdata:latest,netdata/netdata:stable,ghcr.io/netdata/netdata:latest,ghcr.io/netdata/netdata:stable,quay.io/netdata/netdata:latest,quay.io/netdata/netdata:stable,$(.github/scripts/gen-docker-tags.py ${{ needs.normalize-tag.outputs.tag }} '')" \ - >> "${GITHUB_ENV}" - - name: Determine which tags to use - id: nightly-tags - if: github.event.inputs.version == 'nightly' - run: | - echo "tags=netdata/netdata:latest,netdata/netdata:edge,ghcr.io/netdata/netdata:latest,ghcr.io/netdata/netdata:edge,quay.io/netdata/netdata:latest,quay.io/netdata/netdata:edge" >> "${GITHUB_ENV}" + - name: Generate Artifact Name + id: artifact-name + run: echo "platform=$(echo ${{ matrix.platform }} | tr '/' '-' | cut -f 2- -d '-')" >> "${GITHUB_OUTPUT}" + - name: Download Cache + id: fetch-cache + uses: actions/download-artifact@v4 + with: + name: cache-${{ steps.artifact-name.outputs.platform }} + path: /tmp/build-cache - name: Mark image as official id: env if: github.repository == 'netdata/netdata' run: echo "OFFICIAL_IMAGE=true" >> "${GITHUB_ENV}" - name: Setup QEMU id: qemu + if: matrix.platform != 'linux/i386' && matrix.platform != 'linux/amd64' uses: docker/setup-qemu-action@v3 - name: Setup Buildx - id: buildx + id: prepare uses: docker/setup-buildx-action@v3 - - name: Docker Hub Login - id: docker-hub-login - if: github.repository == 'netdata/netdata' - uses: docker/login-action@v3 - with: - username: ${{ secrets.DOCKER_HUB_USERNAME }} - password: ${{ secrets.DOCKER_HUB_PASSWORD }} - - name: GitHub Container Registry Login - id: ghcr-login - if: github.repository == 'netdata/netdata' - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.repository_owner }} - password: ${{ secrets.GITHUB_TOKEN }} - - name: Quay.io Login - id: quay-login + - name: Registry Login + id: login if: github.repository == 'netdata/netdata' uses: docker/login-action@v3 with: registry: quay.io username: ${{ secrets.NETDATABOT_QUAY_USERNAME }} password: ${{ secrets.NETDATABOT_QUAY_TOKEN }} - - name: Docker Build + - name: Build Image id: build uses: docker/build-push-action@v5 with: - platforms: linux/amd64,linux/i386,linux/arm/v7,linux/arm64,linux/ppc64le - push: ${{ github.repository == 'netdata/netdata' }} - tags: ${{ env.tags }} + platforms: ${{ matrix.platform }} + cache-from: type=local,src=/tmp/build-cache build-args: OFFICIAL_IMAGE=${{ env.OFFICIAL_IMAGE }} + outputs: type=image,name=quay.io/netdata/netdata,push-by-digest=true,name-canonical=true,push=true + - name: Export Digest + id: export-digest + if: github.repository == 'netdata/netdata' + run: | + mkdir -p /tmp/digests + digest="${{ steps.build.outputs.digest }}" + touch "/tmp/digests/${digest#sha256:}" + - name: Upload digest + id: upload-digest + if: github.repository == 'netdata/netdata' + uses: actions/upload-artifact@v4 + with: + name: quay-digests-${{ steps.artifact-name.outputs.platform }} + path: /tmp/digests/* + if-no-files-found: error + retention-days: 1 - name: Failure Notification uses: rtCamp/action-slack-notify@v2 env: SLACK_COLOR: 'danger' SLACK_FOOTER: '' SLACK_ICON_EMOJI: ':github-actions:' - SLACK_TITLE: 'Docker Build failed:' + SLACK_TITLE: 'Quay.io upload failed:' SLACK_USERNAME: 'GitHub Actions' SLACK_MESSAGE: |- - ${{ github.repository }}: Failed to build or publish Docker images. - CHeckout: ${{ steps.checkout.outcome }} - Generate release tags: ${{ steps.release-tags.outcome }} - Generate nightly tags: ${{ steps.nightly-tags.outcome }} + ${{ github.repository }}: Creating or uploading Docker image for ${{ matrix.platform }} on Quay.io failed. + Checkout: ${{ steps.checkout.outcome }} + Determine artifact name: ${{ steps.artifact-name.outcome }} + Fetch build cache: ${{ steps.fetch-cache.outcome }} Setup environment: ${{ steps.env.outcome }} Setup QEMU: ${{ steps.qemu.outcome }} - Setup buildx: ${{ steps.buildx.outcome }} - Login to DockerHub: ${{ steps.docker-hub-login.outcome }} - Login to GHCR: ${{ steps.ghcr-login.outcome }} - Login to Quay: ${{ steps.quay-login.outcome }} - Build and publish images: ${{ steps.build.outcome }} + Setup buildx: ${{ steps.prepare.outcome }} + Login to registry: ${{ steps.login.outcome }} + Build image: ${{ steps.build.outcome }} + Export digest: ${{ steps.export-digest.outcome }} + Upload digest: ${{ steps.upload-digest.outcome }} SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }} if: >- ${{ failure() - && github.event_name != 'pull_request' - && startsWith(github.ref, 'refs/heads/master') && github.repository == 'netdata/netdata' }} - - name: Trigger Helmchart PR - if: github.event_name == 'workflow_dispatch' && github.event.inputs.version != 'nightly' && github.repository == 'netdata/netdata' - uses: benc-uk/workflow-dispatch@v1 + + publish-quay: + name: Consolidate and tag images for Quay.io + if: github.event_name == 'workflow_dispatch' + needs: + - build-images-quay + - gen-tags + runs-on: ubuntu-latest + steps: + - name: Checkout + id: checkout + uses: actions/checkout@v4 + - name: Download digests + id: fetch-digests + uses: actions/download-artifact@v4 with: - token: ${{ secrets.NETDATABOT_GITHUB_TOKEN }} - repo: netdata/helmchart - workflow: Agent Version PR - ref: refs/heads/master - inputs: '{"agent_version": "${{ needs.normalize-tag.outputs.tag }}"}' - - name: Trigger MSI build - if: github.event_name == 'workflow_dispatch' && github.event.inputs.version != 'nightly' && github.repository == 'netdata/netdata' - uses: benc-uk/workflow-dispatch@v1 + path: /tmp/digests + pattern: quay-digests-* + merge-multiple: true + - name: Setup Buildx + id: prepare + uses: docker/setup-buildx-action@v3 + - name: Registry Login + id: login + if: github.repository == 'netdata/netdata' + uses: docker/login-action@v3 with: - token: ${{ secrets.NETDATABOT_GITHUB_TOKEN }} - repo: netdata/msi-installer - workflow: Build - ref: refs/heads/master - inputs: '{"tag": "${{ needs.normalize-tag.outputs.tag }}", "pwd": "${{ secrets.MSI_CODE_SIGNING_PASSWORD }}"}' + registry: quay.io + username: ${{ secrets.NETDATABOT_QUAY_USERNAME }} + password: ${{ secrets.NETDATABOT_QUAY_TOKEN }} + - name: Create and Push Manifest + id: manifest + if: github.repository == 'netdata/netdata' + run: docker buildx imagetools create $(.github/scripts/gen-docker-imagetool-args.py /tmp/digests 'quay.io' ${{ needs.gen-tags.outputs.tags }}) + - name: Failure Notification + uses: rtCamp/action-slack-notify@v2 + env: + SLACK_COLOR: 'danger' + SLACK_FOOTER: '' + SLACK_ICON_EMOJI: ':github-actions:' + SLACK_TITLE: 'Publishing Docker images on Quay.io failed:' + SLACK_USERNAME: 'GitHub Actions' + SLACK_MESSAGE: |- + ${{ github.repository }}: Publishing Docker images on Quay.io failed. + Checkout: ${{ steps.checkout.outcome }} + Download digests: ${{ steps.fetch-digests.outcome }} + Setup buildx: ${{ steps.prepare.outcome }} + Login to registry: ${{ steps.login.outcome }} + Create and push manifest: ${{ steps.manifest.outcome }} + SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }} + if: >- + ${{ + failure() + && github.repository == 'netdata/netdata' + }} - docker-dbg-publish: + build-images-ghcr: + name: Push Images to GHCR if: github.event_name == 'workflow_dispatch' - name: Docker Build and Publish (Debugging Image) needs: - - docker-test - - normalize-tag + - build-images + - gen-tags + strategy: + matrix: + platform: + - linux/amd64 + - linux/i386 + - linux/arm/v7 + - linux/arm64 + - linux/ppc64le runs-on: ubuntu-latest steps: - name: Checkout id: checkout uses: actions/checkout@v4 with: + fetch-depth: 0 submodules: recursive - - name: Determine which tags to use - id: release-tags - if: github.event.inputs.version != 'nightly' - run: | - echo "tags=netdata/netdata-debug:latest,netdata/netdata-debug:stable,ghcr.io/netdata/netdata-debug:latest,ghcr.io/netdata/netdata-debug:stable,quay.io/netdata/netdata-debug:latest,quay.io/netdata/netdata-debug:stable,$(.github/scripts/gen-docker-tags.py ${{ needs.normalize-tag.outputs.tag }} '-debug')" \ - >> "${GITHUB_ENV}" - - name: Determine which tags to use - id: nightly-tags - if: github.event.inputs.version == 'nightly' - run: | - echo "tags=netdata/netdata-debug:latest,netdata/netdata-debug:edge,ghcr.io/netdata/netdata-debug:latest,ghcr.io/netdata/netdata-debug:edge,quay.io/netdata/netdata-debug:latest,quay.io/netdata/netdata-debug:edge" >> "${GITHUB_ENV}" + - name: Generate Artifact Name + id: artifact-name + run: echo "platform=$(echo ${{ matrix.platform }} | tr '/' '-' | cut -f 2- -d '-')" >> "${GITHUB_OUTPUT}" + - name: Download Cache + id: fetch-cache + uses: actions/download-artifact@v4 + with: + name: cache-${{ steps.artifact-name.outputs.platform }} + path: /tmp/build-cache - name: Mark image as official id: env if: github.repository == 'netdata/netdata' run: echo "OFFICIAL_IMAGE=true" >> "${GITHUB_ENV}" - name: Setup QEMU id: qemu + if: matrix.platform != 'linux/i386' && matrix.platform != 'linux/amd64' uses: docker/setup-qemu-action@v3 - name: Setup Buildx - id: buildx + id: prepare uses: docker/setup-buildx-action@v3 - - name: Docker Hub Login - id: docker-hub-login + - name: Registry Login + id: login if: github.repository == 'netdata/netdata' uses: docker/login-action@v3 with: - username: ${{ secrets.DOCKER_HUB_USERNAME }} - password: ${{ secrets.DOCKER_HUB_PASSWORD }} - - name: GitHub Container Registry Login - id: ghcr-login + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Build Image + id: build + uses: docker/build-push-action@v5 + with: + platforms: ${{ matrix.platform }} + cache-from: type=local,src=/tmp/build-cache + build-args: OFFICIAL_IMAGE=${{ env.OFFICIAL_IMAGE }} + outputs: type=image,name=ghcr.io/netdata/netdata,push-by-digest=true,name-canonical=true,push=true + - name: Export Digest + id: export-digest + if: github.repository == 'netdata/netdata' + run: | + mkdir -p /tmp/digests + digest="${{ steps.build.outputs.digest }}" + touch "/tmp/digests/${digest#sha256:}" + - name: Upload digest + id: upload-digest + if: github.repository == 'netdata/netdata' + uses: actions/upload-artifact@v4 + with: + name: ghcr-digests-${{ steps.artifact-name.outputs.platform }} + path: /tmp/digests/* + if-no-files-found: error + retention-days: 1 + - name: Failure Notification + uses: rtCamp/action-slack-notify@v2 + env: + SLACK_COLOR: 'danger' + SLACK_FOOTER: '' + SLACK_ICON_EMOJI: ':github-actions:' + SLACK_TITLE: 'GHCR upload failed:' + SLACK_USERNAME: 'GitHub Actions' + SLACK_MESSAGE: |- + ${{ github.repository }}: Creating or uploading Docker image for ${{ matrix.platform }} on GHCR failed. + Checkout: ${{ steps.checkout.outcome }} + Determine artifact name: ${{ steps.artifact-name.outcome }} + Fetch build cache: ${{ steps.fetch-cache.outcome }} + Setup environment: ${{ steps.env.outcome }} + Setup QEMU: ${{ steps.qemu.outcome }} + Setup buildx: ${{ steps.prepare.outcome }} + Login to registry: ${{ steps.login.outcome }} + Build image: ${{ steps.build.outcome }} + Export digest: ${{ steps.export-digest.outcome }} + Upload digest: ${{ steps.upload-digest.outcome }} + SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }} + if: >- + ${{ + failure() + && github.repository == 'netdata/netdata' + }} + + publish-ghcr: + name: Consolidate and tag images for GHCR + if: github.event_name == 'workflow_dispatch' + needs: + - build-images-ghcr + - gen-tags + runs-on: ubuntu-latest + steps: + - name: Checkout + id: checkout + uses: actions/checkout@v4 + - name: Download digests + id: fetch-digests + uses: actions/download-artifact@v4 + with: + path: /tmp/digests + pattern: ghcr-digests-* + merge-multiple: true + - name: Setup Buildx + id: prepare + uses: docker/setup-buildx-action@v3 + - name: Registry Login + id: login if: github.repository == 'netdata/netdata' uses: docker/login-action@v3 with: registry: ghcr.io username: ${{ github.repository_owner }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Quay.io Login - id: quay-login + - name: Create and Push Manifest + id: manifest if: github.repository == 'netdata/netdata' - uses: docker/login-action@v3 + run: docker buildx imagetools create $(.github/scripts/gen-docker-imagetool-args.py /tmp/digests 'ghcr.io' ${{ needs.gen-tags.outputs.tags }}) + - name: Failure Notification + uses: rtCamp/action-slack-notify@v2 + env: + SLACK_COLOR: 'danger' + SLACK_FOOTER: '' + SLACK_ICON_EMOJI: ':github-actions:' + SLACK_TITLE: 'Publishing Docker images on GHCR failed:' + SLACK_USERNAME: 'GitHub Actions' + SLACK_MESSAGE: |- + ${{ github.repository }}: Publishing Docker images on GHCR failed. + Checkout: ${{ steps.checkout.outcome }} + Download digests: ${{ steps.fetch-digests.outcome }} + Setup buildx: ${{ steps.prepare.outcome }} + Login to registry: ${{ steps.login.outcome }} + Create and push manifest: ${{ steps.manifest.outcome }} + SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }} + if: >- + ${{ + failure() + && github.repository == 'netdata/netdata' + }} + + trigger-subsequent-workflows: + if: github.event_name == 'workflow_dispatch' + name: Trigger subsquent workflows for newly added versions + needs: + - publish-docker-hub + - gen-tags + runs-on: ubuntu-latest + steps: + - name: Checkout + id: checkout + uses: actions/checkout@v4 with: - registry: quay.io - username: ${{ secrets.NETDATABOT_QUAY_USERNAME }} - password: ${{ secrets.NETDATABOT_QUAY_TOKEN }} - - name: Docker Build - id: build - uses: docker/build-push-action@v5 + submodules: recursive + - name: Trigger Helmchart PR + if: github.event_name == 'workflow_dispatch' && github.event.inputs.version != 'nightly' && github.repository == 'netdata/netdata' + id: trigger-helmchart + uses: benc-uk/workflow-dispatch@v1 + with: + token: ${{ secrets.NETDATABOT_GITHUB_TOKEN }} + repo: netdata/helmchart + workflow: Agent Version PR + ref: refs/heads/master + inputs: '{"agent_version": "${{ inputs.version }}"}' + - name: Trigger MSI build + if: github.event_name == 'workflow_dispatch' && github.event.inputs.version != 'nightly' && github.repository == 'netdata/netdata' + id: trigger-msi + uses: benc-uk/workflow-dispatch@v1 with: - platforms: linux/amd64,linux/i386,linux/arm/v7,linux/arm64,linux/ppc64le - push: ${{ github.repository == 'netdata/netdata' }} - tags: ${{ env.tags }} - build-args: | - OFFICIAL_IMAGE=${{ env.OFFICIAL_IMAGE }} - DEBUG_BUILD=1 + token: ${{ secrets.NETDATABOT_GITHUB_TOKEN }} + repo: netdata/msi-installer + workflow: Build + ref: refs/heads/master + inputs: '{"tag": "stable", "pwd": "${{ secrets.MSI_CODE_SIGNING_PASSWORD }}"}' - name: Failure Notification uses: rtCamp/action-slack-notify@v2 env: SLACK_COLOR: 'danger' SLACK_FOOTER: '' SLACK_ICON_EMOJI: ':github-actions:' - SLACK_TITLE: 'Docker Debug Build failed:' + SLACK_TITLE: ':' SLACK_USERNAME: 'GitHub Actions' SLACK_MESSAGE: |- - ${{ github.repository }}: Failed to build or publish Docker debug images. + ${{ github.repository }}: Version cascade failed Checkout: ${{ steps.checkout.outcome }} - Generate release tags: ${{ steps.release-tags.outcome }} - Generate nightly tags: ${{ steps.nightly-tags.outcome }} - Setup environment: ${{ steps.env.outcome }} - Setup QEMU: ${{ steps.qemu.outcome }} - Setup buildx: ${{ steps.buildx.outcome }} - Login to DockerHub: ${{ steps.docker-hub-login.outcome }} - Login to GHCR: ${{ steps.ghcr-login.outcome }} - Login to Quay: ${{ steps.quay-login.outcome }} - Build and publish images: ${{ steps.build.outcome }} + Trigger Helmchart PR: ${{ steps.trigger-helmchart.outcome }} + Trigger MSI build: ${{ steps.trigger-msi.outcome }} SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }} if: >- ${{ diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml deleted file mode 100644 index a0554b167..000000000 --- a/.github/workflows/docs.yml +++ /dev/null @@ -1,29 +0,0 @@ ---- -name: Docs -on: - push: - branches: - - master - paths: - - '**.md' - pull_request: - paths: - - '**.md' -env: - DISABLE_TELEMETRY: 1 -jobs: - markdown-link-check: - name: Broken Links - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - submodules: recursive - - name: Run link check - uses: gaurav-nelson/github-action-markdown-link-check@v1 - with: - use-quiet-mode: 'no' - use-verbose-mode: 'yes' - check-modified-files-only: 'yes' - config-file: '.mlc_config.json' diff --git a/.github/workflows/generate-integrations.yml b/.github/workflows/generate-integrations.yml index 4128e9925..f7d4df866 100644 --- a/.github/workflows/generate-integrations.yml +++ b/.github/workflows/generate-integrations.yml @@ -6,15 +6,15 @@ on: branches: - master paths: # If any of these files change, we need to regenerate integrations.js. - - 'collectors/**/metadata.yaml' - - 'exporting/**/metadata.yaml' - - 'health/notifications/**/metadata.yaml' + - 'src/collectors/**/metadata.yaml' + - 'src/go/collectors/**/metadata.yaml' + - 'src/exporting/**/metadata.yaml' + - 'src/health/notifications/**/metadata.yaml' - 'integrations/templates/**' - 'integrations/categories.yaml' - 'integrations/deploy.yaml' - 'integrations/cloud-notifications/metadata.yaml' - 'integrations/gen_integrations.py' - - 'packaging/go.d.version' workflow_dispatch: null concurrency: # This keeps multiple instances of the job from running concurrently for the same ref. group: integrations-${{ github.ref }} @@ -31,17 +31,6 @@ jobs: with: fetch-depth: 1 submodules: recursive - - name: Get Go Ref - id: get-go-ref - run: echo "go_ref=$(cat packaging/go.d.version)" >> "${GITHUB_ENV}" - - name: Checkout Go - id: checkout-go - uses: actions/checkout@v4 - with: - fetch-depth: 1 - path: go.d.plugin - repository: netdata/go.d.plugin - ref: ${{ env.go_ref }} - name: Prepare Dependencies id: prep-deps run: | @@ -58,7 +47,7 @@ jobs: id: generate-integrations-documentation run: | python3 integrations/gen_docs_integrations.py - - name: Generate collectors/COLLECTORS.md + - name: Generate src/collectors/COLLECTORS.md id: generate-collectors-md run: | python3 integrations/gen_doc_collector_page.py @@ -67,7 +56,7 @@ jobs: run: rm -rf go.d.plugin virtualenv - name: Create PR id: create-pr - uses: peter-evans/create-pull-request@v5 + uses: peter-evans/create-pull-request@v6 with: token: ${{ secrets.NETDATABOT_GITHUB_TOKEN }} commit-message: Regenerate integrations.js @@ -95,7 +84,7 @@ jobs: Prepare Dependencies: ${{ steps.prep-deps.outcome }} Generate Integrations: ${{ steps.generate.outcome }} Generate Integrations Documentation: ${{ steps.generate-integrations-documentation.outcome }} - Generate collectors/COLLECTORS.md: ${{ steps.generate-collectors-md.outcome }} + Generate src/collectors/COLLECTORS.md: ${{ steps.generate-collectors-md.outcome }} Clean Up Temporary Data: ${{ steps.clean.outcome }} Create PR: ${{ steps.create-pr.outcome }} SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }} diff --git a/.github/workflows/go-tests.yml b/.github/workflows/go-tests.yml new file mode 100644 index 000000000..9e5550507 --- /dev/null +++ b/.github/workflows/go-tests.yml @@ -0,0 +1,124 @@ +--- +# Ci code for building release artifacts. +name: Go Tests +on: + push: # Master branch checks only validate the build and generate artifacts for testing. + branches: + - master + pull_request: null # PR checks only validate the build and generate artifacts for testing. +concurrency: # This keeps multiple instances of the job from running concurrently for the same ref and event type. + group: go-test-${{ github.ref }}-${{ github.event_name }} + cancel-in-progress: true +jobs: + file-check: # Check what files changed if we’re being run in a PR or on a push. + name: Check Modified Files + runs-on: ubuntu-latest + outputs: + run: ${{ steps.check-run.outputs.run }} + steps: + - name: Checkout + id: checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + submodules: recursive + - name: Check files + id: check-files + uses: tj-actions/changed-files@v43 + with: + since_last_remote_commit: ${{ github.event_name != 'pull_request' }} + files: | + **/*.cmake + CMakeLists.txt + .github/workflows/go-tests.yml + packaging/cmake/ + src/go/** + files_ignore: | + **/*.md + src/go/**/metadata.yaml + - name: List all changed files in pattern + continue-on-error: true + env: + ALL_CHANGED_FILES: ${{ steps.check-files.outputs.all_changed_files }} + run: | + for file in ${ALL_CHANGED_FILES}; do + echo "$file was changed" + done + - name: Check Run + id: check-run + run: | + if [ "${{ steps.check-files.outputs.any_modified }}" == "true" ] || [ "${{ github.event_name }}" == "workflow_dispatch" ]; then + echo 'run=true' >> "${GITHUB_OUTPUT}" + else + echo 'run=false' >> "${GITHUB_OUTPUT}" + fi + + matrix: + name: Generate Build Matrix + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.get-version.outputs.matrix }} + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Install dependencies + run: sudo apt-get update && sudo apt-get upgrade -y && sudo apt-get install -y python3-packaging + - name: Get Go version and modules + id: get-version + run: .github/scripts/get-go-version.py + + tests: + name: Go toolchain tests + runs-on: ubuntu-latest + needs: + - file-check + - matrix + strategy: + fail-fast: false + matrix: ${{ fromJson(needs.matrix.outputs.matrix) }} + steps: + - name: Skip Check + id: skip + if: needs.file-check.outputs.run != 'true' + run: echo "SKIPPED" + - name: Install Go + uses: actions/setup-go@v5 + with: + go-version: ${{ matrix.version }} + - name: Checkout + if: needs.file-check.outputs.run == 'true' + uses: actions/checkout@v4 + with: + submodules: recursive + - name: Go mod download + if: needs.file-check.outputs.run == 'true' + run: go mod download + working-directory: ${{ matrix.module }} + - name: Compile + if: needs.file-check.outputs.run == 'true' + run: | + CGO_ENABLED=0 go build -o /tmp/go-test-build ${{ matrix.build_target }} + /tmp/go-test-build --help || true + working-directory: ${{ matrix.module }} + - name: Go fmt + if: needs.file-check.outputs.run == 'true' + run: | + go fmt ./... | tee modified-files + [ "$(wc -l modified-files | cut -f 1 -d ' ')" -eq 0 ] || exit 1 + working-directory: ${{ matrix.module }} + - name: Go vet + if: needs.file-check.outputs.run == 'true' + run: go vet ./... + working-directory: ${{ matrix.module }} + - name: Set up gotestfmt + if: needs.file-check.outputs.run == 'true' + uses: GoTestTools/gotestfmt-action@v2 + with: + token: ${{ secrets.GITHUB_TOKEN }} + version: v2.0.0 + - name: Go test + if: needs.file-check.outputs.run == 'true' + run: | + set -euo pipefail + go test -json ./... -race -count=1 2>&1 | gotestfmt -hide all + working-directory: ${{ matrix.module }} diff --git a/.github/workflows/kickstart-upload.yml b/.github/workflows/kickstart-upload.yml new file mode 100644 index 000000000..77c26d7bc --- /dev/null +++ b/.github/workflows/kickstart-upload.yml @@ -0,0 +1,54 @@ +--- +# Upload the kickstart script to the repo server +name: Upload Kickstart Script +on: + push: + branches: + - master + paths: + - .github/workflows/kickstart-upload.yml + - packaging/installer/kickstart.sh + workflow_dispatch: null +concurrency: + group: kickstart-upload + cancel-in-progress: true +jobs: + upload: + name: Upload Kickstart Script + runs-on: ubuntu-latest + if: github.repository == 'netdata/netdata' + steps: + - name: Checkout + id: checkout + uses: actions/checkout@v4 + with: + fetch-depth: 1 + - name: SSH setup + id: ssh-setup + uses: shimataro/ssh-key-action@v2 + with: + key: ${{ secrets.NETDATABOT_PACKAGES_SSH_KEY }} + name: id_ecdsa + known_hosts: ${{ secrets.PACKAGES_KNOWN_HOSTS }} + - name: Upload + id: upload + run: rsync -vp packaging/installer/kickstart.sh netdatabot@packages.netdata.cloud:/home/netdatabot/incoming/kickstart.sh + - name: Failure Notification + uses: rtCamp/action-slack-notify@v2 + env: + SLACK_COLOR: 'danger' + SLACK_ICON_EMOJI: ':github-actions:' + SLACK_TITLE: 'Kickstart upload failed:' + SLACK_USERNAME: 'GitHub Actions' + SLACK_MESSAGE: |- + ${{ github.repository }}: Failed to upload updated kickstart script to repo server. + Checkout: ${{ steps.checkout.outcome }} + Import SSH Key: ${{ steps.ssh-setup.outcome }} + Upload: ${{ steps.upload.outcome }} + SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }} + if: >- + ${{ + failure() + && startsWith(github.ref, 'refs/heads/master') + && github.repository == 'netdata/netdata' + }} diff --git a/.github/workflows/labeler.yml b/.github/workflows/labeler.yml index a1e3b52fe..3f66b98f4 100644 --- a/.github/workflows/labeler.yml +++ b/.github/workflows/labeler.yml @@ -14,7 +14,7 @@ jobs: contents: read pull-requests: write steps: - - uses: actions/labeler@v4 + - uses: actions/labeler@v5 if: github.repository == 'netdata/netdata' with: repo-token: "${{ secrets.GITHUB_TOKEN }}" diff --git a/.github/workflows/monitor-releases.yml b/.github/workflows/monitor-releases.yml index 649cf68aa..7962c9861 100644 --- a/.github/workflows/monitor-releases.yml +++ b/.github/workflows/monitor-releases.yml @@ -27,7 +27,7 @@ jobs: with: token: ${{ secrets.NETDATABOT_GITHUB_TOKEN }} - name: Init python environment - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 id: init-python with: python-version: "3.12" diff --git a/.github/workflows/packaging.yml b/.github/workflows/packaging.yml index eb936c4d9..7e72fb3f0 100644 --- a/.github/workflows/packaging.yml +++ b/.github/workflows/packaging.yml @@ -40,39 +40,47 @@ jobs: fetch-depth: 0 submodules: recursive - name: Check files - id: file-check - uses: tj-actions/changed-files@v40 + id: check-files + uses: tj-actions/changed-files@v43 with: since_last_remote_commit: ${{ github.event_name != 'pull_request' }} files: | - **.c - **.cc - **.h - **.hh - **.in + **/*.c + **/*.cc + **/*.h + **/*.hh + **/*.in + **/*.patch + **/*.cmake netdata.spec.in - configure.ac - **/Makefile* - Makefile* + contrib/debian/ + CMakeLists.txt .github/data/distros.yml .github/workflows/packaging.yml .github/scripts/gen-matrix-packaging.py .github/scripts/pkg-test.sh - build/** + packaging/cmake/ packaging/*.sh - packaging/*.checksums packaging/*.version - contrib/debian/** - aclk/aclk-schemas/ - ml/dlib/ - mqtt_websockets - web/server/h2o/libh2o + packaging/*.checksums + src/aclk/aclk-schemas/ + src/ml/dlib/ + src/fluent-bit/ + src/web/server/h2o/libh2o/ files_ignore: | - **.md + **/*.md + - name: List all changed files in pattern + continue-on-error: true + env: + ALL_CHANGED_FILES: ${{ steps.check-files.outputs.all_changed_files }} + run: | + for file in ${ALL_CHANGED_FILES}; do + echo "$file was changed" + done - name: Check Run id: check-run run: | - if [ "${{ steps.file-check.outputs.any_modified }}" == "true" ] || [ "${{ github.event_name }}" == "workflow_dispatch" ]; then + if [ "${{ steps.check-files.outputs.any_modified }}" == "true" ] || [ "${{ github.event_name }}" == "workflow_dispatch" ]; then echo 'run=true' >> "${GITHUB_OUTPUT}" else echo 'run=false' >> "${GITHUB_OUTPUT}" @@ -207,6 +215,18 @@ jobs: with: fetch-depth: 0 # We need full history for versioning submodules: recursive + - name: Set Sentry telemetry env vars + id: set-telemetry-env-vars + run: | + if [ "${{ github.repository }}" = 'netdata/netdata' ] && \ + [ "${{ matrix.bundle_sentry }}" = 'true' ] && \ + [ "${{ github.event_name }}" = 'workflow_dispatch' ]; then + echo "RELEASE_PIPELINE=Production" >> "${GITHUB_ENV}" + echo "UPLOAD_SENTRY=true" >> "${GITHUB_ENV}" + else + echo "RELEASE_PIPELINE=Unknown" >> "${GITHUB_ENV}" + echo "UPLOAD_SENTRY=false" >> "${GITHUB_ENV}" + fi - name: Setup QEMU id: qemu if: matrix.platform != 'linux/amd64' && matrix.platform != 'linux/i386' && needs.file-check.outputs.run == 'true' @@ -221,7 +241,7 @@ jobs: - name: Fetch images id: fetch-images if: needs.file-check.outputs.run == 'true' - uses: nick-invision/retry@v2 + uses: nick-invision/retry@v3 with: max_attempts: 3 retry_wait_seconds: 30 @@ -235,12 +255,15 @@ jobs: shell: bash run: | docker run --security-opt seccomp=unconfined -e DISABLE_TELEMETRY=1 -e VERSION=${{ needs.version-check.outputs.version }} \ + -e ENABLE_SENTRY=${{ matrix.bundle_sentry }} -e RELEASE_PIPELINE=${{ env.RELEASE_PIPELINE }} \ + -e BUILD_DESTINATION=${{ matrix.distro }}${{ matrix.version }}_${{ matrix.arch }} -e UPLOAD_SENTRY=${{ env.UPLOAD_SENTRY }} \ + -e SENTRY_AUTH_TOKEN=${{ secrets.SENTRY_CLI_TOKEN }} -e NETDATA_SENTRY_DSN=${{ secrets.SENTRY_DSN }} \ --platform=${{ matrix.platform }} -v "$PWD":/netdata netdata/package-builders:${{ matrix.distro }}${{ matrix.version }}-v1 - name: Save Packages id: artifacts if: needs.file-check.outputs.run == 'true' continue-on-error: true - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: ${{ matrix.distro }}-${{ matrix.version }}-${{ matrix.arch }}-packages path: ${{ github.workspace }}/artifacts/* diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 2fa51cc52..91f2d5493 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -46,7 +46,7 @@ jobs: - name: Generate Nightly Changleog id: nightly-changelog if: steps.target.outputs.run == 'true' && steps.target.outputs.type == 'nightly' - uses: heinrichreimer/github-changelog-generator-action@v2.3 + uses: heinrichreimer/github-changelog-generator-action@v2.4 with: bugLabels: IGNOREBUGS excludeLabels: "stale,duplicate,question,invalid,wontfix,discussion,no changelog" @@ -59,7 +59,7 @@ jobs: - name: Generate Release Changelog id: release-changelog if: steps.target.outputs.run == 'true' && steps.target.outputs.type != 'nightly' - uses: heinrichreimer/github-changelog-generator-action@v2.3 + uses: heinrichreimer/github-changelog-generator-action@v2.4 with: bugLabels: IGNOREBUGS excludeLabels: "stale,duplicate,question,invalid,wontfix,discussion,no changelog" diff --git a/.github/workflows/repoconfig-packages.yml b/.github/workflows/repoconfig-packages.yml index df8fac204..f3d65a3e1 100644 --- a/.github/workflows/repoconfig-packages.yml +++ b/.github/workflows/repoconfig-packages.yml @@ -81,7 +81,7 @@ jobs: # Unlike normally, we do not need a deep clone or submodules for this. - name: Fetch base image id: fetch-images - uses: nick-invision/retry@v2 + uses: nick-invision/retry@v3 with: max_attempts: 3 retry_wait_seconds: 30 diff --git a/.github/workflows/review.yml b/.github/workflows/review.yml index 8cb61f2cf..c7b038135 100644 --- a/.github/workflows/review.yml +++ b/.github/workflows/review.yml @@ -16,8 +16,8 @@ jobs: outputs: actionlint: ${{ steps.actionlint.outputs.run }} clangformat: ${{ steps.clangformat.outputs.run }} - eslint: ${{ steps.eslint.outputs.run }} flake8: ${{ steps.flake8.outputs.run }} + golangci-lint: ${{ steps.golangci-lint.outputs.run }} hadolint: ${{ steps.hadolint.outputs.run }} shellcheck: ${{ steps.shellcheck.outputs.run }} yamllint: ${{ steps.yamllint.outputs.run }} @@ -49,17 +49,6 @@ jobs: # else # echo "run=false" >> "${GITHUB_OUTPUT}" # fi - - name: Check files for eslint - id: eslint - run: | - if [ "${{ contains(github.event.pull_request.labels.*.name, 'run-ci/eslint') }}" = "true" ]; then - echo "run=true" >> "${GITHUB_OUTPUT}" - elif git diff --name-only origin/${{ github.base_ref }} HEAD | grep -v "web/gui/v1" | grep -v "web/gui/v2" | grep -v "integrations/" | grep -Eq '.*\.js' ; then - echo "run=true" >> "${GITHUB_OUTPUT}" - echo 'JS files have changed, need to run ESLint.' - else - echo "run=false" >> "${GITHUB_OUTPUT}" - fi - name: Check files for flake8 id: flake8 run: | @@ -71,6 +60,17 @@ jobs: else echo "run=false" >> "${GITHUB_OUTPUT}" fi + - name: Check files for golangci-lint + id: golangci-lint + run: | + if [ "${{ contains(github.event.pull_request.labels.*.name, 'run-ci/golangci-lint') }}" = "true" ]; then + echo "run=true" >> "${GITHUB_OUTPUT}" + elif git diff --name-only origin/${{ github.base_ref }} HEAD -- | grep -Eq '.*\.go' ; then + echo "run=true" >> $GITHUB_OUTPUT + echo 'Go code has changed, need to run golangci-lint.' + else + echo "run=false" >> $GITHUB_OUTPUT + fi - name: Check files for hadolint id: hadolint run: | @@ -155,26 +155,6 @@ jobs: exit 1 fi - eslint: - name: eslint - needs: prep-review - if: needs.prep-review.outputs.eslint == 'true' - runs-on: ubuntu-latest - steps: - - name: Git clone repository - uses: actions/checkout@v4 - with: - submodules: recursive - fetch-depth: 0 - - name: Install eslint - run: npm install eslint -D - - name: Run eslint - uses: reviewdog/action-eslint@v1 - with: - github_token: ${{ secrets.GITHUB_TOKEN }} - reporter: github-pr-check - eslint_flags: '.' - flake8: name: flake8 needs: prep-review @@ -187,7 +167,7 @@ jobs: submodules: recursive fetch-depth: 0 - name: Setup Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: "3.10" - name: Run flake8 @@ -196,6 +176,26 @@ jobs: github_token: ${{ secrets.GITHUB_TOKEN }} reporter: github-pr-check + golangci-lint: + name: golangci-lint + needs: prep-review + if: needs.prep-review.outputs.golangci-lint == 'true' + strategy: + matrix: + tree: + - src/go/collectors/go.d.plugin + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Run golangci-lint + uses: reviewdog/action-golangci-lint@v2 + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + reporter: github-pr-check + golangci_lint_flags: '--timeout=10m' + workdir: ${{ matrix.tree }} + hadolint: name: hadolint needs: prep-review |