diff options
Diffstat (limited to '.github')
24 files changed, 421 insertions, 219 deletions
diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index d038ce680..95116696f 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -5,9 +5,8 @@ * @Ferroin # Ownership by directory structure -.github/ @Ferroin @tkatsoulas +.github/ @netdata/agent-sre src/aclk/ @stelfrag -contrib/debian @Ferroin @tkatsoulas src/collectors/ @thiagoftsm src/collectors/ebpf.plugin/ @thiagoftsm src/collectors/charts.d.plugin/ @ilyam8 @Ferroin @@ -18,36 +17,37 @@ src/collectors/cups.plugin/ @thiagoftsm src/exporting/ @thiagoftsm src/daemon/ @thiagoftsm @vkalintiris src/database/ @thiagoftsm @vkalintiris -docs/ @tkatsoulas @Ancairon +docs/ @Ancairon src/go/ @ilyam8 src/health/ @thiagoftsm @vkalintiris src/health/health.d/ @thiagoftsm src/health/notifications/ @Ferroin @thiagoftsm src/ml/ @vkalintiris src/libnetdata/ @thiagoftsm @vkalintiris -packaging/ @Ferroin @tkatsoulas +packaging/ @netdata/agent-sre packaging/cmake @Ferroin @vkalintiris src/registry/ @novykh src/streaming/ @thiagoftsm -system/ @Ferroin @tkatsoulas -tests/ @Ferroin @vkalintiris @tkatsoulas +system/ @netdata/agent-sre +tests/ @netdata/agent-sre @vkalintiris src/web/ @thiagoftsm @vkalintiris src/web/gui/ @novykh src/logsmanagement/ @thiagoftsm # Ownership by filetype (overwrites ownership by directory) -*.md @tkatsoulas @Ancairon -*.mdx @tkatsoulas @Ancairon -Dockerfile* @Ferroin @tkatsoulas +*.md @Ancairon +*.mdx @Ancairon +Dockerfile* @netdata/agent-sre # Ownership of specific files -.gitignore @Ferroin @tkatsoulas @vkalintiris -.codacy.yml @Ferroin @tkatsoulas -.yamllint.yml @Ferroin @tkatsoulas -CMakeLists.txt @Ferroin @vkalintiris -netdata.spec.in @Ferroin @tkatsoulas -netdata-installer.sh @Ferroin @tkatsoulas -packaging/version @netdatabot @Ferroin @tkatsoulas +.gitignore @netdata/agent-sre @vkalintiris +.codacy.yml @netdata/agent-sre +.yamllint.yml @netdata/agent-sre +CMakeLists.txt @netdata/agent-sre @vkalintiris +netdata.spec.in @netdata/agent-sre +netdata-installer.sh @netdata/agent-sre +packaging/version @netdatabot @netdata/agent-sre +packaging/repoconfig/CMakelists.txt @netdata/agent-sre -LICENSE.md @Ferroin @tkatsoulas @vkalintiris -CHANGELOG.md @netdatabot @Ferroin @tkatsoulas +LICENSE.md @netdata/agent-sre @vkalintiris +CHANGELOG.md @netdatabot @netdata/agent-sre diff --git a/.github/data/distros.yml b/.github/data/distros.yml index 811c78f07..838cb72fc 100644 --- a/.github/data/distros.yml +++ b/.github/data/distros.yml @@ -16,6 +16,15 @@ arch_order: # sort order for per-architecture jobs in CI - armhfp - arm64 - aarch64 +default_sentry: &default_sentry # Default configuration for Sentry usage + amd64: false + x86_64: false + i386: false + armhf: false + armhfp: false + arm64: false + aarch64: false +default_builder_rev: &def_builder_rev v1 include: - &alpine distro: alpine @@ -23,6 +32,7 @@ include: support_type: Community notes: '' eol_check: false + bundle_sentry: *default_sentry env_prep: | apk add -U bash jsonc_removal: | @@ -30,23 +40,22 @@ include: test: ebpf-core: true - <<: *alpine - version: "3.19" + version: "3.20" support_type: Core notes: '' eol_check: true - bundle_sentry: false - <<: *alpine - version: "3.18" + version: "3.19" support_type: Core notes: '' eol_check: true - <<: *alpine - version: "3.17" + version: "3.18" support_type: Intermediate notes: '' eol_check: true - <<: *alpine - version: "3.16" + version: "3.17" support_type: Intermediate notes: '' eol_check: true @@ -56,7 +65,7 @@ include: support_type: Intermediate notes: '' eol_check: false - bundle_sentry: false + bundle_sentry: *default_sentry env_prep: | pacman --noconfirm -Syu && pacman --noconfirm -Sy grep libffi test: @@ -68,10 +77,11 @@ include: support_type: Core notes: '' eol_check: 'amazon-linux' - bundle_sentry: false + bundle_sentry: *default_sentry packages: &amzn_packages type: rpm repo_distro: amazonlinux/2 + builder_rev: *def_builder_rev arches: - x86_64 - aarch64 @@ -89,13 +99,15 @@ include: - distro: centos version: "7" + base_image: "netdata/legacy:centos7" support_type: Core notes: '' eol_check: false - bundle_sentry: false + bundle_sentry: *default_sentry packages: type: rpm repo_distro: el/7 + builder_rev: *def_builder_rev alt_links: - el/7Server - el/7Client @@ -114,21 +126,16 @@ include: jsonc_removal: | dnf remove -y json-c-devel eol_check: true - bundle_sentry: false + bundle_sentry: *default_sentry packages: &cs_packages type: rpm repo_distro: el/c9s + builder_rev: *def_builder_rev arches: - x86_64 - aarch64 test: ebpf-core: true - - <<: *centos_stream - version: '8' - base_image: 'quay.io/centos/centos:stream8' - packages: - <<: *cs_packages - repo_distro: el/c8s - &debian distro: debian @@ -137,7 +144,9 @@ include: notes: '' base_image: debian:bookworm eol_check: true - bundle_sentry: true + bundle_sentry: + <<: *default_sentry + amd64: true env_prep: | apt-get update jsonc_removal: | @@ -145,6 +154,7 @@ include: packages: &debian_packages type: deb repo_distro: debian/bookworm + builder_rev: v2 arches: - i386 - amd64 @@ -155,7 +165,6 @@ include: - <<: *debian version: "11" base_image: debian:bullseye - bundle_sentry: false packages: <<: *debian_packages repo_distro: debian/bullseye @@ -164,7 +173,7 @@ include: - <<: *debian version: "10" base_image: debian:buster - bundle_sentry: false + bundle_sentry: *default_sentry packages: <<: *debian_packages repo_distro: debian/buster @@ -173,47 +182,55 @@ include: - &fedora distro: fedora - version: "39" + version: "40" support_type: Core notes: '' eol_check: true - bundle_sentry: false + bundle_sentry: *default_sentry jsonc_removal: | dnf remove -y json-c-devel packages: &fedora_packages type: rpm - repo_distro: fedora/39 + repo_distro: fedora/40 + builder_rev: *def_builder_rev arches: - x86_64 - aarch64 test: ebpf-core: true - <<: *fedora - version: "38" + version: "39" packages: <<: *fedora_packages - repo_distro: fedora/38 + repo_distro: fedora/39 test: ebpf-core: true - &opensuse distro: opensuse - version: "15.5" + version: "15.6" support_type: Core notes: '' eol_check: true - bundle_sentry: false - base_image: opensuse/leap:15.5 + bundle_sentry: *default_sentry + base_image: opensuse/leap:15.6 jsonc_removal: | zypper rm -y libjson-c-devel packages: &opensuse_packages type: rpm - repo_distro: opensuse/15.5 + repo_distro: opensuse/15.6 + builder_rev: *def_builder_rev arches: - x86_64 - aarch64 test: ebpf-core: true + - <<: *opensuse + version: "15.5" + base_image: opensuse/leap:15.5 + packages: + <<: *opensuse_packages + repo_distro: opensuse/15.5 - &oracle distro: oraclelinux @@ -221,12 +238,13 @@ include: support_type: Core notes: '' eol_check: true - bundle_sentry: false + bundle_sentry: *default_sentry jsonc_removal: | dnf remove -y json-c-devel packages: &oracle_packages type: rpm repo_distro: ol/8 + builder_rev: *def_builder_rev arches: - x86_64 - aarch64 @@ -246,10 +264,11 @@ include: jsonc_removal: | dnf remove -y json-c-devel eol_check: true - bundle_sentry: false + bundle_sentry: *default_sentry packages: &rocky_packages type: rpm repo_distro: el/9 + builder_rev: *def_builder_rev alt_links: - el/9Server - el/9Client @@ -271,18 +290,21 @@ include: - &ubuntu distro: ubuntu - version: "22.04" + version: "24.04" support_type: Core notes: '' eol_check: true - bundle_sentry: false + bundle_sentry: + <<: *default_sentry + amd64: true env_prep: | rm -f /etc/apt/apt.conf.d/docker && apt-get update jsonc_removal: | apt-get remove -y libjson-c-dev packages: &ubuntu_packages type: deb - repo_distro: ubuntu/jammy + repo_distro: ubuntu/noble + builder_rev: v2 arches: - amd64 - armhf @@ -295,6 +317,11 @@ include: <<: *ubuntu_packages repo_distro: ubuntu/mantic - <<: *ubuntu + version: "22.04" + packages: + <<: *ubuntu_packages + repo_distro: ubuntu/jammy + - <<: *ubuntu version: "20.04" packages: <<: *ubuntu_packages @@ -302,16 +329,25 @@ include: legacy: # Info for platforms we used to support and still need to handle packages for - <<: *fedora version: "37" - bundle_sentry: false packages: <<: *fedora_packages repo_distro: fedora/37 + - <<: *fedora + version: "38" + packages: + <<: *fedora_packages + repo_distro: fedora/38 - <<: *opensuse version: "15.4" - bundle_sentry: false packages: <<: *opensuse_packages repo_distro: opensuse/15.4 + - <<: *centos_stream + version: '8' + base_image: 'quay.io/centos/centos:stream8' + packages: + <<: *cs_packages + repo_distro: el/c8s no_include: # Info for platforms not covered in CI - distro: docker version: "19.03 or newer" diff --git a/.github/labeler.yml b/.github/labeler.yml index d7c357509..0cbec181d 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -137,7 +137,7 @@ collectors/freeipmi: - any-glob-to-any-file: - src/collectors/freeipmi.plugin/** -collectors/go.d.plugin: +collectors/go.d: - any: - changed-files: - any-glob-to-any-file: @@ -215,6 +215,12 @@ collectors/timex: - any-glob-to-any-file: - src/collectors/timex.plugin/** +collectors/windows: + - any: + - changed-files: + - any-glob-to-any-file: + - src/collectors/windows.plugin/** + collectors/xenstat: - any: - changed-files: @@ -252,7 +258,6 @@ area/packaging: - any: - changed-files: - any-glob-to-any-file: - - contrib/** - packaging/** - system/** - Dockerfile* diff --git a/.github/scripts/build-static.sh b/.github/scripts/build-static.sh index e81051438..9b29a3d99 100755 --- a/.github/scripts/build-static.sh +++ b/.github/scripts/build-static.sh @@ -22,7 +22,7 @@ prepare_build() { build_static() { progress "Building static ${BUILDARCH}" ( - USER="" ./packaging/makeself/build-static.sh "${BUILDARCH}" + EXTRA_INSTALL_FLAGS="${EXTRA_INSTALL_FLAGS}" USER="" ./packaging/makeself/build-static.sh "${BUILDARCH}" ) >&2 } diff --git a/.github/scripts/gen-docker-tags.py b/.github/scripts/gen-docker-tags.py index c45b991d9..f60e1cd41 100755 --- a/.github/scripts/gen-docker-tags.py +++ b/.github/scripts/gen-docker-tags.py @@ -31,4 +31,6 @@ match version: ) ]) + tags = tags + tuple([f'{r}:stable' for r in REPOS]) + print(','.join(tags)) diff --git a/.github/scripts/gen-matrix-packaging.py b/.github/scripts/gen-matrix-packaging.py index b695e33ab..c00ff3181 100755 --- a/.github/scripts/gen-matrix-packaging.py +++ b/.github/scripts/gen-matrix-packaging.py @@ -27,8 +27,9 @@ for i, v in enumerate(data['include']): 'repo_distro': data['include'][i]['packages']['repo_distro'], 'format': data['include'][i]['packages']['type'], 'base_image': data['include'][i]['base_image'] if 'base_image' in data['include'][i] else ':'.join([data['include'][i]['distro'], data['include'][i]['version']]), + 'builder_rev': data['include'][i]['packages']['builder_rev'], 'platform': data['platform_map'][arch], - 'bundle_sentry': data['include'][i]['bundle_sentry'], + 'bundle_sentry': data['include'][i]['bundle_sentry'][arch], 'arch': arch }) diff --git a/.github/scripts/package-upload.sh b/.github/scripts/package-upload.sh index 13d63b4a7..9305ffa67 100755 --- a/.github/scripts/package-upload.sh +++ b/.github/scripts/package-upload.sh @@ -2,18 +2,19 @@ set -e -host="packages.netdata.cloud" user="netdatabot" -distro="${1}" -arch="${2}" -format="${3}" -repo="${4}" +host="${1}" +distro="${2}" +arch="${3}" +format="${4}" +repo="${5}" +pkg_src="${6:-./artifacts}" staging="${TMPDIR:-/tmp}/package-staging" prefix="/home/netdatabot/incoming/${repo}/" -packages="$(find artifacts -name "*.${format}")" +packages="$(find "${pkg_src}" -name "*.${format}")" mkdir -p "${staging}" diff --git a/.github/scripts/pkg-test.sh b/.github/scripts/pkg-test.sh index 6120d3fc3..f0c0dc11a 100755 --- a/.github/scripts/pkg-test.sh +++ b/.github/scripts/pkg-test.sh @@ -21,7 +21,9 @@ install_debian_like() { ! -name '*dbgsym*' ! -name '*cups*' ! -name '*freeipmi*') || exit 3 # Install testing tools - apt-get install -y --no-install-recommends curl "${netcat}" jq || exit 1 + apt-get install -y --no-install-recommends curl dpkg-dev "${netcat}" jq || exit 1 + + dpkg-architecture --equal amd64 || NETDATA_SKIP_EBPF=1 } install_fedora_like() { @@ -40,6 +42,8 @@ install_fedora_like() { # Install testing tools "${PKGMGR}" install -y curl nc jq || exit 1 + + [ "$(rpm --eval '%{_build_arch}')" = "x86_64" ] || NETDATA_SKIP_EBPF=1 } install_centos() { @@ -62,6 +66,8 @@ install_centos() { # Install testing tools # shellcheck disable=SC2086 "${PKGMGR}" install -y ${opts} curl nc jq || exit 1 + + [ "$(rpm --eval '%{_build_arch}')" = "x86_64" ] || NETDATA_SKIP_EBPF=1 } install_amazon_linux() { @@ -78,6 +84,8 @@ install_amazon_linux() { # Install testing tools # shellcheck disable=SC2086 "${PKGMGR}" install -y ${opts} curl nc jq || exit 1 + + [ "$(rpm --eval '%{_build_arch}')" = "x86_64" ] || NETDATA_SKIP_EBPF=1 } install_suse_like() { @@ -90,6 +98,8 @@ install_suse_like() { # Install testing tools zypper install -y --allow-downgrade --no-recommends curl netcat-openbsd jq || exit 1 + + [ "$(rpm --eval '%{_build_arch}')" = "x86_64" ] || NETDATA_SKIP_EBPF=1 } dump_log() { @@ -103,7 +113,7 @@ case "${DISTRO}" in fedora | oraclelinux) install_fedora_like ;; - centos| centos-stream | rockylinux | almalinux) + centos | centos-stream | rockylinux | almalinux) install_centos ;; amazonlinux) @@ -120,6 +130,13 @@ esac trap dump_log EXIT +export NETDATA_LIBEXEC_PREFIX=/usr/libexec/netdata +export NETDATA_SKIP_LIBEXEC_PARTS="logs-management|freeipmi|xenstat|nfacct|cups" + +if [ -n "${NETDATA_SKIP_EBPF}" ]; then + export NETDATA_SKIP_LIBEXEC_PARTS="${NETDATA_SKIP_LIBEXEC_PARTS}|ebpf" +fi + /usr/sbin/netdata -D > ./netdata.log 2>&1 & "${SCRIPT_DIR}/../../packaging/runtime-check.sh" || exit 1 diff --git a/.github/scripts/run-updater-check.sh b/.github/scripts/run-updater-check.sh index 2e70a10af..456a0e5d0 100755 --- a/.github/scripts/run-updater-check.sh +++ b/.github/scripts/run-updater-check.sh @@ -4,7 +4,7 @@ echo ">>> Installing CI support packages..." /netdata/.github/scripts/ci-support-pkgs.sh mkdir -p /etc/cron.daily # Needed to make auto-update checking work correctly on some platforms. echo ">>> Installing Netdata..." -/netdata/packaging/installer/kickstart.sh --dont-wait --build-only --disable-telemetry || exit 1 +/netdata/packaging/installer/kickstart.sh --dont-wait --build-only --dont-start-it --disable-telemetry "${EXTRA_INSTALL_FLAGS:+--local-build-options "${EXTRA_INSTALL_FLAGS}"}" || exit 1 echo "::group::>>> Pre-Update Environment File Contents" cat /etc/netdata/.environment echo "::endgroup::" diff --git a/.github/scripts/upload-new-version-tags.sh b/.github/scripts/upload-new-version-tags.sh index a9b0cd303..ffdfadfa0 100755 --- a/.github/scripts/upload-new-version-tags.sh +++ b/.github/scripts/upload-new-version-tags.sh @@ -2,9 +2,10 @@ set -e -host="packages.netdata.cloud" user="netdatabot" +host="${1}" + prefix="/var/www/html/releases" staging="${TMPDIR:-/tmp}/staging-new-releases" diff --git a/.github/workflows/add-to-project.yml b/.github/workflows/add-to-project.yml index 838917146..7716cab07 100644 --- a/.github/workflows/add-to-project.yml +++ b/.github/workflows/add-to-project.yml @@ -13,13 +13,13 @@ jobs: runs-on: ubuntu-latest steps: - name: Add issues to Agent project board - uses: actions/add-to-project@v0.6.0 + uses: actions/add-to-project@v1.0.1 with: project-url: https://github.com/orgs/netdata/projects/32 github-token: ${{ secrets.NETDATABOT_ORG_GITHUB_TOKEN }} - name: Add issues to Product Bug project board - uses: actions/add-to-project@v0.6.0 + uses: actions/add-to-project@v1.0.1 with: project-url: https://github.com/orgs/netdata/projects/45 github-token: ${{ secrets.NETDATABOT_ORG_GITHUB_TOKEN }} diff --git a/.github/workflows/build-macos.yml b/.github/workflows/build-macos.yml index dd6e41540..d3bfa6d77 100644 --- a/.github/workflows/build-macos.yml +++ b/.github/workflows/build-macos.yml @@ -25,7 +25,7 @@ jobs: submodules: recursive - name: Check files id: check-files - uses: tj-actions/changed-files@v43 + uses: tj-actions/changed-files@v44 with: since_last_remote_commit: ${{ github.event_name != 'pull_request' }} files: | @@ -52,6 +52,7 @@ jobs: files_ignore: | netdata.spec.in **/*.md + packaging/repoconfig/ - name: List all changed files in pattern continue-on-error: true env: @@ -105,7 +106,7 @@ jobs: id: install-nd-dep if: needs.file-check.outputs.run == 'true' run: | - bash ./packaging/installer/install-required-packages.sh --dont-wait --non-interactive netdata + bash ./packaging/installer/install-required-packages.sh --dont-wait --non-interactive netdata-all - name: Build from source id: build-source if: needs.file-check.outputs.run == 'true' diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 7ca83d867..cd48a63bc 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -25,6 +25,7 @@ jobs: runs-on: ubuntu-latest outputs: run: ${{ steps.check-run.outputs.run }} + skip-go: ${{ steps.check-go.outputs.skip-go }} steps: - name: Checkout id: checkout @@ -32,9 +33,9 @@ jobs: with: fetch-depth: 0 submodules: recursive - - name: Check files - id: check-files - uses: tj-actions/changed-files@v43 + - name: Check source files + id: check-source-files + uses: tj-actions/changed-files@v44 with: since_last_remote_commit: ${{ github.event_name != 'pull_request' }} files: | @@ -44,6 +45,19 @@ jobs: **/*.hh **/*.in **/*.patch + src/aclk/aclk-schemas/ + src/ml/dlib/ + src/fluent-bit/ + src/web/server/h2o/libh2o/ + files_ignore: | + netdata.spec.in + **/*.md + - name: Check build files + id: check-build-files + uses: tj-actions/changed-files@v44 + with: + since_last_remote_commit: ${{ github.event_name != 'pull_request' }} + files: | **/*.cmake CMakeLists.txt netdata-installer.sh @@ -59,29 +73,40 @@ jobs: packaging/*.sh packaging/*.version packaging/*.checksums - src/aclk/aclk-schemas/ - src/ml/dlib/ - src/fluent-bit/ - src/web/server/h2o/libh2o/ files_ignore: | - netdata.spec.in **/*.md + packaging/repoconfig/ - name: List all changed files in pattern continue-on-error: true env: - ALL_CHANGED_FILES: ${{ steps.check-files.outputs.all_changed_files }} + CHANGED_SOURCE_FILES: ${{ steps.check-source-files.outputs.all_changed_files }} + CHANGED_BUILD_FILES: ${{ steps.check-build-files.outputs.all_changed_files }} run: | - for file in ${ALL_CHANGED_FILES}; do + for file in ${CHANGED_SOURCE_FILES} ${CHANGED_BUILD_FILES} ; do echo "$file was changed" done - name: Check Run id: check-run run: | - if [ "${{ steps.check-files.outputs.any_modified }}" == "true" ] || [ "${{ github.event_name }}" == "workflow_dispatch" ]; then + if [ "${{ steps.check-source-files.outputs.any_modified }}" == "true" ] || [ "${{ steps.check-build-files.outputs.any_modified }}" == "true" ] || [ "${{ github.event_name }}" == "workflow_dispatch" ]; then echo 'run=true' >> "${GITHUB_OUTPUT}" else echo 'run=false' >> "${GITHUB_OUTPUT}" fi + - name: Check Go + id: check-go + env: + OTHER_CHANGED_FILES: ${{ steps.check-source-files.outputs.other_changed_files }} + run: | + if [ '${{ github.event_name }}' == 'pull_request' ]; then + if echo "${OTHER_CHANGED_FILES}" | grep -q '.*/(.*\.go|go\.mod|go\.sum)$' || [ "${{ steps.check-build-files.outputs.any_modified }}" == "true" ]; then + echo 'skip-go=' >> "${GITHUB_OUTPUT}" + else + echo 'skip-go=--disable-go' >> "${GITHUB_OUTPUT}" + fi + else + echo 'skip-go=' >> "${GITHUB_OUTPUT}" + fi build-dist: # Build the distribution tarball and store it as an artifact. name: Build Distribution Tarball @@ -204,7 +229,9 @@ jobs: key: ${{ steps.cache-key.outputs.key }} - name: Build if: github.event_name != 'workflow_dispatch' && needs.file-check.outputs.run == 'true' # Don’t use retries on PRs. - run: .github/scripts/build-static.sh ${{ matrix.arch }} + run: | + export EXTRA_INSTALL_FLAGS=${{ needs.file-check.outputs.skip-go }} + .github/scripts/build-static.sh ${{ matrix.arch }} - name: Build if: github.event_name == 'workflow_dispatch' && needs.file-check.outputs.run == 'true' id: build @@ -212,7 +239,9 @@ jobs: with: timeout_minutes: 180 max_attempts: 3 - command: .github/scripts/build-static.sh ${{ matrix.arch }} + command: | + export EXTRA_INSTALL_FLAGS=${{ needs.file-check.outputs.skip-go }} + .github/scripts/build-static.sh ${{ matrix.arch }} - name: Store id: store if: needs.file-check.outputs.run == 'true' @@ -259,7 +288,8 @@ jobs: - name: Prepare tools id: prepare run: | - sudo apt-get update && sudo apt-get install -y python3-ruamel.yaml + sudo apt-get update || true + sudo apt-get install -y python3-ruamel.yaml - name: Read build matrix id: set-matrix run: | @@ -415,7 +445,7 @@ jobs: - name: Fetch test environment id: fetch if: needs.file-check.outputs.run == 'true' - uses: Wandalen/wretry.action@v1 + uses: Wandalen/wretry.action@v3 with: action: actions/download-artifact@v4 with: | @@ -432,19 +462,19 @@ jobs: if: needs.file-check.outputs.run == 'true' run: | docker run --security-opt seccomp=unconfined -w /netdata test:${{ matrix.artifact_key }} \ - /bin/sh -c './netdata-installer.sh --dont-wait --dont-start-it --disable-cloud --one-time-build' + /bin/sh -c './netdata-installer.sh --dont-wait --dont-start-it --disable-cloud --one-time-build ${{ needs.file-check.outputs.skip-go }}' - name: netdata-installer on ${{ matrix.distro }}, require cloud id: build-cloud if: needs.file-check.outputs.run == 'true' run: | docker run --security-opt seccomp=unconfined -w /netdata test:${{ matrix.artifact_key }} \ - /bin/sh -c './netdata-installer.sh --dont-wait --dont-start-it --require-cloud --one-time-build' + /bin/sh -c './netdata-installer.sh --dont-wait --dont-start-it --require-cloud --one-time-build ${{ needs.file-check.outputs.skip-go }}' - name: netdata-installer on ${{ matrix.distro }}, require cloud, no JSON-C id: build-no-jsonc if: matrix.jsonc_removal != '' && needs.file-check.outputs.run == 'true' run: | docker run --security-opt seccomp=unconfined -w /netdata test:${{ matrix.artifact_key }} \ - /bin/sh -c '/rmjsonc.sh && ./netdata-installer.sh --dont-wait --dont-start-it --require-cloud --one-time-build' + /bin/sh -c '/rmjsonc.sh && ./netdata-installer.sh --dont-wait --dont-start-it --require-cloud --one-time-build ${{ needs.file-check.outputs.skip-go }}' - name: Failure Notification uses: rtCamp/action-slack-notify@v2 env: @@ -503,7 +533,7 @@ jobs: - name: Fetch dist tarball artifacts id: fetch-tarball if: needs.file-check.outputs.run == 'true' - uses: Wandalen/wretry.action@v1 + uses: Wandalen/wretry.action@v3 with: action: actions/download-artifact@v4 with: | @@ -529,7 +559,7 @@ jobs: - name: Fetch test environment id: fetch-test-environment if: needs.file-check.outputs.run == 'true' - uses: Wandalen/wretry.action@v1 + uses: Wandalen/wretry.action@v3 with: action: actions/download-artifact@v4 with: | @@ -545,8 +575,9 @@ jobs: id: updater-check if: needs.file-check.outputs.run == 'true' run: | - docker run --security-opt seccomp=unconfined -e DISABLE_TELEMETRY=1 --network host -w /netdata test:${{ matrix.artifact_key }} \ - /netdata/.github/scripts/run-updater-check.sh + docker run --security-opt seccomp=unconfined -e DISABLE_TELEMETRY=1 --network host -w /netdata \ + -e EXTRA_INSTALL_FLAGS=${{ needs.file-check.outputs.skip-go }} \ + test:${{ matrix.artifact_key }} /netdata/.github/scripts/run-updater-check.sh - name: Failure Notification uses: rtCamp/action-slack-notify@v2 env: @@ -596,7 +627,7 @@ jobs: - name: Retrieve Build Artifacts id: fetch-dist if: needs.file-check.outputs.run == 'true' - uses: Wandalen/wretry.action@v1 + uses: Wandalen/wretry.action@v3 with: action: actions/download-artifact@v4 with: | @@ -634,10 +665,9 @@ jobs: SLACK_USERNAME: 'GitHub Actions' SLACK_MESSAGE: |- ${{ github.repository }}: Failed to prepare release artifacts for upload. - CHeckout: ${{ steps.checkout.outcome }} + Checkout: ${{ steps.checkout.outcome }} Prepare environment: ${{ steps.prepare.outcome }} Fetch dist tarball: ${{ steps.fetch-dist.outcome }} - Fetch static builds: ${{ steps.fetch-static.outcome }} Consolidate artifacts: ${{ steps.consolidate.outcome }} Store: ${{ steps.store.outcome }} SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }} @@ -675,7 +705,7 @@ jobs: - name: Fetch artifacts id: fetch if: needs.file-check.outputs.run == 'true' - uses: Wandalen/wretry.action@v1 + uses: Wandalen/wretry.action@v3 with: action: actions/download-artifact@v4 with: | @@ -744,7 +774,7 @@ jobs: - name: Fetch artifacts id: fetch-artifacts if: needs.file-check.outputs.run == 'true' - uses: Wandalen/wretry.action@v1 + uses: Wandalen/wretry.action@v3 with: action: actions/download-artifact@v4 with: | @@ -798,7 +828,7 @@ jobs: steps: - name: Retrieve Artifacts id: fetch - uses: Wandalen/wretry.action@v1 + uses: Wandalen/wretry.action@v3 with: action: actions/download-artifact@v4 with: | @@ -814,10 +844,10 @@ jobs: credentials_json: ${{ secrets.GCS_STORAGE_SERVICE_KEY_JSON }} - name: Setup GCS id: gcs-setup - uses: google-github-actions/setup-gcloud@v2.0.1 + uses: google-github-actions/setup-gcloud@v2.1.0 - name: Upload Artifacts id: upload - uses: google-github-actions/upload-cloud-storage@v2.0.0 + uses: google-github-actions/upload-cloud-storage@v2.1.0 with: destination: ${{ secrets.GCP_NIGHTLY_STORAGE_BUCKET }} gzip: false @@ -867,7 +897,7 @@ jobs: token: ${{ secrets.NETDATABOT_GITHUB_TOKEN }} - name: Retrieve Artifacts id: fetch - uses: Wandalen/wretry.action@v1 + uses: Wandalen/wretry.action@v3 with: action: actions/download-artifact@v4 with: | @@ -922,11 +952,17 @@ jobs: key: ${{ secrets.NETDATABOT_PACKAGES_SSH_KEY }} name: id_ecdsa known_hosts: ${{ secrets.PACKAGES_KNOWN_HOSTS }} - - name: Sync newer releases + - name: Sync release info to packages.netdata.cloud id: sync-releases + continue-on-error: true + if: github.event_name == 'workflow_dispatch' && github.repository == 'netdata/netdata' && steps.check-latest-version.outputs.versions_needs_update == 'true' + run: | + .github/scripts/upload-new-version-tags.sh packages.netdata.cloud + - name: Sync release info to packages2.netdata.cloud + id: sync-releases2 if: github.event_name == 'workflow_dispatch' && github.repository == 'netdata/netdata' && steps.check-latest-version.outputs.versions_needs_update == 'true' run: | - .github/scripts/upload-new-version-tags.sh + .github/scripts/upload-new-version-tags.sh packages.netdata.cloud - name: Failure Notification uses: rtCamp/action-slack-notify@v2 env: @@ -947,7 +983,8 @@ jobs: Setup python environment: ${{ steps.setup-python.outcome }} Check the nearly published release against the advertised: ${{ steps.check-latest-version.outcome }} Setup ssh: ${{ steps.ssh-setup.outcome }} - Sync with the releases: ${{ steps.sync-releases.outcome }} + Sync release info to packages.netdata.cloud: ${{ steps.sync-releases.outcome }} + Sync release info to packages2.netdata.cloud: ${{ steps.sync-releases2.outcome }} SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }} if: >- ${{ @@ -985,7 +1022,7 @@ jobs: uses: actions/checkout@v4 - name: Retrieve Artifacts id: fetch - uses: Wandalen/wretry.action@v1 + uses: Wandalen/wretry.action@v3 with: action: actions/download-artifact@v4 with: | diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 01ada7422..2ddcd822b 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -16,6 +16,7 @@ jobs: runs-on: ubuntu-latest outputs: run: ${{ steps.check-run.outputs.run }} + skip-go: ${{ steps.check-go.outputs.skip-go }} steps: - name: Checkout id: checkout @@ -23,9 +24,9 @@ jobs: with: fetch-depth: 0 submodules: recursive - - name: Check files - id: check-files - uses: tj-actions/changed-files@v43 + - name: Check source files + id: check-source-files + uses: tj-actions/changed-files@v44 with: since_last_remote_commit: ${{ github.event_name != 'pull_request' }} files: | @@ -35,41 +36,61 @@ jobs: **/*.hh **/*.in **/*.patch + src/aclk/aclk-schemas/ + src/ml/dlib/ + src/fluent-bit/ + src/web/server/h2o/libh2o/ + files_ignore: | + netdata.spec.in + **/*.md + - name: Check build files + id: check-build-files + uses: tj-actions/changed-files@v44 + with: + since_last_remote_commit: ${{ github.event_name != 'pull_request' }} + files: | **/*.cmake CMakeLists.txt .gitignore .github/data/distros.yml .github/workflows/build.yml - .github/scripts/build-static.sh - .github/scripts/get-static-cache-key.sh - .github/scripts/gen-matrix-build.py - .github/scripts/run-updater-check.sh packaging/cmake/ packaging/*.version packaging/*.checksums - src/aclk/aclk-schemas/ - src/ml/dlib/ - src/fluent-bit/ - src/web/server/h2o/libh2o/ files_ignore: | - netdata.spec.in **/*.md + packaging/repoconfig/ - name: List all changed files in pattern continue-on-error: true env: - ALL_CHANGED_FILES: ${{ steps.check-files.outputs.all_changed_files }} + CHANGED_SOURCE_FILES: ${{ steps.check-source-files.outputs.all_changed_files }} + CHANGED_BUILD_FILES: ${{ steps.check-build-files.outputs.all_changed_files }} run: | - for file in ${ALL_CHANGED_FILES}; do + for file in ${CHANGED_SOURCE_FILES} ${CHANGED_BUILD_FILES} ; do echo "$file was changed" done - name: Check Run id: check-run run: | - if [ "${{ steps.check-files.outputs.any_modified }}" == "true" ] || [ "${{ github.event_name }}" == "workflow_dispatch" ]; then + if [ "${{ steps.check-source-files.outputs.any_modified }}" == "true" ] || [ "${{ steps.check-build-files.outputs.any_modified }}" == "true" ] || [ "${{ github.event_name }}" == "workflow_dispatch" ]; then echo 'run=true' >> "${GITHUB_OUTPUT}" else echo 'run=false' >> "${GITHUB_OUTPUT}" fi + - name: Check Go + id: check-go + env: + OTHER_CHANGED_FILES: ${{ steps.check-source-files.outputs.other_changed_files }} + run: | + if [ '${{ github.event_name }}' == 'pull_request' ]; then + if echo "${OTHER_CHANGED_FILES}" | grep -q '.*/(.*\.go|go\.mod|go\.sum)$' || [ "${{ steps.check-build-files.outputs.any_modified }}" == "true" ]; then + echo 'skip-go=' >> "${GITHUB_OUTPUT}" + else + echo 'skip-go=--disable-go' >> "${GITHUB_OUTPUT}" + fi + else + echo 'skip-go=' >> "${GITHUB_OUTPUT}" + fi libressl-checks: name: LibreSSL @@ -94,7 +115,8 @@ jobs: ./packaging/installer/install-required-packages.sh --dont-wait --non-interactive netdata; apk del openssl openssl-dev; apk add libressl libressl-dev protobuf-dev; - ./netdata-installer.sh --disable-telemetry --dont-start-it --dont-wait --one-time-build;' + ./netdata-installer.sh --disable-telemetry --dont-start-it --dont-wait --one-time-build --disable-go;' + clang-checks: name: Clang needs: @@ -134,7 +156,7 @@ jobs: run: ./packaging/installer/install-required-packages.sh --dont-wait --non-interactive netdata - name: Build netdata if: needs.file-check.outputs.run == 'true' - run: ./netdata-installer.sh --dont-start-it --disable-telemetry --dont-wait --install-prefix /tmp/install --one-time-build + run: ./netdata-installer.sh --dont-start-it --disable-telemetry --dont-wait --install-prefix /tmp/install --one-time-build ${{ needs.file-check.outputs.skip-go }} - name: Check that repo is clean if: needs.file-check.outputs.run == 'true' run: | diff --git a/.github/workflows/cloud_regression.yml b/.github/workflows/cloud_regression.yml index 01fcdca4d..67a55ed2a 100644 --- a/.github/workflows/cloud_regression.yml +++ b/.github/workflows/cloud_regression.yml @@ -22,21 +22,24 @@ jobs: PR_COMMIT_HASH: ${{ github.event.pull_request.head.sha }} id: output-workflow-dispatch-params run: | - if [ ${{ github.event_name }} == 'pull_request_target' ]; then + if [ "${{ github.event_name }}" == 'pull_request_target' ]; then NETDATA_CUSTOM_REPO="$PR_REPO_NAME" NETDATA_CUSTOM_BRANCH="$PR_BRANCH_NAME" NETDATA_CUSTOM_PR_NUMBER="${{ github.event.number }}" NETDATA_CUSTOM_COMMIT_HASH="$PR_COMMIT_HASH" - elif [ ${{ github.event_name }} == 'push' ]; then + elif [ "${{ github.event_name }}" == 'push' ]; then NETDATA_CUSTOM_REPO="netdata/netdata" NETDATA_CUSTOM_BRANCH="master" NETDATA_CUSTOM_PR_NUMBER="" NETDATA_CUSTOM_COMMIT_HASH="${{ github.sha }}" fi - echo "netdata_repo=${NETDATA_CUSTOM_REPO}" >> $GITHUB_OUTPUT - echo "netdata_branch=${NETDATA_CUSTOM_BRANCH}" >> $GITHUB_OUTPUT - echo "netdata_pr_number=${NETDATA_CUSTOM_PR_NUMBER}" >> $GITHUB_OUTPUT - echo "netdata_commit_hash=${NETDATA_CUSTOM_COMMIT_HASH}" >> $GITHUB_OUTPUT + + { + echo "netdata_repo=${NETDATA_CUSTOM_REPO}" + echo "netdata_branch=${NETDATA_CUSTOM_BRANCH}" + echo "netdata_pr_number=${NETDATA_CUSTOM_PR_NUMBER}" + echo "netdata_commit_hash=${NETDATA_CUSTOM_COMMIT_HASH}" + } >> "$GITHUB_OUTPUT" - name: Trigger Full Cloud Regression uses: aurelien-baudet/workflow-dispatch@v2 diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index aa1528a17..0d22aee98 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -31,6 +31,7 @@ jobs: runs-on: ubuntu-latest outputs: run: ${{ steps.check-run.outputs.run }} + skip-go: ${{ steps.check-go.outputs.skip-go }} steps: - name: Checkout id: checkout @@ -39,10 +40,10 @@ jobs: with: fetch-depth: 0 submodules: recursive - - name: Check files - id: check-files + - name: Check source files + id: check-source-files if: github.event_name != 'workflow_dispatch' - uses: tj-actions/changed-files@v43 + uses: tj-actions/changed-files@v44 with: since_last_remote_commit: ${{ github.event_name != 'pull_request' }} files: | @@ -52,7 +53,20 @@ jobs: **/*.hh **/*.in **/*.patch - **/*.cmake + src/aclk/aclk-schemas/ + src/ml/dlib/ + src/fluent-bit/ + src/web/server/h2o/libh2o/ + files_ignore: | + netdata.spec.in + **/*.md + - name: Check build system files + id: check-build-files + if: github.event_name != 'workflow_dispatch' + uses: tj-actions/changed-files@v44 + with: + since_last_remote_commit: ${{ github.event_name != 'pull_request' }} + files: | .dockerignore CMakeLists.txt netdata-installer.sh @@ -66,30 +80,41 @@ jobs: packaging/runtime-check.sh packaging/*.version packaging/*.checksums - src/aclk/aclk-schemas/ - src/ml/dlib/ - src/fluent-bit/ - src/web/server/h2o/libh2o/ files_ignore: | - netdata.spec.in **/*.md + packaging/repoconfig/ - name: List all changed files in pattern continue-on-error: true if: github.event_name != 'workflow_dispatch' env: - ALL_CHANGED_FILES: ${{ steps.check-files.outputs.all_changed_files }} + CHANGED_SOURCE_FILES: ${{ steps.check-source-files.outputs.all_changed_files }} + CHANGED_BUILD_FILES: ${{ steps.check-build-files.outputs.all_changed_files }} run: | - for file in ${ALL_CHANGED_FILES}; do + for file in ${CHANGED_SOURCE_FILES} ${CHANGED_BUILD_FILES} ; do echo "$file was changed" done - name: Check Run id: check-run run: | - if [ "${{ steps.check-files.outputs.any_modified }}" == "true" ] || [ "${{ github.event_name }}" == "workflow_dispatch" ]; then + if [ "${{ steps.check-source-files.outputs.any_modified }}" == "true" ] || [ "${{ steps.check-build-files.outputs.any_modified }}" == "true" ] || [ "${{ github.event_name }}" == "workflow_dispatch" ]; then echo 'run=true' >> "${GITHUB_OUTPUT}" else echo 'run=false' >> "${GITHUB_OUTPUT}" fi + - name: Check Go + id: check-go + env: + OTHER_CHANGED_FILES: ${{ steps.check-source-files.outputs.other_changed_files }} + run: | + if [ '${{ github.event_name }}' == 'pull_request' ]; then + if echo "${OTHER_CHANGED_FILES}" | grep -q '.*/(.*\.go|go\.mod|go\.sum)$' || [ "${{ steps.check-build-files.outputs.any_modified }}" == "true" ]; then + echo 'skip-go=' >> "${GITHUB_OUTPUT}" + else + echo 'skip-go=--disable-go' >> "${GITHUB_OUTPUT}" + fi + else + echo 'skip-go=' >> "${GITHUB_OUTPUT}" + fi build-images: name: Build Docker Images @@ -143,7 +168,9 @@ jobs: tags: netdata/netdata:test load: true cache-to: type=local,dest=/tmp/build-cache,mode=max - build-args: OFFICIAL_IMAGE=${{ env.OFFICIAL_IMAGE }} + build-args: | + OFFICIAL_IMAGE=${{ env.OFFICIAL_IMAGE }} + EXTRA_INSTALL_OPTS=${{ needs.file-check.outputs.skip-go }} - name: Test Image id: test if: needs.file-check.outputs.run == 'true' && matrix.platform == 'linux/amd64' @@ -257,8 +284,8 @@ jobs: with: platforms: ${{ matrix.platform }} cache-from: type=local,src=/tmp/build-cache - build-args: OFFICIAL_IMAGE=${{ env.OFFICIAL_IMAGE }} outputs: type=image,name=netdata/netdata,push-by-digest=true,name-canonical=true,push=true + build-args: OFFICIAL_IMAGE=${{ env.OFFICIAL_IMAGE }} - name: Export Digest id: export-digest if: github.repository == 'netdata/netdata' @@ -333,7 +360,7 @@ jobs: - name: Create and Push Manifest id: manifest if: github.repository == 'netdata/netdata' - run: docker buildx imagetools create $(.github/scripts/gen-docker-imagetool-args.py /tmp/digests '' ${{ needs.gen-tags.outputs.tags }}) + run: docker buildx imagetools create $(.github/scripts/gen-docker-imagetool-args.py /tmp/digests '' "${{ needs.gen-tags.outputs.tags }}") - name: Failure Notification uses: rtCamp/action-slack-notify@v2 env: @@ -489,7 +516,7 @@ jobs: - name: Create and Push Manifest id: manifest if: github.repository == 'netdata/netdata' - run: docker buildx imagetools create $(.github/scripts/gen-docker-imagetool-args.py /tmp/digests 'quay.io' ${{ needs.gen-tags.outputs.tags }}) + run: docker buildx imagetools create $(.github/scripts/gen-docker-imagetool-args.py /tmp/digests 'quay.io' "${{ needs.gen-tags.outputs.tags }}") - name: Failure Notification uses: rtCamp/action-slack-notify@v2 env: @@ -645,7 +672,7 @@ jobs: - name: Create and Push Manifest id: manifest if: github.repository == 'netdata/netdata' - run: docker buildx imagetools create $(.github/scripts/gen-docker-imagetool-args.py /tmp/digests 'ghcr.io' ${{ needs.gen-tags.outputs.tags }}) + run: docker buildx imagetools create $(.github/scripts/gen-docker-imagetool-args.py /tmp/digests 'ghcr.io' "${{ needs.gen-tags.outputs.tags }}") - name: Failure Notification uses: rtCamp/action-slack-notify@v2 env: @@ -690,7 +717,7 @@ jobs: repo: netdata/helmchart workflow: Agent Version PR ref: refs/heads/master - inputs: '{"agent_version": "${{ inputs.version }}"}' + inputs: '{"agent_version": "v${{ inputs.version }}"}' - name: Trigger MSI build if: github.event_name == 'workflow_dispatch' && github.event.inputs.version != 'nightly' && github.repository == 'netdata/netdata' id: trigger-msi diff --git a/.github/workflows/generate-integrations.yml b/.github/workflows/generate-integrations.yml index f7d4df866..8287f9295 100644 --- a/.github/workflows/generate-integrations.yml +++ b/.github/workflows/generate-integrations.yml @@ -79,9 +79,7 @@ jobs: SLACK_MESSAGE: |- ${{ github.repository }}: Failed to create PR rebuilding integrations.js Checkout Agent: ${{ steps.checkout-agent.outcome }} - Get Go Ref: ${{ steps.get-go-ref.outcome }} - Checkout Go: ${{ steps.checkout-go.outcome }} - Prepare Dependencies: ${{ steps.prep-deps.outcome }} + Prep python env and deps: ${{ steps.prep-deps.outcome }} Generate Integrations: ${{ steps.generate.outcome }} Generate Integrations Documentation: ${{ steps.generate-integrations-documentation.outcome }} Generate src/collectors/COLLECTORS.md: ${{ steps.generate-collectors-md.outcome }} diff --git a/.github/workflows/go-tests.yml b/.github/workflows/go-tests.yml index 9e5550507..93596454b 100644 --- a/.github/workflows/go-tests.yml +++ b/.github/workflows/go-tests.yml @@ -24,7 +24,7 @@ jobs: submodules: recursive - name: Check files id: check-files - uses: tj-actions/changed-files@v43 + uses: tj-actions/changed-files@v44 with: since_last_remote_commit: ${{ github.event_name != 'pull_request' }} files: | @@ -36,6 +36,7 @@ jobs: files_ignore: | **/*.md src/go/**/metadata.yaml + packaging/repoconfig/ - name: List all changed files in pattern continue-on-error: true env: @@ -62,7 +63,9 @@ jobs: - name: Checkout uses: actions/checkout@v4 - name: Install dependencies - run: sudo apt-get update && sudo apt-get upgrade -y && sudo apt-get install -y python3-packaging + run: | + sudo apt-get update || true + sudo apt-get install -y python3-packaging - name: Get Go version and modules id: get-version run: .github/scripts/get-go-version.py diff --git a/.github/workflows/kickstart-upload.yml b/.github/workflows/kickstart-upload.yml index 77c26d7bc..46505a4a3 100644 --- a/.github/workflows/kickstart-upload.yml +++ b/.github/workflows/kickstart-upload.yml @@ -30,9 +30,13 @@ jobs: key: ${{ secrets.NETDATABOT_PACKAGES_SSH_KEY }} name: id_ecdsa known_hosts: ${{ secrets.PACKAGES_KNOWN_HOSTS }} - - name: Upload - id: upload + - name: Upload to packages.netdata.cloud + id: upload-primary + continue-on-error: true run: rsync -vp packaging/installer/kickstart.sh netdatabot@packages.netdata.cloud:/home/netdatabot/incoming/kickstart.sh + - name: Upload to packages2.netdata.cloud + id: upload-packages2 + run: rsync -vp packaging/installer/kickstart.sh netdatabot@packages2.netdata.cloud:/home/netdatabot/incoming/kickstart.sh - name: Failure Notification uses: rtCamp/action-slack-notify@v2 env: @@ -44,7 +48,8 @@ jobs: ${{ github.repository }}: Failed to upload updated kickstart script to repo server. Checkout: ${{ steps.checkout.outcome }} Import SSH Key: ${{ steps.ssh-setup.outcome }} - Upload: ${{ steps.upload.outcome }} + Upload to packages.netdata.cloud: ${{ steps.upload-packages.outcome }} + Upload to packages2.netdata.cloud: ${{ steps.upload-packages2.outcome }} SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }} if: >- ${{ diff --git a/.github/workflows/monitor-releases.yml b/.github/workflows/monitor-releases.yml index 7962c9861..e4269f3c1 100644 --- a/.github/workflows/monitor-releases.yml +++ b/.github/workflows/monitor-releases.yml @@ -1,5 +1,5 @@ --- -name: Monitor-releases +name: Monitor releases on: release: @@ -12,13 +12,13 @@ on: default: 'stable' -concurrency: # This keeps multiple instances of the job from running concurrently for the same ref and event type. +concurrency: group: monitor-{{ github.event.inputs.channel }}-releases-${{ github.ref }}-${{ github.event_name }} cancel-in-progress: true jobs: - update-stable-agents-metadata: - name: update-stable-agents-metadata + update-agents-metadata: + name: update-agents-metadata runs-on: ubuntu-latest steps: - name: Checkout @@ -26,7 +26,15 @@ jobs: uses: actions/checkout@v4 with: token: ${{ secrets.NETDATABOT_GITHUB_TOKEN }} - - name: Init python environment + - name: Overwrite defaults + id: ow-defaults + run: | + if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then + echo "RELEASE_CHANNEL=${{ github.event.inputs.channel }}" >> "${GITHUB_ENV}" + else + echo "RELEASE_CHANNEL=stable" >> "${GITHUB_ENV}" + fi + - name: Init Python environment uses: actions/setup-python@v5 id: init-python with: @@ -38,7 +46,7 @@ jobs: - name: Check for newer versions id: check-newer-releases run: | - python .github/scripts/check_latest_versions_per_channel.py "${{ github.event.inputs.channel }}" + python .github/scripts/check_latest_versions_per_channel.py "${{ env.RELEASE_CHANNEL }}" - name: SSH setup id: ssh-setup if: github.event_name == 'workflow_dispatch' && github.repository == 'netdata/netdata' && steps.check-newer-releases.outputs.versions_needs_update == 'true' @@ -47,11 +55,17 @@ jobs: key: ${{ secrets.NETDATABOT_PACKAGES_SSH_KEY }} name: id_ecdsa known_hosts: ${{ secrets.PACKAGES_KNOWN_HOSTS }} - - name: Sync newer releases + - name: Sync newer releases to packages.netdata.cloud id: sync-releases + continue-on-error: true + if: github.event_name == 'workflow_dispatch' && github.repository == 'netdata/netdata' && steps.check-newer-releases.outputs.versions_needs_update == 'true' + run: | + .github/scripts/upload-new-version-tags.sh packages.netdata.cloud + - name: Sync newer releases to packages2.netdata.cloud + id: sync-releases2 if: github.event_name == 'workflow_dispatch' && github.repository == 'netdata/netdata' && steps.check-newer-releases.outputs.versions_needs_update == 'true' run: | - .github/scripts/upload-new-version-tags.sh + .github/scripts/upload-new-version-tags.sh packages2.netdata.cloud - name: Failure Notification uses: rtCamp/action-slack-notify@v2 env: @@ -63,10 +77,12 @@ jobs: SLACK_MESSAGE: |- ${{ github.repository }}: Failed to update stable Agent's metadata. Checkout: ${{ steps.checkout.outcome }} + Overwrite inputs: ${{ steps.ow-defaults.outcome }} Init python: ${{ steps.init-python.outcome }} Setup python: ${{ steps.setup-python.outcome }} Check for newer stable releaes: ${{ steps.check-newer-releases.outcome }} Setup ssh: ${{ steps.ssh-setup.outcome }} Syncing newer release to packages.netdata.cloud : ${{ steps.sync-releases.outcome }} + Syncing newer release to packages2.netdata.cloud : ${{ steps.sync-releases2.outcome }} SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }} if: failure() diff --git a/.github/workflows/packaging.yml b/.github/workflows/packaging.yml index 7e72fb3f0..ebfba8072 100644 --- a/.github/workflows/packaging.yml +++ b/.github/workflows/packaging.yml @@ -41,7 +41,7 @@ jobs: submodules: recursive - name: Check files id: check-files - uses: tj-actions/changed-files@v43 + uses: tj-actions/changed-files@v44 with: since_last_remote_commit: ${{ github.event_name != 'pull_request' }} files: | @@ -53,7 +53,6 @@ jobs: **/*.patch **/*.cmake netdata.spec.in - contrib/debian/ CMakeLists.txt .github/data/distros.yml .github/workflows/packaging.yml @@ -69,6 +68,7 @@ jobs: src/web/server/h2o/libh2o/ files_ignore: | **/*.md + packaging/repoconfig/ - name: List all changed files in pattern continue-on-error: true env: @@ -98,7 +98,8 @@ jobs: - name: Prepare tools id: prepare run: | - sudo apt-get update && sudo apt-get install -y python3-ruamel.yaml + sudo apt-get update || true + sudo apt-get install -y python3-ruamel.yaml - name: Read build matrix id: set-matrix run: | @@ -248,7 +249,7 @@ jobs: timeout_seconds: 900 command: | docker pull --platform ${{ matrix.platform }} ${{ matrix.base_image }} - docker pull --platform ${{ matrix.platform }} netdata/package-builders:${{ matrix.distro }}${{ matrix.version }}-v1 + docker pull --platform ${{ matrix.platform }} netdata/package-builders:${{ matrix.distro }}${{ matrix.version }}-${{ matrix.builder_rev }} - name: Build Packages id: build if: needs.file-check.outputs.run == 'true' @@ -258,7 +259,7 @@ jobs: -e ENABLE_SENTRY=${{ matrix.bundle_sentry }} -e RELEASE_PIPELINE=${{ env.RELEASE_PIPELINE }} \ -e BUILD_DESTINATION=${{ matrix.distro }}${{ matrix.version }}_${{ matrix.arch }} -e UPLOAD_SENTRY=${{ env.UPLOAD_SENTRY }} \ -e SENTRY_AUTH_TOKEN=${{ secrets.SENTRY_CLI_TOKEN }} -e NETDATA_SENTRY_DSN=${{ secrets.SENTRY_DSN }} \ - --platform=${{ matrix.platform }} -v "$PWD":/netdata netdata/package-builders:${{ matrix.distro }}${{ matrix.version }}-v1 + --platform=${{ matrix.platform }} -v "$PWD":/netdata netdata/package-builders:${{ matrix.distro }}${{ matrix.version }}-${{ matrix.builder_rev }} - name: Save Packages id: artifacts if: needs.file-check.outputs.run == 'true' @@ -300,9 +301,21 @@ jobs: known_hosts: ${{ secrets.PACKAGES_KNOWN_HOSTS }} - name: Upload to packages.netdata.cloud id: package-upload + continue-on-error: true + if: github.event_name == 'workflow_dispatch' && github.repository == 'netdata/netdata' && needs.file-check.outputs.run == 'true' + run: | + .github/scripts/package-upload.sh \ + packages.netdata.cloud \ + ${{ matrix.repo_distro }} \ + ${{ matrix.arch }} \ + ${{ matrix.format }} \ + ${{ needs.version-check.outputs.repo }} + - name: Upload to packages2.netdata.cloud + id: package2-upload if: github.event_name == 'workflow_dispatch' && github.repository == 'netdata/netdata' && needs.file-check.outputs.run == 'true' run: | .github/scripts/package-upload.sh \ + packages2.netdata.cloud \ ${{ matrix.repo_distro }} \ ${{ matrix.arch }} \ ${{ matrix.format }} \ @@ -325,6 +338,7 @@ jobs: Publish to PackageCloud: ${{ steps.upload.outcome }} Import SSH Key: ${{ steps.ssh-setup.outcome }} Publish to packages.netdata.cloud: ${{ steps.package-upload.outcome }} + Publish to packages2.netdata.cloud: ${{ steps.package2-upload.outcome }} SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }} if: >- ${{ diff --git a/.github/workflows/platform-eol-check.yml b/.github/workflows/platform-eol-check.yml index ae290a973..185640f2f 100644 --- a/.github/workflows/platform-eol-check.yml +++ b/.github/workflows/platform-eol-check.yml @@ -26,7 +26,8 @@ jobs: - name: Prepare tools id: prepare run: | - sudo apt-get update && sudo apt-get install -y python3-ruamel.yaml + sudo apt-get update || true + sudo apt-get install -y python3-ruamel.yaml - name: Read build matrix id: set-matrix run: | @@ -118,7 +119,7 @@ jobs: if: steps.check.outputs.pending == 'true' && steps.existing.outputs.exists == 'false' uses: imjohnbo/issue-bot@v3 with: - assignees: Ferroin, tkatsoulas + assignees: Ferroin labels: area/packaging, needs triage title: ${{ steps.title.outputs.title }} body: | diff --git a/.github/workflows/repoconfig-packages.yml b/.github/workflows/repoconfig-packages.yml index f3d65a3e1..8b43a2358 100644 --- a/.github/workflows/repoconfig-packages.yml +++ b/.github/workflows/repoconfig-packages.yml @@ -31,7 +31,8 @@ jobs: - name: Prepare tools id: prepare run: | - sudo apt-get update && sudo apt-get install -y python3-ruamel.yaml + sudo apt-get update || true + sudo apt-get install -y python3-ruamel.yaml - name: Read build matrix id: set-matrix run: | @@ -102,14 +103,8 @@ jobs: env: PKG_CLOUD_TOKEN: ${{ secrets.PACKAGE_CLOUD_API_KEY }} run: | - printf "Packages to upload:\n%s" "$(ls artifacts/*.${{ matrix.format }})" + printf "Packages to upload:\n%s" "$(ls packaging/repoconfig/artifacts/*.${{ matrix.format }})" for pkgfile in artifacts/*.${{ matrix.format }} ; do - .github/scripts/package_cloud_wrapper.sh yank "${REPO_PREFIX}/${{ matrix.pkgclouddistro }}" \ - "$(basename "${pkgfile}")" || true - .github/scripts/package_cloud_wrapper.sh push "${REPO_PREFIX}/${{ matrix.pkgclouddistro }}" "${pkgfile}" - .github/scripts/package_cloud_wrapper.sh yank "${REPO_PREFIX}-edge/${{ matrix.pkgclouddistro }}" \ - "$(basename "${pkgfile}")" || true - .github/scripts/package_cloud_wrapper.sh push "${REPO_PREFIX}-edge/${{ matrix.pkgclouddistro }}" "${pkgfile}" .github/scripts/package_cloud_wrapper.sh yank "${REPO_PREFIX}-repoconfig/${{ matrix.pkgclouddistro }}" \ "$(basename "${pkgfile}")" || true .github/scripts/package_cloud_wrapper.sh push "${REPO_PREFIX}-repoconfig/${{ matrix.pkgclouddistro }}" "${pkgfile}" @@ -124,16 +119,32 @@ jobs: known_hosts: ${{ secrets.PACKAGES_KNOWN_HOSTS }} - name: Upload to packages.netdata.cloud id: package-upload + continue-on-error: true + if: github.event_name != 'pull_request' && github.repository == 'netdata/netdata' + run: | + # shellcheck disable=SC2043 + for arch in ${{ matrix.arches }}; do + .github/scripts/package-upload.sh \ + packages.netdata.cloud \ + "${{ matrix.pkgclouddistro }}" \ + "${arch}" \ + "${{ matrix.format }}" \ + netdata/netdata-repoconfig \ + packaging/repoconfig/artifacts + done + - name: Upload to packages2.netdata.cloud + id: package2-upload if: github.event_name != 'pull_request' && github.repository == 'netdata/netdata' run: | + # shellcheck disable=SC2043 for arch in ${{ matrix.arches }}; do - for suffix in '' -edge -repoconfig ; do .github/scripts/package-upload.sh \ - ${{ matrix.pkgclouddistro }} \ - ${arch} \ - ${{ matrix.format }} \ - netdata/netdata${suffix} - done + packages2.netdata.cloud \ + "${{ matrix.pkgclouddistro }}" \ + "${arch}" \ + "${{ matrix.format }}" \ + netdata/netdata-repoconfig \ + packaging/repoconfig/artifacts done - name: Failure Notification if: ${{ failure() && github.repository == 'netdata/netdata' }} @@ -152,4 +163,5 @@ jobs: Publish to PackageCloud: ${{ steps.publish.outcome }} Import SSH Key: ${{ steps.ssh-setup.outcome }} Publish to packages.netdata.cloud: ${{ steps.package-upload.outcome }} + Publish to packages2.netdata.cloud: ${{ steps.package2-upload.outcome }} SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }} diff --git a/.github/workflows/review.yml b/.github/workflows/review.yml index c7b038135..30b09ec06 100644 --- a/.github/workflows/review.yml +++ b/.github/workflows/review.yml @@ -15,7 +15,7 @@ jobs: runs-on: ubuntu-latest outputs: actionlint: ${{ steps.actionlint.outputs.run }} - clangformat: ${{ steps.clangformat.outputs.run }} + # clangformat: ${{ steps.clangformat.outputs.run }} flake8: ${{ steps.flake8.outputs.run }} golangci-lint: ${{ steps.golangci-lint.outputs.run }} hadolint: ${{ steps.hadolint.outputs.run }} @@ -65,11 +65,11 @@ jobs: run: | if [ "${{ contains(github.event.pull_request.labels.*.name, 'run-ci/golangci-lint') }}" = "true" ]; then echo "run=true" >> "${GITHUB_OUTPUT}" - elif git diff --name-only origin/${{ github.base_ref }} HEAD -- | grep -Eq '.*\.go' ; then - echo "run=true" >> $GITHUB_OUTPUT + elif git diff --name-only origin/"${{ github.base_ref }}" HEAD -- | grep -Eq '.*\.go'; then + echo "run=true" >> "${GITHUB_OUTPUT}" echo 'Go code has changed, need to run golangci-lint.' else - echo "run=false" >> $GITHUB_OUTPUT + echo "run=false" >> "${GITHUB_OUTPUT}" fi - name: Check files for hadolint id: hadolint @@ -122,38 +122,38 @@ jobs: github_token: ${{ secrets.GITHUB_TOKEN }} reporter: github-pr-check - clang-format: - name: clang-format - needs: prep-review - if: needs.prep-review.outputs.clangformat == 'true' - runs-on: ubuntu-latest - steps: - - name: Git clone repository - uses: actions/checkout@v4 - with: - submodules: false - fetch-depth: 0 - - name: Check for label - id: label - run: | - if [ "${{ contains(github.event.pull_request.labels.*.name, 'run-ci/clang-format') }}" = "true" ]; then - echo 'check-all=true' >> "${GITHUB_OUTPUT}" - else - echo 'check-all=false' >> "${GITHUB_OUTPUT}" - fi - - name: Run clang-format - run: | - if [ "${{ steps.label.outputs.check-all }}" == 'true' ]; then - find . -regex '.*\.\(c\|cpp\|cxx\|h\|hpp\|hxx\)$' -exec clang-format -i --style=file '{}' \; - else - git diff --name-only origin/${{ github.base_ref }} HEAD | grep -E '\.cpp$|\.cxx$|\.c$|\.hpp$|\.hxx$|\.h$' | \ - xargs -n 1 -r clang-format -i --style=file - fi - git status --porcelain=v1 > /tmp/porcelain - if [ -s /tmp/porcelain ]; then - cat /tmp/porcelain - exit 1 - fi +# clang-format: +# name: clang-format +# needs: prep-review +# if: needs.prep-review.outputs.clangformat == 'true' +# runs-on: ubuntu-latest +# steps: +# - name: Git clone repository +# uses: actions/checkout@v4 +# with: +# submodules: false +# fetch-depth: 0 +# - name: Check for label +# id: label +# run: | +# if [ "${{ contains(github.event.pull_request.labels.*.name, 'run-ci/clang-format') }}" = "true" ]; then +# echo 'check-all=true' >> "${GITHUB_OUTPUT}" +# else +# echo 'check-all=false' >> "${GITHUB_OUTPUT}" +# fi +# - name: Run clang-format +# run: | +# if [ "${{ steps.label.outputs.check-all }}" == 'true' ]; then +# find . -regex '.*\.\(c\|cpp\|cxx\|h\|hpp\|hxx\)$' -exec clang-format -i --style=file '{}' \; +# else +# git diff --name-only origin/${{ github.base_ref }} HEAD | grep -E '\.cpp$|\.cxx$|\.c$|\.hpp$|\.hxx$|\.h$' | \ +# xargs -n 1 -r clang-format -i --style=file +# fi +# git status --porcelain=v1 > /tmp/porcelain +# if [ -s /tmp/porcelain ]; then +# cat /tmp/porcelain +# exit 1 +# fi flake8: name: flake8 |