diff options
Diffstat (limited to '')
| -rw-r--r-- | .github/CODEOWNERS | 22 | ||||
| -rw-r--r-- | .github/codeql/python-config.yml | 10 | ||||
| -rw-r--r-- | .github/data/distros.yml | 29 | ||||
| -rwxr-xr-x | .github/scripts/pkg-test.sh | 8 | ||||
| -rw-r--r-- | .github/workflows/add-to-project.yml | 4 | ||||
| -rw-r--r-- | .github/workflows/build.yml | 87 | ||||
| -rw-r--r-- | .github/workflows/codeql.yml | 117 | ||||
| -rw-r--r-- | .github/workflows/packaging.yml | 11 | ||||
| -rw-r--r-- | .github/workflows/review.yml | 23 | ||||
| -rw-r--r-- | .github/workflows/tests.yml | 56 |
10 files changed, 275 insertions, 92 deletions
diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index dac9f84c..c513b71d 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -10,28 +10,28 @@ aclk/ @stelfrag @underhood build/ @Ferroin contrib/debian @Ferroin -collectors/ @vlvkobal -collectors/ebpf.plugin/ @thiagoftsm @vlvkobal +collectors/ @thiagoftsm +collectors/ebpf.plugin/ @thiagoftsm collectors/charts.d.plugin/ @ilyam8 @Ferroin -collectors/freebsd.plugin/ @vlvkobal @thiagoftsm -collectors/macos.plugin/ @vlvkobal @thiagoftsm +collectors/freebsd.plugin/ @thiagoftsm +collectors/macos.plugin/ @thiagoftsm collectors/python.d.plugin/ @ilyam8 -collectors/cups.plugin/ @simonnagl @vlvkobal @thiagoftsm -exporting/ @vlvkobal @thiagoftsm +collectors/cups.plugin/ @simonnagl @thiagoftsm +exporting/ @thiagoftsm daemon/ @thiagoftsm @vkalintiris database/ @thiagoftsm @vkalintiris docs/ @DShreve2 -health/ @thiagoftsm @vlvkobal @vkalintiris -health/health.d/ @thiagoftsm @vlvkobal -health/notifications/ @Ferroin @thiagoftsm +health/ @thiagoftsm @vkalintiris @MrZammler +health/health.d/ @thiagoftsm @MrZammler +health/notifications/ @Ferroin @thiagoftsm @MrZammler ml/ @andrewm4894 @vkalintiris libnetdata/ @thiagoftsm @vkalintiris packaging/ @Ferroin registry/ @jacekkolasa -streaming/ @thiagoftsm @vlvkobal +streaming/ @thiagoftsm system/ @Ferroin tests/ @Ferroin @vkalintiris -web/ @thiagoftsm @vlvkobal @vkalintiris +web/ @thiagoftsm @vkalintiris web/gui/ @jacekkolasa # Ownership by filetype (overwrites ownership by directory) diff --git a/.github/codeql/python-config.yml b/.github/codeql/python-config.yml new file mode 100644 index 00000000..c82727ce --- /dev/null +++ b/.github/codeql/python-config.yml @@ -0,0 +1,10 @@ +paths-ignore: + - .github + - build_external/ + - ml/dlib + - ml/json + - tests/api + - web/gui + - collectors/python.d.plugin/python_modules/pyyaml* + - collectors/python.d.plugin/python_modules/third_party + - collectors/python.d.plugin/python_modules/urllib3 diff --git a/.github/data/distros.yml b/.github/data/distros.yml index 8ef32785..0f571864 100644 --- a/.github/data/distros.yml +++ b/.github/data/distros.yml @@ -27,13 +27,13 @@ include: test: ebpf-core: true - <<: *alpine + version: "3.17" + - <<: *alpine version: "3.16" - <<: *alpine version: "3.15" - <<: *alpine version: "3.14" - - <<: *alpine - version: "3.13" - distro: archlinux version: latest @@ -99,7 +99,7 @@ include: - &fedora distro: fedora - version: "36" + version: "37" jsonc_removal: | dnf remove -y json-c-devel packages: &fedora_packages @@ -107,6 +107,16 @@ include: repo_distro: fedora/36 arches: - x86_64 + - aarch64 + test: + ebpf-core: true + - <<: *fedora + version: "36" + packages: + <<: *fedora_packages + repo_distro: fedora/36 + arches: + - x86_64 - armhfp - aarch64 test: @@ -116,6 +126,10 @@ include: packages: <<: *fedora_packages repo_distro: fedora/35 + arches: + - x86_64 + - armhfp + - aarch64 test: ebpf-core: true @@ -162,14 +176,14 @@ include: - &ubuntu distro: ubuntu - version: "22.04" + version: "22.10" env_prep: | rm -f /etc/apt/apt.conf.d/docker && apt-get update jsonc_removal: | apt-get remove -y libjson-c-dev packages: &ubuntu_packages type: deb - repo_distro: ubuntu/jammy + repo_distro: ubuntu/kinetic arches: - amd64 - armhf @@ -177,6 +191,11 @@ include: test: ebpf-core: true - <<: *ubuntu + version: "22.04" + packages: + <<: *ubuntu_packages + repo_distro: ubuntu/jammy + - <<: *ubuntu version: "20.04" packages: <<: *ubuntu_packages diff --git a/.github/scripts/pkg-test.sh b/.github/scripts/pkg-test.sh index 179c0c41..e3bc3e7d 100755 --- a/.github/scripts/pkg-test.sh +++ b/.github/scripts/pkg-test.sh @@ -4,13 +4,19 @@ install_debian_like() { # This is needed to ensure package installs don't prompt for any user input. export DEBIAN_FRONTEND=noninteractive + if apt-cache show netcat 2>&1 | grep -q "No packages found"; then + netcat="netcat-traditional" + else + netcat="netcat" + fi + apt-get update # Install Netdata apt-get install -y /netdata/artifacts/netdata_"${VERSION}"*_*.deb || exit 1 # Install testing tools - apt-get install -y --no-install-recommends curl netcat jq || exit 1 + apt-get install -y --no-install-recommends curl "${netcat}" jq || exit 1 } install_fedora_like() { diff --git a/.github/workflows/add-to-project.yml b/.github/workflows/add-to-project.yml index ae58cfce..a80d8b41 100644 --- a/.github/workflows/add-to-project.yml +++ b/.github/workflows/add-to-project.yml @@ -13,13 +13,13 @@ jobs: runs-on: ubuntu-latest steps: - name: Add issues to Agent project board - uses: actions/add-to-project@v0.3.0 + uses: actions/add-to-project@v0.4.0 with: project-url: https://github.com/orgs/netdata/projects/32 github-token: ${{ secrets.NETDATABOT_ORG_GITHUB_TOKEN }} - name: Add issues to Product Bug project board - uses: actions/add-to-project@v0.3.0 + uses: actions/add-to-project@v0.4.0 with: project-url: https://github.com/orgs/netdata/projects/45 github-token: ${{ secrets.NETDATABOT_ORG_GITHUB_TOKEN }} diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 26849312..53f1590f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -670,16 +670,18 @@ jobs: with: name: final-artifacts path: final-artifacts - - name: Setup Gcloud - id: gcloud - uses: google-github-actions/setup-gcloud@v0.6.0 + - name: Authenticate to GCS + id: gcs-auth + uses: google-github-actions/auth@v1 with: project_id: ${{ secrets.GCP_NIGHTLY_STORAGE_PROJECT }} - service_account_key: ${{ secrets.GCP_STORAGE_SERVICE_ACCOUNT_KEY }} - export_default_credentials: true + credentials_json: ${{ secrets.GCS_STORAGE_SERVICE_KEY_JSON }} + - name: Setup GCS + id: gcs-setup + uses: google-github-actions/setup-gcloud@v1.0.1 - name: Upload Artifacts id: upload - uses: google-github-actions/upload-cloud-storage@v0.10.2 + uses: google-github-actions/upload-cloud-storage@v1.0.0 with: destination: ${{ secrets.GCP_NIGHTLY_STORAGE_BUCKET }} gzip: false @@ -696,7 +698,8 @@ jobs: SLACK_MESSAGE: |- ${{ github.repository }}: Failed to upload nightly release artifacts. Fetch artifacts: ${{ steps.fetch.outcome }} - Setup GCloud: ${{ steps.gcloud.outcome }} + Authenticatie GCS: ${{ steps.gcs-auth.outcome }} + Setup GCS: ${{ steps.gcs-setup.outcome }} Upload artifacts: ${{ steps.upload.outcome }} SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }} if: >- @@ -706,6 +709,76 @@ jobs: && github.event_name != 'pull_request' }} + create-nightly: # Create a nightly build release in netdata/netdata-nightlies + name: Create Nightly Release + runs-on: ubuntu-latest + if: github.event_name == 'workflow_dispatch' && github.event.inputs.type == 'nightly' && github.repository == 'netdata/netdata' + needs: + - updater-check + - source-build + - artifact-verification-dist + - artifact-verification-static + steps: + - name: Checkout Main Repo + id: checkout-main + uses: actions/checkout@v3 + with: + path: main + - name: Checkout Nightly Repo + id: checkout-nightly + uses: actions/checkout@v3 + with: + repository: netdata/netdata-nightlies + path: nightlies + token: ${{ secrets.NETDATABOT_GITHUB_TOKEN }} + - name: Retrieve Artifacts + id: fetch + uses: actions/download-artifact@v3 + with: + name: final-artifacts + path: final-artifacts + - name: Prepare version info + id: version + run: | + # shellcheck disable=SC2129 + echo "version=$(cat main/packaging/version)" >> "${GITHUB_OUTPUT}" + echo "commit=$(cd nightlies && git rev-parse HEAD)" >> "${GITHUB_OUTPUT}" + echo "date=$(date +%F)" >> "${GITHUB_OUTPUT}" + - name: Create Release + id: create-release + uses: ncipollo/release-action@v1 + with: + allowUpdates: false + artifactErrorsFailBuild: true + artifacts: 'final-artifacts/sha256sums.txt,final-artifacts/netdata-*.tar.gz,final-artifacts/netdata-*.gz.run' + owner: netdata + repo: netdata-nightlies + body: Netdata nightly build for ${{ steps.version.outputs.date }}. + commit: ${{ steps.version.outputs.commit }} + tag: ${{ steps.version.outputs.version }} + token: ${{ secrets.NETDATABOT_GITHUB_TOKEN }} + - name: Failure Notification + uses: rtCamp/action-slack-notify@v2 + env: + SLACK_COLOR: 'danger' + SLACK_FOOTER: '' + SLACK_ICON_EMOJI: ':github-actions:' + SLACK_TITLE: 'Failed to draft release:' + SLACK_USERNAME: 'GitHub Actions' + SLACK_MESSAGE: |- + ${{ github.repository }}: Failed to create nightly release or attach artifacts. + Checkout netdata/netdata: ${{ steps.checkout-main.outcome }} + Checkout netdata/netdata-nightlies: ${{ steps.checkout-nightly.outcome }} + Fetch artifacts: ${{ steps.fetch.outcome }} + Prepare version info: ${{ steps.version.outcome }} + Create release: ${{ steps.create-release.outcome }} + SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }} + if: >- + ${{ + failure() + && github.event_name == 'workflow_dispatch' + }} + normalize-tag: # Fix the release tag if needed name: Normalize Release Tag runs-on: ubuntu-latest diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml new file mode 100644 index 00000000..021376a2 --- /dev/null +++ b/.github/workflows/codeql.yml @@ -0,0 +1,117 @@ +--- +# Run CodeQL to analyze C/C++ and Python code. +name: CodeQL +on: + pull_request: + types: [opened, reopened, labeled, synchronize] + branches: [master] + push: + branches: [master] + schedule: + - cron: "27 2 * * 1" +env: + DISABLE_TELEMETRY: 1 +concurrency: + group: codeql-${{ github.ref }} + cancel-in-progress: true +jobs: + prepare: + name: Prepare Jobs + runs-on: ubuntu-latest + outputs: + cpp: ${{ steps.cpp.outputs.run }} + python: ${{ steps.python.outputs.run }} + steps: + - name: Clone repository + uses: actions/checkout@v3 + with: + submodules: recursive + fetch-depth: 0 + - name: Check if we should always run + id: always + run: | + if [ "${{ github.event_name }}" = "pull_request" ]; then + if [ "${{ contains(github.event.pull_request.labels.*.name, 'run-ci/codeql') }}" = "true" ]; then + echo '::set-output name=run::true' + echo '::notice::Found ci/codeql label, unconditionally running all CodeQL checks.' + else + echo '::set-output name=run::false' + fi + else + echo '::set-output name=run::true' + fi + - name: Check for C/C++ changes + id: cpp + run: | + if [ "${{ steps.always.outputs.run }}" = "false" ]; then + if git diff --name-only origin/${{ github.base_ref }} HEAD | grep -Eq '.*\.[ch](xx|\+\+)?' ; then + echo '::set-output name=run::true' + echo '::notice::C/C++ code has changed, need to run CodeQL.' + else + echo '::set-output name=run::false' + fi + else + echo '::set-output name=run::true' + fi + - name: Check for python changes + id: python + run: | + if [ "${{ steps.always.outputs.run }}" = "false" ]; then + if git diff --name-only origin/${{ github.base_ref }} HEAD | grep -Eq 'collectors/python.d.plugin/.*\.py' ; then + echo '::set-output name=run::true' + echo '::notice::Python code has changed, need to run CodeQL.' + else + echo '::set-output name=run::false' + fi + else + echo '::set-output name=run::true' + fi + + analyze-cpp: + name: Analyze C/C++ + runs-on: ubuntu-latest + needs: prepare + if: needs.prepare.outputs.cpp == 'true' + permissions: + security-events: write + steps: + - name: Git clone repository + uses: actions/checkout@v3 + with: + submodules: recursive + fetch-depth: 0 + - name: Initialize CodeQL + uses: github/codeql-action/init@v2 + with: + languages: cpp + - name: Prepare environment + run: ./packaging/installer/install-required-packages.sh --dont-wait --non-interactive netdata + - name: Build netdata + run: ./netdata-installer.sh --dont-start-it --disable-telemetry --dont-wait --install /tmp/install --one-time-build + - name: Run CodeQL + uses: github/codeql-action/analyze@v2 + with: + category: "/language:cpp" + + analyze-python: + name: Analyze Python + runs-on: ubuntu-latest + needs: prepare + if: needs.prepare.outputs.python == 'true' + permissions: + security-events: write + steps: + - name: Git clone repository + uses: actions/checkout@v3 + with: + submodules: recursive + fetch-depth: 0 + - name: Initialize CodeQL + uses: github/codeql-action/init@v2 + with: + config-file: ./.github/codeql/python-config.yml + languages: python + - name: Run CodeQL + uses: github/codeql-action/analyze@v2 + with: + category: "/language:python" diff --git a/.github/workflows/packaging.yml b/.github/workflows/packaging.yml index ec4e42c0..ddd8356e 100644 --- a/.github/workflows/packaging.yml +++ b/.github/workflows/packaging.yml @@ -3,9 +3,13 @@ name: Packages on: pull_request: + types: + - opened + - reopened + - labeled + - synchronize branches: - master - - develop push: branches: - master @@ -45,9 +49,8 @@ jobs: from ruamel.yaml import YAML import json import re - FULL_CI_REGEX = '/actions run full ci' + import os ALWAYS_RUN_ARCHES = ["amd64", "x86_64"] - PR_BODY = """${{ github.event.pull_request.body }}""" yaml = YAML(typ='safe') entries = list() run_limited = False @@ -55,7 +58,7 @@ jobs: with open('.github/data/distros.yml') as f: data = yaml.load(f) - if "${{ github.event_name }}" == "pull_request" and re.search(FULL_CI_REGEX, PR_BODY, re.I) is None: + if "${{ github.event_name }}" == "pull_request" and "${{ !contains(github.event.pull_request.labels.*.name, 'run-ci/packaging') }}": run_limited = True for i, v in enumerate(data['include']): diff --git a/.github/workflows/review.yml b/.github/workflows/review.yml index f631b929..5679b246 100644 --- a/.github/workflows/review.yml +++ b/.github/workflows/review.yml @@ -2,7 +2,8 @@ # Runs various ReviewDog based checks against PR with suggested changes to improve quality name: Review on: - pull_request: null + pull_request: + types: [opened, reopened, labeled, synchronize] env: DISABLE_TELEMETRY: 1 concurrency: @@ -27,7 +28,9 @@ jobs: - name: Check files for actionlint id: actionlint run: | - if git diff --name-only origin/${{ github.base_ref }} HEAD | grep -Eq '\.github/workflows/.*' ; then + if [ "${{ contains(github.event.pull_request.labels.*.name, 'run-ci/actionlint') }}" = "true" ]; then + echo '::set-output name=run::true' + elif git diff --name-only origin/${{ github.base_ref }} HEAD | grep -Eq '\.github/workflows/.*' ; then echo '::set-output name=run::true' echo 'GitHub Actions workflows have changed, need to run actionlint.' else @@ -36,7 +39,9 @@ jobs: - name: Check files for eslint id: eslint run: | - if git diff --name-only origin/${{ github.base_ref }} HEAD | grep -v "web/gui/dashboard" | grep -Eq '.*\.js|node\.d\.plugin\.in' ; then + if [ "${{ contains(github.event.pull_request.labels.*.name, 'run-ci/eslint') }}" = "true" ]; then + echo '::set-output name=run::true' + elif git diff --name-only origin/${{ github.base_ref }} HEAD | grep -v "web/gui/dashboard" | grep -Eq '.*\.js|node\.d\.plugin\.in' ; then echo '::set-output name=run::true' echo 'JS files have changed, need to run ESLint.' else @@ -45,7 +50,9 @@ jobs: - name: Check files for hadolint id: hadolint run: | - if git diff --name-only origin/${{ github.base_ref }} HEAD | grep -Eq '.*Dockerfile.*' ; then + if [ "${{ contains(github.event.pull_request.labels.*.name, 'run-ci/hadolint') }}" = "true" ]; then + echo '::set-output name=run::true' + elif git diff --name-only origin/${{ github.base_ref }} HEAD | grep -Eq '.*Dockerfile.*' ; then echo '::set-output name=run::true' echo 'Dockerfiles have changed, need to run Hadolint.' else @@ -54,7 +61,9 @@ jobs: - name: Check files for shellcheck id: shellcheck run: | - if git diff --name-only origin/${{ github.base_ref }} HEAD | grep -Eq '.*\.sh.*' ; then + if [ "${{ contains(github.event.pull_request.labels.*.name, 'run-ci/shellcheck') }}" = "true" ]; then + echo '::set-output name=run::true' + elif git diff --name-only origin/${{ github.base_ref }} HEAD | grep -Eq '.*\.sh.*' ; then echo '::set-output name=run::true' echo 'Shell scripts have changed, need to run shellcheck.' else @@ -63,7 +72,9 @@ jobs: - name: Check files for yamllint id: yamllint run: | - if git diff --name-only origin/${{ github.base_ref }} HEAD | grep -Eq '.*\.ya?ml|python\.d/.*\.conf' ; then + if [ "${{ contains(github.event.pull_request.labels.*.name, 'run-ci/yamllint') }}" = "true" ]; then + echo '::set-output name=run::true' + elif git diff --name-only origin/${{ github.base_ref }} HEAD | grep -Eq '.*\.ya?ml|python\.d/.*\.conf' ; then echo '::set-output name=run::true' echo 'YAML files have changed, need to run yamllint.' else diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 6272a6d6..d4838685 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -39,59 +39,3 @@ jobs: CFLAGS: "-O1 -DNETDATA_INTERNAL_CHECKS=1 -DNETDATA_VERIFY_LOCKS=1" run: | ./tests/run-unit-tests.sh - - unit-tests-cmocka: - name: Unit Tests (cmocka) - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v3 - with: - submodules: recursive - - name: Prepare environment - run: | - ./packaging/installer/install-required-packages.sh --dont-wait --non-interactive netdata-all - sudo apt-get install -y libjson-c-dev libipmimonitoring-dev libcups2-dev libsnappy-dev \ - libprotobuf-dev libprotoc-dev libssl-dev protobuf-compiler \ - libnetfilter-acct-dev libmongoc-dev libcmocka-dev libzstd-dev - - name: Configure - run: | - autoreconf -ivf - ./configure --disable-ml --disable-dependency-tracking - # XXX: Work-around for bug with libbson-1.0 in Ubuntu 18.04 - # See: https://bugs.launchpad.net/ubuntu/+source/libmongoc/+bug/1790771 - # https://jira.mongodb.org/browse/CDRIVER-2818 - - name: Fix libbson - run: | - pushd /usr/lib || exit 1 - sudo ln -s /usr/include . - popd || exit 1 - - name: Build - run: | - mkdir build-tmp - cd build-tmp - cmake \ - -D UNIT_TESTING=1 \ - -D BUILD_TESTING=1 \ - -D CMAKE_BUILD_TYPE="Debug" \ - -D BSON_LIBRARY=/usr/lib/x86_64-linux-gnu/libbson-1.0.so \ - -D MONGOC_LIBRARY=/usr/lib/x86_64-linux-gnu/libmongoc-1.0.so \ - .. - make - - name: Run ctest - run: | - cd build-tmp - ctest - - name: Prepare Artifacts - if: always() - run: | - mkdir logs - pushd build-tmp || exit 1 - find . -type f -name '*.log' -exec cp {} ../logs/ \; - popd || exit 1 - - name: Upload Artifacts - uses: actions/upload-artifact@v3 - if: always() - with: - name: logs - path: logs |