From c21c3b0befeb46a51b6bf3758ffa30813bea0ff0 Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Sat, 9 Mar 2024 14:19:22 +0100 Subject: Adding upstream version 1.44.3. Signed-off-by: Daniel Baumann --- .github/CODEOWNERS | 9 +- .github/data/distros.yml | 7 +- .github/labeler.yml | 3 + .github/scripts/check_latest_versions.py | 33 +++++ .../scripts/check_latest_versions_per_channel.py | 9 ++ .github/scripts/modules/github_actions.py | 27 ++++ .github/scripts/modules/requirements.txt | 1 + .github/scripts/modules/version_manipulation.py | 141 +++++++++++++++++++++ .github/scripts/upload-new-version-tags.sh | 18 +++ .github/workflows/build.yml | 39 +++++- .github/workflows/checks.yml | 2 +- .github/workflows/docker.yml | 2 +- .github/workflows/monitor-releases.yml | 72 +++++++++++ .github/workflows/packaging.yml | 2 +- .github/workflows/review.yml | 1 + 15 files changed, 353 insertions(+), 13 deletions(-) create mode 100755 .github/scripts/check_latest_versions.py create mode 100644 .github/scripts/check_latest_versions_per_channel.py create mode 100644 .github/scripts/modules/github_actions.py create mode 100644 .github/scripts/modules/requirements.txt create mode 100644 .github/scripts/modules/version_manipulation.py create mode 100755 .github/scripts/upload-new-version-tags.sh create mode 100644 .github/workflows/monitor-releases.yml (limited to '.github') diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index bc4fd0b32..7f368ceb7 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -19,11 +19,11 @@ collectors/cups.plugin/ @thiagoftsm exporting/ @thiagoftsm daemon/ @thiagoftsm @vkalintiris database/ @thiagoftsm @vkalintiris -docs/ @tkatsoulas @andrewm4894 @Ancairon +docs/ @tkatsoulas @Ancairon health/ @thiagoftsm @vkalintiris @MrZammler health/health.d/ @thiagoftsm @MrZammler health/notifications/ @Ferroin @thiagoftsm @MrZammler -ml/ @andrewm4894 @vkalintiris +ml/ @vkalintiris libnetdata/ @thiagoftsm @vkalintiris packaging/ @Ferroin @tkatsoulas registry/ @novykh @@ -32,11 +32,12 @@ system/ @Ferroin @tkatsoulas tests/ @Ferroin @vkalintiris @tkatsoulas web/ @thiagoftsm @vkalintiris web/gui/ @novykh +logsmanagement/ @Dim-P @thiagoftsm # Ownership by filetype (overwrites ownership by directory) *.am @Ferroin @tkatsoulas -*.md @tkatsoulas @andrewm4894 @Ancairon -*.mdx @tkatsoulas @andrewm4894 @Ancairon +*.md @tkatsoulas @Ancairon +*.mdx @tkatsoulas @Ancairon Dockerfile* @Ferroin @tkatsoulas # Ownership of specific files diff --git a/.github/data/distros.yml b/.github/data/distros.yml index bd71fd15f..9175a5c73 100644 --- a/.github/data/distros.yml +++ b/.github/data/distros.yml @@ -44,11 +44,6 @@ include: support_type: Intermediate notes: '' eol_check: true - - <<: *alpine - version: "3.15" - support_type: Intermediate - notes: '' - eol_check: true - distro: archlinux version: latest @@ -251,6 +246,7 @@ include: alt_links: - el/9Server - el/9Client + - el/9RedHatVirtualizationHost arches: - x86_64 - aarch64 @@ -264,6 +260,7 @@ include: alt_links: - el/8Server - el/8Client + - el/8RedHatVirtualizationHost - &ubuntu distro: ubuntu diff --git a/.github/labeler.yml b/.github/labeler.yml index 1dd4d472a..0ea825ef4 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -153,3 +153,6 @@ area/tests: area/web: - web/** + +area/logs-management: + - logsmanagement/** diff --git a/.github/scripts/check_latest_versions.py b/.github/scripts/check_latest_versions.py new file mode 100755 index 000000000..67b11f8d5 --- /dev/null +++ b/.github/scripts/check_latest_versions.py @@ -0,0 +1,33 @@ +import sys +import os +import modules.version_manipulation as ndvm +import modules.github_actions as cigh + + +def main(command_line_args): + """ + Inputs: Single version or multiple versions + Outputs: + Create files with the versions that needed update under temp_dir/staging-new-releases + Setting the GitHub outputs, 'versions_needs_update' to 'true' + """ + versions = [str(arg) for arg in command_line_args] + # Create a temp output folder for the release that need update + staging = os.path.join(os.environ.get('TMPDIR', '/tmp'), 'staging-new-releases') + os.makedirs(staging, exist_ok=True) + for version in versions: + temp_value = ndvm.compare_version_with_remote(version) + if temp_value: + path, filename = ndvm.get_release_path_and_filename(version) + release_path = os.path.join(staging, path) + os.makedirs(release_path, exist_ok=True) + file_release_path = os.path.join(release_path, filename) + with open(file_release_path, "w") as file: + print("Creating local copy of the release version update at: ", file_release_path) + file.write(version) + if cigh.run_as_github_action(): + cigh.update_github_output("versions_needs_update", "true") + + +if __name__ == "__main__": + main(sys.argv[1:]) diff --git a/.github/scripts/check_latest_versions_per_channel.py b/.github/scripts/check_latest_versions_per_channel.py new file mode 100644 index 000000000..885e5a98c --- /dev/null +++ b/.github/scripts/check_latest_versions_per_channel.py @@ -0,0 +1,9 @@ +import check_latest_versions +import modules.version_manipulation as ndvm +import sys + +if __name__ == "__main__": + channel = sys.argv[1] + sorted_agents_by_major = ndvm.sort_and_grouby_major_agents_of_channel(channel) + latest_per_major = [values[0] for values in sorted_agents_by_major.values()] + check_latest_versions.main(latest_per_major) diff --git a/.github/scripts/modules/github_actions.py b/.github/scripts/modules/github_actions.py new file mode 100644 index 000000000..1d653a77b --- /dev/null +++ b/.github/scripts/modules/github_actions.py @@ -0,0 +1,27 @@ +import os + + +def update_github_env(key, value): + try: + env_file = os.getenv('GITHUB_ENV') + print(env_file) + with open(env_file, "a") as file: + file.write(f"{key}={value}") + print(f"Updated GITHUB_ENV with {key}={value}") + except Exception as e: + print(f"Error updating GITHUB_ENV. Error: {e}") + + +def update_github_output(key, value): + try: + env_file = os.getenv('GITHUB_OUTPUT') + print(env_file) + with open(env_file, "a") as file: + file.write(f"{key}={value}") + print(f"Updated GITHUB_OUTPUT with {key}={value}") + except Exception as e: + print(f"Error updating GITHUB_OUTPUT. Error: {e}") + + +def run_as_github_action(): + return os.environ.get('GITHUB_ACTIONS') == 'true' diff --git a/.github/scripts/modules/requirements.txt b/.github/scripts/modules/requirements.txt new file mode 100644 index 000000000..fbec796fa --- /dev/null +++ b/.github/scripts/modules/requirements.txt @@ -0,0 +1 @@ +PyGithub==2.1.1 diff --git a/.github/scripts/modules/version_manipulation.py b/.github/scripts/modules/version_manipulation.py new file mode 100644 index 000000000..cc346fb54 --- /dev/null +++ b/.github/scripts/modules/version_manipulation.py @@ -0,0 +1,141 @@ +import os +import re +import requests +from itertools import groupby +from github import Github +from github.GithubException import GithubException + +repos_URL = { + "stable": "netdata/netdata", + "nightly": "netdata/netdata-nightlies" +} + +GH_TOKEN = os.getenv("GH_TOKEN") +if GH_TOKEN is None or GH_TOKEN != "": + print("Token is not defined or empty, continuing with limitation on requests per sec towards Github API") + + +def identify_channel(_version): + nightly_pattern = r'v(\d+)\.(\d+)\.(\d+)-(\d+)-nightly' + stable_pattern = r'v(\d+)\.(\d+)\.(\d+)' + if re.match(nightly_pattern, _version): + _channel = "nightly" + _pattern = nightly_pattern + elif re.match(stable_pattern, _version): + _channel = "stable" + _pattern = stable_pattern + else: + print("Invalid version format.") + return None + return _channel, _pattern + + +def padded_version(item): + key_value = '10000' + for value in item[1:]: + key_value += f'{value:05}' + return int(key_value) + + +def extract_version(title): + if identify_channel(title): + _, _pattern = identify_channel(title) + try: + match = re.match(_pattern, title) + if match: + return tuple(map(int, match.groups())) + except Exception as e: + print(f"Unexpected error: {e}") + return None + + +def get_release_path_and_filename(_version): + nightly_pattern = r'v(\d+)\.(\d+)\.(\d+)-(\d+)-nightly' + stable_pattern = r'v(\d+)\.(\d+)\.(\d+)' + if match := re.match(nightly_pattern, _version): + msb = match.group(1) + _path = "nightly" + _filename = f"v{msb}" + elif match := re.match(stable_pattern, _version): + msb = match.group(1) + _path = "stable" + _filename = f"v{msb}" + else: + print("Invalid version format.") + exit(1) + return (_path, _filename) + + +def compare_version_with_remote(version): + """ + If the version = fun (version) you need to update the version in the + remote. If the version remote doesn't exist, returns the version + :param channel: any version of the agent + :return: the greater from version and version remote. + """ + + prefix = "https://packages.netdata.cloud/releases" + path, filename = get_release_path_and_filename(version) + + remote_url = f"{prefix}/{path}/{filename}" + response = requests.get(remote_url) + + if response.status_code == 200: + version_remote = response.text.rstrip() + + version_components = extract_version(version) + remote_version_components = extract_version(version_remote) + + absolute_version = padded_version(version_components) + absolute_remote_version = padded_version(remote_version_components) + + if absolute_version > absolute_remote_version: + print(f"Version in the remote: {version_remote}, is older than the current: {version}, I need to update") + return (version) + else: + print(f"Version in the remote: {version_remote}, is newer than the current: {version}, no action needed") + return (None) + else: + # Remote version not found + print(f"Version in the remote not found, updating the predefined latest path with the version: {version}") + return (version) + + +def sort_and_grouby_major_agents_of_channel(channel): + """ + Fetches the GH API and read either netdata/netdata or netdata/netdata-nightlies repo. It fetches all of their + releases implements a grouping by their major release number. + Every k,v in this dictionary is in the form; "vX": [descending ordered list of Agents in this major release]. + :param channel: "nightly" or "stable" + :return: None or dict() with the Agents grouped by major version # (vX) + """ + try: + G = Github(GH_TOKEN) + repo = G.get_repo(repos_URL[channel]) + releases = repo.get_releases() + except GithubException as e: + print(f"GitHub API request failed: {e}") + return None + + except Exception as e: + print(f"An unexpected error occurred: {e}") + return None + + extracted_titles = [extract_version(item.title) for item in releases if + extract_version(item.title) is not None] + # Necessary sorting for implement the group by + extracted_titles.sort(key=lambda x: x[0]) + # Group titles by major version + grouped_by_major = {major: list(group) for major, group in groupby(extracted_titles, key=lambda x: x[0])} + sorted_grouped_by_major = {} + for key, values in grouped_by_major.items(): + sorted_values = sorted(values, key=padded_version, reverse=True) + sorted_grouped_by_major[key] = sorted_values + # Transform them in the correct form + if channel == "stable": + result_dict = {f"v{key}": [f"v{a}.{b}.{c}" for a, b, c in values] for key, values in + sorted_grouped_by_major.items()} + else: + result_dict = {f"v{key}": [f"v{a}.{b}.{c}-{d}-nightly" for a, b, c, d in values] for key, values in + sorted_grouped_by_major.items()} + return result_dict diff --git a/.github/scripts/upload-new-version-tags.sh b/.github/scripts/upload-new-version-tags.sh new file mode 100755 index 000000000..a9b0cd303 --- /dev/null +++ b/.github/scripts/upload-new-version-tags.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +set -e + +host="packages.netdata.cloud" +user="netdatabot" + +prefix="/var/www/html/releases" +staging="${TMPDIR:-/tmp}/staging-new-releases" + +mkdir -p "${staging}" + +for source_dir in "${staging}"/*; do + if [ -d "${source_dir}" ]; then + base_name=$(basename "${source_dir}") + scp -r "${source_dir}"/* "${user}@${host}:${prefix}/${base_name}" + fi +done diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 2aabbcf2d..4a6debc46 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -34,7 +34,7 @@ jobs: submodules: recursive - name: Check files id: check-files - uses: tj-actions/changed-files@v39 + uses: tj-actions/changed-files@v40 with: since_last_remote_commit: ${{ github.event_name != 'pull_request' }} files: | @@ -865,6 +865,37 @@ jobs: makeLatest: true tag: ${{ steps.version.outputs.version }} token: ${{ secrets.NETDATABOT_GITHUB_TOKEN }} + - name: Checkout netdata main Repo # Checkout back to netdata/netdata repo to the update latest packaged versions + id: checkout-netdata + uses: actions/checkout@v4 + with: + token: ${{ secrets.NETDATABOT_GITHUB_TOKEN }} + - name: Init python environment for publish release metadata + uses: actions/setup-python@v4 + id: init-python + with: + python-version: "3.12" + - name: Setup python environment + id: setup-python + run: | + pip install -r .github/scripts/modules/requirements.txt + - name: Check if the version is latest and published + id: check-latest-version + run: | + python .github/scripts/check_latest_versions.py ${{ steps.version.outputs.version }} + - name: SSH setup + id: ssh-setup + if: github.event_name == 'workflow_dispatch' && github.repository == 'netdata/netdata' && steps.check-latest-version.outputs.versions_needs_update == 'true' + uses: shimataro/ssh-key-action@v2 + with: + key: ${{ secrets.NETDATABOT_PACKAGES_SSH_KEY }} + name: id_ecdsa + known_hosts: ${{ secrets.PACKAGES_KNOWN_HOSTS }} + - name: Sync newer releases + id: sync-releases + if: github.event_name == 'workflow_dispatch' && github.repository == 'netdata/netdata' && steps.check-latest-version.outputs.versions_needs_update == 'true' + run: | + .github/scripts/upload-new-version-tags.sh - name: Failure Notification uses: rtCamp/action-slack-notify@v2 env: @@ -880,6 +911,12 @@ jobs: Fetch artifacts: ${{ steps.fetch.outcome }} Prepare version info: ${{ steps.version.outcome }} Create release: ${{ steps.create-release.outcome }} + Checkout back netdata/netdata: ${{ steps.checkout-netdata.outcome }} + Init python environment: ${{ steps.init-python.outcome }} + Setup python environment: ${{ steps.setup-python.outcome }} + Check the nearly published release against the advertised: ${{ steps.check-latest-version.outcome }} + Setup ssh: ${{ steps.ssh-setup.outcome }} + Sync with the releases: ${{ steps.sync-releases.outcome }} SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }} if: >- ${{ diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index e99704adb..1308f45fa 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -25,7 +25,7 @@ jobs: submodules: recursive - name: Check files id: check-files - uses: tj-actions/changed-files@v39 + uses: tj-actions/changed-files@v40 with: since_last_remote_commit: ${{ github.event_name != 'pull_request' }} files: | diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index c1dfc55eb..b7fe0a866 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -31,7 +31,7 @@ jobs: submodules: recursive - name: Check files id: file-check - uses: tj-actions/changed-files@v39 + uses: tj-actions/changed-files@v40 with: since_last_remote_commit: ${{ github.event_name != 'pull_request' }} files: | diff --git a/.github/workflows/monitor-releases.yml b/.github/workflows/monitor-releases.yml new file mode 100644 index 000000000..649cf68aa --- /dev/null +++ b/.github/workflows/monitor-releases.yml @@ -0,0 +1,72 @@ +--- +name: Monitor-releases + +on: + release: + types: [released, deleted] + workflow_dispatch: + inputs: + channel: + description: 'Specify the release channel' + required: true + default: 'stable' + + +concurrency: # This keeps multiple instances of the job from running concurrently for the same ref and event type. + group: monitor-{{ github.event.inputs.channel }}-releases-${{ github.ref }}-${{ github.event_name }} + cancel-in-progress: true + +jobs: + update-stable-agents-metadata: + name: update-stable-agents-metadata + runs-on: ubuntu-latest + steps: + - name: Checkout + id: checkout + uses: actions/checkout@v4 + with: + token: ${{ secrets.NETDATABOT_GITHUB_TOKEN }} + - name: Init python environment + uses: actions/setup-python@v4 + id: init-python + with: + python-version: "3.12" + - name: Setup python environment + id: setup-python + run: | + pip install -r .github/scripts/modules/requirements.txt + - name: Check for newer versions + id: check-newer-releases + run: | + python .github/scripts/check_latest_versions_per_channel.py "${{ github.event.inputs.channel }}" + - name: SSH setup + id: ssh-setup + if: github.event_name == 'workflow_dispatch' && github.repository == 'netdata/netdata' && steps.check-newer-releases.outputs.versions_needs_update == 'true' + uses: shimataro/ssh-key-action@v2 + with: + key: ${{ secrets.NETDATABOT_PACKAGES_SSH_KEY }} + name: id_ecdsa + known_hosts: ${{ secrets.PACKAGES_KNOWN_HOSTS }} + - name: Sync newer releases + id: sync-releases + if: github.event_name == 'workflow_dispatch' && github.repository == 'netdata/netdata' && steps.check-newer-releases.outputs.versions_needs_update == 'true' + run: | + .github/scripts/upload-new-version-tags.sh + - name: Failure Notification + uses: rtCamp/action-slack-notify@v2 + env: + SLACK_COLOR: 'danger' + SLACK_FOOTER: '' + SLACK_ICON_EMOJI: ':github-actions:' + SLACK_TITLE: 'Failed to prepare changelog:' + SLACK_USERNAME: 'GitHub Actions' + SLACK_MESSAGE: |- + ${{ github.repository }}: Failed to update stable Agent's metadata. + Checkout: ${{ steps.checkout.outcome }} + Init python: ${{ steps.init-python.outcome }} + Setup python: ${{ steps.setup-python.outcome }} + Check for newer stable releaes: ${{ steps.check-newer-releases.outcome }} + Setup ssh: ${{ steps.ssh-setup.outcome }} + Syncing newer release to packages.netdata.cloud : ${{ steps.sync-releases.outcome }} + SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }} + if: failure() diff --git a/.github/workflows/packaging.yml b/.github/workflows/packaging.yml index 9b15cf94b..eb936c4d9 100644 --- a/.github/workflows/packaging.yml +++ b/.github/workflows/packaging.yml @@ -41,7 +41,7 @@ jobs: submodules: recursive - name: Check files id: file-check - uses: tj-actions/changed-files@v39 + uses: tj-actions/changed-files@v40 with: since_last_remote_commit: ${{ github.event_name != 'pull_request' }} files: | diff --git a/.github/workflows/review.yml b/.github/workflows/review.yml index 732a4a5a1..8cb61f2cf 100644 --- a/.github/workflows/review.yml +++ b/.github/workflows/review.yml @@ -234,6 +234,7 @@ jobs: ./.git/* packaging/makeself/makeself.sh packaging/makeself/makeself-header.sh + ./fluent-bit/* yamllint: name: yamllint -- cgit v1.2.3