From 81581f9719bc56f01d5aa08952671d65fda9867a Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Mon, 8 May 2023 18:27:08 +0200 Subject: Merging upstream version 1.39.0. Signed-off-by: Daniel Baumann --- .github/scripts/ci-support-pkgs.sh | 17 +++++---- .github/scripts/gen-matrix-build.py | 2 +- .github/scripts/gen-matrix-eol-check.py | 29 ++++++++++++++++ .github/scripts/gen-matrix-packaging.py | 2 +- .github/scripts/gen-matrix-repoconfig.py | 2 +- .github/scripts/pkg-test.sh | 36 +++++++++++++++++-- .github/scripts/platform-impending-eol.py | 58 +++++++++++++++++++++++++++++++ .github/scripts/run-updater-check.sh | 1 + 8 files changed, 134 insertions(+), 13 deletions(-) create mode 100755 .github/scripts/gen-matrix-eol-check.py create mode 100755 .github/scripts/platform-impending-eol.py (limited to '.github/scripts') diff --git a/.github/scripts/ci-support-pkgs.sh b/.github/scripts/ci-support-pkgs.sh index bfa9c83a5..9ba11b68e 100755 --- a/.github/scripts/ci-support-pkgs.sh +++ b/.github/scripts/ci-support-pkgs.sh @@ -5,10 +5,13 @@ set -e -if [ -f /etc/centos-release ] || [ -f /etc/redhat-release ] || [ -f /etc/fedora-release ] || [ -f /etc/almalinux-release ]; then - # Alma, Fedora, CentOS, Redhat - dnf install -y procps-ng cronie cronie-anacron || yum install -y procps-ng cronie cronie-anacron -elif [ -f /etc/arch-release ]; then - # Arch - pacman -S --noconfirm cronie -fi +. /etc/os-release + +case "${ID}" in + amzn|almalinux|centos|fedora) + dnf install -y procps-ng cronie cronie-anacron || yum install -y procps-ng cronie cronie-anacron + ;; + arch) + pacman -S --noconfirm cronie + ;; +esac diff --git a/.github/scripts/gen-matrix-build.py b/.github/scripts/gen-matrix-build.py index 28406470f..3185e8836 100755 --- a/.github/scripts/gen-matrix-build.py +++ b/.github/scripts/gen-matrix-build.py @@ -17,7 +17,7 @@ for i, v in enumerate(data['include']): } if 'base_image' in v: - e['distro'] = ':'.join([v['base_image'], str(v['version'])]) + e['distro'] = v['base_image'] else: e['distro'] = ':'.join([v['distro'], str(v['version'])]) diff --git a/.github/scripts/gen-matrix-eol-check.py b/.github/scripts/gen-matrix-eol-check.py new file mode 100755 index 000000000..638527284 --- /dev/null +++ b/.github/scripts/gen-matrix-eol-check.py @@ -0,0 +1,29 @@ +#!/usr/bin/env python3 +'''Generate the build matrix for the EOL check jobs.''' + +import json + +from ruamel.yaml import YAML + +yaml = YAML(typ='safe') +entries = list() + +with open('.github/data/distros.yml') as f: + data = yaml.load(f) + +for item in data['include']: + if 'eol_check' in item and item['eol_check']: + if isinstance(item['eol_check'], str): + distro = item['eol_check'] + else: + distro = item['distro'] + + entries.append({ + 'distro': distro, + 'release': item['version'], + 'full_name': f'{ item["distro"] } { item["version"] }' + }) + +entries.sort(key=lambda k: (k['distro'], k['release'])) +matrix = json.dumps({'include': entries}, sort_keys=True) +print(matrix) diff --git a/.github/scripts/gen-matrix-packaging.py b/.github/scripts/gen-matrix-packaging.py index 01e9ec790..9347cd767 100755 --- a/.github/scripts/gen-matrix-packaging.py +++ b/.github/scripts/gen-matrix-packaging.py @@ -26,7 +26,7 @@ for i, v in enumerate(data['include']): 'version': data['include'][i]['version'], 'repo_distro': data['include'][i]['packages']['repo_distro'], 'format': data['include'][i]['packages']['type'], - 'base_image': data['include'][i]['base_image'] if 'base_image' in data['include'][i] else data['include'][i]['distro'], + 'base_image': data['include'][i]['base_image'] if 'base_image' in data['include'][i] else ':'.join([data['include'][i]['distro'], data['include'][i]['version']]), 'platform': data['platform_map'][arch], 'arch': arch }) diff --git a/.github/scripts/gen-matrix-repoconfig.py b/.github/scripts/gen-matrix-repoconfig.py index 46f671697..264cd53e7 100755 --- a/.github/scripts/gen-matrix-repoconfig.py +++ b/.github/scripts/gen-matrix-repoconfig.py @@ -17,7 +17,7 @@ for i, v in enumerate(data['include']): 'version': data['include'][i]['version'], 'pkgclouddistro': data['include'][i]['packages']['repo_distro'], 'format': data['include'][i]['packages']['type'], - 'base_image': data['include'][i]['base_image'] if 'base_image' in data['include'][i] else data['include'][i]['distro'], + 'base_image': data['include'][i]['base_image'] if 'base_image' in data['include'][i] else ':'.join([data['include'][i]['distro'], data['include'][i]['version']]), 'platform': data['platform_map']['amd64'], 'arches': ' '.join(['"' + x + '"' for x in data['include'][i]['packages']['arches']]) }) diff --git a/.github/scripts/pkg-test.sh b/.github/scripts/pkg-test.sh index e3bc3e7d4..45b8c320b 100755 --- a/.github/scripts/pkg-test.sh +++ b/.github/scripts/pkg-test.sh @@ -13,6 +13,7 @@ install_debian_like() { apt-get update # Install Netdata + # Strange quoting is required here so that glob matching works. apt-get install -y /netdata/artifacts/netdata_"${VERSION}"*_*.deb || exit 1 # Install testing tools @@ -27,8 +28,13 @@ install_fedora_like() { pkg_version="$(echo "${VERSION}" | tr - .)" + if [ "${PKGMGR}" = "dnf" ]; then + opts="--allowerasing" + fi + # Install Netdata - "$PKGMGR" install -y /netdata/artifacts/netdata-"${pkg_version}"-*.rpm + # Strange quoting is required here so that glob matching works. + "$PKGMGR" install -y /netdata/artifacts/netdata-"${pkg_version}"-*.rpm || exit 1 # Install testing tools "$PKGMGR" install -y curl nc jq || exit 1 @@ -50,9 +56,29 @@ install_centos() { "$PKGMGR" install -y epel-release || exit 1 # Install Netdata - "$PKGMGR" install -y /netdata/artifacts/netdata-"${pkg_version}"-*.rpm + # Strange quoting is required here so that glob matching works. + "$PKGMGR" install -y /netdata/artifacts/netdata-"${pkg_version}"-*.rpm || exit 1 # Install testing tools + # shellcheck disable=SC2086 + "$PKGMGR" install -y ${opts} curl nc jq || exit 1 +} + +install_amazon_linux() { + PKGMGR="$( (command -v dnf > /dev/null && echo "dnf") || echo "yum")" + + pkg_version="$(echo "${VERSION}" | tr - .)" + + if [ "${PKGMGR}" = "dnf" ]; then + opts="--allowerasing" + fi + + # Install Netdata + # Strange quoting is required here so that glob matching works. + "$PKGMGR" install -y /netdata/artifacts/netdata-"${pkg_version}"-*.rpm || exit 1 + + # Install testing tools + # shellcheck disable=SC2086 "$PKGMGR" install -y ${opts} curl nc jq || exit 1 } @@ -63,7 +89,8 @@ install_suse_like() { pkg_version="$(echo "${VERSION}" | tr - .)" # Install Netdata - zypper install -y --allow-unsigned-rpm /netdata/artifacts/netdata-"${pkg_version}"-*.rpm + # Strange quoting is required here so that glob matching works. + zypper install -y --allow-unsigned-rpm /netdata/artifacts/netdata-"${pkg_version}"-*.rpm || exit 1 # Install testing tools zypper install -y --no-recommends curl netcat-openbsd jq || exit 1 @@ -114,6 +141,9 @@ case "${DISTRO}" in centos | rockylinux | almalinux) install_centos ;; + amazonlinux) + install_amazon_linux + ;; opensuse) install_suse_like ;; diff --git a/.github/scripts/platform-impending-eol.py b/.github/scripts/platform-impending-eol.py new file mode 100755 index 000000000..c57e5edde --- /dev/null +++ b/.github/scripts/platform-impending-eol.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python3 +'''Check if a given distro is going to be EOL soon. + + This queries the public API of https://endoflife.date to fetch EOL dates. + + ‘soon’ is defined by LEAD_DAYS, currently 30 days.''' + +import datetime +import json +import sys +import urllib.request + +URL_BASE = 'https://endoflife.date/api' +NOW = datetime.date.today() +LEAD_DAYS = datetime.timedelta(days=30) + +DISTRO = sys.argv[1] +RELEASE = sys.argv[2] + +EXIT_NOT_IMPENDING = 0 +EXIT_IMPENDING = 1 +EXIT_NO_DATA = 2 +EXIT_FAILURE = 3 + +try: + with urllib.request.urlopen(f'{ URL_BASE }/{ DISTRO }/{ RELEASE }.json') as response: + match response.status: + case 200: + data = json.load(response) + case _: + print( + f'Failed to retrieve data for { DISTRO } { RELEASE } ' + + f'(status: { response.status }).', + file=sys.stderr + ) + sys.exit(EXIT_FAILURE) +except urllib.error.HTTPError as e: + match e.code: + case 404: + print(f'No data available for { DISTRO } { RELEASE }.', file=sys.stderr) + sys.exit(EXIT_NO_DATA) + case _: + print( + f'Failed to retrieve data for { DISTRO } { RELEASE } ' + + f'(status: { e.code }).', + file=sys.stderr + ) + sys.exit(EXIT_FAILURE) + +eol = datetime.date.fromisoformat(data['eol']) + +offset = abs(eol - NOW) + +if offset <= LEAD_DAYS: + print(data['eol']) + sys.exit(EXIT_IMPENDING) +else: + sys.exit(EXIT_NOT_IMPENDING) diff --git a/.github/scripts/run-updater-check.sh b/.github/scripts/run-updater-check.sh index a96a1d6ef..1224d8f67 100755 --- a/.github/scripts/run-updater-check.sh +++ b/.github/scripts/run-updater-check.sh @@ -2,6 +2,7 @@ echo ">>> Installing CI support packages..." /netdata/.github/scripts/ci-support-pkgs.sh +mkdir -p /etc/cron.daily # Needed to make auto-update checking work correctly on some platforms. echo ">>> Installing Netdata..." /netdata/packaging/installer/kickstart.sh --dont-wait --build-only --disable-telemetry || exit 1 echo "::group::>>> Pre-Update Environment File Contents" -- cgit v1.2.3