From d079b656b4719739b2247dcd9d46e9bec793095a Mon Sep 17 00:00:00 2001
From: Daniel Baumann <daniel.baumann@progress-linux.org>
Date: Mon, 6 Feb 2023 17:11:34 +0100
Subject: Merging upstream version 1.38.0.

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
---
 .github/scripts/docker-test.sh                |  4 ++
 .github/scripts/gen-docker-tags.py            | 13 ++++--
 .github/scripts/gen-matrix-build.py           | 34 ++++++++++++++++
 .github/scripts/gen-matrix-packaging.py       | 36 +++++++++++++++++
 .github/scripts/gen-matrix-repoconfig.py      | 27 +++++++++++++
 .github/scripts/get-static-cache-key.sh       |  2 +-
 .github/scripts/package-upload.sh             |  2 +-
 .github/scripts/prepare-release-base.sh       | 58 ++++++++++++++-------------
 .github/scripts/run-updater-check.sh          | 21 ++++++++--
 .github/scripts/run_install_with_dist_file.sh |  2 +-
 10 files changed, 162 insertions(+), 37 deletions(-)
 create mode 100755 .github/scripts/gen-matrix-build.py
 create mode 100755 .github/scripts/gen-matrix-packaging.py
 create mode 100755 .github/scripts/gen-matrix-repoconfig.py

(limited to '.github/scripts')

diff --git a/.github/scripts/docker-test.sh b/.github/scripts/docker-test.sh
index 22821d17e..0f5fa469c 100755
--- a/.github/scripts/docker-test.sh
+++ b/.github/scripts/docker-test.sh
@@ -26,6 +26,10 @@ wait_for() {
     sleep 1
     if [ "$i" -gt "$timeout" ]; then
       printf "Timed out!\n"
+      docker ps -a
+      echo "::group::Netdata container logs"
+      docker logs netdata 2>&1
+      echo "::endgroup::"
       return 1
     fi
     i="$((i + 1))"
diff --git a/.github/scripts/gen-docker-tags.py b/.github/scripts/gen-docker-tags.py
index df4dc0263..8c88d3b5e 100755
--- a/.github/scripts/gen-docker-tags.py
+++ b/.github/scripts/gen-docker-tags.py
@@ -6,9 +6,14 @@ version = sys.argv[1].split('.')
 suffix = sys.argv[2]
 
 REPO = f'netdata/netdata{suffix}'
+GHCR = f'ghcr.io/{REPO}'
+QUAY = f'quay.io/{REPO}'
 
-MAJOR = ':'.join([REPO, version[0]])
-MINOR = ':'.join([REPO, '.'.join(version[0:2])])
-PATCH = ':'.join([REPO, '.'.join(version[0:3])])
+tags = []
 
-print(','.join([MAJOR, MINOR, PATCH]))
+for repo in [REPO, GHCR, QUAY]:
+    tags.append(':'.join([repo, version[0]]))
+    tags.append(':'.join([repo, '.'.join(version[0:2])]))
+    tags.append(':'.join([repo, '.'.join(version[0:3])]))
+
+print(','.join(tags))
diff --git a/.github/scripts/gen-matrix-build.py b/.github/scripts/gen-matrix-build.py
new file mode 100755
index 000000000..28406470f
--- /dev/null
+++ b/.github/scripts/gen-matrix-build.py
@@ -0,0 +1,34 @@
+#!/usr/bin/env python3
+
+import json
+
+from ruamel.yaml import YAML
+
+yaml = YAML(typ='safe')
+entries = []
+
+with open('.github/data/distros.yml') as f:
+    data = yaml.load(f)
+
+for i, v in enumerate(data['include']):
+    e = {
+      'artifact_key': v['distro'] + str(v['version']).replace('.', ''),
+      'version': v['version'],
+    }
+
+    if 'base_image' in v:
+        e['distro'] = ':'.join([v['base_image'], str(v['version'])])
+    else:
+        e['distro'] = ':'.join([v['distro'], str(v['version'])])
+
+    if 'env_prep' in v:
+        e['env_prep'] = v['env_prep']
+
+    if 'jsonc_removal' in v:
+        e['jsonc_removal'] = v['jsonc_removal']
+
+    entries.append(e)
+
+entries.sort(key=lambda k: k['distro'])
+matrix = json.dumps({'include': entries}, sort_keys=True)
+print(matrix)
diff --git a/.github/scripts/gen-matrix-packaging.py b/.github/scripts/gen-matrix-packaging.py
new file mode 100755
index 000000000..01e9ec790
--- /dev/null
+++ b/.github/scripts/gen-matrix-packaging.py
@@ -0,0 +1,36 @@
+#!/usr/bin/env python3
+
+import json
+import sys
+
+from ruamel.yaml import YAML
+
+ALWAYS_RUN_ARCHES = ["amd64", "x86_64"]
+SHORT_RUN = sys.argv[1]
+yaml = YAML(typ='safe')
+entries = list()
+run_limited = False
+
+with open('.github/data/distros.yml') as f:
+    data = yaml.load(f)
+
+if bool(int(SHORT_RUN)):
+    run_limited = True
+
+for i, v in enumerate(data['include']):
+    if 'packages' in data['include'][i]:
+        for arch in data['include'][i]['packages']['arches']:
+            if arch in ALWAYS_RUN_ARCHES or not run_limited:
+                entries.append({
+                    'distro': data['include'][i]['distro'],
+                    'version': data['include'][i]['version'],
+                    'repo_distro': data['include'][i]['packages']['repo_distro'],
+                    'format': data['include'][i]['packages']['type'],
+                    'base_image': data['include'][i]['base_image'] if 'base_image' in data['include'][i] else data['include'][i]['distro'],
+                    'platform': data['platform_map'][arch],
+                    'arch': arch
+                })
+
+entries.sort(key=lambda k: (data['arch_order'].index(k['arch']), k['distro'], k['version']))
+matrix = json.dumps({'include': entries}, sort_keys=True)
+print(matrix)
diff --git a/.github/scripts/gen-matrix-repoconfig.py b/.github/scripts/gen-matrix-repoconfig.py
new file mode 100755
index 000000000..46f671697
--- /dev/null
+++ b/.github/scripts/gen-matrix-repoconfig.py
@@ -0,0 +1,27 @@
+#!/usr/bin/env python3
+
+import json
+
+from ruamel.yaml import YAML
+
+yaml = YAML(typ='safe')
+entries = list()
+
+with open('.github/data/distros.yml') as f:
+    data = yaml.load(f)
+
+for i, v in enumerate(data['include']):
+    if 'packages' in data['include'][i]:
+        entries.append({
+            'distro': data['include'][i]['distro'],
+            'version': data['include'][i]['version'],
+            'pkgclouddistro': data['include'][i]['packages']['repo_distro'],
+            'format': data['include'][i]['packages']['type'],
+            'base_image': data['include'][i]['base_image'] if 'base_image' in data['include'][i] else data['include'][i]['distro'],
+            'platform': data['platform_map']['amd64'],
+            'arches': ' '.join(['"' + x + '"' for x in data['include'][i]['packages']['arches']])
+        })
+
+entries.sort(key=lambda k: (k['distro'], k['version']))
+matrix = json.dumps({'include': entries}, sort_keys=True)
+print(matrix)
diff --git a/.github/scripts/get-static-cache-key.sh b/.github/scripts/get-static-cache-key.sh
index d9fa28597..3b07088f4 100755
--- a/.github/scripts/get-static-cache-key.sh
+++ b/.github/scripts/get-static-cache-key.sh
@@ -12,4 +12,4 @@ docker run -it --rm --platform "${platform}" netdata/static-builder sh -c 'apk l
 
 h="$(sha256sum /tmp/static-cache-key-data | cut -f 1 -d ' ')"
 
-echo "::set-output name=key::static-${arch}-${h}"
+echo "key=static-${arch}-${h}" >> "${GITHUB_OUTPUT}"
diff --git a/.github/scripts/package-upload.sh b/.github/scripts/package-upload.sh
index fd8a8cda2..13d63b4a7 100755
--- a/.github/scripts/package-upload.sh
+++ b/.github/scripts/package-upload.sh
@@ -19,7 +19,7 @@ mkdir -p "${staging}"
 
 case "${format}" in
     deb)
-        src="${staging}/$(echo "${distro}" | cut -f 1 -d '/')/pool/"
+        src="${staging}/${distro}"
         mkdir -p "${src}"
 
         for pkg in ${packages}; do
diff --git a/.github/scripts/prepare-release-base.sh b/.github/scripts/prepare-release-base.sh
index 7c24f6b66..06a2da160 100755
--- a/.github/scripts/prepare-release-base.sh
+++ b/.github/scripts/prepare-release-base.sh
@@ -97,7 +97,7 @@ git config user.email "bot@netdata.cloud"
 
 if [ "${REPO}" != "netdata/netdata" ] && [ -z "${RELEASE_TEST}" ]; then
     echo "::notice::Not running in the netdata/netdata repository, not queueing a release build."
-    echo "::set-output name=run::false"
+    echo "run=false" >> "${GITHUB_OUTPUT}"
 elif [ "${EVENT_NAME}" = 'schedule' ] || [ "${EVENT_TYPE}" = 'nightly' ]; then
     echo "::notice::Preparing a nightly release build."
     LAST_TAG=$(git describe --abbrev=0 --tags)
@@ -107,15 +107,16 @@ elif [ "${EVENT_NAME}" = 'schedule' ] || [ "${EVENT_TYPE}" = 'nightly' ]; then
     HEAD_COMMIT="$(git rev-parse HEAD)"
     if [ "${EVENT_NAME}" = 'schedule' ] && [ "${LAST_VERSION_COMMIT}" = "${HEAD_COMMIT}" ] && grep -qE '.*-nightly$' packaging/version; then
         echo "::notice::No commits since last nightly build, not publishing a new nightly build."
-        echo "::set-output name=run::false"
+        echo "run=false" >> "${GITHUB_OUTPUT}"
     else
         echo "${NEW_VERSION}" > packaging/version || exit 1
-        echo "::set-output name=run::true"
-        echo "::set-output name=message::Update changelog and version for nightly build: ${NEW_VERSION}."
-        echo "::set-output name=ref::master"
-        echo "::set-output name=type::nightly"
-        echo "::set-output name=branch::master"
-        echo "::set-output name=version::nightly"
+        # shellcheck disable=SC2129
+        echo "run=true" >> "${GITHUB_OUTPUT}"
+        echo "message=Update changelog and version for nightly build: ${NEW_VERSION}." >> "${GITHUB_OUTPUT}"
+        echo "ref=master" >> "${GITHUB_OUTPUT}"
+        echo "type=nightly" >> "${GITHUB_OUTPUT}"
+        echo "branch=master" >> "${GITHUB_OUTPUT}"
+        echo "version=nightly" >> "${GITHUB_OUTPUT}"
     fi
 elif [ "${EVENT_TYPE}" = 'patch' ] && [ "${EVENT_VERSION}" != "nightly" ]; then
     echo "::notice::Preparing a patch release build."
@@ -130,12 +131,13 @@ elif [ "${EVENT_TYPE}" = 'patch' ] && [ "${EVENT_VERSION}" != "nightly" ]; then
     major_matches || exit 1
     check_newer_patch_version || exit 1
     echo "${EVENT_VERSION}" > packaging/version || exit 1
-    echo "::set-output name=run::true"
-    echo "::set-output name=message::Patch release ${EVENT_VERSION}."
-    echo "::set-output name=ref::${EVENT_VERSION}"
-    echo "::set-output name=type::release"
-    echo "::set-output name=branch::${branch_name}"
-    echo "::set-output name=version::$(tr -d 'v' < packaging/version)"
+    # shellcheck disable=SC2129
+    echo "run=true" >> "${GITHUB_OUTPUT}"
+    echo "message=Patch release ${EVENT_VERSION}." >> "${GITHUB_OUTPUT}"
+    echo "ref=${EVENT_VERSION}" >> "${GITHUB_OUTPUT}"
+    echo "type=release" >> "${GITHUB_OUTPUT}"
+    echo "branch=${branch_name}" >> "${GITHUB_OUTPUT}"
+    echo "version=$(tr -d 'v' < packaging/version)" >> "${GITHUB_OUTPUT}"
 elif [ "${EVENT_TYPE}" = 'minor' ] && [ "${EVENT_VERSION}" != "nightly" ]; then
     echo "::notice::Preparing a minor release build."
     check_version_format || exit 1
@@ -149,13 +151,14 @@ elif [ "${EVENT_TYPE}" = 'minor' ] && [ "${EVENT_VERSION}" != "nightly" ]; then
         exit 1
     fi
     echo "${EVENT_VERSION}" > packaging/version || exit 1
-    echo "::set-output name=run::true"
-    echo "::set-output name=message::Minor release ${EVENT_VERSION}."
-    echo "::set-output name=ref::${EVENT_VERSION}"
-    echo "::set-output name=type::release"
-    echo "::set-output name=branch::master"
-    echo "::set-output name=new-branch::${branch_name}"
-    echo "::set-output name=version::$(tr -d 'v' < packaging/version)"
+    # shellcheck disable=SC2129
+    echo "run=true" >> "${GITHUB_OUTPUT}"
+    echo "message=Minor release ${EVENT_VERSION}." >> "${GITHUB_OUTPUT}"
+    echo "ref=${EVENT_VERSION}" >> "${GITHUB_OUTPUT}"
+    echo "type=release" >> "${GITHUB_OUTPUT}"
+    echo "branch=master" >> "${GITHUB_OUTPUT}"
+    echo "new-branch=${branch_name}" >> "${GITHUB_OUTPUT}"
+    echo "version=$(tr -d 'v' < packaging/version)" >> "${GITHUB_OUTPUT}"
 elif [ "${EVENT_TYPE}" = 'major' ] && [ "${EVENT_VERSION}" != "nightly" ]; then
     echo "::notice::Preparing a major release build."
     check_version_format || exit 1
@@ -164,12 +167,13 @@ elif [ "${EVENT_TYPE}" = 'major' ] && [ "${EVENT_VERSION}" != "nightly" ]; then
     check_newer_major_version || exit 1
     check_for_existing_tag || exit 1
     echo "${EVENT_VERSION}" > packaging/version || exit 1
-    echo "::set-output name=run::true"
-    echo "::set-output name=message::Major release ${EVENT_VERSION}"
-    echo "::set-output name=ref::${EVENT_VERSION}"
-    echo "::set-output name=type::release"
-    echo "::set-output name=branch::master"
-    echo "::set-output name=version::$(tr -d 'v' < packaging/version)"
+    # shellcheck disable=SC2129
+    echo "run=true" >> "${GITHUB_OUTPUT}"
+    echo "message=Major release ${EVENT_VERSION}" >> "${GITHUB_OUTPUT}"
+    echo "ref=${EVENT_VERSION}" >> "${GITHUB_OUTPUT}"
+    echo "type=release" >> "${GITHUB_OUTPUT}"
+    echo "branch=master" >> "${GITHUB_OUTPUT}"
+    echo "version=$(tr -d 'v' < packaging/version)" >> "${GITHUB_OUTPUT}"
 else
     echo '::error::Unrecognized release type or invalid version.'
     exit 1
diff --git a/.github/scripts/run-updater-check.sh b/.github/scripts/run-updater-check.sh
index 31ab71de8..a96a1d6ef 100755
--- a/.github/scripts/run-updater-check.sh
+++ b/.github/scripts/run-updater-check.sh
@@ -4,11 +4,26 @@ echo ">>> Installing CI support packages..."
 /netdata/.github/scripts/ci-support-pkgs.sh
 echo ">>> Installing Netdata..."
 /netdata/packaging/installer/kickstart.sh --dont-wait --build-only --disable-telemetry || exit 1
-echo "::group::Environment File Contents"
+echo "::group::>>> Pre-Update Environment File Contents"
 cat /etc/netdata/.environment
 echo "::endgroup::"
+echo "::group::>>> Pre-Update Netdata Build Info"
+netdata -W buildinfo
+echo "::endgroup::"
 echo ">>> Updating Netdata..."
-export NETDATA_NIGHTLIES_BASEURL="http://localhost:8080/artifacts/" # Pull the tarball from the local web server.
-/netdata/packaging/installer/netdata-updater.sh --not-running-from-cron --no-updater-self-update || exit 1
+export NETDATA_BASE_URL="http://localhost:8080/artifacts/" # Pull the tarball from the local web server.
+timeout 3600 /netdata/packaging/installer/netdata-updater.sh --not-running-from-cron --no-updater-self-update
+
+case "$?" in
+    124) echo "!!! Updater timed out." ; exit 1 ;;
+    0) ;;
+    *) echo "!!! Updater failed." ; exit 1 ;;
+esac
+echo "::group::>>> Post-Update Environment File Contents"
+cat /etc/netdata/.environment
+echo "::endgroup::"
+echo "::group::>>> Post-Update Netdata Build Info"
+netdata -W buildinfo
+echo "::endgroup::"
 echo ">>> Checking if update was successful..."
 /netdata/.github/scripts/check-updater.sh || exit 1
diff --git a/.github/scripts/run_install_with_dist_file.sh b/.github/scripts/run_install_with_dist_file.sh
index d59e8b134..74652efdd 100755
--- a/.github/scripts/run_install_with_dist_file.sh
+++ b/.github/scripts/run_install_with_dist_file.sh
@@ -33,7 +33,7 @@ docker run \
   -v "${PWD}:/netdata" \
   -w /netdata \
   "ubuntu:latest" \
-  /bin/bash -c "./install-required-packages.sh --dont-wait --non-interactive netdata && apt install wget && ./netdata-installer.sh --dont-wait --require-cloud --disable-telemetry --install /tmp --one-time-build && echo \"Validating netdata instance is running\" && wget -O - 'http://127.0.0.1:19999/api/v1/info' | grep version"
+  /bin/bash -c "./install-required-packages.sh --dont-wait --non-interactive netdata && apt install wget && ./netdata-installer.sh --dont-wait --require-cloud --disable-telemetry --install-prefix /tmp --one-time-build && echo \"Validating netdata instance is running\" && wget -O - 'http://127.0.0.1:19999/api/v1/info' | grep version"
 popd || exit 1
 
 echo "All Done!"
-- 
cgit v1.2.3