From 310edf444908b09ea6d00c03baceb7925f3bb7a2 Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Thu, 21 Mar 2024 18:19:04 +0100 Subject: Merging upstream version 1.45.0. Signed-off-by: Daniel Baumann --- .github/scripts/build-artifacts.sh | 82 --------------------------- .github/scripts/build-dist.sh | 71 ----------------------- .github/scripts/docker-test.sh | 50 +++------------- .github/scripts/gen-docker-imagetool-args.py | 27 +++++++++ .github/scripts/gen-docker-tags.py | 35 ++++++++---- .github/scripts/gen-matrix-build.py | 3 + .github/scripts/gen-matrix-packaging.py | 1 + .github/scripts/get-go-version.py | 39 +++++++++++++ .github/scripts/get-static-cache-key.sh | 1 + .github/scripts/pkg-test.sh | 41 +------------- .github/scripts/run-updater-check.sh | 3 +- .github/scripts/run_install_with_dist_file.sh | 39 ------------- 12 files changed, 109 insertions(+), 283 deletions(-) delete mode 100755 .github/scripts/build-artifacts.sh delete mode 100755 .github/scripts/build-dist.sh create mode 100755 .github/scripts/gen-docker-imagetool-args.py create mode 100755 .github/scripts/get-go-version.py delete mode 100755 .github/scripts/run_install_with_dist_file.sh (limited to '.github/scripts') diff --git a/.github/scripts/build-artifacts.sh b/.github/scripts/build-artifacts.sh deleted file mode 100755 index 569c79a5a..000000000 --- a/.github/scripts/build-artifacts.sh +++ /dev/null @@ -1,82 +0,0 @@ -#!/bin/sh -# -# Builds the netdata-vX.y.Z-xxxx.tar.gz source tarball (dist) -# and netdata-vX.Y.Z-xxxx.gz.run (static x86_64) artifacts. - -set -e - -# shellcheck source=.github/scripts/functions.sh -. "$(dirname "$0")/functions.sh" - -NAME="${NAME:-netdata}" -VERSION="${VERSION:-"$(git describe)"}" -BASENAME="$NAME-$VERSION" - -prepare_build() { - progress "Preparing build" - ( - test -d artifacts || mkdir -p artifacts - echo "${VERSION}" > packaging/version - ) >&2 -} - -build_dist() { - progress "Building dist" - ( - command -v git > /dev/null && [ -d .git ] && git clean -d -f - autoreconf -ivf - ./configure \ - --prefix=/usr \ - --sysconfdir=/etc \ - --localstatedir=/var \ - --libexecdir=/usr/libexec \ - --with-zlib \ - --with-math \ - --with-user=netdata \ - --disable-dependency-tracking \ - CFLAGS=-O2 - make dist - mv "${BASENAME}.tar.gz" artifacts/ - ) >&2 -} - -build_static_x86_64() { - progress "Building static x86_64" - ( - command -v git > /dev/null && [ -d .git ] && git clean -d -f - USER="" ./packaging/makeself/build-x86_64-static.sh - ) >&2 -} - -prepare_assets() { - progress "Preparing assets" - ( - cp packaging/version artifacts/latest-version.txt - - cd artifacts || exit 1 - ln -f "${BASENAME}.tar.gz" netdata-latest.tar.gz - ln -f "${BASENAME}.gz.run" netdata-latest.gz.run - sha256sum -b ./* > "sha256sums.txt" - ) >&2 -} - -steps="prepare_build build_dist build_static_x86_64" -steps="$steps prepare_assets" - -_main() { - for step in $steps; do - if ! run "$step"; then - if [ -t 1 ]; then - debug - else - fail "Build failed" - fi - fi - done - - echo "🎉 All Done!" -} - -if [ -n "$0" ] && [ x"$0" != x"-bash" ]; then - _main "$@" -fi diff --git a/.github/scripts/build-dist.sh b/.github/scripts/build-dist.sh deleted file mode 100755 index 027b62147..000000000 --- a/.github/scripts/build-dist.sh +++ /dev/null @@ -1,71 +0,0 @@ -#!/bin/sh -# -# Builds the netdata-vX.y.Z-xxxx.tar.gz source tarball (dist) - -set -e - -# shellcheck source=.github/scripts/functions.sh -. "$(dirname "$0")/functions.sh" - -NAME="${NAME:-netdata}" -VERSION="${VERSION:-"$(git describe --always)"}" -BASENAME="$NAME-$VERSION" - -prepare_build() { - progress "Preparing build" - ( - test -d artifacts || mkdir -p artifacts - echo "${VERSION}" > packaging/version - ) >&2 -} - -build_dist() { - progress "Building dist" - ( - command -v git > /dev/null && [ -d .git ] && git clean -d -f - autoreconf -ivf - ./configure \ - --prefix=/usr \ - --sysconfdir=/etc \ - --localstatedir=/var \ - --libexecdir=/usr/libexec \ - --with-zlib \ - --with-math \ - --with-user=netdata \ - --disable-dependency-tracking \ - CFLAGS=-O2 - make dist - mv "${BASENAME}.tar.gz" artifacts/ - ) >&2 -} - -prepare_assets() { - progress "Preparing assets" - ( - cp packaging/version artifacts/latest-version.txt - cd artifacts || exit 1 - ln -f "${BASENAME}.tar.gz" netdata-latest.tar.gz - ln -f "${BASENAME}.gz.run" netdata-latest.gz.run - sha256sum -b ./* > "sha256sums.txt" - ) >&2 -} - -steps="prepare_build build_dist prepare_assets" - -_main() { - for step in $steps; do - if ! run "$step"; then - if [ -t 1 ]; then - debug - else - fail "Build failed" - fi - fi - done - - echo "🎉 All Done!" -} - -if [ -n "$0" ] && [ x"$0" != x"-bash" ]; then - _main "$@" -fi diff --git a/.github/scripts/docker-test.sh b/.github/scripts/docker-test.sh index 0f5fa469c..0ef064501 100755 --- a/.github/scripts/docker-test.sh +++ b/.github/scripts/docker-test.sh @@ -1,41 +1,8 @@ #!/bin/sh -export DEBIAN_FRONTEND=noninteractive - -wait_for() { - host="${1}" - port="${2}" - name="${3}" - timeout="30" - - if command -v nc > /dev/null ; then - netcat="nc" - elif command -v netcat > /dev/null ; then - netcat="netcat" - else - printf "Unable to find a usable netcat command.\n" - return 1 - fi - - printf "Waiting for %s on %s:%s ... " "${name}" "${host}" "${port}" - - sleep 30 +SCRIPT_DIR="$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd -P)" - i=0 - while ! ${netcat} -z "${host}" "${port}"; do - sleep 1 - if [ "$i" -gt "$timeout" ]; then - printf "Timed out!\n" - docker ps -a - echo "::group::Netdata container logs" - docker logs netdata 2>&1 - echo "::endgroup::" - return 1 - fi - i="$((i + 1))" - done - printf "OK\n" -} +export DEBIAN_FRONTEND=noninteractive if [ -z "$(command -v nc 2>/dev/null)" ] && [ -z "$(command -v netcat 2>/dev/null)" ]; then sudo apt-get update && sudo apt-get upgrade -y && sudo apt-get install -y netcat @@ -55,10 +22,9 @@ docker run -d --name=netdata \ --security-opt apparmor=unconfined \ netdata/netdata:test -wait_for localhost 19999 netdata || exit 1 - -curl -sS http://127.0.0.1:19999/api/v1/info > ./response || exit 1 - -cat ./response - -jq '.version' ./response || exit 1 +if ! "${SCRIPT_DIR}/../../packaging/runtime-check.sh"; then + docker ps -a + echo "::group::Netdata container logs" + docker logs netdata 2>&1 + echo "::endgroup::" +fi diff --git a/.github/scripts/gen-docker-imagetool-args.py b/.github/scripts/gen-docker-imagetool-args.py new file mode 100755 index 000000000..c0eaa1cfc --- /dev/null +++ b/.github/scripts/gen-docker-imagetool-args.py @@ -0,0 +1,27 @@ +#!/usr/bin/env python3 + +import sys + +from pathlib import Path + +DIGEST_PATH = Path(sys.argv[1]) +TAG_PREFIX = sys.argv[2] +TAGS = sys.argv[3] + +if TAG_PREFIX: + PUSH_TAGS = tuple([ + t for t in TAGS.split(',') if t.startswith(TAG_PREFIX) + ]) +else: + PUSH_TAGS = tuple([ + t for t in TAGS.split(',') if t.startswith('netdata/') + ]) + +IMAGE_NAME = PUSH_TAGS[0].split(':')[0] + +images = [] + +for f in DIGEST_PATH.glob('*'): + images.append(f'{IMAGE_NAME}@sha256:{f.name}') + +print(f'-t {" -t ".join(PUSH_TAGS)} {" ".join(images)}') diff --git a/.github/scripts/gen-docker-tags.py b/.github/scripts/gen-docker-tags.py index 8c88d3b5e..c45b991d9 100755 --- a/.github/scripts/gen-docker-tags.py +++ b/.github/scripts/gen-docker-tags.py @@ -2,18 +2,33 @@ import sys -version = sys.argv[1].split('.') -suffix = sys.argv[2] +github_event = sys.argv[1] +version = sys.argv[2] -REPO = f'netdata/netdata{suffix}' -GHCR = f'ghcr.io/{REPO}' -QUAY = f'quay.io/{REPO}' +REPO = 'netdata/netdata' -tags = [] +REPOS = ( + REPO, + f'quay.io/{REPO}', + f'ghcr.io/{REPO}', +) -for repo in [REPO, GHCR, QUAY]: - tags.append(':'.join([repo, version[0]])) - tags.append(':'.join([repo, '.'.join(version[0:2])])) - tags.append(':'.join([repo, '.'.join(version[0:3])])) +match version: + case '': + tags = (f'{REPO}:test',) + case 'nightly': + tags = tuple([ + f'{r}:{t}' for r in REPOS for t in ('edge', 'latest') + ]) + case _: + v = f'v{version}'.split('.') + + tags = tuple([ + f'{r}:{t}' for r in REPOS for t in ( + v[0], + '.'.join(v[0:2]), + '.'.join(v[0:3]), + ) + ]) print(','.join(tags)) diff --git a/.github/scripts/gen-matrix-build.py b/.github/scripts/gen-matrix-build.py index 3185e8836..9a70a44ce 100755 --- a/.github/scripts/gen-matrix-build.py +++ b/.github/scripts/gen-matrix-build.py @@ -11,6 +11,9 @@ with open('.github/data/distros.yml') as f: data = yaml.load(f) for i, v in enumerate(data['include']): + if v['test'].get('skip-local-build', False): + continue + e = { 'artifact_key': v['distro'] + str(v['version']).replace('.', ''), 'version': v['version'], diff --git a/.github/scripts/gen-matrix-packaging.py b/.github/scripts/gen-matrix-packaging.py index 9347cd767..b695e33ab 100755 --- a/.github/scripts/gen-matrix-packaging.py +++ b/.github/scripts/gen-matrix-packaging.py @@ -28,6 +28,7 @@ for i, v in enumerate(data['include']): 'format': data['include'][i]['packages']['type'], 'base_image': data['include'][i]['base_image'] if 'base_image' in data['include'][i] else ':'.join([data['include'][i]['distro'], data['include'][i]['version']]), 'platform': data['platform_map'][arch], + 'bundle_sentry': data['include'][i]['bundle_sentry'], 'arch': arch }) diff --git a/.github/scripts/get-go-version.py b/.github/scripts/get-go-version.py new file mode 100755 index 000000000..105c537c8 --- /dev/null +++ b/.github/scripts/get-go-version.py @@ -0,0 +1,39 @@ +#!/usr/bin/env python3 + +import json +import os +import pathlib + +from packaging.version import parse + +SCRIPT_PATH = pathlib.Path(__file__).parents[0] +REPO_ROOT = SCRIPT_PATH.parents[1] +GO_SRC = REPO_ROOT / 'src' / 'go' + +GITHUB_OUTPUT = pathlib.Path(os.environ['GITHUB_OUTPUT']) + +version = parse('1.0.0') +modules = [] + +for modfile in GO_SRC.glob('**/go.mod'): + moddata = modfile.read_text() + + for line in moddata.splitlines(): + if line.startswith('go '): + version = max(version, parse(line.split()[1])) + break + + for main in modfile.parent.glob('**/main.go'): + mainpath = main.relative_to(modfile.parent).parent + + if 'examples' in mainpath.parts: + continue + + modules.append({ + 'module': str(modfile.parent), + 'version': str(version), + 'build_target': f'github.com/netdata/netdata/go/{ modfile.parts[-2] }/{ str(mainpath) }/', + }) + +with GITHUB_OUTPUT.open('a') as f: + f.write(f'matrix={ json.dumps({"include": modules}) }\n') diff --git a/.github/scripts/get-static-cache-key.sh b/.github/scripts/get-static-cache-key.sh index 5093b3327..e45ae30bd 100755 --- a/.github/scripts/get-static-cache-key.sh +++ b/.github/scripts/get-static-cache-key.sh @@ -8,6 +8,7 @@ docker pull --platform "${platform}" netdata/static-builder:${builder_rev} # shellcheck disable=SC2046 cat $(find packaging/makeself/jobs -type f ! -regex '.*\(netdata\|-makeself\).*') > /tmp/static-cache-key-data +cat packaging/makeself/bundled-packages.version >> /tmp/static-cache-key-data docker run -it --rm --platform "${platform}" netdata/static-builder:${builder_rev} sh -c 'apk list -I 2>/dev/null' >> /tmp/static-cache-key-data diff --git a/.github/scripts/pkg-test.sh b/.github/scripts/pkg-test.sh index 35767bf2e..6120d3fc3 100755 --- a/.github/scripts/pkg-test.sh +++ b/.github/scripts/pkg-test.sh @@ -1,5 +1,7 @@ #!/bin/sh +SCRIPT_DIR="$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd -P)" + install_debian_like() { # This is needed to ensure package installs don't prompt for any user input. export DEBIAN_FRONTEND=noninteractive @@ -94,37 +96,6 @@ dump_log() { cat ./netdata.log } -wait_for() { - host="${1}" - port="${2}" - name="${3}" - timeout="30" - - if command -v nc > /dev/null ; then - netcat="nc" - elif command -v netcat > /dev/null ; then - netcat="netcat" - else - printf "Unable to find a usable netcat command.\n" - return 1 - fi - - printf "Waiting for %s on %s:%s ... " "${name}" "${host}" "${port}" - - sleep 30 - - i=0 - while ! ${netcat} -z "${host}" "${port}"; do - sleep 1 - if [ "$i" -gt "$timeout" ]; then - printf "Timed out!\n" - return 1 - fi - i="$((i + 1))" - done - printf "OK\n" -} - case "${DISTRO}" in debian | ubuntu) install_debian_like @@ -151,12 +122,6 @@ trap dump_log EXIT /usr/sbin/netdata -D > ./netdata.log 2>&1 & -wait_for localhost 19999 netdata || exit 1 - -curl -sS http://127.0.0.1:19999/api/v1/info > ./response || exit 1 - -cat ./response - -jq '.version' ./response || exit 1 +"${SCRIPT_DIR}/../../packaging/runtime-check.sh" || exit 1 trap - EXIT diff --git a/.github/scripts/run-updater-check.sh b/.github/scripts/run-updater-check.sh index 1224d8f67..2e70a10af 100755 --- a/.github/scripts/run-updater-check.sh +++ b/.github/scripts/run-updater-check.sh @@ -12,7 +12,8 @@ echo "::group::>>> Pre-Update Netdata Build Info" netdata -W buildinfo echo "::endgroup::" echo ">>> Updating Netdata..." -export NETDATA_BASE_URL="http://localhost:8080/artifacts/" # Pull the tarball from the local web server. +export NETDATA_BASE_URL="http://localhost:8080/artifacts" # Pull the tarball from the local web server. +echo 'NETDATA_ACCEPT_MAJOR_VERSIONS="1 9999"' > /etc/netdata/netdata-updater.conf timeout 3600 /netdata/packaging/installer/netdata-updater.sh --not-running-from-cron --no-updater-self-update case "$?" in diff --git a/.github/scripts/run_install_with_dist_file.sh b/.github/scripts/run_install_with_dist_file.sh deleted file mode 100755 index 74652efdd..000000000 --- a/.github/scripts/run_install_with_dist_file.sh +++ /dev/null @@ -1,39 +0,0 @@ -#!/usr/bin/env bash -# -# This script is evaluating netdata installation with the source from make dist -# -# Copyright: SPDX-License-Identifier: GPL-3.0-or-later -# -# Author : Pavlos Emm. Katsoulakis &2 "Usage: %s \n" "$(basename "$0")" - exit 1 -fi - -distfile="${1}" -shift - -printf >&2 "Opening dist archive %s ... " "${distfile}" -tar -xovf "${distfile}" -distdir="$(echo "${distfile}" | rev | cut -d. -f3- | rev)" -cp -a packaging/installer/install-required-packages.sh "${distdir}/install-required-packages.sh" -if [ ! -d "${distdir}" ]; then - printf >&2 "ERROR: %s is not a directory" "${distdir}" - exit 2 -fi - -printf >&2 "Entering %s and starting docker run ..." "${distdir}" - -pushd "${distdir}" || exit 1 -docker run \ - -e DISABLE_TELEMETRY=1 \ - -v "${PWD}:/netdata" \ - -w /netdata \ - "ubuntu:latest" \ - /bin/bash -c "./install-required-packages.sh --dont-wait --non-interactive netdata && apt install wget && ./netdata-installer.sh --dont-wait --require-cloud --disable-telemetry --install-prefix /tmp --one-time-build && echo \"Validating netdata instance is running\" && wget -O - 'http://127.0.0.1:19999/api/v1/info' | grep version" -popd || exit 1 - -echo "All Done!" -- cgit v1.2.3