summaryrefslogtreecommitdiffstats
path: root/.github/scripts
diff options
context:
space:
mode:
Diffstat (limited to '.github/scripts')
-rwxr-xr-x.github/scripts/build-artifacts.sh82
-rwxr-xr-x.github/scripts/build-dist.sh71
-rwxr-xr-x.github/scripts/build-static.sh2
-rwxr-xr-x.github/scripts/docker-test.sh50
-rwxr-xr-x.github/scripts/gen-docker-imagetool-args.py27
-rwxr-xr-x.github/scripts/gen-docker-tags.py37
-rwxr-xr-x.github/scripts/gen-matrix-build.py3
-rwxr-xr-x.github/scripts/gen-matrix-packaging.py2
-rwxr-xr-x.github/scripts/get-go-version.py39
-rwxr-xr-x.github/scripts/get-static-cache-key.sh1
-rwxr-xr-x.github/scripts/package-upload.sh13
-rwxr-xr-x.github/scripts/pkg-test.sh60
-rwxr-xr-x.github/scripts/run-updater-check.sh5
-rwxr-xr-x.github/scripts/run_install_with_dist_file.sh39
-rwxr-xr-x.github/scripts/upload-new-version-tags.sh3
15 files changed, 141 insertions, 293 deletions
diff --git a/.github/scripts/build-artifacts.sh b/.github/scripts/build-artifacts.sh
deleted file mode 100755
index 569c79a5a..000000000
--- a/.github/scripts/build-artifacts.sh
+++ /dev/null
@@ -1,82 +0,0 @@
-#!/bin/sh
-#
-# Builds the netdata-vX.y.Z-xxxx.tar.gz source tarball (dist)
-# and netdata-vX.Y.Z-xxxx.gz.run (static x86_64) artifacts.
-
-set -e
-
-# shellcheck source=.github/scripts/functions.sh
-. "$(dirname "$0")/functions.sh"
-
-NAME="${NAME:-netdata}"
-VERSION="${VERSION:-"$(git describe)"}"
-BASENAME="$NAME-$VERSION"
-
-prepare_build() {
- progress "Preparing build"
- (
- test -d artifacts || mkdir -p artifacts
- echo "${VERSION}" > packaging/version
- ) >&2
-}
-
-build_dist() {
- progress "Building dist"
- (
- command -v git > /dev/null && [ -d .git ] && git clean -d -f
- autoreconf -ivf
- ./configure \
- --prefix=/usr \
- --sysconfdir=/etc \
- --localstatedir=/var \
- --libexecdir=/usr/libexec \
- --with-zlib \
- --with-math \
- --with-user=netdata \
- --disable-dependency-tracking \
- CFLAGS=-O2
- make dist
- mv "${BASENAME}.tar.gz" artifacts/
- ) >&2
-}
-
-build_static_x86_64() {
- progress "Building static x86_64"
- (
- command -v git > /dev/null && [ -d .git ] && git clean -d -f
- USER="" ./packaging/makeself/build-x86_64-static.sh
- ) >&2
-}
-
-prepare_assets() {
- progress "Preparing assets"
- (
- cp packaging/version artifacts/latest-version.txt
-
- cd artifacts || exit 1
- ln -f "${BASENAME}.tar.gz" netdata-latest.tar.gz
- ln -f "${BASENAME}.gz.run" netdata-latest.gz.run
- sha256sum -b ./* > "sha256sums.txt"
- ) >&2
-}
-
-steps="prepare_build build_dist build_static_x86_64"
-steps="$steps prepare_assets"
-
-_main() {
- for step in $steps; do
- if ! run "$step"; then
- if [ -t 1 ]; then
- debug
- else
- fail "Build failed"
- fi
- fi
- done
-
- echo "🎉 All Done!"
-}
-
-if [ -n "$0" ] && [ x"$0" != x"-bash" ]; then
- _main "$@"
-fi
diff --git a/.github/scripts/build-dist.sh b/.github/scripts/build-dist.sh
deleted file mode 100755
index 027b62147..000000000
--- a/.github/scripts/build-dist.sh
+++ /dev/null
@@ -1,71 +0,0 @@
-#!/bin/sh
-#
-# Builds the netdata-vX.y.Z-xxxx.tar.gz source tarball (dist)
-
-set -e
-
-# shellcheck source=.github/scripts/functions.sh
-. "$(dirname "$0")/functions.sh"
-
-NAME="${NAME:-netdata}"
-VERSION="${VERSION:-"$(git describe --always)"}"
-BASENAME="$NAME-$VERSION"
-
-prepare_build() {
- progress "Preparing build"
- (
- test -d artifacts || mkdir -p artifacts
- echo "${VERSION}" > packaging/version
- ) >&2
-}
-
-build_dist() {
- progress "Building dist"
- (
- command -v git > /dev/null && [ -d .git ] && git clean -d -f
- autoreconf -ivf
- ./configure \
- --prefix=/usr \
- --sysconfdir=/etc \
- --localstatedir=/var \
- --libexecdir=/usr/libexec \
- --with-zlib \
- --with-math \
- --with-user=netdata \
- --disable-dependency-tracking \
- CFLAGS=-O2
- make dist
- mv "${BASENAME}.tar.gz" artifacts/
- ) >&2
-}
-
-prepare_assets() {
- progress "Preparing assets"
- (
- cp packaging/version artifacts/latest-version.txt
- cd artifacts || exit 1
- ln -f "${BASENAME}.tar.gz" netdata-latest.tar.gz
- ln -f "${BASENAME}.gz.run" netdata-latest.gz.run
- sha256sum -b ./* > "sha256sums.txt"
- ) >&2
-}
-
-steps="prepare_build build_dist prepare_assets"
-
-_main() {
- for step in $steps; do
- if ! run "$step"; then
- if [ -t 1 ]; then
- debug
- else
- fail "Build failed"
- fi
- fi
- done
-
- echo "🎉 All Done!"
-}
-
-if [ -n "$0" ] && [ x"$0" != x"-bash" ]; then
- _main "$@"
-fi
diff --git a/.github/scripts/build-static.sh b/.github/scripts/build-static.sh
index e81051438..9b29a3d99 100755
--- a/.github/scripts/build-static.sh
+++ b/.github/scripts/build-static.sh
@@ -22,7 +22,7 @@ prepare_build() {
build_static() {
progress "Building static ${BUILDARCH}"
(
- USER="" ./packaging/makeself/build-static.sh "${BUILDARCH}"
+ EXTRA_INSTALL_FLAGS="${EXTRA_INSTALL_FLAGS}" USER="" ./packaging/makeself/build-static.sh "${BUILDARCH}"
) >&2
}
diff --git a/.github/scripts/docker-test.sh b/.github/scripts/docker-test.sh
index 0f5fa469c..0ef064501 100755
--- a/.github/scripts/docker-test.sh
+++ b/.github/scripts/docker-test.sh
@@ -1,41 +1,8 @@
#!/bin/sh
-export DEBIAN_FRONTEND=noninteractive
-
-wait_for() {
- host="${1}"
- port="${2}"
- name="${3}"
- timeout="30"
-
- if command -v nc > /dev/null ; then
- netcat="nc"
- elif command -v netcat > /dev/null ; then
- netcat="netcat"
- else
- printf "Unable to find a usable netcat command.\n"
- return 1
- fi
-
- printf "Waiting for %s on %s:%s ... " "${name}" "${host}" "${port}"
-
- sleep 30
+SCRIPT_DIR="$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd -P)"
- i=0
- while ! ${netcat} -z "${host}" "${port}"; do
- sleep 1
- if [ "$i" -gt "$timeout" ]; then
- printf "Timed out!\n"
- docker ps -a
- echo "::group::Netdata container logs"
- docker logs netdata 2>&1
- echo "::endgroup::"
- return 1
- fi
- i="$((i + 1))"
- done
- printf "OK\n"
-}
+export DEBIAN_FRONTEND=noninteractive
if [ -z "$(command -v nc 2>/dev/null)" ] && [ -z "$(command -v netcat 2>/dev/null)" ]; then
sudo apt-get update && sudo apt-get upgrade -y && sudo apt-get install -y netcat
@@ -55,10 +22,9 @@ docker run -d --name=netdata \
--security-opt apparmor=unconfined \
netdata/netdata:test
-wait_for localhost 19999 netdata || exit 1
-
-curl -sS http://127.0.0.1:19999/api/v1/info > ./response || exit 1
-
-cat ./response
-
-jq '.version' ./response || exit 1
+if ! "${SCRIPT_DIR}/../../packaging/runtime-check.sh"; then
+ docker ps -a
+ echo "::group::Netdata container logs"
+ docker logs netdata 2>&1
+ echo "::endgroup::"
+fi
diff --git a/.github/scripts/gen-docker-imagetool-args.py b/.github/scripts/gen-docker-imagetool-args.py
new file mode 100755
index 000000000..c0eaa1cfc
--- /dev/null
+++ b/.github/scripts/gen-docker-imagetool-args.py
@@ -0,0 +1,27 @@
+#!/usr/bin/env python3
+
+import sys
+
+from pathlib import Path
+
+DIGEST_PATH = Path(sys.argv[1])
+TAG_PREFIX = sys.argv[2]
+TAGS = sys.argv[3]
+
+if TAG_PREFIX:
+ PUSH_TAGS = tuple([
+ t for t in TAGS.split(',') if t.startswith(TAG_PREFIX)
+ ])
+else:
+ PUSH_TAGS = tuple([
+ t for t in TAGS.split(',') if t.startswith('netdata/')
+ ])
+
+IMAGE_NAME = PUSH_TAGS[0].split(':')[0]
+
+images = []
+
+for f in DIGEST_PATH.glob('*'):
+ images.append(f'{IMAGE_NAME}@sha256:{f.name}')
+
+print(f'-t {" -t ".join(PUSH_TAGS)} {" ".join(images)}')
diff --git a/.github/scripts/gen-docker-tags.py b/.github/scripts/gen-docker-tags.py
index 8c88d3b5e..f60e1cd41 100755
--- a/.github/scripts/gen-docker-tags.py
+++ b/.github/scripts/gen-docker-tags.py
@@ -2,18 +2,35 @@
import sys
-version = sys.argv[1].split('.')
-suffix = sys.argv[2]
+github_event = sys.argv[1]
+version = sys.argv[2]
-REPO = f'netdata/netdata{suffix}'
-GHCR = f'ghcr.io/{REPO}'
-QUAY = f'quay.io/{REPO}'
+REPO = 'netdata/netdata'
-tags = []
+REPOS = (
+ REPO,
+ f'quay.io/{REPO}',
+ f'ghcr.io/{REPO}',
+)
-for repo in [REPO, GHCR, QUAY]:
- tags.append(':'.join([repo, version[0]]))
- tags.append(':'.join([repo, '.'.join(version[0:2])]))
- tags.append(':'.join([repo, '.'.join(version[0:3])]))
+match version:
+ case '':
+ tags = (f'{REPO}:test',)
+ case 'nightly':
+ tags = tuple([
+ f'{r}:{t}' for r in REPOS for t in ('edge', 'latest')
+ ])
+ case _:
+ v = f'v{version}'.split('.')
+
+ tags = tuple([
+ f'{r}:{t}' for r in REPOS for t in (
+ v[0],
+ '.'.join(v[0:2]),
+ '.'.join(v[0:3]),
+ )
+ ])
+
+ tags = tags + tuple([f'{r}:stable' for r in REPOS])
print(','.join(tags))
diff --git a/.github/scripts/gen-matrix-build.py b/.github/scripts/gen-matrix-build.py
index 3185e8836..9a70a44ce 100755
--- a/.github/scripts/gen-matrix-build.py
+++ b/.github/scripts/gen-matrix-build.py
@@ -11,6 +11,9 @@ with open('.github/data/distros.yml') as f:
data = yaml.load(f)
for i, v in enumerate(data['include']):
+ if v['test'].get('skip-local-build', False):
+ continue
+
e = {
'artifact_key': v['distro'] + str(v['version']).replace('.', ''),
'version': v['version'],
diff --git a/.github/scripts/gen-matrix-packaging.py b/.github/scripts/gen-matrix-packaging.py
index 9347cd767..c00ff3181 100755
--- a/.github/scripts/gen-matrix-packaging.py
+++ b/.github/scripts/gen-matrix-packaging.py
@@ -27,7 +27,9 @@ for i, v in enumerate(data['include']):
'repo_distro': data['include'][i]['packages']['repo_distro'],
'format': data['include'][i]['packages']['type'],
'base_image': data['include'][i]['base_image'] if 'base_image' in data['include'][i] else ':'.join([data['include'][i]['distro'], data['include'][i]['version']]),
+ 'builder_rev': data['include'][i]['packages']['builder_rev'],
'platform': data['platform_map'][arch],
+ 'bundle_sentry': data['include'][i]['bundle_sentry'][arch],
'arch': arch
})
diff --git a/.github/scripts/get-go-version.py b/.github/scripts/get-go-version.py
new file mode 100755
index 000000000..105c537c8
--- /dev/null
+++ b/.github/scripts/get-go-version.py
@@ -0,0 +1,39 @@
+#!/usr/bin/env python3
+
+import json
+import os
+import pathlib
+
+from packaging.version import parse
+
+SCRIPT_PATH = pathlib.Path(__file__).parents[0]
+REPO_ROOT = SCRIPT_PATH.parents[1]
+GO_SRC = REPO_ROOT / 'src' / 'go'
+
+GITHUB_OUTPUT = pathlib.Path(os.environ['GITHUB_OUTPUT'])
+
+version = parse('1.0.0')
+modules = []
+
+for modfile in GO_SRC.glob('**/go.mod'):
+ moddata = modfile.read_text()
+
+ for line in moddata.splitlines():
+ if line.startswith('go '):
+ version = max(version, parse(line.split()[1]))
+ break
+
+ for main in modfile.parent.glob('**/main.go'):
+ mainpath = main.relative_to(modfile.parent).parent
+
+ if 'examples' in mainpath.parts:
+ continue
+
+ modules.append({
+ 'module': str(modfile.parent),
+ 'version': str(version),
+ 'build_target': f'github.com/netdata/netdata/go/{ modfile.parts[-2] }/{ str(mainpath) }/',
+ })
+
+with GITHUB_OUTPUT.open('a') as f:
+ f.write(f'matrix={ json.dumps({"include": modules}) }\n')
diff --git a/.github/scripts/get-static-cache-key.sh b/.github/scripts/get-static-cache-key.sh
index 5093b3327..e45ae30bd 100755
--- a/.github/scripts/get-static-cache-key.sh
+++ b/.github/scripts/get-static-cache-key.sh
@@ -8,6 +8,7 @@ docker pull --platform "${platform}" netdata/static-builder:${builder_rev}
# shellcheck disable=SC2046
cat $(find packaging/makeself/jobs -type f ! -regex '.*\(netdata\|-makeself\).*') > /tmp/static-cache-key-data
+cat packaging/makeself/bundled-packages.version >> /tmp/static-cache-key-data
docker run -it --rm --platform "${platform}" netdata/static-builder:${builder_rev} sh -c 'apk list -I 2>/dev/null' >> /tmp/static-cache-key-data
diff --git a/.github/scripts/package-upload.sh b/.github/scripts/package-upload.sh
index 13d63b4a7..9305ffa67 100755
--- a/.github/scripts/package-upload.sh
+++ b/.github/scripts/package-upload.sh
@@ -2,18 +2,19 @@
set -e
-host="packages.netdata.cloud"
user="netdatabot"
-distro="${1}"
-arch="${2}"
-format="${3}"
-repo="${4}"
+host="${1}"
+distro="${2}"
+arch="${3}"
+format="${4}"
+repo="${5}"
+pkg_src="${6:-./artifacts}"
staging="${TMPDIR:-/tmp}/package-staging"
prefix="/home/netdatabot/incoming/${repo}/"
-packages="$(find artifacts -name "*.${format}")"
+packages="$(find "${pkg_src}" -name "*.${format}")"
mkdir -p "${staging}"
diff --git a/.github/scripts/pkg-test.sh b/.github/scripts/pkg-test.sh
index 35767bf2e..f0c0dc11a 100755
--- a/.github/scripts/pkg-test.sh
+++ b/.github/scripts/pkg-test.sh
@@ -1,5 +1,7 @@
#!/bin/sh
+SCRIPT_DIR="$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd -P)"
+
install_debian_like() {
# This is needed to ensure package installs don't prompt for any user input.
export DEBIAN_FRONTEND=noninteractive
@@ -19,7 +21,9 @@ install_debian_like() {
! -name '*dbgsym*' ! -name '*cups*' ! -name '*freeipmi*') || exit 3
# Install testing tools
- apt-get install -y --no-install-recommends curl "${netcat}" jq || exit 1
+ apt-get install -y --no-install-recommends curl dpkg-dev "${netcat}" jq || exit 1
+
+ dpkg-architecture --equal amd64 || NETDATA_SKIP_EBPF=1
}
install_fedora_like() {
@@ -38,6 +42,8 @@ install_fedora_like() {
# Install testing tools
"${PKGMGR}" install -y curl nc jq || exit 1
+
+ [ "$(rpm --eval '%{_build_arch}')" = "x86_64" ] || NETDATA_SKIP_EBPF=1
}
install_centos() {
@@ -60,6 +66,8 @@ install_centos() {
# Install testing tools
# shellcheck disable=SC2086
"${PKGMGR}" install -y ${opts} curl nc jq || exit 1
+
+ [ "$(rpm --eval '%{_build_arch}')" = "x86_64" ] || NETDATA_SKIP_EBPF=1
}
install_amazon_linux() {
@@ -76,6 +84,8 @@ install_amazon_linux() {
# Install testing tools
# shellcheck disable=SC2086
"${PKGMGR}" install -y ${opts} curl nc jq || exit 1
+
+ [ "$(rpm --eval '%{_build_arch}')" = "x86_64" ] || NETDATA_SKIP_EBPF=1
}
install_suse_like() {
@@ -88,43 +98,14 @@ install_suse_like() {
# Install testing tools
zypper install -y --allow-downgrade --no-recommends curl netcat-openbsd jq || exit 1
+
+ [ "$(rpm --eval '%{_build_arch}')" = "x86_64" ] || NETDATA_SKIP_EBPF=1
}
dump_log() {
cat ./netdata.log
}
-wait_for() {
- host="${1}"
- port="${2}"
- name="${3}"
- timeout="30"
-
- if command -v nc > /dev/null ; then
- netcat="nc"
- elif command -v netcat > /dev/null ; then
- netcat="netcat"
- else
- printf "Unable to find a usable netcat command.\n"
- return 1
- fi
-
- printf "Waiting for %s on %s:%s ... " "${name}" "${host}" "${port}"
-
- sleep 30
-
- i=0
- while ! ${netcat} -z "${host}" "${port}"; do
- sleep 1
- if [ "$i" -gt "$timeout" ]; then
- printf "Timed out!\n"
- return 1
- fi
- i="$((i + 1))"
- done
- printf "OK\n"
-}
-
case "${DISTRO}" in
debian | ubuntu)
install_debian_like
@@ -132,7 +113,7 @@ case "${DISTRO}" in
fedora | oraclelinux)
install_fedora_like
;;
- centos| centos-stream | rockylinux | almalinux)
+ centos | centos-stream | rockylinux | almalinux)
install_centos
;;
amazonlinux)
@@ -149,14 +130,15 @@ esac
trap dump_log EXIT
-/usr/sbin/netdata -D > ./netdata.log 2>&1 &
-
-wait_for localhost 19999 netdata || exit 1
+export NETDATA_LIBEXEC_PREFIX=/usr/libexec/netdata
+export NETDATA_SKIP_LIBEXEC_PARTS="logs-management|freeipmi|xenstat|nfacct|cups"
-curl -sS http://127.0.0.1:19999/api/v1/info > ./response || exit 1
+if [ -n "${NETDATA_SKIP_EBPF}" ]; then
+ export NETDATA_SKIP_LIBEXEC_PARTS="${NETDATA_SKIP_LIBEXEC_PARTS}|ebpf"
+fi
-cat ./response
+/usr/sbin/netdata -D > ./netdata.log 2>&1 &
-jq '.version' ./response || exit 1
+"${SCRIPT_DIR}/../../packaging/runtime-check.sh" || exit 1
trap - EXIT
diff --git a/.github/scripts/run-updater-check.sh b/.github/scripts/run-updater-check.sh
index 1224d8f67..456a0e5d0 100755
--- a/.github/scripts/run-updater-check.sh
+++ b/.github/scripts/run-updater-check.sh
@@ -4,7 +4,7 @@ echo ">>> Installing CI support packages..."
/netdata/.github/scripts/ci-support-pkgs.sh
mkdir -p /etc/cron.daily # Needed to make auto-update checking work correctly on some platforms.
echo ">>> Installing Netdata..."
-/netdata/packaging/installer/kickstart.sh --dont-wait --build-only --disable-telemetry || exit 1
+/netdata/packaging/installer/kickstart.sh --dont-wait --build-only --dont-start-it --disable-telemetry "${EXTRA_INSTALL_FLAGS:+--local-build-options "${EXTRA_INSTALL_FLAGS}"}" || exit 1
echo "::group::>>> Pre-Update Environment File Contents"
cat /etc/netdata/.environment
echo "::endgroup::"
@@ -12,7 +12,8 @@ echo "::group::>>> Pre-Update Netdata Build Info"
netdata -W buildinfo
echo "::endgroup::"
echo ">>> Updating Netdata..."
-export NETDATA_BASE_URL="http://localhost:8080/artifacts/" # Pull the tarball from the local web server.
+export NETDATA_BASE_URL="http://localhost:8080/artifacts" # Pull the tarball from the local web server.
+echo 'NETDATA_ACCEPT_MAJOR_VERSIONS="1 9999"' > /etc/netdata/netdata-updater.conf
timeout 3600 /netdata/packaging/installer/netdata-updater.sh --not-running-from-cron --no-updater-self-update
case "$?" in
diff --git a/.github/scripts/run_install_with_dist_file.sh b/.github/scripts/run_install_with_dist_file.sh
deleted file mode 100755
index 74652efdd..000000000
--- a/.github/scripts/run_install_with_dist_file.sh
+++ /dev/null
@@ -1,39 +0,0 @@
-#!/usr/bin/env bash
-#
-# This script is evaluating netdata installation with the source from make dist
-#
-# Copyright: SPDX-License-Identifier: GPL-3.0-or-later
-#
-# Author : Pavlos Emm. Katsoulakis <paul@netdata.cloud)
-
-set -e
-
-if [ $# -ne 1 ]; then
- printf >&2 "Usage: %s <dist_file>\n" "$(basename "$0")"
- exit 1
-fi
-
-distfile="${1}"
-shift
-
-printf >&2 "Opening dist archive %s ... " "${distfile}"
-tar -xovf "${distfile}"
-distdir="$(echo "${distfile}" | rev | cut -d. -f3- | rev)"
-cp -a packaging/installer/install-required-packages.sh "${distdir}/install-required-packages.sh"
-if [ ! -d "${distdir}" ]; then
- printf >&2 "ERROR: %s is not a directory" "${distdir}"
- exit 2
-fi
-
-printf >&2 "Entering %s and starting docker run ..." "${distdir}"
-
-pushd "${distdir}" || exit 1
-docker run \
- -e DISABLE_TELEMETRY=1 \
- -v "${PWD}:/netdata" \
- -w /netdata \
- "ubuntu:latest" \
- /bin/bash -c "./install-required-packages.sh --dont-wait --non-interactive netdata && apt install wget && ./netdata-installer.sh --dont-wait --require-cloud --disable-telemetry --install-prefix /tmp --one-time-build && echo \"Validating netdata instance is running\" && wget -O - 'http://127.0.0.1:19999/api/v1/info' | grep version"
-popd || exit 1
-
-echo "All Done!"
diff --git a/.github/scripts/upload-new-version-tags.sh b/.github/scripts/upload-new-version-tags.sh
index a9b0cd303..ffdfadfa0 100755
--- a/.github/scripts/upload-new-version-tags.sh
+++ b/.github/scripts/upload-new-version-tags.sh
@@ -2,9 +2,10 @@
set -e
-host="packages.netdata.cloud"
user="netdatabot"
+host="${1}"
+
prefix="/var/www/html/releases"
staging="${TMPDIR:-/tmp}/staging-new-releases"
generated by cgit v1.2.3 (git 2.39.1) at 2024-12-28 04:55:20 +0000