Merging upstream version 1.35.0.

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

9 files changed, 213 insertions, 139 deletions

diff --git a/packaging/makeself/build-static.sh b/packaging/makeself/build-static.sh
index f38353721..c7c5a4bcc 100755
--- a/packaging/makeself/build-static.sh
+++ b/packaging/makeself/build-static.sh
@@ -9,57 +9,36 @@ BUILDARCH="${1}"
 
 set -e
 
-case ${BUILDARCH} in
-  x86_64) platform=linux/amd64 ;;
-  armv7l) platform=linux/arm/v7 ;;
-  aarch64) platform=linux/arm64/v8 ;;
-  ppc64le) platform=linux/ppc64le ;;
-  *)
-    echo "Unknown target architecture '${BUILDARCH}'."
+platform="$("$(dirname "${0}")/uname2platform.sh" "${BUILDARCH}")"
+
+if [ -z "${platform}" ]; then
     exit 1
-    ;;
-esac
+fi
 
-DOCKER_CONTAINER_NAME="netdata-package-${BUILDARCH}-static-alpine315"
+DOCKER_IMAGE_NAME="netdata/static-builder"
 
 if [ "${BUILDARCH}" != "$(uname -m)" ] && [ "$(uname -m)" = 'x86_64' ] && [ -z "${SKIP_EMULATION}" ]; then
     docker run --rm --privileged multiarch/qemu-user-static --reset -p yes || exit 1
 fi
 
-if ! docker inspect "${DOCKER_CONTAINER_NAME}" > /dev/null 2>&1; then
-  # To run interactively:
-  #   docker run -it netdata-package-x86_64-static /bin/sh
-  # (add -v host-dir:guest-dir:rw arguments to mount volumes)
-  #
-  # To remove images in order to re-create:
-  #   docker rm -v $(sudo docker ps -a -q -f status=exited)
-  #   docker rmi netdata-package-x86_64-static
-  #
-  # This command maps the current directory to
-  #   /usr/src/netdata.git
-  # inside the container and runs the script install-alpine-packages.sh
-  # (also inside the container)
-  #
-  if docker inspect alpine:3.15 > /dev/null 2>&1; then
-    run docker image remove alpine:3.15
-    run docker pull --platform=${platform}  alpine:3.15
-  fi
-
-  run docker run --platform=${platform} -v "$(pwd)":/netdata:rw alpine:3.15 \
-    /bin/sh /netdata/packaging/makeself/install-alpine-packages.sh
+if docker inspect "${DOCKER_IMAGE_NAME}" > /dev/null 2>&1; then
+    img_platform="$(docker image inspect netdata/static-builder --format '{{.Os}}/{{.Architecture}}/{{.Variant}}')"
+    if [ "${img_platform%'/'}" != "${platform}" ]; then
+        docker image rm "${DOCKER_IMAGE_NAME}" || exit 1
+    fi
+fi
 
-  # save the changes made permanently
-  id=$(docker ps -l -q)
-  run docker commit "${id}" "${DOCKER_CONTAINER_NAME}"
+if ! docker inspect "${DOCKER_IMAGE_NAME}" > /dev/null 2>&1; then
+    docker pull --platform "${platform}" "${DOCKER_IMAGE_NAME}"
 fi
 
 # Run the build script inside the container
 if [ -t 1 ]; then
-  run docker run -e BUILDARCH="${BUILDARCH}" -a stdin -a stdout -a stderr -i -t -v "$(pwd)":/netdata:rw \
-    "${DOCKER_CONTAINER_NAME}" \
+  run docker run --rm -e BUILDARCH="${BUILDARCH}" -a stdin -a stdout -a stderr -i -t -v "$(pwd)":/netdata:rw \
+    "${DOCKER_IMAGE_NAME}" \
     /bin/sh /netdata/packaging/makeself/build.sh "${@}"
 else
-  run docker run -e BUILDARCH="${BUILDARCH}" -v "$(pwd)":/netdata:rw \
-    -e GITHUB_ACTIONS="${GITHUB_ACTIONS}" "${DOCKER_CONTAINER_NAME}" \
+  run docker run --rm -e BUILDARCH="${BUILDARCH}" -v "$(pwd)":/netdata:rw \
+    -e GITHUB_ACTIONS="${GITHUB_ACTIONS}" "${DOCKER_IMAGE_NAME}" \
     /bin/sh /netdata/packaging/makeself/build.sh "${@}"
 fi
diff --git a/packaging/makeself/functions.sh b/packaging/makeself/functions.sh
index afc8a9ac9..31c28d85c 100755
--- a/packaging/makeself/functions.sh
+++ b/packaging/makeself/functions.sh
@@ -29,34 +29,64 @@ set -euo pipefail
 # -----------------------------------------------------------------------------
 
 fetch() {
-  local dir="${1}" url="${2}" sha256="${3}"
+  local dir="${1}" url="${2}" sha256="${3}" key="${4}"
   local tar="${dir}.tar.gz"
+  local cache="${NETDATA_SOURCE_PATH}/artifacts/cache/${BUILDARCH}/${key}"
 
-  if [ ! -f "${NETDATA_MAKESELF_PATH}/tmp/${tar}" ]; then
-    run wget -O "${NETDATA_MAKESELF_PATH}/tmp/${tar}" "${url}"
+  if [ -d "${NETDATA_MAKESELF_PATH}/tmp/${dir}" ]; then
+    rm -rf "${NETDATA_MAKESELF_PATH}/tmp/${dir}"
   fi
 
-  # Check SHA256 of gzip'd tar file (apparently alpine's sha256sum requires
-  # two empty spaces between the checksum and the file's path)
-  set +e
-  echo "${sha256}  ${NETDATA_MAKESELF_PATH}/tmp/${tar}" | sha256sum -c -s
-  local rc=$?
-  if [ ${rc} -ne 0 ]; then
-      echo >&2 "SHA256 verification of tar file ${tar} failed (rc=${rc})"
-      echo >&2 "expected: ${sha256}, got $(sha256sum "${NETDATA_MAKESELF_PATH}/tmp/${tar}")"
-      exit 1
-  fi
-  set -e
+  if [ -d "${cache}/${dir}" ]; then
+    echo "Found cached copy of build directory for ${key}, using it."
+    cp -a "${cache}/${dir}" "${NETDATA_MAKESELF_PATH}/tmp/"
+    CACHE_HIT=1
+  else
+    echo "No cached copy of build directory for ${key} found, fetching sources instead."
+
+    if [ ! -f "${NETDATA_MAKESELF_PATH}/tmp/${tar}" ]; then
+      run wget -O "${NETDATA_MAKESELF_PATH}/tmp/${tar}" "${url}"
+    fi
+
+    # Check SHA256 of gzip'd tar file (apparently alpine's sha256sum requires
+    # two empty spaces between the checksum and the file's path)
+    set +e
+    echo "${sha256}  ${NETDATA_MAKESELF_PATH}/tmp/${tar}" | sha256sum -c -s
+    local rc=$?
+    if [ ${rc} -ne 0 ]; then
+        echo >&2 "SHA256 verification of tar file ${tar} failed (rc=${rc})"
+        echo >&2 "expected: ${sha256}, got $(sha256sum "${NETDATA_MAKESELF_PATH}/tmp/${tar}")"
+        exit 1
+    fi
+    set -e
 
-  if [ ! -d "${NETDATA_MAKESELF_PATH}/tmp/${dir}" ]; then
     cd "${NETDATA_MAKESELF_PATH}/tmp"
     run tar -zxpf "${tar}"
     cd -
+
+    CACHE_HIT=0
   fi
 
   run cd "${NETDATA_MAKESELF_PATH}/tmp/${dir}"
 }
 
+store_cache() {
+    key="${1}"
+    src="${2}"
+
+    cache="${NETDATA_SOURCE_PATH}/artifacts/cache/${BUILDARCH}/${key}"
+
+    if [ "${CACHE_HIT:-0}" -eq 0 ]; then
+        if [ -d "${cache}" ]; then
+            rm -rf "${cache}"
+        fi
+
+        mkdir -p "${cache}"
+
+        cp -a "${src}" "${cache}"
+    fi
+}
+
 # -----------------------------------------------------------------------------
 
 # load the functions of the netdata-installer.sh
diff --git a/packaging/makeself/jobs/20-openssl.install.sh b/packaging/makeself/jobs/20-openssl.install.sh
index 96d53fc86..5d9aa068e 100755
--- a/packaging/makeself/jobs/20-openssl.install.sh
+++ b/packaging/makeself/jobs/20-openssl.install.sh
@@ -9,19 +9,40 @@
 
 version="$(cat "$(dirname "${0}")/../openssl.version")"
 
-export CFLAGS='-fno-lto'
+export CFLAGS='-fno-lto -pipe'
 export LDFLAGS='-static'
 export PKG_CONFIG="pkg-config --static"
 
-# Might be bind-mounted
-if [ ! -d "${NETDATA_MAKESELF_PATH}/tmp/openssl" ]; then
+if [ -d "${NETDATA_MAKESELF_PATH}/tmp/openssl" ]; then
+  rm -rf "${NETDATA_MAKESELF_PATH}/tmp/openssl"
+fi
+
+if [ -d "${NETDATA_MAKESELF_PATH}/tmp/openssl" ]; then
+  rm -rf "${NETDATA_MAKESELF_PATH}/tmp/openssl"
+fi
+
+cache="${NETDATA_SOURCE_PATH}/artifacts/cache/${BUILDARCH}/openssl"
+
+if [ -d "${cache}" ]; then
+  echo "Found cached copy of build directory for openssl, using it."
+  cp -a "${cache}/openssl" "${NETDATA_MAKESELF_PATH}/tmp/"
+  CACHE_HIT=1
+else
+  echo "No cached copy of build directory for openssl found, fetching sources instead."
   run git clone --branch "${version}" --single-branch --depth 1 git://git.openssl.org/openssl.git "${NETDATA_MAKESELF_PATH}/tmp/openssl"
+  CACHE_HIT=0
 fi
+
 cd "${NETDATA_MAKESELF_PATH}/tmp/openssl" || exit 1
 
-run ./config -static no-tests --prefix=/openssl-static --openssldir=/opt/netdata/etc/ssl
-run make -j "$(nproc)"
+if [ "${CACHE_HIT:-0}" -eq 0 ]; then
+    run ./config -static no-tests --prefix=/openssl-static --openssldir=/opt/netdata/etc/ssl
+    run make -j "$(nproc)"
+fi
+
 run make -j "$(nproc)" install_sw
 
+store_cache openssl "${NETDATA_MAKESELF_PATH}/tmp/openssl"
+
 # shellcheck disable=SC2015
 [ "${GITHUB_ACTIONS}" = "true" ] && echo "::endgroup::" || true
diff --git a/packaging/makeself/jobs/50-bash-5.1.16.install.sh b/packaging/makeself/jobs/50-bash-5.1.16.install.sh
index d5cdfff1c..d9f6e5bb0 100755
--- a/packaging/makeself/jobs/50-bash-5.1.16.install.sh
+++ b/packaging/makeself/jobs/50-bash-5.1.16.install.sh
@@ -4,35 +4,43 @@
 # shellcheck source=packaging/makeself/functions.sh
 . "$(dirname "${0}")/../functions.sh" "${@}" || exit 1
 
+version="5.1.16"
+
 # shellcheck disable=SC2015
 [ "${GITHUB_ACTIONS}" = "true" ] && echo "::group::building bash" || true
 
-fetch "bash-5.1.16" "http://ftp.gnu.org/gnu/bash/bash-5.1.16.tar.gz" \
-    5bac17218d3911834520dad13cd1f85ab944e1c09ae1aba55906be1f8192f558
+fetch "bash-${version}" "http://ftp.gnu.org/gnu/bash/bash-${version}.tar.gz" \
+    5bac17218d3911834520dad13cd1f85ab944e1c09ae1aba55906be1f8192f558 bash
 
+export CFLAGS="-pipe"
 export PKG_CONFIG_PATH="/openssl-static/lib/pkgconfig"
 
-run ./configure \
-  --prefix="${NETDATA_INSTALL_PATH}" \
-  --without-bash-malloc \
-  --enable-static-link \
-  --enable-net-redirections \
-  --enable-array-variables \
-  --disable-progcomp \
-  --disable-profiling \
-  --disable-nls
-
-run make clean
-run make -j "$(nproc)"
-
-cat > examples/loadables/Makefile << EOF
-all:
-clean:
-install:
-EOF
+if [ "${CACHE_HIT:-0}" -eq 0 ]; then
+    run ./configure \
+        --prefix="${NETDATA_INSTALL_PATH}" \
+        --without-bash-malloc \
+        --enable-static-link \
+        --enable-net-redirections \
+        --enable-array-variables \
+        --disable-progcomp \
+        --disable-profiling \
+        --disable-nls \
+        --disable-dependency-tracking
+
+    run make clean
+    run make -j "$(nproc)"
+
+    cat > examples/loadables/Makefile <<-EOF
+	all:
+	clean:
+	install:
+	EOF
+fi
 
 run make install
 
+store_cache bash "${NETDATA_MAKESELF_PATH}/tmp/bash-${version}"
+
 if [ "${NETDATA_BUILD_WITH_DEBUG}" -eq 0 ]; then
   run strip "${NETDATA_INSTALL_PATH}"/bin/bash
 fi
diff --git a/packaging/makeself/jobs/50-curl-7.82.0.install.sh b/packaging/makeself/jobs/50-curl-7.82.0.install.sh
index f8fa4042c..251d35095 100755
--- a/packaging/makeself/jobs/50-curl-7.82.0.install.sh
+++ b/packaging/makeself/jobs/50-curl-7.82.0.install.sh
@@ -4,48 +4,56 @@
 # shellcheck source=packaging/makeself/functions.sh
 . "$(dirname "${0}")/../functions.sh" "${@}" || exit 1
 
+version="7.82.0"
+
 # shellcheck disable=SC2015
 [ "${GITHUB_ACTIONS}" = "true" ] && echo "::group::Building cURL" || true
 
-fetch "curl-7.82.0" "https://curl.haxx.se/download/curl-7.82.0.tar.gz" \
-    910cc5fe279dc36e2cca534172c94364cf3fcf7d6494ba56e6c61a390881ddce
+fetch "curl-${version}" "https://curl.haxx.se/download/curl-${version}.tar.gz" \
+    910cc5fe279dc36e2cca534172c94364cf3fcf7d6494ba56e6c61a390881ddce curl
 
-export CFLAGS="-I/openssl-static/include"
+export CFLAGS="-I/openssl-static/include -pipe"
 export LDFLAGS="-static -L/openssl-static/lib"
 export PKG_CONFIG="pkg-config --static"
 export PKG_CONFIG_PATH="/openssl-static/lib/pkgconfig"
 
-run autoreconf -fi
-
-run ./configure \
-  --prefix="${NETDATA_INSTALL_PATH}" \
-  --enable-optimize \
-  --disable-shared \
-  --enable-static \
-  --enable-http \
-  --disable-ldap \
-  --disable-ldaps \
-  --enable-proxy \
-  --disable-dict \
-  --disable-telnet \
-  --disable-tftp \
-  --disable-pop3 \
-  --disable-imap \
-  --disable-smb \
-  --disable-smtp \
-  --disable-gopher \
-  --enable-ipv6 \
-  --enable-cookies \
-  --with-ca-fallback \
-  --with-openssl
-
-# Curl autoconf does not honour the curl_LDFLAGS environment variable
-run sed -i -e "s/LDFLAGS =/LDFLAGS = -all-static/" src/Makefile
-
-run make clean
-run make -j "$(nproc)"
+if [ "${CACHE_HIT:-0}" -eq 0 ]; then
+    run autoreconf -fi
+
+    run ./configure \
+        --prefix="${NETDATA_INSTALL_PATH}" \
+        --enable-optimize \
+        --disable-shared \
+        --enable-static \
+        --enable-http \
+        --disable-ldap \
+        --disable-ldaps \
+        --enable-proxy \
+        --disable-dict \
+        --disable-telnet \
+        --disable-tftp \
+        --disable-pop3 \
+        --disable-imap \
+        --disable-smb \
+        --disable-smtp \
+        --disable-gopher \
+        --enable-ipv6 \
+        --enable-cookies \
+        --with-ca-fallback \
+        --with-openssl \
+        --disable-dependency-tracking
+
+    # Curl autoconf does not honour the curl_LDFLAGS environment variable
+    run sed -i -e "s/LDFLAGS =/LDFLAGS = -all-static/" src/Makefile
+
+    run make clean
+    run make -j "$(nproc)"
+fi
+
 run make install
 
+store_cache curl "${NETDATA_MAKESELF_PATH}/tmp/curl-${version}"
+
 if [ "${NETDATA_BUILD_WITH_DEBUG}" -eq 0 ]; then
   run strip "${NETDATA_INSTALL_PATH}"/bin/curl
 fi
diff --git a/packaging/makeself/jobs/50-fping-5.1.install.sh b/packaging/makeself/jobs/50-fping-5.1.install.sh
index e1435e944..644b5524a 100755
--- a/packaging/makeself/jobs/50-fping-5.1.install.sh
+++ b/packaging/makeself/jobs/50-fping-5.1.install.sh
@@ -4,31 +4,39 @@
 # shellcheck source=packaging/makeself/functions.sh
 . "$(dirname "${0}")/../functions.sh" "${@}" || exit 1
 
+version="5.1"
+
 # shellcheck disable=SC2015
 [ "${GITHUB_ACTIONS}" = "true" ] && echo "::group::Building fping" || true
 
-fetch "fping-5.1" "https://fping.org/dist/fping-5.1.tar.gz" \
-    1ee5268c063d76646af2b4426052e7d81a42b657e6a77d8e7d3d2e60fd7409fe
+fetch "fping-${version}" "https://fping.org/dist/fping-${version}.tar.gz" \
+    1ee5268c063d76646af2b4426052e7d81a42b657e6a77d8e7d3d2e60fd7409fe fping
 
-export CFLAGS="-static -I/openssl-static/include"
+export CFLAGS="-static -I/openssl-static/include -pipe"
 export LDFLAGS="-static -L/openssl-static/lib"
 export PKG_CONFIG_PATH="/openssl-static/lib/pkgconfig"
 
-run ./configure \
-  --prefix="${NETDATA_INSTALL_PATH}" \
-  --enable-ipv4 \
-  --enable-ipv6
-
-cat > doc/Makefile << EOF
-all:
-clean:
-install:
-EOF
+if [ "${CACHE_HIT:-0}" -eq 0 ]; then
+    run ./configure \
+        --prefix="${NETDATA_INSTALL_PATH}" \
+        --enable-ipv4 \
+        --enable-ipv6 \
+        --disable-dependency-tracking
+
+    cat > doc/Makefile <<-EOF
+	all:
+	clean:
+	install:
+	EOF
+
+    run make clean
+    run make -j "$(nproc)"
+fi
 
-run make clean
-run make -j "$(nproc)"
 run make install
 
+store_cache fping "${NETDATA_MAKESELF_PATH}/tmp/fping-${version}"
+
 if [ "${NETDATA_BUILD_WITH_DEBUG}" -eq 0 ]; then
   run strip "${NETDATA_INSTALL_PATH}"/bin/fping
 fi
diff --git a/packaging/makeself/jobs/50-ioping-1.2.install.sh b/packaging/makeself/jobs/50-ioping-1.2.install.sh
index 9e9505022..160f45b98 100755
--- a/packaging/makeself/jobs/50-ioping-1.2.install.sh
+++ b/packaging/makeself/jobs/50-ioping-1.2.install.sh
@@ -4,19 +4,26 @@
 # shellcheck source=packaging/makeself/functions.sh
 . "$(dirname "${0}")/../functions.sh" "${@}" || exit 1
 
+version='1.2'
+
 # shellcheck disable=SC2015
 [ "${GITHUB_ACTIONS}" = "true" ] && echo "::group::Building ioping" || true
 
-fetch "ioping-1.2" "https://github.com/koct9i/ioping/archive/v1.2.tar.gz" \
-    d3e4497c653a1e96df67c72ce2b70da18e9f5e3b93179a5bb57a6e30ceacfa75
+fetch "ioping-${version}" "https://github.com/koct9i/ioping/archive/v${version}.tar.gz" \
+    d3e4497c653a1e96df67c72ce2b70da18e9f5e3b93179a5bb57a6e30ceacfa75 ioping
+
+export CFLAGS="-static -pipe"
 
-export CFLAGS="-static"
+if [ "${CACHE_HIT:-0}" -eq 0 ]; then
+    run make clean
+    run make -j "$(nproc)"
+fi
 
-run make clean
-run make -j "$(nproc)"
 run mkdir -p "${NETDATA_INSTALL_PATH}"/usr/libexec/netdata/plugins.d/
 run install -o root -g root -m 4750 ioping "${NETDATA_INSTALL_PATH}"/usr/libexec/netdata/plugins.d/
 
+store_cache ioping "${NETDATA_MAKESELF_PATH}/tmp/ioping-${version}"
+
 if [ "${NETDATA_BUILD_WITH_DEBUG}" -eq 0 ]; then
   run strip "${NETDATA_INSTALL_PATH}"/usr/libexec/netdata/plugins.d/ioping
 fi
diff --git a/packaging/makeself/jobs/70-netdata-git.install.sh b/packaging/makeself/jobs/70-netdata-git.install.sh
index 98fc4dbe8..ea6902a44 100755
--- a/packaging/makeself/jobs/70-netdata-git.install.sh
+++ b/packaging/makeself/jobs/70-netdata-git.install.sh
@@ -7,9 +7,9 @@
 cd "${NETDATA_SOURCE_PATH}" || exit 1
 
 if [ "${NETDATA_BUILD_WITH_DEBUG}" -eq 0 ]; then
-  export CFLAGS="-static -O3 -I/openssl-static/include"
+  export CFLAGS="-static -O2 -I/openssl-static/include -pipe"
 else
-  export CFLAGS="-static -O1 -ggdb -Wall -Wextra -Wformat-signedness -fstack-protector-all -D_FORTIFY_SOURCE=2 -DNETDATA_INTERNAL_CHECKS=1 -I/openssl-static/include"
+  export CFLAGS="-static -O1 -pipe -ggdb -Wall -Wextra -Wformat-signedness -fstack-protector-all -D_FORTIFY_SOURCE=2 -DNETDATA_INTERNAL_CHECKS=1 -I/openssl-static/include"
 fi
 
 export LDFLAGS="-static -L/openssl-static/lib"
@@ -32,7 +32,8 @@ run ./netdata-installer.sh \
   --dont-start-it \
   --require-cloud \
   --use-system-protobuf \
-  --dont-scrub-cflags-even-though-it-may-break-things
+  --dont-scrub-cflags-even-though-it-may-break-things \
+  --one-time-build
 
 # shellcheck disable=SC2015
 [ "${GITHUB_ACTIONS}" = "true" ] && echo "::group::Finishing netdata install" || true
@@ -53,11 +54,5 @@ if run readelf -l "${NETDATA_INSTALL_PATH}"/bin/netdata | grep 'INTERP'; then
   exit 1
 fi
 
-if [ "${NETDATA_BUILD_WITH_DEBUG}" -eq 0 ]; then
-  run strip "${NETDATA_INSTALL_PATH}"/bin/netdata
-  run strip "${NETDATA_INSTALL_PATH}"/usr/libexec/netdata/plugins.d/apps.plugin
-  run strip "${NETDATA_INSTALL_PATH}"/usr/libexec/netdata/plugins.d/cgroup-network
-fi
-
 # shellcheck disable=SC2015
 [ "${GITHUB_ACTIONS}" = "true" ] && echo "::endgroup::" || true
diff --git a/packaging/makeself/uname2platform.sh b/packaging/makeself/uname2platform.sh
new file mode 100755
index 000000000..7eab706ec
--- /dev/null
+++ b/packaging/makeself/uname2platform.sh
@@ -0,0 +1,18 @@
+#!/bin/sh
+#
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+set -e
+
+BUILDARCH="${1}"
+
+case "${BUILDARCH}" in
+  x86_64) echo "linux/amd64" ;;
+  armv7l) echo "linux/arm/v7" ;;
+  aarch64) echo "linux/arm64/v8" ;;
+  ppc64le) echo "linux/ppc64le" ;;
+  *)
+    echo "Unknown target architecture '${BUILDARCH}'." >&2
+    exit 1
+    ;;
+esac