summaryrefslogtreecommitdiffstats
path: root/.travis/package_management
diff options
context:
space:
mode:
authorDaniel Baumann <daniel.baumann@progress-linux.org>2021-02-07 11:45:55 +0000
committerDaniel Baumann <daniel.baumann@progress-linux.org>2021-02-07 11:45:55 +0000
commita8220ab2d293bb7f4b014b79d16b2fb05090fa93 (patch)
tree77f0a30f016c0925cf7ee9292e644bba183c2774 /.travis/package_management
parentAdding upstream version 1.19.0. (diff)
downloadnetdata-a8220ab2d293bb7f4b014b79d16b2fb05090fa93.tar.xz
netdata-a8220ab2d293bb7f4b014b79d16b2fb05090fa93.zip
Adding upstream version 1.29.0.upstream/1.29.0
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to '')
-rw-r--r--.travis/package_management/build.sh32
-rwxr-xr-x.travis/package_management/build_judy.sh36
-rwxr-xr-x.travis/package_management/build_libuv.sh36
-rwxr-xr-x.travis/package_management/build_package_in_container.sh82
-rwxr-xr-x.travis/package_management/common.py182
-rwxr-xr-x.travis/package_management/configure_deb_lxc_environment.py90
-rwxr-xr-x.travis/package_management/configure_rpm_lxc_environment.py102
-rwxr-xr-x.travis/package_management/create_lxc_for_build.sh101
-rw-r--r--.travis/package_management/functions.sh33
-rwxr-xr-x.travis/package_management/old_package_purging.sh93
-rwxr-xr-x.travis/package_management/package_cloud_wrapper.sh48
-rwxr-xr-x.travis/package_management/prepare_packages.sh63
-rwxr-xr-x.travis/package_management/trigger_deb_lxc_build.py86
-rwxr-xr-x.travis/package_management/trigger_rpm_lxc_build.py55
-rwxr-xr-x.travis/package_management/yank_stale_pkg.sh35
15 files changed, 1074 insertions, 0 deletions
diff --git a/.travis/package_management/build.sh b/.travis/package_management/build.sh
new file mode 100644
index 00000000..bafaecc5
--- /dev/null
+++ b/.travis/package_management/build.sh
@@ -0,0 +1,32 @@
+#!/usr/bin/env bash
+
+UNPACKAGED_NETDATA_PATH="$1"
+LATEST_RELEASE_VERSION="$2"
+
+if [ -z "${LATEST_RELEASE_VERSION}" ]; then
+ echo "Parameter 'LATEST_RELEASE_VERSION' not defined"
+ exit 1
+fi
+
+if [ -z "${UNPACKAGED_NETDATA_PATH}" ]; then
+ echo "Parameter 'UNPACKAGED_NETDATA_PATH' not defined"
+ exit 1
+fi
+
+echo "Running changelog generation mechanism since ${LATEST_RELEASE_VERSION}"
+
+echo "Entering ${UNPACKAGED_NETDATA_PATH}"
+cd "${UNPACKAGED_NETDATA_PATH}"
+
+echo "Linking debian -> contrib/debian"
+ln -sf contrib/debian debian
+
+echo "Executing dpkg-buildpackage"
+# pre/post options are after 1.18.8, is simpler to just check help for their existence than parsing version
+if dpkg-buildpackage --help | grep "\-\-post\-clean" 2> /dev/null > /dev/null; then
+ dpkg-buildpackage --post-clean --pre-clean --build=binary -us -uc
+else
+ dpkg-buildpackage -b -us -uc
+fi
+
+echo "DEB build script completed!"
diff --git a/.travis/package_management/build_judy.sh b/.travis/package_management/build_judy.sh
new file mode 100755
index 00000000..202ea044
--- /dev/null
+++ b/.travis/package_management/build_judy.sh
@@ -0,0 +1,36 @@
+#!/usr/bin/env bash
+#
+# Build Judy from source, you need to run this script as root.
+#
+# Copyright: SPDX-License-Identifier: GPL-3.0-or-later
+#
+# Author : Pavlos Emm. Katsoulakis (paul@netdata.cloud)
+set -e
+JUDY_VER="1.0.5"
+JUDY_DIR="/opt/judy-${JUDY_VER}"
+
+# If we are not in netdata git repo, at the top level directory, fail
+TOP_LEVEL=$(basename "$(git rev-parse --show-toplevel)")
+CWD=$(git rev-parse --show-cdup)
+if [ -n "$CWD" ] || [ ! "${TOP_LEVEL}" == "netdata" ]; then
+ echo "Run as .travis/package_management/$(basename "$0") from top level directory of netdata git repository"
+ echo "Build Judy package from source code failed"
+ exit 1
+fi
+
+echo "Fetching judy source tarball"
+wget -O /opt/judy.tar.gz http://downloads.sourceforge.net/project/judy/judy/Judy-${JUDY_VER}/Judy-${JUDY_VER}.tar.gz
+
+echo "Entering /opt directory and extracting tarball"
+cd /opt && tar -xf judy.tar.gz && rm judy.tar.gz
+
+echo "Entering ${JUDY_DIR}"
+cd "${JUDY_DIR}"
+
+echo "Running configure"
+CFLAGS="-O2 -s" CXXFLAGS="-O2 -s" ./configure
+
+echo "Compiling and installing"
+make && make install
+
+echo "Done, enjoy Judy!"
diff --git a/.travis/package_management/build_libuv.sh b/.travis/package_management/build_libuv.sh
new file mode 100755
index 00000000..c30eede6
--- /dev/null
+++ b/.travis/package_management/build_libuv.sh
@@ -0,0 +1,36 @@
+#!/usr/bin/env bash
+#
+# Build libuv from source, you need to run this script as root.
+#
+# Copyright: SPDX-License-Identifier: GPL-3.0-or-later
+#
+# Author : Pavlos Emm. Katsoulakis <paul@netdata.cloud>
+set -e
+LIBUV_VERSION="v1.32.0"
+# Their folder is libuv-1.32.0 while the tarball version is v1.32.0, so fix that until they fix it...
+LIBUV_DIR="/opt/libuv-${LIBUV_VERSION/v/}"
+
+# If we are not in netdata git repo, at the top level directory, fail
+TOP_LEVEL=$(basename "$(git rev-parse --show-toplevel)")
+CWD=$(git rev-parse --show-cdup)
+if [ -n "$CWD" ] || [ ! "${TOP_LEVEL}" == "netdata" ]; then
+ echo "Run as .travis/package_management/$(basename "$0") from top level directory of netdata git repository"
+ echo "Build libuv package from source code failed"
+ exit 1
+fi
+
+echo "Fetching libuv from github"
+wget -O /opt/libuv.tar.gz "https://github.com/libuv/libuv/archive/${LIBUV_VERSION}.tar.gz"
+
+echo "Entering /opt and extracting source"
+cd /opt && tar -xf libuv.tar.gz && rm libuv.tar.gz
+
+echo "Entering ${LIBUV_DIR}"
+cd "${LIBUV_DIR}"
+
+echo "Compiling and installing"
+sh autogen.sh
+./configure
+make && make check && make install
+
+echo "Done, enjoy libuv!"
diff --git a/.travis/package_management/build_package_in_container.sh b/.travis/package_management/build_package_in_container.sh
new file mode 100755
index 00000000..95a68e7a
--- /dev/null
+++ b/.travis/package_management/build_package_in_container.sh
@@ -0,0 +1,82 @@
+#!/usr/bin/env bash
+#
+# Entry point for package build process
+#
+# Copyright: SPDX-License-Identifier: GPL-3.0-or-later
+#
+# Author : Pavlos Emm. Katsoulakis (paul@netdata.cloud)
+#shellcheck disable=SC1091
+set -e
+
+# If we are not in netdata git repo, at the top level directory, fail
+TOP_LEVEL=$(basename "$(git rev-parse --show-toplevel)")
+CWD=$(git rev-parse --show-cdup)
+if [ -n "$CWD" ] || [ ! "${TOP_LEVEL}" == "netdata" ]; then
+ echo "Run as .travis/package_management/$(basename "$0") from top level directory of netdata git repository"
+ echo "Docker build process aborted"
+ exit 1
+fi
+
+source .travis/package_management/functions.sh || (echo "Failed to load packaging library" && exit 1)
+
+# Check for presence of mandatory environment variables
+if [ -z "${BUILD_STRING}" ]; then
+ echo "No Distribution was defined. Make sure BUILD_STRING is set on the environment before running this script"
+ exit 1
+fi
+
+if [ -z "${BUILDER_NAME}" ]; then
+ echo "No builder account and container name defined. Make sure BUILDER_NAME is set on the environment before running this script"
+ exit 1
+fi
+
+if [ -z "${BUILD_DISTRO}" ]; then
+ echo "No build distro information defined. Make sure BUILD_DISTRO is set on the environment before running this script"
+ exit 1
+fi
+
+if [ -z "${BUILD_RELEASE}" ]; then
+ echo "No build release information defined. Make sure BUILD_RELEASE is set on the environment before running this script"
+ exit 1
+fi
+
+if [ -z "${PACKAGE_TYPE}" ]; then
+ echo "No build release information defined. Make sure PACKAGE_TYPE is set on the environment before running this script"
+ exit 1
+fi
+
+# Detect architecture and load extra variables needed
+detect_arch_from_commit
+
+case "${BUILD_ARCH}" in
+"all")
+ echo "* * * Building all architectures, amd64 and i386 * * *"
+ echo "Building for amd64.."
+ export CONTAINER_NAME="${BUILDER_NAME}-${BUILD_DISTRO}${BUILD_RELEASE}-amd64"
+ export LXC_CONTAINER_ROOT="/var/lib/lxc/${CONTAINER_NAME}/rootfs"
+ .travis/package_management/trigger_${PACKAGE_TYPE}_lxc_build.py "${CONTAINER_NAME}"
+
+ echo "Building for arm64.."
+ export CONTAINER_NAME="${BUILDER_NAME}-${BUILD_DISTRO}${BUILD_RELEASE}-arm64"
+ export LXC_CONTAINER_ROOT="/var/lib/lxc/${CONTAINER_NAME}/rootfs"
+ .travis/package_management/trigger_${PACKAGE_TYPE}_lxc_build.py "${CONTAINER_NAME}"
+
+ echo "Building for i386.."
+ export CONTAINER_NAME="${BUILDER_NAME}-${BUILD_DISTRO}${BUILD_RELEASE}-i386"
+ export LXC_CONTAINER_ROOT="/var/lib/lxc/${CONTAINER_NAME}/rootfs"
+ .travis/package_management/trigger_${PACKAGE_TYPE}_lxc_build.py "${CONTAINER_NAME}"
+
+ ;;
+"amd64"|"arm64"|"i386")
+ echo "Building for ${BUILD_ARCH}.."
+ export CONTAINER_NAME="${BUILDER_NAME}-${BUILD_DISTRO}${BUILD_RELEASE}-${BUILD_ARCH}"
+ export LXC_CONTAINER_ROOT="/var/lib/lxc/${CONTAINER_NAME}/rootfs"
+ .travis/package_management/trigger_${PACKAGE_TYPE}_lxc_build.py "${CONTAINER_NAME}"
+ ;;
+*)
+ echo "Unknown build architecture '${BUILD_ARCH}', nothing to do for build"
+ exit 1
+ ;;
+esac
+
+echo "Build process completed!"
diff --git a/.travis/package_management/common.py b/.travis/package_management/common.py
new file mode 100755
index 00000000..4cc04b93
--- /dev/null
+++ b/.travis/package_management/common.py
@@ -0,0 +1,182 @@
+#
+#
+# Python library with commonly used functions within the package management scope
+#
+# Author : Pavlos Emm. Katsoulakis <paul@netdata.cloud>
+
+import lxc
+import subprocess
+import os
+import sys
+import tempfile
+import shutil
+
+def fetch_version(orig_build_version):
+ tag = None
+ friendly_version = ""
+
+ # TODO: Checksum validations
+ if str(orig_build_version).count(".latest") == 1:
+ version_list=str(orig_build_version).replace('v', '').split('.')
+ minor = version_list[3] if int(version_list[2]) == 0 else (version_list[2] + version_list[3])
+ friendly_version='.'.join(version_list[0:2]) + "." + minor
+ else:
+ friendly_version = orig_build_version.replace('v', '')
+ tag = friendly_version # Go to stable tag
+ print("Version set to %s from %s" % (friendly_version, orig_build_version))
+
+ return friendly_version, tag
+
+def replace_tag(tag_name, spec, new_tag_content):
+ print("Fixing tag %s in %s" % (tag_name, spec))
+
+ ifp = open(spec, "r")
+ config = ifp.readlines()
+ ifp.close()
+
+ source_line = -1
+ for line in config:
+ if str(line).count(tag_name + ":") > 0:
+ source_line = config.index(line)
+ print("Found line: %s in item %d" % (line, source_line))
+ break
+
+ if source_line >= 0:
+ print("Replacing line %s with %s in spec file" %(config[source_line], new_tag_content))
+ config[source_line] = "%s: %s\n" % (tag_name, new_tag_content)
+ config_str = ''.join(config)
+ ofp = open(spec, 'w')
+ ofp.write(config_str)
+ ofp.close()
+
+def run_command(container, command):
+ print("Running command: %s" % command)
+ command_result = container.attach_wait(lxc.attach_run_command, command, stdout=sys.stdout.buffer, stderr=sys.stdout.buffer)
+
+ if command_result != 0:
+ raise Exception("Command failed with exit code %d" % command_result)
+
+def run_command_in_host(cmd, cwd=None):
+ print("Issue command in host: %s, cwd:%s" % (str(cmd), str(cwd)))
+
+ proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=cwd)
+ o, e = proc.communicate()
+ print('Output: ' + o.decode('ascii'))
+ print('Error: ' + e.decode('ascii'))
+ print('code: ' + str(proc.returncode))
+
+def prepare_repo(container):
+ if str(os.environ["REPO_TOOL"]).count("zypper") == 1:
+ run_command(container, [os.environ["REPO_TOOL"], "clean", "-a"])
+ run_command(container, [os.environ["REPO_TOOL"], "--no-gpg-checks", "update", "-y"])
+
+ elif str(os.environ["REPO_TOOL"]).count("yum") == 1:
+ run_command(container, [os.environ["REPO_TOOL"], "clean", "all"])
+ run_command(container, [os.environ["REPO_TOOL"], "update", "-y"])
+
+ if os.environ["BUILD_STRING"].count("el/7") == 1 and os.environ["BUILD_ARCH"].count("i386") == 1:
+ print ("Skipping epel-release install for %s-%s" % (os.environ["BUILD_STRING"], os.environ["BUILD_ARCH"]))
+ else:
+ run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "epel-release"])
+
+ elif str(os.environ["REPO_TOOL"]).count("apt-get") == 1:
+ if str(os.environ["BUILD_STRING"]).count("debian/jessie") == 1:
+ run_command(container, ["bash", "-c", "echo deb http://archive.debian.org/debian/ jessie-backports main contrib non-free >> /etc/apt/sources.list.d/99-archived.list"])
+ run_command(container, [os.environ["REPO_TOOL"], "update", "-y", '-o', 'Acquire::Check-Valid-Until=false'])
+ else:
+ run_command(container, [os.environ["REPO_TOOL"], "update", "-y"])
+ else:
+ run_command(container, [os.environ["REPO_TOOL"], "update", "-y"])
+
+ run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "sudo"])
+ run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "wget"])
+ run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "bash"])
+
+def install_common_dependendencies(container):
+ if str(os.environ["REPO_TOOL"]).count("zypper") == 1:
+ run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "gcc-c++"])
+ run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "json-glib-devel"])
+ run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "freeipmi-devel"])
+ run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "cups-devel"])
+ run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "snappy-devel"])
+ run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "protobuf-devel"])
+ run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "protobuf-c"])
+
+ elif str(os.environ["REPO_TOOL"]).count("yum") == 1:
+ run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "gcc-c++"])
+ run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "json-c-devel"])
+ run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "freeipmi-devel"])
+ run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "cups-devel"])
+ run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "snappy-devel"])
+ run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "protobuf-devel"])
+ run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "protobuf-c-devel"])
+ run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "protobuf-compiler"])
+ run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "libwebsockets-devel"])
+
+ elif str(os.environ["REPO_TOOL"]).count("apt-get") == 1:
+ run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "g++"])
+ run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "libipmimonitoring-dev"])
+ run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "libjson-c-dev"])
+ run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "libcups2-dev"])
+ run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "libsnappy-dev"])
+ run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "libprotobuf-dev"])
+ run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "libprotoc-dev"])
+ run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "protobuf-compiler"])
+ if os.environ["BUILD_STRING"].count("debian/jessie") == 1:
+ run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "snappy"])
+ else:
+ run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "gcc-c++"])
+ run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "cups-devel"])
+ run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "freeipmi-devel"])
+ run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "json-c-devel"])
+ run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "snappy-devel"])
+ run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "protobuf-devel"])
+ run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "protobuf-c-devel"])
+ run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "protobuf-compiler"])
+ run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "libwebsockets-devel"])
+
+ if os.environ["BUILD_STRING"].count("el/6") <= 0:
+ run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "autogen"])
+
+def prepare_version_source(dest_archive, pkg_friendly_version, tag=None):
+ print(".0 Preparing local implementation tarball for version %s" % pkg_friendly_version)
+ tar_file = os.environ['LXC_CONTAINER_ROOT'] + dest_archive
+
+ print(".0 Copy repo to prepare it for tarball generation")
+ tmp_src = tempfile.mkdtemp(prefix='netdata-source-')
+ run_command_in_host(['cp', '-r', '.', tmp_src])
+
+ if tag is not None:
+ print(".1 Checking out tag %s" % tag)
+ run_command_in_host(['git', 'fetch', '--all'], tmp_src)
+
+ # TODO: Keep in mind that tricky 'v' there, needs to be removed once we clear our versioning scheme
+ run_command_in_host(['git', 'checkout', 'v%s' % pkg_friendly_version], tmp_src)
+
+ print(".2 Tagging the code with version: %s" % pkg_friendly_version)
+ run_command_in_host(['git', 'tag', '-a', pkg_friendly_version, '-m', 'Tagging while packaging on %s' % os.environ["CONTAINER_NAME"]], tmp_src)
+
+ print(".3 Run autoreconf -ivf")
+ run_command_in_host(['autoreconf', '-ivf'], tmp_src)
+
+ print(".4 Run configure")
+ run_command_in_host(['./configure', '--prefix=/usr', '--sysconfdir=/etc', '--localstatedir=/var', '--libdir=/usr/lib', '--libexecdir=/usr/libexec', '--with-math', '--with-zlib', '--with-user=netdata'], tmp_src)
+
+ print(".5 Run make dist")
+ run_command_in_host(['make', 'dist'], tmp_src)
+
+ print(".6 Copy generated tarbal to desired path")
+ generated_tarball = '%s/netdata-%s.tar.gz' % (tmp_src, pkg_friendly_version)
+
+ if os.path.exists(generated_tarball):
+ run_command_in_host(['sudo', 'cp', generated_tarball, tar_file])
+
+ print(".7 Fixing permissions on tarball")
+ run_command_in_host(['sudo', 'chmod', '777', tar_file])
+
+ print(".8 Bring over netdata.spec, Remove temp directory");
+ run_command_in_host(['cp', '%s/netdata.spec' % tmp_src, 'netdata.spec'])
+ shutil.rmtree(tmp_src)
+ else:
+ print("I could not find (%s) on the disk, stopping the build. Kindly check the logs and try again" % generated_tarball)
+ sys.exit(1)
diff --git a/.travis/package_management/configure_deb_lxc_environment.py b/.travis/package_management/configure_deb_lxc_environment.py
new file mode 100755
index 00000000..627493bf
--- /dev/null
+++ b/.travis/package_management/configure_deb_lxc_environment.py
@@ -0,0 +1,90 @@
+#!/usr/bin/env python3
+#
+# Prepare the build environment within the container
+# The script attaches to the running container and does the following:
+# 1) Create the container
+# 2) Start the container up
+# 3) Create the builder user
+# 4) Prepare the environment for DEB build
+#
+# Copyright: SPDX-License-Identifier: GPL-3.0-or-later
+#
+# Author : Pavlos Emm. Katsoulakis <paul@netdata.cloud>
+
+import common
+import os
+import sys
+import lxc
+
+if len(sys.argv) != 2:
+ print('You need to provide a container name to get things started')
+ sys.exit(1)
+container_name=sys.argv[1]
+
+# Setup the container object
+print("Defining container %s" % container_name)
+container = lxc.Container(container_name)
+if not container.defined:
+ raise Exception("Container %s not defined!" % container_name)
+
+# Start the container
+if not container.start():
+ raise Exception("Failed to start the container")
+
+if not container.running or not container.state == "RUNNING":
+ raise Exception('Container %s is not running, configuration process aborted ' % container_name)
+
+# Wait for connectivity
+print("Waiting for container connectivity to start configuration sequence")
+if not container.get_ips(timeout=30):
+ raise Exception("Timeout while waiting for container")
+
+build_path = "/home/%s" % os.environ['BUILDER_NAME']
+
+# Run the required activities now
+# 1. Create the builder user
+print("1. Adding user %s" % os.environ['BUILDER_NAME'])
+common.run_command(container, ["useradd", "-m", os.environ['BUILDER_NAME']])
+
+# Fetch package dependencies for the build
+print("2. Preparing repo on LXC container")
+common.prepare_repo(container)
+
+print("2.1 Install .DEB build support packages")
+common.run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "dpkg-dev"])
+common.run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "libdistro-info-perl"])
+common.run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "dh-make"])
+common.run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "dh-systemd"])
+common.run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "dh-autoreconf"])
+common.run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "git-buildpackage"])
+
+print("2.2 Add more dependencies")
+common.run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "libnetfilter-acct-dev"])
+common.run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "libcups2-dev"])
+
+print ("3.1 Run install-required-packages scriptlet")
+common.run_command(container, ["wget", "-T", "15", "-O", "%s/.install-required-packages.sh" % build_path, "https://raw.githubusercontent.com/netdata/netdata/master/packaging/installer/install-required-packages.sh"])
+common.run_command(container, ["bash", "%s/.install-required-packages.sh" % build_path, "netdata", "--dont-wait", "--non-interactive"])
+
+print("3.2 Installing package dependencies within LXC container")
+common.install_common_dependendencies(container)
+
+friendly_version=""
+dest_archive=""
+download_url=""
+tag = None
+friendly_version, tag = common.fetch_version(os.environ['BUILD_VERSION'])
+
+tar_file="%s/netdata-%s.tar.gz" % (os.path.dirname(dest_archive), friendly_version)
+
+print("5. I will be building version '%s' of netdata." % os.environ['BUILD_VERSION'])
+dest_archive="%s/netdata-%s.tar.gz" % (build_path, friendly_version)
+
+common.prepare_version_source(dest_archive, friendly_version, tag=tag)
+
+print("6. Installing build.sh script to build path")
+common.run_command_in_host(['sudo', 'cp', '.travis/package_management/build.sh', "%s/%s/build.sh" % (os.environ['LXC_CONTAINER_ROOT'], build_path)])
+common.run_command_in_host(['sudo', 'chmod', '777', "%s/%s/build.sh" % (os.environ['LXC_CONTAINER_ROOT'], build_path)])
+common.run_command_in_host(['sudo', 'ln', '-sf', 'contrib/debian', 'debian'])
+
+print("Done!")
diff --git a/.travis/package_management/configure_rpm_lxc_environment.py b/.travis/package_management/configure_rpm_lxc_environment.py
new file mode 100755
index 00000000..4dca0bf3
--- /dev/null
+++ b/.travis/package_management/configure_rpm_lxc_environment.py
@@ -0,0 +1,102 @@
+#!/usr/bin/env python3
+#
+#
+# Prepare the build environment within the container
+# The script attaches to the running container and does the following:
+# 1) Create the container
+# 2) Start the container up
+# 3) Create the builder user
+# 4) Prepare the environment for RPM build
+#
+# Copyright: SPDX-License-Identifier: GPL-3.0-or-later
+#
+# Author : Pavlos Emm. Katsoulakis <paul@netdata.cloud>
+
+import common
+import os
+import sys
+import lxc
+
+if len(sys.argv) != 2:
+ print('You need to provide a container name to get things started')
+ sys.exit(1)
+container_name=sys.argv[1]
+
+# Setup the container object
+print("Defining container %s" % container_name)
+container = lxc.Container(container_name)
+if not container.defined:
+ raise Exception("Container %s not defined!" % container_name)
+
+# Start the container
+if not container.start():
+ raise Exception("Failed to start the container")
+
+if not container.running or not container.state == "RUNNING":
+ raise Exception('Container %s is not running, configuration process aborted ' % container_name)
+
+# Wait for connectivity
+print("Waiting for container connectivity to start configuration sequence")
+if not container.get_ips(timeout=30):
+ raise Exception("Timeout while waiting for container")
+
+# Run the required activities now
+# Create the builder user
+print("1. Adding user %s" % os.environ['BUILDER_NAME'])
+common.run_command(container, ["useradd", "-m", os.environ['BUILDER_NAME']])
+
+# Fetch package dependencies for the build
+print("2.1 Preparing repo on LXC container")
+common.prepare_repo(container)
+
+common.run_command(container, ["wget", "-T", "15", "-O", "/home/%s/.install-required-packages.sh" % (os.environ['BUILDER_NAME']), "https://raw.githubusercontent.com/netdata/netdata/master/packaging/installer/install-required-packages.sh"])
+common.run_command(container, ["bash", "/home/%s/.install-required-packages.sh" % (os.environ['BUILDER_NAME']), "netdata", "--dont-wait", "--non-interactive"])
+
+# Exceptional cases, not available everywhere
+#
+print("2.2 Running uncommon dependencies and preparing LXC environment")
+# Not on Centos-7
+if os.environ["BUILD_STRING"].count("el/7") <= 0:
+ common.run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "libnetfilter_acct-devel"])
+
+# Not on Centos-6
+if os.environ["BUILD_STRING"].count("el/6") <= 0:
+ common.run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "autoconf-archive"])
+
+print("2.3 Installing common dependencies")
+common.install_common_dependendencies(container)
+
+print("3. Setting up macros")
+common.run_command(container, ["sudo", "-u", os.environ['BUILDER_NAME'], "/bin/echo", "'%_topdir %(echo /home/" + os.environ['BUILDER_NAME'] + ")/rpmbuild' > /home/" + os.environ['BUILDER_NAME'] + "/.rpmmacros"])
+
+print("4. Create rpmbuild directory")
+common.run_command(container, ["sudo", "-u", os.environ['BUILDER_NAME'], "mkdir", "-p", "/home/" + os.environ['BUILDER_NAME'] + "/rpmbuild/BUILD"])
+common.run_command(container, ["sudo", "-u", os.environ['BUILDER_NAME'], "mkdir", "-p", "/home/" + os.environ['BUILDER_NAME'] + "/rpmbuild/RPMS"])
+common.run_command(container, ["sudo", "-u", os.environ['BUILDER_NAME'], "mkdir", "-p", "/home/" + os.environ['BUILDER_NAME'] + "/rpmbuild/SOURCES"])
+common.run_command(container, ["sudo", "-u", os.environ['BUILDER_NAME'], "mkdir", "-p", "/home/" + os.environ['BUILDER_NAME'] + "/rpmbuild/SPECS"])
+common.run_command(container, ["sudo", "-u", os.environ['BUILDER_NAME'], "mkdir", "-p", "/home/" + os.environ['BUILDER_NAME'] + "/rpmbuild/SRPMS"])
+common.run_command(container, ["sudo", "-u", os.environ['BUILDER_NAME'], "ls", "-ltrR", "/home/" + os.environ['BUILDER_NAME'] + "/rpmbuild"])
+
+# Download the source
+rpm_friendly_version=""
+dest_archive=""
+download_url=""
+spec_file="/home/%s/rpmbuild/SPECS/netdata.spec" % os.environ['BUILDER_NAME']
+tag = None
+rpm_friendly_version, tag = common.fetch_version(os.environ['BUILD_VERSION'])
+tar_file="%s/netdata-%s.tar.gz" % (os.path.dirname(dest_archive), rpm_friendly_version)
+
+print("5. I will be building version '%s' of netdata." % os.environ['BUILD_VERSION'])
+dest_archive="/home/%s/rpmbuild/SOURCES/netdata-%s.tar.gz" % (os.environ['BUILDER_NAME'], rpm_friendly_version)
+
+common.prepare_version_source(dest_archive, rpm_friendly_version, tag=tag)
+
+# Extract the spec file in place
+print("6. Extract spec file from the source")
+common.run_command_in_host(['sudo', 'cp', 'netdata.spec', os.environ['LXC_CONTAINER_ROOT'] + spec_file])
+common.run_command_in_host(['sudo', 'chmod', '777', os.environ['LXC_CONTAINER_ROOT'] + spec_file])
+
+print("7. Temporary hack: Change Source0 to %s on spec file %s" % (dest_archive, spec_file))
+common.replace_tag("Source0", os.environ['LXC_CONTAINER_ROOT'] + spec_file, tar_file)
+
+print('Done!')
diff --git a/.travis/package_management/create_lxc_for_build.sh b/.travis/package_management/create_lxc_for_build.sh
new file mode 100755
index 00000000..d733687a
--- /dev/null
+++ b/.travis/package_management/create_lxc_for_build.sh
@@ -0,0 +1,101 @@
+#!/usr/bin/env bash
+#
+# This script generates an LXC container and starts it up
+# Once the script completes successfully, a container has become available for usage
+# The container image to be used and the container name to be set, are part of variables
+# that must be present for the script to work
+#
+# Copyright: SPDX-License-Identifier: GPL-3.0-or-later
+#
+# Author : Pavlos Emm. Katsoulakis (paul@netdata.cloud)
+# shellcheck disable=SC1091
+set -e
+
+source .travis/package_management/functions.sh || (echo "Failed to load packaging library" && exit 1)
+
+# If we are not in netdata git repo, at the top level directory, fail
+TOP_LEVEL=$(basename "$(git rev-parse --show-toplevel)")
+CWD=$(git rev-parse --show-cdup)
+if [ -n "$CWD" ] || [ ! "${TOP_LEVEL}" == "netdata" ]; then
+ echo "Run as .travis/package_management/$(basename "$0") from top level directory of netdata git repository"
+ echo "LXC Container creation aborted"
+ exit 1
+fi
+
+# Check for presence of mandatory environment variables
+if [ -z "${BUILD_STRING}" ]; then
+ echo "No Distribution was defined. Make sure BUILD_STRING is set on the environment before running this script"
+ exit 1
+fi
+
+if [ -z "${BUILDER_NAME}" ]; then
+ echo "No builder account and container name defined. Make sure BUILDER_NAME is set on the environment before running this script"
+ exit 1
+fi
+
+if [ -z "${BUILD_DISTRO}" ]; then
+ echo "No build distro information defined. Make sure BUILD_DISTRO is set on the environment before running this script"
+ exit 1
+fi
+
+if [ -z "${BUILD_RELEASE}" ]; then
+ echo "No build release information defined. Make sure BUILD_RELEASE is set on the environment before running this script"
+ exit 1
+fi
+
+if [ -z "${PACKAGE_TYPE}" ]; then
+ echo "No build release information defined. Make sure PACKAGE_TYPE is set on the environment before running this script"
+ exit 1
+fi
+
+# Detect architecture and load extra variables needed
+detect_arch_from_commit
+
+echo "Creating LXC container ${BUILDER_NAME}/${BUILD_STRING}/${BUILD_ARCH}...."
+
+case "${BUILD_ARCH}" in
+"all")
+ # i386
+ echo "Creating LXC Container for i386.."
+ export CONTAINER_NAME="${BUILDER_NAME}-${BUILD_DISTRO}${BUILD_RELEASE}-i386"
+ export LXC_CONTAINER_ROOT="/var/lib/lxc/${CONTAINER_NAME}/rootfs"
+ lxc-create -n "${CONTAINER_NAME}" -t "download" -- --dist "${BUILD_DISTRO}" --release "${BUILD_RELEASE}" --arch "i386" --no-validate
+
+ echo "Container(s) ready. Configuring container(s).."
+ .travis/package_management/configure_${PACKAGE_TYPE}_lxc_environment.py "${CONTAINER_NAME}"
+
+ # amd64
+ echo "Creating LXC Container for amd64.."
+ export CONTAINER_NAME="${BUILDER_NAME}-${BUILD_DISTRO}${BUILD_RELEASE}-amd64"
+ export LXC_CONTAINER_ROOT="/var/lib/lxc/${CONTAINER_NAME}/rootfs"
+ lxc-create -n "${CONTAINER_NAME}" -t "download" -- --dist "${BUILD_DISTRO}" --release "${BUILD_RELEASE}" --arch "amd64" --no-validate
+
+ echo "Container(s) ready. Configuring container(s).."
+ .travis/package_management/configure_${PACKAGE_TYPE}_lxc_environment.py "${CONTAINER_NAME}"
+
+ # arm64
+ echo "Creating LXC Container for arm64.."
+ export CONTAINER_NAME="${BUILDER_NAME}-${BUILD_DISTRO}${BUILD_RELEASE}-arm64"
+ export LXC_CONTAINER_ROOT="/var/lib/lxc/${CONTAINER_NAME}/rootfs"
+ lxc-create -n "${CONTAINER_NAME}" -t "download" -- --dist "${BUILD_DISTRO}" --release "${BUILD_RELEASE}" --arch "arm64" --no-validate
+
+ echo "Container(s) ready. Configuring container(s).."
+ .travis/package_management/configure_${PACKAGE_TYPE}_lxc_environment.py "${CONTAINER_NAME}"
+ ;;
+"i386"|"amd64"|"arm64")
+ # amd64 or i386
+ echo "Creating LXC Container for ${BUILD_ARCH}.."
+ export CONTAINER_NAME="${BUILDER_NAME}-${BUILD_DISTRO}${BUILD_RELEASE}-${BUILD_ARCH}"
+ export LXC_CONTAINER_ROOT="/var/lib/lxc/${CONTAINER_NAME}/rootfs"
+ lxc-create -n "${CONTAINER_NAME}" -t "download" -- --dist "${BUILD_DISTRO}" --release "${BUILD_RELEASE}" --arch "${BUILD_ARCH}" --no-validate
+
+ echo "Container(s) ready. Configuring container(s).."
+ .travis/package_management/configure_${PACKAGE_TYPE}_lxc_environment.py "${CONTAINER_NAME}"
+ ;;
+*)
+ echo "Unknown BUILD_ARCH value '${BUILD_ARCH}' given, process failed"
+ exit 1
+ ;;
+esac
+
+echo "..LXC creation complete!"
diff --git a/.travis/package_management/functions.sh b/.travis/package_management/functions.sh
new file mode 100644
index 00000000..0c00d2fc
--- /dev/null
+++ b/.travis/package_management/functions.sh
@@ -0,0 +1,33 @@
+# no-shebang-needed-its-a-library
+#
+# Utility functions for packaging in travis CI
+#
+# Copyright: SPDX-License-Identifier: GPL-3.0-or-later
+#
+# Author : Pavlos Emm. Katsoulakis (paul@netdata.cloud)
+#shellcheck disable=SC2148
+set -e
+
+function detect_arch_from_commit {
+ case "${TRAVIS_COMMIT_MESSAGE}" in
+ "[Package amd64"*)
+ export BUILD_ARCH="amd64"
+ ;;
+ "[Package i386"*)
+ export BUILD_ARCH="i386"
+ ;;
+ "[Package ALL"*)
+ export BUILD_ARCH="all"
+ ;;
+ "[Package arm64"*)
+ export BUILD_ARCH="arm64"
+ ;;
+
+ *)
+ echo "Unknown build architecture in '${TRAVIS_COMMIT_MESSAGE}'. Assuming amd64"
+ export BUILD_ARCH="amd64"
+ ;;
+ esac
+
+ echo "Detected build architecture ${BUILD_ARCH}"
+}
diff --git a/.travis/package_management/old_package_purging.sh b/.travis/package_management/old_package_purging.sh
new file mode 100755
index 00000000..89ecd753
--- /dev/null
+++ b/.travis/package_management/old_package_purging.sh
@@ -0,0 +1,93 @@
+#!/usr/bin/env bash
+#
+# Script to handle package cloud retention policy
+# Our open source subscription is limited,
+# so we use this sript to control the number of packages maintained historically
+#
+# Dependencies:
+# - PACKAGE_CLOUD_RETENTION_DAYS
+# This is to indicate for how many days back we want to maintain the various RPM and DEB packages on package cloud
+#
+# Copyright : SPDX-License-Identifier: GPL-3.0-or-later
+#
+# Author : Pavlos Emm. Katsoulakis <paul@netdata.cloud>
+#
+set -e
+
+delete_files_for_version() {
+ local v="$1"
+
+ # Delete the selected filenames in version
+ FILES_IN_VERSION=$(jq --sort-keys --arg v "${v}" '.[] | select ( .version | contains($v))' "${PKG_LIST_FILE}" | grep filename | cut -d':' -f 2)
+
+ # Iterate through the files and delete them
+ for pkg in ${FILES_IN_VERSION/\\n/}; do
+ pkg=${pkg/,/}
+ pkg=${pkg/\"/}
+ pkg=${pkg/\"/}
+ echo "Attempting yank on ${pkg}.."
+ .travis/package_management/package_cloud_wrapper.sh yank "${PACKAGING_USER}/${DEPLOY_REPO}" "${pkg}" || echo "Nothing to yank or error on ${pkg}"
+ done
+}
+
+# If we are not in netdata git repo, at the top level directory, fail
+TOP_LEVEL=$(basename "$(git rev-parse --show-toplevel)")
+CWD=$(git rev-parse --show-cdup)
+if [ -n "$CWD" ] || [ ! "${TOP_LEVEL}" == "netdata" ]; then
+ echo "Run as .travis/package_management/$(basename "$0") from top level directory of netdata git repository"
+ echo "Old packages yanking cancelled"
+ exit 1
+fi
+
+if [ -z "${PACKAGING_USER}" ]; then
+ echo "No PACKAGING_USER variable found"
+ exit 1
+fi
+
+if [ -z "${DEPLOY_REPO}" ]; then
+ echo "No DEPLOY_REPO variable found"
+ exit 1
+fi
+
+if [ -z ${PKG_CLOUD_TOKEN} ]; then
+ echo "No PKG_CLOUD_TOKEN variable found"
+ exit 1
+fi
+
+if [ -z ${PACKAGE_CLOUD_RETENTION_DAYS} ]; then
+ echo "No PACKAGE_CLOUD_RETENTION_DAYS variable found"
+ exit 1
+fi
+
+TMP_DIR="$(mktemp -d /tmp/netdata-old-package-yanking-XXXXXX)"
+PKG_LIST_FILE="${TMP_DIR}/complete_package_list.json"
+DATE_EPOCH="1970-01-01"
+DATE_UNTIL_TO_DELETE=$(date --date="${PACKAGE_CLOUD_RETENTION_DAYS} day ago" +%Y-%m-%d)
+
+
+echo "Created temp directory: ${TMP_DIR}"
+echo "We will be purging contents up until ${DATE_UNTIL_TO_DELETE}"
+
+echo "Calling package could to retrieve all available packages on ${PACKAGING_USER}/${DEPLOY_REPO}"
+curl -sS "https://${PKG_CLOUD_TOKEN}:@packagecloud.io/api/v1/repos/${PACKAGING_USER}/${DEPLOY_REPO}/packages.json" > "${PKG_LIST_FILE}"
+
+# Get versions within the desired duration
+#
+VERSIONS_TO_PURGE=$(jq --arg s "${DATE_EPOCH}" --arg e "${DATE_UNTIL_TO_DELETE}" '
+[($s, $e) | strptime("%Y-%m-%d")[0:3]] as $r
+ | map(select(
+ (.created_at[:19] | strptime("%Y-%m-%dT%H:%M:%S")[0:3]) as $d
+ | $d >= $r[0] and $d <= $r[1]
+))' "${PKG_LIST_FILE}" | grep '"version":' | sort -u | sed -e 's/ //g' | cut -d':' -f2)
+
+echo "We will be deleting the following versions: ${VERSIONS_TO_PURGE}"
+for v in ${VERSIONS_TO_PURGE/\n//}; do
+ v=${v/\"/}
+ v=${v/\"/}
+ v=${v/,/}
+ echo "Remove all files for version $v"
+ delete_files_for_version "${v}"
+done
+
+# Done, clean up
+[ -d "${TMP_DIR}" ] && rm -rf "${TMP_DIR}"
diff --git a/.travis/package_management/package_cloud_wrapper.sh b/.travis/package_management/package_cloud_wrapper.sh
new file mode 100755
index 00000000..48a372d3
--- /dev/null
+++ b/.travis/package_management/package_cloud_wrapper.sh
@@ -0,0 +1,48 @@
+#!/usr/bin/env bash
+#
+# This is a tool to help removal of packages from packagecloud.io
+# It utilizes the package_cloud utility provided from packagecloud.io
+#
+# Depends on:
+# 1) package cloud gem (detects absence and installs it)
+#
+# Requires:
+# 1) PKG_CLOUD_TOKEN variable exported
+# 2) To properly install package_cloud when not found, it requires: ruby gcc gcc-c++ ruby-devel
+#
+# Copyright: SPDX-License-Identifier: GPL-3.0-or-later
+#
+# Author : Pavlos Emm. Katsoulakis (paul@netdata.cloud)
+#shellcheck disable=SC2068,SC2145
+set -e
+PKG_CLOUD_CONFIG="$HOME/.package_cloud_configuration.cfg"
+
+# If we are not in netdata git repo, at the top level directory, fail
+TOP_LEVEL=$(basename "$(git rev-parse --show-toplevel)")
+CWD=$(git rev-parse --show-cdup)
+if [ -n "$CWD" ] || [ ! "${TOP_LEVEL}" == "netdata" ]; then
+ echo "Run as .travis/package_management/$(basename "$0") from top level directory of netdata git repository"
+ echo "Docker build process aborted"
+ exit 1
+fi
+
+# Install dependency if not there
+if ! command -v package_cloud > /dev/null 2>&1; then
+ echo "No package cloud gem found, installing"
+ gem install -V package_cloud || (echo "Package cloud installation failed. you might want to check if required dependencies are there (ruby gcc gcc-c++ ruby-devel)" && exit 1)
+else
+ echo "Found package_cloud gem, continuing"
+fi
+
+# Check for required token and prepare config
+if [ -z "${PKG_CLOUD_TOKEN}" ]; then
+ echo "Please set PKG_CLOUD_TOKEN to be able to use ${0}"
+ exit 1
+fi
+echo "{\"url\":\"https://packagecloud.io\",\"token\":\"${PKG_CLOUD_TOKEN}\"}" > "${PKG_CLOUD_CONFIG}"
+
+echo "Executing package_cloud with config ${PKG_CLOUD_CONFIG} and parameters $@"
+package_cloud $@ --config="${PKG_CLOUD_CONFIG}"
+
+rm -rf "${PKG_CLOUD_CONFIG}"
+echo "Done!"
diff --git a/.travis/package_management/prepare_packages.sh b/.travis/package_management/prepare_packages.sh
new file mode 100755
index 00000000..12ed07cc
--- /dev/null
+++ b/.travis/package_management/prepare_packages.sh
@@ -0,0 +1,63 @@
+#!/usr/bin/env bash
+#
+# Utility that gathers generated packages,
+# puts them together in a local folder for deploy facility to pick up
+#
+# Copyright: SPDX-License-Identifier: GPL-3.0-or-later
+#
+# Author : Pavlos Emm. Katsoulakis (paul@netdata.cloud)
+#shellcheck disable=SC2068
+set -e
+
+# If we are not in netdata git repo, at the top level directory, fail
+TOP_LEVEL=$(basename "$(git rev-parse --show-toplevel)")
+CWD=$(git rev-parse --show-cdup)
+if [ -n "$CWD" ] || [ ! "${TOP_LEVEL}" == "netdata" ]; then
+ echo "Run as .travis/package_management/$(basename "$0") from top level directory of netdata git repository"
+ echo "Package preparation aborted"
+ exit 1
+fi
+
+export LXC_ROOT="/var/lib/lxc"
+
+# Go through the containers created for packaging and pick up all generated packages
+CREATED_CONTAINERS=$(ls -A "${LXC_ROOT}")
+for d in ${CREATED_CONTAINERS[@]}; do
+ echo "Picking up packaging contents from ${d}"
+
+ # Pick up any RPMS from builder
+ RPM_BUILD_PATH="${LXC_ROOT}/${d}/rootfs/home/${BUILDER_NAME}/rpmbuild"
+ if [ -d "${RPM_BUILD_PATH}" ]; then
+ echo "Checking folder ${RPM_BUILD_PATH} for RPMS and SRPMS"
+
+ if [ -d "${RPM_BUILD_PATH}/RPMS" ]; then
+ echo "Copying any RPMS in '${RPM_BUILD_PATH}', copying over the following:"
+ ls -ltrR "${RPM_BUILD_PATH}/RPMS"
+ [[ -d "${RPM_BUILD_PATH}/RPMS/x86_64" ]] && cp -r "${RPM_BUILD_PATH}"/RPMS/x86_64/* "${PACKAGES_DIRECTORY}"
+ [[ -d "${RPM_BUILD_PATH}/RPMS/i386" ]] && cp -r "${RPM_BUILD_PATH}"/RPMS/i386/* "${PACKAGES_DIRECTORY}"
+ [[ -d "${RPM_BUILD_PATH}/RPMS/i686" ]] && cp -r "${RPM_BUILD_PATH}"/RPMS/i686/* "${PACKAGES_DIRECTORY}"
+ fi
+
+ if [ -d "${RPM_BUILD_PATH}/SRPMS" ]; then
+ echo "Copying any SRPMS in '${RPM_BUILD_PATH}', copying over the following:"
+ ls -ltrR "${RPM_BUILD_PATH}/SRPMS"
+ [[ -d "${RPM_BUILD_PATH}/SRPMS/x86_64" ]] && cp -r "${RPM_BUILD_PATH}"/SRPMS/x86_64/* "${PACKAGES_DIRECTORY}"
+ [[ -d "${RPM_BUILD_PATH}/SRPMS/i386" ]] && cp -r "${RPM_BUILD_PATH}"/SRPMS/i386/* "${PACKAGES_DIRECTORY}"
+ [[ -d "${RPM_BUILD_PATH}/SRPMS/i686" ]] && cp -r "${RPM_BUILD_PATH}"/SRPMS/i686/* "${PACKAGES_DIRECTORY}"
+ fi
+ else
+ DEB_BUILD_PATH="${LXC_ROOT}/${d}/rootfs/home/${BUILDER_NAME}"
+ echo "Checking folder ${DEB_BUILD_PATH} for DEB packages"
+ if [ -d "${DEB_BUILD_PATH}" ]; then
+ cp "${DEB_BUILD_PATH}"/netdata*.ddeb "${PACKAGES_DIRECTORY}" || echo "Could not copy any .ddeb files"
+ cp "${DEB_BUILD_PATH}"/netdata*.deb "${PACKAGES_DIRECTORY}" || echo "Could not copy any .deb files"
+ cp "${DEB_BUILD_PATH}"/netdata*.buildinfo "${PACKAGES_DIRECTORY}" || echo "Could not copy any .buildinfo files"
+ cp "${DEB_BUILD_PATH}"/netdata*.changes "${PACKAGES_DIRECTORY}" || echo "Could not copy any .changes files"
+ else
+ echo "Folder ${DEB_BUILD_PATH} does not exist or not a directory, nothing to do for package preparation"
+ fi
+ fi
+done
+
+chmod -R 777 "${PACKAGES_DIRECTORY}"
+echo "Packaging contents ready to ship!"
diff --git a/.travis/package_management/trigger_deb_lxc_build.py b/.travis/package_management/trigger_deb_lxc_build.py
new file mode 100755
index 00000000..464a7715
--- /dev/null
+++ b/.travis/package_management/trigger_deb_lxc_build.py
@@ -0,0 +1,86 @@
+#!/usr/bin/env python3
+#
+# This script is responsible for running the RPM build on the running container
+#
+# Copyright: SPDX-License-Identifier: GPL-3.0-or-later
+#
+# Author : Pavlos Emm. Katsoulakis <paul@netdata.cloud>
+
+import common
+import os
+import sys
+import lxc
+
+if len(sys.argv) != 2:
+ print('You need to provide a container name to get things started')
+ sys.exit(1)
+container_name=sys.argv[1]
+
+# Load the container, break if its not there
+print("Starting up container %s" % container_name)
+container = lxc.Container(container_name)
+if not container.defined:
+ raise Exception("Container %s does not exist!" % container_name)
+
+# Check if the container is running, attempt to start it up in case its not running
+if not container.running or not container.state == "RUNNING":
+ print('Container %s is not running, attempt to start it up' % container_name)
+
+ # Start the container
+ if not container.start():
+ raise Exception("Failed to start the container")
+
+ if not container.running or not container.state == "RUNNING":
+ raise Exception('Container %s is not running, configuration process aborted ' % container_name)
+
+# Wait for connectivity
+if not container.get_ips(timeout=30):
+ raise Exception("Timeout while waiting for container")
+
+build_path = "/home/%s" % os.environ['BUILDER_NAME']
+
+print("Setting up EMAIL and DEBFULLNAME variables required by the build tools")
+os.environ["EMAIL"] = "bot@netdata.cloud"
+os.environ["DEBFULLNAME"] = "Netdata builder"
+
+# Run the build process on the container
+new_version, tag = common.fetch_version(os.environ['BUILD_VERSION'])
+print("Starting DEB build process for version %s" % new_version)
+
+netdata_tarball = "%s/netdata-%s.tar.gz" % (build_path, new_version)
+unpacked_netdata = netdata_tarball.replace(".tar.gz", "")
+
+print("Extracting tarball %s" % netdata_tarball)
+common.run_command(container, ["sudo", "-u", os.environ['BUILDER_NAME'], "tar", "xf", netdata_tarball, "-C", build_path])
+
+print("Checking version consistency")
+since_version = os.environ["LATEST_RELEASE_VERSION"]
+if str(since_version).replace('v', '') == str(new_version) and str(new_version).count('.') == 2:
+ s = since_version.split('.')
+ if int(s[2]) > 0:
+ patch_prev = str(int(s[2]) - 1)
+ since_version = s[0] + '.' + s[1] + '.' + patch_prev
+ else:
+ prev = str(int(s[1]) - 1)
+ since_version = s[0] + '.' + prev + '.' + s[2]
+
+ print("We seem to be building a new stable release, reduce by one since_version option. New since_version:%s" % since_version)
+
+print("Fixing changelog tags")
+changelog_in_host = "contrib/debian/changelog"
+common.run_command_in_host(['sed', '-i', 's/PREVIOUS_PACKAGE_VERSION/%s-1/g' % since_version.replace("v", ""), changelog_in_host])
+common.run_command_in_host(['sed', '-i', 's/PREVIOUS_PACKAGE_DATE/%s/g' % os.environ["LATEST_RELEASE_DATE"], changelog_in_host])
+
+print("Executing gbp dch command..")
+common.run_command_in_host(['gbp', 'dch', '--release', '--ignore-branch', '--spawn-editor=snapshot', '--since=%s' % since_version, '--new-version=%s' % new_version])
+
+print("Copying over changelog to the destination machine")
+common.run_command_in_host(['sudo', 'cp', 'debian/changelog', "%s/%s/netdata-%s/contrib/debian/" % (os.environ['LXC_CONTAINER_ROOT'], build_path, new_version)])
+
+print("Running debian build script since %s" % since_version)
+common.run_command(container, ["sudo", "-u", os.environ['BUILDER_NAME'], "%s/build.sh" % build_path, unpacked_netdata, new_version])
+
+print("Listing contents on build path")
+common.run_command(container, ["sudo", "-u", os.environ['BUILDER_NAME'], "ls", "-ltr", build_path])
+
+print('Done!')
diff --git a/.travis/package_management/trigger_rpm_lxc_build.py b/.travis/package_management/trigger_rpm_lxc_build.py
new file mode 100755
index 00000000..f9e109c7
--- /dev/null
+++ b/.travis/package_management/trigger_rpm_lxc_build.py
@@ -0,0 +1,55 @@
+#!/usr/bin/env python3
+#
+# This script is responsible for running the RPM build on the running container
+#
+# Copyright: SPDX-License-Identifier: GPL-3.0-or-later
+#
+# Author : Pavlos Emm. Katsoulakis <paul@netdata.cloud>
+
+import common
+import os
+import sys
+import lxc
+
+if len(sys.argv) != 2:
+ print('You need to provide a container name to get things started')
+ sys.exit(1)
+container_name=sys.argv[1]
+
+# Load the container, break if its not there
+print("Starting up container %s" % container_name)
+container = lxc.Container(container_name)
+if not container.defined:
+ raise Exception("Container %s does not exist!" % container_name)
+
+# Check if the container is running, attempt to start it up in case its not running
+if not container.running or not container.state == "RUNNING":
+ print('Container %s is not running, attempt to start it up' % container_name)
+
+ # Start the container
+ if not container.start():
+ raise Exception("Failed to start the container")
+
+ if not container.running or not container.state == "RUNNING":
+ raise Exception('Container %s is not running, configuration process aborted ' % container_name)
+
+# Wait for connectivity
+if not container.get_ips(timeout=30):
+ raise Exception("Timeout while waiting for container")
+
+print("Adding builder specific dependencies to the LXC container")
+common.run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "rpm-build"])
+common.run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "rpm-devel"])
+common.run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "rpmlint"])
+common.run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "make"])
+common.run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "python"])
+common.run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "bash"])
+common.run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "diffutils"])
+common.run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "patch"])
+common.run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "rpmdevtools"])
+
+# Run the build process on the container
+print("Starting RPM build process")
+common.run_command(container, ["sudo", "-u", os.environ['BUILDER_NAME'], "rpmbuild", "-ba", "--rebuild", "/home/%s/rpmbuild/SPECS/netdata.spec" % os.environ['BUILDER_NAME']])
+
+print('Done!')
diff --git a/.travis/package_management/yank_stale_pkg.sh b/.travis/package_management/yank_stale_pkg.sh
new file mode 100755
index 00000000..3f766971
--- /dev/null
+++ b/.travis/package_management/yank_stale_pkg.sh
@@ -0,0 +1,35 @@
+#!/usr/bin/env bash
+#
+# This script is responsible for the removal of stale RPM/DEB files.
+# It runs on the pre-deploy step and takes care of the removal of the files
+# prior to the upload of the freshly built ones
+#
+# Copyright: SPDX-License-Identifier: GPL-3.0-or-later
+#
+# Author : Pavlos Emm. Katsoulakis (paul@netdata.cloud)
+#shellcheck disable=SC2010,SC2068
+set -e
+
+# If we are not in netdata git repo, at the top level directory, fail
+TOP_LEVEL=$(basename "$(git rev-parse --show-toplevel)")
+CWD=$(git rev-parse --show-cdup)
+if [ -n "$CWD" ] || [ ! "${TOP_LEVEL}" == "netdata" ]; then
+ echo "Run as .travis/package_management/$(basename "$0") from top level directory of netdata git repository"
+ echo "Package yanking cancelled"
+ exit 1
+fi
+
+PACKAGES_DIR="$1"
+DISTRO="$2"
+PACKAGES_LIST="$(ls -AR "${PACKAGES_DIR}" | grep -e '\.rpm' -e '\.deb' -e '\.ddeb' )"
+
+if [ ! -d "${PACKAGES_DIR}" ] || [ -z "${PACKAGES_LIST}" ]; then
+ echo "Folder ${PACKAGES_DIR} does not seem to be a valid directory or is empty. No packages to check for yanking"
+ exit 1
+fi
+
+for pkg in ${PACKAGES_LIST[@]}; do
+ echo "Attempting yank on ${pkg}.."
+ .travis/package_management/package_cloud_wrapper.sh yank "${PACKAGING_USER}/${DEPLOY_REPO}/${DISTRO}" "${pkg}" || echo "Nothing to yank or error on ${pkg}"
+done
+