diff options
Diffstat (limited to '.travis/package_management')
| -rwxr-xr-x | .travis/package_management/build_package_in_container.sh | 82 | ||||
| -rwxr-xr-x | .travis/package_management/common.py | 46 | ||||
| -rwxr-xr-x | .travis/package_management/configure_deb_lxc_environment.py | 65 | ||||
| -rwxr-xr-x | .travis/package_management/configure_rpm_lxc_environment.py | 159 | ||||
| -rwxr-xr-x | .travis/package_management/create_lxc_for_build.sh | 101 | ||||
| -rw-r--r-- | .travis/package_management/functions.sh | 33 | ||||
| -rwxr-xr-x | .travis/package_management/package_cloud_wrapper.sh | 48 | ||||
| -rwxr-xr-x | .travis/package_management/prepare_packages.sh | 56 | ||||
| -rwxr-xr-x | .travis/package_management/trigger_deb_lxc_build.py | 51 | ||||
| -rwxr-xr-x | .travis/package_management/trigger_rpm_lxc_build.py | 55 | ||||
| -rwxr-xr-x | .travis/package_management/yank_stale_rpm.sh | 35 |
11 files changed, 731 insertions, 0 deletions
diff --git a/.travis/package_management/build_package_in_container.sh b/.travis/package_management/build_package_in_container.sh new file mode 100755 index 000000000..95a68e7a8 --- /dev/null +++ b/.travis/package_management/build_package_in_container.sh @@ -0,0 +1,82 @@ +#!/usr/bin/env bash +# +# Entry point for package build process +# +# Copyright: SPDX-License-Identifier: GPL-3.0-or-later +# +# Author : Pavlos Emm. Katsoulakis (paul@netdata.cloud) +#shellcheck disable=SC1091 +set -e + +# If we are not in netdata git repo, at the top level directory, fail +TOP_LEVEL=$(basename "$(git rev-parse --show-toplevel)") +CWD=$(git rev-parse --show-cdup) +if [ -n "$CWD" ] || [ ! "${TOP_LEVEL}" == "netdata" ]; then + echo "Run as .travis/package_management/$(basename "$0") from top level directory of netdata git repository" + echo "Docker build process aborted" + exit 1 +fi + +source .travis/package_management/functions.sh || (echo "Failed to load packaging library" && exit 1) + +# Check for presence of mandatory environment variables +if [ -z "${BUILD_STRING}" ]; then + echo "No Distribution was defined. Make sure BUILD_STRING is set on the environment before running this script" + exit 1 +fi + +if [ -z "${BUILDER_NAME}" ]; then + echo "No builder account and container name defined. Make sure BUILDER_NAME is set on the environment before running this script" + exit 1 +fi + +if [ -z "${BUILD_DISTRO}" ]; then + echo "No build distro information defined. Make sure BUILD_DISTRO is set on the environment before running this script" + exit 1 +fi + +if [ -z "${BUILD_RELEASE}" ]; then + echo "No build release information defined. Make sure BUILD_RELEASE is set on the environment before running this script" + exit 1 +fi + +if [ -z "${PACKAGE_TYPE}" ]; then + echo "No build release information defined. Make sure PACKAGE_TYPE is set on the environment before running this script" + exit 1 +fi + +# Detect architecture and load extra variables needed +detect_arch_from_commit + +case "${BUILD_ARCH}" in +"all") + echo "* * * Building all architectures, amd64 and i386 * * *" + echo "Building for amd64.." + export CONTAINER_NAME="${BUILDER_NAME}-${BUILD_DISTRO}${BUILD_RELEASE}-amd64" + export LXC_CONTAINER_ROOT="/var/lib/lxc/${CONTAINER_NAME}/rootfs" + .travis/package_management/trigger_${PACKAGE_TYPE}_lxc_build.py "${CONTAINER_NAME}" + + echo "Building for arm64.." + export CONTAINER_NAME="${BUILDER_NAME}-${BUILD_DISTRO}${BUILD_RELEASE}-arm64" + export LXC_CONTAINER_ROOT="/var/lib/lxc/${CONTAINER_NAME}/rootfs" + .travis/package_management/trigger_${PACKAGE_TYPE}_lxc_build.py "${CONTAINER_NAME}" + + echo "Building for i386.." + export CONTAINER_NAME="${BUILDER_NAME}-${BUILD_DISTRO}${BUILD_RELEASE}-i386" + export LXC_CONTAINER_ROOT="/var/lib/lxc/${CONTAINER_NAME}/rootfs" + .travis/package_management/trigger_${PACKAGE_TYPE}_lxc_build.py "${CONTAINER_NAME}" + + ;; +"amd64"|"arm64"|"i386") + echo "Building for ${BUILD_ARCH}.." + export CONTAINER_NAME="${BUILDER_NAME}-${BUILD_DISTRO}${BUILD_RELEASE}-${BUILD_ARCH}" + export LXC_CONTAINER_ROOT="/var/lib/lxc/${CONTAINER_NAME}/rootfs" + .travis/package_management/trigger_${PACKAGE_TYPE}_lxc_build.py "${CONTAINER_NAME}" + ;; +*) + echo "Unknown build architecture '${BUILD_ARCH}', nothing to do for build" + exit 1 + ;; +esac + +echo "Build process completed!" diff --git a/.travis/package_management/common.py b/.travis/package_management/common.py new file mode 100755 index 000000000..6cf59293d --- /dev/null +++ b/.travis/package_management/common.py @@ -0,0 +1,46 @@ +# +# +# Python library with commonly used functions within the package management scope +# +# Author : Pavlos Emm. Katsoulakis <paul@netdata.cloud> + +import lxc +import subprocess + +def replace_tag(tag_name, spec, new_tag_content): + print("Fixing tag %s in %s" % (tag_name, spec)) + + ifp = open(spec, "r") + config = ifp.readlines() + ifp.close() + + source_line = -1 + for line in config: + if str(line).count(tag_name + ":") > 0: + source_line = config.index(line) + print("Found line: %s in item %d" % (line, source_line)) + break + + if source_line >= 0: + print("Replacing line %s with %s in spec file" %(config[source_line], new_tag_content)) + config[source_line] = "%s: %s\n" % (tag_name, new_tag_content) + config_str = ''.join(config) + ofp = open(spec, 'w') + ofp.write(config_str) + ofp.close() + +def run_command(container, command): + print("Running command: %s" % command) + command_result = container.attach_wait(lxc.attach_run_command, command) + + if command_result != 0: + raise Exception("Command failed with exit code %d" % command_result) + +def run_command_in_host(cmd): + print("Issue command in host: %s" % str(cmd)) + + proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + o, e = proc.communicate() + print('Output: ' + o.decode('ascii')) + print('Error: ' + e.decode('ascii')) + print('code: ' + str(proc.returncode)) diff --git a/.travis/package_management/configure_deb_lxc_environment.py b/.travis/package_management/configure_deb_lxc_environment.py new file mode 100755 index 000000000..58999ad38 --- /dev/null +++ b/.travis/package_management/configure_deb_lxc_environment.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python3 +# +# Prepare the build environment within the container +# The script attaches to the running container and does the following: +# 1) Create the container +# 2) Start the container up +# 3) Create the builder user +# 4) Prepare the environment for DEB build +# +# Copyright: SPDX-License-Identifier: GPL-3.0-or-later +# +# Author : Pavlos Emm. Katsoulakis <paul@netdata.cloud> + +import common +import os +import sys +import lxc + +if len(sys.argv) != 2: + print('You need to provide a container name to get things started') + sys.exit(1) +container_name=sys.argv[1] + +# Setup the container object +print("Defining container %s" % container_name) +container = lxc.Container(container_name) +if not container.defined: + raise Exception("Container %s not defined!" % container_name) + +# Start the container +if not container.start(): + raise Exception("Failed to start the container") + +if not container.running or not container.state == "RUNNING": + raise Exception('Container %s is not running, configuration process aborted ' % container_name) + +# Wait for connectivity +print("Waiting for container connectivity to start configuration sequence") +if not container.get_ips(timeout=30): + raise Exception("Timeout while waiting for container") + +# Run the required activities now +# 1. Create the builder user +print("1. Adding user %s" % os.environ['BUILDER_NAME']) +common.run_command(container, ["useradd", "-m", os.environ['BUILDER_NAME']]) + +# Fetch package dependencies for the build +print("2. Installing package dependencies within LXC container") +common.run_command(container, ["apt-get", "update", "-y"]) +common.run_command(container, ["apt-get", "install", "-y", "sudo"]) +common.run_command(container, ["apt-get", "install", "-y", "wget"]) +common.run_command(container, ["apt-get", "install", "-y", "bash"]) +common.run_command(container, ["wget", "-T", "15", "-O", "~/.install-required-packages.sh", "https://raw.githubusercontent.com/netdata/netdata-demo-site/master/install-required-packages.sh"]) +common.run_command(container, ["bash", "~/.install-required-packages.sh", "netdata", "--dont-wait", "--non-interactive"]) + +# Download the source +dest_archive="/home/%s/netdata-%s.tar.gz" % (os.environ['BUILDER_NAME'],os.environ['BUILD_VERSION']) +release_url="https://github.com/netdata/netdata/releases/download/%s/netdata-%s.tar.gz" % (os.environ['BUILD_VERSION'], os.environ['BUILD_VERSION']) +print("3. Fetch netdata source (%s -> %s)" % (release_url, dest_archive)) +common.run_command(container, ["sudo", "-u", os.environ['BUILDER_NAME'], "wget", "-T", "15", "--output-document=" + dest_archive, release_url]) + +print("4. Extracting directory contents to /home " + os.environ['BUILDER_NAME']) +common.run_command(container, ["sudo", "-u", os.environ['BUILDER_NAME'], "tar", "xf", dest_archive, "-C", "/home/" + os.environ['BUILDER_NAME']]) + +print("Done!") diff --git a/.travis/package_management/configure_rpm_lxc_environment.py b/.travis/package_management/configure_rpm_lxc_environment.py new file mode 100755 index 000000000..644e027b7 --- /dev/null +++ b/.travis/package_management/configure_rpm_lxc_environment.py @@ -0,0 +1,159 @@ +#!/usr/bin/env python3 +# +# +# Prepare the build environment within the container +# The script attaches to the running container and does the following: +# 1) Create the container +# 2) Start the container up +# 3) Create the builder user +# 4) Prepare the environment for RPM build +# +# Copyright: SPDX-License-Identifier: GPL-3.0-or-later +# +# Author : Pavlos Emm. Katsoulakis <paul@netdata.cloud> + +import common +import os +import sys +import lxc + +if len(sys.argv) != 2: + print('You need to provide a container name to get things started') + sys.exit(1) +container_name=sys.argv[1] + +# Setup the container object +print("Defining container %s" % container_name) +container = lxc.Container(container_name) +if not container.defined: + raise Exception("Container %s not defined!" % container_name) + +# Start the container +if not container.start(): + raise Exception("Failed to start the container") + +if not container.running or not container.state == "RUNNING": + raise Exception('Container %s is not running, configuration process aborted ' % container_name) + +# Wait for connectivity +print("Waiting for container connectivity to start configuration sequence") +if not container.get_ips(timeout=30): + raise Exception("Timeout while waiting for container") + +# Run the required activities now +# Create the builder user +print("1. Adding user %s" % os.environ['BUILDER_NAME']) +common.run_command(container, ["useradd", "-m", os.environ['BUILDER_NAME']]) + +# Fetch package dependencies for the build +print("2. Installing package dependencies within LXC container") +if str(os.environ["REPO_TOOL"]).count("zypper") == 1: + common.run_command(container, [os.environ["REPO_TOOL"], "clean", "-a"]) + common.run_command(container, [os.environ["REPO_TOOL"], "--no-gpg-checks", "update", "-y"]) + common.run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "json-glib-devel"]) + +elif str(os.environ["REPO_TOOL"]).count("yum") == 1: + common.run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "json-c-devel"]) + common.run_command(container, [os.environ["REPO_TOOL"], "clean", "all"]) + common.run_command(container, [os.environ["REPO_TOOL"], "update", "-y"]) + common.run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "epel-release"]) +else: + common.run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "json-c-devel"]) + common.run_command(container, [os.environ["REPO_TOOL"], "update", "-y"]) + +common.run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "sudo"]) +common.run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "wget"]) +common.run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "bash"]) +common.run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "freeipmi-devel"]) +common.run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "cups-devel"]) + +# Exceptional cases, not available everywhere +# + +# Not on Centos-7 +if os.environ["BUILD_STRING"].count("el/7") <= 0: + common.run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "libnetfilter_acct-devel"]) + +# Not on Centos-6 +if os.environ["BUILD_STRING"].count("el/6") <= 0: + common.run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "autoconf-archive"]) + +common.run_command(container, ["wget", "-T", "15", "-O", "/home/%s/.install-required-packages.sh" % (os.environ['BUILDER_NAME']), "https://raw.githubusercontent.com/netdata/netdata-demo-site/master/install-required-packages.sh"]) +common.run_command(container, ["bash", "/home/%s/.install-required-packages.sh" % (os.environ['BUILDER_NAME']), "netdata", "--dont-wait", "--non-interactive"]) + +print("3. Setting up macros") +common.run_command(container, ["sudo", "-u", os.environ['BUILDER_NAME'], "/bin/echo", "'%_topdir %(echo /home/" + os.environ['BUILDER_NAME'] + ")/rpmbuild' > /home/" + os.environ['BUILDER_NAME'] + "/.rpmmacros"]) + +print("4. Create rpmbuild directory") +common.run_command(container, ["sudo", "-u", os.environ['BUILDER_NAME'], "mkdir", "-p", "/home/" + os.environ['BUILDER_NAME'] + "/rpmbuild/BUILD"]) +common.run_command(container, ["sudo", "-u", os.environ['BUILDER_NAME'], "mkdir", "-p", "/home/" + os.environ['BUILDER_NAME'] + "/rpmbuild/RPMS"]) +common.run_command(container, ["sudo", "-u", os.environ['BUILDER_NAME'], "mkdir", "-p", "/home/" + os.environ['BUILDER_NAME'] + "/rpmbuild/SOURCES"]) +common.run_command(container, ["sudo", "-u", os.environ['BUILDER_NAME'], "mkdir", "-p", "/home/" + os.environ['BUILDER_NAME'] + "/rpmbuild/SPECS"]) +common.run_command(container, ["sudo", "-u", os.environ['BUILDER_NAME'], "mkdir", "-p", "/home/" + os.environ['BUILDER_NAME'] + "/rpmbuild/SRPMS"]) +common.run_command(container, ["sudo", "-u", os.environ['BUILDER_NAME'], "ls", "-ltrR", "/home/" + os.environ['BUILDER_NAME'] + "/rpmbuild"]) + +# Download the source +rpm_friendly_version="" +dest_archive="" +download_url="" +spec_file="/home/%s/rpmbuild/SPECS/netdata.spec" % os.environ['BUILDER_NAME'] + +# TODO: Checksum validations +if str(os.environ['BUILD_VERSION']).count(".latest") == 1: + version_list=str(os.environ['BUILD_VERSION']).replace('v', '').split('.') + rpm_friendly_version='.'.join(version_list[0:3]) + "." + version_list[3] + + print("Building latest nightly version of netdata..(%s)" % os.environ['BUILD_VERSION']) + dest_archive="/home/%s/rpmbuild/SOURCES/netdata-%s.tar.gz" % (os.environ['BUILDER_NAME'], rpm_friendly_version) + + print("5. Preparing local latest implementation tarball for version %s" % rpm_friendly_version) + tar_file = os.environ['LXC_CONTAINER_ROOT'] + dest_archive + + print("5.1 Tagging the code with latest version: %s" % rpm_friendly_version) + common.run_command_in_host(['git', 'tag', '-a', rpm_friendly_version, '-m', 'Tagging while packaging on %s' % os.environ["CONTAINER_NAME"]]) + + print("5.2 Run autoreconf -ivf") + common.run_command_in_host(['autoreconf', '-ivf']) + + print("5.3 Run configure") + common.run_command_in_host(['./configure', '--with-math', '--with-zlib', '--with-user=netdata']) + + print("5.4 Run make dist") + common.run_command_in_host(['make', 'dist']) + + print("5.5 Copy generated tarbal to desired path") + if os.path.exists('netdata-%s.tar.gz' % rpm_friendly_version): + common.run_command_in_host(['sudo', 'cp', 'netdata-%s.tar.gz' % rpm_friendly_version, tar_file]) + + print("5.6 Fixing permissions on tarball") + common.run_command_in_host(['sudo', 'chmod', '777', tar_file]) + else: + print("I could not find (%s) on the disk, stopping the build. Kindly check the logs and try again" % 'netdata-%s.tar.gz' % rpm_friendly_version) + sys.exit(1) + + # Extract the spec file in place + print("6. Extract spec file from the source") + common.run_command_in_host(['sudo', 'cp', 'netdata.spec', os.environ['LXC_CONTAINER_ROOT'] + spec_file]) + common.run_command_in_host(['sudo', 'chmod', '777', os.environ['LXC_CONTAINER_ROOT'] + spec_file]) + + print("7. Temporary hack: Change Source0 to %s on spec file %s" % (dest_archive, spec_file)) + common.replace_tag("Source0", os.environ['LXC_CONTAINER_ROOT'] + spec_file, tar_file) +else: + rpm_friendly_version = os.environ['BUILD_VERSION'] + + print("Building latest stable version of netdata.. (%s)" % os.environ['BUILD_VERSION']) + dest_archive="/home/%s/rpmbuild/SOURCES/netdata-%s.tar.gz" % (os.environ['BUILDER_NAME'],os.environ['BUILD_VERSION']) + download_url="https://github.com/netdata/netdata/releases/download/%s/netdata-%s.tar.gz" % (os.environ['BUILD_VERSION'], os.environ['BUILD_VERSION']) + + print("5. Fetch netdata source into the repo structure(%s -> %s)" % (download_url, dest_archive)) + tar_file="%s/netdata-%s.tar.gz" % (os.path.dirname(dest_archive), rpm_friendly_version) + common.run_command(container, ["sudo", "-u", os.environ['BUILDER_NAME'], "wget", "-T", "15", "--output-document=" + dest_archive, download_url]) + + print("6.Extract spec file from the source") + common.run_command(container, ["sudo", "-u", os.environ['BUILDER_NAME'], "tar", "--to-command=cat > %s" % spec_file, "-xvf", dest_archive, "netdata-%s/netdata.spec" % os.environ['BUILD_VERSION']]) + + print("7. Temporary hack: Adjust version string on the spec file (%s) to %s and Source0 to %s" % (os.environ['LXC_CONTAINER_ROOT'] + spec_file, rpm_friendly_version, download_url)) + common.replace_tag("Version", os.environ['LXC_CONTAINER_ROOT'] + spec_file, rpm_friendly_version) + common.replace_tag("Source0", os.environ['LXC_CONTAINER_ROOT'] + spec_file, tar_file) + +print('Done!') diff --git a/.travis/package_management/create_lxc_for_build.sh b/.travis/package_management/create_lxc_for_build.sh new file mode 100755 index 000000000..ae855a742 --- /dev/null +++ b/.travis/package_management/create_lxc_for_build.sh @@ -0,0 +1,101 @@ +#!/usr/bin/env bash +# +# This script generates an LXC container and starts it up +# Once the script completes successfully, a container has become available for usage +# The container image to be used and the container name to be set, are part of variables +# that must be present for the script to work +# +# Copyright: SPDX-License-Identifier: GPL-3.0-or-later +# +# Author : Pavlos Emm. Katsoulakis (paul@netdata.cloud) +# shellcheck disable=SC1091 +set -e + +source .travis/package_management/functions.sh || (echo "Failed to load packaging library" && exit 1) + +# If we are not in netdata git repo, at the top level directory, fail +TOP_LEVEL=$(basename "$(git rev-parse --show-toplevel)") +CWD=$(git rev-parse --show-cdup) +if [ -n "$CWD" ] || [ ! "${TOP_LEVEL}" == "netdata" ]; then + echo "Run as .travis/package_management/$(basename "$0") from top level directory of netdata git repository" + echo "LXC Container creation aborted" + exit 1 +fi + +# Check for presence of mandatory environment variables +if [ -z "${BUILD_STRING}" ]; then + echo "No Distribution was defined. Make sure BUILD_STRING is set on the environment before running this script" + exit 1 +fi + +if [ -z "${BUILDER_NAME}" ]; then + echo "No builder account and container name defined. Make sure BUILDER_NAME is set on the environment before running this script" + exit 1 +fi + +if [ -z "${BUILD_DISTRO}" ]; then + echo "No build distro information defined. Make sure BUILD_DISTRO is set on the environment before running this script" + exit 1 +fi + +if [ -z "${BUILD_RELEASE}" ]; then + echo "No build release information defined. Make sure BUILD_RELEASE is set on the environment before running this script" + exit 1 +fi + +if [ -z "${PACKAGE_TYPE}" ]; then + echo "No build release information defined. Make sure PACKAGE_TYPE is set on the environment before running this script" + exit 1 +fi + +# Detect architecture and load extra variables needed +detect_arch_from_commit + +echo "Creating LXC container ${BUILDER_NAME}/${BUILD_STRING}/${BUILD_ARCH}...." + +case "${BUILD_ARCH}" in +"all") + # i386 + echo "Creating LXC Container for i386.." + export CONTAINER_NAME="${BUILDER_NAME}-${BUILD_DISTRO}${BUILD_RELEASE}-i386" + export LXC_CONTAINER_ROOT="/var/lib/lxc/${CONTAINER_NAME}/rootfs" + lxc-create -n "${CONTAINER_NAME}" -t "download" -- --dist "${BUILD_DISTRO}" --release "${BUILD_RELEASE}" --arch "i386" --no-validate + + echo "Container(s) ready. Configuring container(s).." + .travis/package_management/configure_${PACKAGE_TYPE}_lxc_environment.py "${CONTAINER_NAME}" + + # amd64 + echo "Creating LXC Container for amd64.." + export CONTAINER_NAME="${BUILDER_NAME}-${BUILD_DISTRO}${BUILD_RELEASE}-amd64" + export LXC_CONTAINER_ROOT="/var/lib/lxc/${CONTAINER_NAME}/rootfs" + lxc-create -n "${CONTAINER_NAME}" -t "download" -- --dist "${BUILD_DISTRO}" --release "${BUILD_RELEASE}" --arch "amd64" --no-validate + + echo "Container(s) ready. Configuring container(s).." + .travis/package_management/configure_${PACKAGE_TYPE}_lxc_environment.py "${CONTAINER_NAME}" + + # arm64 + echo "Creating LXC Container for arm64.." + export CONTAINER_NAME="${BUILDER_NAME}-${BUILD_DISTRO}${BUILD_RELEASE}-arm64" + export LXC_CONTAINER_ROOT="/var/lib/lxc/${CONTAINER_NAME}/rootfs" + lxc-create -n "${CONTAINER_NAME}" -t "download" -- --dist "${BUILD_DISTRO}" --release "${BUILD_RELEASE}" --arch "arm64" --no-validate + + echo "Container(s) ready. Configuring container(s).." + .travis/package_management/configure_${PACKAGE_TYPE}_lxc_environment.py "${CONTAINER_NAME}" + ;; +"i386"|"amd64"|"arm64") + # AMD64 or i386 + echo "Creating LXC Container for ${BUILD_ARCH}.." + export CONTAINER_NAME="${BUILDER_NAME}-${BUILD_DISTRO}${BUILD_RELEASE}-${BUILD_ARCH}" + export LXC_CONTAINER_ROOT="/var/lib/lxc/${CONTAINER_NAME}/rootfs" + lxc-create -n "${CONTAINER_NAME}" -t "download" -- --dist "${BUILD_DISTRO}" --release "${BUILD_RELEASE}" --arch "${BUILD_ARCH}" --no-validate + + echo "Container(s) ready. Configuring container(s).." + .travis/package_management/configure_${PACKAGE_TYPE}_lxc_environment.py "${CONTAINER_NAME}" + ;; +*) + echo "Unknown BUILD_ARCH value '${BUILD_ARCH}' given, process failed" + exit 1 + ;; +esac + +echo "..LXC creation complete!" diff --git a/.travis/package_management/functions.sh b/.travis/package_management/functions.sh new file mode 100644 index 000000000..9a467ffe1 --- /dev/null +++ b/.travis/package_management/functions.sh @@ -0,0 +1,33 @@ +# no-shebang-needed-its-a-library +# +# Utility functions for packaging in travis CI +# +# Copyright: SPDX-License-Identifier: GPL-3.0-or-later +# +# Author : Pavlos Emm. Katsoulakis (paul@netdata.cloud) +#shellcheck disable=SC2148 +set -e + +function detect_arch_from_commit { + case "${TRAVIS_COMMIT_MESSAGE}" in + "[Package AMD64"*) + export BUILD_ARCH="amd64" + ;; + "[Package i386"*) + export BUILD_ARCH="i386" + ;; + "[Package ALL"*) + export BUILD_ARCH="all" + ;; + "[Package arm64"*) + export BUILD_ARCH="arm64" + ;; + + *) + echo "Unknown build architecture '${BUILD_ARCH}' provided" + exit 1 + ;; + esac + + echo "Detected build architecture ${BUILD_ARCH}" +} diff --git a/.travis/package_management/package_cloud_wrapper.sh b/.travis/package_management/package_cloud_wrapper.sh new file mode 100755 index 000000000..48a372d37 --- /dev/null +++ b/.travis/package_management/package_cloud_wrapper.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash +# +# This is a tool to help removal of packages from packagecloud.io +# It utilizes the package_cloud utility provided from packagecloud.io +# +# Depends on: +# 1) package cloud gem (detects absence and installs it) +# +# Requires: +# 1) PKG_CLOUD_TOKEN variable exported +# 2) To properly install package_cloud when not found, it requires: ruby gcc gcc-c++ ruby-devel +# +# Copyright: SPDX-License-Identifier: GPL-3.0-or-later +# +# Author : Pavlos Emm. Katsoulakis (paul@netdata.cloud) +#shellcheck disable=SC2068,SC2145 +set -e +PKG_CLOUD_CONFIG="$HOME/.package_cloud_configuration.cfg" + +# If we are not in netdata git repo, at the top level directory, fail +TOP_LEVEL=$(basename "$(git rev-parse --show-toplevel)") +CWD=$(git rev-parse --show-cdup) +if [ -n "$CWD" ] || [ ! "${TOP_LEVEL}" == "netdata" ]; then + echo "Run as .travis/package_management/$(basename "$0") from top level directory of netdata git repository" + echo "Docker build process aborted" + exit 1 +fi + +# Install dependency if not there +if ! command -v package_cloud > /dev/null 2>&1; then + echo "No package cloud gem found, installing" + gem install -V package_cloud || (echo "Package cloud installation failed. you might want to check if required dependencies are there (ruby gcc gcc-c++ ruby-devel)" && exit 1) +else + echo "Found package_cloud gem, continuing" +fi + +# Check for required token and prepare config +if [ -z "${PKG_CLOUD_TOKEN}" ]; then + echo "Please set PKG_CLOUD_TOKEN to be able to use ${0}" + exit 1 +fi +echo "{\"url\":\"https://packagecloud.io\",\"token\":\"${PKG_CLOUD_TOKEN}\"}" > "${PKG_CLOUD_CONFIG}" + +echo "Executing package_cloud with config ${PKG_CLOUD_CONFIG} and parameters $@" +package_cloud $@ --config="${PKG_CLOUD_CONFIG}" + +rm -rf "${PKG_CLOUD_CONFIG}" +echo "Done!" diff --git a/.travis/package_management/prepare_packages.sh b/.travis/package_management/prepare_packages.sh new file mode 100755 index 000000000..1fb26a95e --- /dev/null +++ b/.travis/package_management/prepare_packages.sh @@ -0,0 +1,56 @@ +#!/usr/bin/env bash +# +# Utility that gathers generated packages, +# puts them together in a local folder for deploy facility to pick up +# +# Copyright: SPDX-License-Identifier: GPL-3.0-or-later +# +# Author : Pavlos Emm. Katsoulakis (paul@netdata.cloud) +#shellcheck disable=SC2068 +set -e + +# If we are not in netdata git repo, at the top level directory, fail +TOP_LEVEL=$(basename "$(git rev-parse --show-toplevel)") +CWD=$(git rev-parse --show-cdup) +if [ -n "$CWD" ] || [ ! "${TOP_LEVEL}" == "netdata" ]; then + echo "Run as .travis/package_management/$(basename "$0") from top level directory of netdata git repository" + echo "Package preparation aborted" + exit 1 +fi + +export LXC_ROOT="/var/lib/lxc" + +# Go through the containers created for packaging and pick up all generated packages +CREATED_CONTAINERS=$(ls -A "${LXC_ROOT}") +for d in ${CREATED_CONTAINERS[@]}; do + echo "Picking up packaging contents from ${d}" + + # Pick up any RPMS from builder + RPM_BUILD_PATH="${LXC_ROOT}/${d}/rootfs/home/${BUILDER_NAME}/rpmbuild" + echo "Checking folder ${RPM_BUILD_PATH} for RPMS and SRPMS" + + if [ -d "${RPM_BUILD_PATH}/RPMS" ]; then + echo "Copying any RPMS in '${RPM_BUILD_PATH}', copying over the following:" + ls -ltrR "${RPM_BUILD_PATH}/RPMS" + [[ -d "${RPM_BUILD_PATH}/RPMS/x86_64" ]] && cp -r "${RPM_BUILD_PATH}"/RPMS/x86_64/* "${PACKAGES_DIRECTORY}" + [[ -d "${RPM_BUILD_PATH}/RPMS/i386" ]] && cp -r "${RPM_BUILD_PATH}"/RPMS/i386/* "${PACKAGES_DIRECTORY}" + [[ -d "${RPM_BUILD_PATH}/RPMS/i686" ]] && cp -r "${RPM_BUILD_PATH}"/RPMS/i686/* "${PACKAGES_DIRECTORY}" + fi + + if [ -d "${RPM_BUILD_PATH}/SRPMS" ]; then + echo "Copying any SRPMS in '${RPM_BUILD_PATH}', copying over the following:" + ls -ltrR "${RPM_BUILD_PATH}/SRPMS" + [[ -d "${RPM_BUILD_PATH}/SRPMS/x86_64" ]] && cp -r "${RPM_BUILD_PATH}"/SRPMS/x86_64/* "${PACKAGES_DIRECTORY}" + [[ -d "${RPM_BUILD_PATH}/SRPMS/i386" ]] && cp -r "${RPM_BUILD_PATH}"/SRPMS/i386/* "${PACKAGES_DIRECTORY}" + [[ -d "${RPM_BUILD_PATH}/SRPMS/i686" ]] && cp -r "${RPM_BUILD_PATH}"/SRPMS/i686/* "${PACKAGES_DIRECTORY}" + fi + + # Pick up any DEBs from builder + DEB_BUILD_PATH="${d}/home/${BUILDER_NAME}/build-area" + echo "Checking folder ${DEB_BUILD_PATH} for DEB packages" + #TODO: During debian clean up we 'll fill this up + +done + +chmod -R 777 "${PACKAGES_DIRECTORY}" +echo "Packaging contents ready to ship!" diff --git a/.travis/package_management/trigger_deb_lxc_build.py b/.travis/package_management/trigger_deb_lxc_build.py new file mode 100755 index 000000000..3040bdd62 --- /dev/null +++ b/.travis/package_management/trigger_deb_lxc_build.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python3 +# +# This script is responsible for running the RPM build on the running container +# +# Copyright: SPDX-License-Identifier: GPL-3.0-or-later +# +# Author : Pavlos Emm. Katsoulakis <paul@netdata.cloud> + +import common +import os +import sys +import lxc + +if len(sys.argv) != 2: + print('You need to provide a container name to get things started') + sys.exit(1) +container_name=sys.argv[1] + +# Load the container, break if its not there +print("Starting up container %s" % container_name) +container = lxc.Container(container_name) +if not container.defined: + raise Exception("Container %s does not exist!" % container_name) + +# Check if the container is running, attempt to start it up in case its not running +if not container.running or not container.state == "RUNNING": + print('Container %s is not running, attempt to start it up' % container_name) + + # Start the container + if not container.start(): + raise Exception("Failed to start the container") + + if not container.running or not container.state == "RUNNING": + raise Exception('Container %s is not running, configuration process aborted ' % container_name) + +# Wait for connectivity +if not container.get_ips(timeout=30): + raise Exception("Timeout while waiting for container") + +print("Setting up EMAIL and DEBFULLNAME variables required by the build tools") +os.environ["EMAIL"] = "bot@netdata.cloud" +os.environ["DEBFULLNAME"] = "Netdata builder" + +# Run the build process on the container +print("Starting DEB build process, running dh-make") +new_version = os.environ["BUILD_VERSION"].replace('v', '') + +print("Building the package") +common.run_command(container, ["sudo", "-u", os.environ['BUILDER_NAME'], "dpkg-buildpackage", "--host-arch", "amd64", "--target-arch", "amd64", "--post-clean", "--pre-clean", "--build=binary", "--release-by=\"Netdata Builder\"", "--build-by=\"Netdata Builder\""]) + +print('Done!') diff --git a/.travis/package_management/trigger_rpm_lxc_build.py b/.travis/package_management/trigger_rpm_lxc_build.py new file mode 100755 index 000000000..f9e109c72 --- /dev/null +++ b/.travis/package_management/trigger_rpm_lxc_build.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python3 +# +# This script is responsible for running the RPM build on the running container +# +# Copyright: SPDX-License-Identifier: GPL-3.0-or-later +# +# Author : Pavlos Emm. Katsoulakis <paul@netdata.cloud> + +import common +import os +import sys +import lxc + +if len(sys.argv) != 2: + print('You need to provide a container name to get things started') + sys.exit(1) +container_name=sys.argv[1] + +# Load the container, break if its not there +print("Starting up container %s" % container_name) +container = lxc.Container(container_name) +if not container.defined: + raise Exception("Container %s does not exist!" % container_name) + +# Check if the container is running, attempt to start it up in case its not running +if not container.running or not container.state == "RUNNING": + print('Container %s is not running, attempt to start it up' % container_name) + + # Start the container + if not container.start(): + raise Exception("Failed to start the container") + + if not container.running or not container.state == "RUNNING": + raise Exception('Container %s is not running, configuration process aborted ' % container_name) + +# Wait for connectivity +if not container.get_ips(timeout=30): + raise Exception("Timeout while waiting for container") + +print("Adding builder specific dependencies to the LXC container") +common.run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "rpm-build"]) +common.run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "rpm-devel"]) +common.run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "rpmlint"]) +common.run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "make"]) +common.run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "python"]) +common.run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "bash"]) +common.run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "diffutils"]) +common.run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "patch"]) +common.run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "rpmdevtools"]) + +# Run the build process on the container +print("Starting RPM build process") +common.run_command(container, ["sudo", "-u", os.environ['BUILDER_NAME'], "rpmbuild", "-ba", "--rebuild", "/home/%s/rpmbuild/SPECS/netdata.spec" % os.environ['BUILDER_NAME']]) + +print('Done!') diff --git a/.travis/package_management/yank_stale_rpm.sh b/.travis/package_management/yank_stale_rpm.sh new file mode 100755 index 000000000..5cf938664 --- /dev/null +++ b/.travis/package_management/yank_stale_rpm.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +# +# This script is responsible for the removal of stale RPM/DEB files. +# It runs on the pre-deploy step and takes care of the removal of the files +# prior to the upload of the freshly built ones +# +# Copyright: SPDX-License-Identifier: GPL-3.0-or-later +# +# Author : Pavlos Emm. Katsoulakis (paul@netdata.cloud) +#shellcheck disable=SC2010,SC2068 +set -e + +# If we are not in netdata git repo, at the top level directory, fail +TOP_LEVEL=$(basename "$(git rev-parse --show-toplevel)") +CWD=$(git rev-parse --show-cdup) +if [ -n "$CWD" ] || [ ! "${TOP_LEVEL}" == "netdata" ]; then + echo "Run as .travis/package_management/$(basename "$0") from top level directory of netdata git repository" + echo "Package yanking cancelled" + exit 1 +fi + +PACKAGES_DIR="$1" +DISTRO="$2" +PACKAGES_LIST="$(ls -AR "${PACKAGES_DIR}" | grep '\.rpm')" + +if [ ! -d "${PACKAGES_DIR}" ] || [ -z "${PACKAGES_LIST}" ]; then + echo "Folder ${PACKAGES_DIR} does not seem to be a valid directory or is empty. No packages to check for yanking" + exit 1 +fi + +for pkg in ${PACKAGES_LIST[@]}; do + echo "Attempting yank on ${pkg}.." + .travis/package_management/package_cloud_wrapper.sh yank "${PACKAGING_USER}/${DEPLOY_REPO}/${DISTRO}" "${pkg}" || echo "Nothing to yank or error on ${pkg}" +done + |