diff options
Diffstat (limited to '.travis')
| -rw-r--r-- | .travis/README.md | 6 | ||||
| -rwxr-xr-x | .travis/check_changelog_last_modification.sh | 21 | ||||
| -rwxr-xr-x | .travis/generate_changelog_for_nightlies.sh | 1 | ||||
| -rwxr-xr-x | .travis/nightlies.sh | 8 | ||||
| -rw-r--r-- | .travis/package_management/build.sh | 31 | ||||
| -rwxr-xr-x | .travis/package_management/common.py | 116 | ||||
| -rwxr-xr-x | .travis/package_management/configure_deb_lxc_environment.py | 62 | ||||
| -rwxr-xr-x | .travis/package_management/configure_rpm_lxc_environment.py | 97 | ||||
| -rwxr-xr-x | .travis/package_management/create_lxc_for_build.sh | 2 | ||||
| -rw-r--r-- | .travis/package_management/functions.sh | 4 | ||||
| -rwxr-xr-x | .travis/package_management/prepare_packages.sh | 51 | ||||
| -rwxr-xr-x | .travis/package_management/trigger_deb_lxc_build.py | 30 | ||||
| -rwxr-xr-x | .travis/package_management/yank_stale_rpm.sh | 2 |
13 files changed, 304 insertions, 127 deletions
diff --git a/.travis/README.md b/.travis/README.md index b7b61ecb4..3b314fa18 100644 --- a/.travis/README.md +++ b/.travis/README.md @@ -138,6 +138,6 @@ We plan to support amd64, x86 and arm64 architectures. As of June 2019 only amd6 The Package deployment can be triggered manually by executing an empty commit with the following message pattern: `[Package PACKAGE_TYPE PACKAGE_ARCH] DESCRIBE_THE_REASONING_HERE`. Travis Yaml configuration allows the user to combine package type and architecture as necessary to regenerate the current stable release (For example tag v1.15.0 as of 4th of May 2019) -Sample patterns to trigger building of packages for all AMD64 supported architecture: -- '[Package AMD64 RPM]': Build & publish all amd64 available RPM packages -- '[Package AMD64 DEB]': Build & publish all amd64 available DEB packages +Sample patterns to trigger building of packages for all amd64 supported architecture: +- '[Package amd64 RPM]': Build & publish all amd64 available RPM packages +- '[Package amd64 DEB]': Build & publish all amd64 available DEB packages diff --git a/.travis/check_changelog_last_modification.sh b/.travis/check_changelog_last_modification.sh new file mode 100755 index 000000000..2665c0627 --- /dev/null +++ b/.travis/check_changelog_last_modification.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash + +set -e + +LAST_MODIFICATION="$(git log -1 --pretty="format:%at" CHANGELOG.md)" +CURRENT_TIME="$(date +"%s")" +TWO_DAYS_IN_SECONDS=172800 + +DIFF=$((CURRENT_TIME - LAST_MODIFICATION)) + +echo "Checking CHANGELOG.md last modification time on GIT.." +echo "CHANGELOG.md timestamp: ${LAST_MODIFICATION}" +echo "Current timestamp: ${CURRENT_TIME}" +echo "Diff: ${DIFF}" + +if [ ${DIFF} -gt ${TWO_DAYS_IN_SECONDS} ]; then + echo "CHANGELOG.md is more than two days old!" + post_message "TRAVIS_MESSAGE" "Hi <!here>, CHANGELOG.md was found more than two days old (Diff: ${DIFF} seconds)" "${NOTIF_CHANNEL}" +else + echo "CHANGELOG.md is less than two days old, fine" +fi diff --git a/.travis/generate_changelog_for_nightlies.sh b/.travis/generate_changelog_for_nightlies.sh index 68491fa9b..b90862880 100755 --- a/.travis/generate_changelog_for_nightlies.sh +++ b/.travis/generate_changelog_for_nightlies.sh @@ -49,6 +49,7 @@ docker run -it -v "$(pwd)":/project markmandel/github-changelog-generator:latest --token "${GITHUB_TOKEN}" \ --since-tag "v1.10.0" \ --unreleased-label "**Next release**" \ + --no-issues \ --exclude-labels "stale,duplicate,question,invalid,wontfix,discussion,no changelog" \ --no-compare-link ${OPTS} diff --git a/.travis/nightlies.sh b/.travis/nightlies.sh index 188b37da0..002461041 100755 --- a/.travis/nightlies.sh +++ b/.travis/nightlies.sh @@ -38,6 +38,12 @@ if [ ! "${TRAVIS_REPO_SLUG}" == "netdata/netdata" ]; then fi echo "--- Running Changelog generation ---" -.travis/generate_changelog_for_nightlies.sh "${LAST_TAG}" "${COMMITS_SINCE_RELEASE}" || echo "Changelog generation has failed, this is a soft error, process continues" +NIGHTLIES_CHANGELOG_FAILED=0 +.travis/generate_changelog_for_nightlies.sh "${LAST_TAG}" "${COMMITS_SINCE_RELEASE}" || NIGHTLIES_CHANGELOG_FAILED=1 + +if [ ${NIGHTLIES_CHANGELOG_FAILED} -eq 1 ]; then + echo "Changelog generation has failed, this is a soft error, process continues" + post_message "TRAVIS_MESSAGE" "Changelog generation job for nightlies failed, possibly due to github issues" "${NOTIF_CHANNEL}" +fi exit "${FAIL}" diff --git a/.travis/package_management/build.sh b/.travis/package_management/build.sh new file mode 100644 index 000000000..beb522a35 --- /dev/null +++ b/.travis/package_management/build.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash + +UNPACKAGED_NETDATA_PATH="$1" +LATEST_RELEASE_VERSION="$2" + +if [ -z "${LATEST_RELEASE_VERSION}" ]; then + echo "Parameter 'LATEST_RELEASE_VERSION' not defined" + exit 1 +fi + +if [ -z "${UNPACKAGED_NETDATA_PATH}" ]; then + echo "Parameter 'UNPACKAGED_NETDATA_PATH' not defined" + exit 1 +fi + +echo "Running changelog generation mechanism since ${LATEST_RELEASE_VERSION}" + +echo "Entering ${UNPACKAGED_NETDATA_PATH}" +cd "${UNPACKAGED_NETDATA_PATH}" + +echo "Linking debian -> contrib/debian" +ln -sf contrib/debian debian + +echo "Executing dpkg-buildpackage" +if dpkg-buildpackage --version 2> /dev/null | grep -q "1.18"; then + dpkg-buildpackage --post-clean --pre-clean --build=binary +else + dpkg-buildpackage -b +fi + +echo "DEB build script completed!" diff --git a/.travis/package_management/common.py b/.travis/package_management/common.py index 6cf59293d..6e7a26023 100755 --- a/.travis/package_management/common.py +++ b/.travis/package_management/common.py @@ -6,6 +6,22 @@ import lxc import subprocess +import os + +def fetch_version(orig_build_version): + tag = None + friendly_version = "" + + # TODO: Checksum validations + if str(orig_build_version).count(".latest") == 1: + version_list=str(orig_build_version).replace('v', '').split('.') + friendly_version='.'.join(version_list[0:2]) + "." + version_list[3] + else: + friendly_version = orig_build_version.replace('v', '') + tag = friendly_version # Go to stable tag + print("Version set to %s from %s" % (friendly_version, orig_build_version)) + + return friendly_version, tag def replace_tag(tag_name, spec, new_tag_content): print("Fixing tag %s in %s" % (tag_name, spec)) @@ -44,3 +60,103 @@ def run_command_in_host(cmd): print('Output: ' + o.decode('ascii')) print('Error: ' + e.decode('ascii')) print('code: ' + str(proc.returncode)) + +def prepare_repo(container): + if str(os.environ["REPO_TOOL"]).count("zypper") == 1: + run_command(container, [os.environ["REPO_TOOL"], "clean", "-a"]) + run_command(container, [os.environ["REPO_TOOL"], "--no-gpg-checks", "update", "-y"]) + + elif str(os.environ["REPO_TOOL"]).count("yum") == 1: + run_command(container, [os.environ["REPO_TOOL"], "clean", "all"]) + run_command(container, [os.environ["REPO_TOOL"], "update", "-y"]) + + if os.environ["BUILD_STRING"].count("el/7") == 1 and os.environ["BUILD_ARCH"].count("i386") == 1: + print ("Skipping epel-release install for %s-%s" % (os.environ["BUILD_STRING"], os.environ["BUILD_ARCH"])) + else: + run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "epel-release"]) + + elif str(os.environ["REPO_TOOL"]).count("apt-get") == 1: + run_command(container, [os.environ["REPO_TOOL"], "update", "-y"]) + else: + run_command(container, [os.environ["REPO_TOOL"], "update", "-y"]) + + run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "sudo"]) + run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "wget"]) + run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "bash"]) + +def install_common_dependendencies(container): + if str(os.environ["REPO_TOOL"]).count("zypper") == 1: + run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "gcc-c++"]) + run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "json-glib-devel"]) + run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "freeipmi-devel"]) + run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "cups-devel"]) + run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "snappy-devel"]) + run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "protobuf-devel"]) + run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "protobuf-c"]) + + elif str(os.environ["REPO_TOOL"]).count("yum") == 1: + run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "gcc-c++"]) + run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "json-c-devel"]) + run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "freeipmi-devel"]) + run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "cups-devel"]) + run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "snappy-devel"]) + run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "protobuf-devel"]) + run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "protobuf-c-devel"]) + run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "protobuf-compiler"]) + + elif str(os.environ["REPO_TOOL"]).count("apt-get") == 1: + run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "g++"]) + run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "libipmimonitoring-dev"]) + run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "libjson-c-dev"]) + run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "libcups2-dev"]) + run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "libsnappy-dev"]) + run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "libprotobuf-dev"]) + run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "libprotoc-dev"]) + run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "protobuf-compiler"]) + if os.environ["BUILD_STRING"].count("debian/jessie") == 1: + run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "snappy"]) + else: + run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "gcc-c++"]) + run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "cups-devel"]) + run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "freeipmi-devel"]) + run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "json-c-devel"]) + run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "snappy-devel"]) + run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "protobuf-devel"]) + run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "protobuf-c-devel"]) + run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "protobuf-compiler"]) + + if os.environ["BUILD_STRING"].count("el/6") <= 0: + run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "autogen"]) + +def prepare_version_source(dest_archive, pkg_friendly_version, tag=None): + print(".0 Preparing local implementation tarball for version %s" % pkg_friendly_version) + tar_file = os.environ['LXC_CONTAINER_ROOT'] + dest_archive + + if tag is not None: + print(".1 Checking out tag %s" % tag) + run_command_in_host(['git', 'fetch', '--all']) + + # TODO: Keep in mind that tricky 'v' there, needs to be removed once we clear our versioning scheme + run_command_in_host(['git', 'checkout', 'v%s' % pkg_friendly_version]) + + print(".2 Tagging the code with version: %s" % pkg_friendly_version) + run_command_in_host(['git', 'tag', '-a', pkg_friendly_version, '-m', 'Tagging while packaging on %s' % os.environ["CONTAINER_NAME"]]) + + print(".3 Run autoreconf -ivf") + run_command_in_host(['autoreconf', '-ivf']) + + print(".4 Run configure") + run_command_in_host(['./configure', '--with-math', '--with-zlib', '--with-user=netdata']) + + print(".5 Run make dist") + run_command_in_host(['make', 'dist']) + + print(".6 Copy generated tarbal to desired path") + if os.path.exists('netdata-%s.tar.gz' % pkg_friendly_version): + run_command_in_host(['sudo', 'cp', 'netdata-%s.tar.gz' % pkg_friendly_version, tar_file]) + + print(".7 Fixing permissions on tarball") + run_command_in_host(['sudo', 'chmod', '777', tar_file]) + else: + print("I could not find (%s) on the disk, stopping the build. Kindly check the logs and try again" % 'netdata-%s.tar.gz' % pkg_friendly_version) + sys.exit(1) diff --git a/.travis/package_management/configure_deb_lxc_environment.py b/.travis/package_management/configure_deb_lxc_environment.py index 58999ad38..12328dde7 100755 --- a/.travis/package_management/configure_deb_lxc_environment.py +++ b/.travis/package_management/configure_deb_lxc_environment.py @@ -39,27 +39,57 @@ print("Waiting for container connectivity to start configuration sequence") if not container.get_ips(timeout=30): raise Exception("Timeout while waiting for container") +build_path = "/home/%s" % os.environ['BUILDER_NAME'] + # Run the required activities now # 1. Create the builder user print("1. Adding user %s" % os.environ['BUILDER_NAME']) common.run_command(container, ["useradd", "-m", os.environ['BUILDER_NAME']]) # Fetch package dependencies for the build -print("2. Installing package dependencies within LXC container") -common.run_command(container, ["apt-get", "update", "-y"]) -common.run_command(container, ["apt-get", "install", "-y", "sudo"]) -common.run_command(container, ["apt-get", "install", "-y", "wget"]) -common.run_command(container, ["apt-get", "install", "-y", "bash"]) -common.run_command(container, ["wget", "-T", "15", "-O", "~/.install-required-packages.sh", "https://raw.githubusercontent.com/netdata/netdata-demo-site/master/install-required-packages.sh"]) -common.run_command(container, ["bash", "~/.install-required-packages.sh", "netdata", "--dont-wait", "--non-interactive"]) - -# Download the source -dest_archive="/home/%s/netdata-%s.tar.gz" % (os.environ['BUILDER_NAME'],os.environ['BUILD_VERSION']) -release_url="https://github.com/netdata/netdata/releases/download/%s/netdata-%s.tar.gz" % (os.environ['BUILD_VERSION'], os.environ['BUILD_VERSION']) -print("3. Fetch netdata source (%s -> %s)" % (release_url, dest_archive)) -common.run_command(container, ["sudo", "-u", os.environ['BUILDER_NAME'], "wget", "-T", "15", "--output-document=" + dest_archive, release_url]) - -print("4. Extracting directory contents to /home " + os.environ['BUILDER_NAME']) -common.run_command(container, ["sudo", "-u", os.environ['BUILDER_NAME'], "tar", "xf", dest_archive, "-C", "/home/" + os.environ['BUILDER_NAME']]) +print("2. Preparing repo on LXC container") +common.prepare_repo(container) + +print("2.1 Install .DEB build support packages") +common.run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "dpkg-dev"]) +common.run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "libdistro-info-perl"]) +common.run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "dh-make"]) +common.run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "dh-systemd"]) +common.run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "dh-autoreconf"]) +common.run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "git-buildpackage"]) + +print("2.2 Add more dependencies") +common.run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "libnetfilter-acct-dev"]) +common.run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "libcups2-dev"]) + +print ("3.1 Run install-required-packages scriptlet") +common.run_command(container, ["wget", "-T", "15", "-O", "%s/.install-required-packages.sh" % build_path, "https://raw.githubusercontent.com/netdata/netdata-demo-site/master/install-required-packages.sh"]) +common.run_command(container, ["bash", "%s/.install-required-packages.sh" % build_path, "netdata", "--dont-wait", "--non-interactive"]) + +print("3.2 Installing package dependencies within LXC container") +common.install_common_dependendencies(container) + +friendly_version="" +dest_archive="" +download_url="" +tag = None +friendly_version, tag = common.fetch_version(os.environ['BUILD_VERSION']) + +tar_file="%s/netdata-%s.tar.gz" % (os.path.dirname(dest_archive), friendly_version) + +print("5. I will be building version '%s' of netdata." % os.environ['BUILD_VERSION']) +dest_archive="%s/netdata-%s.tar.gz" % (build_path, friendly_version) + +if str(os.environ["BUILD_STRING"]).count("debian/jessie") == 1: + print("5.1 We are building for Jessie, adjusting control file") + common.run_command_in_host(['sudo', 'rm', 'contrib/debian/control']) + common.run_command_in_host(['sudo', 'cp', 'contrib/debian/control.jessie', 'contrib/debian/control']) + +common.prepare_version_source(dest_archive, friendly_version, tag=tag) + +print("6. Installing build.sh script to build path") +common.run_command_in_host(['sudo', 'cp', '.travis/package_management/build.sh', "%s/%s/build.sh" % (os.environ['LXC_CONTAINER_ROOT'], build_path)]) +common.run_command_in_host(['sudo', 'chmod', '777', "%s/%s/build.sh" % (os.environ['LXC_CONTAINER_ROOT'], build_path)]) +common.run_command_in_host(['sudo', 'ln', '-sf', 'contrib/debian', 'debian']) print("Done!") diff --git a/.travis/package_management/configure_rpm_lxc_environment.py b/.travis/package_management/configure_rpm_lxc_environment.py index 644e027b7..79d34608f 100755 --- a/.travis/package_management/configure_rpm_lxc_environment.py +++ b/.travis/package_management/configure_rpm_lxc_environment.py @@ -46,30 +46,15 @@ print("1. Adding user %s" % os.environ['BUILDER_NAME']) common.run_command(container, ["useradd", "-m", os.environ['BUILDER_NAME']]) # Fetch package dependencies for the build -print("2. Installing package dependencies within LXC container") -if str(os.environ["REPO_TOOL"]).count("zypper") == 1: - common.run_command(container, [os.environ["REPO_TOOL"], "clean", "-a"]) - common.run_command(container, [os.environ["REPO_TOOL"], "--no-gpg-checks", "update", "-y"]) - common.run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "json-glib-devel"]) - -elif str(os.environ["REPO_TOOL"]).count("yum") == 1: - common.run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "json-c-devel"]) - common.run_command(container, [os.environ["REPO_TOOL"], "clean", "all"]) - common.run_command(container, [os.environ["REPO_TOOL"], "update", "-y"]) - common.run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "epel-release"]) -else: - common.run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "json-c-devel"]) - common.run_command(container, [os.environ["REPO_TOOL"], "update", "-y"]) - -common.run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "sudo"]) -common.run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "wget"]) -common.run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "bash"]) -common.run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "freeipmi-devel"]) -common.run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "cups-devel"]) +print("2.1 Preparing repo on LXC container") +common.prepare_repo(container) + +common.run_command(container, ["wget", "-T", "15", "-O", "/home/%s/.install-required-packages.sh" % (os.environ['BUILDER_NAME']), "https://raw.githubusercontent.com/netdata/netdata-demo-site/master/install-required-packages.sh"]) +common.run_command(container, ["bash", "/home/%s/.install-required-packages.sh" % (os.environ['BUILDER_NAME']), "netdata", "--dont-wait", "--non-interactive"]) # Exceptional cases, not available everywhere # - +print("2.2 Running uncommon dependencies and preparing LXC environment") # Not on Centos-7 if os.environ["BUILD_STRING"].count("el/7") <= 0: common.run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "libnetfilter_acct-devel"]) @@ -78,8 +63,8 @@ if os.environ["BUILD_STRING"].count("el/7") <= 0: if os.environ["BUILD_STRING"].count("el/6") <= 0: common.run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "autoconf-archive"]) -common.run_command(container, ["wget", "-T", "15", "-O", "/home/%s/.install-required-packages.sh" % (os.environ['BUILDER_NAME']), "https://raw.githubusercontent.com/netdata/netdata-demo-site/master/install-required-packages.sh"]) -common.run_command(container, ["bash", "/home/%s/.install-required-packages.sh" % (os.environ['BUILDER_NAME']), "netdata", "--dont-wait", "--non-interactive"]) +print("2.3 Installing common dependencies") +common.install_common_dependendencies(container) print("3. Setting up macros") common.run_command(container, ["sudo", "-u", os.environ['BUILDER_NAME'], "/bin/echo", "'%_topdir %(echo /home/" + os.environ['BUILDER_NAME'] + ")/rpmbuild' > /home/" + os.environ['BUILDER_NAME'] + "/.rpmmacros"]) @@ -97,63 +82,21 @@ rpm_friendly_version="" dest_archive="" download_url="" spec_file="/home/%s/rpmbuild/SPECS/netdata.spec" % os.environ['BUILDER_NAME'] +tag = None +rpm_friendly_version, tag = common.fetch_version(os.environ['BUILD_VERSION']) +tar_file="%s/netdata-%s.tar.gz" % (os.path.dirname(dest_archive), rpm_friendly_version) -# TODO: Checksum validations -if str(os.environ['BUILD_VERSION']).count(".latest") == 1: - version_list=str(os.environ['BUILD_VERSION']).replace('v', '').split('.') - rpm_friendly_version='.'.join(version_list[0:3]) + "." + version_list[3] - - print("Building latest nightly version of netdata..(%s)" % os.environ['BUILD_VERSION']) - dest_archive="/home/%s/rpmbuild/SOURCES/netdata-%s.tar.gz" % (os.environ['BUILDER_NAME'], rpm_friendly_version) - - print("5. Preparing local latest implementation tarball for version %s" % rpm_friendly_version) - tar_file = os.environ['LXC_CONTAINER_ROOT'] + dest_archive - - print("5.1 Tagging the code with latest version: %s" % rpm_friendly_version) - common.run_command_in_host(['git', 'tag', '-a', rpm_friendly_version, '-m', 'Tagging while packaging on %s' % os.environ["CONTAINER_NAME"]]) - - print("5.2 Run autoreconf -ivf") - common.run_command_in_host(['autoreconf', '-ivf']) - - print("5.3 Run configure") - common.run_command_in_host(['./configure', '--with-math', '--with-zlib', '--with-user=netdata']) - - print("5.4 Run make dist") - common.run_command_in_host(['make', 'dist']) - - print("5.5 Copy generated tarbal to desired path") - if os.path.exists('netdata-%s.tar.gz' % rpm_friendly_version): - common.run_command_in_host(['sudo', 'cp', 'netdata-%s.tar.gz' % rpm_friendly_version, tar_file]) - - print("5.6 Fixing permissions on tarball") - common.run_command_in_host(['sudo', 'chmod', '777', tar_file]) - else: - print("I could not find (%s) on the disk, stopping the build. Kindly check the logs and try again" % 'netdata-%s.tar.gz' % rpm_friendly_version) - sys.exit(1) - - # Extract the spec file in place - print("6. Extract spec file from the source") - common.run_command_in_host(['sudo', 'cp', 'netdata.spec', os.environ['LXC_CONTAINER_ROOT'] + spec_file]) - common.run_command_in_host(['sudo', 'chmod', '777', os.environ['LXC_CONTAINER_ROOT'] + spec_file]) - - print("7. Temporary hack: Change Source0 to %s on spec file %s" % (dest_archive, spec_file)) - common.replace_tag("Source0", os.environ['LXC_CONTAINER_ROOT'] + spec_file, tar_file) -else: - rpm_friendly_version = os.environ['BUILD_VERSION'] - - print("Building latest stable version of netdata.. (%s)" % os.environ['BUILD_VERSION']) - dest_archive="/home/%s/rpmbuild/SOURCES/netdata-%s.tar.gz" % (os.environ['BUILDER_NAME'],os.environ['BUILD_VERSION']) - download_url="https://github.com/netdata/netdata/releases/download/%s/netdata-%s.tar.gz" % (os.environ['BUILD_VERSION'], os.environ['BUILD_VERSION']) +print("5. I will be building version '%s' of netdata." % os.environ['BUILD_VERSION']) +dest_archive="/home/%s/rpmbuild/SOURCES/netdata-%s.tar.gz" % (os.environ['BUILDER_NAME'], rpm_friendly_version) - print("5. Fetch netdata source into the repo structure(%s -> %s)" % (download_url, dest_archive)) - tar_file="%s/netdata-%s.tar.gz" % (os.path.dirname(dest_archive), rpm_friendly_version) - common.run_command(container, ["sudo", "-u", os.environ['BUILDER_NAME'], "wget", "-T", "15", "--output-document=" + dest_archive, download_url]) +common.prepare_version_source(dest_archive, rpm_friendly_version, tag=tag) - print("6.Extract spec file from the source") - common.run_command(container, ["sudo", "-u", os.environ['BUILDER_NAME'], "tar", "--to-command=cat > %s" % spec_file, "-xvf", dest_archive, "netdata-%s/netdata.spec" % os.environ['BUILD_VERSION']]) +# Extract the spec file in place +print("6. Extract spec file from the source") +common.run_command_in_host(['sudo', 'cp', 'netdata.spec', os.environ['LXC_CONTAINER_ROOT'] + spec_file]) +common.run_command_in_host(['sudo', 'chmod', '777', os.environ['LXC_CONTAINER_ROOT'] + spec_file]) - print("7. Temporary hack: Adjust version string on the spec file (%s) to %s and Source0 to %s" % (os.environ['LXC_CONTAINER_ROOT'] + spec_file, rpm_friendly_version, download_url)) - common.replace_tag("Version", os.environ['LXC_CONTAINER_ROOT'] + spec_file, rpm_friendly_version) - common.replace_tag("Source0", os.environ['LXC_CONTAINER_ROOT'] + spec_file, tar_file) +print("7. Temporary hack: Change Source0 to %s on spec file %s" % (dest_archive, spec_file)) +common.replace_tag("Source0", os.environ['LXC_CONTAINER_ROOT'] + spec_file, tar_file) print('Done!') diff --git a/.travis/package_management/create_lxc_for_build.sh b/.travis/package_management/create_lxc_for_build.sh index ae855a742..d733687a8 100755 --- a/.travis/package_management/create_lxc_for_build.sh +++ b/.travis/package_management/create_lxc_for_build.sh @@ -83,7 +83,7 @@ case "${BUILD_ARCH}" in .travis/package_management/configure_${PACKAGE_TYPE}_lxc_environment.py "${CONTAINER_NAME}" ;; "i386"|"amd64"|"arm64") - # AMD64 or i386 + # amd64 or i386 echo "Creating LXC Container for ${BUILD_ARCH}.." export CONTAINER_NAME="${BUILDER_NAME}-${BUILD_DISTRO}${BUILD_RELEASE}-${BUILD_ARCH}" export LXC_CONTAINER_ROOT="/var/lib/lxc/${CONTAINER_NAME}/rootfs" diff --git a/.travis/package_management/functions.sh b/.travis/package_management/functions.sh index 9a467ffe1..0c4425fa5 100644 --- a/.travis/package_management/functions.sh +++ b/.travis/package_management/functions.sh @@ -10,7 +10,7 @@ set -e function detect_arch_from_commit { case "${TRAVIS_COMMIT_MESSAGE}" in - "[Package AMD64"*) + "[Package amd64"*) export BUILD_ARCH="amd64" ;; "[Package i386"*) @@ -24,7 +24,7 @@ function detect_arch_from_commit { ;; *) - echo "Unknown build architecture '${BUILD_ARCH}' provided" + echo "Unknown build architecture in '${TRAVIS_COMMIT_MESSAGE}'. No BUILD_ARCH can be provided" exit 1 ;; esac diff --git a/.travis/package_management/prepare_packages.sh b/.travis/package_management/prepare_packages.sh index 1fb26a95e..12ed07cc7 100755 --- a/.travis/package_management/prepare_packages.sh +++ b/.travis/package_management/prepare_packages.sh @@ -27,29 +27,36 @@ for d in ${CREATED_CONTAINERS[@]}; do # Pick up any RPMS from builder RPM_BUILD_PATH="${LXC_ROOT}/${d}/rootfs/home/${BUILDER_NAME}/rpmbuild" - echo "Checking folder ${RPM_BUILD_PATH} for RPMS and SRPMS" - - if [ -d "${RPM_BUILD_PATH}/RPMS" ]; then - echo "Copying any RPMS in '${RPM_BUILD_PATH}', copying over the following:" - ls -ltrR "${RPM_BUILD_PATH}/RPMS" - [[ -d "${RPM_BUILD_PATH}/RPMS/x86_64" ]] && cp -r "${RPM_BUILD_PATH}"/RPMS/x86_64/* "${PACKAGES_DIRECTORY}" - [[ -d "${RPM_BUILD_PATH}/RPMS/i386" ]] && cp -r "${RPM_BUILD_PATH}"/RPMS/i386/* "${PACKAGES_DIRECTORY}" - [[ -d "${RPM_BUILD_PATH}/RPMS/i686" ]] && cp -r "${RPM_BUILD_PATH}"/RPMS/i686/* "${PACKAGES_DIRECTORY}" - fi - - if [ -d "${RPM_BUILD_PATH}/SRPMS" ]; then - echo "Copying any SRPMS in '${RPM_BUILD_PATH}', copying over the following:" - ls -ltrR "${RPM_BUILD_PATH}/SRPMS" - [[ -d "${RPM_BUILD_PATH}/SRPMS/x86_64" ]] && cp -r "${RPM_BUILD_PATH}"/SRPMS/x86_64/* "${PACKAGES_DIRECTORY}" - [[ -d "${RPM_BUILD_PATH}/SRPMS/i386" ]] && cp -r "${RPM_BUILD_PATH}"/SRPMS/i386/* "${PACKAGES_DIRECTORY}" - [[ -d "${RPM_BUILD_PATH}/SRPMS/i686" ]] && cp -r "${RPM_BUILD_PATH}"/SRPMS/i686/* "${PACKAGES_DIRECTORY}" + if [ -d "${RPM_BUILD_PATH}" ]; then + echo "Checking folder ${RPM_BUILD_PATH} for RPMS and SRPMS" + + if [ -d "${RPM_BUILD_PATH}/RPMS" ]; then + echo "Copying any RPMS in '${RPM_BUILD_PATH}', copying over the following:" + ls -ltrR "${RPM_BUILD_PATH}/RPMS" + [[ -d "${RPM_BUILD_PATH}/RPMS/x86_64" ]] && cp -r "${RPM_BUILD_PATH}"/RPMS/x86_64/* "${PACKAGES_DIRECTORY}" + [[ -d "${RPM_BUILD_PATH}/RPMS/i386" ]] && cp -r "${RPM_BUILD_PATH}"/RPMS/i386/* "${PACKAGES_DIRECTORY}" + [[ -d "${RPM_BUILD_PATH}/RPMS/i686" ]] && cp -r "${RPM_BUILD_PATH}"/RPMS/i686/* "${PACKAGES_DIRECTORY}" + fi + + if [ -d "${RPM_BUILD_PATH}/SRPMS" ]; then + echo "Copying any SRPMS in '${RPM_BUILD_PATH}', copying over the following:" + ls -ltrR "${RPM_BUILD_PATH}/SRPMS" + [[ -d "${RPM_BUILD_PATH}/SRPMS/x86_64" ]] && cp -r "${RPM_BUILD_PATH}"/SRPMS/x86_64/* "${PACKAGES_DIRECTORY}" + [[ -d "${RPM_BUILD_PATH}/SRPMS/i386" ]] && cp -r "${RPM_BUILD_PATH}"/SRPMS/i386/* "${PACKAGES_DIRECTORY}" + [[ -d "${RPM_BUILD_PATH}/SRPMS/i686" ]] && cp -r "${RPM_BUILD_PATH}"/SRPMS/i686/* "${PACKAGES_DIRECTORY}" + fi + else + DEB_BUILD_PATH="${LXC_ROOT}/${d}/rootfs/home/${BUILDER_NAME}" + echo "Checking folder ${DEB_BUILD_PATH} for DEB packages" + if [ -d "${DEB_BUILD_PATH}" ]; then + cp "${DEB_BUILD_PATH}"/netdata*.ddeb "${PACKAGES_DIRECTORY}" || echo "Could not copy any .ddeb files" + cp "${DEB_BUILD_PATH}"/netdata*.deb "${PACKAGES_DIRECTORY}" || echo "Could not copy any .deb files" + cp "${DEB_BUILD_PATH}"/netdata*.buildinfo "${PACKAGES_DIRECTORY}" || echo "Could not copy any .buildinfo files" + cp "${DEB_BUILD_PATH}"/netdata*.changes "${PACKAGES_DIRECTORY}" || echo "Could not copy any .changes files" + else + echo "Folder ${DEB_BUILD_PATH} does not exist or not a directory, nothing to do for package preparation" + fi fi - - # Pick up any DEBs from builder - DEB_BUILD_PATH="${d}/home/${BUILDER_NAME}/build-area" - echo "Checking folder ${DEB_BUILD_PATH} for DEB packages" - #TODO: During debian clean up we 'll fill this up - done chmod -R 777 "${PACKAGES_DIRECTORY}" diff --git a/.travis/package_management/trigger_deb_lxc_build.py b/.travis/package_management/trigger_deb_lxc_build.py index 3040bdd62..a0235a73d 100755 --- a/.travis/package_management/trigger_deb_lxc_build.py +++ b/.travis/package_management/trigger_deb_lxc_build.py @@ -37,15 +37,37 @@ if not container.running or not container.state == "RUNNING": if not container.get_ips(timeout=30): raise Exception("Timeout while waiting for container") +build_path = "/home/%s" % os.environ['BUILDER_NAME'] + print("Setting up EMAIL and DEBFULLNAME variables required by the build tools") os.environ["EMAIL"] = "bot@netdata.cloud" os.environ["DEBFULLNAME"] = "Netdata builder" # Run the build process on the container -print("Starting DEB build process, running dh-make") -new_version = os.environ["BUILD_VERSION"].replace('v', '') +new_version, tag = common.fetch_version(os.environ['BUILD_VERSION']) +print("Starting DEB build process for version %s" % new_version) + +netdata_tarball = "%s/netdata-%s.tar.gz" % (build_path, new_version) +unpacked_netdata = netdata_tarball.replace(".tar.gz", "") + +print("Extracting tarball %s" % netdata_tarball) +common.run_command(container, ["sudo", "-u", os.environ['BUILDER_NAME'], "tar", "xf", netdata_tarball, "-C", build_path]) + +print("Fixing changelog tags") +changelog_in_host = "contrib/debian/changelog" +common.run_command_in_host(['sed', '-i', 's/PREVIOUS_PACKAGE_VERSION/%s-1/g' % os.environ["LATEST_RELEASE_VERSION"].replace("v", ""), changelog_in_host]) +common.run_command_in_host(['sed', '-i', 's/PREVIOUS_PACKAGE_DATE/%s/g' % os.environ["LATEST_RELEASE_DATE"], changelog_in_host]) + +print("Executing gbp dch command..") +common.run_command_in_host(['gbp', 'dch', '--release', '--ignore-branch', '--spawn-editor=snapshot', '--since=%s' % os.environ["LATEST_RELEASE_VERSION"], '--new-version=%s' % new_version]) + +print("Copying over changelog to the destination machine") +common.run_command_in_host(['sudo', 'cp', 'debian/changelog', "%s/%s/netdata-%s/contrib/debian/" % (os.environ['LXC_CONTAINER_ROOT'], build_path, new_version)]) + +print("Running debian build script since %s" % os.environ["LATEST_RELEASE_VERSION"]) +common.run_command(container, ["sudo", "-u", os.environ['BUILDER_NAME'], "%s/build.sh" % build_path, unpacked_netdata, new_version]) -print("Building the package") -common.run_command(container, ["sudo", "-u", os.environ['BUILDER_NAME'], "dpkg-buildpackage", "--host-arch", "amd64", "--target-arch", "amd64", "--post-clean", "--pre-clean", "--build=binary", "--release-by=\"Netdata Builder\"", "--build-by=\"Netdata Builder\""]) +print("Listing contents on build path") +common.run_command(container, ["sudo", "-u", os.environ['BUILDER_NAME'], "ls", "-ltr", build_path]) print('Done!') diff --git a/.travis/package_management/yank_stale_rpm.sh b/.travis/package_management/yank_stale_rpm.sh index 5cf938664..3f7669712 100755 --- a/.travis/package_management/yank_stale_rpm.sh +++ b/.travis/package_management/yank_stale_rpm.sh @@ -21,7 +21,7 @@ fi PACKAGES_DIR="$1" DISTRO="$2" -PACKAGES_LIST="$(ls -AR "${PACKAGES_DIR}" | grep '\.rpm')" +PACKAGES_LIST="$(ls -AR "${PACKAGES_DIR}" | grep -e '\.rpm' -e '\.deb' -e '\.ddeb' )" if [ ! -d "${PACKAGES_DIR}" ] || [ -z "${PACKAGES_LIST}" ]; then echo "Folder ${PACKAGES_DIR} does not seem to be a valid directory or is empty. No packages to check for yanking" |