diff options
Diffstat (limited to '')
| -rw-r--r-- | .travis.yml | 409 | ||||
| -rw-r--r-- | .travis/README.md | 46 | ||||
| -rwxr-xr-x | .travis/labeler.sh | 15 | ||||
| -rwxr-xr-x | .travis/package_management/build_package_in_container.sh | 82 | ||||
| -rwxr-xr-x | .travis/package_management/common.py | 46 | ||||
| -rwxr-xr-x | .travis/package_management/configure_deb_lxc_environment.py | 65 | ||||
| -rwxr-xr-x | .travis/package_management/configure_rpm_lxc_environment.py | 159 | ||||
| -rwxr-xr-x | .travis/package_management/create_lxc_for_build.sh | 101 | ||||
| -rw-r--r-- | .travis/package_management/functions.sh | 33 | ||||
| -rwxr-xr-x | .travis/package_management/package_cloud_wrapper.sh | 48 | ||||
| -rwxr-xr-x | .travis/package_management/prepare_packages.sh | 56 | ||||
| -rwxr-xr-x | .travis/package_management/trigger_deb_lxc_build.py | 51 | ||||
| -rwxr-xr-x | .travis/package_management/trigger_rpm_lxc_build.py | 55 | ||||
| -rwxr-xr-x | .travis/package_management/yank_stale_rpm.sh | 35 |
14 files changed, 1174 insertions, 27 deletions
diff --git a/.travis.yml b/.travis.yml index e1b89dfa1..cb9d72901 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,13 +1,18 @@ +dist: xenial sudo: true language: c services: - docker + + # This is a hook to help us introduce "soft" errors on our process matrix: allow_failures: - env: ALLOW_SOFT_FAILURE_HERE=true + + # Install dependencies for all, once # install: @@ -16,12 +21,23 @@ install: - sudo pip install git-semver - docker info - source tests/installer/slack.sh +- export NOTIF_CHANNEL="automation-beta" +- if [ "${TRAVIS_REPO_SLUG}" = "netdata/netdata" ]; then export NOTIF_CHANNEL="automation"; fi; +- export BUILD_VERSION="$(cat packaging/version | cut -d'-' -f1)" +- if [[ "${TRAVIS_COMMIT_MESSAGE}" = *"[Build latest]"* ]]; then export BUILD_VERSION="$(cat packaging/version | cut -d'-' -f1,2 | sed -e 's/-/./g').latest"; fi; +- export DEPLOY_REPO="netdata" # Default production packaging repository +- if [[ "${TRAVIS_COMMIT_MESSAGE}" = *"[Build latest]"* ]]; then export DEPLOY_REPO="netdata-edge"; fi; +- export PACKAGING_USER="$(echo ${TRAVIS_REPO_SLUG} | cut -d'/' -f1)" + + # Setup notification system # notifications: webhooks: https://app.fossa.io/hooks/travisci + + # Define the stage sequence and conditionals # stages: @@ -29,6 +45,8 @@ stages: - name: Code quality, linting, syntax, code style - name: Build process - name: Artifacts validation +- name: Artifacts validation on bare OS, stable to current lifecycle checks + if: branch = master AND (type = pull_request OR type = cron) # Nightly operations - name: Nightly operations @@ -43,6 +61,148 @@ stages: - name: Publish for release if: branch = master AND type != pull_request AND type != cron AND commit_message =~ /(\[netdata release candidate\]|\[netdata major release\]|\[netdata minor release\]|\[netdata patch release\])/ + # Build DEB packages under special conditions + # Ubuntu +- name: "Package ubuntu/disco" + if: type != cron AND branch = master AND commit_message =~ /(\[Package arm64 DEB Ubuntu\]|\[Package arm64 DEB\]|\[Package i386 DEB Ubuntu\]|\[Package i386 DEB\]|\[Package AMD64 DEB Ubuntu\]|\[Package AMD64 DEB\])/ +- name: "Package ubuntu/cosmic" + if: type != cron AND branch = master AND commit_message =~ /(\[Package arm64 DEB Ubuntu\]|\[Package arm64 DEB\]|\[Package i386 DEB Ubuntu\]|\[Package i386 DEB\]|\[Package AMD64 DEB Ubuntu\]|\[Package AMD64 DEB\])/ +- name: "Package ubuntu/bionic" + if: type != cron AND branch = master AND commit_message =~ /(\[Package arm64 DEB Ubuntu\]|\[Package arm64 DEB\]|\[Package i386 DEB Ubuntu\]|\[Package i386 DEB\]|\[Package AMD64 DEB Ubuntu\]|\[Package AMD64 DEB\])/ +- name: "Package ubuntu/artful" + if: type != cron AND branch = master AND commit_message =~ /(\[Package arm64 DEB Ubuntu\]|\[Package arm64 DEB\]|\[Package i386 DEB Ubuntu\]|\[Package i386 DEB\]|\[Package AMD64 DEB Ubuntu\]|\[Package AMD64 DEB\])/ + + # Debian +- name: "Package debian/buster" + if: type != cron AND branch = master AND commit_message =~ /(\[Package arm64 DEB Debian\]|\[Package arm64 DEB\]|\[Package i386 DEB Debian\]|\[Package i386 DEB\]|\[Package AMD64 DEB Debian\]|\[Package AMD64 DEB\])/ +- name: "Package debian/stretch" + if: type != cron AND branch = master AND commit_message =~ /(\[Package arm64 DEB Debian\]|\[Package arm64 DEB\]|\[Package i386 DEB Debian\]|\[Package i386 DEB\]|\[Package AMD64 DEB Debian\]|\[Package AMD64 DEB\])/ +- name: "Package debian/jessie" + if: type != cron AND branch = master AND commit_message =~ /(\[Package arm64 DEB Debian\]|\[Package arm64 DEB\]|\[Package i386 DEB Debian\]|\[Package i386 DEB\]|\[Package AMD64 DEB Debian\]|\[Package AMD64 DEB\])/ +- name: "Package debian/wheezy" + if: type != cron AND branch = master AND commit_message =~ /(\[Package arm64 DEB Debian\]|\[Package arm64 DEB\]|\[Package i386 DEB Debian\]|\[Package i386 DEB\]|\[Package AMD64 DEB Debian\]|\[Package AMD64 DEB\])/ + + # Build RPM packages under special conditions + # Enterprise linux (Covers CentOS, Redhat, Amazon linux) +- name: "Package Enterprise Linux 7" + if: type != cron AND branch = master AND commit_message =~ /(\[Package arm64 RPM Enterprise Linux\]|\[Package arm64 RPM\]|\[Package i386 RPM Enterprise Linux\]|\[Package i386 RPM\]|\[Package AMD64 RPM Enterprise Linux\]|\[Package AMD64 RPM\])/ +- name: "Package Enterprise linux 6" + if: type != cron AND branch = master AND commit_message =~ /(\[Package i386 RPM Enterprise Linux\]|\[Package i386 RPM\]|\[Package AMD64 RPM Enterprise Linux\]|\[Package AMD64 RPM\])/ + + # Fedora +- name: "Package Fedora 30" + if: type != cron AND branch = master AND commit_message =~ /(\[Package arm64 RPM Fedora\]|\[Package arm64 RPM\]|\[Package AMD64 RPM Fedora\]|\[Package AMD64 RPM\])/ +- name: "Package Fedora 29" + if: type != cron AND branch = master AND commit_message =~ /(\[Package arm64 RPM Fedora\]|\[Package arm64 RPM\]|\[Package AMD64 RPM Fedora\]|\[Package AMD64 RPM\])/ +- name: "Package Fedora 28" + if: type != cron AND branch = master AND commit_message =~ /(\[Package arm64 RPM Fedora\]|\[Package arm64 RPM\]|\[Package AMD64 RPM Fedora\]|\[Package AMD64 RPM\])/ + + # OpenSuSE +- name: "Package OpenSuSE 15.1" + if: type != cron AND branch = master AND commit_message =~ /(\[Package arm64 RPM openSuSE\]|\[Package arm64 RPM\]|\[Package AMD64 RPM openSuSE\]|\[Package AMD64 RPM\])/ +- name: "Package OpenSuSE 15.0" + if: type != cron AND branch = master AND commit_message =~ /(\[Package arm64 RPM openSuSE\]|\[Package arm64 RPM\]|\[Package AMD64 RPM openSuSE\]|\[Package AMD64 RPM\])/ + + + +# DEB and RPM template flows +- stage: &_RPM_TEMPLATE + name: "Build & Publish RPM package" + before_install: + - sudo apt-get install -y wget lxc lxc-templates + - source tests/installer/slack.sh + before_script: + - post_message "TRAVIS_MESSAGE" "Starting package preparation and publishing for ${BUILD_STRING}.${BUILD_ARCH}" "${NOTIF_CHANNEL}" + - export PACKAGES_DIRECTORY="$(mktemp -d -t netdata-packaging-contents-dir-XXXXXX)" && echo "Created packaging directory ${PACKAGES_DIRECTORY}" + script: + - echo "Creating LXC environment for the build" && sudo -E .travis/package_management/create_lxc_for_build.sh + - echo "Building package in container" && sudo -E .travis/package_management/build_package_in_container.sh + - sudo chmod -R 755 "/var/lib/lxc" + - echo "Preparing RPM packaging contents for upload" && sudo -E .travis/package_management/prepare_packages.sh + after_failure: post_message "TRAVIS_MESSAGE" "Failed to build RPM for ${BUILD_STRING}.${BUILD_ARCH}" + + before_deploy: + - .travis/package_management/yank_stale_rpm.sh "${PACKAGES_DIRECTORY}" "${BUILD_STRING}" || echo "No stale RPM found" + deploy: + # Beta packages deployment + - provider: packagecloud + repository: "${DEPLOY_REPO}" + username: "${PACKAGING_USER}" + token: "${PKG_CLOUD_TOKEN}" + dist: "${BUILD_STRING}" + local_dir: "${PACKAGES_DIRECTORY}" + skip_cleanup: true + on: + # Only deploy on ${USER}/netdata, master branch, when packages directory is created + repo: ${TRAVIS_REPO_SLUG} + branch: master + condition: -d "${PACKAGES_DIRECTORY}" + # Production release packages deployment + - provider: packagecloud + repository: "netdata" + username: "netdata" + token: "${PKG_CLOUD_TOKEN}" + dist: "${BUILD_STRING}" + local_dir: "${PACKAGES_DIRECTORY}" + skip_cleanup: true + on: + # Only deploy on ${USER}/netdata, master branch, when packages directory is created + repo: "netdata/netdata" + branch: "master" + condition: -d "${PACKAGES_DIRECTORY}" + after_deploy: + - if [ -n "${BUILDER_NAME}" ]; then rm -rf /home/${BUILDER_NAME}/* && echo "Cleared /home/${BUILDER_NAME} directory" || echo "Failed to clean /home/${BUILDER_NAME} directory"; fi; + - if [ -d "${PACKAGES_DIRECTORY}" ]; then rm -rf "${PACKAGES_DIRECTORY}"; fi; + + + # TODO: This section is stale, will be aligned with the RPM implementation when we get to DEB packaging +- stage: &_DEB_TEMPLATE + name: "Build & Publish DEB package" + before_install: + - sudo apt-get install -y wget lxc lxc-templates + - source tests/installer/slack.sh + before_script: + - post_message "TRAVIS_MESSAGE" "Starting package preparation and publishing for ${BUILD_STRING}.${BUILD_ARCH}" "${NOTIF_CHANNEL}" + - export PACKAGES_DIRECTORY="$(mktemp -d -t netdata-packaging-contents-dir-XXXXXX)" && echo "Created packaging directory ${PACKAGES_DIRECTORY}" + script: + - echo "Creating LXC environment for the build" && sudo -E .travis/package_management/create_lxc_for_build.sh + - echo "Building package in container" && sudo -E .travis/package_management/build_package_in_container.sh + - sudo chmod -R 755 "/var/lib/lxc" + - echo "Preparing DEB packaging contents for upload" && sudo -E .travis/package_management/prepare_packages.sh + after_failure: post_message "TRAVIS_MESSAGE" "Failed to build DEB for ${BUILD_STRING}.${BUILD_ARCH}" + before_deploy: + - .travis/package_management/yank_stale_rpm.sh "${PACKAGES_DIRECTORY}" "${BUILD_STRING}" || echo "No stale DEB found" + deploy: + # Beta packages deployment + - provider: packagecloud + repository: "${DEPLOY_REPO}" + username: "${PACKAGING_USER}" + token: "${PKG_CLOUD_TOKEN}" + dist: "${BUILD_STRING}" + local_dir: "${PACKAGES_DIRECTORY}" + skip_cleanup: true + on: + # Only deploy on ${USER}/netdata, master branch, when build-area directory is created + repo: ${TRAVIS_REPO_SLUG} + branch: master + condition: -d "${PACKAGES_DIRECTORY}" + # Production release packages deployment + - provider: packagecloud + repository: "netdata" + username: "netdata" + token: "${PKG_CLOUD_TOKEN}" + dist: "${BUILD_STRING}" + local_dir: "${PACKAGES_DIRECTORY}" + skip_cleanup: true + on: + # Only deploy on ${USER}/netdata, master branch, when build-area directory is created + repo: "netdata/netdata" + branch: master + condition: -d "${PACKAGES_DIRECTORY}" + after_deploy: + - if [ -n "${BUILDER_NAME}" ]; then rm -rf /home/${BUILDER_NAME}/* && echo "Cleared /home/${BUILDER_NAME} directory" || echo "Failed to clean /home/${BUILDER_NAME} directory"; fi; + - if [ -d "${PACKAGES_DIRECTORY}" ]; then rm -rf "${PACKAGES_DIRECTORY}"; fi; + # Define stage implementation details @@ -88,7 +248,7 @@ jobs: - echo "packaging/version:" && cat packaging/version - docker run -it -v "${PWD}:/netdata:rw" -v "/tmp/netdata-makedist-test:/netdata_install:rw" -w /netdata "netdata/os-test:ubuntu1804" make clean || echo "Nothing to clean" - docker run -it -v "${PWD}:/netdata:rw" -v "/tmp/netdata-makedist-test:/netdata_install:rw" -w /netdata "netdata/os-test:ubuntu1804" make distclean || echo "Nothing to distclean" - - docker run -it -v "${PWD}:/netdata:rw" -v "/tmp/netdata-makedist-test:/netdata_install:rw" -w /netdata "netdata/os-test:ubuntu1804" autoreconf -ivf && ./configure --prefix=/netdata_install/usr --sysconfdir=/netdata_install/etc --localstatedir=/netdata_install/var --with-zlib --with-math --with-user=netdata CFLAGS=-O2 + - docker run -it -v "${PWD}:/netdata:rw" -v "/tmp/netdata-makedist-test:/netdata_install:rw" -w /netdata "netdata/os-test:ubuntu1804" /bin/bash -c "autoreconf -ivf && ./configure --prefix=/netdata_install/usr --sysconfdir=/netdata_install/etc --localstatedir=/netdata_install/var --with-zlib --with-math --with-user=netdata CFLAGS=-O2" - docker run -it -v "${PWD}:/netdata:rw" -v "/tmp/netdata-makedist-test:/netdata_install:rw" -w /netdata "netdata/os-test:ubuntu1804" make dist - docker run -it -v "${PWD}:/netdata:rw" -v "/tmp/netdata-makedist-test:/netdata_install:rw" -w /netdata "netdata/os-test:ubuntu1804" ls -ltr ./netdata-$(git describe).tar.gz || ls -ltr ./netdata-$(cat packaging/version | tr -d '\n').tar.gz - .travis/run_install_with_dist_file.sh @@ -135,10 +295,69 @@ jobs: + - stage: "Artifacts validation on bare OS, stable to current lifecycle checks" + + # Ubuntu runs + name: Run netdata lifecycle on Ubuntu 16.04 (xenial) + script: sudo -E tests/updater_checks.sh + after_failure: post_message "TRAVIS_MESSAGE" "Netdata updater process failed on bare Ubuntu 16.04" + + - name: Run netdata lifecycle, on Ubuntu 19.04 (Containerized) + script: docker run -it -v "${PWD}:/netdata:rw" -w /netdata "ubuntu:19.04" tests/updater_checks.sh + after_failure: post_message "TRAVIS_MESSAGE" "Netdata updater process failed on bare Ubuntu 19.04" + + # Centos runs + - name: Run netdata lifecycle on CentOS 7 (Containerized) + script: docker run -it -v "${PWD}:/netdata:rw" -w /netdata "centos:7" tests/updater_checks.sh + after_failure: post_message "TRAVIS_MESSAGE" "Netdata updater process failed on bare CentOS 7" + + # Debian runs + - name: Run netdata lifecycle, on Debian 9 (Containerized) + script: docker run -it -v "${PWD}:/netdata:rw" -w /netdata "debian:stretch" tests/updater_checks.sh + after_failure: post_message "TRAVIS_MESSAGE" "Netdata updater process failed on bare Debian 9 (stretch)" + + # openSuSE runs + - name: Run netdata lifecycle, on openSuSE 15.0 + script: docker run -it -v "${PWD}:/netdata:rw" -w /netdata "opensuse/leap:15.0" tests/updater_checks.sh + after_failure: post_message "TRAVIS_MESSAGE" "Netdata updater process failed on bare opensuse/leap:15.0" + + - name: Run netdata lifecycle, on openSuSE 15.1 + script: docker run -it -v "${PWD}:/netdata:rw" -w /netdata "opensuse/leap:15.1" tests/updater_checks.sh + after_failure: post_message "TRAVIS_MESSAGE" "Netdata updater process failed on bare opensuse/leap:15.1" + + - name: Run netdata lifecycle, on openSuSE Tumbleweed + script: docker run -it -v "${PWD}:/netdata:rw" -w /netdata "opensuse/tumbleweed:latest" tests/updater_checks.sh + after_failure: post_message "TRAVIS_MESSAGE" "Netdata updater process failed on bare opensuse/tumbleweed:latest" + + # Alpine runs + - name: Run netdata lifecycle, on Alpine linux + script: docker run -it -v "${PWD}:/netdata:rw" -w /netdata "alpine" tests/updater_checks.sh + after_failure: post_message "TRAVIS_MESSAGE" "Netdata updater process failed on bare Alpine" + + # Arch linux runs + - name: Run netdata lifecycle, on ArchLinux + script: docker run -it -v "${PWD}:/netdata:rw" -w /netdata "archlinux/base:latest" tests/updater_checks.sh + after_failure: post_message "TRAVIS_MESSAGE" "Netdata updater process failed on bare archlinux/base:latest" + + # Fedora runs + - name: Run netdata lifecycle, on Fedora 28 + script: docker run -it -v "${PWD}:/netdata:rw" -w /netdata "fedora:28" tests/updater_checks.sh + after_failure: post_message "TRAVIS_MESSAGE" "Netdata updater process failed on bare Fedora 28" + + - name: Run netdata lifecycle, on Fedora 29 + script: docker run -it -v "${PWD}:/netdata:rw" -w /netdata "fedora:29" tests/updater_checks.sh + after_failure: post_message "TRAVIS_MESSAGE" "Netdata updater process failed on bare Fedora 29" + + - name: Run netdata lifecycle, on Fedora 30 (Containerized) + script: docker run -it -v "${PWD}:/netdata:rw" -w /netdata "fedora:30" tests/updater_checks.sh + after_failure: post_message "TRAVIS_MESSAGE" "Netdata updater process failed on bare Fedora 30" + + + - stage: Packaging for release name: Generate changelog and TAG the release (only on special commit msg) - before_script: post_message "TRAVIS_MESSAGE" "Packaging step for release initiated" + before_script: post_message "TRAVIS_MESSAGE" "Packaging step for release initiated" "${NOTIF_CHANNEL}" script: - echo "GIT Branch:" && git branch - echo "Last commit:" && git log -1 @@ -154,11 +373,177 @@ jobs: - # We only publish if a TAG has been set during packaging + # ###### Packaging workflow section ###### + # References: + # https://us.images.linuxcontainers.org + # https://packagecloud.io/docs#install_repo + + # Ubuntu distros build + # + - stage: + <<: *_DEB_TEMPLATE + stage: "Package ubuntu/disco" + env: + - BUILDER_NAME="builder" BUILD_DISTRO="ubuntu" BUILD_RELEASE="disco" BUILD_STRING="ubuntu/disco" + - PACKAGE_TYPE="deb" REPO_TOOL="apt-get" + - ALLOW_SOFT_FAILURE_HERE=true + + + + - stage: + <<: *_DEB_TEMPLATE + stage: "Package ubuntu/cosmic" + env: + - BUILDER_NAME="builder" BUILD_DISTRO="ubuntu" BUILD_RELEASE="cosmic" BUILD_STRING="ubuntu/cosmic" + - PACKAGE_TYPE="deb" REPO_TOOL="apt-get" + - ALLOW_SOFT_FAILURE_HERE=true + + + + - stage: + <<: *_DEB_TEMPLATE + stage: "Package ubuntu/bionic" + env: + - BUILDER_NAME="builder" BUILD_DISTRO="ubuntu" BUILD_RELEASE="bionic" BUILD_STRING="ubuntu/bionic" + - PACKAGE_TYPE="deb" REPO_TOOL="apt-get" + - ALLOW_SOFT_FAILURE_HERE=true + + + + - stage: + <<: *_DEB_TEMPLATE + stage: "Package ubuntu/artful" + env: + - BUILDER_NAME="builder" BUILD_DISTRO="ubuntu" BUILD_RELEASE="artful" BUILD_STRING="ubuntu/artful" + - PACKAGE_TYPE="deb" REPO_TOOL="apt-get" + - ALLOW_SOFT_FAILURE_HERE=true + + + + # Debian distros build + - stage: + <<: *_DEB_TEMPLATE + stage: "Package debian/buster" + env: + - BUILDER_NAME="builder" BUILD_DISTRO="debian" BUILD_RELEASE="buster" BUILD_STRING="debian/buster" + - PACKAGE_TYPE="deb" REPO_TOOL="apt-get" + - ALLOW_SOFT_FAILURE_HERE=true + + + + - stage: + <<: *_DEB_TEMPLATE + stage: "Package debian/stretch" + env: + - BUILDER_NAME="builder" BUILD_DISTRO="debian" BUILD_RELEASE="stretch" BUILD_STRING="debian/stretch" + - PACKAGE_TYPE="deb" REPO_TOOL="apt-get" + - ALLOW_SOFT_FAILURE_HERE=true + + + + - stage: + <<: *_DEB_TEMPLATE + stage: "Package debian/jessie" + env: + - BUILDER_NAME="builder" BUILD_DISTRO="debian" BUILD_RELEASE="jessie" BUILD_STRING="debian/jessie" + - PACKAGE_TYPE="deb" REPO_TOOL="apt-get" + - ALLOW_SOFT_FAILURE_HERE=true + + + + - stage: + <<: *_DEB_TEMPLATE + stage: "Package debian/wheezy" + env: + - BUILDER_NAME="builder" BUILD_DISTRO="debian" BUILD_RELEASE="wheezy" BUILD_STRING="debian/wheezy" + - PACKAGE_TYPE="deb" REPO_TOOL="apt-get" + - ALLOW_SOFT_FAILURE_HERE=true + + + + # Enterprise linux builds (Centos, Redhat, Amazon linux (el/6)) + # + - stage: + <<: *_RPM_TEMPLATE + stage: "Package Enterprise Linux 7" + env: + - BUILDER_NAME="builder" BUILD_DISTRO="centos" BUILD_RELEASE="7" BUILD_STRING="el/7" + - PACKAGE_TYPE="rpm" REPO_TOOL="yum" + - ALLOW_SOFT_FAILURE_HERE=true + + + + - stage: + <<: *_RPM_TEMPLATE + stage: "Package Enterprise linux 6" + env: + - BUILDER_NAME="builder" BUILD_DISTRO="centos" BUILD_RELEASE="6" BUILD_STRING="el/6" + - PACKAGE_TYPE="rpm" REPO_TOOL="yum" + - ALLOW_SOFT_FAILURE_HERE=true + + + + # Fedora distros build + # + - stage: + <<: *_RPM_TEMPLATE + stage: "Package Fedora 30" + env: + - BUILDER_NAME="builder" BUILD_DISTRO="fedora" BUILD_RELEASE="30" BUILD_STRING="fedora/30" + - PACKAGE_TYPE="rpm" REPO_TOOL="dnf" + - ALLOW_SOFT_FAILURE_HERE=true + + + + - stage: + <<: *_RPM_TEMPLATE + stage: "Package Fedora 29" + env: + - BUILDER_NAME="builder" BUILD_DISTRO="fedora" BUILD_RELEASE="29" BUILD_STRING="fedora/29" + - PACKAGE_TYPE="rpm" REPO_TOOL="dnf" + - ALLOW_SOFT_FAILURE_HERE=true + + + + - stage: + <<: *_RPM_TEMPLATE + stage: "Package Fedora 28" + env: + - BUILDER_NAME="builder" BUILD_DISTRO="fedora" BUILD_RELEASE="28" BUILD_STRING="fedora/28" + - PACKAGE_TYPE="rpm" REPO_TOOL="dnf" + - ALLOW_SOFT_FAILURE_HERE=true + + + + # Opensuse distros build + # + - stage: + <<: *_RPM_TEMPLATE + stage: "Package OpenSuSE 15.1" + env: + - BUILDER_NAME="builder" BUILD_DISTRO="opensuse" BUILD_RELEASE="15.0" BUILD_STRING="opensuse/15.1" + - PACKAGE_TYPE="rpm" REPO_TOOL="zypper" + - ALLOW_SOFT_FAILURE_HERE=true + + + + - stage: + <<: *_RPM_TEMPLATE + stage: "Package OpenSuSE 15.0" + env: + - BUILDER_NAME="builder" BUILD_DISTRO="opensuse" BUILD_RELEASE="15.0" BUILD_STRING="opensuse/15.0" + - PACKAGE_TYPE="rpm" REPO_TOOL="zypper" + - ALLOW_SOFT_FAILURE_HERE=true + # ###### End of packaging workflow section ###### # + # ############################################### # + + + + # We only publish if a TAG has been set during packaging - stage: Publish for release name: Build & Publish docker images - before_script: post_message "TRAVIS_MESSAGE" "Publishing docker images" + before_script: post_message "TRAVIS_MESSAGE" "Publishing docker images" "${NOTIF_CHANNEL}" script: - echo "GIT Branch:" && git branch - echo "Last commit:" && git log -1 @@ -175,7 +560,7 @@ jobs: if: tag !~ /(-rc)/ - name: Create release draft - before_script: post_message "TRAVIS_MESSAGE" "Drafting release on github" + before_script: post_message "TRAVIS_MESSAGE" "Drafting release on github" "${NOTIF_CHANNEL}" script: - echo "GIT Branch:" && git branch - echo "Last commit:" && git log -1 @@ -191,12 +576,12 @@ jobs: - # This is the nightly pre-execution step (Jobs, preparatory steps for nightly, etc) + # This is the nightly pre-execution step (Jobs, preparatory steps for nightly, etc) - stage: Nightly operations name: Run coverity scan # Just notify people that Nightly ops triggered, use the first step as a hook to do that - before_script: post_message "TRAVIS_MESSAGE" "Starting nightly operations" + before_script: post_message "TRAVIS_MESSAGE" "Starting nightly operations" "${NOTIF_CHANNEL}" script: ./coverity-install.sh && ./coverity-scan.sh || echo "Coverity failed :(" - name: Kickstart files integrity testing (extended) @@ -207,7 +592,7 @@ jobs: # This is generating the changelog for nightly release and publish it - name: Generate nightly changelog - before_script: post_message "TRAVIS_MESSAGE" "Starting changelog generation for nightlies" + before_script: post_message "TRAVIS_MESSAGE" "Starting changelog generation for nightlies" "${NOTIF_CHANNEL}" script: ".travis/nightlies.sh" after_failure: post_message "TRAVIS_MESSAGE" "<!here> Nightly changelog generation failed" git: @@ -215,12 +600,12 @@ jobs: - # This is the nightly execution step - # + # This is the nightly execution step + # - stage: Nightly release name: Build & Publish docker images - before_script: post_message "TRAVIS_MESSAGE" "Publishing docker images for nightlies" + before_script: post_message "TRAVIS_MESSAGE" "Publishing docker images for nightlies" "${NOTIF_CHANNEL}" script: - echo "GIT Branch:" && git branch - echo "Last commit:" && git log -1 @@ -236,7 +621,7 @@ jobs: env: ALLOW_SOFT_FAILURE_HERE=true - name: Create nightly release artifacts, publish to GCS - before_script: post_message "TRAVIS_MESSAGE" "Starting artifacts generation for nightlies" + before_script: post_message "TRAVIS_MESSAGE" "Starting artifacts generation for nightlies" "${NOTIF_CHANNEL}" script: - echo "GIT Branch:" && git branch - echo "Last commit:" && git log -1 diff --git a/.travis/README.md b/.travis/README.md index 03ac2edd6..b7b61ecb4 100644 --- a/.travis/README.md +++ b/.travis/README.md @@ -95,3 +95,49 @@ During packaging we are preparing the release changelog information and run the ## Publish for release The publishing stage is the most complex part in publishing. This is the stage were we generate and publish docker images, prepare the release artifacts and get ready with the release draft. + +### Package Management workflows +As part of our goal to provide the best support to our customers, we have created a set of CI workflows to automatically produce +DEB and RPM for multiple distributions. These workflows are implemented under the templated stages '_DEB_TEMPLATE' and '_RPM_TEMPLATE'. +We currently plan to actively support the following Operating Systems, with a plan to further expand this list following our users needs. + +### Operating systems supported +The following distributions are supported +- Debian versions + - Buster (TBD - not released yet, check [debian releases](https://www.debian.org/releases/) for details) + - Stretch + - Jessie + - Wheezy + +- Ubuntu versions + - Disco + - Cosmic + - Bionic + - artful + +- Enterprise Linux versions (Covers Redhat, CentOS, and Amazon Linux with version 6) + - Version 8 (TBD) + - Version 7 + - Version 6 + +- Fedora versions + - Version 31 (TBD) + - Version 30 + - Version 29 + - Version 28 + +- OpenSuSE versions + - 15.1 + - 15.0 + +- Gentoo distributions + - TBD + +### Architectures supported +We plan to support amd64, x86 and arm64 architectures. As of June 2019 only amd64 and x86 will become available, as we are still working on solving issues with the architecture. + +The Package deployment can be triggered manually by executing an empty commit with the following message pattern: `[Package PACKAGE_TYPE PACKAGE_ARCH] DESCRIBE_THE_REASONING_HERE`. +Travis Yaml configuration allows the user to combine package type and architecture as necessary to regenerate the current stable release (For example tag v1.15.0 as of 4th of May 2019) +Sample patterns to trigger building of packages for all AMD64 supported architecture: +- '[Package AMD64 RPM]': Build & publish all amd64 available RPM packages +- '[Package AMD64 DEB]': Build & publish all amd64 available DEB packages diff --git a/.travis/labeler.sh b/.travis/labeler.sh index e8d7d228d..7863084d9 100755 --- a/.travis/labeler.sh +++ b/.travis/labeler.sh @@ -36,21 +36,6 @@ echo "===== Looking up available labels =====" LABELS_FILE=/tmp/labels hub issue labels >$LABELS_FILE -echo "===== Categorizing issues =====" -# This won't touch issues which already have at least one label assigned -for STATE in "open" "closed"; do - for ISSUE in $(hub issue -f "%I %l%n" -s "$STATE" -d "$(date +%F -d '1 day ago')" | grep -v -f $LABELS_FILE); do - echo "-------- Processing $STATE issue no. $ISSUE --------" - BODY="$(curl -H "Authorization: token $GITHUB_TOKEN" "https://api.github.com/repos/netdata/netdata/issues/$ISSUE" 2>/dev/null | jq .body)" - case "${BODY}" in - *"# Question summary"*) new_labels "$ISSUE" "question" "no changelog" ;; - *"# Bug report summary"*) new_labels "$ISSUE" "needs triage" "bug" ;; - *"# Feature idea summary"*) new_labels "$ISSUE" "needs triage" "feature request" ;; - *) new_labels "$ISSUE" "needs triage" "no changelog" ;; - esac - done -done - # Change all 'area' labels assigned to PR saving non-area labels. echo "===== Categorizing PRs =====" NEW_LABELS=/tmp/new_labels diff --git a/.travis/package_management/build_package_in_container.sh b/.travis/package_management/build_package_in_container.sh new file mode 100755 index 000000000..95a68e7a8 --- /dev/null +++ b/.travis/package_management/build_package_in_container.sh @@ -0,0 +1,82 @@ +#!/usr/bin/env bash +# +# Entry point for package build process +# +# Copyright: SPDX-License-Identifier: GPL-3.0-or-later +# +# Author : Pavlos Emm. Katsoulakis (paul@netdata.cloud) +#shellcheck disable=SC1091 +set -e + +# If we are not in netdata git repo, at the top level directory, fail +TOP_LEVEL=$(basename "$(git rev-parse --show-toplevel)") +CWD=$(git rev-parse --show-cdup) +if [ -n "$CWD" ] || [ ! "${TOP_LEVEL}" == "netdata" ]; then + echo "Run as .travis/package_management/$(basename "$0") from top level directory of netdata git repository" + echo "Docker build process aborted" + exit 1 +fi + +source .travis/package_management/functions.sh || (echo "Failed to load packaging library" && exit 1) + +# Check for presence of mandatory environment variables +if [ -z "${BUILD_STRING}" ]; then + echo "No Distribution was defined. Make sure BUILD_STRING is set on the environment before running this script" + exit 1 +fi + +if [ -z "${BUILDER_NAME}" ]; then + echo "No builder account and container name defined. Make sure BUILDER_NAME is set on the environment before running this script" + exit 1 +fi + +if [ -z "${BUILD_DISTRO}" ]; then + echo "No build distro information defined. Make sure BUILD_DISTRO is set on the environment before running this script" + exit 1 +fi + +if [ -z "${BUILD_RELEASE}" ]; then + echo "No build release information defined. Make sure BUILD_RELEASE is set on the environment before running this script" + exit 1 +fi + +if [ -z "${PACKAGE_TYPE}" ]; then + echo "No build release information defined. Make sure PACKAGE_TYPE is set on the environment before running this script" + exit 1 +fi + +# Detect architecture and load extra variables needed +detect_arch_from_commit + +case "${BUILD_ARCH}" in +"all") + echo "* * * Building all architectures, amd64 and i386 * * *" + echo "Building for amd64.." + export CONTAINER_NAME="${BUILDER_NAME}-${BUILD_DISTRO}${BUILD_RELEASE}-amd64" + export LXC_CONTAINER_ROOT="/var/lib/lxc/${CONTAINER_NAME}/rootfs" + .travis/package_management/trigger_${PACKAGE_TYPE}_lxc_build.py "${CONTAINER_NAME}" + + echo "Building for arm64.." + export CONTAINER_NAME="${BUILDER_NAME}-${BUILD_DISTRO}${BUILD_RELEASE}-arm64" + export LXC_CONTAINER_ROOT="/var/lib/lxc/${CONTAINER_NAME}/rootfs" + .travis/package_management/trigger_${PACKAGE_TYPE}_lxc_build.py "${CONTAINER_NAME}" + + echo "Building for i386.." + export CONTAINER_NAME="${BUILDER_NAME}-${BUILD_DISTRO}${BUILD_RELEASE}-i386" + export LXC_CONTAINER_ROOT="/var/lib/lxc/${CONTAINER_NAME}/rootfs" + .travis/package_management/trigger_${PACKAGE_TYPE}_lxc_build.py "${CONTAINER_NAME}" + + ;; +"amd64"|"arm64"|"i386") + echo "Building for ${BUILD_ARCH}.." + export CONTAINER_NAME="${BUILDER_NAME}-${BUILD_DISTRO}${BUILD_RELEASE}-${BUILD_ARCH}" + export LXC_CONTAINER_ROOT="/var/lib/lxc/${CONTAINER_NAME}/rootfs" + .travis/package_management/trigger_${PACKAGE_TYPE}_lxc_build.py "${CONTAINER_NAME}" + ;; +*) + echo "Unknown build architecture '${BUILD_ARCH}', nothing to do for build" + exit 1 + ;; +esac + +echo "Build process completed!" diff --git a/.travis/package_management/common.py b/.travis/package_management/common.py new file mode 100755 index 000000000..6cf59293d --- /dev/null +++ b/.travis/package_management/common.py @@ -0,0 +1,46 @@ +# +# +# Python library with commonly used functions within the package management scope +# +# Author : Pavlos Emm. Katsoulakis <paul@netdata.cloud> + +import lxc +import subprocess + +def replace_tag(tag_name, spec, new_tag_content): + print("Fixing tag %s in %s" % (tag_name, spec)) + + ifp = open(spec, "r") + config = ifp.readlines() + ifp.close() + + source_line = -1 + for line in config: + if str(line).count(tag_name + ":") > 0: + source_line = config.index(line) + print("Found line: %s in item %d" % (line, source_line)) + break + + if source_line >= 0: + print("Replacing line %s with %s in spec file" %(config[source_line], new_tag_content)) + config[source_line] = "%s: %s\n" % (tag_name, new_tag_content) + config_str = ''.join(config) + ofp = open(spec, 'w') + ofp.write(config_str) + ofp.close() + +def run_command(container, command): + print("Running command: %s" % command) + command_result = container.attach_wait(lxc.attach_run_command, command) + + if command_result != 0: + raise Exception("Command failed with exit code %d" % command_result) + +def run_command_in_host(cmd): + print("Issue command in host: %s" % str(cmd)) + + proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + o, e = proc.communicate() + print('Output: ' + o.decode('ascii')) + print('Error: ' + e.decode('ascii')) + print('code: ' + str(proc.returncode)) diff --git a/.travis/package_management/configure_deb_lxc_environment.py b/.travis/package_management/configure_deb_lxc_environment.py new file mode 100755 index 000000000..58999ad38 --- /dev/null +++ b/.travis/package_management/configure_deb_lxc_environment.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python3 +# +# Prepare the build environment within the container +# The script attaches to the running container and does the following: +# 1) Create the container +# 2) Start the container up +# 3) Create the builder user +# 4) Prepare the environment for DEB build +# +# Copyright: SPDX-License-Identifier: GPL-3.0-or-later +# +# Author : Pavlos Emm. Katsoulakis <paul@netdata.cloud> + +import common +import os +import sys +import lxc + +if len(sys.argv) != 2: + print('You need to provide a container name to get things started') + sys.exit(1) +container_name=sys.argv[1] + +# Setup the container object +print("Defining container %s" % container_name) +container = lxc.Container(container_name) +if not container.defined: + raise Exception("Container %s not defined!" % container_name) + +# Start the container +if not container.start(): + raise Exception("Failed to start the container") + +if not container.running or not container.state == "RUNNING": + raise Exception('Container %s is not running, configuration process aborted ' % container_name) + +# Wait for connectivity +print("Waiting for container connectivity to start configuration sequence") +if not container.get_ips(timeout=30): + raise Exception("Timeout while waiting for container") + +# Run the required activities now +# 1. Create the builder user +print("1. Adding user %s" % os.environ['BUILDER_NAME']) +common.run_command(container, ["useradd", "-m", os.environ['BUILDER_NAME']]) + +# Fetch package dependencies for the build +print("2. Installing package dependencies within LXC container") +common.run_command(container, ["apt-get", "update", "-y"]) +common.run_command(container, ["apt-get", "install", "-y", "sudo"]) +common.run_command(container, ["apt-get", "install", "-y", "wget"]) +common.run_command(container, ["apt-get", "install", "-y", "bash"]) +common.run_command(container, ["wget", "-T", "15", "-O", "~/.install-required-packages.sh", "https://raw.githubusercontent.com/netdata/netdata-demo-site/master/install-required-packages.sh"]) +common.run_command(container, ["bash", "~/.install-required-packages.sh", "netdata", "--dont-wait", "--non-interactive"]) + +# Download the source +dest_archive="/home/%s/netdata-%s.tar.gz" % (os.environ['BUILDER_NAME'],os.environ['BUILD_VERSION']) +release_url="https://github.com/netdata/netdata/releases/download/%s/netdata-%s.tar.gz" % (os.environ['BUILD_VERSION'], os.environ['BUILD_VERSION']) +print("3. Fetch netdata source (%s -> %s)" % (release_url, dest_archive)) +common.run_command(container, ["sudo", "-u", os.environ['BUILDER_NAME'], "wget", "-T", "15", "--output-document=" + dest_archive, release_url]) + +print("4. Extracting directory contents to /home " + os.environ['BUILDER_NAME']) +common.run_command(container, ["sudo", "-u", os.environ['BUILDER_NAME'], "tar", "xf", dest_archive, "-C", "/home/" + os.environ['BUILDER_NAME']]) + +print("Done!") diff --git a/.travis/package_management/configure_rpm_lxc_environment.py b/.travis/package_management/configure_rpm_lxc_environment.py new file mode 100755 index 000000000..644e027b7 --- /dev/null +++ b/.travis/package_management/configure_rpm_lxc_environment.py @@ -0,0 +1,159 @@ +#!/usr/bin/env python3 +# +# +# Prepare the build environment within the container +# The script attaches to the running container and does the following: +# 1) Create the container +# 2) Start the container up +# 3) Create the builder user +# 4) Prepare the environment for RPM build +# +# Copyright: SPDX-License-Identifier: GPL-3.0-or-later +# +# Author : Pavlos Emm. Katsoulakis <paul@netdata.cloud> + +import common +import os +import sys +import lxc + +if len(sys.argv) != 2: + print('You need to provide a container name to get things started') + sys.exit(1) +container_name=sys.argv[1] + +# Setup the container object +print("Defining container %s" % container_name) +container = lxc.Container(container_name) +if not container.defined: + raise Exception("Container %s not defined!" % container_name) + +# Start the container +if not container.start(): + raise Exception("Failed to start the container") + +if not container.running or not container.state == "RUNNING": + raise Exception('Container %s is not running, configuration process aborted ' % container_name) + +# Wait for connectivity +print("Waiting for container connectivity to start configuration sequence") +if not container.get_ips(timeout=30): + raise Exception("Timeout while waiting for container") + +# Run the required activities now +# Create the builder user +print("1. Adding user %s" % os.environ['BUILDER_NAME']) +common.run_command(container, ["useradd", "-m", os.environ['BUILDER_NAME']]) + +# Fetch package dependencies for the build +print("2. Installing package dependencies within LXC container") +if str(os.environ["REPO_TOOL"]).count("zypper") == 1: + common.run_command(container, [os.environ["REPO_TOOL"], "clean", "-a"]) + common.run_command(container, [os.environ["REPO_TOOL"], "--no-gpg-checks", "update", "-y"]) + common.run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "json-glib-devel"]) + +elif str(os.environ["REPO_TOOL"]).count("yum") == 1: + common.run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "json-c-devel"]) + common.run_command(container, [os.environ["REPO_TOOL"], "clean", "all"]) + common.run_command(container, [os.environ["REPO_TOOL"], "update", "-y"]) + common.run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "epel-release"]) +else: + common.run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "json-c-devel"]) + common.run_command(container, [os.environ["REPO_TOOL"], "update", "-y"]) + +common.run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "sudo"]) +common.run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "wget"]) +common.run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "bash"]) +common.run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "freeipmi-devel"]) +common.run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "cups-devel"]) + +# Exceptional cases, not available everywhere +# + +# Not on Centos-7 +if os.environ["BUILD_STRING"].count("el/7") <= 0: + common.run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "libnetfilter_acct-devel"]) + +# Not on Centos-6 +if os.environ["BUILD_STRING"].count("el/6") <= 0: + common.run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "autoconf-archive"]) + +common.run_command(container, ["wget", "-T", "15", "-O", "/home/%s/.install-required-packages.sh" % (os.environ['BUILDER_NAME']), "https://raw.githubusercontent.com/netdata/netdata-demo-site/master/install-required-packages.sh"]) +common.run_command(container, ["bash", "/home/%s/.install-required-packages.sh" % (os.environ['BUILDER_NAME']), "netdata", "--dont-wait", "--non-interactive"]) + +print("3. Setting up macros") +common.run_command(container, ["sudo", "-u", os.environ['BUILDER_NAME'], "/bin/echo", "'%_topdir %(echo /home/" + os.environ['BUILDER_NAME'] + ")/rpmbuild' > /home/" + os.environ['BUILDER_NAME'] + "/.rpmmacros"]) + +print("4. Create rpmbuild directory") +common.run_command(container, ["sudo", "-u", os.environ['BUILDER_NAME'], "mkdir", "-p", "/home/" + os.environ['BUILDER_NAME'] + "/rpmbuild/BUILD"]) +common.run_command(container, ["sudo", "-u", os.environ['BUILDER_NAME'], "mkdir", "-p", "/home/" + os.environ['BUILDER_NAME'] + "/rpmbuild/RPMS"]) +common.run_command(container, ["sudo", "-u", os.environ['BUILDER_NAME'], "mkdir", "-p", "/home/" + os.environ['BUILDER_NAME'] + "/rpmbuild/SOURCES"]) +common.run_command(container, ["sudo", "-u", os.environ['BUILDER_NAME'], "mkdir", "-p", "/home/" + os.environ['BUILDER_NAME'] + "/rpmbuild/SPECS"]) +common.run_command(container, ["sudo", "-u", os.environ['BUILDER_NAME'], "mkdir", "-p", "/home/" + os.environ['BUILDER_NAME'] + "/rpmbuild/SRPMS"]) +common.run_command(container, ["sudo", "-u", os.environ['BUILDER_NAME'], "ls", "-ltrR", "/home/" + os.environ['BUILDER_NAME'] + "/rpmbuild"]) + +# Download the source +rpm_friendly_version="" +dest_archive="" +download_url="" +spec_file="/home/%s/rpmbuild/SPECS/netdata.spec" % os.environ['BUILDER_NAME'] + +# TODO: Checksum validations +if str(os.environ['BUILD_VERSION']).count(".latest") == 1: + version_list=str(os.environ['BUILD_VERSION']).replace('v', '').split('.') + rpm_friendly_version='.'.join(version_list[0:3]) + "." + version_list[3] + + print("Building latest nightly version of netdata..(%s)" % os.environ['BUILD_VERSION']) + dest_archive="/home/%s/rpmbuild/SOURCES/netdata-%s.tar.gz" % (os.environ['BUILDER_NAME'], rpm_friendly_version) + + print("5. Preparing local latest implementation tarball for version %s" % rpm_friendly_version) + tar_file = os.environ['LXC_CONTAINER_ROOT'] + dest_archive + + print("5.1 Tagging the code with latest version: %s" % rpm_friendly_version) + common.run_command_in_host(['git', 'tag', '-a', rpm_friendly_version, '-m', 'Tagging while packaging on %s' % os.environ["CONTAINER_NAME"]]) + + print("5.2 Run autoreconf -ivf") + common.run_command_in_host(['autoreconf', '-ivf']) + + print("5.3 Run configure") + common.run_command_in_host(['./configure', '--with-math', '--with-zlib', '--with-user=netdata']) + + print("5.4 Run make dist") + common.run_command_in_host(['make', 'dist']) + + print("5.5 Copy generated tarbal to desired path") + if os.path.exists('netdata-%s.tar.gz' % rpm_friendly_version): + common.run_command_in_host(['sudo', 'cp', 'netdata-%s.tar.gz' % rpm_friendly_version, tar_file]) + + print("5.6 Fixing permissions on tarball") + common.run_command_in_host(['sudo', 'chmod', '777', tar_file]) + else: + print("I could not find (%s) on the disk, stopping the build. Kindly check the logs and try again" % 'netdata-%s.tar.gz' % rpm_friendly_version) + sys.exit(1) + + # Extract the spec file in place + print("6. Extract spec file from the source") + common.run_command_in_host(['sudo', 'cp', 'netdata.spec', os.environ['LXC_CONTAINER_ROOT'] + spec_file]) + common.run_command_in_host(['sudo', 'chmod', '777', os.environ['LXC_CONTAINER_ROOT'] + spec_file]) + + print("7. Temporary hack: Change Source0 to %s on spec file %s" % (dest_archive, spec_file)) + common.replace_tag("Source0", os.environ['LXC_CONTAINER_ROOT'] + spec_file, tar_file) +else: + rpm_friendly_version = os.environ['BUILD_VERSION'] + + print("Building latest stable version of netdata.. (%s)" % os.environ['BUILD_VERSION']) + dest_archive="/home/%s/rpmbuild/SOURCES/netdata-%s.tar.gz" % (os.environ['BUILDER_NAME'],os.environ['BUILD_VERSION']) + download_url="https://github.com/netdata/netdata/releases/download/%s/netdata-%s.tar.gz" % (os.environ['BUILD_VERSION'], os.environ['BUILD_VERSION']) + + print("5. Fetch netdata source into the repo structure(%s -> %s)" % (download_url, dest_archive)) + tar_file="%s/netdata-%s.tar.gz" % (os.path.dirname(dest_archive), rpm_friendly_version) + common.run_command(container, ["sudo", "-u", os.environ['BUILDER_NAME'], "wget", "-T", "15", "--output-document=" + dest_archive, download_url]) + + print("6.Extract spec file from the source") + common.run_command(container, ["sudo", "-u", os.environ['BUILDER_NAME'], "tar", "--to-command=cat > %s" % spec_file, "-xvf", dest_archive, "netdata-%s/netdata.spec" % os.environ['BUILD_VERSION']]) + + print("7. Temporary hack: Adjust version string on the spec file (%s) to %s and Source0 to %s" % (os.environ['LXC_CONTAINER_ROOT'] + spec_file, rpm_friendly_version, download_url)) + common.replace_tag("Version", os.environ['LXC_CONTAINER_ROOT'] + spec_file, rpm_friendly_version) + common.replace_tag("Source0", os.environ['LXC_CONTAINER_ROOT'] + spec_file, tar_file) + +print('Done!') diff --git a/.travis/package_management/create_lxc_for_build.sh b/.travis/package_management/create_lxc_for_build.sh new file mode 100755 index 000000000..ae855a742 --- /dev/null +++ b/.travis/package_management/create_lxc_for_build.sh @@ -0,0 +1,101 @@ +#!/usr/bin/env bash +# +# This script generates an LXC container and starts it up +# Once the script completes successfully, a container has become available for usage +# The container image to be used and the container name to be set, are part of variables +# that must be present for the script to work +# +# Copyright: SPDX-License-Identifier: GPL-3.0-or-later +# +# Author : Pavlos Emm. Katsoulakis (paul@netdata.cloud) +# shellcheck disable=SC1091 +set -e + +source .travis/package_management/functions.sh || (echo "Failed to load packaging library" && exit 1) + +# If we are not in netdata git repo, at the top level directory, fail +TOP_LEVEL=$(basename "$(git rev-parse --show-toplevel)") +CWD=$(git rev-parse --show-cdup) +if [ -n "$CWD" ] || [ ! "${TOP_LEVEL}" == "netdata" ]; then + echo "Run as .travis/package_management/$(basename "$0") from top level directory of netdata git repository" + echo "LXC Container creation aborted" + exit 1 +fi + +# Check for presence of mandatory environment variables +if [ -z "${BUILD_STRING}" ]; then + echo "No Distribution was defined. Make sure BUILD_STRING is set on the environment before running this script" + exit 1 +fi + +if [ -z "${BUILDER_NAME}" ]; then + echo "No builder account and container name defined. Make sure BUILDER_NAME is set on the environment before running this script" + exit 1 +fi + +if [ -z "${BUILD_DISTRO}" ]; then + echo "No build distro information defined. Make sure BUILD_DISTRO is set on the environment before running this script" + exit 1 +fi + +if [ -z "${BUILD_RELEASE}" ]; then + echo "No build release information defined. Make sure BUILD_RELEASE is set on the environment before running this script" + exit 1 +fi + +if [ -z "${PACKAGE_TYPE}" ]; then + echo "No build release information defined. Make sure PACKAGE_TYPE is set on the environment before running this script" + exit 1 +fi + +# Detect architecture and load extra variables needed +detect_arch_from_commit + +echo "Creating LXC container ${BUILDER_NAME}/${BUILD_STRING}/${BUILD_ARCH}...." + +case "${BUILD_ARCH}" in +"all") + # i386 + echo "Creating LXC Container for i386.." + export CONTAINER_NAME="${BUILDER_NAME}-${BUILD_DISTRO}${BUILD_RELEASE}-i386" + export LXC_CONTAINER_ROOT="/var/lib/lxc/${CONTAINER_NAME}/rootfs" + lxc-create -n "${CONTAINER_NAME}" -t "download" -- --dist "${BUILD_DISTRO}" --release "${BUILD_RELEASE}" --arch "i386" --no-validate + + echo "Container(s) ready. Configuring container(s).." + .travis/package_management/configure_${PACKAGE_TYPE}_lxc_environment.py "${CONTAINER_NAME}" + + # amd64 + echo "Creating LXC Container for amd64.." + export CONTAINER_NAME="${BUILDER_NAME}-${BUILD_DISTRO}${BUILD_RELEASE}-amd64" + export LXC_CONTAINER_ROOT="/var/lib/lxc/${CONTAINER_NAME}/rootfs" + lxc-create -n "${CONTAINER_NAME}" -t "download" -- --dist "${BUILD_DISTRO}" --release "${BUILD_RELEASE}" --arch "amd64" --no-validate + + echo "Container(s) ready. Configuring container(s).." + .travis/package_management/configure_${PACKAGE_TYPE}_lxc_environment.py "${CONTAINER_NAME}" + + # arm64 + echo "Creating LXC Container for arm64.." + export CONTAINER_NAME="${BUILDER_NAME}-${BUILD_DISTRO}${BUILD_RELEASE}-arm64" + export LXC_CONTAINER_ROOT="/var/lib/lxc/${CONTAINER_NAME}/rootfs" + lxc-create -n "${CONTAINER_NAME}" -t "download" -- --dist "${BUILD_DISTRO}" --release "${BUILD_RELEASE}" --arch "arm64" --no-validate + + echo "Container(s) ready. Configuring container(s).." + .travis/package_management/configure_${PACKAGE_TYPE}_lxc_environment.py "${CONTAINER_NAME}" + ;; +"i386"|"amd64"|"arm64") + # AMD64 or i386 + echo "Creating LXC Container for ${BUILD_ARCH}.." + export CONTAINER_NAME="${BUILDER_NAME}-${BUILD_DISTRO}${BUILD_RELEASE}-${BUILD_ARCH}" + export LXC_CONTAINER_ROOT="/var/lib/lxc/${CONTAINER_NAME}/rootfs" + lxc-create -n "${CONTAINER_NAME}" -t "download" -- --dist "${BUILD_DISTRO}" --release "${BUILD_RELEASE}" --arch "${BUILD_ARCH}" --no-validate + + echo "Container(s) ready. Configuring container(s).." + .travis/package_management/configure_${PACKAGE_TYPE}_lxc_environment.py "${CONTAINER_NAME}" + ;; +*) + echo "Unknown BUILD_ARCH value '${BUILD_ARCH}' given, process failed" + exit 1 + ;; +esac + +echo "..LXC creation complete!" diff --git a/.travis/package_management/functions.sh b/.travis/package_management/functions.sh new file mode 100644 index 000000000..9a467ffe1 --- /dev/null +++ b/.travis/package_management/functions.sh @@ -0,0 +1,33 @@ +# no-shebang-needed-its-a-library +# +# Utility functions for packaging in travis CI +# +# Copyright: SPDX-License-Identifier: GPL-3.0-or-later +# +# Author : Pavlos Emm. Katsoulakis (paul@netdata.cloud) +#shellcheck disable=SC2148 +set -e + +function detect_arch_from_commit { + case "${TRAVIS_COMMIT_MESSAGE}" in + "[Package AMD64"*) + export BUILD_ARCH="amd64" + ;; + "[Package i386"*) + export BUILD_ARCH="i386" + ;; + "[Package ALL"*) + export BUILD_ARCH="all" + ;; + "[Package arm64"*) + export BUILD_ARCH="arm64" + ;; + + *) + echo "Unknown build architecture '${BUILD_ARCH}' provided" + exit 1 + ;; + esac + + echo "Detected build architecture ${BUILD_ARCH}" +} diff --git a/.travis/package_management/package_cloud_wrapper.sh b/.travis/package_management/package_cloud_wrapper.sh new file mode 100755 index 000000000..48a372d37 --- /dev/null +++ b/.travis/package_management/package_cloud_wrapper.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash +# +# This is a tool to help removal of packages from packagecloud.io +# It utilizes the package_cloud utility provided from packagecloud.io +# +# Depends on: +# 1) package cloud gem (detects absence and installs it) +# +# Requires: +# 1) PKG_CLOUD_TOKEN variable exported +# 2) To properly install package_cloud when not found, it requires: ruby gcc gcc-c++ ruby-devel +# +# Copyright: SPDX-License-Identifier: GPL-3.0-or-later +# +# Author : Pavlos Emm. Katsoulakis (paul@netdata.cloud) +#shellcheck disable=SC2068,SC2145 +set -e +PKG_CLOUD_CONFIG="$HOME/.package_cloud_configuration.cfg" + +# If we are not in netdata git repo, at the top level directory, fail +TOP_LEVEL=$(basename "$(git rev-parse --show-toplevel)") +CWD=$(git rev-parse --show-cdup) +if [ -n "$CWD" ] || [ ! "${TOP_LEVEL}" == "netdata" ]; then + echo "Run as .travis/package_management/$(basename "$0") from top level directory of netdata git repository" + echo "Docker build process aborted" + exit 1 +fi + +# Install dependency if not there +if ! command -v package_cloud > /dev/null 2>&1; then + echo "No package cloud gem found, installing" + gem install -V package_cloud || (echo "Package cloud installation failed. you might want to check if required dependencies are there (ruby gcc gcc-c++ ruby-devel)" && exit 1) +else + echo "Found package_cloud gem, continuing" +fi + +# Check for required token and prepare config +if [ -z "${PKG_CLOUD_TOKEN}" ]; then + echo "Please set PKG_CLOUD_TOKEN to be able to use ${0}" + exit 1 +fi +echo "{\"url\":\"https://packagecloud.io\",\"token\":\"${PKG_CLOUD_TOKEN}\"}" > "${PKG_CLOUD_CONFIG}" + +echo "Executing package_cloud with config ${PKG_CLOUD_CONFIG} and parameters $@" +package_cloud $@ --config="${PKG_CLOUD_CONFIG}" + +rm -rf "${PKG_CLOUD_CONFIG}" +echo "Done!" diff --git a/.travis/package_management/prepare_packages.sh b/.travis/package_management/prepare_packages.sh new file mode 100755 index 000000000..1fb26a95e --- /dev/null +++ b/.travis/package_management/prepare_packages.sh @@ -0,0 +1,56 @@ +#!/usr/bin/env bash +# +# Utility that gathers generated packages, +# puts them together in a local folder for deploy facility to pick up +# +# Copyright: SPDX-License-Identifier: GPL-3.0-or-later +# +# Author : Pavlos Emm. Katsoulakis (paul@netdata.cloud) +#shellcheck disable=SC2068 +set -e + +# If we are not in netdata git repo, at the top level directory, fail +TOP_LEVEL=$(basename "$(git rev-parse --show-toplevel)") +CWD=$(git rev-parse --show-cdup) +if [ -n "$CWD" ] || [ ! "${TOP_LEVEL}" == "netdata" ]; then + echo "Run as .travis/package_management/$(basename "$0") from top level directory of netdata git repository" + echo "Package preparation aborted" + exit 1 +fi + +export LXC_ROOT="/var/lib/lxc" + +# Go through the containers created for packaging and pick up all generated packages +CREATED_CONTAINERS=$(ls -A "${LXC_ROOT}") +for d in ${CREATED_CONTAINERS[@]}; do + echo "Picking up packaging contents from ${d}" + + # Pick up any RPMS from builder + RPM_BUILD_PATH="${LXC_ROOT}/${d}/rootfs/home/${BUILDER_NAME}/rpmbuild" + echo "Checking folder ${RPM_BUILD_PATH} for RPMS and SRPMS" + + if [ -d "${RPM_BUILD_PATH}/RPMS" ]; then + echo "Copying any RPMS in '${RPM_BUILD_PATH}', copying over the following:" + ls -ltrR "${RPM_BUILD_PATH}/RPMS" + [[ -d "${RPM_BUILD_PATH}/RPMS/x86_64" ]] && cp -r "${RPM_BUILD_PATH}"/RPMS/x86_64/* "${PACKAGES_DIRECTORY}" + [[ -d "${RPM_BUILD_PATH}/RPMS/i386" ]] && cp -r "${RPM_BUILD_PATH}"/RPMS/i386/* "${PACKAGES_DIRECTORY}" + [[ -d "${RPM_BUILD_PATH}/RPMS/i686" ]] && cp -r "${RPM_BUILD_PATH}"/RPMS/i686/* "${PACKAGES_DIRECTORY}" + fi + + if [ -d "${RPM_BUILD_PATH}/SRPMS" ]; then + echo "Copying any SRPMS in '${RPM_BUILD_PATH}', copying over the following:" + ls -ltrR "${RPM_BUILD_PATH}/SRPMS" + [[ -d "${RPM_BUILD_PATH}/SRPMS/x86_64" ]] && cp -r "${RPM_BUILD_PATH}"/SRPMS/x86_64/* "${PACKAGES_DIRECTORY}" + [[ -d "${RPM_BUILD_PATH}/SRPMS/i386" ]] && cp -r "${RPM_BUILD_PATH}"/SRPMS/i386/* "${PACKAGES_DIRECTORY}" + [[ -d "${RPM_BUILD_PATH}/SRPMS/i686" ]] && cp -r "${RPM_BUILD_PATH}"/SRPMS/i686/* "${PACKAGES_DIRECTORY}" + fi + + # Pick up any DEBs from builder + DEB_BUILD_PATH="${d}/home/${BUILDER_NAME}/build-area" + echo "Checking folder ${DEB_BUILD_PATH} for DEB packages" + #TODO: During debian clean up we 'll fill this up + +done + +chmod -R 777 "${PACKAGES_DIRECTORY}" +echo "Packaging contents ready to ship!" diff --git a/.travis/package_management/trigger_deb_lxc_build.py b/.travis/package_management/trigger_deb_lxc_build.py new file mode 100755 index 000000000..3040bdd62 --- /dev/null +++ b/.travis/package_management/trigger_deb_lxc_build.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python3 +# +# This script is responsible for running the RPM build on the running container +# +# Copyright: SPDX-License-Identifier: GPL-3.0-or-later +# +# Author : Pavlos Emm. Katsoulakis <paul@netdata.cloud> + +import common +import os +import sys +import lxc + +if len(sys.argv) != 2: + print('You need to provide a container name to get things started') + sys.exit(1) +container_name=sys.argv[1] + +# Load the container, break if its not there +print("Starting up container %s" % container_name) +container = lxc.Container(container_name) +if not container.defined: + raise Exception("Container %s does not exist!" % container_name) + +# Check if the container is running, attempt to start it up in case its not running +if not container.running or not container.state == "RUNNING": + print('Container %s is not running, attempt to start it up' % container_name) + + # Start the container + if not container.start(): + raise Exception("Failed to start the container") + + if not container.running or not container.state == "RUNNING": + raise Exception('Container %s is not running, configuration process aborted ' % container_name) + +# Wait for connectivity +if not container.get_ips(timeout=30): + raise Exception("Timeout while waiting for container") + +print("Setting up EMAIL and DEBFULLNAME variables required by the build tools") +os.environ["EMAIL"] = "bot@netdata.cloud" +os.environ["DEBFULLNAME"] = "Netdata builder" + +# Run the build process on the container +print("Starting DEB build process, running dh-make") +new_version = os.environ["BUILD_VERSION"].replace('v', '') + +print("Building the package") +common.run_command(container, ["sudo", "-u", os.environ['BUILDER_NAME'], "dpkg-buildpackage", "--host-arch", "amd64", "--target-arch", "amd64", "--post-clean", "--pre-clean", "--build=binary", "--release-by=\"Netdata Builder\"", "--build-by=\"Netdata Builder\""]) + +print('Done!') diff --git a/.travis/package_management/trigger_rpm_lxc_build.py b/.travis/package_management/trigger_rpm_lxc_build.py new file mode 100755 index 000000000..f9e109c72 --- /dev/null +++ b/.travis/package_management/trigger_rpm_lxc_build.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python3 +# +# This script is responsible for running the RPM build on the running container +# +# Copyright: SPDX-License-Identifier: GPL-3.0-or-later +# +# Author : Pavlos Emm. Katsoulakis <paul@netdata.cloud> + +import common +import os +import sys +import lxc + +if len(sys.argv) != 2: + print('You need to provide a container name to get things started') + sys.exit(1) +container_name=sys.argv[1] + +# Load the container, break if its not there +print("Starting up container %s" % container_name) +container = lxc.Container(container_name) +if not container.defined: + raise Exception("Container %s does not exist!" % container_name) + +# Check if the container is running, attempt to start it up in case its not running +if not container.running or not container.state == "RUNNING": + print('Container %s is not running, attempt to start it up' % container_name) + + # Start the container + if not container.start(): + raise Exception("Failed to start the container") + + if not container.running or not container.state == "RUNNING": + raise Exception('Container %s is not running, configuration process aborted ' % container_name) + +# Wait for connectivity +if not container.get_ips(timeout=30): + raise Exception("Timeout while waiting for container") + +print("Adding builder specific dependencies to the LXC container") +common.run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "rpm-build"]) +common.run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "rpm-devel"]) +common.run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "rpmlint"]) +common.run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "make"]) +common.run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "python"]) +common.run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "bash"]) +common.run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "diffutils"]) +common.run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "patch"]) +common.run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "rpmdevtools"]) + +# Run the build process on the container +print("Starting RPM build process") +common.run_command(container, ["sudo", "-u", os.environ['BUILDER_NAME'], "rpmbuild", "-ba", "--rebuild", "/home/%s/rpmbuild/SPECS/netdata.spec" % os.environ['BUILDER_NAME']]) + +print('Done!') diff --git a/.travis/package_management/yank_stale_rpm.sh b/.travis/package_management/yank_stale_rpm.sh new file mode 100755 index 000000000..5cf938664 --- /dev/null +++ b/.travis/package_management/yank_stale_rpm.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +# +# This script is responsible for the removal of stale RPM/DEB files. +# It runs on the pre-deploy step and takes care of the removal of the files +# prior to the upload of the freshly built ones +# +# Copyright: SPDX-License-Identifier: GPL-3.0-or-later +# +# Author : Pavlos Emm. Katsoulakis (paul@netdata.cloud) +#shellcheck disable=SC2010,SC2068 +set -e + +# If we are not in netdata git repo, at the top level directory, fail +TOP_LEVEL=$(basename "$(git rev-parse --show-toplevel)") +CWD=$(git rev-parse --show-cdup) +if [ -n "$CWD" ] || [ ! "${TOP_LEVEL}" == "netdata" ]; then + echo "Run as .travis/package_management/$(basename "$0") from top level directory of netdata git repository" + echo "Package yanking cancelled" + exit 1 +fi + +PACKAGES_DIR="$1" +DISTRO="$2" +PACKAGES_LIST="$(ls -AR "${PACKAGES_DIR}" | grep '\.rpm')" + +if [ ! -d "${PACKAGES_DIR}" ] || [ -z "${PACKAGES_LIST}" ]; then + echo "Folder ${PACKAGES_DIR} does not seem to be a valid directory or is empty. No packages to check for yanking" + exit 1 +fi + +for pkg in ${PACKAGES_LIST[@]}; do + echo "Attempting yank on ${pkg}.." + .travis/package_management/package_cloud_wrapper.sh yank "${PACKAGING_USER}/${DEPLOY_REPO}/${DISTRO}" "${pkg}" || echo "Nothing to yank or error on ${pkg}" +done + |