diff options
Diffstat (limited to '')
| -rw-r--r-- | .travis.yml | 1102 | ||||
| -rw-r--r-- | .travis/README.md | 6 | ||||
| -rwxr-xr-x | .travis/check_changelog_last_modification.sh | 21 | ||||
| -rwxr-xr-x | .travis/generate_changelog_for_nightlies.sh | 1 | ||||
| -rwxr-xr-x | .travis/nightlies.sh | 8 | ||||
| -rw-r--r-- | .travis/package_management/build.sh | 31 | ||||
| -rwxr-xr-x | .travis/package_management/common.py | 116 | ||||
| -rwxr-xr-x | .travis/package_management/configure_deb_lxc_environment.py | 62 | ||||
| -rwxr-xr-x | .travis/package_management/configure_rpm_lxc_environment.py | 97 | ||||
| -rwxr-xr-x | .travis/package_management/create_lxc_for_build.sh | 2 | ||||
| -rw-r--r-- | .travis/package_management/functions.sh | 4 | ||||
| -rwxr-xr-x | .travis/package_management/prepare_packages.sh | 51 | ||||
| -rwxr-xr-x | .travis/package_management/trigger_deb_lxc_build.py | 30 | ||||
| -rwxr-xr-x | .travis/package_management/yank_stale_rpm.sh | 2 |
14 files changed, 838 insertions, 695 deletions
diff --git a/.travis.yml b/.travis.yml index cb9d72901..a239a0c64 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,7 +2,7 @@ dist: xenial sudo: true language: c services: -- docker + - docker @@ -16,18 +16,20 @@ matrix: # Install dependencies for all, once # install: -- sudo apt-get install -y libcap2-bin zlib1g-dev uuid-dev fakeroot libipmimonitoring-dev libmnl-dev libnetfilter-acct-dev gnupg python-pip -- sudo apt install -y --only-upgrade docker-ce -- sudo pip install git-semver -- docker info -- source tests/installer/slack.sh -- export NOTIF_CHANNEL="automation-beta" -- if [ "${TRAVIS_REPO_SLUG}" = "netdata/netdata" ]; then export NOTIF_CHANNEL="automation"; fi; -- export BUILD_VERSION="$(cat packaging/version | cut -d'-' -f1)" -- if [[ "${TRAVIS_COMMIT_MESSAGE}" = *"[Build latest]"* ]]; then export BUILD_VERSION="$(cat packaging/version | cut -d'-' -f1,2 | sed -e 's/-/./g').latest"; fi; -- export DEPLOY_REPO="netdata" # Default production packaging repository -- if [[ "${TRAVIS_COMMIT_MESSAGE}" = *"[Build latest]"* ]]; then export DEPLOY_REPO="netdata-edge"; fi; -- export PACKAGING_USER="$(echo ${TRAVIS_REPO_SLUG} | cut -d'/' -f1)" + - sudo apt-get install -y libcap2-bin zlib1g-dev uuid-dev fakeroot libipmimonitoring-dev libmnl-dev libnetfilter-acct-dev gnupg python-pip + - sudo apt install -y --only-upgrade docker-ce + - sudo pip install git-semver + - docker info + - source tests/installer/slack.sh + - export NOTIF_CHANNEL="automation-beta" + - if [ "${TRAVIS_REPO_SLUG}" = "netdata/netdata" ]; then export NOTIF_CHANNEL="automation"; fi; + - export BUILD_VERSION="$(cat packaging/version | cut -d'-' -f1)" + - export LATEST_RELEASE_VERSION="$(cat packaging/version | cut -d'-' -f1)" + - export LATEST_RELEASE_DATE="$(git log -1 --format=%aD "${LATEST_RELEASE_VERSION}" | cat)" + - if [[ "${TRAVIS_COMMIT_MESSAGE}" = *"[Build latest]"* ]]; then export BUILD_VERSION="$(cat packaging/version | cut -d'-' -f1,2 | sed -e 's/-/./g').latest"; fi; + - export DEPLOY_REPO="netdata" # Default production packaging repository + - if [[ "${TRAVIS_COMMIT_MESSAGE}" = *"[Build latest]"* ]]; then export DEPLOY_REPO="netdata-edge"; fi; + - export PACKAGING_USER="$(echo ${TRAVIS_REPO_SLUG} | cut -d'/' -f1)" @@ -41,167 +43,148 @@ notifications: # Define the stage sequence and conditionals # stages: -# Mandatory runs, we always want these executed -- name: Code quality, linting, syntax, code style -- name: Build process -- name: Artifacts validation -- name: Artifacts validation on bare OS, stable to current lifecycle checks - if: branch = master AND (type = pull_request OR type = cron) - - # Nightly operations -- name: Nightly operations - if: branch = master AND type = cron -- name: Nightly release - if: branch = master AND type = cron - - # Scheduled releases -- name: Packaging for release - if: branch = master AND type != pull_request AND type != cron - -- name: Publish for release - if: branch = master AND type != pull_request AND type != cron AND commit_message =~ /(\[netdata release candidate\]|\[netdata major release\]|\[netdata minor release\]|\[netdata patch release\])/ - - # Build DEB packages under special conditions - # Ubuntu -- name: "Package ubuntu/disco" - if: type != cron AND branch = master AND commit_message =~ /(\[Package arm64 DEB Ubuntu\]|\[Package arm64 DEB\]|\[Package i386 DEB Ubuntu\]|\[Package i386 DEB\]|\[Package AMD64 DEB Ubuntu\]|\[Package AMD64 DEB\])/ -- name: "Package ubuntu/cosmic" - if: type != cron AND branch = master AND commit_message =~ /(\[Package arm64 DEB Ubuntu\]|\[Package arm64 DEB\]|\[Package i386 DEB Ubuntu\]|\[Package i386 DEB\]|\[Package AMD64 DEB Ubuntu\]|\[Package AMD64 DEB\])/ -- name: "Package ubuntu/bionic" - if: type != cron AND branch = master AND commit_message =~ /(\[Package arm64 DEB Ubuntu\]|\[Package arm64 DEB\]|\[Package i386 DEB Ubuntu\]|\[Package i386 DEB\]|\[Package AMD64 DEB Ubuntu\]|\[Package AMD64 DEB\])/ -- name: "Package ubuntu/artful" - if: type != cron AND branch = master AND commit_message =~ /(\[Package arm64 DEB Ubuntu\]|\[Package arm64 DEB\]|\[Package i386 DEB Ubuntu\]|\[Package i386 DEB\]|\[Package AMD64 DEB Ubuntu\]|\[Package AMD64 DEB\])/ - - # Debian -- name: "Package debian/buster" - if: type != cron AND branch = master AND commit_message =~ /(\[Package arm64 DEB Debian\]|\[Package arm64 DEB\]|\[Package i386 DEB Debian\]|\[Package i386 DEB\]|\[Package AMD64 DEB Debian\]|\[Package AMD64 DEB\])/ -- name: "Package debian/stretch" - if: type != cron AND branch = master AND commit_message =~ /(\[Package arm64 DEB Debian\]|\[Package arm64 DEB\]|\[Package i386 DEB Debian\]|\[Package i386 DEB\]|\[Package AMD64 DEB Debian\]|\[Package AMD64 DEB\])/ -- name: "Package debian/jessie" - if: type != cron AND branch = master AND commit_message =~ /(\[Package arm64 DEB Debian\]|\[Package arm64 DEB\]|\[Package i386 DEB Debian\]|\[Package i386 DEB\]|\[Package AMD64 DEB Debian\]|\[Package AMD64 DEB\])/ -- name: "Package debian/wheezy" - if: type != cron AND branch = master AND commit_message =~ /(\[Package arm64 DEB Debian\]|\[Package arm64 DEB\]|\[Package i386 DEB Debian\]|\[Package i386 DEB\]|\[Package AMD64 DEB Debian\]|\[Package AMD64 DEB\])/ - - # Build RPM packages under special conditions - # Enterprise linux (Covers CentOS, Redhat, Amazon linux) -- name: "Package Enterprise Linux 7" - if: type != cron AND branch = master AND commit_message =~ /(\[Package arm64 RPM Enterprise Linux\]|\[Package arm64 RPM\]|\[Package i386 RPM Enterprise Linux\]|\[Package i386 RPM\]|\[Package AMD64 RPM Enterprise Linux\]|\[Package AMD64 RPM\])/ -- name: "Package Enterprise linux 6" - if: type != cron AND branch = master AND commit_message =~ /(\[Package i386 RPM Enterprise Linux\]|\[Package i386 RPM\]|\[Package AMD64 RPM Enterprise Linux\]|\[Package AMD64 RPM\])/ - - # Fedora -- name: "Package Fedora 30" - if: type != cron AND branch = master AND commit_message =~ /(\[Package arm64 RPM Fedora\]|\[Package arm64 RPM\]|\[Package AMD64 RPM Fedora\]|\[Package AMD64 RPM\])/ -- name: "Package Fedora 29" - if: type != cron AND branch = master AND commit_message =~ /(\[Package arm64 RPM Fedora\]|\[Package arm64 RPM\]|\[Package AMD64 RPM Fedora\]|\[Package AMD64 RPM\])/ -- name: "Package Fedora 28" - if: type != cron AND branch = master AND commit_message =~ /(\[Package arm64 RPM Fedora\]|\[Package arm64 RPM\]|\[Package AMD64 RPM Fedora\]|\[Package AMD64 RPM\])/ - - # OpenSuSE -- name: "Package OpenSuSE 15.1" - if: type != cron AND branch = master AND commit_message =~ /(\[Package arm64 RPM openSuSE\]|\[Package arm64 RPM\]|\[Package AMD64 RPM openSuSE\]|\[Package AMD64 RPM\])/ -- name: "Package OpenSuSE 15.0" - if: type != cron AND branch = master AND commit_message =~ /(\[Package arm64 RPM openSuSE\]|\[Package arm64 RPM\]|\[Package AMD64 RPM openSuSE\]|\[Package AMD64 RPM\])/ - - - -# DEB and RPM template flows -- stage: &_RPM_TEMPLATE - name: "Build & Publish RPM package" - before_install: - - sudo apt-get install -y wget lxc lxc-templates - - source tests/installer/slack.sh - before_script: - - post_message "TRAVIS_MESSAGE" "Starting package preparation and publishing for ${BUILD_STRING}.${BUILD_ARCH}" "${NOTIF_CHANNEL}" - - export PACKAGES_DIRECTORY="$(mktemp -d -t netdata-packaging-contents-dir-XXXXXX)" && echo "Created packaging directory ${PACKAGES_DIRECTORY}" - script: - - echo "Creating LXC environment for the build" && sudo -E .travis/package_management/create_lxc_for_build.sh - - echo "Building package in container" && sudo -E .travis/package_management/build_package_in_container.sh - - sudo chmod -R 755 "/var/lib/lxc" - - echo "Preparing RPM packaging contents for upload" && sudo -E .travis/package_management/prepare_packages.sh - after_failure: post_message "TRAVIS_MESSAGE" "Failed to build RPM for ${BUILD_STRING}.${BUILD_ARCH}" - - before_deploy: - - .travis/package_management/yank_stale_rpm.sh "${PACKAGES_DIRECTORY}" "${BUILD_STRING}" || echo "No stale RPM found" - deploy: - # Beta packages deployment - - provider: packagecloud - repository: "${DEPLOY_REPO}" - username: "${PACKAGING_USER}" - token: "${PKG_CLOUD_TOKEN}" - dist: "${BUILD_STRING}" - local_dir: "${PACKAGES_DIRECTORY}" - skip_cleanup: true - on: - # Only deploy on ${USER}/netdata, master branch, when packages directory is created - repo: ${TRAVIS_REPO_SLUG} - branch: master - condition: -d "${PACKAGES_DIRECTORY}" - # Production release packages deployment - - provider: packagecloud - repository: "netdata" - username: "netdata" - token: "${PKG_CLOUD_TOKEN}" - dist: "${BUILD_STRING}" - local_dir: "${PACKAGES_DIRECTORY}" - skip_cleanup: true - on: - # Only deploy on ${USER}/netdata, master branch, when packages directory is created - repo: "netdata/netdata" - branch: "master" - condition: -d "${PACKAGES_DIRECTORY}" - after_deploy: - - if [ -n "${BUILDER_NAME}" ]; then rm -rf /home/${BUILDER_NAME}/* && echo "Cleared /home/${BUILDER_NAME} directory" || echo "Failed to clean /home/${BUILDER_NAME} directory"; fi; - - if [ -d "${PACKAGES_DIRECTORY}" ]; then rm -rf "${PACKAGES_DIRECTORY}"; fi; - - - # TODO: This section is stale, will be aligned with the RPM implementation when we get to DEB packaging -- stage: &_DEB_TEMPLATE - name: "Build & Publish DEB package" - before_install: - - sudo apt-get install -y wget lxc lxc-templates - - source tests/installer/slack.sh - before_script: - - post_message "TRAVIS_MESSAGE" "Starting package preparation and publishing for ${BUILD_STRING}.${BUILD_ARCH}" "${NOTIF_CHANNEL}" - - export PACKAGES_DIRECTORY="$(mktemp -d -t netdata-packaging-contents-dir-XXXXXX)" && echo "Created packaging directory ${PACKAGES_DIRECTORY}" - script: - - echo "Creating LXC environment for the build" && sudo -E .travis/package_management/create_lxc_for_build.sh - - echo "Building package in container" && sudo -E .travis/package_management/build_package_in_container.sh - - sudo chmod -R 755 "/var/lib/lxc" - - echo "Preparing DEB packaging contents for upload" && sudo -E .travis/package_management/prepare_packages.sh - after_failure: post_message "TRAVIS_MESSAGE" "Failed to build DEB for ${BUILD_STRING}.${BUILD_ARCH}" - before_deploy: - - .travis/package_management/yank_stale_rpm.sh "${PACKAGES_DIRECTORY}" "${BUILD_STRING}" || echo "No stale DEB found" - deploy: - # Beta packages deployment - - provider: packagecloud - repository: "${DEPLOY_REPO}" - username: "${PACKAGING_USER}" - token: "${PKG_CLOUD_TOKEN}" - dist: "${BUILD_STRING}" - local_dir: "${PACKAGES_DIRECTORY}" - skip_cleanup: true - on: - # Only deploy on ${USER}/netdata, master branch, when build-area directory is created - repo: ${TRAVIS_REPO_SLUG} - branch: master - condition: -d "${PACKAGES_DIRECTORY}" - # Production release packages deployment - - provider: packagecloud - repository: "netdata" - username: "netdata" - token: "${PKG_CLOUD_TOKEN}" - dist: "${BUILD_STRING}" - local_dir: "${PACKAGES_DIRECTORY}" - skip_cleanup: true - on: - # Only deploy on ${USER}/netdata, master branch, when build-area directory is created - repo: "netdata/netdata" - branch: master - condition: -d "${PACKAGES_DIRECTORY}" - after_deploy: - - if [ -n "${BUILDER_NAME}" ]; then rm -rf /home/${BUILDER_NAME}/* && echo "Cleared /home/${BUILDER_NAME} directory" || echo "Failed to clean /home/${BUILDER_NAME} directory"; fi; - - if [ -d "${PACKAGES_DIRECTORY}" ]; then rm -rf "${PACKAGES_DIRECTORY}"; fi; + # Mandatory runs, we always want these executed + - name: Code quality, linting, syntax, code style + - name: Build process + - name: Artifacts validation + - name: Artifacts validation on bare OS, stable to current lifecycle checks + if: branch = master AND (type = pull_request OR type = cron) + + # Nightly operations + - name: Nightly operations + if: branch = master AND type = cron + - name: Nightly release + if: branch = master AND type = cron + + # Scheduled releases + - name: Packaging for release + if: branch = master AND type != pull_request AND type != cron + + - name: Publish for release + if: branch = master AND type != pull_request AND type != cron AND commit_message =~ /(\[netdata release candidate\]|\[netdata major release\]|\[netdata minor release\]|\[netdata patch release\])/ + + # Build DEB packages under special conditions + # Ubuntu + - name: "Package ubuntu/disco" + if: type != cron AND type != pull_request AND branch = master AND commit_message =~ /(\[Package arm64 DEB Ubuntu\]|\[Package arm64 DEB\]|\[Package i386 DEB Ubuntu\]|\[Package i386 DEB\]|\[Package amd64 DEB Ubuntu\]|\[Package amd64 DEB\])/ + - name: "Package ubuntu/cosmic" + if: type != cron AND type != pull_request AND branch = master AND commit_message =~ /(\[Package arm64 DEB Ubuntu\]|\[Package arm64 DEB\]|\[Package i386 DEB Ubuntu\]|\[Package i386 DEB\]|\[Package amd64 DEB Ubuntu\]|\[Package amd64 DEB\])/ + - name: "Package ubuntu/bionic" + if: type != cron AND type != pull_request AND branch = master AND commit_message =~ /(\[Package arm64 DEB Ubuntu\]|\[Package arm64 DEB\]|\[Package i386 DEB Ubuntu\]|\[Package i386 DEB\]|\[Package amd64 DEB Ubuntu\]|\[Package amd64 DEB\])/ + + # Debian + - name: "Package debian/buster" + if: type != cron AND type != pull_request AND branch = master AND commit_message =~ /(\[Package arm64 DEB Debian\]|\[Package arm64 DEB\]|\[Package i386 DEB Debian\]|\[Package i386 DEB\]|\[Package amd64 DEB Debian\]|\[Package amd64 DEB\])/ + - name: "Package debian/stretch" + if: type != cron AND type != pull_request AND branch = master AND commit_message =~ /(\[Package arm64 DEB Debian\]|\[Package arm64 DEB\]|\[Package i386 DEB Debian\]|\[Package i386 DEB\]|\[Package amd64 DEB Debian\]|\[Package amd64 DEB\])/ + - name: "Package debian/jessie" + if: type != cron AND type != pull_request AND branch = master AND commit_message =~ /(\[Package arm64 DEB Debian\]|\[Package arm64 DEB\]|\[Package i386 DEB Debian\]|\[Package i386 DEB\]|\[Package amd64 DEB Debian\]|\[Package amd64 DEB\])/ + + # Build RPM packages under special conditions + # Enterprise linux (Covers CentOS, Redhat, Amazon linux) + - name: "Package Enterprise Linux 7" + if: type != cron AND type != pull_request AND branch = master AND commit_message =~ /(\[Package arm64 RPM Enterprise Linux\]|\[Package arm64 RPM\]|\[Package i386 RPM Enterprise Linux\]|\[Package i386 RPM\]|\[Package amd64 RPM Enterprise Linux\]|\[Package amd64 RPM\])/ + - name: "Package Enterprise linux 6" + if: type != cron AND type != pull_request AND branch = master AND commit_message =~ /(\[Package i386 RPM Enterprise Linux\]|\[Package i386 RPM\]|\[Package amd64 RPM Enterprise Linux\]|\[Package amd64 RPM\])/ + + # Fedora + - name: "Package Fedora 30" + if: type != cron AND type != pull_request AND branch = master AND commit_message =~ /(\[Package arm64 RPM Fedora\]|\[Package arm64 RPM\]|\[Package amd64 RPM Fedora\]|\[Package amd64 RPM\])/ + - name: "Package Fedora 29" + if: type != cron AND type != pull_request AND branch = master AND commit_message =~ /(\[Package arm64 RPM Fedora\]|\[Package arm64 RPM\]|\[Package amd64 RPM Fedora\]|\[Package amd64 RPM\])/ + - name: "Package Fedora 28" + if: type != cron AND type != pull_request AND branch = master AND commit_message =~ /(\[Package arm64 RPM Fedora\]|\[Package arm64 RPM\]|\[Package amd64 RPM Fedora\]|\[Package amd64 RPM\])/ + + # OpenSuSE + - name: "Package OpenSuSE 15.1" + if: type != cron AND type != pull_request AND branch = master AND commit_message =~ /(\[Package arm64 RPM openSuSE\]|\[Package arm64 RPM\]|\[Package amd64 RPM openSuSE\]|\[Package amd64 RPM\])/ + - name: "Package OpenSuSE 15.0" + if: type != cron AND type != pull_request AND branch = master AND commit_message =~ /(\[Package arm64 RPM openSuSE\]|\[Package arm64 RPM\]|\[Package amd64 RPM openSuSE\]|\[Package amd64 RPM\])/ + + + + # DEB and RPM template flows + - stage: &_RPM_TEMPLATE + name: "Build & Publish RPM package" + before_install: + - sudo apt-get install -y wget lxc lxc-templates + - source tests/installer/slack.sh + before_script: + - post_message "TRAVIS_MESSAGE" "Starting package preparation and publishing for ${BUILD_STRING}.${BUILD_ARCH}" "${NOTIF_CHANNEL}" + - export PACKAGES_DIRECTORY="$(mktemp -d -t netdata-packaging-contents-dir-XXXXXX)" && echo "Created packaging directory ${PACKAGES_DIRECTORY}" + script: + - echo "GIT Branch:" && git branch + - echo "Last commit:" && git log -1 + - echo "GIT Describe:" && git describe + - echo "packaging/version:" && cat packaging/version + - echo "Creating LXC environment for the build" && sudo -E .travis/package_management/create_lxc_for_build.sh + - echo "Building package in container" && sudo -E .travis/package_management/build_package_in_container.sh + - sudo chmod -R 755 "/var/lib/lxc" + - echo "Preparing RPM packaging contents for upload" && sudo -E .travis/package_management/prepare_packages.sh + git: + depth: false + after_failure: post_message "TRAVIS_MESSAGE" "Failed to build RPM for ${BUILD_STRING}.${BUILD_ARCH}" + before_deploy: + - .travis/package_management/yank_stale_rpm.sh "${PACKAGES_DIRECTORY}" "${BUILD_STRING}" || echo "No stale RPM found" + deploy: + - provider: packagecloud + repository: "${DEPLOY_REPO}" + username: "${PACKAGING_USER}" + token: "${PKG_CLOUD_TOKEN}" + dist: "${BUILD_STRING}" + local_dir: "${PACKAGES_DIRECTORY}" + skip_cleanup: true + on: + # Only deploy on ${USER}/netdata, master branch, when packages directory is created + repo: ${TRAVIS_REPO_SLUG} + branch: "master" + condition: -d "${PACKAGES_DIRECTORY}" + after_deploy: + - if [ -n "${BUILDER_NAME}" ]; then rm -rf /home/${BUILDER_NAME}/* && echo "Cleared /home/${BUILDER_NAME} directory" || echo "Failed to clean /home/${BUILDER_NAME} directory"; fi; + - if [ -d "${PACKAGES_DIRECTORY}" ]; then rm -rf "${PACKAGES_DIRECTORY}"; fi; + + + + # TODO: This section is stale, will be aligned with the RPM implementation when we get to DEB packaging + - stage: &_DEB_TEMPLATE + name: "Build & Publish DEB package" + before_install: + - sudo apt-get install -y wget lxc lxc-templates dh-make git-buildpackage build-essential libdistro-info-perl + - source tests/installer/slack.sh + before_script: + - post_message "TRAVIS_MESSAGE" "Starting package preparation and publishing for ${BUILD_STRING}.${BUILD_ARCH}" "${NOTIF_CHANNEL}" + - export PACKAGES_DIRECTORY="$(mktemp -d -t netdata-packaging-contents-dir-XXXXXX)" && echo "Created packaging directory ${PACKAGES_DIRECTORY}" + script: + - echo "GIT Branch:" && git branch + - echo "Last commit:" && git log -1 + - echo "GIT Describe:" && git describe + - echo "packaging/version:" && cat packaging/version + - echo "Creating LXC environment for the build" && sudo -E .travis/package_management/create_lxc_for_build.sh + - echo "Building package in container" && sudo -E .travis/package_management/build_package_in_container.sh + - sudo chown -R root:travis "/var/lib/lxc" + - sudo chmod -R 750 "/var/lib/lxc" + - echo "Preparing DEB packaging contents for upload" && sudo -E .travis/package_management/prepare_packages.sh + git: + depth: false + after_failure: post_message "TRAVIS_MESSAGE" "Failed to build DEB for ${BUILD_STRING}.${BUILD_ARCH}" + before_deploy: + - .travis/package_management/yank_stale_rpm.sh "${PACKAGES_DIRECTORY}" "${BUILD_STRING}" || echo "No stale DEB found" + deploy: + - provider: packagecloud + repository: "${DEPLOY_REPO}" + username: "${PACKAGING_USER}" + token: "${PKG_CLOUD_TOKEN}" + dist: "${BUILD_STRING}" + local_dir: "${PACKAGES_DIRECTORY}" + skip_cleanup: true + on: + # Only deploy on ${USER}/netdata, master branch, when build-area directory is created + repo: ${TRAVIS_REPO_SLUG} + branch: "master" + condition: -d "${PACKAGES_DIRECTORY}" + after_deploy: + - if [ -n "${BUILDER_NAME}" ]; then rm -rf /home/${BUILDER_NAME}/* && echo "Cleared /home/${BUILDER_NAME} directory" || echo "Failed to clean /home/${BUILDER_NAME} directory"; fi; + - if [ -d "${PACKAGES_DIRECTORY}" ]; then rm -rf "${PACKAGES_DIRECTORY}"; fi; @@ -209,464 +192,447 @@ stages: # jobs: include: - # Do code quality, syntax checking and other pre-build activities - - stage: Code quality, linting, syntax, code style + # Do code quality, syntax checking and other pre-build activities + - stage: Code quality, linting, syntax, code style - name: Run shellchecking on BASH - script: shellcheck --format=gcc $(find . -name '*.sh.in' -not -iwholename '*.git*') + name: Run shellchecking on BASH + script: shellcheck --format=gcc $(find . -name '*.sh.in' -not -iwholename '*.git*') - # This falls under same stage defined earlier - - name: Run checksum checks on kickstart files - script: ./tests/installer/checksums.sh - env: LOCAL_ONLY="true" + # This falls under same stage defined earlier + - name: Run checksum checks on kickstart files + script: ./tests/installer/checksums.sh + env: LOCAL_ONLY="true" - # This falls under same stage defined earlier - - name: Web Dashboard pre-generated file consistency checks (dashboard.js) - script: cp web/gui/dashboard.js /tmp/dashboard.js && ./build/build.sh && diff /tmp/dashboard.js web/gui/dashboard.js + # This falls under same stage defined earlier + - name: Web Dashboard pre-generated file consistency checks (dashboard.js) + script: cp web/gui/dashboard.js /tmp/dashboard.js && ./build/build.sh && diff /tmp/dashboard.js web/gui/dashboard.js - # Ensure netdata code builds successfully - - stage: Build process + # Ensure netdata code builds successfully + - stage: Build process - name: Standard netdata build - script: fakeroot ./netdata-installer.sh --install $HOME --dont-wait --dont-start-it --enable-plugin-nfacct --enable-plugin-freeipmi --disable-lto - env: CFLAGS='-O1 -DNETDATA_INTERNAL_CHECKS=1 -DNETDATA_VERIFY_LOCKS=1' - after_failure: post_message "TRAVIS_MESSAGE" "<!here> standard netdata build is failing (Still dont know which one, will improve soon)" + name: Standard netdata build + script: fakeroot ./netdata-installer.sh --install $HOME --dont-wait --dont-start-it --enable-plugin-nfacct --enable-plugin-freeipmi --disable-lto + env: CFLAGS='-O1 -DNETDATA_INTERNAL_CHECKS=1 -DNETDATA_VERIFY_LOCKS=1' + after_failure: post_message "TRAVIS_MESSAGE" "<!here> standard netdata build is failing (Still dont know which one, will improve soon)" - - name: Docker container build process (alpine installation) - script: packaging/docker/build.sh - env: DEVEL="true" - after_failure: post_message "TRAVIS_MESSAGE" "Docker build process failed" + - name: Docker container build process (alpine installation) + script: packaging/docker/build.sh + env: DEVEL="true" + after_failure: post_message "TRAVIS_MESSAGE" "Docker build process failed" - - name: Run 'make dist' validation - before_script: mkdir /tmp/netdata-makedist-test - script: - - echo "GIT Branch:" && git branch - - echo "Last commit:" && git log -1 - - echo "GIT Describe:" && git describe - - echo "packaging/version:" && cat packaging/version - - docker run -it -v "${PWD}:/netdata:rw" -v "/tmp/netdata-makedist-test:/netdata_install:rw" -w /netdata "netdata/os-test:ubuntu1804" make clean || echo "Nothing to clean" - - docker run -it -v "${PWD}:/netdata:rw" -v "/tmp/netdata-makedist-test:/netdata_install:rw" -w /netdata "netdata/os-test:ubuntu1804" make distclean || echo "Nothing to distclean" - - docker run -it -v "${PWD}:/netdata:rw" -v "/tmp/netdata-makedist-test:/netdata_install:rw" -w /netdata "netdata/os-test:ubuntu1804" /bin/bash -c "autoreconf -ivf && ./configure --prefix=/netdata_install/usr --sysconfdir=/netdata_install/etc --localstatedir=/netdata_install/var --with-zlib --with-math --with-user=netdata CFLAGS=-O2" - - docker run -it -v "${PWD}:/netdata:rw" -v "/tmp/netdata-makedist-test:/netdata_install:rw" -w /netdata "netdata/os-test:ubuntu1804" make dist - - docker run -it -v "${PWD}:/netdata:rw" -v "/tmp/netdata-makedist-test:/netdata_install:rw" -w /netdata "netdata/os-test:ubuntu1804" ls -ltr ./netdata-$(git describe).tar.gz || ls -ltr ./netdata-$(cat packaging/version | tr -d '\n').tar.gz - - .travis/run_install_with_dist_file.sh - - docker run -it -v "${PWD}:/netdata:rw" -v "/tmp/netdata-makedist-test:/netdata_install:rw" -w /netdata "netdata/os-test:ubuntu1804" make distclean - git: - depth: false - after_script: rm -rf /tmp/netdata-makedist-test - after_failure: post_message "TRAVIS_MESSAGE" "'make dist' failed" + - name: Run 'make dist' validation + before_script: mkdir /tmp/netdata-makedist-test + script: + - echo "GIT Branch:" && git branch + - echo "Last commit:" && git log -1 + - echo "GIT Describe:" && git describe + - echo "packaging/version:" && cat packaging/version + - docker run -it -v "${PWD}:/netdata:rw" -v "/tmp/netdata-makedist-test:/netdata_install:rw" -w /netdata "netdata/os-test:ubuntu1804" make clean || echo "Nothing to clean" + - docker run -it -v "${PWD}:/netdata:rw" -v "/tmp/netdata-makedist-test:/netdata_install:rw" -w /netdata "netdata/os-test:ubuntu1804" make distclean || echo "Nothing to distclean" + - docker run -it -v "${PWD}:/netdata:rw" -v "/tmp/netdata-makedist-test:/netdata_install:rw" -w /netdata "netdata/os-test:ubuntu1804" /bin/bash -c "autoreconf -ivf && ./configure --prefix=/netdata_install/usr --sysconfdir=/netdata_install/etc --localstatedir=/netdata_install/var --with-zlib --with-math --with-user=netdata CFLAGS=-O2" + - docker run -it -v "${PWD}:/netdata:rw" -v "/tmp/netdata-makedist-test:/netdata_install:rw" -w /netdata "netdata/os-test:ubuntu1804" make dist + - docker run -it -v "${PWD}:/netdata:rw" -v "/tmp/netdata-makedist-test:/netdata_install:rw" -w /netdata "netdata/os-test:ubuntu1804" ls -ltr ./netdata-$(git describe).tar.gz || ls -ltr ./netdata-$(cat packaging/version | tr -d '\n').tar.gz + - .travis/run_install_with_dist_file.sh + - docker run -it -v "${PWD}:/netdata:rw" -v "/tmp/netdata-makedist-test:/netdata_install:rw" -w /netdata "netdata/os-test:ubuntu1804" make distclean + git: + depth: false + after_script: rm -rf /tmp/netdata-makedist-test + after_failure: post_message "TRAVIS_MESSAGE" "'make dist' failed" - - stage: Artifacts validation + - stage: Artifacts validation - name: Unit Testing - script: - - fakeroot ./netdata-installer.sh --install $HOME --dont-wait --dont-start-it --enable-plugin-nfacct --enable-plugin-freeipmi --disable-lto - - $HOME/netdata/usr/sbin/netdata -W unittest - env: CFLAGS='-O1 -DNETDATA_INTERNAL_CHECKS=1 -DNETDATA_VERIFY_LOCKS=1' - after_failure: post_message "TRAVIS_MESSAGE" "Unit testing failed" - - - name: Build/install on ubuntu 14.04 (not containerized) - script: fakeroot ./netdata-installer.sh --dont-wait --dont-start-it --install $HOME - after_failure: post_message "TRAVIS_MESSAGE" "Build/Install failed on ubuntu 14.04" + name: Unit Testing + script: + - fakeroot ./netdata-installer.sh --install $HOME --dont-wait --dont-start-it --enable-plugin-nfacct --enable-plugin-freeipmi --disable-lto + - $HOME/netdata/usr/sbin/netdata -W unittest + env: CFLAGS='-O1 -DNETDATA_INTERNAL_CHECKS=1 -DNETDATA_VERIFY_LOCKS=1' + after_failure: post_message "TRAVIS_MESSAGE" "Unit testing failed" + + - name: Build/install on ubuntu 14.04 (not containerized) + script: fakeroot ./netdata-installer.sh --dont-wait --dont-start-it --install $HOME + after_failure: post_message "TRAVIS_MESSAGE" "Build/Install failed on ubuntu 14.04" - - name: Build/Install for ubuntu 18.04 (not containerized) - script: fakeroot ./netdata-installer.sh --dont-wait --dont-start-it --install $HOME - after_failure: post_message "TRAVIS_MESSAGE" "Build/Install failed on ubuntu 18.04" + - name: Build/Install for ubuntu 18.04 (not containerized) + script: fakeroot ./netdata-installer.sh --dont-wait --dont-start-it --install $HOME + after_failure: post_message "TRAVIS_MESSAGE" "Build/Install failed on ubuntu 18.04" - - name: Run netdata lifecycle, on ubuntu 18.04 (Containerized) - script: docker run -it -v "${PWD}:/netdata:rw" -w /netdata "netdata/os-test:ubuntu1804" bats --tap tests/lifecycle.bats - after_failure: post_message "TRAVIS_MESSAGE" "Netdata lifecycle test script failed on ubuntu 18.04" + - name: Run netdata lifecycle, on ubuntu 18.04 (Containerized) + script: docker run -it -v "${PWD}:/netdata:rw" -w /netdata "netdata/os-test:ubuntu1804" bats --tap tests/lifecycle.bats + after_failure: post_message "TRAVIS_MESSAGE" "Netdata lifecycle test script failed on ubuntu 18.04" - - name: Run netdata lifecycle from stable to current, on CentOS 7 (Containerized) - script: docker run -it -v "${PWD}:/netdata:rw" -w /netdata "netdata/os-test:centos7" tests/updater_checks.sh - after_failure: post_message "TRAVIS_MESSAGE" "Netdata updater process failed on CentOS 7" - - - name: Build/install for CentOS 6 (Containerized) - script: docker run -it -v "${PWD}:/netdata:rw" -w /netdata "netdata/os-test:centos6" ./netdata-installer.sh --dont-wait --dont-start-it --install /tmp - after_failure: post_message "TRAVIS_MESSAGE" "Build/Install failed on CentOS 6" + - name: Run netdata lifecycle from stable to current, on CentOS 7 (Containerized) + script: docker run -it -v "${PWD}:/netdata:rw" -w /netdata "netdata/os-test:centos7" tests/updater_checks.sh + after_failure: post_message "TRAVIS_MESSAGE" "Netdata updater process failed on CentOS 7" + + - name: Build/install for CentOS 6 (Containerized) + script: docker run -it -v "${PWD}:/netdata:rw" -w /netdata "netdata/os-test:centos6" ./netdata-installer.sh --dont-wait --dont-start-it --install /tmp + after_failure: post_message "TRAVIS_MESSAGE" "Build/Install failed on CentOS 6" - - name: Build/install for CentOS 7 (Containerized) - script: docker run -it -v "${PWD}:/netdata:rw" -w /netdata "netdata/os-test:centos7" ./netdata-installer.sh --dont-wait --dont-start-it --install /tmp - after_failure: post_message "TRAVIS_MESSAGE" "Build/Install failed on CentOS 7" + - name: Build/install for CentOS 7 (Containerized) + script: docker run -it -v "${PWD}:/netdata:rw" -w /netdata "netdata/os-test:centos7" ./netdata-installer.sh --dont-wait --dont-start-it --install /tmp + after_failure: post_message "TRAVIS_MESSAGE" "Build/Install failed on CentOS 7" - - stage: "Artifacts validation on bare OS, stable to current lifecycle checks" - - # Ubuntu runs - name: Run netdata lifecycle on Ubuntu 16.04 (xenial) - script: sudo -E tests/updater_checks.sh - after_failure: post_message "TRAVIS_MESSAGE" "Netdata updater process failed on bare Ubuntu 16.04" + - stage: "Artifacts validation on bare OS, stable to current lifecycle checks" + + # Ubuntu runs + name: Run netdata lifecycle on Ubuntu 16.04 (xenial) + script: sudo -E tests/updater_checks.sh + after_failure: post_message "TRAVIS_MESSAGE" "Netdata updater process failed on bare Ubuntu 16.04" - - name: Run netdata lifecycle, on Ubuntu 19.04 (Containerized) - script: docker run -it -v "${PWD}:/netdata:rw" -w /netdata "ubuntu:19.04" tests/updater_checks.sh - after_failure: post_message "TRAVIS_MESSAGE" "Netdata updater process failed on bare Ubuntu 19.04" + - name: Run netdata lifecycle, on Ubuntu 19.04 (Containerized) + script: docker run -it -v "${PWD}:/netdata:rw" -w /netdata "ubuntu:19.04" tests/updater_checks.sh + after_failure: post_message "TRAVIS_MESSAGE" "Netdata updater process failed on bare Ubuntu 19.04" - # Centos runs - - name: Run netdata lifecycle on CentOS 7 (Containerized) - script: docker run -it -v "${PWD}:/netdata:rw" -w /netdata "centos:7" tests/updater_checks.sh - after_failure: post_message "TRAVIS_MESSAGE" "Netdata updater process failed on bare CentOS 7" + # Centos runs + - name: Run netdata lifecycle on CentOS 7 (Containerized) + script: docker run -it -v "${PWD}:/netdata:rw" -w /netdata "centos:7" tests/updater_checks.sh + after_failure: post_message "TRAVIS_MESSAGE" "Netdata updater process failed on bare CentOS 7" - # Debian runs - - name: Run netdata lifecycle, on Debian 9 (Containerized) - script: docker run -it -v "${PWD}:/netdata:rw" -w /netdata "debian:stretch" tests/updater_checks.sh - after_failure: post_message "TRAVIS_MESSAGE" "Netdata updater process failed on bare Debian 9 (stretch)" + # Debian runs + - name: Run netdata lifecycle, on Debian 9 (Containerized) + script: docker run -it -v "${PWD}:/netdata:rw" -w /netdata "debian:stretch" tests/updater_checks.sh + after_failure: post_message "TRAVIS_MESSAGE" "Netdata updater process failed on bare Debian 9 (stretch)" - # openSuSE runs - - name: Run netdata lifecycle, on openSuSE 15.0 - script: docker run -it -v "${PWD}:/netdata:rw" -w /netdata "opensuse/leap:15.0" tests/updater_checks.sh - after_failure: post_message "TRAVIS_MESSAGE" "Netdata updater process failed on bare opensuse/leap:15.0" + # openSuSE runs + - name: Run netdata lifecycle, on openSuSE 15.0 + script: docker run -it -v "${PWD}:/netdata:rw" -w /netdata "opensuse/leap:15.0" tests/updater_checks.sh + after_failure: post_message "TRAVIS_MESSAGE" "Netdata updater process failed on bare opensuse/leap:15.0" - - name: Run netdata lifecycle, on openSuSE 15.1 - script: docker run -it -v "${PWD}:/netdata:rw" -w /netdata "opensuse/leap:15.1" tests/updater_checks.sh - after_failure: post_message "TRAVIS_MESSAGE" "Netdata updater process failed on bare opensuse/leap:15.1" + - name: Run netdata lifecycle, on openSuSE 15.1 + script: docker run -it -v "${PWD}:/netdata:rw" -w /netdata "opensuse/leap:15.1" tests/updater_checks.sh + after_failure: post_message "TRAVIS_MESSAGE" "Netdata updater process failed on bare opensuse/leap:15.1" - - name: Run netdata lifecycle, on openSuSE Tumbleweed - script: docker run -it -v "${PWD}:/netdata:rw" -w /netdata "opensuse/tumbleweed:latest" tests/updater_checks.sh - after_failure: post_message "TRAVIS_MESSAGE" "Netdata updater process failed on bare opensuse/tumbleweed:latest" + - name: Run netdata lifecycle, on openSuSE Tumbleweed + script: docker run -it -v "${PWD}:/netdata:rw" -w /netdata "opensuse/tumbleweed:latest" tests/updater_checks.sh + after_failure: post_message "TRAVIS_MESSAGE" "Netdata updater process failed on bare opensuse/tumbleweed:latest" - # Alpine runs - - name: Run netdata lifecycle, on Alpine linux - script: docker run -it -v "${PWD}:/netdata:rw" -w /netdata "alpine" tests/updater_checks.sh - after_failure: post_message "TRAVIS_MESSAGE" "Netdata updater process failed on bare Alpine" + # Alpine runs + - name: Run netdata lifecycle, on Alpine linux + script: docker run -it -v "${PWD}:/netdata:rw" -w /netdata "alpine" tests/updater_checks.sh + after_failure: post_message "TRAVIS_MESSAGE" "Netdata updater process failed on bare Alpine" - # Arch linux runs - - name: Run netdata lifecycle, on ArchLinux - script: docker run -it -v "${PWD}:/netdata:rw" -w /netdata "archlinux/base:latest" tests/updater_checks.sh - after_failure: post_message "TRAVIS_MESSAGE" "Netdata updater process failed on bare archlinux/base:latest" + # Arch linux runs + - name: Run netdata lifecycle, on ArchLinux + script: docker run -it -v "${PWD}:/netdata:rw" -w /netdata "archlinux/base:latest" tests/updater_checks.sh + after_failure: post_message "TRAVIS_MESSAGE" "Netdata updater process failed on bare archlinux/base:latest" - # Fedora runs - - name: Run netdata lifecycle, on Fedora 28 - script: docker run -it -v "${PWD}:/netdata:rw" -w /netdata "fedora:28" tests/updater_checks.sh - after_failure: post_message "TRAVIS_MESSAGE" "Netdata updater process failed on bare Fedora 28" + # Fedora runs + - name: Run netdata lifecycle, on Fedora 28 + script: docker run -it -v "${PWD}:/netdata:rw" -w /netdata "fedora:28" tests/updater_checks.sh + after_failure: post_message "TRAVIS_MESSAGE" "Netdata updater process failed on bare Fedora 28" - - name: Run netdata lifecycle, on Fedora 29 - script: docker run -it -v "${PWD}:/netdata:rw" -w /netdata "fedora:29" tests/updater_checks.sh - after_failure: post_message "TRAVIS_MESSAGE" "Netdata updater process failed on bare Fedora 29" + - name: Run netdata lifecycle, on Fedora 29 + script: docker run -it -v "${PWD}:/netdata:rw" -w /netdata "fedora:29" tests/updater_checks.sh + after_failure: post_message "TRAVIS_MESSAGE" "Netdata updater process failed on bare Fedora 29" - - name: Run netdata lifecycle, on Fedora 30 (Containerized) - script: docker run -it -v "${PWD}:/netdata:rw" -w /netdata "fedora:30" tests/updater_checks.sh - after_failure: post_message "TRAVIS_MESSAGE" "Netdata updater process failed on bare Fedora 30" + - name: Run netdata lifecycle, on Fedora 30 (Containerized) + script: docker run -it -v "${PWD}:/netdata:rw" -w /netdata "fedora:30" tests/updater_checks.sh + after_failure: post_message "TRAVIS_MESSAGE" "Netdata updater process failed on bare Fedora 30" - - stage: Packaging for release + - stage: Packaging for release - name: Generate changelog and TAG the release (only on special commit msg) - before_script: post_message "TRAVIS_MESSAGE" "Packaging step for release initiated" "${NOTIF_CHANNEL}" - script: - - echo "GIT Branch:" && git branch - - echo "Last commit:" && git log -1 - - echo "GIT Describe:" && git describe - - echo "packaging/version:" && cat packaging/version - - .travis/generate_changelog_and_tag_release.sh - after_failure: post_message "TRAVIS_MESSAGE" "<!here> Packaging for release failed" - git: - depth: false + name: Generate changelog and TAG the release (only on special commit msg) + before_script: post_message "TRAVIS_MESSAGE" "Packaging step for release initiated" "${NOTIF_CHANNEL}" + script: + - echo "GIT Branch:" && git branch + - echo "Last commit:" && git log -1 + - echo "GIT Describe:" && git describe + - echo "packaging/version:" && cat packaging/version + - .travis/generate_changelog_and_tag_release.sh + after_failure: post_message "TRAVIS_MESSAGE" "<!here> Packaging for release failed" + git: + depth: false - - name: Run labeler on github issues - script: .travis/labeler.sh # labeler should be replaced with GitHub Actions when they hit GA + - name: Run labeler on github issues + script: .travis/labeler.sh # labeler should be replaced with GitHub Actions when they hit GA - # ###### Packaging workflow section ###### - # References: - # https://us.images.linuxcontainers.org - # https://packagecloud.io/docs#install_repo + # ###### Packaging workflow section ###### + # References: + # https://us.images.linuxcontainers.org + # https://packagecloud.io/docs#install_repo - # Ubuntu distros build - # - - stage: - <<: *_DEB_TEMPLATE - stage: "Package ubuntu/disco" - env: - - BUILDER_NAME="builder" BUILD_DISTRO="ubuntu" BUILD_RELEASE="disco" BUILD_STRING="ubuntu/disco" - - PACKAGE_TYPE="deb" REPO_TOOL="apt-get" - - ALLOW_SOFT_FAILURE_HERE=true + # Ubuntu distros build + # + - stage: + <<: *_DEB_TEMPLATE + stage: "Package ubuntu/disco" + env: + - BUILDER_NAME="builder" BUILD_DISTRO="ubuntu" BUILD_RELEASE="disco" BUILD_STRING="ubuntu/disco" + - PACKAGE_TYPE="deb" REPO_TOOL="apt-get" + - ALLOW_SOFT_FAILURE_HERE=true - - stage: - <<: *_DEB_TEMPLATE - stage: "Package ubuntu/cosmic" - env: - - BUILDER_NAME="builder" BUILD_DISTRO="ubuntu" BUILD_RELEASE="cosmic" BUILD_STRING="ubuntu/cosmic" - - PACKAGE_TYPE="deb" REPO_TOOL="apt-get" - - ALLOW_SOFT_FAILURE_HERE=true + - stage: + <<: *_DEB_TEMPLATE + stage: "Package ubuntu/cosmic" + env: + - BUILDER_NAME="builder" BUILD_DISTRO="ubuntu" BUILD_RELEASE="cosmic" BUILD_STRING="ubuntu/cosmic" + - PACKAGE_TYPE="deb" REPO_TOOL="apt-get" + - ALLOW_SOFT_FAILURE_HERE=true - - stage: - <<: *_DEB_TEMPLATE - stage: "Package ubuntu/bionic" - env: - - BUILDER_NAME="builder" BUILD_DISTRO="ubuntu" BUILD_RELEASE="bionic" BUILD_STRING="ubuntu/bionic" - - PACKAGE_TYPE="deb" REPO_TOOL="apt-get" - - ALLOW_SOFT_FAILURE_HERE=true + - stage: + <<: *_DEB_TEMPLATE + stage: "Package ubuntu/bionic" + env: + - BUILDER_NAME="builder" BUILD_DISTRO="ubuntu" BUILD_RELEASE="bionic" BUILD_STRING="ubuntu/bionic" + - PACKAGE_TYPE="deb" REPO_TOOL="apt-get" + - ALLOW_SOFT_FAILURE_HERE=true - - stage: - <<: *_DEB_TEMPLATE - stage: "Package ubuntu/artful" - env: - - BUILDER_NAME="builder" BUILD_DISTRO="ubuntu" BUILD_RELEASE="artful" BUILD_STRING="ubuntu/artful" - - PACKAGE_TYPE="deb" REPO_TOOL="apt-get" - - ALLOW_SOFT_FAILURE_HERE=true + # Debian distros build + - stage: + <<: *_DEB_TEMPLATE + stage: "Package debian/buster" + env: + - BUILDER_NAME="builder" BUILD_DISTRO="debian" BUILD_RELEASE="buster" BUILD_STRING="debian/buster" + - PACKAGE_TYPE="deb" REPO_TOOL="apt-get" + - ALLOW_SOFT_FAILURE_HERE=true - # Debian distros build - - stage: - <<: *_DEB_TEMPLATE - stage: "Package debian/buster" - env: - - BUILDER_NAME="builder" BUILD_DISTRO="debian" BUILD_RELEASE="buster" BUILD_STRING="debian/buster" - - PACKAGE_TYPE="deb" REPO_TOOL="apt-get" - - ALLOW_SOFT_FAILURE_HERE=true + - stage: + <<: *_DEB_TEMPLATE + stage: "Package debian/stretch" + env: + - BUILDER_NAME="builder" BUILD_DISTRO="debian" BUILD_RELEASE="stretch" BUILD_STRING="debian/stretch" + - PACKAGE_TYPE="deb" REPO_TOOL="apt-get" + - ALLOW_SOFT_FAILURE_HERE=true - - stage: - <<: *_DEB_TEMPLATE - stage: "Package debian/stretch" - env: - - BUILDER_NAME="builder" BUILD_DISTRO="debian" BUILD_RELEASE="stretch" BUILD_STRING="debian/stretch" - - PACKAGE_TYPE="deb" REPO_TOOL="apt-get" - - ALLOW_SOFT_FAILURE_HERE=true + - stage: + <<: *_DEB_TEMPLATE + stage: "Package debian/jessie" + env: + - BUILDER_NAME="builder" BUILD_DISTRO="debian" BUILD_RELEASE="jessie" BUILD_STRING="debian/jessie" + - PACKAGE_TYPE="deb" REPO_TOOL="apt-get" + - ALLOW_SOFT_FAILURE_HERE=true - - stage: - <<: *_DEB_TEMPLATE - stage: "Package debian/jessie" - env: - - BUILDER_NAME="builder" BUILD_DISTRO="debian" BUILD_RELEASE="jessie" BUILD_STRING="debian/jessie" - - PACKAGE_TYPE="deb" REPO_TOOL="apt-get" - - ALLOW_SOFT_FAILURE_HERE=true + # Enterprise linux builds (Centos, Redhat, Amazon linux (el/6)) + # + - stage: + <<: *_RPM_TEMPLATE + stage: "Package Enterprise Linux 7" + env: + - BUILDER_NAME="builder" BUILD_DISTRO="centos" BUILD_RELEASE="7" BUILD_STRING="el/7" + - PACKAGE_TYPE="rpm" REPO_TOOL="yum" + - ALLOW_SOFT_FAILURE_HERE=true - - stage: - <<: *_DEB_TEMPLATE - stage: "Package debian/wheezy" - env: - - BUILDER_NAME="builder" BUILD_DISTRO="debian" BUILD_RELEASE="wheezy" BUILD_STRING="debian/wheezy" - - PACKAGE_TYPE="deb" REPO_TOOL="apt-get" - - ALLOW_SOFT_FAILURE_HERE=true + - stage: + <<: *_RPM_TEMPLATE + stage: "Package Enterprise linux 6" + env: + - BUILDER_NAME="builder" BUILD_DISTRO="centos" BUILD_RELEASE="6" BUILD_STRING="el/6" + - PACKAGE_TYPE="rpm" REPO_TOOL="yum" + - ALLOW_SOFT_FAILURE_HERE=true + + # Fedora distros build + # + - stage: + <<: *_RPM_TEMPLATE + stage: "Package Fedora 30" + env: + - BUILDER_NAME="builder" BUILD_DISTRO="fedora" BUILD_RELEASE="30" BUILD_STRING="fedora/30" + - PACKAGE_TYPE="rpm" REPO_TOOL="dnf" + - ALLOW_SOFT_FAILURE_HERE=true - # Enterprise linux builds (Centos, Redhat, Amazon linux (el/6)) - # - - stage: - <<: *_RPM_TEMPLATE - stage: "Package Enterprise Linux 7" - env: - - BUILDER_NAME="builder" BUILD_DISTRO="centos" BUILD_RELEASE="7" BUILD_STRING="el/7" - - PACKAGE_TYPE="rpm" REPO_TOOL="yum" - - ALLOW_SOFT_FAILURE_HERE=true + + + - stage: + <<: *_RPM_TEMPLATE + stage: "Package Fedora 29" + env: + - BUILDER_NAME="builder" BUILD_DISTRO="fedora" BUILD_RELEASE="29" BUILD_STRING="fedora/29" + - PACKAGE_TYPE="rpm" REPO_TOOL="dnf" + - ALLOW_SOFT_FAILURE_HERE=true - - stage: - <<: *_RPM_TEMPLATE - stage: "Package Enterprise linux 6" - env: - - BUILDER_NAME="builder" BUILD_DISTRO="centos" BUILD_RELEASE="6" BUILD_STRING="el/6" - - PACKAGE_TYPE="rpm" REPO_TOOL="yum" - - ALLOW_SOFT_FAILURE_HERE=true - - - - # Fedora distros build - # - - stage: - <<: *_RPM_TEMPLATE - stage: "Package Fedora 30" - env: - - BUILDER_NAME="builder" BUILD_DISTRO="fedora" BUILD_RELEASE="30" BUILD_STRING="fedora/30" - - PACKAGE_TYPE="rpm" REPO_TOOL="dnf" - - ALLOW_SOFT_FAILURE_HERE=true - - - - - stage: - <<: *_RPM_TEMPLATE - stage: "Package Fedora 29" - env: - - BUILDER_NAME="builder" BUILD_DISTRO="fedora" BUILD_RELEASE="29" BUILD_STRING="fedora/29" - - PACKAGE_TYPE="rpm" REPO_TOOL="dnf" - - ALLOW_SOFT_FAILURE_HERE=true - - - - - stage: - <<: *_RPM_TEMPLATE - stage: "Package Fedora 28" - env: - - BUILDER_NAME="builder" BUILD_DISTRO="fedora" BUILD_RELEASE="28" BUILD_STRING="fedora/28" - - PACKAGE_TYPE="rpm" REPO_TOOL="dnf" - - ALLOW_SOFT_FAILURE_HERE=true - - - - # Opensuse distros build - # - - stage: - <<: *_RPM_TEMPLATE - stage: "Package OpenSuSE 15.1" - env: - - BUILDER_NAME="builder" BUILD_DISTRO="opensuse" BUILD_RELEASE="15.0" BUILD_STRING="opensuse/15.1" - - PACKAGE_TYPE="rpm" REPO_TOOL="zypper" - - ALLOW_SOFT_FAILURE_HERE=true - - - - - stage: - <<: *_RPM_TEMPLATE - stage: "Package OpenSuSE 15.0" - env: - - BUILDER_NAME="builder" BUILD_DISTRO="opensuse" BUILD_RELEASE="15.0" BUILD_STRING="opensuse/15.0" - - PACKAGE_TYPE="rpm" REPO_TOOL="zypper" - - ALLOW_SOFT_FAILURE_HERE=true - # ###### End of packaging workflow section ###### # - # ############################################### # - - - - # We only publish if a TAG has been set during packaging - - stage: Publish for release - - name: Build & Publish docker images - before_script: post_message "TRAVIS_MESSAGE" "Publishing docker images" "${NOTIF_CHANNEL}" - script: - - echo "GIT Branch:" && git branch - - echo "Last commit:" && git log -1 - - echo "GIT Describe:" && git describe - - echo "packaging/version:" && cat packaging/version - - packaging/docker/check_login.sh - - packaging/docker/build.sh - - packaging/docker/publish.sh - after_failure: post_message "TRAVIS_MESSAGE" "<!here> Docker image publishing failed" - git: - depth: false - env: ALLOW_SOFT_FAILURE_HERE=true - # We don't run on release candidates - if: tag !~ /(-rc)/ - - - name: Create release draft - before_script: post_message "TRAVIS_MESSAGE" "Drafting release on github" "${NOTIF_CHANNEL}" - script: - - echo "GIT Branch:" && git branch - - echo "Last commit:" && git log -1 - - echo "GIT Describe:" && git describe - - echo "packaging/version:" && cat packaging/version - - echo "Generating release artifacts" && .travis/create_artifacts.sh # Could/should be a common storage to put this and share between jobs - - .travis/draft_release.sh - git: - depth: false - after_failure: post_message "TRAVIS_MESSAGE" "<!here> Draft release submission failed" - # We don't run on release candidates - if: tag !~ /(-rc)/ - - - - # This is the nightly pre-execution step (Jobs, preparatory steps for nightly, etc) - - stage: Nightly operations - - name: Run coverity scan - # Just notify people that Nightly ops triggered, use the first step as a hook to do that - before_script: post_message "TRAVIS_MESSAGE" "Starting nightly operations" "${NOTIF_CHANNEL}" - script: ./coverity-install.sh && ./coverity-scan.sh || echo "Coverity failed :(" - - - name: Kickstart files integrity testing (extended) - script: ./tests/installer/checksums.sh - - - name: Run labeler on github issues - script: .travis/labeler.sh # labeler should be replaced with GitHub Actions when they hit GA - - # This is generating the changelog for nightly release and publish it - - name: Generate nightly changelog - before_script: post_message "TRAVIS_MESSAGE" "Starting changelog generation for nightlies" "${NOTIF_CHANNEL}" - script: ".travis/nightlies.sh" - after_failure: post_message "TRAVIS_MESSAGE" "<!here> Nightly changelog generation failed" - git: - depth: false - - - - # This is the nightly execution step - # - - stage: Nightly release - - name: Build & Publish docker images - before_script: post_message "TRAVIS_MESSAGE" "Publishing docker images for nightlies" "${NOTIF_CHANNEL}" - script: - - echo "GIT Branch:" && git branch - - echo "Last commit:" && git log -1 - - echo "GIT Describe:" && git describe - - echo "packaging/version:" && cat packaging/version - - docker info - - packaging/docker/check_login.sh - - packaging/docker/build.sh - - packaging/docker/publish.sh - after_failure: post_message "TRAVIS_MESSAGE" "<!here> Nightly docker image publish failed" - git: - depth: false - env: ALLOW_SOFT_FAILURE_HERE=true - - - name: Create nightly release artifacts, publish to GCS - before_script: post_message "TRAVIS_MESSAGE" "Starting artifacts generation for nightlies" "${NOTIF_CHANNEL}" - script: - - echo "GIT Branch:" && git branch - - echo "Last commit:" && git log -1 - - echo "GIT Describe:" && git describe - - echo "packaging/version:" && cat packaging/version - - .travis/create_artifacts.sh - after_failure: post_message "TRAVIS_MESSAGE" "<!here> Nightly artifacts generation failed" - git: - depth: false - before_deploy: - echo "Preparing creds under ${TRAVIS_REPO_SLUG}"; - if [ "${TRAVIS_REPO_SLUG}" == "netdata/netdata" ]; then - openssl aes-256-cbc -K $encrypted_8daf19481253_key -iv $encrypted_8daf19481253_iv -in .travis/gcs-credentials.json.enc -out .travis/gcs-credentials.json -d; - else - echo "Beta deployment stage in progress"; - openssl aes-256-cbc -K $encrypted_8daf19481253_key -iv $encrypted_8daf19481253_iv -in .travis/gcs-credentials.json.enc -out .travis/gcs-credentials.json -d; - fi; - deploy: - # Beta storage, used for testing purposes - - provider: gcs - edge: - branch: gcs-ng - project_id: netdata-storage - credentials: .travis/gcs-credentials.json - bucket: "netdata-dev-nightlies" - skip_cleanup: true - local_dir: "artifacts" - on: - # Only deploy on netdata/netdata, master branch, when artifacts directory is created - repo: ${TRAVIS_REPO_SLUG} - branch: master - condition: -d "artifacts" && ${TRAVIS_REPO_SLUG} != "netdata/netdata" - - # Production storage - - provider: gcs - edge: - branch: gcs-ng - project_id: netdata-storage - credentials: .travis/gcs-credentials.json - bucket: "netdata-nightlies" - skip_cleanup: true - local_dir: "artifacts" - on: - # Only deploy on netdata/netdata, master branch, when artifacts directory is created - repo: netdata/netdata - branch: master - condition: -d "artifacts" && ${TRAVIS_REPO_SLUG} = "netdata/netdata" - after_deploy: rm -f .travis/gcs-credentials.json + - stage: + <<: *_RPM_TEMPLATE + stage: "Package Fedora 28" + env: + - BUILDER_NAME="builder" BUILD_DISTRO="fedora" BUILD_RELEASE="28" BUILD_STRING="fedora/28" + - PACKAGE_TYPE="rpm" REPO_TOOL="dnf" + - ALLOW_SOFT_FAILURE_HERE=true + + + + # Opensuse distros build + # + - stage: + <<: *_RPM_TEMPLATE + stage: "Package OpenSuSE 15.1" + env: + - BUILDER_NAME="builder" BUILD_DISTRO="opensuse" BUILD_RELEASE="15.0" BUILD_STRING="opensuse/15.1" + - PACKAGE_TYPE="rpm" REPO_TOOL="zypper" + - ALLOW_SOFT_FAILURE_HERE=true + + + + - stage: + <<: *_RPM_TEMPLATE + stage: "Package OpenSuSE 15.0" + env: + - BUILDER_NAME="builder" BUILD_DISTRO="opensuse" BUILD_RELEASE="15.0" BUILD_STRING="opensuse/15.0" + - PACKAGE_TYPE="rpm" REPO_TOOL="zypper" + - ALLOW_SOFT_FAILURE_HERE=true + # ###### End of packaging workflow section ###### # + # ############################################### # + + + + # We only publish if a TAG has been set during packaging + - stage: Publish for release + + name: Build & Publish docker images + before_script: post_message "TRAVIS_MESSAGE" "Publishing docker images" "${NOTIF_CHANNEL}" + script: + - echo "GIT Branch:" && git branch + - echo "Last commit:" && git log -1 + - echo "GIT Describe:" && git describe + - echo "packaging/version:" && cat packaging/version + - packaging/docker/check_login.sh + - echo "Switching to latest master branch, to pick up tagging if any" && git checkout master && git pull + - packaging/docker/build.sh + - packaging/docker/publish.sh + after_failure: post_message "TRAVIS_MESSAGE" "<!here> Docker image publishing failed" + git: + depth: false + env: ALLOW_SOFT_FAILURE_HERE=true + # We don't run on release candidates + if: tag !~ /(-rc)/ + + - name: Create release draft + before_script: post_message "TRAVIS_MESSAGE" "Drafting release on github" "${NOTIF_CHANNEL}" + script: + - echo "GIT Branch:" && git branch + - echo "Last commit:" && git log -1 + - echo "GIT Describe:" && git describe + - echo "packaging/version:" && cat packaging/version + - echo "Generating release artifacts" && .travis/create_artifacts.sh # Could/should be a common storage to put this and share between jobs + - .travis/draft_release.sh + git: + depth: false + after_failure: post_message "TRAVIS_MESSAGE" "<!here> Draft release submission failed" + # We don't run on release candidates + if: tag !~ /(-rc)/ + + + + # This is the nightly pre-execution step (Jobs, preparatory steps for nightly, etc) + - stage: Nightly operations + + name: Run coverity scan + # Just notify people that Nightly ops triggered, use the first step as a hook to do that + before_script: post_message "TRAVIS_MESSAGE" "Starting nightly operations" "${NOTIF_CHANNEL}" + script: ./coverity-install.sh && ./coverity-scan.sh || echo "Coverity failed :(" + + - name: Kickstart files integrity testing (extended) + script: ./tests/installer/checksums.sh + + - name: Run labeler on github issues + script: .travis/labeler.sh # labeler should be replaced with GitHub Actions when they hit GA + + # This is generating the changelog for nightly release and publish it + - name: Generate nightly changelog + before_script: post_message "TRAVIS_MESSAGE" "Starting changelog generation for nightlies" "${NOTIF_CHANNEL}" + script: + - ".travis/nightlies.sh" + - ".travis/check_changelog_last_modification.sh" + after_failure: post_message "TRAVIS_MESSAGE" "<!here> Nightly changelog generation failed" + git: + depth: false + + + + # This is the nightly execution step + # + - stage: Nightly release + + name: Build & Publish docker images + before_script: post_message "TRAVIS_MESSAGE" "Publishing docker images for nightlies" "${NOTIF_CHANNEL}" + script: + - echo "GIT Branch:" && git branch + - echo "Last commit:" && git log -1 + - echo "GIT Describe:" && git describe + - echo "packaging/version:" && cat packaging/version + - docker info + - packaging/docker/check_login.sh + - packaging/docker/build.sh + - packaging/docker/publish.sh + after_failure: post_message "TRAVIS_MESSAGE" "<!here> Nightly docker image publish failed" + git: + depth: false + env: ALLOW_SOFT_FAILURE_HERE=true + + - name: Create nightly release artifacts, publish to GCS + before_script: post_message "TRAVIS_MESSAGE" "Starting artifacts generation for nightlies" "${NOTIF_CHANNEL}" + script: + - echo "GIT Branch:" && git branch + - echo "Last commit:" && git log -1 + - echo "GIT Describe:" && git describe + - echo "packaging/version:" && cat packaging/version + - .travis/create_artifacts.sh + after_failure: post_message "TRAVIS_MESSAGE" "<!here> Nightly artifacts generation failed" + git: + depth: false + before_deploy: + echo "Preparing creds under ${TRAVIS_REPO_SLUG}"; + if [ "${TRAVIS_REPO_SLUG}" == "netdata/netdata" ]; then + openssl aes-256-cbc -K $encrypted_8daf19481253_key -iv $encrypted_8daf19481253_iv -in .travis/gcs-credentials.json.enc -out .travis/gcs-credentials.json -d; + else + echo "Beta deployment stage in progress"; + openssl aes-256-cbc -K $encrypted_8daf19481253_key -iv $encrypted_8daf19481253_iv -in .travis/gcs-credentials.json.enc -out .travis/gcs-credentials.json -d; + fi; + deploy: + # Beta storage, used for testing purposes + - provider: gcs + edge: + branch: gcs-ng + project_id: netdata-storage + credentials: .travis/gcs-credentials.json + bucket: "netdata-dev-nightlies" + skip_cleanup: true + local_dir: "artifacts" + on: + # Only deploy on netdata/netdata, master branch, when artifacts directory is created + repo: ${TRAVIS_REPO_SLUG} + branch: master + condition: -d "artifacts" && ${TRAVIS_REPO_SLUG} != "netdata/netdata" + + # Production storage + - provider: gcs + edge: + branch: gcs-ng + project_id: netdata-storage + credentials: .travis/gcs-credentials.json + bucket: "netdata-nightlies" + skip_cleanup: true + local_dir: "artifacts" + on: + # Only deploy on netdata/netdata, master branch, when artifacts directory is created + repo: netdata/netdata + branch: master + condition: -d "artifacts" && ${TRAVIS_REPO_SLUG} = "netdata/netdata" + after_deploy: rm -f .travis/gcs-credentials.json diff --git a/.travis/README.md b/.travis/README.md index b7b61ecb4..3b314fa18 100644 --- a/.travis/README.md +++ b/.travis/README.md @@ -138,6 +138,6 @@ We plan to support amd64, x86 and arm64 architectures. As of June 2019 only amd6 The Package deployment can be triggered manually by executing an empty commit with the following message pattern: `[Package PACKAGE_TYPE PACKAGE_ARCH] DESCRIBE_THE_REASONING_HERE`. Travis Yaml configuration allows the user to combine package type and architecture as necessary to regenerate the current stable release (For example tag v1.15.0 as of 4th of May 2019) -Sample patterns to trigger building of packages for all AMD64 supported architecture: -- '[Package AMD64 RPM]': Build & publish all amd64 available RPM packages -- '[Package AMD64 DEB]': Build & publish all amd64 available DEB packages +Sample patterns to trigger building of packages for all amd64 supported architecture: +- '[Package amd64 RPM]': Build & publish all amd64 available RPM packages +- '[Package amd64 DEB]': Build & publish all amd64 available DEB packages diff --git a/.travis/check_changelog_last_modification.sh b/.travis/check_changelog_last_modification.sh new file mode 100755 index 000000000..2665c0627 --- /dev/null +++ b/.travis/check_changelog_last_modification.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash + +set -e + +LAST_MODIFICATION="$(git log -1 --pretty="format:%at" CHANGELOG.md)" +CURRENT_TIME="$(date +"%s")" +TWO_DAYS_IN_SECONDS=172800 + +DIFF=$((CURRENT_TIME - LAST_MODIFICATION)) + +echo "Checking CHANGELOG.md last modification time on GIT.." +echo "CHANGELOG.md timestamp: ${LAST_MODIFICATION}" +echo "Current timestamp: ${CURRENT_TIME}" +echo "Diff: ${DIFF}" + +if [ ${DIFF} -gt ${TWO_DAYS_IN_SECONDS} ]; then + echo "CHANGELOG.md is more than two days old!" + post_message "TRAVIS_MESSAGE" "Hi <!here>, CHANGELOG.md was found more than two days old (Diff: ${DIFF} seconds)" "${NOTIF_CHANNEL}" +else + echo "CHANGELOG.md is less than two days old, fine" +fi diff --git a/.travis/generate_changelog_for_nightlies.sh b/.travis/generate_changelog_for_nightlies.sh index 68491fa9b..b90862880 100755 --- a/.travis/generate_changelog_for_nightlies.sh +++ b/.travis/generate_changelog_for_nightlies.sh @@ -49,6 +49,7 @@ docker run -it -v "$(pwd)":/project markmandel/github-changelog-generator:latest --token "${GITHUB_TOKEN}" \ --since-tag "v1.10.0" \ --unreleased-label "**Next release**" \ + --no-issues \ --exclude-labels "stale,duplicate,question,invalid,wontfix,discussion,no changelog" \ --no-compare-link ${OPTS} diff --git a/.travis/nightlies.sh b/.travis/nightlies.sh index 188b37da0..002461041 100755 --- a/.travis/nightlies.sh +++ b/.travis/nightlies.sh @@ -38,6 +38,12 @@ if [ ! "${TRAVIS_REPO_SLUG}" == "netdata/netdata" ]; then fi echo "--- Running Changelog generation ---" -.travis/generate_changelog_for_nightlies.sh "${LAST_TAG}" "${COMMITS_SINCE_RELEASE}" || echo "Changelog generation has failed, this is a soft error, process continues" +NIGHTLIES_CHANGELOG_FAILED=0 +.travis/generate_changelog_for_nightlies.sh "${LAST_TAG}" "${COMMITS_SINCE_RELEASE}" || NIGHTLIES_CHANGELOG_FAILED=1 + +if [ ${NIGHTLIES_CHANGELOG_FAILED} -eq 1 ]; then + echo "Changelog generation has failed, this is a soft error, process continues" + post_message "TRAVIS_MESSAGE" "Changelog generation job for nightlies failed, possibly due to github issues" "${NOTIF_CHANNEL}" +fi exit "${FAIL}" diff --git a/.travis/package_management/build.sh b/.travis/package_management/build.sh new file mode 100644 index 000000000..beb522a35 --- /dev/null +++ b/.travis/package_management/build.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash + +UNPACKAGED_NETDATA_PATH="$1" +LATEST_RELEASE_VERSION="$2" + +if [ -z "${LATEST_RELEASE_VERSION}" ]; then + echo "Parameter 'LATEST_RELEASE_VERSION' not defined" + exit 1 +fi + +if [ -z "${UNPACKAGED_NETDATA_PATH}" ]; then + echo "Parameter 'UNPACKAGED_NETDATA_PATH' not defined" + exit 1 +fi + +echo "Running changelog generation mechanism since ${LATEST_RELEASE_VERSION}" + +echo "Entering ${UNPACKAGED_NETDATA_PATH}" +cd "${UNPACKAGED_NETDATA_PATH}" + +echo "Linking debian -> contrib/debian" +ln -sf contrib/debian debian + +echo "Executing dpkg-buildpackage" +if dpkg-buildpackage --version 2> /dev/null | grep -q "1.18"; then + dpkg-buildpackage --post-clean --pre-clean --build=binary +else + dpkg-buildpackage -b +fi + +echo "DEB build script completed!" diff --git a/.travis/package_management/common.py b/.travis/package_management/common.py index 6cf59293d..6e7a26023 100755 --- a/.travis/package_management/common.py +++ b/.travis/package_management/common.py @@ -6,6 +6,22 @@ import lxc import subprocess +import os + +def fetch_version(orig_build_version): + tag = None + friendly_version = "" + + # TODO: Checksum validations + if str(orig_build_version).count(".latest") == 1: + version_list=str(orig_build_version).replace('v', '').split('.') + friendly_version='.'.join(version_list[0:2]) + "." + version_list[3] + else: + friendly_version = orig_build_version.replace('v', '') + tag = friendly_version # Go to stable tag + print("Version set to %s from %s" % (friendly_version, orig_build_version)) + + return friendly_version, tag def replace_tag(tag_name, spec, new_tag_content): print("Fixing tag %s in %s" % (tag_name, spec)) @@ -44,3 +60,103 @@ def run_command_in_host(cmd): print('Output: ' + o.decode('ascii')) print('Error: ' + e.decode('ascii')) print('code: ' + str(proc.returncode)) + +def prepare_repo(container): + if str(os.environ["REPO_TOOL"]).count("zypper") == 1: + run_command(container, [os.environ["REPO_TOOL"], "clean", "-a"]) + run_command(container, [os.environ["REPO_TOOL"], "--no-gpg-checks", "update", "-y"]) + + elif str(os.environ["REPO_TOOL"]).count("yum") == 1: + run_command(container, [os.environ["REPO_TOOL"], "clean", "all"]) + run_command(container, [os.environ["REPO_TOOL"], "update", "-y"]) + + if os.environ["BUILD_STRING"].count("el/7") == 1 and os.environ["BUILD_ARCH"].count("i386") == 1: + print ("Skipping epel-release install for %s-%s" % (os.environ["BUILD_STRING"], os.environ["BUILD_ARCH"])) + else: + run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "epel-release"]) + + elif str(os.environ["REPO_TOOL"]).count("apt-get") == 1: + run_command(container, [os.environ["REPO_TOOL"], "update", "-y"]) + else: + run_command(container, [os.environ["REPO_TOOL"], "update", "-y"]) + + run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "sudo"]) + run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "wget"]) + run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "bash"]) + +def install_common_dependendencies(container): + if str(os.environ["REPO_TOOL"]).count("zypper") == 1: + run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "gcc-c++"]) + run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "json-glib-devel"]) + run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "freeipmi-devel"]) + run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "cups-devel"]) + run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "snappy-devel"]) + run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "protobuf-devel"]) + run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "protobuf-c"]) + + elif str(os.environ["REPO_TOOL"]).count("yum") == 1: + run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "gcc-c++"]) + run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "json-c-devel"]) + run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "freeipmi-devel"]) + run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "cups-devel"]) + run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "snappy-devel"]) + run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "protobuf-devel"]) + run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "protobuf-c-devel"]) + run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "protobuf-compiler"]) + + elif str(os.environ["REPO_TOOL"]).count("apt-get") == 1: + run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "g++"]) + run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "libipmimonitoring-dev"]) + run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "libjson-c-dev"]) + run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "libcups2-dev"]) + run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "libsnappy-dev"]) + run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "libprotobuf-dev"]) + run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "libprotoc-dev"]) + run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "protobuf-compiler"]) + if os.environ["BUILD_STRING"].count("debian/jessie") == 1: + run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "snappy"]) + else: + run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "gcc-c++"]) + run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "cups-devel"]) + run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "freeipmi-devel"]) + run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "json-c-devel"]) + run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "snappy-devel"]) + run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "protobuf-devel"]) + run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "protobuf-c-devel"]) + run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "protobuf-compiler"]) + + if os.environ["BUILD_STRING"].count("el/6") <= 0: + run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "autogen"]) + +def prepare_version_source(dest_archive, pkg_friendly_version, tag=None): + print(".0 Preparing local implementation tarball for version %s" % pkg_friendly_version) + tar_file = os.environ['LXC_CONTAINER_ROOT'] + dest_archive + + if tag is not None: + print(".1 Checking out tag %s" % tag) + run_command_in_host(['git', 'fetch', '--all']) + + # TODO: Keep in mind that tricky 'v' there, needs to be removed once we clear our versioning scheme + run_command_in_host(['git', 'checkout', 'v%s' % pkg_friendly_version]) + + print(".2 Tagging the code with version: %s" % pkg_friendly_version) + run_command_in_host(['git', 'tag', '-a', pkg_friendly_version, '-m', 'Tagging while packaging on %s' % os.environ["CONTAINER_NAME"]]) + + print(".3 Run autoreconf -ivf") + run_command_in_host(['autoreconf', '-ivf']) + + print(".4 Run configure") + run_command_in_host(['./configure', '--with-math', '--with-zlib', '--with-user=netdata']) + + print(".5 Run make dist") + run_command_in_host(['make', 'dist']) + + print(".6 Copy generated tarbal to desired path") + if os.path.exists('netdata-%s.tar.gz' % pkg_friendly_version): + run_command_in_host(['sudo', 'cp', 'netdata-%s.tar.gz' % pkg_friendly_version, tar_file]) + + print(".7 Fixing permissions on tarball") + run_command_in_host(['sudo', 'chmod', '777', tar_file]) + else: + print("I could not find (%s) on the disk, stopping the build. Kindly check the logs and try again" % 'netdata-%s.tar.gz' % pkg_friendly_version) + sys.exit(1) diff --git a/.travis/package_management/configure_deb_lxc_environment.py b/.travis/package_management/configure_deb_lxc_environment.py index 58999ad38..12328dde7 100755 --- a/.travis/package_management/configure_deb_lxc_environment.py +++ b/.travis/package_management/configure_deb_lxc_environment.py @@ -39,27 +39,57 @@ print("Waiting for container connectivity to start configuration sequence") if not container.get_ips(timeout=30): raise Exception("Timeout while waiting for container") +build_path = "/home/%s" % os.environ['BUILDER_NAME'] + # Run the required activities now # 1. Create the builder user print("1. Adding user %s" % os.environ['BUILDER_NAME']) common.run_command(container, ["useradd", "-m", os.environ['BUILDER_NAME']]) # Fetch package dependencies for the build -print("2. Installing package dependencies within LXC container") -common.run_command(container, ["apt-get", "update", "-y"]) -common.run_command(container, ["apt-get", "install", "-y", "sudo"]) -common.run_command(container, ["apt-get", "install", "-y", "wget"]) -common.run_command(container, ["apt-get", "install", "-y", "bash"]) -common.run_command(container, ["wget", "-T", "15", "-O", "~/.install-required-packages.sh", "https://raw.githubusercontent.com/netdata/netdata-demo-site/master/install-required-packages.sh"]) -common.run_command(container, ["bash", "~/.install-required-packages.sh", "netdata", "--dont-wait", "--non-interactive"]) - -# Download the source -dest_archive="/home/%s/netdata-%s.tar.gz" % (os.environ['BUILDER_NAME'],os.environ['BUILD_VERSION']) -release_url="https://github.com/netdata/netdata/releases/download/%s/netdata-%s.tar.gz" % (os.environ['BUILD_VERSION'], os.environ['BUILD_VERSION']) -print("3. Fetch netdata source (%s -> %s)" % (release_url, dest_archive)) -common.run_command(container, ["sudo", "-u", os.environ['BUILDER_NAME'], "wget", "-T", "15", "--output-document=" + dest_archive, release_url]) - -print("4. Extracting directory contents to /home " + os.environ['BUILDER_NAME']) -common.run_command(container, ["sudo", "-u", os.environ['BUILDER_NAME'], "tar", "xf", dest_archive, "-C", "/home/" + os.environ['BUILDER_NAME']]) +print("2. Preparing repo on LXC container") +common.prepare_repo(container) + +print("2.1 Install .DEB build support packages") +common.run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "dpkg-dev"]) +common.run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "libdistro-info-perl"]) +common.run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "dh-make"]) +common.run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "dh-systemd"]) +common.run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "dh-autoreconf"]) +common.run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "git-buildpackage"]) + +print("2.2 Add more dependencies") +common.run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "libnetfilter-acct-dev"]) +common.run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "libcups2-dev"]) + +print ("3.1 Run install-required-packages scriptlet") +common.run_command(container, ["wget", "-T", "15", "-O", "%s/.install-required-packages.sh" % build_path, "https://raw.githubusercontent.com/netdata/netdata-demo-site/master/install-required-packages.sh"]) +common.run_command(container, ["bash", "%s/.install-required-packages.sh" % build_path, "netdata", "--dont-wait", "--non-interactive"]) + +print("3.2 Installing package dependencies within LXC container") +common.install_common_dependendencies(container) + +friendly_version="" +dest_archive="" +download_url="" +tag = None +friendly_version, tag = common.fetch_version(os.environ['BUILD_VERSION']) + +tar_file="%s/netdata-%s.tar.gz" % (os.path.dirname(dest_archive), friendly_version) + +print("5. I will be building version '%s' of netdata." % os.environ['BUILD_VERSION']) +dest_archive="%s/netdata-%s.tar.gz" % (build_path, friendly_version) + +if str(os.environ["BUILD_STRING"]).count("debian/jessie") == 1: + print("5.1 We are building for Jessie, adjusting control file") + common.run_command_in_host(['sudo', 'rm', 'contrib/debian/control']) + common.run_command_in_host(['sudo', 'cp', 'contrib/debian/control.jessie', 'contrib/debian/control']) + +common.prepare_version_source(dest_archive, friendly_version, tag=tag) + +print("6. Installing build.sh script to build path") +common.run_command_in_host(['sudo', 'cp', '.travis/package_management/build.sh', "%s/%s/build.sh" % (os.environ['LXC_CONTAINER_ROOT'], build_path)]) +common.run_command_in_host(['sudo', 'chmod', '777', "%s/%s/build.sh" % (os.environ['LXC_CONTAINER_ROOT'], build_path)]) +common.run_command_in_host(['sudo', 'ln', '-sf', 'contrib/debian', 'debian']) print("Done!") diff --git a/.travis/package_management/configure_rpm_lxc_environment.py b/.travis/package_management/configure_rpm_lxc_environment.py index 644e027b7..79d34608f 100755 --- a/.travis/package_management/configure_rpm_lxc_environment.py +++ b/.travis/package_management/configure_rpm_lxc_environment.py @@ -46,30 +46,15 @@ print("1. Adding user %s" % os.environ['BUILDER_NAME']) common.run_command(container, ["useradd", "-m", os.environ['BUILDER_NAME']]) # Fetch package dependencies for the build -print("2. Installing package dependencies within LXC container") -if str(os.environ["REPO_TOOL"]).count("zypper") == 1: - common.run_command(container, [os.environ["REPO_TOOL"], "clean", "-a"]) - common.run_command(container, [os.environ["REPO_TOOL"], "--no-gpg-checks", "update", "-y"]) - common.run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "json-glib-devel"]) - -elif str(os.environ["REPO_TOOL"]).count("yum") == 1: - common.run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "json-c-devel"]) - common.run_command(container, [os.environ["REPO_TOOL"], "clean", "all"]) - common.run_command(container, [os.environ["REPO_TOOL"], "update", "-y"]) - common.run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "epel-release"]) -else: - common.run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "json-c-devel"]) - common.run_command(container, [os.environ["REPO_TOOL"], "update", "-y"]) - -common.run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "sudo"]) -common.run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "wget"]) -common.run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "bash"]) -common.run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "freeipmi-devel"]) -common.run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "cups-devel"]) +print("2.1 Preparing repo on LXC container") +common.prepare_repo(container) + +common.run_command(container, ["wget", "-T", "15", "-O", "/home/%s/.install-required-packages.sh" % (os.environ['BUILDER_NAME']), "https://raw.githubusercontent.com/netdata/netdata-demo-site/master/install-required-packages.sh"]) +common.run_command(container, ["bash", "/home/%s/.install-required-packages.sh" % (os.environ['BUILDER_NAME']), "netdata", "--dont-wait", "--non-interactive"]) # Exceptional cases, not available everywhere # - +print("2.2 Running uncommon dependencies and preparing LXC environment") # Not on Centos-7 if os.environ["BUILD_STRING"].count("el/7") <= 0: common.run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "libnetfilter_acct-devel"]) @@ -78,8 +63,8 @@ if os.environ["BUILD_STRING"].count("el/7") <= 0: if os.environ["BUILD_STRING"].count("el/6") <= 0: common.run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "autoconf-archive"]) -common.run_command(container, ["wget", "-T", "15", "-O", "/home/%s/.install-required-packages.sh" % (os.environ['BUILDER_NAME']), "https://raw.githubusercontent.com/netdata/netdata-demo-site/master/install-required-packages.sh"]) -common.run_command(container, ["bash", "/home/%s/.install-required-packages.sh" % (os.environ['BUILDER_NAME']), "netdata", "--dont-wait", "--non-interactive"]) +print("2.3 Installing common dependencies") +common.install_common_dependendencies(container) print("3. Setting up macros") common.run_command(container, ["sudo", "-u", os.environ['BUILDER_NAME'], "/bin/echo", "'%_topdir %(echo /home/" + os.environ['BUILDER_NAME'] + ")/rpmbuild' > /home/" + os.environ['BUILDER_NAME'] + "/.rpmmacros"]) @@ -97,63 +82,21 @@ rpm_friendly_version="" dest_archive="" download_url="" spec_file="/home/%s/rpmbuild/SPECS/netdata.spec" % os.environ['BUILDER_NAME'] +tag = None +rpm_friendly_version, tag = common.fetch_version(os.environ['BUILD_VERSION']) +tar_file="%s/netdata-%s.tar.gz" % (os.path.dirname(dest_archive), rpm_friendly_version) -# TODO: Checksum validations -if str(os.environ['BUILD_VERSION']).count(".latest") == 1: - version_list=str(os.environ['BUILD_VERSION']).replace('v', '').split('.') - rpm_friendly_version='.'.join(version_list[0:3]) + "." + version_list[3] - - print("Building latest nightly version of netdata..(%s)" % os.environ['BUILD_VERSION']) - dest_archive="/home/%s/rpmbuild/SOURCES/netdata-%s.tar.gz" % (os.environ['BUILDER_NAME'], rpm_friendly_version) - - print("5. Preparing local latest implementation tarball for version %s" % rpm_friendly_version) - tar_file = os.environ['LXC_CONTAINER_ROOT'] + dest_archive - - print("5.1 Tagging the code with latest version: %s" % rpm_friendly_version) - common.run_command_in_host(['git', 'tag', '-a', rpm_friendly_version, '-m', 'Tagging while packaging on %s' % os.environ["CONTAINER_NAME"]]) - - print("5.2 Run autoreconf -ivf") - common.run_command_in_host(['autoreconf', '-ivf']) - - print("5.3 Run configure") - common.run_command_in_host(['./configure', '--with-math', '--with-zlib', '--with-user=netdata']) - - print("5.4 Run make dist") - common.run_command_in_host(['make', 'dist']) - - print("5.5 Copy generated tarbal to desired path") - if os.path.exists('netdata-%s.tar.gz' % rpm_friendly_version): - common.run_command_in_host(['sudo', 'cp', 'netdata-%s.tar.gz' % rpm_friendly_version, tar_file]) - - print("5.6 Fixing permissions on tarball") - common.run_command_in_host(['sudo', 'chmod', '777', tar_file]) - else: - print("I could not find (%s) on the disk, stopping the build. Kindly check the logs and try again" % 'netdata-%s.tar.gz' % rpm_friendly_version) - sys.exit(1) - - # Extract the spec file in place - print("6. Extract spec file from the source") - common.run_command_in_host(['sudo', 'cp', 'netdata.spec', os.environ['LXC_CONTAINER_ROOT'] + spec_file]) - common.run_command_in_host(['sudo', 'chmod', '777', os.environ['LXC_CONTAINER_ROOT'] + spec_file]) - - print("7. Temporary hack: Change Source0 to %s on spec file %s" % (dest_archive, spec_file)) - common.replace_tag("Source0", os.environ['LXC_CONTAINER_ROOT'] + spec_file, tar_file) -else: - rpm_friendly_version = os.environ['BUILD_VERSION'] - - print("Building latest stable version of netdata.. (%s)" % os.environ['BUILD_VERSION']) - dest_archive="/home/%s/rpmbuild/SOURCES/netdata-%s.tar.gz" % (os.environ['BUILDER_NAME'],os.environ['BUILD_VERSION']) - download_url="https://github.com/netdata/netdata/releases/download/%s/netdata-%s.tar.gz" % (os.environ['BUILD_VERSION'], os.environ['BUILD_VERSION']) +print("5. I will be building version '%s' of netdata." % os.environ['BUILD_VERSION']) +dest_archive="/home/%s/rpmbuild/SOURCES/netdata-%s.tar.gz" % (os.environ['BUILDER_NAME'], rpm_friendly_version) - print("5. Fetch netdata source into the repo structure(%s -> %s)" % (download_url, dest_archive)) - tar_file="%s/netdata-%s.tar.gz" % (os.path.dirname(dest_archive), rpm_friendly_version) - common.run_command(container, ["sudo", "-u", os.environ['BUILDER_NAME'], "wget", "-T", "15", "--output-document=" + dest_archive, download_url]) +common.prepare_version_source(dest_archive, rpm_friendly_version, tag=tag) - print("6.Extract spec file from the source") - common.run_command(container, ["sudo", "-u", os.environ['BUILDER_NAME'], "tar", "--to-command=cat > %s" % spec_file, "-xvf", dest_archive, "netdata-%s/netdata.spec" % os.environ['BUILD_VERSION']]) +# Extract the spec file in place +print("6. Extract spec file from the source") +common.run_command_in_host(['sudo', 'cp', 'netdata.spec', os.environ['LXC_CONTAINER_ROOT'] + spec_file]) +common.run_command_in_host(['sudo', 'chmod', '777', os.environ['LXC_CONTAINER_ROOT'] + spec_file]) - print("7. Temporary hack: Adjust version string on the spec file (%s) to %s and Source0 to %s" % (os.environ['LXC_CONTAINER_ROOT'] + spec_file, rpm_friendly_version, download_url)) - common.replace_tag("Version", os.environ['LXC_CONTAINER_ROOT'] + spec_file, rpm_friendly_version) - common.replace_tag("Source0", os.environ['LXC_CONTAINER_ROOT'] + spec_file, tar_file) +print("7. Temporary hack: Change Source0 to %s on spec file %s" % (dest_archive, spec_file)) +common.replace_tag("Source0", os.environ['LXC_CONTAINER_ROOT'] + spec_file, tar_file) print('Done!') diff --git a/.travis/package_management/create_lxc_for_build.sh b/.travis/package_management/create_lxc_for_build.sh index ae855a742..d733687a8 100755 --- a/.travis/package_management/create_lxc_for_build.sh +++ b/.travis/package_management/create_lxc_for_build.sh @@ -83,7 +83,7 @@ case "${BUILD_ARCH}" in .travis/package_management/configure_${PACKAGE_TYPE}_lxc_environment.py "${CONTAINER_NAME}" ;; "i386"|"amd64"|"arm64") - # AMD64 or i386 + # amd64 or i386 echo "Creating LXC Container for ${BUILD_ARCH}.." export CONTAINER_NAME="${BUILDER_NAME}-${BUILD_DISTRO}${BUILD_RELEASE}-${BUILD_ARCH}" export LXC_CONTAINER_ROOT="/var/lib/lxc/${CONTAINER_NAME}/rootfs" diff --git a/.travis/package_management/functions.sh b/.travis/package_management/functions.sh index 9a467ffe1..0c4425fa5 100644 --- a/.travis/package_management/functions.sh +++ b/.travis/package_management/functions.sh @@ -10,7 +10,7 @@ set -e function detect_arch_from_commit { case "${TRAVIS_COMMIT_MESSAGE}" in - "[Package AMD64"*) + "[Package amd64"*) export BUILD_ARCH="amd64" ;; "[Package i386"*) @@ -24,7 +24,7 @@ function detect_arch_from_commit { ;; *) - echo "Unknown build architecture '${BUILD_ARCH}' provided" + echo "Unknown build architecture in '${TRAVIS_COMMIT_MESSAGE}'. No BUILD_ARCH can be provided" exit 1 ;; esac diff --git a/.travis/package_management/prepare_packages.sh b/.travis/package_management/prepare_packages.sh index 1fb26a95e..12ed07cc7 100755 --- a/.travis/package_management/prepare_packages.sh +++ b/.travis/package_management/prepare_packages.sh @@ -27,29 +27,36 @@ for d in ${CREATED_CONTAINERS[@]}; do # Pick up any RPMS from builder RPM_BUILD_PATH="${LXC_ROOT}/${d}/rootfs/home/${BUILDER_NAME}/rpmbuild" - echo "Checking folder ${RPM_BUILD_PATH} for RPMS and SRPMS" - - if [ -d "${RPM_BUILD_PATH}/RPMS" ]; then - echo "Copying any RPMS in '${RPM_BUILD_PATH}', copying over the following:" - ls -ltrR "${RPM_BUILD_PATH}/RPMS" - [[ -d "${RPM_BUILD_PATH}/RPMS/x86_64" ]] && cp -r "${RPM_BUILD_PATH}"/RPMS/x86_64/* "${PACKAGES_DIRECTORY}" - [[ -d "${RPM_BUILD_PATH}/RPMS/i386" ]] && cp -r "${RPM_BUILD_PATH}"/RPMS/i386/* "${PACKAGES_DIRECTORY}" - [[ -d "${RPM_BUILD_PATH}/RPMS/i686" ]] && cp -r "${RPM_BUILD_PATH}"/RPMS/i686/* "${PACKAGES_DIRECTORY}" - fi - - if [ -d "${RPM_BUILD_PATH}/SRPMS" ]; then - echo "Copying any SRPMS in '${RPM_BUILD_PATH}', copying over the following:" - ls -ltrR "${RPM_BUILD_PATH}/SRPMS" - [[ -d "${RPM_BUILD_PATH}/SRPMS/x86_64" ]] && cp -r "${RPM_BUILD_PATH}"/SRPMS/x86_64/* "${PACKAGES_DIRECTORY}" - [[ -d "${RPM_BUILD_PATH}/SRPMS/i386" ]] && cp -r "${RPM_BUILD_PATH}"/SRPMS/i386/* "${PACKAGES_DIRECTORY}" - [[ -d "${RPM_BUILD_PATH}/SRPMS/i686" ]] && cp -r "${RPM_BUILD_PATH}"/SRPMS/i686/* "${PACKAGES_DIRECTORY}" + if [ -d "${RPM_BUILD_PATH}" ]; then + echo "Checking folder ${RPM_BUILD_PATH} for RPMS and SRPMS" + + if [ -d "${RPM_BUILD_PATH}/RPMS" ]; then + echo "Copying any RPMS in '${RPM_BUILD_PATH}', copying over the following:" + ls -ltrR "${RPM_BUILD_PATH}/RPMS" + [[ -d "${RPM_BUILD_PATH}/RPMS/x86_64" ]] && cp -r "${RPM_BUILD_PATH}"/RPMS/x86_64/* "${PACKAGES_DIRECTORY}" + [[ -d "${RPM_BUILD_PATH}/RPMS/i386" ]] && cp -r "${RPM_BUILD_PATH}"/RPMS/i386/* "${PACKAGES_DIRECTORY}" + [[ -d "${RPM_BUILD_PATH}/RPMS/i686" ]] && cp -r "${RPM_BUILD_PATH}"/RPMS/i686/* "${PACKAGES_DIRECTORY}" + fi + + if [ -d "${RPM_BUILD_PATH}/SRPMS" ]; then + echo "Copying any SRPMS in '${RPM_BUILD_PATH}', copying over the following:" + ls -ltrR "${RPM_BUILD_PATH}/SRPMS" + [[ -d "${RPM_BUILD_PATH}/SRPMS/x86_64" ]] && cp -r "${RPM_BUILD_PATH}"/SRPMS/x86_64/* "${PACKAGES_DIRECTORY}" + [[ -d "${RPM_BUILD_PATH}/SRPMS/i386" ]] && cp -r "${RPM_BUILD_PATH}"/SRPMS/i386/* "${PACKAGES_DIRECTORY}" + [[ -d "${RPM_BUILD_PATH}/SRPMS/i686" ]] && cp -r "${RPM_BUILD_PATH}"/SRPMS/i686/* "${PACKAGES_DIRECTORY}" + fi + else + DEB_BUILD_PATH="${LXC_ROOT}/${d}/rootfs/home/${BUILDER_NAME}" + echo "Checking folder ${DEB_BUILD_PATH} for DEB packages" + if [ -d "${DEB_BUILD_PATH}" ]; then + cp "${DEB_BUILD_PATH}"/netdata*.ddeb "${PACKAGES_DIRECTORY}" || echo "Could not copy any .ddeb files" + cp "${DEB_BUILD_PATH}"/netdata*.deb "${PACKAGES_DIRECTORY}" || echo "Could not copy any .deb files" + cp "${DEB_BUILD_PATH}"/netdata*.buildinfo "${PACKAGES_DIRECTORY}" || echo "Could not copy any .buildinfo files" + cp "${DEB_BUILD_PATH}"/netdata*.changes "${PACKAGES_DIRECTORY}" || echo "Could not copy any .changes files" + else + echo "Folder ${DEB_BUILD_PATH} does not exist or not a directory, nothing to do for package preparation" + fi fi - - # Pick up any DEBs from builder - DEB_BUILD_PATH="${d}/home/${BUILDER_NAME}/build-area" - echo "Checking folder ${DEB_BUILD_PATH} for DEB packages" - #TODO: During debian clean up we 'll fill this up - done chmod -R 777 "${PACKAGES_DIRECTORY}" diff --git a/.travis/package_management/trigger_deb_lxc_build.py b/.travis/package_management/trigger_deb_lxc_build.py index 3040bdd62..a0235a73d 100755 --- a/.travis/package_management/trigger_deb_lxc_build.py +++ b/.travis/package_management/trigger_deb_lxc_build.py @@ -37,15 +37,37 @@ if not container.running or not container.state == "RUNNING": if not container.get_ips(timeout=30): raise Exception("Timeout while waiting for container") +build_path = "/home/%s" % os.environ['BUILDER_NAME'] + print("Setting up EMAIL and DEBFULLNAME variables required by the build tools") os.environ["EMAIL"] = "bot@netdata.cloud" os.environ["DEBFULLNAME"] = "Netdata builder" # Run the build process on the container -print("Starting DEB build process, running dh-make") -new_version = os.environ["BUILD_VERSION"].replace('v', '') +new_version, tag = common.fetch_version(os.environ['BUILD_VERSION']) +print("Starting DEB build process for version %s" % new_version) + +netdata_tarball = "%s/netdata-%s.tar.gz" % (build_path, new_version) +unpacked_netdata = netdata_tarball.replace(".tar.gz", "") + +print("Extracting tarball %s" % netdata_tarball) +common.run_command(container, ["sudo", "-u", os.environ['BUILDER_NAME'], "tar", "xf", netdata_tarball, "-C", build_path]) + +print("Fixing changelog tags") +changelog_in_host = "contrib/debian/changelog" +common.run_command_in_host(['sed', '-i', 's/PREVIOUS_PACKAGE_VERSION/%s-1/g' % os.environ["LATEST_RELEASE_VERSION"].replace("v", ""), changelog_in_host]) +common.run_command_in_host(['sed', '-i', 's/PREVIOUS_PACKAGE_DATE/%s/g' % os.environ["LATEST_RELEASE_DATE"], changelog_in_host]) + +print("Executing gbp dch command..") +common.run_command_in_host(['gbp', 'dch', '--release', '--ignore-branch', '--spawn-editor=snapshot', '--since=%s' % os.environ["LATEST_RELEASE_VERSION"], '--new-version=%s' % new_version]) + +print("Copying over changelog to the destination machine") +common.run_command_in_host(['sudo', 'cp', 'debian/changelog', "%s/%s/netdata-%s/contrib/debian/" % (os.environ['LXC_CONTAINER_ROOT'], build_path, new_version)]) + +print("Running debian build script since %s" % os.environ["LATEST_RELEASE_VERSION"]) +common.run_command(container, ["sudo", "-u", os.environ['BUILDER_NAME'], "%s/build.sh" % build_path, unpacked_netdata, new_version]) -print("Building the package") -common.run_command(container, ["sudo", "-u", os.environ['BUILDER_NAME'], "dpkg-buildpackage", "--host-arch", "amd64", "--target-arch", "amd64", "--post-clean", "--pre-clean", "--build=binary", "--release-by=\"Netdata Builder\"", "--build-by=\"Netdata Builder\""]) +print("Listing contents on build path") +common.run_command(container, ["sudo", "-u", os.environ['BUILDER_NAME'], "ls", "-ltr", build_path]) print('Done!') diff --git a/.travis/package_management/yank_stale_rpm.sh b/.travis/package_management/yank_stale_rpm.sh index 5cf938664..3f7669712 100755 --- a/.travis/package_management/yank_stale_rpm.sh +++ b/.travis/package_management/yank_stale_rpm.sh @@ -21,7 +21,7 @@ fi PACKAGES_DIR="$1" DISTRO="$2" -PACKAGES_LIST="$(ls -AR "${PACKAGES_DIR}" | grep '\.rpm')" +PACKAGES_LIST="$(ls -AR "${PACKAGES_DIR}" | grep -e '\.rpm' -e '\.deb' -e '\.ddeb' )" if [ ! -d "${PACKAGES_DIR}" ] || [ -z "${PACKAGES_LIST}" ]; then echo "Folder ${PACKAGES_DIR} does not seem to be a valid directory or is empty. No packages to check for yanking" |