diff options
Diffstat (limited to '')
| -rw-r--r-- | .travis.yml | 458 |
1 files changed, 458 insertions, 0 deletions
diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 000000000..c7a6074d6 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,458 @@ +--- +dist: focal +language: c + +addons: + apt: + packages: ['moreutils'] + +env: + global: + - RELEASE_CHANNEL=nightly + +before_install: + - exec > >(ts -s '%H:%M:%.S ') 2>&1 + - source .travis/utils.sh + +# Install dependencies for all, once +# +install: + - sudo apt-get install -y libuv1-dev liblz4-dev libjudy-dev libcap2-bin zlib1g-dev uuid-dev fakeroot libipmimonitoring-dev libmnl-dev libnetfilter-acct-dev gnupg python3-pip + - sudo pip install git-semver==0.3.2 # 11/Sep/2019: git-semver tip was broken, so we had to force last good run of it + - source tests/installer/slack.sh + - export NOTIF_CHANNEL="automation-beta" + - if [ "${TRAVIS_REPO_SLUG}" = "netdata/netdata" ]; then export NOTIF_CHANNEL="automation"; fi; + - export BUILD_VERSION="$(cat packaging/version | cut -d'-' -f1)" + - export LATEST_RELEASE_VERSION="$(cat packaging/version | cut -d'-' -f1)" + - export LATEST_RELEASE_DATE="$(git log -1 --format=%aD "${LATEST_RELEASE_VERSION}" | cat)" + - if [[ "${TRAVIS_COMMIT_MESSAGE}" = *"[Build latest]"* ]]; then export BUILD_VERSION="$(cat packaging/version | cut -d'-' -f1,2 | sed -e 's/-/./g').latest"; fi; + - export DEPLOY_REPO="netdata" # Default production packaging repository + - export PACKAGING_USER="netdata" # Standard package cloud account + - if [[ "${TRAVIS_COMMIT_MESSAGE}" = *"[Build latest]"* ]]; then export DEPLOY_REPO="netdata-edge"; fi; + - export PACKAGE_CLOUD_RETENTION_DAYS=30 + - if [ ! "${TRAVIS_REPO_SLUG}" = "netdata/netdata" ]; then export DEPLOY_REPO="netdata-devel"; fi; + # These are release-related artifacts and have to be evaluated before we start doing conditional checks inside stages + - source ".travis/tagger.sh" + - export GIT_TAG="$(git tag --points-at)" + + +# Setup notification system +# +notifications: + webhooks: + urls: + - https://app.fossa.io/hooks/travisci + +# Define the stage sequence and conditionals +# +stages: + # Mandatory runs, we always want these executed + - name: Build process + if: commit_message =~ /^((?!\[Package (amd64|arm64|i386) (DEB|RPM)( .*)?\]).)*$/ + + # Nightly operations + - name: Nightly operations + if: branch = master AND type = cron AND env(RUN_NIGHTLY) = yes + + - name: Nightly release + if: branch = master AND type = cron AND env(RUN_NIGHTLY) = yes + - name: Trigger deb and rpm package build (nightly release) + if: branch = master AND type = cron AND env(RUN_NIGHTLY) = yes + + # Scheduled releases + - name: Support activities on main branch + if: branch = master AND type != pull_request AND type != cron AND repo = netdata/netdata + + # We don't run on release candidates + - name: Publish for release + if: >- + branch = master + AND type != pull_request + AND type != cron + AND tag !~ /(-rc)/ + AND commit_message =~ /\[netdata (release candidate|(major|minor|patch) release)\]/ + - name: Trigger deb and rpm package build (release) + if: >- + branch = master + AND type != pull_request + AND type != cron + AND tag !~ /(-rc)/ + AND commit_message =~ /\[netdata (release candidate|(major|minor|patch) release)\]/ + + # Build DEB packages under special conditions + - name: Package ubuntu/* and debian/* + if: type != cron AND type != pull_request AND branch = master + + # Build RPM packages under special conditions + - name: Package centos, fedora and opensuse + if: type != cron AND type != pull_request AND branch = master + + +# Define stage implementation details +# +jobs: + # This is a hook to help us introduce "soft" errors on our process + allow_failures: + - env: ALLOW_SOFT_FAILURE_HERE=true + include: + # Ensure netdata code builds successfully + - stage: Build process + + name: Standard netdata build + script: fakeroot ./netdata-installer.sh --install $HOME --dont-wait --dont-start-it --enable-plugin-nfacct --enable-plugin-freeipmi --disable-lto + env: CFLAGS='-O1 -Wall -Wextra -Wformat-signedness -fstack-protector-all -fno-common -DNETDATA_INTERNAL_CHECKS=1 -D_FORTIFY_SOURCE=2 -DNETDATA_VERIFY_LOCKS=1' + after_failure: post_message "TRAVIS_MESSAGE" "<!here> standard netdata build is failing (Still dont know which one, will improve soon)" + + - name: Build/Install for ubuntu 20.04 (not containerized) + script: fakeroot ./netdata-installer.sh --dont-wait --dont-start-it --install $HOME + after_failure: post_message "TRAVIS_MESSAGE" "Build/Install failed on ubuntu 18.04" + + - name: Build/install for CentOS 7 (Containerized) + script: docker run -it -v "${PWD}:/netdata:rw" -w /netdata "netdata/os-test:centos7" ./netdata-installer.sh --dont-wait --dont-start-it --install /tmp + after_failure: post_message "TRAVIS_MESSAGE" "Build/Install failed on CentOS 7" + + - name: DEB package test + git: + depth: false + before_install: + - sudo apt-get install -y wget lxc python3-lxc lxc-templates dh-make git-buildpackage build-essential libdistro-info-perl + before_script: + - export PACKAGES_DIRECTORY="$(mktemp -d -t netdata-packaging-contents-dir-XXXXXX)" && echo "Created packaging directory ${PACKAGES_DIRECTORY}" + script: + - echo "GIT Branch:" && git branch + - echo "Last commit:" && git log -1 + - echo "GIT Describe:" && git describe + - echo "packaging/version:" && cat packaging/version + - echo "Creating LXC environment for the build" && sudo -E .travis/package_management/create_lxc_for_build.sh + - echo "Building package in container" && sudo -E .travis/package_management/build_package_in_container.sh + - sudo chown -R root:travis "/var/lib/lxc" + - sudo chmod -R 750 "/var/lib/lxc" + - echo "Preparing DEB packaging contents for upload" && sudo -E .travis/package_management/prepare_packages.sh + env: + - BUILDER_NAME="builder" BUILD_DISTRO="debian" BUILD_RELEASE="buster" BUILD_STRING="debian/buster" + - PACKAGE_TYPE="deb" REPO_TOOL="apt-get" + after_failure: post_message "TRAVIS_MESSAGE" "Netdata DEB package build check failed." + + - name: RPM package test + git: + depth: false + before_install: + - sudo apt-get install -y wget lxc lxc-templates python3-lxc + before_script: + - export PACKAGES_DIRECTORY="$(mktemp -d -t netdata-packaging-contents-dir-XXXXXX)" && echo "Created packaging directory ${PACKAGES_DIRECTORY}" + script: + - echo "GIT Branch:" && git branch + - echo "Last commit:" && git log -1 + - echo "GIT Describe:" && git describe + - echo "packaging/version:" && cat packaging/version + - echo "Creating LXC environment for the build" && sudo -E .travis/package_management/create_lxc_for_build.sh + - echo "Building package in container" && sudo -E .travis/package_management/build_package_in_container.sh + - sudo chmod -R 755 "/var/lib/lxc" + - echo "Preparing RPM packaging contents for upload" && sudo -E .travis/package_management/prepare_packages.sh + env: + - BUILDER_NAME="builder" BUILD_DISTRO="fedora" BUILD_RELEASE="32" BUILD_STRING="fedora/32" + - PACKAGE_TYPE="rpm" REPO_TOOL="dnf" + after_failure: post_message "TRAVIS_MESSAGE" "Netdata RPM package build check failed." + + - stage: Support activities on main branch + name: Generate changelog for release (only on special and tagged commit msg) + before_script: post_message "TRAVIS_MESSAGE" "Support activities on main branch initiated" "${NOTIF_CHANNEL}" + script: + - echo "GIT Branch:" && git branch + - echo "Last commit:" && git log -1 + - echo "GIT Describe:" && git describe + - echo "packaging/version:" && cat packaging/version + - if [[ -z "${GIT_TAG}" ]]; then echo "Running set tag for release" && set_tag_for_release; fi; + - .travis/generate_changelog_and_tag_release.sh + after_failure: post_message "TRAVIS_MESSAGE" "<!here> Changelog generation and tag of release, failed" + git: + depth: false + if: commit_message =~ /\[netdata (release candidate|(major|minor|patch) release)\]/ AND tag !~ /(-rc)/ OR (env(GIT_TAG) IS present AND NOT env(GIT_TAG) IS blank) + + # ###### Packaging workflow section ###### + # References: + # https://us.images.linuxcontainers.org + # https://packagecloud.io/docs#install_repo + + # TODO: This section is stale, will be aligned with the RPM implementation when we get to DEB packaging + - stage: Package ubuntu/* and debian/* + _template: &DEB_TEMPLATE + git: + depth: false + before_install: + - sudo apt-get install -y wget lxc python3-lxc lxc-templates dh-make git-buildpackage build-essential libdistro-info-perl + - source tests/installer/slack.sh + before_script: + - export PACKAGES_DIRECTORY="$(mktemp -d -t netdata-packaging-contents-dir-XXXXXX)" && echo "Created packaging directory ${PACKAGES_DIRECTORY}" + script: + - echo "GIT Branch:" && git branch + - echo "Last commit:" && git log -1 + - echo "GIT Describe:" && git describe + - echo "packaging/version:" && cat packaging/version + - echo "Creating LXC environment for the build" && sudo -E .travis/package_management/create_lxc_for_build.sh + - echo "Building package in container" && sudo -E .travis/package_management/build_package_in_container.sh + - sudo chown -R root:travis "/var/lib/lxc" + - sudo chmod -R 750 "/var/lib/lxc" + - echo "Preparing DEB packaging contents for upload" && sudo -E .travis/package_management/prepare_packages.sh + after_failure: post_message "TRAVIS_MESSAGE" "Failed to build DEB for ${BUILD_STRING}.${BUILD_ARCH}" + before_deploy: + - .travis/package_management/yank_stale_pkg.sh "${PACKAGES_DIRECTORY}" "${BUILD_STRING}" || echo "No stale DEB found" + deploy: + - provider: packagecloud + repository: "${DEPLOY_REPO}" + username: "${PACKAGING_USER}" + token: "${PKG_CLOUD_TOKEN}" + dist: "${BUILD_STRING}" + local_dir: "${PACKAGES_DIRECTORY}" + skip_cleanup: true + on: + # Only deploy on ${USER}/netdata, master branch, when build-area directory is created + repo: ${TRAVIS_REPO_SLUG} + branch: "master" + condition: -d "${PACKAGES_DIRECTORY}" + after_deploy: + - if [ -n "${BUILDER_NAME}" ]; then rm -rf /home/${BUILDER_NAME}/* && echo "Cleared /home/${BUILDER_NAME} directory" || echo "Failed to clean /home/${BUILDER_NAME} directory"; fi; + - if [ -d "${PACKAGES_DIRECTORY}" ]; then rm -rf "${PACKAGES_DIRECTORY}"; fi; + + name: "Build & Publish DEB package for ubuntu/bionic" + <<: *DEB_TEMPLATE + if: commit_message =~ /\[Package (amd64|arm64|i386) DEB( Ubuntu)?\]/ + env: + - BUILDER_NAME="builder" BUILD_DISTRO="ubuntu" BUILD_RELEASE="bionic" BUILD_STRING="ubuntu/bionic" + - PACKAGE_TYPE="deb" REPO_TOOL="apt-get" + - ALLOW_SOFT_FAILURE_HERE=true + + - name: "Build & Publish DEB package for ubuntu/xenial" + <<: *DEB_TEMPLATE + if: commit_message =~ /\[Package (amd64|arm64|i386) DEB( Ubuntu)?\]/ + env: + - BUILDER_NAME="builder" BUILD_DISTRO="ubuntu" BUILD_RELEASE="xenial" BUILD_STRING="ubuntu/xenial" + - PACKAGE_TYPE="deb" REPO_TOOL="apt-get" + - ALLOW_SOFT_FAILURE_HERE=true + + - name: "Build & Publish DEB package for ubuntu/focal" + <<: *DEB_TEMPLATE + if: commit_message =~ /\[Package (amd64|arm64) DEB( Ubuntu)?\]/ + env: + - BUILDER_NAME="builder" BUILD_DISTRO="ubuntu" BUILD_RELEASE="focal" BUILD_STRING="ubuntu/focal" + - PACKAGE_TYPE="deb" REPO_TOOL="apt-get" + - ALLOW_SOFT_FAILURE_HERE=true + + - name: "Build & Publish DEB package for ubuntu/groovy" + <<: *DEB_TEMPLATE + if: commit_message =~ /\[Package (amd64|arm64) DEB( Ubuntu)?\]/ + env: + - BUILDER_NAME="builder" BUILD_DISTRO="ubuntu" BUILD_RELEASE="groovy" BUILD_STRING="ubuntu/groovy" + - PACKAGE_TYPE="deb" REPO_TOOL="apt-get" + - ALLOW_SOFT_FAILURE_HERE=true + + - name: "Build & Publish DEB package for debian/buster" + <<: *DEB_TEMPLATE + if: commit_message =~ /\[Package (amd64|arm64|i386) DEB( Debian)?\]/ + env: + - BUILDER_NAME="builder" BUILD_DISTRO="debian" BUILD_RELEASE="buster" BUILD_STRING="debian/buster" + - PACKAGE_TYPE="deb" REPO_TOOL="apt-get" + - ALLOW_SOFT_FAILURE_HERE=true + + - name: "Build & Publish DEB package for debian/stretch" + <<: *DEB_TEMPLATE + if: commit_message =~ /\[Package (amd64|arm64|i386) DEB( Debian)?\]/ + env: + - BUILDER_NAME="builder" BUILD_DISTRO="debian" BUILD_RELEASE="stretch" BUILD_STRING="debian/stretch" + - PACKAGE_TYPE="deb" REPO_TOOL="apt-get" + - ALLOW_SOFT_FAILURE_HERE=true + + - stage: Package centos, fedora and opensuse + _template: &RPM_TEMPLATE + git: + depth: false + before_install: + - sudo apt-get install -y wget lxc lxc-templates python3-lxc + - source tests/installer/slack.sh + before_script: + - export PACKAGES_DIRECTORY="$(mktemp -d -t netdata-packaging-contents-dir-XXXXXX)" && echo "Created packaging directory ${PACKAGES_DIRECTORY}" + script: + - echo "GIT Branch:" && git branch + - echo "Last commit:" && git log -1 + - echo "GIT Describe:" && git describe + - echo "packaging/version:" && cat packaging/version + - echo "Creating LXC environment for the build" && sudo -E .travis/package_management/create_lxc_for_build.sh + - echo "Building package in container" && sudo -E .travis/package_management/build_package_in_container.sh + - sudo chmod -R 755 "/var/lib/lxc" + - echo "Preparing RPM packaging contents for upload" && sudo -E .travis/package_management/prepare_packages.sh + after_failure: post_message "TRAVIS_MESSAGE" "Failed to build RPM for ${BUILD_STRING}.${BUILD_ARCH}" + before_deploy: + - .travis/package_management/yank_stale_pkg.sh "${PACKAGES_DIRECTORY}" "${BUILD_STRING}" || echo "No stale RPM found" + deploy: + - provider: packagecloud + repository: "${DEPLOY_REPO}" + username: "${PACKAGING_USER}" + token: "${PKG_CLOUD_TOKEN}" + dist: "${BUILD_STRING}" + local_dir: "${PACKAGES_DIRECTORY}" + skip_cleanup: true + on: + # Only deploy on ${USER}/netdata, master branch, when packages directory is created + repo: ${TRAVIS_REPO_SLUG} + branch: "master" + condition: -d "${PACKAGES_DIRECTORY}" + after_deploy: + - if [ -n "${BUILDER_NAME}" ]; then rm -rf /home/${BUILDER_NAME}/* && echo "Cleared /home/${BUILDER_NAME} directory" || echo "Failed to clean /home/${BUILDER_NAME} directory"; fi; + - if [ -d "${PACKAGES_DIRECTORY}" ]; then rm -rf "${PACKAGES_DIRECTORY}"; fi; + + name: "Build & Publish RPM package for Enterprise Linux 7" + <<: *RPM_TEMPLATE + if: commit_message =~ /\[Package (amd64|arm64) RPM( Enterprise Linux)?\]/ + env: + - BUILDER_NAME="builder" BUILD_DISTRO="centos" BUILD_RELEASE="7" BUILD_STRING="el/7" + - PACKAGE_TYPE="rpm" REPO_TOOL="yum" + - ALLOW_SOFT_FAILURE_HERE=true + + - name: "Build & Publish RPM package for Fedora 32" + <<: *RPM_TEMPLATE + if: commit_message =~ /\[Package (amd64|arm64) RPM( Fedora)?\]/ + env: + - BUILDER_NAME="builder" BUILD_DISTRO="fedora" BUILD_RELEASE="32" BUILD_STRING="fedora/32" + - PACKAGE_TYPE="rpm" REPO_TOOL="dnf" + - ALLOW_SOFT_FAILURE_HERE=true + + - name: "Build & Publish RPM package for Fedora 33" + <<: *RPM_TEMPLATE + if: commit_message =~ /\[Package (amd64|arm64) RPM( Fedora)?\]/ + env: + - BUILDER_NAME="builder" BUILD_DISTRO="fedora" BUILD_RELEASE="33" BUILD_STRING="fedora/33" + - PACKAGE_TYPE="rpm" REPO_TOOL="dnf" + - ALLOW_SOFT_FAILURE_HERE=true + + - name: "Build & Publish RPM package for openSUSE 15.2" + <<: *RPM_TEMPLATE + if: commit_message =~ /\[Package (amd64|arm64) RPM( openSUSE)?\]/ + env: + - BUILDER_NAME="builder" BUILD_DISTRO="opensuse" BUILD_RELEASE="15.2" BUILD_STRING="opensuse/15.2" + - PACKAGE_TYPE="rpm" REPO_TOOL="zypper" + - ALLOW_SOFT_FAILURE_HERE=true + # ###### End of packaging workflow section ###### # + # ############################################### # + + + # We only publish if a TAG has been set during packaging + - stage: Publish for release + name: Create release draft + git: + depth: false + env: + - RELEASE_CHANNEL=stable + before_script: post_message "TRAVIS_MESSAGE" "Drafting release on github" "${NOTIF_CHANNEL}" + script: + - echo "GIT Branch:" && git branch + - echo "Last commit:" && git log -1 + - echo "GIT Describe:" && git describe + - echo "packaging/version:" && cat packaging/version + - echo "Generating release artifacts" && .travis/create_artifacts.sh # Could/should be a common storage to put this and share between jobs + - .travis/draft_release.sh + after_failure: post_message "TRAVIS_MESSAGE" "<!here> Draft release submission failed" + + - name: Trigger Docker image build and publish + script: .travis/trigger_docker_build.sh "${GITHUB_TOKEN}" "${BUILD_VERSION}" + after_failure: post_message "TRAVIS_MESSAGE" "<!here> Failed to trigger docker build during release" "${NOTIF_CHANNEL}" + + - stage: Trigger deb and rpm package build (release) + name: Trigger deb and rpm package build + script: .travis/trigger_package_generation.sh + after_failure: post_message "TRAVIS_MESSAGE" "<!here> Failed to trigger deb and rpm package build during release" "${NOTIF_CHANNEL}" + + + # This is the nightly pre-execution step (Jobs, preparatory steps for nightly, etc) + - stage: Nightly operations + name: Kickstart files integrity testing (extended) + script: ./tests/installer/checksums.sh + + # This is generating the changelog for nightly release and publish it + - name: Generate nightly changelog + script: + - ".travis/nightlies.sh" + - ".travis/check_changelog_last_modification.sh" + after_failure: post_message "TRAVIS_MESSAGE" "<!here> Nightly changelog generation failed" + git: + depth: false + + - name: Clean up package cloud nightly repository from old versions + script: + - DEPLOY_REPO="netdata-edge" .travis/package_management/old_package_purging.sh + - DEPLOY_REPO="netdata-devel" .travis/package_management/old_package_purging.sh + + # This is the nightly execution step + # + - stage: Nightly release + _template: &NIGHTLY_TEMPLATE + git: + depth: false + script: + - echo "GIT Branch:" && git branch + - echo "Last commit:" && git log -1 + - echo "GIT Describe:" && git describe + - echo "packaging/version:" && cat packaging/version + - "sudo echo '{\"experimental\": true}' > /etc/docker/daemon.json && sudo systemctl restart docker" + - packaging/docker/check_login.sh + && tick packaging/docker/build.sh + && packaging/docker/publish.sh + after_failure: post_message "TRAVIS_MESSAGE" "<!here> Nightly docker image publish failed" + + name: Create nightly release artifacts, publish to GCS + script: + - echo "GIT Branch:" && git branch + - echo "Last commit:" && git log -1 + - echo "GIT Describe:" && git describe + - echo "packaging/version:" && cat packaging/version + - .travis/create_artifacts.sh + after_failure: post_message "TRAVIS_MESSAGE" "<!here> Nightly artifacts generation failed" + git: + depth: false + before_deploy: + echo "Preparing creds under ${TRAVIS_REPO_SLUG}"; + if [ "${TRAVIS_REPO_SLUG}" == "netdata/netdata" ]; then + openssl aes-256-cbc -K $encrypted_8daf19481253_key -iv $encrypted_8daf19481253_iv -in .travis/gcs-credentials.json.enc -out .travis/gcs-credentials.json -d; + else + echo "Beta deployment stage in progress"; + openssl aes-256-cbc -K $encrypted_8daf19481253_key -iv $encrypted_8daf19481253_iv -in .travis/gcs-credentials.json.enc -out .travis/gcs-credentials.json -d; + fi; + deploy: + # Beta storage, used for testing purposes + - provider: gcs + edge: + branch: gcs-ng + project_id: netdata-storage + credentials: .travis/gcs-credentials.json + bucket: "netdata-dev-nightlies" + skip_cleanup: true + local_dir: "artifacts" + on: + # Only deploy on netdata/netdata, master branch, when artifacts directory is created + repo: ${TRAVIS_REPO_SLUG} + branch: master + condition: -d "artifacts" && ${TRAVIS_REPO_SLUG} != "netdata/netdata" + + # Production storage + - provider: gcs + edge: + branch: gcs-ng + project_id: netdata-storage + credentials: .travis/gcs-credentials.json + bucket: "netdata-nightlies" + skip_cleanup: true + local_dir: "artifacts" + on: + # Only deploy on netdata/netdata, master branch, when artifacts directory is created + repo: netdata/netdata + branch: master + condition: -d "artifacts" && ${TRAVIS_REPO_SLUG} = "netdata/netdata" + after_deploy: rm -f .travis/gcs-credentials.json + + - name: Trigger Docker image build and publish + script: .travis/trigger_docker_build.sh "${GITHUB_TOKEN}" "nightly" + after_failure: post_message "TRAVIS_MESSAGE" "<!here> Failed to trigger docker build during nightly release" "${NOTIF_CHANNEL}" + + - stage: Trigger deb and rpm package build (nightly release) + name: Trigger deb and rpm package build + script: .travis/trigger_package_generation.sh "[Build latest]" + after_failure: post_message "TRAVIS_MESSAGE" "<!here> Failed to trigger deb and rpm package build during nightly release" "${NOTIF_CHANNEL}" |