diff options
Diffstat (limited to '')
| -rw-r--r-- | .travis.yml | 68 | ||||
| -rw-r--r-- | .travis/README.md | 149 | ||||
| -rwxr-xr-x | .travis/check_changelog_last_modification.sh | 38 | ||||
| -rwxr-xr-x | .travis/create_artifacts.sh | 77 | ||||
| -rwxr-xr-x | .travis/create_changelog.sh | 47 | ||||
| -rw-r--r-- | .travis/current_build_status | 1 | ||||
| -rwxr-xr-x | .travis/draft_release.sh | 65 | ||||
| -rw-r--r-- | .travis/gcs-credentials.json.enc | bin | 0 -> 2320 bytes | |||
| -rwxr-xr-x | .travis/generate_changelog_and_tag_release.sh | 64 | ||||
| -rwxr-xr-x | .travis/generate_changelog_for_nightlies.sh | 50 | ||||
| -rwxr-xr-x | .travis/nightlies.sh | 51 | ||||
| -rwxr-xr-x | .travis/tagger.sh | 61 | ||||
| -rwxr-xr-x | .travis/trigger_artifact_build.sh | 20 | ||||
| -rwxr-xr-x | .travis/trigger_docker_build.sh | 19 | ||||
| -rwxr-xr-x | .travis/trigger_package_build.sh | 20 | ||||
| -rw-r--r-- | .travis/utils.sh | 29 |
16 files changed, 759 insertions, 0 deletions
diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..e729815 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,68 @@ +--- +dist: focal +language: c + +addons: + apt: + packages: ['moreutils'] + +env: + global: + - RELEASE_CHANNEL=nightly + +before_install: + - exec > >(ts -s '%H:%M:%.S ') 2>&1 + - source .travis/utils.sh + +# Install dependencies for all, once +# +install: + - sudo apt-get install -y libuv1-dev liblz4-dev libcap2-bin zlib1g-dev uuid-dev fakeroot libipmimonitoring-dev libmnl-dev libnetfilter-acct-dev gnupg python3-pip + - sudo pip3 install git-semver==0.3.2 # 11/Sep/2019: git-semver tip was broken, so we had to force last good run of it + - source tests/installer/slack.sh + - export NOTIF_CHANNEL="automation-beta" + - if [ "${TRAVIS_REPO_SLUG}" = "netdata/netdata" ]; then export NOTIF_CHANNEL="automation"; fi; + - export BUILD_VERSION="$(cat packaging/version | cut -d'-' -f1)" + - export LATEST_RELEASE_VERSION="$(cat packaging/version | cut -d'-' -f1)" + - export LATEST_RELEASE_DATE="$(git log -1 --format=%aD "${LATEST_RELEASE_VERSION}" | cat)" + - if [[ "${TRAVIS_COMMIT_MESSAGE}" = *"[Build latest]"* ]]; then export BUILD_VERSION="$(cat packaging/version | cut -d'-' -f1,2 | sed -e 's/-/./g').latest"; fi; + - export DEPLOY_REPO="netdata" # Default production packaging repository + - export PACKAGING_USER="netdata" # Standard package cloud account + - if [[ "${TRAVIS_COMMIT_MESSAGE}" = *"[Build latest]"* ]]; then export DEPLOY_REPO="netdata-edge"; fi; + - export PACKAGE_CLOUD_RETENTION_DAYS=30 + - if [ ! "${TRAVIS_REPO_SLUG}" = "netdata/netdata" ]; then export DEPLOY_REPO="netdata-devel"; fi; + # These are release-related artifacts and have to be evaluated before we start doing conditional checks inside stages + - source ".travis/tagger.sh" + - export GIT_TAG="$(git tag --points-at)" + - git submodule update --init --recursive + + +# Setup notification system +# +notifications: + webhooks: + urls: + - https://app.fossa.io/hooks/travisci + +# Define the stage sequence and conditionals +# +stages: + # Mandatory runs, we always want these executed + - name: Build process + if: commit_message =~ /^((?!\[Package (amd64|arm64|i386) (DEB|RPM)( .*)?\]).)*$/ + + +# Define stage implementation details +# +jobs: + # This is a hook to help us introduce "soft" errors on our process + allow_failures: + - env: ALLOW_SOFT_FAILURE_HERE=true + include: + # Ensure netdata code builds successfully + - stage: Build process + + name: Standard netdata build + script: fakeroot ./netdata-installer.sh --install $HOME --dont-wait --dont-start-it --enable-plugin-nfacct --enable-plugin-freeipmi --disable-lto + env: CFLAGS='-O1 -Wall -Wextra -Wformat-signedness -fstack-protector-all -fno-common -DNETDATA_INTERNAL_CHECKS=1 -D_FORTIFY_SOURCE=2 -DNETDATA_VERIFY_LOCKS=1' + after_failure: post_message "TRAVIS_MESSAGE" "<!here> standard netdata build is failing (Still dont know which one, will improve soon)" diff --git a/.travis/README.md b/.travis/README.md new file mode 100644 index 0000000..8927dd4 --- /dev/null +++ b/.travis/README.md @@ -0,0 +1,149 @@ +<!-- +--- +title: "Description of CI build configuration" +custom_edit_url: https://github.com/netdata/netdata/edit/master/.travis/README.md +--- +--> + +# Description of CI build configuration + +## Variables needed by travis + +- GITHUB_TOKEN - GitHub token with push access to repository +- DOCKER_USERNAME - Username (netdatabot) with write access to docker hub repository +- DOCKER_PWD - Password to docker hub +- encrypted_8daf19481253_key - key needed by openssl to decrypt GCS credentials file +- encrypted_8daf19481253_iv - IV needed by openssl to decrypt GCS credentials file +- COVERITY_SCAN_TOKEN - Token to allow coverity test analysis uploads +- SLACK_USERNAME - This is required for the slack notifications triggered by travis pipeline +- SLACK_CHANNEL - This is the channel that Travis will be posting messages +- SLACK_NOTIFY_WEBHOOK_URL - This is the incoming URL webhook as provided by slack integration. Visit Apps integration in slack to generate the required hook +- SLACK_BOT_NAME - This is the name your bot will appear with on slack + +## CI workflow details +Our CI pipeline is designed to help us identify and mitigate risks at all stages of implementation. +To accommodate this need, we used [Travis CI](http://www.travis-ci.com) as our CI/CD tool. +Our main areas of concern are: +1) Only push code that is working. That means fail fast so that we can improve before we reach the public + +2) Reduce the time to market to minimum, by streamlining the release process. + That means a lot of testing, a lot of consistency checks, a lot of validations + +3) Generated artifacts consistency. We should not allow broken software to reach the public. + When this happens, it's embarrassing and we struggle to eliminate it. + +4) We are an innovative company, so we love to automate :) + + +Having said that, here's a brief introduction to Netdata's improved CI/CD pipeline with Travis. +Our CI/CD lifecycle contains three different execution entry points: +1) A user opens a pull request to netdata/master: Travis will run a pipeline on the branch under that PR +2) A merge or commit happens on netdata/master. This will trigger travis to run, but we have two distinct cases in this scenario: + a) A user merges a pull request to netdata/master: Travis will run on master, after the merge. + b) A user runs a commit/merge with a special keyword (mentioned later). + This triggers a release for either minor, major or release candidate versions, depending the keyword +3) A scheduled job runs on master once per day: Travis will run on master at the scheduled interval + +To accommodate all three entry points our CI/CD workflow has a set of steps that run on all three entry points. +Once all these steps are successful, then our pipeline executes another subset of steps for entry points 2 and 3. +In travis terms the "steps" are "Stages" and within each stage we execute a set of activities called "jobs" in travis. + +### Always run: Stages that running on all three execution entry points + +## Code quality, linting, syntax, code style +At this early stage we iterate through a set of basic quality control checks: +- Shell checking: Run linters for our various BASH scripts +- Checksum validators: Run validators to ensure our installers and documentation are in sync +- Dashboard validator: We provide a pre-generated dashboard.js script file that we need to make sure its up to date. We validate that. + +## Build process +At this stage, basically, we build :-) +We do a baseline check of our build artifacts to guarantee they are not broken +Briefly our activities include: +- Verify docker builds successfully +- Run the standard Netdata installer, to make sure we build & run properly +- Do the same through 'make dist', as this is our stable channel for our kickstart files + +## Artifacts validation +At this point we know our software is building, we need to go through the a set of checks, to guarantee +that our product meets certain expectations. At the current stage, we are focusing on basic capabilities +like installing in different distributions, running the full lifecycle of install-run-update-install and so on. +We are still working on enriching this with more and more use cases, to get us closer to achieving full stability of our software. +Briefly we currently evaluate the following activities: +- Basic software unit testing (only run when changes happen that require it) +- Non containerized build and install on ubuntu 14.04 +- Non containerized build and install on ubuntu 18.04 +- Running the full Netdata lifecycle (install, update, uninstall) on ubuntu 18.04 +- Build and install on CentOS 7 +(More to come) + +### Nightly operations: Stages that run daily under cronjob +The nightly stages are related to the daily nightly activities, that produce our daily latest releases. +We also maintain a couple of cronjobs that run during the night to provide us with deeper insights, +like for example coverity scanning or extended kickstart checksum checks + +## Nightly operations +At this stage we run scheduled jobs and execute the nightly changelog generator, coverity scans, +labeler for our issues and extended kickstart files checksum validations. + +## Nightly release +During this stage we are building and publishing latest docker images, prepare the nightly artifacts +and deploy them (the artifacts) to our google cloud service provider. + + +### Publishing +Publishing is responsible for executing the major/minor/patch releases and is separated +in two stages: packaging preparation process and publishing. + +## Packaging for release +During packaging we are preparing the release changelog information and run the labeler. + +## Publish for release +The publishing stage is the most complex part in publishing. This is the stage were we generate and publish docker images, +prepare the release artifacts and get ready with the release draft. + +### Package Management workflows +As part of our goal to provide the best support to our customers, we have created a set of CI workflows to automatically produce +DEB and RPM for multiple distributions. These workflows are implemented under the templated stages '_DEB_TEMPLATE' and '_RPM_TEMPLATE'. +We currently plan to actively support the following Operating Systems, with a plan to further expand this list following our users needs. + +### Operating systems supported +The following distributions are supported +- Debian versions + - Buster (TBD - not released yet, check [debian releases](https://www.debian.org/releases/) for details) + - Stretch + - Jessie + - Wheezy + +- Ubuntu versions + - Disco + - Cosmic + - Bionic + - artful + +- Enterprise Linux versions (Covers Red Hat, CentOS, and Amazon Linux with version 6) + - Version 8 (TBD) + - Version 7 + - Version 6 + +- Fedora versions + - Version 31 (TBD) + - Version 30 + - Version 29 + - Version 28 + +- openSUSE versions + - 15.1 + - 15.0 + +- Gentoo distributions + - TBD + +### Architectures supported +We plan to support amd64, x86 and arm64 architectures. As of June 2019 only amd64 and x86 will become available, as we are still working on solving issues with the architecture. + +The Package deployment can be triggered manually by executing an empty commit with the following message pattern: `[Package PACKAGE_TYPE PACKAGE_ARCH] DESCRIBE_THE_REASONING_HERE`. +Travis Yaml configuration allows the user to combine package type and architecture as necessary to regenerate the current stable release (For example tag v1.15.0 as of 4th of May 2019) +Sample patterns to trigger building of packages for all amd64 supported architecture: +- '[Package amd64 RPM]': Build & publish all amd64 available RPM packages +- '[Package amd64 DEB]': Build & publish all amd64 available DEB packages diff --git a/.travis/check_changelog_last_modification.sh b/.travis/check_changelog_last_modification.sh new file mode 100755 index 0000000..ae0da87 --- /dev/null +++ b/.travis/check_changelog_last_modification.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash +# +# This scriptlet validates nightlies age and notifies is if it gets too old +# +# Copyright: SPDX-License-Identifier: GPL-3.0-or-later +# +# Author : Pavlos Emm. Katsoulakis (paul@netdata.cloud) + +set -e + +# If we are not in netdata git repo, at the top level directory, fail +TOP_LEVEL=$(basename "$(git rev-parse --show-toplevel)") +CWD=$(git rev-parse --show-cdup || echo "") +if [ -n "${CWD}" ] || [ ! "${TOP_LEVEL}" == "netdata" ]; then + echo "Run as .travis/$(basename "$0") from top level directory of netdata git repository" + echo "Changelog age checker exited abnormally" + exit 1 +fi + +source tests/installer/slack.sh || echo "I could not load slack library" + +LAST_MODIFICATION="$(git log -1 --pretty="format:%at" CHANGELOG.md)" +CURRENT_TIME="$(date +"%s")" +TWO_DAYS_IN_SECONDS=172800 + +DIFF=$((CURRENT_TIME - LAST_MODIFICATION)) + +echo "Checking CHANGELOG.md last modification time on GIT.." +echo "CHANGELOG.md timestamp: ${LAST_MODIFICATION}" +echo "Current timestamp: ${CURRENT_TIME}" +echo "Diff: ${DIFF}" + +if [ ${DIFF} -gt ${TWO_DAYS_IN_SECONDS} ]; then + echo "CHANGELOG.md is more than two days old!" + post_message "TRAVIS_MESSAGE" "Hi <!here>, CHANGELOG.md was found more than two days old (Diff: ${DIFF} seconds)" "${NOTIF_CHANNEL}" +else + echo "CHANGELOG.md is less than two days old, fine" +fi diff --git a/.travis/create_artifacts.sh b/.travis/create_artifacts.sh new file mode 100755 index 0000000..2742891 --- /dev/null +++ b/.travis/create_artifacts.sh @@ -0,0 +1,77 @@ +#!/usr/bin/env bash +# +# Artifacts creation script. +# This script generates two things: +# 1) The static binary that can run on all linux distros (built-in dependencies etc) +# 2) The distribution source tarball +# +# Copyright: SPDX-License-Identifier: GPL-3.0-or-later +# +# Author: Paul Emm. Katsoulakis <paul@netdata.cloud> +# +# shellcheck disable=SC2230 + +set -e + +# If we are not in netdata git repo, at the top level directory, fail +TOP_LEVEL=$(basename "$(git rev-parse --show-toplevel)") +CWD=$(git rev-parse --show-cdup || echo "") +if [ -n "${CWD}" ] || [ ! "${TOP_LEVEL}" == "netdata" ]; then + echo "Run as .travis/$(basename "$0") from top level directory of netdata git repository" + exit 1 +fi + +if [ ! "${TRAVIS_REPO_SLUG}" == "netdata/netdata" ]; then + echo "Beta mode on ${TRAVIS_REPO_SLUG}, not running anything here" + exit 0 +fi + +echo "--- Initialize git configuration ---" +git checkout "${1-master}" +git pull + +if [ "${RELEASE_CHANNEL}" == stable ]; then + echo "--- Set default release channel to stable ---" + sed -i 's/^RELEASE_CHANNEL="nightly" *#/RELEASE_CHANNEL="stable" #/' \ + netdata-installer.sh \ + packaging/makeself/install-or-update.sh +fi + +# Everything from this directory will be uploaded to GCS +mkdir -p artifacts +BASENAME="netdata-$(git describe)" + +# Make sure stdout is in blocking mode. If we don't, then conda create will barf during downloads. +# See https://github.com/travis-ci/travis-ci/issues/4704#issuecomment-348435959 for details. +python -c 'import os,sys,fcntl; flags = fcntl.fcntl(sys.stdout, fcntl.F_GETFL); fcntl.fcntl(sys.stdout, fcntl.F_SETFL, flags&~os.O_NONBLOCK);' +echo "--- Create tarball ---" +command -v git > /dev/null && [ -d .git ] && git clean -d -f +autoreconf -ivf +./configure --prefix=/usr --sysconfdir=/etc --localstatedir=/var --libexecdir=/usr/libexec --with-zlib --with-math --with-user=netdata CFLAGS=-O2 +make dist +mv "${BASENAME}.tar.gz" artifacts/ + +echo "--- Create self-extractor ---" +sxarches="x86_64 armv7l aarch64" +for arch in ${sxarches}; do + git clean -d -f + rm -rf packating/makeself/tmp + ./packaging/makeself/build-static.sh ${arch} +done + +# Needed for GCS +echo "--- Copy artifacts to separate directory ---" +#shellcheck disable=SC2164 +cp packaging/version artifacts/latest-version.txt +cd artifacts +ln -s "${BASENAME}.tar.gz" netdata-latest.tar.gz + +for arch in ${sxarches}; do + ln -s "netdata-${arch}-$(git describe).gz.run" netdata-${arch}-latest.gz.run +done + +ln -s "${BASENAME}.gz.run" netdata-latest.gz.run + +sha256sum -b ./* > "sha256sums.txt" +echo "checksums:" +cat sha256sums.txt diff --git a/.travis/create_changelog.sh b/.travis/create_changelog.sh new file mode 100755 index 0000000..83584aa --- /dev/null +++ b/.travis/create_changelog.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash +# +# Copyright: SPDX-License-Identifier: GPL-3.0-or-later +# +# Author : Pavlos Emm. Katsoulakis (paul@netdata.cloud) +set -e + +# If we are not in netdata git repo, at the top level directory, fail +TOP_LEVEL=$(basename "$(git rev-parse --show-toplevel)") +CWD=$(git rev-parse --show-cdup || echo "") +if [ -n "$CWD" ] || [ ! "${TOP_LEVEL}" == "netdata" ]; then + echo "Run as .travis/$(basename "$0") from top level directory of netdata git repository" + echo "Changelog creation aborted" + exit 1 +fi + +ORGANIZATION=$(echo "$TRAVIS_REPO_SLUG" | awk -F '/' '{print $1}') +PROJECT=$(echo "$TRAVIS_REPO_SLUG" | awk -F '/' '{print $2}') +GIT_MAIL=${GIT_MAIL:-"bot@netdata.cloud"} +GIT_USER=${GIT_USER:-"netdatabot"} + +if [ -z ${GIT_TAG+x} ]; then + OPTS="" +else + OPTS="--future-release ${GIT_TAG}" +fi + +if [ ! "${TRAVIS_REPO_SLUG}" == "netdata/netdata" ]; then + echo "Beta mode on ${TRAVIS_REPO_SLUG}, nothing else to do here" + exit 0 +fi + +echo "--- Creating changelog ---" +git checkout master +git pull + +docker login -u "${DOCKER_USERNAME}" -p "${DOCKER_PWD}" +docker run -it -v "$(pwd)":/project markmandel/github-changelog-generator:latest \ + --user "${ORGANIZATION}" \ + --project "${PROJECT}" \ + --token "${GITHUB_TOKEN}" \ + --since-tag "v1.10.0" \ + --unreleased-label "**Next release**" \ + --no-issues \ + --exclude-labels "stale,duplicate,question,invalid,wontfix,discussion,no changelog" \ + --max-issues 500 \ + --bug-labels IGNOREBUGS ${OPTS} diff --git a/.travis/current_build_status b/.travis/current_build_status new file mode 100644 index 0000000..11a6d0a --- /dev/null +++ b/.travis/current_build_status @@ -0,0 +1 @@ +changes-#18220 diff --git a/.travis/draft_release.sh b/.travis/draft_release.sh new file mode 100755 index 0000000..ddc0f9a --- /dev/null +++ b/.travis/draft_release.sh @@ -0,0 +1,65 @@ +#!/bin/bash +# +# Draft release generator. +# This utility is responsible for submitting a draft release to github repo +# It is agnostic of other processes, when executed it will draft a release, +# based on the most recent reachable tag. +# +# Requirements: +# - GITHUB_TOKEN variable set with GitHub token. Access level: repo.public_repo +# - artifacts directory in place +# - The directory is created by create_artifacts.sh mechanism +# - The artifacts need to be created with the same tag, obviously +# +# Copyright: SPDX-License-Identifier: GPL-3.0-or-later +# +# Author: Pavlos Emm. Katsoulakis <paul@netdata.cloud> + +set -e + +if [ ! -f .gitignore ]; then + echo "Run as ./travis/$(basename "$0") from top level directory of git repository" + exit 1 +fi + +echo "--- Initialize git configuration ---" +git checkout master +git pull + + +if [[ $(git describe) =~ -rc* ]]; then + echo "This is a release candidate tag, we do not generate a release draft" + exit 0 +fi + +# Load the tag, if any +GIT_TAG=$(git describe) + +if [ ! "${TRAVIS_REPO_SLUG}" == "netdata/netdata" ]; then + echo "Beta mode on ${TRAVIS_REPO_SLUG}, i was about to run for release (${GIT_TAG}), but i am emulating, so bye" + exit 0 +fi; + +echo "---- CREATING RELEASE DRAFT WITH ASSETS -----" +# Download hub +HUB_VERSION=${HUB_VERSION:-"2.5.1"} +wget "https://github.com/github/hub/releases/download/v${HUB_VERSION}/hub-linux-amd64-${HUB_VERSION}.tgz" -O "/tmp/hub-linux-amd64-${HUB_VERSION}.tgz" +tar -C /tmp -xvf "/tmp/hub-linux-amd64-${HUB_VERSION}.tgz" +export PATH=$PATH:"/tmp/hub-linux-amd64-${HUB_VERSION}/bin" + +# Create a release draft +if [ -z ${GIT_TAG+x} ]; then + echo "Variable GIT_TAG is not set. Something went terribly wrong! Exiting." + exit 1 +fi +if [ "${GIT_TAG}" != "$(git tag --points-at)" ]; then + echo "ERROR! Current commit is not tagged. Stopping release creation." + exit 1 +fi +until hub release create --draft \ + -a "artifacts/netdata-${GIT_TAG}.tar.gz" \ + -a "artifacts/netdata-${GIT_TAG}.gz.run" \ + -a "artifacts/sha256sums.txt" \ + -m "${GIT_TAG}" "${GIT_TAG}"; do + sleep 5 +done diff --git a/.travis/gcs-credentials.json.enc b/.travis/gcs-credentials.json.enc Binary files differnew file mode 100644 index 0000000..5d1e7b2 --- /dev/null +++ b/.travis/gcs-credentials.json.enc diff --git a/.travis/generate_changelog_and_tag_release.sh b/.travis/generate_changelog_and_tag_release.sh new file mode 100755 index 0000000..bf5555b --- /dev/null +++ b/.travis/generate_changelog_and_tag_release.sh @@ -0,0 +1,64 @@ +#!/bin/bash +# +# Script to automatically do a couple of things: +# - generate a new tag according to semver (https://semver.org/) +# - generate CHANGELOG.md by using https://github.com/skywinder/github-changelog-generator +# +# Tags are generated by searching for a keyword in last commit message. Keywords are: +# - [patch] or [fix] to bump patch number +# - [minor], [feature] or [feat] to bump minor number +# - [major] or [breaking change] to bump major number +# All keywords MUST be surrounded with square braces. +# +# Script uses git mechanisms for locking, so it can be used in parallel builds +# +# Requirements: +# - GITHUB_TOKEN variable set with GitHub token. Access level: repo.public_repo +# - docker +# +# This is a modified version of: +# https://github.com/paulfantom/travis-helper/blob/master/releasing/releaser.sh +# +# Copyright: SPDX-License-Identifier: GPL-3.0-or-later +# +# Author: Pavlos Emm. Katsoulakis <paul@netdata.cloud> +# Author: Pawel Krupa (@paulfantom) +set -e + +if [ ! -f .gitignore ]; then + echo "Run as ./travis/$(basename "$0") from top level directory of git repository" + exit 1 +fi + +echo "--- Changelog generator script starting ---" +# If we dont have a produced TAG there is nothing to do, so bail out happy +if [ -z "${GIT_TAG}" ]; then + echo "GIT_TAG is empty, that is not suppose to happen (Value: $GIT_TAG)" + exit 1 +fi + +if [ ! "${TRAVIS_REPO_SLUG}" == "netdata/netdata" ]; then + echo "Beta mode on ${TRAVIS_REPO_SLUG}, nothing to do on the changelog generator and tagging script for (${GIT_TAG}), bye" + exit 0 +fi + +echo "--- Initialize git configuration ---" +export GIT_MAIL="bot@netdata.cloud" +export GIT_USER="netdatabot" +git checkout master +git pull + +echo "---- UPDATE VERSION FILE ----" +echo "$GIT_TAG" >packaging/version +git add packaging/version + +echo "---- Create CHANGELOG -----" +./.travis/create_changelog.sh +git add CHANGELOG.md + +echo "---- COMMIT AND PUSH CHANGES ----" +git commit -m "[ci skip] release $GIT_TAG" --author "${GIT_USER} <${GIT_MAIL}>" +git tag "$GIT_TAG" -a -m "Automatic tag generation for travis build no. $TRAVIS_BUILD_NUMBER" +git push "https://${GITHUB_TOKEN}:@$(git config --get remote.origin.url | sed -e 's/^https:\/\///')" +git push "https://${GITHUB_TOKEN}:@$(git config --get remote.origin.url | sed -e 's/^https:\/\///')" --tags +# After those operations output of command `git describe` should be identical with a value of GIT_TAG diff --git a/.travis/generate_changelog_for_nightlies.sh b/.travis/generate_changelog_for_nightlies.sh new file mode 100755 index 0000000..59173af --- /dev/null +++ b/.travis/generate_changelog_for_nightlies.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash +# +# Changelog generation scriptlet, for nightlies +# +# Copyright: SPDX-License-Identifier: GPL-3.0-or-later +# +# Author : Pawel Krupa (paulfantom) +# Author : Pavlos Emm. Katsoulakis (paul@netdata.cloud) +set -e + +# If we are not in netdata git repo, at the top level directory, fail +TOP_LEVEL=$(basename "$(git rev-parse --show-toplevel)") +CWD=$(git rev-parse --show-cdup || echo "") +if [ -n "$CWD" ] || [ ! "${TOP_LEVEL}" == "netdata" ]; then + echo "Run as .travis/$(basename "$0") from top level directory of netdata git repository" + echo "Changelog generation process aborted" + exit 1 +fi + +LAST_TAG="$1" +COMMITS_SINCE_RELEASE="$2" +NEW_VERSION="${LAST_TAG}-$((COMMITS_SINCE_RELEASE + 1))-nightly" +GIT_MAIL=${GIT_MAIL:-"bot@netdata.cloud"} +GIT_USER=${GIT_USER:-"netdatabot"} +PUSH_URL=$(git config --get remote.origin.url | sed -e 's/^https:\/\///') +FAIL=0 + +if [ ! "${TRAVIS_REPO_SLUG}" == "netdata/netdata" ]; then + echo "Beta mode on ${TRAVIS_REPO_SLUG}, nothing else to do here" + exit 0 +fi + +echo "Running changelog creation mechanism" +.travis/create_changelog.sh + +echo "Changelog created! Adding packaging/version(${NEW_VERSION}) and CHANGELOG.md to the repository" +echo "${NEW_VERSION}" > packaging/version +git add packaging/version && echo "1) Added packaging/version to repository" || FAIL=1 +git add CHANGELOG.md && echo "2) Added changelog file to repository" || FAIL=1 +git commit -m '[ci skip] create nightly packages and update changelog' --author "${GIT_USER} <${GIT_MAIL}>" && echo "3) Committed changes to repository" || FAIL=1 +git push "https://${GITHUB_TOKEN}:@${PUSH_URL}" && echo "4) Pushed changes to remote ${PUSH_URL}" || FAIL=1 + +# In case of a failure, wrap it up and bail out cleanly +if [ $FAIL -eq 1 ]; then + git clean -xfd + echo "Changelog generation failed during github UPDATE!" + exit 1 +fi + +echo "Changelog generation completed successfully!" diff --git a/.travis/nightlies.sh b/.travis/nightlies.sh new file mode 100755 index 0000000..7240a78 --- /dev/null +++ b/.travis/nightlies.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash +# +# This is the nightly changelog generation script +# It is responsible for two major activities: +# 1) Update packaging/version with the current nightly version +# 2) Generate the changelog for the mentioned version +# +# Copyright: SPDX-License-Identifier: GPL-3.0-or-later +# +# Author : Pawel Krupa (paulfantom) +# Author : Pavlos Emm. Katsoulakis (paul@netdata.cloud) +set -e +FAIL=0 + +source tests/installer/slack.sh || echo "Could not load slack library" + +# If we are not in netdata git repo, at the top level directory, fail +TOP_LEVEL=$(basename "$(git rev-parse --show-toplevel)") +CWD=$(git rev-parse --show-cdup || echo "") +if [ -n "${CWD}" ] || [ ! "${TOP_LEVEL}" == "netdata" ]; then + echo "Run as .travis/$(basename "$0") from top level directory of netdata git repository" + echo "Changelog generation process aborted" + exit 1 +fi + +LAST_TAG=$(git describe --abbrev=0 --tags) +COMMITS_SINCE_RELEASE=$(git rev-list "${LAST_TAG}"..HEAD --count) +PREVIOUS_NIGHTLY_COUNT="$(rev <packaging/version | cut -d- -f 2 | rev)" + +# If no commits since release, just stop +if [ "${COMMITS_SINCE_RELEASE}" == "${PREVIOUS_NIGHTLY_COUNT}" ]; then + echo "No changes since last nightly release, nothing else to do" + exit 0 +fi + +if [ ! "${TRAVIS_REPO_SLUG}" == "netdata/netdata" ]; then + echo "Beta mode -- nothing to do for ${TRAVIS_REPO_SLUG} on the nightlies script, bye" + exit 0 +fi + +echo "--- Running Changelog generation ---" +echo "We got $COMMITS_SINCE_RELEASE changes since $LAST_TAG, re-generating changelog" +NIGHTLIES_CHANGELOG_FAILED=0 +.travis/generate_changelog_for_nightlies.sh "${LAST_TAG}" "${COMMITS_SINCE_RELEASE}" || NIGHTLIES_CHANGELOG_FAILED=1 + +if [ ${NIGHTLIES_CHANGELOG_FAILED} -eq 1 ]; then + echo "Changelog generation has failed, this is a soft error, process continues" + post_message "TRAVIS_MESSAGE" "Changelog generation job for nightlies failed, possibly due to github issues" "${NOTIF_CHANNEL}" || echo "Slack notification failed" +fi + +exit "${FAIL}" diff --git a/.travis/tagger.sh b/.travis/tagger.sh new file mode 100755 index 0000000..a775a82 --- /dev/null +++ b/.travis/tagger.sh @@ -0,0 +1,61 @@ +# #BASH library +# +# Tags are generated by searching for a keyword in last commit message. Keywords are: +# - [patch] or [fix] to bump patch number +# - [minor], [feature] or [feat] to bump minor number +# - [major] or [breaking change] to bump major number +# All keywords MUST be surrounded with square braces. +# +# Requirements: +# - GITHUB_TOKEN variable set with GitHub token. Access level: repo.public_repo +# - git-semver python package (pip install git-semver) +# +# Original script is available at https://github.com/paulfantom/travis-helper/blob/master/releasing/releaser.sh +# +# Copyright: SPDX-License-Identifier: GPL-3.0-or-later +# +# Author : Pawel Krupa (paulfantom) +# Author : Pavlos Emm. Katsoulakis (paul@netdata.cloud) + +# Figure out what will be new release candidate tag based only on previous ones. +# This assumes that RELEASES are in format of "v0.1.2" and prereleases (RCs) are using "v0.1.2-rc0" +function set_tag_release_candidate() { + LAST_TAG=$(git semver) + echo "${0}: Last tag found is: ${LAST_TAG}" + + if [[ $LAST_TAG =~ -rc* ]]; then + VERSION=$(echo "$LAST_TAG" | cut -d'-' -f 1) + LAST_RC=$(echo "$LAST_TAG" | cut -d'c' -f 2) + RC=$((LAST_RC + 1)) + else + VERSION="$(git semver --next-minor)" + RC=0 + echo "${0}: Warning: Will set version to ${VERSION} (Last tag: ${LAST_TAG}) while tagged for release candidate generation" + fi + + GIT_TAG="v${VERSION}-rc${RC}" + echo "${0}: Generated a new tag, set to: (${GIT_TAG})" +} + +function set_tag_for_release() { + echo "${0}: Checking for tag existence" + if [ -z "${GIT_TAG}" ]; then + echo "${0}: No tag was found, generating a new tag" + git semver + + echo "${0}: Last commit message: ${TRAVIS_COMMIT_MESSAGE}" + + # Figure out next tag based on commit message + case "${TRAVIS_COMMIT_MESSAGE}" in + *"[netdata patch release]"*) GIT_TAG="v$(git semver --next-patch)" ;; + *"[netdata minor release]"*) GIT_TAG="v$(git semver --next-minor)" ;; + *"[netdata major release]"*) GIT_TAG="v$(git semver --next-major)" ;; + *"[netdata release candidate]"*) set_tag_release_candidate ;; + *) + echo "${0}: Keyword not detected. Nothing to set for GIT_TAG" + ;; + esac + else + echo "${0}: We seem to already have a GIT_TAG set to (${GIT_TAG})" + fi +} diff --git a/.travis/trigger_artifact_build.sh b/.travis/trigger_artifact_build.sh new file mode 100755 index 0000000..3ec5d02 --- /dev/null +++ b/.travis/trigger_artifact_build.sh @@ -0,0 +1,20 @@ +#!/bin/sh + +token="${1}" +version="${2}" +type="${3}" + +resp="$(curl -X POST \ + -H 'Accept: application/vnd.github.v3+json' \ + -H "Authorization: Bearer ${token}" \ + "https://api.github.com/repos/netdata/netdata/actions/workflows/build.yml/dispatches" \ + -d "{\"ref\": \"master\", \"inputs\": {\"version\": \"${version}\", \"type\": \"${type}\"}}")" + +if [ -z "${resp}" ]; then + echo "Successfully triggered release artifact build." + exit 0 +else + echo "Failed to trigger release artifact build. Output:" + echo "${resp}" + exit 1 +fi diff --git a/.travis/trigger_docker_build.sh b/.travis/trigger_docker_build.sh new file mode 100755 index 0000000..7a3dd50 --- /dev/null +++ b/.travis/trigger_docker_build.sh @@ -0,0 +1,19 @@ +#!/bin/sh + +token="${1}" +version="${2}" + +resp="$(curl -X POST \ + -H 'Accept: application/vnd.github.v3+json' \ + -H "Authorization: Bearer ${token}" \ + "https://api.github.com/repos/netdata/netdata/actions/workflows/docker.yml/dispatches" \ + -d "{\"ref\": \"master\", \"inputs\": {\"version\": \"${version}\"}}")" + +if [ -z "${resp}" ]; then + echo "Successfully triggered Docker image build." + exit 0 +else + echo "Failed to trigger Docker image build. Output:" + echo "${resp}" + exit 1 +fi diff --git a/.travis/trigger_package_build.sh b/.travis/trigger_package_build.sh new file mode 100755 index 0000000..d1ecd48 --- /dev/null +++ b/.travis/trigger_package_build.sh @@ -0,0 +1,20 @@ +#!/bin/sh + +token="${1}" +version="${2}" +pkgtype="${3}" + +resp="$(curl -X POST \ + -H 'Accept: application/vnd.github.v3+json' \ + -H "Authorization: Bearer ${token}" \ + "https://api.github.com/repos/netdata/netdata/actions/workflows/packaging.yml/dispatches" \ + -d "{\"ref\": \"master\", \"inputs\": {\"version\": \"${version}\", \"type\": \"${pkgtype}\"}}")" + +if [ -z "${resp}" ]; then + echo "Successfully triggered binary package builds." + exit 0 +else + echo "Failed to trigger binary package builds. Output:" + echo "${resp}" + exit 1 +fi diff --git a/.travis/utils.sh b/.travis/utils.sh new file mode 100644 index 0000000..d084a7a --- /dev/null +++ b/.travis/utils.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash + +# Prevent travis from timing out after 10 minutes of no output +tick() { + (while true; do sleep 300; echo; done) & + local PID=$! + disown + + "$@" + local RET=$? + + kill $PID + return $RET +} +export -f tick + +retry() { + local tries=$1 + shift + + local i=0 + while [ "$i" -lt "$tries" ]; do + "$@" && return 0 + sleep $((2**((i++)))) + done + + return 1 +} +export -f retry |