diff options
Diffstat (limited to '.travis')
| -rw-r--r-- | .travis/README.md | 91 | ||||
| -rwxr-xr-x | .travis/create_artifacts.sh | 36 | ||||
| -rw-r--r-- | .travis/gcs-credentials.json.enc | bin | 0 -> 2320 bytes | |||
| -rwxr-xr-x | .travis/generate_changelog.sh | 33 | ||||
| -rwxr-xr-x | .travis/labeler.sh | 97 | ||||
| -rwxr-xr-x | .travis/nightlies.sh | 45 | ||||
| -rwxr-xr-x | .travis/releaser.sh | 94 | ||||
| -rwxr-xr-x | .travis/tagger.sh | 59 |
8 files changed, 455 insertions, 0 deletions
diff --git a/.travis/README.md b/.travis/README.md new file mode 100644 index 0000000..d67df29 --- /dev/null +++ b/.travis/README.md @@ -0,0 +1,91 @@ +# Description of CI build configuration + +## Variables needed by travis + +- GITHUB_TOKEN - GitHub token with push access to repository +- DOCKER_USERNAME - Username (netdatabot) with write access to docker hub repository +- DOCKER_PASSWORD - Password to docker hub +- encrypted_8daf19481253_key - key needed by openssl to decrypt GCS credentials file +- encrypted_8daf19481253_iv - IV needed by openssl to decrypt GCS credentials file +- COVERITY_SCAN_TOKEN - Token to allow coverity test analysis uploads + +## Stages + +### Test + +Unit tests and coverage tests are executed here. Stage consists of 2 parallel jobs: + - C tests - executed every time + - dashboard.js - test if source files create the same file as it is in current repo + - coverity test - executed only when pipeline was triggered from cron + +### Build + +Stage is executed every time and consists of 5 parallel jobs which execute containerized and non-containerized +installations of netdata. Jobs are run on following operating systems: + - OSX + - ubuntu 14.04 + - ubuntu 16.04 (containerized) + - CentOS 6 (containerized) + - CentOS 7 (containerized) + - alpine (containerized) + +Images for system containers are stored on dockerhub and are created from Dockerfiles located in +[netdata/helper-images](https://github.com/netdata/helper-images) repository. + +### Packaging + +This stage is executed only on "master" brach and allows us to create a new tag just looking at git commit message. +It executes one script called `releaser.sh` which is responsible for creating a release on GitHub by using +[hub](https://github.com/github/hub). This script is also executing other scripts which can also be used in other +CI jobs: + - `.travis/tagger.sh` + - `.travis/generate_changelog.sh` + - `packaging/docker/build.sh` + - `.travis/create_artifacts.sh` + +Alternatively new release can be also created by pushing new tag to master branch. +Additionally this step is also executing `.travis/labeler.sh` which is a temporary workaround to automatically label +issues and PR. This script should be replaced with GitHub Actions when they are available to public. + +##### tagger.sh + +Script responsible to find out what will be the next tag based on a keyword in last commit message. Keywords are: + - `[netdata patch release]` to bump patch number + - `[netdata minor release]` to bump minor number + - `[netdata major release]` to bump major number + - `[netdata release candidate]` to create a new release candidate (appends or modifies suffix `-rcX` of previous tag) +All keywords MUST be surrounded with square brackets. +Tag is then stored in `GIT_TAG` variable. + +##### generate_changelog.sh + +Automatic changelog generator which updates our CHANGELOG.md file based on GitHub features (mostly labels and pull +requests). Internally it uses +[github-changelog-generator](https://github.com/github-changelog-generator/github-changelog-generator) and more +information can be found on that project site. + +##### build.sh and create_artifacts.sh + +Scripts used to build new container images and provide release artifacts (tar.gz and makeself archives) + +### Nightlies + +##### Tarball and self-extractor build AND Nightly docker images + +As names might suggest those two jobs are responsible for nightly netdata package creation and are run every day (in +cron). Combined they produce: + - docker images + - tar.gz archive (soon to be removed) + - self-extracting package + +This is achieved by running 2 scripts described earlier: + - `create_artifacts.sh` + - `build.sh` + +Artifacts are pushed to GCS and container images are stored in docker hub. + +##### Changelog generation + +This job is responsible for regenerating changelog every day by executing `generate_changelog.sh` script. This is done +only once a day due to github rate limiter. + diff --git a/.travis/create_artifacts.sh b/.travis/create_artifacts.sh new file mode 100755 index 0000000..ca0724e --- /dev/null +++ b/.travis/create_artifacts.sh @@ -0,0 +1,36 @@ +#!/bin/bash +# shellcheck disable=SC2230 + +set -e + +if [ ! -f .gitignore ]; then + echo "Run as ./travis/$(basename "$0") from top level directory of git repository" + exit 1 +fi + +# Everything from this directory will be uploaded to GCS +mkdir -p artifacts +BASENAME="netdata-$(git describe)" + +# Make sure stdout is in blocking mode. If we don't, then conda create will barf during downloads. +# See https://github.com/travis-ci/travis-ci/issues/4704#issuecomment-348435959 for details. +python -c 'import os,sys,fcntl; flags = fcntl.fcntl(sys.stdout, fcntl.F_GETFL); fcntl.fcntl(sys.stdout, fcntl.F_SETFL, flags&~os.O_NONBLOCK);' +echo "--- Create tarball ---" +autoreconf -ivf +./configure --prefix=/usr --sysconfdir=/etc --localstatedir=/var --with-zlib --with-math --with-user=netdata CFLAGS=-O2 +make dist +mv "${BASENAME}.tar.gz" artifacts/ + +echo "--- Create self-extractor ---" +./packaging/makeself/build-x86_64-static.sh + +# Needed fo GCS +echo "--- Copy artifacts to separate directory ---" +#shellcheck disable=SC2164 +cd artifacts +ln -s "${BASENAME}.tar.gz" netdata-latest.tar.gz +ln -s "${BASENAME}.gz.run" netdata-latest.gz.run +sha256sum -b ./* >"sha256sums.txt" +echo "checksums:" +cat sha256sums.txt + diff --git a/.travis/gcs-credentials.json.enc b/.travis/gcs-credentials.json.enc Binary files differnew file mode 100644 index 0000000..5d1e7b2 --- /dev/null +++ b/.travis/gcs-credentials.json.enc diff --git a/.travis/generate_changelog.sh b/.travis/generate_changelog.sh new file mode 100755 index 0000000..d1b72e0 --- /dev/null +++ b/.travis/generate_changelog.sh @@ -0,0 +1,33 @@ +#!/bin/bash + +set -e + +if [ ! -f .gitignore ]; then + echo "Run as ./travis/$(basename "$0") from top level directory of git repository" + exit 1 +fi + +ORGANIZATION=$(echo "$TRAVIS_REPO_SLUG" | awk -F '/' '{print $1}') +PROJECT=$(echo "$TRAVIS_REPO_SLUG" | awk -F '/' '{print $2}') +GIT_MAIL=${GIT_MAIL:-"pawel+bot@netdata.cloud"} +GIT_USER=${GIT_USER:-"netdatabot"} + +if [ -z ${GIT_TAG+x} ]; then + OPTS="" +else + OPTS="--future-release ${GIT_TAG}" +fi + +echo "--- Creating changelog ---" +git checkout master +git pull +#docker run -it --rm -v "$(pwd)":/usr/local/src/your-app ferrarimarco/github-changelog-generator:1.14.3 \ +docker run -it -v "$(pwd)":/project markmandel/github-changelog-generator:latest \ + --user "${ORGANIZATION}" \ + --project "${PROJECT}" \ + --token "${GITHUB_TOKEN}" \ + --since-tag "v1.10.0" \ + --unreleased-label "**Next release**" \ + --exclude-labels "stale,duplicate,question,invalid,wontfix,discussion,no changelog" \ + --no-compare-link ${OPTS} + diff --git a/.travis/labeler.sh b/.travis/labeler.sh new file mode 100755 index 0000000..e8d7d22 --- /dev/null +++ b/.travis/labeler.sh @@ -0,0 +1,97 @@ +#!/bin/bash + +# This is a simple script which should apply labels to unlabelled issues from last 3 days. +# It will soon be deprecated by GitHub Actions so no futher development on it is planned. + +# Previously this was using POST to only add labels. But this method seems to be failing with larger number of requests +new_labels() { + ISSUE="$1" + URL="https://api.github.com/repos/netdata/netdata/issues/$ISSUE/labels" + # deduplicate array and add quotes + SET=( $(for i in "${@:2}"; do [ "$i" != "" ] && echo "\"$i\""; done | sort -u) ) + # implode array to string + LABELS="${SET[*]}" + # add commas between quotes (replace spaces) + LABELS="${LABELS//\" \"/\",\"}" + # remove duplicate quotes in case parameters were already quoted + LABELS="${LABELS//\"\"/\"}" + echo "-------- Assigning labels to #${ISSUE}: ${LABELS} --------" + curl -H "Authorization: token $GITHUB_TOKEN" -d "{\"labels\":[${LABELS}]}" -X PUT "${URL}" &>/dev/null +} + +if [ "$GITHUB_TOKEN" == "" ]; then + echo "GITHUB_TOKEN is needed" + exit 1 +fi + +if ! [ -x "$(command -v hub)" ]; then + echo "===== Download HUB =====" + HUB_VERSION=${HUB_VERSION:-"2.5.1"} + wget "https://github.com/github/hub/releases/download/v${HUB_VERSION}/hub-linux-amd64-${HUB_VERSION}.tgz" -O "/tmp/hub-linux-amd64-${HUB_VERSION}.tgz" + tar -C /tmp -xvf "/tmp/hub-linux-amd64-${HUB_VERSION}.tgz" &>/dev/null + export PATH=$PATH:"/tmp/hub-linux-amd64-${HUB_VERSION}/bin" +fi + +echo "===== Looking up available labels =====" +LABELS_FILE=/tmp/labels +hub issue labels >$LABELS_FILE + +echo "===== Categorizing issues =====" +# This won't touch issues which already have at least one label assigned +for STATE in "open" "closed"; do + for ISSUE in $(hub issue -f "%I %l%n" -s "$STATE" -d "$(date +%F -d '1 day ago')" | grep -v -f $LABELS_FILE); do + echo "-------- Processing $STATE issue no. $ISSUE --------" + BODY="$(curl -H "Authorization: token $GITHUB_TOKEN" "https://api.github.com/repos/netdata/netdata/issues/$ISSUE" 2>/dev/null | jq .body)" + case "${BODY}" in + *"# Question summary"*) new_labels "$ISSUE" "question" "no changelog" ;; + *"# Bug report summary"*) new_labels "$ISSUE" "needs triage" "bug" ;; + *"# Feature idea summary"*) new_labels "$ISSUE" "needs triage" "feature request" ;; + *) new_labels "$ISSUE" "needs triage" "no changelog" ;; + esac + done +done + +# Change all 'area' labels assigned to PR saving non-area labels. +echo "===== Categorizing PRs =====" +NEW_LABELS=/tmp/new_labels +for PR in $(hub pr list -s all -f "%I%n" -L 10); do + echo "----- Processing PR #$PR -----" + echo "" >$NEW_LABELS + NEW_SET="" + DIFF_URL="https://github.com/netdata/netdata/pull/$PR.diff" + for FILE in $(curl -L "${DIFF_URL}" 2>/dev/null | grep "diff --git a/" | cut -d' ' -f3 | sort | uniq); do + LABEL="" + case "${FILE}" in + *".md") AREA="docs" ;; + *"/collectors/python.d.plugin/"*) AREA="external/python" ;; + *"/collectors/charts.d.plugin/"*) AREA="external" ;; + *"/collectors/node.d.plugin/"*) AREA="external" ;; + *"/.travis"*) AREA="ci" ;; + *"/.github/*.md"*) AREA="docs" ;; + *"/.github/"*) AREA="ci" ;; + *"/build/"*) AREA="packaging" ;; + *"/contrib/"*) AREA="packaging" ;; + *"/diagrams/"*) AREA="docs" ;; + *"/installer/"*) AREA="packaging" ;; + *"/makeself/"*) AREA="packaging" ;; + *"/system/"*) AREA="packaging" ;; + *"/netdata-installer.sh"*) AREA="packaging" ;; + *) AREA=$(echo "$FILE" | cut -d'/' -f2) ;; + esac + LABEL="area/$AREA" + echo "Selecting $LABEL due to $FILE" + if grep "$LABEL" "$LABELS_FILE"; then + echo "$LABEL" >>$NEW_LABELS + if [[ $LABEL =~ "external" ]]; then + echo "area/collectors" >>$NEW_LABELS + fi + else + echo "-------- Label '$LABEL' not available --------" + fi + done + NEW_SET=$(sort $NEW_LABELS | uniq) + if [ ! -z "$NEW_SET" ]; then + PREV=$(curl -H "Authorization: token $GITHUB_TOKEN" "https://api.github.com/repos/netdata/netdata/issues/$PR/labels" 2>/dev/null | jq '.[].name' | grep -v "area") + new_labels "$PR" ${NEW_SET} "${PREV[*]}" + fi +done diff --git a/.travis/nightlies.sh b/.travis/nightlies.sh new file mode 100755 index 0000000..fd133d0 --- /dev/null +++ b/.travis/nightlies.sh @@ -0,0 +1,45 @@ +#!/bin/bash + +BAD_THING_HAPPENED=0 + +if [ ! -f .gitignore ]; then + echo "Run as ./travis/$(basename "$0") from top level directory of git repository" + exit 1 +fi + +export GIT_MAIL="pawel+bot@netdata.cloud" +export GIT_USER="netdatabot" +echo "--- Initialize git configuration ---" +git config user.email "${GIT_MAIL}" +git config user.name "${GIT_USER}" + +echo "--- UPDATE VERSION FILE ---" +LAST_TAG=$(git describe --abbrev=0 --tags) +NO_COMMITS=$(git rev-list "$LAST_TAG"..HEAD --count) +if [ "$NO_COMMITS" == "$(rev <packaging/version | cut -d- -f 2 | rev)" ]; then + echo "Nothing changed since last nightly build" + exit 0 +fi +echo "$LAST_TAG-$((NO_COMMITS + 1))-nightly" >packaging/version +git add packaging/version || exit 1 + +echo "--- GENERATE CHANGELOG ---" +if .travis/generate_changelog.sh; then + git add CHANGELOG.md + + echo "--- UPLOAD FILE CHANGES ---" + git commit -m '[ci skip] create nightly packages and update changelog' + git push "https://${GITHUB_TOKEN}:@$(git config --get remote.origin.url | sed -e 's/^https:\/\///')" +else + git clean -xfd + BAD_THING_HAPPENED=1 +fi + +echo "--- BUILD & PUBLISH DOCKER IMAGES ---" +export REPOSITORY="netdata/netdata" +packaging/docker/build.sh || BAD_THING_HAPPENED=1 + +echo "--- BUILD ARTIFACTS ---" +.travis/create_artifacts.sh || BAD_THING_HAPPENED=1 + +exit "${BAD_THING_HAPPENED}" diff --git a/.travis/releaser.sh b/.travis/releaser.sh new file mode 100755 index 0000000..870dec5 --- /dev/null +++ b/.travis/releaser.sh @@ -0,0 +1,94 @@ +#!/bin/bash +# SPDX-License-Identifier: MIT +# Copyright (C) 2018 Pawel Krupa (@paulfantom) - All Rights Reserved +# Permission to copy and modify is granted under the MIT license +# +# Original script is available at https://github.com/paulfantom/travis-helper/blob/master/releasing/releaser.sh +# +# Script to automatically do a couple of things: +# - generate a new tag according to semver (https://semver.org/) +# - generate CHANGELOG.md by using https://github.com/skywinder/github-changelog-generator +# - create draft of GitHub releases by using https://github.com/github/hub +# +# Tags are generated by searching for a keyword in last commit message. Keywords are: +# - [patch] or [fix] to bump patch number +# - [minor], [feature] or [feat] to bump minor number +# - [major] or [breaking change] to bump major number +# All keywords MUST be surrounded with square braces. +# +# Script uses git mechanisms for locking, so it can be used in parallel builds +# +# Requirements: +# - GITHUB_TOKEN variable set with GitHub token. Access level: repo.public_repo +# - docker + +set -e + +if [ ! -f .gitignore ]; then + echo "Run as ./travis/$(basename "$0") from top level directory of git repository" + exit 1 +fi + +export GIT_MAIL="pawel+bot@netdata.cloud" +export GIT_USER="netdatabot" +echo "--- Initialize git configuration ---" +git config user.email "${GIT_MAIL}" +git config user.name "${GIT_USER}" +git checkout master +git pull + +echo "---- FIGURING OUT TAGS ----" +# tagger.sh is sourced since we need environment variables it sets +#shellcheck source=/dev/null +source .travis/tagger.sh || exit 0 +# variable GIT_TAG is produced by tagger.sh script + +echo "---- UPDATE VERSION FILE ----" +echo "$GIT_TAG" >packaging/version +git add packaging/version + +echo "---- GENERATE CHANGELOG -----" +./.travis/generate_changelog.sh +git add CHANGELOG.md + +echo "---- COMMIT AND PUSH CHANGES ----" +git commit -m "[ci skip] release $GIT_TAG" +git tag "$GIT_TAG" -a -m "Automatic tag generation for travis build no. $TRAVIS_BUILD_NUMBER" +git push "https://${GITHUB_TOKEN}:@$(git config --get remote.origin.url | sed -e 's/^https:\/\///')" +git push "https://${GITHUB_TOKEN}:@$(git config --get remote.origin.url | sed -e 's/^https:\/\///')" --tags +# After those operations output of command `git describe` should be identical with a value of GIT_TAG + +if [[ $(git describe) =~ -rc* ]]; then + echo "This is a release candidate tag, exiting without artifact building" + exit 0 +fi + +echo "---- CREATING TAGGED DOCKER CONTAINERS ----" +export REPOSITORY="netdata/netdata" +./packaging/docker/build.sh + +echo "---- CREATING RELEASE ARTIFACTS -----" +# Artifacts are stored in `artifacts/` directory +./.travis/create_artifacts.sh + +echo "---- CREATING RELEASE DRAFT WITH ASSETS -----" +# Download hub +HUB_VERSION=${HUB_VERSION:-"2.5.1"} +wget "https://github.com/github/hub/releases/download/v${HUB_VERSION}/hub-linux-amd64-${HUB_VERSION}.tgz" -O "/tmp/hub-linux-amd64-${HUB_VERSION}.tgz" +tar -C /tmp -xvf "/tmp/hub-linux-amd64-${HUB_VERSION}.tgz" +export PATH=$PATH:"/tmp/hub-linux-amd64-${HUB_VERSION}/bin" + +# Create a release draft +if [ -z ${GIT_TAG+x} ]; then + echo "Variable GIT_TAG is not set. Something went terribly wrong! Exiting." + exit 1 +fi +if [ "${GIT_TAG}" != "$(git tag --points-at)" ]; then + echo "ERROR! Current commit is not tagged. Stopping release creation." + exit 1 +fi +hub release create --draft \ + -a "artifacts/netdata-${GIT_TAG}.tar.gz" \ + -a "artifacts/netdata-${GIT_TAG}.gz.run" \ + -a "artifacts/sha256sums.txt" \ + -m "${GIT_TAG}" "${GIT_TAG}" diff --git a/.travis/tagger.sh b/.travis/tagger.sh new file mode 100755 index 0000000..e72c572 --- /dev/null +++ b/.travis/tagger.sh @@ -0,0 +1,59 @@ +#!/bin/bash +# SPDX-License-Identifier: MIT +# Copyright (C) 2018 Pawel Krupa (@paulfantom) - All Rights Reserved +# Permission to copy and modify is granted under the MIT license +# +# Original script is available at https://github.com/paulfantom/travis-helper/blob/master/releasing/releaser.sh +# +# Tags are generated by searching for a keyword in last commit message. Keywords are: +# - [patch] or [fix] to bump patch number +# - [minor], [feature] or [feat] to bump minor number +# - [major] or [breaking change] to bump major number +# All keywords MUST be surrounded with square braces. +# +# Requirements: +# - GITHUB_TOKEN variable set with GitHub token. Access level: repo.public_repo +# - git-semver python package (pip install git-semver) + +# exported variables are needed by releaser.sh + +set -e + +if [ ! -f .gitignore ]; then + echo "Run as ./travis/$(basename "$0") from top level directory of git repository" + exit 1 +fi + +# Figure out what will be new release candidate tag based only on previous ones. +# This assumes that RELEASES are in format of "v0.1.2" and prereleases (RCs) are using "v0.1.2-rc0" +function release_candidate() { + LAST_TAG=$(git semver) + if [[ $LAST_TAG =~ -rc* ]]; then + VERSION=$(echo "$LAST_TAG" | cut -d'-' -f 1) + LAST_RC=$(echo "$LAST_TAG" | cut -d'c' -f 2) + RC=$((LAST_RC + 1)) + else + VERSION="$(git semver --next-minor)" + RC=0 + fi + GIT_TAG="v$VERSION-rc$RC" +} + +# Check if current commit is tagged or not +GIT_TAG=$(git tag --points-at) +if [ -z "${GIT_TAG}" ]; then + git semver + # Figure out next tag based on commit message + echo "Last commit message: $TRAVIS_COMMIT_MESSAGE" + case "${TRAVIS_COMMIT_MESSAGE}" in + *"[netdata patch release]"*) GIT_TAG="v$(git semver --next-patch)" ;; + *"[netdata minor release]"*) GIT_TAG="v$(git semver --next-minor)" ;; + *"[netdata major release]"*) GIT_TAG="v$(git semver --next-major)" ;; + *"[netdata release candidate]"*) release_candidate ;; + *) + echo "Keyword not detected. Exiting..." + exit 0 + ;; + esac +fi +export GIT_TAG |