summaryrefslogtreecommitdiffstats
path: root/.travis
diff options
context:
space:
mode:
Diffstat (limited to '.travis')
-rw-r--r--.travis/README.md91
-rwxr-xr-x.travis/create_artifacts.sh36
-rw-r--r--.travis/gcs-credentials.json.encbin0 -> 2320 bytes
-rwxr-xr-x.travis/generate_changelog.sh33
-rwxr-xr-x.travis/labeler.sh97
-rwxr-xr-x.travis/nightlies.sh45
-rwxr-xr-x.travis/releaser.sh94
-rwxr-xr-x.travis/tagger.sh59
8 files changed, 455 insertions, 0 deletions
diff --git a/.travis/README.md b/.travis/README.md
new file mode 100644
index 0000000..d67df29
--- /dev/null
+++ b/.travis/README.md
@@ -0,0 +1,91 @@
+# Description of CI build configuration
+
+## Variables needed by travis
+
+- GITHUB_TOKEN - GitHub token with push access to repository
+- DOCKER_USERNAME - Username (netdatabot) with write access to docker hub repository
+- DOCKER_PASSWORD - Password to docker hub
+- encrypted_8daf19481253_key - key needed by openssl to decrypt GCS credentials file
+- encrypted_8daf19481253_iv - IV needed by openssl to decrypt GCS credentials file
+- COVERITY_SCAN_TOKEN - Token to allow coverity test analysis uploads
+
+## Stages
+
+### Test
+
+Unit tests and coverage tests are executed here. Stage consists of 2 parallel jobs:
+ - C tests - executed every time
+ - dashboard.js - test if source files create the same file as it is in current repo
+ - coverity test - executed only when pipeline was triggered from cron
+
+### Build
+
+Stage is executed every time and consists of 5 parallel jobs which execute containerized and non-containerized
+installations of netdata. Jobs are run on following operating systems:
+ - OSX
+ - ubuntu 14.04
+ - ubuntu 16.04 (containerized)
+ - CentOS 6 (containerized)
+ - CentOS 7 (containerized)
+ - alpine (containerized)
+
+Images for system containers are stored on dockerhub and are created from Dockerfiles located in
+[netdata/helper-images](https://github.com/netdata/helper-images) repository.
+
+### Packaging
+
+This stage is executed only on "master" brach and allows us to create a new tag just looking at git commit message.
+It executes one script called `releaser.sh` which is responsible for creating a release on GitHub by using
+[hub](https://github.com/github/hub). This script is also executing other scripts which can also be used in other
+CI jobs:
+ - `.travis/tagger.sh`
+ - `.travis/generate_changelog.sh`
+ - `packaging/docker/build.sh`
+ - `.travis/create_artifacts.sh`
+
+Alternatively new release can be also created by pushing new tag to master branch.
+Additionally this step is also executing `.travis/labeler.sh` which is a temporary workaround to automatically label
+issues and PR. This script should be replaced with GitHub Actions when they are available to public.
+
+##### tagger.sh
+
+Script responsible to find out what will be the next tag based on a keyword in last commit message. Keywords are:
+ - `[netdata patch release]` to bump patch number
+ - `[netdata minor release]` to bump minor number
+ - `[netdata major release]` to bump major number
+ - `[netdata release candidate]` to create a new release candidate (appends or modifies suffix `-rcX` of previous tag)
+All keywords MUST be surrounded with square brackets.
+Tag is then stored in `GIT_TAG` variable.
+
+##### generate_changelog.sh
+
+Automatic changelog generator which updates our CHANGELOG.md file based on GitHub features (mostly labels and pull
+requests). Internally it uses
+[github-changelog-generator](https://github.com/github-changelog-generator/github-changelog-generator) and more
+information can be found on that project site.
+
+##### build.sh and create_artifacts.sh
+
+Scripts used to build new container images and provide release artifacts (tar.gz and makeself archives)
+
+### Nightlies
+
+##### Tarball and self-extractor build AND Nightly docker images
+
+As names might suggest those two jobs are responsible for nightly netdata package creation and are run every day (in
+cron). Combined they produce:
+ - docker images
+ - tar.gz archive (soon to be removed)
+ - self-extracting package
+
+This is achieved by running 2 scripts described earlier:
+ - `create_artifacts.sh`
+ - `build.sh`
+
+Artifacts are pushed to GCS and container images are stored in docker hub.
+
+##### Changelog generation
+
+This job is responsible for regenerating changelog every day by executing `generate_changelog.sh` script. This is done
+only once a day due to github rate limiter.
+
diff --git a/.travis/create_artifacts.sh b/.travis/create_artifacts.sh
new file mode 100755
index 0000000..ca0724e
--- /dev/null
+++ b/.travis/create_artifacts.sh
@@ -0,0 +1,36 @@
+#!/bin/bash
+# shellcheck disable=SC2230
+
+set -e
+
+if [ ! -f .gitignore ]; then
+ echo "Run as ./travis/$(basename "$0") from top level directory of git repository"
+ exit 1
+fi
+
+# Everything from this directory will be uploaded to GCS
+mkdir -p artifacts
+BASENAME="netdata-$(git describe)"
+
+# Make sure stdout is in blocking mode. If we don't, then conda create will barf during downloads.
+# See https://github.com/travis-ci/travis-ci/issues/4704#issuecomment-348435959 for details.
+python -c 'import os,sys,fcntl; flags = fcntl.fcntl(sys.stdout, fcntl.F_GETFL); fcntl.fcntl(sys.stdout, fcntl.F_SETFL, flags&~os.O_NONBLOCK);'
+echo "--- Create tarball ---"
+autoreconf -ivf
+./configure --prefix=/usr --sysconfdir=/etc --localstatedir=/var --with-zlib --with-math --with-user=netdata CFLAGS=-O2
+make dist
+mv "${BASENAME}.tar.gz" artifacts/
+
+echo "--- Create self-extractor ---"
+./packaging/makeself/build-x86_64-static.sh
+
+# Needed fo GCS
+echo "--- Copy artifacts to separate directory ---"
+#shellcheck disable=SC2164
+cd artifacts
+ln -s "${BASENAME}.tar.gz" netdata-latest.tar.gz
+ln -s "${BASENAME}.gz.run" netdata-latest.gz.run
+sha256sum -b ./* >"sha256sums.txt"
+echo "checksums:"
+cat sha256sums.txt
+
diff --git a/.travis/gcs-credentials.json.enc b/.travis/gcs-credentials.json.enc
new file mode 100644
index 0000000..5d1e7b2
--- /dev/null
+++ b/.travis/gcs-credentials.json.enc
Binary files differ
diff --git a/.travis/generate_changelog.sh b/.travis/generate_changelog.sh
new file mode 100755
index 0000000..d1b72e0
--- /dev/null
+++ b/.travis/generate_changelog.sh
@@ -0,0 +1,33 @@
+#!/bin/bash
+
+set -e
+
+if [ ! -f .gitignore ]; then
+ echo "Run as ./travis/$(basename "$0") from top level directory of git repository"
+ exit 1
+fi
+
+ORGANIZATION=$(echo "$TRAVIS_REPO_SLUG" | awk -F '/' '{print $1}')
+PROJECT=$(echo "$TRAVIS_REPO_SLUG" | awk -F '/' '{print $2}')
+GIT_MAIL=${GIT_MAIL:-"pawel+bot@netdata.cloud"}
+GIT_USER=${GIT_USER:-"netdatabot"}
+
+if [ -z ${GIT_TAG+x} ]; then
+ OPTS=""
+else
+ OPTS="--future-release ${GIT_TAG}"
+fi
+
+echo "--- Creating changelog ---"
+git checkout master
+git pull
+#docker run -it --rm -v "$(pwd)":/usr/local/src/your-app ferrarimarco/github-changelog-generator:1.14.3 \
+docker run -it -v "$(pwd)":/project markmandel/github-changelog-generator:latest \
+ --user "${ORGANIZATION}" \
+ --project "${PROJECT}" \
+ --token "${GITHUB_TOKEN}" \
+ --since-tag "v1.10.0" \
+ --unreleased-label "**Next release**" \
+ --exclude-labels "stale,duplicate,question,invalid,wontfix,discussion,no changelog" \
+ --no-compare-link ${OPTS}
+
diff --git a/.travis/labeler.sh b/.travis/labeler.sh
new file mode 100755
index 0000000..e8d7d22
--- /dev/null
+++ b/.travis/labeler.sh
@@ -0,0 +1,97 @@
+#!/bin/bash
+
+# This is a simple script which should apply labels to unlabelled issues from last 3 days.
+# It will soon be deprecated by GitHub Actions so no futher development on it is planned.
+
+# Previously this was using POST to only add labels. But this method seems to be failing with larger number of requests
+new_labels() {
+ ISSUE="$1"
+ URL="https://api.github.com/repos/netdata/netdata/issues/$ISSUE/labels"
+ # deduplicate array and add quotes
+ SET=( $(for i in "${@:2}"; do [ "$i" != "" ] && echo "\"$i\""; done | sort -u) )
+ # implode array to string
+ LABELS="${SET[*]}"
+ # add commas between quotes (replace spaces)
+ LABELS="${LABELS//\" \"/\",\"}"
+ # remove duplicate quotes in case parameters were already quoted
+ LABELS="${LABELS//\"\"/\"}"
+ echo "-------- Assigning labels to #${ISSUE}: ${LABELS} --------"
+ curl -H "Authorization: token $GITHUB_TOKEN" -d "{\"labels\":[${LABELS}]}" -X PUT "${URL}" &>/dev/null
+}
+
+if [ "$GITHUB_TOKEN" == "" ]; then
+ echo "GITHUB_TOKEN is needed"
+ exit 1
+fi
+
+if ! [ -x "$(command -v hub)" ]; then
+ echo "===== Download HUB ====="
+ HUB_VERSION=${HUB_VERSION:-"2.5.1"}
+ wget "https://github.com/github/hub/releases/download/v${HUB_VERSION}/hub-linux-amd64-${HUB_VERSION}.tgz" -O "/tmp/hub-linux-amd64-${HUB_VERSION}.tgz"
+ tar -C /tmp -xvf "/tmp/hub-linux-amd64-${HUB_VERSION}.tgz" &>/dev/null
+ export PATH=$PATH:"/tmp/hub-linux-amd64-${HUB_VERSION}/bin"
+fi
+
+echo "===== Looking up available labels ====="
+LABELS_FILE=/tmp/labels
+hub issue labels >$LABELS_FILE
+
+echo "===== Categorizing issues ====="
+# This won't touch issues which already have at least one label assigned
+for STATE in "open" "closed"; do
+ for ISSUE in $(hub issue -f "%I %l%n" -s "$STATE" -d "$(date +%F -d '1 day ago')" | grep -v -f $LABELS_FILE); do
+ echo "-------- Processing $STATE issue no. $ISSUE --------"
+ BODY="$(curl -H "Authorization: token $GITHUB_TOKEN" "https://api.github.com/repos/netdata/netdata/issues/$ISSUE" 2>/dev/null | jq .body)"
+ case "${BODY}" in
+ *"# Question summary"*) new_labels "$ISSUE" "question" "no changelog" ;;
+ *"# Bug report summary"*) new_labels "$ISSUE" "needs triage" "bug" ;;
+ *"# Feature idea summary"*) new_labels "$ISSUE" "needs triage" "feature request" ;;
+ *) new_labels "$ISSUE" "needs triage" "no changelog" ;;
+ esac
+ done
+done
+
+# Change all 'area' labels assigned to PR saving non-area labels.
+echo "===== Categorizing PRs ====="
+NEW_LABELS=/tmp/new_labels
+for PR in $(hub pr list -s all -f "%I%n" -L 10); do
+ echo "----- Processing PR #$PR -----"
+ echo "" >$NEW_LABELS
+ NEW_SET=""
+ DIFF_URL="https://github.com/netdata/netdata/pull/$PR.diff"
+ for FILE in $(curl -L "${DIFF_URL}" 2>/dev/null | grep "diff --git a/" | cut -d' ' -f3 | sort | uniq); do
+ LABEL=""
+ case "${FILE}" in
+ *".md") AREA="docs" ;;
+ *"/collectors/python.d.plugin/"*) AREA="external/python" ;;
+ *"/collectors/charts.d.plugin/"*) AREA="external" ;;
+ *"/collectors/node.d.plugin/"*) AREA="external" ;;
+ *"/.travis"*) AREA="ci" ;;
+ *"/.github/*.md"*) AREA="docs" ;;
+ *"/.github/"*) AREA="ci" ;;
+ *"/build/"*) AREA="packaging" ;;
+ *"/contrib/"*) AREA="packaging" ;;
+ *"/diagrams/"*) AREA="docs" ;;
+ *"/installer/"*) AREA="packaging" ;;
+ *"/makeself/"*) AREA="packaging" ;;
+ *"/system/"*) AREA="packaging" ;;
+ *"/netdata-installer.sh"*) AREA="packaging" ;;
+ *) AREA=$(echo "$FILE" | cut -d'/' -f2) ;;
+ esac
+ LABEL="area/$AREA"
+ echo "Selecting $LABEL due to $FILE"
+ if grep "$LABEL" "$LABELS_FILE"; then
+ echo "$LABEL" >>$NEW_LABELS
+ if [[ $LABEL =~ "external" ]]; then
+ echo "area/collectors" >>$NEW_LABELS
+ fi
+ else
+ echo "-------- Label '$LABEL' not available --------"
+ fi
+ done
+ NEW_SET=$(sort $NEW_LABELS | uniq)
+ if [ ! -z "$NEW_SET" ]; then
+ PREV=$(curl -H "Authorization: token $GITHUB_TOKEN" "https://api.github.com/repos/netdata/netdata/issues/$PR/labels" 2>/dev/null | jq '.[].name' | grep -v "area")
+ new_labels "$PR" ${NEW_SET} "${PREV[*]}"
+ fi
+done
diff --git a/.travis/nightlies.sh b/.travis/nightlies.sh
new file mode 100755
index 0000000..fd133d0
--- /dev/null
+++ b/.travis/nightlies.sh
@@ -0,0 +1,45 @@
+#!/bin/bash
+
+BAD_THING_HAPPENED=0
+
+if [ ! -f .gitignore ]; then
+ echo "Run as ./travis/$(basename "$0") from top level directory of git repository"
+ exit 1
+fi
+
+export GIT_MAIL="pawel+bot@netdata.cloud"
+export GIT_USER="netdatabot"
+echo "--- Initialize git configuration ---"
+git config user.email "${GIT_MAIL}"
+git config user.name "${GIT_USER}"
+
+echo "--- UPDATE VERSION FILE ---"
+LAST_TAG=$(git describe --abbrev=0 --tags)
+NO_COMMITS=$(git rev-list "$LAST_TAG"..HEAD --count)
+if [ "$NO_COMMITS" == "$(rev <packaging/version | cut -d- -f 2 | rev)" ]; then
+ echo "Nothing changed since last nightly build"
+ exit 0
+fi
+echo "$LAST_TAG-$((NO_COMMITS + 1))-nightly" >packaging/version
+git add packaging/version || exit 1
+
+echo "--- GENERATE CHANGELOG ---"
+if .travis/generate_changelog.sh; then
+ git add CHANGELOG.md
+
+ echo "--- UPLOAD FILE CHANGES ---"
+ git commit -m '[ci skip] create nightly packages and update changelog'
+ git push "https://${GITHUB_TOKEN}:@$(git config --get remote.origin.url | sed -e 's/^https:\/\///')"
+else
+ git clean -xfd
+ BAD_THING_HAPPENED=1
+fi
+
+echo "--- BUILD & PUBLISH DOCKER IMAGES ---"
+export REPOSITORY="netdata/netdata"
+packaging/docker/build.sh || BAD_THING_HAPPENED=1
+
+echo "--- BUILD ARTIFACTS ---"
+.travis/create_artifacts.sh || BAD_THING_HAPPENED=1
+
+exit "${BAD_THING_HAPPENED}"
diff --git a/.travis/releaser.sh b/.travis/releaser.sh
new file mode 100755
index 0000000..870dec5
--- /dev/null
+++ b/.travis/releaser.sh
@@ -0,0 +1,94 @@
+#!/bin/bash
+# SPDX-License-Identifier: MIT
+# Copyright (C) 2018 Pawel Krupa (@paulfantom) - All Rights Reserved
+# Permission to copy and modify is granted under the MIT license
+#
+# Original script is available at https://github.com/paulfantom/travis-helper/blob/master/releasing/releaser.sh
+#
+# Script to automatically do a couple of things:
+# - generate a new tag according to semver (https://semver.org/)
+# - generate CHANGELOG.md by using https://github.com/skywinder/github-changelog-generator
+# - create draft of GitHub releases by using https://github.com/github/hub
+#
+# Tags are generated by searching for a keyword in last commit message. Keywords are:
+# - [patch] or [fix] to bump patch number
+# - [minor], [feature] or [feat] to bump minor number
+# - [major] or [breaking change] to bump major number
+# All keywords MUST be surrounded with square braces.
+#
+# Script uses git mechanisms for locking, so it can be used in parallel builds
+#
+# Requirements:
+# - GITHUB_TOKEN variable set with GitHub token. Access level: repo.public_repo
+# - docker
+
+set -e
+
+if [ ! -f .gitignore ]; then
+ echo "Run as ./travis/$(basename "$0") from top level directory of git repository"
+ exit 1
+fi
+
+export GIT_MAIL="pawel+bot@netdata.cloud"
+export GIT_USER="netdatabot"
+echo "--- Initialize git configuration ---"
+git config user.email "${GIT_MAIL}"
+git config user.name "${GIT_USER}"
+git checkout master
+git pull
+
+echo "---- FIGURING OUT TAGS ----"
+# tagger.sh is sourced since we need environment variables it sets
+#shellcheck source=/dev/null
+source .travis/tagger.sh || exit 0
+# variable GIT_TAG is produced by tagger.sh script
+
+echo "---- UPDATE VERSION FILE ----"
+echo "$GIT_TAG" >packaging/version
+git add packaging/version
+
+echo "---- GENERATE CHANGELOG -----"
+./.travis/generate_changelog.sh
+git add CHANGELOG.md
+
+echo "---- COMMIT AND PUSH CHANGES ----"
+git commit -m "[ci skip] release $GIT_TAG"
+git tag "$GIT_TAG" -a -m "Automatic tag generation for travis build no. $TRAVIS_BUILD_NUMBER"
+git push "https://${GITHUB_TOKEN}:@$(git config --get remote.origin.url | sed -e 's/^https:\/\///')"
+git push "https://${GITHUB_TOKEN}:@$(git config --get remote.origin.url | sed -e 's/^https:\/\///')" --tags
+# After those operations output of command `git describe` should be identical with a value of GIT_TAG
+
+if [[ $(git describe) =~ -rc* ]]; then
+ echo "This is a release candidate tag, exiting without artifact building"
+ exit 0
+fi
+
+echo "---- CREATING TAGGED DOCKER CONTAINERS ----"
+export REPOSITORY="netdata/netdata"
+./packaging/docker/build.sh
+
+echo "---- CREATING RELEASE ARTIFACTS -----"
+# Artifacts are stored in `artifacts/` directory
+./.travis/create_artifacts.sh
+
+echo "---- CREATING RELEASE DRAFT WITH ASSETS -----"
+# Download hub
+HUB_VERSION=${HUB_VERSION:-"2.5.1"}
+wget "https://github.com/github/hub/releases/download/v${HUB_VERSION}/hub-linux-amd64-${HUB_VERSION}.tgz" -O "/tmp/hub-linux-amd64-${HUB_VERSION}.tgz"
+tar -C /tmp -xvf "/tmp/hub-linux-amd64-${HUB_VERSION}.tgz"
+export PATH=$PATH:"/tmp/hub-linux-amd64-${HUB_VERSION}/bin"
+
+# Create a release draft
+if [ -z ${GIT_TAG+x} ]; then
+ echo "Variable GIT_TAG is not set. Something went terribly wrong! Exiting."
+ exit 1
+fi
+if [ "${GIT_TAG}" != "$(git tag --points-at)" ]; then
+ echo "ERROR! Current commit is not tagged. Stopping release creation."
+ exit 1
+fi
+hub release create --draft \
+ -a "artifacts/netdata-${GIT_TAG}.tar.gz" \
+ -a "artifacts/netdata-${GIT_TAG}.gz.run" \
+ -a "artifacts/sha256sums.txt" \
+ -m "${GIT_TAG}" "${GIT_TAG}"
diff --git a/.travis/tagger.sh b/.travis/tagger.sh
new file mode 100755
index 0000000..e72c572
--- /dev/null
+++ b/.travis/tagger.sh
@@ -0,0 +1,59 @@
+#!/bin/bash
+# SPDX-License-Identifier: MIT
+# Copyright (C) 2018 Pawel Krupa (@paulfantom) - All Rights Reserved
+# Permission to copy and modify is granted under the MIT license
+#
+# Original script is available at https://github.com/paulfantom/travis-helper/blob/master/releasing/releaser.sh
+#
+# Tags are generated by searching for a keyword in last commit message. Keywords are:
+# - [patch] or [fix] to bump patch number
+# - [minor], [feature] or [feat] to bump minor number
+# - [major] or [breaking change] to bump major number
+# All keywords MUST be surrounded with square braces.
+#
+# Requirements:
+# - GITHUB_TOKEN variable set with GitHub token. Access level: repo.public_repo
+# - git-semver python package (pip install git-semver)
+
+# exported variables are needed by releaser.sh
+
+set -e
+
+if [ ! -f .gitignore ]; then
+ echo "Run as ./travis/$(basename "$0") from top level directory of git repository"
+ exit 1
+fi
+
+# Figure out what will be new release candidate tag based only on previous ones.
+# This assumes that RELEASES are in format of "v0.1.2" and prereleases (RCs) are using "v0.1.2-rc0"
+function release_candidate() {
+ LAST_TAG=$(git semver)
+ if [[ $LAST_TAG =~ -rc* ]]; then
+ VERSION=$(echo "$LAST_TAG" | cut -d'-' -f 1)
+ LAST_RC=$(echo "$LAST_TAG" | cut -d'c' -f 2)
+ RC=$((LAST_RC + 1))
+ else
+ VERSION="$(git semver --next-minor)"
+ RC=0
+ fi
+ GIT_TAG="v$VERSION-rc$RC"
+}
+
+# Check if current commit is tagged or not
+GIT_TAG=$(git tag --points-at)
+if [ -z "${GIT_TAG}" ]; then
+ git semver
+ # Figure out next tag based on commit message
+ echo "Last commit message: $TRAVIS_COMMIT_MESSAGE"
+ case "${TRAVIS_COMMIT_MESSAGE}" in
+ *"[netdata patch release]"*) GIT_TAG="v$(git semver --next-patch)" ;;
+ *"[netdata minor release]"*) GIT_TAG="v$(git semver --next-minor)" ;;
+ *"[netdata major release]"*) GIT_TAG="v$(git semver --next-major)" ;;
+ *"[netdata release candidate]"*) release_candidate ;;
+ *)
+ echo "Keyword not detected. Exiting..."
+ exit 0
+ ;;
+ esac
+fi
+export GIT_TAG