11 files changed, 297 insertions, 2 deletions
diff --git a/.travis/README.md b/.travis/README.md
new file mode 100644
index 000000000..5a51b2a7c
--- /dev/null
+++ b/.travis/README.md
@@ -0,0 +1,80 @@
+# Description of CI build configuration
+
+## Variables needed by travis
+
+- GITHUB_TOKEN - GitHub token with push access to repository
+- DOCKER_USERNAME - Username (netdatabot) with write access to docker hub repository
+- DOCKER_PASSWORD - Password to docker hub
+- encrypted_decb6f6387c4_key - Something to do with package releasing (soon to be deprecated)
+- encrypted_decb6f6387c4_iv - Something to do with package releasing (soon to be deprecated)
+- OLD_DOCKER_USERNAME - Username used to push images to firehol/netdata # TODO: remove after deprecating that repo
+- OLD_DOCKER_PASSWORD - Password used to push images to firehol/netdata # TODO: remove after deprecating that repo
+
+## Stages
+
+### Test
+
+Unit tests and coverage tests are executed here. Stage consists of 2 parallel jobs:
+  - C tests - executed every time
+  - coverity test - executed only when pipeline was triggered from cron
+
+### Build
+
+Stage is executed every time and consists of 5 parallel jobs which execute containerized and non-containerized
+installations of netdata. Jobs are run on following operating systems:
+  - OSX
+  - ubuntu 14.04
+  - ubuntu 16.04 (containerized)
+  - CentOS 6 (containerized)
+  - CentOS 7 (containerized)
+  - alpine (containerized)
+
+### Release
+
+This stage is executed only on "master" brach and allows us to create a new tag just looking at git commit message.
+It also has an option to automatically generate changelog based on GitHub labels and sync it with GitHub release.
+For the sake of simplicity and to use travis features this stage cannot be integrated with next stage.
+
+Releases are generated by searching for a keyword in last commit message. Keywords are:
+ - [patch] or [fix] to bump patch number
+ - [minor], [feature] or [feat] to bump minor number
+ - [major] or [breaking change] to bump major number
+All keywords MUST be surrounded with square braces.
+Alternative is to push a tag to master branch.
+
+### Packaging
+
+This stage is executed only on "master" branch and it is separated into 3 jobs:
+  - Update Changelog/Create release
+  - Nightly tarball and self-extractor build
+  - Nightly docker images
+
+##### Update Changelog/Create release
+
+This job is running one script called `releaser.sh`, which is responsible for a couple of things. First of all it 
+automatically updates our CHANGELOG.md file based on GitHub features (mostly labels and pull requests). Apart from 
+that it can also create a new git tag and a github draft release connected to that tag.
+Releases are generated by searching for a keyword in last commit message. Keywords are:
+ - `[netdata patch release]` to bump patch number
+ - `[netdata minor release]` to bump minor number
+ - `[netdata major release]` to bump major number
+All keywords MUST be surrounded with square brackets.
+
+Alternatively new release can be also created by pushing new tag to master branch.
+
+##### Nightly tarball and self-extractor build AND Nightly docker images
+
+As names might suggest those two jobs are responsible for nightly netdata package creation and are run every day (in
+cron). Combined they produce:
+  - docker images
+  - tar.gz archive (soon to be removed)
+  - self-extracting package
+
+Currently "Nightly tarball and self-extractor build" is using old firehol script and it is planed to be replaced with
+new design.
+
+##### Nightly changelog generation
+
+This job is responsible for regenerating changelog every day by executing `generate_changelog.sh` script. This is done
+only once a day due to github rate limiter.
+
diff --git a/.travis/containerized_build.sh b/.travis/containerized_build.sh
new file mode 100755
index 000000000..314a2ec39
--- /dev/null
+++ b/.travis/containerized_build.sh
@@ -0,0 +1,7 @@
+#!/bin/bash
+
+set -e
+
+docker build -t dev-image -f ".travis/images/Dockerfile.$1" .
+
+docker run -it -w /code dev-image ./netdata-installer.sh --dont-wait --dont-start-it --install /tmp
diff --git a/.travis/create_artifacts.sh b/.travis/create_artifacts.sh
new file mode 100755
index 000000000..40ba9c85f
--- /dev/null
+++ b/.travis/create_artifacts.sh
@@ -0,0 +1,33 @@
+#!/bin/bash
+# shellcheck disable=SC2230
+
+if [ ! -f .gitignore ]
+then
+  echo "Run as ./travis/$(basename "$0") from top level directory of git repository"
+  exit 1
+fi
+
+# Make sure stdout is in blocking mode. If we don't, then conda create will barf during downloads.
+# See https://github.com/travis-ci/travis-ci/issues/4704#issuecomment-348435959 for details.
+python -c 'import os,sys,fcntl; flags = fcntl.fcntl(sys.stdout, fcntl.F_GETFL); fcntl.fcntl(sys.stdout, fcntl.F_SETFL, flags&~os.O_NONBLOCK);'
+echo "--- Create tarball ---"
+autoreconf -ivf
+./configure
+make dist
+echo "--- Create self-extractor ---"
+./makeself/build-x86_64-static.sh
+
+echo "--- Create checksums ---"
+GIT_TAG=$(git tag --points-at)
+if [ "${GIT_TAG}" != "" ]; then
+    ln -s netdata-latest.gz.run "netdata-${GIT_TAG}.gz.run"
+    ln -s netdata-*.tar.gz "netdata-${GIT_TAG}.tar.gz"
+    sha256sum -b "netdata-${GIT_TAG}.gz.run" "netdata-${GIT_TAG}.tar.gz" > "sha256sums.txt"
+else
+    sha256sum -b ./*.tar.gz ./*.gz.run > "sha256sums.txt"
+fi
+
+echo "checksums:"
+cat sha256sums.txt
+
+
diff --git a/.travis/deploy-if-have-key b/.travis/deploy-if-have-key
index 1933eeb2e..8b3b40f7e 100755
--- a/.travis/deploy-if-have-key
+++ b/.travis/deploy-if-have-key
@@ -46,6 +46,10 @@ fi
 
 ssh-keyscan -H firehol.org >> ~/.ssh/known_hosts
 ssh travis@firehol.org mkdir -p uploads/netdata/$TRAVIS_BRANCH/
-scp -p *.tar.* travis@firehol.org:uploads/netdata/$TRAVIS_BRANCH/
-scp -p *.gz.run* travis@firehol.org:uploads/netdata/$TRAVIS_BRANCH/
+scp -p *.tar.gz travis@firehol.org:uploads/netdata/$TRAVIS_BRANCH/
+scp -p *.tar.gz.sha travis@firehol.org:uploads/netdata/$TRAVIS_BRANCH/
+scp -p *.tar.gz.asc travis@firehol.org:uploads/netdata/$TRAVIS_BRANCH/
+scp -p *.gz.run travis@firehol.org:uploads/netdata/$TRAVIS_BRANCH/
+scp -p *.gz.run.sha travis@firehol.org:uploads/netdata/$TRAVIS_BRANCH/
+scp -p *.gz.run.asc travis@firehol.org:uploads/netdata/$TRAVIS_BRANCH/
 ssh travis@firehol.org touch uploads/netdata/$TRAVIS_BRANCH/complete.txt
diff --git a/.travis/firehol_create_artifacts.sh b/.travis/firehol_create_artifacts.sh
new file mode 100755
index 000000000..3fcb910e7
--- /dev/null
+++ b/.travis/firehol_create_artifacts.sh
@@ -0,0 +1,31 @@
+#!/bin/bash
+# shellcheck disable=SC2230
+
+# WARNING: This script is deprecated and placed here until @paulfantom figures out how to fully replace it
+
+if [ ! -f .gitignore ]
+then
+  echo "Run as ./travis/$(basename "$0") from top level directory of git repository"
+  exit 1
+fi
+
+eval "$(ssh-agent -s)"
+./.travis/decrypt-if-have-key decb6f6387c4
+export KEYSERVER=ipv4.pool.sks-keyservers.net
+./packaging/gpg-recv-key phil@firehol.org "0762 9FF7 89EA 6156 012F  9F50 C406 9602 1359 9237"
+./packaging/gpg-recv-key costa@tsaousis.gr "4DFF 624A E564 3B51 2872  1F40 29CA 3358 89B9 A863"
+# Run the commit hooks in case the developer didn't
+git diff 4b825dc642cb6eb9a060e54bf8d69288fbee4904 | ./packaging/check-files -
+fakeroot ./packaging/git-build
+# Make sure stdout is in blocking mode. If we don't, then conda create will barf during downloads.
+# See https://github.com/travis-ci/travis-ci/issues/4704#issuecomment-348435959 for details.
+python -c 'import os,sys,fcntl; flags = fcntl.fcntl(sys.stdout, fcntl.F_GETFL); fcntl.fcntl(sys.stdout, fcntl.F_SETFL, flags&~os.O_NONBLOCK);'
+echo "--- Create tarball ---"
+make dist
+echo "--- Create self-extractor ---"
+./makeself/build-x86_64-static.sh
+echo "--- Create checksums ---"
+for i in *.tar.gz; do sha512sum -b "$i" > "$i.sha"; done #FIXME remove?
+for i in *.gz.run; do sha512sum -b "$i" > "$i.sha"; done #FIXME remove?
+sha256sum -b ./*.tar.gz ./*.gz.run > "sha256sums.txt"
+./.travis/deploy-if-have-key
diff --git a/.travis/generate_changelog.sh b/.travis/generate_changelog.sh
new file mode 100755
index 000000000..bc8be1023
--- /dev/null
+++ b/.travis/generate_changelog.sh
@@ -0,0 +1,36 @@
+#!/bin/bash
+
+set -e
+
+if [ ! -f .gitignore ]
+then
+  echo "Run as ./travis/$(basename "$0") from top level directory of git repository"
+  exit 1
+fi
+
+ORGANIZATION=$(echo "$TRAVIS_REPO_SLUG" | awk -F '/' '{print $1}')
+PROJECT=$(echo "$TRAVIS_REPO_SLUG" | awk -F '/' '{print $2}')
+GIT_MAIL="pawel+bot@netdata.cloud"
+GIT_USER="netdatabot"
+
+echo "--- Initialize git configuration ---"
+git config user.email "${GIT_MAIL}"
+git config user.name "${GIT_USER}"
+
+echo "--- Creating changelog ---"
+git checkout master
+git pull
+#docker run -it --rm -v "$(pwd)":/usr/local/src/your-app ferrarimarco/github-changelog-generator:1.14.3 \
+docker run -it -v "$(pwd)":/project markmandel/github-changelog-generator:latest \
+                                                        --user "${ORGANIZATION}" \
+                                                        --project "${PROJECT}" \
+                                                        --token "${GITHUB_TOKEN}" \
+                                                        --since-tag "v1.10.0" \
+                                                        --unreleased-label "**Next release**" \
+                                                        --no-compare-link \
+                                                        --exclude-labels duplicate,question,invalid,wontfix,discussion,documentation
+
+echo "--- Uploading changelog ---"
+git add CHANGELOG.md
+git commit -m '[ci skip] Automatic changelog update'
+git push "https://${GITHUB_TOKEN}:@$(git config --get remote.origin.url | sed -e 's/^https:\/\///')"
diff --git a/.travis/images/Dockerfile.alpine b/.travis/images/Dockerfile.alpine
new file mode 100644
index 000000000..15f527165
--- /dev/null
+++ b/.travis/images/Dockerfile.alpine
@@ -0,0 +1,5 @@
+FROM alpine:latest
+
+RUN apk add bash gcc make autoconf automake pkgconfig zlib-dev libuuid git libmnl-dev util-linux-dev build-base
+
+COPY . /code
diff --git a/.travis/images/Dockerfile.centos6 b/.travis/images/Dockerfile.centos6
new file mode 100644
index 000000000..c286fb95c
--- /dev/null
+++ b/.travis/images/Dockerfile.centos6
@@ -0,0 +1,5 @@
+FROM centos:6
+
+RUN yum install -y gcc make autoconf automake pkg-config zlib-devel libuuid-devel git
+
+COPY . /code
diff --git a/.travis/images/Dockerfile.centos7 b/.travis/images/Dockerfile.centos7
new file mode 100644
index 000000000..d94359dec
--- /dev/null
+++ b/.travis/images/Dockerfile.centos7
@@ -0,0 +1,5 @@
+FROM centos:7
+
+RUN yum install -y gcc make autoconf automake pkg-config zlib-devel libuuid-devel git
+
+COPY . /code
diff --git a/.travis/images/Dockerfile.ubuntu1804 b/.travis/images/Dockerfile.ubuntu1804
new file mode 100644
index 000000000..30ac7cf71
--- /dev/null
+++ b/.travis/images/Dockerfile.ubuntu1804
@@ -0,0 +1,6 @@
+FROM ubuntu:18.04
+
+RUN apt-get update && \
+    apt-get install -y gcc make autoconf automake pkg-config zlib1g-dev uuid-dev git
+
+COPY . /code
diff --git a/.travis/releaser.sh b/.travis/releaser.sh
new file mode 100755
index 000000000..9f7ecd4ee
--- /dev/null
+++ b/.travis/releaser.sh
@@ -0,0 +1,83 @@
+#!/bin/bash
+# SPDX-License-Identifier: MIT
+# Copyright (C) 2018 Pawel Krupa (@paulfantom) - All Rights Reserved
+# Permission to copy and modify is granted under the MIT license
+#
+# Original script is available at https://github.com/paulfantom/travis-helper/blob/master/releasing/releaser.sh
+#
+# Script to automatically do a couple of things:
+#   - generate a new tag according to semver (https://semver.org/)
+#   - generate CHANGELOG.md by using https://github.com/skywinder/github-changelog-generator
+#   - create draft of GitHub releases by using https://github.com/github/hub
+#
+# Tags are generated by searching for a keyword in last commit message. Keywords are:
+#  - [patch] or [fix] to bump patch number
+#  - [minor], [feature] or [feat] to bump minor number
+#  - [major] or [breaking change] to bump major number
+# All keywords MUST be surrounded with square braces.
+#
+# Script uses git mechanisms for locking, so it can be used in parallel builds
+#
+# Requirements:
+#   - GITHUB_TOKEN variable set with GitHub token. Access level: repo.public_repo
+#   - docker
+#   - git-semver python package (pip install git-semver)
+
+set -e
+
+if [ ! -f .gitignore ]
+then
+  echo "Run as ./travis/$(basename "$0") from top level directory of git repository"
+  exit 1
+fi
+
+echo "---- GENERATING CHANGELOG -----"
+./.travis/generate_changelog.sh
+
+echo "---- FIGURING OUT TAGS ----"
+# Check if current commit is tagged or not
+GIT_TAG=$(git tag --points-at)
+if [ -z "${GIT_TAG}" ]; then
+  git semver
+  # Figure out next tag based on commit message
+  GIT_TAG=HEAD
+  echo "Last commit message: $TRAVIS_COMMIT_MESSAGE"
+  case "${TRAVIS_COMMIT_MESSAGE}" in
+    *"[netdata patch release]"* ) GIT_TAG="v$(git semver --next-patch)" ;;
+    *"[netdata minor release]"* ) GIT_TAG="v$(git semver --next-minor)" ;;
+    *"[netdata major release]"* ) GIT_TAG="v$(git semver --next-major)" ;;
+    *) echo "Keyword not detected. Doing nothing" ;;
+  esac
+
+  # Tag it!
+  if [ "$GIT_TAG" != "HEAD" ]; then
+      echo "Assigning a new tag: $GIT_TAG"
+      git tag "$GIT_TAG" -a -m "Automatic tag generation for travis build no. $TRAVIS_BUILD_NUMBER"
+      # git is able to push due to configuration already being initialized in `generate_changelog.sh` script
+      git push "https://${GITHUB_TOKEN}:@$(git config --get remote.origin.url | sed -e 's/^https:\/\///')" --tags
+  fi
+fi
+
+if [ "${GIT_TAG}" == "HEAD" ]; then
+    echo "Not creating a release since neither of two conditions was met:"
+    echo "  - keyword in commit message"
+    echo "  - commit is tagged"
+    exit 0
+fi
+
+echo "---- CREATING TAGGED DOCKER CONTAINERS ----"
+export REPOSITORY="netdata/netdata"
+./docker/build.sh
+
+echo "---- CREATING RELEASE ARTIFACTS -----"
+./.travis/create_artifacts.sh
+
+echo "---- CREATING RELEASE DRAFT WITH ASSETS -----"
+# Download hub
+HUB_VERSION=${HUB_VERSION:-"2.5.1"}
+wget "https://github.com/github/hub/releases/download/v${HUB_VERSION}/hub-linux-amd64-${HUB_VERSION}.tgz" -O "/tmp/hub-linux-amd64-${HUB_VERSION}.tgz"
+tar -C /tmp -xvf "/tmp/hub-linux-amd64-${HUB_VERSION}.tgz"
+export PATH=$PATH:"/tmp/hub-linux-amd64-${HUB_VERSION}/bin"
+
+# Create a release draft
+hub release create --draft -a "netdata-${GIT_TAG}.tar.gz" -a "netdata-${GIT_TAG}.gz.run" -a "sha256sums.txt" -m "${GIT_TAG}" "${GIT_TAG}"