From 1e6c93250172946eeb38e94a92a1fd12c9d3011e Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Wed, 7 Nov 2018 13:22:44 +0100 Subject: Merging upstream version 1.11.0+dfsg. Signed-off-by: Daniel Baumann --- .travis/README.md | 80 ++++++++++++++++++++++++++++++++++ .travis/containerized_build.sh | 7 +++ .travis/create_artifacts.sh | 33 ++++++++++++++ .travis/deploy-if-have-key | 8 +++- .travis/firehol_create_artifacts.sh | 31 ++++++++++++++ .travis/generate_changelog.sh | 36 ++++++++++++++++ .travis/images/Dockerfile.alpine | 5 +++ .travis/images/Dockerfile.centos6 | 5 +++ .travis/images/Dockerfile.centos7 | 5 +++ .travis/images/Dockerfile.ubuntu1804 | 6 +++ .travis/releaser.sh | 83 ++++++++++++++++++++++++++++++++++++ 11 files changed, 297 insertions(+), 2 deletions(-) create mode 100644 .travis/README.md create mode 100755 .travis/containerized_build.sh create mode 100755 .travis/create_artifacts.sh create mode 100755 .travis/firehol_create_artifacts.sh create mode 100755 .travis/generate_changelog.sh create mode 100644 .travis/images/Dockerfile.alpine create mode 100644 .travis/images/Dockerfile.centos6 create mode 100644 .travis/images/Dockerfile.centos7 create mode 100644 .travis/images/Dockerfile.ubuntu1804 create mode 100755 .travis/releaser.sh (limited to '.travis') diff --git a/.travis/README.md b/.travis/README.md new file mode 100644 index 000000000..5a51b2a7c --- /dev/null +++ b/.travis/README.md @@ -0,0 +1,80 @@ +# Description of CI build configuration + +## Variables needed by travis + +- GITHUB_TOKEN - GitHub token with push access to repository +- DOCKER_USERNAME - Username (netdatabot) with write access to docker hub repository +- DOCKER_PASSWORD - Password to docker hub +- encrypted_decb6f6387c4_key - Something to do with package releasing (soon to be deprecated) +- encrypted_decb6f6387c4_iv - Something to do with package releasing (soon to be deprecated) +- OLD_DOCKER_USERNAME - Username used to push images to firehol/netdata # TODO: remove after deprecating that repo +- OLD_DOCKER_PASSWORD - Password used to push images to firehol/netdata # TODO: remove after deprecating that repo + +## Stages + +### Test + +Unit tests and coverage tests are executed here. Stage consists of 2 parallel jobs: + - C tests - executed every time + - coverity test - executed only when pipeline was triggered from cron + +### Build + +Stage is executed every time and consists of 5 parallel jobs which execute containerized and non-containerized +installations of netdata. Jobs are run on following operating systems: + - OSX + - ubuntu 14.04 + - ubuntu 16.04 (containerized) + - CentOS 6 (containerized) + - CentOS 7 (containerized) + - alpine (containerized) + +### Release + +This stage is executed only on "master" brach and allows us to create a new tag just looking at git commit message. +It also has an option to automatically generate changelog based on GitHub labels and sync it with GitHub release. +For the sake of simplicity and to use travis features this stage cannot be integrated with next stage. + +Releases are generated by searching for a keyword in last commit message. Keywords are: + - [patch] or [fix] to bump patch number + - [minor], [feature] or [feat] to bump minor number + - [major] or [breaking change] to bump major number +All keywords MUST be surrounded with square braces. +Alternative is to push a tag to master branch. + +### Packaging + +This stage is executed only on "master" branch and it is separated into 3 jobs: + - Update Changelog/Create release + - Nightly tarball and self-extractor build + - Nightly docker images + +##### Update Changelog/Create release + +This job is running one script called `releaser.sh`, which is responsible for a couple of things. First of all it +automatically updates our CHANGELOG.md file based on GitHub features (mostly labels and pull requests). Apart from +that it can also create a new git tag and a github draft release connected to that tag. +Releases are generated by searching for a keyword in last commit message. Keywords are: + - `[netdata patch release]` to bump patch number + - `[netdata minor release]` to bump minor number + - `[netdata major release]` to bump major number +All keywords MUST be surrounded with square brackets. + +Alternatively new release can be also created by pushing new tag to master branch. + +##### Nightly tarball and self-extractor build AND Nightly docker images + +As names might suggest those two jobs are responsible for nightly netdata package creation and are run every day (in +cron). Combined they produce: + - docker images + - tar.gz archive (soon to be removed) + - self-extracting package + +Currently "Nightly tarball and self-extractor build" is using old firehol script and it is planed to be replaced with +new design. + +##### Nightly changelog generation + +This job is responsible for regenerating changelog every day by executing `generate_changelog.sh` script. This is done +only once a day due to github rate limiter. + diff --git a/.travis/containerized_build.sh b/.travis/containerized_build.sh new file mode 100755 index 000000000..314a2ec39 --- /dev/null +++ b/.travis/containerized_build.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +set -e + +docker build -t dev-image -f ".travis/images/Dockerfile.$1" . + +docker run -it -w /code dev-image ./netdata-installer.sh --dont-wait --dont-start-it --install /tmp diff --git a/.travis/create_artifacts.sh b/.travis/create_artifacts.sh new file mode 100755 index 000000000..40ba9c85f --- /dev/null +++ b/.travis/create_artifacts.sh @@ -0,0 +1,33 @@ +#!/bin/bash +# shellcheck disable=SC2230 + +if [ ! -f .gitignore ] +then + echo "Run as ./travis/$(basename "$0") from top level directory of git repository" + exit 1 +fi + +# Make sure stdout is in blocking mode. If we don't, then conda create will barf during downloads. +# See https://github.com/travis-ci/travis-ci/issues/4704#issuecomment-348435959 for details. +python -c 'import os,sys,fcntl; flags = fcntl.fcntl(sys.stdout, fcntl.F_GETFL); fcntl.fcntl(sys.stdout, fcntl.F_SETFL, flags&~os.O_NONBLOCK);' +echo "--- Create tarball ---" +autoreconf -ivf +./configure +make dist +echo "--- Create self-extractor ---" +./makeself/build-x86_64-static.sh + +echo "--- Create checksums ---" +GIT_TAG=$(git tag --points-at) +if [ "${GIT_TAG}" != "" ]; then + ln -s netdata-latest.gz.run "netdata-${GIT_TAG}.gz.run" + ln -s netdata-*.tar.gz "netdata-${GIT_TAG}.tar.gz" + sha256sum -b "netdata-${GIT_TAG}.gz.run" "netdata-${GIT_TAG}.tar.gz" > "sha256sums.txt" +else + sha256sum -b ./*.tar.gz ./*.gz.run > "sha256sums.txt" +fi + +echo "checksums:" +cat sha256sums.txt + + diff --git a/.travis/deploy-if-have-key b/.travis/deploy-if-have-key index 1933eeb2e..8b3b40f7e 100755 --- a/.travis/deploy-if-have-key +++ b/.travis/deploy-if-have-key @@ -46,6 +46,10 @@ fi ssh-keyscan -H firehol.org >> ~/.ssh/known_hosts ssh travis@firehol.org mkdir -p uploads/netdata/$TRAVIS_BRANCH/ -scp -p *.tar.* travis@firehol.org:uploads/netdata/$TRAVIS_BRANCH/ -scp -p *.gz.run* travis@firehol.org:uploads/netdata/$TRAVIS_BRANCH/ +scp -p *.tar.gz travis@firehol.org:uploads/netdata/$TRAVIS_BRANCH/ +scp -p *.tar.gz.sha travis@firehol.org:uploads/netdata/$TRAVIS_BRANCH/ +scp -p *.tar.gz.asc travis@firehol.org:uploads/netdata/$TRAVIS_BRANCH/ +scp -p *.gz.run travis@firehol.org:uploads/netdata/$TRAVIS_BRANCH/ +scp -p *.gz.run.sha travis@firehol.org:uploads/netdata/$TRAVIS_BRANCH/ +scp -p *.gz.run.asc travis@firehol.org:uploads/netdata/$TRAVIS_BRANCH/ ssh travis@firehol.org touch uploads/netdata/$TRAVIS_BRANCH/complete.txt diff --git a/.travis/firehol_create_artifacts.sh b/.travis/firehol_create_artifacts.sh new file mode 100755 index 000000000..3fcb910e7 --- /dev/null +++ b/.travis/firehol_create_artifacts.sh @@ -0,0 +1,31 @@ +#!/bin/bash +# shellcheck disable=SC2230 + +# WARNING: This script is deprecated and placed here until @paulfantom figures out how to fully replace it + +if [ ! -f .gitignore ] +then + echo "Run as ./travis/$(basename "$0") from top level directory of git repository" + exit 1 +fi + +eval "$(ssh-agent -s)" +./.travis/decrypt-if-have-key decb6f6387c4 +export KEYSERVER=ipv4.pool.sks-keyservers.net +./packaging/gpg-recv-key phil@firehol.org "0762 9FF7 89EA 6156 012F 9F50 C406 9602 1359 9237" +./packaging/gpg-recv-key costa@tsaousis.gr "4DFF 624A E564 3B51 2872 1F40 29CA 3358 89B9 A863" +# Run the commit hooks in case the developer didn't +git diff 4b825dc642cb6eb9a060e54bf8d69288fbee4904 | ./packaging/check-files - +fakeroot ./packaging/git-build +# Make sure stdout is in blocking mode. If we don't, then conda create will barf during downloads. +# See https://github.com/travis-ci/travis-ci/issues/4704#issuecomment-348435959 for details. +python -c 'import os,sys,fcntl; flags = fcntl.fcntl(sys.stdout, fcntl.F_GETFL); fcntl.fcntl(sys.stdout, fcntl.F_SETFL, flags&~os.O_NONBLOCK);' +echo "--- Create tarball ---" +make dist +echo "--- Create self-extractor ---" +./makeself/build-x86_64-static.sh +echo "--- Create checksums ---" +for i in *.tar.gz; do sha512sum -b "$i" > "$i.sha"; done #FIXME remove? +for i in *.gz.run; do sha512sum -b "$i" > "$i.sha"; done #FIXME remove? +sha256sum -b ./*.tar.gz ./*.gz.run > "sha256sums.txt" +./.travis/deploy-if-have-key diff --git a/.travis/generate_changelog.sh b/.travis/generate_changelog.sh new file mode 100755 index 000000000..bc8be1023 --- /dev/null +++ b/.travis/generate_changelog.sh @@ -0,0 +1,36 @@ +#!/bin/bash + +set -e + +if [ ! -f .gitignore ] +then + echo "Run as ./travis/$(basename "$0") from top level directory of git repository" + exit 1 +fi + +ORGANIZATION=$(echo "$TRAVIS_REPO_SLUG" | awk -F '/' '{print $1}') +PROJECT=$(echo "$TRAVIS_REPO_SLUG" | awk -F '/' '{print $2}') +GIT_MAIL="pawel+bot@netdata.cloud" +GIT_USER="netdatabot" + +echo "--- Initialize git configuration ---" +git config user.email "${GIT_MAIL}" +git config user.name "${GIT_USER}" + +echo "--- Creating changelog ---" +git checkout master +git pull +#docker run -it --rm -v "$(pwd)":/usr/local/src/your-app ferrarimarco/github-changelog-generator:1.14.3 \ +docker run -it -v "$(pwd)":/project markmandel/github-changelog-generator:latest \ + --user "${ORGANIZATION}" \ + --project "${PROJECT}" \ + --token "${GITHUB_TOKEN}" \ + --since-tag "v1.10.0" \ + --unreleased-label "**Next release**" \ + --no-compare-link \ + --exclude-labels duplicate,question,invalid,wontfix,discussion,documentation + +echo "--- Uploading changelog ---" +git add CHANGELOG.md +git commit -m '[ci skip] Automatic changelog update' +git push "https://${GITHUB_TOKEN}:@$(git config --get remote.origin.url | sed -e 's/^https:\/\///')" diff --git a/.travis/images/Dockerfile.alpine b/.travis/images/Dockerfile.alpine new file mode 100644 index 000000000..15f527165 --- /dev/null +++ b/.travis/images/Dockerfile.alpine @@ -0,0 +1,5 @@ +FROM alpine:latest + +RUN apk add bash gcc make autoconf automake pkgconfig zlib-dev libuuid git libmnl-dev util-linux-dev build-base + +COPY . /code diff --git a/.travis/images/Dockerfile.centos6 b/.travis/images/Dockerfile.centos6 new file mode 100644 index 000000000..c286fb95c --- /dev/null +++ b/.travis/images/Dockerfile.centos6 @@ -0,0 +1,5 @@ +FROM centos:6 + +RUN yum install -y gcc make autoconf automake pkg-config zlib-devel libuuid-devel git + +COPY . /code diff --git a/.travis/images/Dockerfile.centos7 b/.travis/images/Dockerfile.centos7 new file mode 100644 index 000000000..d94359dec --- /dev/null +++ b/.travis/images/Dockerfile.centos7 @@ -0,0 +1,5 @@ +FROM centos:7 + +RUN yum install -y gcc make autoconf automake pkg-config zlib-devel libuuid-devel git + +COPY . /code diff --git a/.travis/images/Dockerfile.ubuntu1804 b/.travis/images/Dockerfile.ubuntu1804 new file mode 100644 index 000000000..30ac7cf71 --- /dev/null +++ b/.travis/images/Dockerfile.ubuntu1804 @@ -0,0 +1,6 @@ +FROM ubuntu:18.04 + +RUN apt-get update && \ + apt-get install -y gcc make autoconf automake pkg-config zlib1g-dev uuid-dev git + +COPY . /code diff --git a/.travis/releaser.sh b/.travis/releaser.sh new file mode 100755 index 000000000..9f7ecd4ee --- /dev/null +++ b/.travis/releaser.sh @@ -0,0 +1,83 @@ +#!/bin/bash +# SPDX-License-Identifier: MIT +# Copyright (C) 2018 Pawel Krupa (@paulfantom) - All Rights Reserved +# Permission to copy and modify is granted under the MIT license +# +# Original script is available at https://github.com/paulfantom/travis-helper/blob/master/releasing/releaser.sh +# +# Script to automatically do a couple of things: +# - generate a new tag according to semver (https://semver.org/) +# - generate CHANGELOG.md by using https://github.com/skywinder/github-changelog-generator +# - create draft of GitHub releases by using https://github.com/github/hub +# +# Tags are generated by searching for a keyword in last commit message. Keywords are: +# - [patch] or [fix] to bump patch number +# - [minor], [feature] or [feat] to bump minor number +# - [major] or [breaking change] to bump major number +# All keywords MUST be surrounded with square braces. +# +# Script uses git mechanisms for locking, so it can be used in parallel builds +# +# Requirements: +# - GITHUB_TOKEN variable set with GitHub token. Access level: repo.public_repo +# - docker +# - git-semver python package (pip install git-semver) + +set -e + +if [ ! -f .gitignore ] +then + echo "Run as ./travis/$(basename "$0") from top level directory of git repository" + exit 1 +fi + +echo "---- GENERATING CHANGELOG -----" +./.travis/generate_changelog.sh + +echo "---- FIGURING OUT TAGS ----" +# Check if current commit is tagged or not +GIT_TAG=$(git tag --points-at) +if [ -z "${GIT_TAG}" ]; then + git semver + # Figure out next tag based on commit message + GIT_TAG=HEAD + echo "Last commit message: $TRAVIS_COMMIT_MESSAGE" + case "${TRAVIS_COMMIT_MESSAGE}" in + *"[netdata patch release]"* ) GIT_TAG="v$(git semver --next-patch)" ;; + *"[netdata minor release]"* ) GIT_TAG="v$(git semver --next-minor)" ;; + *"[netdata major release]"* ) GIT_TAG="v$(git semver --next-major)" ;; + *) echo "Keyword not detected. Doing nothing" ;; + esac + + # Tag it! + if [ "$GIT_TAG" != "HEAD" ]; then + echo "Assigning a new tag: $GIT_TAG" + git tag "$GIT_TAG" -a -m "Automatic tag generation for travis build no. $TRAVIS_BUILD_NUMBER" + # git is able to push due to configuration already being initialized in `generate_changelog.sh` script + git push "https://${GITHUB_TOKEN}:@$(git config --get remote.origin.url | sed -e 's/^https:\/\///')" --tags + fi +fi + +if [ "${GIT_TAG}" == "HEAD" ]; then + echo "Not creating a release since neither of two conditions was met:" + echo " - keyword in commit message" + echo " - commit is tagged" + exit 0 +fi + +echo "---- CREATING TAGGED DOCKER CONTAINERS ----" +export REPOSITORY="netdata/netdata" +./docker/build.sh + +echo "---- CREATING RELEASE ARTIFACTS -----" +./.travis/create_artifacts.sh + +echo "---- CREATING RELEASE DRAFT WITH ASSETS -----" +# Download hub +HUB_VERSION=${HUB_VERSION:-"2.5.1"} +wget "https://github.com/github/hub/releases/download/v${HUB_VERSION}/hub-linux-amd64-${HUB_VERSION}.tgz" -O "/tmp/hub-linux-amd64-${HUB_VERSION}.tgz" +tar -C /tmp -xvf "/tmp/hub-linux-amd64-${HUB_VERSION}.tgz" +export PATH=$PATH:"/tmp/hub-linux-amd64-${HUB_VERSION}/bin" + +# Create a release draft +hub release create --draft -a "netdata-${GIT_TAG}.tar.gz" -a "netdata-${GIT_TAG}.gz.run" -a "sha256sums.txt" -m "${GIT_TAG}" "${GIT_TAG}" -- cgit v1.2.3