Merging upstream version 1.18.0.

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

11 files changed, 177 insertions, 104 deletions

diff --git a/.travis/generate_changelog_for_release.sh b/.travis/create_changelog.sh
index d7066aa9..8d4c12ee 100755
--- a/.travis/generate_changelog_for_release.sh
+++ b/.travis/create_changelog.sh
@@ -1,16 +1,24 @@
-#!/bin/bash
-
+#!/usr/bin/env bash
+#
+# Copyright: SPDX-License-Identifier: GPL-3.0-or-later
+#
+# Author  : Pavlos Emm. Katsoulakis (paul@netdata.cloud)
 set -e
 
-if [ ! -f .gitignore ]; then
-	echo "Run as ./travis/$(basename "$0") from top level directory of git repository"
-	exit 1
+# If we are not in netdata git repo, at the top level directory, fail
+TOP_LEVEL=$(basename "$(git rev-parse --show-toplevel)")
+CWD=$(git rev-parse --show-cdup || echo "")
+if [ -n "$CWD" ] || [ ! "${TOP_LEVEL}" == "netdata" ]; then
+    echo "Run as .travis/$(basename "$0") from top level directory of netdata git repository"
+    echo "Changelog creation aborted"
+    exit 1
 fi
 
 ORGANIZATION=$(echo "$TRAVIS_REPO_SLUG" | awk -F '/' '{print $1}')
 PROJECT=$(echo "$TRAVIS_REPO_SLUG" | awk -F '/' '{print $2}')
 GIT_MAIL=${GIT_MAIL:-"bot@netdata.cloud"}
 GIT_USER=${GIT_USER:-"netdatabot"}
+
 if [ -z ${GIT_TAG+x} ]; then
 	OPTS=""
 else
@@ -18,20 +26,21 @@ else
 fi
 
 if [ ! "${TRAVIS_REPO_SLUG}" == "netdata/netdata" ]; then
-	echo "Beta mode on ${TRAVIS_REPO_SLUG}, nothing else to do"
+	echo "Beta mode on ${TRAVIS_REPO_SLUG}, nothing else to do here"
 	exit 0
 fi
 
 echo "--- Creating changelog ---"
 git checkout master
 git pull
-#docker run -it --rm -v "$(pwd)":/usr/local/src/your-app ferrarimarco/github-changelog-generator:1.14.3 \
+
 docker run -it -v "$(pwd)":/project markmandel/github-changelog-generator:latest \
 	--user "${ORGANIZATION}" \
 	--project "${PROJECT}" \
 	--token "${GITHUB_TOKEN}" \
 	--since-tag "v1.10.0" \
-	--no-issues \
 	--unreleased-label "**Next release**" \
+	--no-issues \
 	--exclude-labels "stale,duplicate,question,invalid,wontfix,discussion,no changelog" \
-	--no-compare-link ${OPTS}
+	--max-issues 500 \
+	--bug-labels IGNOREBUGS ${OPTS}
diff --git a/.travis/generate_changelog_and_tag_release.sh b/.travis/generate_changelog_and_tag_release.sh
index 4613ab14..bf5555b4 100755
--- a/.travis/generate_changelog_and_tag_release.sh
+++ b/.travis/generate_changelog_and_tag_release.sh
@@ -30,14 +30,11 @@ if [ ! -f .gitignore ]; then
 	exit 1
 fi
 
-echo "--- Executing Tagging facility to determine TAG ---"
-source .travis/tagger.sh
-
-echo "--- Changelog generator and tagger script starting ---"
-# If tagger script hasn't produced a TAG, there is nothing to do so bail out happy
+echo "--- Changelog generator script starting ---"
+# If we dont have a produced TAG there is nothing to do, so bail out happy
 if [ -z "${GIT_TAG}" ]; then
-	echo "GIT_TAG is empty, nothing to do for now (Value: $GIT_TAG)"
-	exit 0
+	echo "GIT_TAG is empty, that is not suppose to happen (Value: $GIT_TAG)"
+	exit 1
 fi
 
 if [ ! "${TRAVIS_REPO_SLUG}" == "netdata/netdata" ]; then
@@ -55,9 +52,9 @@ echo "---- UPDATE VERSION FILE ----"
 echo "$GIT_TAG" >packaging/version
 git add packaging/version
 
-#echo "---- GENERATE CHANGELOG -----"
-#./.travis/generate_changelog_for_release.sh
-#git add CHANGELOG.md
+echo "---- Create CHANGELOG -----"
+./.travis/create_changelog.sh
+git add CHANGELOG.md
 
 echo "---- COMMIT AND PUSH CHANGES ----"
 git commit -m "[ci skip] release $GIT_TAG" --author "${GIT_USER} <${GIT_MAIL}>"
diff --git a/.travis/generate_changelog_for_nightlies.sh b/.travis/generate_changelog_for_nightlies.sh
index b9086288..59173af3 100755
--- a/.travis/generate_changelog_for_nightlies.sh
+++ b/.travis/generate_changelog_for_nightlies.sh
@@ -1,6 +1,6 @@
-#!/bin/bash
+#!/usr/bin/env bash
 #
-# Changelog generation scriptlet.
+# Changelog generation scriptlet, for nightlies
 #
 # Copyright: SPDX-License-Identifier: GPL-3.0-or-later
 #
@@ -20,38 +20,18 @@ fi
 LAST_TAG="$1"
 COMMITS_SINCE_RELEASE="$2"
 NEW_VERSION="${LAST_TAG}-$((COMMITS_SINCE_RELEASE + 1))-nightly"
-ORG=$(echo "$TRAVIS_REPO_SLUG" | cut -d '/' -f1)
-PROJECT=$(echo "$TRAVIS_REPO_SLUG" | cut -d '/' -f 2)
 GIT_MAIL=${GIT_MAIL:-"bot@netdata.cloud"}
 GIT_USER=${GIT_USER:-"netdatabot"}
 PUSH_URL=$(git config --get remote.origin.url | sed -e 's/^https:\/\///')
 FAIL=0
-if [ -z ${GIT_TAG+x} ]; then
-	OPTS=""
-else
-	OPTS="--future-release ${GIT_TAG}"
-fi
-echo "We got $COMMITS_SINCE_RELEASE changes since $LAST_TAG, re-generating changelog"
 
 if [ ! "${TRAVIS_REPO_SLUG}" == "netdata/netdata" ]; then
 	echo "Beta mode on ${TRAVIS_REPO_SLUG}, nothing else to do here"
 	exit 0
 fi
 
-git checkout master
-git pull
-
-echo "Running project markmandel for github changelog generation"
-#docker run -it --rm -v "$(pwd)":/usr/local/src/your-app ferrarimarco/github-changelog-generator:1.14.3 \
-docker run -it -v "$(pwd)":/project markmandel/github-changelog-generator:latest \
-	--user "${ORG}" \
-	--project "${PROJECT}" \
-	--token "${GITHUB_TOKEN}" \
-	--since-tag "v1.10.0" \
-	--unreleased-label "**Next release**" \
-	--no-issues \
-	--exclude-labels "stale,duplicate,question,invalid,wontfix,discussion,no changelog" \
-	--no-compare-link ${OPTS}
+echo "Running changelog creation mechanism"
+.travis/create_changelog.sh
 
 echo "Changelog created! Adding packaging/version(${NEW_VERSION}) and CHANGELOG.md to the repository"
 echo "${NEW_VERSION}" > packaging/version
diff --git a/.travis/nightlies.sh b/.travis/nightlies.sh
index 13a813d7..e60ee279 100755
--- a/.travis/nightlies.sh
+++ b/.travis/nightlies.sh
@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 #
 # This is the nightly changelog generation script
 # It is responsible for two major activities:
@@ -29,7 +29,7 @@ PREVIOUS_NIGHTLY_COUNT="$(rev <packaging/version | cut -d- -f 2 | rev)"
 
 # If no commits since release, just stop
 if [ "${COMMITS_SINCE_RELEASE}" == "${PREVIOUS_NIGHTLY_COUNT}" ]; then
-	echo "No changes since last nighthly release"
+	echo "No changes since last nighthly release, nothing else to do"
 	exit 0
 fi
 
@@ -39,6 +39,7 @@ if [ ! "${TRAVIS_REPO_SLUG}" == "netdata/netdata" ]; then
 fi
 
 echo "--- Running Changelog generation ---"
+echo "We got $COMMITS_SINCE_RELEASE changes since $LAST_TAG, re-generating changelog"
 NIGHTLIES_CHANGELOG_FAILED=0
 .travis/generate_changelog_for_nightlies.sh "${LAST_TAG}" "${COMMITS_SINCE_RELEASE}" || NIGHTLIES_CHANGELOG_FAILED=1
 
diff --git a/.travis/package_management/build.sh b/.travis/package_management/build.sh
index a5217715..bafaecc5 100644
--- a/.travis/package_management/build.sh
+++ b/.travis/package_management/build.sh
@@ -24,9 +24,9 @@ ln -sf contrib/debian debian
 echo "Executing dpkg-buildpackage"
 # pre/post options are after 1.18.8, is simpler to just check help for their existence than parsing version
 if dpkg-buildpackage --help | grep "\-\-post\-clean" 2> /dev/null > /dev/null; then
-	dpkg-buildpackage --post-clean --pre-clean --build=binary
+	dpkg-buildpackage --post-clean --pre-clean --build=binary -us -uc
 else
-	dpkg-buildpackage -b
+	dpkg-buildpackage -b -us -uc
 fi
 
 echo "DEB build script completed!"
diff --git a/.travis/package_management/build_judy.sh b/.travis/package_management/build_judy.sh
new file mode 100755
index 00000000..202ea044
--- /dev/null
+++ b/.travis/package_management/build_judy.sh
@@ -0,0 +1,36 @@
+#!/usr/bin/env bash
+#
+# Build Judy from source, you need to run this script as root.
+#
+# Copyright: SPDX-License-Identifier: GPL-3.0-or-later
+#
+# Author  : Pavlos Emm. Katsoulakis (paul@netdata.cloud)
+set -e
+JUDY_VER="1.0.5"
+JUDY_DIR="/opt/judy-${JUDY_VER}"
+
+# If we are not in netdata git repo, at the top level directory, fail
+TOP_LEVEL=$(basename "$(git rev-parse --show-toplevel)")
+CWD=$(git rev-parse --show-cdup)
+if [ -n "$CWD" ] || [ ! "${TOP_LEVEL}" == "netdata" ]; then
+    echo "Run as .travis/package_management/$(basename "$0") from top level directory of netdata git repository"
+    echo "Build Judy package from source code failed"
+    exit 1
+fi
+
+echo "Fetching judy source tarball"
+wget -O /opt/judy.tar.gz http://downloads.sourceforge.net/project/judy/judy/Judy-${JUDY_VER}/Judy-${JUDY_VER}.tar.gz
+
+echo "Entering /opt directory and extracting tarball"
+cd /opt && tar -xf judy.tar.gz && rm judy.tar.gz
+
+echo "Entering ${JUDY_DIR}"
+cd "${JUDY_DIR}"
+
+echo "Running configure"
+CFLAGS="-O2 -s" CXXFLAGS="-O2 -s" ./configure
+
+echo "Compiling and installing"
+make && make install
+
+echo "Done, enjoy Judy!"
diff --git a/.travis/package_management/build_libuv.sh b/.travis/package_management/build_libuv.sh
new file mode 100755
index 00000000..c30eede6
--- /dev/null
+++ b/.travis/package_management/build_libuv.sh
@@ -0,0 +1,36 @@
+#!/usr/bin/env bash
+#
+# Build libuv from source, you need to run this script as root.
+#
+# Copyright: SPDX-License-Identifier: GPL-3.0-or-later
+#
+# Author  : Pavlos Emm. Katsoulakis <paul@netdata.cloud>
+set -e
+LIBUV_VERSION="v1.32.0"
+# Their folder is libuv-1.32.0 while the tarball version is v1.32.0, so fix that until they fix it...
+LIBUV_DIR="/opt/libuv-${LIBUV_VERSION/v/}"
+
+# If we are not in netdata git repo, at the top level directory, fail
+TOP_LEVEL=$(basename "$(git rev-parse --show-toplevel)")
+CWD=$(git rev-parse --show-cdup)
+if [ -n "$CWD" ] || [ ! "${TOP_LEVEL}" == "netdata" ]; then
+    echo "Run as .travis/package_management/$(basename "$0") from top level directory of netdata git repository"
+    echo "Build libuv package from source code failed"
+    exit 1
+fi
+
+echo "Fetching libuv from github"
+wget -O /opt/libuv.tar.gz "https://github.com/libuv/libuv/archive/${LIBUV_VERSION}.tar.gz"
+
+echo "Entering /opt and extracting source"
+cd /opt && tar -xf libuv.tar.gz && rm libuv.tar.gz
+
+echo "Entering ${LIBUV_DIR}"
+cd "${LIBUV_DIR}"
+
+echo "Compiling and installing"
+sh autogen.sh
+./configure
+make && make check && make install
+
+echo "Done, enjoy libuv!"
diff --git a/.travis/package_management/common.py b/.travis/package_management/common.py
index a72fe483..8a510697 100755
--- a/.travis/package_management/common.py
+++ b/.travis/package_management/common.py
@@ -7,6 +7,9 @@
 import lxc
 import subprocess
 import os
+import sys
+import tempfile
+import shutil
 
 def fetch_version(orig_build_version):
     tag = None
@@ -53,10 +56,10 @@ def run_command(container, command):
     if command_result != 0:
         raise Exception("Command failed with exit code %d" % command_result)
 
-def run_command_in_host(cmd):
-    print("Issue command in host: %s" % str(cmd))
+def run_command_in_host(cmd, cwd=None):
+    print("Issue command in host: %s, cwd:%s" % (str(cmd), str(cwd)))
 
-    proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=cwd)
     o, e = proc.communicate()
     print('Output: ' + o.decode('ascii'))
     print('Error: '  + e.decode('ascii'))
@@ -133,31 +136,41 @@ def prepare_version_source(dest_archive, pkg_friendly_version, tag=None):
     print(".0 Preparing local implementation tarball for version %s" % pkg_friendly_version)
     tar_file = os.environ['LXC_CONTAINER_ROOT'] + dest_archive
 
+    print(".0 Copy repo to prepare it for tarball generation")
+    tmp_src = tempfile.mkdtemp(prefix='netdata-source-')
+    run_command_in_host(['cp', '-r', '.', tmp_src])
+
     if tag is not None:
         print(".1 Checking out tag %s" % tag)
-        run_command_in_host(['git', 'fetch', '--all'])
+        run_command_in_host(['git', 'fetch', '--all'], tmp_src)
 
         # TODO: Keep in mind that tricky 'v' there, needs to be removed once we clear our versioning scheme
-        run_command_in_host(['git', 'checkout', 'v%s' % pkg_friendly_version])
+        run_command_in_host(['git', 'checkout', 'v%s' % pkg_friendly_version], tmp_src)
 
     print(".2 Tagging the code with version: %s" % pkg_friendly_version)
-    run_command_in_host(['git', 'tag', '-a', pkg_friendly_version, '-m', 'Tagging while packaging on %s' % os.environ["CONTAINER_NAME"]])
+    run_command_in_host(['git', 'tag', '-a', pkg_friendly_version, '-m', 'Tagging while packaging on %s' % os.environ["CONTAINER_NAME"]], tmp_src)
 
     print(".3 Run autoreconf -ivf")
-    run_command_in_host(['autoreconf', '-ivf'])
+    run_command_in_host(['autoreconf', '-ivf'], tmp_src)
 
     print(".4 Run configure")
-    run_command_in_host(['./configure', '--prefix=/usr', '--sysconfdir=/etc', '--localstatedir=/var', '--libdir=/usr/lib', '--libexecdir=/usr/libexec', '--with-math', '--with-zlib', '--with-user=netdata'])
+    run_command_in_host(['./configure', '--prefix=/usr', '--sysconfdir=/etc', '--localstatedir=/var', '--libdir=/usr/lib', '--libexecdir=/usr/libexec', '--with-math', '--with-zlib', '--with-user=netdata'], tmp_src)
 
     print(".5 Run make dist")
-    run_command_in_host(['make', 'dist'])
+    run_command_in_host(['make', 'dist'], tmp_src)
 
     print(".6 Copy generated tarbal to desired path")
-    if os.path.exists('netdata-%s.tar.gz' % pkg_friendly_version):
-        run_command_in_host(['sudo', 'cp', 'netdata-%s.tar.gz' % pkg_friendly_version, tar_file])
+    generated_tarball = '%s/netdata-%s.tar.gz' % (tmp_src, pkg_friendly_version)
+
+    if os.path.exists(generated_tarball):
+        run_command_in_host(['sudo', 'cp', generated_tarball, tar_file])
 
         print(".7 Fixing permissions on tarball")
         run_command_in_host(['sudo', 'chmod', '777', tar_file])
+
+        print(".8 Bring over netdata.spec, Remove temp directory");
+        run_command_in_host(['cp', '%s/netdata.spec' % tmp_src, 'netdata.spec'])
+        shutil.rmtree(tmp_src)
     else:
-        print("I could not find (%s) on the disk, stopping the build. Kindly check the logs and try again" % 'netdata-%s.tar.gz' % pkg_friendly_version)
+        print("I could not find (%s) on the disk, stopping the build. Kindly check the logs and try again" % generated_tarball)
         sys.exit(1)
diff --git a/.travis/package_management/trigger_deb_lxc_build.py b/.travis/package_management/trigger_deb_lxc_build.py
index 46b64e6a..464a7715 100755
--- a/.travis/package_management/trigger_deb_lxc_build.py
+++ b/.travis/package_management/trigger_deb_lxc_build.py
@@ -57,8 +57,13 @@ print("Checking version consistency")
 since_version = os.environ["LATEST_RELEASE_VERSION"]
 if str(since_version).replace('v', '') == str(new_version) and str(new_version).count('.') == 2:
     s = since_version.split('.')
-    prev = str(int(s[1]) - 1)
-    since_version = s[0] + '.' + prev + s[2]
+    if int(s[2]) > 0:
+        patch_prev = str(int(s[2]) - 1)
+        since_version = s[0] + '.' + s[1] + '.'  + patch_prev
+    else:
+        prev = str(int(s[1]) - 1)
+        since_version = s[0] + '.' + prev + '.' + s[2]
+
     print("We seem to be building a new stable release, reduce by one since_version option. New since_version:%s" % since_version)
 
 print("Fixing changelog tags")
diff --git a/.travis/tagger.sh b/.travis/tagger.sh
index adac35b4..a775a826 100755
--- a/.travis/tagger.sh
+++ b/.travis/tagger.sh
@@ -1,6 +1,4 @@
-#!/bin/bash
-#
-# Original script is available at https://github.com/paulfantom/travis-helper/blob/master/releasing/releaser.sh
+# #BASH library
 #
 # Tags are generated by searching for a keyword in last commit message. Keywords are:
 #  - [patch] or [fix] to bump patch number
@@ -12,8 +10,6 @@
 #   - GITHUB_TOKEN variable set with GitHub token. Access level: repo.public_repo
 #   - git-semver python package (pip install git-semver)
 #
-# Note: Exported variables needed by .travis/draft_release.sh
-#
 # Original script is available at https://github.com/paulfantom/travis-helper/blob/master/releasing/releaser.sh
 #
 # Copyright: SPDX-License-Identifier: GPL-3.0-or-later
@@ -21,24 +17,11 @@
 # Author  : Pawel Krupa (paulfantom)
 # Author  : Pavlos Emm. Katsoulakis (paul@netdata.cloud)
 
-set -e
-
-
-# If we are not in netdata git repo, at the top level directory, fail
-TOP_LEVEL=$(basename "$(git rev-parse --show-toplevel)")
-CWD=$(git rev-parse --show-cdup || echo "")
-if [ -n "${CWD}" ] || [ ! "${TOP_LEVEL}" == "netdata" ]; then
-    echo "Run as .travis/$(basename "$0") from top level directory of netdata git repository"
-    echo "Changelog generation process aborted"
-    exit 1
-fi
-
-
 # Figure out what will be new release candidate tag based only on previous ones.
 # This assumes that RELEASES are in format of "v0.1.2" and prereleases (RCs) are using "v0.1.2-rc0"
 function set_tag_release_candidate() {
 	LAST_TAG=$(git semver)
-	echo "Last tag found is: ${LAST_TAG}"
+	echo "${0}: Last tag found is: ${LAST_TAG}"
 
 	if [[ $LAST_TAG =~ -rc* ]]; then
 		VERSION=$(echo "$LAST_TAG" | cut -d'-' -f 1)
@@ -47,31 +30,32 @@ function set_tag_release_candidate() {
 	else
 		VERSION="$(git semver --next-minor)"
 		RC=0
-		echo "Warning: Will set version to ${VERSION} (Last tag: ${LAST_TAG}) while tagged for release candidate generation"
+		echo "${0}: Warning: Will set version to ${VERSION} (Last tag: ${LAST_TAG}) while tagged for release candidate generation"
 	fi
+
 	GIT_TAG="v${VERSION}-rc${RC}"
+	echo "${0}: Generated a new tag, set to: (${GIT_TAG})"
 }
 
-echo "Determining TAG"
-
-# Check if current commit is tagged or not
-GIT_TAG=$(git tag --points-at)
-
-if [ -z "${GIT_TAG}" ]; then
-	git semver
-	# Figure out next tag based on commit message
-	echo "Last commit message: ${TRAVIS_COMMIT_MESSAGE}"
-	case "${TRAVIS_COMMIT_MESSAGE}" in
-	*"[netdata patch release]"*) GIT_TAG="v$(git semver --next-patch)" ;;
-	*"[netdata minor release]"*) GIT_TAG="v$(git semver --next-minor)" ;;
-	*"[netdata major release]"*) GIT_TAG="v$(git semver --next-major)" ;;
-	*"[netdata release candidate]"*) set_tag_release_candidate ;;
-	*)
-		echo "Keyword not detected. Exiting..."
-		exit 0
-		;;
-	esac
-fi
-
-echo "Setting up GIT_TAG to ${GIT_TAG}"
-export GIT_TAG
+function set_tag_for_release() {
+	echo "${0}: Checking for tag existence"
+	if [ -z "${GIT_TAG}" ]; then
+		echo "${0}: No tag was found, generating a new tag"
+		git semver
+
+		echo "${0}: Last commit message: ${TRAVIS_COMMIT_MESSAGE}"
+
+		# Figure out next tag based on commit message
+		case "${TRAVIS_COMMIT_MESSAGE}" in
+		*"[netdata patch release]"*) GIT_TAG="v$(git semver --next-patch)" ;;
+		*"[netdata minor release]"*) GIT_TAG="v$(git semver --next-minor)" ;;
+		*"[netdata major release]"*) GIT_TAG="v$(git semver --next-major)" ;;
+		*"[netdata release candidate]"*) set_tag_release_candidate ;;
+		*)
+			echo "${0}: Keyword not detected. Nothing to set for GIT_TAG"
+			;;
+		esac
+	else
+		echo "${0}: We seem to already have a GIT_TAG set to (${GIT_TAG})"
+	fi
+}
diff --git a/.travis/trigger_package_generation.sh b/.travis/trigger_package_generation.sh
index aee69be5..713679c1 100755
--- a/.travis/trigger_package_generation.sh
+++ b/.travis/trigger_package_generation.sh
@@ -45,7 +45,19 @@ push_change
 echo "---- Waiting for ${WAIT_TIME} seconds before triggering next process ----"
 sleep "${WAIT_TIME}"
 
+commit_change "i386" "DEB"
+push_change
+
+echo "---- Waiting for ${WAIT_TIME} seconds before triggering next process ----"
+sleep "${WAIT_TIME}"
+
 commit_change "amd64" "RPM"
 push_change
 
+echo "---- Waiting for ${WAIT_TIME} seconds before triggering next process ----"
+sleep "${WAIT_TIME}"
+
+commit_change "i386" "RPM"
+push_change
+
 echo "---- Done! ----"