diff options
Diffstat (limited to '')
18 files changed, 36 insertions, 1248 deletions
diff --git a/.travis/package_management/old_package_purging.sh b/.github/scripts/old_package_purging.sh index 89ecd753f..c90c4b780 100755 --- a/.travis/package_management/old_package_purging.sh +++ b/.github/scripts/old_package_purging.sh @@ -26,7 +26,7 @@ delete_files_for_version() { pkg=${pkg/\"/} pkg=${pkg/\"/} echo "Attempting yank on ${pkg}.." - .travis/package_management/package_cloud_wrapper.sh yank "${PACKAGING_USER}/${DEPLOY_REPO}" "${pkg}" || echo "Nothing to yank or error on ${pkg}" + .github/scripts/package_cloud_wrapper.sh yank "${REPO}" "${pkg}" || echo "Nothing to yank or error on ${pkg}" done } @@ -34,18 +34,13 @@ delete_files_for_version() { TOP_LEVEL=$(basename "$(git rev-parse --show-toplevel)") CWD=$(git rev-parse --show-cdup) if [ -n "$CWD" ] || [ ! "${TOP_LEVEL}" == "netdata" ]; then - echo "Run as .travis/package_management/$(basename "$0") from top level directory of netdata git repository" + echo "Run as .github/scripts/$(basename "$0") from top level directory of netdata git repository" echo "Old packages yanking cancelled" exit 1 fi -if [ -z "${PACKAGING_USER}" ]; then - echo "No PACKAGING_USER variable found" - exit 1 -fi - -if [ -z "${DEPLOY_REPO}" ]; then - echo "No DEPLOY_REPO variable found" +if [ -z "${REPO}" ]; then + echo "No REPO variable found" exit 1 fi @@ -68,8 +63,8 @@ DATE_UNTIL_TO_DELETE=$(date --date="${PACKAGE_CLOUD_RETENTION_DAYS} day ago" +%Y echo "Created temp directory: ${TMP_DIR}" echo "We will be purging contents up until ${DATE_UNTIL_TO_DELETE}" -echo "Calling package could to retrieve all available packages on ${PACKAGING_USER}/${DEPLOY_REPO}" -curl -sS "https://${PKG_CLOUD_TOKEN}:@packagecloud.io/api/v1/repos/${PACKAGING_USER}/${DEPLOY_REPO}/packages.json" > "${PKG_LIST_FILE}" +echo "Calling package could to retrieve all available packages on ${REPO}" +curl -sS "https://${PKG_CLOUD_TOKEN}:@packagecloud.io/api/v1/repos/${REPO}/packages.json" > "${PKG_LIST_FILE}" # Get versions within the desired duration # diff --git a/.travis/package_management/package_cloud_wrapper.sh b/.github/scripts/package_cloud_wrapper.sh index 48a372d37..7640ef484 100755 --- a/.travis/package_management/package_cloud_wrapper.sh +++ b/.github/scripts/package_cloud_wrapper.sh @@ -21,7 +21,7 @@ PKG_CLOUD_CONFIG="$HOME/.package_cloud_configuration.cfg" TOP_LEVEL=$(basename "$(git rev-parse --show-toplevel)") CWD=$(git rev-parse --show-cdup) if [ -n "$CWD" ] || [ ! "${TOP_LEVEL}" == "netdata" ]; then - echo "Run as .travis/package_management/$(basename "$0") from top level directory of netdata git repository" + echo "Run as .github/scripts/$(basename "$0") from top level directory of netdata git repository" echo "Docker build process aborted" exit 1 fi @@ -29,7 +29,7 @@ fi # Install dependency if not there if ! command -v package_cloud > /dev/null 2>&1; then echo "No package cloud gem found, installing" - gem install -V package_cloud || (echo "Package cloud installation failed. you might want to check if required dependencies are there (ruby gcc gcc-c++ ruby-devel)" && exit 1) + sudo gem install -V package_cloud || (echo "Package cloud installation failed. you might want to check if required dependencies are there (ruby gcc gcc-c++ ruby-devel)" && exit 1) else echo "Found package_cloud gem, continuing" fi diff --git a/.travis.yml b/.travis.yml index 8c95f5d1d..d5cf8ea10 100644 --- a/.travis.yml +++ b/.travis.yml @@ -56,8 +56,6 @@ stages: - name: Nightly release if: branch = master AND type = cron AND env(RUN_NIGHTLY) = yes - - name: Trigger deb and rpm package build (nightly release) - if: branch = master AND type = cron AND env(RUN_NIGHTLY) = yes # Scheduled releases - name: Support activities on main branch @@ -71,21 +69,6 @@ stages: AND type != cron AND tag !~ /(-rc)/ AND commit_message =~ /\[netdata (release candidate|(major|minor|patch) release)\]/ - - name: Trigger deb and rpm package build (release) - if: >- - branch = master - AND type != pull_request - AND type != cron - AND tag !~ /(-rc)/ - AND commit_message =~ /\[netdata (release candidate|(major|minor|patch) release)\]/ - - # Build DEB packages under special conditions - - name: Package ubuntu/* and debian/* - if: type != cron AND type != pull_request AND branch = master - - # Build RPM packages under special conditions - - name: Package centos, fedora and opensuse - if: type != cron AND type != pull_request AND branch = master # Define stage implementation details @@ -103,57 +86,6 @@ jobs: env: CFLAGS='-O1 -Wall -Wextra -Wformat-signedness -fstack-protector-all -fno-common -DNETDATA_INTERNAL_CHECKS=1 -D_FORTIFY_SOURCE=2 -DNETDATA_VERIFY_LOCKS=1' after_failure: post_message "TRAVIS_MESSAGE" "<!here> standard netdata build is failing (Still dont know which one, will improve soon)" - - name: Build/Install for ubuntu 20.04 (not containerized) - script: fakeroot ./netdata-installer.sh --dont-wait --dont-start-it --install $HOME - after_failure: post_message "TRAVIS_MESSAGE" "Build/Install failed on ubuntu 18.04" - - - name: Build/install for CentOS 7 (Containerized) - script: docker run -it -v "${PWD}:/netdata:rw" -w /netdata "netdata/os-test:centos7" ./netdata-installer.sh --dont-wait --dont-start-it --install /tmp - after_failure: post_message "TRAVIS_MESSAGE" "Build/Install failed on CentOS 7" - - - name: DEB package test - git: - depth: false - before_install: - - sudo apt-get install -y wget lxc python3-lxc lxc-templates dh-make git-buildpackage build-essential libdistro-info-perl - before_script: - - export PACKAGES_DIRECTORY="$(mktemp -d -t netdata-packaging-contents-dir-XXXXXX)" && echo "Created packaging directory ${PACKAGES_DIRECTORY}" - script: - - echo "GIT Branch:" && git branch - - echo "Last commit:" && git log -1 - - echo "GIT Describe:" && git describe - - echo "packaging/version:" && cat packaging/version - - echo "Creating LXC environment for the build" && sudo -E .travis/package_management/create_lxc_for_build.sh - - echo "Building package in container" && sudo -E .travis/package_management/build_package_in_container.sh - - sudo chown -R root:travis "/var/lib/lxc" - - sudo chmod -R 750 "/var/lib/lxc" - - echo "Preparing DEB packaging contents for upload" && sudo -E .travis/package_management/prepare_packages.sh - env: - - BUILDER_NAME="builder" BUILD_DISTRO="debian" BUILD_RELEASE="buster" BUILD_STRING="debian/buster" - - PACKAGE_TYPE="deb" REPO_TOOL="apt-get" - after_failure: post_message "TRAVIS_MESSAGE" "Netdata DEB package build check failed." - - - name: RPM package test - git: - depth: false - before_install: - - sudo apt-get install -y wget lxc lxc-templates python3-lxc - before_script: - - export PACKAGES_DIRECTORY="$(mktemp -d -t netdata-packaging-contents-dir-XXXXXX)" && echo "Created packaging directory ${PACKAGES_DIRECTORY}" - script: - - echo "GIT Branch:" && git branch - - echo "Last commit:" && git log -1 - - echo "GIT Describe:" && git describe - - echo "packaging/version:" && cat packaging/version - - echo "Creating LXC environment for the build" && sudo -E .travis/package_management/create_lxc_for_build.sh - - echo "Building package in container" && sudo -E .travis/package_management/build_package_in_container.sh - - sudo chmod -R 755 "/var/lib/lxc" - - echo "Preparing RPM packaging contents for upload" && sudo -E .travis/package_management/prepare_packages.sh - env: - - BUILDER_NAME="builder" BUILD_DISTRO="fedora" BUILD_RELEASE="32" BUILD_STRING="fedora/32" - - PACKAGE_TYPE="rpm" REPO_TOOL="dnf" - after_failure: post_message "TRAVIS_MESSAGE" "Netdata RPM package build check failed." - - stage: Support activities on main branch name: Generate changelog for release (only on special and tagged commit msg) before_script: post_message "TRAVIS_MESSAGE" "Support activities on main branch initiated" "${NOTIF_CHANNEL}" @@ -169,172 +101,6 @@ jobs: depth: false if: commit_message =~ /\[netdata (release candidate|(major|minor|patch) release)\]/ AND tag !~ /(-rc)/ OR (env(GIT_TAG) IS present AND NOT env(GIT_TAG) IS blank) - # ###### Packaging workflow section ###### - # References: - # https://us.images.linuxcontainers.org - # https://packagecloud.io/docs#install_repo - - # TODO: This section is stale, will be aligned with the RPM implementation when we get to DEB packaging - - stage: Package ubuntu/* and debian/* - _template: &DEB_TEMPLATE - git: - depth: false - before_install: - - sudo apt-get install -y wget lxc python3-lxc lxc-templates dh-make git-buildpackage build-essential libdistro-info-perl - - source tests/installer/slack.sh - before_script: - - export PACKAGES_DIRECTORY="$(mktemp -d -t netdata-packaging-contents-dir-XXXXXX)" && echo "Created packaging directory ${PACKAGES_DIRECTORY}" - script: - - echo "GIT Branch:" && git branch - - echo "Last commit:" && git log -1 - - echo "GIT Describe:" && git describe - - echo "packaging/version:" && cat packaging/version - - echo "Creating LXC environment for the build" && sudo -E .travis/package_management/create_lxc_for_build.sh - - echo "Building package in container" && sudo -E .travis/package_management/build_package_in_container.sh - - sudo chown -R root:travis "/var/lib/lxc" - - sudo chmod -R 750 "/var/lib/lxc" - - echo "Preparing DEB packaging contents for upload" && sudo -E .travis/package_management/prepare_packages.sh - after_failure: post_message "TRAVIS_MESSAGE" "Failed to build DEB for ${BUILD_STRING}.${BUILD_ARCH}" - before_deploy: - - .travis/package_management/yank_stale_pkg.sh "${PACKAGES_DIRECTORY}" "${BUILD_STRING}" || echo "No stale DEB found" - deploy: - - provider: packagecloud - repository: "${DEPLOY_REPO}" - username: "${PACKAGING_USER}" - token: "${PKG_CLOUD_TOKEN}" - dist: "${BUILD_STRING}" - local_dir: "${PACKAGES_DIRECTORY}" - skip_cleanup: true - on: - # Only deploy on ${USER}/netdata, master branch, when build-area directory is created - repo: ${TRAVIS_REPO_SLUG} - branch: "master" - condition: -d "${PACKAGES_DIRECTORY}" - after_deploy: - - if [ -n "${BUILDER_NAME}" ]; then rm -rf /home/${BUILDER_NAME}/* && echo "Cleared /home/${BUILDER_NAME} directory" || echo "Failed to clean /home/${BUILDER_NAME} directory"; fi; - - if [ -d "${PACKAGES_DIRECTORY}" ]; then rm -rf "${PACKAGES_DIRECTORY}"; fi; - - name: "Build & Publish DEB package for ubuntu/bionic" - <<: *DEB_TEMPLATE - if: commit_message =~ /\[Package (amd64|arm64|i386) DEB( Ubuntu)?\]/ - env: - - BUILDER_NAME="builder" BUILD_DISTRO="ubuntu" BUILD_RELEASE="bionic" BUILD_STRING="ubuntu/bionic" - - PACKAGE_TYPE="deb" REPO_TOOL="apt-get" - - ALLOW_SOFT_FAILURE_HERE=true - - - name: "Build & Publish DEB package for ubuntu/xenial" - <<: *DEB_TEMPLATE - if: commit_message =~ /\[Package (amd64|arm64|i386) DEB( Ubuntu)?\]/ - env: - - BUILDER_NAME="builder" BUILD_DISTRO="ubuntu" BUILD_RELEASE="xenial" BUILD_STRING="ubuntu/xenial" - - PACKAGE_TYPE="deb" REPO_TOOL="apt-get" - - ALLOW_SOFT_FAILURE_HERE=true - - - name: "Build & Publish DEB package for ubuntu/focal" - <<: *DEB_TEMPLATE - if: commit_message =~ /\[Package (amd64|arm64) DEB( Ubuntu)?\]/ - env: - - BUILDER_NAME="builder" BUILD_DISTRO="ubuntu" BUILD_RELEASE="focal" BUILD_STRING="ubuntu/focal" - - PACKAGE_TYPE="deb" REPO_TOOL="apt-get" - - ALLOW_SOFT_FAILURE_HERE=true - - - name: "Build & Publish DEB package for ubuntu/groovy" - <<: *DEB_TEMPLATE - if: commit_message =~ /\[Package (amd64|arm64) DEB( Ubuntu)?\]/ - env: - - BUILDER_NAME="builder" BUILD_DISTRO="ubuntu" BUILD_RELEASE="groovy" BUILD_STRING="ubuntu/groovy" - - PACKAGE_TYPE="deb" REPO_TOOL="apt-get" - - ALLOW_SOFT_FAILURE_HERE=true - - - name: "Build & Publish DEB package for debian/buster" - <<: *DEB_TEMPLATE - if: commit_message =~ /\[Package (amd64|arm64|i386) DEB( Debian)?\]/ - env: - - BUILDER_NAME="builder" BUILD_DISTRO="debian" BUILD_RELEASE="buster" BUILD_STRING="debian/buster" - - PACKAGE_TYPE="deb" REPO_TOOL="apt-get" - - ALLOW_SOFT_FAILURE_HERE=true - - - name: "Build & Publish DEB package for debian/stretch" - <<: *DEB_TEMPLATE - if: commit_message =~ /\[Package (amd64|arm64|i386) DEB( Debian)?\]/ - env: - - BUILDER_NAME="builder" BUILD_DISTRO="debian" BUILD_RELEASE="stretch" BUILD_STRING="debian/stretch" - - PACKAGE_TYPE="deb" REPO_TOOL="apt-get" - - ALLOW_SOFT_FAILURE_HERE=true - - - stage: Package centos, fedora and opensuse - _template: &RPM_TEMPLATE - git: - depth: false - before_install: - - sudo apt-get install -y wget lxc lxc-templates python3-lxc - - source tests/installer/slack.sh - before_script: - - export PACKAGES_DIRECTORY="$(mktemp -d -t netdata-packaging-contents-dir-XXXXXX)" && echo "Created packaging directory ${PACKAGES_DIRECTORY}" - script: - - echo "GIT Branch:" && git branch - - echo "Last commit:" && git log -1 - - echo "GIT Describe:" && git describe - - echo "packaging/version:" && cat packaging/version - - echo "Creating LXC environment for the build" && sudo -E .travis/package_management/create_lxc_for_build.sh - - echo "Building package in container" && sudo -E .travis/package_management/build_package_in_container.sh - - sudo chmod -R 755 "/var/lib/lxc" - - echo "Preparing RPM packaging contents for upload" && sudo -E .travis/package_management/prepare_packages.sh - after_failure: post_message "TRAVIS_MESSAGE" "Failed to build RPM for ${BUILD_STRING}.${BUILD_ARCH}" - before_deploy: - - .travis/package_management/yank_stale_pkg.sh "${PACKAGES_DIRECTORY}" "${BUILD_STRING}" || echo "No stale RPM found" - deploy: - - provider: packagecloud - repository: "${DEPLOY_REPO}" - username: "${PACKAGING_USER}" - token: "${PKG_CLOUD_TOKEN}" - dist: "${BUILD_STRING}" - local_dir: "${PACKAGES_DIRECTORY}" - skip_cleanup: true - on: - # Only deploy on ${USER}/netdata, master branch, when packages directory is created - repo: ${TRAVIS_REPO_SLUG} - branch: "master" - condition: -d "${PACKAGES_DIRECTORY}" - after_deploy: - - if [ -n "${BUILDER_NAME}" ]; then rm -rf /home/${BUILDER_NAME}/* && echo "Cleared /home/${BUILDER_NAME} directory" || echo "Failed to clean /home/${BUILDER_NAME} directory"; fi; - - if [ -d "${PACKAGES_DIRECTORY}" ]; then rm -rf "${PACKAGES_DIRECTORY}"; fi; - - name: "Build & Publish RPM package for Enterprise Linux 7" - <<: *RPM_TEMPLATE - if: commit_message =~ /\[Package (amd64|arm64) RPM( Enterprise Linux)?\]/ - env: - - BUILDER_NAME="builder" BUILD_DISTRO="centos" BUILD_RELEASE="7" BUILD_STRING="el/7" - - PACKAGE_TYPE="rpm" REPO_TOOL="yum" - - ALLOW_SOFT_FAILURE_HERE=true - - - name: "Build & Publish RPM package for Fedora 32" - <<: *RPM_TEMPLATE - if: commit_message =~ /\[Package (amd64|arm64) RPM( Fedora)?\]/ - env: - - BUILDER_NAME="builder" BUILD_DISTRO="fedora" BUILD_RELEASE="32" BUILD_STRING="fedora/32" - - PACKAGE_TYPE="rpm" REPO_TOOL="dnf" - - ALLOW_SOFT_FAILURE_HERE=true - - - name: "Build & Publish RPM package for Fedora 33" - <<: *RPM_TEMPLATE - if: commit_message =~ /\[Package (amd64|arm64) RPM( Fedora)?\]/ - env: - - BUILDER_NAME="builder" BUILD_DISTRO="fedora" BUILD_RELEASE="33" BUILD_STRING="fedora/33" - - PACKAGE_TYPE="rpm" REPO_TOOL="dnf" - - ALLOW_SOFT_FAILURE_HERE=true - - - name: "Build & Publish RPM package for openSUSE 15.2" - <<: *RPM_TEMPLATE - if: commit_message =~ /\[Package (amd64|arm64) RPM( openSUSE)?\]/ - env: - - BUILDER_NAME="builder" BUILD_DISTRO="opensuse" BUILD_RELEASE="15.2" BUILD_STRING="opensuse/15.2" - - PACKAGE_TYPE="rpm" REPO_TOOL="zypper" - - ALLOW_SOFT_FAILURE_HERE=true - # ###### End of packaging workflow section ###### # - # ############################################### # - - # We only publish if a TAG has been set during packaging - stage: Publish for release name: Create release draft @@ -358,9 +124,10 @@ jobs: - .travis/trigger_docker_build.sh "${GITHUB_TOKEN}" "${BUILD_VERSION}" after_failure: post_message "TRAVIS_MESSAGE" "<!here> Failed to trigger docker build during release" "${NOTIF_CHANNEL}" - - stage: Trigger deb and rpm package build (release) - name: Trigger deb and rpm package build - script: .travis/trigger_package_generation.sh + - name: Trigger DEB and RPM package build + script: + - git checkout "${TRAVIS_BRANCH}" && export BUILD_VERSION="$(cat packaging/version | sed 's/^v//' | cut -d'-' -f1)" + - .travis/trigger_package_build.sh "${GITHUB_TOKEN}" "${BUILD_VERSION}" "release" after_failure: post_message "TRAVIS_MESSAGE" "<!here> Failed to trigger deb and rpm package build during release" "${NOTIF_CHANNEL}" @@ -378,11 +145,6 @@ jobs: git: depth: false - - name: Clean up package cloud nightly repository from old versions - script: - - DEPLOY_REPO="netdata-edge" .travis/package_management/old_package_purging.sh - - DEPLOY_REPO="netdata-devel" .travis/package_management/old_package_purging.sh - # This is the nightly execution step # - stage: Nightly release @@ -454,7 +216,8 @@ jobs: script: .travis/trigger_docker_build.sh "${GITHUB_TOKEN}" "nightly" after_failure: post_message "TRAVIS_MESSAGE" "<!here> Failed to trigger docker build during nightly release" "${NOTIF_CHANNEL}" - - stage: Trigger deb and rpm package build (nightly release) - name: Trigger deb and rpm package build - script: .travis/trigger_package_generation.sh "[Build latest]" + - name: Trigger DEB and RPM package build + script: + - git checkout "${TRAVIS_BRANCH}" && export BUILD_VERSION="$(cat packaging/version | sed 's/^v//')" + - .travis/trigger_package_build.sh "${GITHUB_TOKEN}" "${BUILD_VERSION}" "nightly" after_failure: post_message "TRAVIS_MESSAGE" "<!here> Failed to trigger deb and rpm package build during nightly release" "${NOTIF_CHANNEL}" diff --git a/.travis/package_management/build.sh b/.travis/package_management/build.sh deleted file mode 100644 index bafaecc5a..000000000 --- a/.travis/package_management/build.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/usr/bin/env bash - -UNPACKAGED_NETDATA_PATH="$1" -LATEST_RELEASE_VERSION="$2" - -if [ -z "${LATEST_RELEASE_VERSION}" ]; then - echo "Parameter 'LATEST_RELEASE_VERSION' not defined" - exit 1 -fi - -if [ -z "${UNPACKAGED_NETDATA_PATH}" ]; then - echo "Parameter 'UNPACKAGED_NETDATA_PATH' not defined" - exit 1 -fi - -echo "Running changelog generation mechanism since ${LATEST_RELEASE_VERSION}" - -echo "Entering ${UNPACKAGED_NETDATA_PATH}" -cd "${UNPACKAGED_NETDATA_PATH}" - -echo "Linking debian -> contrib/debian" -ln -sf contrib/debian debian - -echo "Executing dpkg-buildpackage" -# pre/post options are after 1.18.8, is simpler to just check help for their existence than parsing version -if dpkg-buildpackage --help | grep "\-\-post\-clean" 2> /dev/null > /dev/null; then - dpkg-buildpackage --post-clean --pre-clean --build=binary -us -uc -else - dpkg-buildpackage -b -us -uc -fi - -echo "DEB build script completed!" diff --git a/.travis/package_management/build_judy.sh b/.travis/package_management/build_judy.sh deleted file mode 100755 index 202ea0449..000000000 --- a/.travis/package_management/build_judy.sh +++ /dev/null @@ -1,36 +0,0 @@ -#!/usr/bin/env bash -# -# Build Judy from source, you need to run this script as root. -# -# Copyright: SPDX-License-Identifier: GPL-3.0-or-later -# -# Author : Pavlos Emm. Katsoulakis (paul@netdata.cloud) -set -e -JUDY_VER="1.0.5" -JUDY_DIR="/opt/judy-${JUDY_VER}" - -# If we are not in netdata git repo, at the top level directory, fail -TOP_LEVEL=$(basename "$(git rev-parse --show-toplevel)") -CWD=$(git rev-parse --show-cdup) -if [ -n "$CWD" ] || [ ! "${TOP_LEVEL}" == "netdata" ]; then - echo "Run as .travis/package_management/$(basename "$0") from top level directory of netdata git repository" - echo "Build Judy package from source code failed" - exit 1 -fi - -echo "Fetching judy source tarball" -wget -O /opt/judy.tar.gz http://downloads.sourceforge.net/project/judy/judy/Judy-${JUDY_VER}/Judy-${JUDY_VER}.tar.gz - -echo "Entering /opt directory and extracting tarball" -cd /opt && tar -xf judy.tar.gz && rm judy.tar.gz - -echo "Entering ${JUDY_DIR}" -cd "${JUDY_DIR}" - -echo "Running configure" -CFLAGS="-O2 -s" CXXFLAGS="-O2 -s" ./configure - -echo "Compiling and installing" -make && make install - -echo "Done, enjoy Judy!" diff --git a/.travis/package_management/build_libuv.sh b/.travis/package_management/build_libuv.sh deleted file mode 100755 index c30eede64..000000000 --- a/.travis/package_management/build_libuv.sh +++ /dev/null @@ -1,36 +0,0 @@ -#!/usr/bin/env bash -# -# Build libuv from source, you need to run this script as root. -# -# Copyright: SPDX-License-Identifier: GPL-3.0-or-later -# -# Author : Pavlos Emm. Katsoulakis <paul@netdata.cloud> -set -e -LIBUV_VERSION="v1.32.0" -# Their folder is libuv-1.32.0 while the tarball version is v1.32.0, so fix that until they fix it... -LIBUV_DIR="/opt/libuv-${LIBUV_VERSION/v/}" - -# If we are not in netdata git repo, at the top level directory, fail -TOP_LEVEL=$(basename "$(git rev-parse --show-toplevel)") -CWD=$(git rev-parse --show-cdup) -if [ -n "$CWD" ] || [ ! "${TOP_LEVEL}" == "netdata" ]; then - echo "Run as .travis/package_management/$(basename "$0") from top level directory of netdata git repository" - echo "Build libuv package from source code failed" - exit 1 -fi - -echo "Fetching libuv from github" -wget -O /opt/libuv.tar.gz "https://github.com/libuv/libuv/archive/${LIBUV_VERSION}.tar.gz" - -echo "Entering /opt and extracting source" -cd /opt && tar -xf libuv.tar.gz && rm libuv.tar.gz - -echo "Entering ${LIBUV_DIR}" -cd "${LIBUV_DIR}" - -echo "Compiling and installing" -sh autogen.sh -./configure -make && make check && make install - -echo "Done, enjoy libuv!" diff --git a/.travis/package_management/build_package_in_container.sh b/.travis/package_management/build_package_in_container.sh deleted file mode 100755 index 95a68e7a8..000000000 --- a/.travis/package_management/build_package_in_container.sh +++ /dev/null @@ -1,82 +0,0 @@ -#!/usr/bin/env bash -# -# Entry point for package build process -# -# Copyright: SPDX-License-Identifier: GPL-3.0-or-later -# -# Author : Pavlos Emm. Katsoulakis (paul@netdata.cloud) -#shellcheck disable=SC1091 -set -e - -# If we are not in netdata git repo, at the top level directory, fail -TOP_LEVEL=$(basename "$(git rev-parse --show-toplevel)") -CWD=$(git rev-parse --show-cdup) -if [ -n "$CWD" ] || [ ! "${TOP_LEVEL}" == "netdata" ]; then - echo "Run as .travis/package_management/$(basename "$0") from top level directory of netdata git repository" - echo "Docker build process aborted" - exit 1 -fi - -source .travis/package_management/functions.sh || (echo "Failed to load packaging library" && exit 1) - -# Check for presence of mandatory environment variables -if [ -z "${BUILD_STRING}" ]; then - echo "No Distribution was defined. Make sure BUILD_STRING is set on the environment before running this script" - exit 1 -fi - -if [ -z "${BUILDER_NAME}" ]; then - echo "No builder account and container name defined. Make sure BUILDER_NAME is set on the environment before running this script" - exit 1 -fi - -if [ -z "${BUILD_DISTRO}" ]; then - echo "No build distro information defined. Make sure BUILD_DISTRO is set on the environment before running this script" - exit 1 -fi - -if [ -z "${BUILD_RELEASE}" ]; then - echo "No build release information defined. Make sure BUILD_RELEASE is set on the environment before running this script" - exit 1 -fi - -if [ -z "${PACKAGE_TYPE}" ]; then - echo "No build release information defined. Make sure PACKAGE_TYPE is set on the environment before running this script" - exit 1 -fi - -# Detect architecture and load extra variables needed -detect_arch_from_commit - -case "${BUILD_ARCH}" in -"all") - echo "* * * Building all architectures, amd64 and i386 * * *" - echo "Building for amd64.." - export CONTAINER_NAME="${BUILDER_NAME}-${BUILD_DISTRO}${BUILD_RELEASE}-amd64" - export LXC_CONTAINER_ROOT="/var/lib/lxc/${CONTAINER_NAME}/rootfs" - .travis/package_management/trigger_${PACKAGE_TYPE}_lxc_build.py "${CONTAINER_NAME}" - - echo "Building for arm64.." - export CONTAINER_NAME="${BUILDER_NAME}-${BUILD_DISTRO}${BUILD_RELEASE}-arm64" - export LXC_CONTAINER_ROOT="/var/lib/lxc/${CONTAINER_NAME}/rootfs" - .travis/package_management/trigger_${PACKAGE_TYPE}_lxc_build.py "${CONTAINER_NAME}" - - echo "Building for i386.." - export CONTAINER_NAME="${BUILDER_NAME}-${BUILD_DISTRO}${BUILD_RELEASE}-i386" - export LXC_CONTAINER_ROOT="/var/lib/lxc/${CONTAINER_NAME}/rootfs" - .travis/package_management/trigger_${PACKAGE_TYPE}_lxc_build.py "${CONTAINER_NAME}" - - ;; -"amd64"|"arm64"|"i386") - echo "Building for ${BUILD_ARCH}.." - export CONTAINER_NAME="${BUILDER_NAME}-${BUILD_DISTRO}${BUILD_RELEASE}-${BUILD_ARCH}" - export LXC_CONTAINER_ROOT="/var/lib/lxc/${CONTAINER_NAME}/rootfs" - .travis/package_management/trigger_${PACKAGE_TYPE}_lxc_build.py "${CONTAINER_NAME}" - ;; -*) - echo "Unknown build architecture '${BUILD_ARCH}', nothing to do for build" - exit 1 - ;; -esac - -echo "Build process completed!" diff --git a/.travis/package_management/common.py b/.travis/package_management/common.py deleted file mode 100755 index 4cc04b93f..000000000 --- a/.travis/package_management/common.py +++ /dev/null @@ -1,182 +0,0 @@ -# -# -# Python library with commonly used functions within the package management scope -# -# Author : Pavlos Emm. Katsoulakis <paul@netdata.cloud> - -import lxc -import subprocess -import os -import sys -import tempfile -import shutil - -def fetch_version(orig_build_version): - tag = None - friendly_version = "" - - # TODO: Checksum validations - if str(orig_build_version).count(".latest") == 1: - version_list=str(orig_build_version).replace('v', '').split('.') - minor = version_list[3] if int(version_list[2]) == 0 else (version_list[2] + version_list[3]) - friendly_version='.'.join(version_list[0:2]) + "." + minor - else: - friendly_version = orig_build_version.replace('v', '') - tag = friendly_version # Go to stable tag - print("Version set to %s from %s" % (friendly_version, orig_build_version)) - - return friendly_version, tag - -def replace_tag(tag_name, spec, new_tag_content): - print("Fixing tag %s in %s" % (tag_name, spec)) - - ifp = open(spec, "r") - config = ifp.readlines() - ifp.close() - - source_line = -1 - for line in config: - if str(line).count(tag_name + ":") > 0: - source_line = config.index(line) - print("Found line: %s in item %d" % (line, source_line)) - break - - if source_line >= 0: - print("Replacing line %s with %s in spec file" %(config[source_line], new_tag_content)) - config[source_line] = "%s: %s\n" % (tag_name, new_tag_content) - config_str = ''.join(config) - ofp = open(spec, 'w') - ofp.write(config_str) - ofp.close() - -def run_command(container, command): - print("Running command: %s" % command) - command_result = container.attach_wait(lxc.attach_run_command, command, stdout=sys.stdout.buffer, stderr=sys.stdout.buffer) - - if command_result != 0: - raise Exception("Command failed with exit code %d" % command_result) - -def run_command_in_host(cmd, cwd=None): - print("Issue command in host: %s, cwd:%s" % (str(cmd), str(cwd))) - - proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=cwd) - o, e = proc.communicate() - print('Output: ' + o.decode('ascii')) - print('Error: ' + e.decode('ascii')) - print('code: ' + str(proc.returncode)) - -def prepare_repo(container): - if str(os.environ["REPO_TOOL"]).count("zypper") == 1: - run_command(container, [os.environ["REPO_TOOL"], "clean", "-a"]) - run_command(container, [os.environ["REPO_TOOL"], "--no-gpg-checks", "update", "-y"]) - - elif str(os.environ["REPO_TOOL"]).count("yum") == 1: - run_command(container, [os.environ["REPO_TOOL"], "clean", "all"]) - run_command(container, [os.environ["REPO_TOOL"], "update", "-y"]) - - if os.environ["BUILD_STRING"].count("el/7") == 1 and os.environ["BUILD_ARCH"].count("i386") == 1: - print ("Skipping epel-release install for %s-%s" % (os.environ["BUILD_STRING"], os.environ["BUILD_ARCH"])) - else: - run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "epel-release"]) - - elif str(os.environ["REPO_TOOL"]).count("apt-get") == 1: - if str(os.environ["BUILD_STRING"]).count("debian/jessie") == 1: - run_command(container, ["bash", "-c", "echo deb http://archive.debian.org/debian/ jessie-backports main contrib non-free >> /etc/apt/sources.list.d/99-archived.list"]) - run_command(container, [os.environ["REPO_TOOL"], "update", "-y", '-o', 'Acquire::Check-Valid-Until=false']) - else: - run_command(container, [os.environ["REPO_TOOL"], "update", "-y"]) - else: - run_command(container, [os.environ["REPO_TOOL"], "update", "-y"]) - - run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "sudo"]) - run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "wget"]) - run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "bash"]) - -def install_common_dependendencies(container): - if str(os.environ["REPO_TOOL"]).count("zypper") == 1: - run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "gcc-c++"]) - run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "json-glib-devel"]) - run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "freeipmi-devel"]) - run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "cups-devel"]) - run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "snappy-devel"]) - run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "protobuf-devel"]) - run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "protobuf-c"]) - - elif str(os.environ["REPO_TOOL"]).count("yum") == 1: - run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "gcc-c++"]) - run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "json-c-devel"]) - run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "freeipmi-devel"]) - run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "cups-devel"]) - run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "snappy-devel"]) - run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "protobuf-devel"]) - run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "protobuf-c-devel"]) - run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "protobuf-compiler"]) - run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "libwebsockets-devel"]) - - elif str(os.environ["REPO_TOOL"]).count("apt-get") == 1: - run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "g++"]) - run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "libipmimonitoring-dev"]) - run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "libjson-c-dev"]) - run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "libcups2-dev"]) - run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "libsnappy-dev"]) - run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "libprotobuf-dev"]) - run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "libprotoc-dev"]) - run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "protobuf-compiler"]) - if os.environ["BUILD_STRING"].count("debian/jessie") == 1: - run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "snappy"]) - else: - run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "gcc-c++"]) - run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "cups-devel"]) - run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "freeipmi-devel"]) - run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "json-c-devel"]) - run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "snappy-devel"]) - run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "protobuf-devel"]) - run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "protobuf-c-devel"]) - run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "protobuf-compiler"]) - run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "libwebsockets-devel"]) - - if os.environ["BUILD_STRING"].count("el/6") <= 0: - run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "autogen"]) - -def prepare_version_source(dest_archive, pkg_friendly_version, tag=None): - print(".0 Preparing local implementation tarball for version %s" % pkg_friendly_version) - tar_file = os.environ['LXC_CONTAINER_ROOT'] + dest_archive - - print(".0 Copy repo to prepare it for tarball generation") - tmp_src = tempfile.mkdtemp(prefix='netdata-source-') - run_command_in_host(['cp', '-r', '.', tmp_src]) - - if tag is not None: - print(".1 Checking out tag %s" % tag) - run_command_in_host(['git', 'fetch', '--all'], tmp_src) - - # TODO: Keep in mind that tricky 'v' there, needs to be removed once we clear our versioning scheme - run_command_in_host(['git', 'checkout', 'v%s' % pkg_friendly_version], tmp_src) - - print(".2 Tagging the code with version: %s" % pkg_friendly_version) - run_command_in_host(['git', 'tag', '-a', pkg_friendly_version, '-m', 'Tagging while packaging on %s' % os.environ["CONTAINER_NAME"]], tmp_src) - - print(".3 Run autoreconf -ivf") - run_command_in_host(['autoreconf', '-ivf'], tmp_src) - - print(".4 Run configure") - run_command_in_host(['./configure', '--prefix=/usr', '--sysconfdir=/etc', '--localstatedir=/var', '--libdir=/usr/lib', '--libexecdir=/usr/libexec', '--with-math', '--with-zlib', '--with-user=netdata'], tmp_src) - - print(".5 Run make dist") - run_command_in_host(['make', 'dist'], tmp_src) - - print(".6 Copy generated tarbal to desired path") - generated_tarball = '%s/netdata-%s.tar.gz' % (tmp_src, pkg_friendly_version) - - if os.path.exists(generated_tarball): - run_command_in_host(['sudo', 'cp', generated_tarball, tar_file]) - - print(".7 Fixing permissions on tarball") - run_command_in_host(['sudo', 'chmod', '777', tar_file]) - - print(".8 Bring over netdata.spec, Remove temp directory"); - run_command_in_host(['cp', '%s/netdata.spec' % tmp_src, 'netdata.spec']) - shutil.rmtree(tmp_src) - else: - print("I could not find (%s) on the disk, stopping the build. Kindly check the logs and try again" % generated_tarball) - sys.exit(1) diff --git a/.travis/package_management/configure_deb_lxc_environment.py b/.travis/package_management/configure_deb_lxc_environment.py deleted file mode 100755 index 627493bf5..000000000 --- a/.travis/package_management/configure_deb_lxc_environment.py +++ /dev/null @@ -1,90 +0,0 @@ -#!/usr/bin/env python3 -# -# Prepare the build environment within the container -# The script attaches to the running container and does the following: -# 1) Create the container -# 2) Start the container up -# 3) Create the builder user -# 4) Prepare the environment for DEB build -# -# Copyright: SPDX-License-Identifier: GPL-3.0-or-later -# -# Author : Pavlos Emm. Katsoulakis <paul@netdata.cloud> - -import common -import os -import sys -import lxc - -if len(sys.argv) != 2: - print('You need to provide a container name to get things started') - sys.exit(1) -container_name=sys.argv[1] - -# Setup the container object -print("Defining container %s" % container_name) -container = lxc.Container(container_name) -if not container.defined: - raise Exception("Container %s not defined!" % container_name) - -# Start the container -if not container.start(): - raise Exception("Failed to start the container") - -if not container.running or not container.state == "RUNNING": - raise Exception('Container %s is not running, configuration process aborted ' % container_name) - -# Wait for connectivity -print("Waiting for container connectivity to start configuration sequence") -if not container.get_ips(timeout=30): - raise Exception("Timeout while waiting for container") - -build_path = "/home/%s" % os.environ['BUILDER_NAME'] - -# Run the required activities now -# 1. Create the builder user -print("1. Adding user %s" % os.environ['BUILDER_NAME']) -common.run_command(container, ["useradd", "-m", os.environ['BUILDER_NAME']]) - -# Fetch package dependencies for the build -print("2. Preparing repo on LXC container") -common.prepare_repo(container) - -print("2.1 Install .DEB build support packages") -common.run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "dpkg-dev"]) -common.run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "libdistro-info-perl"]) -common.run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "dh-make"]) -common.run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "dh-systemd"]) -common.run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "dh-autoreconf"]) -common.run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "git-buildpackage"]) - -print("2.2 Add more dependencies") -common.run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "libnetfilter-acct-dev"]) -common.run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "libcups2-dev"]) - -print ("3.1 Run install-required-packages scriptlet") -common.run_command(container, ["wget", "-T", "15", "-O", "%s/.install-required-packages.sh" % build_path, "https://raw.githubusercontent.com/netdata/netdata/master/packaging/installer/install-required-packages.sh"]) -common.run_command(container, ["bash", "%s/.install-required-packages.sh" % build_path, "netdata", "--dont-wait", "--non-interactive"]) - -print("3.2 Installing package dependencies within LXC container") -common.install_common_dependendencies(container) - -friendly_version="" -dest_archive="" -download_url="" -tag = None -friendly_version, tag = common.fetch_version(os.environ['BUILD_VERSION']) - -tar_file="%s/netdata-%s.tar.gz" % (os.path.dirname(dest_archive), friendly_version) - -print("5. I will be building version '%s' of netdata." % os.environ['BUILD_VERSION']) -dest_archive="%s/netdata-%s.tar.gz" % (build_path, friendly_version) - -common.prepare_version_source(dest_archive, friendly_version, tag=tag) - -print("6. Installing build.sh script to build path") -common.run_command_in_host(['sudo', 'cp', '.travis/package_management/build.sh', "%s/%s/build.sh" % (os.environ['LXC_CONTAINER_ROOT'], build_path)]) -common.run_command_in_host(['sudo', 'chmod', '777', "%s/%s/build.sh" % (os.environ['LXC_CONTAINER_ROOT'], build_path)]) -common.run_command_in_host(['sudo', 'ln', '-sf', 'contrib/debian', 'debian']) - -print("Done!") diff --git a/.travis/package_management/configure_rpm_lxc_environment.py b/.travis/package_management/configure_rpm_lxc_environment.py deleted file mode 100755 index 4dca0bf38..000000000 --- a/.travis/package_management/configure_rpm_lxc_environment.py +++ /dev/null @@ -1,102 +0,0 @@ -#!/usr/bin/env python3 -# -# -# Prepare the build environment within the container -# The script attaches to the running container and does the following: -# 1) Create the container -# 2) Start the container up -# 3) Create the builder user -# 4) Prepare the environment for RPM build -# -# Copyright: SPDX-License-Identifier: GPL-3.0-or-later -# -# Author : Pavlos Emm. Katsoulakis <paul@netdata.cloud> - -import common -import os -import sys -import lxc - -if len(sys.argv) != 2: - print('You need to provide a container name to get things started') - sys.exit(1) -container_name=sys.argv[1] - -# Setup the container object -print("Defining container %s" % container_name) -container = lxc.Container(container_name) -if not container.defined: - raise Exception("Container %s not defined!" % container_name) - -# Start the container -if not container.start(): - raise Exception("Failed to start the container") - -if not container.running or not container.state == "RUNNING": - raise Exception('Container %s is not running, configuration process aborted ' % container_name) - -# Wait for connectivity -print("Waiting for container connectivity to start configuration sequence") -if not container.get_ips(timeout=30): - raise Exception("Timeout while waiting for container") - -# Run the required activities now -# Create the builder user -print("1. Adding user %s" % os.environ['BUILDER_NAME']) -common.run_command(container, ["useradd", "-m", os.environ['BUILDER_NAME']]) - -# Fetch package dependencies for the build -print("2.1 Preparing repo on LXC container") -common.prepare_repo(container) - -common.run_command(container, ["wget", "-T", "15", "-O", "/home/%s/.install-required-packages.sh" % (os.environ['BUILDER_NAME']), "https://raw.githubusercontent.com/netdata/netdata/master/packaging/installer/install-required-packages.sh"]) -common.run_command(container, ["bash", "/home/%s/.install-required-packages.sh" % (os.environ['BUILDER_NAME']), "netdata", "--dont-wait", "--non-interactive"]) - -# Exceptional cases, not available everywhere -# -print("2.2 Running uncommon dependencies and preparing LXC environment") -# Not on Centos-7 -if os.environ["BUILD_STRING"].count("el/7") <= 0: - common.run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "libnetfilter_acct-devel"]) - -# Not on Centos-6 -if os.environ["BUILD_STRING"].count("el/6") <= 0: - common.run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "autoconf-archive"]) - -print("2.3 Installing common dependencies") -common.install_common_dependendencies(container) - -print("3. Setting up macros") -common.run_command(container, ["sudo", "-u", os.environ['BUILDER_NAME'], "/bin/echo", "'%_topdir %(echo /home/" + os.environ['BUILDER_NAME'] + ")/rpmbuild' > /home/" + os.environ['BUILDER_NAME'] + "/.rpmmacros"]) - -print("4. Create rpmbuild directory") -common.run_command(container, ["sudo", "-u", os.environ['BUILDER_NAME'], "mkdir", "-p", "/home/" + os.environ['BUILDER_NAME'] + "/rpmbuild/BUILD"]) -common.run_command(container, ["sudo", "-u", os.environ['BUILDER_NAME'], "mkdir", "-p", "/home/" + os.environ['BUILDER_NAME'] + "/rpmbuild/RPMS"]) -common.run_command(container, ["sudo", "-u", os.environ['BUILDER_NAME'], "mkdir", "-p", "/home/" + os.environ['BUILDER_NAME'] + "/rpmbuild/SOURCES"]) -common.run_command(container, ["sudo", "-u", os.environ['BUILDER_NAME'], "mkdir", "-p", "/home/" + os.environ['BUILDER_NAME'] + "/rpmbuild/SPECS"]) -common.run_command(container, ["sudo", "-u", os.environ['BUILDER_NAME'], "mkdir", "-p", "/home/" + os.environ['BUILDER_NAME'] + "/rpmbuild/SRPMS"]) -common.run_command(container, ["sudo", "-u", os.environ['BUILDER_NAME'], "ls", "-ltrR", "/home/" + os.environ['BUILDER_NAME'] + "/rpmbuild"]) - -# Download the source -rpm_friendly_version="" -dest_archive="" -download_url="" -spec_file="/home/%s/rpmbuild/SPECS/netdata.spec" % os.environ['BUILDER_NAME'] -tag = None -rpm_friendly_version, tag = common.fetch_version(os.environ['BUILD_VERSION']) -tar_file="%s/netdata-%s.tar.gz" % (os.path.dirname(dest_archive), rpm_friendly_version) - -print("5. I will be building version '%s' of netdata." % os.environ['BUILD_VERSION']) -dest_archive="/home/%s/rpmbuild/SOURCES/netdata-%s.tar.gz" % (os.environ['BUILDER_NAME'], rpm_friendly_version) - -common.prepare_version_source(dest_archive, rpm_friendly_version, tag=tag) - -# Extract the spec file in place -print("6. Extract spec file from the source") -common.run_command_in_host(['sudo', 'cp', 'netdata.spec', os.environ['LXC_CONTAINER_ROOT'] + spec_file]) -common.run_command_in_host(['sudo', 'chmod', '777', os.environ['LXC_CONTAINER_ROOT'] + spec_file]) - -print("7. Temporary hack: Change Source0 to %s on spec file %s" % (dest_archive, spec_file)) -common.replace_tag("Source0", os.environ['LXC_CONTAINER_ROOT'] + spec_file, tar_file) - -print('Done!') diff --git a/.travis/package_management/create_lxc_for_build.sh b/.travis/package_management/create_lxc_for_build.sh deleted file mode 100755 index d733687a8..000000000 --- a/.travis/package_management/create_lxc_for_build.sh +++ /dev/null @@ -1,101 +0,0 @@ -#!/usr/bin/env bash -# -# This script generates an LXC container and starts it up -# Once the script completes successfully, a container has become available for usage -# The container image to be used and the container name to be set, are part of variables -# that must be present for the script to work -# -# Copyright: SPDX-License-Identifier: GPL-3.0-or-later -# -# Author : Pavlos Emm. Katsoulakis (paul@netdata.cloud) -# shellcheck disable=SC1091 -set -e - -source .travis/package_management/functions.sh || (echo "Failed to load packaging library" && exit 1) - -# If we are not in netdata git repo, at the top level directory, fail -TOP_LEVEL=$(basename "$(git rev-parse --show-toplevel)") -CWD=$(git rev-parse --show-cdup) -if [ -n "$CWD" ] || [ ! "${TOP_LEVEL}" == "netdata" ]; then - echo "Run as .travis/package_management/$(basename "$0") from top level directory of netdata git repository" - echo "LXC Container creation aborted" - exit 1 -fi - -# Check for presence of mandatory environment variables -if [ -z "${BUILD_STRING}" ]; then - echo "No Distribution was defined. Make sure BUILD_STRING is set on the environment before running this script" - exit 1 -fi - -if [ -z "${BUILDER_NAME}" ]; then - echo "No builder account and container name defined. Make sure BUILDER_NAME is set on the environment before running this script" - exit 1 -fi - -if [ -z "${BUILD_DISTRO}" ]; then - echo "No build distro information defined. Make sure BUILD_DISTRO is set on the environment before running this script" - exit 1 -fi - -if [ -z "${BUILD_RELEASE}" ]; then - echo "No build release information defined. Make sure BUILD_RELEASE is set on the environment before running this script" - exit 1 -fi - -if [ -z "${PACKAGE_TYPE}" ]; then - echo "No build release information defined. Make sure PACKAGE_TYPE is set on the environment before running this script" - exit 1 -fi - -# Detect architecture and load extra variables needed -detect_arch_from_commit - -echo "Creating LXC container ${BUILDER_NAME}/${BUILD_STRING}/${BUILD_ARCH}...." - -case "${BUILD_ARCH}" in -"all") - # i386 - echo "Creating LXC Container for i386.." - export CONTAINER_NAME="${BUILDER_NAME}-${BUILD_DISTRO}${BUILD_RELEASE}-i386" - export LXC_CONTAINER_ROOT="/var/lib/lxc/${CONTAINER_NAME}/rootfs" - lxc-create -n "${CONTAINER_NAME}" -t "download" -- --dist "${BUILD_DISTRO}" --release "${BUILD_RELEASE}" --arch "i386" --no-validate - - echo "Container(s) ready. Configuring container(s).." - .travis/package_management/configure_${PACKAGE_TYPE}_lxc_environment.py "${CONTAINER_NAME}" - - # amd64 - echo "Creating LXC Container for amd64.." - export CONTAINER_NAME="${BUILDER_NAME}-${BUILD_DISTRO}${BUILD_RELEASE}-amd64" - export LXC_CONTAINER_ROOT="/var/lib/lxc/${CONTAINER_NAME}/rootfs" - lxc-create -n "${CONTAINER_NAME}" -t "download" -- --dist "${BUILD_DISTRO}" --release "${BUILD_RELEASE}" --arch "amd64" --no-validate - - echo "Container(s) ready. Configuring container(s).." - .travis/package_management/configure_${PACKAGE_TYPE}_lxc_environment.py "${CONTAINER_NAME}" - - # arm64 - echo "Creating LXC Container for arm64.." - export CONTAINER_NAME="${BUILDER_NAME}-${BUILD_DISTRO}${BUILD_RELEASE}-arm64" - export LXC_CONTAINER_ROOT="/var/lib/lxc/${CONTAINER_NAME}/rootfs" - lxc-create -n "${CONTAINER_NAME}" -t "download" -- --dist "${BUILD_DISTRO}" --release "${BUILD_RELEASE}" --arch "arm64" --no-validate - - echo "Container(s) ready. Configuring container(s).." - .travis/package_management/configure_${PACKAGE_TYPE}_lxc_environment.py "${CONTAINER_NAME}" - ;; -"i386"|"amd64"|"arm64") - # amd64 or i386 - echo "Creating LXC Container for ${BUILD_ARCH}.." - export CONTAINER_NAME="${BUILDER_NAME}-${BUILD_DISTRO}${BUILD_RELEASE}-${BUILD_ARCH}" - export LXC_CONTAINER_ROOT="/var/lib/lxc/${CONTAINER_NAME}/rootfs" - lxc-create -n "${CONTAINER_NAME}" -t "download" -- --dist "${BUILD_DISTRO}" --release "${BUILD_RELEASE}" --arch "${BUILD_ARCH}" --no-validate - - echo "Container(s) ready. Configuring container(s).." - .travis/package_management/configure_${PACKAGE_TYPE}_lxc_environment.py "${CONTAINER_NAME}" - ;; -*) - echo "Unknown BUILD_ARCH value '${BUILD_ARCH}' given, process failed" - exit 1 - ;; -esac - -echo "..LXC creation complete!" diff --git a/.travis/package_management/functions.sh b/.travis/package_management/functions.sh deleted file mode 100644 index 0c00d2fcb..000000000 --- a/.travis/package_management/functions.sh +++ /dev/null @@ -1,33 +0,0 @@ -# no-shebang-needed-its-a-library -# -# Utility functions for packaging in travis CI -# -# Copyright: SPDX-License-Identifier: GPL-3.0-or-later -# -# Author : Pavlos Emm. Katsoulakis (paul@netdata.cloud) -#shellcheck disable=SC2148 -set -e - -function detect_arch_from_commit { - case "${TRAVIS_COMMIT_MESSAGE}" in - "[Package amd64"*) - export BUILD_ARCH="amd64" - ;; - "[Package i386"*) - export BUILD_ARCH="i386" - ;; - "[Package ALL"*) - export BUILD_ARCH="all" - ;; - "[Package arm64"*) - export BUILD_ARCH="arm64" - ;; - - *) - echo "Unknown build architecture in '${TRAVIS_COMMIT_MESSAGE}'. Assuming amd64" - export BUILD_ARCH="amd64" - ;; - esac - - echo "Detected build architecture ${BUILD_ARCH}" -} diff --git a/.travis/package_management/prepare_packages.sh b/.travis/package_management/prepare_packages.sh deleted file mode 100755 index 12ed07cc7..000000000 --- a/.travis/package_management/prepare_packages.sh +++ /dev/null @@ -1,63 +0,0 @@ -#!/usr/bin/env bash -# -# Utility that gathers generated packages, -# puts them together in a local folder for deploy facility to pick up -# -# Copyright: SPDX-License-Identifier: GPL-3.0-or-later -# -# Author : Pavlos Emm. Katsoulakis (paul@netdata.cloud) -#shellcheck disable=SC2068 -set -e - -# If we are not in netdata git repo, at the top level directory, fail -TOP_LEVEL=$(basename "$(git rev-parse --show-toplevel)") -CWD=$(git rev-parse --show-cdup) -if [ -n "$CWD" ] || [ ! "${TOP_LEVEL}" == "netdata" ]; then - echo "Run as .travis/package_management/$(basename "$0") from top level directory of netdata git repository" - echo "Package preparation aborted" - exit 1 -fi - -export LXC_ROOT="/var/lib/lxc" - -# Go through the containers created for packaging and pick up all generated packages -CREATED_CONTAINERS=$(ls -A "${LXC_ROOT}") -for d in ${CREATED_CONTAINERS[@]}; do - echo "Picking up packaging contents from ${d}" - - # Pick up any RPMS from builder - RPM_BUILD_PATH="${LXC_ROOT}/${d}/rootfs/home/${BUILDER_NAME}/rpmbuild" - if [ -d "${RPM_BUILD_PATH}" ]; then - echo "Checking folder ${RPM_BUILD_PATH} for RPMS and SRPMS" - - if [ -d "${RPM_BUILD_PATH}/RPMS" ]; then - echo "Copying any RPMS in '${RPM_BUILD_PATH}', copying over the following:" - ls -ltrR "${RPM_BUILD_PATH}/RPMS" - [[ -d "${RPM_BUILD_PATH}/RPMS/x86_64" ]] && cp -r "${RPM_BUILD_PATH}"/RPMS/x86_64/* "${PACKAGES_DIRECTORY}" - [[ -d "${RPM_BUILD_PATH}/RPMS/i386" ]] && cp -r "${RPM_BUILD_PATH}"/RPMS/i386/* "${PACKAGES_DIRECTORY}" - [[ -d "${RPM_BUILD_PATH}/RPMS/i686" ]] && cp -r "${RPM_BUILD_PATH}"/RPMS/i686/* "${PACKAGES_DIRECTORY}" - fi - - if [ -d "${RPM_BUILD_PATH}/SRPMS" ]; then - echo "Copying any SRPMS in '${RPM_BUILD_PATH}', copying over the following:" - ls -ltrR "${RPM_BUILD_PATH}/SRPMS" - [[ -d "${RPM_BUILD_PATH}/SRPMS/x86_64" ]] && cp -r "${RPM_BUILD_PATH}"/SRPMS/x86_64/* "${PACKAGES_DIRECTORY}" - [[ -d "${RPM_BUILD_PATH}/SRPMS/i386" ]] && cp -r "${RPM_BUILD_PATH}"/SRPMS/i386/* "${PACKAGES_DIRECTORY}" - [[ -d "${RPM_BUILD_PATH}/SRPMS/i686" ]] && cp -r "${RPM_BUILD_PATH}"/SRPMS/i686/* "${PACKAGES_DIRECTORY}" - fi - else - DEB_BUILD_PATH="${LXC_ROOT}/${d}/rootfs/home/${BUILDER_NAME}" - echo "Checking folder ${DEB_BUILD_PATH} for DEB packages" - if [ -d "${DEB_BUILD_PATH}" ]; then - cp "${DEB_BUILD_PATH}"/netdata*.ddeb "${PACKAGES_DIRECTORY}" || echo "Could not copy any .ddeb files" - cp "${DEB_BUILD_PATH}"/netdata*.deb "${PACKAGES_DIRECTORY}" || echo "Could not copy any .deb files" - cp "${DEB_BUILD_PATH}"/netdata*.buildinfo "${PACKAGES_DIRECTORY}" || echo "Could not copy any .buildinfo files" - cp "${DEB_BUILD_PATH}"/netdata*.changes "${PACKAGES_DIRECTORY}" || echo "Could not copy any .changes files" - else - echo "Folder ${DEB_BUILD_PATH} does not exist or not a directory, nothing to do for package preparation" - fi - fi -done - -chmod -R 777 "${PACKAGES_DIRECTORY}" -echo "Packaging contents ready to ship!" diff --git a/.travis/package_management/trigger_deb_lxc_build.py b/.travis/package_management/trigger_deb_lxc_build.py deleted file mode 100755 index 464a7715f..000000000 --- a/.travis/package_management/trigger_deb_lxc_build.py +++ /dev/null @@ -1,86 +0,0 @@ -#!/usr/bin/env python3 -# -# This script is responsible for running the RPM build on the running container -# -# Copyright: SPDX-License-Identifier: GPL-3.0-or-later -# -# Author : Pavlos Emm. Katsoulakis <paul@netdata.cloud> - -import common -import os -import sys -import lxc - -if len(sys.argv) != 2: - print('You need to provide a container name to get things started') - sys.exit(1) -container_name=sys.argv[1] - -# Load the container, break if its not there -print("Starting up container %s" % container_name) -container = lxc.Container(container_name) -if not container.defined: - raise Exception("Container %s does not exist!" % container_name) - -# Check if the container is running, attempt to start it up in case its not running -if not container.running or not container.state == "RUNNING": - print('Container %s is not running, attempt to start it up' % container_name) - - # Start the container - if not container.start(): - raise Exception("Failed to start the container") - - if not container.running or not container.state == "RUNNING": - raise Exception('Container %s is not running, configuration process aborted ' % container_name) - -# Wait for connectivity -if not container.get_ips(timeout=30): - raise Exception("Timeout while waiting for container") - -build_path = "/home/%s" % os.environ['BUILDER_NAME'] - -print("Setting up EMAIL and DEBFULLNAME variables required by the build tools") -os.environ["EMAIL"] = "bot@netdata.cloud" -os.environ["DEBFULLNAME"] = "Netdata builder" - -# Run the build process on the container -new_version, tag = common.fetch_version(os.environ['BUILD_VERSION']) -print("Starting DEB build process for version %s" % new_version) - -netdata_tarball = "%s/netdata-%s.tar.gz" % (build_path, new_version) -unpacked_netdata = netdata_tarball.replace(".tar.gz", "") - -print("Extracting tarball %s" % netdata_tarball) -common.run_command(container, ["sudo", "-u", os.environ['BUILDER_NAME'], "tar", "xf", netdata_tarball, "-C", build_path]) - -print("Checking version consistency") -since_version = os.environ["LATEST_RELEASE_VERSION"] -if str(since_version).replace('v', '') == str(new_version) and str(new_version).count('.') == 2: - s = since_version.split('.') - if int(s[2]) > 0: - patch_prev = str(int(s[2]) - 1) - since_version = s[0] + '.' + s[1] + '.' + patch_prev - else: - prev = str(int(s[1]) - 1) - since_version = s[0] + '.' + prev + '.' + s[2] - - print("We seem to be building a new stable release, reduce by one since_version option. New since_version:%s" % since_version) - -print("Fixing changelog tags") -changelog_in_host = "contrib/debian/changelog" -common.run_command_in_host(['sed', '-i', 's/PREVIOUS_PACKAGE_VERSION/%s-1/g' % since_version.replace("v", ""), changelog_in_host]) -common.run_command_in_host(['sed', '-i', 's/PREVIOUS_PACKAGE_DATE/%s/g' % os.environ["LATEST_RELEASE_DATE"], changelog_in_host]) - -print("Executing gbp dch command..") -common.run_command_in_host(['gbp', 'dch', '--release', '--ignore-branch', '--spawn-editor=snapshot', '--since=%s' % since_version, '--new-version=%s' % new_version]) - -print("Copying over changelog to the destination machine") -common.run_command_in_host(['sudo', 'cp', 'debian/changelog', "%s/%s/netdata-%s/contrib/debian/" % (os.environ['LXC_CONTAINER_ROOT'], build_path, new_version)]) - -print("Running debian build script since %s" % since_version) -common.run_command(container, ["sudo", "-u", os.environ['BUILDER_NAME'], "%s/build.sh" % build_path, unpacked_netdata, new_version]) - -print("Listing contents on build path") -common.run_command(container, ["sudo", "-u", os.environ['BUILDER_NAME'], "ls", "-ltr", build_path]) - -print('Done!') diff --git a/.travis/package_management/trigger_rpm_lxc_build.py b/.travis/package_management/trigger_rpm_lxc_build.py deleted file mode 100755 index f9e109c72..000000000 --- a/.travis/package_management/trigger_rpm_lxc_build.py +++ /dev/null @@ -1,55 +0,0 @@ -#!/usr/bin/env python3 -# -# This script is responsible for running the RPM build on the running container -# -# Copyright: SPDX-License-Identifier: GPL-3.0-or-later -# -# Author : Pavlos Emm. Katsoulakis <paul@netdata.cloud> - -import common -import os -import sys -import lxc - -if len(sys.argv) != 2: - print('You need to provide a container name to get things started') - sys.exit(1) -container_name=sys.argv[1] - -# Load the container, break if its not there -print("Starting up container %s" % container_name) -container = lxc.Container(container_name) -if not container.defined: - raise Exception("Container %s does not exist!" % container_name) - -# Check if the container is running, attempt to start it up in case its not running -if not container.running or not container.state == "RUNNING": - print('Container %s is not running, attempt to start it up' % container_name) - - # Start the container - if not container.start(): - raise Exception("Failed to start the container") - - if not container.running or not container.state == "RUNNING": - raise Exception('Container %s is not running, configuration process aborted ' % container_name) - -# Wait for connectivity -if not container.get_ips(timeout=30): - raise Exception("Timeout while waiting for container") - -print("Adding builder specific dependencies to the LXC container") -common.run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "rpm-build"]) -common.run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "rpm-devel"]) -common.run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "rpmlint"]) -common.run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "make"]) -common.run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "python"]) -common.run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "bash"]) -common.run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "diffutils"]) -common.run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "patch"]) -common.run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "rpmdevtools"]) - -# Run the build process on the container -print("Starting RPM build process") -common.run_command(container, ["sudo", "-u", os.environ['BUILDER_NAME'], "rpmbuild", "-ba", "--rebuild", "/home/%s/rpmbuild/SPECS/netdata.spec" % os.environ['BUILDER_NAME']]) - -print('Done!') diff --git a/.travis/package_management/yank_stale_pkg.sh b/.travis/package_management/yank_stale_pkg.sh deleted file mode 100755 index 3f7669712..000000000 --- a/.travis/package_management/yank_stale_pkg.sh +++ /dev/null @@ -1,35 +0,0 @@ -#!/usr/bin/env bash -# -# This script is responsible for the removal of stale RPM/DEB files. -# It runs on the pre-deploy step and takes care of the removal of the files -# prior to the upload of the freshly built ones -# -# Copyright: SPDX-License-Identifier: GPL-3.0-or-later -# -# Author : Pavlos Emm. Katsoulakis (paul@netdata.cloud) -#shellcheck disable=SC2010,SC2068 -set -e - -# If we are not in netdata git repo, at the top level directory, fail -TOP_LEVEL=$(basename "$(git rev-parse --show-toplevel)") -CWD=$(git rev-parse --show-cdup) -if [ -n "$CWD" ] || [ ! "${TOP_LEVEL}" == "netdata" ]; then - echo "Run as .travis/package_management/$(basename "$0") from top level directory of netdata git repository" - echo "Package yanking cancelled" - exit 1 -fi - -PACKAGES_DIR="$1" -DISTRO="$2" -PACKAGES_LIST="$(ls -AR "${PACKAGES_DIR}" | grep -e '\.rpm' -e '\.deb' -e '\.ddeb' )" - -if [ ! -d "${PACKAGES_DIR}" ] || [ -z "${PACKAGES_LIST}" ]; then - echo "Folder ${PACKAGES_DIR} does not seem to be a valid directory or is empty. No packages to check for yanking" - exit 1 -fi - -for pkg in ${PACKAGES_LIST[@]}; do - echo "Attempting yank on ${pkg}.." - .travis/package_management/package_cloud_wrapper.sh yank "${PACKAGING_USER}/${DEPLOY_REPO}/${DISTRO}" "${pkg}" || echo "Nothing to yank or error on ${pkg}" -done - diff --git a/.travis/trigger_package_build.sh b/.travis/trigger_package_build.sh new file mode 100755 index 000000000..d1ecd48ca --- /dev/null +++ b/.travis/trigger_package_build.sh @@ -0,0 +1,20 @@ +#!/bin/sh + +token="${1}" +version="${2}" +pkgtype="${3}" + +resp="$(curl -X POST \ + -H 'Accept: application/vnd.github.v3+json' \ + -H "Authorization: Bearer ${token}" \ + "https://api.github.com/repos/netdata/netdata/actions/workflows/packaging.yml/dispatches" \ + -d "{\"ref\": \"master\", \"inputs\": {\"version\": \"${version}\", \"type\": \"${pkgtype}\"}}")" + +if [ -z "${resp}" ]; then + echo "Successfully triggered binary package builds." + exit 0 +else + echo "Failed to trigger binary package builds. Output:" + echo "${resp}" + exit 1 +fi diff --git a/.travis/trigger_package_generation.sh b/.travis/trigger_package_generation.sh deleted file mode 100755 index b58832cf4..000000000 --- a/.travis/trigger_package_generation.sh +++ /dev/null @@ -1,57 +0,0 @@ -#!/usr/bin/env bash -# -# Trigger .RPM and .DEB package generation processes -# -# Copyright: SPDX-License-Identifier: GPL-3.0-or-later -# -# Author: Pavlos Emm. Katsoulakis <paul@netdata.cloud> -set -e -WAIT_TIME=15 -BUILD_NIGHTLY="$1" - -commit_change() { - local ARCH="$1" - local PKG="$2" - local GIT_MAIL="bot@netdata.cloud" - local GIT_USER="netdatabot" - - echo "---- Committing ${ARCH} .${PKG} package generation ----" - git commit --allow-empty --author "${GIT_USER} <${GIT_MAIL}>" -m "[Package ${ARCH} ${PKG}]${BUILD_NIGHTLY} Package build process trigger" -} - -push_change() { - - echo "---- Push changes to repository ----" - git push "https://${GITHUB_TOKEN}:@$(git config --get remote.origin.url | sed -e 's/^https:\/\///')" -} - -# If we are not in netdata git repo, at the top level directory, fail -TOP_LEVEL=$(basename "$(git rev-parse --show-toplevel)") -CWD=$(git rev-parse --show-cdup || echo "") -if [ -n "${CWD}" ] || [ ! "${TOP_LEVEL}" == "netdata" ]; then - echo "Run as .travis/$(basename "$0") from top level directory of netdata git repository" - echo "Changelog generation process aborted" - exit 1 -fi - -echo "--- Initialize git configuration ---" -git checkout master -git fetch --all -git pull - -commit_change "amd64" "DEB" -push_change - -echo "---- Waiting for ${WAIT_TIME} seconds before triggering next process ----" -sleep "${WAIT_TIME}" - -commit_change "i386" "DEB" -push_change - -echo "---- Waiting for ${WAIT_TIME} seconds before triggering next process ----" -sleep "${WAIT_TIME}" - -commit_change "amd64" "RPM" -push_change - -echo "---- Done! ----" |