diff options
author | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-05-05 11:19:16 +0000 |
---|---|---|
committer | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-05-05 12:07:37 +0000 |
commit | b485aab7e71c1625cfc27e0f92c9509f42378458 (patch) | |
tree | ae9abe108601079d1679194de237c9a435ae5b55 /packaging | |
parent | Adding upstream version 1.44.3. (diff) | |
download | netdata-b485aab7e71c1625cfc27e0f92c9509f42378458.tar.xz netdata-b485aab7e71c1625cfc27e0f92c9509f42378458.zip |
Adding upstream version 1.45.3+dfsg.
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to '')
77 files changed, 3984 insertions, 548 deletions
diff --git a/packaging/PLATFORM_SUPPORT.md b/packaging/PLATFORM_SUPPORT.md index 5448e5da4..591b850af 100644 --- a/packaging/PLATFORM_SUPPORT.md +++ b/packaging/PLATFORM_SUPPORT.md @@ -61,8 +61,8 @@ to work on these platforms with minimal user effort. | Alpine Linux | 3.18 | No | The latest release of Alpine Linux is guaranteed to remain at **Core** tier due to usage for our Docker images | | Alma Linux | 9.x | x86\_64, AArch64 | Also includes support for Rocky Linux and other ABI compatible RHEL derivatives | | Alma Linux | 8.x | x86\_64, AArch64 | Also includes support for Rocky Linux and other ABI compatible RHEL derivatives | -| Amazon Linux | 2023 | x86\_64, AArch64 | | -| Amazon Linux | 2 | x86\_64, AArch64 | | +| Amazon Linux | 2023 | x86\_64, AArch64 | | +| Amazon Linux | 2 | x86\_64, AArch64 | | | CentOS | 7.x | x86\_64 | | | Docker | 19.03 or newer | x86\_64, i386, ARMv7, AArch64, POWER8+ | See our [Docker documentation](https://github.com/netdata/netdata/blob/master/packaging/docker/README.md) for more info on using Netdata on Docker | | Debian | 12.x | x86\_64, i386, ARMv7, AArch64 | | @@ -71,15 +71,14 @@ to work on these platforms with minimal user effort. | Fedora | 39 | x86\_64, AArch64 | | | Fedora | 38 | x86\_64, AArch64 | | | Fedora | 37 | x86\_64, AArch64 | | -| openSUSE | Leap 15.4 | x86\_64, AArch64 | | | openSUSE | Leap 15.5 | x86\_64, AArch64 | | +| openSUSE | Leap 15.4 | x86\_64, AArch64 | | | Oracle Linux | 9.x | x86\_64, AArch64 | | | Oracle Linux | 8.x | x86\_64, AArch64 | | | Red Hat Enterprise Linux | 9.x | x86\_64, AArch64 | | | Red Hat Enterprise Linux | 8.x | x86\_64, AArch64 | | | Red Hat Enterprise Linux | 7.x | x86\_64 | | | Ubuntu | 23.10 | x86\_64, AArch64, ARMv7 | | -| Ubuntu | 23.04 | x86\_64, AArch64, ARMv7 | | | Ubuntu | 22.04 | x86\_64, ARMv7, AArch64 | | | Ubuntu | 20.04 | x86\_64, ARMv7, AArch64 | | @@ -159,9 +158,12 @@ This is a list of platforms that we have supported in the recent past but no lon | Alpine Linux | 3.14 | EOL as of 2023-05-01 | | Alpine Linux | 3.13 | EOL as of 2022-11-01 | | Debian | 9.x | EOL as of 2022-06-30 | +| Fedora | 37 | EOL as of 2023-12-05 | | Fedora | 36 | EOL as of 2023-05-18 | | Fedora | 35 | EOL as of 2022-12-13 | +| openSUSE | Leap 15.4 | EOL as of 2023-12-07 | | openSUSE | Leap 15.3 | EOL as of 2022-12-01 | +| Ubuntu | 23.04 | EOL as of 2024-01-20 | | Ubuntu | 22.10 | EOL as of 2023-07-20 | | Ubuntu | 21.10 | EOL as of 2022-07-31 | | Ubuntu | 18.04 | EOL as of 2023-04-02 | @@ -175,9 +177,9 @@ means that they generally do not support non-local username mappings or exotic n We currently provide static builds for the following CPU architectures: -- 32-bit x86 - 64-bit x86 - ARMv7 +- ARMv6 - AArch64 - POWER8+ @@ -187,3 +189,8 @@ We currently provide static builds for the following CPU architectures: Our IPMI collector is based on FreeIPMI. Due to upstream limitations in FreeIPMI, we are unable to support our IPMI collector on POWER-based hardware. + +### Systemd + +Many of our systemd integrations are not supported in our static builds. This is due to a general refusal by the +systemd developers to support static linking (or any C runtime other than glibc), and is not something we can resolve. diff --git a/packaging/bundle-ebpf-co-re.sh b/packaging/bundle-ebpf-co-re.sh index 460709b67..572333cc9 100755 --- a/packaging/bundle-ebpf-co-re.sh +++ b/packaging/bundle-ebpf-co-re.sh @@ -6,4 +6,4 @@ CORE_VERSION="$(cat "${SRCDIR}/packaging/ebpf-co-re.version")" CORE_TARBALL="netdata-ebpf-co-re-glibc-${CORE_VERSION}.tar.xz" curl -sSL --connect-timeout 10 --retry 3 "https://github.com/netdata/ebpf-co-re/releases/download/${CORE_VERSION}/${CORE_TARBALL}" > "${CORE_TARBALL}" || exit 1 grep "${CORE_TARBALL}" "${SRCDIR}/packaging/ebpf-co-re.checksums" | sha256sum -c - || exit 1 -tar -xa --no-same-owner -f "${CORE_TARBALL}" -C "${SRCDIR}/collectors/ebpf.plugin" || exit 1 +tar -xa --no-same-owner -f "${CORE_TARBALL}" -C "${SRCDIR}/src/libnetdata/ebpf" || exit 1 diff --git a/packaging/bundle-protobuf.sh b/packaging/bundle-protobuf.sh deleted file mode 100755 index d715dfe3d..000000000 --- a/packaging/bundle-protobuf.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/sh - -PROTOBUF_TARBALL="protobuf-cpp-$(cat "${1}/packaging/protobuf.version").tar.gz" -PROTOBUF_BUILD_PATH="${1}/externaldeps/protobuf/protobuf-$(cat "${1}/packaging/protobuf.version")" - -mkdir -p "${1}/externaldeps/protobuf" || exit 1 -curl -sSL --connect-timeout 10 --retry 3 "https://github.com/protocolbuffers/protobuf/releases/download/v$(cat "${1}/packaging/protobuf.version")/${PROTOBUF_TARBALL}" > "${PROTOBUF_TARBALL}" || exit 1 -sha256sum -c "${1}/packaging/protobuf.checksums" || exit 1 -tar -xz --no-same-owner -f "${PROTOBUF_TARBALL}" -C "${1}/externaldeps/protobuf" || exit 1 -OLDPWD="${PWD}" -cd "${PROTOBUF_BUILD_PATH}" || exit 1 -./configure --disable-shared --without-zlib --disable-dependency-tracking --with-pic || exit 1 -make -j "$(nproc)" || exit 1 -cd "${OLDPWD}" || exit 1 - -cp -a "${PROTOBUF_BUILD_PATH}/src" "${1}/externaldeps/protobuf" || exit 1 diff --git a/packaging/check-for-go-toolchain.sh b/packaging/check-for-go-toolchain.sh new file mode 100644 index 000000000..cd9bbfb5e --- /dev/null +++ b/packaging/check-for-go-toolchain.sh @@ -0,0 +1,176 @@ +#!/bin/sh +# +# Copyright (c) 2024 Netdata Inc. +# SPDX-License-Identifier: GPL-v3+ +# +# Check if we need to install a Go toolchain. +# +# Scripts that use this should call the ensure_go_toolchain function +# after sourcing this file to handle things correctly. +# +# If a working Go toolchain is either present or was installed, then the +# function will return 0. If a working Go toolchain is not present and one +# cannot be installed, then it will instead return 1, with the variable +# GOLANG_FAILURE_REASON set to an error message indicating what went wrong. + +GOLANG_MIN_MAJOR_VERSION='1' +GOLANG_MIN_MINOR_VERSION='21' +GOLANG_MIN_PATCH_VERSION='0' +GOLANG_MIN_VERSION="${GOLANG_MIN_MAJOR_VERSION}.${GOLANG_MIN_MINOR_VERSION}.${GOLANG_MIN_PATCH_VERSION}" + +GOLANG_TEMP_PATH="${TMPDIR}/go-toolchain" + +check_go_version() { + version="$("${go}" version | awk '{ print $3 }' | sed 's/^go//')" + version_major="$(echo "${version}" | cut -f 1 -d '.')" + version_minor="$(echo "${version}" | cut -f 2 -d '.')" + version_patch="$(echo "${version}" | cut -f 3 -d '.')" + + if [ -z "${version_major}" ] || [ "${version_major}" -lt "${GOLANG_MIN_MAJOR_VERSION}" ]; then + return 1 + elif [ "${version_major}" -gt "${GOLANG_MIN_MAJOR_VERSION}" ]; then + return 0 + fi + + if [ -z "${version_minor}" ] || [ "${version_minor}" -lt "${GOLANG_MIN_MINOR_VERSION}" ]; then + return 1 + elif [ "${version_minor}" -gt "${GOLANG_MIN_MINOR_VERSION}" ]; then + return 0 + fi + + if [ -n "${version_patch}" ] && [ "${version_patch}" -ge "${GOLANG_MIN_PATCH_VERSION}" ]; then + return 0 + fi + + return 1 +} + +install_go_toolchain() { + GOLANG_ARCHIVE_NAME="${GOLANG_TEMP_PATH}/golang.tar.gz" + GOLANG_CHECKSUM_FILE="${GOLANG_TEMP_PATH}/golang.sha256sums" + + case "$(uname -s)" in + Linux) + case "$(uname -m)" in + i?86) + GOLANG_ARCHIVE_URL="https://go.dev/dl/go1.22.0.linux-386.tar.gz" + GOLANG_ARCHIVE_CHECKSUM="1e209c4abde069067ac9afb341c8003db6a210f8173c77777f02d3a524313da3" + ;; + x86_64) + GOLANG_ARCHIVE_URL="https://go.dev/dl/go1.22.0.linux-amd64.tar.gz" + GOLANG_ARCHIVE_CHECKSUM="f6c8a87aa03b92c4b0bf3d558e28ea03006eb29db78917daec5cfb6ec1046265" + ;; + aarch64) + GOLANG_ARCHIVE_URL="https://go.dev/dl/go1.22.0.linux-arm64.tar.gz" + GOLANG_ARCHIVE_CHECKSUM="6a63fef0e050146f275bf02a0896badfe77c11b6f05499bb647e7bd613a45a10" + ;; + armv*) + GOLANG_ARCHIVE_URL="https://go.dev/dl/go1.22.0.linux-armv6l.tar.gz" + GOLANG_ARCHIVE_CHECKSUM="0525f92f79df7ed5877147bce7b955f159f3962711b69faac66bc7121d36dcc4" + ;; + ppc64le) + GOLANG_ARCHIVE_URL="https://go.dev/dl/go1.22.0.linux-ppc64le.tar.gz" + GOLANG_ARCHIVE_CHECKSUM="0e57f421df9449066f00155ce98a5be93744b3d81b00ee4c2c9b511be2a31d93" + ;; + riscv64) + GOLANG_ARCHIVE_URL="https://go.dev/dl/go1.22.0.linux-riscv64.tar.gz" + GOLANG_ARCHIVE_CHECKSUM="afe9cedcdbd6fdff27c57efd30aa5ce0f666f471fed5fa96cd4fb38d6b577086" + ;; + s390x) + GOLANG_ARCHIVE_URL="https://go.dev/dl/go1.22.0.linux-s390x.tar.gz" + GOLANG_ARCHIVE_CHECKSUM="2e546a3583ba7bd3988f8f476245698f6a93dfa9fe206a8ca8f85c1ceecb2446" + ;; + *) + GOLANG_FAILURE_REASON="Linux $(uname -m) platform is not supported out-of-box by Go, you must install a toolchain for it yourself." + return 1 + ;; + esac + ;; + FreeBSD) + case "$(uname -m)" in + 386) + GOLANG_ARCHIVE_URL="https://go.dev/dl/go1.22.0.freebsd-386.tar.gz" + GOLANG_ARCHIVE_CHECKSUM="b8065da37783e8b9e7086365a54d74537e832c92311b61101a66989ab2458d8e" + ;; + amd64) + GOLANG_ARCHIVE_URL="https://go.dev/dl/go1.22.0.freebsd-amd64.tar.gz" + GOLANG_ARCHIVE_CHECKSUM="50f421c7f217083ac94aab1e09400cb9c2fea7d337679ec11f1638a11460da30" + ;; + arm) + GOLANG_ARCHIVE_URL="https://go.dev/dl/go1.22.0.freebsd-arm.tar.gz" + GOLANG_ARCHIVE_CHECKSUM="c9c8b305f90903536f4981bad9f029828c2483b3216ca1783777344fbe603f2d" + ;; + arm64) + GOLANG_ARCHIVE_URL="https://go.dev/dl/go1.22.0.freebsd-arm64.tar.gz" + GOLANG_ARCHIVE_CHECKSUM="e23385e5c640787fa02cd58f2301ea09e162c4d99f8ca9fa6d52766f428a933d" + ;; + riscv64) + GOLANG_ARCHIVE_URL="https://go.dev/dl/go1.22.0.freebsd-riscv64.tar.gz" + GOLANG_ARCHIVE_CHECKSUM="c8f94d1de6024546194d58e7b9370dc7ea06176aad94a675b0062c25c40cb645" + ;; + *) + GOLANG_FAILURE_REASON="FreeBSD $(uname -m) platform is not supported out-of-box by Go, you must install a toolchain for it yourself." + return 1 + ;; + esac + ;; + *) + GOLANG_FAILURE_REASON="We do not support automatic handling of a Go toolchain on this system, you must install one manually." + return 1 + ;; + esac + + if [ -d '/usr/local/go' ]; then + if [ -f '/usr/local/go/.installed-by-netdata' ]; then + rm -rf /usr/local/go + else + GOLANG_FAILURE_REASON="Refusing to overwrite existing Go toolchain install at /usr/local/go, it needs to be updated manually." + return 1 + fi + fi + + mkdir -p "${GOLANG_TEMP_PATH}" + + if ! curl --fail -q -sSL --connect-timeout 10 --retry 3 --output "${GOLANG_ARCHIVE_NAME}" "${GOLANG_ARCHIVE_URL}"; then + GOLANG_FAILURE_REASON="Failed to download Go toolchain." + return 1 + fi + + echo "${GOLANG_ARCHIVE_CHECKSUM} ${GOLANG_ARCHIVE_NAME}" > "${GOLANG_CHECKSUM_FILE}" + + if ! sha256sum -c "${GOLANG_CHECKSUM_FILE}"; then + GOLANG_FAILURE_REASON="Invalid checksum for downloaded Go toolchain." + return 1 + fi + + if ! tar -C /usr/local/ -xzf "${GOLANG_ARCHIVE_NAME}"; then + GOLANG_FAILURE_REASON="Failed to extract Go toolchain." + return 1 + fi + + touch /usr/local/go/.installed-by-netdata + + rm -rf "${GOLANG_TEMP_PATH}" +} + +ensure_go_toolchain() { + go="$(PATH="/usr/local/go/bin:${PATH}" command -v go 2>/dev/null)" + + need_go_install=0 + + if [ -z "${go}" ]; then + need_go_install=1 + elif ! check_go_version; then + need_go_install=1 + fi + + if [ "${need_go_install}" -eq 1 ]; then + if ! install_go_toolchain; then + return 1 + fi + + rm -rf "${GOLANG_TEMP_PATH}" || true + fi + + return 0 +} diff --git a/packaging/cmake/Modules/FindGo.cmake b/packaging/cmake/Modules/FindGo.cmake new file mode 100644 index 000000000..454a0051e --- /dev/null +++ b/packaging/cmake/Modules/FindGo.cmake @@ -0,0 +1,39 @@ +# Custom CMake module to find the Go toolchain +# +# Copyright (c) 2024 Netdata Inc +# +# SPDX-License-Identifier: GPL-3.0-or-later +# +# This is a relatively orthodox CMake Find Module. It can be used by +# simply including it and then invoking `find_package(Go)`. +# +# Version handling is done by CMake itself via the +# find_package_handle_standard_args() function, so `find_package(Go 1.21)` +# will also work correctly. + +if(GO_FOUND) + return() +endif() + +# Two passes are needed here so that we prefer a copy in `/usr/local/go/bin` over a system copy. +find_program(GO_EXECUTABLE go PATHS /usr/local/go/bin DOC "Go toolchain" NO_DEFAULT_PATH) +find_program(GO_EXECUTABLE go DOC "Go toolchain") + +if (GO_EXECUTABLE) + execute_process( + COMMAND ${GO_EXECUTABLE} version + OUTPUT_VARIABLE GO_VERSION_STRING + RESULT_VARIABLE RESULT + ) + if (RESULT EQUAL 0) + string(REGEX MATCH "go([0-9]+\\.[0-9]+(\\.[0-9]+)?)" GO_VERSION_STRING "${GO_VERSION_STRING}") + string(REGEX MATCH "([0-9]+\\.[0-9]+(\\.[0-9]+)?)" GO_VERSION_STRING "${GO_VERSION_STRING}") + endif() +endif() + +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args( + Go + REQUIRED_VARS GO_EXECUTABLE + VERSION_VAR GO_VERSION_STRING +) diff --git a/packaging/cmake/Modules/NetdataCompilerFlags.cmake b/packaging/cmake/Modules/NetdataCompilerFlags.cmake new file mode 100644 index 000000000..894e244ce --- /dev/null +++ b/packaging/cmake/Modules/NetdataCompilerFlags.cmake @@ -0,0 +1,75 @@ +# Functions to simplify handling of extra compiler flags. +# +# Copyright (c) 2024 Netdata Inc. +# SPDX-License-Identifier: GPL-3.0-or-later + +include(CheckCCompilerFlag) +include(CheckCXXCompilerFlag) + +# Construct a pre-processor safe name +# +# This takes a specified value, and assigns the generated name to the +# specified target. +function(make_cpp_safe_name value target) + string(REPLACE "-" "_" tmp "${value}") + string(REPLACE "=" "_" tmp "${tmp}") + set(${target} "${tmp}" PARENT_SCOPE) +endfunction() + +# Conditionally add an extra compiler flag to C and C++ flags. +# +# If the language flags already match the `match` argument, skip this flag. +# Otherwise, check for support for `flag` and if support is found, add it to +# the language-specific `target` flag group. +function(add_simple_extra_compiler_flag match flag target) + set(CMAKE_REQUIRED_FLAGS "-Werror") + + make_cpp_safe_name("${flag}" flag_name) + + if(NOT ${CMAKE_C_FLAGS} MATCHES ${match}) + check_c_compiler_flag("${flag}" HAVE_C_${flag_name}) + if(HAVE_C_${flag_name}) + set(${target}_C_FLAGS "${${target}_C_FLAGS} ${flag}" PARENT_SCOPE) + endif() + endif() + + if(NOT ${CMAKE_CXX_FLAGS} MATCHES ${match}) + check_cxx_compiler_flag("${flag}" HAVE_CXX_${flag_name}) + if(HAVE_CXX_${flag_name}) + set(${target}_CXX_FLAGS "${${target}_CXX_FLAGS} ${flag}" PARENT_SCOPE) + endif() + endif() +endfunction() + +# Same as add_simple_extra_compiler_flag, but check for a second flag if the +# first one is unsupported. +function(add_double_extra_compiler_flag match flag1 flag2 target) + set(CMAKE_REQUIRED_FLAGS "-Werror") + + make_cpp_safe_name("${flag1}" flag1_name) + make_cpp_safe_name("${flag2}" flag2_name) + + if(NOT ${CMAKE_C_FLAGS} MATCHES ${match}) + check_c_compiler_flag("${flag1}" HAVE_C_${flag1_name}) + if(HAVE_C_${flag1_name}) + set(${target}_C_FLAGS "${${target}_C_FLAGS} ${flag1}" PARENT_SCOPE) + else() + check_c_compiler_flag("${flag2}" HAVE_C_${flag2_name}) + if(HAVE_C_${flag2_name}) + set(${target}_C_FLAGS "${${target}_C_FLAGS} ${flag2}" PARENT_SCOPE) + endif() + endif() + endif() + + if(NOT ${CMAKE_CXX_FLAGS} MATCHES ${match}) + check_cxx_compiler_flag("${flag1}" HAVE_CXX_${flag1_name}) + if(HAVE_CXX_${flag1_name}) + set(${target}_CXX_FLAGS "${${target}_CXX_FLAGS} ${flag1}" PARENT_SCOPE) + else() + check_cxx_compiler_flag("${flag2}" HAVE_CXX_${flag2_name}) + if(HAVE_CXX_${flag2_name}) + set(${target}_CXX_FLAGS "${${target}_CXX_FLAGS} ${flag2}" PARENT_SCOPE) + endif() + endif() + endif() +endfunction() diff --git a/packaging/cmake/Modules/NetdataDetectSystemd.cmake b/packaging/cmake/Modules/NetdataDetectSystemd.cmake new file mode 100644 index 000000000..ecac7aaf8 --- /dev/null +++ b/packaging/cmake/Modules/NetdataDetectSystemd.cmake @@ -0,0 +1,42 @@ +# CMake Module to handle all the systemd-related checks for Netdata. +# +# Copyright (c) 2024 Netdata Inc. +# SPDX-License-Identifier: GPL-3.0-or-later + +macro(detect_systemd) + find_library(SYSTEMD_LIBRARY NAMES systemd) + + set(ENABLE_DSYSTEMD_DBUS NO) + pkg_check_modules(SYSTEMD libsystemd) + + if(SYSTEMD_FOUND) + set(CMAKE_REQUIRED_LIBRARIES_BEFORE_SYSTEMD "${CMAKE_REQUIRED_LIBRARIES}") + set(CMAKE_REQUIRED_LIBRARIES "${CMAKE_REQUIRED_LIBRARIES};${SYSTEMD_LIBRARIES}") + + check_c_source_compiles(" + #include <systemd/sd-journal.h> + + int main() { + int x = SD_JOURNAL_OS_ROOT; + return 0; + }" HAVE_SD_JOURNAL_OS_ROOT) + + check_symbol_exists(SD_JOURNAL_OS_ROOT "systemd/sd-journal.h" HAVE_SD_JOURNAL_OS_ROOT) + check_symbol_exists(sd_journal_open_files_fd "systemd/sd-journal.h" HAVE_SD_JOURNAL_OPEN_FILES_FD) + check_symbol_exists(sd_journal_restart_fields "systemd/sd-journal.h" HAVE_SD_JOURNAL_RESTART_FIELDS) + check_symbol_exists(sd_journal_get_seqnum "systemd/sd-journal.h" HAVE_SD_JOURNAL_GET_SEQNUM) + + check_symbol_exists(sd_bus_default_system "systemd/sd-bus.h" HAVE_SD_BUS_DEFAULT_SYSTEM) + check_symbol_exists(sd_bus_call_method "systemd/sd-bus.h" HAVE_SD_BUS_CALL_METHOD) + check_symbol_exists(sd_bus_message_enter_container "systemd/sd-bus.h" HAVE_SD_BUS_MESSAGE_ENTER_CONTAINER) + check_symbol_exists(sd_bus_message_read "systemd/sd-bus.h" HAVE_SD_BUS_MESSAGE_READ) + check_symbol_exists(sd_bus_message_exit_container "systemd/sd-bus.h" HAVE_SD_BUS_MESSAGE_EXIT_CONTAINER) + + set(CMAKE_REQUIRED_LIBRARIES "${CMAKE_REQUIRED_LIBRARIES_BEFORE_SYSTEMD}") + + set(HAVE_SYSTEMD True) + if(HAVE_SD_BUS_DEFAULT_SYSTEM AND HAVE_SD_BUS_CALL_METHOD AND HAVE_SD_BUS_MESSAGE_ENTER_CONTAINER AND HAVE_SD_BUS_MESSAGE_READ AND HAVE_SD_BUS_MESSAGE_EXIT_CONTAINER) + set(ENABLE_SYSTEMD_DBUS YES) + endif() + endif() +endmacro() diff --git a/packaging/cmake/Modules/NetdataFetchContentExtra.cmake b/packaging/cmake/Modules/NetdataFetchContentExtra.cmake new file mode 100644 index 000000000..1de1dcef9 --- /dev/null +++ b/packaging/cmake/Modules/NetdataFetchContentExtra.cmake @@ -0,0 +1,27 @@ +# Extra tools for working with FetchContent on older CMake +# +# Copyright (c) 2024 Netdata Inc. +# SPDX-License-Identifier: GPL-3.0-or-later + +# FetchContent_MakeAvailable_NoInstall +# +# Add a sub-project with FetchContent, but with the EXCLUDE_FROM_ALL +# argument for the add_subdirectory part. +# +# CMake 3.28 and newer provide a way to do this with an extra argument +# on FetchContent_Declare, but older versions need you to implement +# the logic yourself. Once we no longer support CMake versions older +# than 3.28, we can get rid of this macro. +# +# Unlike FetchContent_MakeAvailble, this only accepts a single project +# to make available. +macro(FetchContent_MakeAvailable_NoInstall name) + include(FetchContent) + + FetchContent_GetProperties(${name}) + + if(NOT ${name}_POPULATED) + FetchContent_Populate(${name}) + add_subdirectory(${${name}_SOURCE_DIR} ${${name}_BINARY_DIR} EXCLUDE_FROM_ALL) + endif() +endmacro() diff --git a/packaging/cmake/Modules/NetdataGoTools.cmake b/packaging/cmake/Modules/NetdataGoTools.cmake new file mode 100644 index 000000000..9bb09cdc5 --- /dev/null +++ b/packaging/cmake/Modules/NetdataGoTools.cmake @@ -0,0 +1,85 @@ +# Macros and functions to assist in working with Go +# +# Copyright (c) 2024 Netdata Inc +# +# SPDX-License-Identifier: GPL + +if(CMAKE_BUILD_TYPE STREQUAL Debug OR CMAKE_BUILD_TYPE STREQUAL RelWithDebInfo) + set(GO_LDFLAGS "-X github.com/netdata/netdata/go/go.d.plugin/pkg/buildinfo.Version=${NETDATA_VERSION}") +else() + set(GO_LDFLAGS "-w -s -X github.com/netdata/netdata/go/go.d.plugin/pkg/buildinfo.Version=${NETDATA_VERSION}") +endif() + +# add_go_target: Add a new target that needs to be built using the Go toolchain. +# +# Takes four arguments, the target name, the output artifact name, the +# source tree for the Go module, and the sub-directory of that source tree +# to pass to `go build`. +# +# The target itself will invoke `go build` in the specified source tree, +# using the `-o` option to produce the final output artifact, and passing +# the requested sub-directory as the final argument. +# +# This will also automatically construct the dependency list for the +# target by finding all Go source files under the specified source tree +# and then appending the go.mod and go.sum files from the root of the +# source tree. +macro(add_go_target target output build_src build_dir) + file(GLOB_RECURSE ${target}_DEPS CONFIGURE_DEPENDS "${build_src}/*.go") + list(APPEND ${target}_DEPS + "${build_src}/go.mod" + "${build_src}/go.sum" + ) + + add_custom_command( + OUTPUT ${output} + COMMAND "${CMAKE_COMMAND}" -E env CGO_ENABLED=0 "${GO_EXECUTABLE}" build -buildvcs=false -ldflags "${GO_LDFLAGS}" -o "${CMAKE_BINARY_DIR}/${output}" "./${build_dir}" + DEPENDS ${${target}_DEPS} + COMMENT "Building Go component ${output}" + WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}/${build_src}" + VERBATIM + ) + add_custom_target( + ${target} ALL + DEPENDS ${output} + ) +endmacro() + +# find_min_go_version: Determine the minimum Go version based on go.mod files +# +# Takes one argument, specifying a source tree to scan for go.mod files. +# +# All files found will be checked for a `go` directive, and the +# MIN_GO_VERSION variable will be set to the highest version +# number found among these directives. +# +# Only works on UNIX-like systems, because it has to process the go.mod +# files in ways that CMake can't do on it's own. +function(find_min_go_version src_tree) + message(STATUS "Determining minimum required version of Go for this build") + + file(GLOB_RECURSE go_mod_files ${src_tree}/go.mod) + + set(result 1.0) + + foreach(f IN ITEMS ${go_mod_files}) + message(VERBOSE "Checking Go version specified in ${f}") + execute_process( + COMMAND grep -E "^go .*$" ${f} + COMMAND cut -f 2 -d " " + RESULT_VARIABLE version_check_result + OUTPUT_VARIABLE go_mod_version + ) + + if(version_check_result EQUAL 0) + string(REGEX MATCH "([0-9]+\\.[0-9]+(\\.[0-9]+)?)" go_mod_version "${go_mod_version}") + + if(go_mod_version VERSION_GREATER result) + set(result "${go_mod_version}") + endif() + endif() + endforeach() + + message(STATUS "Minimum required Go version determined to be ${result}") + set(MIN_GO_VERSION "${result}" PARENT_SCOPE) +endfunction() diff --git a/packaging/cmake/Modules/NetdataProtobuf.cmake b/packaging/cmake/Modules/NetdataProtobuf.cmake new file mode 100644 index 000000000..d4ae3aec6 --- /dev/null +++ b/packaging/cmake/Modules/NetdataProtobuf.cmake @@ -0,0 +1,225 @@ +# Macros and functions for handling of Protobuf +# +# Copyright (c) 2024 Netdata Inc. +# SPDX-License-Identifier: GPL-3.0-or-later + +macro(netdata_protobuf_21_tags) + set(PROTOBUF_TAG f0dc78d7e6e331b8c6bb2d5283e06aa26883ca7c) # v21.12 + set(NEED_ABSL False) +endmacro() + +macro(netdata_protobuf_25_tags) + set(PROTOBUF_TAG 4a2aef570deb2bfb8927426558701e8bfc26f2a4) # v25.3 + set(NEED_ABSL True) + set(ABSL_TAG 2f9e432cce407ce0ae50676696666f33a77d42ac) # 20240116.1 +endmacro() + +# Determine what version of protobuf and abseil to bundle. +# +# This is unfortunately very complicated because we support systems +# older than what Google officially supports for C++. +macro(netdata_set_bundled_protobuf_tags) + netdata_protobuf_21_tags() + + if(NOT USE_CXX_11) + if(CMAKE_CXX_COMPILER_ID STREQUAL GNU) + if(CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL 7.3.1) + netdata_protobuf_25_tags() + endif() + elseif(CMAKE_CXX_COMPILER_ID STREQUAL Clang) + if(CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL 7.0.0) + netdata_protobuf_25_tags() + endif() + elseif(CMAKE_CXX_COMPILER_ID STREQUAL AppleClang) + if(CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL 12) + netdata_protobuf_25_tags() + endif() + endif() + endif() +endmacro() + +# Prepare a vendored copy of Protobuf for use with Netdata. +function(netdata_bundle_protobuf) + include(FetchContent) + include(NetdataFetchContentExtra) + + netdata_set_bundled_protobuf_tags() + + set(FETCHCONTENT_TRY_FIND_PACKAGE_MODE NEVER) + + string(REPLACE "-fsanitize=address" "" CMAKE_C_FLAGS ${CMAKE_C_FLAGS}) + string(REPLACE "-fsanitize=address" "" CMAKE_CXX_FLAGS ${CMAKE_CXX_FLAGS}) + + # ignore debhelper + set(FETCHCONTENT_FULLY_DISCONNECTED Off) + + if(NEED_ABSL) + set(ABSL_PROPAGATE_CXX_STD On) + set(ABSL_ENABLE_INSTALL Off) + + message(STATUS "Preparing bundled Abseil (required by bundled Protobuf)") + FetchContent_Declare(absl + GIT_REPOSITORY https://github.com/abseil/abseil-cpp + GIT_TAG ${ABSL_TAG} + ) + FetchContent_MakeAvailable_NoInstall(absl) + message(STATUS "Finished preparing bundled Abseil") + endif() + + set(protobuf_INSTALL Off) + set(protobuf_BUILD_LIBPROTOC Off) + set(protobuf_BUILD_TESTS Off) + set(protobuf_BUILD_SHARED_LIBS Off) + + message(STATUS "Preparing bundled Protobuf") + FetchContent_Declare(protobuf + GIT_REPOSITORY https://github.com/protocolbuffers/protobuf.git + GIT_TAG ${PROTOBUF_TAG} + ) + FetchContent_MakeAvailable_NoInstall(protobuf) + message(STATUS "Finished preparing bundled Protobuf.") + + set(BUNDLED_PROTOBUF True PARENT_SCOPE) +endfunction() + +# Handle detection of Protobuf +macro(netdata_detect_protobuf) + if(NOT ENABLE_BUNDLED_PROTOBUF) + if (NOT BUILD_SHARED_LIBS) + set(Protobuf_USE_STATIC_LIBS On) + endif() + + # The FindProtobuf CMake module shipped by upstream CMake is + # broken for Protobuf version 22.0 and newer because it does + # not correctly pull in the new Abseil dependencies. Protobuf + # itself sometimes ships a CMake Package Configuration module + # that _does_ work correctly, so use that in preference to the + # Find module shipped with CMake. + # + # The code below works by first attempting to use find_package + # in config mode, and then checking for the existence of the + # target we actually use that gets defined by the protobuf + # CMake Package Configuration Module to determine if that + # worked. A bit of extra logic is required in the case of the + # config mode working, because some systems ship compatibility + # logic for the old FindProtobuf module while others do not. + # + # Upstream bug reference: https://gitlab.kitware.com/cmake/cmake/-/issues/24321 + find_package(Protobuf CONFIG) + + if(NOT TARGET protobuf::libprotobuf) + message(STATUS "Could not find Protobuf using Config mode, falling back to Module mode") + find_package(Protobuf REQUIRED) + endif() + endif() + + if(TARGET protobuf::libprotobuf) + if(NOT Protobuf_PROTOC_EXECUTABLE AND TARGET protobuf::protoc) + get_target_property(Protobuf_PROTOC_EXECUTABLE protobuf::protoc + IMPORTED_LOCATION_RELEASE) + if(NOT EXISTS "${Protobuf_PROTOC_EXECUTABLE}") + get_target_property(Protobuf_PROTOC_EXECUTABLE protobuf::protoc + IMPORTED_LOCATION_RELWITHDEBINFO) + endif() + if(NOT EXISTS "${Protobuf_PROTOC_EXECUTABLE}") + get_target_property(Protobuf_PROTOC_EXECUTABLE protobuf::protoc + IMPORTED_LOCATION_MINSIZEREL) + endif() + if(NOT EXISTS "${Protobuf_PROTOC_EXECUTABLE}") + get_target_property(Protobuf_PROTOC_EXECUTABLE protobuf::protoc + IMPORTED_LOCATION_DEBUG) + endif() + if(NOT EXISTS "${Protobuf_PROTOC_EXECUTABLE}") + get_target_property(Protobuf_PROTOC_EXECUTABLE protobuf::protoc + IMPORTED_LOCATION_NOCONFIG) + endif() + if(NOT Protobuf_PROTOC_EXECUTABLE) + set(Protobuf_PROTOC_EXECUTABLE protobuf::protoc) + endif() + endif() + + # It is technically possible that this may still not + # be set by this point, so we need to check it and + # fail noisily if it isn't because the build won't + # work without it. + if(NOT Protobuf_PROTOC_EXECUTABLE) + message(FATAL_ERROR "Could not determine the location of the protobuf compiler for the detected version of protobuf.") + endif() + + set(NETDATA_PROTOBUF_PROTOC_EXECUTABLE ${Protobuf_PROTOC_EXECUTABLE}) + set(NETDATA_PROTOBUF_LIBS protobuf::libprotobuf) + get_target_property(NETDATA_PROTOBUF_CFLAGS_OTHER + protobuf::libprotobuf + INTERFACE_COMPILE_DEFINITIONS) + get_target_property(NETDATA_PROTOBUF_INCLUDE_DIRS + protobuf::libprotobuf + INTERFACE_INCLUDE_DIRECTORIES) + + if(NETDATA_PROTOBUF_CFLAGS_OTHER STREQUAL NETDATA_PROTOBUF_CFLAGS_OTHER-NOTFOUND) + set(NETDATA_PROTOBUF_CFLAGS_OTHER "") + endif() + + if(NETDATA_PROTOBUF_INCLUDE_DIRS STREQUAL NETDATA_PROTOBUF_INCLUDE_DIRS-NOTFOUND) + set(NETDATA_PROTOBUF_INCLUDE_DIRS "") + endif() + else() + set(NETDATA_PROTOBUF_PROTOC_EXECUTABLE ${PROTOBUF_PROTOC_EXECUTABLE}) + set(NETDATA_PROTOBUF_CFLAGS_OTHER ${PROTOBUF_CFLAGS_OTHER}) + set(NETDATA_PROTOBUF_INCLUDE_DIRS ${PROTOBUF_INCLUDE_DIRS}) + set(NETDATA_PROTOBUF_LIBS ${PROTOBUF_LIBRARIES}) + endif() + + set(ENABLE_PROTOBUF True) + set(HAVE_PROTOBUF True) +endmacro() + +# Helper function to compile protocol definitions into C++ code. +function(netdata_protoc_generate_cpp INC_DIR OUT_DIR SRCS HDRS) + if(NOT ARGN) + message(SEND_ERROR "Error: protoc_generate_cpp() called without any proto files") + return() + endif() + + set(${INC_DIR}) + set(${OUT_DIR}) + set(${SRCS}) + set(${HDRS}) + + foreach(FIL ${ARGN}) + get_filename_component(ABS_FIL ${FIL} ABSOLUTE) + get_filename_component(DIR ${ABS_FIL} DIRECTORY) + get_filename_component(FIL_WE ${FIL} NAME_WE) + + set(GENERATED_PB_CC "${DIR}/${FIL_WE}.pb.cc") + list(APPEND ${SRCS} ${GENERATED_PB_CC}) + + set(GENERATED_PB_H "${DIR}/${FIL_WE}.pb.h") + list(APPEND ${HDRS} ${GENERATED_PB_H}) + + list(APPEND _PROTOC_INCLUDE_DIRS ${INC_DIR}) + + if(ENABLE_BUNDLED_PROTOBUF) + list(APPEND _PROTOC_INCLUDE_DIRS ${CMAKE_BINARY_DIR}/_deps/protobuf-src/src/) + endif() + + add_custom_command(OUTPUT ${GENERATED_PB_CC} ${GENERATED_PB_H} + COMMAND ${NETDATA_PROTOBUF_PROTOC_EXECUTABLE} + ARGS "-I$<JOIN:${_PROTOC_INCLUDE_DIRS},;-I>" --cpp_out=${OUT_DIR} ${ABS_FIL} + DEPENDS ${ABS_FIL} ${NETDATA_PROTOBUF_PROTOC_EXECUTABLE} + COMMENT "Running C++ protocol buffer compiler on ${FIL}" + COMMAND_EXPAND_LISTS) + endforeach() + + set_source_files_properties(${${SRCS}} ${${HDRS}} PROPERTIES GENERATED TRUE) + set_source_files_properties(${${SRCS}} ${${HDRS}} PROPERTIES COMPILE_OPTIONS -Wno-deprecated-declarations) + + set(${SRCS} ${${SRCS}} PARENT_SCOPE) + set(${HDRS} ${${HDRS}} PARENT_SCOPE) +endfunction() + +# Add protobuf to a specified target. +function(netdata_add_protobuf _target) + target_compile_definitions(${_target} PRIVATE ${NETDATA_PROTOBUF_CFLAGS_OTHER}) + target_include_directories(${_target} PRIVATE ${NETDATA_PROTOBUF_INCLUDE_DIRS}) + target_link_libraries(${_target} PRIVATE ${NETDATA_PROTOBUF_LIBS}) +endfunction() diff --git a/packaging/cmake/Modules/NetdataYAML.cmake b/packaging/cmake/Modules/NetdataYAML.cmake new file mode 100644 index 000000000..73af55167 --- /dev/null +++ b/packaging/cmake/Modules/NetdataYAML.cmake @@ -0,0 +1,65 @@ +# Functions and macros for handling of libYAML +# +# Copyright (c) 2024 Netdata Inc. +# SPDX-License-Identifier: GPL-3.0-or-later + +# Handle bundling of libyaml. +# +# This pulls it in as a sub-project using FetchContent functionality. +# +# This needs to be a function and not a macro for variable scoping +# reasons. All the things we care about from the sub-project are exposed +# as targets, which are globally scoped and not function scoped. +function(netdata_bundle_libyaml) + include(FetchContent) + include(NetdataFetchContentExtra) + + if(ENABLE_BUNDLED_LIBYAML) + set(FETCHCONTENT_TRY_FIND_PACKAGE_MODE NEVER) + endif() + + set(FETCHCONTENT_FULLY_DISCONNECTED Off) + + FetchContent_Declare(yaml + GIT_REPOSITORY https://github.com/yaml/libyaml + GIT_TAG 2c891fc7a770e8ba2fec34fc6b545c672beb37e6 # v0.2.5 + ) + + FetchContent_MakeAvailable_NoInstall(yaml) +endfunction() + +# Handle setup of libyaml for the build. +# +# This will attempt to find libyaml using pkg_check_modules. If it finds +# a usable copy, that will be used. If not, it will bundle a vendored copy +# as a sub-project. +# +# Irrespective of how libyaml is to be included, library names, +# include directories, and compile definitions will be specified in the +# NETDATA_YAML_* variables for later use. +macro(netdata_detect_libyaml) + set(HAVE_LIBYAML True) + + pkg_check_modules(YAML yaml-0.1) + + if(ENABLE_BUNDLED_LIBYAML OR NOT YAML_FOUND) + netdata_bundle_libyaml() + set(NETDATA_YAML_LDFLAGS yaml) + get_target_property(NETDATA_YAML_INCLUDE_DIRS yaml INTERFACE_INCLUDE_DIRECTORIES) + get_target_property(NETDATA_YAML_CFLAGS_OTHER yaml INTERFACE_COMPILE_DEFINITIONS) + else() + set(NETDATA_YAML_LDFLAGS ${YAML_LDFLAGS}) + set(NETDATA_YAML_CFLAGS_OTHER ${YAML_CFLAGS_OTHER}) + set(NETDATA_YAML_INCLUDE_DIRS ${YAML_INCLUDE_DIRS}) + endif() +endmacro() + +# Add libyaml as a public link dependency of the specified target. +# +# The specified target must already exist, and the netdata_detect_libyaml +# macro must have already been run at least once for this to work correctly. +function(netdata_add_libyaml_to_target _target) + target_include_directories(${_target} PUBLIC ${NETDATA_YAML_INCLUDE_DIRS}) + target_compile_definitions(${_target} PUBLIC ${NETDATA_YAML_CFLAGS_OTHER}) + target_link_libraries(${_target} PUBLIC ${NETDATA_YAML_LDFLAGS}) +endfunction() diff --git a/packaging/cmake/config.cmake.h.in b/packaging/cmake/config.cmake.h.in new file mode 100644 index 000000000..276b051f9 --- /dev/null +++ b/packaging/cmake/config.cmake.h.in @@ -0,0 +1,166 @@ +/* This file was generated by CMAKE from config.cmake.h.in */ + +#define likely(x) __builtin_expect(!!(x), 1) +#define unlikely(x) __builtin_expect(!!(x), 0) + +#define __always_unused __attribute__((unused)) +#define __maybe_unused __attribute__((unused)) + +#cmakedefine SIZEOF_VOID_P ${SIZEOF_VOID_P} + +// platform +#cmakedefine COMPILED_FOR_FREEBSD +#cmakedefine COMPILED_FOR_LINUX +#cmakedefine COMPILED_FOR_MACOS + +// checked headers + +#cmakedefine HAVE_NETINET_IN_H +#cmakedefine HAVE_RESOLV_H +#cmakedefine HAVE_NETDB_H +#cmakedefine HAVE_SYS_PRCTL_H +#cmakedefine HAVE_SYS_STAT_H +#cmakedefine HAVE_SYS_VFS_H +#cmakedefine HAVE_SYS_STATFS_H +#cmakedefine HAVE_LINUX_MAGIC_H +#cmakedefine HAVE_SYS_MOUNT_H +#cmakedefine HAVE_SYS_STATVFS_H +#cmakedefine HAVE_INTTYPES_H +#cmakedefine HAVE_STDINT_H +#cmakedefine HAVE_SYS_CAPABILITY_H + +#cmakedefine HAVE_CAPABILITY +#cmakedefine HAVE_PROTOBUF +#cmakedefine BUNDLED_PROTOBUF +#cmakedefine HAVE_MONGOC +#cmakedefine HAVE_LIBDATACHANNEL + +// checked symbols + +#cmakedefine MAJOR_IN_SYSMACROS +#cmakedefine MAJOR_IN_MKDEV +#cmakedefine HAVE_CLOCK_GETTIME +#cmakedefine HAVE_STRERROR_R +#cmakedefine HAVE_FINITE +#cmakedefine HAVE_ISFINITE +#cmakedefine HAVE_RECVMMSG +#cmakedefine HAVE_NICE +#cmakedefine HAVE_GETPRIORITY +#cmakedefine HAVE_DLSYM + +#cmakedefine HAVE_BACKTRACE +#cmakedefine HAVE_CLOSE_RANGE +#cmakedefine HAVE_SCHED_GETSCHEDULER +#cmakedefine HAVE_SCHED_SETSCHEDULER +#cmakedefine HAVE_SCHED_GET_PRIORITY_MIN +#cmakedefine HAVE_SCHED_GET_PRIORITY_MAX + +#cmakedefine HAVE_SYSTEMD +#cmakedefine HAVE_SD_JOURNAL_OS_ROOT +#cmakedefine HAVE_SD_JOURNAL_OPEN_FILES_FD +#cmakedefine HAVE_SD_JOURNAL_RESTART_FIELDS +#cmakedefine HAVE_SD_JOURNAL_GET_SEQNUM +#cmakedefine ENABLE_SYSTEMD_DBUS + +// checked source compilation + +#cmakedefine HAVE_PTHREAD_GETNAME_NP +#cmakedefine HAVE_ACCEPT4 +#cmakedefine STRERROR_R_CHAR_P +#cmakedefine HAVE_C__GENERIC +#cmakedefine HAVE_C_MALLOPT +#cmakedefine HAVE_SETNS + +#cmakedefine HAVE_FUNC_ATTRIBUTE_FORMAT +#cmakedefine HAVE_FUNC_ATTRIBUTE_MALLOC +#cmakedefine HAVE_FUNC_ATTRIBUTE_NOINLINE +#cmakedefine HAVE_FUNC_ATTRIBUTE_NORETURN +#cmakedefine HAVE_FUNC_ATTRIBUTE_RETURNS_NONNULL +#cmakedefine HAVE_FUNC_ATTRIBUTE_WARN_UNUSED_RESULT + +// enabled features + +#cmakedefine ENABLE_OPENSSL +#cmakedefine ENABLE_CLOUD +#cmakedefine ENABLE_ACLK +#cmakedefine ENABLE_ML +#cmakedefine ENABLE_EXPORTING_MONGODB +#cmakedefine ENABLE_H2O +#cmakedefine ENABLE_DBENGINE +#cmakedefine ENABLE_HTTPS +#cmakedefine ENABLE_LZ4 +#cmakedefine ENABLE_ZSTD +#cmakedefine ENABLE_BROTLI + +#cmakedefine ENABLE_LOGSMANAGEMENT +#cmakedefine ENABLE_LOGSMANAGEMENT_TESTS +#cmakedefine ENABLE_PROMETHEUS_REMOTE_WRITE + +// enabled plugins + +#cmakedefine ENABLE_PLUGIN_DEBUGFS +#cmakedefine ENABLE_PLUGIN_APPS +#cmakedefine ENABLE_PLUGIN_FREEIPMI +#cmakedefine ENABLE_PLUGIN_NFACCT +#cmakedefine ENABLE_PLUGIN_XENSTAT +#cmakedefine ENABLE_PLUGIN_PERF +#cmakedefine ENABLE_PLUGIN_SLABINFO +#cmakedefine ENABLE_PLUGIN_CUPS +#cmakedefine ENABLE_PLUGIN_CGROUP_NETWORK +#cmakedefine ENABLE_PLUGIN_EBPF + +// enabled sentry + +#cmakedefine ENABLE_SENTRY +#cmakedefine NETDATA_SENTRY_ENVIRONMENT "@NETDATA_SENTRY_ENVIRONMENT@" +#cmakedefine NETDATA_SENTRY_RELEASE "@NETDATA_SENTRY_RELEASE@" +#cmakedefine NETDATA_SENTRY_DIST "@NETDATA_SENTRY_DIST@" +#cmakedefine NETDATA_SENTRY_DSN "@NETDATA_SENTRY_DSN@" +// enabled bundling + +#cmakedefine ENABLE_BUNDLED_JSONC +#cmakedefine ENABLE_BUNDLED_YAML +#cmakedefine ENABLE_BUNDLED_PROTOBUF + +// directory paths + +#cmakedefine CACHE_DIR "@CACHE_DIR@" +#cmakedefine CONFIG_DIR "@CONFIG_DIR@" +#cmakedefine LIBCONFIG_DIR "@LIBCONFIG_DIR@" +#cmakedefine LOG_DIR "@LOG_DIR@" +#cmakedefine PLUGINS_DIR "@PLUGINS_DIR@" +#cmakedefine WEB_DIR "@WEB_DIR@" +#cmakedefine VARLIB_DIR "@VARLIB_DIR@" + +// config command, user and version + +#cmakedefine CONFIGURE_COMMAND "@CONFIGURE_COMMAND@" +#cmakedefine NETDATA_USER "@NETDATA_USER@" + +#cmakedefine VERSION "@GIT_DESCRIBE_OUTPUT@" + +#define ENABLE_JSONC 1 + +#cmakedefine HAVE_LIBYAML +#cmakedefine HAVE_LIBMNL + +// /* Enable GNU extensions on systems that have them. */ +// #ifndef _GNU_SOURCE +// # define _GNU_SOURCE 1 +// #endif + +// #cmakedefine HAVE_CRYPTO + +// #cmakedefine ENABLE_PROMETHEUS_REMOTE_WRITE + +// /* NSA spy stuff */ +// #define ENABLE_HTTPS 1 +// #cmakedefine01 HAVE_X509_VERIFY_PARAM_set1_host + +#define HAVE_CRYPTO +#define HAVE_X509_VERIFY_PARAM_set1_host 1 + +/* Enable GNU extensions on systems that have them. */ +#ifndef _GNU_SOURCE +#define _GNU_SOURCE 1 +#endif diff --git a/packaging/current_libbpf.checksums b/packaging/current_libbpf.checksums index 8279c1dd3..e6e780bae 100644 --- a/packaging/current_libbpf.checksums +++ b/packaging/current_libbpf.checksums @@ -1 +1 @@ -05e4ccdd3bc8532290eebc37b37455b253071244d30e42412a7071d89221f1c8 v1.2.2p_netdata.tar.gz +842a285050a103168347689fa8b7bdd677465534862b9e85808d0a8f793805b8 v1.3.0p_netdata.tar.gz diff --git a/packaging/current_libbpf.version b/packaging/current_libbpf.version index b5dc2d8bd..816b50271 100644 --- a/packaging/current_libbpf.version +++ b/packaging/current_libbpf.version @@ -1 +1 @@ -1.2.2p_netdata +1.3.0p_netdata diff --git a/packaging/dag/README.md b/packaging/dag/README.md new file mode 100644 index 000000000..4b17c5447 --- /dev/null +++ b/packaging/dag/README.md @@ -0,0 +1,23 @@ +- Install Dagger CLI: + ``` + cd /usr/local + curl -L https://dl.dagger.io/dagger/install.sh | sudo sh + ``` +- Install python requirements: + ``` + pip install -r packaging/dag/requirements.txt + ``` + +Now you can run something like this: + +``` +dagger run python packaging/dag/main.py build -p linux/x86_64 -d debian12 +``` + +or + +``` +dagger run python packaging/dag/main.py test +```. + + diff --git a/packaging/dag/build_command.py b/packaging/dag/build_command.py new file mode 100644 index 000000000..dcb1b6c8a --- /dev/null +++ b/packaging/dag/build_command.py @@ -0,0 +1,65 @@ +import click +import asyncio +import sys +import dagger +import pathlib +import uuid + +from nd import ( + Distribution, + NetdataInstaller, + FeatureFlags, + Endpoint, + AgentContext, + SUPPORTED_PLATFORMS, + SUPPORTED_DISTRIBUTIONS, +) + + +def run_async(func): + def wrapper(*args, **kwargs): + return asyncio.run(func(*args, **kwargs)) + + return wrapper + + +@run_async +async def simple_build(platform, distro): + config = dagger.Config(log_output=sys.stdout) + + async with dagger.Connection(config) as client: + repo_root = pathlib.Path("/netdata") + prefix_path = pathlib.Path("/opt/netdata") + + installer = NetdataInstaller( + platform, distro, repo_root, prefix_path, FeatureFlags.DBEngine + ) + + endpoint = Endpoint("node", 19999) + api_key = uuid.uuid4() + allow_children = False + + agent_ctx = AgentContext( + client, platform, distro, installer, endpoint, api_key, allow_children + ) + + await agent_ctx.build_container() + + +@click.command() +@click.option( + "--platform", + "-p", + type=click.Choice(sorted([str(p) for p in SUPPORTED_PLATFORMS])), + help="Specify the platform.", +) +@click.option( + "--distribution", + "-d", + type=click.Choice(sorted([str(p) for p in SUPPORTED_DISTRIBUTIONS])), + help="Specify the distribution.", +) +def build(platform, distribution): + platform = dagger.Platform(platform) + distro = Distribution(distribution) + simple_build(platform, distro) diff --git a/packaging/dag/files/child_stream.conf b/packaging/dag/files/child_stream.conf new file mode 100644 index 000000000..ed78bd3fb --- /dev/null +++ b/packaging/dag/files/child_stream.conf @@ -0,0 +1,10 @@ +[stream] + enabled = {{ enabled }} + destination = {{ destination }} + api key = {{ api_key }} + timeout seconds = {{ timeout_seconds }} + default port = {{ default_port }} + send charts matching = {{ send_charts_matching }} + buffer size bytes = {{ buffer_size_bytes }} + reconnect delay seconds = {{ reconnect_delay_seconds }} + initial clock resync iterations = {{ initial_clock_resync_iterations }} diff --git a/packaging/dag/files/cmake-aarch64.sha256 b/packaging/dag/files/cmake-aarch64.sha256 new file mode 100644 index 000000000..122b26e99 --- /dev/null +++ b/packaging/dag/files/cmake-aarch64.sha256 @@ -0,0 +1 @@ +a83e01ed1cdf44c2e33e0726513b9a35a8c09e3b5a126fd720b3c8a9d5552368 cmake-aarch64.sh diff --git a/packaging/dag/files/cmake-x86_64.sha256 b/packaging/dag/files/cmake-x86_64.sha256 new file mode 100644 index 000000000..d5adc8aa1 --- /dev/null +++ b/packaging/dag/files/cmake-x86_64.sha256 @@ -0,0 +1 @@ +8c449dabb2b2563ec4e6d5e0fb0ae09e729680efab71527b59015131cea4a042 cmake-x86_64.sh diff --git a/packaging/dag/files/ol8-epel.repo b/packaging/dag/files/ol8-epel.repo new file mode 100644 index 000000000..587cc3577 --- /dev/null +++ b/packaging/dag/files/ol8-epel.repo @@ -0,0 +1,6 @@ +[ol8_developer_EPEL] +name=Oracle Linux $releasever EPEL Packages for Development ($basearch) +baseurl=https://yum$ociregion.$ocidomain/repo/OracleLinux/OL8/developer/EPEL/$basearch/ +gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-oracle +gpgcheck=1 +enabled=1 diff --git a/packaging/dag/files/ol9-epel.repo b/packaging/dag/files/ol9-epel.repo new file mode 100644 index 000000000..c40007f1f --- /dev/null +++ b/packaging/dag/files/ol9-epel.repo @@ -0,0 +1,6 @@ +[ol9_developer_EPEL] +name=Oracle Linux $releasever EPEL Packages for Development ($basearch) +baseurl=https://yum$ociregion.$ocidomain/repo/OracleLinux/OL9/developer/EPEL/$basearch/ +gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-oracle +gpgcheck=1 +enabled=1 diff --git a/packaging/dag/files/parent_stream.conf b/packaging/dag/files/parent_stream.conf new file mode 100644 index 000000000..15f303f97 --- /dev/null +++ b/packaging/dag/files/parent_stream.conf @@ -0,0 +1,7 @@ +[{{ api_key }}] + enabled = {{ enabled }} + allow from = {{ allow_from }} + default history = {{ default_history }} + health enabled by default = {{ health_enabled_by_default }} + default postpone alarms on connect seconds = {{ default_postpone_alarms_on_connect_seconds }} + multiple connections = {{ multiple_connections }} diff --git a/packaging/dag/imageutils.py b/packaging/dag/imageutils.py new file mode 100644 index 000000000..fd1e8ad26 --- /dev/null +++ b/packaging/dag/imageutils.py @@ -0,0 +1,1580 @@ +import os +from pathlib import Path + +import dagger + + +_ALPINE_COMMON_PACKAGES = [ + "alpine-sdk", + "autoconf", + "automake", + "bash", + "binutils", + "bison", + "cmake", + "curl", + "curl-static", + "elfutils-dev", + "flex", + "gcc", + "git", + "gnutls-dev", + "gzip", + "jq", + "libelf-static", + "libmnl-dev", + "libmnl-static", + "libtool", + "libuv-dev", + "libuv-static", + "lz4-dev", + "lz4-static", + "make", + "mongo-c-driver-dev", + "mongo-c-driver-static", + "musl-fts-dev", + "ncurses", + "ncurses-static", + "netcat-openbsd", + "ninja", + "openssh", + "pcre2-dev", + "pkgconfig", + "protobuf-dev", + "snappy-dev", + "snappy-static", + "util-linux-dev", + "wget", + "xz", + "yaml-dev", + "yaml-static", + "zlib-dev", + "zlib-static", + "zstd-dev", + "zstd-static", +] + + +def build_alpine_3_18( + client: dagger.Client, platform: dagger.Platform +) -> dagger.Container: + ctr = client.container(platform=platform).from_("alpine:3.18") + + pkgs = [pkg for pkg in _ALPINE_COMMON_PACKAGES] + + ctr = ctr.with_exec(["apk", "add", "--no-cache"] + pkgs) + + return ctr + + +def build_alpine_3_19( + client: dagger.Client, platform: dagger.Platform +) -> dagger.Container: + ctr = client.container(platform=platform).from_("alpine:3.19") + + pkgs = [pkg for pkg in _ALPINE_COMMON_PACKAGES] + + ctr = ctr.with_exec(["apk", "add", "--no-cache"] + pkgs) + + return ctr + + +def static_build_openssl( + client: dagger.Client, ctr: dagger.Container +) -> dagger.Container: + tree = ( + client.git(url="https://github.com/openssl/openssl", keep_git_dir=True) + .tag("openssl-3.1.4") + .tree() + ) + + # + # TODO: verify 32-bit builds + # + ctr = ( + ctr.with_directory("/openssl", tree) + .with_workdir("/openssl") + .with_env_variable("CFLAGS", "-fno-lto -pipe") + .with_env_variable("LDFLAGS", "-static") + .with_env_variable("PKG_CONFIG", "pkg-config --static") + .with_exec( + [ + "sed", + "-i", + "s/disable('static', 'pic', 'threads');/disable('static', 'pic');/", + "Configure", + ] + ) + .with_exec( + [ + "./config", + "-static", + "threads", + "no-tests", + "--prefix=/openssl-static", + "--openssldir=/opt/netdata/etc/ssl", + ] + ) + .with_exec(["make", "V=1", "-j", str(os.cpu_count())]) + .with_exec(["make", "V=1", "-j", str(os.cpu_count()), "install_sw"]) + .with_exec(["ln", "-s", "/openssl-static/lib", "/openssl-static/lib64"]) + .with_exec(["perl", "configdata.pm", "--dump"]) + ) + + return ctr + + +def static_build_bash(client: dagger.Client, ctr: dagger.Container) -> dagger.Container: + tree = ( + client.git(url="https://git.savannah.gnu.org/git/bash.git", keep_git_dir=True) + .tag("bash-5.1") + .tree() + ) + + ctr = ( + ctr.with_directory("/bash", tree) + .with_workdir("/bash") + .with_env_variable("CFLAGS", "-pipe") + .with_env_variable("LDFLAGS", "") + .with_env_variable("PKG_CONFIG", "pkg-config --static") + .with_env_variable("PKG_CONFIG_PATH", "/openssl-static/lib64/pkgconfig") + .with_exec( + [ + "./configure", + "--prefix", + "/opt/netdata", + "--without-bash-malloc", + "--enable-static-link", + "--enable-net-redirections", + "--enable-array-variables", + "--disable-progcomp", + "--disable-profiling", + "--disable-nls", + "--disable-dependency-tracking", + ] + ) + .with_exec( + [ + "echo", + "-e", + "all:\nclean:\ninstall:\n", + ">", + "examples/loadables/Makefile", + ] + ) + .with_exec(["make", "clean"]) + # see: https://gitweb.gentoo.org/repo/gentoo.git/tree/app-shells/bash/files/bash-5.1-parallel_make.patch?id=4c2ebbf4b8bc660beb98cc2d845c73375d6e4f50 + .with_exec(["make", "V=1", "-j", "2", "install"]) + .with_exec(["strip", "/opt/netdata/bin/bash"]) + ) + + return ctr + + +def static_build_curl(client: dagger.Client, ctr: dagger.Container) -> dagger.Container: + tree = ( + client.git(url="https://github.com/curl/curl", keep_git_dir=True) + .tag("curl-8_4_0") + .tree() + ) + + ctr = ( + ctr.with_directory("/curl", tree) + .with_workdir("/curl") + .with_env_variable("CFLAGS", "-I/openssl-static/include -pipe") + .with_env_variable("LDFLAGS", "-static -L/openssl-static/lib64") + .with_env_variable("PKG_CONFIG", "pkg-config --static") + .with_env_variable("PKG_CONFIG_PATH", "/openssl-static/lib64/pkgconfig") + .with_exec(["autoreconf", "-ifv"]) + .with_exec( + [ + "./configure", + "--prefix=/curl-static", + "--enable-optimize", + "--disable-shared", + "--enable-static", + "--enable-http", + "--disable-ldap", + "--disable-ldaps", + "--enable-proxy", + "--disable-dict", + "--disable-telnet", + "--disable-tftp", + "--disable-pop3", + "--disable-imap", + "--disable-smb", + "--disable-smtp", + "--disable-gopher", + "--enable-ipv6", + "--enable-cookies", + "--with-ca-fallback", + "--with-openssl", + "--disable-dependency-tracking", + ] + ) + .with_exec( + ["sed", "-i", "-e", "s/LDFLAGS =/LDFLAGS = -all-static/", "src/Makefile"] + ) + .with_exec(["make", "clean"]) + .with_exec(["make", "V=1", "-j", str(os.cpu_count()), "install"]) + .with_exec(["cp", "/curl-static/bin/curl", "/opt/netdata/bin/curl"]) + .with_exec(["strip", "/opt/netdata/bin/curl"]) + ) + + return ctr + + +def static_build_ioping( + client: dagger.Client, ctr: dagger.Container +) -> dagger.Container: + tree = ( + client.git(url="https://github.com/koct9i/ioping", keep_git_dir=True) + .tag("v1.3") + .tree() + ) + + ctr = ( + ctr.with_directory("/ioping", tree) + .with_workdir("/ioping") + .with_env_variable("CFLAGS", "-static -pipe") + .with_exec(["mkdir", "-p", "/opt/netdata/usr/libexec/netdata/plugins.d"]) + .with_exec(["make", "V=1"]) + .with_exec( + [ + "install", + "-o", + "root", + "-g", + "root", + "-m", + "4750", + "ioping", + "/opt/netdata/usr/libexec/netdata/plugins.d", + ] + ) + .with_exec(["strip", "/opt/netdata/usr/libexec/netdata/plugins.d/ioping"]) + ) + + return ctr + + +def static_build_libnetfilter_acct( + client: dagger.Client, ctr: dagger.Container +) -> dagger.Container: + tree = ( + client.git(url="git://git.netfilter.org/libnetfilter_acct", keep_git_dir=True) + .tag("libnetfilter_acct-1.0.3") + .tree() + ) + + ctr = ( + ctr.with_directory("/libnetfilter_acct", tree) + .with_workdir("/libnetfilter_acct") + .with_env_variable("CFLAGS", "-static -I/usr/include/libmnl -pipe") + .with_env_variable("LDFLAGS", "-static -L/usr/lib -lmnl") + .with_env_variable("PKG_CONFIG", "pkg-config --static") + .with_env_variable("PKG_CONFIG_PATH", "/usr/lib/pkgconfig") + .with_exec(["autoreconf", "-ifv"]) + .with_exec( + [ + "./configure", + "--prefix=/libnetfilter_acct-static", + "--exec-prefix=/libnetfilter_acct-static", + ] + ) + .with_exec(["make", "clean"]) + .with_exec(["make", "V=1", "-j", str(os.cpu_count()), "install"]) + ) + + return ctr + + +def static_build_netdata( + client: dagger.Client, ctr: dagger.Container +) -> dagger.Container: + CFLAGS = [ + "-ffunction-sections", + "-fdata-sections", + "-static", + "-O2", + "-funroll-loops", + "-I/openssl-static/include", + "-I/libnetfilter_acct-static/include/libnetfilter_acct", + "-I/curl-local/include/curl", + "-I/usr/include/libmnl", + "-pipe", + ] + + LDFLAGS = [ + "-Wl,--gc-sections", + "-static", + "-L/openssl-static/lib64", + "-L/libnetfilter_acct-static/lib", + "-lnetfilter_acct", + "-L/usr/lib", + "-lmnl", + "-L/usr/lib", + "-lzstd", + "-L/curl-local/lib", + ] + + PKG_CONFIG = [ + "pkg-config", + "--static", + ] + + PKG_CONFIG_PATH = [ + "/openssl-static/lib64/pkgconfig", + "/libnetfilter_acct-static/lib/pkgconfig", + "/usr/lib/pkgconfig", + "/curl-local/lib/pkgconfig", + ] + + CMAKE_FLAGS = [ + "-DOPENSSL_ROOT_DIR=/openssl-static", + "-DOPENSSL_LIBRARIES=/openssl-static/lib64", + "-DCMAKE_INCLUDE_DIRECTORIES_PROJECT_BEFORE=/openssl-static", + "-DLWS_OPENSSL_INCLUDE_DIRS=/openssl-static/include", + "-DLWS_OPENSSL_LIBRARIES=/openssl-static/lib64/libssl.a;/openssl-static/lib64/libcrypto.a", + ] + + NETDATA_INSTALLER_CMD = [ + "./netdata-installer.sh", + "--install-prefix", + "/opt", + "--dont-wait", + "--dont-start-it", + "--disable-exporting-mongodb", + "--require-cloud", + "--use-system-protobuf", + "--dont-scrub-cflags-even-though-it-may-break-things", + "--one-time-build", + "--enable-lto", + ] + + ctr = ( + ctr.with_workdir("/netdata") + .with_env_variable("NETDATA_CMAKE_OPTIONS", "-DCMAKE_BUILD_TYPE=Debug") + .with_env_variable("CFLAGS", " ".join(CFLAGS)) + .with_env_variable("LDFLAGS", " ".join(LDFLAGS)) + .with_env_variable("PKG_CONFIG", " ".join(PKG_CONFIG)) + .with_env_variable("PKG_CONFIG_PATH", ":".join(PKG_CONFIG_PATH)) + .with_env_variable("CMAKE_FLAGS", " ".join(CMAKE_FLAGS)) + .with_env_variable("EBPF_LIBC", "static") + .with_env_variable("IS_NETDATA_STATIC_BINARY", "yes") + .with_exec(NETDATA_INSTALLER_CMD) + ) + + return ctr + + +def static_build(client, repo_path): + cmake_build_release_path = os.path.join(repo_path, "cmake-build-release") + + ctr = build_alpine_3_18(client, dagger.Platform("linux/x86_64")) + ctr = static_build_openssl(client, ctr) + ctr = static_build_bash(client, ctr) + ctr = static_build_curl(client, ctr) + ctr = static_build_ioping(client, ctr) + ctr = static_build_libnetfilter_acct(client, ctr) + + ctr = ctr.with_directory( + "/netdata", + client.host().directory(repo_path), + exclude=[ + f"{cmake_build_release_path}/*", + "fluent-bit/build", + ], + ) + + # TODO: link bin/sbin + + ctr = static_build_netdata(client, ctr) + + build_dir = ctr.directory("/opt/netdata") + artifact_dir = os.path.join(Path.home(), "ci/netdata-static") + output_task = build_dir.export(artifact_dir) + return output_task + + +_CENTOS_COMMON_PACKAGES = [ + "autoconf", + "autoconf-archive", + "autogen", + "automake", + "bison", + "bison-devel", + "cmake", + "cups-devel", + "curl", + "diffutils", + "elfutils-libelf-devel", + "findutils", + "flex", + "flex-devel", + "freeipmi-devel", + "gcc", + "gcc-c++", + "git-core", + "golang", + "json-c-devel", + "libyaml-devel", + "libatomic", + "libcurl-devel", + "libmnl-devel", + "libnetfilter_acct-devel", + "libtool", + "libuuid-devel", + "libuv-devel", + "libzstd-devel", + "lm_sensors", + "lz4-devel", + "make", + "ninja-build", + "openssl-devel", + "openssl-perl", + "patch", + "pcre2-devel", + "pkgconfig", + "pkgconfig(libmongoc-1.0)", + "procps", + "protobuf-c-devel", + "protobuf-compiler", + "protobuf-devel", + "rpm-build", + "rpm-devel", + "rpmdevtools", + "snappy-devel", + "systemd-devel", + "wget", + "zlib-devel", +] + + +def build_amazon_linux_2( + client: dagger.Client, platform: dagger.Platform +) -> dagger.Container: + ctr = client.container(platform=platform).from_("amazonlinux:2") + + pkgs = [pkg for pkg in _CENTOS_COMMON_PACKAGES] + + ctr = ( + ctr.with_exec(["yum", "update", "-y"]) + .with_exec(["yum", "install", "-y"] + pkgs) + .with_exec(["yum", "clean", "all"]) + .with_exec(["c_rehash"]) + .with_exec( + [ + "mkdir", + "-p", + "/root/rpmbuild/BUILD", + "/root/rpmbuild/RPMS", + "/root/rpmbuild/SOURCES", + "/root/rpmbuild/SPECS", + "/root/rpmbuild/SRPMS", + ] + ) + ) + + if platform == "linux/x86_64": + machine = "x86_64" + elif platform == "linux/arm64": + machine = "aarch64" + else: + raise Exception( + "Amaxon Linux 2 supports only linux/amd64 and linux/arm64 platforms." + ) + + + checksum_path = Path(__file__).parent / f"files/cmake-{machine}.sha256" + + ctr = ( + ctr.with_file( + f"cmake-{machine}.sha256", + client.host().file(checksum_path.as_posix()), + ) + .with_exec( + [ + "curl", + "--fail", + "-sSL", + "--connect-timeout", + "20", + "--retry", + "3", + "--output", + f"cmake-{machine}.sh", + f"https://github.com/Kitware/CMake/releases/download/v3.27.6/cmake-3.27.6-linux-{machine}.sh", + ] + ) + .with_exec(["sha256sum", "-c", f"cmake-{machine}.sha256"]) + .with_exec(["chmod", "u+x", f"./cmake-{machine}.sh"]) + .with_exec([f"./cmake-{machine}.sh", "--skip-license", "--prefix=/usr/local"]) + ) + + return ctr + + +def build_centos_7( + client: dagger.Client, platform: dagger.Platform +) -> dagger.Container: + ctr = client.container(platform=platform).from_("centos:7") + + pkgs = [pkg for pkg in _CENTOS_COMMON_PACKAGES] + ["bash"] + + ctr = ( + ctr.with_exec(["yum", "install", "-y", "epel-release"]) + .with_exec(["yum", "update", "-y"]) + .with_exec(["yum", "install", "-y"] + pkgs) + .with_exec(["yum", "clean", "all"]) + .with_exec(["c_rehash"]) + .with_exec( + [ + "mkdir", + "-p", + "/root/rpmbuild/BUILD", + "/root/rpmbuild/RPMS", + "/root/rpmbuild/SOURCES", + "/root/rpmbuild/SPECS", + "/root/rpmbuild/SRPMS", + ] + ) + ) + + if platform == "linux/x86_64": + machine = "x86_64" + elif platform == "linux/arm64": + machine = "aarch64" + else: + raise Exception("CentOS 7 supports only linux/amd64 and linux/arm64 platforms.") + + checksum_path = Path(__file__).parent / f"files/cmake-{machine}.sha256" + + ctr = ( + ctr.with_file( + f"cmake-{machine}.sha256", + client.host().file(checksum_path.as_posix()), + ) + .with_exec( + [ + "curl", + "--fail", + "-sSL", + "--connect-timeout", + "20", + "--retry", + "3", + "--output", + f"cmake-{machine}.sh", + f"https://github.com/Kitware/CMake/releases/download/v3.27.6/cmake-3.27.6-linux-{machine}.sh", + ] + ) + .with_exec(["sha256sum", "-c", f"cmake-{machine}.sha256"]) + .with_exec(["chmod", "u+x", f"./cmake-{machine}.sh"]) + .with_exec([f"./cmake-{machine}.sh", "--skip-license", "--prefix=/usr/local"]) + ) + + return ctr + + +_ROCKY_LINUX_COMMON_PACKAGES = [ + "autoconf", + "autoconf-archive", + "automake", + "bash", + "bison", + "cmake", + "cups-devel", + "curl", + "libcurl-devel", + "diffutils", + "elfutils-libelf-devel", + "findutils", + "flex", + "freeipmi-devel", + "gcc", + "gcc-c++", + "git", + "golang", + "json-c-devel", + "libatomic", + "libmnl-devel", + "libtool", + "libuuid-devel", + "libuv-devel", + "libyaml-devel", + "libzstd-devel", + "lm_sensors", + "lz4-devel", + "make", + "ninja-build", + "nc", + "openssl-devel", + "openssl-perl", + "patch", + "pcre2-devel", + "pkgconfig", + "pkgconfig(libmongoc-1.0)", + "procps", + "protobuf-c-devel", + "protobuf-compiler", + "protobuf-devel", + "python3", + "python3-pyyaml", + "rpm-build", + "rpm-devel", + "rpmdevtools", + "snappy-devel", + "systemd-devel", + "wget", + "zlib-devel", +] + + +def build_rocky_linux_8( + client: dagger.Client, platform: dagger.Platform +) -> dagger.Container: + ctr = client.container(platform=platform).from_("rockylinux:8") + + pkgs = [pkg for pkg in _ROCKY_LINUX_COMMON_PACKAGES] + ["autogen"] + + ctr = ( + ctr.with_exec(["dnf", "distro-sync", "-y", "--nodocs"]) + .with_exec( + [ + "dnf", + "install", + "-y", + "--nodocs", + "dnf-command(config-manager)", + "epel-release", + ] + ) + .with_exec(["dnf", "config-manager", "--set-enabled", "powertools"]) + .with_exec(["dnf", "clean", "packages"]) + .with_exec( + [ + "dnf", + "install", + "-y", + "--nodocs", + "--setopt=install_weak_deps=False", + "--setopt=diskspacecheck=False", + ] + + pkgs + ) + .with_exec(["rm", "-rf", "/var/cache/dnf"]) + .with_exec(["c_rehash"]) + .with_exec( + [ + "mkdir", + "-p", + "/root/rpmbuild/BUILD", + "/root/rpmbuild/RPMS", + "/root/rpmbuild/SOURCES", + "/root/rpmbuild/SPECS", + "/root/rpmbuild/SRPMS", + ] + ) + ) + + return ctr + + +def build_rocky_linux_9( + client: dagger.Client, platform: dagger.Platform +) -> dagger.Container: + ctr = client.container(platform=platform).from_("rockylinux:9") + + pkgs = [pkg for pkg in _ROCKY_LINUX_COMMON_PACKAGES] + + ctr = ( + ctr.with_exec(["dnf", "distro-sync", "-y", "--nodocs"]) + .with_exec( + [ + "dnf", + "install", + "-y", + "--nodocs", + "dnf-command(config-manager)", + "epel-release", + ] + ) + .with_exec(["dnf", "config-manager", "--set-enabled", "crb"]) + .with_exec(["dnf", "clean", "packages"]) + .with_exec( + [ + "dnf", + "install", + "-y", + "--allowerasing", + "--nodocs", + "--setopt=install_weak_deps=False", + "--setopt=diskspacecheck=False", + ] + + pkgs + ) + .with_exec(["rm", "-rf", "/var/cache/dnf"]) + .with_exec(["c_rehash"]) + .with_exec( + [ + "mkdir", + "-p", + "/root/rpmbuild/BUILD", + "/root/rpmbuild/RPMS", + "/root/rpmbuild/SOURCES", + "/root/rpmbuild/SPECS", + "/root/rpmbuild/SRPMS", + ] + ) + ) + + return ctr + + +_CENTOS_STREAM_COMMON_PACKAGES = [ + "autoconf", + "autoconf-archive", + "automake", + "bash", + "bison", + "cmake", + "cups-devel", + "curl", + "libcurl-devel", + "libyaml-devel", + "diffutils", + "elfutils-libelf-devel", + "findutils", + "flex", + "freeipmi-devel", + "gcc", + "gcc-c++", + "git", + "golang", + "json-c-devel", + "libatomic", + "libmnl-devel", + "libtool", + "libuuid-devel", + "libuv-devel", + # "libzstd-devel", + "lm_sensors", + "lz4-devel", + "make", + "ninja-build", + "nc", + "openssl-devel", + "openssl-perl", + "patch", + "pcre2-devel", + "pkgconfig", + "pkgconfig(libmongoc-1.0)", + "procps", + "protobuf-c-devel", + "protobuf-compiler", + "protobuf-devel", + "python3", + "python3-pyyaml", + "rpm-build", + "rpm-devel", + "rpmdevtools", + "snappy-devel", + "systemd-devel", + "wget", + "zlib-devel", +] + + +def build_centos_stream_8( + client: dagger.Client, platform: dagger.Platform +) -> dagger.Container: + ctr = client.container(platform=platform).from_("quay.io/centos/centos:stream8") + + pkgs = [pkg for pkg in _CENTOS_STREAM_COMMON_PACKAGES] + ["autogen"] + + ctr = ( + ctr.with_exec(["dnf", "distro-sync", "-y", "--nodocs"]) + .with_exec( + [ + "dnf", + "install", + "-y", + "--nodocs", + "dnf-command(config-manager)", + "epel-release", + ] + ) + .with_exec(["dnf", "config-manager", "--set-enabled", "powertools"]) + .with_exec(["dnf", "clean", "packages"]) + .with_exec( + [ + "dnf", + "install", + "-y", + "--nodocs", + "--setopt=install_weak_deps=False", + "--setopt=diskspacecheck=False", + ] + + pkgs + ) + .with_exec(["rm", "-rf", "/var/cache/dnf"]) + .with_exec(["c_rehash"]) + .with_exec( + [ + "mkdir", + "-p", + "/root/rpmbuild/BUILD", + "/root/rpmbuild/RPMS", + "/root/rpmbuild/SOURCES", + "/root/rpmbuild/SPECS", + "/root/rpmbuild/SRPMS", + ] + ) + ) + + return ctr + + +def build_centos_stream_9( + client: dagger.Client, platform: dagger.Platform +) -> dagger.Container: + ctr = client.container(platform=platform).from_("quay.io/centos/centos:stream9") + + pkgs = [pkg for pkg in _CENTOS_STREAM_COMMON_PACKAGES] + + ctr = ( + ctr.with_exec(["dnf", "distro-sync", "-y", "--nodocs"]) + .with_exec( + [ + "dnf", + "install", + "-y", + "--nodocs", + "dnf-command(config-manager)", + "epel-release", + ] + ) + .with_exec(["dnf", "config-manager", "--set-enabled", "crb"]) + .with_exec(["dnf", "clean", "packages"]) + .with_exec( + [ + "dnf", + "install", + "-y", + "--allowerasing", + "--nodocs", + "--setopt=install_weak_deps=False", + "--setopt=diskspacecheck=False", + ] + + pkgs + ) + .with_exec(["rm", "-rf", "/var/cache/dnf"]) + .with_exec(["c_rehash"]) + .with_exec( + [ + "mkdir", + "-p", + "/root/rpmbuild/BUILD", + "/root/rpmbuild/RPMS", + "/root/rpmbuild/SOURCES", + "/root/rpmbuild/SPECS", + "/root/rpmbuild/SRPMS", + ] + ) + ) + + return ctr + + +_ORACLE_LINUX_COMMON_PACKAGES = list(_ROCKY_LINUX_COMMON_PACKAGES) + + +def build_oracle_linux_9( + client: dagger.Client, platform: dagger.Platform +) -> dagger.Container: + ctr = client.container(platform=platform).from_("oraclelinux:9") + + pkgs = [pkg for pkg in _ORACLE_LINUX_COMMON_PACKAGES] + + repo_path = str(Path(__file__).parent.parent.parent) + this_path = os.path.join(repo_path, "packaging/dag") + + ctr = ( + ctr.with_file( + "/etc/yum.repos.d/ol9-epel.repo", + client.host().file(f"{this_path}/ol9-epel.repo"), + ) + .with_exec(["dnf", "config-manager", "--set-enabled", "ol9_codeready_builder"]) + .with_exec(["dnf", "config-manager", "--set-enabled", "ol9_developer_EPEL"]) + .with_exec(["dnf", "distro-sync", "-y", "--nodocs"]) + .with_exec(["dnf", "clean", "-y", "packages"]) + .with_exec( + [ + "dnf", + "install", + "-y", + "--nodocs", + "--setopt=install_weak_deps=False", + "--setopt=diskspacecheck=False", + ] + + pkgs + ) + .with_exec(["rm", "-rf", "/var/cache/dnf"]) + .with_exec(["c_rehash"]) + .with_exec( + [ + "mkdir", + "-p", + "/root/rpmbuild/BUILD", + "/root/rpmbuild/RPMS", + "/root/rpmbuild/SOURCES", + "/root/rpmbuild/SPECS", + "/root/rpmbuild/SRPMS", + ] + ) + ) + + return ctr + + +def build_oracle_linux_8( + client: dagger.Client, platform: dagger.Platform +) -> dagger.Container: + ctr = client.container(platform=platform).from_("oraclelinux:8") + + pkgs = [pkg for pkg in _ORACLE_LINUX_COMMON_PACKAGES] + ["autogen"] + + repo_path = str(Path(__file__).parent.parent.parent) + this_path = os.path.join(repo_path, "packaging/dag") + + ctr = ( + ctr.with_file( + "/etc/yum.repos.d/ol8-epel.repo", + client.host().file(f"{this_path}/ol8-epel.repo"), + ) + .with_exec(["dnf", "config-manager", "--set-enabled", "ol8_codeready_builder"]) + .with_exec(["dnf", "distro-sync", "-y", "--nodocs"]) + .with_exec(["dnf", "clean", "-y", "packages"]) + .with_exec( + [ + "dnf", + "install", + "-y", + "--nodocs", + "--setopt=install_weak_deps=False", + "--setopt=diskspacecheck=False", + ] + + pkgs + ) + .with_exec(["rm", "-rf", "/var/cache/dnf"]) + .with_exec(["c_rehash"]) + .with_exec( + [ + "mkdir", + "-p", + "/root/rpmbuild/BUILD", + "/root/rpmbuild/RPMS", + "/root/rpmbuild/SOURCES", + "/root/rpmbuild/SPECS", + "/root/rpmbuild/SRPMS", + ] + ) + ) + + return ctr + + +_OPENSUSE_COMMON_PACKAGES = [ + "autoconf", + "autoconf-archive", + "autogen", + "automake", + "bison", + "cmake", + "cups", + "cups-devel", + "curl", + "diffutils", + "flex", + "freeipmi-devel", + "gcc", + "gcc-c++", + "git-core", + "go", + "json-glib-devel", + "judy-devel", + "libatomic1", + "libcurl-devel", + "libelf-devel", + "liblz4-devel", + "libjson-c-devel", + "libyaml-devel", + "libmnl0", + "libmnl-devel", + "libnetfilter_acct1", + "libnetfilter_acct-devel", + "libpcre2-8-0", + "libopenssl-devel", + "libtool", + "libuv-devel", + "libuuid-devel", + "libzstd-devel", + "make", + "ninja", + "patch", + "pkg-config", + "protobuf-devel", + "rpm-build", + "rpm-devel", + "rpmdevtools", + "snappy-devel", + "systemd-devel", + "tar", + "wget", + "xen-devel", +] + + +def build_opensuse_tumbleweed( + client: dagger.Client, platform: dagger.Platform +) -> dagger.Container: + ctr = client.container(platform=platform).from_("opensuse/tumbleweed:latest") + + pkgs = [pkg for pkg in _OPENSUSE_COMMON_PACKAGES] + ["protobuf-c"] + + ctr = ( + ctr.with_exec(["zypper", "update", "-y"]) + .with_exec( + [ + "zypper", + "install", + "-y", + "--allow-downgrade", + ] + + pkgs + ) + .with_exec(["zypper", "clean"]) + .with_exec(["rm", "-rf", "/var/cache/zypp/*/*"]) + .with_exec(["c_rehash"]) + .with_exec( + [ + "mkdir", + "-p", + "/usr/src/packages/BUILD", + "/usr/src/packages/RPMS", + "/usr/src/packages/SOURCES", + "/usr/src/packages/SPECS", + "/usr/src/packages/SRPMS", + ] + ) + ) + + return ctr + + +def build_opensuse_15_5( + client: dagger.Client, platform: dagger.Platform +) -> dagger.Container: + ctr = client.container(platform=platform).from_("opensuse/leap:15.5") + + pkgs = [pkg for pkg in _OPENSUSE_COMMON_PACKAGES] + ["libprotobuf-c-devel"] + + ctr = ( + ctr.with_exec(["zypper", "update", "-y"]) + .with_exec( + [ + "zypper", + "install", + "-y", + "--allow-downgrade", + ] + + pkgs + ) + .with_exec(["zypper", "clean"]) + .with_exec(["rm", "-rf", "/var/cache/zypp/*/*"]) + .with_exec(["c_rehash"]) + .with_exec( + [ + "mkdir", + "-p", + "/usr/src/packages/BUILD", + "/usr/src/packages/RPMS", + "/usr/src/packages/SOURCES", + "/usr/src/packages/SPECS", + "/usr/src/packages/SRPMS", + ] + ) + ) + + return ctr + + +def build_opensuse_15_4( + client: dagger.Client, platform: dagger.Platform +) -> dagger.Container: + crt = client.container(platform=platform).from_("opensuse/leap:15.4") + + pkgs = [pkg for pkg in _OPENSUSE_COMMON_PACKAGES] + ["libprotobuf-c-devel"] + + crt = ( + crt.with_exec(["zypper", "update", "-y"]) + .with_exec( + [ + "zypper", + "install", + "-y", + "--allow-downgrade", + ] + + pkgs + ) + .with_exec(["zypper", "clean"]) + .with_exec(["rm", "-rf", "/var/cache/zypp/*/*"]) + .with_exec(["c_rehash"]) + .with_exec( + [ + "mkdir", + "-p", + "/usr/src/packages/BUILD", + "/usr/src/packages/RPMS", + "/usr/src/packages/SOURCES", + "/usr/src/packages/SPECS", + "/usr/src/packages/SRPMS", + ] + ) + ) + + return crt + + +_FEDORA_COMMON_PACKAGES = [ + "autoconf", + "autoconf-archive", + "autogen", + "automake", + "bash", + "bison", + "cmake", + "cups-devel", + "curl", + "diffutils", + "elfutils-libelf-devel", + "findutils", + "flex", + "freeipmi-devel", + "gcc", + "gcc-c++", + "git-core", + "golang", + "json-c-devel", + "libcurl-devel", + "libyaml-devel", + "Judy-devel", + "libatomic", + "libmnl-devel", + "libnetfilter_acct-devel", + "libtool", + "libuuid-devel", + "libuv-devel", + "libzstd-devel", + "lz4-devel", + "make", + "ninja-build", + "openssl-devel", + "openssl-perl", + "patch", + "pcre2-devel", + "pkgconfig", +] + + +def build_fedora_37( + client: dagger.Client, platform: dagger.Platform +) -> dagger.Container: + ctr = client.container(platform=platform).from_("fedora:37") + + pkgs = [pkg for pkg in _FEDORA_COMMON_PACKAGES] + + ctr = ( + ctr.with_exec(["dnf", "distro-sync", "-y", "--nodocs"]) + .with_exec(["dnf", "clean", "-y", "packages"]) + .with_exec( + [ + "dnf", + "install", + "-y", + "--nodocs", + "--setopt=install_weak_deps=False", + "--setopt=diskspacecheck=False", + ] + + pkgs + ) + .with_exec(["rm", "-rf", "/var/cache/dnf"]) + .with_exec(["c_rehash"]) + .with_exec( + [ + "mkdir", + "-p", + "/root/rpmbuild/BUILD", + "/root/rpmbuild/RPMS", + "/root/rpmbuild/SOURCES", + "/root/rpmbuild/SPECS", + "/root/rpmbuild/SRPMS", + ] + ) + ) + + return ctr + + +def build_fedora_38( + client: dagger.Client, platform: dagger.Platform +) -> dagger.Container: + ctr = client.container(platform=platform).from_("fedora:38") + + pkgs = [pkg for pkg in _FEDORA_COMMON_PACKAGES] + + ctr = ( + ctr.with_exec(["dnf", "distro-sync", "-y", "--nodocs"]) + .with_exec(["dnf", "clean", "-y", "packages"]) + .with_exec( + [ + "dnf", + "install", + "-y", + "--nodocs", + "--setopt=install_weak_deps=False", + "--setopt=diskspacecheck=False", + ] + + pkgs + ) + .with_exec(["rm", "-rf", "/var/cache/dnf"]) + .with_exec(["c_rehash"]) + .with_exec( + [ + "mkdir", + "-p", + "/root/rpmbuild/BUILD", + "/root/rpmbuild/RPMS", + "/root/rpmbuild/SOURCES", + "/root/rpmbuild/SPECS", + "/root/rpmbuild/SRPMS", + ] + ) + ) + + return ctr + + +def build_fedora_39( + client: dagger.Client, platform: dagger.Platform +) -> dagger.Container: + ctr = client.container(platform=platform).from_("fedora:39") + + pkgs = [pkg for pkg in _FEDORA_COMMON_PACKAGES] + + ctr = ( + ctr.with_exec(["dnf", "distro-sync", "-y", "--nodocs"]) + .with_exec(["dnf", "clean", "-y", "packages"]) + .with_exec( + [ + "dnf", + "install", + "-y", + "--nodocs", + "--setopt=install_weak_deps=False", + "--setopt=diskspacecheck=False", + ] + + pkgs + ) + .with_exec(["rm", "-rf", "/var/cache/dnf"]) + .with_exec(["c_rehash"]) + .with_exec( + [ + "mkdir", + "-p", + "/root/rpmbuild/BUILD", + "/root/rpmbuild/RPMS", + "/root/rpmbuild/SOURCES", + "/root/rpmbuild/SPECS", + "/root/rpmbuild/SRPMS", + ] + ) + ) + + return ctr + + +_DEBIAN_COMMON_PACKAGES = [ + "autoconf", + "autoconf-archive", + "autogen", + "automake", + "bison", + "build-essential", + "ca-certificates", + "cmake", + "curl", + "dh-autoreconf", + "dh-make", + "dpkg-dev", + "flex", + "g++", + "gcc", + "git-buildpackage", + "git-core", + "golang", + "libatomic1", + "libcurl4-openssl-dev", + "libcups2-dev", + "libdistro-info-perl", + "libelf-dev", + "libipmimonitoring-dev", + "libjson-c-dev", + "libyaml-dev", + "libjudy-dev", + "liblz4-dev", + "libmnl-dev", + "libmongoc-dev", + "libnetfilter-acct-dev", + "libpcre2-dev", + "libprotobuf-dev", + "libprotoc-dev", + "libsnappy-dev", + "libsystemd-dev", + "libssl-dev", + "libtool", + "libuv1-dev", + "libzstd-dev", + "make", + "ninja-build", + "pkg-config", + "protobuf-compiler", + "systemd", + "uuid-dev", + "wget", + "zlib1g-dev", +] + + +def build_debian_10( + client: dagger.Client, platform: dagger.Platform +) -> dagger.Container: + ctr = client.container(platform=platform).from_("debian:buster") + + pkgs = [pkg for pkg in _DEBIAN_COMMON_PACKAGES] + ["dh-systemd", "libxen-dev"] + + ctr = ( + ctr.with_env_variable("DEBIAN_FRONTEND", "noninteractive") + .with_exec(["apt-get", "update"]) + .with_exec(["apt-get", "upgrade", "-y"]) + .with_exec(["apt-get", "install", "-y", "--no-install-recommends"] + pkgs) + .with_exec(["apt-get", "clean"]) + .with_exec(["c_rehash"]) + .with_exec(["rm", "-rf", "/var/lib/apt/lists/*"]) + ) + + return ctr + + +def build_debian_11( + client: dagger.Client, platform: dagger.Platform +) -> dagger.Container: + ctr = client.container(platform=platform).from_("debian:bullseye") + + pkgs = [pkg for pkg in _DEBIAN_COMMON_PACKAGES] + ["libxen-dev"] + + ctr = ( + ctr.with_env_variable("DEBIAN_FRONTEND", "noninteractive") + .with_exec(["apt-get", "update"]) + .with_exec(["apt-get", "upgrade", "-y"]) + .with_exec(["apt-get", "install", "-y", "--no-install-recommends"] + pkgs) + .with_exec(["apt-get", "clean"]) + .with_exec(["c_rehash"]) + .with_exec(["rm", "-rf", "/var/lib/apt/lists/*"]) + ) + + return ctr + + +def build_debian_12( + client: dagger.Client, platform: dagger.Platform +) -> dagger.Container: + ctr = client.container(platform=platform).from_("debian:bookworm") + + pkgs = [pkg for pkg in _DEBIAN_COMMON_PACKAGES] + + if platform != dagger.Platform("linux/i386"): + pkgs.append("libxen-dev") + + ctr = ( + ctr.with_env_variable("DEBIAN_FRONTEND", "noninteractive") + .with_exec(["apt-get", "update"]) + .with_exec(["apt-get", "upgrade", "-y"]) + .with_exec(["apt-get", "install", "-y", "--no-install-recommends"] + pkgs) + .with_exec(["apt-get", "clean"]) + .with_exec(["c_rehash"]) + .with_exec(["rm", "-rf", "/var/lib/apt/lists/*"]) + ) + + return ctr + + +_UBUNTU_COMMON_PACKAGES = [ + "autoconf", + "autoconf-archive", + "autogen", + "automake", + "bison", + "build-essential", + "ca-certificates", + "cmake", + "curl", + "dh-autoreconf", + "dh-make", + "dpkg-dev", + "flex", + "g++", + "gcc", + "git-buildpackage", + "git-core", + "golang", + "libatomic1", + "libcurl4-openssl-dev", + "libcups2-dev", + "libdistro-info-perl", + "libelf-dev", + "libipmimonitoring-dev", + "libjson-c-dev", + "libyaml-dev", + "libjudy-dev", + "liblz4-dev", + "libmnl-dev", + "libmongoc-dev", + "libnetfilter-acct-dev", + "libpcre2-dev", + "libprotobuf-dev", + "libprotoc-dev", + "libsnappy-dev", + "libsystemd-dev", + "libssl-dev", + "libtool", + "libuv1-dev", + "libxen-dev", + "libzstd-dev", + "make", + "ninja-build", + "pkg-config", + "protobuf-compiler", + "systemd", + "uuid-dev", + "wget", + "zlib1g-dev", +] + + +def build_ubuntu_20_04( + client: dagger.Client, platform: dagger.Platform +) -> dagger.Container: + ctr = client.container(platform=platform).from_("ubuntu:20.04") + + pkgs = [pkg for pkg in _UBUNTU_COMMON_PACKAGES] + ["dh-systemd"] + + ctr = ( + ctr.with_env_variable("DEBIAN_FRONTEND", "noninteractive") + .with_exec(["apt-get", "update"]) + .with_exec(["apt-get", "upgrade", "-y"]) + .with_exec(["apt-get", "install", "-y", "--no-install-recommends"] + pkgs) + .with_exec(["apt-get", "clean"]) + .with_exec(["c_rehash"]) + .with_exec(["rm", "-rf", "/var/lib/apt/lists/*"]) + ) + + # + # FIXME: add kitware for cmake on arm-hf + # + + return ctr + + +def build_ubuntu_22_04( + client: dagger.Client, platform: dagger.Platform +) -> dagger.Container: + ctr = client.container(platform=platform).from_("ubuntu:22.04") + + pkgs = [pkg for pkg in _UBUNTU_COMMON_PACKAGES] + + ctr = ( + ctr.with_env_variable("DEBIAN_FRONTEND", "noninteractive") + .with_exec(["apt-get", "update"]) + .with_exec(["apt-get", "upgrade", "-y"]) + .with_exec(["apt-get", "install", "-y", "--no-install-recommends"] + pkgs) + .with_exec(["apt-get", "clean"]) + .with_exec(["c_rehash"]) + .with_exec(["rm", "-rf", "/var/lib/apt/lists/*"]) + ) + + return ctr + + +def build_ubuntu_23_04( + client: dagger.Client, platform: dagger.Platform +) -> dagger.Container: + ctr = client.container(platform=platform).from_("ubuntu:23.04") + + pkgs = [pkg for pkg in _UBUNTU_COMMON_PACKAGES] + + ctr = ( + ctr.with_env_variable("DEBIAN_FRONTEND", "noninteractive") + .with_exec(["apt-get", "update"]) + .with_exec(["apt-get", "upgrade", "-y"]) + .with_exec(["apt-get", "install", "-y", "--no-install-recommends"] + pkgs) + .with_exec(["apt-get", "clean"]) + .with_exec(["c_rehash"]) + .with_exec(["rm", "-rf", "/var/lib/apt/lists/*"]) + ) + + return ctr + + +def build_ubuntu_23_10( + client: dagger.Client, platform: dagger.Platform +) -> dagger.Container: + ctr = client.container(platform=platform).from_("ubuntu:23.10") + + pkgs = [pkg for pkg in _UBUNTU_COMMON_PACKAGES] + + ctr = ( + ctr.with_env_variable("DEBIAN_FRONTEND", "noninteractive") + .with_exec(["apt-get", "update"]) + .with_exec(["apt-get", "upgrade", "-y"]) + .with_exec(["apt-get", "install", "-y", "--no-install-recommends"] + pkgs) + .with_exec(["apt-get", "clean"]) + .with_exec(["c_rehash"]) + .with_exec(["rm", "-rf", "/var/lib/apt/lists/*"]) + ) + + return ctr + + +def install_cargo(ctr: dagger.Container) -> dagger.Container: + bin_paths = [ + "/root/.cargo/bin", + "/usr/local/sbin", + "/usr/local/bin", + "/usr/sbin", + "/usr/bin", + "/sbin", + "/bin", + ] + + ctr = ( + ctr.with_workdir("/") + .with_exec(["sh", "-c", "curl https://sh.rustup.rs -sSf | sh -s -- -y"]) + .with_env_variable("PATH", ":".join(bin_paths)) + .with_exec(["cargo", "new", "--bin", "hello"]) + .with_workdir("/hello") + .with_exec(["cargo", "run", "-v", "-v"]) + ) + + return ctr diff --git a/packaging/dag/main.py b/packaging/dag/main.py new file mode 100755 index 000000000..c7e9670cf --- /dev/null +++ b/packaging/dag/main.py @@ -0,0 +1,18 @@ +#!/usr/bin/env python3 + +import click + +from test_command import test +from build_command import build + + +@click.group() +def cli(): + pass + + +cli.add_command(test) +cli.add_command(build) + +if __name__ == "__main__": + cli() diff --git a/packaging/dag/nd.py b/packaging/dag/nd.py new file mode 100644 index 000000000..d59adf30a --- /dev/null +++ b/packaging/dag/nd.py @@ -0,0 +1,409 @@ +from typing import List + +import enum +import os +import pathlib +import uuid + +import dagger +import jinja2 + +import imageutils + + +class Platform: + def __init__(self, platform: str): + self.platform = dagger.Platform(platform) + + def escaped(self) -> str: + return str(self.platform).removeprefix("linux/").replace("/", "_") + + def __eq__(self, other): + if isinstance(other, Platform): + return self.platform == other.platform + elif isinstance(other, dagger.Platform): + return self.platform == other + else: + return NotImplemented + + def __ne__(self, other): + return not (self == other) + + def __hash__(self): + return hash(self.platform) + + def __str__(self) -> str: + return str(self.platform) + + +SUPPORTED_PLATFORMS = set( + [ + Platform("linux/x86_64"), + Platform("linux/arm64"), + Platform("linux/i386"), + Platform("linux/arm/v7"), + Platform("linux/arm/v6"), + Platform("linux/ppc64le"), + Platform("linux/s390x"), + Platform("linux/riscv64"), + ] +) + + +SUPPORTED_DISTRIBUTIONS = set( + [ + "alpine_3_18", + "alpine_3_19", + "amazonlinux2", + "centos7", + "centos-stream8", + "centos-stream9", + "debian10", + "debian11", + "debian12", + "fedora37", + "fedora38", + "fedora39", + "opensuse15.4", + "opensuse15.5", + "opensusetumbleweed", + "oraclelinux8", + "oraclelinux9", + "rockylinux8", + "rockylinux9", + "ubuntu20.04", + "ubuntu22.04", + "ubuntu23.04", + "ubuntu23.10", + ] +) + + +class Distribution: + def __init__(self, display_name): + self.display_name = display_name + + if self.display_name == "alpine_3_18": + self.docker_tag = "alpine:3.18" + self.builder = imageutils.build_alpine_3_18 + self.platforms = SUPPORTED_PLATFORMS + elif self.display_name == "alpine_3_19": + self.docker_tag = "alpine:3.19" + self.builder = imageutils.build_alpine_3_19 + self.platforms = SUPPORTED_PLATFORMS + elif self.display_name == "amazonlinux2": + self.docker_tag = "amazonlinux:2" + self.builder = imageutils.build_amazon_linux_2 + self.platforms = SUPPORTED_PLATFORMS + elif self.display_name == "centos7": + self.docker_tag = "centos:7" + self.builder = imageutils.build_centos_7 + self.platforms = SUPPORTED_PLATFORMS + elif self.display_name == "centos-stream8": + self.docker_tag = "quay.io/centos/centos:stream8" + self.builder = imageutils.build_centos_stream_8 + self.platforms = SUPPORTED_PLATFORMS + elif self.display_name == "centos-stream9": + self.docker_tag = "quay.io/centos/centos:stream9" + self.builder = imageutils.build_centos_stream_9 + self.platforms = SUPPORTED_PLATFORMS + elif self.display_name == "debian10": + self.docker_tag = "debian:10" + self.builder = imageutils.build_debian_10 + self.platforms = SUPPORTED_PLATFORMS + elif self.display_name == "debian11": + self.docker_tag = "debian:11" + self.builder = imageutils.build_debian_11 + self.platforms = SUPPORTED_PLATFORMS + elif self.display_name == "debian12": + self.docker_tag = "debian:12" + self.builder = imageutils.build_debian_12 + self.platforms = SUPPORTED_PLATFORMS + elif self.display_name == "fedora37": + self.docker_tag = "fedora:37" + self.builder = imageutils.build_fedora_37 + self.platforms = SUPPORTED_PLATFORMS + elif self.display_name == "fedora38": + self.docker_tag = "fedora:38" + self.builder = imageutils.build_fedora_38 + self.platforms = SUPPORTED_PLATFORMS + elif self.display_name == "fedora39": + self.docker_tag = "fedora:39" + self.platforms = SUPPORTED_PLATFORMS + self.builder = imageutils.build_fedora_39 + elif self.display_name == "opensuse15.4": + self.docker_tag = "opensuse/leap:15.4" + self.builder = imageutils.build_opensuse_15_4 + self.platforms = SUPPORTED_PLATFORMS + elif self.display_name == "opensuse15.5": + self.docker_tag = "opensuse/leap:15.5" + self.builder = imageutils.build_opensuse_15_5 + self.platforms = SUPPORTED_PLATFORMS + elif self.display_name == "opensusetumbleweed": + self.docker_tag = "opensuse/tumbleweed:latest" + self.builder = imageutils.build_opensuse_tumbleweed + self.platforms = SUPPORTED_PLATFORMS + elif self.display_name == "oraclelinux8": + self.docker_tag = "oraclelinux:8" + self.builder = imageutils.build_oracle_linux_8 + self.platforms = SUPPORTED_PLATFORMS + elif self.display_name == "oraclelinux9": + self.docker_tag = "oraclelinux:9" + self.builder = imageutils.build_oracle_linux_9 + self.platforms = SUPPORTED_PLATFORMS + elif self.display_name == "rockylinux8": + self.docker_tag = "rockylinux:8" + self.builder = imageutils.build_rocky_linux_8 + self.platforms = SUPPORTED_PLATFORMS + elif self.display_name == "rockylinux9": + self.docker_tag = "rockylinux:9" + self.builder = imageutils.build_rocky_linux_9 + self.platforms = SUPPORTED_PLATFORMS + elif self.display_name == "ubuntu20.04": + self.docker_tag = "ubuntu:20.04" + self.builder = imageutils.build_ubuntu_20_04 + self.platforms = SUPPORTED_PLATFORMS + elif self.display_name == "ubuntu22.04": + self.docker_tag = "ubuntu:22.04" + self.builder = imageutils.build_ubuntu_22_04 + self.platforms = SUPPORTED_PLATFORMS + elif self.display_name == "ubuntu23.04": + self.docker_tag = "ubuntu:23.04" + self.builder = imageutils.build_ubuntu_23_04 + self.platforms = SUPPORTED_PLATFORMS + elif self.display_name == "ubuntu23.10": + self.docker_tag = "ubuntu:23.10" + self.builder = imageutils.build_ubuntu_23_10 + self.platforms = SUPPORTED_PLATFORMS + else: + raise ValueError(f"Unknown distribution: {self.display_name}") + + def _cache_volume( + self, client: dagger.Client, platform: dagger.Platform, path: str + ) -> dagger.CacheVolume: + tag = "_".join([self.display_name, Platform(platform).escaped()]) + return client.cache_volume(f"{path}-{tag}") + + def build( + self, client: dagger.Client, platform: dagger.Platform + ) -> dagger.Container: + if platform not in self.platforms: + raise ValueError( + f"Building {self.display_name} is not supported on {platform}." + ) + + ctr = self.builder(client, platform) + ctr = imageutils.install_cargo(ctr) + + return ctr + + +class FeatureFlags(enum.Flag): + DBEngine = enum.auto() + GoPlugin = enum.auto() + ExtendedBPF = enum.auto() + LogsManagement = enum.auto() + MachineLearning = enum.auto() + BundledProtobuf = enum.auto() + + +class NetdataInstaller: + def __init__( + self, + platform: Platform, + distro: Distribution, + repo_root: pathlib.Path, + prefix: pathlib.Path, + features: FeatureFlags, + ): + self.platform = platform + self.distro = distro + self.repo_root = repo_root + self.prefix = prefix + self.features = features + + def _mount_repo( + self, client: dagger.Client, ctr: dagger.Container, repo_root: pathlib.Path + ) -> dagger.Container: + host_repo_root = pathlib.Path(__file__).parent.parent.parent.as_posix() + exclude_dirs = ["build", "fluent-bit/build", "packaging/dag"] + + # The installer builds/stores intermediate artifacts under externaldeps/ + # We add a volume to speed up rebuilds. The volume has to be unique + # per platform/distro in order to avoid mixing unrelated artifacts + # together. + externaldeps = self.distro._cache_volume(client, self.platform, "externaldeps") + + ctr = ( + ctr.with_directory( + self.repo_root.as_posix(), client.host().directory(host_repo_root) + ) + .with_workdir(self.repo_root.as_posix()) + .with_mounted_cache( + os.path.join(self.repo_root, "externaldeps"), externaldeps + ) + ) + + return ctr + + def install(self, client: dagger.Client, ctr: dagger.Container) -> dagger.Container: + args = ["--dont-wait", "--dont-start-it", "--disable-telemetry"] + + if FeatureFlags.DBEngine not in self.features: + args.append("--disable-dbengine") + + if FeatureFlags.GoPlugin not in self.features: + args.append("--disable-go") + + if FeatureFlags.ExtendedBPF not in self.features: + args.append("--disable-ebpf") + + if FeatureFlags.LogsManagement not in self.features: + args.append("--disable-logsmanagement") + + if FeatureFlags.MachineLearning not in self.features: + args.append("--disable-ml") + + if FeatureFlags.BundledProtobuf not in self.features: + args.append("--use-system-protobuf") + + args.extend(["--install-prefix", self.prefix.parent.as_posix()]) + + ctr = self._mount_repo(client, ctr, self.repo_root.as_posix()) + + ctr = ctr.with_env_variable( + "NETDATA_CMAKE_OPTIONS", "-DCMAKE_BUILD_TYPE=Debug" + ).with_exec(["./netdata-installer.sh"] + args) + + return ctr + + +class Endpoint: + def __init__(self, hostname: str, port: int): + self.hostname = hostname + self.port = port + + def __str__(self): + return ":".join([self.hostname, str(self.port)]) + + +class ChildStreamConf: + def __init__( + self, + installer: NetdataInstaller, + destinations: List[Endpoint], + api_key: uuid.UUID, + ): + self.installer = installer + self.substitutions = { + "enabled": "yes", + "destination": " ".join([str(dst) for dst in destinations]), + "api_key": api_key, + "timeout_seconds": 60, + "default_port": 19999, + "send_charts_matching": "*", + "buffer_size_bytes": 1024 * 1024, + "reconnect_delay_seconds": 5, + "initial_clock_resync_iterations": 60, + } + + def render(self) -> str: + tmpl_path = pathlib.Path(__file__).parent / "files/child_stream.conf" + with open(tmpl_path) as fp: + tmpl = jinja2.Template(fp.read()) + + return tmpl.render(**self.substitutions) + + +class ParentStreamConf: + def __init__(self, installer: NetdataInstaller, api_key: uuid.UUID): + self.installer = installer + self.substitutions = { + "api_key": str(api_key), + "enabled": "yes", + "allow_from": "*", + "default_history": 3600, + "health_enabled_by_default": "auto", + "default_postpone_alarms_on_connect_seconds": 60, + "multiple_connections": "allow", + } + + def render(self) -> str: + tmpl_path = pathlib.Path(__file__).parent / "files/parent_stream.conf" + with open(tmpl_path) as fp: + tmpl = jinja2.Template(fp.read()) + + return tmpl.render(**self.substitutions) + + +class StreamConf: + def __init__(self, child_conf: ChildStreamConf, parent_conf: ParentStreamConf): + self.child_conf = child_conf + self.parent_conf = parent_conf + + def render(self) -> str: + child_section = self.child_conf.render() if self.child_conf else "" + parent_section = self.parent_conf.render() if self.parent_conf else "" + return "\n".join([child_section, parent_section]) + + +class AgentContext: + def __init__( + self, + client: dagger.Client, + platform: dagger.Platform, + distro: Distribution, + installer: NetdataInstaller, + endpoint: Endpoint, + api_key: uuid.UUID, + allow_children: bool, + ): + self.client = client + self.platform = platform + self.distro = distro + self.installer = installer + self.endpoint = endpoint + self.api_key = api_key + self.allow_children = allow_children + + self.parent_contexts = [] + + self.built_distro = False + self.built_agent = False + + def add_parent(self, parent_context: "AgentContext"): + self.parent_contexts.append(parent_context) + + def build_container(self) -> dagger.Container: + ctr = self.distro.build(self.client, self.platform) + ctr = self.installer.install(self.client, ctr) + + if len(self.parent_contexts) == 0 and not self.allow_children: + return ctr.with_exposed_port(self.endpoint.port) + + destinations = [parent_ctx.endpoint for parent_ctx in self.parent_contexts] + child_stream_conf = ChildStreamConf(self.installer, destinations, self.api_key) + + parent_stream_conf = None + if self.allow_children: + parent_stream_conf = ParentStreamConf(self.installer, self.api_key) + + stream_conf = StreamConf(child_stream_conf, parent_stream_conf) + + # write the stream conf to localhost and cp it in the container + host_stream_conf_path = pathlib.Path( + f"/tmp/{self.endpoint.hostname}_stream.conf" + ) + with open(host_stream_conf_path, "w") as fp: + fp.write(stream_conf.render()) + + ctr_stream_conf_path = self.installer.prefix / "etc/netdata/stream.conf" + + ctr = ctr.with_file( + ctr_stream_conf_path.as_posix(), + self.client.host().file(host_stream_conf_path.as_posix()), + ) + + ctr = ctr.with_exposed_port(self.endpoint.port) + + return ctr diff --git a/packaging/dag/requirements.txt b/packaging/dag/requirements.txt new file mode 100644 index 000000000..8cc243646 --- /dev/null +++ b/packaging/dag/requirements.txt @@ -0,0 +1,3 @@ +click==8.1.7 +dagger-io==0.9.7 +Jinja2==3.1.3 diff --git a/packaging/dag/test_command.py b/packaging/dag/test_command.py new file mode 100644 index 000000000..2418d142e --- /dev/null +++ b/packaging/dag/test_command.py @@ -0,0 +1,128 @@ +import click +import asyncio +import sys +import pathlib +import dagger +import uuid +import httpx + +from nd import Distribution, NetdataInstaller, FeatureFlags, Endpoint, AgentContext + + +def run_async(func): + def wrapper(*args, **kwargs): + return asyncio.run(func(*args, **kwargs)) + + return wrapper + + +@run_async +async def simple_test(): + config = dagger.Config(log_output=sys.stdout) + + async with dagger.Connection(config) as client: + platform = dagger.Platform("linux/x86_64") + distro = Distribution("debian10") + + repo_root = pathlib.Path("/netdata") + prefix_path = pathlib.Path("/opt/netdata") + installer = NetdataInstaller( + platform, distro, repo_root, prefix_path, FeatureFlags.DBEngine + ) + + api_key = uuid.uuid4() + + # + # parent + # + parent_endpoint = Endpoint("parent1", 22000) + parent_ctx = AgentContext( + client, platform, distro, installer, parent_endpoint, api_key, True + ) + parent_cmd = installer.prefix / "usr/sbin/netdata" + parent_args = [ + parent_cmd.as_posix(), + "-D", + "-i", + "0.0.0.0", + "-p", + str(parent_endpoint.port), + ] + + parent_ctr = parent_ctx.build_container() + parent_ctr = parent_ctr.with_exec(parent_args) + parent_svc = parent_ctr.as_service() + + # + # child + # + child_endpoint = Endpoint("child1", 21000) + child_ctx = AgentContext( + client, platform, distro, installer, child_endpoint, api_key, False + ) + child_ctx.add_parent(parent_ctx) + child_cmd = installer.prefix / "usr/sbin/netdata" + child_args = [ + child_cmd.as_posix(), + "-D", + "-i", + "0.0.0.0", + "-p", + str(child_endpoint.port), + ] + + child_ctr = child_ctx.build_container() + child_ctr = child_ctr.with_service_binding(parent_endpoint.hostname, parent_svc) + child_ctr = child_ctr.with_exec(child_args) + child_svc = child_ctr.as_service() + + # + # endpoints + # + parent_tunnel, child_tunnel = await asyncio.gather( + client.host().tunnel(parent_svc, native=True).start(), + client.host().tunnel(child_svc, native=True).start(), + ) + + parent_endpoint, child_endpoint = await asyncio.gather( + parent_tunnel.endpoint(), + child_tunnel.endpoint(), + ) + + await asyncio.sleep(10) + + # + # run tests + # + + async with httpx.AsyncClient() as http: + resp = await http.get(f"http://{parent_endpoint}/api/v1/info") + + # + # Check that the child was connected + # + jd = resp.json() + assert ( + "hosts-available" in jd + ), "Could not find 'host-available' key in api/v1/info" + assert jd["hosts-available"] == 2, "Child did not connect to parent" + + # + # Check bearer protection + # + forbidden_urls = [ + f"http://{parent_endpoint}/api/v2/bearer_protection", + f"http://{parent_endpoint}/api/v2/bearer_get_token", + ] + + for url in forbidden_urls: + async with httpx.AsyncClient() as http: + resp = await http.get(url) + assert ( + resp.status_code == httpx.codes.UNAVAILABLE_FOR_LEGAL_REASONS + ), "Bearer protection is broken" + + +@click.command(help="Run a simple parent/child test") +def test(): + simple_test() diff --git a/packaging/docker/Dockerfile b/packaging/docker/Dockerfile index 8e7c9a7b1..b12af313d 100644 --- a/packaging/docker/Dockerfile +++ b/packaging/docker/Dockerfile @@ -29,7 +29,7 @@ RUN chmod +x netdata-installer.sh && \ cp -rp /deps/* /usr/local/ && \ /bin/echo -e "INSTALL_TYPE='oci'\nPREBUILT_ARCH='$(uname -m)'" > ./system/.install-type && \ CFLAGS="$(packaging/docker/gen-cflags.sh)" LDFLAGS="-Wl,--gc-sections" ./netdata-installer.sh --dont-wait --dont-start-it --use-system-protobuf \ - ${EXTRA_INSTALL_OPTS} --disable-ebpf --one-time-build --enable-lto "$([ "$RELEASE_CHANNEL" = stable ] && echo --stable-channel)" + ${EXTRA_INSTALL_OPTS} --disable-ebpf --install-no-prefix / "$([ "$RELEASE_CHANNEL" = stable ] && echo --stable-channel)" # files to one directory RUN mkdir -p /app/usr/sbin/ \ @@ -122,7 +122,9 @@ RUN addgroup --gid ${NETDATA_GID} --system "${DOCKER_GRP}" && \ freeipmi.plugin \ go.d.plugin \ perf.plugin \ + ndsudo \ slabinfo.plugin \ + network-viewer.plugin \ systemd-journal.plugin; do \ [ -f "/usr/libexec/netdata/plugins.d/$name" ] && chmod 4755 "/usr/libexec/netdata/plugins.d/$name"; \ done && \ diff --git a/packaging/docker/README.md b/packaging/docker/README.md index 528ef5926..fbe5ba433 100644 --- a/packaging/docker/README.md +++ b/packaging/docker/README.md @@ -12,6 +12,16 @@ import TabItem from '@theme/TabItem'; # Install Netdata with Docker +## Limitations running the Agent in Docker + +We do not officially support running our Docker images with the Docker CLI `--user` option or the Docker Compose +`user:` parameter. Such usage will usually still work, but some features will not be available when run this +way. Note that the agent will drop privileges appropriately inside the container during startup, meaning that even +when run without these options almost nothing in the container will actually run with an effective UID of 0. + +Our POWER8+ Docker images do not support our FreeIPMI collector. This is a technical limitation in FreeIPMI itself, +and unfortunately not something we can realistically work around. + ## Create a new Netdata Agent container You can create a new Agent container using either `docker run` or `docker-compose`. After using any method, you can @@ -24,12 +34,13 @@ along with their descriptions. <details open> <summary>Privileges</summary> -| Component | Privileges | Description | -|:---------------:|:-----------------------------:|--------------------------------------------------------------------------------------------------------------------------| -| cgroups.plugin | host PID mode, SYS_ADMIN | Container network interfaces monitoring. Map virtual interfaces in the system namespace to interfaces inside containers. | -| proc.plugin | host network mode | Host system networking stack monitoring. | -| go.d.plugin | host network mode | Monitoring applications running on the host and inside containers. | -| local-listeners | host network mode, SYS_PTRACE | Discovering local services/applications. Map open (listening) ports to running services/applications. | +| Component | Privileges | Description | +|:---------------------:|:-----------------------------:|--------------------------------------------------------------------------------------------------------------------------| +| cgroups.plugin | host PID mode, SYS_ADMIN | Container network interfaces monitoring. Map virtual interfaces in the system namespace to interfaces inside containers. | +| proc.plugin | host network mode | Host system networking stack monitoring. | +| go.d.plugin | host network mode | Monitoring applications running on the host and inside containers. | +| local-listeners | host network mode, SYS_PTRACE | Discovering local services/applications. Map open (listening) ports to running services/applications. | +| network-viewer.plugin | host network mode, SYS_ADMIN | Discovering all current network sockets and building a network-map. | </details> @@ -161,6 +172,43 @@ Add `- /run/dbus:/run/dbus:ro` to the netdata service `volumes`. </TabItem> </Tabs> +### With NVIDIA GPUs monitoring + + +Monitoring NVIDIA GPUs requires: + +- Using official [NVIDIA driver](https://www.nvidia.com/Download/index.aspx). +- Installing [NVIDIA Container Toolkit](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/install-guide.html). +- Allowing the Netdata container to access GPU resources. + + +<Tabs> +<TabItem value="docker_run" label="docker run"> + +<h3> Using the <code>docker run</code> command </h3> + +Add `--gpus 'all,capabilities=utility'` to your `docker run`. + +</TabItem> +<TabItem value="docker compose" label="docker-compose"> + +<h3> Using the <code>docker-compose</code> command</h3> + +Add the following to the netdata service. + +```yaml + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: all + capabilities: [gpu] +``` + +</TabItem> +</Tabs> + ### With host-editable configuration Use a [bind mount](https://docs.docker.com/storage/bind-mounts/) for `/etc/netdata` rather than a volume. @@ -246,7 +294,7 @@ volumes: ### With SSL/TLS enabled HTTP Proxy For a permanent installation on a public server, you -should [secure the Netdata instance](https://github.com/netdata/netdata/blob/master/docs/netdata-security.md). This +should [secure the Netdata instance](https://github.com/netdata/netdata/blob/master/docs/category-overview-pages/secure-nodes.md). This section contains an example of how to install Netdata with an SSL reverse proxy and basic authentication. You can use the following `docker-compose.yml` and Caddyfile files to run Netdata with Docker. Replace the domains and @@ -426,6 +474,62 @@ volumes: You can run the socket proxy in its own Docker Compose file and leave it on a private network that you can add to other services that require access. +### Rootless mode + +Netdata can be run successfully in a non-root environment, such as [rootless Docker](https://docs.docker.com/engine/security/rootless/). + +However, it should be noted that Netdata's data collection capabilities are considerably restricted in rootless Docker +due to its inherent limitations. While Netdata can function in a rootless environment, it cannot access certain +resources that require elevated privileges. The following components do not work: + +- container network interfaces monitoring (cgroup-network helper) +- disk I/O and file descriptors of applications and processes (apps.plugin) +- debugfs.plugin +- freeipmi.plugin +- perf.plugin +- slabinfo.plugin +- systemd-journal.plugin + +This method creates a [volume](https://docs.docker.com/storage/volumes/) for Netdata's configuration files +_within the container_ at `/etc/netdata`. +See the [configure section](#configure-agent-containers) for details. If you want to access the configuration files from +your _host_ machine, see [host-editable configuration](#with-host-editable-configuration). + +<Tabs> +<TabItem value="docker_run" label="docker run"> + +<h3> Using the <code>docker run</code> command </h3> + +Run the following command in your terminal to start a new container. + +```bash +docker run -d --name=netdata \ + --hostname=$(hostname) \ + -p 19999:19999 \ + -v netdataconfig:/etc/netdata \ + -v netdatalib:/var/lib/netdata \ + -v netdatacache:/var/cache/netdata \ + -v /etc/passwd:/host/etc/passwd:ro \ + -v /etc/group:/host/etc/group:ro \ + -v /etc/localtime:/etc/localtime:ro \ + -v /proc:/host/proc:ro \ + -v /sys:/host/sys:ro \ + -v /etc/os-release:/host/etc/os-release:ro \ + -v /run/user/$UID/docker.sock:/var/run/docker.sock:ro \ + --restart unless-stopped \ + --security-opt apparmor=unconfined \ + netdata/netdata +``` + +</TabItem> + +</Tabs> + +> :bookmark_tabs: Note +> +> If you plan to Claim the node to Netdata Cloud, you can find the command with the right parameters by clicking the " +> Add Nodes" button in your Space's "Nodes" view. + ## Docker tags See our full list of Docker images at [Docker Hub](https://hub.docker.com/r/netdata/netdata). diff --git a/packaging/docker/run.sh b/packaging/docker/run.sh index 415573320..6ba16d1ce 100755 --- a/packaging/docker/run.sh +++ b/packaging/docker/run.sh @@ -14,38 +14,6 @@ if [ ! -w / ] && [ "${EUID}" -eq 0 ]; then echo >&2 "WARNING: For more information, see https://learn.netdata.cloud/docs/agent/claim#known-issues-on-older-hosts-with-seccomp-enabled" fi -if [ ! "${DISABLE_TELEMETRY:-0}" -eq 0 ] || - [ -n "$DISABLE_TELEMETRY" ] || - [ ! "${DO_NOT_TRACK:-0}" -eq 0 ] || - [ -n "$DO_NOT_TRACK" ]; then - touch /etc/netdata/.opt-out-from-anonymous-statistics -fi - -chmod o+rX / 2>/dev/null || echo "Unable to change permissions without errors." - -BALENA_PGID=$(stat -c %g /var/run/balena.sock 2>/dev/null || true) -DOCKER_PGID=$(stat -c %g /var/run/docker.sock 2>/dev/null || true) - -re='^[0-9]+$' -if [[ $BALENA_PGID =~ $re ]]; then - echo "Netdata detected balena-engine.sock" - DOCKER_HOST='/var/run/balena-engine.sock' - PGID="$BALENA_PGID" -elif [[ $DOCKER_PGID =~ $re ]]; then - echo "Netdata detected docker.sock" - DOCKER_HOST="/var/run/docker.sock" - PGID="$DOCKER_PGID" -fi -export PGID -export DOCKER_HOST - -if [ -n "${PGID}" ]; then - echo "Creating docker group ${PGID}" - addgroup --gid "${PGID}" "docker" || echo >&2 "Could not add group docker with ID ${PGID}, its already there probably" - echo "Assign netdata user to docker group ${PGID}" - usermod --append --groups "docker" "${DOCKER_USR}" || echo >&2 "Could not add netdata user to group docker with ID ${PGID}" -fi - # Needed to read Proxmox VMs and (LXC) containers configuration files (name resolution + CPU and memory limits) function add_netdata_to_proxmox_conf_files_group() { group_guid="$(stat -c %g /host/etc/pve 2>/dev/null || true)" @@ -68,10 +36,65 @@ function add_netdata_to_proxmox_conf_files_group() { fi } -if [ -d "/host/etc/pve" ]; then - add_netdata_to_proxmox_conf_files_group || true +if [ ! "${DISABLE_TELEMETRY:-0}" -eq 0 ] || + [ -n "$DISABLE_TELEMETRY" ] || + [ ! "${DO_NOT_TRACK:-0}" -eq 0 ] || + [ -n "$DO_NOT_TRACK" ]; then + touch /etc/netdata/.opt-out-from-anonymous-statistics fi +chmod o+rX / 2>/dev/null || echo "Unable to change permissions without errors." + +if [ "${EUID}" -eq 0 ]; then + if [ -n "${NETDATA_EXTRA_APK_PACKAGES}" ]; then + echo >&2 "WARNING: Netdata’s Docker images have switched from Alpine to Debian as a base platform. Supplementary package support is now handled through the NETDATA_EXTRA_DEB_PACKAGES variable instead of NETDATA_EXTRA_APK_PACKAGES." + echo >&2 "WARNING: The container will still run, but supplementary packages listed in NETDATA_EXTRA_APK_PACKAGES will not be installed." + echo >&2 "WARNING: To remove these messages, either undefine NETDATA_EXTRA_APK_PACKAGES, or define it to an empty string." + fi + + if [ -n "${NETDATA_EXTRA_DEB_PACKAGES}" ]; then + echo "Fetching APT repository metadata." + if ! apt-get update; then + echo "Failed to fetch APT repository metadata." + else + echo "Installing supplementary packages." + export DEBIAN_FRONTEND="noninteractive" + # shellcheck disable=SC2086 + if ! apt-get install -y --no-install-recommends ${NETDATA_EXTRA_DEB_PACKAGES}; then + echo "Failed to install supplementary packages." + fi + fi + fi + + BALENA_PGID=$(stat -c %g /var/run/balena.sock 2>/dev/null || true) + DOCKER_PGID=$(stat -c %g /var/run/docker.sock 2>/dev/null || true) + + re='^[0-9]+$' + if [[ $BALENA_PGID =~ $re ]]; then + echo "Netdata detected balena-engine.sock" + DOCKER_HOST='/var/run/balena-engine.sock' + PGID="$BALENA_PGID" + elif [[ $DOCKER_PGID =~ $re ]]; then + echo "Netdata detected docker.sock" + DOCKER_HOST="/var/run/docker.sock" + PGID="$DOCKER_PGID" + fi + export PGID + export DOCKER_HOST + + if [ -n "${PGID}" ]; then + echo "Creating docker group ${PGID}" + addgroup --gid "${PGID}" "docker" || echo >&2 "Could not add group docker with ID ${PGID}, its already there probably" + echo "Assign netdata user to docker group ${PGID}" + usermod --append --groups "docker" "${DOCKER_USR}" || echo >&2 "Could not add netdata user to group docker with ID ${PGID}" + fi + + if [ -d "/host/etc/pve" ]; then + add_netdata_to_proxmox_conf_files_group || true + fi +else + echo >&2 "WARNING: Entrypoint started as non-root user. This is not officially supported and some features may not be available." +fi if mountpoint -q /etc/netdata; then echo "Copying stock configuration to /etc/netdata" @@ -97,24 +120,4 @@ if [ -n "${NETDATA_CLAIM_URL}" ] && [ -n "${NETDATA_CLAIM_TOKEN}" ] && [ ! -f /v -daemon-not-running fi -if [ -n "${NETDATA_EXTRA_APK_PACKAGES}" ]; then - echo >&2 "WARNING: Netdata’s Docker images have switched from Alpine to Debian as a base platform. Supplementary package support is now handled through the NETDATA_EXTRA_DEB_PACKAGES variable instead of NETDATA_EXTRA_APK_PACKAGES." - echo >&2 "WARNING: The container will still run, but supplementary packages listed in NETDATA_EXTRA_APK_PACKAGES will not be installed." - echo >&2 "WARNING: To remove these messages, either undefine NETDATA_EXTRA_APK_PACKAGES, or define it to an empty string." -fi - -if [ -n "${NETDATA_EXTRA_DEB_PACKAGES}" ]; then - echo "Fetching APT repository metadata." - if ! apt-get update; then - echo "Failed to fetch APT repository metadata." - else - echo "Installing supplementary packages." - export DEBIAN_FRONTEND="noninteractive" - # shellcheck disable=SC2086 - if ! apt-get install -y --no-install-recommends ${NETDATA_EXTRA_DEB_PACKAGES}; then - echo "Failed to install supplementary packages." - fi - fi -fi - exec /usr/sbin/netdata -u "${DOCKER_USR}" -D -s /host -p "${NETDATA_LISTENER_PORT}" "$@" diff --git a/packaging/ebpf-co-re.checksums b/packaging/ebpf-co-re.checksums index c51f3ef5f..b9e55d875 100644 --- a/packaging/ebpf-co-re.checksums +++ b/packaging/ebpf-co-re.checksums @@ -1 +1 @@ -7ef8d2a0f485b4c81942f66c50e1aedcd568b7997a933c50c0ebbd8353543c08 netdata-ebpf-co-re-glibc-v1.2.8.tar.xz +208e843a1c981071bc84eeb2dba8d64947701e0624abf6669daac3f73feacf40 netdata-ebpf-co-re-glibc-v1.3.1.tar.xz diff --git a/packaging/ebpf-co-re.version b/packaging/ebpf-co-re.version index d1f79a941..757407982 100644 --- a/packaging/ebpf-co-re.version +++ b/packaging/ebpf-co-re.version @@ -1 +1 @@ -v1.2.8 +v1.3.1 diff --git a/packaging/ebpf.checksums b/packaging/ebpf.checksums index 28f023d52..f0c3db759 100644 --- a/packaging/ebpf.checksums +++ b/packaging/ebpf.checksums @@ -1,3 +1,3 @@ -9035b6b8dda5230c1ddc44991518a3ee069bd497ad5a8e5448b79dc4b8c51c43 ./netdata-kernel-collector-glibc-v1.2.8.tar.xz -e5b1a141475f75c60c282a2e3ce8e3914893e75d474c976bad95f66d4c9846c5 ./netdata-kernel-collector-musl-v1.2.8.tar.xz -d6081a2fedc9435d1ab430697cb101123cebaac07b62fb91d790ca526923f4e3 ./netdata-kernel-collector-static-v1.2.8.tar.xz +3beba8ddcc0394324e1444dc239f87ba82f68f88b4446b37598e75584f0b8e5f ./netdata-kernel-collector-glibc-v1.3.1.tar.xz +945c511ad69eecf58d74b6d35b886cc9867e939bd1eca4e1cbe3f01181a3f036 ./netdata-kernel-collector-musl-v1.3.1.tar.xz +5e0c7a230bcb54cbc51cb86518d2199fa6fb2344a02a88a467387dec537aa45a ./netdata-kernel-collector-static-v1.3.1.tar.xz diff --git a/packaging/ebpf.version b/packaging/ebpf.version index d1f79a941..757407982 100644 --- a/packaging/ebpf.version +++ b/packaging/ebpf.version @@ -1 +1 @@ -v1.2.8 +v1.3.1 diff --git a/packaging/go.d.checksums b/packaging/go.d.checksums deleted file mode 100644 index addfc0a50..000000000 --- a/packaging/go.d.checksums +++ /dev/null @@ -1,20 +0,0 @@ -9eeb1a06764fcc7f75e25d56916d1566d4a7206de778856165f407409f091470 *config.tar.gz -9eeb1a06764fcc7f75e25d56916d1566d4a7206de778856165f407409f091470 *go.d.plugin-config-v0.58.0.tar.gz -cb9bbbc164e16fdef46ddb3a9aafa354fe83a2765a5aa25a7ceeaa4c60d90eb7 *go.d.plugin-v0.58.0.darwin-amd64.tar.gz -6d5123955f87ebf30e5faf17c8502616ca84f156ae5e6108cb4a83b79dd0fa6b *go.d.plugin-v0.58.0.darwin-arm64.tar.gz -04a3ceebb345556cfc3f5dd5230c31d06cf59f8f6a6d85c4e8cfb1497ac2c793 *go.d.plugin-v0.58.0.freebsd-386.tar.gz -9b530d2d7d387c81d0551888b0aa6b55290910f75c5f01a0d399ca29fa83757f *go.d.plugin-v0.58.0.freebsd-amd64.tar.gz -45f4d0884b3993d3758f63a453ace96207ebdb9f2d97f89d5e42795ca743c6b6 *go.d.plugin-v0.58.0.freebsd-arm.tar.gz -d4222a6812255946f5d367cd59e8d6284c36b44baaba2925f7268bc42368a41a *go.d.plugin-v0.58.0.freebsd-arm64.tar.gz -4d71efc97a8f32db36f1d3f925e97531f846d9c39d66fbe63f00097f9a6cd425 *go.d.plugin-v0.58.0.linux-386.tar.gz -287db876af5a5b093ee91ef937f4ee59ebc5fdf79e403a48042b9f3cf58c716f *go.d.plugin-v0.58.0.linux-amd64.tar.gz -c0a4f1a20e2d93e1df7adab651b9feb7ca481b0b04e4e12323cad7b8f39e8590 *go.d.plugin-v0.58.0.linux-arm.tar.gz -b94adb6df7fc3a04cda1078e82c2d97a514c12dcc12f5dba7cec2259a34c89bb *go.d.plugin-v0.58.0.linux-arm64.tar.gz -5570b0ebc7c1a45c00301b0212531ee178cc06cb47912330ebc3d3d20bed6b13 *go.d.plugin-v0.58.0.linux-mips.tar.gz -6a850631e1978fdb6ff27923c3779f85e985dd0adb3cfb3767a482777e1802c8 *go.d.plugin-v0.58.0.linux-mips64.tar.gz -1ac22842fa52b97efac45f39f36e9fe69bd9a47497d91653563e02c2855ea5ff *go.d.plugin-v0.58.0.linux-mips64le.tar.gz -2487214cf11430e4152fbccf17205764d91e731aa236b2edb994d8242d33db26 *go.d.plugin-v0.58.0.linux-mipsle.tar.gz -547e4196cd1ebe07054de74f64bcea5ff704376138a495d6b66a6d3f46b22c5f *go.d.plugin-v0.58.0.linux-ppc64.tar.gz -3917e4c798cca7d5f944eb983f8facc2227abff88fc12398a277ee38010540cd *go.d.plugin-v0.58.0.linux-ppc64le.tar.gz -089bff22c63c1b79a0081e3c52e26eacafbea3698f967e5d18cee0c7dd0f88f9 *go.d.plugin-vendor-v0.58.0.tar.gz -089bff22c63c1b79a0081e3c52e26eacafbea3698f967e5d18cee0c7dd0f88f9 *vendor.tar.gz diff --git a/packaging/go.d.version b/packaging/go.d.version deleted file mode 100644 index 0bf661714..000000000 --- a/packaging/go.d.version +++ /dev/null @@ -1 +0,0 @@ -v0.58.0 diff --git a/packaging/installer/README.md b/packaging/installer/README.md index a99e869b8..9260fb6e4 100644 --- a/packaging/installer/README.md +++ b/packaging/installer/README.md @@ -3,23 +3,16 @@ import { InstallRegexLink, InstallBoxRegexLink } from '@site/src/components/Inst import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; -# Install Netdata +# Netdata Agent Installation -This document will guide you through installing the open-source Netdata monitoring Agent on Linux, Docker, Kubernetes, and many others, often with one command. +Netdata is very flexible and can be used to monitor all kinds of infrastructure. Read more about possible [Deployment guides](https://github.com/netdata/netdata/blob/master/docs/deployment-guides/README.md) to understand what better suites your needs. -Netdata is very flexible and can be used to monitor all kinds of infrastructure. Read more about possible [Deployment strategies](https://github.com/netdata/netdata/blob/master/docs/category-overview-pages/deployment-strategies.md) to understand what better suites your needs. +## Install through Netdata Cloud -## Get started +Netdata is a free and open-source (FOSS) monitoring agent that collects thousands of hardware and software metrics from any physical or virtual system (we call them _nodes_). These metrics are organized in an easy-to-use and -navigate interface. -Netdata is a free and open-source (FOSS) monitoring agent that collects thousands of hardware and software metrics from -any physical or virtual system (we call them _nodes_). These metrics are organized in an easy-to-use and -navigate interface. - -Together with [Netdata Cloud](https://github.com/netdata/netdata/blob/master/docs/quickstart/infrastructure.md), you can monitor your entire infrastructure in -real time and troubleshoot problems that threaten the health of your nodes. - -Netdata runs permanently on all your physical/virtual servers, containers, cloud deployments, and edge/IoT devices. It -runs on Linux distributions (Ubuntu, Debian, CentOS, and more), container/microservice platforms (Kubernetes clusters, -Docker), and many other operating systems (FreeBSD, macOS), with no `sudo` required. +Netdata runs permanently on all your physical/virtual servers, containers, cloud deployments, and edge/IoT devices. +It runs on Linux distributions (Ubuntu, Debian, CentOS, and more), container/microservice platforms (Kubernetes clusters, Docker), and many other operating systems (FreeBSD, macOS), with no `sudo` required. To install Netdata in minutes on your platform: @@ -27,113 +20,27 @@ To install Netdata in minutes on your platform: 2. You will be presented with an empty space, and a prompt to "Connect Nodes" with the install command for each platform 3. Select the platform you want to install Netdata to, copy and paste the script into your node's terminal, and run it -Upon installation completing successfully, you should be able to see the node live in your Netdata Space and live charts -in the Overview tab. [Read more about the cloud features](https://github.com/netdata/netdata/blob/master/docs/quickstart/infrastructure.md). - -Where you go from here is based on your use case, immediate needs, and experience with monitoring and troubleshooting, -but we have some hints on what you might want to do next. - -### What's next? - -Explore our [general advanced installation options and troubleshooting](#advanced-installation-options-and-troubleshooting), specific options -for the [single line installer](#install-on-linux-with-one-line-installer), or [other installation methods](#other-installation-methods). - -#### Configuration - -Discover the recommended way to [configure Netdata's settings or behavior](https://github.com/netdata/netdata/blob/master/docs/configure/nodes.md) using our built-in -`edit-config` script, then apply that knowledge to mission-critical tweaks, such as [changing how long Netdata stores -metrics](https://github.com/netdata/netdata/blob/master/docs/store/change-metrics-storage.md). - -#### Data collection - -If Netdata didn't autodetect all the hardware, containers, services, or applications running on your node, you should -learn more about [how data collectors work](https://github.com/netdata/netdata/blob/master/collectors/README.md). If there's a [supported -collector](https://github.com/netdata/netdata/blob/master/collectors/COLLECTORS.md) for metrics you need, [configure the collector](https://github.com/netdata/netdata/blob/master/collectors/REFERENCE.md) -or read about its requirements to configure your endpoint to publish metrics in the correct format and endpoint. - -#### Alerts & notifications - -Netdata comes with hundreds of preconfigured alerts, designed by our monitoring gurus in parallel with our open-source -community, but you may want to [edit alerts](https://github.com/netdata/netdata/blob/master/health/REFERENCE.md) or -[enable notifications](https://github.com/netdata/netdata/blob/master/docs/monitor/enable-notifications.md) to customize your Netdata experience. - -#### Make your deployment production ready - -Go through our [deployment strategies](https://github.com/netdata/netdata/edit/master/docs/category-overview-pages/deployment-strategies.md), -for suggested configuration changes for production deployments. - -## Install on Linux with one-line installer - -The **recommended** way to install Netdata on a Linux node (physical, virtual, container, IoT) is our one-line -[kickstart script](https://github.com/netdata/netdata/blob/master/packaging/installer/methods/kickstart.md). -This script automatically installs dependencies and builds Netdata from its source code. - -To install, copy the script, paste it into your node's terminal, and hit `Enter` to begin the installation process. +Upon installation completing successfully, you should be able to see the node live in your Netdata Space and live charts in the Overview tab. [Take a look at our Dashboards and Charts](https://github.com/netdata/netdata/blob/master/docs/category-overview-pages/accessing-netdata-dashboards.md) section to read more about Netdata's features. - <Tabs> - <TabItem value="wget" label=<code>wget</code>> +## Maintaining a Netdata Agent installation - <OneLineInstallWget/> +For actions like starting, stopping, restarting, updating and uninstalling the Netdata Agent take a look at your specific installation platform in the current section of our Documentation. - </TabItem> - <TabItem value="curl" label=<code>curl</code>> +## Configuration - <OneLineInstallCurl/> +If you are looking to configure your Netdata Agent installation, refer to the [respective section in our Documentation](https://github.com/netdata/netdata/blob/master/docs/netdata-agent/configuration.md). - </TabItem> -</Tabs> +## Data collection -> ### Note -> -> If you plan to also claim the node to Netdata Cloud, make sure to replace `YOUR_CLAIM_TOKEN` with the claim token of your space, and `YOUR_ROOM_ID` with the ID of the room you are claiming to. -> You can leave the room id blank to have your node claimed to the default "All nodes" room. +If Netdata didn't autodetect all the hardware, containers, services, or applications running on your node, you should learn more about [how data collectors work](https://github.com/netdata/netdata/blob/master/src/collectors/README.md). If there's a [supported collector](https://github.com/netdata/netdata/blob/master/src/collectors/COLLECTORS.md) for metrics you need, [configure the collector](https://github.com/netdata/netdata/blob/master/src/collectors/REFERENCE.md) or read about its requirements to configure your endpoint to publish metrics in the correct format and endpoint. -Jump up to [what's next](#whats-next) to learn how to view your new dashboard and take your next steps in monitoring and -troubleshooting with Netdata. +## Alerts & notifications -## Other installation methods +Netdata comes with hundreds of pre-configured alerts, designed by our monitoring gurus in parallel with our open-source community, but you may want to [edit alerts](https://github.com/netdata/netdata/blob/master/src/health/REFERENCE.md) or [enable notifications](https://github.com/netdata/netdata/blob/master/docs/monitor/enable-notifications.md) to customize your Netdata experience. -<InstallRegexLink> - <InstallBoxRegexLink - to="[](https://github.com/netdata/netdata/blob/master/packaging/docker/README.md)" - os="Run with Docker" - svg="docker" /> - <InstallBoxRegexLink - to="[](https://github.com/netdata/netdata/blob/master/packaging/installer/methods/kubernetes.md)" - os="Deploy on Kubernetes" - svg="kubernetes" /> - <InstallBoxRegexLink - to="[](https://github.com/netdata/netdata/blob/master/packaging/installer/methods/macos.md)" - os="Install on macOS" - svg="macos" /> - <InstallBoxRegexLink - to="[](https://github.com/netdata/netdata/blob/master/packaging/installer/methods/packages.md)" - os="Native DEB/RPM packages" - svg="linux" /> - <InstallBoxRegexLink - to="[](https://github.com/netdata/netdata/blob/master/packaging/installer/methods/manual.md)" - os="Linux from Git" - svg="linux" /> - <InstallBoxRegexLink - to="[](https://github.com/netdata/netdata/blob/master/packaging/installer/methods/source.md)" - os="Linux from source" - svg="linux" /> - <InstallBoxRegexLink - to="[](https://github.com/netdata/netdata/blob/master/packaging/installer/methods/offline.md)" - os="Linux for offline nodes" - svg="linux" /> -</InstallRegexLink> +## Make your deployment production ready -- [Native DEB/RPM packages](https://github.com/netdata/netdata/blob/master/packaging/installer/methods/packages.md) -- [Run with Docker](https://github.com/netdata/netdata/blob/master/packaging/docker/README.md) -- [Deploy on Kubernetes](https://github.com/netdata/netdata/blob/master/packaging/installer/methods/kubernetes.md) -- [Install on macOS](https://github.com/netdata/netdata/blob/master/packaging/installer/methods/macos.md) -- [Linux from Git](https://github.com/netdata/netdata/blob/master/packaging/installer/methods/manual.md) -- [Linux from source](https://github.com/netdata/netdata/blob/master/packaging/installer/methods/source.md) -- [Linux for offline nodes](https://github.com/netdata/netdata/blob/master/packaging/installer/methods/offline.md) - -The full list of all installation methods for various systems is available in [Netdata Learn](https://learn.netdata.cloud), -under [Installation](https://github.com/netdata/netdata/blob/master/docs/category-overview-pages/installation-overview.md). +Go through our [deployment guides](https://github.com/netdata/netdata/blob/master/docs/deployment-guides/README.md), for suggested configuration changes for production deployments. ## Advanced installation options and troubleshooting @@ -146,7 +53,7 @@ option when you install or update Netdata using the [automatic one-line installa script](#automatic-one-line-installation-script). ```bash -wget -O /tmp/netdata-kickstart.sh https://my-netdata.io/kickstart.sh && sh /tmp/netdata-kickstart.sh --no-updates +wget -O /tmp/netdata-kickstart.sh https://get.netdata.cloud/kickstart.sh && sh /tmp/netdata-kickstart.sh --no-updates ``` With automatic updates disabled, you can choose exactly when and how you [update @@ -215,7 +122,7 @@ There are three potential workarounds for this: affect many projects other than just Netdata, and there are unfortunately a number of other services out there that do not provide IPv6 connectivity, so taking this route is likely to save you time in the future as well. 2. If you are using a system that we publish native packages for (see our [platform support - policy](https://github.com/netdata/netdata/blob/master/packaging/PLATFORM_SUPPORT.md) for more details), + policy](https://github.com/netdata/netdata/blob/master/docs/netdata-agent/versions-and-platforms.md) for more details), you can manually set up our native package repositories as outlined in our [native package install documentation](https://github.com/netdata/netdata/blob/master/packaging/installer/methods/packages.md). Our official package repositories do provide service over IPv6, so they work without issue on hosts without IPv4 connectivity. diff --git a/packaging/installer/REINSTALL.md b/packaging/installer/REINSTALL.md index 82cea498a..91dd3d5f3 100644 --- a/packaging/installer/REINSTALL.md +++ b/packaging/installer/REINSTALL.md @@ -18,7 +18,7 @@ used, read the contents of the `.environment` file and look for a `REINSTALL_OPT optional parameters. ```bash -wget -O /tmp/netdata-kickstart.sh https://my-netdata.io/kickstart.sh && sh /tmp/netdata-kickstart.sh --reinstall +wget -O /tmp/netdata-kickstart.sh https://get.netdata.cloud/kickstart.sh && sh /tmp/netdata-kickstart.sh --reinstall ``` ### Performing a clean reinstall @@ -35,7 +35,7 @@ used, read the contents of the `.environment` file and look for a `REINSTALL_OPT optional parameters. ```bash -wget -O /tmp/netdata-kickstart.sh https://my-netdata.io/kickstart.sh && sh /tmp/netdata-kickstart.sh --reinstall-clean +wget -O /tmp/netdata-kickstart.sh https://get.netdata.cloud/kickstart.sh && sh /tmp/netdata-kickstart.sh --reinstall-clean ``` ### Changing the install type of an existing installation diff --git a/packaging/installer/UNINSTALL.md b/packaging/installer/UNINSTALL.md index a66bd7a28..dc1dec0c2 100644 --- a/packaging/installer/UNINSTALL.md +++ b/packaging/installer/UNINSTALL.md @@ -8,13 +8,13 @@ The recommended method to uninstall Netdata on a system is to use our kickstart installer script with the `--uninstall` option like so: ```sh -wget -O /tmp/netdata-kickstart.sh https://my-netdata.io/kickstart.sh && sh /tmp/netdata-kickstart.sh --uninstall +wget -O /tmp/netdata-kickstart.sh https://get.netdata.cloud/kickstart.sh && sh /tmp/netdata-kickstart.sh --uninstall ``` Or (if you have curl but not wget): ```sh -curl https://my-netdata.io/kickstart.sh > /tmp/netdata-kickstart.sh && sh /tmp/netdata-kickstart.sh --uninstall +curl https://get.netdata.cloud/kickstart.sh > /tmp/netdata-kickstart.sh && sh /tmp/netdata-kickstart.sh --uninstall ``` This will work in most cases without you needing to do anything more other than accepting removal of configuration diff --git a/packaging/installer/UPDATE.md b/packaging/installer/UPDATE.md index 7275ee524..50d8c8270 100644 --- a/packaging/installer/UPDATE.md +++ b/packaging/installer/UPDATE.md @@ -52,7 +52,7 @@ installation script in dry-run mode to attempt to determine what method to use t command: ```bash -wget -O /tmp/netdata-kickstart.sh https://my-netdata.io/kickstart.sh && sh /tmp/netdata-kickstart.sh --dry-run +wget -O /tmp/netdata-kickstart.sh https://get.netdata.cloud/kickstart.sh && sh /tmp/netdata-kickstart.sh --dry-run ``` Note that if you installed Netdata using an installation prefix, you will need to add an `--install-prefix` option @@ -75,7 +75,7 @@ If you installed Netdata using an installation prefix, you will need to add an ` that prefix to this command to make sure it finds Netdata. ```bash -wget -O /tmp/netdata-kickstart.sh https://my-netdata.io/kickstart.sh && sh /tmp/netdata-kickstart.sh +wget -O /tmp/netdata-kickstart.sh https://get.netdata.cloud/kickstart.sh && sh /tmp/netdata-kickstart.sh ``` ### Issues with older binpkg installs @@ -204,6 +204,8 @@ The following configuration options are currently supported: as a scheduled task. This random delay helps avoid issues resulting from too many nodes trying to reconnect to the Cloud at the same time. The default value is 3600, which corresponds to one hour. Most users should not ever need to change this. +- `NETDATA_MAJOR_VERSION_UPDATES`: If set to a value other than 0, then new major versions will be installed + without user confirmation. Must be set to a non-zero value for automated updates to install new major versions. - `NETDATA_NO_SYSTEMD_JOURNAL`: If set to a value other than 0, skip attempting to install the `netdata-plugin-systemd-journal` package on supported systems on update. This optional package will be installed by default on supported systems by the updater if this option is not set. Only affects systems using native packages. diff --git a/packaging/installer/functions.sh b/packaging/installer/functions.sh index dd3158d6d..9060930ce 100644 --- a/packaging/installer/functions.sh +++ b/packaging/installer/functions.sh @@ -192,6 +192,133 @@ netdata_banner() { } # ----------------------------------------------------------------------------- +# Feature management and configuration commands + +enable_feature() { + NETDATA_CMAKE_OPTIONS="$(echo "${NETDATA_CMAKE_OPTIONS}" | sed -e "s/-DENABLE_${1}=Off[[:space:]]*//g" -e "s/-DENABLE_${1}=On[[:space:]]*//g")" + if [ "${2}" -eq 1 ]; then + NETDATA_CMAKE_OPTIONS="$(echo "${NETDATA_CMAKE_OPTIONS}" | sed "s/$/ -DENABLE_${1}=On/")" + else + NETDATA_CMAKE_OPTIONS="$(echo "${NETDATA_CMAKE_OPTIONS}" | sed "s/$/ -DENABLE_${1}=Off/")" + fi +} + +check_for_module() { + if [ -z "${pkgconf}" ]; then + pkgconf="$(command -v pkgconf 2>/dev/null)" + [ -z "${pkgconf}" ] && pkgconf="$(command -v pkg-config 2>/dev/null)" + [ -z "${pkgconf}" ] && fatal "Unable to find a usable pkgconf/pkg-config command, cannot build Netdata." I0013 + fi + + "${pkgconf}" "${1}" + return "${?}" +} + +check_for_feature() { + feature_name="${1}" + feature_state="${2}" + shift 2 + feature_modules="${*}" + + if [ -z "${feature_state}" ]; then + # shellcheck disable=SC2086 + if check_for_module ${feature_modules}; then + enable_feature "${feature_name}" 1 + else + enable_feature "${feature_name}" 0 + fi + else + enable_feature "${feature_name}" "${feature_state}" + fi +} + +prepare_cmake_options() { + NETDATA_CMAKE_OPTIONS="-S ./ -B ${NETDATA_BUILD_DIR} ${CMAKE_OPTS} ${NETDATA_PREFIX+-DCMAKE_INSTALL_PREFIX="${NETDATA_PREFIX}"} ${NETDATA_USER:+-DNETDATA_USER=${NETDATA_USER}} ${NETDATA_CMAKE_OPTIONS} " + + NEED_OLD_CXX=0 + + if [ "${FORCE_LEGACY_CXX:-0}" -eq 1 ]; then + NEED_OLD_CXX=1 + else + if command -v gcc >/dev/null 2>&1; then + if [ "$(gcc --version | head -n 1 | sed 's/(.*) //' | cut -f 2 -d ' ' | cut -f 1 -d '.')" -lt 5 ]; then + NEED_OLD_CXX=1 + fi + fi + + if command -v clang >/dev/null 2>&1; then + if [ "$(clang --version | head -n 1 | cut -f 3 -d ' ' | cut -f 1 -d '.')" -lt 4 ]; then + NEED_OLD_CXX=1 + fi + fi + fi + + if [ "${NEED_OLD_CXX}" -eq 1 ]; then + NETDATA_CMAKE_OPTIONS="${NETDATA_CMAKE_OPTIONS} -DUSE_CXX_11=On" + fi + + if [ "${ENABLE_GO:-1}" -eq 1 ]; then + enable_feature PLUGIN_GO 1 + else + enable_feature PLUGIN_GO 0 + fi + + if [ "${USE_SYSTEM_PROTOBUF:-0}" -eq 1 ]; then + enable_feature BUNDLED_PROTOBUF 0 + else + enable_feature BUNDLED_PROTOBUF 1 + fi + + if [ -z "${ENABLE_SYSTEMD_JOURNAL}" ]; then + if check_for_module libsystemd; then + if check_for_module libelogind; then + ENABLE_SYSTEMD_JOURNAL=0 + else + ENABLE_SYSTEMD_JOURNAL=1 + fi + else + ENABLE_SYSTEMD_JOURNAL=0 + fi + fi + + enable_feature PLUGIN_SYSTEMD_JOURNAL "${ENABLE_SYSTEMD_JOURNAL}" + + if command -v cups-config >/dev/null 2>&1 || check_for_module libcups || check_for_module cups; then + ENABLE_CUPS=1 + else + ENABLE_CUPS=0 + fi + + enable_feature PLUGIN_CUPS "${ENABLE_CUPS}" + + IS_LINUX=0 + [ "$(uname -s)" = "Linux" ] && IS_LINUX=1 + enable_feature PLUGIN_DEBUGFS "${IS_LINUX}" + enable_feature PLUGIN_PERF "${IS_LINUX}" + enable_feature PLUGIN_SLABINFO "${IS_LINUX}" + enable_feature PLUGIN_CGROUP_NETWORK "${IS_LINUX}" + enable_feature PLUGIN_LOCAL_LISTENERS "${IS_LINUX}" + enable_feature PLUGIN_NETWORK_VIEWER "${IS_LINUX}" + enable_feature PLUGIN_EBPF "${ENABLE_EBPF:-0}" + enable_feature PLUGIN_LOGS_MANAGEMENT "${ENABLE_LOGS_MANAGEMENT:-0}" + enable_feature LOGS_MANAGEMENT_TESTS "${ENABLE_LOGS_MANAGEMENT_TESTS:-0}" + + enable_feature ACLK "${ENABLE_CLOUD:-1}" + enable_feature CLOUD "${ENABLE_CLOUD:-1}" + enable_feature BUNDLED_JSONC "${NETDATA_BUILD_JSON_C:-0}" + enable_feature DBENGINE "${ENABLE_DBENGINE:-1}" + enable_feature H2O "${ENABLE_H2O:-1}" + enable_feature ML "${NETDATA_ENABLE_ML:-1}" + enable_feature PLUGIN_APPS "${ENABLE_APPS:-1}" + + check_for_feature EXPORTER_PROMETHEUS_REMOTE_WRITE "${EXPORTER_PROMETHEUS}" snappy + check_for_feature EXPORTER_MONGODB "${EXPORTER_MONGODB}" libmongoc-1.0 + check_for_feature PLUGIN_FREEIPMI "${ENABLE_FREEIPMI}" libipmimonitoring + check_for_feature PLUGIN_NFACCT "${ENABLE_NFACCT}" libnetfilter_acct libnml + check_for_feature PLUGIN_XENSTAT "${ENABLE_XENSTAT}" xenstat xenlight +} + +# ----------------------------------------------------------------------------- # portable service command service_cmd="$(command -v service 2> /dev/null || true)" @@ -463,45 +590,6 @@ get_systemd_service_dir() { fi } -install_non_systemd_init() { - [ "${UID}" != 0 ] && return 1 - key="$(get_os_key)" - - if [ -d /etc/init.d ] && [ ! -f /etc/init.d/netdata ]; then - if expr "${key}" : "^(gentoo|alpine).*"; then - echo >&2 "Installing OpenRC init file..." - run cp system/openrc/init.d/netdata /etc/init.d/netdata && - run chmod 755 /etc/init.d/netdata && - run rc-update add netdata default && - return 0 - - elif expr "${key}" : "^devuan*" || [ "${key}" = "debian-7" ] || [ "${key}" = "ubuntu-12.04" ] || [ "${key}" = "ubuntu-14.04" ]; then - echo >&2 "Installing LSB init file..." - run cp system/lsb/init.d/netdata /etc/init.d/netdata && - run chmod 755 /etc/init.d/netdata && - run update-rc.d netdata defaults && - run update-rc.d netdata enable && - return 0 - elif expr "${key}" : "^(amzn-201[5678]|ol|CentOS release 6|Red Hat Enterprise Linux Server release 6|Scientific Linux CERN SLC release 6|CloudLinux Server release 6).*"; then - echo >&2 "Installing init.d file..." - run cp system/initd/init.d/netdata /etc/init.d/netdata && - run chmod 755 /etc/init.d/netdata && - run chkconfig netdata on && - return 0 - else - warning "Could not determine what type of init script to install on this system." - return 1 - fi - elif [ -f /etc/init.d/netdata ]; then - echo >&2 "file '/etc/init.d/netdata' already exists." - return 0 - else - warning "Could not determine what type of init script to install on this system." - fi - - return 1 -} - run_install_service_script() { if [ -z "${tmpdir}" ]; then tmpdir="${TMPDIR:-/tmp}" @@ -565,90 +653,7 @@ install_netdata_service() { if [ -x "${NETDATA_PREFIX}/usr/libexec/netdata/install-service.sh" ]; then run_install_service_script && return 0 else - # This is used by netdata-installer.sh - # shellcheck disable=SC2034 - NETDATA_STOP_CMD="netdatacli shutdown-agent" - - NETDATA_START_CMD="netdata" - NETDATA_INSTALLER_START_CMD="" - - uname="$(uname 2> /dev/null)" - - if [ "${uname}" = "Darwin" ]; then - if [ -f "/Library/LaunchDaemons/com.github.netdata.plist" ]; then - echo >&2 "file '/Library/LaunchDaemons/com.github.netdata.plist' already exists." - return 0 - else - echo >&2 "Installing MacOS X plist file..." - # This is used by netdata-installer.sh - # shellcheck disable=SC2034 - run cp system/launchd/netdata.plist /Library/LaunchDaemons/com.github.netdata.plist && - run launchctl load /Library/LaunchDaemons/com.github.netdata.plist && - NETDATA_START_CMD="launchctl start com.github.netdata" && - NETDATA_STOP_CMD="launchctl stop com.github.netdata" - return 0 - fi - - elif [ "${uname}" = "FreeBSD" ]; then - # This is used by netdata-installer.sh - # shellcheck disable=SC2034 - run cp system/freebsd/rc.d/netdata /etc/rc.d/netdata && NETDATA_START_CMD="service netdata start" && - NETDATA_STOP_CMD="service netdata stop" && - NETDATA_INSTALLER_START_CMD="service netdata onestart" && - myret=$? - - echo >&2 "Note: To explicitly enable netdata automatic start, set 'netdata_enable' to 'YES' in /etc/rc.conf" - echo >&2 "" - - return "${myret}" - - elif issystemd; then - # systemd is running on this system - NETDATA_START_CMD="systemctl start netdata" - # This is used by netdata-installer.sh - # shellcheck disable=SC2034 - NETDATA_STOP_CMD="systemctl stop netdata" - NETDATA_INSTALLER_START_CMD="${NETDATA_START_CMD}" - - SYSTEMD_DIRECTORY="$(get_systemd_service_dir)" - - if [ "${SYSTEMD_DIRECTORY}x" != "x" ]; then - ENABLE_NETDATA_IF_PREVIOUSLY_ENABLED="run systemctl enable netdata" - IS_NETDATA_ENABLED="$(systemctl is-enabled netdata 2> /dev/null || echo "Netdata not there")" - if [ "${IS_NETDATA_ENABLED}" = "disabled" ]; then - echo >&2 "Netdata was there and disabled, make sure we don't re-enable it ourselves" - ENABLE_NETDATA_IF_PREVIOUSLY_ENABLED="true" - fi - - echo >&2 "Installing systemd service..." - run cp system/systemd/netdata.service "${SYSTEMD_DIRECTORY}/netdata.service" && - run systemctl daemon-reload && - ${ENABLE_NETDATA_IF_PREVIOUSLY_ENABLED} && - return 0 - else - warning "Could not find a systemd service directory, unable to install Netdata systemd service." - fi - else - install_non_systemd_init - ret=$? - - if [ ${ret} -eq 0 ]; then - if [ -n "${service_cmd}" ]; then - NETDATA_START_CMD="service netdata start" - # This is used by netdata-installer.sh - # shellcheck disable=SC2034 - NETDATA_STOP_CMD="service netdata stop" - elif [ -n "${rcservice_cmd}" ]; then - NETDATA_START_CMD="rc-service netdata start" - # This is used by netdata-installer.sh - # shellcheck disable=SC2034 - NETDATA_STOP_CMD="rc-service netdata stop" - fi - NETDATA_INSTALLER_START_CMD="${NETDATA_START_CMD}" - fi - - return ${ret} - fi + warning "Could not find service install script, not installing Netdata as a system service." fi fi @@ -928,6 +933,11 @@ portable_add_user() { echo >&2 "User '${username}' already exists." return 0 fi + elif command -v dscl > /dev/null 2>&1; then + if dscl . read /Users/"${username}" >/dev/null 2>&1; then + echo >&2 "User '${username}' already exists." + return 0 + fi else if cut -d ':' -f 1 < /etc/passwd | grep "^${username}$" 1> /dev/null 2>&1; then echo >&2 "User '${username}' already exists." @@ -946,7 +956,13 @@ portable_add_user() { elif command -v adduser 1> /dev/null 2>&1; then run adduser -h "${homedir}" -s "${nologin}" -D -G "${username}" "${username}" && return 0 elif command -v sysadminctl 1> /dev/null 2>&1; then - run sysadminctl -addUser "${username}" && return 0 + gid=$(dscl . read /Groups/"${username}" 2>/dev/null | grep PrimaryGroupID | grep -Eo "[0-9]+") + if run sysadminctl -addUser "${username}" -shell /usr/bin/false -home /var/empty -GID "$gid"; then + # FIXME: I think the proper solution is to create a role account: + # -roleAccount + name starting with _ and UID in 200-400 range. + run dscl . create /Users/"${username}" IsHidden 1 + return 0 + fi fi warning "Failed to add ${username} user account!" @@ -1061,9 +1077,11 @@ install_netdata_updater() { cat "${NETDATA_SOURCE_DIR}/packaging/installer/netdata-updater.sh" > "${NETDATA_PREFIX}/usr/libexec/netdata/netdata-updater.sh" || return 1 fi - if issystemd && [ -n "$(get_systemd_service_dir)" ]; then - cat "${NETDATA_SOURCE_DIR}/system/systemd/netdata-updater.timer" > "$(get_systemd_service_dir)/netdata-updater.timer" - cat "${NETDATA_SOURCE_DIR}/system/systemd/netdata-updater.service" > "$(get_systemd_service_dir)/netdata-updater.service" + # these files are installed by cmake + libsysdir="${NETDATA_PREFIX}/usr/lib/netdata/system/systemd/" + if [ -d "${libsysdir}" ] && issystemd && [ -n "$(get_systemd_service_dir)" ]; then + cat "${libsysdir}/netdata-updater.timer" > "$(get_systemd_service_dir)/netdata-updater.timer" + cat "${libsysdir}/netdata-updater.service" > "$(get_systemd_service_dir)/netdata-updater.service" fi sed -i -e "s|THIS_SHOULD_BE_REPLACED_BY_INSTALLER_SCRIPT|${NETDATA_USER_CONFIG_DIR}/.environment|" "${NETDATA_PREFIX}/usr/libexec/netdata/netdata-updater.sh" || return 1 diff --git a/packaging/installer/install-required-packages.sh b/packaging/installer/install-required-packages.sh index bdd529397..179cbd5e9 100755 --- a/packaging/installer/install-required-packages.sh +++ b/packaging/installer/install-required-packages.sh @@ -946,7 +946,7 @@ declare -A pkg_postfix=( ) declare -A pkg_pkg_config=( - ['alpine']="pkgconfig" + ['alpine']="pkgconf" ['arch']="pkgconfig" ['centos']="pkgconfig" ['debian']="pkg-config" diff --git a/packaging/installer/kickstart.sh b/packaging/installer/kickstart.sh index f7c078274..6d9f4ea0e 100755 --- a/packaging/installer/kickstart.sh +++ b/packaging/installer/kickstart.sh @@ -2,7 +2,7 @@ # # SPDX-License-Identifier: GPL-3.0-or-later # -# Next unused error code: F0516 +# Next unused error code: F0517 # ====================================================================== # Constants @@ -21,10 +21,10 @@ KICKSTART_SOURCE="$( )" DEFAULT_PLUGIN_PACKAGES="" PATH="${PATH}:/usr/local/bin:/usr/local/sbin" -REPOCONFIG_DEB_VERSION="2-1" -REPOCONFIG_RPM_VERSION="2-1" +REPOCONFIG_DEB_VERSION="2-2" +REPOCONFIG_RPM_VERSION="2-2" START_TIME="$(date +%s)" -STATIC_INSTALL_ARCHES="x86_64 armv7l aarch64 ppc64le" +STATIC_INSTALL_ARCHES="x86_64 armv7l armv6l aarch64 ppc64le" # ====================================================================== # URLs used throughout the script @@ -125,6 +125,7 @@ main() { ;; esac + handle_existing_install set_tmpdir if [ -n "${INSTALL_VERSION}" ]; then @@ -402,6 +403,9 @@ support_list() { } success_banner() { + printf >&2 "%s\n" "To view your system's real-time performance metrics, open your web browser and enter http://NODE:19999." + printf >&2 "%s\n\n" "Replace NODE with the IP address or hostname of your Netdata server to access the dashboard." + printf >&2 "%s\n\n" "Official documentation can be found online at ${DOCS_URL}." if [ -z "${CLAIM_TOKEN}" ]; then @@ -591,6 +595,8 @@ check_for_remote_file() { if echo "${url}" | grep -Eq "^file:///"; then [ -e "${url#file://}" ] || return 1 + elif [ -n "${NETDATA_ASSUME_REMOTE_FILES_ARE_PRESENT}" ]; then + return 0 elif [ -n "${CURL}" ]; then "${CURL}" --output /dev/null --silent --head --fail "${url}" || return 1 elif command -v wget > /dev/null 2>&1; then @@ -635,9 +641,9 @@ get_redirect() { url="${1}" if [ -n "${CURL}" ]; then - run sh -c "${CURL} ${url} -s -L -I -o /dev/null -w '%{url_effective}' | grep -o '[^/]*$'" || return 1 + run sh -c "${CURL} ${url} -s -L -I -o /dev/null -w '%{url_effective}' | grep -Eo '[^/]+$'" || return 1 elif command -v wget > /dev/null 2>&1; then - run sh -c "wget -S -O /dev/null ${url} 2>&1 | grep -m 1 Location | grep -o '[^/]*$'" || return 1 + run sh -c "wget -S -O /dev/null ${url} 2>&1 | grep -m 1 Location | grep -Eo '[^/]+$'" || return 1 else fatal "${ERROR_F0003}" F0003 fi @@ -1325,7 +1331,7 @@ netdata_avail_check() { ;; centos|fedora|ol|amzn) # shellcheck disable=SC2086 - ${pm_cmd} search --nogpgcheck -v netdata | grep -qE 'Repo *: netdata(-edge)?$' + LC_ALL=C ${pm_cmd} search --nogpgcheck -v netdata | grep -qE 'Repo *: netdata(-edge)?$' return $? ;; opensuse) @@ -1341,7 +1347,7 @@ check_special_native_deps() { if [ "${DISTRO_COMPAT_NAME}" = "centos" ] && [ "${SYSVERSION}" -gt 6 ]; then progress "EPEL is required on this system, checking if it’s available." - if ${pm_cmd} search --nogpgcheck -v epel-release | grep -q "No matches found"; then + if LC_ALL=C ${pm_cmd} search --nogpgcheck -v epel-release | grep -q "No matches found"; then warning "Unable to find a suitable source for libuv, cannot install using native packages on this system." return 1 else @@ -1356,6 +1362,14 @@ check_special_native_deps() { fi } +cleanup_apt_cache() { + cache_dir="/var/cache/apt/archives" + + if [ -d "${cache_dir}" ]; then + run_as_root find "${cache_dir}" -type f -name 'netdata*.deb' -delete + fi +} + common_rpm_opts() { pkg_type="rpm" pkg_suffix=".noarch" @@ -1422,6 +1436,7 @@ try_package_install() { install_subcmd="install" fi needs_early_refresh=1 + needs_apt_cache_cleanup=1 pm_cmd="apt-get" repo_subcmd="update" pkg_type="deb" @@ -1496,15 +1511,21 @@ try_package_install() { deb) repoconfig_file="${repoconfig_name}${pkg_vsep}${REPOCONFIG_DEB_VERSION}${pkg_suffix}.${pkg_type}" repoconfig_url="${REPOCONFIG_DEB_URL_PREFIX}/${repo_prefix}/${repoconfig_file}" + ref_check_url="${REPOCONFIG_DEB_URL_PREFIX}" ;; rpm) repoconfig_file="${repoconfig_name}${pkg_vsep}${REPOCONFIG_RPM_VERSION}${pkg_suffix}.${pkg_type}" repoconfig_url="${REPOCONFIG_RPM_URL_PREFIX}/${repo_prefix}/${SYSARCH}/${repoconfig_file}" + ref_check_url="${REPOCONFIG_RPM_URL_PREFIX}" ;; esac if ! pkg_installed "${repoconfig_name}"; then progress "Checking for availability of repository configuration package." + if ! check_for_remote_file "${ref_check_url}"; then + NETDATA_ASSUME_REMOTE_FILES_ARE_PRESENT=1 + fi + if ! check_for_remote_file "${repoconfig_url}"; then warning "No repository configuration package available for ${DISTRO} ${SYSVERSION}. Cannot install native packages on this system." return 2 @@ -1520,6 +1541,10 @@ try_package_install() { warning "${failed_refresh_msg}" return 2 fi + + if [ -n "${needs_apt_cache_cleanup}" ]; then + cleanup_apt_cache + fi fi # shellcheck disable=SC2086 @@ -1643,6 +1668,10 @@ try_static_install() { progress "Attempting to install using static build..." fi + if ! check_for_remote_file "${NETDATA_TARBALL_BASEURL}"; then + NETDATA_ASSUME_REMOTE_FILES_ARE_PRESENT=1 + fi + # Check status code first, so that we can provide nicer fallback for dry runs. if check_for_remote_file "${NETDATA_STATIC_ARCHIVE_URL}"; then netdata_agent="${NETDATA_STATIC_ARCHIVE_NAME}" @@ -1650,7 +1679,7 @@ try_static_install() { netdata_agent="${NETDATA_STATIC_ARCHIVE_OLD_NAME}" export NETDATA_STATIC_ARCHIVE_URL="${NETDATA_STATIC_ARCHIVE_OLD_URL}" else - warning "There is no static build available for ${SYSARCH} CPUs. This usually means we simply do not currently provide static builds for ${SYSARCH} CPUs." + warning "Could not find a ${SELECTED_RELEASE_CHANNEL} static build for ${SYSARCH} CPUs. This usually means there is some networking issue preventing access to https://github.com/ from this system." return 2 fi @@ -1750,8 +1779,14 @@ install_local_build_dependencies() { fi # shellcheck disable=SC2086 - if ! run_as_root "${bash}" "${tmpdir}/install-required-packages.sh" ${opts} netdata; then - warning "Failed to install all required packages, but installation might still be possible." + if [ "$(uname -s)" = "Darwin" ]; then + if ! run "${bash}" "${tmpdir}/install-required-packages.sh" ${opts} netdata; then + warning "Failed to install all required packages, but installation might still be possible." + fi + else + if ! run_as_root "${bash}" "${tmpdir}/install-required-packages.sh" ${opts} netdata; then + warning "Failed to install all required packages, but installation might still be possible." + fi fi } @@ -1875,13 +1910,21 @@ prepare_offline_install_source() { static|'') set_static_archive_urls "${SELECTED_RELEASE_CHANNEL}" "x86_64" + if ! check_for_remote_file "${NETDATA_TARBALL_BASEURL}"; then + NETDATA_ASSUME_REMOTE_FILES_ARE_PRESENT=1 + fi + if check_for_remote_file "${NETDATA_STATIC_ARCHIVE_URL}"; then for arch in ${STATIC_INSTALL_ARCHES}; do set_static_archive_urls "${SELECTED_RELEASE_CHANNEL}" "${arch}" - progress "Fetching ${NETDATA_STATIC_ARCHIVE_URL}" - if ! download "${NETDATA_STATIC_ARCHIVE_URL}" "netdata-${arch}-latest.gz.run"; then - warning "Failed to download static installer archive for ${arch}. ${BADNET_MSG}." + if check_for_remote_file "${NETDATA_STATIC_ARCH_URL}"; then + progress "Fetching ${NETDATA_STATIC_ARCHIVE_URL}" + if ! download "${NETDATA_STATIC_ARCHIVE_URL}" "netdata-${arch}-latest.gz.run"; then + warning "Failed to download static installer archive for ${arch}. ${BADNET_MSG}." + fi + else + progress "Skipping ${NETDATA_STATIC_ARCHIVE_URL} as it does not exist on the server." fi done legacy=0 @@ -1895,6 +1938,10 @@ prepare_offline_install_source() { fi fi + if ! find . -name '*.gz.run'; then + fatal "Did not actually download any static installer archives, cannot continue. ${BADNET_MSG}." F0516 + fi + progress "Fetching ${NETDATA_STATIC_ARCHIVE_CHECKSUM_URL}" if ! download "${NETDATA_STATIC_ARCHIVE_CHECKSUM_URL}" "sha256sums.txt"; then fatal "Failed to download checksum file. ${BADNET_MSG}." F0506 @@ -2325,8 +2372,4 @@ confirm_root_support get_system_info confirm_install_prefix -if [ -z "${ACTION}" ]; then - handle_existing_install -fi - main diff --git a/packaging/installer/methods/ansible.md b/packaging/installer/methods/ansible.md index 6ce4e8f0f..85c982fa1 100644 --- a/packaging/installer/methods/ansible.md +++ b/packaging/installer/methods/ansible.md @@ -130,7 +130,7 @@ hostname of the node, the playbook disables that local dashboard by setting `web security boost by not allowing any unwanted access to the local dashboard. You can read more about this decision, or other ways you might lock down the local dashboard, in our [node security -doc](https://github.com/netdata/netdata/blob/master/docs/netdata-security.md). +doc](https://github.com/netdata/netdata/blob/master/docs/security-and-privacy-design/README.md). > Curious about why Netdata's dashboard is open by default? Read our [blog > post](https://www.netdata.cloud/blog/netdata-agent-dashboard/) on that zero-configuration design decision. diff --git a/packaging/installer/methods/aws.md b/packaging/installer/methods/aws.md index c0b92a036..bd3b091a8 100644 --- a/packaging/installer/methods/aws.md +++ b/packaging/installer/methods/aws.md @@ -11,7 +11,7 @@ learn_rel_path: "Installation/Install on specific environments" Netdata is fully compatible with Amazon Web Services (AWS). You can install Netdata on cloud instances to monitor the apps/services running there, or use -multiple instances in a [parent-child streaming](https://github.com/netdata/netdata/blob/master/streaming/README.md) configuration. +multiple instances in a [parent-child streaming](https://github.com/netdata/netdata/blob/master/src/streaming/README.md) configuration. ## Recommended installation method diff --git a/packaging/installer/methods/azure.md b/packaging/installer/methods/azure.md index 4c39a00ad..0d8e1ed14 100644 --- a/packaging/installer/methods/azure.md +++ b/packaging/installer/methods/azure.md @@ -11,7 +11,7 @@ learn_rel_path: "Installation/Install on specific environments" Netdata is fully compatible with Azure. You can install Netdata on cloud instances to monitor the apps/services running there, or use -multiple instances in a [parent-child streaming](https://github.com/netdata/netdata/blob/master/streaming/README.md) configuration. +multiple instances in a [parent-child streaming](https://github.com/netdata/netdata/blob/master/src/streaming/README.md) configuration. ## Recommended installation method diff --git a/packaging/installer/methods/freebsd.md b/packaging/installer/methods/freebsd.md index 21670cdc9..b34a3b879 100644 --- a/packaging/installer/methods/freebsd.md +++ b/packaging/installer/methods/freebsd.md @@ -33,7 +33,7 @@ The simplest method is to use the single line [kickstart script](https://learn.n If you have a Netdata cloud account then clicking on the **Connect Nodes** button will generate the kickstart command you should use. Use the command from the "Linux" tab, it should look something like this: ```sh -wget -O /tmp/netdata-kickstart.sh https://my-netdata.io/kickstart.sh && sh /tmp/netdata-kickstart.sh --claim-token <CLAIM_TOKEN> --claim-url https://app.netdata.cloud +wget -O /tmp/netdata-kickstart.sh https://get.netdata.cloud/kickstart.sh && sh /tmp/netdata-kickstart.sh --claim-token <CLAIM_TOKEN> --claim-url https://app.netdata.cloud ``` Please respond in the affirmative for any relevant prompts during the installation process. diff --git a/packaging/installer/methods/gcp.md b/packaging/installer/methods/gcp.md index 0b16b1096..873afbcdb 100644 --- a/packaging/installer/methods/gcp.md +++ b/packaging/installer/methods/gcp.md @@ -12,7 +12,7 @@ learn_rel_path: "Installation/Install on specific environments" Netdata is fully compatible with the Google Cloud Platform (GCP). You can install Netdata on cloud instances to monitor the apps/services running there, or use -multiple instances in a [parent-child streaming](https://github.com/netdata/netdata/blob/master/streaming/README.md) configuration. +multiple instances in a [parent-child streaming](https://github.com/netdata/netdata/blob/master/src/streaming/README.md) configuration. ## Recommended installation method diff --git a/packaging/installer/methods/kickstart.md b/packaging/installer/methods/kickstart.md index b21f4dde9..043b3ecc4 100644 --- a/packaging/installer/methods/kickstart.md +++ b/packaging/installer/methods/kickstart.md @@ -1,13 +1,3 @@ -<!-- -title: "Install Netdata with kickstart.sh" -description: "The kickstart.sh script installs Netdata from source, including all dependencies required to connect to Netdata Cloud, with a single command." -custom_edit_url: "https://github.com/netdata/netdata/edit/master/packaging/installer/methods/kickstart.md" -sidebar_label: "One line installer (kickstart.sh)" -learn_status: "Published" -learn_rel_path: "Installation/Installation methods" -sidebar_position: 10 ---> - import { OneLineInstallWget, OneLineInstallCurl } from '@site/src/components/OneLineInstall/' import { Install, InstallBox } from '@site/src/components/Install/' import Tabs from '@theme/Tabs'; @@ -19,28 +9,13 @@ import TabItem from '@theme/TabItem'; `kickstart.sh` is the recommended way of installing Netdata. -This script works on all Linux distributions and macOS environments, by detecting the optimal method of installing Netdata directly to the operating system (it will never install a docker image of Netdata - to run Netdata in a container [check Installing with Docker](https://learn.netdata.cloud/docs/installing/docker)). - -If you are installing on macOS, make sure to check the [install documentation for macOS](https://github.com/netdata/netdata/blob/master/packaging/installer/methods/macos.md) before continuing. - - -## Verify script integrity - -To use `md5sum` to verify the integrity of the `kickstart.sh` script you will download using the one-line command above, -run the following: - -```bash -[ "<checksum-will-be-added-in-documentation-processing>" = "$(curl -Ss https://my-netdata.io/kickstart.sh | md5sum | cut -d ' ' -f 1)" ] && echo "OK, VALID" || echo "FAILED, INVALID" -``` - -If the script is valid, this command will return `OK, VALID`. - +This script works on all Linux distributions and macOS environments, by detecting the optimal method of installing Netdata directly to the operating system. ## Installation > :bulb: Tip > -> If you are unsure whether you want nightly or stable releases, read the [installation guide](https://github.com/netdata/netdata/blob/master/packaging/installer/README.md#nightly-vs-stable-releases). +> If you are unsure whether you want nightly or stable releases, read the [related section](https://github.com/netdata/netdata/blob/master/packaging/installer/README.md#nightly-vs-stable-releases) of our Documentation, detailing the pros and cons of each release type. To install Netdata, run the following as your normal user: @@ -58,10 +33,20 @@ To install Netdata, run the following as your normal user: </Tabs> > :bookmark_tabs: Note -> +> > If you plan to also connect the node to Netdata Cloud, make sure to replace `YOUR_CLAIM_TOKEN` with the claim token of your space, > and `YOUR_ROOM_ID` with the ID of the room you are willing to connect the node to. +## Verify script integrity + +To use `md5sum` to verify the integrity of the `kickstart.sh` script you will download using the one-line command above, +run the following: + +```bash +[ "@KICKSTART_CHECKSUM@" = "$(curl -Ss https://get.netdata.cloud/kickstart.sh | md5sum | cut -d ' ' -f 1)" ] && echo "OK, VALID" || echo "FAILED, INVALID" +``` + +If the script is valid, this command will return `OK, VALID`. ## What does `kickstart.sh` do? @@ -78,6 +63,116 @@ The `kickstart.sh` script does the following after being downloaded and run usin versions, unless you override that with an [optional parameter](#optional-parameters-to-alter-your-installation). - Prints a message whether installation succeeded or failed for QA purposes. +## Start stop or restart the Netdata Agent + +You will most often need to _restart_ the Agent to load new or edited configuration files. + +> **Note** +> Stopping or restarting the Netdata Agent will cause gaps in stored metrics until the `netdata` process initiates collectors and the database engine. +> +> You do not need to restart the Netdata Agent between changes to health configuration files, see the relevant section on [reloading health configuration](https://github.com/netdata/netdata/blob/master/src/health/REFERENCE.md#reload-health-configuration). + +### Using `systemctl` or `service` + +This is the recommended way to start, stop, or restart the Netdata daemon. + +- To **start** Netdata, run `sudo systemctl start netdata`. +- To **stop** Netdata, run `sudo systemctl stop netdata`. +- To **restart** Netdata, run `sudo systemctl restart netdata`. + +If the above commands fail, or you know that you're using a non-systemd system, try using the `service` command: + +- Starting: `sudo service netdata start`. +- Stopping: `sudo service netdata stop`. +- Restarting: `sudo service netdata restart`. + +### Using the `netdata` command + +Use the `netdata` command, typically located at `/usr/sbin/netdata`, to start the Netdata daemon: + +```bash +sudo netdata +``` + +If you start the daemon this way, close it with `sudo killall netdata`. + +### Shutdown using `netdatacli` + +The Netdata Agent also comes with a [CLI tool](https://github.com/netdata/netdata/blob/master/src/cli/README.md) capable of performing shutdowns. Start the Agent back up using your preferred method listed above. + +```bash +sudo netdatacli shutdown-agent +``` + +## Starting Netdata at boot + +In the `system` directory you can find scripts and configurations for the +various distros. + +### systemd + +The installer already installs `netdata.service` if it detects a systemd system. + +To install `netdata.service` by hand, run: + +```sh +# stop Netdata +killall netdata + +# copy netdata.service to systemd +cp system/netdata.service /etc/systemd/system/ + +# let systemd know there is a new service +systemctl daemon-reload + +# enable Netdata at boot +systemctl enable netdata + +# start Netdata +systemctl start netdata +``` + +### init.d + +In the system directory you can find `netdata-lsb`. Copy it to the proper place according to your distribution's documentation. For Ubuntu, this can be done via running the following commands as root. + +```sh +# copy the Netdata startup file to /etc/init.d +cp system/netdata-lsb /etc/init.d/netdata + +# make sure it is executable +chmod +x /etc/init.d/netdata + +# enable it +update-rc.d netdata defaults +``` + +### openrc / Gentoo Linux + +In the `system` directory you can find `netdata-openrc`. Copy it to the proper +place according to your distribution documentation. + +### CentOS / Red Hat Enterprise Linux + +For older versions of RHEL/CentOS that don't have systemd, an init script is included in the system directory. This can be installed by running the following commands as root. + +```sh +# copy the Netdata startup file to /etc/init.d +cp system/netdata-init-d /etc/init.d/netdata + +# make sure it is executable +chmod +x /etc/init.d/netdata + +# enable it +chkconfig --add netdata +``` + +_There have been some recent work on the init script, see the following PR <https://github.com/netdata/netdata/pull/403>_ + +### Other operating systems + +You can start Netdata by running it from `/etc/rc.local` or your system's equivalent. + ## Optional parameters to alter your installation The `kickstart.sh` script accepts a number of optional parameters to control how the installation process works: @@ -147,7 +242,7 @@ By default, the kickstart script will provide a Netdata agent installation that - `--claim-rooms` Specify a comma-separated list of tokens for each War Room this node should appear in. - `--claim-proxy` - Specify a proxy to use when connecting to the cloud in the form of `http://[user:pass@]host:ip` for an HTTP(S) proxy. See [connecting through a proxy](https://github.com/netdata/netdata/blob/master/claim/README.md#connect-through-a-proxy) for details. + Specify a proxy to use when connecting to the cloud in the form of `http://[user:pass@]host:ip` for an HTTP(S) proxy. See [connecting through a proxy](https://github.com/netdata/netdata/blob/master/src/claim/README.md#connect-through-a-proxy) for details. - `--claim-only` If there is an existing install, only try to claim it without attempting to update it. If there is no existing install, install and claim Netdata normally. - `--require-cloud` @@ -177,11 +272,12 @@ By default, the agent is sending anonymous telemetry data to help us take identi Uninstall an existing installation of Netdata. Fails if there is no existing install. ### other options + - `--dry-run` Show what the installer would do, but don’t actually do any of it. - `--dont-start-it` Don’t auto-start the daemon after installing. This parameter is not guaranteed to work. -- `--override-distro` +- `--distro-override` Override the distro detection logic and assume the system is using a specific Linux distribution and release. Takes a single argument consisting of the values of the `ID`, `VERSION_ID`, and `VERSION_CODENAME` fields from `/etc/os-release` for the desired distribution. The following options are mutually exclusive and specify special operations other than trying to install Netdata normally or update an existing install: @@ -204,10 +300,9 @@ should not need to use special values for any of these): those to work, or have a different tool to do the same thing on your system, you can specify it here. - `DISABLE_TELEMETRY`: If set to a value other than 0, behave as if `--disable-telemetry` was specified. - ## Native packages -We publish official DEB/RPM packages for a number of common Linux distributions as part of our releases and nightly +We publish [official DEB/RPM packages](https://github.com/netdata/netdata/blob/master/packaging/installer/methods/packages.md) for a number of common Linux distributions as part of our releases and nightly builds. These packages are available for 64-bit x86 systems. Depending on the distribution and release they may also be available for 32-bit x86, ARMv7, and AArch64 systems. If a native package is available, it will be used as the default installation method. This allows you to handle Netdata updates as part of your usual system update procedure. @@ -217,7 +312,7 @@ you can do so by adding `--native-only` to the options you pass to the installer ## Static builds -We publish pre-built static builds of Netdata for Linux systems. Currently, these are published for 64-bit x86, ARMv7, +We publish pre-built [static builds](https://github.com/netdata/netdata/blob/master/packaging/makeself/README.md) of Netdata for Linux systems. Currently, these are published for 64-bit x86, ARMv7, AArch64, and POWER8+ hardware. These static builds are able to operate in a mostly self-contained manner and only require a POSIX compliant shell and a supported init system. These static builds install under `/opt/netdata`. If you are on a platform which we provide static builds for but do not provide native packages for, a static build diff --git a/packaging/installer/methods/macos.md b/packaging/installer/methods/macos.md index 480a41283..07ba9f989 100644 --- a/packaging/installer/methods/macos.md +++ b/packaging/installer/methods/macos.md @@ -9,8 +9,8 @@ learn_rel_path: "Installation/Install on specific environments" # Install Netdata on macOS Netdata works on macOS, albeit with some limitations. -The number of charts displaying system metrics is limited, but you can use any of Netdata's [external plugins](https://github.com/netdata/netdata/blob/master/collectors/plugins.d/README.md) to monitor any services you might have installed on your macOS system. -You could also use a macOS system as the parent node in a [streaming configuration](https://github.com/netdata/netdata/blob/master/streaming/README.md). +The number of charts displaying system metrics is limited, but you can use any of Netdata's [external plugins](https://github.com/netdata/netdata/blob/master/src/collectors/plugins.d/README.md) to monitor any services you might have installed on your macOS system. +You could also use a macOS system as the parent node in a [streaming configuration](https://github.com/netdata/netdata/blob/master/src/streaming/README.md). You can install Netdata in one of the three following ways: @@ -21,7 +21,7 @@ You can install Netdata in one of the three following ways: Each of these installation option requires [Homebrew](https://brew.sh/) for handling dependencies. > The Netdata Homebrew package is community-created and -maintained. -> Community-maintained packages _may_ receive support from Netdata, but are only a best-effort affair. Learn more about [Netdata's platform support policy](https://github.com/netdata/netdata/blob/master/packaging/PLATFORM_SUPPORT.md). +> Community-maintained packages _may_ receive support from Netdata, but are only a best-effort affair. Learn more about [Netdata's platform support policy](https://github.com/netdata/netdata/blob/master/docs/netdata-agent/versions-and-platforms.md). ## Install Netdata with our automatic one-line installation script @@ -29,13 +29,13 @@ Each of these installation option requires [Homebrew](https://brew.sh/) for hand To install Netdata using our automatic [kickstart](https://github.com/netdata/netdata/blob/master/packaging/installer/README.md#automatic-one-line-installation-script) open a new terminal and run: ```bash -curl https://my-netdata.io/kickstart.sh > /tmp/netdata-kickstart.sh && sh /tmp/netdata-kickstart.sh +curl https://get.netdata.cloud/kickstart.sh > /tmp/netdata-kickstart.sh && sh /tmp/netdata-kickstart.sh ``` The Netdata Agent is installed under `/usr/local/netdata`. Dependencies are handled via Homebrew. **Automatically connect to Netdata Cloud during installation** -The `kickstart.sh` script accepts additional parameters to automatically [connect](https://github.com/netdata/netdata/blob/master/claim/README.md) your node to Netdata +The `kickstart.sh` script accepts additional parameters to automatically [connect](https://github.com/netdata/netdata/blob/master/src/claim/README.md) your node to Netdata Cloud immediately after installation. Find the `token` and `rooms` strings by [signing in to Netdata Cloud](https://app.netdata.cloud/sign-in?cloudRoute=/spaces), then clicking on **Connect Nodes** in the [Spaces management area](https://github.com/netdata/netdata/blob/master/docs/cloud/manage/organize-your-infrastrucutre-invite-your-team.md#netdata-cloud-spaces). @@ -44,16 +44,16 @@ area](https://github.com/netdata/netdata/blob/master/docs/cloud/manage/organize- after the install. - `--claim-rooms`: Specify a comma-separated list of tokens for each War Room this node should appear in. - `--claim-proxy`: Specify a proxy to use when connecting to the cloud in the form of `http://[user:pass@]host:ip` for an HTTP(S) proxy. - See [connecting through a proxy](https://github.com/netdata/netdata/blob/master/claim/README.md#connect-through-a-proxy) for details. + See [connecting through a proxy](https://github.com/netdata/netdata/blob/master/src/claim/README.md#connect-through-a-proxy) for details. - `--claim-url`: Specify a URL to use when connecting to the cloud. Defaults to `https://app.netdata.cloud`. For example: ```bash -curl https://my-netdata.io/kickstart.sh > /tmp/netdata-kickstart.sh && sh /tmp/netdata-kickstart.sh --install-prefix /usr/local/ --claim-token TOKEN --claim-rooms ROOM1,ROOM2 --claim-url https://app.netdata.cloud +curl https://get.netdata.cloud/kickstart.sh > /tmp/netdata-kickstart.sh && sh /tmp/netdata-kickstart.sh --install-prefix /usr/local/ --claim-token TOKEN --claim-rooms ROOM1,ROOM2 --claim-url https://app.netdata.cloud ``` The Netdata Agent is installed under `/usr/local/netdata` on your machine. Your machine will also show up as a node in your Netdata Cloud. -If you experience issues while claiming your node, follow the steps in our [Troubleshooting](https://github.com/netdata/netdata/blob/master/claim/README.md#troubleshooting) documentation. +If you experience issues while claiming your node, follow the steps in our [Troubleshooting](https://github.com/netdata/netdata/blob/master/src/claim/README.md#troubleshooting) documentation. ## Install Netdata via Homebrew ### For macOS Intel @@ -94,7 +94,7 @@ We don't recommend installing Netdata from source on macOS, as it can be difficu ``` 2. Click **Install** on the Software Update popup window that appears. -3. Use the same terminal session to install some of Netdata's prerequisites using Homebrew. If you don't want to use [Netdata Cloud](https://github.com/netdata/netdata/blob/master/docs/quickstart/infrastructure.md), you can omit `cmake`. +3. Use the same terminal session to install some of Netdata's prerequisites using Homebrew. If you don't want to use [Netdata Cloud](https://github.com/netdata/netdata/blob/master/docs/netdata-cloud/README.md), you can omit `cmake`. ```bash brew install ossp-uuid autoconf automake pkg-config libuv lz4 json-c openssl libtool cmake diff --git a/packaging/installer/methods/manual.md b/packaging/installer/methods/manual.md index 269b67c1a..dd4e5a0a0 100644 --- a/packaging/installer/methods/manual.md +++ b/packaging/installer/methods/manual.md @@ -218,9 +218,9 @@ cd netdata ### Connect node to Netdata Cloud during installation Unlike the [`kickstart.sh`](https://github.com/netdata/netdata/blob/master/packaging/installer/methods/kickstart.md), the `netdata-installer.sh` script does -not allow you to automatically [connect](https://github.com/netdata/netdata/blob/master/claim/README.md) your node to Netdata Cloud immediately after installation. +not allow you to automatically [connect](https://github.com/netdata/netdata/blob/master/src/claim/README.md) your node to Netdata Cloud immediately after installation. -See the [connect to cloud](https://github.com/netdata/netdata/blob/master/claim/README.md) doc for details on connecting a node with a manual installation of Netdata. +See the [connect to cloud](https://github.com/netdata/netdata/blob/master/src/claim/README.md) doc for details on connecting a node with a manual installation of Netdata. ### 'nonrepresentable section on output' errors diff --git a/packaging/installer/methods/offline.md b/packaging/installer/methods/offline.md index f2b6cc415..fd69f74c7 100644 --- a/packaging/installer/methods/offline.md +++ b/packaging/installer/methods/offline.md @@ -31,13 +31,13 @@ be as a regular user from any internet connected system that has the following t To prepare the offline installation source, simply run: ```bash -wget -O /tmp/netdata-kickstart.sh https://my-netdata.io/kickstart.sh && sh /tmp/netdata-kickstart.sh --prepare-offline-install-source ./netdata-offline +wget -O /tmp/netdata-kickstart.sh https://get.netdata.cloud/kickstart.sh && sh /tmp/netdata-kickstart.sh --prepare-offline-install-source ./netdata-offline ``` or ```bash -curl https://my-netdata.io/kickstart.sh > /tmp/netdata-kickstart.sh && sh /tmp/netdata-kickstart.sh --prepare-offline-install-source ./netdata-offline +curl https://get.netdata.cloud/kickstart.sh > /tmp/netdata-kickstart.sh && sh /tmp/netdata-kickstart.sh --prepare-offline-install-source ./netdata-offline ``` > The exact name used for the directory does not matter, you can specify any other name you want in place of `./netdata-offline`. diff --git a/packaging/installer/methods/packages.md b/packaging/installer/methods/packages.md index d49e21394..bd0072d21 100644 --- a/packaging/installer/methods/packages.md +++ b/packaging/installer/methods/packages.md @@ -12,7 +12,7 @@ sidebar_position: 20 For most common Linux distributions that use either DEB or RPM packages, Netdata provides pre-built native packages for current releases in-line with -our [official platform support policy](https://github.com/netdata/netdata/blob/master/packaging/PLATFORM_SUPPORT.md). +our [official platform support policy](https://github.com/netdata/netdata/blob/master/docs/netdata-agent/versions-and-platforms.md). These packages will be used by default when attempting to install on a supported platform using our [kickstart.sh installer script](https://github.com/netdata/netdata/blob/master/packaging/installer/methods/kickstart.md). diff --git a/packaging/installer/methods/source.md b/packaging/installer/methods/source.md index 8f34218a2..b70e39850 100644 --- a/packaging/installer/methods/source.md +++ b/packaging/installer/methods/source.md @@ -46,7 +46,7 @@ Additionally, the following build time features require additional dependencies: ## Preparing the source tree Certain features in Netdata require custom versions of specific libraries, -which the the build system will link statically into Netdata. These +which the build system will link statically into Netdata. These libraries and their header files must be copied into specific locations in the source tree to be used. diff --git a/packaging/installer/methods/synology.md b/packaging/installer/methods/synology.md index 3910859b4..10ead7215 100644 --- a/packaging/installer/methods/synology.md +++ b/packaging/installer/methods/synology.md @@ -27,23 +27,22 @@ will install the content into `/opt/netdata`, making future removal safe and sim When Netdata is first installed, it will run as _root_. This may or may not be acceptable for you, and since other installations run it as the `netdata` user, you might wish to do the same. This requires some extra work: -1. Create a group `netdata` via the Synology group interface. Give it no access to anything. -2. Create a user `netdata` via the Synology user interface. Give it no access to anything and a random password. Assign +1. Create a group `netdata` via the Synology group interface. Give it no access to anything. +2. Create a user `netdata` via the Synology user interface. Give it no access to anything and a random password. Assign the user to the `netdata` group. Netdata will chuid to this user when running. -3. Change ownership of the following directories, as defined in - [Netdata Security](https://github.com/netdata/netdata/blob/master/docs/netdata-security.md#security-design): +3. Change ownership of the following directories: -```sh -chown -R root:netdata /opt/netdata/usr/share/netdata -chown -R netdata:netdata /opt/netdata/var/lib/netdata /opt/netdata/var/cache/netdata -chown -R netdata:root /opt/netdata/var/log/netdata -``` + ```sh + chown -R root:netdata /opt/netdata/usr/share/netdata + chown -R netdata:netdata /opt/netdata/var/lib/netdata /opt/netdata/var/cache/netdata + chown -R netdata:root /opt/netdata/var/log/netdata + ``` 4. Restart Netdata -```sh -/etc/rc.netdata restart -``` + ```sh + /etc/rc.netdata restart + ``` ## Create startup script @@ -59,6 +58,6 @@ installed. You'll have to do this manually: [ -x /etc/rc.netdata ] && /etc/rc.netdata start ``` -3. Make sure `/etc/rc.netdata` is executable: `chmod 0755 /etc/rc.netdata`. +3. Make sure `/etc/rc.local` is executable: `chmod 0755 /etc/rc.local`. diff --git a/packaging/installer/methods/systems.md b/packaging/installer/methods/systems.md index e53c4f4a0..acefdeb0a 100644 --- a/packaging/installer/methods/systems.md +++ b/packaging/installer/methods/systems.md @@ -14,5 +14,5 @@ If you have a standard environment that is not yet listed here, just use the [one line installer kickstart.sh](https://github.com/netdata/netdata/blob/master/packaging/installer/methods/kickstart.md) If your environment is somewhat old or unusual, check our -[platform support policy](https://github.com/netdata/netdata/blob/master/packaging/PLATFORM_SUPPORT.md). +[platform support policy](https://github.com/netdata/netdata/blob/master/docs/netdata-agent/versions-and-platforms.md). diff --git a/packaging/installer/netdata-uninstaller.sh b/packaging/installer/netdata-uninstaller.sh index 4326ebe25..1627ca6b5 100755 --- a/packaging/installer/netdata-uninstaller.sh +++ b/packaging/installer/netdata-uninstaller.sh @@ -520,6 +520,15 @@ portable_del_user_from_group() { groupname="${1}" username="${2}" + if command -v getent > /dev/null 2>&1; then + getent group "${1:-""}" | grep -q "${2}" + else + grep "^${1}:" /etc/group | grep -q "${2}" + fi + + ret=$? + [ "${ret}" != "0" ] && return 0 + # username is not in group info "Deleting ${username} user from ${groupname} group ..." @@ -714,6 +723,10 @@ trap quit_msg EXIT info "Stopping a possibly running netdata..." stop_all_netdata +if [ "$(uname -s)" = "Darwin" ]; then + launchctl unload /Library/LaunchDaemons/com.github.netdata.plist 2>/dev/null +fi + #### REMOVE NETDATA FILES rm_file /etc/logrotate.d/netdata rm_file /etc/systemd/system/netdata.service @@ -732,6 +745,7 @@ rm_file /etc/periodic/daily/netdata-updater rm_file /etc/cron.daily/netdata-updater rm_file /etc/cron.d/netdata-updater rm_file /etc/cron.d/netdata-updater-daily +rm_file /Library/LaunchDaemons/com.github.netdata.plist if [ -n "${NETDATA_PREFIX}" ] && [ -d "${NETDATA_PREFIX}" ] && [ "netdata" = "$(basename "$NETDATA_PREFIX")" ] ; then @@ -758,8 +772,10 @@ fi FILE_REMOVAL_STATUS=1 -#### REMOVE NETDATA USER FROM ADDED GROUPS -if [ -n "$NETDATA_ADDED_TO_GROUPS" ]; then +#### REMOVE USER +if user_input "Do you want to delete 'netdata' system user ? "; then + portable_del_user "netdata" || : +elif [ -n "$NETDATA_ADDED_TO_GROUPS" ]; then if user_input "Do you want to delete 'netdata' from following groups: '$NETDATA_ADDED_TO_GROUPS' ? "; then for group in $NETDATA_ADDED_TO_GROUPS; do portable_del_user_from_group "${group}" "netdata" @@ -767,11 +783,6 @@ if [ -n "$NETDATA_ADDED_TO_GROUPS" ]; then fi fi -#### REMOVE USER -if user_input "Do you want to delete 'netdata' system user ? "; then - portable_del_user "netdata" || : -fi - ### REMOVE GROUP if user_input "Do you want to delete 'netdata' system group ? "; then portable_del_group "netdata" || : diff --git a/packaging/installer/netdata-updater.sh b/packaging/installer/netdata-updater.sh index 80faea0a0..220de6e4c 100755 --- a/packaging/installer/netdata-updater.sh +++ b/packaging/installer/netdata-updater.sh @@ -28,7 +28,7 @@ # Author: Pavlos Emm. Katsoulakis <paul@netdata.cloud> # Author: Austin S. Hemmelgarn <austin@netdata.cloud> -# Next unused error code: U001B +# Next unused error code: U001D set -e @@ -36,10 +36,12 @@ PACKAGES_SCRIPT="https://raw.githubusercontent.com/netdata/netdata/master/packag NETDATA_STABLE_BASE_URL="${NETDATA_BASE_URL:-https://github.com/netdata/netdata/releases}" NETDATA_NIGHTLY_BASE_URL="${NETDATA_BASE_URL:-https://github.com/netdata/netdata-nightlies/releases}" +NETDATA_DEFAULT_ACCEPT_MAJOR_VERSIONS="1 2" # Following variables are intended to be overridden by the updater config file. NETDATA_UPDATER_JITTER=3600 NETDATA_NO_SYSTEMD_JOURNAL=0 +NETDATA_ACCEPT_MAJOR_VERSIONS='' script_dir="$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd -P)" @@ -171,6 +173,38 @@ _get_scheduler_type() { fi } +confirm() { + prompt="${1} [y/n]" + + while true; do + echo "${prompt}" + read -r yn + + case "$yn" in + [Yy]*) return 0;; + [Nn]*) return 1;; + *) echo "Please answer yes or no.";; + esac + done +} + +warn_major_update() { + nmv_suffix="New major versions generally involve breaking changes, and may not work in the same way as older versions." + + if [ "${INTERACTIVE}" -eq 0 ]; then + warning "Would update to a new major version of Netdata. ${nmv_suffix}" + warning "To install the new major version anyway, either run the updater interactively, or include the new major version number in the NETDATA_ACCEPT_MAJOR_VERSIONS variable in ${UPDATER_CONFIG_PATH}." + fatal "Aborting update to new major version to avoid breaking things." U001B + else + warning "This update will install a new major version of Netdata. ${nmv_suffix}" + if confirm "Are you sure you want to update to a new major version of Netdata?"; then + notice "User accepted update to new major version of Netdata." + else + fatal "Aborting update to new major version at user request." U001C + fi + fi +} + install_build_dependencies() { bash="$(command -v bash 2> /dev/null)" @@ -366,7 +400,7 @@ _safe_download() { check_for_curl if [ -n "${curl}" ]; then - "${curl}" -sSL --connect-timeout 10 --retry 3 "${url}" > "${dest}" + "${curl}" -fsSL --connect-timeout 10 --retry 3 "${url}" > "${dest}" return $? elif command -v wget > /dev/null 2>&1; then wget -T 15 -O - "${url}" > "${dest}" @@ -394,19 +428,33 @@ download() { get_netdata_latest_tag() { url="${1}/latest" - dest="${2}" check_for_curl if [ -n "${curl}" ]; then - tag=$("${curl}" "${url}" -s -L -I -o /dev/null -w '%{url_effective}' | grep -m 1 -o '[^/]*$') + tag=$("${curl}" "${url}" -s -L -I -o /dev/null -w '%{url_effective}' | grep -Eom 1 '[^/]*/?$') elif command -v wget >/dev/null 2>&1; then - tag=$(wget -S -O /dev/null "${url}" 2>&1 | grep -m 1 Location | grep -o '[^/]*$') + tag=$(wget -S -O /dev/null "${url}" 2>&1 | grep -m 1 Location | grep -Eo '[^/]*/?$') else fatal "I need curl or wget to proceed, but neither of them are available on this system." U0006 fi - echo "${tag}" >"${dest}" + # Fallback case for simpler local testing. + if echo "${tag}" | grep -Eq 'latest/?$'; then + if _safe_download "${url}/latest-version.txt" ./ndupdate-version.txt; then + tag="$(cat ./ndupdate-version.txt)" + + if grep -q 'Not Found' ./ndupdate-version.txt; then + tag="latest" + fi + + rm -f ./ndupdate-version.txt + else + tag="latest" + fi + fi + + echo "${tag}" } newer_commit_date() { @@ -494,9 +542,9 @@ parse_version() { get_latest_version() { if [ "${RELEASE_CHANNEL}" = "stable" ]; then - get_netdata_latest_tag "${NETDATA_STABLE_BASE_URL}" /dev/stdout + get_netdata_latest_tag "${NETDATA_STABLE_BASE_URL}" else - get_netdata_latest_tag "${NETDATA_NIGHTLY_BASE_URL}" /dev/stdout + get_netdata_latest_tag "${NETDATA_NIGHTLY_BASE_URL}" fi } @@ -513,6 +561,7 @@ update_available() { info "Force update requested" return 0 fi + basepath="$(dirname "$(dirname "$(dirname "${NETDATA_LIB_DIR}")")")" searchpath="${basepath}/bin:${basepath}/sbin:${basepath}/usr/bin:${basepath}/usr/sbin:${PATH}" searchpath="${basepath}/netdata/bin:${basepath}/netdata/sbin:${basepath}/netdata/usr/bin:${basepath}/netdata/usr/sbin:${searchpath}" @@ -542,6 +591,27 @@ update_available() { return 1 else info "Update available" + + if [ "${current_version}" -ne 0 ] && [ "${latest_version}" -ne 0 ]; then + current_major="$(${ndbinary} -v | cut -f 2 -d ' ' | cut -f 1 -d '.' | tr -d 'v')" + latest_major="$(echo "${latest_tag}" | cut -f 1 -d '.' | tr -d 'v')" + + if [ "${current_major}" -ne "${latest_major}" ]; then + update_safe=0 + + for v in ${NETDATA_ACCEPT_MAJOR_VERSIONS}; do + if [ "${latest_major}" -eq "${v}" ]; then + update_safe=1 + break + fi + done + + if [ "${update_safe}" -eq 0 ]; then + warn_major_update + fi + fi + fi + return 0 fi } @@ -553,6 +623,7 @@ set_tarball_urls() { if [ -e /opt/netdata/etc/netdata/.install-type ]; then # shellcheck disable=SC1091 . /opt/netdata/etc/netdata/.install-type + [ -z "${PREBUILT_ARCH:-}" ] && PREBUILT_ARCH="$(uname -m)" filename="netdata-${PREBUILT_ARCH}-latest.gz.run" else filename="netdata-x86_64-latest.gz.run" @@ -560,11 +631,11 @@ set_tarball_urls() { fi if [ "$1" = "stable" ]; then - latest="$(get_netdata_latest_tag "${NETDATA_STABLE_BASE_URL}" /dev/stdout)" + latest="$(get_netdata_latest_tag "${NETDATA_STABLE_BASE_URL}")" export NETDATA_TARBALL_URL="${NETDATA_STABLE_BASE_URL}/download/$latest/${filename}" export NETDATA_TARBALL_CHECKSUM_URL="${NETDATA_STABLE_BASE_URL}/download/$latest/sha256sums.txt" else - tag="$(get_netdata_latest_tag "${NETDATA_NIGHTLY_BASE_URL}" /dev/stdout)" + tag="$(get_netdata_latest_tag "${NETDATA_NIGHTLY_BASE_URL}")" export NETDATA_TARBALL_URL="${NETDATA_NIGHTLY_BASE_URL}/download/${tag}/${filename}" export NETDATA_TARBALL_CHECKSUM_URL="${NETDATA_NIGHTLY_BASE_URL}/download/${tag}/sha256sums.txt" fi @@ -713,6 +784,15 @@ update_static() { exit 0 } +get_new_binpkg_major() { + case "${pm_cmd}" in + apt-get) apt-get --just-print upgrade 2>&1 | grep Inst | grep ' netdata ' | cut -f 3 -d ' ' | tr -d '[]' | cut -f 1 -d '.' ;; + yum) yum check-update netdata | grep -E '^netdata ' | awk '{print $2}' | cut -f 1 -d '.' ;; + dnf) dnf check-update netdata | grep -E '^netdata ' | awk '{print $2}' | cut -f 1 -d '.' ;; + zypper) zypper list-updates | grep '| netdata |' | cut -f 5 -d '|' | tr -d ' ' | cut -f 1 -d '.' ;; + esac +} + update_binpkg() { os_release_file= if [ -s "/etc/os-release" ] && [ -r "/etc/os-release" ]; then @@ -823,6 +903,24 @@ update_binpkg() { fi done + current_major="$(netdata -v | cut -f 2 -d ' ' | cut -f 1 -d '.' | tr -d 'v')" + latest_major="$(get_new_binpkg_major)" + + if [ -n "${latest_major}" ] && [ "${latest_major}" -ne "${current_major}" ]; then + update_safe=0 + + for v in ${NETDATA_ACCEPT_MAJOR_VERSIONS}; do + if [ "${latest_major}" -eq "${v}" ]; then + update_safe=1 + break + fi + done + + if [ "${update_safe}" -eq 0 ]; then + warn_major_update + fi + fi + # shellcheck disable=SC2086 env ${env} ${pm_cmd} ${upgrade_subcmd} ${pkg_install_opts} netdata >&3 2>&3 || fatal "Failed to update Netdata package." U000F @@ -901,11 +999,14 @@ if [ -r "$(dirname "${ENVIRONMENT_FILE}")/.install-type" ]; then . "$(dirname "${ENVIRONMENT_FILE}")/.install-type" || fatal "Failed to source $(dirname "${ENVIRONMENT_FILE}")/.install-type" U0015 fi -if [ -r "$(dirname "${ENVIRONMENT_FILE}")/netdata-updater.conf" ]; then +UPDATER_CONFIG_PATH="$(dirname "${ENVIRONMENT_FILE}")/netdata-updater.conf" +if [ -r "${UPDATER_CONFIG_PATH}" ]; then # shellcheck source=/dev/null - . "$(dirname "${ENVIRONMENT_FILE}")/netdata-updater.conf" + . "${UPDATER_CONFIG_PATH}" fi +[ -z "${NETDATA_ACCEPT_MAJOR_VERSIONS}" ] && NETDATA_ACCEPT_MAJOR_VERSIONS="${NETDATA_DEFAULT_ACCEPT_MAJOR_VERSIONS}" + while [ -n "${1}" ]; do case "${1}" in --not-running-from-cron) NETDATA_NOT_RUNNING_FROM_CRON=1 ;; diff --git a/packaging/makeself/build-static.sh b/packaging/makeself/build-static.sh index 0c46c12af..260581ed1 100755 --- a/packaging/makeself/build-static.sh +++ b/packaging/makeself/build-static.sh @@ -26,8 +26,13 @@ fi DOCKER_IMAGE_NAME="netdata/static-builder:v1" -if [ "${BUILDARCH}" != "$(uname -m)" ] && [ "$(uname -m)" = 'x86_64' ] && [ -z "${SKIP_EMULATION}" ]; then - ${docker} run --rm --privileged multiarch/qemu-user-static --reset -p yes || exit 1 +if [ "${BUILDARCH}" != "$(uname -m)" ] && [ -z "${SKIP_EMULATION}" ]; then + if [ "$(uname -m)" = "x86_64" ]; then + ${docker} run --rm --privileged multiarch/qemu-user-static --reset -p yes || exit 1 + else + echo "Automatic cross-architecture builds are only supported on x86_64 hosts." + exit 1 + fi fi if ${docker} inspect "${DOCKER_IMAGE_NAME}" > /dev/null 2>&1; then @@ -49,10 +54,10 @@ fi # Run the build script inside the container if [ -t 1 ]; then run ${docker} run --rm -e BUILDARCH="${BUILDARCH}" -a stdin -a stdout -a stderr -i -t -v "$(pwd)":/netdata:rw \ - "${DOCKER_IMAGE_NAME}" \ + --platform "${platform}" "${DOCKER_IMAGE_NAME}" \ /bin/sh /netdata/packaging/makeself/build.sh "${@}" else run ${docker} run --rm -e BUILDARCH="${BUILDARCH}" -v "$(pwd)":/netdata:rw \ - -e GITHUB_ACTIONS="${GITHUB_ACTIONS}" "${DOCKER_IMAGE_NAME}" \ + -e GITHUB_ACTIONS="${GITHUB_ACTIONS}" --platform "${platform}" "${DOCKER_IMAGE_NAME}" \ /bin/sh /netdata/packaging/makeself/build.sh "${@}" fi diff --git a/packaging/makeself/bundled-packages b/packaging/makeself/bundled-packages.version index 02ee4469d..02ee4469d 100644 --- a/packaging/makeself/bundled-packages +++ b/packaging/makeself/bundled-packages.version diff --git a/packaging/makeself/install-or-update.sh b/packaging/makeself/install-or-update.sh index e4c133459..964d2aa5d 100755 --- a/packaging/makeself/install-or-update.sh +++ b/packaging/makeself/install-or-update.sh @@ -172,7 +172,7 @@ fi progress "changing plugins ownership and permissions" -for x in apps.plugin perf.plugin slabinfo.plugin debugfs.plugin freeipmi.plugin ioping cgroup-network local-listeners ebpf.plugin nfacct.plugin xenstat.plugin python.d.plugin charts.d.plugin go.d.plugin ioping.plugin cgroup-network-helper.sh; do +for x in ndsudo apps.plugin perf.plugin slabinfo.plugin debugfs.plugin freeipmi.plugin ioping cgroup-network local-listeners network-viewer.plugin ebpf.plugin nfacct.plugin xenstat.plugin python.d.plugin charts.d.plugin go.d.plugin ioping.plugin cgroup-network-helper.sh; do f="usr/libexec/netdata/plugins.d/${x}" if [ -f "${f}" ]; then run chown root:${NETDATA_GROUP} "${f}" @@ -190,15 +190,15 @@ if command -v setcap >/dev/null 2>&1; then run setcap "cap_sys_admin=ep" "usr/libexec/netdata/plugins.d/perf.plugin" fi - run setcap "cap_net_admin,cap_net_raw=eip" "usr/libexec/netdata/plugins.d/go.d.plugin" + run setcap "cap_dac_read_search+epi cap_net_admin+epi cap_net_raw=eip" "usr/libexec/netdata/plugins.d/go.d.plugin" else - for x in apps.plugin perf.plugin slabinfo.plugin debugfs.plugin; do + for x in ndsudo apps.plugin perf.plugin slabinfo.plugin debugfs.plugin; do f="usr/libexec/netdata/plugins.d/${x}" run chmod 4750 "${f}" done fi -for x in freeipmi.plugin ioping cgroup-network local-listeners ebpf.plugin nfacct.plugin xenstat.plugin; do +for x in freeipmi.plugin ioping cgroup-network local-listeners network-viewer.plugin ebpf.plugin nfacct.plugin xenstat.plugin; do f="usr/libexec/netdata/plugins.d/${x}" if [ -f "${f}" ]; then diff --git a/packaging/makeself/jobs/20-openssl.install.sh b/packaging/makeself/jobs/20-openssl.install.sh index 1158a6330..a07f9c947 100755 --- a/packaging/makeself/jobs/20-openssl.install.sh +++ b/packaging/makeself/jobs/20-openssl.install.sh @@ -4,7 +4,7 @@ # shellcheck source=packaging/makeself/functions.sh . "$(dirname "${0}")/../functions.sh" "${@}" || exit 1 # Source of truth for all the packages we bundle in static builds -. "$(dirname "${0}")/../bundled-packages" +. "$(dirname "${0}")/../bundled-packages.version" # shellcheck disable=SC2015 [ "${GITHUB_ACTIONS}" = "true" ] && echo "::group::Building OpenSSL" || true diff --git a/packaging/makeself/jobs/50-bash-5.1.16.install.sh b/packaging/makeself/jobs/50-bash-5.1.16.install.sh index 7a302f2ee..cc74d0fc8 100755 --- a/packaging/makeself/jobs/50-bash-5.1.16.install.sh +++ b/packaging/makeself/jobs/50-bash-5.1.16.install.sh @@ -4,7 +4,7 @@ # shellcheck source=packaging/makeself/functions.sh . "$(dirname "${0}")/../functions.sh" "${@}" || exit 1 # Source of truth for all the packages we bundle in static builds -. "$(dirname "${0}")/../bundled-packages" +. "$(dirname "${0}")/../bundled-packages.version" # shellcheck disable=SC2015 [ "${GITHUB_ACTIONS}" = "true" ] && echo "::group::building bash" || true diff --git a/packaging/makeself/jobs/50-curl.install.sh b/packaging/makeself/jobs/50-curl.install.sh index 824b30562..54f55480f 100755 --- a/packaging/makeself/jobs/50-curl.install.sh +++ b/packaging/makeself/jobs/50-curl.install.sh @@ -4,7 +4,7 @@ # shellcheck source=packaging/makeself/functions.sh . "$(dirname "${0}")/../functions.sh" "${@}" || exit 1 # Source of truth for all the packages we bundle in static builds -. "$(dirname "${0}")/../bundled-packages" +. "$(dirname "${0}")/../bundled-packages.version" # shellcheck disable=SC2015 [ "${GITHUB_ACTIONS}" = "true" ] && echo "::group::Building cURL" || true diff --git a/packaging/makeself/jobs/50-ioping-1.3.install.sh b/packaging/makeself/jobs/50-ioping-1.3.install.sh index 6bd538e35..de6cb3241 100755 --- a/packaging/makeself/jobs/50-ioping-1.3.install.sh +++ b/packaging/makeself/jobs/50-ioping-1.3.install.sh @@ -4,7 +4,7 @@ # shellcheck source=packaging/makeself/functions.sh . "$(dirname "${0}")/../functions.sh" "${@}" || exit 1 # Source of truth for all the packages we bundle in static builds -. "$(dirname "${0}")/../bundled-packages" || exit 1 +. "$(dirname "${0}")/../bundled-packages.version" || exit 1 # shellcheck disable=SC2015 [ "${GITHUB_ACTIONS}" = "true" ] && echo "::group::Building ioping" || true diff --git a/packaging/makeself/jobs/50-libnetfilter_acct-1.0.3.install.sh b/packaging/makeself/jobs/50-libnetfilter_acct-1.0.3.install.sh index 829752178..efde6976f 100755 --- a/packaging/makeself/jobs/50-libnetfilter_acct-1.0.3.install.sh +++ b/packaging/makeself/jobs/50-libnetfilter_acct-1.0.3.install.sh @@ -7,7 +7,7 @@ # shellcheck source=packaging/makeself/functions.sh . "$(dirname "${0}")/../functions.sh" "${@}" || exit 1 # Source of truth for all the packages we bundle in static builds -. "$(dirname "${0}")/../bundled-packages" || exit 1 +. "$(dirname "${0}")/../bundled-packages.version" || exit 1 # shellcheck disable=SC2015 [ "${GITHUB_ACTIONS}" = "true" ] && echo "::group::building libnetfilter_acct" || true diff --git a/packaging/makeself/jobs/70-netdata-git.install.sh b/packaging/makeself/jobs/70-netdata-git.install.sh index 83d28bf87..0373599a9 100755 --- a/packaging/makeself/jobs/70-netdata-git.install.sh +++ b/packaging/makeself/jobs/70-netdata-git.install.sh @@ -36,6 +36,7 @@ run ./netdata-installer.sh \ --use-system-protobuf \ --dont-scrub-cflags-even-though-it-may-break-things \ --one-time-build \ + --disable-logsmanagement \ --enable-lto # shellcheck disable=SC2015 diff --git a/packaging/makeself/jobs/90-netdata-runtime-check.sh b/packaging/makeself/jobs/90-netdata-runtime-check.sh index a3c94ffcb..fc1b239b3 100755 --- a/packaging/makeself/jobs/90-netdata-runtime-check.sh +++ b/packaging/makeself/jobs/90-netdata-runtime-check.sh @@ -8,47 +8,10 @@ dump_log() { cat ./netdata.log } -wait_for() { - host="${1}" - port="${2}" - name="${3}" - timeout="30" - - if command -v nc > /dev/null ; then - netcat="nc" - elif command -v netcat > /dev/null ; then - netcat="netcat" - else - printf "Unable to find a usable netcat command.\n" - return 1 - fi - - printf "Waiting for %s on %s:%s ... " "${name}" "${host}" "${port}" - - sleep 30 - - i=0 - while ! ${netcat} -z "${host}" "${port}"; do - sleep 1 - if [ "$i" -gt "$timeout" ]; then - printf "Timed out!\n" - return 1 - fi - i="$((i + 1))" - done - printf "OK\n" -} - trap dump_log EXIT "${NETDATA_INSTALL_PATH}/bin/netdata" -D > ./netdata.log 2>&1 & -wait_for localhost 19999 netdata || exit 1 - -curl -sS http://127.0.0.1:19999/api/v1/info > ./response || exit 1 - -cat ./response - -jq '.version' ./response || exit 1 +"${NETDATA_SOURCE_PATH}/packaging/runtime-check.sh" || exit 1 trap - EXIT diff --git a/packaging/makeself/uname2platform.sh b/packaging/makeself/uname2platform.sh index 7eab706ec..34d76ff9f 100755 --- a/packaging/makeself/uname2platform.sh +++ b/packaging/makeself/uname2platform.sh @@ -8,6 +8,7 @@ BUILDARCH="${1}" case "${BUILDARCH}" in x86_64) echo "linux/amd64" ;; + armv6l) echo "linux/arm/v6" ;; armv7l) echo "linux/arm/v7" ;; aarch64) echo "linux/arm64/v8" ;; ppc64le) echo "linux/ppc64le" ;; diff --git a/packaging/protobuf.checksums b/packaging/protobuf.checksums deleted file mode 100644 index 4a025c5fb..000000000 --- a/packaging/protobuf.checksums +++ /dev/null @@ -1 +0,0 @@ -89ac31a93832e204db6d73b1e80f39f142d5747b290f17340adce5be5b122f94 protobuf-cpp-3.19.4.tar.gz diff --git a/packaging/protobuf.version b/packaging/protobuf.version deleted file mode 100644 index de24deecf..000000000 --- a/packaging/protobuf.version +++ /dev/null @@ -1 +0,0 @@ -3.19.4 diff --git a/packaging/repoconfig/netdata-repo.spec b/packaging/repoconfig/netdata-repo.spec index 242178ba7..3c99b921f 100644 --- a/packaging/repoconfig/netdata-repo.spec +++ b/packaging/repoconfig/netdata-repo.spec @@ -49,21 +49,20 @@ install -pm 644 %{SOURCE3} ./netdata-edge.repo %endif %if 0%{?centos_ver} -# Amazon Linux 2 looks like CentOS, but with extra macros. -%if 0%{?amzn2} -install -pm 644 %{SOURCE8} ./netdata.repo -install -pm 644 %{SOURCE9} ./netdata-edge.repo -%else install -pm 644 %{SOURCE4} ./netdata.repo install -pm 644 %{SOURCE5} ./netdata-edge.repo %endif -%endif %if 0%{?oraclelinux} install -pm 644 %{SOURCE6} ./netdata.repo install -pm 644 %{SOURCE7} ./netdata-edge.repo %endif +%if 0%{?amzn} +install -pm 644 %{SOURCE8} ./netdata.repo +install -pm 644 %{SOURCE9} ./netdata-edge.repo +%endif + %build true diff --git a/packaging/runtime-check.sh b/packaging/runtime-check.sh new file mode 100755 index 000000000..de6e22071 --- /dev/null +++ b/packaging/runtime-check.sh @@ -0,0 +1,50 @@ +#!/bin/sh + +wait_for() { + host="${1}" + port="${2}" + name="${3}" + timeout="30" + + if command -v nc > /dev/null ; then + netcat="nc" + elif command -v netcat > /dev/null ; then + netcat="netcat" + else + printf "Unable to find a usable netcat command.\n" + return 1 + fi + + printf "Waiting for %s on %s:%s ... " "${name}" "${host}" "${port}" + + sleep 30 + + i=0 + while ! ${netcat} -z "${host}" "${port}"; do + sleep 1 + if [ "$i" -gt "$timeout" ]; then + printf "Timed out!\n" + return 2 + fi + i="$((i + 1))" + done + printf "OK\n" +} + +wait_for localhost 19999 netdata + +case $? in + 1) exit 2 ;; + 2) exit 3 ;; +esac + +curl -sfS http://127.0.0.1:19999/api/v1/info > ./response || exit 1 + +cat ./response + +jq '.version' ./response || exit 1 + +curl -sfS http://127.0.0.1:19999/index.html || exit 1 +curl -sfS http://127.0.0.1:19999/v0/index.html || exit 1 +curl -sfS http://127.0.0.1:19999/v1/index.html || exit 1 +curl -sfS http://127.0.0.1:19999/v2/index.html || exit 1 diff --git a/coverity-scan.sh b/packaging/utils/coverity-scan.sh index 8466b21a7..b69fea561 100755 --- a/coverity-scan.sh +++ b/packaging/utils/coverity-scan.sh @@ -42,11 +42,31 @@ INSTALL_DIR="/opt" # the version of coverity to use COVERITY_BUILD_VERSION="${COVERITY_BUILD_VERSION:-cov-analysis-linux64-2023.6.2}" -# TODO: For some reasons this does not fully load on Debian 10 (Haven't checked if it happens on other distros yet), it breaks -source packaging/installer/functions.sh || echo "Failed to fully load the functions library" +SCRIPT_SOURCE="$( + self=${0} + while [ -L "${self}" ] + do + cd "${self%/*}" || exit 1 + self=$(readlink "${self}") + done + cd "${self%/*}" || exit 1 + echo "$(pwd -P)/${self##*/}" +)" +REPO_ROOT="$(dirname "${SCRIPT_SOURCE}")/../.." + +. "${REPO_ROOT}/packaging/installer/functions.sh" + +JOBS=$(find_processors) +[ -z "${JOBS}" ] && JOBS=1 + +if command -v ninja 2>&1; then + ninja="$(command -v ninja)" +fi -cpus=$(find_processors) -[ -z "${cpus}" ] && cpus=1 +CMAKE_OPTS="${ninja:+-G Ninja}" +BUILD_OPTS="VERBOSE=1" +[ -n "${ninja}" ] && BUILD_OPTS="-v" +NETDATA_BUILD_DIR="${NETDATA_BUILD_DIR:-./build/}" if [ -f ".coverity-scan.conf" ]; then source ".coverity-scan.conf" @@ -98,23 +118,27 @@ scanit() { fatal "The command '${covbuild}' is not executable. Export variable COVERITY_BUILD_PATH or set it in .coverity-scan.conf" fi + cd "${REPO_ROOT}" || exit 1 + version="$(grep "^#define PACKAGE_VERSION" config.h | cut -d '"' -f 2)" progress "Working on netdata version: ${version}" progress "Cleaning up old builds..." - run make clean || echo >&2 "Nothing to clean" + rm -rf "${NETDATA_BUILD_DIR}" [ -d "cov-int" ] && rm -rf "cov-int" [ -f netdata-coverity-analysis.tgz ] && run rm netdata-coverity-analysis.tgz progress "Configuring netdata source..." + USE_SYSTEM_PROTOBUF=1 + ENABLE_GO=0 + prepare_cmake_options - run autoreconf -ivf - run ./configure ${OTHER_OPTIONS} + run cmake ${NETDATA_CMAKE_OPTIONS} progress "Analyzing netdata..." - run "${covbuild}" --dir cov-int make -j${cpus} + run "${covbuild}" --dir cov-int cmake --build "${NETDATA_BUILD_DIR}" --parallel ${JOBS} -- ${BUILD_OPTS} echo >&2 "Compressing analysis..." run tar czvf netdata-coverity-analysis.tgz cov-int @@ -170,21 +194,6 @@ installit() { return 0 } -OTHER_OPTIONS="--disable-lto" -OTHER_OPTIONS+=" --with-zlib" -OTHER_OPTIONS+=" --with-math" -OTHER_OPTIONS+=" --enable-lz4" -OTHER_OPTIONS+=" --enable-openssl" -OTHER_OPTIONS+=" --enable-jsonc" -OTHER_OPTIONS+=" --enable-plugin-nfacct" -OTHER_OPTIONS+=" --enable-plugin-freeipmi" -OTHER_OPTIONS+=" --enable-plugin-cups" -OTHER_OPTIONS+=" --enable-exporting-prometheus-remote-write" -# TODO: enable these plugins too -#OTHER_OPTIONS+=" --enable-plugin-xenstat" -#OTHER_OPTIONS+=" --enable-exporting-kinesis" -#OTHER_OPTIONS+=" --enable-exporting-mongodb" - FOUND_OPTS="NO" while [ -n "${1}" ]; do if [ "${1}" = "--with-install" ]; then diff --git a/packaging/version b/packaging/version index 0ade2bb9e..7b8b065c0 100644 --- a/packaging/version +++ b/packaging/version @@ -1 +1 @@ -v1.44.3 +v1.45.3 diff --git a/packaging/yaml.checksums b/packaging/yaml.checksums deleted file mode 100644 index 563c273d4..000000000 --- a/packaging/yaml.checksums +++ /dev/null @@ -1 +0,0 @@ -c642ae9b75fee120b2d96c712538bd2cf283228d2337df2cf2988e3c02678ef4 yaml-0.2.5.tar.gz diff --git a/packaging/yaml.version b/packaging/yaml.version deleted file mode 100644 index 3a4036fb4..000000000 --- a/packaging/yaml.version +++ /dev/null @@ -1 +0,0 @@ -0.2.5 |