From b5f8ee61a7f7e9bd291dd26b0585d03eb686c941 Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Sun, 5 May 2024 13:19:16 +0200 Subject: Adding upstream version 1.46.3. Signed-off-by: Daniel Baumann --- packaging/dag/README.md | 23 + packaging/dag/build_command.py | 65 ++ packaging/dag/files/child_stream.conf | 10 + packaging/dag/files/cmake-aarch64.sha256 | 1 + packaging/dag/files/cmake-x86_64.sha256 | 1 + packaging/dag/files/ol8-epel.repo | 6 + packaging/dag/files/ol9-epel.repo | 6 + packaging/dag/files/parent_stream.conf | 7 + packaging/dag/imageutils.py | 1580 ++++++++++++++++++++++++++++++ packaging/dag/main.py | 18 + packaging/dag/nd.py | 409 ++++++++ packaging/dag/requirements.txt | 3 + packaging/dag/test_command.py | 128 +++ 13 files changed, 2257 insertions(+) create mode 100644 packaging/dag/README.md create mode 100644 packaging/dag/build_command.py create mode 100644 packaging/dag/files/child_stream.conf create mode 100644 packaging/dag/files/cmake-aarch64.sha256 create mode 100644 packaging/dag/files/cmake-x86_64.sha256 create mode 100644 packaging/dag/files/ol8-epel.repo create mode 100644 packaging/dag/files/ol9-epel.repo create mode 100644 packaging/dag/files/parent_stream.conf create mode 100644 packaging/dag/imageutils.py create mode 100755 packaging/dag/main.py create mode 100644 packaging/dag/nd.py create mode 100644 packaging/dag/requirements.txt create mode 100644 packaging/dag/test_command.py (limited to 'packaging/dag') diff --git a/packaging/dag/README.md b/packaging/dag/README.md new file mode 100644 index 000000000..4b17c5447 --- /dev/null +++ b/packaging/dag/README.md @@ -0,0 +1,23 @@ +- Install Dagger CLI: + ``` + cd /usr/local + curl -L https://dl.dagger.io/dagger/install.sh | sudo sh + ``` +- Install python requirements: + ``` + pip install -r packaging/dag/requirements.txt + ``` + +Now you can run something like this: + +``` +dagger run python packaging/dag/main.py build -p linux/x86_64 -d debian12 +``` + +or + +``` +dagger run python packaging/dag/main.py test +```. + + diff --git a/packaging/dag/build_command.py b/packaging/dag/build_command.py new file mode 100644 index 000000000..dcb1b6c8a --- /dev/null +++ b/packaging/dag/build_command.py @@ -0,0 +1,65 @@ +import click +import asyncio +import sys +import dagger +import pathlib +import uuid + +from nd import ( + Distribution, + NetdataInstaller, + FeatureFlags, + Endpoint, + AgentContext, + SUPPORTED_PLATFORMS, + SUPPORTED_DISTRIBUTIONS, +) + + +def run_async(func): + def wrapper(*args, **kwargs): + return asyncio.run(func(*args, **kwargs)) + + return wrapper + + +@run_async +async def simple_build(platform, distro): + config = dagger.Config(log_output=sys.stdout) + + async with dagger.Connection(config) as client: + repo_root = pathlib.Path("/netdata") + prefix_path = pathlib.Path("/opt/netdata") + + installer = NetdataInstaller( + platform, distro, repo_root, prefix_path, FeatureFlags.DBEngine + ) + + endpoint = Endpoint("node", 19999) + api_key = uuid.uuid4() + allow_children = False + + agent_ctx = AgentContext( + client, platform, distro, installer, endpoint, api_key, allow_children + ) + + await agent_ctx.build_container() + + +@click.command() +@click.option( + "--platform", + "-p", + type=click.Choice(sorted([str(p) for p in SUPPORTED_PLATFORMS])), + help="Specify the platform.", +) +@click.option( + "--distribution", + "-d", + type=click.Choice(sorted([str(p) for p in SUPPORTED_DISTRIBUTIONS])), + help="Specify the distribution.", +) +def build(platform, distribution): + platform = dagger.Platform(platform) + distro = Distribution(distribution) + simple_build(platform, distro) diff --git a/packaging/dag/files/child_stream.conf b/packaging/dag/files/child_stream.conf new file mode 100644 index 000000000..ed78bd3fb --- /dev/null +++ b/packaging/dag/files/child_stream.conf @@ -0,0 +1,10 @@ +[stream] + enabled = {{ enabled }} + destination = {{ destination }} + api key = {{ api_key }} + timeout seconds = {{ timeout_seconds }} + default port = {{ default_port }} + send charts matching = {{ send_charts_matching }} + buffer size bytes = {{ buffer_size_bytes }} + reconnect delay seconds = {{ reconnect_delay_seconds }} + initial clock resync iterations = {{ initial_clock_resync_iterations }} diff --git a/packaging/dag/files/cmake-aarch64.sha256 b/packaging/dag/files/cmake-aarch64.sha256 new file mode 100644 index 000000000..122b26e99 --- /dev/null +++ b/packaging/dag/files/cmake-aarch64.sha256 @@ -0,0 +1 @@ +a83e01ed1cdf44c2e33e0726513b9a35a8c09e3b5a126fd720b3c8a9d5552368 cmake-aarch64.sh diff --git a/packaging/dag/files/cmake-x86_64.sha256 b/packaging/dag/files/cmake-x86_64.sha256 new file mode 100644 index 000000000..d5adc8aa1 --- /dev/null +++ b/packaging/dag/files/cmake-x86_64.sha256 @@ -0,0 +1 @@ +8c449dabb2b2563ec4e6d5e0fb0ae09e729680efab71527b59015131cea4a042 cmake-x86_64.sh diff --git a/packaging/dag/files/ol8-epel.repo b/packaging/dag/files/ol8-epel.repo new file mode 100644 index 000000000..587cc3577 --- /dev/null +++ b/packaging/dag/files/ol8-epel.repo @@ -0,0 +1,6 @@ +[ol8_developer_EPEL] +name=Oracle Linux $releasever EPEL Packages for Development ($basearch) +baseurl=https://yum$ociregion.$ocidomain/repo/OracleLinux/OL8/developer/EPEL/$basearch/ +gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-oracle +gpgcheck=1 +enabled=1 diff --git a/packaging/dag/files/ol9-epel.repo b/packaging/dag/files/ol9-epel.repo new file mode 100644 index 000000000..c40007f1f --- /dev/null +++ b/packaging/dag/files/ol9-epel.repo @@ -0,0 +1,6 @@ +[ol9_developer_EPEL] +name=Oracle Linux $releasever EPEL Packages for Development ($basearch) +baseurl=https://yum$ociregion.$ocidomain/repo/OracleLinux/OL9/developer/EPEL/$basearch/ +gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-oracle +gpgcheck=1 +enabled=1 diff --git a/packaging/dag/files/parent_stream.conf b/packaging/dag/files/parent_stream.conf new file mode 100644 index 000000000..15f303f97 --- /dev/null +++ b/packaging/dag/files/parent_stream.conf @@ -0,0 +1,7 @@ +[{{ api_key }}] + enabled = {{ enabled }} + allow from = {{ allow_from }} + default history = {{ default_history }} + health enabled by default = {{ health_enabled_by_default }} + default postpone alarms on connect seconds = {{ default_postpone_alarms_on_connect_seconds }} + multiple connections = {{ multiple_connections }} diff --git a/packaging/dag/imageutils.py b/packaging/dag/imageutils.py new file mode 100644 index 000000000..fd1e8ad26 --- /dev/null +++ b/packaging/dag/imageutils.py @@ -0,0 +1,1580 @@ +import os +from pathlib import Path + +import dagger + + +_ALPINE_COMMON_PACKAGES = [ + "alpine-sdk", + "autoconf", + "automake", + "bash", + "binutils", + "bison", + "cmake", + "curl", + "curl-static", + "elfutils-dev", + "flex", + "gcc", + "git", + "gnutls-dev", + "gzip", + "jq", + "libelf-static", + "libmnl-dev", + "libmnl-static", + "libtool", + "libuv-dev", + "libuv-static", + "lz4-dev", + "lz4-static", + "make", + "mongo-c-driver-dev", + "mongo-c-driver-static", + "musl-fts-dev", + "ncurses", + "ncurses-static", + "netcat-openbsd", + "ninja", + "openssh", + "pcre2-dev", + "pkgconfig", + "protobuf-dev", + "snappy-dev", + "snappy-static", + "util-linux-dev", + "wget", + "xz", + "yaml-dev", + "yaml-static", + "zlib-dev", + "zlib-static", + "zstd-dev", + "zstd-static", +] + + +def build_alpine_3_18( + client: dagger.Client, platform: dagger.Platform +) -> dagger.Container: + ctr = client.container(platform=platform).from_("alpine:3.18") + + pkgs = [pkg for pkg in _ALPINE_COMMON_PACKAGES] + + ctr = ctr.with_exec(["apk", "add", "--no-cache"] + pkgs) + + return ctr + + +def build_alpine_3_19( + client: dagger.Client, platform: dagger.Platform +) -> dagger.Container: + ctr = client.container(platform=platform).from_("alpine:3.19") + + pkgs = [pkg for pkg in _ALPINE_COMMON_PACKAGES] + + ctr = ctr.with_exec(["apk", "add", "--no-cache"] + pkgs) + + return ctr + + +def static_build_openssl( + client: dagger.Client, ctr: dagger.Container +) -> dagger.Container: + tree = ( + client.git(url="https://github.com/openssl/openssl", keep_git_dir=True) + .tag("openssl-3.1.4") + .tree() + ) + + # + # TODO: verify 32-bit builds + # + ctr = ( + ctr.with_directory("/openssl", tree) + .with_workdir("/openssl") + .with_env_variable("CFLAGS", "-fno-lto -pipe") + .with_env_variable("LDFLAGS", "-static") + .with_env_variable("PKG_CONFIG", "pkg-config --static") + .with_exec( + [ + "sed", + "-i", + "s/disable('static', 'pic', 'threads');/disable('static', 'pic');/", + "Configure", + ] + ) + .with_exec( + [ + "./config", + "-static", + "threads", + "no-tests", + "--prefix=/openssl-static", + "--openssldir=/opt/netdata/etc/ssl", + ] + ) + .with_exec(["make", "V=1", "-j", str(os.cpu_count())]) + .with_exec(["make", "V=1", "-j", str(os.cpu_count()), "install_sw"]) + .with_exec(["ln", "-s", "/openssl-static/lib", "/openssl-static/lib64"]) + .with_exec(["perl", "configdata.pm", "--dump"]) + ) + + return ctr + + +def static_build_bash(client: dagger.Client, ctr: dagger.Container) -> dagger.Container: + tree = ( + client.git(url="https://git.savannah.gnu.org/git/bash.git", keep_git_dir=True) + .tag("bash-5.1") + .tree() + ) + + ctr = ( + ctr.with_directory("/bash", tree) + .with_workdir("/bash") + .with_env_variable("CFLAGS", "-pipe") + .with_env_variable("LDFLAGS", "") + .with_env_variable("PKG_CONFIG", "pkg-config --static") + .with_env_variable("PKG_CONFIG_PATH", "/openssl-static/lib64/pkgconfig") + .with_exec( + [ + "./configure", + "--prefix", + "/opt/netdata", + "--without-bash-malloc", + "--enable-static-link", + "--enable-net-redirections", + "--enable-array-variables", + "--disable-progcomp", + "--disable-profiling", + "--disable-nls", + "--disable-dependency-tracking", + ] + ) + .with_exec( + [ + "echo", + "-e", + "all:\nclean:\ninstall:\n", + ">", + "examples/loadables/Makefile", + ] + ) + .with_exec(["make", "clean"]) + # see: https://gitweb.gentoo.org/repo/gentoo.git/tree/app-shells/bash/files/bash-5.1-parallel_make.patch?id=4c2ebbf4b8bc660beb98cc2d845c73375d6e4f50 + .with_exec(["make", "V=1", "-j", "2", "install"]) + .with_exec(["strip", "/opt/netdata/bin/bash"]) + ) + + return ctr + + +def static_build_curl(client: dagger.Client, ctr: dagger.Container) -> dagger.Container: + tree = ( + client.git(url="https://github.com/curl/curl", keep_git_dir=True) + .tag("curl-8_4_0") + .tree() + ) + + ctr = ( + ctr.with_directory("/curl", tree) + .with_workdir("/curl") + .with_env_variable("CFLAGS", "-I/openssl-static/include -pipe") + .with_env_variable("LDFLAGS", "-static -L/openssl-static/lib64") + .with_env_variable("PKG_CONFIG", "pkg-config --static") + .with_env_variable("PKG_CONFIG_PATH", "/openssl-static/lib64/pkgconfig") + .with_exec(["autoreconf", "-ifv"]) + .with_exec( + [ + "./configure", + "--prefix=/curl-static", + "--enable-optimize", + "--disable-shared", + "--enable-static", + "--enable-http", + "--disable-ldap", + "--disable-ldaps", + "--enable-proxy", + "--disable-dict", + "--disable-telnet", + "--disable-tftp", + "--disable-pop3", + "--disable-imap", + "--disable-smb", + "--disable-smtp", + "--disable-gopher", + "--enable-ipv6", + "--enable-cookies", + "--with-ca-fallback", + "--with-openssl", + "--disable-dependency-tracking", + ] + ) + .with_exec( + ["sed", "-i", "-e", "s/LDFLAGS =/LDFLAGS = -all-static/", "src/Makefile"] + ) + .with_exec(["make", "clean"]) + .with_exec(["make", "V=1", "-j", str(os.cpu_count()), "install"]) + .with_exec(["cp", "/curl-static/bin/curl", "/opt/netdata/bin/curl"]) + .with_exec(["strip", "/opt/netdata/bin/curl"]) + ) + + return ctr + + +def static_build_ioping( + client: dagger.Client, ctr: dagger.Container +) -> dagger.Container: + tree = ( + client.git(url="https://github.com/koct9i/ioping", keep_git_dir=True) + .tag("v1.3") + .tree() + ) + + ctr = ( + ctr.with_directory("/ioping", tree) + .with_workdir("/ioping") + .with_env_variable("CFLAGS", "-static -pipe") + .with_exec(["mkdir", "-p", "/opt/netdata/usr/libexec/netdata/plugins.d"]) + .with_exec(["make", "V=1"]) + .with_exec( + [ + "install", + "-o", + "root", + "-g", + "root", + "-m", + "4750", + "ioping", + "/opt/netdata/usr/libexec/netdata/plugins.d", + ] + ) + .with_exec(["strip", "/opt/netdata/usr/libexec/netdata/plugins.d/ioping"]) + ) + + return ctr + + +def static_build_libnetfilter_acct( + client: dagger.Client, ctr: dagger.Container +) -> dagger.Container: + tree = ( + client.git(url="git://git.netfilter.org/libnetfilter_acct", keep_git_dir=True) + .tag("libnetfilter_acct-1.0.3") + .tree() + ) + + ctr = ( + ctr.with_directory("/libnetfilter_acct", tree) + .with_workdir("/libnetfilter_acct") + .with_env_variable("CFLAGS", "-static -I/usr/include/libmnl -pipe") + .with_env_variable("LDFLAGS", "-static -L/usr/lib -lmnl") + .with_env_variable("PKG_CONFIG", "pkg-config --static") + .with_env_variable("PKG_CONFIG_PATH", "/usr/lib/pkgconfig") + .with_exec(["autoreconf", "-ifv"]) + .with_exec( + [ + "./configure", + "--prefix=/libnetfilter_acct-static", + "--exec-prefix=/libnetfilter_acct-static", + ] + ) + .with_exec(["make", "clean"]) + .with_exec(["make", "V=1", "-j", str(os.cpu_count()), "install"]) + ) + + return ctr + + +def static_build_netdata( + client: dagger.Client, ctr: dagger.Container +) -> dagger.Container: + CFLAGS = [ + "-ffunction-sections", + "-fdata-sections", + "-static", + "-O2", + "-funroll-loops", + "-I/openssl-static/include", + "-I/libnetfilter_acct-static/include/libnetfilter_acct", + "-I/curl-local/include/curl", + "-I/usr/include/libmnl", + "-pipe", + ] + + LDFLAGS = [ + "-Wl,--gc-sections", + "-static", + "-L/openssl-static/lib64", + "-L/libnetfilter_acct-static/lib", + "-lnetfilter_acct", + "-L/usr/lib", + "-lmnl", + "-L/usr/lib", + "-lzstd", + "-L/curl-local/lib", + ] + + PKG_CONFIG = [ + "pkg-config", + "--static", + ] + + PKG_CONFIG_PATH = [ + "/openssl-static/lib64/pkgconfig", + "/libnetfilter_acct-static/lib/pkgconfig", + "/usr/lib/pkgconfig", + "/curl-local/lib/pkgconfig", + ] + + CMAKE_FLAGS = [ + "-DOPENSSL_ROOT_DIR=/openssl-static", + "-DOPENSSL_LIBRARIES=/openssl-static/lib64", + "-DCMAKE_INCLUDE_DIRECTORIES_PROJECT_BEFORE=/openssl-static", + "-DLWS_OPENSSL_INCLUDE_DIRS=/openssl-static/include", + "-DLWS_OPENSSL_LIBRARIES=/openssl-static/lib64/libssl.a;/openssl-static/lib64/libcrypto.a", + ] + + NETDATA_INSTALLER_CMD = [ + "./netdata-installer.sh", + "--install-prefix", + "/opt", + "--dont-wait", + "--dont-start-it", + "--disable-exporting-mongodb", + "--require-cloud", + "--use-system-protobuf", + "--dont-scrub-cflags-even-though-it-may-break-things", + "--one-time-build", + "--enable-lto", + ] + + ctr = ( + ctr.with_workdir("/netdata") + .with_env_variable("NETDATA_CMAKE_OPTIONS", "-DCMAKE_BUILD_TYPE=Debug") + .with_env_variable("CFLAGS", " ".join(CFLAGS)) + .with_env_variable("LDFLAGS", " ".join(LDFLAGS)) + .with_env_variable("PKG_CONFIG", " ".join(PKG_CONFIG)) + .with_env_variable("PKG_CONFIG_PATH", ":".join(PKG_CONFIG_PATH)) + .with_env_variable("CMAKE_FLAGS", " ".join(CMAKE_FLAGS)) + .with_env_variable("EBPF_LIBC", "static") + .with_env_variable("IS_NETDATA_STATIC_BINARY", "yes") + .with_exec(NETDATA_INSTALLER_CMD) + ) + + return ctr + + +def static_build(client, repo_path): + cmake_build_release_path = os.path.join(repo_path, "cmake-build-release") + + ctr = build_alpine_3_18(client, dagger.Platform("linux/x86_64")) + ctr = static_build_openssl(client, ctr) + ctr = static_build_bash(client, ctr) + ctr = static_build_curl(client, ctr) + ctr = static_build_ioping(client, ctr) + ctr = static_build_libnetfilter_acct(client, ctr) + + ctr = ctr.with_directory( + "/netdata", + client.host().directory(repo_path), + exclude=[ + f"{cmake_build_release_path}/*", + "fluent-bit/build", + ], + ) + + # TODO: link bin/sbin + + ctr = static_build_netdata(client, ctr) + + build_dir = ctr.directory("/opt/netdata") + artifact_dir = os.path.join(Path.home(), "ci/netdata-static") + output_task = build_dir.export(artifact_dir) + return output_task + + +_CENTOS_COMMON_PACKAGES = [ + "autoconf", + "autoconf-archive", + "autogen", + "automake", + "bison", + "bison-devel", + "cmake", + "cups-devel", + "curl", + "diffutils", + "elfutils-libelf-devel", + "findutils", + "flex", + "flex-devel", + "freeipmi-devel", + "gcc", + "gcc-c++", + "git-core", + "golang", + "json-c-devel", + "libyaml-devel", + "libatomic", + "libcurl-devel", + "libmnl-devel", + "libnetfilter_acct-devel", + "libtool", + "libuuid-devel", + "libuv-devel", + "libzstd-devel", + "lm_sensors", + "lz4-devel", + "make", + "ninja-build", + "openssl-devel", + "openssl-perl", + "patch", + "pcre2-devel", + "pkgconfig", + "pkgconfig(libmongoc-1.0)", + "procps", + "protobuf-c-devel", + "protobuf-compiler", + "protobuf-devel", + "rpm-build", + "rpm-devel", + "rpmdevtools", + "snappy-devel", + "systemd-devel", + "wget", + "zlib-devel", +] + + +def build_amazon_linux_2( + client: dagger.Client, platform: dagger.Platform +) -> dagger.Container: + ctr = client.container(platform=platform).from_("amazonlinux:2") + + pkgs = [pkg for pkg in _CENTOS_COMMON_PACKAGES] + + ctr = ( + ctr.with_exec(["yum", "update", "-y"]) + .with_exec(["yum", "install", "-y"] + pkgs) + .with_exec(["yum", "clean", "all"]) + .with_exec(["c_rehash"]) + .with_exec( + [ + "mkdir", + "-p", + "/root/rpmbuild/BUILD", + "/root/rpmbuild/RPMS", + "/root/rpmbuild/SOURCES", + "/root/rpmbuild/SPECS", + "/root/rpmbuild/SRPMS", + ] + ) + ) + + if platform == "linux/x86_64": + machine = "x86_64" + elif platform == "linux/arm64": + machine = "aarch64" + else: + raise Exception( + "Amaxon Linux 2 supports only linux/amd64 and linux/arm64 platforms." + ) + + + checksum_path = Path(__file__).parent / f"files/cmake-{machine}.sha256" + + ctr = ( + ctr.with_file( + f"cmake-{machine}.sha256", + client.host().file(checksum_path.as_posix()), + ) + .with_exec( + [ + "curl", + "--fail", + "-sSL", + "--connect-timeout", + "20", + "--retry", + "3", + "--output", + f"cmake-{machine}.sh", + f"https://github.com/Kitware/CMake/releases/download/v3.27.6/cmake-3.27.6-linux-{machine}.sh", + ] + ) + .with_exec(["sha256sum", "-c", f"cmake-{machine}.sha256"]) + .with_exec(["chmod", "u+x", f"./cmake-{machine}.sh"]) + .with_exec([f"./cmake-{machine}.sh", "--skip-license", "--prefix=/usr/local"]) + ) + + return ctr + + +def build_centos_7( + client: dagger.Client, platform: dagger.Platform +) -> dagger.Container: + ctr = client.container(platform=platform).from_("centos:7") + + pkgs = [pkg for pkg in _CENTOS_COMMON_PACKAGES] + ["bash"] + + ctr = ( + ctr.with_exec(["yum", "install", "-y", "epel-release"]) + .with_exec(["yum", "update", "-y"]) + .with_exec(["yum", "install", "-y"] + pkgs) + .with_exec(["yum", "clean", "all"]) + .with_exec(["c_rehash"]) + .with_exec( + [ + "mkdir", + "-p", + "/root/rpmbuild/BUILD", + "/root/rpmbuild/RPMS", + "/root/rpmbuild/SOURCES", + "/root/rpmbuild/SPECS", + "/root/rpmbuild/SRPMS", + ] + ) + ) + + if platform == "linux/x86_64": + machine = "x86_64" + elif platform == "linux/arm64": + machine = "aarch64" + else: + raise Exception("CentOS 7 supports only linux/amd64 and linux/arm64 platforms.") + + checksum_path = Path(__file__).parent / f"files/cmake-{machine}.sha256" + + ctr = ( + ctr.with_file( + f"cmake-{machine}.sha256", + client.host().file(checksum_path.as_posix()), + ) + .with_exec( + [ + "curl", + "--fail", + "-sSL", + "--connect-timeout", + "20", + "--retry", + "3", + "--output", + f"cmake-{machine}.sh", + f"https://github.com/Kitware/CMake/releases/download/v3.27.6/cmake-3.27.6-linux-{machine}.sh", + ] + ) + .with_exec(["sha256sum", "-c", f"cmake-{machine}.sha256"]) + .with_exec(["chmod", "u+x", f"./cmake-{machine}.sh"]) + .with_exec([f"./cmake-{machine}.sh", "--skip-license", "--prefix=/usr/local"]) + ) + + return ctr + + +_ROCKY_LINUX_COMMON_PACKAGES = [ + "autoconf", + "autoconf-archive", + "automake", + "bash", + "bison", + "cmake", + "cups-devel", + "curl", + "libcurl-devel", + "diffutils", + "elfutils-libelf-devel", + "findutils", + "flex", + "freeipmi-devel", + "gcc", + "gcc-c++", + "git", + "golang", + "json-c-devel", + "libatomic", + "libmnl-devel", + "libtool", + "libuuid-devel", + "libuv-devel", + "libyaml-devel", + "libzstd-devel", + "lm_sensors", + "lz4-devel", + "make", + "ninja-build", + "nc", + "openssl-devel", + "openssl-perl", + "patch", + "pcre2-devel", + "pkgconfig", + "pkgconfig(libmongoc-1.0)", + "procps", + "protobuf-c-devel", + "protobuf-compiler", + "protobuf-devel", + "python3", + "python3-pyyaml", + "rpm-build", + "rpm-devel", + "rpmdevtools", + "snappy-devel", + "systemd-devel", + "wget", + "zlib-devel", +] + + +def build_rocky_linux_8( + client: dagger.Client, platform: dagger.Platform +) -> dagger.Container: + ctr = client.container(platform=platform).from_("rockylinux:8") + + pkgs = [pkg for pkg in _ROCKY_LINUX_COMMON_PACKAGES] + ["autogen"] + + ctr = ( + ctr.with_exec(["dnf", "distro-sync", "-y", "--nodocs"]) + .with_exec( + [ + "dnf", + "install", + "-y", + "--nodocs", + "dnf-command(config-manager)", + "epel-release", + ] + ) + .with_exec(["dnf", "config-manager", "--set-enabled", "powertools"]) + .with_exec(["dnf", "clean", "packages"]) + .with_exec( + [ + "dnf", + "install", + "-y", + "--nodocs", + "--setopt=install_weak_deps=False", + "--setopt=diskspacecheck=False", + ] + + pkgs + ) + .with_exec(["rm", "-rf", "/var/cache/dnf"]) + .with_exec(["c_rehash"]) + .with_exec( + [ + "mkdir", + "-p", + "/root/rpmbuild/BUILD", + "/root/rpmbuild/RPMS", + "/root/rpmbuild/SOURCES", + "/root/rpmbuild/SPECS", + "/root/rpmbuild/SRPMS", + ] + ) + ) + + return ctr + + +def build_rocky_linux_9( + client: dagger.Client, platform: dagger.Platform +) -> dagger.Container: + ctr = client.container(platform=platform).from_("rockylinux:9") + + pkgs = [pkg for pkg in _ROCKY_LINUX_COMMON_PACKAGES] + + ctr = ( + ctr.with_exec(["dnf", "distro-sync", "-y", "--nodocs"]) + .with_exec( + [ + "dnf", + "install", + "-y", + "--nodocs", + "dnf-command(config-manager)", + "epel-release", + ] + ) + .with_exec(["dnf", "config-manager", "--set-enabled", "crb"]) + .with_exec(["dnf", "clean", "packages"]) + .with_exec( + [ + "dnf", + "install", + "-y", + "--allowerasing", + "--nodocs", + "--setopt=install_weak_deps=False", + "--setopt=diskspacecheck=False", + ] + + pkgs + ) + .with_exec(["rm", "-rf", "/var/cache/dnf"]) + .with_exec(["c_rehash"]) + .with_exec( + [ + "mkdir", + "-p", + "/root/rpmbuild/BUILD", + "/root/rpmbuild/RPMS", + "/root/rpmbuild/SOURCES", + "/root/rpmbuild/SPECS", + "/root/rpmbuild/SRPMS", + ] + ) + ) + + return ctr + + +_CENTOS_STREAM_COMMON_PACKAGES = [ + "autoconf", + "autoconf-archive", + "automake", + "bash", + "bison", + "cmake", + "cups-devel", + "curl", + "libcurl-devel", + "libyaml-devel", + "diffutils", + "elfutils-libelf-devel", + "findutils", + "flex", + "freeipmi-devel", + "gcc", + "gcc-c++", + "git", + "golang", + "json-c-devel", + "libatomic", + "libmnl-devel", + "libtool", + "libuuid-devel", + "libuv-devel", + # "libzstd-devel", + "lm_sensors", + "lz4-devel", + "make", + "ninja-build", + "nc", + "openssl-devel", + "openssl-perl", + "patch", + "pcre2-devel", + "pkgconfig", + "pkgconfig(libmongoc-1.0)", + "procps", + "protobuf-c-devel", + "protobuf-compiler", + "protobuf-devel", + "python3", + "python3-pyyaml", + "rpm-build", + "rpm-devel", + "rpmdevtools", + "snappy-devel", + "systemd-devel", + "wget", + "zlib-devel", +] + + +def build_centos_stream_8( + client: dagger.Client, platform: dagger.Platform +) -> dagger.Container: + ctr = client.container(platform=platform).from_("quay.io/centos/centos:stream8") + + pkgs = [pkg for pkg in _CENTOS_STREAM_COMMON_PACKAGES] + ["autogen"] + + ctr = ( + ctr.with_exec(["dnf", "distro-sync", "-y", "--nodocs"]) + .with_exec( + [ + "dnf", + "install", + "-y", + "--nodocs", + "dnf-command(config-manager)", + "epel-release", + ] + ) + .with_exec(["dnf", "config-manager", "--set-enabled", "powertools"]) + .with_exec(["dnf", "clean", "packages"]) + .with_exec( + [ + "dnf", + "install", + "-y", + "--nodocs", + "--setopt=install_weak_deps=False", + "--setopt=diskspacecheck=False", + ] + + pkgs + ) + .with_exec(["rm", "-rf", "/var/cache/dnf"]) + .with_exec(["c_rehash"]) + .with_exec( + [ + "mkdir", + "-p", + "/root/rpmbuild/BUILD", + "/root/rpmbuild/RPMS", + "/root/rpmbuild/SOURCES", + "/root/rpmbuild/SPECS", + "/root/rpmbuild/SRPMS", + ] + ) + ) + + return ctr + + +def build_centos_stream_9( + client: dagger.Client, platform: dagger.Platform +) -> dagger.Container: + ctr = client.container(platform=platform).from_("quay.io/centos/centos:stream9") + + pkgs = [pkg for pkg in _CENTOS_STREAM_COMMON_PACKAGES] + + ctr = ( + ctr.with_exec(["dnf", "distro-sync", "-y", "--nodocs"]) + .with_exec( + [ + "dnf", + "install", + "-y", + "--nodocs", + "dnf-command(config-manager)", + "epel-release", + ] + ) + .with_exec(["dnf", "config-manager", "--set-enabled", "crb"]) + .with_exec(["dnf", "clean", "packages"]) + .with_exec( + [ + "dnf", + "install", + "-y", + "--allowerasing", + "--nodocs", + "--setopt=install_weak_deps=False", + "--setopt=diskspacecheck=False", + ] + + pkgs + ) + .with_exec(["rm", "-rf", "/var/cache/dnf"]) + .with_exec(["c_rehash"]) + .with_exec( + [ + "mkdir", + "-p", + "/root/rpmbuild/BUILD", + "/root/rpmbuild/RPMS", + "/root/rpmbuild/SOURCES", + "/root/rpmbuild/SPECS", + "/root/rpmbuild/SRPMS", + ] + ) + ) + + return ctr + + +_ORACLE_LINUX_COMMON_PACKAGES = list(_ROCKY_LINUX_COMMON_PACKAGES) + + +def build_oracle_linux_9( + client: dagger.Client, platform: dagger.Platform +) -> dagger.Container: + ctr = client.container(platform=platform).from_("oraclelinux:9") + + pkgs = [pkg for pkg in _ORACLE_LINUX_COMMON_PACKAGES] + + repo_path = str(Path(__file__).parent.parent.parent) + this_path = os.path.join(repo_path, "packaging/dag") + + ctr = ( + ctr.with_file( + "/etc/yum.repos.d/ol9-epel.repo", + client.host().file(f"{this_path}/ol9-epel.repo"), + ) + .with_exec(["dnf", "config-manager", "--set-enabled", "ol9_codeready_builder"]) + .with_exec(["dnf", "config-manager", "--set-enabled", "ol9_developer_EPEL"]) + .with_exec(["dnf", "distro-sync", "-y", "--nodocs"]) + .with_exec(["dnf", "clean", "-y", "packages"]) + .with_exec( + [ + "dnf", + "install", + "-y", + "--nodocs", + "--setopt=install_weak_deps=False", + "--setopt=diskspacecheck=False", + ] + + pkgs + ) + .with_exec(["rm", "-rf", "/var/cache/dnf"]) + .with_exec(["c_rehash"]) + .with_exec( + [ + "mkdir", + "-p", + "/root/rpmbuild/BUILD", + "/root/rpmbuild/RPMS", + "/root/rpmbuild/SOURCES", + "/root/rpmbuild/SPECS", + "/root/rpmbuild/SRPMS", + ] + ) + ) + + return ctr + + +def build_oracle_linux_8( + client: dagger.Client, platform: dagger.Platform +) -> dagger.Container: + ctr = client.container(platform=platform).from_("oraclelinux:8") + + pkgs = [pkg for pkg in _ORACLE_LINUX_COMMON_PACKAGES] + ["autogen"] + + repo_path = str(Path(__file__).parent.parent.parent) + this_path = os.path.join(repo_path, "packaging/dag") + + ctr = ( + ctr.with_file( + "/etc/yum.repos.d/ol8-epel.repo", + client.host().file(f"{this_path}/ol8-epel.repo"), + ) + .with_exec(["dnf", "config-manager", "--set-enabled", "ol8_codeready_builder"]) + .with_exec(["dnf", "distro-sync", "-y", "--nodocs"]) + .with_exec(["dnf", "clean", "-y", "packages"]) + .with_exec( + [ + "dnf", + "install", + "-y", + "--nodocs", + "--setopt=install_weak_deps=False", + "--setopt=diskspacecheck=False", + ] + + pkgs + ) + .with_exec(["rm", "-rf", "/var/cache/dnf"]) + .with_exec(["c_rehash"]) + .with_exec( + [ + "mkdir", + "-p", + "/root/rpmbuild/BUILD", + "/root/rpmbuild/RPMS", + "/root/rpmbuild/SOURCES", + "/root/rpmbuild/SPECS", + "/root/rpmbuild/SRPMS", + ] + ) + ) + + return ctr + + +_OPENSUSE_COMMON_PACKAGES = [ + "autoconf", + "autoconf-archive", + "autogen", + "automake", + "bison", + "cmake", + "cups", + "cups-devel", + "curl", + "diffutils", + "flex", + "freeipmi-devel", + "gcc", + "gcc-c++", + "git-core", + "go", + "json-glib-devel", + "judy-devel", + "libatomic1", + "libcurl-devel", + "libelf-devel", + "liblz4-devel", + "libjson-c-devel", + "libyaml-devel", + "libmnl0", + "libmnl-devel", + "libnetfilter_acct1", + "libnetfilter_acct-devel", + "libpcre2-8-0", + "libopenssl-devel", + "libtool", + "libuv-devel", + "libuuid-devel", + "libzstd-devel", + "make", + "ninja", + "patch", + "pkg-config", + "protobuf-devel", + "rpm-build", + "rpm-devel", + "rpmdevtools", + "snappy-devel", + "systemd-devel", + "tar", + "wget", + "xen-devel", +] + + +def build_opensuse_tumbleweed( + client: dagger.Client, platform: dagger.Platform +) -> dagger.Container: + ctr = client.container(platform=platform).from_("opensuse/tumbleweed:latest") + + pkgs = [pkg for pkg in _OPENSUSE_COMMON_PACKAGES] + ["protobuf-c"] + + ctr = ( + ctr.with_exec(["zypper", "update", "-y"]) + .with_exec( + [ + "zypper", + "install", + "-y", + "--allow-downgrade", + ] + + pkgs + ) + .with_exec(["zypper", "clean"]) + .with_exec(["rm", "-rf", "/var/cache/zypp/*/*"]) + .with_exec(["c_rehash"]) + .with_exec( + [ + "mkdir", + "-p", + "/usr/src/packages/BUILD", + "/usr/src/packages/RPMS", + "/usr/src/packages/SOURCES", + "/usr/src/packages/SPECS", + "/usr/src/packages/SRPMS", + ] + ) + ) + + return ctr + + +def build_opensuse_15_5( + client: dagger.Client, platform: dagger.Platform +) -> dagger.Container: + ctr = client.container(platform=platform).from_("opensuse/leap:15.5") + + pkgs = [pkg for pkg in _OPENSUSE_COMMON_PACKAGES] + ["libprotobuf-c-devel"] + + ctr = ( + ctr.with_exec(["zypper", "update", "-y"]) + .with_exec( + [ + "zypper", + "install", + "-y", + "--allow-downgrade", + ] + + pkgs + ) + .with_exec(["zypper", "clean"]) + .with_exec(["rm", "-rf", "/var/cache/zypp/*/*"]) + .with_exec(["c_rehash"]) + .with_exec( + [ + "mkdir", + "-p", + "/usr/src/packages/BUILD", + "/usr/src/packages/RPMS", + "/usr/src/packages/SOURCES", + "/usr/src/packages/SPECS", + "/usr/src/packages/SRPMS", + ] + ) + ) + + return ctr + + +def build_opensuse_15_4( + client: dagger.Client, platform: dagger.Platform +) -> dagger.Container: + crt = client.container(platform=platform).from_("opensuse/leap:15.4") + + pkgs = [pkg for pkg in _OPENSUSE_COMMON_PACKAGES] + ["libprotobuf-c-devel"] + + crt = ( + crt.with_exec(["zypper", "update", "-y"]) + .with_exec( + [ + "zypper", + "install", + "-y", + "--allow-downgrade", + ] + + pkgs + ) + .with_exec(["zypper", "clean"]) + .with_exec(["rm", "-rf", "/var/cache/zypp/*/*"]) + .with_exec(["c_rehash"]) + .with_exec( + [ + "mkdir", + "-p", + "/usr/src/packages/BUILD", + "/usr/src/packages/RPMS", + "/usr/src/packages/SOURCES", + "/usr/src/packages/SPECS", + "/usr/src/packages/SRPMS", + ] + ) + ) + + return crt + + +_FEDORA_COMMON_PACKAGES = [ + "autoconf", + "autoconf-archive", + "autogen", + "automake", + "bash", + "bison", + "cmake", + "cups-devel", + "curl", + "diffutils", + "elfutils-libelf-devel", + "findutils", + "flex", + "freeipmi-devel", + "gcc", + "gcc-c++", + "git-core", + "golang", + "json-c-devel", + "libcurl-devel", + "libyaml-devel", + "Judy-devel", + "libatomic", + "libmnl-devel", + "libnetfilter_acct-devel", + "libtool", + "libuuid-devel", + "libuv-devel", + "libzstd-devel", + "lz4-devel", + "make", + "ninja-build", + "openssl-devel", + "openssl-perl", + "patch", + "pcre2-devel", + "pkgconfig", +] + + +def build_fedora_37( + client: dagger.Client, platform: dagger.Platform +) -> dagger.Container: + ctr = client.container(platform=platform).from_("fedora:37") + + pkgs = [pkg for pkg in _FEDORA_COMMON_PACKAGES] + + ctr = ( + ctr.with_exec(["dnf", "distro-sync", "-y", "--nodocs"]) + .with_exec(["dnf", "clean", "-y", "packages"]) + .with_exec( + [ + "dnf", + "install", + "-y", + "--nodocs", + "--setopt=install_weak_deps=False", + "--setopt=diskspacecheck=False", + ] + + pkgs + ) + .with_exec(["rm", "-rf", "/var/cache/dnf"]) + .with_exec(["c_rehash"]) + .with_exec( + [ + "mkdir", + "-p", + "/root/rpmbuild/BUILD", + "/root/rpmbuild/RPMS", + "/root/rpmbuild/SOURCES", + "/root/rpmbuild/SPECS", + "/root/rpmbuild/SRPMS", + ] + ) + ) + + return ctr + + +def build_fedora_38( + client: dagger.Client, platform: dagger.Platform +) -> dagger.Container: + ctr = client.container(platform=platform).from_("fedora:38") + + pkgs = [pkg for pkg in _FEDORA_COMMON_PACKAGES] + + ctr = ( + ctr.with_exec(["dnf", "distro-sync", "-y", "--nodocs"]) + .with_exec(["dnf", "clean", "-y", "packages"]) + .with_exec( + [ + "dnf", + "install", + "-y", + "--nodocs", + "--setopt=install_weak_deps=False", + "--setopt=diskspacecheck=False", + ] + + pkgs + ) + .with_exec(["rm", "-rf", "/var/cache/dnf"]) + .with_exec(["c_rehash"]) + .with_exec( + [ + "mkdir", + "-p", + "/root/rpmbuild/BUILD", + "/root/rpmbuild/RPMS", + "/root/rpmbuild/SOURCES", + "/root/rpmbuild/SPECS", + "/root/rpmbuild/SRPMS", + ] + ) + ) + + return ctr + + +def build_fedora_39( + client: dagger.Client, platform: dagger.Platform +) -> dagger.Container: + ctr = client.container(platform=platform).from_("fedora:39") + + pkgs = [pkg for pkg in _FEDORA_COMMON_PACKAGES] + + ctr = ( + ctr.with_exec(["dnf", "distro-sync", "-y", "--nodocs"]) + .with_exec(["dnf", "clean", "-y", "packages"]) + .with_exec( + [ + "dnf", + "install", + "-y", + "--nodocs", + "--setopt=install_weak_deps=False", + "--setopt=diskspacecheck=False", + ] + + pkgs + ) + .with_exec(["rm", "-rf", "/var/cache/dnf"]) + .with_exec(["c_rehash"]) + .with_exec( + [ + "mkdir", + "-p", + "/root/rpmbuild/BUILD", + "/root/rpmbuild/RPMS", + "/root/rpmbuild/SOURCES", + "/root/rpmbuild/SPECS", + "/root/rpmbuild/SRPMS", + ] + ) + ) + + return ctr + + +_DEBIAN_COMMON_PACKAGES = [ + "autoconf", + "autoconf-archive", + "autogen", + "automake", + "bison", + "build-essential", + "ca-certificates", + "cmake", + "curl", + "dh-autoreconf", + "dh-make", + "dpkg-dev", + "flex", + "g++", + "gcc", + "git-buildpackage", + "git-core", + "golang", + "libatomic1", + "libcurl4-openssl-dev", + "libcups2-dev", + "libdistro-info-perl", + "libelf-dev", + "libipmimonitoring-dev", + "libjson-c-dev", + "libyaml-dev", + "libjudy-dev", + "liblz4-dev", + "libmnl-dev", + "libmongoc-dev", + "libnetfilter-acct-dev", + "libpcre2-dev", + "libprotobuf-dev", + "libprotoc-dev", + "libsnappy-dev", + "libsystemd-dev", + "libssl-dev", + "libtool", + "libuv1-dev", + "libzstd-dev", + "make", + "ninja-build", + "pkg-config", + "protobuf-compiler", + "systemd", + "uuid-dev", + "wget", + "zlib1g-dev", +] + + +def build_debian_10( + client: dagger.Client, platform: dagger.Platform +) -> dagger.Container: + ctr = client.container(platform=platform).from_("debian:buster") + + pkgs = [pkg for pkg in _DEBIAN_COMMON_PACKAGES] + ["dh-systemd", "libxen-dev"] + + ctr = ( + ctr.with_env_variable("DEBIAN_FRONTEND", "noninteractive") + .with_exec(["apt-get", "update"]) + .with_exec(["apt-get", "upgrade", "-y"]) + .with_exec(["apt-get", "install", "-y", "--no-install-recommends"] + pkgs) + .with_exec(["apt-get", "clean"]) + .with_exec(["c_rehash"]) + .with_exec(["rm", "-rf", "/var/lib/apt/lists/*"]) + ) + + return ctr + + +def build_debian_11( + client: dagger.Client, platform: dagger.Platform +) -> dagger.Container: + ctr = client.container(platform=platform).from_("debian:bullseye") + + pkgs = [pkg for pkg in _DEBIAN_COMMON_PACKAGES] + ["libxen-dev"] + + ctr = ( + ctr.with_env_variable("DEBIAN_FRONTEND", "noninteractive") + .with_exec(["apt-get", "update"]) + .with_exec(["apt-get", "upgrade", "-y"]) + .with_exec(["apt-get", "install", "-y", "--no-install-recommends"] + pkgs) + .with_exec(["apt-get", "clean"]) + .with_exec(["c_rehash"]) + .with_exec(["rm", "-rf", "/var/lib/apt/lists/*"]) + ) + + return ctr + + +def build_debian_12( + client: dagger.Client, platform: dagger.Platform +) -> dagger.Container: + ctr = client.container(platform=platform).from_("debian:bookworm") + + pkgs = [pkg for pkg in _DEBIAN_COMMON_PACKAGES] + + if platform != dagger.Platform("linux/i386"): + pkgs.append("libxen-dev") + + ctr = ( + ctr.with_env_variable("DEBIAN_FRONTEND", "noninteractive") + .with_exec(["apt-get", "update"]) + .with_exec(["apt-get", "upgrade", "-y"]) + .with_exec(["apt-get", "install", "-y", "--no-install-recommends"] + pkgs) + .with_exec(["apt-get", "clean"]) + .with_exec(["c_rehash"]) + .with_exec(["rm", "-rf", "/var/lib/apt/lists/*"]) + ) + + return ctr + + +_UBUNTU_COMMON_PACKAGES = [ + "autoconf", + "autoconf-archive", + "autogen", + "automake", + "bison", + "build-essential", + "ca-certificates", + "cmake", + "curl", + "dh-autoreconf", + "dh-make", + "dpkg-dev", + "flex", + "g++", + "gcc", + "git-buildpackage", + "git-core", + "golang", + "libatomic1", + "libcurl4-openssl-dev", + "libcups2-dev", + "libdistro-info-perl", + "libelf-dev", + "libipmimonitoring-dev", + "libjson-c-dev", + "libyaml-dev", + "libjudy-dev", + "liblz4-dev", + "libmnl-dev", + "libmongoc-dev", + "libnetfilter-acct-dev", + "libpcre2-dev", + "libprotobuf-dev", + "libprotoc-dev", + "libsnappy-dev", + "libsystemd-dev", + "libssl-dev", + "libtool", + "libuv1-dev", + "libxen-dev", + "libzstd-dev", + "make", + "ninja-build", + "pkg-config", + "protobuf-compiler", + "systemd", + "uuid-dev", + "wget", + "zlib1g-dev", +] + + +def build_ubuntu_20_04( + client: dagger.Client, platform: dagger.Platform +) -> dagger.Container: + ctr = client.container(platform=platform).from_("ubuntu:20.04") + + pkgs = [pkg for pkg in _UBUNTU_COMMON_PACKAGES] + ["dh-systemd"] + + ctr = ( + ctr.with_env_variable("DEBIAN_FRONTEND", "noninteractive") + .with_exec(["apt-get", "update"]) + .with_exec(["apt-get", "upgrade", "-y"]) + .with_exec(["apt-get", "install", "-y", "--no-install-recommends"] + pkgs) + .with_exec(["apt-get", "clean"]) + .with_exec(["c_rehash"]) + .with_exec(["rm", "-rf", "/var/lib/apt/lists/*"]) + ) + + # + # FIXME: add kitware for cmake on arm-hf + # + + return ctr + + +def build_ubuntu_22_04( + client: dagger.Client, platform: dagger.Platform +) -> dagger.Container: + ctr = client.container(platform=platform).from_("ubuntu:22.04") + + pkgs = [pkg for pkg in _UBUNTU_COMMON_PACKAGES] + + ctr = ( + ctr.with_env_variable("DEBIAN_FRONTEND", "noninteractive") + .with_exec(["apt-get", "update"]) + .with_exec(["apt-get", "upgrade", "-y"]) + .with_exec(["apt-get", "install", "-y", "--no-install-recommends"] + pkgs) + .with_exec(["apt-get", "clean"]) + .with_exec(["c_rehash"]) + .with_exec(["rm", "-rf", "/var/lib/apt/lists/*"]) + ) + + return ctr + + +def build_ubuntu_23_04( + client: dagger.Client, platform: dagger.Platform +) -> dagger.Container: + ctr = client.container(platform=platform).from_("ubuntu:23.04") + + pkgs = [pkg for pkg in _UBUNTU_COMMON_PACKAGES] + + ctr = ( + ctr.with_env_variable("DEBIAN_FRONTEND", "noninteractive") + .with_exec(["apt-get", "update"]) + .with_exec(["apt-get", "upgrade", "-y"]) + .with_exec(["apt-get", "install", "-y", "--no-install-recommends"] + pkgs) + .with_exec(["apt-get", "clean"]) + .with_exec(["c_rehash"]) + .with_exec(["rm", "-rf", "/var/lib/apt/lists/*"]) + ) + + return ctr + + +def build_ubuntu_23_10( + client: dagger.Client, platform: dagger.Platform +) -> dagger.Container: + ctr = client.container(platform=platform).from_("ubuntu:23.10") + + pkgs = [pkg for pkg in _UBUNTU_COMMON_PACKAGES] + + ctr = ( + ctr.with_env_variable("DEBIAN_FRONTEND", "noninteractive") + .with_exec(["apt-get", "update"]) + .with_exec(["apt-get", "upgrade", "-y"]) + .with_exec(["apt-get", "install", "-y", "--no-install-recommends"] + pkgs) + .with_exec(["apt-get", "clean"]) + .with_exec(["c_rehash"]) + .with_exec(["rm", "-rf", "/var/lib/apt/lists/*"]) + ) + + return ctr + + +def install_cargo(ctr: dagger.Container) -> dagger.Container: + bin_paths = [ + "/root/.cargo/bin", + "/usr/local/sbin", + "/usr/local/bin", + "/usr/sbin", + "/usr/bin", + "/sbin", + "/bin", + ] + + ctr = ( + ctr.with_workdir("/") + .with_exec(["sh", "-c", "curl https://sh.rustup.rs -sSf | sh -s -- -y"]) + .with_env_variable("PATH", ":".join(bin_paths)) + .with_exec(["cargo", "new", "--bin", "hello"]) + .with_workdir("/hello") + .with_exec(["cargo", "run", "-v", "-v"]) + ) + + return ctr diff --git a/packaging/dag/main.py b/packaging/dag/main.py new file mode 100755 index 000000000..c7e9670cf --- /dev/null +++ b/packaging/dag/main.py @@ -0,0 +1,18 @@ +#!/usr/bin/env python3 + +import click + +from test_command import test +from build_command import build + + +@click.group() +def cli(): + pass + + +cli.add_command(test) +cli.add_command(build) + +if __name__ == "__main__": + cli() diff --git a/packaging/dag/nd.py b/packaging/dag/nd.py new file mode 100644 index 000000000..d59adf30a --- /dev/null +++ b/packaging/dag/nd.py @@ -0,0 +1,409 @@ +from typing import List + +import enum +import os +import pathlib +import uuid + +import dagger +import jinja2 + +import imageutils + + +class Platform: + def __init__(self, platform: str): + self.platform = dagger.Platform(platform) + + def escaped(self) -> str: + return str(self.platform).removeprefix("linux/").replace("/", "_") + + def __eq__(self, other): + if isinstance(other, Platform): + return self.platform == other.platform + elif isinstance(other, dagger.Platform): + return self.platform == other + else: + return NotImplemented + + def __ne__(self, other): + return not (self == other) + + def __hash__(self): + return hash(self.platform) + + def __str__(self) -> str: + return str(self.platform) + + +SUPPORTED_PLATFORMS = set( + [ + Platform("linux/x86_64"), + Platform("linux/arm64"), + Platform("linux/i386"), + Platform("linux/arm/v7"), + Platform("linux/arm/v6"), + Platform("linux/ppc64le"), + Platform("linux/s390x"), + Platform("linux/riscv64"), + ] +) + + +SUPPORTED_DISTRIBUTIONS = set( + [ + "alpine_3_18", + "alpine_3_19", + "amazonlinux2", + "centos7", + "centos-stream8", + "centos-stream9", + "debian10", + "debian11", + "debian12", + "fedora37", + "fedora38", + "fedora39", + "opensuse15.4", + "opensuse15.5", + "opensusetumbleweed", + "oraclelinux8", + "oraclelinux9", + "rockylinux8", + "rockylinux9", + "ubuntu20.04", + "ubuntu22.04", + "ubuntu23.04", + "ubuntu23.10", + ] +) + + +class Distribution: + def __init__(self, display_name): + self.display_name = display_name + + if self.display_name == "alpine_3_18": + self.docker_tag = "alpine:3.18" + self.builder = imageutils.build_alpine_3_18 + self.platforms = SUPPORTED_PLATFORMS + elif self.display_name == "alpine_3_19": + self.docker_tag = "alpine:3.19" + self.builder = imageutils.build_alpine_3_19 + self.platforms = SUPPORTED_PLATFORMS + elif self.display_name == "amazonlinux2": + self.docker_tag = "amazonlinux:2" + self.builder = imageutils.build_amazon_linux_2 + self.platforms = SUPPORTED_PLATFORMS + elif self.display_name == "centos7": + self.docker_tag = "centos:7" + self.builder = imageutils.build_centos_7 + self.platforms = SUPPORTED_PLATFORMS + elif self.display_name == "centos-stream8": + self.docker_tag = "quay.io/centos/centos:stream8" + self.builder = imageutils.build_centos_stream_8 + self.platforms = SUPPORTED_PLATFORMS + elif self.display_name == "centos-stream9": + self.docker_tag = "quay.io/centos/centos:stream9" + self.builder = imageutils.build_centos_stream_9 + self.platforms = SUPPORTED_PLATFORMS + elif self.display_name == "debian10": + self.docker_tag = "debian:10" + self.builder = imageutils.build_debian_10 + self.platforms = SUPPORTED_PLATFORMS + elif self.display_name == "debian11": + self.docker_tag = "debian:11" + self.builder = imageutils.build_debian_11 + self.platforms = SUPPORTED_PLATFORMS + elif self.display_name == "debian12": + self.docker_tag = "debian:12" + self.builder = imageutils.build_debian_12 + self.platforms = SUPPORTED_PLATFORMS + elif self.display_name == "fedora37": + self.docker_tag = "fedora:37" + self.builder = imageutils.build_fedora_37 + self.platforms = SUPPORTED_PLATFORMS + elif self.display_name == "fedora38": + self.docker_tag = "fedora:38" + self.builder = imageutils.build_fedora_38 + self.platforms = SUPPORTED_PLATFORMS + elif self.display_name == "fedora39": + self.docker_tag = "fedora:39" + self.platforms = SUPPORTED_PLATFORMS + self.builder = imageutils.build_fedora_39 + elif self.display_name == "opensuse15.4": + self.docker_tag = "opensuse/leap:15.4" + self.builder = imageutils.build_opensuse_15_4 + self.platforms = SUPPORTED_PLATFORMS + elif self.display_name == "opensuse15.5": + self.docker_tag = "opensuse/leap:15.5" + self.builder = imageutils.build_opensuse_15_5 + self.platforms = SUPPORTED_PLATFORMS + elif self.display_name == "opensusetumbleweed": + self.docker_tag = "opensuse/tumbleweed:latest" + self.builder = imageutils.build_opensuse_tumbleweed + self.platforms = SUPPORTED_PLATFORMS + elif self.display_name == "oraclelinux8": + self.docker_tag = "oraclelinux:8" + self.builder = imageutils.build_oracle_linux_8 + self.platforms = SUPPORTED_PLATFORMS + elif self.display_name == "oraclelinux9": + self.docker_tag = "oraclelinux:9" + self.builder = imageutils.build_oracle_linux_9 + self.platforms = SUPPORTED_PLATFORMS + elif self.display_name == "rockylinux8": + self.docker_tag = "rockylinux:8" + self.builder = imageutils.build_rocky_linux_8 + self.platforms = SUPPORTED_PLATFORMS + elif self.display_name == "rockylinux9": + self.docker_tag = "rockylinux:9" + self.builder = imageutils.build_rocky_linux_9 + self.platforms = SUPPORTED_PLATFORMS + elif self.display_name == "ubuntu20.04": + self.docker_tag = "ubuntu:20.04" + self.builder = imageutils.build_ubuntu_20_04 + self.platforms = SUPPORTED_PLATFORMS + elif self.display_name == "ubuntu22.04": + self.docker_tag = "ubuntu:22.04" + self.builder = imageutils.build_ubuntu_22_04 + self.platforms = SUPPORTED_PLATFORMS + elif self.display_name == "ubuntu23.04": + self.docker_tag = "ubuntu:23.04" + self.builder = imageutils.build_ubuntu_23_04 + self.platforms = SUPPORTED_PLATFORMS + elif self.display_name == "ubuntu23.10": + self.docker_tag = "ubuntu:23.10" + self.builder = imageutils.build_ubuntu_23_10 + self.platforms = SUPPORTED_PLATFORMS + else: + raise ValueError(f"Unknown distribution: {self.display_name}") + + def _cache_volume( + self, client: dagger.Client, platform: dagger.Platform, path: str + ) -> dagger.CacheVolume: + tag = "_".join([self.display_name, Platform(platform).escaped()]) + return client.cache_volume(f"{path}-{tag}") + + def build( + self, client: dagger.Client, platform: dagger.Platform + ) -> dagger.Container: + if platform not in self.platforms: + raise ValueError( + f"Building {self.display_name} is not supported on {platform}." + ) + + ctr = self.builder(client, platform) + ctr = imageutils.install_cargo(ctr) + + return ctr + + +class FeatureFlags(enum.Flag): + DBEngine = enum.auto() + GoPlugin = enum.auto() + ExtendedBPF = enum.auto() + LogsManagement = enum.auto() + MachineLearning = enum.auto() + BundledProtobuf = enum.auto() + + +class NetdataInstaller: + def __init__( + self, + platform: Platform, + distro: Distribution, + repo_root: pathlib.Path, + prefix: pathlib.Path, + features: FeatureFlags, + ): + self.platform = platform + self.distro = distro + self.repo_root = repo_root + self.prefix = prefix + self.features = features + + def _mount_repo( + self, client: dagger.Client, ctr: dagger.Container, repo_root: pathlib.Path + ) -> dagger.Container: + host_repo_root = pathlib.Path(__file__).parent.parent.parent.as_posix() + exclude_dirs = ["build", "fluent-bit/build", "packaging/dag"] + + # The installer builds/stores intermediate artifacts under externaldeps/ + # We add a volume to speed up rebuilds. The volume has to be unique + # per platform/distro in order to avoid mixing unrelated artifacts + # together. + externaldeps = self.distro._cache_volume(client, self.platform, "externaldeps") + + ctr = ( + ctr.with_directory( + self.repo_root.as_posix(), client.host().directory(host_repo_root) + ) + .with_workdir(self.repo_root.as_posix()) + .with_mounted_cache( + os.path.join(self.repo_root, "externaldeps"), externaldeps + ) + ) + + return ctr + + def install(self, client: dagger.Client, ctr: dagger.Container) -> dagger.Container: + args = ["--dont-wait", "--dont-start-it", "--disable-telemetry"] + + if FeatureFlags.DBEngine not in self.features: + args.append("--disable-dbengine") + + if FeatureFlags.GoPlugin not in self.features: + args.append("--disable-go") + + if FeatureFlags.ExtendedBPF not in self.features: + args.append("--disable-ebpf") + + if FeatureFlags.LogsManagement not in self.features: + args.append("--disable-logsmanagement") + + if FeatureFlags.MachineLearning not in self.features: + args.append("--disable-ml") + + if FeatureFlags.BundledProtobuf not in self.features: + args.append("--use-system-protobuf") + + args.extend(["--install-prefix", self.prefix.parent.as_posix()]) + + ctr = self._mount_repo(client, ctr, self.repo_root.as_posix()) + + ctr = ctr.with_env_variable( + "NETDATA_CMAKE_OPTIONS", "-DCMAKE_BUILD_TYPE=Debug" + ).with_exec(["./netdata-installer.sh"] + args) + + return ctr + + +class Endpoint: + def __init__(self, hostname: str, port: int): + self.hostname = hostname + self.port = port + + def __str__(self): + return ":".join([self.hostname, str(self.port)]) + + +class ChildStreamConf: + def __init__( + self, + installer: NetdataInstaller, + destinations: List[Endpoint], + api_key: uuid.UUID, + ): + self.installer = installer + self.substitutions = { + "enabled": "yes", + "destination": " ".join([str(dst) for dst in destinations]), + "api_key": api_key, + "timeout_seconds": 60, + "default_port": 19999, + "send_charts_matching": "*", + "buffer_size_bytes": 1024 * 1024, + "reconnect_delay_seconds": 5, + "initial_clock_resync_iterations": 60, + } + + def render(self) -> str: + tmpl_path = pathlib.Path(__file__).parent / "files/child_stream.conf" + with open(tmpl_path) as fp: + tmpl = jinja2.Template(fp.read()) + + return tmpl.render(**self.substitutions) + + +class ParentStreamConf: + def __init__(self, installer: NetdataInstaller, api_key: uuid.UUID): + self.installer = installer + self.substitutions = { + "api_key": str(api_key), + "enabled": "yes", + "allow_from": "*", + "default_history": 3600, + "health_enabled_by_default": "auto", + "default_postpone_alarms_on_connect_seconds": 60, + "multiple_connections": "allow", + } + + def render(self) -> str: + tmpl_path = pathlib.Path(__file__).parent / "files/parent_stream.conf" + with open(tmpl_path) as fp: + tmpl = jinja2.Template(fp.read()) + + return tmpl.render(**self.substitutions) + + +class StreamConf: + def __init__(self, child_conf: ChildStreamConf, parent_conf: ParentStreamConf): + self.child_conf = child_conf + self.parent_conf = parent_conf + + def render(self) -> str: + child_section = self.child_conf.render() if self.child_conf else "" + parent_section = self.parent_conf.render() if self.parent_conf else "" + return "\n".join([child_section, parent_section]) + + +class AgentContext: + def __init__( + self, + client: dagger.Client, + platform: dagger.Platform, + distro: Distribution, + installer: NetdataInstaller, + endpoint: Endpoint, + api_key: uuid.UUID, + allow_children: bool, + ): + self.client = client + self.platform = platform + self.distro = distro + self.installer = installer + self.endpoint = endpoint + self.api_key = api_key + self.allow_children = allow_children + + self.parent_contexts = [] + + self.built_distro = False + self.built_agent = False + + def add_parent(self, parent_context: "AgentContext"): + self.parent_contexts.append(parent_context) + + def build_container(self) -> dagger.Container: + ctr = self.distro.build(self.client, self.platform) + ctr = self.installer.install(self.client, ctr) + + if len(self.parent_contexts) == 0 and not self.allow_children: + return ctr.with_exposed_port(self.endpoint.port) + + destinations = [parent_ctx.endpoint for parent_ctx in self.parent_contexts] + child_stream_conf = ChildStreamConf(self.installer, destinations, self.api_key) + + parent_stream_conf = None + if self.allow_children: + parent_stream_conf = ParentStreamConf(self.installer, self.api_key) + + stream_conf = StreamConf(child_stream_conf, parent_stream_conf) + + # write the stream conf to localhost and cp it in the container + host_stream_conf_path = pathlib.Path( + f"/tmp/{self.endpoint.hostname}_stream.conf" + ) + with open(host_stream_conf_path, "w") as fp: + fp.write(stream_conf.render()) + + ctr_stream_conf_path = self.installer.prefix / "etc/netdata/stream.conf" + + ctr = ctr.with_file( + ctr_stream_conf_path.as_posix(), + self.client.host().file(host_stream_conf_path.as_posix()), + ) + + ctr = ctr.with_exposed_port(self.endpoint.port) + + return ctr diff --git a/packaging/dag/requirements.txt b/packaging/dag/requirements.txt new file mode 100644 index 000000000..66edabfea --- /dev/null +++ b/packaging/dag/requirements.txt @@ -0,0 +1,3 @@ +click==8.1.7 +dagger-io==0.9.7 +Jinja2==3.1.4 diff --git a/packaging/dag/test_command.py b/packaging/dag/test_command.py new file mode 100644 index 000000000..2418d142e --- /dev/null +++ b/packaging/dag/test_command.py @@ -0,0 +1,128 @@ +import click +import asyncio +import sys +import pathlib +import dagger +import uuid +import httpx + +from nd import Distribution, NetdataInstaller, FeatureFlags, Endpoint, AgentContext + + +def run_async(func): + def wrapper(*args, **kwargs): + return asyncio.run(func(*args, **kwargs)) + + return wrapper + + +@run_async +async def simple_test(): + config = dagger.Config(log_output=sys.stdout) + + async with dagger.Connection(config) as client: + platform = dagger.Platform("linux/x86_64") + distro = Distribution("debian10") + + repo_root = pathlib.Path("/netdata") + prefix_path = pathlib.Path("/opt/netdata") + installer = NetdataInstaller( + platform, distro, repo_root, prefix_path, FeatureFlags.DBEngine + ) + + api_key = uuid.uuid4() + + # + # parent + # + parent_endpoint = Endpoint("parent1", 22000) + parent_ctx = AgentContext( + client, platform, distro, installer, parent_endpoint, api_key, True + ) + parent_cmd = installer.prefix / "usr/sbin/netdata" + parent_args = [ + parent_cmd.as_posix(), + "-D", + "-i", + "0.0.0.0", + "-p", + str(parent_endpoint.port), + ] + + parent_ctr = parent_ctx.build_container() + parent_ctr = parent_ctr.with_exec(parent_args) + parent_svc = parent_ctr.as_service() + + # + # child + # + child_endpoint = Endpoint("child1", 21000) + child_ctx = AgentContext( + client, platform, distro, installer, child_endpoint, api_key, False + ) + child_ctx.add_parent(parent_ctx) + child_cmd = installer.prefix / "usr/sbin/netdata" + child_args = [ + child_cmd.as_posix(), + "-D", + "-i", + "0.0.0.0", + "-p", + str(child_endpoint.port), + ] + + child_ctr = child_ctx.build_container() + child_ctr = child_ctr.with_service_binding(parent_endpoint.hostname, parent_svc) + child_ctr = child_ctr.with_exec(child_args) + child_svc = child_ctr.as_service() + + # + # endpoints + # + parent_tunnel, child_tunnel = await asyncio.gather( + client.host().tunnel(parent_svc, native=True).start(), + client.host().tunnel(child_svc, native=True).start(), + ) + + parent_endpoint, child_endpoint = await asyncio.gather( + parent_tunnel.endpoint(), + child_tunnel.endpoint(), + ) + + await asyncio.sleep(10) + + # + # run tests + # + + async with httpx.AsyncClient() as http: + resp = await http.get(f"http://{parent_endpoint}/api/v1/info") + + # + # Check that the child was connected + # + jd = resp.json() + assert ( + "hosts-available" in jd + ), "Could not find 'host-available' key in api/v1/info" + assert jd["hosts-available"] == 2, "Child did not connect to parent" + + # + # Check bearer protection + # + forbidden_urls = [ + f"http://{parent_endpoint}/api/v2/bearer_protection", + f"http://{parent_endpoint}/api/v2/bearer_get_token", + ] + + for url in forbidden_urls: + async with httpx.AsyncClient() as http: + resp = await http.get(url) + assert ( + resp.status_code == httpx.codes.UNAVAILABLE_FOR_LEGAL_REASONS + ), "Bearer protection is broken" + + +@click.command(help="Run a simple parent/child test") +def test(): + simple_test() -- cgit v1.2.3