Merging upstream version 1.45.0.

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

1 files changed, 409 insertions, 0 deletions

diff --git a/packaging/dag/nd.py b/packaging/dag/nd.py
new file mode 100644
index 000000000..d59adf30a
--- /dev/null
+++ b/packaging/dag/nd.py
@@ -0,0 +1,409 @@
+from typing import List
+
+import enum
+import os
+import pathlib
+import uuid
+
+import dagger
+import jinja2
+
+import imageutils
+
+
+class Platform:
+    def __init__(self, platform: str):
+        self.platform = dagger.Platform(platform)
+
+    def escaped(self) -> str:
+        return str(self.platform).removeprefix("linux/").replace("/", "_")
+
+    def __eq__(self, other):
+        if isinstance(other, Platform):
+            return self.platform == other.platform
+        elif isinstance(other, dagger.Platform):
+            return self.platform == other
+        else:
+            return NotImplemented
+
+    def __ne__(self, other):
+        return not (self == other)
+
+    def __hash__(self):
+        return hash(self.platform)
+
+    def __str__(self) -> str:
+        return str(self.platform)
+
+
+SUPPORTED_PLATFORMS = set(
+    [
+        Platform("linux/x86_64"),
+        Platform("linux/arm64"),
+        Platform("linux/i386"),
+        Platform("linux/arm/v7"),
+        Platform("linux/arm/v6"),
+        Platform("linux/ppc64le"),
+        Platform("linux/s390x"),
+        Platform("linux/riscv64"),
+    ]
+)
+
+
+SUPPORTED_DISTRIBUTIONS = set(
+    [
+        "alpine_3_18",
+        "alpine_3_19",
+        "amazonlinux2",
+        "centos7",
+        "centos-stream8",
+        "centos-stream9",
+        "debian10",
+        "debian11",
+        "debian12",
+        "fedora37",
+        "fedora38",
+        "fedora39",
+        "opensuse15.4",
+        "opensuse15.5",
+        "opensusetumbleweed",
+        "oraclelinux8",
+        "oraclelinux9",
+        "rockylinux8",
+        "rockylinux9",
+        "ubuntu20.04",
+        "ubuntu22.04",
+        "ubuntu23.04",
+        "ubuntu23.10",
+    ]
+)
+
+
+class Distribution:
+    def __init__(self, display_name):
+        self.display_name = display_name
+
+        if self.display_name == "alpine_3_18":
+            self.docker_tag = "alpine:3.18"
+            self.builder = imageutils.build_alpine_3_18
+            self.platforms = SUPPORTED_PLATFORMS
+        elif self.display_name == "alpine_3_19":
+            self.docker_tag = "alpine:3.19"
+            self.builder = imageutils.build_alpine_3_19
+            self.platforms = SUPPORTED_PLATFORMS
+        elif self.display_name == "amazonlinux2":
+            self.docker_tag = "amazonlinux:2"
+            self.builder = imageutils.build_amazon_linux_2
+            self.platforms = SUPPORTED_PLATFORMS
+        elif self.display_name == "centos7":
+            self.docker_tag = "centos:7"
+            self.builder = imageutils.build_centos_7
+            self.platforms = SUPPORTED_PLATFORMS
+        elif self.display_name == "centos-stream8":
+            self.docker_tag = "quay.io/centos/centos:stream8"
+            self.builder = imageutils.build_centos_stream_8
+            self.platforms = SUPPORTED_PLATFORMS
+        elif self.display_name == "centos-stream9":
+            self.docker_tag = "quay.io/centos/centos:stream9"
+            self.builder = imageutils.build_centos_stream_9
+            self.platforms = SUPPORTED_PLATFORMS
+        elif self.display_name == "debian10":
+            self.docker_tag = "debian:10"
+            self.builder = imageutils.build_debian_10
+            self.platforms = SUPPORTED_PLATFORMS
+        elif self.display_name == "debian11":
+            self.docker_tag = "debian:11"
+            self.builder = imageutils.build_debian_11
+            self.platforms = SUPPORTED_PLATFORMS
+        elif self.display_name == "debian12":
+            self.docker_tag = "debian:12"
+            self.builder = imageutils.build_debian_12
+            self.platforms = SUPPORTED_PLATFORMS
+        elif self.display_name == "fedora37":
+            self.docker_tag = "fedora:37"
+            self.builder = imageutils.build_fedora_37
+            self.platforms = SUPPORTED_PLATFORMS
+        elif self.display_name == "fedora38":
+            self.docker_tag = "fedora:38"
+            self.builder = imageutils.build_fedora_38
+            self.platforms = SUPPORTED_PLATFORMS
+        elif self.display_name == "fedora39":
+            self.docker_tag = "fedora:39"
+            self.platforms = SUPPORTED_PLATFORMS
+            self.builder = imageutils.build_fedora_39
+        elif self.display_name == "opensuse15.4":
+            self.docker_tag = "opensuse/leap:15.4"
+            self.builder = imageutils.build_opensuse_15_4
+            self.platforms = SUPPORTED_PLATFORMS
+        elif self.display_name == "opensuse15.5":
+            self.docker_tag = "opensuse/leap:15.5"
+            self.builder = imageutils.build_opensuse_15_5
+            self.platforms = SUPPORTED_PLATFORMS
+        elif self.display_name == "opensusetumbleweed":
+            self.docker_tag = "opensuse/tumbleweed:latest"
+            self.builder = imageutils.build_opensuse_tumbleweed
+            self.platforms = SUPPORTED_PLATFORMS
+        elif self.display_name == "oraclelinux8":
+            self.docker_tag = "oraclelinux:8"
+            self.builder = imageutils.build_oracle_linux_8
+            self.platforms = SUPPORTED_PLATFORMS
+        elif self.display_name == "oraclelinux9":
+            self.docker_tag = "oraclelinux:9"
+            self.builder = imageutils.build_oracle_linux_9
+            self.platforms = SUPPORTED_PLATFORMS
+        elif self.display_name == "rockylinux8":
+            self.docker_tag = "rockylinux:8"
+            self.builder = imageutils.build_rocky_linux_8
+            self.platforms = SUPPORTED_PLATFORMS
+        elif self.display_name == "rockylinux9":
+            self.docker_tag = "rockylinux:9"
+            self.builder = imageutils.build_rocky_linux_9
+            self.platforms = SUPPORTED_PLATFORMS
+        elif self.display_name == "ubuntu20.04":
+            self.docker_tag = "ubuntu:20.04"
+            self.builder = imageutils.build_ubuntu_20_04
+            self.platforms = SUPPORTED_PLATFORMS
+        elif self.display_name == "ubuntu22.04":
+            self.docker_tag = "ubuntu:22.04"
+            self.builder = imageutils.build_ubuntu_22_04
+            self.platforms = SUPPORTED_PLATFORMS
+        elif self.display_name == "ubuntu23.04":
+            self.docker_tag = "ubuntu:23.04"
+            self.builder = imageutils.build_ubuntu_23_04
+            self.platforms = SUPPORTED_PLATFORMS
+        elif self.display_name == "ubuntu23.10":
+            self.docker_tag = "ubuntu:23.10"
+            self.builder = imageutils.build_ubuntu_23_10
+            self.platforms = SUPPORTED_PLATFORMS
+        else:
+            raise ValueError(f"Unknown distribution: {self.display_name}")
+
+    def _cache_volume(
+        self, client: dagger.Client, platform: dagger.Platform, path: str
+    ) -> dagger.CacheVolume:
+        tag = "_".join([self.display_name, Platform(platform).escaped()])
+        return client.cache_volume(f"{path}-{tag}")
+
+    def build(
+        self, client: dagger.Client, platform: dagger.Platform
+    ) -> dagger.Container:
+        if platform not in self.platforms:
+            raise ValueError(
+                f"Building {self.display_name} is not supported on {platform}."
+            )
+
+        ctr = self.builder(client, platform)
+        ctr = imageutils.install_cargo(ctr)
+
+        return ctr
+
+
+class FeatureFlags(enum.Flag):
+    DBEngine = enum.auto()
+    GoPlugin = enum.auto()
+    ExtendedBPF = enum.auto()
+    LogsManagement = enum.auto()
+    MachineLearning = enum.auto()
+    BundledProtobuf = enum.auto()
+
+
+class NetdataInstaller:
+    def __init__(
+        self,
+        platform: Platform,
+        distro: Distribution,
+        repo_root: pathlib.Path,
+        prefix: pathlib.Path,
+        features: FeatureFlags,
+    ):
+        self.platform = platform
+        self.distro = distro
+        self.repo_root = repo_root
+        self.prefix = prefix
+        self.features = features
+
+    def _mount_repo(
+        self, client: dagger.Client, ctr: dagger.Container, repo_root: pathlib.Path
+    ) -> dagger.Container:
+        host_repo_root = pathlib.Path(__file__).parent.parent.parent.as_posix()
+        exclude_dirs = ["build", "fluent-bit/build", "packaging/dag"]
+
+        # The installer builds/stores intermediate artifacts under externaldeps/
+        # We add a volume to speed up rebuilds. The volume has to be unique
+        # per platform/distro in order to avoid mixing unrelated artifacts
+        # together.
+        externaldeps = self.distro._cache_volume(client, self.platform, "externaldeps")
+
+        ctr = (
+            ctr.with_directory(
+                self.repo_root.as_posix(), client.host().directory(host_repo_root)
+            )
+            .with_workdir(self.repo_root.as_posix())
+            .with_mounted_cache(
+                os.path.join(self.repo_root, "externaldeps"), externaldeps
+            )
+        )
+
+        return ctr
+
+    def install(self, client: dagger.Client, ctr: dagger.Container) -> dagger.Container:
+        args = ["--dont-wait", "--dont-start-it", "--disable-telemetry"]
+
+        if FeatureFlags.DBEngine not in self.features:
+            args.append("--disable-dbengine")
+
+        if FeatureFlags.GoPlugin not in self.features:
+            args.append("--disable-go")
+
+        if FeatureFlags.ExtendedBPF not in self.features:
+            args.append("--disable-ebpf")
+
+        if FeatureFlags.LogsManagement not in self.features:
+            args.append("--disable-logsmanagement")
+
+        if FeatureFlags.MachineLearning not in self.features:
+            args.append("--disable-ml")
+
+        if FeatureFlags.BundledProtobuf not in self.features:
+            args.append("--use-system-protobuf")
+
+        args.extend(["--install-prefix", self.prefix.parent.as_posix()])
+
+        ctr = self._mount_repo(client, ctr, self.repo_root.as_posix())
+
+        ctr = ctr.with_env_variable(
+            "NETDATA_CMAKE_OPTIONS", "-DCMAKE_BUILD_TYPE=Debug"
+        ).with_exec(["./netdata-installer.sh"] + args)
+
+        return ctr
+
+
+class Endpoint:
+    def __init__(self, hostname: str, port: int):
+        self.hostname = hostname
+        self.port = port
+
+    def __str__(self):
+        return ":".join([self.hostname, str(self.port)])
+
+
+class ChildStreamConf:
+    def __init__(
+        self,
+        installer: NetdataInstaller,
+        destinations: List[Endpoint],
+        api_key: uuid.UUID,
+    ):
+        self.installer = installer
+        self.substitutions = {
+            "enabled": "yes",
+            "destination": " ".join([str(dst) for dst in destinations]),
+            "api_key": api_key,
+            "timeout_seconds": 60,
+            "default_port": 19999,
+            "send_charts_matching": "*",
+            "buffer_size_bytes": 1024 * 1024,
+            "reconnect_delay_seconds": 5,
+            "initial_clock_resync_iterations": 60,
+        }
+
+    def render(self) -> str:
+        tmpl_path = pathlib.Path(__file__).parent / "files/child_stream.conf"
+        with open(tmpl_path) as fp:
+            tmpl = jinja2.Template(fp.read())
+
+        return tmpl.render(**self.substitutions)
+
+
+class ParentStreamConf:
+    def __init__(self, installer: NetdataInstaller, api_key: uuid.UUID):
+        self.installer = installer
+        self.substitutions = {
+            "api_key": str(api_key),
+            "enabled": "yes",
+            "allow_from": "*",
+            "default_history": 3600,
+            "health_enabled_by_default": "auto",
+            "default_postpone_alarms_on_connect_seconds": 60,
+            "multiple_connections": "allow",
+        }
+
+    def render(self) -> str:
+        tmpl_path = pathlib.Path(__file__).parent / "files/parent_stream.conf"
+        with open(tmpl_path) as fp:
+            tmpl = jinja2.Template(fp.read())
+
+        return tmpl.render(**self.substitutions)
+
+
+class StreamConf:
+    def __init__(self, child_conf: ChildStreamConf, parent_conf: ParentStreamConf):
+        self.child_conf = child_conf
+        self.parent_conf = parent_conf
+
+    def render(self) -> str:
+        child_section = self.child_conf.render() if self.child_conf else ""
+        parent_section = self.parent_conf.render() if self.parent_conf else ""
+        return "\n".join([child_section, parent_section])
+
+
+class AgentContext:
+    def __init__(
+        self,
+        client: dagger.Client,
+        platform: dagger.Platform,
+        distro: Distribution,
+        installer: NetdataInstaller,
+        endpoint: Endpoint,
+        api_key: uuid.UUID,
+        allow_children: bool,
+    ):
+        self.client = client
+        self.platform = platform
+        self.distro = distro
+        self.installer = installer
+        self.endpoint = endpoint
+        self.api_key = api_key
+        self.allow_children = allow_children
+
+        self.parent_contexts = []
+
+        self.built_distro = False
+        self.built_agent = False
+
+    def add_parent(self, parent_context: "AgentContext"):
+        self.parent_contexts.append(parent_context)
+
+    def build_container(self) -> dagger.Container:
+        ctr = self.distro.build(self.client, self.platform)
+        ctr = self.installer.install(self.client, ctr)
+
+        if len(self.parent_contexts) == 0 and not self.allow_children:
+            return ctr.with_exposed_port(self.endpoint.port)
+
+        destinations = [parent_ctx.endpoint for parent_ctx in self.parent_contexts]
+        child_stream_conf = ChildStreamConf(self.installer, destinations, self.api_key)
+
+        parent_stream_conf = None
+        if self.allow_children:
+            parent_stream_conf = ParentStreamConf(self.installer, self.api_key)
+
+        stream_conf = StreamConf(child_stream_conf, parent_stream_conf)
+
+        # write the stream conf to localhost and cp it in the container
+        host_stream_conf_path = pathlib.Path(
+            f"/tmp/{self.endpoint.hostname}_stream.conf"
+        )
+        with open(host_stream_conf_path, "w") as fp:
+            fp.write(stream_conf.render())
+
+        ctr_stream_conf_path = self.installer.prefix / "etc/netdata/stream.conf"
+
+        ctr = ctr.with_file(
+            ctr_stream_conf_path.as_posix(),
+            self.client.host().file(host_stream_conf_path.as_posix()),
+        )
+
+        ctr = ctr.with_exposed_port(self.endpoint.port)
+
+        return ctr