diff options
Diffstat (limited to 'packaging/dag/nd.py')
| -rw-r--r-- | packaging/dag/nd.py | 409 |
1 files changed, 409 insertions, 0 deletions
diff --git a/packaging/dag/nd.py b/packaging/dag/nd.py new file mode 100644 index 00000000..d59adf30 --- /dev/null +++ b/packaging/dag/nd.py @@ -0,0 +1,409 @@ +from typing import List + +import enum +import os +import pathlib +import uuid + +import dagger +import jinja2 + +import imageutils + + +class Platform: + def __init__(self, platform: str): + self.platform = dagger.Platform(platform) + + def escaped(self) -> str: + return str(self.platform).removeprefix("linux/").replace("/", "_") + + def __eq__(self, other): + if isinstance(other, Platform): + return self.platform == other.platform + elif isinstance(other, dagger.Platform): + return self.platform == other + else: + return NotImplemented + + def __ne__(self, other): + return not (self == other) + + def __hash__(self): + return hash(self.platform) + + def __str__(self) -> str: + return str(self.platform) + + +SUPPORTED_PLATFORMS = set( + [ + Platform("linux/x86_64"), + Platform("linux/arm64"), + Platform("linux/i386"), + Platform("linux/arm/v7"), + Platform("linux/arm/v6"), + Platform("linux/ppc64le"), + Platform("linux/s390x"), + Platform("linux/riscv64"), + ] +) + + +SUPPORTED_DISTRIBUTIONS = set( + [ + "alpine_3_18", + "alpine_3_19", + "amazonlinux2", + "centos7", + "centos-stream8", + "centos-stream9", + "debian10", + "debian11", + "debian12", + "fedora37", + "fedora38", + "fedora39", + "opensuse15.4", + "opensuse15.5", + "opensusetumbleweed", + "oraclelinux8", + "oraclelinux9", + "rockylinux8", + "rockylinux9", + "ubuntu20.04", + "ubuntu22.04", + "ubuntu23.04", + "ubuntu23.10", + ] +) + + +class Distribution: + def __init__(self, display_name): + self.display_name = display_name + + if self.display_name == "alpine_3_18": + self.docker_tag = "alpine:3.18" + self.builder = imageutils.build_alpine_3_18 + self.platforms = SUPPORTED_PLATFORMS + elif self.display_name == "alpine_3_19": + self.docker_tag = "alpine:3.19" + self.builder = imageutils.build_alpine_3_19 + self.platforms = SUPPORTED_PLATFORMS + elif self.display_name == "amazonlinux2": + self.docker_tag = "amazonlinux:2" + self.builder = imageutils.build_amazon_linux_2 + self.platforms = SUPPORTED_PLATFORMS + elif self.display_name == "centos7": + self.docker_tag = "centos:7" + self.builder = imageutils.build_centos_7 + self.platforms = SUPPORTED_PLATFORMS + elif self.display_name == "centos-stream8": + self.docker_tag = "quay.io/centos/centos:stream8" + self.builder = imageutils.build_centos_stream_8 + self.platforms = SUPPORTED_PLATFORMS + elif self.display_name == "centos-stream9": + self.docker_tag = "quay.io/centos/centos:stream9" + self.builder = imageutils.build_centos_stream_9 + self.platforms = SUPPORTED_PLATFORMS + elif self.display_name == "debian10": + self.docker_tag = "debian:10" + self.builder = imageutils.build_debian_10 + self.platforms = SUPPORTED_PLATFORMS + elif self.display_name == "debian11": + self.docker_tag = "debian:11" + self.builder = imageutils.build_debian_11 + self.platforms = SUPPORTED_PLATFORMS + elif self.display_name == "debian12": + self.docker_tag = "debian:12" + self.builder = imageutils.build_debian_12 + self.platforms = SUPPORTED_PLATFORMS + elif self.display_name == "fedora37": + self.docker_tag = "fedora:37" + self.builder = imageutils.build_fedora_37 + self.platforms = SUPPORTED_PLATFORMS + elif self.display_name == "fedora38": + self.docker_tag = "fedora:38" + self.builder = imageutils.build_fedora_38 + self.platforms = SUPPORTED_PLATFORMS + elif self.display_name == "fedora39": + self.docker_tag = "fedora:39" + self.platforms = SUPPORTED_PLATFORMS + self.builder = imageutils.build_fedora_39 + elif self.display_name == "opensuse15.4": + self.docker_tag = "opensuse/leap:15.4" + self.builder = imageutils.build_opensuse_15_4 + self.platforms = SUPPORTED_PLATFORMS + elif self.display_name == "opensuse15.5": + self.docker_tag = "opensuse/leap:15.5" + self.builder = imageutils.build_opensuse_15_5 + self.platforms = SUPPORTED_PLATFORMS + elif self.display_name == "opensusetumbleweed": + self.docker_tag = "opensuse/tumbleweed:latest" + self.builder = imageutils.build_opensuse_tumbleweed + self.platforms = SUPPORTED_PLATFORMS + elif self.display_name == "oraclelinux8": + self.docker_tag = "oraclelinux:8" + self.builder = imageutils.build_oracle_linux_8 + self.platforms = SUPPORTED_PLATFORMS + elif self.display_name == "oraclelinux9": + self.docker_tag = "oraclelinux:9" + self.builder = imageutils.build_oracle_linux_9 + self.platforms = SUPPORTED_PLATFORMS + elif self.display_name == "rockylinux8": + self.docker_tag = "rockylinux:8" + self.builder = imageutils.build_rocky_linux_8 + self.platforms = SUPPORTED_PLATFORMS + elif self.display_name == "rockylinux9": + self.docker_tag = "rockylinux:9" + self.builder = imageutils.build_rocky_linux_9 + self.platforms = SUPPORTED_PLATFORMS + elif self.display_name == "ubuntu20.04": + self.docker_tag = "ubuntu:20.04" + self.builder = imageutils.build_ubuntu_20_04 + self.platforms = SUPPORTED_PLATFORMS + elif self.display_name == "ubuntu22.04": + self.docker_tag = "ubuntu:22.04" + self.builder = imageutils.build_ubuntu_22_04 + self.platforms = SUPPORTED_PLATFORMS + elif self.display_name == "ubuntu23.04": + self.docker_tag = "ubuntu:23.04" + self.builder = imageutils.build_ubuntu_23_04 + self.platforms = SUPPORTED_PLATFORMS + elif self.display_name == "ubuntu23.10": + self.docker_tag = "ubuntu:23.10" + self.builder = imageutils.build_ubuntu_23_10 + self.platforms = SUPPORTED_PLATFORMS + else: + raise ValueError(f"Unknown distribution: {self.display_name}") + + def _cache_volume( + self, client: dagger.Client, platform: dagger.Platform, path: str + ) -> dagger.CacheVolume: + tag = "_".join([self.display_name, Platform(platform).escaped()]) + return client.cache_volume(f"{path}-{tag}") + + def build( + self, client: dagger.Client, platform: dagger.Platform + ) -> dagger.Container: + if platform not in self.platforms: + raise ValueError( + f"Building {self.display_name} is not supported on {platform}." + ) + + ctr = self.builder(client, platform) + ctr = imageutils.install_cargo(ctr) + + return ctr + + +class FeatureFlags(enum.Flag): + DBEngine = enum.auto() + GoPlugin = enum.auto() + ExtendedBPF = enum.auto() + LogsManagement = enum.auto() + MachineLearning = enum.auto() + BundledProtobuf = enum.auto() + + +class NetdataInstaller: + def __init__( + self, + platform: Platform, + distro: Distribution, + repo_root: pathlib.Path, + prefix: pathlib.Path, + features: FeatureFlags, + ): + self.platform = platform + self.distro = distro + self.repo_root = repo_root + self.prefix = prefix + self.features = features + + def _mount_repo( + self, client: dagger.Client, ctr: dagger.Container, repo_root: pathlib.Path + ) -> dagger.Container: + host_repo_root = pathlib.Path(__file__).parent.parent.parent.as_posix() + exclude_dirs = ["build", "fluent-bit/build", "packaging/dag"] + + # The installer builds/stores intermediate artifacts under externaldeps/ + # We add a volume to speed up rebuilds. The volume has to be unique + # per platform/distro in order to avoid mixing unrelated artifacts + # together. + externaldeps = self.distro._cache_volume(client, self.platform, "externaldeps") + + ctr = ( + ctr.with_directory( + self.repo_root.as_posix(), client.host().directory(host_repo_root) + ) + .with_workdir(self.repo_root.as_posix()) + .with_mounted_cache( + os.path.join(self.repo_root, "externaldeps"), externaldeps + ) + ) + + return ctr + + def install(self, client: dagger.Client, ctr: dagger.Container) -> dagger.Container: + args = ["--dont-wait", "--dont-start-it", "--disable-telemetry"] + + if FeatureFlags.DBEngine not in self.features: + args.append("--disable-dbengine") + + if FeatureFlags.GoPlugin not in self.features: + args.append("--disable-go") + + if FeatureFlags.ExtendedBPF not in self.features: + args.append("--disable-ebpf") + + if FeatureFlags.LogsManagement not in self.features: + args.append("--disable-logsmanagement") + + if FeatureFlags.MachineLearning not in self.features: + args.append("--disable-ml") + + if FeatureFlags.BundledProtobuf not in self.features: + args.append("--use-system-protobuf") + + args.extend(["--install-prefix", self.prefix.parent.as_posix()]) + + ctr = self._mount_repo(client, ctr, self.repo_root.as_posix()) + + ctr = ctr.with_env_variable( + "NETDATA_CMAKE_OPTIONS", "-DCMAKE_BUILD_TYPE=Debug" + ).with_exec(["./netdata-installer.sh"] + args) + + return ctr + + +class Endpoint: + def __init__(self, hostname: str, port: int): + self.hostname = hostname + self.port = port + + def __str__(self): + return ":".join([self.hostname, str(self.port)]) + + +class ChildStreamConf: + def __init__( + self, + installer: NetdataInstaller, + destinations: List[Endpoint], + api_key: uuid.UUID, + ): + self.installer = installer + self.substitutions = { + "enabled": "yes", + "destination": " ".join([str(dst) for dst in destinations]), + "api_key": api_key, + "timeout_seconds": 60, + "default_port": 19999, + "send_charts_matching": "*", + "buffer_size_bytes": 1024 * 1024, + "reconnect_delay_seconds": 5, + "initial_clock_resync_iterations": 60, + } + + def render(self) -> str: + tmpl_path = pathlib.Path(__file__).parent / "files/child_stream.conf" + with open(tmpl_path) as fp: + tmpl = jinja2.Template(fp.read()) + + return tmpl.render(**self.substitutions) + + +class ParentStreamConf: + def __init__(self, installer: NetdataInstaller, api_key: uuid.UUID): + self.installer = installer + self.substitutions = { + "api_key": str(api_key), + "enabled": "yes", + "allow_from": "*", + "default_history": 3600, + "health_enabled_by_default": "auto", + "default_postpone_alarms_on_connect_seconds": 60, + "multiple_connections": "allow", + } + + def render(self) -> str: + tmpl_path = pathlib.Path(__file__).parent / "files/parent_stream.conf" + with open(tmpl_path) as fp: + tmpl = jinja2.Template(fp.read()) + + return tmpl.render(**self.substitutions) + + +class StreamConf: + def __init__(self, child_conf: ChildStreamConf, parent_conf: ParentStreamConf): + self.child_conf = child_conf + self.parent_conf = parent_conf + + def render(self) -> str: + child_section = self.child_conf.render() if self.child_conf else "" + parent_section = self.parent_conf.render() if self.parent_conf else "" + return "\n".join([child_section, parent_section]) + + +class AgentContext: + def __init__( + self, + client: dagger.Client, + platform: dagger.Platform, + distro: Distribution, + installer: NetdataInstaller, + endpoint: Endpoint, + api_key: uuid.UUID, + allow_children: bool, + ): + self.client = client + self.platform = platform + self.distro = distro + self.installer = installer + self.endpoint = endpoint + self.api_key = api_key + self.allow_children = allow_children + + self.parent_contexts = [] + + self.built_distro = False + self.built_agent = False + + def add_parent(self, parent_context: "AgentContext"): + self.parent_contexts.append(parent_context) + + def build_container(self) -> dagger.Container: + ctr = self.distro.build(self.client, self.platform) + ctr = self.installer.install(self.client, ctr) + + if len(self.parent_contexts) == 0 and not self.allow_children: + return ctr.with_exposed_port(self.endpoint.port) + + destinations = [parent_ctx.endpoint for parent_ctx in self.parent_contexts] + child_stream_conf = ChildStreamConf(self.installer, destinations, self.api_key) + + parent_stream_conf = None + if self.allow_children: + parent_stream_conf = ParentStreamConf(self.installer, self.api_key) + + stream_conf = StreamConf(child_stream_conf, parent_stream_conf) + + # write the stream conf to localhost and cp it in the container + host_stream_conf_path = pathlib.Path( + f"/tmp/{self.endpoint.hostname}_stream.conf" + ) + with open(host_stream_conf_path, "w") as fp: + fp.write(stream_conf.render()) + + ctr_stream_conf_path = self.installer.prefix / "etc/netdata/stream.conf" + + ctr = ctr.with_file( + ctr_stream_conf_path.as_posix(), + self.client.host().file(host_stream_conf_path.as_posix()), + ) + + ctr = ctr.with_exposed_port(self.endpoint.port) + + return ctr |