diff options
author | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-04-07 18:45:59 +0000 |
---|---|---|
committer | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-04-07 18:45:59 +0000 |
commit | 19fcec84d8d7d21e796c7624e521b60d28ee21ed (patch) | |
tree | 42d26aa27d1e3f7c0b8bd3fd14e7d7082f5008dc /src/cephadm | |
parent | Initial commit. (diff) | |
download | ceph-19fcec84d8d7d21e796c7624e521b60d28ee21ed.tar.xz ceph-19fcec84d8d7d21e796c7624e521b60d28ee21ed.zip |
Adding upstream version 16.2.11+ds.upstream/16.2.11+dsupstream
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to '')
-rw-r--r-- | src/cephadm/CMakeLists.txt | 4 | ||||
-rwxr-xr-x | src/cephadm/cephadm | 9248 | ||||
-rw-r--r-- | src/cephadm/samples/alertmanager.json | 27 | ||||
-rw-r--r-- | src/cephadm/samples/custom_container.json | 35 | ||||
-rw-r--r-- | src/cephadm/samples/grafana.json | 90 | ||||
-rw-r--r-- | src/cephadm/samples/nfs.json | 14 | ||||
-rw-r--r-- | src/cephadm/samples/prometheus.json | 17 | ||||
-rw-r--r-- | src/cephadm/samples/rgw_ssl.json | 101 | ||||
-rw-r--r-- | src/cephadm/tests/__init__.py | 0 | ||||
-rw-r--r-- | src/cephadm/tests/fixtures.py | 126 | ||||
-rw-r--r-- | src/cephadm/tests/test_cephadm.py | 2413 | ||||
-rw-r--r-- | src/cephadm/tests/test_networks.py | 235 | ||||
-rw-r--r-- | src/cephadm/tox.ini | 62 | ||||
-rwxr-xr-x | src/cephadm/vstart-cleanup.sh | 6 | ||||
-rwxr-xr-x | src/cephadm/vstart-smoke.sh | 86 |
15 files changed, 12464 insertions, 0 deletions
diff --git a/src/cephadm/CMakeLists.txt b/src/cephadm/CMakeLists.txt new file mode 100644 index 000000000..13be09bae --- /dev/null +++ b/src/cephadm/CMakeLists.txt @@ -0,0 +1,4 @@ +if(WITH_TESTS) + include(AddCephTest) + add_tox_test(cephadm TOX_ENVS py3 mypy flake8) +endif() diff --git a/src/cephadm/cephadm b/src/cephadm/cephadm new file mode 100755 index 000000000..d69655946 --- /dev/null +++ b/src/cephadm/cephadm @@ -0,0 +1,9248 @@ +#!/usr/bin/python3 + +import asyncio +import asyncio.subprocess +import argparse +import datetime +import fcntl +import ipaddress +import json +import logging +from logging.config import dictConfig +import os +import platform +import pwd +import random +import shlex +import shutil +import socket +import string +import subprocess +import sys +import tempfile +import time +import errno +import struct +from socketserver import ThreadingMixIn +from http.server import BaseHTTPRequestHandler, HTTPServer +import signal +import io +from contextlib import redirect_stdout +import ssl +from enum import Enum + +from typing import Dict, List, Tuple, Optional, Union, Any, NoReturn, Callable, IO, Sequence, TypeVar, cast, Set, Iterable + +import re +import uuid + +from configparser import ConfigParser +from functools import wraps +from glob import glob +from io import StringIO +from threading import Thread, RLock +from urllib.error import HTTPError, URLError +from urllib.request import urlopen +from pathlib import Path + +FuncT = TypeVar('FuncT', bound=Callable) + +# Default container images ----------------------------------------------------- +DEFAULT_IMAGE = 'quay.io/ceph/ceph:v16' +DEFAULT_IMAGE_IS_MASTER = False +DEFAULT_IMAGE_RELEASE = 'pacific' +DEFAULT_PROMETHEUS_IMAGE = 'quay.io/prometheus/prometheus:v2.33.4' +DEFAULT_NODE_EXPORTER_IMAGE = 'quay.io/prometheus/node-exporter:v1.3.1' +DEFAULT_ALERT_MANAGER_IMAGE = 'quay.io/prometheus/alertmanager:v0.23.0' +DEFAULT_GRAFANA_IMAGE = 'quay.io/ceph/ceph-grafana:8.3.5' +DEFAULT_HAPROXY_IMAGE = 'docker.io/library/haproxy:2.3' +DEFAULT_KEEPALIVED_IMAGE = 'docker.io/arcts/keepalived' +DEFAULT_SNMP_GATEWAY_IMAGE = 'docker.io/maxwo/snmp-notifier:v1.2.1' +DEFAULT_REGISTRY = 'docker.io' # normalize unqualified digests to this +# ------------------------------------------------------------------------------ + +LATEST_STABLE_RELEASE = 'pacific' +DATA_DIR = '/var/lib/ceph' +LOG_DIR = '/var/log/ceph' +LOCK_DIR = '/run/cephadm' +LOGROTATE_DIR = '/etc/logrotate.d' +SYSCTL_DIR = '/etc/sysctl.d' +UNIT_DIR = '/etc/systemd/system' +LOG_DIR_MODE = 0o770 +DATA_DIR_MODE = 0o700 +CONTAINER_INIT = True +MIN_PODMAN_VERSION = (2, 0, 2) +CGROUPS_SPLIT_PODMAN_VERSION = (2, 1, 0) +CUSTOM_PS1 = r'[ceph: \u@\h \W]\$ ' +DEFAULT_TIMEOUT = None # in seconds +DEFAULT_RETRY = 15 +SHELL_DEFAULT_CONF = '/etc/ceph/ceph.conf' +SHELL_DEFAULT_KEYRING = '/etc/ceph/ceph.client.admin.keyring' +DATEFMT = '%Y-%m-%dT%H:%M:%S.%fZ' + +logger: logging.Logger = None # type: ignore + +""" +You can invoke cephadm in two ways: + +1. The normal way, at the command line. + +2. By piping the script to the python3 binary. In this latter case, you should + prepend one or more lines to the beginning of the script. + + For arguments, + + injected_argv = [...] + + e.g., + + injected_argv = ['ls'] + + For reading stdin from the '--config-json -' argument, + + injected_stdin = '...' +""" +cached_stdin = None + + +################################## + + +async def run_func(func: Callable, cmd: str) -> subprocess.CompletedProcess: + logger.debug(f'running function {func.__name__}, with parms: {cmd}') + response = func(cmd) + return response + + +async def concurrent_tasks(func: Callable, cmd_list: List[str]) -> List[Any]: + tasks = [] + for cmd in cmd_list: + tasks.append(run_func(func, cmd)) + + data = await asyncio.gather(*tasks) + + return data + + +class EndPoint: + """EndPoint representing an ip:port format""" + + def __init__(self, ip: str, port: int) -> None: + self.ip = ip + self.port = port + + def __str__(self) -> str: + return f'{self.ip}:{self.port}' + + def __repr__(self) -> str: + return f'{self.ip}:{self.port}' + + +class BaseConfig: + + def __init__(self) -> None: + self.image: str = '' + self.docker: bool = False + self.data_dir: str = DATA_DIR + self.log_dir: str = LOG_DIR + self.logrotate_dir: str = LOGROTATE_DIR + self.sysctl_dir: str = SYSCTL_DIR + self.unit_dir: str = UNIT_DIR + self.verbose: bool = False + self.timeout: Optional[int] = DEFAULT_TIMEOUT + self.retry: int = DEFAULT_RETRY + self.env: List[str] = [] + self.memory_request: Optional[int] = None + self.memory_limit: Optional[int] = None + + self.container_init: bool = CONTAINER_INIT + self.container_engine: Optional[ContainerEngine] = None + + def set_from_args(self, args: argparse.Namespace) -> None: + argdict: Dict[str, Any] = vars(args) + for k, v in argdict.items(): + if hasattr(self, k): + setattr(self, k, v) + + +class CephadmContext: + + def __init__(self) -> None: + self.__dict__['_args'] = None + self.__dict__['_conf'] = BaseConfig() + + def set_args(self, args: argparse.Namespace) -> None: + self._conf.set_from_args(args) + self._args = args + + def has_function(self) -> bool: + return 'func' in self._args + + def __contains__(self, name: str) -> bool: + return hasattr(self, name) + + def __getattr__(self, name: str) -> Any: + if '_conf' in self.__dict__ and hasattr(self._conf, name): + return getattr(self._conf, name) + elif '_args' in self.__dict__ and hasattr(self._args, name): + return getattr(self._args, name) + else: + return super().__getattribute__(name) + + def __setattr__(self, name: str, value: Any) -> None: + if hasattr(self._conf, name): + setattr(self._conf, name, value) + elif hasattr(self._args, name): + setattr(self._args, name, value) + else: + super().__setattr__(name, value) + + +class ContainerEngine: + def __init__(self) -> None: + self.path = find_program(self.EXE) + + @classmethod + @property + def EXE(cls) -> str: + raise NotImplementedError() + + def __str__(self) -> str: + return f'{self.EXE} ({self.path})' + + +class Podman(ContainerEngine): + EXE = 'podman' + + def __init__(self) -> None: + super().__init__() + self._version: Optional[Tuple[int, ...]] = None + + @property + def version(self) -> Tuple[int, ...]: + if self._version is None: + raise RuntimeError('Please call `get_version` first') + return self._version + + def get_version(self, ctx: CephadmContext) -> None: + out, _, _ = call_throws(ctx, [self.path, 'version', '--format', '{{.Client.Version}}']) + self._version = _parse_podman_version(out) + + def __str__(self) -> str: + version = '.'.join(map(str, self.version)) + return f'{self.EXE} ({self.path}) version {version}' + + +class Docker(ContainerEngine): + EXE = 'docker' + + +CONTAINER_PREFERENCE = (Podman, Docker) # prefer podman to docker + + +# Log and console output config +logging_config = { + 'version': 1, + 'disable_existing_loggers': True, + 'formatters': { + 'cephadm': { + 'format': '%(asctime)s %(thread)x %(levelname)s %(message)s' + }, + }, + 'handlers': { + 'console': { + 'level': 'INFO', + 'class': 'logging.StreamHandler', + }, + 'log_file': { + 'level': 'DEBUG', + 'class': 'logging.handlers.WatchedFileHandler', + 'formatter': 'cephadm', + 'filename': '%s/cephadm.log' % LOG_DIR, + } + }, + 'loggers': { + '': { + 'level': 'DEBUG', + 'handlers': ['console', 'log_file'], + } + } +} + + +class termcolor: + yellow = '\033[93m' + red = '\033[31m' + end = '\033[0m' + + +class Error(Exception): + pass + + +class TimeoutExpired(Error): + pass + + +class UnauthorizedRegistryError(Error): + pass + +################################## + + +class Ceph(object): + daemons = ('mon', 'mgr', 'mds', 'osd', 'rgw', 'rbd-mirror', + 'crash', 'cephfs-mirror') + +################################## + + +class OSD(object): + @staticmethod + def get_sysctl_settings() -> List[str]: + return [ + '# allow a large number of OSDs', + 'fs.aio-max-nr = 1048576', + 'kernel.pid_max = 4194304', + ] + + +################################## + + +class SNMPGateway: + """Defines an SNMP gateway between Prometheus and SNMP monitoring Frameworks""" + daemon_type = 'snmp-gateway' + SUPPORTED_VERSIONS = ['V2c', 'V3'] + default_image = DEFAULT_SNMP_GATEWAY_IMAGE + DEFAULT_PORT = 9464 + env_filename = 'snmp-gateway.conf' + + def __init__(self, + ctx: CephadmContext, + fsid: str, + daemon_id: Union[int, str], + config_json: Dict[str, Any], + image: Optional[str] = None) -> None: + self.ctx = ctx + self.fsid = fsid + self.daemon_id = daemon_id + self.image = image or SNMPGateway.default_image + + self.uid = config_json.get('uid', 0) + self.gid = config_json.get('gid', 0) + + self.destination = config_json.get('destination', '') + self.snmp_version = config_json.get('snmp_version', 'V2c') + self.snmp_community = config_json.get('snmp_community', 'public') + self.log_level = config_json.get('log_level', 'info') + self.snmp_v3_auth_username = config_json.get('snmp_v3_auth_username', '') + self.snmp_v3_auth_password = config_json.get('snmp_v3_auth_password', '') + self.snmp_v3_auth_protocol = config_json.get('snmp_v3_auth_protocol', '') + self.snmp_v3_priv_protocol = config_json.get('snmp_v3_priv_protocol', '') + self.snmp_v3_priv_password = config_json.get('snmp_v3_priv_password', '') + self.snmp_v3_engine_id = config_json.get('snmp_v3_engine_id', '') + + self.validate() + + @classmethod + def init(cls, ctx: CephadmContext, fsid: str, + daemon_id: Union[int, str]) -> 'SNMPGateway': + assert ctx.config_json + return cls(ctx, fsid, daemon_id, + get_parm(ctx.config_json), ctx.image) + + @staticmethod + def get_version(ctx: CephadmContext, fsid: str, daemon_id: str) -> Optional[str]: + """Return the version of the notifer from it's http endpoint""" + path = os.path.join(ctx.data_dir, fsid, f'snmp-gateway.{daemon_id}', 'unit.meta') + try: + with open(path, 'r') as env: + metadata = json.loads(env.read()) + except (OSError, json.JSONDecodeError): + return None + + ports = metadata.get('ports', []) + if not ports: + return None + + try: + with urlopen(f'http://127.0.0.1:{ports[0]}/') as r: + html = r.read().decode('utf-8').split('\n') + except (HTTPError, URLError): + return None + + for h in html: + stripped = h.strip() + if stripped.startswith(('<pre>', '<PRE>')) and \ + stripped.endswith(('</pre>', '</PRE>')): + # <pre>(version=1.2.1, branch=HEAD, revision=7... + return stripped.split(',')[0].split('version=')[1] + + return None + + @property + def port(self) -> int: + if not self.ctx.tcp_ports: + return self.DEFAULT_PORT + else: + if len(self.ctx.tcp_ports) > 0: + return int(self.ctx.tcp_ports.split()[0]) + else: + return self.DEFAULT_PORT + + def get_daemon_args(self) -> List[str]: + v3_args = [] + base_args = [ + f'--web.listen-address=:{self.port}', + f'--snmp.destination={self.destination}', + f'--snmp.version={self.snmp_version}', + f'--log.level={self.log_level}', + '--snmp.trap-description-template=/etc/snmp_notifier/description-template.tpl' + ] + + if self.snmp_version == 'V3': + # common auth settings + v3_args.extend([ + '--snmp.authentication-enabled', + f'--snmp.authentication-protocol={self.snmp_v3_auth_protocol}', + f'--snmp.security-engine-id={self.snmp_v3_engine_id}' + ]) + # authPriv setting is applied if we have a privacy protocol setting + if self.snmp_v3_priv_protocol: + v3_args.extend([ + '--snmp.private-enabled', + f'--snmp.private-protocol={self.snmp_v3_priv_protocol}' + ]) + + return base_args + v3_args + + @property + def data_dir(self) -> str: + return os.path.join(self.ctx.data_dir, self.ctx.fsid, f'{self.daemon_type}.{self.daemon_id}') + + @property + def conf_file_path(self) -> str: + return os.path.join(self.data_dir, self.env_filename) + + def create_daemon_conf(self) -> None: + """Creates the environment file holding 'secrets' passed to the snmp-notifier daemon""" + with open(os.open(self.conf_file_path, os.O_CREAT | os.O_WRONLY, 0o600), 'w') as f: + if self.snmp_version == 'V2c': + f.write(f'SNMP_NOTIFIER_COMMUNITY={self.snmp_community}\n') + else: + f.write(f'SNMP_NOTIFIER_AUTH_USERNAME={self.snmp_v3_auth_username}\n') + f.write(f'SNMP_NOTIFIER_AUTH_PASSWORD={self.snmp_v3_auth_password}\n') + if self.snmp_v3_priv_password: + f.write(f'SNMP_NOTIFIER_PRIV_PASSWORD={self.snmp_v3_priv_password}\n') + + def validate(self) -> None: + """Validate the settings + + Raises: + Error: if the fsid doesn't look like an fsid + Error: if the snmp version is not supported + Error: destination IP and port address missing + """ + if not is_fsid(self.fsid): + raise Error(f'not a valid fsid: {self.fsid}') + + if self.snmp_version not in SNMPGateway.SUPPORTED_VERSIONS: + raise Error(f'not a valid snmp version: {self.snmp_version}') + + if not self.destination: + raise Error('config is missing destination attribute(<ip>:<port>) of the target SNMP listener') + + +################################## +class Monitoring(object): + """Define the configs for the monitoring containers""" + + port_map = { + 'prometheus': [9095], # Avoid default 9090, due to conflict with cockpit UI + 'node-exporter': [9100], + 'grafana': [3000], + 'alertmanager': [9093, 9094], + } + + components = { + 'prometheus': { + 'image': DEFAULT_PROMETHEUS_IMAGE, + 'cpus': '2', + 'memory': '4GB', + 'args': [ + '--config.file=/etc/prometheus/prometheus.yml', + '--storage.tsdb.path=/prometheus', + ], + 'config-json-files': [ + 'prometheus.yml', + ], + }, + 'node-exporter': { + 'image': DEFAULT_NODE_EXPORTER_IMAGE, + 'cpus': '1', + 'memory': '1GB', + 'args': [ + '--no-collector.timex', + ], + }, + 'grafana': { + 'image': DEFAULT_GRAFANA_IMAGE, + 'cpus': '2', + 'memory': '4GB', + 'args': [], + 'config-json-files': [ + 'grafana.ini', + 'provisioning/datasources/ceph-dashboard.yml', + 'certs/cert_file', + 'certs/cert_key', + ], + }, + 'alertmanager': { + 'image': DEFAULT_ALERT_MANAGER_IMAGE, + 'cpus': '2', + 'memory': '2GB', + 'args': [ + '--cluster.listen-address=:{}'.format(port_map['alertmanager'][1]), + ], + 'config-json-files': [ + 'alertmanager.yml', + ], + 'config-json-args': [ + 'peers', + ], + }, + } # type: ignore + + @staticmethod + def get_version(ctx, container_id, daemon_type): + # type: (CephadmContext, str, str) -> str + """ + :param: daemon_type Either "prometheus", "alertmanager" or "node-exporter" + """ + assert daemon_type in ('prometheus', 'alertmanager', 'node-exporter') + cmd = daemon_type.replace('-', '_') + code = -1 + err = '' + out = '' + version = '' + if daemon_type == 'alertmanager': + for cmd in ['alertmanager', 'prometheus-alertmanager']: + out, err, code = call(ctx, [ + ctx.container_engine.path, 'exec', container_id, cmd, + '--version' + ], verbosity=CallVerbosity.DEBUG) + if code == 0: + break + cmd = 'alertmanager' # reset cmd for version extraction + else: + out, err, code = call(ctx, [ + ctx.container_engine.path, 'exec', container_id, cmd, '--version' + ], verbosity=CallVerbosity.DEBUG) + if code == 0: + if err.startswith('%s, version ' % cmd): + version = err.split(' ')[2] + elif out.startswith('%s, version ' % cmd): + version = out.split(' ')[2] + return version + +################################## + + +def populate_files(config_dir, config_files, uid, gid): + # type: (str, Dict, int, int) -> None + """create config files for different services""" + for fname in config_files: + config_file = os.path.join(config_dir, fname) + config_content = dict_get_join(config_files, fname) + logger.info('Write file: %s' % (config_file)) + with open(config_file, 'w', encoding='utf-8') as f: + os.fchown(f.fileno(), uid, gid) + os.fchmod(f.fileno(), 0o600) + f.write(config_content) + + +class NFSGanesha(object): + """Defines a NFS-Ganesha container""" + + daemon_type = 'nfs' + entrypoint = '/usr/bin/ganesha.nfsd' + daemon_args = ['-F', '-L', 'STDERR'] + + required_files = ['ganesha.conf'] + + port_map = { + 'nfs': 2049, + } + + def __init__(self, + ctx, + fsid, + daemon_id, + config_json, + image=DEFAULT_IMAGE): + # type: (CephadmContext, str, Union[int, str], Dict, str) -> None + self.ctx = ctx + self.fsid = fsid + self.daemon_id = daemon_id + self.image = image + + # config-json options + self.pool = dict_get(config_json, 'pool', require=True) + self.namespace = dict_get(config_json, 'namespace') + self.userid = dict_get(config_json, 'userid') + self.extra_args = dict_get(config_json, 'extra_args', []) + self.files = dict_get(config_json, 'files', {}) + self.rgw = dict_get(config_json, 'rgw', {}) + + # validate the supplied args + self.validate() + + @classmethod + def init(cls, ctx, fsid, daemon_id): + # type: (CephadmContext, str, Union[int, str]) -> NFSGanesha + return cls(ctx, fsid, daemon_id, get_parm(ctx.config_json), ctx.image) + + def get_container_mounts(self, data_dir): + # type: (str) -> Dict[str, str] + mounts = dict() + mounts[os.path.join(data_dir, 'config')] = '/etc/ceph/ceph.conf:z' + mounts[os.path.join(data_dir, 'keyring')] = '/etc/ceph/keyring:z' + mounts[os.path.join(data_dir, 'etc/ganesha')] = '/etc/ganesha:z' + if self.rgw: + cluster = self.rgw.get('cluster', 'ceph') + rgw_user = self.rgw.get('user', 'admin') + mounts[os.path.join(data_dir, 'keyring.rgw')] = \ + '/var/lib/ceph/radosgw/%s-%s/keyring:z' % (cluster, rgw_user) + return mounts + + @staticmethod + def get_container_envs(): + # type: () -> List[str] + envs = [ + 'CEPH_CONF=%s' % ('/etc/ceph/ceph.conf') + ] + return envs + + @staticmethod + def get_version(ctx, container_id): + # type: (CephadmContext, str) -> Optional[str] + version = None + out, err, code = call(ctx, + [ctx.container_engine.path, 'exec', container_id, + NFSGanesha.entrypoint, '-v'], + verbosity=CallVerbosity.DEBUG) + if code == 0: + match = re.search(r'NFS-Ganesha Release\s*=\s*[V]*([\d.]+)', out) + if match: + version = match.group(1) + return version + + def validate(self): + # type: () -> None + if not is_fsid(self.fsid): + raise Error('not an fsid: %s' % self.fsid) + if not self.daemon_id: + raise Error('invalid daemon_id: %s' % self.daemon_id) + if not self.image: + raise Error('invalid image: %s' % self.image) + + # check for the required files + if self.required_files: + for fname in self.required_files: + if fname not in self.files: + raise Error('required file missing from config-json: %s' % fname) + + # check for an RGW config + if self.rgw: + if not self.rgw.get('keyring'): + raise Error('RGW keyring is missing') + if not self.rgw.get('user'): + raise Error('RGW user is missing') + + def get_daemon_name(self): + # type: () -> str + return '%s.%s' % (self.daemon_type, self.daemon_id) + + def get_container_name(self, desc=None): + # type: (Optional[str]) -> str + cname = 'ceph-%s-%s' % (self.fsid, self.get_daemon_name()) + if desc: + cname = '%s-%s' % (cname, desc) + return cname + + def get_daemon_args(self): + # type: () -> List[str] + return self.daemon_args + self.extra_args + + def create_daemon_dirs(self, data_dir, uid, gid): + # type: (str, int, int) -> None + """Create files under the container data dir""" + if not os.path.isdir(data_dir): + raise OSError('data_dir is not a directory: %s' % (data_dir)) + + logger.info('Creating ganesha config...') + + # create the ganesha conf dir + config_dir = os.path.join(data_dir, 'etc/ganesha') + makedirs(config_dir, uid, gid, 0o755) + + # populate files from the config-json + populate_files(config_dir, self.files, uid, gid) + + # write the RGW keyring + if self.rgw: + keyring_path = os.path.join(data_dir, 'keyring.rgw') + with open(keyring_path, 'w') as f: + os.fchmod(f.fileno(), 0o600) + os.fchown(f.fileno(), uid, gid) + f.write(self.rgw.get('keyring', '')) + +################################## + + +class CephIscsi(object): + """Defines a Ceph-Iscsi container""" + + daemon_type = 'iscsi' + entrypoint = '/usr/bin/rbd-target-api' + + required_files = ['iscsi-gateway.cfg'] + + def __init__(self, + ctx, + fsid, + daemon_id, + config_json, + image=DEFAULT_IMAGE): + # type: (CephadmContext, str, Union[int, str], Dict, str) -> None + self.ctx = ctx + self.fsid = fsid + self.daemon_id = daemon_id + self.image = image + + # config-json options + self.files = dict_get(config_json, 'files', {}) + + # validate the supplied args + self.validate() + + @classmethod + def init(cls, ctx, fsid, daemon_id): + # type: (CephadmContext, str, Union[int, str]) -> CephIscsi + return cls(ctx, fsid, daemon_id, + get_parm(ctx.config_json), ctx.image) + + @staticmethod + def get_container_mounts(data_dir, log_dir): + # type: (str, str) -> Dict[str, str] + mounts = dict() + mounts[os.path.join(data_dir, 'config')] = '/etc/ceph/ceph.conf:z' + mounts[os.path.join(data_dir, 'keyring')] = '/etc/ceph/keyring:z' + mounts[os.path.join(data_dir, 'iscsi-gateway.cfg')] = '/etc/ceph/iscsi-gateway.cfg:z' + mounts[os.path.join(data_dir, 'configfs')] = '/sys/kernel/config' + mounts[log_dir] = '/var/log:z' + mounts['/dev'] = '/dev' + return mounts + + @staticmethod + def get_container_binds(): + # type: () -> List[List[str]] + binds = [] + lib_modules = ['type=bind', + 'source=/lib/modules', + 'destination=/lib/modules', + 'ro=true'] + binds.append(lib_modules) + return binds + + @staticmethod + def get_version(ctx, container_id): + # type: (CephadmContext, str) -> Optional[str] + version = None + out, err, code = call(ctx, + [ctx.container_engine.path, 'exec', container_id, + '/usr/bin/python3', '-c', "import pkg_resources; print(pkg_resources.require('ceph_iscsi')[0].version)"], + verbosity=CallVerbosity.DEBUG) + if code == 0: + version = out.strip() + return version + + def validate(self): + # type: () -> None + if not is_fsid(self.fsid): + raise Error('not an fsid: %s' % self.fsid) + if not self.daemon_id: + raise Error('invalid daemon_id: %s' % self.daemon_id) + if not self.image: + raise Error('invalid image: %s' % self.image) + + # check for the required files + if self.required_files: + for fname in self.required_files: + if fname not in self.files: + raise Error('required file missing from config-json: %s' % fname) + + def get_daemon_name(self): + # type: () -> str + return '%s.%s' % (self.daemon_type, self.daemon_id) + + def get_container_name(self, desc=None): + # type: (Optional[str]) -> str + cname = 'ceph-%s-%s' % (self.fsid, self.get_daemon_name()) + if desc: + cname = '%s-%s' % (cname, desc) + return cname + + def create_daemon_dirs(self, data_dir, uid, gid): + # type: (str, int, int) -> None + """Create files under the container data dir""" + if not os.path.isdir(data_dir): + raise OSError('data_dir is not a directory: %s' % (data_dir)) + + logger.info('Creating ceph-iscsi config...') + configfs_dir = os.path.join(data_dir, 'configfs') + makedirs(configfs_dir, uid, gid, 0o755) + + # populate files from the config-json + populate_files(data_dir, self.files, uid, gid) + + @staticmethod + def configfs_mount_umount(data_dir, mount=True): + # type: (str, bool) -> List[str] + mount_path = os.path.join(data_dir, 'configfs') + if mount: + cmd = 'if ! grep -qs {0} /proc/mounts; then ' \ + 'mount -t configfs none {0}; fi'.format(mount_path) + else: + cmd = 'if grep -qs {0} /proc/mounts; then ' \ + 'umount {0}; fi'.format(mount_path) + return cmd.split() + + def get_tcmu_runner_container(self): + # type: () -> CephContainer + tcmu_container = get_container(self.ctx, self.fsid, self.daemon_type, self.daemon_id) + tcmu_container.entrypoint = '/usr/bin/tcmu-runner' + tcmu_container.cname = self.get_container_name(desc='tcmu') + # remove extra container args for tcmu container. + # extra args could cause issue with forking service type + tcmu_container.container_args = [] + return tcmu_container + +################################## + + +class HAproxy(object): + """Defines an HAproxy container""" + daemon_type = 'haproxy' + required_files = ['haproxy.cfg'] + default_image = DEFAULT_HAPROXY_IMAGE + + def __init__(self, + ctx: CephadmContext, + fsid: str, daemon_id: Union[int, str], + config_json: Dict, image: str) -> None: + self.ctx = ctx + self.fsid = fsid + self.daemon_id = daemon_id + self.image = image + + # config-json options + self.files = dict_get(config_json, 'files', {}) + + self.validate() + + @classmethod + def init(cls, ctx: CephadmContext, + fsid: str, daemon_id: Union[int, str]) -> 'HAproxy': + return cls(ctx, fsid, daemon_id, get_parm(ctx.config_json), + ctx.image) + + def create_daemon_dirs(self, data_dir: str, uid: int, gid: int) -> None: + """Create files under the container data dir""" + if not os.path.isdir(data_dir): + raise OSError('data_dir is not a directory: %s' % (data_dir)) + + # create additional directories in data dir for HAproxy to use + if not os.path.isdir(os.path.join(data_dir, 'haproxy')): + makedirs(os.path.join(data_dir, 'haproxy'), uid, gid, DATA_DIR_MODE) + + data_dir = os.path.join(data_dir, 'haproxy') + populate_files(data_dir, self.files, uid, gid) + + def get_daemon_args(self) -> List[str]: + return ['haproxy', '-f', '/var/lib/haproxy/haproxy.cfg'] + + def validate(self): + # type: () -> None + if not is_fsid(self.fsid): + raise Error('not an fsid: %s' % self.fsid) + if not self.daemon_id: + raise Error('invalid daemon_id: %s' % self.daemon_id) + if not self.image: + raise Error('invalid image: %s' % self.image) + + # check for the required files + if self.required_files: + for fname in self.required_files: + if fname not in self.files: + raise Error('required file missing from config-json: %s' % fname) + + def get_daemon_name(self): + # type: () -> str + return '%s.%s' % (self.daemon_type, self.daemon_id) + + def get_container_name(self, desc=None): + # type: (Optional[str]) -> str + cname = 'ceph-%s-%s' % (self.fsid, self.get_daemon_name()) + if desc: + cname = '%s-%s' % (cname, desc) + return cname + + def extract_uid_gid_haproxy(self) -> Tuple[int, int]: + # better directory for this? + return extract_uid_gid(self.ctx, file_path='/var/lib') + + @staticmethod + def get_container_mounts(data_dir: str) -> Dict[str, str]: + mounts = dict() + mounts[os.path.join(data_dir, 'haproxy')] = '/var/lib/haproxy' + return mounts + + @staticmethod + def get_sysctl_settings() -> List[str]: + return [ + '# IP forwarding and non-local bind', + 'net.ipv4.ip_forward = 1', + 'net.ipv4.ip_nonlocal_bind = 1', + ] + +################################## + + +class Keepalived(object): + """Defines an Keepalived container""" + daemon_type = 'keepalived' + required_files = ['keepalived.conf'] + default_image = DEFAULT_KEEPALIVED_IMAGE + + def __init__(self, + ctx: CephadmContext, + fsid: str, daemon_id: Union[int, str], + config_json: Dict, image: str) -> None: + self.ctx = ctx + self.fsid = fsid + self.daemon_id = daemon_id + self.image = image + + # config-json options + self.files = dict_get(config_json, 'files', {}) + + self.validate() + + @classmethod + def init(cls, ctx: CephadmContext, fsid: str, + daemon_id: Union[int, str]) -> 'Keepalived': + return cls(ctx, fsid, daemon_id, + get_parm(ctx.config_json), ctx.image) + + def create_daemon_dirs(self, data_dir: str, uid: int, gid: int) -> None: + """Create files under the container data dir""" + if not os.path.isdir(data_dir): + raise OSError('data_dir is not a directory: %s' % (data_dir)) + + # create additional directories in data dir for keepalived to use + if not os.path.isdir(os.path.join(data_dir, 'keepalived')): + makedirs(os.path.join(data_dir, 'keepalived'), uid, gid, DATA_DIR_MODE) + + # populate files from the config-json + populate_files(data_dir, self.files, uid, gid) + + def validate(self): + # type: () -> None + if not is_fsid(self.fsid): + raise Error('not an fsid: %s' % self.fsid) + if not self.daemon_id: + raise Error('invalid daemon_id: %s' % self.daemon_id) + if not self.image: + raise Error('invalid image: %s' % self.image) + + # check for the required files + if self.required_files: + for fname in self.required_files: + if fname not in self.files: + raise Error('required file missing from config-json: %s' % fname) + + def get_daemon_name(self): + # type: () -> str + return '%s.%s' % (self.daemon_type, self.daemon_id) + + def get_container_name(self, desc=None): + # type: (Optional[str]) -> str + cname = 'ceph-%s-%s' % (self.fsid, self.get_daemon_name()) + if desc: + cname = '%s-%s' % (cname, desc) + return cname + + @staticmethod + def get_container_envs(): + # type: () -> List[str] + envs = [ + 'KEEPALIVED_AUTOCONF=false', + 'KEEPALIVED_CONF=/etc/keepalived/keepalived.conf', + 'KEEPALIVED_CMD=/usr/sbin/keepalived -n -l -f /etc/keepalived/keepalived.conf', + 'KEEPALIVED_DEBUG=false' + ] + return envs + + @staticmethod + def get_sysctl_settings() -> List[str]: + return [ + '# IP forwarding and non-local bind', + 'net.ipv4.ip_forward = 1', + 'net.ipv4.ip_nonlocal_bind = 1', + ] + + def extract_uid_gid_keepalived(self) -> Tuple[int, int]: + # better directory for this? + return extract_uid_gid(self.ctx, file_path='/var/lib') + + @staticmethod + def get_container_mounts(data_dir: str) -> Dict[str, str]: + mounts = dict() + mounts[os.path.join(data_dir, 'keepalived.conf')] = '/etc/keepalived/keepalived.conf' + return mounts + +################################## + + +class CustomContainer(object): + """Defines a custom container""" + daemon_type = 'container' + + def __init__(self, + fsid: str, daemon_id: Union[int, str], + config_json: Dict, image: str) -> None: + self.fsid = fsid + self.daemon_id = daemon_id + self.image = image + + # config-json options + self.entrypoint = dict_get(config_json, 'entrypoint') + self.uid = dict_get(config_json, 'uid', 65534) # nobody + self.gid = dict_get(config_json, 'gid', 65534) # nobody + self.volume_mounts = dict_get(config_json, 'volume_mounts', {}) + self.args = dict_get(config_json, 'args', []) + self.envs = dict_get(config_json, 'envs', []) + self.privileged = dict_get(config_json, 'privileged', False) + self.bind_mounts = dict_get(config_json, 'bind_mounts', []) + self.ports = dict_get(config_json, 'ports', []) + self.dirs = dict_get(config_json, 'dirs', []) + self.files = dict_get(config_json, 'files', {}) + + @classmethod + def init(cls, ctx: CephadmContext, + fsid: str, daemon_id: Union[int, str]) -> 'CustomContainer': + return cls(fsid, daemon_id, + get_parm(ctx.config_json), ctx.image) + + def create_daemon_dirs(self, data_dir: str, uid: int, gid: int) -> None: + """ + Create dirs/files below the container data directory. + """ + logger.info('Creating custom container configuration ' + 'dirs/files in {} ...'.format(data_dir)) + + if not os.path.isdir(data_dir): + raise OSError('data_dir is not a directory: %s' % data_dir) + + for dir_path in self.dirs: + logger.info('Creating directory: {}'.format(dir_path)) + dir_path = os.path.join(data_dir, dir_path.strip('/')) + makedirs(dir_path, uid, gid, 0o755) + + for file_path in self.files: + logger.info('Creating file: {}'.format(file_path)) + content = dict_get_join(self.files, file_path) + file_path = os.path.join(data_dir, file_path.strip('/')) + with open(file_path, 'w', encoding='utf-8') as f: + os.fchown(f.fileno(), uid, gid) + os.fchmod(f.fileno(), 0o600) + f.write(content) + + def get_daemon_args(self) -> List[str]: + return [] + + def get_container_args(self) -> List[str]: + return self.args + + def get_container_envs(self) -> List[str]: + return self.envs + + def get_container_mounts(self, data_dir: str) -> Dict[str, str]: + """ + Get the volume mounts. Relative source paths will be located below + `/var/lib/ceph/<cluster-fsid>/<daemon-name>`. + + Example: + { + /foo/conf: /conf + foo/conf: /conf + } + becomes + { + /foo/conf: /conf + /var/lib/ceph/<cluster-fsid>/<daemon-name>/foo/conf: /conf + } + """ + mounts = {} + for source, destination in self.volume_mounts.items(): + source = os.path.join(data_dir, source) + mounts[source] = destination + return mounts + + def get_container_binds(self, data_dir: str) -> List[List[str]]: + """ + Get the bind mounts. Relative `source=...` paths will be located below + `/var/lib/ceph/<cluster-fsid>/<daemon-name>`. + + Example: + [ + 'type=bind', + 'source=lib/modules', + 'destination=/lib/modules', + 'ro=true' + ] + becomes + [ + ... + 'source=/var/lib/ceph/<cluster-fsid>/<daemon-name>/lib/modules', + ... + ] + """ + binds = self.bind_mounts.copy() + for bind in binds: + for index, value in enumerate(bind): + match = re.match(r'^source=(.+)$', value) + if match: + bind[index] = 'source={}'.format(os.path.join( + data_dir, match.group(1))) + return binds + +################################## + + +def touch(file_path: str, uid: Optional[int] = None, gid: Optional[int] = None) -> None: + Path(file_path).touch() + if uid and gid: + os.chown(file_path, uid, gid) + + +################################## + + +def dict_get(d: Dict, key: str, default: Any = None, require: bool = False) -> Any: + """ + Helper function to get a key from a dictionary. + :param d: The dictionary to process. + :param key: The name of the key to get. + :param default: The default value in case the key does not + exist. Default is `None`. + :param require: Set to `True` if the key is required. An + exception will be raised if the key does not exist in + the given dictionary. + :return: Returns the value of the given key. + :raises: :exc:`self.Error` if the given key does not exist + and `require` is set to `True`. + """ + if require and key not in d.keys(): + raise Error('{} missing from dict'.format(key)) + return d.get(key, default) # type: ignore + +################################## + + +def dict_get_join(d: Dict, key: str) -> Any: + """ + Helper function to get the value of a given key from a dictionary. + `List` values will be converted to a string by joining them with a + line break. + :param d: The dictionary to process. + :param key: The name of the key to get. + :return: Returns the value of the given key. If it was a `list`, it + will be joining with a line break. + """ + value = d.get(key) + if isinstance(value, list): + value = '\n'.join(map(str, value)) + return value + +################################## + + +def get_supported_daemons(): + # type: () -> List[str] + supported_daemons = list(Ceph.daemons) + supported_daemons.extend(Monitoring.components) + supported_daemons.append(NFSGanesha.daemon_type) + supported_daemons.append(CephIscsi.daemon_type) + supported_daemons.append(CustomContainer.daemon_type) + supported_daemons.append(CephadmDaemon.daemon_type) + supported_daemons.append(HAproxy.daemon_type) + supported_daemons.append(Keepalived.daemon_type) + supported_daemons.append(SNMPGateway.daemon_type) + assert len(supported_daemons) == len(set(supported_daemons)) + return supported_daemons + +################################## + + +class PortOccupiedError(Error): + pass + + +def attempt_bind(ctx, s, address, port): + # type: (CephadmContext, socket.socket, str, int) -> None + try: + s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + s.bind((address, port)) + except OSError as e: + if e.errno == errno.EADDRINUSE: + msg = 'Cannot bind to IP %s port %d: %s' % (address, port, e) + logger.warning(msg) + raise PortOccupiedError(msg) + else: + raise Error(e) + except Exception as e: + raise Error(e) + finally: + s.close() + + +def port_in_use(ctx, port_num): + # type: (CephadmContext, int) -> bool + """Detect whether a port is in use on the local machine - IPv4 and IPv6""" + logger.info('Verifying port %d ...' % port_num) + + def _port_in_use(af: socket.AddressFamily, address: str) -> bool: + try: + s = socket.socket(af, socket.SOCK_STREAM) + attempt_bind(ctx, s, address, port_num) + except PortOccupiedError: + return True + except OSError as e: + if e.errno in (errno.EAFNOSUPPORT, errno.EADDRNOTAVAIL): + # Ignore EAFNOSUPPORT and EADDRNOTAVAIL as two interfaces are + # being tested here and one might be intentionally be disabled. + # In that case no error should be raised. + return False + else: + raise e + return False + return any(_port_in_use(af, address) for af, address in ( + (socket.AF_INET, '0.0.0.0'), + (socket.AF_INET6, '::') + )) + + +def check_ip_port(ctx, ep): + # type: (CephadmContext, EndPoint) -> None + if not ctx.skip_ping_check: + logger.info(f'Verifying IP {ep.ip} port {ep.port} ...') + if is_ipv6(ep.ip): + s = socket.socket(socket.AF_INET6, socket.SOCK_STREAM) + ip = unwrap_ipv6(ep.ip) + else: + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + ip = ep.ip + attempt_bind(ctx, s, ip, ep.port) + +################################## + + +# this is an abbreviated version of +# https://github.com/benediktschmitt/py-filelock/blob/master/filelock.py +# that drops all of the compatibility (this is Unix/Linux only). + +class Timeout(TimeoutError): + """ + Raised when the lock could not be acquired in *timeout* + seconds. + """ + + def __init__(self, lock_file: str) -> None: + """ + """ + #: The path of the file lock. + self.lock_file = lock_file + return None + + def __str__(self) -> str: + temp = "The file lock '{}' could not be acquired."\ + .format(self.lock_file) + return temp + + +class _Acquire_ReturnProxy(object): + def __init__(self, lock: 'FileLock') -> None: + self.lock = lock + return None + + def __enter__(self) -> 'FileLock': + return self.lock + + def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> None: + self.lock.release() + return None + + +class FileLock(object): + def __init__(self, ctx: CephadmContext, name: str, timeout: int = -1) -> None: + if not os.path.exists(LOCK_DIR): + os.mkdir(LOCK_DIR, 0o700) + self._lock_file = os.path.join(LOCK_DIR, name + '.lock') + self.ctx = ctx + + # The file descriptor for the *_lock_file* as it is returned by the + # os.open() function. + # This file lock is only NOT None, if the object currently holds the + # lock. + self._lock_file_fd: Optional[int] = None + self.timeout = timeout + # The lock counter is used for implementing the nested locking + # mechanism. Whenever the lock is acquired, the counter is increased and + # the lock is only released, when this value is 0 again. + self._lock_counter = 0 + return None + + @property + def is_locked(self) -> bool: + return self._lock_file_fd is not None + + def acquire(self, timeout: Optional[int] = None, poll_intervall: float = 0.05) -> _Acquire_ReturnProxy: + """ + Acquires the file lock or fails with a :exc:`Timeout` error. + .. code-block:: python + # You can use this method in the context manager (recommended) + with lock.acquire(): + pass + # Or use an equivalent try-finally construct: + lock.acquire() + try: + pass + finally: + lock.release() + :arg float timeout: + The maximum time waited for the file lock. + If ``timeout < 0``, there is no timeout and this method will + block until the lock could be acquired. + If ``timeout`` is None, the default :attr:`~timeout` is used. + :arg float poll_intervall: + We check once in *poll_intervall* seconds if we can acquire the + file lock. + :raises Timeout: + if the lock could not be acquired in *timeout* seconds. + .. versionchanged:: 2.0.0 + This method returns now a *proxy* object instead of *self*, + so that it can be used in a with statement without side effects. + """ + + # Use the default timeout, if no timeout is provided. + if timeout is None: + timeout = self.timeout + + # Increment the number right at the beginning. + # We can still undo it, if something fails. + self._lock_counter += 1 + + lock_id = id(self) + lock_filename = self._lock_file + start_time = time.time() + try: + while True: + if not self.is_locked: + logger.debug('Acquiring lock %s on %s', lock_id, + lock_filename) + self._acquire() + + if self.is_locked: + logger.debug('Lock %s acquired on %s', lock_id, + lock_filename) + break + elif timeout >= 0 and time.time() - start_time > timeout: + logger.warning('Timeout acquiring lock %s on %s', lock_id, + lock_filename) + raise Timeout(self._lock_file) + else: + logger.debug( + 'Lock %s not acquired on %s, waiting %s seconds ...', + lock_id, lock_filename, poll_intervall + ) + time.sleep(poll_intervall) + except Exception: + # Something did go wrong, so decrement the counter. + self._lock_counter = max(0, self._lock_counter - 1) + + raise + return _Acquire_ReturnProxy(lock=self) + + def release(self, force: bool = False) -> None: + """ + Releases the file lock. + Please note, that the lock is only completly released, if the lock + counter is 0. + Also note, that the lock file itself is not automatically deleted. + :arg bool force: + If true, the lock counter is ignored and the lock is released in + every case. + """ + if self.is_locked: + self._lock_counter -= 1 + + if self._lock_counter == 0 or force: + # lock_id = id(self) + # lock_filename = self._lock_file + + # Can't log in shutdown: + # File "/usr/lib64/python3.9/logging/__init__.py", line 1175, in _open + # NameError: name 'open' is not defined + # logger.debug('Releasing lock %s on %s', lock_id, lock_filename) + self._release() + self._lock_counter = 0 + # logger.debug('Lock %s released on %s', lock_id, lock_filename) + + return None + + def __enter__(self) -> 'FileLock': + self.acquire() + return self + + def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> None: + self.release() + return None + + def __del__(self) -> None: + self.release(force=True) + return None + + def _acquire(self) -> None: + open_mode = os.O_RDWR | os.O_CREAT | os.O_TRUNC + fd = os.open(self._lock_file, open_mode) + + try: + fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB) + except (IOError, OSError): + os.close(fd) + else: + self._lock_file_fd = fd + return None + + def _release(self) -> None: + # Do not remove the lockfile: + # + # https://github.com/benediktschmitt/py-filelock/issues/31 + # https://stackoverflow.com/questions/17708885/flock-removing-locked-file-without-race-condition + fd = self._lock_file_fd + self._lock_file_fd = None + fcntl.flock(fd, fcntl.LOCK_UN) # type: ignore + os.close(fd) # type: ignore + return None + + +################################## +# Popen wrappers, lifted from ceph-volume + +class CallVerbosity(Enum): + SILENT = 0 + # log stdout/stderr to logger.debug + DEBUG = 1 + # On a non-zero exit status, it will forcefully set + # logging ON for the terminal + VERBOSE_ON_FAILURE = 2 + # log at info (instead of debug) level. + VERBOSE = 3 + + +if sys.version_info < (3, 8): + import itertools + import threading + import warnings + from asyncio import events + + class ThreadedChildWatcher(asyncio.AbstractChildWatcher): + """Threaded child watcher implementation. + The watcher uses a thread per process + for waiting for the process finish. + It doesn't require subscription on POSIX signal + but a thread creation is not free. + The watcher has O(1) complexity, its performance doesn't depend + on amount of spawn processes. + """ + + def __init__(self) -> None: + self._pid_counter = itertools.count(0) + self._threads: Dict[Any, Any] = {} + + def is_active(self) -> bool: + return True + + def close(self) -> None: + self._join_threads() + + def _join_threads(self) -> None: + """Internal: Join all non-daemon threads""" + threads = [thread for thread in list(self._threads.values()) + if thread.is_alive() and not thread.daemon] + for thread in threads: + thread.join() + + def __enter__(self) -> Any: + return self + + def __exit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> None: + pass + + def __del__(self, _warn: Any = warnings.warn) -> None: + threads = [thread for thread in list(self._threads.values()) + if thread.is_alive()] + if threads: + _warn(f'{self.__class__} has registered but not finished child processes', + ResourceWarning, + source=self) + + def add_child_handler(self, pid: Any, callback: Any, *args: Any) -> None: + loop = events.get_event_loop() + thread = threading.Thread(target=self._do_waitpid, + name=f'waitpid-{next(self._pid_counter)}', + args=(loop, pid, callback, args), + daemon=True) + self._threads[pid] = thread + thread.start() + + def remove_child_handler(self, pid: Any) -> bool: + # asyncio never calls remove_child_handler() !!! + # The method is no-op but is implemented because + # abstract base classe requires it + return True + + def attach_loop(self, loop: Any) -> None: + pass + + def _do_waitpid(self, loop: Any, expected_pid: Any, callback: Any, args: Any) -> None: + assert expected_pid > 0 + + try: + pid, status = os.waitpid(expected_pid, 0) + except ChildProcessError: + # The child process is already reaped + # (may happen if waitpid() is called elsewhere). + pid = expected_pid + returncode = 255 + logger.warning( + 'Unknown child process pid %d, will report returncode 255', + pid) + else: + if os.WIFEXITED(status): + returncode = os.WEXITSTATUS(status) + elif os.WIFSIGNALED(status): + returncode = -os.WTERMSIG(status) + else: + raise ValueError(f'unknown wait status {status}') + if loop.get_debug(): + logger.debug('process %s exited with returncode %s', + expected_pid, returncode) + + if loop.is_closed(): + logger.warning('Loop %r that handles pid %r is closed', loop, pid) + else: + loop.call_soon_threadsafe(callback, pid, returncode, *args) + + self._threads.pop(expected_pid) + + # unlike SafeChildWatcher which handles SIGCHLD in the main thread, + # ThreadedChildWatcher runs in a separated thread, hence allows us to + # run create_subprocess_exec() in non-main thread, see + # https://bugs.python.org/issue35621 + asyncio.set_child_watcher(ThreadedChildWatcher()) + + +try: + from asyncio import run as async_run # type: ignore[attr-defined] +except ImportError: + def async_run(coro): # type: ignore + loop = asyncio.new_event_loop() + try: + asyncio.set_event_loop(loop) + return loop.run_until_complete(coro) + finally: + try: + loop.run_until_complete(loop.shutdown_asyncgens()) + finally: + asyncio.set_event_loop(None) + loop.close() + + +def call(ctx: CephadmContext, + command: List[str], + desc: Optional[str] = None, + verbosity: CallVerbosity = CallVerbosity.VERBOSE_ON_FAILURE, + timeout: Optional[int] = DEFAULT_TIMEOUT, + **kwargs: Any) -> Tuple[str, str, int]: + """ + Wrap subprocess.Popen to + + - log stdout/stderr to a logger, + - decode utf-8 + - cleanly return out, err, returncode + + :param timeout: timeout in seconds + """ + + prefix = command[0] if desc is None else desc + if prefix: + prefix += ': ' + timeout = timeout or ctx.timeout + + async def tee(reader: asyncio.StreamReader) -> str: + collected = StringIO() + async for line in reader: + message = line.decode('utf-8') + collected.write(message) + if verbosity == CallVerbosity.VERBOSE: + logger.info(prefix + message.rstrip()) + elif verbosity != CallVerbosity.SILENT: + logger.debug(prefix + message.rstrip()) + return collected.getvalue() + + async def run_with_timeout() -> Tuple[str, str, int]: + process = await asyncio.create_subprocess_exec( + *command, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + env=os.environ.copy()) + assert process.stdout + assert process.stderr + try: + stdout, stderr = await asyncio.gather(tee(process.stdout), + tee(process.stderr)) + returncode = await asyncio.wait_for(process.wait(), timeout) + except asyncio.TimeoutError: + logger.info(prefix + f'timeout after {timeout} seconds') + return '', '', 124 + else: + return stdout, stderr, returncode + + stdout, stderr, returncode = async_run(run_with_timeout()) + if returncode != 0 and verbosity == CallVerbosity.VERBOSE_ON_FAILURE: + logger.info('Non-zero exit code %d from %s', + returncode, ' '.join(command)) + for line in stdout.splitlines(): + logger.info(prefix + 'stdout ' + line) + for line in stderr.splitlines(): + logger.info(prefix + 'stderr ' + line) + return stdout, stderr, returncode + + +def call_throws( + ctx: CephadmContext, + command: List[str], + desc: Optional[str] = None, + verbosity: CallVerbosity = CallVerbosity.VERBOSE_ON_FAILURE, + timeout: Optional[int] = DEFAULT_TIMEOUT, + **kwargs: Any) -> Tuple[str, str, int]: + out, err, ret = call(ctx, command, desc, verbosity, timeout, **kwargs) + if ret: + for s in (out, err): + if s.strip() and len(s.splitlines()) <= 2: # readable message? + raise RuntimeError(f'Failed command: {" ".join(command)}: {s}') + raise RuntimeError('Failed command: %s' % ' '.join(command)) + return out, err, ret + + +def call_timeout(ctx, command, timeout): + # type: (CephadmContext, List[str], int) -> int + logger.debug('Running command (timeout=%s): %s' + % (timeout, ' '.join(command))) + + def raise_timeout(command, timeout): + # type: (List[str], int) -> NoReturn + msg = 'Command `%s` timed out after %s seconds' % (command, timeout) + logger.debug(msg) + raise TimeoutExpired(msg) + + try: + return subprocess.call(command, timeout=timeout, env=os.environ.copy()) + except subprocess.TimeoutExpired: + raise_timeout(command, timeout) + +################################## + + +def json_loads_retry(cli_func: Callable[[], str]) -> Any: + for sleep_secs in [1, 4, 4]: + try: + return json.loads(cli_func()) + except json.JSONDecodeError: + logger.debug('Invalid JSON. Retrying in %s seconds...' % sleep_secs) + time.sleep(sleep_secs) + return json.loads(cli_func()) + + +def is_available(ctx, what, func): + # type: (CephadmContext, str, Callable[[], bool]) -> None + """ + Wait for a service to become available + + :param what: the name of the service + :param func: the callable object that determines availability + """ + retry = ctx.retry + logger.info('Waiting for %s...' % what) + num = 1 + while True: + if func(): + logger.info('%s is available' + % what) + break + elif num > retry: + raise Error('%s not available after %s tries' + % (what, retry)) + + logger.info('%s not available, waiting (%s/%s)...' + % (what, num, retry)) + + num += 1 + time.sleep(2) + + +def read_config(fn): + # type: (Optional[str]) -> ConfigParser + cp = ConfigParser() + if fn: + cp.read(fn) + return cp + + +def pathify(p): + # type: (str) -> str + p = os.path.expanduser(p) + return os.path.abspath(p) + + +def get_file_timestamp(fn): + # type: (str) -> Optional[str] + try: + mt = os.path.getmtime(fn) + return datetime.datetime.fromtimestamp( + mt, tz=datetime.timezone.utc + ).strftime(DATEFMT) + except Exception: + return None + + +def try_convert_datetime(s): + # type: (str) -> Optional[str] + # This is super irritating because + # 1) podman and docker use different formats + # 2) python's strptime can't parse either one + # + # I've seen: + # docker 18.09.7: 2020-03-03T09:21:43.636153304Z + # podman 1.7.0: 2020-03-03T15:52:30.136257504-06:00 + # 2020-03-03 15:52:30.136257504 -0600 CST + # (In the podman case, there is a different string format for + # 'inspect' and 'inspect --format {{.Created}}'!!) + + # In *all* cases, the 9 digit second precision is too much for + # python's strptime. Shorten it to 6 digits. + p = re.compile(r'(\.[\d]{6})[\d]*') + s = p.sub(r'\1', s) + + # replace trailing Z with -0000, since (on python 3.6.8) it won't parse + if s and s[-1] == 'Z': + s = s[:-1] + '-0000' + + # cut off the redundant 'CST' part that strptime can't parse, if + # present. + v = s.split(' ') + s = ' '.join(v[0:3]) + + # try parsing with several format strings + fmts = [ + '%Y-%m-%dT%H:%M:%S.%f%z', + '%Y-%m-%d %H:%M:%S.%f %z', + ] + for f in fmts: + try: + # return timestamp normalized to UTC, rendered as DATEFMT. + return datetime.datetime.strptime(s, f).astimezone(tz=datetime.timezone.utc).strftime(DATEFMT) + except ValueError: + pass + return None + + +def _parse_podman_version(version_str): + # type: (str) -> Tuple[int, ...] + def to_int(val: str, org_e: Optional[Exception] = None) -> int: + if not val and org_e: + raise org_e + try: + return int(val) + except ValueError as e: + return to_int(val[0:-1], org_e or e) + + return tuple(map(to_int, version_str.split('.'))) + + +def get_hostname(): + # type: () -> str + return socket.gethostname() + + +def get_fqdn(): + # type: () -> str + return socket.getfqdn() or socket.gethostname() + + +def get_arch(): + # type: () -> str + return platform.uname().machine + + +def generate_service_id(): + # type: () -> str + return get_hostname() + '.' + ''.join(random.choice(string.ascii_lowercase) + for _ in range(6)) + + +def generate_password(): + # type: () -> str + return ''.join(random.choice(string.ascii_lowercase + string.digits) + for i in range(10)) + + +def normalize_container_id(i): + # type: (str) -> str + # docker adds the sha256: prefix, but AFAICS both + # docker (18.09.7 in bionic at least) and podman + # both always use sha256, so leave off the prefix + # for consistency. + prefix = 'sha256:' + if i.startswith(prefix): + i = i[len(prefix):] + return i + + +def make_fsid(): + # type: () -> str + return str(uuid.uuid1()) + + +def is_fsid(s): + # type: (str) -> bool + try: + uuid.UUID(s) + except ValueError: + return False + return True + + +def validate_fsid(func: FuncT) -> FuncT: + @wraps(func) + def _validate_fsid(ctx: CephadmContext) -> Any: + if 'fsid' in ctx and ctx.fsid: + if not is_fsid(ctx.fsid): + raise Error('not an fsid: %s' % ctx.fsid) + return func(ctx) + return cast(FuncT, _validate_fsid) + + +def infer_fsid(func: FuncT) -> FuncT: + """ + If we only find a single fsid in /var/lib/ceph/*, use that + """ + @infer_config + @wraps(func) + def _infer_fsid(ctx: CephadmContext) -> Any: + if 'fsid' in ctx and ctx.fsid: + logger.debug('Using specified fsid: %s' % ctx.fsid) + return func(ctx) + + fsids = set() + + cp = read_config(ctx.config) + if cp.has_option('global', 'fsid'): + fsids.add(cp.get('global', 'fsid')) + + daemon_list = list_daemons(ctx, detail=False) + for daemon in daemon_list: + if not is_fsid(daemon['fsid']): + # 'unknown' fsid + continue + elif 'name' not in ctx or not ctx.name: + # ctx.name not specified + fsids.add(daemon['fsid']) + elif daemon['name'] == ctx.name: + # ctx.name is a match + fsids.add(daemon['fsid']) + fsids = sorted(fsids) + + if not fsids: + # some commands do not always require an fsid + pass + elif len(fsids) == 1: + logger.info('Inferring fsid %s' % fsids[0]) + ctx.fsid = fsids[0] + else: + raise Error('Cannot infer an fsid, one must be specified (using --fsid): %s' % fsids) + return func(ctx) + + return cast(FuncT, _infer_fsid) + + +def infer_config(func: FuncT) -> FuncT: + """ + If we find a MON daemon, use the config from that container + """ + @wraps(func) + def _infer_config(ctx: CephadmContext) -> Any: + ctx.config = ctx.config if 'config' in ctx else None + if ctx.config: + logger.debug('Using specified config: %s' % ctx.config) + return func(ctx) + + def config_path(daemon_type: str, daemon_name: str) -> str: + data_dir = get_data_dir(ctx.fsid, ctx.data_dir, daemon_type, daemon_name) + return os.path.join(data_dir, 'config') + + if 'fsid' in ctx and ctx.fsid: + name = ctx.name if 'name' in ctx else None + if not name: + daemon_list = list_daemons(ctx, detail=False) + for daemon in daemon_list: + if ( + daemon.get('name', '').startswith('mon.') + and daemon.get('fsid', '') == ctx.fsid + and daemon.get('style', '') == 'cephadm:v1' + and os.path.exists(config_path('mon', daemon['name'].split('.', 1)[1])) + ): + name = daemon['name'] + break + if name: + ctx.config = config_path(name.split('.', 1)[0], name.split('.', 1)[1]) + if ctx.config: + logger.info('Inferring config %s' % ctx.config) + elif os.path.exists(SHELL_DEFAULT_CONF): + logger.debug('Using default config: %s' % SHELL_DEFAULT_CONF) + ctx.config = SHELL_DEFAULT_CONF + return func(ctx) + + return cast(FuncT, _infer_config) + + +def _get_default_image(ctx: CephadmContext) -> str: + if DEFAULT_IMAGE_IS_MASTER: + warn = """This is a development version of cephadm. +For information regarding the latest stable release: + https://docs.ceph.com/docs/{}/cephadm/install +""".format(LATEST_STABLE_RELEASE) + for line in warn.splitlines(): + logger.warning('{}{}{}'.format(termcolor.yellow, line, termcolor.end)) + return DEFAULT_IMAGE + + +def infer_image(func: FuncT) -> FuncT: + """ + Use the most recent ceph image + """ + @wraps(func) + def _infer_image(ctx: CephadmContext) -> Any: + if not ctx.image: + ctx.image = os.environ.get('CEPHADM_IMAGE') + if not ctx.image: + ctx.image = get_last_local_ceph_image(ctx, ctx.container_engine.path) + if not ctx.image: + ctx.image = _get_default_image(ctx) + return func(ctx) + + return cast(FuncT, _infer_image) + + +def default_image(func: FuncT) -> FuncT: + @wraps(func) + def _default_image(ctx: CephadmContext) -> Any: + if not ctx.image: + if 'name' in ctx and ctx.name: + type_ = ctx.name.split('.', 1)[0] + if type_ in Monitoring.components: + ctx.image = Monitoring.components[type_]['image'] + if type_ == 'haproxy': + ctx.image = HAproxy.default_image + if type_ == 'keepalived': + ctx.image = Keepalived.default_image + if type_ == SNMPGateway.daemon_type: + ctx.image = SNMPGateway.default_image + if not ctx.image: + ctx.image = os.environ.get('CEPHADM_IMAGE') + if not ctx.image: + ctx.image = _get_default_image(ctx) + + return func(ctx) + + return cast(FuncT, _default_image) + + +def get_last_local_ceph_image(ctx: CephadmContext, container_path: str) -> Optional[str]: + """ + :return: The most recent local ceph image (already pulled) + """ + out, _, _ = call_throws(ctx, + [container_path, 'images', + '--filter', 'label=ceph=True', + '--filter', 'dangling=false', + '--format', '{{.Repository}}@{{.Digest}}']) + return _filter_last_local_ceph_image(out) + + +def _filter_last_local_ceph_image(out): + # type: (str) -> Optional[str] + for image in out.splitlines(): + if image and not image.endswith('@'): + logger.info('Using recent ceph image %s' % image) + return image + return None + + +def write_tmp(s, uid, gid): + # type: (str, int, int) -> IO[str] + tmp_f = tempfile.NamedTemporaryFile(mode='w', + prefix='ceph-tmp') + os.fchown(tmp_f.fileno(), uid, gid) + tmp_f.write(s) + tmp_f.flush() + + return tmp_f + + +def makedirs(dir, uid, gid, mode): + # type: (str, int, int, int) -> None + if not os.path.exists(dir): + os.makedirs(dir, mode=mode) + else: + os.chmod(dir, mode) + os.chown(dir, uid, gid) + os.chmod(dir, mode) # the above is masked by umask... + + +def get_data_dir(fsid, data_dir, t, n): + # type: (str, str, str, Union[int, str]) -> str + return os.path.join(data_dir, fsid, '%s.%s' % (t, n)) + + +def get_log_dir(fsid, log_dir): + # type: (str, str) -> str + return os.path.join(log_dir, fsid) + + +def make_data_dir_base(fsid, data_dir, uid, gid): + # type: (str, str, int, int) -> str + data_dir_base = os.path.join(data_dir, fsid) + makedirs(data_dir_base, uid, gid, DATA_DIR_MODE) + makedirs(os.path.join(data_dir_base, 'crash'), uid, gid, DATA_DIR_MODE) + makedirs(os.path.join(data_dir_base, 'crash', 'posted'), uid, gid, + DATA_DIR_MODE) + return data_dir_base + + +def make_data_dir(ctx, fsid, daemon_type, daemon_id, uid=None, gid=None): + # type: (CephadmContext, str, str, Union[int, str], Optional[int], Optional[int]) -> str + if uid is None or gid is None: + uid, gid = extract_uid_gid(ctx) + make_data_dir_base(fsid, ctx.data_dir, uid, gid) + data_dir = get_data_dir(fsid, ctx.data_dir, daemon_type, daemon_id) + makedirs(data_dir, uid, gid, DATA_DIR_MODE) + return data_dir + + +def make_log_dir(ctx, fsid, uid=None, gid=None): + # type: (CephadmContext, str, Optional[int], Optional[int]) -> str + if uid is None or gid is None: + uid, gid = extract_uid_gid(ctx) + log_dir = get_log_dir(fsid, ctx.log_dir) + makedirs(log_dir, uid, gid, LOG_DIR_MODE) + return log_dir + + +def make_var_run(ctx, fsid, uid, gid): + # type: (CephadmContext, str, int, int) -> None + call_throws(ctx, ['install', '-d', '-m0770', '-o', str(uid), '-g', str(gid), + '/var/run/ceph/%s' % fsid]) + + +def copy_tree(ctx, src, dst, uid=None, gid=None): + # type: (CephadmContext, List[str], str, Optional[int], Optional[int]) -> None + """ + Copy a directory tree from src to dst + """ + if uid is None or gid is None: + (uid, gid) = extract_uid_gid(ctx) + + for src_dir in src: + dst_dir = dst + if os.path.isdir(dst): + dst_dir = os.path.join(dst, os.path.basename(src_dir)) + + logger.debug('copy directory `%s` -> `%s`' % (src_dir, dst_dir)) + shutil.rmtree(dst_dir, ignore_errors=True) + shutil.copytree(src_dir, dst_dir) # dirs_exist_ok needs python 3.8 + + for dirpath, dirnames, filenames in os.walk(dst_dir): + logger.debug('chown %s:%s `%s`' % (uid, gid, dirpath)) + os.chown(dirpath, uid, gid) + for filename in filenames: + logger.debug('chown %s:%s `%s`' % (uid, gid, filename)) + os.chown(os.path.join(dirpath, filename), uid, gid) + + +def copy_files(ctx, src, dst, uid=None, gid=None): + # type: (CephadmContext, List[str], str, Optional[int], Optional[int]) -> None + """ + Copy a files from src to dst + """ + if uid is None or gid is None: + (uid, gid) = extract_uid_gid(ctx) + + for src_file in src: + dst_file = dst + if os.path.isdir(dst): + dst_file = os.path.join(dst, os.path.basename(src_file)) + + logger.debug('copy file `%s` -> `%s`' % (src_file, dst_file)) + shutil.copyfile(src_file, dst_file) + + logger.debug('chown %s:%s `%s`' % (uid, gid, dst_file)) + os.chown(dst_file, uid, gid) + + +def move_files(ctx, src, dst, uid=None, gid=None): + # type: (CephadmContext, List[str], str, Optional[int], Optional[int]) -> None + """ + Move files from src to dst + """ + if uid is None or gid is None: + (uid, gid) = extract_uid_gid(ctx) + + for src_file in src: + dst_file = dst + if os.path.isdir(dst): + dst_file = os.path.join(dst, os.path.basename(src_file)) + + if os.path.islink(src_file): + # shutil.move() in py2 does not handle symlinks correctly + src_rl = os.readlink(src_file) + logger.debug("symlink '%s' -> '%s'" % (dst_file, src_rl)) + os.symlink(src_rl, dst_file) + os.unlink(src_file) + else: + logger.debug("move file '%s' -> '%s'" % (src_file, dst_file)) + shutil.move(src_file, dst_file) + logger.debug('chown %s:%s `%s`' % (uid, gid, dst_file)) + os.chown(dst_file, uid, gid) + + +def recursive_chown(path: str, uid: int, gid: int) -> None: + for dirpath, dirnames, filenames in os.walk(path): + os.chown(dirpath, uid, gid) + for filename in filenames: + os.chown(os.path.join(dirpath, filename), uid, gid) + + +# copied from distutils +def find_executable(executable: str, path: Optional[str] = None) -> Optional[str]: + """Tries to find 'executable' in the directories listed in 'path'. + A string listing directories separated by 'os.pathsep'; defaults to + os.environ['PATH']. Returns the complete filename or None if not found. + """ + _, ext = os.path.splitext(executable) + if (sys.platform == 'win32') and (ext != '.exe'): + executable = executable + '.exe' + + if os.path.isfile(executable): + return executable + + if path is None: + path = os.environ.get('PATH', None) + if path is None: + try: + path = os.confstr('CS_PATH') + except (AttributeError, ValueError): + # os.confstr() or CS_PATH is not available + path = os.defpath + # bpo-35755: Don't use os.defpath if the PATH environment variable is + # set to an empty string + + # PATH='' doesn't match, whereas PATH=':' looks in the current directory + if not path: + return None + + paths = path.split(os.pathsep) + for p in paths: + f = os.path.join(p, executable) + if os.path.isfile(f): + # the file exists, we have a shot at spawn working + return f + return None + + +def find_program(filename): + # type: (str) -> str + name = find_executable(filename) + if name is None: + raise ValueError('%s not found' % filename) + return name + + +def find_container_engine(ctx: CephadmContext) -> Optional[ContainerEngine]: + if ctx.docker: + return Docker() + else: + for i in CONTAINER_PREFERENCE: + try: + return i() + except Exception: + pass + return None + + +def check_container_engine(ctx: CephadmContext) -> ContainerEngine: + engine = ctx.container_engine + if not isinstance(engine, CONTAINER_PREFERENCE): + # See https://github.com/python/mypy/issues/8993 + exes: List[str] = [i.EXE for i in CONTAINER_PREFERENCE] # type: ignore + raise Error('No container engine binary found ({}). Try run `apt/dnf/yum/zypper install <container engine>`'.format(' or '.join(exes))) + elif isinstance(engine, Podman): + engine.get_version(ctx) + if engine.version < MIN_PODMAN_VERSION: + raise Error('podman version %d.%d.%d or later is required' % MIN_PODMAN_VERSION) + return engine + + +def get_unit_name(fsid, daemon_type, daemon_id=None): + # type: (str, str, Optional[Union[int, str]]) -> str + # accept either name or type + id + if daemon_type == CephadmDaemon.daemon_type and daemon_id is not None: + return 'ceph-%s-%s.%s' % (fsid, daemon_type, daemon_id) + elif daemon_id is not None: + return 'ceph-%s@%s.%s' % (fsid, daemon_type, daemon_id) + else: + return 'ceph-%s@%s' % (fsid, daemon_type) + + +def get_unit_name_by_daemon_name(ctx: CephadmContext, fsid: str, name: str) -> str: + daemon = get_daemon_description(ctx, fsid, name) + try: + return daemon['systemd_unit'] + except KeyError: + raise Error('Failed to get unit name for {}'.format(daemon)) + + +def check_unit(ctx, unit_name): + # type: (CephadmContext, str) -> Tuple[bool, str, bool] + # NOTE: we ignore the exit code here because systemctl outputs + # various exit codes based on the state of the service, but the + # string result is more explicit (and sufficient). + enabled = False + installed = False + try: + out, err, code = call(ctx, ['systemctl', 'is-enabled', unit_name], + verbosity=CallVerbosity.DEBUG) + if code == 0: + enabled = True + installed = True + elif 'disabled' in out: + installed = True + except Exception as e: + logger.warning('unable to run systemctl: %s' % e) + enabled = False + installed = False + + state = 'unknown' + try: + out, err, code = call(ctx, ['systemctl', 'is-active', unit_name], + verbosity=CallVerbosity.DEBUG) + out = out.strip() + if out in ['active']: + state = 'running' + elif out in ['inactive']: + state = 'stopped' + elif out in ['failed', 'auto-restart']: + state = 'error' + else: + state = 'unknown' + except Exception as e: + logger.warning('unable to run systemctl: %s' % e) + state = 'unknown' + return (enabled, state, installed) + + +def check_units(ctx, units, enabler=None): + # type: (CephadmContext, List[str], Optional[Packager]) -> bool + for u in units: + (enabled, state, installed) = check_unit(ctx, u) + if enabled and state == 'running': + logger.info('Unit %s is enabled and running' % u) + return True + if enabler is not None: + if installed: + logger.info('Enabling unit %s' % u) + enabler.enable_service(u) + return False + + +def is_container_running(ctx: CephadmContext, c: 'CephContainer') -> bool: + return bool(get_running_container_name(ctx, c)) + + +def get_running_container_name(ctx: CephadmContext, c: 'CephContainer') -> Optional[str]: + for name in [c.cname, c.old_cname]: + out, err, ret = call(ctx, [ + ctx.container_engine.path, 'container', 'inspect', + '--format', '{{.State.Status}}', name + ]) + if out.strip() == 'running': + return name + return None + + +def get_legacy_config_fsid(cluster, legacy_dir=None): + # type: (str, Optional[str]) -> Optional[str] + config_file = '/etc/ceph/%s.conf' % cluster + if legacy_dir is not None: + config_file = os.path.abspath(legacy_dir + config_file) + + if os.path.exists(config_file): + config = read_config(config_file) + if config.has_section('global') and config.has_option('global', 'fsid'): + return config.get('global', 'fsid') + return None + + +def get_legacy_daemon_fsid(ctx, cluster, + daemon_type, daemon_id, legacy_dir=None): + # type: (CephadmContext, str, str, Union[int, str], Optional[str]) -> Optional[str] + fsid = None + if daemon_type == 'osd': + try: + fsid_file = os.path.join(ctx.data_dir, + daemon_type, + 'ceph-%s' % daemon_id, + 'ceph_fsid') + if legacy_dir is not None: + fsid_file = os.path.abspath(legacy_dir + fsid_file) + with open(fsid_file, 'r') as f: + fsid = f.read().strip() + except IOError: + pass + if not fsid: + fsid = get_legacy_config_fsid(cluster, legacy_dir=legacy_dir) + return fsid + + +def get_daemon_args(ctx, fsid, daemon_type, daemon_id): + # type: (CephadmContext, str, str, Union[int, str]) -> List[str] + r = list() # type: List[str] + + if daemon_type in Ceph.daemons and daemon_type != 'crash': + r += [ + '--setuser', 'ceph', + '--setgroup', 'ceph', + '--default-log-to-file=false', + '--default-log-to-stderr=true', + '--default-log-stderr-prefix=debug ', + ] + if daemon_type == 'mon': + r += [ + '--default-mon-cluster-log-to-file=false', + '--default-mon-cluster-log-to-stderr=true', + ] + elif daemon_type in Monitoring.components: + metadata = Monitoring.components[daemon_type] + r += metadata.get('args', list()) + # set ip and port to bind to for nodeexporter,alertmanager,prometheus + if daemon_type != 'grafana': + ip = '' + port = Monitoring.port_map[daemon_type][0] + if 'meta_json' in ctx and ctx.meta_json: + meta = json.loads(ctx.meta_json) or {} + if 'ip' in meta and meta['ip']: + ip = meta['ip'] + if 'ports' in meta and meta['ports']: + port = meta['ports'][0] + r += [f'--web.listen-address={ip}:{port}'] + if daemon_type == 'prometheus': + config = get_parm(ctx.config_json) + retention_time = config.get('retention_time', '15d') + r += [f'--storage.tsdb.retention.time={retention_time}'] + scheme = 'http' + host = get_fqdn() + r += [f'--web.external-url={scheme}://{host}:{port}'] + if daemon_type == 'alertmanager': + config = get_parm(ctx.config_json) + peers = config.get('peers', list()) # type: ignore + for peer in peers: + r += ['--cluster.peer={}'.format(peer)] + # some alertmanager, by default, look elsewhere for a config + r += ['--config.file=/etc/alertmanager/alertmanager.yml'] + if daemon_type == 'node-exporter': + r += ['--path.procfs=/host/proc', + '--path.sysfs=/host/sys', + '--path.rootfs=/rootfs'] + elif daemon_type == NFSGanesha.daemon_type: + nfs_ganesha = NFSGanesha.init(ctx, fsid, daemon_id) + r += nfs_ganesha.get_daemon_args() + elif daemon_type == HAproxy.daemon_type: + haproxy = HAproxy.init(ctx, fsid, daemon_id) + r += haproxy.get_daemon_args() + elif daemon_type == CustomContainer.daemon_type: + cc = CustomContainer.init(ctx, fsid, daemon_id) + r.extend(cc.get_daemon_args()) + elif daemon_type == SNMPGateway.daemon_type: + sc = SNMPGateway.init(ctx, fsid, daemon_id) + r.extend(sc.get_daemon_args()) + + return r + + +def create_daemon_dirs(ctx, fsid, daemon_type, daemon_id, uid, gid, + config=None, keyring=None): + # type: (CephadmContext, str, str, Union[int, str], int, int, Optional[str], Optional[str]) -> None + data_dir = make_data_dir(ctx, fsid, daemon_type, daemon_id, uid=uid, gid=gid) + + if daemon_type in Ceph.daemons: + make_log_dir(ctx, fsid, uid=uid, gid=gid) + + if config: + config_path = os.path.join(data_dir, 'config') + with open(config_path, 'w') as f: + os.fchown(f.fileno(), uid, gid) + os.fchmod(f.fileno(), 0o600) + f.write(config) + + if keyring: + keyring_path = os.path.join(data_dir, 'keyring') + with open(keyring_path, 'w') as f: + os.fchmod(f.fileno(), 0o600) + os.fchown(f.fileno(), uid, gid) + f.write(keyring) + + if daemon_type in Monitoring.components.keys(): + config_json: Dict[str, Any] = dict() + if 'config_json' in ctx: + config_json = get_parm(ctx.config_json) + + # Set up directories specific to the monitoring component + config_dir = '' + data_dir_root = '' + if daemon_type == 'prometheus': + data_dir_root = get_data_dir(fsid, ctx.data_dir, + daemon_type, daemon_id) + config_dir = 'etc/prometheus' + makedirs(os.path.join(data_dir_root, config_dir), uid, gid, 0o755) + makedirs(os.path.join(data_dir_root, config_dir, 'alerting'), uid, gid, 0o755) + makedirs(os.path.join(data_dir_root, 'data'), uid, gid, 0o755) + recursive_chown(os.path.join(data_dir_root, 'etc'), uid, gid) + recursive_chown(os.path.join(data_dir_root, 'data'), uid, gid) + elif daemon_type == 'grafana': + data_dir_root = get_data_dir(fsid, ctx.data_dir, + daemon_type, daemon_id) + config_dir = 'etc/grafana' + makedirs(os.path.join(data_dir_root, config_dir), uid, gid, 0o755) + makedirs(os.path.join(data_dir_root, config_dir, 'certs'), uid, gid, 0o755) + makedirs(os.path.join(data_dir_root, config_dir, 'provisioning/datasources'), uid, gid, 0o755) + makedirs(os.path.join(data_dir_root, 'data'), uid, gid, 0o755) + touch(os.path.join(data_dir_root, 'data', 'grafana.db'), uid, gid) + elif daemon_type == 'alertmanager': + data_dir_root = get_data_dir(fsid, ctx.data_dir, + daemon_type, daemon_id) + config_dir = 'etc/alertmanager' + makedirs(os.path.join(data_dir_root, config_dir), uid, gid, 0o755) + makedirs(os.path.join(data_dir_root, config_dir, 'data'), uid, gid, 0o755) + + # populate the config directory for the component from the config-json + if 'files' in config_json: + for fname in config_json['files']: + content = dict_get_join(config_json['files'], fname) + if os.path.isabs(fname): + fpath = os.path.join(data_dir_root, fname.lstrip(os.path.sep)) + else: + fpath = os.path.join(data_dir_root, config_dir, fname) + with open(fpath, 'w', encoding='utf-8') as f: + os.fchown(f.fileno(), uid, gid) + os.fchmod(f.fileno(), 0o600) + f.write(content) + + elif daemon_type == NFSGanesha.daemon_type: + nfs_ganesha = NFSGanesha.init(ctx, fsid, daemon_id) + nfs_ganesha.create_daemon_dirs(data_dir, uid, gid) + + elif daemon_type == CephIscsi.daemon_type: + ceph_iscsi = CephIscsi.init(ctx, fsid, daemon_id) + ceph_iscsi.create_daemon_dirs(data_dir, uid, gid) + + elif daemon_type == HAproxy.daemon_type: + haproxy = HAproxy.init(ctx, fsid, daemon_id) + haproxy.create_daemon_dirs(data_dir, uid, gid) + + elif daemon_type == Keepalived.daemon_type: + keepalived = Keepalived.init(ctx, fsid, daemon_id) + keepalived.create_daemon_dirs(data_dir, uid, gid) + + elif daemon_type == CustomContainer.daemon_type: + cc = CustomContainer.init(ctx, fsid, daemon_id) + cc.create_daemon_dirs(data_dir, uid, gid) + + elif daemon_type == SNMPGateway.daemon_type: + sg = SNMPGateway.init(ctx, fsid, daemon_id) + sg.create_daemon_conf() + + +def get_parm(option): + # type: (str) -> Dict[str, str] + + if not option: + return dict() + + global cached_stdin + if option == '-': + if cached_stdin is not None: + j = cached_stdin + else: + j = sys.stdin.read() + cached_stdin = j + else: + # inline json string + if option[0] == '{' and option[-1] == '}': + j = option + # json file + elif os.path.exists(option): + with open(option, 'r') as f: + j = f.read() + else: + raise Error('Config file {} not found'.format(option)) + + try: + js = json.loads(j) + except ValueError as e: + raise Error('Invalid JSON in {}: {}'.format(option, e)) + else: + return js + + +def get_config_and_keyring(ctx): + # type: (CephadmContext) -> Tuple[Optional[str], Optional[str]] + config = None + keyring = None + + if 'config_json' in ctx and ctx.config_json: + d = get_parm(ctx.config_json) + config = d.get('config') + keyring = d.get('keyring') + if config and keyring: + return config, keyring + + if 'config' in ctx and ctx.config: + try: + with open(ctx.config, 'r') as f: + config = f.read() + except FileNotFoundError as e: + raise Error(e) + + if 'key' in ctx and ctx.key: + keyring = '[%s]\n\tkey = %s\n' % (ctx.name, ctx.key) + elif 'keyring' in ctx and ctx.keyring: + try: + with open(ctx.keyring, 'r') as f: + keyring = f.read() + except FileNotFoundError as e: + raise Error(e) + + return config, keyring + + +def get_container_binds(ctx, fsid, daemon_type, daemon_id): + # type: (CephadmContext, str, str, Union[int, str, None]) -> List[List[str]] + binds = list() + + if daemon_type == CephIscsi.daemon_type: + binds.extend(CephIscsi.get_container_binds()) + elif daemon_type == CustomContainer.daemon_type: + assert daemon_id + cc = CustomContainer.init(ctx, fsid, daemon_id) + data_dir = get_data_dir(fsid, ctx.data_dir, daemon_type, daemon_id) + binds.extend(cc.get_container_binds(data_dir)) + + return binds + + +def get_container_mounts(ctx, fsid, daemon_type, daemon_id, + no_config=False): + # type: (CephadmContext, str, str, Union[int, str, None], Optional[bool]) -> Dict[str, str] + mounts = dict() + + if daemon_type in Ceph.daemons: + if fsid: + run_path = os.path.join('/var/run/ceph', fsid) + if os.path.exists(run_path): + mounts[run_path] = '/var/run/ceph:z' + log_dir = get_log_dir(fsid, ctx.log_dir) + mounts[log_dir] = '/var/log/ceph:z' + crash_dir = '/var/lib/ceph/%s/crash' % fsid + if os.path.exists(crash_dir): + mounts[crash_dir] = '/var/lib/ceph/crash:z' + + if daemon_type in Ceph.daemons and daemon_id: + data_dir = get_data_dir(fsid, ctx.data_dir, daemon_type, daemon_id) + if daemon_type == 'rgw': + cdata_dir = '/var/lib/ceph/radosgw/ceph-rgw.%s' % (daemon_id) + else: + cdata_dir = '/var/lib/ceph/%s/ceph-%s' % (daemon_type, daemon_id) + if daemon_type != 'crash': + mounts[data_dir] = cdata_dir + ':z' + if not no_config: + mounts[data_dir + '/config'] = '/etc/ceph/ceph.conf:z' + if daemon_type in ['rbd-mirror', 'cephfs-mirror', 'crash']: + # these do not search for their keyrings in a data directory + mounts[data_dir + '/keyring'] = '/etc/ceph/ceph.client.%s.%s.keyring' % (daemon_type, daemon_id) + + if daemon_type in ['mon', 'osd', 'clusterless-ceph-volume']: + mounts['/dev'] = '/dev' # FIXME: narrow this down? + mounts['/run/udev'] = '/run/udev' + if daemon_type in ['osd', 'clusterless-ceph-volume']: + mounts['/sys'] = '/sys' # for numa.cc, pick_address, cgroups, ... + mounts['/run/lvm'] = '/run/lvm' + mounts['/run/lock/lvm'] = '/run/lock/lvm' + if daemon_type == 'osd': + # selinux-policy in the container may not match the host. + if HostFacts(ctx).selinux_enabled: + selinux_folder = '/var/lib/ceph/%s/selinux' % fsid + if not os.path.exists(selinux_folder): + os.makedirs(selinux_folder, mode=0o755) + mounts[selinux_folder] = '/sys/fs/selinux:ro' + mounts['/'] = '/rootfs' + + try: + if ctx.shared_ceph_folder: # make easy manager modules/ceph-volume development + ceph_folder = pathify(ctx.shared_ceph_folder) + if os.path.exists(ceph_folder): + mounts[ceph_folder + '/src/ceph-volume/ceph_volume'] = '/usr/lib/python3.6/site-packages/ceph_volume' + mounts[ceph_folder + '/src/cephadm/cephadm'] = '/usr/sbin/cephadm' + mounts[ceph_folder + '/src/pybind/mgr'] = '/usr/share/ceph/mgr' + mounts[ceph_folder + '/src/python-common/ceph'] = '/usr/lib/python3.6/site-packages/ceph' + mounts[ceph_folder + '/monitoring/ceph-mixin/dashboards_out'] = '/etc/grafana/dashboards/ceph-dashboard' + mounts[ceph_folder + '/monitoring/ceph-mixin/prometheus_alerts.yml'] = '/etc/prometheus/ceph/ceph_default_alerts.yml' + else: + logger.error('{}{}{}'.format(termcolor.red, + 'Ceph shared source folder does not exist.', + termcolor.end)) + except AttributeError: + pass + + if daemon_type in Monitoring.components and daemon_id: + data_dir = get_data_dir(fsid, ctx.data_dir, daemon_type, daemon_id) + if daemon_type == 'prometheus': + mounts[os.path.join(data_dir, 'etc/prometheus')] = '/etc/prometheus:Z' + mounts[os.path.join(data_dir, 'data')] = '/prometheus:Z' + elif daemon_type == 'node-exporter': + mounts['/proc'] = '/host/proc:ro' + mounts['/sys'] = '/host/sys:ro' + mounts['/'] = '/rootfs:ro' + elif daemon_type == 'grafana': + mounts[os.path.join(data_dir, 'etc/grafana/grafana.ini')] = '/etc/grafana/grafana.ini:Z' + mounts[os.path.join(data_dir, 'etc/grafana/provisioning/datasources')] = '/etc/grafana/provisioning/datasources:Z' + mounts[os.path.join(data_dir, 'etc/grafana/certs')] = '/etc/grafana/certs:Z' + mounts[os.path.join(data_dir, 'data/grafana.db')] = '/var/lib/grafana/grafana.db:Z' + elif daemon_type == 'alertmanager': + mounts[os.path.join(data_dir, 'etc/alertmanager')] = '/etc/alertmanager:Z' + + if daemon_type == NFSGanesha.daemon_type: + assert daemon_id + data_dir = get_data_dir(fsid, ctx.data_dir, daemon_type, daemon_id) + nfs_ganesha = NFSGanesha.init(ctx, fsid, daemon_id) + mounts.update(nfs_ganesha.get_container_mounts(data_dir)) + + if daemon_type == HAproxy.daemon_type: + assert daemon_id + data_dir = get_data_dir(fsid, ctx.data_dir, daemon_type, daemon_id) + mounts.update(HAproxy.get_container_mounts(data_dir)) + + if daemon_type == CephIscsi.daemon_type: + assert daemon_id + data_dir = get_data_dir(fsid, ctx.data_dir, daemon_type, daemon_id) + log_dir = get_log_dir(fsid, ctx.log_dir) + mounts.update(CephIscsi.get_container_mounts(data_dir, log_dir)) + + if daemon_type == Keepalived.daemon_type: + assert daemon_id + data_dir = get_data_dir(fsid, ctx.data_dir, daemon_type, daemon_id) + mounts.update(Keepalived.get_container_mounts(data_dir)) + + if daemon_type == CustomContainer.daemon_type: + assert daemon_id + cc = CustomContainer.init(ctx, fsid, daemon_id) + data_dir = get_data_dir(fsid, ctx.data_dir, daemon_type, daemon_id) + mounts.update(cc.get_container_mounts(data_dir)) + + return mounts + + +def get_ceph_volume_container(ctx: CephadmContext, + privileged: bool = True, + cname: str = '', + volume_mounts: Dict[str, str] = {}, + bind_mounts: Optional[List[List[str]]] = None, + args: List[str] = [], + envs: Optional[List[str]] = None) -> 'CephContainer': + if envs is None: + envs = [] + envs.append('CEPH_VOLUME_SKIP_RESTORECON=yes') + envs.append('CEPH_VOLUME_DEBUG=1') + + return CephContainer( + ctx, + image=ctx.image, + entrypoint='/usr/sbin/ceph-volume', + args=args, + volume_mounts=volume_mounts, + bind_mounts=bind_mounts, + envs=envs, + privileged=privileged, + cname=cname, + memory_request=ctx.memory_request, + memory_limit=ctx.memory_limit, + ) + + +def get_container(ctx: CephadmContext, + fsid: str, daemon_type: str, daemon_id: Union[int, str], + privileged: bool = False, + ptrace: bool = False, + container_args: Optional[List[str]] = None) -> 'CephContainer': + entrypoint: str = '' + name: str = '' + ceph_args: List[str] = [] + envs: List[str] = [] + host_network: bool = True + + if daemon_type in Ceph.daemons: + envs.append('TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES=134217728') + if container_args is None: + container_args = [] + if daemon_type in ['mon', 'osd']: + # mon and osd need privileged in order for libudev to query devices + privileged = True + if daemon_type == 'rgw': + entrypoint = '/usr/bin/radosgw' + name = 'client.rgw.%s' % daemon_id + elif daemon_type == 'rbd-mirror': + entrypoint = '/usr/bin/rbd-mirror' + name = 'client.rbd-mirror.%s' % daemon_id + elif daemon_type == 'cephfs-mirror': + entrypoint = '/usr/bin/cephfs-mirror' + name = 'client.cephfs-mirror.%s' % daemon_id + elif daemon_type == 'crash': + entrypoint = '/usr/bin/ceph-crash' + name = 'client.crash.%s' % daemon_id + elif daemon_type in ['mon', 'mgr', 'mds', 'osd']: + entrypoint = '/usr/bin/ceph-' + daemon_type + name = '%s.%s' % (daemon_type, daemon_id) + elif daemon_type in Monitoring.components: + entrypoint = '' + elif daemon_type == NFSGanesha.daemon_type: + entrypoint = NFSGanesha.entrypoint + name = '%s.%s' % (daemon_type, daemon_id) + envs.extend(NFSGanesha.get_container_envs()) + elif daemon_type == HAproxy.daemon_type: + name = '%s.%s' % (daemon_type, daemon_id) + container_args.extend(['--user=root']) # haproxy 2.4 defaults to a different user + elif daemon_type == Keepalived.daemon_type: + name = '%s.%s' % (daemon_type, daemon_id) + envs.extend(Keepalived.get_container_envs()) + container_args.extend(['--cap-add=NET_ADMIN', '--cap-add=NET_RAW']) + elif daemon_type == CephIscsi.daemon_type: + entrypoint = CephIscsi.entrypoint + name = '%s.%s' % (daemon_type, daemon_id) + # So the container can modprobe iscsi_target_mod and have write perms + # to configfs we need to make this a privileged container. + privileged = True + elif daemon_type == CustomContainer.daemon_type: + cc = CustomContainer.init(ctx, fsid, daemon_id) + entrypoint = cc.entrypoint + host_network = False + envs.extend(cc.get_container_envs()) + container_args.extend(cc.get_container_args()) + + if daemon_type in Monitoring.components: + uid, gid = extract_uid_gid_monitoring(ctx, daemon_type) + monitoring_args = [ + '--user', + str(uid), + # FIXME: disable cpu/memory limits for the time being (not supported + # by ubuntu 18.04 kernel!) + ] + container_args.extend(monitoring_args) + if daemon_type == 'node-exporter': + # in order to support setting '--path.procfs=/host/proc','--path.sysfs=/host/sys', + # '--path.rootfs=/rootfs' for node-exporter we need to disable selinux separation + # between the node-exporter container and the host to avoid selinux denials + container_args.extend(['--security-opt', 'label=disable']) + elif daemon_type == 'crash': + ceph_args = ['-n', name] + elif daemon_type in Ceph.daemons: + ceph_args = ['-n', name, '-f'] + elif daemon_type == SNMPGateway.daemon_type: + sg = SNMPGateway.init(ctx, fsid, daemon_id) + container_args.append( + f'--env-file={sg.conf_file_path}' + ) + + # if using podman, set -d, --conmon-pidfile & --cidfile flags + # so service can have Type=Forking + if isinstance(ctx.container_engine, Podman): + runtime_dir = '/run' + container_args.extend([ + '-d', '--log-driver', 'journald', + '--conmon-pidfile', + runtime_dir + '/ceph-%s@%s.%s.service-pid' % (fsid, daemon_type, daemon_id), + '--cidfile', + runtime_dir + '/ceph-%s@%s.%s.service-cid' % (fsid, daemon_type, daemon_id), + ]) + if ctx.container_engine.version >= CGROUPS_SPLIT_PODMAN_VERSION: + container_args.append('--cgroups=split') + + return CephContainer.for_daemon( + ctx, + fsid=fsid, + daemon_type=daemon_type, + daemon_id=str(daemon_id), + entrypoint=entrypoint, + args=ceph_args + get_daemon_args(ctx, fsid, daemon_type, daemon_id), + container_args=container_args, + volume_mounts=get_container_mounts(ctx, fsid, daemon_type, daemon_id), + bind_mounts=get_container_binds(ctx, fsid, daemon_type, daemon_id), + envs=envs, + privileged=privileged, + ptrace=ptrace, + host_network=host_network, + ) + + +def extract_uid_gid(ctx, img='', file_path='/var/lib/ceph'): + # type: (CephadmContext, str, Union[str, List[str]]) -> Tuple[int, int] + + if not img: + img = ctx.image + + if isinstance(file_path, str): + paths = [file_path] + else: + paths = file_path + + ex: Optional[Tuple[str, RuntimeError]] = None + + for fp in paths: + try: + out = CephContainer( + ctx, + image=img, + entrypoint='stat', + args=['-c', '%u %g', fp] + ).run() + uid, gid = out.split(' ') + return int(uid), int(gid) + except RuntimeError as e: + ex = (fp, e) + if ex: + raise Error(f'Failed to extract uid/gid for path {ex[0]}: {ex[1]}') + + raise RuntimeError('uid/gid not found') + + +def deploy_daemon(ctx, fsid, daemon_type, daemon_id, c, uid, gid, + config=None, keyring=None, + osd_fsid=None, + reconfig=False, + ports=None): + # type: (CephadmContext, str, str, Union[int, str], Optional[CephContainer], int, int, Optional[str], Optional[str], Optional[str], Optional[bool], Optional[List[int]]) -> None + + ports = ports or [] + if any([port_in_use(ctx, port) for port in ports]): + if daemon_type == 'mgr': + # non-fatal for mgr when we are in mgr_standby_modules=false, but we can't + # tell whether that is the case here. + logger.warning( + f"ceph-mgr TCP port(s) {','.join(map(str, ports))} already in use" + ) + else: + raise Error("TCP Port(s) '{}' required for {} already in use".format(','.join(map(str, ports)), daemon_type)) + + data_dir = get_data_dir(fsid, ctx.data_dir, daemon_type, daemon_id) + if reconfig and not os.path.exists(data_dir): + raise Error('cannot reconfig, data path %s does not exist' % data_dir) + if daemon_type == 'mon' and not os.path.exists(data_dir): + assert config + assert keyring + # tmp keyring file + tmp_keyring = write_tmp(keyring, uid, gid) + + # tmp config file + tmp_config = write_tmp(config, uid, gid) + + # --mkfs + create_daemon_dirs(ctx, fsid, daemon_type, daemon_id, uid, gid) + mon_dir = get_data_dir(fsid, ctx.data_dir, 'mon', daemon_id) + log_dir = get_log_dir(fsid, ctx.log_dir) + CephContainer( + ctx, + image=ctx.image, + entrypoint='/usr/bin/ceph-mon', + args=[ + '--mkfs', + '-i', str(daemon_id), + '--fsid', fsid, + '-c', '/tmp/config', + '--keyring', '/tmp/keyring', + ] + get_daemon_args(ctx, fsid, 'mon', daemon_id), + volume_mounts={ + log_dir: '/var/log/ceph:z', + mon_dir: '/var/lib/ceph/mon/ceph-%s:z' % (daemon_id), + tmp_keyring.name: '/tmp/keyring:z', + tmp_config.name: '/tmp/config:z', + }, + ).run() + + # write conf + with open(mon_dir + '/config', 'w') as f: + os.fchown(f.fileno(), uid, gid) + os.fchmod(f.fileno(), 0o600) + f.write(config) + else: + # dirs, conf, keyring + create_daemon_dirs( + ctx, + fsid, daemon_type, daemon_id, + uid, gid, + config, keyring) + + if not reconfig: + if daemon_type == CephadmDaemon.daemon_type: + port = next(iter(ports), None) # get first tcp port provided or None + + if ctx.config_json == '-': + config_js = get_parm('-') + else: + config_js = get_parm(ctx.config_json) + assert isinstance(config_js, dict) + assert isinstance(daemon_id, str) + + cephadm_exporter = CephadmDaemon(ctx, fsid, daemon_id, port) + cephadm_exporter.deploy_daemon_unit(config_js) + else: + if c: + deploy_daemon_units(ctx, fsid, uid, gid, daemon_type, daemon_id, + c, osd_fsid=osd_fsid, ports=ports) + else: + raise RuntimeError('attempting to deploy a daemon without a container image') + + if not os.path.exists(data_dir + '/unit.created'): + with open(data_dir + '/unit.created', 'w') as f: + os.fchmod(f.fileno(), 0o600) + os.fchown(f.fileno(), uid, gid) + f.write('mtime is time the daemon deployment was created\n') + + with open(data_dir + '/unit.configured', 'w') as f: + f.write('mtime is time we were last configured\n') + os.fchmod(f.fileno(), 0o600) + os.fchown(f.fileno(), uid, gid) + + update_firewalld(ctx, daemon_type) + + # Open ports explicitly required for the daemon + if ports: + fw = Firewalld(ctx) + fw.open_ports(ports) + fw.apply_rules() + + if reconfig and daemon_type not in Ceph.daemons: + # ceph daemons do not need a restart; others (presumably) do to pick + # up the new config + call_throws(ctx, ['systemctl', 'reset-failed', + get_unit_name(fsid, daemon_type, daemon_id)]) + call_throws(ctx, ['systemctl', 'restart', + get_unit_name(fsid, daemon_type, daemon_id)]) + + +def _write_container_cmd_to_bash(ctx, file_obj, container, comment=None, background=False): + # type: (CephadmContext, IO[str], CephContainer, Optional[str], Optional[bool]) -> None + if comment: + # Sometimes adding a comment, especially if there are multiple containers in one + # unit file, makes it easier to read and grok. + file_obj.write('# ' + comment + '\n') + # Sometimes, adding `--rm` to a run_cmd doesn't work. Let's remove the container manually + file_obj.write('! ' + ' '.join(container.rm_cmd(old_cname=True)) + ' 2> /dev/null\n') + file_obj.write('! ' + ' '.join(container.rm_cmd()) + ' 2> /dev/null\n') + # Sometimes, `podman rm` doesn't find the container. Then you'll have to add `--storage` + if isinstance(ctx.container_engine, Podman): + file_obj.write( + '! ' + + ' '.join([shlex.quote(a) for a in container.rm_cmd(storage=True)]) + + ' 2> /dev/null\n') + file_obj.write( + '! ' + + ' '.join([shlex.quote(a) for a in container.rm_cmd(old_cname=True, storage=True)]) + + ' 2> /dev/null\n') + + # container run command + file_obj.write( + ' '.join([shlex.quote(a) for a in container.run_cmd()]) + + (' &' if background else '') + '\n') + + +def clean_cgroup(ctx: CephadmContext, fsid: str, unit_name: str) -> None: + # systemd may fail to cleanup cgroups from previous stopped unit, which will cause next "systemctl start" to fail. + # see https://tracker.ceph.com/issues/50998 + + CGROUPV2_PATH = Path('/sys/fs/cgroup') + if not (CGROUPV2_PATH / 'system.slice').exists(): + # Only unified cgroup is affected, skip if not the case + return + + slice_name = 'system-ceph\\x2d{}.slice'.format(fsid.replace('-', '\\x2d')) + cg_path = CGROUPV2_PATH / 'system.slice' / slice_name / f'{unit_name}.service' + if not cg_path.exists(): + return + + def cg_trim(path: Path) -> None: + for p in path.iterdir(): + if p.is_dir(): + cg_trim(p) + path.rmdir() + try: + cg_trim(cg_path) + except OSError: + logger.warning(f'Failed to trim old cgroups {cg_path}') + + +def deploy_daemon_units( + ctx: CephadmContext, + fsid: str, + uid: int, + gid: int, + daemon_type: str, + daemon_id: Union[int, str], + c: 'CephContainer', + enable: bool = True, + start: bool = True, + osd_fsid: Optional[str] = None, + ports: Optional[List[int]] = None, +) -> None: + # cmd + data_dir = get_data_dir(fsid, ctx.data_dir, daemon_type, daemon_id) + with open(data_dir + '/unit.run.new', 'w') as f, \ + open(data_dir + '/unit.meta.new', 'w') as metaf: + f.write('set -e\n') + + if daemon_type in Ceph.daemons: + install_path = find_program('install') + f.write('{install_path} -d -m0770 -o {uid} -g {gid} /var/run/ceph/{fsid}\n'.format(install_path=install_path, fsid=fsid, uid=uid, gid=gid)) + + # pre-start cmd(s) + if daemon_type == 'osd': + # osds have a pre-start step + assert osd_fsid + simple_fn = os.path.join('/etc/ceph/osd', + '%s-%s.json.adopted-by-cephadm' % (daemon_id, osd_fsid)) + if os.path.exists(simple_fn): + f.write('# Simple OSDs need chown on startup:\n') + for n in ['block', 'block.db', 'block.wal']: + p = os.path.join(data_dir, n) + f.write('[ ! -L {p} ] || chown {uid}:{gid} {p}\n'.format(p=p, uid=uid, gid=gid)) + else: + # if ceph-volume does not support 'ceph-volume activate', we must + # do 'ceph-volume lvm activate'. + test_cv = get_ceph_volume_container( + ctx, + args=['activate', '--bad-option'], + volume_mounts=get_container_mounts(ctx, fsid, daemon_type, daemon_id), + bind_mounts=get_container_binds(ctx, fsid, daemon_type, daemon_id), + cname='ceph-%s-%s.%s-activate-test' % (fsid, daemon_type, daemon_id), + ) + out, err, ret = call(ctx, test_cv.run_cmd(), verbosity=CallVerbosity.SILENT) + # bad: ceph-volume: error: unrecognized arguments: activate --bad-option + # good: ceph-volume: error: unrecognized arguments: --bad-option + if 'unrecognized arguments: activate' in err: + # older ceph-volume without top-level activate or --no-tmpfs + cmd = [ + 'lvm', 'activate', + str(daemon_id), osd_fsid, + '--no-systemd', + ] + else: + cmd = [ + 'activate', + '--osd-id', str(daemon_id), + '--osd-uuid', osd_fsid, + '--no-systemd', + '--no-tmpfs', + ] + + prestart = get_ceph_volume_container( + ctx, + args=cmd, + volume_mounts=get_container_mounts(ctx, fsid, daemon_type, daemon_id), + bind_mounts=get_container_binds(ctx, fsid, daemon_type, daemon_id), + cname='ceph-%s-%s.%s-activate' % (fsid, daemon_type, daemon_id), + ) + _write_container_cmd_to_bash(ctx, f, prestart, 'LVM OSDs use ceph-volume lvm activate') + elif daemon_type == CephIscsi.daemon_type: + f.write(' '.join(CephIscsi.configfs_mount_umount(data_dir, mount=True)) + '\n') + ceph_iscsi = CephIscsi.init(ctx, fsid, daemon_id) + tcmu_container = ceph_iscsi.get_tcmu_runner_container() + _write_container_cmd_to_bash(ctx, f, tcmu_container, 'iscsi tcmu-runner container', background=True) + + _write_container_cmd_to_bash(ctx, f, c, '%s.%s' % (daemon_type, str(daemon_id))) + + # some metadata about the deploy + meta: Dict[str, Any] = {} + if 'meta_json' in ctx and ctx.meta_json: + meta = json.loads(ctx.meta_json) or {} + meta.update({ + 'memory_request': int(ctx.memory_request) if ctx.memory_request else None, + 'memory_limit': int(ctx.memory_limit) if ctx.memory_limit else None, + }) + if not meta.get('ports'): + meta['ports'] = ports + metaf.write(json.dumps(meta, indent=4) + '\n') + + os.fchmod(f.fileno(), 0o600) + os.fchmod(metaf.fileno(), 0o600) + os.rename(data_dir + '/unit.run.new', + data_dir + '/unit.run') + os.rename(data_dir + '/unit.meta.new', + data_dir + '/unit.meta') + + # post-stop command(s) + with open(data_dir + '/unit.poststop.new', 'w') as f: + if daemon_type == 'osd': + assert osd_fsid + poststop = get_ceph_volume_container( + ctx, + args=[ + 'lvm', 'deactivate', + str(daemon_id), osd_fsid, + ], + volume_mounts=get_container_mounts(ctx, fsid, daemon_type, daemon_id), + bind_mounts=get_container_binds(ctx, fsid, daemon_type, daemon_id), + cname='ceph-%s-%s.%s-deactivate' % (fsid, daemon_type, + daemon_id), + ) + _write_container_cmd_to_bash(ctx, f, poststop, 'deactivate osd') + elif daemon_type == CephIscsi.daemon_type: + # make sure we also stop the tcmu container + ceph_iscsi = CephIscsi.init(ctx, fsid, daemon_id) + tcmu_container = ceph_iscsi.get_tcmu_runner_container() + f.write('! ' + ' '.join(tcmu_container.stop_cmd()) + '\n') + f.write(' '.join(CephIscsi.configfs_mount_umount(data_dir, mount=False)) + '\n') + os.fchmod(f.fileno(), 0o600) + os.rename(data_dir + '/unit.poststop.new', + data_dir + '/unit.poststop') + + # post-stop command(s) + with open(data_dir + '/unit.stop.new', 'w') as f: + f.write('! ' + ' '.join(c.stop_cmd()) + '\n') + f.write('! ' + ' '.join(c.stop_cmd(old_cname=True)) + '\n') + + os.fchmod(f.fileno(), 0o600) + os.rename(data_dir + '/unit.stop.new', + data_dir + '/unit.stop') + + if c: + with open(data_dir + '/unit.image.new', 'w') as f: + f.write(c.image + '\n') + os.fchmod(f.fileno(), 0o600) + os.rename(data_dir + '/unit.image.new', + data_dir + '/unit.image') + + # sysctl + install_sysctl(ctx, fsid, daemon_type) + + # systemd + install_base_units(ctx, fsid) + unit = get_unit_file(ctx, fsid) + unit_file = 'ceph-%s@.service' % (fsid) + with open(ctx.unit_dir + '/' + unit_file + '.new', 'w') as f: + f.write(unit) + os.rename(ctx.unit_dir + '/' + unit_file + '.new', + ctx.unit_dir + '/' + unit_file) + call_throws(ctx, ['systemctl', 'daemon-reload']) + + unit_name = get_unit_name(fsid, daemon_type, daemon_id) + call(ctx, ['systemctl', 'stop', unit_name], + verbosity=CallVerbosity.DEBUG) + call(ctx, ['systemctl', 'reset-failed', unit_name], + verbosity=CallVerbosity.DEBUG) + if enable: + call_throws(ctx, ['systemctl', 'enable', unit_name]) + if start: + clean_cgroup(ctx, fsid, unit_name) + call_throws(ctx, ['systemctl', 'start', unit_name]) + + +class Firewalld(object): + def __init__(self, ctx): + # type: (CephadmContext) -> None + self.ctx = ctx + self.available = self.check() + + def check(self): + # type: () -> bool + self.cmd = find_executable('firewall-cmd') + if not self.cmd: + logger.debug('firewalld does not appear to be present') + return False + (enabled, state, _) = check_unit(self.ctx, 'firewalld.service') + if not enabled: + logger.debug('firewalld.service is not enabled') + return False + if state != 'running': + logger.debug('firewalld.service is not running') + return False + + logger.info('firewalld ready') + return True + + def enable_service_for(self, daemon_type): + # type: (str) -> None + if not self.available: + logger.debug('Not possible to enable service <%s>. firewalld.service is not available' % daemon_type) + return + + if daemon_type == 'mon': + svc = 'ceph-mon' + elif daemon_type in ['mgr', 'mds', 'osd']: + svc = 'ceph' + elif daemon_type == NFSGanesha.daemon_type: + svc = 'nfs' + else: + return + + if not self.cmd: + raise RuntimeError('command not defined') + + out, err, ret = call(self.ctx, [self.cmd, '--permanent', '--query-service', svc], verbosity=CallVerbosity.DEBUG) + if ret: + logger.info('Enabling firewalld service %s in current zone...' % svc) + out, err, ret = call(self.ctx, [self.cmd, '--permanent', '--add-service', svc]) + if ret: + raise RuntimeError( + 'unable to add service %s to current zone: %s' % (svc, err)) + else: + logger.debug('firewalld service %s is enabled in current zone' % svc) + + def open_ports(self, fw_ports): + # type: (List[int]) -> None + if not self.available: + logger.debug('Not possible to open ports <%s>. firewalld.service is not available' % fw_ports) + return + + if not self.cmd: + raise RuntimeError('command not defined') + + for port in fw_ports: + tcp_port = str(port) + '/tcp' + out, err, ret = call(self.ctx, [self.cmd, '--permanent', '--query-port', tcp_port], verbosity=CallVerbosity.DEBUG) + if ret: + logger.info('Enabling firewalld port %s in current zone...' % tcp_port) + out, err, ret = call(self.ctx, [self.cmd, '--permanent', '--add-port', tcp_port]) + if ret: + raise RuntimeError('unable to add port %s to current zone: %s' % + (tcp_port, err)) + else: + logger.debug('firewalld port %s is enabled in current zone' % tcp_port) + + def close_ports(self, fw_ports): + # type: (List[int]) -> None + if not self.available: + logger.debug('Not possible to close ports <%s>. firewalld.service is not available' % fw_ports) + return + + if not self.cmd: + raise RuntimeError('command not defined') + + for port in fw_ports: + tcp_port = str(port) + '/tcp' + out, err, ret = call(self.ctx, [self.cmd, '--permanent', '--query-port', tcp_port], verbosity=CallVerbosity.DEBUG) + if not ret: + logger.info('Disabling port %s in current zone...' % tcp_port) + out, err, ret = call(self.ctx, [self.cmd, '--permanent', '--remove-port', tcp_port]) + if ret: + raise RuntimeError('unable to remove port %s from current zone: %s' % + (tcp_port, err)) + else: + logger.info(f'Port {tcp_port} disabled') + else: + logger.info(f'firewalld port {tcp_port} already closed') + + def apply_rules(self): + # type: () -> None + if not self.available: + return + + if not self.cmd: + raise RuntimeError('command not defined') + + call_throws(self.ctx, [self.cmd, '--reload']) + + +def update_firewalld(ctx, daemon_type): + # type: (CephadmContext, str) -> None + if not ('skip_firewalld' in ctx and ctx.skip_firewalld): + firewall = Firewalld(ctx) + firewall.enable_service_for(daemon_type) + firewall.apply_rules() + + +def install_sysctl(ctx: CephadmContext, fsid: str, daemon_type: str) -> None: + """ + Set up sysctl settings + """ + def _write(conf: Path, lines: List[str]) -> None: + lines = [ + '# created by cephadm', + '', + *lines, + '', + ] + with open(conf, 'w') as f: + f.write('\n'.join(lines)) + + conf = Path(ctx.sysctl_dir).joinpath(f'90-ceph-{fsid}-{daemon_type}.conf') + lines: Optional[List] = None + + if daemon_type == 'osd': + lines = OSD.get_sysctl_settings() + elif daemon_type == 'haproxy': + lines = HAproxy.get_sysctl_settings() + elif daemon_type == 'keepalived': + lines = Keepalived.get_sysctl_settings() + + # apply the sysctl settings + if lines: + Path(ctx.sysctl_dir).mkdir(mode=0o755, exist_ok=True) + _write(conf, lines) + call_throws(ctx, ['sysctl', '--system']) + + +def migrate_sysctl_dir(ctx: CephadmContext, fsid: str) -> None: + """ + Cephadm once used '/usr/lib/sysctl.d' for storing sysctl configuration. + This moves it to '/etc/sysctl.d'. + """ + deprecated_location: str = '/usr/lib/sysctl.d' + deprecated_confs: List[str] = glob(f'{deprecated_location}/90-ceph-{fsid}-*.conf') + if not deprecated_confs: + return + + file_count: int = len(deprecated_confs) + logger.info(f'Found sysctl {file_count} files in deprecated location {deprecated_location}. Starting Migration.') + for conf in deprecated_confs: + try: + shutil.move(conf, ctx.sysctl_dir) + file_count -= 1 + except shutil.Error as err: + if str(err).endswith('already exists'): + logger.warning(f'Destination file already exists. Deleting {conf}.') + try: + os.unlink(conf) + file_count -= 1 + except OSError as del_err: + logger.warning(f'Could not remove {conf}: {del_err}.') + else: + logger.warning(f'Could not move {conf} from {deprecated_location} to {ctx.sysctl_dir}: {err}') + + # Log successful migration + if file_count == 0: + logger.info(f'Successfully migrated sysctl config to {ctx.sysctl_dir}.') + return + + # Log partially successful / unsuccessful migration + files_processed: int = len(deprecated_confs) + if file_count < files_processed: + status: str = f'partially successful (failed {file_count}/{files_processed})' + elif file_count == files_processed: + status = 'unsuccessful' + logger.warning(f'Migration of sysctl configuration {status}. You may want to perform a migration manually.') + + +def install_base_units(ctx, fsid): + # type: (CephadmContext, str) -> None + """ + Set up ceph.target and ceph-$fsid.target units. + """ + # global unit + existed = os.path.exists(ctx.unit_dir + '/ceph.target') + with open(ctx.unit_dir + '/ceph.target.new', 'w') as f: + f.write('[Unit]\n' + 'Description=All Ceph clusters and services\n' + '\n' + '[Install]\n' + 'WantedBy=multi-user.target\n') + os.rename(ctx.unit_dir + '/ceph.target.new', + ctx.unit_dir + '/ceph.target') + if not existed: + # we disable before enable in case a different ceph.target + # (from the traditional package) is present; while newer + # systemd is smart enough to disable the old + # (/lib/systemd/...) and enable the new (/etc/systemd/...), + # some older versions of systemd error out with EEXIST. + call_throws(ctx, ['systemctl', 'disable', 'ceph.target']) + call_throws(ctx, ['systemctl', 'enable', 'ceph.target']) + call_throws(ctx, ['systemctl', 'start', 'ceph.target']) + + # cluster unit + existed = os.path.exists(ctx.unit_dir + '/ceph-%s.target' % fsid) + with open(ctx.unit_dir + '/ceph-%s.target.new' % fsid, 'w') as f: + f.write( + '[Unit]\n' + 'Description=Ceph cluster {fsid}\n' + 'PartOf=ceph.target\n' + 'Before=ceph.target\n' + '\n' + '[Install]\n' + 'WantedBy=multi-user.target ceph.target\n'.format( + fsid=fsid) + ) + os.rename(ctx.unit_dir + '/ceph-%s.target.new' % fsid, + ctx.unit_dir + '/ceph-%s.target' % fsid) + if not existed: + call_throws(ctx, ['systemctl', 'enable', 'ceph-%s.target' % fsid]) + call_throws(ctx, ['systemctl', 'start', 'ceph-%s.target' % fsid]) + + # logrotate for the cluster + with open(ctx.logrotate_dir + '/ceph-%s' % fsid, 'w') as f: + """ + This is a bit sloppy in that the killall/pkill will touch all ceph daemons + in all containers, but I don't see an elegant way to send SIGHUP *just* to + the daemons for this cluster. (1) systemd kill -s will get the signal to + podman, but podman will exit. (2) podman kill will get the signal to the + first child (bash), but that isn't the ceph daemon. This is simpler and + should be harmless. + """ + f.write("""# created by cephadm +/var/log/ceph/%s/*.log { + rotate 7 + daily + compress + sharedscripts + postrotate + killall -q -1 ceph-mon ceph-mgr ceph-mds ceph-osd ceph-fuse radosgw rbd-mirror cephfs-mirror || pkill -1 -x 'ceph-mon|ceph-mgr|ceph-mds|ceph-osd|ceph-fuse|radosgw|rbd-mirror|cephfs-mirror' || true + endscript + missingok + notifempty + su root root +} +""" % fsid) + + +def get_unit_file(ctx, fsid): + # type: (CephadmContext, str) -> str + extra_args = '' + if isinstance(ctx.container_engine, Podman): + extra_args = ('ExecStartPre=-/bin/rm -f %t/%n-pid %t/%n-cid\n' + 'ExecStopPost=-/bin/rm -f %t/%n-pid %t/%n-cid\n' + 'Type=forking\n' + 'PIDFile=%t/%n-pid\n') + if ctx.container_engine.version >= CGROUPS_SPLIT_PODMAN_VERSION: + extra_args += 'Delegate=yes\n' + + docker = isinstance(ctx.container_engine, Docker) + u = """# generated by cephadm +[Unit] +Description=Ceph %i for {fsid} + +# According to: +# http://www.freedesktop.org/wiki/Software/systemd/NetworkTarget +# these can be removed once ceph-mon will dynamically change network +# configuration. +After=network-online.target local-fs.target time-sync.target{docker_after} +Wants=network-online.target local-fs.target time-sync.target +{docker_requires} + +PartOf=ceph-{fsid}.target +Before=ceph-{fsid}.target + +[Service] +LimitNOFILE=1048576 +LimitNPROC=1048576 +EnvironmentFile=-/etc/environment +ExecStart=/bin/bash {data_dir}/{fsid}/%i/unit.run +ExecStop=-/bin/bash -c '{container_path} stop ceph-{fsid}-%i ; bash {data_dir}/{fsid}/%i/unit.stop' +ExecStopPost=-/bin/bash {data_dir}/{fsid}/%i/unit.poststop +KillMode=none +Restart=on-failure +RestartSec=10s +TimeoutStartSec=120 +TimeoutStopSec=120 +StartLimitInterval=30min +StartLimitBurst=5 +{extra_args} +[Install] +WantedBy=ceph-{fsid}.target +""".format(container_path=ctx.container_engine.path, + fsid=fsid, + data_dir=ctx.data_dir, + extra_args=extra_args, + # if docker, we depend on docker.service + docker_after=' docker.service' if docker else '', + docker_requires='Requires=docker.service\n' if docker else '') + + return u + +################################## + + +class CephContainer: + def __init__(self, + ctx: CephadmContext, + image: str, + entrypoint: str, + args: List[str] = [], + volume_mounts: Dict[str, str] = {}, + cname: str = '', + container_args: List[str] = [], + envs: Optional[List[str]] = None, + privileged: bool = False, + ptrace: bool = False, + bind_mounts: Optional[List[List[str]]] = None, + init: Optional[bool] = None, + host_network: bool = True, + memory_request: Optional[str] = None, + memory_limit: Optional[str] = None, + ) -> None: + self.ctx = ctx + self.image = image + self.entrypoint = entrypoint + self.args = args + self.volume_mounts = volume_mounts + self._cname = cname + self.container_args = container_args + self.envs = envs + self.privileged = privileged + self.ptrace = ptrace + self.bind_mounts = bind_mounts if bind_mounts else [] + self.init = init if init else ctx.container_init + self.host_network = host_network + self.memory_request = memory_request + self.memory_limit = memory_limit + + @classmethod + def for_daemon(cls, + ctx: CephadmContext, + fsid: str, + daemon_type: str, + daemon_id: str, + entrypoint: str, + args: List[str] = [], + volume_mounts: Dict[str, str] = {}, + container_args: List[str] = [], + envs: Optional[List[str]] = None, + privileged: bool = False, + ptrace: bool = False, + bind_mounts: Optional[List[List[str]]] = None, + init: Optional[bool] = None, + host_network: bool = True, + memory_request: Optional[str] = None, + memory_limit: Optional[str] = None, + ) -> 'CephContainer': + return cls( + ctx, + image=ctx.image, + entrypoint=entrypoint, + args=args, + volume_mounts=volume_mounts, + cname='ceph-%s-%s.%s' % (fsid, daemon_type, daemon_id), + container_args=container_args, + envs=envs, + privileged=privileged, + ptrace=ptrace, + bind_mounts=bind_mounts, + init=init, + host_network=host_network, + memory_request=memory_request, + memory_limit=memory_limit, + ) + + @property + def cname(self) -> str: + """ + podman adds the current container name to the /etc/hosts + file. Turns out, python's `socket.getfqdn()` differs from + `hostname -f`, when we have the container names containing + dots in it.: + + # podman run --name foo.bar.baz.com ceph/ceph /bin/bash + [root@sebastians-laptop /]# cat /etc/hosts + 127.0.0.1 localhost + ::1 localhost + 127.0.1.1 sebastians-laptop foo.bar.baz.com + [root@sebastians-laptop /]# hostname -f + sebastians-laptop + [root@sebastians-laptop /]# python3 -c 'import socket; print(socket.getfqdn())' + foo.bar.baz.com + + Fascinatingly, this doesn't happen when using dashes. + """ + return self._cname.replace('.', '-') + + @cname.setter + def cname(self, val: str) -> None: + self._cname = val + + @property + def old_cname(self) -> str: + return self._cname + + def run_cmd(self) -> List[str]: + cmd_args: List[str] = [ + str(self.ctx.container_engine.path), + 'run', + '--rm', + '--ipc=host', + # some containers (ahem, haproxy) override this, but we want a fast + # shutdown always (and, more importantly, a successful exit even if we + # fall back to SIGKILL). + '--stop-signal=SIGTERM', + ] + + if isinstance(self.ctx.container_engine, Podman): + if os.path.exists('/etc/ceph/podman-auth.json'): + cmd_args.append('--authfile=/etc/ceph/podman-auth.json') + + envs: List[str] = [ + '-e', 'CONTAINER_IMAGE=%s' % self.image, + '-e', 'NODE_NAME=%s' % get_hostname(), + ] + vols: List[str] = [] + binds: List[str] = [] + + if self.memory_request: + cmd_args.extend(['-e', 'POD_MEMORY_REQUEST', str(self.memory_request)]) + if self.memory_limit: + cmd_args.extend(['-e', 'POD_MEMORY_LIMIT', str(self.memory_limit)]) + cmd_args.extend(['--memory', str(self.memory_limit)]) + + if self.host_network: + cmd_args.append('--net=host') + if self.entrypoint: + cmd_args.extend(['--entrypoint', self.entrypoint]) + if self.privileged: + cmd_args.extend([ + '--privileged', + # let OSD etc read block devs that haven't been chowned + '--group-add=disk']) + if self.ptrace and not self.privileged: + # if privileged, the SYS_PTRACE cap is already added + # in addition, --cap-add and --privileged are mutually + # exclusive since podman >= 2.0 + cmd_args.append('--cap-add=SYS_PTRACE') + if self.init: + cmd_args.append('--init') + envs += ['-e', 'CEPH_USE_RANDOM_NONCE=1'] + if self.cname: + cmd_args.extend(['--name', self.cname]) + if self.envs: + for env in self.envs: + envs.extend(['-e', env]) + + vols = sum( + [['-v', '%s:%s' % (host_dir, container_dir)] + for host_dir, container_dir in self.volume_mounts.items()], []) + binds = sum([['--mount', '{}'.format(','.join(bind))] + for bind in self.bind_mounts], []) + + return \ + cmd_args + self.container_args + \ + envs + vols + binds + \ + [self.image] + self.args # type: ignore + + def shell_cmd(self, cmd: List[str]) -> List[str]: + cmd_args: List[str] = [ + str(self.ctx.container_engine.path), + 'run', + '--rm', + '--ipc=host', + ] + envs: List[str] = [ + '-e', 'CONTAINER_IMAGE=%s' % self.image, + '-e', 'NODE_NAME=%s' % get_hostname(), + ] + vols: List[str] = [] + binds: List[str] = [] + + if self.host_network: + cmd_args.append('--net=host') + if self.ctx.no_hosts: + cmd_args.append('--no-hosts') + if self.privileged: + cmd_args.extend([ + '--privileged', + # let OSD etc read block devs that haven't been chowned + '--group-add=disk', + ]) + if self.init: + cmd_args.append('--init') + envs += ['-e', 'CEPH_USE_RANDOM_NONCE=1'] + if self.envs: + for env in self.envs: + envs.extend(['-e', env]) + + vols = sum( + [['-v', '%s:%s' % (host_dir, container_dir)] + for host_dir, container_dir in self.volume_mounts.items()], []) + binds = sum([['--mount', '{}'.format(','.join(bind))] + for bind in self.bind_mounts], []) + + return cmd_args + self.container_args + envs + vols + binds + [ + '--entrypoint', cmd[0], + self.image, + ] + cmd[1:] + + def exec_cmd(self, cmd): + # type: (List[str]) -> List[str] + cname = get_running_container_name(self.ctx, self) + if not cname: + raise Error('unable to find container "{}"'.format(self.cname)) + return [ + str(self.ctx.container_engine.path), + 'exec', + ] + self.container_args + [ + self.cname, + ] + cmd + + def rm_cmd(self, old_cname: bool = False, storage: bool = False) -> List[str]: + ret = [ + str(self.ctx.container_engine.path), + 'rm', '-f', + ] + if storage: + ret.append('--storage') + if old_cname: + ret.append(self.old_cname) + else: + ret.append(self.cname) + return ret + + def stop_cmd(self, old_cname: bool = False) -> List[str]: + ret = [ + str(self.ctx.container_engine.path), + 'stop', self.old_cname if old_cname else self.cname, + ] + return ret + + def run(self, timeout=DEFAULT_TIMEOUT): + # type: (Optional[int]) -> str + out, _, _ = call_throws(self.ctx, self.run_cmd(), + desc=self.entrypoint, timeout=timeout) + return out + +################################## + + +@infer_image +def command_version(ctx): + # type: (CephadmContext) -> int + c = CephContainer(ctx, ctx.image, 'ceph', ['--version']) + out, err, ret = call(ctx, c.run_cmd(), desc=c.entrypoint) + if not ret: + print(out.strip()) + return ret + +################################## + + +@default_image +def command_pull(ctx): + # type: (CephadmContext) -> int + + try: + _pull_image(ctx, ctx.image, ctx.insecure) + except UnauthorizedRegistryError: + err_str = 'Failed to pull container image. Check that host(s) are logged into the registry' + logger.debug(f'Pulling image for `command_pull` failed: {err_str}') + raise Error(err_str) + return command_inspect_image(ctx) + + +def _pull_image(ctx, image, insecure=False): + # type: (CephadmContext, str, bool) -> None + logger.info('Pulling container image %s...' % image) + + ignorelist = [ + 'error creating read-write layer with ID', + 'net/http: TLS handshake timeout', + 'Digest did not match, expected', + ] + + cmd = [ctx.container_engine.path, 'pull', image] + if isinstance(ctx.container_engine, Podman): + if insecure: + cmd.append('--tls-verify=false') + + if os.path.exists('/etc/ceph/podman-auth.json'): + cmd.append('--authfile=/etc/ceph/podman-auth.json') + cmd_str = ' '.join(cmd) + + for sleep_secs in [1, 4, 25]: + out, err, ret = call(ctx, cmd) + if not ret: + return + + if 'unauthorized' in err: + raise UnauthorizedRegistryError() + + if not any(pattern in err for pattern in ignorelist): + raise Error('Failed command: %s' % cmd_str) + + logger.info('`%s` failed transiently. Retrying. waiting %s seconds...' % (cmd_str, sleep_secs)) + time.sleep(sleep_secs) + + raise Error('Failed command: %s: maximum retries reached' % cmd_str) + +################################## + + +@infer_image +def command_inspect_image(ctx): + # type: (CephadmContext) -> int + out, err, ret = call_throws(ctx, [ + ctx.container_engine.path, 'inspect', + '--format', '{{.ID}},{{.RepoDigests}}', + ctx.image]) + if ret: + return errno.ENOENT + info_from = get_image_info_from_inspect(out.strip(), ctx.image) + + ver = CephContainer(ctx, ctx.image, 'ceph', ['--version']).run().strip() + info_from['ceph_version'] = ver + + print(json.dumps(info_from, indent=4, sort_keys=True)) + return 0 + + +def normalize_image_digest(digest: str) -> str: + """ + Normal case: + >>> normalize_image_digest('ceph/ceph', 'docker.io') + 'docker.io/ceph/ceph' + + No change: + >>> normalize_image_digest('quay.ceph.io/ceph/ceph', 'docker.io') + 'quay.ceph.io/ceph/ceph' + + >>> normalize_image_digest('docker.io/ubuntu', 'docker.io') + 'docker.io/ubuntu' + + >>> normalize_image_digest('localhost/ceph', 'docker.io') + 'localhost/ceph' + """ + known_shortnames = [ + 'ceph/ceph', + 'ceph/daemon', + 'ceph/daemon-base', + ] + for image in known_shortnames: + if digest.startswith(image): + return f'{DEFAULT_REGISTRY}/{digest}' + return digest + + +def get_image_info_from_inspect(out, image): + # type: (str, str) -> Dict[str, Union[str,List[str]]] + image_id, digests = out.split(',', 1) + if not out: + raise Error('inspect {}: empty result'.format(image)) + r = { + 'image_id': normalize_container_id(image_id) + } # type: Dict[str, Union[str,List[str]]] + if digests: + r['repo_digests'] = list(map(normalize_image_digest, digests[1:-1].split(' '))) + return r + +################################## + + +def check_subnet(subnets: str) -> Tuple[int, List[int], str]: + """Determine whether the given string is a valid subnet + + :param subnets: subnet string, a single definition or comma separated list of CIDR subnets + :returns: return code, IP version list of the subnets and msg describing any errors validation errors + """ + + rc = 0 + versions = set() + errors = [] + subnet_list = subnets.split(',') + for subnet in subnet_list: + # ensure the format of the string is as expected address/netmask + subnet = subnet.strip() + if not re.search(r'\/\d+$', subnet): + rc = 1 + errors.append(f'{subnet} is not in CIDR format (address/netmask)') + continue + try: + v = ipaddress.ip_network(subnet).version + versions.add(v) + except ValueError as e: + rc = 1 + errors.append(f'{subnet} invalid: {str(e)}') + + return rc, list(versions), ', '.join(errors) + + +def unwrap_ipv6(address): + # type: (str) -> str + if address.startswith('[') and address.endswith(']'): + return address[1:-1] + return address + + +def wrap_ipv6(address): + # type: (str) -> str + + # We cannot assume it's already wrapped or even an IPv6 address if + # it's already wrapped it'll not pass (like if it's a hostname) and trigger + # the ValueError + try: + if ipaddress.ip_address(address).version == 6: + return f'[{address}]' + except ValueError: + pass + + return address + + +def is_ipv6(address): + # type: (str) -> bool + address = unwrap_ipv6(address) + try: + return ipaddress.ip_address(address).version == 6 + except ValueError: + logger.warning('Address: {} is not a valid IP address'.format(address)) + return False + + +def ip_in_subnets(ip_addr: str, subnets: str) -> bool: + """Determine if the ip_addr belongs to any of the subnets list.""" + subnet_list = [x.strip() for x in subnets.split(',')] + for subnet in subnet_list: + ip_address = unwrap_ipv6(ip_addr) if is_ipv6(ip_addr) else ip_addr + if ipaddress.ip_address(ip_address) in ipaddress.ip_network(subnet): + return True + return False + + +def parse_mon_addrv(addrv_arg: str) -> List[EndPoint]: + """Parse mon-addrv param into a list of mon end points.""" + r = re.compile(r':(\d+)$') + addrv_args = [] + addr_arg = addrv_arg + if addr_arg[0] != '[' or addr_arg[-1] != ']': + raise Error(f'--mon-addrv value {addr_arg} must use square backets') + + for addr in addr_arg[1: -1].split(','): + hasport = r.findall(addr) + if not hasport: + raise Error(f'--mon-addrv value {addr_arg} must include port number') + port_str = hasport[0] + addr = re.sub(r'^v\d+:', '', addr) # strip off v1: or v2: prefix + base_ip = addr[0:-(len(port_str)) - 1] + addrv_args.append(EndPoint(base_ip, int(port_str))) + + return addrv_args + + +def parse_mon_ip(mon_ip: str) -> List[EndPoint]: + """Parse mon-ip param into a list of mon end points.""" + r = re.compile(r':(\d+)$') + addrv_args = [] + hasport = r.findall(mon_ip) + if hasport: + port_str = hasport[0] + base_ip = mon_ip[0:-(len(port_str)) - 1] + addrv_args.append(EndPoint(base_ip, int(port_str))) + else: + # No port provided: use fixed ports for ceph monitor + addrv_args.append(EndPoint(mon_ip, 3300)) + addrv_args.append(EndPoint(mon_ip, 6789)) + + return addrv_args + + +def build_addrv_params(addrv: List[EndPoint]) -> str: + """Convert mon end-points (ip:port) into the format: [v[1|2]:ip:port1]""" + if len(addrv) > 2: + raise Error('Detected a local mon-addrv list with more than 2 entries.') + port_to_ver: Dict[int, str] = {6789: 'v1', 3300: 'v2'} + addr_arg_list: List[str] = [] + for ep in addrv: + if ep.port in port_to_ver: + ver = port_to_ver[ep.port] + else: + ver = 'v2' # default mon protocol version if port is not provided + logger.warning(f'Using msgr2 protocol for unrecognized port {ep}') + addr_arg_list.append(f'{ver}:{ep.ip}:{ep.port}') + + addr_arg = '[{0}]'.format(','.join(addr_arg_list)) + return addr_arg + + +def get_public_net_from_cfg(ctx: CephadmContext) -> Optional[str]: + """Get mon public network from configuration file.""" + cp = read_config(ctx.config) + if not cp.has_option('global', 'public_network'): + return None + + # Ensure all public CIDR networks are valid + public_network = cp.get('global', 'public_network').strip('"').strip("'") + rc, _, err_msg = check_subnet(public_network) + if rc: + raise Error(f'Invalid public_network {public_network} parameter: {err_msg}') + + # Ensure all public CIDR networks are configured locally + configured_subnets = set([x.strip() for x in public_network.split(',')]) + local_subnets = set([x[0] for x in list_networks(ctx).items()]) + valid_public_net = False + for net in configured_subnets: + if net in local_subnets: + valid_public_net = True + else: + logger.warning(f'The public CIDR network {net} (from -c conf file) is not configured locally.') + if not valid_public_net: + raise Error(f'None of the public CIDR network(s) {configured_subnets} (from -c conf file) is configured locally.') + + # Ensure public_network is compatible with the provided mon-ip (or mon-addrv) + if ctx.mon_ip: + if not ip_in_subnets(ctx.mon_ip, public_network): + raise Error(f'The provided --mon-ip {ctx.mon_ip} does not belong to any public_network(s) {public_network}') + elif ctx.mon_addrv: + addrv_args = parse_mon_addrv(ctx.mon_addrv) + for addrv in addrv_args: + if not ip_in_subnets(addrv.ip, public_network): + raise Error(f'The provided --mon-addrv {addrv.ip} ip does not belong to any public_network(s) {public_network}') + + logger.debug(f'Using mon public network from configuration file {public_network}') + return public_network + + +def infer_mon_network(ctx: CephadmContext, mon_eps: List[EndPoint]) -> Optional[str]: + """Infer mon public network from local network.""" + # Make sure IP is configured locally, and then figure out the CIDR network + mon_networks = [] + for net, ifaces in list_networks(ctx).items(): + # build local_ips list for the specified network + local_ips: List[str] = [] + for _, ls in ifaces.items(): + local_ips.extend([ipaddress.ip_address(ip) for ip in ls]) + + # check if any of mon ips belong to this net + for mon_ep in mon_eps: + try: + if ipaddress.ip_address(unwrap_ipv6(mon_ep.ip)) in local_ips: + mon_networks.append(net) + logger.info(f'Mon IP `{mon_ep.ip}` is in CIDR network `{net}`') + except ValueError as e: + logger.warning(f'Cannot infer CIDR network for mon IP `{mon_ep.ip}` : {e}') + + if not mon_networks: + raise Error('Cannot infer CIDR network. Pass --skip-mon-network to configure it later') + else: + logger.debug(f'Inferred mon public CIDR from local network configuration {mon_networks}') + + mon_networks = list(set(mon_networks)) # remove duplicates + return ','.join(mon_networks) + + +def prepare_mon_addresses(ctx: CephadmContext) -> Tuple[str, bool, Optional[str]]: + """Get mon public network configuration.""" + ipv6 = False + addrv_args: List[EndPoint] = [] + mon_addrv: str = '' # i.e: [v2:192.168.100.1:3300,v1:192.168.100.1:6789] + + if ctx.mon_ip: + ipv6 = is_ipv6(ctx.mon_ip) + if ipv6: + ctx.mon_ip = wrap_ipv6(ctx.mon_ip) + addrv_args = parse_mon_ip(ctx.mon_ip) + mon_addrv = build_addrv_params(addrv_args) + elif ctx.mon_addrv: + ipv6 = ctx.mon_addrv.count('[') > 1 + addrv_args = parse_mon_addrv(ctx.mon_addrv) + mon_addrv = ctx.mon_addrv + else: + raise Error('must specify --mon-ip or --mon-addrv') + + if addrv_args: + for end_point in addrv_args: + check_ip_port(ctx, end_point) + + logger.debug(f'Base mon IP(s) is {addrv_args}, mon addrv is {mon_addrv}') + mon_network = None + if not ctx.skip_mon_network: + mon_network = get_public_net_from_cfg(ctx) or infer_mon_network(ctx, addrv_args) + + return (mon_addrv, ipv6, mon_network) + + +def prepare_cluster_network(ctx: CephadmContext) -> Tuple[str, bool]: + # the cluster network may not exist on this node, so all we can do is + # validate that the address given is valid ipv4 or ipv6 subnet + ipv6_cluster_network = False + cp = read_config(ctx.config) + cluster_network = ctx.cluster_network + if cluster_network is None and cp.has_option('global', 'cluster_network'): + cluster_network = cp.get('global', 'cluster_network').strip('"').strip("'") + + if cluster_network: + cluser_nets = set([x.strip() for x in cluster_network.split(',')]) + local_subnets = set([x[0] for x in list_networks(ctx).items()]) + for net in cluser_nets: + if net not in local_subnets: + logger.warning(f'The cluster CIDR network {net} is not configured locally.') + + rc, versions, err_msg = check_subnet(cluster_network) + if rc: + raise Error(f'Invalid --cluster-network parameter: {err_msg}') + ipv6_cluster_network = True if 6 in versions else False + else: + logger.info('Internal network (--cluster-network) has not ' + 'been provided, OSD replication will default to ' + 'the public_network') + + return cluster_network, ipv6_cluster_network + + +def create_initial_keys( + ctx: CephadmContext, + uid: int, gid: int, + mgr_id: str +) -> Tuple[str, str, str, Any, Any]: # type: ignore + + _image = ctx.image + + # create some initial keys + logger.info('Creating initial keys...') + mon_key = CephContainer( + ctx, + image=_image, + entrypoint='/usr/bin/ceph-authtool', + args=['--gen-print-key'], + ).run().strip() + admin_key = CephContainer( + ctx, + image=_image, + entrypoint='/usr/bin/ceph-authtool', + args=['--gen-print-key'], + ).run().strip() + mgr_key = CephContainer( + ctx, + image=_image, + entrypoint='/usr/bin/ceph-authtool', + args=['--gen-print-key'], + ).run().strip() + + keyring = ('[mon.]\n' + '\tkey = %s\n' + '\tcaps mon = allow *\n' + '[client.admin]\n' + '\tkey = %s\n' + '\tcaps mon = allow *\n' + '\tcaps mds = allow *\n' + '\tcaps mgr = allow *\n' + '\tcaps osd = allow *\n' + '[mgr.%s]\n' + '\tkey = %s\n' + '\tcaps mon = profile mgr\n' + '\tcaps mds = allow *\n' + '\tcaps osd = allow *\n' + % (mon_key, admin_key, mgr_id, mgr_key)) + + admin_keyring = write_tmp('[client.admin]\n' + '\tkey = ' + admin_key + '\n', + uid, gid) + + # tmp keyring file + bootstrap_keyring = write_tmp(keyring, uid, gid) + return (mon_key, mgr_key, admin_key, + bootstrap_keyring, admin_keyring) + + +def create_initial_monmap( + ctx: CephadmContext, + uid: int, gid: int, + fsid: str, + mon_id: str, mon_addr: str +) -> Any: + logger.info('Creating initial monmap...') + monmap = write_tmp('', 0, 0) + out = CephContainer( + ctx, + image=ctx.image, + entrypoint='/usr/bin/monmaptool', + args=[ + '--create', + '--clobber', + '--fsid', fsid, + '--addv', mon_id, mon_addr, + '/tmp/monmap' + ], + volume_mounts={ + monmap.name: '/tmp/monmap:z', + }, + ).run() + logger.debug(f'monmaptool for {mon_id} {mon_addr} on {out}') + + # pass monmap file to ceph user for use by ceph-mon --mkfs below + os.fchown(monmap.fileno(), uid, gid) + return monmap + + +def prepare_create_mon( + ctx: CephadmContext, + uid: int, gid: int, + fsid: str, mon_id: str, + bootstrap_keyring_path: str, + monmap_path: str +) -> Tuple[str, str]: + logger.info('Creating mon...') + create_daemon_dirs(ctx, fsid, 'mon', mon_id, uid, gid) + mon_dir = get_data_dir(fsid, ctx.data_dir, 'mon', mon_id) + log_dir = get_log_dir(fsid, ctx.log_dir) + out = CephContainer( + ctx, + image=ctx.image, + entrypoint='/usr/bin/ceph-mon', + args=[ + '--mkfs', + '-i', mon_id, + '--fsid', fsid, + '-c', '/dev/null', + '--monmap', '/tmp/monmap', + '--keyring', '/tmp/keyring', + ] + get_daemon_args(ctx, fsid, 'mon', mon_id), + volume_mounts={ + log_dir: '/var/log/ceph:z', + mon_dir: '/var/lib/ceph/mon/ceph-%s:z' % (mon_id), + bootstrap_keyring_path: '/tmp/keyring:z', + monmap_path: '/tmp/monmap:z', + }, + ).run() + logger.debug(f'create mon.{mon_id} on {out}') + return (mon_dir, log_dir) + + +def create_mon( + ctx: CephadmContext, + uid: int, gid: int, + fsid: str, mon_id: str +) -> None: + mon_c = get_container(ctx, fsid, 'mon', mon_id) + ctx.meta_json = json.dumps({'service_name': 'mon'}) + deploy_daemon(ctx, fsid, 'mon', mon_id, mon_c, uid, gid, + config=None, keyring=None) + + +def wait_for_mon( + ctx: CephadmContext, + mon_id: str, mon_dir: str, + admin_keyring_path: str, config_path: str +) -> None: + logger.info('Waiting for mon to start...') + c = CephContainer( + ctx, + image=ctx.image, + entrypoint='/usr/bin/ceph', + args=[ + 'status'], + volume_mounts={ + mon_dir: '/var/lib/ceph/mon/ceph-%s:z' % (mon_id), + admin_keyring_path: '/etc/ceph/ceph.client.admin.keyring:z', + config_path: '/etc/ceph/ceph.conf:z', + }, + ) + + # wait for the service to become available + def is_mon_available(): + # type: () -> bool + timeout = ctx.timeout if ctx.timeout else 60 # seconds + out, err, ret = call(ctx, c.run_cmd(), + desc=c.entrypoint, + timeout=timeout) + return ret == 0 + + is_available(ctx, 'mon', is_mon_available) + + +def create_mgr( + ctx: CephadmContext, + uid: int, gid: int, + fsid: str, mgr_id: str, mgr_key: str, + config: str, clifunc: Callable +) -> None: + logger.info('Creating mgr...') + mgr_keyring = '[mgr.%s]\n\tkey = %s\n' % (mgr_id, mgr_key) + mgr_c = get_container(ctx, fsid, 'mgr', mgr_id) + # Note:the default port used by the Prometheus node exporter is opened in fw + ctx.meta_json = json.dumps({'service_name': 'mgr'}) + deploy_daemon(ctx, fsid, 'mgr', mgr_id, mgr_c, uid, gid, + config=config, keyring=mgr_keyring, ports=[9283]) + + # wait for the service to become available + logger.info('Waiting for mgr to start...') + + def is_mgr_available(): + # type: () -> bool + timeout = ctx.timeout if ctx.timeout else 60 # seconds + try: + out = clifunc(['status', '-f', 'json-pretty'], timeout=timeout) + j = json.loads(out) + return j.get('mgrmap', {}).get('available', False) + except Exception as e: + logger.debug('status failed: %s' % e) + return False + is_available(ctx, 'mgr', is_mgr_available) + + +def prepare_ssh( + ctx: CephadmContext, + cli: Callable, wait_for_mgr_restart: Callable +) -> None: + + cli(['cephadm', 'set-user', ctx.ssh_user]) + + if ctx.ssh_config: + logger.info('Using provided ssh config...') + mounts = { + pathify(ctx.ssh_config.name): '/tmp/cephadm-ssh-config:z', + } + cli(['cephadm', 'set-ssh-config', '-i', '/tmp/cephadm-ssh-config'], extra_mounts=mounts) + + if ctx.ssh_private_key and ctx.ssh_public_key: + logger.info('Using provided ssh keys...') + mounts = { + pathify(ctx.ssh_private_key.name): '/tmp/cephadm-ssh-key:z', + pathify(ctx.ssh_public_key.name): '/tmp/cephadm-ssh-key.pub:z' + } + cli(['cephadm', 'set-priv-key', '-i', '/tmp/cephadm-ssh-key'], extra_mounts=mounts) + cli(['cephadm', 'set-pub-key', '-i', '/tmp/cephadm-ssh-key.pub'], extra_mounts=mounts) + else: + logger.info('Generating ssh key...') + cli(['cephadm', 'generate-key']) + ssh_pub = cli(['cephadm', 'get-pub-key']) + + with open(ctx.output_pub_ssh_key, 'w') as f: + f.write(ssh_pub) + logger.info('Wrote public SSH key to %s' % ctx.output_pub_ssh_key) + + logger.info('Adding key to %s@localhost authorized_keys...' % ctx.ssh_user) + try: + s_pwd = pwd.getpwnam(ctx.ssh_user) + except KeyError: + raise Error('Cannot find uid/gid for ssh-user: %s' % (ctx.ssh_user)) + ssh_uid = s_pwd.pw_uid + ssh_gid = s_pwd.pw_gid + ssh_dir = os.path.join(s_pwd.pw_dir, '.ssh') + + if not os.path.exists(ssh_dir): + makedirs(ssh_dir, ssh_uid, ssh_gid, 0o700) + + auth_keys_file = '%s/authorized_keys' % ssh_dir + add_newline = False + + if os.path.exists(auth_keys_file): + with open(auth_keys_file, 'r') as f: + f.seek(0, os.SEEK_END) + if f.tell() > 0: + f.seek(f.tell() - 1, os.SEEK_SET) # go to last char + if f.read() != '\n': + add_newline = True + + with open(auth_keys_file, 'a') as f: + os.fchown(f.fileno(), ssh_uid, ssh_gid) # just in case we created it + os.fchmod(f.fileno(), 0o600) # just in case we created it + if add_newline: + f.write('\n') + f.write(ssh_pub.strip() + '\n') + + host = get_hostname() + logger.info('Adding host %s...' % host) + try: + args = ['orch', 'host', 'add', host] + if ctx.mon_ip: + args.append(unwrap_ipv6(ctx.mon_ip)) + elif ctx.mon_addrv: + addrv_args = parse_mon_addrv(ctx.mon_addrv) + args.append(unwrap_ipv6(addrv_args[0].ip)) + cli(args) + except RuntimeError as e: + raise Error('Failed to add host <%s>: %s' % (host, e)) + + for t in ['mon', 'mgr']: + if not ctx.orphan_initial_daemons: + logger.info('Deploying %s service with default placement...' % t) + cli(['orch', 'apply', t]) + else: + logger.info('Deploying unmanaged %s service...' % t) + cli(['orch', 'apply', t, '--unmanaged']) + + if not ctx.orphan_initial_daemons: + logger.info('Deploying crash service with default placement...') + cli(['orch', 'apply', 'crash']) + + if not ctx.skip_monitoring_stack: + for t in ['prometheus', 'grafana', 'node-exporter', 'alertmanager']: + logger.info('Deploying %s service with default placement...' % t) + cli(['orch', 'apply', t]) + + +def enable_cephadm_mgr_module( + cli: Callable, wait_for_mgr_restart: Callable +) -> None: + + logger.info('Enabling cephadm module...') + cli(['mgr', 'module', 'enable', 'cephadm']) + wait_for_mgr_restart() + logger.info('Setting orchestrator backend to cephadm...') + cli(['orch', 'set', 'backend', 'cephadm']) + + +def prepare_dashboard( + ctx: CephadmContext, + uid: int, gid: int, + cli: Callable, wait_for_mgr_restart: Callable +) -> None: + + # Configure SSL port (cephadm only allows to configure dashboard SSL port) + # if the user does not want to use SSL he can change this setting once the cluster is up + cli(['config', 'set', 'mgr', 'mgr/dashboard/ssl_server_port', str(ctx.ssl_dashboard_port)]) + + # configuring dashboard parameters + logger.info('Enabling the dashboard module...') + cli(['mgr', 'module', 'enable', 'dashboard']) + wait_for_mgr_restart() + + # dashboard crt and key + if ctx.dashboard_key and ctx.dashboard_crt: + logger.info('Using provided dashboard certificate...') + mounts = { + pathify(ctx.dashboard_crt.name): '/tmp/dashboard.crt:z', + pathify(ctx.dashboard_key.name): '/tmp/dashboard.key:z' + } + cli(['dashboard', 'set-ssl-certificate', '-i', '/tmp/dashboard.crt'], extra_mounts=mounts) + cli(['dashboard', 'set-ssl-certificate-key', '-i', '/tmp/dashboard.key'], extra_mounts=mounts) + else: + logger.info('Generating a dashboard self-signed certificate...') + cli(['dashboard', 'create-self-signed-cert']) + + logger.info('Creating initial admin user...') + password = ctx.initial_dashboard_password or generate_password() + tmp_password_file = write_tmp(password, uid, gid) + cmd = ['dashboard', 'ac-user-create', ctx.initial_dashboard_user, '-i', '/tmp/dashboard.pw', 'administrator', '--force-password'] + if not ctx.dashboard_password_noupdate: + cmd.append('--pwd-update-required') + cli(cmd, extra_mounts={pathify(tmp_password_file.name): '/tmp/dashboard.pw:z'}) + logger.info('Fetching dashboard port number...') + out = cli(['config', 'get', 'mgr', 'mgr/dashboard/ssl_server_port']) + port = int(out) + + # Open dashboard port + if not ('skip_firewalld' in ctx and ctx.skip_firewalld): + fw = Firewalld(ctx) + fw.open_ports([port]) + fw.apply_rules() + + logger.info('Ceph Dashboard is now available at:\n\n' + '\t URL: https://%s:%s/\n' + '\t User: %s\n' + '\tPassword: %s\n' % ( + get_fqdn(), port, + ctx.initial_dashboard_user, + password)) + + +def prepare_bootstrap_config( + ctx: CephadmContext, + fsid: str, mon_addr: str, image: str + +) -> str: + + cp = read_config(ctx.config) + if not cp.has_section('global'): + cp.add_section('global') + cp.set('global', 'fsid', fsid) + cp.set('global', 'mon_host', mon_addr) + cp.set('global', 'container_image', image) + + if not cp.has_section('mon'): + cp.add_section('mon') + if ( + not cp.has_option('mon', 'auth_allow_insecure_global_id_reclaim') + and not cp.has_option('mon', 'auth allow insecure global id reclaim') + ): + cp.set('mon', 'auth_allow_insecure_global_id_reclaim', 'false') + + if ctx.single_host_defaults: + logger.info('Adjusting default settings to suit single-host cluster...') + # replicate across osds, not hosts + if ( + not cp.has_option('global', 'osd_crush_chooseleaf_type') + and not cp.has_option('global', 'osd crush chooseleaf type') + ): + cp.set('global', 'osd_crush_chooseleaf_type', '0') + # replica 2x + if ( + not cp.has_option('global', 'osd_pool_default_size') + and not cp.has_option('global', 'osd pool default size') + ): + cp.set('global', 'osd_pool_default_size', '2') + # disable mgr standby modules (so we can colocate multiple mgrs on one host) + if not cp.has_section('mgr'): + cp.add_section('mgr') + if ( + not cp.has_option('mgr', 'mgr_standby_modules') + and not cp.has_option('mgr', 'mgr standby modules') + ): + cp.set('mgr', 'mgr_standby_modules', 'false') + if ctx.log_to_file: + cp.set('global', 'log_to_file', 'true') + cp.set('global', 'log_to_stderr', 'false') + cp.set('global', 'log_to_journald', 'false') + cp.set('global', 'mon_cluster_log_to_file', 'true') + cp.set('global', 'mon_cluster_log_to_stderr', 'false') + cp.set('global', 'mon_cluster_log_to_journald', 'false') + + cpf = StringIO() + cp.write(cpf) + config = cpf.getvalue() + + if ctx.registry_json or ctx.registry_url: + command_registry_login(ctx) + + return config + + +def finish_bootstrap_config( + ctx: CephadmContext, + fsid: str, + config: str, + mon_id: str, mon_dir: str, + mon_network: Optional[str], ipv6: bool, + cli: Callable, + cluster_network: Optional[str], ipv6_cluster_network: bool + +) -> None: + if not ctx.no_minimize_config: + logger.info('Assimilating anything we can from ceph.conf...') + cli([ + 'config', 'assimilate-conf', + '-i', '/var/lib/ceph/mon/ceph-%s/config' % mon_id + ], { + mon_dir: '/var/lib/ceph/mon/ceph-%s:z' % mon_id + }) + logger.info('Generating new minimal ceph.conf...') + cli([ + 'config', 'generate-minimal-conf', + '-o', '/var/lib/ceph/mon/ceph-%s/config' % mon_id + ], { + mon_dir: '/var/lib/ceph/mon/ceph-%s:z' % mon_id + }) + # re-read our minimized config + with open(mon_dir + '/config', 'r') as f: + config = f.read() + logger.info('Restarting the monitor...') + call_throws(ctx, [ + 'systemctl', + 'restart', + get_unit_name(fsid, 'mon', mon_id) + ]) + elif 'image' in ctx and ctx.image: + # we still want to assimilate the given container image if provided + cli(['config', 'set', 'global', 'container_image', f'{ctx.image}']) + + if mon_network: + logger.info(f'Setting mon public_network to {mon_network}') + cli(['config', 'set', 'mon', 'public_network', mon_network]) + + if cluster_network: + logger.info(f'Setting cluster_network to {cluster_network}') + cli(['config', 'set', 'global', 'cluster_network', cluster_network]) + + if ipv6 or ipv6_cluster_network: + logger.info('Enabling IPv6 (ms_bind_ipv6) binding') + cli(['config', 'set', 'global', 'ms_bind_ipv6', 'true']) + + with open(ctx.output_config, 'w') as f: + f.write(config) + logger.info('Wrote config to %s' % ctx.output_config) + pass + + +# funcs to process spec file for apply spec +def _parse_yaml_docs(f: Iterable[str]) -> List[List[str]]: + docs = [] + current_doc = [] # type: List[str] + for line in f: + if re.search(r'^---\s+', line): + if current_doc: + docs.append(current_doc) + current_doc = [] + else: + current_doc.append(line.rstrip()) + if current_doc: + docs.append(current_doc) + return docs + + +def _parse_yaml_obj(doc: List[str]) -> Dict[str, str]: + # note: this only parses the first layer of yaml + obj = {} # type: Dict[str, str] + current_key = '' + for line in doc: + if line.startswith(' '): + obj[current_key] += line.strip() + elif line.endswith(':'): + current_key = line.strip(':') + obj[current_key] = '' + else: + current_key, val = line.split(':') + obj[current_key] = val.strip() + return obj + + +def parse_yaml_objs(f: Iterable[str]) -> List[Dict[str, str]]: + objs = [] + for d in _parse_yaml_docs(f): + objs.append(_parse_yaml_obj(d)) + return objs + + +def _distribute_ssh_keys(ctx: CephadmContext, host_spec: Dict[str, str], bootstrap_hostname: str) -> int: + # copy ssh key to hosts in host spec (used for apply spec) + ssh_key = '/etc/ceph/ceph.pub' + if ctx.ssh_public_key: + ssh_key = ctx.ssh_public_key.name + + if bootstrap_hostname != host_spec['hostname']: + if 'addr' in host_spec: + addr = host_spec['addr'] + else: + addr = host_spec['hostname'] + out, err, code = call(ctx, ['sudo', '-u', ctx.ssh_user, 'ssh-copy-id', '-f', '-i', ssh_key, '-o StrictHostKeyChecking=no', '%s@%s' % (ctx.ssh_user, addr)]) + if code: + logger.info('\nCopying ssh key to host %s at address %s failed!\n' % (host_spec['hostname'], addr)) + return 1 + else: + logger.info('Added ssh key to host %s at address %s\n' % (host_spec['hostname'], addr)) + return 0 + + +@default_image +def command_bootstrap(ctx): + # type: (CephadmContext) -> int + + ctx.error_code = 0 + + if not ctx.output_config: + ctx.output_config = os.path.join(ctx.output_dir, 'ceph.conf') + if not ctx.output_keyring: + ctx.output_keyring = os.path.join(ctx.output_dir, + 'ceph.client.admin.keyring') + if not ctx.output_pub_ssh_key: + ctx.output_pub_ssh_key = os.path.join(ctx.output_dir, 'ceph.pub') + + # verify output files + for f in [ctx.output_config, ctx.output_keyring, + ctx.output_pub_ssh_key]: + if not ctx.allow_overwrite: + if os.path.exists(f): + raise Error('%s already exists; delete or pass ' + '--allow-overwrite to overwrite' % f) + dirname = os.path.dirname(f) + if dirname and not os.path.exists(dirname): + fname = os.path.basename(f) + logger.info(f'Creating directory {dirname} for {fname}') + try: + # use makedirs to create intermediate missing dirs + os.makedirs(dirname, 0o755) + except PermissionError: + raise Error(f'Unable to create {dirname} due to permissions failure. Retry with root, or sudo or preallocate the directory.') + + (user_conf, _) = get_config_and_keyring(ctx) + + if not ctx.skip_prepare_host: + command_prepare_host(ctx) + else: + logger.info('Skip prepare_host') + + # initial vars + fsid = ctx.fsid or make_fsid() + if not is_fsid(fsid): + raise Error('not an fsid: %s' % fsid) + logger.info('Cluster fsid: %s' % fsid) + + hostname = get_hostname() + if '.' in hostname and not ctx.allow_fqdn_hostname: + raise Error('hostname is a fully qualified domain name (%s); either fix (e.g., "sudo hostname %s" or similar) or pass --allow-fqdn-hostname' % (hostname, hostname.split('.')[0])) + mon_id = ctx.mon_id or hostname + mgr_id = ctx.mgr_id or generate_service_id() + + lock = FileLock(ctx, fsid) + lock.acquire() + + (addr_arg, ipv6, mon_network) = prepare_mon_addresses(ctx) + cluster_network, ipv6_cluster_network = prepare_cluster_network(ctx) + + config = prepare_bootstrap_config(ctx, fsid, addr_arg, ctx.image) + + if not ctx.skip_pull: + try: + _pull_image(ctx, ctx.image) + except UnauthorizedRegistryError: + err_str = 'Failed to pull container image. Check that correct registry credentials are provided in bootstrap by --registry-url, --registry-username, --registry-password, or supply --registry-json with credentials' + logger.debug(f'Pulling image for bootstrap on {hostname} failed: {err_str}') + raise Error(err_str) + + image_ver = CephContainer(ctx, ctx.image, 'ceph', ['--version']).run().strip() + logger.info(f'Ceph version: {image_ver}') + + if not ctx.allow_mismatched_release: + image_release = image_ver.split()[4] + if image_release not in \ + [DEFAULT_IMAGE_RELEASE, LATEST_STABLE_RELEASE]: + raise Error( + f'Container release {image_release} != cephadm release {DEFAULT_IMAGE_RELEASE};' + ' please use matching version of cephadm (pass --allow-mismatched-release to continue anyway)' + ) + + logger.info('Extracting ceph user uid/gid from container image...') + (uid, gid) = extract_uid_gid(ctx) + + # create some initial keys + (mon_key, mgr_key, admin_key, bootstrap_keyring, admin_keyring) = \ + create_initial_keys(ctx, uid, gid, mgr_id) + + monmap = create_initial_monmap(ctx, uid, gid, fsid, mon_id, addr_arg) + (mon_dir, log_dir) = \ + prepare_create_mon(ctx, uid, gid, fsid, mon_id, + bootstrap_keyring.name, monmap.name) + + with open(mon_dir + '/config', 'w') as f: + os.fchown(f.fileno(), uid, gid) + os.fchmod(f.fileno(), 0o600) + f.write(config) + + make_var_run(ctx, fsid, uid, gid) + create_mon(ctx, uid, gid, fsid, mon_id) + + # config to issue various CLI commands + tmp_config = write_tmp(config, uid, gid) + + # a CLI helper to reduce our typing + def cli(cmd, extra_mounts={}, timeout=DEFAULT_TIMEOUT): + # type: (List[str], Dict[str, str], Optional[int]) -> str + mounts = { + log_dir: '/var/log/ceph:z', + admin_keyring.name: '/etc/ceph/ceph.client.admin.keyring:z', + tmp_config.name: '/etc/ceph/ceph.conf:z', + } + for k, v in extra_mounts.items(): + mounts[k] = v + timeout = timeout or ctx.timeout + return CephContainer( + ctx, + image=ctx.image, + entrypoint='/usr/bin/ceph', + args=cmd, + volume_mounts=mounts, + ).run(timeout=timeout) + + wait_for_mon(ctx, mon_id, mon_dir, admin_keyring.name, tmp_config.name) + + finish_bootstrap_config(ctx, fsid, config, mon_id, mon_dir, + mon_network, ipv6, cli, + cluster_network, ipv6_cluster_network) + + # output files + with open(ctx.output_keyring, 'w') as f: + os.fchmod(f.fileno(), 0o600) + f.write('[client.admin]\n' + '\tkey = ' + admin_key + '\n') + logger.info('Wrote keyring to %s' % ctx.output_keyring) + + # create mgr + create_mgr(ctx, uid, gid, fsid, mgr_id, mgr_key, config, cli) + + if user_conf: + # user given config settings were already assimilated earlier + # but if the given settings contained any attributes in + # the mgr (e.g. mgr/cephadm/container_image_prometheus) + # they don't seem to be stored if there isn't a mgr yet. + # Since re-assimilating the same conf settings should be + # idempotent we can just do it again here. + with tempfile.NamedTemporaryFile(buffering=0) as tmp: + tmp.write(user_conf.encode('utf-8')) + cli(['config', 'assimilate-conf', + '-i', '/var/lib/ceph/user.conf'], + {tmp.name: '/var/lib/ceph/user.conf:z'}) + + # wait for mgr to restart (after enabling a module) + def wait_for_mgr_restart() -> None: + # first get latest mgrmap epoch from the mon. try newer 'mgr + # stat' command first, then fall back to 'mgr dump' if + # necessary + try: + j = json_loads_retry(lambda: cli(['mgr', 'stat'])) + except Exception: + j = json_loads_retry(lambda: cli(['mgr', 'dump'])) + epoch = j['epoch'] + + # wait for mgr to have it + logger.info('Waiting for the mgr to restart...') + + def mgr_has_latest_epoch(): + # type: () -> bool + try: + out = cli(['tell', 'mgr', 'mgr_status']) + j = json.loads(out) + return j['mgrmap_epoch'] >= epoch + except Exception as e: + logger.debug('tell mgr mgr_status failed: %s' % e) + return False + is_available(ctx, 'mgr epoch %d' % epoch, mgr_has_latest_epoch) + + enable_cephadm_mgr_module(cli, wait_for_mgr_restart) + + # ssh + if not ctx.skip_ssh: + prepare_ssh(ctx, cli, wait_for_mgr_restart) + + if ctx.registry_url and ctx.registry_username and ctx.registry_password: + registry_credentials = {'url': ctx.registry_url, 'username': ctx.registry_username, 'password': ctx.registry_password} + cli(['config-key', 'set', 'mgr/cephadm/registry_credentials', json.dumps(registry_credentials)]) + + cli(['config', 'set', 'mgr', 'mgr/cephadm/container_init', str(ctx.container_init), '--force']) + + if ctx.with_exporter: + cli(['config-key', 'set', 'mgr/cephadm/exporter_enabled', 'true']) + if ctx.exporter_config: + logger.info('Applying custom cephadm exporter settings') + # validated within the parser, so we can just apply to the store + with tempfile.NamedTemporaryFile(buffering=0) as tmp: + tmp.write(json.dumps(ctx.exporter_config).encode('utf-8')) + mounts = { + tmp.name: '/tmp/exporter-config.json:z' + } + cli(['cephadm', 'set-exporter-config', '-i', '/tmp/exporter-config.json'], extra_mounts=mounts) + logger.info('-> Use ceph orch apply cephadm-exporter to deploy') + else: + # generate a default SSL configuration for the exporter(s) + logger.info('Generating a default cephadm exporter configuration (self-signed)') + cli(['cephadm', 'generate-exporter-config']) + # + # deploy the service (commented out until the cephadm changes are in the ceph container build) + logger.info('Deploying cephadm exporter service with default placement...') + cli(['orch', 'apply', 'cephadm-exporter']) + + if not ctx.skip_dashboard: + prepare_dashboard(ctx, uid, gid, cli, wait_for_mgr_restart) + + if ctx.output_config == '/etc/ceph/ceph.conf' and not ctx.skip_admin_label and not ctx.no_minimize_config: + logger.info('Enabling client.admin keyring and conf on hosts with "admin" label') + try: + cli(['orch', 'client-keyring', 'set', 'client.admin', 'label:_admin']) + cli(['orch', 'host', 'label', 'add', get_hostname(), '_admin']) + except Exception: + logger.info('Unable to set up "admin" label; assuming older version of Ceph') + + if ctx.apply_spec: + logger.info('Applying %s to cluster' % ctx.apply_spec) + # copy ssh key to hosts in spec file + with open(ctx.apply_spec) as f: + try: + for spec in parse_yaml_objs(f): + if spec.get('service_type') == 'host': + _distribute_ssh_keys(ctx, spec, hostname) + except ValueError: + logger.info('Unable to parse %s succesfully' % ctx.apply_spec) + + mounts = {} + mounts[pathify(ctx.apply_spec)] = '/tmp/spec.yml:ro' + try: + out = cli(['orch', 'apply', '-i', '/tmp/spec.yml'], extra_mounts=mounts) + logger.info(out) + except Exception: + ctx.error_code = -errno.EINVAL + logger.info('\nApplying %s to cluster failed!\n' % ctx.apply_spec) + + # enable autotune for osd_memory_target + logger.info('Enabling autotune for osd_memory_target') + cli(['config', 'set', 'osd', 'osd_memory_target_autotune', 'true']) + + # Notify the Dashboard to show the 'Expand cluster' page on first log in. + cli(['config-key', 'set', 'mgr/dashboard/cluster/status', 'INSTALLED']) + + logger.info('You can access the Ceph CLI as following in case of multi-cluster or non-default config:\n\n' + '\tsudo %s shell --fsid %s -c %s -k %s\n' % ( + sys.argv[0], + fsid, + ctx.output_config, + ctx.output_keyring)) + + logger.info('Or, if you are only running a single cluster on this host:\n\n\tsudo %s shell \n' % (sys.argv[0])) + + logger.info('Please consider enabling telemetry to help improve Ceph:\n\n' + '\tceph telemetry on\n\n' + 'For more information see:\n\n' + '\thttps://docs.ceph.com/en/pacific/mgr/telemetry/\n') + logger.info('Bootstrap complete.') + return ctx.error_code + +################################## + + +def command_registry_login(ctx: CephadmContext) -> int: + if ctx.registry_json: + logger.info('Pulling custom registry login info from %s.' % ctx.registry_json) + d = get_parm(ctx.registry_json) + if d.get('url') and d.get('username') and d.get('password'): + ctx.registry_url = d.get('url') + ctx.registry_username = d.get('username') + ctx.registry_password = d.get('password') + registry_login(ctx, ctx.registry_url, ctx.registry_username, ctx.registry_password) + else: + raise Error('json provided for custom registry login did not include all necessary fields. ' + 'Please setup json file as\n' + '{\n' + ' "url": "REGISTRY_URL",\n' + ' "username": "REGISTRY_USERNAME",\n' + ' "password": "REGISTRY_PASSWORD"\n' + '}\n') + elif ctx.registry_url and ctx.registry_username and ctx.registry_password: + registry_login(ctx, ctx.registry_url, ctx.registry_username, ctx.registry_password) + else: + raise Error('Invalid custom registry arguments received. To login to a custom registry include ' + '--registry-url, --registry-username and --registry-password ' + 'options or --registry-json option') + return 0 + + +def registry_login(ctx: CephadmContext, url: Optional[str], username: Optional[str], password: Optional[str]) -> None: + logger.info('Logging into custom registry.') + try: + engine = ctx.container_engine + cmd = [engine.path, 'login', + '-u', username, '-p', password, + url] + if isinstance(engine, Podman): + cmd.append('--authfile=/etc/ceph/podman-auth.json') + out, _, _ = call_throws(ctx, cmd) + if isinstance(engine, Podman): + os.chmod('/etc/ceph/podman-auth.json', 0o600) + except Exception: + raise Error('Failed to login to custom registry @ %s as %s with given password' % (ctx.registry_url, ctx.registry_username)) + +################################## + + +def extract_uid_gid_monitoring(ctx, daemon_type): + # type: (CephadmContext, str) -> Tuple[int, int] + + if daemon_type == 'prometheus': + uid, gid = extract_uid_gid(ctx, file_path='/etc/prometheus') + elif daemon_type == 'node-exporter': + uid, gid = 65534, 65534 + elif daemon_type == 'grafana': + uid, gid = extract_uid_gid(ctx, file_path='/var/lib/grafana') + elif daemon_type == 'alertmanager': + uid, gid = extract_uid_gid(ctx, file_path=['/etc/alertmanager', '/etc/prometheus']) + else: + raise Error('{} not implemented yet'.format(daemon_type)) + return uid, gid + + +def get_container_with_extra_args(ctx: CephadmContext, + fsid: str, daemon_type: str, daemon_id: Union[int, str], + privileged: bool = False, + ptrace: bool = False, + container_args: Optional[List[str]] = None) -> 'CephContainer': + # wrapper for get_container that additionally adds extra_container_args if present + # used for deploying daemons with additional podman/docker container arguments + c = get_container(ctx, fsid, daemon_type, daemon_id, privileged, ptrace, container_args) + if 'extra_container_args' in ctx and ctx.extra_container_args: + c.container_args.extend(ctx.extra_container_args) + return c + + +@default_image +def command_deploy(ctx): + # type: (CephadmContext) -> None + daemon_type, daemon_id = ctx.name.split('.', 1) + + lock = FileLock(ctx, ctx.fsid) + lock.acquire() + + if daemon_type not in get_supported_daemons(): + raise Error('daemon type %s not recognized' % daemon_type) + + redeploy = False + unit_name = get_unit_name(ctx.fsid, daemon_type, daemon_id) + (_, state, _) = check_unit(ctx, unit_name) + if state == 'running' or is_container_running(ctx, CephContainer.for_daemon(ctx, ctx.fsid, daemon_type, daemon_id, 'bash')): + redeploy = True + + if ctx.reconfig: + logger.info('%s daemon %s ...' % ('Reconfig', ctx.name)) + elif redeploy: + logger.info('%s daemon %s ...' % ('Redeploy', ctx.name)) + else: + logger.info('%s daemon %s ...' % ('Deploy', ctx.name)) + + # Migrate sysctl conf files from /usr/lib to /etc + migrate_sysctl_dir(ctx, ctx.fsid) + + # Get and check ports explicitly required to be opened + daemon_ports = [] # type: List[int] + + # only check port in use if not reconfig or redeploy since service + # we are redeploying/reconfiguring will already be using the port + if not ctx.reconfig and not redeploy: + if ctx.tcp_ports: + daemon_ports = list(map(int, ctx.tcp_ports.split())) + + if daemon_type in Ceph.daemons: + config, keyring = get_config_and_keyring(ctx) + uid, gid = extract_uid_gid(ctx) + make_var_run(ctx, ctx.fsid, uid, gid) + + c = get_container_with_extra_args(ctx, ctx.fsid, daemon_type, daemon_id, + ptrace=ctx.allow_ptrace) + deploy_daemon(ctx, ctx.fsid, daemon_type, daemon_id, c, uid, gid, + config=config, keyring=keyring, + osd_fsid=ctx.osd_fsid, + reconfig=ctx.reconfig, + ports=daemon_ports) + + elif daemon_type in Monitoring.components: + # monitoring daemon - prometheus, grafana, alertmanager, node-exporter + # Default Checks + # make sure provided config-json is sufficient + config = get_parm(ctx.config_json) # type: ignore + required_files = Monitoring.components[daemon_type].get('config-json-files', list()) + required_args = Monitoring.components[daemon_type].get('config-json-args', list()) + if required_files: + if not config or not all(c in config.get('files', {}).keys() for c in required_files): # type: ignore + raise Error('{} deployment requires config-json which must ' + 'contain file content for {}'.format(daemon_type.capitalize(), ', '.join(required_files))) + if required_args: + if not config or not all(c in config.keys() for c in required_args): # type: ignore + raise Error('{} deployment requires config-json which must ' + 'contain arg for {}'.format(daemon_type.capitalize(), ', '.join(required_args))) + + uid, gid = extract_uid_gid_monitoring(ctx, daemon_type) + c = get_container_with_extra_args(ctx, ctx.fsid, daemon_type, daemon_id) + deploy_daemon(ctx, ctx.fsid, daemon_type, daemon_id, c, uid, gid, + reconfig=ctx.reconfig, + ports=daemon_ports) + + elif daemon_type == NFSGanesha.daemon_type: + if not ctx.reconfig and not redeploy and not daemon_ports: + daemon_ports = list(NFSGanesha.port_map.values()) + + config, keyring = get_config_and_keyring(ctx) + # TODO: extract ganesha uid/gid (997, 994) ? + uid, gid = extract_uid_gid(ctx) + c = get_container_with_extra_args(ctx, ctx.fsid, daemon_type, daemon_id) + deploy_daemon(ctx, ctx.fsid, daemon_type, daemon_id, c, uid, gid, + config=config, keyring=keyring, + reconfig=ctx.reconfig, + ports=daemon_ports) + + elif daemon_type == CephIscsi.daemon_type: + config, keyring = get_config_and_keyring(ctx) + uid, gid = extract_uid_gid(ctx) + c = get_container_with_extra_args(ctx, ctx.fsid, daemon_type, daemon_id) + deploy_daemon(ctx, ctx.fsid, daemon_type, daemon_id, c, uid, gid, + config=config, keyring=keyring, + reconfig=ctx.reconfig, + ports=daemon_ports) + + elif daemon_type == HAproxy.daemon_type: + haproxy = HAproxy.init(ctx, ctx.fsid, daemon_id) + uid, gid = haproxy.extract_uid_gid_haproxy() + c = get_container_with_extra_args(ctx, ctx.fsid, daemon_type, daemon_id) + deploy_daemon(ctx, ctx.fsid, daemon_type, daemon_id, c, uid, gid, + reconfig=ctx.reconfig, + ports=daemon_ports) + + elif daemon_type == Keepalived.daemon_type: + keepalived = Keepalived.init(ctx, ctx.fsid, daemon_id) + uid, gid = keepalived.extract_uid_gid_keepalived() + c = get_container_with_extra_args(ctx, ctx.fsid, daemon_type, daemon_id) + deploy_daemon(ctx, ctx.fsid, daemon_type, daemon_id, c, uid, gid, + reconfig=ctx.reconfig, + ports=daemon_ports) + + elif daemon_type == CustomContainer.daemon_type: + cc = CustomContainer.init(ctx, ctx.fsid, daemon_id) + if not ctx.reconfig and not redeploy: + daemon_ports.extend(cc.ports) + c = get_container_with_extra_args(ctx, ctx.fsid, daemon_type, daemon_id, + privileged=cc.privileged, + ptrace=ctx.allow_ptrace) + deploy_daemon(ctx, ctx.fsid, daemon_type, daemon_id, c, + uid=cc.uid, gid=cc.gid, config=None, + keyring=None, reconfig=ctx.reconfig, + ports=daemon_ports) + + elif daemon_type == CephadmDaemon.daemon_type: + # get current user gid and uid + uid = os.getuid() + gid = os.getgid() + config_js = get_parm(ctx.config_json) # type: Dict[str, str] + if not daemon_ports: + logger.info('cephadm-exporter will use default port ({})'.format(CephadmDaemon.default_port)) + daemon_ports = [CephadmDaemon.default_port] + + CephadmDaemon.validate_config(config_js) + + deploy_daemon(ctx, ctx.fsid, daemon_type, daemon_id, None, + uid, gid, ports=daemon_ports) + + elif daemon_type == SNMPGateway.daemon_type: + sc = SNMPGateway.init(ctx, ctx.fsid, daemon_id) + c = get_container_with_extra_args(ctx, ctx.fsid, daemon_type, daemon_id) + deploy_daemon(ctx, ctx.fsid, daemon_type, daemon_id, c, + sc.uid, sc.gid, + ports=daemon_ports) + + else: + raise Error('daemon type {} not implemented in command_deploy function' + .format(daemon_type)) + +################################## + + +@infer_image +def command_run(ctx): + # type: (CephadmContext) -> int + (daemon_type, daemon_id) = ctx.name.split('.', 1) + c = get_container(ctx, ctx.fsid, daemon_type, daemon_id) + command = c.run_cmd() + return call_timeout(ctx, command, ctx.timeout) + +################################## + + +@infer_fsid +@infer_config +@infer_image +@validate_fsid +def command_shell(ctx): + # type: (CephadmContext) -> int + cp = read_config(ctx.config) + if cp.has_option('global', 'fsid') and \ + cp.get('global', 'fsid') != ctx.fsid: + raise Error('fsid does not match ceph.conf') + + if ctx.name: + if '.' in ctx.name: + (daemon_type, daemon_id) = ctx.name.split('.', 1) + else: + daemon_type = ctx.name + daemon_id = None + else: + daemon_type = 'osd' # get the most mounts + daemon_id = None + + if ctx.fsid and daemon_type in Ceph.daemons: + make_log_dir(ctx, ctx.fsid) + + if daemon_id and not ctx.fsid: + raise Error('must pass --fsid to specify cluster') + + # use /etc/ceph files by default, if present. we do this instead of + # making these defaults in the arg parser because we don't want an error + # if they don't exist. + if not ctx.keyring and os.path.exists(SHELL_DEFAULT_KEYRING): + ctx.keyring = SHELL_DEFAULT_KEYRING + + container_args: List[str] = ['-i'] + mounts = get_container_mounts(ctx, ctx.fsid, daemon_type, daemon_id, + no_config=True if ctx.config else False) + binds = get_container_binds(ctx, ctx.fsid, daemon_type, daemon_id) + if ctx.config: + mounts[pathify(ctx.config)] = '/etc/ceph/ceph.conf:z' + if ctx.keyring: + mounts[pathify(ctx.keyring)] = '/etc/ceph/ceph.keyring:z' + if ctx.mount: + for _mount in ctx.mount: + split_src_dst = _mount.split(':') + mount = pathify(split_src_dst[0]) + filename = os.path.basename(split_src_dst[0]) + if len(split_src_dst) > 1: + dst = split_src_dst[1] + if len(split_src_dst) == 3: + dst = '{}:{}'.format(dst, split_src_dst[2]) + mounts[mount] = dst + else: + mounts[mount] = '/mnt/{}'.format(filename) + if ctx.command: + command = ctx.command + else: + command = ['bash'] + container_args += [ + '-t', + '-e', 'LANG=C', + '-e', 'PS1=%s' % CUSTOM_PS1, + ] + if ctx.fsid: + home = os.path.join(ctx.data_dir, ctx.fsid, 'home') + if not os.path.exists(home): + logger.debug('Creating root home at %s' % home) + makedirs(home, 0, 0, 0o660) + if os.path.exists('/etc/skel'): + for f in os.listdir('/etc/skel'): + if f.startswith('.bash'): + shutil.copyfile(os.path.join('/etc/skel', f), + os.path.join(home, f)) + mounts[home] = '/root' + + for i in ctx.volume: + a, b = i.split(':', 1) + mounts[a] = b + + c = CephContainer( + ctx, + image=ctx.image, + entrypoint='doesnotmatter', + args=[], + container_args=container_args, + volume_mounts=mounts, + bind_mounts=binds, + envs=ctx.env, + privileged=True) + command = c.shell_cmd(command) + + return call_timeout(ctx, command, ctx.timeout) + +################################## + + +@infer_fsid +def command_enter(ctx): + # type: (CephadmContext) -> int + if not ctx.fsid: + raise Error('must pass --fsid to specify cluster') + (daemon_type, daemon_id) = ctx.name.split('.', 1) + container_args = ['-i'] # type: List[str] + if ctx.command: + command = ctx.command + else: + command = ['sh'] + container_args += [ + '-t', + '-e', 'LANG=C', + '-e', 'PS1=%s' % CUSTOM_PS1, + ] + c = CephContainer( + ctx, + image=ctx.image, + entrypoint='doesnotmatter', + container_args=container_args, + cname='ceph-%s-%s.%s' % (ctx.fsid, daemon_type, daemon_id), + ) + command = c.exec_cmd(command) + return call_timeout(ctx, command, ctx.timeout) + +################################## + + +@infer_fsid +@infer_image +@validate_fsid +def command_ceph_volume(ctx): + # type: (CephadmContext) -> None + cp = read_config(ctx.config) + if cp.has_option('global', 'fsid') and \ + cp.get('global', 'fsid') != ctx.fsid: + raise Error('fsid does not match ceph.conf') + + if ctx.fsid: + make_log_dir(ctx, ctx.fsid) + + lock = FileLock(ctx, ctx.fsid) + lock.acquire() + + (uid, gid) = (0, 0) # ceph-volume runs as root + mounts = get_container_mounts(ctx, ctx.fsid, 'osd', None) + + tmp_config = None + tmp_keyring = None + + (config, keyring) = get_config_and_keyring(ctx) + + if config: + # tmp config file + tmp_config = write_tmp(config, uid, gid) + mounts[tmp_config.name] = '/etc/ceph/ceph.conf:z' + + if keyring: + # tmp keyring file + tmp_keyring = write_tmp(keyring, uid, gid) + mounts[tmp_keyring.name] = '/var/lib/ceph/bootstrap-osd/ceph.keyring:z' + + c = get_ceph_volume_container( + ctx, + envs=ctx.env, + args=ctx.command, + volume_mounts=mounts, + ) + + out, err, code = call_throws(ctx, c.run_cmd()) + if not code: + print(out) + +################################## + + +@infer_fsid +def command_unit(ctx): + # type: (CephadmContext) -> int + if not ctx.fsid: + raise Error('must pass --fsid to specify cluster') + + unit_name = get_unit_name_by_daemon_name(ctx, ctx.fsid, ctx.name) + + _, _, code = call( + ctx, + ['systemctl', ctx.command, unit_name], + verbosity=CallVerbosity.VERBOSE, + desc='' + ) + return code + +################################## + + +@infer_fsid +def command_logs(ctx): + # type: (CephadmContext) -> None + if not ctx.fsid: + raise Error('must pass --fsid to specify cluster') + + unit_name = get_unit_name_by_daemon_name(ctx, ctx.fsid, ctx.name) + + cmd = [find_program('journalctl')] + cmd.extend(['-u', unit_name]) + if ctx.command: + cmd.extend(ctx.command) + + # call this directly, without our wrapper, so that we get an unmolested + # stdout with logger prefixing. + logger.debug('Running command: %s' % ' '.join(cmd)) + subprocess.call(cmd, env=os.environ.copy()) # type: ignore + +################################## + + +def list_networks(ctx): + # type: (CephadmContext) -> Dict[str,Dict[str, Set[str]]] + + # sadly, 18.04's iproute2 4.15.0-2ubun doesn't support the -j flag, + # so we'll need to use a regex to parse 'ip' command output. + # + # out, _, _ = call_throws(['ip', '-j', 'route', 'ls']) + # j = json.loads(out) + # for x in j: + res = _list_ipv4_networks(ctx) + res.update(_list_ipv6_networks(ctx)) + return res + + +def _list_ipv4_networks(ctx: CephadmContext) -> Dict[str, Dict[str, Set[str]]]: + execstr: Optional[str] = find_executable('ip') + if not execstr: + raise FileNotFoundError("unable to find 'ip' command") + out, _, _ = call_throws(ctx, [execstr, 'route', 'ls']) + return _parse_ipv4_route(out) + + +def _parse_ipv4_route(out: str) -> Dict[str, Dict[str, Set[str]]]: + r = {} # type: Dict[str, Dict[str, Set[str]]] + p = re.compile(r'^(\S+) (?:via \S+)? ?dev (\S+) (.*)scope link (.*)src (\S+)') + for line in out.splitlines(): + m = p.findall(line) + if not m: + continue + net = m[0][0] + if '/' not in net: # aggregate /32 mask for single host sub-networks + net += '/32' + iface = m[0][1] + ip = m[0][4] + if net not in r: + r[net] = {} + if iface not in r[net]: + r[net][iface] = set() + r[net][iface].add(ip) + return r + + +def _list_ipv6_networks(ctx: CephadmContext) -> Dict[str, Dict[str, Set[str]]]: + execstr: Optional[str] = find_executable('ip') + if not execstr: + raise FileNotFoundError("unable to find 'ip' command") + routes, _, _ = call_throws(ctx, [execstr, '-6', 'route', 'ls']) + ips, _, _ = call_throws(ctx, [execstr, '-6', 'addr', 'ls']) + return _parse_ipv6_route(routes, ips) + + +def _parse_ipv6_route(routes: str, ips: str) -> Dict[str, Dict[str, Set[str]]]: + r = {} # type: Dict[str, Dict[str, Set[str]]] + route_p = re.compile(r'^(\S+) dev (\S+) proto (\S+) metric (\S+) .*pref (\S+)$') + ip_p = re.compile(r'^\s+inet6 (\S+)/(.*)scope (.*)$') + iface_p = re.compile(r'^(\d+): (\S+): (.*)$') + for line in routes.splitlines(): + m = route_p.findall(line) + if not m or m[0][0].lower() == 'default': + continue + net = m[0][0] + if '/' not in net: # aggregate /128 mask for single host sub-networks + net += '/128' + iface = m[0][1] + if iface == 'lo': # skip loopback devices + continue + if net not in r: + r[net] = {} + if iface not in r[net]: + r[net][iface] = set() + + iface = None + for line in ips.splitlines(): + m = ip_p.findall(line) + if not m: + m = iface_p.findall(line) + if m: + # drop @... suffix, if present + iface = m[0][1].split('@')[0] + continue + ip = m[0][0] + # find the network it belongs to + net = [n for n in r.keys() + if ipaddress.ip_address(ip) in ipaddress.ip_network(n)] + if net and iface in r[net[0]]: + assert iface + r[net[0]][iface].add(ip) + + return r + + +def command_list_networks(ctx): + # type: (CephadmContext) -> None + r = list_networks(ctx) + + def serialize_sets(obj: Any) -> Any: + return list(obj) if isinstance(obj, set) else obj + + print(json.dumps(r, indent=4, default=serialize_sets)) + +################################## + + +def command_ls(ctx): + # type: (CephadmContext) -> None + ls = list_daemons(ctx, detail=not ctx.no_detail, + legacy_dir=ctx.legacy_dir) + print(json.dumps(ls, indent=4)) + + +def with_units_to_int(v: str) -> int: + if v.endswith('iB'): + v = v[:-2] + elif v.endswith('B'): + v = v[:-1] + mult = 1 + if v[-1].upper() == 'K': + mult = 1024 + v = v[:-1] + elif v[-1].upper() == 'M': + mult = 1024 * 1024 + v = v[:-1] + elif v[-1].upper() == 'G': + mult = 1024 * 1024 * 1024 + v = v[:-1] + elif v[-1].upper() == 'T': + mult = 1024 * 1024 * 1024 * 1024 + v = v[:-1] + return int(float(v) * mult) + + +def list_daemons(ctx, detail=True, legacy_dir=None): + # type: (CephadmContext, bool, Optional[str]) -> List[Dict[str, str]] + host_version: Optional[str] = None + ls = [] + container_path = ctx.container_engine.path + + data_dir = ctx.data_dir + if legacy_dir is not None: + data_dir = os.path.abspath(legacy_dir + data_dir) + + # keep track of ceph versions we see + seen_versions = {} # type: Dict[str, Optional[str]] + + # keep track of image digests + seen_digests = {} # type: Dict[str, List[str]] + + # keep track of memory and cpu usage we've seen + seen_memusage = {} # type: Dict[str, int] + seen_cpuperc = {} # type: Dict[str, str] + out, err, code = call( + ctx, + [container_path, 'stats', '--format', '{{.ID}},{{.MemUsage}}', '--no-stream'], + verbosity=CallVerbosity.DEBUG + ) + seen_memusage_cid_len, seen_memusage = _parse_mem_usage(code, out) + + out, err, code = call( + ctx, + [container_path, 'stats', '--format', '{{.ID}},{{.CPUPerc}}', '--no-stream'], + verbosity=CallVerbosity.DEBUG + ) + seen_cpuperc_cid_len, seen_cpuperc = _parse_cpu_perc(code, out) + + # /var/lib/ceph + if os.path.exists(data_dir): + for i in os.listdir(data_dir): + if i in ['mon', 'osd', 'mds', 'mgr']: + daemon_type = i + for j in os.listdir(os.path.join(data_dir, i)): + if '-' not in j: + continue + (cluster, daemon_id) = j.split('-', 1) + fsid = get_legacy_daemon_fsid(ctx, + cluster, daemon_type, daemon_id, + legacy_dir=legacy_dir) + legacy_unit_name = 'ceph-%s@%s' % (daemon_type, daemon_id) + val: Dict[str, Any] = { + 'style': 'legacy', + 'name': '%s.%s' % (daemon_type, daemon_id), + 'fsid': fsid if fsid is not None else 'unknown', + 'systemd_unit': legacy_unit_name, + } + if detail: + (val['enabled'], val['state'], _) = \ + check_unit(ctx, legacy_unit_name) + if not host_version: + try: + out, err, code = call(ctx, + ['ceph', '-v'], + verbosity=CallVerbosity.DEBUG) + if not code and out.startswith('ceph version '): + host_version = out.split(' ')[2] + except Exception: + pass + val['host_version'] = host_version + ls.append(val) + elif is_fsid(i): + fsid = str(i) # convince mypy that fsid is a str here + for j in os.listdir(os.path.join(data_dir, i)): + if '.' in j and os.path.isdir(os.path.join(data_dir, fsid, j)): + name = j + (daemon_type, daemon_id) = j.split('.', 1) + unit_name = get_unit_name(fsid, + daemon_type, + daemon_id) + else: + continue + val = { + 'style': 'cephadm:v1', + 'name': name, + 'fsid': fsid, + 'systemd_unit': unit_name, + } + if detail: + # get container id + (val['enabled'], val['state'], _) = \ + check_unit(ctx, unit_name) + container_id = None + image_name = None + image_id = None + image_digests = None + version = None + start_stamp = None + + out, err, code = get_container_stats(ctx, container_path, fsid, daemon_type, daemon_id) + if not code: + (container_id, image_name, image_id, start, + version) = out.strip().split(',') + image_id = normalize_container_id(image_id) + daemon_type = name.split('.', 1)[0] + start_stamp = try_convert_datetime(start) + + # collect digests for this image id + image_digests = seen_digests.get(image_id) + if not image_digests: + out, err, code = call( + ctx, + [ + container_path, 'image', 'inspect', image_id, + '--format', '{{.RepoDigests}}', + ], + verbosity=CallVerbosity.DEBUG) + if not code: + image_digests = list(set(map( + normalize_image_digest, + out.strip()[1:-1].split(' ')))) + seen_digests[image_id] = image_digests + + # identify software version inside the container (if we can) + if not version or '.' not in version: + version = seen_versions.get(image_id, None) + if daemon_type == NFSGanesha.daemon_type: + version = NFSGanesha.get_version(ctx, container_id) + if daemon_type == CephIscsi.daemon_type: + version = CephIscsi.get_version(ctx, container_id) + elif not version: + if daemon_type in Ceph.daemons: + out, err, code = call(ctx, + [container_path, 'exec', container_id, + 'ceph', '-v'], + verbosity=CallVerbosity.DEBUG) + if not code and \ + out.startswith('ceph version '): + version = out.split(' ')[2] + seen_versions[image_id] = version + elif daemon_type == 'grafana': + out, err, code = call(ctx, + [container_path, 'exec', container_id, + 'grafana-server', '-v'], + verbosity=CallVerbosity.DEBUG) + if not code and \ + out.startswith('Version '): + version = out.split(' ')[1] + seen_versions[image_id] = version + elif daemon_type in ['prometheus', + 'alertmanager', + 'node-exporter']: + version = Monitoring.get_version(ctx, container_id, daemon_type) + seen_versions[image_id] = version + elif daemon_type == 'haproxy': + out, err, code = call(ctx, + [container_path, 'exec', container_id, + 'haproxy', '-v'], + verbosity=CallVerbosity.DEBUG) + if not code and \ + out.startswith('HA-Proxy version '): + version = out.split(' ')[2] + seen_versions[image_id] = version + elif daemon_type == 'keepalived': + out, err, code = call(ctx, + [container_path, 'exec', container_id, + 'keepalived', '--version'], + verbosity=CallVerbosity.DEBUG) + if not code and \ + err.startswith('Keepalived '): + version = err.split(' ')[1] + if version[0] == 'v': + version = version[1:] + seen_versions[image_id] = version + elif daemon_type == CustomContainer.daemon_type: + # Because a custom container can contain + # everything, we do not know which command + # to execute to get the version. + pass + elif daemon_type == SNMPGateway.daemon_type: + version = SNMPGateway.get_version(ctx, fsid, daemon_id) + seen_versions[image_id] = version + else: + logger.warning('version for unknown daemon type %s' % daemon_type) + else: + vfile = os.path.join(data_dir, fsid, j, 'unit.image') # type: ignore + try: + with open(vfile, 'r') as f: + image_name = f.read().strip() or None + except IOError: + pass + + # unit.meta? + mfile = os.path.join(data_dir, fsid, j, 'unit.meta') # type: ignore + try: + with open(mfile, 'r') as f: + meta = json.loads(f.read()) + val.update(meta) + except IOError: + pass + + val['container_id'] = container_id + val['container_image_name'] = image_name + val['container_image_id'] = image_id + val['container_image_digests'] = image_digests + if container_id: + val['memory_usage'] = seen_memusage.get(container_id[0:seen_memusage_cid_len]) + val['cpu_percentage'] = seen_cpuperc.get(container_id[0:seen_cpuperc_cid_len]) + val['version'] = version + val['started'] = start_stamp + val['created'] = get_file_timestamp( + os.path.join(data_dir, fsid, j, 'unit.created') + ) + val['deployed'] = get_file_timestamp( + os.path.join(data_dir, fsid, j, 'unit.image')) + val['configured'] = get_file_timestamp( + os.path.join(data_dir, fsid, j, 'unit.configured')) + ls.append(val) + + return ls + + +def _parse_mem_usage(code: int, out: str) -> Tuple[int, Dict[str, int]]: + # keep track of memory usage we've seen + seen_memusage = {} # type: Dict[str, int] + seen_memusage_cid_len = 0 + if not code: + for line in out.splitlines(): + (cid, usage) = line.split(',') + (used, limit) = usage.split(' / ') + try: + seen_memusage[cid] = with_units_to_int(used) + if not seen_memusage_cid_len: + seen_memusage_cid_len = len(cid) + except ValueError: + logger.info('unable to parse memory usage line\n>{}'.format(line)) + pass + return seen_memusage_cid_len, seen_memusage + + +def _parse_cpu_perc(code: int, out: str) -> Tuple[int, Dict[str, str]]: + seen_cpuperc = {} + seen_cpuperc_cid_len = 0 + if not code: + for line in out.splitlines(): + (cid, cpuperc) = line.split(',') + try: + seen_cpuperc[cid] = cpuperc + if not seen_cpuperc_cid_len: + seen_cpuperc_cid_len = len(cid) + except ValueError: + logger.info('unable to parse cpu percentage line\n>{}'.format(line)) + pass + return seen_cpuperc_cid_len, seen_cpuperc + + +def get_daemon_description(ctx, fsid, name, detail=False, legacy_dir=None): + # type: (CephadmContext, str, str, bool, Optional[str]) -> Dict[str, str] + + for d in list_daemons(ctx, detail=detail, legacy_dir=legacy_dir): + if d['fsid'] != fsid: + continue + if d['name'] != name: + continue + return d + raise Error('Daemon not found: {}. See `cephadm ls`'.format(name)) + + +def get_container_stats(ctx: CephadmContext, container_path: str, fsid: str, daemon_type: str, daemon_id: str) -> Tuple[str, str, int]: + c = CephContainer.for_daemon(ctx, fsid, daemon_type, daemon_id, 'bash') + out, err, code = '', '', -1 + for name in (c.cname, c.old_cname): + cmd = [ + container_path, 'inspect', + '--format', '{{.Id}},{{.Config.Image}},{{.Image}},{{.Created}},{{index .Config.Labels "io.ceph.version"}}', + name + ] + out, err, code = call(ctx, cmd, verbosity=CallVerbosity.DEBUG) + if not code: + break + return out, err, code + +################################## + + +@default_image +def command_adopt(ctx): + # type: (CephadmContext) -> None + + if not ctx.skip_pull: + try: + _pull_image(ctx, ctx.image) + except UnauthorizedRegistryError: + err_str = 'Failed to pull container image. Host may not be logged into container registry. Try `cephadm registry-login --registry-url <url> --registry-username <username> --registry-password <password>` or supply login info via a json file with `cephadm registry-login --registry-json <file>`' + logger.debug(f'Pulling image for `command_adopt` failed: {err_str}') + raise Error(err_str) + + (daemon_type, daemon_id) = ctx.name.split('.', 1) + + # legacy check + if ctx.style != 'legacy': + raise Error('adoption of style %s not implemented' % ctx.style) + + # lock + fsid = get_legacy_daemon_fsid(ctx, + ctx.cluster, + daemon_type, + daemon_id, + legacy_dir=ctx.legacy_dir) + if not fsid: + raise Error('could not detect legacy fsid; set fsid in ceph.conf') + lock = FileLock(ctx, fsid) + lock.acquire() + + # call correct adoption + if daemon_type in Ceph.daemons: + command_adopt_ceph(ctx, daemon_type, daemon_id, fsid) + elif daemon_type == 'prometheus': + command_adopt_prometheus(ctx, daemon_id, fsid) + elif daemon_type == 'grafana': + command_adopt_grafana(ctx, daemon_id, fsid) + elif daemon_type == 'node-exporter': + raise Error('adoption of node-exporter not implemented') + elif daemon_type == 'alertmanager': + command_adopt_alertmanager(ctx, daemon_id, fsid) + else: + raise Error('daemon type %s not recognized' % daemon_type) + + +class AdoptOsd(object): + def __init__(self, ctx, osd_data_dir, osd_id): + # type: (CephadmContext, str, str) -> None + self.ctx = ctx + self.osd_data_dir = osd_data_dir + self.osd_id = osd_id + + def check_online_osd(self): + # type: () -> Tuple[Optional[str], Optional[str]] + + osd_fsid, osd_type = None, None + + path = os.path.join(self.osd_data_dir, 'fsid') + try: + with open(path, 'r') as f: + osd_fsid = f.read().strip() + logger.info('Found online OSD at %s' % path) + except IOError: + logger.info('Unable to read OSD fsid from %s' % path) + if os.path.exists(os.path.join(self.osd_data_dir, 'type')): + with open(os.path.join(self.osd_data_dir, 'type')) as f: + osd_type = f.read().strip() + else: + logger.info('"type" file missing for OSD data dir') + + return osd_fsid, osd_type + + def check_offline_lvm_osd(self): + # type: () -> Tuple[Optional[str], Optional[str]] + osd_fsid, osd_type = None, None + + c = get_ceph_volume_container( + self.ctx, + args=['lvm', 'list', '--format=json'], + ) + out, err, code = call_throws(self.ctx, c.run_cmd()) + if not code: + try: + js = json.loads(out) + if self.osd_id in js: + logger.info('Found offline LVM OSD {}'.format(self.osd_id)) + osd_fsid = js[self.osd_id][0]['tags']['ceph.osd_fsid'] + for device in js[self.osd_id]: + if device['tags']['ceph.type'] == 'block': + osd_type = 'bluestore' + break + if device['tags']['ceph.type'] == 'data': + osd_type = 'filestore' + break + except ValueError as e: + logger.info('Invalid JSON in ceph-volume lvm list: {}'.format(e)) + + return osd_fsid, osd_type + + def check_offline_simple_osd(self): + # type: () -> Tuple[Optional[str], Optional[str]] + osd_fsid, osd_type = None, None + + osd_file = glob('/etc/ceph/osd/{}-[a-f0-9-]*.json'.format(self.osd_id)) + if len(osd_file) == 1: + with open(osd_file[0], 'r') as f: + try: + js = json.loads(f.read()) + logger.info('Found offline simple OSD {}'.format(self.osd_id)) + osd_fsid = js['fsid'] + osd_type = js['type'] + if osd_type != 'filestore': + # need this to be mounted for the adopt to work, as it + # needs to move files from this directory + call_throws(self.ctx, ['mount', js['data']['path'], self.osd_data_dir]) + except ValueError as e: + logger.info('Invalid JSON in {}: {}'.format(osd_file, e)) + + return osd_fsid, osd_type + + def change_cluster_name(self) -> None: + logger.info('Attempting to convert osd cluster name to ceph . . .') + c = get_ceph_volume_container( + self.ctx, + args=['lvm', 'list', '{}'.format(self.osd_id), '--format=json'], + ) + out, err, code = call_throws(self.ctx, c.run_cmd()) + if code: + raise Exception(f'Failed to get list of LVs: {err}\nceph-volume failed with rc {code}') + try: + js = json.loads(out) + if not js: + raise RuntimeError(f'Failed to find osd.{self.osd_id}') + device: Optional[Dict[Any, Any]] = None + for d in js[self.osd_id]: + if d['type'] == 'block': + device = d + break + if not device: + raise RuntimeError(f'Failed to find block device for osd.{self.osd_id}') + vg = device['vg_name'] + out, err, code = call_throws(self.ctx, ['lvchange', '--deltag', f'ceph.cluster_name={self.ctx.cluster}', vg]) + if code: + raise RuntimeError(f"Can't delete tag ceph.cluster_name={self.ctx.cluster} on osd.{self.osd_id}.\nlvchange failed with rc {code}") + out, err, code = call_throws(self.ctx, ['lvchange', '--addtag', 'ceph.cluster_name=ceph', vg]) + if code: + raise RuntimeError(f"Can't add tag ceph.cluster_name=ceph on osd.{self.osd_id}.\nlvchange failed with rc {code}") + logger.info('Successfully converted osd cluster name') + except (Exception, RuntimeError) as e: + logger.info(f'Failed to convert osd cluster name: {e}') + + +def command_adopt_ceph(ctx, daemon_type, daemon_id, fsid): + # type: (CephadmContext, str, str, str) -> None + + (uid, gid) = extract_uid_gid(ctx) + + data_dir_src = ('/var/lib/ceph/%s/%s-%s' % + (daemon_type, ctx.cluster, daemon_id)) + data_dir_src = os.path.abspath(ctx.legacy_dir + data_dir_src) + + if not os.path.exists(data_dir_src): + raise Error("{}.{} data directory '{}' does not exist. " + 'Incorrect ID specified, or daemon already adopted?'.format( + daemon_type, daemon_id, data_dir_src)) + + osd_fsid = None + if daemon_type == 'osd': + adopt_osd = AdoptOsd(ctx, data_dir_src, daemon_id) + osd_fsid, osd_type = adopt_osd.check_online_osd() + if not osd_fsid: + osd_fsid, osd_type = adopt_osd.check_offline_lvm_osd() + if not osd_fsid: + osd_fsid, osd_type = adopt_osd.check_offline_simple_osd() + if not osd_fsid: + raise Error('Unable to find OSD {}'.format(daemon_id)) + elif ctx.cluster != 'ceph': + adopt_osd.change_cluster_name() + logger.info('objectstore_type is %s' % osd_type) + assert osd_type + if osd_type == 'filestore': + raise Error('FileStore is not supported by cephadm') + + # NOTE: implicit assumption here that the units correspond to the + # cluster we are adopting based on the /etc/{defaults,sysconfig}/ceph + # CLUSTER field. + unit_name = 'ceph-%s@%s' % (daemon_type, daemon_id) + (enabled, state, _) = check_unit(ctx, unit_name) + if state == 'running': + logger.info('Stopping old systemd unit %s...' % unit_name) + call_throws(ctx, ['systemctl', 'stop', unit_name]) + if enabled: + logger.info('Disabling old systemd unit %s...' % unit_name) + call_throws(ctx, ['systemctl', 'disable', unit_name]) + + # data + logger.info('Moving data...') + data_dir_dst = make_data_dir(ctx, fsid, daemon_type, daemon_id, + uid=uid, gid=gid) + move_files(ctx, glob(os.path.join(data_dir_src, '*')), + data_dir_dst, + uid=uid, gid=gid) + logger.debug('Remove dir `%s`' % (data_dir_src)) + if os.path.ismount(data_dir_src): + call_throws(ctx, ['umount', data_dir_src]) + os.rmdir(data_dir_src) + + logger.info('Chowning content...') + call_throws(ctx, ['chown', '-c', '-R', '%d.%d' % (uid, gid), data_dir_dst]) + + if daemon_type == 'mon': + # rename *.ldb -> *.sst, in case they are coming from ubuntu + store = os.path.join(data_dir_dst, 'store.db') + num_renamed = 0 + if os.path.exists(store): + for oldf in os.listdir(store): + if oldf.endswith('.ldb'): + newf = oldf.replace('.ldb', '.sst') + oldp = os.path.join(store, oldf) + newp = os.path.join(store, newf) + logger.debug('Renaming %s -> %s' % (oldp, newp)) + os.rename(oldp, newp) + if num_renamed: + logger.info('Renamed %d leveldb *.ldb files to *.sst', + num_renamed) + if daemon_type == 'osd': + for n in ['block', 'block.db', 'block.wal']: + p = os.path.join(data_dir_dst, n) + if os.path.exists(p): + logger.info('Chowning %s...' % p) + os.chown(p, uid, gid) + # disable the ceph-volume 'simple' mode files on the host + simple_fn = os.path.join('/etc/ceph/osd', + '%s-%s.json' % (daemon_id, osd_fsid)) + if os.path.exists(simple_fn): + new_fn = simple_fn + '.adopted-by-cephadm' + logger.info('Renaming %s -> %s', simple_fn, new_fn) + os.rename(simple_fn, new_fn) + logger.info('Disabling host unit ceph-volume@ simple unit...') + call(ctx, ['systemctl', 'disable', + 'ceph-volume@simple-%s-%s.service' % (daemon_id, osd_fsid)]) + else: + # assume this is an 'lvm' c-v for now, but don't error + # out if it's not. + logger.info('Disabling host unit ceph-volume@ lvm unit...') + call(ctx, ['systemctl', 'disable', + 'ceph-volume@lvm-%s-%s.service' % (daemon_id, osd_fsid)]) + + # config + config_src = '/etc/ceph/%s.conf' % (ctx.cluster) + config_src = os.path.abspath(ctx.legacy_dir + config_src) + config_dst = os.path.join(data_dir_dst, 'config') + copy_files(ctx, [config_src], config_dst, uid=uid, gid=gid) + + # logs + logger.info('Moving logs...') + log_dir_src = ('/var/log/ceph/%s-%s.%s.log*' % + (ctx.cluster, daemon_type, daemon_id)) + log_dir_src = os.path.abspath(ctx.legacy_dir + log_dir_src) + log_dir_dst = make_log_dir(ctx, fsid, uid=uid, gid=gid) + move_files(ctx, glob(log_dir_src), + log_dir_dst, + uid=uid, gid=gid) + + logger.info('Creating new units...') + make_var_run(ctx, fsid, uid, gid) + c = get_container(ctx, fsid, daemon_type, daemon_id) + deploy_daemon_units(ctx, fsid, uid, gid, daemon_type, daemon_id, c, + enable=True, # unconditionally enable the new unit + start=(state == 'running' or ctx.force_start), + osd_fsid=osd_fsid) + update_firewalld(ctx, daemon_type) + + +def command_adopt_prometheus(ctx, daemon_id, fsid): + # type: (CephadmContext, str, str) -> None + daemon_type = 'prometheus' + (uid, gid) = extract_uid_gid_monitoring(ctx, daemon_type) + + _stop_and_disable(ctx, 'prometheus') + + data_dir_dst = make_data_dir(ctx, fsid, daemon_type, daemon_id, + uid=uid, gid=gid) + + # config + config_src = '/etc/prometheus/prometheus.yml' + config_src = os.path.abspath(ctx.legacy_dir + config_src) + config_dst = os.path.join(data_dir_dst, 'etc/prometheus') + makedirs(config_dst, uid, gid, 0o755) + copy_files(ctx, [config_src], config_dst, uid=uid, gid=gid) + + # data + data_src = '/var/lib/prometheus/metrics/' + data_src = os.path.abspath(ctx.legacy_dir + data_src) + data_dst = os.path.join(data_dir_dst, 'data') + copy_tree(ctx, [data_src], data_dst, uid=uid, gid=gid) + + make_var_run(ctx, fsid, uid, gid) + c = get_container(ctx, fsid, daemon_type, daemon_id) + deploy_daemon(ctx, fsid, daemon_type, daemon_id, c, uid, gid) + update_firewalld(ctx, daemon_type) + + +def command_adopt_grafana(ctx, daemon_id, fsid): + # type: (CephadmContext, str, str) -> None + + daemon_type = 'grafana' + (uid, gid) = extract_uid_gid_monitoring(ctx, daemon_type) + + _stop_and_disable(ctx, 'grafana-server') + + data_dir_dst = make_data_dir(ctx, fsid, daemon_type, daemon_id, + uid=uid, gid=gid) + + # config + config_src = '/etc/grafana/grafana.ini' + config_src = os.path.abspath(ctx.legacy_dir + config_src) + config_dst = os.path.join(data_dir_dst, 'etc/grafana') + makedirs(config_dst, uid, gid, 0o755) + copy_files(ctx, [config_src], config_dst, uid=uid, gid=gid) + + prov_src = '/etc/grafana/provisioning/' + prov_src = os.path.abspath(ctx.legacy_dir + prov_src) + prov_dst = os.path.join(data_dir_dst, 'etc/grafana') + copy_tree(ctx, [prov_src], prov_dst, uid=uid, gid=gid) + + # cert + cert = '/etc/grafana/grafana.crt' + key = '/etc/grafana/grafana.key' + if os.path.exists(cert) and os.path.exists(key): + cert_src = '/etc/grafana/grafana.crt' + cert_src = os.path.abspath(ctx.legacy_dir + cert_src) + makedirs(os.path.join(data_dir_dst, 'etc/grafana/certs'), uid, gid, 0o755) + cert_dst = os.path.join(data_dir_dst, 'etc/grafana/certs/cert_file') + copy_files(ctx, [cert_src], cert_dst, uid=uid, gid=gid) + + key_src = '/etc/grafana/grafana.key' + key_src = os.path.abspath(ctx.legacy_dir + key_src) + key_dst = os.path.join(data_dir_dst, 'etc/grafana/certs/cert_key') + copy_files(ctx, [key_src], key_dst, uid=uid, gid=gid) + + _adjust_grafana_ini(os.path.join(config_dst, 'grafana.ini')) + else: + logger.debug('Skipping ssl, missing cert {} or key {}'.format(cert, key)) + + # data - possible custom dashboards/plugins + data_src = '/var/lib/grafana/' + data_src = os.path.abspath(ctx.legacy_dir + data_src) + data_dst = os.path.join(data_dir_dst, 'data') + copy_tree(ctx, [data_src], data_dst, uid=uid, gid=gid) + + make_var_run(ctx, fsid, uid, gid) + c = get_container(ctx, fsid, daemon_type, daemon_id) + deploy_daemon(ctx, fsid, daemon_type, daemon_id, c, uid, gid) + update_firewalld(ctx, daemon_type) + + +def command_adopt_alertmanager(ctx, daemon_id, fsid): + # type: (CephadmContext, str, str) -> None + + daemon_type = 'alertmanager' + (uid, gid) = extract_uid_gid_monitoring(ctx, daemon_type) + + _stop_and_disable(ctx, 'prometheus-alertmanager') + + data_dir_dst = make_data_dir(ctx, fsid, daemon_type, daemon_id, + uid=uid, gid=gid) + + # config + config_src = '/etc/prometheus/alertmanager.yml' + config_src = os.path.abspath(ctx.legacy_dir + config_src) + config_dst = os.path.join(data_dir_dst, 'etc/alertmanager') + makedirs(config_dst, uid, gid, 0o755) + copy_files(ctx, [config_src], config_dst, uid=uid, gid=gid) + + # data + data_src = '/var/lib/prometheus/alertmanager/' + data_src = os.path.abspath(ctx.legacy_dir + data_src) + data_dst = os.path.join(data_dir_dst, 'etc/alertmanager/data') + copy_tree(ctx, [data_src], data_dst, uid=uid, gid=gid) + + make_var_run(ctx, fsid, uid, gid) + c = get_container(ctx, fsid, daemon_type, daemon_id) + deploy_daemon(ctx, fsid, daemon_type, daemon_id, c, uid, gid) + update_firewalld(ctx, daemon_type) + + +def _adjust_grafana_ini(filename): + # type: (str) -> None + + # Update cert_file, cert_key pathnames in server section + # ConfigParser does not preserve comments + try: + with open(filename, 'r') as grafana_ini: + lines = grafana_ini.readlines() + with open('{}.new'.format(filename), 'w') as grafana_ini: + server_section = False + for line in lines: + if line.startswith('['): + server_section = False + if line.startswith('[server]'): + server_section = True + if server_section: + line = re.sub(r'^cert_file.*', + 'cert_file = /etc/grafana/certs/cert_file', line) + line = re.sub(r'^cert_key.*', + 'cert_key = /etc/grafana/certs/cert_key', line) + grafana_ini.write(line) + os.rename('{}.new'.format(filename), filename) + except OSError as err: + raise Error('Cannot update {}: {}'.format(filename, err)) + + +def _stop_and_disable(ctx, unit_name): + # type: (CephadmContext, str) -> None + + (enabled, state, _) = check_unit(ctx, unit_name) + if state == 'running': + logger.info('Stopping old systemd unit %s...' % unit_name) + call_throws(ctx, ['systemctl', 'stop', unit_name]) + if enabled: + logger.info('Disabling old systemd unit %s...' % unit_name) + call_throws(ctx, ['systemctl', 'disable', unit_name]) + +################################## + + +def command_rm_daemon(ctx): + # type: (CephadmContext) -> None + lock = FileLock(ctx, ctx.fsid) + lock.acquire() + + (daemon_type, daemon_id) = ctx.name.split('.', 1) + unit_name = get_unit_name_by_daemon_name(ctx, ctx.fsid, ctx.name) + + if daemon_type in ['mon', 'osd'] and not ctx.force: + raise Error('must pass --force to proceed: ' + 'this command may destroy precious data!') + + call(ctx, ['systemctl', 'stop', unit_name], + verbosity=CallVerbosity.DEBUG) + call(ctx, ['systemctl', 'reset-failed', unit_name], + verbosity=CallVerbosity.DEBUG) + call(ctx, ['systemctl', 'disable', unit_name], + verbosity=CallVerbosity.DEBUG) + data_dir = get_data_dir(ctx.fsid, ctx.data_dir, daemon_type, daemon_id) + if daemon_type in ['mon', 'osd', 'prometheus'] and \ + not ctx.force_delete_data: + # rename it out of the way -- do not delete + backup_dir = os.path.join(ctx.data_dir, ctx.fsid, 'removed') + if not os.path.exists(backup_dir): + makedirs(backup_dir, 0, 0, DATA_DIR_MODE) + dirname = '%s.%s_%s' % (daemon_type, daemon_id, + datetime.datetime.utcnow().strftime(DATEFMT)) + os.rename(data_dir, + os.path.join(backup_dir, dirname)) + else: + if daemon_type == CephadmDaemon.daemon_type: + CephadmDaemon.uninstall(ctx, ctx.fsid, daemon_type, daemon_id) + call_throws(ctx, ['rm', '-rf', data_dir]) + + if 'tcp_ports' in ctx and ctx.tcp_ports is not None: + ports: List[int] = [int(p) for p in ctx.tcp_ports.split()] + try: + fw = Firewalld(ctx) + fw.close_ports(ports) + fw.apply_rules() + except RuntimeError as e: + # in case we cannot close the ports we will remove + # the daemon but keep them open. + logger.warning(f' Error when trying to close ports: {e}') + + +################################## + + +def _zap(ctx: CephadmContext, what: str) -> None: + mounts = get_container_mounts(ctx, ctx.fsid, 'clusterless-ceph-volume', None) + c = get_ceph_volume_container(ctx, + args=['lvm', 'zap', '--destroy', what], + volume_mounts=mounts, + envs=ctx.env) + logger.info(f'Zapping {what}...') + out, err, code = call_throws(ctx, c.run_cmd()) + + +@infer_image +def _zap_osds(ctx: CephadmContext) -> None: + # assume fsid lock already held + + # list + mounts = get_container_mounts(ctx, ctx.fsid, 'clusterless-ceph-volume', None) + c = get_ceph_volume_container(ctx, + args=['inventory', '--format', 'json'], + volume_mounts=mounts, + envs=ctx.env) + out, err, code = call_throws(ctx, c.run_cmd()) + if code: + raise Error('failed to list osd inventory') + try: + ls = json.loads(out) + except ValueError as e: + raise Error(f'Invalid JSON in ceph-volume inventory: {e}') + + for i in ls: + matches = [lv.get('cluster_fsid') == ctx.fsid and i.get('ceph_device') for lv in i.get('lvs', [])] + if any(matches) and all(matches): + _zap(ctx, i.get('path')) + elif any(matches): + lv_names = [lv['name'] for lv in i.get('lvs', [])] + # TODO: we need to map the lv_names back to device paths (the vg + # id isn't part of the output here!) + logger.warning(f'Not zapping LVs (not implemented): {lv_names}') + + +def command_zap_osds(ctx: CephadmContext) -> None: + if not ctx.force: + raise Error('must pass --force to proceed: ' + 'this command may destroy precious data!') + + lock = FileLock(ctx, ctx.fsid) + lock.acquire() + + _zap_osds(ctx) + +################################## + + +def get_ceph_cluster_count(ctx: CephadmContext) -> int: + return len([c for c in os.listdir(ctx.data_dir) if is_fsid(c)]) + + +def command_rm_cluster(ctx): + # type: (CephadmContext) -> None + if not ctx.force: + raise Error('must pass --force to proceed: ' + 'this command may destroy precious data!') + + lock = FileLock(ctx, ctx.fsid) + lock.acquire() + + def disable_systemd_service(unit_name: str) -> None: + call(ctx, ['systemctl', 'stop', unit_name], + verbosity=CallVerbosity.DEBUG) + call(ctx, ['systemctl', 'reset-failed', unit_name], + verbosity=CallVerbosity.DEBUG) + call(ctx, ['systemctl', 'disable', unit_name], + verbosity=CallVerbosity.DEBUG) + + # stop + disable individual daemon units + for d in list_daemons(ctx, detail=False): + if d['fsid'] != ctx.fsid: + continue + if d['style'] != 'cephadm:v1': + continue + disable_systemd_service(get_unit_name(ctx.fsid, d['name'])) + + # cluster units + for unit_name in ['ceph-%s.target' % ctx.fsid]: + disable_systemd_service(unit_name) + + slice_name = 'system-ceph\\x2d{}.slice'.format(ctx.fsid.replace('-', '\\x2d')) + call(ctx, ['systemctl', 'stop', slice_name], + verbosity=CallVerbosity.DEBUG) + + # osds? + if ctx.zap_osds: + _zap_osds(ctx) + + # rm units + call_throws(ctx, ['rm', '-f', ctx.unit_dir + + '/ceph-%s@.service' % ctx.fsid]) + call_throws(ctx, ['rm', '-f', ctx.unit_dir + + '/ceph-%s.target' % ctx.fsid]) + call_throws(ctx, ['rm', '-rf', + ctx.unit_dir + '/ceph-%s.target.wants' % ctx.fsid]) + # rm data + call_throws(ctx, ['rm', '-rf', ctx.data_dir + '/' + ctx.fsid]) + + if not ctx.keep_logs: + # rm logs + call_throws(ctx, ['rm', '-rf', ctx.log_dir + '/' + ctx.fsid]) + call_throws(ctx, ['rm', '-rf', ctx.log_dir + + '/*.wants/ceph-%s@*' % ctx.fsid]) + + # rm logrotate config + call_throws(ctx, ['rm', '-f', ctx.logrotate_dir + '/ceph-%s' % ctx.fsid]) + + # if last cluster on host remove shared files + if get_ceph_cluster_count(ctx) == 0: + disable_systemd_service('ceph.target') + + # rm shared ceph target files + call_throws(ctx, ['rm', '-f', ctx.unit_dir + '/multi-user.target.wants/ceph.target']) + call_throws(ctx, ['rm', '-f', ctx.unit_dir + '/ceph.target']) + + # rm cephadm logrotate config + call_throws(ctx, ['rm', '-f', ctx.logrotate_dir + '/cephadm']) + + if not ctx.keep_logs: + # remove all cephadm logs + for fname in glob(f'{ctx.log_dir}/cephadm.log*'): + os.remove(fname) + + # rm sysctl settings + sysctl_dirs: List[Path] = [Path(ctx.sysctl_dir), Path('/usr/lib/sysctl.d')] + + for sysctl_dir in sysctl_dirs: + for p in sysctl_dir.glob(f'90-ceph-{ctx.fsid}-*.conf'): + p.unlink() + + # cleanup remaining ceph directories + ceph_dirs = [f'/run/ceph/{ctx.fsid}', f'/tmp/var/lib/ceph/{ctx.fsid}', f'/var/run/ceph/{ctx.fsid}'] + for dd in ceph_dirs: + shutil.rmtree(dd, ignore_errors=True) + + # clean up config, keyring, and pub key files + files = ['/etc/ceph/ceph.conf', '/etc/ceph/ceph.pub', '/etc/ceph/ceph.client.admin.keyring'] + if os.path.exists(files[0]): + valid_fsid = False + with open(files[0]) as f: + if ctx.fsid in f.read(): + valid_fsid = True + if valid_fsid: + # rm configuration files on /etc/ceph + for n in range(0, len(files)): + if os.path.exists(files[n]): + os.remove(files[n]) + +################################## + + +def check_time_sync(ctx, enabler=None): + # type: (CephadmContext, Optional[Packager]) -> bool + units = [ + 'chrony.service', # 18.04 (at least) + 'chronyd.service', # el / opensuse + 'systemd-timesyncd.service', + 'ntpd.service', # el7 (at least) + 'ntp.service', # 18.04 (at least) + 'ntpsec.service', # 20.04 (at least) / buster + 'openntpd.service', # ubuntu / debian + ] + if not check_units(ctx, units, enabler): + logger.warning('No time sync service is running; checked for %s' % units) + return False + return True + + +def command_check_host(ctx: CephadmContext) -> None: + errors = [] + commands = ['systemctl', 'lvcreate'] + + try: + engine = check_container_engine(ctx) + logger.info(f'{engine} is present') + except Error as e: + errors.append(str(e)) + + for command in commands: + try: + find_program(command) + logger.info('%s is present' % command) + except ValueError: + errors.append('%s binary does not appear to be installed' % command) + + # check for configured+running chronyd or ntp + if not check_time_sync(ctx): + errors.append('No time synchronization is active') + + if 'expect_hostname' in ctx and ctx.expect_hostname: + if get_hostname().lower() != ctx.expect_hostname.lower(): + errors.append('hostname "%s" does not match expected hostname "%s"' % ( + get_hostname(), ctx.expect_hostname)) + else: + logger.info('Hostname "%s" matches what is expected.', + ctx.expect_hostname) + + if errors: + raise Error('\nERROR: '.join(errors)) + + logger.info('Host looks OK') + +################################## + + +def command_prepare_host(ctx: CephadmContext) -> None: + logger.info('Verifying podman|docker is present...') + pkg = None + try: + check_container_engine(ctx) + except Error as e: + logger.warning(str(e)) + if not pkg: + pkg = create_packager(ctx) + pkg.install_podman() + + logger.info('Verifying lvm2 is present...') + if not find_executable('lvcreate'): + if not pkg: + pkg = create_packager(ctx) + pkg.install(['lvm2']) + + logger.info('Verifying time synchronization is in place...') + if not check_time_sync(ctx): + if not pkg: + pkg = create_packager(ctx) + pkg.install(['chrony']) + # check again, and this time try to enable + # the service + check_time_sync(ctx, enabler=pkg) + + if 'expect_hostname' in ctx and ctx.expect_hostname and ctx.expect_hostname != get_hostname(): + logger.warning('Adjusting hostname from %s -> %s...' % (get_hostname(), ctx.expect_hostname)) + call_throws(ctx, ['hostname', ctx.expect_hostname]) + with open('/etc/hostname', 'w') as f: + f.write(ctx.expect_hostname + '\n') + + logger.info('Repeating the final host check...') + command_check_host(ctx) + +################################## + + +class CustomValidation(argparse.Action): + + def _check_name(self, values: str) -> None: + try: + (daemon_type, daemon_id) = values.split('.', 1) + except ValueError: + raise argparse.ArgumentError(self, + 'must be of the format <type>.<id>. For example, osd.1 or prometheus.myhost.com') + + daemons = get_supported_daemons() + if daemon_type not in daemons: + raise argparse.ArgumentError(self, + 'name must declare the type of daemon e.g. ' + '{}'.format(', '.join(daemons))) + + def __call__(self, parser: argparse.ArgumentParser, namespace: argparse.Namespace, values: Union[str, Sequence[Any], None], + option_string: Optional[str] = None) -> None: + assert isinstance(values, str) + if self.dest == 'name': + self._check_name(values) + setattr(namespace, self.dest, values) + elif self.dest == 'exporter_config': + cfg = get_parm(values) + # run the class' validate method, and convert to an argparse error + # if problems are found + try: + CephadmDaemon.validate_config(cfg) + except Error as e: + raise argparse.ArgumentError(self, + str(e)) + setattr(namespace, self.dest, cfg) + +################################## + + +def get_distro(): + # type: () -> Tuple[Optional[str], Optional[str], Optional[str]] + distro = None + distro_version = None + distro_codename = None + with open('/etc/os-release', 'r') as f: + for line in f.readlines(): + line = line.strip() + if '=' not in line or line.startswith('#'): + continue + (var, val) = line.split('=', 1) + if val[0] == '"' and val[-1] == '"': + val = val[1:-1] + if var == 'ID': + distro = val.lower() + elif var == 'VERSION_ID': + distro_version = val.lower() + elif var == 'VERSION_CODENAME': + distro_codename = val.lower() + return distro, distro_version, distro_codename + + +class Packager(object): + def __init__(self, ctx: CephadmContext, + stable: Optional[str] = None, version: Optional[str] = None, + branch: Optional[str] = None, commit: Optional[str] = None): + assert \ + (stable and not version and not branch and not commit) or \ + (not stable and version and not branch and not commit) or \ + (not stable and not version and branch) or \ + (not stable and not version and not branch and not commit) + self.ctx = ctx + self.stable = stable + self.version = version + self.branch = branch + self.commit = commit + + def validate(self) -> None: + """Validate parameters before writing any state to disk.""" + pass + + def add_repo(self) -> None: + raise NotImplementedError + + def rm_repo(self) -> None: + raise NotImplementedError + + def install(self, ls: List[str]) -> None: + raise NotImplementedError + + def install_podman(self) -> None: + raise NotImplementedError + + def query_shaman(self, distro: str, distro_version: Any, branch: Optional[str], commit: Optional[str]) -> str: + # query shaman + logger.info('Fetching repo metadata from shaman and chacra...') + shaman_url = 'https://shaman.ceph.com/api/repos/ceph/{branch}/{sha1}/{distro}/{distro_version}/repo/?arch={arch}'.format( + distro=distro, + distro_version=distro_version, + branch=branch, + sha1=commit or 'latest', + arch=get_arch() + ) + try: + shaman_response = urlopen(shaman_url) + except HTTPError as err: + logger.error('repository not found in shaman (might not be available yet)') + raise Error('%s, failed to fetch %s' % (err, shaman_url)) + chacra_url = '' + try: + chacra_url = shaman_response.geturl() + chacra_response = urlopen(chacra_url) + except HTTPError as err: + logger.error('repository not found in chacra (might not be available yet)') + raise Error('%s, failed to fetch %s' % (err, chacra_url)) + return chacra_response.read().decode('utf-8') + + def repo_gpgkey(self) -> Tuple[str, str]: + if self.ctx.gpg_url: + return self.ctx.gpg_url, 'manual' + if self.stable or self.version: + return 'https://download.ceph.com/keys/release.gpg', 'release' + else: + return 'https://download.ceph.com/keys/autobuild.gpg', 'autobuild' + + def enable_service(self, service: str) -> None: + """ + Start and enable the service (typically using systemd). + """ + call_throws(self.ctx, ['systemctl', 'enable', '--now', service]) + + +class Apt(Packager): + DISTRO_NAMES = { + 'ubuntu': 'ubuntu', + 'debian': 'debian', + } + + def __init__(self, ctx: CephadmContext, + stable: Optional[str], version: Optional[str], branch: Optional[str], commit: Optional[str], + distro: Optional[str], distro_version: Optional[str], distro_codename: Optional[str]) -> None: + super(Apt, self).__init__(ctx, stable=stable, version=version, + branch=branch, commit=commit) + assert distro + self.ctx = ctx + self.distro = self.DISTRO_NAMES[distro] + self.distro_codename = distro_codename + self.distro_version = distro_version + + def repo_path(self) -> str: + return '/etc/apt/sources.list.d/ceph.list' + + def add_repo(self) -> None: + + url, name = self.repo_gpgkey() + logger.info('Installing repo GPG key from %s...' % url) + try: + response = urlopen(url) + except HTTPError as err: + logger.error('failed to fetch GPG repo key from %s: %s' % ( + url, err)) + raise Error('failed to fetch GPG key') + key = response.read() + with open('/etc/apt/trusted.gpg.d/ceph.%s.gpg' % name, 'wb') as f: + f.write(key) + + if self.version: + content = 'deb %s/debian-%s/ %s main\n' % ( + self.ctx.repo_url, self.version, self.distro_codename) + elif self.stable: + content = 'deb %s/debian-%s/ %s main\n' % ( + self.ctx.repo_url, self.stable, self.distro_codename) + else: + content = self.query_shaman(self.distro, self.distro_codename, self.branch, + self.commit) + + logger.info('Installing repo file at %s...' % self.repo_path()) + with open(self.repo_path(), 'w') as f: + f.write(content) + + self.update() + + def rm_repo(self) -> None: + for name in ['autobuild', 'release', 'manual']: + p = '/etc/apt/trusted.gpg.d/ceph.%s.gpg' % name + if os.path.exists(p): + logger.info('Removing repo GPG key %s...' % p) + os.unlink(p) + if os.path.exists(self.repo_path()): + logger.info('Removing repo at %s...' % self.repo_path()) + os.unlink(self.repo_path()) + + if self.distro == 'ubuntu': + self.rm_kubic_repo() + + def install(self, ls: List[str]) -> None: + logger.info('Installing packages %s...' % ls) + call_throws(self.ctx, ['apt-get', 'install', '-y'] + ls) + + def update(self) -> None: + logger.info('Updating package list...') + call_throws(self.ctx, ['apt-get', 'update']) + + def install_podman(self) -> None: + if self.distro == 'ubuntu': + logger.info('Setting up repo for podman...') + self.add_kubic_repo() + self.update() + + logger.info('Attempting podman install...') + try: + self.install(['podman']) + except Error: + logger.info('Podman did not work. Falling back to docker...') + self.install(['docker.io']) + + def kubic_repo_url(self) -> str: + return 'https://download.opensuse.org/repositories/devel:/kubic:/' \ + 'libcontainers:/stable/xUbuntu_%s/' % self.distro_version + + def kubic_repo_path(self) -> str: + return '/etc/apt/sources.list.d/devel:kubic:libcontainers:stable.list' + + def kubric_repo_gpgkey_url(self) -> str: + return '%s/Release.key' % self.kubic_repo_url() + + def kubric_repo_gpgkey_path(self) -> str: + return '/etc/apt/trusted.gpg.d/kubic.release.gpg' + + def add_kubic_repo(self) -> None: + url = self.kubric_repo_gpgkey_url() + logger.info('Installing repo GPG key from %s...' % url) + try: + response = urlopen(url) + except HTTPError as err: + logger.error('failed to fetch GPG repo key from %s: %s' % ( + url, err)) + raise Error('failed to fetch GPG key') + key = response.read().decode('utf-8') + tmp_key = write_tmp(key, 0, 0) + keyring = self.kubric_repo_gpgkey_path() + call_throws(self.ctx, ['apt-key', '--keyring', keyring, 'add', tmp_key.name]) + + logger.info('Installing repo file at %s...' % self.kubic_repo_path()) + content = 'deb %s /\n' % self.kubic_repo_url() + with open(self.kubic_repo_path(), 'w') as f: + f.write(content) + + def rm_kubic_repo(self) -> None: + keyring = self.kubric_repo_gpgkey_path() + if os.path.exists(keyring): + logger.info('Removing repo GPG key %s...' % keyring) + os.unlink(keyring) + + p = self.kubic_repo_path() + if os.path.exists(p): + logger.info('Removing repo at %s...' % p) + os.unlink(p) + + +class YumDnf(Packager): + DISTRO_NAMES = { + 'centos': ('centos', 'el'), + 'rhel': ('centos', 'el'), + 'scientific': ('centos', 'el'), + 'rocky': ('centos', 'el'), + 'almalinux': ('centos', 'el'), + 'ol': ('centos', 'el'), + 'fedora': ('fedora', 'fc'), + 'mariner': ('mariner', 'cm'), + } + + def __init__(self, ctx: CephadmContext, + stable: Optional[str], version: Optional[str], branch: Optional[str], commit: Optional[str], + distro: Optional[str], distro_version: Optional[str]) -> None: + super(YumDnf, self).__init__(ctx, stable=stable, version=version, + branch=branch, commit=commit) + assert distro + assert distro_version + self.ctx = ctx + self.major = int(distro_version.split('.')[0]) + self.distro_normalized = self.DISTRO_NAMES[distro][0] + self.distro_code = self.DISTRO_NAMES[distro][1] + str(self.major) + if (self.distro_code == 'fc' and self.major >= 30) or \ + (self.distro_code == 'el' and self.major >= 8): + self.tool = 'dnf' + elif (self.distro_code == 'cm'): + self.tool = 'tdnf' + else: + self.tool = 'yum' + + def custom_repo(self, **kw: Any) -> str: + """ + Repo files need special care in that a whole line should not be present + if there is no value for it. Because we were using `format()` we could + not conditionally add a line for a repo file. So the end result would + contain a key with a missing value (say if we were passing `None`). + + For example, it could look like:: + + [ceph repo] + name= ceph repo + proxy= + gpgcheck= + + Which breaks. This function allows us to conditionally add lines, + preserving an order and be more careful. + + Previously, and for historical purposes, this is how the template used + to look:: + + custom_repo = + [{repo_name}] + name={name} + baseurl={baseurl} + enabled={enabled} + gpgcheck={gpgcheck} + type={_type} + gpgkey={gpgkey} + proxy={proxy} + + """ + lines = [] + + # by using tuples (vs a dict) we preserve the order of what we want to + # return, like starting with a [repo name] + tmpl = ( + ('reponame', '[%s]'), + ('name', 'name=%s'), + ('baseurl', 'baseurl=%s'), + ('enabled', 'enabled=%s'), + ('gpgcheck', 'gpgcheck=%s'), + ('_type', 'type=%s'), + ('gpgkey', 'gpgkey=%s'), + ('proxy', 'proxy=%s'), + ('priority', 'priority=%s'), + ) + + for line in tmpl: + tmpl_key, tmpl_value = line # key values from tmpl + + # ensure that there is an actual value (not None nor empty string) + if tmpl_key in kw and kw.get(tmpl_key) not in (None, ''): + lines.append(tmpl_value % kw.get(tmpl_key)) + + return '\n'.join(lines) + + def repo_path(self) -> str: + return '/etc/yum.repos.d/ceph.repo' + + def repo_baseurl(self) -> str: + assert self.stable or self.version + if self.version: + return '%s/rpm-%s/%s' % (self.ctx.repo_url, self.version, + self.distro_code) + else: + return '%s/rpm-%s/%s' % (self.ctx.repo_url, self.stable, + self.distro_code) + + def validate(self) -> None: + if self.distro_code.startswith('fc'): + raise Error('Ceph team does not build Fedora specific packages and therefore cannot add repos for this distro') + if self.distro_code == 'el7': + if self.stable and self.stable >= 'pacific': + raise Error('Ceph does not support pacific or later for this version of this linux distro and therefore cannot add a repo for it') + if self.version and self.version.split('.')[0] >= '16': + raise Error('Ceph does not support 16.y.z or later for this version of this linux distro and therefore cannot add a repo for it') + + if self.stable or self.version: + # we know that yum & dnf require there to be a + # $base_url/$arch/repodata/repomd.xml so we can test if this URL + # is gettable in order to validate the inputs + test_url = self.repo_baseurl() + '/noarch/repodata/repomd.xml' + try: + urlopen(test_url) + except HTTPError as err: + logger.error('unable to fetch repo metadata: %r', err) + raise Error('failed to fetch repository metadata. please check' + ' the provided parameters are correct and try again') + + def add_repo(self) -> None: + if self.stable or self.version: + content = '' + for n, t in { + 'Ceph': '$basearch', + 'Ceph-noarch': 'noarch', + 'Ceph-source': 'SRPMS'}.items(): + content += '[%s]\n' % (n) + content += self.custom_repo( + name='Ceph %s' % t, + baseurl=self.repo_baseurl() + '/' + t, + enabled=1, + gpgcheck=1, + gpgkey=self.repo_gpgkey()[0], + ) + content += '\n\n' + else: + content = self.query_shaman(self.distro_normalized, self.major, + self.branch, + self.commit) + + logger.info('Writing repo to %s...' % self.repo_path()) + with open(self.repo_path(), 'w') as f: + f.write(content) + + if self.distro_code.startswith('el'): + logger.info('Enabling EPEL...') + call_throws(self.ctx, [self.tool, 'install', '-y', 'epel-release']) + + def rm_repo(self) -> None: + if os.path.exists(self.repo_path()): + os.unlink(self.repo_path()) + + def install(self, ls: List[str]) -> None: + logger.info('Installing packages %s...' % ls) + call_throws(self.ctx, [self.tool, 'install', '-y'] + ls) + + def install_podman(self) -> None: + self.install(['podman']) + + +class Zypper(Packager): + DISTRO_NAMES = [ + 'sles', + 'opensuse-tumbleweed', + 'opensuse-leap' + ] + + def __init__(self, ctx: CephadmContext, + stable: Optional[str], version: Optional[str], branch: Optional[str], commit: Optional[str], + distro: Optional[str], distro_version: Optional[str]) -> None: + super(Zypper, self).__init__(ctx, stable=stable, version=version, + branch=branch, commit=commit) + assert distro is not None + self.ctx = ctx + self.tool = 'zypper' + self.distro = 'opensuse' + self.distro_version = '15.1' + if 'tumbleweed' not in distro and distro_version is not None: + self.distro_version = distro_version + + def custom_repo(self, **kw: Any) -> str: + """ + See YumDnf for format explanation. + """ + lines = [] + + # by using tuples (vs a dict) we preserve the order of what we want to + # return, like starting with a [repo name] + tmpl = ( + ('reponame', '[%s]'), + ('name', 'name=%s'), + ('baseurl', 'baseurl=%s'), + ('enabled', 'enabled=%s'), + ('gpgcheck', 'gpgcheck=%s'), + ('_type', 'type=%s'), + ('gpgkey', 'gpgkey=%s'), + ('proxy', 'proxy=%s'), + ('priority', 'priority=%s'), + ) + + for line in tmpl: + tmpl_key, tmpl_value = line # key values from tmpl + + # ensure that there is an actual value (not None nor empty string) + if tmpl_key in kw and kw.get(tmpl_key) not in (None, ''): + lines.append(tmpl_value % kw.get(tmpl_key)) + + return '\n'.join(lines) + + def repo_path(self) -> str: + return '/etc/zypp/repos.d/ceph.repo' + + def repo_baseurl(self) -> str: + assert self.stable or self.version + if self.version: + return '%s/rpm-%s/%s' % (self.ctx.repo_url, + self.stable, self.distro) + else: + return '%s/rpm-%s/%s' % (self.ctx.repo_url, + self.stable, self.distro) + + def add_repo(self) -> None: + if self.stable or self.version: + content = '' + for n, t in { + 'Ceph': '$basearch', + 'Ceph-noarch': 'noarch', + 'Ceph-source': 'SRPMS'}.items(): + content += '[%s]\n' % (n) + content += self.custom_repo( + name='Ceph %s' % t, + baseurl=self.repo_baseurl() + '/' + t, + enabled=1, + gpgcheck=1, + gpgkey=self.repo_gpgkey()[0], + ) + content += '\n\n' + else: + content = self.query_shaman(self.distro, self.distro_version, + self.branch, + self.commit) + + logger.info('Writing repo to %s...' % self.repo_path()) + with open(self.repo_path(), 'w') as f: + f.write(content) + + def rm_repo(self) -> None: + if os.path.exists(self.repo_path()): + os.unlink(self.repo_path()) + + def install(self, ls: List[str]) -> None: + logger.info('Installing packages %s...' % ls) + call_throws(self.ctx, [self.tool, 'in', '-y'] + ls) + + def install_podman(self) -> None: + self.install(['podman']) + + +def create_packager(ctx: CephadmContext, + stable: Optional[str] = None, version: Optional[str] = None, + branch: Optional[str] = None, commit: Optional[str] = None) -> Packager: + distro, distro_version, distro_codename = get_distro() + if distro in YumDnf.DISTRO_NAMES: + return YumDnf(ctx, stable=stable, version=version, + branch=branch, commit=commit, + distro=distro, distro_version=distro_version) + elif distro in Apt.DISTRO_NAMES: + return Apt(ctx, stable=stable, version=version, + branch=branch, commit=commit, + distro=distro, distro_version=distro_version, + distro_codename=distro_codename) + elif distro in Zypper.DISTRO_NAMES: + return Zypper(ctx, stable=stable, version=version, + branch=branch, commit=commit, + distro=distro, distro_version=distro_version) + raise Error('Distro %s version %s not supported' % (distro, distro_version)) + + +def command_add_repo(ctx: CephadmContext) -> None: + if ctx.version and ctx.release: + raise Error('you can specify either --release or --version but not both') + if not ctx.version and not ctx.release and not ctx.dev and not ctx.dev_commit: + raise Error('please supply a --release, --version, --dev or --dev-commit argument') + if ctx.version: + try: + (x, y, z) = ctx.version.split('.') + except Exception: + raise Error('version must be in the form x.y.z (e.g., 15.2.0)') + if ctx.release: + # Pacific =/= pacific in this case, set to undercase to avoid confision + ctx.release = ctx.release.lower() + + pkg = create_packager(ctx, stable=ctx.release, + version=ctx.version, + branch=ctx.dev, + commit=ctx.dev_commit) + pkg.validate() + pkg.add_repo() + logger.info('Completed adding repo.') + + +def command_rm_repo(ctx: CephadmContext) -> None: + pkg = create_packager(ctx) + pkg.rm_repo() + + +def command_install(ctx: CephadmContext) -> None: + pkg = create_packager(ctx) + pkg.install(ctx.packages) + + +def command_rescan_disks(ctx: CephadmContext) -> str: + + def probe_hba(scan_path: str) -> None: + """Tell the adapter to rescan""" + with open(scan_path, 'w') as f: + f.write('- - -') + + cmd = ctx.func.__name__.replace('command_', '') + logger.info(f'{cmd}: starting') + start = time.time() + + all_scan_files = glob('/sys/class/scsi_host/*/scan') + scan_files = [] + skipped = [] + for scan_path in all_scan_files: + adapter_name = os.path.basename(os.path.dirname(scan_path)) + proc_name = read_file([os.path.join(os.path.dirname(scan_path), 'proc_name')]) + if proc_name in ['unknown', 'usb-storage']: + skipped.append(os.path.basename(scan_path)) + logger.info(f'{cmd}: rescan skipping incompatible host adapter {adapter_name} : {proc_name}') + continue + + scan_files.append(scan_path) + + if not scan_files: + logger.info(f'{cmd}: no compatible HBAs found') + return 'Ok. No compatible HBAs found' + + responses = async_run(concurrent_tasks(probe_hba, scan_files)) + failures = [r for r in responses if r] + + logger.info(f'{cmd}: Complete. {len(scan_files)} adapters rescanned, {len(failures)} failures, {len(skipped)} skipped') + + elapsed = time.time() - start + if failures: + plural = 's' if len(failures) > 1 else '' + if len(failures) == len(scan_files): + return f'Failed. All {len(scan_files)} rescan requests failed' + else: + return f'Partial. {len(scan_files) - len(failures)} successful, {len(failures)} failure{plural} against: {", ".join(failures)}' + + return f'Ok. {len(all_scan_files)} adapters detected: {len(scan_files)} rescanned, {len(skipped)} skipped, {len(failures)} failed ({elapsed:.2f}s)' + +################################## + + +def get_ipv4_address(ifname): + # type: (str) -> str + def _extract(sock: socket.socket, offset: int) -> str: + return socket.inet_ntop( + socket.AF_INET, + fcntl.ioctl( + sock.fileno(), + offset, + struct.pack('256s', bytes(ifname[:15], 'utf-8')) + )[20:24]) + + s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + try: + addr = _extract(s, 35093) # '0x8915' = SIOCGIFADDR + dq_mask = _extract(s, 35099) # 0x891b = SIOCGIFNETMASK + except OSError: + # interface does not have an ipv4 address + return '' + + dec_mask = sum([bin(int(i)).count('1') + for i in dq_mask.split('.')]) + return '{}/{}'.format(addr, dec_mask) + + +def get_ipv6_address(ifname): + # type: (str) -> str + if not os.path.exists('/proc/net/if_inet6'): + return '' + + raw = read_file(['/proc/net/if_inet6']) + data = raw.splitlines() + # based on docs @ https://www.tldp.org/HOWTO/Linux+IPv6-HOWTO/ch11s04.html + # field 0 is ipv6, field 2 is scope + for iface_setting in data: + field = iface_setting.split() + if field[-1] == ifname: + ipv6_raw = field[0] + ipv6_fmtd = ':'.join([ipv6_raw[_p:_p + 4] for _p in range(0, len(field[0]), 4)]) + # apply naming rules using ipaddress module + ipv6 = ipaddress.ip_address(ipv6_fmtd) + return '{}/{}'.format(str(ipv6), int('0x{}'.format(field[2]), 16)) + return '' + + +def bytes_to_human(num, mode='decimal'): + # type: (float, str) -> str + """Convert a bytes value into it's human-readable form. + + :param num: number, in bytes, to convert + :param mode: Either decimal (default) or binary to determine divisor + :returns: string representing the bytes value in a more readable format + """ + unit_list = ['', 'KB', 'MB', 'GB', 'TB', 'PB', 'EB', 'ZB'] + divisor = 1000.0 + yotta = 'YB' + + if mode == 'binary': + unit_list = ['', 'KiB', 'MiB', 'GiB', 'TiB', 'PiB', 'EiB', 'ZiB'] + divisor = 1024.0 + yotta = 'YiB' + + for unit in unit_list: + if abs(num) < divisor: + return '%3.1f%s' % (num, unit) + num /= divisor + return '%.1f%s' % (num, yotta) + + +def read_file(path_list, file_name=''): + # type: (List[str], str) -> str + """Returns the content of the first file found within the `path_list` + + :param path_list: list of file paths to search + :param file_name: optional file_name to be applied to a file path + :returns: content of the file or 'Unknown' + """ + for path in path_list: + if file_name: + file_path = os.path.join(path, file_name) + else: + file_path = path + if os.path.exists(file_path): + with open(file_path, 'r') as f: + try: + content = f.read().strip() + except OSError: + # sysfs may populate the file, but for devices like + # virtio reads can fail + return 'Unknown' + else: + return content + return 'Unknown' + +################################## + + +class HostFacts(): + _dmi_path_list = ['/sys/class/dmi/id'] + _nic_path_list = ['/sys/class/net'] + _apparmor_path_list = ['/etc/apparmor'] + _disk_vendor_workarounds = { + '0x1af4': 'Virtio Block Device' + } + _excluded_block_devices = ('sr', 'zram', 'dm-') + + def __init__(self, ctx: CephadmContext): + self.ctx: CephadmContext = ctx + self.cpu_model: str = 'Unknown' + self.cpu_count: int = 0 + self.cpu_cores: int = 0 + self.cpu_threads: int = 0 + self.interfaces: Dict[str, Any] = {} + + self._meminfo: List[str] = read_file(['/proc/meminfo']).splitlines() + self._get_cpuinfo() + self._process_nics() + self.arch: str = platform.processor() + self.kernel: str = platform.release() + + def _get_cpuinfo(self): + # type: () -> None + """Determine cpu information via /proc/cpuinfo""" + raw = read_file(['/proc/cpuinfo']) + output = raw.splitlines() + cpu_set = set() + + for line in output: + field = [f.strip() for f in line.split(':')] + if 'model name' in line: + self.cpu_model = field[1] + if 'physical id' in line: + cpu_set.add(field[1]) + if 'siblings' in line: + self.cpu_threads = int(field[1].strip()) + if 'cpu cores' in line: + self.cpu_cores = int(field[1].strip()) + pass + self.cpu_count = len(cpu_set) + + def _get_block_devs(self): + # type: () -> List[str] + """Determine the list of block devices by looking at /sys/block""" + return [dev for dev in os.listdir('/sys/block') + if not dev.startswith(HostFacts._excluded_block_devices)] + + def _get_devs_by_type(self, rota='0'): + # type: (str) -> List[str] + """Filter block devices by a given rotational attribute (0=flash, 1=spinner)""" + devs = list() + for blk_dev in self._get_block_devs(): + rot_path = '/sys/block/{}/queue/rotational'.format(blk_dev) + rot_value = read_file([rot_path]) + if rot_value == rota: + devs.append(blk_dev) + return devs + + @property + def operating_system(self): + # type: () -> str + """Determine OS version""" + raw_info = read_file(['/etc/os-release']) + os_release = raw_info.splitlines() + rel_str = 'Unknown' + rel_dict = dict() + + for line in os_release: + if '=' in line: + var_name, var_value = line.split('=') + rel_dict[var_name] = var_value.strip('"') + + # Would normally use PRETTY_NAME, but NAME and VERSION are more + # consistent + if all(_v in rel_dict for _v in ['NAME', 'VERSION']): + rel_str = '{} {}'.format(rel_dict['NAME'], rel_dict['VERSION']) + return rel_str + + @property + def hostname(self): + # type: () -> str + """Return the hostname""" + return platform.node() + + @property + def subscribed(self): + # type: () -> str + """Highlevel check to see if the host is subscribed to receive updates/support""" + def _red_hat(): + # type: () -> str + # RHEL 7 and RHEL 8 + entitlements_dir = '/etc/pki/entitlement' + if os.path.exists(entitlements_dir): + pems = glob('{}/*.pem'.format(entitlements_dir)) + if len(pems) >= 2: + return 'Yes' + + return 'No' + + os_name = self.operating_system + if os_name.upper().startswith('RED HAT'): + return _red_hat() + + return 'Unknown' + + @property + def hdd_count(self): + # type: () -> int + """Return a count of HDDs (spinners)""" + return len(self._get_devs_by_type(rota='1')) + + def _get_capacity(self, dev): + # type: (str) -> int + """Determine the size of a given device + + The kernel always bases device size calculations based on a 512 byte + sector. For more information see + https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git/tree/include/linux/types.h?h=v5.15.63#n120 + """ + size_path = os.path.join('/sys/block', dev, 'size') + size_blocks = int(read_file([size_path])) + return size_blocks * 512 + + def _get_capacity_by_type(self, rota='0'): + # type: (str) -> int + """Return the total capacity of a category of device (flash or hdd)""" + devs = self._get_devs_by_type(rota=rota) + capacity = 0 + for dev in devs: + capacity += self._get_capacity(dev) + return capacity + + def _dev_list(self, dev_list): + # type: (List[str]) -> List[Dict[str, object]] + """Return a 'pretty' name list for each device in the `dev_list`""" + disk_list = list() + + for dev in dev_list: + disk_model = read_file(['/sys/block/{}/device/model'.format(dev)]).strip() + disk_rev = read_file(['/sys/block/{}/device/rev'.format(dev)]).strip() + disk_wwid = read_file(['/sys/block/{}/device/wwid'.format(dev)]).strip() + vendor = read_file(['/sys/block/{}/device/vendor'.format(dev)]).strip() + disk_vendor = HostFacts._disk_vendor_workarounds.get(vendor, vendor) + disk_size_bytes = self._get_capacity(dev) + disk_list.append({ + 'description': '{} {} ({})'.format(disk_vendor, disk_model, bytes_to_human(disk_size_bytes)), + 'vendor': disk_vendor, + 'model': disk_model, + 'rev': disk_rev, + 'wwid': disk_wwid, + 'dev_name': dev, + 'disk_size_bytes': disk_size_bytes, + }) + return disk_list + + @property + def hdd_list(self): + # type: () -> List[Dict[str, object]] + """Return a list of devices that are HDDs (spinners)""" + devs = self._get_devs_by_type(rota='1') + return self._dev_list(devs) + + @property + def flash_list(self): + # type: () -> List[Dict[str, object]] + """Return a list of devices that are flash based (SSD, NVMe)""" + devs = self._get_devs_by_type(rota='0') + return self._dev_list(devs) + + @property + def hdd_capacity_bytes(self): + # type: () -> int + """Return the total capacity for all HDD devices (bytes)""" + return self._get_capacity_by_type(rota='1') + + @property + def hdd_capacity(self): + # type: () -> str + """Return the total capacity for all HDD devices (human readable format)""" + return bytes_to_human(self.hdd_capacity_bytes) + + @property + def cpu_load(self): + # type: () -> Dict[str, float] + """Return the cpu load average data for the host""" + raw = read_file(['/proc/loadavg']).strip() + data = raw.split() + return { + '1min': float(data[0]), + '5min': float(data[1]), + '15min': float(data[2]), + } + + @property + def flash_count(self): + # type: () -> int + """Return the number of flash devices in the system (SSD, NVMe)""" + return len(self._get_devs_by_type(rota='0')) + + @property + def flash_capacity_bytes(self): + # type: () -> int + """Return the total capacity for all flash devices (bytes)""" + return self._get_capacity_by_type(rota='0') + + @property + def flash_capacity(self): + # type: () -> str + """Return the total capacity for all Flash devices (human readable format)""" + return bytes_to_human(self.flash_capacity_bytes) + + def _process_nics(self): + # type: () -> None + """Look at the NIC devices and extract network related metadata""" + # from https://github.com/torvalds/linux/blob/master/include/uapi/linux/if_arp.h + hw_lookup = { + '1': 'ethernet', + '32': 'infiniband', + '772': 'loopback', + } + + for nic_path in HostFacts._nic_path_list: + if not os.path.exists(nic_path): + continue + for iface in os.listdir(nic_path): + + lower_devs_list = [os.path.basename(link.replace('lower_', '')) for link in glob(os.path.join(nic_path, iface, 'lower_*'))] + upper_devs_list = [os.path.basename(link.replace('upper_', '')) for link in glob(os.path.join(nic_path, iface, 'upper_*'))] + + try: + mtu = int(read_file([os.path.join(nic_path, iface, 'mtu')])) + except ValueError: + mtu = 0 + + operstate = read_file([os.path.join(nic_path, iface, 'operstate')]) + try: + speed = int(read_file([os.path.join(nic_path, iface, 'speed')])) + except (OSError, ValueError): + # OSError : device doesn't support the ethtool get_link_ksettings + # ValueError : raised when the read fails, and returns Unknown + # + # Either way, we show a -1 when speed isn't available + speed = -1 + + if os.path.exists(os.path.join(nic_path, iface, 'bridge')): + nic_type = 'bridge' + elif os.path.exists(os.path.join(nic_path, iface, 'bonding')): + nic_type = 'bonding' + else: + nic_type = hw_lookup.get(read_file([os.path.join(nic_path, iface, 'type')]), 'Unknown') + + dev_link = os.path.join(nic_path, iface, 'device') + if os.path.exists(dev_link): + iftype = 'physical' + driver_path = os.path.join(dev_link, 'driver') + if os.path.exists(driver_path): + driver = os.path.basename(os.path.realpath(driver_path)) + else: + driver = 'Unknown' + + else: + iftype = 'logical' + driver = '' + + self.interfaces[iface] = { + 'mtu': mtu, + 'upper_devs_list': upper_devs_list, + 'lower_devs_list': lower_devs_list, + 'operstate': operstate, + 'iftype': iftype, + 'nic_type': nic_type, + 'driver': driver, + 'speed': speed, + 'ipv4_address': get_ipv4_address(iface), + 'ipv6_address': get_ipv6_address(iface), + } + + @property + def nic_count(self): + # type: () -> int + """Return a total count of all physical NICs detected in the host""" + phys_devs = [] + for iface in self.interfaces: + if self.interfaces[iface]['iftype'] == 'physical': + phys_devs.append(iface) + return len(phys_devs) + + def _get_mem_data(self, field_name): + # type: (str) -> int + for line in self._meminfo: + if line.startswith(field_name): + _d = line.split() + return int(_d[1]) + return 0 + + @property + def memory_total_kb(self): + # type: () -> int + """Determine the memory installed (kb)""" + return self._get_mem_data('MemTotal') + + @property + def memory_free_kb(self): + # type: () -> int + """Determine the memory free (not cache, immediately usable)""" + return self._get_mem_data('MemFree') + + @property + def memory_available_kb(self): + # type: () -> int + """Determine the memory available to new applications without swapping""" + return self._get_mem_data('MemAvailable') + + @property + def vendor(self): + # type: () -> str + """Determine server vendor from DMI data in sysfs""" + return read_file(HostFacts._dmi_path_list, 'sys_vendor') + + @property + def model(self): + # type: () -> str + """Determine server model information from DMI data in sysfs""" + family = read_file(HostFacts._dmi_path_list, 'product_family') + product = read_file(HostFacts._dmi_path_list, 'product_name') + if family == 'Unknown' and product: + return '{}'.format(product) + + return '{} ({})'.format(family, product) + + @property + def bios_version(self): + # type: () -> str + """Determine server BIOS version from DMI data in sysfs""" + return read_file(HostFacts._dmi_path_list, 'bios_version') + + @property + def bios_date(self): + # type: () -> str + """Determine server BIOS date from DMI data in sysfs""" + return read_file(HostFacts._dmi_path_list, 'bios_date') + + @property + def timestamp(self): + # type: () -> float + """Return the current time as Epoch seconds""" + return time.time() + + @property + def system_uptime(self): + # type: () -> float + """Return the system uptime (in secs)""" + raw_time = read_file(['/proc/uptime']) + up_secs, _ = raw_time.split() + return float(up_secs) + + @property + def kernel_security(self): + # type: () -> Dict[str, str] + """Determine the security features enabled in the kernel - SELinux, AppArmor""" + def _fetch_selinux() -> Dict[str, str]: + """Get the selinux status""" + security = {} + try: + out, err, code = call(self.ctx, ['sestatus'], + verbosity=CallVerbosity.DEBUG) + security['type'] = 'SELinux' + status, mode, policy = '', '', '' + for line in out.split('\n'): + if line.startswith('SELinux status:'): + k, v = line.split(':') + status = v.strip() + elif line.startswith('Current mode:'): + k, v = line.split(':') + mode = v.strip() + elif line.startswith('Loaded policy name:'): + k, v = line.split(':') + policy = v.strip() + if status == 'disabled': + security['description'] = 'SELinux: Disabled' + else: + security['description'] = 'SELinux: Enabled({}, {})'.format(mode, policy) + except Exception as e: + logger.info('unable to get selinux status: %s' % e) + return security + + def _fetch_apparmor() -> Dict[str, str]: + """Read the apparmor profiles directly, returning an overview of AppArmor status""" + security = {} + for apparmor_path in HostFacts._apparmor_path_list: + if os.path.exists(apparmor_path): + security['type'] = 'AppArmor' + security['description'] = 'AppArmor: Enabled' + try: + profiles = read_file(['/sys/kernel/security/apparmor/profiles']) + if len(profiles) == 0: + return {} + except OSError: + pass + else: + summary = {} # type: Dict[str, int] + for line in profiles.split('\n'): + item, mode = line.split(' ') + mode = mode.strip('()') + if mode in summary: + summary[mode] += 1 + else: + summary[mode] = 0 + summary_str = ','.join(['{} {}'.format(v, k) for k, v in summary.items()]) + security = {**security, **summary} # type: ignore + security['description'] += '({})'.format(summary_str) + + return security + return {} + + ret = {} + if os.path.exists('/sys/kernel/security/lsm'): + lsm = read_file(['/sys/kernel/security/lsm']).strip() + if 'selinux' in lsm: + ret = _fetch_selinux() + elif 'apparmor' in lsm: + ret = _fetch_apparmor() + else: + return { + 'type': 'Unknown', + 'description': 'Linux Security Module framework is active, but is not using SELinux or AppArmor' + } + + if ret: + return ret + + return { + 'type': 'None', + 'description': 'Linux Security Module framework is not available' + } + + @property + def selinux_enabled(self) -> bool: + return (self.kernel_security['type'] == 'SELinux') and \ + (self.kernel_security['description'] != 'SELinux: Disabled') + + @property + def kernel_parameters(self): + # type: () -> Dict[str, str] + """Get kernel parameters required/used in Ceph clusters""" + + k_param = {} + out, _, _ = call_throws(self.ctx, ['sysctl', '-a'], verbosity=CallVerbosity.SILENT) + if out: + param_list = out.split('\n') + param_dict = {param.split(' = ')[0]: param.split(' = ')[-1] for param in param_list} + + # return only desired parameters + if 'net.ipv4.ip_nonlocal_bind' in param_dict: + k_param['net.ipv4.ip_nonlocal_bind'] = param_dict['net.ipv4.ip_nonlocal_bind'] + + return k_param + + @staticmethod + def _process_net_data(tcp_file: str, protocol: str = 'tcp') -> List[int]: + listening_ports = [] + # Connections state documentation + # tcp - https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/include/net/tcp_states.h + # udp - uses 07 (TCP_CLOSE or UNCONN, since udp is stateless. test with netcat -ul <port>) + listening_state = { + 'tcp': '0A', + 'udp': '07' + } + + if protocol not in listening_state.keys(): + return [] + + if os.path.exists(tcp_file): + with open(tcp_file) as f: + tcp_data = f.readlines()[1:] + + for con in tcp_data: + con_info = con.strip().split() + if con_info[3] == listening_state[protocol]: + local_port = int(con_info[1].split(':')[1], 16) + listening_ports.append(local_port) + + return listening_ports + + @property + def tcp_ports_used(self) -> List[int]: + return HostFacts._process_net_data('/proc/net/tcp') + + @property + def tcp6_ports_used(self) -> List[int]: + return HostFacts._process_net_data('/proc/net/tcp6') + + @property + def udp_ports_used(self) -> List[int]: + return HostFacts._process_net_data('/proc/net/udp', 'udp') + + @property + def udp6_ports_used(self) -> List[int]: + return HostFacts._process_net_data('/proc/net/udp6', 'udp') + + def dump(self): + # type: () -> str + """Return the attributes of this HostFacts object as json""" + data = { + k: getattr(self, k) for k in dir(self) + if not k.startswith('_') + and isinstance(getattr(self, k), (float, int, str, list, dict, tuple)) + } + return json.dumps(data, indent=2, sort_keys=True) + +################################## + + +def command_gather_facts(ctx: CephadmContext) -> None: + """gather_facts is intended to provide host releated metadata to the caller""" + host = HostFacts(ctx) + print(host.dump()) + + +################################## + + +class CephadmCache: + task_types = ['disks', 'daemons', 'host', 'http_server'] + + def __init__(self) -> None: + self.started_epoch_secs = time.time() + self.tasks = { + 'daemons': 'inactive', + 'disks': 'inactive', + 'host': 'inactive', + 'http_server': 'inactive', + } + self.errors: list = [] + self.disks: dict = {} + self.daemons: dict = {} + self.host: dict = {} + self.lock = RLock() + + @property + def health(self) -> dict: + return { + 'started_epoch_secs': self.started_epoch_secs, + 'tasks': self.tasks, + 'errors': self.errors, + } + + def to_json(self) -> dict: + return { + 'health': self.health, + 'host': self.host, + 'daemons': self.daemons, + 'disks': self.disks, + } + + def update_health(self, task_type: str, task_status: str, error_msg: Optional[str] = None) -> None: + assert task_type in CephadmCache.task_types + with self.lock: + self.tasks[task_type] = task_status + if error_msg: + self.errors.append(error_msg) + + def update_task(self, task_type: str, content: dict) -> None: + assert task_type in CephadmCache.task_types + assert isinstance(content, dict) + with self.lock: + current = getattr(self, task_type) + for k in content: + current[k] = content[k] + + setattr(self, task_type, current) + + +class CephadmHTTPServer(ThreadingMixIn, HTTPServer): + allow_reuse_address = True + daemon_threads = True + cephadm_cache: CephadmCache + token: str + + +class CephadmDaemonHandler(BaseHTTPRequestHandler): + server: CephadmHTTPServer + api_version = 'v1' + valid_routes = [ + f'/{api_version}/metadata', + f'/{api_version}/metadata/health', + f'/{api_version}/metadata/disks', + f'/{api_version}/metadata/daemons', + f'/{api_version}/metadata/host', + ] + + class Decorators: + @classmethod + def authorize(cls, f: Any) -> Any: + """Implement a basic token check. + + The token is installed at deployment time and must be provided to + ensure we only respond to callers who know our token i.e. mgr + """ + + def wrapper(self: Any, *args: Any, **kwargs: Any) -> Any: + auth = self.headers.get('Authorization', None) + if auth != 'Bearer ' + self.server.token: + self.send_error(401) + return + f(self, *args, **kwargs) + + return wrapper + + def _help_page(self) -> str: + return """<!DOCTYPE html> +<html> +<head><title>cephadm metadata exporter</title></head> +<style> +body {{ + font-family: sans-serif; + font-size: 0.8em; +}} +table {{ + border-width: 0px; + border-spacing: 0px; + margin-left:20px; +}} +tr:hover {{ + background: PowderBlue; +}} +td,th {{ + padding: 5px; +}} +</style> +<body> + <h1>cephadm metadata exporter {api_version}</h1> + <table> + <thead> + <tr><th>Endpoint</th><th>Methods</th><th>Response</th><th>Description</th></tr> + </thead> + <tr><td><a href='{api_version}/metadata'>{api_version}/metadata</a></td><td>GET</td><td>JSON</td><td>Return <b>all</b> metadata for the host</td></tr> + <tr><td><a href='{api_version}/metadata/daemons'>{api_version}/metadata/daemons</a></td><td>GET</td><td>JSON</td><td>Return daemon and systemd states for ceph daemons (ls)</td></tr> + <tr><td><a href='{api_version}/metadata/disks'>{api_version}/metadata/disks</a></td><td>GET</td><td>JSON</td><td>show disk inventory (ceph-volume)</td></tr> + <tr><td><a href='{api_version}/metadata/health'>{api_version}/metadata/health</a></td><td>GET</td><td>JSON</td><td>Show current health of the exporter sub-tasks</td></tr> + <tr><td><a href='{api_version}/metadata/host'>{api_version}/metadata/host</a></td><td>GET</td><td>JSON</td><td>Show host metadata (gather-facts)</td></tr> + </table> +</body> +</html>""".format(api_version=CephadmDaemonHandler.api_version) + + def _fetch_root(self) -> None: + self.send_response(200) + self.send_header('Content-type', 'text/html; charset=utf-8') + self.end_headers() + self.wfile.write(self._help_page().encode('utf-8')) + + @Decorators.authorize + def do_GET(self) -> None: + """Handle *all* GET requests""" + + if self.path == '/': + # provide a html response if someone hits the root url, to document the + # available api endpoints + return self._fetch_root() + elif self.path in CephadmDaemonHandler.valid_routes: + u = self.path.split('/')[-1] + data = json.dumps({}) + status_code = 200 + + tasks = self.server.cephadm_cache.health.get('tasks', {}) + assert tasks + + # We're using the http status code to help indicate thread health + # - 200 (OK): request successful + # - 204 (No Content): access to a cache relating to a dead thread + # - 206 (Partial content): one or more theads are inactive + # - 500 (Server Error): all threads inactive + if u == 'metadata': + data = json.dumps(self.server.cephadm_cache.to_json()) + if all([tasks[task_name] == 'inactive' for task_name in tasks if task_name != 'http_server']): + # All the subtasks are dead! + status_code = 500 + elif any([tasks[task_name] == 'inactive' for task_name in tasks if task_name != 'http_server']): + status_code = 206 + + # Individual GETs against the a tasks endpoint will also return a 503 if the corresponding thread is inactive + elif u == 'daemons': + data = json.dumps(self.server.cephadm_cache.daemons) + if tasks['daemons'] == 'inactive': + status_code = 204 + elif u == 'disks': + data = json.dumps(self.server.cephadm_cache.disks) + if tasks['disks'] == 'inactive': + status_code = 204 + elif u == 'host': + data = json.dumps(self.server.cephadm_cache.host) + if tasks['host'] == 'inactive': + status_code = 204 + + # a GET against health will always return a 200, since the op is always successful + elif u == 'health': + data = json.dumps(self.server.cephadm_cache.health) + + self.send_response(status_code) + self.send_header('Content-type', 'application/json') + self.end_headers() + self.wfile.write(data.encode('utf-8')) + else: + # Invalid GET URL + bad_request_msg = 'Valid URLs are: {}'.format(', '.join(CephadmDaemonHandler.valid_routes)) + self.send_response(404, message=bad_request_msg) # reason + self.send_header('Content-type', 'application/json') + self.end_headers() + self.wfile.write(json.dumps({'message': bad_request_msg}).encode('utf-8')) + + def log_message(self, format: str, *args: Any) -> None: + rqst = ' '.join(str(a) for a in args) + logger.info(f'client:{self.address_string()} [{self.log_date_time_string()}] {rqst}') + + +class CephadmDaemon(): + + daemon_type = 'cephadm-exporter' + default_port = 9443 + key_name = 'key' + crt_name = 'crt' + token_name = 'token' + config_requirements = [ + key_name, + crt_name, + token_name, + ] + loop_delay = 1 + thread_check_interval = 5 + + def __init__(self, ctx: CephadmContext, fsid: str, daemon_id: Optional[str] = None, port: Optional[int] = None) -> None: + self.ctx = ctx + self.fsid = fsid + self.daemon_id = daemon_id + if not port: + self.port = CephadmDaemon.default_port + else: + self.port = port + self.workers: List[Thread] = [] + self.http_server: CephadmHTTPServer + self.stop = False + self.cephadm_cache = CephadmCache() + self.errors: List[str] = [] + self.token = read_file([os.path.join(self.daemon_path, CephadmDaemon.token_name)]) + + @classmethod + def validate_config(cls, config: dict) -> None: + reqs = ', '.join(CephadmDaemon.config_requirements) + errors = [] + + if not config or not all([k_name in config for k_name in CephadmDaemon.config_requirements]): + raise Error(f'config must contain the following fields : {reqs}') + + if not all([isinstance(config[k_name], str) for k_name in CephadmDaemon.config_requirements]): + errors.append(f'the following fields must be strings: {reqs}') + + crt = config[CephadmDaemon.crt_name] + key = config[CephadmDaemon.key_name] + token = config[CephadmDaemon.token_name] + + if not crt.startswith('-----BEGIN CERTIFICATE-----') or not crt.endswith('-----END CERTIFICATE-----\n'): + errors.append('crt field is not a valid SSL certificate') + if not key.startswith('-----BEGIN PRIVATE KEY-----') or not key.endswith('-----END PRIVATE KEY-----\n'): + errors.append('key is not a valid SSL private key') + if len(token) < 8: + errors.append("'token' must be more than 8 characters long") + + if 'port' in config: + try: + p = int(config['port']) + if p <= 1024: + raise ValueError + except (TypeError, ValueError): + errors.append('port must be an integer > 1024') + + if errors: + raise Error('Parameter errors : {}'.format(', '.join(errors))) + + @property + def port_active(self) -> bool: + return port_in_use(self.ctx, self.port) + + @property + def can_run(self) -> bool: + # if port is in use + if self.port_active: + self.errors.append(f'TCP port {self.port} already in use, unable to bind') + if not os.path.exists(os.path.join(self.daemon_path, CephadmDaemon.key_name)): + self.errors.append(f"Key file '{CephadmDaemon.key_name}' is missing from {self.daemon_path}") + if not os.path.exists(os.path.join(self.daemon_path, CephadmDaemon.crt_name)): + self.errors.append(f"Certificate file '{CephadmDaemon.crt_name}' is missing from {self.daemon_path}") + if self.token == 'Unknown': + self.errors.append(f"Authentication token '{CephadmDaemon.token_name}' is missing from {self.daemon_path}") + return len(self.errors) == 0 + + @staticmethod + def _unit_name(fsid: str, daemon_id: str) -> str: + return '{}.service'.format(get_unit_name(fsid, CephadmDaemon.daemon_type, daemon_id)) + + @property + def unit_name(self) -> str: + assert self.daemon_id is not None + return CephadmDaemon._unit_name(self.fsid, self.daemon_id) + + @property + def daemon_path(self) -> str: + return os.path.join( + self.ctx.data_dir, + self.fsid, + f'{self.daemon_type}.{self.daemon_id}' + ) + + @property + def binary_path(self) -> str: + path = os.path.realpath(__file__) + assert os.path.isfile(path) + return path + + def _handle_thread_exception(self, exc: Exception, thread_type: str) -> None: + e_msg = f'{exc.__class__.__name__} exception: {str(exc)}' + thread_info = getattr(self.cephadm_cache, thread_type) + errors = thread_info.get('scrape_errors', []) + errors.append(e_msg) + logger.error(e_msg) + logger.exception(exc) + self.cephadm_cache.update_task( + thread_type, + { + 'scrape_errors': errors, + 'data': None, + } + ) + + def _scrape_host_facts(self, refresh_interval: int = 10) -> None: + ctr = 0 + exception_encountered = False + + while True: + + if self.stop or exception_encountered: + break + + if ctr >= refresh_interval: + ctr = 0 + logger.debug('executing host-facts scrape') + errors = [] + s_time = time.time() + + try: + facts = HostFacts(self.ctx) + except Exception as e: + self._handle_thread_exception(e, 'host') + exception_encountered = True + else: + elapsed = time.time() - s_time + try: + data = json.loads(facts.dump()) + except json.decoder.JSONDecodeError: + errors.append('host-facts provided invalid JSON') + logger.warning(errors[-1]) + data = {} + self.cephadm_cache.update_task( + 'host', + { + 'scrape_timestamp': s_time, + 'scrape_duration_secs': elapsed, + 'scrape_errors': errors, + 'data': data, + } + ) + logger.debug(f'completed host-facts scrape - {elapsed}s') + + time.sleep(CephadmDaemon.loop_delay) + ctr += CephadmDaemon.loop_delay + logger.info('host-facts thread stopped') + + def _scrape_ceph_volume(self, refresh_interval: int = 15) -> None: + # we're invoking the ceph_volume command, so we need to set the args that it + # expects to use + self.ctx.command = 'inventory --format=json'.split() + self.ctx.fsid = self.fsid + + ctr = 0 + exception_encountered = False + + while True: + if self.stop or exception_encountered: + break + + if ctr >= refresh_interval: + ctr = 0 + logger.debug('executing ceph-volume scrape') + errors = [] + s_time = time.time() + stream = io.StringIO() + try: + with redirect_stdout(stream): + command_ceph_volume(self.ctx) + except Exception as e: + self._handle_thread_exception(e, 'disks') + exception_encountered = True + else: + elapsed = time.time() - s_time + + # if the call to ceph-volume returns junk with the + # json, it won't parse + stdout = stream.getvalue() + + data = [] + if stdout: + try: + data = json.loads(stdout) + except json.decoder.JSONDecodeError: + errors.append('ceph-volume thread provided bad json data') + logger.warning(errors[-1]) + else: + errors.append('ceph-volume did not return any data') + logger.warning(errors[-1]) + + self.cephadm_cache.update_task( + 'disks', + { + 'scrape_timestamp': s_time, + 'scrape_duration_secs': elapsed, + 'scrape_errors': errors, + 'data': data, + } + ) + + logger.debug(f'completed ceph-volume scrape - {elapsed}s') + time.sleep(CephadmDaemon.loop_delay) + ctr += CephadmDaemon.loop_delay + + logger.info('ceph-volume thread stopped') + + def _scrape_list_daemons(self, refresh_interval: int = 20) -> None: + ctr = 0 + exception_encountered = False + while True: + if self.stop or exception_encountered: + break + + if ctr >= refresh_interval: + ctr = 0 + logger.debug('executing list-daemons scrape') + errors = [] + s_time = time.time() + + try: + # list daemons should ideally be invoked with a fsid + data = list_daemons(self.ctx) + except Exception as e: + self._handle_thread_exception(e, 'daemons') + exception_encountered = True + else: + if not isinstance(data, list): + errors.append('list-daemons did not supply a list?') + logger.warning(errors[-1]) + data = [] + elapsed = time.time() - s_time + self.cephadm_cache.update_task( + 'daemons', + { + 'scrape_timestamp': s_time, + 'scrape_duration_secs': elapsed, + 'scrape_errors': errors, + 'data': data, + } + ) + logger.debug(f'completed list-daemons scrape - {elapsed}s') + + time.sleep(CephadmDaemon.loop_delay) + ctr += CephadmDaemon.loop_delay + logger.info('list-daemons thread stopped') + + def _create_thread(self, target: Any, name: str, refresh_interval: Optional[int] = None) -> Thread: + if refresh_interval: + t = Thread(target=target, args=(refresh_interval,)) + else: + t = Thread(target=target) + t.daemon = True + t.name = name + self.cephadm_cache.update_health(name, 'active') + t.start() + + start_msg = f'Started {name} thread' + if refresh_interval: + logger.info(f'{start_msg}, with a refresh interval of {refresh_interval}s') + else: + logger.info(f'{start_msg}') + return t + + def reload(self, *args: Any) -> None: + """reload -HUP received + + This is a placeholder function only, and serves to provide the hook that could + be exploited later if the exporter evolves to incorporate a config file + """ + logger.info('Reload request received - ignoring, no action needed') + + def shutdown(self, *args: Any) -> None: + logger.info('Shutdown request received') + self.stop = True + self.http_server.shutdown() + + def run(self) -> None: + logger.info(f"cephadm exporter starting for FSID '{self.fsid}'") + if not self.can_run: + logger.error('Unable to start the exporter daemon') + for e in self.errors: + logger.error(e) + return + + # register signal handlers for running under systemd control + signal.signal(signal.SIGTERM, self.shutdown) + signal.signal(signal.SIGINT, self.shutdown) + signal.signal(signal.SIGHUP, self.reload) + logger.debug('Signal handlers attached') + + host_facts = self._create_thread(self._scrape_host_facts, 'host', 5) + self.workers.append(host_facts) + + daemons = self._create_thread(self._scrape_list_daemons, 'daemons', 20) + self.workers.append(daemons) + + disks = self._create_thread(self._scrape_ceph_volume, 'disks', 20) + self.workers.append(disks) + + self.http_server = CephadmHTTPServer(('0.0.0.0', self.port), CephadmDaemonHandler) # IPv4 only + self.http_server.socket = ssl.wrap_socket(self.http_server.socket, + keyfile=os.path.join(self.daemon_path, CephadmDaemon.key_name), + certfile=os.path.join(self.daemon_path, CephadmDaemon.crt_name), + server_side=True) + + self.http_server.cephadm_cache = self.cephadm_cache + self.http_server.token = self.token + server_thread = self._create_thread(self.http_server.serve_forever, 'http_server') + logger.info(f'https server listening on {self.http_server.server_address[0]}:{self.http_server.server_port}') + + ctr = 0 + while server_thread.is_alive(): + if self.stop: + break + + if ctr >= CephadmDaemon.thread_check_interval: + ctr = 0 + for worker in self.workers: + if self.cephadm_cache.tasks[worker.name] == 'inactive': + continue + if not worker.is_alive(): + logger.warning(f'{worker.name} thread not running') + stop_time = datetime.datetime.now().strftime('%Y/%m/%d %H:%M:%S') + self.cephadm_cache.update_health(worker.name, 'inactive', f'{worker.name} stopped at {stop_time}') + + time.sleep(CephadmDaemon.loop_delay) + ctr += CephadmDaemon.loop_delay + + logger.info('Main http server thread stopped') + + @property + def unit_run(self) -> str: + + return """set -e +{py3} {bin_path} exporter --fsid {fsid} --id {daemon_id} --port {port} &""".format( + py3=shutil.which('python3'), + bin_path=self.binary_path, + fsid=self.fsid, + daemon_id=self.daemon_id, + port=self.port + ) + + @property + def unit_file(self) -> str: + docker = isinstance(self.ctx.container_engine, Docker) + return """#generated by cephadm +[Unit] +Description=cephadm exporter service for cluster {fsid} +After=network-online.target{docker_after} +Wants=network-online.target +{docker_requires} + +PartOf=ceph-{fsid}.target +Before=ceph-{fsid}.target + +[Service] +Type=forking +ExecStart=/bin/bash {daemon_path}/unit.run +ExecReload=/bin/kill -HUP $MAINPID +Restart=on-failure +RestartSec=10s + +[Install] +WantedBy=ceph-{fsid}.target +""".format(fsid=self.fsid, + daemon_path=self.daemon_path, + # if docker, we depend on docker.service + docker_after=' docker.service' if docker else '', + docker_requires='Requires=docker.service\n' if docker else '') + + def deploy_daemon_unit(self, config: Optional[dict] = None) -> None: + """deploy a specific unit file for cephadm + + The normal deploy_daemon_units doesn't apply for this + daemon since it's not a container, so we just create a + simple service definition and add it to the fsid's target + """ + if not config: + raise Error('Attempting to deploy cephadm daemon without a config') + assert isinstance(config, dict) + + # Create the required config files in the daemons dir, with restricted permissions + for filename in config: + with open(os.open(os.path.join(self.daemon_path, filename), os.O_CREAT | os.O_WRONLY, mode=0o600), 'w') as f: + f.write(config[filename]) + + # When __file__ is <stdin> we're being invoked over remoto via the orchestrator, so + # we pick up the file from where the orchestrator placed it - otherwise we'll + # copy it to the binary location for this cluster + if not __file__ == '<stdin>': + try: + shutil.copy(__file__, + self.binary_path) + except shutil.SameFileError: + pass + + with open(os.path.join(self.daemon_path, 'unit.run'), 'w') as f: + f.write(self.unit_run) + + with open( + os.path.join(self.ctx.unit_dir, + f'{self.unit_name}.new'), + 'w' + ) as f: + f.write(self.unit_file) + os.rename( + os.path.join(self.ctx.unit_dir, f'{self.unit_name}.new'), + os.path.join(self.ctx.unit_dir, self.unit_name)) + + call_throws(self.ctx, ['systemctl', 'daemon-reload']) + call(self.ctx, ['systemctl', 'stop', self.unit_name], + verbosity=CallVerbosity.DEBUG) + call(self.ctx, ['systemctl', 'reset-failed', self.unit_name], + verbosity=CallVerbosity.DEBUG) + call_throws(self.ctx, ['systemctl', 'enable', '--now', self.unit_name]) + + @classmethod + def uninstall(cls, ctx: CephadmContext, fsid: str, daemon_type: str, daemon_id: str) -> None: + unit_name = CephadmDaemon._unit_name(fsid, daemon_id) + unit_path = os.path.join(ctx.unit_dir, unit_name) + unit_run = os.path.join(ctx.data_dir, fsid, f'{daemon_type}.{daemon_id}', 'unit.run') + port = None + try: + with open(unit_run, 'r') as u: + contents = u.read().strip(' &') + except OSError: + logger.warning(f'Unable to access the unit.run file @ {unit_run}') + return + + port = None + for line in contents.split('\n'): + if '--port ' in line: + try: + port = int(line.split('--port ')[-1]) + except ValueError: + logger.warning('Unexpected format in unit.run file: port is not numeric') + logger.warning('Unable to remove the systemd file and close the port') + return + break + + if port: + fw = Firewalld(ctx) + try: + fw.close_ports([port]) + except RuntimeError: + logger.error(f'Unable to close port {port}') + + stdout, stderr, rc = call(ctx, ['rm', '-f', unit_path]) + if rc: + logger.error(f'Unable to remove the systemd file @ {unit_path}') + else: + logger.info(f'removed systemd unit file @ {unit_path}') + stdout, stderr, rc = call(ctx, ['systemctl', 'daemon-reload']) + + +def command_exporter(ctx: CephadmContext) -> None: + exporter = CephadmDaemon(ctx, ctx.fsid, daemon_id=ctx.id, port=ctx.port) + + if ctx.fsid not in os.listdir(ctx.data_dir): + raise Error(f"cluster fsid '{ctx.fsid}' not found in '{ctx.data_dir}'") + + exporter.run() + +################################## + + +def systemd_target_state(ctx: CephadmContext, target_name: str, subsystem: str = 'ceph') -> bool: + # TODO: UNITTEST + return os.path.exists( + os.path.join( + ctx.unit_dir, + f'{subsystem}.target.wants', + target_name + ) + ) + + +def target_exists(ctx: CephadmContext) -> bool: + return os.path.exists(ctx.unit_dir + '/ceph.target') + + +@infer_fsid +def command_maintenance(ctx: CephadmContext) -> str: + if not ctx.fsid: + raise Error('failed - must pass --fsid to specify cluster') + + target = f'ceph-{ctx.fsid}.target' + + if ctx.maintenance_action.lower() == 'enter': + logger.info('Requested to place host into maintenance') + if systemd_target_state(ctx, target): + _out, _err, code = call(ctx, + ['systemctl', 'disable', target], + verbosity=CallVerbosity.DEBUG) + if code: + logger.error(f'Failed to disable the {target} target') + return 'failed - to disable the target' + else: + # stopping a target waits by default + _out, _err, code = call(ctx, + ['systemctl', 'stop', target], + verbosity=CallVerbosity.DEBUG) + if code: + logger.error(f'Failed to stop the {target} target') + return 'failed - to disable the target' + else: + return f'success - systemd target {target} disabled' + + else: + return 'skipped - target already disabled' + + else: + logger.info('Requested to exit maintenance state') + # if we've never deployed a daemon on this host there will be no systemd + # target to disable so attempting a disable will fail. We still need to + # return success here or host will be permanently stuck in maintenance mode + # as no daemons can be deployed so no systemd target will ever exist to disable. + if not target_exists(ctx): + return 'skipped - systemd target not present on this host. Host removed from maintenance mode.' + # exit maintenance request + if not systemd_target_state(ctx, target): + _out, _err, code = call(ctx, + ['systemctl', 'enable', target], + verbosity=CallVerbosity.DEBUG) + if code: + logger.error(f'Failed to enable the {target} target') + return 'failed - unable to enable the target' + else: + # starting a target waits by default + _out, _err, code = call(ctx, + ['systemctl', 'start', target], + verbosity=CallVerbosity.DEBUG) + if code: + logger.error(f'Failed to start the {target} target') + return 'failed - unable to start the target' + else: + return f'success - systemd target {target} enabled and started' + return f'success - systemd target {target} enabled and started' + +################################## + + +def _get_parser(): + # type: () -> argparse.ArgumentParser + parser = argparse.ArgumentParser( + description='Bootstrap Ceph daemons with systemd and containers.', + formatter_class=argparse.ArgumentDefaultsHelpFormatter) + parser.add_argument( + '--image', + help='container image. Can also be set via the "CEPHADM_IMAGE" ' + 'env var') + parser.add_argument( + '--docker', + action='store_true', + help='use docker instead of podman') + parser.add_argument( + '--data-dir', + default=DATA_DIR, + help='base directory for daemon data') + parser.add_argument( + '--log-dir', + default=LOG_DIR, + help='base directory for daemon logs') + parser.add_argument( + '--logrotate-dir', + default=LOGROTATE_DIR, + help='location of logrotate configuration files') + parser.add_argument( + '--sysctl-dir', + default=SYSCTL_DIR, + help='location of sysctl configuration files') + parser.add_argument( + '--unit-dir', + default=UNIT_DIR, + help='base directory for systemd units') + parser.add_argument( + '--verbose', '-v', + action='store_true', + help='Show debug-level log messages') + parser.add_argument( + '--timeout', + type=int, + default=DEFAULT_TIMEOUT, + help='timeout in seconds') + parser.add_argument( + '--retry', + type=int, + default=DEFAULT_RETRY, + help='max number of retries') + parser.add_argument( + '--env', '-e', + action='append', + default=[], + help='set environment variable') + parser.add_argument( + '--no-container-init', + action='store_true', + default=not CONTAINER_INIT, + help='Do not run podman/docker with `--init`') + + subparsers = parser.add_subparsers(help='sub-command') + + parser_version = subparsers.add_parser( + 'version', help='get ceph version from container') + parser_version.set_defaults(func=command_version) + + parser_pull = subparsers.add_parser( + 'pull', help='pull the default container image') + parser_pull.set_defaults(func=command_pull) + parser_pull.add_argument( + '--insecure', + action='store_true', + help=argparse.SUPPRESS, + ) + + parser_inspect_image = subparsers.add_parser( + 'inspect-image', help='inspect local container image') + parser_inspect_image.set_defaults(func=command_inspect_image) + + parser_ls = subparsers.add_parser( + 'ls', help='list daemon instances on this host') + parser_ls.set_defaults(func=command_ls) + parser_ls.add_argument( + '--no-detail', + action='store_true', + help='Do not include daemon status') + parser_ls.add_argument( + '--legacy-dir', + default='/', + help='base directory for legacy daemon data') + + parser_list_networks = subparsers.add_parser( + 'list-networks', help='list IP networks') + parser_list_networks.set_defaults(func=command_list_networks) + + parser_adopt = subparsers.add_parser( + 'adopt', help='adopt daemon deployed with a different tool') + parser_adopt.set_defaults(func=command_adopt) + parser_adopt.add_argument( + '--name', '-n', + required=True, + help='daemon name (type.id)') + parser_adopt.add_argument( + '--style', + required=True, + help='deployment style (legacy, ...)') + parser_adopt.add_argument( + '--cluster', + default='ceph', + help='cluster name') + parser_adopt.add_argument( + '--legacy-dir', + default='/', + help='base directory for legacy daemon data') + parser_adopt.add_argument( + '--config-json', + help='Additional configuration information in JSON format') + parser_adopt.add_argument( + '--skip-firewalld', + action='store_true', + help='Do not configure firewalld') + parser_adopt.add_argument( + '--skip-pull', + action='store_true', + help='do not pull the default image before adopting') + parser_adopt.add_argument( + '--force-start', + action='store_true', + help='start newly adoped daemon, even if it was not running previously') + parser_adopt.add_argument( + '--container-init', + action='store_true', + default=CONTAINER_INIT, + help=argparse.SUPPRESS) + + parser_rm_daemon = subparsers.add_parser( + 'rm-daemon', help='remove daemon instance') + parser_rm_daemon.set_defaults(func=command_rm_daemon) + parser_rm_daemon.add_argument( + '--name', '-n', + required=True, + action=CustomValidation, + help='daemon name (type.id)') + parser_rm_daemon.add_argument( + '--tcp-ports', + help='List of tcp ports to close in the host firewall') + parser_rm_daemon.add_argument( + '--fsid', + required=True, + help='cluster FSID') + parser_rm_daemon.add_argument( + '--force', + action='store_true', + help='proceed, even though this may destroy valuable data') + parser_rm_daemon.add_argument( + '--force-delete-data', + action='store_true', + help='delete valuable daemon data instead of making a backup') + + parser_rm_cluster = subparsers.add_parser( + 'rm-cluster', help='remove all daemons for a cluster') + parser_rm_cluster.set_defaults(func=command_rm_cluster) + parser_rm_cluster.add_argument( + '--fsid', + required=True, + help='cluster FSID') + parser_rm_cluster.add_argument( + '--force', + action='store_true', + help='proceed, even though this may destroy valuable data') + parser_rm_cluster.add_argument( + '--keep-logs', + action='store_true', + help='do not remove log files') + parser_rm_cluster.add_argument( + '--zap-osds', + action='store_true', + help='zap OSD devices for this cluster') + + parser_run = subparsers.add_parser( + 'run', help='run a ceph daemon, in a container, in the foreground') + parser_run.set_defaults(func=command_run) + parser_run.add_argument( + '--name', '-n', + required=True, + help='daemon name (type.id)') + parser_run.add_argument( + '--fsid', + required=True, + help='cluster FSID') + + parser_shell = subparsers.add_parser( + 'shell', help='run an interactive shell inside a daemon container') + parser_shell.set_defaults(func=command_shell) + parser_shell.add_argument( + '--shared_ceph_folder', + metavar='CEPH_SOURCE_FOLDER', + help='Development mode. Several folders in containers are volumes mapped to different sub-folders in the ceph source folder') + parser_shell.add_argument( + '--fsid', + help='cluster FSID') + parser_shell.add_argument( + '--name', '-n', + help='daemon name (type.id)') + parser_shell.add_argument( + '--config', '-c', + help='ceph.conf to pass through to the container') + parser_shell.add_argument( + '--keyring', '-k', + help='ceph.keyring to pass through to the container') + parser_shell.add_argument( + '--mount', '-m', + help=('mount a file or directory in the container. ' + 'Support multiple mounts. ' + 'ie: `--mount /foo /bar:/bar`. ' + 'When no destination is passed, default is /mnt'), + nargs='+') + parser_shell.add_argument( + '--env', '-e', + action='append', + default=[], + help='set environment variable') + parser_shell.add_argument( + '--volume', '-v', + action='append', + default=[], + help='set environment variable') + parser_shell.add_argument( + 'command', nargs=argparse.REMAINDER, + help='command (optional)') + parser_shell.add_argument( + '--no-hosts', + action='store_true', + help='dont pass /etc/hosts through to the container') + + parser_enter = subparsers.add_parser( + 'enter', help='run an interactive shell inside a running daemon container') + parser_enter.set_defaults(func=command_enter) + parser_enter.add_argument( + '--fsid', + help='cluster FSID') + parser_enter.add_argument( + '--name', '-n', + required=True, + help='daemon name (type.id)') + parser_enter.add_argument( + 'command', nargs=argparse.REMAINDER, + help='command') + + parser_ceph_volume = subparsers.add_parser( + 'ceph-volume', help='run ceph-volume inside a container') + parser_ceph_volume.set_defaults(func=command_ceph_volume) + parser_ceph_volume.add_argument( + '--shared_ceph_folder', + metavar='CEPH_SOURCE_FOLDER', + help='Development mode. Several folders in containers are volumes mapped to different sub-folders in the ceph source folder') + parser_ceph_volume.add_argument( + '--fsid', + help='cluster FSID') + parser_ceph_volume.add_argument( + '--config-json', + help='JSON file with config and (client.bootrap-osd) key') + parser_ceph_volume.add_argument( + '--config', '-c', + help='ceph conf file') + parser_ceph_volume.add_argument( + '--keyring', '-k', + help='ceph.keyring to pass through to the container') + parser_ceph_volume.add_argument( + 'command', nargs=argparse.REMAINDER, + help='command') + + parser_zap_osds = subparsers.add_parser( + 'zap-osds', help='zap all OSDs associated with a particular fsid') + parser_zap_osds.set_defaults(func=command_zap_osds) + parser_zap_osds.add_argument( + '--fsid', + required=True, + help='cluster FSID') + parser_zap_osds.add_argument( + '--force', + action='store_true', + help='proceed, even though this may destroy valuable data') + + parser_unit = subparsers.add_parser( + 'unit', help="operate on the daemon's systemd unit") + parser_unit.set_defaults(func=command_unit) + parser_unit.add_argument( + 'command', + help='systemd command (start, stop, restart, enable, disable, ...)') + parser_unit.add_argument( + '--fsid', + help='cluster FSID') + parser_unit.add_argument( + '--name', '-n', + required=True, + help='daemon name (type.id)') + + parser_logs = subparsers.add_parser( + 'logs', help='print journald logs for a daemon container') + parser_logs.set_defaults(func=command_logs) + parser_logs.add_argument( + '--fsid', + help='cluster FSID') + parser_logs.add_argument( + '--name', '-n', + required=True, + help='daemon name (type.id)') + parser_logs.add_argument( + 'command', nargs='*', + help='additional journalctl args') + + parser_bootstrap = subparsers.add_parser( + 'bootstrap', help='bootstrap a cluster (mon + mgr daemons)') + parser_bootstrap.set_defaults(func=command_bootstrap) + parser_bootstrap.add_argument( + '--config', '-c', + help='ceph conf file to incorporate') + parser_bootstrap.add_argument( + '--mon-id', + required=False, + help='mon id (default: local hostname)') + group = parser_bootstrap.add_mutually_exclusive_group() + group.add_argument( + '--mon-addrv', + help='mon IPs (e.g., [v2:localipaddr:3300,v1:localipaddr:6789])') + group.add_argument( + '--mon-ip', + help='mon IP') + parser_bootstrap.add_argument( + '--mgr-id', + required=False, + help='mgr id (default: randomly generated)') + parser_bootstrap.add_argument( + '--fsid', + help='cluster FSID') + parser_bootstrap.add_argument( + '--output-dir', + default='/etc/ceph', + help='directory to write config, keyring, and pub key files') + parser_bootstrap.add_argument( + '--output-keyring', + help='location to write keyring file with new cluster admin and mon keys') + parser_bootstrap.add_argument( + '--output-config', + help='location to write conf file to connect to new cluster') + parser_bootstrap.add_argument( + '--output-pub-ssh-key', + help="location to write the cluster's public SSH key") + parser_bootstrap.add_argument( + '--skip-admin-label', + action='store_true', + help='do not create admin label for ceph.conf and client.admin keyring distribution') + parser_bootstrap.add_argument( + '--skip-ssh', + action='store_true', + help='skip setup of ssh key on local host') + parser_bootstrap.add_argument( + '--initial-dashboard-user', + default='admin', + help='Initial user for the dashboard') + parser_bootstrap.add_argument( + '--initial-dashboard-password', + help='Initial password for the initial dashboard user') + parser_bootstrap.add_argument( + '--ssl-dashboard-port', + type=int, + default=8443, + help='Port number used to connect with dashboard using SSL') + parser_bootstrap.add_argument( + '--dashboard-key', + type=argparse.FileType('r'), + help='Dashboard key') + parser_bootstrap.add_argument( + '--dashboard-crt', + type=argparse.FileType('r'), + help='Dashboard certificate') + + parser_bootstrap.add_argument( + '--ssh-config', + type=argparse.FileType('r'), + help='SSH config') + parser_bootstrap.add_argument( + '--ssh-private-key', + type=argparse.FileType('r'), + help='SSH private key') + parser_bootstrap.add_argument( + '--ssh-public-key', + type=argparse.FileType('r'), + help='SSH public key') + parser_bootstrap.add_argument( + '--ssh-user', + default='root', + help='set user for SSHing to cluster hosts, passwordless sudo will be needed for non-root users') + parser_bootstrap.add_argument( + '--skip-mon-network', + action='store_true', + help='set mon public_network based on bootstrap mon ip') + parser_bootstrap.add_argument( + '--skip-dashboard', + action='store_true', + help='do not enable the Ceph Dashboard') + parser_bootstrap.add_argument( + '--dashboard-password-noupdate', + action='store_true', + help='stop forced dashboard password change') + parser_bootstrap.add_argument( + '--no-minimize-config', + action='store_true', + help='do not assimilate and minimize the config file') + parser_bootstrap.add_argument( + '--skip-ping-check', + action='store_true', + help='do not verify that mon IP is pingable') + parser_bootstrap.add_argument( + '--skip-pull', + action='store_true', + help='do not pull the default image before bootstrapping') + parser_bootstrap.add_argument( + '--skip-firewalld', + action='store_true', + help='Do not configure firewalld') + parser_bootstrap.add_argument( + '--allow-overwrite', + action='store_true', + help='allow overwrite of existing --output-* config/keyring/ssh files') + parser_bootstrap.add_argument( + '--allow-fqdn-hostname', + action='store_true', + help='allow hostname that is fully-qualified (contains ".")') + parser_bootstrap.add_argument( + '--allow-mismatched-release', + action='store_true', + help="allow bootstrap of ceph that doesn't match this version of cephadm") + parser_bootstrap.add_argument( + '--skip-prepare-host', + action='store_true', + help='Do not prepare host') + parser_bootstrap.add_argument( + '--orphan-initial-daemons', + action='store_true', + help='Set mon and mgr service to `unmanaged`, Do not create the crash service') + parser_bootstrap.add_argument( + '--skip-monitoring-stack', + action='store_true', + help='Do not automatically provision monitoring stack (prometheus, grafana, alertmanager, node-exporter)') + parser_bootstrap.add_argument( + '--apply-spec', + help='Apply cluster spec after bootstrap (copy ssh key, add hosts and apply services)') + parser_bootstrap.add_argument( + '--shared_ceph_folder', + metavar='CEPH_SOURCE_FOLDER', + help='Development mode. Several folders in containers are volumes mapped to different sub-folders in the ceph source folder') + + parser_bootstrap.add_argument( + '--registry-url', + help='url for custom registry') + parser_bootstrap.add_argument( + '--registry-username', + help='username for custom registry') + parser_bootstrap.add_argument( + '--registry-password', + help='password for custom registry') + parser_bootstrap.add_argument( + '--registry-json', + help='json file with custom registry login info (URL, Username, Password)') + parser_bootstrap.add_argument( + '--container-init', + action='store_true', + default=CONTAINER_INIT, + help=argparse.SUPPRESS) + parser_bootstrap.add_argument( + '--with-exporter', + action='store_true', + help='Automatically deploy cephadm metadata exporter to each node') + parser_bootstrap.add_argument( + '--exporter-config', + action=CustomValidation, + help=f'Exporter configuration information in JSON format (providing: {", ".join(CephadmDaemon.config_requirements)}, port information)') + parser_bootstrap.add_argument( + '--cluster-network', + help='subnet to use for cluster replication, recovery and heartbeats (in CIDR notation network/mask)') + parser_bootstrap.add_argument( + '--single-host-defaults', + action='store_true', + help='adjust configuration defaults to suit a single-host cluster') + parser_bootstrap.add_argument( + '--log-to-file', + action='store_true', + help='configure cluster to log to traditional log files in /var/log/ceph/$fsid') + + parser_deploy = subparsers.add_parser( + 'deploy', help='deploy a daemon') + parser_deploy.set_defaults(func=command_deploy) + parser_deploy.add_argument( + '--name', + required=True, + action=CustomValidation, + help='daemon name (type.id)') + parser_deploy.add_argument( + '--fsid', + required=True, + help='cluster FSID') + parser_deploy.add_argument( + '--config', '-c', + help='config file for new daemon') + parser_deploy.add_argument( + '--config-json', + help='Additional configuration information in JSON format') + parser_deploy.add_argument( + '--keyring', + help='keyring for new daemon') + parser_deploy.add_argument( + '--key', + help='key for new daemon') + parser_deploy.add_argument( + '--osd-fsid', + help='OSD uuid, if creating an OSD container') + parser_deploy.add_argument( + '--skip-firewalld', + action='store_true', + help='Do not configure firewalld') + parser_deploy.add_argument( + '--tcp-ports', + help='List of tcp ports to open in the host firewall') + parser_deploy.add_argument( + '--reconfig', + action='store_true', + help='Reconfigure a previously deployed daemon') + parser_deploy.add_argument( + '--allow-ptrace', + action='store_true', + help='Allow SYS_PTRACE on daemon container') + parser_deploy.add_argument( + '--container-init', + action='store_true', + default=CONTAINER_INIT, + help=argparse.SUPPRESS) + parser_deploy.add_argument( + '--memory-request', + help='Container memory request/target' + ) + parser_deploy.add_argument( + '--memory-limit', + help='Container memory hard limit' + ) + parser_deploy.add_argument( + '--meta-json', + help='JSON dict of additional metadata' + ) + parser_deploy.add_argument( + '--extra-container-args', + action='append', + default=[], + help='Additional container arguments to apply to deamon' + ) + + parser_check_host = subparsers.add_parser( + 'check-host', help='check host configuration') + parser_check_host.set_defaults(func=command_check_host) + parser_check_host.add_argument( + '--expect-hostname', + help='Check that hostname matches an expected value') + + parser_prepare_host = subparsers.add_parser( + 'prepare-host', help='prepare a host for cephadm use') + parser_prepare_host.set_defaults(func=command_prepare_host) + parser_prepare_host.add_argument( + '--expect-hostname', + help='Set hostname') + + parser_add_repo = subparsers.add_parser( + 'add-repo', help='configure package repository') + parser_add_repo.set_defaults(func=command_add_repo) + parser_add_repo.add_argument( + '--release', + help='use latest version of a named release (e.g., {})'.format(LATEST_STABLE_RELEASE)) + parser_add_repo.add_argument( + '--version', + help='use specific upstream version (x.y.z)') + parser_add_repo.add_argument( + '--dev', + help='use specified bleeding edge build from git branch or tag') + parser_add_repo.add_argument( + '--dev-commit', + help='use specified bleeding edge build from git commit') + parser_add_repo.add_argument( + '--gpg-url', + help='specify alternative GPG key location') + parser_add_repo.add_argument( + '--repo-url', + default='https://download.ceph.com', + help='specify alternative repo location') + # TODO: proxy? + + parser_rm_repo = subparsers.add_parser( + 'rm-repo', help='remove package repository configuration') + parser_rm_repo.set_defaults(func=command_rm_repo) + + parser_install = subparsers.add_parser( + 'install', help='install ceph package(s)') + parser_install.set_defaults(func=command_install) + parser_install.add_argument( + 'packages', nargs='*', + default=['cephadm'], + help='packages') + + parser_registry_login = subparsers.add_parser( + 'registry-login', help='log host into authenticated registry') + parser_registry_login.set_defaults(func=command_registry_login) + parser_registry_login.add_argument( + '--registry-url', + help='url for custom registry') + parser_registry_login.add_argument( + '--registry-username', + help='username for custom registry') + parser_registry_login.add_argument( + '--registry-password', + help='password for custom registry') + parser_registry_login.add_argument( + '--registry-json', + help='json file with custom registry login info (URL, Username, Password)') + parser_registry_login.add_argument( + '--fsid', + help='cluster FSID') + + parser_gather_facts = subparsers.add_parser( + 'gather-facts', help='gather and return host related information (JSON format)') + parser_gather_facts.set_defaults(func=command_gather_facts) + + parser_exporter = subparsers.add_parser( + 'exporter', help='Start cephadm in exporter mode (web service), providing host/daemon/disk metadata') + parser_exporter.add_argument( + '--fsid', + required=True, + type=str, + help='fsid of the cephadm exporter to run against') + parser_exporter.add_argument( + '--port', + type=int, + default=int(CephadmDaemon.default_port), + help='port number for the cephadm exporter service') + parser_exporter.add_argument( + '--id', + type=str, + default=get_hostname().split('.')[0], + help='daemon identifer for the exporter') + parser_exporter.set_defaults(func=command_exporter) + + parser_maintenance = subparsers.add_parser( + 'host-maintenance', help='Manage the maintenance state of a host') + parser_maintenance.add_argument( + '--fsid', + help='cluster FSID') + parser_maintenance.add_argument( + 'maintenance_action', + type=str, + choices=['enter', 'exit'], + help='Maintenance action - enter maintenance, or exit maintenance') + parser_maintenance.set_defaults(func=command_maintenance) + + parser_disk_rescan = subparsers.add_parser( + 'disk-rescan', help='rescan all HBAs to detect new/removed devices') + parser_disk_rescan.set_defaults(func=command_rescan_disks) + + return parser + + +def _parse_args(av: List[str]) -> argparse.Namespace: + parser = _get_parser() + + args = parser.parse_args(av) + if 'command' in args and args.command and args.command[0] == '--': + args.command.pop(0) + + # workaround argparse to deprecate the subparser `--container-init` flag + # container_init and no_container_init must always be mutually exclusive + container_init_args = ('--container-init', '--no-container-init') + if set(container_init_args).issubset(av): + parser.error('argument %s: not allowed with argument %s' % (container_init_args)) + elif '--container-init' in av: + args.no_container_init = not args.container_init + else: + args.container_init = not args.no_container_init + assert args.container_init is not args.no_container_init + + return args + + +def cephadm_init_ctx(args: List[str]) -> CephadmContext: + ctx = CephadmContext() + ctx.set_args(_parse_args(args)) + return ctx + + +def cephadm_init_logging(ctx: CephadmContext, args: List[str]) -> None: + """Configure the logging for cephadm as well as updating the system + to have the expected log dir and logrotate configuration. + """ + global logger + if not os.path.exists(LOG_DIR): + os.makedirs(LOG_DIR) + dictConfig(logging_config) + logger = logging.getLogger() + + if not os.path.exists(ctx.logrotate_dir + '/cephadm'): + with open(ctx.logrotate_dir + '/cephadm', 'w') as f: + f.write("""# created by cephadm +/var/log/ceph/cephadm.log { + rotate 7 + daily + compress + missingok + notifempty + su root root +} +""") + + if ctx.verbose: + for handler in logger.handlers: + if handler.name == 'console': + handler.setLevel(logging.DEBUG) + logger.debug('%s\ncephadm %s' % ('-' * 80, args)) + + +def cephadm_require_root() -> None: + """Exit if the process is not running as root.""" + if os.geteuid() != 0: + sys.stderr.write('ERROR: cephadm should be run as root\n') + sys.exit(1) + + +def main() -> None: + av: List[str] = [] + av = sys.argv[1:] + + ctx = cephadm_init_ctx(av) + if not ctx.has_function(): + sys.stderr.write('No command specified; pass -h or --help for usage\n') + sys.exit(1) + + cephadm_require_root() + cephadm_init_logging(ctx, av) + try: + # podman or docker? + ctx.container_engine = find_container_engine(ctx) + if ctx.func not in \ + [ + command_check_host, + command_prepare_host, + command_add_repo, + command_rm_repo, + command_install + ]: + check_container_engine(ctx) + # command handler + r = ctx.func(ctx) + except Error as e: + if ctx.verbose: + raise + logger.error('ERROR: %s' % e) + sys.exit(1) + if not r: + r = 0 + sys.exit(r) + + +if __name__ == '__main__': + main() diff --git a/src/cephadm/samples/alertmanager.json b/src/cephadm/samples/alertmanager.json new file mode 100644 index 000000000..bacbad300 --- /dev/null +++ b/src/cephadm/samples/alertmanager.json @@ -0,0 +1,27 @@ +{ + "files": { + "alertmanager.yml": [ + "global:", + " resolve_timeout: 5m", + "", + "route:", + " group_by: ['alertname']", + " group_wait: 10s", + " group_interval: 10s", + " repeat_interval: 1h", + " receiver: 'web.hook'", + "receivers:", + "- name: 'web.hook'", + " webhook_configs:", + " - url: 'http://127.0.0.1:5001/'", + "inhibit_rules:", + " - source_match:", + " severity: 'critical'", + " target_match:", + " severity: 'warning'", + " equal: ['alertname', 'dev', 'instance']" + ] + }, + "peers": [] +} + diff --git a/src/cephadm/samples/custom_container.json b/src/cephadm/samples/custom_container.json new file mode 100644 index 000000000..194a44d2a --- /dev/null +++ b/src/cephadm/samples/custom_container.json @@ -0,0 +1,35 @@ +{ + "image": "docker.io/prom/alertmanager:v0.20.0", + "ports": [9093, 9094], + "args": [ + "-p", "9093:9093", + "-p", "9094:9094" + ], + "dirs": ["etc/alertmanager"], + "files": { + "etc/alertmanager/alertmanager.yml": [ + "global:", + " resolve_timeout: 5m", + "", + "route:", + " group_by: ['alertname']", + " group_wait: 10s", + " group_interval: 10s", + " repeat_interval: 1h", + " receiver: 'web.hook'", + "receivers:", + "- name: 'web.hook'", + " webhook_configs:", + " - url: 'http://127.0.0.1:5001/'", + "inhibit_rules:", + " - source_match:", + " severity: 'critical'", + " target_match:", + " severity: 'warning'", + " equal: ['alertname', 'dev', 'instance']" + ] + }, + "volume_mounts": { + "etc/alertmanager": "/etc/alertmanager" + } +} diff --git a/src/cephadm/samples/grafana.json b/src/cephadm/samples/grafana.json new file mode 100644 index 000000000..0e0689b7e --- /dev/null +++ b/src/cephadm/samples/grafana.json @@ -0,0 +1,90 @@ +{ + "files": { + "grafana.ini": [ + "[users]", + " default_theme = light", + "[auth.anonymous]", + " enabled = true", + " org_name = 'Main Org.'", + " org_role = 'Viewer'", + "[server]", + " domain = 'bootstrap.storage.lab'", + " protocol = https", + " cert_file = /etc/grafana/certs/cert_file", + " cert_key = /etc/grafana/certs/cert_key", + " http_port = 3000", + " http_addr = localhost", + "[security]", + " admin_user = admin", + " admin_password = admin", + " allow_embedding = true" + ], + "provisioning/datasources/ceph-dashboard.yml": [ + "deleteDatasources:", + " - name: 'Dashboard'", + " orgId: 1", + " ", + "datasources:", + " - name: 'Dashboard'", + " type: 'prometheus'", + " access: 'proxy'", + " orgId: 1", + " url: 'http://localhost:9095'", + " basicAuth: false", + " isDefault: true", + " editable: false" + ], + "certs/cert_file": [ + "-----BEGIN CERTIFICATE-----", + "MIIDLTCCAhWgAwIBAgIUEH0mq6u93LKsWlNXst5pxWcuqkQwDQYJKoZIhvcNAQEL", + "BQAwJjELMAkGA1UECgwCSVQxFzAVBgNVBAMMDmNlcGgtZGFzaGJvYXJkMB4XDTIw", + "MDEwNTIyNDYyMFoXDTMwMDEwMjIyNDYyMFowJjELMAkGA1UECgwCSVQxFzAVBgNV", + "BAMMDmNlcGgtZGFzaGJvYXJkMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKC", + "AQEAqxh6eO0NTZJe+DoKZG/kozJCf+83eB3gWzwXoNinRmV/49f5WPR20DIxAe0R", + "saO6XynJXTrhvXT1bsARUq+LSmjWNFoYXopFuOJhGdWn4dmpuHwtpcFv2kjzNOKj", + "U2EG8j6bsRp1jFAzn7kdbSWT0UHySRXp9DPAjDiF3LjykMXiJMReccFXrB1pRi93", + "nJxED8d6oT5GazGB44svb+Zi6ABamZu5SDJC1Fr/O5rWFNQkH4hQEqDPj1817H9O", + "sm0mZiNy77ZQuAzOgZN153L3QOsyJismwNHfAMGMH9mzPKOjyhc13VlZyeEzml8p", + "ZpWQ2gi8P2r/FAr8bFL3MFnHKwIDAQABo1MwUTAdBgNVHQ4EFgQUZg3v7MX4J+hx", + "w3HENCrUkMK8tbwwHwYDVR0jBBgwFoAUZg3v7MX4J+hxw3HENCrUkMK8tbwwDwYD", + "VR0TAQH/BAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAQEAaR/XPGKwUgVwH3KXAb6+", + "s9NTAt6lCmFdQz1ngoqFSizW7KGSXnOgd6xTiUCR0Tjjo2zKCwhIINaI6mwqMbrg", + "BOjb7diaqwFaitRs27AtdmaqMGndUqEBUn/k64Ld3VPGL4p0W2W+tXsyzZg1qQIn", + "JXb7c4+oWzXny7gHFheYQTwnHzDcNOf9vJiMGyYYvU1xTOGucu6dwtOVDDe1Z4Nq", + "AyIYWDScRr2FeAOXyx4aW2v5bjpTxvP+79/OOBbQ+p4y5F4PDrPeOSweGoo6huTR", + "+T+YI9Jfw2XCgV7NHWhfdt3fHHwUQzO6WszWU557pmCODLvXWsQ8P+GRiG7Nywm3", + "uA==", + "-----END CERTIFICATE-----" + ], + "certs/cert_key": [ + "-----BEGIN PRIVATE KEY-----", + "MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQCrGHp47Q1Nkl74", + "Ogpkb+SjMkJ/7zd4HeBbPBeg2KdGZX/j1/lY9HbQMjEB7RGxo7pfKcldOuG9dPVu", + "wBFSr4tKaNY0WhheikW44mEZ1afh2am4fC2lwW/aSPM04qNTYQbyPpuxGnWMUDOf", + "uR1tJZPRQfJJFen0M8CMOIXcuPKQxeIkxF5xwVesHWlGL3ecnEQPx3qhPkZrMYHj", + "iy9v5mLoAFqZm7lIMkLUWv87mtYU1CQfiFASoM+PXzXsf06ybSZmI3LvtlC4DM6B", + "k3XncvdA6zImKybA0d8AwYwf2bM8o6PKFzXdWVnJ4TOaXylmlZDaCLw/av8UCvxs", + "UvcwWccrAgMBAAECggEAeBv0BiYrm5QwdUORfhaKxAIJavRM1Vbr5EBYOgM90o54", + "bEN2ePsM2XUSsE5ziGfu8tVL1dX7GNwdW8UbpBc1ymO0VAYXa27YKUVKcy9o7oS1", + "v5v1E5Kq6esiSLL9gw/vJ2nKNFblxD2dL/hs7u1dSp5n7uSiW1tlRUp8toljRzts", + "1Cenp0J/a82HwWDE8j/H9NvitTOZ2cdwJ76V8GkBynlvr2ARjRfZGx0WXEJmoZYD", + "YUQVU303DB6Q2tkFco4LbPofkuhhMPhXsz3fZ/blHj/c78tqP9L5sQ29oqoPE1pS", + "DBOwKC/eoi5FY34RdLNL0dKq9MzbuYqEcCfZOJgxoQKBgQDf+5XF+aXQz2OmSaj6", + "1Yr+3KAKdfX/AYp22X1Wy4zWcZlgujgwQ1FG0zay8HVBM0/xn4UgOtcKCoXibePh", + "ag1t8aZINdRE1JcMzKmZoSvU9Xk30CNvygizuJVEKsJFPDbPzCpauDSplzcQb4pZ", + "wepucPuowkPMBx0iU3x0qSThWwKBgQDDjYs7d30xxSqWWXyCOZshy7UtHMNfqP15", + "kDfTXIZzuHvDf6ZNci10VY1eDZbpZfHgc6x1ElbKv2H4dYsgkENJZUi1YQDpVPKq", + "4N5teNykgAuagiR7dRFltSju3S7hIE6HInTv3hShaFPymlEE7zuBMuEUcuvYz5YN", + "RjxsvypKcQKBgCuuV+Y1KqZPW8K5SNAqRyIvCrMfkCr8NPG6tpvvtHa5zsyzZHPd", + "HQOv+1HoXSWrCSM5FfBUKU3XAYdIIRH76cSQRPp+LPiDcTXY0Baa/P5aJRrCZ7bM", + "cugBznJt2FdCR/o8eeIZXIPabq2w4w1gKQUC2cFuqWQn2wGvwGzL89pTAoGAAfpx", + "mSVpT9KVzrWTC+I3To04BP/QfixAfDVYSzwZZBxOrDijXw8zpISlDHmIuE2+t62T", + "5g9Mb3qmLBRMVwT+mUR8CtGzZ6jjV5U0yti5KrTc6TA93D3f8i51/oygR8jC4p0X", + "n8GYZdWfW8nx3eHpsTHpkwJinmvjMbkvLU51yBECgYAnUAMyhNOWjbYS5QWd8i1W", + "SFQansVDeeT98RebrzmGwlgrCImHItJz0Tz8gkNB3+S2B2balqT0WHaDxQ8vCtwX", + "xB4wd+gMomgdYtHGRnRwj1UyRXDk0c1TgGdRjOn3URaezBMibHTQSbFgPciJgAuU", + "mEl75h1ToBX9yvnH39o50g==", + "-----END PRIVATE KEY-----" + ] + } +} diff --git a/src/cephadm/samples/nfs.json b/src/cephadm/samples/nfs.json new file mode 100644 index 000000000..2e6625101 --- /dev/null +++ b/src/cephadm/samples/nfs.json @@ -0,0 +1,14 @@ +{ + "pool" : "nfs-ganesha", + "namespace" : "nfs-ns", + "files": { + "ganesha.conf": [ + "RADOS_URLS {", + " userid = admin;", + "}", + "", + "%url rados://nfs-ganesha/nfs-ns/conf-nfs.a", + "" + ] + } +} diff --git a/src/cephadm/samples/prometheus.json b/src/cephadm/samples/prometheus.json new file mode 100644 index 000000000..64727fb59 --- /dev/null +++ b/src/cephadm/samples/prometheus.json @@ -0,0 +1,17 @@ +{ + "files": { + "prometheus.yml": [ + "global:", + " scrape_interval: 5s", + " evaluation_interval: 10s", + "", + "rule_files: ", + " - '/etc/prometheus/alerting/*'", + "", + "scrape_configs:", + " - job_name: 'prometheus'", + " static_configs:", + " - targets: ['localhost:9095']" + ] + } +} diff --git a/src/cephadm/samples/rgw_ssl.json b/src/cephadm/samples/rgw_ssl.json new file mode 100644 index 000000000..3fe6fea1c --- /dev/null +++ b/src/cephadm/samples/rgw_ssl.json @@ -0,0 +1,101 @@ +{ + "rgw_realm": "default", + "rgw_zone": "default", + "service_type": "rgw", + "placement": { + "hosts": [{ + "hostname": "ironic-moliver", + "name": "", + "network": "" + }], + "count": 1 + }, + "ssl": true, + "rgw_frontend_port": 4343, + "rgw_frontend_ssl_certificate": [ + "-----BEGIN CERTIFICATE-----", + "MIIFmjCCA4KgAwIBAgIJAIZ2n35bmwXTMA0GCSqGSIb3DQEBCwUAMGIxCzAJBgNV", + "BAYTAkFVMQwwCgYDVQQIDANOU1cxHTAbBgNVBAoMFEV4YW1wbGUgUkdXIFNTTCBp", + "bmMuMSYwJAYDVQQDDB1yZ3ctZW5kcG9pbnQuZXhhbXBsZS1jZXBoLmNvbTAeFw0y", + "MDAyMDcwMDEzNTFaFw0zMDAyMDQwMDEzNTFaMGIxCzAJBgNVBAYTAkFVMQwwCgYD", + "VQQIDANOU1cxHTAbBgNVBAoMFEV4YW1wbGUgUkdXIFNTTCBpbmMuMSYwJAYDVQQD", + "DB1yZ3ctZW5kcG9pbnQuZXhhbXBsZS1jZXBoLmNvbTCCAiIwDQYJKoZIhvcNAQEB", + "BQADggIPADCCAgoCggIBAMptGJ523QkEbc37za8iuCTahj0Zr6hy+ToSX/Vfdzxj", + "iYHuD2PiZZyJB7t2eOqiA8sQ5N513EUtf2ZIBwtnnqFIzD5TqI3BxRajUTlOyXUX", + "onMwQwXu2ifDUy3LCmuQfzanOTWvVLac1NmkWbJHpJCXYbUnPb1Nvd0QjTTEH1jt", + "5bDHhfxwCIYK6PY+MqC72a09wB2ZF+EKsSdqghOKmibfJHtoJdsqGeLrysBLrzUJ", + "e/5ZW3V4Z85T2lja5KZnWgRofrUy5TmJV10HO4Hht92xvWvEi/rmjg2AVYZFUQQx", + "xKXpUBbF5T46eSVmaT7IH88Yp5ytgBTaigym7ETCjohp/DfCaK1DUehh0ce7iUq2", + "yCLviZsX4WdPYxzkoLflNrqm4YZP6iKcZSUR/A+qPKCzCXgMXFNA1JxilDwEq35F", + "zGN++ehJqdNmOQ1eQScsLwZQa6mC97d+upWdCvyntf1+S6vNcXhtRQpjNM4W37oW", + "r5nicsGA3/0rpDEHZW85KlkdWO1uCS/6ftgt8UUMaf5ew3PigzusqymBWTlMOjtW", + "uAQXxgZZvkRp+xdspn/uTCAP+bNShGD6Q+TO3U6IjTqHk83sGKCvg2dyU/dqgPr9", + "2IIzgQBFGk0W0nM/E83E8hUSwX17COLL3drhPZb4VRMChQ8PAa6u9nIymkX2wSVv", + "AgMBAAGjUzBRMB0GA1UdDgQWBBSsZHuY7KK80RrZHp+Gx+k16skuRDAfBgNVHSME", + "GDAWgBSsZHuY7KK80RrZHp+Gx+k16skuRDAPBgNVHRMBAf8EBTADAQH/MA0GCSqG", + "SIb3DQEBCwUAA4ICAQAE+BLtnu0p8FtK7vrBCRcCdvycWaSFGJUt7r5Nm8TD7sKw", + "bWeDLgXrRouyA7n6yt/JqQbXYcxt4MLAM0P6NQd5BlNrrnDk4rBnJiJgejppNE+S", + "BazR7Dv0uYcs8kPT4DPpwzv4aJ2aXCBaxYrq8Rx2xOqANCPVOrtPUk9yGpaQ5adU", + "GfxkVbpgIEz1c71PeQuK1KUU/Wpk7cpm+FQCizl9ftP2lHWsGhSLCuyWoMTjt68P", + "gYEWoV54eo/bzwj2ei6TcfNo+uHyzEiiG2qEvMh/cnYUFzs8O1t0mN19WPB1pSh1", + "faci5lGdtkRbLgP0g5RvpagE7Lw3mCc5Om8jmHs4mPfuVkssBVV23CrFpqLLrDX3", + "Acwb/zRGvA7T4WESBTJMYFOLgm0W0Y+AN8RcYNU9QbDhe++Te0uz/3Sy3GN2Xg5z", + "MxfD1+34x6KvMfCh8NjII2mFQ9ukcfrhcfO3oWDLlwsqlVbhkZxNiUOEIx9nzHcF", + "kWpZ2ypBDH45h2o3LyqvGjsu/BFkeG6JpEDCWbClKWcjKxOrLVDufhSDduffDjja", + "zOsgQJg0Yf//Ubb5p0c54GjHM/XDXEcV3m3sEtbmMYz6xGwuag4bx8P2E/QY8sFp", + "JxgIdS8vdl6YhDCjKJ2XzI30JwCdftgDIAiWSE0ivoDc+8+gG1nb11GT52HFzA==", + "-----END CERTIFICATE-----", + "-----BEGIN PRIVATE KEY-----", + "MIIJQwIBADANBgkqhkiG9w0BAQEFAASCCS0wggkpAgEAAoICAQDKbRiedt0JBG3N", + "+82vIrgk2oY9Ga+ocvk6El/1X3c8Y4mB7g9j4mWciQe7dnjqogPLEOTeddxFLX9m", + "SAcLZ56hSMw+U6iNwcUWo1E5Tsl1F6JzMEMF7tonw1MtywprkH82pzk1r1S2nNTZ", + "pFmyR6SQl2G1Jz29Tb3dEI00xB9Y7eWwx4X8cAiGCuj2PjKgu9mtPcAdmRfhCrEn", + "aoITipom3yR7aCXbKhni68rAS681CXv+WVt1eGfOU9pY2uSmZ1oEaH61MuU5iVdd", + "BzuB4bfdsb1rxIv65o4NgFWGRVEEMcSl6VAWxeU+OnklZmk+yB/PGKecrYAU2ooM", + "puxEwo6Iafw3wmitQ1HoYdHHu4lKtsgi74mbF+FnT2Mc5KC35Ta6puGGT+oinGUl", + "EfwPqjygswl4DFxTQNScYpQ8BKt+RcxjfvnoSanTZjkNXkEnLC8GUGupgve3frqV", + "nQr8p7X9fkurzXF4bUUKYzTOFt+6Fq+Z4nLBgN/9K6QxB2VvOSpZHVjtbgkv+n7Y", + "LfFFDGn+XsNz4oM7rKspgVk5TDo7VrgEF8YGWb5EafsXbKZ/7kwgD/mzUoRg+kPk", + "zt1OiI06h5PN7Bigr4NnclP3aoD6/diCM4EARRpNFtJzPxPNxPIVEsF9ewjiy93a", + "4T2W+FUTAoUPDwGurvZyMppF9sElbwIDAQABAoICAQC4sATwP563pXTRpNYq3lCI", + "P2COyqq70/qUA0PNygYt8Nr60srz5RG0WknVvefgm2U+lvFaDsqjyzkbhsf2ndnb", + "aWH/07BLdeluGB/5W2rvDFtJIVVlSmF8OffgJgohzbpjkPrfglKWMkz5LbwwrrD0", + "w0mAUIdB+nYqBfnvlKjNKHCSc9hJU6ZTNg0K7gCfKgUWzOpFlvJ0fp7XSZPYZHL0", + "2E6e0Y0Ig0cPBPb9r4/xoe+hRsHtUafUVik3PK+1K0K0FurUQ9VkQ2yUEg83F0v8", + "Vzht5OuaRVSB+P8O/JtIamfywAY0YOYhepQhjWikwU5UUzhJ+PqNDD87/+g9bA1B", + "xC25eoDxThiQlgDmRoH18ZsWDVf9TuJnm4cpxnZYX6ip+BLm/aidT39auZo0Fl+r", + "cJxRn0Qlm0Vm4Tc/6ZG6PQWB+Q6CjVFdoxeOvEQcTSuKA6VZBStLmqX++5In1Lmj", + "hVr3/aueHiZvXS5bNIdd2IfzatR+nP+uxzM/ryJRvGO2B2XTS00Cvv/lH84BDJYV", + "yt1PJIBoM9Dh7aUAHmKNVfRt83xzvcSPZx9VmSzA6wwqCQcO1GJk6keAuxOuligu", + "YdSFcfChOg90WvBcl+NzMblLkwrFSBQR7kgG0+dedv+Wkm4xO4T7B4W2G5+VIJKG", + "mrEAq6XQMFnfEJzNVg7JUQKCAQEA91eMvphoVVz+cxa4Ew7OokNXk5kSlvmQ8+Ij", + "ngFBvniXPZecxVzFEJglSthH5KI2ZqxwF3GJhKjxUihwf6K13Hx54EM7S/qV57ie", + "kVeKdAs+SGv+hRk1gQOoPBInbtKGKTni1V8T7iNginLueC/YikFugzv6IxiliBSG", + "3R7zjRepOW69aEoCPecx9amU4CkAwgeLJgBloBoqWD8sKM+bl7p5juQCU2sQ9D4/", + "kLnpG9+zPRUNjI4sog3L1wql3zthI6/4gf0TNuDhJTZ68vpMSi02pOUkVa0MmVOA", + "ex16luIp0BhxG/sUAeoevFL4KqR0CBbyAstbt2E/oPYOWMJ4MwKCAQEA0YMNXY7I", + "RNFOmiZ2Wn8kENCyJguqbOMd/li2+ercgp3MaSgTjC5KDFvZqTwXXlrURPu9hcyv", + "sJBSjp45g1T1LsUUq8UJgOIQgxykurIstGg33TAR+TN9VUu/xonLQF23GD8M6Vzd", + "EcZEVlBY33hgNXw4mRcBPnaoG5FZCBfHOgdBCExoYKW/RNKcmu0q+h9uhDBCbopv", + "04ROzw+HW1qc5qvNPR47buZ9+5QdonVK8s2bguMJ0phXwdSxL21wsjIsXyAO9m7w", + "qLHOq/hVokM0Fki09Exg4ppB8cLHC2ITpsVSgn4Dcz5zRtyvhozSKX4R9kMC64a0", + "AgMPVMllmGlR1QKCAQBIIGCrh7gNBIQyvXJKJGw/RxH3uZCBNB9/7vbh8Y3hZDr+", + "PAL8TpQsxaFCVRWJ53+jqy84tuQaKkXM5nv/zEvqEuZbbl+RRW6HVv/udC+srUap", + "Scy7tWEz0QQzGDwlhgCXbwjlnccrD2fsl51QsOsdTf1TCZ9ksqA6sXmua4MsJrUz", + "SUa0bbh/oraf46bFQ0+0RQzftQftixPEDg/rirbdpQQjlfvTpYoZHzncE0qV1ULo", + "UgZUcXU0gH9rovBBy4gFJyB5j3oV67fb6SorRrAOhWbE6QkSbtcYsw/pVuxTqXn1", + "89qwBSSNdl8mHa++h5xKa56BEBobvKEYaAhA+9yfAoIBAQDPFEE5n87Cdj7CjhGd", + "EN2M4Tmz8QPZ7AHRS85O5pxdXaqUpU/s1jPKU16nrwVJ9WypYkjI3q8oTP3MiQc/", + "j9FnENSFkpL6GHdJoB4Rido11myg6spZDVNr4xsCGWATlo1KIceZQHghAV66EWBG", + "QKyXMNigN+S64Hz4AomFPjtkV5cnpJ3mKO0MET9IwfIglsCdVzXSHHK7FaLvdeHL", + "oZxDQrvxFNiZnKgY6SUBVf1mT2LN06n5xSm4I4md3wXsmzrQKtefK7gihNxJjYLW", + "hqYNAIAalwOL9fwIAQTLc30I8S/EWtj+J1O5TpcO3lE7QahvR3yzXsi81Flq7ETG", + "iBKhAoIBAGHGpnjrLlCarNk9axh4Dw1OjgEvwPlEqsiWXt2tylLeab0OGC47MmJx", + "RmKwgVukMuxkQb8v4ANSRtih7R+E+qXfexjEFYtzh/uaRP1Z7ZrO/oqq0oLbPpsx", + "yTSRDL1i5/fgdIlKVH3N4IF7E8Pc3REgYIwLQxYjTdgVHEAM65XegQ2Lkpr4iae3", + "hm4IsD2PrsVITrlsLg65XnfcbsCs/OfQ5GuUp+xUBw5e0bQBmsWEiCaCjrq/EHJa", + "/oeJRqS7lyGYDC+wiSsE70x4dvu1um2F+V1Jw4LWjhu8Z8dNSXPSf8vLqXGkWAlk", + "805lq+iy7Mkhb+dlr4R9WhMWDyGwgYs=", + "-----END PRIVATE KEY-----" + ] +} diff --git a/src/cephadm/tests/__init__.py b/src/cephadm/tests/__init__.py new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/src/cephadm/tests/__init__.py diff --git a/src/cephadm/tests/fixtures.py b/src/cephadm/tests/fixtures.py new file mode 100644 index 000000000..45990d910 --- /dev/null +++ b/src/cephadm/tests/fixtures.py @@ -0,0 +1,126 @@ +import mock +import os +import pytest +import time + +from contextlib import contextmanager +from pyfakefs import fake_filesystem + +from typing import Callable, Dict, List, Optional + + +with mock.patch('builtins.open', create=True): + from importlib.machinery import SourceFileLoader + cd = SourceFileLoader('cephadm', 'cephadm').load_module() + + +def mock_docker(): + docker = mock.Mock(cd.Docker) + docker.path = '/usr/bin/docker' + return docker + + +def mock_podman(): + podman = mock.Mock(cd.Podman) + podman.path = '/usr/bin/podman' + podman.version = (2, 1, 0) + return podman + + +def _daemon_path(): + return os.getcwd() + + +def mock_bad_firewalld(): + def raise_bad_firewalld(): + raise Exception('Called bad firewalld') + f = mock.Mock(cd.Firewalld) + f.enable_service_for = lambda _ : raise_bad_firewalld() + f.apply_rules = lambda : raise_bad_firewalld() + f.open_ports = lambda _ : raise_bad_firewalld() + +def _mock_scrape_host(obj, interval): + try: + raise ValueError("wah") + except Exception as e: + obj._handle_thread_exception(e, 'host') + + +def _mock_run(obj): + t = obj._create_thread(obj._scrape_host_facts, 'host', 5) + time.sleep(1) + if not t.is_alive(): + obj.cephadm_cache.update_health('host', "inactive", "host thread stopped") + + +@pytest.fixture +def exporter(): + with mock.patch('cephadm.CephadmDaemon.daemon_path', _daemon_path()), \ + mock.patch('cephadm.CephadmDaemon.can_run', return_value=True), \ + mock.patch('cephadm.CephadmDaemon.run', _mock_run), \ + mock.patch('cephadm.CephadmDaemon._scrape_host_facts', _mock_scrape_host): + + ctx = cd.CephadmContext() + exporter = cd.CephadmDaemon(ctx, fsid='foobar', daemon_id='test') + assert exporter.token == 'MyAccessToken' + yield exporter + + +@pytest.fixture() +def cephadm_fs( + fs: fake_filesystem.FakeFilesystem, +): + """ + use pyfakefs to stub filesystem calls + """ + uid = os.getuid() + gid = os.getgid() + + with mock.patch('os.fchown'), \ + mock.patch('os.fchmod'), \ + mock.patch('platform.processor', return_value='x86_64'), \ + mock.patch('cephadm.extract_uid_gid', return_value=(uid, gid)): + + if not fake_filesystem.is_root(): + fake_filesystem.set_uid(0) + + fs.create_dir(cd.DATA_DIR) + fs.create_dir(cd.LOG_DIR) + fs.create_dir(cd.LOCK_DIR) + fs.create_dir(cd.LOGROTATE_DIR) + fs.create_dir(cd.UNIT_DIR) + + yield fs + + +@contextmanager +def with_cephadm_ctx( + cmd: List[str], + container_engine: Callable = mock_podman(), + list_networks: Optional[Dict[str,Dict[str,List[str]]]] = None, + hostname: Optional[str] = None, +): + """ + :param cmd: cephadm command argv + :param container_engine: container engine mock (podman or docker) + :param list_networks: mock 'list-networks' return + :param hostname: mock 'socket.gethostname' return + """ + if not hostname: + hostname = 'host1' + + with mock.patch('cephadm.attempt_bind'), \ + mock.patch('cephadm.call', return_value=('', '', 0)), \ + mock.patch('cephadm.call_timeout', return_value=0), \ + mock.patch('cephadm.find_executable', return_value='foo'), \ + mock.patch('cephadm.is_available', return_value=True), \ + mock.patch('cephadm.json_loads_retry', return_value={'epoch' : 1}), \ + mock.patch('socket.gethostname', return_value=hostname): + ctx: cd.CephadmContext = cd.cephadm_init_ctx(cmd) + ctx.container_engine = container_engine + if list_networks is not None: + with mock.patch('cephadm.list_networks', return_value=list_networks): + yield ctx + else: + yield ctx + diff --git a/src/cephadm/tests/test_cephadm.py b/src/cephadm/tests/test_cephadm.py new file mode 100644 index 000000000..7383ac0ee --- /dev/null +++ b/src/cephadm/tests/test_cephadm.py @@ -0,0 +1,2413 @@ +# type: ignore + +import errno +import json +import mock +import os +import pytest +import socket +import sys +import time +import threading +import unittest + +from http.server import HTTPServer +from textwrap import dedent +from urllib.request import Request, urlopen +from urllib.error import HTTPError + +from typing import List, Optional + +from .fixtures import ( + cephadm_fs, + exporter, + mock_docker, + mock_podman, + with_cephadm_ctx, + mock_bad_firewalld, +) + +from pyfakefs import fake_filesystem +from pyfakefs import fake_filesystem_unittest + +with mock.patch('builtins.open', create=True): + from importlib.machinery import SourceFileLoader + cd = SourceFileLoader('cephadm', 'cephadm').load_module() + + +def get_ceph_conf( + fsid='00000000-0000-0000-0000-0000deadbeef', + mon_host='[v2:192.168.1.1:3300/0,v1:192.168.1.1:6789/0]'): + return f''' +# minimal ceph.conf for {fsid} +[global] + fsid = {fsid} + mon_host = {mon_host} +''' + +class TestCephAdm(object): + + def test_docker_unit_file(self): + ctx = cd.CephadmContext() + ctx.container_engine = mock_docker() + r = cd.get_unit_file(ctx, '9b9d7609-f4d5-4aba-94c8-effa764d96c9') + assert 'Requires=docker.service' in r + ctx.container_engine = mock_podman() + r = cd.get_unit_file(ctx, '9b9d7609-f4d5-4aba-94c8-effa764d96c9') + assert 'Requires=docker.service' not in r + + @mock.patch('cephadm.logger') + def test_attempt_bind(self, logger): + ctx = None + address = None + port = 0 + + def os_error(errno): + _os_error = OSError() + _os_error.errno = errno + return _os_error + + for side_effect, expected_exception in ( + (os_error(errno.EADDRINUSE), cd.PortOccupiedError), + (os_error(errno.EAFNOSUPPORT), cd.Error), + (os_error(errno.EADDRNOTAVAIL), cd.Error), + (None, None), + ): + _socket = mock.Mock() + _socket.bind.side_effect = side_effect + try: + cd.attempt_bind(ctx, _socket, address, port) + except Exception as e: + assert isinstance(e, expected_exception) + else: + if expected_exception is not None: + assert False + + @mock.patch('cephadm.attempt_bind') + @mock.patch('cephadm.logger') + def test_port_in_use(self, logger, attempt_bind): + empty_ctx = None + + assert cd.port_in_use(empty_ctx, 9100) == False + + attempt_bind.side_effect = cd.PortOccupiedError('msg') + assert cd.port_in_use(empty_ctx, 9100) == True + + os_error = OSError() + os_error.errno = errno.EADDRNOTAVAIL + attempt_bind.side_effect = os_error + assert cd.port_in_use(empty_ctx, 9100) == False + + os_error = OSError() + os_error.errno = errno.EAFNOSUPPORT + attempt_bind.side_effect = os_error + assert cd.port_in_use(empty_ctx, 9100) == False + + @mock.patch('socket.socket') + @mock.patch('cephadm.logger') + def test_check_ip_port_success(self, logger, _socket): + ctx = cd.CephadmContext() + ctx.skip_ping_check = False # enables executing port check with `check_ip_port` + + for address, address_family in ( + ('0.0.0.0', socket.AF_INET), + ('::', socket.AF_INET6), + ): + try: + cd.check_ip_port(ctx, cd.EndPoint(address, 9100)) + except: + assert False + else: + assert _socket.call_args == mock.call(address_family, socket.SOCK_STREAM) + + @mock.patch('socket.socket') + @mock.patch('cephadm.logger') + def test_check_ip_port_failure(self, logger, _socket): + ctx = cd.CephadmContext() + ctx.skip_ping_check = False # enables executing port check with `check_ip_port` + + def os_error(errno): + _os_error = OSError() + _os_error.errno = errno + return _os_error + + for address, address_family in ( + ('0.0.0.0', socket.AF_INET), + ('::', socket.AF_INET6), + ): + for side_effect, expected_exception in ( + (os_error(errno.EADDRINUSE), cd.PortOccupiedError), + (os_error(errno.EADDRNOTAVAIL), cd.Error), + (os_error(errno.EAFNOSUPPORT), cd.Error), + (None, None), + ): + mock_socket_obj = mock.Mock() + mock_socket_obj.bind.side_effect = side_effect + _socket.return_value = mock_socket_obj + try: + cd.check_ip_port(ctx, cd.EndPoint(address, 9100)) + except Exception as e: + assert isinstance(e, expected_exception) + else: + if side_effect is not None: + assert False + + + def test_is_not_fsid(self): + assert not cd.is_fsid('no-uuid') + + def test_is_fsid(self): + assert cd.is_fsid('e863154d-33c7-4350-bca5-921e0467e55b') + + def test__get_parser_image(self): + args = cd._parse_args(['--image', 'foo', 'version']) + assert args.image == 'foo' + + def test_parse_mem_usage(self): + cd.logger = mock.Mock() + len, summary = cd._parse_mem_usage(0, 'c6290e3f1489,-- / --') + assert summary == {} + + def test_CustomValidation(self): + assert cd._parse_args(['deploy', '--name', 'mon.a', '--fsid', 'fsid']) + + with pytest.raises(SystemExit): + cd._parse_args(['deploy', '--name', 'wrong', '--fsid', 'fsid']) + + @pytest.mark.parametrize("test_input, expected", [ + ("1.6.2", (1,6,2)), + ("1.6.2-stable2", (1,6,2)), + ]) + def test_parse_podman_version(self, test_input, expected): + assert cd._parse_podman_version(test_input) == expected + + def test_parse_podman_version_invalid(self): + with pytest.raises(ValueError) as res: + cd._parse_podman_version('inval.id') + assert 'inval' in str(res.value) + + def test_is_ipv6(self): + cd.logger = mock.Mock() + for good in ("[::1]", "::1", + "fff:ffff:ffff:ffff:ffff:ffff:ffff:ffff"): + assert cd.is_ipv6(good) + for bad in ("127.0.0.1", + "ffff:ffff:ffff:ffff:ffff:ffff:ffff:fffg", + "1:2:3:4:5:6:7:8:9", "fd00::1::1", "[fg::1]"): + assert not cd.is_ipv6(bad) + + def test_unwrap_ipv6(self): + def unwrap_test(address, expected): + assert cd.unwrap_ipv6(address) == expected + + tests = [ + ('::1', '::1'), ('[::1]', '::1'), + ('[fde4:8dba:82e1:0:5054:ff:fe6a:357]', 'fde4:8dba:82e1:0:5054:ff:fe6a:357'), + ('can actually be any string', 'can actually be any string'), + ('[but needs to be stripped] ', '[but needs to be stripped] ')] + for address, expected in tests: + unwrap_test(address, expected) + + def test_wrap_ipv6(self): + def wrap_test(address, expected): + assert cd.wrap_ipv6(address) == expected + + tests = [ + ('::1', '[::1]'), ('[::1]', '[::1]'), + ('fde4:8dba:82e1:0:5054:ff:fe6a:357', + '[fde4:8dba:82e1:0:5054:ff:fe6a:357]'), + ('myhost.example.com', 'myhost.example.com'), + ('192.168.0.1', '192.168.0.1'), + ('', ''), ('fd00::1::1', 'fd00::1::1')] + for address, expected in tests: + wrap_test(address, expected) + + @mock.patch('cephadm.Firewalld', mock_bad_firewalld) + @mock.patch('cephadm.logger') + def test_skip_firewalld(self, logger, cephadm_fs): + """ + test --skip-firewalld actually skips changing firewall + """ + + ctx = cd.CephadmContext() + with pytest.raises(Exception): + cd.update_firewalld(ctx, 'mon') + + ctx.skip_firewalld = True + cd.update_firewalld(ctx, 'mon') + + ctx.skip_firewalld = False + with pytest.raises(Exception): + cd.update_firewalld(ctx, 'mon') + + ctx = cd.CephadmContext() + ctx.ssl_dashboard_port = 8888 + ctx.dashboard_key = None + ctx.dashboard_password_noupdate = True + ctx.initial_dashboard_password = 'password' + ctx.initial_dashboard_user = 'User' + with pytest.raises(Exception): + cd.prepare_dashboard(ctx, 0, 0, lambda _, extra_mounts=None, ___=None : '5', lambda : None) + + ctx.skip_firewalld = True + cd.prepare_dashboard(ctx, 0, 0, lambda _, extra_mounts=None, ___=None : '5', lambda : None) + + ctx.skip_firewalld = False + with pytest.raises(Exception): + cd.prepare_dashboard(ctx, 0, 0, lambda _, extra_mounts=None, ___=None : '5', lambda : None) + + @mock.patch('cephadm.call_throws') + @mock.patch('cephadm.get_parm') + def test_registry_login(self, get_parm, call_throws): + # test normal valid login with url, username and password specified + call_throws.return_value = '', '', 0 + ctx: cd.CephadmContext = cd.cephadm_init_ctx( + ['registry-login', '--registry-url', 'sample-url', + '--registry-username', 'sample-user', '--registry-password', + 'sample-pass']) + ctx.container_engine = mock_docker() + retval = cd.command_registry_login(ctx) + assert retval == 0 + + # test bad login attempt with invalid arguments given + ctx: cd.CephadmContext = cd.cephadm_init_ctx( + ['registry-login', '--registry-url', 'bad-args-url']) + with pytest.raises(Exception) as e: + assert cd.command_registry_login(ctx) + assert str(e.value) == ('Invalid custom registry arguments received. To login to a custom registry include ' + '--registry-url, --registry-username and --registry-password options or --registry-json option') + + # test normal valid login with json file + get_parm.return_value = {"url": "sample-url", "username": "sample-username", "password": "sample-password"} + ctx: cd.CephadmContext = cd.cephadm_init_ctx( + ['registry-login', '--registry-json', 'sample-json']) + ctx.container_engine = mock_docker() + retval = cd.command_registry_login(ctx) + assert retval == 0 + + # test bad login attempt with bad json file + get_parm.return_value = {"bad-json": "bad-json"} + ctx: cd.CephadmContext = cd.cephadm_init_ctx( + ['registry-login', '--registry-json', 'sample-json']) + with pytest.raises(Exception) as e: + assert cd.command_registry_login(ctx) + assert str(e.value) == ("json provided for custom registry login did not include all necessary fields. " + "Please setup json file as\n" + "{\n" + " \"url\": \"REGISTRY_URL\",\n" + " \"username\": \"REGISTRY_USERNAME\",\n" + " \"password\": \"REGISTRY_PASSWORD\"\n" + "}\n") + + # test login attempt with valid arguments where login command fails + call_throws.side_effect = Exception + ctx: cd.CephadmContext = cd.cephadm_init_ctx( + ['registry-login', '--registry-url', 'sample-url', + '--registry-username', 'sample-user', '--registry-password', + 'sample-pass']) + with pytest.raises(Exception) as e: + cd.command_registry_login(ctx) + assert str(e.value) == "Failed to login to custom registry @ sample-url as sample-user with given password" + + def test_get_image_info_from_inspect(self): + # podman + out = """204a01f9b0b6710dd0c0af7f37ce7139c47ff0f0105d778d7104c69282dfbbf1,[docker.io/ceph/ceph@sha256:1cc9b824e1b076cdff52a9aa3f0cc8557d879fb2fbbba0cafed970aca59a3992]""" + r = cd.get_image_info_from_inspect(out, 'registry/ceph/ceph:latest') + print(r) + assert r == { + 'image_id': '204a01f9b0b6710dd0c0af7f37ce7139c47ff0f0105d778d7104c69282dfbbf1', + 'repo_digests': ['docker.io/ceph/ceph@sha256:1cc9b824e1b076cdff52a9aa3f0cc8557d879fb2fbbba0cafed970aca59a3992'] + } + + # docker + out = """sha256:16f4549cf7a8f112bbebf7946749e961fbbd1b0838627fe619aab16bc17ce552,[quay.ceph.io/ceph-ci/ceph@sha256:4e13da36c1bd6780b312a985410ae678984c37e6a9493a74c87e4a50b9bda41f]""" + r = cd.get_image_info_from_inspect(out, 'registry/ceph/ceph:latest') + assert r == { + 'image_id': '16f4549cf7a8f112bbebf7946749e961fbbd1b0838627fe619aab16bc17ce552', + 'repo_digests': ['quay.ceph.io/ceph-ci/ceph@sha256:4e13da36c1bd6780b312a985410ae678984c37e6a9493a74c87e4a50b9bda41f'] + } + + # multiple digests (podman) + out = """e935122ab143a64d92ed1fbb27d030cf6e2f0258207be1baf1b509c466aeeb42,[docker.io/prom/prometheus@sha256:e4ca62c0d62f3e886e684806dfe9d4e0cda60d54986898173c1083856cfda0f4 docker.io/prom/prometheus@sha256:efd99a6be65885c07c559679a0df4ec709604bcdd8cd83f0d00a1a683b28fb6a]""" + r = cd.get_image_info_from_inspect(out, 'registry/prom/prometheus:latest') + assert r == { + 'image_id': 'e935122ab143a64d92ed1fbb27d030cf6e2f0258207be1baf1b509c466aeeb42', + 'repo_digests': [ + 'docker.io/prom/prometheus@sha256:e4ca62c0d62f3e886e684806dfe9d4e0cda60d54986898173c1083856cfda0f4', + 'docker.io/prom/prometheus@sha256:efd99a6be65885c07c559679a0df4ec709604bcdd8cd83f0d00a1a683b28fb6a', + ] + } + + + def test_dict_get(self): + result = cd.dict_get({'a': 1}, 'a', require=True) + assert result == 1 + result = cd.dict_get({'a': 1}, 'b') + assert result is None + result = cd.dict_get({'a': 1}, 'b', default=2) + assert result == 2 + + def test_dict_get_error(self): + with pytest.raises(cd.Error): + cd.dict_get({'a': 1}, 'b', require=True) + + def test_dict_get_join(self): + result = cd.dict_get_join({'foo': ['a', 'b']}, 'foo') + assert result == 'a\nb' + result = cd.dict_get_join({'foo': [1, 2]}, 'foo') + assert result == '1\n2' + result = cd.dict_get_join({'bar': 'a'}, 'bar') + assert result == 'a' + result = cd.dict_get_join({'a': 1}, 'a') + assert result == 1 + + def test_last_local_images(self): + out = ''' +docker.io/ceph/daemon-base@ +docker.io/ceph/ceph:v15.2.5 +docker.io/ceph/daemon-base:octopus + ''' + image = cd._filter_last_local_ceph_image(out) + assert image == 'docker.io/ceph/ceph:v15.2.5' + + def test_normalize_image_digest(self): + s = 'myhostname:5000/ceph/ceph@sha256:753886ad9049004395ae990fbb9b096923b5a518b819283141ee8716ddf55ad1' + assert cd.normalize_image_digest(s) == s + + s = 'ceph/ceph:latest' + assert cd.normalize_image_digest(s) == f'{cd.DEFAULT_REGISTRY}/{s}' + + @pytest.mark.parametrize('fsid, ceph_conf, list_daemons, result, err, ', + [ + ( + None, + None, + [], + None, + None, + ), + ( + '00000000-0000-0000-0000-0000deadbeef', + None, + [], + '00000000-0000-0000-0000-0000deadbeef', + None, + ), + ( + '00000000-0000-0000-0000-0000deadbeef', + None, + [ + {'fsid': '10000000-0000-0000-0000-0000deadbeef'}, + {'fsid': '20000000-0000-0000-0000-0000deadbeef'}, + ], + '00000000-0000-0000-0000-0000deadbeef', + None, + ), + ( + None, + None, + [ + {'fsid': '00000000-0000-0000-0000-0000deadbeef'}, + ], + '00000000-0000-0000-0000-0000deadbeef', + None, + ), + ( + None, + None, + [ + {'fsid': '10000000-0000-0000-0000-0000deadbeef'}, + {'fsid': '20000000-0000-0000-0000-0000deadbeef'}, + ], + None, + r'Cannot infer an fsid', + ), + ( + None, + get_ceph_conf(fsid='00000000-0000-0000-0000-0000deadbeef'), + [], + '00000000-0000-0000-0000-0000deadbeef', + None, + ), + ( + None, + get_ceph_conf(fsid='00000000-0000-0000-0000-0000deadbeef'), + [ + {'fsid': '00000000-0000-0000-0000-0000deadbeef'}, + ], + '00000000-0000-0000-0000-0000deadbeef', + None, + ), + ( + None, + get_ceph_conf(fsid='00000000-0000-0000-0000-0000deadbeef'), + [ + {'fsid': '10000000-0000-0000-0000-0000deadbeef'}, + {'fsid': '20000000-0000-0000-0000-0000deadbeef'}, + ], + None, + r'Cannot infer an fsid', + ), + ]) + @mock.patch('cephadm.call') + def test_infer_fsid(self, _call, fsid, ceph_conf, list_daemons, result, err, cephadm_fs): + # build the context + ctx = cd.CephadmContext() + ctx.fsid = fsid + + # mock the decorator + mock_fn = mock.Mock() + mock_fn.return_value = 0 + infer_fsid = cd.infer_fsid(mock_fn) + + # mock the ceph.conf file content + if ceph_conf: + f = cephadm_fs.create_file('ceph.conf', contents=ceph_conf) + ctx.config = f.path + + # test + with mock.patch('cephadm.list_daemons', return_value=list_daemons): + if err: + with pytest.raises(cd.Error, match=err): + infer_fsid(ctx) + else: + infer_fsid(ctx) + assert ctx.fsid == result + + @pytest.mark.parametrize('fsid, config, name, list_daemons, result, ', + [ + ( + None, + '/foo/bar.conf', + None, + [], + '/foo/bar.conf', + ), + ( + '00000000-0000-0000-0000-0000deadbeef', + None, + None, + [], + cd.SHELL_DEFAULT_CONF, + ), + ( + '00000000-0000-0000-0000-0000deadbeef', + None, + None, + [{'name': 'mon.a', 'fsid': '00000000-0000-0000-0000-0000deadbeef', 'style': 'cephadm:v1'}], + '/var/lib/ceph/00000000-0000-0000-0000-0000deadbeef/mon.a/config', + ), + ( + '00000000-0000-0000-0000-0000deadbeef', + None, + None, + [{'name': 'mon.a', 'fsid': 'aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa', 'style': 'cephadm:v1'}], + cd.SHELL_DEFAULT_CONF, + ), + ( + '00000000-0000-0000-0000-0000deadbeef', + None, + None, + [{'name': 'mon.a', 'fsid': '00000000-0000-0000-0000-0000deadbeef', 'style': 'legacy'}], + cd.SHELL_DEFAULT_CONF, + ), + ( + '00000000-0000-0000-0000-0000deadbeef', + None, + None, + [{'name': 'osd.0'}], + cd.SHELL_DEFAULT_CONF, + ), + ( + '00000000-0000-0000-0000-0000deadbeef', + '/foo/bar.conf', + 'mon.a', + [{'name': 'mon.a', 'style': 'cephadm:v1'}], + '/foo/bar.conf', + ), + ( + '00000000-0000-0000-0000-0000deadbeef', + None, + 'mon.a', + [], + '/var/lib/ceph/00000000-0000-0000-0000-0000deadbeef/mon.a/config', + ), + ( + '00000000-0000-0000-0000-0000deadbeef', + None, + 'osd.0', + [], + '/var/lib/ceph/00000000-0000-0000-0000-0000deadbeef/osd.0/config', + ), + ( + None, + None, + None, + [], + cd.SHELL_DEFAULT_CONF, + ), + ]) + @mock.patch('cephadm.call') + @mock.patch('cephadm.logger') + def test_infer_config(self, logger, _call, fsid, config, name, list_daemons, result, cephadm_fs): + # build the context + ctx = cd.CephadmContext() + ctx.fsid = fsid + ctx.config = config + ctx.name = name + + # mock the decorator + mock_fn = mock.Mock() + mock_fn.return_value = 0 + infer_config = cd.infer_config(mock_fn) + + # mock the config file + cephadm_fs.create_file(result) + + # test + with mock.patch('cephadm.list_daemons', return_value=list_daemons): + infer_config(ctx) + assert ctx.config == result + + @mock.patch('cephadm.call') + def test_extract_uid_gid_fail(self, _call): + err = """Error: container_linux.go:370: starting container process caused: process_linux.go:459: container init caused: process_linux.go:422: setting cgroup config for procHooks process caused: Unit libpod-056038e1126191fba41d8a037275136f2d7aeec9710b9ee +ff792c06d8544b983.scope not found.: OCI runtime error""" + _call.return_value = ('', err, 127) + ctx = cd.CephadmContext() + ctx.container_engine = mock_podman() + with pytest.raises(cd.Error, match='OCI'): + cd.extract_uid_gid(ctx) + + @pytest.mark.parametrize('test_input, expected', [ + ([cd.make_fsid(), cd.make_fsid(), cd.make_fsid()], 3), + ([cd.make_fsid(), 'invalid-fsid', cd.make_fsid(), '0b87e50c-8e77-11ec-b890-'], 2), + (['f6860ec2-8e76-11ec-', '0b87e50c-8e77-11ec-b890-', ''], 0), + ([], 0), + ]) + def test_get_ceph_cluster_count(self, test_input, expected): + ctx = cd.CephadmContext() + with mock.patch('os.listdir', return_value=test_input): + assert cd.get_ceph_cluster_count(ctx) == expected + + def test_set_image_minimize_config(self): + def throw_cmd(cmd): + raise cd.Error(' '.join(cmd)) + ctx = cd.CephadmContext() + ctx.image = 'test_image' + ctx.no_minimize_config = True + fake_cli = lambda cmd, __=None, ___=None: throw_cmd(cmd) + with pytest.raises(cd.Error, match='config set global container_image test_image'): + cd.finish_bootstrap_config( + ctx=ctx, + fsid=cd.make_fsid(), + config='', + mon_id='a', mon_dir='mon_dir', + mon_network=None, ipv6=False, + cli=fake_cli, + cluster_network=None, + ipv6_cluster_network=False + ) + + +class TestCustomContainer(unittest.TestCase): + cc: cd.CustomContainer + + def setUp(self): + self.cc = cd.CustomContainer( + 'e863154d-33c7-4350-bca5-921e0467e55b', + 'container', + config_json={ + 'entrypoint': 'bash', + 'gid': 1000, + 'args': [ + '--no-healthcheck', + '-p 6800:6800' + ], + 'envs': ['SECRET=password'], + 'ports': [8080, 8443], + 'volume_mounts': { + '/CONFIG_DIR': '/foo/conf', + 'bar/config': '/bar:ro' + }, + 'bind_mounts': [ + [ + 'type=bind', + 'source=/CONFIG_DIR', + 'destination=/foo/conf', + '' + ], + [ + 'type=bind', + 'source=bar/config', + 'destination=/bar:ro', + 'ro=true' + ] + ] + }, + image='docker.io/library/hello-world:latest' + ) + + def test_entrypoint(self): + self.assertEqual(self.cc.entrypoint, 'bash') + + def test_uid_gid(self): + self.assertEqual(self.cc.uid, 65534) + self.assertEqual(self.cc.gid, 1000) + + def test_ports(self): + self.assertEqual(self.cc.ports, [8080, 8443]) + + def test_get_container_args(self): + result = self.cc.get_container_args() + self.assertEqual(result, [ + '--no-healthcheck', + '-p 6800:6800' + ]) + + def test_get_container_envs(self): + result = self.cc.get_container_envs() + self.assertEqual(result, ['SECRET=password']) + + def test_get_container_mounts(self): + result = self.cc.get_container_mounts('/xyz') + self.assertDictEqual(result, { + '/CONFIG_DIR': '/foo/conf', + '/xyz/bar/config': '/bar:ro' + }) + + def test_get_container_binds(self): + result = self.cc.get_container_binds('/xyz') + self.assertEqual(result, [ + [ + 'type=bind', + 'source=/CONFIG_DIR', + 'destination=/foo/conf', + '' + ], + [ + 'type=bind', + 'source=/xyz/bar/config', + 'destination=/bar:ro', + 'ro=true' + ] + ]) + + +class TestCephadmExporter(object): + exporter: cd.CephadmDaemon + files_created: List[str] = [] + crt = """-----BEGIN CERTIFICATE----- +MIIC1zCCAb8CEFHoZE2MfUVzo53fzzBKAT0wDQYJKoZIhvcNAQENBQAwKjENMAsG +A1UECgwEQ2VwaDEZMBcGA1UECwwQY2VwaGFkbS1leHBvcnRlcjAeFw0yMDExMjUy +MzEwNTVaFw0zMDExMjMyMzEwNTVaMCoxDTALBgNVBAoMBENlcGgxGTAXBgNVBAsM +EGNlcGhhZG0tZXhwb3J0ZXIwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIB +AQCsTfcJcXbREqfx1zTUuEmK+lJn9WWjk0URRF1Z+QgPkascNdkX16PnvhbGwXmF +BTdAcNl7V0U+z4EsGJ7hJsB7qTq6Rb6wNl7r0OxjeWOmB9xbF4Q/KR5yrbM1DA9A +B5fNswrUXViku5Y2jlOAz+ZMBhYxMx0edqhxSn297j04Z6RF4Mvkc43v0FH7Ju7k +O5+0VbdzcOdu37DFpoE4Ll2MZ/GuAHcJ8SD06sEdzFEjRCraav976743XcUlhZGX +ZTTG/Zf/a+wuCjtMG3od7vRFfuRrM5oTE133DuQ5deR7ybcZNDyopDjHF8xB1bAk +IOz4SbP6Q25K99Czm1K+3kMLAgMBAAEwDQYJKoZIhvcNAQENBQADggEBACmtvZb8 +dJGHx/WC0/JHxnEJCJM2qnn87ELzbbIQL1w1Yb/I6JQYPgq+WiQPaHaLL9eYsm0l +dFwvrh+WC0JpXDfADnUnkTSB/WpZ2nC+2JxBptrQEuIcqNXpcJd0bKDiHunv04JI +uEVpTAK05dBV38qNmIlu4HyB4OEnuQpyOr9xpIhdxuJ95O9K0j5BIw98ZaEwYNUP +Rm3YlQwfS6R5xaBvL9kyfxyAD2joNj44q6w/5zj4egXVIA5VpkQm8DmMtu0Pd2NG +dzfYRmqrDolh+rty8HiyIxzeDJQ5bj6LKbUkmABvX50nDySVyMfHmt461/n7W65R +CHFLoOmfJJik+Uc=\n-----END CERTIFICATE----- +""" + key = """-----BEGIN PRIVATE KEY----- +MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQCsTfcJcXbREqfx +1zTUuEmK+lJn9WWjk0URRF1Z+QgPkascNdkX16PnvhbGwXmFBTdAcNl7V0U+z4Es +GJ7hJsB7qTq6Rb6wNl7r0OxjeWOmB9xbF4Q/KR5yrbM1DA9AB5fNswrUXViku5Y2 +jlOAz+ZMBhYxMx0edqhxSn297j04Z6RF4Mvkc43v0FH7Ju7kO5+0VbdzcOdu37DF +poE4Ll2MZ/GuAHcJ8SD06sEdzFEjRCraav976743XcUlhZGXZTTG/Zf/a+wuCjtM +G3od7vRFfuRrM5oTE133DuQ5deR7ybcZNDyopDjHF8xB1bAkIOz4SbP6Q25K99Cz +m1K+3kMLAgMBAAECggEASnAwToMXWsGdjqxzpYasNv9oBIOO0nk4OHp5ffpJUjiT +XM+ip1tA80g7HMjPD/mt4gge3NtaDgWlf4Bve0O7mnEE7x5cgFIs9eG/jkYOF9eD +ilMBjivcfJywNDWujPH60iIMhqyBNEHaZl1ck+S9UJC8m6rCZLvMj40n/5riFfBy +1sjf2uOwcfWrjSj9Ju4wlMI6khSSz2aYC7glQQ/fo2+YArbEUcy60iloPQ6wEgZK +okoVWZA9AehwLcnRjkwd9EVmMMtRGPE/AcP4s/kKA0tRDRicPLN727Ke/yxv+Ppo +hbIZIcOn7soOFAENcodJ4YRSCd++QfCNaVAi7vwWWQKBgQDeBY4vvr+H0brbSjQg +O7Fpqub/fxZY3UoHWDqWs2X4o3qhDqaTQODpuYtCm8YQE//55JoLWKAD0evq5dLS +YLrtC1Vyxf+TA7opCUjWBe+liyndbJdB5q0zF7qdWUtQKGVSWyUWhK8gHa6M64fP +oi83DD7F0OGusTWGtfbceErk/wKBgQDGrJLRo/5xnAH5VmPfNu+S6h0M2qM6CYwe +Y5wHFG2uQQct73adf53SkhvZVmOzJsWQbVnlDOKMhqazcs+7VWRgO5X3naWVcctE +Hggw9MgpbXAWFOI5sNYsCYE58E+fTHjE6O4A3MhMCsze+CIC3sKuPQBBiL9bWSOX +8POswqfl9QKBgDe/nVxPwTgRaaH2l/AgDQRDbY1qE+psZlJBzTRaB5jPM9ONIjaH +a/JELLuk8a7H1tagmC2RK1zKMTriSnWY5FbxKZuQLAR2QyBavHdBNlOTBggbZD+f +9I2Hv8wSx95wxkBPsphc6Lxft5ya55czWjewU3LIaGK9DHuu5TWm3udxAoGBAJGP +PsJ59KIoOwoDUYjpJv3sqPwR9CVBeXeKY3aMcQ+KdUgiejVKmsb8ZYsG0GUhsv3u +ID7BAfsTbG9tXuVR2wjmnymcRwUHKnXtyvKTZVN06vpCsryx4zjAff2FI9ECpjke +r8HSAK41+4QhKEoSC3C9IMLi/dBfrsRTtTSOKZVBAoGBAI2dl5HEIFpufaI4toWM +LO5HFrlXgRDGoc/+Byr5/8ZZpYpU115Ol/q6M+l0koV2ygJ9jeJJEllFWykIDS6F +XxazFI74swAqobHb2ZS/SLhoVxE82DdSeXrjkTvUjNtrW5zs1gIMKBR4nD6H8AqL +iMN28C2bKGao5UHvdER1rGy7 +-----END PRIVATE KEY----- +""" + token = "MyAccessToken" + + @classmethod + def setup_class(cls): + # create the ssl files + fname = os.path.join(os.getcwd(), 'crt') + with open(fname, 'w') as crt: + crt.write(cls.crt) + cls.files_created.append(fname) + fname = os.path.join(os.getcwd(), 'key') + with open(fname, 'w') as crt: + crt.write(cls.key) + cls.files_created.append(fname) + fname = os.path.join(os.getcwd(), 'token') + with open(fname, 'w') as crt: + crt.write(cls.token) + cls.files_created.append(fname) + # start a simple http instance to test the requesthandler + cls.server = HTTPServer(('0.0.0.0', 9443), cd.CephadmDaemonHandler) + cls.server.cephadm_cache = cd.CephadmCache() + cls.server.token = cls.token + t = threading.Thread(target=cls.server.serve_forever) + t.daemon = True + t.start() + + @classmethod + def teardown_class(cls): + cls.server.shutdown() + assert len(cls.files_created) > 0 + for f in cls.files_created: + os.remove(f) + + def setup_method(self): + # re-init the cache for every test + TestCephadmExporter.server.cephadm_cache = cd.CephadmCache() + + def teardown_method(self): + pass + + def test_files_ready(self): + assert os.path.exists(os.path.join(os.getcwd(), 'crt')) + assert os.path.exists(os.path.join(os.getcwd(), 'key')) + assert os.path.exists(os.path.join(os.getcwd(), 'token')) + + def test_can_run(self, exporter): + assert exporter.can_run + + def test_token_valid(self, exporter): + assert exporter.token == self.token + + def test_unit_name(self,exporter): + assert exporter.unit_name + assert exporter.unit_name == "ceph-foobar-cephadm-exporter.test.service" + + def test_unit_run(self,exporter): + assert exporter.unit_run + lines = exporter.unit_run.split('\n') + assert len(lines) == 2 + assert "cephadm exporter --fsid foobar --id test --port 9443 &" in lines[1] + + def test_binary_path(self, exporter): + assert os.path.isfile(exporter.binary_path) + + def test_systemd_unit(self, exporter): + assert exporter.unit_file + + def test_validate_passes(self, exporter): + config = { + "crt": self.crt, + "key": self.key, + "token": self.token, + } + cd.CephadmDaemon.validate_config(config) + + def test_validate_fails(self, exporter): + config = { + "key": self.key, + "token": self.token, + } + with pytest.raises(cd.Error): + cd.CephadmDaemon.validate_config(config) + + def test_port_active(self, exporter): + assert exporter.port_active == True + + def test_rqst_health_200(self): + hdrs={"Authorization":f"Bearer {TestCephadmExporter.token}"} + req=Request("http://localhost:9443/v1/metadata/health",headers=hdrs) + r = urlopen(req) + assert r.status == 200 + + def test_rqst_all_inactive_500(self): + hdrs={"Authorization":f"Bearer {TestCephadmExporter.token}"} + req=Request("http://localhost:9443/v1/metadata",headers=hdrs) + try: + r = urlopen(req) + except HTTPError as e: + assert e.code == 500 + + def test_rqst_no_auth_401(self): + req=Request("http://localhost:9443/v1/metadata") + try: + urlopen(req) + except HTTPError as e: + assert e.code == 401 + + def test_rqst_bad_auth_401(self): + hdrs={"Authorization":f"Bearer BogusAuthToken"} + req=Request("http://localhost:9443/v1/metadata",headers=hdrs) + try: + urlopen(req) + except HTTPError as e: + assert e.code == 401 + + def test_rqst_badURL_404(self): + hdrs={"Authorization":f"Bearer {TestCephadmExporter.token}"} + req=Request("http://localhost:9443/v1/metazoic",headers=hdrs) + try: + urlopen(req) + except HTTPError as e: + assert e.code == 404 + + def test_rqst_inactive_task_204(self): + # all tasks initialise as inactive, and then 'go' active as their thread starts + # so we can pick any task to check for an inactive response (no content) + hdrs={"Authorization":f"Bearer {TestCephadmExporter.token}"} + req=Request("http://localhost:9443/v1/metadata/disks",headers=hdrs) + r = urlopen(req) + assert r.status == 204 + + def test_rqst_active_task_200(self): + TestCephadmExporter.server.cephadm_cache.tasks['host'] = 'active' + hdrs={"Authorization":f"Bearer {TestCephadmExporter.token}"} + req=Request("http://localhost:9443/v1/metadata/host",headers=hdrs) + r = urlopen(req) + assert r.status == 200 + + def test_rqst_all_206(self): + TestCephadmExporter.server.cephadm_cache.tasks['disks'] = 'active' + hdrs={"Authorization":f"Bearer {TestCephadmExporter.token}"} + req=Request("http://localhost:9443/v1/metadata",headers=hdrs) + r = urlopen(req) + assert r.status == 206 + + def test_rqst_disks_200(self): + TestCephadmExporter.server.cephadm_cache.tasks['disks'] = 'active' + hdrs={"Authorization":f"Bearer {TestCephadmExporter.token}"} + req=Request("http://localhost:9443/v1/metadata/disks",headers=hdrs) + r = urlopen(req) + assert r.status == 200 + + def test_thread_exception(self, exporter): + # run is patched to invoke a mocked scrape_host thread that will raise so + # we check here that the exception handler updates the cache object as we'd + # expect with the error + exporter.run() + assert exporter.cephadm_cache.host['scrape_errors'] + assert exporter.cephadm_cache.host['scrape_errors'] == ['ValueError exception: wah'] + assert exporter.cephadm_cache.errors == ['host thread stopped'] + + # Test the requesthandler does the right thing with invalid methods... + # ie. return a "501" - Not Implemented / Unsupported Method + def test_invalid_method_HEAD(self): + hdrs={"Authorization":f"Bearer {TestCephadmExporter.token}"} + req=Request("http://localhost:9443/v1/metadata/health",headers=hdrs, method="HEAD") + with pytest.raises(HTTPError, match=r"HTTP Error 501: .*") as e: + urlopen(req) + + def test_invalid_method_DELETE(self): + hdrs={"Authorization":f"Bearer {TestCephadmExporter.token}"} + req=Request("http://localhost:9443/v1/metadata/health",headers=hdrs, method="DELETE") + with pytest.raises(HTTPError, match=r"HTTP Error 501: .*") as e: + urlopen(req) + + def test_invalid_method_POST(self): + hdrs={"Authorization":f"Bearer {TestCephadmExporter.token}"} + req=Request("http://localhost:9443/v1/metadata/health",headers=hdrs, method="POST") + with pytest.raises(HTTPError, match=r"HTTP Error 501: .*") as e: + urlopen(req) + + def test_invalid_method_PUT(self): + hdrs={"Authorization":f"Bearer {TestCephadmExporter.token}"} + req=Request("http://localhost:9443/v1/metadata/health",headers=hdrs, method="PUT") + with pytest.raises(HTTPError, match=r"HTTP Error 501: .*") as e: + urlopen(req) + + def test_invalid_method_CONNECT(self): + hdrs={"Authorization":f"Bearer {TestCephadmExporter.token}"} + req=Request("http://localhost:9443/v1/metadata/health",headers=hdrs, method="CONNECT") + with pytest.raises(HTTPError, match=r"HTTP Error 501: .*") as e: + urlopen(req) + + def test_invalid_method_TRACE(self): + hdrs={"Authorization":f"Bearer {TestCephadmExporter.token}"} + req=Request("http://localhost:9443/v1/metadata/health",headers=hdrs, method="TRACE") + with pytest.raises(HTTPError, match=r"HTTP Error 501: .*") as e: + urlopen(req) + + def test_invalid_method_OPTIONS(self): + hdrs={"Authorization":f"Bearer {TestCephadmExporter.token}"} + req=Request("http://localhost:9443/v1/metadata/health",headers=hdrs, method="OPTIONS") + with pytest.raises(HTTPError, match=r"HTTP Error 501: .*") as e: + urlopen(req) + + def test_invalid_method_PATCH(self): + hdrs={"Authorization":f"Bearer {TestCephadmExporter.token}"} + req=Request("http://localhost:9443/v1/metadata/health",headers=hdrs, method="PATCH") + with pytest.raises(HTTPError, match=r"HTTP Error 501: .*") as e: + urlopen(req) + + def test_ipv4_subnet(self): + rc, v, msg = cd.check_subnet('192.168.1.0/24') + assert rc == 0 and v[0] == 4 + + def test_ipv4_subnet_list(self): + rc, v, msg = cd.check_subnet('192.168.1.0/24,10.90.90.0/24') + assert rc == 0 and not msg + + def test_ipv4_subnet_badlist(self): + rc, v, msg = cd.check_subnet('192.168.1.0/24,192.168.1.1') + assert rc == 1 and msg + + def test_ipv4_subnet_mixed(self): + rc, v, msg = cd.check_subnet('192.168.100.0/24,fe80::/64') + assert rc == 0 and v == [4,6] + + def test_ipv6_subnet(self): + rc, v, msg = cd.check_subnet('fe80::/64') + assert rc == 0 and v[0] == 6 + + def test_subnet_mask_missing(self): + rc, v, msg = cd.check_subnet('192.168.1.58') + assert rc == 1 and msg + + def test_subnet_mask_junk(self): + rc, v, msg = cd.check_subnet('wah') + assert rc == 1 and msg + + +class TestMaintenance: + systemd_target = "ceph.00000000-0000-0000-0000-000000c0ffee.target" + fsid = '0ea8cdd0-1bbf-11ec-a9c7-5254002763fa' + + def test_systemd_target_OK(self, tmp_path): + base = tmp_path + wants = base / "ceph.target.wants" + wants.mkdir() + target = wants / TestMaintenance.systemd_target + target.touch() + ctx = cd.CephadmContext() + ctx.unit_dir = str(base) + + assert cd.systemd_target_state(ctx, target.name) + + def test_systemd_target_NOTOK(self, tmp_path): + base = tmp_path + ctx = cd.CephadmContext() + ctx.unit_dir = str(base) + assert not cd.systemd_target_state(ctx, TestMaintenance.systemd_target) + + def test_parser_OK(self): + args = cd._parse_args(['host-maintenance', 'enter']) + assert args.maintenance_action == 'enter' + + def test_parser_BAD(self): + with pytest.raises(SystemExit): + cd._parse_args(['host-maintenance', 'wah']) + + @mock.patch('os.listdir', return_value=[]) + @mock.patch('cephadm.call') + @mock.patch('cephadm.systemd_target_state') + def test_enter_failure_1(self, _target_state, _call, _listdir): + _call.return_value = '', '', 999 + _target_state.return_value = True + ctx: cd.CephadmContext = cd.cephadm_init_ctx( + ['host-maintenance', 'enter', '--fsid', TestMaintenance.fsid]) + ctx.container_engine = mock_podman() + retval = cd.command_maintenance(ctx) + assert retval.startswith('failed') + + @mock.patch('os.listdir', return_value=[]) + @mock.patch('cephadm.call') + @mock.patch('cephadm.systemd_target_state') + def test_enter_failure_2(self, _target_state, _call, _listdir): + _call.side_effect = [('', '', 0), ('', '', 999), ('', '', 0), ('', '', 999)] + _target_state.return_value = True + ctx: cd.CephadmContext = cd.cephadm_init_ctx( + ['host-maintenance', 'enter', '--fsid', TestMaintenance.fsid]) + ctx.container_engine = mock_podman() + retval = cd.command_maintenance(ctx) + assert retval.startswith('failed') + + @mock.patch('os.listdir', return_value=[]) + @mock.patch('cephadm.call') + @mock.patch('cephadm.systemd_target_state') + @mock.patch('cephadm.target_exists') + def test_exit_failure_1(self, _target_exists, _target_state, _call, _listdir): + _call.return_value = '', '', 999 + _target_state.return_value = False + _target_exists.return_value = True + ctx: cd.CephadmContext = cd.cephadm_init_ctx( + ['host-maintenance', 'exit', '--fsid', TestMaintenance.fsid]) + ctx.container_engine = mock_podman() + retval = cd.command_maintenance(ctx) + assert retval.startswith('failed') + + @mock.patch('os.listdir', return_value=[]) + @mock.patch('cephadm.call') + @mock.patch('cephadm.systemd_target_state') + @mock.patch('cephadm.target_exists') + def test_exit_failure_2(self, _target_exists, _target_state, _call, _listdir): + _call.side_effect = [('', '', 0), ('', '', 999), ('', '', 0), ('', '', 999)] + _target_state.return_value = False + _target_exists.return_value = True + ctx: cd.CephadmContext = cd.cephadm_init_ctx( + ['host-maintenance', 'exit', '--fsid', TestMaintenance.fsid]) + ctx.container_engine = mock_podman() + retval = cd.command_maintenance(ctx) + assert retval.startswith('failed') + + +class TestMonitoring(object): + @mock.patch('cephadm.call') + def test_get_version_alertmanager(self, _call): + ctx = cd.CephadmContext() + ctx.container_engine = mock_podman() + daemon_type = 'alertmanager' + + # binary `prometheus` + _call.return_value = '', '{}, version 0.16.1'.format(daemon_type), 0 + version = cd.Monitoring.get_version(ctx, 'container_id', daemon_type) + assert version == '0.16.1' + + # binary `prometheus-alertmanager` + _call.side_effect = ( + ('', '', 1), + ('', '{}, version 0.16.1'.format(daemon_type), 0), + ) + version = cd.Monitoring.get_version(ctx, 'container_id', daemon_type) + assert version == '0.16.1' + + @mock.patch('cephadm.call') + def test_get_version_prometheus(self, _call): + ctx = cd.CephadmContext() + ctx.container_engine = mock_podman() + daemon_type = 'prometheus' + _call.return_value = '', '{}, version 0.16.1'.format(daemon_type), 0 + version = cd.Monitoring.get_version(ctx, 'container_id', daemon_type) + assert version == '0.16.1' + + def test_prometheus_external_url(self): + ctx = cd.CephadmContext() + ctx.config_json = json.dumps({'files': {}, 'retention_time': '15d'}) + daemon_type = 'prometheus' + daemon_id = 'home' + fsid = 'aaf5a720-13fe-4a3b-82b9-2d99b7fd9704' + args = cd.get_daemon_args(ctx, fsid, daemon_type, daemon_id) + assert any([x.startswith('--web.external-url=http://') for x in args]) + + @mock.patch('cephadm.call') + def test_get_version_node_exporter(self, _call): + ctx = cd.CephadmContext() + ctx.container_engine = mock_podman() + daemon_type = 'node-exporter' + _call.return_value = '', '{}, version 0.16.1'.format(daemon_type.replace('-', '_')), 0 + version = cd.Monitoring.get_version(ctx, 'container_id', daemon_type) + assert version == '0.16.1' + + def test_create_daemon_dirs_prometheus(self, cephadm_fs): + """ + Ensures the required and optional files given in the configuration are + created and mapped correctly inside the container. Tests absolute and + relative file paths given in the configuration. + """ + + fsid = 'aaf5a720-13fe-4a3b-82b9-2d99b7fd9704' + daemon_type = 'prometheus' + uid, gid = 50, 50 + daemon_id = 'home' + ctx = cd.CephadmContext() + ctx.data_dir = '/somedir' + ctx.config_json = json.dumps({ + 'files': { + 'prometheus.yml': 'foo', + '/etc/prometheus/alerting/ceph_alerts.yml': 'bar' + } + }) + + cd.create_daemon_dirs(ctx, + fsid, + daemon_type, + daemon_id, + uid, + gid, + config=None, + keyring=None) + + prefix = '{data_dir}/{fsid}/{daemon_type}.{daemon_id}'.format( + data_dir=ctx.data_dir, + fsid=fsid, + daemon_type=daemon_type, + daemon_id=daemon_id + ) + + expected = { + 'etc/prometheus/prometheus.yml': 'foo', + 'etc/prometheus/alerting/ceph_alerts.yml': 'bar', + } + + for file,content in expected.items(): + file = os.path.join(prefix, file) + assert os.path.exists(file) + with open(file) as f: + assert f.read() == content + + # assert uid/gid after redeploy + new_uid = uid+1 + new_gid = gid+1 + cd.create_daemon_dirs(ctx, + fsid, + daemon_type, + daemon_id, + new_uid, + new_gid, + config=None, + keyring=None) + for file,content in expected.items(): + file = os.path.join(prefix, file) + assert os.stat(file).st_uid == new_uid + assert os.stat(file).st_gid == new_gid + + +class TestBootstrap(object): + + @staticmethod + def _get_cmd(*args): + return [ + 'bootstrap', + '--allow-mismatched-release', + '--skip-prepare-host', + '--skip-dashboard', + *args, + ] + + +###############################################3 + + def test_config(self, cephadm_fs): + conf_file = 'foo' + cmd = self._get_cmd( + '--mon-ip', '192.168.1.1', + '--skip-mon-network', + '--config', conf_file, + ) + + with with_cephadm_ctx(cmd) as ctx: + msg = r'No such file or directory' + with pytest.raises(cd.Error, match=msg): + cd.command_bootstrap(ctx) + + cephadm_fs.create_file(conf_file) + with with_cephadm_ctx(cmd) as ctx: + retval = cd.command_bootstrap(ctx) + assert retval == 0 + + def test_no_mon_addr(self, cephadm_fs): + cmd = self._get_cmd() + with with_cephadm_ctx(cmd) as ctx: + msg = r'must specify --mon-ip or --mon-addrv' + with pytest.raises(cd.Error, match=msg): + cd.command_bootstrap(ctx) + + def test_skip_mon_network(self, cephadm_fs): + cmd = self._get_cmd('--mon-ip', '192.168.1.1') + + with with_cephadm_ctx(cmd, list_networks={}) as ctx: + msg = r'--skip-mon-network' + with pytest.raises(cd.Error, match=msg): + cd.command_bootstrap(ctx) + + cmd += ['--skip-mon-network'] + with with_cephadm_ctx(cmd, list_networks={}) as ctx: + retval = cd.command_bootstrap(ctx) + assert retval == 0 + + @pytest.mark.parametrize('mon_ip, list_networks, result', + [ + # IPv4 + ( + 'eth0', + {'192.168.1.0/24': {'eth0': ['192.168.1.1']}}, + False, + ), + ( + '0.0.0.0', + {'192.168.1.0/24': {'eth0': ['192.168.1.1']}}, + False, + ), + ( + '192.168.1.0', + {'192.168.1.0/24': {'eth0': ['192.168.1.1']}}, + False, + ), + ( + '192.168.1.1', + {'192.168.1.0/24': {'eth0': ['192.168.1.1']}}, + True, + ), + ( + '192.168.1.1:1234', + {'192.168.1.0/24': {'eth0': ['192.168.1.1']}}, + True, + ), + ( + '192.168.1.1:0123', + {'192.168.1.0/24': {'eth0': ['192.168.1.1']}}, + True, + ), + # IPv6 + ( + '::', + {'192.168.1.0/24': {'eth0': ['192.168.1.1']}}, + False, + ), + ( + '::ffff:192.168.1.0', + {"ffff::/64": {"eth0": ["::ffff:c0a8:101"]}}, + False, + ), + ( + '::ffff:192.168.1.1', + {"ffff::/64": {"eth0": ["::ffff:c0a8:101"]}}, + True, + ), + ( + '::ffff:c0a8:101', + {"ffff::/64": {"eth0": ["::ffff:c0a8:101"]}}, + True, + ), + ( + '[::ffff:c0a8:101]:1234', + {"ffff::/64": {"eth0": ["::ffff:c0a8:101"]}}, + True, + ), + ( + '[::ffff:c0a8:101]:0123', + {"ffff::/64": {"eth0": ["::ffff:c0a8:101"]}}, + True, + ), + ( + '0000:0000:0000:0000:0000:FFFF:C0A8:0101', + {"ffff::/64": {"eth0": ["::ffff:c0a8:101"]}}, + True, + ), + ]) + def test_mon_ip(self, mon_ip, list_networks, result, cephadm_fs): + cmd = self._get_cmd('--mon-ip', mon_ip) + if not result: + with with_cephadm_ctx(cmd, list_networks=list_networks) as ctx: + msg = r'--skip-mon-network' + with pytest.raises(cd.Error, match=msg): + cd.command_bootstrap(ctx) + else: + with with_cephadm_ctx(cmd, list_networks=list_networks) as ctx: + retval = cd.command_bootstrap(ctx) + assert retval == 0 + + @pytest.mark.parametrize('mon_addrv, list_networks, err', + [ + # IPv4 + ( + '192.168.1.1', + {'192.168.1.0/24': {'eth0': ['192.168.1.1']}}, + r'must use square backets', + ), + ( + '[192.168.1.1]', + {'192.168.1.0/24': {'eth0': ['192.168.1.1']}}, + r'must include port number', + ), + ( + '[192.168.1.1:1234]', + {'192.168.1.0/24': {'eth0': ['192.168.1.1']}}, + None, + ), + ( + '[192.168.1.1:0123]', + {'192.168.1.0/24': {'eth0': ['192.168.1.1']}}, + None, + ), + ( + '[v2:192.168.1.1:3300,v1:192.168.1.1:6789]', + {'192.168.1.0/24': {'eth0': ['192.168.1.1']}}, + None, + ), + # IPv6 + ( + '[::ffff:192.168.1.1:1234]', + {'ffff::/64': {'eth0': ['::ffff:c0a8:101']}}, + None, + ), + ( + '[::ffff:192.168.1.1:0123]', + {'ffff::/64': {'eth0': ['::ffff:c0a8:101']}}, + None, + ), + ( + '[0000:0000:0000:0000:0000:FFFF:C0A8:0101:1234]', + {'ffff::/64': {'eth0': ['::ffff:c0a8:101']}}, + None, + ), + ( + '[v2:0000:0000:0000:0000:0000:FFFF:C0A8:0101:3300,v1:0000:0000:0000:0000:0000:FFFF:C0A8:0101:6789]', + {'ffff::/64': {'eth0': ['::ffff:c0a8:101']}}, + None, + ), + ]) + def test_mon_addrv(self, mon_addrv, list_networks, err, cephadm_fs): + cmd = self._get_cmd('--mon-addrv', mon_addrv) + if err: + with with_cephadm_ctx(cmd, list_networks=list_networks) as ctx: + with pytest.raises(cd.Error, match=err): + cd.command_bootstrap(ctx) + else: + with with_cephadm_ctx(cmd, list_networks=list_networks) as ctx: + retval = cd.command_bootstrap(ctx) + assert retval == 0 + + def test_allow_fqdn_hostname(self, cephadm_fs): + hostname = 'foo.bar' + cmd = self._get_cmd( + '--mon-ip', '192.168.1.1', + '--skip-mon-network', + ) + + with with_cephadm_ctx(cmd, hostname=hostname) as ctx: + msg = r'--allow-fqdn-hostname' + with pytest.raises(cd.Error, match=msg): + cd.command_bootstrap(ctx) + + cmd += ['--allow-fqdn-hostname'] + with with_cephadm_ctx(cmd, hostname=hostname) as ctx: + retval = cd.command_bootstrap(ctx) + assert retval == 0 + + @pytest.mark.parametrize('fsid, err', + [ + ('', None), + ('00000000-0000-0000-0000-0000deadbeef', None), + ('00000000-0000-0000-0000-0000deadbeez', 'not an fsid'), + ]) + def test_fsid(self, fsid, err, cephadm_fs): + cmd = self._get_cmd( + '--mon-ip', '192.168.1.1', + '--skip-mon-network', + '--fsid', fsid, + ) + + with with_cephadm_ctx(cmd) as ctx: + if err: + with pytest.raises(cd.Error, match=err): + cd.command_bootstrap(ctx) + else: + retval = cd.command_bootstrap(ctx) + assert retval == 0 + + +class TestShell(object): + + def test_fsid(self, cephadm_fs): + fsid = '00000000-0000-0000-0000-0000deadbeef' + + cmd = ['shell', '--fsid', fsid] + with with_cephadm_ctx(cmd) as ctx: + retval = cd.command_shell(ctx) + assert retval == 0 + assert ctx.fsid == fsid + + cmd = ['shell', '--fsid', '00000000-0000-0000-0000-0000deadbeez'] + with with_cephadm_ctx(cmd) as ctx: + err = 'not an fsid' + with pytest.raises(cd.Error, match=err): + retval = cd.command_shell(ctx) + assert retval == 1 + assert ctx.fsid == None + + s = get_ceph_conf(fsid=fsid) + f = cephadm_fs.create_file('ceph.conf', contents=s) + + cmd = ['shell', '--fsid', fsid, '--config', f.path] + with with_cephadm_ctx(cmd) as ctx: + retval = cd.command_shell(ctx) + assert retval == 0 + assert ctx.fsid == fsid + + cmd = ['shell', '--fsid', '10000000-0000-0000-0000-0000deadbeef', '--config', f.path] + with with_cephadm_ctx(cmd) as ctx: + err = 'fsid does not match ceph.conf' + with pytest.raises(cd.Error, match=err): + retval = cd.command_shell(ctx) + assert retval == 1 + assert ctx.fsid == None + + def test_name(self, cephadm_fs): + cmd = ['shell', '--name', 'foo'] + with with_cephadm_ctx(cmd) as ctx: + retval = cd.command_shell(ctx) + assert retval == 0 + + cmd = ['shell', '--name', 'foo.bar'] + with with_cephadm_ctx(cmd) as ctx: + err = r'must pass --fsid' + with pytest.raises(cd.Error, match=err): + retval = cd.command_shell(ctx) + assert retval == 1 + + fsid = '00000000-0000-0000-0000-0000deadbeef' + cmd = ['shell', '--name', 'foo.bar', '--fsid', fsid] + with with_cephadm_ctx(cmd) as ctx: + retval = cd.command_shell(ctx) + assert retval == 0 + + def test_config(self, cephadm_fs): + cmd = ['shell'] + with with_cephadm_ctx(cmd) as ctx: + retval = cd.command_shell(ctx) + assert retval == 0 + assert ctx.config == None + + cephadm_fs.create_file(cd.SHELL_DEFAULT_CONF) + with with_cephadm_ctx(cmd) as ctx: + retval = cd.command_shell(ctx) + assert retval == 0 + assert ctx.config == cd.SHELL_DEFAULT_CONF + + cmd = ['shell', '--config', 'foo'] + with with_cephadm_ctx(cmd) as ctx: + retval = cd.command_shell(ctx) + assert retval == 0 + assert ctx.config == 'foo' + + def test_keyring(self, cephadm_fs): + cmd = ['shell'] + with with_cephadm_ctx(cmd) as ctx: + retval = cd.command_shell(ctx) + assert retval == 0 + assert ctx.keyring == None + + cephadm_fs.create_file(cd.SHELL_DEFAULT_KEYRING) + with with_cephadm_ctx(cmd) as ctx: + retval = cd.command_shell(ctx) + assert retval == 0 + assert ctx.keyring == cd.SHELL_DEFAULT_KEYRING + + cmd = ['shell', '--keyring', 'foo'] + with with_cephadm_ctx(cmd) as ctx: + retval = cd.command_shell(ctx) + assert retval == 0 + assert ctx.keyring == 'foo' + + @mock.patch('cephadm.CephContainer') + def test_mount_no_dst(self, m_ceph_container, cephadm_fs): + cmd = ['shell', '--mount', '/etc/foo'] + with with_cephadm_ctx(cmd) as ctx: + retval = cd.command_shell(ctx) + assert retval == 0 + assert m_ceph_container.call_args.kwargs['volume_mounts']['/etc/foo'] == '/mnt/foo' + + @mock.patch('cephadm.CephContainer') + def test_mount_with_dst_no_opt(self, m_ceph_container, cephadm_fs): + cmd = ['shell', '--mount', '/etc/foo:/opt/foo/bar'] + with with_cephadm_ctx(cmd) as ctx: + retval = cd.command_shell(ctx) + assert retval == 0 + assert m_ceph_container.call_args.kwargs['volume_mounts']['/etc/foo'] == '/opt/foo/bar' + + @mock.patch('cephadm.CephContainer') + def test_mount_with_dst_and_opt(self, m_ceph_container, cephadm_fs): + cmd = ['shell', '--mount', '/etc/foo:/opt/foo/bar:Z'] + with with_cephadm_ctx(cmd) as ctx: + retval = cd.command_shell(ctx) + assert retval == 0 + assert m_ceph_container.call_args.kwargs['volume_mounts']['/etc/foo'] == '/opt/foo/bar:Z' + +class TestCephVolume(object): + + @staticmethod + def _get_cmd(*args): + return [ + 'ceph-volume', + *args, + '--', 'inventory', '--format', 'json' + ] + + def test_noop(self, cephadm_fs): + cmd = self._get_cmd() + with with_cephadm_ctx(cmd) as ctx: + cd.command_ceph_volume(ctx) + assert ctx.fsid == None + assert ctx.config == None + assert ctx.keyring == None + assert ctx.config_json == None + + def test_fsid(self, cephadm_fs): + fsid = '00000000-0000-0000-0000-0000deadbeef' + + cmd = self._get_cmd('--fsid', fsid) + with with_cephadm_ctx(cmd) as ctx: + cd.command_ceph_volume(ctx) + assert ctx.fsid == fsid + + cmd = self._get_cmd('--fsid', '00000000-0000-0000-0000-0000deadbeez') + with with_cephadm_ctx(cmd) as ctx: + err = 'not an fsid' + with pytest.raises(cd.Error, match=err): + retval = cd.command_shell(ctx) + assert retval == 1 + assert ctx.fsid == None + + s = get_ceph_conf(fsid=fsid) + f = cephadm_fs.create_file('ceph.conf', contents=s) + + cmd = self._get_cmd('--fsid', fsid, '--config', f.path) + with with_cephadm_ctx(cmd) as ctx: + cd.command_ceph_volume(ctx) + assert ctx.fsid == fsid + + cmd = self._get_cmd('--fsid', '10000000-0000-0000-0000-0000deadbeef', '--config', f.path) + with with_cephadm_ctx(cmd) as ctx: + err = 'fsid does not match ceph.conf' + with pytest.raises(cd.Error, match=err): + cd.command_ceph_volume(ctx) + assert ctx.fsid == None + + def test_config(self, cephadm_fs): + cmd = self._get_cmd('--config', 'foo') + with with_cephadm_ctx(cmd) as ctx: + err = r'No such file or directory' + with pytest.raises(cd.Error, match=err): + cd.command_ceph_volume(ctx) + + cephadm_fs.create_file('bar') + cmd = self._get_cmd('--config', 'bar') + with with_cephadm_ctx(cmd) as ctx: + cd.command_ceph_volume(ctx) + assert ctx.config == 'bar' + + def test_keyring(self, cephadm_fs): + cmd = self._get_cmd('--keyring', 'foo') + with with_cephadm_ctx(cmd) as ctx: + err = r'No such file or directory' + with pytest.raises(cd.Error, match=err): + cd.command_ceph_volume(ctx) + + cephadm_fs.create_file('bar') + cmd = self._get_cmd('--keyring', 'bar') + with with_cephadm_ctx(cmd) as ctx: + cd.command_ceph_volume(ctx) + assert ctx.keyring == 'bar' + + +class TestIscsi: + def test_unit_run(self, cephadm_fs): + fsid = '9b9d7609-f4d5-4aba-94c8-effa764d96c9' + config_json = { + 'files': {'iscsi-gateway.cfg': ''} + } + with with_cephadm_ctx(['--image=ceph/ceph'], list_networks={}) as ctx: + import json + ctx.config_json = json.dumps(config_json) + ctx.fsid = fsid + cd.get_parm.return_value = config_json + c = cd.get_container(ctx, fsid, 'iscsi', 'daemon_id') + + cd.make_data_dir(ctx, fsid, 'iscsi', 'daemon_id') + cd.deploy_daemon_units( + ctx, + fsid, + 0, 0, + 'iscsi', + 'daemon_id', + c, + True, True + ) + + with open('/var/lib/ceph/9b9d7609-f4d5-4aba-94c8-effa764d96c9/iscsi.daemon_id/unit.run') as f: + assert f.read() == """set -e +if ! grep -qs /var/lib/ceph/9b9d7609-f4d5-4aba-94c8-effa764d96c9/iscsi.daemon_id/configfs /proc/mounts; then mount -t configfs none /var/lib/ceph/9b9d7609-f4d5-4aba-94c8-effa764d96c9/iscsi.daemon_id/configfs; fi +# iscsi tcmu-runner container +! /usr/bin/podman rm -f ceph-9b9d7609-f4d5-4aba-94c8-effa764d96c9-iscsi.daemon_id-tcmu 2> /dev/null +! /usr/bin/podman rm -f ceph-9b9d7609-f4d5-4aba-94c8-effa764d96c9-iscsi-daemon_id-tcmu 2> /dev/null +/usr/bin/podman run --rm --ipc=host --stop-signal=SIGTERM --net=host --entrypoint /usr/bin/tcmu-runner --privileged --group-add=disk --init --name ceph-9b9d7609-f4d5-4aba-94c8-effa764d96c9-iscsi-daemon_id-tcmu -e CONTAINER_IMAGE=ceph/ceph -e NODE_NAME=host1 -e CEPH_USE_RANDOM_NONCE=1 -v /var/lib/ceph/9b9d7609-f4d5-4aba-94c8-effa764d96c9/iscsi.daemon_id/config:/etc/ceph/ceph.conf:z -v /var/lib/ceph/9b9d7609-f4d5-4aba-94c8-effa764d96c9/iscsi.daemon_id/keyring:/etc/ceph/keyring:z -v /var/lib/ceph/9b9d7609-f4d5-4aba-94c8-effa764d96c9/iscsi.daemon_id/iscsi-gateway.cfg:/etc/ceph/iscsi-gateway.cfg:z -v /var/lib/ceph/9b9d7609-f4d5-4aba-94c8-effa764d96c9/iscsi.daemon_id/configfs:/sys/kernel/config -v /var/log/ceph/9b9d7609-f4d5-4aba-94c8-effa764d96c9:/var/log:z -v /dev:/dev --mount type=bind,source=/lib/modules,destination=/lib/modules,ro=true ceph/ceph & +# iscsi.daemon_id +! /usr/bin/podman rm -f ceph-9b9d7609-f4d5-4aba-94c8-effa764d96c9-iscsi.daemon_id 2> /dev/null +! /usr/bin/podman rm -f ceph-9b9d7609-f4d5-4aba-94c8-effa764d96c9-iscsi-daemon_id 2> /dev/null +/usr/bin/podman run --rm --ipc=host --stop-signal=SIGTERM --net=host --entrypoint /usr/bin/rbd-target-api --privileged --group-add=disk --init --name ceph-9b9d7609-f4d5-4aba-94c8-effa764d96c9-iscsi-daemon_id -e CONTAINER_IMAGE=ceph/ceph -e NODE_NAME=host1 -e CEPH_USE_RANDOM_NONCE=1 -v /var/lib/ceph/9b9d7609-f4d5-4aba-94c8-effa764d96c9/iscsi.daemon_id/config:/etc/ceph/ceph.conf:z -v /var/lib/ceph/9b9d7609-f4d5-4aba-94c8-effa764d96c9/iscsi.daemon_id/keyring:/etc/ceph/keyring:z -v /var/lib/ceph/9b9d7609-f4d5-4aba-94c8-effa764d96c9/iscsi.daemon_id/iscsi-gateway.cfg:/etc/ceph/iscsi-gateway.cfg:z -v /var/lib/ceph/9b9d7609-f4d5-4aba-94c8-effa764d96c9/iscsi.daemon_id/configfs:/sys/kernel/config -v /var/log/ceph/9b9d7609-f4d5-4aba-94c8-effa764d96c9:/var/log:z -v /dev:/dev --mount type=bind,source=/lib/modules,destination=/lib/modules,ro=true ceph/ceph +""" + + def test_get_container(self): + """ + Due to a combination of socket.getfqdn() and podman's behavior to + add the container name into the /etc/hosts file, we cannot use periods + in container names. But we need to be able to detect old existing containers. + Assert this behaviour. I think we can remove this in Ceph R + """ + fsid = '9b9d7609-f4d5-4aba-94c8-effa764d96c9' + with with_cephadm_ctx(['--image=ceph/ceph'], list_networks={}) as ctx: + ctx.fsid = fsid + c = cd.get_container(ctx, fsid, 'iscsi', 'something') + assert c.cname == 'ceph-9b9d7609-f4d5-4aba-94c8-effa764d96c9-iscsi-something' + assert c.old_cname == 'ceph-9b9d7609-f4d5-4aba-94c8-effa764d96c9-iscsi.something' + + +class TestCheckHost: + + @mock.patch('cephadm.find_executable', return_value='foo') + @mock.patch('cephadm.check_time_sync', return_value=True) + def test_container_engine(self, find_executable, check_time_sync): + ctx = cd.CephadmContext() + + ctx.container_engine = None + err = r'No container engine binary found' + with pytest.raises(cd.Error, match=err): + cd.command_check_host(ctx) + + ctx.container_engine = mock_podman() + cd.command_check_host(ctx) + + ctx.container_engine = mock_docker() + cd.command_check_host(ctx) + + +class TestRmRepo: + + @pytest.mark.parametrize('os_release', + [ + # Apt + dedent(""" + NAME="Ubuntu" + VERSION="20.04 LTS (Focal Fossa)" + ID=ubuntu + ID_LIKE=debian + PRETTY_NAME="Ubuntu 20.04 LTS" + VERSION_ID="20.04" + HOME_URL="https://www.ubuntu.com/" + SUPPORT_URL="https://help.ubuntu.com/" + BUG_REPORT_URL="https://bugs.launchpad.net/ubuntu/" + PRIVACY_POLICY_URL="https://www.ubuntu.com/legal/terms-and-policies/privacy-policy" + VERSION_CODENAME=focal + UBUNTU_CODENAME=focal + """), + + # YumDnf + dedent(""" + NAME="CentOS Linux" + VERSION="8 (Core)" + ID="centos" + ID_LIKE="rhel fedora" + VERSION_ID="8" + PLATFORM_ID="platform:el8" + PRETTY_NAME="CentOS Linux 8 (Core)" + ANSI_COLOR="0;31" + CPE_NAME="cpe:/o:centos:centos:8" + HOME_URL="https://www.centos.org/" + BUG_REPORT_URL="https://bugs.centos.org/" + + CENTOS_MANTISBT_PROJECT="CentOS-8" + CENTOS_MANTISBT_PROJECT_VERSION="8" + REDHAT_SUPPORT_PRODUCT="centos" + REDHAT_SUPPORT_PRODUCT_VERSION="8" + """), + + # Zypper + dedent(""" + NAME="openSUSE Tumbleweed" + # VERSION="20210810" + ID="opensuse-tumbleweed" + ID_LIKE="opensuse suse" + VERSION_ID="20210810" + PRETTY_NAME="openSUSE Tumbleweed" + ANSI_COLOR="0;32" + CPE_NAME="cpe:/o:opensuse:tumbleweed:20210810" + BUG_REPORT_URL="https://bugs.opensuse.org" + HOME_URL="https://www.opensuse.org/" + DOCUMENTATION_URL="https://en.opensuse.org/Portal:Tumbleweed" + LOGO="distributor-logo" + """), + ]) + @mock.patch('cephadm.find_executable', return_value='foo') + def test_container_engine(self, find_executable, os_release, cephadm_fs): + cephadm_fs.create_file('/etc/os-release', contents=os_release) + ctx = cd.CephadmContext() + + ctx.container_engine = None + cd.command_rm_repo(ctx) + + ctx.container_engine = mock_podman() + cd.command_rm_repo(ctx) + + ctx.container_engine = mock_docker() + cd.command_rm_repo(ctx) + + +class TestValidateRepo: + + @pytest.mark.parametrize('values', + [ + # Apt - no checks + dict( + version="", + release="pacific", + err_text="", + os_release=dedent(""" + NAME="Ubuntu" + VERSION="20.04 LTS (Focal Fossa)" + ID=ubuntu + ID_LIKE=debian + PRETTY_NAME="Ubuntu 20.04 LTS" + VERSION_ID="20.04" + HOME_URL="https://www.ubuntu.com/" + SUPPORT_URL="https://help.ubuntu.com/" + BUG_REPORT_URL="https://bugs.launchpad.net/ubuntu/" + PRIVACY_POLICY_URL="https://www.ubuntu.com/legal/terms-and-policies/privacy-policy" + VERSION_CODENAME=focal + UBUNTU_CODENAME=focal + """)), + + # YumDnf on Centos8 - OK + dict( + version="", + release="pacific", + err_text="", + os_release=dedent(""" + NAME="CentOS Linux" + VERSION="8 (Core)" + ID="centos" + ID_LIKE="rhel fedora" + VERSION_ID="8" + PLATFORM_ID="platform:el8" + PRETTY_NAME="CentOS Linux 8 (Core)" + ANSI_COLOR="0;31" + CPE_NAME="cpe:/o:centos:centos:8" + HOME_URL="https://www.centos.org/" + BUG_REPORT_URL="https://bugs.centos.org/" + + CENTOS_MANTISBT_PROJECT="CentOS-8" + CENTOS_MANTISBT_PROJECT_VERSION="8" + REDHAT_SUPPORT_PRODUCT="centos" + REDHAT_SUPPORT_PRODUCT_VERSION="8" + """)), + + # YumDnf on Fedora - Fedora not supported + dict( + version="", + release="pacific", + err_text="does not build Fedora", + os_release=dedent(""" + NAME="Fedora Linux" + VERSION="35 (Cloud Edition)" + ID=fedora + VERSION_ID=35 + VERSION_CODENAME="" + PLATFORM_ID="platform:f35" + PRETTY_NAME="Fedora Linux 35 (Cloud Edition)" + ANSI_COLOR="0;38;2;60;110;180" + LOGO=fedora-logo-icon + CPE_NAME="cpe:/o:fedoraproject:fedora:35" + HOME_URL="https://fedoraproject.org/" + DOCUMENTATION_URL="https://docs.fedoraproject.org/en-US/fedora/f35/system-administrators-guide/" + SUPPORT_URL="https://ask.fedoraproject.org/" + BUG_REPORT_URL="https://bugzilla.redhat.com/" + REDHAT_BUGZILLA_PRODUCT="Fedora" + REDHAT_BUGZILLA_PRODUCT_VERSION=35 + REDHAT_SUPPORT_PRODUCT="Fedora" + REDHAT_SUPPORT_PRODUCT_VERSION=35 + PRIVACY_POLICY_URL="https://fedoraproject.org/wiki/Legal:PrivacyPolicy" + VARIANT="Cloud Edition" + VARIANT_ID=cloud + """)), + + # YumDnf on Centos 7 - no pacific + dict( + version="", + release="pacific", + err_text="does not support pacific", + os_release=dedent(""" + NAME="CentOS Linux" + VERSION="7 (Core)" + ID="centos" + ID_LIKE="rhel fedora" + VERSION_ID="7" + PRETTY_NAME="CentOS Linux 7 (Core)" + ANSI_COLOR="0;31" + CPE_NAME="cpe:/o:centos:centos:7" + HOME_URL="https://www.centos.org/" + BUG_REPORT_URL="https://bugs.centos.org/" + + CENTOS_MANTISBT_PROJECT="CentOS-7" + CENTOS_MANTISBT_PROJECT_VERSION="7" + REDHAT_SUPPORT_PRODUCT="centos" + REDHAT_SUPPORT_PRODUCT_VERSION="7" + """)), + + # YumDnf on Centos 7 - nothing after pacific + dict( + version="", + release="zillions", + err_text="does not support pacific", + os_release=dedent(""" + NAME="CentOS Linux" + VERSION="7 (Core)" + ID="centos" + ID_LIKE="rhel fedora" + VERSION_ID="7" + PRETTY_NAME="CentOS Linux 7 (Core)" + ANSI_COLOR="0;31" + CPE_NAME="cpe:/o:centos:centos:7" + HOME_URL="https://www.centos.org/" + BUG_REPORT_URL="https://bugs.centos.org/" + + CENTOS_MANTISBT_PROJECT="CentOS-7" + CENTOS_MANTISBT_PROJECT_VERSION="7" + REDHAT_SUPPORT_PRODUCT="centos" + REDHAT_SUPPORT_PRODUCT_VERSION="7" + """)), + + # YumDnf on Centos 7 - nothing v16 or higher + dict( + version="v16.1.3", + release="", + err_text="does not support", + os_release=dedent(""" + NAME="CentOS Linux" + VERSION="7 (Core)" + ID="centos" + ID_LIKE="rhel fedora" + VERSION_ID="7" + PRETTY_NAME="CentOS Linux 7 (Core)" + ANSI_COLOR="0;31" + CPE_NAME="cpe:/o:centos:centos:7" + HOME_URL="https://www.centos.org/" + BUG_REPORT_URL="https://bugs.centos.org/" + + CENTOS_MANTISBT_PROJECT="CentOS-7" + CENTOS_MANTISBT_PROJECT_VERSION="7" + REDHAT_SUPPORT_PRODUCT="centos" + REDHAT_SUPPORT_PRODUCT_VERSION="7" + """)), + ]) + @mock.patch('cephadm.find_executable', return_value='foo') + def test_distro_validation(self, find_executable, values, cephadm_fs): + os_release = values['os_release'] + release = values['release'] + version = values['version'] + err_text = values['err_text'] + + cephadm_fs.create_file('/etc/os-release', contents=os_release) + ctx = cd.CephadmContext() + ctx.repo_url = 'http://localhost' + pkg = cd.create_packager(ctx, stable=release, version=version) + + if err_text: + with pytest.raises(cd.Error, match=err_text): + pkg.validate() + else: + with mock.patch('cephadm.urlopen', return_value=None): + pkg.validate() + + @pytest.mark.parametrize('values', + [ + # Apt - not checked + dict( + version="", + release="pacific", + err_text="", + os_release=dedent(""" + NAME="Ubuntu" + VERSION="20.04 LTS (Focal Fossa)" + ID=ubuntu + ID_LIKE=debian + PRETTY_NAME="Ubuntu 20.04 LTS" + VERSION_ID="20.04" + HOME_URL="https://www.ubuntu.com/" + SUPPORT_URL="https://help.ubuntu.com/" + BUG_REPORT_URL="https://bugs.launchpad.net/ubuntu/" + PRIVACY_POLICY_URL="https://www.ubuntu.com/legal/terms-and-policies/privacy-policy" + VERSION_CODENAME=focal + UBUNTU_CODENAME=focal + """)), + + # YumDnf on Centos8 - force failure + dict( + version="", + release="foobar", + err_text="failed to fetch repository metadata", + os_release=dedent(""" + NAME="CentOS Linux" + VERSION="8 (Core)" + ID="centos" + ID_LIKE="rhel fedora" + VERSION_ID="8" + PLATFORM_ID="platform:el8" + PRETTY_NAME="CentOS Linux 8 (Core)" + ANSI_COLOR="0;31" + CPE_NAME="cpe:/o:centos:centos:8" + HOME_URL="https://www.centos.org/" + BUG_REPORT_URL="https://bugs.centos.org/" + + CENTOS_MANTISBT_PROJECT="CentOS-8" + CENTOS_MANTISBT_PROJECT_VERSION="8" + REDHAT_SUPPORT_PRODUCT="centos" + REDHAT_SUPPORT_PRODUCT_VERSION="8" + """)), + ]) + @mock.patch('cephadm.find_executable', return_value='foo') + def test_http_validation(self, find_executable, values, cephadm_fs): + from urllib.error import HTTPError + + os_release = values['os_release'] + release = values['release'] + version = values['version'] + err_text = values['err_text'] + + cephadm_fs.create_file('/etc/os-release', contents=os_release) + ctx = cd.CephadmContext() + ctx.repo_url = 'http://localhost' + pkg = cd.create_packager(ctx, stable=release, version=version) + + with mock.patch('cephadm.urlopen') as _urlopen: + _urlopen.side_effect = HTTPError(ctx.repo_url, 404, "not found", None, fp=None) + if err_text: + with pytest.raises(cd.Error, match=err_text): + pkg.validate() + else: + pkg.validate() + + +class TestPull: + + @mock.patch('time.sleep') + @mock.patch('cephadm.call', return_value=('', '', 0)) + @mock.patch('cephadm.get_image_info_from_inspect', return_value={}) + def test_error(self, get_image_info_from_inspect, call, sleep): + ctx = cd.CephadmContext() + ctx.container_engine = mock_podman() + ctx.insecure = False + + call.return_value = ('', '', 0) + retval = cd.command_pull(ctx) + assert retval == 0 + + err = 'maximum retries reached' + + call.return_value = ('', 'foobar', 1) + with pytest.raises(cd.Error) as e: + cd.command_pull(ctx) + assert err not in str(e.value) + + call.return_value = ('', 'net/http: TLS handshake timeout', 1) + with pytest.raises(cd.Error) as e: + cd.command_pull(ctx) + assert err in str(e.value) + + @mock.patch('cephadm.logger') + @mock.patch('cephadm.get_image_info_from_inspect', return_value={}) + @mock.patch('cephadm.get_last_local_ceph_image', return_value='last_local_ceph_image') + def test_image(self, get_last_local_ceph_image, get_image_info_from_inspect, logger): + cmd = ['pull'] + with with_cephadm_ctx(cmd) as ctx: + retval = cd.command_pull(ctx) + assert retval == 0 + assert ctx.image == cd.DEFAULT_IMAGE + + with mock.patch.dict(os.environ, {"CEPHADM_IMAGE": 'cephadm_image_environ'}): + cmd = ['pull'] + with with_cephadm_ctx(cmd) as ctx: + retval = cd.command_pull(ctx) + assert retval == 0 + assert ctx.image == 'cephadm_image_environ' + + cmd = ['--image', 'cephadm_image_param', 'pull'] + with with_cephadm_ctx(cmd) as ctx: + retval = cd.command_pull(ctx) + assert retval == 0 + assert ctx.image == 'cephadm_image_param' + + +class TestApplySpec: + + def test_parse_yaml(self, cephadm_fs): + yaml = '''--- +service_type: host +hostname: vm-00 +addr: 192.168.122.44 +labels: + - example1 + - example2 +--- +service_type: host +hostname: vm-01 +addr: 192.168.122.247 +labels: + - grafana +--- +service_type: host +hostname: vm-02 +addr: 192.168.122.165 +--- +--- +service_type: rgw +service_id: myrgw +spec: + rgw_frontend_ssl_certificate: | + -----BEGIN PRIVATE KEY----- + V2VyIGRhcyBsaWVzdCBpc3QgZG9vZi4gTG9yZW0gaXBzdW0gZG9sb3Igc2l0IGFt + ZXQsIGNvbnNldGV0dXIgc2FkaXBzY2luZyBlbGl0ciwgc2VkIGRpYW0gbm9udW15 + IGVpcm1vZCB0ZW1wb3IgaW52aWR1bnQgdXQgbGFib3JlIGV0IGRvbG9yZSBtYWdu + YSBhbGlxdXlhbSBlcmF0LCBzZWQgZGlhbSB2b2x1cHR1YS4gQXQgdmVybyBlb3Mg + ZXQgYWNjdXNhbSBldCBqdXN0byBkdW8= + -----END PRIVATE KEY----- + -----BEGIN CERTIFICATE----- + V2VyIGRhcyBsaWVzdCBpc3QgZG9vZi4gTG9yZW0gaXBzdW0gZG9sb3Igc2l0IGFt + ZXQsIGNvbnNldGV0dXIgc2FkaXBzY2luZyBlbGl0ciwgc2VkIGRpYW0gbm9udW15 + IGVpcm1vZCB0ZW1wb3IgaW52aWR1bnQgdXQgbGFib3JlIGV0IGRvbG9yZSBtYWdu + YSBhbGlxdXlhbSBlcmF0LCBzZWQgZGlhbSB2b2x1cHR1YS4gQXQgdmVybyBlb3Mg + ZXQgYWNjdXNhbSBldCBqdXN0byBkdW8= + -----END CERTIFICATE----- + ssl: true +--- +''' + + cephadm_fs.create_file('spec.yml', contents=yaml) + retdic = [{'service_type': 'host', 'hostname': 'vm-00', 'addr': '192.168.122.44', 'labels': '- example1- example2'}, + {'service_type': 'host', 'hostname': 'vm-01', 'addr': '192.168.122.247', 'labels': '- grafana'}, + {'service_type': 'host', 'hostname': 'vm-02', 'addr': '192.168.122.165'}, + {'service_id': 'myrgw', + 'service_type': 'rgw', + 'spec': + 'rgw_frontend_ssl_certificate: |-----BEGIN PRIVATE ' + 'KEY-----V2VyIGRhcyBsaWVzdCBpc3QgZG9vZi4gTG9yZW0gaXBzdW0gZG9sb3Igc2l0IGFtZXQsIGNvbnNldGV0dXIgc2FkaXBzY2luZyBlbGl0ciwgc2VkIGRpYW0gbm9udW15IGVpcm1vZCB0ZW1wb3IgaW52aWR1bnQgdXQgbGFib3JlIGV0IGRvbG9yZSBtYWduYSBhbGlxdXlhbSBlcmF0LCBzZWQgZGlhbSB2b2x1cHR1YS4gQXQgdmVybyBlb3MgZXQgYWNjdXNhbSBldCBqdXN0byBkdW8=-----END ' + 'PRIVATE KEY----------BEGIN ' + 'CERTIFICATE-----V2VyIGRhcyBsaWVzdCBpc3QgZG9vZi4gTG9yZW0gaXBzdW0gZG9sb3Igc2l0IGFtZXQsIGNvbnNldGV0dXIgc2FkaXBzY2luZyBlbGl0ciwgc2VkIGRpYW0gbm9udW15IGVpcm1vZCB0ZW1wb3IgaW52aWR1bnQgdXQgbGFib3JlIGV0IGRvbG9yZSBtYWduYSBhbGlxdXlhbSBlcmF0LCBzZWQgZGlhbSB2b2x1cHR1YS4gQXQgdmVybyBlb3MgZXQgYWNjdXNhbSBldCBqdXN0byBkdW8=-----END ' + 'CERTIFICATE-----ssl: true'}] + + with open('spec.yml') as f: + dic = cd.parse_yaml_objs(f) + assert dic == retdic + + @mock.patch('cephadm.call', return_value=('', '', 0)) + def test_distribute_ssh_keys(self, call): + ctx = cd.CephadmContext() + ctx.ssh_public_key = None + ctx.ssh_user = 'root' + + host_spec = {'service_type': 'host', 'hostname': 'vm-02', 'addr': '192.168.122.165'} + + retval = cd._distribute_ssh_keys(ctx, host_spec, 'bootstrap_hostname') + + assert retval == 0 + + call.return_value = ('', '', 1) + + retval = cd._distribute_ssh_keys(ctx, host_spec, 'bootstrap_hostname') + + assert retval == 1 + + +class TestSNMPGateway: + V2c_config = { + 'snmp_community': 'public', + 'destination': '192.168.1.10:162', + 'snmp_version': 'V2c', + } + V3_no_priv_config = { + 'destination': '192.168.1.10:162', + 'snmp_version': 'V3', + 'snmp_v3_auth_username': 'myuser', + 'snmp_v3_auth_password': 'mypassword', + 'snmp_v3_auth_protocol': 'SHA', + 'snmp_v3_engine_id': '8000C53F00000000', + } + V3_priv_config = { + 'destination': '192.168.1.10:162', + 'snmp_version': 'V3', + 'snmp_v3_auth_username': 'myuser', + 'snmp_v3_auth_password': 'mypassword', + 'snmp_v3_auth_protocol': 'SHA', + 'snmp_v3_priv_protocol': 'DES', + 'snmp_v3_priv_password': 'mysecret', + 'snmp_v3_engine_id': '8000C53F00000000', + } + no_destination_config = { + 'snmp_version': 'V3', + 'snmp_v3_auth_username': 'myuser', + 'snmp_v3_auth_password': 'mypassword', + 'snmp_v3_auth_protocol': 'SHA', + 'snmp_v3_priv_protocol': 'DES', + 'snmp_v3_priv_password': 'mysecret', + 'snmp_v3_engine_id': '8000C53F00000000', + } + bad_version_config = { + 'snmp_community': 'public', + 'destination': '192.168.1.10:162', + 'snmp_version': 'V1', + } + + def test_unit_run_V2c(self, cephadm_fs): + fsid = 'ca734440-3dc6-11ec-9b98-5254002537a6' + with with_cephadm_ctx(['--image=docker.io/maxwo/snmp-notifier:v1.2.1'], list_networks={}) as ctx: + import json + ctx.config_json = json.dumps(self.V2c_config) + ctx.fsid = fsid + ctx.tcp_ports = '9464' + cd.get_parm.return_value = self.V2c_config + c = cd.get_container(ctx, fsid, 'snmp-gateway', 'daemon_id') + + cd.make_data_dir(ctx, fsid, 'snmp-gateway', 'daemon_id') + + cd.create_daemon_dirs(ctx, fsid, 'snmp-gateway', 'daemon_id', 0, 0) + with open(f'/var/lib/ceph/{fsid}/snmp-gateway.daemon_id/snmp-gateway.conf', 'r') as f: + conf = f.read().rstrip() + assert conf == 'SNMP_NOTIFIER_COMMUNITY=public' + + cd.deploy_daemon_units( + ctx, + fsid, + 0, 0, + 'snmp-gateway', + 'daemon_id', + c, + True, True + ) + with open(f'/var/lib/ceph/{fsid}/snmp-gateway.daemon_id/unit.run', 'r') as f: + run_cmd = f.readlines()[-1].rstrip() + assert run_cmd.endswith('docker.io/maxwo/snmp-notifier:v1.2.1 --web.listen-address=:9464 --snmp.destination=192.168.1.10:162 --snmp.version=V2c --log.level=info --snmp.trap-description-template=/etc/snmp_notifier/description-template.tpl') + + def test_unit_run_V3_noPriv(self, cephadm_fs): + fsid = 'ca734440-3dc6-11ec-9b98-5254002537a6' + with with_cephadm_ctx(['--image=docker.io/maxwo/snmp-notifier:v1.2.1'], list_networks={}) as ctx: + import json + ctx.config_json = json.dumps(self.V3_no_priv_config) + ctx.fsid = fsid + ctx.tcp_ports = '9465' + cd.get_parm.return_value = self.V3_no_priv_config + c = cd.get_container(ctx, fsid, 'snmp-gateway', 'daemon_id') + + cd.make_data_dir(ctx, fsid, 'snmp-gateway', 'daemon_id') + + cd.create_daemon_dirs(ctx, fsid, 'snmp-gateway', 'daemon_id', 0, 0) + with open(f'/var/lib/ceph/{fsid}/snmp-gateway.daemon_id/snmp-gateway.conf', 'r') as f: + conf = f.read() + assert conf == 'SNMP_NOTIFIER_AUTH_USERNAME=myuser\nSNMP_NOTIFIER_AUTH_PASSWORD=mypassword\n' + + cd.deploy_daemon_units( + ctx, + fsid, + 0, 0, + 'snmp-gateway', + 'daemon_id', + c, + True, True + ) + with open(f'/var/lib/ceph/{fsid}/snmp-gateway.daemon_id/unit.run', 'r') as f: + run_cmd = f.readlines()[-1].rstrip() + assert run_cmd.endswith('docker.io/maxwo/snmp-notifier:v1.2.1 --web.listen-address=:9465 --snmp.destination=192.168.1.10:162 --snmp.version=V3 --log.level=info --snmp.trap-description-template=/etc/snmp_notifier/description-template.tpl --snmp.authentication-enabled --snmp.authentication-protocol=SHA --snmp.security-engine-id=8000C53F00000000') + + def test_unit_run_V3_Priv(self, cephadm_fs): + fsid = 'ca734440-3dc6-11ec-9b98-5254002537a6' + with with_cephadm_ctx(['--image=docker.io/maxwo/snmp-notifier:v1.2.1'], list_networks={}) as ctx: + import json + ctx.config_json = json.dumps(self.V3_priv_config) + ctx.fsid = fsid + ctx.tcp_ports = '9464' + cd.get_parm.return_value = self.V3_priv_config + c = cd.get_container(ctx, fsid, 'snmp-gateway', 'daemon_id') + + cd.make_data_dir(ctx, fsid, 'snmp-gateway', 'daemon_id') + + cd.create_daemon_dirs(ctx, fsid, 'snmp-gateway', 'daemon_id', 0, 0) + with open(f'/var/lib/ceph/{fsid}/snmp-gateway.daemon_id/snmp-gateway.conf', 'r') as f: + conf = f.read() + assert conf == 'SNMP_NOTIFIER_AUTH_USERNAME=myuser\nSNMP_NOTIFIER_AUTH_PASSWORD=mypassword\nSNMP_NOTIFIER_PRIV_PASSWORD=mysecret\n' + + cd.deploy_daemon_units( + ctx, + fsid, + 0, 0, + 'snmp-gateway', + 'daemon_id', + c, + True, True + ) + with open(f'/var/lib/ceph/{fsid}/snmp-gateway.daemon_id/unit.run', 'r') as f: + run_cmd = f.readlines()[-1].rstrip() + assert run_cmd.endswith('docker.io/maxwo/snmp-notifier:v1.2.1 --web.listen-address=:9464 --snmp.destination=192.168.1.10:162 --snmp.version=V3 --log.level=info --snmp.trap-description-template=/etc/snmp_notifier/description-template.tpl --snmp.authentication-enabled --snmp.authentication-protocol=SHA --snmp.security-engine-id=8000C53F00000000 --snmp.private-enabled --snmp.private-protocol=DES') + + def test_unit_run_no_dest(self, cephadm_fs): + fsid = 'ca734440-3dc6-11ec-9b98-5254002537a6' + with with_cephadm_ctx(['--image=docker.io/maxwo/snmp-notifier:v1.2.1'], list_networks={}) as ctx: + import json + ctx.config_json = json.dumps(self.no_destination_config) + ctx.fsid = fsid + ctx.tcp_ports = '9464' + cd.get_parm.return_value = self.no_destination_config + + with pytest.raises(Exception) as e: + c = cd.get_container(ctx, fsid, 'snmp-gateway', 'daemon_id') + assert str(e.value) == "config is missing destination attribute(<ip>:<port>) of the target SNMP listener" + + def test_unit_run_bad_version(self, cephadm_fs): + fsid = 'ca734440-3dc6-11ec-9b98-5254002537a6' + with with_cephadm_ctx(['--image=docker.io/maxwo/snmp-notifier:v1.2.1'], list_networks={}) as ctx: + import json + ctx.config_json = json.dumps(self.bad_version_config) + ctx.fsid = fsid + ctx.tcp_ports = '9464' + cd.get_parm.return_value = self.bad_version_config + + with pytest.raises(Exception) as e: + c = cd.get_container(ctx, fsid, 'snmp-gateway', 'daemon_id') + assert str(e.value) == 'not a valid snmp version: V1' + +class TestNetworkValidation: + + def test_ipv4_subnet(self): + rc, v, msg = cd.check_subnet('192.168.1.0/24') + assert rc == 0 and v[0] == 4 + + def test_ipv4_subnet_list(self): + rc, v, msg = cd.check_subnet('192.168.1.0/24,10.90.90.0/24') + assert rc == 0 and not msg + + def test_ipv4_subnet_list_with_spaces(self): + rc, v, msg = cd.check_subnet('192.168.1.0/24, 10.90.90.0/24 ') + assert rc == 0 and not msg + + def test_ipv4_subnet_badlist(self): + rc, v, msg = cd.check_subnet('192.168.1.0/24,192.168.1.1') + assert rc == 1 and msg + + def test_ipv4_subnet_mixed(self): + rc, v, msg = cd.check_subnet('192.168.100.0/24,fe80::/64') + assert rc == 0 and v == [4,6] + + def test_ipv6_subnet(self): + rc, v, msg = cd.check_subnet('fe80::/64') + assert rc == 0 and v[0] == 6 + + def test_subnet_mask_missing(self): + rc, v, msg = cd.check_subnet('192.168.1.58') + assert rc == 1 and msg + + def test_subnet_mask_junk(self): + rc, v, msg = cd.check_subnet('wah') + assert rc == 1 and msg + + def test_ip_in_subnet(self): + # valid ip and only one valid subnet + rc = cd.ip_in_subnets('192.168.100.1', '192.168.100.0/24') + assert rc is True + + # valid ip and valid subnets list without spaces + rc = cd.ip_in_subnets('192.168.100.1', '192.168.100.0/24,10.90.90.0/24') + assert rc is True + + # valid ip and valid subnets list with spaces + rc = cd.ip_in_subnets('10.90.90.2', '192.168.1.0/24, 192.168.100.0/24, 10.90.90.0/24') + assert rc is True + + # valid ip that doesn't belong to any subnet + rc = cd.ip_in_subnets('192.168.100.2', '192.168.50.0/24, 10.90.90.0/24') + assert rc is False + + # valid ip that doesn't belong to the subnet (only 14 hosts) + rc = cd.ip_in_subnets('192.168.100.20', '192.168.100.0/28') + assert rc is False + + # valid ip and valid IPV6 network + rc = cd.ip_in_subnets('fe80::5054:ff:fef4:873a', 'fe80::/64') + assert rc is True + + # valid wrapped ip and valid IPV6 network + rc = cd.ip_in_subnets('[fe80::5054:ff:fef4:873a]', 'fe80::/64') + assert rc is True + + # valid ip and that doesn't belong to IPV6 network + rc = cd.ip_in_subnets('fe80::5054:ff:fef4:873a', '2001:db8:85a3::/64') + assert rc is False + + # invalid IPv4 and valid subnets list + with pytest.raises(Exception): + rc = cd.ip_in_sublets('10.90.200.', '192.168.1.0/24, 192.168.100.0/24, 10.90.90.0/24') + + # invalid IPv6 and valid subnets list + with pytest.raises(Exception): + rc = cd.ip_in_sublets('fe80:2030:31:24', 'fe80::/64') + + @pytest.mark.parametrize("conf", [ + """[global] +public_network='1.1.1.0/24,2.2.2.0/24' +cluster_network="3.3.3.0/24, 4.4.4.0/24" +""", + """[global] +public_network=" 1.1.1.0/24,2.2.2.0/24 " +cluster_network=3.3.3.0/24, 4.4.4.0/24 +""", + """[global] + public_network= 1.1.1.0/24, 2.2.2.0/24 + cluster_network='3.3.3.0/24,4.4.4.0/24' +"""]) + @mock.patch('cephadm.list_networks') + def test_get_networks_from_conf(self, _list_networks, conf, cephadm_fs): + cephadm_fs.create_file('ceph.conf', contents=conf) + _list_networks.return_value = {'1.1.1.0/24': {'eth0': ['1.1.1.1']}, + '2.2.2.0/24': {'eth1': ['2.2.2.2']}, + '3.3.3.0/24': {'eth2': ['3.3.3.3']}, + '4.4.4.0/24': {'eth3': ['4.4.4.4']}} + ctx = cd.CephadmContext() + ctx.config = 'ceph.conf' + ctx.mon_ip = '1.1.1.1' + ctx.cluster_network = None + # what the cephadm module does with the public network string is + # [x.strip() for x in out.split(',')] + # so we must make sure our output, through that alteration, + # generates correctly formatted networks + def _str_to_networks(s): + return [x.strip() for x in s.split(',')] + public_network = cd.get_public_net_from_cfg(ctx) + assert _str_to_networks(public_network) == ['1.1.1.0/24', '2.2.2.0/24'] + cluster_network, ipv6 = cd.prepare_cluster_network(ctx) + assert not ipv6 + assert _str_to_networks(cluster_network) == ['3.3.3.0/24', '4.4.4.0/24'] + +class TestRescan(fake_filesystem_unittest.TestCase): + + def setUp(self): + self.setUpPyfakefs() + if not fake_filesystem.is_root(): + fake_filesystem.set_uid(0) + + self.fs.create_dir('/sys/class') + self.ctx = cd.CephadmContext() + self.ctx.func = cd.command_rescan_disks + + def test_no_hbas(self): + out = cd.command_rescan_disks(self.ctx) + assert out == 'Ok. No compatible HBAs found' + + def test_success(self): + self.fs.create_file('/sys/class/scsi_host/host0/scan') + self.fs.create_file('/sys/class/scsi_host/host1/scan') + out = cd.command_rescan_disks(self.ctx) + assert out.startswith('Ok. 2 adapters detected: 2 rescanned, 0 skipped, 0 failed') + + def test_skip_usb_adapter(self): + self.fs.create_file('/sys/class/scsi_host/host0/scan') + self.fs.create_file('/sys/class/scsi_host/host1/scan') + self.fs.create_file('/sys/class/scsi_host/host1/proc_name', contents='usb-storage') + out = cd.command_rescan_disks(self.ctx) + assert out.startswith('Ok. 2 adapters detected: 1 rescanned, 1 skipped, 0 failed') + + def test_skip_unknown_adapter(self): + self.fs.create_file('/sys/class/scsi_host/host0/scan') + self.fs.create_file('/sys/class/scsi_host/host1/scan') + self.fs.create_file('/sys/class/scsi_host/host1/proc_name', contents='unknown') + out = cd.command_rescan_disks(self.ctx) + assert out.startswith('Ok. 2 adapters detected: 1 rescanned, 1 skipped, 0 failed') diff --git a/src/cephadm/tests/test_networks.py b/src/cephadm/tests/test_networks.py new file mode 100644 index 000000000..6fa47ea27 --- /dev/null +++ b/src/cephadm/tests/test_networks.py @@ -0,0 +1,235 @@ +import json +from textwrap import dedent +from unittest import mock + +import pytest + +from tests.fixtures import with_cephadm_ctx, cephadm_fs + +with mock.patch('builtins.open', create=True): + from importlib.machinery import SourceFileLoader + cd = SourceFileLoader('cephadm', 'cephadm').load_module() + + +class TestCommandListNetworks: + @pytest.mark.parametrize("test_input, expected", [ + ( + dedent(""" + default via 192.168.178.1 dev enxd89ef3f34260 proto dhcp metric 100 + 10.0.0.0/8 via 10.4.0.1 dev tun0 proto static metric 50 + 10.3.0.0/21 via 10.4.0.1 dev tun0 proto static metric 50 + 10.4.0.1 dev tun0 proto kernel scope link src 10.4.0.2 metric 50 + 137.1.0.0/16 via 10.4.0.1 dev tun0 proto static metric 50 + 138.1.0.0/16 via 10.4.0.1 dev tun0 proto static metric 50 + 139.1.0.0/16 via 10.4.0.1 dev tun0 proto static metric 50 + 140.1.0.0/17 via 10.4.0.1 dev tun0 proto static metric 50 + 141.1.0.0/16 via 10.4.0.1 dev tun0 proto static metric 50 + 172.16.100.34 via 172.16.100.34 dev eth1 proto kernel scope link src 172.16.100.34 + 192.168.122.1 dev ens3 proto dhcp scope link src 192.168.122.236 metric 100 + 169.254.0.0/16 dev docker0 scope link metric 1000 + 172.17.0.0/16 dev docker0 proto kernel scope link src 172.17.0.1 + 192.168.39.0/24 dev virbr1 proto kernel scope link src 192.168.39.1 linkdown + 192.168.122.0/24 dev virbr0 proto kernel scope link src 192.168.122.1 linkdown + 192.168.178.0/24 dev enxd89ef3f34260 proto kernel scope link src 192.168.178.28 metric 100 + 192.168.178.1 dev enxd89ef3f34260 proto static scope link metric 100 + 195.135.221.12 via 192.168.178.1 dev enxd89ef3f34260 proto static metric 100 + """), + { + '172.16.100.34/32': {'eth1': {'172.16.100.34'}}, + '192.168.122.1/32': {'ens3': {'192.168.122.236'}}, + '10.4.0.1/32': {'tun0': {'10.4.0.2'}}, + '172.17.0.0/16': {'docker0': {'172.17.0.1'}}, + '192.168.39.0/24': {'virbr1': {'192.168.39.1'}}, + '192.168.122.0/24': {'virbr0': {'192.168.122.1'}}, + '192.168.178.0/24': {'enxd89ef3f34260': {'192.168.178.28'}} + } + ), ( + dedent(""" + default via 10.3.64.1 dev eno1 proto static metric 100 + 10.3.64.0/24 dev eno1 proto kernel scope link src 10.3.64.23 metric 100 + 10.3.64.0/24 dev eno1 proto kernel scope link src 10.3.64.27 metric 100 + 10.88.0.0/16 dev cni-podman0 proto kernel scope link src 10.88.0.1 linkdown + 172.21.0.0/20 via 172.21.3.189 dev tun0 + 172.21.1.0/20 via 172.21.3.189 dev tun0 + 172.21.2.1 via 172.21.3.189 dev tun0 + 172.21.3.1 dev tun0 proto kernel scope link src 172.21.3.2 + 172.21.4.0/24 via 172.21.3.1 dev tun0 + 172.21.5.0/24 via 172.21.3.1 dev tun0 + 172.21.6.0/24 via 172.21.3.1 dev tun0 + 172.21.7.0/24 via 172.21.3.1 dev tun0 + 192.168.122.0/24 dev virbr0 proto kernel scope link src 192.168.122.1 linkdown + 192.168.122.0/24 dev virbr0 proto kernel scope link src 192.168.122.1 linkdown + 192.168.122.0/24 dev virbr0 proto kernel scope link src 192.168.122.1 linkdown + 192.168.122.0/24 dev virbr0 proto kernel scope link src 192.168.122.1 linkdown + """), + { + '10.3.64.0/24': {'eno1': {'10.3.64.23', '10.3.64.27'}}, + '10.88.0.0/16': {'cni-podman0': {'10.88.0.1'}}, + '172.21.3.1/32': {'tun0': {'172.21.3.2'}}, + '192.168.122.0/24': {'virbr0': {'192.168.122.1'}} + } + ), + ]) + def test_parse_ipv4_route(self, test_input, expected): + assert cd._parse_ipv4_route(test_input) == expected + + @pytest.mark.parametrize("test_routes, test_ips, expected", [ + ( + dedent(""" + ::1 dev lo proto kernel metric 256 pref medium + fe80::/64 dev eno1 proto kernel metric 100 pref medium + fe80::/64 dev br-3d443496454c proto kernel metric 256 linkdown pref medium + fe80::/64 dev tun0 proto kernel metric 256 pref medium + fe80::/64 dev br-4355f5dbb528 proto kernel metric 256 pref medium + fe80::/64 dev docker0 proto kernel metric 256 linkdown pref medium + fe80::/64 dev cni-podman0 proto kernel metric 256 linkdown pref medium + fe80::/64 dev veth88ba1e8 proto kernel metric 256 pref medium + fe80::/64 dev vethb6e5fc7 proto kernel metric 256 pref medium + fe80::/64 dev vethaddb245 proto kernel metric 256 pref medium + fe80::/64 dev vethbd14d6b proto kernel metric 256 pref medium + fe80::/64 dev veth13e8fd2 proto kernel metric 256 pref medium + fe80::/64 dev veth1d3aa9e proto kernel metric 256 pref medium + fe80::/64 dev vethe485ca9 proto kernel metric 256 pref medium + """), + dedent(""" + 1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 state UNKNOWN qlen 1000 + inet6 ::1/128 scope host + valid_lft forever preferred_lft forever + 2: eno1: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 state UP qlen 1000 + inet6 fe80::225:90ff:fee5:26e8/64 scope link noprefixroute + valid_lft forever preferred_lft forever + 6: br-3d443496454c: <NO-CARRIER,BROADCAST,MULTICAST,UP> mtu 1500 state DOWN + inet6 fe80::42:23ff:fe9d:ee4/64 scope link + valid_lft forever preferred_lft forever + 7: br-4355f5dbb528: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 state UP + inet6 fe80::42:6eff:fe35:41fe/64 scope link + valid_lft forever preferred_lft forever + 8: docker0: <NO-CARRIER,BROADCAST,MULTICAST,UP> mtu 1500 state DOWN + inet6 fe80::42:faff:fee6:40a0/64 scope link + valid_lft forever preferred_lft forever + 11: tun0: <POINTOPOINT,MULTICAST,NOARP,UP,LOWER_UP> mtu 1500 state UNKNOWN qlen 100 + inet6 fe80::98a6:733e:dafd:350/64 scope link stable-privacy + valid_lft forever preferred_lft forever + 28: cni-podman0: <NO-CARRIER,BROADCAST,MULTICAST,UP> mtu 1500 state DOWN qlen 1000 + inet6 fe80::3449:cbff:fe89:b87e/64 scope link + valid_lft forever preferred_lft forever + 31: vethaddb245@if30: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 state UP + inet6 fe80::90f7:3eff:feed:a6bb/64 scope link + valid_lft forever preferred_lft forever + 33: veth88ba1e8@if32: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 state UP + inet6 fe80::d:f5ff:fe73:8c82/64 scope link + valid_lft forever preferred_lft forever + 35: vethbd14d6b@if34: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 state UP + inet6 fe80::b44f:8ff:fe6f:813d/64 scope link + valid_lft forever preferred_lft forever + 37: vethb6e5fc7@if36: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 state UP + inet6 fe80::4869:c6ff:feaa:8afe/64 scope link + valid_lft forever preferred_lft forever + 39: veth13e8fd2@if38: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 state UP + inet6 fe80::78f4:71ff:fefe:eb40/64 scope link + valid_lft forever preferred_lft forever + 41: veth1d3aa9e@if40: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 state UP + inet6 fe80::24bd:88ff:fe28:5b18/64 scope link + valid_lft forever preferred_lft forever + 43: vethe485ca9@if42: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 state UP + inet6 fe80::6425:87ff:fe42:b9f0/64 scope link + valid_lft forever preferred_lft forever + """), + { + "fe80::/64": { + "eno1": {"fe80::225:90ff:fee5:26e8"}, + "br-3d443496454c": {"fe80::42:23ff:fe9d:ee4"}, + "tun0": {"fe80::98a6:733e:dafd:350"}, + "br-4355f5dbb528": {"fe80::42:6eff:fe35:41fe"}, + "docker0": {"fe80::42:faff:fee6:40a0"}, + "cni-podman0": {"fe80::3449:cbff:fe89:b87e"}, + "veth88ba1e8": {"fe80::d:f5ff:fe73:8c82"}, + "vethb6e5fc7": {"fe80::4869:c6ff:feaa:8afe"}, + "vethaddb245": {"fe80::90f7:3eff:feed:a6bb"}, + "vethbd14d6b": {"fe80::b44f:8ff:fe6f:813d"}, + "veth13e8fd2": {"fe80::78f4:71ff:fefe:eb40"}, + "veth1d3aa9e": {"fe80::24bd:88ff:fe28:5b18"}, + "vethe485ca9": {"fe80::6425:87ff:fe42:b9f0"}, + } + } + ), + ( + dedent(""" + ::1 dev lo proto kernel metric 256 pref medium + 2001:1458:301:eb::100:1a dev ens20f0 proto kernel metric 100 pref medium + 2001:1458:301:eb::/64 dev ens20f0 proto ra metric 100 pref medium + fd01:1458:304:5e::/64 dev ens20f0 proto ra metric 100 pref medium + fe80::/64 dev ens20f0 proto kernel metric 100 pref medium + default proto ra metric 100 + nexthop via fe80::46ec:ce00:b8a0:d3c8 dev ens20f0 weight 1 + nexthop via fe80::46ec:ce00:b8a2:33c8 dev ens20f0 weight 1 pref medium + """), + dedent(""" + 1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 state UNKNOWN qlen 1000 + inet6 ::1/128 scope host + valid_lft forever preferred_lft forever + 2: ens20f0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 state UP qlen 1000 + inet6 2001:1458:301:eb::100:1a/128 scope global dynamic noprefixroute + valid_lft 590879sec preferred_lft 590879sec + inet6 fe80::2e60:cff:fef8:da41/64 scope link noprefixroute + valid_lft forever preferred_lft forever + inet6 fe80::2e60:cff:fef8:da41/64 scope link noprefixroute + valid_lft forever preferred_lft forever + inet6 fe80::2e60:cff:fef8:da41/64 scope link noprefixroute + valid_lft forever preferred_lft forever + """), + { + '2001:1458:301:eb::100:1a/128': { + 'ens20f0': { + '2001:1458:301:eb::100:1a' + }, + }, + '2001:1458:301:eb::/64': { + 'ens20f0': set(), + }, + 'fe80::/64': { + 'ens20f0': {'fe80::2e60:cff:fef8:da41'}, + }, + 'fd01:1458:304:5e::/64': { + 'ens20f0': set() + }, + } + ), + ( + dedent(""" + ::1 dev lo proto kernel metric 256 pref medium + fe80::/64 dev ceph-brx proto kernel metric 256 pref medium + fe80::/64 dev brx.0 proto kernel metric 256 pref medium + default via fe80::327c:5e00:6487:71e0 dev enp3s0f1 proto ra metric 1024 expires 1790sec hoplimit 64 pref medium """), + dedent(""" + 1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 state UNKNOWN qlen 1000 + inet6 ::1/128 scope host + valid_lft forever preferred_lft forever + 5: enp3s0f1: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 state UP qlen 1000 + inet6 fe80::ec4:7aff:fe8f:cb83/64 scope link noprefixroute + valid_lft forever preferred_lft forever + 6: ceph-brx: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 state UP qlen 1000 + inet6 fe80::d8a1:69ff:fede:8f58/64 scope link + valid_lft forever preferred_lft forever + 7: brx.0@eno1: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 state UP qlen 1000 + inet6 fe80::a4cb:54ff:fecc:f2a2/64 scope link + valid_lft forever preferred_lft forever + """), + { + 'fe80::/64': { + 'brx.0': {'fe80::a4cb:54ff:fecc:f2a2'}, + 'ceph-brx': {'fe80::d8a1:69ff:fede:8f58'} + } + } + ), + ]) + def test_parse_ipv6_route(self, test_routes, test_ips, expected): + assert cd._parse_ipv6_route(test_routes, test_ips) == expected + + @mock.patch.object(cd, 'call_throws', return_value=('10.4.0.1 dev tun0 proto kernel scope link src 10.4.0.2 metric 50\n', '', '')) + def test_command_list_networks(self, cephadm_fs, capsys): + with with_cephadm_ctx([]) as ctx: + cd.command_list_networks(ctx) + assert json.loads(capsys.readouterr().out) == { + '10.4.0.1/32': {'tun0': ['10.4.0.2']} + }
\ No newline at end of file diff --git a/src/cephadm/tox.ini b/src/cephadm/tox.ini new file mode 100644 index 000000000..079605afc --- /dev/null +++ b/src/cephadm/tox.ini @@ -0,0 +1,62 @@ +[tox] +envlist = + py3 + mypy + fix + flake8 +skipsdist = true + +[flake8] +max-line-length = 100 +inline-quotes = ' +ignore = + E501, + W503, +exclude = + .tox, + .vagrant, + __pycache__, + *.pyc, + templates, + .eggs +statistics = True + +[autopep8] +addopts = + --max-line-length {[flake8]max-line-length} + --ignore "{[flake8]ignore}" + --exclude "{[flake8]exclude}" + --in-place + --recursive + --ignore-local-config + +[testenv] +skip_install=true +deps = + pyfakefs == 5.0 + mock + pytest +commands=pytest {posargs} + +[testenv:mypy] +basepython = python3 +deps = mypy==0.790 +commands = mypy --config-file ../mypy.ini {posargs:cephadm} + +[testenv:fix] +basepython = python3 +deps = + autopep8 +commands = + python --version + autopep8 {[autopep8]addopts} {posargs: cephadm} + +[testenv:flake8] +basepython = python3 +allowlist_externals = bash +deps = + flake8 == 5.0.4 + flake8-quotes +commands = + flake8 --config=tox.ini {posargs:cephadm} + bash -c "test $(grep 'docker.io' cephadm | wc -l) == 11" diff --git a/src/cephadm/vstart-cleanup.sh b/src/cephadm/vstart-cleanup.sh new file mode 100755 index 000000000..facbdd100 --- /dev/null +++ b/src/cephadm/vstart-cleanup.sh @@ -0,0 +1,6 @@ +#!/bin/sh -ex + +bin/ceph mon rm `hostname` +for f in `bin/ceph orch ls | grep -v NAME | awk '{print $1}'` ; do + bin/ceph orch rm $f --force +done diff --git a/src/cephadm/vstart-smoke.sh b/src/cephadm/vstart-smoke.sh new file mode 100755 index 000000000..ecdb59d18 --- /dev/null +++ b/src/cephadm/vstart-smoke.sh @@ -0,0 +1,86 @@ +#!/bin/bash -ex + +# this is a smoke test, meant to be run against vstart.sh. + +host="$(hostname)" + +bin/init-ceph stop || true +MON=1 OSD=1 MDS=0 MGR=1 ../src/vstart.sh -d -n -x -l --cephadm + +export CEPH_DEV=1 + +bin/ceph orch ls +bin/ceph orch apply mds foo 1 +bin/ceph orch ls | grep foo +while ! bin/ceph orch ps | grep mds.foo ; do sleep 1 ; done +bin/ceph orch ps + +bin/ceph orch host ls + +bin/ceph orch rm crash +! bin/ceph orch ls | grep crash +bin/ceph orch apply crash '*' +bin/ceph orch ls | grep crash + +while ! bin/ceph orch ps | grep crash ; do sleep 1 ; done +bin/ceph orch ps | grep crash.$host | grep running +bin/ceph orch ls | grep crash | grep 1/1 +bin/ceph orch daemon rm crash.$host +while ! bin/ceph orch ps | grep crash ; do sleep 1 ; done + +bin/ceph orch daemon stop crash.$host +bin/ceph orch daemon start crash.$host +bin/ceph orch daemon restart crash.$host +bin/ceph orch daemon reconfig crash.$host +bin/ceph orch daemon redeploy crash.$host + +bin/ceph orch host ls | grep $host +bin/ceph orch host label add $host fooxyz +bin/ceph orch host ls | grep $host | grep fooxyz +bin/ceph orch host label rm $host fooxyz +! bin/ceph orch host ls | grep $host | grep fooxyz +bin/ceph orch host set-addr $host $host + +bin/ceph cephadm check-host $host +#! bin/ceph cephadm check-host $host 1.2.3.4 +#bin/ceph orch host set-addr $host 1.2.3.4 +#! bin/ceph cephadm check-host $host +bin/ceph orch host set-addr $host $host +bin/ceph cephadm check-host $host + +bin/ceph orch apply mgr 1 +bin/ceph orch rm mgr --force # we don't want a mgr to take over for ours + +bin/ceph orch daemon add mon $host:127.0.0.1 + +while ! bin/ceph mon dump | grep 'epoch 2' ; do sleep 1 ; done + +bin/ceph orch apply rbd-mirror 1 + +bin/ceph orch apply node-exporter '*' +bin/ceph orch apply prometheus 1 +bin/ceph orch apply alertmanager 1 +bin/ceph orch apply grafana 1 + +while ! bin/ceph dashboard get-grafana-api-url | grep $host ; do sleep 1 ; done + +bin/ceph orch apply rgw foo --placement=1 + +bin/ceph orch ps +bin/ceph orch ls + +# clean up +bin/ceph orch rm mds.foo +bin/ceph orch rm rgw.myrealm.myzone +bin/ceph orch rm rbd-mirror +bin/ceph orch rm node-exporter +bin/ceph orch rm alertmanager +bin/ceph orch rm grafana +bin/ceph orch rm prometheus +bin/ceph orch rm crash + +bin/ceph mon rm $host +! bin/ceph orch daemon rm mon.$host +bin/ceph orch daemon rm mon.$host --force + +echo OK |