summaryrefslogtreecommitdiffstats
path: root/src/cephadm
diff options
context:
space:
mode:
Diffstat (limited to 'src/cephadm')
-rw-r--r--src/cephadm/CMakeLists.txt4
-rwxr-xr-xsrc/cephadm/cephadm9248
-rw-r--r--src/cephadm/samples/alertmanager.json27
-rw-r--r--src/cephadm/samples/custom_container.json35
-rw-r--r--src/cephadm/samples/grafana.json90
-rw-r--r--src/cephadm/samples/nfs.json14
-rw-r--r--src/cephadm/samples/prometheus.json17
-rw-r--r--src/cephadm/samples/rgw_ssl.json101
-rw-r--r--src/cephadm/tests/__init__.py0
-rw-r--r--src/cephadm/tests/fixtures.py126
-rw-r--r--src/cephadm/tests/test_cephadm.py2413
-rw-r--r--src/cephadm/tests/test_networks.py235
-rw-r--r--src/cephadm/tox.ini62
-rwxr-xr-xsrc/cephadm/vstart-cleanup.sh6
-rwxr-xr-xsrc/cephadm/vstart-smoke.sh86
15 files changed, 12464 insertions, 0 deletions
diff --git a/src/cephadm/CMakeLists.txt b/src/cephadm/CMakeLists.txt
new file mode 100644
index 000000000..13be09bae
--- /dev/null
+++ b/src/cephadm/CMakeLists.txt
@@ -0,0 +1,4 @@
+if(WITH_TESTS)
+ include(AddCephTest)
+ add_tox_test(cephadm TOX_ENVS py3 mypy flake8)
+endif()
diff --git a/src/cephadm/cephadm b/src/cephadm/cephadm
new file mode 100755
index 000000000..d69655946
--- /dev/null
+++ b/src/cephadm/cephadm
@@ -0,0 +1,9248 @@
+#!/usr/bin/python3
+
+import asyncio
+import asyncio.subprocess
+import argparse
+import datetime
+import fcntl
+import ipaddress
+import json
+import logging
+from logging.config import dictConfig
+import os
+import platform
+import pwd
+import random
+import shlex
+import shutil
+import socket
+import string
+import subprocess
+import sys
+import tempfile
+import time
+import errno
+import struct
+from socketserver import ThreadingMixIn
+from http.server import BaseHTTPRequestHandler, HTTPServer
+import signal
+import io
+from contextlib import redirect_stdout
+import ssl
+from enum import Enum
+
+from typing import Dict, List, Tuple, Optional, Union, Any, NoReturn, Callable, IO, Sequence, TypeVar, cast, Set, Iterable
+
+import re
+import uuid
+
+from configparser import ConfigParser
+from functools import wraps
+from glob import glob
+from io import StringIO
+from threading import Thread, RLock
+from urllib.error import HTTPError, URLError
+from urllib.request import urlopen
+from pathlib import Path
+
+FuncT = TypeVar('FuncT', bound=Callable)
+
+# Default container images -----------------------------------------------------
+DEFAULT_IMAGE = 'quay.io/ceph/ceph:v16'
+DEFAULT_IMAGE_IS_MASTER = False
+DEFAULT_IMAGE_RELEASE = 'pacific'
+DEFAULT_PROMETHEUS_IMAGE = 'quay.io/prometheus/prometheus:v2.33.4'
+DEFAULT_NODE_EXPORTER_IMAGE = 'quay.io/prometheus/node-exporter:v1.3.1'
+DEFAULT_ALERT_MANAGER_IMAGE = 'quay.io/prometheus/alertmanager:v0.23.0'
+DEFAULT_GRAFANA_IMAGE = 'quay.io/ceph/ceph-grafana:8.3.5'
+DEFAULT_HAPROXY_IMAGE = 'docker.io/library/haproxy:2.3'
+DEFAULT_KEEPALIVED_IMAGE = 'docker.io/arcts/keepalived'
+DEFAULT_SNMP_GATEWAY_IMAGE = 'docker.io/maxwo/snmp-notifier:v1.2.1'
+DEFAULT_REGISTRY = 'docker.io' # normalize unqualified digests to this
+# ------------------------------------------------------------------------------
+
+LATEST_STABLE_RELEASE = 'pacific'
+DATA_DIR = '/var/lib/ceph'
+LOG_DIR = '/var/log/ceph'
+LOCK_DIR = '/run/cephadm'
+LOGROTATE_DIR = '/etc/logrotate.d'
+SYSCTL_DIR = '/etc/sysctl.d'
+UNIT_DIR = '/etc/systemd/system'
+LOG_DIR_MODE = 0o770
+DATA_DIR_MODE = 0o700
+CONTAINER_INIT = True
+MIN_PODMAN_VERSION = (2, 0, 2)
+CGROUPS_SPLIT_PODMAN_VERSION = (2, 1, 0)
+CUSTOM_PS1 = r'[ceph: \u@\h \W]\$ '
+DEFAULT_TIMEOUT = None # in seconds
+DEFAULT_RETRY = 15
+SHELL_DEFAULT_CONF = '/etc/ceph/ceph.conf'
+SHELL_DEFAULT_KEYRING = '/etc/ceph/ceph.client.admin.keyring'
+DATEFMT = '%Y-%m-%dT%H:%M:%S.%fZ'
+
+logger: logging.Logger = None # type: ignore
+
+"""
+You can invoke cephadm in two ways:
+
+1. The normal way, at the command line.
+
+2. By piping the script to the python3 binary. In this latter case, you should
+ prepend one or more lines to the beginning of the script.
+
+ For arguments,
+
+ injected_argv = [...]
+
+ e.g.,
+
+ injected_argv = ['ls']
+
+ For reading stdin from the '--config-json -' argument,
+
+ injected_stdin = '...'
+"""
+cached_stdin = None
+
+
+##################################
+
+
+async def run_func(func: Callable, cmd: str) -> subprocess.CompletedProcess:
+ logger.debug(f'running function {func.__name__}, with parms: {cmd}')
+ response = func(cmd)
+ return response
+
+
+async def concurrent_tasks(func: Callable, cmd_list: List[str]) -> List[Any]:
+ tasks = []
+ for cmd in cmd_list:
+ tasks.append(run_func(func, cmd))
+
+ data = await asyncio.gather(*tasks)
+
+ return data
+
+
+class EndPoint:
+ """EndPoint representing an ip:port format"""
+
+ def __init__(self, ip: str, port: int) -> None:
+ self.ip = ip
+ self.port = port
+
+ def __str__(self) -> str:
+ return f'{self.ip}:{self.port}'
+
+ def __repr__(self) -> str:
+ return f'{self.ip}:{self.port}'
+
+
+class BaseConfig:
+
+ def __init__(self) -> None:
+ self.image: str = ''
+ self.docker: bool = False
+ self.data_dir: str = DATA_DIR
+ self.log_dir: str = LOG_DIR
+ self.logrotate_dir: str = LOGROTATE_DIR
+ self.sysctl_dir: str = SYSCTL_DIR
+ self.unit_dir: str = UNIT_DIR
+ self.verbose: bool = False
+ self.timeout: Optional[int] = DEFAULT_TIMEOUT
+ self.retry: int = DEFAULT_RETRY
+ self.env: List[str] = []
+ self.memory_request: Optional[int] = None
+ self.memory_limit: Optional[int] = None
+
+ self.container_init: bool = CONTAINER_INIT
+ self.container_engine: Optional[ContainerEngine] = None
+
+ def set_from_args(self, args: argparse.Namespace) -> None:
+ argdict: Dict[str, Any] = vars(args)
+ for k, v in argdict.items():
+ if hasattr(self, k):
+ setattr(self, k, v)
+
+
+class CephadmContext:
+
+ def __init__(self) -> None:
+ self.__dict__['_args'] = None
+ self.__dict__['_conf'] = BaseConfig()
+
+ def set_args(self, args: argparse.Namespace) -> None:
+ self._conf.set_from_args(args)
+ self._args = args
+
+ def has_function(self) -> bool:
+ return 'func' in self._args
+
+ def __contains__(self, name: str) -> bool:
+ return hasattr(self, name)
+
+ def __getattr__(self, name: str) -> Any:
+ if '_conf' in self.__dict__ and hasattr(self._conf, name):
+ return getattr(self._conf, name)
+ elif '_args' in self.__dict__ and hasattr(self._args, name):
+ return getattr(self._args, name)
+ else:
+ return super().__getattribute__(name)
+
+ def __setattr__(self, name: str, value: Any) -> None:
+ if hasattr(self._conf, name):
+ setattr(self._conf, name, value)
+ elif hasattr(self._args, name):
+ setattr(self._args, name, value)
+ else:
+ super().__setattr__(name, value)
+
+
+class ContainerEngine:
+ def __init__(self) -> None:
+ self.path = find_program(self.EXE)
+
+ @classmethod
+ @property
+ def EXE(cls) -> str:
+ raise NotImplementedError()
+
+ def __str__(self) -> str:
+ return f'{self.EXE} ({self.path})'
+
+
+class Podman(ContainerEngine):
+ EXE = 'podman'
+
+ def __init__(self) -> None:
+ super().__init__()
+ self._version: Optional[Tuple[int, ...]] = None
+
+ @property
+ def version(self) -> Tuple[int, ...]:
+ if self._version is None:
+ raise RuntimeError('Please call `get_version` first')
+ return self._version
+
+ def get_version(self, ctx: CephadmContext) -> None:
+ out, _, _ = call_throws(ctx, [self.path, 'version', '--format', '{{.Client.Version}}'])
+ self._version = _parse_podman_version(out)
+
+ def __str__(self) -> str:
+ version = '.'.join(map(str, self.version))
+ return f'{self.EXE} ({self.path}) version {version}'
+
+
+class Docker(ContainerEngine):
+ EXE = 'docker'
+
+
+CONTAINER_PREFERENCE = (Podman, Docker) # prefer podman to docker
+
+
+# Log and console output config
+logging_config = {
+ 'version': 1,
+ 'disable_existing_loggers': True,
+ 'formatters': {
+ 'cephadm': {
+ 'format': '%(asctime)s %(thread)x %(levelname)s %(message)s'
+ },
+ },
+ 'handlers': {
+ 'console': {
+ 'level': 'INFO',
+ 'class': 'logging.StreamHandler',
+ },
+ 'log_file': {
+ 'level': 'DEBUG',
+ 'class': 'logging.handlers.WatchedFileHandler',
+ 'formatter': 'cephadm',
+ 'filename': '%s/cephadm.log' % LOG_DIR,
+ }
+ },
+ 'loggers': {
+ '': {
+ 'level': 'DEBUG',
+ 'handlers': ['console', 'log_file'],
+ }
+ }
+}
+
+
+class termcolor:
+ yellow = '\033[93m'
+ red = '\033[31m'
+ end = '\033[0m'
+
+
+class Error(Exception):
+ pass
+
+
+class TimeoutExpired(Error):
+ pass
+
+
+class UnauthorizedRegistryError(Error):
+ pass
+
+##################################
+
+
+class Ceph(object):
+ daemons = ('mon', 'mgr', 'mds', 'osd', 'rgw', 'rbd-mirror',
+ 'crash', 'cephfs-mirror')
+
+##################################
+
+
+class OSD(object):
+ @staticmethod
+ def get_sysctl_settings() -> List[str]:
+ return [
+ '# allow a large number of OSDs',
+ 'fs.aio-max-nr = 1048576',
+ 'kernel.pid_max = 4194304',
+ ]
+
+
+##################################
+
+
+class SNMPGateway:
+ """Defines an SNMP gateway between Prometheus and SNMP monitoring Frameworks"""
+ daemon_type = 'snmp-gateway'
+ SUPPORTED_VERSIONS = ['V2c', 'V3']
+ default_image = DEFAULT_SNMP_GATEWAY_IMAGE
+ DEFAULT_PORT = 9464
+ env_filename = 'snmp-gateway.conf'
+
+ def __init__(self,
+ ctx: CephadmContext,
+ fsid: str,
+ daemon_id: Union[int, str],
+ config_json: Dict[str, Any],
+ image: Optional[str] = None) -> None:
+ self.ctx = ctx
+ self.fsid = fsid
+ self.daemon_id = daemon_id
+ self.image = image or SNMPGateway.default_image
+
+ self.uid = config_json.get('uid', 0)
+ self.gid = config_json.get('gid', 0)
+
+ self.destination = config_json.get('destination', '')
+ self.snmp_version = config_json.get('snmp_version', 'V2c')
+ self.snmp_community = config_json.get('snmp_community', 'public')
+ self.log_level = config_json.get('log_level', 'info')
+ self.snmp_v3_auth_username = config_json.get('snmp_v3_auth_username', '')
+ self.snmp_v3_auth_password = config_json.get('snmp_v3_auth_password', '')
+ self.snmp_v3_auth_protocol = config_json.get('snmp_v3_auth_protocol', '')
+ self.snmp_v3_priv_protocol = config_json.get('snmp_v3_priv_protocol', '')
+ self.snmp_v3_priv_password = config_json.get('snmp_v3_priv_password', '')
+ self.snmp_v3_engine_id = config_json.get('snmp_v3_engine_id', '')
+
+ self.validate()
+
+ @classmethod
+ def init(cls, ctx: CephadmContext, fsid: str,
+ daemon_id: Union[int, str]) -> 'SNMPGateway':
+ assert ctx.config_json
+ return cls(ctx, fsid, daemon_id,
+ get_parm(ctx.config_json), ctx.image)
+
+ @staticmethod
+ def get_version(ctx: CephadmContext, fsid: str, daemon_id: str) -> Optional[str]:
+ """Return the version of the notifer from it's http endpoint"""
+ path = os.path.join(ctx.data_dir, fsid, f'snmp-gateway.{daemon_id}', 'unit.meta')
+ try:
+ with open(path, 'r') as env:
+ metadata = json.loads(env.read())
+ except (OSError, json.JSONDecodeError):
+ return None
+
+ ports = metadata.get('ports', [])
+ if not ports:
+ return None
+
+ try:
+ with urlopen(f'http://127.0.0.1:{ports[0]}/') as r:
+ html = r.read().decode('utf-8').split('\n')
+ except (HTTPError, URLError):
+ return None
+
+ for h in html:
+ stripped = h.strip()
+ if stripped.startswith(('<pre>', '<PRE>')) and \
+ stripped.endswith(('</pre>', '</PRE>')):
+ # <pre>(version=1.2.1, branch=HEAD, revision=7...
+ return stripped.split(',')[0].split('version=')[1]
+
+ return None
+
+ @property
+ def port(self) -> int:
+ if not self.ctx.tcp_ports:
+ return self.DEFAULT_PORT
+ else:
+ if len(self.ctx.tcp_ports) > 0:
+ return int(self.ctx.tcp_ports.split()[0])
+ else:
+ return self.DEFAULT_PORT
+
+ def get_daemon_args(self) -> List[str]:
+ v3_args = []
+ base_args = [
+ f'--web.listen-address=:{self.port}',
+ f'--snmp.destination={self.destination}',
+ f'--snmp.version={self.snmp_version}',
+ f'--log.level={self.log_level}',
+ '--snmp.trap-description-template=/etc/snmp_notifier/description-template.tpl'
+ ]
+
+ if self.snmp_version == 'V3':
+ # common auth settings
+ v3_args.extend([
+ '--snmp.authentication-enabled',
+ f'--snmp.authentication-protocol={self.snmp_v3_auth_protocol}',
+ f'--snmp.security-engine-id={self.snmp_v3_engine_id}'
+ ])
+ # authPriv setting is applied if we have a privacy protocol setting
+ if self.snmp_v3_priv_protocol:
+ v3_args.extend([
+ '--snmp.private-enabled',
+ f'--snmp.private-protocol={self.snmp_v3_priv_protocol}'
+ ])
+
+ return base_args + v3_args
+
+ @property
+ def data_dir(self) -> str:
+ return os.path.join(self.ctx.data_dir, self.ctx.fsid, f'{self.daemon_type}.{self.daemon_id}')
+
+ @property
+ def conf_file_path(self) -> str:
+ return os.path.join(self.data_dir, self.env_filename)
+
+ def create_daemon_conf(self) -> None:
+ """Creates the environment file holding 'secrets' passed to the snmp-notifier daemon"""
+ with open(os.open(self.conf_file_path, os.O_CREAT | os.O_WRONLY, 0o600), 'w') as f:
+ if self.snmp_version == 'V2c':
+ f.write(f'SNMP_NOTIFIER_COMMUNITY={self.snmp_community}\n')
+ else:
+ f.write(f'SNMP_NOTIFIER_AUTH_USERNAME={self.snmp_v3_auth_username}\n')
+ f.write(f'SNMP_NOTIFIER_AUTH_PASSWORD={self.snmp_v3_auth_password}\n')
+ if self.snmp_v3_priv_password:
+ f.write(f'SNMP_NOTIFIER_PRIV_PASSWORD={self.snmp_v3_priv_password}\n')
+
+ def validate(self) -> None:
+ """Validate the settings
+
+ Raises:
+ Error: if the fsid doesn't look like an fsid
+ Error: if the snmp version is not supported
+ Error: destination IP and port address missing
+ """
+ if not is_fsid(self.fsid):
+ raise Error(f'not a valid fsid: {self.fsid}')
+
+ if self.snmp_version not in SNMPGateway.SUPPORTED_VERSIONS:
+ raise Error(f'not a valid snmp version: {self.snmp_version}')
+
+ if not self.destination:
+ raise Error('config is missing destination attribute(<ip>:<port>) of the target SNMP listener')
+
+
+##################################
+class Monitoring(object):
+ """Define the configs for the monitoring containers"""
+
+ port_map = {
+ 'prometheus': [9095], # Avoid default 9090, due to conflict with cockpit UI
+ 'node-exporter': [9100],
+ 'grafana': [3000],
+ 'alertmanager': [9093, 9094],
+ }
+
+ components = {
+ 'prometheus': {
+ 'image': DEFAULT_PROMETHEUS_IMAGE,
+ 'cpus': '2',
+ 'memory': '4GB',
+ 'args': [
+ '--config.file=/etc/prometheus/prometheus.yml',
+ '--storage.tsdb.path=/prometheus',
+ ],
+ 'config-json-files': [
+ 'prometheus.yml',
+ ],
+ },
+ 'node-exporter': {
+ 'image': DEFAULT_NODE_EXPORTER_IMAGE,
+ 'cpus': '1',
+ 'memory': '1GB',
+ 'args': [
+ '--no-collector.timex',
+ ],
+ },
+ 'grafana': {
+ 'image': DEFAULT_GRAFANA_IMAGE,
+ 'cpus': '2',
+ 'memory': '4GB',
+ 'args': [],
+ 'config-json-files': [
+ 'grafana.ini',
+ 'provisioning/datasources/ceph-dashboard.yml',
+ 'certs/cert_file',
+ 'certs/cert_key',
+ ],
+ },
+ 'alertmanager': {
+ 'image': DEFAULT_ALERT_MANAGER_IMAGE,
+ 'cpus': '2',
+ 'memory': '2GB',
+ 'args': [
+ '--cluster.listen-address=:{}'.format(port_map['alertmanager'][1]),
+ ],
+ 'config-json-files': [
+ 'alertmanager.yml',
+ ],
+ 'config-json-args': [
+ 'peers',
+ ],
+ },
+ } # type: ignore
+
+ @staticmethod
+ def get_version(ctx, container_id, daemon_type):
+ # type: (CephadmContext, str, str) -> str
+ """
+ :param: daemon_type Either "prometheus", "alertmanager" or "node-exporter"
+ """
+ assert daemon_type in ('prometheus', 'alertmanager', 'node-exporter')
+ cmd = daemon_type.replace('-', '_')
+ code = -1
+ err = ''
+ out = ''
+ version = ''
+ if daemon_type == 'alertmanager':
+ for cmd in ['alertmanager', 'prometheus-alertmanager']:
+ out, err, code = call(ctx, [
+ ctx.container_engine.path, 'exec', container_id, cmd,
+ '--version'
+ ], verbosity=CallVerbosity.DEBUG)
+ if code == 0:
+ break
+ cmd = 'alertmanager' # reset cmd for version extraction
+ else:
+ out, err, code = call(ctx, [
+ ctx.container_engine.path, 'exec', container_id, cmd, '--version'
+ ], verbosity=CallVerbosity.DEBUG)
+ if code == 0:
+ if err.startswith('%s, version ' % cmd):
+ version = err.split(' ')[2]
+ elif out.startswith('%s, version ' % cmd):
+ version = out.split(' ')[2]
+ return version
+
+##################################
+
+
+def populate_files(config_dir, config_files, uid, gid):
+ # type: (str, Dict, int, int) -> None
+ """create config files for different services"""
+ for fname in config_files:
+ config_file = os.path.join(config_dir, fname)
+ config_content = dict_get_join(config_files, fname)
+ logger.info('Write file: %s' % (config_file))
+ with open(config_file, 'w', encoding='utf-8') as f:
+ os.fchown(f.fileno(), uid, gid)
+ os.fchmod(f.fileno(), 0o600)
+ f.write(config_content)
+
+
+class NFSGanesha(object):
+ """Defines a NFS-Ganesha container"""
+
+ daemon_type = 'nfs'
+ entrypoint = '/usr/bin/ganesha.nfsd'
+ daemon_args = ['-F', '-L', 'STDERR']
+
+ required_files = ['ganesha.conf']
+
+ port_map = {
+ 'nfs': 2049,
+ }
+
+ def __init__(self,
+ ctx,
+ fsid,
+ daemon_id,
+ config_json,
+ image=DEFAULT_IMAGE):
+ # type: (CephadmContext, str, Union[int, str], Dict, str) -> None
+ self.ctx = ctx
+ self.fsid = fsid
+ self.daemon_id = daemon_id
+ self.image = image
+
+ # config-json options
+ self.pool = dict_get(config_json, 'pool', require=True)
+ self.namespace = dict_get(config_json, 'namespace')
+ self.userid = dict_get(config_json, 'userid')
+ self.extra_args = dict_get(config_json, 'extra_args', [])
+ self.files = dict_get(config_json, 'files', {})
+ self.rgw = dict_get(config_json, 'rgw', {})
+
+ # validate the supplied args
+ self.validate()
+
+ @classmethod
+ def init(cls, ctx, fsid, daemon_id):
+ # type: (CephadmContext, str, Union[int, str]) -> NFSGanesha
+ return cls(ctx, fsid, daemon_id, get_parm(ctx.config_json), ctx.image)
+
+ def get_container_mounts(self, data_dir):
+ # type: (str) -> Dict[str, str]
+ mounts = dict()
+ mounts[os.path.join(data_dir, 'config')] = '/etc/ceph/ceph.conf:z'
+ mounts[os.path.join(data_dir, 'keyring')] = '/etc/ceph/keyring:z'
+ mounts[os.path.join(data_dir, 'etc/ganesha')] = '/etc/ganesha:z'
+ if self.rgw:
+ cluster = self.rgw.get('cluster', 'ceph')
+ rgw_user = self.rgw.get('user', 'admin')
+ mounts[os.path.join(data_dir, 'keyring.rgw')] = \
+ '/var/lib/ceph/radosgw/%s-%s/keyring:z' % (cluster, rgw_user)
+ return mounts
+
+ @staticmethod
+ def get_container_envs():
+ # type: () -> List[str]
+ envs = [
+ 'CEPH_CONF=%s' % ('/etc/ceph/ceph.conf')
+ ]
+ return envs
+
+ @staticmethod
+ def get_version(ctx, container_id):
+ # type: (CephadmContext, str) -> Optional[str]
+ version = None
+ out, err, code = call(ctx,
+ [ctx.container_engine.path, 'exec', container_id,
+ NFSGanesha.entrypoint, '-v'],
+ verbosity=CallVerbosity.DEBUG)
+ if code == 0:
+ match = re.search(r'NFS-Ganesha Release\s*=\s*[V]*([\d.]+)', out)
+ if match:
+ version = match.group(1)
+ return version
+
+ def validate(self):
+ # type: () -> None
+ if not is_fsid(self.fsid):
+ raise Error('not an fsid: %s' % self.fsid)
+ if not self.daemon_id:
+ raise Error('invalid daemon_id: %s' % self.daemon_id)
+ if not self.image:
+ raise Error('invalid image: %s' % self.image)
+
+ # check for the required files
+ if self.required_files:
+ for fname in self.required_files:
+ if fname not in self.files:
+ raise Error('required file missing from config-json: %s' % fname)
+
+ # check for an RGW config
+ if self.rgw:
+ if not self.rgw.get('keyring'):
+ raise Error('RGW keyring is missing')
+ if not self.rgw.get('user'):
+ raise Error('RGW user is missing')
+
+ def get_daemon_name(self):
+ # type: () -> str
+ return '%s.%s' % (self.daemon_type, self.daemon_id)
+
+ def get_container_name(self, desc=None):
+ # type: (Optional[str]) -> str
+ cname = 'ceph-%s-%s' % (self.fsid, self.get_daemon_name())
+ if desc:
+ cname = '%s-%s' % (cname, desc)
+ return cname
+
+ def get_daemon_args(self):
+ # type: () -> List[str]
+ return self.daemon_args + self.extra_args
+
+ def create_daemon_dirs(self, data_dir, uid, gid):
+ # type: (str, int, int) -> None
+ """Create files under the container data dir"""
+ if not os.path.isdir(data_dir):
+ raise OSError('data_dir is not a directory: %s' % (data_dir))
+
+ logger.info('Creating ganesha config...')
+
+ # create the ganesha conf dir
+ config_dir = os.path.join(data_dir, 'etc/ganesha')
+ makedirs(config_dir, uid, gid, 0o755)
+
+ # populate files from the config-json
+ populate_files(config_dir, self.files, uid, gid)
+
+ # write the RGW keyring
+ if self.rgw:
+ keyring_path = os.path.join(data_dir, 'keyring.rgw')
+ with open(keyring_path, 'w') as f:
+ os.fchmod(f.fileno(), 0o600)
+ os.fchown(f.fileno(), uid, gid)
+ f.write(self.rgw.get('keyring', ''))
+
+##################################
+
+
+class CephIscsi(object):
+ """Defines a Ceph-Iscsi container"""
+
+ daemon_type = 'iscsi'
+ entrypoint = '/usr/bin/rbd-target-api'
+
+ required_files = ['iscsi-gateway.cfg']
+
+ def __init__(self,
+ ctx,
+ fsid,
+ daemon_id,
+ config_json,
+ image=DEFAULT_IMAGE):
+ # type: (CephadmContext, str, Union[int, str], Dict, str) -> None
+ self.ctx = ctx
+ self.fsid = fsid
+ self.daemon_id = daemon_id
+ self.image = image
+
+ # config-json options
+ self.files = dict_get(config_json, 'files', {})
+
+ # validate the supplied args
+ self.validate()
+
+ @classmethod
+ def init(cls, ctx, fsid, daemon_id):
+ # type: (CephadmContext, str, Union[int, str]) -> CephIscsi
+ return cls(ctx, fsid, daemon_id,
+ get_parm(ctx.config_json), ctx.image)
+
+ @staticmethod
+ def get_container_mounts(data_dir, log_dir):
+ # type: (str, str) -> Dict[str, str]
+ mounts = dict()
+ mounts[os.path.join(data_dir, 'config')] = '/etc/ceph/ceph.conf:z'
+ mounts[os.path.join(data_dir, 'keyring')] = '/etc/ceph/keyring:z'
+ mounts[os.path.join(data_dir, 'iscsi-gateway.cfg')] = '/etc/ceph/iscsi-gateway.cfg:z'
+ mounts[os.path.join(data_dir, 'configfs')] = '/sys/kernel/config'
+ mounts[log_dir] = '/var/log:z'
+ mounts['/dev'] = '/dev'
+ return mounts
+
+ @staticmethod
+ def get_container_binds():
+ # type: () -> List[List[str]]
+ binds = []
+ lib_modules = ['type=bind',
+ 'source=/lib/modules',
+ 'destination=/lib/modules',
+ 'ro=true']
+ binds.append(lib_modules)
+ return binds
+
+ @staticmethod
+ def get_version(ctx, container_id):
+ # type: (CephadmContext, str) -> Optional[str]
+ version = None
+ out, err, code = call(ctx,
+ [ctx.container_engine.path, 'exec', container_id,
+ '/usr/bin/python3', '-c', "import pkg_resources; print(pkg_resources.require('ceph_iscsi')[0].version)"],
+ verbosity=CallVerbosity.DEBUG)
+ if code == 0:
+ version = out.strip()
+ return version
+
+ def validate(self):
+ # type: () -> None
+ if not is_fsid(self.fsid):
+ raise Error('not an fsid: %s' % self.fsid)
+ if not self.daemon_id:
+ raise Error('invalid daemon_id: %s' % self.daemon_id)
+ if not self.image:
+ raise Error('invalid image: %s' % self.image)
+
+ # check for the required files
+ if self.required_files:
+ for fname in self.required_files:
+ if fname not in self.files:
+ raise Error('required file missing from config-json: %s' % fname)
+
+ def get_daemon_name(self):
+ # type: () -> str
+ return '%s.%s' % (self.daemon_type, self.daemon_id)
+
+ def get_container_name(self, desc=None):
+ # type: (Optional[str]) -> str
+ cname = 'ceph-%s-%s' % (self.fsid, self.get_daemon_name())
+ if desc:
+ cname = '%s-%s' % (cname, desc)
+ return cname
+
+ def create_daemon_dirs(self, data_dir, uid, gid):
+ # type: (str, int, int) -> None
+ """Create files under the container data dir"""
+ if not os.path.isdir(data_dir):
+ raise OSError('data_dir is not a directory: %s' % (data_dir))
+
+ logger.info('Creating ceph-iscsi config...')
+ configfs_dir = os.path.join(data_dir, 'configfs')
+ makedirs(configfs_dir, uid, gid, 0o755)
+
+ # populate files from the config-json
+ populate_files(data_dir, self.files, uid, gid)
+
+ @staticmethod
+ def configfs_mount_umount(data_dir, mount=True):
+ # type: (str, bool) -> List[str]
+ mount_path = os.path.join(data_dir, 'configfs')
+ if mount:
+ cmd = 'if ! grep -qs {0} /proc/mounts; then ' \
+ 'mount -t configfs none {0}; fi'.format(mount_path)
+ else:
+ cmd = 'if grep -qs {0} /proc/mounts; then ' \
+ 'umount {0}; fi'.format(mount_path)
+ return cmd.split()
+
+ def get_tcmu_runner_container(self):
+ # type: () -> CephContainer
+ tcmu_container = get_container(self.ctx, self.fsid, self.daemon_type, self.daemon_id)
+ tcmu_container.entrypoint = '/usr/bin/tcmu-runner'
+ tcmu_container.cname = self.get_container_name(desc='tcmu')
+ # remove extra container args for tcmu container.
+ # extra args could cause issue with forking service type
+ tcmu_container.container_args = []
+ return tcmu_container
+
+##################################
+
+
+class HAproxy(object):
+ """Defines an HAproxy container"""
+ daemon_type = 'haproxy'
+ required_files = ['haproxy.cfg']
+ default_image = DEFAULT_HAPROXY_IMAGE
+
+ def __init__(self,
+ ctx: CephadmContext,
+ fsid: str, daemon_id: Union[int, str],
+ config_json: Dict, image: str) -> None:
+ self.ctx = ctx
+ self.fsid = fsid
+ self.daemon_id = daemon_id
+ self.image = image
+
+ # config-json options
+ self.files = dict_get(config_json, 'files', {})
+
+ self.validate()
+
+ @classmethod
+ def init(cls, ctx: CephadmContext,
+ fsid: str, daemon_id: Union[int, str]) -> 'HAproxy':
+ return cls(ctx, fsid, daemon_id, get_parm(ctx.config_json),
+ ctx.image)
+
+ def create_daemon_dirs(self, data_dir: str, uid: int, gid: int) -> None:
+ """Create files under the container data dir"""
+ if not os.path.isdir(data_dir):
+ raise OSError('data_dir is not a directory: %s' % (data_dir))
+
+ # create additional directories in data dir for HAproxy to use
+ if not os.path.isdir(os.path.join(data_dir, 'haproxy')):
+ makedirs(os.path.join(data_dir, 'haproxy'), uid, gid, DATA_DIR_MODE)
+
+ data_dir = os.path.join(data_dir, 'haproxy')
+ populate_files(data_dir, self.files, uid, gid)
+
+ def get_daemon_args(self) -> List[str]:
+ return ['haproxy', '-f', '/var/lib/haproxy/haproxy.cfg']
+
+ def validate(self):
+ # type: () -> None
+ if not is_fsid(self.fsid):
+ raise Error('not an fsid: %s' % self.fsid)
+ if not self.daemon_id:
+ raise Error('invalid daemon_id: %s' % self.daemon_id)
+ if not self.image:
+ raise Error('invalid image: %s' % self.image)
+
+ # check for the required files
+ if self.required_files:
+ for fname in self.required_files:
+ if fname not in self.files:
+ raise Error('required file missing from config-json: %s' % fname)
+
+ def get_daemon_name(self):
+ # type: () -> str
+ return '%s.%s' % (self.daemon_type, self.daemon_id)
+
+ def get_container_name(self, desc=None):
+ # type: (Optional[str]) -> str
+ cname = 'ceph-%s-%s' % (self.fsid, self.get_daemon_name())
+ if desc:
+ cname = '%s-%s' % (cname, desc)
+ return cname
+
+ def extract_uid_gid_haproxy(self) -> Tuple[int, int]:
+ # better directory for this?
+ return extract_uid_gid(self.ctx, file_path='/var/lib')
+
+ @staticmethod
+ def get_container_mounts(data_dir: str) -> Dict[str, str]:
+ mounts = dict()
+ mounts[os.path.join(data_dir, 'haproxy')] = '/var/lib/haproxy'
+ return mounts
+
+ @staticmethod
+ def get_sysctl_settings() -> List[str]:
+ return [
+ '# IP forwarding and non-local bind',
+ 'net.ipv4.ip_forward = 1',
+ 'net.ipv4.ip_nonlocal_bind = 1',
+ ]
+
+##################################
+
+
+class Keepalived(object):
+ """Defines an Keepalived container"""
+ daemon_type = 'keepalived'
+ required_files = ['keepalived.conf']
+ default_image = DEFAULT_KEEPALIVED_IMAGE
+
+ def __init__(self,
+ ctx: CephadmContext,
+ fsid: str, daemon_id: Union[int, str],
+ config_json: Dict, image: str) -> None:
+ self.ctx = ctx
+ self.fsid = fsid
+ self.daemon_id = daemon_id
+ self.image = image
+
+ # config-json options
+ self.files = dict_get(config_json, 'files', {})
+
+ self.validate()
+
+ @classmethod
+ def init(cls, ctx: CephadmContext, fsid: str,
+ daemon_id: Union[int, str]) -> 'Keepalived':
+ return cls(ctx, fsid, daemon_id,
+ get_parm(ctx.config_json), ctx.image)
+
+ def create_daemon_dirs(self, data_dir: str, uid: int, gid: int) -> None:
+ """Create files under the container data dir"""
+ if not os.path.isdir(data_dir):
+ raise OSError('data_dir is not a directory: %s' % (data_dir))
+
+ # create additional directories in data dir for keepalived to use
+ if not os.path.isdir(os.path.join(data_dir, 'keepalived')):
+ makedirs(os.path.join(data_dir, 'keepalived'), uid, gid, DATA_DIR_MODE)
+
+ # populate files from the config-json
+ populate_files(data_dir, self.files, uid, gid)
+
+ def validate(self):
+ # type: () -> None
+ if not is_fsid(self.fsid):
+ raise Error('not an fsid: %s' % self.fsid)
+ if not self.daemon_id:
+ raise Error('invalid daemon_id: %s' % self.daemon_id)
+ if not self.image:
+ raise Error('invalid image: %s' % self.image)
+
+ # check for the required files
+ if self.required_files:
+ for fname in self.required_files:
+ if fname not in self.files:
+ raise Error('required file missing from config-json: %s' % fname)
+
+ def get_daemon_name(self):
+ # type: () -> str
+ return '%s.%s' % (self.daemon_type, self.daemon_id)
+
+ def get_container_name(self, desc=None):
+ # type: (Optional[str]) -> str
+ cname = 'ceph-%s-%s' % (self.fsid, self.get_daemon_name())
+ if desc:
+ cname = '%s-%s' % (cname, desc)
+ return cname
+
+ @staticmethod
+ def get_container_envs():
+ # type: () -> List[str]
+ envs = [
+ 'KEEPALIVED_AUTOCONF=false',
+ 'KEEPALIVED_CONF=/etc/keepalived/keepalived.conf',
+ 'KEEPALIVED_CMD=/usr/sbin/keepalived -n -l -f /etc/keepalived/keepalived.conf',
+ 'KEEPALIVED_DEBUG=false'
+ ]
+ return envs
+
+ @staticmethod
+ def get_sysctl_settings() -> List[str]:
+ return [
+ '# IP forwarding and non-local bind',
+ 'net.ipv4.ip_forward = 1',
+ 'net.ipv4.ip_nonlocal_bind = 1',
+ ]
+
+ def extract_uid_gid_keepalived(self) -> Tuple[int, int]:
+ # better directory for this?
+ return extract_uid_gid(self.ctx, file_path='/var/lib')
+
+ @staticmethod
+ def get_container_mounts(data_dir: str) -> Dict[str, str]:
+ mounts = dict()
+ mounts[os.path.join(data_dir, 'keepalived.conf')] = '/etc/keepalived/keepalived.conf'
+ return mounts
+
+##################################
+
+
+class CustomContainer(object):
+ """Defines a custom container"""
+ daemon_type = 'container'
+
+ def __init__(self,
+ fsid: str, daemon_id: Union[int, str],
+ config_json: Dict, image: str) -> None:
+ self.fsid = fsid
+ self.daemon_id = daemon_id
+ self.image = image
+
+ # config-json options
+ self.entrypoint = dict_get(config_json, 'entrypoint')
+ self.uid = dict_get(config_json, 'uid', 65534) # nobody
+ self.gid = dict_get(config_json, 'gid', 65534) # nobody
+ self.volume_mounts = dict_get(config_json, 'volume_mounts', {})
+ self.args = dict_get(config_json, 'args', [])
+ self.envs = dict_get(config_json, 'envs', [])
+ self.privileged = dict_get(config_json, 'privileged', False)
+ self.bind_mounts = dict_get(config_json, 'bind_mounts', [])
+ self.ports = dict_get(config_json, 'ports', [])
+ self.dirs = dict_get(config_json, 'dirs', [])
+ self.files = dict_get(config_json, 'files', {})
+
+ @classmethod
+ def init(cls, ctx: CephadmContext,
+ fsid: str, daemon_id: Union[int, str]) -> 'CustomContainer':
+ return cls(fsid, daemon_id,
+ get_parm(ctx.config_json), ctx.image)
+
+ def create_daemon_dirs(self, data_dir: str, uid: int, gid: int) -> None:
+ """
+ Create dirs/files below the container data directory.
+ """
+ logger.info('Creating custom container configuration '
+ 'dirs/files in {} ...'.format(data_dir))
+
+ if not os.path.isdir(data_dir):
+ raise OSError('data_dir is not a directory: %s' % data_dir)
+
+ for dir_path in self.dirs:
+ logger.info('Creating directory: {}'.format(dir_path))
+ dir_path = os.path.join(data_dir, dir_path.strip('/'))
+ makedirs(dir_path, uid, gid, 0o755)
+
+ for file_path in self.files:
+ logger.info('Creating file: {}'.format(file_path))
+ content = dict_get_join(self.files, file_path)
+ file_path = os.path.join(data_dir, file_path.strip('/'))
+ with open(file_path, 'w', encoding='utf-8') as f:
+ os.fchown(f.fileno(), uid, gid)
+ os.fchmod(f.fileno(), 0o600)
+ f.write(content)
+
+ def get_daemon_args(self) -> List[str]:
+ return []
+
+ def get_container_args(self) -> List[str]:
+ return self.args
+
+ def get_container_envs(self) -> List[str]:
+ return self.envs
+
+ def get_container_mounts(self, data_dir: str) -> Dict[str, str]:
+ """
+ Get the volume mounts. Relative source paths will be located below
+ `/var/lib/ceph/<cluster-fsid>/<daemon-name>`.
+
+ Example:
+ {
+ /foo/conf: /conf
+ foo/conf: /conf
+ }
+ becomes
+ {
+ /foo/conf: /conf
+ /var/lib/ceph/<cluster-fsid>/<daemon-name>/foo/conf: /conf
+ }
+ """
+ mounts = {}
+ for source, destination in self.volume_mounts.items():
+ source = os.path.join(data_dir, source)
+ mounts[source] = destination
+ return mounts
+
+ def get_container_binds(self, data_dir: str) -> List[List[str]]:
+ """
+ Get the bind mounts. Relative `source=...` paths will be located below
+ `/var/lib/ceph/<cluster-fsid>/<daemon-name>`.
+
+ Example:
+ [
+ 'type=bind',
+ 'source=lib/modules',
+ 'destination=/lib/modules',
+ 'ro=true'
+ ]
+ becomes
+ [
+ ...
+ 'source=/var/lib/ceph/<cluster-fsid>/<daemon-name>/lib/modules',
+ ...
+ ]
+ """
+ binds = self.bind_mounts.copy()
+ for bind in binds:
+ for index, value in enumerate(bind):
+ match = re.match(r'^source=(.+)$', value)
+ if match:
+ bind[index] = 'source={}'.format(os.path.join(
+ data_dir, match.group(1)))
+ return binds
+
+##################################
+
+
+def touch(file_path: str, uid: Optional[int] = None, gid: Optional[int] = None) -> None:
+ Path(file_path).touch()
+ if uid and gid:
+ os.chown(file_path, uid, gid)
+
+
+##################################
+
+
+def dict_get(d: Dict, key: str, default: Any = None, require: bool = False) -> Any:
+ """
+ Helper function to get a key from a dictionary.
+ :param d: The dictionary to process.
+ :param key: The name of the key to get.
+ :param default: The default value in case the key does not
+ exist. Default is `None`.
+ :param require: Set to `True` if the key is required. An
+ exception will be raised if the key does not exist in
+ the given dictionary.
+ :return: Returns the value of the given key.
+ :raises: :exc:`self.Error` if the given key does not exist
+ and `require` is set to `True`.
+ """
+ if require and key not in d.keys():
+ raise Error('{} missing from dict'.format(key))
+ return d.get(key, default) # type: ignore
+
+##################################
+
+
+def dict_get_join(d: Dict, key: str) -> Any:
+ """
+ Helper function to get the value of a given key from a dictionary.
+ `List` values will be converted to a string by joining them with a
+ line break.
+ :param d: The dictionary to process.
+ :param key: The name of the key to get.
+ :return: Returns the value of the given key. If it was a `list`, it
+ will be joining with a line break.
+ """
+ value = d.get(key)
+ if isinstance(value, list):
+ value = '\n'.join(map(str, value))
+ return value
+
+##################################
+
+
+def get_supported_daemons():
+ # type: () -> List[str]
+ supported_daemons = list(Ceph.daemons)
+ supported_daemons.extend(Monitoring.components)
+ supported_daemons.append(NFSGanesha.daemon_type)
+ supported_daemons.append(CephIscsi.daemon_type)
+ supported_daemons.append(CustomContainer.daemon_type)
+ supported_daemons.append(CephadmDaemon.daemon_type)
+ supported_daemons.append(HAproxy.daemon_type)
+ supported_daemons.append(Keepalived.daemon_type)
+ supported_daemons.append(SNMPGateway.daemon_type)
+ assert len(supported_daemons) == len(set(supported_daemons))
+ return supported_daemons
+
+##################################
+
+
+class PortOccupiedError(Error):
+ pass
+
+
+def attempt_bind(ctx, s, address, port):
+ # type: (CephadmContext, socket.socket, str, int) -> None
+ try:
+ s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+ s.bind((address, port))
+ except OSError as e:
+ if e.errno == errno.EADDRINUSE:
+ msg = 'Cannot bind to IP %s port %d: %s' % (address, port, e)
+ logger.warning(msg)
+ raise PortOccupiedError(msg)
+ else:
+ raise Error(e)
+ except Exception as e:
+ raise Error(e)
+ finally:
+ s.close()
+
+
+def port_in_use(ctx, port_num):
+ # type: (CephadmContext, int) -> bool
+ """Detect whether a port is in use on the local machine - IPv4 and IPv6"""
+ logger.info('Verifying port %d ...' % port_num)
+
+ def _port_in_use(af: socket.AddressFamily, address: str) -> bool:
+ try:
+ s = socket.socket(af, socket.SOCK_STREAM)
+ attempt_bind(ctx, s, address, port_num)
+ except PortOccupiedError:
+ return True
+ except OSError as e:
+ if e.errno in (errno.EAFNOSUPPORT, errno.EADDRNOTAVAIL):
+ # Ignore EAFNOSUPPORT and EADDRNOTAVAIL as two interfaces are
+ # being tested here and one might be intentionally be disabled.
+ # In that case no error should be raised.
+ return False
+ else:
+ raise e
+ return False
+ return any(_port_in_use(af, address) for af, address in (
+ (socket.AF_INET, '0.0.0.0'),
+ (socket.AF_INET6, '::')
+ ))
+
+
+def check_ip_port(ctx, ep):
+ # type: (CephadmContext, EndPoint) -> None
+ if not ctx.skip_ping_check:
+ logger.info(f'Verifying IP {ep.ip} port {ep.port} ...')
+ if is_ipv6(ep.ip):
+ s = socket.socket(socket.AF_INET6, socket.SOCK_STREAM)
+ ip = unwrap_ipv6(ep.ip)
+ else:
+ s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+ ip = ep.ip
+ attempt_bind(ctx, s, ip, ep.port)
+
+##################################
+
+
+# this is an abbreviated version of
+# https://github.com/benediktschmitt/py-filelock/blob/master/filelock.py
+# that drops all of the compatibility (this is Unix/Linux only).
+
+class Timeout(TimeoutError):
+ """
+ Raised when the lock could not be acquired in *timeout*
+ seconds.
+ """
+
+ def __init__(self, lock_file: str) -> None:
+ """
+ """
+ #: The path of the file lock.
+ self.lock_file = lock_file
+ return None
+
+ def __str__(self) -> str:
+ temp = "The file lock '{}' could not be acquired."\
+ .format(self.lock_file)
+ return temp
+
+
+class _Acquire_ReturnProxy(object):
+ def __init__(self, lock: 'FileLock') -> None:
+ self.lock = lock
+ return None
+
+ def __enter__(self) -> 'FileLock':
+ return self.lock
+
+ def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> None:
+ self.lock.release()
+ return None
+
+
+class FileLock(object):
+ def __init__(self, ctx: CephadmContext, name: str, timeout: int = -1) -> None:
+ if not os.path.exists(LOCK_DIR):
+ os.mkdir(LOCK_DIR, 0o700)
+ self._lock_file = os.path.join(LOCK_DIR, name + '.lock')
+ self.ctx = ctx
+
+ # The file descriptor for the *_lock_file* as it is returned by the
+ # os.open() function.
+ # This file lock is only NOT None, if the object currently holds the
+ # lock.
+ self._lock_file_fd: Optional[int] = None
+ self.timeout = timeout
+ # The lock counter is used for implementing the nested locking
+ # mechanism. Whenever the lock is acquired, the counter is increased and
+ # the lock is only released, when this value is 0 again.
+ self._lock_counter = 0
+ return None
+
+ @property
+ def is_locked(self) -> bool:
+ return self._lock_file_fd is not None
+
+ def acquire(self, timeout: Optional[int] = None, poll_intervall: float = 0.05) -> _Acquire_ReturnProxy:
+ """
+ Acquires the file lock or fails with a :exc:`Timeout` error.
+ .. code-block:: python
+ # You can use this method in the context manager (recommended)
+ with lock.acquire():
+ pass
+ # Or use an equivalent try-finally construct:
+ lock.acquire()
+ try:
+ pass
+ finally:
+ lock.release()
+ :arg float timeout:
+ The maximum time waited for the file lock.
+ If ``timeout < 0``, there is no timeout and this method will
+ block until the lock could be acquired.
+ If ``timeout`` is None, the default :attr:`~timeout` is used.
+ :arg float poll_intervall:
+ We check once in *poll_intervall* seconds if we can acquire the
+ file lock.
+ :raises Timeout:
+ if the lock could not be acquired in *timeout* seconds.
+ .. versionchanged:: 2.0.0
+ This method returns now a *proxy* object instead of *self*,
+ so that it can be used in a with statement without side effects.
+ """
+
+ # Use the default timeout, if no timeout is provided.
+ if timeout is None:
+ timeout = self.timeout
+
+ # Increment the number right at the beginning.
+ # We can still undo it, if something fails.
+ self._lock_counter += 1
+
+ lock_id = id(self)
+ lock_filename = self._lock_file
+ start_time = time.time()
+ try:
+ while True:
+ if not self.is_locked:
+ logger.debug('Acquiring lock %s on %s', lock_id,
+ lock_filename)
+ self._acquire()
+
+ if self.is_locked:
+ logger.debug('Lock %s acquired on %s', lock_id,
+ lock_filename)
+ break
+ elif timeout >= 0 and time.time() - start_time > timeout:
+ logger.warning('Timeout acquiring lock %s on %s', lock_id,
+ lock_filename)
+ raise Timeout(self._lock_file)
+ else:
+ logger.debug(
+ 'Lock %s not acquired on %s, waiting %s seconds ...',
+ lock_id, lock_filename, poll_intervall
+ )
+ time.sleep(poll_intervall)
+ except Exception:
+ # Something did go wrong, so decrement the counter.
+ self._lock_counter = max(0, self._lock_counter - 1)
+
+ raise
+ return _Acquire_ReturnProxy(lock=self)
+
+ def release(self, force: bool = False) -> None:
+ """
+ Releases the file lock.
+ Please note, that the lock is only completly released, if the lock
+ counter is 0.
+ Also note, that the lock file itself is not automatically deleted.
+ :arg bool force:
+ If true, the lock counter is ignored and the lock is released in
+ every case.
+ """
+ if self.is_locked:
+ self._lock_counter -= 1
+
+ if self._lock_counter == 0 or force:
+ # lock_id = id(self)
+ # lock_filename = self._lock_file
+
+ # Can't log in shutdown:
+ # File "/usr/lib64/python3.9/logging/__init__.py", line 1175, in _open
+ # NameError: name 'open' is not defined
+ # logger.debug('Releasing lock %s on %s', lock_id, lock_filename)
+ self._release()
+ self._lock_counter = 0
+ # logger.debug('Lock %s released on %s', lock_id, lock_filename)
+
+ return None
+
+ def __enter__(self) -> 'FileLock':
+ self.acquire()
+ return self
+
+ def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> None:
+ self.release()
+ return None
+
+ def __del__(self) -> None:
+ self.release(force=True)
+ return None
+
+ def _acquire(self) -> None:
+ open_mode = os.O_RDWR | os.O_CREAT | os.O_TRUNC
+ fd = os.open(self._lock_file, open_mode)
+
+ try:
+ fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
+ except (IOError, OSError):
+ os.close(fd)
+ else:
+ self._lock_file_fd = fd
+ return None
+
+ def _release(self) -> None:
+ # Do not remove the lockfile:
+ #
+ # https://github.com/benediktschmitt/py-filelock/issues/31
+ # https://stackoverflow.com/questions/17708885/flock-removing-locked-file-without-race-condition
+ fd = self._lock_file_fd
+ self._lock_file_fd = None
+ fcntl.flock(fd, fcntl.LOCK_UN) # type: ignore
+ os.close(fd) # type: ignore
+ return None
+
+
+##################################
+# Popen wrappers, lifted from ceph-volume
+
+class CallVerbosity(Enum):
+ SILENT = 0
+ # log stdout/stderr to logger.debug
+ DEBUG = 1
+ # On a non-zero exit status, it will forcefully set
+ # logging ON for the terminal
+ VERBOSE_ON_FAILURE = 2
+ # log at info (instead of debug) level.
+ VERBOSE = 3
+
+
+if sys.version_info < (3, 8):
+ import itertools
+ import threading
+ import warnings
+ from asyncio import events
+
+ class ThreadedChildWatcher(asyncio.AbstractChildWatcher):
+ """Threaded child watcher implementation.
+ The watcher uses a thread per process
+ for waiting for the process finish.
+ It doesn't require subscription on POSIX signal
+ but a thread creation is not free.
+ The watcher has O(1) complexity, its performance doesn't depend
+ on amount of spawn processes.
+ """
+
+ def __init__(self) -> None:
+ self._pid_counter = itertools.count(0)
+ self._threads: Dict[Any, Any] = {}
+
+ def is_active(self) -> bool:
+ return True
+
+ def close(self) -> None:
+ self._join_threads()
+
+ def _join_threads(self) -> None:
+ """Internal: Join all non-daemon threads"""
+ threads = [thread for thread in list(self._threads.values())
+ if thread.is_alive() and not thread.daemon]
+ for thread in threads:
+ thread.join()
+
+ def __enter__(self) -> Any:
+ return self
+
+ def __exit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> None:
+ pass
+
+ def __del__(self, _warn: Any = warnings.warn) -> None:
+ threads = [thread for thread in list(self._threads.values())
+ if thread.is_alive()]
+ if threads:
+ _warn(f'{self.__class__} has registered but not finished child processes',
+ ResourceWarning,
+ source=self)
+
+ def add_child_handler(self, pid: Any, callback: Any, *args: Any) -> None:
+ loop = events.get_event_loop()
+ thread = threading.Thread(target=self._do_waitpid,
+ name=f'waitpid-{next(self._pid_counter)}',
+ args=(loop, pid, callback, args),
+ daemon=True)
+ self._threads[pid] = thread
+ thread.start()
+
+ def remove_child_handler(self, pid: Any) -> bool:
+ # asyncio never calls remove_child_handler() !!!
+ # The method is no-op but is implemented because
+ # abstract base classe requires it
+ return True
+
+ def attach_loop(self, loop: Any) -> None:
+ pass
+
+ def _do_waitpid(self, loop: Any, expected_pid: Any, callback: Any, args: Any) -> None:
+ assert expected_pid > 0
+
+ try:
+ pid, status = os.waitpid(expected_pid, 0)
+ except ChildProcessError:
+ # The child process is already reaped
+ # (may happen if waitpid() is called elsewhere).
+ pid = expected_pid
+ returncode = 255
+ logger.warning(
+ 'Unknown child process pid %d, will report returncode 255',
+ pid)
+ else:
+ if os.WIFEXITED(status):
+ returncode = os.WEXITSTATUS(status)
+ elif os.WIFSIGNALED(status):
+ returncode = -os.WTERMSIG(status)
+ else:
+ raise ValueError(f'unknown wait status {status}')
+ if loop.get_debug():
+ logger.debug('process %s exited with returncode %s',
+ expected_pid, returncode)
+
+ if loop.is_closed():
+ logger.warning('Loop %r that handles pid %r is closed', loop, pid)
+ else:
+ loop.call_soon_threadsafe(callback, pid, returncode, *args)
+
+ self._threads.pop(expected_pid)
+
+ # unlike SafeChildWatcher which handles SIGCHLD in the main thread,
+ # ThreadedChildWatcher runs in a separated thread, hence allows us to
+ # run create_subprocess_exec() in non-main thread, see
+ # https://bugs.python.org/issue35621
+ asyncio.set_child_watcher(ThreadedChildWatcher())
+
+
+try:
+ from asyncio import run as async_run # type: ignore[attr-defined]
+except ImportError:
+ def async_run(coro): # type: ignore
+ loop = asyncio.new_event_loop()
+ try:
+ asyncio.set_event_loop(loop)
+ return loop.run_until_complete(coro)
+ finally:
+ try:
+ loop.run_until_complete(loop.shutdown_asyncgens())
+ finally:
+ asyncio.set_event_loop(None)
+ loop.close()
+
+
+def call(ctx: CephadmContext,
+ command: List[str],
+ desc: Optional[str] = None,
+ verbosity: CallVerbosity = CallVerbosity.VERBOSE_ON_FAILURE,
+ timeout: Optional[int] = DEFAULT_TIMEOUT,
+ **kwargs: Any) -> Tuple[str, str, int]:
+ """
+ Wrap subprocess.Popen to
+
+ - log stdout/stderr to a logger,
+ - decode utf-8
+ - cleanly return out, err, returncode
+
+ :param timeout: timeout in seconds
+ """
+
+ prefix = command[0] if desc is None else desc
+ if prefix:
+ prefix += ': '
+ timeout = timeout or ctx.timeout
+
+ async def tee(reader: asyncio.StreamReader) -> str:
+ collected = StringIO()
+ async for line in reader:
+ message = line.decode('utf-8')
+ collected.write(message)
+ if verbosity == CallVerbosity.VERBOSE:
+ logger.info(prefix + message.rstrip())
+ elif verbosity != CallVerbosity.SILENT:
+ logger.debug(prefix + message.rstrip())
+ return collected.getvalue()
+
+ async def run_with_timeout() -> Tuple[str, str, int]:
+ process = await asyncio.create_subprocess_exec(
+ *command,
+ stdout=asyncio.subprocess.PIPE,
+ stderr=asyncio.subprocess.PIPE,
+ env=os.environ.copy())
+ assert process.stdout
+ assert process.stderr
+ try:
+ stdout, stderr = await asyncio.gather(tee(process.stdout),
+ tee(process.stderr))
+ returncode = await asyncio.wait_for(process.wait(), timeout)
+ except asyncio.TimeoutError:
+ logger.info(prefix + f'timeout after {timeout} seconds')
+ return '', '', 124
+ else:
+ return stdout, stderr, returncode
+
+ stdout, stderr, returncode = async_run(run_with_timeout())
+ if returncode != 0 and verbosity == CallVerbosity.VERBOSE_ON_FAILURE:
+ logger.info('Non-zero exit code %d from %s',
+ returncode, ' '.join(command))
+ for line in stdout.splitlines():
+ logger.info(prefix + 'stdout ' + line)
+ for line in stderr.splitlines():
+ logger.info(prefix + 'stderr ' + line)
+ return stdout, stderr, returncode
+
+
+def call_throws(
+ ctx: CephadmContext,
+ command: List[str],
+ desc: Optional[str] = None,
+ verbosity: CallVerbosity = CallVerbosity.VERBOSE_ON_FAILURE,
+ timeout: Optional[int] = DEFAULT_TIMEOUT,
+ **kwargs: Any) -> Tuple[str, str, int]:
+ out, err, ret = call(ctx, command, desc, verbosity, timeout, **kwargs)
+ if ret:
+ for s in (out, err):
+ if s.strip() and len(s.splitlines()) <= 2: # readable message?
+ raise RuntimeError(f'Failed command: {" ".join(command)}: {s}')
+ raise RuntimeError('Failed command: %s' % ' '.join(command))
+ return out, err, ret
+
+
+def call_timeout(ctx, command, timeout):
+ # type: (CephadmContext, List[str], int) -> int
+ logger.debug('Running command (timeout=%s): %s'
+ % (timeout, ' '.join(command)))
+
+ def raise_timeout(command, timeout):
+ # type: (List[str], int) -> NoReturn
+ msg = 'Command `%s` timed out after %s seconds' % (command, timeout)
+ logger.debug(msg)
+ raise TimeoutExpired(msg)
+
+ try:
+ return subprocess.call(command, timeout=timeout, env=os.environ.copy())
+ except subprocess.TimeoutExpired:
+ raise_timeout(command, timeout)
+
+##################################
+
+
+def json_loads_retry(cli_func: Callable[[], str]) -> Any:
+ for sleep_secs in [1, 4, 4]:
+ try:
+ return json.loads(cli_func())
+ except json.JSONDecodeError:
+ logger.debug('Invalid JSON. Retrying in %s seconds...' % sleep_secs)
+ time.sleep(sleep_secs)
+ return json.loads(cli_func())
+
+
+def is_available(ctx, what, func):
+ # type: (CephadmContext, str, Callable[[], bool]) -> None
+ """
+ Wait for a service to become available
+
+ :param what: the name of the service
+ :param func: the callable object that determines availability
+ """
+ retry = ctx.retry
+ logger.info('Waiting for %s...' % what)
+ num = 1
+ while True:
+ if func():
+ logger.info('%s is available'
+ % what)
+ break
+ elif num > retry:
+ raise Error('%s not available after %s tries'
+ % (what, retry))
+
+ logger.info('%s not available, waiting (%s/%s)...'
+ % (what, num, retry))
+
+ num += 1
+ time.sleep(2)
+
+
+def read_config(fn):
+ # type: (Optional[str]) -> ConfigParser
+ cp = ConfigParser()
+ if fn:
+ cp.read(fn)
+ return cp
+
+
+def pathify(p):
+ # type: (str) -> str
+ p = os.path.expanduser(p)
+ return os.path.abspath(p)
+
+
+def get_file_timestamp(fn):
+ # type: (str) -> Optional[str]
+ try:
+ mt = os.path.getmtime(fn)
+ return datetime.datetime.fromtimestamp(
+ mt, tz=datetime.timezone.utc
+ ).strftime(DATEFMT)
+ except Exception:
+ return None
+
+
+def try_convert_datetime(s):
+ # type: (str) -> Optional[str]
+ # This is super irritating because
+ # 1) podman and docker use different formats
+ # 2) python's strptime can't parse either one
+ #
+ # I've seen:
+ # docker 18.09.7: 2020-03-03T09:21:43.636153304Z
+ # podman 1.7.0: 2020-03-03T15:52:30.136257504-06:00
+ # 2020-03-03 15:52:30.136257504 -0600 CST
+ # (In the podman case, there is a different string format for
+ # 'inspect' and 'inspect --format {{.Created}}'!!)
+
+ # In *all* cases, the 9 digit second precision is too much for
+ # python's strptime. Shorten it to 6 digits.
+ p = re.compile(r'(\.[\d]{6})[\d]*')
+ s = p.sub(r'\1', s)
+
+ # replace trailing Z with -0000, since (on python 3.6.8) it won't parse
+ if s and s[-1] == 'Z':
+ s = s[:-1] + '-0000'
+
+ # cut off the redundant 'CST' part that strptime can't parse, if
+ # present.
+ v = s.split(' ')
+ s = ' '.join(v[0:3])
+
+ # try parsing with several format strings
+ fmts = [
+ '%Y-%m-%dT%H:%M:%S.%f%z',
+ '%Y-%m-%d %H:%M:%S.%f %z',
+ ]
+ for f in fmts:
+ try:
+ # return timestamp normalized to UTC, rendered as DATEFMT.
+ return datetime.datetime.strptime(s, f).astimezone(tz=datetime.timezone.utc).strftime(DATEFMT)
+ except ValueError:
+ pass
+ return None
+
+
+def _parse_podman_version(version_str):
+ # type: (str) -> Tuple[int, ...]
+ def to_int(val: str, org_e: Optional[Exception] = None) -> int:
+ if not val and org_e:
+ raise org_e
+ try:
+ return int(val)
+ except ValueError as e:
+ return to_int(val[0:-1], org_e or e)
+
+ return tuple(map(to_int, version_str.split('.')))
+
+
+def get_hostname():
+ # type: () -> str
+ return socket.gethostname()
+
+
+def get_fqdn():
+ # type: () -> str
+ return socket.getfqdn() or socket.gethostname()
+
+
+def get_arch():
+ # type: () -> str
+ return platform.uname().machine
+
+
+def generate_service_id():
+ # type: () -> str
+ return get_hostname() + '.' + ''.join(random.choice(string.ascii_lowercase)
+ for _ in range(6))
+
+
+def generate_password():
+ # type: () -> str
+ return ''.join(random.choice(string.ascii_lowercase + string.digits)
+ for i in range(10))
+
+
+def normalize_container_id(i):
+ # type: (str) -> str
+ # docker adds the sha256: prefix, but AFAICS both
+ # docker (18.09.7 in bionic at least) and podman
+ # both always use sha256, so leave off the prefix
+ # for consistency.
+ prefix = 'sha256:'
+ if i.startswith(prefix):
+ i = i[len(prefix):]
+ return i
+
+
+def make_fsid():
+ # type: () -> str
+ return str(uuid.uuid1())
+
+
+def is_fsid(s):
+ # type: (str) -> bool
+ try:
+ uuid.UUID(s)
+ except ValueError:
+ return False
+ return True
+
+
+def validate_fsid(func: FuncT) -> FuncT:
+ @wraps(func)
+ def _validate_fsid(ctx: CephadmContext) -> Any:
+ if 'fsid' in ctx and ctx.fsid:
+ if not is_fsid(ctx.fsid):
+ raise Error('not an fsid: %s' % ctx.fsid)
+ return func(ctx)
+ return cast(FuncT, _validate_fsid)
+
+
+def infer_fsid(func: FuncT) -> FuncT:
+ """
+ If we only find a single fsid in /var/lib/ceph/*, use that
+ """
+ @infer_config
+ @wraps(func)
+ def _infer_fsid(ctx: CephadmContext) -> Any:
+ if 'fsid' in ctx and ctx.fsid:
+ logger.debug('Using specified fsid: %s' % ctx.fsid)
+ return func(ctx)
+
+ fsids = set()
+
+ cp = read_config(ctx.config)
+ if cp.has_option('global', 'fsid'):
+ fsids.add(cp.get('global', 'fsid'))
+
+ daemon_list = list_daemons(ctx, detail=False)
+ for daemon in daemon_list:
+ if not is_fsid(daemon['fsid']):
+ # 'unknown' fsid
+ continue
+ elif 'name' not in ctx or not ctx.name:
+ # ctx.name not specified
+ fsids.add(daemon['fsid'])
+ elif daemon['name'] == ctx.name:
+ # ctx.name is a match
+ fsids.add(daemon['fsid'])
+ fsids = sorted(fsids)
+
+ if not fsids:
+ # some commands do not always require an fsid
+ pass
+ elif len(fsids) == 1:
+ logger.info('Inferring fsid %s' % fsids[0])
+ ctx.fsid = fsids[0]
+ else:
+ raise Error('Cannot infer an fsid, one must be specified (using --fsid): %s' % fsids)
+ return func(ctx)
+
+ return cast(FuncT, _infer_fsid)
+
+
+def infer_config(func: FuncT) -> FuncT:
+ """
+ If we find a MON daemon, use the config from that container
+ """
+ @wraps(func)
+ def _infer_config(ctx: CephadmContext) -> Any:
+ ctx.config = ctx.config if 'config' in ctx else None
+ if ctx.config:
+ logger.debug('Using specified config: %s' % ctx.config)
+ return func(ctx)
+
+ def config_path(daemon_type: str, daemon_name: str) -> str:
+ data_dir = get_data_dir(ctx.fsid, ctx.data_dir, daemon_type, daemon_name)
+ return os.path.join(data_dir, 'config')
+
+ if 'fsid' in ctx and ctx.fsid:
+ name = ctx.name if 'name' in ctx else None
+ if not name:
+ daemon_list = list_daemons(ctx, detail=False)
+ for daemon in daemon_list:
+ if (
+ daemon.get('name', '').startswith('mon.')
+ and daemon.get('fsid', '') == ctx.fsid
+ and daemon.get('style', '') == 'cephadm:v1'
+ and os.path.exists(config_path('mon', daemon['name'].split('.', 1)[1]))
+ ):
+ name = daemon['name']
+ break
+ if name:
+ ctx.config = config_path(name.split('.', 1)[0], name.split('.', 1)[1])
+ if ctx.config:
+ logger.info('Inferring config %s' % ctx.config)
+ elif os.path.exists(SHELL_DEFAULT_CONF):
+ logger.debug('Using default config: %s' % SHELL_DEFAULT_CONF)
+ ctx.config = SHELL_DEFAULT_CONF
+ return func(ctx)
+
+ return cast(FuncT, _infer_config)
+
+
+def _get_default_image(ctx: CephadmContext) -> str:
+ if DEFAULT_IMAGE_IS_MASTER:
+ warn = """This is a development version of cephadm.
+For information regarding the latest stable release:
+ https://docs.ceph.com/docs/{}/cephadm/install
+""".format(LATEST_STABLE_RELEASE)
+ for line in warn.splitlines():
+ logger.warning('{}{}{}'.format(termcolor.yellow, line, termcolor.end))
+ return DEFAULT_IMAGE
+
+
+def infer_image(func: FuncT) -> FuncT:
+ """
+ Use the most recent ceph image
+ """
+ @wraps(func)
+ def _infer_image(ctx: CephadmContext) -> Any:
+ if not ctx.image:
+ ctx.image = os.environ.get('CEPHADM_IMAGE')
+ if not ctx.image:
+ ctx.image = get_last_local_ceph_image(ctx, ctx.container_engine.path)
+ if not ctx.image:
+ ctx.image = _get_default_image(ctx)
+ return func(ctx)
+
+ return cast(FuncT, _infer_image)
+
+
+def default_image(func: FuncT) -> FuncT:
+ @wraps(func)
+ def _default_image(ctx: CephadmContext) -> Any:
+ if not ctx.image:
+ if 'name' in ctx and ctx.name:
+ type_ = ctx.name.split('.', 1)[0]
+ if type_ in Monitoring.components:
+ ctx.image = Monitoring.components[type_]['image']
+ if type_ == 'haproxy':
+ ctx.image = HAproxy.default_image
+ if type_ == 'keepalived':
+ ctx.image = Keepalived.default_image
+ if type_ == SNMPGateway.daemon_type:
+ ctx.image = SNMPGateway.default_image
+ if not ctx.image:
+ ctx.image = os.environ.get('CEPHADM_IMAGE')
+ if not ctx.image:
+ ctx.image = _get_default_image(ctx)
+
+ return func(ctx)
+
+ return cast(FuncT, _default_image)
+
+
+def get_last_local_ceph_image(ctx: CephadmContext, container_path: str) -> Optional[str]:
+ """
+ :return: The most recent local ceph image (already pulled)
+ """
+ out, _, _ = call_throws(ctx,
+ [container_path, 'images',
+ '--filter', 'label=ceph=True',
+ '--filter', 'dangling=false',
+ '--format', '{{.Repository}}@{{.Digest}}'])
+ return _filter_last_local_ceph_image(out)
+
+
+def _filter_last_local_ceph_image(out):
+ # type: (str) -> Optional[str]
+ for image in out.splitlines():
+ if image and not image.endswith('@'):
+ logger.info('Using recent ceph image %s' % image)
+ return image
+ return None
+
+
+def write_tmp(s, uid, gid):
+ # type: (str, int, int) -> IO[str]
+ tmp_f = tempfile.NamedTemporaryFile(mode='w',
+ prefix='ceph-tmp')
+ os.fchown(tmp_f.fileno(), uid, gid)
+ tmp_f.write(s)
+ tmp_f.flush()
+
+ return tmp_f
+
+
+def makedirs(dir, uid, gid, mode):
+ # type: (str, int, int, int) -> None
+ if not os.path.exists(dir):
+ os.makedirs(dir, mode=mode)
+ else:
+ os.chmod(dir, mode)
+ os.chown(dir, uid, gid)
+ os.chmod(dir, mode) # the above is masked by umask...
+
+
+def get_data_dir(fsid, data_dir, t, n):
+ # type: (str, str, str, Union[int, str]) -> str
+ return os.path.join(data_dir, fsid, '%s.%s' % (t, n))
+
+
+def get_log_dir(fsid, log_dir):
+ # type: (str, str) -> str
+ return os.path.join(log_dir, fsid)
+
+
+def make_data_dir_base(fsid, data_dir, uid, gid):
+ # type: (str, str, int, int) -> str
+ data_dir_base = os.path.join(data_dir, fsid)
+ makedirs(data_dir_base, uid, gid, DATA_DIR_MODE)
+ makedirs(os.path.join(data_dir_base, 'crash'), uid, gid, DATA_DIR_MODE)
+ makedirs(os.path.join(data_dir_base, 'crash', 'posted'), uid, gid,
+ DATA_DIR_MODE)
+ return data_dir_base
+
+
+def make_data_dir(ctx, fsid, daemon_type, daemon_id, uid=None, gid=None):
+ # type: (CephadmContext, str, str, Union[int, str], Optional[int], Optional[int]) -> str
+ if uid is None or gid is None:
+ uid, gid = extract_uid_gid(ctx)
+ make_data_dir_base(fsid, ctx.data_dir, uid, gid)
+ data_dir = get_data_dir(fsid, ctx.data_dir, daemon_type, daemon_id)
+ makedirs(data_dir, uid, gid, DATA_DIR_MODE)
+ return data_dir
+
+
+def make_log_dir(ctx, fsid, uid=None, gid=None):
+ # type: (CephadmContext, str, Optional[int], Optional[int]) -> str
+ if uid is None or gid is None:
+ uid, gid = extract_uid_gid(ctx)
+ log_dir = get_log_dir(fsid, ctx.log_dir)
+ makedirs(log_dir, uid, gid, LOG_DIR_MODE)
+ return log_dir
+
+
+def make_var_run(ctx, fsid, uid, gid):
+ # type: (CephadmContext, str, int, int) -> None
+ call_throws(ctx, ['install', '-d', '-m0770', '-o', str(uid), '-g', str(gid),
+ '/var/run/ceph/%s' % fsid])
+
+
+def copy_tree(ctx, src, dst, uid=None, gid=None):
+ # type: (CephadmContext, List[str], str, Optional[int], Optional[int]) -> None
+ """
+ Copy a directory tree from src to dst
+ """
+ if uid is None or gid is None:
+ (uid, gid) = extract_uid_gid(ctx)
+
+ for src_dir in src:
+ dst_dir = dst
+ if os.path.isdir(dst):
+ dst_dir = os.path.join(dst, os.path.basename(src_dir))
+
+ logger.debug('copy directory `%s` -> `%s`' % (src_dir, dst_dir))
+ shutil.rmtree(dst_dir, ignore_errors=True)
+ shutil.copytree(src_dir, dst_dir) # dirs_exist_ok needs python 3.8
+
+ for dirpath, dirnames, filenames in os.walk(dst_dir):
+ logger.debug('chown %s:%s `%s`' % (uid, gid, dirpath))
+ os.chown(dirpath, uid, gid)
+ for filename in filenames:
+ logger.debug('chown %s:%s `%s`' % (uid, gid, filename))
+ os.chown(os.path.join(dirpath, filename), uid, gid)
+
+
+def copy_files(ctx, src, dst, uid=None, gid=None):
+ # type: (CephadmContext, List[str], str, Optional[int], Optional[int]) -> None
+ """
+ Copy a files from src to dst
+ """
+ if uid is None or gid is None:
+ (uid, gid) = extract_uid_gid(ctx)
+
+ for src_file in src:
+ dst_file = dst
+ if os.path.isdir(dst):
+ dst_file = os.path.join(dst, os.path.basename(src_file))
+
+ logger.debug('copy file `%s` -> `%s`' % (src_file, dst_file))
+ shutil.copyfile(src_file, dst_file)
+
+ logger.debug('chown %s:%s `%s`' % (uid, gid, dst_file))
+ os.chown(dst_file, uid, gid)
+
+
+def move_files(ctx, src, dst, uid=None, gid=None):
+ # type: (CephadmContext, List[str], str, Optional[int], Optional[int]) -> None
+ """
+ Move files from src to dst
+ """
+ if uid is None or gid is None:
+ (uid, gid) = extract_uid_gid(ctx)
+
+ for src_file in src:
+ dst_file = dst
+ if os.path.isdir(dst):
+ dst_file = os.path.join(dst, os.path.basename(src_file))
+
+ if os.path.islink(src_file):
+ # shutil.move() in py2 does not handle symlinks correctly
+ src_rl = os.readlink(src_file)
+ logger.debug("symlink '%s' -> '%s'" % (dst_file, src_rl))
+ os.symlink(src_rl, dst_file)
+ os.unlink(src_file)
+ else:
+ logger.debug("move file '%s' -> '%s'" % (src_file, dst_file))
+ shutil.move(src_file, dst_file)
+ logger.debug('chown %s:%s `%s`' % (uid, gid, dst_file))
+ os.chown(dst_file, uid, gid)
+
+
+def recursive_chown(path: str, uid: int, gid: int) -> None:
+ for dirpath, dirnames, filenames in os.walk(path):
+ os.chown(dirpath, uid, gid)
+ for filename in filenames:
+ os.chown(os.path.join(dirpath, filename), uid, gid)
+
+
+# copied from distutils
+def find_executable(executable: str, path: Optional[str] = None) -> Optional[str]:
+ """Tries to find 'executable' in the directories listed in 'path'.
+ A string listing directories separated by 'os.pathsep'; defaults to
+ os.environ['PATH']. Returns the complete filename or None if not found.
+ """
+ _, ext = os.path.splitext(executable)
+ if (sys.platform == 'win32') and (ext != '.exe'):
+ executable = executable + '.exe'
+
+ if os.path.isfile(executable):
+ return executable
+
+ if path is None:
+ path = os.environ.get('PATH', None)
+ if path is None:
+ try:
+ path = os.confstr('CS_PATH')
+ except (AttributeError, ValueError):
+ # os.confstr() or CS_PATH is not available
+ path = os.defpath
+ # bpo-35755: Don't use os.defpath if the PATH environment variable is
+ # set to an empty string
+
+ # PATH='' doesn't match, whereas PATH=':' looks in the current directory
+ if not path:
+ return None
+
+ paths = path.split(os.pathsep)
+ for p in paths:
+ f = os.path.join(p, executable)
+ if os.path.isfile(f):
+ # the file exists, we have a shot at spawn working
+ return f
+ return None
+
+
+def find_program(filename):
+ # type: (str) -> str
+ name = find_executable(filename)
+ if name is None:
+ raise ValueError('%s not found' % filename)
+ return name
+
+
+def find_container_engine(ctx: CephadmContext) -> Optional[ContainerEngine]:
+ if ctx.docker:
+ return Docker()
+ else:
+ for i in CONTAINER_PREFERENCE:
+ try:
+ return i()
+ except Exception:
+ pass
+ return None
+
+
+def check_container_engine(ctx: CephadmContext) -> ContainerEngine:
+ engine = ctx.container_engine
+ if not isinstance(engine, CONTAINER_PREFERENCE):
+ # See https://github.com/python/mypy/issues/8993
+ exes: List[str] = [i.EXE for i in CONTAINER_PREFERENCE] # type: ignore
+ raise Error('No container engine binary found ({}). Try run `apt/dnf/yum/zypper install <container engine>`'.format(' or '.join(exes)))
+ elif isinstance(engine, Podman):
+ engine.get_version(ctx)
+ if engine.version < MIN_PODMAN_VERSION:
+ raise Error('podman version %d.%d.%d or later is required' % MIN_PODMAN_VERSION)
+ return engine
+
+
+def get_unit_name(fsid, daemon_type, daemon_id=None):
+ # type: (str, str, Optional[Union[int, str]]) -> str
+ # accept either name or type + id
+ if daemon_type == CephadmDaemon.daemon_type and daemon_id is not None:
+ return 'ceph-%s-%s.%s' % (fsid, daemon_type, daemon_id)
+ elif daemon_id is not None:
+ return 'ceph-%s@%s.%s' % (fsid, daemon_type, daemon_id)
+ else:
+ return 'ceph-%s@%s' % (fsid, daemon_type)
+
+
+def get_unit_name_by_daemon_name(ctx: CephadmContext, fsid: str, name: str) -> str:
+ daemon = get_daemon_description(ctx, fsid, name)
+ try:
+ return daemon['systemd_unit']
+ except KeyError:
+ raise Error('Failed to get unit name for {}'.format(daemon))
+
+
+def check_unit(ctx, unit_name):
+ # type: (CephadmContext, str) -> Tuple[bool, str, bool]
+ # NOTE: we ignore the exit code here because systemctl outputs
+ # various exit codes based on the state of the service, but the
+ # string result is more explicit (and sufficient).
+ enabled = False
+ installed = False
+ try:
+ out, err, code = call(ctx, ['systemctl', 'is-enabled', unit_name],
+ verbosity=CallVerbosity.DEBUG)
+ if code == 0:
+ enabled = True
+ installed = True
+ elif 'disabled' in out:
+ installed = True
+ except Exception as e:
+ logger.warning('unable to run systemctl: %s' % e)
+ enabled = False
+ installed = False
+
+ state = 'unknown'
+ try:
+ out, err, code = call(ctx, ['systemctl', 'is-active', unit_name],
+ verbosity=CallVerbosity.DEBUG)
+ out = out.strip()
+ if out in ['active']:
+ state = 'running'
+ elif out in ['inactive']:
+ state = 'stopped'
+ elif out in ['failed', 'auto-restart']:
+ state = 'error'
+ else:
+ state = 'unknown'
+ except Exception as e:
+ logger.warning('unable to run systemctl: %s' % e)
+ state = 'unknown'
+ return (enabled, state, installed)
+
+
+def check_units(ctx, units, enabler=None):
+ # type: (CephadmContext, List[str], Optional[Packager]) -> bool
+ for u in units:
+ (enabled, state, installed) = check_unit(ctx, u)
+ if enabled and state == 'running':
+ logger.info('Unit %s is enabled and running' % u)
+ return True
+ if enabler is not None:
+ if installed:
+ logger.info('Enabling unit %s' % u)
+ enabler.enable_service(u)
+ return False
+
+
+def is_container_running(ctx: CephadmContext, c: 'CephContainer') -> bool:
+ return bool(get_running_container_name(ctx, c))
+
+
+def get_running_container_name(ctx: CephadmContext, c: 'CephContainer') -> Optional[str]:
+ for name in [c.cname, c.old_cname]:
+ out, err, ret = call(ctx, [
+ ctx.container_engine.path, 'container', 'inspect',
+ '--format', '{{.State.Status}}', name
+ ])
+ if out.strip() == 'running':
+ return name
+ return None
+
+
+def get_legacy_config_fsid(cluster, legacy_dir=None):
+ # type: (str, Optional[str]) -> Optional[str]
+ config_file = '/etc/ceph/%s.conf' % cluster
+ if legacy_dir is not None:
+ config_file = os.path.abspath(legacy_dir + config_file)
+
+ if os.path.exists(config_file):
+ config = read_config(config_file)
+ if config.has_section('global') and config.has_option('global', 'fsid'):
+ return config.get('global', 'fsid')
+ return None
+
+
+def get_legacy_daemon_fsid(ctx, cluster,
+ daemon_type, daemon_id, legacy_dir=None):
+ # type: (CephadmContext, str, str, Union[int, str], Optional[str]) -> Optional[str]
+ fsid = None
+ if daemon_type == 'osd':
+ try:
+ fsid_file = os.path.join(ctx.data_dir,
+ daemon_type,
+ 'ceph-%s' % daemon_id,
+ 'ceph_fsid')
+ if legacy_dir is not None:
+ fsid_file = os.path.abspath(legacy_dir + fsid_file)
+ with open(fsid_file, 'r') as f:
+ fsid = f.read().strip()
+ except IOError:
+ pass
+ if not fsid:
+ fsid = get_legacy_config_fsid(cluster, legacy_dir=legacy_dir)
+ return fsid
+
+
+def get_daemon_args(ctx, fsid, daemon_type, daemon_id):
+ # type: (CephadmContext, str, str, Union[int, str]) -> List[str]
+ r = list() # type: List[str]
+
+ if daemon_type in Ceph.daemons and daemon_type != 'crash':
+ r += [
+ '--setuser', 'ceph',
+ '--setgroup', 'ceph',
+ '--default-log-to-file=false',
+ '--default-log-to-stderr=true',
+ '--default-log-stderr-prefix=debug ',
+ ]
+ if daemon_type == 'mon':
+ r += [
+ '--default-mon-cluster-log-to-file=false',
+ '--default-mon-cluster-log-to-stderr=true',
+ ]
+ elif daemon_type in Monitoring.components:
+ metadata = Monitoring.components[daemon_type]
+ r += metadata.get('args', list())
+ # set ip and port to bind to for nodeexporter,alertmanager,prometheus
+ if daemon_type != 'grafana':
+ ip = ''
+ port = Monitoring.port_map[daemon_type][0]
+ if 'meta_json' in ctx and ctx.meta_json:
+ meta = json.loads(ctx.meta_json) or {}
+ if 'ip' in meta and meta['ip']:
+ ip = meta['ip']
+ if 'ports' in meta and meta['ports']:
+ port = meta['ports'][0]
+ r += [f'--web.listen-address={ip}:{port}']
+ if daemon_type == 'prometheus':
+ config = get_parm(ctx.config_json)
+ retention_time = config.get('retention_time', '15d')
+ r += [f'--storage.tsdb.retention.time={retention_time}']
+ scheme = 'http'
+ host = get_fqdn()
+ r += [f'--web.external-url={scheme}://{host}:{port}']
+ if daemon_type == 'alertmanager':
+ config = get_parm(ctx.config_json)
+ peers = config.get('peers', list()) # type: ignore
+ for peer in peers:
+ r += ['--cluster.peer={}'.format(peer)]
+ # some alertmanager, by default, look elsewhere for a config
+ r += ['--config.file=/etc/alertmanager/alertmanager.yml']
+ if daemon_type == 'node-exporter':
+ r += ['--path.procfs=/host/proc',
+ '--path.sysfs=/host/sys',
+ '--path.rootfs=/rootfs']
+ elif daemon_type == NFSGanesha.daemon_type:
+ nfs_ganesha = NFSGanesha.init(ctx, fsid, daemon_id)
+ r += nfs_ganesha.get_daemon_args()
+ elif daemon_type == HAproxy.daemon_type:
+ haproxy = HAproxy.init(ctx, fsid, daemon_id)
+ r += haproxy.get_daemon_args()
+ elif daemon_type == CustomContainer.daemon_type:
+ cc = CustomContainer.init(ctx, fsid, daemon_id)
+ r.extend(cc.get_daemon_args())
+ elif daemon_type == SNMPGateway.daemon_type:
+ sc = SNMPGateway.init(ctx, fsid, daemon_id)
+ r.extend(sc.get_daemon_args())
+
+ return r
+
+
+def create_daemon_dirs(ctx, fsid, daemon_type, daemon_id, uid, gid,
+ config=None, keyring=None):
+ # type: (CephadmContext, str, str, Union[int, str], int, int, Optional[str], Optional[str]) -> None
+ data_dir = make_data_dir(ctx, fsid, daemon_type, daemon_id, uid=uid, gid=gid)
+
+ if daemon_type in Ceph.daemons:
+ make_log_dir(ctx, fsid, uid=uid, gid=gid)
+
+ if config:
+ config_path = os.path.join(data_dir, 'config')
+ with open(config_path, 'w') as f:
+ os.fchown(f.fileno(), uid, gid)
+ os.fchmod(f.fileno(), 0o600)
+ f.write(config)
+
+ if keyring:
+ keyring_path = os.path.join(data_dir, 'keyring')
+ with open(keyring_path, 'w') as f:
+ os.fchmod(f.fileno(), 0o600)
+ os.fchown(f.fileno(), uid, gid)
+ f.write(keyring)
+
+ if daemon_type in Monitoring.components.keys():
+ config_json: Dict[str, Any] = dict()
+ if 'config_json' in ctx:
+ config_json = get_parm(ctx.config_json)
+
+ # Set up directories specific to the monitoring component
+ config_dir = ''
+ data_dir_root = ''
+ if daemon_type == 'prometheus':
+ data_dir_root = get_data_dir(fsid, ctx.data_dir,
+ daemon_type, daemon_id)
+ config_dir = 'etc/prometheus'
+ makedirs(os.path.join(data_dir_root, config_dir), uid, gid, 0o755)
+ makedirs(os.path.join(data_dir_root, config_dir, 'alerting'), uid, gid, 0o755)
+ makedirs(os.path.join(data_dir_root, 'data'), uid, gid, 0o755)
+ recursive_chown(os.path.join(data_dir_root, 'etc'), uid, gid)
+ recursive_chown(os.path.join(data_dir_root, 'data'), uid, gid)
+ elif daemon_type == 'grafana':
+ data_dir_root = get_data_dir(fsid, ctx.data_dir,
+ daemon_type, daemon_id)
+ config_dir = 'etc/grafana'
+ makedirs(os.path.join(data_dir_root, config_dir), uid, gid, 0o755)
+ makedirs(os.path.join(data_dir_root, config_dir, 'certs'), uid, gid, 0o755)
+ makedirs(os.path.join(data_dir_root, config_dir, 'provisioning/datasources'), uid, gid, 0o755)
+ makedirs(os.path.join(data_dir_root, 'data'), uid, gid, 0o755)
+ touch(os.path.join(data_dir_root, 'data', 'grafana.db'), uid, gid)
+ elif daemon_type == 'alertmanager':
+ data_dir_root = get_data_dir(fsid, ctx.data_dir,
+ daemon_type, daemon_id)
+ config_dir = 'etc/alertmanager'
+ makedirs(os.path.join(data_dir_root, config_dir), uid, gid, 0o755)
+ makedirs(os.path.join(data_dir_root, config_dir, 'data'), uid, gid, 0o755)
+
+ # populate the config directory for the component from the config-json
+ if 'files' in config_json:
+ for fname in config_json['files']:
+ content = dict_get_join(config_json['files'], fname)
+ if os.path.isabs(fname):
+ fpath = os.path.join(data_dir_root, fname.lstrip(os.path.sep))
+ else:
+ fpath = os.path.join(data_dir_root, config_dir, fname)
+ with open(fpath, 'w', encoding='utf-8') as f:
+ os.fchown(f.fileno(), uid, gid)
+ os.fchmod(f.fileno(), 0o600)
+ f.write(content)
+
+ elif daemon_type == NFSGanesha.daemon_type:
+ nfs_ganesha = NFSGanesha.init(ctx, fsid, daemon_id)
+ nfs_ganesha.create_daemon_dirs(data_dir, uid, gid)
+
+ elif daemon_type == CephIscsi.daemon_type:
+ ceph_iscsi = CephIscsi.init(ctx, fsid, daemon_id)
+ ceph_iscsi.create_daemon_dirs(data_dir, uid, gid)
+
+ elif daemon_type == HAproxy.daemon_type:
+ haproxy = HAproxy.init(ctx, fsid, daemon_id)
+ haproxy.create_daemon_dirs(data_dir, uid, gid)
+
+ elif daemon_type == Keepalived.daemon_type:
+ keepalived = Keepalived.init(ctx, fsid, daemon_id)
+ keepalived.create_daemon_dirs(data_dir, uid, gid)
+
+ elif daemon_type == CustomContainer.daemon_type:
+ cc = CustomContainer.init(ctx, fsid, daemon_id)
+ cc.create_daemon_dirs(data_dir, uid, gid)
+
+ elif daemon_type == SNMPGateway.daemon_type:
+ sg = SNMPGateway.init(ctx, fsid, daemon_id)
+ sg.create_daemon_conf()
+
+
+def get_parm(option):
+ # type: (str) -> Dict[str, str]
+
+ if not option:
+ return dict()
+
+ global cached_stdin
+ if option == '-':
+ if cached_stdin is not None:
+ j = cached_stdin
+ else:
+ j = sys.stdin.read()
+ cached_stdin = j
+ else:
+ # inline json string
+ if option[0] == '{' and option[-1] == '}':
+ j = option
+ # json file
+ elif os.path.exists(option):
+ with open(option, 'r') as f:
+ j = f.read()
+ else:
+ raise Error('Config file {} not found'.format(option))
+
+ try:
+ js = json.loads(j)
+ except ValueError as e:
+ raise Error('Invalid JSON in {}: {}'.format(option, e))
+ else:
+ return js
+
+
+def get_config_and_keyring(ctx):
+ # type: (CephadmContext) -> Tuple[Optional[str], Optional[str]]
+ config = None
+ keyring = None
+
+ if 'config_json' in ctx and ctx.config_json:
+ d = get_parm(ctx.config_json)
+ config = d.get('config')
+ keyring = d.get('keyring')
+ if config and keyring:
+ return config, keyring
+
+ if 'config' in ctx and ctx.config:
+ try:
+ with open(ctx.config, 'r') as f:
+ config = f.read()
+ except FileNotFoundError as e:
+ raise Error(e)
+
+ if 'key' in ctx and ctx.key:
+ keyring = '[%s]\n\tkey = %s\n' % (ctx.name, ctx.key)
+ elif 'keyring' in ctx and ctx.keyring:
+ try:
+ with open(ctx.keyring, 'r') as f:
+ keyring = f.read()
+ except FileNotFoundError as e:
+ raise Error(e)
+
+ return config, keyring
+
+
+def get_container_binds(ctx, fsid, daemon_type, daemon_id):
+ # type: (CephadmContext, str, str, Union[int, str, None]) -> List[List[str]]
+ binds = list()
+
+ if daemon_type == CephIscsi.daemon_type:
+ binds.extend(CephIscsi.get_container_binds())
+ elif daemon_type == CustomContainer.daemon_type:
+ assert daemon_id
+ cc = CustomContainer.init(ctx, fsid, daemon_id)
+ data_dir = get_data_dir(fsid, ctx.data_dir, daemon_type, daemon_id)
+ binds.extend(cc.get_container_binds(data_dir))
+
+ return binds
+
+
+def get_container_mounts(ctx, fsid, daemon_type, daemon_id,
+ no_config=False):
+ # type: (CephadmContext, str, str, Union[int, str, None], Optional[bool]) -> Dict[str, str]
+ mounts = dict()
+
+ if daemon_type in Ceph.daemons:
+ if fsid:
+ run_path = os.path.join('/var/run/ceph', fsid)
+ if os.path.exists(run_path):
+ mounts[run_path] = '/var/run/ceph:z'
+ log_dir = get_log_dir(fsid, ctx.log_dir)
+ mounts[log_dir] = '/var/log/ceph:z'
+ crash_dir = '/var/lib/ceph/%s/crash' % fsid
+ if os.path.exists(crash_dir):
+ mounts[crash_dir] = '/var/lib/ceph/crash:z'
+
+ if daemon_type in Ceph.daemons and daemon_id:
+ data_dir = get_data_dir(fsid, ctx.data_dir, daemon_type, daemon_id)
+ if daemon_type == 'rgw':
+ cdata_dir = '/var/lib/ceph/radosgw/ceph-rgw.%s' % (daemon_id)
+ else:
+ cdata_dir = '/var/lib/ceph/%s/ceph-%s' % (daemon_type, daemon_id)
+ if daemon_type != 'crash':
+ mounts[data_dir] = cdata_dir + ':z'
+ if not no_config:
+ mounts[data_dir + '/config'] = '/etc/ceph/ceph.conf:z'
+ if daemon_type in ['rbd-mirror', 'cephfs-mirror', 'crash']:
+ # these do not search for their keyrings in a data directory
+ mounts[data_dir + '/keyring'] = '/etc/ceph/ceph.client.%s.%s.keyring' % (daemon_type, daemon_id)
+
+ if daemon_type in ['mon', 'osd', 'clusterless-ceph-volume']:
+ mounts['/dev'] = '/dev' # FIXME: narrow this down?
+ mounts['/run/udev'] = '/run/udev'
+ if daemon_type in ['osd', 'clusterless-ceph-volume']:
+ mounts['/sys'] = '/sys' # for numa.cc, pick_address, cgroups, ...
+ mounts['/run/lvm'] = '/run/lvm'
+ mounts['/run/lock/lvm'] = '/run/lock/lvm'
+ if daemon_type == 'osd':
+ # selinux-policy in the container may not match the host.
+ if HostFacts(ctx).selinux_enabled:
+ selinux_folder = '/var/lib/ceph/%s/selinux' % fsid
+ if not os.path.exists(selinux_folder):
+ os.makedirs(selinux_folder, mode=0o755)
+ mounts[selinux_folder] = '/sys/fs/selinux:ro'
+ mounts['/'] = '/rootfs'
+
+ try:
+ if ctx.shared_ceph_folder: # make easy manager modules/ceph-volume development
+ ceph_folder = pathify(ctx.shared_ceph_folder)
+ if os.path.exists(ceph_folder):
+ mounts[ceph_folder + '/src/ceph-volume/ceph_volume'] = '/usr/lib/python3.6/site-packages/ceph_volume'
+ mounts[ceph_folder + '/src/cephadm/cephadm'] = '/usr/sbin/cephadm'
+ mounts[ceph_folder + '/src/pybind/mgr'] = '/usr/share/ceph/mgr'
+ mounts[ceph_folder + '/src/python-common/ceph'] = '/usr/lib/python3.6/site-packages/ceph'
+ mounts[ceph_folder + '/monitoring/ceph-mixin/dashboards_out'] = '/etc/grafana/dashboards/ceph-dashboard'
+ mounts[ceph_folder + '/monitoring/ceph-mixin/prometheus_alerts.yml'] = '/etc/prometheus/ceph/ceph_default_alerts.yml'
+ else:
+ logger.error('{}{}{}'.format(termcolor.red,
+ 'Ceph shared source folder does not exist.',
+ termcolor.end))
+ except AttributeError:
+ pass
+
+ if daemon_type in Monitoring.components and daemon_id:
+ data_dir = get_data_dir(fsid, ctx.data_dir, daemon_type, daemon_id)
+ if daemon_type == 'prometheus':
+ mounts[os.path.join(data_dir, 'etc/prometheus')] = '/etc/prometheus:Z'
+ mounts[os.path.join(data_dir, 'data')] = '/prometheus:Z'
+ elif daemon_type == 'node-exporter':
+ mounts['/proc'] = '/host/proc:ro'
+ mounts['/sys'] = '/host/sys:ro'
+ mounts['/'] = '/rootfs:ro'
+ elif daemon_type == 'grafana':
+ mounts[os.path.join(data_dir, 'etc/grafana/grafana.ini')] = '/etc/grafana/grafana.ini:Z'
+ mounts[os.path.join(data_dir, 'etc/grafana/provisioning/datasources')] = '/etc/grafana/provisioning/datasources:Z'
+ mounts[os.path.join(data_dir, 'etc/grafana/certs')] = '/etc/grafana/certs:Z'
+ mounts[os.path.join(data_dir, 'data/grafana.db')] = '/var/lib/grafana/grafana.db:Z'
+ elif daemon_type == 'alertmanager':
+ mounts[os.path.join(data_dir, 'etc/alertmanager')] = '/etc/alertmanager:Z'
+
+ if daemon_type == NFSGanesha.daemon_type:
+ assert daemon_id
+ data_dir = get_data_dir(fsid, ctx.data_dir, daemon_type, daemon_id)
+ nfs_ganesha = NFSGanesha.init(ctx, fsid, daemon_id)
+ mounts.update(nfs_ganesha.get_container_mounts(data_dir))
+
+ if daemon_type == HAproxy.daemon_type:
+ assert daemon_id
+ data_dir = get_data_dir(fsid, ctx.data_dir, daemon_type, daemon_id)
+ mounts.update(HAproxy.get_container_mounts(data_dir))
+
+ if daemon_type == CephIscsi.daemon_type:
+ assert daemon_id
+ data_dir = get_data_dir(fsid, ctx.data_dir, daemon_type, daemon_id)
+ log_dir = get_log_dir(fsid, ctx.log_dir)
+ mounts.update(CephIscsi.get_container_mounts(data_dir, log_dir))
+
+ if daemon_type == Keepalived.daemon_type:
+ assert daemon_id
+ data_dir = get_data_dir(fsid, ctx.data_dir, daemon_type, daemon_id)
+ mounts.update(Keepalived.get_container_mounts(data_dir))
+
+ if daemon_type == CustomContainer.daemon_type:
+ assert daemon_id
+ cc = CustomContainer.init(ctx, fsid, daemon_id)
+ data_dir = get_data_dir(fsid, ctx.data_dir, daemon_type, daemon_id)
+ mounts.update(cc.get_container_mounts(data_dir))
+
+ return mounts
+
+
+def get_ceph_volume_container(ctx: CephadmContext,
+ privileged: bool = True,
+ cname: str = '',
+ volume_mounts: Dict[str, str] = {},
+ bind_mounts: Optional[List[List[str]]] = None,
+ args: List[str] = [],
+ envs: Optional[List[str]] = None) -> 'CephContainer':
+ if envs is None:
+ envs = []
+ envs.append('CEPH_VOLUME_SKIP_RESTORECON=yes')
+ envs.append('CEPH_VOLUME_DEBUG=1')
+
+ return CephContainer(
+ ctx,
+ image=ctx.image,
+ entrypoint='/usr/sbin/ceph-volume',
+ args=args,
+ volume_mounts=volume_mounts,
+ bind_mounts=bind_mounts,
+ envs=envs,
+ privileged=privileged,
+ cname=cname,
+ memory_request=ctx.memory_request,
+ memory_limit=ctx.memory_limit,
+ )
+
+
+def get_container(ctx: CephadmContext,
+ fsid: str, daemon_type: str, daemon_id: Union[int, str],
+ privileged: bool = False,
+ ptrace: bool = False,
+ container_args: Optional[List[str]] = None) -> 'CephContainer':
+ entrypoint: str = ''
+ name: str = ''
+ ceph_args: List[str] = []
+ envs: List[str] = []
+ host_network: bool = True
+
+ if daemon_type in Ceph.daemons:
+ envs.append('TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES=134217728')
+ if container_args is None:
+ container_args = []
+ if daemon_type in ['mon', 'osd']:
+ # mon and osd need privileged in order for libudev to query devices
+ privileged = True
+ if daemon_type == 'rgw':
+ entrypoint = '/usr/bin/radosgw'
+ name = 'client.rgw.%s' % daemon_id
+ elif daemon_type == 'rbd-mirror':
+ entrypoint = '/usr/bin/rbd-mirror'
+ name = 'client.rbd-mirror.%s' % daemon_id
+ elif daemon_type == 'cephfs-mirror':
+ entrypoint = '/usr/bin/cephfs-mirror'
+ name = 'client.cephfs-mirror.%s' % daemon_id
+ elif daemon_type == 'crash':
+ entrypoint = '/usr/bin/ceph-crash'
+ name = 'client.crash.%s' % daemon_id
+ elif daemon_type in ['mon', 'mgr', 'mds', 'osd']:
+ entrypoint = '/usr/bin/ceph-' + daemon_type
+ name = '%s.%s' % (daemon_type, daemon_id)
+ elif daemon_type in Monitoring.components:
+ entrypoint = ''
+ elif daemon_type == NFSGanesha.daemon_type:
+ entrypoint = NFSGanesha.entrypoint
+ name = '%s.%s' % (daemon_type, daemon_id)
+ envs.extend(NFSGanesha.get_container_envs())
+ elif daemon_type == HAproxy.daemon_type:
+ name = '%s.%s' % (daemon_type, daemon_id)
+ container_args.extend(['--user=root']) # haproxy 2.4 defaults to a different user
+ elif daemon_type == Keepalived.daemon_type:
+ name = '%s.%s' % (daemon_type, daemon_id)
+ envs.extend(Keepalived.get_container_envs())
+ container_args.extend(['--cap-add=NET_ADMIN', '--cap-add=NET_RAW'])
+ elif daemon_type == CephIscsi.daemon_type:
+ entrypoint = CephIscsi.entrypoint
+ name = '%s.%s' % (daemon_type, daemon_id)
+ # So the container can modprobe iscsi_target_mod and have write perms
+ # to configfs we need to make this a privileged container.
+ privileged = True
+ elif daemon_type == CustomContainer.daemon_type:
+ cc = CustomContainer.init(ctx, fsid, daemon_id)
+ entrypoint = cc.entrypoint
+ host_network = False
+ envs.extend(cc.get_container_envs())
+ container_args.extend(cc.get_container_args())
+
+ if daemon_type in Monitoring.components:
+ uid, gid = extract_uid_gid_monitoring(ctx, daemon_type)
+ monitoring_args = [
+ '--user',
+ str(uid),
+ # FIXME: disable cpu/memory limits for the time being (not supported
+ # by ubuntu 18.04 kernel!)
+ ]
+ container_args.extend(monitoring_args)
+ if daemon_type == 'node-exporter':
+ # in order to support setting '--path.procfs=/host/proc','--path.sysfs=/host/sys',
+ # '--path.rootfs=/rootfs' for node-exporter we need to disable selinux separation
+ # between the node-exporter container and the host to avoid selinux denials
+ container_args.extend(['--security-opt', 'label=disable'])
+ elif daemon_type == 'crash':
+ ceph_args = ['-n', name]
+ elif daemon_type in Ceph.daemons:
+ ceph_args = ['-n', name, '-f']
+ elif daemon_type == SNMPGateway.daemon_type:
+ sg = SNMPGateway.init(ctx, fsid, daemon_id)
+ container_args.append(
+ f'--env-file={sg.conf_file_path}'
+ )
+
+ # if using podman, set -d, --conmon-pidfile & --cidfile flags
+ # so service can have Type=Forking
+ if isinstance(ctx.container_engine, Podman):
+ runtime_dir = '/run'
+ container_args.extend([
+ '-d', '--log-driver', 'journald',
+ '--conmon-pidfile',
+ runtime_dir + '/ceph-%s@%s.%s.service-pid' % (fsid, daemon_type, daemon_id),
+ '--cidfile',
+ runtime_dir + '/ceph-%s@%s.%s.service-cid' % (fsid, daemon_type, daemon_id),
+ ])
+ if ctx.container_engine.version >= CGROUPS_SPLIT_PODMAN_VERSION:
+ container_args.append('--cgroups=split')
+
+ return CephContainer.for_daemon(
+ ctx,
+ fsid=fsid,
+ daemon_type=daemon_type,
+ daemon_id=str(daemon_id),
+ entrypoint=entrypoint,
+ args=ceph_args + get_daemon_args(ctx, fsid, daemon_type, daemon_id),
+ container_args=container_args,
+ volume_mounts=get_container_mounts(ctx, fsid, daemon_type, daemon_id),
+ bind_mounts=get_container_binds(ctx, fsid, daemon_type, daemon_id),
+ envs=envs,
+ privileged=privileged,
+ ptrace=ptrace,
+ host_network=host_network,
+ )
+
+
+def extract_uid_gid(ctx, img='', file_path='/var/lib/ceph'):
+ # type: (CephadmContext, str, Union[str, List[str]]) -> Tuple[int, int]
+
+ if not img:
+ img = ctx.image
+
+ if isinstance(file_path, str):
+ paths = [file_path]
+ else:
+ paths = file_path
+
+ ex: Optional[Tuple[str, RuntimeError]] = None
+
+ for fp in paths:
+ try:
+ out = CephContainer(
+ ctx,
+ image=img,
+ entrypoint='stat',
+ args=['-c', '%u %g', fp]
+ ).run()
+ uid, gid = out.split(' ')
+ return int(uid), int(gid)
+ except RuntimeError as e:
+ ex = (fp, e)
+ if ex:
+ raise Error(f'Failed to extract uid/gid for path {ex[0]}: {ex[1]}')
+
+ raise RuntimeError('uid/gid not found')
+
+
+def deploy_daemon(ctx, fsid, daemon_type, daemon_id, c, uid, gid,
+ config=None, keyring=None,
+ osd_fsid=None,
+ reconfig=False,
+ ports=None):
+ # type: (CephadmContext, str, str, Union[int, str], Optional[CephContainer], int, int, Optional[str], Optional[str], Optional[str], Optional[bool], Optional[List[int]]) -> None
+
+ ports = ports or []
+ if any([port_in_use(ctx, port) for port in ports]):
+ if daemon_type == 'mgr':
+ # non-fatal for mgr when we are in mgr_standby_modules=false, but we can't
+ # tell whether that is the case here.
+ logger.warning(
+ f"ceph-mgr TCP port(s) {','.join(map(str, ports))} already in use"
+ )
+ else:
+ raise Error("TCP Port(s) '{}' required for {} already in use".format(','.join(map(str, ports)), daemon_type))
+
+ data_dir = get_data_dir(fsid, ctx.data_dir, daemon_type, daemon_id)
+ if reconfig and not os.path.exists(data_dir):
+ raise Error('cannot reconfig, data path %s does not exist' % data_dir)
+ if daemon_type == 'mon' and not os.path.exists(data_dir):
+ assert config
+ assert keyring
+ # tmp keyring file
+ tmp_keyring = write_tmp(keyring, uid, gid)
+
+ # tmp config file
+ tmp_config = write_tmp(config, uid, gid)
+
+ # --mkfs
+ create_daemon_dirs(ctx, fsid, daemon_type, daemon_id, uid, gid)
+ mon_dir = get_data_dir(fsid, ctx.data_dir, 'mon', daemon_id)
+ log_dir = get_log_dir(fsid, ctx.log_dir)
+ CephContainer(
+ ctx,
+ image=ctx.image,
+ entrypoint='/usr/bin/ceph-mon',
+ args=[
+ '--mkfs',
+ '-i', str(daemon_id),
+ '--fsid', fsid,
+ '-c', '/tmp/config',
+ '--keyring', '/tmp/keyring',
+ ] + get_daemon_args(ctx, fsid, 'mon', daemon_id),
+ volume_mounts={
+ log_dir: '/var/log/ceph:z',
+ mon_dir: '/var/lib/ceph/mon/ceph-%s:z' % (daemon_id),
+ tmp_keyring.name: '/tmp/keyring:z',
+ tmp_config.name: '/tmp/config:z',
+ },
+ ).run()
+
+ # write conf
+ with open(mon_dir + '/config', 'w') as f:
+ os.fchown(f.fileno(), uid, gid)
+ os.fchmod(f.fileno(), 0o600)
+ f.write(config)
+ else:
+ # dirs, conf, keyring
+ create_daemon_dirs(
+ ctx,
+ fsid, daemon_type, daemon_id,
+ uid, gid,
+ config, keyring)
+
+ if not reconfig:
+ if daemon_type == CephadmDaemon.daemon_type:
+ port = next(iter(ports), None) # get first tcp port provided or None
+
+ if ctx.config_json == '-':
+ config_js = get_parm('-')
+ else:
+ config_js = get_parm(ctx.config_json)
+ assert isinstance(config_js, dict)
+ assert isinstance(daemon_id, str)
+
+ cephadm_exporter = CephadmDaemon(ctx, fsid, daemon_id, port)
+ cephadm_exporter.deploy_daemon_unit(config_js)
+ else:
+ if c:
+ deploy_daemon_units(ctx, fsid, uid, gid, daemon_type, daemon_id,
+ c, osd_fsid=osd_fsid, ports=ports)
+ else:
+ raise RuntimeError('attempting to deploy a daemon without a container image')
+
+ if not os.path.exists(data_dir + '/unit.created'):
+ with open(data_dir + '/unit.created', 'w') as f:
+ os.fchmod(f.fileno(), 0o600)
+ os.fchown(f.fileno(), uid, gid)
+ f.write('mtime is time the daemon deployment was created\n')
+
+ with open(data_dir + '/unit.configured', 'w') as f:
+ f.write('mtime is time we were last configured\n')
+ os.fchmod(f.fileno(), 0o600)
+ os.fchown(f.fileno(), uid, gid)
+
+ update_firewalld(ctx, daemon_type)
+
+ # Open ports explicitly required for the daemon
+ if ports:
+ fw = Firewalld(ctx)
+ fw.open_ports(ports)
+ fw.apply_rules()
+
+ if reconfig and daemon_type not in Ceph.daemons:
+ # ceph daemons do not need a restart; others (presumably) do to pick
+ # up the new config
+ call_throws(ctx, ['systemctl', 'reset-failed',
+ get_unit_name(fsid, daemon_type, daemon_id)])
+ call_throws(ctx, ['systemctl', 'restart',
+ get_unit_name(fsid, daemon_type, daemon_id)])
+
+
+def _write_container_cmd_to_bash(ctx, file_obj, container, comment=None, background=False):
+ # type: (CephadmContext, IO[str], CephContainer, Optional[str], Optional[bool]) -> None
+ if comment:
+ # Sometimes adding a comment, especially if there are multiple containers in one
+ # unit file, makes it easier to read and grok.
+ file_obj.write('# ' + comment + '\n')
+ # Sometimes, adding `--rm` to a run_cmd doesn't work. Let's remove the container manually
+ file_obj.write('! ' + ' '.join(container.rm_cmd(old_cname=True)) + ' 2> /dev/null\n')
+ file_obj.write('! ' + ' '.join(container.rm_cmd()) + ' 2> /dev/null\n')
+ # Sometimes, `podman rm` doesn't find the container. Then you'll have to add `--storage`
+ if isinstance(ctx.container_engine, Podman):
+ file_obj.write(
+ '! '
+ + ' '.join([shlex.quote(a) for a in container.rm_cmd(storage=True)])
+ + ' 2> /dev/null\n')
+ file_obj.write(
+ '! '
+ + ' '.join([shlex.quote(a) for a in container.rm_cmd(old_cname=True, storage=True)])
+ + ' 2> /dev/null\n')
+
+ # container run command
+ file_obj.write(
+ ' '.join([shlex.quote(a) for a in container.run_cmd()])
+ + (' &' if background else '') + '\n')
+
+
+def clean_cgroup(ctx: CephadmContext, fsid: str, unit_name: str) -> None:
+ # systemd may fail to cleanup cgroups from previous stopped unit, which will cause next "systemctl start" to fail.
+ # see https://tracker.ceph.com/issues/50998
+
+ CGROUPV2_PATH = Path('/sys/fs/cgroup')
+ if not (CGROUPV2_PATH / 'system.slice').exists():
+ # Only unified cgroup is affected, skip if not the case
+ return
+
+ slice_name = 'system-ceph\\x2d{}.slice'.format(fsid.replace('-', '\\x2d'))
+ cg_path = CGROUPV2_PATH / 'system.slice' / slice_name / f'{unit_name}.service'
+ if not cg_path.exists():
+ return
+
+ def cg_trim(path: Path) -> None:
+ for p in path.iterdir():
+ if p.is_dir():
+ cg_trim(p)
+ path.rmdir()
+ try:
+ cg_trim(cg_path)
+ except OSError:
+ logger.warning(f'Failed to trim old cgroups {cg_path}')
+
+
+def deploy_daemon_units(
+ ctx: CephadmContext,
+ fsid: str,
+ uid: int,
+ gid: int,
+ daemon_type: str,
+ daemon_id: Union[int, str],
+ c: 'CephContainer',
+ enable: bool = True,
+ start: bool = True,
+ osd_fsid: Optional[str] = None,
+ ports: Optional[List[int]] = None,
+) -> None:
+ # cmd
+ data_dir = get_data_dir(fsid, ctx.data_dir, daemon_type, daemon_id)
+ with open(data_dir + '/unit.run.new', 'w') as f, \
+ open(data_dir + '/unit.meta.new', 'w') as metaf:
+ f.write('set -e\n')
+
+ if daemon_type in Ceph.daemons:
+ install_path = find_program('install')
+ f.write('{install_path} -d -m0770 -o {uid} -g {gid} /var/run/ceph/{fsid}\n'.format(install_path=install_path, fsid=fsid, uid=uid, gid=gid))
+
+ # pre-start cmd(s)
+ if daemon_type == 'osd':
+ # osds have a pre-start step
+ assert osd_fsid
+ simple_fn = os.path.join('/etc/ceph/osd',
+ '%s-%s.json.adopted-by-cephadm' % (daemon_id, osd_fsid))
+ if os.path.exists(simple_fn):
+ f.write('# Simple OSDs need chown on startup:\n')
+ for n in ['block', 'block.db', 'block.wal']:
+ p = os.path.join(data_dir, n)
+ f.write('[ ! -L {p} ] || chown {uid}:{gid} {p}\n'.format(p=p, uid=uid, gid=gid))
+ else:
+ # if ceph-volume does not support 'ceph-volume activate', we must
+ # do 'ceph-volume lvm activate'.
+ test_cv = get_ceph_volume_container(
+ ctx,
+ args=['activate', '--bad-option'],
+ volume_mounts=get_container_mounts(ctx, fsid, daemon_type, daemon_id),
+ bind_mounts=get_container_binds(ctx, fsid, daemon_type, daemon_id),
+ cname='ceph-%s-%s.%s-activate-test' % (fsid, daemon_type, daemon_id),
+ )
+ out, err, ret = call(ctx, test_cv.run_cmd(), verbosity=CallVerbosity.SILENT)
+ # bad: ceph-volume: error: unrecognized arguments: activate --bad-option
+ # good: ceph-volume: error: unrecognized arguments: --bad-option
+ if 'unrecognized arguments: activate' in err:
+ # older ceph-volume without top-level activate or --no-tmpfs
+ cmd = [
+ 'lvm', 'activate',
+ str(daemon_id), osd_fsid,
+ '--no-systemd',
+ ]
+ else:
+ cmd = [
+ 'activate',
+ '--osd-id', str(daemon_id),
+ '--osd-uuid', osd_fsid,
+ '--no-systemd',
+ '--no-tmpfs',
+ ]
+
+ prestart = get_ceph_volume_container(
+ ctx,
+ args=cmd,
+ volume_mounts=get_container_mounts(ctx, fsid, daemon_type, daemon_id),
+ bind_mounts=get_container_binds(ctx, fsid, daemon_type, daemon_id),
+ cname='ceph-%s-%s.%s-activate' % (fsid, daemon_type, daemon_id),
+ )
+ _write_container_cmd_to_bash(ctx, f, prestart, 'LVM OSDs use ceph-volume lvm activate')
+ elif daemon_type == CephIscsi.daemon_type:
+ f.write(' '.join(CephIscsi.configfs_mount_umount(data_dir, mount=True)) + '\n')
+ ceph_iscsi = CephIscsi.init(ctx, fsid, daemon_id)
+ tcmu_container = ceph_iscsi.get_tcmu_runner_container()
+ _write_container_cmd_to_bash(ctx, f, tcmu_container, 'iscsi tcmu-runner container', background=True)
+
+ _write_container_cmd_to_bash(ctx, f, c, '%s.%s' % (daemon_type, str(daemon_id)))
+
+ # some metadata about the deploy
+ meta: Dict[str, Any] = {}
+ if 'meta_json' in ctx and ctx.meta_json:
+ meta = json.loads(ctx.meta_json) or {}
+ meta.update({
+ 'memory_request': int(ctx.memory_request) if ctx.memory_request else None,
+ 'memory_limit': int(ctx.memory_limit) if ctx.memory_limit else None,
+ })
+ if not meta.get('ports'):
+ meta['ports'] = ports
+ metaf.write(json.dumps(meta, indent=4) + '\n')
+
+ os.fchmod(f.fileno(), 0o600)
+ os.fchmod(metaf.fileno(), 0o600)
+ os.rename(data_dir + '/unit.run.new',
+ data_dir + '/unit.run')
+ os.rename(data_dir + '/unit.meta.new',
+ data_dir + '/unit.meta')
+
+ # post-stop command(s)
+ with open(data_dir + '/unit.poststop.new', 'w') as f:
+ if daemon_type == 'osd':
+ assert osd_fsid
+ poststop = get_ceph_volume_container(
+ ctx,
+ args=[
+ 'lvm', 'deactivate',
+ str(daemon_id), osd_fsid,
+ ],
+ volume_mounts=get_container_mounts(ctx, fsid, daemon_type, daemon_id),
+ bind_mounts=get_container_binds(ctx, fsid, daemon_type, daemon_id),
+ cname='ceph-%s-%s.%s-deactivate' % (fsid, daemon_type,
+ daemon_id),
+ )
+ _write_container_cmd_to_bash(ctx, f, poststop, 'deactivate osd')
+ elif daemon_type == CephIscsi.daemon_type:
+ # make sure we also stop the tcmu container
+ ceph_iscsi = CephIscsi.init(ctx, fsid, daemon_id)
+ tcmu_container = ceph_iscsi.get_tcmu_runner_container()
+ f.write('! ' + ' '.join(tcmu_container.stop_cmd()) + '\n')
+ f.write(' '.join(CephIscsi.configfs_mount_umount(data_dir, mount=False)) + '\n')
+ os.fchmod(f.fileno(), 0o600)
+ os.rename(data_dir + '/unit.poststop.new',
+ data_dir + '/unit.poststop')
+
+ # post-stop command(s)
+ with open(data_dir + '/unit.stop.new', 'w') as f:
+ f.write('! ' + ' '.join(c.stop_cmd()) + '\n')
+ f.write('! ' + ' '.join(c.stop_cmd(old_cname=True)) + '\n')
+
+ os.fchmod(f.fileno(), 0o600)
+ os.rename(data_dir + '/unit.stop.new',
+ data_dir + '/unit.stop')
+
+ if c:
+ with open(data_dir + '/unit.image.new', 'w') as f:
+ f.write(c.image + '\n')
+ os.fchmod(f.fileno(), 0o600)
+ os.rename(data_dir + '/unit.image.new',
+ data_dir + '/unit.image')
+
+ # sysctl
+ install_sysctl(ctx, fsid, daemon_type)
+
+ # systemd
+ install_base_units(ctx, fsid)
+ unit = get_unit_file(ctx, fsid)
+ unit_file = 'ceph-%s@.service' % (fsid)
+ with open(ctx.unit_dir + '/' + unit_file + '.new', 'w') as f:
+ f.write(unit)
+ os.rename(ctx.unit_dir + '/' + unit_file + '.new',
+ ctx.unit_dir + '/' + unit_file)
+ call_throws(ctx, ['systemctl', 'daemon-reload'])
+
+ unit_name = get_unit_name(fsid, daemon_type, daemon_id)
+ call(ctx, ['systemctl', 'stop', unit_name],
+ verbosity=CallVerbosity.DEBUG)
+ call(ctx, ['systemctl', 'reset-failed', unit_name],
+ verbosity=CallVerbosity.DEBUG)
+ if enable:
+ call_throws(ctx, ['systemctl', 'enable', unit_name])
+ if start:
+ clean_cgroup(ctx, fsid, unit_name)
+ call_throws(ctx, ['systemctl', 'start', unit_name])
+
+
+class Firewalld(object):
+ def __init__(self, ctx):
+ # type: (CephadmContext) -> None
+ self.ctx = ctx
+ self.available = self.check()
+
+ def check(self):
+ # type: () -> bool
+ self.cmd = find_executable('firewall-cmd')
+ if not self.cmd:
+ logger.debug('firewalld does not appear to be present')
+ return False
+ (enabled, state, _) = check_unit(self.ctx, 'firewalld.service')
+ if not enabled:
+ logger.debug('firewalld.service is not enabled')
+ return False
+ if state != 'running':
+ logger.debug('firewalld.service is not running')
+ return False
+
+ logger.info('firewalld ready')
+ return True
+
+ def enable_service_for(self, daemon_type):
+ # type: (str) -> None
+ if not self.available:
+ logger.debug('Not possible to enable service <%s>. firewalld.service is not available' % daemon_type)
+ return
+
+ if daemon_type == 'mon':
+ svc = 'ceph-mon'
+ elif daemon_type in ['mgr', 'mds', 'osd']:
+ svc = 'ceph'
+ elif daemon_type == NFSGanesha.daemon_type:
+ svc = 'nfs'
+ else:
+ return
+
+ if not self.cmd:
+ raise RuntimeError('command not defined')
+
+ out, err, ret = call(self.ctx, [self.cmd, '--permanent', '--query-service', svc], verbosity=CallVerbosity.DEBUG)
+ if ret:
+ logger.info('Enabling firewalld service %s in current zone...' % svc)
+ out, err, ret = call(self.ctx, [self.cmd, '--permanent', '--add-service', svc])
+ if ret:
+ raise RuntimeError(
+ 'unable to add service %s to current zone: %s' % (svc, err))
+ else:
+ logger.debug('firewalld service %s is enabled in current zone' % svc)
+
+ def open_ports(self, fw_ports):
+ # type: (List[int]) -> None
+ if not self.available:
+ logger.debug('Not possible to open ports <%s>. firewalld.service is not available' % fw_ports)
+ return
+
+ if not self.cmd:
+ raise RuntimeError('command not defined')
+
+ for port in fw_ports:
+ tcp_port = str(port) + '/tcp'
+ out, err, ret = call(self.ctx, [self.cmd, '--permanent', '--query-port', tcp_port], verbosity=CallVerbosity.DEBUG)
+ if ret:
+ logger.info('Enabling firewalld port %s in current zone...' % tcp_port)
+ out, err, ret = call(self.ctx, [self.cmd, '--permanent', '--add-port', tcp_port])
+ if ret:
+ raise RuntimeError('unable to add port %s to current zone: %s' %
+ (tcp_port, err))
+ else:
+ logger.debug('firewalld port %s is enabled in current zone' % tcp_port)
+
+ def close_ports(self, fw_ports):
+ # type: (List[int]) -> None
+ if not self.available:
+ logger.debug('Not possible to close ports <%s>. firewalld.service is not available' % fw_ports)
+ return
+
+ if not self.cmd:
+ raise RuntimeError('command not defined')
+
+ for port in fw_ports:
+ tcp_port = str(port) + '/tcp'
+ out, err, ret = call(self.ctx, [self.cmd, '--permanent', '--query-port', tcp_port], verbosity=CallVerbosity.DEBUG)
+ if not ret:
+ logger.info('Disabling port %s in current zone...' % tcp_port)
+ out, err, ret = call(self.ctx, [self.cmd, '--permanent', '--remove-port', tcp_port])
+ if ret:
+ raise RuntimeError('unable to remove port %s from current zone: %s' %
+ (tcp_port, err))
+ else:
+ logger.info(f'Port {tcp_port} disabled')
+ else:
+ logger.info(f'firewalld port {tcp_port} already closed')
+
+ def apply_rules(self):
+ # type: () -> None
+ if not self.available:
+ return
+
+ if not self.cmd:
+ raise RuntimeError('command not defined')
+
+ call_throws(self.ctx, [self.cmd, '--reload'])
+
+
+def update_firewalld(ctx, daemon_type):
+ # type: (CephadmContext, str) -> None
+ if not ('skip_firewalld' in ctx and ctx.skip_firewalld):
+ firewall = Firewalld(ctx)
+ firewall.enable_service_for(daemon_type)
+ firewall.apply_rules()
+
+
+def install_sysctl(ctx: CephadmContext, fsid: str, daemon_type: str) -> None:
+ """
+ Set up sysctl settings
+ """
+ def _write(conf: Path, lines: List[str]) -> None:
+ lines = [
+ '# created by cephadm',
+ '',
+ *lines,
+ '',
+ ]
+ with open(conf, 'w') as f:
+ f.write('\n'.join(lines))
+
+ conf = Path(ctx.sysctl_dir).joinpath(f'90-ceph-{fsid}-{daemon_type}.conf')
+ lines: Optional[List] = None
+
+ if daemon_type == 'osd':
+ lines = OSD.get_sysctl_settings()
+ elif daemon_type == 'haproxy':
+ lines = HAproxy.get_sysctl_settings()
+ elif daemon_type == 'keepalived':
+ lines = Keepalived.get_sysctl_settings()
+
+ # apply the sysctl settings
+ if lines:
+ Path(ctx.sysctl_dir).mkdir(mode=0o755, exist_ok=True)
+ _write(conf, lines)
+ call_throws(ctx, ['sysctl', '--system'])
+
+
+def migrate_sysctl_dir(ctx: CephadmContext, fsid: str) -> None:
+ """
+ Cephadm once used '/usr/lib/sysctl.d' for storing sysctl configuration.
+ This moves it to '/etc/sysctl.d'.
+ """
+ deprecated_location: str = '/usr/lib/sysctl.d'
+ deprecated_confs: List[str] = glob(f'{deprecated_location}/90-ceph-{fsid}-*.conf')
+ if not deprecated_confs:
+ return
+
+ file_count: int = len(deprecated_confs)
+ logger.info(f'Found sysctl {file_count} files in deprecated location {deprecated_location}. Starting Migration.')
+ for conf in deprecated_confs:
+ try:
+ shutil.move(conf, ctx.sysctl_dir)
+ file_count -= 1
+ except shutil.Error as err:
+ if str(err).endswith('already exists'):
+ logger.warning(f'Destination file already exists. Deleting {conf}.')
+ try:
+ os.unlink(conf)
+ file_count -= 1
+ except OSError as del_err:
+ logger.warning(f'Could not remove {conf}: {del_err}.')
+ else:
+ logger.warning(f'Could not move {conf} from {deprecated_location} to {ctx.sysctl_dir}: {err}')
+
+ # Log successful migration
+ if file_count == 0:
+ logger.info(f'Successfully migrated sysctl config to {ctx.sysctl_dir}.')
+ return
+
+ # Log partially successful / unsuccessful migration
+ files_processed: int = len(deprecated_confs)
+ if file_count < files_processed:
+ status: str = f'partially successful (failed {file_count}/{files_processed})'
+ elif file_count == files_processed:
+ status = 'unsuccessful'
+ logger.warning(f'Migration of sysctl configuration {status}. You may want to perform a migration manually.')
+
+
+def install_base_units(ctx, fsid):
+ # type: (CephadmContext, str) -> None
+ """
+ Set up ceph.target and ceph-$fsid.target units.
+ """
+ # global unit
+ existed = os.path.exists(ctx.unit_dir + '/ceph.target')
+ with open(ctx.unit_dir + '/ceph.target.new', 'w') as f:
+ f.write('[Unit]\n'
+ 'Description=All Ceph clusters and services\n'
+ '\n'
+ '[Install]\n'
+ 'WantedBy=multi-user.target\n')
+ os.rename(ctx.unit_dir + '/ceph.target.new',
+ ctx.unit_dir + '/ceph.target')
+ if not existed:
+ # we disable before enable in case a different ceph.target
+ # (from the traditional package) is present; while newer
+ # systemd is smart enough to disable the old
+ # (/lib/systemd/...) and enable the new (/etc/systemd/...),
+ # some older versions of systemd error out with EEXIST.
+ call_throws(ctx, ['systemctl', 'disable', 'ceph.target'])
+ call_throws(ctx, ['systemctl', 'enable', 'ceph.target'])
+ call_throws(ctx, ['systemctl', 'start', 'ceph.target'])
+
+ # cluster unit
+ existed = os.path.exists(ctx.unit_dir + '/ceph-%s.target' % fsid)
+ with open(ctx.unit_dir + '/ceph-%s.target.new' % fsid, 'w') as f:
+ f.write(
+ '[Unit]\n'
+ 'Description=Ceph cluster {fsid}\n'
+ 'PartOf=ceph.target\n'
+ 'Before=ceph.target\n'
+ '\n'
+ '[Install]\n'
+ 'WantedBy=multi-user.target ceph.target\n'.format(
+ fsid=fsid)
+ )
+ os.rename(ctx.unit_dir + '/ceph-%s.target.new' % fsid,
+ ctx.unit_dir + '/ceph-%s.target' % fsid)
+ if not existed:
+ call_throws(ctx, ['systemctl', 'enable', 'ceph-%s.target' % fsid])
+ call_throws(ctx, ['systemctl', 'start', 'ceph-%s.target' % fsid])
+
+ # logrotate for the cluster
+ with open(ctx.logrotate_dir + '/ceph-%s' % fsid, 'w') as f:
+ """
+ This is a bit sloppy in that the killall/pkill will touch all ceph daemons
+ in all containers, but I don't see an elegant way to send SIGHUP *just* to
+ the daemons for this cluster. (1) systemd kill -s will get the signal to
+ podman, but podman will exit. (2) podman kill will get the signal to the
+ first child (bash), but that isn't the ceph daemon. This is simpler and
+ should be harmless.
+ """
+ f.write("""# created by cephadm
+/var/log/ceph/%s/*.log {
+ rotate 7
+ daily
+ compress
+ sharedscripts
+ postrotate
+ killall -q -1 ceph-mon ceph-mgr ceph-mds ceph-osd ceph-fuse radosgw rbd-mirror cephfs-mirror || pkill -1 -x 'ceph-mon|ceph-mgr|ceph-mds|ceph-osd|ceph-fuse|radosgw|rbd-mirror|cephfs-mirror' || true
+ endscript
+ missingok
+ notifempty
+ su root root
+}
+""" % fsid)
+
+
+def get_unit_file(ctx, fsid):
+ # type: (CephadmContext, str) -> str
+ extra_args = ''
+ if isinstance(ctx.container_engine, Podman):
+ extra_args = ('ExecStartPre=-/bin/rm -f %t/%n-pid %t/%n-cid\n'
+ 'ExecStopPost=-/bin/rm -f %t/%n-pid %t/%n-cid\n'
+ 'Type=forking\n'
+ 'PIDFile=%t/%n-pid\n')
+ if ctx.container_engine.version >= CGROUPS_SPLIT_PODMAN_VERSION:
+ extra_args += 'Delegate=yes\n'
+
+ docker = isinstance(ctx.container_engine, Docker)
+ u = """# generated by cephadm
+[Unit]
+Description=Ceph %i for {fsid}
+
+# According to:
+# http://www.freedesktop.org/wiki/Software/systemd/NetworkTarget
+# these can be removed once ceph-mon will dynamically change network
+# configuration.
+After=network-online.target local-fs.target time-sync.target{docker_after}
+Wants=network-online.target local-fs.target time-sync.target
+{docker_requires}
+
+PartOf=ceph-{fsid}.target
+Before=ceph-{fsid}.target
+
+[Service]
+LimitNOFILE=1048576
+LimitNPROC=1048576
+EnvironmentFile=-/etc/environment
+ExecStart=/bin/bash {data_dir}/{fsid}/%i/unit.run
+ExecStop=-/bin/bash -c '{container_path} stop ceph-{fsid}-%i ; bash {data_dir}/{fsid}/%i/unit.stop'
+ExecStopPost=-/bin/bash {data_dir}/{fsid}/%i/unit.poststop
+KillMode=none
+Restart=on-failure
+RestartSec=10s
+TimeoutStartSec=120
+TimeoutStopSec=120
+StartLimitInterval=30min
+StartLimitBurst=5
+{extra_args}
+[Install]
+WantedBy=ceph-{fsid}.target
+""".format(container_path=ctx.container_engine.path,
+ fsid=fsid,
+ data_dir=ctx.data_dir,
+ extra_args=extra_args,
+ # if docker, we depend on docker.service
+ docker_after=' docker.service' if docker else '',
+ docker_requires='Requires=docker.service\n' if docker else '')
+
+ return u
+
+##################################
+
+
+class CephContainer:
+ def __init__(self,
+ ctx: CephadmContext,
+ image: str,
+ entrypoint: str,
+ args: List[str] = [],
+ volume_mounts: Dict[str, str] = {},
+ cname: str = '',
+ container_args: List[str] = [],
+ envs: Optional[List[str]] = None,
+ privileged: bool = False,
+ ptrace: bool = False,
+ bind_mounts: Optional[List[List[str]]] = None,
+ init: Optional[bool] = None,
+ host_network: bool = True,
+ memory_request: Optional[str] = None,
+ memory_limit: Optional[str] = None,
+ ) -> None:
+ self.ctx = ctx
+ self.image = image
+ self.entrypoint = entrypoint
+ self.args = args
+ self.volume_mounts = volume_mounts
+ self._cname = cname
+ self.container_args = container_args
+ self.envs = envs
+ self.privileged = privileged
+ self.ptrace = ptrace
+ self.bind_mounts = bind_mounts if bind_mounts else []
+ self.init = init if init else ctx.container_init
+ self.host_network = host_network
+ self.memory_request = memory_request
+ self.memory_limit = memory_limit
+
+ @classmethod
+ def for_daemon(cls,
+ ctx: CephadmContext,
+ fsid: str,
+ daemon_type: str,
+ daemon_id: str,
+ entrypoint: str,
+ args: List[str] = [],
+ volume_mounts: Dict[str, str] = {},
+ container_args: List[str] = [],
+ envs: Optional[List[str]] = None,
+ privileged: bool = False,
+ ptrace: bool = False,
+ bind_mounts: Optional[List[List[str]]] = None,
+ init: Optional[bool] = None,
+ host_network: bool = True,
+ memory_request: Optional[str] = None,
+ memory_limit: Optional[str] = None,
+ ) -> 'CephContainer':
+ return cls(
+ ctx,
+ image=ctx.image,
+ entrypoint=entrypoint,
+ args=args,
+ volume_mounts=volume_mounts,
+ cname='ceph-%s-%s.%s' % (fsid, daemon_type, daemon_id),
+ container_args=container_args,
+ envs=envs,
+ privileged=privileged,
+ ptrace=ptrace,
+ bind_mounts=bind_mounts,
+ init=init,
+ host_network=host_network,
+ memory_request=memory_request,
+ memory_limit=memory_limit,
+ )
+
+ @property
+ def cname(self) -> str:
+ """
+ podman adds the current container name to the /etc/hosts
+ file. Turns out, python's `socket.getfqdn()` differs from
+ `hostname -f`, when we have the container names containing
+ dots in it.:
+
+ # podman run --name foo.bar.baz.com ceph/ceph /bin/bash
+ [root@sebastians-laptop /]# cat /etc/hosts
+ 127.0.0.1 localhost
+ ::1 localhost
+ 127.0.1.1 sebastians-laptop foo.bar.baz.com
+ [root@sebastians-laptop /]# hostname -f
+ sebastians-laptop
+ [root@sebastians-laptop /]# python3 -c 'import socket; print(socket.getfqdn())'
+ foo.bar.baz.com
+
+ Fascinatingly, this doesn't happen when using dashes.
+ """
+ return self._cname.replace('.', '-')
+
+ @cname.setter
+ def cname(self, val: str) -> None:
+ self._cname = val
+
+ @property
+ def old_cname(self) -> str:
+ return self._cname
+
+ def run_cmd(self) -> List[str]:
+ cmd_args: List[str] = [
+ str(self.ctx.container_engine.path),
+ 'run',
+ '--rm',
+ '--ipc=host',
+ # some containers (ahem, haproxy) override this, but we want a fast
+ # shutdown always (and, more importantly, a successful exit even if we
+ # fall back to SIGKILL).
+ '--stop-signal=SIGTERM',
+ ]
+
+ if isinstance(self.ctx.container_engine, Podman):
+ if os.path.exists('/etc/ceph/podman-auth.json'):
+ cmd_args.append('--authfile=/etc/ceph/podman-auth.json')
+
+ envs: List[str] = [
+ '-e', 'CONTAINER_IMAGE=%s' % self.image,
+ '-e', 'NODE_NAME=%s' % get_hostname(),
+ ]
+ vols: List[str] = []
+ binds: List[str] = []
+
+ if self.memory_request:
+ cmd_args.extend(['-e', 'POD_MEMORY_REQUEST', str(self.memory_request)])
+ if self.memory_limit:
+ cmd_args.extend(['-e', 'POD_MEMORY_LIMIT', str(self.memory_limit)])
+ cmd_args.extend(['--memory', str(self.memory_limit)])
+
+ if self.host_network:
+ cmd_args.append('--net=host')
+ if self.entrypoint:
+ cmd_args.extend(['--entrypoint', self.entrypoint])
+ if self.privileged:
+ cmd_args.extend([
+ '--privileged',
+ # let OSD etc read block devs that haven't been chowned
+ '--group-add=disk'])
+ if self.ptrace and not self.privileged:
+ # if privileged, the SYS_PTRACE cap is already added
+ # in addition, --cap-add and --privileged are mutually
+ # exclusive since podman >= 2.0
+ cmd_args.append('--cap-add=SYS_PTRACE')
+ if self.init:
+ cmd_args.append('--init')
+ envs += ['-e', 'CEPH_USE_RANDOM_NONCE=1']
+ if self.cname:
+ cmd_args.extend(['--name', self.cname])
+ if self.envs:
+ for env in self.envs:
+ envs.extend(['-e', env])
+
+ vols = sum(
+ [['-v', '%s:%s' % (host_dir, container_dir)]
+ for host_dir, container_dir in self.volume_mounts.items()], [])
+ binds = sum([['--mount', '{}'.format(','.join(bind))]
+ for bind in self.bind_mounts], [])
+
+ return \
+ cmd_args + self.container_args + \
+ envs + vols + binds + \
+ [self.image] + self.args # type: ignore
+
+ def shell_cmd(self, cmd: List[str]) -> List[str]:
+ cmd_args: List[str] = [
+ str(self.ctx.container_engine.path),
+ 'run',
+ '--rm',
+ '--ipc=host',
+ ]
+ envs: List[str] = [
+ '-e', 'CONTAINER_IMAGE=%s' % self.image,
+ '-e', 'NODE_NAME=%s' % get_hostname(),
+ ]
+ vols: List[str] = []
+ binds: List[str] = []
+
+ if self.host_network:
+ cmd_args.append('--net=host')
+ if self.ctx.no_hosts:
+ cmd_args.append('--no-hosts')
+ if self.privileged:
+ cmd_args.extend([
+ '--privileged',
+ # let OSD etc read block devs that haven't been chowned
+ '--group-add=disk',
+ ])
+ if self.init:
+ cmd_args.append('--init')
+ envs += ['-e', 'CEPH_USE_RANDOM_NONCE=1']
+ if self.envs:
+ for env in self.envs:
+ envs.extend(['-e', env])
+
+ vols = sum(
+ [['-v', '%s:%s' % (host_dir, container_dir)]
+ for host_dir, container_dir in self.volume_mounts.items()], [])
+ binds = sum([['--mount', '{}'.format(','.join(bind))]
+ for bind in self.bind_mounts], [])
+
+ return cmd_args + self.container_args + envs + vols + binds + [
+ '--entrypoint', cmd[0],
+ self.image,
+ ] + cmd[1:]
+
+ def exec_cmd(self, cmd):
+ # type: (List[str]) -> List[str]
+ cname = get_running_container_name(self.ctx, self)
+ if not cname:
+ raise Error('unable to find container "{}"'.format(self.cname))
+ return [
+ str(self.ctx.container_engine.path),
+ 'exec',
+ ] + self.container_args + [
+ self.cname,
+ ] + cmd
+
+ def rm_cmd(self, old_cname: bool = False, storage: bool = False) -> List[str]:
+ ret = [
+ str(self.ctx.container_engine.path),
+ 'rm', '-f',
+ ]
+ if storage:
+ ret.append('--storage')
+ if old_cname:
+ ret.append(self.old_cname)
+ else:
+ ret.append(self.cname)
+ return ret
+
+ def stop_cmd(self, old_cname: bool = False) -> List[str]:
+ ret = [
+ str(self.ctx.container_engine.path),
+ 'stop', self.old_cname if old_cname else self.cname,
+ ]
+ return ret
+
+ def run(self, timeout=DEFAULT_TIMEOUT):
+ # type: (Optional[int]) -> str
+ out, _, _ = call_throws(self.ctx, self.run_cmd(),
+ desc=self.entrypoint, timeout=timeout)
+ return out
+
+##################################
+
+
+@infer_image
+def command_version(ctx):
+ # type: (CephadmContext) -> int
+ c = CephContainer(ctx, ctx.image, 'ceph', ['--version'])
+ out, err, ret = call(ctx, c.run_cmd(), desc=c.entrypoint)
+ if not ret:
+ print(out.strip())
+ return ret
+
+##################################
+
+
+@default_image
+def command_pull(ctx):
+ # type: (CephadmContext) -> int
+
+ try:
+ _pull_image(ctx, ctx.image, ctx.insecure)
+ except UnauthorizedRegistryError:
+ err_str = 'Failed to pull container image. Check that host(s) are logged into the registry'
+ logger.debug(f'Pulling image for `command_pull` failed: {err_str}')
+ raise Error(err_str)
+ return command_inspect_image(ctx)
+
+
+def _pull_image(ctx, image, insecure=False):
+ # type: (CephadmContext, str, bool) -> None
+ logger.info('Pulling container image %s...' % image)
+
+ ignorelist = [
+ 'error creating read-write layer with ID',
+ 'net/http: TLS handshake timeout',
+ 'Digest did not match, expected',
+ ]
+
+ cmd = [ctx.container_engine.path, 'pull', image]
+ if isinstance(ctx.container_engine, Podman):
+ if insecure:
+ cmd.append('--tls-verify=false')
+
+ if os.path.exists('/etc/ceph/podman-auth.json'):
+ cmd.append('--authfile=/etc/ceph/podman-auth.json')
+ cmd_str = ' '.join(cmd)
+
+ for sleep_secs in [1, 4, 25]:
+ out, err, ret = call(ctx, cmd)
+ if not ret:
+ return
+
+ if 'unauthorized' in err:
+ raise UnauthorizedRegistryError()
+
+ if not any(pattern in err for pattern in ignorelist):
+ raise Error('Failed command: %s' % cmd_str)
+
+ logger.info('`%s` failed transiently. Retrying. waiting %s seconds...' % (cmd_str, sleep_secs))
+ time.sleep(sleep_secs)
+
+ raise Error('Failed command: %s: maximum retries reached' % cmd_str)
+
+##################################
+
+
+@infer_image
+def command_inspect_image(ctx):
+ # type: (CephadmContext) -> int
+ out, err, ret = call_throws(ctx, [
+ ctx.container_engine.path, 'inspect',
+ '--format', '{{.ID}},{{.RepoDigests}}',
+ ctx.image])
+ if ret:
+ return errno.ENOENT
+ info_from = get_image_info_from_inspect(out.strip(), ctx.image)
+
+ ver = CephContainer(ctx, ctx.image, 'ceph', ['--version']).run().strip()
+ info_from['ceph_version'] = ver
+
+ print(json.dumps(info_from, indent=4, sort_keys=True))
+ return 0
+
+
+def normalize_image_digest(digest: str) -> str:
+ """
+ Normal case:
+ >>> normalize_image_digest('ceph/ceph', 'docker.io')
+ 'docker.io/ceph/ceph'
+
+ No change:
+ >>> normalize_image_digest('quay.ceph.io/ceph/ceph', 'docker.io')
+ 'quay.ceph.io/ceph/ceph'
+
+ >>> normalize_image_digest('docker.io/ubuntu', 'docker.io')
+ 'docker.io/ubuntu'
+
+ >>> normalize_image_digest('localhost/ceph', 'docker.io')
+ 'localhost/ceph'
+ """
+ known_shortnames = [
+ 'ceph/ceph',
+ 'ceph/daemon',
+ 'ceph/daemon-base',
+ ]
+ for image in known_shortnames:
+ if digest.startswith(image):
+ return f'{DEFAULT_REGISTRY}/{digest}'
+ return digest
+
+
+def get_image_info_from_inspect(out, image):
+ # type: (str, str) -> Dict[str, Union[str,List[str]]]
+ image_id, digests = out.split(',', 1)
+ if not out:
+ raise Error('inspect {}: empty result'.format(image))
+ r = {
+ 'image_id': normalize_container_id(image_id)
+ } # type: Dict[str, Union[str,List[str]]]
+ if digests:
+ r['repo_digests'] = list(map(normalize_image_digest, digests[1:-1].split(' ')))
+ return r
+
+##################################
+
+
+def check_subnet(subnets: str) -> Tuple[int, List[int], str]:
+ """Determine whether the given string is a valid subnet
+
+ :param subnets: subnet string, a single definition or comma separated list of CIDR subnets
+ :returns: return code, IP version list of the subnets and msg describing any errors validation errors
+ """
+
+ rc = 0
+ versions = set()
+ errors = []
+ subnet_list = subnets.split(',')
+ for subnet in subnet_list:
+ # ensure the format of the string is as expected address/netmask
+ subnet = subnet.strip()
+ if not re.search(r'\/\d+$', subnet):
+ rc = 1
+ errors.append(f'{subnet} is not in CIDR format (address/netmask)')
+ continue
+ try:
+ v = ipaddress.ip_network(subnet).version
+ versions.add(v)
+ except ValueError as e:
+ rc = 1
+ errors.append(f'{subnet} invalid: {str(e)}')
+
+ return rc, list(versions), ', '.join(errors)
+
+
+def unwrap_ipv6(address):
+ # type: (str) -> str
+ if address.startswith('[') and address.endswith(']'):
+ return address[1:-1]
+ return address
+
+
+def wrap_ipv6(address):
+ # type: (str) -> str
+
+ # We cannot assume it's already wrapped or even an IPv6 address if
+ # it's already wrapped it'll not pass (like if it's a hostname) and trigger
+ # the ValueError
+ try:
+ if ipaddress.ip_address(address).version == 6:
+ return f'[{address}]'
+ except ValueError:
+ pass
+
+ return address
+
+
+def is_ipv6(address):
+ # type: (str) -> bool
+ address = unwrap_ipv6(address)
+ try:
+ return ipaddress.ip_address(address).version == 6
+ except ValueError:
+ logger.warning('Address: {} is not a valid IP address'.format(address))
+ return False
+
+
+def ip_in_subnets(ip_addr: str, subnets: str) -> bool:
+ """Determine if the ip_addr belongs to any of the subnets list."""
+ subnet_list = [x.strip() for x in subnets.split(',')]
+ for subnet in subnet_list:
+ ip_address = unwrap_ipv6(ip_addr) if is_ipv6(ip_addr) else ip_addr
+ if ipaddress.ip_address(ip_address) in ipaddress.ip_network(subnet):
+ return True
+ return False
+
+
+def parse_mon_addrv(addrv_arg: str) -> List[EndPoint]:
+ """Parse mon-addrv param into a list of mon end points."""
+ r = re.compile(r':(\d+)$')
+ addrv_args = []
+ addr_arg = addrv_arg
+ if addr_arg[0] != '[' or addr_arg[-1] != ']':
+ raise Error(f'--mon-addrv value {addr_arg} must use square backets')
+
+ for addr in addr_arg[1: -1].split(','):
+ hasport = r.findall(addr)
+ if not hasport:
+ raise Error(f'--mon-addrv value {addr_arg} must include port number')
+ port_str = hasport[0]
+ addr = re.sub(r'^v\d+:', '', addr) # strip off v1: or v2: prefix
+ base_ip = addr[0:-(len(port_str)) - 1]
+ addrv_args.append(EndPoint(base_ip, int(port_str)))
+
+ return addrv_args
+
+
+def parse_mon_ip(mon_ip: str) -> List[EndPoint]:
+ """Parse mon-ip param into a list of mon end points."""
+ r = re.compile(r':(\d+)$')
+ addrv_args = []
+ hasport = r.findall(mon_ip)
+ if hasport:
+ port_str = hasport[0]
+ base_ip = mon_ip[0:-(len(port_str)) - 1]
+ addrv_args.append(EndPoint(base_ip, int(port_str)))
+ else:
+ # No port provided: use fixed ports for ceph monitor
+ addrv_args.append(EndPoint(mon_ip, 3300))
+ addrv_args.append(EndPoint(mon_ip, 6789))
+
+ return addrv_args
+
+
+def build_addrv_params(addrv: List[EndPoint]) -> str:
+ """Convert mon end-points (ip:port) into the format: [v[1|2]:ip:port1]"""
+ if len(addrv) > 2:
+ raise Error('Detected a local mon-addrv list with more than 2 entries.')
+ port_to_ver: Dict[int, str] = {6789: 'v1', 3300: 'v2'}
+ addr_arg_list: List[str] = []
+ for ep in addrv:
+ if ep.port in port_to_ver:
+ ver = port_to_ver[ep.port]
+ else:
+ ver = 'v2' # default mon protocol version if port is not provided
+ logger.warning(f'Using msgr2 protocol for unrecognized port {ep}')
+ addr_arg_list.append(f'{ver}:{ep.ip}:{ep.port}')
+
+ addr_arg = '[{0}]'.format(','.join(addr_arg_list))
+ return addr_arg
+
+
+def get_public_net_from_cfg(ctx: CephadmContext) -> Optional[str]:
+ """Get mon public network from configuration file."""
+ cp = read_config(ctx.config)
+ if not cp.has_option('global', 'public_network'):
+ return None
+
+ # Ensure all public CIDR networks are valid
+ public_network = cp.get('global', 'public_network').strip('"').strip("'")
+ rc, _, err_msg = check_subnet(public_network)
+ if rc:
+ raise Error(f'Invalid public_network {public_network} parameter: {err_msg}')
+
+ # Ensure all public CIDR networks are configured locally
+ configured_subnets = set([x.strip() for x in public_network.split(',')])
+ local_subnets = set([x[0] for x in list_networks(ctx).items()])
+ valid_public_net = False
+ for net in configured_subnets:
+ if net in local_subnets:
+ valid_public_net = True
+ else:
+ logger.warning(f'The public CIDR network {net} (from -c conf file) is not configured locally.')
+ if not valid_public_net:
+ raise Error(f'None of the public CIDR network(s) {configured_subnets} (from -c conf file) is configured locally.')
+
+ # Ensure public_network is compatible with the provided mon-ip (or mon-addrv)
+ if ctx.mon_ip:
+ if not ip_in_subnets(ctx.mon_ip, public_network):
+ raise Error(f'The provided --mon-ip {ctx.mon_ip} does not belong to any public_network(s) {public_network}')
+ elif ctx.mon_addrv:
+ addrv_args = parse_mon_addrv(ctx.mon_addrv)
+ for addrv in addrv_args:
+ if not ip_in_subnets(addrv.ip, public_network):
+ raise Error(f'The provided --mon-addrv {addrv.ip} ip does not belong to any public_network(s) {public_network}')
+
+ logger.debug(f'Using mon public network from configuration file {public_network}')
+ return public_network
+
+
+def infer_mon_network(ctx: CephadmContext, mon_eps: List[EndPoint]) -> Optional[str]:
+ """Infer mon public network from local network."""
+ # Make sure IP is configured locally, and then figure out the CIDR network
+ mon_networks = []
+ for net, ifaces in list_networks(ctx).items():
+ # build local_ips list for the specified network
+ local_ips: List[str] = []
+ for _, ls in ifaces.items():
+ local_ips.extend([ipaddress.ip_address(ip) for ip in ls])
+
+ # check if any of mon ips belong to this net
+ for mon_ep in mon_eps:
+ try:
+ if ipaddress.ip_address(unwrap_ipv6(mon_ep.ip)) in local_ips:
+ mon_networks.append(net)
+ logger.info(f'Mon IP `{mon_ep.ip}` is in CIDR network `{net}`')
+ except ValueError as e:
+ logger.warning(f'Cannot infer CIDR network for mon IP `{mon_ep.ip}` : {e}')
+
+ if not mon_networks:
+ raise Error('Cannot infer CIDR network. Pass --skip-mon-network to configure it later')
+ else:
+ logger.debug(f'Inferred mon public CIDR from local network configuration {mon_networks}')
+
+ mon_networks = list(set(mon_networks)) # remove duplicates
+ return ','.join(mon_networks)
+
+
+def prepare_mon_addresses(ctx: CephadmContext) -> Tuple[str, bool, Optional[str]]:
+ """Get mon public network configuration."""
+ ipv6 = False
+ addrv_args: List[EndPoint] = []
+ mon_addrv: str = '' # i.e: [v2:192.168.100.1:3300,v1:192.168.100.1:6789]
+
+ if ctx.mon_ip:
+ ipv6 = is_ipv6(ctx.mon_ip)
+ if ipv6:
+ ctx.mon_ip = wrap_ipv6(ctx.mon_ip)
+ addrv_args = parse_mon_ip(ctx.mon_ip)
+ mon_addrv = build_addrv_params(addrv_args)
+ elif ctx.mon_addrv:
+ ipv6 = ctx.mon_addrv.count('[') > 1
+ addrv_args = parse_mon_addrv(ctx.mon_addrv)
+ mon_addrv = ctx.mon_addrv
+ else:
+ raise Error('must specify --mon-ip or --mon-addrv')
+
+ if addrv_args:
+ for end_point in addrv_args:
+ check_ip_port(ctx, end_point)
+
+ logger.debug(f'Base mon IP(s) is {addrv_args}, mon addrv is {mon_addrv}')
+ mon_network = None
+ if not ctx.skip_mon_network:
+ mon_network = get_public_net_from_cfg(ctx) or infer_mon_network(ctx, addrv_args)
+
+ return (mon_addrv, ipv6, mon_network)
+
+
+def prepare_cluster_network(ctx: CephadmContext) -> Tuple[str, bool]:
+ # the cluster network may not exist on this node, so all we can do is
+ # validate that the address given is valid ipv4 or ipv6 subnet
+ ipv6_cluster_network = False
+ cp = read_config(ctx.config)
+ cluster_network = ctx.cluster_network
+ if cluster_network is None and cp.has_option('global', 'cluster_network'):
+ cluster_network = cp.get('global', 'cluster_network').strip('"').strip("'")
+
+ if cluster_network:
+ cluser_nets = set([x.strip() for x in cluster_network.split(',')])
+ local_subnets = set([x[0] for x in list_networks(ctx).items()])
+ for net in cluser_nets:
+ if net not in local_subnets:
+ logger.warning(f'The cluster CIDR network {net} is not configured locally.')
+
+ rc, versions, err_msg = check_subnet(cluster_network)
+ if rc:
+ raise Error(f'Invalid --cluster-network parameter: {err_msg}')
+ ipv6_cluster_network = True if 6 in versions else False
+ else:
+ logger.info('Internal network (--cluster-network) has not '
+ 'been provided, OSD replication will default to '
+ 'the public_network')
+
+ return cluster_network, ipv6_cluster_network
+
+
+def create_initial_keys(
+ ctx: CephadmContext,
+ uid: int, gid: int,
+ mgr_id: str
+) -> Tuple[str, str, str, Any, Any]: # type: ignore
+
+ _image = ctx.image
+
+ # create some initial keys
+ logger.info('Creating initial keys...')
+ mon_key = CephContainer(
+ ctx,
+ image=_image,
+ entrypoint='/usr/bin/ceph-authtool',
+ args=['--gen-print-key'],
+ ).run().strip()
+ admin_key = CephContainer(
+ ctx,
+ image=_image,
+ entrypoint='/usr/bin/ceph-authtool',
+ args=['--gen-print-key'],
+ ).run().strip()
+ mgr_key = CephContainer(
+ ctx,
+ image=_image,
+ entrypoint='/usr/bin/ceph-authtool',
+ args=['--gen-print-key'],
+ ).run().strip()
+
+ keyring = ('[mon.]\n'
+ '\tkey = %s\n'
+ '\tcaps mon = allow *\n'
+ '[client.admin]\n'
+ '\tkey = %s\n'
+ '\tcaps mon = allow *\n'
+ '\tcaps mds = allow *\n'
+ '\tcaps mgr = allow *\n'
+ '\tcaps osd = allow *\n'
+ '[mgr.%s]\n'
+ '\tkey = %s\n'
+ '\tcaps mon = profile mgr\n'
+ '\tcaps mds = allow *\n'
+ '\tcaps osd = allow *\n'
+ % (mon_key, admin_key, mgr_id, mgr_key))
+
+ admin_keyring = write_tmp('[client.admin]\n'
+ '\tkey = ' + admin_key + '\n',
+ uid, gid)
+
+ # tmp keyring file
+ bootstrap_keyring = write_tmp(keyring, uid, gid)
+ return (mon_key, mgr_key, admin_key,
+ bootstrap_keyring, admin_keyring)
+
+
+def create_initial_monmap(
+ ctx: CephadmContext,
+ uid: int, gid: int,
+ fsid: str,
+ mon_id: str, mon_addr: str
+) -> Any:
+ logger.info('Creating initial monmap...')
+ monmap = write_tmp('', 0, 0)
+ out = CephContainer(
+ ctx,
+ image=ctx.image,
+ entrypoint='/usr/bin/monmaptool',
+ args=[
+ '--create',
+ '--clobber',
+ '--fsid', fsid,
+ '--addv', mon_id, mon_addr,
+ '/tmp/monmap'
+ ],
+ volume_mounts={
+ monmap.name: '/tmp/monmap:z',
+ },
+ ).run()
+ logger.debug(f'monmaptool for {mon_id} {mon_addr} on {out}')
+
+ # pass monmap file to ceph user for use by ceph-mon --mkfs below
+ os.fchown(monmap.fileno(), uid, gid)
+ return monmap
+
+
+def prepare_create_mon(
+ ctx: CephadmContext,
+ uid: int, gid: int,
+ fsid: str, mon_id: str,
+ bootstrap_keyring_path: str,
+ monmap_path: str
+) -> Tuple[str, str]:
+ logger.info('Creating mon...')
+ create_daemon_dirs(ctx, fsid, 'mon', mon_id, uid, gid)
+ mon_dir = get_data_dir(fsid, ctx.data_dir, 'mon', mon_id)
+ log_dir = get_log_dir(fsid, ctx.log_dir)
+ out = CephContainer(
+ ctx,
+ image=ctx.image,
+ entrypoint='/usr/bin/ceph-mon',
+ args=[
+ '--mkfs',
+ '-i', mon_id,
+ '--fsid', fsid,
+ '-c', '/dev/null',
+ '--monmap', '/tmp/monmap',
+ '--keyring', '/tmp/keyring',
+ ] + get_daemon_args(ctx, fsid, 'mon', mon_id),
+ volume_mounts={
+ log_dir: '/var/log/ceph:z',
+ mon_dir: '/var/lib/ceph/mon/ceph-%s:z' % (mon_id),
+ bootstrap_keyring_path: '/tmp/keyring:z',
+ monmap_path: '/tmp/monmap:z',
+ },
+ ).run()
+ logger.debug(f'create mon.{mon_id} on {out}')
+ return (mon_dir, log_dir)
+
+
+def create_mon(
+ ctx: CephadmContext,
+ uid: int, gid: int,
+ fsid: str, mon_id: str
+) -> None:
+ mon_c = get_container(ctx, fsid, 'mon', mon_id)
+ ctx.meta_json = json.dumps({'service_name': 'mon'})
+ deploy_daemon(ctx, fsid, 'mon', mon_id, mon_c, uid, gid,
+ config=None, keyring=None)
+
+
+def wait_for_mon(
+ ctx: CephadmContext,
+ mon_id: str, mon_dir: str,
+ admin_keyring_path: str, config_path: str
+) -> None:
+ logger.info('Waiting for mon to start...')
+ c = CephContainer(
+ ctx,
+ image=ctx.image,
+ entrypoint='/usr/bin/ceph',
+ args=[
+ 'status'],
+ volume_mounts={
+ mon_dir: '/var/lib/ceph/mon/ceph-%s:z' % (mon_id),
+ admin_keyring_path: '/etc/ceph/ceph.client.admin.keyring:z',
+ config_path: '/etc/ceph/ceph.conf:z',
+ },
+ )
+
+ # wait for the service to become available
+ def is_mon_available():
+ # type: () -> bool
+ timeout = ctx.timeout if ctx.timeout else 60 # seconds
+ out, err, ret = call(ctx, c.run_cmd(),
+ desc=c.entrypoint,
+ timeout=timeout)
+ return ret == 0
+
+ is_available(ctx, 'mon', is_mon_available)
+
+
+def create_mgr(
+ ctx: CephadmContext,
+ uid: int, gid: int,
+ fsid: str, mgr_id: str, mgr_key: str,
+ config: str, clifunc: Callable
+) -> None:
+ logger.info('Creating mgr...')
+ mgr_keyring = '[mgr.%s]\n\tkey = %s\n' % (mgr_id, mgr_key)
+ mgr_c = get_container(ctx, fsid, 'mgr', mgr_id)
+ # Note:the default port used by the Prometheus node exporter is opened in fw
+ ctx.meta_json = json.dumps({'service_name': 'mgr'})
+ deploy_daemon(ctx, fsid, 'mgr', mgr_id, mgr_c, uid, gid,
+ config=config, keyring=mgr_keyring, ports=[9283])
+
+ # wait for the service to become available
+ logger.info('Waiting for mgr to start...')
+
+ def is_mgr_available():
+ # type: () -> bool
+ timeout = ctx.timeout if ctx.timeout else 60 # seconds
+ try:
+ out = clifunc(['status', '-f', 'json-pretty'], timeout=timeout)
+ j = json.loads(out)
+ return j.get('mgrmap', {}).get('available', False)
+ except Exception as e:
+ logger.debug('status failed: %s' % e)
+ return False
+ is_available(ctx, 'mgr', is_mgr_available)
+
+
+def prepare_ssh(
+ ctx: CephadmContext,
+ cli: Callable, wait_for_mgr_restart: Callable
+) -> None:
+
+ cli(['cephadm', 'set-user', ctx.ssh_user])
+
+ if ctx.ssh_config:
+ logger.info('Using provided ssh config...')
+ mounts = {
+ pathify(ctx.ssh_config.name): '/tmp/cephadm-ssh-config:z',
+ }
+ cli(['cephadm', 'set-ssh-config', '-i', '/tmp/cephadm-ssh-config'], extra_mounts=mounts)
+
+ if ctx.ssh_private_key and ctx.ssh_public_key:
+ logger.info('Using provided ssh keys...')
+ mounts = {
+ pathify(ctx.ssh_private_key.name): '/tmp/cephadm-ssh-key:z',
+ pathify(ctx.ssh_public_key.name): '/tmp/cephadm-ssh-key.pub:z'
+ }
+ cli(['cephadm', 'set-priv-key', '-i', '/tmp/cephadm-ssh-key'], extra_mounts=mounts)
+ cli(['cephadm', 'set-pub-key', '-i', '/tmp/cephadm-ssh-key.pub'], extra_mounts=mounts)
+ else:
+ logger.info('Generating ssh key...')
+ cli(['cephadm', 'generate-key'])
+ ssh_pub = cli(['cephadm', 'get-pub-key'])
+
+ with open(ctx.output_pub_ssh_key, 'w') as f:
+ f.write(ssh_pub)
+ logger.info('Wrote public SSH key to %s' % ctx.output_pub_ssh_key)
+
+ logger.info('Adding key to %s@localhost authorized_keys...' % ctx.ssh_user)
+ try:
+ s_pwd = pwd.getpwnam(ctx.ssh_user)
+ except KeyError:
+ raise Error('Cannot find uid/gid for ssh-user: %s' % (ctx.ssh_user))
+ ssh_uid = s_pwd.pw_uid
+ ssh_gid = s_pwd.pw_gid
+ ssh_dir = os.path.join(s_pwd.pw_dir, '.ssh')
+
+ if not os.path.exists(ssh_dir):
+ makedirs(ssh_dir, ssh_uid, ssh_gid, 0o700)
+
+ auth_keys_file = '%s/authorized_keys' % ssh_dir
+ add_newline = False
+
+ if os.path.exists(auth_keys_file):
+ with open(auth_keys_file, 'r') as f:
+ f.seek(0, os.SEEK_END)
+ if f.tell() > 0:
+ f.seek(f.tell() - 1, os.SEEK_SET) # go to last char
+ if f.read() != '\n':
+ add_newline = True
+
+ with open(auth_keys_file, 'a') as f:
+ os.fchown(f.fileno(), ssh_uid, ssh_gid) # just in case we created it
+ os.fchmod(f.fileno(), 0o600) # just in case we created it
+ if add_newline:
+ f.write('\n')
+ f.write(ssh_pub.strip() + '\n')
+
+ host = get_hostname()
+ logger.info('Adding host %s...' % host)
+ try:
+ args = ['orch', 'host', 'add', host]
+ if ctx.mon_ip:
+ args.append(unwrap_ipv6(ctx.mon_ip))
+ elif ctx.mon_addrv:
+ addrv_args = parse_mon_addrv(ctx.mon_addrv)
+ args.append(unwrap_ipv6(addrv_args[0].ip))
+ cli(args)
+ except RuntimeError as e:
+ raise Error('Failed to add host <%s>: %s' % (host, e))
+
+ for t in ['mon', 'mgr']:
+ if not ctx.orphan_initial_daemons:
+ logger.info('Deploying %s service with default placement...' % t)
+ cli(['orch', 'apply', t])
+ else:
+ logger.info('Deploying unmanaged %s service...' % t)
+ cli(['orch', 'apply', t, '--unmanaged'])
+
+ if not ctx.orphan_initial_daemons:
+ logger.info('Deploying crash service with default placement...')
+ cli(['orch', 'apply', 'crash'])
+
+ if not ctx.skip_monitoring_stack:
+ for t in ['prometheus', 'grafana', 'node-exporter', 'alertmanager']:
+ logger.info('Deploying %s service with default placement...' % t)
+ cli(['orch', 'apply', t])
+
+
+def enable_cephadm_mgr_module(
+ cli: Callable, wait_for_mgr_restart: Callable
+) -> None:
+
+ logger.info('Enabling cephadm module...')
+ cli(['mgr', 'module', 'enable', 'cephadm'])
+ wait_for_mgr_restart()
+ logger.info('Setting orchestrator backend to cephadm...')
+ cli(['orch', 'set', 'backend', 'cephadm'])
+
+
+def prepare_dashboard(
+ ctx: CephadmContext,
+ uid: int, gid: int,
+ cli: Callable, wait_for_mgr_restart: Callable
+) -> None:
+
+ # Configure SSL port (cephadm only allows to configure dashboard SSL port)
+ # if the user does not want to use SSL he can change this setting once the cluster is up
+ cli(['config', 'set', 'mgr', 'mgr/dashboard/ssl_server_port', str(ctx.ssl_dashboard_port)])
+
+ # configuring dashboard parameters
+ logger.info('Enabling the dashboard module...')
+ cli(['mgr', 'module', 'enable', 'dashboard'])
+ wait_for_mgr_restart()
+
+ # dashboard crt and key
+ if ctx.dashboard_key and ctx.dashboard_crt:
+ logger.info('Using provided dashboard certificate...')
+ mounts = {
+ pathify(ctx.dashboard_crt.name): '/tmp/dashboard.crt:z',
+ pathify(ctx.dashboard_key.name): '/tmp/dashboard.key:z'
+ }
+ cli(['dashboard', 'set-ssl-certificate', '-i', '/tmp/dashboard.crt'], extra_mounts=mounts)
+ cli(['dashboard', 'set-ssl-certificate-key', '-i', '/tmp/dashboard.key'], extra_mounts=mounts)
+ else:
+ logger.info('Generating a dashboard self-signed certificate...')
+ cli(['dashboard', 'create-self-signed-cert'])
+
+ logger.info('Creating initial admin user...')
+ password = ctx.initial_dashboard_password or generate_password()
+ tmp_password_file = write_tmp(password, uid, gid)
+ cmd = ['dashboard', 'ac-user-create', ctx.initial_dashboard_user, '-i', '/tmp/dashboard.pw', 'administrator', '--force-password']
+ if not ctx.dashboard_password_noupdate:
+ cmd.append('--pwd-update-required')
+ cli(cmd, extra_mounts={pathify(tmp_password_file.name): '/tmp/dashboard.pw:z'})
+ logger.info('Fetching dashboard port number...')
+ out = cli(['config', 'get', 'mgr', 'mgr/dashboard/ssl_server_port'])
+ port = int(out)
+
+ # Open dashboard port
+ if not ('skip_firewalld' in ctx and ctx.skip_firewalld):
+ fw = Firewalld(ctx)
+ fw.open_ports([port])
+ fw.apply_rules()
+
+ logger.info('Ceph Dashboard is now available at:\n\n'
+ '\t URL: https://%s:%s/\n'
+ '\t User: %s\n'
+ '\tPassword: %s\n' % (
+ get_fqdn(), port,
+ ctx.initial_dashboard_user,
+ password))
+
+
+def prepare_bootstrap_config(
+ ctx: CephadmContext,
+ fsid: str, mon_addr: str, image: str
+
+) -> str:
+
+ cp = read_config(ctx.config)
+ if not cp.has_section('global'):
+ cp.add_section('global')
+ cp.set('global', 'fsid', fsid)
+ cp.set('global', 'mon_host', mon_addr)
+ cp.set('global', 'container_image', image)
+
+ if not cp.has_section('mon'):
+ cp.add_section('mon')
+ if (
+ not cp.has_option('mon', 'auth_allow_insecure_global_id_reclaim')
+ and not cp.has_option('mon', 'auth allow insecure global id reclaim')
+ ):
+ cp.set('mon', 'auth_allow_insecure_global_id_reclaim', 'false')
+
+ if ctx.single_host_defaults:
+ logger.info('Adjusting default settings to suit single-host cluster...')
+ # replicate across osds, not hosts
+ if (
+ not cp.has_option('global', 'osd_crush_chooseleaf_type')
+ and not cp.has_option('global', 'osd crush chooseleaf type')
+ ):
+ cp.set('global', 'osd_crush_chooseleaf_type', '0')
+ # replica 2x
+ if (
+ not cp.has_option('global', 'osd_pool_default_size')
+ and not cp.has_option('global', 'osd pool default size')
+ ):
+ cp.set('global', 'osd_pool_default_size', '2')
+ # disable mgr standby modules (so we can colocate multiple mgrs on one host)
+ if not cp.has_section('mgr'):
+ cp.add_section('mgr')
+ if (
+ not cp.has_option('mgr', 'mgr_standby_modules')
+ and not cp.has_option('mgr', 'mgr standby modules')
+ ):
+ cp.set('mgr', 'mgr_standby_modules', 'false')
+ if ctx.log_to_file:
+ cp.set('global', 'log_to_file', 'true')
+ cp.set('global', 'log_to_stderr', 'false')
+ cp.set('global', 'log_to_journald', 'false')
+ cp.set('global', 'mon_cluster_log_to_file', 'true')
+ cp.set('global', 'mon_cluster_log_to_stderr', 'false')
+ cp.set('global', 'mon_cluster_log_to_journald', 'false')
+
+ cpf = StringIO()
+ cp.write(cpf)
+ config = cpf.getvalue()
+
+ if ctx.registry_json or ctx.registry_url:
+ command_registry_login(ctx)
+
+ return config
+
+
+def finish_bootstrap_config(
+ ctx: CephadmContext,
+ fsid: str,
+ config: str,
+ mon_id: str, mon_dir: str,
+ mon_network: Optional[str], ipv6: bool,
+ cli: Callable,
+ cluster_network: Optional[str], ipv6_cluster_network: bool
+
+) -> None:
+ if not ctx.no_minimize_config:
+ logger.info('Assimilating anything we can from ceph.conf...')
+ cli([
+ 'config', 'assimilate-conf',
+ '-i', '/var/lib/ceph/mon/ceph-%s/config' % mon_id
+ ], {
+ mon_dir: '/var/lib/ceph/mon/ceph-%s:z' % mon_id
+ })
+ logger.info('Generating new minimal ceph.conf...')
+ cli([
+ 'config', 'generate-minimal-conf',
+ '-o', '/var/lib/ceph/mon/ceph-%s/config' % mon_id
+ ], {
+ mon_dir: '/var/lib/ceph/mon/ceph-%s:z' % mon_id
+ })
+ # re-read our minimized config
+ with open(mon_dir + '/config', 'r') as f:
+ config = f.read()
+ logger.info('Restarting the monitor...')
+ call_throws(ctx, [
+ 'systemctl',
+ 'restart',
+ get_unit_name(fsid, 'mon', mon_id)
+ ])
+ elif 'image' in ctx and ctx.image:
+ # we still want to assimilate the given container image if provided
+ cli(['config', 'set', 'global', 'container_image', f'{ctx.image}'])
+
+ if mon_network:
+ logger.info(f'Setting mon public_network to {mon_network}')
+ cli(['config', 'set', 'mon', 'public_network', mon_network])
+
+ if cluster_network:
+ logger.info(f'Setting cluster_network to {cluster_network}')
+ cli(['config', 'set', 'global', 'cluster_network', cluster_network])
+
+ if ipv6 or ipv6_cluster_network:
+ logger.info('Enabling IPv6 (ms_bind_ipv6) binding')
+ cli(['config', 'set', 'global', 'ms_bind_ipv6', 'true'])
+
+ with open(ctx.output_config, 'w') as f:
+ f.write(config)
+ logger.info('Wrote config to %s' % ctx.output_config)
+ pass
+
+
+# funcs to process spec file for apply spec
+def _parse_yaml_docs(f: Iterable[str]) -> List[List[str]]:
+ docs = []
+ current_doc = [] # type: List[str]
+ for line in f:
+ if re.search(r'^---\s+', line):
+ if current_doc:
+ docs.append(current_doc)
+ current_doc = []
+ else:
+ current_doc.append(line.rstrip())
+ if current_doc:
+ docs.append(current_doc)
+ return docs
+
+
+def _parse_yaml_obj(doc: List[str]) -> Dict[str, str]:
+ # note: this only parses the first layer of yaml
+ obj = {} # type: Dict[str, str]
+ current_key = ''
+ for line in doc:
+ if line.startswith(' '):
+ obj[current_key] += line.strip()
+ elif line.endswith(':'):
+ current_key = line.strip(':')
+ obj[current_key] = ''
+ else:
+ current_key, val = line.split(':')
+ obj[current_key] = val.strip()
+ return obj
+
+
+def parse_yaml_objs(f: Iterable[str]) -> List[Dict[str, str]]:
+ objs = []
+ for d in _parse_yaml_docs(f):
+ objs.append(_parse_yaml_obj(d))
+ return objs
+
+
+def _distribute_ssh_keys(ctx: CephadmContext, host_spec: Dict[str, str], bootstrap_hostname: str) -> int:
+ # copy ssh key to hosts in host spec (used for apply spec)
+ ssh_key = '/etc/ceph/ceph.pub'
+ if ctx.ssh_public_key:
+ ssh_key = ctx.ssh_public_key.name
+
+ if bootstrap_hostname != host_spec['hostname']:
+ if 'addr' in host_spec:
+ addr = host_spec['addr']
+ else:
+ addr = host_spec['hostname']
+ out, err, code = call(ctx, ['sudo', '-u', ctx.ssh_user, 'ssh-copy-id', '-f', '-i', ssh_key, '-o StrictHostKeyChecking=no', '%s@%s' % (ctx.ssh_user, addr)])
+ if code:
+ logger.info('\nCopying ssh key to host %s at address %s failed!\n' % (host_spec['hostname'], addr))
+ return 1
+ else:
+ logger.info('Added ssh key to host %s at address %s\n' % (host_spec['hostname'], addr))
+ return 0
+
+
+@default_image
+def command_bootstrap(ctx):
+ # type: (CephadmContext) -> int
+
+ ctx.error_code = 0
+
+ if not ctx.output_config:
+ ctx.output_config = os.path.join(ctx.output_dir, 'ceph.conf')
+ if not ctx.output_keyring:
+ ctx.output_keyring = os.path.join(ctx.output_dir,
+ 'ceph.client.admin.keyring')
+ if not ctx.output_pub_ssh_key:
+ ctx.output_pub_ssh_key = os.path.join(ctx.output_dir, 'ceph.pub')
+
+ # verify output files
+ for f in [ctx.output_config, ctx.output_keyring,
+ ctx.output_pub_ssh_key]:
+ if not ctx.allow_overwrite:
+ if os.path.exists(f):
+ raise Error('%s already exists; delete or pass '
+ '--allow-overwrite to overwrite' % f)
+ dirname = os.path.dirname(f)
+ if dirname and not os.path.exists(dirname):
+ fname = os.path.basename(f)
+ logger.info(f'Creating directory {dirname} for {fname}')
+ try:
+ # use makedirs to create intermediate missing dirs
+ os.makedirs(dirname, 0o755)
+ except PermissionError:
+ raise Error(f'Unable to create {dirname} due to permissions failure. Retry with root, or sudo or preallocate the directory.')
+
+ (user_conf, _) = get_config_and_keyring(ctx)
+
+ if not ctx.skip_prepare_host:
+ command_prepare_host(ctx)
+ else:
+ logger.info('Skip prepare_host')
+
+ # initial vars
+ fsid = ctx.fsid or make_fsid()
+ if not is_fsid(fsid):
+ raise Error('not an fsid: %s' % fsid)
+ logger.info('Cluster fsid: %s' % fsid)
+
+ hostname = get_hostname()
+ if '.' in hostname and not ctx.allow_fqdn_hostname:
+ raise Error('hostname is a fully qualified domain name (%s); either fix (e.g., "sudo hostname %s" or similar) or pass --allow-fqdn-hostname' % (hostname, hostname.split('.')[0]))
+ mon_id = ctx.mon_id or hostname
+ mgr_id = ctx.mgr_id or generate_service_id()
+
+ lock = FileLock(ctx, fsid)
+ lock.acquire()
+
+ (addr_arg, ipv6, mon_network) = prepare_mon_addresses(ctx)
+ cluster_network, ipv6_cluster_network = prepare_cluster_network(ctx)
+
+ config = prepare_bootstrap_config(ctx, fsid, addr_arg, ctx.image)
+
+ if not ctx.skip_pull:
+ try:
+ _pull_image(ctx, ctx.image)
+ except UnauthorizedRegistryError:
+ err_str = 'Failed to pull container image. Check that correct registry credentials are provided in bootstrap by --registry-url, --registry-username, --registry-password, or supply --registry-json with credentials'
+ logger.debug(f'Pulling image for bootstrap on {hostname} failed: {err_str}')
+ raise Error(err_str)
+
+ image_ver = CephContainer(ctx, ctx.image, 'ceph', ['--version']).run().strip()
+ logger.info(f'Ceph version: {image_ver}')
+
+ if not ctx.allow_mismatched_release:
+ image_release = image_ver.split()[4]
+ if image_release not in \
+ [DEFAULT_IMAGE_RELEASE, LATEST_STABLE_RELEASE]:
+ raise Error(
+ f'Container release {image_release} != cephadm release {DEFAULT_IMAGE_RELEASE};'
+ ' please use matching version of cephadm (pass --allow-mismatched-release to continue anyway)'
+ )
+
+ logger.info('Extracting ceph user uid/gid from container image...')
+ (uid, gid) = extract_uid_gid(ctx)
+
+ # create some initial keys
+ (mon_key, mgr_key, admin_key, bootstrap_keyring, admin_keyring) = \
+ create_initial_keys(ctx, uid, gid, mgr_id)
+
+ monmap = create_initial_monmap(ctx, uid, gid, fsid, mon_id, addr_arg)
+ (mon_dir, log_dir) = \
+ prepare_create_mon(ctx, uid, gid, fsid, mon_id,
+ bootstrap_keyring.name, monmap.name)
+
+ with open(mon_dir + '/config', 'w') as f:
+ os.fchown(f.fileno(), uid, gid)
+ os.fchmod(f.fileno(), 0o600)
+ f.write(config)
+
+ make_var_run(ctx, fsid, uid, gid)
+ create_mon(ctx, uid, gid, fsid, mon_id)
+
+ # config to issue various CLI commands
+ tmp_config = write_tmp(config, uid, gid)
+
+ # a CLI helper to reduce our typing
+ def cli(cmd, extra_mounts={}, timeout=DEFAULT_TIMEOUT):
+ # type: (List[str], Dict[str, str], Optional[int]) -> str
+ mounts = {
+ log_dir: '/var/log/ceph:z',
+ admin_keyring.name: '/etc/ceph/ceph.client.admin.keyring:z',
+ tmp_config.name: '/etc/ceph/ceph.conf:z',
+ }
+ for k, v in extra_mounts.items():
+ mounts[k] = v
+ timeout = timeout or ctx.timeout
+ return CephContainer(
+ ctx,
+ image=ctx.image,
+ entrypoint='/usr/bin/ceph',
+ args=cmd,
+ volume_mounts=mounts,
+ ).run(timeout=timeout)
+
+ wait_for_mon(ctx, mon_id, mon_dir, admin_keyring.name, tmp_config.name)
+
+ finish_bootstrap_config(ctx, fsid, config, mon_id, mon_dir,
+ mon_network, ipv6, cli,
+ cluster_network, ipv6_cluster_network)
+
+ # output files
+ with open(ctx.output_keyring, 'w') as f:
+ os.fchmod(f.fileno(), 0o600)
+ f.write('[client.admin]\n'
+ '\tkey = ' + admin_key + '\n')
+ logger.info('Wrote keyring to %s' % ctx.output_keyring)
+
+ # create mgr
+ create_mgr(ctx, uid, gid, fsid, mgr_id, mgr_key, config, cli)
+
+ if user_conf:
+ # user given config settings were already assimilated earlier
+ # but if the given settings contained any attributes in
+ # the mgr (e.g. mgr/cephadm/container_image_prometheus)
+ # they don't seem to be stored if there isn't a mgr yet.
+ # Since re-assimilating the same conf settings should be
+ # idempotent we can just do it again here.
+ with tempfile.NamedTemporaryFile(buffering=0) as tmp:
+ tmp.write(user_conf.encode('utf-8'))
+ cli(['config', 'assimilate-conf',
+ '-i', '/var/lib/ceph/user.conf'],
+ {tmp.name: '/var/lib/ceph/user.conf:z'})
+
+ # wait for mgr to restart (after enabling a module)
+ def wait_for_mgr_restart() -> None:
+ # first get latest mgrmap epoch from the mon. try newer 'mgr
+ # stat' command first, then fall back to 'mgr dump' if
+ # necessary
+ try:
+ j = json_loads_retry(lambda: cli(['mgr', 'stat']))
+ except Exception:
+ j = json_loads_retry(lambda: cli(['mgr', 'dump']))
+ epoch = j['epoch']
+
+ # wait for mgr to have it
+ logger.info('Waiting for the mgr to restart...')
+
+ def mgr_has_latest_epoch():
+ # type: () -> bool
+ try:
+ out = cli(['tell', 'mgr', 'mgr_status'])
+ j = json.loads(out)
+ return j['mgrmap_epoch'] >= epoch
+ except Exception as e:
+ logger.debug('tell mgr mgr_status failed: %s' % e)
+ return False
+ is_available(ctx, 'mgr epoch %d' % epoch, mgr_has_latest_epoch)
+
+ enable_cephadm_mgr_module(cli, wait_for_mgr_restart)
+
+ # ssh
+ if not ctx.skip_ssh:
+ prepare_ssh(ctx, cli, wait_for_mgr_restart)
+
+ if ctx.registry_url and ctx.registry_username and ctx.registry_password:
+ registry_credentials = {'url': ctx.registry_url, 'username': ctx.registry_username, 'password': ctx.registry_password}
+ cli(['config-key', 'set', 'mgr/cephadm/registry_credentials', json.dumps(registry_credentials)])
+
+ cli(['config', 'set', 'mgr', 'mgr/cephadm/container_init', str(ctx.container_init), '--force'])
+
+ if ctx.with_exporter:
+ cli(['config-key', 'set', 'mgr/cephadm/exporter_enabled', 'true'])
+ if ctx.exporter_config:
+ logger.info('Applying custom cephadm exporter settings')
+ # validated within the parser, so we can just apply to the store
+ with tempfile.NamedTemporaryFile(buffering=0) as tmp:
+ tmp.write(json.dumps(ctx.exporter_config).encode('utf-8'))
+ mounts = {
+ tmp.name: '/tmp/exporter-config.json:z'
+ }
+ cli(['cephadm', 'set-exporter-config', '-i', '/tmp/exporter-config.json'], extra_mounts=mounts)
+ logger.info('-> Use ceph orch apply cephadm-exporter to deploy')
+ else:
+ # generate a default SSL configuration for the exporter(s)
+ logger.info('Generating a default cephadm exporter configuration (self-signed)')
+ cli(['cephadm', 'generate-exporter-config'])
+ #
+ # deploy the service (commented out until the cephadm changes are in the ceph container build)
+ logger.info('Deploying cephadm exporter service with default placement...')
+ cli(['orch', 'apply', 'cephadm-exporter'])
+
+ if not ctx.skip_dashboard:
+ prepare_dashboard(ctx, uid, gid, cli, wait_for_mgr_restart)
+
+ if ctx.output_config == '/etc/ceph/ceph.conf' and not ctx.skip_admin_label and not ctx.no_minimize_config:
+ logger.info('Enabling client.admin keyring and conf on hosts with "admin" label')
+ try:
+ cli(['orch', 'client-keyring', 'set', 'client.admin', 'label:_admin'])
+ cli(['orch', 'host', 'label', 'add', get_hostname(), '_admin'])
+ except Exception:
+ logger.info('Unable to set up "admin" label; assuming older version of Ceph')
+
+ if ctx.apply_spec:
+ logger.info('Applying %s to cluster' % ctx.apply_spec)
+ # copy ssh key to hosts in spec file
+ with open(ctx.apply_spec) as f:
+ try:
+ for spec in parse_yaml_objs(f):
+ if spec.get('service_type') == 'host':
+ _distribute_ssh_keys(ctx, spec, hostname)
+ except ValueError:
+ logger.info('Unable to parse %s succesfully' % ctx.apply_spec)
+
+ mounts = {}
+ mounts[pathify(ctx.apply_spec)] = '/tmp/spec.yml:ro'
+ try:
+ out = cli(['orch', 'apply', '-i', '/tmp/spec.yml'], extra_mounts=mounts)
+ logger.info(out)
+ except Exception:
+ ctx.error_code = -errno.EINVAL
+ logger.info('\nApplying %s to cluster failed!\n' % ctx.apply_spec)
+
+ # enable autotune for osd_memory_target
+ logger.info('Enabling autotune for osd_memory_target')
+ cli(['config', 'set', 'osd', 'osd_memory_target_autotune', 'true'])
+
+ # Notify the Dashboard to show the 'Expand cluster' page on first log in.
+ cli(['config-key', 'set', 'mgr/dashboard/cluster/status', 'INSTALLED'])
+
+ logger.info('You can access the Ceph CLI as following in case of multi-cluster or non-default config:\n\n'
+ '\tsudo %s shell --fsid %s -c %s -k %s\n' % (
+ sys.argv[0],
+ fsid,
+ ctx.output_config,
+ ctx.output_keyring))
+
+ logger.info('Or, if you are only running a single cluster on this host:\n\n\tsudo %s shell \n' % (sys.argv[0]))
+
+ logger.info('Please consider enabling telemetry to help improve Ceph:\n\n'
+ '\tceph telemetry on\n\n'
+ 'For more information see:\n\n'
+ '\thttps://docs.ceph.com/en/pacific/mgr/telemetry/\n')
+ logger.info('Bootstrap complete.')
+ return ctx.error_code
+
+##################################
+
+
+def command_registry_login(ctx: CephadmContext) -> int:
+ if ctx.registry_json:
+ logger.info('Pulling custom registry login info from %s.' % ctx.registry_json)
+ d = get_parm(ctx.registry_json)
+ if d.get('url') and d.get('username') and d.get('password'):
+ ctx.registry_url = d.get('url')
+ ctx.registry_username = d.get('username')
+ ctx.registry_password = d.get('password')
+ registry_login(ctx, ctx.registry_url, ctx.registry_username, ctx.registry_password)
+ else:
+ raise Error('json provided for custom registry login did not include all necessary fields. '
+ 'Please setup json file as\n'
+ '{\n'
+ ' "url": "REGISTRY_URL",\n'
+ ' "username": "REGISTRY_USERNAME",\n'
+ ' "password": "REGISTRY_PASSWORD"\n'
+ '}\n')
+ elif ctx.registry_url and ctx.registry_username and ctx.registry_password:
+ registry_login(ctx, ctx.registry_url, ctx.registry_username, ctx.registry_password)
+ else:
+ raise Error('Invalid custom registry arguments received. To login to a custom registry include '
+ '--registry-url, --registry-username and --registry-password '
+ 'options or --registry-json option')
+ return 0
+
+
+def registry_login(ctx: CephadmContext, url: Optional[str], username: Optional[str], password: Optional[str]) -> None:
+ logger.info('Logging into custom registry.')
+ try:
+ engine = ctx.container_engine
+ cmd = [engine.path, 'login',
+ '-u', username, '-p', password,
+ url]
+ if isinstance(engine, Podman):
+ cmd.append('--authfile=/etc/ceph/podman-auth.json')
+ out, _, _ = call_throws(ctx, cmd)
+ if isinstance(engine, Podman):
+ os.chmod('/etc/ceph/podman-auth.json', 0o600)
+ except Exception:
+ raise Error('Failed to login to custom registry @ %s as %s with given password' % (ctx.registry_url, ctx.registry_username))
+
+##################################
+
+
+def extract_uid_gid_monitoring(ctx, daemon_type):
+ # type: (CephadmContext, str) -> Tuple[int, int]
+
+ if daemon_type == 'prometheus':
+ uid, gid = extract_uid_gid(ctx, file_path='/etc/prometheus')
+ elif daemon_type == 'node-exporter':
+ uid, gid = 65534, 65534
+ elif daemon_type == 'grafana':
+ uid, gid = extract_uid_gid(ctx, file_path='/var/lib/grafana')
+ elif daemon_type == 'alertmanager':
+ uid, gid = extract_uid_gid(ctx, file_path=['/etc/alertmanager', '/etc/prometheus'])
+ else:
+ raise Error('{} not implemented yet'.format(daemon_type))
+ return uid, gid
+
+
+def get_container_with_extra_args(ctx: CephadmContext,
+ fsid: str, daemon_type: str, daemon_id: Union[int, str],
+ privileged: bool = False,
+ ptrace: bool = False,
+ container_args: Optional[List[str]] = None) -> 'CephContainer':
+ # wrapper for get_container that additionally adds extra_container_args if present
+ # used for deploying daemons with additional podman/docker container arguments
+ c = get_container(ctx, fsid, daemon_type, daemon_id, privileged, ptrace, container_args)
+ if 'extra_container_args' in ctx and ctx.extra_container_args:
+ c.container_args.extend(ctx.extra_container_args)
+ return c
+
+
+@default_image
+def command_deploy(ctx):
+ # type: (CephadmContext) -> None
+ daemon_type, daemon_id = ctx.name.split('.', 1)
+
+ lock = FileLock(ctx, ctx.fsid)
+ lock.acquire()
+
+ if daemon_type not in get_supported_daemons():
+ raise Error('daemon type %s not recognized' % daemon_type)
+
+ redeploy = False
+ unit_name = get_unit_name(ctx.fsid, daemon_type, daemon_id)
+ (_, state, _) = check_unit(ctx, unit_name)
+ if state == 'running' or is_container_running(ctx, CephContainer.for_daemon(ctx, ctx.fsid, daemon_type, daemon_id, 'bash')):
+ redeploy = True
+
+ if ctx.reconfig:
+ logger.info('%s daemon %s ...' % ('Reconfig', ctx.name))
+ elif redeploy:
+ logger.info('%s daemon %s ...' % ('Redeploy', ctx.name))
+ else:
+ logger.info('%s daemon %s ...' % ('Deploy', ctx.name))
+
+ # Migrate sysctl conf files from /usr/lib to /etc
+ migrate_sysctl_dir(ctx, ctx.fsid)
+
+ # Get and check ports explicitly required to be opened
+ daemon_ports = [] # type: List[int]
+
+ # only check port in use if not reconfig or redeploy since service
+ # we are redeploying/reconfiguring will already be using the port
+ if not ctx.reconfig and not redeploy:
+ if ctx.tcp_ports:
+ daemon_ports = list(map(int, ctx.tcp_ports.split()))
+
+ if daemon_type in Ceph.daemons:
+ config, keyring = get_config_and_keyring(ctx)
+ uid, gid = extract_uid_gid(ctx)
+ make_var_run(ctx, ctx.fsid, uid, gid)
+
+ c = get_container_with_extra_args(ctx, ctx.fsid, daemon_type, daemon_id,
+ ptrace=ctx.allow_ptrace)
+ deploy_daemon(ctx, ctx.fsid, daemon_type, daemon_id, c, uid, gid,
+ config=config, keyring=keyring,
+ osd_fsid=ctx.osd_fsid,
+ reconfig=ctx.reconfig,
+ ports=daemon_ports)
+
+ elif daemon_type in Monitoring.components:
+ # monitoring daemon - prometheus, grafana, alertmanager, node-exporter
+ # Default Checks
+ # make sure provided config-json is sufficient
+ config = get_parm(ctx.config_json) # type: ignore
+ required_files = Monitoring.components[daemon_type].get('config-json-files', list())
+ required_args = Monitoring.components[daemon_type].get('config-json-args', list())
+ if required_files:
+ if not config or not all(c in config.get('files', {}).keys() for c in required_files): # type: ignore
+ raise Error('{} deployment requires config-json which must '
+ 'contain file content for {}'.format(daemon_type.capitalize(), ', '.join(required_files)))
+ if required_args:
+ if not config or not all(c in config.keys() for c in required_args): # type: ignore
+ raise Error('{} deployment requires config-json which must '
+ 'contain arg for {}'.format(daemon_type.capitalize(), ', '.join(required_args)))
+
+ uid, gid = extract_uid_gid_monitoring(ctx, daemon_type)
+ c = get_container_with_extra_args(ctx, ctx.fsid, daemon_type, daemon_id)
+ deploy_daemon(ctx, ctx.fsid, daemon_type, daemon_id, c, uid, gid,
+ reconfig=ctx.reconfig,
+ ports=daemon_ports)
+
+ elif daemon_type == NFSGanesha.daemon_type:
+ if not ctx.reconfig and not redeploy and not daemon_ports:
+ daemon_ports = list(NFSGanesha.port_map.values())
+
+ config, keyring = get_config_and_keyring(ctx)
+ # TODO: extract ganesha uid/gid (997, 994) ?
+ uid, gid = extract_uid_gid(ctx)
+ c = get_container_with_extra_args(ctx, ctx.fsid, daemon_type, daemon_id)
+ deploy_daemon(ctx, ctx.fsid, daemon_type, daemon_id, c, uid, gid,
+ config=config, keyring=keyring,
+ reconfig=ctx.reconfig,
+ ports=daemon_ports)
+
+ elif daemon_type == CephIscsi.daemon_type:
+ config, keyring = get_config_and_keyring(ctx)
+ uid, gid = extract_uid_gid(ctx)
+ c = get_container_with_extra_args(ctx, ctx.fsid, daemon_type, daemon_id)
+ deploy_daemon(ctx, ctx.fsid, daemon_type, daemon_id, c, uid, gid,
+ config=config, keyring=keyring,
+ reconfig=ctx.reconfig,
+ ports=daemon_ports)
+
+ elif daemon_type == HAproxy.daemon_type:
+ haproxy = HAproxy.init(ctx, ctx.fsid, daemon_id)
+ uid, gid = haproxy.extract_uid_gid_haproxy()
+ c = get_container_with_extra_args(ctx, ctx.fsid, daemon_type, daemon_id)
+ deploy_daemon(ctx, ctx.fsid, daemon_type, daemon_id, c, uid, gid,
+ reconfig=ctx.reconfig,
+ ports=daemon_ports)
+
+ elif daemon_type == Keepalived.daemon_type:
+ keepalived = Keepalived.init(ctx, ctx.fsid, daemon_id)
+ uid, gid = keepalived.extract_uid_gid_keepalived()
+ c = get_container_with_extra_args(ctx, ctx.fsid, daemon_type, daemon_id)
+ deploy_daemon(ctx, ctx.fsid, daemon_type, daemon_id, c, uid, gid,
+ reconfig=ctx.reconfig,
+ ports=daemon_ports)
+
+ elif daemon_type == CustomContainer.daemon_type:
+ cc = CustomContainer.init(ctx, ctx.fsid, daemon_id)
+ if not ctx.reconfig and not redeploy:
+ daemon_ports.extend(cc.ports)
+ c = get_container_with_extra_args(ctx, ctx.fsid, daemon_type, daemon_id,
+ privileged=cc.privileged,
+ ptrace=ctx.allow_ptrace)
+ deploy_daemon(ctx, ctx.fsid, daemon_type, daemon_id, c,
+ uid=cc.uid, gid=cc.gid, config=None,
+ keyring=None, reconfig=ctx.reconfig,
+ ports=daemon_ports)
+
+ elif daemon_type == CephadmDaemon.daemon_type:
+ # get current user gid and uid
+ uid = os.getuid()
+ gid = os.getgid()
+ config_js = get_parm(ctx.config_json) # type: Dict[str, str]
+ if not daemon_ports:
+ logger.info('cephadm-exporter will use default port ({})'.format(CephadmDaemon.default_port))
+ daemon_ports = [CephadmDaemon.default_port]
+
+ CephadmDaemon.validate_config(config_js)
+
+ deploy_daemon(ctx, ctx.fsid, daemon_type, daemon_id, None,
+ uid, gid, ports=daemon_ports)
+
+ elif daemon_type == SNMPGateway.daemon_type:
+ sc = SNMPGateway.init(ctx, ctx.fsid, daemon_id)
+ c = get_container_with_extra_args(ctx, ctx.fsid, daemon_type, daemon_id)
+ deploy_daemon(ctx, ctx.fsid, daemon_type, daemon_id, c,
+ sc.uid, sc.gid,
+ ports=daemon_ports)
+
+ else:
+ raise Error('daemon type {} not implemented in command_deploy function'
+ .format(daemon_type))
+
+##################################
+
+
+@infer_image
+def command_run(ctx):
+ # type: (CephadmContext) -> int
+ (daemon_type, daemon_id) = ctx.name.split('.', 1)
+ c = get_container(ctx, ctx.fsid, daemon_type, daemon_id)
+ command = c.run_cmd()
+ return call_timeout(ctx, command, ctx.timeout)
+
+##################################
+
+
+@infer_fsid
+@infer_config
+@infer_image
+@validate_fsid
+def command_shell(ctx):
+ # type: (CephadmContext) -> int
+ cp = read_config(ctx.config)
+ if cp.has_option('global', 'fsid') and \
+ cp.get('global', 'fsid') != ctx.fsid:
+ raise Error('fsid does not match ceph.conf')
+
+ if ctx.name:
+ if '.' in ctx.name:
+ (daemon_type, daemon_id) = ctx.name.split('.', 1)
+ else:
+ daemon_type = ctx.name
+ daemon_id = None
+ else:
+ daemon_type = 'osd' # get the most mounts
+ daemon_id = None
+
+ if ctx.fsid and daemon_type in Ceph.daemons:
+ make_log_dir(ctx, ctx.fsid)
+
+ if daemon_id and not ctx.fsid:
+ raise Error('must pass --fsid to specify cluster')
+
+ # use /etc/ceph files by default, if present. we do this instead of
+ # making these defaults in the arg parser because we don't want an error
+ # if they don't exist.
+ if not ctx.keyring and os.path.exists(SHELL_DEFAULT_KEYRING):
+ ctx.keyring = SHELL_DEFAULT_KEYRING
+
+ container_args: List[str] = ['-i']
+ mounts = get_container_mounts(ctx, ctx.fsid, daemon_type, daemon_id,
+ no_config=True if ctx.config else False)
+ binds = get_container_binds(ctx, ctx.fsid, daemon_type, daemon_id)
+ if ctx.config:
+ mounts[pathify(ctx.config)] = '/etc/ceph/ceph.conf:z'
+ if ctx.keyring:
+ mounts[pathify(ctx.keyring)] = '/etc/ceph/ceph.keyring:z'
+ if ctx.mount:
+ for _mount in ctx.mount:
+ split_src_dst = _mount.split(':')
+ mount = pathify(split_src_dst[0])
+ filename = os.path.basename(split_src_dst[0])
+ if len(split_src_dst) > 1:
+ dst = split_src_dst[1]
+ if len(split_src_dst) == 3:
+ dst = '{}:{}'.format(dst, split_src_dst[2])
+ mounts[mount] = dst
+ else:
+ mounts[mount] = '/mnt/{}'.format(filename)
+ if ctx.command:
+ command = ctx.command
+ else:
+ command = ['bash']
+ container_args += [
+ '-t',
+ '-e', 'LANG=C',
+ '-e', 'PS1=%s' % CUSTOM_PS1,
+ ]
+ if ctx.fsid:
+ home = os.path.join(ctx.data_dir, ctx.fsid, 'home')
+ if not os.path.exists(home):
+ logger.debug('Creating root home at %s' % home)
+ makedirs(home, 0, 0, 0o660)
+ if os.path.exists('/etc/skel'):
+ for f in os.listdir('/etc/skel'):
+ if f.startswith('.bash'):
+ shutil.copyfile(os.path.join('/etc/skel', f),
+ os.path.join(home, f))
+ mounts[home] = '/root'
+
+ for i in ctx.volume:
+ a, b = i.split(':', 1)
+ mounts[a] = b
+
+ c = CephContainer(
+ ctx,
+ image=ctx.image,
+ entrypoint='doesnotmatter',
+ args=[],
+ container_args=container_args,
+ volume_mounts=mounts,
+ bind_mounts=binds,
+ envs=ctx.env,
+ privileged=True)
+ command = c.shell_cmd(command)
+
+ return call_timeout(ctx, command, ctx.timeout)
+
+##################################
+
+
+@infer_fsid
+def command_enter(ctx):
+ # type: (CephadmContext) -> int
+ if not ctx.fsid:
+ raise Error('must pass --fsid to specify cluster')
+ (daemon_type, daemon_id) = ctx.name.split('.', 1)
+ container_args = ['-i'] # type: List[str]
+ if ctx.command:
+ command = ctx.command
+ else:
+ command = ['sh']
+ container_args += [
+ '-t',
+ '-e', 'LANG=C',
+ '-e', 'PS1=%s' % CUSTOM_PS1,
+ ]
+ c = CephContainer(
+ ctx,
+ image=ctx.image,
+ entrypoint='doesnotmatter',
+ container_args=container_args,
+ cname='ceph-%s-%s.%s' % (ctx.fsid, daemon_type, daemon_id),
+ )
+ command = c.exec_cmd(command)
+ return call_timeout(ctx, command, ctx.timeout)
+
+##################################
+
+
+@infer_fsid
+@infer_image
+@validate_fsid
+def command_ceph_volume(ctx):
+ # type: (CephadmContext) -> None
+ cp = read_config(ctx.config)
+ if cp.has_option('global', 'fsid') and \
+ cp.get('global', 'fsid') != ctx.fsid:
+ raise Error('fsid does not match ceph.conf')
+
+ if ctx.fsid:
+ make_log_dir(ctx, ctx.fsid)
+
+ lock = FileLock(ctx, ctx.fsid)
+ lock.acquire()
+
+ (uid, gid) = (0, 0) # ceph-volume runs as root
+ mounts = get_container_mounts(ctx, ctx.fsid, 'osd', None)
+
+ tmp_config = None
+ tmp_keyring = None
+
+ (config, keyring) = get_config_and_keyring(ctx)
+
+ if config:
+ # tmp config file
+ tmp_config = write_tmp(config, uid, gid)
+ mounts[tmp_config.name] = '/etc/ceph/ceph.conf:z'
+
+ if keyring:
+ # tmp keyring file
+ tmp_keyring = write_tmp(keyring, uid, gid)
+ mounts[tmp_keyring.name] = '/var/lib/ceph/bootstrap-osd/ceph.keyring:z'
+
+ c = get_ceph_volume_container(
+ ctx,
+ envs=ctx.env,
+ args=ctx.command,
+ volume_mounts=mounts,
+ )
+
+ out, err, code = call_throws(ctx, c.run_cmd())
+ if not code:
+ print(out)
+
+##################################
+
+
+@infer_fsid
+def command_unit(ctx):
+ # type: (CephadmContext) -> int
+ if not ctx.fsid:
+ raise Error('must pass --fsid to specify cluster')
+
+ unit_name = get_unit_name_by_daemon_name(ctx, ctx.fsid, ctx.name)
+
+ _, _, code = call(
+ ctx,
+ ['systemctl', ctx.command, unit_name],
+ verbosity=CallVerbosity.VERBOSE,
+ desc=''
+ )
+ return code
+
+##################################
+
+
+@infer_fsid
+def command_logs(ctx):
+ # type: (CephadmContext) -> None
+ if not ctx.fsid:
+ raise Error('must pass --fsid to specify cluster')
+
+ unit_name = get_unit_name_by_daemon_name(ctx, ctx.fsid, ctx.name)
+
+ cmd = [find_program('journalctl')]
+ cmd.extend(['-u', unit_name])
+ if ctx.command:
+ cmd.extend(ctx.command)
+
+ # call this directly, without our wrapper, so that we get an unmolested
+ # stdout with logger prefixing.
+ logger.debug('Running command: %s' % ' '.join(cmd))
+ subprocess.call(cmd, env=os.environ.copy()) # type: ignore
+
+##################################
+
+
+def list_networks(ctx):
+ # type: (CephadmContext) -> Dict[str,Dict[str, Set[str]]]
+
+ # sadly, 18.04's iproute2 4.15.0-2ubun doesn't support the -j flag,
+ # so we'll need to use a regex to parse 'ip' command output.
+ #
+ # out, _, _ = call_throws(['ip', '-j', 'route', 'ls'])
+ # j = json.loads(out)
+ # for x in j:
+ res = _list_ipv4_networks(ctx)
+ res.update(_list_ipv6_networks(ctx))
+ return res
+
+
+def _list_ipv4_networks(ctx: CephadmContext) -> Dict[str, Dict[str, Set[str]]]:
+ execstr: Optional[str] = find_executable('ip')
+ if not execstr:
+ raise FileNotFoundError("unable to find 'ip' command")
+ out, _, _ = call_throws(ctx, [execstr, 'route', 'ls'])
+ return _parse_ipv4_route(out)
+
+
+def _parse_ipv4_route(out: str) -> Dict[str, Dict[str, Set[str]]]:
+ r = {} # type: Dict[str, Dict[str, Set[str]]]
+ p = re.compile(r'^(\S+) (?:via \S+)? ?dev (\S+) (.*)scope link (.*)src (\S+)')
+ for line in out.splitlines():
+ m = p.findall(line)
+ if not m:
+ continue
+ net = m[0][0]
+ if '/' not in net: # aggregate /32 mask for single host sub-networks
+ net += '/32'
+ iface = m[0][1]
+ ip = m[0][4]
+ if net not in r:
+ r[net] = {}
+ if iface not in r[net]:
+ r[net][iface] = set()
+ r[net][iface].add(ip)
+ return r
+
+
+def _list_ipv6_networks(ctx: CephadmContext) -> Dict[str, Dict[str, Set[str]]]:
+ execstr: Optional[str] = find_executable('ip')
+ if not execstr:
+ raise FileNotFoundError("unable to find 'ip' command")
+ routes, _, _ = call_throws(ctx, [execstr, '-6', 'route', 'ls'])
+ ips, _, _ = call_throws(ctx, [execstr, '-6', 'addr', 'ls'])
+ return _parse_ipv6_route(routes, ips)
+
+
+def _parse_ipv6_route(routes: str, ips: str) -> Dict[str, Dict[str, Set[str]]]:
+ r = {} # type: Dict[str, Dict[str, Set[str]]]
+ route_p = re.compile(r'^(\S+) dev (\S+) proto (\S+) metric (\S+) .*pref (\S+)$')
+ ip_p = re.compile(r'^\s+inet6 (\S+)/(.*)scope (.*)$')
+ iface_p = re.compile(r'^(\d+): (\S+): (.*)$')
+ for line in routes.splitlines():
+ m = route_p.findall(line)
+ if not m or m[0][0].lower() == 'default':
+ continue
+ net = m[0][0]
+ if '/' not in net: # aggregate /128 mask for single host sub-networks
+ net += '/128'
+ iface = m[0][1]
+ if iface == 'lo': # skip loopback devices
+ continue
+ if net not in r:
+ r[net] = {}
+ if iface not in r[net]:
+ r[net][iface] = set()
+
+ iface = None
+ for line in ips.splitlines():
+ m = ip_p.findall(line)
+ if not m:
+ m = iface_p.findall(line)
+ if m:
+ # drop @... suffix, if present
+ iface = m[0][1].split('@')[0]
+ continue
+ ip = m[0][0]
+ # find the network it belongs to
+ net = [n for n in r.keys()
+ if ipaddress.ip_address(ip) in ipaddress.ip_network(n)]
+ if net and iface in r[net[0]]:
+ assert iface
+ r[net[0]][iface].add(ip)
+
+ return r
+
+
+def command_list_networks(ctx):
+ # type: (CephadmContext) -> None
+ r = list_networks(ctx)
+
+ def serialize_sets(obj: Any) -> Any:
+ return list(obj) if isinstance(obj, set) else obj
+
+ print(json.dumps(r, indent=4, default=serialize_sets))
+
+##################################
+
+
+def command_ls(ctx):
+ # type: (CephadmContext) -> None
+ ls = list_daemons(ctx, detail=not ctx.no_detail,
+ legacy_dir=ctx.legacy_dir)
+ print(json.dumps(ls, indent=4))
+
+
+def with_units_to_int(v: str) -> int:
+ if v.endswith('iB'):
+ v = v[:-2]
+ elif v.endswith('B'):
+ v = v[:-1]
+ mult = 1
+ if v[-1].upper() == 'K':
+ mult = 1024
+ v = v[:-1]
+ elif v[-1].upper() == 'M':
+ mult = 1024 * 1024
+ v = v[:-1]
+ elif v[-1].upper() == 'G':
+ mult = 1024 * 1024 * 1024
+ v = v[:-1]
+ elif v[-1].upper() == 'T':
+ mult = 1024 * 1024 * 1024 * 1024
+ v = v[:-1]
+ return int(float(v) * mult)
+
+
+def list_daemons(ctx, detail=True, legacy_dir=None):
+ # type: (CephadmContext, bool, Optional[str]) -> List[Dict[str, str]]
+ host_version: Optional[str] = None
+ ls = []
+ container_path = ctx.container_engine.path
+
+ data_dir = ctx.data_dir
+ if legacy_dir is not None:
+ data_dir = os.path.abspath(legacy_dir + data_dir)
+
+ # keep track of ceph versions we see
+ seen_versions = {} # type: Dict[str, Optional[str]]
+
+ # keep track of image digests
+ seen_digests = {} # type: Dict[str, List[str]]
+
+ # keep track of memory and cpu usage we've seen
+ seen_memusage = {} # type: Dict[str, int]
+ seen_cpuperc = {} # type: Dict[str, str]
+ out, err, code = call(
+ ctx,
+ [container_path, 'stats', '--format', '{{.ID}},{{.MemUsage}}', '--no-stream'],
+ verbosity=CallVerbosity.DEBUG
+ )
+ seen_memusage_cid_len, seen_memusage = _parse_mem_usage(code, out)
+
+ out, err, code = call(
+ ctx,
+ [container_path, 'stats', '--format', '{{.ID}},{{.CPUPerc}}', '--no-stream'],
+ verbosity=CallVerbosity.DEBUG
+ )
+ seen_cpuperc_cid_len, seen_cpuperc = _parse_cpu_perc(code, out)
+
+ # /var/lib/ceph
+ if os.path.exists(data_dir):
+ for i in os.listdir(data_dir):
+ if i in ['mon', 'osd', 'mds', 'mgr']:
+ daemon_type = i
+ for j in os.listdir(os.path.join(data_dir, i)):
+ if '-' not in j:
+ continue
+ (cluster, daemon_id) = j.split('-', 1)
+ fsid = get_legacy_daemon_fsid(ctx,
+ cluster, daemon_type, daemon_id,
+ legacy_dir=legacy_dir)
+ legacy_unit_name = 'ceph-%s@%s' % (daemon_type, daemon_id)
+ val: Dict[str, Any] = {
+ 'style': 'legacy',
+ 'name': '%s.%s' % (daemon_type, daemon_id),
+ 'fsid': fsid if fsid is not None else 'unknown',
+ 'systemd_unit': legacy_unit_name,
+ }
+ if detail:
+ (val['enabled'], val['state'], _) = \
+ check_unit(ctx, legacy_unit_name)
+ if not host_version:
+ try:
+ out, err, code = call(ctx,
+ ['ceph', '-v'],
+ verbosity=CallVerbosity.DEBUG)
+ if not code and out.startswith('ceph version '):
+ host_version = out.split(' ')[2]
+ except Exception:
+ pass
+ val['host_version'] = host_version
+ ls.append(val)
+ elif is_fsid(i):
+ fsid = str(i) # convince mypy that fsid is a str here
+ for j in os.listdir(os.path.join(data_dir, i)):
+ if '.' in j and os.path.isdir(os.path.join(data_dir, fsid, j)):
+ name = j
+ (daemon_type, daemon_id) = j.split('.', 1)
+ unit_name = get_unit_name(fsid,
+ daemon_type,
+ daemon_id)
+ else:
+ continue
+ val = {
+ 'style': 'cephadm:v1',
+ 'name': name,
+ 'fsid': fsid,
+ 'systemd_unit': unit_name,
+ }
+ if detail:
+ # get container id
+ (val['enabled'], val['state'], _) = \
+ check_unit(ctx, unit_name)
+ container_id = None
+ image_name = None
+ image_id = None
+ image_digests = None
+ version = None
+ start_stamp = None
+
+ out, err, code = get_container_stats(ctx, container_path, fsid, daemon_type, daemon_id)
+ if not code:
+ (container_id, image_name, image_id, start,
+ version) = out.strip().split(',')
+ image_id = normalize_container_id(image_id)
+ daemon_type = name.split('.', 1)[0]
+ start_stamp = try_convert_datetime(start)
+
+ # collect digests for this image id
+ image_digests = seen_digests.get(image_id)
+ if not image_digests:
+ out, err, code = call(
+ ctx,
+ [
+ container_path, 'image', 'inspect', image_id,
+ '--format', '{{.RepoDigests}}',
+ ],
+ verbosity=CallVerbosity.DEBUG)
+ if not code:
+ image_digests = list(set(map(
+ normalize_image_digest,
+ out.strip()[1:-1].split(' '))))
+ seen_digests[image_id] = image_digests
+
+ # identify software version inside the container (if we can)
+ if not version or '.' not in version:
+ version = seen_versions.get(image_id, None)
+ if daemon_type == NFSGanesha.daemon_type:
+ version = NFSGanesha.get_version(ctx, container_id)
+ if daemon_type == CephIscsi.daemon_type:
+ version = CephIscsi.get_version(ctx, container_id)
+ elif not version:
+ if daemon_type in Ceph.daemons:
+ out, err, code = call(ctx,
+ [container_path, 'exec', container_id,
+ 'ceph', '-v'],
+ verbosity=CallVerbosity.DEBUG)
+ if not code and \
+ out.startswith('ceph version '):
+ version = out.split(' ')[2]
+ seen_versions[image_id] = version
+ elif daemon_type == 'grafana':
+ out, err, code = call(ctx,
+ [container_path, 'exec', container_id,
+ 'grafana-server', '-v'],
+ verbosity=CallVerbosity.DEBUG)
+ if not code and \
+ out.startswith('Version '):
+ version = out.split(' ')[1]
+ seen_versions[image_id] = version
+ elif daemon_type in ['prometheus',
+ 'alertmanager',
+ 'node-exporter']:
+ version = Monitoring.get_version(ctx, container_id, daemon_type)
+ seen_versions[image_id] = version
+ elif daemon_type == 'haproxy':
+ out, err, code = call(ctx,
+ [container_path, 'exec', container_id,
+ 'haproxy', '-v'],
+ verbosity=CallVerbosity.DEBUG)
+ if not code and \
+ out.startswith('HA-Proxy version '):
+ version = out.split(' ')[2]
+ seen_versions[image_id] = version
+ elif daemon_type == 'keepalived':
+ out, err, code = call(ctx,
+ [container_path, 'exec', container_id,
+ 'keepalived', '--version'],
+ verbosity=CallVerbosity.DEBUG)
+ if not code and \
+ err.startswith('Keepalived '):
+ version = err.split(' ')[1]
+ if version[0] == 'v':
+ version = version[1:]
+ seen_versions[image_id] = version
+ elif daemon_type == CustomContainer.daemon_type:
+ # Because a custom container can contain
+ # everything, we do not know which command
+ # to execute to get the version.
+ pass
+ elif daemon_type == SNMPGateway.daemon_type:
+ version = SNMPGateway.get_version(ctx, fsid, daemon_id)
+ seen_versions[image_id] = version
+ else:
+ logger.warning('version for unknown daemon type %s' % daemon_type)
+ else:
+ vfile = os.path.join(data_dir, fsid, j, 'unit.image') # type: ignore
+ try:
+ with open(vfile, 'r') as f:
+ image_name = f.read().strip() or None
+ except IOError:
+ pass
+
+ # unit.meta?
+ mfile = os.path.join(data_dir, fsid, j, 'unit.meta') # type: ignore
+ try:
+ with open(mfile, 'r') as f:
+ meta = json.loads(f.read())
+ val.update(meta)
+ except IOError:
+ pass
+
+ val['container_id'] = container_id
+ val['container_image_name'] = image_name
+ val['container_image_id'] = image_id
+ val['container_image_digests'] = image_digests
+ if container_id:
+ val['memory_usage'] = seen_memusage.get(container_id[0:seen_memusage_cid_len])
+ val['cpu_percentage'] = seen_cpuperc.get(container_id[0:seen_cpuperc_cid_len])
+ val['version'] = version
+ val['started'] = start_stamp
+ val['created'] = get_file_timestamp(
+ os.path.join(data_dir, fsid, j, 'unit.created')
+ )
+ val['deployed'] = get_file_timestamp(
+ os.path.join(data_dir, fsid, j, 'unit.image'))
+ val['configured'] = get_file_timestamp(
+ os.path.join(data_dir, fsid, j, 'unit.configured'))
+ ls.append(val)
+
+ return ls
+
+
+def _parse_mem_usage(code: int, out: str) -> Tuple[int, Dict[str, int]]:
+ # keep track of memory usage we've seen
+ seen_memusage = {} # type: Dict[str, int]
+ seen_memusage_cid_len = 0
+ if not code:
+ for line in out.splitlines():
+ (cid, usage) = line.split(',')
+ (used, limit) = usage.split(' / ')
+ try:
+ seen_memusage[cid] = with_units_to_int(used)
+ if not seen_memusage_cid_len:
+ seen_memusage_cid_len = len(cid)
+ except ValueError:
+ logger.info('unable to parse memory usage line\n>{}'.format(line))
+ pass
+ return seen_memusage_cid_len, seen_memusage
+
+
+def _parse_cpu_perc(code: int, out: str) -> Tuple[int, Dict[str, str]]:
+ seen_cpuperc = {}
+ seen_cpuperc_cid_len = 0
+ if not code:
+ for line in out.splitlines():
+ (cid, cpuperc) = line.split(',')
+ try:
+ seen_cpuperc[cid] = cpuperc
+ if not seen_cpuperc_cid_len:
+ seen_cpuperc_cid_len = len(cid)
+ except ValueError:
+ logger.info('unable to parse cpu percentage line\n>{}'.format(line))
+ pass
+ return seen_cpuperc_cid_len, seen_cpuperc
+
+
+def get_daemon_description(ctx, fsid, name, detail=False, legacy_dir=None):
+ # type: (CephadmContext, str, str, bool, Optional[str]) -> Dict[str, str]
+
+ for d in list_daemons(ctx, detail=detail, legacy_dir=legacy_dir):
+ if d['fsid'] != fsid:
+ continue
+ if d['name'] != name:
+ continue
+ return d
+ raise Error('Daemon not found: {}. See `cephadm ls`'.format(name))
+
+
+def get_container_stats(ctx: CephadmContext, container_path: str, fsid: str, daemon_type: str, daemon_id: str) -> Tuple[str, str, int]:
+ c = CephContainer.for_daemon(ctx, fsid, daemon_type, daemon_id, 'bash')
+ out, err, code = '', '', -1
+ for name in (c.cname, c.old_cname):
+ cmd = [
+ container_path, 'inspect',
+ '--format', '{{.Id}},{{.Config.Image}},{{.Image}},{{.Created}},{{index .Config.Labels "io.ceph.version"}}',
+ name
+ ]
+ out, err, code = call(ctx, cmd, verbosity=CallVerbosity.DEBUG)
+ if not code:
+ break
+ return out, err, code
+
+##################################
+
+
+@default_image
+def command_adopt(ctx):
+ # type: (CephadmContext) -> None
+
+ if not ctx.skip_pull:
+ try:
+ _pull_image(ctx, ctx.image)
+ except UnauthorizedRegistryError:
+ err_str = 'Failed to pull container image. Host may not be logged into container registry. Try `cephadm registry-login --registry-url <url> --registry-username <username> --registry-password <password>` or supply login info via a json file with `cephadm registry-login --registry-json <file>`'
+ logger.debug(f'Pulling image for `command_adopt` failed: {err_str}')
+ raise Error(err_str)
+
+ (daemon_type, daemon_id) = ctx.name.split('.', 1)
+
+ # legacy check
+ if ctx.style != 'legacy':
+ raise Error('adoption of style %s not implemented' % ctx.style)
+
+ # lock
+ fsid = get_legacy_daemon_fsid(ctx,
+ ctx.cluster,
+ daemon_type,
+ daemon_id,
+ legacy_dir=ctx.legacy_dir)
+ if not fsid:
+ raise Error('could not detect legacy fsid; set fsid in ceph.conf')
+ lock = FileLock(ctx, fsid)
+ lock.acquire()
+
+ # call correct adoption
+ if daemon_type in Ceph.daemons:
+ command_adopt_ceph(ctx, daemon_type, daemon_id, fsid)
+ elif daemon_type == 'prometheus':
+ command_adopt_prometheus(ctx, daemon_id, fsid)
+ elif daemon_type == 'grafana':
+ command_adopt_grafana(ctx, daemon_id, fsid)
+ elif daemon_type == 'node-exporter':
+ raise Error('adoption of node-exporter not implemented')
+ elif daemon_type == 'alertmanager':
+ command_adopt_alertmanager(ctx, daemon_id, fsid)
+ else:
+ raise Error('daemon type %s not recognized' % daemon_type)
+
+
+class AdoptOsd(object):
+ def __init__(self, ctx, osd_data_dir, osd_id):
+ # type: (CephadmContext, str, str) -> None
+ self.ctx = ctx
+ self.osd_data_dir = osd_data_dir
+ self.osd_id = osd_id
+
+ def check_online_osd(self):
+ # type: () -> Tuple[Optional[str], Optional[str]]
+
+ osd_fsid, osd_type = None, None
+
+ path = os.path.join(self.osd_data_dir, 'fsid')
+ try:
+ with open(path, 'r') as f:
+ osd_fsid = f.read().strip()
+ logger.info('Found online OSD at %s' % path)
+ except IOError:
+ logger.info('Unable to read OSD fsid from %s' % path)
+ if os.path.exists(os.path.join(self.osd_data_dir, 'type')):
+ with open(os.path.join(self.osd_data_dir, 'type')) as f:
+ osd_type = f.read().strip()
+ else:
+ logger.info('"type" file missing for OSD data dir')
+
+ return osd_fsid, osd_type
+
+ def check_offline_lvm_osd(self):
+ # type: () -> Tuple[Optional[str], Optional[str]]
+ osd_fsid, osd_type = None, None
+
+ c = get_ceph_volume_container(
+ self.ctx,
+ args=['lvm', 'list', '--format=json'],
+ )
+ out, err, code = call_throws(self.ctx, c.run_cmd())
+ if not code:
+ try:
+ js = json.loads(out)
+ if self.osd_id in js:
+ logger.info('Found offline LVM OSD {}'.format(self.osd_id))
+ osd_fsid = js[self.osd_id][0]['tags']['ceph.osd_fsid']
+ for device in js[self.osd_id]:
+ if device['tags']['ceph.type'] == 'block':
+ osd_type = 'bluestore'
+ break
+ if device['tags']['ceph.type'] == 'data':
+ osd_type = 'filestore'
+ break
+ except ValueError as e:
+ logger.info('Invalid JSON in ceph-volume lvm list: {}'.format(e))
+
+ return osd_fsid, osd_type
+
+ def check_offline_simple_osd(self):
+ # type: () -> Tuple[Optional[str], Optional[str]]
+ osd_fsid, osd_type = None, None
+
+ osd_file = glob('/etc/ceph/osd/{}-[a-f0-9-]*.json'.format(self.osd_id))
+ if len(osd_file) == 1:
+ with open(osd_file[0], 'r') as f:
+ try:
+ js = json.loads(f.read())
+ logger.info('Found offline simple OSD {}'.format(self.osd_id))
+ osd_fsid = js['fsid']
+ osd_type = js['type']
+ if osd_type != 'filestore':
+ # need this to be mounted for the adopt to work, as it
+ # needs to move files from this directory
+ call_throws(self.ctx, ['mount', js['data']['path'], self.osd_data_dir])
+ except ValueError as e:
+ logger.info('Invalid JSON in {}: {}'.format(osd_file, e))
+
+ return osd_fsid, osd_type
+
+ def change_cluster_name(self) -> None:
+ logger.info('Attempting to convert osd cluster name to ceph . . .')
+ c = get_ceph_volume_container(
+ self.ctx,
+ args=['lvm', 'list', '{}'.format(self.osd_id), '--format=json'],
+ )
+ out, err, code = call_throws(self.ctx, c.run_cmd())
+ if code:
+ raise Exception(f'Failed to get list of LVs: {err}\nceph-volume failed with rc {code}')
+ try:
+ js = json.loads(out)
+ if not js:
+ raise RuntimeError(f'Failed to find osd.{self.osd_id}')
+ device: Optional[Dict[Any, Any]] = None
+ for d in js[self.osd_id]:
+ if d['type'] == 'block':
+ device = d
+ break
+ if not device:
+ raise RuntimeError(f'Failed to find block device for osd.{self.osd_id}')
+ vg = device['vg_name']
+ out, err, code = call_throws(self.ctx, ['lvchange', '--deltag', f'ceph.cluster_name={self.ctx.cluster}', vg])
+ if code:
+ raise RuntimeError(f"Can't delete tag ceph.cluster_name={self.ctx.cluster} on osd.{self.osd_id}.\nlvchange failed with rc {code}")
+ out, err, code = call_throws(self.ctx, ['lvchange', '--addtag', 'ceph.cluster_name=ceph', vg])
+ if code:
+ raise RuntimeError(f"Can't add tag ceph.cluster_name=ceph on osd.{self.osd_id}.\nlvchange failed with rc {code}")
+ logger.info('Successfully converted osd cluster name')
+ except (Exception, RuntimeError) as e:
+ logger.info(f'Failed to convert osd cluster name: {e}')
+
+
+def command_adopt_ceph(ctx, daemon_type, daemon_id, fsid):
+ # type: (CephadmContext, str, str, str) -> None
+
+ (uid, gid) = extract_uid_gid(ctx)
+
+ data_dir_src = ('/var/lib/ceph/%s/%s-%s' %
+ (daemon_type, ctx.cluster, daemon_id))
+ data_dir_src = os.path.abspath(ctx.legacy_dir + data_dir_src)
+
+ if not os.path.exists(data_dir_src):
+ raise Error("{}.{} data directory '{}' does not exist. "
+ 'Incorrect ID specified, or daemon already adopted?'.format(
+ daemon_type, daemon_id, data_dir_src))
+
+ osd_fsid = None
+ if daemon_type == 'osd':
+ adopt_osd = AdoptOsd(ctx, data_dir_src, daemon_id)
+ osd_fsid, osd_type = adopt_osd.check_online_osd()
+ if not osd_fsid:
+ osd_fsid, osd_type = adopt_osd.check_offline_lvm_osd()
+ if not osd_fsid:
+ osd_fsid, osd_type = adopt_osd.check_offline_simple_osd()
+ if not osd_fsid:
+ raise Error('Unable to find OSD {}'.format(daemon_id))
+ elif ctx.cluster != 'ceph':
+ adopt_osd.change_cluster_name()
+ logger.info('objectstore_type is %s' % osd_type)
+ assert osd_type
+ if osd_type == 'filestore':
+ raise Error('FileStore is not supported by cephadm')
+
+ # NOTE: implicit assumption here that the units correspond to the
+ # cluster we are adopting based on the /etc/{defaults,sysconfig}/ceph
+ # CLUSTER field.
+ unit_name = 'ceph-%s@%s' % (daemon_type, daemon_id)
+ (enabled, state, _) = check_unit(ctx, unit_name)
+ if state == 'running':
+ logger.info('Stopping old systemd unit %s...' % unit_name)
+ call_throws(ctx, ['systemctl', 'stop', unit_name])
+ if enabled:
+ logger.info('Disabling old systemd unit %s...' % unit_name)
+ call_throws(ctx, ['systemctl', 'disable', unit_name])
+
+ # data
+ logger.info('Moving data...')
+ data_dir_dst = make_data_dir(ctx, fsid, daemon_type, daemon_id,
+ uid=uid, gid=gid)
+ move_files(ctx, glob(os.path.join(data_dir_src, '*')),
+ data_dir_dst,
+ uid=uid, gid=gid)
+ logger.debug('Remove dir `%s`' % (data_dir_src))
+ if os.path.ismount(data_dir_src):
+ call_throws(ctx, ['umount', data_dir_src])
+ os.rmdir(data_dir_src)
+
+ logger.info('Chowning content...')
+ call_throws(ctx, ['chown', '-c', '-R', '%d.%d' % (uid, gid), data_dir_dst])
+
+ if daemon_type == 'mon':
+ # rename *.ldb -> *.sst, in case they are coming from ubuntu
+ store = os.path.join(data_dir_dst, 'store.db')
+ num_renamed = 0
+ if os.path.exists(store):
+ for oldf in os.listdir(store):
+ if oldf.endswith('.ldb'):
+ newf = oldf.replace('.ldb', '.sst')
+ oldp = os.path.join(store, oldf)
+ newp = os.path.join(store, newf)
+ logger.debug('Renaming %s -> %s' % (oldp, newp))
+ os.rename(oldp, newp)
+ if num_renamed:
+ logger.info('Renamed %d leveldb *.ldb files to *.sst',
+ num_renamed)
+ if daemon_type == 'osd':
+ for n in ['block', 'block.db', 'block.wal']:
+ p = os.path.join(data_dir_dst, n)
+ if os.path.exists(p):
+ logger.info('Chowning %s...' % p)
+ os.chown(p, uid, gid)
+ # disable the ceph-volume 'simple' mode files on the host
+ simple_fn = os.path.join('/etc/ceph/osd',
+ '%s-%s.json' % (daemon_id, osd_fsid))
+ if os.path.exists(simple_fn):
+ new_fn = simple_fn + '.adopted-by-cephadm'
+ logger.info('Renaming %s -> %s', simple_fn, new_fn)
+ os.rename(simple_fn, new_fn)
+ logger.info('Disabling host unit ceph-volume@ simple unit...')
+ call(ctx, ['systemctl', 'disable',
+ 'ceph-volume@simple-%s-%s.service' % (daemon_id, osd_fsid)])
+ else:
+ # assume this is an 'lvm' c-v for now, but don't error
+ # out if it's not.
+ logger.info('Disabling host unit ceph-volume@ lvm unit...')
+ call(ctx, ['systemctl', 'disable',
+ 'ceph-volume@lvm-%s-%s.service' % (daemon_id, osd_fsid)])
+
+ # config
+ config_src = '/etc/ceph/%s.conf' % (ctx.cluster)
+ config_src = os.path.abspath(ctx.legacy_dir + config_src)
+ config_dst = os.path.join(data_dir_dst, 'config')
+ copy_files(ctx, [config_src], config_dst, uid=uid, gid=gid)
+
+ # logs
+ logger.info('Moving logs...')
+ log_dir_src = ('/var/log/ceph/%s-%s.%s.log*' %
+ (ctx.cluster, daemon_type, daemon_id))
+ log_dir_src = os.path.abspath(ctx.legacy_dir + log_dir_src)
+ log_dir_dst = make_log_dir(ctx, fsid, uid=uid, gid=gid)
+ move_files(ctx, glob(log_dir_src),
+ log_dir_dst,
+ uid=uid, gid=gid)
+
+ logger.info('Creating new units...')
+ make_var_run(ctx, fsid, uid, gid)
+ c = get_container(ctx, fsid, daemon_type, daemon_id)
+ deploy_daemon_units(ctx, fsid, uid, gid, daemon_type, daemon_id, c,
+ enable=True, # unconditionally enable the new unit
+ start=(state == 'running' or ctx.force_start),
+ osd_fsid=osd_fsid)
+ update_firewalld(ctx, daemon_type)
+
+
+def command_adopt_prometheus(ctx, daemon_id, fsid):
+ # type: (CephadmContext, str, str) -> None
+ daemon_type = 'prometheus'
+ (uid, gid) = extract_uid_gid_monitoring(ctx, daemon_type)
+
+ _stop_and_disable(ctx, 'prometheus')
+
+ data_dir_dst = make_data_dir(ctx, fsid, daemon_type, daemon_id,
+ uid=uid, gid=gid)
+
+ # config
+ config_src = '/etc/prometheus/prometheus.yml'
+ config_src = os.path.abspath(ctx.legacy_dir + config_src)
+ config_dst = os.path.join(data_dir_dst, 'etc/prometheus')
+ makedirs(config_dst, uid, gid, 0o755)
+ copy_files(ctx, [config_src], config_dst, uid=uid, gid=gid)
+
+ # data
+ data_src = '/var/lib/prometheus/metrics/'
+ data_src = os.path.abspath(ctx.legacy_dir + data_src)
+ data_dst = os.path.join(data_dir_dst, 'data')
+ copy_tree(ctx, [data_src], data_dst, uid=uid, gid=gid)
+
+ make_var_run(ctx, fsid, uid, gid)
+ c = get_container(ctx, fsid, daemon_type, daemon_id)
+ deploy_daemon(ctx, fsid, daemon_type, daemon_id, c, uid, gid)
+ update_firewalld(ctx, daemon_type)
+
+
+def command_adopt_grafana(ctx, daemon_id, fsid):
+ # type: (CephadmContext, str, str) -> None
+
+ daemon_type = 'grafana'
+ (uid, gid) = extract_uid_gid_monitoring(ctx, daemon_type)
+
+ _stop_and_disable(ctx, 'grafana-server')
+
+ data_dir_dst = make_data_dir(ctx, fsid, daemon_type, daemon_id,
+ uid=uid, gid=gid)
+
+ # config
+ config_src = '/etc/grafana/grafana.ini'
+ config_src = os.path.abspath(ctx.legacy_dir + config_src)
+ config_dst = os.path.join(data_dir_dst, 'etc/grafana')
+ makedirs(config_dst, uid, gid, 0o755)
+ copy_files(ctx, [config_src], config_dst, uid=uid, gid=gid)
+
+ prov_src = '/etc/grafana/provisioning/'
+ prov_src = os.path.abspath(ctx.legacy_dir + prov_src)
+ prov_dst = os.path.join(data_dir_dst, 'etc/grafana')
+ copy_tree(ctx, [prov_src], prov_dst, uid=uid, gid=gid)
+
+ # cert
+ cert = '/etc/grafana/grafana.crt'
+ key = '/etc/grafana/grafana.key'
+ if os.path.exists(cert) and os.path.exists(key):
+ cert_src = '/etc/grafana/grafana.crt'
+ cert_src = os.path.abspath(ctx.legacy_dir + cert_src)
+ makedirs(os.path.join(data_dir_dst, 'etc/grafana/certs'), uid, gid, 0o755)
+ cert_dst = os.path.join(data_dir_dst, 'etc/grafana/certs/cert_file')
+ copy_files(ctx, [cert_src], cert_dst, uid=uid, gid=gid)
+
+ key_src = '/etc/grafana/grafana.key'
+ key_src = os.path.abspath(ctx.legacy_dir + key_src)
+ key_dst = os.path.join(data_dir_dst, 'etc/grafana/certs/cert_key')
+ copy_files(ctx, [key_src], key_dst, uid=uid, gid=gid)
+
+ _adjust_grafana_ini(os.path.join(config_dst, 'grafana.ini'))
+ else:
+ logger.debug('Skipping ssl, missing cert {} or key {}'.format(cert, key))
+
+ # data - possible custom dashboards/plugins
+ data_src = '/var/lib/grafana/'
+ data_src = os.path.abspath(ctx.legacy_dir + data_src)
+ data_dst = os.path.join(data_dir_dst, 'data')
+ copy_tree(ctx, [data_src], data_dst, uid=uid, gid=gid)
+
+ make_var_run(ctx, fsid, uid, gid)
+ c = get_container(ctx, fsid, daemon_type, daemon_id)
+ deploy_daemon(ctx, fsid, daemon_type, daemon_id, c, uid, gid)
+ update_firewalld(ctx, daemon_type)
+
+
+def command_adopt_alertmanager(ctx, daemon_id, fsid):
+ # type: (CephadmContext, str, str) -> None
+
+ daemon_type = 'alertmanager'
+ (uid, gid) = extract_uid_gid_monitoring(ctx, daemon_type)
+
+ _stop_and_disable(ctx, 'prometheus-alertmanager')
+
+ data_dir_dst = make_data_dir(ctx, fsid, daemon_type, daemon_id,
+ uid=uid, gid=gid)
+
+ # config
+ config_src = '/etc/prometheus/alertmanager.yml'
+ config_src = os.path.abspath(ctx.legacy_dir + config_src)
+ config_dst = os.path.join(data_dir_dst, 'etc/alertmanager')
+ makedirs(config_dst, uid, gid, 0o755)
+ copy_files(ctx, [config_src], config_dst, uid=uid, gid=gid)
+
+ # data
+ data_src = '/var/lib/prometheus/alertmanager/'
+ data_src = os.path.abspath(ctx.legacy_dir + data_src)
+ data_dst = os.path.join(data_dir_dst, 'etc/alertmanager/data')
+ copy_tree(ctx, [data_src], data_dst, uid=uid, gid=gid)
+
+ make_var_run(ctx, fsid, uid, gid)
+ c = get_container(ctx, fsid, daemon_type, daemon_id)
+ deploy_daemon(ctx, fsid, daemon_type, daemon_id, c, uid, gid)
+ update_firewalld(ctx, daemon_type)
+
+
+def _adjust_grafana_ini(filename):
+ # type: (str) -> None
+
+ # Update cert_file, cert_key pathnames in server section
+ # ConfigParser does not preserve comments
+ try:
+ with open(filename, 'r') as grafana_ini:
+ lines = grafana_ini.readlines()
+ with open('{}.new'.format(filename), 'w') as grafana_ini:
+ server_section = False
+ for line in lines:
+ if line.startswith('['):
+ server_section = False
+ if line.startswith('[server]'):
+ server_section = True
+ if server_section:
+ line = re.sub(r'^cert_file.*',
+ 'cert_file = /etc/grafana/certs/cert_file', line)
+ line = re.sub(r'^cert_key.*',
+ 'cert_key = /etc/grafana/certs/cert_key', line)
+ grafana_ini.write(line)
+ os.rename('{}.new'.format(filename), filename)
+ except OSError as err:
+ raise Error('Cannot update {}: {}'.format(filename, err))
+
+
+def _stop_and_disable(ctx, unit_name):
+ # type: (CephadmContext, str) -> None
+
+ (enabled, state, _) = check_unit(ctx, unit_name)
+ if state == 'running':
+ logger.info('Stopping old systemd unit %s...' % unit_name)
+ call_throws(ctx, ['systemctl', 'stop', unit_name])
+ if enabled:
+ logger.info('Disabling old systemd unit %s...' % unit_name)
+ call_throws(ctx, ['systemctl', 'disable', unit_name])
+
+##################################
+
+
+def command_rm_daemon(ctx):
+ # type: (CephadmContext) -> None
+ lock = FileLock(ctx, ctx.fsid)
+ lock.acquire()
+
+ (daemon_type, daemon_id) = ctx.name.split('.', 1)
+ unit_name = get_unit_name_by_daemon_name(ctx, ctx.fsid, ctx.name)
+
+ if daemon_type in ['mon', 'osd'] and not ctx.force:
+ raise Error('must pass --force to proceed: '
+ 'this command may destroy precious data!')
+
+ call(ctx, ['systemctl', 'stop', unit_name],
+ verbosity=CallVerbosity.DEBUG)
+ call(ctx, ['systemctl', 'reset-failed', unit_name],
+ verbosity=CallVerbosity.DEBUG)
+ call(ctx, ['systemctl', 'disable', unit_name],
+ verbosity=CallVerbosity.DEBUG)
+ data_dir = get_data_dir(ctx.fsid, ctx.data_dir, daemon_type, daemon_id)
+ if daemon_type in ['mon', 'osd', 'prometheus'] and \
+ not ctx.force_delete_data:
+ # rename it out of the way -- do not delete
+ backup_dir = os.path.join(ctx.data_dir, ctx.fsid, 'removed')
+ if not os.path.exists(backup_dir):
+ makedirs(backup_dir, 0, 0, DATA_DIR_MODE)
+ dirname = '%s.%s_%s' % (daemon_type, daemon_id,
+ datetime.datetime.utcnow().strftime(DATEFMT))
+ os.rename(data_dir,
+ os.path.join(backup_dir, dirname))
+ else:
+ if daemon_type == CephadmDaemon.daemon_type:
+ CephadmDaemon.uninstall(ctx, ctx.fsid, daemon_type, daemon_id)
+ call_throws(ctx, ['rm', '-rf', data_dir])
+
+ if 'tcp_ports' in ctx and ctx.tcp_ports is not None:
+ ports: List[int] = [int(p) for p in ctx.tcp_ports.split()]
+ try:
+ fw = Firewalld(ctx)
+ fw.close_ports(ports)
+ fw.apply_rules()
+ except RuntimeError as e:
+ # in case we cannot close the ports we will remove
+ # the daemon but keep them open.
+ logger.warning(f' Error when trying to close ports: {e}')
+
+
+##################################
+
+
+def _zap(ctx: CephadmContext, what: str) -> None:
+ mounts = get_container_mounts(ctx, ctx.fsid, 'clusterless-ceph-volume', None)
+ c = get_ceph_volume_container(ctx,
+ args=['lvm', 'zap', '--destroy', what],
+ volume_mounts=mounts,
+ envs=ctx.env)
+ logger.info(f'Zapping {what}...')
+ out, err, code = call_throws(ctx, c.run_cmd())
+
+
+@infer_image
+def _zap_osds(ctx: CephadmContext) -> None:
+ # assume fsid lock already held
+
+ # list
+ mounts = get_container_mounts(ctx, ctx.fsid, 'clusterless-ceph-volume', None)
+ c = get_ceph_volume_container(ctx,
+ args=['inventory', '--format', 'json'],
+ volume_mounts=mounts,
+ envs=ctx.env)
+ out, err, code = call_throws(ctx, c.run_cmd())
+ if code:
+ raise Error('failed to list osd inventory')
+ try:
+ ls = json.loads(out)
+ except ValueError as e:
+ raise Error(f'Invalid JSON in ceph-volume inventory: {e}')
+
+ for i in ls:
+ matches = [lv.get('cluster_fsid') == ctx.fsid and i.get('ceph_device') for lv in i.get('lvs', [])]
+ if any(matches) and all(matches):
+ _zap(ctx, i.get('path'))
+ elif any(matches):
+ lv_names = [lv['name'] for lv in i.get('lvs', [])]
+ # TODO: we need to map the lv_names back to device paths (the vg
+ # id isn't part of the output here!)
+ logger.warning(f'Not zapping LVs (not implemented): {lv_names}')
+
+
+def command_zap_osds(ctx: CephadmContext) -> None:
+ if not ctx.force:
+ raise Error('must pass --force to proceed: '
+ 'this command may destroy precious data!')
+
+ lock = FileLock(ctx, ctx.fsid)
+ lock.acquire()
+
+ _zap_osds(ctx)
+
+##################################
+
+
+def get_ceph_cluster_count(ctx: CephadmContext) -> int:
+ return len([c for c in os.listdir(ctx.data_dir) if is_fsid(c)])
+
+
+def command_rm_cluster(ctx):
+ # type: (CephadmContext) -> None
+ if not ctx.force:
+ raise Error('must pass --force to proceed: '
+ 'this command may destroy precious data!')
+
+ lock = FileLock(ctx, ctx.fsid)
+ lock.acquire()
+
+ def disable_systemd_service(unit_name: str) -> None:
+ call(ctx, ['systemctl', 'stop', unit_name],
+ verbosity=CallVerbosity.DEBUG)
+ call(ctx, ['systemctl', 'reset-failed', unit_name],
+ verbosity=CallVerbosity.DEBUG)
+ call(ctx, ['systemctl', 'disable', unit_name],
+ verbosity=CallVerbosity.DEBUG)
+
+ # stop + disable individual daemon units
+ for d in list_daemons(ctx, detail=False):
+ if d['fsid'] != ctx.fsid:
+ continue
+ if d['style'] != 'cephadm:v1':
+ continue
+ disable_systemd_service(get_unit_name(ctx.fsid, d['name']))
+
+ # cluster units
+ for unit_name in ['ceph-%s.target' % ctx.fsid]:
+ disable_systemd_service(unit_name)
+
+ slice_name = 'system-ceph\\x2d{}.slice'.format(ctx.fsid.replace('-', '\\x2d'))
+ call(ctx, ['systemctl', 'stop', slice_name],
+ verbosity=CallVerbosity.DEBUG)
+
+ # osds?
+ if ctx.zap_osds:
+ _zap_osds(ctx)
+
+ # rm units
+ call_throws(ctx, ['rm', '-f', ctx.unit_dir
+ + '/ceph-%s@.service' % ctx.fsid])
+ call_throws(ctx, ['rm', '-f', ctx.unit_dir
+ + '/ceph-%s.target' % ctx.fsid])
+ call_throws(ctx, ['rm', '-rf',
+ ctx.unit_dir + '/ceph-%s.target.wants' % ctx.fsid])
+ # rm data
+ call_throws(ctx, ['rm', '-rf', ctx.data_dir + '/' + ctx.fsid])
+
+ if not ctx.keep_logs:
+ # rm logs
+ call_throws(ctx, ['rm', '-rf', ctx.log_dir + '/' + ctx.fsid])
+ call_throws(ctx, ['rm', '-rf', ctx.log_dir
+ + '/*.wants/ceph-%s@*' % ctx.fsid])
+
+ # rm logrotate config
+ call_throws(ctx, ['rm', '-f', ctx.logrotate_dir + '/ceph-%s' % ctx.fsid])
+
+ # if last cluster on host remove shared files
+ if get_ceph_cluster_count(ctx) == 0:
+ disable_systemd_service('ceph.target')
+
+ # rm shared ceph target files
+ call_throws(ctx, ['rm', '-f', ctx.unit_dir + '/multi-user.target.wants/ceph.target'])
+ call_throws(ctx, ['rm', '-f', ctx.unit_dir + '/ceph.target'])
+
+ # rm cephadm logrotate config
+ call_throws(ctx, ['rm', '-f', ctx.logrotate_dir + '/cephadm'])
+
+ if not ctx.keep_logs:
+ # remove all cephadm logs
+ for fname in glob(f'{ctx.log_dir}/cephadm.log*'):
+ os.remove(fname)
+
+ # rm sysctl settings
+ sysctl_dirs: List[Path] = [Path(ctx.sysctl_dir), Path('/usr/lib/sysctl.d')]
+
+ for sysctl_dir in sysctl_dirs:
+ for p in sysctl_dir.glob(f'90-ceph-{ctx.fsid}-*.conf'):
+ p.unlink()
+
+ # cleanup remaining ceph directories
+ ceph_dirs = [f'/run/ceph/{ctx.fsid}', f'/tmp/var/lib/ceph/{ctx.fsid}', f'/var/run/ceph/{ctx.fsid}']
+ for dd in ceph_dirs:
+ shutil.rmtree(dd, ignore_errors=True)
+
+ # clean up config, keyring, and pub key files
+ files = ['/etc/ceph/ceph.conf', '/etc/ceph/ceph.pub', '/etc/ceph/ceph.client.admin.keyring']
+ if os.path.exists(files[0]):
+ valid_fsid = False
+ with open(files[0]) as f:
+ if ctx.fsid in f.read():
+ valid_fsid = True
+ if valid_fsid:
+ # rm configuration files on /etc/ceph
+ for n in range(0, len(files)):
+ if os.path.exists(files[n]):
+ os.remove(files[n])
+
+##################################
+
+
+def check_time_sync(ctx, enabler=None):
+ # type: (CephadmContext, Optional[Packager]) -> bool
+ units = [
+ 'chrony.service', # 18.04 (at least)
+ 'chronyd.service', # el / opensuse
+ 'systemd-timesyncd.service',
+ 'ntpd.service', # el7 (at least)
+ 'ntp.service', # 18.04 (at least)
+ 'ntpsec.service', # 20.04 (at least) / buster
+ 'openntpd.service', # ubuntu / debian
+ ]
+ if not check_units(ctx, units, enabler):
+ logger.warning('No time sync service is running; checked for %s' % units)
+ return False
+ return True
+
+
+def command_check_host(ctx: CephadmContext) -> None:
+ errors = []
+ commands = ['systemctl', 'lvcreate']
+
+ try:
+ engine = check_container_engine(ctx)
+ logger.info(f'{engine} is present')
+ except Error as e:
+ errors.append(str(e))
+
+ for command in commands:
+ try:
+ find_program(command)
+ logger.info('%s is present' % command)
+ except ValueError:
+ errors.append('%s binary does not appear to be installed' % command)
+
+ # check for configured+running chronyd or ntp
+ if not check_time_sync(ctx):
+ errors.append('No time synchronization is active')
+
+ if 'expect_hostname' in ctx and ctx.expect_hostname:
+ if get_hostname().lower() != ctx.expect_hostname.lower():
+ errors.append('hostname "%s" does not match expected hostname "%s"' % (
+ get_hostname(), ctx.expect_hostname))
+ else:
+ logger.info('Hostname "%s" matches what is expected.',
+ ctx.expect_hostname)
+
+ if errors:
+ raise Error('\nERROR: '.join(errors))
+
+ logger.info('Host looks OK')
+
+##################################
+
+
+def command_prepare_host(ctx: CephadmContext) -> None:
+ logger.info('Verifying podman|docker is present...')
+ pkg = None
+ try:
+ check_container_engine(ctx)
+ except Error as e:
+ logger.warning(str(e))
+ if not pkg:
+ pkg = create_packager(ctx)
+ pkg.install_podman()
+
+ logger.info('Verifying lvm2 is present...')
+ if not find_executable('lvcreate'):
+ if not pkg:
+ pkg = create_packager(ctx)
+ pkg.install(['lvm2'])
+
+ logger.info('Verifying time synchronization is in place...')
+ if not check_time_sync(ctx):
+ if not pkg:
+ pkg = create_packager(ctx)
+ pkg.install(['chrony'])
+ # check again, and this time try to enable
+ # the service
+ check_time_sync(ctx, enabler=pkg)
+
+ if 'expect_hostname' in ctx and ctx.expect_hostname and ctx.expect_hostname != get_hostname():
+ logger.warning('Adjusting hostname from %s -> %s...' % (get_hostname(), ctx.expect_hostname))
+ call_throws(ctx, ['hostname', ctx.expect_hostname])
+ with open('/etc/hostname', 'w') as f:
+ f.write(ctx.expect_hostname + '\n')
+
+ logger.info('Repeating the final host check...')
+ command_check_host(ctx)
+
+##################################
+
+
+class CustomValidation(argparse.Action):
+
+ def _check_name(self, values: str) -> None:
+ try:
+ (daemon_type, daemon_id) = values.split('.', 1)
+ except ValueError:
+ raise argparse.ArgumentError(self,
+ 'must be of the format <type>.<id>. For example, osd.1 or prometheus.myhost.com')
+
+ daemons = get_supported_daemons()
+ if daemon_type not in daemons:
+ raise argparse.ArgumentError(self,
+ 'name must declare the type of daemon e.g. '
+ '{}'.format(', '.join(daemons)))
+
+ def __call__(self, parser: argparse.ArgumentParser, namespace: argparse.Namespace, values: Union[str, Sequence[Any], None],
+ option_string: Optional[str] = None) -> None:
+ assert isinstance(values, str)
+ if self.dest == 'name':
+ self._check_name(values)
+ setattr(namespace, self.dest, values)
+ elif self.dest == 'exporter_config':
+ cfg = get_parm(values)
+ # run the class' validate method, and convert to an argparse error
+ # if problems are found
+ try:
+ CephadmDaemon.validate_config(cfg)
+ except Error as e:
+ raise argparse.ArgumentError(self,
+ str(e))
+ setattr(namespace, self.dest, cfg)
+
+##################################
+
+
+def get_distro():
+ # type: () -> Tuple[Optional[str], Optional[str], Optional[str]]
+ distro = None
+ distro_version = None
+ distro_codename = None
+ with open('/etc/os-release', 'r') as f:
+ for line in f.readlines():
+ line = line.strip()
+ if '=' not in line or line.startswith('#'):
+ continue
+ (var, val) = line.split('=', 1)
+ if val[0] == '"' and val[-1] == '"':
+ val = val[1:-1]
+ if var == 'ID':
+ distro = val.lower()
+ elif var == 'VERSION_ID':
+ distro_version = val.lower()
+ elif var == 'VERSION_CODENAME':
+ distro_codename = val.lower()
+ return distro, distro_version, distro_codename
+
+
+class Packager(object):
+ def __init__(self, ctx: CephadmContext,
+ stable: Optional[str] = None, version: Optional[str] = None,
+ branch: Optional[str] = None, commit: Optional[str] = None):
+ assert \
+ (stable and not version and not branch and not commit) or \
+ (not stable and version and not branch and not commit) or \
+ (not stable and not version and branch) or \
+ (not stable and not version and not branch and not commit)
+ self.ctx = ctx
+ self.stable = stable
+ self.version = version
+ self.branch = branch
+ self.commit = commit
+
+ def validate(self) -> None:
+ """Validate parameters before writing any state to disk."""
+ pass
+
+ def add_repo(self) -> None:
+ raise NotImplementedError
+
+ def rm_repo(self) -> None:
+ raise NotImplementedError
+
+ def install(self, ls: List[str]) -> None:
+ raise NotImplementedError
+
+ def install_podman(self) -> None:
+ raise NotImplementedError
+
+ def query_shaman(self, distro: str, distro_version: Any, branch: Optional[str], commit: Optional[str]) -> str:
+ # query shaman
+ logger.info('Fetching repo metadata from shaman and chacra...')
+ shaman_url = 'https://shaman.ceph.com/api/repos/ceph/{branch}/{sha1}/{distro}/{distro_version}/repo/?arch={arch}'.format(
+ distro=distro,
+ distro_version=distro_version,
+ branch=branch,
+ sha1=commit or 'latest',
+ arch=get_arch()
+ )
+ try:
+ shaman_response = urlopen(shaman_url)
+ except HTTPError as err:
+ logger.error('repository not found in shaman (might not be available yet)')
+ raise Error('%s, failed to fetch %s' % (err, shaman_url))
+ chacra_url = ''
+ try:
+ chacra_url = shaman_response.geturl()
+ chacra_response = urlopen(chacra_url)
+ except HTTPError as err:
+ logger.error('repository not found in chacra (might not be available yet)')
+ raise Error('%s, failed to fetch %s' % (err, chacra_url))
+ return chacra_response.read().decode('utf-8')
+
+ def repo_gpgkey(self) -> Tuple[str, str]:
+ if self.ctx.gpg_url:
+ return self.ctx.gpg_url, 'manual'
+ if self.stable or self.version:
+ return 'https://download.ceph.com/keys/release.gpg', 'release'
+ else:
+ return 'https://download.ceph.com/keys/autobuild.gpg', 'autobuild'
+
+ def enable_service(self, service: str) -> None:
+ """
+ Start and enable the service (typically using systemd).
+ """
+ call_throws(self.ctx, ['systemctl', 'enable', '--now', service])
+
+
+class Apt(Packager):
+ DISTRO_NAMES = {
+ 'ubuntu': 'ubuntu',
+ 'debian': 'debian',
+ }
+
+ def __init__(self, ctx: CephadmContext,
+ stable: Optional[str], version: Optional[str], branch: Optional[str], commit: Optional[str],
+ distro: Optional[str], distro_version: Optional[str], distro_codename: Optional[str]) -> None:
+ super(Apt, self).__init__(ctx, stable=stable, version=version,
+ branch=branch, commit=commit)
+ assert distro
+ self.ctx = ctx
+ self.distro = self.DISTRO_NAMES[distro]
+ self.distro_codename = distro_codename
+ self.distro_version = distro_version
+
+ def repo_path(self) -> str:
+ return '/etc/apt/sources.list.d/ceph.list'
+
+ def add_repo(self) -> None:
+
+ url, name = self.repo_gpgkey()
+ logger.info('Installing repo GPG key from %s...' % url)
+ try:
+ response = urlopen(url)
+ except HTTPError as err:
+ logger.error('failed to fetch GPG repo key from %s: %s' % (
+ url, err))
+ raise Error('failed to fetch GPG key')
+ key = response.read()
+ with open('/etc/apt/trusted.gpg.d/ceph.%s.gpg' % name, 'wb') as f:
+ f.write(key)
+
+ if self.version:
+ content = 'deb %s/debian-%s/ %s main\n' % (
+ self.ctx.repo_url, self.version, self.distro_codename)
+ elif self.stable:
+ content = 'deb %s/debian-%s/ %s main\n' % (
+ self.ctx.repo_url, self.stable, self.distro_codename)
+ else:
+ content = self.query_shaman(self.distro, self.distro_codename, self.branch,
+ self.commit)
+
+ logger.info('Installing repo file at %s...' % self.repo_path())
+ with open(self.repo_path(), 'w') as f:
+ f.write(content)
+
+ self.update()
+
+ def rm_repo(self) -> None:
+ for name in ['autobuild', 'release', 'manual']:
+ p = '/etc/apt/trusted.gpg.d/ceph.%s.gpg' % name
+ if os.path.exists(p):
+ logger.info('Removing repo GPG key %s...' % p)
+ os.unlink(p)
+ if os.path.exists(self.repo_path()):
+ logger.info('Removing repo at %s...' % self.repo_path())
+ os.unlink(self.repo_path())
+
+ if self.distro == 'ubuntu':
+ self.rm_kubic_repo()
+
+ def install(self, ls: List[str]) -> None:
+ logger.info('Installing packages %s...' % ls)
+ call_throws(self.ctx, ['apt-get', 'install', '-y'] + ls)
+
+ def update(self) -> None:
+ logger.info('Updating package list...')
+ call_throws(self.ctx, ['apt-get', 'update'])
+
+ def install_podman(self) -> None:
+ if self.distro == 'ubuntu':
+ logger.info('Setting up repo for podman...')
+ self.add_kubic_repo()
+ self.update()
+
+ logger.info('Attempting podman install...')
+ try:
+ self.install(['podman'])
+ except Error:
+ logger.info('Podman did not work. Falling back to docker...')
+ self.install(['docker.io'])
+
+ def kubic_repo_url(self) -> str:
+ return 'https://download.opensuse.org/repositories/devel:/kubic:/' \
+ 'libcontainers:/stable/xUbuntu_%s/' % self.distro_version
+
+ def kubic_repo_path(self) -> str:
+ return '/etc/apt/sources.list.d/devel:kubic:libcontainers:stable.list'
+
+ def kubric_repo_gpgkey_url(self) -> str:
+ return '%s/Release.key' % self.kubic_repo_url()
+
+ def kubric_repo_gpgkey_path(self) -> str:
+ return '/etc/apt/trusted.gpg.d/kubic.release.gpg'
+
+ def add_kubic_repo(self) -> None:
+ url = self.kubric_repo_gpgkey_url()
+ logger.info('Installing repo GPG key from %s...' % url)
+ try:
+ response = urlopen(url)
+ except HTTPError as err:
+ logger.error('failed to fetch GPG repo key from %s: %s' % (
+ url, err))
+ raise Error('failed to fetch GPG key')
+ key = response.read().decode('utf-8')
+ tmp_key = write_tmp(key, 0, 0)
+ keyring = self.kubric_repo_gpgkey_path()
+ call_throws(self.ctx, ['apt-key', '--keyring', keyring, 'add', tmp_key.name])
+
+ logger.info('Installing repo file at %s...' % self.kubic_repo_path())
+ content = 'deb %s /\n' % self.kubic_repo_url()
+ with open(self.kubic_repo_path(), 'w') as f:
+ f.write(content)
+
+ def rm_kubic_repo(self) -> None:
+ keyring = self.kubric_repo_gpgkey_path()
+ if os.path.exists(keyring):
+ logger.info('Removing repo GPG key %s...' % keyring)
+ os.unlink(keyring)
+
+ p = self.kubic_repo_path()
+ if os.path.exists(p):
+ logger.info('Removing repo at %s...' % p)
+ os.unlink(p)
+
+
+class YumDnf(Packager):
+ DISTRO_NAMES = {
+ 'centos': ('centos', 'el'),
+ 'rhel': ('centos', 'el'),
+ 'scientific': ('centos', 'el'),
+ 'rocky': ('centos', 'el'),
+ 'almalinux': ('centos', 'el'),
+ 'ol': ('centos', 'el'),
+ 'fedora': ('fedora', 'fc'),
+ 'mariner': ('mariner', 'cm'),
+ }
+
+ def __init__(self, ctx: CephadmContext,
+ stable: Optional[str], version: Optional[str], branch: Optional[str], commit: Optional[str],
+ distro: Optional[str], distro_version: Optional[str]) -> None:
+ super(YumDnf, self).__init__(ctx, stable=stable, version=version,
+ branch=branch, commit=commit)
+ assert distro
+ assert distro_version
+ self.ctx = ctx
+ self.major = int(distro_version.split('.')[0])
+ self.distro_normalized = self.DISTRO_NAMES[distro][0]
+ self.distro_code = self.DISTRO_NAMES[distro][1] + str(self.major)
+ if (self.distro_code == 'fc' and self.major >= 30) or \
+ (self.distro_code == 'el' and self.major >= 8):
+ self.tool = 'dnf'
+ elif (self.distro_code == 'cm'):
+ self.tool = 'tdnf'
+ else:
+ self.tool = 'yum'
+
+ def custom_repo(self, **kw: Any) -> str:
+ """
+ Repo files need special care in that a whole line should not be present
+ if there is no value for it. Because we were using `format()` we could
+ not conditionally add a line for a repo file. So the end result would
+ contain a key with a missing value (say if we were passing `None`).
+
+ For example, it could look like::
+
+ [ceph repo]
+ name= ceph repo
+ proxy=
+ gpgcheck=
+
+ Which breaks. This function allows us to conditionally add lines,
+ preserving an order and be more careful.
+
+ Previously, and for historical purposes, this is how the template used
+ to look::
+
+ custom_repo =
+ [{repo_name}]
+ name={name}
+ baseurl={baseurl}
+ enabled={enabled}
+ gpgcheck={gpgcheck}
+ type={_type}
+ gpgkey={gpgkey}
+ proxy={proxy}
+
+ """
+ lines = []
+
+ # by using tuples (vs a dict) we preserve the order of what we want to
+ # return, like starting with a [repo name]
+ tmpl = (
+ ('reponame', '[%s]'),
+ ('name', 'name=%s'),
+ ('baseurl', 'baseurl=%s'),
+ ('enabled', 'enabled=%s'),
+ ('gpgcheck', 'gpgcheck=%s'),
+ ('_type', 'type=%s'),
+ ('gpgkey', 'gpgkey=%s'),
+ ('proxy', 'proxy=%s'),
+ ('priority', 'priority=%s'),
+ )
+
+ for line in tmpl:
+ tmpl_key, tmpl_value = line # key values from tmpl
+
+ # ensure that there is an actual value (not None nor empty string)
+ if tmpl_key in kw and kw.get(tmpl_key) not in (None, ''):
+ lines.append(tmpl_value % kw.get(tmpl_key))
+
+ return '\n'.join(lines)
+
+ def repo_path(self) -> str:
+ return '/etc/yum.repos.d/ceph.repo'
+
+ def repo_baseurl(self) -> str:
+ assert self.stable or self.version
+ if self.version:
+ return '%s/rpm-%s/%s' % (self.ctx.repo_url, self.version,
+ self.distro_code)
+ else:
+ return '%s/rpm-%s/%s' % (self.ctx.repo_url, self.stable,
+ self.distro_code)
+
+ def validate(self) -> None:
+ if self.distro_code.startswith('fc'):
+ raise Error('Ceph team does not build Fedora specific packages and therefore cannot add repos for this distro')
+ if self.distro_code == 'el7':
+ if self.stable and self.stable >= 'pacific':
+ raise Error('Ceph does not support pacific or later for this version of this linux distro and therefore cannot add a repo for it')
+ if self.version and self.version.split('.')[0] >= '16':
+ raise Error('Ceph does not support 16.y.z or later for this version of this linux distro and therefore cannot add a repo for it')
+
+ if self.stable or self.version:
+ # we know that yum & dnf require there to be a
+ # $base_url/$arch/repodata/repomd.xml so we can test if this URL
+ # is gettable in order to validate the inputs
+ test_url = self.repo_baseurl() + '/noarch/repodata/repomd.xml'
+ try:
+ urlopen(test_url)
+ except HTTPError as err:
+ logger.error('unable to fetch repo metadata: %r', err)
+ raise Error('failed to fetch repository metadata. please check'
+ ' the provided parameters are correct and try again')
+
+ def add_repo(self) -> None:
+ if self.stable or self.version:
+ content = ''
+ for n, t in {
+ 'Ceph': '$basearch',
+ 'Ceph-noarch': 'noarch',
+ 'Ceph-source': 'SRPMS'}.items():
+ content += '[%s]\n' % (n)
+ content += self.custom_repo(
+ name='Ceph %s' % t,
+ baseurl=self.repo_baseurl() + '/' + t,
+ enabled=1,
+ gpgcheck=1,
+ gpgkey=self.repo_gpgkey()[0],
+ )
+ content += '\n\n'
+ else:
+ content = self.query_shaman(self.distro_normalized, self.major,
+ self.branch,
+ self.commit)
+
+ logger.info('Writing repo to %s...' % self.repo_path())
+ with open(self.repo_path(), 'w') as f:
+ f.write(content)
+
+ if self.distro_code.startswith('el'):
+ logger.info('Enabling EPEL...')
+ call_throws(self.ctx, [self.tool, 'install', '-y', 'epel-release'])
+
+ def rm_repo(self) -> None:
+ if os.path.exists(self.repo_path()):
+ os.unlink(self.repo_path())
+
+ def install(self, ls: List[str]) -> None:
+ logger.info('Installing packages %s...' % ls)
+ call_throws(self.ctx, [self.tool, 'install', '-y'] + ls)
+
+ def install_podman(self) -> None:
+ self.install(['podman'])
+
+
+class Zypper(Packager):
+ DISTRO_NAMES = [
+ 'sles',
+ 'opensuse-tumbleweed',
+ 'opensuse-leap'
+ ]
+
+ def __init__(self, ctx: CephadmContext,
+ stable: Optional[str], version: Optional[str], branch: Optional[str], commit: Optional[str],
+ distro: Optional[str], distro_version: Optional[str]) -> None:
+ super(Zypper, self).__init__(ctx, stable=stable, version=version,
+ branch=branch, commit=commit)
+ assert distro is not None
+ self.ctx = ctx
+ self.tool = 'zypper'
+ self.distro = 'opensuse'
+ self.distro_version = '15.1'
+ if 'tumbleweed' not in distro and distro_version is not None:
+ self.distro_version = distro_version
+
+ def custom_repo(self, **kw: Any) -> str:
+ """
+ See YumDnf for format explanation.
+ """
+ lines = []
+
+ # by using tuples (vs a dict) we preserve the order of what we want to
+ # return, like starting with a [repo name]
+ tmpl = (
+ ('reponame', '[%s]'),
+ ('name', 'name=%s'),
+ ('baseurl', 'baseurl=%s'),
+ ('enabled', 'enabled=%s'),
+ ('gpgcheck', 'gpgcheck=%s'),
+ ('_type', 'type=%s'),
+ ('gpgkey', 'gpgkey=%s'),
+ ('proxy', 'proxy=%s'),
+ ('priority', 'priority=%s'),
+ )
+
+ for line in tmpl:
+ tmpl_key, tmpl_value = line # key values from tmpl
+
+ # ensure that there is an actual value (not None nor empty string)
+ if tmpl_key in kw and kw.get(tmpl_key) not in (None, ''):
+ lines.append(tmpl_value % kw.get(tmpl_key))
+
+ return '\n'.join(lines)
+
+ def repo_path(self) -> str:
+ return '/etc/zypp/repos.d/ceph.repo'
+
+ def repo_baseurl(self) -> str:
+ assert self.stable or self.version
+ if self.version:
+ return '%s/rpm-%s/%s' % (self.ctx.repo_url,
+ self.stable, self.distro)
+ else:
+ return '%s/rpm-%s/%s' % (self.ctx.repo_url,
+ self.stable, self.distro)
+
+ def add_repo(self) -> None:
+ if self.stable or self.version:
+ content = ''
+ for n, t in {
+ 'Ceph': '$basearch',
+ 'Ceph-noarch': 'noarch',
+ 'Ceph-source': 'SRPMS'}.items():
+ content += '[%s]\n' % (n)
+ content += self.custom_repo(
+ name='Ceph %s' % t,
+ baseurl=self.repo_baseurl() + '/' + t,
+ enabled=1,
+ gpgcheck=1,
+ gpgkey=self.repo_gpgkey()[0],
+ )
+ content += '\n\n'
+ else:
+ content = self.query_shaman(self.distro, self.distro_version,
+ self.branch,
+ self.commit)
+
+ logger.info('Writing repo to %s...' % self.repo_path())
+ with open(self.repo_path(), 'w') as f:
+ f.write(content)
+
+ def rm_repo(self) -> None:
+ if os.path.exists(self.repo_path()):
+ os.unlink(self.repo_path())
+
+ def install(self, ls: List[str]) -> None:
+ logger.info('Installing packages %s...' % ls)
+ call_throws(self.ctx, [self.tool, 'in', '-y'] + ls)
+
+ def install_podman(self) -> None:
+ self.install(['podman'])
+
+
+def create_packager(ctx: CephadmContext,
+ stable: Optional[str] = None, version: Optional[str] = None,
+ branch: Optional[str] = None, commit: Optional[str] = None) -> Packager:
+ distro, distro_version, distro_codename = get_distro()
+ if distro in YumDnf.DISTRO_NAMES:
+ return YumDnf(ctx, stable=stable, version=version,
+ branch=branch, commit=commit,
+ distro=distro, distro_version=distro_version)
+ elif distro in Apt.DISTRO_NAMES:
+ return Apt(ctx, stable=stable, version=version,
+ branch=branch, commit=commit,
+ distro=distro, distro_version=distro_version,
+ distro_codename=distro_codename)
+ elif distro in Zypper.DISTRO_NAMES:
+ return Zypper(ctx, stable=stable, version=version,
+ branch=branch, commit=commit,
+ distro=distro, distro_version=distro_version)
+ raise Error('Distro %s version %s not supported' % (distro, distro_version))
+
+
+def command_add_repo(ctx: CephadmContext) -> None:
+ if ctx.version and ctx.release:
+ raise Error('you can specify either --release or --version but not both')
+ if not ctx.version and not ctx.release and not ctx.dev and not ctx.dev_commit:
+ raise Error('please supply a --release, --version, --dev or --dev-commit argument')
+ if ctx.version:
+ try:
+ (x, y, z) = ctx.version.split('.')
+ except Exception:
+ raise Error('version must be in the form x.y.z (e.g., 15.2.0)')
+ if ctx.release:
+ # Pacific =/= pacific in this case, set to undercase to avoid confision
+ ctx.release = ctx.release.lower()
+
+ pkg = create_packager(ctx, stable=ctx.release,
+ version=ctx.version,
+ branch=ctx.dev,
+ commit=ctx.dev_commit)
+ pkg.validate()
+ pkg.add_repo()
+ logger.info('Completed adding repo.')
+
+
+def command_rm_repo(ctx: CephadmContext) -> None:
+ pkg = create_packager(ctx)
+ pkg.rm_repo()
+
+
+def command_install(ctx: CephadmContext) -> None:
+ pkg = create_packager(ctx)
+ pkg.install(ctx.packages)
+
+
+def command_rescan_disks(ctx: CephadmContext) -> str:
+
+ def probe_hba(scan_path: str) -> None:
+ """Tell the adapter to rescan"""
+ with open(scan_path, 'w') as f:
+ f.write('- - -')
+
+ cmd = ctx.func.__name__.replace('command_', '')
+ logger.info(f'{cmd}: starting')
+ start = time.time()
+
+ all_scan_files = glob('/sys/class/scsi_host/*/scan')
+ scan_files = []
+ skipped = []
+ for scan_path in all_scan_files:
+ adapter_name = os.path.basename(os.path.dirname(scan_path))
+ proc_name = read_file([os.path.join(os.path.dirname(scan_path), 'proc_name')])
+ if proc_name in ['unknown', 'usb-storage']:
+ skipped.append(os.path.basename(scan_path))
+ logger.info(f'{cmd}: rescan skipping incompatible host adapter {adapter_name} : {proc_name}')
+ continue
+
+ scan_files.append(scan_path)
+
+ if not scan_files:
+ logger.info(f'{cmd}: no compatible HBAs found')
+ return 'Ok. No compatible HBAs found'
+
+ responses = async_run(concurrent_tasks(probe_hba, scan_files))
+ failures = [r for r in responses if r]
+
+ logger.info(f'{cmd}: Complete. {len(scan_files)} adapters rescanned, {len(failures)} failures, {len(skipped)} skipped')
+
+ elapsed = time.time() - start
+ if failures:
+ plural = 's' if len(failures) > 1 else ''
+ if len(failures) == len(scan_files):
+ return f'Failed. All {len(scan_files)} rescan requests failed'
+ else:
+ return f'Partial. {len(scan_files) - len(failures)} successful, {len(failures)} failure{plural} against: {", ".join(failures)}'
+
+ return f'Ok. {len(all_scan_files)} adapters detected: {len(scan_files)} rescanned, {len(skipped)} skipped, {len(failures)} failed ({elapsed:.2f}s)'
+
+##################################
+
+
+def get_ipv4_address(ifname):
+ # type: (str) -> str
+ def _extract(sock: socket.socket, offset: int) -> str:
+ return socket.inet_ntop(
+ socket.AF_INET,
+ fcntl.ioctl(
+ sock.fileno(),
+ offset,
+ struct.pack('256s', bytes(ifname[:15], 'utf-8'))
+ )[20:24])
+
+ s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+ try:
+ addr = _extract(s, 35093) # '0x8915' = SIOCGIFADDR
+ dq_mask = _extract(s, 35099) # 0x891b = SIOCGIFNETMASK
+ except OSError:
+ # interface does not have an ipv4 address
+ return ''
+
+ dec_mask = sum([bin(int(i)).count('1')
+ for i in dq_mask.split('.')])
+ return '{}/{}'.format(addr, dec_mask)
+
+
+def get_ipv6_address(ifname):
+ # type: (str) -> str
+ if not os.path.exists('/proc/net/if_inet6'):
+ return ''
+
+ raw = read_file(['/proc/net/if_inet6'])
+ data = raw.splitlines()
+ # based on docs @ https://www.tldp.org/HOWTO/Linux+IPv6-HOWTO/ch11s04.html
+ # field 0 is ipv6, field 2 is scope
+ for iface_setting in data:
+ field = iface_setting.split()
+ if field[-1] == ifname:
+ ipv6_raw = field[0]
+ ipv6_fmtd = ':'.join([ipv6_raw[_p:_p + 4] for _p in range(0, len(field[0]), 4)])
+ # apply naming rules using ipaddress module
+ ipv6 = ipaddress.ip_address(ipv6_fmtd)
+ return '{}/{}'.format(str(ipv6), int('0x{}'.format(field[2]), 16))
+ return ''
+
+
+def bytes_to_human(num, mode='decimal'):
+ # type: (float, str) -> str
+ """Convert a bytes value into it's human-readable form.
+
+ :param num: number, in bytes, to convert
+ :param mode: Either decimal (default) or binary to determine divisor
+ :returns: string representing the bytes value in a more readable format
+ """
+ unit_list = ['', 'KB', 'MB', 'GB', 'TB', 'PB', 'EB', 'ZB']
+ divisor = 1000.0
+ yotta = 'YB'
+
+ if mode == 'binary':
+ unit_list = ['', 'KiB', 'MiB', 'GiB', 'TiB', 'PiB', 'EiB', 'ZiB']
+ divisor = 1024.0
+ yotta = 'YiB'
+
+ for unit in unit_list:
+ if abs(num) < divisor:
+ return '%3.1f%s' % (num, unit)
+ num /= divisor
+ return '%.1f%s' % (num, yotta)
+
+
+def read_file(path_list, file_name=''):
+ # type: (List[str], str) -> str
+ """Returns the content of the first file found within the `path_list`
+
+ :param path_list: list of file paths to search
+ :param file_name: optional file_name to be applied to a file path
+ :returns: content of the file or 'Unknown'
+ """
+ for path in path_list:
+ if file_name:
+ file_path = os.path.join(path, file_name)
+ else:
+ file_path = path
+ if os.path.exists(file_path):
+ with open(file_path, 'r') as f:
+ try:
+ content = f.read().strip()
+ except OSError:
+ # sysfs may populate the file, but for devices like
+ # virtio reads can fail
+ return 'Unknown'
+ else:
+ return content
+ return 'Unknown'
+
+##################################
+
+
+class HostFacts():
+ _dmi_path_list = ['/sys/class/dmi/id']
+ _nic_path_list = ['/sys/class/net']
+ _apparmor_path_list = ['/etc/apparmor']
+ _disk_vendor_workarounds = {
+ '0x1af4': 'Virtio Block Device'
+ }
+ _excluded_block_devices = ('sr', 'zram', 'dm-')
+
+ def __init__(self, ctx: CephadmContext):
+ self.ctx: CephadmContext = ctx
+ self.cpu_model: str = 'Unknown'
+ self.cpu_count: int = 0
+ self.cpu_cores: int = 0
+ self.cpu_threads: int = 0
+ self.interfaces: Dict[str, Any] = {}
+
+ self._meminfo: List[str] = read_file(['/proc/meminfo']).splitlines()
+ self._get_cpuinfo()
+ self._process_nics()
+ self.arch: str = platform.processor()
+ self.kernel: str = platform.release()
+
+ def _get_cpuinfo(self):
+ # type: () -> None
+ """Determine cpu information via /proc/cpuinfo"""
+ raw = read_file(['/proc/cpuinfo'])
+ output = raw.splitlines()
+ cpu_set = set()
+
+ for line in output:
+ field = [f.strip() for f in line.split(':')]
+ if 'model name' in line:
+ self.cpu_model = field[1]
+ if 'physical id' in line:
+ cpu_set.add(field[1])
+ if 'siblings' in line:
+ self.cpu_threads = int(field[1].strip())
+ if 'cpu cores' in line:
+ self.cpu_cores = int(field[1].strip())
+ pass
+ self.cpu_count = len(cpu_set)
+
+ def _get_block_devs(self):
+ # type: () -> List[str]
+ """Determine the list of block devices by looking at /sys/block"""
+ return [dev for dev in os.listdir('/sys/block')
+ if not dev.startswith(HostFacts._excluded_block_devices)]
+
+ def _get_devs_by_type(self, rota='0'):
+ # type: (str) -> List[str]
+ """Filter block devices by a given rotational attribute (0=flash, 1=spinner)"""
+ devs = list()
+ for blk_dev in self._get_block_devs():
+ rot_path = '/sys/block/{}/queue/rotational'.format(blk_dev)
+ rot_value = read_file([rot_path])
+ if rot_value == rota:
+ devs.append(blk_dev)
+ return devs
+
+ @property
+ def operating_system(self):
+ # type: () -> str
+ """Determine OS version"""
+ raw_info = read_file(['/etc/os-release'])
+ os_release = raw_info.splitlines()
+ rel_str = 'Unknown'
+ rel_dict = dict()
+
+ for line in os_release:
+ if '=' in line:
+ var_name, var_value = line.split('=')
+ rel_dict[var_name] = var_value.strip('"')
+
+ # Would normally use PRETTY_NAME, but NAME and VERSION are more
+ # consistent
+ if all(_v in rel_dict for _v in ['NAME', 'VERSION']):
+ rel_str = '{} {}'.format(rel_dict['NAME'], rel_dict['VERSION'])
+ return rel_str
+
+ @property
+ def hostname(self):
+ # type: () -> str
+ """Return the hostname"""
+ return platform.node()
+
+ @property
+ def subscribed(self):
+ # type: () -> str
+ """Highlevel check to see if the host is subscribed to receive updates/support"""
+ def _red_hat():
+ # type: () -> str
+ # RHEL 7 and RHEL 8
+ entitlements_dir = '/etc/pki/entitlement'
+ if os.path.exists(entitlements_dir):
+ pems = glob('{}/*.pem'.format(entitlements_dir))
+ if len(pems) >= 2:
+ return 'Yes'
+
+ return 'No'
+
+ os_name = self.operating_system
+ if os_name.upper().startswith('RED HAT'):
+ return _red_hat()
+
+ return 'Unknown'
+
+ @property
+ def hdd_count(self):
+ # type: () -> int
+ """Return a count of HDDs (spinners)"""
+ return len(self._get_devs_by_type(rota='1'))
+
+ def _get_capacity(self, dev):
+ # type: (str) -> int
+ """Determine the size of a given device
+
+ The kernel always bases device size calculations based on a 512 byte
+ sector. For more information see
+ https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git/tree/include/linux/types.h?h=v5.15.63#n120
+ """
+ size_path = os.path.join('/sys/block', dev, 'size')
+ size_blocks = int(read_file([size_path]))
+ return size_blocks * 512
+
+ def _get_capacity_by_type(self, rota='0'):
+ # type: (str) -> int
+ """Return the total capacity of a category of device (flash or hdd)"""
+ devs = self._get_devs_by_type(rota=rota)
+ capacity = 0
+ for dev in devs:
+ capacity += self._get_capacity(dev)
+ return capacity
+
+ def _dev_list(self, dev_list):
+ # type: (List[str]) -> List[Dict[str, object]]
+ """Return a 'pretty' name list for each device in the `dev_list`"""
+ disk_list = list()
+
+ for dev in dev_list:
+ disk_model = read_file(['/sys/block/{}/device/model'.format(dev)]).strip()
+ disk_rev = read_file(['/sys/block/{}/device/rev'.format(dev)]).strip()
+ disk_wwid = read_file(['/sys/block/{}/device/wwid'.format(dev)]).strip()
+ vendor = read_file(['/sys/block/{}/device/vendor'.format(dev)]).strip()
+ disk_vendor = HostFacts._disk_vendor_workarounds.get(vendor, vendor)
+ disk_size_bytes = self._get_capacity(dev)
+ disk_list.append({
+ 'description': '{} {} ({})'.format(disk_vendor, disk_model, bytes_to_human(disk_size_bytes)),
+ 'vendor': disk_vendor,
+ 'model': disk_model,
+ 'rev': disk_rev,
+ 'wwid': disk_wwid,
+ 'dev_name': dev,
+ 'disk_size_bytes': disk_size_bytes,
+ })
+ return disk_list
+
+ @property
+ def hdd_list(self):
+ # type: () -> List[Dict[str, object]]
+ """Return a list of devices that are HDDs (spinners)"""
+ devs = self._get_devs_by_type(rota='1')
+ return self._dev_list(devs)
+
+ @property
+ def flash_list(self):
+ # type: () -> List[Dict[str, object]]
+ """Return a list of devices that are flash based (SSD, NVMe)"""
+ devs = self._get_devs_by_type(rota='0')
+ return self._dev_list(devs)
+
+ @property
+ def hdd_capacity_bytes(self):
+ # type: () -> int
+ """Return the total capacity for all HDD devices (bytes)"""
+ return self._get_capacity_by_type(rota='1')
+
+ @property
+ def hdd_capacity(self):
+ # type: () -> str
+ """Return the total capacity for all HDD devices (human readable format)"""
+ return bytes_to_human(self.hdd_capacity_bytes)
+
+ @property
+ def cpu_load(self):
+ # type: () -> Dict[str, float]
+ """Return the cpu load average data for the host"""
+ raw = read_file(['/proc/loadavg']).strip()
+ data = raw.split()
+ return {
+ '1min': float(data[0]),
+ '5min': float(data[1]),
+ '15min': float(data[2]),
+ }
+
+ @property
+ def flash_count(self):
+ # type: () -> int
+ """Return the number of flash devices in the system (SSD, NVMe)"""
+ return len(self._get_devs_by_type(rota='0'))
+
+ @property
+ def flash_capacity_bytes(self):
+ # type: () -> int
+ """Return the total capacity for all flash devices (bytes)"""
+ return self._get_capacity_by_type(rota='0')
+
+ @property
+ def flash_capacity(self):
+ # type: () -> str
+ """Return the total capacity for all Flash devices (human readable format)"""
+ return bytes_to_human(self.flash_capacity_bytes)
+
+ def _process_nics(self):
+ # type: () -> None
+ """Look at the NIC devices and extract network related metadata"""
+ # from https://github.com/torvalds/linux/blob/master/include/uapi/linux/if_arp.h
+ hw_lookup = {
+ '1': 'ethernet',
+ '32': 'infiniband',
+ '772': 'loopback',
+ }
+
+ for nic_path in HostFacts._nic_path_list:
+ if not os.path.exists(nic_path):
+ continue
+ for iface in os.listdir(nic_path):
+
+ lower_devs_list = [os.path.basename(link.replace('lower_', '')) for link in glob(os.path.join(nic_path, iface, 'lower_*'))]
+ upper_devs_list = [os.path.basename(link.replace('upper_', '')) for link in glob(os.path.join(nic_path, iface, 'upper_*'))]
+
+ try:
+ mtu = int(read_file([os.path.join(nic_path, iface, 'mtu')]))
+ except ValueError:
+ mtu = 0
+
+ operstate = read_file([os.path.join(nic_path, iface, 'operstate')])
+ try:
+ speed = int(read_file([os.path.join(nic_path, iface, 'speed')]))
+ except (OSError, ValueError):
+ # OSError : device doesn't support the ethtool get_link_ksettings
+ # ValueError : raised when the read fails, and returns Unknown
+ #
+ # Either way, we show a -1 when speed isn't available
+ speed = -1
+
+ if os.path.exists(os.path.join(nic_path, iface, 'bridge')):
+ nic_type = 'bridge'
+ elif os.path.exists(os.path.join(nic_path, iface, 'bonding')):
+ nic_type = 'bonding'
+ else:
+ nic_type = hw_lookup.get(read_file([os.path.join(nic_path, iface, 'type')]), 'Unknown')
+
+ dev_link = os.path.join(nic_path, iface, 'device')
+ if os.path.exists(dev_link):
+ iftype = 'physical'
+ driver_path = os.path.join(dev_link, 'driver')
+ if os.path.exists(driver_path):
+ driver = os.path.basename(os.path.realpath(driver_path))
+ else:
+ driver = 'Unknown'
+
+ else:
+ iftype = 'logical'
+ driver = ''
+
+ self.interfaces[iface] = {
+ 'mtu': mtu,
+ 'upper_devs_list': upper_devs_list,
+ 'lower_devs_list': lower_devs_list,
+ 'operstate': operstate,
+ 'iftype': iftype,
+ 'nic_type': nic_type,
+ 'driver': driver,
+ 'speed': speed,
+ 'ipv4_address': get_ipv4_address(iface),
+ 'ipv6_address': get_ipv6_address(iface),
+ }
+
+ @property
+ def nic_count(self):
+ # type: () -> int
+ """Return a total count of all physical NICs detected in the host"""
+ phys_devs = []
+ for iface in self.interfaces:
+ if self.interfaces[iface]['iftype'] == 'physical':
+ phys_devs.append(iface)
+ return len(phys_devs)
+
+ def _get_mem_data(self, field_name):
+ # type: (str) -> int
+ for line in self._meminfo:
+ if line.startswith(field_name):
+ _d = line.split()
+ return int(_d[1])
+ return 0
+
+ @property
+ def memory_total_kb(self):
+ # type: () -> int
+ """Determine the memory installed (kb)"""
+ return self._get_mem_data('MemTotal')
+
+ @property
+ def memory_free_kb(self):
+ # type: () -> int
+ """Determine the memory free (not cache, immediately usable)"""
+ return self._get_mem_data('MemFree')
+
+ @property
+ def memory_available_kb(self):
+ # type: () -> int
+ """Determine the memory available to new applications without swapping"""
+ return self._get_mem_data('MemAvailable')
+
+ @property
+ def vendor(self):
+ # type: () -> str
+ """Determine server vendor from DMI data in sysfs"""
+ return read_file(HostFacts._dmi_path_list, 'sys_vendor')
+
+ @property
+ def model(self):
+ # type: () -> str
+ """Determine server model information from DMI data in sysfs"""
+ family = read_file(HostFacts._dmi_path_list, 'product_family')
+ product = read_file(HostFacts._dmi_path_list, 'product_name')
+ if family == 'Unknown' and product:
+ return '{}'.format(product)
+
+ return '{} ({})'.format(family, product)
+
+ @property
+ def bios_version(self):
+ # type: () -> str
+ """Determine server BIOS version from DMI data in sysfs"""
+ return read_file(HostFacts._dmi_path_list, 'bios_version')
+
+ @property
+ def bios_date(self):
+ # type: () -> str
+ """Determine server BIOS date from DMI data in sysfs"""
+ return read_file(HostFacts._dmi_path_list, 'bios_date')
+
+ @property
+ def timestamp(self):
+ # type: () -> float
+ """Return the current time as Epoch seconds"""
+ return time.time()
+
+ @property
+ def system_uptime(self):
+ # type: () -> float
+ """Return the system uptime (in secs)"""
+ raw_time = read_file(['/proc/uptime'])
+ up_secs, _ = raw_time.split()
+ return float(up_secs)
+
+ @property
+ def kernel_security(self):
+ # type: () -> Dict[str, str]
+ """Determine the security features enabled in the kernel - SELinux, AppArmor"""
+ def _fetch_selinux() -> Dict[str, str]:
+ """Get the selinux status"""
+ security = {}
+ try:
+ out, err, code = call(self.ctx, ['sestatus'],
+ verbosity=CallVerbosity.DEBUG)
+ security['type'] = 'SELinux'
+ status, mode, policy = '', '', ''
+ for line in out.split('\n'):
+ if line.startswith('SELinux status:'):
+ k, v = line.split(':')
+ status = v.strip()
+ elif line.startswith('Current mode:'):
+ k, v = line.split(':')
+ mode = v.strip()
+ elif line.startswith('Loaded policy name:'):
+ k, v = line.split(':')
+ policy = v.strip()
+ if status == 'disabled':
+ security['description'] = 'SELinux: Disabled'
+ else:
+ security['description'] = 'SELinux: Enabled({}, {})'.format(mode, policy)
+ except Exception as e:
+ logger.info('unable to get selinux status: %s' % e)
+ return security
+
+ def _fetch_apparmor() -> Dict[str, str]:
+ """Read the apparmor profiles directly, returning an overview of AppArmor status"""
+ security = {}
+ for apparmor_path in HostFacts._apparmor_path_list:
+ if os.path.exists(apparmor_path):
+ security['type'] = 'AppArmor'
+ security['description'] = 'AppArmor: Enabled'
+ try:
+ profiles = read_file(['/sys/kernel/security/apparmor/profiles'])
+ if len(profiles) == 0:
+ return {}
+ except OSError:
+ pass
+ else:
+ summary = {} # type: Dict[str, int]
+ for line in profiles.split('\n'):
+ item, mode = line.split(' ')
+ mode = mode.strip('()')
+ if mode in summary:
+ summary[mode] += 1
+ else:
+ summary[mode] = 0
+ summary_str = ','.join(['{} {}'.format(v, k) for k, v in summary.items()])
+ security = {**security, **summary} # type: ignore
+ security['description'] += '({})'.format(summary_str)
+
+ return security
+ return {}
+
+ ret = {}
+ if os.path.exists('/sys/kernel/security/lsm'):
+ lsm = read_file(['/sys/kernel/security/lsm']).strip()
+ if 'selinux' in lsm:
+ ret = _fetch_selinux()
+ elif 'apparmor' in lsm:
+ ret = _fetch_apparmor()
+ else:
+ return {
+ 'type': 'Unknown',
+ 'description': 'Linux Security Module framework is active, but is not using SELinux or AppArmor'
+ }
+
+ if ret:
+ return ret
+
+ return {
+ 'type': 'None',
+ 'description': 'Linux Security Module framework is not available'
+ }
+
+ @property
+ def selinux_enabled(self) -> bool:
+ return (self.kernel_security['type'] == 'SELinux') and \
+ (self.kernel_security['description'] != 'SELinux: Disabled')
+
+ @property
+ def kernel_parameters(self):
+ # type: () -> Dict[str, str]
+ """Get kernel parameters required/used in Ceph clusters"""
+
+ k_param = {}
+ out, _, _ = call_throws(self.ctx, ['sysctl', '-a'], verbosity=CallVerbosity.SILENT)
+ if out:
+ param_list = out.split('\n')
+ param_dict = {param.split(' = ')[0]: param.split(' = ')[-1] for param in param_list}
+
+ # return only desired parameters
+ if 'net.ipv4.ip_nonlocal_bind' in param_dict:
+ k_param['net.ipv4.ip_nonlocal_bind'] = param_dict['net.ipv4.ip_nonlocal_bind']
+
+ return k_param
+
+ @staticmethod
+ def _process_net_data(tcp_file: str, protocol: str = 'tcp') -> List[int]:
+ listening_ports = []
+ # Connections state documentation
+ # tcp - https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/include/net/tcp_states.h
+ # udp - uses 07 (TCP_CLOSE or UNCONN, since udp is stateless. test with netcat -ul <port>)
+ listening_state = {
+ 'tcp': '0A',
+ 'udp': '07'
+ }
+
+ if protocol not in listening_state.keys():
+ return []
+
+ if os.path.exists(tcp_file):
+ with open(tcp_file) as f:
+ tcp_data = f.readlines()[1:]
+
+ for con in tcp_data:
+ con_info = con.strip().split()
+ if con_info[3] == listening_state[protocol]:
+ local_port = int(con_info[1].split(':')[1], 16)
+ listening_ports.append(local_port)
+
+ return listening_ports
+
+ @property
+ def tcp_ports_used(self) -> List[int]:
+ return HostFacts._process_net_data('/proc/net/tcp')
+
+ @property
+ def tcp6_ports_used(self) -> List[int]:
+ return HostFacts._process_net_data('/proc/net/tcp6')
+
+ @property
+ def udp_ports_used(self) -> List[int]:
+ return HostFacts._process_net_data('/proc/net/udp', 'udp')
+
+ @property
+ def udp6_ports_used(self) -> List[int]:
+ return HostFacts._process_net_data('/proc/net/udp6', 'udp')
+
+ def dump(self):
+ # type: () -> str
+ """Return the attributes of this HostFacts object as json"""
+ data = {
+ k: getattr(self, k) for k in dir(self)
+ if not k.startswith('_')
+ and isinstance(getattr(self, k), (float, int, str, list, dict, tuple))
+ }
+ return json.dumps(data, indent=2, sort_keys=True)
+
+##################################
+
+
+def command_gather_facts(ctx: CephadmContext) -> None:
+ """gather_facts is intended to provide host releated metadata to the caller"""
+ host = HostFacts(ctx)
+ print(host.dump())
+
+
+##################################
+
+
+class CephadmCache:
+ task_types = ['disks', 'daemons', 'host', 'http_server']
+
+ def __init__(self) -> None:
+ self.started_epoch_secs = time.time()
+ self.tasks = {
+ 'daemons': 'inactive',
+ 'disks': 'inactive',
+ 'host': 'inactive',
+ 'http_server': 'inactive',
+ }
+ self.errors: list = []
+ self.disks: dict = {}
+ self.daemons: dict = {}
+ self.host: dict = {}
+ self.lock = RLock()
+
+ @property
+ def health(self) -> dict:
+ return {
+ 'started_epoch_secs': self.started_epoch_secs,
+ 'tasks': self.tasks,
+ 'errors': self.errors,
+ }
+
+ def to_json(self) -> dict:
+ return {
+ 'health': self.health,
+ 'host': self.host,
+ 'daemons': self.daemons,
+ 'disks': self.disks,
+ }
+
+ def update_health(self, task_type: str, task_status: str, error_msg: Optional[str] = None) -> None:
+ assert task_type in CephadmCache.task_types
+ with self.lock:
+ self.tasks[task_type] = task_status
+ if error_msg:
+ self.errors.append(error_msg)
+
+ def update_task(self, task_type: str, content: dict) -> None:
+ assert task_type in CephadmCache.task_types
+ assert isinstance(content, dict)
+ with self.lock:
+ current = getattr(self, task_type)
+ for k in content:
+ current[k] = content[k]
+
+ setattr(self, task_type, current)
+
+
+class CephadmHTTPServer(ThreadingMixIn, HTTPServer):
+ allow_reuse_address = True
+ daemon_threads = True
+ cephadm_cache: CephadmCache
+ token: str
+
+
+class CephadmDaemonHandler(BaseHTTPRequestHandler):
+ server: CephadmHTTPServer
+ api_version = 'v1'
+ valid_routes = [
+ f'/{api_version}/metadata',
+ f'/{api_version}/metadata/health',
+ f'/{api_version}/metadata/disks',
+ f'/{api_version}/metadata/daemons',
+ f'/{api_version}/metadata/host',
+ ]
+
+ class Decorators:
+ @classmethod
+ def authorize(cls, f: Any) -> Any:
+ """Implement a basic token check.
+
+ The token is installed at deployment time and must be provided to
+ ensure we only respond to callers who know our token i.e. mgr
+ """
+
+ def wrapper(self: Any, *args: Any, **kwargs: Any) -> Any:
+ auth = self.headers.get('Authorization', None)
+ if auth != 'Bearer ' + self.server.token:
+ self.send_error(401)
+ return
+ f(self, *args, **kwargs)
+
+ return wrapper
+
+ def _help_page(self) -> str:
+ return """<!DOCTYPE html>
+<html>
+<head><title>cephadm metadata exporter</title></head>
+<style>
+body {{
+ font-family: sans-serif;
+ font-size: 0.8em;
+}}
+table {{
+ border-width: 0px;
+ border-spacing: 0px;
+ margin-left:20px;
+}}
+tr:hover {{
+ background: PowderBlue;
+}}
+td,th {{
+ padding: 5px;
+}}
+</style>
+<body>
+ <h1>cephadm metadata exporter {api_version}</h1>
+ <table>
+ <thead>
+ <tr><th>Endpoint</th><th>Methods</th><th>Response</th><th>Description</th></tr>
+ </thead>
+ <tr><td><a href='{api_version}/metadata'>{api_version}/metadata</a></td><td>GET</td><td>JSON</td><td>Return <b>all</b> metadata for the host</td></tr>
+ <tr><td><a href='{api_version}/metadata/daemons'>{api_version}/metadata/daemons</a></td><td>GET</td><td>JSON</td><td>Return daemon and systemd states for ceph daemons (ls)</td></tr>
+ <tr><td><a href='{api_version}/metadata/disks'>{api_version}/metadata/disks</a></td><td>GET</td><td>JSON</td><td>show disk inventory (ceph-volume)</td></tr>
+ <tr><td><a href='{api_version}/metadata/health'>{api_version}/metadata/health</a></td><td>GET</td><td>JSON</td><td>Show current health of the exporter sub-tasks</td></tr>
+ <tr><td><a href='{api_version}/metadata/host'>{api_version}/metadata/host</a></td><td>GET</td><td>JSON</td><td>Show host metadata (gather-facts)</td></tr>
+ </table>
+</body>
+</html>""".format(api_version=CephadmDaemonHandler.api_version)
+
+ def _fetch_root(self) -> None:
+ self.send_response(200)
+ self.send_header('Content-type', 'text/html; charset=utf-8')
+ self.end_headers()
+ self.wfile.write(self._help_page().encode('utf-8'))
+
+ @Decorators.authorize
+ def do_GET(self) -> None:
+ """Handle *all* GET requests"""
+
+ if self.path == '/':
+ # provide a html response if someone hits the root url, to document the
+ # available api endpoints
+ return self._fetch_root()
+ elif self.path in CephadmDaemonHandler.valid_routes:
+ u = self.path.split('/')[-1]
+ data = json.dumps({})
+ status_code = 200
+
+ tasks = self.server.cephadm_cache.health.get('tasks', {})
+ assert tasks
+
+ # We're using the http status code to help indicate thread health
+ # - 200 (OK): request successful
+ # - 204 (No Content): access to a cache relating to a dead thread
+ # - 206 (Partial content): one or more theads are inactive
+ # - 500 (Server Error): all threads inactive
+ if u == 'metadata':
+ data = json.dumps(self.server.cephadm_cache.to_json())
+ if all([tasks[task_name] == 'inactive' for task_name in tasks if task_name != 'http_server']):
+ # All the subtasks are dead!
+ status_code = 500
+ elif any([tasks[task_name] == 'inactive' for task_name in tasks if task_name != 'http_server']):
+ status_code = 206
+
+ # Individual GETs against the a tasks endpoint will also return a 503 if the corresponding thread is inactive
+ elif u == 'daemons':
+ data = json.dumps(self.server.cephadm_cache.daemons)
+ if tasks['daemons'] == 'inactive':
+ status_code = 204
+ elif u == 'disks':
+ data = json.dumps(self.server.cephadm_cache.disks)
+ if tasks['disks'] == 'inactive':
+ status_code = 204
+ elif u == 'host':
+ data = json.dumps(self.server.cephadm_cache.host)
+ if tasks['host'] == 'inactive':
+ status_code = 204
+
+ # a GET against health will always return a 200, since the op is always successful
+ elif u == 'health':
+ data = json.dumps(self.server.cephadm_cache.health)
+
+ self.send_response(status_code)
+ self.send_header('Content-type', 'application/json')
+ self.end_headers()
+ self.wfile.write(data.encode('utf-8'))
+ else:
+ # Invalid GET URL
+ bad_request_msg = 'Valid URLs are: {}'.format(', '.join(CephadmDaemonHandler.valid_routes))
+ self.send_response(404, message=bad_request_msg) # reason
+ self.send_header('Content-type', 'application/json')
+ self.end_headers()
+ self.wfile.write(json.dumps({'message': bad_request_msg}).encode('utf-8'))
+
+ def log_message(self, format: str, *args: Any) -> None:
+ rqst = ' '.join(str(a) for a in args)
+ logger.info(f'client:{self.address_string()} [{self.log_date_time_string()}] {rqst}')
+
+
+class CephadmDaemon():
+
+ daemon_type = 'cephadm-exporter'
+ default_port = 9443
+ key_name = 'key'
+ crt_name = 'crt'
+ token_name = 'token'
+ config_requirements = [
+ key_name,
+ crt_name,
+ token_name,
+ ]
+ loop_delay = 1
+ thread_check_interval = 5
+
+ def __init__(self, ctx: CephadmContext, fsid: str, daemon_id: Optional[str] = None, port: Optional[int] = None) -> None:
+ self.ctx = ctx
+ self.fsid = fsid
+ self.daemon_id = daemon_id
+ if not port:
+ self.port = CephadmDaemon.default_port
+ else:
+ self.port = port
+ self.workers: List[Thread] = []
+ self.http_server: CephadmHTTPServer
+ self.stop = False
+ self.cephadm_cache = CephadmCache()
+ self.errors: List[str] = []
+ self.token = read_file([os.path.join(self.daemon_path, CephadmDaemon.token_name)])
+
+ @classmethod
+ def validate_config(cls, config: dict) -> None:
+ reqs = ', '.join(CephadmDaemon.config_requirements)
+ errors = []
+
+ if not config or not all([k_name in config for k_name in CephadmDaemon.config_requirements]):
+ raise Error(f'config must contain the following fields : {reqs}')
+
+ if not all([isinstance(config[k_name], str) for k_name in CephadmDaemon.config_requirements]):
+ errors.append(f'the following fields must be strings: {reqs}')
+
+ crt = config[CephadmDaemon.crt_name]
+ key = config[CephadmDaemon.key_name]
+ token = config[CephadmDaemon.token_name]
+
+ if not crt.startswith('-----BEGIN CERTIFICATE-----') or not crt.endswith('-----END CERTIFICATE-----\n'):
+ errors.append('crt field is not a valid SSL certificate')
+ if not key.startswith('-----BEGIN PRIVATE KEY-----') or not key.endswith('-----END PRIVATE KEY-----\n'):
+ errors.append('key is not a valid SSL private key')
+ if len(token) < 8:
+ errors.append("'token' must be more than 8 characters long")
+
+ if 'port' in config:
+ try:
+ p = int(config['port'])
+ if p <= 1024:
+ raise ValueError
+ except (TypeError, ValueError):
+ errors.append('port must be an integer > 1024')
+
+ if errors:
+ raise Error('Parameter errors : {}'.format(', '.join(errors)))
+
+ @property
+ def port_active(self) -> bool:
+ return port_in_use(self.ctx, self.port)
+
+ @property
+ def can_run(self) -> bool:
+ # if port is in use
+ if self.port_active:
+ self.errors.append(f'TCP port {self.port} already in use, unable to bind')
+ if not os.path.exists(os.path.join(self.daemon_path, CephadmDaemon.key_name)):
+ self.errors.append(f"Key file '{CephadmDaemon.key_name}' is missing from {self.daemon_path}")
+ if not os.path.exists(os.path.join(self.daemon_path, CephadmDaemon.crt_name)):
+ self.errors.append(f"Certificate file '{CephadmDaemon.crt_name}' is missing from {self.daemon_path}")
+ if self.token == 'Unknown':
+ self.errors.append(f"Authentication token '{CephadmDaemon.token_name}' is missing from {self.daemon_path}")
+ return len(self.errors) == 0
+
+ @staticmethod
+ def _unit_name(fsid: str, daemon_id: str) -> str:
+ return '{}.service'.format(get_unit_name(fsid, CephadmDaemon.daemon_type, daemon_id))
+
+ @property
+ def unit_name(self) -> str:
+ assert self.daemon_id is not None
+ return CephadmDaemon._unit_name(self.fsid, self.daemon_id)
+
+ @property
+ def daemon_path(self) -> str:
+ return os.path.join(
+ self.ctx.data_dir,
+ self.fsid,
+ f'{self.daemon_type}.{self.daemon_id}'
+ )
+
+ @property
+ def binary_path(self) -> str:
+ path = os.path.realpath(__file__)
+ assert os.path.isfile(path)
+ return path
+
+ def _handle_thread_exception(self, exc: Exception, thread_type: str) -> None:
+ e_msg = f'{exc.__class__.__name__} exception: {str(exc)}'
+ thread_info = getattr(self.cephadm_cache, thread_type)
+ errors = thread_info.get('scrape_errors', [])
+ errors.append(e_msg)
+ logger.error(e_msg)
+ logger.exception(exc)
+ self.cephadm_cache.update_task(
+ thread_type,
+ {
+ 'scrape_errors': errors,
+ 'data': None,
+ }
+ )
+
+ def _scrape_host_facts(self, refresh_interval: int = 10) -> None:
+ ctr = 0
+ exception_encountered = False
+
+ while True:
+
+ if self.stop or exception_encountered:
+ break
+
+ if ctr >= refresh_interval:
+ ctr = 0
+ logger.debug('executing host-facts scrape')
+ errors = []
+ s_time = time.time()
+
+ try:
+ facts = HostFacts(self.ctx)
+ except Exception as e:
+ self._handle_thread_exception(e, 'host')
+ exception_encountered = True
+ else:
+ elapsed = time.time() - s_time
+ try:
+ data = json.loads(facts.dump())
+ except json.decoder.JSONDecodeError:
+ errors.append('host-facts provided invalid JSON')
+ logger.warning(errors[-1])
+ data = {}
+ self.cephadm_cache.update_task(
+ 'host',
+ {
+ 'scrape_timestamp': s_time,
+ 'scrape_duration_secs': elapsed,
+ 'scrape_errors': errors,
+ 'data': data,
+ }
+ )
+ logger.debug(f'completed host-facts scrape - {elapsed}s')
+
+ time.sleep(CephadmDaemon.loop_delay)
+ ctr += CephadmDaemon.loop_delay
+ logger.info('host-facts thread stopped')
+
+ def _scrape_ceph_volume(self, refresh_interval: int = 15) -> None:
+ # we're invoking the ceph_volume command, so we need to set the args that it
+ # expects to use
+ self.ctx.command = 'inventory --format=json'.split()
+ self.ctx.fsid = self.fsid
+
+ ctr = 0
+ exception_encountered = False
+
+ while True:
+ if self.stop or exception_encountered:
+ break
+
+ if ctr >= refresh_interval:
+ ctr = 0
+ logger.debug('executing ceph-volume scrape')
+ errors = []
+ s_time = time.time()
+ stream = io.StringIO()
+ try:
+ with redirect_stdout(stream):
+ command_ceph_volume(self.ctx)
+ except Exception as e:
+ self._handle_thread_exception(e, 'disks')
+ exception_encountered = True
+ else:
+ elapsed = time.time() - s_time
+
+ # if the call to ceph-volume returns junk with the
+ # json, it won't parse
+ stdout = stream.getvalue()
+
+ data = []
+ if stdout:
+ try:
+ data = json.loads(stdout)
+ except json.decoder.JSONDecodeError:
+ errors.append('ceph-volume thread provided bad json data')
+ logger.warning(errors[-1])
+ else:
+ errors.append('ceph-volume did not return any data')
+ logger.warning(errors[-1])
+
+ self.cephadm_cache.update_task(
+ 'disks',
+ {
+ 'scrape_timestamp': s_time,
+ 'scrape_duration_secs': elapsed,
+ 'scrape_errors': errors,
+ 'data': data,
+ }
+ )
+
+ logger.debug(f'completed ceph-volume scrape - {elapsed}s')
+ time.sleep(CephadmDaemon.loop_delay)
+ ctr += CephadmDaemon.loop_delay
+
+ logger.info('ceph-volume thread stopped')
+
+ def _scrape_list_daemons(self, refresh_interval: int = 20) -> None:
+ ctr = 0
+ exception_encountered = False
+ while True:
+ if self.stop or exception_encountered:
+ break
+
+ if ctr >= refresh_interval:
+ ctr = 0
+ logger.debug('executing list-daemons scrape')
+ errors = []
+ s_time = time.time()
+
+ try:
+ # list daemons should ideally be invoked with a fsid
+ data = list_daemons(self.ctx)
+ except Exception as e:
+ self._handle_thread_exception(e, 'daemons')
+ exception_encountered = True
+ else:
+ if not isinstance(data, list):
+ errors.append('list-daemons did not supply a list?')
+ logger.warning(errors[-1])
+ data = []
+ elapsed = time.time() - s_time
+ self.cephadm_cache.update_task(
+ 'daemons',
+ {
+ 'scrape_timestamp': s_time,
+ 'scrape_duration_secs': elapsed,
+ 'scrape_errors': errors,
+ 'data': data,
+ }
+ )
+ logger.debug(f'completed list-daemons scrape - {elapsed}s')
+
+ time.sleep(CephadmDaemon.loop_delay)
+ ctr += CephadmDaemon.loop_delay
+ logger.info('list-daemons thread stopped')
+
+ def _create_thread(self, target: Any, name: str, refresh_interval: Optional[int] = None) -> Thread:
+ if refresh_interval:
+ t = Thread(target=target, args=(refresh_interval,))
+ else:
+ t = Thread(target=target)
+ t.daemon = True
+ t.name = name
+ self.cephadm_cache.update_health(name, 'active')
+ t.start()
+
+ start_msg = f'Started {name} thread'
+ if refresh_interval:
+ logger.info(f'{start_msg}, with a refresh interval of {refresh_interval}s')
+ else:
+ logger.info(f'{start_msg}')
+ return t
+
+ def reload(self, *args: Any) -> None:
+ """reload -HUP received
+
+ This is a placeholder function only, and serves to provide the hook that could
+ be exploited later if the exporter evolves to incorporate a config file
+ """
+ logger.info('Reload request received - ignoring, no action needed')
+
+ def shutdown(self, *args: Any) -> None:
+ logger.info('Shutdown request received')
+ self.stop = True
+ self.http_server.shutdown()
+
+ def run(self) -> None:
+ logger.info(f"cephadm exporter starting for FSID '{self.fsid}'")
+ if not self.can_run:
+ logger.error('Unable to start the exporter daemon')
+ for e in self.errors:
+ logger.error(e)
+ return
+
+ # register signal handlers for running under systemd control
+ signal.signal(signal.SIGTERM, self.shutdown)
+ signal.signal(signal.SIGINT, self.shutdown)
+ signal.signal(signal.SIGHUP, self.reload)
+ logger.debug('Signal handlers attached')
+
+ host_facts = self._create_thread(self._scrape_host_facts, 'host', 5)
+ self.workers.append(host_facts)
+
+ daemons = self._create_thread(self._scrape_list_daemons, 'daemons', 20)
+ self.workers.append(daemons)
+
+ disks = self._create_thread(self._scrape_ceph_volume, 'disks', 20)
+ self.workers.append(disks)
+
+ self.http_server = CephadmHTTPServer(('0.0.0.0', self.port), CephadmDaemonHandler) # IPv4 only
+ self.http_server.socket = ssl.wrap_socket(self.http_server.socket,
+ keyfile=os.path.join(self.daemon_path, CephadmDaemon.key_name),
+ certfile=os.path.join(self.daemon_path, CephadmDaemon.crt_name),
+ server_side=True)
+
+ self.http_server.cephadm_cache = self.cephadm_cache
+ self.http_server.token = self.token
+ server_thread = self._create_thread(self.http_server.serve_forever, 'http_server')
+ logger.info(f'https server listening on {self.http_server.server_address[0]}:{self.http_server.server_port}')
+
+ ctr = 0
+ while server_thread.is_alive():
+ if self.stop:
+ break
+
+ if ctr >= CephadmDaemon.thread_check_interval:
+ ctr = 0
+ for worker in self.workers:
+ if self.cephadm_cache.tasks[worker.name] == 'inactive':
+ continue
+ if not worker.is_alive():
+ logger.warning(f'{worker.name} thread not running')
+ stop_time = datetime.datetime.now().strftime('%Y/%m/%d %H:%M:%S')
+ self.cephadm_cache.update_health(worker.name, 'inactive', f'{worker.name} stopped at {stop_time}')
+
+ time.sleep(CephadmDaemon.loop_delay)
+ ctr += CephadmDaemon.loop_delay
+
+ logger.info('Main http server thread stopped')
+
+ @property
+ def unit_run(self) -> str:
+
+ return """set -e
+{py3} {bin_path} exporter --fsid {fsid} --id {daemon_id} --port {port} &""".format(
+ py3=shutil.which('python3'),
+ bin_path=self.binary_path,
+ fsid=self.fsid,
+ daemon_id=self.daemon_id,
+ port=self.port
+ )
+
+ @property
+ def unit_file(self) -> str:
+ docker = isinstance(self.ctx.container_engine, Docker)
+ return """#generated by cephadm
+[Unit]
+Description=cephadm exporter service for cluster {fsid}
+After=network-online.target{docker_after}
+Wants=network-online.target
+{docker_requires}
+
+PartOf=ceph-{fsid}.target
+Before=ceph-{fsid}.target
+
+[Service]
+Type=forking
+ExecStart=/bin/bash {daemon_path}/unit.run
+ExecReload=/bin/kill -HUP $MAINPID
+Restart=on-failure
+RestartSec=10s
+
+[Install]
+WantedBy=ceph-{fsid}.target
+""".format(fsid=self.fsid,
+ daemon_path=self.daemon_path,
+ # if docker, we depend on docker.service
+ docker_after=' docker.service' if docker else '',
+ docker_requires='Requires=docker.service\n' if docker else '')
+
+ def deploy_daemon_unit(self, config: Optional[dict] = None) -> None:
+ """deploy a specific unit file for cephadm
+
+ The normal deploy_daemon_units doesn't apply for this
+ daemon since it's not a container, so we just create a
+ simple service definition and add it to the fsid's target
+ """
+ if not config:
+ raise Error('Attempting to deploy cephadm daemon without a config')
+ assert isinstance(config, dict)
+
+ # Create the required config files in the daemons dir, with restricted permissions
+ for filename in config:
+ with open(os.open(os.path.join(self.daemon_path, filename), os.O_CREAT | os.O_WRONLY, mode=0o600), 'w') as f:
+ f.write(config[filename])
+
+ # When __file__ is <stdin> we're being invoked over remoto via the orchestrator, so
+ # we pick up the file from where the orchestrator placed it - otherwise we'll
+ # copy it to the binary location for this cluster
+ if not __file__ == '<stdin>':
+ try:
+ shutil.copy(__file__,
+ self.binary_path)
+ except shutil.SameFileError:
+ pass
+
+ with open(os.path.join(self.daemon_path, 'unit.run'), 'w') as f:
+ f.write(self.unit_run)
+
+ with open(
+ os.path.join(self.ctx.unit_dir,
+ f'{self.unit_name}.new'),
+ 'w'
+ ) as f:
+ f.write(self.unit_file)
+ os.rename(
+ os.path.join(self.ctx.unit_dir, f'{self.unit_name}.new'),
+ os.path.join(self.ctx.unit_dir, self.unit_name))
+
+ call_throws(self.ctx, ['systemctl', 'daemon-reload'])
+ call(self.ctx, ['systemctl', 'stop', self.unit_name],
+ verbosity=CallVerbosity.DEBUG)
+ call(self.ctx, ['systemctl', 'reset-failed', self.unit_name],
+ verbosity=CallVerbosity.DEBUG)
+ call_throws(self.ctx, ['systemctl', 'enable', '--now', self.unit_name])
+
+ @classmethod
+ def uninstall(cls, ctx: CephadmContext, fsid: str, daemon_type: str, daemon_id: str) -> None:
+ unit_name = CephadmDaemon._unit_name(fsid, daemon_id)
+ unit_path = os.path.join(ctx.unit_dir, unit_name)
+ unit_run = os.path.join(ctx.data_dir, fsid, f'{daemon_type}.{daemon_id}', 'unit.run')
+ port = None
+ try:
+ with open(unit_run, 'r') as u:
+ contents = u.read().strip(' &')
+ except OSError:
+ logger.warning(f'Unable to access the unit.run file @ {unit_run}')
+ return
+
+ port = None
+ for line in contents.split('\n'):
+ if '--port ' in line:
+ try:
+ port = int(line.split('--port ')[-1])
+ except ValueError:
+ logger.warning('Unexpected format in unit.run file: port is not numeric')
+ logger.warning('Unable to remove the systemd file and close the port')
+ return
+ break
+
+ if port:
+ fw = Firewalld(ctx)
+ try:
+ fw.close_ports([port])
+ except RuntimeError:
+ logger.error(f'Unable to close port {port}')
+
+ stdout, stderr, rc = call(ctx, ['rm', '-f', unit_path])
+ if rc:
+ logger.error(f'Unable to remove the systemd file @ {unit_path}')
+ else:
+ logger.info(f'removed systemd unit file @ {unit_path}')
+ stdout, stderr, rc = call(ctx, ['systemctl', 'daemon-reload'])
+
+
+def command_exporter(ctx: CephadmContext) -> None:
+ exporter = CephadmDaemon(ctx, ctx.fsid, daemon_id=ctx.id, port=ctx.port)
+
+ if ctx.fsid not in os.listdir(ctx.data_dir):
+ raise Error(f"cluster fsid '{ctx.fsid}' not found in '{ctx.data_dir}'")
+
+ exporter.run()
+
+##################################
+
+
+def systemd_target_state(ctx: CephadmContext, target_name: str, subsystem: str = 'ceph') -> bool:
+ # TODO: UNITTEST
+ return os.path.exists(
+ os.path.join(
+ ctx.unit_dir,
+ f'{subsystem}.target.wants',
+ target_name
+ )
+ )
+
+
+def target_exists(ctx: CephadmContext) -> bool:
+ return os.path.exists(ctx.unit_dir + '/ceph.target')
+
+
+@infer_fsid
+def command_maintenance(ctx: CephadmContext) -> str:
+ if not ctx.fsid:
+ raise Error('failed - must pass --fsid to specify cluster')
+
+ target = f'ceph-{ctx.fsid}.target'
+
+ if ctx.maintenance_action.lower() == 'enter':
+ logger.info('Requested to place host into maintenance')
+ if systemd_target_state(ctx, target):
+ _out, _err, code = call(ctx,
+ ['systemctl', 'disable', target],
+ verbosity=CallVerbosity.DEBUG)
+ if code:
+ logger.error(f'Failed to disable the {target} target')
+ return 'failed - to disable the target'
+ else:
+ # stopping a target waits by default
+ _out, _err, code = call(ctx,
+ ['systemctl', 'stop', target],
+ verbosity=CallVerbosity.DEBUG)
+ if code:
+ logger.error(f'Failed to stop the {target} target')
+ return 'failed - to disable the target'
+ else:
+ return f'success - systemd target {target} disabled'
+
+ else:
+ return 'skipped - target already disabled'
+
+ else:
+ logger.info('Requested to exit maintenance state')
+ # if we've never deployed a daemon on this host there will be no systemd
+ # target to disable so attempting a disable will fail. We still need to
+ # return success here or host will be permanently stuck in maintenance mode
+ # as no daemons can be deployed so no systemd target will ever exist to disable.
+ if not target_exists(ctx):
+ return 'skipped - systemd target not present on this host. Host removed from maintenance mode.'
+ # exit maintenance request
+ if not systemd_target_state(ctx, target):
+ _out, _err, code = call(ctx,
+ ['systemctl', 'enable', target],
+ verbosity=CallVerbosity.DEBUG)
+ if code:
+ logger.error(f'Failed to enable the {target} target')
+ return 'failed - unable to enable the target'
+ else:
+ # starting a target waits by default
+ _out, _err, code = call(ctx,
+ ['systemctl', 'start', target],
+ verbosity=CallVerbosity.DEBUG)
+ if code:
+ logger.error(f'Failed to start the {target} target')
+ return 'failed - unable to start the target'
+ else:
+ return f'success - systemd target {target} enabled and started'
+ return f'success - systemd target {target} enabled and started'
+
+##################################
+
+
+def _get_parser():
+ # type: () -> argparse.ArgumentParser
+ parser = argparse.ArgumentParser(
+ description='Bootstrap Ceph daemons with systemd and containers.',
+ formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+ parser.add_argument(
+ '--image',
+ help='container image. Can also be set via the "CEPHADM_IMAGE" '
+ 'env var')
+ parser.add_argument(
+ '--docker',
+ action='store_true',
+ help='use docker instead of podman')
+ parser.add_argument(
+ '--data-dir',
+ default=DATA_DIR,
+ help='base directory for daemon data')
+ parser.add_argument(
+ '--log-dir',
+ default=LOG_DIR,
+ help='base directory for daemon logs')
+ parser.add_argument(
+ '--logrotate-dir',
+ default=LOGROTATE_DIR,
+ help='location of logrotate configuration files')
+ parser.add_argument(
+ '--sysctl-dir',
+ default=SYSCTL_DIR,
+ help='location of sysctl configuration files')
+ parser.add_argument(
+ '--unit-dir',
+ default=UNIT_DIR,
+ help='base directory for systemd units')
+ parser.add_argument(
+ '--verbose', '-v',
+ action='store_true',
+ help='Show debug-level log messages')
+ parser.add_argument(
+ '--timeout',
+ type=int,
+ default=DEFAULT_TIMEOUT,
+ help='timeout in seconds')
+ parser.add_argument(
+ '--retry',
+ type=int,
+ default=DEFAULT_RETRY,
+ help='max number of retries')
+ parser.add_argument(
+ '--env', '-e',
+ action='append',
+ default=[],
+ help='set environment variable')
+ parser.add_argument(
+ '--no-container-init',
+ action='store_true',
+ default=not CONTAINER_INIT,
+ help='Do not run podman/docker with `--init`')
+
+ subparsers = parser.add_subparsers(help='sub-command')
+
+ parser_version = subparsers.add_parser(
+ 'version', help='get ceph version from container')
+ parser_version.set_defaults(func=command_version)
+
+ parser_pull = subparsers.add_parser(
+ 'pull', help='pull the default container image')
+ parser_pull.set_defaults(func=command_pull)
+ parser_pull.add_argument(
+ '--insecure',
+ action='store_true',
+ help=argparse.SUPPRESS,
+ )
+
+ parser_inspect_image = subparsers.add_parser(
+ 'inspect-image', help='inspect local container image')
+ parser_inspect_image.set_defaults(func=command_inspect_image)
+
+ parser_ls = subparsers.add_parser(
+ 'ls', help='list daemon instances on this host')
+ parser_ls.set_defaults(func=command_ls)
+ parser_ls.add_argument(
+ '--no-detail',
+ action='store_true',
+ help='Do not include daemon status')
+ parser_ls.add_argument(
+ '--legacy-dir',
+ default='/',
+ help='base directory for legacy daemon data')
+
+ parser_list_networks = subparsers.add_parser(
+ 'list-networks', help='list IP networks')
+ parser_list_networks.set_defaults(func=command_list_networks)
+
+ parser_adopt = subparsers.add_parser(
+ 'adopt', help='adopt daemon deployed with a different tool')
+ parser_adopt.set_defaults(func=command_adopt)
+ parser_adopt.add_argument(
+ '--name', '-n',
+ required=True,
+ help='daemon name (type.id)')
+ parser_adopt.add_argument(
+ '--style',
+ required=True,
+ help='deployment style (legacy, ...)')
+ parser_adopt.add_argument(
+ '--cluster',
+ default='ceph',
+ help='cluster name')
+ parser_adopt.add_argument(
+ '--legacy-dir',
+ default='/',
+ help='base directory for legacy daemon data')
+ parser_adopt.add_argument(
+ '--config-json',
+ help='Additional configuration information in JSON format')
+ parser_adopt.add_argument(
+ '--skip-firewalld',
+ action='store_true',
+ help='Do not configure firewalld')
+ parser_adopt.add_argument(
+ '--skip-pull',
+ action='store_true',
+ help='do not pull the default image before adopting')
+ parser_adopt.add_argument(
+ '--force-start',
+ action='store_true',
+ help='start newly adoped daemon, even if it was not running previously')
+ parser_adopt.add_argument(
+ '--container-init',
+ action='store_true',
+ default=CONTAINER_INIT,
+ help=argparse.SUPPRESS)
+
+ parser_rm_daemon = subparsers.add_parser(
+ 'rm-daemon', help='remove daemon instance')
+ parser_rm_daemon.set_defaults(func=command_rm_daemon)
+ parser_rm_daemon.add_argument(
+ '--name', '-n',
+ required=True,
+ action=CustomValidation,
+ help='daemon name (type.id)')
+ parser_rm_daemon.add_argument(
+ '--tcp-ports',
+ help='List of tcp ports to close in the host firewall')
+ parser_rm_daemon.add_argument(
+ '--fsid',
+ required=True,
+ help='cluster FSID')
+ parser_rm_daemon.add_argument(
+ '--force',
+ action='store_true',
+ help='proceed, even though this may destroy valuable data')
+ parser_rm_daemon.add_argument(
+ '--force-delete-data',
+ action='store_true',
+ help='delete valuable daemon data instead of making a backup')
+
+ parser_rm_cluster = subparsers.add_parser(
+ 'rm-cluster', help='remove all daemons for a cluster')
+ parser_rm_cluster.set_defaults(func=command_rm_cluster)
+ parser_rm_cluster.add_argument(
+ '--fsid',
+ required=True,
+ help='cluster FSID')
+ parser_rm_cluster.add_argument(
+ '--force',
+ action='store_true',
+ help='proceed, even though this may destroy valuable data')
+ parser_rm_cluster.add_argument(
+ '--keep-logs',
+ action='store_true',
+ help='do not remove log files')
+ parser_rm_cluster.add_argument(
+ '--zap-osds',
+ action='store_true',
+ help='zap OSD devices for this cluster')
+
+ parser_run = subparsers.add_parser(
+ 'run', help='run a ceph daemon, in a container, in the foreground')
+ parser_run.set_defaults(func=command_run)
+ parser_run.add_argument(
+ '--name', '-n',
+ required=True,
+ help='daemon name (type.id)')
+ parser_run.add_argument(
+ '--fsid',
+ required=True,
+ help='cluster FSID')
+
+ parser_shell = subparsers.add_parser(
+ 'shell', help='run an interactive shell inside a daemon container')
+ parser_shell.set_defaults(func=command_shell)
+ parser_shell.add_argument(
+ '--shared_ceph_folder',
+ metavar='CEPH_SOURCE_FOLDER',
+ help='Development mode. Several folders in containers are volumes mapped to different sub-folders in the ceph source folder')
+ parser_shell.add_argument(
+ '--fsid',
+ help='cluster FSID')
+ parser_shell.add_argument(
+ '--name', '-n',
+ help='daemon name (type.id)')
+ parser_shell.add_argument(
+ '--config', '-c',
+ help='ceph.conf to pass through to the container')
+ parser_shell.add_argument(
+ '--keyring', '-k',
+ help='ceph.keyring to pass through to the container')
+ parser_shell.add_argument(
+ '--mount', '-m',
+ help=('mount a file or directory in the container. '
+ 'Support multiple mounts. '
+ 'ie: `--mount /foo /bar:/bar`. '
+ 'When no destination is passed, default is /mnt'),
+ nargs='+')
+ parser_shell.add_argument(
+ '--env', '-e',
+ action='append',
+ default=[],
+ help='set environment variable')
+ parser_shell.add_argument(
+ '--volume', '-v',
+ action='append',
+ default=[],
+ help='set environment variable')
+ parser_shell.add_argument(
+ 'command', nargs=argparse.REMAINDER,
+ help='command (optional)')
+ parser_shell.add_argument(
+ '--no-hosts',
+ action='store_true',
+ help='dont pass /etc/hosts through to the container')
+
+ parser_enter = subparsers.add_parser(
+ 'enter', help='run an interactive shell inside a running daemon container')
+ parser_enter.set_defaults(func=command_enter)
+ parser_enter.add_argument(
+ '--fsid',
+ help='cluster FSID')
+ parser_enter.add_argument(
+ '--name', '-n',
+ required=True,
+ help='daemon name (type.id)')
+ parser_enter.add_argument(
+ 'command', nargs=argparse.REMAINDER,
+ help='command')
+
+ parser_ceph_volume = subparsers.add_parser(
+ 'ceph-volume', help='run ceph-volume inside a container')
+ parser_ceph_volume.set_defaults(func=command_ceph_volume)
+ parser_ceph_volume.add_argument(
+ '--shared_ceph_folder',
+ metavar='CEPH_SOURCE_FOLDER',
+ help='Development mode. Several folders in containers are volumes mapped to different sub-folders in the ceph source folder')
+ parser_ceph_volume.add_argument(
+ '--fsid',
+ help='cluster FSID')
+ parser_ceph_volume.add_argument(
+ '--config-json',
+ help='JSON file with config and (client.bootrap-osd) key')
+ parser_ceph_volume.add_argument(
+ '--config', '-c',
+ help='ceph conf file')
+ parser_ceph_volume.add_argument(
+ '--keyring', '-k',
+ help='ceph.keyring to pass through to the container')
+ parser_ceph_volume.add_argument(
+ 'command', nargs=argparse.REMAINDER,
+ help='command')
+
+ parser_zap_osds = subparsers.add_parser(
+ 'zap-osds', help='zap all OSDs associated with a particular fsid')
+ parser_zap_osds.set_defaults(func=command_zap_osds)
+ parser_zap_osds.add_argument(
+ '--fsid',
+ required=True,
+ help='cluster FSID')
+ parser_zap_osds.add_argument(
+ '--force',
+ action='store_true',
+ help='proceed, even though this may destroy valuable data')
+
+ parser_unit = subparsers.add_parser(
+ 'unit', help="operate on the daemon's systemd unit")
+ parser_unit.set_defaults(func=command_unit)
+ parser_unit.add_argument(
+ 'command',
+ help='systemd command (start, stop, restart, enable, disable, ...)')
+ parser_unit.add_argument(
+ '--fsid',
+ help='cluster FSID')
+ parser_unit.add_argument(
+ '--name', '-n',
+ required=True,
+ help='daemon name (type.id)')
+
+ parser_logs = subparsers.add_parser(
+ 'logs', help='print journald logs for a daemon container')
+ parser_logs.set_defaults(func=command_logs)
+ parser_logs.add_argument(
+ '--fsid',
+ help='cluster FSID')
+ parser_logs.add_argument(
+ '--name', '-n',
+ required=True,
+ help='daemon name (type.id)')
+ parser_logs.add_argument(
+ 'command', nargs='*',
+ help='additional journalctl args')
+
+ parser_bootstrap = subparsers.add_parser(
+ 'bootstrap', help='bootstrap a cluster (mon + mgr daemons)')
+ parser_bootstrap.set_defaults(func=command_bootstrap)
+ parser_bootstrap.add_argument(
+ '--config', '-c',
+ help='ceph conf file to incorporate')
+ parser_bootstrap.add_argument(
+ '--mon-id',
+ required=False,
+ help='mon id (default: local hostname)')
+ group = parser_bootstrap.add_mutually_exclusive_group()
+ group.add_argument(
+ '--mon-addrv',
+ help='mon IPs (e.g., [v2:localipaddr:3300,v1:localipaddr:6789])')
+ group.add_argument(
+ '--mon-ip',
+ help='mon IP')
+ parser_bootstrap.add_argument(
+ '--mgr-id',
+ required=False,
+ help='mgr id (default: randomly generated)')
+ parser_bootstrap.add_argument(
+ '--fsid',
+ help='cluster FSID')
+ parser_bootstrap.add_argument(
+ '--output-dir',
+ default='/etc/ceph',
+ help='directory to write config, keyring, and pub key files')
+ parser_bootstrap.add_argument(
+ '--output-keyring',
+ help='location to write keyring file with new cluster admin and mon keys')
+ parser_bootstrap.add_argument(
+ '--output-config',
+ help='location to write conf file to connect to new cluster')
+ parser_bootstrap.add_argument(
+ '--output-pub-ssh-key',
+ help="location to write the cluster's public SSH key")
+ parser_bootstrap.add_argument(
+ '--skip-admin-label',
+ action='store_true',
+ help='do not create admin label for ceph.conf and client.admin keyring distribution')
+ parser_bootstrap.add_argument(
+ '--skip-ssh',
+ action='store_true',
+ help='skip setup of ssh key on local host')
+ parser_bootstrap.add_argument(
+ '--initial-dashboard-user',
+ default='admin',
+ help='Initial user for the dashboard')
+ parser_bootstrap.add_argument(
+ '--initial-dashboard-password',
+ help='Initial password for the initial dashboard user')
+ parser_bootstrap.add_argument(
+ '--ssl-dashboard-port',
+ type=int,
+ default=8443,
+ help='Port number used to connect with dashboard using SSL')
+ parser_bootstrap.add_argument(
+ '--dashboard-key',
+ type=argparse.FileType('r'),
+ help='Dashboard key')
+ parser_bootstrap.add_argument(
+ '--dashboard-crt',
+ type=argparse.FileType('r'),
+ help='Dashboard certificate')
+
+ parser_bootstrap.add_argument(
+ '--ssh-config',
+ type=argparse.FileType('r'),
+ help='SSH config')
+ parser_bootstrap.add_argument(
+ '--ssh-private-key',
+ type=argparse.FileType('r'),
+ help='SSH private key')
+ parser_bootstrap.add_argument(
+ '--ssh-public-key',
+ type=argparse.FileType('r'),
+ help='SSH public key')
+ parser_bootstrap.add_argument(
+ '--ssh-user',
+ default='root',
+ help='set user for SSHing to cluster hosts, passwordless sudo will be needed for non-root users')
+ parser_bootstrap.add_argument(
+ '--skip-mon-network',
+ action='store_true',
+ help='set mon public_network based on bootstrap mon ip')
+ parser_bootstrap.add_argument(
+ '--skip-dashboard',
+ action='store_true',
+ help='do not enable the Ceph Dashboard')
+ parser_bootstrap.add_argument(
+ '--dashboard-password-noupdate',
+ action='store_true',
+ help='stop forced dashboard password change')
+ parser_bootstrap.add_argument(
+ '--no-minimize-config',
+ action='store_true',
+ help='do not assimilate and minimize the config file')
+ parser_bootstrap.add_argument(
+ '--skip-ping-check',
+ action='store_true',
+ help='do not verify that mon IP is pingable')
+ parser_bootstrap.add_argument(
+ '--skip-pull',
+ action='store_true',
+ help='do not pull the default image before bootstrapping')
+ parser_bootstrap.add_argument(
+ '--skip-firewalld',
+ action='store_true',
+ help='Do not configure firewalld')
+ parser_bootstrap.add_argument(
+ '--allow-overwrite',
+ action='store_true',
+ help='allow overwrite of existing --output-* config/keyring/ssh files')
+ parser_bootstrap.add_argument(
+ '--allow-fqdn-hostname',
+ action='store_true',
+ help='allow hostname that is fully-qualified (contains ".")')
+ parser_bootstrap.add_argument(
+ '--allow-mismatched-release',
+ action='store_true',
+ help="allow bootstrap of ceph that doesn't match this version of cephadm")
+ parser_bootstrap.add_argument(
+ '--skip-prepare-host',
+ action='store_true',
+ help='Do not prepare host')
+ parser_bootstrap.add_argument(
+ '--orphan-initial-daemons',
+ action='store_true',
+ help='Set mon and mgr service to `unmanaged`, Do not create the crash service')
+ parser_bootstrap.add_argument(
+ '--skip-monitoring-stack',
+ action='store_true',
+ help='Do not automatically provision monitoring stack (prometheus, grafana, alertmanager, node-exporter)')
+ parser_bootstrap.add_argument(
+ '--apply-spec',
+ help='Apply cluster spec after bootstrap (copy ssh key, add hosts and apply services)')
+ parser_bootstrap.add_argument(
+ '--shared_ceph_folder',
+ metavar='CEPH_SOURCE_FOLDER',
+ help='Development mode. Several folders in containers are volumes mapped to different sub-folders in the ceph source folder')
+
+ parser_bootstrap.add_argument(
+ '--registry-url',
+ help='url for custom registry')
+ parser_bootstrap.add_argument(
+ '--registry-username',
+ help='username for custom registry')
+ parser_bootstrap.add_argument(
+ '--registry-password',
+ help='password for custom registry')
+ parser_bootstrap.add_argument(
+ '--registry-json',
+ help='json file with custom registry login info (URL, Username, Password)')
+ parser_bootstrap.add_argument(
+ '--container-init',
+ action='store_true',
+ default=CONTAINER_INIT,
+ help=argparse.SUPPRESS)
+ parser_bootstrap.add_argument(
+ '--with-exporter',
+ action='store_true',
+ help='Automatically deploy cephadm metadata exporter to each node')
+ parser_bootstrap.add_argument(
+ '--exporter-config',
+ action=CustomValidation,
+ help=f'Exporter configuration information in JSON format (providing: {", ".join(CephadmDaemon.config_requirements)}, port information)')
+ parser_bootstrap.add_argument(
+ '--cluster-network',
+ help='subnet to use for cluster replication, recovery and heartbeats (in CIDR notation network/mask)')
+ parser_bootstrap.add_argument(
+ '--single-host-defaults',
+ action='store_true',
+ help='adjust configuration defaults to suit a single-host cluster')
+ parser_bootstrap.add_argument(
+ '--log-to-file',
+ action='store_true',
+ help='configure cluster to log to traditional log files in /var/log/ceph/$fsid')
+
+ parser_deploy = subparsers.add_parser(
+ 'deploy', help='deploy a daemon')
+ parser_deploy.set_defaults(func=command_deploy)
+ parser_deploy.add_argument(
+ '--name',
+ required=True,
+ action=CustomValidation,
+ help='daemon name (type.id)')
+ parser_deploy.add_argument(
+ '--fsid',
+ required=True,
+ help='cluster FSID')
+ parser_deploy.add_argument(
+ '--config', '-c',
+ help='config file for new daemon')
+ parser_deploy.add_argument(
+ '--config-json',
+ help='Additional configuration information in JSON format')
+ parser_deploy.add_argument(
+ '--keyring',
+ help='keyring for new daemon')
+ parser_deploy.add_argument(
+ '--key',
+ help='key for new daemon')
+ parser_deploy.add_argument(
+ '--osd-fsid',
+ help='OSD uuid, if creating an OSD container')
+ parser_deploy.add_argument(
+ '--skip-firewalld',
+ action='store_true',
+ help='Do not configure firewalld')
+ parser_deploy.add_argument(
+ '--tcp-ports',
+ help='List of tcp ports to open in the host firewall')
+ parser_deploy.add_argument(
+ '--reconfig',
+ action='store_true',
+ help='Reconfigure a previously deployed daemon')
+ parser_deploy.add_argument(
+ '--allow-ptrace',
+ action='store_true',
+ help='Allow SYS_PTRACE on daemon container')
+ parser_deploy.add_argument(
+ '--container-init',
+ action='store_true',
+ default=CONTAINER_INIT,
+ help=argparse.SUPPRESS)
+ parser_deploy.add_argument(
+ '--memory-request',
+ help='Container memory request/target'
+ )
+ parser_deploy.add_argument(
+ '--memory-limit',
+ help='Container memory hard limit'
+ )
+ parser_deploy.add_argument(
+ '--meta-json',
+ help='JSON dict of additional metadata'
+ )
+ parser_deploy.add_argument(
+ '--extra-container-args',
+ action='append',
+ default=[],
+ help='Additional container arguments to apply to deamon'
+ )
+
+ parser_check_host = subparsers.add_parser(
+ 'check-host', help='check host configuration')
+ parser_check_host.set_defaults(func=command_check_host)
+ parser_check_host.add_argument(
+ '--expect-hostname',
+ help='Check that hostname matches an expected value')
+
+ parser_prepare_host = subparsers.add_parser(
+ 'prepare-host', help='prepare a host for cephadm use')
+ parser_prepare_host.set_defaults(func=command_prepare_host)
+ parser_prepare_host.add_argument(
+ '--expect-hostname',
+ help='Set hostname')
+
+ parser_add_repo = subparsers.add_parser(
+ 'add-repo', help='configure package repository')
+ parser_add_repo.set_defaults(func=command_add_repo)
+ parser_add_repo.add_argument(
+ '--release',
+ help='use latest version of a named release (e.g., {})'.format(LATEST_STABLE_RELEASE))
+ parser_add_repo.add_argument(
+ '--version',
+ help='use specific upstream version (x.y.z)')
+ parser_add_repo.add_argument(
+ '--dev',
+ help='use specified bleeding edge build from git branch or tag')
+ parser_add_repo.add_argument(
+ '--dev-commit',
+ help='use specified bleeding edge build from git commit')
+ parser_add_repo.add_argument(
+ '--gpg-url',
+ help='specify alternative GPG key location')
+ parser_add_repo.add_argument(
+ '--repo-url',
+ default='https://download.ceph.com',
+ help='specify alternative repo location')
+ # TODO: proxy?
+
+ parser_rm_repo = subparsers.add_parser(
+ 'rm-repo', help='remove package repository configuration')
+ parser_rm_repo.set_defaults(func=command_rm_repo)
+
+ parser_install = subparsers.add_parser(
+ 'install', help='install ceph package(s)')
+ parser_install.set_defaults(func=command_install)
+ parser_install.add_argument(
+ 'packages', nargs='*',
+ default=['cephadm'],
+ help='packages')
+
+ parser_registry_login = subparsers.add_parser(
+ 'registry-login', help='log host into authenticated registry')
+ parser_registry_login.set_defaults(func=command_registry_login)
+ parser_registry_login.add_argument(
+ '--registry-url',
+ help='url for custom registry')
+ parser_registry_login.add_argument(
+ '--registry-username',
+ help='username for custom registry')
+ parser_registry_login.add_argument(
+ '--registry-password',
+ help='password for custom registry')
+ parser_registry_login.add_argument(
+ '--registry-json',
+ help='json file with custom registry login info (URL, Username, Password)')
+ parser_registry_login.add_argument(
+ '--fsid',
+ help='cluster FSID')
+
+ parser_gather_facts = subparsers.add_parser(
+ 'gather-facts', help='gather and return host related information (JSON format)')
+ parser_gather_facts.set_defaults(func=command_gather_facts)
+
+ parser_exporter = subparsers.add_parser(
+ 'exporter', help='Start cephadm in exporter mode (web service), providing host/daemon/disk metadata')
+ parser_exporter.add_argument(
+ '--fsid',
+ required=True,
+ type=str,
+ help='fsid of the cephadm exporter to run against')
+ parser_exporter.add_argument(
+ '--port',
+ type=int,
+ default=int(CephadmDaemon.default_port),
+ help='port number for the cephadm exporter service')
+ parser_exporter.add_argument(
+ '--id',
+ type=str,
+ default=get_hostname().split('.')[0],
+ help='daemon identifer for the exporter')
+ parser_exporter.set_defaults(func=command_exporter)
+
+ parser_maintenance = subparsers.add_parser(
+ 'host-maintenance', help='Manage the maintenance state of a host')
+ parser_maintenance.add_argument(
+ '--fsid',
+ help='cluster FSID')
+ parser_maintenance.add_argument(
+ 'maintenance_action',
+ type=str,
+ choices=['enter', 'exit'],
+ help='Maintenance action - enter maintenance, or exit maintenance')
+ parser_maintenance.set_defaults(func=command_maintenance)
+
+ parser_disk_rescan = subparsers.add_parser(
+ 'disk-rescan', help='rescan all HBAs to detect new/removed devices')
+ parser_disk_rescan.set_defaults(func=command_rescan_disks)
+
+ return parser
+
+
+def _parse_args(av: List[str]) -> argparse.Namespace:
+ parser = _get_parser()
+
+ args = parser.parse_args(av)
+ if 'command' in args and args.command and args.command[0] == '--':
+ args.command.pop(0)
+
+ # workaround argparse to deprecate the subparser `--container-init` flag
+ # container_init and no_container_init must always be mutually exclusive
+ container_init_args = ('--container-init', '--no-container-init')
+ if set(container_init_args).issubset(av):
+ parser.error('argument %s: not allowed with argument %s' % (container_init_args))
+ elif '--container-init' in av:
+ args.no_container_init = not args.container_init
+ else:
+ args.container_init = not args.no_container_init
+ assert args.container_init is not args.no_container_init
+
+ return args
+
+
+def cephadm_init_ctx(args: List[str]) -> CephadmContext:
+ ctx = CephadmContext()
+ ctx.set_args(_parse_args(args))
+ return ctx
+
+
+def cephadm_init_logging(ctx: CephadmContext, args: List[str]) -> None:
+ """Configure the logging for cephadm as well as updating the system
+ to have the expected log dir and logrotate configuration.
+ """
+ global logger
+ if not os.path.exists(LOG_DIR):
+ os.makedirs(LOG_DIR)
+ dictConfig(logging_config)
+ logger = logging.getLogger()
+
+ if not os.path.exists(ctx.logrotate_dir + '/cephadm'):
+ with open(ctx.logrotate_dir + '/cephadm', 'w') as f:
+ f.write("""# created by cephadm
+/var/log/ceph/cephadm.log {
+ rotate 7
+ daily
+ compress
+ missingok
+ notifempty
+ su root root
+}
+""")
+
+ if ctx.verbose:
+ for handler in logger.handlers:
+ if handler.name == 'console':
+ handler.setLevel(logging.DEBUG)
+ logger.debug('%s\ncephadm %s' % ('-' * 80, args))
+
+
+def cephadm_require_root() -> None:
+ """Exit if the process is not running as root."""
+ if os.geteuid() != 0:
+ sys.stderr.write('ERROR: cephadm should be run as root\n')
+ sys.exit(1)
+
+
+def main() -> None:
+ av: List[str] = []
+ av = sys.argv[1:]
+
+ ctx = cephadm_init_ctx(av)
+ if not ctx.has_function():
+ sys.stderr.write('No command specified; pass -h or --help for usage\n')
+ sys.exit(1)
+
+ cephadm_require_root()
+ cephadm_init_logging(ctx, av)
+ try:
+ # podman or docker?
+ ctx.container_engine = find_container_engine(ctx)
+ if ctx.func not in \
+ [
+ command_check_host,
+ command_prepare_host,
+ command_add_repo,
+ command_rm_repo,
+ command_install
+ ]:
+ check_container_engine(ctx)
+ # command handler
+ r = ctx.func(ctx)
+ except Error as e:
+ if ctx.verbose:
+ raise
+ logger.error('ERROR: %s' % e)
+ sys.exit(1)
+ if not r:
+ r = 0
+ sys.exit(r)
+
+
+if __name__ == '__main__':
+ main()
diff --git a/src/cephadm/samples/alertmanager.json b/src/cephadm/samples/alertmanager.json
new file mode 100644
index 000000000..bacbad300
--- /dev/null
+++ b/src/cephadm/samples/alertmanager.json
@@ -0,0 +1,27 @@
+{
+ "files": {
+ "alertmanager.yml": [
+ "global:",
+ " resolve_timeout: 5m",
+ "",
+ "route:",
+ " group_by: ['alertname']",
+ " group_wait: 10s",
+ " group_interval: 10s",
+ " repeat_interval: 1h",
+ " receiver: 'web.hook'",
+ "receivers:",
+ "- name: 'web.hook'",
+ " webhook_configs:",
+ " - url: 'http://127.0.0.1:5001/'",
+ "inhibit_rules:",
+ " - source_match:",
+ " severity: 'critical'",
+ " target_match:",
+ " severity: 'warning'",
+ " equal: ['alertname', 'dev', 'instance']"
+ ]
+ },
+ "peers": []
+}
+
diff --git a/src/cephadm/samples/custom_container.json b/src/cephadm/samples/custom_container.json
new file mode 100644
index 000000000..194a44d2a
--- /dev/null
+++ b/src/cephadm/samples/custom_container.json
@@ -0,0 +1,35 @@
+{
+ "image": "docker.io/prom/alertmanager:v0.20.0",
+ "ports": [9093, 9094],
+ "args": [
+ "-p", "9093:9093",
+ "-p", "9094:9094"
+ ],
+ "dirs": ["etc/alertmanager"],
+ "files": {
+ "etc/alertmanager/alertmanager.yml": [
+ "global:",
+ " resolve_timeout: 5m",
+ "",
+ "route:",
+ " group_by: ['alertname']",
+ " group_wait: 10s",
+ " group_interval: 10s",
+ " repeat_interval: 1h",
+ " receiver: 'web.hook'",
+ "receivers:",
+ "- name: 'web.hook'",
+ " webhook_configs:",
+ " - url: 'http://127.0.0.1:5001/'",
+ "inhibit_rules:",
+ " - source_match:",
+ " severity: 'critical'",
+ " target_match:",
+ " severity: 'warning'",
+ " equal: ['alertname', 'dev', 'instance']"
+ ]
+ },
+ "volume_mounts": {
+ "etc/alertmanager": "/etc/alertmanager"
+ }
+}
diff --git a/src/cephadm/samples/grafana.json b/src/cephadm/samples/grafana.json
new file mode 100644
index 000000000..0e0689b7e
--- /dev/null
+++ b/src/cephadm/samples/grafana.json
@@ -0,0 +1,90 @@
+{
+ "files": {
+ "grafana.ini": [
+ "[users]",
+ " default_theme = light",
+ "[auth.anonymous]",
+ " enabled = true",
+ " org_name = 'Main Org.'",
+ " org_role = 'Viewer'",
+ "[server]",
+ " domain = 'bootstrap.storage.lab'",
+ " protocol = https",
+ " cert_file = /etc/grafana/certs/cert_file",
+ " cert_key = /etc/grafana/certs/cert_key",
+ " http_port = 3000",
+ " http_addr = localhost",
+ "[security]",
+ " admin_user = admin",
+ " admin_password = admin",
+ " allow_embedding = true"
+ ],
+ "provisioning/datasources/ceph-dashboard.yml": [
+ "deleteDatasources:",
+ " - name: 'Dashboard'",
+ " orgId: 1",
+ " ",
+ "datasources:",
+ " - name: 'Dashboard'",
+ " type: 'prometheus'",
+ " access: 'proxy'",
+ " orgId: 1",
+ " url: 'http://localhost:9095'",
+ " basicAuth: false",
+ " isDefault: true",
+ " editable: false"
+ ],
+ "certs/cert_file": [
+ "-----BEGIN CERTIFICATE-----",
+ "MIIDLTCCAhWgAwIBAgIUEH0mq6u93LKsWlNXst5pxWcuqkQwDQYJKoZIhvcNAQEL",
+ "BQAwJjELMAkGA1UECgwCSVQxFzAVBgNVBAMMDmNlcGgtZGFzaGJvYXJkMB4XDTIw",
+ "MDEwNTIyNDYyMFoXDTMwMDEwMjIyNDYyMFowJjELMAkGA1UECgwCSVQxFzAVBgNV",
+ "BAMMDmNlcGgtZGFzaGJvYXJkMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKC",
+ "AQEAqxh6eO0NTZJe+DoKZG/kozJCf+83eB3gWzwXoNinRmV/49f5WPR20DIxAe0R",
+ "saO6XynJXTrhvXT1bsARUq+LSmjWNFoYXopFuOJhGdWn4dmpuHwtpcFv2kjzNOKj",
+ "U2EG8j6bsRp1jFAzn7kdbSWT0UHySRXp9DPAjDiF3LjykMXiJMReccFXrB1pRi93",
+ "nJxED8d6oT5GazGB44svb+Zi6ABamZu5SDJC1Fr/O5rWFNQkH4hQEqDPj1817H9O",
+ "sm0mZiNy77ZQuAzOgZN153L3QOsyJismwNHfAMGMH9mzPKOjyhc13VlZyeEzml8p",
+ "ZpWQ2gi8P2r/FAr8bFL3MFnHKwIDAQABo1MwUTAdBgNVHQ4EFgQUZg3v7MX4J+hx",
+ "w3HENCrUkMK8tbwwHwYDVR0jBBgwFoAUZg3v7MX4J+hxw3HENCrUkMK8tbwwDwYD",
+ "VR0TAQH/BAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAQEAaR/XPGKwUgVwH3KXAb6+",
+ "s9NTAt6lCmFdQz1ngoqFSizW7KGSXnOgd6xTiUCR0Tjjo2zKCwhIINaI6mwqMbrg",
+ "BOjb7diaqwFaitRs27AtdmaqMGndUqEBUn/k64Ld3VPGL4p0W2W+tXsyzZg1qQIn",
+ "JXb7c4+oWzXny7gHFheYQTwnHzDcNOf9vJiMGyYYvU1xTOGucu6dwtOVDDe1Z4Nq",
+ "AyIYWDScRr2FeAOXyx4aW2v5bjpTxvP+79/OOBbQ+p4y5F4PDrPeOSweGoo6huTR",
+ "+T+YI9Jfw2XCgV7NHWhfdt3fHHwUQzO6WszWU557pmCODLvXWsQ8P+GRiG7Nywm3",
+ "uA==",
+ "-----END CERTIFICATE-----"
+ ],
+ "certs/cert_key": [
+ "-----BEGIN PRIVATE KEY-----",
+ "MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQCrGHp47Q1Nkl74",
+ "Ogpkb+SjMkJ/7zd4HeBbPBeg2KdGZX/j1/lY9HbQMjEB7RGxo7pfKcldOuG9dPVu",
+ "wBFSr4tKaNY0WhheikW44mEZ1afh2am4fC2lwW/aSPM04qNTYQbyPpuxGnWMUDOf",
+ "uR1tJZPRQfJJFen0M8CMOIXcuPKQxeIkxF5xwVesHWlGL3ecnEQPx3qhPkZrMYHj",
+ "iy9v5mLoAFqZm7lIMkLUWv87mtYU1CQfiFASoM+PXzXsf06ybSZmI3LvtlC4DM6B",
+ "k3XncvdA6zImKybA0d8AwYwf2bM8o6PKFzXdWVnJ4TOaXylmlZDaCLw/av8UCvxs",
+ "UvcwWccrAgMBAAECggEAeBv0BiYrm5QwdUORfhaKxAIJavRM1Vbr5EBYOgM90o54",
+ "bEN2ePsM2XUSsE5ziGfu8tVL1dX7GNwdW8UbpBc1ymO0VAYXa27YKUVKcy9o7oS1",
+ "v5v1E5Kq6esiSLL9gw/vJ2nKNFblxD2dL/hs7u1dSp5n7uSiW1tlRUp8toljRzts",
+ "1Cenp0J/a82HwWDE8j/H9NvitTOZ2cdwJ76V8GkBynlvr2ARjRfZGx0WXEJmoZYD",
+ "YUQVU303DB6Q2tkFco4LbPofkuhhMPhXsz3fZ/blHj/c78tqP9L5sQ29oqoPE1pS",
+ "DBOwKC/eoi5FY34RdLNL0dKq9MzbuYqEcCfZOJgxoQKBgQDf+5XF+aXQz2OmSaj6",
+ "1Yr+3KAKdfX/AYp22X1Wy4zWcZlgujgwQ1FG0zay8HVBM0/xn4UgOtcKCoXibePh",
+ "ag1t8aZINdRE1JcMzKmZoSvU9Xk30CNvygizuJVEKsJFPDbPzCpauDSplzcQb4pZ",
+ "wepucPuowkPMBx0iU3x0qSThWwKBgQDDjYs7d30xxSqWWXyCOZshy7UtHMNfqP15",
+ "kDfTXIZzuHvDf6ZNci10VY1eDZbpZfHgc6x1ElbKv2H4dYsgkENJZUi1YQDpVPKq",
+ "4N5teNykgAuagiR7dRFltSju3S7hIE6HInTv3hShaFPymlEE7zuBMuEUcuvYz5YN",
+ "RjxsvypKcQKBgCuuV+Y1KqZPW8K5SNAqRyIvCrMfkCr8NPG6tpvvtHa5zsyzZHPd",
+ "HQOv+1HoXSWrCSM5FfBUKU3XAYdIIRH76cSQRPp+LPiDcTXY0Baa/P5aJRrCZ7bM",
+ "cugBznJt2FdCR/o8eeIZXIPabq2w4w1gKQUC2cFuqWQn2wGvwGzL89pTAoGAAfpx",
+ "mSVpT9KVzrWTC+I3To04BP/QfixAfDVYSzwZZBxOrDijXw8zpISlDHmIuE2+t62T",
+ "5g9Mb3qmLBRMVwT+mUR8CtGzZ6jjV5U0yti5KrTc6TA93D3f8i51/oygR8jC4p0X",
+ "n8GYZdWfW8nx3eHpsTHpkwJinmvjMbkvLU51yBECgYAnUAMyhNOWjbYS5QWd8i1W",
+ "SFQansVDeeT98RebrzmGwlgrCImHItJz0Tz8gkNB3+S2B2balqT0WHaDxQ8vCtwX",
+ "xB4wd+gMomgdYtHGRnRwj1UyRXDk0c1TgGdRjOn3URaezBMibHTQSbFgPciJgAuU",
+ "mEl75h1ToBX9yvnH39o50g==",
+ "-----END PRIVATE KEY-----"
+ ]
+ }
+}
diff --git a/src/cephadm/samples/nfs.json b/src/cephadm/samples/nfs.json
new file mode 100644
index 000000000..2e6625101
--- /dev/null
+++ b/src/cephadm/samples/nfs.json
@@ -0,0 +1,14 @@
+{
+ "pool" : "nfs-ganesha",
+ "namespace" : "nfs-ns",
+ "files": {
+ "ganesha.conf": [
+ "RADOS_URLS {",
+ " userid = admin;",
+ "}",
+ "",
+ "%url rados://nfs-ganesha/nfs-ns/conf-nfs.a",
+ ""
+ ]
+ }
+}
diff --git a/src/cephadm/samples/prometheus.json b/src/cephadm/samples/prometheus.json
new file mode 100644
index 000000000..64727fb59
--- /dev/null
+++ b/src/cephadm/samples/prometheus.json
@@ -0,0 +1,17 @@
+{
+ "files": {
+ "prometheus.yml": [
+ "global:",
+ " scrape_interval: 5s",
+ " evaluation_interval: 10s",
+ "",
+ "rule_files: ",
+ " - '/etc/prometheus/alerting/*'",
+ "",
+ "scrape_configs:",
+ " - job_name: 'prometheus'",
+ " static_configs:",
+ " - targets: ['localhost:9095']"
+ ]
+ }
+}
diff --git a/src/cephadm/samples/rgw_ssl.json b/src/cephadm/samples/rgw_ssl.json
new file mode 100644
index 000000000..3fe6fea1c
--- /dev/null
+++ b/src/cephadm/samples/rgw_ssl.json
@@ -0,0 +1,101 @@
+{
+ "rgw_realm": "default",
+ "rgw_zone": "default",
+ "service_type": "rgw",
+ "placement": {
+ "hosts": [{
+ "hostname": "ironic-moliver",
+ "name": "",
+ "network": ""
+ }],
+ "count": 1
+ },
+ "ssl": true,
+ "rgw_frontend_port": 4343,
+ "rgw_frontend_ssl_certificate": [
+ "-----BEGIN CERTIFICATE-----",
+ "MIIFmjCCA4KgAwIBAgIJAIZ2n35bmwXTMA0GCSqGSIb3DQEBCwUAMGIxCzAJBgNV",
+ "BAYTAkFVMQwwCgYDVQQIDANOU1cxHTAbBgNVBAoMFEV4YW1wbGUgUkdXIFNTTCBp",
+ "bmMuMSYwJAYDVQQDDB1yZ3ctZW5kcG9pbnQuZXhhbXBsZS1jZXBoLmNvbTAeFw0y",
+ "MDAyMDcwMDEzNTFaFw0zMDAyMDQwMDEzNTFaMGIxCzAJBgNVBAYTAkFVMQwwCgYD",
+ "VQQIDANOU1cxHTAbBgNVBAoMFEV4YW1wbGUgUkdXIFNTTCBpbmMuMSYwJAYDVQQD",
+ "DB1yZ3ctZW5kcG9pbnQuZXhhbXBsZS1jZXBoLmNvbTCCAiIwDQYJKoZIhvcNAQEB",
+ "BQADggIPADCCAgoCggIBAMptGJ523QkEbc37za8iuCTahj0Zr6hy+ToSX/Vfdzxj",
+ "iYHuD2PiZZyJB7t2eOqiA8sQ5N513EUtf2ZIBwtnnqFIzD5TqI3BxRajUTlOyXUX",
+ "onMwQwXu2ifDUy3LCmuQfzanOTWvVLac1NmkWbJHpJCXYbUnPb1Nvd0QjTTEH1jt",
+ "5bDHhfxwCIYK6PY+MqC72a09wB2ZF+EKsSdqghOKmibfJHtoJdsqGeLrysBLrzUJ",
+ "e/5ZW3V4Z85T2lja5KZnWgRofrUy5TmJV10HO4Hht92xvWvEi/rmjg2AVYZFUQQx",
+ "xKXpUBbF5T46eSVmaT7IH88Yp5ytgBTaigym7ETCjohp/DfCaK1DUehh0ce7iUq2",
+ "yCLviZsX4WdPYxzkoLflNrqm4YZP6iKcZSUR/A+qPKCzCXgMXFNA1JxilDwEq35F",
+ "zGN++ehJqdNmOQ1eQScsLwZQa6mC97d+upWdCvyntf1+S6vNcXhtRQpjNM4W37oW",
+ "r5nicsGA3/0rpDEHZW85KlkdWO1uCS/6ftgt8UUMaf5ew3PigzusqymBWTlMOjtW",
+ "uAQXxgZZvkRp+xdspn/uTCAP+bNShGD6Q+TO3U6IjTqHk83sGKCvg2dyU/dqgPr9",
+ "2IIzgQBFGk0W0nM/E83E8hUSwX17COLL3drhPZb4VRMChQ8PAa6u9nIymkX2wSVv",
+ "AgMBAAGjUzBRMB0GA1UdDgQWBBSsZHuY7KK80RrZHp+Gx+k16skuRDAfBgNVHSME",
+ "GDAWgBSsZHuY7KK80RrZHp+Gx+k16skuRDAPBgNVHRMBAf8EBTADAQH/MA0GCSqG",
+ "SIb3DQEBCwUAA4ICAQAE+BLtnu0p8FtK7vrBCRcCdvycWaSFGJUt7r5Nm8TD7sKw",
+ "bWeDLgXrRouyA7n6yt/JqQbXYcxt4MLAM0P6NQd5BlNrrnDk4rBnJiJgejppNE+S",
+ "BazR7Dv0uYcs8kPT4DPpwzv4aJ2aXCBaxYrq8Rx2xOqANCPVOrtPUk9yGpaQ5adU",
+ "GfxkVbpgIEz1c71PeQuK1KUU/Wpk7cpm+FQCizl9ftP2lHWsGhSLCuyWoMTjt68P",
+ "gYEWoV54eo/bzwj2ei6TcfNo+uHyzEiiG2qEvMh/cnYUFzs8O1t0mN19WPB1pSh1",
+ "faci5lGdtkRbLgP0g5RvpagE7Lw3mCc5Om8jmHs4mPfuVkssBVV23CrFpqLLrDX3",
+ "Acwb/zRGvA7T4WESBTJMYFOLgm0W0Y+AN8RcYNU9QbDhe++Te0uz/3Sy3GN2Xg5z",
+ "MxfD1+34x6KvMfCh8NjII2mFQ9ukcfrhcfO3oWDLlwsqlVbhkZxNiUOEIx9nzHcF",
+ "kWpZ2ypBDH45h2o3LyqvGjsu/BFkeG6JpEDCWbClKWcjKxOrLVDufhSDduffDjja",
+ "zOsgQJg0Yf//Ubb5p0c54GjHM/XDXEcV3m3sEtbmMYz6xGwuag4bx8P2E/QY8sFp",
+ "JxgIdS8vdl6YhDCjKJ2XzI30JwCdftgDIAiWSE0ivoDc+8+gG1nb11GT52HFzA==",
+ "-----END CERTIFICATE-----",
+ "-----BEGIN PRIVATE KEY-----",
+ "MIIJQwIBADANBgkqhkiG9w0BAQEFAASCCS0wggkpAgEAAoICAQDKbRiedt0JBG3N",
+ "+82vIrgk2oY9Ga+ocvk6El/1X3c8Y4mB7g9j4mWciQe7dnjqogPLEOTeddxFLX9m",
+ "SAcLZ56hSMw+U6iNwcUWo1E5Tsl1F6JzMEMF7tonw1MtywprkH82pzk1r1S2nNTZ",
+ "pFmyR6SQl2G1Jz29Tb3dEI00xB9Y7eWwx4X8cAiGCuj2PjKgu9mtPcAdmRfhCrEn",
+ "aoITipom3yR7aCXbKhni68rAS681CXv+WVt1eGfOU9pY2uSmZ1oEaH61MuU5iVdd",
+ "BzuB4bfdsb1rxIv65o4NgFWGRVEEMcSl6VAWxeU+OnklZmk+yB/PGKecrYAU2ooM",
+ "puxEwo6Iafw3wmitQ1HoYdHHu4lKtsgi74mbF+FnT2Mc5KC35Ta6puGGT+oinGUl",
+ "EfwPqjygswl4DFxTQNScYpQ8BKt+RcxjfvnoSanTZjkNXkEnLC8GUGupgve3frqV",
+ "nQr8p7X9fkurzXF4bUUKYzTOFt+6Fq+Z4nLBgN/9K6QxB2VvOSpZHVjtbgkv+n7Y",
+ "LfFFDGn+XsNz4oM7rKspgVk5TDo7VrgEF8YGWb5EafsXbKZ/7kwgD/mzUoRg+kPk",
+ "zt1OiI06h5PN7Bigr4NnclP3aoD6/diCM4EARRpNFtJzPxPNxPIVEsF9ewjiy93a",
+ "4T2W+FUTAoUPDwGurvZyMppF9sElbwIDAQABAoICAQC4sATwP563pXTRpNYq3lCI",
+ "P2COyqq70/qUA0PNygYt8Nr60srz5RG0WknVvefgm2U+lvFaDsqjyzkbhsf2ndnb",
+ "aWH/07BLdeluGB/5W2rvDFtJIVVlSmF8OffgJgohzbpjkPrfglKWMkz5LbwwrrD0",
+ "w0mAUIdB+nYqBfnvlKjNKHCSc9hJU6ZTNg0K7gCfKgUWzOpFlvJ0fp7XSZPYZHL0",
+ "2E6e0Y0Ig0cPBPb9r4/xoe+hRsHtUafUVik3PK+1K0K0FurUQ9VkQ2yUEg83F0v8",
+ "Vzht5OuaRVSB+P8O/JtIamfywAY0YOYhepQhjWikwU5UUzhJ+PqNDD87/+g9bA1B",
+ "xC25eoDxThiQlgDmRoH18ZsWDVf9TuJnm4cpxnZYX6ip+BLm/aidT39auZo0Fl+r",
+ "cJxRn0Qlm0Vm4Tc/6ZG6PQWB+Q6CjVFdoxeOvEQcTSuKA6VZBStLmqX++5In1Lmj",
+ "hVr3/aueHiZvXS5bNIdd2IfzatR+nP+uxzM/ryJRvGO2B2XTS00Cvv/lH84BDJYV",
+ "yt1PJIBoM9Dh7aUAHmKNVfRt83xzvcSPZx9VmSzA6wwqCQcO1GJk6keAuxOuligu",
+ "YdSFcfChOg90WvBcl+NzMblLkwrFSBQR7kgG0+dedv+Wkm4xO4T7B4W2G5+VIJKG",
+ "mrEAq6XQMFnfEJzNVg7JUQKCAQEA91eMvphoVVz+cxa4Ew7OokNXk5kSlvmQ8+Ij",
+ "ngFBvniXPZecxVzFEJglSthH5KI2ZqxwF3GJhKjxUihwf6K13Hx54EM7S/qV57ie",
+ "kVeKdAs+SGv+hRk1gQOoPBInbtKGKTni1V8T7iNginLueC/YikFugzv6IxiliBSG",
+ "3R7zjRepOW69aEoCPecx9amU4CkAwgeLJgBloBoqWD8sKM+bl7p5juQCU2sQ9D4/",
+ "kLnpG9+zPRUNjI4sog3L1wql3zthI6/4gf0TNuDhJTZ68vpMSi02pOUkVa0MmVOA",
+ "ex16luIp0BhxG/sUAeoevFL4KqR0CBbyAstbt2E/oPYOWMJ4MwKCAQEA0YMNXY7I",
+ "RNFOmiZ2Wn8kENCyJguqbOMd/li2+ercgp3MaSgTjC5KDFvZqTwXXlrURPu9hcyv",
+ "sJBSjp45g1T1LsUUq8UJgOIQgxykurIstGg33TAR+TN9VUu/xonLQF23GD8M6Vzd",
+ "EcZEVlBY33hgNXw4mRcBPnaoG5FZCBfHOgdBCExoYKW/RNKcmu0q+h9uhDBCbopv",
+ "04ROzw+HW1qc5qvNPR47buZ9+5QdonVK8s2bguMJ0phXwdSxL21wsjIsXyAO9m7w",
+ "qLHOq/hVokM0Fki09Exg4ppB8cLHC2ITpsVSgn4Dcz5zRtyvhozSKX4R9kMC64a0",
+ "AgMPVMllmGlR1QKCAQBIIGCrh7gNBIQyvXJKJGw/RxH3uZCBNB9/7vbh8Y3hZDr+",
+ "PAL8TpQsxaFCVRWJ53+jqy84tuQaKkXM5nv/zEvqEuZbbl+RRW6HVv/udC+srUap",
+ "Scy7tWEz0QQzGDwlhgCXbwjlnccrD2fsl51QsOsdTf1TCZ9ksqA6sXmua4MsJrUz",
+ "SUa0bbh/oraf46bFQ0+0RQzftQftixPEDg/rirbdpQQjlfvTpYoZHzncE0qV1ULo",
+ "UgZUcXU0gH9rovBBy4gFJyB5j3oV67fb6SorRrAOhWbE6QkSbtcYsw/pVuxTqXn1",
+ "89qwBSSNdl8mHa++h5xKa56BEBobvKEYaAhA+9yfAoIBAQDPFEE5n87Cdj7CjhGd",
+ "EN2M4Tmz8QPZ7AHRS85O5pxdXaqUpU/s1jPKU16nrwVJ9WypYkjI3q8oTP3MiQc/",
+ "j9FnENSFkpL6GHdJoB4Rido11myg6spZDVNr4xsCGWATlo1KIceZQHghAV66EWBG",
+ "QKyXMNigN+S64Hz4AomFPjtkV5cnpJ3mKO0MET9IwfIglsCdVzXSHHK7FaLvdeHL",
+ "oZxDQrvxFNiZnKgY6SUBVf1mT2LN06n5xSm4I4md3wXsmzrQKtefK7gihNxJjYLW",
+ "hqYNAIAalwOL9fwIAQTLc30I8S/EWtj+J1O5TpcO3lE7QahvR3yzXsi81Flq7ETG",
+ "iBKhAoIBAGHGpnjrLlCarNk9axh4Dw1OjgEvwPlEqsiWXt2tylLeab0OGC47MmJx",
+ "RmKwgVukMuxkQb8v4ANSRtih7R+E+qXfexjEFYtzh/uaRP1Z7ZrO/oqq0oLbPpsx",
+ "yTSRDL1i5/fgdIlKVH3N4IF7E8Pc3REgYIwLQxYjTdgVHEAM65XegQ2Lkpr4iae3",
+ "hm4IsD2PrsVITrlsLg65XnfcbsCs/OfQ5GuUp+xUBw5e0bQBmsWEiCaCjrq/EHJa",
+ "/oeJRqS7lyGYDC+wiSsE70x4dvu1um2F+V1Jw4LWjhu8Z8dNSXPSf8vLqXGkWAlk",
+ "805lq+iy7Mkhb+dlr4R9WhMWDyGwgYs=",
+ "-----END PRIVATE KEY-----"
+ ]
+}
diff --git a/src/cephadm/tests/__init__.py b/src/cephadm/tests/__init__.py
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/src/cephadm/tests/__init__.py
diff --git a/src/cephadm/tests/fixtures.py b/src/cephadm/tests/fixtures.py
new file mode 100644
index 000000000..45990d910
--- /dev/null
+++ b/src/cephadm/tests/fixtures.py
@@ -0,0 +1,126 @@
+import mock
+import os
+import pytest
+import time
+
+from contextlib import contextmanager
+from pyfakefs import fake_filesystem
+
+from typing import Callable, Dict, List, Optional
+
+
+with mock.patch('builtins.open', create=True):
+ from importlib.machinery import SourceFileLoader
+ cd = SourceFileLoader('cephadm', 'cephadm').load_module()
+
+
+def mock_docker():
+ docker = mock.Mock(cd.Docker)
+ docker.path = '/usr/bin/docker'
+ return docker
+
+
+def mock_podman():
+ podman = mock.Mock(cd.Podman)
+ podman.path = '/usr/bin/podman'
+ podman.version = (2, 1, 0)
+ return podman
+
+
+def _daemon_path():
+ return os.getcwd()
+
+
+def mock_bad_firewalld():
+ def raise_bad_firewalld():
+ raise Exception('Called bad firewalld')
+ f = mock.Mock(cd.Firewalld)
+ f.enable_service_for = lambda _ : raise_bad_firewalld()
+ f.apply_rules = lambda : raise_bad_firewalld()
+ f.open_ports = lambda _ : raise_bad_firewalld()
+
+def _mock_scrape_host(obj, interval):
+ try:
+ raise ValueError("wah")
+ except Exception as e:
+ obj._handle_thread_exception(e, 'host')
+
+
+def _mock_run(obj):
+ t = obj._create_thread(obj._scrape_host_facts, 'host', 5)
+ time.sleep(1)
+ if not t.is_alive():
+ obj.cephadm_cache.update_health('host', "inactive", "host thread stopped")
+
+
+@pytest.fixture
+def exporter():
+ with mock.patch('cephadm.CephadmDaemon.daemon_path', _daemon_path()), \
+ mock.patch('cephadm.CephadmDaemon.can_run', return_value=True), \
+ mock.patch('cephadm.CephadmDaemon.run', _mock_run), \
+ mock.patch('cephadm.CephadmDaemon._scrape_host_facts', _mock_scrape_host):
+
+ ctx = cd.CephadmContext()
+ exporter = cd.CephadmDaemon(ctx, fsid='foobar', daemon_id='test')
+ assert exporter.token == 'MyAccessToken'
+ yield exporter
+
+
+@pytest.fixture()
+def cephadm_fs(
+ fs: fake_filesystem.FakeFilesystem,
+):
+ """
+ use pyfakefs to stub filesystem calls
+ """
+ uid = os.getuid()
+ gid = os.getgid()
+
+ with mock.patch('os.fchown'), \
+ mock.patch('os.fchmod'), \
+ mock.patch('platform.processor', return_value='x86_64'), \
+ mock.patch('cephadm.extract_uid_gid', return_value=(uid, gid)):
+
+ if not fake_filesystem.is_root():
+ fake_filesystem.set_uid(0)
+
+ fs.create_dir(cd.DATA_DIR)
+ fs.create_dir(cd.LOG_DIR)
+ fs.create_dir(cd.LOCK_DIR)
+ fs.create_dir(cd.LOGROTATE_DIR)
+ fs.create_dir(cd.UNIT_DIR)
+
+ yield fs
+
+
+@contextmanager
+def with_cephadm_ctx(
+ cmd: List[str],
+ container_engine: Callable = mock_podman(),
+ list_networks: Optional[Dict[str,Dict[str,List[str]]]] = None,
+ hostname: Optional[str] = None,
+):
+ """
+ :param cmd: cephadm command argv
+ :param container_engine: container engine mock (podman or docker)
+ :param list_networks: mock 'list-networks' return
+ :param hostname: mock 'socket.gethostname' return
+ """
+ if not hostname:
+ hostname = 'host1'
+
+ with mock.patch('cephadm.attempt_bind'), \
+ mock.patch('cephadm.call', return_value=('', '', 0)), \
+ mock.patch('cephadm.call_timeout', return_value=0), \
+ mock.patch('cephadm.find_executable', return_value='foo'), \
+ mock.patch('cephadm.is_available', return_value=True), \
+ mock.patch('cephadm.json_loads_retry', return_value={'epoch' : 1}), \
+ mock.patch('socket.gethostname', return_value=hostname):
+ ctx: cd.CephadmContext = cd.cephadm_init_ctx(cmd)
+ ctx.container_engine = container_engine
+ if list_networks is not None:
+ with mock.patch('cephadm.list_networks', return_value=list_networks):
+ yield ctx
+ else:
+ yield ctx
+
diff --git a/src/cephadm/tests/test_cephadm.py b/src/cephadm/tests/test_cephadm.py
new file mode 100644
index 000000000..7383ac0ee
--- /dev/null
+++ b/src/cephadm/tests/test_cephadm.py
@@ -0,0 +1,2413 @@
+# type: ignore
+
+import errno
+import json
+import mock
+import os
+import pytest
+import socket
+import sys
+import time
+import threading
+import unittest
+
+from http.server import HTTPServer
+from textwrap import dedent
+from urllib.request import Request, urlopen
+from urllib.error import HTTPError
+
+from typing import List, Optional
+
+from .fixtures import (
+ cephadm_fs,
+ exporter,
+ mock_docker,
+ mock_podman,
+ with_cephadm_ctx,
+ mock_bad_firewalld,
+)
+
+from pyfakefs import fake_filesystem
+from pyfakefs import fake_filesystem_unittest
+
+with mock.patch('builtins.open', create=True):
+ from importlib.machinery import SourceFileLoader
+ cd = SourceFileLoader('cephadm', 'cephadm').load_module()
+
+
+def get_ceph_conf(
+ fsid='00000000-0000-0000-0000-0000deadbeef',
+ mon_host='[v2:192.168.1.1:3300/0,v1:192.168.1.1:6789/0]'):
+ return f'''
+# minimal ceph.conf for {fsid}
+[global]
+ fsid = {fsid}
+ mon_host = {mon_host}
+'''
+
+class TestCephAdm(object):
+
+ def test_docker_unit_file(self):
+ ctx = cd.CephadmContext()
+ ctx.container_engine = mock_docker()
+ r = cd.get_unit_file(ctx, '9b9d7609-f4d5-4aba-94c8-effa764d96c9')
+ assert 'Requires=docker.service' in r
+ ctx.container_engine = mock_podman()
+ r = cd.get_unit_file(ctx, '9b9d7609-f4d5-4aba-94c8-effa764d96c9')
+ assert 'Requires=docker.service' not in r
+
+ @mock.patch('cephadm.logger')
+ def test_attempt_bind(self, logger):
+ ctx = None
+ address = None
+ port = 0
+
+ def os_error(errno):
+ _os_error = OSError()
+ _os_error.errno = errno
+ return _os_error
+
+ for side_effect, expected_exception in (
+ (os_error(errno.EADDRINUSE), cd.PortOccupiedError),
+ (os_error(errno.EAFNOSUPPORT), cd.Error),
+ (os_error(errno.EADDRNOTAVAIL), cd.Error),
+ (None, None),
+ ):
+ _socket = mock.Mock()
+ _socket.bind.side_effect = side_effect
+ try:
+ cd.attempt_bind(ctx, _socket, address, port)
+ except Exception as e:
+ assert isinstance(e, expected_exception)
+ else:
+ if expected_exception is not None:
+ assert False
+
+ @mock.patch('cephadm.attempt_bind')
+ @mock.patch('cephadm.logger')
+ def test_port_in_use(self, logger, attempt_bind):
+ empty_ctx = None
+
+ assert cd.port_in_use(empty_ctx, 9100) == False
+
+ attempt_bind.side_effect = cd.PortOccupiedError('msg')
+ assert cd.port_in_use(empty_ctx, 9100) == True
+
+ os_error = OSError()
+ os_error.errno = errno.EADDRNOTAVAIL
+ attempt_bind.side_effect = os_error
+ assert cd.port_in_use(empty_ctx, 9100) == False
+
+ os_error = OSError()
+ os_error.errno = errno.EAFNOSUPPORT
+ attempt_bind.side_effect = os_error
+ assert cd.port_in_use(empty_ctx, 9100) == False
+
+ @mock.patch('socket.socket')
+ @mock.patch('cephadm.logger')
+ def test_check_ip_port_success(self, logger, _socket):
+ ctx = cd.CephadmContext()
+ ctx.skip_ping_check = False # enables executing port check with `check_ip_port`
+
+ for address, address_family in (
+ ('0.0.0.0', socket.AF_INET),
+ ('::', socket.AF_INET6),
+ ):
+ try:
+ cd.check_ip_port(ctx, cd.EndPoint(address, 9100))
+ except:
+ assert False
+ else:
+ assert _socket.call_args == mock.call(address_family, socket.SOCK_STREAM)
+
+ @mock.patch('socket.socket')
+ @mock.patch('cephadm.logger')
+ def test_check_ip_port_failure(self, logger, _socket):
+ ctx = cd.CephadmContext()
+ ctx.skip_ping_check = False # enables executing port check with `check_ip_port`
+
+ def os_error(errno):
+ _os_error = OSError()
+ _os_error.errno = errno
+ return _os_error
+
+ for address, address_family in (
+ ('0.0.0.0', socket.AF_INET),
+ ('::', socket.AF_INET6),
+ ):
+ for side_effect, expected_exception in (
+ (os_error(errno.EADDRINUSE), cd.PortOccupiedError),
+ (os_error(errno.EADDRNOTAVAIL), cd.Error),
+ (os_error(errno.EAFNOSUPPORT), cd.Error),
+ (None, None),
+ ):
+ mock_socket_obj = mock.Mock()
+ mock_socket_obj.bind.side_effect = side_effect
+ _socket.return_value = mock_socket_obj
+ try:
+ cd.check_ip_port(ctx, cd.EndPoint(address, 9100))
+ except Exception as e:
+ assert isinstance(e, expected_exception)
+ else:
+ if side_effect is not None:
+ assert False
+
+
+ def test_is_not_fsid(self):
+ assert not cd.is_fsid('no-uuid')
+
+ def test_is_fsid(self):
+ assert cd.is_fsid('e863154d-33c7-4350-bca5-921e0467e55b')
+
+ def test__get_parser_image(self):
+ args = cd._parse_args(['--image', 'foo', 'version'])
+ assert args.image == 'foo'
+
+ def test_parse_mem_usage(self):
+ cd.logger = mock.Mock()
+ len, summary = cd._parse_mem_usage(0, 'c6290e3f1489,-- / --')
+ assert summary == {}
+
+ def test_CustomValidation(self):
+ assert cd._parse_args(['deploy', '--name', 'mon.a', '--fsid', 'fsid'])
+
+ with pytest.raises(SystemExit):
+ cd._parse_args(['deploy', '--name', 'wrong', '--fsid', 'fsid'])
+
+ @pytest.mark.parametrize("test_input, expected", [
+ ("1.6.2", (1,6,2)),
+ ("1.6.2-stable2", (1,6,2)),
+ ])
+ def test_parse_podman_version(self, test_input, expected):
+ assert cd._parse_podman_version(test_input) == expected
+
+ def test_parse_podman_version_invalid(self):
+ with pytest.raises(ValueError) as res:
+ cd._parse_podman_version('inval.id')
+ assert 'inval' in str(res.value)
+
+ def test_is_ipv6(self):
+ cd.logger = mock.Mock()
+ for good in ("[::1]", "::1",
+ "fff:ffff:ffff:ffff:ffff:ffff:ffff:ffff"):
+ assert cd.is_ipv6(good)
+ for bad in ("127.0.0.1",
+ "ffff:ffff:ffff:ffff:ffff:ffff:ffff:fffg",
+ "1:2:3:4:5:6:7:8:9", "fd00::1::1", "[fg::1]"):
+ assert not cd.is_ipv6(bad)
+
+ def test_unwrap_ipv6(self):
+ def unwrap_test(address, expected):
+ assert cd.unwrap_ipv6(address) == expected
+
+ tests = [
+ ('::1', '::1'), ('[::1]', '::1'),
+ ('[fde4:8dba:82e1:0:5054:ff:fe6a:357]', 'fde4:8dba:82e1:0:5054:ff:fe6a:357'),
+ ('can actually be any string', 'can actually be any string'),
+ ('[but needs to be stripped] ', '[but needs to be stripped] ')]
+ for address, expected in tests:
+ unwrap_test(address, expected)
+
+ def test_wrap_ipv6(self):
+ def wrap_test(address, expected):
+ assert cd.wrap_ipv6(address) == expected
+
+ tests = [
+ ('::1', '[::1]'), ('[::1]', '[::1]'),
+ ('fde4:8dba:82e1:0:5054:ff:fe6a:357',
+ '[fde4:8dba:82e1:0:5054:ff:fe6a:357]'),
+ ('myhost.example.com', 'myhost.example.com'),
+ ('192.168.0.1', '192.168.0.1'),
+ ('', ''), ('fd00::1::1', 'fd00::1::1')]
+ for address, expected in tests:
+ wrap_test(address, expected)
+
+ @mock.patch('cephadm.Firewalld', mock_bad_firewalld)
+ @mock.patch('cephadm.logger')
+ def test_skip_firewalld(self, logger, cephadm_fs):
+ """
+ test --skip-firewalld actually skips changing firewall
+ """
+
+ ctx = cd.CephadmContext()
+ with pytest.raises(Exception):
+ cd.update_firewalld(ctx, 'mon')
+
+ ctx.skip_firewalld = True
+ cd.update_firewalld(ctx, 'mon')
+
+ ctx.skip_firewalld = False
+ with pytest.raises(Exception):
+ cd.update_firewalld(ctx, 'mon')
+
+ ctx = cd.CephadmContext()
+ ctx.ssl_dashboard_port = 8888
+ ctx.dashboard_key = None
+ ctx.dashboard_password_noupdate = True
+ ctx.initial_dashboard_password = 'password'
+ ctx.initial_dashboard_user = 'User'
+ with pytest.raises(Exception):
+ cd.prepare_dashboard(ctx, 0, 0, lambda _, extra_mounts=None, ___=None : '5', lambda : None)
+
+ ctx.skip_firewalld = True
+ cd.prepare_dashboard(ctx, 0, 0, lambda _, extra_mounts=None, ___=None : '5', lambda : None)
+
+ ctx.skip_firewalld = False
+ with pytest.raises(Exception):
+ cd.prepare_dashboard(ctx, 0, 0, lambda _, extra_mounts=None, ___=None : '5', lambda : None)
+
+ @mock.patch('cephadm.call_throws')
+ @mock.patch('cephadm.get_parm')
+ def test_registry_login(self, get_parm, call_throws):
+ # test normal valid login with url, username and password specified
+ call_throws.return_value = '', '', 0
+ ctx: cd.CephadmContext = cd.cephadm_init_ctx(
+ ['registry-login', '--registry-url', 'sample-url',
+ '--registry-username', 'sample-user', '--registry-password',
+ 'sample-pass'])
+ ctx.container_engine = mock_docker()
+ retval = cd.command_registry_login(ctx)
+ assert retval == 0
+
+ # test bad login attempt with invalid arguments given
+ ctx: cd.CephadmContext = cd.cephadm_init_ctx(
+ ['registry-login', '--registry-url', 'bad-args-url'])
+ with pytest.raises(Exception) as e:
+ assert cd.command_registry_login(ctx)
+ assert str(e.value) == ('Invalid custom registry arguments received. To login to a custom registry include '
+ '--registry-url, --registry-username and --registry-password options or --registry-json option')
+
+ # test normal valid login with json file
+ get_parm.return_value = {"url": "sample-url", "username": "sample-username", "password": "sample-password"}
+ ctx: cd.CephadmContext = cd.cephadm_init_ctx(
+ ['registry-login', '--registry-json', 'sample-json'])
+ ctx.container_engine = mock_docker()
+ retval = cd.command_registry_login(ctx)
+ assert retval == 0
+
+ # test bad login attempt with bad json file
+ get_parm.return_value = {"bad-json": "bad-json"}
+ ctx: cd.CephadmContext = cd.cephadm_init_ctx(
+ ['registry-login', '--registry-json', 'sample-json'])
+ with pytest.raises(Exception) as e:
+ assert cd.command_registry_login(ctx)
+ assert str(e.value) == ("json provided for custom registry login did not include all necessary fields. "
+ "Please setup json file as\n"
+ "{\n"
+ " \"url\": \"REGISTRY_URL\",\n"
+ " \"username\": \"REGISTRY_USERNAME\",\n"
+ " \"password\": \"REGISTRY_PASSWORD\"\n"
+ "}\n")
+
+ # test login attempt with valid arguments where login command fails
+ call_throws.side_effect = Exception
+ ctx: cd.CephadmContext = cd.cephadm_init_ctx(
+ ['registry-login', '--registry-url', 'sample-url',
+ '--registry-username', 'sample-user', '--registry-password',
+ 'sample-pass'])
+ with pytest.raises(Exception) as e:
+ cd.command_registry_login(ctx)
+ assert str(e.value) == "Failed to login to custom registry @ sample-url as sample-user with given password"
+
+ def test_get_image_info_from_inspect(self):
+ # podman
+ out = """204a01f9b0b6710dd0c0af7f37ce7139c47ff0f0105d778d7104c69282dfbbf1,[docker.io/ceph/ceph@sha256:1cc9b824e1b076cdff52a9aa3f0cc8557d879fb2fbbba0cafed970aca59a3992]"""
+ r = cd.get_image_info_from_inspect(out, 'registry/ceph/ceph:latest')
+ print(r)
+ assert r == {
+ 'image_id': '204a01f9b0b6710dd0c0af7f37ce7139c47ff0f0105d778d7104c69282dfbbf1',
+ 'repo_digests': ['docker.io/ceph/ceph@sha256:1cc9b824e1b076cdff52a9aa3f0cc8557d879fb2fbbba0cafed970aca59a3992']
+ }
+
+ # docker
+ out = """sha256:16f4549cf7a8f112bbebf7946749e961fbbd1b0838627fe619aab16bc17ce552,[quay.ceph.io/ceph-ci/ceph@sha256:4e13da36c1bd6780b312a985410ae678984c37e6a9493a74c87e4a50b9bda41f]"""
+ r = cd.get_image_info_from_inspect(out, 'registry/ceph/ceph:latest')
+ assert r == {
+ 'image_id': '16f4549cf7a8f112bbebf7946749e961fbbd1b0838627fe619aab16bc17ce552',
+ 'repo_digests': ['quay.ceph.io/ceph-ci/ceph@sha256:4e13da36c1bd6780b312a985410ae678984c37e6a9493a74c87e4a50b9bda41f']
+ }
+
+ # multiple digests (podman)
+ out = """e935122ab143a64d92ed1fbb27d030cf6e2f0258207be1baf1b509c466aeeb42,[docker.io/prom/prometheus@sha256:e4ca62c0d62f3e886e684806dfe9d4e0cda60d54986898173c1083856cfda0f4 docker.io/prom/prometheus@sha256:efd99a6be65885c07c559679a0df4ec709604bcdd8cd83f0d00a1a683b28fb6a]"""
+ r = cd.get_image_info_from_inspect(out, 'registry/prom/prometheus:latest')
+ assert r == {
+ 'image_id': 'e935122ab143a64d92ed1fbb27d030cf6e2f0258207be1baf1b509c466aeeb42',
+ 'repo_digests': [
+ 'docker.io/prom/prometheus@sha256:e4ca62c0d62f3e886e684806dfe9d4e0cda60d54986898173c1083856cfda0f4',
+ 'docker.io/prom/prometheus@sha256:efd99a6be65885c07c559679a0df4ec709604bcdd8cd83f0d00a1a683b28fb6a',
+ ]
+ }
+
+
+ def test_dict_get(self):
+ result = cd.dict_get({'a': 1}, 'a', require=True)
+ assert result == 1
+ result = cd.dict_get({'a': 1}, 'b')
+ assert result is None
+ result = cd.dict_get({'a': 1}, 'b', default=2)
+ assert result == 2
+
+ def test_dict_get_error(self):
+ with pytest.raises(cd.Error):
+ cd.dict_get({'a': 1}, 'b', require=True)
+
+ def test_dict_get_join(self):
+ result = cd.dict_get_join({'foo': ['a', 'b']}, 'foo')
+ assert result == 'a\nb'
+ result = cd.dict_get_join({'foo': [1, 2]}, 'foo')
+ assert result == '1\n2'
+ result = cd.dict_get_join({'bar': 'a'}, 'bar')
+ assert result == 'a'
+ result = cd.dict_get_join({'a': 1}, 'a')
+ assert result == 1
+
+ def test_last_local_images(self):
+ out = '''
+docker.io/ceph/daemon-base@
+docker.io/ceph/ceph:v15.2.5
+docker.io/ceph/daemon-base:octopus
+ '''
+ image = cd._filter_last_local_ceph_image(out)
+ assert image == 'docker.io/ceph/ceph:v15.2.5'
+
+ def test_normalize_image_digest(self):
+ s = 'myhostname:5000/ceph/ceph@sha256:753886ad9049004395ae990fbb9b096923b5a518b819283141ee8716ddf55ad1'
+ assert cd.normalize_image_digest(s) == s
+
+ s = 'ceph/ceph:latest'
+ assert cd.normalize_image_digest(s) == f'{cd.DEFAULT_REGISTRY}/{s}'
+
+ @pytest.mark.parametrize('fsid, ceph_conf, list_daemons, result, err, ',
+ [
+ (
+ None,
+ None,
+ [],
+ None,
+ None,
+ ),
+ (
+ '00000000-0000-0000-0000-0000deadbeef',
+ None,
+ [],
+ '00000000-0000-0000-0000-0000deadbeef',
+ None,
+ ),
+ (
+ '00000000-0000-0000-0000-0000deadbeef',
+ None,
+ [
+ {'fsid': '10000000-0000-0000-0000-0000deadbeef'},
+ {'fsid': '20000000-0000-0000-0000-0000deadbeef'},
+ ],
+ '00000000-0000-0000-0000-0000deadbeef',
+ None,
+ ),
+ (
+ None,
+ None,
+ [
+ {'fsid': '00000000-0000-0000-0000-0000deadbeef'},
+ ],
+ '00000000-0000-0000-0000-0000deadbeef',
+ None,
+ ),
+ (
+ None,
+ None,
+ [
+ {'fsid': '10000000-0000-0000-0000-0000deadbeef'},
+ {'fsid': '20000000-0000-0000-0000-0000deadbeef'},
+ ],
+ None,
+ r'Cannot infer an fsid',
+ ),
+ (
+ None,
+ get_ceph_conf(fsid='00000000-0000-0000-0000-0000deadbeef'),
+ [],
+ '00000000-0000-0000-0000-0000deadbeef',
+ None,
+ ),
+ (
+ None,
+ get_ceph_conf(fsid='00000000-0000-0000-0000-0000deadbeef'),
+ [
+ {'fsid': '00000000-0000-0000-0000-0000deadbeef'},
+ ],
+ '00000000-0000-0000-0000-0000deadbeef',
+ None,
+ ),
+ (
+ None,
+ get_ceph_conf(fsid='00000000-0000-0000-0000-0000deadbeef'),
+ [
+ {'fsid': '10000000-0000-0000-0000-0000deadbeef'},
+ {'fsid': '20000000-0000-0000-0000-0000deadbeef'},
+ ],
+ None,
+ r'Cannot infer an fsid',
+ ),
+ ])
+ @mock.patch('cephadm.call')
+ def test_infer_fsid(self, _call, fsid, ceph_conf, list_daemons, result, err, cephadm_fs):
+ # build the context
+ ctx = cd.CephadmContext()
+ ctx.fsid = fsid
+
+ # mock the decorator
+ mock_fn = mock.Mock()
+ mock_fn.return_value = 0
+ infer_fsid = cd.infer_fsid(mock_fn)
+
+ # mock the ceph.conf file content
+ if ceph_conf:
+ f = cephadm_fs.create_file('ceph.conf', contents=ceph_conf)
+ ctx.config = f.path
+
+ # test
+ with mock.patch('cephadm.list_daemons', return_value=list_daemons):
+ if err:
+ with pytest.raises(cd.Error, match=err):
+ infer_fsid(ctx)
+ else:
+ infer_fsid(ctx)
+ assert ctx.fsid == result
+
+ @pytest.mark.parametrize('fsid, config, name, list_daemons, result, ',
+ [
+ (
+ None,
+ '/foo/bar.conf',
+ None,
+ [],
+ '/foo/bar.conf',
+ ),
+ (
+ '00000000-0000-0000-0000-0000deadbeef',
+ None,
+ None,
+ [],
+ cd.SHELL_DEFAULT_CONF,
+ ),
+ (
+ '00000000-0000-0000-0000-0000deadbeef',
+ None,
+ None,
+ [{'name': 'mon.a', 'fsid': '00000000-0000-0000-0000-0000deadbeef', 'style': 'cephadm:v1'}],
+ '/var/lib/ceph/00000000-0000-0000-0000-0000deadbeef/mon.a/config',
+ ),
+ (
+ '00000000-0000-0000-0000-0000deadbeef',
+ None,
+ None,
+ [{'name': 'mon.a', 'fsid': 'aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa', 'style': 'cephadm:v1'}],
+ cd.SHELL_DEFAULT_CONF,
+ ),
+ (
+ '00000000-0000-0000-0000-0000deadbeef',
+ None,
+ None,
+ [{'name': 'mon.a', 'fsid': '00000000-0000-0000-0000-0000deadbeef', 'style': 'legacy'}],
+ cd.SHELL_DEFAULT_CONF,
+ ),
+ (
+ '00000000-0000-0000-0000-0000deadbeef',
+ None,
+ None,
+ [{'name': 'osd.0'}],
+ cd.SHELL_DEFAULT_CONF,
+ ),
+ (
+ '00000000-0000-0000-0000-0000deadbeef',
+ '/foo/bar.conf',
+ 'mon.a',
+ [{'name': 'mon.a', 'style': 'cephadm:v1'}],
+ '/foo/bar.conf',
+ ),
+ (
+ '00000000-0000-0000-0000-0000deadbeef',
+ None,
+ 'mon.a',
+ [],
+ '/var/lib/ceph/00000000-0000-0000-0000-0000deadbeef/mon.a/config',
+ ),
+ (
+ '00000000-0000-0000-0000-0000deadbeef',
+ None,
+ 'osd.0',
+ [],
+ '/var/lib/ceph/00000000-0000-0000-0000-0000deadbeef/osd.0/config',
+ ),
+ (
+ None,
+ None,
+ None,
+ [],
+ cd.SHELL_DEFAULT_CONF,
+ ),
+ ])
+ @mock.patch('cephadm.call')
+ @mock.patch('cephadm.logger')
+ def test_infer_config(self, logger, _call, fsid, config, name, list_daemons, result, cephadm_fs):
+ # build the context
+ ctx = cd.CephadmContext()
+ ctx.fsid = fsid
+ ctx.config = config
+ ctx.name = name
+
+ # mock the decorator
+ mock_fn = mock.Mock()
+ mock_fn.return_value = 0
+ infer_config = cd.infer_config(mock_fn)
+
+ # mock the config file
+ cephadm_fs.create_file(result)
+
+ # test
+ with mock.patch('cephadm.list_daemons', return_value=list_daemons):
+ infer_config(ctx)
+ assert ctx.config == result
+
+ @mock.patch('cephadm.call')
+ def test_extract_uid_gid_fail(self, _call):
+ err = """Error: container_linux.go:370: starting container process caused: process_linux.go:459: container init caused: process_linux.go:422: setting cgroup config for procHooks process caused: Unit libpod-056038e1126191fba41d8a037275136f2d7aeec9710b9ee
+ff792c06d8544b983.scope not found.: OCI runtime error"""
+ _call.return_value = ('', err, 127)
+ ctx = cd.CephadmContext()
+ ctx.container_engine = mock_podman()
+ with pytest.raises(cd.Error, match='OCI'):
+ cd.extract_uid_gid(ctx)
+
+ @pytest.mark.parametrize('test_input, expected', [
+ ([cd.make_fsid(), cd.make_fsid(), cd.make_fsid()], 3),
+ ([cd.make_fsid(), 'invalid-fsid', cd.make_fsid(), '0b87e50c-8e77-11ec-b890-'], 2),
+ (['f6860ec2-8e76-11ec-', '0b87e50c-8e77-11ec-b890-', ''], 0),
+ ([], 0),
+ ])
+ def test_get_ceph_cluster_count(self, test_input, expected):
+ ctx = cd.CephadmContext()
+ with mock.patch('os.listdir', return_value=test_input):
+ assert cd.get_ceph_cluster_count(ctx) == expected
+
+ def test_set_image_minimize_config(self):
+ def throw_cmd(cmd):
+ raise cd.Error(' '.join(cmd))
+ ctx = cd.CephadmContext()
+ ctx.image = 'test_image'
+ ctx.no_minimize_config = True
+ fake_cli = lambda cmd, __=None, ___=None: throw_cmd(cmd)
+ with pytest.raises(cd.Error, match='config set global container_image test_image'):
+ cd.finish_bootstrap_config(
+ ctx=ctx,
+ fsid=cd.make_fsid(),
+ config='',
+ mon_id='a', mon_dir='mon_dir',
+ mon_network=None, ipv6=False,
+ cli=fake_cli,
+ cluster_network=None,
+ ipv6_cluster_network=False
+ )
+
+
+class TestCustomContainer(unittest.TestCase):
+ cc: cd.CustomContainer
+
+ def setUp(self):
+ self.cc = cd.CustomContainer(
+ 'e863154d-33c7-4350-bca5-921e0467e55b',
+ 'container',
+ config_json={
+ 'entrypoint': 'bash',
+ 'gid': 1000,
+ 'args': [
+ '--no-healthcheck',
+ '-p 6800:6800'
+ ],
+ 'envs': ['SECRET=password'],
+ 'ports': [8080, 8443],
+ 'volume_mounts': {
+ '/CONFIG_DIR': '/foo/conf',
+ 'bar/config': '/bar:ro'
+ },
+ 'bind_mounts': [
+ [
+ 'type=bind',
+ 'source=/CONFIG_DIR',
+ 'destination=/foo/conf',
+ ''
+ ],
+ [
+ 'type=bind',
+ 'source=bar/config',
+ 'destination=/bar:ro',
+ 'ro=true'
+ ]
+ ]
+ },
+ image='docker.io/library/hello-world:latest'
+ )
+
+ def test_entrypoint(self):
+ self.assertEqual(self.cc.entrypoint, 'bash')
+
+ def test_uid_gid(self):
+ self.assertEqual(self.cc.uid, 65534)
+ self.assertEqual(self.cc.gid, 1000)
+
+ def test_ports(self):
+ self.assertEqual(self.cc.ports, [8080, 8443])
+
+ def test_get_container_args(self):
+ result = self.cc.get_container_args()
+ self.assertEqual(result, [
+ '--no-healthcheck',
+ '-p 6800:6800'
+ ])
+
+ def test_get_container_envs(self):
+ result = self.cc.get_container_envs()
+ self.assertEqual(result, ['SECRET=password'])
+
+ def test_get_container_mounts(self):
+ result = self.cc.get_container_mounts('/xyz')
+ self.assertDictEqual(result, {
+ '/CONFIG_DIR': '/foo/conf',
+ '/xyz/bar/config': '/bar:ro'
+ })
+
+ def test_get_container_binds(self):
+ result = self.cc.get_container_binds('/xyz')
+ self.assertEqual(result, [
+ [
+ 'type=bind',
+ 'source=/CONFIG_DIR',
+ 'destination=/foo/conf',
+ ''
+ ],
+ [
+ 'type=bind',
+ 'source=/xyz/bar/config',
+ 'destination=/bar:ro',
+ 'ro=true'
+ ]
+ ])
+
+
+class TestCephadmExporter(object):
+ exporter: cd.CephadmDaemon
+ files_created: List[str] = []
+ crt = """-----BEGIN CERTIFICATE-----
+MIIC1zCCAb8CEFHoZE2MfUVzo53fzzBKAT0wDQYJKoZIhvcNAQENBQAwKjENMAsG
+A1UECgwEQ2VwaDEZMBcGA1UECwwQY2VwaGFkbS1leHBvcnRlcjAeFw0yMDExMjUy
+MzEwNTVaFw0zMDExMjMyMzEwNTVaMCoxDTALBgNVBAoMBENlcGgxGTAXBgNVBAsM
+EGNlcGhhZG0tZXhwb3J0ZXIwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIB
+AQCsTfcJcXbREqfx1zTUuEmK+lJn9WWjk0URRF1Z+QgPkascNdkX16PnvhbGwXmF
+BTdAcNl7V0U+z4EsGJ7hJsB7qTq6Rb6wNl7r0OxjeWOmB9xbF4Q/KR5yrbM1DA9A
+B5fNswrUXViku5Y2jlOAz+ZMBhYxMx0edqhxSn297j04Z6RF4Mvkc43v0FH7Ju7k
+O5+0VbdzcOdu37DFpoE4Ll2MZ/GuAHcJ8SD06sEdzFEjRCraav976743XcUlhZGX
+ZTTG/Zf/a+wuCjtMG3od7vRFfuRrM5oTE133DuQ5deR7ybcZNDyopDjHF8xB1bAk
+IOz4SbP6Q25K99Czm1K+3kMLAgMBAAEwDQYJKoZIhvcNAQENBQADggEBACmtvZb8
+dJGHx/WC0/JHxnEJCJM2qnn87ELzbbIQL1w1Yb/I6JQYPgq+WiQPaHaLL9eYsm0l
+dFwvrh+WC0JpXDfADnUnkTSB/WpZ2nC+2JxBptrQEuIcqNXpcJd0bKDiHunv04JI
+uEVpTAK05dBV38qNmIlu4HyB4OEnuQpyOr9xpIhdxuJ95O9K0j5BIw98ZaEwYNUP
+Rm3YlQwfS6R5xaBvL9kyfxyAD2joNj44q6w/5zj4egXVIA5VpkQm8DmMtu0Pd2NG
+dzfYRmqrDolh+rty8HiyIxzeDJQ5bj6LKbUkmABvX50nDySVyMfHmt461/n7W65R
+CHFLoOmfJJik+Uc=\n-----END CERTIFICATE-----
+"""
+ key = """-----BEGIN PRIVATE KEY-----
+MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQCsTfcJcXbREqfx
+1zTUuEmK+lJn9WWjk0URRF1Z+QgPkascNdkX16PnvhbGwXmFBTdAcNl7V0U+z4Es
+GJ7hJsB7qTq6Rb6wNl7r0OxjeWOmB9xbF4Q/KR5yrbM1DA9AB5fNswrUXViku5Y2
+jlOAz+ZMBhYxMx0edqhxSn297j04Z6RF4Mvkc43v0FH7Ju7kO5+0VbdzcOdu37DF
+poE4Ll2MZ/GuAHcJ8SD06sEdzFEjRCraav976743XcUlhZGXZTTG/Zf/a+wuCjtM
+G3od7vRFfuRrM5oTE133DuQ5deR7ybcZNDyopDjHF8xB1bAkIOz4SbP6Q25K99Cz
+m1K+3kMLAgMBAAECggEASnAwToMXWsGdjqxzpYasNv9oBIOO0nk4OHp5ffpJUjiT
+XM+ip1tA80g7HMjPD/mt4gge3NtaDgWlf4Bve0O7mnEE7x5cgFIs9eG/jkYOF9eD
+ilMBjivcfJywNDWujPH60iIMhqyBNEHaZl1ck+S9UJC8m6rCZLvMj40n/5riFfBy
+1sjf2uOwcfWrjSj9Ju4wlMI6khSSz2aYC7glQQ/fo2+YArbEUcy60iloPQ6wEgZK
+okoVWZA9AehwLcnRjkwd9EVmMMtRGPE/AcP4s/kKA0tRDRicPLN727Ke/yxv+Ppo
+hbIZIcOn7soOFAENcodJ4YRSCd++QfCNaVAi7vwWWQKBgQDeBY4vvr+H0brbSjQg
+O7Fpqub/fxZY3UoHWDqWs2X4o3qhDqaTQODpuYtCm8YQE//55JoLWKAD0evq5dLS
+YLrtC1Vyxf+TA7opCUjWBe+liyndbJdB5q0zF7qdWUtQKGVSWyUWhK8gHa6M64fP
+oi83DD7F0OGusTWGtfbceErk/wKBgQDGrJLRo/5xnAH5VmPfNu+S6h0M2qM6CYwe
+Y5wHFG2uQQct73adf53SkhvZVmOzJsWQbVnlDOKMhqazcs+7VWRgO5X3naWVcctE
+Hggw9MgpbXAWFOI5sNYsCYE58E+fTHjE6O4A3MhMCsze+CIC3sKuPQBBiL9bWSOX
+8POswqfl9QKBgDe/nVxPwTgRaaH2l/AgDQRDbY1qE+psZlJBzTRaB5jPM9ONIjaH
+a/JELLuk8a7H1tagmC2RK1zKMTriSnWY5FbxKZuQLAR2QyBavHdBNlOTBggbZD+f
+9I2Hv8wSx95wxkBPsphc6Lxft5ya55czWjewU3LIaGK9DHuu5TWm3udxAoGBAJGP
+PsJ59KIoOwoDUYjpJv3sqPwR9CVBeXeKY3aMcQ+KdUgiejVKmsb8ZYsG0GUhsv3u
+ID7BAfsTbG9tXuVR2wjmnymcRwUHKnXtyvKTZVN06vpCsryx4zjAff2FI9ECpjke
+r8HSAK41+4QhKEoSC3C9IMLi/dBfrsRTtTSOKZVBAoGBAI2dl5HEIFpufaI4toWM
+LO5HFrlXgRDGoc/+Byr5/8ZZpYpU115Ol/q6M+l0koV2ygJ9jeJJEllFWykIDS6F
+XxazFI74swAqobHb2ZS/SLhoVxE82DdSeXrjkTvUjNtrW5zs1gIMKBR4nD6H8AqL
+iMN28C2bKGao5UHvdER1rGy7
+-----END PRIVATE KEY-----
+"""
+ token = "MyAccessToken"
+
+ @classmethod
+ def setup_class(cls):
+ # create the ssl files
+ fname = os.path.join(os.getcwd(), 'crt')
+ with open(fname, 'w') as crt:
+ crt.write(cls.crt)
+ cls.files_created.append(fname)
+ fname = os.path.join(os.getcwd(), 'key')
+ with open(fname, 'w') as crt:
+ crt.write(cls.key)
+ cls.files_created.append(fname)
+ fname = os.path.join(os.getcwd(), 'token')
+ with open(fname, 'w') as crt:
+ crt.write(cls.token)
+ cls.files_created.append(fname)
+ # start a simple http instance to test the requesthandler
+ cls.server = HTTPServer(('0.0.0.0', 9443), cd.CephadmDaemonHandler)
+ cls.server.cephadm_cache = cd.CephadmCache()
+ cls.server.token = cls.token
+ t = threading.Thread(target=cls.server.serve_forever)
+ t.daemon = True
+ t.start()
+
+ @classmethod
+ def teardown_class(cls):
+ cls.server.shutdown()
+ assert len(cls.files_created) > 0
+ for f in cls.files_created:
+ os.remove(f)
+
+ def setup_method(self):
+ # re-init the cache for every test
+ TestCephadmExporter.server.cephadm_cache = cd.CephadmCache()
+
+ def teardown_method(self):
+ pass
+
+ def test_files_ready(self):
+ assert os.path.exists(os.path.join(os.getcwd(), 'crt'))
+ assert os.path.exists(os.path.join(os.getcwd(), 'key'))
+ assert os.path.exists(os.path.join(os.getcwd(), 'token'))
+
+ def test_can_run(self, exporter):
+ assert exporter.can_run
+
+ def test_token_valid(self, exporter):
+ assert exporter.token == self.token
+
+ def test_unit_name(self,exporter):
+ assert exporter.unit_name
+ assert exporter.unit_name == "ceph-foobar-cephadm-exporter.test.service"
+
+ def test_unit_run(self,exporter):
+ assert exporter.unit_run
+ lines = exporter.unit_run.split('\n')
+ assert len(lines) == 2
+ assert "cephadm exporter --fsid foobar --id test --port 9443 &" in lines[1]
+
+ def test_binary_path(self, exporter):
+ assert os.path.isfile(exporter.binary_path)
+
+ def test_systemd_unit(self, exporter):
+ assert exporter.unit_file
+
+ def test_validate_passes(self, exporter):
+ config = {
+ "crt": self.crt,
+ "key": self.key,
+ "token": self.token,
+ }
+ cd.CephadmDaemon.validate_config(config)
+
+ def test_validate_fails(self, exporter):
+ config = {
+ "key": self.key,
+ "token": self.token,
+ }
+ with pytest.raises(cd.Error):
+ cd.CephadmDaemon.validate_config(config)
+
+ def test_port_active(self, exporter):
+ assert exporter.port_active == True
+
+ def test_rqst_health_200(self):
+ hdrs={"Authorization":f"Bearer {TestCephadmExporter.token}"}
+ req=Request("http://localhost:9443/v1/metadata/health",headers=hdrs)
+ r = urlopen(req)
+ assert r.status == 200
+
+ def test_rqst_all_inactive_500(self):
+ hdrs={"Authorization":f"Bearer {TestCephadmExporter.token}"}
+ req=Request("http://localhost:9443/v1/metadata",headers=hdrs)
+ try:
+ r = urlopen(req)
+ except HTTPError as e:
+ assert e.code == 500
+
+ def test_rqst_no_auth_401(self):
+ req=Request("http://localhost:9443/v1/metadata")
+ try:
+ urlopen(req)
+ except HTTPError as e:
+ assert e.code == 401
+
+ def test_rqst_bad_auth_401(self):
+ hdrs={"Authorization":f"Bearer BogusAuthToken"}
+ req=Request("http://localhost:9443/v1/metadata",headers=hdrs)
+ try:
+ urlopen(req)
+ except HTTPError as e:
+ assert e.code == 401
+
+ def test_rqst_badURL_404(self):
+ hdrs={"Authorization":f"Bearer {TestCephadmExporter.token}"}
+ req=Request("http://localhost:9443/v1/metazoic",headers=hdrs)
+ try:
+ urlopen(req)
+ except HTTPError as e:
+ assert e.code == 404
+
+ def test_rqst_inactive_task_204(self):
+ # all tasks initialise as inactive, and then 'go' active as their thread starts
+ # so we can pick any task to check for an inactive response (no content)
+ hdrs={"Authorization":f"Bearer {TestCephadmExporter.token}"}
+ req=Request("http://localhost:9443/v1/metadata/disks",headers=hdrs)
+ r = urlopen(req)
+ assert r.status == 204
+
+ def test_rqst_active_task_200(self):
+ TestCephadmExporter.server.cephadm_cache.tasks['host'] = 'active'
+ hdrs={"Authorization":f"Bearer {TestCephadmExporter.token}"}
+ req=Request("http://localhost:9443/v1/metadata/host",headers=hdrs)
+ r = urlopen(req)
+ assert r.status == 200
+
+ def test_rqst_all_206(self):
+ TestCephadmExporter.server.cephadm_cache.tasks['disks'] = 'active'
+ hdrs={"Authorization":f"Bearer {TestCephadmExporter.token}"}
+ req=Request("http://localhost:9443/v1/metadata",headers=hdrs)
+ r = urlopen(req)
+ assert r.status == 206
+
+ def test_rqst_disks_200(self):
+ TestCephadmExporter.server.cephadm_cache.tasks['disks'] = 'active'
+ hdrs={"Authorization":f"Bearer {TestCephadmExporter.token}"}
+ req=Request("http://localhost:9443/v1/metadata/disks",headers=hdrs)
+ r = urlopen(req)
+ assert r.status == 200
+
+ def test_thread_exception(self, exporter):
+ # run is patched to invoke a mocked scrape_host thread that will raise so
+ # we check here that the exception handler updates the cache object as we'd
+ # expect with the error
+ exporter.run()
+ assert exporter.cephadm_cache.host['scrape_errors']
+ assert exporter.cephadm_cache.host['scrape_errors'] == ['ValueError exception: wah']
+ assert exporter.cephadm_cache.errors == ['host thread stopped']
+
+ # Test the requesthandler does the right thing with invalid methods...
+ # ie. return a "501" - Not Implemented / Unsupported Method
+ def test_invalid_method_HEAD(self):
+ hdrs={"Authorization":f"Bearer {TestCephadmExporter.token}"}
+ req=Request("http://localhost:9443/v1/metadata/health",headers=hdrs, method="HEAD")
+ with pytest.raises(HTTPError, match=r"HTTP Error 501: .*") as e:
+ urlopen(req)
+
+ def test_invalid_method_DELETE(self):
+ hdrs={"Authorization":f"Bearer {TestCephadmExporter.token}"}
+ req=Request("http://localhost:9443/v1/metadata/health",headers=hdrs, method="DELETE")
+ with pytest.raises(HTTPError, match=r"HTTP Error 501: .*") as e:
+ urlopen(req)
+
+ def test_invalid_method_POST(self):
+ hdrs={"Authorization":f"Bearer {TestCephadmExporter.token}"}
+ req=Request("http://localhost:9443/v1/metadata/health",headers=hdrs, method="POST")
+ with pytest.raises(HTTPError, match=r"HTTP Error 501: .*") as e:
+ urlopen(req)
+
+ def test_invalid_method_PUT(self):
+ hdrs={"Authorization":f"Bearer {TestCephadmExporter.token}"}
+ req=Request("http://localhost:9443/v1/metadata/health",headers=hdrs, method="PUT")
+ with pytest.raises(HTTPError, match=r"HTTP Error 501: .*") as e:
+ urlopen(req)
+
+ def test_invalid_method_CONNECT(self):
+ hdrs={"Authorization":f"Bearer {TestCephadmExporter.token}"}
+ req=Request("http://localhost:9443/v1/metadata/health",headers=hdrs, method="CONNECT")
+ with pytest.raises(HTTPError, match=r"HTTP Error 501: .*") as e:
+ urlopen(req)
+
+ def test_invalid_method_TRACE(self):
+ hdrs={"Authorization":f"Bearer {TestCephadmExporter.token}"}
+ req=Request("http://localhost:9443/v1/metadata/health",headers=hdrs, method="TRACE")
+ with pytest.raises(HTTPError, match=r"HTTP Error 501: .*") as e:
+ urlopen(req)
+
+ def test_invalid_method_OPTIONS(self):
+ hdrs={"Authorization":f"Bearer {TestCephadmExporter.token}"}
+ req=Request("http://localhost:9443/v1/metadata/health",headers=hdrs, method="OPTIONS")
+ with pytest.raises(HTTPError, match=r"HTTP Error 501: .*") as e:
+ urlopen(req)
+
+ def test_invalid_method_PATCH(self):
+ hdrs={"Authorization":f"Bearer {TestCephadmExporter.token}"}
+ req=Request("http://localhost:9443/v1/metadata/health",headers=hdrs, method="PATCH")
+ with pytest.raises(HTTPError, match=r"HTTP Error 501: .*") as e:
+ urlopen(req)
+
+ def test_ipv4_subnet(self):
+ rc, v, msg = cd.check_subnet('192.168.1.0/24')
+ assert rc == 0 and v[0] == 4
+
+ def test_ipv4_subnet_list(self):
+ rc, v, msg = cd.check_subnet('192.168.1.0/24,10.90.90.0/24')
+ assert rc == 0 and not msg
+
+ def test_ipv4_subnet_badlist(self):
+ rc, v, msg = cd.check_subnet('192.168.1.0/24,192.168.1.1')
+ assert rc == 1 and msg
+
+ def test_ipv4_subnet_mixed(self):
+ rc, v, msg = cd.check_subnet('192.168.100.0/24,fe80::/64')
+ assert rc == 0 and v == [4,6]
+
+ def test_ipv6_subnet(self):
+ rc, v, msg = cd.check_subnet('fe80::/64')
+ assert rc == 0 and v[0] == 6
+
+ def test_subnet_mask_missing(self):
+ rc, v, msg = cd.check_subnet('192.168.1.58')
+ assert rc == 1 and msg
+
+ def test_subnet_mask_junk(self):
+ rc, v, msg = cd.check_subnet('wah')
+ assert rc == 1 and msg
+
+
+class TestMaintenance:
+ systemd_target = "ceph.00000000-0000-0000-0000-000000c0ffee.target"
+ fsid = '0ea8cdd0-1bbf-11ec-a9c7-5254002763fa'
+
+ def test_systemd_target_OK(self, tmp_path):
+ base = tmp_path
+ wants = base / "ceph.target.wants"
+ wants.mkdir()
+ target = wants / TestMaintenance.systemd_target
+ target.touch()
+ ctx = cd.CephadmContext()
+ ctx.unit_dir = str(base)
+
+ assert cd.systemd_target_state(ctx, target.name)
+
+ def test_systemd_target_NOTOK(self, tmp_path):
+ base = tmp_path
+ ctx = cd.CephadmContext()
+ ctx.unit_dir = str(base)
+ assert not cd.systemd_target_state(ctx, TestMaintenance.systemd_target)
+
+ def test_parser_OK(self):
+ args = cd._parse_args(['host-maintenance', 'enter'])
+ assert args.maintenance_action == 'enter'
+
+ def test_parser_BAD(self):
+ with pytest.raises(SystemExit):
+ cd._parse_args(['host-maintenance', 'wah'])
+
+ @mock.patch('os.listdir', return_value=[])
+ @mock.patch('cephadm.call')
+ @mock.patch('cephadm.systemd_target_state')
+ def test_enter_failure_1(self, _target_state, _call, _listdir):
+ _call.return_value = '', '', 999
+ _target_state.return_value = True
+ ctx: cd.CephadmContext = cd.cephadm_init_ctx(
+ ['host-maintenance', 'enter', '--fsid', TestMaintenance.fsid])
+ ctx.container_engine = mock_podman()
+ retval = cd.command_maintenance(ctx)
+ assert retval.startswith('failed')
+
+ @mock.patch('os.listdir', return_value=[])
+ @mock.patch('cephadm.call')
+ @mock.patch('cephadm.systemd_target_state')
+ def test_enter_failure_2(self, _target_state, _call, _listdir):
+ _call.side_effect = [('', '', 0), ('', '', 999), ('', '', 0), ('', '', 999)]
+ _target_state.return_value = True
+ ctx: cd.CephadmContext = cd.cephadm_init_ctx(
+ ['host-maintenance', 'enter', '--fsid', TestMaintenance.fsid])
+ ctx.container_engine = mock_podman()
+ retval = cd.command_maintenance(ctx)
+ assert retval.startswith('failed')
+
+ @mock.patch('os.listdir', return_value=[])
+ @mock.patch('cephadm.call')
+ @mock.patch('cephadm.systemd_target_state')
+ @mock.patch('cephadm.target_exists')
+ def test_exit_failure_1(self, _target_exists, _target_state, _call, _listdir):
+ _call.return_value = '', '', 999
+ _target_state.return_value = False
+ _target_exists.return_value = True
+ ctx: cd.CephadmContext = cd.cephadm_init_ctx(
+ ['host-maintenance', 'exit', '--fsid', TestMaintenance.fsid])
+ ctx.container_engine = mock_podman()
+ retval = cd.command_maintenance(ctx)
+ assert retval.startswith('failed')
+
+ @mock.patch('os.listdir', return_value=[])
+ @mock.patch('cephadm.call')
+ @mock.patch('cephadm.systemd_target_state')
+ @mock.patch('cephadm.target_exists')
+ def test_exit_failure_2(self, _target_exists, _target_state, _call, _listdir):
+ _call.side_effect = [('', '', 0), ('', '', 999), ('', '', 0), ('', '', 999)]
+ _target_state.return_value = False
+ _target_exists.return_value = True
+ ctx: cd.CephadmContext = cd.cephadm_init_ctx(
+ ['host-maintenance', 'exit', '--fsid', TestMaintenance.fsid])
+ ctx.container_engine = mock_podman()
+ retval = cd.command_maintenance(ctx)
+ assert retval.startswith('failed')
+
+
+class TestMonitoring(object):
+ @mock.patch('cephadm.call')
+ def test_get_version_alertmanager(self, _call):
+ ctx = cd.CephadmContext()
+ ctx.container_engine = mock_podman()
+ daemon_type = 'alertmanager'
+
+ # binary `prometheus`
+ _call.return_value = '', '{}, version 0.16.1'.format(daemon_type), 0
+ version = cd.Monitoring.get_version(ctx, 'container_id', daemon_type)
+ assert version == '0.16.1'
+
+ # binary `prometheus-alertmanager`
+ _call.side_effect = (
+ ('', '', 1),
+ ('', '{}, version 0.16.1'.format(daemon_type), 0),
+ )
+ version = cd.Monitoring.get_version(ctx, 'container_id', daemon_type)
+ assert version == '0.16.1'
+
+ @mock.patch('cephadm.call')
+ def test_get_version_prometheus(self, _call):
+ ctx = cd.CephadmContext()
+ ctx.container_engine = mock_podman()
+ daemon_type = 'prometheus'
+ _call.return_value = '', '{}, version 0.16.1'.format(daemon_type), 0
+ version = cd.Monitoring.get_version(ctx, 'container_id', daemon_type)
+ assert version == '0.16.1'
+
+ def test_prometheus_external_url(self):
+ ctx = cd.CephadmContext()
+ ctx.config_json = json.dumps({'files': {}, 'retention_time': '15d'})
+ daemon_type = 'prometheus'
+ daemon_id = 'home'
+ fsid = 'aaf5a720-13fe-4a3b-82b9-2d99b7fd9704'
+ args = cd.get_daemon_args(ctx, fsid, daemon_type, daemon_id)
+ assert any([x.startswith('--web.external-url=http://') for x in args])
+
+ @mock.patch('cephadm.call')
+ def test_get_version_node_exporter(self, _call):
+ ctx = cd.CephadmContext()
+ ctx.container_engine = mock_podman()
+ daemon_type = 'node-exporter'
+ _call.return_value = '', '{}, version 0.16.1'.format(daemon_type.replace('-', '_')), 0
+ version = cd.Monitoring.get_version(ctx, 'container_id', daemon_type)
+ assert version == '0.16.1'
+
+ def test_create_daemon_dirs_prometheus(self, cephadm_fs):
+ """
+ Ensures the required and optional files given in the configuration are
+ created and mapped correctly inside the container. Tests absolute and
+ relative file paths given in the configuration.
+ """
+
+ fsid = 'aaf5a720-13fe-4a3b-82b9-2d99b7fd9704'
+ daemon_type = 'prometheus'
+ uid, gid = 50, 50
+ daemon_id = 'home'
+ ctx = cd.CephadmContext()
+ ctx.data_dir = '/somedir'
+ ctx.config_json = json.dumps({
+ 'files': {
+ 'prometheus.yml': 'foo',
+ '/etc/prometheus/alerting/ceph_alerts.yml': 'bar'
+ }
+ })
+
+ cd.create_daemon_dirs(ctx,
+ fsid,
+ daemon_type,
+ daemon_id,
+ uid,
+ gid,
+ config=None,
+ keyring=None)
+
+ prefix = '{data_dir}/{fsid}/{daemon_type}.{daemon_id}'.format(
+ data_dir=ctx.data_dir,
+ fsid=fsid,
+ daemon_type=daemon_type,
+ daemon_id=daemon_id
+ )
+
+ expected = {
+ 'etc/prometheus/prometheus.yml': 'foo',
+ 'etc/prometheus/alerting/ceph_alerts.yml': 'bar',
+ }
+
+ for file,content in expected.items():
+ file = os.path.join(prefix, file)
+ assert os.path.exists(file)
+ with open(file) as f:
+ assert f.read() == content
+
+ # assert uid/gid after redeploy
+ new_uid = uid+1
+ new_gid = gid+1
+ cd.create_daemon_dirs(ctx,
+ fsid,
+ daemon_type,
+ daemon_id,
+ new_uid,
+ new_gid,
+ config=None,
+ keyring=None)
+ for file,content in expected.items():
+ file = os.path.join(prefix, file)
+ assert os.stat(file).st_uid == new_uid
+ assert os.stat(file).st_gid == new_gid
+
+
+class TestBootstrap(object):
+
+ @staticmethod
+ def _get_cmd(*args):
+ return [
+ 'bootstrap',
+ '--allow-mismatched-release',
+ '--skip-prepare-host',
+ '--skip-dashboard',
+ *args,
+ ]
+
+
+###############################################3
+
+ def test_config(self, cephadm_fs):
+ conf_file = 'foo'
+ cmd = self._get_cmd(
+ '--mon-ip', '192.168.1.1',
+ '--skip-mon-network',
+ '--config', conf_file,
+ )
+
+ with with_cephadm_ctx(cmd) as ctx:
+ msg = r'No such file or directory'
+ with pytest.raises(cd.Error, match=msg):
+ cd.command_bootstrap(ctx)
+
+ cephadm_fs.create_file(conf_file)
+ with with_cephadm_ctx(cmd) as ctx:
+ retval = cd.command_bootstrap(ctx)
+ assert retval == 0
+
+ def test_no_mon_addr(self, cephadm_fs):
+ cmd = self._get_cmd()
+ with with_cephadm_ctx(cmd) as ctx:
+ msg = r'must specify --mon-ip or --mon-addrv'
+ with pytest.raises(cd.Error, match=msg):
+ cd.command_bootstrap(ctx)
+
+ def test_skip_mon_network(self, cephadm_fs):
+ cmd = self._get_cmd('--mon-ip', '192.168.1.1')
+
+ with with_cephadm_ctx(cmd, list_networks={}) as ctx:
+ msg = r'--skip-mon-network'
+ with pytest.raises(cd.Error, match=msg):
+ cd.command_bootstrap(ctx)
+
+ cmd += ['--skip-mon-network']
+ with with_cephadm_ctx(cmd, list_networks={}) as ctx:
+ retval = cd.command_bootstrap(ctx)
+ assert retval == 0
+
+ @pytest.mark.parametrize('mon_ip, list_networks, result',
+ [
+ # IPv4
+ (
+ 'eth0',
+ {'192.168.1.0/24': {'eth0': ['192.168.1.1']}},
+ False,
+ ),
+ (
+ '0.0.0.0',
+ {'192.168.1.0/24': {'eth0': ['192.168.1.1']}},
+ False,
+ ),
+ (
+ '192.168.1.0',
+ {'192.168.1.0/24': {'eth0': ['192.168.1.1']}},
+ False,
+ ),
+ (
+ '192.168.1.1',
+ {'192.168.1.0/24': {'eth0': ['192.168.1.1']}},
+ True,
+ ),
+ (
+ '192.168.1.1:1234',
+ {'192.168.1.0/24': {'eth0': ['192.168.1.1']}},
+ True,
+ ),
+ (
+ '192.168.1.1:0123',
+ {'192.168.1.0/24': {'eth0': ['192.168.1.1']}},
+ True,
+ ),
+ # IPv6
+ (
+ '::',
+ {'192.168.1.0/24': {'eth0': ['192.168.1.1']}},
+ False,
+ ),
+ (
+ '::ffff:192.168.1.0',
+ {"ffff::/64": {"eth0": ["::ffff:c0a8:101"]}},
+ False,
+ ),
+ (
+ '::ffff:192.168.1.1',
+ {"ffff::/64": {"eth0": ["::ffff:c0a8:101"]}},
+ True,
+ ),
+ (
+ '::ffff:c0a8:101',
+ {"ffff::/64": {"eth0": ["::ffff:c0a8:101"]}},
+ True,
+ ),
+ (
+ '[::ffff:c0a8:101]:1234',
+ {"ffff::/64": {"eth0": ["::ffff:c0a8:101"]}},
+ True,
+ ),
+ (
+ '[::ffff:c0a8:101]:0123',
+ {"ffff::/64": {"eth0": ["::ffff:c0a8:101"]}},
+ True,
+ ),
+ (
+ '0000:0000:0000:0000:0000:FFFF:C0A8:0101',
+ {"ffff::/64": {"eth0": ["::ffff:c0a8:101"]}},
+ True,
+ ),
+ ])
+ def test_mon_ip(self, mon_ip, list_networks, result, cephadm_fs):
+ cmd = self._get_cmd('--mon-ip', mon_ip)
+ if not result:
+ with with_cephadm_ctx(cmd, list_networks=list_networks) as ctx:
+ msg = r'--skip-mon-network'
+ with pytest.raises(cd.Error, match=msg):
+ cd.command_bootstrap(ctx)
+ else:
+ with with_cephadm_ctx(cmd, list_networks=list_networks) as ctx:
+ retval = cd.command_bootstrap(ctx)
+ assert retval == 0
+
+ @pytest.mark.parametrize('mon_addrv, list_networks, err',
+ [
+ # IPv4
+ (
+ '192.168.1.1',
+ {'192.168.1.0/24': {'eth0': ['192.168.1.1']}},
+ r'must use square backets',
+ ),
+ (
+ '[192.168.1.1]',
+ {'192.168.1.0/24': {'eth0': ['192.168.1.1']}},
+ r'must include port number',
+ ),
+ (
+ '[192.168.1.1:1234]',
+ {'192.168.1.0/24': {'eth0': ['192.168.1.1']}},
+ None,
+ ),
+ (
+ '[192.168.1.1:0123]',
+ {'192.168.1.0/24': {'eth0': ['192.168.1.1']}},
+ None,
+ ),
+ (
+ '[v2:192.168.1.1:3300,v1:192.168.1.1:6789]',
+ {'192.168.1.0/24': {'eth0': ['192.168.1.1']}},
+ None,
+ ),
+ # IPv6
+ (
+ '[::ffff:192.168.1.1:1234]',
+ {'ffff::/64': {'eth0': ['::ffff:c0a8:101']}},
+ None,
+ ),
+ (
+ '[::ffff:192.168.1.1:0123]',
+ {'ffff::/64': {'eth0': ['::ffff:c0a8:101']}},
+ None,
+ ),
+ (
+ '[0000:0000:0000:0000:0000:FFFF:C0A8:0101:1234]',
+ {'ffff::/64': {'eth0': ['::ffff:c0a8:101']}},
+ None,
+ ),
+ (
+ '[v2:0000:0000:0000:0000:0000:FFFF:C0A8:0101:3300,v1:0000:0000:0000:0000:0000:FFFF:C0A8:0101:6789]',
+ {'ffff::/64': {'eth0': ['::ffff:c0a8:101']}},
+ None,
+ ),
+ ])
+ def test_mon_addrv(self, mon_addrv, list_networks, err, cephadm_fs):
+ cmd = self._get_cmd('--mon-addrv', mon_addrv)
+ if err:
+ with with_cephadm_ctx(cmd, list_networks=list_networks) as ctx:
+ with pytest.raises(cd.Error, match=err):
+ cd.command_bootstrap(ctx)
+ else:
+ with with_cephadm_ctx(cmd, list_networks=list_networks) as ctx:
+ retval = cd.command_bootstrap(ctx)
+ assert retval == 0
+
+ def test_allow_fqdn_hostname(self, cephadm_fs):
+ hostname = 'foo.bar'
+ cmd = self._get_cmd(
+ '--mon-ip', '192.168.1.1',
+ '--skip-mon-network',
+ )
+
+ with with_cephadm_ctx(cmd, hostname=hostname) as ctx:
+ msg = r'--allow-fqdn-hostname'
+ with pytest.raises(cd.Error, match=msg):
+ cd.command_bootstrap(ctx)
+
+ cmd += ['--allow-fqdn-hostname']
+ with with_cephadm_ctx(cmd, hostname=hostname) as ctx:
+ retval = cd.command_bootstrap(ctx)
+ assert retval == 0
+
+ @pytest.mark.parametrize('fsid, err',
+ [
+ ('', None),
+ ('00000000-0000-0000-0000-0000deadbeef', None),
+ ('00000000-0000-0000-0000-0000deadbeez', 'not an fsid'),
+ ])
+ def test_fsid(self, fsid, err, cephadm_fs):
+ cmd = self._get_cmd(
+ '--mon-ip', '192.168.1.1',
+ '--skip-mon-network',
+ '--fsid', fsid,
+ )
+
+ with with_cephadm_ctx(cmd) as ctx:
+ if err:
+ with pytest.raises(cd.Error, match=err):
+ cd.command_bootstrap(ctx)
+ else:
+ retval = cd.command_bootstrap(ctx)
+ assert retval == 0
+
+
+class TestShell(object):
+
+ def test_fsid(self, cephadm_fs):
+ fsid = '00000000-0000-0000-0000-0000deadbeef'
+
+ cmd = ['shell', '--fsid', fsid]
+ with with_cephadm_ctx(cmd) as ctx:
+ retval = cd.command_shell(ctx)
+ assert retval == 0
+ assert ctx.fsid == fsid
+
+ cmd = ['shell', '--fsid', '00000000-0000-0000-0000-0000deadbeez']
+ with with_cephadm_ctx(cmd) as ctx:
+ err = 'not an fsid'
+ with pytest.raises(cd.Error, match=err):
+ retval = cd.command_shell(ctx)
+ assert retval == 1
+ assert ctx.fsid == None
+
+ s = get_ceph_conf(fsid=fsid)
+ f = cephadm_fs.create_file('ceph.conf', contents=s)
+
+ cmd = ['shell', '--fsid', fsid, '--config', f.path]
+ with with_cephadm_ctx(cmd) as ctx:
+ retval = cd.command_shell(ctx)
+ assert retval == 0
+ assert ctx.fsid == fsid
+
+ cmd = ['shell', '--fsid', '10000000-0000-0000-0000-0000deadbeef', '--config', f.path]
+ with with_cephadm_ctx(cmd) as ctx:
+ err = 'fsid does not match ceph.conf'
+ with pytest.raises(cd.Error, match=err):
+ retval = cd.command_shell(ctx)
+ assert retval == 1
+ assert ctx.fsid == None
+
+ def test_name(self, cephadm_fs):
+ cmd = ['shell', '--name', 'foo']
+ with with_cephadm_ctx(cmd) as ctx:
+ retval = cd.command_shell(ctx)
+ assert retval == 0
+
+ cmd = ['shell', '--name', 'foo.bar']
+ with with_cephadm_ctx(cmd) as ctx:
+ err = r'must pass --fsid'
+ with pytest.raises(cd.Error, match=err):
+ retval = cd.command_shell(ctx)
+ assert retval == 1
+
+ fsid = '00000000-0000-0000-0000-0000deadbeef'
+ cmd = ['shell', '--name', 'foo.bar', '--fsid', fsid]
+ with with_cephadm_ctx(cmd) as ctx:
+ retval = cd.command_shell(ctx)
+ assert retval == 0
+
+ def test_config(self, cephadm_fs):
+ cmd = ['shell']
+ with with_cephadm_ctx(cmd) as ctx:
+ retval = cd.command_shell(ctx)
+ assert retval == 0
+ assert ctx.config == None
+
+ cephadm_fs.create_file(cd.SHELL_DEFAULT_CONF)
+ with with_cephadm_ctx(cmd) as ctx:
+ retval = cd.command_shell(ctx)
+ assert retval == 0
+ assert ctx.config == cd.SHELL_DEFAULT_CONF
+
+ cmd = ['shell', '--config', 'foo']
+ with with_cephadm_ctx(cmd) as ctx:
+ retval = cd.command_shell(ctx)
+ assert retval == 0
+ assert ctx.config == 'foo'
+
+ def test_keyring(self, cephadm_fs):
+ cmd = ['shell']
+ with with_cephadm_ctx(cmd) as ctx:
+ retval = cd.command_shell(ctx)
+ assert retval == 0
+ assert ctx.keyring == None
+
+ cephadm_fs.create_file(cd.SHELL_DEFAULT_KEYRING)
+ with with_cephadm_ctx(cmd) as ctx:
+ retval = cd.command_shell(ctx)
+ assert retval == 0
+ assert ctx.keyring == cd.SHELL_DEFAULT_KEYRING
+
+ cmd = ['shell', '--keyring', 'foo']
+ with with_cephadm_ctx(cmd) as ctx:
+ retval = cd.command_shell(ctx)
+ assert retval == 0
+ assert ctx.keyring == 'foo'
+
+ @mock.patch('cephadm.CephContainer')
+ def test_mount_no_dst(self, m_ceph_container, cephadm_fs):
+ cmd = ['shell', '--mount', '/etc/foo']
+ with with_cephadm_ctx(cmd) as ctx:
+ retval = cd.command_shell(ctx)
+ assert retval == 0
+ assert m_ceph_container.call_args.kwargs['volume_mounts']['/etc/foo'] == '/mnt/foo'
+
+ @mock.patch('cephadm.CephContainer')
+ def test_mount_with_dst_no_opt(self, m_ceph_container, cephadm_fs):
+ cmd = ['shell', '--mount', '/etc/foo:/opt/foo/bar']
+ with with_cephadm_ctx(cmd) as ctx:
+ retval = cd.command_shell(ctx)
+ assert retval == 0
+ assert m_ceph_container.call_args.kwargs['volume_mounts']['/etc/foo'] == '/opt/foo/bar'
+
+ @mock.patch('cephadm.CephContainer')
+ def test_mount_with_dst_and_opt(self, m_ceph_container, cephadm_fs):
+ cmd = ['shell', '--mount', '/etc/foo:/opt/foo/bar:Z']
+ with with_cephadm_ctx(cmd) as ctx:
+ retval = cd.command_shell(ctx)
+ assert retval == 0
+ assert m_ceph_container.call_args.kwargs['volume_mounts']['/etc/foo'] == '/opt/foo/bar:Z'
+
+class TestCephVolume(object):
+
+ @staticmethod
+ def _get_cmd(*args):
+ return [
+ 'ceph-volume',
+ *args,
+ '--', 'inventory', '--format', 'json'
+ ]
+
+ def test_noop(self, cephadm_fs):
+ cmd = self._get_cmd()
+ with with_cephadm_ctx(cmd) as ctx:
+ cd.command_ceph_volume(ctx)
+ assert ctx.fsid == None
+ assert ctx.config == None
+ assert ctx.keyring == None
+ assert ctx.config_json == None
+
+ def test_fsid(self, cephadm_fs):
+ fsid = '00000000-0000-0000-0000-0000deadbeef'
+
+ cmd = self._get_cmd('--fsid', fsid)
+ with with_cephadm_ctx(cmd) as ctx:
+ cd.command_ceph_volume(ctx)
+ assert ctx.fsid == fsid
+
+ cmd = self._get_cmd('--fsid', '00000000-0000-0000-0000-0000deadbeez')
+ with with_cephadm_ctx(cmd) as ctx:
+ err = 'not an fsid'
+ with pytest.raises(cd.Error, match=err):
+ retval = cd.command_shell(ctx)
+ assert retval == 1
+ assert ctx.fsid == None
+
+ s = get_ceph_conf(fsid=fsid)
+ f = cephadm_fs.create_file('ceph.conf', contents=s)
+
+ cmd = self._get_cmd('--fsid', fsid, '--config', f.path)
+ with with_cephadm_ctx(cmd) as ctx:
+ cd.command_ceph_volume(ctx)
+ assert ctx.fsid == fsid
+
+ cmd = self._get_cmd('--fsid', '10000000-0000-0000-0000-0000deadbeef', '--config', f.path)
+ with with_cephadm_ctx(cmd) as ctx:
+ err = 'fsid does not match ceph.conf'
+ with pytest.raises(cd.Error, match=err):
+ cd.command_ceph_volume(ctx)
+ assert ctx.fsid == None
+
+ def test_config(self, cephadm_fs):
+ cmd = self._get_cmd('--config', 'foo')
+ with with_cephadm_ctx(cmd) as ctx:
+ err = r'No such file or directory'
+ with pytest.raises(cd.Error, match=err):
+ cd.command_ceph_volume(ctx)
+
+ cephadm_fs.create_file('bar')
+ cmd = self._get_cmd('--config', 'bar')
+ with with_cephadm_ctx(cmd) as ctx:
+ cd.command_ceph_volume(ctx)
+ assert ctx.config == 'bar'
+
+ def test_keyring(self, cephadm_fs):
+ cmd = self._get_cmd('--keyring', 'foo')
+ with with_cephadm_ctx(cmd) as ctx:
+ err = r'No such file or directory'
+ with pytest.raises(cd.Error, match=err):
+ cd.command_ceph_volume(ctx)
+
+ cephadm_fs.create_file('bar')
+ cmd = self._get_cmd('--keyring', 'bar')
+ with with_cephadm_ctx(cmd) as ctx:
+ cd.command_ceph_volume(ctx)
+ assert ctx.keyring == 'bar'
+
+
+class TestIscsi:
+ def test_unit_run(self, cephadm_fs):
+ fsid = '9b9d7609-f4d5-4aba-94c8-effa764d96c9'
+ config_json = {
+ 'files': {'iscsi-gateway.cfg': ''}
+ }
+ with with_cephadm_ctx(['--image=ceph/ceph'], list_networks={}) as ctx:
+ import json
+ ctx.config_json = json.dumps(config_json)
+ ctx.fsid = fsid
+ cd.get_parm.return_value = config_json
+ c = cd.get_container(ctx, fsid, 'iscsi', 'daemon_id')
+
+ cd.make_data_dir(ctx, fsid, 'iscsi', 'daemon_id')
+ cd.deploy_daemon_units(
+ ctx,
+ fsid,
+ 0, 0,
+ 'iscsi',
+ 'daemon_id',
+ c,
+ True, True
+ )
+
+ with open('/var/lib/ceph/9b9d7609-f4d5-4aba-94c8-effa764d96c9/iscsi.daemon_id/unit.run') as f:
+ assert f.read() == """set -e
+if ! grep -qs /var/lib/ceph/9b9d7609-f4d5-4aba-94c8-effa764d96c9/iscsi.daemon_id/configfs /proc/mounts; then mount -t configfs none /var/lib/ceph/9b9d7609-f4d5-4aba-94c8-effa764d96c9/iscsi.daemon_id/configfs; fi
+# iscsi tcmu-runner container
+! /usr/bin/podman rm -f ceph-9b9d7609-f4d5-4aba-94c8-effa764d96c9-iscsi.daemon_id-tcmu 2> /dev/null
+! /usr/bin/podman rm -f ceph-9b9d7609-f4d5-4aba-94c8-effa764d96c9-iscsi-daemon_id-tcmu 2> /dev/null
+/usr/bin/podman run --rm --ipc=host --stop-signal=SIGTERM --net=host --entrypoint /usr/bin/tcmu-runner --privileged --group-add=disk --init --name ceph-9b9d7609-f4d5-4aba-94c8-effa764d96c9-iscsi-daemon_id-tcmu -e CONTAINER_IMAGE=ceph/ceph -e NODE_NAME=host1 -e CEPH_USE_RANDOM_NONCE=1 -v /var/lib/ceph/9b9d7609-f4d5-4aba-94c8-effa764d96c9/iscsi.daemon_id/config:/etc/ceph/ceph.conf:z -v /var/lib/ceph/9b9d7609-f4d5-4aba-94c8-effa764d96c9/iscsi.daemon_id/keyring:/etc/ceph/keyring:z -v /var/lib/ceph/9b9d7609-f4d5-4aba-94c8-effa764d96c9/iscsi.daemon_id/iscsi-gateway.cfg:/etc/ceph/iscsi-gateway.cfg:z -v /var/lib/ceph/9b9d7609-f4d5-4aba-94c8-effa764d96c9/iscsi.daemon_id/configfs:/sys/kernel/config -v /var/log/ceph/9b9d7609-f4d5-4aba-94c8-effa764d96c9:/var/log:z -v /dev:/dev --mount type=bind,source=/lib/modules,destination=/lib/modules,ro=true ceph/ceph &
+# iscsi.daemon_id
+! /usr/bin/podman rm -f ceph-9b9d7609-f4d5-4aba-94c8-effa764d96c9-iscsi.daemon_id 2> /dev/null
+! /usr/bin/podman rm -f ceph-9b9d7609-f4d5-4aba-94c8-effa764d96c9-iscsi-daemon_id 2> /dev/null
+/usr/bin/podman run --rm --ipc=host --stop-signal=SIGTERM --net=host --entrypoint /usr/bin/rbd-target-api --privileged --group-add=disk --init --name ceph-9b9d7609-f4d5-4aba-94c8-effa764d96c9-iscsi-daemon_id -e CONTAINER_IMAGE=ceph/ceph -e NODE_NAME=host1 -e CEPH_USE_RANDOM_NONCE=1 -v /var/lib/ceph/9b9d7609-f4d5-4aba-94c8-effa764d96c9/iscsi.daemon_id/config:/etc/ceph/ceph.conf:z -v /var/lib/ceph/9b9d7609-f4d5-4aba-94c8-effa764d96c9/iscsi.daemon_id/keyring:/etc/ceph/keyring:z -v /var/lib/ceph/9b9d7609-f4d5-4aba-94c8-effa764d96c9/iscsi.daemon_id/iscsi-gateway.cfg:/etc/ceph/iscsi-gateway.cfg:z -v /var/lib/ceph/9b9d7609-f4d5-4aba-94c8-effa764d96c9/iscsi.daemon_id/configfs:/sys/kernel/config -v /var/log/ceph/9b9d7609-f4d5-4aba-94c8-effa764d96c9:/var/log:z -v /dev:/dev --mount type=bind,source=/lib/modules,destination=/lib/modules,ro=true ceph/ceph
+"""
+
+ def test_get_container(self):
+ """
+ Due to a combination of socket.getfqdn() and podman's behavior to
+ add the container name into the /etc/hosts file, we cannot use periods
+ in container names. But we need to be able to detect old existing containers.
+ Assert this behaviour. I think we can remove this in Ceph R
+ """
+ fsid = '9b9d7609-f4d5-4aba-94c8-effa764d96c9'
+ with with_cephadm_ctx(['--image=ceph/ceph'], list_networks={}) as ctx:
+ ctx.fsid = fsid
+ c = cd.get_container(ctx, fsid, 'iscsi', 'something')
+ assert c.cname == 'ceph-9b9d7609-f4d5-4aba-94c8-effa764d96c9-iscsi-something'
+ assert c.old_cname == 'ceph-9b9d7609-f4d5-4aba-94c8-effa764d96c9-iscsi.something'
+
+
+class TestCheckHost:
+
+ @mock.patch('cephadm.find_executable', return_value='foo')
+ @mock.patch('cephadm.check_time_sync', return_value=True)
+ def test_container_engine(self, find_executable, check_time_sync):
+ ctx = cd.CephadmContext()
+
+ ctx.container_engine = None
+ err = r'No container engine binary found'
+ with pytest.raises(cd.Error, match=err):
+ cd.command_check_host(ctx)
+
+ ctx.container_engine = mock_podman()
+ cd.command_check_host(ctx)
+
+ ctx.container_engine = mock_docker()
+ cd.command_check_host(ctx)
+
+
+class TestRmRepo:
+
+ @pytest.mark.parametrize('os_release',
+ [
+ # Apt
+ dedent("""
+ NAME="Ubuntu"
+ VERSION="20.04 LTS (Focal Fossa)"
+ ID=ubuntu
+ ID_LIKE=debian
+ PRETTY_NAME="Ubuntu 20.04 LTS"
+ VERSION_ID="20.04"
+ HOME_URL="https://www.ubuntu.com/"
+ SUPPORT_URL="https://help.ubuntu.com/"
+ BUG_REPORT_URL="https://bugs.launchpad.net/ubuntu/"
+ PRIVACY_POLICY_URL="https://www.ubuntu.com/legal/terms-and-policies/privacy-policy"
+ VERSION_CODENAME=focal
+ UBUNTU_CODENAME=focal
+ """),
+
+ # YumDnf
+ dedent("""
+ NAME="CentOS Linux"
+ VERSION="8 (Core)"
+ ID="centos"
+ ID_LIKE="rhel fedora"
+ VERSION_ID="8"
+ PLATFORM_ID="platform:el8"
+ PRETTY_NAME="CentOS Linux 8 (Core)"
+ ANSI_COLOR="0;31"
+ CPE_NAME="cpe:/o:centos:centos:8"
+ HOME_URL="https://www.centos.org/"
+ BUG_REPORT_URL="https://bugs.centos.org/"
+
+ CENTOS_MANTISBT_PROJECT="CentOS-8"
+ CENTOS_MANTISBT_PROJECT_VERSION="8"
+ REDHAT_SUPPORT_PRODUCT="centos"
+ REDHAT_SUPPORT_PRODUCT_VERSION="8"
+ """),
+
+ # Zypper
+ dedent("""
+ NAME="openSUSE Tumbleweed"
+ # VERSION="20210810"
+ ID="opensuse-tumbleweed"
+ ID_LIKE="opensuse suse"
+ VERSION_ID="20210810"
+ PRETTY_NAME="openSUSE Tumbleweed"
+ ANSI_COLOR="0;32"
+ CPE_NAME="cpe:/o:opensuse:tumbleweed:20210810"
+ BUG_REPORT_URL="https://bugs.opensuse.org"
+ HOME_URL="https://www.opensuse.org/"
+ DOCUMENTATION_URL="https://en.opensuse.org/Portal:Tumbleweed"
+ LOGO="distributor-logo"
+ """),
+ ])
+ @mock.patch('cephadm.find_executable', return_value='foo')
+ def test_container_engine(self, find_executable, os_release, cephadm_fs):
+ cephadm_fs.create_file('/etc/os-release', contents=os_release)
+ ctx = cd.CephadmContext()
+
+ ctx.container_engine = None
+ cd.command_rm_repo(ctx)
+
+ ctx.container_engine = mock_podman()
+ cd.command_rm_repo(ctx)
+
+ ctx.container_engine = mock_docker()
+ cd.command_rm_repo(ctx)
+
+
+class TestValidateRepo:
+
+ @pytest.mark.parametrize('values',
+ [
+ # Apt - no checks
+ dict(
+ version="",
+ release="pacific",
+ err_text="",
+ os_release=dedent("""
+ NAME="Ubuntu"
+ VERSION="20.04 LTS (Focal Fossa)"
+ ID=ubuntu
+ ID_LIKE=debian
+ PRETTY_NAME="Ubuntu 20.04 LTS"
+ VERSION_ID="20.04"
+ HOME_URL="https://www.ubuntu.com/"
+ SUPPORT_URL="https://help.ubuntu.com/"
+ BUG_REPORT_URL="https://bugs.launchpad.net/ubuntu/"
+ PRIVACY_POLICY_URL="https://www.ubuntu.com/legal/terms-and-policies/privacy-policy"
+ VERSION_CODENAME=focal
+ UBUNTU_CODENAME=focal
+ """)),
+
+ # YumDnf on Centos8 - OK
+ dict(
+ version="",
+ release="pacific",
+ err_text="",
+ os_release=dedent("""
+ NAME="CentOS Linux"
+ VERSION="8 (Core)"
+ ID="centos"
+ ID_LIKE="rhel fedora"
+ VERSION_ID="8"
+ PLATFORM_ID="platform:el8"
+ PRETTY_NAME="CentOS Linux 8 (Core)"
+ ANSI_COLOR="0;31"
+ CPE_NAME="cpe:/o:centos:centos:8"
+ HOME_URL="https://www.centos.org/"
+ BUG_REPORT_URL="https://bugs.centos.org/"
+
+ CENTOS_MANTISBT_PROJECT="CentOS-8"
+ CENTOS_MANTISBT_PROJECT_VERSION="8"
+ REDHAT_SUPPORT_PRODUCT="centos"
+ REDHAT_SUPPORT_PRODUCT_VERSION="8"
+ """)),
+
+ # YumDnf on Fedora - Fedora not supported
+ dict(
+ version="",
+ release="pacific",
+ err_text="does not build Fedora",
+ os_release=dedent("""
+ NAME="Fedora Linux"
+ VERSION="35 (Cloud Edition)"
+ ID=fedora
+ VERSION_ID=35
+ VERSION_CODENAME=""
+ PLATFORM_ID="platform:f35"
+ PRETTY_NAME="Fedora Linux 35 (Cloud Edition)"
+ ANSI_COLOR="0;38;2;60;110;180"
+ LOGO=fedora-logo-icon
+ CPE_NAME="cpe:/o:fedoraproject:fedora:35"
+ HOME_URL="https://fedoraproject.org/"
+ DOCUMENTATION_URL="https://docs.fedoraproject.org/en-US/fedora/f35/system-administrators-guide/"
+ SUPPORT_URL="https://ask.fedoraproject.org/"
+ BUG_REPORT_URL="https://bugzilla.redhat.com/"
+ REDHAT_BUGZILLA_PRODUCT="Fedora"
+ REDHAT_BUGZILLA_PRODUCT_VERSION=35
+ REDHAT_SUPPORT_PRODUCT="Fedora"
+ REDHAT_SUPPORT_PRODUCT_VERSION=35
+ PRIVACY_POLICY_URL="https://fedoraproject.org/wiki/Legal:PrivacyPolicy"
+ VARIANT="Cloud Edition"
+ VARIANT_ID=cloud
+ """)),
+
+ # YumDnf on Centos 7 - no pacific
+ dict(
+ version="",
+ release="pacific",
+ err_text="does not support pacific",
+ os_release=dedent("""
+ NAME="CentOS Linux"
+ VERSION="7 (Core)"
+ ID="centos"
+ ID_LIKE="rhel fedora"
+ VERSION_ID="7"
+ PRETTY_NAME="CentOS Linux 7 (Core)"
+ ANSI_COLOR="0;31"
+ CPE_NAME="cpe:/o:centos:centos:7"
+ HOME_URL="https://www.centos.org/"
+ BUG_REPORT_URL="https://bugs.centos.org/"
+
+ CENTOS_MANTISBT_PROJECT="CentOS-7"
+ CENTOS_MANTISBT_PROJECT_VERSION="7"
+ REDHAT_SUPPORT_PRODUCT="centos"
+ REDHAT_SUPPORT_PRODUCT_VERSION="7"
+ """)),
+
+ # YumDnf on Centos 7 - nothing after pacific
+ dict(
+ version="",
+ release="zillions",
+ err_text="does not support pacific",
+ os_release=dedent("""
+ NAME="CentOS Linux"
+ VERSION="7 (Core)"
+ ID="centos"
+ ID_LIKE="rhel fedora"
+ VERSION_ID="7"
+ PRETTY_NAME="CentOS Linux 7 (Core)"
+ ANSI_COLOR="0;31"
+ CPE_NAME="cpe:/o:centos:centos:7"
+ HOME_URL="https://www.centos.org/"
+ BUG_REPORT_URL="https://bugs.centos.org/"
+
+ CENTOS_MANTISBT_PROJECT="CentOS-7"
+ CENTOS_MANTISBT_PROJECT_VERSION="7"
+ REDHAT_SUPPORT_PRODUCT="centos"
+ REDHAT_SUPPORT_PRODUCT_VERSION="7"
+ """)),
+
+ # YumDnf on Centos 7 - nothing v16 or higher
+ dict(
+ version="v16.1.3",
+ release="",
+ err_text="does not support",
+ os_release=dedent("""
+ NAME="CentOS Linux"
+ VERSION="7 (Core)"
+ ID="centos"
+ ID_LIKE="rhel fedora"
+ VERSION_ID="7"
+ PRETTY_NAME="CentOS Linux 7 (Core)"
+ ANSI_COLOR="0;31"
+ CPE_NAME="cpe:/o:centos:centos:7"
+ HOME_URL="https://www.centos.org/"
+ BUG_REPORT_URL="https://bugs.centos.org/"
+
+ CENTOS_MANTISBT_PROJECT="CentOS-7"
+ CENTOS_MANTISBT_PROJECT_VERSION="7"
+ REDHAT_SUPPORT_PRODUCT="centos"
+ REDHAT_SUPPORT_PRODUCT_VERSION="7"
+ """)),
+ ])
+ @mock.patch('cephadm.find_executable', return_value='foo')
+ def test_distro_validation(self, find_executable, values, cephadm_fs):
+ os_release = values['os_release']
+ release = values['release']
+ version = values['version']
+ err_text = values['err_text']
+
+ cephadm_fs.create_file('/etc/os-release', contents=os_release)
+ ctx = cd.CephadmContext()
+ ctx.repo_url = 'http://localhost'
+ pkg = cd.create_packager(ctx, stable=release, version=version)
+
+ if err_text:
+ with pytest.raises(cd.Error, match=err_text):
+ pkg.validate()
+ else:
+ with mock.patch('cephadm.urlopen', return_value=None):
+ pkg.validate()
+
+ @pytest.mark.parametrize('values',
+ [
+ # Apt - not checked
+ dict(
+ version="",
+ release="pacific",
+ err_text="",
+ os_release=dedent("""
+ NAME="Ubuntu"
+ VERSION="20.04 LTS (Focal Fossa)"
+ ID=ubuntu
+ ID_LIKE=debian
+ PRETTY_NAME="Ubuntu 20.04 LTS"
+ VERSION_ID="20.04"
+ HOME_URL="https://www.ubuntu.com/"
+ SUPPORT_URL="https://help.ubuntu.com/"
+ BUG_REPORT_URL="https://bugs.launchpad.net/ubuntu/"
+ PRIVACY_POLICY_URL="https://www.ubuntu.com/legal/terms-and-policies/privacy-policy"
+ VERSION_CODENAME=focal
+ UBUNTU_CODENAME=focal
+ """)),
+
+ # YumDnf on Centos8 - force failure
+ dict(
+ version="",
+ release="foobar",
+ err_text="failed to fetch repository metadata",
+ os_release=dedent("""
+ NAME="CentOS Linux"
+ VERSION="8 (Core)"
+ ID="centos"
+ ID_LIKE="rhel fedora"
+ VERSION_ID="8"
+ PLATFORM_ID="platform:el8"
+ PRETTY_NAME="CentOS Linux 8 (Core)"
+ ANSI_COLOR="0;31"
+ CPE_NAME="cpe:/o:centos:centos:8"
+ HOME_URL="https://www.centos.org/"
+ BUG_REPORT_URL="https://bugs.centos.org/"
+
+ CENTOS_MANTISBT_PROJECT="CentOS-8"
+ CENTOS_MANTISBT_PROJECT_VERSION="8"
+ REDHAT_SUPPORT_PRODUCT="centos"
+ REDHAT_SUPPORT_PRODUCT_VERSION="8"
+ """)),
+ ])
+ @mock.patch('cephadm.find_executable', return_value='foo')
+ def test_http_validation(self, find_executable, values, cephadm_fs):
+ from urllib.error import HTTPError
+
+ os_release = values['os_release']
+ release = values['release']
+ version = values['version']
+ err_text = values['err_text']
+
+ cephadm_fs.create_file('/etc/os-release', contents=os_release)
+ ctx = cd.CephadmContext()
+ ctx.repo_url = 'http://localhost'
+ pkg = cd.create_packager(ctx, stable=release, version=version)
+
+ with mock.patch('cephadm.urlopen') as _urlopen:
+ _urlopen.side_effect = HTTPError(ctx.repo_url, 404, "not found", None, fp=None)
+ if err_text:
+ with pytest.raises(cd.Error, match=err_text):
+ pkg.validate()
+ else:
+ pkg.validate()
+
+
+class TestPull:
+
+ @mock.patch('time.sleep')
+ @mock.patch('cephadm.call', return_value=('', '', 0))
+ @mock.patch('cephadm.get_image_info_from_inspect', return_value={})
+ def test_error(self, get_image_info_from_inspect, call, sleep):
+ ctx = cd.CephadmContext()
+ ctx.container_engine = mock_podman()
+ ctx.insecure = False
+
+ call.return_value = ('', '', 0)
+ retval = cd.command_pull(ctx)
+ assert retval == 0
+
+ err = 'maximum retries reached'
+
+ call.return_value = ('', 'foobar', 1)
+ with pytest.raises(cd.Error) as e:
+ cd.command_pull(ctx)
+ assert err not in str(e.value)
+
+ call.return_value = ('', 'net/http: TLS handshake timeout', 1)
+ with pytest.raises(cd.Error) as e:
+ cd.command_pull(ctx)
+ assert err in str(e.value)
+
+ @mock.patch('cephadm.logger')
+ @mock.patch('cephadm.get_image_info_from_inspect', return_value={})
+ @mock.patch('cephadm.get_last_local_ceph_image', return_value='last_local_ceph_image')
+ def test_image(self, get_last_local_ceph_image, get_image_info_from_inspect, logger):
+ cmd = ['pull']
+ with with_cephadm_ctx(cmd) as ctx:
+ retval = cd.command_pull(ctx)
+ assert retval == 0
+ assert ctx.image == cd.DEFAULT_IMAGE
+
+ with mock.patch.dict(os.environ, {"CEPHADM_IMAGE": 'cephadm_image_environ'}):
+ cmd = ['pull']
+ with with_cephadm_ctx(cmd) as ctx:
+ retval = cd.command_pull(ctx)
+ assert retval == 0
+ assert ctx.image == 'cephadm_image_environ'
+
+ cmd = ['--image', 'cephadm_image_param', 'pull']
+ with with_cephadm_ctx(cmd) as ctx:
+ retval = cd.command_pull(ctx)
+ assert retval == 0
+ assert ctx.image == 'cephadm_image_param'
+
+
+class TestApplySpec:
+
+ def test_parse_yaml(self, cephadm_fs):
+ yaml = '''---
+service_type: host
+hostname: vm-00
+addr: 192.168.122.44
+labels:
+ - example1
+ - example2
+---
+service_type: host
+hostname: vm-01
+addr: 192.168.122.247
+labels:
+ - grafana
+---
+service_type: host
+hostname: vm-02
+addr: 192.168.122.165
+---
+---
+service_type: rgw
+service_id: myrgw
+spec:
+ rgw_frontend_ssl_certificate: |
+ -----BEGIN PRIVATE KEY-----
+ V2VyIGRhcyBsaWVzdCBpc3QgZG9vZi4gTG9yZW0gaXBzdW0gZG9sb3Igc2l0IGFt
+ ZXQsIGNvbnNldGV0dXIgc2FkaXBzY2luZyBlbGl0ciwgc2VkIGRpYW0gbm9udW15
+ IGVpcm1vZCB0ZW1wb3IgaW52aWR1bnQgdXQgbGFib3JlIGV0IGRvbG9yZSBtYWdu
+ YSBhbGlxdXlhbSBlcmF0LCBzZWQgZGlhbSB2b2x1cHR1YS4gQXQgdmVybyBlb3Mg
+ ZXQgYWNjdXNhbSBldCBqdXN0byBkdW8=
+ -----END PRIVATE KEY-----
+ -----BEGIN CERTIFICATE-----
+ V2VyIGRhcyBsaWVzdCBpc3QgZG9vZi4gTG9yZW0gaXBzdW0gZG9sb3Igc2l0IGFt
+ ZXQsIGNvbnNldGV0dXIgc2FkaXBzY2luZyBlbGl0ciwgc2VkIGRpYW0gbm9udW15
+ IGVpcm1vZCB0ZW1wb3IgaW52aWR1bnQgdXQgbGFib3JlIGV0IGRvbG9yZSBtYWdu
+ YSBhbGlxdXlhbSBlcmF0LCBzZWQgZGlhbSB2b2x1cHR1YS4gQXQgdmVybyBlb3Mg
+ ZXQgYWNjdXNhbSBldCBqdXN0byBkdW8=
+ -----END CERTIFICATE-----
+ ssl: true
+---
+'''
+
+ cephadm_fs.create_file('spec.yml', contents=yaml)
+ retdic = [{'service_type': 'host', 'hostname': 'vm-00', 'addr': '192.168.122.44', 'labels': '- example1- example2'},
+ {'service_type': 'host', 'hostname': 'vm-01', 'addr': '192.168.122.247', 'labels': '- grafana'},
+ {'service_type': 'host', 'hostname': 'vm-02', 'addr': '192.168.122.165'},
+ {'service_id': 'myrgw',
+ 'service_type': 'rgw',
+ 'spec':
+ 'rgw_frontend_ssl_certificate: |-----BEGIN PRIVATE '
+ 'KEY-----V2VyIGRhcyBsaWVzdCBpc3QgZG9vZi4gTG9yZW0gaXBzdW0gZG9sb3Igc2l0IGFtZXQsIGNvbnNldGV0dXIgc2FkaXBzY2luZyBlbGl0ciwgc2VkIGRpYW0gbm9udW15IGVpcm1vZCB0ZW1wb3IgaW52aWR1bnQgdXQgbGFib3JlIGV0IGRvbG9yZSBtYWduYSBhbGlxdXlhbSBlcmF0LCBzZWQgZGlhbSB2b2x1cHR1YS4gQXQgdmVybyBlb3MgZXQgYWNjdXNhbSBldCBqdXN0byBkdW8=-----END '
+ 'PRIVATE KEY----------BEGIN '
+ 'CERTIFICATE-----V2VyIGRhcyBsaWVzdCBpc3QgZG9vZi4gTG9yZW0gaXBzdW0gZG9sb3Igc2l0IGFtZXQsIGNvbnNldGV0dXIgc2FkaXBzY2luZyBlbGl0ciwgc2VkIGRpYW0gbm9udW15IGVpcm1vZCB0ZW1wb3IgaW52aWR1bnQgdXQgbGFib3JlIGV0IGRvbG9yZSBtYWduYSBhbGlxdXlhbSBlcmF0LCBzZWQgZGlhbSB2b2x1cHR1YS4gQXQgdmVybyBlb3MgZXQgYWNjdXNhbSBldCBqdXN0byBkdW8=-----END '
+ 'CERTIFICATE-----ssl: true'}]
+
+ with open('spec.yml') as f:
+ dic = cd.parse_yaml_objs(f)
+ assert dic == retdic
+
+ @mock.patch('cephadm.call', return_value=('', '', 0))
+ def test_distribute_ssh_keys(self, call):
+ ctx = cd.CephadmContext()
+ ctx.ssh_public_key = None
+ ctx.ssh_user = 'root'
+
+ host_spec = {'service_type': 'host', 'hostname': 'vm-02', 'addr': '192.168.122.165'}
+
+ retval = cd._distribute_ssh_keys(ctx, host_spec, 'bootstrap_hostname')
+
+ assert retval == 0
+
+ call.return_value = ('', '', 1)
+
+ retval = cd._distribute_ssh_keys(ctx, host_spec, 'bootstrap_hostname')
+
+ assert retval == 1
+
+
+class TestSNMPGateway:
+ V2c_config = {
+ 'snmp_community': 'public',
+ 'destination': '192.168.1.10:162',
+ 'snmp_version': 'V2c',
+ }
+ V3_no_priv_config = {
+ 'destination': '192.168.1.10:162',
+ 'snmp_version': 'V3',
+ 'snmp_v3_auth_username': 'myuser',
+ 'snmp_v3_auth_password': 'mypassword',
+ 'snmp_v3_auth_protocol': 'SHA',
+ 'snmp_v3_engine_id': '8000C53F00000000',
+ }
+ V3_priv_config = {
+ 'destination': '192.168.1.10:162',
+ 'snmp_version': 'V3',
+ 'snmp_v3_auth_username': 'myuser',
+ 'snmp_v3_auth_password': 'mypassword',
+ 'snmp_v3_auth_protocol': 'SHA',
+ 'snmp_v3_priv_protocol': 'DES',
+ 'snmp_v3_priv_password': 'mysecret',
+ 'snmp_v3_engine_id': '8000C53F00000000',
+ }
+ no_destination_config = {
+ 'snmp_version': 'V3',
+ 'snmp_v3_auth_username': 'myuser',
+ 'snmp_v3_auth_password': 'mypassword',
+ 'snmp_v3_auth_protocol': 'SHA',
+ 'snmp_v3_priv_protocol': 'DES',
+ 'snmp_v3_priv_password': 'mysecret',
+ 'snmp_v3_engine_id': '8000C53F00000000',
+ }
+ bad_version_config = {
+ 'snmp_community': 'public',
+ 'destination': '192.168.1.10:162',
+ 'snmp_version': 'V1',
+ }
+
+ def test_unit_run_V2c(self, cephadm_fs):
+ fsid = 'ca734440-3dc6-11ec-9b98-5254002537a6'
+ with with_cephadm_ctx(['--image=docker.io/maxwo/snmp-notifier:v1.2.1'], list_networks={}) as ctx:
+ import json
+ ctx.config_json = json.dumps(self.V2c_config)
+ ctx.fsid = fsid
+ ctx.tcp_ports = '9464'
+ cd.get_parm.return_value = self.V2c_config
+ c = cd.get_container(ctx, fsid, 'snmp-gateway', 'daemon_id')
+
+ cd.make_data_dir(ctx, fsid, 'snmp-gateway', 'daemon_id')
+
+ cd.create_daemon_dirs(ctx, fsid, 'snmp-gateway', 'daemon_id', 0, 0)
+ with open(f'/var/lib/ceph/{fsid}/snmp-gateway.daemon_id/snmp-gateway.conf', 'r') as f:
+ conf = f.read().rstrip()
+ assert conf == 'SNMP_NOTIFIER_COMMUNITY=public'
+
+ cd.deploy_daemon_units(
+ ctx,
+ fsid,
+ 0, 0,
+ 'snmp-gateway',
+ 'daemon_id',
+ c,
+ True, True
+ )
+ with open(f'/var/lib/ceph/{fsid}/snmp-gateway.daemon_id/unit.run', 'r') as f:
+ run_cmd = f.readlines()[-1].rstrip()
+ assert run_cmd.endswith('docker.io/maxwo/snmp-notifier:v1.2.1 --web.listen-address=:9464 --snmp.destination=192.168.1.10:162 --snmp.version=V2c --log.level=info --snmp.trap-description-template=/etc/snmp_notifier/description-template.tpl')
+
+ def test_unit_run_V3_noPriv(self, cephadm_fs):
+ fsid = 'ca734440-3dc6-11ec-9b98-5254002537a6'
+ with with_cephadm_ctx(['--image=docker.io/maxwo/snmp-notifier:v1.2.1'], list_networks={}) as ctx:
+ import json
+ ctx.config_json = json.dumps(self.V3_no_priv_config)
+ ctx.fsid = fsid
+ ctx.tcp_ports = '9465'
+ cd.get_parm.return_value = self.V3_no_priv_config
+ c = cd.get_container(ctx, fsid, 'snmp-gateway', 'daemon_id')
+
+ cd.make_data_dir(ctx, fsid, 'snmp-gateway', 'daemon_id')
+
+ cd.create_daemon_dirs(ctx, fsid, 'snmp-gateway', 'daemon_id', 0, 0)
+ with open(f'/var/lib/ceph/{fsid}/snmp-gateway.daemon_id/snmp-gateway.conf', 'r') as f:
+ conf = f.read()
+ assert conf == 'SNMP_NOTIFIER_AUTH_USERNAME=myuser\nSNMP_NOTIFIER_AUTH_PASSWORD=mypassword\n'
+
+ cd.deploy_daemon_units(
+ ctx,
+ fsid,
+ 0, 0,
+ 'snmp-gateway',
+ 'daemon_id',
+ c,
+ True, True
+ )
+ with open(f'/var/lib/ceph/{fsid}/snmp-gateway.daemon_id/unit.run', 'r') as f:
+ run_cmd = f.readlines()[-1].rstrip()
+ assert run_cmd.endswith('docker.io/maxwo/snmp-notifier:v1.2.1 --web.listen-address=:9465 --snmp.destination=192.168.1.10:162 --snmp.version=V3 --log.level=info --snmp.trap-description-template=/etc/snmp_notifier/description-template.tpl --snmp.authentication-enabled --snmp.authentication-protocol=SHA --snmp.security-engine-id=8000C53F00000000')
+
+ def test_unit_run_V3_Priv(self, cephadm_fs):
+ fsid = 'ca734440-3dc6-11ec-9b98-5254002537a6'
+ with with_cephadm_ctx(['--image=docker.io/maxwo/snmp-notifier:v1.2.1'], list_networks={}) as ctx:
+ import json
+ ctx.config_json = json.dumps(self.V3_priv_config)
+ ctx.fsid = fsid
+ ctx.tcp_ports = '9464'
+ cd.get_parm.return_value = self.V3_priv_config
+ c = cd.get_container(ctx, fsid, 'snmp-gateway', 'daemon_id')
+
+ cd.make_data_dir(ctx, fsid, 'snmp-gateway', 'daemon_id')
+
+ cd.create_daemon_dirs(ctx, fsid, 'snmp-gateway', 'daemon_id', 0, 0)
+ with open(f'/var/lib/ceph/{fsid}/snmp-gateway.daemon_id/snmp-gateway.conf', 'r') as f:
+ conf = f.read()
+ assert conf == 'SNMP_NOTIFIER_AUTH_USERNAME=myuser\nSNMP_NOTIFIER_AUTH_PASSWORD=mypassword\nSNMP_NOTIFIER_PRIV_PASSWORD=mysecret\n'
+
+ cd.deploy_daemon_units(
+ ctx,
+ fsid,
+ 0, 0,
+ 'snmp-gateway',
+ 'daemon_id',
+ c,
+ True, True
+ )
+ with open(f'/var/lib/ceph/{fsid}/snmp-gateway.daemon_id/unit.run', 'r') as f:
+ run_cmd = f.readlines()[-1].rstrip()
+ assert run_cmd.endswith('docker.io/maxwo/snmp-notifier:v1.2.1 --web.listen-address=:9464 --snmp.destination=192.168.1.10:162 --snmp.version=V3 --log.level=info --snmp.trap-description-template=/etc/snmp_notifier/description-template.tpl --snmp.authentication-enabled --snmp.authentication-protocol=SHA --snmp.security-engine-id=8000C53F00000000 --snmp.private-enabled --snmp.private-protocol=DES')
+
+ def test_unit_run_no_dest(self, cephadm_fs):
+ fsid = 'ca734440-3dc6-11ec-9b98-5254002537a6'
+ with with_cephadm_ctx(['--image=docker.io/maxwo/snmp-notifier:v1.2.1'], list_networks={}) as ctx:
+ import json
+ ctx.config_json = json.dumps(self.no_destination_config)
+ ctx.fsid = fsid
+ ctx.tcp_ports = '9464'
+ cd.get_parm.return_value = self.no_destination_config
+
+ with pytest.raises(Exception) as e:
+ c = cd.get_container(ctx, fsid, 'snmp-gateway', 'daemon_id')
+ assert str(e.value) == "config is missing destination attribute(<ip>:<port>) of the target SNMP listener"
+
+ def test_unit_run_bad_version(self, cephadm_fs):
+ fsid = 'ca734440-3dc6-11ec-9b98-5254002537a6'
+ with with_cephadm_ctx(['--image=docker.io/maxwo/snmp-notifier:v1.2.1'], list_networks={}) as ctx:
+ import json
+ ctx.config_json = json.dumps(self.bad_version_config)
+ ctx.fsid = fsid
+ ctx.tcp_ports = '9464'
+ cd.get_parm.return_value = self.bad_version_config
+
+ with pytest.raises(Exception) as e:
+ c = cd.get_container(ctx, fsid, 'snmp-gateway', 'daemon_id')
+ assert str(e.value) == 'not a valid snmp version: V1'
+
+class TestNetworkValidation:
+
+ def test_ipv4_subnet(self):
+ rc, v, msg = cd.check_subnet('192.168.1.0/24')
+ assert rc == 0 and v[0] == 4
+
+ def test_ipv4_subnet_list(self):
+ rc, v, msg = cd.check_subnet('192.168.1.0/24,10.90.90.0/24')
+ assert rc == 0 and not msg
+
+ def test_ipv4_subnet_list_with_spaces(self):
+ rc, v, msg = cd.check_subnet('192.168.1.0/24, 10.90.90.0/24 ')
+ assert rc == 0 and not msg
+
+ def test_ipv4_subnet_badlist(self):
+ rc, v, msg = cd.check_subnet('192.168.1.0/24,192.168.1.1')
+ assert rc == 1 and msg
+
+ def test_ipv4_subnet_mixed(self):
+ rc, v, msg = cd.check_subnet('192.168.100.0/24,fe80::/64')
+ assert rc == 0 and v == [4,6]
+
+ def test_ipv6_subnet(self):
+ rc, v, msg = cd.check_subnet('fe80::/64')
+ assert rc == 0 and v[0] == 6
+
+ def test_subnet_mask_missing(self):
+ rc, v, msg = cd.check_subnet('192.168.1.58')
+ assert rc == 1 and msg
+
+ def test_subnet_mask_junk(self):
+ rc, v, msg = cd.check_subnet('wah')
+ assert rc == 1 and msg
+
+ def test_ip_in_subnet(self):
+ # valid ip and only one valid subnet
+ rc = cd.ip_in_subnets('192.168.100.1', '192.168.100.0/24')
+ assert rc is True
+
+ # valid ip and valid subnets list without spaces
+ rc = cd.ip_in_subnets('192.168.100.1', '192.168.100.0/24,10.90.90.0/24')
+ assert rc is True
+
+ # valid ip and valid subnets list with spaces
+ rc = cd.ip_in_subnets('10.90.90.2', '192.168.1.0/24, 192.168.100.0/24, 10.90.90.0/24')
+ assert rc is True
+
+ # valid ip that doesn't belong to any subnet
+ rc = cd.ip_in_subnets('192.168.100.2', '192.168.50.0/24, 10.90.90.0/24')
+ assert rc is False
+
+ # valid ip that doesn't belong to the subnet (only 14 hosts)
+ rc = cd.ip_in_subnets('192.168.100.20', '192.168.100.0/28')
+ assert rc is False
+
+ # valid ip and valid IPV6 network
+ rc = cd.ip_in_subnets('fe80::5054:ff:fef4:873a', 'fe80::/64')
+ assert rc is True
+
+ # valid wrapped ip and valid IPV6 network
+ rc = cd.ip_in_subnets('[fe80::5054:ff:fef4:873a]', 'fe80::/64')
+ assert rc is True
+
+ # valid ip and that doesn't belong to IPV6 network
+ rc = cd.ip_in_subnets('fe80::5054:ff:fef4:873a', '2001:db8:85a3::/64')
+ assert rc is False
+
+ # invalid IPv4 and valid subnets list
+ with pytest.raises(Exception):
+ rc = cd.ip_in_sublets('10.90.200.', '192.168.1.0/24, 192.168.100.0/24, 10.90.90.0/24')
+
+ # invalid IPv6 and valid subnets list
+ with pytest.raises(Exception):
+ rc = cd.ip_in_sublets('fe80:2030:31:24', 'fe80::/64')
+
+ @pytest.mark.parametrize("conf", [
+ """[global]
+public_network='1.1.1.0/24,2.2.2.0/24'
+cluster_network="3.3.3.0/24, 4.4.4.0/24"
+""",
+ """[global]
+public_network=" 1.1.1.0/24,2.2.2.0/24 "
+cluster_network=3.3.3.0/24, 4.4.4.0/24
+""",
+ """[global]
+ public_network= 1.1.1.0/24, 2.2.2.0/24
+ cluster_network='3.3.3.0/24,4.4.4.0/24'
+"""])
+ @mock.patch('cephadm.list_networks')
+ def test_get_networks_from_conf(self, _list_networks, conf, cephadm_fs):
+ cephadm_fs.create_file('ceph.conf', contents=conf)
+ _list_networks.return_value = {'1.1.1.0/24': {'eth0': ['1.1.1.1']},
+ '2.2.2.0/24': {'eth1': ['2.2.2.2']},
+ '3.3.3.0/24': {'eth2': ['3.3.3.3']},
+ '4.4.4.0/24': {'eth3': ['4.4.4.4']}}
+ ctx = cd.CephadmContext()
+ ctx.config = 'ceph.conf'
+ ctx.mon_ip = '1.1.1.1'
+ ctx.cluster_network = None
+ # what the cephadm module does with the public network string is
+ # [x.strip() for x in out.split(',')]
+ # so we must make sure our output, through that alteration,
+ # generates correctly formatted networks
+ def _str_to_networks(s):
+ return [x.strip() for x in s.split(',')]
+ public_network = cd.get_public_net_from_cfg(ctx)
+ assert _str_to_networks(public_network) == ['1.1.1.0/24', '2.2.2.0/24']
+ cluster_network, ipv6 = cd.prepare_cluster_network(ctx)
+ assert not ipv6
+ assert _str_to_networks(cluster_network) == ['3.3.3.0/24', '4.4.4.0/24']
+
+class TestRescan(fake_filesystem_unittest.TestCase):
+
+ def setUp(self):
+ self.setUpPyfakefs()
+ if not fake_filesystem.is_root():
+ fake_filesystem.set_uid(0)
+
+ self.fs.create_dir('/sys/class')
+ self.ctx = cd.CephadmContext()
+ self.ctx.func = cd.command_rescan_disks
+
+ def test_no_hbas(self):
+ out = cd.command_rescan_disks(self.ctx)
+ assert out == 'Ok. No compatible HBAs found'
+
+ def test_success(self):
+ self.fs.create_file('/sys/class/scsi_host/host0/scan')
+ self.fs.create_file('/sys/class/scsi_host/host1/scan')
+ out = cd.command_rescan_disks(self.ctx)
+ assert out.startswith('Ok. 2 adapters detected: 2 rescanned, 0 skipped, 0 failed')
+
+ def test_skip_usb_adapter(self):
+ self.fs.create_file('/sys/class/scsi_host/host0/scan')
+ self.fs.create_file('/sys/class/scsi_host/host1/scan')
+ self.fs.create_file('/sys/class/scsi_host/host1/proc_name', contents='usb-storage')
+ out = cd.command_rescan_disks(self.ctx)
+ assert out.startswith('Ok. 2 adapters detected: 1 rescanned, 1 skipped, 0 failed')
+
+ def test_skip_unknown_adapter(self):
+ self.fs.create_file('/sys/class/scsi_host/host0/scan')
+ self.fs.create_file('/sys/class/scsi_host/host1/scan')
+ self.fs.create_file('/sys/class/scsi_host/host1/proc_name', contents='unknown')
+ out = cd.command_rescan_disks(self.ctx)
+ assert out.startswith('Ok. 2 adapters detected: 1 rescanned, 1 skipped, 0 failed')
diff --git a/src/cephadm/tests/test_networks.py b/src/cephadm/tests/test_networks.py
new file mode 100644
index 000000000..6fa47ea27
--- /dev/null
+++ b/src/cephadm/tests/test_networks.py
@@ -0,0 +1,235 @@
+import json
+from textwrap import dedent
+from unittest import mock
+
+import pytest
+
+from tests.fixtures import with_cephadm_ctx, cephadm_fs
+
+with mock.patch('builtins.open', create=True):
+ from importlib.machinery import SourceFileLoader
+ cd = SourceFileLoader('cephadm', 'cephadm').load_module()
+
+
+class TestCommandListNetworks:
+ @pytest.mark.parametrize("test_input, expected", [
+ (
+ dedent("""
+ default via 192.168.178.1 dev enxd89ef3f34260 proto dhcp metric 100
+ 10.0.0.0/8 via 10.4.0.1 dev tun0 proto static metric 50
+ 10.3.0.0/21 via 10.4.0.1 dev tun0 proto static metric 50
+ 10.4.0.1 dev tun0 proto kernel scope link src 10.4.0.2 metric 50
+ 137.1.0.0/16 via 10.4.0.1 dev tun0 proto static metric 50
+ 138.1.0.0/16 via 10.4.0.1 dev tun0 proto static metric 50
+ 139.1.0.0/16 via 10.4.0.1 dev tun0 proto static metric 50
+ 140.1.0.0/17 via 10.4.0.1 dev tun0 proto static metric 50
+ 141.1.0.0/16 via 10.4.0.1 dev tun0 proto static metric 50
+ 172.16.100.34 via 172.16.100.34 dev eth1 proto kernel scope link src 172.16.100.34
+ 192.168.122.1 dev ens3 proto dhcp scope link src 192.168.122.236 metric 100
+ 169.254.0.0/16 dev docker0 scope link metric 1000
+ 172.17.0.0/16 dev docker0 proto kernel scope link src 172.17.0.1
+ 192.168.39.0/24 dev virbr1 proto kernel scope link src 192.168.39.1 linkdown
+ 192.168.122.0/24 dev virbr0 proto kernel scope link src 192.168.122.1 linkdown
+ 192.168.178.0/24 dev enxd89ef3f34260 proto kernel scope link src 192.168.178.28 metric 100
+ 192.168.178.1 dev enxd89ef3f34260 proto static scope link metric 100
+ 195.135.221.12 via 192.168.178.1 dev enxd89ef3f34260 proto static metric 100
+ """),
+ {
+ '172.16.100.34/32': {'eth1': {'172.16.100.34'}},
+ '192.168.122.1/32': {'ens3': {'192.168.122.236'}},
+ '10.4.0.1/32': {'tun0': {'10.4.0.2'}},
+ '172.17.0.0/16': {'docker0': {'172.17.0.1'}},
+ '192.168.39.0/24': {'virbr1': {'192.168.39.1'}},
+ '192.168.122.0/24': {'virbr0': {'192.168.122.1'}},
+ '192.168.178.0/24': {'enxd89ef3f34260': {'192.168.178.28'}}
+ }
+ ), (
+ dedent("""
+ default via 10.3.64.1 dev eno1 proto static metric 100
+ 10.3.64.0/24 dev eno1 proto kernel scope link src 10.3.64.23 metric 100
+ 10.3.64.0/24 dev eno1 proto kernel scope link src 10.3.64.27 metric 100
+ 10.88.0.0/16 dev cni-podman0 proto kernel scope link src 10.88.0.1 linkdown
+ 172.21.0.0/20 via 172.21.3.189 dev tun0
+ 172.21.1.0/20 via 172.21.3.189 dev tun0
+ 172.21.2.1 via 172.21.3.189 dev tun0
+ 172.21.3.1 dev tun0 proto kernel scope link src 172.21.3.2
+ 172.21.4.0/24 via 172.21.3.1 dev tun0
+ 172.21.5.0/24 via 172.21.3.1 dev tun0
+ 172.21.6.0/24 via 172.21.3.1 dev tun0
+ 172.21.7.0/24 via 172.21.3.1 dev tun0
+ 192.168.122.0/24 dev virbr0 proto kernel scope link src 192.168.122.1 linkdown
+ 192.168.122.0/24 dev virbr0 proto kernel scope link src 192.168.122.1 linkdown
+ 192.168.122.0/24 dev virbr0 proto kernel scope link src 192.168.122.1 linkdown
+ 192.168.122.0/24 dev virbr0 proto kernel scope link src 192.168.122.1 linkdown
+ """),
+ {
+ '10.3.64.0/24': {'eno1': {'10.3.64.23', '10.3.64.27'}},
+ '10.88.0.0/16': {'cni-podman0': {'10.88.0.1'}},
+ '172.21.3.1/32': {'tun0': {'172.21.3.2'}},
+ '192.168.122.0/24': {'virbr0': {'192.168.122.1'}}
+ }
+ ),
+ ])
+ def test_parse_ipv4_route(self, test_input, expected):
+ assert cd._parse_ipv4_route(test_input) == expected
+
+ @pytest.mark.parametrize("test_routes, test_ips, expected", [
+ (
+ dedent("""
+ ::1 dev lo proto kernel metric 256 pref medium
+ fe80::/64 dev eno1 proto kernel metric 100 pref medium
+ fe80::/64 dev br-3d443496454c proto kernel metric 256 linkdown pref medium
+ fe80::/64 dev tun0 proto kernel metric 256 pref medium
+ fe80::/64 dev br-4355f5dbb528 proto kernel metric 256 pref medium
+ fe80::/64 dev docker0 proto kernel metric 256 linkdown pref medium
+ fe80::/64 dev cni-podman0 proto kernel metric 256 linkdown pref medium
+ fe80::/64 dev veth88ba1e8 proto kernel metric 256 pref medium
+ fe80::/64 dev vethb6e5fc7 proto kernel metric 256 pref medium
+ fe80::/64 dev vethaddb245 proto kernel metric 256 pref medium
+ fe80::/64 dev vethbd14d6b proto kernel metric 256 pref medium
+ fe80::/64 dev veth13e8fd2 proto kernel metric 256 pref medium
+ fe80::/64 dev veth1d3aa9e proto kernel metric 256 pref medium
+ fe80::/64 dev vethe485ca9 proto kernel metric 256 pref medium
+ """),
+ dedent("""
+ 1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 state UNKNOWN qlen 1000
+ inet6 ::1/128 scope host
+ valid_lft forever preferred_lft forever
+ 2: eno1: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 state UP qlen 1000
+ inet6 fe80::225:90ff:fee5:26e8/64 scope link noprefixroute
+ valid_lft forever preferred_lft forever
+ 6: br-3d443496454c: <NO-CARRIER,BROADCAST,MULTICAST,UP> mtu 1500 state DOWN
+ inet6 fe80::42:23ff:fe9d:ee4/64 scope link
+ valid_lft forever preferred_lft forever
+ 7: br-4355f5dbb528: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 state UP
+ inet6 fe80::42:6eff:fe35:41fe/64 scope link
+ valid_lft forever preferred_lft forever
+ 8: docker0: <NO-CARRIER,BROADCAST,MULTICAST,UP> mtu 1500 state DOWN
+ inet6 fe80::42:faff:fee6:40a0/64 scope link
+ valid_lft forever preferred_lft forever
+ 11: tun0: <POINTOPOINT,MULTICAST,NOARP,UP,LOWER_UP> mtu 1500 state UNKNOWN qlen 100
+ inet6 fe80::98a6:733e:dafd:350/64 scope link stable-privacy
+ valid_lft forever preferred_lft forever
+ 28: cni-podman0: <NO-CARRIER,BROADCAST,MULTICAST,UP> mtu 1500 state DOWN qlen 1000
+ inet6 fe80::3449:cbff:fe89:b87e/64 scope link
+ valid_lft forever preferred_lft forever
+ 31: vethaddb245@if30: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 state UP
+ inet6 fe80::90f7:3eff:feed:a6bb/64 scope link
+ valid_lft forever preferred_lft forever
+ 33: veth88ba1e8@if32: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 state UP
+ inet6 fe80::d:f5ff:fe73:8c82/64 scope link
+ valid_lft forever preferred_lft forever
+ 35: vethbd14d6b@if34: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 state UP
+ inet6 fe80::b44f:8ff:fe6f:813d/64 scope link
+ valid_lft forever preferred_lft forever
+ 37: vethb6e5fc7@if36: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 state UP
+ inet6 fe80::4869:c6ff:feaa:8afe/64 scope link
+ valid_lft forever preferred_lft forever
+ 39: veth13e8fd2@if38: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 state UP
+ inet6 fe80::78f4:71ff:fefe:eb40/64 scope link
+ valid_lft forever preferred_lft forever
+ 41: veth1d3aa9e@if40: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 state UP
+ inet6 fe80::24bd:88ff:fe28:5b18/64 scope link
+ valid_lft forever preferred_lft forever
+ 43: vethe485ca9@if42: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 state UP
+ inet6 fe80::6425:87ff:fe42:b9f0/64 scope link
+ valid_lft forever preferred_lft forever
+ """),
+ {
+ "fe80::/64": {
+ "eno1": {"fe80::225:90ff:fee5:26e8"},
+ "br-3d443496454c": {"fe80::42:23ff:fe9d:ee4"},
+ "tun0": {"fe80::98a6:733e:dafd:350"},
+ "br-4355f5dbb528": {"fe80::42:6eff:fe35:41fe"},
+ "docker0": {"fe80::42:faff:fee6:40a0"},
+ "cni-podman0": {"fe80::3449:cbff:fe89:b87e"},
+ "veth88ba1e8": {"fe80::d:f5ff:fe73:8c82"},
+ "vethb6e5fc7": {"fe80::4869:c6ff:feaa:8afe"},
+ "vethaddb245": {"fe80::90f7:3eff:feed:a6bb"},
+ "vethbd14d6b": {"fe80::b44f:8ff:fe6f:813d"},
+ "veth13e8fd2": {"fe80::78f4:71ff:fefe:eb40"},
+ "veth1d3aa9e": {"fe80::24bd:88ff:fe28:5b18"},
+ "vethe485ca9": {"fe80::6425:87ff:fe42:b9f0"},
+ }
+ }
+ ),
+ (
+ dedent("""
+ ::1 dev lo proto kernel metric 256 pref medium
+ 2001:1458:301:eb::100:1a dev ens20f0 proto kernel metric 100 pref medium
+ 2001:1458:301:eb::/64 dev ens20f0 proto ra metric 100 pref medium
+ fd01:1458:304:5e::/64 dev ens20f0 proto ra metric 100 pref medium
+ fe80::/64 dev ens20f0 proto kernel metric 100 pref medium
+ default proto ra metric 100
+ nexthop via fe80::46ec:ce00:b8a0:d3c8 dev ens20f0 weight 1
+ nexthop via fe80::46ec:ce00:b8a2:33c8 dev ens20f0 weight 1 pref medium
+ """),
+ dedent("""
+ 1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 state UNKNOWN qlen 1000
+ inet6 ::1/128 scope host
+ valid_lft forever preferred_lft forever
+ 2: ens20f0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 state UP qlen 1000
+ inet6 2001:1458:301:eb::100:1a/128 scope global dynamic noprefixroute
+ valid_lft 590879sec preferred_lft 590879sec
+ inet6 fe80::2e60:cff:fef8:da41/64 scope link noprefixroute
+ valid_lft forever preferred_lft forever
+ inet6 fe80::2e60:cff:fef8:da41/64 scope link noprefixroute
+ valid_lft forever preferred_lft forever
+ inet6 fe80::2e60:cff:fef8:da41/64 scope link noprefixroute
+ valid_lft forever preferred_lft forever
+ """),
+ {
+ '2001:1458:301:eb::100:1a/128': {
+ 'ens20f0': {
+ '2001:1458:301:eb::100:1a'
+ },
+ },
+ '2001:1458:301:eb::/64': {
+ 'ens20f0': set(),
+ },
+ 'fe80::/64': {
+ 'ens20f0': {'fe80::2e60:cff:fef8:da41'},
+ },
+ 'fd01:1458:304:5e::/64': {
+ 'ens20f0': set()
+ },
+ }
+ ),
+ (
+ dedent("""
+ ::1 dev lo proto kernel metric 256 pref medium
+ fe80::/64 dev ceph-brx proto kernel metric 256 pref medium
+ fe80::/64 dev brx.0 proto kernel metric 256 pref medium
+ default via fe80::327c:5e00:6487:71e0 dev enp3s0f1 proto ra metric 1024 expires 1790sec hoplimit 64 pref medium """),
+ dedent("""
+ 1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 state UNKNOWN qlen 1000
+ inet6 ::1/128 scope host
+ valid_lft forever preferred_lft forever
+ 5: enp3s0f1: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 state UP qlen 1000
+ inet6 fe80::ec4:7aff:fe8f:cb83/64 scope link noprefixroute
+ valid_lft forever preferred_lft forever
+ 6: ceph-brx: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 state UP qlen 1000
+ inet6 fe80::d8a1:69ff:fede:8f58/64 scope link
+ valid_lft forever preferred_lft forever
+ 7: brx.0@eno1: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 state UP qlen 1000
+ inet6 fe80::a4cb:54ff:fecc:f2a2/64 scope link
+ valid_lft forever preferred_lft forever
+ """),
+ {
+ 'fe80::/64': {
+ 'brx.0': {'fe80::a4cb:54ff:fecc:f2a2'},
+ 'ceph-brx': {'fe80::d8a1:69ff:fede:8f58'}
+ }
+ }
+ ),
+ ])
+ def test_parse_ipv6_route(self, test_routes, test_ips, expected):
+ assert cd._parse_ipv6_route(test_routes, test_ips) == expected
+
+ @mock.patch.object(cd, 'call_throws', return_value=('10.4.0.1 dev tun0 proto kernel scope link src 10.4.0.2 metric 50\n', '', ''))
+ def test_command_list_networks(self, cephadm_fs, capsys):
+ with with_cephadm_ctx([]) as ctx:
+ cd.command_list_networks(ctx)
+ assert json.loads(capsys.readouterr().out) == {
+ '10.4.0.1/32': {'tun0': ['10.4.0.2']}
+ } \ No newline at end of file
diff --git a/src/cephadm/tox.ini b/src/cephadm/tox.ini
new file mode 100644
index 000000000..079605afc
--- /dev/null
+++ b/src/cephadm/tox.ini
@@ -0,0 +1,62 @@
+[tox]
+envlist =
+ py3
+ mypy
+ fix
+ flake8
+skipsdist = true
+
+[flake8]
+max-line-length = 100
+inline-quotes = '
+ignore =
+ E501,
+ W503,
+exclude =
+ .tox,
+ .vagrant,
+ __pycache__,
+ *.pyc,
+ templates,
+ .eggs
+statistics = True
+
+[autopep8]
+addopts =
+ --max-line-length {[flake8]max-line-length}
+ --ignore "{[flake8]ignore}"
+ --exclude "{[flake8]exclude}"
+ --in-place
+ --recursive
+ --ignore-local-config
+
+[testenv]
+skip_install=true
+deps =
+ pyfakefs == 5.0
+ mock
+ pytest
+commands=pytest {posargs}
+
+[testenv:mypy]
+basepython = python3
+deps = mypy==0.790
+commands = mypy --config-file ../mypy.ini {posargs:cephadm}
+
+[testenv:fix]
+basepython = python3
+deps =
+ autopep8
+commands =
+ python --version
+ autopep8 {[autopep8]addopts} {posargs: cephadm}
+
+[testenv:flake8]
+basepython = python3
+allowlist_externals = bash
+deps =
+ flake8 == 5.0.4
+ flake8-quotes
+commands =
+ flake8 --config=tox.ini {posargs:cephadm}
+ bash -c "test $(grep 'docker.io' cephadm | wc -l) == 11"
diff --git a/src/cephadm/vstart-cleanup.sh b/src/cephadm/vstart-cleanup.sh
new file mode 100755
index 000000000..facbdd100
--- /dev/null
+++ b/src/cephadm/vstart-cleanup.sh
@@ -0,0 +1,6 @@
+#!/bin/sh -ex
+
+bin/ceph mon rm `hostname`
+for f in `bin/ceph orch ls | grep -v NAME | awk '{print $1}'` ; do
+ bin/ceph orch rm $f --force
+done
diff --git a/src/cephadm/vstart-smoke.sh b/src/cephadm/vstart-smoke.sh
new file mode 100755
index 000000000..ecdb59d18
--- /dev/null
+++ b/src/cephadm/vstart-smoke.sh
@@ -0,0 +1,86 @@
+#!/bin/bash -ex
+
+# this is a smoke test, meant to be run against vstart.sh.
+
+host="$(hostname)"
+
+bin/init-ceph stop || true
+MON=1 OSD=1 MDS=0 MGR=1 ../src/vstart.sh -d -n -x -l --cephadm
+
+export CEPH_DEV=1
+
+bin/ceph orch ls
+bin/ceph orch apply mds foo 1
+bin/ceph orch ls | grep foo
+while ! bin/ceph orch ps | grep mds.foo ; do sleep 1 ; done
+bin/ceph orch ps
+
+bin/ceph orch host ls
+
+bin/ceph orch rm crash
+! bin/ceph orch ls | grep crash
+bin/ceph orch apply crash '*'
+bin/ceph orch ls | grep crash
+
+while ! bin/ceph orch ps | grep crash ; do sleep 1 ; done
+bin/ceph orch ps | grep crash.$host | grep running
+bin/ceph orch ls | grep crash | grep 1/1
+bin/ceph orch daemon rm crash.$host
+while ! bin/ceph orch ps | grep crash ; do sleep 1 ; done
+
+bin/ceph orch daemon stop crash.$host
+bin/ceph orch daemon start crash.$host
+bin/ceph orch daemon restart crash.$host
+bin/ceph orch daemon reconfig crash.$host
+bin/ceph orch daemon redeploy crash.$host
+
+bin/ceph orch host ls | grep $host
+bin/ceph orch host label add $host fooxyz
+bin/ceph orch host ls | grep $host | grep fooxyz
+bin/ceph orch host label rm $host fooxyz
+! bin/ceph orch host ls | grep $host | grep fooxyz
+bin/ceph orch host set-addr $host $host
+
+bin/ceph cephadm check-host $host
+#! bin/ceph cephadm check-host $host 1.2.3.4
+#bin/ceph orch host set-addr $host 1.2.3.4
+#! bin/ceph cephadm check-host $host
+bin/ceph orch host set-addr $host $host
+bin/ceph cephadm check-host $host
+
+bin/ceph orch apply mgr 1
+bin/ceph orch rm mgr --force # we don't want a mgr to take over for ours
+
+bin/ceph orch daemon add mon $host:127.0.0.1
+
+while ! bin/ceph mon dump | grep 'epoch 2' ; do sleep 1 ; done
+
+bin/ceph orch apply rbd-mirror 1
+
+bin/ceph orch apply node-exporter '*'
+bin/ceph orch apply prometheus 1
+bin/ceph orch apply alertmanager 1
+bin/ceph orch apply grafana 1
+
+while ! bin/ceph dashboard get-grafana-api-url | grep $host ; do sleep 1 ; done
+
+bin/ceph orch apply rgw foo --placement=1
+
+bin/ceph orch ps
+bin/ceph orch ls
+
+# clean up
+bin/ceph orch rm mds.foo
+bin/ceph orch rm rgw.myrealm.myzone
+bin/ceph orch rm rbd-mirror
+bin/ceph orch rm node-exporter
+bin/ceph orch rm alertmanager
+bin/ceph orch rm grafana
+bin/ceph orch rm prometheus
+bin/ceph orch rm crash
+
+bin/ceph mon rm $host
+! bin/ceph orch daemon rm mon.$host
+bin/ceph orch daemon rm mon.$host --force
+
+echo OK
generated by cgit v1.2.3 (git 2.39.1) at 2024-11-18 22:39:23 +0000