diff options
Diffstat (limited to 'src/pybind/mgr/dashboard/controllers/host.py')
| -rw-r--r-- | src/pybind/mgr/dashboard/controllers/host.py | 507 |
1 files changed, 507 insertions, 0 deletions
diff --git a/src/pybind/mgr/dashboard/controllers/host.py b/src/pybind/mgr/dashboard/controllers/host.py new file mode 100644 index 000000000..22dca02c4 --- /dev/null +++ b/src/pybind/mgr/dashboard/controllers/host.py @@ -0,0 +1,507 @@ +# -*- coding: utf-8 -*- +from __future__ import absolute_import + +import copy +import os +import time +from typing import Dict, List, Optional + +import cherrypy +from mgr_util import merge_dicts +from orchestrator import HostSpec + +from .. import mgr +from ..exceptions import DashboardException +from ..security import Scope +from ..services.ceph_service import CephService +from ..services.exception import handle_orchestrator_error +from ..services.orchestrator import OrchClient, OrchFeature +from ..tools import TaskManager, merge_list_of_dicts_by_key, str_to_bool +from . import APIDoc, APIRouter, BaseController, Endpoint, EndpointDoc, \ + ReadPermission, RESTController, Task, UIRouter, UpdatePermission, \ + allow_empty_body +from ._version import APIVersion +from .orchestrator import raise_if_no_orchestrator + +LIST_HOST_SCHEMA = { + "hostname": (str, "Hostname"), + "services": ([{ + "type": (str, "type of service"), + "id": (str, "Service Id"), + }], "Services related to the host"), + "ceph_version": (str, "Ceph version"), + "addr": (str, "Host address"), + "labels": ([str], "Labels related to the host"), + "service_type": (str, ""), + "sources": ({ + "ceph": (bool, ""), + "orchestrator": (bool, "") + }, "Host Sources"), + "status": (str, "") +} + +INVENTORY_SCHEMA = { + "name": (str, "Hostname"), + "addr": (str, "Host address"), + "devices": ([{ + "rejected_reasons": ([str], ""), + "available": (bool, "If the device can be provisioned to an OSD"), + "path": (str, "Device path"), + "sys_api": ({ + "removable": (str, ""), + "ro": (str, ""), + "vendor": (str, ""), + "model": (str, ""), + "rev": (str, ""), + "sas_address": (str, ""), + "sas_device_handle": (str, ""), + "support_discard": (str, ""), + "rotational": (str, ""), + "nr_requests": (str, ""), + "scheduler_mode": (str, ""), + "partitions": ({ + "partition_name": ({ + "start": (str, ""), + "sectors": (str, ""), + "sectorsize": (int, ""), + "size": (int, ""), + "human_readable_size": (str, ""), + "holders": ([str], "") + }, "") + }, ""), + "sectors": (int, ""), + "sectorsize": (str, ""), + "size": (int, ""), + "human_readable_size": (str, ""), + "path": (str, ""), + "locked": (int, "") + }, ""), + "lvs": ([{ + "name": (str, ""), + "osd_id": (str, ""), + "cluster_name": (str, ""), + "type": (str, ""), + "osd_fsid": (str, ""), + "cluster_fsid": (str, ""), + "osdspec_affinity": (str, ""), + "block_uuid": (str, ""), + }], ""), + "human_readable_type": (str, "Device type. ssd or hdd"), + "device_id": (str, "Device's udev ID"), + "lsm_data": ({ + "serialNum": (str, ""), + "transport": (str, ""), + "mediaType": (str, ""), + "rpm": (str, ""), + "linkSpeed": (str, ""), + "health": (str, ""), + "ledSupport": ({ + "IDENTsupport": (str, ""), + "IDENTstatus": (str, ""), + "FAILsupport": (str, ""), + "FAILstatus": (str, ""), + }, ""), + "errors": ([str], "") + }, ""), + "osd_ids": ([int], "Device OSD IDs") + }], "Host devices"), + "labels": ([str], "Host labels") +} + + +def host_task(name, metadata, wait_for=10.0): + return Task("host/{}".format(name), metadata, wait_for) + + +def merge_hosts_by_hostname(ceph_hosts, orch_hosts): + # type: (List[dict], List[HostSpec]) -> List[dict] + """ + Merge Ceph hosts with orchestrator hosts by hostnames. + + :param ceph_hosts: hosts returned from mgr + :type ceph_hosts: list of dict + :param orch_hosts: hosts returned from ochestrator + :type orch_hosts: list of HostSpec + :return list of dict + """ + hosts = copy.deepcopy(ceph_hosts) + orch_hosts_map = {host.hostname: host.to_json() for host in orch_hosts} + + # Sort labels. + for hostname in orch_hosts_map: + orch_hosts_map[hostname]['labels'].sort() + + # Hosts in both Ceph and Orchestrator. + for host in hosts: + hostname = host['hostname'] + if hostname in orch_hosts_map: + host.update(orch_hosts_map[hostname]) + host['sources']['orchestrator'] = True + orch_hosts_map.pop(hostname) + + # Hosts only in Orchestrator. + orch_hosts_only = [ + merge_dicts( + { + 'ceph_version': '', + 'services': [], + 'sources': { + 'ceph': False, + 'orchestrator': True + } + }, orch_hosts_map[hostname]) for hostname in orch_hosts_map + ] + hosts.extend(orch_hosts_only) + return hosts + + +def get_hosts(sources=None): + """ + Get hosts from various sources. + """ + from_ceph = True + from_orchestrator = True + if sources: + _sources = sources.split(',') + from_ceph = 'ceph' in _sources + from_orchestrator = 'orchestrator' in _sources + + ceph_hosts = [] + if from_ceph: + ceph_hosts = [ + merge_dicts( + server, { + 'addr': '', + 'labels': [], + 'sources': { + 'ceph': True, + 'orchestrator': False + }, + 'status': '' + }) for server in mgr.list_servers() + ] + if from_orchestrator: + orch = OrchClient.instance() + if orch.available(): + return merge_hosts_by_hostname(ceph_hosts, orch.hosts.list()) + return ceph_hosts + + +def get_host(hostname: str) -> Dict: + """ + Get a specific host from Ceph or Orchestrator (if available). + :param hostname: The name of the host to fetch. + :raises: cherrypy.HTTPError: If host not found. + """ + for host in get_hosts(): + if host['hostname'] == hostname: + return host + raise cherrypy.HTTPError(404) + + +def get_device_osd_map(): + """Get mappings from inventory devices to OSD IDs. + + :return: Returns a dictionary containing mappings. Note one device might + shared between multiple OSDs. + e.g. { + 'node1': { + 'nvme0n1': [0, 1], + 'vdc': [0], + 'vdb': [1] + }, + 'node2': { + 'vdc': [2] + } + } + :rtype: dict + """ + result: dict = {} + for osd_id, osd_metadata in mgr.get('osd_metadata').items(): + hostname = osd_metadata.get('hostname') + devices = osd_metadata.get('devices') + if not hostname or not devices: + continue + if hostname not in result: + result[hostname] = {} + # for OSD contains multiple devices, devices is in `sda,sdb` + for device in devices.split(','): + if device not in result[hostname]: + result[hostname][device] = [int(osd_id)] + else: + result[hostname][device].append(int(osd_id)) + return result + + +def get_inventories(hosts: Optional[List[str]] = None, + refresh: Optional[bool] = None) -> List[dict]: + """Get inventories from the Orchestrator and link devices with OSD IDs. + + :param hosts: Hostnames to query. + :param refresh: Ask the Orchestrator to refresh the inventories. Note the this is an + asynchronous operation, the updated version of inventories need to + be re-qeuried later. + :return: Returns list of inventory. + :rtype: list + """ + do_refresh = False + if refresh is not None: + do_refresh = str_to_bool(refresh) + orch = OrchClient.instance() + inventory_hosts = [host.to_json() + for host in orch.inventory.list(hosts=hosts, refresh=do_refresh)] + device_osd_map = get_device_osd_map() + for inventory_host in inventory_hosts: + host_osds = device_osd_map.get(inventory_host['name']) + for device in inventory_host['devices']: + if host_osds: # pragma: no cover + dev_name = os.path.basename(device['path']) + device['osd_ids'] = sorted(host_osds.get(dev_name, [])) + else: + device['osd_ids'] = [] + return inventory_hosts + + +@allow_empty_body +def add_host(hostname: str, addr: Optional[str] = None, + labels: Optional[List[str]] = None, + status: Optional[str] = None): + orch_client = OrchClient.instance() + host = Host() + host.check_orchestrator_host_op(orch_client, hostname) + orch_client.hosts.add(hostname, addr, labels) + if status == 'maintenance': + orch_client.hosts.enter_maintenance(hostname) + + +@APIRouter('/host', Scope.HOSTS) +@APIDoc("Get Host Details", "Host") +class Host(RESTController): + @EndpointDoc("List Host Specifications", + parameters={ + 'sources': (str, 'Host Sources'), + 'facts': (bool, 'Host Facts') + }, + responses={200: LIST_HOST_SCHEMA}) + @RESTController.MethodMap(version=APIVersion(1, 1)) + def list(self, sources=None, facts=False): + hosts = get_hosts(sources) + orch = OrchClient.instance() + if str_to_bool(facts): + if orch.available(): + if not orch.get_missing_features(['get_facts']): + hosts_facts = orch.hosts.get_facts() + return merge_list_of_dicts_by_key(hosts, hosts_facts, 'hostname') + + raise DashboardException( + code='invalid_orchestrator_backend', # pragma: no cover + msg="Please enable the cephadm orchestrator backend " + "(try `ceph orch set backend cephadm`)", + component='orchestrator', + http_status_code=400) + + raise DashboardException(code='orchestrator_status_unavailable', # pragma: no cover + msg="Please configure and enable the orchestrator if you " + "really want to gather facts from hosts", + component='orchestrator', + http_status_code=400) + return hosts + + @raise_if_no_orchestrator([OrchFeature.HOST_LIST, OrchFeature.HOST_ADD]) + @handle_orchestrator_error('host') + @host_task('add', {'hostname': '{hostname}'}) + @EndpointDoc('', + parameters={ + 'hostname': (str, 'Hostname'), + 'addr': (str, 'Network Address'), + 'labels': ([str], 'Host Labels'), + 'status': (str, 'Host Status'), + }, + responses={200: None, 204: None}) + @RESTController.MethodMap(version=APIVersion.EXPERIMENTAL) + def create(self, hostname: str, + addr: Optional[str] = None, + labels: Optional[List[str]] = None, + status: Optional[str] = None): # pragma: no cover - requires realtime env + add_host(hostname, addr, labels, status) + + @raise_if_no_orchestrator([OrchFeature.HOST_LIST, OrchFeature.HOST_REMOVE]) + @handle_orchestrator_error('host') + @host_task('remove', {'hostname': '{hostname}'}) + @allow_empty_body + def delete(self, hostname): # pragma: no cover - requires realtime env + orch_client = OrchClient.instance() + self.check_orchestrator_host_op(orch_client, hostname, False) + orch_client.hosts.remove(hostname) + + def check_orchestrator_host_op(self, orch_client, hostname, add=True): # pragma:no cover + """Check if we can adding or removing a host with orchestrator + + :param orch_client: Orchestrator client + :param add: True for adding host operation, False for removing host + :raise DashboardException + """ + host = orch_client.hosts.get(hostname) + if add and host: + raise DashboardException( + code='orchestrator_add_existed_host', + msg='{} is already in orchestrator'.format(hostname), + component='orchestrator') + if not add and not host: + raise DashboardException( + code='orchestrator_remove_nonexistent_host', + msg='Remove a non-existent host {} from orchestrator'.format(hostname), + component='orchestrator') + + @RESTController.Resource('GET') + def devices(self, hostname): + # (str) -> List + return CephService.get_devices_by_host(hostname) + + @RESTController.Resource('GET') + def smart(self, hostname): + # type: (str) -> dict + return CephService.get_smart_data_by_host(hostname) + + @RESTController.Resource('GET') + @raise_if_no_orchestrator([OrchFeature.DEVICE_LIST]) + @handle_orchestrator_error('host') + @EndpointDoc('Get inventory of a host', + parameters={ + 'hostname': (str, 'Hostname'), + 'refresh': (str, 'Trigger asynchronous refresh'), + }, + responses={200: INVENTORY_SCHEMA}) + def inventory(self, hostname, refresh=None): + inventory = get_inventories([hostname], refresh) + if inventory: + return inventory[0] + return {} + + @RESTController.Resource('POST') + @UpdatePermission + @raise_if_no_orchestrator([OrchFeature.DEVICE_BLINK_LIGHT]) + @handle_orchestrator_error('host') + @host_task('identify_device', ['{hostname}', '{device}'], wait_for=2.0) + def identify_device(self, hostname, device, duration): + # type: (str, str, int) -> None + """ + Identify a device by switching on the device light for N seconds. + :param hostname: The hostname of the device to process. + :param device: The device identifier to process, e.g. ``/dev/dm-0`` or + ``ABC1234DEF567-1R1234_ABC8DE0Q``. + :param duration: The duration in seconds how long the LED should flash. + """ + orch = OrchClient.instance() + TaskManager.current_task().set_progress(0) + orch.blink_device_light(hostname, device, 'ident', True) + for i in range(int(duration)): + percentage = int(round(i / float(duration) * 100)) + TaskManager.current_task().set_progress(percentage) + time.sleep(1) + orch.blink_device_light(hostname, device, 'ident', False) + TaskManager.current_task().set_progress(100) + + @RESTController.Resource('GET') + @raise_if_no_orchestrator([OrchFeature.DAEMON_LIST]) + def daemons(self, hostname: str) -> List[dict]: + orch = OrchClient.instance() + daemons = orch.services.list_daemons(hostname=hostname) + return [d.to_dict() for d in daemons] + + @handle_orchestrator_error('host') + def get(self, hostname: str) -> Dict: + """ + Get the specified host. + :raises: cherrypy.HTTPError: If host not found. + """ + return get_host(hostname) + + @raise_if_no_orchestrator([OrchFeature.HOST_LABEL_ADD, + OrchFeature.HOST_LABEL_REMOVE, + OrchFeature.HOST_MAINTENANCE_ENTER, + OrchFeature.HOST_MAINTENANCE_EXIT, + OrchFeature.HOST_DRAIN]) + @handle_orchestrator_error('host') + @EndpointDoc('', + parameters={ + 'hostname': (str, 'Hostname'), + 'update_labels': (bool, 'Update Labels'), + 'labels': ([str], 'Host Labels'), + 'maintenance': (bool, 'Enter/Exit Maintenance'), + 'force': (bool, 'Force Enter Maintenance'), + 'drain': (bool, 'Drain Host') + }, + responses={200: None, 204: None}) + @RESTController.MethodMap(version=APIVersion.EXPERIMENTAL) + def set(self, hostname: str, update_labels: bool = False, + labels: List[str] = None, maintenance: bool = False, + force: bool = False, drain: bool = False): + """ + Update the specified host. + Note, this is only supported when Ceph Orchestrator is enabled. + :param hostname: The name of the host to be processed. + :param update_labels: To update the labels. + :param labels: List of labels. + :param maintenance: Enter/Exit maintenance mode. + :param force: Force enter maintenance mode. + :param drain: Drain host + """ + orch = OrchClient.instance() + host = get_host(hostname) + + if maintenance: + status = host['status'] + if status != 'maintenance': + orch.hosts.enter_maintenance(hostname, force) + + if status == 'maintenance': + orch.hosts.exit_maintenance(hostname) + + if drain: + orch.hosts.drain(hostname) + + if update_labels: + # only allow List[str] type for labels + if not isinstance(labels, list): + raise DashboardException( + msg='Expected list of labels. Please check API documentation.', + http_status_code=400, + component='orchestrator') + current_labels = set(host['labels']) + # Remove labels. + remove_labels = list(current_labels.difference(set(labels))) + for label in remove_labels: + orch.hosts.remove_label(hostname, label) + # Add labels. + add_labels = list(set(labels).difference(current_labels)) + for label in add_labels: + orch.hosts.add_label(hostname, label) + + +@UIRouter('/host', Scope.HOSTS) +class HostUi(BaseController): + @Endpoint('GET') + @ReadPermission + @handle_orchestrator_error('host') + def labels(self) -> List[str]: + """ + Get all host labels. + Note, host labels are only supported when Ceph Orchestrator is enabled. + If Ceph Orchestrator is not enabled, an empty list is returned. + :return: A list of all host labels. + """ + labels = [] + orch = OrchClient.instance() + if orch.available(): + for host in orch.hosts.list(): + labels.extend(host.labels) + labels.sort() + return list(set(labels)) # Filter duplicate labels. + + @Endpoint('GET') + @ReadPermission + @raise_if_no_orchestrator([OrchFeature.DEVICE_LIST]) + @handle_orchestrator_error('host') + def inventory(self, refresh=None): + return get_inventories(None, refresh) |