Adding upstream version 16.2.11+ds.upstream/16.2.11+dsupstream

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

41 files changed, 5490 insertions, 0 deletions

diff --git a/src/pybind/mgr/rook/.gitignore b/src/pybind/mgr/rook/.gitignore
new file mode 100644
index 000000000..211c13153
--- /dev/null
+++ b/src/pybind/mgr/rook/.gitignore
@@ -0,0 +1 @@
+rook_client
diff --git a/src/pybind/mgr/rook/CMakeLists.txt b/src/pybind/mgr/rook/CMakeLists.txt
new file mode 100644
index 000000000..206e165e6
--- /dev/null
+++ b/src/pybind/mgr/rook/CMakeLists.txt
@@ -0,0 +1,15 @@
+include(ExternalProject)
+
+ExternalProject_Add(mgr-rook-client
+  SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/rook-client-python/rook_client"
+  # use INSTALL_DIR for destination dir
+  INSTALL_DIR "${CMAKE_CURRENT_SOURCE_DIR}/rook_client"
+  CONFIGURE_COMMAND ""
+  BUILD_COMMAND ${CMAKE_COMMAND} -E make_directory <INSTALL_DIR>
+  COMMAND       ${CMAKE_COMMAND} -E copy_directory <SOURCE_DIR>/ceph <INSTALL_DIR>/ceph
+  COMMAND       ${CMAKE_COMMAND} -E copy <SOURCE_DIR>/__init__.py <INSTALL_DIR>
+  COMMAND       ${CMAKE_COMMAND} -E copy <SOURCE_DIR>/_helper.py <INSTALL_DIR>
+  BUILD_BYPRODUCTS "<INSTALL_DIR>/__init__.py"
+  INSTALL_COMMAND "")
+
+add_dependencies(ceph-mgr mgr-rook-client)
diff --git a/src/pybind/mgr/rook/__init__.py b/src/pybind/mgr/rook/__init__.py
new file mode 100644
index 000000000..b9f08fc80
--- /dev/null
+++ b/src/pybind/mgr/rook/__init__.py
@@ -0,0 +1,2 @@
+
+from .module import RookOrchestrator
diff --git a/src/pybind/mgr/rook/generate_rook_ceph_client.sh b/src/pybind/mgr/rook/generate_rook_ceph_client.sh
new file mode 100755
index 000000000..c9ad15ce0
--- /dev/null
+++ b/src/pybind/mgr/rook/generate_rook_ceph_client.sh
@@ -0,0 +1,14 @@
+#!/bin/sh
+
+set -e
+
+script_location="$(dirname "$(readlink -f "$0")")"
+cd "$script_location"
+
+rm -rf rook_client
+
+
+cp -r ./rook-client-python/rook_client .
+rm -rf rook_client/cassandra
+rm -rf rook_client/edgefs
+rm -rf rook_client/tests
diff --git a/src/pybind/mgr/rook/module.py b/src/pybind/mgr/rook/module.py
new file mode 100644
index 000000000..70512567a
--- /dev/null
+++ b/src/pybind/mgr/rook/module.py
@@ -0,0 +1,512 @@
+import threading
+import functools
+import os
+import json
+
+from ceph.deployment import inventory
+from ceph.deployment.service_spec import ServiceSpec, NFSServiceSpec, RGWSpec, PlacementSpec
+from ceph.utils import datetime_now
+
+from typing import List, Dict, Optional, Callable, Any, TypeVar, Tuple
+
+try:
+    from ceph.deployment.drive_group import DriveGroupSpec
+except ImportError:
+    pass  # just for type checking
+
+try:
+    from kubernetes import client, config
+    from kubernetes.client.rest import ApiException
+
+    kubernetes_imported = True
+
+    # https://github.com/kubernetes-client/python/issues/895
+    from kubernetes.client.models.v1_container_image import V1ContainerImage
+    def names(self: Any, names: Any) -> None:
+        self._names = names
+    V1ContainerImage.names = V1ContainerImage.names.setter(names)
+
+except ImportError:
+    kubernetes_imported = False
+    client = None
+    config = None
+
+from mgr_module import MgrModule, Option
+import orchestrator
+from orchestrator import handle_orch_error, OrchResult, raise_if_exception
+
+from .rook_cluster import RookCluster
+
+T = TypeVar('T')
+FuncT = TypeVar('FuncT', bound=Callable)
+ServiceSpecT = TypeVar('ServiceSpecT', bound=ServiceSpec)
+
+
+
+class RookEnv(object):
+    def __init__(self) -> None:
+        # POD_NAMESPACE already exist for Rook 0.9
+        self.namespace = os.environ.get('POD_NAMESPACE', 'rook-ceph')
+
+        # ROOK_CEPH_CLUSTER_CRD_NAME is new is Rook 1.0
+        self.cluster_name = os.environ.get('ROOK_CEPH_CLUSTER_CRD_NAME', self.namespace)
+
+        self.operator_namespace = os.environ.get('ROOK_OPERATOR_NAMESPACE', self.namespace)
+        self.crd_version = os.environ.get('ROOK_CEPH_CLUSTER_CRD_VERSION', 'v1')
+        self.api_name = "ceph.rook.io/" + self.crd_version
+
+    def api_version_match(self) -> bool:
+        return self.crd_version == 'v1'
+
+    def has_namespace(self) -> bool:
+        return 'POD_NAMESPACE' in os.environ
+
+
+class RookOrchestrator(MgrModule, orchestrator.Orchestrator):
+    """
+    Writes are a two-phase thing, firstly sending
+    the write to the k8s API (fast) and then waiting
+    for the corresponding change to appear in the
+    Ceph cluster (slow)
+
+    Right now, we are calling the k8s API synchronously.
+    """
+
+    MODULE_OPTIONS: List[Option] = [
+        # TODO: configure k8s API addr instead of assuming local
+    ]
+
+    @staticmethod
+    def can_run() -> Tuple[bool, str]:
+        if not kubernetes_imported:
+            return False, "`kubernetes` python module not found"
+        if not RookEnv().api_version_match():
+            return False, "Rook version unsupported."
+        return True, ''
+
+    def available(self) -> Tuple[bool, str, Dict[str, Any]]:
+        if not kubernetes_imported:
+            return False, "`kubernetes` python module not found", {}
+        elif not self._rook_env.has_namespace():
+            return False, "ceph-mgr not running in Rook cluster", {}
+
+        try:
+            self.k8s.list_namespaced_pod(self._rook_env.namespace)
+        except ApiException as e:
+            return False, "Cannot reach Kubernetes API: {}".format(e), {}
+        else:
+            return True, "", {}
+
+    def __init__(self, *args: Any, **kwargs: Any) -> None:
+        super(RookOrchestrator, self).__init__(*args, **kwargs)
+
+        self._initialized = threading.Event()
+        self._k8s_CoreV1_api: Optional[client.CoreV1Api] = None
+        self._k8s_BatchV1_api: Optional[client.BatchV1Api] = None
+        self._rook_cluster: Optional[RookCluster] = None
+        self._rook_env = RookEnv()
+
+        self._shutdown = threading.Event()
+
+    def shutdown(self) -> None:
+        self._shutdown.set()
+
+    @property
+    def k8s(self):
+        # type: () -> client.CoreV1Api
+        self._initialized.wait()
+        assert self._k8s_CoreV1_api is not None
+        return self._k8s_CoreV1_api
+
+    @property
+    def rook_cluster(self):
+        # type: () -> RookCluster
+        self._initialized.wait()
+        assert self._rook_cluster is not None
+        return self._rook_cluster
+
+    def serve(self) -> None:
+        # For deployed clusters, we should always be running inside
+        # a Rook cluster.  For development convenience, also support
+        # running outside (reading ~/.kube config)
+
+        if self._rook_env.has_namespace():
+            config.load_incluster_config()
+        else:
+            self.log.warning("DEVELOPMENT ONLY: Reading kube config from ~")
+            config.load_kube_config()
+
+            # So that I can do port forwarding from my workstation - jcsp
+            from kubernetes.client import configuration
+            configuration.verify_ssl = False
+
+        self._k8s_CoreV1_api = client.CoreV1Api()
+        self._k8s_BatchV1_api = client.BatchV1Api()
+
+        try:
+            # XXX mystery hack -- I need to do an API call from
+            # this context, or subsequent API usage from handle_command
+            # fails with SSLError('bad handshake').  Suspect some kind of
+            # thread context setup in SSL lib?
+            self._k8s_CoreV1_api.list_namespaced_pod(self._rook_env.namespace)
+        except ApiException:
+            # Ignore here to make self.available() fail with a proper error message
+            pass
+
+        self._rook_cluster = RookCluster(
+            self._k8s_CoreV1_api,
+            self._k8s_BatchV1_api,
+            self._rook_env)
+
+        self._initialized.set()
+
+        while not self._shutdown.is_set():
+            self._shutdown.wait(5)
+
+    @handle_orch_error
+    def get_inventory(self, host_filter: Optional[orchestrator.InventoryFilter] = None, refresh: bool = False) -> List[orchestrator.InventoryHost]:
+        host_list = None
+        if host_filter and host_filter.hosts:
+            # Explicit host list
+            host_list = host_filter.hosts
+        elif host_filter and host_filter.labels:
+            # TODO: query k8s API to resolve to host list, and pass
+            # it into RookCluster.get_discovered_devices
+            raise NotImplementedError()
+
+        discovered_devs = self.rook_cluster.get_discovered_devices(host_list)
+
+        result = []
+        for host_name, host_devs in discovered_devs.items():
+            devs = []
+            for d in host_devs:
+                if 'cephVolumeData' in d and d['cephVolumeData']:
+                    devs.append(inventory.Device.from_json(json.loads(d['cephVolumeData'])))
+                else:
+                    devs.append(inventory.Device(
+                        path = '/dev/' + d['name'],
+                        sys_api = dict(
+                            rotational = '1' if d['rotational'] else '0',
+                            size = d['size']
+                        ),
+                        available = False,
+                        rejected_reasons=['device data coming from ceph-volume not provided'],
+                    ))
+
+            result.append(orchestrator.InventoryHost(host_name, inventory.Devices(devs)))
+
+        return result
+
+    @handle_orch_error
+    def get_hosts(self):
+        # type: () -> List[orchestrator.HostSpec]
+        return [orchestrator.HostSpec(n) for n in self.rook_cluster.get_node_names()]
+
+    @handle_orch_error
+    def describe_service(self,
+                         service_type: Optional[str] = None,
+                         service_name: Optional[str] = None,
+                         refresh: bool = False) -> List[orchestrator.ServiceDescription]:
+        now = datetime_now()
+
+        # CephCluster
+        cl = self.rook_cluster.rook_api_get(
+            "cephclusters/{0}".format(self.rook_cluster.rook_env.cluster_name))
+        self.log.debug('CephCluster %s' % cl)
+        image_name = cl['spec'].get('cephVersion', {}).get('image', None)
+        num_nodes = len(self.rook_cluster.get_node_names())
+
+        spec = {}
+        if service_type == 'mon' or service_type is None:
+            spec['mon'] = orchestrator.ServiceDescription(
+                spec=ServiceSpec(
+                    'mon',
+                    placement=PlacementSpec(
+                        count=cl['spec'].get('mon', {}).get('count', 1),
+                    ),
+                ),
+                size=cl['spec'].get('mon', {}).get('count', 1),
+                container_image_name=image_name,
+                last_refresh=now,
+            )
+        if service_type == 'mgr' or service_type is None:
+            spec['mgr'] = orchestrator.ServiceDescription(
+                spec=ServiceSpec(
+                    'mgr',
+                    placement=PlacementSpec.from_string('count:1'),
+                ),
+                size=1,
+                container_image_name=image_name,
+                last_refresh=now,
+            )
+        if not cl['spec'].get('crashCollector', {}).get('disable', False):
+            spec['crash'] = orchestrator.ServiceDescription(
+                spec=ServiceSpec(
+                    'crash',
+                    placement=PlacementSpec.from_string('*'),
+                ),
+                size=num_nodes,
+                container_image_name=image_name,
+                last_refresh=now,
+            )
+
+        if service_type == 'mds' or service_type is None:
+            # CephFilesystems
+            all_fs = self.rook_cluster.rook_api_get(
+                "cephfilesystems/")
+            self.log.debug('CephFilesystems %s' % all_fs)
+            for fs in all_fs.get('items', []):
+                svc = 'mds.' + fs['metadata']['name']
+                if svc in spec:
+                    continue
+                # FIXME: we are conflating active (+ standby) with count
+                active = fs['spec'].get('metadataServer', {}).get('activeCount', 1)
+                total_mds = active
+                if fs['spec'].get('metadataServer', {}).get('activeStandby', False):
+                    total_mds = active * 2
+                    spec[svc] = orchestrator.ServiceDescription(
+                        spec=ServiceSpec(
+                            service_type='mds',
+                            service_id=fs['metadata']['name'],
+                            placement=PlacementSpec(count=active),
+                        ),
+                        size=total_mds,
+                        container_image_name=image_name,
+                        last_refresh=now,
+                    )
+
+        if service_type == 'rgw' or service_type is None:
+            # CephObjectstores
+            all_zones = self.rook_cluster.rook_api_get(
+                "cephobjectstores/")
+            self.log.debug('CephObjectstores %s' % all_zones)
+            for zone in all_zones.get('items', []):
+                rgw_realm = zone['metadata']['name']
+                rgw_zone = rgw_realm
+                svc = 'rgw.' + rgw_realm + '.' + rgw_zone
+                if svc in spec:
+                    continue
+                active = zone['spec']['gateway']['instances'];
+                if 'securePort' in zone['spec']['gateway']:
+                    ssl = True
+                    port = zone['spec']['gateway']['securePort']
+                else:
+                    ssl = False
+                    port = zone['spec']['gateway']['port'] or 80
+                spec[svc] = orchestrator.ServiceDescription(
+                    spec=RGWSpec(
+                        service_id=rgw_realm + '.' + rgw_zone,
+                        rgw_realm=rgw_realm,
+                        rgw_zone=rgw_zone,
+                        ssl=ssl,
+                        rgw_frontend_port=port,
+                        placement=PlacementSpec(count=active),
+                    ),
+                    size=active,
+                    container_image_name=image_name,
+                    last_refresh=now,
+                )
+
+        if service_type == 'nfs' or service_type is None:
+            # CephNFSes
+            all_nfs = self.rook_cluster.rook_api_get(
+                "cephnfses/")
+            self.log.warning('CephNFS %s' % all_nfs)
+            for nfs in all_nfs.get('items', []):
+                nfs_name = nfs['metadata']['name']
+                svc = 'nfs.' + nfs_name
+                if svc in spec:
+                    continue
+                active = nfs['spec'].get('server', {}).get('active')
+                spec[svc] = orchestrator.ServiceDescription(
+                    spec=NFSServiceSpec(
+                        service_id=nfs_name,
+                        placement=PlacementSpec(count=active),
+                    ),
+                    size=active,
+                    last_refresh=now,
+                )
+
+        for dd in self._list_daemons():
+            if dd.service_name() not in spec:
+                continue
+            service = spec[dd.service_name()]
+            service.running += 1
+            if not service.container_image_id:
+                service.container_image_id = dd.container_image_id
+            if not service.container_image_name:
+                service.container_image_name = dd.container_image_name
+            if service.last_refresh is None or not dd.last_refresh or dd.last_refresh < service.last_refresh:
+                service.last_refresh = dd.last_refresh
+            if service.created is None or dd.created is None or dd.created < service.created:
+                service.created = dd.created
+
+        return [v for k, v in spec.items()]
+
+    @handle_orch_error
+    def list_daemons(self,
+                     service_name: Optional[str] = None,
+                     daemon_type: Optional[str] = None,
+                     daemon_id: Optional[str] = None,
+                     host: Optional[str] = None,
+                     refresh: bool = False) -> List[orchestrator.DaemonDescription]:
+        return self._list_daemons(service_name=service_name,
+                                  daemon_type=daemon_type,
+                                  daemon_id=daemon_id,
+                                  host=host,
+                                  refresh=refresh)
+
+    def _list_daemons(self,
+                      service_name: Optional[str] = None,
+                      daemon_type: Optional[str] = None,
+                      daemon_id: Optional[str] = None,
+                      host: Optional[str] = None,
+                      refresh: bool = False) -> List[orchestrator.DaemonDescription]:
+        pods = self.rook_cluster.describe_pods(daemon_type, daemon_id, host)
+        self.log.debug('pods %s' % pods)
+        result = []
+        for p in pods:
+            sd = orchestrator.DaemonDescription()
+            sd.hostname = p['hostname']
+            sd.daemon_type = p['labels']['app'].replace('rook-ceph-', '')
+            status = {
+                'Pending': orchestrator.DaemonDescriptionStatus.starting,
+                'Running': orchestrator.DaemonDescriptionStatus.running,
+                'Succeeded': orchestrator.DaemonDescriptionStatus.stopped,
+                'Failed': orchestrator.DaemonDescriptionStatus.error,
+                'Unknown': orchestrator.DaemonDescriptionStatus.unknown,
+            }[p['phase']]
+            sd.status = status
+
+            if 'ceph_daemon_id' in p['labels']:
+                sd.daemon_id = p['labels']['ceph_daemon_id']
+            elif 'ceph-osd-id' in p['labels']:
+                sd.daemon_id = p['labels']['ceph-osd-id']
+            else:
+                # Unknown type -- skip it
+                continue
+
+            if service_name is not None and service_name != sd.service_name():
+                continue
+            sd.container_image_name = p['container_image_name']
+            sd.container_image_id = p['container_image_id']
+            sd.created = p['created']
+            sd.last_configured = p['created']
+            sd.last_deployed = p['created']
+            sd.started = p['started']
+            sd.last_refresh = p['refreshed']
+            result.append(sd)
+
+        return result
+
+    @handle_orch_error
+    def remove_service(self, service_name: str, force: bool = False) -> str:
+        service_type, service_name = service_name.split('.', 1)
+        if service_type == 'mds':
+            return self.rook_cluster.rm_service('cephfilesystems', service_name)
+        elif service_type == 'rgw':
+            return self.rook_cluster.rm_service('cephobjectstores', service_name)
+        elif service_type == 'nfs':
+            return self.rook_cluster.rm_service('cephnfses', service_name)
+        else:
+            raise orchestrator.OrchestratorError(f'Service type {service_type} not supported')
+
+    @handle_orch_error
+    def apply_mon(self, spec):
+        # type: (ServiceSpec) -> str
+        if spec.placement.hosts or spec.placement.label:
+            raise RuntimeError("Host list or label is not supported by rook.")
+
+        return self.rook_cluster.update_mon_count(spec.placement.count)
+
+    @handle_orch_error
+    def apply_mds(self, spec):
+        # type: (ServiceSpec) -> str
+        return self.rook_cluster.apply_filesystem(spec)
+
+    @handle_orch_error
+    def apply_rgw(self, spec):
+        # type: (RGWSpec) -> str
+        return self.rook_cluster.apply_objectstore(spec)
+
+    @handle_orch_error
+    def apply_nfs(self, spec):
+        # type: (NFSServiceSpec) -> str
+        return self.rook_cluster.apply_nfsgw(spec)
+
+    @handle_orch_error
+    def remove_daemons(self, names: List[str]) -> List[str]:
+        return self.rook_cluster.remove_pods(names)
+
+    @handle_orch_error
+    def create_osds(self, drive_group):
+        # type: (DriveGroupSpec) -> str
+        """ Creates OSDs from a drive group specification.
+
+        $: ceph orch osd create -i <dg.file>
+
+        The drivegroup file must only contain one spec at a time.
+        """
+
+        targets = []  # type: List[str]
+        if drive_group.data_devices and drive_group.data_devices.paths:
+            targets += [d.path for d in drive_group.data_devices.paths]
+        if drive_group.data_directories:
+            targets += drive_group.data_directories
+
+        all_hosts = raise_if_exception(self.get_hosts())
+
+        matching_hosts = drive_group.placement.filter_matching_hosts(lambda label=None, as_hostspec=None: all_hosts)
+
+        assert len(matching_hosts) == 1
+
+        if not self.rook_cluster.node_exists(matching_hosts[0]):
+            raise RuntimeError("Node '{0}' is not in the Kubernetes "
+                               "cluster".format(matching_hosts))
+
+        # Validate whether cluster CRD can accept individual OSD
+        # creations (i.e. not useAllDevices)
+        if not self.rook_cluster.can_create_osd():
+            raise RuntimeError("Rook cluster configuration does not "
+                               "support OSD creation.")
+
+        return self.rook_cluster.add_osds(drive_group, matching_hosts)
+
+        # TODO: this was the code to update the progress reference:
+        """
+        @handle_orch_error
+        def has_osds(matching_hosts: List[str]) -> bool:
+
+            # Find OSD pods on this host
+            pod_osd_ids = set()
+            pods = self.k8s.list_namespaced_pod(self._rook_env.namespace,
+                                                label_selector="rook_cluster={},app=rook-ceph-osd".format(self._rook_env.cluster_name),
+                                                field_selector="spec.nodeName={0}".format(
+                                                    matching_hosts[0]
+                                                )).items
+            for p in pods:
+                pod_osd_ids.add(int(p.metadata.labels['ceph-osd-id']))
+
+            self.log.debug('pod_osd_ids={0}'.format(pod_osd_ids))
+
+            found = []
+            osdmap = self.get("osd_map")
+            for osd in osdmap['osds']:
+                osd_id = osd['osd']
+                if osd_id not in pod_osd_ids:
+                    continue
+
+                metadata = self.get_metadata('osd', "%s" % osd_id)
+                if metadata and metadata['devices'] in targets:
+                    found.append(osd_id)
+                else:
+                    self.log.info("ignoring osd {0} {1}".format(
+                        osd_id, metadata['devices'] if metadata else 'DNE'
+                    ))
+
+            return found is not None
+        """
+
+    @handle_orch_error
+    def blink_device_light(self, ident_fault: str, on: bool, locs: List[orchestrator.DeviceLightLoc]) -> List[str]:
+        return self.rook_cluster.blink_light(ident_fault, on, locs)
diff --git a/src/pybind/mgr/rook/requirements.txt b/src/pybind/mgr/rook/requirements.txt
new file mode 100644
index 000000000..378de08e6
--- /dev/null
+++ b/src/pybind/mgr/rook/requirements.txt
@@ -0,0 +1,2 @@
+kubernetes
+jsonpatch
\ No newline at end of file
diff --git a/src/pybind/mgr/rook/rook-client-python/.github/workflows/generate.yml b/src/pybind/mgr/rook/rook-client-python/.github/workflows/generate.yml
new file mode 100644
index 000000000..97ac5a222
--- /dev/null
+++ b/src/pybind/mgr/rook/rook-client-python/.github/workflows/generate.yml
@@ -0,0 +1,21 @@
+name: Python application
+
+on: [push, pull_request]
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python 3.8
+      uses: actions/setup-python@v1
+      with:
+        python-version: 3.8
+    - name: clone rook
+      run: |
+        git clone --depth 1 https://github.com/rook/rook.git
+    - name: Run generate.sh
+      run: |
+        ./generate.sh $PWD/rook
diff --git a/src/pybind/mgr/rook/rook-client-python/.gitignore b/src/pybind/mgr/rook/rook-client-python/.gitignore
new file mode 100644
index 000000000..83ace910c
--- /dev/null
+++ b/src/pybind/mgr/rook/rook-client-python/.gitignore
@@ -0,0 +1,11 @@
+.coverage
+.eggs/
+.idea/
+.mypy_cache/
+rook_client.egg-info/
+venv/
+downloads/
+__pycache__
+*.pyc
+rook_client/tests/test_README.py
+.tox
diff --git a/src/pybind/mgr/rook/rook-client-python/README.md b/src/pybind/mgr/rook/rook-client-python/README.md
new file mode 100644
index 000000000..6f039ab3e
--- /dev/null
+++ b/src/pybind/mgr/rook/rook-client-python/README.md
@@ -0,0 +1,75 @@
+# `rook-client` Python API Classes  
+
+Automatically generated models for Rook's custom resource definitions.
+
+Right now, it supports three operators:
+
+* Ceph
+* Edgefs
+* Cassandra
+
+It is used to type check client code against the Rook API
+
+Inspired by https://github.com/kubernetes-client/python/tree/master/kubernetes/client/models
+
+Main uses case is the mgr/rook orchestrator module of the Ceph MGR
+
+## Installing `rook-client`
+
+To install the package, run:
+
+```bash
+pip install -e 'git+https://github.com/rook/rook#egg=rook-client&subdirectory=build/codegen/python'
+```
+
+
+## Example
+
+```python
+def objectstore(api_name, name, namespace, instances):
+    from rook_client.ceph import cephobjectstore as cos
+    rook_os = cos.CephObjectStore(
+        apiVersion=api_name,
+        metadata=dict(
+            name=name,
+            namespace=namespace
+        ),
+        spec=cos.Spec(
+            metadataPool=cos.MetadataPool(
+                failureDomain='host',
+                replicated=cos.Replicated(
+                    size=1
+                )
+            ),
+            dataPool=cos.DataPool(
+                failureDomain='osd',
+                replicated=cos.Replicated(
+                    size=1
+                )
+            ),
+            gateway=cos.Gateway(
+                type='s3',
+                port=80,
+                instances=instances
+            )
+        )
+    )
+    return rook_os.to_json()
+```
+
+## Demo
+
+![](rook-python-client-demo.gif)
+
+## Regenerate
+
+To re-generate the python files, run 
+
+```bash
+./generate.sh
+```
+
+## Rational
+
+The Rook CRDs are evolving over time and to verify that clients make use of the correct definitions, 
+this project provides generated API classes to let users write correct clients easily.
\ No newline at end of file
diff --git a/src/pybind/mgr/rook/rook-client-python/conftest.py b/src/pybind/mgr/rook/rook-client-python/conftest.py
new file mode 100644
index 000000000..77d6eb358
--- /dev/null
+++ b/src/pybind/mgr/rook/rook-client-python/conftest.py
@@ -0,0 +1,11 @@
+import pytest
+
+def pytest_addoption(parser):
+    parser.addoption(
+        "--crd_base", action="store", help="base path to the crd yamls"
+    )
+
+
+@pytest.fixture
+def crd_base(request):
+    return request.config.getoption("--crd_base")
diff --git a/src/pybind/mgr/rook/rook-client-python/generate.sh b/src/pybind/mgr/rook/rook-client-python/generate.sh
new file mode 100755
index 000000000..5aa189276
--- /dev/null
+++ b/src/pybind/mgr/rook/rook-client-python/generate.sh
@@ -0,0 +1,37 @@
+#!/bin/bash
+
+set -ex
+
+if [ -z "$1" ] ; then
+    rook_base="${GOPATH:-$HOME/go}/src/github.com/rook/rook"
+else
+    rook_base="$1"
+fi
+crd_base="$rook_base/cluster/examples/kubernetes"
+
+cd "$(dirname "$0")"
+
+if ! [ -x "$(command -v python3)" ]; then
+  echo 'Error: python3 is not installed.' >&2
+  exit 1
+fi
+
+if [ ! -d venv ]
+then
+  python3 -m venv venv
+  . venv/bin/activate
+  pip install -r requirements.txt
+else
+  . venv/bin/activate 
+fi
+
+python generate_model_classes.py "$crd_base/ceph/common.yaml" "rook_client/ceph"
+python generate_model_classes.py "$crd_base/cassandra/operator.yaml" "rook_client/cassandra"
+python generate_model_classes.py "$crd_base/edgefs/operator.yaml" "rook_client/edgefs"
+
+
+python setup.py develop
+
+tox --skip-missing-interpreters=true -- --crd_base="$crd_base"
+
+deactivate
diff --git a/src/pybind/mgr/rook/rook-client-python/generate_model_classes.py b/src/pybind/mgr/rook/rook-client-python/generate_model_classes.py
new file mode 100644
index 000000000..760dd5e23
--- /dev/null
+++ b/src/pybind/mgr/rook/rook-client-python/generate_model_classes.py
@@ -0,0 +1,298 @@
+"""
+Generate Python files containing data Python models classes for
+all properties of the all CRDs in the file
+
+**Note**: generate_model_classes.py is independent of Rook or Ceph. It can be used for all
+  CRDs.
+
+For example:
+  python3 -m venv venv
+  pip install -r requirements.txt
+  python generate_model_classes.py <crds.yaml> <output-folder>
+  python setup.py develop
+
+Usage:
+  generate_model_classes.py <crds.yaml> <output-folder>
+"""
+import os
+from abc import ABC, abstractmethod
+from collections import OrderedDict
+from typing import List, Union, Iterator, Optional
+
+import yaml
+try:
+    from dataclasses import dataclass
+except ImportError:
+    from attr import dataclass  # type: ignore
+
+header = '''"""
+This file is automatically generated.
+Do not modify.
+"""
+
+try:
+    from typing import Any, Optional, Union, List
+except ImportError:
+    pass
+
+from .._helper import _omit, CrdObject, CrdObjectList, CrdClass
+
+'''
+
+@dataclass  # type: ignore
+class CRDBase(ABC):
+    name: str
+    nullable: bool
+    required: bool
+
+    @property
+    def py_name(self):
+        return self.name
+
+    @property
+    @abstractmethod
+    def py_type(self):
+        ...
+
+    @abstractmethod
+    def flatten(self):
+        ...
+
+    def py_property(self):
+        return f"""
+@property
+def {self.py_name}(self):
+    # type: () -> {self.py_property_return_type}
+    return self._property_impl('{self.py_name}')
+
+@{self.py_name}.setter
+def {self.py_name}(self, new_val):
+    # type: ({self.py_param_type}) -> None
+    self._{self.py_name} = new_val
+    """.strip()
+
+    @property
+    def py_param(self):
+        if not self.has_default:
+            return f'{self.py_name},  # type: {self.py_param_type}'
+        return f'{self.py_name}=_omit,  # type: {self.py_param_type}'
+
+    @property
+    def has_default(self):
+        return not self.required
+
+    @property
+    def py_param_type(self):
+        return f'Optional[{self.py_type}]' if (self.nullable or not self.required) else self.py_type
+
+    @property
+    def py_property_return_type(self):
+        return f'Optional[{self.py_type}]' if (self.nullable) else self.py_type
+
+@dataclass
+class CRDAttribute(CRDBase):
+    type: str
+    default_value: str='_omit'
+
+    @property
+    def py_param(self):
+        if not self.has_default:
+            return f'{self.py_name},  # type: {self.py_param_type}'
+        return f'{self.py_name}={self.default_value},  # type: {self.py_param_type}'
+
+    @property
+    def has_default(self):
+        return not self.required or self.default_value != '_omit'
+
+    @property
+    def py_type(self):
+        return {
+            'integer': 'int',
+            'boolean': 'bool',
+            'string': 'str',
+            'object': 'Any',
+            'number': 'float',
+        }[self.type]
+
+    def flatten(self):
+        yield from ()
+
+    def toplevel(self):
+        return ''
+
+
+@dataclass
+class CRDList(CRDBase):
+    items: 'CRDClass'
+
+    @property
+    def py_name(self):
+        return self.name
+
+    @property
+    def py_type(self):
+        return self.name[0].upper() + self.name[1:] + 'List'
+
+    @property
+    def py_param_type(self):
+        inner = f'Union[List[{self.items.py_type}], CrdObjectList]'
+        return f'Optional[{inner}]' if (self.nullable or not self.required) else inner
+
+    @property
+    def py_property_return_type(self):
+        inner = f'Union[List[{self.items.py_type}], CrdObjectList]'
+        return f'Optional[{inner}]' if (self.nullable) else inner
+
+    def flatten(self):
+        yield from self.items.flatten()
+        yield self
+
+    def toplevel(self):
+        py_type = self.items.py_type
+        if py_type == 'Any':
+            py_type = 'None'
+
+        return f"""
+class {self.py_type}(CrdObjectList):
+{indent('_items_type = ' + py_type)}
+""".strip()
+
+
+@dataclass
+class CRDClass(CRDBase):
+    attrs: List[Union[CRDAttribute, 'CRDClass']]
+    base_class: str = 'CrdObject'
+
+    def toplevel(self):
+        ps = '\n\n'.join(a.py_property() for a in self.attrs)
+        return f"""class {self.py_type}({self.base_class}):
+{indent(self.py_properties())}        
+
+{indent(self.py_init())}
+
+{indent(ps)}
+""".strip()
+
+    @property
+    def sub_classes(self) -> List["CRDClass"]:
+        return [a for a in self.attrs if isinstance(a, CRDClass)]
+
+    @property
+    def py_type(self):
+        return self.name[0].upper() + self.name[1:]
+
+    def py_properties(self):
+        def a_to_tuple(a):
+            return ', '.join((f"'{a.name}'",
+                       f"'{a.py_name}'",
+                       a.py_type.replace('Any', 'object'),
+                       str(a.required),
+                       str(a.nullable)))
+
+        attrlist = ',\n'.join([f'({a_to_tuple(a)})' for a in self.attrs])
+        return f"""_properties = [\n{indent(attrlist)}\n]"""
+
+    def flatten(self) -> Iterator['CRDClass']:
+        for sub_cls in self.attrs:
+            yield from sub_cls.flatten()
+        yield self
+
+    def py_init(self):
+        sorted_attrs = sorted(self.attrs, key=lambda a: a.has_default)
+        params = '\n'.join(a.py_param for a in sorted_attrs)
+        params_set = '\n'.join(f'{a.py_name}={a.py_name},' for a in sorted_attrs)
+        return f"""
+def __init__(self,
+{indent(params, indent=4+9)}
+             ):
+    super({self.py_type}, self).__init__(
+{indent(params_set, indent=8)}
+    )
+""".strip()
+
+def indent(s, indent=4):
+    return '\n'.join(' '*indent + l for l in s.splitlines())
+
+
+def handle_property(elem_name, elem: dict, required: bool):
+    nullable = elem.get('nullable', False)
+    if 'properties' in elem:
+        ps = elem['properties']
+        required_elems = elem.get('required', [])
+        sub_props = [handle_property(k, v, k in required_elems) for k, v in ps.items()]
+        return CRDClass(elem_name, nullable, required, sub_props)
+    elif 'items' in elem:
+        item = handle_property(elem_name + 'Item', elem['items'], False)
+        return CRDList(elem_name, nullable, required, item)
+    elif 'type' in elem:
+        return CRDAttribute(elem_name, nullable, required, elem['type'])
+    elif elem == {}:
+        return CRDAttribute(elem_name, nullable, required, 'object')
+    assert False, str((elem_name, elem))
+
+
+def handle_crd(c_dict) -> Optional[CRDClass]:
+    try:
+        name = c_dict['spec']['names']['kind']
+        s = c_dict['spec']['validation']['openAPIV3Schema']
+    except (KeyError, TypeError):
+        return None
+    s['required'] = ['spec']
+    c = handle_property(name, s, True)
+    k8s_attrs = [CRDAttribute('apiVersion', False, True, 'string'),
+                 CRDAttribute('metadata', False, True, 'object'),
+                 CRDAttribute('status', False, False, 'object')]
+    return CRDClass(c.name, False, True, k8s_attrs + c.attrs, base_class='CrdClass')
+
+
+def local(yaml_filename):
+    with open(yaml_filename) as f:
+        yamls = yaml.safe_load_all(f.read())
+        for y in yamls:
+            try:
+                yield y
+            except AttributeError:
+                pass
+
+
+def remove_duplicates(items):
+    return OrderedDict.fromkeys(items).keys()
+
+
+def get_toplevels(crd):
+    elems = list(crd.flatten())
+
+    def dup_elems(l):
+        ds = set([x for x in l if l.count(x) > 1])
+        return ds
+
+    names = [t.name for t in elems]
+    for dup_name in dup_elems(names):
+        dups = set(e.toplevel() for e in elems if e.name == dup_name)
+        assert len(dups) == 1, str(dups)
+        
+    return remove_duplicates(cls.toplevel() for cls in elems)
+
+
+def main(yaml_filename, outfolder):
+    for crd in local(yaml_filename):
+        valid_crd = handle_crd(crd)
+        if valid_crd is not None:
+            try:
+                os.mkdir(outfolder)
+            except FileExistsError:
+                pass
+            open(f'{outfolder}/__init__.py', 'w').close()
+
+            with open(f'{outfolder}/{valid_crd.name.lower()}.py', 'w') as f:
+                f.write(header)
+                classes = get_toplevels(valid_crd)
+                f.write('\n\n\n'.join(classes))
+                f.write('\n')
+
+
+if __name__ == '__main__':
+    from docopt import docopt
+    args = docopt(__doc__)
+    yaml_filename = '/dev/stdin' if args["<crds.yaml>"] == '-' else args["<crds.yaml>"]
+    main(yaml_filename, args["<output-folder>"])
\ No newline at end of file
diff --git a/src/pybind/mgr/rook/rook-client-python/mypy.ini b/src/pybind/mgr/rook/rook-client-python/mypy.ini
new file mode 100644
index 000000000..cd26ff97e
--- /dev/null
+++ b/src/pybind/mgr/rook/rook-client-python/mypy.ini
@@ -0,0 +1,7 @@
+[mypy]
+strict_optional = True
+no_implicit_optional = True
+ignore_missing_imports = True
+warn_incomplete_stub = True
+check_untyped_defs = True
+show_error_context = True
diff --git a/src/pybind/mgr/rook/rook-client-python/requirements.txt b/src/pybind/mgr/rook/rook-client-python/requirements.txt
new file mode 100644
index 000000000..093d2a3db
--- /dev/null
+++ b/src/pybind/mgr/rook/rook-client-python/requirements.txt
@@ -0,0 +1,7 @@
+pytest
+requests
+pyyaml
+markdown
+-e git+https://github.com/ryneeverett/mkcodes.git#egg=mkcodes
+docopt
+tox
diff --git a/src/pybind/mgr/rook/rook-client-python/rook-python-client-demo.gif b/src/pybind/mgr/rook/rook-client-python/rook-python-client-demo.gif
new file mode 100644
index 000000000..0a9139a13
--- /dev/null
+++ b/src/pybind/mgr/rook/rook-client-python/rook-python-client-demo.gif
diff --git a/src/pybind/mgr/rook/rook-client-python/rook_client/__init__.py b/src/pybind/mgr/rook/rook-client-python/rook_client/__init__.py
new file mode 100644
index 000000000..3fa2272dd
--- /dev/null
+++ b/src/pybind/mgr/rook/rook-client-python/rook_client/__init__.py
@@ -0,0 +1 @@
+from ._helper import STRICT
\ No newline at end of file
diff --git a/src/pybind/mgr/rook/rook-client-python/rook_client/_helper.py b/src/pybind/mgr/rook/rook-client-python/rook_client/_helper.py
new file mode 100644
index 000000000..b4c7793f0
--- /dev/null
+++ b/src/pybind/mgr/rook/rook-client-python/rook_client/_helper.py
@@ -0,0 +1,108 @@
+import logging
+try:
+    from typing import List, Dict, Any, Optional
+except ImportError:
+    pass
+
+logger = logging.getLogger(__name__)
+
+# Tricking mypy to think `_omit`'s type is NoneType
+# To make us not add things like `Union[Optional[str], OmitType]`
+NoneType = type(None)
+_omit = None  # type: NoneType
+_omit = object()  # type: ignore
+
+
+# Don't add any additionalProperties to objects. Useful for completeness testing
+STRICT = False
+
+
+def _property_from_json(data, breadcrumb, name, py_name, typ, required, nullable):
+    if not required and name not in data:
+        return _omit
+    obj = data[name]
+    if nullable and obj is None:
+        return obj
+    if issubclass(typ, CrdObject) or issubclass(typ, CrdObjectList):
+        return typ.from_json(obj, breadcrumb + '.' + name)
+    return obj
+
+
+class CrdObject(object):
+    _properties = []  # type: List
+
+    def __init__(self, **kwargs):
+        for prop in self._properties:
+            setattr(self, prop[1], kwargs.pop(prop[1]))
+        if kwargs:
+            raise TypeError(
+                '{} got unexpected arguments {}'.format(self.__class__.__name__, kwargs.keys()))
+        self._additionalProperties = {}  # type: Dict[str, Any]
+
+    def _property_impl(self, name):
+        obj = getattr(self, '_' + name)
+        if obj is _omit:
+            raise AttributeError(name + ' not found')
+        return obj
+
+    def _property_to_json(self, name, py_name, typ, required, nullable):
+        obj = getattr(self, '_' + py_name)
+        if issubclass(typ, CrdObject) or issubclass(typ, CrdObjectList):
+            if nullable and obj is None:
+                return obj
+            if not required and obj is _omit:
+                return obj
+            return obj.to_json()
+        else:
+            return obj
+
+    def to_json(self):
+        # type: () -> Dict[str, Any]
+        res = {p[0]: self._property_to_json(*p) for p in self._properties}
+        res.update(self._additionalProperties)
+        return {k: v for k, v in res.items() if v is not _omit}
+
+    @classmethod
+    def from_json(cls, data, breadcrumb=''):
+        try:
+            sanitized = {
+                p[1]: _property_from_json(data, breadcrumb, *p) for p in cls._properties
+            }
+            extra = {k:v for k,v in data.items() if k not in sanitized}
+            ret = cls(**sanitized)
+            ret._additionalProperties = {} if STRICT else extra
+            return ret
+        except (TypeError, AttributeError, KeyError):
+            logger.exception(breadcrumb)
+            raise
+
+
+class CrdClass(CrdObject):
+    @classmethod
+    def from_json(cls, data, breadcrumb=''):
+        kind = data['kind']
+        if kind != cls.__name__:
+            raise ValueError("kind mismatch: {} != {}".format(kind, cls.__name__))
+        return super(CrdClass, cls).from_json(data, breadcrumb)
+
+    def to_json(self):
+        ret = super(CrdClass, self).to_json()
+        ret['kind'] = self.__class__.__name__
+        return ret
+
+
+class CrdObjectList(list):
+    # Py3: Replace `Any` with `TypeVar('T_CrdObject', bound='CrdObject')`
+    _items_type = None  # type: Optional[Any]
+
+    def to_json(self):
+        # type: () -> List
+        if self._items_type is None:
+            return self
+        return [e.to_json() for e in self]
+
+    @classmethod
+    def from_json(cls, data, breadcrumb=''):
+        if cls._items_type is None:
+            return cls(data)
+        return cls(cls._items_type.from_json(e, breadcrumb + '[]') for e in data)
diff --git a/src/pybind/mgr/rook/rook-client-python/rook_client/cassandra/__init__.py b/src/pybind/mgr/rook/rook-client-python/rook_client/cassandra/__init__.py
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/src/pybind/mgr/rook/rook-client-python/rook_client/cassandra/__init__.py
diff --git a/src/pybind/mgr/rook/rook-client-python/rook_client/cassandra/cluster.py b/src/pybind/mgr/rook/rook-client-python/rook_client/cassandra/cluster.py
new file mode 100644
index 000000000..56dc8a8fe
--- /dev/null
+++ b/src/pybind/mgr/rook/rook-client-python/rook_client/cassandra/cluster.py
@@ -0,0 +1,308 @@
+"""
+This file is automatically generated.
+Do not modify.
+"""
+
+try:
+    from typing import Any, Optional, Union, List
+except ImportError:
+    pass
+
+from .._helper import _omit, CrdObject, CrdObjectList, CrdClass
+
+class Storage(CrdObject):
+    _properties = [
+        ('volumeClaimTemplates', 'volumeClaimTemplates', object, True, False)
+    ]        
+
+    def __init__(self,
+                 volumeClaimTemplates,  # type: Any
+                 ):
+        super(Storage, self).__init__(
+            volumeClaimTemplates=volumeClaimTemplates,
+        )
+
+    @property
+    def volumeClaimTemplates(self):
+        # type: () -> Any
+        return self._property_impl('volumeClaimTemplates')
+    
+    @volumeClaimTemplates.setter
+    def volumeClaimTemplates(self, new_val):
+        # type: (Any) -> None
+        self._volumeClaimTemplates = new_val
+
+
+class Resources(CrdObject):
+    _properties = [
+        ('cassandra', 'cassandra', object, False, False),
+        ('sidecar', 'sidecar', object, False, False)
+    ]        
+
+    def __init__(self,
+                 cassandra=_omit,  # type: Optional[Any]
+                 sidecar=_omit,  # type: Optional[Any]
+                 ):
+        super(Resources, self).__init__(
+            cassandra=cassandra,
+            sidecar=sidecar,
+        )
+
+    @property
+    def cassandra(self):
+        # type: () -> Any
+        return self._property_impl('cassandra')
+    
+    @cassandra.setter
+    def cassandra(self, new_val):
+        # type: (Optional[Any]) -> None
+        self._cassandra = new_val
+    
+    @property
+    def sidecar(self):
+        # type: () -> Any
+        return self._property_impl('sidecar')
+    
+    @sidecar.setter
+    def sidecar(self, new_val):
+        # type: (Optional[Any]) -> None
+        self._sidecar = new_val
+
+
+class RacksItem(CrdObject):
+    _properties = [
+        ('name', 'name', str, True, False),
+        ('members', 'members', int, True, False),
+        ('configMapName', 'configMapName', str, False, False),
+        ('storage', 'storage', Storage, True, False),
+        ('placement', 'placement', object, False, False),
+        ('resources', 'resources', Resources, True, False),
+        ('sidecarImage', 'sidecarImage', object, False, False)
+    ]        
+
+    def __init__(self,
+                 name,  # type: str
+                 members,  # type: int
+                 storage,  # type: Storage
+                 resources,  # type: Resources
+                 configMapName=_omit,  # type: Optional[str]
+                 placement=_omit,  # type: Optional[Any]
+                 sidecarImage=_omit,  # type: Optional[Any]
+                 ):
+        super(RacksItem, self).__init__(
+            name=name,
+            members=members,
+            storage=storage,
+            resources=resources,
+            configMapName=configMapName,
+            placement=placement,
+            sidecarImage=sidecarImage,
+        )
+
+    @property
+    def name(self):
+        # type: () -> str
+        return self._property_impl('name')
+    
+    @name.setter
+    def name(self, new_val):
+        # type: (str) -> None
+        self._name = new_val
+    
+    @property
+    def members(self):
+        # type: () -> int
+        return self._property_impl('members')
+    
+    @members.setter
+    def members(self, new_val):
+        # type: (int) -> None
+        self._members = new_val
+    
+    @property
+    def configMapName(self):
+        # type: () -> str
+        return self._property_impl('configMapName')
+    
+    @configMapName.setter
+    def configMapName(self, new_val):
+        # type: (Optional[str]) -> None
+        self._configMapName = new_val
+    
+    @property
+    def storage(self):
+        # type: () -> Storage
+        return self._property_impl('storage')
+    
+    @storage.setter
+    def storage(self, new_val):
+        # type: (Storage) -> None
+        self._storage = new_val
+    
+    @property
+    def placement(self):
+        # type: () -> Any
+        return self._property_impl('placement')
+    
+    @placement.setter
+    def placement(self, new_val):
+        # type: (Optional[Any]) -> None
+        self._placement = new_val
+    
+    @property
+    def resources(self):
+        # type: () -> Resources
+        return self._property_impl('resources')
+    
+    @resources.setter
+    def resources(self, new_val):
+        # type: (Resources) -> None
+        self._resources = new_val
+    
+    @property
+    def sidecarImage(self):
+        # type: () -> Any
+        return self._property_impl('sidecarImage')
+    
+    @sidecarImage.setter
+    def sidecarImage(self, new_val):
+        # type: (Optional[Any]) -> None
+        self._sidecarImage = new_val
+
+
+class RacksList(CrdObjectList):
+    _items_type = RacksItem
+
+
+class Datacenter(CrdObject):
+    _properties = [
+        ('name', 'name', str, True, False),
+        ('racks', 'racks', RacksList, False, False)
+    ]        
+
+    def __init__(self,
+                 name,  # type: str
+                 racks=_omit,  # type: Optional[Union[List[RacksItem], CrdObjectList]]
+                 ):
+        super(Datacenter, self).__init__(
+            name=name,
+            racks=racks,
+        )
+
+    @property
+    def name(self):
+        # type: () -> str
+        return self._property_impl('name')
+    
+    @name.setter
+    def name(self, new_val):
+        # type: (str) -> None
+        self._name = new_val
+    
+    @property
+    def racks(self):
+        # type: () -> Union[List[RacksItem], CrdObjectList]
+        return self._property_impl('racks')
+    
+    @racks.setter
+    def racks(self, new_val):
+        # type: (Optional[Union[List[RacksItem], CrdObjectList]]) -> None
+        self._racks = new_val
+
+
+class Spec(CrdObject):
+    _properties = [
+        ('version', 'version', str, True, False),
+        ('datacenter', 'datacenter', Datacenter, True, False)
+    ]        
+
+    def __init__(self,
+                 version,  # type: str
+                 datacenter,  # type: Datacenter
+                 ):
+        super(Spec, self).__init__(
+            version=version,
+            datacenter=datacenter,
+        )
+
+    @property
+    def version(self):
+        # type: () -> str
+        return self._property_impl('version')
+    
+    @version.setter
+    def version(self, new_val):
+        # type: (str) -> None
+        self._version = new_val
+    
+    @property
+    def datacenter(self):
+        # type: () -> Datacenter
+        return self._property_impl('datacenter')
+    
+    @datacenter.setter
+    def datacenter(self, new_val):
+        # type: (Datacenter) -> None
+        self._datacenter = new_val
+
+
+class Cluster(CrdClass):
+    _properties = [
+        ('apiVersion', 'apiVersion', str, True, False),
+        ('metadata', 'metadata', object, True, False),
+        ('status', 'status', object, False, False),
+        ('spec', 'spec', Spec, True, False)
+    ]        
+
+    def __init__(self,
+                 apiVersion,  # type: str
+                 metadata,  # type: Any
+                 spec,  # type: Spec
+                 status=_omit,  # type: Optional[Any]
+                 ):
+        super(Cluster, self).__init__(
+            apiVersion=apiVersion,
+            metadata=metadata,
+            spec=spec,
+            status=status,
+        )
+
+    @property
+    def apiVersion(self):
+        # type: () -> str
+        return self._property_impl('apiVersion')
+    
+    @apiVersion.setter
+    def apiVersion(self, new_val):
+        # type: (str) -> None
+        self._apiVersion = new_val
+    
+    @property
+    def metadata(self):
+        # type: () -> Any
+        return self._property_impl('metadata')
+    
+    @metadata.setter
+    def metadata(self, new_val):
+        # type: (Any) -> None
+        self._metadata = new_val
+    
+    @property
+    def status(self):
+        # type: () -> Any
+        return self._property_impl('status')
+    
+    @status.setter
+    def status(self, new_val):
+        # type: (Optional[Any]) -> None
+        self._status = new_val
+    
+    @property
+    def spec(self):
+        # type: () -> Spec
+        return self._property_impl('spec')
+    
+    @spec.setter
+    def spec(self, new_val):
+        # type: (Spec) -> None
+        self._spec = new_val
diff --git a/src/pybind/mgr/rook/rook-client-python/rook_client/ceph/__init__.py b/src/pybind/mgr/rook/rook-client-python/rook_client/ceph/__init__.py
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/src/pybind/mgr/rook/rook-client-python/rook_client/ceph/__init__.py
diff --git a/src/pybind/mgr/rook/rook-client-python/rook_client/ceph/cephclient.py b/src/pybind/mgr/rook/rook-client-python/rook_client/ceph/cephclient.py
new file mode 100644
index 000000000..52894eadf
--- /dev/null
+++ b/src/pybind/mgr/rook/rook-client-python/rook_client/ceph/cephclient.py
@@ -0,0 +1,95 @@
+"""
+This file is automatically generated.
+Do not modify.
+"""
+
+try:
+    from typing import Any, Optional, Union, List
+except ImportError:
+    pass
+
+from .._helper import _omit, CrdObject, CrdObjectList, CrdClass
+
+class Spec(CrdObject):
+    _properties = [
+        ('caps', 'caps', object, False, False)
+    ]        
+
+    def __init__(self,
+                 caps=_omit,  # type: Optional[Any]
+                 ):
+        super(Spec, self).__init__(
+            caps=caps,
+        )
+
+    @property
+    def caps(self):
+        # type: () -> Any
+        return self._property_impl('caps')
+    
+    @caps.setter
+    def caps(self, new_val):
+        # type: (Optional[Any]) -> None
+        self._caps = new_val
+
+
+class CephClient(CrdClass):
+    _properties = [
+        ('apiVersion', 'apiVersion', str, True, False),
+        ('metadata', 'metadata', object, True, False),
+        ('status', 'status', object, False, False),
+        ('spec', 'spec', Spec, True, False)
+    ]        
+
+    def __init__(self,
+                 apiVersion,  # type: str
+                 metadata,  # type: Any
+                 spec,  # type: Spec
+                 status=_omit,  # type: Optional[Any]
+                 ):
+        super(CephClient, self).__init__(
+            apiVersion=apiVersion,
+            metadata=metadata,
+            spec=spec,
+            status=status,
+        )
+
+    @property
+    def apiVersion(self):
+        # type: () -> str
+        return self._property_impl('apiVersion')
+    
+    @apiVersion.setter
+    def apiVersion(self, new_val):
+        # type: (str) -> None
+        self._apiVersion = new_val
+    
+    @property
+    def metadata(self):
+        # type: () -> Any
+        return self._property_impl('metadata')
+    
+    @metadata.setter
+    def metadata(self, new_val):
+        # type: (Any) -> None
+        self._metadata = new_val
+    
+    @property
+    def status(self):
+        # type: () -> Any
+        return self._property_impl('status')
+    
+    @status.setter
+    def status(self, new_val):
+        # type: (Optional[Any]) -> None
+        self._status = new_val
+    
+    @property
+    def spec(self):
+        # type: () -> Spec
+        return self._property_impl('spec')
+    
+    @spec.setter
+    def spec(self, new_val):
+        # type: (Spec) -> None
+        self._spec = new_val
diff --git a/src/pybind/mgr/rook/rook-client-python/rook_client/ceph/cephcluster.py b/src/pybind/mgr/rook/rook-client-python/rook_client/ceph/cephcluster.py
new file mode 100644
index 000000000..d168c3c7c
--- /dev/null
+++ b/src/pybind/mgr/rook/rook-client-python/rook_client/ceph/cephcluster.py
@@ -0,0 +1,1119 @@
+"""
+This file is automatically generated.
+Do not modify.
+"""
+
+try:
+    from typing import Any, Optional, Union, List
+except ImportError:
+    pass
+
+from .._helper import _omit, CrdObject, CrdObjectList, CrdClass
+
+class CephVersion(CrdObject):
+    _properties = [
+        ('allowUnsupported', 'allowUnsupported', bool, False, False),
+        ('image', 'image', str, False, False)
+    ]        
+
+    def __init__(self,
+                 allowUnsupported=_omit,  # type: Optional[bool]
+                 image=_omit,  # type: Optional[str]
+                 ):
+        super(CephVersion, self).__init__(
+            allowUnsupported=allowUnsupported,
+            image=image,
+        )
+
+    @property
+    def allowUnsupported(self):
+        # type: () -> bool
+        return self._property_impl('allowUnsupported')
+    
+    @allowUnsupported.setter
+    def allowUnsupported(self, new_val):
+        # type: (Optional[bool]) -> None
+        self._allowUnsupported = new_val
+    
+    @property
+    def image(self):
+        # type: () -> str
+        return self._property_impl('image')
+    
+    @image.setter
+    def image(self, new_val):
+        # type: (Optional[str]) -> None
+        self._image = new_val
+
+
+class Dashboard(CrdObject):
+    _properties = [
+        ('enabled', 'enabled', bool, False, False),
+        ('urlPrefix', 'urlPrefix', str, False, False),
+        ('port', 'port', int, False, False),
+        ('ssl', 'ssl', bool, False, False)
+    ]        
+
+    def __init__(self,
+                 enabled=_omit,  # type: Optional[bool]
+                 urlPrefix=_omit,  # type: Optional[str]
+                 port=_omit,  # type: Optional[int]
+                 ssl=_omit,  # type: Optional[bool]
+                 ):
+        super(Dashboard, self).__init__(
+            enabled=enabled,
+            urlPrefix=urlPrefix,
+            port=port,
+            ssl=ssl,
+        )
+
+    @property
+    def enabled(self):
+        # type: () -> bool
+        return self._property_impl('enabled')
+    
+    @enabled.setter
+    def enabled(self, new_val):
+        # type: (Optional[bool]) -> None
+        self._enabled = new_val
+    
+    @property
+    def urlPrefix(self):
+        # type: () -> str
+        return self._property_impl('urlPrefix')
+    
+    @urlPrefix.setter
+    def urlPrefix(self, new_val):
+        # type: (Optional[str]) -> None
+        self._urlPrefix = new_val
+    
+    @property
+    def port(self):
+        # type: () -> int
+        return self._property_impl('port')
+    
+    @port.setter
+    def port(self, new_val):
+        # type: (Optional[int]) -> None
+        self._port = new_val
+    
+    @property
+    def ssl(self):
+        # type: () -> bool
+        return self._property_impl('ssl')
+    
+    @ssl.setter
+    def ssl(self, new_val):
+        # type: (Optional[bool]) -> None
+        self._ssl = new_val
+
+
+class DisruptionManagement(CrdObject):
+    _properties = [
+        ('machineDisruptionBudgetNamespace', 'machineDisruptionBudgetNamespace', str, False, False),
+        ('managePodBudgets', 'managePodBudgets', bool, False, False),
+        ('osdMaintenanceTimeout', 'osdMaintenanceTimeout', int, False, False),
+        ('manageMachineDisruptionBudgets', 'manageMachineDisruptionBudgets', bool, False, False)
+    ]        
+
+    def __init__(self,
+                 machineDisruptionBudgetNamespace=_omit,  # type: Optional[str]
+                 managePodBudgets=_omit,  # type: Optional[bool]
+                 osdMaintenanceTimeout=_omit,  # type: Optional[int]
+                 manageMachineDisruptionBudgets=_omit,  # type: Optional[bool]
+                 ):
+        super(DisruptionManagement, self).__init__(
+            machineDisruptionBudgetNamespace=machineDisruptionBudgetNamespace,
+            managePodBudgets=managePodBudgets,
+            osdMaintenanceTimeout=osdMaintenanceTimeout,
+            manageMachineDisruptionBudgets=manageMachineDisruptionBudgets,
+        )
+
+    @property
+    def machineDisruptionBudgetNamespace(self):
+        # type: () -> str
+        return self._property_impl('machineDisruptionBudgetNamespace')
+    
+    @machineDisruptionBudgetNamespace.setter
+    def machineDisruptionBudgetNamespace(self, new_val):
+        # type: (Optional[str]) -> None
+        self._machineDisruptionBudgetNamespace = new_val
+    
+    @property
+    def managePodBudgets(self):
+        # type: () -> bool
+        return self._property_impl('managePodBudgets')
+    
+    @managePodBudgets.setter
+    def managePodBudgets(self, new_val):
+        # type: (Optional[bool]) -> None
+        self._managePodBudgets = new_val
+    
+    @property
+    def osdMaintenanceTimeout(self):
+        # type: () -> int
+        return self._property_impl('osdMaintenanceTimeout')
+    
+    @osdMaintenanceTimeout.setter
+    def osdMaintenanceTimeout(self, new_val):
+        # type: (Optional[int]) -> None
+        self._osdMaintenanceTimeout = new_val
+    
+    @property
+    def manageMachineDisruptionBudgets(self):
+        # type: () -> bool
+        return self._property_impl('manageMachineDisruptionBudgets')
+    
+    @manageMachineDisruptionBudgets.setter
+    def manageMachineDisruptionBudgets(self, new_val):
+        # type: (Optional[bool]) -> None
+        self._manageMachineDisruptionBudgets = new_val
+
+
+class Mon(CrdObject):
+    _properties = [
+        ('allowMultiplePerNode', 'allowMultiplePerNode', bool, False, False),
+        ('count', 'count', int, False, False),
+        ('volumeClaimTemplate', 'volumeClaimTemplate', object, False, False)
+    ]        
+
+    def __init__(self,
+                 allowMultiplePerNode=_omit,  # type: Optional[bool]
+                 count=_omit,  # type: Optional[int]
+                 volumeClaimTemplate=_omit,  # type: Optional[Any]
+                 ):
+        super(Mon, self).__init__(
+            allowMultiplePerNode=allowMultiplePerNode,
+            count=count,
+            volumeClaimTemplate=volumeClaimTemplate,
+        )
+
+    @property
+    def allowMultiplePerNode(self):
+        # type: () -> bool
+        return self._property_impl('allowMultiplePerNode')
+    
+    @allowMultiplePerNode.setter
+    def allowMultiplePerNode(self, new_val):
+        # type: (Optional[bool]) -> None
+        self._allowMultiplePerNode = new_val
+    
+    @property
+    def count(self):
+        # type: () -> int
+        return self._property_impl('count')
+    
+    @count.setter
+    def count(self, new_val):
+        # type: (Optional[int]) -> None
+        self._count = new_val
+    
+    @property
+    def volumeClaimTemplate(self):
+        # type: () -> Any
+        return self._property_impl('volumeClaimTemplate')
+    
+    @volumeClaimTemplate.setter
+    def volumeClaimTemplate(self, new_val):
+        # type: (Optional[Any]) -> None
+        self._volumeClaimTemplate = new_val
+
+
+class ModulesItem(CrdObject):
+    _properties = [
+        ('name', 'name', str, False, False),
+        ('enabled', 'enabled', bool, False, False)
+    ]        
+
+    def __init__(self,
+                 name=_omit,  # type: Optional[str]
+                 enabled=_omit,  # type: Optional[bool]
+                 ):
+        super(ModulesItem, self).__init__(
+            name=name,
+            enabled=enabled,
+        )
+
+    @property
+    def name(self):
+        # type: () -> str
+        return self._property_impl('name')
+    
+    @name.setter
+    def name(self, new_val):
+        # type: (Optional[str]) -> None
+        self._name = new_val
+    
+    @property
+    def enabled(self):
+        # type: () -> bool
+        return self._property_impl('enabled')
+    
+    @enabled.setter
+    def enabled(self, new_val):
+        # type: (Optional[bool]) -> None
+        self._enabled = new_val
+
+
+class ModulesList(CrdObjectList):
+    _items_type = ModulesItem
+
+
+class Mgr(CrdObject):
+    _properties = [
+        ('modules', 'modules', ModulesList, False, False)
+    ]        
+
+    def __init__(self,
+                 modules=_omit,  # type: Optional[Union[List[ModulesItem], CrdObjectList]]
+                 ):
+        super(Mgr, self).__init__(
+            modules=modules,
+        )
+
+    @property
+    def modules(self):
+        # type: () -> Union[List[ModulesItem], CrdObjectList]
+        return self._property_impl('modules')
+    
+    @modules.setter
+    def modules(self, new_val):
+        # type: (Optional[Union[List[ModulesItem], CrdObjectList]]) -> None
+        self._modules = new_val
+
+
+class Network(CrdObject):
+    _properties = [
+        ('hostNetwork', 'hostNetwork', bool, False, False),
+        ('provider', 'provider', str, False, False),
+        ('selectors', 'selectors', object, False, False)
+    ]        
+
+    def __init__(self,
+                 hostNetwork=_omit,  # type: Optional[bool]
+                 provider=_omit,  # type: Optional[str]
+                 selectors=_omit,  # type: Optional[Any]
+                 ):
+        super(Network, self).__init__(
+            hostNetwork=hostNetwork,
+            provider=provider,
+            selectors=selectors,
+        )
+
+    @property
+    def hostNetwork(self):
+        # type: () -> bool
+        return self._property_impl('hostNetwork')
+    
+    @hostNetwork.setter
+    def hostNetwork(self, new_val):
+        # type: (Optional[bool]) -> None
+        self._hostNetwork = new_val
+    
+    @property
+    def provider(self):
+        # type: () -> str
+        return self._property_impl('provider')
+    
+    @provider.setter
+    def provider(self, new_val):
+        # type: (Optional[str]) -> None
+        self._provider = new_val
+    
+    @property
+    def selectors(self):
+        # type: () -> Any
+        return self._property_impl('selectors')
+    
+    @selectors.setter
+    def selectors(self, new_val):
+        # type: (Optional[Any]) -> None
+        self._selectors = new_val
+
+
+class Config(CrdObject):
+    _properties = [
+        ('metadataDevice', 'metadataDevice', str, False, False),
+        ('storeType', 'storeType', str, False, False),
+        ('databaseSizeMB', 'databaseSizeMB', str, False, False),
+        ('walSizeMB', 'walSizeMB', str, False, False),
+        ('journalSizeMB', 'journalSizeMB', str, False, False),
+        ('osdsPerDevice', 'osdsPerDevice', str, False, False),
+        ('encryptedDevice', 'encryptedDevice', str, False, False)
+    ]        
+
+    def __init__(self,
+                 metadataDevice=_omit,  # type: Optional[str]
+                 storeType=_omit,  # type: Optional[str]
+                 databaseSizeMB=_omit,  # type: Optional[str]
+                 walSizeMB=_omit,  # type: Optional[str]
+                 journalSizeMB=_omit,  # type: Optional[str]
+                 osdsPerDevice=_omit,  # type: Optional[str]
+                 encryptedDevice=_omit,  # type: Optional[str]
+                 ):
+        super(Config, self).__init__(
+            metadataDevice=metadataDevice,
+            storeType=storeType,
+            databaseSizeMB=databaseSizeMB,
+            walSizeMB=walSizeMB,
+            journalSizeMB=journalSizeMB,
+            osdsPerDevice=osdsPerDevice,
+            encryptedDevice=encryptedDevice,
+        )
+
+    @property
+    def metadataDevice(self):
+        # type: () -> str
+        return self._property_impl('metadataDevice')
+    
+    @metadataDevice.setter
+    def metadataDevice(self, new_val):
+        # type: (Optional[str]) -> None
+        self._metadataDevice = new_val
+    
+    @property
+    def storeType(self):
+        # type: () -> str
+        return self._property_impl('storeType')
+    
+    @storeType.setter
+    def storeType(self, new_val):
+        # type: (Optional[str]) -> None
+        self._storeType = new_val
+    
+    @property
+    def databaseSizeMB(self):
+        # type: () -> str
+        return self._property_impl('databaseSizeMB')
+    
+    @databaseSizeMB.setter
+    def databaseSizeMB(self, new_val):
+        # type: (Optional[str]) -> None
+        self._databaseSizeMB = new_val
+    
+    @property
+    def walSizeMB(self):
+        # type: () -> str
+        return self._property_impl('walSizeMB')
+    
+    @walSizeMB.setter
+    def walSizeMB(self, new_val):
+        # type: (Optional[str]) -> None
+        self._walSizeMB = new_val
+    
+    @property
+    def journalSizeMB(self):
+        # type: () -> str
+        return self._property_impl('journalSizeMB')
+    
+    @journalSizeMB.setter
+    def journalSizeMB(self, new_val):
+        # type: (Optional[str]) -> None
+        self._journalSizeMB = new_val
+    
+    @property
+    def osdsPerDevice(self):
+        # type: () -> str
+        return self._property_impl('osdsPerDevice')
+    
+    @osdsPerDevice.setter
+    def osdsPerDevice(self, new_val):
+        # type: (Optional[str]) -> None
+        self._osdsPerDevice = new_val
+    
+    @property
+    def encryptedDevice(self):
+        # type: () -> str
+        return self._property_impl('encryptedDevice')
+    
+    @encryptedDevice.setter
+    def encryptedDevice(self, new_val):
+        # type: (Optional[str]) -> None
+        self._encryptedDevice = new_val
+
+
+class DirectoriesItem(CrdObject):
+    _properties = [
+        ('path', 'path', str, False, False)
+    ]        
+
+    def __init__(self,
+                 path=_omit,  # type: Optional[str]
+                 ):
+        super(DirectoriesItem, self).__init__(
+            path=path,
+        )
+
+    @property
+    def path(self):
+        # type: () -> str
+        return self._property_impl('path')
+    
+    @path.setter
+    def path(self, new_val):
+        # type: (Optional[str]) -> None
+        self._path = new_val
+
+
+class DirectoriesList(CrdObjectList):
+    _items_type = DirectoriesItem
+
+
+class DevicesItem(CrdObject):
+    _properties = [
+        ('name', 'name', str, False, False),
+        ('config', 'config', object, False, False)
+    ]        
+
+    def __init__(self,
+                 name=_omit,  # type: Optional[str]
+                 config=_omit,  # type: Optional[Any]
+                 ):
+        super(DevicesItem, self).__init__(
+            name=name,
+            config=config,
+        )
+
+    @property
+    def name(self):
+        # type: () -> str
+        return self._property_impl('name')
+    
+    @name.setter
+    def name(self, new_val):
+        # type: (Optional[str]) -> None
+        self._name = new_val
+    
+    @property
+    def config(self):
+        # type: () -> Any
+        return self._property_impl('config')
+    
+    @config.setter
+    def config(self, new_val):
+        # type: (Optional[Any]) -> None
+        self._config = new_val
+
+
+class DevicesList(CrdObjectList):
+    _items_type = DevicesItem
+
+
+class NodesItem(CrdObject):
+    _properties = [
+        ('name', 'name', str, False, False),
+        ('config', 'config', Config, False, False),
+        ('useAllDevices', 'useAllDevices', bool, False, False),
+        ('deviceFilter', 'deviceFilter', str, False, False),
+        ('devicePathFilter', 'devicePathFilter', str, False, False),
+        ('directories', 'directories', DirectoriesList, False, False),
+        ('devices', 'devices', DevicesList, False, False),
+        ('resources', 'resources', object, False, False)
+    ]        
+
+    def __init__(self,
+                 name=_omit,  # type: Optional[str]
+                 config=_omit,  # type: Optional[Config]
+                 useAllDevices=_omit,  # type: Optional[bool]
+                 deviceFilter=_omit,  # type: Optional[str]
+                 devicePathFilter=_omit,  # type: Optional[str]
+                 directories=_omit,  # type: Optional[Union[List[DirectoriesItem], CrdObjectList]]
+                 devices=_omit,  # type: Optional[Union[List[DevicesItem], CrdObjectList]]
+                 resources=_omit,  # type: Optional[Any]
+                 ):
+        super(NodesItem, self).__init__(
+            name=name,
+            config=config,
+            useAllDevices=useAllDevices,
+            deviceFilter=deviceFilter,
+            devicePathFilter=devicePathFilter,
+            directories=directories,
+            devices=devices,
+            resources=resources,
+        )
+
+    @property
+    def name(self):
+        # type: () -> str
+        return self._property_impl('name')
+    
+    @name.setter
+    def name(self, new_val):
+        # type: (Optional[str]) -> None
+        self._name = new_val
+    
+    @property
+    def config(self):
+        # type: () -> Config
+        return self._property_impl('config')
+    
+    @config.setter
+    def config(self, new_val):
+        # type: (Optional[Config]) -> None
+        self._config = new_val
+    
+    @property
+    def useAllDevices(self):
+        # type: () -> bool
+        return self._property_impl('useAllDevices')
+    
+    @useAllDevices.setter
+    def useAllDevices(self, new_val):
+        # type: (Optional[bool]) -> None
+        self._useAllDevices = new_val
+    
+    @property
+    def deviceFilter(self):
+        # type: () -> str
+        return self._property_impl('deviceFilter')
+    
+    @deviceFilter.setter
+    def deviceFilter(self, new_val):
+        # type: (Optional[str]) -> None
+        self._deviceFilter = new_val
+    
+    @property
+    def devicePathFilter(self):
+        # type: () -> str
+        return self._property_impl('devicePathFilter')
+    
+    @devicePathFilter.setter
+    def devicePathFilter(self, new_val):
+        # type: (Optional[str]) -> None
+        self._devicePathFilter = new_val
+    
+    @property
+    def directories(self):
+        # type: () -> Union[List[DirectoriesItem], CrdObjectList]
+        return self._property_impl('directories')
+    
+    @directories.setter
+    def directories(self, new_val):
+        # type: (Optional[Union[List[DirectoriesItem], CrdObjectList]]) -> None
+        self._directories = new_val
+    
+    @property
+    def devices(self):
+        # type: () -> Union[List[DevicesItem], CrdObjectList]
+        return self._property_impl('devices')
+    
+    @devices.setter
+    def devices(self, new_val):
+        # type: (Optional[Union[List[DevicesItem], CrdObjectList]]) -> None
+        self._devices = new_val
+    
+    @property
+    def resources(self):
+        # type: () -> Any
+        return self._property_impl('resources')
+    
+    @resources.setter
+    def resources(self, new_val):
+        # type: (Optional[Any]) -> None
+        self._resources = new_val
+
+
+class NodesList(CrdObjectList):
+    _items_type = NodesItem
+
+
+class Storage(CrdObject):
+    _properties = [
+        ('disruptionManagement', 'disruptionManagement', DisruptionManagement, False, False),
+        ('useAllNodes', 'useAllNodes', bool, False, False),
+        ('nodes', 'nodes', NodesList, False, False),
+        ('useAllDevices', 'useAllDevices', bool, False, False),
+        ('deviceFilter', 'deviceFilter', str, False, False),
+        ('devicePathFilter', 'devicePathFilter', str, False, False),
+        ('directories', 'directories', DirectoriesList, False, False),
+        ('config', 'config', object, False, False),
+        ('storageClassDeviceSets', 'storageClassDeviceSets', object, False, False)
+    ]        
+
+    def __init__(self,
+                 disruptionManagement=_omit,  # type: Optional[DisruptionManagement]
+                 useAllNodes=_omit,  # type: Optional[bool]
+                 nodes=_omit,  # type: Optional[Union[List[NodesItem], CrdObjectList]]
+                 useAllDevices=_omit,  # type: Optional[bool]
+                 deviceFilter=_omit,  # type: Optional[str]
+                 devicePathFilter=_omit,  # type: Optional[str]
+                 directories=_omit,  # type: Optional[Union[List[DirectoriesItem], CrdObjectList]]
+                 config=_omit,  # type: Optional[Any]
+                 storageClassDeviceSets=_omit,  # type: Optional[Any]
+                 ):
+        super(Storage, self).__init__(
+            disruptionManagement=disruptionManagement,
+            useAllNodes=useAllNodes,
+            nodes=nodes,
+            useAllDevices=useAllDevices,
+            deviceFilter=deviceFilter,
+            devicePathFilter=devicePathFilter,
+            directories=directories,
+            config=config,
+            storageClassDeviceSets=storageClassDeviceSets,
+        )
+
+    @property
+    def disruptionManagement(self):
+        # type: () -> DisruptionManagement
+        return self._property_impl('disruptionManagement')
+    
+    @disruptionManagement.setter
+    def disruptionManagement(self, new_val):
+        # type: (Optional[DisruptionManagement]) -> None
+        self._disruptionManagement = new_val
+    
+    @property
+    def useAllNodes(self):
+        # type: () -> bool
+        return self._property_impl('useAllNodes')
+    
+    @useAllNodes.setter
+    def useAllNodes(self, new_val):
+        # type: (Optional[bool]) -> None
+        self._useAllNodes = new_val
+    
+    @property
+    def nodes(self):
+        # type: () -> Union[List[NodesItem], CrdObjectList]
+        return self._property_impl('nodes')
+    
+    @nodes.setter
+    def nodes(self, new_val):
+        # type: (Optional[Union[List[NodesItem], CrdObjectList]]) -> None
+        self._nodes = new_val
+    
+    @property
+    def useAllDevices(self):
+        # type: () -> bool
+        return self._property_impl('useAllDevices')
+    
+    @useAllDevices.setter
+    def useAllDevices(self, new_val):
+        # type: (Optional[bool]) -> None
+        self._useAllDevices = new_val
+    
+    @property
+    def deviceFilter(self):
+        # type: () -> str
+        return self._property_impl('deviceFilter')
+    
+    @deviceFilter.setter
+    def deviceFilter(self, new_val):
+        # type: (Optional[str]) -> None
+        self._deviceFilter = new_val
+    
+    @property
+    def devicePathFilter(self):
+        # type: () -> str
+        return self._property_impl('devicePathFilter')
+    
+    @devicePathFilter.setter
+    def devicePathFilter(self, new_val):
+        # type: (Optional[str]) -> None
+        self._devicePathFilter = new_val
+    
+    @property
+    def directories(self):
+        # type: () -> Union[List[DirectoriesItem], CrdObjectList]
+        return self._property_impl('directories')
+    
+    @directories.setter
+    def directories(self, new_val):
+        # type: (Optional[Union[List[DirectoriesItem], CrdObjectList]]) -> None
+        self._directories = new_val
+    
+    @property
+    def config(self):
+        # type: () -> Any
+        return self._property_impl('config')
+    
+    @config.setter
+    def config(self, new_val):
+        # type: (Optional[Any]) -> None
+        self._config = new_val
+    
+    @property
+    def storageClassDeviceSets(self):
+        # type: () -> Any
+        return self._property_impl('storageClassDeviceSets')
+    
+    @storageClassDeviceSets.setter
+    def storageClassDeviceSets(self, new_val):
+        # type: (Optional[Any]) -> None
+        self._storageClassDeviceSets = new_val
+
+
+class Monitoring(CrdObject):
+    _properties = [
+        ('enabled', 'enabled', bool, False, False),
+        ('rulesNamespace', 'rulesNamespace', str, False, False)
+    ]        
+
+    def __init__(self,
+                 enabled=_omit,  # type: Optional[bool]
+                 rulesNamespace=_omit,  # type: Optional[str]
+                 ):
+        super(Monitoring, self).__init__(
+            enabled=enabled,
+            rulesNamespace=rulesNamespace,
+        )
+
+    @property
+    def enabled(self):
+        # type: () -> bool
+        return self._property_impl('enabled')
+    
+    @enabled.setter
+    def enabled(self, new_val):
+        # type: (Optional[bool]) -> None
+        self._enabled = new_val
+    
+    @property
+    def rulesNamespace(self):
+        # type: () -> str
+        return self._property_impl('rulesNamespace')
+    
+    @rulesNamespace.setter
+    def rulesNamespace(self, new_val):
+        # type: (Optional[str]) -> None
+        self._rulesNamespace = new_val
+
+
+class RbdMirroring(CrdObject):
+    _properties = [
+        ('workers', 'workers', int, False, False)
+    ]        
+
+    def __init__(self,
+                 workers=_omit,  # type: Optional[int]
+                 ):
+        super(RbdMirroring, self).__init__(
+            workers=workers,
+        )
+
+    @property
+    def workers(self):
+        # type: () -> int
+        return self._property_impl('workers')
+    
+    @workers.setter
+    def workers(self, new_val):
+        # type: (Optional[int]) -> None
+        self._workers = new_val
+
+
+class External(CrdObject):
+    _properties = [
+        ('enable', 'enable', bool, False, False)
+    ]        
+
+    def __init__(self,
+                 enable=_omit,  # type: Optional[bool]
+                 ):
+        super(External, self).__init__(
+            enable=enable,
+        )
+
+    @property
+    def enable(self):
+        # type: () -> bool
+        return self._property_impl('enable')
+    
+    @enable.setter
+    def enable(self, new_val):
+        # type: (Optional[bool]) -> None
+        self._enable = new_val
+
+
+class Spec(CrdObject):
+    _properties = [
+        ('annotations', 'annotations', object, False, False),
+        ('cephVersion', 'cephVersion', CephVersion, False, False),
+        ('dashboard', 'dashboard', Dashboard, False, False),
+        ('dataDirHostPath', 'dataDirHostPath', str, False, False),
+        ('disruptionManagement', 'disruptionManagement', DisruptionManagement, False, False),
+        ('skipUpgradeChecks', 'skipUpgradeChecks', bool, False, False),
+        ('continueUpgradeAfterChecksEvenIfNotHealthy', 'continueUpgradeAfterChecksEvenIfNotHealthy', bool, False, False),
+        ('mon', 'mon', Mon, False, False),
+        ('mgr', 'mgr', Mgr, False, False),
+        ('network', 'network', Network, False, False),
+        ('storage', 'storage', Storage, False, False),
+        ('monitoring', 'monitoring', Monitoring, False, False),
+        ('rbdMirroring', 'rbdMirroring', RbdMirroring, False, False),
+        ('removeOSDsIfOutAndSafeToRemove', 'removeOSDsIfOutAndSafeToRemove', bool, False, False),
+        ('external', 'external', External, False, False),
+        ('placement', 'placement', object, False, False),
+        ('resources', 'resources', object, False, False)
+    ]        
+
+    def __init__(self,
+                 annotations=_omit,  # type: Optional[Any]
+                 cephVersion=_omit,  # type: Optional[CephVersion]
+                 dashboard=_omit,  # type: Optional[Dashboard]
+                 dataDirHostPath=_omit,  # type: Optional[str]
+                 disruptionManagement=_omit,  # type: Optional[DisruptionManagement]
+                 skipUpgradeChecks=_omit,  # type: Optional[bool]
+                 continueUpgradeAfterChecksEvenIfNotHealthy=_omit,  # type: Optional[bool]
+                 mon=_omit,  # type: Optional[Mon]
+                 mgr=_omit,  # type: Optional[Mgr]
+                 network=_omit,  # type: Optional[Network]
+                 storage=_omit,  # type: Optional[Storage]
+                 monitoring=_omit,  # type: Optional[Monitoring]
+                 rbdMirroring=_omit,  # type: Optional[RbdMirroring]
+                 removeOSDsIfOutAndSafeToRemove=_omit,  # type: Optional[bool]
+                 external=_omit,  # type: Optional[External]
+                 placement=_omit,  # type: Optional[Any]
+                 resources=_omit,  # type: Optional[Any]
+                 ):
+        super(Spec, self).__init__(
+            annotations=annotations,
+            cephVersion=cephVersion,
+            dashboard=dashboard,
+            dataDirHostPath=dataDirHostPath,
+            disruptionManagement=disruptionManagement,
+            skipUpgradeChecks=skipUpgradeChecks,
+            continueUpgradeAfterChecksEvenIfNotHealthy=continueUpgradeAfterChecksEvenIfNotHealthy,
+            mon=mon,
+            mgr=mgr,
+            network=network,
+            storage=storage,
+            monitoring=monitoring,
+            rbdMirroring=rbdMirroring,
+            removeOSDsIfOutAndSafeToRemove=removeOSDsIfOutAndSafeToRemove,
+            external=external,
+            placement=placement,
+            resources=resources,
+        )
+
+    @property
+    def annotations(self):
+        # type: () -> Any
+        return self._property_impl('annotations')
+    
+    @annotations.setter
+    def annotations(self, new_val):
+        # type: (Optional[Any]) -> None
+        self._annotations = new_val
+    
+    @property
+    def cephVersion(self):
+        # type: () -> CephVersion
+        return self._property_impl('cephVersion')
+    
+    @cephVersion.setter
+    def cephVersion(self, new_val):
+        # type: (Optional[CephVersion]) -> None
+        self._cephVersion = new_val
+    
+    @property
+    def dashboard(self):
+        # type: () -> Dashboard
+        return self._property_impl('dashboard')
+    
+    @dashboard.setter
+    def dashboard(self, new_val):
+        # type: (Optional[Dashboard]) -> None
+        self._dashboard = new_val
+    
+    @property
+    def dataDirHostPath(self):
+        # type: () -> str
+        return self._property_impl('dataDirHostPath')
+    
+    @dataDirHostPath.setter
+    def dataDirHostPath(self, new_val):
+        # type: (Optional[str]) -> None
+        self._dataDirHostPath = new_val
+    
+    @property
+    def disruptionManagement(self):
+        # type: () -> DisruptionManagement
+        return self._property_impl('disruptionManagement')
+    
+    @disruptionManagement.setter
+    def disruptionManagement(self, new_val):
+        # type: (Optional[DisruptionManagement]) -> None
+        self._disruptionManagement = new_val
+    
+    @property
+    def skipUpgradeChecks(self):
+        # type: () -> bool
+        return self._property_impl('skipUpgradeChecks')
+    
+    @skipUpgradeChecks.setter
+    def skipUpgradeChecks(self, new_val):
+        # type: (Optional[bool]) -> None
+        self._skipUpgradeChecks = new_val
+    
+    @property
+    def continueUpgradeAfterChecksEvenIfNotHealthy(self):
+        # type: () -> bool
+        return self._property_impl('continueUpgradeAfterChecksEvenIfNotHealthy')
+    
+    @continueUpgradeAfterChecksEvenIfNotHealthy.setter
+    def continueUpgradeAfterChecksEvenIfNotHealthy(self, new_val):
+        # type: (Optional[bool]) -> None
+        self._continueUpgradeAfterChecksEvenIfNotHealthy = new_val
+    
+    @property
+    def mon(self):
+        # type: () -> Mon
+        return self._property_impl('mon')
+    
+    @mon.setter
+    def mon(self, new_val):
+        # type: (Optional[Mon]) -> None
+        self._mon = new_val
+    
+    @property
+    def mgr(self):
+        # type: () -> Mgr
+        return self._property_impl('mgr')
+    
+    @mgr.setter
+    def mgr(self, new_val):
+        # type: (Optional[Mgr]) -> None
+        self._mgr = new_val
+    
+    @property
+    def network(self):
+        # type: () -> Network
+        return self._property_impl('network')
+    
+    @network.setter
+    def network(self, new_val):
+        # type: (Optional[Network]) -> None
+        self._network = new_val
+    
+    @property
+    def storage(self):
+        # type: () -> Storage
+        return self._property_impl('storage')
+    
+    @storage.setter
+    def storage(self, new_val):
+        # type: (Optional[Storage]) -> None
+        self._storage = new_val
+    
+    @property
+    def monitoring(self):
+        # type: () -> Monitoring
+        return self._property_impl('monitoring')
+    
+    @monitoring.setter
+    def monitoring(self, new_val):
+        # type: (Optional[Monitoring]) -> None
+        self._monitoring = new_val
+    
+    @property
+    def rbdMirroring(self):
+        # type: () -> RbdMirroring
+        return self._property_impl('rbdMirroring')
+    
+    @rbdMirroring.setter
+    def rbdMirroring(self, new_val):
+        # type: (Optional[RbdMirroring]) -> None
+        self._rbdMirroring = new_val
+    
+    @property
+    def removeOSDsIfOutAndSafeToRemove(self):
+        # type: () -> bool
+        return self._property_impl('removeOSDsIfOutAndSafeToRemove')
+    
+    @removeOSDsIfOutAndSafeToRemove.setter
+    def removeOSDsIfOutAndSafeToRemove(self, new_val):
+        # type: (Optional[bool]) -> None
+        self._removeOSDsIfOutAndSafeToRemove = new_val
+    
+    @property
+    def external(self):
+        # type: () -> External
+        return self._property_impl('external')
+    
+    @external.setter
+    def external(self, new_val):
+        # type: (Optional[External]) -> None
+        self._external = new_val
+    
+    @property
+    def placement(self):
+        # type: () -> Any
+        return self._property_impl('placement')
+    
+    @placement.setter
+    def placement(self, new_val):
+        # type: (Optional[Any]) -> None
+        self._placement = new_val
+    
+    @property
+    def resources(self):
+        # type: () -> Any
+        return self._property_impl('resources')
+    
+    @resources.setter
+    def resources(self, new_val):
+        # type: (Optional[Any]) -> None
+        self._resources = new_val
+
+
+class CephCluster(CrdClass):
+    _properties = [
+        ('apiVersion', 'apiVersion', str, True, False),
+        ('metadata', 'metadata', object, True, False),
+        ('status', 'status', object, False, False),
+        ('spec', 'spec', Spec, True, False)
+    ]        
+
+    def __init__(self,
+                 apiVersion,  # type: str
+                 metadata,  # type: Any
+                 spec,  # type: Spec
+                 status=_omit,  # type: Optional[Any]
+                 ):
+        super(CephCluster, self).__init__(
+            apiVersion=apiVersion,
+            metadata=metadata,
+            spec=spec,
+            status=status,
+        )
+
+    @property
+    def apiVersion(self):
+        # type: () -> str
+        return self._property_impl('apiVersion')
+    
+    @apiVersion.setter
+    def apiVersion(self, new_val):
+        # type: (str) -> None
+        self._apiVersion = new_val
+    
+    @property
+    def metadata(self):
+        # type: () -> Any
+        return self._property_impl('metadata')
+    
+    @metadata.setter
+    def metadata(self, new_val):
+        # type: (Any) -> None
+        self._metadata = new_val
+    
+    @property
+    def status(self):
+        # type: () -> Any
+        return self._property_impl('status')
+    
+    @status.setter
+    def status(self, new_val):
+        # type: (Optional[Any]) -> None
+        self._status = new_val
+    
+    @property
+    def spec(self):
+        # type: () -> Spec
+        return self._property_impl('spec')
+    
+    @spec.setter
+    def spec(self, new_val):
+        # type: (Spec) -> None
+        self._spec = new_val
diff --git a/src/pybind/mgr/rook/rook-client-python/rook_client/ceph/cephfilesystem.py b/src/pybind/mgr/rook/rook-client-python/rook_client/ceph/cephfilesystem.py
new file mode 100644
index 000000000..ac217711d
--- /dev/null
+++ b/src/pybind/mgr/rook/rook-client-python/rook_client/ceph/cephfilesystem.py
@@ -0,0 +1,370 @@
+"""
+This file is automatically generated.
+Do not modify.
+"""
+
+try:
+    from typing import Any, Optional, Union, List
+except ImportError:
+    pass
+
+from .._helper import _omit, CrdObject, CrdObjectList, CrdClass
+
+class MetadataServer(CrdObject):
+    _properties = [
+        ('activeCount', 'activeCount', int, False, False),
+        ('activeStandby', 'activeStandby', bool, False, False),
+        ('annotations', 'annotations', object, False, False),
+        ('placement', 'placement', object, False, False),
+        ('resources', 'resources', object, False, False)
+    ]        
+
+    def __init__(self,
+                 activeCount=_omit,  # type: Optional[int]
+                 activeStandby=_omit,  # type: Optional[bool]
+                 annotations=_omit,  # type: Optional[Any]
+                 placement=_omit,  # type: Optional[Any]
+                 resources=_omit,  # type: Optional[Any]
+                 ):
+        super(MetadataServer, self).__init__(
+            activeCount=activeCount,
+            activeStandby=activeStandby,
+            annotations=annotations,
+            placement=placement,
+            resources=resources,
+        )
+
+    @property
+    def activeCount(self):
+        # type: () -> int
+        return self._property_impl('activeCount')
+    
+    @activeCount.setter
+    def activeCount(self, new_val):
+        # type: (Optional[int]) -> None
+        self._activeCount = new_val
+    
+    @property
+    def activeStandby(self):
+        # type: () -> bool
+        return self._property_impl('activeStandby')
+    
+    @activeStandby.setter
+    def activeStandby(self, new_val):
+        # type: (Optional[bool]) -> None
+        self._activeStandby = new_val
+    
+    @property
+    def annotations(self):
+        # type: () -> Any
+        return self._property_impl('annotations')
+    
+    @annotations.setter
+    def annotations(self, new_val):
+        # type: (Optional[Any]) -> None
+        self._annotations = new_val
+    
+    @property
+    def placement(self):
+        # type: () -> Any
+        return self._property_impl('placement')
+    
+    @placement.setter
+    def placement(self, new_val):
+        # type: (Optional[Any]) -> None
+        self._placement = new_val
+    
+    @property
+    def resources(self):
+        # type: () -> Any
+        return self._property_impl('resources')
+    
+    @resources.setter
+    def resources(self, new_val):
+        # type: (Optional[Any]) -> None
+        self._resources = new_val
+
+
+class Replicated(CrdObject):
+    _properties = [
+        ('size', 'size', int, False, False)
+    ]        
+
+    def __init__(self,
+                 size=_omit,  # type: Optional[int]
+                 ):
+        super(Replicated, self).__init__(
+            size=size,
+        )
+
+    @property
+    def size(self):
+        # type: () -> int
+        return self._property_impl('size')
+    
+    @size.setter
+    def size(self, new_val):
+        # type: (Optional[int]) -> None
+        self._size = new_val
+
+
+class ErasureCoded(CrdObject):
+    _properties = [
+        ('dataChunks', 'dataChunks', int, False, False),
+        ('codingChunks', 'codingChunks', int, False, False)
+    ]        
+
+    def __init__(self,
+                 dataChunks=_omit,  # type: Optional[int]
+                 codingChunks=_omit,  # type: Optional[int]
+                 ):
+        super(ErasureCoded, self).__init__(
+            dataChunks=dataChunks,
+            codingChunks=codingChunks,
+        )
+
+    @property
+    def dataChunks(self):
+        # type: () -> int
+        return self._property_impl('dataChunks')
+    
+    @dataChunks.setter
+    def dataChunks(self, new_val):
+        # type: (Optional[int]) -> None
+        self._dataChunks = new_val
+    
+    @property
+    def codingChunks(self):
+        # type: () -> int
+        return self._property_impl('codingChunks')
+    
+    @codingChunks.setter
+    def codingChunks(self, new_val):
+        # type: (Optional[int]) -> None
+        self._codingChunks = new_val
+
+
+class MetadataPool(CrdObject):
+    _properties = [
+        ('failureDomain', 'failureDomain', str, False, False),
+        ('replicated', 'replicated', Replicated, False, False),
+        ('erasureCoded', 'erasureCoded', ErasureCoded, False, False)
+    ]        
+
+    def __init__(self,
+                 failureDomain=_omit,  # type: Optional[str]
+                 replicated=_omit,  # type: Optional[Replicated]
+                 erasureCoded=_omit,  # type: Optional[ErasureCoded]
+                 ):
+        super(MetadataPool, self).__init__(
+            failureDomain=failureDomain,
+            replicated=replicated,
+            erasureCoded=erasureCoded,
+        )
+
+    @property
+    def failureDomain(self):
+        # type: () -> str
+        return self._property_impl('failureDomain')
+    
+    @failureDomain.setter
+    def failureDomain(self, new_val):
+        # type: (Optional[str]) -> None
+        self._failureDomain = new_val
+    
+    @property
+    def replicated(self):
+        # type: () -> Replicated
+        return self._property_impl('replicated')
+    
+    @replicated.setter
+    def replicated(self, new_val):
+        # type: (Optional[Replicated]) -> None
+        self._replicated = new_val
+    
+    @property
+    def erasureCoded(self):
+        # type: () -> ErasureCoded
+        return self._property_impl('erasureCoded')
+    
+    @erasureCoded.setter
+    def erasureCoded(self, new_val):
+        # type: (Optional[ErasureCoded]) -> None
+        self._erasureCoded = new_val
+
+
+class DataPoolsItem(CrdObject):
+    _properties = [
+        ('failureDomain', 'failureDomain', str, False, False),
+        ('replicated', 'replicated', Replicated, False, False),
+        ('erasureCoded', 'erasureCoded', ErasureCoded, False, False)
+    ]        
+
+    def __init__(self,
+                 failureDomain=_omit,  # type: Optional[str]
+                 replicated=_omit,  # type: Optional[Replicated]
+                 erasureCoded=_omit,  # type: Optional[ErasureCoded]
+                 ):
+        super(DataPoolsItem, self).__init__(
+            failureDomain=failureDomain,
+            replicated=replicated,
+            erasureCoded=erasureCoded,
+        )
+
+    @property
+    def failureDomain(self):
+        # type: () -> str
+        return self._property_impl('failureDomain')
+    
+    @failureDomain.setter
+    def failureDomain(self, new_val):
+        # type: (Optional[str]) -> None
+        self._failureDomain = new_val
+    
+    @property
+    def replicated(self):
+        # type: () -> Replicated
+        return self._property_impl('replicated')
+    
+    @replicated.setter
+    def replicated(self, new_val):
+        # type: (Optional[Replicated]) -> None
+        self._replicated = new_val
+    
+    @property
+    def erasureCoded(self):
+        # type: () -> ErasureCoded
+        return self._property_impl('erasureCoded')
+    
+    @erasureCoded.setter
+    def erasureCoded(self, new_val):
+        # type: (Optional[ErasureCoded]) -> None
+        self._erasureCoded = new_val
+
+
+class DataPoolsList(CrdObjectList):
+    _items_type = DataPoolsItem
+
+
+class Spec(CrdObject):
+    _properties = [
+        ('metadataServer', 'metadataServer', MetadataServer, False, False),
+        ('metadataPool', 'metadataPool', MetadataPool, False, False),
+        ('dataPools', 'dataPools', DataPoolsList, False, False),
+        ('preservePoolsOnDelete', 'preservePoolsOnDelete', bool, False, False)
+    ]        
+
+    def __init__(self,
+                 metadataServer=_omit,  # type: Optional[MetadataServer]
+                 metadataPool=_omit,  # type: Optional[MetadataPool]
+                 dataPools=_omit,  # type: Optional[Union[List[DataPoolsItem], CrdObjectList]]
+                 preservePoolsOnDelete=_omit,  # type: Optional[bool]
+                 ):
+        super(Spec, self).__init__(
+            metadataServer=metadataServer,
+            metadataPool=metadataPool,
+            dataPools=dataPools,
+            preservePoolsOnDelete=preservePoolsOnDelete,
+        )
+
+    @property
+    def metadataServer(self):
+        # type: () -> MetadataServer
+        return self._property_impl('metadataServer')
+    
+    @metadataServer.setter
+    def metadataServer(self, new_val):
+        # type: (Optional[MetadataServer]) -> None
+        self._metadataServer = new_val
+    
+    @property
+    def metadataPool(self):
+        # type: () -> MetadataPool
+        return self._property_impl('metadataPool')
+    
+    @metadataPool.setter
+    def metadataPool(self, new_val):
+        # type: (Optional[MetadataPool]) -> None
+        self._metadataPool = new_val
+    
+    @property
+    def dataPools(self):
+        # type: () -> Union[List[DataPoolsItem], CrdObjectList]
+        return self._property_impl('dataPools')
+    
+    @dataPools.setter
+    def dataPools(self, new_val):
+        # type: (Optional[Union[List[DataPoolsItem], CrdObjectList]]) -> None
+        self._dataPools = new_val
+    
+    @property
+    def preservePoolsOnDelete(self):
+        # type: () -> bool
+        return self._property_impl('preservePoolsOnDelete')
+    
+    @preservePoolsOnDelete.setter
+    def preservePoolsOnDelete(self, new_val):
+        # type: (Optional[bool]) -> None
+        self._preservePoolsOnDelete = new_val
+
+
+class CephFilesystem(CrdClass):
+    _properties = [
+        ('apiVersion', 'apiVersion', str, True, False),
+        ('metadata', 'metadata', object, True, False),
+        ('status', 'status', object, False, False),
+        ('spec', 'spec', Spec, True, False)
+    ]        
+
+    def __init__(self,
+                 apiVersion,  # type: str
+                 metadata,  # type: Any
+                 spec,  # type: Spec
+                 status=_omit,  # type: Optional[Any]
+                 ):
+        super(CephFilesystem, self).__init__(
+            apiVersion=apiVersion,
+            metadata=metadata,
+            spec=spec,
+            status=status,
+        )
+
+    @property
+    def apiVersion(self):
+        # type: () -> str
+        return self._property_impl('apiVersion')
+    
+    @apiVersion.setter
+    def apiVersion(self, new_val):
+        # type: (str) -> None
+        self._apiVersion = new_val
+    
+    @property
+    def metadata(self):
+        # type: () -> Any
+        return self._property_impl('metadata')
+    
+    @metadata.setter
+    def metadata(self, new_val):
+        # type: (Any) -> None
+        self._metadata = new_val
+    
+    @property
+    def status(self):
+        # type: () -> Any
+        return self._property_impl('status')
+    
+    @status.setter
+    def status(self, new_val):
+        # type: (Optional[Any]) -> None
+        self._status = new_val
+    
+    @property
+    def spec(self):
+        # type: () -> Spec
+        return self._property_impl('spec')
+    
+    @spec.setter
+    def spec(self, new_val):
+        # type: (Spec) -> None
+        self._spec = new_val
diff --git a/src/pybind/mgr/rook/rook-client-python/rook_client/ceph/cephnfs.py b/src/pybind/mgr/rook/rook-client-python/rook_client/ceph/cephnfs.py
new file mode 100644
index 000000000..c46533ec9
--- /dev/null
+++ b/src/pybind/mgr/rook/rook-client-python/rook_client/ceph/cephnfs.py
@@ -0,0 +1,206 @@
+"""
+This file is automatically generated.
+Do not modify.
+"""
+
+try:
+    from typing import Any, Optional, Union, List
+except ImportError:
+    pass
+
+from .._helper import _omit, CrdObject, CrdObjectList, CrdClass
+
+class Rados(CrdObject):
+    _properties = [
+        ('pool', 'pool', str, False, False),
+        ('namespace', 'namespace', str, False, False)
+    ]        
+
+    def __init__(self,
+                 pool=_omit,  # type: Optional[str]
+                 namespace=_omit,  # type: Optional[str]
+                 ):
+        super(Rados, self).__init__(
+            pool=pool,
+            namespace=namespace,
+        )
+
+    @property
+    def pool(self):
+        # type: () -> str
+        return self._property_impl('pool')
+    
+    @pool.setter
+    def pool(self, new_val):
+        # type: (Optional[str]) -> None
+        self._pool = new_val
+    
+    @property
+    def namespace(self):
+        # type: () -> str
+        return self._property_impl('namespace')
+    
+    @namespace.setter
+    def namespace(self, new_val):
+        # type: (Optional[str]) -> None
+        self._namespace = new_val
+
+
+class Server(CrdObject):
+    _properties = [
+        ('active', 'active', int, False, False),
+        ('annotations', 'annotations', object, False, False),
+        ('placement', 'placement', object, False, False),
+        ('resources', 'resources', object, False, False)
+    ]        
+
+    def __init__(self,
+                 active=_omit,  # type: Optional[int]
+                 annotations=_omit,  # type: Optional[Any]
+                 placement=_omit,  # type: Optional[Any]
+                 resources=_omit,  # type: Optional[Any]
+                 ):
+        super(Server, self).__init__(
+            active=active,
+            annotations=annotations,
+            placement=placement,
+            resources=resources,
+        )
+
+    @property
+    def active(self):
+        # type: () -> int
+        return self._property_impl('active')
+    
+    @active.setter
+    def active(self, new_val):
+        # type: (Optional[int]) -> None
+        self._active = new_val
+    
+    @property
+    def annotations(self):
+        # type: () -> Any
+        return self._property_impl('annotations')
+    
+    @annotations.setter
+    def annotations(self, new_val):
+        # type: (Optional[Any]) -> None
+        self._annotations = new_val
+    
+    @property
+    def placement(self):
+        # type: () -> Any
+        return self._property_impl('placement')
+    
+    @placement.setter
+    def placement(self, new_val):
+        # type: (Optional[Any]) -> None
+        self._placement = new_val
+    
+    @property
+    def resources(self):
+        # type: () -> Any
+        return self._property_impl('resources')
+    
+    @resources.setter
+    def resources(self, new_val):
+        # type: (Optional[Any]) -> None
+        self._resources = new_val
+
+
+class Spec(CrdObject):
+    _properties = [
+        ('rados', 'rados', Rados, False, False),
+        ('server', 'server', Server, False, False)
+    ]        
+
+    def __init__(self,
+                 rados=_omit,  # type: Optional[Rados]
+                 server=_omit,  # type: Optional[Server]
+                 ):
+        super(Spec, self).__init__(
+            rados=rados,
+            server=server,
+        )
+
+    @property
+    def rados(self):
+        # type: () -> Rados
+        return self._property_impl('rados')
+    
+    @rados.setter
+    def rados(self, new_val):
+        # type: (Optional[Rados]) -> None
+        self._rados = new_val
+    
+    @property
+    def server(self):
+        # type: () -> Server
+        return self._property_impl('server')
+    
+    @server.setter
+    def server(self, new_val):
+        # type: (Optional[Server]) -> None
+        self._server = new_val
+
+
+class CephNFS(CrdClass):
+    _properties = [
+        ('apiVersion', 'apiVersion', str, True, False),
+        ('metadata', 'metadata', object, True, False),
+        ('status', 'status', object, False, False),
+        ('spec', 'spec', Spec, True, False)
+    ]        
+
+    def __init__(self,
+                 apiVersion,  # type: str
+                 metadata,  # type: Any
+                 spec,  # type: Spec
+                 status=_omit,  # type: Optional[Any]
+                 ):
+        super(CephNFS, self).__init__(
+            apiVersion=apiVersion,
+            metadata=metadata,
+            spec=spec,
+            status=status,
+        )
+
+    @property
+    def apiVersion(self):
+        # type: () -> str
+        return self._property_impl('apiVersion')
+    
+    @apiVersion.setter
+    def apiVersion(self, new_val):
+        # type: (str) -> None
+        self._apiVersion = new_val
+    
+    @property
+    def metadata(self):
+        # type: () -> Any
+        return self._property_impl('metadata')
+    
+    @metadata.setter
+    def metadata(self, new_val):
+        # type: (Any) -> None
+        self._metadata = new_val
+    
+    @property
+    def status(self):
+        # type: () -> Any
+        return self._property_impl('status')
+    
+    @status.setter
+    def status(self, new_val):
+        # type: (Optional[Any]) -> None
+        self._status = new_val
+    
+    @property
+    def spec(self):
+        # type: () -> Spec
+        return self._property_impl('spec')
+    
+    @spec.setter
+    def spec(self, new_val):
+        # type: (Spec) -> None
+        self._spec = new_val
diff --git a/src/pybind/mgr/rook/rook-client-python/rook_client/ceph/cephobjectstore.py b/src/pybind/mgr/rook/rook-client-python/rook_client/ceph/cephobjectstore.py
new file mode 100644
index 000000000..025080929
--- /dev/null
+++ b/src/pybind/mgr/rook/rook-client-python/rook_client/ceph/cephobjectstore.py
@@ -0,0 +1,405 @@
+"""
+This file is automatically generated.
+Do not modify.
+"""
+
+try:
+    from typing import Any, Optional, Union, List
+except ImportError:
+    pass
+
+from .._helper import _omit, CrdObject, CrdObjectList, CrdClass
+
+class Gateway(CrdObject):
+    _properties = [
+        ('type', 'type', str, False, False),
+        ('sslCertificateRef', 'sslCertificateRef', object, False, False),
+        ('port', 'port', int, False, False),
+        ('securePort', 'securePort', object, False, False),
+        ('instances', 'instances', int, False, False),
+        ('annotations', 'annotations', object, False, False),
+        ('placement', 'placement', object, False, False),
+        ('resources', 'resources', object, False, False)
+    ]        
+
+    def __init__(self,
+                 type=_omit,  # type: Optional[str]
+                 sslCertificateRef=_omit,  # type: Optional[Any]
+                 port=_omit,  # type: Optional[int]
+                 securePort=_omit,  # type: Optional[Any]
+                 instances=_omit,  # type: Optional[int]
+                 annotations=_omit,  # type: Optional[Any]
+                 placement=_omit,  # type: Optional[Any]
+                 resources=_omit,  # type: Optional[Any]
+                 ):
+        super(Gateway, self).__init__(
+            type=type,
+            sslCertificateRef=sslCertificateRef,
+            port=port,
+            securePort=securePort,
+            instances=instances,
+            annotations=annotations,
+            placement=placement,
+            resources=resources,
+        )
+
+    @property
+    def type(self):
+        # type: () -> str
+        return self._property_impl('type')
+    
+    @type.setter
+    def type(self, new_val):
+        # type: (Optional[str]) -> None
+        self._type = new_val
+    
+    @property
+    def sslCertificateRef(self):
+        # type: () -> Any
+        return self._property_impl('sslCertificateRef')
+    
+    @sslCertificateRef.setter
+    def sslCertificateRef(self, new_val):
+        # type: (Optional[Any]) -> None
+        self._sslCertificateRef = new_val
+    
+    @property
+    def port(self):
+        # type: () -> int
+        return self._property_impl('port')
+    
+    @port.setter
+    def port(self, new_val):
+        # type: (Optional[int]) -> None
+        self._port = new_val
+    
+    @property
+    def securePort(self):
+        # type: () -> Any
+        return self._property_impl('securePort')
+    
+    @securePort.setter
+    def securePort(self, new_val):
+        # type: (Optional[Any]) -> None
+        self._securePort = new_val
+    
+    @property
+    def instances(self):
+        # type: () -> int
+        return self._property_impl('instances')
+    
+    @instances.setter
+    def instances(self, new_val):
+        # type: (Optional[int]) -> None
+        self._instances = new_val
+    
+    @property
+    def annotations(self):
+        # type: () -> Any
+        return self._property_impl('annotations')
+    
+    @annotations.setter
+    def annotations(self, new_val):
+        # type: (Optional[Any]) -> None
+        self._annotations = new_val
+    
+    @property
+    def placement(self):
+        # type: () -> Any
+        return self._property_impl('placement')
+    
+    @placement.setter
+    def placement(self, new_val):
+        # type: (Optional[Any]) -> None
+        self._placement = new_val
+    
+    @property
+    def resources(self):
+        # type: () -> Any
+        return self._property_impl('resources')
+    
+    @resources.setter
+    def resources(self, new_val):
+        # type: (Optional[Any]) -> None
+        self._resources = new_val
+
+
+class Replicated(CrdObject):
+    _properties = [
+        ('size', 'size', int, False, False)
+    ]        
+
+    def __init__(self,
+                 size=_omit,  # type: Optional[int]
+                 ):
+        super(Replicated, self).__init__(
+            size=size,
+        )
+
+    @property
+    def size(self):
+        # type: () -> int
+        return self._property_impl('size')
+    
+    @size.setter
+    def size(self, new_val):
+        # type: (Optional[int]) -> None
+        self._size = new_val
+
+
+class ErasureCoded(CrdObject):
+    _properties = [
+        ('dataChunks', 'dataChunks', int, False, False),
+        ('codingChunks', 'codingChunks', int, False, False)
+    ]        
+
+    def __init__(self,
+                 dataChunks=_omit,  # type: Optional[int]
+                 codingChunks=_omit,  # type: Optional[int]
+                 ):
+        super(ErasureCoded, self).__init__(
+            dataChunks=dataChunks,
+            codingChunks=codingChunks,
+        )
+
+    @property
+    def dataChunks(self):
+        # type: () -> int
+        return self._property_impl('dataChunks')
+    
+    @dataChunks.setter
+    def dataChunks(self, new_val):
+        # type: (Optional[int]) -> None
+        self._dataChunks = new_val
+    
+    @property
+    def codingChunks(self):
+        # type: () -> int
+        return self._property_impl('codingChunks')
+    
+    @codingChunks.setter
+    def codingChunks(self, new_val):
+        # type: (Optional[int]) -> None
+        self._codingChunks = new_val
+
+
+class MetadataPool(CrdObject):
+    _properties = [
+        ('failureDomain', 'failureDomain', str, False, False),
+        ('replicated', 'replicated', Replicated, False, False),
+        ('erasureCoded', 'erasureCoded', ErasureCoded, False, False)
+    ]        
+
+    def __init__(self,
+                 failureDomain=_omit,  # type: Optional[str]
+                 replicated=_omit,  # type: Optional[Replicated]
+                 erasureCoded=_omit,  # type: Optional[ErasureCoded]
+                 ):
+        super(MetadataPool, self).__init__(
+            failureDomain=failureDomain,
+            replicated=replicated,
+            erasureCoded=erasureCoded,
+        )
+
+    @property
+    def failureDomain(self):
+        # type: () -> str
+        return self._property_impl('failureDomain')
+    
+    @failureDomain.setter
+    def failureDomain(self, new_val):
+        # type: (Optional[str]) -> None
+        self._failureDomain = new_val
+    
+    @property
+    def replicated(self):
+        # type: () -> Replicated
+        return self._property_impl('replicated')
+    
+    @replicated.setter
+    def replicated(self, new_val):
+        # type: (Optional[Replicated]) -> None
+        self._replicated = new_val
+    
+    @property
+    def erasureCoded(self):
+        # type: () -> ErasureCoded
+        return self._property_impl('erasureCoded')
+    
+    @erasureCoded.setter
+    def erasureCoded(self, new_val):
+        # type: (Optional[ErasureCoded]) -> None
+        self._erasureCoded = new_val
+
+
+class DataPool(CrdObject):
+    _properties = [
+        ('failureDomain', 'failureDomain', str, False, False),
+        ('replicated', 'replicated', Replicated, False, False),
+        ('erasureCoded', 'erasureCoded', ErasureCoded, False, False)
+    ]        
+
+    def __init__(self,
+                 failureDomain=_omit,  # type: Optional[str]
+                 replicated=_omit,  # type: Optional[Replicated]
+                 erasureCoded=_omit,  # type: Optional[ErasureCoded]
+                 ):
+        super(DataPool, self).__init__(
+            failureDomain=failureDomain,
+            replicated=replicated,
+            erasureCoded=erasureCoded,
+        )
+
+    @property
+    def failureDomain(self):
+        # type: () -> str
+        return self._property_impl('failureDomain')
+    
+    @failureDomain.setter
+    def failureDomain(self, new_val):
+        # type: (Optional[str]) -> None
+        self._failureDomain = new_val
+    
+    @property
+    def replicated(self):
+        # type: () -> Replicated
+        return self._property_impl('replicated')
+    
+    @replicated.setter
+    def replicated(self, new_val):
+        # type: (Optional[Replicated]) -> None
+        self._replicated = new_val
+    
+    @property
+    def erasureCoded(self):
+        # type: () -> ErasureCoded
+        return self._property_impl('erasureCoded')
+    
+    @erasureCoded.setter
+    def erasureCoded(self, new_val):
+        # type: (Optional[ErasureCoded]) -> None
+        self._erasureCoded = new_val
+
+
+class Spec(CrdObject):
+    _properties = [
+        ('gateway', 'gateway', Gateway, False, False),
+        ('metadataPool', 'metadataPool', MetadataPool, False, False),
+        ('dataPool', 'dataPool', DataPool, False, False),
+        ('preservePoolsOnDelete', 'preservePoolsOnDelete', bool, False, False)
+    ]        
+
+    def __init__(self,
+                 gateway=_omit,  # type: Optional[Gateway]
+                 metadataPool=_omit,  # type: Optional[MetadataPool]
+                 dataPool=_omit,  # type: Optional[DataPool]
+                 preservePoolsOnDelete=_omit,  # type: Optional[bool]
+                 ):
+        super(Spec, self).__init__(
+            gateway=gateway,
+            metadataPool=metadataPool,
+            dataPool=dataPool,
+            preservePoolsOnDelete=preservePoolsOnDelete,
+        )
+
+    @property
+    def gateway(self):
+        # type: () -> Gateway
+        return self._property_impl('gateway')
+    
+    @gateway.setter
+    def gateway(self, new_val):
+        # type: (Optional[Gateway]) -> None
+        self._gateway = new_val
+    
+    @property
+    def metadataPool(self):
+        # type: () -> MetadataPool
+        return self._property_impl('metadataPool')
+    
+    @metadataPool.setter
+    def metadataPool(self, new_val):
+        # type: (Optional[MetadataPool]) -> None
+        self._metadataPool = new_val
+    
+    @property
+    def dataPool(self):
+        # type: () -> DataPool
+        return self._property_impl('dataPool')
+    
+    @dataPool.setter
+    def dataPool(self, new_val):
+        # type: (Optional[DataPool]) -> None
+        self._dataPool = new_val
+    
+    @property
+    def preservePoolsOnDelete(self):
+        # type: () -> bool
+        return self._property_impl('preservePoolsOnDelete')
+    
+    @preservePoolsOnDelete.setter
+    def preservePoolsOnDelete(self, new_val):
+        # type: (Optional[bool]) -> None
+        self._preservePoolsOnDelete = new_val
+
+
+class CephObjectStore(CrdClass):
+    _properties = [
+        ('apiVersion', 'apiVersion', str, True, False),
+        ('metadata', 'metadata', object, True, False),
+        ('status', 'status', object, False, False),
+        ('spec', 'spec', Spec, True, False)
+    ]        
+
+    def __init__(self,
+                 apiVersion,  # type: str
+                 metadata,  # type: Any
+                 spec,  # type: Spec
+                 status=_omit,  # type: Optional[Any]
+                 ):
+        super(CephObjectStore, self).__init__(
+            apiVersion=apiVersion,
+            metadata=metadata,
+            spec=spec,
+            status=status,
+        )
+
+    @property
+    def apiVersion(self):
+        # type: () -> str
+        return self._property_impl('apiVersion')
+    
+    @apiVersion.setter
+    def apiVersion(self, new_val):
+        # type: (str) -> None
+        self._apiVersion = new_val
+    
+    @property
+    def metadata(self):
+        # type: () -> Any
+        return self._property_impl('metadata')
+    
+    @metadata.setter
+    def metadata(self, new_val):
+        # type: (Any) -> None
+        self._metadata = new_val
+    
+    @property
+    def status(self):
+        # type: () -> Any
+        return self._property_impl('status')
+    
+    @status.setter
+    def status(self, new_val):
+        # type: (Optional[Any]) -> None
+        self._status = new_val
+    
+    @property
+    def spec(self):
+        # type: () -> Spec
+        return self._property_impl('spec')
+    
+    @spec.setter
+    def spec(self, new_val):
+        # type: (Spec) -> None
+        self._spec = new_val
diff --git a/src/pybind/mgr/rook/rook-client-python/rook_client/edgefs/__init__.py b/src/pybind/mgr/rook/rook-client-python/rook_client/edgefs/__init__.py
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/src/pybind/mgr/rook/rook-client-python/rook_client/edgefs/__init__.py
diff --git a/src/pybind/mgr/rook/rook-client-python/rook_client/edgefs/cluster.py b/src/pybind/mgr/rook/rook-client-python/rook_client/edgefs/cluster.py
new file mode 100644
index 000000000..18501bcc2
--- /dev/null
+++ b/src/pybind/mgr/rook/rook-client-python/rook_client/edgefs/cluster.py
@@ -0,0 +1,285 @@
+"""
+This file is automatically generated.
+Do not modify.
+"""
+
+try:
+    from typing import Any, Optional, Union, List
+except ImportError:
+    pass
+
+from .._helper import _omit, CrdObject, CrdObjectList, CrdClass
+
+class Dashboard(CrdObject):
+    _properties = [
+        ('localAddr', 'localAddr', str, False, False)
+    ]        
+
+    def __init__(self,
+                 localAddr=_omit,  # type: Optional[str]
+                 ):
+        super(Dashboard, self).__init__(
+            localAddr=localAddr,
+        )
+
+    @property
+    def localAddr(self):
+        # type: () -> str
+        return self._property_impl('localAddr')
+    
+    @localAddr.setter
+    def localAddr(self, new_val):
+        # type: (Optional[str]) -> None
+        self._localAddr = new_val
+
+
+class Network(CrdObject):
+    _properties = [
+        ('serverIfName', 'serverIfName', str, False, False),
+        ('brokerIfName', 'brokerIfName', str, False, False)
+    ]        
+
+    def __init__(self,
+                 serverIfName=_omit,  # type: Optional[str]
+                 brokerIfName=_omit,  # type: Optional[str]
+                 ):
+        super(Network, self).__init__(
+            serverIfName=serverIfName,
+            brokerIfName=brokerIfName,
+        )
+
+    @property
+    def serverIfName(self):
+        # type: () -> str
+        return self._property_impl('serverIfName')
+    
+    @serverIfName.setter
+    def serverIfName(self, new_val):
+        # type: (Optional[str]) -> None
+        self._serverIfName = new_val
+    
+    @property
+    def brokerIfName(self):
+        # type: () -> str
+        return self._property_impl('brokerIfName')
+    
+    @brokerIfName.setter
+    def brokerIfName(self, new_val):
+        # type: (Optional[str]) -> None
+        self._brokerIfName = new_val
+
+
+class NodesList(CrdObjectList):
+    _items_type = None
+
+
+class Storage(CrdObject):
+    _properties = [
+        ('nodes', 'nodes', NodesList, False, False),
+        ('useAllDevices', 'useAllDevices', object, False, False),
+        ('useAllNodes', 'useAllNodes', bool, False, False)
+    ]        
+
+    def __init__(self,
+                 nodes=_omit,  # type: Optional[Union[List[Any], CrdObjectList]]
+                 useAllDevices=_omit,  # type: Optional[Any]
+                 useAllNodes=_omit,  # type: Optional[bool]
+                 ):
+        super(Storage, self).__init__(
+            nodes=nodes,
+            useAllDevices=useAllDevices,
+            useAllNodes=useAllNodes,
+        )
+
+    @property
+    def nodes(self):
+        # type: () -> Union[List[Any], CrdObjectList]
+        return self._property_impl('nodes')
+    
+    @nodes.setter
+    def nodes(self, new_val):
+        # type: (Optional[Union[List[Any], CrdObjectList]]) -> None
+        self._nodes = new_val
+    
+    @property
+    def useAllDevices(self):
+        # type: () -> Any
+        return self._property_impl('useAllDevices')
+    
+    @useAllDevices.setter
+    def useAllDevices(self, new_val):
+        # type: (Optional[Any]) -> None
+        self._useAllDevices = new_val
+    
+    @property
+    def useAllNodes(self):
+        # type: () -> bool
+        return self._property_impl('useAllNodes')
+    
+    @useAllNodes.setter
+    def useAllNodes(self, new_val):
+        # type: (Optional[bool]) -> None
+        self._useAllNodes = new_val
+
+
+class Spec(CrdObject):
+    _properties = [
+        ('edgefsImageName', 'edgefsImageName', str, True, False),
+        ('dataDirHostPath', 'dataDirHostPath', str, True, False),
+        ('devicesResurrectMode', 'devicesResurrectMode', str, False, False),
+        ('dashboard', 'dashboard', Dashboard, False, False),
+        ('network', 'network', Network, False, False),
+        ('skipHostPrepare', 'skipHostPrepare', bool, False, False),
+        ('storage', 'storage', Storage, False, False)
+    ]        
+
+    def __init__(self,
+                 edgefsImageName,  # type: str
+                 dataDirHostPath,  # type: str
+                 devicesResurrectMode=_omit,  # type: Optional[str]
+                 dashboard=_omit,  # type: Optional[Dashboard]
+                 network=_omit,  # type: Optional[Network]
+                 skipHostPrepare=_omit,  # type: Optional[bool]
+                 storage=_omit,  # type: Optional[Storage]
+                 ):
+        super(Spec, self).__init__(
+            edgefsImageName=edgefsImageName,
+            dataDirHostPath=dataDirHostPath,
+            devicesResurrectMode=devicesResurrectMode,
+            dashboard=dashboard,
+            network=network,
+            skipHostPrepare=skipHostPrepare,
+            storage=storage,
+        )
+
+    @property
+    def edgefsImageName(self):
+        # type: () -> str
+        return self._property_impl('edgefsImageName')
+    
+    @edgefsImageName.setter
+    def edgefsImageName(self, new_val):
+        # type: (str) -> None
+        self._edgefsImageName = new_val
+    
+    @property
+    def dataDirHostPath(self):
+        # type: () -> str
+        return self._property_impl('dataDirHostPath')
+    
+    @dataDirHostPath.setter
+    def dataDirHostPath(self, new_val):
+        # type: (str) -> None
+        self._dataDirHostPath = new_val
+    
+    @property
+    def devicesResurrectMode(self):
+        # type: () -> str
+        return self._property_impl('devicesResurrectMode')
+    
+    @devicesResurrectMode.setter
+    def devicesResurrectMode(self, new_val):
+        # type: (Optional[str]) -> None
+        self._devicesResurrectMode = new_val
+    
+    @property
+    def dashboard(self):
+        # type: () -> Dashboard
+        return self._property_impl('dashboard')
+    
+    @dashboard.setter
+    def dashboard(self, new_val):
+        # type: (Optional[Dashboard]) -> None
+        self._dashboard = new_val
+    
+    @property
+    def network(self):
+        # type: () -> Network
+        return self._property_impl('network')
+    
+    @network.setter
+    def network(self, new_val):
+        # type: (Optional[Network]) -> None
+        self._network = new_val
+    
+    @property
+    def skipHostPrepare(self):
+        # type: () -> bool
+        return self._property_impl('skipHostPrepare')
+    
+    @skipHostPrepare.setter
+    def skipHostPrepare(self, new_val):
+        # type: (Optional[bool]) -> None
+        self._skipHostPrepare = new_val
+    
+    @property
+    def storage(self):
+        # type: () -> Storage
+        return self._property_impl('storage')
+    
+    @storage.setter
+    def storage(self, new_val):
+        # type: (Optional[Storage]) -> None
+        self._storage = new_val
+
+
+class Cluster(CrdClass):
+    _properties = [
+        ('apiVersion', 'apiVersion', str, True, False),
+        ('metadata', 'metadata', object, True, False),
+        ('status', 'status', object, False, False),
+        ('spec', 'spec', Spec, True, False)
+    ]        
+
+    def __init__(self,
+                 apiVersion,  # type: str
+                 metadata,  # type: Any
+                 spec,  # type: Spec
+                 status=_omit,  # type: Optional[Any]
+                 ):
+        super(Cluster, self).__init__(
+            apiVersion=apiVersion,
+            metadata=metadata,
+            spec=spec,
+            status=status,
+        )
+
+    @property
+    def apiVersion(self):
+        # type: () -> str
+        return self._property_impl('apiVersion')
+    
+    @apiVersion.setter
+    def apiVersion(self, new_val):
+        # type: (str) -> None
+        self._apiVersion = new_val
+    
+    @property
+    def metadata(self):
+        # type: () -> Any
+        return self._property_impl('metadata')
+    
+    @metadata.setter
+    def metadata(self, new_val):
+        # type: (Any) -> None
+        self._metadata = new_val
+    
+    @property
+    def status(self):
+        # type: () -> Any
+        return self._property_impl('status')
+    
+    @status.setter
+    def status(self, new_val):
+        # type: (Optional[Any]) -> None
+        self._status = new_val
+    
+    @property
+    def spec(self):
+        # type: () -> Spec
+        return self._property_impl('spec')
+    
+    @spec.setter
+    def spec(self, new_val):
+        # type: (Spec) -> None
+        self._spec = new_val
diff --git a/src/pybind/mgr/rook/rook-client-python/rook_client/edgefs/isgw.py b/src/pybind/mgr/rook/rook-client-python/rook_client/edgefs/isgw.py
new file mode 100644
index 000000000..eda8d32e4
--- /dev/null
+++ b/src/pybind/mgr/rook/rook-client-python/rook_client/edgefs/isgw.py
@@ -0,0 +1,161 @@
+"""
+This file is automatically generated.
+Do not modify.
+"""
+
+try:
+    from typing import Any, Optional, Union, List
+except ImportError:
+    pass
+
+from .._helper import _omit, CrdObject, CrdObjectList, CrdClass
+
+class ClientsList(CrdObjectList):
+    _items_type = str
+
+
+class Config(CrdObject):
+    _properties = [
+        ('server', 'server', str, False, False),
+        ('clients', 'clients', ClientsList, False, False)
+    ]        
+
+    def __init__(self,
+                 server=_omit,  # type: Optional[str]
+                 clients=_omit,  # type: Optional[Union[List[str], CrdObjectList]]
+                 ):
+        super(Config, self).__init__(
+            server=server,
+            clients=clients,
+        )
+
+    @property
+    def server(self):
+        # type: () -> str
+        return self._property_impl('server')
+    
+    @server.setter
+    def server(self, new_val):
+        # type: (Optional[str]) -> None
+        self._server = new_val
+    
+    @property
+    def clients(self):
+        # type: () -> Union[List[str], CrdObjectList]
+        return self._property_impl('clients')
+    
+    @clients.setter
+    def clients(self, new_val):
+        # type: (Optional[Union[List[str], CrdObjectList]]) -> None
+        self._clients = new_val
+
+
+class Spec(CrdObject):
+    _properties = [
+        ('direction', 'direction', str, True, False),
+        ('remoteURL', 'remoteURL', str, False, False),
+        ('config', 'config', Config, False, False)
+    ]        
+
+    def __init__(self,
+                 direction,  # type: str
+                 remoteURL=_omit,  # type: Optional[str]
+                 config=_omit,  # type: Optional[Config]
+                 ):
+        super(Spec, self).__init__(
+            direction=direction,
+            remoteURL=remoteURL,
+            config=config,
+        )
+
+    @property
+    def direction(self):
+        # type: () -> str
+        return self._property_impl('direction')
+    
+    @direction.setter
+    def direction(self, new_val):
+        # type: (str) -> None
+        self._direction = new_val
+    
+    @property
+    def remoteURL(self):
+        # type: () -> str
+        return self._property_impl('remoteURL')
+    
+    @remoteURL.setter
+    def remoteURL(self, new_val):
+        # type: (Optional[str]) -> None
+        self._remoteURL = new_val
+    
+    @property
+    def config(self):
+        # type: () -> Config
+        return self._property_impl('config')
+    
+    @config.setter
+    def config(self, new_val):
+        # type: (Optional[Config]) -> None
+        self._config = new_val
+
+
+class ISGW(CrdClass):
+    _properties = [
+        ('apiVersion', 'apiVersion', str, True, False),
+        ('metadata', 'metadata', object, True, False),
+        ('status', 'status', object, False, False),
+        ('spec', 'spec', Spec, True, False)
+    ]        
+
+    def __init__(self,
+                 apiVersion,  # type: str
+                 metadata,  # type: Any
+                 spec,  # type: Spec
+                 status=_omit,  # type: Optional[Any]
+                 ):
+        super(ISGW, self).__init__(
+            apiVersion=apiVersion,
+            metadata=metadata,
+            spec=spec,
+            status=status,
+        )
+
+    @property
+    def apiVersion(self):
+        # type: () -> str
+        return self._property_impl('apiVersion')
+    
+    @apiVersion.setter
+    def apiVersion(self, new_val):
+        # type: (str) -> None
+        self._apiVersion = new_val
+    
+    @property
+    def metadata(self):
+        # type: () -> Any
+        return self._property_impl('metadata')
+    
+    @metadata.setter
+    def metadata(self, new_val):
+        # type: (Any) -> None
+        self._metadata = new_val
+    
+    @property
+    def status(self):
+        # type: () -> Any
+        return self._property_impl('status')
+    
+    @status.setter
+    def status(self, new_val):
+        # type: (Optional[Any]) -> None
+        self._status = new_val
+    
+    @property
+    def spec(self):
+        # type: () -> Spec
+        return self._property_impl('spec')
+    
+    @spec.setter
+    def spec(self, new_val):
+        # type: (Spec) -> None
+        self._spec = new_val
diff --git a/src/pybind/mgr/rook/rook-client-python/rook_client/edgefs/nfs.py b/src/pybind/mgr/rook/rook-client-python/rook_client/edgefs/nfs.py
new file mode 100644
index 000000000..ed8a9d8ed
--- /dev/null
+++ b/src/pybind/mgr/rook/rook-client-python/rook_client/edgefs/nfs.py
@@ -0,0 +1,95 @@
+"""
+This file is automatically generated.
+Do not modify.
+"""
+
+try:
+    from typing import Any, Optional, Union, List
+except ImportError:
+    pass
+
+from .._helper import _omit, CrdObject, CrdObjectList, CrdClass
+
+class Spec(CrdObject):
+    _properties = [
+        ('instances', 'instances', int, True, False)
+    ]        
+
+    def __init__(self,
+                 instances,  # type: int
+                 ):
+        super(Spec, self).__init__(
+            instances=instances,
+        )
+
+    @property
+    def instances(self):
+        # type: () -> int
+        return self._property_impl('instances')
+    
+    @instances.setter
+    def instances(self, new_val):
+        # type: (int) -> None
+        self._instances = new_val
+
+
+class NFS(CrdClass):
+    _properties = [
+        ('apiVersion', 'apiVersion', str, True, False),
+        ('metadata', 'metadata', object, True, False),
+        ('status', 'status', object, False, False),
+        ('spec', 'spec', Spec, True, False)
+    ]        
+
+    def __init__(self,
+                 apiVersion,  # type: str
+                 metadata,  # type: Any
+                 spec,  # type: Spec
+                 status=_omit,  # type: Optional[Any]
+                 ):
+        super(NFS, self).__init__(
+            apiVersion=apiVersion,
+            metadata=metadata,
+            spec=spec,
+            status=status,
+        )
+
+    @property
+    def apiVersion(self):
+        # type: () -> str
+        return self._property_impl('apiVersion')
+    
+    @apiVersion.setter
+    def apiVersion(self, new_val):
+        # type: (str) -> None
+        self._apiVersion = new_val
+    
+    @property
+    def metadata(self):
+        # type: () -> Any
+        return self._property_impl('metadata')
+    
+    @metadata.setter
+    def metadata(self, new_val):
+        # type: (Any) -> None
+        self._metadata = new_val
+    
+    @property
+    def status(self):
+        # type: () -> Any
+        return self._property_impl('status')
+    
+    @status.setter
+    def status(self, new_val):
+        # type: (Optional[Any]) -> None
+        self._status = new_val
+    
+    @property
+    def spec(self):
+        # type: () -> Spec
+        return self._property_impl('spec')
+    
+    @spec.setter
+    def spec(self, new_val):
+        # type: (Spec) -> None
+        self._spec = new_val
diff --git a/src/pybind/mgr/rook/rook-client-python/rook_client/edgefs/s3.py b/src/pybind/mgr/rook/rook-client-python/rook_client/edgefs/s3.py
new file mode 100644
index 000000000..a2995e1ea
--- /dev/null
+++ b/src/pybind/mgr/rook/rook-client-python/rook_client/edgefs/s3.py
@@ -0,0 +1,95 @@
+"""
+This file is automatically generated.
+Do not modify.
+"""
+
+try:
+    from typing import Any, Optional, Union, List
+except ImportError:
+    pass
+
+from .._helper import _omit, CrdObject, CrdObjectList, CrdClass
+
+class Spec(CrdObject):
+    _properties = [
+        ('instances', 'instances', int, True, False)
+    ]        
+
+    def __init__(self,
+                 instances,  # type: int
+                 ):
+        super(Spec, self).__init__(
+            instances=instances,
+        )
+
+    @property
+    def instances(self):
+        # type: () -> int
+        return self._property_impl('instances')
+    
+    @instances.setter
+    def instances(self, new_val):
+        # type: (int) -> None
+        self._instances = new_val
+
+
+class S3(CrdClass):
+    _properties = [
+        ('apiVersion', 'apiVersion', str, True, False),
+        ('metadata', 'metadata', object, True, False),
+        ('status', 'status', object, False, False),
+        ('spec', 'spec', Spec, True, False)
+    ]        
+
+    def __init__(self,
+                 apiVersion,  # type: str
+                 metadata,  # type: Any
+                 spec,  # type: Spec
+                 status=_omit,  # type: Optional[Any]
+                 ):
+        super(S3, self).__init__(
+            apiVersion=apiVersion,
+            metadata=metadata,
+            spec=spec,
+            status=status,
+        )
+
+    @property
+    def apiVersion(self):
+        # type: () -> str
+        return self._property_impl('apiVersion')
+    
+    @apiVersion.setter
+    def apiVersion(self, new_val):
+        # type: (str) -> None
+        self._apiVersion = new_val
+    
+    @property
+    def metadata(self):
+        # type: () -> Any
+        return self._property_impl('metadata')
+    
+    @metadata.setter
+    def metadata(self, new_val):
+        # type: (Any) -> None
+        self._metadata = new_val
+    
+    @property
+    def status(self):
+        # type: () -> Any
+        return self._property_impl('status')
+    
+    @status.setter
+    def status(self, new_val):
+        # type: (Optional[Any]) -> None
+        self._status = new_val
+    
+    @property
+    def spec(self):
+        # type: () -> Spec
+        return self._property_impl('spec')
+    
+    @spec.setter
+    def spec(self, new_val):
+        # type: (Spec) -> None
+        self._spec = new_val
diff --git a/src/pybind/mgr/rook/rook-client-python/rook_client/edgefs/s3x.py b/src/pybind/mgr/rook/rook-client-python/rook_client/edgefs/s3x.py
new file mode 100644
index 000000000..0326a8eff
--- /dev/null
+++ b/src/pybind/mgr/rook/rook-client-python/rook_client/edgefs/s3x.py
@@ -0,0 +1,95 @@
+"""
+This file is automatically generated.
+Do not modify.
+"""
+
+try:
+    from typing import Any, Optional, Union, List
+except ImportError:
+    pass
+
+from .._helper import _omit, CrdObject, CrdObjectList, CrdClass
+
+class Spec(CrdObject):
+    _properties = [
+        ('instances', 'instances', int, True, False)
+    ]        
+
+    def __init__(self,
+                 instances,  # type: int
+                 ):
+        super(Spec, self).__init__(
+            instances=instances,
+        )
+
+    @property
+    def instances(self):
+        # type: () -> int
+        return self._property_impl('instances')
+    
+    @instances.setter
+    def instances(self, new_val):
+        # type: (int) -> None
+        self._instances = new_val
+
+
+class S3X(CrdClass):
+    _properties = [
+        ('apiVersion', 'apiVersion', str, True, False),
+        ('metadata', 'metadata', object, True, False),
+        ('status', 'status', object, False, False),
+        ('spec', 'spec', Spec, True, False)
+    ]        
+
+    def __init__(self,
+                 apiVersion,  # type: str
+                 metadata,  # type: Any
+                 spec,  # type: Spec
+                 status=_omit,  # type: Optional[Any]
+                 ):
+        super(S3X, self).__init__(
+            apiVersion=apiVersion,
+            metadata=metadata,
+            spec=spec,
+            status=status,
+        )
+
+    @property
+    def apiVersion(self):
+        # type: () -> str
+        return self._property_impl('apiVersion')
+    
+    @apiVersion.setter
+    def apiVersion(self, new_val):
+        # type: (str) -> None
+        self._apiVersion = new_val
+    
+    @property
+    def metadata(self):
+        # type: () -> Any
+        return self._property_impl('metadata')
+    
+    @metadata.setter
+    def metadata(self, new_val):
+        # type: (Any) -> None
+        self._metadata = new_val
+    
+    @property
+    def status(self):
+        # type: () -> Any
+        return self._property_impl('status')
+    
+    @status.setter
+    def status(self, new_val):
+        # type: (Optional[Any]) -> None
+        self._status = new_val
+    
+    @property
+    def spec(self):
+        # type: () -> Spec
+        return self._property_impl('spec')
+    
+    @spec.setter
+    def spec(self, new_val):
+        # type: (Spec) -> None
+        self._spec = new_val
diff --git a/src/pybind/mgr/rook/rook-client-python/rook_client/edgefs/swift.py b/src/pybind/mgr/rook/rook-client-python/rook_client/edgefs/swift.py
new file mode 100644
index 000000000..ab4cacb58
--- /dev/null
+++ b/src/pybind/mgr/rook/rook-client-python/rook_client/edgefs/swift.py
@@ -0,0 +1,95 @@
+"""
+This file is automatically generated.
+Do not modify.
+"""
+
+try:
+    from typing import Any, Optional, Union, List
+except ImportError:
+    pass
+
+from .._helper import _omit, CrdObject, CrdObjectList, CrdClass
+
+class Spec(CrdObject):
+    _properties = [
+        ('instances', 'instances', int, True, False)
+    ]        
+
+    def __init__(self,
+                 instances,  # type: int
+                 ):
+        super(Spec, self).__init__(
+            instances=instances,
+        )
+
+    @property
+    def instances(self):
+        # type: () -> int
+        return self._property_impl('instances')
+    
+    @instances.setter
+    def instances(self, new_val):
+        # type: (int) -> None
+        self._instances = new_val
+
+
+class SWIFT(CrdClass):
+    _properties = [
+        ('apiVersion', 'apiVersion', str, True, False),
+        ('metadata', 'metadata', object, True, False),
+        ('status', 'status', object, False, False),
+        ('spec', 'spec', Spec, True, False)
+    ]        
+
+    def __init__(self,
+                 apiVersion,  # type: str
+                 metadata,  # type: Any
+                 spec,  # type: Spec
+                 status=_omit,  # type: Optional[Any]
+                 ):
+        super(SWIFT, self).__init__(
+            apiVersion=apiVersion,
+            metadata=metadata,
+            spec=spec,
+            status=status,
+        )
+
+    @property
+    def apiVersion(self):
+        # type: () -> str
+        return self._property_impl('apiVersion')
+    
+    @apiVersion.setter
+    def apiVersion(self, new_val):
+        # type: (str) -> None
+        self._apiVersion = new_val
+    
+    @property
+    def metadata(self):
+        # type: () -> Any
+        return self._property_impl('metadata')
+    
+    @metadata.setter
+    def metadata(self, new_val):
+        # type: (Any) -> None
+        self._metadata = new_val
+    
+    @property
+    def status(self):
+        # type: () -> Any
+        return self._property_impl('status')
+    
+    @status.setter
+    def status(self, new_val):
+        # type: (Optional[Any]) -> None
+        self._status = new_val
+    
+    @property
+    def spec(self):
+        # type: () -> Spec
+        return self._property_impl('spec')
+    
+    @spec.setter
+    def spec(self, new_val):
+        # type: (Spec) -> None
+        self._spec = new_val
diff --git a/src/pybind/mgr/rook/rook-client-python/rook_client/tests/__init__.py b/src/pybind/mgr/rook/rook-client-python/rook_client/tests/__init__.py
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/src/pybind/mgr/rook/rook-client-python/rook_client/tests/__init__.py
diff --git a/src/pybind/mgr/rook/rook-client-python/rook_client/tests/test_README.py b/src/pybind/mgr/rook/rook-client-python/rook_client/tests/test_README.py
new file mode 100644
index 000000000..aa9261a2a
--- /dev/null
+++ b/src/pybind/mgr/rook/rook-client-python/rook_client/tests/test_README.py
@@ -0,0 +1,29 @@
+def objectstore(api_name, name, namespace, instances):
+    from rook_client.ceph import cephobjectstore as cos
+    rook_os = cos.CephObjectStore(
+        apiVersion=api_name,
+        metadata=dict(
+            name=name,
+            namespace=namespace
+        ),
+        spec=cos.Spec(
+            metadataPool=cos.MetadataPool(
+                failureDomain='host',
+                replicated=cos.Replicated(
+                    size=1
+                )
+            ),
+            dataPool=cos.DataPool(
+                failureDomain='osd',
+                replicated=cos.Replicated(
+                    size=1
+                )
+            ),
+            gateway=cos.Gateway(
+                type='s3',
+                port=80,
+                instances=instances
+            )
+        )
+    )
+    return rook_os.to_json()
diff --git a/src/pybind/mgr/rook/rook-client-python/rook_client/tests/test_examples.py b/src/pybind/mgr/rook/rook-client-python/rook_client/tests/test_examples.py
new file mode 100644
index 000000000..1cfd078a5
--- /dev/null
+++ b/src/pybind/mgr/rook/rook-client-python/rook_client/tests/test_examples.py
@@ -0,0 +1,51 @@
+from os.path import expanduser, dirname, realpath
+
+import yaml
+import pytest
+
+import rook_client
+from rook_client.cassandra.cluster import Cluster as CassandraCluster
+from rook_client.ceph.cephcluster import CephCluster
+from rook_client.ceph.cephfilesystem import CephFilesystem
+from rook_client.ceph.cephnfs import CephNFS
+from rook_client.ceph.cephobjectstore import CephObjectStore
+from rook_client.edgefs.cluster import Cluster as EdgefsCluster
+
+
+def _load_example(crd_base, what):
+    with open(expanduser('{crd_base}/{what}').format(crd_base=crd_base, what=what)) as f:
+        return f.read()
+
+
+@pytest.mark.parametrize(
+    "strict,cls,filename",
+    [
+        (True, CephCluster, "ceph/cluster-external.yaml"),
+        (True, CephCluster, "ceph/cluster-minimal.yaml"),
+        (True, CephCluster, "ceph/cluster-on-pvc.yaml"),
+        (True, CephCluster, "ceph/cluster.yaml"),
+        (True, CephFilesystem, "ceph/filesystem-ec.yaml"),
+        (True, CephFilesystem, "ceph/filesystem-test.yaml"),
+        (True, CephFilesystem, "ceph/filesystem.yaml"),
+        (True, CephObjectStore, "ceph/object-ec.yaml"),
+        (True, CephObjectStore, "ceph/object-openshift.yaml"),
+        (True, CephObjectStore, "ceph/object-test.yaml"),
+        (True, CephObjectStore, "ceph/object.yaml"),
+        (True, CephNFS, "ceph/nfs.yaml"),
+
+        # schema invalid:
+        # (False, CassandraCluster, "cassandra/cluster.yaml"),
+        (False, EdgefsCluster, "edgefs/cluster.yaml"),
+    ],
+)
+def test_exact_match(strict, cls, filename, crd_base):
+    crds = yaml.safe_load_all(_load_example(crd_base, filename))
+    rook_client.STRICT = strict
+    [crd] = [e for e in crds if e.get('kind', '') == cls.__name__]
+
+    c = cls.from_json(crd)
+    assert crd == c.to_json()
+
+
+
+
diff --git a/src/pybind/mgr/rook/rook-client-python/rook_client/tests/test_properties.py b/src/pybind/mgr/rook/rook-client-python/rook_client/tests/test_properties.py
new file mode 100644
index 000000000..24ec38f5d
--- /dev/null
+++ b/src/pybind/mgr/rook/rook-client-python/rook_client/tests/test_properties.py
@@ -0,0 +1,13 @@
+from copy import deepcopy
+
+import pytest
+
+from rook_client.ceph import cephfilesystem as cfs
+
+
+def test_omit():
+    ec = cfs.ErasureCoded()
+    with pytest.raises(AttributeError):
+        ec.codingChunks
+
+    assert not hasattr(ec, 'codingChunks')
diff --git a/src/pybind/mgr/rook/rook-client-python/setup.py b/src/pybind/mgr/rook/rook-client-python/setup.py
new file mode 100644
index 000000000..356a3015c
--- /dev/null
+++ b/src/pybind/mgr/rook/rook-client-python/setup.py
@@ -0,0 +1,17 @@
+from setuptools import setup, find_packages
+
+with open("README.md", "r") as fh:
+    long_description = fh.read()
+
+setup(
+    name='rook-client',
+    version='1.0.0',
+    packages=find_packages(),
+    url='',
+    license='MIT',
+    author='Sebastian Wagner',
+    author_email='swagner@suse.com',
+    description='Client model classes for the CRDs exposed by Rook',
+    long_description=long_description,
+    long_description_content_type="text/markdown",
+)
diff --git a/src/pybind/mgr/rook/rook-client-python/tox.ini b/src/pybind/mgr/rook/rook-client-python/tox.ini
new file mode 100644
index 000000000..0b164c3a7
--- /dev/null
+++ b/src/pybind/mgr/rook/rook-client-python/tox.ini
@@ -0,0 +1,23 @@
+[tox]
+envlist = py2,py36,py37,py38,mypy
+skipsdist = true
+
+
+[testenv]
+deps = -rrequirements.txt
+commands = pytest {posargs}
+
+
+[testenv:mypy]
+basepython = python3
+deps =
+    -rrequirements.txt
+    mypy
+commands = 
+  mkcodes --github --output rook_client/tests/test_\{name\}.py README.md
+  mypy --config-file=mypy.ini \
+      rook_client \
+      conftest.py \
+      generate_model_classes.py \
+      setup.py
+      
\ No newline at end of file
diff --git a/src/pybind/mgr/rook/rook_client/__init__.py b/src/pybind/mgr/rook/rook_client/__init__.py
new file mode 100644
index 000000000..3fa2272dd
--- /dev/null
+++ b/src/pybind/mgr/rook/rook_client/__init__.py
@@ -0,0 +1 @@
+from ._helper import STRICT
\ No newline at end of file
diff --git a/src/pybind/mgr/rook/rook_client/_helper.py b/src/pybind/mgr/rook/rook_client/_helper.py
new file mode 100644
index 000000000..b4c7793f0
--- /dev/null
+++ b/src/pybind/mgr/rook/rook_client/_helper.py
@@ -0,0 +1,108 @@
+import logging
+try:
+    from typing import List, Dict, Any, Optional
+except ImportError:
+    pass
+
+logger = logging.getLogger(__name__)
+
+# Tricking mypy to think `_omit`'s type is NoneType
+# To make us not add things like `Union[Optional[str], OmitType]`
+NoneType = type(None)
+_omit = None  # type: NoneType
+_omit = object()  # type: ignore
+
+
+# Don't add any additionalProperties to objects. Useful for completeness testing
+STRICT = False
+
+
+def _property_from_json(data, breadcrumb, name, py_name, typ, required, nullable):
+    if not required and name not in data:
+        return _omit
+    obj = data[name]
+    if nullable and obj is None:
+        return obj
+    if issubclass(typ, CrdObject) or issubclass(typ, CrdObjectList):
+        return typ.from_json(obj, breadcrumb + '.' + name)
+    return obj
+
+
+class CrdObject(object):
+    _properties = []  # type: List
+
+    def __init__(self, **kwargs):
+        for prop in self._properties:
+            setattr(self, prop[1], kwargs.pop(prop[1]))
+        if kwargs:
+            raise TypeError(
+                '{} got unexpected arguments {}'.format(self.__class__.__name__, kwargs.keys()))
+        self._additionalProperties = {}  # type: Dict[str, Any]
+
+    def _property_impl(self, name):
+        obj = getattr(self, '_' + name)
+        if obj is _omit:
+            raise AttributeError(name + ' not found')
+        return obj
+
+    def _property_to_json(self, name, py_name, typ, required, nullable):
+        obj = getattr(self, '_' + py_name)
+        if issubclass(typ, CrdObject) or issubclass(typ, CrdObjectList):
+            if nullable and obj is None:
+                return obj
+            if not required and obj is _omit:
+                return obj
+            return obj.to_json()
+        else:
+            return obj
+
+    def to_json(self):
+        # type: () -> Dict[str, Any]
+        res = {p[0]: self._property_to_json(*p) for p in self._properties}
+        res.update(self._additionalProperties)
+        return {k: v for k, v in res.items() if v is not _omit}
+
+    @classmethod
+    def from_json(cls, data, breadcrumb=''):
+        try:
+            sanitized = {
+                p[1]: _property_from_json(data, breadcrumb, *p) for p in cls._properties
+            }
+            extra = {k:v for k,v in data.items() if k not in sanitized}
+            ret = cls(**sanitized)
+            ret._additionalProperties = {} if STRICT else extra
+            return ret
+        except (TypeError, AttributeError, KeyError):
+            logger.exception(breadcrumb)
+            raise
+
+
+class CrdClass(CrdObject):
+    @classmethod
+    def from_json(cls, data, breadcrumb=''):
+        kind = data['kind']
+        if kind != cls.__name__:
+            raise ValueError("kind mismatch: {} != {}".format(kind, cls.__name__))
+        return super(CrdClass, cls).from_json(data, breadcrumb)
+
+    def to_json(self):
+        ret = super(CrdClass, self).to_json()
+        ret['kind'] = self.__class__.__name__
+        return ret
+
+
+class CrdObjectList(list):
+    # Py3: Replace `Any` with `TypeVar('T_CrdObject', bound='CrdObject')`
+    _items_type = None  # type: Optional[Any]
+
+    def to_json(self):
+        # type: () -> List
+        if self._items_type is None:
+            return self
+        return [e.to_json() for e in self]
+
+    @classmethod
+    def from_json(cls, data, breadcrumb=''):
+        if cls._items_type is None:
+            return cls(data)
+        return cls(cls._items_type.from_json(e, breadcrumb + '[]') for e in data)
diff --git a/src/pybind/mgr/rook/rook_cluster.py b/src/pybind/mgr/rook/rook_cluster.py
new file mode 100644
index 000000000..323ac6888
--- /dev/null
+++ b/src/pybind/mgr/rook/rook_cluster.py
@@ -0,0 +1,797 @@
+"""
+This module wrap's Rook + Kubernetes APIs to expose the calls
+needed to implement an orchestrator module.  While the orchestrator
+module exposes an async API, this module simply exposes blocking API
+call methods.
+
+This module is runnable outside of ceph-mgr, useful for testing.
+"""
+import datetime
+import threading
+import logging
+import json
+from contextlib import contextmanager
+from time import sleep
+
+import jsonpatch
+from urllib.parse import urljoin
+
+# Optional kubernetes imports to enable MgrModule.can_run
+# to behave cleanly.
+from urllib3.exceptions import ProtocolError
+
+from ceph.deployment.drive_group import DriveGroupSpec
+from ceph.deployment.service_spec import ServiceSpec, NFSServiceSpec, RGWSpec
+from ceph.utils import datetime_now
+from mgr_module import NFS_POOL_NAME
+from mgr_util import merge_dicts
+
+from typing import Optional, TypeVar, List, Callable, Any, cast, Generic, \
+    Iterable, Dict, Iterator, Type
+
+try:
+    from kubernetes import client, watch
+    from kubernetes.client.rest import ApiException
+except ImportError:
+    class ApiException(Exception):  # type: ignore
+        status = 0
+
+from .rook_client.ceph import cephfilesystem as cfs
+from .rook_client.ceph import cephnfs as cnfs
+from .rook_client.ceph import cephobjectstore as cos
+from .rook_client.ceph import cephcluster as ccl
+from .rook_client._helper import CrdClass
+
+import orchestrator
+
+try:
+    from rook.module import RookEnv
+except ImportError:
+    pass  # just used for type checking.
+
+
+T = TypeVar('T')
+FuncT = TypeVar('FuncT', bound=Callable)
+
+CrdClassT = TypeVar('CrdClassT', bound=CrdClass)
+
+
+log = logging.getLogger(__name__)
+
+
+def __urllib3_supports_read_chunked() -> bool:
+    # There is a bug in CentOS 7 as it ships a urllib3 which is lower
+    # than required by kubernetes-client
+    try:
+        from urllib3.response import HTTPResponse
+        return hasattr(HTTPResponse, 'read_chunked')
+    except ImportError:
+        return False
+
+
+_urllib3_supports_read_chunked = __urllib3_supports_read_chunked()
+
+class ApplyException(orchestrator.OrchestratorError):
+    """
+    For failures to update the Rook CRDs, usually indicating
+    some kind of interference between our attempted update
+    and other conflicting activity.
+    """
+
+
+def threaded(f: Callable[..., None]) -> Callable[..., threading.Thread]:
+    def wrapper(*args: Any, **kwargs: Any) -> threading.Thread:
+        t = threading.Thread(target=f, args=args, kwargs=kwargs)
+        t.start()
+        return t
+
+    return cast(Callable[..., threading.Thread], wrapper)
+
+
+class KubernetesResource(Generic[T]):
+    def __init__(self, api_func: Callable, **kwargs: Any) -> None:
+        """
+        Generic kubernetes Resource parent class
+
+        The api fetch and watch methods should be common across resource types,
+
+        Exceptions in the runner thread are propagated to the caller.
+
+        :param api_func: kubernetes client api function that is passed to the watcher
+        :param filter_func: signature: ``(Item) -> bool``.
+        """
+        self.kwargs = kwargs
+        self.api_func = api_func
+
+        # ``_items`` is accessed by different threads. I assume assignment is atomic.
+        self._items: Dict[str, T] = dict()
+        self.thread = None  # type: Optional[threading.Thread]
+        self.exception: Optional[Exception] = None
+        if not _urllib3_supports_read_chunked:
+            logging.info('urllib3 is too old. Fallback to full fetches')
+
+    def _fetch(self) -> str:
+        """ Execute the requested api method as a one-off fetch"""
+        response = self.api_func(**self.kwargs)
+        # metadata is a client.V1ListMeta object type
+        metadata = response.metadata  # type: client.V1ListMeta
+        self._items = {item.metadata.name: item for item in response.items}
+        log.info('Full fetch of {}. result: {}'.format(self.api_func, len(self._items)))
+        return metadata.resource_version
+
+    @property
+    def items(self) -> Iterable[T]:
+        """
+        Returns the items of the request.
+        Creates the watcher as a side effect.
+        :return:
+        """
+        if self.exception:
+            e = self.exception
+            self.exception = None
+            raise e  # Propagate the exception to the user.
+        if not self.thread or not self.thread.is_alive():
+            resource_version = self._fetch()
+            if _urllib3_supports_read_chunked:
+                # Start a thread which will use the kubernetes watch client against a resource
+                log.debug("Attaching resource watcher for k8s {}".format(self.api_func))
+                self.thread = self._watch(resource_version)
+
+        return self._items.values()
+
+    @threaded
+    def _watch(self, res_ver: Optional[str]) -> None:
+        """ worker thread that runs the kubernetes watch """
+
+        self.exception = None
+
+        w = watch.Watch()
+
+        try:
+            # execute generator to continually watch resource for changes
+            for event in w.stream(self.api_func, resource_version=res_ver, watch=True,
+                                  **self.kwargs):
+                self.health = ''
+                item = event['object']
+                try:
+                    name = item.metadata.name
+                except AttributeError:
+                    raise AttributeError(
+                        "{} doesn't contain a metadata.name. Unable to track changes".format(
+                            self.api_func))
+
+                log.info('{} event: {}'.format(event['type'], name))
+
+                if event['type'] in ('ADDED', 'MODIFIED'):
+                    self._items = merge_dicts(self._items, {name: item})
+                elif event['type'] == 'DELETED':
+                    self._items = {k:v for k,v in self._items.items() if k != name}
+                elif event['type'] == 'BOOKMARK':
+                    pass
+                elif event['type'] == 'ERROR':
+                    raise ApiException(str(event))
+                else:
+                    raise KeyError('Unknown watch event {}'.format(event['type']))
+        except ProtocolError as e:
+            if 'Connection broken' in str(e):
+                log.info('Connection reset.')
+                return
+            raise
+        except ApiException as e:
+            log.exception('K8s API failed. {}'.format(self.api_func))
+            self.exception = e
+            raise
+        except Exception as e:
+            log.exception("Watcher failed. ({})".format(self.api_func))
+            self.exception = e
+            raise
+
+
+class RookCluster(object):
+    # import of client.CoreV1Api must be optional at import time.
+    # Instead allow mgr/rook to be imported anyway.
+    def __init__(self, coreV1_api: 'client.CoreV1Api', batchV1_api: 'client.BatchV1Api', rook_env: 'RookEnv'):
+        self.rook_env = rook_env  # type: RookEnv
+        self.coreV1_api = coreV1_api  # client.CoreV1Api
+        self.batchV1_api = batchV1_api
+
+        #  TODO: replace direct k8s calls with Rook API calls
+        # when they're implemented
+        self.inventory_maps: KubernetesResource[client.V1ConfigMapList] = KubernetesResource(self.coreV1_api.list_namespaced_config_map,
+                                                 namespace=self.rook_env.operator_namespace,
+                                                 label_selector="app=rook-discover")
+
+        self.rook_pods: KubernetesResource[client.V1Pod] = KubernetesResource(self.coreV1_api.list_namespaced_pod,
+                                            namespace=self.rook_env.namespace,
+                                            label_selector="rook_cluster={0}".format(
+                                                self.rook_env.namespace))
+        self.nodes: KubernetesResource[client.V1Node] = KubernetesResource(self.coreV1_api.list_node)
+
+    def rook_url(self, path: str) -> str:
+        prefix = "/apis/ceph.rook.io/%s/namespaces/%s/" % (
+            self.rook_env.crd_version, self.rook_env.namespace)
+        return urljoin(prefix, path)
+
+    def rook_api_call(self, verb: str, path: str, **kwargs: Any) -> Any:
+        full_path = self.rook_url(path)
+        log.debug("[%s] %s" % (verb, full_path))
+
+        return self.coreV1_api.api_client.call_api(
+            full_path,
+            verb,
+            auth_settings=['BearerToken'],
+            response_type="object",
+            _return_http_data_only=True,
+            _preload_content=True,
+            **kwargs)
+
+    def rook_api_get(self, path: str, **kwargs: Any) -> Any:
+        return self.rook_api_call("GET", path, **kwargs)
+
+    def rook_api_delete(self, path: str) -> Any:
+        return self.rook_api_call("DELETE", path)
+
+    def rook_api_patch(self, path: str, **kwargs: Any) -> Any:
+        return self.rook_api_call("PATCH", path,
+                                  header_params={"Content-Type": "application/json-patch+json"},
+                                  **kwargs)
+
+    def rook_api_post(self, path: str, **kwargs: Any) -> Any:
+        return self.rook_api_call("POST", path, **kwargs)
+
+    def get_discovered_devices(self, nodenames: Optional[List[str]] = None) -> Dict[str, dict]:
+        def predicate(item: client.V1ConfigMapList) -> bool:
+            if nodenames is not None:
+                return item.metadata.labels['rook.io/node'] in nodenames
+            else:
+                return True
+
+        try:
+            result = [i for i in self.inventory_maps.items if predicate(i)]
+        except ApiException as dummy_e:
+            log.exception("Failed to fetch device metadata")
+            raise
+
+        nodename_to_devices = {}
+        for i in result:
+            drives = json.loads(i.data['devices'])
+            nodename_to_devices[i.metadata.labels['rook.io/node']] = drives
+
+        return nodename_to_devices
+
+    def get_nfs_conf_url(self, nfs_cluster: str, instance: str) -> Optional[str]:
+        #
+        # Fetch cephnfs object for "nfs_cluster" and then return a rados://
+        # URL for the instance within that cluster. If the fetch fails, just
+        # return None.
+        #
+        try:
+            ceph_nfs = self.rook_api_get("cephnfses/{0}".format(nfs_cluster))
+        except ApiException as e:
+            log.info("Unable to fetch cephnfs object: {}".format(e.status))
+            return None
+
+        pool = ceph_nfs['spec']['rados']['pool']
+        namespace = ceph_nfs['spec']['rados'].get('namespace', None)
+
+        if namespace == None:
+            url = "rados://{0}/conf-{1}.{2}".format(pool, nfs_cluster, instance)
+        else:
+            url = "rados://{0}/{1}/conf-{2}.{3}".format(pool, namespace, nfs_cluster, instance)
+        return url
+
+    def describe_pods(self,
+                      service_type: Optional[str],
+                      service_id: Optional[str],
+                      nodename: Optional[str]) -> List[Dict[str, Any]]:
+        """
+        Go query the k8s API about deployment, containers related to this
+        filesystem
+
+        Example Rook Pod labels for a mgr daemon:
+        Labels:         app=rook-ceph-mgr
+                        pod-template-hash=2171958073
+                        rook_cluster=rook
+        And MDS containers additionally have `rook_filesystem` label
+
+        Label filter is rook_cluster=<cluster namespace>
+                        rook_file_system=<self.fs_name>
+        """
+        def predicate(item):
+            # type: (client.V1Pod) -> bool
+            metadata = item.metadata
+            if service_type is not None:
+                if metadata.labels['app'] != "rook-ceph-{0}".format(service_type):
+                    return False
+
+                if service_id is not None:
+                    try:
+                        k, v = {
+                            "mds": ("rook_file_system", service_id),
+                            "osd": ("ceph-osd-id", service_id),
+                            "mon": ("mon", service_id),
+                            "mgr": ("mgr", service_id),
+                            "ceph_nfs": ("ceph_nfs", service_id),
+                            "rgw": ("ceph_rgw", service_id),
+                        }[service_type]
+                    except KeyError:
+                        raise orchestrator.OrchestratorValidationError(
+                            '{} not supported'.format(service_type))
+                    if metadata.labels[k] != v:
+                        return False
+
+            if nodename is not None:
+                if item.spec.node_name != nodename:
+                    return False
+            return True
+
+        refreshed = datetime_now()
+        pods = [i for i in self.rook_pods.items if predicate(i)]
+
+        pods_summary = []
+        prefix = 'sha256:'
+
+        for p in pods:
+            d = p.to_dict()
+
+            image_name = None
+            for c in d['spec']['containers']:
+                # look at the first listed container in the pod...
+                image_name = c['image']
+                break
+
+            image_id = d['status']['container_statuses'][0]['image_id']
+            image_id = image_id.split(prefix)[1] if prefix in image_id else image_id
+
+            s = {
+                "name": d['metadata']['name'],
+                "hostname": d['spec']['node_name'],
+                "labels": d['metadata']['labels'],
+                'phase': d['status']['phase'],
+                'container_image_name': image_name,
+                'container_image_id': image_id,
+                'refreshed': refreshed,
+                # these may get set below...
+                'started': None,
+                'created': None,
+            }
+
+            # note: we want UTC
+            if d['metadata'].get('creation_timestamp', None):
+                s['created'] = d['metadata']['creation_timestamp'].astimezone(
+                    tz=datetime.timezone.utc)
+            if d['status'].get('start_time', None):
+                s['started'] = d['status']['start_time'].astimezone(
+                    tz=datetime.timezone.utc)
+
+            pods_summary.append(s)
+
+        return pods_summary
+
+    def remove_pods(self, names: List[str]) -> List[str]:
+        pods = [i for i in self.rook_pods.items]
+        for p in pods:
+            d = p.to_dict()
+            daemon_type = d['metadata']['labels']['app'].replace('rook-ceph-','')
+            daemon_id = d['metadata']['labels']['ceph_daemon_id']
+            name = daemon_type + '.' + daemon_id
+            if name in names:
+                self.coreV1_api.delete_namespaced_pod(
+                    d['metadata']['name'],
+                    self.rook_env.namespace,
+                    body=client.V1DeleteOptions()
+                )
+        return [f'Removed Pod {n}' for n in names]
+
+    def get_node_names(self) -> List[str]:
+        return [i.metadata.name for i in self.nodes.items]
+
+    @contextmanager
+    def ignore_409(self, what: str) -> Iterator[None]:
+        try:
+            yield
+        except ApiException as e:
+            if e.status == 409:
+                # Idempotent, succeed.
+                log.info("{} already exists".format(what))
+            else:
+                raise
+
+    def apply_filesystem(self, spec: ServiceSpec) -> str:
+        # TODO use spec.placement
+        # TODO warn if spec.extended has entries we don't kow how
+        #      to action.
+        def _update_fs(new: cfs.CephFilesystem) -> cfs.CephFilesystem:
+            new.spec.metadataServer.activeCount = spec.placement.count or 1
+            return new
+
+        def _create_fs() -> cfs.CephFilesystem:
+            return cfs.CephFilesystem(
+                apiVersion=self.rook_env.api_name,
+                metadata=dict(
+                    name=spec.service_id,
+                    namespace=self.rook_env.namespace,
+                ),
+                spec=cfs.Spec(
+                    metadataServer=cfs.MetadataServer(
+                        activeCount=spec.placement.count or 1,
+                        activeStandby=True
+                    )
+                )
+            )
+        assert spec.service_id is not None
+        return self._create_or_patch(
+            cfs.CephFilesystem, 'cephfilesystems', spec.service_id,
+            _update_fs, _create_fs)
+
+    def apply_objectstore(self, spec: RGWSpec) -> str:
+        assert spec.service_id is not None
+
+        name = spec.service_id
+
+        if '.' in spec.service_id:
+            # rook does not like . in the name.  this is could
+            # there because it is a legacy rgw spec that was named
+            # like $realm.$zone, except that I doubt there were any
+            # users of this code.  Instead, focus on future users and
+            # translate . to - (fingers crossed!) instead.
+            name = spec.service_id.replace('.', '-')
+
+        # FIXME: pass realm and/or zone through to the CR
+
+        def _create_zone() -> cos.CephObjectStore:
+            port = None
+            secure_port = None
+            if spec.ssl:
+                secure_port = spec.get_port()
+            else:
+                port = spec.get_port()
+            return cos.CephObjectStore(
+                apiVersion=self.rook_env.api_name,
+                metadata=dict(
+                    name=name,
+                    namespace=self.rook_env.namespace
+                ),
+                spec=cos.Spec(
+                    gateway=cos.Gateway(
+                        type='s3',
+                        port=port,
+                        securePort=secure_port,
+                        instances=spec.placement.count or 1,
+                    )
+                )
+            )
+
+        def _update_zone(new: cos.CephObjectStore) -> cos.CephObjectStore:
+            new.spec.gateway.instances = spec.placement.count or 1
+            return new
+
+        return self._create_or_patch(
+            cos.CephObjectStore, 'cephobjectstores', name,
+            _update_zone, _create_zone)
+
+    def apply_nfsgw(self, spec: NFSServiceSpec) -> str:
+        # TODO use spec.placement
+        # TODO warn if spec.extended has entries we don't kow how
+        #      to action.
+        # TODO Number of pods should be based on the list of hosts in the
+        #      PlacementSpec.
+        count = spec.placement.count or 1
+        def _update_nfs(new: cnfs.CephNFS) -> cnfs.CephNFS:
+            new.spec.server.active = count
+            return new
+
+        def _create_nfs() -> cnfs.CephNFS:
+            rook_nfsgw = cnfs.CephNFS(
+                    apiVersion=self.rook_env.api_name,
+                    metadata=dict(
+                        name=spec.service_id,
+                        namespace=self.rook_env.namespace,
+                        ),
+                    spec=cnfs.Spec(
+                        rados=cnfs.Rados(
+                            pool=NFS_POOL_NAME,
+                            ),
+                        server=cnfs.Server(
+                            active=count
+                            )
+                        )
+                    )
+
+            rook_nfsgw.spec.rados.namespace = cast(str, spec.service_id)
+
+            return rook_nfsgw
+
+        assert spec.service_id is not None
+        return self._create_or_patch(cnfs.CephNFS, 'cephnfses', spec.service_id,
+                _update_nfs, _create_nfs)
+
+    def rm_service(self, rooktype: str, service_id: str) -> str:
+
+        objpath = "{0}/{1}".format(rooktype, service_id)
+
+        try:
+            self.rook_api_delete(objpath)
+        except ApiException as e:
+            if e.status == 404:
+                log.info("{0} service '{1}' does not exist".format(rooktype, service_id))
+                # Idempotent, succeed.
+            else:
+                raise
+
+        return f'Removed {objpath}'
+
+    def can_create_osd(self) -> bool:
+        current_cluster = self.rook_api_get(
+            "cephclusters/{0}".format(self.rook_env.cluster_name))
+        use_all_nodes = current_cluster['spec'].get('useAllNodes', False)
+
+        # If useAllNodes is set, then Rook will not be paying attention
+        # to anything we put in 'nodes', so can't do OSD creation.
+        return not use_all_nodes
+
+    def node_exists(self, node_name: str) -> bool:
+        return node_name in self.get_node_names()
+
+    def update_mon_count(self, newcount: Optional[int]) -> str:
+        def _update_mon_count(current, new):
+            # type: (ccl.CephCluster, ccl.CephCluster) -> ccl.CephCluster
+            if newcount is None:
+                raise orchestrator.OrchestratorError('unable to set mon count to None')
+            new.spec.mon.count = newcount
+            return new
+        return self._patch(ccl.CephCluster, 'cephclusters', self.rook_env.cluster_name, _update_mon_count)
+
+    def add_osds(self, drive_group, matching_hosts):
+        # type: (DriveGroupSpec, List[str]) -> str
+        """
+        Rook currently (0.8) can only do single-drive OSDs, so we
+        treat all drive groups as just a list of individual OSDs.
+        """
+        block_devices = drive_group.data_devices.paths if drive_group.data_devices else []
+        directories = drive_group.data_directories
+
+        assert drive_group.objectstore in ("bluestore", "filestore")
+
+        def _add_osds(current_cluster, new_cluster):
+            # type: (ccl.CephCluster, ccl.CephCluster) -> ccl.CephCluster
+
+            # FIXME: this is all not really atomic, because jsonpatch doesn't
+            # let us do "test" operations that would check if items with
+            # matching names were in existing lists.
+
+            if not hasattr(new_cluster.spec.storage, 'nodes'):
+                new_cluster.spec.storage.nodes = ccl.NodesList()
+
+            current_nodes = getattr(current_cluster.spec.storage, 'nodes', ccl.NodesList())
+            matching_host = matching_hosts[0]
+
+            if matching_host not in [n.name for n in current_nodes]:
+                pd = ccl.NodesItem(
+                    name=matching_host,
+                    config=ccl.Config(
+                        storeType=drive_group.objectstore
+                    )
+                )
+
+                if block_devices:
+                    pd.devices = ccl.DevicesList(
+                        ccl.DevicesItem(name=d.path) for d in block_devices
+                    )
+                if directories:
+                    pd.directories = ccl.DirectoriesList(
+                        ccl.DirectoriesItem(path=p) for p in directories
+                    )
+                new_cluster.spec.storage.nodes.append(pd)
+            else:
+                for _node in new_cluster.spec.storage.nodes:
+                    current_node = _node  # type: ccl.NodesItem
+                    if current_node.name == matching_host:
+                        if block_devices:
+                            if not hasattr(current_node, 'devices'):
+                                current_node.devices = ccl.DevicesList()
+                            new_devices = list(set(block_devices) - set([d.name for d in current_node.devices]))
+                            current_node.devices.extend(
+                                ccl.DevicesItem(name=n.path) for n in new_devices
+                            )
+
+                        if directories:
+                            if not hasattr(current_node, 'directories'):
+                                current_node.directories = ccl.DirectoriesList()
+                            new_dirs = list(set(directories) - set([d.path for d in current_node.directories]))
+                            current_node.directories.extend(
+                                ccl.DirectoriesItem(path=n) for n in new_dirs
+                            )
+            return new_cluster
+
+        return self._patch(ccl.CephCluster, 'cephclusters', self.rook_env.cluster_name, _add_osds)
+
+    def _patch(self, crd: Type, crd_name: str, cr_name: str, func: Callable[[CrdClassT, CrdClassT], CrdClassT]) -> str:
+        current_json = self.rook_api_get(
+            "{}/{}".format(crd_name, cr_name)
+        )
+
+        current = crd.from_json(current_json)
+        new = crd.from_json(current_json)  # no deepcopy.
+
+        new = func(current, new)
+
+        patch = list(jsonpatch.make_patch(current_json, new.to_json()))
+
+        log.info('patch for {}/{}: \n{}'.format(crd_name, cr_name, patch))
+
+        if len(patch) == 0:
+            return "No change"
+
+        try:
+            self.rook_api_patch(
+                "{}/{}".format(crd_name, cr_name),
+                body=patch)
+        except ApiException as e:
+            log.exception("API exception: {0}".format(e))
+            raise ApplyException(
+                "Failed to update {}/{}: {}".format(crd_name, cr_name, e))
+
+        return "Success"
+
+    def _create_or_patch(self,
+                         crd: Type,
+                         crd_name: str,
+                         cr_name: str,
+                         update_func: Callable[[CrdClassT], CrdClassT],
+                         create_func: Callable[[], CrdClassT]) -> str:
+        try:
+            current_json = self.rook_api_get(
+                "{}/{}".format(crd_name, cr_name)
+            )
+        except ApiException as e:
+            if e.status == 404:
+                current_json = None
+            else:
+                raise
+
+        if current_json:
+            new = crd.from_json(current_json)  # no deepcopy.
+
+            new = update_func(new)
+
+            patch = list(jsonpatch.make_patch(current_json, new.to_json()))
+
+            log.info('patch for {}/{}: \n{}'.format(crd_name, cr_name, patch))
+
+            if len(patch) == 0:
+                return "No change"
+
+            try:
+                self.rook_api_patch(
+                    "{}/{}".format(crd_name, cr_name),
+                    body=patch)
+            except ApiException as e:
+                log.exception("API exception: {0}".format(e))
+                raise ApplyException(
+                    "Failed to update {}/{}: {}".format(crd_name, cr_name, e))
+            return "Updated"
+        else:
+            new = create_func()
+            with self.ignore_409("{} {} already exists".format(crd_name,
+                                                               cr_name)):
+                self.rook_api_post("{}/".format(crd_name),
+                                   body=new.to_json())
+            return "Created"
+    def get_ceph_image(self) -> str:
+        try:
+            api_response = self.coreV1_api.list_namespaced_pod(self.rook_env.namespace,
+                                                               label_selector="app=rook-ceph-mon",
+                                                               timeout_seconds=10)
+            if api_response.items:
+                return api_response.items[-1].spec.containers[0].image
+            else:
+                raise orchestrator.OrchestratorError(
+                        "Error getting ceph image. Cluster without monitors")
+        except ApiException as e:
+            raise orchestrator.OrchestratorError("Error getting ceph image: {}".format(e))
+
+
+    def _execute_blight_job(self, ident_fault: str, on: bool, loc: orchestrator.DeviceLightLoc) -> str:
+        operation_id = str(hash(loc))
+        message = ""
+
+        # job definition
+        job_metadata = client.V1ObjectMeta(name=operation_id,
+                                           namespace= self.rook_env.namespace,
+                                           labels={"ident": operation_id})
+        pod_metadata = client.V1ObjectMeta(labels={"ident": operation_id})
+        pod_container = client.V1Container(name="ceph-lsmcli-command",
+                                           security_context=client.V1SecurityContext(privileged=True),
+                                           image=self.get_ceph_image(),
+                                           command=["lsmcli",],
+                                           args=['local-disk-%s-led-%s' % (ident_fault,'on' if on else 'off'),
+                                                 '--path', loc.path or loc.dev,],
+                                           volume_mounts=[client.V1VolumeMount(name="devices", mount_path="/dev"),
+                                                          client.V1VolumeMount(name="run-udev", mount_path="/run/udev")])
+        pod_spec = client.V1PodSpec(containers=[pod_container],
+                                    active_deadline_seconds=30, # Max time to terminate pod
+                                    restart_policy="Never",
+                                    node_selector= {"kubernetes.io/hostname": loc.host},
+                                    volumes=[client.V1Volume(name="devices",
+                                                             host_path=client.V1HostPathVolumeSource(path="/dev")),
+                                             client.V1Volume(name="run-udev",
+                                                             host_path=client.V1HostPathVolumeSource(path="/run/udev"))])
+        pod_template = client.V1PodTemplateSpec(metadata=pod_metadata,
+                                                  spec=pod_spec)
+        job_spec = client.V1JobSpec(active_deadline_seconds=60, # Max time to terminate job
+                                    ttl_seconds_after_finished=10, # Alfa. Lifetime after finishing (either Complete or Failed)
+                                    backoff_limit=0,
+                                    template=pod_template)
+        job = client.V1Job(api_version="batch/v1",
+                           kind="Job",
+                           metadata=job_metadata,
+                           spec=job_spec)
+
+        # delete previous job if it exists
+        try:
+            try:
+                api_response = self.batchV1_api.delete_namespaced_job(operation_id,
+                                                                      self.rook_env.namespace,
+                                                                      propagation_policy="Background")
+            except ApiException as e:
+                if e.status != 404: # No problem if the job does not exist
+                    raise
+
+            # wait until the job is not present
+            deleted = False
+            retries = 0
+            while not deleted and retries < 10:
+                api_response = self.batchV1_api.list_namespaced_job(self.rook_env.namespace,
+                                                                    label_selector="ident=%s" % operation_id,
+                                                                    timeout_seconds=10)
+                deleted = not api_response.items
+                if retries > 5:
+                    sleep(0.1)
+                retries += 1
+            if retries == 10 and not deleted:
+                raise orchestrator.OrchestratorError(
+                    "Light <{}> in <{}:{}> cannot be executed. Cannot delete previous job <{}>".format(
+                            on, loc.host, loc.path or loc.dev, operation_id))
+
+            # create the job
+            api_response = self.batchV1_api.create_namespaced_job(self.rook_env.namespace, job)
+
+            # get the result
+            finished = False
+            while not finished:
+                api_response = self.batchV1_api.read_namespaced_job(operation_id,
+                                                                    self.rook_env.namespace)
+                finished = api_response.status.succeeded or api_response.status.failed
+                if finished:
+                    message = api_response.status.conditions[-1].message
+
+            # get the result of the lsmcli command
+            api_response=self.coreV1_api.list_namespaced_pod(self.rook_env.namespace,
+                                                             label_selector="ident=%s" % operation_id,
+                                                             timeout_seconds=10)
+            if api_response.items:
+                pod_name = api_response.items[-1].metadata.name
+                message = self.coreV1_api.read_namespaced_pod_log(pod_name,
+                                                                  self.rook_env.namespace)
+
+        except ApiException as e:
+            log.exception('K8s API failed. {}'.format(e))
+            raise
+
+        # Finally, delete the job.
+        # The job uses <ttl_seconds_after_finished>. This makes that the TTL controller delete automatically the job.
+        # This feature is in Alpha state, so extra explicit delete operations trying to delete the Job has been used strategically
+        try:
+            api_response = self.batchV1_api.delete_namespaced_job(operation_id,
+                                                                  self.rook_env.namespace,
+                                                                  propagation_policy="Background")
+        except ApiException as e:
+            if e.status != 404: # No problem if the job does not exist
+                raise
+
+        return message
+
+    def blink_light(self, ident_fault, on, locs):
+        # type: (str, bool, List[orchestrator.DeviceLightLoc]) -> List[str]
+        return [self._execute_blight_job(ident_fault, on, loc) for loc in locs]