Adding upstream version 16.2.11+ds.upstream/16.2.11+dsupstream

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

1 files changed, 433 insertions, 0 deletions

diff --git a/qa/tasks/mgr/dashboard/test_pool.py b/qa/tasks/mgr/dashboard/test_pool.py
new file mode 100644
index 000000000..055ba2b00
--- /dev/null
+++ b/qa/tasks/mgr/dashboard/test_pool.py
@@ -0,0 +1,433 @@
+# -*- coding: utf-8 -*-
+from __future__ import absolute_import
+
+import logging
+import time
+from contextlib import contextmanager
+
+from .helper import DashboardTestCase, JAny, JList, JObj, JUnion
+
+log = logging.getLogger(__name__)
+
+
+class PoolTest(DashboardTestCase):
+    AUTH_ROLES = ['pool-manager']
+
+    pool_schema = JObj(sub_elems={
+        'pool_name': str,
+        'type': str,
+        'application_metadata': JList(str),
+        'flags': int,
+        'flags_names': str,
+    }, allow_unknown=True)
+
+    pool_list_stat_schema = JObj(sub_elems={
+        'latest': JUnion([int, float]),
+        'rate': float,
+        'rates': JList(JAny(none=False)),
+    })
+
+    pool_list_stats_schema = JObj(sub_elems={
+        'avail_raw': pool_list_stat_schema,
+        'bytes_used': pool_list_stat_schema,
+        'max_avail': pool_list_stat_schema,
+        'percent_used': pool_list_stat_schema,
+        'rd_bytes': pool_list_stat_schema,
+        'wr_bytes': pool_list_stat_schema,
+        'rd': pool_list_stat_schema,
+        'wr': pool_list_stat_schema,
+    }, allow_unknown=True)
+
+    pool_rbd_conf_schema = JList(JObj(sub_elems={
+        'name': str,
+        'value': str,
+        'source': int
+    }))
+
+    @contextmanager
+    def __yield_pool(self, name=None, data=None, deletion_name=None):
+        """
+        Use either just a name or whole description of a pool to create one.
+        This also validates the correct creation and deletion after the pool was used.
+
+        :param name: Name of the pool
+        :param data: Describes the pool in full length
+        :param deletion_name: Only needed if the pool was renamed
+        :return:
+        """
+        data = self._create_pool(name, data)
+        yield data
+        self._delete_pool(deletion_name or data['pool'])
+
+    def _create_pool(self, name, data):
+        data = data or {
+            'pool': name,
+            'pg_num': '32',
+            'pool_type': 'replicated',
+            'compression_algorithm': 'snappy',
+            'compression_mode': 'passive',
+            'compression_max_blob_size': '131072',
+            'compression_required_ratio': '0.875',
+            'application_metadata': ['rbd'],
+            'configuration': {
+                'rbd_qos_bps_limit': 1024000,
+                'rbd_qos_iops_limit': 5000,
+            }
+        }
+        self._task_post('/api/pool/', data)
+        self.assertStatus(201)
+        self._validate_pool_properties(data, self._get_pool(data['pool']))
+        return data
+
+    def _delete_pool(self, name):
+        self._task_delete('/api/pool/' + name)
+        self.assertStatus(204)
+
+    def _validate_pool_properties(self, data, pool, timeout=DashboardTestCase.TIMEOUT_HEALTH_CLEAR):
+        # pylint: disable=too-many-branches
+        for prop, value in data.items():
+            if prop == 'pool_type':
+                self.assertEqual(pool['type'], value)
+            elif prop == 'size':
+                self.assertEqual(pool[prop], int(value),
+                                 '{}: {} != {}'.format(prop, pool[prop], value))
+            elif prop == 'pg_num':
+                self._check_pg_num(pool['pool_name'], int(value))
+            elif prop == 'application_metadata':
+                self.assertIsInstance(pool[prop], list)
+                self.assertEqual(value, pool[prop])
+            elif prop == 'pool':
+                self.assertEqual(pool['pool_name'], value)
+            elif prop.startswith('compression'):
+                if value is not None:
+                    if prop.endswith('size'):
+                        value = int(value)
+                    elif prop.endswith('ratio'):
+                        value = float(value)
+                    self.assertEqual(pool['options'][prop], value)
+                else:
+                    self.assertEqual(pool['options'], {})
+            elif prop == 'configuration':
+                # configuration cannot really be checked here for two reasons:
+                #   1.  The default value cannot be given to this method, which becomes relevant
+                #       when resetting a value, because it's not always zero.
+                #   2.  The expected `source` cannot be given to this method, and it cannot
+                #       reliably be determined (see 1)
+                pass
+            else:
+                self.assertEqual(pool[prop], value, '{}: {} != {}'.format(prop, pool[prop], value))
+
+        self.wait_for_health_clear(timeout)
+
+    def _get_pool(self, pool_name):
+        pool = self._get("/api/pool/" + pool_name)
+        self.assertStatus(200)
+        self.assertSchemaBody(self.pool_schema)
+        return pool
+
+    def _check_pg_num(self, pool_name, pg_num):
+        """
+        If both properties have not the same value, the cluster goes into a warning state, which
+        will only happen during a pg update on an existing pool. The test that does that is
+        currently commented out because our QA systems can't deal with the change. Feel free to test
+        it locally.
+        """
+        self.wait_until_equal(
+            lambda: self._get_pool(pool_name)['pg_placement_num'],
+            expect_val=pg_num,
+            timeout=180
+        )
+
+        pool = self._get_pool(pool_name)
+
+        for prop in ['pg_num', 'pg_placement_num']:
+            self.assertEqual(pool[prop], int(pg_num),
+                             '{}: {} != {}'.format(prop, pool[prop], pg_num))
+
+    @DashboardTestCase.RunAs('test', 'test', [{'pool': ['create', 'update', 'delete']}])
+    def test_read_access_permissions(self):
+        self._get('/api/pool')
+        self.assertStatus(403)
+        self._get('/api/pool/bla')
+        self.assertStatus(403)
+
+    @DashboardTestCase.RunAs('test', 'test', [{'pool': ['read', 'update', 'delete']}])
+    def test_create_access_permissions(self):
+        self._task_post('/api/pool/', {})
+        self.assertStatus(403)
+
+    @DashboardTestCase.RunAs('test', 'test', [{'pool': ['read', 'create', 'update']}])
+    def test_delete_access_permissions(self):
+        self._delete('/api/pool/ddd')
+        self.assertStatus(403)
+
+    def test_pool_configuration(self):
+        pool_name = 'device_health_metrics'
+        data = self._get('/api/pool/{}/configuration'.format(pool_name))
+        self.assertStatus(200)
+        self.assertSchema(data, JList(JObj({
+            'name': str,
+            'value': str,
+            'source': int
+        })))
+
+    def test_pool_list(self):
+        data = self._get("/api/pool")
+        self.assertStatus(200)
+
+        cluster_pools = self.ceph_cluster.mon_manager.list_pools()
+        self.assertEqual(len(cluster_pools), len(data))
+        self.assertSchemaBody(JList(self.pool_schema))
+        for pool in data:
+            self.assertNotIn('pg_status', pool)
+            self.assertNotIn('stats', pool)
+            self.assertIn(pool['pool_name'], cluster_pools)
+
+    def test_pool_list_attrs(self):
+        data = self._get("/api/pool?attrs=type,flags")
+        self.assertStatus(200)
+
+        cluster_pools = self.ceph_cluster.mon_manager.list_pools()
+        self.assertEqual(len(cluster_pools), len(data))
+        for pool in data:
+            self.assertIn('pool_name', pool)
+            self.assertIn('type', pool)
+            self.assertIn('flags', pool)
+            self.assertNotIn('flags_names', pool)
+            self.assertNotIn('pg_status', pool)
+            self.assertNotIn('stats', pool)
+            self.assertIn(pool['pool_name'], cluster_pools)
+
+    def test_pool_list_stats(self):
+        data = self._get("/api/pool?stats=true")
+        self.assertStatus(200)
+
+        cluster_pools = self.ceph_cluster.mon_manager.list_pools()
+        self.assertEqual(len(cluster_pools), len(data))
+        self.assertSchemaBody(JList(self.pool_schema))
+        for pool in data:
+            self.assertIn('pool_name', pool)
+            self.assertIn('type', pool)
+            self.assertIn('application_metadata', pool)
+            self.assertIn('flags', pool)
+            self.assertIn('pg_status', pool)
+            self.assertSchema(pool['stats'], self.pool_list_stats_schema)
+            self.assertIn('flags_names', pool)
+            self.assertIn(pool['pool_name'], cluster_pools)
+
+    def test_pool_get(self):
+        cluster_pools = self.ceph_cluster.mon_manager.list_pools()
+        pool = self._get("/api/pool/{}?stats=true&attrs=type,flags,stats"
+                         .format(cluster_pools[0]))
+        self.assertEqual(pool['pool_name'], cluster_pools[0])
+        self.assertIn('type', pool)
+        self.assertIn('flags', pool)
+        self.assertNotIn('pg_status', pool)
+        self.assertSchema(pool['stats'], self.pool_list_stats_schema)
+        self.assertNotIn('flags_names', pool)
+        self.assertSchema(pool['configuration'], self.pool_rbd_conf_schema)
+
+    def test_pool_create_with_two_applications(self):
+        self.__yield_pool(None, {
+            'pool': 'dashboard_pool1',
+            'pg_num': '32',
+            'pool_type': 'replicated',
+            'application_metadata': ['rbd', 'sth'],
+        })
+
+    def test_pool_create_with_ecp_and_rule(self):
+        self._ceph_cmd(['osd', 'crush', 'rule', 'create-erasure', 'ecrule'])
+        self._ceph_cmd(
+            ['osd', 'erasure-code-profile', 'set', 'ecprofile', 'crush-failure-domain=osd'])
+        self.__yield_pool(None, {
+            'pool': 'dashboard_pool2',
+            'pg_num': '32',
+            'pool_type': 'erasure',
+            'application_metadata': ['rbd'],
+            'erasure_code_profile': 'ecprofile',
+            'crush_rule': 'ecrule',
+        })
+        self._ceph_cmd(['osd', 'erasure-code-profile', 'rm', 'ecprofile'])
+
+    def test_pool_create_with_compression(self):
+        pool = {
+            'pool': 'dashboard_pool3',
+            'pg_num': '32',
+            'pool_type': 'replicated',
+            'compression_algorithm': 'zstd',
+            'compression_mode': 'aggressive',
+            'compression_max_blob_size': '10000000',
+            'compression_required_ratio': '0.8',
+            'application_metadata': ['rbd'],
+            'configuration': {
+                'rbd_qos_bps_limit': 2048,
+                'rbd_qos_iops_limit': None,
+            },
+        }
+        with self.__yield_pool(None, pool):
+            expected_configuration = [{
+                'name': 'rbd_qos_bps_limit',
+                'source': 1,
+                'value': '2048',
+            }, {
+                'name': 'rbd_qos_iops_limit',
+                'source': 0,
+                'value': '0',
+            }]
+            new_pool = self._get_pool(pool['pool'])
+            for conf in expected_configuration:
+                self.assertIn(conf, new_pool['configuration'])
+
+    def test_pool_create_with_quotas(self):
+        pools = [
+            {
+                'pool_data': {
+                    'pool': 'dashboard_pool_quota1',
+                    'pg_num': '32',
+                    'pool_type': 'replicated',
+                },
+                'pool_quotas_to_check': {
+                    'quota_max_objects': 0,
+                    'quota_max_bytes': 0,
+                }
+            },
+            {
+                'pool_data': {
+                    'pool': 'dashboard_pool_quota2',
+                    'pg_num': '32',
+                    'pool_type': 'replicated',
+                    'quota_max_objects': 1024,
+                    'quota_max_bytes': 1000,
+                },
+                'pool_quotas_to_check': {
+                    'quota_max_objects': 1024,
+                    'quota_max_bytes': 1000,
+                }
+            }
+        ]
+
+        for pool in pools:
+            pool_name = pool['pool_data']['pool']
+            with self.__yield_pool(pool_name, pool['pool_data']):
+                self._validate_pool_properties(pool['pool_quotas_to_check'],
+                                               self._get_pool(pool_name))
+
+    def test_pool_update_name(self):
+        name = 'pool_update'
+        updated_name = 'pool_updated_name'
+        with self.__yield_pool(name, None, updated_name):
+            props = {'pool': updated_name}
+            self._task_put('/api/pool/{}'.format(name), props)
+            time.sleep(5)
+            self.assertStatus(200)
+            self._validate_pool_properties(props, self._get_pool(updated_name))
+
+    def test_pool_update_metadata(self):
+        pool_name = 'pool_update_metadata'
+        with self.__yield_pool(pool_name):
+            props = {'application_metadata': ['rbd', 'sth']}
+            self._task_put('/api/pool/{}'.format(pool_name), props)
+            self._validate_pool_properties(props, self._get_pool(pool_name),
+                                           self.TIMEOUT_HEALTH_CLEAR * 2)
+
+            properties = {'application_metadata': ['rgw']}
+            self._task_put('/api/pool/' + pool_name, properties)
+            self._validate_pool_properties(properties, self._get_pool(pool_name),
+                                           self.TIMEOUT_HEALTH_CLEAR * 2)
+
+            properties = {'application_metadata': ['rbd', 'sth']}
+            self._task_put('/api/pool/' + pool_name, properties)
+            self._validate_pool_properties(properties, self._get_pool(pool_name),
+                                           self.TIMEOUT_HEALTH_CLEAR * 2)
+
+            properties = {'application_metadata': ['rgw']}
+            self._task_put('/api/pool/' + pool_name, properties)
+            self._validate_pool_properties(properties, self._get_pool(pool_name),
+                                           self.TIMEOUT_HEALTH_CLEAR * 2)
+
+    def test_pool_update_configuration(self):
+        pool_name = 'pool_update_configuration'
+        with self.__yield_pool(pool_name):
+            configuration = {
+                'rbd_qos_bps_limit': 1024,
+                'rbd_qos_iops_limit': None,
+            }
+            expected_configuration = [{
+                'name': 'rbd_qos_bps_limit',
+                'source': 1,
+                'value': '1024',
+            }, {
+                'name': 'rbd_qos_iops_limit',
+                'source': 0,
+                'value': '0',
+            }]
+            self._task_put('/api/pool/' + pool_name, {'configuration': configuration})
+            time.sleep(5)
+            pool_config = self._get_pool(pool_name)['configuration']
+            for conf in expected_configuration:
+                self.assertIn(conf, pool_config)
+
+    def test_pool_update_compression(self):
+        pool_name = 'pool_update_compression'
+        with self.__yield_pool(pool_name):
+            properties = {
+                'compression_algorithm': 'zstd',
+                'compression_mode': 'aggressive',
+                'compression_max_blob_size': '10000000',
+                'compression_required_ratio': '0.8',
+            }
+            self._task_put('/api/pool/' + pool_name, properties)
+            time.sleep(5)
+            self._validate_pool_properties(properties, self._get_pool(pool_name))
+
+    def test_pool_update_unset_compression(self):
+        pool_name = 'pool_update_unset_compression'
+        with self.__yield_pool(pool_name):
+            self._task_put('/api/pool/' + pool_name, {'compression_mode': 'unset'})
+            time.sleep(5)
+            self._validate_pool_properties({
+                'compression_algorithm': None,
+                'compression_mode': None,
+                'compression_max_blob_size': None,
+                'compression_required_ratio': None,
+            }, self._get_pool(pool_name))
+
+    def test_pool_update_quotas(self):
+        pool_name = 'pool_update_quotas'
+        with self.__yield_pool(pool_name):
+            properties = {
+                'quota_max_objects': 1024,
+                'quota_max_bytes': 1000,
+            }
+            self._task_put('/api/pool/' + pool_name, properties)
+            time.sleep(5)
+            self._validate_pool_properties(properties, self._get_pool(pool_name))
+
+    def test_pool_create_fail(self):
+        data = {'pool_type': u'replicated', 'rule_name': u'dnf', 'pg_num': u'8', 'pool': u'sadfs'}
+        self._task_post('/api/pool/', data)
+        self.assertStatus(400)
+        self.assertJsonBody({
+            'component': 'pool',
+            'code': "2",
+            'detail': "[errno -2] specified rule dnf doesn't exist"
+        })
+
+    def test_pool_info(self):
+        self._get("/ui-api/pool/info")
+        self.assertSchemaBody(JObj({
+            'pool_names': JList(str),
+            'compression_algorithms': JList(str),
+            'compression_modes': JList(str),
+            'is_all_bluestore': bool,
+            'bluestore_compression_algorithm': str,
+            'osd_count': int,
+            'crush_rules_replicated': JList(JObj({}, allow_unknown=True)),
+            'crush_rules_erasure': JList(JObj({}, allow_unknown=True)),
+            'pg_autoscale_default_mode': str,
+            'pg_autoscale_modes': JList(str),
+            'erasure_code_profiles': JList(JObj({}, allow_unknown=True)),
+            'used_rules': JObj({}, allow_unknown=True),
+            'used_profiles': JObj({}, allow_unknown=True),
+            'nodes': JList(JObj({}, allow_unknown=True)),
+        }))