path: root/src/pybind/mgr/pg_autoscaler/module.py

"""
Automatically scale pg_num based on how much data is stored in each pool.
"""

import json
import mgr_util
import threading
from typing import Any, Dict, List, Optional, Set, Tuple, TYPE_CHECKING, Union
import uuid
from prettytable import PrettyTable
from mgr_module import HealthChecksT, CLIReadCommand, CLIWriteCommand, CRUSHMap, MgrModule, Option, OSDMap

"""
Some terminology is made up for the purposes of this module:

 - "raw pgs": pg count after applying replication, i.e. the real resource
              consumption of a pool.
 - "grow/shrink" - increase/decrease the pg_num in a pool
 - "crush subtree" - non-overlapping domains in crush hierarchy: used as
                     units of resource management.
"""

INTERVAL = 5

PG_NUM_MIN = 32  # unless specified on a per-pool basis

if TYPE_CHECKING:
    import sys
    if sys.version_info >= (3, 8):
        from typing import Literal
    else:
        from typing_extensions import Literal

    PassT = Literal['first', 'second', 'third']


def nearest_power_of_two(n: int) -> int:
    v = int(n)

    v -= 1
    v |= v >> 1
    v |= v >> 2
    v |= v >> 4
    v |= v >> 8
    v |= v >> 16

    # High bound power of two
    v += 1

    # Low bound power of tow
    x = v >> 1

    return x if (v - n) > (n - x) else v


def effective_target_ratio(target_ratio: float,
                           total_target_ratio: float,
                           total_target_bytes: int,
                           capacity: int) -> float:
    """
    Returns the target ratio after normalizing for ratios across pools and
    adjusting for capacity reserved by pools that have target_size_bytes set.
    """
    target_ratio = float(target_ratio)
    if total_target_ratio:
        target_ratio = target_ratio / total_target_ratio

    if total_target_bytes and capacity:
        fraction_available = 1.0 - min(1.0, float(total_target_bytes) / capacity)
        target_ratio *= fraction_available

    return target_ratio


class PgAdjustmentProgress(object):
    """
    Keeps the initial and target pg_num values
    """

    def __init__(self, pool_id: int, pg_num: int, pg_num_target: int) -> None:
        self.ev_id = str(uuid.uuid4())
        self.pool_id = pool_id
        self.reset(pg_num, pg_num_target)

    def reset(self, pg_num: int, pg_num_target: int) -> None:
        self.pg_num = pg_num
        self.pg_num_target = pg_num_target

    def update(self, module: MgrModule, progress: float) -> None:
        desc = 'increasing' if self.pg_num < self.pg_num_target else 'decreasing'
        module.remote('progress', 'update', self.ev_id,
                      ev_msg="PG autoscaler %s pool %d PGs from %d to %d" %
                      (desc, self.pool_id, self.pg_num, self.pg_num_target),
                      ev_progress=progress,
                      refs=[("pool", self.pool_id)])


class CrushSubtreeResourceStatus:
    def __init__(self) -> None:
        self.root_ids: List[int] = []
        self.osds: Set[int] = set()
        self.osd_count: Optional[int] = None  # Number of OSDs
        self.pg_target: Optional[int] = None  # Ideal full-capacity PG count?
        self.pg_current = 0  # How many PGs already?
        self.pg_left = 0
        self.capacity: Optional[int] = None  # Total capacity of OSDs in subtree
        self.pool_ids: List[int] = []
        self.pool_names: List[str] = []
        self.pool_count: Optional[int] = None
        self.pool_used = 0
        self.total_target_ratio = 0.0
        self.total_target_bytes = 0  # including replication / EC overhead


class PgAutoscaler(MgrModule):
    """
    PG autoscaler.
    """
    NATIVE_OPTIONS = [
        'mon_target_pg_per_osd',
        'mon_max_pg_per_osd',
    ]

    MODULE_OPTIONS = [
        Option(
            name='sleep_interval',
            type='secs',
            default=60),

        Option(
            name='noautoscale',
            type='bool',
            desc='global autoscale flag',
            long_desc=('Option to turn on/off the autoscaler for all pools'),
            default=False),
    ]

    def __init__(self, *args: Any, **kwargs: Any) -> None:
        super(PgAutoscaler, self).__init__(*args, **kwargs)
        self._shutdown = threading.Event()
        self._event: Dict[int, PgAdjustmentProgress] = {}

        # So much of what we do peeks at the osdmap that it's easiest
        # to just keep a copy of the pythonized version.
        self._osd_map = None
        if TYPE_CHECKING:
            self.sleep_interval = 60
            self.mon_target_pg_per_osd = 0
            self.noautoscale = False

    def config_notify(self) -> None:
        for opt in self.NATIVE_OPTIONS:
            setattr(self,
                    opt,
                    self.get_ceph_option(opt))
            self.log.debug(' native option %s = %s', opt, getattr(self, opt))
        for opt in self.MODULE_OPTIONS:
            setattr(self,
                    opt['name'],
                    self.get_module_option(opt['name']))
            self.log.debug(' mgr option %s = %s',
                           opt['name'], getattr(self, opt['name']))

    @CLIReadCommand('osd pool autoscale-status')
    def _command_autoscale_status(self, format: str = 'plain') -> Tuple[int, str, str]:
        """
        report on pool pg_num sizing recommendation and intent
        """
        osdmap = self.get_osdmap()
        pools = osdmap.get_pools_by_name()
        ps, root_map = self._get_pool_status(osdmap, pools)

        if format in ('json', 'json-pretty'):
            return 0, json.dumps(ps, indent=4, sort_keys=True), ''
        else:
            table = PrettyTable(['POOL', 'SIZE', 'TARGET SIZE',
                                 'RATE', 'RAW CAPACITY',
                                 'RATIO', 'TARGET RATIO',
                                 'EFFECTIVE RATIO',
                                 'BIAS',
                                 'PG_NUM',
#                                 'IDEAL',
                                 'NEW PG_NUM', 'AUTOSCALE',
                                 'BULK'],
                                border=False)
            table.left_padding_width = 0
            table.right_padding_width = 2
            table.align['POOL'] = 'l'
            table.align['SIZE'] = 'r'
            table.align['TARGET SIZE'] = 'r'
            table.align['RATE'] = 'r'
            table.align['RAW CAPACITY'] = 'r'
            table.align['RATIO'] = 'r'
            table.align['TARGET RATIO'] = 'r'
            table.align['EFFECTIVE RATIO'] = 'r'
            table.align['BIAS'] = 'r'
            table.align['PG_NUM'] = 'r'
#            table.align['IDEAL'] = 'r'
            table.align['NEW PG_NUM'] = 'r'
            table.align['AUTOSCALE'] = 'l'
            table.align['BULK'] = 'l'
            for p in ps:
                if p['would_adjust']:
                    final = str(p['pg_num_final'])
                else:
                    final = ''
                if p['target_bytes'] > 0:
                    ts = mgr_util.format_bytes(p['target_bytes'], 6)
                else:
                    ts = ''
                if p['target_ratio'] > 0.0:
                    tr = '%.4f' % p['target_ratio']
                else:
                    tr = ''
                if p['effective_target_ratio'] > 0.0:
                    etr = '%.4f' % p['effective_target_ratio']
                else:
                    etr = ''
                table.add_row([
                    p['pool_name'],
                    mgr_util.format_bytes(p['logical_used'], 6),
                    ts,
                    p['raw_used_rate'],
                    mgr_util.format_bytes(p['subtree_capacity'], 6),
                    '%.4f' % p['capacity_ratio'],
                    tr,
                    etr,
                    p['bias'],
                    p['pg_num_target'],
#                    p['pg_num_ideal'],
                    final,
                    p['pg_autoscale_mode'],
                    str(p['bulk'])
                ])
            return 0, table.get_string(), ''

    def complete_all_progress_events(self) -> None:
        for pool_id in list(self._event):
            ev = self._event[pool_id]
            self.remote('progress', 'complete', ev.ev_id)
            del self._event[pool_id]

    def set_autoscale_mode_all_pools(self, status: str) -> None:
        osdmap = self.get_osdmap()
        pools = osdmap.get_pools_by_name()
        for pool_name, _ in pools.items():
            self.mon_command({
                'prefix': 'osd pool set',
                'pool': pool_name,
                'var': 'pg_autoscale_mode',
                'val': status
            })
    @CLIWriteCommand("osd pool get noautoscale")
    def get_noautoscale(self) -> Tuple[int, str, str]:
        """
        Get the noautoscale flag to see if all pools
        are setting the autoscaler on or off as well
        as newly created pools in the future.
        """

        if self.noautoscale == None:
            raise TypeError("noautoscale cannot be None")
        elif self.noautoscale:
            return 0, "", "noautoscale is on"
        else:
            return 0, "", "noautoscale is off"

    @CLIWriteCommand("osd pool unset noautoscale")
    def unset_noautoscale(self) -> Tuple[int, str, str]:
        """
        Unset the noautoscale flag so all pools will
        have autoscale enabled (including newly created
        pools in the future).
        """
        if not self.noautoscale:
            return 0, "", "noautoscale is already unset!"
        else:
            self.set_module_option("noautoscale", False)
            self.mon_command({
                'prefix': 'config set',
                'who': 'global',
                'name': 'osd_pool_default_pg_autoscale_mode',
                'value': 'on'
            })
            self.set_autoscale_mode_all_pools("on")
            return 0, "", "noautoscale is unset, all pools now have autoscale on"

    @CLIWriteCommand("osd pool set noautoscale")
    def set_noautoscale(self) -> Tuple[int, str, str]:
        """
        set the noautoscale for all pools (including
        newly created pools in the future)
        and complete all on-going progress events
        regarding PG-autoscaling.
        """
        if self.noautoscale:
            return 0, "", "noautoscale is already set!"
        else:
            self.set_module_option("noautoscale", True)
            self.mon_command({
                'prefix': 'config set',
                'who': 'global',
                'name': 'osd_pool_default_pg_autoscale_mode',
                'value': 'off'
            })
            self.set_autoscale_mode_all_pools("off")
            self.complete_all_progress_events()
            return 0, "", "noautoscale is set, all pools now have autoscale off"

    def serve(self) -> None:
        self.config_notify()
        while not self._shutdown.is_set():
            self._maybe_adjust()
            self._update_progress_events()
            self._shutdown.wait(timeout=self.sleep_interval)

    def shutdown(self) -> None:
        self.log.info('Stopping pg_autoscaler')
        self._shutdown.set()

    def identify_subtrees_and_overlaps(self,
                                       osdmap: OSDMap,
                                       crush: CRUSHMap,
                                       result: Dict[int, CrushSubtreeResourceStatus],
                                       overlapped_roots: Set[int],
                                       roots: List[CrushSubtreeResourceStatus]) -> \
        Tuple[List[CrushSubtreeResourceStatus],
              Set[int]]:

        # We identify subtrees and overlapping roots from osdmap
        for pool_id, pool in osdmap.get_pools().items():
            crush_rule = crush.get_rule_by_id(pool['crush_rule'])
            assert crush_rule is not None
            cr_name = crush_rule['rule_name']
            root_id = crush.get_rule_root(cr_name)
            assert root_id is not None
            osds = set(crush.get_osds_under(root_id))

            # Are there overlapping roots?
            s = None
            for prev_root_id, prev in result.items():
                if osds & prev.osds:
                    s = prev
                    if prev_root_id != root_id:
                        overlapped_roots.add(prev_root_id)
                        overlapped_roots.add(root_id)
                        self.log.warning("pool %s won't scale due to overlapping roots: %s",
                                       pool['pool_name'], overlapped_roots)
                        self.log.warning("Please See: https://docs.ceph.com/en/"
                                         "latest/rados/operations/placement-groups"
                                         "/#automated-scaling")
                    break
            if not s:
                s = CrushSubtreeResourceStatus()
                roots.append(s)
            result[root_id] = s
            s.root_ids.append(root_id)
            s.osds |= osds
            s.pool_ids.append(pool_id)
            s.pool_names.append(pool['pool_name'])
            s.pg_current += pool['pg_num_target'] * pool['size']
            target_ratio = pool['options'].get('target_size_ratio', 0.0)
            if target_ratio:
                s.total_target_ratio += target_ratio
            else:
                target_bytes = pool['options'].get('target_size_bytes', 0)
                if target_bytes:
                    s.total_target_bytes += target_bytes * osdmap.pool_raw_used_rate(pool_id)
        return roots, overlapped_roots

    def get_subtree_resource_status(self,
                                    osdmap: OSDMap,
                                    crush: CRUSHMap) -> Tuple[Dict[int, CrushSubtreeResourceStatus],
                                                              Set[int]]:
        """
        For each CRUSH subtree of interest (i.e. the roots under which
        we have pools), calculate the current resource usages and targets,
        such as how many PGs there are, vs. how many PGs we would
        like there to be.
        """
        result: Dict[int, CrushSubtreeResourceStatus] = {}
        roots: List[CrushSubtreeResourceStatus] = []
        overlapped_roots: Set[int] = set()
        # identify subtrees and overlapping roots
        roots, overlapped_roots = self.identify_subtrees_and_overlaps(osdmap,
                                                                      crush, result, overlapped_roots, roots)
        # finish subtrees
        all_stats = self.get('osd_stats')
        for s in roots:
            assert s.osds is not None
            s.osd_count = len(s.osds)
            s.pg_target = s.osd_count * self.mon_target_pg_per_osd
            s.pg_left = s.pg_target
            s.pool_count = len(s.pool_ids)
            capacity = 0
            for osd_stats in all_stats['osd_stats']:
                if osd_stats['osd'] in s.osds:
                    # Intentionally do not apply the OSD's reweight to
                    # this, because we want to calculate PG counts based
                    # on the physical storage available, not how it is
                    # reweighted right now.
                    capacity += osd_stats['kb'] * 1024

            s.capacity = capacity
            self.log.debug('root_ids %s pools %s with %d osds, pg_target %d',
                           s.root_ids,
                           s.pool_ids,
                           s.osd_count,
                           s.pg_target)

        return result, overlapped_roots

    def _calc_final_pg_target(
            self,
            p: Dict[str, Any],
            pool_name: str,
            root_map: Dict[int, CrushSubtreeResourceStatus],
            root_id: int,
            capacity_ratio: float,
            bias: float,
            even_pools: Dict[str, Dict[str, Any]],
            bulk_pools: Dict[str, Dict[str, Any]],
            func_pass: 'PassT',
            bulk: bool,
    ) -> Union[Tuple[float, int, int], Tuple[None, None, None]]:
        """
        `profile` determines behaviour of the autoscaler.
        `first_pass` flag used to determine if this is the first
        pass where the caller tries to calculate/adjust pools that has
        used_ratio > even_ratio else this is the second pass,
        we calculate final_ratio by giving it 1 / pool_count
        of the root we are currently looking at.
        """
        if func_pass == 'first':
            # first pass to deal with small pools (no bulk flag)
            # calculating final_pg_target based on capacity ratio
            # we also keep track of bulk_pools to be used in second pass
            if not bulk:
                final_ratio = capacity_ratio
                pg_left = root_map[root_id].pg_left
                assert pg_left is not None
                used_pg = final_ratio * pg_left
                root_map[root_id].pg_left -= int(used_pg)
                root_map[root_id].pool_used += 1
                pool_pg_target = used_pg / p['size'] * bias
            else:
                bulk_pools[pool_name] = p
                return None, None, None

        elif func_pass == 'second':
            # second pass we calculate the final_pg_target
            # for pools that have used_ratio > even_ratio
            # and we keep track of even pools to be used in third pass
            pool_count = root_map[root_id].pool_count
            assert pool_count is not None
            even_ratio = 1 / (pool_count - root_map[root_id].pool_used)
            used_ratio = capacity_ratio

            if used_ratio > even_ratio:
                root_map[root_id].pool_used += 1
            else:
                even_pools[pool_name] = p
                return None, None, None

            final_ratio = max(used_ratio, even_ratio)
            pg_left = root_map[root_id].pg_left
            assert pg_left is not None
            used_pg = final_ratio * pg_left
            root_map[root_id].pg_left -= int(used_pg)
            pool_pg_target = used_pg / p['size'] * bias

        else:
            # third pass we just split the pg_left to all even_pools
            pool_count = root_map[root_id].pool_count
            assert pool_count is not None
            final_ratio = 1 / (pool_count - root_map[root_id].pool_used)
            pool_pg_target = (final_ratio * root_map[root_id].pg_left) / p['size'] * bias

        min_pg = p.get('options', {}).get('pg_num_min', PG_NUM_MIN)
        max_pg = p.get('options', {}).get('pg_num_max')
        final_pg_target = max(min_pg, nearest_power_of_two(pool_pg_target))
        if max_pg and max_pg < final_pg_target:
            final_pg_target = max_pg
        self.log.info("Pool '{0}' root_id {1} using {2} of space, bias {3}, "
                      "pg target {4} quantized to {5} (current {6})".format(
                      p['pool_name'],
                      root_id,
                      capacity_ratio,
                      bias,
                      pool_pg_target,
                      final_pg_target,
                      p['pg_num_target']
        ))
        return final_ratio, pool_pg_target, final_pg_target

    def _get_pool_pg_targets(
            self,
            osdmap: OSDMap,
            pools: Dict[str, Dict[str, Any]],
            crush_map: CRUSHMap,
            root_map: Dict[int, CrushSubtreeResourceStatus],
            pool_stats: Dict[int, Dict[str, int]],
            ret: List[Dict[str, Any]],
            threshold: float,
            func_pass: 'PassT',
            overlapped_roots: Set[int],
    ) -> Tuple[List[Dict[str, Any]], Dict[str, Dict[str, Any]] , Dict[str, Dict[str, Any]]]:
        """
        Calculates final_pg_target of each pools and determine if it needs
        scaling, this depends on the profile of the autoscaler. For scale-down,
        we start out with a full complement of pgs and only descrease it when other
        pools needs more pgs due to increased usage. For scale-up, we start out with
        the minimal amount of pgs and only scale when there is increase in usage.
        """
        even_pools: Dict[str, Dict[str, Any]] = {}
        bulk_pools: Dict[str, Dict[str, Any]] = {}
        for pool_name, p in pools.items():
            pool_id = p['pool']
            if pool_id not in pool_stats:
                # race with pool deletion; skip
                continue

            # FIXME: we assume there is only one take per pool, but that
            # may not be true.
            crush_rule = crush_map.get_rule_by_id(p['crush_rule'])
            assert crush_rule is not None
            cr_name = crush_rule['rule_name']
            root_id = crush_map.get_rule_root(cr_name)
            assert root_id is not None
            if root_id in overlapped_roots:
                # skip pools
                # with overlapping roots
                self.log.warn("pool %d contains an overlapping root %d"
                              "... skipping scaling", pool_id, root_id)
                continue
            capacity = root_map[root_id].capacity
            assert capacity is not None
            if capacity == 0:
                self.log.debug('skipping empty subtree %s', cr_name)
                continue

            raw_used_rate = osdmap.pool_raw_used_rate(pool_id)

            pool_logical_used = pool_stats[pool_id]['stored']
            bias = p['options'].get('pg_autoscale_bias', 1.0)
            target_bytes = 0
            # ratio takes precedence if both are set
            if p['options'].get('target_size_ratio', 0.0) == 0.0:
                target_bytes = p['options'].get('target_size_bytes', 0)

            # What proportion of space are we using?
            actual_raw_used = pool_logical_used * raw_used_rate
            actual_capacity_ratio = float(actual_raw_used) / capacity

            pool_raw_used = max(pool_logical_used, target_bytes) * raw_used_rate
            capacity_ratio = float(pool_raw_used) / capacity

            self.log.info("effective_target_ratio {0} {1} {2} {3}".format(
                p['options'].get('target_size_ratio', 0.0),
                root_map[root_id].total_target_ratio,
                root_map[root_id].total_target_bytes,
                capacity))

            target_ratio = effective_target_ratio(p['options'].get('target_size_ratio', 0.0),
                                                  root_map[root_id].total_target_ratio,
                                                  root_map[root_id].total_target_bytes,
                                                  capacity)

            # determine if the pool is a bulk
            bulk = False
            flags = p['flags_names'].split(",")
            if "bulk" in flags:
                bulk = True

            capacity_ratio = max(capacity_ratio, target_ratio)
            final_ratio, pool_pg_target, final_pg_target = self._calc_final_pg_target(
                p, pool_name, root_map, root_id,
                capacity_ratio, bias, even_pools,
                bulk_pools, func_pass, bulk)

            if final_ratio is None:
                continue

            adjust = False
            if (final_pg_target > p['pg_num_target'] * threshold or
                    final_pg_target < p['pg_num_target'] / threshold) and \
                    final_ratio >= 0.0 and \
                    final_ratio <= 1.0:
                adjust = True

            assert pool_pg_target is not None
            ret.append({
                'pool_id': pool_id,
                'pool_name': p['pool_name'],
                'crush_root_id': root_id,
                'pg_autoscale_mode': p['pg_autoscale_mode'],
                'pg_num_target': p['pg_num_target'],
                'logical_used': pool_logical_used,
                'target_bytes': target_bytes,
                'raw_used_rate': raw_used_rate,
                'subtree_capacity': capacity,
                'actual_raw_used': actual_raw_used,
                'raw_used': pool_raw_used,
                'actual_capacity_ratio': actual_capacity_ratio,
                'capacity_ratio': capacity_ratio,
                'target_ratio': p['options'].get('target_size_ratio', 0.0),
                'effective_target_ratio': target_ratio,
                'pg_num_ideal': int(pool_pg_target),
                'pg_num_final': final_pg_target,
                'would_adjust': adjust,
                'bias': p.get('options', {}).get('pg_autoscale_bias', 1.0),
                'bulk': bulk,
            })

        return ret, bulk_pools, even_pools

    def _get_pool_status(
            self,
            osdmap: OSDMap,
            pools: Dict[str, Dict[str, Any]],
            threshold: float = 3.0,
    ) -> Tuple[List[Dict[str, Any]],
               Dict[int, CrushSubtreeResourceStatus]]:
        assert threshold >= 2.0

        crush_map = osdmap.get_crush()
        root_map, overlapped_roots = self.get_subtree_resource_status(osdmap, crush_map)
        df = self.get('df')
        pool_stats = dict([(p['id'], p['stats']) for p in df['pools']])

        ret: List[Dict[str, Any]] = []

        # Iterate over all pools to determine how they should be sized.
        # First call of _get_pool_pg_targets() is to find/adjust pools that uses more capacaity than
        # the even_ratio of other pools and we adjust those first.
        # Second call make use of the even_pools we keep track of in the first call.
        # All we need to do is iterate over those and give them 1/pool_count of the
        # total pgs.

        ret, bulk_pools, _  = self._get_pool_pg_targets(osdmap, pools, crush_map, root_map,
                                                  pool_stats, ret, threshold, 'first', overlapped_roots)

        ret, _, even_pools = self._get_pool_pg_targets(osdmap, bulk_pools, crush_map, root_map,
                                                  pool_stats, ret, threshold, 'second', overlapped_roots)

        ret, _, _ = self._get_pool_pg_targets(osdmap, even_pools, crush_map, root_map,
                                         pool_stats, ret, threshold, 'third', overlapped_roots)

        return (ret, root_map)

    def _update_progress_events(self) -> None:
        if self.noautoscale:
            return
        osdmap = self.get_osdmap()
        pools = osdmap.get_pools()
        for pool_id in list(self._event):
            ev = self._event[pool_id]
            pool_data = pools.get(pool_id)
            if pool_data is None or pool_data['pg_num'] == pool_data['pg_num_target'] or ev.pg_num == ev.pg_num_target:
                # pool is gone or we've reached our target
                self.remote('progress', 'complete', ev.ev_id)
                del self._event[pool_id]
                continue
            ev.update(self, (ev.pg_num - pool_data['pg_num']) / (ev.pg_num - ev.pg_num_target))

    def _maybe_adjust(self) -> None:
        if self.noautoscale:
            return
        self.log.info('_maybe_adjust')
        osdmap = self.get_osdmap()
        if osdmap.get_require_osd_release() < 'nautilus':
            return
        pools = osdmap.get_pools_by_name()
        self.log.debug("pool: {0}".format(json.dumps(pools, indent=4,
                                sort_keys=True)))
        ps, root_map = self._get_pool_status(osdmap, pools)

        # Anyone in 'warn', set the health message for them and then
        # drop them from consideration.
        too_few = []
        too_many = []
        bytes_and_ratio = []
        health_checks: Dict[str, Dict[str, Union[int, str, List[str]]]] = {}

        total_bytes = dict([(r, 0) for r in iter(root_map)])
        total_target_bytes = dict([(r, 0.0) for r in iter(root_map)])
        target_bytes_pools: Dict[int, List[int]] = dict([(r, []) for r in iter(root_map)])

        for p in ps:
            pool_id = p['pool_id']
            pool_opts = pools[p['pool_name']]['options']
            if pool_opts.get('target_size_ratio', 0) > 0 and pool_opts.get('target_size_bytes', 0) > 0:
                bytes_and_ratio.append(
                    'Pool %s has target_size_bytes and target_size_ratio set' % p['pool_name'])
            total_bytes[p['crush_root_id']] += max(
                p['actual_raw_used'],
                p['target_bytes'] * p['raw_used_rate'])
            if p['target_bytes'] > 0:
                total_target_bytes[p['crush_root_id']] += p['target_bytes'] * p['raw_used_rate']
                target_bytes_pools[p['crush_root_id']].append(p['pool_name'])
            if not p['would_adjust']:
                continue
            if p['pg_autoscale_mode'] == 'warn':
                msg = 'Pool %s has %d placement groups, should have %d' % (
                    p['pool_name'],
                    p['pg_num_target'],
                    p['pg_num_final'])
                if p['pg_num_final'] > p['pg_num_target']:
                    too_few.append(msg)
                else:
                    too_many.append(msg)

            if p['pg_autoscale_mode'] == 'on':
                # Note that setting pg_num actually sets pg_num_target (see
                # OSDMonitor.cc)
                r = self.mon_command({
                    'prefix': 'osd pool set',
                    'pool': p['pool_name'],
                    'var': 'pg_num',
                    'val': str(p['pg_num_final'])
                })

                # create new event or update existing one to reflect
                # progress from current state to the new pg_num_target
                pool_data = pools[p['pool_name']]
                pg_num = pool_data['pg_num']
                new_target = p['pg_num_final']
                if pool_id in self._event:
                    self._event[pool_id].reset(pg_num, new_target)
                else:
                    self._event[pool_id] = PgAdjustmentProgress(pool_id, pg_num, new_target)
                self._event[pool_id].update(self, 0.0)

                if r[0] != 0:
                    # FIXME: this is a serious and unexpected thing,
                    # we should expose it as a cluster log error once
                    # the hook for doing that from ceph-mgr modules is
                    # in.
                    self.log.error("pg_num adjustment on {0} to {1} failed: {2}"
                                   .format(p['pool_name'],
                                           p['pg_num_final'], r))

        if too_few:
            summary = "{0} pools have too few placement groups".format(
                len(too_few))
            health_checks['POOL_TOO_FEW_PGS'] = {
                'severity': 'warning',
                'summary': summary,
                'count': len(too_few),
                'detail': too_few
            }
        if too_many:
            summary = "{0} pools have too many placement groups".format(
                len(too_many))
            health_checks['POOL_TOO_MANY_PGS'] = {
                'severity': 'warning',
                'summary': summary,
                'count': len(too_many),
                'detail': too_many
            }

        too_much_target_bytes = []
        for root_id, total in total_bytes.items():
            total_target = int(total_target_bytes[root_id])
            capacity = root_map[root_id].capacity
            assert capacity is not None
            if total_target > 0 and total > capacity and capacity:
                too_much_target_bytes.append(
                    'Pools %s overcommit available storage by %.03fx due to '
                    'target_size_bytes %s on pools %s' % (
                        root_map[root_id].pool_names,
                        total / capacity,
                        mgr_util.format_bytes(total_target, 5, colored=False),
                        target_bytes_pools[root_id]
                    )
                )
            elif total_target > capacity and capacity:
                too_much_target_bytes.append(
                    'Pools %s overcommit available storage by %.03fx due to '
                    'collective target_size_bytes of %s' % (
                        root_map[root_id].pool_names,
                        total / capacity,
                        mgr_util.format_bytes(total_target, 5, colored=False),
                    )
                )
        if too_much_target_bytes:
            health_checks['POOL_TARGET_SIZE_BYTES_OVERCOMMITTED'] = {
                'severity': 'warning',
                'summary': "%d subtrees have overcommitted pool target_size_bytes" % len(too_much_target_bytes),
                'count': len(too_much_target_bytes),
                'detail': too_much_target_bytes,
            }

        if bytes_and_ratio:
            health_checks['POOL_HAS_TARGET_SIZE_BYTES_AND_RATIO'] = {
                'severity': 'warning',
                'summary': "%d pools have both target_size_bytes and target_size_ratio set" % len(bytes_and_ratio),
                'count': len(bytes_and_ratio),
                'detail': bytes_and_ratio,
            }

        self.set_health_checks(health_checks)